diff --git a/.ci/bwcVersions b/.ci/bwcVersions
index 81181301895e..6dabe613c827 100644
--- a/.ci/bwcVersions
+++ b/.ci/bwcVersions
@@ -57,6 +57,8 @@ BWC_VERSION:
   - "7.17.6"
   - "7.17.7"
   - "7.17.8"
+  - "7.17.9"
+  - "7.17.10"
   - "8.0.0"
   - "8.0.1"
   - "8.1.0"
@@ -78,5 +80,8 @@ BWC_VERSION:
   - "8.5.0"
   - "8.5.1"
   - "8.5.2"
+  - "8.5.3"
   - "8.6.0"
+  - "8.6.1"
+  - "8.6.2"
   - "8.7.0"
diff --git a/.ci/jobs.t/elastic+elasticsearch+dra-snapshot.yml b/.ci/jobs.t/elastic+elasticsearch+dra-snapshot.yml
new file mode 100644
index 000000000000..37fdd85ee656
--- /dev/null
+++ b/.ci/jobs.t/elastic+elasticsearch+dra-snapshot.yml
@@ -0,0 +1,67 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+dra-snapshot
+    workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+dra-snapshot
+    display-name: "elastic / elasticsearch # %BRANCH% - DRA snapshot"
+    description: "Publishing Daily Releasable Artifacts (DRAs) of Elasticsearch %BRANCH% snapshots.\n"
+    node: "ubuntu-20.04"
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+      - shell: |
+            #!/usr/local/bin/runbld --redirect-stderr
+            WORKFLOW="snapshot"
+            RM_BRANCH="%BRANCH%" && [[ "%BRANCH%" == "main" ]] && RM_BRANCH=master
+            ES_VERSION=$(cat build-tools-internal/version.properties \
+                  | grep elasticsearch \
+                  | sed "s/elasticsearch *= *//g")
+            VERSION_SUFFIX="" && [[ "$WORKFLOW" == "snapshot" ]] && VERSION_SUFFIX="-SNAPSHOT"
+            BEATS_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh beats $RM_BRANCH $ES_VERSION $WORKFLOW)"
+            ML_CPP_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh ml-cpp $RM_BRANCH $ES_VERSION $WORKFLOW)"
+            set -euo pipefail
+            set +x
+            VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID)
+            export VAULT_TOKEN
+            $WORKSPACE/.ci/scripts/run-gradle.sh -Ddra.artifacts=true \
+                -Ddra.artifacts.dependency.beats=${BEATS_BUILD_ID} \
+                -Ddra.artifacts.dependency.ml-cpp=${ML_CPP_BUILD_ID} \
+                -Ddra.workflow=$WORKFLOW \
+                -Dcsv=$WORKSPACE/build/distributions/dependencies-${ES_VERSION}${VERSION_SUFFIX}.csv \
+                buildReleaseArtifacts \
+                exportCompressedDockerImages \
+                :distribution:generateDependenciesReport
+
+            unset VAULT_TOKEN
+            set -x
+            $WORKSPACE/x-pack/plugin/sql/connectors/tableau/package.sh asm qualifier="$VERSION_SUFFIX"
+
+            # we regenerate this file as part of the release manager invocation
+            rm $WORKSPACE/build/distributions/elasticsearch-jdbc-${ES_VERSION}${VERSION_SUFFIX}.taco.sha512
+
+            # Allow other users access to read the artifacts so they are readable in the
+            # container
+            find $WORKSPACE -type f -path "*/build/distributions/*" -exec chmod a+r {} \;
+
+            # Allow other users write access to create checksum files
+            find $WORKSPACE -type d -path "*/build/distributions" -exec chmod a+w {} \;
+
+            # Artifacts should be generated
+            docker run --rm \
+                   --name release-manager \
+                   -e VAULT_ADDR \
+                   -e VAULT_ROLE_ID \
+                   -e VAULT_SECRET_ID \
+                   --mount type=bind,readonly=false,src="$PWD",target=/artifacts \
+                   docker.elastic.co/infra/release-manager:latest \
+                   cli collect \
+                   --project elasticsearch \
+                   --branch "$RM_BRANCH" \
+                   --commit "$GIT_COMMIT" \
+                   --workflow "$WORKFLOW" \
+                   --version "$ES_VERSION" \
+                   --artifact-set main \
+                   --dependency beats:https://artifacts-${WORKFLOW}.elastic.co/beats/${BEATS_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json \
+                   --dependency ml-cpp:https://artifacts-${WORKFLOW}.elastic.co/ml-cpp/${ML_CPP_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json
diff --git a/.ci/jobs.t/elastic+elasticsearch+dra-staging-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+dra-staging-trigger.yml
new file mode 100644
index 000000000000..87df5e07c2e4
--- /dev/null
+++ b/.ci/jobs.t/elastic+elasticsearch+dra-staging-trigger.yml
@@ -0,0 +1,6 @@
+---
+jjbb-template: periodic-trigger-lgc.yml
+vars:
+  - periodic-job: elastic+elasticsearch+%BRANCH%+dra-staging
+  - lgc-job: elastic+elasticsearch+%BRANCH%+intake
+  - cron: "H H/12 * * *"
diff --git a/.ci/jobs.t/elastic+elasticsearch+dra-staging.yml b/.ci/jobs.t/elastic+elasticsearch+dra-staging.yml
index d558872abeb9..40a759e20ba2 100644
--- a/.ci/jobs.t/elastic+elasticsearch+dra-staging.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+dra-staging.yml
@@ -1,6 +1,6 @@
 ---
 - job:
-    name: elastic+elasticsearch+%BRANCH%+dra-snapshot
+    name: elastic+elasticsearch+%BRANCH%+dra-staging
     workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+dra-staging
     display-name: "elastic / elasticsearch # %BRANCH% - DRA staging"
     description: "Publishing Daily Releasable Artifacts (DRAs) of Elasticsearch %BRANCH% staging.\n"
@@ -13,6 +13,12 @@
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
       - shell: |
             #!/usr/local/bin/runbld --redirect-stderr
+
+            # Don't publish main branch to staging
+            if [ "%BRANCH%" == "main" ]; then
+              exit 0
+            fi
+
             WORKFLOW="staging"
             RM_BRANCH="%BRANCH%" && [[ "%BRANCH%" == "main" ]] && RM_BRANCH=master
             ES_VERSION=$(cat build-tools-internal/version.properties \
diff --git a/.ci/jobs.t/elastic+elasticsearch+intake.yml b/.ci/jobs.t/elastic+elasticsearch+intake.yml
index f4aa32ec7e75..138318897b52 100644
--- a/.ci/jobs.t/elastic+elasticsearch+intake.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+intake.yml
@@ -54,6 +54,13 @@
               kill-phase-on: NEVER
               current-parameters: true
               git-revision: true
+      - multijob:
+          name: Publish snapshot artifacts
+          projects:
+            - name: elastic+elasticsearch+%BRANCH%+dra-snapshot
+              kill-phase-on: NEVER
+              current-parameters: true
+              git-revision: true
       - multijob:
           name: Update last good commit
           projects:
diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-arm.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-arm.yml
index a5e590bc4a76..773db8ff2f92 100644
--- a/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-arm.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-arm.yml
@@ -3,8 +3,7 @@
     name: elastic+elasticsearch+%BRANCH%+multijob+platform-support-arm
     display-name: "elastic / elasticsearch # %BRANCH% - arm compatibility"
     description: "Elasticsearch %BRANCH% ARM (aarch64) compatibility testing.\n"
-    # Don't use ramdisk for now as we are exhausting memory and causing oomkiller to trigger
-    # child-workspace: "/dev/shm/elastic+elasticsearch+%BRANCH%+multijob+platform-support-arm"
+    child-workspace: "/dev/shm/elastic+elasticsearch+%BRANCH%+multijob+platform-support-arm"
     project-type: matrix
     node: master
     scm:
@@ -17,6 +16,15 @@
           values:
             - "almalinux-8-aarch64&&immutable"
             - "ubuntu-1804-aarch64&&immutable"
+      - axis:
+          type: user-defined
+          name: GRADLE_TASK
+          values:
+            - 'checkPart1'
+            - 'checkPart2'
+            - 'checkPart3'
+            - 'bwcTestSnapshots'
+            - 'checkRestCompat'
     builders:
       - inject:
           properties-file: '.ci/java-versions-aarch64.properties'
@@ -28,4 +36,4 @@
             JAVA16_HOME=$HOME/.java/jdk16
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr
-          $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true check
+          $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true $GRADLE_TASK
diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml
index 6aec8b04a2f3..23f647f5af88 100644
--- a/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml
@@ -5,6 +5,7 @@
     description: "Elasticsearch %BRANCH% unix compatibility testing.\n"
     project-type: matrix
     node: master
+    child-workspace: "/var/lib/jenkins/workspace/elastic+elasticsearch+%BRANCH%+multijob+platform-support-unix"
     scm:
       - git:
           wipe-workspace: false
diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+dra-snapshots-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+dra-snapshots-trigger.yml
deleted file mode 100644
index 7f1c639770ae..000000000000
--- a/.ci/jobs.t/elastic+elasticsearch+periodic+dra-snapshots-trigger.yml
+++ /dev/null
@@ -1,6 +0,0 @@
----
-jjbb-template: periodic-trigger-lgc.yml
-vars:
-  - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+dra-snapshot
-  - lgc-job: elastic+elasticsearch+%BRANCH%+intake
-  - cron: "H H/12 * * *"
diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+dra-snapshots.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+dra-snapshots.yml
deleted file mode 100644
index e0c7e7d122d3..000000000000
--- a/.ci/jobs.t/elastic+elasticsearch+periodic+dra-snapshots.yml
+++ /dev/null
@@ -1,67 +0,0 @@
----
-- job:
-    name: elastic+elasticsearch+%BRANCH%+periodic+dra-snapshot
-    workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic+dra-snapshot
-    display-name: "elastic / elasticsearch # %BRANCH% - DRA snapshot"
-    description: "Publishing Daily Releasable Artifacts (DRAs) of Elasticsearch %BRANCH% snapshots.\n"
-    node: "ubuntu-20.04"
-    builders:
-      - inject:
-          properties-file: '.ci/java-versions.properties'
-          properties-content: |
-            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
-            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
-      - shell: |
-            #!/usr/local/bin/runbld --redirect-stderr
-            WORKFLOW="snapshot"
-            RM_BRANCH="%BRANCH%" && [[ "%BRANCH%" == "main" ]] && RM_BRANCH=master
-            ES_VERSION=$(cat build-tools-internal/version.properties \
-                  | grep elasticsearch \
-                  | sed "s/elasticsearch *= *//g")
-            VERSION_SUFFIX="" && [[ "$WORKFLOW" == "snapshot" ]] && VERSION_SUFFIX="-SNAPSHOT"
-            BEATS_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh beats $RM_BRANCH $ES_VERSION $WORKFLOW)"
-            ML_CPP_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh ml-cpp $RM_BRANCH $ES_VERSION $WORKFLOW)"
-            set -euo pipefail
-            set +x
-            VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID)
-            export VAULT_TOKEN
-            $WORKSPACE/.ci/scripts/run-gradle.sh -Ddra.artifacts=true \
-                -Ddra.artifacts.dependency.beats=${BEATS_BUILD_ID} \
-                -Ddra.artifacts.dependency.ml-cpp=${ML_CPP_BUILD_ID} \
-                -Ddra.workflow=$WORKFLOW \
-                -Dcsv=$WORKSPACE/build/distributions/dependencies-${ES_VERSION}${VERSION_SUFFIX}.csv \
-                buildReleaseArtifacts \
-                exportCompressedDockerImages \
-                :distribution:generateDependenciesReport
-
-            unset VAULT_TOKEN
-            set -x
-            $WORKSPACE/x-pack/plugin/sql/connectors/tableau/package.sh asm qualifier="$VERSION_SUFFIX"
-
-            # we regenerate this file as part of the release manager invocation
-            rm $WORKSPACE/build/distributions/elasticsearch-jdbc-${ES_VERSION}${VERSION_SUFFIX}.taco.sha512
-
-            # Allow other users access to read the artifacts so they are readable in the
-            # container
-            find $WORKSPACE -type f -path "*/build/distributions/*" -exec chmod a+r {} \;
-
-            # Allow other users write access to create checksum files
-            find $WORKSPACE -type d -path "*/build/distributions" -exec chmod a+w {} \;
-
-            # Artifacts should be generated
-            docker run --rm \
-                   --name release-manager \
-                   -e VAULT_ADDR \
-                   -e VAULT_ROLE_ID \
-                   -e VAULT_SECRET_ID \
-                   --mount type=bind,readonly=false,src="$PWD",target=/artifacts \
-                   docker.elastic.co/infra/release-manager:latest \
-                   cli collect \
-                   --project elasticsearch \
-                   --branch "$RM_BRANCH" \
-                   --commit "$GIT_COMMIT" \
-                   --workflow "$WORKFLOW" \
-                   --version "$ES_VERSION" \
-                   --artifact-set main \
-                   --dependency beats:https://artifacts-${WORKFLOW}.elastic.co/beats/${BEATS_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json \
-                   --dependency ml-cpp:https://artifacts-${WORKFLOW}.elastic.co/ml-cpp/${ML_CPP_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json
diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+ear.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+ear.yml
index 47a8d4f48cc5..de4886c9f8eb 100644
--- a/.ci/jobs.t/elastic+elasticsearch+periodic+ear.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+periodic+ear.yml
@@ -16,7 +16,7 @@
           #!/bin/bash
           # Configure a dm-crypt volume backed by a file
           set -e
-          dd if=/dev/zero of=dm-crypt.img bs=1 count=0 seek=60GB
+          dd if=/dev/zero of=dm-crypt.img bs=1 count=0 seek=80GB
           dd if=/dev/urandom of=key.secret bs=2k count=1
           LOOP=$(losetup -f)
           sudo losetup $LOOP dm-crypt.img
diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+java-preview-features-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+java-preview-features-trigger.yml
deleted file mode 100644
index 9b1893362c05..000000000000
--- a/.ci/jobs.t/elastic+elasticsearch+periodic+java-preview-features-trigger.yml
+++ /dev/null
@@ -1,6 +0,0 @@
----
-jjbb-template: periodic-trigger-lgc.yml
-vars:
-  - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+java-preview-features
-  - lgc-job: elastic+elasticsearch+%BRANCH%+intake
-  - cron: "H H/12 * * *"
diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+java-preview-features.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+java-preview-features.yml
deleted file mode 100644
index 592bb2c3941c..000000000000
--- a/.ci/jobs.t/elastic+elasticsearch+periodic+java-preview-features.yml
+++ /dev/null
@@ -1,38 +0,0 @@
----
-- job:
-    name: "elastic+elasticsearch+%BRANCH%+periodic+java-preview-features"
-    display-name: "elastic / elasticsearch # %BRANCH% - java preview features"
-    description: "Testing of the Elasticsearch %BRANCH% branch with java preview features enabled.\n"
-    project-type: matrix
-    child-workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic+java-preview-features
-    node: master
-    scm:
-      - git:
-          wipe-workspace: false
-    axes:
-      - axis:
-          type: slave
-          name: nodes
-          values:
-            - "general-purpose"
-      # We shred out these jobs to avoid running out of memory given since we use a ramdisk workspace
-      - axis:
-          type: user-defined
-          name: GRADLE_TASK
-          values:
-            - 'checkPart1'
-            - 'checkPart2'
-            - 'checkPart3'
-            - 'bwcTestSnapshots'
-            - 'checkRestCompat'
-    builders:
-      - inject:
-          properties-file: '.ci/java-versions.properties'
-          properties-content: |
-            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
-            RUNTIME_JAVA_HOME=$HOME/.java/openjdk19
-            JAVA11_HOME=$HOME/.java/java11
-            JAVA16_HOME=$HOME/.java/openjdk16
-      - shell: |
-          #!/usr/local/bin/runbld --redirect-stderr
-          $WORKSPACE/.ci/scripts/run-gradle.sh -Dtests.jvm.argline="--enable-preview" -Dbwc.checkout.align=true $GRADLE_TASK
diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+release-tests.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+release-tests.yml
index 0ce470d37b34..7447deb49dcf 100644
--- a/.ci/jobs.t/elastic+elasticsearch+periodic+release-tests.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+periodic+release-tests.yml
@@ -23,7 +23,9 @@
 
           mkdir -p ${BEATS_DIR}
           curl --fail -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
-          curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
+          curl --fail -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
+          curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
+          curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
 
           # Fetch ML artifacts
           export ML_IVY_REPO=$(mktemp -d)
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+bwc-snapshots-windows.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+bwc-snapshots-windows.yml
new file mode 100644
index 000000000000..9cd24f1391e0
--- /dev/null
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+bwc-snapshots-windows.yml
@@ -0,0 +1,55 @@
+---
+- job:
+    name: "elastic+elasticsearch+pull-request+bwc-snapshots-windows"
+    display-name: "elastic / elasticsearch - pull request bwc windows"
+    description: "Testing of Elasticsearch pull requests - bwc windows"
+    project-type: matrix
+    node: master
+    child-workspace: "C:\\Users\\jenkins\\workspace\\bwc-snapshots\\${BUILD_NUMBER}"
+    scm:
+      - git:
+          refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
+          branches:
+            - "${ghprbActualCommit}"
+    triggers:
+      - github-pull-request:
+          org-list:
+            - elastic
+          allow-whitelist-orgs-as-admins: true
+          trigger-phrase: '.*run\W+elasticsearch-ci/bwc-snapshots-windows.*'
+          github-hooks: true
+          status-context: elasticsearch-ci/bwc-snapshots-windows
+          cancel-builds-on-update: true
+          black-list-target-branches:
+            - 6.8
+          excluded-regions:
+            - ^docs/.*
+          white-list-labels:
+            - 'test-windows'
+          black-list-labels:
+            - '>test-mute'
+    axes:
+      - axis:
+          type: slave
+          name: nodes
+          values:
+            - "windows-immutable"
+      - axis:
+          type: yaml
+          filename: ".ci/snapshotBwcVersions"
+          name: "BWC_VERSION"
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$USERPROFILE\\.java\\$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$USERPROFILE\\.java\\$ES_RUNTIME_JAVA
+            JAVA11_HOME=$USERPROFILE\\.java\\java11
+            JAVA16_HOME=$USERPROFILE\\.java\\openjdk16
+      - batch: |
+          del /f /s /q %USERPROFILE%\.gradle\init.d\*.*
+          mkdir %USERPROFILE%\.gradle\init.d
+          copy .ci\init.gradle %USERPROFILE%\.gradle\init.d\
+          (
+            echo call %GRADLEW_BAT% --max-workers=4 -Dbwc.checkout.align=true v%BWC_VERSION%#bwcTest ^|^| exit /b 1
+          ) | java -jar "C:\Program Files\infra\bin\runbld" --redirect-stderr -
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-1-windows.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-1-windows.yml
index dbb54e37fb23..f4ded4d1eeca 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-1-windows.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-1-windows.yml
@@ -33,6 +33,7 @@
           properties-content: |
             JAVA_HOME=$USERPROFILE\\.java\\$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$USERPROFILE\\.java\\$ES_RUNTIME_JAVA
+            JAVA11_HOME=$USERPROFILE\\.java\\java11
             JAVA16_HOME=$USERPROFILE\\.java\\openjdk16
             GRADLE_TASK=checkPart1
       - batch: |
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-2-windows.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-2-windows.yml
index ab367188cb8e..53295df00627 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-2-windows.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-2-windows.yml
@@ -33,6 +33,7 @@
           properties-content: |
             JAVA_HOME=$USERPROFILE\\.java\\$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$USERPROFILE\\.java\\$ES_RUNTIME_JAVA
+            JAVA11_HOME=$USERPROFILE\\.java\\java11
             JAVA16_HOME=$USERPROFILE\\.java\\openjdk16
             GRADLE_TASK=checkPart2
       - batch: |
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-3-windows.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-3-windows.yml
index a287c8c0bd9c..80e4bf7f47c0 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-3-windows.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-3-windows.yml
@@ -34,6 +34,7 @@
           properties-content: |
             JAVA_HOME=$USERPROFILE\\.java\\$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$USERPROFILE\\.java\\$ES_RUNTIME_JAVA
+            JAVA11_HOME=$USERPROFILE\\.java\\java11
             JAVA16_HOME=$USERPROFILE\\.java\\openjdk16
             GRADLE_TASK=checkPart3
       - batch: |
diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+release-tests.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+release-tests.yml
index 54242eee118b..125b0a00b6c8 100644
--- a/.ci/jobs.t/elastic+elasticsearch+pull-request+release-tests.yml
+++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+release-tests.yml
@@ -45,13 +45,16 @@
 
           mkdir -p ${BEATS_DIR}
           curl --fail -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
-          curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
+          curl --fail -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
+          curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
+          curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
 
           # Fetch ML artifacts
           export ML_IVY_REPO=$(mktemp -d)
           mkdir -p ${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}
           curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/ml-cpp-${ES_VERSION}-deps.zip" https://artifacts-snapshot.elastic.co/ml-cpp/${ES_VERSION}-SNAPSHOT/downloads/ml-cpp/ml-cpp-${ES_VERSION}-SNAPSHOT-deps.zip
           curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/ml-cpp-${ES_VERSION}-nodeps.zip" https://artifacts-snapshot.elastic.co/ml-cpp/${ES_VERSION}-SNAPSHOT/downloads/ml-cpp/ml-cpp-${ES_VERSION}-SNAPSHOT-nodeps.zip
+          curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/ml-cpp-${ES_VERSION}.zip" https://artifacts-snapshot.elastic.co/ml-cpp/${ES_VERSION}-SNAPSHOT/downloads/ml-cpp/ml-cpp-${ES_VERSION}-SNAPSHOT.zip
 
           $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dbuild.snapshot=false -Dbuild.ml_cpp.repo=file://${ML_IVY_REPO} \
             -Dtests.jvm.argline=-Dbuild.snapshot=false -Dlicense.key=${WORKSPACE}/x-pack/license-tools/src/test/resources/public.key -Dbuild.id=deadbeef build
diff --git a/.ci/matrix-runtime-javas.yml b/.ci/matrix-runtime-javas.yml
index a6b2d4a15d84..07582c4892d5 100644
--- a/.ci/matrix-runtime-javas.yml
+++ b/.ci/matrix-runtime-javas.yml
@@ -10,3 +10,4 @@ ES_RUNTIME_JAVA:
   - openjdk17
   - openjdk18
   - openjdk19
+  - openjdk20
diff --git a/.ci/packer_cache.sh b/.ci/packer_cache.sh
index 5a7b3c792d7f..43d1c5a82b90 100755
--- a/.ci/packer_cache.sh
+++ b/.ci/packer_cache.sh
@@ -43,4 +43,4 @@ fi
 ## Gradle is able to resolve dependencies resolved with earlier gradle versions
 ## therefore we run main _AFTER_ we run 6.8 which uses an earlier gradle version
 export JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA}
-./gradlew --parallel clean -s resolveAllDependencies -Dorg.gradle.warning.mode=none
+./gradlew --parallel clean -s resolveAllDependencies -Dorg.gradle.warning.mode=none -Drecurse.bwc=true
diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions
index dd700ed35e2c..c89f884223c0 100644
--- a/.ci/snapshotBwcVersions
+++ b/.ci/snapshotBwcVersions
@@ -1,5 +1,4 @@
 BWC_VERSION:
-  - "7.17.8"
-  - "8.5.2"
-  - "8.6.0"
+  - "7.17.10"
+  - "8.6.2"
   - "8.7.0"
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 7a95e4eaef1b..6a3d07df917e 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -16,6 +16,6 @@ x-pack/plugin/core/src/main/resources/monitoring-logstash.json @elastic/infra-mo
 x-pack/plugin/core/src/main/resources/monitoring-mb-ilm-policy.json @elastic/infra-monitoring-ui
 x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java @elastic/infra-monitoring-ui
 
-# Elastic Agent
-x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet @elastic/elastic-agent-control-plane
-x-pack/plugin/core/src/main/resources/fleet-* @elastic/elastic-agent-control-plane
+# Fleet
+x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet @elastic/fleet
+x-pack/plugin/core/src/main/resources/fleet-* @elastic/fleet
diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml
index 0bae5de1d0f8..b434572cf6f3 100644
--- a/.github/ISSUE_TEMPLATE/bug.yml
+++ b/.github/ISSUE_TEMPLATE/bug.yml
@@ -4,22 +4,27 @@ labels: [">bug", "needs:triage"]
 body:
   - type: markdown
     attributes:
-      value: |
+      value: >
         Github is reserved for bug reports and feature requests; it is
         not the place for general questions. If you have a question or
         an unconfirmed bug, please visit the [forums](https://discuss.elastic.co/c/elasticsearch).
-        Please also check your OS is [supported](https://www.elastic.co/support/matrix#show_os).
-        If it is not, the issue is likely to be closed.
+        Please also check your OS is [supported](https://www.elastic.co/support/matrix#show_os),
+        and that the version of Elasticsearch has not passed [end-of-life](https://www.elastic.co/support/eol).
+        If you are using an unsupported OS or an unsupported version then the issue is likely to be closed.
+
 
         For security vulnerabilities please only send reports to security@elastic.co.
         See https://www.elastic.co/community/security for more information.
 
+
         Please fill in the following details to help us reproduce the bug:
   - type: input
     id: es_version
     attributes:
       label: Elasticsearch Version
-      description: The version of Elasticsearch you are running, found with `bin/elasticsearch --version`
+      description: >-
+        The version of Elasticsearch you are running, found with `bin/elasticsearch --version`.
+        Ensure this version has not [passed end-of-life](https://www.elastic.co/support/eol).
     validations:
       required: true
   - type: input
diff --git a/BUILDING.md b/BUILDING.md
index fa2258608144..692a7a559de2 100644
--- a/BUILDING.md
+++ b/BUILDING.md
@@ -78,7 +78,7 @@ For updated or newly added dependencies you need to add an entry to this verific
       </component>
 ```
 
-In case of updating a dependency, ensure to remove the unused entry of the outdated dependency manually from the verifcation.xml file.
+In case of updating a dependency, ensure to remove the unused entry of the outdated dependency manually from the `verification-metadata.xml` file.
 
 You can also automate the generation of this entry by running your build using the `--write-verification-metadata` commandline option:
 ```
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 61c31c7665d7..00af53743fd3 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -870,6 +870,17 @@ benefits of this kind of change are very small, and in our experience it is not
 worth investing the substantial effort needed to review them. This especially
 includes changes suggested by tools.
 
+We normally immediately reject PRs which target platforms or system
+configurations that are not in the [official support
+matrix](https://www.elastic.co/support/matrix). We choose to support particular
+platforms with care because we must work to ensure that every Elasticsearch
+release works completely on every platform, and we must spend time
+investigating test failures and performance regressions there too. We cannot
+determine whether PRs which target unsupported platforms or configurations meet
+our quality standards, nor can we guarantee that the change they introduce will
+continue to work in future releases. We do not want Elasticsearch to suddenly
+stop working on a particular platform after an upgrade.
+
 We sometimes reject contributions due to the low quality of the submission
 since low-quality submissions tend to take unreasonable effort to review
 properly. Quality is rather subjective so it is hard to describe exactly how to
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java
index 2a8bf1b91ce7..c69c357b1378 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java
@@ -131,7 +131,7 @@ public void setUp() throws Exception {
             );
         }
         Metadata metadata = mb.build();
-        RoutingTable.Builder rb = RoutingTable.builder();
+        RoutingTable.Builder rb = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         for (int i = 1; i <= numIndices; i++) {
             rb.addAsNew(metadata.index("test_" + i));
         }
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java
index 09080029ba25..f933957c7486 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java
@@ -81,7 +81,8 @@ public static AllocationService createAllocationService(Settings settings, Clust
             NoopGatewayAllocator.INSTANCE,
             new BalancedShardsAllocator(settings),
             EmptyClusterInfoService.INSTANCE,
-            EmptySnapshotsInfoService.INSTANCE
+            EmptySnapshotsInfoService.INSTANCE,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
     }
 
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java
index 20e8106d93c0..deceab114f4f 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.health.HealthIndicatorResult;
 import org.elasticsearch.health.node.HealthInfo;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.openjdk.jmh.annotations.Benchmark;
@@ -45,6 +46,7 @@
 
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
@@ -127,7 +129,7 @@ public void setUp() throws Exception {
                 .numberOfReplicas(numReplicas)
                 .build();
 
-            final IndexRoutingTable.Builder indexRountingTableBuilder = new IndexRoutingTable.Builder(indexMetadata.getIndex());
+            final IndexRoutingTable.Builder indexRountingTableBuilder = IndexRoutingTable.builder(indexMetadata.getIndex());
             for (int shardIdNumber = 0; shardIdNumber < numShards; shardIdNumber++) {
                 ShardId shardId = new ShardId(indexMetadata.getIndex(), shardIdNumber);
                 final IndexShardRoutingTable.Builder shardBuilder = new IndexShardRoutingTable.Builder(shardId);
@@ -135,7 +137,8 @@ public void setUp() throws Exception {
                     shardId,
                     true,
                     RecoverySource.ExistingStoreRecoverySource.INSTANCE,
-                    decidersNoUnassignedInfo
+                    decidersNoUnassignedInfo,
+                    ShardRouting.Role.DEFAULT
                 );
                 shardBuilder.addShard(shardRouting);
                 if (shardIdNumber < numReplicas) {
@@ -144,7 +147,8 @@ public void setUp() throws Exception {
                             shardId,
                             false,
                             RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-                            decidersNoUnassignedInfo
+                            decidersNoUnassignedInfo,
+                            ShardRouting.Role.DEFAULT
                         )
                     );
                 }
@@ -171,7 +175,7 @@ public void setUp() throws Exception {
             new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet())
         );
         clusterService.getClusterApplierService().setInitialState(initialClusterState);
-        indicatorService = new ShardsAvailabilityHealthIndicatorService(clusterService, allocationService);
+        indicatorService = new ShardsAvailabilityHealthIndicatorService(clusterService, allocationService, new SystemIndices(List.of()));
     }
 
     private int toInt(String v) {
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/TestShardRoutingRoleStrategies.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/TestShardRoutingRoleStrategies.java
new file mode 100644
index 000000000000..2c6207285595
--- /dev/null
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/TestShardRoutingRoleStrategies.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.benchmark.routing.allocation;
+
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingRoleStrategy;
+
+public class TestShardRoutingRoleStrategies {
+
+    /**
+     * A strategy which only returns the default role in all situations. This is deliberately not available to production code to avoid any
+     * possibility of using it instead of the strategy provided by the plugin (if so configured).
+     */
+    public static final ShardRoutingRoleStrategy DEFAULT_ROLE_ONLY = new ShardRoutingRoleStrategy() {
+        @Override
+        public ShardRouting.Role newReplicaRole() {
+            return ShardRouting.Role.DEFAULT;
+        }
+
+        @Override
+        public ShardRouting.Role newEmptyRole(int copyIndex) {
+            return ShardRouting.Role.DEFAULT;
+        }
+    };
+
+    private TestShardRoutingRoleStrategies() {
+        // no instances
+    }
+}
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java
index 47f1b4f41353..7e40acff82c8 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java
@@ -40,7 +40,7 @@
 import org.elasticsearch.script.ScoreScript;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.SourceProvider;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -90,7 +90,7 @@ public class ScriptScoreBenchmark {
     private final SearchLookup lookup = new SearchLookup(
         fieldTypes::get,
         (mft, lookup, fdo) -> mft.fielddataBuilder(FieldDataContext.noRuntimeFields("benchmark")).build(fieldDataCache, breakerService),
-        new SourceLookup.ReaderSourceProvider()
+        SourceProvider.fromStoredFields()
     );
 
     @Param({ "expression", "metal", "painless_cast", "painless_def" })
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/DistanceFunctionBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/DistanceFunctionBenchmark.java
new file mode 100644
index 000000000000..1ee1d6217598
--- /dev/null
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/DistanceFunctionBenchmark.java
@@ -0,0 +1,467 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.benchmark.vector;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.script.field.vectors.BinaryDenseVector;
+import org.elasticsearch.script.field.vectors.ByteBinaryDenseVector;
+import org.elasticsearch.script.field.vectors.ByteKnnDenseVector;
+import org.elasticsearch.script.field.vectors.KnnDenseVector;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OperationsPerInvocation;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+/**
+ * Various benchmarks for the distance functions
+ * used by indexed and non-indexed vectors.
+ * Parameters include element, dims, function, and type.
+ * For individual local tests it may be useful to increase
+ * fork, measurement, and operations per invocation. (Note
+ * to also update the benchmark loop if operations per invocation
+ * is increased.)
+ */
+@Fork(1)
+@Warmup(iterations = 1)
+@Measurement(iterations = 2)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@OperationsPerInvocation(25000)
+@State(Scope.Benchmark)
+public class DistanceFunctionBenchmark {
+
+    @Param({ "float", "byte" })
+    private String element;
+
+    @Param({ "96" })
+    private int dims;
+
+    @Param({ "dot", "cosine", "l1", "l2" })
+    private String function;
+
+    @Param({ "knn", "binary" })
+    private String type;
+
+    private abstract static class BenchmarkFunction {
+
+        final int dims;
+
+        private BenchmarkFunction(int dims) {
+            this.dims = dims;
+        }
+
+        abstract void execute(Consumer<Object> consumer);
+    }
+
+    private abstract static class KnnFloatBenchmarkFunction extends BenchmarkFunction {
+
+        final float[] docVector;
+        final float[] queryVector;
+
+        private KnnFloatBenchmarkFunction(int dims, boolean normalize) {
+            super(dims);
+
+            docVector = new float[dims];
+            queryVector = new float[dims];
+
+            float docMagnitude = 0f;
+            float queryMagnitude = 0f;
+
+            for (int i = 0; i < dims; ++i) {
+                docVector[i] = (float) (dims - i);
+                queryVector[i] = (float) i;
+
+                docMagnitude += (float) (dims - i);
+                queryMagnitude += (float) i;
+            }
+
+            docMagnitude /= dims;
+            queryMagnitude /= dims;
+
+            if (normalize) {
+                for (int i = 0; i < dims; ++i) {
+                    docVector[i] /= docMagnitude;
+                    queryVector[i] /= queryMagnitude;
+                }
+            }
+        }
+    }
+
+    private abstract static class BinaryFloatBenchmarkFunction extends BenchmarkFunction {
+
+        final BytesRef docVector;
+        final float[] queryVector;
+
+        private BinaryFloatBenchmarkFunction(int dims, boolean normalize) {
+            super(dims);
+
+            float[] docVector = new float[dims];
+            queryVector = new float[dims];
+
+            float docMagnitude = 0f;
+            float queryMagnitude = 0f;
+
+            for (int i = 0; i < dims; ++i) {
+                docVector[i] = (float) (dims - i);
+                queryVector[i] = (float) i;
+
+                docMagnitude += (float) (dims - i);
+                queryMagnitude += (float) i;
+            }
+
+            docMagnitude /= dims;
+            queryMagnitude /= dims;
+
+            ByteBuffer byteBuffer = ByteBuffer.allocate(dims * 4 + 4);
+
+            for (int i = 0; i < dims; ++i) {
+                if (normalize) {
+                    docVector[i] /= docMagnitude;
+                    queryVector[i] /= queryMagnitude;
+                }
+
+                byteBuffer.putFloat(docVector[i]);
+            }
+
+            byteBuffer.putFloat(docMagnitude);
+            this.docVector = new BytesRef(byteBuffer.array());
+        }
+    }
+
+    private abstract static class KnnByteBenchmarkFunction extends BenchmarkFunction {
+
+        final byte[] docVector;
+        final byte[] queryVector;
+
+        final float queryMagnitude;
+
+        private KnnByteBenchmarkFunction(int dims) {
+            super(dims);
+
+            ByteBuffer docVector = ByteBuffer.allocate(dims);
+            queryVector = new byte[dims];
+
+            float queryMagnitude = 0f;
+
+            for (int i = 0; i < dims; ++i) {
+                docVector.put((byte) (dims - i));
+                queryVector[i] = (byte) i;
+
+                queryMagnitude += (float) i;
+            }
+
+            this.docVector = docVector.array();
+            this.queryMagnitude = queryMagnitude / dims;
+        }
+    }
+
+    private abstract static class BinaryByteBenchmarkFunction extends BenchmarkFunction {
+
+        final BytesRef docVector;
+        final byte[] queryVector;
+
+        final float queryMagnitude;
+
+        private BinaryByteBenchmarkFunction(int dims) {
+            super(dims);
+
+            ByteBuffer docVector = ByteBuffer.allocate(dims + 4);
+            queryVector = new byte[dims];
+
+            float docMagnitude = 0f;
+            float queryMagnitude = 0f;
+
+            for (int i = 0; i < dims; ++i) {
+                docVector.put((byte) (dims - i));
+                queryVector[i] = (byte) i;
+
+                docMagnitude += (float) (dims - i);
+                queryMagnitude += (float) i;
+            }
+
+            docVector.putFloat(docMagnitude / dims);
+            this.docVector = new BytesRef(docVector.array());
+            this.queryMagnitude = queryMagnitude / dims;
+
+        }
+    }
+
+    private static class DotKnnFloatBenchmarkFunction extends KnnFloatBenchmarkFunction {
+
+        private DotKnnFloatBenchmarkFunction(int dims) {
+            super(dims, false);
+        }
+
+        @Override
+        public void execute(Consumer<Object> consumer) {
+            new KnnDenseVector(docVector).dotProduct(queryVector);
+        }
+    }
+
+    private static class DotKnnByteBenchmarkFunction extends KnnByteBenchmarkFunction {
+
+        private DotKnnByteBenchmarkFunction(int dims) {
+            super(dims);
+        }
+
+        @Override
+        public void execute(Consumer<Object> consumer) {
+            new ByteKnnDenseVector(docVector).dotProduct(queryVector);
+        }
+    }
+
+    private static class DotBinaryFloatBenchmarkFunction extends BinaryFloatBenchmarkFunction {
+
+        private DotBinaryFloatBenchmarkFunction(int dims) {
+            super(dims, false);
+        }
+
+        @Override
+        public void execute(Consumer<Object> consumer) {
+            new BinaryDenseVector(docVector, dims, Version.CURRENT).dotProduct(queryVector);
+        }
+    }
+
+    private static class DotBinaryByteBenchmarkFunction extends BinaryByteBenchmarkFunction {
+
+        private DotBinaryByteBenchmarkFunction(int dims) {
+            super(dims);
+        }
+
+        @Override
+        public void execute(Consumer<Object> consumer) {
+            new ByteBinaryDenseVector(docVector, dims).dotProduct(queryVector);
+        }
+    }
+
+    private static class CosineKnnFloatBenchmarkFunction extends KnnFloatBenchmarkFunction {
+
+        private CosineKnnFloatBenchmarkFunction(int dims) {
+            super(dims, true);
+        }
+
+        @Override
+        public void execute(Consumer<Object> consumer) {
+            new KnnDenseVector(docVector).cosineSimilarity(queryVector, false);
+        }
+    }
+
+    private static class CosineKnnByteBenchmarkFunction extends KnnByteBenchmarkFunction {
+
+        private CosineKnnByteBenchmarkFunction(int dims) {
+            super(dims);
+        }
+
+        @Override
+        public void execute(Consumer<Object> consumer) {
+            new ByteKnnDenseVector(docVector).cosineSimilarity(queryVector, queryMagnitude);
+        }
+    }
+
+    private static class CosineBinaryFloatBenchmarkFunction extends BinaryFloatBenchmarkFunction {
+
+        private CosineBinaryFloatBenchmarkFunction(int dims) {
+            super(dims, true);
+        }
+
+        @Override
+        public void execute(Consumer<Object> consumer) {
+            new BinaryDenseVector(docVector, dims, Version.CURRENT).cosineSimilarity(queryVector, false);
+        }
+    }
+
+    private static class CosineBinaryByteBenchmarkFunction extends BinaryByteBenchmarkFunction {
+
+        private CosineBinaryByteBenchmarkFunction(int dims) {
+            super(dims);
+        }
+
+        @Override
+        public void execute(Consumer<Object> consumer) {
+            new ByteBinaryDenseVector(docVector, dims).cosineSimilarity(queryVector, queryMagnitude);
+        }
+    }
+
+    private static class L1KnnFloatBenchmarkFunction extends KnnFloatBenchmarkFunction {
+
+        private L1KnnFloatBenchmarkFunction(int dims) {
+            super(dims, false);
+        }
+
+        @Override
+        public void execute(Consumer<Object> consumer) {
+            new KnnDenseVector(docVector).l1Norm(queryVector);
+        }
+    }
+
+    private static class L1KnnByteBenchmarkFunction extends KnnByteBenchmarkFunction {
+
+        private L1KnnByteBenchmarkFunction(int dims) {
+            super(dims);
+        }
+
+        @Override
+        public void execute(Consumer<Object> consumer) {
+            new ByteKnnDenseVector(docVector).l1Norm(queryVector);
+        }
+    }
+
+    private static class L1BinaryFloatBenchmarkFunction extends BinaryFloatBenchmarkFunction {
+
+        private L1BinaryFloatBenchmarkFunction(int dims) {
+            super(dims, true);
+        }
+
+        @Override
+        public void execute(Consumer<Object> consumer) {
+            new BinaryDenseVector(docVector, dims, Version.CURRENT).l1Norm(queryVector);
+        }
+    }
+
+    private static class L1BinaryByteBenchmarkFunction extends BinaryByteBenchmarkFunction {
+
+        private L1BinaryByteBenchmarkFunction(int dims) {
+            super(dims);
+        }
+
+        @Override
+        public void execute(Consumer<Object> consumer) {
+            new ByteBinaryDenseVector(docVector, dims).l1Norm(queryVector);
+        }
+    }
+
+    private static class L2KnnFloatBenchmarkFunction extends KnnFloatBenchmarkFunction {
+
+        private L2KnnFloatBenchmarkFunction(int dims) {
+            super(dims, false);
+        }
+
+        @Override
+        public void execute(Consumer<Object> consumer) {
+            new KnnDenseVector(docVector).l2Norm(queryVector);
+        }
+    }
+
+    private static class L2KnnByteBenchmarkFunction extends KnnByteBenchmarkFunction {
+
+        private L2KnnByteBenchmarkFunction(int dims) {
+            super(dims);
+        }
+
+        @Override
+        public void execute(Consumer<Object> consumer) {
+            new ByteKnnDenseVector(docVector).l2Norm(queryVector);
+        }
+    }
+
+    private static class L2BinaryFloatBenchmarkFunction extends BinaryFloatBenchmarkFunction {
+
+        private L2BinaryFloatBenchmarkFunction(int dims) {
+            super(dims, true);
+        }
+
+        @Override
+        public void execute(Consumer<Object> consumer) {
+            new BinaryDenseVector(docVector, dims, Version.CURRENT).l1Norm(queryVector);
+        }
+    }
+
+    private static class L2BinaryByteBenchmarkFunction extends BinaryByteBenchmarkFunction {
+
+        private L2BinaryByteBenchmarkFunction(int dims) {
+            super(dims);
+        }
+
+        @Override
+        public void execute(Consumer<Object> consumer) {
+            consumer.accept(new ByteBinaryDenseVector(docVector, dims).l2Norm(queryVector));
+        }
+    }
+
+    private BenchmarkFunction benchmarkFunction;
+
+    @Setup
+    public void setBenchmarkFunction() {
+        switch (element) {
+            case "float" -> {
+                switch (function) {
+                    case "dot" -> benchmarkFunction = switch (type) {
+                            case "knn" -> new DotKnnFloatBenchmarkFunction(dims);
+                            case "binary" -> new DotBinaryFloatBenchmarkFunction(dims);
+                            default -> throw new UnsupportedOperationException("unexpected type [" + type + "]");
+                        };
+                    case "cosine" -> benchmarkFunction = switch (type) {
+                            case "knn" -> new CosineKnnFloatBenchmarkFunction(dims);
+                            case "binary" -> new CosineBinaryFloatBenchmarkFunction(dims);
+                            default -> throw new UnsupportedOperationException("unexpected type [" + type + "]");
+                        };
+                    case "l1" -> benchmarkFunction = switch (type) {
+                            case "knn" -> new L1KnnFloatBenchmarkFunction(dims);
+                            case "binary" -> new L1BinaryFloatBenchmarkFunction(dims);
+                            default -> throw new UnsupportedOperationException("unexpected type [" + type + "]");
+                        };
+                    case "l2" -> benchmarkFunction = switch (type) {
+                            case "knn" -> new L2KnnFloatBenchmarkFunction(dims);
+                            case "binary" -> new L2BinaryFloatBenchmarkFunction(dims);
+                            default -> throw new UnsupportedOperationException("unexpected type [" + type + "]");
+                        };
+                    default -> throw new UnsupportedOperationException("unexpected function [" + function + "]");
+                }
+            }
+            case "byte" -> {
+                switch (function) {
+                    case "dot" -> benchmarkFunction = switch (type) {
+                            case "knn" -> new DotKnnByteBenchmarkFunction(dims);
+                            case "binary" -> new DotBinaryByteBenchmarkFunction(dims);
+                            default -> throw new UnsupportedOperationException("unexpected type [" + type + "]");
+                        };
+                    case "cosine" -> benchmarkFunction = switch (type) {
+                            case "knn" -> new CosineKnnByteBenchmarkFunction(dims);
+                            case "binary" -> new CosineBinaryByteBenchmarkFunction(dims);
+                            default -> throw new UnsupportedOperationException("unexpected type [" + type + "]");
+                        };
+                    case "l1" -> benchmarkFunction = switch (type) {
+                            case "knn" -> new L1KnnByteBenchmarkFunction(dims);
+                            case "binary" -> new L1BinaryByteBenchmarkFunction(dims);
+                            default -> throw new UnsupportedOperationException("unexpected type [" + type + "]");
+                        };
+                    case "l2" -> benchmarkFunction = switch (type) {
+                            case "knn" -> new L2KnnByteBenchmarkFunction(dims);
+                            case "binary" -> new L2BinaryByteBenchmarkFunction(dims);
+                            default -> throw new UnsupportedOperationException("unexpected type [" + type + "]");
+                        };
+                    default -> throw new UnsupportedOperationException("unexpected function [" + function + "]");
+                }
+            }
+            default -> throw new UnsupportedOperationException("unexpected element [" + element + "]");
+        }
+        ;
+    }
+
+    @Benchmark
+    public void benchmark() throws IOException {
+        for (int i = 0; i < 25000; ++i) {
+            benchmarkFunction.execute(Object::toString);
+        }
+    }
+}
diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PrecommitTaskPlugin.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PrecommitTaskPlugin.java
index f6a5db279792..49148330e02e 100644
--- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PrecommitTaskPlugin.java
+++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PrecommitTaskPlugin.java
@@ -31,10 +31,10 @@ public void apply(Project project) {
                         "lifecycle-base",
                         p -> project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(precommit))
                 );
-        project.getPluginManager().withPlugin("java", p -> {
+        project.getPluginManager().withPlugin("java-base", p -> {
             // run compilation as part of precommit
             project.getExtensions().getByType(JavaPluginExtension.class).getSourceSets().all(sourceSet ->
-                    precommit.configure(t -> t.shouldRunAfter(sourceSet.getClassesTaskName()))
+                    precommit.configure(t -> t.dependsOn(sourceSet.getClassesTaskName()))
             );
             // make sure tests run after all precommit tasks
             project.getTasks().withType(Test.class).configureEach(t -> t.mustRunAfter(precommit));
diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle
index 707faf8749c1..3d7edf7ef86f 100644
--- a/build-tools-internal/build.gradle
+++ b/build-tools-internal/build.gradle
@@ -115,6 +115,10 @@ gradlePlugin {
       id = 'elasticsearch.java'
       implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchJavaPlugin'
     }
+    legacyInternalJavaRestTest {
+      id = 'elasticsearch.legacy-java-rest-test'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.rest.LegacyJavaRestTestPlugin'
+    }
     internalJavaRestTest {
       id = 'elasticsearch.internal-java-rest-test'
       implementationClass = 'org.elasticsearch.gradle.internal.test.rest.InternalJavaRestTestPlugin'
@@ -167,9 +171,17 @@ gradlePlugin {
       id = 'elasticsearch.validate-rest-spec'
       implementationClass = 'org.elasticsearch.gradle.internal.precommit.ValidateRestSpecPlugin'
     }
+    legacyYamlRestCompatTest {
+      id = 'elasticsearch.legacy-yaml-rest-compat-test'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.rest.compat.compat.LegacyYamlRestCompatTestPlugin'
+    }
     yamlRestCompatTest {
       id = 'elasticsearch.yaml-rest-compat-test'
-      implementationClass = 'org.elasticsearch.gradle.internal.rest.compat.YamlRestCompatTestPlugin'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.rest.compat.compat.YamlRestCompatTestPlugin'
+    }
+    legacyYamlRestTest {
+      id = 'elasticsearch.legacy-yaml-rest-test'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.rest.LegacyYamlRestTestPlugin'
     }
     yamlRestTest {
       id = 'elasticsearch.internal-yaml-rest-test'
@@ -220,7 +232,7 @@ configurations {
 
 dependencies {
   constraints {
-      integTestImplementation('org.ow2.asm:asm:9.3')
+      integTestImplementation('org.ow2.asm:asm:9.4')
       implementation "org.yaml:snakeyaml:${versions.snakeyaml}"
   }
   components.all(JacksonAlignmentRule)
@@ -324,6 +336,10 @@ tasks.register("bootstrapPerformanceTests", Copy) {
           branchWrapper:"${-> gradle.gradleVersion}".toString()])
 }
 
+tasks.named("jar") {
+  exclude("classpath.index")
+}
+
 def resolveMainWrapperVersion() {
   new URL("https://raw.githubusercontent.com/elastic/elasticsearch/main/build-tools-internal/src/main/resources/minimumGradleVersion").text.trim()
 }
diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy
index 2fa7ed6faaee..2756b9745bc7 100644
--- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy
+++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy
@@ -13,6 +13,7 @@ abstract class AbstractRestResourcesFuncTest extends AbstractGradleFuncTest {
 
     def setup() {
         subProject(":test:framework") << "apply plugin: 'elasticsearch.java'"
+        subProject(":test:test-clusters") << "apply plugin: 'elasticsearch.java'"
         subProject(":test:yaml-rest-runner") << "apply plugin: 'elasticsearch.java'"
 
         subProject(":rest-api-spec") << """
diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy
index 017b7333f8de..31ecffc07f63 100644
--- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy
+++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy
@@ -174,12 +174,12 @@ class TestingConventionsPrecommitPluginFuncTest extends AbstractGradleInternalPl
         given:
         clazz(dir('src/yamlRestTest/java'), "org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase")
         buildFile << """
-        apply plugin:'elasticsearch.internal-yaml-rest-test'
-        
+        apply plugin:'elasticsearch.legacy-yaml-rest-test'
+
         dependencies {
             yamlRestTestImplementation "org.apache.lucene:tests.util:1.0"
             yamlRestTestImplementation "org.junit:junit:4.42"
-        }    
+        }
         """
 
         clazz(dir("src/yamlRestTest/java"), "org.acme.valid.SomeMatchingIT", "org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase") {
@@ -216,11 +216,11 @@ class TestingConventionsPrecommitPluginFuncTest extends AbstractGradleInternalPl
         buildFile << """
         import org.elasticsearch.gradle.internal.precommit.TestingConventionsCheckTask
         apply plugin:'$pluginName'
-        
+
         dependencies {
             ${sourceSetName}Implementation "org.apache.lucene:tests.util:1.0"
             ${sourceSetName}Implementation "org.junit:junit:4.42"
-        }    
+        }
         tasks.withType(TestingConventionsCheckTask).configureEach {
             suffix 'IT'
             suffix 'Tests'
@@ -252,19 +252,19 @@ class TestingConventionsPrecommitPluginFuncTest extends AbstractGradleInternalPl
         )
 
         where:
-        pluginName                              | taskName                                | sourceSetName
-        "elasticsearch.internal-java-rest-test" | ":javaRestTestTestingConventions"       | "javaRestTest"
+        pluginName                              | taskName                                 | sourceSetName
+        "elasticsearch.legacy-java-rest-test"   | ":javaRestTestTestingConventions"        | "javaRestTest"
         "elasticsearch.internal-cluster-test"   | ":internalClusterTestTestingConventions" | "internalClusterTest"
     }
 
     private void simpleJavaBuild() {
         buildFile << """
         apply plugin:'java'
-                
+
         dependencies {
             testImplementation "org.apache.lucene:tests.util:1.0"
             testImplementation "org.junit:junit:4.42"
-        }    
+        }
         """
     }
 }
diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/InternalYamlRestTestPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/InternalYamlRestTestPluginFuncTest.groovy
deleted file mode 100644
index e70be2b4de4d..000000000000
--- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/InternalYamlRestTestPluginFuncTest.groovy
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.gradle.internal.test.rest
-
-import spock.lang.IgnoreIf
-
-import org.elasticsearch.gradle.VersionProperties
-import org.elasticsearch.gradle.fixtures.AbstractRestResourcesFuncTest
-import org.gradle.testkit.runner.TaskOutcome
-
-@IgnoreIf({ os.isWindows() })
-class InternalYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
-
-    def "yamlRestTest does nothing when there are no tests"() {
-        given:
-        // RestIntegTestTask not cc compatible due to
-        configurationCacheCompatible = false
-        buildFile << """
-        plugins {
-          id 'elasticsearch.internal-yaml-rest-test'
-        }
-        """
-
-        when:
-        def result = gradleRunner("yamlRestTest").build()
-
-        then:
-        result.task(':yamlRestTest').outcome == TaskOutcome.NO_SOURCE
-        result.task(':copyYamlTestsTask').outcome == TaskOutcome.NO_SOURCE
-        result.task(':copyRestApiSpecsTask').outcome == TaskOutcome.NO_SOURCE
-    }
-
-    def "yamlRestTest executes and copies api and tests to correct source set"() {
-        given:
-        // RestIntegTestTask not cc compatible due to
-        configurationCacheCompatible = false
-        internalBuild()
-        buildFile << """
-            apply plugin: 'elasticsearch.internal-yaml-rest-test'
-
-            dependencies {
-               yamlRestTestImplementation "junit:junit:4.12"
-            }
-
-            // can't actually spin up test cluster from this test
-           tasks.withType(Test).configureEach{ enabled = false }
-
-           tasks.register("printYamlRestTestClasspath").configure {
-               doLast {
-                   println sourceSets.yamlRestTest.runtimeClasspath.asPath
-               }
-           }
-        """
-        String api = "foo.json"
-        setupRestResources([api])
-        addRestTestsToProject(["10_basic.yml"], "yamlRestTest")
-        file("src/yamlRestTest/java/MockIT.java") << "import org.junit.Test;class MockIT { @Test public void doNothing() { }}"
-
-        when:
-        def result = gradleRunner("yamlRestTest", "printYamlRestTestClasspath").build()
-
-        then:
-        result.task(':yamlRestTest').outcome == TaskOutcome.SKIPPED
-        result.task(':copyRestApiSpecsTask').outcome == TaskOutcome.SUCCESS
-        result.task(':copyYamlTestsTask').outcome == TaskOutcome.NO_SOURCE
-
-        file("/build/restResources/yamlSpecs/rest-api-spec/api/" + api).exists()
-        file("/build/resources/yamlRestTest/rest-api-spec/test/10_basic.yml").exists()
-        file("/build/classes/java/yamlRestTest/MockIT.class").exists()
-
-        // check that our copied specs and tests are on the yamlRestTest classpath
-        result.output.contains("./build/restResources/yamlSpecs")
-        result.output.contains("./build/restResources/yamlTests")
-
-        when:
-        result = gradleRunner("yamlRestTest").build()
-
-        then:
-        result.task(':yamlRestTest').outcome == TaskOutcome.SKIPPED
-        result.task(':copyRestApiSpecsTask').outcome == TaskOutcome.UP_TO_DATE
-        result.task(':copyYamlTestsTask').outcome == TaskOutcome.NO_SOURCE
-    }
-
-    def "#type projects are wired into test cluster setup"() {
-        given:
-        internalBuild()
-        localDistroSetup()
-        def distroVersion = VersionProperties.getElasticsearch()
-
-        def subProjectBuildFile = subProject(pluginProjectPath)
-        subProjectBuildFile << """
-            apply plugin: 'elasticsearch.esplugin'
-            apply plugin: 'elasticsearch.internal-yaml-rest-test'
-
-            dependencies {
-               yamlRestTestImplementation "junit:junit:4.12"
-            }
-
-            esplugin {
-                description = 'test plugin'
-                classname = 'com.acme.plugin.TestPlugin'
-            }
-
-            // for testing purposes only
-            configurations.compileOnly.dependencies.clear()
-
-            testClusters {
-              yamlRestTest {
-                  version = "$distroVersion"
-                  testDistribution = 'INTEG_TEST'
-              }
-            }
-        """
-        def testFile = new File(subProjectBuildFile.parentFile, 'src/yamlRestTest/java/org/acme/SomeTestIT.java')
-        testFile.parentFile.mkdirs()
-        testFile << """
-        package org.acme;
-
-        import org.junit.Test;
-
-        public class SomeTestIT {
-            @Test
-            public void someMethod() {
-            }
-        }
-        """
-
-        when:
-        def result = gradleRunner("yamlRestTest", "--console", 'plain', '--stacktrace').buildAndFail()
-
-        then:
-        result.task(":distribution:archives:integ-test-zip:buildExpanded").outcome == TaskOutcome.SUCCESS
-        result.getOutput().contains(expectedInstallLog)
-
-        where:
-        type     | pluginProjectPath   | expectedInstallLog
-        "plugin" | ":plugins:plugin-a" | "installing 1 plugins in a single transaction"
-        "module" | ":modules:module-a" | "Installing 1 modules"
-    }
-
-    private void localDistroSetup() {
-        settingsFile << """
-        include ":distribution:archives:integ-test-zip"
-        """
-        def distProjectFolder = file("distribution/archives/integ-test-zip")
-        file(distProjectFolder, 'current-marker.txt') << "current"
-
-        def elasticPluginScript = file(distProjectFolder, 'src/bin/elasticsearch-plugin')
-        elasticPluginScript << """#!/bin/bash
-@echo off
-echo "Installing plugin \$0"
-"""
-        assert elasticPluginScript.setExecutable(true)
-
-        def elasticKeystoreScript = file(distProjectFolder, 'src/bin/elasticsearch-keystore')
-        elasticKeystoreScript << """#!/bin/bash
-@echo off
-echo "Installing keystore \$0"
-"""
-        assert elasticKeystoreScript.setExecutable(true)
-
-        def elasticScript = file(distProjectFolder, 'src/bin/elasticsearch')
-        elasticScript << """#!/bin/bash
-@echo off
-echo "Running elasticsearch \$0"
-"""
-        assert elasticScript.setExecutable(true)
-
-        file(distProjectFolder, 'src/config/elasticsearch.properties') << "some propes"
-        file(distProjectFolder, 'src/config/jvm.options') << """
--Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
--XX:ErrorFile=logs/hs_err_pid%p.log
--XX:HeapDumpPath=data
-"""
-        file(distProjectFolder, 'build.gradle') << """
-            import org.gradle.api.internal.artifacts.ArtifactAttributes;
-
-            apply plugin:'distribution'
-            def buildExpanded = tasks.register("buildExpanded", Copy) {
-                into("build/local")
-
-                into('es-dummy-dist') {
-                    from('src')
-                    from('current-marker.txt')
-                }
-            }
-
-            configurations {
-                extracted {
-                    attributes {
-                          attribute(ArtifactAttributes.ARTIFACT_FORMAT, "directory")
-                    }
-                }
-            }
-            artifacts {
-                it.add("extracted", buildExpanded)
-            }
-        """
-    }
-}
diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy
new file mode 100644
index 000000000000..cf0cab3b952f
--- /dev/null
+++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy
@@ -0,0 +1,412 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test.rest
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.databind.SequenceWriter
+import com.fasterxml.jackson.databind.node.ObjectNode
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
+import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.fixtures.AbstractRestResourcesFuncTest
+import org.elasticsearch.gradle.VersionProperties
+import org.gradle.testkit.runner.TaskOutcome
+
+class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
+
+    def compatibleVersion = Version.fromString(VersionProperties.getVersions().get("elasticsearch")).getMajor() - 1
+    def specIntermediateDir = "restResources/v${compatibleVersion}/yamlSpecs"
+    def testIntermediateDir = "restResources/v${compatibleVersion}/yamlTests"
+    def transformTask  = ":yamlRestTestV${compatibleVersion}CompatTransform"
+    def YAML_FACTORY = new YAMLFactory()
+    def MAPPER = new ObjectMapper(YAML_FACTORY)
+    def READER = MAPPER.readerFor(ObjectNode.class)
+    def WRITER = MAPPER.writerFor(ObjectNode.class)
+
+    def setup() {
+        // not cc compatible due to:
+        // 1. TestClustersPlugin not cc compatible due to listener registration
+        // 2. RestIntegTestTask not cc compatible due to
+        configurationCacheCompatible = false
+    }
+    def "yamlRestTestVxCompatTest does nothing when there are no tests"() {
+        given:
+        subProject(":distribution:bwc:maintenance") << """
+        configurations { checkout }
+        artifacts {
+            checkout(new File(projectDir, "checkoutDir"))
+        }
+        """
+
+        buildFile << """
+        plugins {
+          id 'elasticsearch.legacy-yaml-rest-compat-test'
+        }
+        """
+
+        when:
+        def result = gradleRunner("yamlRestTestV${compatibleVersion}CompatTest", '--stacktrace').build()
+
+        then:
+        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.NO_SOURCE
+        result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE
+        result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE
+        result.task(transformTask).outcome == TaskOutcome.NO_SOURCE
+    }
+
+    def "yamlRestTestVxCompatTest executes and copies api and transforms tests from :bwc:maintenance"() {
+        given:
+        internalBuild()
+
+        subProject(":distribution:bwc:maintenance") << """
+        configurations { checkout }
+        artifacts {
+            checkout(new File(projectDir, "checkoutDir"))
+        }
+        """
+
+        buildFile << """
+            apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
+
+            // avoids a dependency problem in this test, the distribution in use here is inconsequential to the test
+            import org.elasticsearch.gradle.testclusters.TestDistribution;
+
+            dependencies {
+               yamlRestTestImplementation "junit:junit:4.12"
+            }
+
+            // can't actually spin up test cluster from this test
+           tasks.withType(Test).configureEach{ enabled = false }
+        """
+
+        String wrongApi = "wrong_version.json"
+        String wrongTest = "wrong_version.yml"
+        String additionalTest = "additional_test.yml"
+        setupRestResources([wrongApi], [wrongTest]) //setups up resources for current version, which should not be used for this test
+        String sourceSetName = "yamlRestTestV" + compatibleVersion + "Compat"
+        addRestTestsToProject([additionalTest], sourceSetName)
+        //intentionally adding to yamlRestTest source set since the .classes are copied from there
+        file("src/yamlRestTest/java/MockIT.java") << "import org.junit.Test;class MockIT { @Test public void doNothing() { }}"
+
+        String api = "foo.json"
+        String test = "10_basic.yml"
+        //add the compatible test and api files, these are the prior version's normal yaml rest tests
+        file("distribution/bwc/maintenance/checkoutDir/rest-api-spec/src/main/resources/rest-api-spec/api/" + api) << ""
+        file("distribution/bwc/maintenance/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/" + test) << ""
+
+        when:
+        def result = gradleRunner("yamlRestTestV${compatibleVersion}CompatTest").build()
+
+        then:
+        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED
+        result.task(':copyRestCompatApiTask').outcome == TaskOutcome.SUCCESS
+        result.task(':copyRestCompatTestTask').outcome == TaskOutcome.SUCCESS
+        result.task(transformTask).outcome == TaskOutcome.SUCCESS
+
+        file("/build/${specIntermediateDir}/rest-api-spec/api/" + api).exists()
+        file("/build/${testIntermediateDir}/original/rest-api-spec/test/" + test).exists()
+        file("/build/${testIntermediateDir}/transformed/rest-api-spec/test/" + test).exists()
+        file("/build/${testIntermediateDir}/original/rest-api-spec/test/" + test).exists()
+        file("/build/${testIntermediateDir}/transformed/rest-api-spec/test/" + test).exists()
+        file("/build/${testIntermediateDir}/transformed/rest-api-spec/test/" + test).text.contains("headers") //transformation adds this
+        file("/build/resources/${sourceSetName}/rest-api-spec/test/" + additionalTest).exists()
+
+        //additionalTest is not copied from the prior version, and thus not in the intermediate directory, nor transformed
+        file("/build/resources/${sourceSetName}/" + testIntermediateDir + "/rest-api-spec/test/" + additionalTest).exists() == false
+        file("/build/resources/${sourceSetName}/rest-api-spec/test/" + additionalTest).text.contains("headers") == false
+
+        file("/build/classes/java/yamlRestTest/MockIT.class").exists() //The "standard" runner is used to execute the compat test
+
+        file("/build/resources/${sourceSetName}/rest-api-spec/api/" + wrongApi).exists() == false
+        file("/build/resources/${sourceSetName}/" + testIntermediateDir + "/rest-api-spec/test/" + wrongTest).exists() == false
+        file("/build/resources/${sourceSetName}/rest-api-spec/test/" + wrongTest).exists() == false
+
+        result.task(':copyRestApiSpecsTask').outcome == TaskOutcome.NO_SOURCE
+        result.task(':copyYamlTestsTask').outcome == TaskOutcome.NO_SOURCE
+
+        when:
+        result = gradleRunner("yamlRestTestV${compatibleVersion}CompatTest").build()
+
+        then:
+        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED
+        result.task(':copyRestCompatApiTask').outcome == TaskOutcome.UP_TO_DATE
+        result.task(':copyRestCompatTestTask').outcome == TaskOutcome.UP_TO_DATE
+        result.task(transformTask).outcome == TaskOutcome.UP_TO_DATE
+    }
+
+    def "yamlRestTestVxCompatTest is wired into check and checkRestCompat"() {
+        given:
+        withVersionCatalogue()
+        subProject(":distribution:bwc:maintenance") << """
+        configurations { checkout }
+        artifacts {
+            checkout(new File(projectDir, "checkoutDir"))
+        }
+        """
+
+        buildFile << """
+        plugins {
+          id 'elasticsearch.legacy-yaml-rest-compat-test'
+        }
+
+        """
+
+        when:
+        def result = gradleRunner("check").build()
+
+        then:
+        result.task(':check').outcome == TaskOutcome.UP_TO_DATE
+        result.task(':checkRestCompat').outcome == TaskOutcome.UP_TO_DATE
+        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.NO_SOURCE
+        result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE
+        result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE
+        result.task(transformTask).outcome == TaskOutcome.NO_SOURCE
+
+        when:
+        buildFile << """
+         ext.bwc_tests_enabled = false
+        """
+        result = gradleRunner("check").build()
+
+        then:
+        result.task(':check').outcome == TaskOutcome.UP_TO_DATE
+        result.task(':checkRestCompat').outcome == TaskOutcome.UP_TO_DATE
+        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED
+        result.task(':copyRestCompatApiTask').outcome == TaskOutcome.SKIPPED
+        result.task(':copyRestCompatTestTask').outcome == TaskOutcome.SKIPPED
+        result.task(transformTask).outcome == TaskOutcome.SKIPPED
+    }
+
+    def "transform task executes and works as configured"() {
+        given:
+        internalBuild()
+
+        subProject(":distribution:bwc:maintenance") << """
+        configurations { checkout }
+        artifacts {
+            checkout(new File(projectDir, "checkoutDir"))
+        }
+        """
+
+        buildFile << """
+            apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
+
+            // avoids a dependency problem in this test, the distribution in use here is inconsequential to the test
+            import org.elasticsearch.gradle.testclusters.TestDistribution;
+
+            dependencies {
+               yamlRestTestImplementation "junit:junit:4.12"
+            }
+            tasks.named("yamlRestTestV${compatibleVersion}CompatTransform").configure({ task ->
+              task.skipTest("test/test/two", "This is a test to skip test two")
+              task.replaceValueInMatch("_type", "_doc")
+              task.replaceValueInMatch("_source.values", ["z", "x", "y"], "one")
+              task.removeMatch("_source.blah")
+              task.removeMatch("_source.junk", "two")
+              task.addMatch("_source.added", [name: 'jake', likes: 'cheese'], "one")
+              task.addWarning("one", "warning1", "warning2")
+              task.addWarningRegex("two", "regex warning here .* [a-z]")
+              task.addAllowedWarning("added allowed warning")
+              task.addAllowedWarningRegex("added allowed warning regex .* [0-9]")
+              task.removeWarning("one", "warning to remove")
+              task.replaceIsTrue("value_to_replace", "replaced_value")
+              task.replaceIsFalse("value_to_replace", "replaced_value")
+              task.replaceKeyInDo("do_.some.key_to_replace", "do_.some.key_that_was_replaced")
+              task.replaceKeyInDo("do_.some.key_to_replace_in_two", "do_.some.key_that_was_replaced_in_two", "two")
+              task.replaceKeyInMatch("match_.some.key_to_replace", "match_.some.key_that_was_replaced")
+              task.replaceKeyInLength("key.in_length_to_replace", "key.in_length_that_was_replaced")
+              task.replaceValueInLength("value_to_replace", 99, "one")
+              task.replaceValueTextByKeyValue("keyvalue", "toreplace", "replacedkeyvalue")
+              task.replaceValueTextByKeyValue("index", "test", "test2", "two")
+            })
+            // can't actually spin up test cluster from this test
+           tasks.withType(Test).configureEach{ enabled = false }
+        """
+
+        setupRestResources([], [])
+
+        file("distribution/bwc/maintenance/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/test.yml" ) << """
+        "one":
+          - do:
+              do_.some.key_to_replace:
+                index: test
+                id: 1
+                keyvalue : toreplace
+              do_.some.key_to_replace_in_two:
+                no_change_here: "because it's not in test 'two'"
+              warnings:
+                - "warning to remove"
+          - match: { _source.values: ["foo"] }
+          - match: { _type: "_foo" }
+          - match: { _source.blah: 1234 }
+          - match: { _source.junk: true }
+          - match: { match_.some.key_to_replace: true }
+          - is_true: "value_to_replace"
+          - is_false: "value_to_replace"
+          - is_true: "value_not_to_replace"
+          - is_false: "value_not_to_replace"
+          - length: { key.in_length_to_replace: 1 }
+          - length: { value_to_replace: 1 }
+        ---
+        "two":
+          - do:
+              get:
+                index: test
+                id: 1
+              do_.some.key_to_replace_in_two:
+                changed_here: "because it is in test 'two'"
+          - match: { _source.values: ["foo"] }
+          - match: { _type: "_foo" }
+          - match: { _source.blah: 1234 }
+          - match: { _source.junk: true }
+          - is_true: "value_to_replace"
+          - is_false: "value_to_replace"
+          - is_true: "value_not_to_replace"
+          - is_false: "value_not_to_replace"
+          - length: { value_not_to_replace: 1 }
+        ---
+        "use cat with no header":
+          - do:
+              cat.indices:
+                {}
+          - match: {}
+        """.stripIndent()
+        when:
+        def result = gradleRunner("yamlRestTestV${compatibleVersion}CompatTest").build()
+
+        then:
+
+        result.task(transformTask).outcome == TaskOutcome.SUCCESS
+
+
+        file("/build/${testIntermediateDir}/transformed/rest-api-spec/test/test.yml" ).exists()
+        List<ObjectNode> actual = READER.readValues(file("/build/${testIntermediateDir}/transformed/rest-api-spec/test/test.yml")).readAll()
+        List<ObjectNode> expectedAll = READER.readValues(
+        """
+        ---
+        setup:
+        - skip:
+            features:
+            - "headers"
+            - "warnings"
+            - "warnings_regex"
+            - "allowed_warnings"
+            - "allowed_warnings_regex"
+        ---
+        one:
+        - do:
+            do_.some.key_that_was_replaced:
+              index: "test"
+              id: 1
+              keyvalue : replacedkeyvalue
+            do_.some.key_to_replace_in_two:
+              no_change_here: "because it's not in test 'two'"
+            warnings:
+            - "warning1"
+            - "warning2"
+            headers:
+              Content-Type: "application/vnd.elasticsearch+json;compatible-with=7"
+              Accept: "application/vnd.elasticsearch+json;compatible-with=7"
+            allowed_warnings:
+            - "added allowed warning"
+            allowed_warnings_regex:
+            - "added allowed warning regex .* [0-9]"
+        - match:
+            _source.values:
+            - "z"
+            - "x"
+            - "y"
+        - match:
+            _type: "_doc"
+        - match: {}
+        - match:
+            _source.junk: true
+        - match:
+            match_.some.key_that_was_replaced: true
+        - is_true: "replaced_value"
+        - is_false: "replaced_value"
+        - is_true: "value_not_to_replace"
+        - is_false: "value_not_to_replace"
+        - length: { key.in_length_that_was_replaced: 1 }
+        - length: { value_to_replace: 99 }
+        - match:
+            _source.added:
+              name: "jake"
+              likes: "cheese"
+
+        ---
+        two:
+        - skip:
+            version: "all"
+            reason: "This is a test to skip test two"
+        - do:
+            get:
+              index: "test2"
+              id: 1
+            do_.some.key_that_was_replaced_in_two:
+                changed_here: "because it is in test 'two'"
+            headers:
+              Content-Type: "application/vnd.elasticsearch+json;compatible-with=7"
+              Accept: "application/vnd.elasticsearch+json;compatible-with=7"
+            warnings_regex:
+            - "regex warning here .* [a-z]"
+            allowed_warnings:
+            - "added allowed warning"
+            allowed_warnings_regex:
+            - "added allowed warning regex .* [0-9]"
+        - match:
+            _source.values:
+            - "foo"
+        - match:
+            _type: "_doc"
+        - match: {}
+        - match: {}
+        - is_true: "replaced_value"
+        - is_false: "replaced_value"
+        - is_true: "value_not_to_replace"
+        - is_false: "value_not_to_replace"
+        - length: { value_not_to_replace: 1 }
+        ---
+        "use cat with no header":
+          - do:
+              cat.indices:
+                {}
+              allowed_warnings:
+                - "added allowed warning"
+              allowed_warnings_regex:
+                - "added allowed warning regex .* [0-9]"
+          - match: {}
+        """.stripIndent()).readAll()
+
+        expectedAll.eachWithIndex{ ObjectNode expected, int i ->
+            if(expected != actual.get(i)) {
+                println("\nTransformed Test:")
+                SequenceWriter sequenceWriter = WRITER.writeValues(System.out)
+                for (ObjectNode transformedTest : actual) {
+                    sequenceWriter.write(transformedTest)
+                }
+                sequenceWriter.close()
+            }
+           assert expected == actual.get(i)
+        }
+
+        when:
+        result = gradleRunner(transformTask).build()
+
+        then:
+        result.task(transformTask).outcome == TaskOutcome.UP_TO_DATE
+
+        when:
+        buildFile.write(buildFile.text.replace("blah", "baz"))
+        result = gradleRunner(transformTask).build()
+
+        then:
+        result.task(transformTask).outcome == TaskOutcome.SUCCESS
+    }
+
+}
diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy
new file mode 100644
index 000000000000..599ad6d2df86
--- /dev/null
+++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy
@@ -0,0 +1,206 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test.rest
+
+import spock.lang.IgnoreIf
+
+import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.fixtures.AbstractRestResourcesFuncTest
+import org.gradle.testkit.runner.TaskOutcome
+
+@IgnoreIf({ os.isWindows() })
+class LegacyYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
+
+    def "yamlRestTest does nothing when there are no tests"() {
+        given:
+        // RestIntegTestTask not cc compatible due to
+        configurationCacheCompatible = false
+        buildFile << """
+        plugins {
+          id 'elasticsearch.legacy-yaml-rest-test'
+        }
+        """
+
+        when:
+        def result = gradleRunner("yamlRestTest").build()
+
+        then:
+        result.task(':yamlRestTest').outcome == TaskOutcome.NO_SOURCE
+        result.task(':copyYamlTestsTask').outcome == TaskOutcome.NO_SOURCE
+        result.task(':copyRestApiSpecsTask').outcome == TaskOutcome.NO_SOURCE
+    }
+
+    def "yamlRestTest executes and copies api and tests to correct source set"() {
+        given:
+        // RestIntegTestTask not cc compatible due to
+        configurationCacheCompatible = false
+        internalBuild()
+        buildFile << """
+            apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+
+            dependencies {
+               yamlRestTestImplementation "junit:junit:4.12"
+            }
+
+            // can't actually spin up test cluster from this test
+           tasks.withType(Test).configureEach{ enabled = false }
+
+           tasks.register("printYamlRestTestClasspath").configure {
+               doLast {
+                   println sourceSets.yamlRestTest.runtimeClasspath.asPath
+               }
+           }
+        """
+        String api = "foo.json"
+        setupRestResources([api])
+        addRestTestsToProject(["10_basic.yml"], "yamlRestTest")
+        file("src/yamlRestTest/java/MockIT.java") << "import org.junit.Test;class MockIT { @Test public void doNothing() { }}"
+
+        when:
+        def result = gradleRunner("yamlRestTest", "printYamlRestTestClasspath").build()
+
+        then:
+        result.task(':yamlRestTest').outcome == TaskOutcome.SKIPPED
+        result.task(':copyRestApiSpecsTask').outcome == TaskOutcome.SUCCESS
+        result.task(':copyYamlTestsTask').outcome == TaskOutcome.NO_SOURCE
+
+        file("/build/restResources/yamlSpecs/rest-api-spec/api/" + api).exists()
+        file("/build/resources/yamlRestTest/rest-api-spec/test/10_basic.yml").exists()
+        file("/build/classes/java/yamlRestTest/MockIT.class").exists()
+
+        // check that our copied specs and tests are on the yamlRestTest classpath
+        result.output.contains("./build/restResources/yamlSpecs")
+        result.output.contains("./build/restResources/yamlTests")
+
+        when:
+        result = gradleRunner("yamlRestTest").build()
+
+        then:
+        result.task(':yamlRestTest').outcome == TaskOutcome.SKIPPED
+        result.task(':copyRestApiSpecsTask').outcome == TaskOutcome.UP_TO_DATE
+        result.task(':copyYamlTestsTask').outcome == TaskOutcome.NO_SOURCE
+    }
+
+    def "#type projects are wired into test cluster setup"() {
+        given:
+        internalBuild()
+        localDistroSetup()
+        def distroVersion = VersionProperties.getElasticsearch()
+
+        def subProjectBuildFile = subProject(pluginProjectPath)
+        subProjectBuildFile << """
+            apply plugin: 'elasticsearch.esplugin'
+            apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+
+            dependencies {
+               yamlRestTestImplementation "junit:junit:4.12"
+            }
+
+            esplugin {
+                description = 'test plugin'
+                classname = 'com.acme.plugin.TestPlugin'
+            }
+
+            // for testing purposes only
+            configurations.compileOnly.dependencies.clear()
+
+            testClusters {
+              yamlRestTest {
+                  version = "$distroVersion"
+                  testDistribution = 'INTEG_TEST'
+              }
+            }
+        """
+        def testFile = new File(subProjectBuildFile.parentFile, 'src/yamlRestTest/java/org/acme/SomeTestIT.java')
+        testFile.parentFile.mkdirs()
+        testFile << """
+        package org.acme;
+
+        import org.junit.Test;
+
+        public class SomeTestIT {
+            @Test
+            public void someMethod() {
+            }
+        }
+        """
+
+        when:
+        def result = gradleRunner("yamlRestTest", "--console", 'plain', '--stacktrace').buildAndFail()
+
+        then:
+        result.task(":distribution:archives:integ-test-zip:buildExpanded").outcome == TaskOutcome.SUCCESS
+        result.getOutput().contains(expectedInstallLog)
+
+        where:
+        type     | pluginProjectPath   | expectedInstallLog
+        "plugin" | ":plugins:plugin-a" | "installing 1 plugins in a single transaction"
+        "module" | ":modules:module-a" | "Installing 1 modules"
+    }
+
+    private void localDistroSetup() {
+        settingsFile << """
+        include ":distribution:archives:integ-test-zip"
+        """
+        def distProjectFolder = file("distribution/archives/integ-test-zip")
+        file(distProjectFolder, 'current-marker.txt') << "current"
+
+        def elasticPluginScript = file(distProjectFolder, 'src/bin/elasticsearch-plugin')
+        elasticPluginScript << """#!/bin/bash
+@echo off
+echo "Installing plugin \$0"
+"""
+        assert elasticPluginScript.setExecutable(true)
+
+        def elasticKeystoreScript = file(distProjectFolder, 'src/bin/elasticsearch-keystore')
+        elasticKeystoreScript << """#!/bin/bash
+@echo off
+echo "Installing keystore \$0"
+"""
+        assert elasticKeystoreScript.setExecutable(true)
+
+        def elasticScript = file(distProjectFolder, 'src/bin/elasticsearch')
+        elasticScript << """#!/bin/bash
+@echo off
+echo "Running elasticsearch \$0"
+"""
+        assert elasticScript.setExecutable(true)
+
+        file(distProjectFolder, 'src/config/elasticsearch.properties') << "some propes"
+        file(distProjectFolder, 'src/config/jvm.options') << """
+-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,level,pid,tags:filecount=32,filesize=64m
+-XX:ErrorFile=logs/hs_err_pid%p.log
+-XX:HeapDumpPath=data
+"""
+        file(distProjectFolder, 'build.gradle') << """
+            import org.gradle.api.internal.artifacts.ArtifactAttributes;
+
+            apply plugin:'distribution'
+            def buildExpanded = tasks.register("buildExpanded", Copy) {
+                into("build/local")
+
+                into('es-dummy-dist') {
+                    from('src')
+                    from('current-marker.txt')
+                }
+            }
+
+            configurations {
+                extracted {
+                    attributes {
+                          attribute(ArtifactAttributes.ARTIFACT_FORMAT, "directory")
+                    }
+                }
+            }
+            artifacts {
+                it.add("extracted", buildExpanded)
+            }
+        """
+    }
+}
diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy
deleted file mode 100644
index 58f894489ab7..000000000000
--- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy
+++ /dev/null
@@ -1,412 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.gradle.internal.test.rest
-
-import com.fasterxml.jackson.databind.ObjectMapper
-import com.fasterxml.jackson.databind.SequenceWriter
-import com.fasterxml.jackson.databind.node.ObjectNode
-import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
-import org.elasticsearch.gradle.Version
-import org.elasticsearch.gradle.fixtures.AbstractRestResourcesFuncTest
-import org.elasticsearch.gradle.VersionProperties
-import org.gradle.testkit.runner.TaskOutcome
-
-class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
-
-    def compatibleVersion = Version.fromString(VersionProperties.getVersions().get("elasticsearch")).getMajor() - 1
-    def specIntermediateDir = "restResources/v${compatibleVersion}/yamlSpecs"
-    def testIntermediateDir = "restResources/v${compatibleVersion}/yamlTests"
-    def transformTask  = ":yamlRestTestV${compatibleVersion}CompatTransform"
-    def YAML_FACTORY = new YAMLFactory()
-    def MAPPER = new ObjectMapper(YAML_FACTORY)
-    def READER = MAPPER.readerFor(ObjectNode.class)
-    def WRITER = MAPPER.writerFor(ObjectNode.class)
-
-    def setup() {
-        // not cc compatible due to:
-        // 1. TestClustersPlugin not cc compatible due to listener registration
-        // 2. RestIntegTestTask not cc compatible due to
-        configurationCacheCompatible = false
-    }
-    def "yamlRestTestVxCompatTest does nothing when there are no tests"() {
-        given:
-        subProject(":distribution:bwc:maintenance") << """
-        configurations { checkout }
-        artifacts {
-            checkout(new File(projectDir, "checkoutDir"))
-        }
-        """
-
-        buildFile << """
-        plugins {
-          id 'elasticsearch.yaml-rest-compat-test'
-        }
-        """
-
-        when:
-        def result = gradleRunner("yamlRestTestV${compatibleVersion}CompatTest", '--stacktrace').build()
-
-        then:
-        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.NO_SOURCE
-        result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE
-        result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE
-        result.task(transformTask).outcome == TaskOutcome.NO_SOURCE
-    }
-
-    def "yamlRestTestVxCompatTest executes and copies api and transforms tests from :bwc:maintenance"() {
-        given:
-        internalBuild()
-
-        subProject(":distribution:bwc:maintenance") << """
-        configurations { checkout }
-        artifacts {
-            checkout(new File(projectDir, "checkoutDir"))
-        }
-        """
-
-        buildFile << """
-            apply plugin: 'elasticsearch.yaml-rest-compat-test'
-
-            // avoids a dependency problem in this test, the distribution in use here is inconsequential to the test
-            import org.elasticsearch.gradle.testclusters.TestDistribution;
-
-            dependencies {
-               yamlRestTestImplementation "junit:junit:4.12"
-            }
-
-            // can't actually spin up test cluster from this test
-           tasks.withType(Test).configureEach{ enabled = false }
-        """
-
-        String wrongApi = "wrong_version.json"
-        String wrongTest = "wrong_version.yml"
-        String additionalTest = "additional_test.yml"
-        setupRestResources([wrongApi], [wrongTest]) //setups up resources for current version, which should not be used for this test
-        String sourceSetName = "yamlRestTestV" + compatibleVersion + "Compat"
-        addRestTestsToProject([additionalTest], sourceSetName)
-        //intentionally adding to yamlRestTest source set since the .classes are copied from there
-        file("src/yamlRestTest/java/MockIT.java") << "import org.junit.Test;class MockIT { @Test public void doNothing() { }}"
-
-        String api = "foo.json"
-        String test = "10_basic.yml"
-        //add the compatible test and api files, these are the prior version's normal yaml rest tests
-        file("distribution/bwc/maintenance/checkoutDir/rest-api-spec/src/main/resources/rest-api-spec/api/" + api) << ""
-        file("distribution/bwc/maintenance/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/" + test) << ""
-
-        when:
-        def result = gradleRunner("yamlRestTestV${compatibleVersion}CompatTest").build()
-
-        then:
-        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED
-        result.task(':copyRestCompatApiTask').outcome == TaskOutcome.SUCCESS
-        result.task(':copyRestCompatTestTask').outcome == TaskOutcome.SUCCESS
-        result.task(transformTask).outcome == TaskOutcome.SUCCESS
-
-        file("/build/${specIntermediateDir}/rest-api-spec/api/" + api).exists()
-        file("/build/${testIntermediateDir}/original/rest-api-spec/test/" + test).exists()
-        file("/build/${testIntermediateDir}/transformed/rest-api-spec/test/" + test).exists()
-        file("/build/${testIntermediateDir}/original/rest-api-spec/test/" + test).exists()
-        file("/build/${testIntermediateDir}/transformed/rest-api-spec/test/" + test).exists()
-        file("/build/${testIntermediateDir}/transformed/rest-api-spec/test/" + test).text.contains("headers") //transformation adds this
-        file("/build/resources/${sourceSetName}/rest-api-spec/test/" + additionalTest).exists()
-
-        //additionalTest is not copied from the prior version, and thus not in the intermediate directory, nor transformed
-        file("/build/resources/${sourceSetName}/" + testIntermediateDir + "/rest-api-spec/test/" + additionalTest).exists() == false
-        file("/build/resources/${sourceSetName}/rest-api-spec/test/" + additionalTest).text.contains("headers") == false
-
-        file("/build/classes/java/yamlRestTest/MockIT.class").exists() //The "standard" runner is used to execute the compat test
-
-        file("/build/resources/${sourceSetName}/rest-api-spec/api/" + wrongApi).exists() == false
-        file("/build/resources/${sourceSetName}/" + testIntermediateDir + "/rest-api-spec/test/" + wrongTest).exists() == false
-        file("/build/resources/${sourceSetName}/rest-api-spec/test/" + wrongTest).exists() == false
-
-        result.task(':copyRestApiSpecsTask').outcome == TaskOutcome.NO_SOURCE
-        result.task(':copyYamlTestsTask').outcome == TaskOutcome.NO_SOURCE
-
-        when:
-        result = gradleRunner("yamlRestTestV${compatibleVersion}CompatTest").build()
-
-        then:
-        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED
-        result.task(':copyRestCompatApiTask').outcome == TaskOutcome.UP_TO_DATE
-        result.task(':copyRestCompatTestTask').outcome == TaskOutcome.UP_TO_DATE
-        result.task(transformTask).outcome == TaskOutcome.UP_TO_DATE
-    }
-
-    def "yamlRestTestVxCompatTest is wired into check and checkRestCompat"() {
-        given:
-        withVersionCatalogue()
-        subProject(":distribution:bwc:maintenance") << """
-        configurations { checkout }
-        artifacts {
-            checkout(new File(projectDir, "checkoutDir"))
-        }
-        """
-
-        buildFile << """
-        plugins {
-          id 'elasticsearch.yaml-rest-compat-test'
-        }
-
-        """
-
-        when:
-        def result = gradleRunner("check").build()
-
-        then:
-        result.task(':check').outcome == TaskOutcome.UP_TO_DATE
-        result.task(':checkRestCompat').outcome == TaskOutcome.UP_TO_DATE
-        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.NO_SOURCE
-        result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE
-        result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE
-        result.task(transformTask).outcome == TaskOutcome.NO_SOURCE
-
-        when:
-        buildFile << """
-         ext.bwc_tests_enabled = false
-        """
-        result = gradleRunner("check").build()
-
-        then:
-        result.task(':check').outcome == TaskOutcome.UP_TO_DATE
-        result.task(':checkRestCompat').outcome == TaskOutcome.UP_TO_DATE
-        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED
-        result.task(':copyRestCompatApiTask').outcome == TaskOutcome.SKIPPED
-        result.task(':copyRestCompatTestTask').outcome == TaskOutcome.SKIPPED
-        result.task(transformTask).outcome == TaskOutcome.SKIPPED
-    }
-
-    def "transform task executes and works as configured"() {
-        given:
-        internalBuild()
-
-        subProject(":distribution:bwc:maintenance") << """
-        configurations { checkout }
-        artifacts {
-            checkout(new File(projectDir, "checkoutDir"))
-        }
-        """
-
-        buildFile << """
-            apply plugin: 'elasticsearch.yaml-rest-compat-test'
-
-            // avoids a dependency problem in this test, the distribution in use here is inconsequential to the test
-            import org.elasticsearch.gradle.testclusters.TestDistribution;
-
-            dependencies {
-               yamlRestTestImplementation "junit:junit:4.12"
-            }
-            tasks.named("yamlRestTestV${compatibleVersion}CompatTransform").configure({ task ->
-              task.skipTest("test/test/two", "This is a test to skip test two")
-              task.replaceValueInMatch("_type", "_doc")
-              task.replaceValueInMatch("_source.values", ["z", "x", "y"], "one")
-              task.removeMatch("_source.blah")
-              task.removeMatch("_source.junk", "two")
-              task.addMatch("_source.added", [name: 'jake', likes: 'cheese'], "one")
-              task.addWarning("one", "warning1", "warning2")
-              task.addWarningRegex("two", "regex warning here .* [a-z]")
-              task.addAllowedWarning("added allowed warning")
-              task.addAllowedWarningRegex("added allowed warning regex .* [0-9]")
-              task.removeWarning("one", "warning to remove")
-              task.replaceIsTrue("value_to_replace", "replaced_value")
-              task.replaceIsFalse("value_to_replace", "replaced_value")
-              task.replaceKeyInDo("do_.some.key_to_replace", "do_.some.key_that_was_replaced")
-              task.replaceKeyInDo("do_.some.key_to_replace_in_two", "do_.some.key_that_was_replaced_in_two", "two")
-              task.replaceKeyInMatch("match_.some.key_to_replace", "match_.some.key_that_was_replaced")
-              task.replaceKeyInLength("key.in_length_to_replace", "key.in_length_that_was_replaced")
-              task.replaceValueInLength("value_to_replace", 99, "one")
-              task.replaceValueTextByKeyValue("keyvalue", "toreplace", "replacedkeyvalue")
-              task.replaceValueTextByKeyValue("index", "test", "test2", "two")
-            })
-            // can't actually spin up test cluster from this test
-           tasks.withType(Test).configureEach{ enabled = false }
-        """
-
-        setupRestResources([], [])
-
-        file("distribution/bwc/maintenance/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/test.yml" ) << """
-        "one":
-          - do:
-              do_.some.key_to_replace:
-                index: test
-                id: 1
-                keyvalue : toreplace
-              do_.some.key_to_replace_in_two:
-                no_change_here: "because it's not in test 'two'"
-              warnings:
-                - "warning to remove"
-          - match: { _source.values: ["foo"] }
-          - match: { _type: "_foo" }
-          - match: { _source.blah: 1234 }
-          - match: { _source.junk: true }
-          - match: { match_.some.key_to_replace: true }
-          - is_true: "value_to_replace"
-          - is_false: "value_to_replace"
-          - is_true: "value_not_to_replace"
-          - is_false: "value_not_to_replace"
-          - length: { key.in_length_to_replace: 1 }
-          - length: { value_to_replace: 1 }
-        ---
-        "two":
-          - do:
-              get:
-                index: test
-                id: 1
-              do_.some.key_to_replace_in_two:
-                changed_here: "because it is in test 'two'"
-          - match: { _source.values: ["foo"] }
-          - match: { _type: "_foo" }
-          - match: { _source.blah: 1234 }
-          - match: { _source.junk: true }
-          - is_true: "value_to_replace"
-          - is_false: "value_to_replace"
-          - is_true: "value_not_to_replace"
-          - is_false: "value_not_to_replace"
-          - length: { value_not_to_replace: 1 }
-        ---
-        "use cat with no header":
-          - do:
-              cat.indices:
-                {}
-          - match: {}
-        """.stripIndent()
-        when:
-        def result = gradleRunner("yamlRestTestV${compatibleVersion}CompatTest").build()
-
-        then:
-
-        result.task(transformTask).outcome == TaskOutcome.SUCCESS
-
-
-        file("/build/${testIntermediateDir}/transformed/rest-api-spec/test/test.yml" ).exists()
-        List<ObjectNode> actual = READER.readValues(file("/build/${testIntermediateDir}/transformed/rest-api-spec/test/test.yml")).readAll()
-        List<ObjectNode> expectedAll = READER.readValues(
-        """
-        ---
-        setup:
-        - skip:
-            features:
-            - "headers"
-            - "warnings"
-            - "warnings_regex"
-            - "allowed_warnings"
-            - "allowed_warnings_regex"
-        ---
-        one:
-        - do:
-            do_.some.key_that_was_replaced:
-              index: "test"
-              id: 1
-              keyvalue : replacedkeyvalue
-            do_.some.key_to_replace_in_two:
-              no_change_here: "because it's not in test 'two'"
-            warnings:
-            - "warning1"
-            - "warning2"
-            headers:
-              Content-Type: "application/vnd.elasticsearch+json;compatible-with=7"
-              Accept: "application/vnd.elasticsearch+json;compatible-with=7"
-            allowed_warnings:
-            - "added allowed warning"
-            allowed_warnings_regex:
-            - "added allowed warning regex .* [0-9]"
-        - match:
-            _source.values:
-            - "z"
-            - "x"
-            - "y"
-        - match:
-            _type: "_doc"
-        - match: {}
-        - match:
-            _source.junk: true
-        - match:
-            match_.some.key_that_was_replaced: true
-        - is_true: "replaced_value"
-        - is_false: "replaced_value"
-        - is_true: "value_not_to_replace"
-        - is_false: "value_not_to_replace"
-        - length: { key.in_length_that_was_replaced: 1 }
-        - length: { value_to_replace: 99 }
-        - match:
-            _source.added:
-              name: "jake"
-              likes: "cheese"
-
-        ---
-        two:
-        - skip:
-            version: "all"
-            reason: "This is a test to skip test two"
-        - do:
-            get:
-              index: "test2"
-              id: 1
-            do_.some.key_that_was_replaced_in_two:
-                changed_here: "because it is in test 'two'"
-            headers:
-              Content-Type: "application/vnd.elasticsearch+json;compatible-with=7"
-              Accept: "application/vnd.elasticsearch+json;compatible-with=7"
-            warnings_regex:
-            - "regex warning here .* [a-z]"
-            allowed_warnings:
-            - "added allowed warning"
-            allowed_warnings_regex:
-            - "added allowed warning regex .* [0-9]"
-        - match:
-            _source.values:
-            - "foo"
-        - match:
-            _type: "_doc"
-        - match: {}
-        - match: {}
-        - is_true: "replaced_value"
-        - is_false: "replaced_value"
-        - is_true: "value_not_to_replace"
-        - is_false: "value_not_to_replace"
-        - length: { value_not_to_replace: 1 }
-        ---
-        "use cat with no header":
-          - do:
-              cat.indices:
-                {}
-              allowed_warnings:
-                - "added allowed warning"
-              allowed_warnings_regex:
-                - "added allowed warning regex .* [0-9]"
-          - match: {}
-        """.stripIndent()).readAll()
-
-        expectedAll.eachWithIndex{ ObjectNode expected, int i ->
-            if(expected != actual.get(i)) {
-                println("\nTransformed Test:")
-                SequenceWriter sequenceWriter = WRITER.writeValues(System.out)
-                for (ObjectNode transformedTest : actual) {
-                    sequenceWriter.write(transformedTest)
-                }
-                sequenceWriter.close()
-            }
-           assert expected == actual.get(i)
-        }
-
-        when:
-        result = gradleRunner(transformTask).build()
-
-        then:
-        result.task(transformTask).outcome == TaskOutcome.UP_TO_DATE
-
-        when:
-        buildFile.write(buildFile.text.replace("blah", "baz"))
-        result = gradleRunner(transformTask).build()
-
-        then:
-        result.task(transformTask).outcome == TaskOutcome.SUCCESS
-    }
-
-}
diff --git a/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle b/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle
index b80c450c5914..a5e74c372129 100644
--- a/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle
+++ b/build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle
@@ -9,6 +9,8 @@
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.internal.ElasticsearchTestBasePlugin
 import org.elasticsearch.gradle.internal.info.BuildParams
+import org.elasticsearch.gradle.internal.test.rest.InternalJavaRestTestPlugin
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
 ext.bwcTaskName = { Version version ->
   return "v${version}#bwcTest"
@@ -36,5 +38,17 @@ plugins.withType(ElasticsearchTestBasePlugin) {
   }
 }
 
+plugins.withType(InternalJavaRestTestPlugin) {
+  tasks.named("javaRestTest") {
+    enabled = false
+  }
+
+  tasks.withType(StandaloneRestIntegTestTask).configureEach {
+    testClassesDirs = sourceSets.javaRestTest.output.classesDirs
+    classpath = sourceSets.javaRestTest.runtimeClasspath
+    usesDefaultDistribution()
+  }
+}
+
 tasks.matching { it.name.equals("check") }.configureEach {dependsOn(bwcTestSnapshots) }
 tasks.matching { it.name.equals("test") }.configureEach {enabled = false}
diff --git a/build-tools-internal/src/main/groovy/elasticsearch.fips.gradle b/build-tools-internal/src/main/groovy/elasticsearch.fips.gradle
index 0fe3e8dd7846..08fbc5b67e97 100644
--- a/build-tools-internal/src/main/groovy/elasticsearch.fips.gradle
+++ b/build-tools-internal/src/main/groovy/elasticsearch.fips.gradle
@@ -9,6 +9,8 @@
 
 import org.elasticsearch.gradle.internal.ExportElasticsearchBuildResourcesTask
 import org.elasticsearch.gradle.internal.info.BuildParams
+import org.elasticsearch.gradle.internal.test.rest.RestTestBasePlugin
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 import org.elasticsearch.gradle.testclusters.TestClustersAware
 import org.elasticsearch.gradle.testclusters.TestDistribution
 
@@ -76,6 +78,14 @@ if (BuildParams.inFipsJvm) {
           keystorePassword 'keystore-password'
         }
       }
+
+      plugins.withType(RestTestBasePlugin) {
+        tasks.withType(StandaloneRestIntegTestTask).configureEach {
+          inputs.files(extraFipsJarsConfiguration).withNormalizer(ClasspathNormalizer)
+          nonInputProperties.systemProperty "tests.cluster.fips.jars.path", "${-> extraFipsJarsConfiguration.asPath}"
+        }
+      }
+
       project.tasks.withType(Test).configureEach { Test task ->
         dependsOn 'fipsResources'
         task.systemProperty('javax.net.ssl.trustStorePassword', 'password')
diff --git a/build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle b/build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle
index 1d1112706fd0..6a6dbe43875c 100644
--- a/build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle
+++ b/build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle
@@ -10,6 +10,8 @@ import org.elasticsearch.gradle.Architecture
 import org.elasticsearch.gradle.OS
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.internal.info.BuildParams
+import org.elasticsearch.gradle.internal.test.rest.RestTestBasePlugin
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
 // gradle has an open issue of failing applying plugins in
 // precompiled script plugins (see https://github.com/gradle/gradle/issues/17004)
@@ -42,4 +44,13 @@ configure(allprojects) {
             }
         }
     }
+
+    project.plugins.withType(RestTestBasePlugin) {
+      tasks.withType(StandaloneRestIntegTestTask).configureEach {
+        if (BuildParams.getIsRuntimeJavaHomeSet() == false) {
+          dependsOn(project.jdks.provisioned_runtime)
+          nonInputProperties.systemProperty("tests.runtime.java", "${-> project.jdks.provisioned_runtime.javaHomePath}")
+        }
+      }
+    }
 }
diff --git a/build-tools-internal/src/main/groovy/elasticsearch.stable-api.gradle b/build-tools-internal/src/main/groovy/elasticsearch.stable-api.gradle
new file mode 100644
index 000000000000..c52bd9d1d52c
--- /dev/null
+++ b/build-tools-internal/src/main/groovy/elasticsearch.stable-api.gradle
@@ -0,0 +1,50 @@
+import org.apache.tools.ant.taskdefs.condition.Os
+import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.internal.BwcVersions
+import org.elasticsearch.gradle.internal.JarApiComparisonTask
+import org.elasticsearch.gradle.internal.info.BuildParams
+
+import static org.elasticsearch.gradle.internal.InternalDistributionBwcSetupPlugin.buildBwcTaskName
+
+configurations {
+  newJar
+}
+
+dependencies {
+  newJar project(":libs:${project.name}")
+}
+
+BuildParams.bwcVersions.withIndexCompatible({ it.onOrAfter(Version.fromString(ext.stableApiSince))
+  && it != VersionProperties.elasticsearchVersion
+}) { bwcVersion, baseName ->
+
+  BwcVersions.UnreleasedVersionInfo unreleasedVersion = BuildParams.bwcVersions.unreleasedInfo(bwcVersion)
+
+  configurations {
+    "oldJar${baseName}" {
+      transitive = false
+    }
+  }
+
+  dependencies {
+    if (unreleasedVersion) {
+      // For unreleased snapshot versions, build them from source
+      "oldJar${baseName}"(files(project(unreleasedVersion.gradleProjectPath).tasks.named(buildBwcTaskName(project.name))))
+    } else {
+      // For released versions, download it
+      "oldJar${baseName}"("org.elasticsearch:${project.name}:${bwcVersion}")
+    }
+  }
+
+  def jarApiComparisonTask = tasks.register(bwcTaskName(bwcVersion), JarApiComparisonTask) {
+    oldJar = configurations."oldJar${baseName}"
+    newJar = configurations.newJar
+  }
+
+  jarApiComparisonTask.configure {
+    onlyIf {
+      !Os.isFamily(Os.FAMILY_WINDOWS)
+    }
+  }
+}
diff --git a/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/DocsTestPlugin.groovy b/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/DocsTestPlugin.groovy
index f56a9fefceac..874141f2135a 100644
--- a/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/DocsTestPlugin.groovy
+++ b/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/DocsTestPlugin.groovy
@@ -38,7 +38,7 @@ class DocsTestPlugin implements Plugin<Project> {
 
     @Override
     void apply(Project project) {
-        project.pluginManager.apply('elasticsearch.internal-yaml-rest-test')
+        project.pluginManager.apply('elasticsearch.legacy-yaml-rest-test')
 
         String distribution = System.getProperty('tests.distribution', 'default')
         // The distribution can be configured with -Dtests.distribution on the command line
diff --git a/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromSnippetsTask.groovy b/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromSnippetsTask.groovy
index b9b77da16bd6..eda86355ee30 100644
--- a/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromSnippetsTask.groovy
+++ b/build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromSnippetsTask.groovy
@@ -210,6 +210,12 @@ class RestTestsFromSnippetsTask extends SnippetsTask {
                 return
             }
             if (snippet.testResponse || snippet.language == 'console-result') {
+                if (previousTest == null) {
+                    throw new InvalidUserDataException("$snippet: No paired previous test")
+                }
+                if (previousTest.path != snippet.path) {
+                    throw new InvalidUserDataException("$snippet: Result can't be first in file")
+                }
                 response(snippet)
                 return
             }
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java
index 28b6c1e66992..ef7c9507d865 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java
@@ -12,6 +12,7 @@
 import org.apache.tools.ant.taskdefs.condition.Os;
 import org.elasticsearch.gradle.LoggedExec;
 import org.elasticsearch.gradle.Version;
+import org.elasticsearch.gradle.internal.info.BuildParams;
 import org.gradle.api.Action;
 import org.gradle.api.GradleException;
 import org.gradle.api.Project;
@@ -37,30 +38,35 @@ public class BwcSetupExtension {
     private static final Version BUILD_TOOL_MINIMUM_VERSION = Version.fromString("7.14.0");
     private final Project project;
     private final Provider<BwcVersions.UnreleasedVersionInfo> unreleasedVersionInfo;
-    private final Provider<InternalDistributionBwcSetupPlugin.BwcTaskThrottle> bwcTaskThrottleProvider;
 
     private Provider<File> checkoutDir;
 
     public BwcSetupExtension(
         Project project,
         Provider<BwcVersions.UnreleasedVersionInfo> unreleasedVersionInfo,
-        Provider<InternalDistributionBwcSetupPlugin.BwcTaskThrottle> bwcTaskThrottleProvider,
         Provider<File> checkoutDir
     ) {
         this.project = project;
         this.unreleasedVersionInfo = unreleasedVersionInfo;
-        this.bwcTaskThrottleProvider = bwcTaskThrottleProvider;
         this.checkoutDir = checkoutDir;
     }
 
     TaskProvider<LoggedExec> bwcTask(String name, Action<LoggedExec> configuration) {
-        return createRunBwcGradleTask(project, name, configuration);
+        return bwcTask(name, configuration, true);
     }
 
-    private TaskProvider<LoggedExec> createRunBwcGradleTask(Project project, String name, Action<LoggedExec> configAction) {
+    TaskProvider<LoggedExec> bwcTask(String name, Action<LoggedExec> configuration, boolean useUniqueUserHome) {
+        return createRunBwcGradleTask(project, name, configuration, useUniqueUserHome);
+    }
+
+    private TaskProvider<LoggedExec> createRunBwcGradleTask(
+        Project project,
+        String name,
+        Action<LoggedExec> configAction,
+        boolean useUniqueUserHome
+    ) {
         return project.getTasks().register(name, LoggedExec.class, loggedExec -> {
             loggedExec.dependsOn("checkoutBwcBranch");
-            loggedExec.usesService(bwcTaskThrottleProvider);
             loggedExec.getWorkingDir().set(checkoutDir.get());
 
             loggedExec.getEnvironment().put("JAVA_HOME", unreleasedVersionInfo.zip(checkoutDir, (version, checkoutDir) -> {
@@ -68,6 +74,11 @@ private TaskProvider<LoggedExec> createRunBwcGradleTask(Project project, String
                 return getJavaHome(Integer.parseInt(minimumCompilerVersion));
             }));
 
+            if (BuildParams.isCi() && Os.isFamily(Os.FAMILY_WINDOWS) == false) {
+                // TODO: Disabled for now until we can figure out why files are getting corrupted
+                // loggedExec.getEnvironment().put("GRADLE_RO_DEP_CACHE", System.getProperty("user.home") + "/gradle_ro_cache");
+            }
+
             if (Os.isFamily(Os.FAMILY_WINDOWS)) {
                 loggedExec.getExecutable().set("cmd");
                 loggedExec.args("/C", "call", new File(checkoutDir.get(), "gradlew").toString());
@@ -75,7 +86,11 @@ private TaskProvider<LoggedExec> createRunBwcGradleTask(Project project, String
                 loggedExec.getExecutable().set(new File(checkoutDir.get(), "gradlew").toString());
             }
 
-            loggedExec.args("-g", project.getGradle().getGradleUserHomeDir().toString());
+            if (useUniqueUserHome) {
+                loggedExec.dependsOn("setupGradleUserHome");
+                loggedExec.args("-g", project.getGradle().getGradleUserHomeDir().getAbsolutePath() + "-" + project.getName());
+            }
+
             if (project.getGradle().getStartParameter().isOffline()) {
                 loggedExec.args("--offline");
             }
@@ -101,6 +116,9 @@ private TaskProvider<LoggedExec> createRunBwcGradleTask(Project project, String
             if (project.getGradle().getStartParameter().isParallelProjectExecutionEnabled()) {
                 loggedExec.args("--parallel");
             }
+            for (File initScript : project.getGradle().getStartParameter().getInitScripts()) {
+                loggedExec.args("-I", initScript.getAbsolutePath());
+            }
             loggedExec.getIndentingConsoleOutput().set(unreleasedVersionInfo.map(v -> v.version().toString()));
             configAction.execute(loggedExec);
         });
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java
index fa578ffa7e20..7a5bead71fb0 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java
@@ -38,8 +38,8 @@ public class ElasticsearchJavaBasePlugin implements Plugin<Project> {
     public void apply(Project project) {
         // make sure the global build info plugin is applied to the root project
         project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
-        // common repositories setup
         project.getPluginManager().apply(JavaBasePlugin.class);
+        // common repositories setup
         project.getPluginManager().apply(RepositoriesSetupPlugin.class);
         project.getPluginManager().apply(ElasticsearchTestBasePlugin.class);
         project.getPluginManager().apply(PrecommitTaskPlugin.class);
@@ -146,6 +146,7 @@ public static void configureCompile(Project project) {
      */
     public static void configureInputNormalization(Project project) {
         project.getNormalization().getRuntimeClasspath().ignore("META-INF/MANIFEST.MF");
+        project.getNormalization().getRuntimeClasspath().ignore("IMPL-JARS/**/META-INF/MANIFEST.MF");
     }
 
     private static Provider<Integer> releaseVersionProviderFromCompileTask(Project project, AbstractCompile compileTask) {
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java
index c33545ada993..c6758092b17e 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java
@@ -54,7 +54,8 @@ public void apply(Project project) {
         project.getTasks().withType(Test.class).configureEach(test -> {
             File testOutputDir = new File(test.getReports().getJunitXml().getOutputLocation().getAsFile().get(), "output");
 
-            ErrorReportingTestListener listener = new ErrorReportingTestListener(test.getTestLogging(), test.getLogger(), testOutputDir);
+            ErrorReportingTestListener listener = new ErrorReportingTestListener(test, testOutputDir);
+            test.getExtensions().getExtraProperties().set("dumpOutputOnFailure", true);
             test.getExtensions().add("errorReportingTestListener", listener);
             test.addTestOutputListener(listener);
             test.addTestListener(listener);
@@ -88,7 +89,7 @@ public void execute(Task t) {
             test.getJvmArgumentProviders().add(nonInputProperties);
             test.getExtensions().add("nonInputProperties", nonInputProperties);
 
-            test.setWorkingDir(project.file(project.getBuildDir() + "/testrun/" + test.getName()));
+            test.setWorkingDir(project.file(project.getBuildDir() + "/testrun/" + test.getName().replace("#", "_")));
             test.setMaxParallelForks(Integer.parseInt(System.getProperty("tests.jvms", BuildParams.getDefaultParallel().toString())));
 
             test.exclude("**/*$*.class");
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java
index f974b02a1c5b..a32358c6db4f 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java
@@ -18,8 +18,7 @@
 import org.gradle.api.Task;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
-import org.gradle.api.services.BuildService;
-import org.gradle.api.services.BuildServiceParameters;
+import org.gradle.api.tasks.Copy;
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.language.base.plugins.LifecycleBasePlugin;
 
@@ -28,6 +27,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 import javax.inject.Inject;
@@ -44,7 +44,6 @@
  */
 public class InternalDistributionBwcSetupPlugin implements Plugin<Project> {
 
-    private static final String BWC_TASK_THROTTLE_SERVICE = "bwcTaskThrottle";
     private ProviderFactory providerFactory;
 
     @Inject
@@ -55,26 +54,19 @@ public InternalDistributionBwcSetupPlugin(ProviderFactory providerFactory) {
     @Override
     public void apply(Project project) {
         project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
-        Provider<BwcTaskThrottle> bwcTaskThrottleProvider = project.getGradle()
-            .getSharedServices()
-            .registerIfAbsent(BWC_TASK_THROTTLE_SERVICE, BwcTaskThrottle.class, spec -> spec.getMaxParallelUsages().set(1));
         BuildParams.getBwcVersions()
             .forPreviousUnreleased(
                 (BwcVersions.UnreleasedVersionInfo unreleasedVersion) -> {
-                    configureBwcProject(project.project(unreleasedVersion.gradleProjectPath()), unreleasedVersion, bwcTaskThrottleProvider);
+                    configureBwcProject(project.project(unreleasedVersion.gradleProjectPath()), unreleasedVersion);
                 }
             );
     }
 
-    private void configureBwcProject(
-        Project project,
-        BwcVersions.UnreleasedVersionInfo versionInfo,
-        Provider<BwcTaskThrottle> bwcTaskThrottleProvider
-    ) {
+    private void configureBwcProject(Project project, BwcVersions.UnreleasedVersionInfo versionInfo) {
         Provider<BwcVersions.UnreleasedVersionInfo> versionInfoProvider = providerFactory.provider(() -> versionInfo);
         Provider<File> checkoutDir = versionInfoProvider.map(info -> new File(project.getBuildDir(), "bwc/checkout-" + info.branch()));
         BwcSetupExtension bwcSetupExtension = project.getExtensions()
-            .create("bwcSetup", BwcSetupExtension.class, project, versionInfoProvider, bwcTaskThrottleProvider, checkoutDir);
+            .create("bwcSetup", BwcSetupExtension.class, project, versionInfoProvider, checkoutDir);
         BwcGitExtension gitExtension = project.getPlugins().apply(InternalBwcGitPlugin.class).getGitExtension();
         Provider<Version> bwcVersion = versionInfoProvider.map(info -> info.version());
         gitExtension.setBwcVersion(versionInfoProvider.map(info -> info.version()));
@@ -87,6 +79,15 @@ private void configureBwcProject(
         TaskProvider<Task> buildBwcTaskProvider = project.getTasks().register("buildBwc");
         List<DistributionProject> distributionProjects = resolveArchiveProjects(checkoutDir.get(), bwcVersion.get());
 
+        // Setup gradle user home directory
+        project.getTasks().register("setupGradleUserHome", Copy.class, copy -> {
+            copy.into(project.getGradle().getGradleUserHomeDir().getAbsolutePath() + "-" + project.getName());
+            copy.from(project.getGradle().getGradleUserHomeDir().getAbsolutePath(), copySpec -> {
+                copySpec.include("gradle.properties");
+                copySpec.include("init.d/*");
+            });
+        });
+
         for (DistributionProject distributionProject : distributionProjects) {
             createBuildBwcTask(
                 bwcSetupExtension,
@@ -120,6 +121,35 @@ private void configureBwcProject(
             buildBwcTaskProvider,
             "assemble"
         );
+
+        // for versions before 8.7.0, we do not need to set up stable API bwc
+        if (bwcVersion.get().before(Version.fromString("8.7.0"))) {
+            return;
+        }
+
+        for (Project stableApiProject : resolveStableProjects(project)) {
+
+            String relativeDir = project.getRootProject().relativePath(stableApiProject.getProjectDir());
+
+            DistributionProjectArtifact stableAnalysisPluginProjectArtifact = new DistributionProjectArtifact(
+                new File(
+                    checkoutDir.get(),
+                    relativeDir + "/build/distributions/" + stableApiProject.getName() + "-" + bwcVersion.get() + "-SNAPSHOT.jar"
+                ),
+                null
+            );
+
+            createBuildBwcTask(
+                bwcSetupExtension,
+                project,
+                bwcVersion,
+                stableApiProject.getName(),
+                "libs/" + stableApiProject.getName(),
+                stableAnalysisPluginProjectArtifact,
+                buildBwcTaskProvider,
+                "assemble"
+            );
+        }
     }
 
     private void registerBwcDistributionArtifacts(Project bwcProject, DistributionProject distributionProject) {
@@ -209,7 +239,16 @@ private static List<DistributionProject> resolveArchiveProjects(File checkoutDir
         }).collect(Collectors.toList());
     }
 
-    private static String buildBwcTaskName(String projectName) {
+    private static List<Project> resolveStableProjects(Project project) {
+        Set<String> stableProjectNames = Set.of("elasticsearch-logging", "elasticsearch-plugin-api", "elasticsearch-plugin-analysis-api");
+        return project.findProject(":libs")
+            .getSubprojects()
+            .stream()
+            .filter(subproject -> stableProjectNames.contains(subproject.getName()))
+            .toList();
+    }
+
+    public static String buildBwcTaskName(String projectName) {
         return "buildBwc"
             + stream(projectName.split("-")).map(i -> i.substring(0, 1).toUpperCase(Locale.ROOT) + i.substring(1))
                 .collect(Collectors.joining());
@@ -237,11 +276,7 @@ static void createBuildBwcTask(
             } else {
                 c.getOutputs().files(expectedOutputFile);
             }
-            c.getOutputs().cacheIf("BWC distribution caching is disabled on 'main' branch", task -> {
-                String gitBranch = System.getenv("GIT_BRANCH");
-                return BuildParams.isCi()
-                    && (gitBranch == null || gitBranch.endsWith("master") == false || gitBranch.endsWith("main") == false);
-            });
+            c.getOutputs().doNotCacheIf("BWC distribution caching is disabled for local builds", task -> BuildParams.isCi() == false);
             c.getArgs().add(projectPath.replace('/', ':') + ":" + assembleTaskName);
             if (project.getGradle().getStartParameter().isBuildCacheEnabled()) {
                 c.getArgs().add("--build-cache");
@@ -323,5 +358,4 @@ private static class DistributionProjectArtifact {
         }
     }
 
-    public abstract class BwcTaskThrottle implements BuildService<BuildServiceParameters.None> {}
 }
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java
index 0a29741be893..51239853aa12 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java
@@ -54,8 +54,8 @@ public class InternalDistributionModuleCheckTaskProvider {
         "org.elasticsearch.geo",
         "org.elasticsearch.logging",
         "org.elasticsearch.lz4",
-        "org.elasticsearch.plugin.analysis.api",
-        "org.elasticsearch.plugin.api",
+        "org.elasticsearch.plugin",
+        "org.elasticsearch.plugin.analysis",
         "org.elasticsearch.pluginclassloader",
         "org.elasticsearch.securesm",
         "org.elasticsearch.server",
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactExtension.java
index fae845b22965..4952085f466b 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactExtension.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactExtension.java
@@ -32,7 +32,7 @@ public void registerTestArtifactFromSourceSet(SourceSet sourceSet) {
         JavaPluginExtension javaPluginExtension = project.getExtensions().getByType(JavaPluginExtension.class);
         javaPluginExtension.registerFeature(name + "Artifacts", featureSpec -> {
             featureSpec.usingSourceSet(sourceSet);
-            featureSpec.capability("org.elasticsearch.gradle", project.getName() + "-" + name + "-artifacts", "1.0");
+            featureSpec.capability("org.elasticsearch.gradle", project.getName() + "-test-artifacts", "1.0");
             // This feature is only used internally in the
             // elasticsearch build so we do not need any publication.
             featureSpec.disablePublication();
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JarApiComparisonTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JarApiComparisonTask.java
new file mode 100644
index 000000000000..b1da632d84cd
--- /dev/null
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JarApiComparisonTask.java
@@ -0,0 +1,224 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal;
+
+import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitTask;
+import org.gradle.api.file.FileCollection;
+import org.gradle.api.provider.Property;
+import org.gradle.api.tasks.CacheableTask;
+import org.gradle.api.tasks.CompileClasspath;
+import org.gradle.api.tasks.TaskAction;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.jar.JarFile;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.zip.ZipEntry;
+
+/**
+ * This implementation of a jar API comparison uses the "javap" tool to compare
+ * the "signatures" of two different jars. We assume that calling out to javap
+ * is not too expensive at this stage of the stable API project. We also assume
+ * that for every public class, method, and field, javap will print a consistent
+ * single line. This should let us make string comparisons, rather than having
+ * to parse the output of javap.
+ * <p>
+ * While the above assumptions appear to hold, they are not guaranteed, and hence
+ * brittle. We could overcome these problems with an ASM implementation of the
+ * Jar Scanner.
+ * <p>
+ * We also assume that we will not be comparing multi-version JARs.
+ * <p>
+ * This "javap" approach has a few further drawbacks:
+ * <ol>
+ *     <li>We don't account for class visibility when examining fields and methods.</li>
+ *     <li>We don't consider what is exported from the module. Is a public method from
+ *     a non-exported package considered part of the stable api?</li>
+ *     <li>Changing method types to their superclass or return types to an implementation
+ *     class will be considered a change by this approach, even though that doesn't break
+ *     an API.</li>
+ *     <li>Finally, moving a method up the class hierarchy is not really a breaking change,
+ *     but it will trip this test.</li>
+ * </ol>
+ */
+@CacheableTask
+public abstract class JarApiComparisonTask extends PrecommitTask {
+
+    @TaskAction
+    public void compare() {
+        FileCollection fileCollection = getOldJar().get();
+        File newJarFile = getNewJar().get().getSingleFile();
+
+        Set<String> oldJarNames = fileCollection.getFiles().stream().map(File::getName).collect(Collectors.toSet());
+        if (oldJarNames.size() > 1) {
+            throw new IllegalStateException("Expected a single original jar, but found: " + oldJarNames);
+        }
+        if (oldJarNames.contains(newJarFile.getName())) {
+            throw new IllegalStateException(
+                "We should be comparing different jars, but original and new jars were both: " + newJarFile.getAbsolutePath()
+            );
+        }
+
+        JarScanner oldJS = new JarScanner(getOldJar().get().getSingleFile().getPath());
+        JarScanner newJS = new JarScanner(newJarFile.getPath());
+        try {
+            JarScanner.compareSignatures(oldJS.jarSignature(), newJS.jarSignature());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @CompileClasspath
+    public abstract Property<FileCollection> getOldJar();
+
+    @CompileClasspath
+    public abstract Property<FileCollection> getNewJar();
+
+    public static class JarScanner {
+
+        private final String path;
+
+        public JarScanner(String path) {
+            this.path = path;
+        }
+
+        private String getPath() {
+            return path;
+        }
+
+        /**
+         * Get a list of class names contained in this jar by looking for file names
+         * that end in ".class"
+         */
+        List<String> classNames() throws IOException {
+            Pattern classEnding = Pattern.compile(".*\\.class$");
+            try (JarFile jf = new JarFile(this.path)) {
+                return jf.stream().map(ZipEntry::getName).filter(classEnding.asMatchPredicate()).collect(Collectors.toList());
+            }
+        }
+
+        /**
+         * Given a path to a file in the jar, get the output of javap as a list of strings.
+         */
+        public List<String> disassembleFromJar(String fileInJarPath, String classpath) {
+            String location = "jar:file://" + getPath() + "!/" + fileInJarPath;
+            return disassemble(location, getPath(), classpath);
+        }
+
+        /**
+         * Invoke javap on a class file, optionally providing a module path or class path
+         */
+        static List<String> disassemble(String location, String modulePath, String classpath) {
+            ProcessBuilder pb = new ProcessBuilder();
+            List<String> command = new ArrayList<>();
+            command.add("javap");
+            if (modulePath != null) {
+                command.add("--module-path");
+                command.add(modulePath);
+            }
+            if (classpath != null) {
+                command.add("--class-path");
+                command.add(classpath);
+            }
+            command.add(location);
+            pb.command(command.toArray(new String[] {}));
+            Process p;
+            try {
+                p = pb.start();
+                p.onExit().get();
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+
+            InputStream streamToRead = p.exitValue() == 0 ? p.getInputStream() : p.getErrorStream();
+
+            try (BufferedReader br = new BufferedReader(new InputStreamReader(streamToRead))) {
+                return br.lines().toList();
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        /**
+         * Given the output of the javap command, that is, the disassembled class file,
+         * return a set of signatures for all public classes, methods, and fields.
+         */
+        public static Set<String> signaturesSet(List<String> javapOutput) {
+            return javapOutput.stream().filter(s -> s.matches("^\\s*public.*")).collect(Collectors.toSet());
+        }
+
+        /**
+         * Given a disassembled module-info.class, return all unqualified exports.
+         */
+        public static Set<String> moduleInfoSignaturesSet(List<String> javapOutput) {
+            return javapOutput.stream()
+                .filter(s -> s.matches("^\\s*exports.*"))
+                .filter(s -> s.matches(".* to$") == false)
+                .collect(Collectors.toSet());
+        }
+
+        /**
+         * Iterate over classes and gather signatures.
+         */
+        public Map<String, Set<String>> jarSignature() throws IOException {
+            return this.classNames().stream().collect(Collectors.toMap(s -> s, s -> {
+                List<String> disassembled = disassembleFromJar(s, null);
+                if ("module-info.class".equals(s)) {
+                    return moduleInfoSignaturesSet(disassembled);
+                }
+                return signaturesSet(disassembled);
+            }));
+        }
+
+        /**
+         * Comparison: The signatures are maps of class names to public class, field, or method
+         * declarations.
+         * </p>
+         * First, we check that the new jar signature contains all the same classes
+         * as the old jar signature. If not, we return an error.
+         * </p>
+         * Second, we iterate over the signature for each class. If a signature from the old
+         * jar is absent in the new jar, we add it to our list of errors.
+         * </p>
+         * Note that it is fine for the new jar to have additional elements, as this
+         * is backwards compatible.
+         */
+        public static void compareSignatures(Map<String, Set<String>> oldSignature, Map<String, Set<String>> newSignature) {
+            Set<String> deletedClasses = new HashSet<>(oldSignature.keySet());
+            deletedClasses.removeAll(newSignature.keySet());
+            if (deletedClasses.size() > 0) {
+                throw new IllegalStateException("Classes from a previous version not found: " + deletedClasses);
+            }
+
+            Map<String, Set<String>> deletedMembersMap = new HashMap<>();
+            for (Map.Entry<String, Set<String>> entry : oldSignature.entrySet()) {
+                Set<String> deletedMembers = new HashSet<>(entry.getValue());
+                deletedMembers.removeAll(newSignature.get(entry.getKey()));
+                if (deletedMembers.size() > 0) {
+                    deletedMembersMap.put(entry.getKey(), Set.copyOf(deletedMembers));
+                }
+            }
+            if (deletedMembersMap.size() > 0) {
+                throw new IllegalStateException(
+                    "Classes from a previous version have been modified, violating backwards compatibility: " + deletedMembersMap
+                );
+            }
+        }
+    }
+}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ResolveAllDependencies.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ResolveAllDependencies.java
index d86ec9001d41..0afa675c9dfc 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ResolveAllDependencies.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ResolveAllDependencies.java
@@ -21,7 +21,7 @@
 import javax.inject.Inject;
 
 import static org.elasticsearch.gradle.DistributionDownloadPlugin.DISTRO_EXTRACTED_CONFIG_PREFIX;
-import static org.elasticsearch.gradle.internal.rest.compat.YamlRestCompatTestPlugin.BWC_MINOR_CONFIG_NAME;
+import static org.elasticsearch.gradle.internal.test.rest.compat.compat.LegacyYamlRestCompatTestPlugin.BWC_MINOR_CONFIG_NAME;
 
 public class ResolveAllDependencies extends DefaultTask {
 
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParams.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParams.java
index 33a859747681..c3feb367c76e 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParams.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParams.java
@@ -18,6 +18,7 @@
 import java.time.ZonedDateTime;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Random;
 import java.util.function.Consumer;
 
 import static java.util.Objects.requireNonNull;
@@ -110,10 +111,18 @@ public static String getTestSeed() {
         return value(testSeed);
     }
 
+    public static Random getRandom() {
+        return new Random(Long.parseUnsignedLong(testSeed.split(":")[0], 16));
+    }
+
     public static Boolean isCi() {
         return value(isCi);
     }
 
+    public static Boolean isGraalVmRuntime() {
+        return value(runtimeJavaDetails.toLowerCase().contains("graalvm"));
+    }
+
     public static Integer getDefaultParallel() {
         return value(defaultParallel);
     }
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPlugin.java
index 95d0ad2be4cc..6adf422133db 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPlugin.java
@@ -11,7 +11,8 @@
 import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin;
 import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin;
 import org.elasticsearch.gradle.internal.test.rest.InternalJavaRestTestPlugin;
-import org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin;
+import org.elasticsearch.gradle.internal.test.rest.LegacyJavaRestTestPlugin;
+import org.elasticsearch.gradle.internal.test.rest.LegacyYamlRestTestPlugin;
 import org.gradle.api.Action;
 import org.gradle.api.NamedDomainObjectProvider;
 import org.gradle.api.Project;
@@ -43,8 +44,8 @@ public TaskProvider<? extends Task> createTask(Project project) {
             });
         });
 
-        project.getPlugins().withType(InternalYamlRestTestPlugin.class, yamlRestTestPlugin -> {
-            NamedDomainObjectProvider<SourceSet> sourceSet = sourceSets.named(InternalYamlRestTestPlugin.SOURCE_SET_NAME);
+        project.getPlugins().withType(LegacyYamlRestTestPlugin.class, yamlRestTestPlugin -> {
+            NamedDomainObjectProvider<SourceSet> sourceSet = sourceSets.named(LegacyYamlRestTestPlugin.SOURCE_SET_NAME);
             setupTaskForSourceSet(project, sourceSet, t -> {
                 t.getSuffixes().convention(List.of("IT"));
                 t.getBaseClasses().convention(List.of("org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase"));
@@ -68,8 +69,17 @@ public TaskProvider<? extends Task> createTask(Project project) {
             });
         });
 
+        project.getPlugins().withType(LegacyJavaRestTestPlugin.class, javaRestTestPlugin -> {
+            NamedDomainObjectProvider<SourceSet> sourceSet = sourceSets.named(LegacyJavaRestTestPlugin.SOURCE_SET_NAME);
+            setupTaskForSourceSet(project, sourceSet, t -> {
+                t.getSuffixes().convention(List.of("IT"));
+                t.getBaseClasses()
+                    .convention(List.of("org.elasticsearch.test.ESIntegTestCase", "org.elasticsearch.test.rest.ESRestTestCase"));
+            });
+        });
+
         project.getPlugins().withType(InternalJavaRestTestPlugin.class, javaRestTestPlugin -> {
-            NamedDomainObjectProvider<SourceSet> sourceSet = sourceSets.named(InternalJavaRestTestPlugin.SOURCE_SET_NAME);
+            NamedDomainObjectProvider<SourceSet> sourceSet = sourceSets.named(LegacyJavaRestTestPlugin.SOURCE_SET_NAME);
             setupTaskForSourceSet(project, sourceSet, t -> {
                 t.getSuffixes().convention(List.of("IT"));
                 t.getBaseClasses()
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/rest/compat/YamlRestCompatTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/rest/compat/YamlRestCompatTestPlugin.java
deleted file mode 100644
index bb245bec61b9..000000000000
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/rest/compat/YamlRestCompatTestPlugin.java
+++ /dev/null
@@ -1,271 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.gradle.internal.rest.compat;
-
-import org.elasticsearch.gradle.Version;
-import org.elasticsearch.gradle.VersionProperties;
-import org.elasticsearch.gradle.internal.ElasticsearchJavaBasePlugin;
-import org.elasticsearch.gradle.internal.test.RestIntegTestTask;
-import org.elasticsearch.gradle.internal.test.RestTestBasePlugin;
-import org.elasticsearch.gradle.internal.test.rest.CopyRestApiTask;
-import org.elasticsearch.gradle.internal.test.rest.CopyRestTestsTask;
-import org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin;
-import org.elasticsearch.gradle.internal.test.rest.RestResourcesExtension;
-import org.elasticsearch.gradle.internal.test.rest.RestResourcesPlugin;
-import org.elasticsearch.gradle.internal.test.rest.RestTestUtil;
-import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
-import org.elasticsearch.gradle.util.GradleUtils;
-import org.gradle.api.Plugin;
-import org.gradle.api.Project;
-import org.gradle.api.Task;
-import org.gradle.api.artifacts.Configuration;
-import org.gradle.api.artifacts.Dependency;
-import org.gradle.api.file.Directory;
-import org.gradle.api.file.ProjectLayout;
-import org.gradle.api.file.RelativePath;
-import org.gradle.api.internal.file.FileOperations;
-import org.gradle.api.plugins.ExtraPropertiesExtension;
-import org.gradle.api.plugins.JavaBasePlugin;
-import org.gradle.api.provider.Provider;
-import org.gradle.api.tasks.SourceSet;
-import org.gradle.api.tasks.SourceSetContainer;
-import org.gradle.api.tasks.Sync;
-import org.gradle.api.tasks.TaskProvider;
-
-import java.io.File;
-import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.Map;
-
-import javax.inject.Inject;
-
-import static org.elasticsearch.gradle.internal.test.rest.RestTestUtil.setupYamlRestTestDependenciesDefaults;
-
-/**
- * Apply this plugin to run the YAML based REST tests from a prior major version against this version's cluster.
- */
-public class YamlRestCompatTestPlugin implements Plugin<Project> {
-    public static final String BWC_MINOR_CONFIG_NAME = "bwcMinor";
-    private static final String REST_COMPAT_CHECK_TASK_NAME = "checkRestCompat";
-    private static final String COMPATIBILITY_APIS_CONFIGURATION = "restCompatSpecs";
-    private static final String COMPATIBILITY_TESTS_CONFIGURATION = "restCompatTests";
-    private static final Path RELATIVE_API_PATH = Path.of("rest-api-spec/api");
-    private static final Path RELATIVE_TEST_PATH = Path.of("rest-api-spec/test");
-    private static final Path RELATIVE_REST_API_RESOURCES = Path.of("rest-api-spec/src/main/resources");
-    private static final Path RELATIVE_REST_CORE = Path.of("rest-api-spec");
-    private static final Path RELATIVE_REST_XPACK = Path.of("x-pack/plugin");
-    private static final Path RELATIVE_REST_PROJECT_RESOURCES = Path.of("src/yamlRestTest/resources");
-    private static final int COMPATIBLE_VERSION = Version.fromString(VersionProperties.getVersions().get("elasticsearch")).getMajor() - 1;
-    private static final String SOURCE_SET_NAME = "yamlRestTestV" + COMPATIBLE_VERSION + "Compat";
-    private ProjectLayout projectLayout;
-    private FileOperations fileOperations;
-
-    @Inject
-    public YamlRestCompatTestPlugin(ProjectLayout projectLayout, FileOperations fileOperations) {
-        this.projectLayout = projectLayout;
-        this.fileOperations = fileOperations;
-    }
-
-    @Override
-    public void apply(Project project) {
-
-        final Path compatRestResourcesDir = Path.of("restResources").resolve("v" + COMPATIBLE_VERSION);
-        final Path compatSpecsDir = compatRestResourcesDir.resolve("yamlSpecs");
-        final Path compatTestsDir = compatRestResourcesDir.resolve("yamlTests");
-
-        project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class);
-        project.getPluginManager().apply(TestClustersPlugin.class);
-        project.getPluginManager().apply(RestTestBasePlugin.class);
-        project.getPluginManager().apply(RestResourcesPlugin.class);
-        project.getPluginManager().apply(InternalYamlRestTestPlugin.class);
-
-        RestResourcesExtension extension = project.getExtensions().getByType(RestResourcesExtension.class);
-
-        // create source set
-        SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-        SourceSet yamlCompatTestSourceSet = sourceSets.create(SOURCE_SET_NAME);
-        SourceSet yamlTestSourceSet = sourceSets.getByName(InternalYamlRestTestPlugin.SOURCE_SET_NAME);
-        GradleUtils.extendSourceSet(project, InternalYamlRestTestPlugin.SOURCE_SET_NAME, SOURCE_SET_NAME);
-
-        // copy compatible rest specs
-        Configuration bwcMinorConfig = project.getConfigurations().create(BWC_MINOR_CONFIG_NAME);
-        Dependency bwcMinor = project.getDependencies()
-            .project(Map.of("path", ":distribution:bwc:maintenance", "configuration", "checkout"));
-        project.getDependencies().add(bwcMinorConfig.getName(), bwcMinor);
-
-        String projectPath = project.getPath();
-        ExtraPropertiesExtension extraProperties = project.getExtensions().getExtraProperties();
-        Provider<CopyRestApiTask> copyCompatYamlSpecTask = project.getTasks()
-            .register("copyRestCompatApiTask", CopyRestApiTask.class, task -> {
-                task.dependsOn(bwcMinorConfig);
-                task.setConfig(bwcMinorConfig);
-                task.setAdditionalConfig(bwcMinorConfig);
-                task.getInclude().set(extension.getRestApi().getInclude());
-                task.getOutputResourceDir().set(projectLayout.getBuildDirectory().dir(compatSpecsDir.toString()));
-                task.setSourceResourceDir(
-                    yamlCompatTestSourceSet.getResources()
-                        .getSrcDirs()
-                        .stream()
-                        .filter(f -> f.isDirectory() && f.getName().equals("resources"))
-                        .findFirst()
-                        .orElse(null)
-                );
-                task.setSkipHasRestTestCheck(true);
-                task.setConfigToFileTree(
-                    config -> fileOperations.fileTree(
-                        config.getSingleFile().toPath().resolve(RELATIVE_REST_API_RESOURCES).resolve(RELATIVE_API_PATH)
-                    )
-                );
-                task.setAdditionalConfigToFileTree(
-                    config -> fileOperations.fileTree(
-                        getCompatProjectPath(projectPath, config.getSingleFile().toPath()).resolve(RELATIVE_REST_PROJECT_RESOURCES)
-                            .resolve(RELATIVE_API_PATH)
-                    )
-                );
-                task.onlyIf(t -> isEnabled(extraProperties));
-            });
-
-        // copy compatible rest tests
-        Provider<CopyRestTestsTask> copyCompatYamlTestTask = project.getTasks()
-            .register("copyRestCompatTestTask", CopyRestTestsTask.class, task -> {
-                task.dependsOn(bwcMinorConfig);
-                task.setCoreConfig(bwcMinorConfig);
-                task.setXpackConfig(bwcMinorConfig);
-                task.setAdditionalConfig(bwcMinorConfig);
-                task.getIncludeCore().set(extension.getRestTests().getIncludeCore());
-                task.getIncludeXpack().set(extension.getRestTests().getIncludeXpack());
-                task.getOutputResourceDir().set(projectLayout.getBuildDirectory().dir(compatTestsDir.resolve("original").toString()));
-                task.setCoreConfigToFileTree(
-                    config -> fileOperations.fileTree(
-                        config.getSingleFile()
-                            .toPath()
-                            .resolve(RELATIVE_REST_CORE)
-                            .resolve(RELATIVE_REST_PROJECT_RESOURCES)
-                            .resolve(RELATIVE_TEST_PATH)
-                    )
-                );
-                task.setXpackConfigToFileTree(
-                    config -> fileOperations.fileTree(
-                        config.getSingleFile()
-                            .toPath()
-                            .resolve(RELATIVE_REST_XPACK)
-                            .resolve(RELATIVE_REST_PROJECT_RESOURCES)
-                            .resolve(RELATIVE_TEST_PATH)
-                    )
-                );
-                task.setAdditionalConfigToFileTree(
-                    config -> fileOperations.fileTree(
-                        getCompatProjectPath(projectPath, config.getSingleFile().toPath()).resolve(RELATIVE_REST_PROJECT_RESOURCES)
-                            .resolve(RELATIVE_TEST_PATH)
-                    )
-                );
-                task.dependsOn(copyCompatYamlSpecTask);
-                task.onlyIf(t -> isEnabled(extraProperties));
-            });
-
-        // copy both local source set apis and compat apis to a single location to be exported as an artifact
-        TaskProvider<Sync> bundleRestCompatApis = project.getTasks().register("bundleRestCompatApis", Sync.class, task -> {
-            task.setDestinationDir(projectLayout.getBuildDirectory().dir("bundledCompatApis").get().getAsFile());
-            task.setIncludeEmptyDirs(false);
-            task.from(copyCompatYamlSpecTask.flatMap(t -> t.getOutputResourceDir().map(d -> d.dir(RELATIVE_API_PATH.toString()))));
-            task.from(yamlCompatTestSourceSet.getProcessResourcesTaskName(), s -> {
-                s.include(RELATIVE_API_PATH + "/*");
-                s.eachFile(
-                    details -> details.setRelativePath(
-                        new RelativePath(true, Arrays.stream(details.getRelativePath().getSegments()).skip(2).toArray(String[]::new))
-                    )
-                );
-            });
-        });
-
-        // transform the copied tests task
-        TaskProvider<RestCompatTestTransformTask> transformCompatTestTask = project.getTasks()
-            .register("yamlRestTestV" + COMPATIBLE_VERSION + "CompatTransform", RestCompatTestTransformTask.class, task -> {
-                task.getSourceDirectory().set(copyCompatYamlTestTask.flatMap(CopyRestTestsTask::getOutputResourceDir));
-                task.getOutputDirectory()
-                    .set(project.getLayout().getBuildDirectory().dir(compatTestsDir.resolve("transformed").toString()));
-                task.onlyIf(t -> isEnabled(extraProperties));
-            });
-
-        // Register compat rest resources with source set
-        yamlCompatTestSourceSet.getOutput().dir(copyCompatYamlSpecTask.map(CopyRestApiTask::getOutputResourceDir));
-        yamlCompatTestSourceSet.getOutput().dir(transformCompatTestTask.map(RestCompatTestTransformTask::getOutputDirectory));
-
-        // Register artifact for transformed compatibility apis and tests
-        Configuration compatRestSpecs = project.getConfigurations().create(COMPATIBILITY_APIS_CONFIGURATION);
-        Configuration compatRestTests = project.getConfigurations().create(COMPATIBILITY_TESTS_CONFIGURATION);
-        project.getArtifacts().add(compatRestSpecs.getName(), bundleRestCompatApis.map(Sync::getDestinationDir));
-        project.getArtifacts()
-            .add(
-                compatRestTests.getName(),
-                transformCompatTestTask.flatMap(t -> t.getOutputDirectory().dir(RELATIVE_TEST_PATH.toString()))
-            );
-
-        // Grab the original rest resources locations so we can omit them from the compatibility testing classpath down below
-        Provider<Directory> originalYamlSpecsDir = project.getTasks()
-            .withType(CopyRestApiTask.class)
-            .named(RestResourcesPlugin.COPY_REST_API_SPECS_TASK)
-            .flatMap(CopyRestApiTask::getOutputResourceDir);
-        Provider<Directory> originalYamlTestsDir = project.getTasks()
-            .withType(CopyRestTestsTask.class)
-            .named(RestResourcesPlugin.COPY_YAML_TESTS_TASK)
-            .flatMap(CopyRestTestsTask::getOutputResourceDir);
-
-        String testTaskName = "yamlRestTestV" + COMPATIBLE_VERSION + "CompatTest";
-
-        // setup the test task
-        Provider<RestIntegTestTask> yamlRestCompatTestTask = RestTestUtil.registerTestTask(project, yamlCompatTestSourceSet, testTaskName);
-        project.getTasks().withType(RestIntegTestTask.class).named(testTaskName).configure(testTask -> {
-            testTask.systemProperty("tests.restCompat", true);
-            // Use test runner and classpath from "normal" yaml source set
-            testTask.setTestClassesDirs(
-                yamlTestSourceSet.getOutput().getClassesDirs().plus(yamlCompatTestSourceSet.getOutput().getClassesDirs())
-            );
-            testTask.setClasspath(
-                yamlCompatTestSourceSet.getRuntimeClasspath()
-                    // remove the "normal" api and tests
-                    .minus(project.files(yamlTestSourceSet.getOutput().getResourcesDir()))
-                    .minus(project.files(originalYamlSpecsDir))
-                    .minus(project.files(originalYamlTestsDir))
-            );
-
-            // run compatibility tests after "normal" tests
-            testTask.mustRunAfter(project.getTasks().named(InternalYamlRestTestPlugin.SOURCE_SET_NAME));
-            testTask.onlyIf(t -> isEnabled(extraProperties));
-        });
-
-        setupYamlRestTestDependenciesDefaults(project, yamlCompatTestSourceSet);
-
-        // setup IDE
-        GradleUtils.setupIdeForTestSourceSet(project, yamlCompatTestSourceSet);
-
-        // add a lifecycle task to allow for a possible future additional rest compatibility without needing to change task names
-        TaskProvider<Task> checkRestCompatTask = project.getTasks().register(REST_COMPAT_CHECK_TASK_NAME, (thisCheckTask) -> {
-            thisCheckTask.setDescription("Runs all REST compatibility checks.");
-            thisCheckTask.setGroup("verification");
-        });
-
-        // wire the lifecycle task into the main check task
-        project.getTasks().named(JavaBasePlugin.CHECK_TASK_NAME).configure(check -> check.dependsOn(checkRestCompatTask));
-
-        // wire the yamlRestCompatTest into the custom lifecycle task
-        project.getTasks().named(REST_COMPAT_CHECK_TASK_NAME).configure(check -> check.dependsOn(yamlRestCompatTestTask));
-
-    }
-
-    private boolean isEnabled(ExtraPropertiesExtension extraProperties) {
-        Object bwcEnabled = extraProperties.getProperties().get("bwc_tests_enabled");
-        return bwcEnabled == null || (Boolean) bwcEnabled;
-    }
-
-    // TODO: implement custom extension that allows us move around of the projects between major versions and still find them
-    private Path getCompatProjectPath(String projectPath, Path checkoutDir) {
-        return checkoutDir.resolve(projectPath.replaceFirst(":", "").replace(":", File.separator));
-    }
-}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/ErrorReportingTestListener.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/ErrorReportingTestListener.java
index d00301e96fab..6659218593f9 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/ErrorReportingTestListener.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/ErrorReportingTestListener.java
@@ -10,12 +10,12 @@
 import org.gradle.api.internal.tasks.testing.logging.FullExceptionFormatter;
 import org.gradle.api.internal.tasks.testing.logging.TestExceptionFormatter;
 import org.gradle.api.logging.Logger;
+import org.gradle.api.tasks.testing.Test;
 import org.gradle.api.tasks.testing.TestDescriptor;
 import org.gradle.api.tasks.testing.TestListener;
 import org.gradle.api.tasks.testing.TestOutputEvent;
 import org.gradle.api.tasks.testing.TestOutputListener;
 import org.gradle.api.tasks.testing.TestResult;
-import org.gradle.api.tasks.testing.logging.TestLogging;
 
 import java.io.BufferedOutputStream;
 import java.io.BufferedReader;
@@ -38,6 +38,7 @@
 public class ErrorReportingTestListener implements TestOutputListener, TestListener {
     private static final String REPRODUCE_WITH_PREFIX = "REPRODUCE WITH";
 
+    private final Test testTask;
     private final TestExceptionFormatter formatter;
     private final File outputDirectory;
     private final Logger taskLogger;
@@ -45,9 +46,10 @@ public class ErrorReportingTestListener implements TestOutputListener, TestListe
     private Map<Descriptor, Deque<String>> reproductionLines = new ConcurrentHashMap<>();
     private Set<Descriptor> failedTests = new LinkedHashSet<>();
 
-    public ErrorReportingTestListener(TestLogging testLogging, Logger taskLogger, File outputDirectory) {
-        this.formatter = new FullExceptionFormatter(testLogging);
-        this.taskLogger = taskLogger;
+    public ErrorReportingTestListener(Test testTask, File outputDirectory) {
+        this.testTask = testTask;
+        this.formatter = new FullExceptionFormatter(testTask.getTestLogging());
+        this.taskLogger = testTask.getLogger();
         this.outputDirectory = outputDirectory;
     }
 
@@ -80,34 +82,37 @@ public void afterSuite(final TestDescriptor suite, TestResult result) {
         Descriptor descriptor = Descriptor.of(suite);
 
         try {
-            // if the test suite failed, report all captured output
-            if (result.getResultType().equals(TestResult.ResultType.FAILURE)) {
-                EventWriter eventWriter = eventWriters.get(descriptor);
-
-                if (eventWriter != null) {
-                    // It's not explicit what the threading guarantees are for TestListener method execution so we'll
-                    // be explicitly safe here to avoid interleaving output from multiple test suites
-                    synchronized (this) {
-                        // make sure we've flushed everything to disk before reading
-                        eventWriter.flush();
-
-                        System.err.println("\n\nSuite: " + suite);
-
-                        try (BufferedReader reader = eventWriter.reader()) {
-                            PrintStream out = System.out;
-                            for (String message = reader.readLine(); message != null; message = reader.readLine()) {
-                                if (message.startsWith("  1> ")) {
-                                    out = System.out;
-                                } else if (message.startsWith("  2> ")) {
-                                    out = System.err;
+            if (isDumpOutputEnabled()) {
+                // if the test suite failed, report all captured output
+                if (result.getResultType().equals(TestResult.ResultType.FAILURE)) {
+                    EventWriter eventWriter = eventWriters.get(descriptor);
+
+                    if (eventWriter != null) {
+                        // It's not explicit what the threading guarantees are for TestListener method execution so we'll
+                        // be explicitly safe here to avoid interleaving output from multiple test suites
+                        synchronized (this) {
+                            // make sure we've flushed everything to disk before reading
+                            eventWriter.flush();
+
+                            System.err.println("\n\nSuite: " + suite);
+
+                            try (BufferedReader reader = eventWriter.reader()) {
+                                PrintStream out = System.out;
+                                for (String message = reader.readLine(); message != null; message = reader.readLine()) {
+                                    if (message.startsWith("  1> ")) {
+                                        out = System.out;
+                                    } else if (message.startsWith("  2> ")) {
+                                        out = System.err;
+                                    }
+
+                                    out.println(message);
                                 }
-
-                                out.println(message);
                             }
                         }
                     }
                 }
             }
+
             if (suite.getParent() == null) {
                 // per test task top level gradle test run suite finished
                 if (getFailedTests().size() > 0) {
@@ -250,4 +255,9 @@ public void close() throws IOException {
             outputFile.delete();
         }
     }
+
+    private boolean isDumpOutputEnabled() {
+        Object errorReportingEnabled = testTask.getExtensions().getExtraProperties().get("dumpOutputOnFailure");
+        return errorReportingEnabled == null || (boolean) errorReportingEnabled;
+    }
 }
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/LegacyRestTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/LegacyRestTestBasePlugin.java
new file mode 100644
index 000000000000..f27020511145
--- /dev/null
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/LegacyRestTestBasePlugin.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test;
+
+import org.elasticsearch.gradle.internal.ElasticsearchJavaBasePlugin;
+import org.elasticsearch.gradle.internal.ElasticsearchTestBasePlugin;
+import org.elasticsearch.gradle.internal.FixtureStop;
+import org.elasticsearch.gradle.internal.InternalTestClustersPlugin;
+import org.elasticsearch.gradle.internal.precommit.InternalPrecommitTasks;
+import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider;
+import org.elasticsearch.gradle.testclusters.ElasticsearchCluster;
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask;
+import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
+import org.elasticsearch.gradle.util.GradleUtils;
+import org.gradle.api.NamedDomainObjectContainer;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.plugins.JavaBasePlugin;
+import org.gradle.api.provider.ProviderFactory;
+import org.gradle.api.tasks.Sync;
+import org.gradle.api.tasks.bundling.Zip;
+
+import javax.inject.Inject;
+
+import static org.elasticsearch.gradle.plugin.BasePluginBuildPlugin.BUNDLE_PLUGIN_TASK_NAME;
+import static org.elasticsearch.gradle.plugin.BasePluginBuildPlugin.EXPLODED_BUNDLE_PLUGIN_TASK_NAME;
+
+/**
+ * @deprecated use {@link RestTestBasePlugin} instead
+ */
+@Deprecated
+public class LegacyRestTestBasePlugin implements Plugin<Project> {
+    private static final String TESTS_REST_CLUSTER = "tests.rest.cluster";
+    private static final String TESTS_CLUSTER = "tests.cluster";
+    private static final String TESTS_CLUSTER_NAME = "tests.clustername";
+    private static final String TESTS_CLUSTER_READINESS = "tests.cluster.readiness";
+
+    private static final String TESTS_CLUSTER_REMOTE_ACCESS = "tests.cluster.remote_access";
+
+    private ProviderFactory providerFactory;
+
+    @Inject
+    public LegacyRestTestBasePlugin(ProviderFactory providerFactory) {
+        this.providerFactory = providerFactory;
+    }
+
+    @Override
+    public void apply(Project project) {
+        project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class);
+        project.getPluginManager().apply(ElasticsearchTestBasePlugin.class);
+        project.getPluginManager().apply(InternalTestClustersPlugin.class);
+        InternalPrecommitTasks.create(project, false);
+        project.getTasks().withType(RestIntegTestTask.class).configureEach(restIntegTestTask -> {
+            @SuppressWarnings("unchecked")
+            NamedDomainObjectContainer<ElasticsearchCluster> testClusters = (NamedDomainObjectContainer<ElasticsearchCluster>) project
+                .getExtensions()
+                .getByName(TestClustersPlugin.EXTENSION_NAME);
+            ElasticsearchCluster cluster = testClusters.maybeCreate(restIntegTestTask.getName());
+            restIntegTestTask.useCluster(cluster);
+            restIntegTestTask.include("**/*IT.class");
+            restIntegTestTask.systemProperty("tests.rest.load_packaged", Boolean.FALSE.toString());
+            if (systemProperty(TESTS_REST_CLUSTER) == null) {
+                if (systemProperty(TESTS_CLUSTER) != null || systemProperty(TESTS_CLUSTER_NAME) != null) {
+                    throw new IllegalArgumentException(
+                        String.format("%s, %s, and %s must all be null or non-null", TESTS_REST_CLUSTER, TESTS_CLUSTER, TESTS_CLUSTER_NAME)
+                    );
+                }
+                SystemPropertyCommandLineArgumentProvider runnerNonInputProperties =
+                    (SystemPropertyCommandLineArgumentProvider) restIntegTestTask.getExtensions().getByName("nonInputProperties");
+                runnerNonInputProperties.systemProperty(TESTS_REST_CLUSTER, () -> String.join(",", cluster.getAllHttpSocketURI()));
+                runnerNonInputProperties.systemProperty(TESTS_CLUSTER, () -> String.join(",", cluster.getAllTransportPortURI()));
+                runnerNonInputProperties.systemProperty(TESTS_CLUSTER_NAME, cluster::getName);
+                runnerNonInputProperties.systemProperty(TESTS_CLUSTER_READINESS, () -> String.join(",", cluster.getAllReadinessPortURI()));
+                runnerNonInputProperties.systemProperty(
+                    TESTS_CLUSTER_REMOTE_ACCESS,
+                    () -> String.join(",", cluster.getAllRemoteAccessPortURI())
+                );
+            } else {
+                if (systemProperty(TESTS_CLUSTER) == null || systemProperty(TESTS_CLUSTER_NAME) == null) {
+                    throw new IllegalArgumentException(
+                        String.format("%s, %s, and %s must all be null or non-null", TESTS_REST_CLUSTER, TESTS_CLUSTER, TESTS_CLUSTER_NAME)
+                    );
+                }
+            }
+        });
+
+        project.getTasks()
+            .named(JavaBasePlugin.CHECK_TASK_NAME)
+            .configure(check -> check.dependsOn(project.getTasks().withType(RestIntegTestTask.class)));
+        project.getTasks()
+            .withType(StandaloneRestIntegTestTask.class)
+            .configureEach(t -> t.finalizedBy(project.getTasks().withType(FixtureStop.class)));
+
+        project.getTasks().withType(StandaloneRestIntegTestTask.class).configureEach(t -> {
+            t.setMaxParallelForks(1);
+            // if this a module or plugin, it may have an associated zip file with it's contents, add that to the test cluster
+            project.getPluginManager().withPlugin("elasticsearch.esplugin", plugin -> {
+                if (GradleUtils.isModuleProject(project.getPath())) {
+                    var bundle = project.getTasks().withType(Sync.class).named(EXPLODED_BUNDLE_PLUGIN_TASK_NAME);
+                    t.getClusters().forEach(c -> c.module(bundle));
+                } else {
+                    var bundle = project.getTasks().withType(Zip.class).named(BUNDLE_PLUGIN_TASK_NAME);
+                    t.getClusters().forEach(c -> c.plugin(bundle));
+                }
+            });
+        });
+    }
+
+    private String systemProperty(String propName) {
+        return providerFactory.systemProperty(propName).getOrNull();
+    }
+}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/RestTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/RestTestBasePlugin.java
deleted file mode 100644
index fc27bfa43798..000000000000
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/RestTestBasePlugin.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.gradle.internal.test;
-
-import org.elasticsearch.gradle.internal.ElasticsearchJavaBasePlugin;
-import org.elasticsearch.gradle.internal.ElasticsearchTestBasePlugin;
-import org.elasticsearch.gradle.internal.FixtureStop;
-import org.elasticsearch.gradle.internal.InternalTestClustersPlugin;
-import org.elasticsearch.gradle.internal.precommit.InternalPrecommitTasks;
-import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider;
-import org.elasticsearch.gradle.testclusters.ElasticsearchCluster;
-import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask;
-import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
-import org.elasticsearch.gradle.util.GradleUtils;
-import org.gradle.api.NamedDomainObjectContainer;
-import org.gradle.api.Plugin;
-import org.gradle.api.Project;
-import org.gradle.api.plugins.JavaBasePlugin;
-import org.gradle.api.provider.ProviderFactory;
-import org.gradle.api.tasks.Sync;
-import org.gradle.api.tasks.bundling.Zip;
-
-import javax.inject.Inject;
-
-import static org.elasticsearch.gradle.plugin.BasePluginBuildPlugin.BUNDLE_PLUGIN_TASK_NAME;
-import static org.elasticsearch.gradle.plugin.BasePluginBuildPlugin.EXPLODED_BUNDLE_PLUGIN_TASK_NAME;
-
-public class RestTestBasePlugin implements Plugin<Project> {
-    private static final String TESTS_REST_CLUSTER = "tests.rest.cluster";
-    private static final String TESTS_CLUSTER = "tests.cluster";
-    private static final String TESTS_CLUSTER_NAME = "tests.clustername";
-    private static final String TESTS_CLUSTER_READINESS = "tests.cluster.readiness";
-
-    private ProviderFactory providerFactory;
-
-    @Inject
-    public RestTestBasePlugin(ProviderFactory providerFactory) {
-        this.providerFactory = providerFactory;
-    }
-
-    @Override
-    public void apply(Project project) {
-        project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class);
-        project.getPluginManager().apply(ElasticsearchTestBasePlugin.class);
-        project.getPluginManager().apply(InternalTestClustersPlugin.class);
-        InternalPrecommitTasks.create(project, false);
-        project.getTasks().withType(RestIntegTestTask.class).configureEach(restIntegTestTask -> {
-            @SuppressWarnings("unchecked")
-            NamedDomainObjectContainer<ElasticsearchCluster> testClusters = (NamedDomainObjectContainer<ElasticsearchCluster>) project
-                .getExtensions()
-                .getByName(TestClustersPlugin.EXTENSION_NAME);
-            ElasticsearchCluster cluster = testClusters.maybeCreate(restIntegTestTask.getName());
-            restIntegTestTask.useCluster(cluster);
-            restIntegTestTask.include("**/*IT.class");
-            restIntegTestTask.systemProperty("tests.rest.load_packaged", Boolean.FALSE.toString());
-            if (systemProperty(TESTS_REST_CLUSTER) == null) {
-                if (systemProperty(TESTS_CLUSTER) != null || systemProperty(TESTS_CLUSTER_NAME) != null) {
-                    throw new IllegalArgumentException(
-                        String.format("%s, %s, and %s must all be null or non-null", TESTS_REST_CLUSTER, TESTS_CLUSTER, TESTS_CLUSTER_NAME)
-                    );
-                }
-                SystemPropertyCommandLineArgumentProvider runnerNonInputProperties =
-                    (SystemPropertyCommandLineArgumentProvider) restIntegTestTask.getExtensions().getByName("nonInputProperties");
-                runnerNonInputProperties.systemProperty(TESTS_REST_CLUSTER, () -> String.join(",", cluster.getAllHttpSocketURI()));
-                runnerNonInputProperties.systemProperty(TESTS_CLUSTER, () -> String.join(",", cluster.getAllTransportPortURI()));
-                runnerNonInputProperties.systemProperty(TESTS_CLUSTER_NAME, cluster::getName);
-                runnerNonInputProperties.systemProperty(TESTS_CLUSTER_READINESS, () -> String.join(",", cluster.getAllReadinessPortURI()));
-            } else {
-                if (systemProperty(TESTS_CLUSTER) == null || systemProperty(TESTS_CLUSTER_NAME) == null) {
-                    throw new IllegalArgumentException(
-                        String.format("%s, %s, and %s must all be null or non-null", TESTS_REST_CLUSTER, TESTS_CLUSTER, TESTS_CLUSTER_NAME)
-                    );
-                }
-            }
-        });
-
-        project.getTasks()
-            .named(JavaBasePlugin.CHECK_TASK_NAME)
-            .configure(check -> check.dependsOn(project.getTasks().withType(RestIntegTestTask.class)));
-        project.getTasks()
-            .withType(StandaloneRestIntegTestTask.class)
-            .configureEach(t -> t.finalizedBy(project.getTasks().withType(FixtureStop.class)));
-
-        project.getTasks().withType(StandaloneRestIntegTestTask.class).configureEach(t ->
-        // if this a module or plugin, it may have an associated zip file with it's contents, add that to the test cluster
-        project.getPluginManager().withPlugin("elasticsearch.esplugin", plugin -> {
-            if (GradleUtils.isModuleProject(project.getPath())) {
-                var bundle = project.getTasks().withType(Sync.class).named(EXPLODED_BUNDLE_PLUGIN_TASK_NAME);
-                t.getClusters().forEach(c -> c.module(bundle));
-            } else {
-                var bundle = project.getTasks().withType(Zip.class).named(BUNDLE_PLUGIN_TASK_NAME);
-                t.getClusters().forEach(c -> c.plugin(bundle));
-            }
-        }));
-    }
-
-    private String systemProperty(String propName) {
-        return providerFactory.systemProperty(propName).getOrNull();
-    }
-}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/StandaloneRestTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/StandaloneRestTestPlugin.java
index 9ffaf396e7fb..163df6cc40e1 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/StandaloneRestTestPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/StandaloneRestTestPlugin.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.gradle.internal.ExportElasticsearchBuildResourcesTask;
 import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin;
 import org.elasticsearch.gradle.internal.precommit.InternalPrecommitTasks;
-import org.elasticsearch.gradle.internal.test.rest.InternalJavaRestTestPlugin;
-import org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin;
+import org.elasticsearch.gradle.internal.test.rest.LegacyJavaRestTestPlugin;
+import org.elasticsearch.gradle.internal.test.rest.LegacyYamlRestTestPlugin;
 import org.elasticsearch.gradle.internal.test.rest.RestTestUtil;
 import org.gradle.api.InvalidUserDataException;
 import org.gradle.api.Plugin;
@@ -32,8 +32,8 @@
  * and run REST tests. Use BuildPlugin if you want to build main code as well
  * as tests.
  *
- * @deprecated use {@link InternalClusterTestPlugin}, {@link InternalJavaRestTestPlugin} or
- * {@link InternalYamlRestTestPlugin} instead.
+ * @deprecated use {@link InternalClusterTestPlugin}, {@link LegacyJavaRestTestPlugin} or
+ * {@link LegacyYamlRestTestPlugin} instead.
  */
 @Deprecated
 public class StandaloneRestTestPlugin implements Plugin<Project> {
@@ -46,7 +46,7 @@ public void apply(final Project project) {
         }
 
         project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
-        project.getPluginManager().apply(RestTestBasePlugin.class);
+        project.getPluginManager().apply(LegacyRestTestBasePlugin.class);
 
         project.getTasks().register("buildResources", ExportElasticsearchBuildResourcesTask.class);
 
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java
index 9eb2efc77835..524f3dfedf95 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.gradle.internal.precommit.FilePermissionsPrecommitPlugin;
 import org.elasticsearch.gradle.internal.precommit.ForbiddenPatternsPrecommitPlugin;
 import org.elasticsearch.gradle.internal.precommit.ForbiddenPatternsTask;
-import org.elasticsearch.gradle.internal.test.rest.InternalJavaRestTestPlugin;
+import org.elasticsearch.gradle.internal.test.rest.LegacyJavaRestTestPlugin;
 import org.elasticsearch.gradle.testclusters.ElasticsearchCluster;
 import org.elasticsearch.gradle.testclusters.TestClustersAware;
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
@@ -62,8 +62,8 @@ public void apply(Project project) {
                 .withType(RestIntegTestTask.class)
                 .configureEach(runner -> runner.systemProperty("tests.ssl.enabled", "true"));
         });
-        project.getPlugins().withType(InternalJavaRestTestPlugin.class).configureEach(restTestPlugin -> {
-            SourceSet testSourceSet = Util.getJavaSourceSets(project).getByName(InternalJavaRestTestPlugin.SOURCE_SET_NAME);
+        project.getPlugins().withType(LegacyJavaRestTestPlugin.class).configureEach(restTestPlugin -> {
+            SourceSet testSourceSet = Util.getJavaSourceSets(project).getByName(LegacyJavaRestTestPlugin.SOURCE_SET_NAME);
             testSourceSet.getResources().srcDir(new File(keyStoreDir, "test/ssl"));
             project.getTasks().named(testSourceSet.getProcessResourcesTaskName()).configure(t -> t.dependsOn(exportKeyStore));
             project.getTasks().withType(TestClustersAware.class).configureEach(clusterAware -> clusterAware.dependsOn(exportKeyStore));
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/InternalJavaRestTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/InternalJavaRestTestPlugin.java
index 9dda731f7211..d18505ca4b11 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/InternalJavaRestTestPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/InternalJavaRestTestPlugin.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.gradle.internal.test.rest;
 
-import org.elasticsearch.gradle.internal.test.RestTestBasePlugin;
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
@@ -33,8 +33,11 @@ public void apply(Project project) {
         SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
         SourceSet javaTestSourceSet = sourceSets.create(SOURCE_SET_NAME);
 
+        project.getDependencies().add(javaTestSourceSet.getImplementationConfigurationName(), project.project(":test:test-clusters"));
+
         // setup the javaRestTest task
-        registerTestTask(project, javaTestSourceSet);
+        // we use a StandloneRestIntegTestTask here so that the conventions of RestTestBasePlugin don't create a test cluster
+        registerTestTask(project, javaTestSourceSet, SOURCE_SET_NAME, StandaloneRestIntegTestTask.class);
 
         // setup dependencies
         setupJavaRestTestDependenciesDefaults(project, javaTestSourceSet);
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/InternalYamlRestTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/InternalYamlRestTestPlugin.java
index dacf119994f4..b0fd142705a0 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/InternalYamlRestTestPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/InternalYamlRestTestPlugin.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.gradle.internal.test.rest;
 
-import org.elasticsearch.gradle.internal.test.RestTestBasePlugin;
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
@@ -34,10 +34,10 @@ public void apply(Project project) {
         SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
         SourceSet yamlTestSourceSet = sourceSets.create(SOURCE_SET_NAME);
 
-        registerTestTask(project, yamlTestSourceSet);
+        registerTestTask(project, yamlTestSourceSet, SOURCE_SET_NAME, StandaloneRestIntegTestTask.class);
 
         // setup the dependencies
-        setupYamlRestTestDependenciesDefaults(project, yamlTestSourceSet);
+        setupYamlRestTestDependenciesDefaults(project, yamlTestSourceSet, true);
 
         // setup the copy for the rest resources
         project.getTasks().withType(CopyRestApiTask.class).configureEach(copyRestApiTask -> {
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/LegacyJavaRestTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/LegacyJavaRestTestPlugin.java
new file mode 100644
index 000000000000..a1b17c110b34
--- /dev/null
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/LegacyJavaRestTestPlugin.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test.rest;
+
+import org.elasticsearch.gradle.internal.test.LegacyRestTestBasePlugin;
+import org.elasticsearch.gradle.util.GradleUtils;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.tasks.SourceSet;
+import org.gradle.api.tasks.SourceSetContainer;
+
+import static org.elasticsearch.gradle.internal.test.rest.RestTestUtil.registerTestTask;
+import static org.elasticsearch.gradle.internal.test.rest.RestTestUtil.setupJavaRestTestDependenciesDefaults;
+
+/**
+ * Apply this plugin to run the Java based REST tests.
+ *
+ * @deprecated use {@link InternalJavaRestTestPlugin}
+ */
+@Deprecated
+public class LegacyJavaRestTestPlugin implements Plugin<Project> {
+
+    public static final String SOURCE_SET_NAME = "javaRestTest";
+
+    @Override
+    public void apply(Project project) {
+        project.getPluginManager().apply(LegacyRestTestBasePlugin.class);
+
+        // create source set
+        SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+        SourceSet javaTestSourceSet = sourceSets.create(SOURCE_SET_NAME);
+
+        // setup the javaRestTest task
+        registerTestTask(project, javaTestSourceSet);
+
+        // setup dependencies
+        setupJavaRestTestDependenciesDefaults(project, javaTestSourceSet);
+
+        // setup IDE
+        GradleUtils.setupIdeForTestSourceSet(project, javaTestSourceSet);
+    }
+}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPlugin.java
new file mode 100644
index 000000000000..4977c0924efb
--- /dev/null
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPlugin.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test.rest;
+
+import org.elasticsearch.gradle.internal.test.LegacyRestTestBasePlugin;
+import org.elasticsearch.gradle.util.GradleUtils;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.tasks.SourceSet;
+import org.gradle.api.tasks.SourceSetContainer;
+
+import static org.elasticsearch.gradle.internal.test.rest.RestTestUtil.registerTestTask;
+import static org.elasticsearch.gradle.internal.test.rest.RestTestUtil.setupYamlRestTestDependenciesDefaults;
+
+/**
+ * Apply this plugin to run the YAML based REST tests.
+ *
+ * @deprecated use {@link InternalYamlRestTestPlugin}
+ */
+@Deprecated
+public class LegacyYamlRestTestPlugin implements Plugin<Project> {
+
+    public static final String SOURCE_SET_NAME = "yamlRestTest";
+
+    @Override
+    public void apply(Project project) {
+        project.getPluginManager().apply(LegacyRestTestBasePlugin.class);
+        project.getPluginManager().apply(RestResourcesPlugin.class);
+
+        // create source set
+        SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+        SourceSet yamlTestSourceSet = sourceSets.create(SOURCE_SET_NAME);
+
+        registerTestTask(project, yamlTestSourceSet);
+
+        // setup the dependencies
+        setupYamlRestTestDependenciesDefaults(project, yamlTestSourceSet);
+
+        // setup the copy for the rest resources
+        project.getTasks().withType(CopyRestApiTask.class).configureEach(copyRestApiTask -> {
+            copyRestApiTask.setSourceResourceDir(
+                yamlTestSourceSet.getResources()
+                    .getSrcDirs()
+                    .stream()
+                    .filter(f -> f.isDirectory() && f.getName().equals("resources"))
+                    .findFirst()
+                    .orElse(null)
+            );
+        });
+
+        // Register rest resources with source set
+        yamlTestSourceSet.getOutput()
+            .dir(
+                project.getTasks()
+                    .withType(CopyRestApiTask.class)
+                    .named(RestResourcesPlugin.COPY_REST_API_SPECS_TASK)
+                    .flatMap(CopyRestApiTask::getOutputResourceDir)
+            );
+
+        yamlTestSourceSet.getOutput()
+            .dir(
+                project.getTasks()
+                    .withType(CopyRestTestsTask.class)
+                    .named(RestResourcesPlugin.COPY_YAML_TESTS_TASK)
+                    .flatMap(CopyRestTestsTask::getOutputResourceDir)
+            );
+
+        GradleUtils.setupIdeForTestSourceSet(project, yamlTestSourceSet);
+    }
+}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesExtension.java
index 8b08c23ac287..a107cfc51226 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesExtension.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesExtension.java
@@ -27,11 +27,11 @@ public RestResourcesExtension(ObjectFactory objects) {
         restTests = new XpackRestResourcesSpec(objects);
     }
 
-    void restApi(Action<? super RestResourcesSpec> spec) {
+    public void restApi(Action<? super RestResourcesSpec> spec) {
         spec.execute(restApi);
     }
 
-    void restTests(Action<? super XpackRestResourcesSpec> spec) {
+    public void restTests(Action<? super XpackRestResourcesSpec> spec) {
         spec.execute(restTests);
     }
 
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesPlugin.java
index e99ef646cdff..309480d9f600 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesPlugin.java
@@ -11,6 +11,7 @@
 import org.gradle.api.Project;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.attributes.Usage;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.SourceSetContainer;
@@ -76,6 +77,9 @@ public class RestResourcesPlugin implements Plugin<Project> {
 
     public static final String COPY_YAML_TESTS_TASK = "copyYamlTestsTask";
     public static final String COPY_REST_API_SPECS_TASK = "copyRestApiSpecsTask";
+    public static final String YAML_TESTS_USAGE = "yaml-tests";
+    public static final String YAML_XPACK_TESTS_USAGE = "yaml-xpack-tests";
+    public static final String YAML_SPEC_USAGE = "yaml-spec";
     private static final String EXTENSION_NAME = "restResources";
 
     @Override
@@ -86,8 +90,14 @@ public void apply(Project project) {
         SourceSet defaultSourceSet = sourceSets.maybeCreate(TEST_SOURCE_SET_NAME);
 
         // tests
-        Configuration testConfig = project.getConfigurations().create("restTestConfig");
-        Configuration xpackTestConfig = project.getConfigurations().create("restXpackTestConfig");
+        Configuration testConfig = project.getConfigurations().create("restTestConfig", config -> {
+            config.setCanBeConsumed(false);
+            config.getAttributes().attribute(Usage.USAGE_ATTRIBUTE, project.getObjects().named(Usage.class, YAML_TESTS_USAGE));
+        });
+        Configuration xpackTestConfig = project.getConfigurations().create("restXpackTestConfig", config -> {
+            config.setCanBeConsumed(false);
+            config.getAttributes().attribute(Usage.USAGE_ATTRIBUTE, project.getObjects().named(Usage.class, YAML_XPACK_TESTS_USAGE));
+        });
         // core
         // we guard this reference to :rest-api-spec with a find to make testing easier
         var restApiSpecProjectAvailable = project.findProject(":rest-api-spec") != null;
@@ -104,8 +114,17 @@ public void apply(Project project) {
                 .project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackTests"));
             project.getDependencies().add(xpackTestConfig.getName(), restXPackTestdependency);
         }
-        project.getConfigurations().create("restTests");
-        project.getConfigurations().create("restXpackTests");
+        project.getConfigurations()
+            .create(
+                "restTests",
+                config -> config.getAttributes().attribute(Usage.USAGE_ATTRIBUTE, project.getObjects().named(Usage.class, YAML_TESTS_USAGE))
+            );
+        project.getConfigurations()
+            .create(
+                "restXpackTests",
+                config -> config.getAttributes()
+                    .attribute(Usage.USAGE_ATTRIBUTE, project.getObjects().named(Usage.class, YAML_XPACK_TESTS_USAGE))
+            );
 
         Provider<CopyRestTestsTask> copyRestYamlTestTask = project.getTasks()
             .register(COPY_YAML_TESTS_TASK, CopyRestTestsTask.class, task -> {
@@ -121,13 +140,24 @@ public void apply(Project project) {
             });
 
         // api
-        Configuration specConfig = project.getConfigurations().create("restSpec"); // name chosen for passivity
+        Configuration specConfig = project.getConfigurations()
+            .create(
+                "restSpec", // name chosen for passivity
+                config -> {
+                    config.setCanBeConsumed(false);
+                    config.getAttributes().attribute(Usage.USAGE_ATTRIBUTE, project.getObjects().named(Usage.class, YAML_SPEC_USAGE));
+                }
+            );
         if (restApiSpecProjectAvailable) {
             Dependency restSpecDependency = project.getDependencies()
                 .project(Map.of("path", ":rest-api-spec", "configuration", "restSpecs"));
             project.getDependencies().add(specConfig.getName(), restSpecDependency);
         }
-        project.getConfigurations().create("restSpecs");
+        project.getConfigurations()
+            .create(
+                "restSpecs",
+                config -> config.getAttributes().attribute(Usage.USAGE_ATTRIBUTE, project.getObjects().named(Usage.class, YAML_SPEC_USAGE))
+            );
 
         Provider<CopyRestApiTask> copyRestYamlApiTask = project.getTasks()
             .register(COPY_REST_API_SPECS_TASK, CopyRestApiTask.class, task -> {
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java
new file mode 100644
index 000000000000..1a7b5bc3ee2a
--- /dev/null
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java
@@ -0,0 +1,319 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test.rest;
+
+import groovy.lang.Closure;
+
+import org.elasticsearch.gradle.Architecture;
+import org.elasticsearch.gradle.DistributionDownloadPlugin;
+import org.elasticsearch.gradle.ElasticsearchDistribution;
+import org.elasticsearch.gradle.ElasticsearchDistributionType;
+import org.elasticsearch.gradle.Version;
+import org.elasticsearch.gradle.VersionProperties;
+import org.elasticsearch.gradle.distribution.ElasticsearchDistributionTypes;
+import org.elasticsearch.gradle.internal.ElasticsearchJavaPlugin;
+import org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin;
+import org.elasticsearch.gradle.internal.info.BuildParams;
+import org.elasticsearch.gradle.plugin.BasePluginBuildPlugin;
+import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
+import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
+import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider;
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask;
+import org.elasticsearch.gradle.transform.UnzipTransform;
+import org.elasticsearch.gradle.util.GradleUtils;
+import org.gradle.api.Action;
+import org.gradle.api.NamedDomainObjectContainer;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.Task;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.artifacts.ProjectDependency;
+import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
+import org.gradle.api.file.FileTree;
+import org.gradle.api.plugins.JavaBasePlugin;
+import org.gradle.api.provider.ProviderFactory;
+import org.gradle.api.tasks.ClasspathNormalizer;
+import org.gradle.api.tasks.PathSensitivity;
+import org.gradle.api.tasks.util.PatternFilterable;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import javax.inject.Inject;
+
+/**
+ * Base plugin used for wiring up build tasks to REST testing tasks using new JUnit rule-based test clusters framework.
+ */
+public class RestTestBasePlugin implements Plugin<Project> {
+
+    private static final String TESTS_RUNTIME_JAVA_SYSPROP = "tests.runtime.java";
+    private static final String DEFAULT_DISTRIBUTION_SYSPROP = "tests.default.distribution";
+    private static final String INTEG_TEST_DISTRIBUTION_SYSPROP = "tests.integ-test.distribution";
+    private static final String BWC_SNAPSHOT_DISTRIBUTION_SYSPROP_PREFIX = "tests.snapshot.distribution.";
+    private static final String BWC_RELEASED_DISTRIBUTION_SYSPROP_PREFIX = "tests.release.distribution.";
+    private static final String TESTS_CLUSTER_MODULES_PATH_SYSPROP = "tests.cluster.modules.path";
+    private static final String TESTS_CLUSTER_PLUGINS_PATH_SYSPROP = "tests.cluster.plugins.path";
+    private static final String DEFAULT_REST_INTEG_TEST_DISTRO = "default_distro";
+    private static final String INTEG_TEST_REST_INTEG_TEST_DISTRO = "integ_test_distro";
+    private static final String MODULES_CONFIGURATION = "clusterModules";
+    private static final String PLUGINS_CONFIGURATION = "clusterPlugins";
+    private static final String EXTRACTED_PLUGINS_CONFIGURATION = "extractedPlugins";
+
+    private final ProviderFactory providerFactory;
+
+    @Inject
+    public RestTestBasePlugin(ProviderFactory providerFactory) {
+        this.providerFactory = providerFactory;
+    }
+
+    @Override
+    public void apply(Project project) {
+        project.getPluginManager().apply(ElasticsearchJavaPlugin.class);
+        project.getPluginManager().apply(InternalDistributionDownloadPlugin.class);
+
+        // Register integ-test and default distributions
+        ElasticsearchDistribution defaultDistro = createDistribution(
+            project,
+            DEFAULT_REST_INTEG_TEST_DISTRO,
+            VersionProperties.getElasticsearch()
+        );
+        ElasticsearchDistribution integTestDistro = createDistribution(
+            project,
+            INTEG_TEST_REST_INTEG_TEST_DISTRO,
+            VersionProperties.getElasticsearch(),
+            ElasticsearchDistributionTypes.INTEG_TEST_ZIP
+        );
+
+        // Create configures for module and plugin dependencies
+        Configuration modulesConfiguration = createPluginConfiguration(project, MODULES_CONFIGURATION, true, false);
+        Configuration pluginsConfiguration = createPluginConfiguration(project, PLUGINS_CONFIGURATION, false, false);
+        Configuration extractedPluginsConfiguration = createPluginConfiguration(project, EXTRACTED_PLUGINS_CONFIGURATION, true, true);
+        extractedPluginsConfiguration.extendsFrom(pluginsConfiguration);
+        configureArtifactTransforms(project);
+
+        // For plugin and module projects, register the current project plugin bundle as a dependency
+        project.getPluginManager().withPlugin("elasticsearch.esplugin", plugin -> {
+            if (GradleUtils.isModuleProject(project.getPath())) {
+                project.getDependencies().add(MODULES_CONFIGURATION, getExplodedBundleDependency(project, project.getPath()));
+            } else {
+                project.getDependencies().add(PLUGINS_CONFIGURATION, getBundleZipTaskDependency(project, project.getPath()));
+            }
+
+        });
+
+        project.getTasks().withType(StandaloneRestIntegTestTask.class).configureEach(task -> {
+            SystemPropertyCommandLineArgumentProvider nonInputSystemProperties = task.getExtensions()
+                .getByType(SystemPropertyCommandLineArgumentProvider.class);
+
+            task.dependsOn(integTestDistro, modulesConfiguration);
+            registerDistributionInputs(task, integTestDistro);
+
+            // Enable parallel execution for these tests since each test gets its own cluster
+            task.setMaxParallelForks(task.getProject().getGradle().getStartParameter().getMaxWorkerCount() / 2);
+
+            // Disable test failure reporting since this stuff is now captured in build scans
+            task.getExtensions().getExtraProperties().set("dumpOutputOnFailure", false);
+
+            // Disable the security manager and syscall filter since the test framework needs to fork processes
+            task.systemProperty("tests.security.manager", "false");
+            task.systemProperty("tests.system_call_filter", "false");
+
+            // Register plugins and modules as task inputs and pass paths as system properties to tests
+            nonInputSystemProperties.systemProperty(TESTS_CLUSTER_MODULES_PATH_SYSPROP, modulesConfiguration::getAsPath);
+            registerConfigurationInputs(task, modulesConfiguration);
+            nonInputSystemProperties.systemProperty(TESTS_CLUSTER_PLUGINS_PATH_SYSPROP, pluginsConfiguration::getAsPath);
+            registerConfigurationInputs(task, extractedPluginsConfiguration);
+
+            // Wire up integ-test distribution by default for all test tasks
+            nonInputSystemProperties.systemProperty(
+                INTEG_TEST_DISTRIBUTION_SYSPROP,
+                () -> integTestDistro.getExtracted().getSingleFile().getPath()
+            );
+            nonInputSystemProperties.systemProperty(TESTS_RUNTIME_JAVA_SYSPROP, BuildParams.getRuntimeJavaHome());
+
+            // Add `usesDefaultDistribution()` extension method to test tasks to indicate they require the default distro
+            task.getExtensions().getExtraProperties().set("usesDefaultDistribution", new Closure<Void>(task) {
+                @Override
+                public Void call(Object... args) {
+                    task.dependsOn(defaultDistro);
+                    registerDistributionInputs(task, defaultDistro);
+
+                    nonInputSystemProperties.systemProperty(
+                        DEFAULT_DISTRIBUTION_SYSPROP,
+                        providerFactory.provider(() -> defaultDistro.getExtracted().getSingleFile().getPath())
+                    );
+                    return null;
+                }
+            });
+
+            // Add `usesBwcDistribution(version)` extension method to test tasks to indicate they require a BWC distribution
+            task.getExtensions().getExtraProperties().set("usesBwcDistribution", new Closure<Void>(task) {
+                @Override
+                public Void call(Object... args) {
+                    if (args.length != 1 && args[0] instanceof Version == false) {
+                        throw new IllegalArgumentException("Expected exactly one argument of type org.elasticsearch.gradle.Version");
+                    }
+
+                    Version version = (Version) args[0];
+                    boolean isReleased = BuildParams.getBwcVersions().unreleasedInfo(version) == null;
+                    String versionString = version.toString();
+                    ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + versionString, versionString);
+
+                    task.dependsOn(bwcDistro);
+                    registerDistributionInputs(task, bwcDistro);
+
+                    nonInputSystemProperties.systemProperty(
+                        (isReleased ? BWC_RELEASED_DISTRIBUTION_SYSPROP_PREFIX : BWC_SNAPSHOT_DISTRIBUTION_SYSPROP_PREFIX) + versionString,
+                        providerFactory.provider(() -> bwcDistro.getExtracted().getSingleFile().getPath())
+                    );
+
+                    if (version.before(BuildParams.getBwcVersions().getMinimumWireCompatibleVersion())) {
+                        // If we are upgrade testing older versions we also need to upgrade to 7.last
+                        this.call(BuildParams.getBwcVersions().getMinimumWireCompatibleVersion());
+                    }
+                    return null;
+                }
+            });
+        });
+
+        project.getTasks()
+            .named(JavaBasePlugin.CHECK_TASK_NAME)
+            .configure(check -> check.dependsOn(project.getTasks().withType(StandaloneRestIntegTestTask.class)));
+    }
+
+    private ElasticsearchDistribution createDistribution(Project project, String name, String version) {
+        return createDistribution(project, name, version, null);
+    }
+
+    private ElasticsearchDistribution createDistribution(Project project, String name, String version, ElasticsearchDistributionType type) {
+        NamedDomainObjectContainer<ElasticsearchDistribution> distributions = DistributionDownloadPlugin.getContainer(project);
+        ElasticsearchDistribution maybeDistro = distributions.findByName(name);
+        if (maybeDistro == null) {
+            return distributions.create(name, distro -> {
+                distro.setVersion(version);
+                distro.setArchitecture(Architecture.current());
+                if (type != null) {
+                    distro.setType(type);
+                }
+            });
+        } else {
+            return maybeDistro;
+        }
+    }
+
+    private FileTree getDistributionFiles(ElasticsearchDistribution distribution, Action<PatternFilterable> patternFilter) {
+        return distribution.getExtracted().getAsFileTree().matching(patternFilter);
+    }
+
+    private void registerConfigurationInputs(Task task, Configuration configuration) {
+        task.getInputs()
+            .files(providerFactory.provider(() -> configuration.getAsFileTree().filter(f -> f.getName().endsWith(".jar") == false)))
+            .withPropertyName(configuration.getName() + "-files")
+            .withPathSensitivity(PathSensitivity.RELATIVE);
+
+        task.getInputs()
+            .files(providerFactory.provider(() -> configuration.getAsFileTree().filter(f -> f.getName().endsWith(".jar"))))
+            .withPropertyName(configuration.getName() + "-classpath")
+            .withNormalizer(ClasspathNormalizer.class);
+    }
+
+    private void registerDistributionInputs(Task task, ElasticsearchDistribution distribution) {
+        task.getInputs()
+            .files(providerFactory.provider(() -> getDistributionFiles(distribution, filter -> filter.exclude("**/*.jar"))))
+            .withPropertyName(distribution.getName() + "-files")
+            .withPathSensitivity(PathSensitivity.RELATIVE);
+
+        task.getInputs()
+            .files(providerFactory.provider(() -> getDistributionFiles(distribution, filter -> filter.include("**/*.jar"))))
+            .withPropertyName(distribution.getName() + "-classpath")
+            .withNormalizer(ClasspathNormalizer.class);
+    }
+
+    private Optional<String> findModulePath(Project project, String pluginName) {
+        return project.getRootProject()
+            .getAllprojects()
+            .stream()
+            .filter(p -> GradleUtils.isModuleProject(p.getPath()))
+            .filter(p -> p.getPlugins().hasPlugin(PluginBuildPlugin.class))
+            .filter(p -> p.getExtensions().getByType(PluginPropertiesExtension.class).getName().equals(pluginName))
+            .findFirst()
+            .map(Project::getPath);
+    }
+
+    private Configuration createPluginConfiguration(Project project, String name, boolean useExploded, boolean isExtended) {
+        return project.getConfigurations().create(name, c -> {
+            if (useExploded) {
+                c.attributes(a -> a.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE));
+            } else {
+                c.attributes(a -> a.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.ZIP_TYPE));
+            }
+            if (isExtended == false) {
+                c.withDependencies(dependencies -> {
+                    // Add dependencies of any modules
+                    Collection<Dependency> additionalDependencies = new LinkedHashSet<>();
+                    for (Iterator<Dependency> iterator = dependencies.iterator(); iterator.hasNext();) {
+                        Dependency dependency = iterator.next();
+                        if (dependency instanceof ProjectDependency projectDependency) {
+                            Project dependencyProject = projectDependency.getDependencyProject();
+                            List<String> extendedPlugins = dependencyProject.getExtensions()
+                                .getByType(PluginPropertiesExtension.class)
+                                .getExtendedPlugins();
+
+                            // Replace project dependency with explicit dependency on exploded configuration to workaround variant bug
+                            if (projectDependency.getTargetConfiguration() == null) {
+                                iterator.remove();
+                                additionalDependencies.add(
+                                    useExploded
+                                        ? getExplodedBundleDependency(project, dependencyProject.getPath())
+                                        : getBundleZipTaskDependency(project, dependencyProject.getPath())
+                                );
+                            }
+
+                            for (String extendedPlugin : extendedPlugins) {
+                                findModulePath(project, extendedPlugin).ifPresent(
+                                    modulePath -> additionalDependencies.add(
+                                        useExploded
+                                            ? getExplodedBundleDependency(project, modulePath)
+                                            : getBundleZipTaskDependency(project, modulePath)
+                                    )
+                                );
+                            }
+                        }
+                    }
+
+                    dependencies.addAll(additionalDependencies);
+                });
+            }
+        });
+    }
+
+    private Dependency getExplodedBundleDependency(Project project, String projectPath) {
+        return project.getDependencies()
+            .project(Map.of("path", projectPath, "configuration", BasePluginBuildPlugin.EXPLODED_BUNDLE_CONFIG));
+    }
+
+    private Dependency getBundleZipTaskDependency(Project project, String projectPath) {
+        Project dependencyProject = project.findProject(projectPath);
+        return project.getDependencies()
+            .create(project.files(dependencyProject.getTasks().named(BasePluginBuildPlugin.BUNDLE_PLUGIN_TASK_NAME)));
+    }
+
+    private void configureArtifactTransforms(Project project) {
+        project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> {
+            transformSpec.getFrom().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.ZIP_TYPE);
+            transformSpec.getTo().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE);
+            transformSpec.getParameters().setAsFiletreeOutput(false);
+        });
+    }
+}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestUtil.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestUtil.java
index 922157333d80..99c25b9e2570 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestUtil.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestUtil.java
@@ -15,6 +15,7 @@
 import org.gradle.api.provider.Provider;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.TaskProvider;
+import org.gradle.api.tasks.testing.Test;
 
 /**
  * Utility class to configure the necessary tasks and dependencies.
@@ -34,8 +35,17 @@ public static Provider<RestIntegTestTask> registerTestTask(Project project, Sour
      * Creates a {@link RestIntegTestTask} task with a custom name for the provided source set
      */
     public static TaskProvider<RestIntegTestTask> registerTestTask(Project project, SourceSet sourceSet, String taskName) {
+        return registerTestTask(project, sourceSet, taskName, RestIntegTestTask.class);
+    }
+
+    /**
+     * Creates a {@link T} task with a custom name for the provided source set
+     *
+     * @param <T> test task type
+     */
+    public static <T extends Test> TaskProvider<T> registerTestTask(Project project, SourceSet sourceSet, String taskName, Class<T> clazz) {
         // lazily create the test task
-        return project.getTasks().register(taskName, RestIntegTestTask.class, testTask -> {
+        return project.getTasks().register(taskName, clazz, testTask -> {
             testTask.setGroup(JavaBasePlugin.VERIFICATION_GROUP);
             testTask.setDescription("Runs the REST tests against an external cluster");
             project.getPlugins().withType(JavaPlugin.class, t -> testTask.mustRunAfter(project.getTasks().named("test")));
@@ -49,10 +59,20 @@ public static TaskProvider<RestIntegTestTask> registerTestTask(Project project,
      * Setup the dependencies needed for the YAML REST tests.
      */
     public static void setupYamlRestTestDependenciesDefaults(Project project, SourceSet sourceSet) {
+        setupYamlRestTestDependenciesDefaults(project, sourceSet, false);
+    }
+
+    /**
+     * Setup the dependencies needed for the YAML REST tests.
+     */
+    public static void setupYamlRestTestDependenciesDefaults(Project project, SourceSet sourceSet, boolean useNewTestClusters) {
         Project yamlTestRunnerProject = project.findProject(":test:yaml-rest-runner");
         // we shield the project dependency to make integration tests easier
         if (yamlTestRunnerProject != null) {
             project.getDependencies().add(sourceSet.getImplementationConfigurationName(), yamlTestRunnerProject);
+            if (useNewTestClusters) {
+                project.getDependencies().add(sourceSet.getImplementationConfigurationName(), project.project(":test:test-clusters"));
+            }
         }
     }
 
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java
new file mode 100644
index 000000000000..273a0a379318
--- /dev/null
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java
@@ -0,0 +1,272 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test.rest.compat.compat;
+
+import org.elasticsearch.gradle.Version;
+import org.elasticsearch.gradle.VersionProperties;
+import org.elasticsearch.gradle.internal.ElasticsearchJavaBasePlugin;
+import org.elasticsearch.gradle.internal.test.LegacyRestTestBasePlugin;
+import org.elasticsearch.gradle.internal.test.rest.CopyRestApiTask;
+import org.elasticsearch.gradle.internal.test.rest.CopyRestTestsTask;
+import org.elasticsearch.gradle.internal.test.rest.LegacyYamlRestTestPlugin;
+import org.elasticsearch.gradle.internal.test.rest.RestResourcesExtension;
+import org.elasticsearch.gradle.internal.test.rest.RestResourcesPlugin;
+import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
+import org.elasticsearch.gradle.util.GradleUtils;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.Task;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.file.Directory;
+import org.gradle.api.file.ProjectLayout;
+import org.gradle.api.file.RelativePath;
+import org.gradle.api.internal.file.FileOperations;
+import org.gradle.api.plugins.ExtraPropertiesExtension;
+import org.gradle.api.plugins.JavaBasePlugin;
+import org.gradle.api.provider.Provider;
+import org.gradle.api.tasks.SourceSet;
+import org.gradle.api.tasks.SourceSetContainer;
+import org.gradle.api.tasks.Sync;
+import org.gradle.api.tasks.TaskProvider;
+import org.gradle.api.tasks.testing.Test;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Map;
+
+import javax.inject.Inject;
+
+import static org.elasticsearch.gradle.internal.test.rest.RestTestUtil.setupYamlRestTestDependenciesDefaults;
+
+/**
+ * Apply this plugin to run the YAML based REST tests from a prior major version against this version's cluster.
+ */
+public abstract class AbstractYamlRestCompatTestPlugin implements Plugin<Project> {
+    public static final String BWC_MINOR_CONFIG_NAME = "bwcMinor";
+    private static final String REST_COMPAT_CHECK_TASK_NAME = "checkRestCompat";
+    private static final String COMPATIBILITY_APIS_CONFIGURATION = "restCompatSpecs";
+    private static final String COMPATIBILITY_TESTS_CONFIGURATION = "restCompatTests";
+    private static final Path RELATIVE_API_PATH = Path.of("rest-api-spec/api");
+    private static final Path RELATIVE_TEST_PATH = Path.of("rest-api-spec/test");
+    private static final Path RELATIVE_REST_API_RESOURCES = Path.of("rest-api-spec/src/main/resources");
+    private static final Path RELATIVE_REST_CORE = Path.of("rest-api-spec");
+    private static final Path RELATIVE_REST_XPACK = Path.of("x-pack/plugin");
+    private static final Path RELATIVE_REST_PROJECT_RESOURCES = Path.of("src/yamlRestTest/resources");
+    private static final int COMPATIBLE_VERSION = Version.fromString(VersionProperties.getVersions().get("elasticsearch")).getMajor() - 1;
+    private static final String SOURCE_SET_NAME = "yamlRestTestV" + COMPATIBLE_VERSION + "Compat";
+    private ProjectLayout projectLayout;
+    private FileOperations fileOperations;
+
+    @Inject
+    public AbstractYamlRestCompatTestPlugin(ProjectLayout projectLayout, FileOperations fileOperations) {
+        this.projectLayout = projectLayout;
+        this.fileOperations = fileOperations;
+    }
+
+    @Override
+    public void apply(Project project) {
+
+        final Path compatRestResourcesDir = Path.of("restResources").resolve("v" + COMPATIBLE_VERSION);
+        final Path compatSpecsDir = compatRestResourcesDir.resolve("yamlSpecs");
+        final Path compatTestsDir = compatRestResourcesDir.resolve("yamlTests");
+
+        project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class);
+        project.getPluginManager().apply(TestClustersPlugin.class);
+        project.getPluginManager().apply(LegacyRestTestBasePlugin.class);
+        project.getPluginManager().apply(RestResourcesPlugin.class);
+        project.getPluginManager().apply(getBasePlugin());
+
+        RestResourcesExtension extension = project.getExtensions().getByType(RestResourcesExtension.class);
+
+        // create source set
+        SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+        SourceSet yamlCompatTestSourceSet = sourceSets.create(SOURCE_SET_NAME);
+        SourceSet yamlTestSourceSet = sourceSets.getByName(LegacyYamlRestTestPlugin.SOURCE_SET_NAME);
+        GradleUtils.extendSourceSet(project, LegacyYamlRestTestPlugin.SOURCE_SET_NAME, SOURCE_SET_NAME);
+
+        // copy compatible rest specs
+        Configuration bwcMinorConfig = project.getConfigurations().create(BWC_MINOR_CONFIG_NAME);
+        Dependency bwcMinor = project.getDependencies()
+            .project(Map.of("path", ":distribution:bwc:maintenance", "configuration", "checkout"));
+        project.getDependencies().add(bwcMinorConfig.getName(), bwcMinor);
+
+        String projectPath = project.getPath();
+        ExtraPropertiesExtension extraProperties = project.getExtensions().getExtraProperties();
+        Provider<CopyRestApiTask> copyCompatYamlSpecTask = project.getTasks()
+            .register("copyRestCompatApiTask", CopyRestApiTask.class, task -> {
+                task.dependsOn(bwcMinorConfig);
+                task.setConfig(bwcMinorConfig);
+                task.setAdditionalConfig(bwcMinorConfig);
+                task.getInclude().set(extension.getRestApi().getInclude());
+                task.getOutputResourceDir().set(projectLayout.getBuildDirectory().dir(compatSpecsDir.toString()));
+                task.setSourceResourceDir(
+                    yamlCompatTestSourceSet.getResources()
+                        .getSrcDirs()
+                        .stream()
+                        .filter(f -> f.isDirectory() && f.getName().equals("resources"))
+                        .findFirst()
+                        .orElse(null)
+                );
+                task.setSkipHasRestTestCheck(true);
+                task.setConfigToFileTree(
+                    config -> fileOperations.fileTree(
+                        config.getSingleFile().toPath().resolve(RELATIVE_REST_API_RESOURCES).resolve(RELATIVE_API_PATH)
+                    )
+                );
+                task.setAdditionalConfigToFileTree(
+                    config -> fileOperations.fileTree(
+                        getCompatProjectPath(projectPath, config.getSingleFile().toPath()).resolve(RELATIVE_REST_PROJECT_RESOURCES)
+                            .resolve(RELATIVE_API_PATH)
+                    )
+                );
+                task.onlyIf(t -> isEnabled(extraProperties));
+            });
+
+        // copy compatible rest tests
+        Provider<CopyRestTestsTask> copyCompatYamlTestTask = project.getTasks()
+            .register("copyRestCompatTestTask", CopyRestTestsTask.class, task -> {
+                task.dependsOn(bwcMinorConfig);
+                task.setCoreConfig(bwcMinorConfig);
+                task.setXpackConfig(bwcMinorConfig);
+                task.setAdditionalConfig(bwcMinorConfig);
+                task.getIncludeCore().set(extension.getRestTests().getIncludeCore());
+                task.getIncludeXpack().set(extension.getRestTests().getIncludeXpack());
+                task.getOutputResourceDir().set(projectLayout.getBuildDirectory().dir(compatTestsDir.resolve("original").toString()));
+                task.setCoreConfigToFileTree(
+                    config -> fileOperations.fileTree(
+                        config.getSingleFile()
+                            .toPath()
+                            .resolve(RELATIVE_REST_CORE)
+                            .resolve(RELATIVE_REST_PROJECT_RESOURCES)
+                            .resolve(RELATIVE_TEST_PATH)
+                    )
+                );
+                task.setXpackConfigToFileTree(
+                    config -> fileOperations.fileTree(
+                        config.getSingleFile()
+                            .toPath()
+                            .resolve(RELATIVE_REST_XPACK)
+                            .resolve(RELATIVE_REST_PROJECT_RESOURCES)
+                            .resolve(RELATIVE_TEST_PATH)
+                    )
+                );
+                task.setAdditionalConfigToFileTree(
+                    config -> fileOperations.fileTree(
+                        getCompatProjectPath(projectPath, config.getSingleFile().toPath()).resolve(RELATIVE_REST_PROJECT_RESOURCES)
+                            .resolve(RELATIVE_TEST_PATH)
+                    )
+                );
+                task.dependsOn(copyCompatYamlSpecTask);
+                task.onlyIf(t -> isEnabled(extraProperties));
+            });
+
+        // copy both local source set apis and compat apis to a single location to be exported as an artifact
+        TaskProvider<Sync> bundleRestCompatApis = project.getTasks().register("bundleRestCompatApis", Sync.class, task -> {
+            task.setDestinationDir(projectLayout.getBuildDirectory().dir("bundledCompatApis").get().getAsFile());
+            task.setIncludeEmptyDirs(false);
+            task.from(copyCompatYamlSpecTask.flatMap(t -> t.getOutputResourceDir().map(d -> d.dir(RELATIVE_API_PATH.toString()))));
+            task.from(yamlCompatTestSourceSet.getProcessResourcesTaskName(), s -> {
+                s.include(RELATIVE_API_PATH + "/*");
+                s.eachFile(
+                    details -> details.setRelativePath(
+                        new RelativePath(true, Arrays.stream(details.getRelativePath().getSegments()).skip(2).toArray(String[]::new))
+                    )
+                );
+            });
+        });
+
+        // transform the copied tests task
+        TaskProvider<RestCompatTestTransformTask> transformCompatTestTask = project.getTasks()
+            .register("yamlRestTestV" + COMPATIBLE_VERSION + "CompatTransform", RestCompatTestTransformTask.class, task -> {
+                task.getSourceDirectory().set(copyCompatYamlTestTask.flatMap(CopyRestTestsTask::getOutputResourceDir));
+                task.getOutputDirectory()
+                    .set(project.getLayout().getBuildDirectory().dir(compatTestsDir.resolve("transformed").toString()));
+                task.onlyIf(t -> isEnabled(extraProperties));
+            });
+
+        // Register compat rest resources with source set
+        yamlCompatTestSourceSet.getOutput().dir(copyCompatYamlSpecTask.map(CopyRestApiTask::getOutputResourceDir));
+        yamlCompatTestSourceSet.getOutput().dir(transformCompatTestTask.map(RestCompatTestTransformTask::getOutputDirectory));
+
+        // Register artifact for transformed compatibility apis and tests
+        Configuration compatRestSpecs = project.getConfigurations().create(COMPATIBILITY_APIS_CONFIGURATION);
+        Configuration compatRestTests = project.getConfigurations().create(COMPATIBILITY_TESTS_CONFIGURATION);
+        project.getArtifacts().add(compatRestSpecs.getName(), bundleRestCompatApis.map(Sync::getDestinationDir));
+        project.getArtifacts()
+            .add(
+                compatRestTests.getName(),
+                transformCompatTestTask.flatMap(t -> t.getOutputDirectory().dir(RELATIVE_TEST_PATH.toString()))
+            );
+
+        // Grab the original rest resources locations so we can omit them from the compatibility testing classpath down below
+        Provider<Directory> originalYamlSpecsDir = project.getTasks()
+            .withType(CopyRestApiTask.class)
+            .named(RestResourcesPlugin.COPY_REST_API_SPECS_TASK)
+            .flatMap(CopyRestApiTask::getOutputResourceDir);
+        Provider<Directory> originalYamlTestsDir = project.getTasks()
+            .withType(CopyRestTestsTask.class)
+            .named(RestResourcesPlugin.COPY_YAML_TESTS_TASK)
+            .flatMap(CopyRestTestsTask::getOutputResourceDir);
+
+        // setup the test task
+        TaskProvider<? extends Test> yamlRestCompatTestTask = registerTestTask(project, yamlCompatTestSourceSet);
+        yamlRestCompatTestTask.configure(testTask -> {
+            testTask.systemProperty("tests.restCompat", true);
+            // Use test runner and classpath from "normal" yaml source set
+            testTask.setTestClassesDirs(
+                yamlTestSourceSet.getOutput().getClassesDirs().plus(yamlCompatTestSourceSet.getOutput().getClassesDirs())
+            );
+            testTask.setClasspath(
+                yamlCompatTestSourceSet.getRuntimeClasspath()
+                    // remove the "normal" api and tests
+                    .minus(project.files(yamlTestSourceSet.getOutput().getResourcesDir()))
+                    .minus(project.files(originalYamlSpecsDir))
+                    .minus(project.files(originalYamlTestsDir))
+            );
+
+            // run compatibility tests after "normal" tests
+            testTask.mustRunAfter(project.getTasks().named(LegacyYamlRestTestPlugin.SOURCE_SET_NAME));
+            testTask.onlyIf(t -> isEnabled(extraProperties));
+        });
+
+        setupYamlRestTestDependenciesDefaults(project, yamlCompatTestSourceSet, true);
+
+        // setup IDE
+        GradleUtils.setupIdeForTestSourceSet(project, yamlCompatTestSourceSet);
+
+        // add a lifecycle task to allow for a possible future additional rest compatibility without needing to change task names
+        TaskProvider<Task> checkRestCompatTask = project.getTasks().register(REST_COMPAT_CHECK_TASK_NAME, (thisCheckTask) -> {
+            thisCheckTask.setDescription("Runs all REST compatibility checks.");
+            thisCheckTask.setGroup("verification");
+        });
+
+        // wire the lifecycle task into the main check task
+        project.getTasks().named(JavaBasePlugin.CHECK_TASK_NAME).configure(check -> check.dependsOn(checkRestCompatTask));
+
+        // wire the yamlRestCompatTest into the custom lifecycle task
+        project.getTasks().named(REST_COMPAT_CHECK_TASK_NAME).configure(check -> check.dependsOn(yamlRestCompatTestTask));
+
+    }
+
+    public abstract TaskProvider<? extends Test> registerTestTask(Project project, SourceSet sourceSet);
+
+    public abstract Class<? extends Plugin<Project>> getBasePlugin();
+
+    private boolean isEnabled(ExtraPropertiesExtension extraProperties) {
+        Object bwcEnabled = extraProperties.getProperties().get("bwc_tests_enabled");
+        return bwcEnabled == null || (Boolean) bwcEnabled;
+    }
+
+    // TODO: implement custom extension that allows us move around of the projects between major versions and still find them
+    private Path getCompatProjectPath(String projectPath, Path checkoutDir) {
+        return checkoutDir.resolve(projectPath.replaceFirst(":", "").replace(":", File.separator));
+    }
+}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/LegacyYamlRestCompatTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/LegacyYamlRestCompatTestPlugin.java
new file mode 100644
index 000000000000..e84c84cc426a
--- /dev/null
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/LegacyYamlRestCompatTestPlugin.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test.rest.compat.compat;
+
+import org.elasticsearch.gradle.internal.test.rest.LegacyYamlRestTestPlugin;
+import org.elasticsearch.gradle.internal.test.rest.RestTestUtil;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.file.ProjectLayout;
+import org.gradle.api.internal.file.FileOperations;
+import org.gradle.api.tasks.SourceSet;
+import org.gradle.api.tasks.TaskProvider;
+import org.gradle.api.tasks.testing.Test;
+
+import javax.inject.Inject;
+
+/**
+ * Apply this plugin to run the YAML based REST tests from a prior major version against this version's cluster.
+ *
+ * @deprecated use {@link YamlRestCompatTestPlugin}
+ */
+@Deprecated
+public class LegacyYamlRestCompatTestPlugin extends AbstractYamlRestCompatTestPlugin {
+    @Inject
+    public LegacyYamlRestCompatTestPlugin(ProjectLayout projectLayout, FileOperations fileOperations) {
+        super(projectLayout, fileOperations);
+    }
+
+    @Override
+    public TaskProvider<? extends Test> registerTestTask(Project project, SourceSet sourceSet) {
+        return RestTestUtil.registerTestTask(project, sourceSet, sourceSet.getTaskName(null, "test"));
+    }
+
+    @Override
+    public Class<? extends Plugin<Project>> getBasePlugin() {
+        return LegacyYamlRestTestPlugin.class;
+    }
+}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java
similarity index 99%
rename from build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java
rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java
index bfb53c23b5f1..eee1c4c21eb0 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.gradle.internal.rest.compat;
+package org.elasticsearch.gradle.internal.test.rest.compat.compat;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/YamlRestCompatTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/YamlRestCompatTestPlugin.java
new file mode 100644
index 000000000000..79588ca722ff
--- /dev/null
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/YamlRestCompatTestPlugin.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test.rest.compat.compat;
+
+import org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin;
+import org.elasticsearch.gradle.internal.test.rest.RestTestUtil;
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.file.ProjectLayout;
+import org.gradle.api.internal.file.FileOperations;
+import org.gradle.api.tasks.SourceSet;
+import org.gradle.api.tasks.TaskProvider;
+import org.gradle.api.tasks.testing.Test;
+
+import javax.inject.Inject;
+
+/**
+ * Apply this plugin to run the YAML based REST tests from a prior major version against this version's cluster.
+ */
+public class YamlRestCompatTestPlugin extends AbstractYamlRestCompatTestPlugin {
+    @Inject
+    public YamlRestCompatTestPlugin(ProjectLayout projectLayout, FileOperations fileOperations) {
+        super(projectLayout, fileOperations);
+    }
+
+    @Override
+    public TaskProvider<? extends Test> registerTestTask(Project project, SourceSet sourceSet) {
+        return RestTestUtil.registerTestTask(project, sourceSet, sourceSet.getTaskName(null, "test"), StandaloneRestIntegTestTask.class);
+    }
+
+    @Override
+    public Class<? extends Plugin<Project>> getBasePlugin() {
+        return InternalYamlRestTestPlugin.class;
+    }
+}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java
index 384774190207..3d59cf337e87 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java
@@ -117,6 +117,7 @@ public void execute(Task task) {
             maybeSkipTask(dockerSupport, buildFixture);
 
             ComposeExtension composeExtension = project.getExtensions().getByType(ComposeExtension.class);
+            composeExtension.setProjectName(project.getName());
             composeExtension.getUseComposeFiles().addAll(Collections.singletonList(DOCKER_COMPOSE_YML));
             composeExtension.getRemoveContainers().set(true);
             composeExtension.getCaptureContainersOutput()
diff --git a/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt b/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt
index 729907f9e42a..64f89d6c57ed 100644
--- a/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt
+++ b/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt
@@ -139,7 +139,9 @@ org.apache.logging.log4j.LogManager#getLogger()
 
 # This is permitted in test code, where we have a Checkstyle rule to guard
 # against unsafe uses. This leniency does not extend to server code.
-java.lang.String#formatted(java.lang.Object[]) @ Uses default locale - use String#format(Locale, String, Object...) instead
+@defaultMessage Uses default locale - use org.elasticsearch.common.Strings#format(String, Object...) instead
+java.lang.String#formatted(java.lang.Object[])
+java.lang.String#format(java.lang.String,java.lang.Object[])
 
 @defaultMessage Unbatched cluster state tasks are a source of performance and stability bugs. Implement the update logic in a executor which is reused across tasks instead.
 org.elasticsearch.cluster.service.MasterService#submitUnbatchedStateUpdateTask(java.lang.String, org.elasticsearch.cluster.ClusterStateUpdateTask)
diff --git a/build-tools-internal/src/main/resources/templates/breaking-changes.asciidoc b/build-tools-internal/src/main/resources/templates/breaking-changes.asciidoc
index a16c7e393ee9..d181c60f6463 100644
--- a/build-tools-internal/src/main/resources/templates/breaking-changes.asciidoc
+++ b/build-tools-internal/src/main/resources/templates/breaking-changes.asciidoc
@@ -7,7 +7,7 @@
 This section discusses the changes that you need to be aware of when migrating
 your application to {es} ${majorDotMinor}.
 
-See also <<release-highlights>> and <<es-release-notes>>.
+See also {ref-bare}/${majorDotMinor}/release-highlights.html[What's new in ${majorDotMinor}] and <<es-release-notes>>.
 <% if (isElasticsearchSnapshot) { %>
 coming::[${majorDotMinorDotRevision}]
 <% } %>
diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.generateMigrationFile.asciidoc b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.generateMigrationFile.asciidoc
index d8be43130260..1c835da9994a 100644
--- a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.generateMigrationFile.asciidoc
+++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/BreakingChangesGeneratorTest.generateMigrationFile.asciidoc
@@ -7,7 +7,7 @@
 This section discusses the changes that you need to be aware of when migrating
 your application to {es} 8.4.
 
-See also <<release-highlights>> and <<es-release-notes>>.
+See also {ref-bare}/8.4/release-highlights.html[What's new in 8.4] and <<es-release-notes>>.
 
 coming::[8.4.0]
 
diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties
index 32617c9623a9..0444d4152ce1 100644
--- a/build-tools-internal/version.properties
+++ b/build-tools-internal/version.properties
@@ -1,32 +1,30 @@
 elasticsearch     = 8.7.0
-lucene            = 9.4.1
+lucene            = 9.5.0
 
 bundled_jdk_vendor = openjdk
-bundled_jdk = 19.0.1+10@afdd2e245b014143b62ccb916125e3ce
+bundled_jdk = 19.0.2+7@fdb695a9d9064ad6b064dc6df578380c
 
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-jackson           = 2.14.0
+jackson           = 2.14.2
 snakeyaml         = 1.33
 icu4j             = 68.2
 supercsv          = 2.4.0
 log4j             = 2.19.0
 slf4j             = 1.6.2
 ecsLogging        = 1.2.0
-
 jna               = 5.10.0
+netty             = 4.1.86.Final
+commons_lang3     = 3.9
+google_oauth_client = 1.34.1
 
-netty             = 4.1.84.Final
-
-commons_lang3                   = 3.9
-
+antlr4            = 4.11.1
 # when updating this version, you need to ensure compatibility with:
 #  - modules/ingest-attachment (transitive dependency, check the upstream POM)
 #  - distribution/tools/plugin-cli
 #  - x-pack/plugin/security
 bouncycastle=1.64
-
 # used by security and idp (need to be in sync due to cross-dependency in testing)
 opensaml = 4.0.1
 
@@ -36,6 +34,7 @@ httpcore          = 4.4.13
 httpasyncclient   = 4.1.5
 commonslogging    = 1.2
 commonscodec      = 1.15
+protobuf          = 3.21.9
 
 # test dependencies
 randomizedrunner  = 2.8.0
diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/StablePluginBuildPluginFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/StablePluginBuildPluginFuncTest.groovy
index fc706497008a..9b0a44ad9710 100644
--- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/StablePluginBuildPluginFuncTest.groovy
+++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/StablePluginBuildPluginFuncTest.groovy
@@ -19,15 +19,11 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.util.stream.Collectors
 
+import static org.elasticsearch.gradle.fixtures.TestClasspathUtils.setupNamedComponentScanner
+
 class StablePluginBuildPluginFuncTest extends AbstractGradleFuncTest {
 
     def setup() {
-        // underlaying TestClusterPlugin and StandaloneRestIntegTestTask are not cc compatible
-        configurationCacheCompatible = false
-    }
-
-    def "can build stable plugin properties"() {
-        given:
         buildFile << """plugins {
                 id 'elasticsearch.stable-esplugin'
             }
@@ -38,8 +34,27 @@ class StablePluginBuildPluginFuncTest extends AbstractGradleFuncTest {
                 name = 'myplugin'
                 description = 'test plugin'
             }
+            repositories {
+              maven {
+                name = "local-test"
+                url = file("local-repo")
+                metadataSources {
+                  artifact()
+                }
+              }
+            }
             """
 
+        // underlaying TestClusterPlugin and StandaloneRestIntegTestTask are not cc compatible
+        configurationCacheCompatible = false
+
+        def version = VersionProperties.elasticsearch
+        setupNamedComponentScanner(dir("local-repo/org/elasticsearch/elasticsearch-plugin-scanner/${version}/"), version)
+
+    }
+
+    def "can build stable plugin properties"() {
+        given:
         when:
         def result = gradleRunner(":pluginProperties").build()
         def props = getPluginProperties()
@@ -62,32 +77,22 @@ class StablePluginBuildPluginFuncTest extends AbstractGradleFuncTest {
     }
 
     def "can scan and create named components file"() {
+        //THIS IS RUNNING A MOCK CONFIGURED IN setup()
         given:
         File jarFolder = new File(testProjectDir.root, "jars")
         jarFolder.mkdirs()
 
-        buildFile << """plugins {
-                id 'elasticsearch.stable-esplugin'
-            }
-
-            version = '1.2.3'
-
-            esplugin {
-                name = 'myplugin'
-                description = 'test plugin'
-            }
-
+        buildFile << """
             dependencies {
                 implementation files('${normalized(StableApiJarMocks.createPluginApiJar(jarFolder.toPath()).toAbsolutePath().toString())}')
                 implementation files('${normalized(StableApiJarMocks.createExtensibleApiJar(jarFolder.toPath()).toAbsolutePath().toString())}')
             }
-
             """
 
         file("src/main/java/org/acme/A.java") << """
             package org.acme;
 
-            import org.elasticsearch.plugin.api.NamedComponent;
+            import org.elasticsearch.plugin.NamedComponent;
             import org.elasticsearch.plugin.scanner.test_classes.ExtensibleClass;
 
             @NamedComponent( "componentA")
@@ -95,18 +100,16 @@ class StablePluginBuildPluginFuncTest extends AbstractGradleFuncTest {
             }
         """
 
-
         when:
-        def result = gradleRunner(":assemble").build()
-        Path namedComponents = file("build/generated-named-components/named_components.json").toPath();
-        def map = new JsonSlurper().parse(namedComponents.toFile())
+        def result = gradleRunner(":assemble", "-i").build()
+
         then:
         result.task(":assemble").outcome == TaskOutcome.SUCCESS
-
-        map  == ["org.elasticsearch.plugin.scanner.test_classes.ExtensibleClass" : (["componentA" : "org.acme.A"]) ]
+        //we expect that a Fake namedcomponent scanner used in this test will be passed a filename to be created
+        File namedComponents = file("build/generated-named-components/named_components.json")
+        namedComponents.exists() == true
     }
 
-
     Map<String, String> getPluginProperties() {
         Path propsFile = file("build/generated-descriptor/stable-plugin-descriptor.properties").toPath();
         Properties rawProps = new Properties()
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/NamedComponentScannerMock.java b/build-tools/src/main/java/org/elasticsearch/gradle/NamedComponentScannerMock.java
new file mode 100644
index 000000000000..cfd3ab566b33
--- /dev/null
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/NamedComponentScannerMock.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+public class NamedComponentScannerMock {
+    public static void main(String[] args) throws IOException {
+        // expect a file name to passed in as a parameter
+        // creating a file so that we can assert about this in a test
+        Path path = Path.of(args[0]);
+        Files.createDirectories(path.getParent());
+        Files.createFile(path);
+    }
+}
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/GenerateNamedComponentsTask.java b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/GenerateNamedComponentsTask.java
index 9a44a5214899..7945bce426cc 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/GenerateNamedComponentsTask.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/GenerateNamedComponentsTask.java
@@ -8,55 +8,60 @@
 
 package org.elasticsearch.gradle.plugin;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import org.elasticsearch.gradle.plugin.scanner.ClassReaders;
-import org.elasticsearch.gradle.plugin.scanner.NamedComponentScanner;
+import org.elasticsearch.gradle.LoggedExec;
 import org.gradle.api.DefaultTask;
-import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.ProjectLayout;
 import org.gradle.api.file.RegularFileProperty;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
-import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.tasks.CompileClasspath;
+import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.TaskAction;
-import org.gradle.workers.WorkAction;
-import org.gradle.workers.WorkParameters;
+import org.gradle.process.ExecOperations;
+import org.gradle.process.ExecResult;
 import org.gradle.workers.WorkerExecutor;
-import org.objectweb.asm.ClassReader;
 
 import java.io.File;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
 
 import javax.inject.Inject;
 
 public abstract class GenerateNamedComponentsTask extends DefaultTask {
     private static final Logger LOGGER = Logging.getLogger(GenerateNamedComponentsTask.class);
+    private static final String NAMED_COMPONENTS_DIR = "generated-named-components/";
     private static final String NAMED_COMPONENTS_FILE = "named_components.json";
+    private static final String NAMED_COMPONENTS_PATH = NAMED_COMPONENTS_DIR + NAMED_COMPONENTS_FILE;
 
     private final WorkerExecutor workerExecutor;
+    private FileCollection pluginScannerClasspath;
     private FileCollection classpath;
+    private ExecOperations execOperations;
+    private ProjectLayout projectLayout;
 
     @Inject
-    public GenerateNamedComponentsTask(WorkerExecutor workerExecutor, ObjectFactory objectFactory, ProjectLayout projectLayout) {
+    public GenerateNamedComponentsTask(WorkerExecutor workerExecutor, ExecOperations execOperations, ProjectLayout projectLayout) {
         this.workerExecutor = workerExecutor;
-        getOutputFile().convention(projectLayout.getBuildDirectory().file("generated-named-components/" + NAMED_COMPONENTS_FILE));
+        this.execOperations = execOperations;
+        this.projectLayout = projectLayout;
+
+        getOutputFile().convention(projectLayout.getBuildDirectory().file(NAMED_COMPONENTS_PATH));
     }
 
     @TaskAction
     public void scanPluginClasses() {
-        workerExecutor.noIsolation().submit(GenerateNamedComponentsAction.class, params -> {
-            params.getClasspath().from(classpath);
-            params.getOutputFile().set(getOutputFile());
+        File outputFile = projectLayout.getBuildDirectory().file(NAMED_COMPONENTS_PATH).get().getAsFile();
+
+        ExecResult execResult = LoggedExec.javaexec(execOperations, spec -> {
+            spec.classpath(pluginScannerClasspath.plus(getClasspath()).getAsPath());
+            spec.getMainClass().set("org.elasticsearch.plugin.scanner.NamedComponentScanner");
+            spec.args(outputFile);
+            spec.setErrorOutput(System.err);
+            spec.setStandardOutput(System.out);
         });
+        execResult.assertNormalExitValue();
     }
 
     @OutputFile
@@ -71,37 +76,13 @@ public void setClasspath(FileCollection classpath) {
         this.classpath = classpath;
     }
 
-    public abstract static class GenerateNamedComponentsAction implements WorkAction<Parameters> {
-        private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
-
-        @Override
-        public void execute() {
-            Set<File> classpathFiles = getParameters().getClasspath().getFiles();
-
-            List<ClassReader> classReaders = ClassReaders.ofPaths(classpathFiles.stream().map(File::toPath)).collect(Collectors.toList());
-
-            NamedComponentScanner namedComponentScanner = new NamedComponentScanner();
-            Map<String, Map<String, String>> namedComponentsMap = namedComponentScanner.scanForNamedClasses(classReaders);
-            writeToFile(namedComponentsMap);
-        }
-
-        private void writeToFile(Map<String, Map<String, String>> namedComponentsMap) {
-            try {
-                String json = OBJECT_MAPPER.writeValueAsString(namedComponentsMap);
-                File file = getParameters().getOutputFile().getAsFile().get();
-                Path of = Path.of(file.getAbsolutePath());
-                Files.writeString(of, json);
-            } catch (Exception e) {
-                e.printStackTrace();
-            }
-
-        }
+    public void setPluginScannerClasspath(FileCollection pluginScannerClasspath) {
+        this.pluginScannerClasspath = pluginScannerClasspath;
     }
 
-    interface Parameters extends WorkParameters {
-
-        ConfigurableFileCollection getClasspath();
-
-        RegularFileProperty getOutputFile();
+    @InputFiles
+    @PathSensitive(PathSensitivity.RELATIVE)
+    public FileCollection getPluginScannerClasspath() {
+        return pluginScannerClasspath;
     }
 }
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/StablePluginBuildPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/StablePluginBuildPlugin.java
index 80f65ea26fa8..ef2d1631d560 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/StablePluginBuildPlugin.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/StablePluginBuildPlugin.java
@@ -8,9 +8,12 @@
 
 package org.elasticsearch.gradle.plugin;
 
+import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.dsl.DependencyHandler;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.RegularFile;
 import org.gradle.api.plugins.JavaPlugin;
@@ -33,12 +36,21 @@ public void apply(Project project) {
         });
 
         final var pluginNamedComponents = project.getTasks().register("pluginNamedComponents", GenerateNamedComponentsTask.class, t -> {
+
             SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME);
             FileCollection dependencyJars = project.getConfigurations().getByName(JavaPlugin.COMPILE_CLASSPATH_CONFIGURATION_NAME);
             FileCollection compiledPluginClasses = mainSourceSet.getOutput().getClassesDirs();
             FileCollection classPath = dependencyJars.plus(compiledPluginClasses);
             t.setClasspath(classPath);
         });
+        Configuration pluginScannerConfig = project.getConfigurations().create("pluginScannerConfig");
+        DependencyHandler dependencyHandler = project.getDependencies();
+        pluginScannerConfig.defaultDependencies(
+            deps -> deps.add(
+                dependencyHandler.create("org.elasticsearch:elasticsearch-plugin-scanner:" + VersionProperties.getElasticsearch())
+            )
+        );
+        pluginNamedComponents.configure(t -> { t.setPluginScannerClasspath(pluginScannerConfig); });
 
         final var pluginExtension = project.getExtensions().getByType(PluginPropertiesExtension.class);
         pluginExtension.getBundleSpec().from(pluginNamedComponents);
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/scanner/ClassReaders.java b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/scanner/ClassReaders.java
deleted file mode 100644
index 40cc66c0a485..000000000000
--- a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/scanner/ClassReaders.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.gradle.plugin.scanner;
-
-import org.objectweb.asm.ClassReader;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.UncheckedIOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.jar.JarFile;
-import java.util.stream.Stream;
-import java.util.zip.ZipFile;
-
-/**
- * A utility class containing methods to create streams of ASM's ClassReader
- *
- * @see ClassReader
- */
-public class ClassReaders {
-    private static final String MODULE_INFO = "module-info.class";
-
-    /**
-     * This method must be used within a try-with-resources statement or similar
-     * control structure.
-     */
-    public static Stream<ClassReader> ofDirWithJars(String path) {
-        if (path == null) {
-            return Stream.empty();
-        }
-        Path dir = Paths.get(path);
-        try {
-            return ofPaths(Files.list(dir));
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-    }
-
-    /**
-     * This method must be used within a try-with-resources statement or similar
-     * control structure.
-     */
-    public static Stream<ClassReader> ofPaths(Stream<Path> list) {
-        return list.filter(Files::exists).flatMap(p -> {
-            if (p.toString().endsWith(".jar")) {
-                return classesInJar(p);
-            } else {
-                return classesInPath(p);
-            }
-        });
-    }
-
-    private static Stream<ClassReader> classesInJar(Path jar) {
-        try {
-            JarFile jf = new JarFile(jar.toFile(), true, ZipFile.OPEN_READ, Runtime.version());
-
-            Stream<ClassReader> classReaderStream = jf.versionedStream()
-                .filter(e -> e.getName().endsWith(".class") && e.getName().equals(MODULE_INFO) == false)
-                .map(e -> {
-                    try (InputStream is = jf.getInputStream(e)) {
-                        byte[] classBytes = is.readAllBytes();
-                        return new ClassReader(classBytes);
-                    } catch (IOException ex) {
-                        throw new UncheckedIOException(ex);
-                    }
-                });
-            return classReaderStream;
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-    }
-
-    private static Stream<ClassReader> classesInPath(Path root) {
-        try {
-            Stream<Path> stream = Files.walk(root);
-            return stream.filter(p -> p.toString().endsWith(".class"))
-                .filter(p -> p.toString().endsWith("module-info.class") == false)
-                .map(p -> {
-                    try (InputStream is = Files.newInputStream(p)) {
-                        byte[] classBytes = is.readAllBytes();
-                        return new ClassReader(classBytes);
-                    } catch (IOException e) {
-                        throw new UncheckedIOException(e);
-                    }
-                });
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-    }
-}
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/scanner/NamedComponentScanner.java b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/scanner/NamedComponentScanner.java
deleted file mode 100644
index 86843c8f20ee..000000000000
--- a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/scanner/NamedComponentScanner.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.gradle.plugin.scanner;
-
-import org.objectweb.asm.AnnotationVisitor;
-import org.objectweb.asm.ClassReader;
-import org.objectweb.asm.Opcodes;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-
-public class NamedComponentScanner {
-
-    // returns a Map<String, Map<String,String> - extensible interface -> map{ namedName -> className }
-    public Map<String, Map<String, String>> scanForNamedClasses(Collection<ClassReader> classReaderStream) {
-        // TODO I don't have access to stable-plugin-api here so I have to hardcode class descriptors
-        ClassScanner extensibleClassScanner = new ClassScanner("Lorg/elasticsearch/plugin/api/Extensible;", (classname, map) -> {
-            map.put(classname, classname);
-            return null;
-        });
-        extensibleClassScanner.visit(classReaderStream.stream());
-
-        ClassScanner namedComponentsScanner = new ClassScanner(
-            "Lorg/elasticsearch/plugin/api/NamedComponent;"/*NamedComponent.class*/,
-            (classname, map) -> new AnnotationVisitor(Opcodes.ASM9) {
-                @Override
-                public void visit(String key, Object value) {
-                    assert key.equals("value");
-                    assert value instanceof String;
-                    map.put(value.toString(), classname);
-                }
-            }
-        );
-
-        namedComponentsScanner.visit(classReaderStream.stream());
-
-        Map<String, Map<String, String>> componentInfo = new HashMap<>();
-        for (var e : namedComponentsScanner.getFoundClasses().entrySet()) {
-            String name = e.getKey();
-            String classnameWithSlashes = e.getValue();
-            String extensibleClassnameWithSlashes = extensibleClassScanner.getFoundClasses().get(classnameWithSlashes);
-            if (extensibleClassnameWithSlashes == null) {
-                throw new RuntimeException(
-                    "Named component " + name + "(" + pathToClassName(classnameWithSlashes) + ") does not extend from an extensible class"
-                );
-            }
-            var named = componentInfo.computeIfAbsent(pathToClassName(extensibleClassnameWithSlashes), k -> new HashMap<>());
-            named.put(name, pathToClassName(classnameWithSlashes));
-        }
-        return componentInfo;
-    }
-
-    private String pathToClassName(String classWithSlashes) {
-        return classWithSlashes.replace('/', '.');
-    }
-}
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java
index 0317b74bf0d3..f6705bdb62fa 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java
@@ -491,6 +491,13 @@ public List<String> getAllReadinessPortURI() {
         return nodes.stream().flatMap(each -> each.getAllReadinessPortURI().stream()).collect(Collectors.toList());
     }
 
+    @Override
+    @Internal
+    public List<String> getAllRemoteAccessPortURI() {
+        waitForAllConditions();
+        return nodes.stream().flatMap(each -> each.getAllRemoteAccessPortURI().stream()).collect(Collectors.toList());
+    }
+
     public void waitForAllConditions() {
         writeUnicastHostsFiles();
 
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
index 85433e5a0772..fcb1af73e854 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
@@ -164,6 +164,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private final Path transportPortFile;
     private final Path httpPortsFile;
     private final Path readinessPortsFile;
+    private final Path remoteAccessPortsFile;
     private final Path esOutputFile;
     private final Path esInputFile;
     private final Path tmpDir;
@@ -215,6 +216,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         transportPortFile = confPathLogs.resolve("transport.ports");
         httpPortsFile = confPathLogs.resolve("http.ports");
         readinessPortsFile = confPathLogs.resolve("readiness.ports");
+        remoteAccessPortsFile = confPathLogs.resolve("remote_cluster.ports");
         esOutputFile = confPathLogs.resolve("es.out");
         esInputFile = workingDir.resolve("es.in");
         tmpDir = workingDir.resolve("tmp");
@@ -999,6 +1001,13 @@ public List<String> getAllReadinessPortURI() {
         return getReadinessPortInternal();
     }
 
+    @Override
+    @Internal
+    public List<String> getAllRemoteAccessPortURI() {
+        waitForAllConditions();
+        return getRemoteAccessPortInternal();
+    }
+
     @Internal
     public File getServerLog() {
         return confPathLogs.resolve(defaultConfig.get("cluster.name") + "_server.json").toFile();
@@ -1022,6 +1031,9 @@ public synchronized void stop(boolean tailLogs) {
             if (Files.exists(readinessPortsFile)) {
                 Files.delete(readinessPortsFile);
             }
+            if (Files.exists(remoteAccessPortsFile)) {
+                Files.delete(remoteAccessPortsFile);
+            }
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
@@ -1047,6 +1059,9 @@ public synchronized void stop(boolean tailLogs) {
             if (Files.exists(readinessPortsFile)) {
                 Files.delete(readinessPortsFile);
             }
+            if (Files.exists(remoteAccessPortsFile)) {
+                Files.delete(remoteAccessPortsFile);
+            }
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
@@ -1519,6 +1534,14 @@ private List<String> getReadinessPortInternal() {
         }
     }
 
+    private List<String> getRemoteAccessPortInternal() {
+        try {
+            return readPortsFile(remoteAccessPortsFile);
+        } catch (IOException e) {
+            return new ArrayList<>();
+        }
+    }
+
     private List<String> readPortsFile(Path file) throws IOException {
         try (Stream<String> lines = Files.lines(file, StandardCharsets.UTF_8)) {
             return lines.map(String::trim).collect(Collectors.toList());
@@ -1673,20 +1696,17 @@ public boolean isHttpSslEnabled() {
     }
 
     void configureHttpWait(WaitForHttpResource wait) {
-        if (settings.containsKey("xpack.security.http.ssl.certificate_authorities")) {
-            wait.setCertificateAuthorities(
-                getConfigDir().resolve(settings.get("xpack.security.http.ssl.certificate_authorities").toString()).toFile()
-            );
-        }
         if (settings.containsKey("xpack.security.http.ssl.certificate")) {
-            wait.setCertificateAuthorities(getConfigDir().resolve(settings.get("xpack.security.http.ssl.certificate").toString()).toFile());
-        }
-        if (settings.containsKey("xpack.security.http.ssl.keystore.path")
-            && settings.containsKey("xpack.security.http.ssl.certificate_authorities") == false) { // Can not set both trust stores and CA
-            wait.setTrustStoreFile(getConfigDir().resolve(settings.get("xpack.security.http.ssl.keystore.path").toString()).toFile());
-        }
-        if (keystoreSettings.containsKey("xpack.security.http.ssl.keystore.secure_password")) {
-            wait.setTrustStorePassword(keystoreSettings.get("xpack.security.http.ssl.keystore.secure_password").toString());
+            wait.setServerCertificate(getConfigDir().resolve(settings.get("xpack.security.http.ssl.certificate").toString()).toFile());
+        } else {
+            if (settings.containsKey("xpack.security.http.ssl.keystore.path")) {
+                wait.setServerKeystoreFile(
+                    getConfigDir().resolve(settings.get("xpack.security.http.ssl.keystore.path").toString()).toFile()
+                );
+            }
+            if (keystoreSettings.containsKey("xpack.security.http.ssl.keystore.secure_password")) {
+                wait.setServerKeystorePassword(keystoreSettings.get("xpack.security.http.ssl.keystore.secure_password").toString());
+            }
         }
     }
 
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/SslTrustResolver.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/SslTrustResolver.java
new file mode 100644
index 000000000000..4c2cbf4defb7
--- /dev/null
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/SslTrustResolver.java
@@ -0,0 +1,219 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.testclusters;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.GeneralSecurityException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.SecureRandom;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
+import java.security.cert.CertificateFactory;
+import java.security.cert.X509Certificate;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Enumeration;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.X509TrustManager;
+
+class SslTrustResolver {
+    private Set<File> certificateAuthorities;
+    private File trustStoreFile;
+    private String trustStorePassword;
+    private File serverCertificate;
+    private File serverKeyStoreFile;
+    private String serverKeyStorePassword;
+
+    public void setCertificateAuthorities(File... certificateAuthorities) {
+        this.certificateAuthorities = new HashSet<>(Arrays.asList(certificateAuthorities));
+    }
+
+    public void setTrustStoreFile(File trustStoreFile) {
+        this.trustStoreFile = trustStoreFile;
+    }
+
+    public void setTrustStorePassword(String trustStorePassword) {
+        this.trustStorePassword = trustStorePassword;
+    }
+
+    public void setServerCertificate(File serverCertificate) {
+        this.serverCertificate = serverCertificate;
+    }
+
+    public void setServerKeystoreFile(File keyStoreFile) {
+        this.serverKeyStoreFile = keyStoreFile;
+    }
+
+    public void setServerKeystorePassword(String keyStorePassword) {
+        this.serverKeyStorePassword = keyStorePassword;
+    }
+
+    public SSLContext getSslContext() throws GeneralSecurityException, IOException {
+        final TrustManager[] trustManagers = buildTrustManagers();
+        if (trustManagers != null) {
+            return createSslContext(trustManagers);
+        } else {
+            return null;
+        }
+    }
+
+    TrustManager[] buildTrustManagers() throws GeneralSecurityException, IOException {
+        var configurationCount = Stream.of(
+            this.certificateAuthorities,
+            this.trustStoreFile,
+            this.serverCertificate,
+            this.serverKeyStoreFile
+        ).filter(Objects::nonNull).count();
+        if (configurationCount == 0) {
+            return null;
+        } else if (configurationCount > 1) {
+            throw new IllegalStateException(
+                String.format(
+                    Locale.ROOT,
+                    "Cannot specify more than one trust method (CA=%s, trustStore=%s, serverCert=%s, serverKeyStore=%s)",
+                    certificateAuthorities,
+                    trustStoreFile,
+                    serverCertificate,
+                    serverKeyStoreFile
+                )
+            );
+        }
+        if (this.certificateAuthorities != null) {
+            return getTrustManagers(buildTrustStoreFromCA(certificateAuthorities));
+        } else if (this.trustStoreFile != null) {
+            return getTrustManagers(readKeyStoreFromFile(trustStoreFile, trustStorePassword));
+        } else if (this.serverCertificate != null) {
+            return buildTrustManagerFromLeafCertificates(head(readCertificates(serverCertificate)));
+        } else if (this.serverKeyStoreFile != null) {
+            return buildTrustManagerFromLeafCertificates(readCertificatesFromKeystore(serverKeyStoreFile, serverKeyStorePassword));
+        } else {
+            // Cannot get here unless the code gets out of sync with the 'configurationCount == 0' check above
+            throw new IllegalStateException("Expected to configure trust, but all configuration values are null");
+        }
+    }
+
+    private SSLContext createSslContext(TrustManager[] trustManagers) throws GeneralSecurityException {
+        SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
+        sslContext.init(new KeyManager[0], trustManagers, new SecureRandom());
+        return sslContext;
+    }
+
+    private TrustManager[] getTrustManagers(KeyStore trustStore) throws GeneralSecurityException {
+        checkForTrustEntry(trustStore);
+        TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+        tmf.init(trustStore);
+        return tmf.getTrustManagers();
+    }
+
+    private void checkForTrustEntry(KeyStore trustStore) throws KeyStoreException {
+        Enumeration<String> enumeration = trustStore.aliases();
+        while (enumeration.hasMoreElements()) {
+            if (trustStore.isCertificateEntry(enumeration.nextElement())) {
+                // found trusted cert entry
+                return;
+            }
+        }
+        throw new IllegalStateException("Trust-store does not contain any trusted certificate entries");
+    }
+
+    private static KeyStore buildTrustStoreFromCA(Set<File> files) throws GeneralSecurityException, IOException {
+        final KeyStore store = KeyStore.getInstance(KeyStore.getDefaultType());
+        store.load(null, null);
+        int counter = 0;
+        for (File ca : files) {
+            for (Certificate certificate : readCertificates(ca)) {
+                store.setCertificateEntry("cert-" + counter, certificate);
+                counter++;
+            }
+        }
+        return store;
+    }
+
+    private static TrustManager[] buildTrustManagerFromLeafCertificates(Collection<? extends Certificate> certificates) {
+        final Set<X509Certificate> trusted = certificates.stream()
+            .filter(X509Certificate.class::isInstance)
+            .map(X509Certificate.class::cast)
+            .collect(Collectors.toUnmodifiableSet());
+
+        var trustManager = new X509TrustManager() {
+            @Override
+            public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
+                final X509Certificate leaf = chain[0];
+                if (trusted.contains(leaf) == false) {
+                    throw new CertificateException("Untrusted leaf certificate: " + leaf.getSubjectX500Principal());
+                }
+            }
+
+            @Override
+            public X509Certificate[] getAcceptedIssuers() {
+                // This doesn't apply when trusting leaf certs, and is only really needed for server trust managers anyways
+                return new X509Certificate[0];
+            }
+
+            @Override
+            public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
+                throw new CertificateException("This trust manager is for client use only and cannot trust other clients");
+            }
+
+        };
+        return new TrustManager[] { trustManager };
+    }
+
+    private static Collection<Certificate> readCertificatesFromKeystore(File file, String password) throws GeneralSecurityException,
+        IOException {
+        var keyStore = readKeyStoreFromFile(file, password);
+        final Set<Certificate> certificates = new HashSet<>(keyStore.size());
+        var enumeration = keyStore.aliases();
+        while (enumeration.hasMoreElements()) {
+            var alias = enumeration.nextElement();
+            if (keyStore.isKeyEntry(alias)) {
+                certificates.add(keyStore.getCertificate(alias));
+            }
+        }
+        return certificates;
+    }
+
+    private static KeyStore readKeyStoreFromFile(File file, String password) throws GeneralSecurityException, IOException {
+        KeyStore keyStore = KeyStore.getInstance(file.getName().endsWith(".jks") ? "JKS" : "PKCS12");
+        try (InputStream input = new FileInputStream(file)) {
+            keyStore.load(input, password == null ? null : password.toCharArray());
+        }
+        return keyStore;
+    }
+
+    private static Collection<? extends Certificate> readCertificates(File pemFile) throws GeneralSecurityException, IOException {
+        final CertificateFactory certFactory = CertificateFactory.getInstance("X.509");
+        try (InputStream input = new FileInputStream(pemFile)) {
+            return certFactory.generateCertificates(input);
+        }
+    }
+
+    private Collection<? extends Certificate> head(Collection<? extends Certificate> certificates) {
+        if (certificates.isEmpty()) {
+            return certificates;
+        } else {
+            return List.of(certificates.iterator().next());
+        }
+    }
+}
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java
index 11ad0a29f5b8..3754f57dc378 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java
@@ -9,8 +9,11 @@
 
 import org.elasticsearch.gradle.FileSystemOperationsAware;
 import org.elasticsearch.gradle.util.GradleUtils;
+import org.gradle.api.Task;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.services.internal.BuildServiceRegistryInternal;
+import org.gradle.api.specs.NotSpec;
+import org.gradle.api.specs.Spec;
 import org.gradle.api.tasks.CacheableTask;
 import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.Nested;
@@ -40,6 +43,11 @@ public class StandaloneRestIntegTestTask extends Test implements TestClustersAwa
     private boolean debugServer = false;
 
     public StandaloneRestIntegTestTask() {
+        Spec<Task> taskSpec = t -> getProject().getTasks()
+            .withType(StandaloneRestIntegTestTask.class)
+            .stream()
+            .filter(task -> task != this)
+            .anyMatch(task -> Collections.disjoint(task.getClusters(), getClusters()) == false);
         this.getOutputs()
             .doNotCacheIf(
                 "Caching disabled for this task since it uses a cluster shared by other tasks",
@@ -49,13 +57,9 @@ public StandaloneRestIntegTestTask() {
                  * avoid any undesired behavior we simply disable the cache if we detect that this task uses a cluster shared between
                  * multiple tasks.
                  */
-                t -> getProject().getTasks()
-                    .withType(StandaloneRestIntegTestTask.class)
-                    .stream()
-                    .filter(task -> task != this)
-                    .anyMatch(task -> Collections.disjoint(task.getClusters(), getClusters()) == false)
+                taskSpec
             );
-
+        this.getOutputs().upToDateWhen(new NotSpec(taskSpec));
         this.getOutputs()
             .doNotCacheIf(
                 "Caching disabled for this task since it is configured to preserve data directory",
@@ -67,11 +71,7 @@ public StandaloneRestIntegTestTask() {
     @Option(option = "debug-server-jvm", description = "Enable debugging configuration, to allow attaching a debugger to elasticsearch.")
     public void setDebugServer(boolean enabled) {
         this.debugServer = enabled;
-    }
-
-    @Override
-    public int getMaxParallelForks() {
-        return 1;
+        systemProperty("tests.cluster.debug.enabled", Boolean.toString(enabled));
     }
 
     @Nested
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java
index e6f0fd9965b6..b6ead59296eb 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java
@@ -121,6 +121,8 @@ public interface TestClusterConfiguration {
 
     List<String> getAllReadinessPortURI();
 
+    List<String> getAllRemoteAccessPortURI();
+
     void stop(boolean tailLogs);
 
     void setNameCustomization(Function<String, String> nameSupplier);
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/WaitForHttpResource.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/WaitForHttpResource.java
index 03368e74cdb7..550dd0fdcf8f 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/WaitForHttpResource.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/WaitForHttpResource.java
@@ -12,31 +12,20 @@
 import org.gradle.api.logging.Logging;
 
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.security.GeneralSecurityException;
-import java.security.KeyStore;
-import java.security.KeyStoreException;
-import java.security.SecureRandom;
-import java.security.cert.Certificate;
-import java.security.cert.CertificateFactory;
-import java.util.Arrays;
 import java.util.Base64;
 import java.util.Collections;
-import java.util.Enumeration;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 import javax.net.ssl.HttpsURLConnection;
-import javax.net.ssl.KeyManager;
 import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManagerFactory;
 
 /**
  * A utility to wait for a specific HTTP resource to be available, optionally with customized TLS trusted CAs.
@@ -47,11 +36,10 @@ public class WaitForHttpResource {
 
     private static final Logger logger = Logging.getLogger(WaitForHttpResource.class);
 
+    private final SslTrustResolver trustResolver;
+    private final URL url;
+
     private Set<Integer> validResponseCodes = Collections.singleton(200);
-    private URL url;
-    private Set<File> certificateAuthorities;
-    private File trustStoreFile;
-    private String trustStorePassword;
     private String username;
     private String password;
 
@@ -61,6 +49,7 @@ public WaitForHttpResource(String protocol, String host, int numberOfNodes) thro
 
     public WaitForHttpResource(URL url) {
         this.url = url;
+        this.trustResolver = new SslTrustResolver();
     }
 
     public void setValidResponseCodes(int... validResponseCodes) {
@@ -71,15 +60,27 @@ public void setValidResponseCodes(int... validResponseCodes) {
     }
 
     public void setCertificateAuthorities(File... certificateAuthorities) {
-        this.certificateAuthorities = new HashSet<>(Arrays.asList(certificateAuthorities));
+        trustResolver.setCertificateAuthorities(certificateAuthorities);
     }
 
     public void setTrustStoreFile(File trustStoreFile) {
-        this.trustStoreFile = trustStoreFile;
+        trustResolver.setTrustStoreFile(trustStoreFile);
     }
 
     public void setTrustStorePassword(String trustStorePassword) {
-        this.trustStorePassword = trustStorePassword;
+        trustResolver.setTrustStorePassword(trustStorePassword);
+    }
+
+    public void setServerCertificate(File serverCertificate) {
+        trustResolver.setServerCertificate(serverCertificate);
+    }
+
+    public void setServerKeystoreFile(File keyStoreFile) {
+        trustResolver.setServerKeystoreFile(keyStoreFile);
+    }
+
+    public void setServerKeystorePassword(String keyStorePassword) {
+        trustResolver.setServerKeystorePassword(keyStorePassword);
     }
 
     public void setUsername(String username) {
@@ -94,13 +95,7 @@ public boolean wait(int durationInMs) throws GeneralSecurityException, Interrupt
         final long waitUntil = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(durationInMs);
         final long sleep = Long.max(durationInMs / 10, 100);
 
-        final SSLContext ssl;
-        final KeyStore trustStore = buildTrustStore();
-        if (trustStore != null) {
-            ssl = createSslContext(trustStore);
-        } else {
-            ssl = null;
-        }
+        final SSLContext ssl = trustResolver.getSslContext();
         IOException failure = null;
         while (true) {
             try {
@@ -159,61 +154,4 @@ private void configureBasicAuth(HttpURLConnection connection) {
             );
         }
     }
-
-    KeyStore buildTrustStore() throws GeneralSecurityException, IOException {
-        if (this.certificateAuthorities != null) {
-            if (trustStoreFile != null) {
-                throw new IllegalStateException("Cannot specify both truststore and CAs");
-            }
-            return buildTrustStoreFromCA();
-        } else if (trustStoreFile != null) {
-            return buildTrustStoreFromFile();
-        } else {
-            return null;
-        }
-    }
-
-    private KeyStore buildTrustStoreFromFile() throws GeneralSecurityException, IOException {
-        KeyStore keyStore = KeyStore.getInstance(trustStoreFile.getName().endsWith(".jks") ? "JKS" : "PKCS12");
-        try (InputStream input = new FileInputStream(trustStoreFile)) {
-            keyStore.load(input, trustStorePassword == null ? null : trustStorePassword.toCharArray());
-        }
-        return keyStore;
-    }
-
-    private KeyStore buildTrustStoreFromCA() throws GeneralSecurityException, IOException {
-        final KeyStore store = KeyStore.getInstance(KeyStore.getDefaultType());
-        store.load(null, null);
-        final CertificateFactory certFactory = CertificateFactory.getInstance("X.509");
-        int counter = 0;
-        for (File ca : certificateAuthorities) {
-            try (InputStream input = new FileInputStream(ca)) {
-                for (Certificate certificate : certFactory.generateCertificates(input)) {
-                    store.setCertificateEntry("cert-" + counter, certificate);
-                    counter++;
-                }
-            }
-        }
-        return store;
-    }
-
-    private SSLContext createSslContext(KeyStore trustStore) throws GeneralSecurityException {
-        checkForTrustEntry(trustStore);
-        TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
-        tmf.init(trustStore);
-        SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
-        sslContext.init(new KeyManager[0], tmf.getTrustManagers(), new SecureRandom());
-        return sslContext;
-    }
-
-    private void checkForTrustEntry(KeyStore trustStore) throws KeyStoreException {
-        Enumeration<String> enumeration = trustStore.aliases();
-        while (enumeration.hasMoreElements()) {
-            if (trustStore.isCertificateEntry(enumeration.nextElement())) {
-                // found trusted cert entry
-                return;
-            }
-        }
-        throw new IllegalStateException("Trust-store does not contain any trusted certificate entries");
-    }
 }
diff --git a/build-tools/src/test/groovy/org/elasticsearch/gradle/plugin/scanner/AnnotatedHierarchyVisitorSpec.groovy b/build-tools/src/test/groovy/org/elasticsearch/gradle/plugin/scanner/AnnotatedHierarchyVisitorSpec.groovy
deleted file mode 100644
index 732fa1c5e0ac..000000000000
--- a/build-tools/src/test/groovy/org/elasticsearch/gradle/plugin/scanner/AnnotatedHierarchyVisitorSpec.groovy
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1 you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.gradle.plugin.scanner
-
-import spock.lang.Specification
-
-import org.elasticsearch.plugin.api.NamedComponent
-import org.elasticsearch.plugin.scanner.test_classes.ExtensibleClass
-import org.elasticsearch.plugin.scanner.test_classes.ExtensibleInterface
-import org.elasticsearch.plugin.scanner.test_classes.ImplementingExtensible
-import org.elasticsearch.plugin.scanner.test_classes.SubClass
-import org.elasticsearch.plugin.api.Extensible
-import org.objectweb.asm.ClassReader
-import org.objectweb.asm.Type
-
-import java.nio.file.Files
-import java.nio.file.Path
-import java.nio.file.Paths
-
-class AnnotatedHierarchyVisitorSpec extends Specification {
-    Set<String> foundClasses
-    AnnotatedHierarchyVisitor visitor
-
-    def setup() {
-        foundClasses = new HashSet<>()
-        visitor =
-            new AnnotatedHierarchyVisitor(
-                Type.getDescriptor(Extensible.class), (className) -> {
-                foundClasses.add(className)
-                return null
-            }
-            )
-    }
-
-    def "empty result when no classes annotated"() {
-        when:
-        performScan(visitor, NamedComponent.class)
-
-        then:
-        foundClasses.empty
-    }
-
-    def "single class found when only one is annotated"() {
-        when:
-        performScan(visitor, ExtensibleClass.class)
-
-        then:
-        foundClasses == [classNameToPath(ExtensibleClass.class)] as Set
-    }
-
-    def "class extending an extensible is also found"() {
-        when:
-        performScan(visitor, ExtensibleClass.class, SubClass.class)
-
-        then:
-        foundClasses == [classNameToPath(ExtensibleClass.class)] as Set
-        visitor.getClassHierarchy() == [(classNameToPath(ExtensibleClass.class)) : [classNameToPath(SubClass.class)] as Set]
-    }
-
-    def "interface extending an extensible is also found"() {
-        when:
-        performScan(visitor, ImplementingExtensible.class, ExtensibleInterface.class)
-
-        then:
-        foundClasses == [classNameToPath(ExtensibleInterface.class)] as Set
-        visitor.getClassHierarchy() ==
-            [(classNameToPath(ExtensibleInterface.class)) : [classNameToPath(ImplementingExtensible.class)] as Set]
-    }
-
-    private String classNameToPath(Class<?> clazz) {
-        return clazz.getCanonicalName().replace(".", "/")
-    }
-
-    private void performScan(AnnotatedHierarchyVisitor classVisitor, Class<?>... classes) throws IOException, URISyntaxException {
-        for (Class<?> clazz : classes) {
-            String className = classNameToPath(clazz) + ".class"
-            def stream = this.getClass().getClassLoader().getResourceAsStream(className)
-            try (InputStream fileInputStream = stream) {
-                ClassReader cr = new ClassReader(fileInputStream)
-                cr.accept(classVisitor, 0)
-            }
-        }
-    }
-
-}
diff --git a/build-tools/src/test/groovy/org/elasticsearch/gradle/plugin/scanner/ClassReadersSpec.groovy b/build-tools/src/test/groovy/org/elasticsearch/gradle/plugin/scanner/ClassReadersSpec.groovy
deleted file mode 100644
index bc448efd2d75..000000000000
--- a/build-tools/src/test/groovy/org/elasticsearch/gradle/plugin/scanner/ClassReadersSpec.groovy
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.gradle.plugin.scanner
-
-import spock.lang.Specification
-
-import org.elasticsearch.gradle.internal.test.InMemoryJavaCompiler;
-
-import org.elasticsearch.gradle.internal.test.JarUtils
-import org.hamcrest.Matchers
-import org.junit.Rule
-import org.junit.rules.TemporaryFolder
-import org.objectweb.asm.ClassReader
-
-import java.nio.file.Files
-import java.nio.file.Path
-import java.util.stream.Collectors
-import java.util.stream.Stream
-
-import static org.hamcrest.MatcherAssert.assertThat
-
-class ClassReadersSpec extends Specification {
-    @Rule
-    TemporaryFolder testProjectDir = new TemporaryFolder()
-
-    private Path tmpDir() throws IOException {
-        return testProjectDir.root.toPath();
-    }
-
-    def "module-info is not returned as a class from jar"() {
-        when:
-        final Path tmp = tmpDir();
-        final Path dirWithJar = tmp.resolve("jars-dir");
-        Files.createDirectories(dirWithJar);
-        Path jar = dirWithJar.resolve("api.jar");
-        JarUtils.createJarWithEntries(
-            jar, Map.of(
-            "module-info.class", InMemoryJavaCompiler.compile(
-            "module-info", """
-            module p {}
-            """)
-            )
-        )
-
-
-        then:
-        try (Stream<ClassReader> classReaderStream = ClassReaders.ofPaths(Stream.of(jar))) {
-
-            assertThat(classReaderStream.collect(Collectors.toList()), Matchers.empty());
-        }
-    }
-
-
-    def "two classes are returned in a stream from jar"() {
-        when:
-        final Path tmp = tmpDir();
-        final Path dirWithJar = tmp.resolve("jars-dir");
-        Files.createDirectories(dirWithJar);
-        Path jar = dirWithJar.resolve("api.jar");
-        JarUtils.createJarWithEntries(
-            jar, Map.of(
-            "p/A.class", InMemoryJavaCompiler.compile(
-            "p.A", """
-            package p;
-            public class A {}
-            """),
-        "p/B.class", InMemoryJavaCompiler.compile(
-            "p.B", """
-            package p;
-            public class B {}
-            """)
-            )
-        );
-
-
-        then:
-        try (Stream<ClassReader> classReaderStream = ClassReaders.ofPaths(Stream.of(jar))) {
-            List<String> collect = classReaderStream.map(cr -> cr.getClassName()).collect(Collectors.toList());
-            assertThat(collect, Matchers.containsInAnyOrder("p/A", "p/B"));
-        }
-    }
-
-
-    def "on a classpath jars and individual classes are returned"() {
-        when:
-        final Path tmp = tmpDir();
-        final Path dirWithJar = tmp.resolve("jars-dir");
-        Files.createDirectories(dirWithJar);
-
-        Path jar = dirWithJar.resolve("a_b.jar");
-        JarUtils.createJarWithEntries(
-            jar, Map.of(
-            "p/A.class", InMemoryJavaCompiler.compile(
-            "p.A", """
-            package p;
-            public class A {}
-            """),
-        "p/B.class", InMemoryJavaCompiler.compile(
-            "p.B", """
-            package p;
-            public class B {}
-            """)
-            )
-        );
-
-        Path jar2 = dirWithJar.resolve("c_d.jar");
-        JarUtils.createJarWithEntries(
-            jar2, Map.of(
-            "p/C.class", InMemoryJavaCompiler.compile(
-            "p.C", """
-            package p;
-            public class C {}
-            """),
-        "p/D.class", InMemoryJavaCompiler.compile(
-            "p.D", """
-            package p;
-            public class D {}
-            """)
-            )
-        );
-
-        InMemoryJavaCompiler.compile(
-            "p.E", """
-            package p;
-            public class E {}
-            """
-        );
-        Files.write(
-            tmp.resolve("E.class"), InMemoryJavaCompiler.compile(
-            "p.E", """
-            package p;
-            public class E {}
-            """)
-        );
-
-
-        then:
-        try (Stream<ClassReader> classReaderStream = ClassReaders.ofPaths(Stream.of(tmp, jar, jar2))) {
-
-            List<String> collect = classReaderStream.map(cr -> cr.getClassName()).collect(Collectors.toList());
-            assertThat(collect, Matchers.containsInAnyOrder("p/A", "p/B", "p/C", "p/D", "p/E"));
-        }
-    }
-
-    def "classes from multiple jars in a dir are returned"() {
-        when:
-        final Path tmp = tmpDir();
-        final Path dirWithJar = tmp.resolve("jars-dir");
-        Files.createDirectories(dirWithJar);
-
-
-        Path jar = dirWithJar.resolve("a_b.jar");
-        JarUtils.createJarWithEntries(
-            jar, Map.of(
-            "p/A.class", InMemoryJavaCompiler.compile(
-            "p.A", """
-            package p;
-            public class A {}
-            """),
-        "p/B.class", InMemoryJavaCompiler.compile(
-            "p.B", """
-            package p;
-            public class B {}
-            """)
-            )
-        );
-
-        Path jar2 = dirWithJar.resolve("c_d.jar");
-        JarUtils.createJarWithEntries(
-            jar2, Map.of(
-            "p/C.class", InMemoryJavaCompiler.compile(
-            "p.C", """
-            package p;
-            public class C {}
-            """),
-        "p/D.class", InMemoryJavaCompiler.compile(
-            "p.D", """
-            package p;
-            public class D {}
-            """)
-            )
-        );
-
-        then:
-        try (Stream<ClassReader> classReaderStream = ClassReaders.ofDirWithJars(dirWithJar.toString())) {
-            List<String> collect = classReaderStream.map(cr -> cr.getClassName()).collect(Collectors.toList());
-            assertThat(collect, Matchers.containsInAnyOrder("p/A", "p/B", "p/C", "p/D"));
-        }
-    }
-}
diff --git a/build-tools/src/test/groovy/org/elasticsearch/gradle/plugin/scanner/ClassScannerSpec.groovy b/build-tools/src/test/groovy/org/elasticsearch/gradle/plugin/scanner/ClassScannerSpec.groovy
deleted file mode 100644
index d03ae28576ae..000000000000
--- a/build-tools/src/test/groovy/org/elasticsearch/gradle/plugin/scanner/ClassScannerSpec.groovy
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.gradle.plugin.scanner
-
-import spock.lang.Specification
-
-import org.elasticsearch.plugin.api.Extensible
-import org.hamcrest.Matchers
-import org.objectweb.asm.ClassReader
-import org.objectweb.asm.Type
-
-import java.nio.file.Paths
-import java.util.stream.Collectors
-import java.util.stream.Stream
-
-import static org.hamcrest.MatcherAssert.assertThat
-
-class ClassScannerSpec extends Specification {
-    static final System.Logger logger = System.getLogger(ClassScannerSpec.class.getName())
-    def "class and interface hierarchy is scanned"() {
-        given:
-        def reader = new ClassScanner(
-            Type.getDescriptor(Extensible.class), (classname, map) -> {
-            map.put(classname, classname)
-            return null
-        }
-        )
-        Stream<ClassReader> classReaderStream = ofClassPath()
-        logger.log(System.Logger.Level.INFO, "classReaderStream size "+ofClassPath().collect(Collectors.toList()).size())
-
-        when:
-        reader.visit(classReaderStream);
-        Map<String, String> extensibleClasses = reader.getFoundClasses()
-
-        then:
-        assertThat(
-            extensibleClasses,
-            Matchers.allOf(
-                Matchers.hasEntry(
-                    "org/elasticsearch/plugin/scanner/test_classes/ExtensibleClass",
-                    "org/elasticsearch/plugin/scanner/test_classes/ExtensibleClass"
-                ),
-                Matchers.hasEntry(
-                    "org/elasticsearch/plugin/scanner/test_classes/ImplementingExtensible",
-                    "org/elasticsearch/plugin/scanner/test_classes/ExtensibleInterface"
-                ),
-                Matchers.hasEntry(
-                    "org/elasticsearch/plugin/scanner/test_classes/SubClass",
-                    "org/elasticsearch/plugin/scanner/test_classes/ExtensibleClass"
-                )
-            )
-        );
-    }
-
-    static Stream<ClassReader> ofClassPath() throws IOException {
-        String classpath = System.getProperty("java.class.path");
-        logger.log(System.Logger.Level.INFO, "classpath "+classpath);
-        return ofClassPath(classpath);
-    }
-
-    static Stream<ClassReader> ofClassPath(String classpath) {
-        if (classpath != null && classpath.equals("") == false) {// todo when do we set cp to "" ?
-            def classpathSeparator = System.getProperty("path.separator")
-            logger.log(System.Logger.Level.INFO, "classpathSeparator "+classpathSeparator);
-
-            String[] pathelements = classpath.split(classpathSeparator);
-            return ClassReaders.ofPaths(Arrays.stream(pathelements).map(Paths::get));
-        }
-        return Stream.empty();
-    }
-
-}
diff --git a/build-tools/src/test/groovy/org/elasticsearch/gradle/plugin/scanner/NamedComponentScannerSpec.groovy b/build-tools/src/test/groovy/org/elasticsearch/gradle/plugin/scanner/NamedComponentScannerSpec.groovy
deleted file mode 100644
index fc48ae6a0ad6..000000000000
--- a/build-tools/src/test/groovy/org/elasticsearch/gradle/plugin/scanner/NamedComponentScannerSpec.groovy
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.plugin.scanner
-
-import net.bytebuddy.ByteBuddy
-import net.bytebuddy.dynamic.DynamicType
-import spock.lang.Specification
-
-import org.elasticsearch.gradle.internal.test.InMemoryJavaCompiler
-import org.elasticsearch.gradle.internal.test.JarUtils
-import org.elasticsearch.gradle.internal.test.StableApiJarMocks
-import org.elasticsearch.gradle.plugin.scanner.ClassReaders
-import org.elasticsearch.gradle.plugin.scanner.NamedComponentScanner
-import org.elasticsearch.plugin.scanner.test_classes.ExtensibleClass
-import org.elasticsearch.plugin.scanner.test_classes.ExtensibleInterface
-import org.elasticsearch.plugin.scanner.test_classes.TestNamedComponent
-import org.elasticsearch.plugin.api.Extensible
-import org.elasticsearch.plugin.api.NamedComponent
-import org.junit.Rule
-import org.junit.rules.TemporaryFolder
-import org.objectweb.asm.ClassReader
-
-import java.nio.file.Files
-import java.nio.file.Path
-import java.nio.file.Paths
-import java.util.stream.Collectors
-
-import static org.hamcrest.MatcherAssert.assertThat
-import static org.hamcrest.Matchers.equalTo
-
-class NamedComponentScannerSpec extends Specification {
-    @Rule
-    TemporaryFolder testProjectDir = new TemporaryFolder()
-
-    private Path tmpDir() throws IOException {
-        return testProjectDir.root.toPath();
-    }
-
-    NamedComponentScanner namedComponentScanner = new NamedComponentScanner();
-
-    def "named component is found when single class provided"() {
-        when:
-        Map<String, Map<String, String>> namedComponents = namedComponentScanner.scanForNamedClasses(
-            classReaderStream(TestNamedComponent.class, ExtensibleInterface.class)
-        )
-
-        then:
-        assertThat(
-            namedComponents,
-            equalTo(
-                Map.of(
-                    ExtensibleInterface.class.getCanonicalName(),
-                    Map.of("test_named_component", TestNamedComponent.class.getCanonicalName())
-                )
-            )
-        )
-
-    }
-
-    def "named components are found when single jar provided"() {
-        given:
-        final Path tmp = tmpDir();
-        final Path dirWithJar = tmp.resolve("jars-dir");
-        Files.createDirectories(dirWithJar);
-        Path jar = dirWithJar.resolve("plugin.jar");
-        JarUtils.createJarWithEntries(
-            jar, Map.of(
-            "p/A.class", InMemoryJavaCompiler.compile(
-            "p.A", """
-            package p;
-            import org.elasticsearch.plugin.api.*;
-            import org.elasticsearch.plugin.scanner.test_classes.*;
-            @NamedComponent("a_component")
-            public class A extends ExtensibleClass {}
-            """
-        ), "p/B.class", InMemoryJavaCompiler.compile(
-            "p.B", """
-            package p;
-            import org.elasticsearch.plugin.api.*;
-            import org.elasticsearch.plugin.scanner.test_classes.*;
-            @NamedComponent("b_component")
-            public class B implements ExtensibleInterface{}
-            """
-        )
-        )
-        );
-        StableApiJarMocks.createPluginApiJar(dirWithJar);
-        StableApiJarMocks.createExtensibleApiJar(dirWithJar);//for instance analysis api
-
-
-        Collection<ClassReader> classReaderStream = ClassReaders.ofDirWithJars(dirWithJar.toString()).collect(Collectors.toList())
-
-        when:
-        Map<String, Map<String, String>> namedComponents = namedComponentScanner.scanForNamedClasses(classReaderStream);
-
-        then:
-        assertThat(
-            namedComponents,
-            equalTo(
-                Map.of(
-                    ExtensibleClass.class.getCanonicalName(),
-                    Map.of("a_component", "p.A"),
-                    ExtensibleInterface.class.getCanonicalName(),
-                    Map.of("b_component", "p.B")
-                )
-            )
-        );
-    }
-
-    def "named components can extend common super class"() {
-        given:
-        Map<String, CharSequence> sources = Map.of(
-            "p.CustomExtensibleInterface",
-            """
-                package p;
-                import org.elasticsearch.plugin.api.*;
-                import org.elasticsearch.plugin.scanner.test_classes.*;
-                public interface CustomExtensibleInterface extends ExtensibleInterface {}
-                """,
-            // note that this class implements a custom interface
-            "p.CustomExtensibleClass",
-            """
-                package p;
-                import org.elasticsearch.plugin.api.*;
-                import org.elasticsearch.plugin.scanner.test_classes.*;
-                public class CustomExtensibleClass implements CustomExtensibleInterface {}
-                """,
-            "p.A",
-            """
-                package p;
-                import org.elasticsearch.plugin.api.*;
-                import org.elasticsearch.plugin.scanner.test_classes.*;
-                @NamedComponent("a_component")
-                public class A extends CustomExtensibleClass {}
-                """,
-            "p.B",
-            """
-                package p;
-                import org.elasticsearch.plugin.api.*;
-                import org.elasticsearch.plugin.scanner.test_classes.*;
-                @NamedComponent("b_component")
-                public class B implements CustomExtensibleInterface{}
-                """
-        );
-        var classToBytes = InMemoryJavaCompiler.compile(sources);
-
-        Map<String, byte[]> jarEntries = new HashMap<>();
-        jarEntries.put("p/CustomExtensibleInterface.class", classToBytes.get("p.CustomExtensibleInterface"));
-        jarEntries.put("p/CustomExtensibleClass.class", classToBytes.get("p.CustomExtensibleClass"));
-        jarEntries.put("p/A.class", classToBytes.get("p.A"));
-        jarEntries.put("p/B.class", classToBytes.get("p.B"));
-
-        final Path tmp = tmpDir();
-        final Path dirWithJar = tmp.resolve("jars-dir");
-        Files.createDirectories(dirWithJar);
-        Path jar = dirWithJar.resolve("plugin.jar");
-        JarUtils.createJarWithEntries(jar, jarEntries);
-
-        StableApiJarMocks.createPluginApiJar(dirWithJar)
-        StableApiJarMocks.createExtensibleApiJar(dirWithJar);//for instance analysis api
-
-        Collection<ClassReader> classReaderStream = ClassReaders.ofDirWithJars(dirWithJar.toString()).collect(Collectors.toList())
-
-        when:
-        Map<String, Map<String, String>> namedComponents = namedComponentScanner.scanForNamedClasses(classReaderStream);
-
-        then:
-        assertThat(
-            namedComponents,
-            equalTo(
-                Map.of(
-                    ExtensibleInterface.class.getCanonicalName(),
-                    Map.of(
-                        "a_component", "p.A",
-                        "b_component", "p.B"
-                    )
-                )
-            )
-        );
-    }
-
-
-
-    private Collection<ClassReader> classReaderStream(Class<?>... classes) {
-            try {
-                return Arrays.stream(classes).map(
-                    clazz -> {
-                        String className = classNameToPath(clazz) + ".class";
-                        def stream = this.getClass().getClassLoader().getResourceAsStream(className)
-                        try (InputStream is = stream) {
-                            byte[] classBytes = is.readAllBytes();
-                            ClassReader classReader = new ClassReader(classBytes);
-                            return classReader;
-                        } catch (IOException e) {
-                            throw new UncheckedIOException(e);
-                        }
-                    }
-                ).collect(Collectors.toList())
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-
-    }
-
-    private String classNameToPath(Class<?> clazz) {
-        return clazz.getCanonicalName().replace(".", "/");
-    }
-
-
-}
diff --git a/build-tools/src/test/groovy/org/elasticsearch/gradle/testclusters/SslTrustResolverSpec.groovy b/build-tools/src/test/groovy/org/elasticsearch/gradle/testclusters/SslTrustResolverSpec.groovy
new file mode 100644
index 000000000000..f1a87cd9ff26
--- /dev/null
+++ b/build-tools/src/test/groovy/org/elasticsearch/gradle/testclusters/SslTrustResolverSpec.groovy
@@ -0,0 +1,117 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.testclusters
+
+import spock.lang.Specification
+
+import java.nio.file.Paths
+import java.security.cert.Certificate
+import java.security.cert.CertificateException
+import java.security.cert.CertificateFactory
+import javax.net.ssl.TrustManager
+import javax.net.ssl.X509TrustManager
+
+class SslTrustResolverSpec extends Specification {
+
+    def "build trust manager from trust store file"() {
+        given:
+        SslTrustResolver resolver = new SslTrustResolver()
+        URL ca = getClass().getResource("/ca.p12")
+
+        when:
+        resolver.setTrustStoreFile(Paths.get(ca.toURI()).toFile())
+        resolver.setTrustStorePassword("password")
+        final TrustManager[] trustManagers = resolver.buildTrustManagers()
+
+        then:
+        trustManagers.length == 1
+        trustManagers[0] instanceof X509TrustManager
+        def issuers = ((X509TrustManager) trustManagers[0]).getAcceptedIssuers()
+        issuers.length == 1
+        issuers[0].subjectX500Principal.toString() == 'CN=Elastic Certificate Tool Autogenerated CA'
+    }
+
+    def "build trust manager from certificate authorities file"() {
+        given:
+        SslTrustResolver resolver = new SslTrustResolver()
+        URL ca = getClass().getResource("/ca.pem")
+
+        when:
+        resolver.setCertificateAuthorities(Paths.get(ca.toURI()).toFile())
+        final TrustManager[] trustManagers = resolver.buildTrustManagers()
+
+        then:
+        trustManagers.length == 1
+        trustManagers[0] instanceof X509TrustManager
+        def issuers = ((X509TrustManager) trustManagers[0]).getAcceptedIssuers()
+        issuers.length == 1
+        issuers[0].subjectX500Principal.toString() == 'CN=Elastic Certificate Tool Autogenerated CA'
+    }
+
+    def "build trust manager from keystore file"() {
+        given:
+        SslTrustResolver resolver = new SslTrustResolver()
+        URL ks = getClass().getResource("/server.p12")
+        Certificate[] serverChain = readCertificates("/server.chain")
+        Certificate[] issuingChain = readCertificates("/issuing.pem")
+        Certificate[] altChain = readCertificates("/ca.pem")
+
+        when:
+        resolver.setServerKeystoreFile(Paths.get(ks.toURI()).toFile())
+        resolver.setServerKeystorePassword("password")
+        final TrustManager[] trustManagers = resolver.buildTrustManagers()
+
+        then:
+        trustManagers.length == 1
+        trustManagers[0] instanceof X509TrustManager
+
+        def trustManager = (X509TrustManager) trustManagers[0]
+        isTrusted(trustManager, serverChain) == true;
+        isTrusted(trustManager, issuingChain) == false;
+        isTrusted(trustManager, altChain) == false;
+    }
+
+    def "build trust manager from server certificate file"() {
+        given:
+        SslTrustResolver resolver = new SslTrustResolver()
+        URL chain = getClass().getResource("/server.chain")
+        Certificate[] serverChain = readCertificates("/server.chain")
+        Certificate[] issuingChain = readCertificates("/issuing.pem")
+        Certificate[] altChain = readCertificates("/ca.pem")
+
+        when:
+        resolver.setServerCertificate(Paths.get(chain.toURI()).toFile())
+        final TrustManager[] trustManagers = resolver.buildTrustManagers()
+
+        then:
+        trustManagers.length == 1
+        trustManagers[0] instanceof X509TrustManager
+
+        def trustManager = (X509TrustManager) trustManagers[0]
+        isTrusted(trustManager, serverChain) == true;
+        isTrusted(trustManager, issuingChain) == false;
+        isTrusted(trustManager, altChain) == false;
+    }
+
+    private Certificate[] readCertificates(String resourceName) {
+        CertificateFactory certificateFactory = CertificateFactory.getInstance("X.509")
+        return getClass().getResource(resourceName).withInputStream { stream ->
+            certificateFactory.generateCertificates(stream)
+        }
+    }
+
+    private boolean isTrusted(X509TrustManager trustManager, Certificate[] certificateChain) {
+        try {
+            trustManager.checkServerTrusted(((java.security.cert.X509Certificate[]) certificateChain), "RSA");
+            return true;
+        } catch(CertificateException ignore) {
+            return false;
+        }
+    }
+}
diff --git a/build-tools/src/test/groovy/org/elasticsearch/gradle/testclusters/WaitForHttpResourceSpec.groovy b/build-tools/src/test/groovy/org/elasticsearch/gradle/testclusters/WaitForHttpResourceSpec.groovy
deleted file mode 100644
index ffa8b569e3e9..000000000000
--- a/build-tools/src/test/groovy/org/elasticsearch/gradle/testclusters/WaitForHttpResourceSpec.groovy
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.gradle.testclusters
-
-import spock.lang.Specification
-
-import java.nio.file.Paths
-import java.security.KeyStore
-import java.security.cert.Certificate
-import java.security.cert.X509Certificate
-
-class WaitForHttpResourceSpec extends Specification {
-
-    def "build trust store from trust store file"() {
-        given:
-        WaitForHttpResource http = new WaitForHttpResource(new URL("https://localhost/"))
-        URL ca = getClass().getResource("/ca.p12")
-
-        when:
-        http.setTrustStoreFile(Paths.get(ca.toURI()).toFile())
-        http.setTrustStorePassword("password")
-        final KeyStore store = http.buildTrustStore()
-        final Certificate certificate = store.getCertificate("ca")
-
-        then:
-        certificate != null
-        certificate instanceof X509Certificate
-        certificate.subjectX500Principal.toString() == 'CN=Elastic Certificate Tool Autogenerated CA'
-    }
-
-    def "build trust store from certificate authorities file"() {
-        given:
-        WaitForHttpResource http = new WaitForHttpResource(new URL("https://localhost/"))
-        URL ca = getClass().getResource("/ca.pem")
-
-        when:
-        http.setCertificateAuthorities(Paths.get(ca.toURI()).toFile())
-        KeyStore store = http.buildTrustStore()
-        Certificate certificate = store.getCertificate("cert-0")
-
-        then:
-        certificate != null
-        certificate instanceof X509Certificate
-        certificate.subjectX500Principal.toString() == "CN=Elastic Certificate Tool Autogenerated CA"
-    }
-}
diff --git a/build-tools/src/test/resources/issuing.p12 b/build-tools/src/test/resources/issuing.p12
new file mode 100644
index 000000000000..85118080aacd
Binary files /dev/null and b/build-tools/src/test/resources/issuing.p12 differ
diff --git a/build-tools/src/test/resources/issuing.pem b/build-tools/src/test/resources/issuing.pem
new file mode 100644
index 000000000000..49d1f9d95052
--- /dev/null
+++ b/build-tools/src/test/resources/issuing.pem
@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIICBjCCAW+gAwIBAgIUY8ypR5RTcCCoHom3Mvq3Cl4MWggwDQYJKoZIhvcNAQEL
+BQAwFTETMBEGA1UEAxMKSXNzdWluZyBDQTAeFw0yMzAxMDQwNjE3MDlaFw0zNjA5
+MTIwNjE3MDlaMBUxEzARBgNVBAMTCklzc3VpbmcgQ0EwgZ8wDQYJKoZIhvcNAQEB
+BQADgY0AMIGJAoGBAKta2elMeFOjCoMaaT3XiQh4bcs8FfG2x96puWu8vy184qBp
+26JDizlpJROx8WmE1hZEoexlagltYZ9bQrNyxrXDFbFw6Ccg+0o8RZmLzJE+v5Th
+vE2ezzGcDDlb+gvFKTmdTiunI1G6y+5qe03YTtJniT5wOSUMOahRf2qc4p/DAgMB
+AAGjUzBRMB0GA1UdDgQWBBSC0hVfH3oNVV5ZPM0fxfMCfVTK0DAfBgNVHSMEGDAW
+gBSC0hVfH3oNVV5ZPM0fxfMCfVTK0DAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3
+DQEBCwUAA4GBAFKZ7KV9XP5Q5baRk3wgvpejBEpLf438icvpxU++WU8neX1vc9qK
+ZNbivyaVG1JthobV9A/lS9pNSlJtGvTjpp8Aq2uM0V2LDaLeNQ/sMrKWpsvYZkKU
+kIkbrBgUHm/zHfuhRhS3X1siKa7muH+T3bH//Vc3x6lO8nfX1iz6XHcy
+-----END CERTIFICATE-----
diff --git a/build-tools/src/test/resources/server.chain b/build-tools/src/test/resources/server.chain
new file mode 100644
index 000000000000..07da615d580a
--- /dev/null
+++ b/build-tools/src/test/resources/server.chain
@@ -0,0 +1,27 @@
+-----BEGIN CERTIFICATE-----
+MIICDjCCAXegAwIBAgIVAO17tpEpOGKg8EH4BIKpYMlPkwtLMA0GCSqGSIb3DQEB
+CwUAMBUxEzARBgNVBAMTCklzc3VpbmcgQ0EwHhcNMjMwMTA0MDYxOTMyWhcNMzYw
+OTEyMDYxOTMyWjARMQ8wDQYDVQQDEwZzZXJ2ZXIwgZ8wDQYJKoZIhvcNAQEBBQAD
+gY0AMIGJAoGBAMQkTJQP3ust+YERFxqjm6Ck+w4ql9Dc7L87mkdJhY7kVrv8hVb5
+UB+UjNYpO/hBKKE3sAqenuni/XfNRHbmVq+c3xNgJxkr9JGOzIf76TBDEORlVe99
+uViqQb+WwGcO+qY7NN5WcNKd8W779Y6CENPlMiyRzdEmgf4AnwmxOTs7AgMBAAGj
+XjBcMB0GA1UdDgQWBBRj8cuUKTNtKJvhy7mMganBGg5RCjAfBgNVHSMEGDAWgBSC
+0hVfH3oNVV5ZPM0fxfMCfVTK0DAPBgNVHREECDAGhwR/AAABMAkGA1UdEwQCMAAw
+DQYJKoZIhvcNAQELBQADgYEASox5Lk7otOz85+G0qQOJyortpOTyxZgJJ5RI3RO5
+7d59vMabbktdj+750fL50G07oeACip6kyglQqAE+I6UPQDepTpvoEv3OxDvilehX
+HCq2f0rKFnFB8ueT9Qx6tzAz1Luz764KU4AzP4rBcJicMZW1cmGsXD3/TDDEE+qJ
+YEY=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIICBjCCAW+gAwIBAgIUY8ypR5RTcCCoHom3Mvq3Cl4MWggwDQYJKoZIhvcNAQEL
+BQAwFTETMBEGA1UEAxMKSXNzdWluZyBDQTAeFw0yMzAxMDQwNjE3MDlaFw0zNjA5
+MTIwNjE3MDlaMBUxEzARBgNVBAMTCklzc3VpbmcgQ0EwgZ8wDQYJKoZIhvcNAQEB
+BQADgY0AMIGJAoGBAKta2elMeFOjCoMaaT3XiQh4bcs8FfG2x96puWu8vy184qBp
+26JDizlpJROx8WmE1hZEoexlagltYZ9bQrNyxrXDFbFw6Ccg+0o8RZmLzJE+v5Th
+vE2ezzGcDDlb+gvFKTmdTiunI1G6y+5qe03YTtJniT5wOSUMOahRf2qc4p/DAgMB
+AAGjUzBRMB0GA1UdDgQWBBSC0hVfH3oNVV5ZPM0fxfMCfVTK0DAfBgNVHSMEGDAW
+gBSC0hVfH3oNVV5ZPM0fxfMCfVTK0DAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3
+DQEBCwUAA4GBAFKZ7KV9XP5Q5baRk3wgvpejBEpLf438icvpxU++WU8neX1vc9qK
+ZNbivyaVG1JthobV9A/lS9pNSlJtGvTjpp8Aq2uM0V2LDaLeNQ/sMrKWpsvYZkKU
+kIkbrBgUHm/zHfuhRhS3X1siKa7muH+T3bH//Vc3x6lO8nfX1iz6XHcy
+-----END CERTIFICATE-----
diff --git a/build-tools/src/test/resources/server.p12 b/build-tools/src/test/resources/server.p12
new file mode 100644
index 000000000000..95674e0fe78e
Binary files /dev/null and b/build-tools/src/test/resources/server.p12 differ
diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/TestClasspathUtils.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/TestClasspathUtils.groovy
index 94e012c95021..e41ace83b75b 100644
--- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/TestClasspathUtils.groovy
+++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/TestClasspathUtils.groovy
@@ -16,17 +16,40 @@ import net.bytebuddy.dynamic.DynamicType
 import net.bytebuddy.implementation.ExceptionMethod
 import net.bytebuddy.implementation.FixedValue
 import net.bytebuddy.implementation.Implementation
+import net.bytebuddy.implementation.MethodDelegation
+
+import org.elasticsearch.gradle.NamedComponentScannerMock
 
 import static org.junit.Assert.fail
 
 class TestClasspathUtils {
 
+    // we cannot access real NamedComponentScanner in libs:plugin-api-scanner so we create a fake class
+    static void setupNamedComponentScanner(File projectRoot, String version) {
+        def value = MethodDelegation.to(NamedComponentScannerMock.class)
+
+        DynamicType.Unloaded<?> dynamicType = new ByteBuddy().subclass(Object.class)
+            .name("org.elasticsearch.plugin.scanner.NamedComponentScanner")
+            .defineMethod("main", void.class, Visibility.PUBLIC, Ownership.STATIC)
+            .withParameters(String[].class)
+            .intercept(value)
+            .make()
+            .include(new ByteBuddy().redefine(NamedComponentScannerMock.class).make())
+
+        try {
+            dynamicType.toJar(targetFile(projectRoot, "elasticsearch-plugin-scanner", version))
+        } catch (IOException e) {
+            e.printStackTrace()
+            fail("Cannot setup jdk jar hell classpath")
+        }
+    }
+
     static void setupJarHellJar(File projectRoot) {
-        generateJdkJarHellCheck(projectRoot, "org.elasticsearch.jdk.JarHell", "current", FixedValue.value(TypeDescription.VOID))
+        generateJarWithClass(projectRoot, "org.elasticsearch.jdk.JarHell","elasticsearch-core", "current", FixedValue.value(TypeDescription.VOID))
     }
 
     static void setupJarHellJar(File projectRoot, String version) {
-        generateJdkJarHellCheck(projectRoot, "org.elasticsearch.jdk.JarHell", version, FixedValue.value(TypeDescription.VOID))
+        generateJarWithClass(projectRoot, "org.elasticsearch.jdk.JarHell", "elasticsearch-core", version, FixedValue.value(TypeDescription.VOID))
     }
 
     static void setupJarJdkClasspath(File projectRoot) {
@@ -39,26 +62,28 @@ class TestClasspathUtils {
     }
 
     private static void generateJdkJarHellCheck(File targetDir, String className, Implementation mainImplementation) {
-        generateJdkJarHellCheck(targetDir, className, "current", mainImplementation)
+        generateJarWithClass(targetDir, className, "elasticsearch-core", "current", mainImplementation)
     }
 
-        private static void generateJdkJarHellCheck(File targetDir, String className, String version, Implementation mainImplementation) {
+
+    private static void generateJarWithClass(File targetDir, String className, String artifactName, String version, Implementation mainImplementation) {
         DynamicType.Unloaded<?> dynamicType = new ByteBuddy().subclass(Object.class)
             .name(className)
             .defineMethod("main", void.class, Visibility.PUBLIC, Ownership.STATIC)
             .withParameters(String[].class)
             .intercept(mainImplementation)
             .make()
+
         try {
-            dynamicType.toJar(targetFile(targetDir, version))
+            dynamicType.toJar(targetFile(targetDir, artifactName, version))
         } catch (IOException e) {
             e.printStackTrace()
             fail("Cannot setup jdk jar hell classpath")
         }
     }
 
-    private static File targetFile(File projectRoot, String version) {
-        File targetFile = new File(projectRoot, "elasticsearch-core-${version}.jar")
+    private static File targetFile(File projectRoot, String artifactName, String version) {
+        File targetFile = new File(projectRoot, "${artifactName}-${version}.jar")
 
         println "targetFile = $targetFile"
         targetFile.getParentFile().mkdirs()
diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/StableApiJarMocks.java b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/StableApiJarMocks.java
index d5d45807f062..a625fef61903 100644
--- a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/StableApiJarMocks.java
+++ b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/StableApiJarMocks.java
@@ -11,8 +11,8 @@
 import net.bytebuddy.ByteBuddy;
 import net.bytebuddy.dynamic.DynamicType;
 
-import org.elasticsearch.plugin.api.Extensible;
-import org.elasticsearch.plugin.api.NamedComponent;
+import org.elasticsearch.plugin.Extensible;
+import org.elasticsearch.plugin.NamedComponent;
 import org.elasticsearch.plugin.scanner.test_classes.ExtensibleClass;
 import org.elasticsearch.plugin.scanner.test_classes.ExtensibleInterface;
 
diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/Extensible.java b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/Extensible.java
new file mode 100644
index 000000000000..34984b90d3c7
--- /dev/null
+++ b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/Extensible.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.plugin;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.TYPE;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target(value = { TYPE })
+public @interface Extensible {
+}
diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/NamedComponent.java b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/NamedComponent.java
new file mode 100644
index 000000000000..53b195313ecc
--- /dev/null
+++ b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/NamedComponent.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugin;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ ElementType.TYPE })
+public @interface NamedComponent {
+    /**
+     * The name used for registration and lookup
+     * @return a name
+     */
+    String value();
+}
diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/api/Extensible.java b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/api/Extensible.java
deleted file mode 100644
index 1a1f10f11c3e..000000000000
--- a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/api/Extensible.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.plugin.api;
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-import static java.lang.annotation.ElementType.TYPE;
-
-@Retention(RetentionPolicy.RUNTIME)
-@Target(value = { TYPE })
-public @interface Extensible {
-}
diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/api/NamedComponent.java b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/api/NamedComponent.java
deleted file mode 100644
index 65a1a0d46abd..000000000000
--- a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/api/NamedComponent.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.plugin.api;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-@Retention(RetentionPolicy.RUNTIME)
-@Target({ ElementType.TYPE })
-public @interface NamedComponent {
-    /**
-     * The name used for registration and lookup
-     * @return a name
-     */
-    String value();
-}
diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/ExtensibleClass.java b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/ExtensibleClass.java
index b76f6dd93f9d..50ea17993d7c 100644
--- a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/ExtensibleClass.java
+++ b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/ExtensibleClass.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.plugin.scanner.test_classes;
 
-import org.elasticsearch.plugin.api.Extensible;
+import org.elasticsearch.plugin.Extensible;
 
 @Extensible
 public class ExtensibleClass {}
diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/ExtensibleInterface.java b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/ExtensibleInterface.java
index c630bf216883..784136bf6f5a 100644
--- a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/ExtensibleInterface.java
+++ b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/ExtensibleInterface.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.plugin.scanner.test_classes;
 
-import org.elasticsearch.plugin.api.Extensible;
+import org.elasticsearch.plugin.Extensible;
 
 @Extensible
 public interface ExtensibleInterface {}
diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/TestNamedComponent.java b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/TestNamedComponent.java
index a98e6b08a797..9a6b9b648e3a 100644
--- a/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/TestNamedComponent.java
+++ b/build-tools/src/testFixtures/java/org/elasticsearch/plugin/scanner/test_classes/TestNamedComponent.java
@@ -8,7 +8,9 @@
 
 package org.elasticsearch.plugin.scanner.test_classes;
 
-@org.elasticsearch.plugin.api.NamedComponent("test_named_component")
+import org.elasticsearch.plugin.NamedComponent;
+
+@NamedComponent("test_named_component")
 public class TestNamedComponent implements ExtensibleInterface {
 
 }
diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle
index be2be6f0461e..500870fb593c 100644
--- a/client/rest-high-level/build.gradle
+++ b/client/rest-high-level/build.gradle
@@ -25,7 +25,6 @@ restResources {
 dependencies {
   api project(':modules:mapper-extras')
   api project(':modules:parent-join')
-  api project(':modules:aggs-matrix-stats')
   api project(':modules:rank-eval')
   api project(':modules:lang-mustache')
   api project(':modules:aggregations')
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/GetAliasesResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/GetAliasesResponse.java
deleted file mode 100644
index c5a1d97b7edf..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/GetAliasesResponse.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client;
-
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.cluster.metadata.AliasMetadata;
-import org.elasticsearch.common.xcontent.StatusToXContentObject;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xcontent.XContentParser.Token;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
-
-/**
- * Response obtained from the get aliases API.
- * The format is pretty horrible as it holds aliases, but at the same time errors can come back through the status and error fields.
- * Such errors are mostly 404 - NOT FOUND for aliases that were specified but not found. In such case the client won't throw exception
- * so it allows to retrieve the returned aliases, while at the same time checking if errors were returned.
- * There's also the case where an exception is returned, like for instance an {@link org.elasticsearch.index.IndexNotFoundException}.
- * We would usually throw such exception, but we configure the client to not throw for 404 to support the case above, hence we also not
- * throw in case an index is not found, although it is a hard error that doesn't come back with aliases.
- */
-public class GetAliasesResponse implements StatusToXContentObject {
-
-    private final RestStatus status;
-    private final String error;
-    private final ElasticsearchException exception;
-
-    private final Map<String, Set<AliasMetadata>> aliases;
-
-    GetAliasesResponse(RestStatus status, String error, Map<String, Set<AliasMetadata>> aliases) {
-        this.status = status;
-        this.error = error;
-        this.aliases = aliases;
-        this.exception = null;
-    }
-
-    private GetAliasesResponse(RestStatus status, ElasticsearchException exception) {
-        this.status = status;
-        this.error = null;
-        this.aliases = Collections.emptyMap();
-        this.exception = exception;
-    }
-
-    @Override
-    public RestStatus status() {
-        return status;
-    }
-
-    /**
-     * Return the possibly returned error, null otherwise
-     */
-    public String getError() {
-        return error;
-    }
-
-    /**
-     * Return the exception that may have been returned
-     */
-    public ElasticsearchException getException() {
-        return exception;
-    }
-
-    /**
-     * Return the requested aliases
-     */
-    public Map<String, Set<AliasMetadata>> getAliases() {
-        return aliases;
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        {
-            if (status != RestStatus.OK) {
-                builder.field("error", error);
-                builder.field("status", status.getStatus());
-            }
-
-            for (Map.Entry<String, Set<AliasMetadata>> entry : aliases.entrySet()) {
-                builder.startObject(entry.getKey());
-                {
-                    builder.startObject("aliases");
-                    {
-                        for (final AliasMetadata alias : entry.getValue()) {
-                            AliasMetadata.Builder.toXContent(alias, builder, ToXContent.EMPTY_PARAMS);
-                        }
-                    }
-                    builder.endObject();
-                }
-                builder.endObject();
-            }
-        }
-        builder.endObject();
-        return builder;
-    }
-
-    /**
-     * Parse the get aliases response
-     */
-    public static GetAliasesResponse fromXContent(XContentParser parser) throws IOException {
-        if (parser.currentToken() == null) {
-            parser.nextToken();
-        }
-        ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser);
-        Map<String, Set<AliasMetadata>> aliases = new HashMap<>();
-
-        String currentFieldName;
-        Token token;
-        String error = null;
-        ElasticsearchException exception = null;
-        RestStatus status = RestStatus.OK;
-
-        while (parser.nextToken() != Token.END_OBJECT) {
-            if (parser.currentToken() == Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-
-                if ("status".equals(currentFieldName)) {
-                    if ((token = parser.nextToken()) != Token.FIELD_NAME) {
-                        ensureExpectedToken(Token.VALUE_NUMBER, token, parser);
-                        status = RestStatus.fromCode(parser.intValue());
-                    }
-                } else if ("error".equals(currentFieldName)) {
-                    token = parser.nextToken();
-                    if (token == Token.VALUE_STRING) {
-                        error = parser.text();
-                    } else if (token == Token.START_OBJECT) {
-                        parser.nextToken();
-                        exception = ElasticsearchException.innerFromXContent(parser, true);
-                    } else if (token == Token.START_ARRAY) {
-                        parser.skipChildren();
-                    }
-                } else {
-                    String indexName = parser.currentName();
-                    if (parser.nextToken() == Token.START_OBJECT) {
-                        Set<AliasMetadata> parseInside = parseAliases(parser);
-                        aliases.put(indexName, parseInside);
-                    }
-                }
-            }
-        }
-        if (exception != null) {
-            assert error == null;
-            assert aliases.isEmpty();
-            return new GetAliasesResponse(status, exception);
-        }
-        return new GetAliasesResponse(status, error, aliases);
-    }
-
-    private static Set<AliasMetadata> parseAliases(XContentParser parser) throws IOException {
-        Set<AliasMetadata> aliases = new HashSet<>();
-        Token token;
-        String currentFieldName = null;
-        while ((token = parser.nextToken()) != Token.END_OBJECT) {
-            if (token == Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if (token == Token.START_OBJECT) {
-                if ("aliases".equals(currentFieldName)) {
-                    while (parser.nextToken() != Token.END_OBJECT) {
-                        AliasMetadata fromXContent = AliasMetadata.Builder.fromXContent(parser);
-                        aliases.add(fromXContent);
-                    }
-                } else {
-                    parser.skipChildren();
-                }
-            } else if (token == Token.START_ARRAY) {
-                parser.skipChildren();
-            }
-        }
-        return aliases;
-    }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/NodesResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/NodesResponse.java
deleted file mode 100644
index a0c38498591f..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/NodesResponse.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client;
-
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ParseField;
-
-/**
- * Base class for responses that are node responses. These responses always contain the cluster
- * name and the {@link NodesResponseHeader}.
- */
-public abstract class NodesResponse {
-
-    private final NodesResponseHeader header;
-    private final String clusterName;
-
-    protected NodesResponse(NodesResponseHeader header, String clusterName) {
-        this.header = header;
-        this.clusterName = clusterName;
-    }
-
-    /**
-     * Get the cluster name associated with all of the nodes.
-     *
-     * @return Never {@code null}.
-     */
-    public String getClusterName() {
-        return clusterName;
-    }
-
-    /**
-     * Gets information about the number of total, successful and failed nodes the request was run on.
-     * Also includes exceptions if relevant.
-     */
-    public NodesResponseHeader getHeader() {
-        return header;
-    }
-
-    public static <T extends NodesResponse> void declareCommonNodesResponseParsing(ConstructingObjectParser<T, Void> parser) {
-        parser.declareObject(ConstructingObjectParser.constructorArg(), NodesResponseHeader::fromXContent, new ParseField("_nodes"));
-        parser.declareString(ConstructingObjectParser.constructorArg(), new ParseField("cluster_name"));
-    }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/NodesResponseHeader.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/NodesResponseHeader.java
deleted file mode 100644
index e22326dc88fb..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/NodesResponseHeader.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.client;
-
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.support.nodes.BaseNodesResponse;
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.rest.action.RestActions;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-
-/**
- * A utility class to parse the Nodes Header returned by
- * {@link RestActions#buildNodesHeader(XContentBuilder, ToXContent.Params, BaseNodesResponse)}.
- */
-public final class NodesResponseHeader {
-
-    public static final ParseField TOTAL = new ParseField("total");
-    public static final ParseField SUCCESSFUL = new ParseField("successful");
-    public static final ParseField FAILED = new ParseField("failed");
-    public static final ParseField FAILURES = new ParseField("failures");
-
-    @SuppressWarnings("unchecked")
-    public static final ConstructingObjectParser<NodesResponseHeader, Void> PARSER = new ConstructingObjectParser<>(
-        "nodes_response_header",
-        true,
-        (a) -> {
-            int i = 0;
-            int total = (Integer) a[i++];
-            int successful = (Integer) a[i++];
-            int failed = (Integer) a[i++];
-            List<ElasticsearchException> failures = (List<ElasticsearchException>) a[i++];
-            return new NodesResponseHeader(total, successful, failed, failures);
-        }
-    );
-
-    static {
-        PARSER.declareInt(ConstructingObjectParser.constructorArg(), TOTAL);
-        PARSER.declareInt(ConstructingObjectParser.constructorArg(), SUCCESSFUL);
-        PARSER.declareInt(ConstructingObjectParser.constructorArg(), FAILED);
-        PARSER.declareObjectArray(
-            ConstructingObjectParser.optionalConstructorArg(),
-            (p, c) -> ElasticsearchException.fromXContent(p),
-            FAILURES
-        );
-    }
-
-    private final int total;
-    private final int successful;
-    private final int failed;
-    private final List<ElasticsearchException> failures;
-
-    public NodesResponseHeader(int total, int successful, int failed, @Nullable List<ElasticsearchException> failures) {
-        this.total = total;
-        this.successful = successful;
-        this.failed = failed;
-        this.failures = failures == null ? Collections.emptyList() : failures;
-    }
-
-    public static NodesResponseHeader fromXContent(XContentParser parser, Void context) throws IOException {
-        return PARSER.parse(parser, context);
-    }
-
-    /** the total number of nodes that the operation was carried on */
-    public int getTotal() {
-        return total;
-    }
-
-    /** the number of nodes that the operation has failed on */
-    public int getFailed() {
-        return failed;
-    }
-
-    /** the number of nodes that the operation was successful on */
-    public int getSuccessful() {
-        return successful;
-    }
-
-    /**
-     * Get the failed node exceptions.
-     *
-     * @return Never {@code null}. Can be empty.
-     */
-    public List<ElasticsearchException> getFailures() {
-        return failures;
-    }
-
-    /**
-     * Determine if there are any node failures in {@link #failures}.
-     *
-     * @return {@code true} if {@link #failures} contains at least 1 exception.
-     */
-    public boolean hasFailures() {
-        return failures.isEmpty() == false;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (o == null || getClass() != o.getClass()) {
-            return false;
-        }
-        NodesResponseHeader that = (NodesResponseHeader) o;
-        return total == that.total && successful == that.successful && failed == that.failed && Objects.equals(failures, that.failures);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(total, successful, failed, failures);
-    }
-
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
index 1cba9261ec09..fca1e5d29efa 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
@@ -9,66 +9,33 @@
 package org.elasticsearch.client;
 
 import org.apache.http.HttpEntity;
-import org.apache.http.client.methods.HttpDelete;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpHead;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.entity.ContentType;
 import org.apache.http.nio.entity.NByteArrayEntity;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.DocWriteRequest;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
-import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
-import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
-import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.explain.ExplainRequest;
-import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
-import org.elasticsearch.action.get.GetRequest;
-import org.elasticsearch.action.get.MultiGetRequest;
 import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.search.ClearScrollRequest;
-import org.elasticsearch.action.search.ClosePointInTimeRequest;
-import org.elasticsearch.action.search.MultiSearchRequest;
-import org.elasticsearch.action.search.OpenPointInTimeRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.client.core.CountRequest;
-import org.elasticsearch.client.core.GetSourceRequest;
-import org.elasticsearch.client.core.MultiTermVectorsRequest;
-import org.elasticsearch.client.core.TermVectorsRequest;
-import org.elasticsearch.client.internal.Requests;
-import org.elasticsearch.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.uid.Versions;
-import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.VersionType;
-import org.elasticsearch.index.rankeval.RankEvalRequest;
-import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest;
-import org.elasticsearch.index.reindex.DeleteByQueryRequest;
-import org.elasticsearch.index.reindex.ReindexRequest;
-import org.elasticsearch.index.reindex.UpdateByQueryRequest;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.rest.action.search.RestSearchAction;
-import org.elasticsearch.script.mustache.MultiSearchTemplateRequest;
-import org.elasticsearch.script.mustache.SearchTemplateRequest;
-import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.xcontent.DeprecationHandler;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.XContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
@@ -78,9 +45,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.charset.Charset;
-import java.util.Arrays;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.StringJoiner;
@@ -92,27 +57,6 @@ private RequestConverters() {
         // Contains only status utility methods
     }
 
-    static Request delete(DeleteRequest deleteRequest) {
-        String endpoint = endpoint(deleteRequest.index(), deleteRequest.id());
-        Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
-
-        Params parameters = new Params();
-        parameters.withRouting(deleteRequest.routing());
-        parameters.withTimeout(deleteRequest.timeout());
-        parameters.withVersion(deleteRequest.version());
-        parameters.withVersionType(deleteRequest.versionType());
-        parameters.withIfSeqNo(deleteRequest.ifSeqNo());
-        parameters.withIfPrimaryTerm(deleteRequest.ifPrimaryTerm());
-        parameters.withRefreshPolicy(deleteRequest.getRefreshPolicy());
-        parameters.withWaitForActiveShards(deleteRequest.waitForActiveShards());
-        request.addParameters(parameters.asMap());
-        return request;
-    }
-
-    static Request info() {
-        return new Request(HttpGet.METHOD_NAME, "/");
-    }
-
     static Request bulk(BulkRequest bulkRequest) throws IOException {
         Request request = new Request(HttpPost.METHOD_NAME, "/_bulk");
 
@@ -246,64 +190,6 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
         return request;
     }
 
-    static Request exists(GetRequest getRequest) {
-        return getStyleRequest(HttpHead.METHOD_NAME, getRequest);
-    }
-
-    static Request get(GetRequest getRequest) {
-        return getStyleRequest(HttpGet.METHOD_NAME, getRequest);
-    }
-
-    private static Request getStyleRequest(String method, GetRequest getRequest) {
-        Request request = new Request(method, endpoint(getRequest.index(), getRequest.id()));
-
-        Params parameters = new Params();
-        parameters.withPreference(getRequest.preference());
-        parameters.withRouting(getRequest.routing());
-        parameters.withRefresh(getRequest.refresh());
-        parameters.withRealtime(getRequest.realtime());
-        parameters.withStoredFields(getRequest.storedFields());
-        parameters.withVersion(getRequest.version());
-        parameters.withVersionType(getRequest.versionType());
-        parameters.withFetchSourceContext(getRequest.fetchSourceContext());
-        request.addParameters(parameters.asMap());
-        return request;
-    }
-
-    static Request sourceExists(GetSourceRequest getSourceRequest) {
-        return sourceRequest(getSourceRequest, HttpHead.METHOD_NAME);
-    }
-
-    static Request getSource(GetSourceRequest getSourceRequest) {
-        return sourceRequest(getSourceRequest, HttpGet.METHOD_NAME);
-    }
-
-    private static Request sourceRequest(GetSourceRequest getSourceRequest, String httpMethodName) {
-        Params parameters = new Params();
-        parameters.withPreference(getSourceRequest.preference());
-        parameters.withRouting(getSourceRequest.routing());
-        parameters.withRefresh(getSourceRequest.refresh());
-        parameters.withRealtime(getSourceRequest.realtime());
-        parameters.withFetchSourceContext(getSourceRequest.fetchSourceContext());
-
-        String endpoint = endpoint(getSourceRequest.index(), "_source", getSourceRequest.id());
-        Request request = new Request(httpMethodName, endpoint);
-        request.addParameters(parameters.asMap());
-        return request;
-    }
-
-    static Request multiGet(MultiGetRequest multiGetRequest) throws IOException {
-        Request request = new Request(HttpPost.METHOD_NAME, "/_mget");
-
-        Params parameters = new Params();
-        parameters.withPreference(multiGetRequest.preference());
-        parameters.withRealtime(multiGetRequest.realtime());
-        parameters.withRefresh(multiGetRequest.refresh());
-        request.addParameters(parameters.asMap());
-        request.setEntity(createEntity(multiGetRequest, REQUEST_BODY_CONTENT_TYPE));
-        return request;
-    }
-
     static Request index(IndexRequest indexRequest) {
         String method = Strings.hasLength(indexRequest.id()) ? HttpPut.METHOD_NAME : HttpPost.METHOD_NAME;
 
@@ -335,57 +221,6 @@ static Request index(IndexRequest indexRequest) {
         return request;
     }
 
-    static Request ping() {
-        return new Request(HttpHead.METHOD_NAME, "/");
-    }
-
-    static Request update(UpdateRequest updateRequest) throws IOException {
-        String endpoint = endpoint(updateRequest.index(), "_update", updateRequest.id());
-        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-
-        Params parameters = new Params();
-        parameters.withRouting(updateRequest.routing());
-        parameters.withTimeout(updateRequest.timeout());
-        parameters.withRefreshPolicy(updateRequest.getRefreshPolicy());
-        parameters.withWaitForActiveShards(updateRequest.waitForActiveShards());
-        parameters.withDocAsUpsert(updateRequest.docAsUpsert());
-        parameters.withFetchSourceContext(updateRequest.fetchSource());
-        parameters.withRetryOnConflict(updateRequest.retryOnConflict());
-        parameters.withVersion(updateRequest.version());
-        parameters.withVersionType(updateRequest.versionType());
-        parameters.withRequireAlias(updateRequest.isRequireAlias());
-
-        // The Java API allows update requests with different content types
-        // set for the partial document and the upsert document. This client
-        // only accepts update requests that have the same content types set
-        // for both doc and upsert.
-        XContentType xContentType = null;
-        if (updateRequest.doc() != null) {
-            xContentType = updateRequest.doc().getContentType();
-        }
-        if (updateRequest.upsertRequest() != null) {
-            XContentType upsertContentType = updateRequest.upsertRequest().getContentType();
-            if ((xContentType != null) && (xContentType != upsertContentType)) {
-                throw new IllegalStateException(
-                    "Update request cannot have different content types for doc ["
-                        + xContentType
-                        + "]"
-                        + " and upsert ["
-                        + upsertContentType
-                        + "] documents"
-                );
-            } else {
-                xContentType = upsertContentType;
-            }
-        }
-        if (xContentType == null) {
-            xContentType = Requests.INDEX_CONTENT_TYPE;
-        }
-        request.addParameters(parameters.asMap());
-        request.setEntity(createEntity(updateRequest, xContentType));
-        return request;
-    }
-
     /**
      * Convert a {@linkplain SearchRequest} into a {@linkplain Request}.
      * @param searchRequest the request to convert
@@ -439,326 +274,6 @@ static Request searchScroll(SearchScrollRequest searchScrollRequest) throws IOEx
         return request;
     }
 
-    static Request clearScroll(ClearScrollRequest clearScrollRequest) throws IOException {
-        Request request = new Request(HttpDelete.METHOD_NAME, "/_search/scroll");
-        request.setEntity(createEntity(clearScrollRequest, REQUEST_BODY_CONTENT_TYPE));
-        return request;
-    }
-
-    static Request openPointInTime(OpenPointInTimeRequest openRequest) {
-        Request request = new Request(HttpPost.METHOD_NAME, endpoint(openRequest.indices(), "_pit"));
-        Params params = new Params();
-        if (OpenPointInTimeRequest.DEFAULT_INDICES_OPTIONS.equals(openRequest.indicesOptions()) == false) {
-            params.withIndicesOptions(openRequest.indicesOptions());
-        }
-        params.withRouting(openRequest.routing());
-        params.withPreference(openRequest.preference());
-        params.putParam("keep_alive", openRequest.keepAlive());
-        request.addParameters(params.asMap());
-        return request;
-    }
-
-    static Request closePointInTime(ClosePointInTimeRequest closeRequest) throws IOException {
-        Request request = new Request(HttpDelete.METHOD_NAME, "/_pit");
-        request.setEntity(createEntity(closeRequest, REQUEST_BODY_CONTENT_TYPE));
-        return request;
-    }
-
-    static Request multiSearch(MultiSearchRequest multiSearchRequest) throws IOException {
-        Request request = new Request(HttpPost.METHOD_NAME, "/_msearch");
-
-        Params params = new Params();
-        params.putParam(RestSearchAction.TYPED_KEYS_PARAM, "true");
-        if (multiSearchRequest.maxConcurrentSearchRequests() != MultiSearchRequest.MAX_CONCURRENT_SEARCH_REQUESTS_DEFAULT) {
-            params.putParam("max_concurrent_searches", Integer.toString(multiSearchRequest.maxConcurrentSearchRequests()));
-        }
-
-        XContent xContent = REQUEST_BODY_CONTENT_TYPE.xContent();
-        byte[] source = MultiSearchRequest.writeMultiLineFormat(multiSearchRequest, xContent);
-        request.addParameters(params.asMap());
-        request.setEntity(new NByteArrayEntity(source, createContentType(xContent.type())));
-        return request;
-    }
-
-    static Request searchTemplate(SearchTemplateRequest searchTemplateRequest) throws IOException {
-        Request request;
-
-        if (searchTemplateRequest.isSimulate()) {
-            request = new Request(HttpGet.METHOD_NAME, "_render/template");
-        } else {
-            SearchRequest searchRequest = searchTemplateRequest.getRequest();
-            String endpoint = endpoint(searchRequest.indices(), "_search/template");
-            request = new Request(HttpPost.METHOD_NAME, endpoint);
-
-            Params params = new Params();
-            addSearchRequestParams(params, searchRequest);
-            request.addParameters(params.asMap());
-        }
-
-        request.setEntity(createEntity(searchTemplateRequest, REQUEST_BODY_CONTENT_TYPE));
-        return request;
-    }
-
-    static Request multiSearchTemplate(MultiSearchTemplateRequest multiSearchTemplateRequest) throws IOException {
-        Request request = new Request(HttpPost.METHOD_NAME, "/_msearch/template");
-
-        Params params = new Params();
-        params.putParam(RestSearchAction.TYPED_KEYS_PARAM, "true");
-        if (multiSearchTemplateRequest.maxConcurrentSearchRequests() != MultiSearchRequest.MAX_CONCURRENT_SEARCH_REQUESTS_DEFAULT) {
-            params.putParam("max_concurrent_searches", Integer.toString(multiSearchTemplateRequest.maxConcurrentSearchRequests()));
-        }
-        request.addParameters(params.asMap());
-
-        XContent xContent = REQUEST_BODY_CONTENT_TYPE.xContent();
-        byte[] source = MultiSearchTemplateRequest.writeMultiLineFormat(multiSearchTemplateRequest, xContent);
-        request.setEntity(new NByteArrayEntity(source, createContentType(xContent.type())));
-        return request;
-    }
-
-    static Request count(CountRequest countRequest) throws IOException {
-        Request request = new Request(HttpPost.METHOD_NAME, endpoint(countRequest.indices(), countRequest.types(), "_count"));
-        Params params = new Params();
-        params.withRouting(countRequest.routing());
-        params.withPreference(countRequest.preference());
-        params.withIndicesOptions(countRequest.indicesOptions());
-        if (countRequest.terminateAfter() != 0) {
-            params.withTerminateAfter(countRequest.terminateAfter());
-        }
-        if (countRequest.minScore() != null) {
-            params.putParam("min_score", String.valueOf(countRequest.minScore()));
-        }
-        request.addParameters(params.asMap());
-        request.setEntity(createEntity(countRequest, REQUEST_BODY_CONTENT_TYPE));
-        return request;
-    }
-
-    static Request explain(ExplainRequest explainRequest) throws IOException {
-        String endpoint = endpoint(explainRequest.index(), "_explain", explainRequest.id());
-        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
-
-        Params params = new Params();
-        params.withStoredFields(explainRequest.storedFields());
-        params.withFetchSourceContext(explainRequest.fetchSourceContext());
-        params.withRouting(explainRequest.routing());
-        params.withPreference(explainRequest.preference());
-        request.addParameters(params.asMap());
-        request.setEntity(createEntity(explainRequest, REQUEST_BODY_CONTENT_TYPE));
-        return request;
-    }
-
-    static Request fieldCaps(FieldCapabilitiesRequest fieldCapabilitiesRequest) throws IOException {
-        String methodName = fieldCapabilitiesRequest.indexFilter() != null ? HttpPost.METHOD_NAME : HttpGet.METHOD_NAME;
-        Request request = new Request(methodName, endpoint(fieldCapabilitiesRequest.indices(), "_field_caps"));
-
-        Params params = new Params();
-        params.withFields(fieldCapabilitiesRequest.fields());
-        if (FieldCapabilitiesRequest.DEFAULT_INDICES_OPTIONS.equals(fieldCapabilitiesRequest.indicesOptions()) == false) {
-            params.withIndicesOptions(fieldCapabilitiesRequest.indicesOptions());
-        }
-        request.addParameters(params.asMap());
-        if (fieldCapabilitiesRequest.indexFilter() != null) {
-            request.setEntity(createEntity(fieldCapabilitiesRequest, REQUEST_BODY_CONTENT_TYPE));
-        }
-        return request;
-    }
-
-    static Request rankEval(RankEvalRequest rankEvalRequest) throws IOException {
-        Request request = new Request(HttpGet.METHOD_NAME, endpoint(rankEvalRequest.indices(), Strings.EMPTY_ARRAY, "_rank_eval"));
-
-        Params params = new Params();
-        if (SearchRequest.DEFAULT_INDICES_OPTIONS.equals(rankEvalRequest.indicesOptions()) == false) {
-            params.withIndicesOptions(rankEvalRequest.indicesOptions());
-        }
-        params.putParam("search_type", rankEvalRequest.searchType().name().toLowerCase(Locale.ROOT));
-        request.addParameters(params.asMap());
-        request.setEntity(createEntity(rankEvalRequest.getRankEvalSpec(), REQUEST_BODY_CONTENT_TYPE));
-        return request;
-    }
-
-    static Request reindex(ReindexRequest reindexRequest) throws IOException {
-        return prepareReindexRequest(reindexRequest, true);
-    }
-
-    static Request deleteByQuery(DeleteByQueryRequest deleteByQueryRequest) throws IOException {
-        return prepareDeleteByQueryRequest(deleteByQueryRequest, true);
-    }
-
-    static Request updateByQuery(UpdateByQueryRequest updateByQueryRequest) throws IOException {
-        return prepareUpdateByQueryRequest(updateByQueryRequest, true);
-    }
-
-    private static Request prepareReindexRequest(ReindexRequest reindexRequest, boolean waitForCompletion) throws IOException {
-        String endpoint = new EndpointBuilder().addPathPart("_reindex").build();
-        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        Params params = new Params().withWaitForCompletion(waitForCompletion)
-            .withRefresh(reindexRequest.isRefresh())
-            .withTimeout(reindexRequest.getTimeout())
-            .withWaitForActiveShards(reindexRequest.getWaitForActiveShards())
-            .withRequestsPerSecond(reindexRequest.getRequestsPerSecond())
-            .withSlices(reindexRequest.getSlices())
-            .withRequireAlias(reindexRequest.getDestination().isRequireAlias());
-
-        if (reindexRequest.getScrollTime() != null) {
-            params.putParam("scroll", reindexRequest.getScrollTime());
-        }
-
-        request.addParameters(params.asMap());
-        request.setEntity(createEntity(reindexRequest, REQUEST_BODY_CONTENT_TYPE));
-        return request;
-    }
-
-    private static Request prepareDeleteByQueryRequest(DeleteByQueryRequest deleteByQueryRequest, boolean waitForCompletion)
-        throws IOException {
-        String endpoint = endpoint(deleteByQueryRequest.indices(), "_delete_by_query");
-        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        Params params = new Params().withRouting(deleteByQueryRequest.getRouting())
-            .withRefresh(deleteByQueryRequest.isRefresh())
-            .withTimeout(deleteByQueryRequest.getTimeout())
-            .withWaitForActiveShards(deleteByQueryRequest.getWaitForActiveShards())
-            .withRequestsPerSecond(deleteByQueryRequest.getRequestsPerSecond())
-            .withWaitForCompletion(waitForCompletion)
-            .withSlices(deleteByQueryRequest.getSlices());
-
-        if (SearchRequest.DEFAULT_INDICES_OPTIONS.equals(deleteByQueryRequest.indicesOptions()) == false) {
-            params = params.withIndicesOptions(deleteByQueryRequest.indicesOptions());
-        }
-
-        if (deleteByQueryRequest.isAbortOnVersionConflict() == false) {
-            params.putParam("conflicts", "proceed");
-        }
-        if (deleteByQueryRequest.getBatchSize() != AbstractBulkByScrollRequest.DEFAULT_SCROLL_SIZE) {
-            params.putParam("scroll_size", Integer.toString(deleteByQueryRequest.getBatchSize()));
-        }
-        if (deleteByQueryRequest.getScrollTime() != AbstractBulkByScrollRequest.DEFAULT_SCROLL_TIMEOUT) {
-            params.putParam("scroll", deleteByQueryRequest.getScrollTime());
-        }
-        if (deleteByQueryRequest.getMaxDocs() > 0) {
-            params.putParam("max_docs", Integer.toString(deleteByQueryRequest.getMaxDocs()));
-        }
-        request.addParameters(params.asMap());
-        request.setEntity(createEntity(deleteByQueryRequest, REQUEST_BODY_CONTENT_TYPE));
-        return request;
-    }
-
-    static Request prepareUpdateByQueryRequest(UpdateByQueryRequest updateByQueryRequest, boolean waitForCompletion) throws IOException {
-        String endpoint = endpoint(updateByQueryRequest.indices(), "_update_by_query");
-        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        Params params = new Params().withRouting(updateByQueryRequest.getRouting())
-            .withPipeline(updateByQueryRequest.getPipeline())
-            .withRefresh(updateByQueryRequest.isRefresh())
-            .withTimeout(updateByQueryRequest.getTimeout())
-            .withWaitForActiveShards(updateByQueryRequest.getWaitForActiveShards())
-            .withRequestsPerSecond(updateByQueryRequest.getRequestsPerSecond())
-            .withWaitForCompletion(waitForCompletion)
-            .withSlices(updateByQueryRequest.getSlices());
-        if (SearchRequest.DEFAULT_INDICES_OPTIONS.equals(updateByQueryRequest.indicesOptions()) == false) {
-            params = params.withIndicesOptions(updateByQueryRequest.indicesOptions());
-        }
-        if (updateByQueryRequest.isAbortOnVersionConflict() == false) {
-            params.putParam("conflicts", "proceed");
-        }
-        if (updateByQueryRequest.getBatchSize() != AbstractBulkByScrollRequest.DEFAULT_SCROLL_SIZE) {
-            params.putParam("scroll_size", Integer.toString(updateByQueryRequest.getBatchSize()));
-        }
-        if (updateByQueryRequest.getScrollTime() != AbstractBulkByScrollRequest.DEFAULT_SCROLL_TIMEOUT) {
-            params.putParam("scroll", updateByQueryRequest.getScrollTime());
-        }
-        if (updateByQueryRequest.getMaxDocs() > 0) {
-            params.putParam("max_docs", Integer.toString(updateByQueryRequest.getMaxDocs()));
-        }
-        request.addParameters(params.asMap());
-        request.setEntity(createEntity(updateByQueryRequest, REQUEST_BODY_CONTENT_TYPE));
-        return request;
-    }
-
-    static Request rethrottleReindex(RethrottleRequest rethrottleRequest) {
-        return rethrottle(rethrottleRequest, "_reindex");
-    }
-
-    static Request rethrottleUpdateByQuery(RethrottleRequest rethrottleRequest) {
-        return rethrottle(rethrottleRequest, "_update_by_query");
-    }
-
-    static Request rethrottleDeleteByQuery(RethrottleRequest rethrottleRequest) {
-        return rethrottle(rethrottleRequest, "_delete_by_query");
-    }
-
-    private static Request rethrottle(RethrottleRequest rethrottleRequest, String firstPathPart) {
-        String endpoint = new EndpointBuilder().addPathPart(firstPathPart)
-            .addPathPart(rethrottleRequest.getTaskId().toString())
-            .addPathPart("_rethrottle")
-            .build();
-        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        Params params = new Params().withRequestsPerSecond(rethrottleRequest.getRequestsPerSecond());
-        // we set "group_by" to "none" because this is the response format we can parse back
-        params.putParam("group_by", "none");
-        request.addParameters(params.asMap());
-        return request;
-    }
-
-    static Request putScript(PutStoredScriptRequest putStoredScriptRequest) throws IOException {
-        String endpoint = new EndpointBuilder().addPathPartAsIs("_scripts").addPathPart(putStoredScriptRequest.id()).build();
-        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        Params params = new Params();
-        params.withTimeout(putStoredScriptRequest.timeout());
-        params.withMasterTimeout(putStoredScriptRequest.masterNodeTimeout());
-        if (Strings.hasText(putStoredScriptRequest.context())) {
-            params.putParam("context", putStoredScriptRequest.context());
-        }
-        request.addParameters(params.asMap());
-        request.setEntity(createEntity(putStoredScriptRequest, REQUEST_BODY_CONTENT_TYPE));
-        return request;
-    }
-
-    static Request termVectors(TermVectorsRequest tvrequest) throws IOException {
-        String endpoint;
-        if (tvrequest.getType() != null) {
-            endpoint = new EndpointBuilder().addPathPart(tvrequest.getIndex(), tvrequest.getType(), tvrequest.getId())
-                .addPathPartAsIs("_termvectors")
-                .build();
-        } else {
-            endpoint = new EndpointBuilder().addPathPart(tvrequest.getIndex())
-                .addPathPartAsIs("_termvectors")
-                .addPathPart(tvrequest.getId())
-                .build();
-        }
-
-        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
-        Params params = new Params();
-        params.withRouting(tvrequest.getRouting());
-        params.withPreference(tvrequest.getPreference());
-        params.withRealtime(tvrequest.getRealtime());
-        request.addParameters(params.asMap());
-        request.setEntity(createEntity(tvrequest, REQUEST_BODY_CONTENT_TYPE));
-        return request;
-    }
-
-    static Request mtermVectors(MultiTermVectorsRequest mtvrequest) throws IOException {
-        String endpoint = "_mtermvectors";
-        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
-        request.setEntity(createEntity(mtvrequest, REQUEST_BODY_CONTENT_TYPE));
-        return request;
-    }
-
-    static Request getScript(GetStoredScriptRequest getStoredScriptRequest) {
-        String endpoint = new EndpointBuilder().addPathPartAsIs("_scripts").addPathPart(getStoredScriptRequest.id()).build();
-        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
-        Params params = new Params();
-        params.withMasterTimeout(getStoredScriptRequest.masterNodeTimeout());
-        request.addParameters(params.asMap());
-        return request;
-    }
-
-    static Request deleteScript(DeleteStoredScriptRequest deleteStoredScriptRequest) {
-        String endpoint = new EndpointBuilder().addPathPartAsIs("_scripts").addPathPart(deleteStoredScriptRequest.id()).build();
-        Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
-        Params params = new Params();
-        params.withTimeout(deleteStoredScriptRequest.timeout());
-        params.withMasterTimeout(deleteStoredScriptRequest.masterNodeTimeout());
-        request.addParameters(params.asMap());
-        return request;
-    }
-
     static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException {
         return createEntity(toXContent, xContentType, ToXContent.EMPTY_PARAMS);
     }
@@ -783,10 +298,6 @@ static String endpoint(String index, String type, String id, String endpoint) {
         return new EndpointBuilder().addPathPart(index, type, id).addPathPartAsIs(endpoint).build();
     }
 
-    static String endpoint(String[] indices) {
-        return new EndpointBuilder().addCommaSeparatedPathParts(indices).build();
-    }
-
     static String endpoint(String[] indices, String endpoint) {
         return new EndpointBuilder().addCommaSeparatedPathParts(indices).addPathPartAsIs(endpoint).build();
     }
@@ -799,13 +310,6 @@ static String endpoint(String[] indices, String[] types, String endpoint) {
             .build();
     }
 
-    static String endpoint(String[] indices, String endpoint, String[] suffixes) {
-        return new EndpointBuilder().addCommaSeparatedPathParts(indices)
-            .addPathPartAsIs(endpoint)
-            .addCommaSeparatedPathParts(suffixes)
-            .build();
-    }
-
     @Deprecated
     static String endpoint(String[] indices, String endpoint, String type) {
         return new EndpointBuilder().addCommaSeparatedPathParts(indices).addPathPartAsIs(endpoint).addPathPart(type).build();
@@ -849,39 +353,6 @@ Map<String, String> asMap() {
             return parameters;
         }
 
-        Params withDocAsUpsert(boolean docAsUpsert) {
-            if (docAsUpsert) {
-                return putParam("doc_as_upsert", Boolean.TRUE.toString());
-            }
-            return this;
-        }
-
-        Params withFetchSourceContext(FetchSourceContext fetchSourceContext) {
-            if (fetchSourceContext != null) {
-                if (fetchSourceContext.fetchSource() == false) {
-                    putParam("_source", Boolean.FALSE.toString());
-                }
-                if (CollectionUtils.isEmpty(fetchSourceContext.includes()) == false) {
-                    putParam("_source_includes", String.join(",", fetchSourceContext.includes()));
-                }
-                if (CollectionUtils.isEmpty(fetchSourceContext.excludes()) == false) {
-                    putParam("_source_excludes", String.join(",", fetchSourceContext.excludes()));
-                }
-            }
-            return this;
-        }
-
-        Params withFields(String[] fields) {
-            if (CollectionUtils.isEmpty(fields) == false) {
-                return putParam("fields", String.join(",", fields));
-            }
-            return this;
-        }
-
-        Params withMasterTimeout(TimeValue masterTimeout) {
-            return putParam("master_timeout", masterTimeout);
-        }
-
         Params withPipeline(String pipeline) {
             return putParam("pipeline", pipeline);
         }
@@ -910,20 +381,6 @@ Params withAllowPartialResults(boolean allowPartialSearchResults) {
             return putParam("allow_partial_search_results", Boolean.toString(allowPartialSearchResults));
         }
 
-        Params withRealtime(boolean realtime) {
-            if (realtime == false) {
-                return putParam("realtime", Boolean.FALSE.toString());
-            }
-            return this;
-        }
-
-        Params withRefresh(boolean refresh) {
-            if (refresh) {
-                return withRefreshPolicy(RefreshPolicy.IMMEDIATE);
-            }
-            return this;
-        }
-
         Params withRefreshPolicy(RefreshPolicy refreshPolicy) {
             if (refreshPolicy != RefreshPolicy.NONE) {
                 return putParam("refresh", refreshPolicy.getValue());
@@ -931,46 +388,10 @@ Params withRefreshPolicy(RefreshPolicy refreshPolicy) {
             return this;
         }
 
-        Params withRequestsPerSecond(float requestsPerSecond) {
-            // the default in AbstractBulkByScrollRequest is Float.POSITIVE_INFINITY,
-            // but we don't want to add that to the URL parameters, instead we use -1
-            if (Float.isFinite(requestsPerSecond)) {
-                return putParam(RethrottleRequest.REQUEST_PER_SECOND_PARAMETER, Float.toString(requestsPerSecond));
-            } else {
-                return putParam(RethrottleRequest.REQUEST_PER_SECOND_PARAMETER, "-1");
-            }
-        }
-
-        Params withRetryOnConflict(int retryOnConflict) {
-            if (retryOnConflict > 0) {
-                return putParam("retry_on_conflict", String.valueOf(retryOnConflict));
-            }
-            return this;
-        }
-
         Params withRouting(String routing) {
             return putParam("routing", routing);
         }
 
-        Params withSlices(int slices) {
-            if (slices == 0) {
-                // translate to "auto" value in rest request so the receiving end doesn't throw error
-                return putParam("slices", AbstractBulkByScrollRequest.AUTO_SLICES_VALUE);
-            }
-            return putParam("slices", String.valueOf(slices));
-        }
-
-        Params withStoredFields(String[] storedFields) {
-            if (CollectionUtils.isEmpty(storedFields) == false) {
-                return putParam("stored_fields", String.join(",", storedFields));
-            }
-            return this;
-        }
-
-        Params withTerminateAfter(int terminateAfter) {
-            return putParam("terminate_after", String.valueOf(terminateAfter));
-        }
-
         Params withTimeout(TimeValue timeout) {
             return putParam("timeout", timeout);
         }
@@ -1049,103 +470,6 @@ Params withIgnoreUnavailable(boolean ignoreUnavailable) {
             putParam("ignore_unavailable", Boolean.toString(ignoreUnavailable));
             return this;
         }
-
-        Params withHuman(boolean human) {
-            if (human) {
-                putParam("human", Boolean.toString(human));
-            }
-            return this;
-        }
-
-        Params withLocal(boolean local) {
-            if (local) {
-                putParam("local", Boolean.toString(local));
-            }
-            return this;
-        }
-
-        Params withIncludeDefaults(boolean includeDefaults) {
-            if (includeDefaults) {
-                return putParam("include_defaults", Boolean.TRUE.toString());
-            }
-            return this;
-        }
-
-        Params withPreserveExisting(boolean preserveExisting) {
-            if (preserveExisting) {
-                return putParam("preserve_existing", Boolean.TRUE.toString());
-            }
-            return this;
-        }
-
-        Params withDetailed(boolean detailed) {
-            if (detailed) {
-                return putParam("detailed", Boolean.TRUE.toString());
-            }
-            return this;
-        }
-
-        Params withWaitForCompletion(Boolean waitForCompletion) {
-            return putParam("wait_for_completion", waitForCompletion.toString());
-        }
-
-        Params withNodes(String[] nodes) {
-            return withNodes(Arrays.asList(nodes));
-        }
-
-        Params withNodes(List<String> nodes) {
-            if (nodes != null && nodes.size() > 0) {
-                return putParam("nodes", String.join(",", nodes));
-            }
-            return this;
-        }
-
-        Params withActions(String[] actions) {
-            return withActions(Arrays.asList(actions));
-        }
-
-        Params withActions(List<String> actions) {
-            if (actions != null && actions.size() > 0) {
-                return putParam("actions", String.join(",", actions));
-            }
-            return this;
-        }
-
-        Params withWaitForStatus(ClusterHealthStatus status) {
-            if (status != null) {
-                return putParam("wait_for_status", status.name().toLowerCase(Locale.ROOT));
-            }
-            return this;
-        }
-
-        Params withWaitForNoRelocatingShards(boolean waitNoRelocatingShards) {
-            if (waitNoRelocatingShards) {
-                return putParam("wait_for_no_relocating_shards", Boolean.TRUE.toString());
-            }
-            return this;
-        }
-
-        Params withWaitForNoInitializingShards(boolean waitNoInitShards) {
-            if (waitNoInitShards) {
-                return putParam("wait_for_no_initializing_shards", Boolean.TRUE.toString());
-            }
-            return this;
-        }
-
-        Params withWaitForNodes(String waitForNodes) {
-            return putParam("wait_for_nodes", waitForNodes);
-        }
-
-        Params withLevel(ClusterHealthRequest.Level level) {
-            return putParam("level", level.name().toLowerCase(Locale.ROOT));
-        }
-
-        Params withWaitForEvents(Priority waitForEvents) {
-            if (waitForEvents != null) {
-                return putParam("wait_for_events", waitForEvents.name().toLowerCase(Locale.ROOT));
-            }
-            return this;
-        }
     }
 
     /**
@@ -1199,11 +523,6 @@ EndpointBuilder addCommaSeparatedPathParts(String[] parts) {
             return this;
         }
 
-        EndpointBuilder addCommaSeparatedPathParts(List<String> parts) {
-            addPathPart(String.join(",", parts));
-            return this;
-        }
-
         EndpointBuilder addPathPartAsIs(String... parts) {
             for (String part : parts) {
                 if (Strings.hasLength(part)) {
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
index 1ccdb1ae58df..9b8e92b65981 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
@@ -19,25 +19,11 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.delete.DeleteResponse;
-import org.elasticsearch.action.explain.ExplainRequest;
-import org.elasticsearch.action.explain.ExplainResponse;
-import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
-import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
-import org.elasticsearch.action.get.GetRequest;
-import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.action.get.MultiGetRequest;
-import org.elasticsearch.action.get.MultiGetResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.action.search.MultiSearchRequest;
-import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollRequest;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder;
 import org.elasticsearch.aggregations.bucket.adjacency.ParsedAdjacencyMatrix;
 import org.elasticsearch.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
@@ -49,23 +35,12 @@
 import org.elasticsearch.client.analytics.ParsedTopMetrics;
 import org.elasticsearch.client.analytics.StringStatsAggregationBuilder;
 import org.elasticsearch.client.analytics.TopMetricsAggregationBuilder;
-import org.elasticsearch.client.core.CountRequest;
-import org.elasticsearch.client.core.CountResponse;
-import org.elasticsearch.client.core.GetSourceRequest;
-import org.elasticsearch.client.core.GetSourceResponse;
-import org.elasticsearch.client.core.MainRequest;
 import org.elasticsearch.client.core.MainResponse;
-import org.elasticsearch.client.core.TermVectorsRequest;
-import org.elasticsearch.client.core.TermVectorsResponse;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.common.util.concurrent.ListenableFuture;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.CheckedFunction;
-import org.elasticsearch.index.reindex.BulkByScrollResponse;
-import org.elasticsearch.index.reindex.DeleteByQueryRequest;
-import org.elasticsearch.index.reindex.ReindexRequest;
-import org.elasticsearch.index.reindex.UpdateByQueryRequest;
 import org.elasticsearch.plugins.spi.NamedXContentProvider;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
@@ -198,7 +173,6 @@
 import java.util.stream.Stream;
 
 import static java.util.Collections.emptySet;
-import static java.util.Collections.singleton;
 import static java.util.stream.Collectors.toList;
 
 /**
@@ -360,223 +334,6 @@ public final Cancellable bulkAsync(BulkRequest bulkRequest, RequestOptions optio
         );
     }
 
-    /**
-     * Executes a reindex request.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html">Reindex API on elastic.co</a>
-     * @param reindexRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return the response
-     */
-    public final BulkByScrollResponse reindex(ReindexRequest reindexRequest, RequestOptions options) throws IOException {
-        return performRequestAndParseEntity(
-            reindexRequest,
-            RequestConverters::reindex,
-            options,
-            BulkByScrollResponse::fromXContent,
-            singleton(409)
-        );
-    }
-
-    /**
-     * Executes a update by query request.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update-by-query.html">
-     *     Update By Query API on elastic.co</a>
-     * @param updateByQueryRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return the response
-     */
-    public final BulkByScrollResponse updateByQuery(UpdateByQueryRequest updateByQueryRequest, RequestOptions options) throws IOException {
-        return performRequestAndParseEntity(
-            updateByQueryRequest,
-            RequestConverters::updateByQuery,
-            options,
-            BulkByScrollResponse::fromXContent,
-            singleton(409)
-        );
-    }
-
-    /**
-     * Executes a delete by query request.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html">
-     *     Delete By Query API on elastic.co</a>
-     * @param deleteByQueryRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return the response
-     */
-    public final BulkByScrollResponse deleteByQuery(DeleteByQueryRequest deleteByQueryRequest, RequestOptions options) throws IOException {
-        return performRequestAndParseEntity(
-            deleteByQueryRequest,
-            RequestConverters::deleteByQuery,
-            options,
-            BulkByScrollResponse::fromXContent,
-            singleton(409)
-        );
-    }
-
-    /**
-     * Get the cluster info otherwise provided when sending an HTTP request to '/'
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return the response
-     */
-    public final MainResponse info(RequestOptions options) throws IOException {
-        return performRequestAndParseEntity(
-            new MainRequest(),
-            (request) -> RequestConverters.info(),
-            options,
-            MainResponse::fromXContent,
-            emptySet()
-        );
-    }
-
-    /**
-     * Retrieves a document by id using the Get API.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html">Get API on elastic.co</a>
-     * @param getRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return the response
-     */
-    public final GetResponse get(GetRequest getRequest, RequestOptions options) throws IOException {
-        return performRequestAndParseEntity(getRequest, RequestConverters::get, options, GetResponse::fromXContent, singleton(404));
-    }
-
-    /**
-     * Retrieves multiple documents by id using the Multi Get API.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-multi-get.html">Multi Get API on elastic.co</a>
-     * @param multiGetRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return the response
-     * @deprecated use {@link #mget(MultiGetRequest, RequestOptions)} instead
-     */
-    @Deprecated
-    public final MultiGetResponse multiGet(MultiGetRequest multiGetRequest, RequestOptions options) throws IOException {
-        return mget(multiGetRequest, options);
-    }
-
-    /**
-     * Retrieves multiple documents by id using the Multi Get API.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-multi-get.html">Multi Get API on elastic.co</a>
-     * @param multiGetRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return the response
-     */
-    public final MultiGetResponse mget(MultiGetRequest multiGetRequest, RequestOptions options) throws IOException {
-        return performRequestAndParseEntity(
-            multiGetRequest,
-            RequestConverters::multiGet,
-            options,
-            MultiGetResponse::fromXContent,
-            singleton(404)
-        );
-    }
-
-    /**
-     * Asynchronously retrieves multiple documents by id using the Multi Get API.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-multi-get.html">Multi Get API on elastic.co</a>
-     * @param multiGetRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @param listener the listener to be notified upon request completion
-     * @deprecated use {@link #mgetAsync(MultiGetRequest, RequestOptions, ActionListener)} instead
-     * @return cancellable that may be used to cancel the request
-     */
-    @Deprecated
-    public final Cancellable multiGetAsync(
-        MultiGetRequest multiGetRequest,
-        RequestOptions options,
-        ActionListener<MultiGetResponse> listener
-    ) {
-        return mgetAsync(multiGetRequest, options, listener);
-    }
-
-    /**
-     * Asynchronously retrieves multiple documents by id using the Multi Get API.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-multi-get.html">Multi Get API on elastic.co</a>
-     * @param multiGetRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @param listener the listener to be notified upon request completion
-     * @return cancellable that may be used to cancel the request
-     */
-    public final Cancellable mgetAsync(MultiGetRequest multiGetRequest, RequestOptions options, ActionListener<MultiGetResponse> listener) {
-        return performRequestAsyncAndParseEntity(
-            multiGetRequest,
-            RequestConverters::multiGet,
-            options,
-            MultiGetResponse::fromXContent,
-            listener,
-            singleton(404)
-        );
-    }
-
-    /**
-     * Checks for the existence of a document. Returns true if it exists, false otherwise.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html">Get API on elastic.co</a>
-     * @param getRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return <code>true</code> if the document exists, <code>false</code> otherwise
-     */
-    public final boolean exists(GetRequest getRequest, RequestOptions options) throws IOException {
-        return performRequest(getRequest, RequestConverters::exists, options, RestHighLevelClient::convertExistsResponse, emptySet());
-    }
-
-    /**
-     * Checks for the existence of a document with a "_source" field. Returns true if it exists, false otherwise.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html#_source">Source exists API
-     * on elastic.co</a>
-     * @param getRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return <code>true</code> if the document and _source field exists, <code>false</code> otherwise
-     */
-    @Deprecated
-    public boolean existsSource(GetRequest getRequest, RequestOptions options) throws IOException {
-        GetSourceRequest getSourceRequest = GetSourceRequest.from(getRequest);
-        return performRequest(
-            getSourceRequest,
-            RequestConverters::sourceExists,
-            options,
-            RestHighLevelClient::convertExistsResponse,
-            emptySet()
-        );
-    }
-
-    /**
-     * Asynchronously checks for the existence of a document with a "_source" field. Returns true if it exists, false otherwise.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html#_source">Source exists API
-     * on elastic.co</a>
-     * @param getRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @param listener the listener to be notified upon request completion
-     * @return cancellable that may be used to cancel the request
-     */
-    @Deprecated
-    public final Cancellable existsSourceAsync(GetRequest getRequest, RequestOptions options, ActionListener<Boolean> listener) {
-        GetSourceRequest getSourceRequest = GetSourceRequest.from(getRequest);
-        return performRequestAsync(
-            getSourceRequest,
-            RequestConverters::sourceExists,
-            options,
-            RestHighLevelClient::convertExistsResponse,
-            listener,
-            emptySet()
-        );
-    }
-
-    /**
-     * Retrieves the source field only of a document using GetSource API.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html#_source">Get Source API
-     * on elastic.co</a>
-     * @param getSourceRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return the response
-     */
-    public GetSourceResponse getSource(GetSourceRequest getSourceRequest, RequestOptions options) throws IOException {
-        return performRequestAndParseEntity(
-            getSourceRequest,
-            RequestConverters::getSource,
-            options,
-            GetSourceResponse::fromXContent,
-            emptySet()
-        );
-    }
-
     /**
      * Index a document using the Index API.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html">Index API on elastic.co</a>
@@ -588,45 +345,6 @@ public final IndexResponse index(IndexRequest indexRequest, RequestOptions optio
         return performRequestAndParseEntity(indexRequest, RequestConverters::index, options, IndexResponse::fromXContent, emptySet());
     }
 
-    /**
-     * Executes a count request using the Count API.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-count.html">Count API on elastic.co</a>
-     * @param countRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return the response
-     */
-    public final CountResponse count(CountRequest countRequest, RequestOptions options) throws IOException {
-        return performRequestAndParseEntity(countRequest, RequestConverters::count, options, CountResponse::fromXContent, emptySet());
-    }
-
-    /**
-     * Updates a document using the Update API.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update.html">Update API on elastic.co</a>
-     * @param updateRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return the response
-     */
-    public final UpdateResponse update(UpdateRequest updateRequest, RequestOptions options) throws IOException {
-        return performRequestAndParseEntity(updateRequest, RequestConverters::update, options, UpdateResponse::fromXContent, emptySet());
-    }
-
-    /**
-     * Deletes a document by id using the Delete API.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete.html">Delete API on elastic.co</a>
-     * @param deleteRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return the response
-     */
-    public final DeleteResponse delete(DeleteRequest deleteRequest, RequestOptions options) throws IOException {
-        return performRequestAndParseEntity(
-            deleteRequest,
-            RequestConverters::delete,
-            options,
-            DeleteResponse::fromXContent,
-            singleton(404)
-        );
-    }
-
     /**
      * Executes a search request using the Search API.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html">Search API on elastic.co</a>
@@ -663,96 +381,6 @@ public final Cancellable searchAsync(SearchRequest searchRequest, RequestOptions
         );
     }
 
-    /**
-     * Executes a multi search using the msearch API.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-multi-search.html">Multi search API on
-     * elastic.co</a>
-     * @param multiSearchRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return the response
-     * @deprecated use {@link #msearch(MultiSearchRequest, RequestOptions)} instead
-     */
-    @Deprecated
-    public final MultiSearchResponse multiSearch(MultiSearchRequest multiSearchRequest, RequestOptions options) throws IOException {
-        return msearch(multiSearchRequest, options);
-    }
-
-    /**
-     * Executes a multi search using the msearch API.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-multi-search.html">Multi search API on
-     * elastic.co</a>
-     * @param multiSearchRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return the response
-     */
-    public final MultiSearchResponse msearch(MultiSearchRequest multiSearchRequest, RequestOptions options) throws IOException {
-        return performRequestAndParseEntity(
-            multiSearchRequest,
-            RequestConverters::multiSearch,
-            options,
-            MultiSearchResponse::fromXContext,
-            emptySet()
-        );
-    }
-
-    /**
-     * Asynchronously executes a multi search using the msearch API.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-multi-search.html">Multi search API on
-     * elastic.co</a>
-     * @param searchRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @param listener the listener to be notified upon request completion
-     * @deprecated use {@link #msearchAsync(MultiSearchRequest, RequestOptions, ActionListener)} instead
-     * @return cancellable that may be used to cancel the request
-     */
-    @Deprecated
-    public final Cancellable multiSearchAsync(
-        MultiSearchRequest searchRequest,
-        RequestOptions options,
-        ActionListener<MultiSearchResponse> listener
-    ) {
-        return msearchAsync(searchRequest, options, listener);
-    }
-
-    /**
-     * Asynchronously executes a multi search using the msearch API.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-multi-search.html">Multi search API on
-     * elastic.co</a>
-     * @param searchRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @param listener the listener to be notified upon request completion
-     * @return cancellable that may be used to cancel the request
-     */
-    public final Cancellable msearchAsync(
-        MultiSearchRequest searchRequest,
-        RequestOptions options,
-        ActionListener<MultiSearchResponse> listener
-    ) {
-        return performRequestAsyncAndParseEntity(
-            searchRequest,
-            RequestConverters::multiSearch,
-            options,
-            MultiSearchResponse::fromXContext,
-            listener,
-            emptySet()
-        );
-    }
-
-    /**
-     * Executes a search using the Search Scroll API.
-     * See <a
-     * href="https://www.elastic.co/guide/en/elasticsearch/reference/master/search-request-body.html#request-body-search-scroll">Search
-     * Scroll API on elastic.co</a>
-     * @param searchScrollRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return the response
-     * @deprecated use {@link #scroll(SearchScrollRequest, RequestOptions)} instead
-     */
-    @Deprecated
-    public final SearchResponse searchScroll(SearchScrollRequest searchScrollRequest, RequestOptions options) throws IOException {
-        return scroll(searchScrollRequest, options);
-    }
-
     /**
      * Executes a search using the Search Scroll API.
      * See <a
@@ -772,106 +400,6 @@ public final SearchResponse scroll(SearchScrollRequest searchScrollRequest, Requ
         );
     }
 
-    /**
-     * Asynchronously executes a search using the Search Scroll API.
-     * See <a
-     * href="https://www.elastic.co/guide/en/elasticsearch/reference/master/search-request-body.html#request-body-search-scroll">Search
-     * Scroll API on elastic.co</a>
-     * @param searchScrollRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @param listener the listener to be notified upon request completion
-     * @deprecated use {@link #scrollAsync(SearchScrollRequest, RequestOptions, ActionListener)} instead
-     * @return cancellable that may be used to cancel the request
-     */
-    @Deprecated
-    public final Cancellable searchScrollAsync(
-        SearchScrollRequest searchScrollRequest,
-        RequestOptions options,
-        ActionListener<SearchResponse> listener
-    ) {
-        return scrollAsync(searchScrollRequest, options, listener);
-    }
-
-    /**
-     * Asynchronously executes a search using the Search Scroll API.
-     * See <a
-     * href="https://www.elastic.co/guide/en/elasticsearch/reference/master/search-request-body.html#request-body-search-scroll">Search
-     * Scroll API on elastic.co</a>
-     * @param searchScrollRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @param listener the listener to be notified upon request completion
-     * @return cancellable that may be used to cancel the request
-     */
-    public final Cancellable scrollAsync(
-        SearchScrollRequest searchScrollRequest,
-        RequestOptions options,
-        ActionListener<SearchResponse> listener
-    ) {
-        return performRequestAsyncAndParseEntity(
-            searchScrollRequest,
-            RequestConverters::searchScroll,
-            options,
-            SearchResponse::fromXContent,
-            listener,
-            emptySet()
-        );
-    }
-
-    /**
-     * Executes a request using the Explain API.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-explain.html">Explain API on elastic.co</a>
-     * @param explainRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return the response
-     */
-    public final ExplainResponse explain(ExplainRequest explainRequest, RequestOptions options) throws IOException {
-        return performRequest(explainRequest, RequestConverters::explain, options, response -> {
-            CheckedFunction<XContentParser, ExplainResponse, IOException> entityParser = parser -> ExplainResponse.fromXContent(
-                parser,
-                convertExistsResponse(response)
-            );
-            return parseEntity(response.getEntity(), entityParser);
-        }, singleton(404));
-    }
-
-    /**
-     * Calls the Term Vectors API
-     *
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-termvectors.html">Term Vectors API on
-     * elastic.co</a>
-     *
-     * @param request   the request
-     * @param options   the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     */
-    public final TermVectorsResponse termvectors(TermVectorsRequest request, RequestOptions options) throws IOException {
-        return performRequestAndParseEntity(
-            request,
-            RequestConverters::termVectors,
-            options,
-            TermVectorsResponse::fromXContent,
-            emptySet()
-        );
-    }
-
-    /**
-     * Executes a request using the Field Capabilities API.
-     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-field-caps.html">Field Capabilities API
-     * on elastic.co</a>.
-     * @param fieldCapabilitiesRequest the request
-     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
-     * @return the response
-     */
-    public final FieldCapabilitiesResponse fieldCaps(FieldCapabilitiesRequest fieldCapabilitiesRequest, RequestOptions options)
-        throws IOException {
-        return performRequestAndParseEntity(
-            fieldCapabilitiesRequest,
-            RequestConverters::fieldCaps,
-            options,
-            FieldCapabilitiesResponse::fromXContent,
-            emptySet()
-        );
-    }
-
     /**
      * @deprecated If creating a new HLRC ReST API call, consider creating new actions instead of reusing server actions. The Validation
      * layer has been added to the ReST client, and requests should extend {@link Validatable} instead of {@link ActionRequest}.
@@ -887,19 +415,6 @@ protected final <Req extends ActionRequest, Resp> Resp performRequestAndParseEnt
         return performRequest(request, requestConverter, options, response -> parseEntity(response.getEntity(), entityParser), ignores);
     }
 
-    /**
-     * Defines a helper method for performing a request and then parsing the returned entity using the provided entityParser.
-     */
-    protected final <Req extends Validatable, Resp> Resp performRequestAndParseEntity(
-        Req request,
-        CheckedFunction<Req, Request, IOException> requestConverter,
-        RequestOptions options,
-        CheckedFunction<XContentParser, Resp, IOException> entityParser,
-        Set<Integer> ignores
-    ) throws IOException {
-        return performRequest(request, requestConverter, options, response -> parseEntity(response.getEntity(), entityParser), ignores);
-    }
-
     /**
      * @deprecated If creating a new HLRC ReST API call, consider creating new actions instead of reusing server actions. The Validation
      * layer has been added to the ReST client, and requests should extend {@link Validatable} instead of {@link ActionRequest}.
@@ -1019,26 +534,6 @@ protected final <Req extends ActionRequest, Resp> Cancellable performRequestAsyn
         return internalPerformRequestAsync(request, requestConverter, options, responseConverter, listener, ignores);
     }
 
-    /**
-     * Defines a helper method for asynchronously performing a request.
-     * @return Cancellable instance that may be used to cancel the request
-     */
-    protected final <Req extends Validatable, Resp> Cancellable performRequestAsync(
-        Req request,
-        CheckedFunction<Req, Request, IOException> requestConverter,
-        RequestOptions options,
-        CheckedFunction<Response, Resp, IOException> responseConverter,
-        ActionListener<Resp> listener,
-        Set<Integer> ignores
-    ) {
-        Optional<ValidationException> validationException = request.validate();
-        if (validationException != null && validationException.isPresent()) {
-            listener.onFailure(validationException.get());
-            return Cancellable.NO_OP;
-        }
-        return internalPerformRequestAsync(request, requestConverter, options, responseConverter, listener, ignores);
-    }
-
     /**
      * Provides common functionality for asynchronously performing a request.
      * @return Cancellable instance that may be used to cancel the request
@@ -1104,37 +599,6 @@ public void onFailure(Exception exception) {
         };
     }
 
-    final <Resp> ResponseListener wrapResponseListener404sOptional(
-        CheckedFunction<Response, Resp, IOException> responseConverter,
-        ActionListener<Optional<Resp>> actionListener
-    ) {
-        return new ResponseListener() {
-            @Override
-            public void onSuccess(Response response) {
-                try {
-                    actionListener.onResponse(Optional.of(responseConverter.apply(response)));
-                } catch (Exception e) {
-                    IOException ioe = new IOException("Unable to parse response body for " + response, e);
-                    onFailure(ioe);
-                }
-            }
-
-            @Override
-            public void onFailure(Exception exception) {
-                if (exception instanceof ResponseException responseException) {
-                    Response response = responseException.getResponse();
-                    if (RestStatus.NOT_FOUND.getStatus() == response.getStatusLine().getStatusCode()) {
-                        actionListener.onResponse(Optional.empty());
-                    } else {
-                        actionListener.onFailure(parseResponseException(responseException));
-                    }
-                } else {
-                    actionListener.onFailure(exception);
-                }
-            }
-        };
-    }
-
     /**
      * Converts a {@link ResponseException} obtained from the low level REST client into an {@link ElasticsearchException}.
      * If a response body was returned, tries to parse it as an error returned from Elasticsearch.
@@ -1179,10 +643,6 @@ protected final <Resp> Resp parseEntity(final HttpEntity entity, final CheckedFu
         }
     }
 
-    protected static boolean convertExistsResponse(Response response) {
-        return response.getStatusLine().getStatusCode() == 200;
-    }
-
     private enum EntityType {
         JSON() {
             @Override
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RethrottleRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RethrottleRequest.java
deleted file mode 100644
index a518d74b14c2..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RethrottleRequest.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client;
-
-import org.elasticsearch.tasks.TaskId;
-
-import java.util.Objects;
-
-/**
- * A request changing throttling of a task.
- */
-public class RethrottleRequest implements Validatable {
-
-    static final String REQUEST_PER_SECOND_PARAMETER = "requests_per_second";
-
-    private final TaskId taskId;
-    private final float requestsPerSecond;
-
-    /**
-     * Create a new {@link RethrottleRequest} which disables any throttling for the given taskId.
-     * @param taskId the task for which throttling will be disabled
-     */
-    public RethrottleRequest(TaskId taskId) {
-        this.taskId = taskId;
-        this.requestsPerSecond = Float.POSITIVE_INFINITY;
-    }
-
-    /**
-     * Create a new {@link RethrottleRequest} which changes the throttling for the given taskId.
-     * @param taskId the task that throttling changes will be applied to
-     * @param requestsPerSecond the number of requests per second that the task should perform. This needs to be a positive value.
-     */
-    public RethrottleRequest(TaskId taskId, float requestsPerSecond) {
-        Objects.requireNonNull(taskId, "taskId cannot be null");
-        if (requestsPerSecond <= 0) {
-            throw new IllegalArgumentException("requestsPerSecond needs to be positive value but was [" + requestsPerSecond + "]");
-        }
-        this.taskId = taskId;
-        this.requestsPerSecond = requestsPerSecond;
-    }
-
-    /**
-     * @return the task Id
-     */
-    public TaskId getTaskId() {
-        return taskId;
-    }
-
-    /**
-     * @return the requests per seconds value
-     */
-    public float getRequestsPerSecond() {
-        return requestsPerSecond;
-    }
-
-    @Override
-    public String toString() {
-        return "RethrottleRequest: taskID = " + taskId + "; reqestsPerSecond = " + requestsPerSecond;
-    }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java
deleted file mode 100644
index a3135b9725eb..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.client;
-
-import org.elasticsearch.core.TimeValue;
-
-import static org.elasticsearch.core.TimeValue.timeValueSeconds;
-
-/**
- * A base request for any requests that supply timeouts.
- *
- * Please note, any requests that use a ackTimeout should set timeout as they
- * represent the same backing field on the server.
- */
-public abstract class TimedRequest implements Validatable {
-
-    public static final TimeValue DEFAULT_ACK_TIMEOUT = timeValueSeconds(30);
-    public static final TimeValue DEFAULT_MASTER_NODE_TIMEOUT = TimeValue.timeValueSeconds(30);
-
-    private TimeValue timeout = DEFAULT_ACK_TIMEOUT;
-    private TimeValue masterTimeout = DEFAULT_MASTER_NODE_TIMEOUT;
-
-    /**
-     * Sets the timeout to wait for the all the nodes to acknowledge
-     * @param timeout timeout as a {@link TimeValue}
-     */
-    public void setTimeout(TimeValue timeout) {
-        this.timeout = timeout;
-    }
-
-    /**
-     * Sets the timeout to connect to the master node
-     * @param masterTimeout timeout as a {@link TimeValue}
-     */
-    public void setMasterTimeout(TimeValue masterTimeout) {
-        this.masterTimeout = masterTimeout;
-    }
-
-    /**
-     * Returns the request timeout
-     */
-    public TimeValue timeout() {
-        return timeout;
-    }
-
-    /**
-     * Returns the timeout for the request to be completed on the master node
-     */
-    public TimeValue masterNodeTimeout() {
-        return masterTimeout;
-    }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/StringStatsAggregationBuilder.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/StringStatsAggregationBuilder.java
index e1dc45e65086..981e6de9c941 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/StringStatsAggregationBuilder.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/StringStatsAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.client.analytics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.index.query.QueryRewriteContext;
@@ -124,7 +124,7 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/TopMetricsAggregationBuilder.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/TopMetricsAggregationBuilder.java
index eb3d9cf0452c..5af136f75bb7 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/TopMetricsAggregationBuilder.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/TopMetricsAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.client.analytics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.index.query.QueryRewriteContext;
@@ -101,7 +101,7 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map<String, O
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_7_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_7_0;
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/DeleteAsyncSearchRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/DeleteAsyncSearchRequest.java
deleted file mode 100644
index 0f7e7579e0b5..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/DeleteAsyncSearchRequest.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client.asyncsearch;
-
-import org.elasticsearch.client.Validatable;
-
-import java.util.Objects;
-
-public class DeleteAsyncSearchRequest implements Validatable {
-
-    private final String id;
-
-    public DeleteAsyncSearchRequest(String id) {
-        this.id = id;
-    }
-
-    public String getId() {
-        return this.id;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (o == null || getClass() != o.getClass()) {
-            return false;
-        }
-        DeleteAsyncSearchRequest request = (DeleteAsyncSearchRequest) o;
-        return Objects.equals(getId(), request.getId());
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(getId());
-    }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/GetAsyncSearchRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/GetAsyncSearchRequest.java
deleted file mode 100644
index d6c40dbe1a56..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/GetAsyncSearchRequest.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client.asyncsearch;
-
-import org.elasticsearch.client.Validatable;
-import org.elasticsearch.client.ValidationException;
-import org.elasticsearch.core.TimeValue;
-
-import java.util.Objects;
-import java.util.Optional;
-
-public class GetAsyncSearchRequest implements Validatable {
-
-    private TimeValue waitForCompletion;
-    private TimeValue keepAlive;
-
-    private final String id;
-
-    public GetAsyncSearchRequest(String id) {
-        this.id = id;
-    }
-
-    public String getId() {
-        return this.id;
-    }
-
-    public TimeValue getWaitForCompletion() {
-        return waitForCompletion;
-    }
-
-    public void setWaitForCompletion(TimeValue waitForCompletion) {
-        this.waitForCompletion = waitForCompletion;
-    }
-
-    public TimeValue getKeepAlive() {
-        return keepAlive;
-    }
-
-    public void setKeepAlive(TimeValue keepAlive) {
-        this.keepAlive = keepAlive;
-    }
-
-    @Override
-    public Optional<ValidationException> validate() {
-        return Optional.empty();
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (o == null || getClass() != o.getClass()) {
-            return false;
-        }
-        GetAsyncSearchRequest request = (GetAsyncSearchRequest) o;
-        return Objects.equals(getId(), request.getId())
-            && Objects.equals(getKeepAlive(), request.getKeepAlive())
-            && Objects.equals(getWaitForCompletion(), request.getWaitForCompletion());
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(getId(), getKeepAlive(), getWaitForCompletion());
-    }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/SubmitAsyncSearchRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/SubmitAsyncSearchRequest.java
deleted file mode 100644
index 9c381645d6ff..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/asyncsearch/SubmitAsyncSearchRequest.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client.asyncsearch;
-
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Validatable;
-import org.elasticsearch.client.ValidationException;
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-
-import java.util.Objects;
-import java.util.Optional;
-
-/**
- * A request to track asynchronously the progress of a search against one or more indices.
- */
-public class SubmitAsyncSearchRequest implements Validatable {
-
-    public static long MIN_KEEP_ALIVE = TimeValue.timeValueMinutes(1).millis();
-
-    private TimeValue waitForCompletionTimeout;
-    private Boolean keepOnCompletion;
-    private TimeValue keepAlive;
-    private final SearchRequest searchRequest;
-    // The following is optional and will only be sent down with the request if explicitely set by the user
-    private Integer batchedReduceSize;
-
-    /**
-     * Creates a new request
-     */
-    public SubmitAsyncSearchRequest(SearchSourceBuilder source, String... indices) {
-        this.searchRequest = new SearchRequest(indices, source);
-    }
-
-    /**
-     * Get the target indices
-     */
-    public String[] getIndices() {
-        return this.searchRequest.indices();
-    }
-
-    /**
-     * Get the minimum time that the request should wait before returning a partial result (defaults to 1 second).
-     */
-    public TimeValue getWaitForCompletionTimeout() {
-        return waitForCompletionTimeout;
-    }
-
-    /**
-     * Sets the minimum time that the request should wait before returning a partial result (defaults to 1 second).
-     */
-    public void setWaitForCompletionTimeout(TimeValue waitForCompletionTimeout) {
-        this.waitForCompletionTimeout = waitForCompletionTimeout;
-    }
-
-    /**
-     * Returns whether the resource resource should be kept on completion or failure (defaults to false).
-     */
-    public Boolean isKeepOnCompletion() {
-        return keepOnCompletion;
-    }
-
-    /**
-     * Determines if the resource should be kept on completion or failure (defaults to false).
-     */
-    public void setKeepOnCompletion(boolean keepOnCompletion) {
-        this.keepOnCompletion = keepOnCompletion;
-    }
-
-    /**
-     * Get the amount of time after which the result will expire (defaults to 5 days).
-     */
-    public TimeValue getKeepAlive() {
-        return keepAlive;
-    }
-
-    /**
-     * Sets the amount of time after which the result will expire (defaults to 5 days).
-     */
-    public void setKeepAlive(TimeValue keepAlive) {
-        this.keepAlive = keepAlive;
-    }
-
-    // setters for request parameters of the wrapped SearchRequest
-    /**
-     * Set the routing value to control the shards that the search will be executed on.
-     * A comma separated list of routing values to control the shards the search will be executed on.
-     */
-    public void setRouting(String routing) {
-        this.searchRequest.routing(routing);
-    }
-
-    /**
-     * Set the routing values to control the shards that the search will be executed on.
-     */
-    public void setRoutings(String... routings) {
-        this.searchRequest.routing(routings);
-    }
-
-    /**
-     * Get the routing value to control the shards that the search will be executed on.
-     */
-    public String getRouting() {
-        return this.searchRequest.routing();
-    }
-
-    /**
-     * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to
-     * {@code _local} to prefer local shards or a custom value, which guarantees that the same order
-     * will be used across different requests.
-     */
-    public void setPreference(String preference) {
-        this.searchRequest.preference(preference);
-    }
-
-    /**
-     * Get the preference to execute the search.
-     */
-    public String getPreference() {
-        return this.searchRequest.preference();
-    }
-
-    /**
-     * Specifies what type of requested indices to ignore and how to deal with indices wildcard expressions.
-     */
-    public void setIndicesOptions(IndicesOptions indicesOptions) {
-        this.searchRequest.indicesOptions(indicesOptions);
-    }
-
-    /**
-     * Get the indices Options.
-     */
-    public IndicesOptions getIndicesOptions() {
-        return this.searchRequest.indicesOptions();
-    }
-
-    /**
-     * The search type to execute, defaults to {@link SearchType#DEFAULT}.
-     */
-    public void setSearchType(SearchType searchType) {
-        this.searchRequest.searchType(searchType);
-    }
-
-    /**
-     * Get the search type to execute, defaults to {@link SearchType#DEFAULT}.
-     */
-    public SearchType getSearchType() {
-        return this.searchRequest.searchType();
-    }
-
-    /**
-     * Sets if this request should allow partial results. (If method is not called,
-     * will default to the cluster level setting).
-     */
-    public void setAllowPartialSearchResults(boolean allowPartialSearchResults) {
-        this.searchRequest.allowPartialSearchResults(allowPartialSearchResults);
-    }
-
-    /**
-     * Gets if this request should allow partial results.
-     */
-    public Boolean getAllowPartialSearchResults() {
-        return this.searchRequest.allowPartialSearchResults();
-    }
-
-    /**
-     * Optional. Sets the number of shard results that should be reduced at once on the coordinating node.
-     * This value should be used as a protection mechanism to reduce the memory overhead per search
-     * request if the potential number of shards in the request can be large. Defaults to 5.
-     */
-    public void setBatchedReduceSize(int batchedReduceSize) {
-        this.batchedReduceSize = batchedReduceSize;
-    }
-
-    /**
-     * Gets the number of shard results that should be reduced at once on the coordinating node.
-     * Returns {@code null} if unset.
-     */
-    public Integer getBatchedReduceSize() {
-        return this.batchedReduceSize;
-    }
-
-    /**
-     * Sets if this request should use the request cache or not, assuming that it can (for
-     * example, if "now" is used, it will never be cached).
-     * By default (if not set) this is turned on for {@link SubmitAsyncSearchRequest}.
-     */
-    public void setRequestCache(Boolean requestCache) {
-        this.searchRequest.requestCache(requestCache);
-    }
-
-    /**
-     * Gets if this request should use the request cache or not, if set.
-     * This defaults to `true` on the server side if unset in the client.
-     */
-    public Boolean getRequestCache() {
-        return this.searchRequest.requestCache();
-    }
-
-    /**
-     * Returns the number of shard requests that should be executed concurrently on a single node.
-     * The default is {@code 5}.
-     */
-    public int getMaxConcurrentShardRequests() {
-        return this.searchRequest.getMaxConcurrentShardRequests();
-    }
-
-    /**
-     * Sets the number of shard requests that should be executed concurrently on a single node.
-     * The default is {@code 5}.
-     */
-    public void setMaxConcurrentShardRequests(int maxConcurrentShardRequests) {
-        this.searchRequest.setMaxConcurrentShardRequests(maxConcurrentShardRequests);
-    }
-
-    /**
-     * Gets if the source of the {@link SearchSourceBuilder} initially used on this request.
-     */
-    public SearchSourceBuilder getSearchSource() {
-        return this.searchRequest.source();
-    }
-
-    @Override
-    public Optional<ValidationException> validate() {
-        final ValidationException validationException = new ValidationException();
-        if (searchRequest.isSuggestOnly()) {
-            validationException.addValidationError("suggest-only queries are not supported");
-        }
-        if (keepAlive != null && keepAlive.getMillis() < MIN_KEEP_ALIVE) {
-            validationException.addValidationError("[keep_alive] must be greater than 1 minute, got: " + keepAlive.toString());
-        }
-        if (validationException.validationErrors().isEmpty()) {
-            return Optional.empty();
-        }
-        return Optional.of(validationException);
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (o == null || getClass() != o.getClass()) {
-            return false;
-        }
-        SubmitAsyncSearchRequest request = (SubmitAsyncSearchRequest) o;
-        return Objects.equals(searchRequest, request.searchRequest)
-            && Objects.equals(getKeepAlive(), request.getKeepAlive())
-            && Objects.equals(getWaitForCompletionTimeout(), request.getWaitForCompletionTimeout())
-            && Objects.equals(isKeepOnCompletion(), request.isKeepOnCompletion());
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(searchRequest, getKeepAlive(), getWaitForCompletionTimeout(), isKeepOnCompletion());
-    }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/ProxyModeInfo.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/ProxyModeInfo.java
deleted file mode 100644
index 0e55c232cf3f..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/ProxyModeInfo.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client.cluster;
-
-import java.util.Objects;
-
-public class ProxyModeInfo implements RemoteConnectionInfo.ModeInfo {
-    static final String NAME = "proxy";
-    static final String PROXY_ADDRESS = "proxy_address";
-    static final String SERVER_NAME = "server_name";
-    static final String NUM_PROXY_SOCKETS_CONNECTED = "num_proxy_sockets_connected";
-    static final String MAX_PROXY_SOCKET_CONNECTIONS = "max_proxy_socket_connections";
-    private final String address;
-    private final String serverName;
-    private final int maxSocketConnections;
-    private final int numSocketsConnected;
-
-    ProxyModeInfo(String address, String serverName, int maxSocketConnections, int numSocketsConnected) {
-        this.address = address;
-        this.serverName = serverName;
-        this.maxSocketConnections = maxSocketConnections;
-        this.numSocketsConnected = numSocketsConnected;
-    }
-
-    @Override
-    public boolean isConnected() {
-        return numSocketsConnected > 0;
-    }
-
-    @Override
-    public String modeName() {
-        return NAME;
-    }
-
-    public String getAddress() {
-        return address;
-    }
-
-    public String getServerName() {
-        return serverName;
-    }
-
-    public int getMaxSocketConnections() {
-        return maxSocketConnections;
-    }
-
-    public int getNumSocketsConnected() {
-        return numSocketsConnected;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        ProxyModeInfo otherProxy = (ProxyModeInfo) o;
-        return maxSocketConnections == otherProxy.maxSocketConnections
-            && numSocketsConnected == otherProxy.numSocketsConnected
-            && Objects.equals(address, otherProxy.address)
-            && Objects.equals(serverName, otherProxy.serverName);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(address, serverName, maxSocketConnections, numSocketsConnected);
-    }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteConnectionInfo.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteConnectionInfo.java
deleted file mode 100644
index f5069d7771d7..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteConnectionInfo.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client.cluster;
-
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentParser;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Objects;
-
-import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
-import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
-
-/**
- * This class encapsulates all remote cluster information to be rendered on
- * {@code _remote/info} requests.
- */
-public final class RemoteConnectionInfo {
-    private static final String CONNECTED = "connected";
-    private static final String MODE = "mode";
-    private static final String INITIAL_CONNECT_TIMEOUT = "initial_connect_timeout";
-    private static final String SKIP_UNAVAILABLE = "skip_unavailable";
-
-    @SuppressWarnings("unchecked")
-    private static final ConstructingObjectParser<RemoteConnectionInfo, String> PARSER = new ConstructingObjectParser<>(
-        "RemoteConnectionInfoObjectParser",
-        false,
-        (args, clusterAlias) -> {
-            String mode = (String) args[1];
-            ModeInfo modeInfo;
-            if (mode.equals(ProxyModeInfo.NAME)) {
-                modeInfo = new ProxyModeInfo((String) args[4], (String) args[5], (int) args[6], (int) args[7]);
-            } else if (mode.equals(SniffModeInfo.NAME)) {
-                modeInfo = new SniffModeInfo((List<String>) args[8], (int) args[9], (int) args[10]);
-            } else {
-                throw new IllegalArgumentException("mode cannot be " + mode);
-            }
-            return new RemoteConnectionInfo(clusterAlias, modeInfo, (String) args[2], (boolean) args[3]);
-        }
-    );
-
-    static {
-        PARSER.declareBoolean(constructorArg(), new ParseField(CONNECTED));
-        PARSER.declareString(constructorArg(), new ParseField(MODE));
-        PARSER.declareString(constructorArg(), new ParseField(INITIAL_CONNECT_TIMEOUT));
-        PARSER.declareBoolean(constructorArg(), new ParseField(SKIP_UNAVAILABLE));
-
-        PARSER.declareString(optionalConstructorArg(), new ParseField(ProxyModeInfo.PROXY_ADDRESS));
-        PARSER.declareString(optionalConstructorArg(), new ParseField(ProxyModeInfo.SERVER_NAME));
-        PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.MAX_PROXY_SOCKET_CONNECTIONS));
-        PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.NUM_PROXY_SOCKETS_CONNECTED));
-
-        PARSER.declareStringArray(optionalConstructorArg(), new ParseField(SniffModeInfo.SEEDS));
-        PARSER.declareInt(optionalConstructorArg(), new ParseField(SniffModeInfo.MAX_CONNECTIONS_PER_CLUSTER));
-        PARSER.declareInt(optionalConstructorArg(), new ParseField(SniffModeInfo.NUM_NODES_CONNECTED));
-    }
-
-    private final ModeInfo modeInfo;
-    // TODO: deprecate and remove this field in favor of initialConnectionTimeout field that is of type TimeValue.
-    // When rest api versioning exists then change org.elasticsearch.transport.RemoteConnectionInfo to properly serialize
-    // the initialConnectionTimeout field so that we can properly parse initialConnectionTimeout as TimeValue
-    private final String initialConnectionTimeoutString;
-    private final String clusterAlias;
-    private final boolean skipUnavailable;
-
-    RemoteConnectionInfo(String clusterAlias, ModeInfo modeInfo, String initialConnectionTimeoutString, boolean skipUnavailable) {
-        this.clusterAlias = clusterAlias;
-        this.modeInfo = modeInfo;
-        this.initialConnectionTimeoutString = initialConnectionTimeoutString;
-        this.skipUnavailable = skipUnavailable;
-    }
-
-    public boolean isConnected() {
-        return modeInfo.isConnected();
-    }
-
-    public String getClusterAlias() {
-        return clusterAlias;
-    }
-
-    public ModeInfo getModeInfo() {
-        return modeInfo;
-    }
-
-    public String getInitialConnectionTimeoutString() {
-        return initialConnectionTimeoutString;
-    }
-
-    public boolean isSkipUnavailable() {
-        return skipUnavailable;
-    }
-
-    public static RemoteConnectionInfo fromXContent(XContentParser parser, String clusterAlias) throws IOException {
-        return PARSER.parse(parser, clusterAlias);
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        RemoteConnectionInfo that = (RemoteConnectionInfo) o;
-        return skipUnavailable == that.skipUnavailable
-            && Objects.equals(modeInfo, that.modeInfo)
-            && Objects.equals(initialConnectionTimeoutString, that.initialConnectionTimeoutString)
-            && Objects.equals(clusterAlias, that.clusterAlias);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(modeInfo, initialConnectionTimeoutString, clusterAlias, skipUnavailable);
-    }
-
-    public interface ModeInfo {
-
-        boolean isConnected();
-
-        String modeName();
-    }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteInfoRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteInfoRequest.java
deleted file mode 100644
index 7672c1054c04..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteInfoRequest.java
+++ /dev/null
@@ -1,17 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.client.cluster;
-
-import org.elasticsearch.client.Validatable;
-
-/**
- * The request object used by the Remote cluster info API.
- */
-public final class RemoteInfoRequest implements Validatable {
-
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteInfoResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteInfoResponse.java
deleted file mode 100644
index 9aff43bf3871..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteInfoResponse.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.client.cluster;
-
-import org.elasticsearch.xcontent.XContentParser;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
-
-/**
- * A response to _remote/info API request.
- */
-public final class RemoteInfoResponse {
-
-    private List<RemoteConnectionInfo> infos;
-
-    RemoteInfoResponse(Collection<RemoteConnectionInfo> infos) {
-        this.infos = List.copyOf(infos);
-    }
-
-    public List<RemoteConnectionInfo> getInfos() {
-        return infos;
-    }
-
-    public static RemoteInfoResponse fromXContent(XContentParser parser) throws IOException {
-        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
-
-        List<RemoteConnectionInfo> infos = new ArrayList<>();
-
-        XContentParser.Token token;
-        while ((token = parser.nextToken()) == XContentParser.Token.FIELD_NAME) {
-            String clusterAlias = parser.currentName();
-            RemoteConnectionInfo info = RemoteConnectionInfo.fromXContent(parser, clusterAlias);
-            infos.add(info);
-        }
-        ensureExpectedToken(XContentParser.Token.END_OBJECT, token, parser);
-        return new RemoteInfoResponse(infos);
-    }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/SniffModeInfo.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/SniffModeInfo.java
deleted file mode 100644
index e08509dd14b6..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/SniffModeInfo.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client.cluster;
-
-import java.util.List;
-import java.util.Objects;
-
-public class SniffModeInfo implements RemoteConnectionInfo.ModeInfo {
-    public static final String NAME = "sniff";
-    static final String SEEDS = "seeds";
-    static final String NUM_NODES_CONNECTED = "num_nodes_connected";
-    static final String MAX_CONNECTIONS_PER_CLUSTER = "max_connections_per_cluster";
-    final List<String> seedNodes;
-    final int maxConnectionsPerCluster;
-    final int numNodesConnected;
-
-    SniffModeInfo(List<String> seedNodes, int maxConnectionsPerCluster, int numNodesConnected) {
-        this.seedNodes = seedNodes;
-        this.maxConnectionsPerCluster = maxConnectionsPerCluster;
-        this.numNodesConnected = numNodesConnected;
-    }
-
-    @Override
-    public boolean isConnected() {
-        return numNodesConnected > 0;
-    }
-
-    @Override
-    public String modeName() {
-        return NAME;
-    }
-
-    public List<String> getSeedNodes() {
-        return seedNodes;
-    }
-
-    public int getMaxConnectionsPerCluster() {
-        return maxConnectionsPerCluster;
-    }
-
-    public int getNumNodesConnected() {
-        return numNodesConnected;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        SniffModeInfo sniff = (SniffModeInfo) o;
-        return maxConnectionsPerCluster == sniff.maxConnectionsPerCluster
-            && numNodesConnected == sniff.numNodesConnected
-            && Objects.equals(seedNodes, sniff.seedNodes);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(seedNodes, maxConnectionsPerCluster, numNodesConnected);
-    }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/BroadcastResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/BroadcastResponse.java
deleted file mode 100644
index ef3b35a0896d..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/BroadcastResponse.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client.core;
-
-import org.elasticsearch.action.support.DefaultShardOperationFailedException;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentParser;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Objects;
-
-/**
- * Represents a response to a request that is broadcast to a collection of shards.
- */
-public class BroadcastResponse {
-
-    private final Shards shards;
-
-    /**
-     * Represents the shard-level summary of the response execution.
-     *
-     * @return the shard-level response summary
-     */
-    public Shards shards() {
-        return shards;
-    }
-
-    protected BroadcastResponse(final Shards shards) {
-        this.shards = Objects.requireNonNull(shards);
-    }
-
-    private static final ParseField SHARDS_FIELD = new ParseField("_shards");
-
-    static final ConstructingObjectParser<BroadcastResponse, Void> PARSER = new ConstructingObjectParser<>(
-        "broadcast_response",
-        a -> new BroadcastResponse((Shards) a[0])
-    );
-
-    static {
-        declareShardsField(PARSER);
-    }
-
-    /**
-     * Parses a broadcast response.
-     *
-     * @param parser the parser
-     * @return a broadcast response parsed from the specified parser
-     * @throws IOException if an I/O exception occurs parsing the response
-     */
-    public static BroadcastResponse fromXContent(final XContentParser parser) throws IOException {
-        return PARSER.parse(parser, null);
-    }
-
-    protected static <T extends BroadcastResponse> void declareShardsField(ConstructingObjectParser<T, Void> parser) {
-        parser.declareObject(ConstructingObjectParser.constructorArg(), Shards.SHARDS_PARSER, SHARDS_FIELD);
-    }
-
-    /**
-     * Represents the results of a collection of shards on which a request was executed against.
-     */
-    public static class Shards {
-
-        private final int total;
-
-        /**
-         * The total number of shards on which a request was executed against.
-         *
-         * @return the total number of shards
-         */
-        public int total() {
-            return total;
-        }
-
-        private final int successful;
-
-        /**
-         * The number of successful shards on which a request was executed against.
-         *
-         * @return the number of successful shards
-         */
-        public int successful() {
-            return successful;
-        }
-
-        private final int skipped;
-
-        /**
-         * The number of shards skipped by the request.
-         *
-         * @return the number of skipped shards
-         */
-        public int skipped() {
-            return skipped;
-        }
-
-        private final int failed;
-
-        /**
-         * The number of shards on which a request failed to be executed against.
-         *
-         * @return the number of failed shards
-         */
-        public int failed() {
-            return failed;
-        }
-
-        private final Collection<DefaultShardOperationFailedException> failures;
-
-        /**
-         * The failures corresponding to the shards on which a request failed to be executed against. Note that the number of failures might
-         * not match {@link #failed()} as some responses group together shard failures.
-         *
-         * @return the failures
-         */
-        public Collection<DefaultShardOperationFailedException> failures() {
-            return failures;
-        }
-
-        Shards(
-            final int total,
-            final int successful,
-            final int skipped,
-            final int failed,
-            final Collection<DefaultShardOperationFailedException> failures
-        ) {
-            this.total = total;
-            this.successful = successful;
-            this.skipped = skipped;
-            this.failed = failed;
-            this.failures = Collections.unmodifiableCollection(Objects.requireNonNull(failures));
-        }
-
-        private static final ParseField TOTAL_FIELD = new ParseField("total");
-        private static final ParseField SUCCESSFUL_FIELD = new ParseField("successful");
-        private static final ParseField SKIPPED_FIELD = new ParseField("skipped");
-        private static final ParseField FAILED_FIELD = new ParseField("failed");
-        private static final ParseField FAILURES_FIELD = new ParseField("failures");
-
-        @SuppressWarnings("unchecked")
-        static final ConstructingObjectParser<Shards, Void> SHARDS_PARSER = new ConstructingObjectParser<>(
-            "shards",
-            a -> new Shards(
-                (int) a[0], // total
-                (int) a[1], // successful
-                a[2] == null ? 0 : (int) a[2], // skipped
-                (int) a[3], // failed
-                a[4] == null ? Collections.emptyList() : (Collection<DefaultShardOperationFailedException>) a[4]
-            )
-        ); // failures
-
-        static {
-            SHARDS_PARSER.declareInt(ConstructingObjectParser.constructorArg(), TOTAL_FIELD);
-            SHARDS_PARSER.declareInt(ConstructingObjectParser.constructorArg(), SUCCESSFUL_FIELD);
-            SHARDS_PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), SKIPPED_FIELD);
-            SHARDS_PARSER.declareInt(ConstructingObjectParser.constructorArg(), FAILED_FIELD);
-            SHARDS_PARSER.declareObjectArray(
-                ConstructingObjectParser.optionalConstructorArg(),
-                DefaultShardOperationFailedException.PARSER,
-                FAILURES_FIELD
-            );
-        }
-
-    }
-
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/CountRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/CountRequest.java
deleted file mode 100644
index 0899eb03311d..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/CountRequest.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client.core;
-
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Validatable;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Objects;
-
-/**
- * Encapsulates a request to _count API against one, several or all indices.
- */
-public final class CountRequest implements Validatable, ToXContentObject {
-
-    private String[] indices = Strings.EMPTY_ARRAY;
-    private String[] types = Strings.EMPTY_ARRAY;
-    private String routing;
-    private String preference;
-    private QueryBuilder query;
-    private IndicesOptions indicesOptions;
-    private int terminateAfter = SearchContext.DEFAULT_TERMINATE_AFTER;
-    private Float minScore;
-
-    public CountRequest() {}
-
-    /**
-     * Constructs a new count request against the indices. No indices provided here means that count will execute on all indices.
-     */
-    public CountRequest(String... indices) {
-        indices(indices);
-    }
-
-    /**
-     * Constructs a new search request against the provided indices with the given search source.
-     *
-     * @deprecated The count api only supports a query. Use {@link #CountRequest(String[], QueryBuilder)} instead.
-     */
-    @Deprecated
-    public CountRequest(String[] indices, SearchSourceBuilder searchSourceBuilder) {
-        indices(indices);
-        this.query = Objects.requireNonNull(searchSourceBuilder, "source must not be null").query();
-    }
-
-    /**
-     * Constructs a new search request against the provided indices with the given query.
-     */
-    public CountRequest(String[] indices, QueryBuilder query) {
-        indices(indices);
-        this.query = Objects.requireNonNull(query, "query must not be null");
-        ;
-    }
-
-    /**
-     * Sets the indices the count will be executed on.
-     */
-    public CountRequest indices(String... indices) {
-        Objects.requireNonNull(indices, "indices must not be null");
-        for (String index : indices) {
-            Objects.requireNonNull(index, "index must not be null");
-        }
-        this.indices = indices;
-        return this;
-    }
-
-    /**
-     * The source of the count request.
-     *
-     * @deprecated The count api only supports a query. Use {@link #query(QueryBuilder)} instead.
-     */
-    @Deprecated
-    public CountRequest source(SearchSourceBuilder searchSourceBuilder) {
-        this.query = Objects.requireNonNull(searchSourceBuilder, "source must not be null").query();
-        return this;
-    }
-
-    /**
-     * Sets the query to execute for this count request.
-     */
-    public CountRequest query(QueryBuilder query) {
-        this.query = Objects.requireNonNull(query, "query must not be null");
-        return this;
-    }
-
-    /**
-     * The document types to execute the count against. Defaults to be executed against all types.
-     *
-     * @deprecated Types are in the process of being removed. Instead of using a type, prefer to
-     * filter on a field on the document.
-     */
-    @Deprecated
-    public CountRequest types(String... types) {
-        Objects.requireNonNull(types, "types must not be null");
-        for (String type : types) {
-            Objects.requireNonNull(type, "type must not be null");
-        }
-        this.types = types;
-        return this;
-    }
-
-    /**
-     * The routing values to control the shards that the search will be executed on.
-     */
-    public CountRequest routing(String routing) {
-        this.routing = routing;
-        return this;
-    }
-
-    /**
-     * A comma separated list of routing values to control the shards the count will be executed on.
-     */
-    public CountRequest routing(String... routings) {
-        this.routing = Strings.arrayToCommaDelimitedString(routings);
-        return this;
-    }
-
-    /**
-     * Returns the indices options used to resolve indices. They tell for instance whether a single index is accepted, whether an empty
-     * array will be converted to _all, and how wildcards will be expanded if needed.
-     *
-     * @see org.elasticsearch.action.support.IndicesOptions
-     */
-    public CountRequest indicesOptions(IndicesOptions indicesOptions) {
-        this.indicesOptions = Objects.requireNonNull(indicesOptions, "indicesOptions must not be null");
-        return this;
-    }
-
-    /**
-     * Sets the preference to execute the count. Defaults to randomize across shards. Can be set to {@code _local} to prefer local shards
-     * or a custom value, which guarantees that the same order will be used across different requests.
-     */
-    public CountRequest preference(String preference) {
-        this.preference = preference;
-        return this;
-    }
-
-    public IndicesOptions indicesOptions() {
-        return this.indicesOptions;
-    }
-
-    public String routing() {
-        return this.routing;
-    }
-
-    public String preference() {
-        return this.preference;
-    }
-
-    public String[] indices() {
-        return Arrays.copyOf(this.indices, this.indices.length);
-    }
-
-    public Float minScore() {
-        return minScore;
-    }
-
-    public CountRequest minScore(Float minScore) {
-        this.minScore = minScore;
-        return this;
-    }
-
-    public int terminateAfter() {
-        return this.terminateAfter;
-    }
-
-    public CountRequest terminateAfter(int terminateAfter) {
-        if (terminateAfter < 0) {
-            throw new IllegalArgumentException("terminateAfter must be > 0");
-        }
-        this.terminateAfter = terminateAfter;
-        return this;
-    }
-
-    /**
-     * @deprecated Types are in the process of being removed. Instead of using a type, prefer to
-     * filter on a field on the document.
-     */
-    @Deprecated
-    public String[] types() {
-        return Arrays.copyOf(this.types, this.types.length);
-    }
-
-    /**
-     * @return the source builder
-     * @deprecated The count api only supports a query. Use {@link #query()} instead.
-     */
-    @Deprecated
-    public SearchSourceBuilder source() {
-        return new SearchSourceBuilder().query(query);
-    }
-
-    /**
-     * @return The provided query to execute with the count request or
-     * <code>null</code> if no query was provided.
-     */
-    public QueryBuilder query() {
-        return query;
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        if (query != null) {
-            builder.field("query", query);
-        }
-        builder.endObject();
-        return builder;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (o == null || getClass() != o.getClass()) {
-            return false;
-        }
-        CountRequest that = (CountRequest) o;
-        return Objects.equals(indicesOptions, that.indicesOptions)
-            && Arrays.equals(indices, that.indices)
-            && Arrays.equals(types, that.types)
-            && Objects.equals(routing, that.routing)
-            && Objects.equals(preference, that.preference)
-            && Objects.equals(terminateAfter, that.terminateAfter)
-            && Objects.equals(minScore, that.minScore)
-            && Objects.equals(query, that.query);
-    }
-
-    @Override
-    public int hashCode() {
-        int result = Objects.hash(indicesOptions, routing, preference, terminateAfter, minScore, query);
-        result = 31 * result + Arrays.hashCode(indices);
-        result = 31 * result + Arrays.hashCode(types);
-        return result;
-    }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/CountResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/CountResponse.java
deleted file mode 100644
index c19245c4d09f..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/CountResponse.java
+++ /dev/null
@@ -1,230 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client.core;
-
-import org.elasticsearch.action.search.ShardSearchFailure;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentParser;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
-
-/**
- * A response to _count API request.
- */
-public final class CountResponse {
-
-    static final ParseField COUNT = new ParseField("count");
-    static final ParseField TERMINATED_EARLY = new ParseField("terminated_early");
-    static final ParseField SHARDS = new ParseField("_shards");
-
-    private final long count;
-    private final Boolean terminatedEarly;
-    private final ShardStats shardStats;
-
-    public CountResponse(long count, Boolean terminatedEarly, ShardStats shardStats) {
-        this.count = count;
-        this.terminatedEarly = terminatedEarly;
-        this.shardStats = shardStats;
-    }
-
-    public ShardStats getShardStats() {
-        return shardStats;
-    }
-
-    /**
-     * Number of documents matching request.
-     */
-    public long getCount() {
-        return count;
-    }
-
-    /**
-     * The total number of shards the search was executed on.
-     */
-    public int getTotalShards() {
-        return shardStats.totalShards;
-    }
-
-    /**
-     * The successful number of shards the search was executed on.
-     */
-    public int getSuccessfulShards() {
-        return shardStats.successfulShards;
-    }
-
-    /**
-     * The number of shards skipped due to pre-filtering
-     */
-    public int getSkippedShards() {
-        return shardStats.skippedShards;
-    }
-
-    /**
-     * The failed number of shards the search was executed on.
-     */
-    public int getFailedShards() {
-        return shardStats.shardFailures.length;
-    }
-
-    /**
-     * The failures that occurred during the search.
-     */
-    public ShardSearchFailure[] getShardFailures() {
-        return shardStats.shardFailures;
-    }
-
-    public RestStatus status() {
-        return RestStatus.status(shardStats.successfulShards, shardStats.totalShards, shardStats.shardFailures);
-    }
-
-    public static CountResponse fromXContent(XContentParser parser) throws IOException {
-        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
-        parser.nextToken();
-        ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser);
-        String currentName = parser.currentName();
-        Boolean terminatedEarly = null;
-        long count = 0;
-        ShardStats shardStats = new ShardStats(-1, -1, 0, ShardSearchFailure.EMPTY_ARRAY);
-
-        for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentName = parser.currentName();
-            } else if (token.isValue()) {
-                if (COUNT.match(currentName, parser.getDeprecationHandler())) {
-                    count = parser.longValue();
-                } else if (TERMINATED_EARLY.match(currentName, parser.getDeprecationHandler())) {
-                    terminatedEarly = parser.booleanValue();
-                } else {
-                    parser.skipChildren();
-                }
-            } else if (token == XContentParser.Token.START_OBJECT) {
-                if (SHARDS.match(currentName, parser.getDeprecationHandler())) {
-                    shardStats = ShardStats.fromXContent(parser);
-                } else {
-                    parser.skipChildren();
-                }
-            }
-        }
-        return new CountResponse(count, terminatedEarly, shardStats);
-    }
-
-    @Override
-    public String toString() {
-        String s = "{"
-            + "count="
-            + count
-            + (isTerminatedEarly() != null ? ", terminatedEarly=" + terminatedEarly : "")
-            + ", "
-            + shardStats
-            + '}';
-        return s;
-    }
-
-    public Boolean isTerminatedEarly() {
-        return terminatedEarly;
-    }
-
-    /**
-     * Encapsulates _shards section of count api response.
-     */
-    public static final class ShardStats {
-
-        static final ParseField FAILED = new ParseField("failed");
-        static final ParseField SKIPPED = new ParseField("skipped");
-        static final ParseField TOTAL = new ParseField("total");
-        static final ParseField SUCCESSFUL = new ParseField("successful");
-        static final ParseField FAILURES = new ParseField("failures");
-
-        private final int successfulShards;
-        private final int totalShards;
-        private final int skippedShards;
-        private final ShardSearchFailure[] shardFailures;
-
-        public ShardStats(int successfulShards, int totalShards, int skippedShards, ShardSearchFailure[] shardFailures) {
-            this.successfulShards = successfulShards;
-            this.totalShards = totalShards;
-            this.skippedShards = skippedShards;
-            this.shardFailures = Arrays.copyOf(shardFailures, shardFailures.length);
-        }
-
-        public int getSuccessfulShards() {
-            return successfulShards;
-        }
-
-        public int getTotalShards() {
-            return totalShards;
-        }
-
-        public int getSkippedShards() {
-            return skippedShards;
-        }
-
-        public ShardSearchFailure[] getShardFailures() {
-            return Arrays.copyOf(shardFailures, shardFailures.length, ShardSearchFailure[].class);
-        }
-
-        static ShardStats fromXContent(XContentParser parser) throws IOException {
-            int successfulShards = -1;
-            int totalShards = -1;
-            int skippedShards = 0; // BWC @see org.elasticsearch.action.search.SearchResponse
-            List<ShardSearchFailure> failures = new ArrayList<>();
-            XContentParser.Token token;
-            String currentName = parser.currentName();
-            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    currentName = parser.currentName();
-                } else if (token.isValue()) {
-                    if (FAILED.match(currentName, parser.getDeprecationHandler())) {
-                        parser.intValue();
-                    } else if (SKIPPED.match(currentName, parser.getDeprecationHandler())) {
-                        skippedShards = parser.intValue();
-                    } else if (TOTAL.match(currentName, parser.getDeprecationHandler())) {
-                        totalShards = parser.intValue();
-                    } else if (SUCCESSFUL.match(currentName, parser.getDeprecationHandler())) {
-                        successfulShards = parser.intValue();
-                    } else {
-                        parser.skipChildren();
-                    }
-                } else if (token == XContentParser.Token.START_ARRAY) {
-                    if (FAILURES.match(currentName, parser.getDeprecationHandler())) {
-                        while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                            failures.add(ShardSearchFailure.fromXContent(parser));
-                        }
-                    } else {
-                        parser.skipChildren();
-                    }
-                } else {
-                    parser.skipChildren();
-                }
-            }
-            return new ShardStats(successfulShards, totalShards, skippedShards, failures.toArray(new ShardSearchFailure[failures.size()]));
-        }
-
-        @Override
-        public String toString() {
-            return "_shards : {"
-                + "total="
-                + totalShards
-                + ", successful="
-                + successfulShards
-                + ", skipped="
-                + skippedShards
-                + ", failed="
-                + (shardFailures != null && shardFailures.length > 0 ? shardFailures.length : 0)
-                + (shardFailures != null && shardFailures.length > 0 ? ", failures: " + Arrays.asList(shardFailures) : "")
-                + '}';
-        }
-    }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/GetSourceRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/GetSourceRequest.java
deleted file mode 100644
index 6e26457a27a5..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/GetSourceRequest.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client.core;
-
-import org.elasticsearch.action.get.GetRequest;
-import org.elasticsearch.client.Validatable;
-import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
-
-public final class GetSourceRequest implements Validatable {
-    private String routing;
-    private String preference;
-
-    private boolean refresh = false;
-    private boolean realtime = true;
-
-    private FetchSourceContext fetchSourceContext;
-
-    private final String index;
-    private final String id;
-
-    public GetSourceRequest(String index, String id) {
-        this.index = index;
-        this.id = id;
-    }
-
-    public static GetSourceRequest from(GetRequest getRequest) {
-        return new GetSourceRequest(getRequest.index(), getRequest.id()).routing(getRequest.routing())
-            .preference(getRequest.preference())
-            .refresh(getRequest.refresh())
-            .realtime(getRequest.realtime())
-            .fetchSourceContext(getRequest.fetchSourceContext());
-    }
-
-    /**
-     * Controls the shard routing of the request. Using this value to hash the shard
-     * and not the id.
-     */
-    public GetSourceRequest routing(String routing) {
-        if (routing != null && routing.length() == 0) {
-            this.routing = null;
-        } else {
-            this.routing = routing;
-        }
-        return this;
-    }
-
-    /**
-     * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to
-     * {@code _local} to prefer local shards or a custom value, which guarantees that the same order
-     * will be used across different requests.
-     */
-    public GetSourceRequest preference(String preference) {
-        this.preference = preference;
-        return this;
-    }
-
-    /**
-     * Should a refresh be executed before this get operation causing the operation to
-     * return the latest value. Note, heavy get should not set this to {@code true}. Defaults
-     * to {@code false}.
-     */
-    public GetSourceRequest refresh(boolean refresh) {
-        this.refresh = refresh;
-        return this;
-    }
-
-    public GetSourceRequest realtime(boolean realtime) {
-        this.realtime = realtime;
-        return this;
-    }
-
-    /**
-     * Allows setting the {@link FetchSourceContext} for this request, controlling if and how _source should be returned.
-     * Note, the {@code fetchSource} field of the context must be set to {@code true}.
-     */
-
-    public GetSourceRequest fetchSourceContext(FetchSourceContext context) {
-        this.fetchSourceContext = context;
-        return this;
-    }
-
-    public String index() {
-        return index;
-    }
-
-    public String id() {
-        return id;
-    }
-
-    public String routing() {
-        return routing;
-    }
-
-    public String preference() {
-        return preference;
-    }
-
-    public boolean refresh() {
-        return refresh;
-    }
-
-    public boolean realtime() {
-        return realtime;
-    }
-
-    public FetchSourceContext fetchSourceContext() {
-        return fetchSourceContext;
-    }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/GetSourceResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/GetSourceResponse.java
deleted file mode 100644
index 45469cf1d1fb..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/GetSourceResponse.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client.core;
-
-import org.elasticsearch.xcontent.XContentParser;
-
-import java.io.IOException;
-import java.util.Map;
-
-public final class GetSourceResponse {
-
-    private final Map<String, Object> source;
-
-    public GetSourceResponse(Map<String, Object> source) {
-        this.source = source;
-    }
-
-    public static GetSourceResponse fromXContent(XContentParser parser) throws IOException {
-        return new GetSourceResponse(parser.map());
-    }
-
-    public Map<String, Object> getSource() {
-        return this.source;
-    }
-
-    @Override
-    public String toString() {
-        return source.toString();
-    }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerJobStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerJobStats.java
deleted file mode 100644
index e404f254e17a..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerJobStats.java
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client.core;
-
-import org.elasticsearch.xcontent.ParseField;
-
-import java.util.Objects;
-
-public abstract class IndexerJobStats {
-    public static ParseField NUM_PAGES = new ParseField("pages_processed");
-    public static ParseField NUM_INPUT_DOCUMENTS = new ParseField("documents_processed");
-    public static ParseField NUM_OUTPUT_DOCUMENTS = new ParseField("documents_indexed");
-    public static ParseField NUM_INVOCATIONS = new ParseField("trigger_count");
-    public static ParseField INDEX_TIME_IN_MS = new ParseField("index_time_in_ms");
-    public static ParseField SEARCH_TIME_IN_MS = new ParseField("search_time_in_ms");
-    public static ParseField PROCESSING_TIME_IN_MS = new ParseField("processing_time_in_ms");
-    public static ParseField INDEX_TOTAL = new ParseField("index_total");
-    public static ParseField SEARCH_TOTAL = new ParseField("search_total");
-    public static ParseField PROCESSING_TOTAL = new ParseField("processing_total");
-    public static ParseField SEARCH_FAILURES = new ParseField("search_failures");
-    public static ParseField INDEX_FAILURES = new ParseField("index_failures");
-
-    protected final long numPages;
-    protected final long numInputDocuments;
-    protected final long numOuputDocuments;
-    protected final long numInvocations;
-    protected final long indexTime;
-    protected final long indexTotal;
-    protected final long searchTime;
-    protected final long searchTotal;
-    protected final long processingTime;
-    protected final long processingTotal;
-    protected final long indexFailures;
-    protected final long searchFailures;
-
-    public IndexerJobStats(
-        long numPages,
-        long numInputDocuments,
-        long numOutputDocuments,
-        long numInvocations,
-        long indexTime,
-        long searchTime,
-        long processingTime,
-        long indexTotal,
-        long searchTotal,
-        long processingTotal,
-        long indexFailures,
-        long searchFailures
-    ) {
-        this.numPages = numPages;
-        this.numInputDocuments = numInputDocuments;
-        this.numOuputDocuments = numOutputDocuments;
-        this.numInvocations = numInvocations;
-        this.indexTime = indexTime;
-        this.indexTotal = indexTotal;
-        this.searchTime = searchTime;
-        this.searchTotal = searchTotal;
-        this.processingTime = processingTime;
-        this.processingTotal = processingTotal;
-        this.indexFailures = indexFailures;
-        this.searchFailures = searchFailures;
-    }
-
-    /**
-     * The number of pages read from the input indices
-     */
-    public long getNumPages() {
-        return numPages;
-    }
-
-    /**
-     * The number of documents read from the input indices
-     */
-    public long getNumDocuments() {
-        return numInputDocuments;
-    }
-
-    /**
-     * Number of times that the job woke up to write documents
-     */
-    public long getNumInvocations() {
-        return numInvocations;
-    }
-
-    /**
-     * Number of documents written
-     */
-    public long getOutputDocuments() {
-        return numOuputDocuments;
-    }
-
-    /**
-     * Number of index failures that have occurred
-     */
-    public long getIndexFailures() {
-        return indexFailures;
-    }
-
-    /**
-     * Number of failures that have occurred
-     */
-    public long getSearchFailures() {
-        return searchFailures;
-    }
-
-    /**
-     * Returns the time spent indexing (cumulative) in milliseconds
-     */
-    public long getIndexTime() {
-        return indexTime;
-    }
-
-    /**
-     * Returns the time spent searching (cumulative) in milliseconds
-     */
-    public long getSearchTime() {
-        return searchTime;
-    }
-
-    /**
-     * Returns the time spent processing (cumulative) in milliseconds
-     */
-    public long getProcessingTime() {
-        return processingTime;
-    }
-
-    /**
-     * Returns the total number of indexing requests that have been processed
-     * (Note: this is not the number of _documents_ that have been indexed)
-     */
-    public long getIndexTotal() {
-        return indexTotal;
-    }
-
-    /**
-     * Returns the total number of search requests that have been made
-     */
-    public long getSearchTotal() {
-        return searchTotal;
-    }
-
-    /**
-     * Returns the total number of processing runs that have been made
-     */
-    public long getProcessingTotal() {
-        return processingTotal;
-    }
-
-    @Override
-    public boolean equals(Object other) {
-        if (this == other) {
-            return true;
-        }
-
-        if (other instanceof IndexerJobStats == false) {
-            return false;
-        }
-
-        IndexerJobStats that = (IndexerJobStats) other;
-        return Objects.equals(this.numPages, that.numPages)
-            && Objects.equals(this.numInputDocuments, that.numInputDocuments)
-            && Objects.equals(this.numOuputDocuments, that.numOuputDocuments)
-            && Objects.equals(this.numInvocations, that.numInvocations)
-            && Objects.equals(this.indexTime, that.indexTime)
-            && Objects.equals(this.searchTime, that.searchTime)
-            && Objects.equals(this.processingTime, that.processingTime)
-            && Objects.equals(this.indexFailures, that.indexFailures)
-            && Objects.equals(this.searchFailures, that.searchFailures)
-            && Objects.equals(this.searchTotal, that.searchTotal)
-            && Objects.equals(this.processingTotal, that.processingTotal)
-            && Objects.equals(this.indexTotal, that.indexTotal);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(
-            numPages,
-            numInputDocuments,
-            numOuputDocuments,
-            numInvocations,
-            indexTime,
-            searchTime,
-            processingTime,
-            indexFailures,
-            searchFailures,
-            searchTotal,
-            indexTotal,
-            processingTotal
-        );
-    }
-
-    @Override
-    public final String toString() {
-        return "{pages="
-            + numPages
-            + ", input_docs="
-            + numInputDocuments
-            + ", output_docs="
-            + numOuputDocuments
-            + ", invocations="
-            + numInvocations
-            + ", index_failures="
-            + indexFailures
-            + ", search_failures="
-            + searchFailures
-            + ", index_time_in_ms="
-            + indexTime
-            + ", index_total="
-            + indexTotal
-            + ", search_time_in_ms="
-            + searchTime
-            + ", search_total="
-            + searchTotal
-            + ", processing_time_in_ms="
-            + processingTime
-            + ", processing_total="
-            + processingTotal
-            + "}";
-    }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerState.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerState.java
deleted file mode 100644
index 95ca55505261..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerState.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client.core;
-
-import java.util.Locale;
-
-/**
- * IndexerState represents the internal state of the indexer.  It
- * is also persistent when changing from started/stopped in case the allocated
- * task is restarted elsewhere.
- */
-public enum IndexerState {
-    /** Indexer is running, but not actively indexing data (e.g. it's idle). */
-    STARTED,
-
-    /** Indexer is actively indexing data. */
-    INDEXING,
-
-    /**
-     * Transition state to where an indexer has acknowledged the stop
-     * but is still in process of halting.
-     */
-    STOPPING,
-
-    /** Indexer is "paused" and ignoring scheduled triggers. */
-    STOPPED,
-
-    /**
-     * Something (internal or external) has requested the indexer abort
-     * and shutdown.
-     */
-    ABORTING;
-
-    public static IndexerState fromString(String name) {
-        return valueOf(name.trim().toUpperCase(Locale.ROOT));
-    }
-
-    public String value() {
-        return name().toLowerCase(Locale.ROOT);
-    }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/MainRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/MainRequest.java
deleted file mode 100644
index 592d98674372..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/MainRequest.java
+++ /dev/null
@@ -1,13 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client.core;
-
-import org.elasticsearch.client.Validatable;
-
-public class MainRequest implements Validatable {}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/MultiTermVectorsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/MultiTermVectorsRequest.java
deleted file mode 100644
index cdd98bb12a56..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/MultiTermVectorsRequest.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client.core;
-
-import org.elasticsearch.client.Validatable;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.elasticsearch.client.core.TermVectorsRequest.createFromTemplate;
-
-public class MultiTermVectorsRequest implements ToXContentObject, Validatable {
-
-    private List<TermVectorsRequest> requests = new ArrayList<>();
-
-    /**
-     * Constructs an empty MultiTermVectorsRequest
-     * After that use {@code add} method to add individual {@code TermVectorsRequest} to it.
-     */
-    public MultiTermVectorsRequest() {}
-
-    /**
-     * Constructs a MultiTermVectorsRequest from the given document ids
-     * and a template {@code TermVectorsRequest}.
-     * Used when individual requests share the same index, type and other settings.
-     * @param ids - ids of documents for which term vectors are requested
-     * @param template - a template {@code TermVectorsRequest} that allows to set all
-     * settings only once for all requests.
-     */
-    public MultiTermVectorsRequest(String[] ids, TermVectorsRequest template) {
-        for (String id : ids) {
-            TermVectorsRequest request = createFromTemplate(template, id);
-            requests.add(request);
-        }
-    }
-
-    /**
-     * Adds another {@code TermVectorsRequest} to this {@code MultiTermVectorsRequest}
-     * @param request - {@code TermVectorsRequest} to add
-     */
-    public void add(TermVectorsRequest request) {
-        requests.add(request);
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        builder.startArray("docs");
-        for (TermVectorsRequest request : requests) {
-            request.toXContent(builder, params);
-        }
-        builder.endArray();
-        builder.endObject();
-        return builder;
-    }
-
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/MultiTermVectorsResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/MultiTermVectorsResponse.java
deleted file mode 100644
index 3f836d714433..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/MultiTermVectorsResponse.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client.core;
-
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentParser;
-
-import java.util.List;
-import java.util.Objects;
-
-import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
-
-public class MultiTermVectorsResponse {
-    private final List<TermVectorsResponse> responses;
-
-    public MultiTermVectorsResponse(List<TermVectorsResponse> responses) {
-        this.responses = responses;
-    }
-
-    private static final ConstructingObjectParser<MultiTermVectorsResponse, Void> PARSER = new ConstructingObjectParser<>(
-        "multi_term_vectors",
-        true,
-        args -> {
-            // as the response comes from server, we are sure that args[0] will be a list of TermVectorsResponse
-            @SuppressWarnings("unchecked")
-            List<TermVectorsResponse> termVectorsResponsesList = (List<TermVectorsResponse>) args[0];
-            return new MultiTermVectorsResponse(termVectorsResponsesList);
-        }
-    );
-
-    static {
-        PARSER.declareObjectArray(constructorArg(), (p, c) -> TermVectorsResponse.fromXContent(p), new ParseField("docs"));
-    }
-
-    public static MultiTermVectorsResponse fromXContent(XContentParser parser) {
-        return PARSER.apply(parser, null);
-    }
-
-    /**
-     * Returns the list of {@code TermVectorsResponse} for this {@code MultiTermVectorsResponse}
-     */
-    public List<TermVectorsResponse> getTermVectorsResponses() {
-        return responses;
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj) return true;
-        if ((obj instanceof MultiTermVectorsResponse) == false) return false;
-        MultiTermVectorsResponse other = (MultiTermVectorsResponse) obj;
-        return Objects.equals(responses, other.responses);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(responses);
-    }
-
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/PageParams.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/PageParams.java
deleted file mode 100644
index c41e17e5d1ed..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/PageParams.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.client.core;
-
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.Objects;
-
-/**
- * Paging parameters for GET requests
- */
-public class PageParams implements ToXContentObject {
-
-    public static final ParseField PAGE = new ParseField("page");
-    public static final ParseField FROM = new ParseField("from");
-    public static final ParseField SIZE = new ParseField("size");
-
-    public static final ConstructingObjectParser<PageParams, Void> PARSER = new ConstructingObjectParser<>(
-        PAGE.getPreferredName(),
-        a -> new PageParams((Integer) a[0], (Integer) a[1])
-    );
-
-    static {
-        PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), FROM);
-        PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), SIZE);
-    }
-
-    private final Integer from;
-    private final Integer size;
-
-    /**
-     * Constructs paging parameters
-     * @param from skips the specified number of items. When {@code null} the default value will be used.
-     * @param size specifies the maximum number of items to obtain. When {@code null} the default value will be used.
-     */
-    public PageParams(@Nullable Integer from, @Nullable Integer size) {
-        this.from = from;
-        this.size = size;
-    }
-
-    public Integer getFrom() {
-        return from;
-    }
-
-    public Integer getSize() {
-        return size;
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        if (from != null) {
-            builder.field(FROM.getPreferredName(), from);
-        }
-        if (size != null) {
-            builder.field(SIZE.getPreferredName(), size);
-        }
-        builder.endObject();
-        return builder;
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(from, size);
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (obj == null) {
-            return false;
-        }
-        if (getClass() != obj.getClass()) {
-            return false;
-        }
-        PageParams other = (PageParams) obj;
-        return Objects.equals(from, other.from) && Objects.equals(size, other.size);
-    }
-
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/TermVectorsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/TermVectorsRequest.java
deleted file mode 100644
index af27dae1c04c..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/TermVectorsRequest.java
+++ /dev/null
@@ -1,284 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client.core;
-
-import org.elasticsearch.client.Validatable;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Map;
-
-public class TermVectorsRequest implements ToXContentObject, Validatable {
-
-    private final String index;
-    @Nullable
-    private final String type;
-    private String id = null;
-    private XContentBuilder docBuilder = null;
-
-    private String routing = null;
-    private String preference = null;
-    private boolean realtime = true;
-    private String[] fields = null;
-    private boolean requestPositions = true;
-    private boolean requestPayloads = true;
-    private boolean requestOffsets = true;
-    private boolean requestFieldStatistics = true;
-    private boolean requestTermStatistics = false;
-    private Map<String, String> perFieldAnalyzer = null;
-    private Map<String, Integer> filterSettings = null;
-
-    /**
-     * Constructs TermVectorRequest for the given document
-     *
-     * @param index - index of the document
-     * @param docId - id of the document
-     */
-    public TermVectorsRequest(String index, String docId) {
-        this.index = index;
-        this.type = null;
-        this.id = docId;
-    }
-
-    /**
-     * Constructs TermVectorRequest for the given document
-     *
-     * @param index - index of the document
-     * @param type - type of the document
-     * @param docId - id of the document
-     *
-     * @deprecated Types are in the process of being removed, use
-     * {@link #TermVectorsRequest(String, String)} instead.
-     */
-    @Deprecated
-    public TermVectorsRequest(String index, String type, String docId) {
-        this.index = index;
-        this.type = type;
-        this.id = docId;
-    }
-
-    /**
-     * Constructs TermVectorRequest for an artificial document
-     *
-     * @param index - index of the document
-     * @param docBuilder - an artificial document
-     */
-    public TermVectorsRequest(String index, XContentBuilder docBuilder) {
-        this.index = index;
-        this.type = null;
-        this.docBuilder = docBuilder;
-    }
-
-    /**
-     * Constructs TermVectorRequest for an artificial document
-     * @param index - index of the document
-     * @param type - type of the document
-     * @param docBuilder - an artificial document
-     *
-     * @deprecated Types are in the process of being removed, use
-     * {@link TermVectorsRequest(String, XContentBuilder)} instead.
-     */
-    @Deprecated
-    public TermVectorsRequest(String index, String type, XContentBuilder docBuilder) {
-        this.index = index;
-        this.type = type;
-        this.docBuilder = docBuilder;
-    }
-
-    /**
-     * Constructs a new TermVectorRequest from a template
-     * using the provided document id
-     * @param template - a term vector request served as a template
-     * @param id - id of the requested document
-     */
-    static TermVectorsRequest createFromTemplate(TermVectorsRequest template, String id) {
-        TermVectorsRequest request = new TermVectorsRequest(template.getIndex(), template.getType(), id);
-        request.realtime = template.getRealtime();
-        request.requestPositions = template.requestPositions;
-        request.requestPayloads = template.requestPayloads;
-        request.requestOffsets = template.requestOffsets;
-        request.requestFieldStatistics = template.requestFieldStatistics;
-        request.requestTermStatistics = template.requestTermStatistics;
-        if (template.routing != null) request.setRouting(template.getRouting());
-        if (template.preference != null) request.setPreference(template.getPreference());
-        if (template.fields != null) request.setFields(template.getFields());
-        if (template.perFieldAnalyzer != null) request.setPerFieldAnalyzer(template.perFieldAnalyzer);
-        if (template.filterSettings != null) request.setFilterSettings(template.filterSettings);
-        return request;
-    }
-
-    /**
-     * Returns the index of the request
-     */
-    public String getIndex() {
-        return index;
-    }
-
-    /**
-     * Returns the type of the request
-     *
-     * @deprecated Types are in the process of being removed.
-     */
-    @Deprecated
-    public String getType() {
-        return type;
-    }
-
-    /**
-     * Returns the id of the request
-     * can be NULL if there is no document ID
-     */
-    public String getId() {
-        return id;
-    }
-
-    /**
-     * Sets the fields for which term vectors information should be retrieved
-     */
-    public void setFields(String... fields) {
-        this.fields = fields;
-    }
-
-    public String[] getFields() {
-        return fields;
-    }
-
-    /**
-     * Sets whether to request term positions
-     */
-    public void setPositions(boolean positions) {
-        this.requestPositions = positions;
-    }
-
-    /**
-     * Sets whether to request term payloads
-     */
-    public void setPayloads(boolean payloads) {
-        this.requestPayloads = payloads;
-    }
-
-    /**
-     * Sets whether to request term offsets
-     */
-    public void setOffsets(boolean offsets) {
-        this.requestOffsets = offsets;
-    }
-
-    /**
-     * Sets whether to request field statistics
-     */
-    public void setFieldStatistics(boolean fieldStatistics) {
-        this.requestFieldStatistics = fieldStatistics;
-    }
-
-    /**
-     * Sets whether to request term statistics
-     */
-    public void setTermStatistics(boolean termStatistics) {
-        this.requestTermStatistics = termStatistics;
-    }
-
-    /**
-     * Sets different analyzers than the one at the fields
-     */
-    public void setPerFieldAnalyzer(Map<String, String> perFieldAnalyzer) {
-        this.perFieldAnalyzer = perFieldAnalyzer;
-    }
-
-    /**
-     * Sets conditions for terms filtering
-     */
-    public void setFilterSettings(Map<String, Integer> filterSettings) {
-        this.filterSettings = filterSettings;
-    }
-
-    /**
-     * Sets a routing to route a request to a particular shard
-     */
-    public void setRouting(String routing) {
-        this.routing = routing;
-    }
-
-    public String getRouting() {
-        return routing;
-    }
-
-    /**
-     * Set a preference of which shard copies to execute the request
-     */
-    public void setPreference(String preference) {
-        this.preference = preference;
-    }
-
-    public String getPreference() {
-        return preference;
-    }
-
-    /**
-     * Sets if the request should be realtime or near-realtime
-     */
-    public void setRealtime(boolean realtime) {
-        this.realtime = realtime;
-    }
-
-    /**
-     * Returns if the request is realtime(true) or near-realtime(false)
-     */
-    public boolean getRealtime() {
-        return realtime;
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        builder.field("_index", index);
-        if (type != null) {
-            builder.field("_type", type);
-        }
-        if (id != null) builder.field("_id", id);
-        if (fields != null) builder.field("fields", fields);
-        // set values only when different from defaults
-        if (requestPositions == false) builder.field("positions", false);
-        if (requestPayloads == false) builder.field("payloads", false);
-        if (requestOffsets == false) builder.field("offsets", false);
-        if (requestFieldStatistics == false) builder.field("field_statistics", false);
-        if (requestTermStatistics) builder.field("term_statistics", true);
-        if (perFieldAnalyzer != null) builder.field("per_field_analyzer", perFieldAnalyzer);
-
-        if (docBuilder != null) {
-            BytesReference doc = BytesReference.bytes(docBuilder);
-            try (InputStream stream = doc.streamInput()) {
-                builder.rawField("doc", stream, docBuilder.contentType());
-            }
-        }
-
-        if (filterSettings != null) {
-            builder.startObject("filter");
-            String[] filterSettingNames = {
-                "max_num_terms",
-                "min_term_freq",
-                "max_term_freq",
-                "min_doc_freq",
-                "max_doc_freq",
-                "min_word_length",
-                "max_word_length" };
-            for (String settingName : filterSettingNames) {
-                if (filterSettings.containsKey(settingName)) builder.field(settingName, filterSettings.get(settingName));
-            }
-            builder.endObject();
-        }
-        builder.endObject();
-        return builder;
-    }
-
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/TermVectorsResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/TermVectorsResponse.java
deleted file mode 100644
index 89764f639e1a..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/TermVectorsResponse.java
+++ /dev/null
@@ -1,468 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client.core;
-
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentParser;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Objects;
-
-import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
-import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
-
-public class TermVectorsResponse {
-    private final String index;
-    private final String id;
-    private final long docVersion;
-    private final boolean found;
-    private final long tookInMillis;
-    private final List<TermVector> termVectorList;
-
-    public TermVectorsResponse(String index, String id, long version, boolean found, long tookInMillis, List<TermVector> termVectorList) {
-        this.index = index;
-        this.id = id;
-        this.docVersion = version;
-        this.found = found;
-        this.tookInMillis = tookInMillis;
-        this.termVectorList = termVectorList;
-    }
-
-    private static final ConstructingObjectParser<TermVectorsResponse, Void> PARSER = new ConstructingObjectParser<>(
-        "term_vectors",
-        true,
-        args -> {
-            // as the response comes from server, we are sure that args[5] will be a list of TermVector
-            @SuppressWarnings("unchecked")
-            List<TermVector> termVectorList = (List<TermVector>) args[5];
-            if (termVectorList != null) {
-                Collections.sort(termVectorList, Comparator.comparing(TermVector::getFieldName));
-            }
-            return new TermVectorsResponse(
-                (String) args[0],
-                (String) args[1],
-                (long) args[2],
-                (boolean) args[3],
-                (long) args[4],
-                termVectorList
-            );
-        }
-    );
-
-    static {
-        PARSER.declareString(constructorArg(), new ParseField("_index"));
-        PARSER.declareString(optionalConstructorArg(), new ParseField("_id"));
-        PARSER.declareLong(constructorArg(), new ParseField("_version"));
-        PARSER.declareBoolean(constructorArg(), new ParseField("found"));
-        PARSER.declareLong(constructorArg(), new ParseField("took"));
-        PARSER.declareNamedObjects(
-            optionalConstructorArg(),
-            (p, c, fieldName) -> TermVector.fromXContent(p, fieldName),
-            new ParseField("term_vectors")
-        );
-    }
-
-    public static TermVectorsResponse fromXContent(XContentParser parser) {
-        return PARSER.apply(parser, null);
-    }
-
-    /**
-     * Returns the index for the response
-     */
-    public String getIndex() {
-        return index;
-    }
-
-    /**
-     * Returns the id of the request
-     * can be NULL if there is no document ID
-     */
-    public String getId() {
-        return id;
-    }
-
-    /**
-     * Returns if the document is found
-     * always <code>true</code> for artificial documents
-     */
-    public boolean getFound() {
-        return found;
-    }
-
-    /**
-     * Returns the document version
-     */
-    public long getDocVersion() {
-        return docVersion;
-    }
-
-    /**
-     * Returns the time that a request took in milliseconds
-     */
-    public long getTookInMillis() {
-        return tookInMillis;
-    }
-
-    /**
-     * Returns the list of term vectors
-     */
-    public List<TermVector> getTermVectorsList() {
-        return termVectorList;
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj) return true;
-        if ((obj instanceof TermVectorsResponse) == false) return false;
-        TermVectorsResponse other = (TermVectorsResponse) obj;
-        return index.equals(other.index)
-            && Objects.equals(id, other.id)
-            && docVersion == other.docVersion
-            && found == other.found
-            && tookInMillis == other.tookInMillis
-            && Objects.equals(termVectorList, other.termVectorList);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(index, id, docVersion, found, tookInMillis, termVectorList);
-    }
-
-    public static final class TermVector {
-
-        private static final ConstructingObjectParser<TermVector, String> PARSER = new ConstructingObjectParser<>(
-            "term_vector",
-            true,
-            (args, ctxFieldName) -> {
-                // as the response comes from server, we are sure that args[1] will be a list of Term
-                @SuppressWarnings("unchecked")
-                List<Term> terms = (List<Term>) args[1];
-                if (terms != null) {
-                    Collections.sort(terms, Comparator.comparing(Term::getTerm));
-                }
-                return new TermVector(ctxFieldName, (FieldStatistics) args[0], terms);
-            }
-        );
-
-        static {
-            PARSER.declareObject(optionalConstructorArg(), (p, c) -> FieldStatistics.fromXContent(p), new ParseField("field_statistics"));
-            PARSER.declareNamedObjects(optionalConstructorArg(), (p, c, term) -> Term.fromXContent(p, term), new ParseField("terms"));
-        }
-
-        private final String fieldName;
-        @Nullable
-        private final FieldStatistics fieldStatistics;
-        @Nullable
-        private final List<Term> terms;
-
-        public TermVector(String fieldName, FieldStatistics fieldStatistics, List<Term> terms) {
-            this.fieldName = fieldName;
-            this.fieldStatistics = fieldStatistics;
-            this.terms = terms;
-        }
-
-        public static TermVector fromXContent(XContentParser parser, String fieldName) {
-            return PARSER.apply(parser, fieldName);
-        }
-
-        /**
-         * Returns the field name of the current term vector
-         */
-        public String getFieldName() {
-            return fieldName;
-        }
-
-        /**
-         * Returns the list of terms for the current term vector
-         */
-        public List<Term> getTerms() {
-            return terms;
-        }
-
-        /**
-         * Returns the field statistics for the current field
-         */
-        public FieldStatistics getFieldStatistics() {
-            return fieldStatistics;
-        }
-
-        @Override
-        public boolean equals(Object obj) {
-            if (this == obj) return true;
-            if ((obj instanceof TermVector) == false) return false;
-            TermVector other = (TermVector) obj;
-            return fieldName.equals(other.fieldName)
-                && Objects.equals(fieldStatistics, other.fieldStatistics)
-                && Objects.equals(terms, other.terms);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(fieldName, fieldStatistics, terms);
-        }
-
-        // Class containing a general field statistics for the field
-        public static final class FieldStatistics {
-
-            private static final ConstructingObjectParser<FieldStatistics, Void> PARSER = new ConstructingObjectParser<>(
-                "field_statistics",
-                true,
-                args -> { return new FieldStatistics((long) args[0], (int) args[1], (long) args[2]); }
-            );
-
-            static {
-                PARSER.declareLong(constructorArg(), new ParseField("sum_doc_freq"));
-                PARSER.declareInt(constructorArg(), new ParseField("doc_count"));
-                PARSER.declareLong(constructorArg(), new ParseField("sum_ttf"));
-            }
-            private final long sumDocFreq;
-            private final int docCount;
-            private final long sumTotalTermFreq;
-
-            public FieldStatistics(long sumDocFreq, int docCount, long sumTotalTermFreq) {
-                this.sumDocFreq = sumDocFreq;
-                this.docCount = docCount;
-                this.sumTotalTermFreq = sumTotalTermFreq;
-            }
-
-            public static FieldStatistics fromXContent(XContentParser parser) {
-                return PARSER.apply(parser, null);
-            }
-
-            /*
-             * Returns how many documents this field contains
-             */
-            public int getDocCount() {
-                return docCount;
-            }
-
-            /**
-             * Returns the sum of document frequencies for all terms in this field
-             */
-            public long getSumDocFreq() {
-                return sumDocFreq;
-            }
-
-            /**
-             * Returns the sum of total term frequencies of all terms in this field
-             */
-            public long getSumTotalTermFreq() {
-                return sumTotalTermFreq;
-            }
-
-            @Override
-            public boolean equals(Object obj) {
-                if (this == obj) return true;
-                if ((obj instanceof FieldStatistics) == false) return false;
-                FieldStatistics other = (FieldStatistics) obj;
-                return docCount == other.docCount && sumDocFreq == other.sumDocFreq && sumTotalTermFreq == other.sumTotalTermFreq;
-            }
-
-            @Override
-            public int hashCode() {
-                return Objects.hash(docCount, sumDocFreq, sumTotalTermFreq);
-            }
-        }
-
-        public static final class Term {
-            private static final ConstructingObjectParser<Term, String> PARSER = new ConstructingObjectParser<>(
-                "token",
-                true,
-                (args, ctxTerm) -> {
-                    // as the response comes from server, we are sure that args[4] will be a list of Token
-                    @SuppressWarnings("unchecked")
-                    List<Token> tokens = (List<Token>) args[4];
-                    if (tokens != null) {
-                        Collections.sort(
-                            tokens,
-                            Comparator.comparing(Token::getPosition, Comparator.nullsFirst(Integer::compareTo))
-                                .thenComparing(Token::getStartOffset, Comparator.nullsFirst(Integer::compareTo))
-                                .thenComparing(Token::getEndOffset, Comparator.nullsFirst(Integer::compareTo))
-                        );
-                    }
-                    return new Term(ctxTerm, (int) args[0], (Integer) args[1], (Long) args[2], (Float) args[3], tokens);
-                }
-            );
-            static {
-                PARSER.declareInt(constructorArg(), new ParseField("term_freq"));
-                PARSER.declareInt(optionalConstructorArg(), new ParseField("doc_freq"));
-                PARSER.declareLong(optionalConstructorArg(), new ParseField("ttf"));
-                PARSER.declareFloat(optionalConstructorArg(), new ParseField("score"));
-                PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> Token.fromXContent(p), new ParseField("tokens"));
-            }
-
-            private final String term;
-            private final int termFreq;
-            @Nullable
-            private final Integer docFreq;
-            @Nullable
-            private final Long totalTermFreq;
-            @Nullable
-            private final Float score;
-            @Nullable
-            private final List<Token> tokens;
-
-            public Term(String term, int termFreq, Integer docFreq, Long totalTermFreq, Float score, List<Token> tokens) {
-                this.term = term;
-                this.termFreq = termFreq;
-                this.docFreq = docFreq;
-                this.totalTermFreq = totalTermFreq;
-                this.score = score;
-                this.tokens = tokens;
-            }
-
-            public static Term fromXContent(XContentParser parser, String term) {
-                return PARSER.apply(parser, term);
-            }
-
-            /**
-             * Returns the string representation of the term
-             */
-            public String getTerm() {
-                return term;
-            }
-
-            /**
-             * Returns term frequency - the number of times this term occurs in the current document
-             */
-            public int getTermFreq() {
-                return termFreq;
-            }
-
-            /**
-             * Returns document frequency - the number of documents in the index that contain this term
-             */
-            public Integer getDocFreq() {
-                return docFreq;
-            }
-
-            /**
-             * Returns total term frequency - the number of times this term occurs across all documents
-             */
-            public Long getTotalTermFreq() {
-                return totalTermFreq;
-            }
-
-            /**
-             * Returns tf-idf score, if the request used some form of terms filtering
-             */
-            public Float getScore() {
-                return score;
-            }
-
-            /**
-             * Returns a list of tokens for the term
-             */
-            public List<Token> getTokens() {
-                return tokens;
-            }
-
-            @Override
-            public boolean equals(Object obj) {
-                if (this == obj) return true;
-                if ((obj instanceof Term) == false) return false;
-                Term other = (Term) obj;
-                return term.equals(other.term)
-                    && termFreq == other.termFreq
-                    && Objects.equals(docFreq, other.docFreq)
-                    && Objects.equals(totalTermFreq, other.totalTermFreq)
-                    && Objects.equals(score, other.score)
-                    && Objects.equals(tokens, other.tokens);
-            }
-
-            @Override
-            public int hashCode() {
-                return Objects.hash(term, termFreq, docFreq, totalTermFreq, score, tokens);
-            }
-        }
-
-        public static final class Token {
-
-            private static final ConstructingObjectParser<Token, Void> PARSER = new ConstructingObjectParser<>(
-                "token",
-                true,
-                args -> { return new Token((Integer) args[0], (Integer) args[1], (Integer) args[2], (String) args[3]); }
-            );
-            static {
-                PARSER.declareInt(optionalConstructorArg(), new ParseField("start_offset"));
-                PARSER.declareInt(optionalConstructorArg(), new ParseField("end_offset"));
-                PARSER.declareInt(optionalConstructorArg(), new ParseField("position"));
-                PARSER.declareString(optionalConstructorArg(), new ParseField("payload"));
-            }
-
-            @Nullable
-            private final Integer startOffset;
-            @Nullable
-            private final Integer endOffset;
-            @Nullable
-            private final Integer position;
-            @Nullable
-            private final String payload;
-
-            public Token(Integer startOffset, Integer endOffset, Integer position, String payload) {
-                this.startOffset = startOffset;
-                this.endOffset = endOffset;
-                this.position = position;
-                this.payload = payload;
-            }
-
-            public static Token fromXContent(XContentParser parser) {
-                return PARSER.apply(parser, null);
-            }
-
-            /**
-             * Returns the start offset of the token in the document's field
-             */
-            public Integer getStartOffset() {
-                return startOffset;
-            }
-
-            /**
-             * Returns the end offset of the token in the document's field
-             */
-            public Integer getEndOffset() {
-                return endOffset;
-            }
-
-            /**
-             * Returns the position of the token in the document's field
-             */
-            public Integer getPosition() {
-                return position;
-            }
-
-            /**
-             * Returns the payload of the token or <code>null</code> if the payload doesn't exist
-             */
-            public String getPayload() {
-                return payload;
-            }
-
-            @Override
-            public boolean equals(Object obj) {
-                if (this == obj) return true;
-                if ((obj instanceof Token) == false) return false;
-                Token other = (Token) obj;
-                return Objects.equals(startOffset, other.startOffset)
-                    && Objects.equals(endOffset, other.endOffset)
-                    && Objects.equals(position, other.position)
-                    && Objects.equals(payload, other.payload);
-            }
-
-            @Override
-            public int hashCode() {
-                return Objects.hash(startOffset, endOffset, position, payload);
-            }
-        }
-    }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/license/LicenseStatus.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/license/LicenseStatus.java
deleted file mode 100644
index 3038b702c84e..000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/license/LicenseStatus.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.client.license;
-
-/**
- * Status of an X-Pack license.
- */
-public enum LicenseStatus {
-
-    ACTIVE("active"),
-    INVALID("invalid"),
-    EXPIRED("expired");
-
-    private final String label;
-
-    LicenseStatus(String label) {
-        this.label = label;
-    }
-
-    public String label() {
-        return label;
-    }
-
-    public static LicenseStatus fromString(String value) {
-        return switch (value) {
-            case "active" -> ACTIVE;
-            case "invalid" -> INVALID;
-            case "expired" -> EXPIRED;
-            default -> throw new IllegalArgumentException("unknown license status [" + value + "]");
-        };
-    }
-}
diff --git a/distribution/archives/integ-test-zip/build.gradle b/distribution/archives/integ-test-zip/build.gradle
index e57c6cf32141..d91d919619ff 100644
--- a/distribution/archives/integ-test-zip/build.gradle
+++ b/distribution/archives/integ-test-zip/build.gradle
@@ -8,7 +8,7 @@
 
 import org.apache.tools.ant.filters.ReplaceTokens
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 // The integ-test-distribution is published to maven
 apply plugin: 'elasticsearch.publish'
diff --git a/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseIT.java b/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseIT.java
index bdb0a76cf970..19afb4932ff2 100644
--- a/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseIT.java
+++ b/distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseIT.java
@@ -10,6 +10,7 @@
 
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
@@ -117,11 +118,7 @@ private ActionFuture<String> start(Request request) {
         client().performRequestAsync(request, new ResponseListener() {
             @Override
             public void onSuccess(Response response) {
-                try {
-                    future.onResponse(EntityUtils.toString(response.getEntity()));
-                } catch (IOException e) {
-                    future.onFailure(e);
-                }
+                ActionListener.completeWith(future, () -> EntityUtils.toString(response.getEntity()));
             }
 
             @Override
diff --git a/distribution/build.gradle b/distribution/build.gradle
index 6b8cdb128042..08920ed173d7 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -116,6 +116,14 @@ def processIntegTestOutputsTaskProvider = tasks.register("processIntegTestOutput
   into integTestOutputs
 }
 
+def integTestConfigFiles = fileTree("${integTestOutputs}/config") {
+  builtBy processIntegTestOutputsTaskProvider
+}
+
+def integTestBinFiles = fileTree("${integTestOutputs}/bin") {
+  builtBy processIntegTestOutputsTaskProvider
+}
+
 def defaultModulesFiles = fileTree("${defaultOutputs}/modules") {
   builtBy processDefaultOutputsTaskProvider
 }
@@ -358,7 +366,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
           filter("tokens" : expansionsForDistribution(distributionType, isTestDistro), ReplaceTokens.class)
         }
         from buildDefaultLog4jConfigTaskProvider
-        from defaultConfigFiles
+        from isTestDistro ? integTestConfigFiles : defaultConfigFiles
       }
     }
 
@@ -388,7 +396,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
         // module provided bin files
         with copySpec {
           eachFile { it.setMode(0755) }
-          from(defaultBinFiles)
+          from(testDistro ? integTestBinFiles : defaultBinFiles)
           if (distributionType != 'zip') {
             exclude '*.bat'
           }
diff --git a/distribution/bwc/build.gradle b/distribution/bwc/build.gradle
index 8827e5e4b808..11b0f99c5026 100644
--- a/distribution/bwc/build.gradle
+++ b/distribution/bwc/build.gradle
@@ -15,10 +15,10 @@ import org.elasticsearch.gradle.internal.info.BuildParams
 BuildParams.getBwcVersions().forPreviousUnreleased { unreleasedVersion ->
     project(unreleasedVersion.gradleProjectPath) {
         Version currentVersion = Version.fromString(version)
-        TaskProvider<Task> resolveAllBwcDepsTaskProvider = bwcSetup.bwcTask("resolveAllBwcDependencies") {
+        TaskProvider<Task> resolveAllBwcDepsTaskProvider = bwcSetup.bwcTask("resolveAllBwcDependencies", {
             t -> t.args("resolveAllDependencies", "-Dorg.gradle.warning.mode=none")
-        }
-        if (currentVersion.getMinor() == 0 && currentVersion.getRevision() == 0) {
+        }, false)
+        if (Boolean.getBoolean("recurse.bwc")) {
             // We only want to resolve dependencies for live versions of main, without cascading this to older versions
             tasks.named("resolveAllDependencies").configure {
                 dependsOn(resolveAllBwcDepsTaskProvider)
diff --git a/distribution/docker/README.md b/distribution/docker/README.md
index 2e22fe099f4f..4c8052cfc26b 100644
--- a/distribution/docker/README.md
+++ b/distribution/docker/README.md
@@ -92,6 +92,33 @@ images, and combining them with a Docker manifest. The Elasticsearch Delivery
 team aren't responsible for this - rather, it happens during our unified release
 process.
 
+To build multi-architecture images on `x86_64` hosts using Docker[^1], you'll
+need [buildx](https://docs.docker.com/build/buildx/install/) and ensure that it
+supports both `linux/amd64` **and** `linux/arm64` targets.
+
+You can verify the supported targets using `docker buildx ls`. For example, the
+following output indicates that support for `linux/arm64` is missing:
+
+```shell
+$ docker buildx ls
+NAME/NODE DRIVER/ENDPOINT STATUS  BUILDKIT PLATFORMS
+default * docker
+  default default         running 20.10.21 linux/amd64, linux/386
+```
+
+On Linux `x86_64` hosts, to enable `linux-arm64` you need to install
+[qemu-user-static-binfmt](https://github.com/multiarch/qemu-user-static).
+Installation details depend on the Linux distribution but, as described in the
+[getting started docs](https://github.com/multiarch/qemu-user-static#getting-started),
+running `docker run --rm --privileged multiarch/qemu-user-static --reset -p yes`
+will add the necessary support (but will not persist across reboots):
+
+```shell
+$ docker buildx ls
+NAME/NODE DRIVER/ENDPOINT STATUS  BUILDKIT PLATFORMS
+default * docker
+  default default         running 20.10.21 linux/amd64, linux/arm64, linux/riscv64, linux/ppc64le, linux/s390x, linux/386, linux/arm/v7, linux/arm/v6
+```
 
 ## Testing
 
@@ -130,3 +157,5 @@ Ideally this import / export stuff should be completely removed.
 [DockerTests]: ../../qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java
 [multi-arch]: https://www.docker.com/blog/multi-arch-build-and-images-the-simple-way/
 [ubi]: https://developers.redhat.com/products/rhel/ubi
+
+[^1]: `podman/buildah` also [supports building multi-platform images](https://github.com/containers/buildah/issues/1590).
diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle
index fe4a1e8b48a7..52b8d7b574d4 100644
--- a/distribution/docker/build.gradle
+++ b/distribution/docker/build.gradle
@@ -8,14 +8,13 @@ import org.elasticsearch.gradle.internal.docker.DockerSupportPlugin
 import org.elasticsearch.gradle.internal.docker.DockerSupportService
 import org.elasticsearch.gradle.internal.docker.ShellRetry
 import org.elasticsearch.gradle.internal.docker.TransformLog4jConfigFilter
-import org.elasticsearch.gradle.internal.dra.DraResolvePlugin
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.util.GradleUtils
 
 import java.nio.file.Path
 import java.time.temporal.ChronoUnit
 
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
 apply plugin: 'elasticsearch.internal-distribution-download'
 apply plugin: 'elasticsearch.dra-artifacts'
@@ -75,11 +74,12 @@ configurations {
   log4jConfig
   tini
   allPlugins
-  filebeat
-  metricbeat
+  filebeat_aarch64
+  filebeat_x86_64
+  metricbeat_aarch64
+  metricbeat_x86_64
 }
 
-String beatsArch = Architecture.current() == Architecture.AARCH64 ? 'arm64' : 'x86_64'
 String tiniArch = Architecture.current() == Architecture.AARCH64 ? 'arm64' : 'amd64'
 
 dependencies {
@@ -88,8 +88,10 @@ dependencies {
   log4jConfig project(path: ":distribution", configuration: 'log4jConfig')
   tini "krallin:tini:0.19.0:${tiniArch}"
   allPlugins project(path: ':plugins', configuration: 'allPlugins')
-  filebeat "beats:filebeat:${VersionProperties.elasticsearch}:linux-${beatsArch}@tar.gz"
-  metricbeat "beats:metricbeat:${VersionProperties.elasticsearch}:linux-${beatsArch}@tar.gz"
+  filebeat_aarch64 "beats:filebeat:${VersionProperties.elasticsearch}:linux-arm64@tar.gz"
+  filebeat_x86_64 "beats:filebeat:${VersionProperties.elasticsearch}:linux-x86_64@tar.gz"
+  metricbeat_aarch64 "beats:metricbeat:${VersionProperties.elasticsearch}:linux-arm64@tar.gz"
+  metricbeat_x86_64 "beats:metricbeat:${VersionProperties.elasticsearch}:linux-x86_64@tar.gz"
 }
 
 ext.expansions = { Architecture architecture, DockerBase base ->
@@ -273,8 +275,8 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) {
         boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra
 
         if (includeBeats) {
-          from configurations.filebeat
-          from configurations.metricbeat
+          from configurations.getByName("filebeat_${architecture.classifier}")
+          from configurations.getByName("metricbeat_${architecture.classifier}")
         }
         // For some reason, the artifact name can differ depending on what repository we used.
         rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
@@ -548,5 +550,5 @@ subprojects { Project subProject ->
 
 tasks.named('resolveAllDependencies') {
   // Don't try and resolve filebeat or metricbeat snapshots as they may not always be available
-  configs = configurations.matching { it.name.endsWith('beat') == false }
+  configs = configurations.matching { it.name.contains('beat') == false }
 }
diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options
index 58d83410cd6d..c999c666ade3 100644
--- a/distribution/src/config/jvm.options
+++ b/distribution/src/config/jvm.options
@@ -72,4 +72,4 @@
 @error.file@
 
 ## GC logging
--Xlog:gc*,gc+age=trace,safepoint:file=@loggc@:utctime,pid,tags:filecount=32,filesize=64m
+-Xlog:gc*,gc+age=trace,safepoint:file=@loggc@:utctime,level,pid,tags:filecount=32,filesize=64m
diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java
index 02f6b6b3e7ab..f6e357881168 100644
--- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java
+++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java
@@ -14,6 +14,7 @@
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexOutput;
 import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.settings.KeyStoreWrapper;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.core.IOUtils;
@@ -51,8 +52,10 @@
 import javax.crypto.spec.SecretKeySpec;
 
 import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
 
@@ -454,6 +457,48 @@ public void testLegacyV3() throws GeneralSecurityException, IOException {
         assertThat(toByteArray(wrapper.getFile("file_setting")), equalTo("file_value".getBytes(StandardCharsets.UTF_8)));
     }
 
+    public void testSerializationNewlyCreated() throws Exception {
+        final KeyStoreWrapper wrapper = KeyStoreWrapper.create();
+        wrapper.setString("string_setting", "string_value".toCharArray());
+
+        // testing when dataBytes[] is null
+        final BytesStreamOutput out = new BytesStreamOutput();
+        wrapper.writeTo(out);
+        final KeyStoreWrapper fromStream = new KeyStoreWrapper(out.bytes().streamInput());
+
+        assertThat(fromStream.getSettingNames(), hasSize(2));
+        assertThat(fromStream.getSettingNames(), containsInAnyOrder("string_setting", "keystore.seed"));
+
+        assertEquals(wrapper.getString("string_setting"), fromStream.getString("string_setting"));
+        assertFalse(wrapper.hasPassword());
+    }
+
+    public void testSerializationWhenLoadedFromFile() throws Exception {
+        final KeyStoreWrapper wrapper = KeyStoreWrapper.create();
+        wrapper.setString("string_setting", "string_value".toCharArray());
+
+        // testing with password and raw dataBytes[]
+        final char[] password = getPossibleKeystorePassword();
+        wrapper.save(env.configFile(), password);
+        final KeyStoreWrapper fromFile = KeyStoreWrapper.load(env.configFile());
+        fromFile.decrypt(password);
+
+        assertThat(fromFile.getSettingNames(), hasSize(2));
+        assertThat(fromFile.getSettingNames(), containsInAnyOrder("string_setting", "keystore.seed"));
+
+        assertEquals(wrapper.getString("string_setting"), fromFile.getString("string_setting"));
+
+        final BytesStreamOutput secondOut = new BytesStreamOutput();
+        fromFile.writeTo(secondOut);
+        final KeyStoreWrapper fromStreamSecond = new KeyStoreWrapper(secondOut.bytes().streamInput());
+
+        assertThat(fromStreamSecond.getSettingNames(), hasSize(2));
+        assertThat(fromStreamSecond.getSettingNames(), containsInAnyOrder("string_setting", "keystore.seed"));
+
+        assertEquals(wrapper.getString("string_setting"), fromStreamSecond.getString("string_setting"));
+        assertEquals(fromFile.hasPassword(), fromStreamSecond.hasPassword());
+    }
+
     private byte[] toByteArray(final InputStream is) throws IOException {
         final ByteArrayOutputStream os = new ByteArrayOutputStream();
         final byte[] buffer = new byte[1024];
diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle
index d1a53a0e11bd..8ea5b2f963b7 100644
--- a/distribution/tools/plugin-cli/build.gradle
+++ b/distribution/tools/plugin-cli/build.gradle
@@ -10,9 +10,18 @@ apply plugin: 'elasticsearch.build'
 
 archivesBaseName = 'elasticsearch-plugin-cli'
 
+tasks.named("dependencyLicenses").configure {
+  mapping from: /asm-.*/, to: 'asm'
+}
+
 dependencies {
   compileOnly project(":server")
   compileOnly project(":libs:elasticsearch-cli")
+  implementation project(":libs:elasticsearch-plugin-api")
+  implementation project(":libs:elasticsearch-plugin-scanner")
+  implementation 'org.ow2.asm:asm:9.4'
+  implementation 'org.ow2.asm:asm-tree:9.4'
+
   api "org.bouncycastle:bcpg-fips:1.0.4"
   api "org.bouncycastle:bc-fips:1.0.2"
   testImplementation project(":test:framework")
diff --git a/distribution/tools/plugin-cli/licenses/asm-LICENSE.txt b/distribution/tools/plugin-cli/licenses/asm-LICENSE.txt
new file mode 100644
index 000000000000..afb064f2f266
--- /dev/null
+++ b/distribution/tools/plugin-cli/licenses/asm-LICENSE.txt
@@ -0,0 +1,26 @@
+Copyright (c) 2012 France Télécom
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. Neither the name of the copyright holders nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/distribution/tools/plugin-cli/licenses/asm-NOTICE.txt b/distribution/tools/plugin-cli/licenses/asm-NOTICE.txt
new file mode 100644
index 000000000000..8d1c8b69c3fc
--- /dev/null
+++ b/distribution/tools/plugin-cli/licenses/asm-NOTICE.txt
@@ -0,0 +1 @@
+ 
diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java
index 5f7b46e2a749..1afdcd17287f 100644
--- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java
+++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java
@@ -38,9 +38,12 @@
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.jdk.JarHell;
+import org.elasticsearch.plugin.scanner.ClassReaders;
+import org.elasticsearch.plugin.scanner.NamedComponentScanner;
 import org.elasticsearch.plugins.Platforms;
 import org.elasticsearch.plugins.PluginDescriptor;
 import org.elasticsearch.plugins.PluginsUtils;
+import org.objectweb.asm.ClassReader;
 
 import java.io.BufferedReader;
 import java.io.Closeable;
@@ -82,6 +85,7 @@
 import java.util.Timer;
 import java.util.TimerTask;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
@@ -197,6 +201,7 @@ public class InstallPluginAction implements Closeable {
     private Environment env;
     private boolean batch;
     private Proxy proxy = null;
+    private NamedComponentScanner scanner = new NamedComponentScanner();
 
     public InstallPluginAction(Terminal terminal, Environment env, boolean batch) {
         this.terminal = terminal;
@@ -208,7 +213,6 @@ public void setProxy(Proxy proxy) {
         this.proxy = proxy;
     }
 
-    // pkg private for testing
     public void execute(List<InstallablePlugin> plugins) throws Exception {
         if (plugins.isEmpty()) {
             throw new UserException(ExitCodes.USAGE, "at least one plugin id is required");
@@ -867,9 +871,24 @@ private PluginDescriptor loadPluginInfo(Path pluginRoot) throws Exception {
         // check for jar hell before any copying
         jarHellCheck(info, pluginRoot, env.pluginsFile(), env.modulesFile());
 
+        if (info.isStable() && hasNamedComponentFile(pluginRoot) == false) {
+            generateNameComponentFile(pluginRoot);
+        }
         return info;
     }
 
+    private void generateNameComponentFile(Path pluginRoot) throws IOException {
+        Stream<ClassReader> classPath = ClassReaders.ofClassPath().stream(); // contains plugin-api
+        List<ClassReader> classReaders = Stream.concat(ClassReaders.ofDirWithJars(pluginRoot).stream(), classPath).toList();
+        Map<String, Map<String, String>> namedComponentsMap = scanner.scanForNamedClasses(classReaders);
+        Path outputFile = pluginRoot.resolve(PluginDescriptor.NAMED_COMPONENTS_FILENAME);
+        scanner.writeToFile(namedComponentsMap, outputFile);
+    }
+
+    private boolean hasNamedComponentFile(Path pluginRoot) {
+        return Files.exists(pluginRoot.resolve(PluginDescriptor.NAMED_COMPONENTS_FILENAME));
+    }
+
     private static final String LIB_TOOLS_PLUGIN_CLI_CLASSPATH_JAR;
 
     static {
diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java
index 9341c469462b..45328397f6ae 100644
--- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java
+++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java
@@ -43,15 +43,19 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.PathUtilsForTesting;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.plugin.scanner.NamedComponentScanner;
 import org.elasticsearch.plugins.Platforms;
 import org.elasticsearch.plugins.PluginDescriptor;
 import org.elasticsearch.plugins.PluginTestUtil;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.PosixPermissionsResetter;
+import org.elasticsearch.test.compiler.InMemoryJavaCompiler;
+import org.elasticsearch.test.jar.JarUtils;
 import org.junit.After;
 import org.junit.Before;
 
@@ -85,6 +89,7 @@
 import java.util.Arrays;
 import java.util.Date;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -123,6 +128,7 @@ public class InstallPluginActionTests extends ESTestCase {
     private MockTerminal terminal;
     private Tuple<Path, Environment> env;
     private Path pluginDir;
+    private NamedComponentScanner namedComponentScanner;
 
     private final boolean isPosix;
     private final boolean isReal;
@@ -130,6 +136,8 @@ public class InstallPluginActionTests extends ESTestCase {
 
     @SuppressForbidden(reason = "sets java.io.tmpdir")
     public InstallPluginActionTests(FileSystem fs, Function<String, Path> temp) {
+        assert "false".equals(System.getProperty("tests.security.manager")) : "-Dtests.security.manager=false has to be set";
+
         this.temp = temp;
         this.isPosix = fs.supportedFileAttributeViews().contains("posix");
         this.isReal = fs == PathUtils.getDefaultFileSystem();
@@ -151,6 +159,7 @@ void jarHellCheck(PluginDescriptor candidateInfo, Path candidate, Path pluginsDi
                 // no jarhell check
             }
         };
+
         defaultAction = new InstallPluginAction(terminal, env.v2(), false);
     }
 
@@ -198,7 +207,9 @@ private static Configuration toPosix(Configuration configuration) {
         return configuration.toBuilder().setAttributeViews("basic", "owner", "posix", "unix").build();
     }
 
-    /** Creates a test environment with bin, config and plugins directories. */
+    /**
+     * Creates a test environment with bin, config and plugins directories.
+     */
     static Tuple<Path, Environment> createEnv(Function<String, Path> temp) throws IOException {
         Path home = temp.apply("install-plugin-command-tests");
         Files.createDirectories(home.resolve("bin"));
@@ -215,7 +226,9 @@ static Path createPluginDir(Function<String, Path> temp) {
         return temp.apply("pluginDir");
     }
 
-    /** creates a fake jar file with empty class files */
+    /**
+     * creates a fake jar file with empty class files
+     */
     static void writeJar(Path jar, String... classes) throws IOException {
         try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(jar))) {
             for (String clazz : classes) {
@@ -236,13 +249,47 @@ static Path writeZip(Path structure, String prefix) throws IOException {
         return zip;
     }
 
-    /** creates a plugin .zip and returns the url for testing */
+    /**
+     * creates a plugin .zip and returns the url for testing
+     */
     static InstallablePlugin createPluginZip(String name, Path structure, String... additionalProps) throws IOException {
         return createPlugin(name, structure, additionalProps);
     }
 
+    static void writeStablePlugin(String name, Path structure, boolean hasNamedComponentFile, String... additionalProps)
+        throws IOException {
+        String[] properties = pluginProperties(name, additionalProps, true);
+        PluginTestUtil.writeStablePluginProperties(structure, properties);
+
+        if (hasNamedComponentFile) {
+            PluginTestUtil.writeNamedComponentsFile(structure, namedComponentsJSON());
+        }
+        Path jar = structure.resolve("plugin.jar");
+
+        JarUtils.createJarWithEntries(jar, Map.of("p/A.class", InMemoryJavaCompiler.compile("p.A", """
+            package p;
+            import org.elasticsearch.plugin.*;
+            import org.elasticsearch.plugins.cli.test_model.*;
+            @NamedComponent("a_component")
+            public class A implements ExtensibleInterface{}
+            """), "p/B.class", InMemoryJavaCompiler.compile("p.B", """
+            package p;
+            import org.elasticsearch.plugin.*;
+            import org.elasticsearch.plugins.cli.test_model.*;
+            @NamedComponent("b_component")
+            public class B implements ExtensibleInterface{}
+            """)));
+    }
+
     static void writePlugin(String name, Path structure, String... additionalProps) throws IOException {
-        String[] properties = Stream.concat(
+        String[] properties = pluginProperties(name, additionalProps, false);
+        PluginTestUtil.writePluginProperties(structure, properties);
+        String className = name.substring(0, 1).toUpperCase(Locale.ENGLISH) + name.substring(1) + "Plugin";
+        writeJar(structure.resolve("plugin.jar"), className);
+    }
+
+    private static String[] pluginProperties(String name, String[] additionalProps, boolean isStable) {
+        return Stream.of(
             Stream.of(
                 "description",
                 "fake desc",
@@ -253,15 +300,12 @@ static void writePlugin(String name, Path structure, String... additionalProps)
                 "elasticsearch.version",
                 Version.CURRENT.toString(),
                 "java.version",
-                System.getProperty("java.specification.version"),
-                "classname",
-                "FakePlugin"
+                System.getProperty("java.specification.version")
+
             ),
+            isStable ? Stream.empty() : Stream.of("classname", "FakePlugin"),
             Arrays.stream(additionalProps)
-        ).toArray(String[]::new);
-        PluginTestUtil.writePluginProperties(structure, properties);
-        String className = name.substring(0, 1).toUpperCase(Locale.ENGLISH) + name.substring(1) + "Plugin";
-        writeJar(structure.resolve("plugin.jar"), className);
+        ).flatMap(Function.identity()).toArray(String[]::new);
     }
 
     static void writePluginSecurityPolicy(Path pluginDir, String... permissions) throws IOException {
@@ -275,6 +319,12 @@ static void writePluginSecurityPolicy(Path pluginDir, String... permissions) thr
         Files.write(pluginDir.resolve("plugin-security.policy"), securityPolicyContent.toString().getBytes(StandardCharsets.UTF_8));
     }
 
+    static InstallablePlugin createStablePlugin(String name, Path structure, boolean hasNamedComponentFile, String... additionalProps)
+        throws IOException {
+        writeStablePlugin(name, structure, hasNamedComponentFile, additionalProps);
+        return new InstallablePlugin(name, writeZip(structure, null).toUri().toURL().toString());
+    }
+
     static InstallablePlugin createPlugin(String name, Path structure, String... additionalProps) throws IOException {
         writePlugin(name, structure, additionalProps);
         return new InstallablePlugin(name, writeZip(structure, null).toUri().toURL().toString());
@@ -309,6 +359,11 @@ void assertPlugin(String name, Path original, Environment environment) throws IO
         assertInstallCleaned(environment);
     }
 
+    void assertNamedComponentFile(String name, Path pluginDir, String expectedContent) throws IOException {
+        Path namedComponents = pluginDir.resolve(name).resolve(PluginDescriptor.NAMED_COMPONENTS_FILENAME);
+        assertThat(Files.readString(namedComponents), equalTo(expectedContent));
+    }
+
     void assertPluginInternal(String name, Path pluginsFile, Path originalPlugin) throws IOException {
         Path got = pluginsFile.resolve(name);
         assertTrue("dir " + name + " exists", Files.exists(got));
@@ -505,7 +560,7 @@ public void testInstallFailsIfPreviouslyRemovedPluginFailed() throws Exception {
         final Path removing = env.v2().pluginsFile().resolve(".removing-failed");
         Files.createDirectory(removing);
         final IllegalStateException e = expectThrows(IllegalStateException.class, () -> installPlugin(pluginZip));
-        final String expected = formatted(
+        final String expected = Strings.format(
             "found file [%s] from a failed attempt to remove the plugin [failed]; execute [elasticsearch-plugin remove failed]",
             removing
         );
@@ -1506,4 +1561,42 @@ public void testInstallMigratedPlugins() throws Exception {
             assertThat(terminal.getErrorOutput(), containsString("[" + id + "] is no longer a plugin"));
         }
     }
+
+    public void testStablePluginWithNamedComponentsFile() throws Exception {
+        InstallablePlugin stablePluginZip = createStablePlugin("stable1", pluginDir, true);
+        installPlugins(List.of(stablePluginZip), env.v1());
+        assertPlugin("stable1", pluginDir, env.v2());
+        assertNamedComponentFile("stable1", env.v2().pluginsFile(), namedComponentsJSON());
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testStablePluginWithoutNamedComponentsFile() throws Exception {
+        // named component will have to be generated upon install
+        InstallablePlugin stablePluginZip = createStablePlugin("stable1", pluginDir, false);
+
+        installPlugins(List.of(stablePluginZip), env.v1());
+
+        assertPlugin("stable1", pluginDir, env.v2());
+        assertNamedComponentFile("stable1", env.v2().pluginsFile(), namedComponentsJSON());
+    }
+
+    private Map<String, Map<String, String>> namedComponentsMap() {
+        Map<String, Map<String, String>> result = new LinkedHashMap<>();
+        Map<String, String> extensibles = new LinkedHashMap<>();
+        extensibles.put("a_component", "p.A");
+        extensibles.put("b_component", "p.B");
+        result.put("org.elasticsearch.plugins.cli.test_model.ExtensibleInterface", extensibles);
+        return result;
+    }
+
+    private static String namedComponentsJSON() {
+        return """
+            {
+              "org.elasticsearch.plugins.cli.test_model.ExtensibleInterface": {
+                "a_component": "p.A",
+                "b_component": "p.B"
+              }
+            }
+            """.replaceAll("[\n\r\s]", "");
+    }
 }
diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/test_model/ExtensibleInterface.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/test_model/ExtensibleInterface.java
new file mode 100644
index 000000000000..517c57a03680
--- /dev/null
+++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/test_model/ExtensibleInterface.java
@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugins.cli.test_model;
+
+import org.elasticsearch.plugin.Extensible;
+
+@Extensible
+public interface ExtensibleInterface {}
diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java
index b89eb166aad8..c13ab9d10681 100644
--- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java
+++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.KeyStoreWrapper;
+import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Nullable;
@@ -133,11 +133,10 @@ class APMJvmOptions {
      * because it will be deleted once Elasticsearch starts.
      *
      * @param settings the Elasticsearch settings to consider
-     * @param keystore a wrapper to access the keystore, or null if there is no keystore
+     * @param secrets a wrapper to access the secrets, or null if there is no secrets
      * @param tmpdir Elasticsearch's temporary directory, where the config file will be written
      */
-    static List<String> apmJvmOptions(Settings settings, @Nullable KeyStoreWrapper keystore, Path tmpdir) throws UserException,
-        IOException {
+    static List<String> apmJvmOptions(Settings settings, @Nullable SecureSettings secrets, Path tmpdir) throws UserException, IOException {
         final Path agentJar = findAgentJar();
 
         if (agentJar == null) {
@@ -158,8 +157,8 @@ static List<String> apmJvmOptions(Settings settings, @Nullable KeyStoreWrapper k
             }
         }
 
-        if (keystore != null) {
-            extractSecureSettings(keystore, propertiesMap);
+        if (secrets != null) {
+            extractSecureSettings(secrets, propertiesMap);
         }
         final Map<String, String> dynamicSettings = extractDynamicSettings(propertiesMap);
 
@@ -180,11 +179,11 @@ static String agentCommandLineOption(Path agentJar, Path tmpPropertiesFile) {
         return "-javaagent:" + agentJar + "=c=" + tmpPropertiesFile;
     }
 
-    private static void extractSecureSettings(KeyStoreWrapper keystore, Map<String, String> propertiesMap) {
-        final Set<String> settingNames = keystore.getSettingNames();
+    private static void extractSecureSettings(SecureSettings secrets, Map<String, String> propertiesMap) {
+        final Set<String> settingNames = secrets.getSettingNames();
         for (String key : List.of("api_key", "secret_token")) {
             if (settingNames.contains("tracing.apm." + key)) {
-                try (SecureString token = keystore.getString("tracing.apm." + key)) {
+                try (SecureString token = secrets.getString("tracing.apm." + key)) {
                     propertiesMap.put(key, token.toString());
                 }
             }
diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ErrorPumpThread.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ErrorPumpThread.java
index d8f0c4471c65..397e6a327d1a 100644
--- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ErrorPumpThread.java
+++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ErrorPumpThread.java
@@ -19,19 +19,14 @@
 import java.util.concurrent.CountDownLatch;
 
 import static org.elasticsearch.bootstrap.BootstrapInfo.SERVER_READY_MARKER;
-import static org.elasticsearch.bootstrap.BootstrapInfo.USER_EXCEPTION_MARKER;
 import static org.elasticsearch.server.cli.ProcessUtil.nonInterruptibleVoid;
 
 /**
  * A thread which reads stderr of the jvm process and writes it to this process' stderr.
  *
- * <p> Two special state markers are watched for. These are ascii control characters which signal
- * to the cli process something has changed in the server process. The two possible special messages are:
- * <ul>
- *     <li>{@link BootstrapInfo#USER_EXCEPTION_MARKER} - signals a bootstrap error has occurred, and is followed
- *     by the error message</li>
- *     <li>{@link BootstrapInfo#SERVER_READY_MARKER} - signals the server is ready so the cli may detach if daemonizing</li>
- * </ul>
+ * <p> The thread watches for a special state marker from the process. The ascii character
+ * {@link BootstrapInfo#SERVER_READY_MARKER} signals the server is ready and the cli may
+ * detach if daemonizing. All other messages are passed through to stderr.
  */
 class ErrorPumpThread extends Thread {
     private final BufferedReader reader;
@@ -43,9 +38,6 @@ class ErrorPumpThread extends Thread {
     // a flag denoting whether the ready marker has been received by the server process
     private volatile boolean ready;
 
-    // an exception message received alongside the user exception marker, if a bootstrap error has occurred
-    private volatile String userExceptionMsg;
-
     // an unexpected io failure that occurred while pumping stderr
     private volatile IOException ioFailure;
 
@@ -66,10 +58,6 @@ String waitUntilReady() throws IOException {
         if (ioFailure != null) {
             throw ioFailure;
         }
-        if (ready == false) {
-            return userExceptionMsg;
-        }
-        assert userExceptionMsg == null;
         return null;
     }
 
@@ -85,10 +73,7 @@ public void run() {
         try {
             String line;
             while ((line = reader.readLine()) != null) {
-                if (line.isEmpty() == false && line.charAt(0) == USER_EXCEPTION_MARKER) {
-                    userExceptionMsg = line.substring(1);
-                    readyOrDead.countDown();
-                } else if (line.isEmpty() == false && line.charAt(0) == SERVER_READY_MARKER) {
+                if (line.isEmpty() == false && line.charAt(0) == SERVER_READY_MARKER) {
                     ready = true;
                     readyOrDead.countDown();
                 } else {
diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmErgonomics.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmErgonomics.java
index 46e3da3ced90..926d5727a1b4 100644
--- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmErgonomics.java
+++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmErgonomics.java
@@ -64,28 +64,42 @@ static boolean tuneG1GCForSmallHeap(final long heapSize) {
     }
 
     static boolean tuneG1GCHeapRegion(final Map<String, JvmOption> finalJvmOptions, final boolean tuneG1GCForSmallHeap) {
-        JvmOption g1GCHeapRegion = finalJvmOptions.get("G1HeapRegionSize");
-        JvmOption g1GC = finalJvmOptions.get("UseG1GC");
-        return (tuneG1GCForSmallHeap && g1GC.getMandatoryValue().equals("true") && g1GCHeapRegion.isCommandLineOrigin() == false);
+        return tuneG1GCForSmallHeap && usingG1GcWithoutCommandLineOriginOption(finalJvmOptions, "G1HeapRegionSize");
     }
 
     static int tuneG1GCReservePercent(final Map<String, JvmOption> finalJvmOptions, final boolean tuneG1GCForSmallHeap) {
-        JvmOption g1GC = finalJvmOptions.get("UseG1GC");
-        JvmOption g1GCReservePercent = finalJvmOptions.get("G1ReservePercent");
-        if (g1GC.getMandatoryValue().equals("true")) {
-            if (g1GCReservePercent.isCommandLineOrigin() == false && tuneG1GCForSmallHeap) {
-                return 15;
-            } else if (g1GCReservePercent.isCommandLineOrigin() == false && tuneG1GCForSmallHeap == false) {
-                return 25;
-            }
+        if (usingG1GcWithoutCommandLineOriginOption(finalJvmOptions, "G1ReservePercent")) {
+            return tuneG1GCForSmallHeap ? 15 : 25;
         }
         return 0;
     }
 
     static boolean tuneG1GCInitiatingHeapOccupancyPercent(final Map<String, JvmOption> finalJvmOptions) {
-        JvmOption g1GC = finalJvmOptions.get("UseG1GC");
-        JvmOption g1GCInitiatingHeapOccupancyPercent = finalJvmOptions.get("InitiatingHeapOccupancyPercent");
-        return g1GCInitiatingHeapOccupancyPercent.isCommandLineOrigin() == false && g1GC.getMandatoryValue().equals("true");
+        return usingG1GcWithoutCommandLineOriginOption(finalJvmOptions, "InitiatingHeapOccupancyPercent");
+    }
+
+    /**
+     * @return <ul>
+     *         <li>{@code true} if `-XX:+UseG1GC` is in the final JVM options and {@code optionName} was not specified.
+     *         <li>{@code false} if either `-XX:-UseG1GC` is in the final JVM options, or {@code optionName} was specified.
+     *         </ul>
+     *
+     * @throws IllegalStateException if neither `-XX:+UseG1GC` nor `-XX:-UseG1GC` is in the final JVM options, or `-XX:+UseG1GC` is selected
+     *                               and {@code optionName} is not in the final JVM options.
+     */
+    private static boolean usingG1GcWithoutCommandLineOriginOption(Map<String, JvmOption> finalJvmOptions, String optionName) {
+        return getRequiredOption(finalJvmOptions, "UseG1GC").getMandatoryValue().equals("true")
+            && getRequiredOption(finalJvmOptions, optionName).isCommandLineOrigin() == false;
+    }
+
+    private static JvmOption getRequiredOption(final Map<String, JvmOption> finalJvmOptions, final String key) {
+        final var jvmOption = finalJvmOptions.get(key);
+        if (jvmOption == null) {
+            throw new IllegalStateException(
+                "JVM option [" + key + "] was unexpectedly missing. Elasticsearch requires this option to be present."
+            );
+        }
+        return jvmOption;
     }
 
     private static final Pattern SYSTEM_PROPERTY = Pattern.compile("^-D(?<key>[\\w+].*?)=(?<value>.*)$");
diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOption.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOption.java
index 39bf2e54dade..60cbcb86c02b 100644
--- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOption.java
+++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOption.java
@@ -8,6 +8,8 @@
 
 package org.elasticsearch.server.cli;
 
+import org.elasticsearch.common.Strings;
+
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -29,6 +31,11 @@ class JvmOption {
     private final String origin;
 
     JvmOption(String value, String origin) {
+        if (origin == null) {
+            throw new IllegalStateException(Strings.format("""
+                Elasticsearch could not determine the origin of JVM option [%s]. \
+                This indicates that it is running in an unsupported configuration.""", value));
+        }
         this.value = value;
         this.origin = origin;
     }
diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java
index b20aad3a0b84..1b92300ac3dd 100644
--- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java
+++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.bootstrap.ServerArgs;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.UserException;
-import org.elasticsearch.common.settings.KeyStoreWrapper;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -70,7 +69,7 @@ SortedMap<Integer, String> invalidLines() {
      * files in the {@code jvm.options.d} directory, and the options given by the {@code ES_JAVA_OPTS} environment
      * variable.
      *
-     * @param keystore        the installation's keystore
+     * @param args            the start-up arguments
      * @param configDir       the ES config dir
      * @param tmpDir          the directory that should be passed to {@code -Djava.io.tmpdir}
      * @param envOptions      the options passed through the ES_JAVA_OPTS env var
@@ -79,8 +78,8 @@ SortedMap<Integer, String> invalidLines() {
      * @throws IOException          if there is a problem reading any of the files
      * @throws UserException        if there is a problem parsing the `jvm.options` file or `jvm.options.d` files
      */
-    static List<String> determineJvmOptions(ServerArgs args, KeyStoreWrapper keystore, Path configDir, Path tmpDir, String envOptions)
-        throws InterruptedException, IOException, UserException {
+    static List<String> determineJvmOptions(ServerArgs args, Path configDir, Path tmpDir, String envOptions) throws InterruptedException,
+        IOException, UserException {
 
         final JvmOptionsParser parser = new JvmOptionsParser();
 
@@ -89,7 +88,7 @@ static List<String> determineJvmOptions(ServerArgs args, KeyStoreWrapper keystor
         substitutions.put("ES_PATH_CONF", configDir.toString());
 
         try {
-            return parser.jvmOptions(args, keystore, configDir, tmpDir, envOptions, substitutions);
+            return parser.jvmOptions(args, configDir, tmpDir, envOptions, substitutions);
         } catch (final JvmOptionsFileParserException e) {
             final String errorMessage = String.format(
                 Locale.ROOT,
@@ -120,7 +119,6 @@ static List<String> determineJvmOptions(ServerArgs args, KeyStoreWrapper keystor
 
     private List<String> jvmOptions(
         ServerArgs args,
-        KeyStoreWrapper keystore,
         final Path config,
         Path tmpDir,
         final String esJavaOpts,
@@ -141,7 +139,7 @@ private List<String> jvmOptions(
         final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
         final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions();
 
-        final List<String> apmOptions = APMJvmOptions.apmJvmOptions(args.nodeSettings(), keystore, tmpDir);
+        final List<String> apmOptions = APMJvmOptions.apmJvmOptions(args.nodeSettings(), args.secrets(), tmpDir);
 
         final List<String> finalJvmOptions = new ArrayList<>(
             systemJvmOptions.size() + substitutedJvmOptions.size() + ergonomicJvmOptions.size() + apmOptions.size()
diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/KeyStoreLoader.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/KeyStoreLoader.java
new file mode 100644
index 000000000000..6741e95e98dc
--- /dev/null
+++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/KeyStoreLoader.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.server.cli;
+
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.common.settings.KeyStoreWrapper;
+import org.elasticsearch.common.settings.SecureSettings;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.env.Environment;
+
+import java.util.Optional;
+
+/**
+ * Implementation of {@link SecureSettingsLoader} for {@link KeyStoreWrapper}
+ */
+public class KeyStoreLoader implements SecureSettingsLoader {
+    @Override
+    public LoadedSecrets load(Environment environment, Terminal terminal) throws Exception {
+        // See if we have a keystore already present
+        KeyStoreWrapper secureSettings = KeyStoreWrapper.load(environment.configFile());
+        // If there's no keystore or the keystore has no password, set an empty password
+        var password = (secureSettings == null || secureSettings.hasPassword() == false)
+            ? new SecureString(new char[0])
+            : new SecureString(terminal.readSecret(KeyStoreWrapper.PROMPT));
+
+        return new LoadedSecrets(secureSettings, Optional.of(password));
+    }
+
+    @Override
+    public SecureSettings bootstrap(Environment environment, SecureString password) throws Exception {
+        return KeyStoreWrapper.bootstrap(environment.configFile(), () -> password);
+    }
+
+    @Override
+    public boolean supportsSecurityAutoConfiguration() {
+        return true;
+    }
+}
diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SecureSettingsLoader.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SecureSettingsLoader.java
new file mode 100644
index 000000000000..98fc47f4e64b
--- /dev/null
+++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SecureSettingsLoader.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.server.cli;
+
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.common.settings.SecureSettings;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.env.Environment;
+
+import java.util.Optional;
+
+/**
+ * An interface for implementing {@link SecureSettings} loaders, that is, implementations that create, initialize and load
+ * secrets stores.
+ */
+public interface SecureSettingsLoader {
+    /**
+     * Loads an existing SecureSettings implementation
+     */
+    LoadedSecrets load(Environment environment, Terminal terminal) throws Exception;
+
+    /**
+     * Loads an existing SecureSettings implementation, creates one if it doesn't exist
+     */
+    SecureSettings bootstrap(Environment environment, SecureString password) throws Exception;
+
+    /**
+     * A load result for loading a SecureSettings implementation from a SecureSettingsLoader
+     * @param secrets the loaded secure settings
+     * @param password an optional password if the implementation required one
+     */
+    record LoadedSecrets(SecureSettings secrets, Optional<SecureString> password) implements AutoCloseable {
+        @Override
+        public void close() throws Exception {
+            if (password.isPresent()) {
+                password.get().close();
+            }
+        }
+    }
+
+    boolean supportsSecurityAutoConfiguration();
+}
diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java
index 73269b8c719f..e473301ad67f 100644
--- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java
+++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.cli.EnvironmentAwareCommand;
-import org.elasticsearch.common.settings.KeyStoreWrapper;
+import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.monitor.jvm.JvmInfo;
@@ -75,20 +75,29 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce
 
         validateConfig(options, env);
 
-        try (KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile())) {
-            // setup security
-            final SecureString keystorePassword = getKeystorePassword(keystore, terminal);
-            env = autoConfigureSecurity(terminal, options, processInfo, env, keystorePassword);
+        var secureSettingsLoader = secureSettingsLoader(env);
+
+        try (
+            var loadedSecrets = secureSettingsLoader.load(env, terminal);
+            var password = (loadedSecrets.password().isPresent()) ? loadedSecrets.password().get() : new SecureString(new char[0]);
+        ) {
+            SecureSettings secrets = loadedSecrets.secrets();
+            if (secureSettingsLoader.supportsSecurityAutoConfiguration()) {
+                env = autoConfigureSecurity(terminal, options, processInfo, env, password);
+                // reload or create the secrets
+                secrets = secureSettingsLoader.bootstrap(env, password);
+            }
 
-            if (keystore != null) {
-                keystore.decrypt(keystorePassword.getChars());
+            // we should have a loaded or bootstrapped secure settings at this point
+            if (secrets == null) {
+                throw new UserException(ExitCodes.CONFIG, "Elasticsearch secure settings not configured");
             }
 
             // install/remove plugins from elasticsearch-plugins.yml
             syncPlugins(terminal, env, processInfo);
 
-            ServerArgs args = createArgs(options, env, keystorePassword, processInfo);
-            this.server = startServer(terminal, processInfo, args, keystore);
+            ServerArgs args = createArgs(options, env, secrets, processInfo);
+            this.server = startServer(terminal, processInfo, args);
         }
 
         if (options.has(daemonizeOption)) {
@@ -127,21 +136,17 @@ private void validateConfig(OptionSet options, Environment env) throws UserExcep
         }
     }
 
-    private static SecureString getKeystorePassword(KeyStoreWrapper keystore, Terminal terminal) {
-        if (keystore != null && keystore.hasPassword()) {
-            return new SecureString(terminal.readSecret(KeyStoreWrapper.PROMPT));
-        } else {
-            return new SecureString(new char[0]);
-        }
-    }
-
-    private Environment autoConfigureSecurity(
+    // Autoconfiguration of SecureSettings is currently only supported for KeyStore based secure settings
+    // package private for testing
+    Environment autoConfigureSecurity(
         Terminal terminal,
         OptionSet options,
         ProcessInfo processInfo,
         Environment env,
         SecureString keystorePassword
     ) throws Exception {
+        assert secureSettingsLoader(env) instanceof KeyStoreLoader;
+
         String autoConfigLibs = "modules/x-pack-core,modules/x-pack-security,lib/tools/security-cli";
         Command cmd = loadTool("auto-configure-node", autoConfigLibs);
         assert cmd instanceof EnvironmentAwareCommand;
@@ -182,7 +187,8 @@ private Environment autoConfigureSecurity(
         return env;
     }
 
-    private void syncPlugins(Terminal terminal, Environment env, ProcessInfo processInfo) throws Exception {
+    // package private for testing
+    void syncPlugins(Terminal terminal, Environment env, ProcessInfo processInfo) throws Exception {
         String pluginCliLibs = "lib/tools/plugin-cli";
         Command cmd = loadTool("sync-plugins", pluginCliLibs);
         assert cmd instanceof EnvironmentAwareCommand;
@@ -201,7 +207,7 @@ private void validatePidFile(Path pidFile) throws UserException {
         }
     }
 
-    private ServerArgs createArgs(OptionSet options, Environment env, SecureString keystorePassword, ProcessInfo processInfo)
+    private ServerArgs createArgs(OptionSet options, Environment env, SecureSettings secrets, ProcessInfo processInfo)
         throws UserException {
         boolean daemonize = options.has(daemonizeOption);
         boolean quiet = options.has(quietOption);
@@ -213,7 +219,7 @@ private ServerArgs createArgs(OptionSet options, Environment env, SecureString k
             }
             validatePidFile(pidFile);
         }
-        return new ServerArgs(daemonize, quiet, pidFile, keystorePassword, env.settings(), env.configFile());
+        return new ServerArgs(daemonize, quiet, pidFile, secrets, env.settings(), env.configFile());
     }
 
     @Override
@@ -229,8 +235,13 @@ protected Command loadTool(String toolname, String libs) {
     }
 
     // protected to allow tests to override
-    protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args, KeyStoreWrapper keystore)
-        throws UserException {
-        return ServerProcess.start(terminal, processInfo, args, keystore);
+    protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) throws UserException {
+        return ServerProcess.start(terminal, processInfo, args);
+    }
+
+    // protected to allow tests to override
+    protected SecureSettingsLoader secureSettingsLoader(Environment env) {
+        // TODO: Use the environment configuration to decide what kind of secrets store to load
+        return new KeyStoreLoader();
     }
 }
diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcess.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcess.java
index 674f9f12c916..208ba9b5fe34 100644
--- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcess.java
+++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerProcess.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
-import org.elasticsearch.common.settings.KeyStoreWrapper;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.SuppressForbidden;
@@ -37,7 +36,7 @@
 /**
  * A helper to control a {@link Process} running the main Elasticsearch server.
  *
- * <p> The process can be started by calling {@link #start(Terminal, ProcessInfo, ServerArgs, KeyStoreWrapper)}.
+ * <p> The process can be started by calling {@link #start(Terminal, ProcessInfo, ServerArgs)}.
  * The process is controlled by internally sending arguments and control signals on stdin,
  * and receiving control signals on stderr. The start method does not return until the
  * server is ready to process requests and has exited the bootstrap thread.
@@ -67,8 +66,8 @@ public class ServerProcess {
 
     // this allows mocking the process building by tests
     interface OptionsBuilder {
-        List<String> getJvmOptions(ServerArgs args, KeyStoreWrapper keyStoreWrapper, Path configDir, Path tmpDir, String envOptions)
-            throws InterruptedException, IOException, UserException;
+        List<String> getJvmOptions(ServerArgs args, Path configDir, Path tmpDir, String envOptions) throws InterruptedException,
+            IOException, UserException;
     }
 
     // this allows mocking the process building by tests
@@ -82,13 +81,11 @@ interface ProcessStarter {
      * @param terminal        A terminal to connect the standard inputs and outputs to for the new process.
      * @param processInfo     Info about the current process, for passing through to the subprocess.
      * @param args            Arguments to the server process.
-     * @param keystore        A keystore for accessing secrets.
      * @return A running server process that is ready for requests
      * @throws UserException If the process failed during bootstrap
      */
-    public static ServerProcess start(Terminal terminal, ProcessInfo processInfo, ServerArgs args, KeyStoreWrapper keystore)
-        throws UserException {
-        return start(terminal, processInfo, args, keystore, JvmOptionsParser::determineJvmOptions, ProcessBuilder::start);
+    public static ServerProcess start(Terminal terminal, ProcessInfo processInfo, ServerArgs args) throws UserException {
+        return start(terminal, processInfo, args, JvmOptionsParser::determineJvmOptions, ProcessBuilder::start);
     }
 
     // package private so tests can mock options building and process starting
@@ -96,7 +93,6 @@ static ServerProcess start(
         Terminal terminal,
         ProcessInfo processInfo,
         ServerArgs args,
-        KeyStoreWrapper keystore,
         OptionsBuilder optionsBuilder,
         ProcessStarter processStarter
     ) throws UserException {
@@ -105,7 +101,7 @@ static ServerProcess start(
 
         boolean success = false;
         try {
-            jvmProcess = createProcess(args, keystore, processInfo, args.configDir(), optionsBuilder, processStarter);
+            jvmProcess = createProcess(args, processInfo, args.configDir(), optionsBuilder, processStarter);
             errorPump = new ErrorPumpThread(terminal.getErrorWriter(), jvmProcess.getErrorStream());
             errorPump.start();
             sendArgs(args, jvmProcess.getOutputStream());
@@ -184,7 +180,6 @@ private static void sendArgs(ServerArgs args, OutputStream processStdin) {
             // so the pump thread can complete, writing out the actual error. All we get here is the failure to write to
             // the process pipe, which isn't helpful to print.
         }
-        args.keystorePassword().close();
     }
 
     private void sendShutdownMarker() {
@@ -199,7 +194,6 @@ private void sendShutdownMarker() {
 
     private static Process createProcess(
         ServerArgs args,
-        KeyStoreWrapper keystore,
         ProcessInfo processInfo,
         Path configDir,
         OptionsBuilder optionsBuilder,
@@ -211,7 +205,7 @@ private static Process createProcess(
             envVars.put("LIBFFI_TMPDIR", tempDir.toString());
         }
 
-        List<String> jvmOptions = optionsBuilder.getJvmOptions(args, keystore, configDir, tempDir, envVars.remove("ES_JAVA_OPTS"));
+        List<String> jvmOptions = optionsBuilder.getJvmOptions(args, configDir, tempDir, envVars.remove("ES_JAVA_OPTS"));
         // also pass through distribution type
         jvmOptions.add("-Des.distribution.type=" + processInfo.sysprops().get("es.distribution.type"));
 
diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmErgonomicsTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmErgonomicsTests.java
index f68a51de85c2..0d4edfc384d4 100644
--- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmErgonomicsTests.java
+++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmErgonomicsTests.java
@@ -19,6 +19,7 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.everyItem;
@@ -179,4 +180,46 @@ public void testMaxDirectMemorySizeChoiceWhenSet() throws Exception {
         );
     }
 
+    @SuppressWarnings("ConstantConditions")
+    public void testMissingOptionHandling() {
+        final Map<String, JvmOption> g1GcOn = Map.of("UseG1GC", new JvmOption("true", ""));
+        final Map<String, JvmOption> g1GcOff = Map.of("UseG1GC", new JvmOption("", ""));
+
+        assertFalse(JvmErgonomics.tuneG1GCHeapRegion(Map.of(), false));
+        assertThat(
+            expectThrows(IllegalStateException.class, () -> JvmErgonomics.tuneG1GCHeapRegion(Map.of(), true)).getMessage(),
+            allOf(containsString("[UseG1GC]"), containsString("unexpectedly missing"))
+        );
+        assertThat(
+            expectThrows(IllegalStateException.class, () -> JvmErgonomics.tuneG1GCHeapRegion(g1GcOn, true)).getMessage(),
+            allOf(containsString("[G1HeapRegionSize]"), containsString("unexpectedly missing"))
+        );
+        assertFalse(JvmErgonomics.tuneG1GCHeapRegion(g1GcOff, randomBoolean()));
+
+        assertThat(
+            expectThrows(IllegalStateException.class, () -> JvmErgonomics.tuneG1GCReservePercent(Map.of(), randomBoolean())).getMessage(),
+            allOf(containsString("[UseG1GC]"), containsString("unexpectedly missing"))
+        );
+        assertThat(
+            expectThrows(IllegalStateException.class, () -> JvmErgonomics.tuneG1GCReservePercent(g1GcOn, randomBoolean())).getMessage(),
+            allOf(containsString("[G1ReservePercent]"), containsString("unexpectedly missing"))
+        );
+        assertEquals(0, JvmErgonomics.tuneG1GCReservePercent(g1GcOff, randomBoolean()));
+
+        assertThat(
+            expectThrows(IllegalStateException.class, () -> JvmErgonomics.tuneG1GCInitiatingHeapOccupancyPercent(Map.of())).getMessage(),
+            allOf(containsString("[UseG1GC]"), containsString("unexpectedly missing"))
+        );
+        assertThat(
+            expectThrows(IllegalStateException.class, () -> JvmErgonomics.tuneG1GCInitiatingHeapOccupancyPercent(g1GcOn)).getMessage(),
+            allOf(containsString("[InitiatingHeapOccupancyPercent]"), containsString("unexpectedly missing"))
+        );
+        assertFalse(JvmErgonomics.tuneG1GCInitiatingHeapOccupancyPercent(g1GcOff));
+
+        assertThat(
+            expectThrows(IllegalStateException.class, () -> new JvmOption("OptionName", null)).getMessage(),
+            allOf(containsString("could not determine the origin of JVM option [OptionName]"), containsString("unsupported"))
+        );
+    }
+
 }
diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java
index 8acc94777bf0..5d63f29ac584 100644
--- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java
+++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.server.cli;
 
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.ESTestCase.WithoutSecurityManager;
 
@@ -291,7 +292,7 @@ public void accept(final int lineNumber, final String line) {
 
         final int javaMajorVersion = randomIntBetween(8, Integer.MAX_VALUE);
         final int smallerJavaMajorVersion = randomIntBetween(7, javaMajorVersion - 1);
-        final String invalidRangeLine = String.format(Locale.ROOT, "%d:%d-XX:+UseG1GC", javaMajorVersion, smallerJavaMajorVersion);
+        final String invalidRangeLine = Strings.format("%d:%d-XX:+UseG1GC", javaMajorVersion, smallerJavaMajorVersion);
         try (StringReader sr = new StringReader(invalidRangeLine); BufferedReader br = new BufferedReader(sr)) {
             assertInvalidLines(br, Collections.singletonMap(1, invalidRangeLine));
         }
@@ -306,8 +307,8 @@ public void accept(final int lineNumber, final String line) {
         );
         try (StringReader sr = new StringReader(numberFormatExceptionsLine); BufferedReader br = new BufferedReader(sr)) {
             final Map<Integer, String> invalidLines = new HashMap<>(2);
-            invalidLines.put(1, formatted("%d:-XX:+UseG1GC", invalidLowerJavaMajorVersion));
-            invalidLines.put(2, formatted("8-%d:-XX:+AggressiveOpts", invalidUpperJavaMajorVersion));
+            invalidLines.put(1, Strings.format("%d:-XX:+UseG1GC", invalidLowerJavaMajorVersion));
+            invalidLines.put(2, Strings.format("8-%d:-XX:+AggressiveOpts", invalidUpperJavaMajorVersion));
             assertInvalidLines(br, invalidLines);
         }
 
@@ -321,7 +322,7 @@ public void accept(final int lineNumber, final String line) {
 
         final int lowerBound = randomIntBetween(9, 16);
         final int upperBound = randomIntBetween(8, lowerBound - 1);
-        final String upperBoundGreaterThanLowerBound = String.format(Locale.ROOT, "%d-%d-XX:+UseG1GC", lowerBound, upperBound);
+        final String upperBoundGreaterThanLowerBound = Strings.format("%d-%d-XX:+UseG1GC", lowerBound, upperBound);
         try (StringReader sr = new StringReader(upperBoundGreaterThanLowerBound); BufferedReader br = new BufferedReader(sr)) {
             assertInvalidLines(br, Collections.singletonMap(1, upperBoundGreaterThanLowerBound));
         }
diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java
index 7a189563801e..f420834f84f3 100644
--- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java
+++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java
@@ -22,6 +22,8 @@
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.cli.EnvironmentAwareCommand;
 import org.elasticsearch.common.settings.KeyStoreWrapper;
+import org.elasticsearch.common.settings.SecureSettings;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.monitor.jvm.JvmInfo;
@@ -32,6 +34,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Locale;
+import java.util.Optional;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
@@ -46,9 +49,12 @@
 
 public class ServerCliTests extends CommandTestCase {
 
+    private SecureSettingsLoader mockSecureSettingsLoader;
+
     @Before
     public void setupMockConfig() throws IOException {
         Files.createFile(configDir.resolve("log4j2.properties"));
+        mockSecureSettingsLoader = null;
     }
 
     @Override
@@ -277,7 +283,7 @@ public void assertKeystorePassword(String password) throws Exception {
             }
         }
         String expectedPassword = password == null ? "" : password;
-        argsValidator = args -> assertThat(args.keystorePassword().toString(), equalTo(expectedPassword));
+        argsValidator = args -> assertThat(((KeyStoreWrapper) args.secrets()).hasPassword(), equalTo(hasPassword));
         autoConfigCallback = (t, options, env, processInfo) -> {
             char[] gotPassword = t.readSecret("");
             assertThat(gotPassword, equalTo(expectedPassword.toCharArray()));
@@ -314,6 +320,60 @@ public void testServerExitsNonZero() throws Exception {
         assertThat(exitCode, equalTo(140));
     }
 
+    public void testSecureSettingsLoaderChoice() throws Exception {
+        var loader = loadWithMockSecureSettingsLoader();
+        assertTrue(loader.loaded);
+        // the mock loader doesn't support autoconfigure, no need to bootstrap a keystore
+        assertFalse(loader.bootstrapped);
+        // assert that we ran the code to verify the environment
+        assertTrue(loader.verifiedEnv);
+    }
+
+    public void testSecureSettingsLoaderWithPassword() throws Exception {
+        var loader = setupMockKeystoreLoader();
+        assertKeystorePassword("aaaaaaaaaaaaaaaaaa");
+        assertTrue(loader.loaded);
+        assertTrue(loader.bootstrapped);
+        // the password we read should match what we passed in
+        assertEquals("aaaaaaaaaaaaaaaaaa", loader.password);
+        // after the command the secrets password is closed
+        assertEquals(
+            "SecureString has already been closed",
+            expectThrows(IllegalStateException.class, () -> loader.secrets.password().get().getChars()).getMessage()
+        );
+    }
+
+    public void testSecureSettingsLoaderWithEmptyPassword() throws Exception {
+        var loader = setupMockKeystoreLoader();
+        assertKeystorePassword("");
+        assertTrue(loader.loaded);
+        assertTrue(loader.bootstrapped);
+        assertEquals("", loader.password);
+    }
+
+    public void testSecureSettingsLoaderWithNullPassword() throws Exception {
+        var loader = setupMockKeystoreLoader();
+        assertKeystorePassword(null); // no keystore exists
+        assertTrue(loader.loaded);
+        assertTrue(loader.bootstrapped);
+        assertEquals("", loader.password);
+    }
+
+    private MockSecureSettingsLoader loadWithMockSecureSettingsLoader() throws Exception {
+        var loader = new MockSecureSettingsLoader();
+        this.mockSecureSettingsLoader = loader;
+        Command command = newCommand();
+        command.main(new String[0], terminal, new ProcessInfo(sysprops, envVars, esHomeDir));
+        command.close();
+        return loader;
+    }
+
+    private KeystoreSecureSettingsLoader setupMockKeystoreLoader() {
+        var loader = new KeystoreSecureSettingsLoader();
+        this.mockSecureSettingsLoader = loader;
+        return loader;
+    }
+
     interface AutoConfigMethod {
         void autoconfig(Terminal terminal, OptionSet options, Environment env, ProcessInfo processInfo) throws UserException;
     }
@@ -436,13 +496,120 @@ protected Command loadTool(String toolname, String libs) {
             }
 
             @Override
-            protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args, KeyStoreWrapper keystore) {
+            Environment autoConfigureSecurity(
+                Terminal terminal,
+                OptionSet options,
+                ProcessInfo processInfo,
+                Environment env,
+                SecureString keystorePassword
+            ) throws Exception {
+                if (mockSecureSettingsLoader != null && mockSecureSettingsLoader.supportsSecurityAutoConfiguration() == false) {
+                    fail("We shouldn't be calling auto configure on loaders that don't support it");
+                }
+                return super.autoConfigureSecurity(terminal, options, processInfo, env, keystorePassword);
+            }
+
+            @Override
+            protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) {
                 if (argsValidator != null) {
                     argsValidator.accept(args);
                 }
                 mockServer.reset();
                 return mockServer;
             }
+
+            @Override
+            void syncPlugins(Terminal terminal, Environment env, ProcessInfo processInfo) throws Exception {
+                if (mockSecureSettingsLoader != null && mockSecureSettingsLoader instanceof MockSecureSettingsLoader mock) {
+                    mock.verifiedEnv = true;
+                    // equals as a pointer, environment shouldn't be changed if autoconfigure is not supported
+                    assertFalse(mockSecureSettingsLoader.supportsSecurityAutoConfiguration());
+                    assertTrue(mock.environment == env);
+                }
+
+                super.syncPlugins(terminal, env, processInfo);
+            }
+
+            @Override
+            protected SecureSettingsLoader secureSettingsLoader(Environment env) {
+                if (mockSecureSettingsLoader != null) {
+                    return mockSecureSettingsLoader;
+                }
+
+                return new KeystoreSecureSettingsLoader();
+            }
         };
     }
+
+    static class MockSecureSettingsLoader implements SecureSettingsLoader {
+        boolean loaded = false;
+        LoadedSecrets secrets = null;
+        String password = null;
+        boolean bootstrapped = false;
+        Environment environment = null;
+        boolean verifiedEnv = false;
+
+        @Override
+        public SecureSettingsLoader.LoadedSecrets load(Environment environment, Terminal terminal) throws IOException {
+            loaded = true;
+            // Stash the environment pointer, so we can compare it. Environment shouldn't be changed for
+            // loaders that don't autoconfigure.
+            this.environment = environment;
+
+            SecureString password = null;
+
+            if (terminal.getReader().ready() == false) {
+                this.password = null;
+            } else {
+                password = new SecureString(terminal.readSecret("Enter a password"));
+                this.password = password.toString();
+            }
+
+            secrets = new SecureSettingsLoader.LoadedSecrets(
+                KeyStoreWrapper.create(),
+                password == null ? Optional.empty() : Optional.of(password)
+            );
+
+            return secrets;
+        }
+
+        @Override
+        public SecureSettings bootstrap(Environment environment, SecureString password) throws Exception {
+            fail("Bootstrap shouldn't be called for loaders that cannot be auto-configured");
+            bootstrapped = true;
+            return KeyStoreWrapper.create();
+        }
+
+        @Override
+        public boolean supportsSecurityAutoConfiguration() {
+            return false;
+        }
+    }
+
+    static class KeystoreSecureSettingsLoader extends KeyStoreLoader {
+        boolean loaded = false;
+        LoadedSecrets secrets = null;
+        String password = null;
+        boolean bootstrapped = false;
+
+        @Override
+        public LoadedSecrets load(Environment environment, Terminal terminal) throws Exception {
+            var result = super.load(environment, terminal);
+            loaded = true;
+            secrets = result;
+            password = result.password().get().toString();
+
+            return result;
+        }
+
+        @Override
+        public SecureSettings bootstrap(Environment environment, SecureString password) throws Exception {
+            this.bootstrapped = true;
+            // make sure we don't fail in fips mode when we run with an empty password
+            if (inFipsJvm() && (password == null || password.isEmpty())) {
+                return KeyStoreWrapper.create();
+            }
+            return super.bootstrap(environment, password);
+        }
+    }
 }
diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessTests.java
index f0fa37227119..c8dfb84363d6 100644
--- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessTests.java
+++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerProcessTests.java
@@ -15,7 +15,8 @@
 import org.elasticsearch.cli.ProcessInfo;
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
-import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.KeyStoreWrapper;
+import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.test.ESTestCase;
@@ -69,6 +70,7 @@ public class ServerProcessTests extends ESTestCase {
     ProcessValidator processValidator;
     MainMethod mainCallback;
     MockElasticsearchProcess process;
+    SecureSettings secrets;
 
     interface MainMethod {
         void main(ServerArgs args, InputStream stdin, PrintStream stderr, AtomicInteger exitCode) throws IOException;
@@ -78,9 +80,9 @@ interface ProcessValidator {
         void validate(ProcessBuilder processBuilder) throws IOException;
     }
 
-    void runForeground() throws Exception {
+    int runForeground() throws Exception {
         var server = startProcess(false, false, "");
-        server.waitFor();
+        return server.waitFor();
     }
 
     @Before
@@ -92,9 +94,10 @@ public void resetEnv() {
         envVars.clear();
         esHomeDir = createTempDir();
         nodeSettings = Settings.builder();
-        optionsBuilder = (args, keystore, configDir, tmpDir, envOptions) -> new ArrayList<>();
+        optionsBuilder = (args, configDir, tmpDir, envOptions) -> new ArrayList<>();
         processValidator = null;
         mainCallback = null;
+        secrets = KeyStoreWrapper.create();
     }
 
     @AfterClass
@@ -192,8 +195,7 @@ public Process destroyForcibly() {
 
     ServerProcess startProcess(boolean daemonize, boolean quiet, String keystorePassword) throws Exception {
         var pinfo = new ProcessInfo(Map.copyOf(sysprops), Map.copyOf(envVars), esHomeDir);
-        SecureString password = new SecureString(keystorePassword.toCharArray());
-        var args = new ServerArgs(daemonize, quiet, null, password, nodeSettings.build(), esHomeDir.resolve("config"));
+        var args = new ServerArgs(daemonize, quiet, null, secrets, nodeSettings.build(), esHomeDir.resolve("config"));
         ServerProcess.ProcessStarter starter = pb -> {
             if (processValidator != null) {
                 processValidator.validate(pb);
@@ -201,7 +203,7 @@ ServerProcess startProcess(boolean daemonize, boolean quiet, String keystorePass
             process = new MockElasticsearchProcess();
             return process;
         };
-        return ServerProcess.start(terminal, pinfo, args, null, optionsBuilder, starter);
+        return ServerProcess.start(terminal, pinfo, args, optionsBuilder, starter);
     }
 
     public void testProcessBuilder() throws Exception {
@@ -228,22 +230,12 @@ public void testPid() throws Exception {
 
     public void testBootstrapError() throws Exception {
         mainCallback = (args, stdin, stderr, exitCode) -> {
-            stderr.println(BootstrapInfo.USER_EXCEPTION_MARKER + "a bootstrap exception");
+            stderr.println("a bootstrap exception");
             exitCode.set(ExitCodes.CONFIG);
         };
-        var e = expectThrows(UserException.class, () -> runForeground());
-        assertThat(e.exitCode, equalTo(ExitCodes.CONFIG));
-        assertThat(e.getMessage(), equalTo("a bootstrap exception"));
-    }
-
-    public void testUserError() throws Exception {
-        mainCallback = (args, stdin, stderr, exitCode) -> {
-            stderr.println(BootstrapInfo.USER_EXCEPTION_MARKER + "a user exception");
-            exitCode.set(ExitCodes.USAGE);
-        };
-        var e = expectThrows(UserException.class, () -> runForeground());
-        assertThat(e.exitCode, equalTo(ExitCodes.USAGE));
-        assertThat(e.getMessage(), equalTo("a user exception"));
+        int exitCode = runForeground();
+        assertThat(exitCode, equalTo(ExitCodes.CONFIG));
+        assertThat(terminal.getErrorOutput(), containsString("a bootstrap exception"));
     }
 
     public void testStartError() throws Exception {
@@ -253,9 +245,7 @@ public void testStartError() throws Exception {
     }
 
     public void testOptionsBuildingInterrupted() throws Exception {
-        optionsBuilder = (args, keystore, configDir, tmpDir, envOptions) -> {
-            throw new InterruptedException("interrupted while get jvm options");
-        };
+        optionsBuilder = (args, configDir, tmpDir, envOptions) -> { throw new InterruptedException("interrupted while get jvm options"); };
         var e = expectThrows(RuntimeException.class, () -> runForeground());
         assertThat(e.getCause().getMessage(), equalTo("interrupted while get jvm options"));
     }
@@ -279,7 +269,7 @@ public void testLibffiEnv() throws Exception {
     }
 
     public void testTempDir() throws Exception {
-        optionsBuilder = (args, keystore, configDir, tmpDir, envOptions) -> {
+        optionsBuilder = (args, configDir, tmpDir, envOptions) -> {
             assertThat(tmpDir.toString(), Files.exists(tmpDir), is(true));
             assertThat(tmpDir.getFileName().toString(), startsWith("elasticsearch-"));
             return new ArrayList<>();
@@ -291,7 +281,7 @@ public void testTempDirWindows() throws Exception {
         Path baseTmpDir = createTempDir();
         sysprops.put("os.name", "Windows 10");
         sysprops.put("java.io.tmpdir", baseTmpDir.toString());
-        optionsBuilder = (args, keystore, configDir, tmpDir, envOptions) -> {
+        optionsBuilder = (args, configDir, tmpDir, envOptions) -> {
             assertThat(tmpDir.toString(), Files.exists(tmpDir), is(true));
             assertThat(tmpDir.getFileName().toString(), equalTo("elasticsearch"));
             assertThat(tmpDir.getParent().toString(), equalTo(baseTmpDir.toString()));
@@ -303,7 +293,7 @@ public void testTempDirWindows() throws Exception {
     public void testTempDirOverride() throws Exception {
         Path customTmpDir = createTempDir();
         envVars.put("ES_TMPDIR", customTmpDir.toString());
-        optionsBuilder = (args, keystore, configDir, tmpDir, envOptions) -> {
+        optionsBuilder = (args, configDir, tmpDir, envOptions) -> {
             assertThat(tmpDir.toString(), equalTo(customTmpDir.toString()));
             return new ArrayList<>();
         };
@@ -329,7 +319,7 @@ public void testTempDirOverrideNotADirectory() throws Exception {
 
     public void testCustomJvmOptions() throws Exception {
         envVars.put("ES_JAVA_OPTS", "-Dmyoption=foo");
-        optionsBuilder = (args, keystore, configDir, tmpDir, envOptions) -> {
+        optionsBuilder = (args, configDir, tmpDir, envOptions) -> {
             assertThat(envOptions, equalTo("-Dmyoption=foo"));
             return new ArrayList<>();
         };
@@ -338,7 +328,7 @@ public void testCustomJvmOptions() throws Exception {
     }
 
     public void testCommandLineSysprops() throws Exception {
-        optionsBuilder = (args, keystore, configDir, tmpDir, envOptions) -> List.of("-Dfoo1=bar", "-Dfoo2=baz");
+        optionsBuilder = (args, configDir, tmpDir, envOptions) -> List.of("-Dfoo1=bar", "-Dfoo2=baz");
         processValidator = pb -> {
             assertThat(pb.command(), contains("-Dfoo1=bar"));
             assertThat(pb.command(), contains("-Dfoo2=bar"));
diff --git a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java
index 637bd09eb2ce..5d1e67bc92fb 100644
--- a/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java
+++ b/distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cli.ProcessInfo;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.common.cli.EnvironmentAwareCommand;
+import org.elasticsearch.common.settings.KeyStoreWrapper;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.server.cli.ServerProcess;
@@ -34,9 +35,12 @@ class WindowsServiceDaemon extends EnvironmentAwareCommand {
 
     @Override
     public void execute(Terminal terminal, OptionSet options, Environment env, ProcessInfo processInfo) throws Exception {
-        var args = new ServerArgs(false, true, null, new SecureString(""), env.settings(), env.configFile());
-        this.server = ServerProcess.start(terminal, processInfo, args, null);
-        // start does not return until the server is ready, and we do not wait for the process
+        // the Windows service daemon doesn't support secure settings implementations other than the keystore
+        try (var loadedSecrets = KeyStoreWrapper.bootstrap(env.configFile(), () -> new SecureString(new char[0]))) {
+            var args = new ServerArgs(false, true, null, loadedSecrets, env.settings(), env.configFile());
+            this.server = ServerProcess.start(terminal, processInfo, args);
+            // start does not return until the server is ready, and we do not wait for the process
+        }
     }
 
     @Override
diff --git a/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceInstallCommandTests.java b/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceInstallCommandTests.java
index ec4b27c779d6..7753e09bd9f4 100644
--- a/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceInstallCommandTests.java
+++ b/distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceInstallCommandTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cli.Command;
 import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.core.Strings;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -153,13 +154,13 @@ public void testPidFile() throws Exception {
     }
 
     public void testDisplayName() throws Exception {
-        assertServiceArgs(Map.of("DisplayName", formatted("\"Elasticsearch %s (elasticsearch-service-x64)\"", Version.CURRENT)));
+        assertServiceArgs(Map.of("DisplayName", Strings.format("\"Elasticsearch %s (elasticsearch-service-x64)\"", Version.CURRENT)));
         envVars.put("SERVICE_DISPLAY_NAME", "my service name");
         assertServiceArgs(Map.of("DisplayName", "\"my service name\""));
     }
 
     public void testDescription() throws Exception {
-        String defaultDescription = formatted("\"Elasticsearch %s Windows Service - https://elastic.co\"", Version.CURRENT);
+        String defaultDescription = Strings.format("\"Elasticsearch %s Windows Service - https://elastic.co\"", Version.CURRENT);
         assertServiceArgs(Map.of("Description", defaultDescription));
         envVars.put("SERVICE_DESCRIPTION", "my description");
         assertServiceArgs(Map.of("Description", "\"my description\""));
diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc
index ce5a7dce1531..8e5464f0b388 100644
--- a/docs/Versions.asciidoc
+++ b/docs/Versions.asciidoc
@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        9.4.1
-:lucene_version_path:   9_4_1
+:lucene_version:        9.5.0
+:lucene_version_path:   9_5_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar
diff --git a/docs/changelog/85656.yaml b/docs/changelog/85656.yaml
new file mode 100644
index 000000000000..3ca31b32c6f6
--- /dev/null
+++ b/docs/changelog/85656.yaml
@@ -0,0 +1,6 @@
+pr: 85656
+summary: Instrument Weight#count in ProfileWeight
+area: Search
+type: enhancement
+issues:
+  - 85203
diff --git a/docs/changelog/86110.yaml b/docs/changelog/86110.yaml
deleted file mode 100644
index 376cadaa5660..000000000000
--- a/docs/changelog/86110.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 86110
-summary: Add LimitedOffsetsEnum to Limited offset token
-area: Search
-type: enhancement
-issues:
- - 86109
diff --git a/docs/changelog/86323.yaml b/docs/changelog/86323.yaml
deleted file mode 100644
index 4abf25204c37..000000000000
--- a/docs/changelog/86323.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 86323
-summary: Bulk merge field-caps responses using mapping hash
-area: Search
-type: enhancement
-issues: []
diff --git a/docs/changelog/88088.yaml b/docs/changelog/88088.yaml
deleted file mode 100644
index fedfd4942ae3..000000000000
--- a/docs/changelog/88088.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 88088
-summary: Move SpatialUtils to geo library
-area: Geo
-type: enhancement
-issues:
- - 86607
diff --git a/docs/changelog/88600.yaml b/docs/changelog/88600.yaml
deleted file mode 100644
index f1d9d8a8730a..000000000000
--- a/docs/changelog/88600.yaml
+++ /dev/null
@@ -1,10 +0,0 @@
-pr: 88600
-summary: Make `categorize_text` aggregation GA
-area: Machine Learning
-type: feature
-issues: []
-highlight:
-  title: Make `categorize_text` aggregation GA
-  body: The `categorize_text` aggregation has been moved from
-        technical preview to general availability.
-  notable: true
diff --git a/docs/changelog/88686.yaml b/docs/changelog/88686.yaml
new file mode 100644
index 000000000000..1346e725df1d
--- /dev/null
+++ b/docs/changelog/88686.yaml
@@ -0,0 +1,6 @@
+pr: 88686
+summary: "Fix: do not allow map key types other than String"
+area: Aggregations
+type: bug
+issues:
+ - 66057
diff --git a/docs/changelog/88952.yaml b/docs/changelog/88952.yaml
deleted file mode 100644
index 2d16663cbcf1..000000000000
--- a/docs/changelog/88952.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 88952
-summary: Prevalidate node removal API
-area: Allocation
-type: enhancement
-issues: []
diff --git a/docs/changelog/89216.yaml b/docs/changelog/89216.yaml
deleted file mode 100644
index ee7dc37dbdf3..000000000000
--- a/docs/changelog/89216.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 89216
-summary: Centroid aggregation for cartesian points and shapes
-area: Geo
-type: enhancement
-issues: []
diff --git a/docs/changelog/89238.yaml b/docs/changelog/89238.yaml
deleted file mode 100644
index 7f434db2aefe..000000000000
--- a/docs/changelog/89238.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 89238
-summary: "ILM: Get policy support wildcard name"
-area: ILM+SLM
-type: enhancement
-issues: []
diff --git a/docs/changelog/89461.yaml b/docs/changelog/89461.yaml
deleted file mode 100644
index f99afbabdbd4..000000000000
--- a/docs/changelog/89461.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 89461
-summary: Add certificate start/expiry dates to SSL Diagnostic message
-area: TLS
-type: enhancement
-issues: []
diff --git a/docs/changelog/89735.yaml b/docs/changelog/89735.yaml
deleted file mode 100644
index 8fd69d53b379..000000000000
--- a/docs/changelog/89735.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 89735
-summary: Operator/ingest
-area: Infra/Core
-type: enhancement
-issues: []
diff --git a/docs/changelog/89950.yaml b/docs/changelog/89950.yaml
deleted file mode 100644
index 2027209fcb2a..000000000000
--- a/docs/changelog/89950.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 89950
-summary: "Synthetic _source: support `field` in many cases"
-area: TSDB
-type: enhancement
-issues: []
diff --git a/docs/changelog/89965.yaml b/docs/changelog/89965.yaml
deleted file mode 100644
index 4616f2806ba0..000000000000
--- a/docs/changelog/89965.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 89965
-summary: "EQL sequences: support join on multi-values"
-area: EQL
-type: bug
-issues: []
diff --git a/docs/changelog/90038.yaml b/docs/changelog/90038.yaml
deleted file mode 100644
index 826b6f9284fe..000000000000
--- a/docs/changelog/90038.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90038
-summary: "Synthetic `_source`: `ignore_malformed` for `ip`"
-area: TSDB
-type: enhancement
-issues: []
diff --git a/docs/changelog/90143.yaml b/docs/changelog/90143.yaml
deleted file mode 100644
index 94124cba8fb5..000000000000
--- a/docs/changelog/90143.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90143
-summary: Operator/index templates
-area: Infra/Core
-type: enhancement
-issues: []
diff --git a/docs/changelog/90196.yaml b/docs/changelog/90196.yaml
deleted file mode 100644
index 8a8115ba6092..000000000000
--- a/docs/changelog/90196.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90196
-summary: "Synthetic `_source`: support `wildcard` field"
-area: TSDB
-type: enhancement
-issues: []
diff --git a/docs/changelog/90200.yaml b/docs/changelog/90200.yaml
deleted file mode 100644
index 36b2017ac712..000000000000
--- a/docs/changelog/90200.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90200
-summary: Add profiling information for knn vector queries
-area: Vector Search
-type: enhancement
-issues: []
diff --git a/docs/changelog/90282.yaml b/docs/changelog/90282.yaml
deleted file mode 100644
index e08ace64533a..000000000000
--- a/docs/changelog/90282.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 90282
-summary: Increase snaphot pool max size to 10
-area: Snapshot/Restore
-type: enhancement
-issues:
- - 89608
diff --git a/docs/changelog/90287.yaml b/docs/changelog/90287.yaml
deleted file mode 100644
index 954b90e61ece..000000000000
--- a/docs/changelog/90287.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90287
-summary: 'Revert "Remove `ImmutableOpenMap` from snapshot services"'
-area: "Infra/Core"
-type: regression
-issues: []
diff --git a/docs/changelog/90296.yaml b/docs/changelog/90296.yaml
deleted file mode 100644
index dd021ec08bf4..000000000000
--- a/docs/changelog/90296.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90296
-summary: Limit shard realocation retries
-area: Allocation
-type: enhancement
-issues: []
diff --git a/docs/changelog/90298.yaml b/docs/changelog/90298.yaml
deleted file mode 100644
index 549f690ed891..000000000000
--- a/docs/changelog/90298.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90298
-summary: Improve date math exclusions in expressions
-area: Infra/Core
-type: enhancement
-issues: []
diff --git a/docs/changelog/90303.yaml b/docs/changelog/90303.yaml
deleted file mode 100644
index 21292e424ad3..000000000000
--- a/docs/changelog/90303.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 90303
-summary: Use `IndexOrDocValues` query for IP range queries
-area: Search
-type: enhancement
-issues:
- - 83658
diff --git a/docs/changelog/90346.yaml b/docs/changelog/90346.yaml
deleted file mode 100644
index bf9388a49175..000000000000
--- a/docs/changelog/90346.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 90346
-summary: Update error states from inside the main state executor
-area: Infra/Core
-type: bug
-issues:
- - 90337
diff --git a/docs/changelog/90381.yaml b/docs/changelog/90381.yaml
deleted file mode 100644
index a7f467510be4..000000000000
--- a/docs/changelog/90381.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90381
-summary: aggregate metric double add a max min validation
-area: Mapping
-type: enhancement
-issues: []
diff --git a/docs/changelog/90391.yaml b/docs/changelog/90391.yaml
deleted file mode 100644
index 4565a9503439..000000000000
--- a/docs/changelog/90391.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 90391
-summary: Remove resize index settings once shards are started
-area: Recovery
-type: enhancement
-issues:
- - 90127
diff --git a/docs/changelog/90399.yaml b/docs/changelog/90399.yaml
deleted file mode 100644
index 485f9eeeedd9..000000000000
--- a/docs/changelog/90399.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-pr: 90399
-summary: Deprecate state field in /_cluster/reroute response
-area: Allocation
-type: deprecation
-issues: []
-deprecation:
-  title: state field is deprecated in /_cluster/reroute response
-  area: REST API
-  details: |-
-    `state` field is deprecated in `/_cluster/reroute` response. Cluster state does not provide meaningful information
-    about the result of reroute/commands execution. There are no guarantees that this exact state would be applied.
-  impact: |-
-    Reroute API users should not rely on `state` field and instead use `explain` to request result of commands execution.
diff --git a/docs/changelog/90425.yaml b/docs/changelog/90425.yaml
deleted file mode 100644
index 6205900f8839..000000000000
--- a/docs/changelog/90425.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90425
-summary: Enhance nested depth tracking when parsing queries
-area: Search
-type: enhancement
-issues: []
diff --git a/docs/changelog/90428.yaml b/docs/changelog/90428.yaml
deleted file mode 100644
index cb65566f7576..000000000000
--- a/docs/changelog/90428.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90428
-summary: Support malformed numbers in synthetic `_source`
-area: TSDB
-type: enhancement
-issues: []
diff --git a/docs/changelog/90447.yaml b/docs/changelog/90447.yaml
new file mode 100644
index 000000000000..198bbb781565
--- /dev/null
+++ b/docs/changelog/90447.yaml
@@ -0,0 +1,5 @@
+pr: 90447
+summary: Add a TSDB rate aggregation
+area: "TSDB"
+type: feature
+issues: []
diff --git a/docs/changelog/90460.yaml b/docs/changelog/90460.yaml
deleted file mode 100644
index 57e70ac3aebe..000000000000
--- a/docs/changelog/90460.yaml
+++ /dev/null
@@ -1,10 +0,0 @@
-pr: 90460
-summary: Deprecate 'remove_binary' default of false for ingest attachment processor
-area: Ingest Node
-type: deprecation
-issues: []
-deprecation:
-  title: Deprecate 'remove_binary' default of false for ingest attachment processor
-  area: CRUD
-  details: The default "remove_binary" option for the attachment processor will be changed from false to true in a later Elasticsearch release. This means that the binary file sent to Elasticsearch will not be retained.
-  impact: Users should update the "remove_binary" option to be explicitly true or false, instead of relying on the default value, so that no default value changes will affect Elasticsearch.
diff --git a/docs/changelog/90482.yaml b/docs/changelog/90482.yaml
deleted file mode 100644
index 61f3e42609ea..000000000000
--- a/docs/changelog/90482.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 90482
-summary: Transport threads and `_hot_threads`
-area: Infra/Core
-type: enhancement
-issues:
- - 90334
diff --git a/docs/changelog/90494.yaml b/docs/changelog/90494.yaml
deleted file mode 100644
index 9c73655f0f0c..000000000000
--- a/docs/changelog/90494.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-pr: 90494
-summary: Fix logstash loadavg (xpack cases)
-area: Monitoring
-type: bug
-issues: []
-highlight:
-  title: "Stack Monitoring: Logstash load average type fixed for metricbeat collection"
-  body: |-
-    Previously, the templates for ingesting logstash load average using metricbeat were set to `long`. This provides only an integer graph.
-
-    The type has been corrected to `half_float`. You can force a rollover to see the change immediately or wait for the next ILM rollover.
-
-    [source,console]
-    ----
-    POST .monitoring-logstash-8-mb/_rollover
-    ----
diff --git a/docs/changelog/90534.yaml b/docs/changelog/90534.yaml
new file mode 100644
index 000000000000..c24735d94879
--- /dev/null
+++ b/docs/changelog/90534.yaml
@@ -0,0 +1,6 @@
+pr: 90534
+summary: Fix fransport handshake starting before tls handshake completes
+area: Network
+type: bug
+issues:
+ - 77999
diff --git a/docs/changelog/90536.yaml b/docs/changelog/90536.yaml
deleted file mode 100644
index b9ec1e33e96c..000000000000
--- a/docs/changelog/90536.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 90536
-summary: Add profiling and documentation for dfs phase
-area: Search
-type: enhancement
-issues:
- - 89713
diff --git a/docs/changelog/90552.yaml b/docs/changelog/90552.yaml
deleted file mode 100644
index ae294d2edd2d..000000000000
--- a/docs/changelog/90552.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 90552
-summary: "Generate 'index.routing_path' from dynamic mapping templates"
-area: TSDB
-type: enhancement
-issues:
-  - 90528
diff --git a/docs/changelog/90553.yaml b/docs/changelog/90553.yaml
deleted file mode 100644
index d6cfdbc48e4f..000000000000
--- a/docs/changelog/90553.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90553
-summary: Upgrade XContent to Jackson 2.14.0 and enable Fast Double Parser
-area: Infra/Core
-type: enhancement
-issues: []
diff --git a/docs/changelog/90585.yaml b/docs/changelog/90585.yaml
deleted file mode 100644
index 77adbb6d274a..000000000000
--- a/docs/changelog/90585.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 90585
-summary: Return docs when using nested mappings in archive indices
-area: Search
-type: enhancement
-issues:
- - 90523
diff --git a/docs/changelog/90589.yaml b/docs/changelog/90589.yaml
deleted file mode 100644
index c0e0f8298d98..000000000000
--- a/docs/changelog/90589.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 90589
-summary: Upgrade to log4j 2.19.0
-area: Infra/Logging
-type: upgrade
-issues:
- - 90584
diff --git a/docs/changelog/90593.yaml b/docs/changelog/90593.yaml
deleted file mode 100644
index 23f8acc7fa38..000000000000
--- a/docs/changelog/90593.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90593
-summary: Test downsample runtime fields and security
-area: Rollup
-type: enhancement
-issues: []
diff --git a/docs/changelog/90604.yaml b/docs/changelog/90604.yaml
deleted file mode 100644
index 94c913cc1c12..000000000000
--- a/docs/changelog/90604.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90604
-summary: Upgrade to Netty 4.1.82.Final
-area: Network
-type: upgrade
-issues: []
diff --git a/docs/changelog/90612.yaml b/docs/changelog/90612.yaml
deleted file mode 100644
index c69cc1fa3d8e..000000000000
--- a/docs/changelog/90612.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90612
-summary: Make `knn` search requests fully cancellable
-area: Vector Search
-type: bug
-issues: []
diff --git a/docs/changelog/90649.yaml b/docs/changelog/90649.yaml
deleted file mode 100644
index 9a377456a088..000000000000
--- a/docs/changelog/90649.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90649
-summary: "[Stack Monitoring] Update ES module mappings"
-area: Monitoring
-type: bug
-issues: []
diff --git a/docs/changelog/90675.yaml b/docs/changelog/90675.yaml
deleted file mode 100644
index 6fda273b6b08..000000000000
--- a/docs/changelog/90675.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90675
-summary: Provide additional information about anomaly score factors
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/90688.yaml b/docs/changelog/90688.yaml
deleted file mode 100644
index f848ca01a7be..000000000000
--- a/docs/changelog/90688.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-pr: 90688
-summary: Fixing a race condition in `EnrichCoordinatorProxyAction` that can leave
-  an item stuck in its queue
-area: Ingest Node
-type: bug
-issues:
- - 90598
diff --git a/docs/changelog/90723.yaml b/docs/changelog/90723.yaml
deleted file mode 100644
index 7ebf4c21f7ee..000000000000
--- a/docs/changelog/90723.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90723
-summary: Add a regex to the output of the `categorize_text` aggregation
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/90724.yaml b/docs/changelog/90724.yaml
deleted file mode 100644
index bafe4c28f264..000000000000
--- a/docs/changelog/90724.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 90724
-summary: Fix `TransportMasterNodeAction` holding a CS reference needlessly
-area: Cluster Coordination
-type: bug
-issues:
- - 89220
diff --git a/docs/changelog/90728.yaml b/docs/changelog/90728.yaml
deleted file mode 100644
index 79efcd0050ef..000000000000
--- a/docs/changelog/90728.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90728
-summary: Add api to update trained model deployment `number_of_allocations`
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/90760.yaml b/docs/changelog/90760.yaml
deleted file mode 100644
index 6656ed2ce041..000000000000
--- a/docs/changelog/90760.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90760
-summary: Add a health section to transform stats
-area: Transform
-type: enhancement
-issues: []
diff --git a/docs/changelog/90764.yaml b/docs/changelog/90764.yaml
deleted file mode 100644
index c8f759ef8200..000000000000
--- a/docs/changelog/90764.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90764
-summary: Allow overriding timestamp field to null in file structure finder
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/90768.yaml b/docs/changelog/90768.yaml
deleted file mode 100644
index f011973a0d99..000000000000
--- a/docs/changelog/90768.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 90768
-summary: Keep track of average shard write load
-area: CRUD
-type: enhancement
-issues:
- - 90102
diff --git a/docs/changelog/90774.yaml b/docs/changelog/90774.yaml
deleted file mode 100644
index 9fb5d11ea15f..000000000000
--- a/docs/changelog/90774.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90774
-summary: Add support for indexing byte-sized knn vectors
-area: Vector Search
-type: feature
-issues: []
diff --git a/docs/changelog/90805.yaml b/docs/changelog/90805.yaml
deleted file mode 100644
index b4147ad94004..000000000000
--- a/docs/changelog/90805.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90805
-summary: Example stable plugin
-area: Infra/Plugins
-type: enhancement
-issues: []
diff --git a/docs/changelog/90806.yaml b/docs/changelog/90806.yaml
deleted file mode 100644
index e97d8ae86476..000000000000
--- a/docs/changelog/90806.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90806
-summary: Don't create IndexCaps objects when recording unmapped fields
-area: Mapping
-type: enhancement
-issues: [90796]
diff --git a/docs/changelog/90807.yaml b/docs/changelog/90807.yaml
deleted file mode 100644
index 0fef9f51dbc6..000000000000
--- a/docs/changelog/90807.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90807
-summary: Check `NodesShutdownMetadata` type before assuming restart
-area: Allocation
-type: bug
-issues: []
diff --git a/docs/changelog/90812.yaml b/docs/changelog/90812.yaml
deleted file mode 100644
index d440c9685317..000000000000
--- a/docs/changelog/90812.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90812
-summary: Alias timestamp to @timestamp in anomaly detection results index
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/90822.yaml b/docs/changelog/90822.yaml
deleted file mode 100644
index 124d053712ab..000000000000
--- a/docs/changelog/90822.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90822
-summary: Fix handling empty key case in the terms aggregation
-area: Mapping
-type: bug
-issues: []
diff --git a/docs/changelog/90835.yaml b/docs/changelog/90835.yaml
deleted file mode 100644
index 8eac3367847b..000000000000
--- a/docs/changelog/90835.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 90835
-summary: "Make `extendedPlugins,` `HasNativeController` and `moduleName` optional\
-  \ in plugin descriptor"
-area: Infra/Plugins
-type: enhancement
-issues: []
diff --git a/docs/changelog/90870.yaml b/docs/changelog/90870.yaml
deleted file mode 100644
index ce68c4cdc6e2..000000000000
--- a/docs/changelog/90870.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90870
-summary: Create placeholder plugin when loading stable plugins
-area: Infra/Plugins
-type: enhancement
-issues: []
diff --git a/docs/changelog/90902.yaml b/docs/changelog/90902.yaml
deleted file mode 100644
index ecebdd1feca9..000000000000
--- a/docs/changelog/90902.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90902
-summary: Index expression exclusions never trigger "not found"
-area: Infra/Core
-type: bug
-issues: []
diff --git a/docs/changelog/90931.yaml b/docs/changelog/90931.yaml
deleted file mode 100644
index 045ca8a1659b..000000000000
--- a/docs/changelog/90931.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90931
-summary: Refactor enrich maintenance coordination logic
-area: Ingest Node
-type: enhancement
-issues: []
diff --git a/docs/changelog/90977.yaml b/docs/changelog/90977.yaml
new file mode 100644
index 000000000000..995a84299920
--- /dev/null
+++ b/docs/changelog/90977.yaml
@@ -0,0 +1,6 @@
+pr: 90977
+summary: '`TransportListTaskAction:` wait for tasks to finish asynchronously'
+area: Task Management
+type: enhancement
+issues:
+ - 89564
diff --git a/docs/changelog/90989.yaml b/docs/changelog/90989.yaml
deleted file mode 100644
index 668c075d3148..000000000000
--- a/docs/changelog/90989.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-pr: 90989
-summary: Deprecate silently ignoring type, fields, copy_to and boost in metadata field definition
-area: Mapping
-type: deprecation
-issues:
- - 35389
-deprecation:
-  title: Deprecate silently ignoring type, fields, copy_to and boost in metadata field definition
-  area: Mapping
-  details: Unsupported parameters like type, fields, copy_to and boost are silently ignored when
-    provided as part of the configuration of a metadata field in the index mappings. They will
-    cause a deprecation warning when used in the mappings for indices that are created from 8.6
-    onwards.
-  impact: To resolve the deprecation warning, remove the mention of type, fields, copy_to or boost
-    from any metadata field definition as part of index mappings. They take no effect so removing
-    them won't have any impact besides resolving the deprecation warning.
diff --git a/docs/changelog/90993.yaml b/docs/changelog/90993.yaml
deleted file mode 100644
index 209522227eeb..000000000000
--- a/docs/changelog/90993.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 90993
-summary: Add "index" and "search" node roles with feature flag and setting
-area: Distributed
-type: feature
-issues: []
diff --git a/docs/changelog/91006.yaml b/docs/changelog/91006.yaml
deleted file mode 100644
index 56ef752f1895..000000000000
--- a/docs/changelog/91006.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91006
-summary: Fix potential issue with graph api's timed out field in response
-area: Graph
-type: bug
-issues: []
diff --git a/docs/changelog/91019.yaml b/docs/changelog/91019.yaml
deleted file mode 100644
index 5753ab4d7fba..000000000000
--- a/docs/changelog/91019.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91019
-summary: Store write load in the `IndexMetadata` during data streams rollovers
-area: Allocation
-type: enhancement
-issues: []
diff --git a/docs/changelog/91021.yaml b/docs/changelog/91021.yaml
deleted file mode 100644
index 5e22c1c76fa4..000000000000
--- a/docs/changelog/91021.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 91021
-summary: Tie snapshot speed to node bandwidth settings
-area: Snapshot/Restore
-type: enhancement
-issues:
- - 57023
diff --git a/docs/changelog/91038.yaml b/docs/changelog/91038.yaml
deleted file mode 100644
index ce0a8d00abb4..000000000000
--- a/docs/changelog/91038.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 91038
-summary: '`DesiredBalance:` expose it via _internal/desired_balance'
-area: Allocation
-type: enhancement
-issues:
- - 90583
diff --git a/docs/changelog/91045.yaml b/docs/changelog/91045.yaml
deleted file mode 100644
index ca5bc4240d9f..000000000000
--- a/docs/changelog/91045.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91045
-summary: Support `aggregate_metric_double` field type in transform aggregations
-area: Transform
-type: enhancement
-issues: []
diff --git a/docs/changelog/91047.yaml b/docs/changelog/91047.yaml
deleted file mode 100644
index 73cbf0c402d8..000000000000
--- a/docs/changelog/91047.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91047
-summary: Security remove datemath special handling
-area: Authorization
-type: bug
-issues: []
diff --git a/docs/changelog/91073.yaml b/docs/changelog/91073.yaml
deleted file mode 100644
index 30d41b6dc0b7..000000000000
--- a/docs/changelog/91073.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 91073
-summary: GeoBoundsAggregations reject sub aggregations
-area: Aggregations
-type: bug
-issues:
- - 91072
diff --git a/docs/changelog/91116.yaml b/docs/changelog/91116.yaml
new file mode 100644
index 000000000000..ed35fa364e47
--- /dev/null
+++ b/docs/changelog/91116.yaml
@@ -0,0 +1,6 @@
+pr: 91116
+summary: Add from parameter to Transform Start API
+area: Transform
+type: enhancement
+issues:
+ - 88646
diff --git a/docs/changelog/91137.yaml b/docs/changelog/91137.yaml
deleted file mode 100644
index 3f54a1599a85..000000000000
--- a/docs/changelog/91137.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91137
-summary: Add a filter parameter to frequent items
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/91140.yaml b/docs/changelog/91140.yaml
deleted file mode 100644
index 733e7bfdda46..000000000000
--- a/docs/changelog/91140.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91140
-summary: Improve H3#hexRing logic and add H3#areNeighborCells method
-area: Geo
-type: enhancement
-issues: []
diff --git a/docs/changelog/91184.yaml b/docs/changelog/91184.yaml
deleted file mode 100644
index 6bacb199d8c0..000000000000
--- a/docs/changelog/91184.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91184
-summary: Add fielddata and scripting support for byte-sized vectors
-area: Vector Search
-type: feature
-issues: []
diff --git a/docs/changelog/91195.yaml b/docs/changelog/91195.yaml
deleted file mode 100644
index f42ed122a3f0..000000000000
--- a/docs/changelog/91195.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 91195
-summary: Allow different decay values depending on the score function
-area: Search
-type: bug
-issues:
-  - 78887
diff --git a/docs/changelog/91231.yaml b/docs/changelog/91231.yaml
deleted file mode 100644
index 1ca88ff84d8d..000000000000
--- a/docs/changelog/91231.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91231
-summary: Fix index expression options for requests with a single name or pattern
-area: Infra/Core
-type: bug
-issues: []
diff --git a/docs/changelog/91234.yaml b/docs/changelog/91234.yaml
deleted file mode 100644
index 1e25efe5aa30..000000000000
--- a/docs/changelog/91234.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 91234
-summary: Low priority trained model deployments
-area: Machine Learning
-type: enhancement
-issues:
- - 91024
diff --git a/docs/changelog/91238.yaml b/docs/changelog/91238.yaml
new file mode 100644
index 000000000000..cfac17b27e60
--- /dev/null
+++ b/docs/changelog/91238.yaml
@@ -0,0 +1,7 @@
+pr: 91238
+summary: Avoiding `BulkProcessor` deadlock in ILMHistoryStore
+area: ILM+SLM
+type: bug
+issues:
+ - 68468
+ - 50440
diff --git a/docs/changelog/91243.yaml b/docs/changelog/91243.yaml
deleted file mode 100644
index ac0433b51769..000000000000
--- a/docs/changelog/91243.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91243
-summary: More actionable error for ancient indices
-area: Infra/Core
-type: enhancement
-issues: []
diff --git a/docs/changelog/91256.yaml b/docs/changelog/91256.yaml
new file mode 100644
index 000000000000..3c829465073b
--- /dev/null
+++ b/docs/changelog/91256.yaml
@@ -0,0 +1,6 @@
+pr: 91256
+summary: Prevalidate node removal API (pt. 2)
+area: Allocation
+type: enhancement
+issues:
+ - 87776
diff --git a/docs/changelog/91271.yaml b/docs/changelog/91271.yaml
deleted file mode 100644
index 43ad13596bc1..000000000000
--- a/docs/changelog/91271.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91271
-summary: Upgrade to Netty 4.1.84
-area: Network
-type: upgrade
-issues: []
diff --git a/docs/changelog/91296.yaml b/docs/changelog/91296.yaml
deleted file mode 100644
index 83889a6929ae..000000000000
--- a/docs/changelog/91296.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91296
-summary: Allow `model_aliases` to be used with Pytorch trained models
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/91298.yaml b/docs/changelog/91298.yaml
deleted file mode 100644
index 77077b3b0b29..000000000000
--- a/docs/changelog/91298.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 91298
-summary: Support `cartesian_bounds` aggregation on point and shape
-area: Geo
-type: enhancement
-issues:
- - 90157
diff --git a/docs/changelog/91299.yaml b/docs/changelog/91299.yaml
deleted file mode 100644
index 9d50b2244b4a..000000000000
--- a/docs/changelog/91299.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91299
-summary: Add ability to filter and sort buckets by `change_point` numeric values
-area: Machine Learning
-type: enhancement
-issues: []
diff --git a/docs/changelog/91306.yaml b/docs/changelog/91306.yaml
deleted file mode 100644
index 4170ecd1973a..000000000000
--- a/docs/changelog/91306.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91306
-summary: Rename `NamedComponent` name parameter to value
-area: Infra/Plugins
-type: enhancement
-issues: []
diff --git a/docs/changelog/91312.yaml b/docs/changelog/91312.yaml
deleted file mode 100644
index e5628ad74190..000000000000
--- a/docs/changelog/91312.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91312
-summary: EQL samples
-area: EQL
-type: feature
-issues: []
diff --git a/docs/changelog/91328.yaml b/docs/changelog/91328.yaml
deleted file mode 100644
index b225d9619ac8..000000000000
--- a/docs/changelog/91328.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91328
-summary: Consolidate field name validation when parsing mappings and documents
-area: Mapping
-type: bug
-issues: []
diff --git a/docs/changelog/91338.yaml b/docs/changelog/91338.yaml
deleted file mode 100644
index 50c08b3ce5f8..000000000000
--- a/docs/changelog/91338.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91338
-summary: Extend systemd startup timeout to 900s
-area: Infra/Core
-type: enhancement
-issues: []
diff --git a/docs/changelog/91343.yaml b/docs/changelog/91343.yaml
deleted file mode 100644
index 27a59ec487d0..000000000000
--- a/docs/changelog/91343.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91343
-summary: Introduce desired-balance allocator
-area: Allocation
-type: enhancement
-issues: []
diff --git a/docs/changelog/91347.yaml b/docs/changelog/91347.yaml
deleted file mode 100644
index 254f5474f1e7..000000000000
--- a/docs/changelog/91347.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91347
-summary: "Use an explicit null check for null receivers in painless, rather than an NPE"
-area: Infra/Scripting
-type: enhancement
-issues: [91236]
diff --git a/docs/changelog/91362.yaml b/docs/changelog/91362.yaml
deleted file mode 100644
index a34bec585229..000000000000
--- a/docs/changelog/91362.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 91362
-summary: "Introduce a phase to use String.equals on constant strings, rather than\
-  \ def equality"
-area: Infra/Core
-type: enhancement
-issues: [91235]
diff --git a/docs/changelog/91364.yaml b/docs/changelog/91364.yaml
deleted file mode 100644
index a2da37f99a37..000000000000
--- a/docs/changelog/91364.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91364
-summary: Fix `BytesRefArray` on append empty `BytesRef`
-area: Infra/Core
-type: bug
-issues: []
diff --git a/docs/changelog/91367.yaml b/docs/changelog/91367.yaml
new file mode 100644
index 000000000000..2ec5842ee132
--- /dev/null
+++ b/docs/changelog/91367.yaml
@@ -0,0 +1,5 @@
+pr: 91367
+summary: Deserialize responses on the handling thread-pool
+area: Network
+type: enhancement
+issues: []
diff --git a/docs/changelog/91398.yaml b/docs/changelog/91398.yaml
new file mode 100644
index 000000000000..2127e983cbef
--- /dev/null
+++ b/docs/changelog/91398.yaml
@@ -0,0 +1,5 @@
+pr: 91398
+summary: Deduplicate Heavy CCR Repository CS Requests
+area: CCR
+type: bug
+issues: []
diff --git a/docs/changelog/91409.yaml b/docs/changelog/91409.yaml
deleted file mode 100644
index 0fc8febae6d7..000000000000
--- a/docs/changelog/91409.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91409
-summary: Remove version limitations for CCS
-area: EQL
-type: enhancement
-issues: []
diff --git a/docs/changelog/91413.yaml b/docs/changelog/91413.yaml
new file mode 100644
index 000000000000..4a6d68004920
--- /dev/null
+++ b/docs/changelog/91413.yaml
@@ -0,0 +1,5 @@
+pr: 91413
+summary: "[Fleet] Add files and files data index templates and ILM policies"
+area: Infra/Plugins
+type: feature
+issues: []
diff --git a/docs/changelog/91425.yaml b/docs/changelog/91425.yaml
deleted file mode 100644
index 3200fcc2655f..000000000000
--- a/docs/changelog/91425.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91425
-summary: Forecast write load during rollovers
-area: Allocation
-type: enhancement
-issues: []
diff --git a/docs/changelog/91438.yaml b/docs/changelog/91438.yaml
deleted file mode 100644
index a0dcb63c3712..000000000000
--- a/docs/changelog/91438.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 91438
-summary: Handle APM global labels as affix setting
-area: Infra/Core
-type: enhancement
-issues:
- - 91278
diff --git a/docs/changelog/91461.yaml b/docs/changelog/91461.yaml
deleted file mode 100644
index 507e23e190b6..000000000000
--- a/docs/changelog/91461.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91461
-summary: Datastream unavailable exception metadata
-area: Infra/Core
-type: bug
-issues: []
diff --git a/docs/changelog/91465.yaml b/docs/changelog/91465.yaml
deleted file mode 100644
index fa8f78a897a4..000000000000
--- a/docs/changelog/91465.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91465
-summary: Support synthetic `_source` for `_doc_count` field
-area: TSDB
-type: enhancement
-issues: []
diff --git a/docs/changelog/91467.yaml b/docs/changelog/91467.yaml
new file mode 100644
index 000000000000..ebf7d2fcd248
--- /dev/null
+++ b/docs/changelog/91467.yaml
@@ -0,0 +1,5 @@
+pr: 91467
+summary: Settings api for stable plugins
+area: Infra/Plugins
+type: enhancement
+issues: []
diff --git a/docs/changelog/91490.yaml b/docs/changelog/91490.yaml
deleted file mode 100644
index a25d1312deed..000000000000
--- a/docs/changelog/91490.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91490
-summary: Avoid potential unsupported operation exception in doc bitset cache
-area: Authorization
-type: bug
-issues: []
diff --git a/docs/changelog/91492.yaml b/docs/changelog/91492.yaml
deleted file mode 100644
index 739b3fa68ab2..000000000000
--- a/docs/changelog/91492.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91492
-summary: Reduce number of object allocations in H3#geoToH3 and speed up computations
-area: Geo
-type: enhancement
-issues: []
diff --git a/docs/changelog/91499.yaml b/docs/changelog/91499.yaml
new file mode 100644
index 000000000000..600c0f27cedd
--- /dev/null
+++ b/docs/changelog/91499.yaml
@@ -0,0 +1,5 @@
+pr: 91499
+summary: Extra `kibana_system` privileges for Fleet transform upgrades
+area: Authorization
+type: enhancement
+issues: []
diff --git a/docs/changelog/91501.yaml b/docs/changelog/91501.yaml
deleted file mode 100644
index 054401a090b8..000000000000
--- a/docs/changelog/91501.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 91501
-summary: Fix compile with hex literals ending with d/f
-area: Infra/Scripting
-type: bug
-issues:
- - 88614
diff --git a/docs/changelog/91506.yaml b/docs/changelog/91506.yaml
deleted file mode 100644
index 09263d2b7ec1..000000000000
--- a/docs/changelog/91506.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91506
-summary: SLM uneahlthy policies diagnosis recommends correct URL in action
-area: Health
-type: bug
-issues: []
diff --git a/docs/changelog/91508.yaml b/docs/changelog/91508.yaml
new file mode 100644
index 000000000000..33356fa3da11
--- /dev/null
+++ b/docs/changelog/91508.yaml
@@ -0,0 +1,5 @@
+pr: 91508
+summary: "Add kibana.stats.elasticsearch_client stats to the monitoring index templates."
+area: Monitoring
+type: enhancement
+issues: []
diff --git a/docs/changelog/91510.yaml b/docs/changelog/91510.yaml
deleted file mode 100644
index 06bbf0bc30eb..000000000000
--- a/docs/changelog/91510.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91510
-summary: Refine bwc version checks on `EqlSearchRequest`
-area: EQL
-type: bug
-issues: []
diff --git a/docs/changelog/91515.yaml b/docs/changelog/91515.yaml
deleted file mode 100644
index 57229217f979..000000000000
--- a/docs/changelog/91515.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 91515
-summary: Use chunked encoding for `RestGetHealthAction`
-area: Health
-type: feature
-issues:
- - 90223
diff --git a/docs/changelog/91528.yaml b/docs/changelog/91528.yaml
deleted file mode 100644
index 7415220c923f..000000000000
--- a/docs/changelog/91528.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91528
-summary: Expose telemetry about search usage
-area: Search
-type: enhancement
-issues: []
diff --git a/docs/changelog/91546.yaml b/docs/changelog/91546.yaml
deleted file mode 100644
index 9dbb77ab15f2..000000000000
--- a/docs/changelog/91546.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 91546
-summary: Copy more settings when creating DF analytics destination index
-area: Machine Learning
-type: bug
-issues:
- - 89795
diff --git a/docs/changelog/91556.yaml b/docs/changelog/91556.yaml
deleted file mode 100644
index 44d08d58730a..000000000000
--- a/docs/changelog/91556.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91556
-summary: Allow plugins to wrap Lucene directories created by the `IndexModule`
-area: Store
-type: enhancement
-issues: []
diff --git a/docs/changelog/91561.yaml b/docs/changelog/91561.yaml
deleted file mode 100644
index 91662e5ca1a9..000000000000
--- a/docs/changelog/91561.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91561
-summary: Forecast average shard size during rollovers
-area: Allocation
-type: enhancement
-issues: []
diff --git a/docs/changelog/91567.yaml b/docs/changelog/91567.yaml
deleted file mode 100644
index c3eeb4dc0302..000000000000
--- a/docs/changelog/91567.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 91567
-summary: Refactor `DatabaseNodeService` as a cluster state listener
-area: Ingest Node
-type: bug
-issues:
- - 86999
diff --git a/docs/changelog/91587.yaml b/docs/changelog/91587.yaml
deleted file mode 100644
index f9c22f961738..000000000000
--- a/docs/changelog/91587.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91587
-summary: "[HealthAPI] Use the `RestCancellableNodeClient` infrastructure"
-area: Health
-type: feature
-issues: []
diff --git a/docs/changelog/91590.yaml b/docs/changelog/91590.yaml
deleted file mode 100644
index fb02f3d260bb..000000000000
--- a/docs/changelog/91590.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 91590
-summary: Clear up forecasted write load and shard size from previous write index during
-  rollovers
-area: Allocation
-type: enhancement
-issues: []
diff --git a/docs/changelog/91603.yaml b/docs/changelog/91603.yaml
deleted file mode 100644
index 8406c6d2fb22..000000000000
--- a/docs/changelog/91603.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91603
-summary: Improve shard balancing
-area: Allocation
-type: enhancement
-issues: []
diff --git a/docs/changelog/91622.yaml b/docs/changelog/91622.yaml
deleted file mode 100644
index f105f27d95c6..000000000000
--- a/docs/changelog/91622.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 91622
-summary: Fix failure when resolving indices from CCS
-area: Transform
-type: bug
-issues:
- - 91550
diff --git a/docs/changelog/91640.yaml b/docs/changelog/91640.yaml
new file mode 100644
index 000000000000..e581b03d523c
--- /dev/null
+++ b/docs/changelog/91640.yaml
@@ -0,0 +1,5 @@
+pr: 91640
+summary: Add profiling plugin
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/91659.yaml b/docs/changelog/91659.yaml
deleted file mode 100644
index 80cc3f5bb5d7..000000000000
--- a/docs/changelog/91659.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 91659
-summary: Avoid NPE when disassociateDeadNodes is executed for a node present in the
-  desired balance
-area: Allocation
-type: bug
-issues: []
diff --git a/docs/changelog/91673.yaml b/docs/changelog/91673.yaml
new file mode 100644
index 000000000000..86fb9f487658
--- /dev/null
+++ b/docs/changelog/91673.yaml
@@ -0,0 +1,5 @@
+pr: 91673
+summary: "Add new H3 api method #h3ToNoChildrenIntersecting"
+area: Geo
+type: enhancement
+issues: []
diff --git a/docs/changelog/91708.yaml b/docs/changelog/91708.yaml
new file mode 100644
index 000000000000..a4b9e0817508
--- /dev/null
+++ b/docs/changelog/91708.yaml
@@ -0,0 +1,6 @@
+pr: 91708
+summary: Expose Health Api telemetry via xpack
+area: Health
+type: enhancement
+issues:
+ - 90877
diff --git a/docs/changelog/91710.yaml b/docs/changelog/91710.yaml
deleted file mode 100644
index c52359d86fd8..000000000000
--- a/docs/changelog/91710.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 91710
-summary: Support fields with commas in data frame analytics `analyzed_fields`
-area: Machine Learning
-type: bug
-issues:
- - 72541
diff --git a/docs/changelog/91713.yaml b/docs/changelog/91713.yaml
new file mode 100644
index 000000000000..c5fe0b73719b
--- /dev/null
+++ b/docs/changelog/91713.yaml
@@ -0,0 +1,6 @@
+pr: 91713
+summary: Fix NPE when method was called on an array type
+area: Infra/Scripting
+type: bug
+issues:
+ - 87562
diff --git a/docs/changelog/91722.yaml b/docs/changelog/91722.yaml
new file mode 100644
index 000000000000..ff403d33f7e4
--- /dev/null
+++ b/docs/changelog/91722.yaml
@@ -0,0 +1,5 @@
+pr: 91722
+summary: Consolidate google-oauth-client to latest version
+area: Snapshot/Restore
+type: upgrade
+issues: []
diff --git a/docs/changelog/91725.yaml b/docs/changelog/91725.yaml
deleted file mode 100644
index d36ad5fe921f..000000000000
--- a/docs/changelog/91725.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 91725
-summary: Align all usages of Jackson to be 2.14.0
-area: Infra/REST API
-type: upgrade
-issues: []
diff --git a/docs/changelog/91730.yaml b/docs/changelog/91730.yaml
new file mode 100644
index 000000000000..3db9189da486
--- /dev/null
+++ b/docs/changelog/91730.yaml
@@ -0,0 +1,6 @@
+pr: 91730
+summary: More accurate total ingest stats
+area: Ingest Node
+type: bug
+issues:
+ - 91358
diff --git a/docs/changelog/91769.yaml b/docs/changelog/91769.yaml
new file mode 100644
index 000000000000..02fb056ad2ea
--- /dev/null
+++ b/docs/changelog/91769.yaml
@@ -0,0 +1,6 @@
+pr: 91769
+summary: Fix synthetic `_source` for sparse `_doc_count` field
+area: TSDB
+type: bug
+issues:
+ - 91731
diff --git a/docs/changelog/91780.yaml b/docs/changelog/91780.yaml
new file mode 100644
index 000000000000..1cab6b9c2960
--- /dev/null
+++ b/docs/changelog/91780.yaml
@@ -0,0 +1,5 @@
+pr: 91780
+summary: Check stable plugin version at install and load time
+area: Infra/Plugins
+type: enhancement
+issues: []
diff --git a/docs/changelog/91781.yaml b/docs/changelog/91781.yaml
new file mode 100644
index 000000000000..7513ec86baf4
--- /dev/null
+++ b/docs/changelog/91781.yaml
@@ -0,0 +1,5 @@
+pr: 91781
+summary: JWT realm - Initial support for access tokens
+area: Authentication
+type: enhancement
+issues: []
diff --git a/docs/changelog/91783.yaml b/docs/changelog/91783.yaml
new file mode 100644
index 000000000000..02f3dd5b10c1
--- /dev/null
+++ b/docs/changelog/91783.yaml
@@ -0,0 +1,5 @@
+pr: 91783
+summary: "EQL Samples: add support for multiple samples per key"
+area: EQL
+type: enhancement
+issues: []
diff --git a/docs/changelog/91802.yaml b/docs/changelog/91802.yaml
new file mode 100644
index 000000000000..9432ac4bab64
--- /dev/null
+++ b/docs/changelog/91802.yaml
@@ -0,0 +1,5 @@
+pr: 91802
+summary: Fix potential leak in `RemoteRecoveryHandler`
+area: Recovery
+type: bug
+issues: []
diff --git a/docs/changelog/91839.yaml b/docs/changelog/91839.yaml
new file mode 100644
index 000000000000..205319b11023
--- /dev/null
+++ b/docs/changelog/91839.yaml
@@ -0,0 +1,5 @@
+pr: 91839
+summary: Speed h3 library by using `FastMath` implementation for trigonometric functions
+area: Geo
+type: enhancement
+issues: []
diff --git a/docs/changelog/91846.yaml b/docs/changelog/91846.yaml
new file mode 100644
index 000000000000..3c9db04c8337
--- /dev/null
+++ b/docs/changelog/91846.yaml
@@ -0,0 +1,5 @@
+pr: 91846
+summary: Upgrade to Netty 4.1.85
+area: Network
+type: upgrade
+issues: []
diff --git a/docs/changelog/91869.yaml b/docs/changelog/91869.yaml
new file mode 100644
index 000000000000..1e38cfd9dd6a
--- /dev/null
+++ b/docs/changelog/91869.yaml
@@ -0,0 +1,5 @@
+pr: 91869
+summary: Load stable plugins as synthetic modules
+area: Infra/Plugins
+type: enhancement
+issues: []
diff --git a/docs/changelog/91873.yaml b/docs/changelog/91873.yaml
new file mode 100644
index 000000000000..3f2fb776bb63
--- /dev/null
+++ b/docs/changelog/91873.yaml
@@ -0,0 +1,5 @@
+pr: 91873
+summary: Record timestamp on API key invalidation
+area: Security
+type: enhancement
+issues: []
diff --git a/docs/changelog/91886.yaml b/docs/changelog/91886.yaml
new file mode 100644
index 000000000000..b32ac9e5b5d0
--- /dev/null
+++ b/docs/changelog/91886.yaml
@@ -0,0 +1,5 @@
+pr: 91886
+summary: Pre-authorize child search transport actions
+area: Authorization
+type: enhancement
+issues: []
diff --git a/docs/changelog/91925.yaml b/docs/changelog/91925.yaml
new file mode 100644
index 000000000000..731ce402d98f
--- /dev/null
+++ b/docs/changelog/91925.yaml
@@ -0,0 +1,5 @@
+pr: 91925
+summary: Secure settings that can fall back to yml in Stateless
+area: Distributed
+type: feature
+issues: []
diff --git a/docs/changelog/91956.yaml b/docs/changelog/91956.yaml
new file mode 100644
index 000000000000..4df8fa057dcc
--- /dev/null
+++ b/docs/changelog/91956.yaml
@@ -0,0 +1,5 @@
+pr: 91956
+summary: Geohex aggregation on `geo_shape` field
+area: Geo
+type: feature
+issues: []
diff --git a/docs/changelog/92017.yaml b/docs/changelog/92017.yaml
new file mode 100644
index 000000000000..84016910b258
--- /dev/null
+++ b/docs/changelog/92017.yaml
@@ -0,0 +1,5 @@
+pr: 92017
+summary: Add commits listener for `InternalEngine` and `CombinedDeletionPolicy`
+area: Engine
+type: enhancement
+issues: []
diff --git a/docs/changelog/92025.yaml b/docs/changelog/92025.yaml
new file mode 100644
index 000000000000..a1ce0d2ee863
--- /dev/null
+++ b/docs/changelog/92025.yaml
@@ -0,0 +1,6 @@
+pr: 92025
+summary: Use primitive types rather than boxing/unboxing for iterating over primitive
+  arrays from defs
+area: Infra/Scripting
+type: enhancement
+issues: []
diff --git a/docs/changelog/92087.yaml b/docs/changelog/92087.yaml
new file mode 100644
index 000000000000..14faa9abd005
--- /dev/null
+++ b/docs/changelog/92087.yaml
@@ -0,0 +1,5 @@
+pr: 92087
+summary: Use all profiling events on startup
+area: Search
+type: bug
+issues: []
diff --git a/docs/changelog/92099.yaml b/docs/changelog/92099.yaml
new file mode 100644
index 000000000000..4b50e6caf7b2
--- /dev/null
+++ b/docs/changelog/92099.yaml
@@ -0,0 +1,6 @@
+pr: 92099
+summary: Add methods to prevent allocating long arrays during child navigation on
+  H3 api
+area: Geo
+type: enhancement
+issues: []
diff --git a/docs/changelog/92101.yaml b/docs/changelog/92101.yaml
new file mode 100644
index 000000000000..509166d7330f
--- /dev/null
+++ b/docs/changelog/92101.yaml
@@ -0,0 +1,5 @@
+pr: 92101
+summary: Add primary term supplier to Engine.IndexCommitListener
+area: Engine
+type: enhancement
+issues: []
diff --git a/docs/changelog/92102.yaml b/docs/changelog/92102.yaml
new file mode 100644
index 000000000000..99a1cbcc4b38
--- /dev/null
+++ b/docs/changelog/92102.yaml
@@ -0,0 +1,5 @@
+pr: 92102
+summary: Short-circuit painless def equality
+area: Infra/Scripting
+type: enhancement
+issues: []
diff --git a/docs/changelog/92104.yaml b/docs/changelog/92104.yaml
new file mode 100644
index 000000000000..19c66c14773d
--- /dev/null
+++ b/docs/changelog/92104.yaml
@@ -0,0 +1,5 @@
+pr: 92104
+summary: Upgrading tika to 2.6.0
+area: Ingest Node
+type: upgrade
+issues: []
diff --git a/docs/changelog/92115.yaml b/docs/changelog/92115.yaml
new file mode 100644
index 000000000000..d8f9defebb67
--- /dev/null
+++ b/docs/changelog/92115.yaml
@@ -0,0 +1,5 @@
+pr: 92115
+summary: Enable bloom filter for `_id` field in tsdb indices
+area: TSDB
+type: enhancement
+issues: []
diff --git a/docs/changelog/92118.yaml b/docs/changelog/92118.yaml
new file mode 100644
index 000000000000..a59cac45cb2c
--- /dev/null
+++ b/docs/changelog/92118.yaml
@@ -0,0 +1,14 @@
+pr: 92118
+summary: Allow more than one KNN search clause
+area: Vector Search
+type: enhancement
+issues:
+ - 91187
+highlight:
+  title: Allow more than one KNN search clause
+  body: "Some vector search scenarios require relevance ranking using a few kNN clauses,\n\
+  e.g. when ranking based on several fields, each with its own vector, or when a document \n\
+  includes a vector for the image and another vector for the text. The user may want to obtain\n\
+   relevance ranking based on a combination of all of these kNN clauses.\n\ncloses\
+    \ https://github.com/elastic/elasticsearch/issues/91187"
+  notable: true
diff --git a/docs/changelog/92123.yaml b/docs/changelog/92123.yaml
new file mode 100644
index 000000000000..d124315711f2
--- /dev/null
+++ b/docs/changelog/92123.yaml
@@ -0,0 +1,5 @@
+pr: 92123
+summary: Align all usages of protobuf to be 3.21.9
+area: Snapshot/Restore
+type: upgrade
+issues: []
diff --git a/docs/changelog/92171.yaml b/docs/changelog/92171.yaml
new file mode 100644
index 000000000000..fa98bc20d576
--- /dev/null
+++ b/docs/changelog/92171.yaml
@@ -0,0 +1,5 @@
+pr: 92171
+summary: Forwarding simulate calls to ingest nodes
+area: Ingest Node
+type: bug
+issues: []
diff --git a/docs/changelog/92197.yaml b/docs/changelog/92197.yaml
new file mode 100644
index 000000000000..a28740217277
--- /dev/null
+++ b/docs/changelog/92197.yaml
@@ -0,0 +1,5 @@
+pr: 92197
+summary: Optimize composite agg with leading global ordinal value source
+area: Aggregations
+type: enhancement
+issues: []
diff --git a/docs/changelog/92199.yaml b/docs/changelog/92199.yaml
new file mode 100644
index 000000000000..af71777f1ff9
--- /dev/null
+++ b/docs/changelog/92199.yaml
@@ -0,0 +1,5 @@
+pr: 92199
+summary: Expose tier balancing stats via internal endpoint
+area: "Allocation"
+type: enhancement
+issues: []
diff --git a/docs/changelog/92204.yaml b/docs/changelog/92204.yaml
new file mode 100644
index 000000000000..ca0b8ff7e738
--- /dev/null
+++ b/docs/changelog/92204.yaml
@@ -0,0 +1,5 @@
+pr: 92204
+summary: Free resources correctly when model loading is cancelled
+area: Machine Learning
+type: bug
+issues: []
diff --git a/docs/changelog/92213.yaml b/docs/changelog/92213.yaml
new file mode 100644
index 000000000000..6cbde4e15bf0
--- /dev/null
+++ b/docs/changelog/92213.yaml
@@ -0,0 +1,5 @@
+pr: 92213
+summary: Make native inference generally available
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/92219.yaml b/docs/changelog/92219.yaml
new file mode 100644
index 000000000000..2b889a9c3210
--- /dev/null
+++ b/docs/changelog/92219.yaml
@@ -0,0 +1,5 @@
+pr: 92219
+summary: Configurable retention period for invalidated or expired API keys
+area: Security
+type: enhancement
+issues: []
diff --git a/docs/changelog/92232.yaml b/docs/changelog/92232.yaml
new file mode 100644
index 000000000000..0ace8ab75f8f
--- /dev/null
+++ b/docs/changelog/92232.yaml
@@ -0,0 +1,6 @@
+pr: 92232
+summary: Better error when `aggregate_metric_double` used in scrolling datafeeds
+area: Machine Learning
+type: enhancement
+issues:
+ - 90592
diff --git a/docs/changelog/92238.yaml b/docs/changelog/92238.yaml
new file mode 100644
index 000000000000..c6d5128452bc
--- /dev/null
+++ b/docs/changelog/92238.yaml
@@ -0,0 +1,5 @@
+pr: 92238
+summary: Diff the list of filenames that are added by each new commit
+area: Engine
+type: enhancement
+issues: []
diff --git a/docs/changelog/92239.yaml b/docs/changelog/92239.yaml
new file mode 100644
index 000000000000..7cb621973752
--- /dev/null
+++ b/docs/changelog/92239.yaml
@@ -0,0 +1,5 @@
+pr: 92239
+summary: Stop the `frequent_items` aggregation reporting a subset when a superset exists
+area: Machine Learning
+type: bug
+issues: []
diff --git a/docs/changelog/92252.yaml b/docs/changelog/92252.yaml
new file mode 100644
index 000000000000..5c45a46a5d3a
--- /dev/null
+++ b/docs/changelog/92252.yaml
@@ -0,0 +1,5 @@
+pr: 92252
+summary: Preventing ILM and SLM runtime state from being stored in a snapshot
+area: ILM+SLM
+type: bug
+issues: []
diff --git a/docs/changelog/92255.yaml b/docs/changelog/92255.yaml
new file mode 100644
index 000000000000..abbb68ce4903
--- /dev/null
+++ b/docs/changelog/92255.yaml
@@ -0,0 +1,5 @@
+pr: 92255
+summary: Avoid capturing cluster state in TBbNA
+area: Stats
+type: bug
+issues: []
diff --git a/docs/changelog/92269.yaml b/docs/changelog/92269.yaml
new file mode 100644
index 000000000000..5e1381248bb8
--- /dev/null
+++ b/docs/changelog/92269.yaml
@@ -0,0 +1,5 @@
+pr: 92269
+summary: Access term dictionary more efficiently
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/92276.yaml b/docs/changelog/92276.yaml
new file mode 100644
index 000000000000..0727c512496a
--- /dev/null
+++ b/docs/changelog/92276.yaml
@@ -0,0 +1,5 @@
+pr: 92276
+summary: Minor TSDB parsing speedup
+area: TSDB
+type: enhancement
+issues: []
diff --git a/docs/changelog/92296.yaml b/docs/changelog/92296.yaml
new file mode 100644
index 000000000000..75ecd5885edb
--- /dev/null
+++ b/docs/changelog/92296.yaml
@@ -0,0 +1,6 @@
+pr: 92296
+summary: "[HealthAPI] Add support for the FEATURE_STATE affected resource"
+area: Health
+type: feature
+issues:
+ - 91353
diff --git a/docs/changelog/92303.yaml b/docs/changelog/92303.yaml
new file mode 100644
index 000000000000..beb3a0580a3b
--- /dev/null
+++ b/docs/changelog/92303.yaml
@@ -0,0 +1,5 @@
+pr: 92303
+summary: Add `forecasted_write_load` and `forecasted_shard_size_in_bytes` to the endpoint
+area: Allocation
+type: enhancement
+issues: []
diff --git a/docs/changelog/92309.yaml b/docs/changelog/92309.yaml
new file mode 100644
index 000000000000..e024ca73f56a
--- /dev/null
+++ b/docs/changelog/92309.yaml
@@ -0,0 +1,5 @@
+pr: 92309
+summary: Increase the number of threads of GET threadpool
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/92314.yaml b/docs/changelog/92314.yaml
new file mode 100644
index 000000000000..1de076290588
--- /dev/null
+++ b/docs/changelog/92314.yaml
@@ -0,0 +1,5 @@
+pr: 92314
+summary: JWT realm - add support for required claims
+area: Authentication
+type: enhancement
+issues: []
diff --git a/docs/changelog/92315.yaml b/docs/changelog/92315.yaml
new file mode 100644
index 000000000000..9180467f5db0
--- /dev/null
+++ b/docs/changelog/92315.yaml
@@ -0,0 +1,5 @@
+pr: 92315
+summary: JWT realm - Simplify token principal calculation
+area: Authentication
+type: enhancement
+issues: []
diff --git a/docs/changelog/92322.yaml b/docs/changelog/92322.yaml
new file mode 100644
index 000000000000..3b7d93fc6a4e
--- /dev/null
+++ b/docs/changelog/92322.yaml
@@ -0,0 +1,5 @@
+pr: 92322
+summary: implement extension pruning in frequent items to improve runtime
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/92328.yaml b/docs/changelog/92328.yaml
new file mode 100644
index 000000000000..e1cfd9ef955a
--- /dev/null
+++ b/docs/changelog/92328.yaml
@@ -0,0 +1,5 @@
+pr: 92328
+summary: Increase the default timeout for the start trained model deployment API
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/92329.yaml b/docs/changelog/92329.yaml
new file mode 100644
index 000000000000..5eab808b7ca9
--- /dev/null
+++ b/docs/changelog/92329.yaml
@@ -0,0 +1,5 @@
+pr: 92329
+summary: Fix tokenization bug when handling normalization in BERT and MPNet
+area: Machine Learning
+type: bug
+issues: []
diff --git a/docs/changelog/92334.yaml b/docs/changelog/92334.yaml
new file mode 100644
index 000000000000..299a7e172a14
--- /dev/null
+++ b/docs/changelog/92334.yaml
@@ -0,0 +1,5 @@
+pr: 92334
+summary: Example stable plugins with settings
+area: Infra/Plugins
+type: enhancement
+issues: []
diff --git a/docs/changelog/92335.yaml b/docs/changelog/92335.yaml
new file mode 100644
index 000000000000..9dc21fdcdc51
--- /dev/null
+++ b/docs/changelog/92335.yaml
@@ -0,0 +1,6 @@
+pr: 92335
+summary: Download the geoip databases only when needed
+area: Ingest Node
+type: bug
+issues:
+ - 90673
diff --git a/docs/changelog/92340.yaml b/docs/changelog/92340.yaml
new file mode 100644
index 000000000000..3ed47c66452b
--- /dev/null
+++ b/docs/changelog/92340.yaml
@@ -0,0 +1,5 @@
+pr: 92340
+summary: Add vector distance scoring to micro benchmarks
+area: Performance
+type: enhancement
+issues: []
diff --git a/docs/changelog/92360.yaml b/docs/changelog/92360.yaml
new file mode 100644
index 000000000000..7f389646f7e7
--- /dev/null
+++ b/docs/changelog/92360.yaml
@@ -0,0 +1,5 @@
+pr: 92360
+summary: Fix missing override for matches in `ProfileWeight`
+area: Search
+type: bug
+issues: []
diff --git a/docs/changelog/92372.yaml b/docs/changelog/92372.yaml
new file mode 100644
index 000000000000..7c882b22236e
--- /dev/null
+++ b/docs/changelog/92372.yaml
@@ -0,0 +1,16 @@
+pr: 92372
+summary: Speed up ingest geoip processors
+area: Ingest Node
+type: bug
+issues: []
+highlight:
+  title: Speed up ingest geoip processors
+  body: |-
+    The `geoip` ingest processor is significantly faster.
+
+    Previous versions of the geoip library needed special permission to execute
+    databinding code, requiring an expensive permissions check and
+    `AccessController.doPrivileged` call. The current version of the geoip
+    library no longer requires that, however, so the expensive code has been
+    removed, resulting in better performance for the ingest geoip processor.
+  notable: true
diff --git a/docs/changelog/92380.yaml b/docs/changelog/92380.yaml
new file mode 100644
index 000000000000..73a1af2a56b5
--- /dev/null
+++ b/docs/changelog/92380.yaml
@@ -0,0 +1,5 @@
+pr: 92380
+summary: Runtime fields to optionally ignore script errors
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/92382.yaml b/docs/changelog/92382.yaml
new file mode 100644
index 000000000000..6621307ae4fb
--- /dev/null
+++ b/docs/changelog/92382.yaml
@@ -0,0 +1,5 @@
+pr: 92382
+summary: Add log level for JVM logs
+area: Infra/Core
+type: enhancement
+issues: []
diff --git a/docs/changelog/92387.yaml b/docs/changelog/92387.yaml
new file mode 100644
index 000000000000..558a94169c93
--- /dev/null
+++ b/docs/changelog/92387.yaml
@@ -0,0 +1,6 @@
+pr: 92387
+summary: Add `jdk.internal.reflect` permission to es codebase
+area: Infra/Core
+type: bug
+issues:
+ - 92356
diff --git a/docs/changelog/92395.yaml b/docs/changelog/92395.yaml
new file mode 100644
index 000000000000..11d0ecdf758e
--- /dev/null
+++ b/docs/changelog/92395.yaml
@@ -0,0 +1,11 @@
+pr: 92395
+summary: Speed up ingest set and append processors
+area: Ingest Node
+type: bug
+issues: []
+highlight:
+  title: Speed up ingest set and append processors
+  body: |-
+    `set` and `append` ingest processors that use mustache templates are
+    significantly faster.
+  notable: true
diff --git a/docs/changelog/92399.yaml b/docs/changelog/92399.yaml
new file mode 100644
index 000000000000..027f3ac85969
--- /dev/null
+++ b/docs/changelog/92399.yaml
@@ -0,0 +1,7 @@
+pr: 92399
+summary: "[HealthAPI] Add size parameter that controls the number of affected resources\
+  \ returned"
+area: Health
+type: feature
+issues:
+ - 91930
diff --git a/docs/changelog/92414.yaml b/docs/changelog/92414.yaml
new file mode 100644
index 000000000000..c062122833c4
--- /dev/null
+++ b/docs/changelog/92414.yaml
@@ -0,0 +1,5 @@
+pr: 92414
+summary: Add the ability to include and exclude values in Frequent items
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/92417.yaml b/docs/changelog/92417.yaml
new file mode 100644
index 000000000000..b766fe0384f1
--- /dev/null
+++ b/docs/changelog/92417.yaml
@@ -0,0 +1,5 @@
+pr: 92417
+summary: Improve anomaly detection results indexing speed
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/92422.yaml b/docs/changelog/92422.yaml
new file mode 100644
index 000000000000..5d77e03e2eb8
--- /dev/null
+++ b/docs/changelog/92422.yaml
@@ -0,0 +1,5 @@
+pr: 92422
+summary: Make `FilterStreamInput` less trappy
+area: Infra/Core
+type: bug
+issues: []
diff --git a/docs/changelog/92428.yaml b/docs/changelog/92428.yaml
new file mode 100644
index 000000000000..dda2ff132feb
--- /dev/null
+++ b/docs/changelog/92428.yaml
@@ -0,0 +1,5 @@
+pr: 92428
+summary: Introduce parameterized rule and executor
+area: Query Languages
+type: enhancement
+issues: []
diff --git a/docs/changelog/92436.yaml b/docs/changelog/92436.yaml
new file mode 100644
index 000000000000..1f8b4a9bf187
--- /dev/null
+++ b/docs/changelog/92436.yaml
@@ -0,0 +1,6 @@
+pr: 92436
+summary: Add `ignore_missing_component_templates` config option
+area: Indices APIs
+type: enhancement
+issues:
+  - 92426
diff --git a/docs/changelog/92455.yaml b/docs/changelog/92455.yaml
new file mode 100644
index 000000000000..e7f77c315e6b
--- /dev/null
+++ b/docs/changelog/92455.yaml
@@ -0,0 +1,5 @@
+pr: 92455
+summary: Correctly handle an exception case for ingest failure
+area: Ingest Node
+type: bug
+issues: []
diff --git a/docs/changelog/92456.yaml b/docs/changelog/92456.yaml
new file mode 100644
index 000000000000..213f6e85870d
--- /dev/null
+++ b/docs/changelog/92456.yaml
@@ -0,0 +1,5 @@
+pr: 92456
+summary: Skip duplicate checks on segments that don't contain the document's timestamp
+area: TSDB
+type: enhancement
+issues: []
diff --git a/docs/changelog/92457.yaml b/docs/changelog/92457.yaml
new file mode 100644
index 000000000000..e8a2e7207cad
--- /dev/null
+++ b/docs/changelog/92457.yaml
@@ -0,0 +1,5 @@
+pr: 92457
+summary: Bump reactor netty version
+area: Snapshot/Restore
+type: upgrade
+issues: []
diff --git a/docs/changelog/92460.yaml b/docs/changelog/92460.yaml
new file mode 100644
index 000000000000..4fc47007ea8a
--- /dev/null
+++ b/docs/changelog/92460.yaml
@@ -0,0 +1,5 @@
+pr: 92460
+summary: Check `GeohexGrid` bounds on geopoint using spherical coordinates
+area: Geo
+type: bug
+issues: []
diff --git a/docs/changelog/92489.yaml b/docs/changelog/92489.yaml
new file mode 100644
index 000000000000..e2f3c4cd19e1
--- /dev/null
+++ b/docs/changelog/92489.yaml
@@ -0,0 +1,5 @@
+pr: 92489
+summary: Make `RecoveryPlannerService` optional
+area: Snapshot/Restore
+type: enhancement
+issues: []
diff --git a/docs/changelog/92490.yaml b/docs/changelog/92490.yaml
new file mode 100644
index 000000000000..5f6f63f3fc27
--- /dev/null
+++ b/docs/changelog/92490.yaml
@@ -0,0 +1,5 @@
+pr: 92490
+summary: Make clean up files step configurable for peer-recovery of replicas
+area: Recovery
+type: enhancement
+issues: []
diff --git a/docs/changelog/92528.yaml b/docs/changelog/92528.yaml
new file mode 100644
index 000000000000..03d12eea5b0e
--- /dev/null
+++ b/docs/changelog/92528.yaml
@@ -0,0 +1,5 @@
+pr: 92528
+summary: Scan stable plugins for named components upon install
+area: Infra/CLI
+type: enhancement
+issues: []
diff --git a/docs/changelog/92546.yaml b/docs/changelog/92546.yaml
new file mode 100644
index 000000000000..1549d512453e
--- /dev/null
+++ b/docs/changelog/92546.yaml
@@ -0,0 +1,5 @@
+pr: 92546
+summary: Delay Connection#onRemoved while pending
+area: Network
+type: bug
+issues: []
diff --git a/docs/changelog/92547.yaml b/docs/changelog/92547.yaml
new file mode 100644
index 000000000000..2e39c93724f4
--- /dev/null
+++ b/docs/changelog/92547.yaml
@@ -0,0 +1,5 @@
+pr: 92547
+summary: Don't create a new `DoubleHistogram` instance for empty buckets
+area: Aggregations
+type: bug
+issues: []
diff --git a/docs/changelog/92558.yaml b/docs/changelog/92558.yaml
new file mode 100644
index 000000000000..f0621449fa20
--- /dev/null
+++ b/docs/changelog/92558.yaml
@@ -0,0 +1,6 @@
+pr: 92558
+summary: Protect `NodeConnectionsService` from stale conns
+area: Network
+type: bug
+issues:
+ - 92029
diff --git a/docs/changelog/92586.yaml b/docs/changelog/92586.yaml
new file mode 100644
index 000000000000..f6d0c58ade30
--- /dev/null
+++ b/docs/changelog/92586.yaml
@@ -0,0 +1,6 @@
+pr: 92586
+summary: "Grok returns a list of matches for repeated pattern names #92092"
+area: Ingest Node
+type: bug
+issues:
+ - 92092
diff --git a/docs/changelog/92587.yaml b/docs/changelog/92587.yaml
new file mode 100644
index 000000000000..bc6b6b5b6e8c
--- /dev/null
+++ b/docs/changelog/92587.yaml
@@ -0,0 +1,5 @@
+pr: 92587
+summary: Upgrade to Netty 4.1.86
+area: Network
+type: upgrade
+issues: []
diff --git a/docs/changelog/92625.yaml b/docs/changelog/92625.yaml
new file mode 100644
index 000000000000..232da48fc06d
--- /dev/null
+++ b/docs/changelog/92625.yaml
@@ -0,0 +1,6 @@
+pr: 92625
+summary: Fix Security's expression resolver to not remove unavailable but authorized
+  names
+area: Authorization
+type: bug
+issues: []
diff --git a/docs/changelog/92640.yaml b/docs/changelog/92640.yaml
new file mode 100644
index 000000000000..8c2dc2415907
--- /dev/null
+++ b/docs/changelog/92640.yaml
@@ -0,0 +1,5 @@
+pr: 92640
+summary: Added new field `rollout_duration_seconds` to fleet-actions
+area: Infra/Core
+type: enhancement
+issues: []
diff --git a/docs/changelog/92659.yaml b/docs/changelog/92659.yaml
new file mode 100644
index 000000000000..4e90acbb1395
--- /dev/null
+++ b/docs/changelog/92659.yaml
@@ -0,0 +1,5 @@
+pr: 92659
+summary: Set a fixed compound file threshold of 1GB
+area: Engine
+type: enhancement
+issues: []
diff --git a/docs/changelog/92668.yaml b/docs/changelog/92668.yaml
new file mode 100644
index 000000000000..ffa45030ec67
--- /dev/null
+++ b/docs/changelog/92668.yaml
@@ -0,0 +1,5 @@
+pr: 92668
+summary: "Introduce ShardRouting.Role"
+area: Allocation
+type: enhancement
+issues: []
diff --git a/docs/changelog/92680.yaml b/docs/changelog/92680.yaml
new file mode 100644
index 000000000000..ebf8e2ff84b2
--- /dev/null
+++ b/docs/changelog/92680.yaml
@@ -0,0 +1,5 @@
+pr: 92680
+summary: Fix `InputStream#readAllBytes` on `InputStreamIndexInput`
+area: Infra/Core
+type: bug
+issues: []
diff --git a/docs/changelog/92692.yaml b/docs/changelog/92692.yaml
new file mode 100644
index 000000000000..eb6830498779
--- /dev/null
+++ b/docs/changelog/92692.yaml
@@ -0,0 +1,6 @@
+pr: 92692
+summary: Allow different filters per `DataStream` in a `DataStreamAlias`
+area: Data streams
+type: bug
+issues:
+ - 92050
diff --git a/docs/changelog/92693.yaml b/docs/changelog/92693.yaml
new file mode 100644
index 000000000000..c6d6fd90c64f
--- /dev/null
+++ b/docs/changelog/92693.yaml
@@ -0,0 +1,6 @@
+pr: 92693
+summary: Delay master task failure notifications until commit
+area: Cluster Coordination
+type: bug
+issues:
+ - 92677
diff --git a/docs/changelog/92695.yaml b/docs/changelog/92695.yaml
new file mode 100644
index 000000000000..d9b404744448
--- /dev/null
+++ b/docs/changelog/92695.yaml
@@ -0,0 +1,5 @@
+pr: 92695
+summary: "[Rest Api Compatibility] Format response media type with parameters"
+area: Infra/REST API
+type: bug
+issues: []
diff --git a/docs/changelog/92711.yaml b/docs/changelog/92711.yaml
new file mode 100644
index 000000000000..9659c89f4194
--- /dev/null
+++ b/docs/changelog/92711.yaml
@@ -0,0 +1,5 @@
+pr: 92711
+summary: Add methods to H3#hexRing to prevent allocating long arrays
+area: Geo
+type: enhancement
+issues: []
diff --git a/docs/changelog/92742.yaml b/docs/changelog/92742.yaml
new file mode 100644
index 000000000000..ac63c5e226c9
--- /dev/null
+++ b/docs/changelog/92742.yaml
@@ -0,0 +1,5 @@
+pr: 92742
+summary: "Improve node-{join,left} logging for troubleshooting"
+area: Cluster Coordination
+type: enhancement
+issues: []
diff --git a/docs/changelog/92744.yaml b/docs/changelog/92744.yaml
new file mode 100644
index 000000000000..7c121d1d22f2
--- /dev/null
+++ b/docs/changelog/92744.yaml
@@ -0,0 +1,5 @@
+pr: 92744
+summary: Repeat `cluster.initial_master_nodes` log warning
+area: Cluster Coordination
+type: enhancement
+issues: []
diff --git a/docs/changelog/92755.yaml b/docs/changelog/92755.yaml
new file mode 100644
index 000000000000..8f52817d2a0e
--- /dev/null
+++ b/docs/changelog/92755.yaml
@@ -0,0 +1,6 @@
+pr: 92755
+summary: Add links to troubleshooting docs
+area: Cluster Coordination
+type: enhancement
+issues:
+ - 92741
diff --git a/docs/changelog/92762.yaml b/docs/changelog/92762.yaml
new file mode 100644
index 000000000000..a5953ce3117f
--- /dev/null
+++ b/docs/changelog/92762.yaml
@@ -0,0 +1,6 @@
+pr: 92762
+summary: Integrate "sourceHasChanged" call into failure handling and retry logic
+area: Transform
+type: bug
+issues:
+ - 92133
diff --git a/docs/changelog/92777.yaml b/docs/changelog/92777.yaml
new file mode 100644
index 000000000000..7159a258e897
--- /dev/null
+++ b/docs/changelog/92777.yaml
@@ -0,0 +1,5 @@
+pr: 92777
+summary: Reduce memory usage of match all bitset
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/92784.yaml b/docs/changelog/92784.yaml
new file mode 100644
index 000000000000..06c8e767da60
--- /dev/null
+++ b/docs/changelog/92784.yaml
@@ -0,0 +1,6 @@
+pr: 92784
+summary: Update the version of asm used by plugin scanner
+area: Infra/Plugins
+type: bug
+issues:
+ - 92782
diff --git a/docs/changelog/92787.yaml b/docs/changelog/92787.yaml
new file mode 100644
index 000000000000..6e7339eac228
--- /dev/null
+++ b/docs/changelog/92787.yaml
@@ -0,0 +1,5 @@
+pr: 92787
+summary: Enable profiling plugin by default
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/92829.yaml b/docs/changelog/92829.yaml
new file mode 100644
index 000000000000..69ad54928f89
--- /dev/null
+++ b/docs/changelog/92829.yaml
@@ -0,0 +1,5 @@
+pr: 92829
+summary: Protect H3 library against integer overflow
+area: Geo
+type: enhancement
+issues: []
diff --git a/docs/changelog/92863.yaml b/docs/changelog/92863.yaml
new file mode 100644
index 000000000000..2d4a7f4e7a5e
--- /dev/null
+++ b/docs/changelog/92863.yaml
@@ -0,0 +1,5 @@
+pr: 92863
+summary: Support retrieving inlined stack frames
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/92871.yaml b/docs/changelog/92871.yaml
new file mode 100644
index 000000000000..f401f50e7461
--- /dev/null
+++ b/docs/changelog/92871.yaml
@@ -0,0 +1,5 @@
+pr: 92871
+summary: Support custom PBKDF2 password hashes
+area: Authentication
+type: enhancement
+issues: []
diff --git a/docs/changelog/92879.yaml b/docs/changelog/92879.yaml
new file mode 100644
index 000000000000..0e9d0d8ab6cc
--- /dev/null
+++ b/docs/changelog/92879.yaml
@@ -0,0 +1,13 @@
+pr: 92879
+summary: The Health API is now generally available
+area: Health
+type: feature
+issues: []
+highlight:
+  title: The Health API is now generally available
+  body: |-
+    Elasticsearch introduces a new Health API designed to report the health of
+    the cluster. The new API provides both a high level overview of the cluster
+    health, and a very detailed report that can include a precise diagnosis and
+    a resolution.
+  notable: true
diff --git a/docs/changelog/92880.yaml b/docs/changelog/92880.yaml
new file mode 100644
index 000000000000..5336987ee2cd
--- /dev/null
+++ b/docs/changelog/92880.yaml
@@ -0,0 +1,5 @@
+pr: 92880
+summary: Cache the creation of parsers within DateProcessor
+area: Ingest Node
+type: enhancement
+issues: []
diff --git a/docs/changelog/92890.yaml b/docs/changelog/92890.yaml
new file mode 100644
index 000000000000..3bd38e6152e5
--- /dev/null
+++ b/docs/changelog/92890.yaml
@@ -0,0 +1,6 @@
+pr: 92890
+summary: Fix unclosed directory stream in `ClassReaders`
+area: Infra/Plugins
+type: bug
+issues:
+ - 92866
diff --git a/docs/changelog/92920.yaml b/docs/changelog/92920.yaml
new file mode 100644
index 000000000000..7c14ecd34ac2
--- /dev/null
+++ b/docs/changelog/92920.yaml
@@ -0,0 +1,6 @@
+pr: 92920
+summary: Annotated highlighter does not match when search contains both annotation and annotated term
+area: Search
+type: bug
+issues:
+  - 91944
diff --git a/docs/changelog/92928.yaml b/docs/changelog/92928.yaml
new file mode 100644
index 000000000000..0f86adf45d1b
--- /dev/null
+++ b/docs/changelog/92928.yaml
@@ -0,0 +1,5 @@
+pr: 92928
+summary: Ensure one-shot wrappers release their delegates
+area: Infra/Core
+type: bug
+issues: []
diff --git a/docs/changelog/92948.yaml b/docs/changelog/92948.yaml
new file mode 100644
index 000000000000..a335e60035ce
--- /dev/null
+++ b/docs/changelog/92948.yaml
@@ -0,0 +1,6 @@
+pr: 92948
+summary: Transform _schedule_now API
+area: Transform
+type: feature
+issues:
+ - 44722
diff --git a/docs/changelog/92950.yaml b/docs/changelog/92950.yaml
new file mode 100644
index 000000000000..464d56bbda2e
--- /dev/null
+++ b/docs/changelog/92950.yaml
@@ -0,0 +1,5 @@
+pr: 92950
+summary: Add monitoring mappings for es ingest metricset
+area: Monitoring
+type: enhancement
+issues: []
diff --git a/docs/changelog/92957.yaml b/docs/changelog/92957.yaml
new file mode 100644
index 000000000000..0afc61ba9ec6
--- /dev/null
+++ b/docs/changelog/92957.yaml
@@ -0,0 +1,5 @@
+pr: 92957
+summary: Upgrade to lucene-9.5.0-snapshot-d19c3e2e0ed
+area: Search
+type: upgrade
+issues: []
diff --git a/docs/changelog/92973.yaml b/docs/changelog/92973.yaml
new file mode 100644
index 000000000000..c20df5ad55e6
--- /dev/null
+++ b/docs/changelog/92973.yaml
@@ -0,0 +1,5 @@
+pr: 92973
+summary: Fix indices resolver for datemath with colon
+area: Infra/Core
+type: bug
+issues: []
diff --git a/docs/changelog/92976.yaml b/docs/changelog/92976.yaml
new file mode 100644
index 000000000000..918606b124b8
--- /dev/null
+++ b/docs/changelog/92976.yaml
@@ -0,0 +1,6 @@
+pr: 92976
+summary: Report recovered files as recovered from snapshot for fully mounted searchable
+  snapshots
+area: Recovery
+type: bug
+issues: []
diff --git a/docs/changelog/93000.yaml b/docs/changelog/93000.yaml
new file mode 100644
index 000000000000..33c933e393b6
--- /dev/null
+++ b/docs/changelog/93000.yaml
@@ -0,0 +1,5 @@
+pr: 93000
+summary: Persist data counts and datafeed timing stats asynchronously
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/93084.yaml b/docs/changelog/93084.yaml
new file mode 100644
index 000000000000..e6baa47190ff
--- /dev/null
+++ b/docs/changelog/93084.yaml
@@ -0,0 +1,6 @@
+pr: 93084
+summary: "[H3] Compute destination point from distance and azimuth using planar 3d\
+  \ math"
+area: Geo
+type: enhancement
+issues: []
diff --git a/docs/changelog/93157.yaml b/docs/changelog/93157.yaml
new file mode 100644
index 000000000000..02d816b7fe82
--- /dev/null
+++ b/docs/changelog/93157.yaml
@@ -0,0 +1,6 @@
+pr: 93157
+summary: Take into account `max_headroom` in disk watermark calculations
+area: Health
+type: bug
+issues:
+ - 93155
diff --git a/docs/changelog/93165.yaml b/docs/changelog/93165.yaml
new file mode 100644
index 000000000000..6b57f80c153e
--- /dev/null
+++ b/docs/changelog/93165.yaml
@@ -0,0 +1,5 @@
+pr: 93165
+summary: Switch to Lucene's new `IntField/LongField/FloatField/DoubleField`
+area: Mapping
+type: enhancement
+issues: []
diff --git a/docs/changelog/93177.yaml b/docs/changelog/93177.yaml
new file mode 100644
index 000000000000..6a539f5372e4
--- /dev/null
+++ b/docs/changelog/93177.yaml
@@ -0,0 +1,5 @@
+pr: 93177
+summary: Fix geo ip database file leak when processing IP arrays
+area: Ingest Node
+type: bug
+issues: []
diff --git a/docs/changelog/93178.yaml b/docs/changelog/93178.yaml
new file mode 100644
index 000000000000..1622c9d98669
--- /dev/null
+++ b/docs/changelog/93178.yaml
@@ -0,0 +1,6 @@
+pr: 93178
+summary: Restore printing bootstrap checks as errors
+area: Infra/CLI
+type: bug
+issues:
+ - 93074
diff --git a/docs/changelog/93179.yaml b/docs/changelog/93179.yaml
new file mode 100644
index 000000000000..bb18523d630d
--- /dev/null
+++ b/docs/changelog/93179.yaml
@@ -0,0 +1,6 @@
+pr: 93179
+summary: Making `JsonProcessor` stricter so that it does not silently drop data
+area: Ingest Node
+type: bug
+issues:
+ - 92898
diff --git a/docs/changelog/93188.yaml b/docs/changelog/93188.yaml
new file mode 100644
index 000000000000..bd4f47f74853
--- /dev/null
+++ b/docs/changelog/93188.yaml
@@ -0,0 +1,5 @@
+pr: 93188
+summary: Adjust range of allowed percentages of deletes in an index
+area: Engine
+type: enhancement
+issues: []
diff --git a/docs/changelog/93203.yaml b/docs/changelog/93203.yaml
new file mode 100644
index 000000000000..771c07d40190
--- /dev/null
+++ b/docs/changelog/93203.yaml
@@ -0,0 +1,5 @@
+pr: 93203
+summary: Support "offset" parameter in `DateHistogramGroupSource`
+area: Transform
+type: enhancement
+issues: []
diff --git a/docs/changelog/93210.yaml b/docs/changelog/93210.yaml
new file mode 100644
index 000000000000..179f4ab9dec8
--- /dev/null
+++ b/docs/changelog/93210.yaml
@@ -0,0 +1,5 @@
+pr: 93210
+summary: Unpromotables skip replication and peer recovery
+area: Allocation
+type: enhancement
+issues: []
diff --git a/docs/changelog/93221.yaml b/docs/changelog/93221.yaml
new file mode 100644
index 000000000000..7ee858d7718c
--- /dev/null
+++ b/docs/changelog/93221.yaml
@@ -0,0 +1,5 @@
+pr: 93221
+summary: Trigger state persistence based on time
+area: Transform
+type: enhancement
+issues: []
diff --git a/docs/changelog/93238.yaml b/docs/changelog/93238.yaml
new file mode 100644
index 000000000000..711b81d529b7
--- /dev/null
+++ b/docs/changelog/93238.yaml
@@ -0,0 +1,5 @@
+pr: 93238
+summary: "Upgrade antlr to 4.11.1 for ql, eql and sql"
+area: Query Languages
+type: upgrade
+issues: []
diff --git a/docs/changelog/93247.yaml b/docs/changelog/93247.yaml
new file mode 100644
index 000000000000..e32d2acde270
--- /dev/null
+++ b/docs/changelog/93247.yaml
@@ -0,0 +1,5 @@
+pr: 93247
+summary: Add `term` query support to `rank_features` mapped field
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/93255.yaml b/docs/changelog/93255.yaml
new file mode 100644
index 000000000000..1492a5b7155d
--- /dev/null
+++ b/docs/changelog/93255.yaml
@@ -0,0 +1,5 @@
+pr: 93255
+summary: Improve frequent items runtime
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/93276.yaml b/docs/changelog/93276.yaml
new file mode 100644
index 000000000000..be00ea97912d
--- /dev/null
+++ b/docs/changelog/93276.yaml
@@ -0,0 +1,7 @@
+pr: 93276
+summary: -|
+  Correctly remove domain from realm when rewriting `Authentication` for compatibility with node versions that don't
+  support domains
+area: Authentication
+type: bug
+issues: [ ]
diff --git a/docs/changelog/93285.yaml b/docs/changelog/93285.yaml
new file mode 100644
index 000000000000..2a539a132a45
--- /dev/null
+++ b/docs/changelog/93285.yaml
@@ -0,0 +1,5 @@
+pr: 93285
+summary: Make `GeoIpProcessor` backing database instance pluggable
+area: Ingest Node
+type: enhancement
+issues: []
diff --git a/docs/changelog/93299.yaml b/docs/changelog/93299.yaml
new file mode 100644
index 000000000000..13c6e64b8f1d
--- /dev/null
+++ b/docs/changelog/93299.yaml
@@ -0,0 +1,6 @@
+pr: 93299
+summary: No length check for source-only keyword fields
+area: Mapping
+type: bug
+issues:
+ - 9304
diff --git a/docs/changelog/93304.yaml b/docs/changelog/93304.yaml
new file mode 100644
index 000000000000..e9d05b8c1b7e
--- /dev/null
+++ b/docs/changelog/93304.yaml
@@ -0,0 +1,5 @@
+pr: 93304
+summary: Improve `frequent_items` performance using global ordinals
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/93306.yaml b/docs/changelog/93306.yaml
new file mode 100644
index 000000000000..6857c777fb79
--- /dev/null
+++ b/docs/changelog/93306.yaml
@@ -0,0 +1,5 @@
+pr: 93306
+summary: Allowed indices matcher supports nested limited roles
+area: Authorization
+type: enhancement
+issues: []
diff --git a/docs/changelog/93324.yaml b/docs/changelog/93324.yaml
new file mode 100644
index 000000000000..12d212bddf48
--- /dev/null
+++ b/docs/changelog/93324.yaml
@@ -0,0 +1,6 @@
+pr: 93324
+summary: Fix data counts race condition when starting a datafeed
+area: Machine Learning
+type: bug
+issues:
+ - 93298
diff --git a/docs/changelog/93329.yaml b/docs/changelog/93329.yaml
new file mode 100644
index 000000000000..4585edad9b6c
--- /dev/null
+++ b/docs/changelog/93329.yaml
@@ -0,0 +1,20 @@
+pr: 93329
+summary: Handle a default/request pipeline and a final pipeline with minimal additional
+  overhead
+area: Ingest Node
+type: bug
+issues:
+ - 92843
+ - 81244
+ - 93118
+highlight:
+  title: Speed up ingest processing with multiple pipelines
+  body: |-
+    Processing documents with both a request/default and a final
+    pipeline is significantly faster.
+
+    Rather than marshalling a document from and to json once per
+    pipeline, a document is now marshalled from json before any
+    pipelines execute and then back to json after all pipelines have
+    executed.
+  notable: true
diff --git a/docs/changelog/93331.yaml b/docs/changelog/93331.yaml
new file mode 100644
index 000000000000..5dd1cddc7207
--- /dev/null
+++ b/docs/changelog/93331.yaml
@@ -0,0 +1,5 @@
+pr: 93331
+summary: Add new `query_vector_builder` option to knn search clause
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/93333.yaml b/docs/changelog/93333.yaml
new file mode 100644
index 000000000000..60048590d2d4
--- /dev/null
+++ b/docs/changelog/93333.yaml
@@ -0,0 +1,5 @@
+pr: 93333
+summary: "Script: Metadata `validateMetadata` optimization"
+area: Infra/Scripting
+type: enhancement
+issues: []
diff --git a/docs/changelog/93340.yaml b/docs/changelog/93340.yaml
new file mode 100644
index 000000000000..7d8c54d6155f
--- /dev/null
+++ b/docs/changelog/93340.yaml
@@ -0,0 +1,6 @@
+pr: 93340
+summary: Nested path info shouldn't be added during `copy_to`
+area: Search
+type: bug
+issues:
+ - 93117
diff --git a/docs/changelog/93370.yaml b/docs/changelog/93370.yaml
new file mode 100644
index 000000000000..6cdc718ea4ad
--- /dev/null
+++ b/docs/changelog/93370.yaml
@@ -0,0 +1,22 @@
+pr: 93370
+summary: Support geo_grid ingest processor
+area: Geo
+type: feature
+issues:
+ - 92473
+highlight:
+  title: Support geo_grid ingest processor
+  body: |-
+    The `geo_grid` ingest processor supports creating indexable geometries from geohash, geotile and H3 cells.
+
+    There already exists a `circle` ingest processor that creates a polygon from a point and radius definition.
+    This concept is useful when there is need to use spatial operations that work with indexable geometries on
+    geometric objects that are not defined spatially (or at least not indexable by lucene).
+    In this case, the string `4/8/5` does not have spatial meaning, until we interpret it as the address
+    of a rectangular `geotile`, and save the bounding box defining its border for further use.
+    Likewise we can interpret `geohash` strings like `u0` as a tile, and H3 strings like `811fbffffffffff`
+    as an hexagonal cell, saving the cell border as a polygon.
+
+    [role="screenshot"]
+    image::../images/spatial/geogrid_h3_children.png[Kibana map with three H3 layers: cell, children and intersecting non-children]
+  notable: true
diff --git a/docs/changelog/93375.yaml b/docs/changelog/93375.yaml
new file mode 100644
index 000000000000..63114927df96
--- /dev/null
+++ b/docs/changelog/93375.yaml
@@ -0,0 +1,5 @@
+pr: 93375
+summary: '`TransportGetTaskAction:` Wait for the task asynchronously'
+area: Task Management
+type: enhancement
+issues: []
diff --git a/docs/changelog/93385.yaml b/docs/changelog/93385.yaml
new file mode 100644
index 000000000000..036bff2571e1
--- /dev/null
+++ b/docs/changelog/93385.yaml
@@ -0,0 +1,5 @@
+pr: 93385
+summary: Upgrade to Lucene 9.5.0
+area: Search
+type: upgrade
+issues: []
diff --git a/docs/changelog/93388.yaml b/docs/changelog/93388.yaml
new file mode 100644
index 000000000000..43ddc40bde73
--- /dev/null
+++ b/docs/changelog/93388.yaml
@@ -0,0 +1,5 @@
+pr: 93388
+summary: Allow `null` to be provided for `dense_vector` field values
+area: Vector Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/93392.yaml b/docs/changelog/93392.yaml
new file mode 100644
index 000000000000..7ed7ffacc087
--- /dev/null
+++ b/docs/changelog/93392.yaml
@@ -0,0 +1,5 @@
+pr: 93392
+summary: Provide locally mounted secure settings implementation
+area: Infra/Core
+type: enhancement
+issues: []
diff --git a/docs/changelog/93417.yaml b/docs/changelog/93417.yaml
new file mode 100644
index 000000000000..422302464625
--- /dev/null
+++ b/docs/changelog/93417.yaml
@@ -0,0 +1,5 @@
+pr: 93417
+summary: Do not refresh all indices in `TransportBulkAction`
+area: CRUD
+type: bug
+issues: []
diff --git a/docs/changelog/93421.yaml b/docs/changelog/93421.yaml
new file mode 100644
index 000000000000..20cb13522f52
--- /dev/null
+++ b/docs/changelog/93421.yaml
@@ -0,0 +1,10 @@
+pr: 93421
+summary: Make `frequent_item_sets` aggregation GA
+area: Machine Learning
+type: feature
+issues: []
+highlight:
+  title: Make `frequent_item_sets` aggregation GA
+  body: The `frequent_item_sets` aggregation has been moved from
+        technical preview to general availability.
+  notable: true
diff --git a/docs/changelog/93426.yaml b/docs/changelog/93426.yaml
new file mode 100644
index 000000000000..c6bf12ccf9d3
--- /dev/null
+++ b/docs/changelog/93426.yaml
@@ -0,0 +1,5 @@
+pr: 93426
+summary: Add ability for Watcher's webhook actions to send additional header
+area: Watcher
+type: enhancement
+issues: []
diff --git a/docs/changelog/93431.yaml b/docs/changelog/93431.yaml
new file mode 100644
index 000000000000..082a84768445
--- /dev/null
+++ b/docs/changelog/93431.yaml
@@ -0,0 +1,6 @@
+pr: 93431
+summary: Fix context leak in list tasks API
+area: Task Management
+type: bug
+issues:
+ - 93428
diff --git a/docs/changelog/93438.yaml b/docs/changelog/93438.yaml
new file mode 100644
index 000000000000..24464b4245d9
--- /dev/null
+++ b/docs/changelog/93438.yaml
@@ -0,0 +1,5 @@
+pr: 93438
+summary: Align all usages of Jackson to be 2.14.2
+area: Infra/Core
+type: upgrade
+issues: []
diff --git a/docs/changelog/93445.yaml b/docs/changelog/93445.yaml
new file mode 100644
index 000000000000..e2f1d39e9bb1
--- /dev/null
+++ b/docs/changelog/93445.yaml
@@ -0,0 +1,6 @@
+pr: 93445
+summary: Support downsampling of histogram as labels
+area: Rollup
+type: bug
+issues:
+ - 93263
diff --git a/docs/changelog/93448.yaml b/docs/changelog/93448.yaml
new file mode 100644
index 000000000000..54ccc0935836
--- /dev/null
+++ b/docs/changelog/93448.yaml
@@ -0,0 +1,5 @@
+pr: 93448
+summary: Speed up retrieval of data for flamegraphs
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/93461.yaml b/docs/changelog/93461.yaml
new file mode 100644
index 000000000000..e1efa406c72f
--- /dev/null
+++ b/docs/changelog/93461.yaml
@@ -0,0 +1,5 @@
+pr: 93461
+summary: Fallback to the actual shard size when forecast is not available
+area: Allocation
+type: bug
+issues: []
diff --git a/docs/changelog/93551.yaml b/docs/changelog/93551.yaml
new file mode 100644
index 000000000000..9c0a81b039e0
--- /dev/null
+++ b/docs/changelog/93551.yaml
@@ -0,0 +1,6 @@
+pr: 93551
+summary: Disable recovery monitor before recovery start
+area: Recovery
+type: bug
+issues:
+ - 93542
diff --git a/docs/java-rest/low-level/configuration.asciidoc b/docs/java-rest/low-level/configuration.asciidoc
index 18f96858c761..e1102305db1d 100644
--- a/docs/java-rest/low-level/configuration.asciidoc
+++ b/docs/java-rest/low-level/configuration.asciidoc
@@ -14,7 +14,7 @@ additional configuration for the low-level Java REST Client.
 Configuring requests timeouts can be done by providing an instance of
 `RequestConfigCallback` while building the `RestClient` through its builder.
 The interface has one method that receives an instance of
-https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/client/config/RequestConfig.Builder.html[`org.apache.http.client.config.RequestConfig.Builder`]
+https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/client/config/RequestConfig.Builder.html[`org.apache.http.client.config.RequestConfig.Builder`]
  as an argument and has the same return type. The request config builder can
 be modified and then returned. In the following example we increase the
 connect timeout (defaults to 1 second) and the socket timeout (defaults to 30
@@ -50,7 +50,7 @@ include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-config-thre
 Configuring basic authentication can be done by providing an
 `HttpClientConfigCallback` while building the `RestClient` through its builder.
 The interface has one method that receives an instance of
-https://hc.apache.org/httpcomponents-asyncclient-dev/httpasyncclient/apidocs/org/apache/http/impl/nio/client/HttpAsyncClientBuilder.html[`org.apache.http.impl.nio.client.HttpAsyncClientBuilder`]
+https://hc.apache.org/httpcomponents-asyncclient-4.1.x/current/httpasyncclient/apidocs/org/apache/http/impl/nio/client/HttpAsyncClientBuilder.html[`org.apache.http.impl.nio.client.HttpAsyncClientBuilder`]
  as an argument and has the same return type. The http client builder can be
 modified and then returned. In the following example we set a default
 credentials provider that requires basic authentication.
@@ -99,7 +99,7 @@ include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-auth-api-ke
 
 Encrypted communication using TLS can also be configured through the
 `HttpClientConfigCallback`. The
-https://hc.apache.org/httpcomponents-asyncclient-dev/httpasyncclient/apidocs/org/apache/http/impl/nio/client/HttpAsyncClientBuilder.html[`org.apache.http.impl.nio.client.HttpAsyncClientBuilder`]
+https://hc.apache.org/httpcomponents-asyncclient-4.1.x/current/httpasyncclient/apidocs/org/apache/http/impl/nio/client/HttpAsyncClientBuilder.html[`org.apache.http.impl.nio.client.HttpAsyncClientBuilder`]
  received as an argument exposes multiple methods to configure encrypted
  communication: `setSSLContext`, `setSSLSessionStrategy` and
  `setConnectionManager`, in order of precedence from the least important.
diff --git a/docs/java-rest/low-level/usage.asciidoc b/docs/java-rest/low-level/usage.asciidoc
index 68b91e06d6fa..439606281e04 100644
--- a/docs/java-rest/low-level/usage.asciidoc
+++ b/docs/java-rest/low-level/usage.asciidoc
@@ -57,7 +57,7 @@ dependencies {
 === Dependencies
 
 The low-level Java REST client internally uses the
-https://hc.apache.org/httpcomponents-asyncclient-dev/[Apache Http Async Client]
+https://hc.apache.org/httpcomponents-asyncclient-4.1.x/[Apache Http Async Client]
  to send http requests. It depends on the following artifacts, namely the async
  http client and its own transitive dependencies:
 
@@ -152,7 +152,7 @@ A `RestClient` instance can be built through the corresponding
 `RestClientBuilder` class, created via `RestClient#builder(HttpHost...)`
 static method. The only required argument is one or more hosts that the
 client will communicate with, provided as instances of
-https://hc.apache.org/httpcomponents-core-ga/httpcore/apidocs/org/apache/http/HttpHost.html[HttpHost]
+https://hc.apache.org/httpcomponents-core-4.4.x/current/httpcore/apidocs/org/apache/http/HttpHost.html[HttpHost]
  as follows:
 
 ["source","java",subs="attributes,callouts,macros"]
@@ -203,7 +203,7 @@ include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-init-reques
 --------------------------------------------------
 <1> Set a callback that allows to modify the default request configuration
 (e.g. request timeouts, authentication, or anything that the
-https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/client/config/RequestConfig.Builder.html[`org.apache.http.client.config.RequestConfig.Builder`]
+https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/client/config/RequestConfig.Builder.html[`org.apache.http.client.config.RequestConfig.Builder`]
  allows to set)
 
 ["source","java",subs="attributes,callouts,macros"]
@@ -212,7 +212,7 @@ include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-init-client
 --------------------------------------------------
 <1> Set a callback that allows to modify the http client configuration
 (e.g. encrypted communication over ssl, or anything that the
-https://hc.apache.org/httpcomponents-asyncclient-dev/httpasyncclient/apidocs/org/apache/http/impl/nio/client/HttpAsyncClientBuilder.html[`org.apache.http.impl.nio.client.HttpAsyncClientBuilder`]
+https://hc.apache.org/httpcomponents-asyncclient-4.1.x/current/httpasyncclient/apidocs/org/apache/http/impl/nio/client/HttpAsyncClientBuilder.html[`org.apache.http.impl.nio.client.HttpAsyncClientBuilder`]
  allows to set)
 
 
@@ -339,9 +339,9 @@ translate to the execution of that request being cancelled, which needs to be
 specifically implemented in the API itself.
 
 The use of the `Cancellable` instance is optional and you can safely ignore this
-if you don't need it. A typical usecase for this would be using this together with 
+if you don't need it. A typical usecase for this would be using this together with
 frameworks like Rx Java or the Kotlin's `suspendCancellableCoRoutine`. Cancelling
-no longer needed requests is a good way to avoid putting unnecessary 
+no longer needed requests is a good way to avoid putting unnecessary
 load on Elasticsearch.
 
 ["source","java",subs="attributes,callouts,macros"]
@@ -366,7 +366,7 @@ include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-response2]
 <2> The host that returned the response
 <3> The response status line, from which you can for instance retrieve the status code
 <4> The response headers, which can also be retrieved by name though `getHeader(String)`
-<5> The response body enclosed in an https://hc.apache.org/httpcomponents-core-ga/httpcore/apidocs/org/apache/http/HttpEntity.html[`org.apache.http.HttpEntity`]
+<5> The response body enclosed in an https://hc.apache.org/httpcomponents-core-4.4.x/current/httpcore/apidocs/org/apache/http/HttpEntity.html[`org.apache.http.HttpEntity`]
  object
 
 When performing a request, an exception is thrown (or received as an argument
@@ -395,13 +395,13 @@ and un-marshalling. Users are free to use the library that they prefer for that
 purpose.
 
 The underlying Apache Async Http Client ships with different
-https://hc.apache.org/httpcomponents-core-ga/httpcore/apidocs/org/apache/http/HttpEntity.html[`org.apache.http.HttpEntity`]
+https://hc.apache.org/httpcomponents-core-4.4.x/current/httpcore/apidocs/org/apache/http/HttpEntity.html[`org.apache.http.HttpEntity`]
  implementations that allow to provide the request body in different formats
 (stream, byte array, string etc.). As for reading the response body, the
 `HttpEntity#getContent` method comes handy which returns an `InputStream`
 reading from the previously buffered response body. As an alternative, it is
 possible to provide a custom
-https://hc.apache.org/httpcomponents-core-ga/httpcore-nio/apidocs/org/apache/http/nio/protocol/HttpAsyncResponseConsumer.html[`org.apache.http.nio.protocol.HttpAsyncResponseConsumer`]
+https://hc.apache.org/httpcomponents-core-4.4.x/current/httpcore-nio/apidocs/org/apache/http/nio/protocol/HttpAsyncResponseConsumer.html[`org.apache.http.nio.protocol.HttpAsyncResponseConsumer`]
  that controls how bytes are read and buffered.
 
 [[java-rest-low-usage-logging]]
diff --git a/docs/plugins/analysis-icu.asciidoc b/docs/plugins/analysis-icu.asciidoc
index 3e6612dd8f4c..f4e8fcf65714 100644
--- a/docs/plugins/analysis-icu.asciidoc
+++ b/docs/plugins/analysis-icu.asciidoc
@@ -2,7 +2,7 @@
 === ICU Analysis Plugin
 
 The ICU Analysis plugin integrates the Lucene ICU module into {es},
-adding extended Unicode support using the http://site.icu-project.org/[ICU]
+adding extended Unicode support using the https://icu.unicode.org/[ICU]
 libraries, including better analysis of Asian languages, Unicode
 normalization, Unicode-aware case folding, collation support, and
 transliteration.
@@ -48,7 +48,7 @@ The following parameters are accepted:
 ==== ICU Normalization Character Filter
 
 Normalizes characters as explained
-http://userguide.icu-project.org/transforms/normalization[here].
+https://unicode-org.github.io/icu/userguide/transforms/normalization/[here].
 It registers itself as the `icu_normalizer` character filter, which is
 available to all indices without any further configuration. The type of
 normalization can be specified with the `name` parameter, which accepts `nfc`,
@@ -202,7 +202,7 @@ The above `analyze` request returns the following:
 ==== ICU Normalization Token Filter
 
 Normalizes characters as explained
-http://userguide.icu-project.org/transforms/normalization[here]. It registers
+https://unicode-org.github.io/icu/userguide/transforms/normalization/[here]. It registers
 itself as the `icu_normalizer` token filter, which is available to all indices
 without any further configuration. The type of normalization can be specified
 with the `name` parameter, which accepts `nfc`, `nfkc`, and `nfkc_cf`
@@ -554,4 +554,4 @@ GET icu_sample/_analyze
 <3> Returns `zdravstvujte`.
 <4> Returns `kon'nichiha`.
 
-For more documentation, Please see the http://userguide.icu-project.org/transforms/general[user guide of ICU Transform].
+For more documentation, Please see the https://unicode-org.github.io/icu/userguide/transforms/[user guide of ICU Transform].
diff --git a/docs/plugins/analysis-nori.asciidoc b/docs/plugins/analysis-nori.asciidoc
index 6a2e7c767bab..1f37a31496c0 100644
--- a/docs/plugins/analysis-nori.asciidoc
+++ b/docs/plugins/analysis-nori.asciidoc
@@ -82,7 +82,7 @@ dictionary to `$ES_HOME/config/userdict_ko.txt`:
 [source,txt]
 -----------------------
 c++                 <1>
-C샤프
+C쁠쁠
 세종
 세종시 세종 시        <2>
 -----------------------
@@ -176,7 +176,7 @@ PUT nori_sample
           "nori_user_dict": {
             "type": "nori_tokenizer",
             "decompound_mode": "mixed",
-            "user_dictionary_rules": ["c++", "C샤프", "세종", "세종시 세종 시"]
+            "user_dictionary_rules": ["c++", "C쁠쁠", "세종", "세종시 세종 시"]
           }
         },
         "analyzer": {
diff --git a/docs/plugins/plugin-script.asciidoc b/docs/plugins/plugin-script.asciidoc
index f04c18115230..9d924a8d2606 100644
--- a/docs/plugins/plugin-script.asciidoc
+++ b/docs/plugins/plugin-script.asciidoc
@@ -188,6 +188,7 @@ purge the configuration files while removing a plugin, use `-p` or `--purge`.
 This can option can be used after a plugin is removed to remove any lingering
 configuration files.
 
+[discrete]
 [[removing-multiple-plugins]]
 === Removing multiple plugins
 
diff --git a/docs/reference/aggregations/bucket.asciidoc b/docs/reference/aggregations/bucket.asciidoc
index a52eb15c9a15..4391d73ebd46 100644
--- a/docs/reference/aggregations/bucket.asciidoc
+++ b/docs/reference/aggregations/bucket.asciidoc
@@ -36,7 +36,7 @@ include::bucket/filter-aggregation.asciidoc[]
 
 include::bucket/filters-aggregation.asciidoc[]
 
-include::bucket/frequent-items-aggregation.asciidoc[]
+include::bucket/frequent-item-sets-aggregation.asciidoc[]
 
 include::bucket/geodistance-aggregation.asciidoc[]
 
diff --git a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
index 405aa6863111..26774c7091d2 100644
--- a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
@@ -80,7 +80,8 @@ time zone.
 One month is the interval between the start day of the month and time of
 day and the same day of the month and time of the following month in the specified
 time zone, so that the day of the month and time of day are the same at the start
-and end.
+and end. Note that the day may differ if an
+<<search-aggregations-bucket-datehistogram-offset-months,`offset` is used that is longer than a month>>.
 
 `quarter`, `1q` ::
 
@@ -543,6 +544,94 @@ NOTE: The start `offset` of each bucket is calculated after `time_zone`
 adjustments have been made.
 // end::offset-note[]
 
+[[search-aggregations-bucket-datehistogram-offset-months]]
+===== Long offsets over calendar intervals
+
+It is typical to use offsets in units smaller than the `calendar_interval`. For example,
+using offsets in hours when the interval is days, or an offset of days when the interval is months.
+If the calendar interval is always of a standard length, or the `offset` is less than one unit of the calendar
+interval (for example less than `+24h` for `days` or less than `+28d` for months),
+then each bucket will have a repeating start. For example `+6h` for `days` will result in all buckets
+starting at 6am each day. However, `+30h` will also result in buckets starting at 6am, except when crossing
+days that change from standard to summer-savings time or vice-versa.
+
+This situation is much more pronounced for months, where each month has a different length
+to at least one of its adjacent months.
+To demonstrate this, consider eight documents each with a date field on the 20th day of each of the
+eight months from January to August of 2022.
+
+When querying for a date histogram over the calendar interval of months, the response will return one bucket per month, each with a single document.
+Each bucket will have a key named after the first day of the month, plus any offset.
+For example, the offset of `+19d` will result in buckets with names like `2022-01-20`.
+
+[source,console,id=datehistogram-aggregation-offset-example-19d]
+--------------------------------------------------
+"buckets": [
+  { "key_as_string": "2022-01-20", "key": 1642636800000, "doc_count": 1 },
+  { "key_as_string": "2022-02-20", "key": 1645315200000, "doc_count": 1 },
+  { "key_as_string": "2022-03-20", "key": 1647734400000, "doc_count": 1 },
+  { "key_as_string": "2022-04-20", "key": 1650412800000, "doc_count": 1 },
+  { "key_as_string": "2022-05-20", "key": 1653004800000, "doc_count": 1 },
+  { "key_as_string": "2022-06-20", "key": 1655683200000, "doc_count": 1 },
+  { "key_as_string": "2022-07-20", "key": 1658275200000, "doc_count": 1 },
+  { "key_as_string": "2022-08-20", "key": 1660953600000, "doc_count": 1 }
+]
+--------------------------------------------------
+// TESTRESPONSE[skip:no setup made for this example yet]
+
+Increasing the offset to `+20d`, each document will appear in a bucket for the previous month,
+with all bucket keys ending with the same day of the month, as normal.
+However, further increasing to `+28d`, 
+what used to be a February bucket has now become `"2022-03-01"`.
+
+[source,console,id=datehistogram-aggregation-offset-example-28d]
+--------------------------------------------------
+"buckets": [
+  { "key_as_string": "2021-12-29", "key": 1640736000000, "doc_count": 1 },
+  { "key_as_string": "2022-01-29", "key": 1643414400000, "doc_count": 1 },
+  { "key_as_string": "2022-03-01", "key": 1646092800000, "doc_count": 1 },
+  { "key_as_string": "2022-03-29", "key": 1648512000000, "doc_count": 1 },
+  { "key_as_string": "2022-04-29", "key": 1651190400000, "doc_count": 1 },
+  { "key_as_string": "2022-05-29", "key": 1653782400000, "doc_count": 1 },
+  { "key_as_string": "2022-06-29", "key": 1656460800000, "doc_count": 1 },
+  { "key_as_string": "2022-07-29", "key": 1659052800000, "doc_count": 1 }
+]
+--------------------------------------------------
+// TESTRESPONSE[skip:no setup made for this example yet]
+
+If we continue to increase the offset, the 30-day months will also shift into the next month,
+so that 3 of the 8 buckets have different days than the other five.
+In fact if we keep going, we will find cases where two documents appear in the same month.
+Documents that were originally 30 days apart can be shifted into the same 31-day month bucket.
+
+For example, for `+50d` we see:
+
+[source,console,id=datehistogram-aggregation-offset-example-50d]
+--------------------------------------------------
+"buckets": [
+  { "key_as_string": "2022-01-20", "key": 1642636800000, "doc_count": 1 },
+  { "key_as_string": "2022-02-20", "key": 1645315200000, "doc_count": 2 },
+  { "key_as_string": "2022-04-20", "key": 1650412800000, "doc_count": 2 },
+  { "key_as_string": "2022-06-20", "key": 1655683200000, "doc_count": 2 },
+  { "key_as_string": "2022-08-20", "key": 1660953600000, "doc_count": 1 }
+]
+--------------------------------------------------
+// TESTRESPONSE[skip:no setup made for this example yet]
+
+It is therefor always important when using `offset` with `calendar_interval` bucket sizes
+to understand the consequences of using offsets larger than the interval size.
+
+More examples:
+
+* If the goal is to, for example, have an annual histogram where each year starts on the 5th February,
+you could use `calendar_interval` of `year` and `offset` of `+33d`, and each year will be shifted identically,
+because the offset includes only January, which is the same length every year.
+However, if the goal is to have the year start on the 5th March instead, this technique will not work because
+the offset includes February, which changes length every four years.
+* If you want a quarterly histogram starting on a date within the first month of the year, it will work,
+but as soon as you push the start date into the second month by having an offset longer than a month, the
+quarters will all start on different dates.
+
 [[date-histogram-keyed-response]]
 ==== Keyed Response
 
@@ -705,7 +794,7 @@ POST /sales/_search?size=0
 --------------------------------------------------
 // TEST[setup:sales]
 
-<1> Documents without a value in the `publish_date` field will fall into the
+<1> Documents without a value in the `date` field will fall into the
 same bucket as documents that have the value `2000-01-01`.
 
 [[date-histogram-order]]
diff --git a/docs/reference/aggregations/bucket/frequent-item-sets-aggregation.asciidoc b/docs/reference/aggregations/bucket/frequent-item-sets-aggregation.asciidoc
new file mode 100644
index 000000000000..01dacd0d6ccd
--- /dev/null
+++ b/docs/reference/aggregations/bucket/frequent-item-sets-aggregation.asciidoc
@@ -0,0 +1,405 @@
+[[search-aggregations-bucket-frequent-item-sets-aggregation]]
+=== Frequent item sets aggregation
+++++
+<titleabbrev>Frequent item sets</titleabbrev>
+++++
+
+A bucket aggregation which finds frequent item sets. It is a form of association
+rules mining that identifies items that often occur together. Items that are
+frequently purchased together or log events that tend to co-occur are examples
+of frequent item sets. Finding frequent item sets helps to discover
+relationships between different data points (items).
+
+The aggregation reports closed item sets. A frequent item set is called closed
+if no superset exists with the same ratio of documents (also known as its
+<<frequent-item-sets-minimum-support,support value>>). For example, we have the two
+following candidates for a frequent item set, which have the same support value:
+1. `apple, orange, banana`
+2. `apple, orange, banana, tomato`.
+Only the second item set (`apple, orange, banana, tomato`) is returned, and the
+first set – which is a subset of the second one – is skipped. Both item sets
+might be returned if their support values are different.
+
+The runtime of the aggregation depends on the data and the provided parameters.
+It might take a significant time for the aggregation to complete. For this
+reason, it is recommended to use <<async-search,async search>> to run your
+requests asynchronously.
+
+
+==== Syntax
+
+A `frequent_item_sets` aggregation looks like this in isolation:
+
+[source,js]
+--------------------------------------------------
+"frequent_item_sets": {
+  "minimum_set_size": 3,
+  "fields": [
+    {"field": "my_field_1"},
+    {"field": "my_field_2"}
+  ]
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+.`frequent_item_sets` Parameters
+|===
+|Parameter Name |Description |Required |Default Value
+|`fields` |(array) Fields to analyze. | Required |
+|`minimum_set_size` | (integer) The <<frequent-item-sets-minimum-set-size,minimum size>> of one item set. | Optional | `1`
+|`minimum_support` | (integer) The <<frequent-item-sets-minimum-support,minimum support>> of one item set. | Optional | `0.1`
+|`size` | (integer) The number of top item sets to return. | Optional | `10`
+|`filter` | (object) Query that filters documents from the analysis | Optional | `match_all`
+|===
+
+
+[discrete]
+[[frequent-item-sets-fields]]
+==== Fields
+
+Supported field types for the analyzed fields are keyword, numeric, ip, date,
+and arrays of these types. You can also add runtime fields to your analyzed
+fields.
+
+If the combined cardinality of the analyzed fields are high, the aggregation
+might require a significant amount of system resources.
+
+You can filter the values for each field by using the `include` and `exclude`
+parameters. The parameters can be regular expression strings or arrays of
+strings of exact terms. The filtered values are removed from the analysis and
+therefore reduce the runtime. If both `include` and `exclude` are defined,
+`exclude` takes precedence; it means `include` is evaluated first and then
+`exclude`.
+
+[discrete]
+[[frequent-item-sets-minimum-set-size]]
+==== Minimum set size
+
+The minimum set size is the minimum number of items the set needs to contain. A
+value of 1 returns the frequency of single items. Only item sets that contain at
+least the number of `minimum_set_size` items are returned. For example, the item
+set `orange, banana, apple` is returned only if the minimum set size is 3 or
+lower.
+
+[discrete]
+[[frequent-item-sets-minimum-support]]
+==== Minimum support
+
+The minimum support value is the ratio of documents that an item set must exist
+in to be considered "frequent". In particular, it is a normalized value between
+0 and 1. It is calculated by dividing the number of documents containing the
+item set by the total number of documents.
+
+For example, if a given item set is contained by five documents and the total
+number of documents is 20, then the support of the item set is 5/20 = 0.25.
+Therefore, this set is returned only if the minimum support is 0.25 or lower.
+As a higher minimum support prunes more items, the calculation is less resource
+intensive. The `minimum_support` parameter has an effect on the required memory
+and the runtime of the aggregation.
+
+
+[discrete]
+[[frequent-item-sets-size]]
+==== Size
+
+This parameter defines the maximum number of item sets to return. The result
+contains top-k item sets; the item sets with the highest support values. This
+parameter has a significant effect on the required memory and the runtime of the
+aggregation.
+
+
+[discrete]
+[[frequent-item-sets-filter]]
+==== Filter
+
+A query to filter documents to use as part of the analysis. Documents that
+don't match the filter are ignored when generating the item sets, however still
+count when calculating the support of an item set.
+
+Use the filter if you want to narrow the item set analysis to fields of interest.
+Use a top-level query to filter the data set.
+
+
+[discrete]
+[[frequent-item-sets-example]]
+==== Examples
+
+In the following examples, we use the e-commerce {kib} sample data set.
+
+
+[discrete]
+==== Aggregation with two analyzed fields and an `exclude` parameter
+
+In the first example, the goal is to find out based on transaction data (1.)
+from what product categories the customers purchase products frequently together
+and (2.) from which cities they make those purchases. We want to exclude results
+where location information is not available (where the city name is `other`).
+Finally, we are interested in sets with three or more items, and want to see the
+first three frequent item sets with the highest support.
+
+Note that we use the <<async-search,async search>> endpoint in this first
+example.
+
+[source,console]
+-------------------------------------------------
+POST /kibana_sample_data_ecommerce/_async_search
+{
+   "size":0,
+   "aggs":{
+      "my_agg":{
+         "frequent_item_sets":{
+            "minimum_set_size":3,
+            "fields":[
+               {
+                  "field":"category.keyword"
+               },
+               {
+                  "field":"geoip.city_name",
+                  "exclude":"other"
+               }
+            ],
+            "size":3
+         }
+      }
+   }
+}
+-------------------------------------------------
+// TEST[skip:setup kibana sample data]
+
+The response of the API call above contains an identifier (`id`) of the async
+search request. You can use the identifier to retrieve the search results:
+
+[source,console]
+-------------------------------------------------
+GET /_async_search/<id>
+-------------------------------------------------
+// TEST[skip:setup kibana sample data]
+
+The API returns a response similar to the following one:
+
+[source,console-result]
+-------------------------------------------------
+(...)
+"aggregations" : {
+    "my_agg" : {
+      "buckets" : [ <1>
+        {
+          "key" : { <2>
+            "category.keyword" : [
+              "Women's Clothing",
+              "Women's Shoes"
+            ],
+            "geoip.city_name" : [
+              "New York"
+            ]
+          },
+          "doc_count" : 217, <3>
+          "support" : 0.04641711229946524 <4>
+        },
+        {
+          "key" : {
+            "category.keyword" : [
+              "Women's Clothing",
+              "Women's Accessories"
+            ],
+            "geoip.city_name" : [
+              "New York"
+            ]
+          },
+          "doc_count" : 135,
+          "support" : 0.028877005347593583
+        },
+        {
+          "key" : {
+            "category.keyword" : [
+              "Men's Clothing",
+              "Men's Shoes"
+            ],
+            "geoip.city_name" : [
+              "Cairo"
+            ]
+          },
+          "doc_count" : 123,
+          "support" : 0.026310160427807486
+        }
+      ],
+    (...)
+  }
+}
+-------------------------------------------------
+// TEST[skip:setup kibana sample data]
+
+<1> The array of returned item sets.
+<2> The `key` object contains one item set. In this case, it consists of two
+values of the `category.keyword` field and one value of the `geoip.city_name`.
+<3> The number of documents that contain the item set.
+<4> The support value of the item set. It is calculated by dividing the number
+of documents containing the item set by the total number of documents.
+
+The response shows that the categories customers purchase from most frequently
+together are `Women's Clothing` and `Women's Shoes` and customers from New York
+tend to buy items from these categories frequently together. In other words,
+customers who buy products labelled `Women's Clothing` more likely buy products
+also from the `Women's Shoes` category and customers from New York most likely
+buy products from these categories together. The item set with the second
+highest support is `Women's Clothing` and `Women's Accessories` with customers
+mostly from New York. Finally, the item set with the third highest support is
+`Men's Clothing` and `Men's Shoes` with customers mostly from Cairo.
+
+
+[discrete]
+==== Aggregation with two analyzed fields and a filter
+
+We take the first example, but want to narrow the item sets to places in Europe.
+For that, we add a filter, and this time, we don't use the `exclude` parameter:
+
+[source,console]
+-------------------------------------------------
+POST /kibana_sample_data_ecommerce/_async_search
+{
+  "size": 0,
+  "aggs": {
+    "my_agg": {
+      "frequent_item_sets": {
+        "minimum_set_size": 3,
+        "fields": [
+          { "field": "category.keyword" },
+          { "field": "geoip.city_name" }
+        ],
+        "size": 3,
+        "filter": {
+          "term": {
+            "geoip.continent_name": "Europe"
+          }
+        }
+      }
+    }
+  }
+}
+-------------------------------------------------
+// TEST[skip:setup kibana sample data]
+
+The result will only show item sets that created from documents matching the
+filter, namely purchases in Europe. Using `filter`, the calculated `support`
+still takes all purchases into acount. That's different than specifying a query
+at the top-level, in which case `support` gets calculated only from purchases in
+Europe.
+
+
+[discrete]
+==== Analyzing numeric values by using a runtime field
+
+The frequent items aggregation enables you to bucket numeric values by using
+<<runtime,runtime fields>>. The next example demonstrates how to use a script to
+add a runtime field to your documents called `price_range`, which is
+calculated from the taxful total price of the individual transactions. The
+runtime field then can be used in the frequent items aggregation as a field to
+analyze.
+
+
+[source,console]
+-------------------------------------------------
+GET kibana_sample_data_ecommerce/_search
+{
+  "runtime_mappings": {
+    "price_range": {
+      "type": "keyword",
+      "script": {
+        "source": """
+           def bucket_start = (long) Math.floor(doc['taxful_total_price'].value / 50) * 50;
+           def bucket_end = bucket_start + 50;
+           emit(bucket_start.toString() + "-" + bucket_end.toString());
+        """
+      }
+    }
+  },
+  "size": 0,
+  "aggs": {
+    "my_agg": {
+      "frequent_item_sets": {
+        "minimum_set_size": 4,
+        "fields": [
+          {
+            "field": "category.keyword"
+          },
+          {
+            "field": "price_range"
+          },
+          {
+            "field": "geoip.city_name"
+          }
+        ],
+        "size": 3
+      }
+    }
+  }
+}
+-------------------------------------------------
+// TEST[skip:setup kibana sample data]
+
+The API returns a response similar to the following one:
+
+[source,console-result]
+-------------------------------------------------
+(...)
+"aggregations" : {
+    "my_agg" : {
+      "buckets" : [
+        {
+          "key" : {
+            "category.keyword" : [
+              "Women's Clothing",
+              "Women's Shoes"
+            ],
+            "price_range" : [
+              "50-100"
+            ],
+            "geoip.city_name" : [
+              "New York"
+            ]
+          },
+          "doc_count" : 100,
+          "support" : 0.0213903743315508
+        },
+        {
+          "key" : {
+            "category.keyword" : [
+              "Women's Clothing",
+              "Women's Shoes"
+            ],
+            "price_range" : [
+              "50-100"
+            ],
+            "geoip.city_name" : [
+              "Dubai"
+            ]
+          },
+          "doc_count" : 59,
+          "support" : 0.012620320855614974
+        },
+        {
+          "key" : {
+            "category.keyword" : [
+              "Men's Clothing",
+              "Men's Shoes"
+            ],
+            "price_range" : [
+              "50-100"
+            ],
+            "geoip.city_name" : [
+              "Marrakesh"
+            ]
+          },
+          "doc_count" : 53,
+          "support" : 0.011336898395721925
+        }
+      ],
+    (...)
+    }
+  }
+-------------------------------------------------
+// TEST[skip:setup kibana sample data]
+
+The response shows the categories that customers purchase from most frequently
+together, the location of the customers who tend to buy items from these
+categories, and the most frequent price ranges of these purchases.
diff --git a/docs/reference/aggregations/bucket/frequent-items-aggregation.asciidoc b/docs/reference/aggregations/bucket/frequent-items-aggregation.asciidoc
deleted file mode 100644
index 811f08c98ca0..000000000000
--- a/docs/reference/aggregations/bucket/frequent-items-aggregation.asciidoc
+++ /dev/null
@@ -1,393 +0,0 @@
-[[search-aggregations-bucket-frequent-items-aggregation]]
-=== Frequent items aggregation
-++++
-<titleabbrev>Frequent items</titleabbrev>
-++++
-
-experimental::[]
-
-A bucket aggregation which finds frequent item sets. It is a form of association 
-rules mining that identifies items that often occur together. Items that are 
-frequently purchased together or log events that tend to co-occur are examples 
-of frequent item sets. Finding frequent item sets helps to discover 
-relationships between different data points (items).
-
-The aggregation reports closed item sets. A frequent item set is called closed 
-if no superset exists with the same ratio of documents (also known as its 
-<<frequent-items-minimum-support,support value>>). For example, we have the two 
-following candidates for a frequent item set, which have the same support value:
-1. `apple, orange, banana`
-2. `apple, orange, banana, tomato`.
-Only the second item set (`apple, orange, banana, tomato`) is returned, and the 
-first set – which is a subset of the second one – is skipped. Both item sets 
-might be returned if their support values are different.
-
-The runtime of the aggregation depends on the data and the provided parameters. 
-It might take a significant time for the aggregation to complete. For this 
-reason, it is recommended to use <<async-search,async search>> to run your 
-requests asynchronously.
-
-
-==== Syntax
-
-A `frequent_items` aggregation looks like this in isolation:
-
-[source,js]
---------------------------------------------------
-"frequent_items": {
-  "minimum_set_size": 3,
-  "fields": [
-    {"field": "my_field_1"},
-    {"field": "my_field_2"}
-  ]
-}
---------------------------------------------------
-// NOTCONSOLE
-
-.`frequent_items` Parameters
-|===
-|Parameter Name |Description |Required |Default Value
-|`fields` |(array) Fields to analyze. | Required |
-|`minimum_set_size` | (integer) The <<frequent-items-minimum-set-size,minimum size>> of one item set. | Optional | `1`
-|`minimum_support` | (integer) The <<frequent-items-minimum-support,minimum support>> of one item set. | Optional | `0.1`
-|`size` | (integer) The number of top item sets to return. | Optional | `10`
-|`filter` | (object) Query that filters documents from the analysis | Optional | `match_all`
-|===
-
-
-[discrete]
-[[frequent-items-fields]]
-==== Fields
-
-Supported field types for the analyzed fields are keyword, numeric, ip, date, 
-and arrays of these types. You can also add runtime fields to your analyzed 
-fields.
-
-If the combined cardinality of the analyzed fields are high, then the 
-aggregation might require a significant amount of system resources.
-
-[discrete]
-[[frequent-items-minimum-set-size]]
-==== Minimum set size
-
-The minimum set size is the minimum number of items the set needs to contain. A 
-value of 1 returns the frequency of single items. Only item sets that contain at 
-least the number of `minimum_set_size` items are returned. For example, the item 
-set `orange, banana, apple` is returned only if the minimum set size is 3 or 
-lower.
-
-[discrete]
-[[frequent-items-minimum-support]]
-==== Minimum support
-
-The minimum support value is the ratio of documents that an item set must exist 
-in to be considered "frequent". In particular, it is a normalized value between 
-0 and 1. It is calculated by dividing the number of documents containing the 
-item set by the total number of documents.
-
-For example, if a given item set is contained by five documents and the total 
-number of documents is 20, then the support of the item set is 5/20 = 0.25. 
-Therefore, this set is returned only if the minimum support is 0.25 or lower. 
-As a higher minimum support prunes more items, the calculation is less resource 
-intensive. The `minimum_support` parameter has an effect on the required memory 
-and the runtime of the aggregation.
-
-
-[discrete]
-[[frequent-items-size]]
-==== Size
-
-This parameter defines the maximum number of item sets to return. The result 
-contains top-k item sets; the item sets with the highest support values. This 
-parameter has a significant effect on the required memory and the runtime of the 
-aggregation.
-
-
-[discrete]
-[[frequent-items-filter]]
-==== Filter
-
-A query to filter documents to use as part of the analysis. Documents that
-don't match the filter are ignored when generating the item sets, however still
-count when calculating the support of an item set.
-
-Use the filter if you want to narrow the item set analysis to fields of interest.
-Use a top-level query to filter the data set.
-
-
-[discrete]
-[[frequent-items-example]]
-==== Examples
-
-In the following examples, we use the e-commerce {kib} sample data set.
-
-
-[discrete]
-==== Aggregation with two analyzed fields
-
-In the first example, the goal is to find out based on transaction data (1.) 
-from what product categories the customers purchase products frequently together 
-and (2.) from which cities they make those purchases. We are interested in sets 
-with three or more items, and want to see the first three frequent item sets 
-with the highest support.
-
-Note that we use the <<async-search,async search>> endpoint in this first 
-example.
-
-[source,console]
--------------------------------------------------
-POST /kibana_sample_data_ecommerce/_async_search
-{
-  "size": 0,
-  "aggs": {
-    "my_agg": {
-      "frequent_items": {
-        "minimum_set_size": 3,
-        "fields": [
-          { "field": "category.keyword" }, 
-          { "field": "geoip.city_name" }
-        ],
-        "size": 3
-      }
-    }
-  }
-}
--------------------------------------------------
-// TEST[skip:setup kibana sample data]
-
-The response of the API call above contains an identifier (`id`) of the async 
-search request. You can use the identifier to retrieve the search results:
-
-[source,console]
--------------------------------------------------
-GET /_async_search/<id>
--------------------------------------------------
-// TEST[skip:setup kibana sample data]
-
-The API returns a response similar to the following one:
-
-[source,console-result]
--------------------------------------------------
-(...)
-"aggregations" : {
-    "my_agg" : {
-      "buckets" : [ <1>
-        {
-          "key" : { <2>
-            "category.keyword" : [
-              "Women's Clothing",
-              "Women's Shoes"
-            ],
-            "geoip.city_name" : [
-              "New York"
-            ]
-          },
-          "doc_count" : 217, <3>
-          "support" : 0.04641711229946524 <4>
-        },
-        {
-          "key" : {
-            "category.keyword" : [
-              "Women's Clothing",
-              "Women's Accessories"
-            ],
-            "geoip.city_name" : [
-              "New York"
-            ]
-          },
-          "doc_count" : 135,
-          "support" : 0.028877005347593583
-        },
-        {
-          "key" : {
-            "category.keyword" : [
-              "Men's Clothing",
-              "Men's Shoes"
-            ],
-            "geoip.city_name" : [
-              "Cairo"
-            ]
-          },
-          "doc_count" : 123,
-          "support" : 0.026310160427807486
-        }
-      ],
-    (...) 
-  }
-}
--------------------------------------------------
-// TEST[skip:setup kibana sample data]
-
-<1> The array of returned item sets.
-<2> The `key` object contains one item set. In this case, it consists of two 
-values of the `category.keyword` field and one value of the `geoip.city_name`.
-<3> The number of documents that contain the item set. 
-<4> The support value of the item set. It is calculated by dividing the number 
-of documents containing the item set by the total number of documents. 
-
-The response shows that the categories customers purchase from most frequently 
-together are `Women's Clothing` and `Women's Shoes` and customers from New York 
-tend to buy items from these categories frequently togeher. In other words, 
-customers who buy products labelled `Women's Clothing` more likely buy products 
-also from the `Women's Shoes` category and customers from New York most likely buy 
-products from these categories together. The item set with the second highest 
-support is `Women's Clothing` and `Women's Accessories` with customers mostly 
-from New York. Finally, the item set with the third highest support is 
-`Men's Clothing` and `Men's Shoes` with customers mostly from Cairo.
-
-
-[discrete]
-==== Aggregation with two analyzed fields and a filter
-
-We take the first example, but want to narrow the item sets to places in Europe.
-For that we add a filter:
-
-[source,console]
--------------------------------------------------
-POST /kibana_sample_data_ecommerce/_async_search
-{
-  "size": 0,
-  "aggs": {
-    "my_agg": {
-      "frequent_items": {
-        "minimum_set_size": 3,
-        "fields": [
-          { "field": "category.keyword" },
-          { "field": "geoip.city_name" }
-        ],
-        "size": 3,
-        "filter": {
-          "term": {
-            "geoip.continent_name": "Europe"
-          }
-        }
-      }
-    }
-  }
-}
--------------------------------------------------
-// TEST[skip:setup kibana sample data]
-
-The result will only show item sets that created from documents matching the
-filter, namely purchases in Europe. Using `filter`, the calculated `support` still
-takes all purchases into acount. That's different than specifying a query at the
-top-level, in which case `support` gets calculated only from purchases in Europe.
-
-
-[discrete]
-==== Analyzing numeric values by using a runtime field
-
-The frequent items aggregation enables you to bucket numeric values by using 
-<<runtime,runtime fields>>. The next example demonstrates how to use a script to 
-add a runtime field to your documents called `price_range`, which is 
-calculated from the taxful total price of the individual transactions. The 
-runtime field then can be used in the frequent items aggregation as a field to 
-analyze.
-
-
-[source,console]
--------------------------------------------------
-GET kibana_sample_data_ecommerce/_search
-{
-  "runtime_mappings": {
-    "price_range": {
-      "type": "keyword",
-      "script": {
-        "source": """
-           def bucket_start = (long) Math.floor(doc['taxful_total_price'].value / 50) * 50;
-           def bucket_end = bucket_start + 50;
-           emit(bucket_start.toString() + "-" + bucket_end.toString());
-        """
-      }
-    }
-  },
-  "size": 0,
-  "aggs": {
-    "my_agg": {
-      "frequent_items": {
-        "minimum_set_size": 4,
-        "fields": [
-          {
-            "field": "category.keyword"
-          },
-          {
-            "field": "price_range"
-          },
-          {
-            "field": "geoip.city_name"
-          }
-        ],
-        "size": 3
-      }
-    }
-  }
-}
--------------------------------------------------
-// TEST[skip:setup kibana sample data]
-
-The API returns a response similar to the following one:
-
-[source,console-result]
--------------------------------------------------
-(...)
-"aggregations" : {
-    "my_agg" : {
-      "buckets" : [
-        {
-          "key" : {
-            "category.keyword" : [
-              "Women's Clothing",
-              "Women's Shoes"
-            ],
-            "price_range" : [
-              "50-100"
-            ],
-            "geoip.city_name" : [
-              "New York"
-            ]
-          },
-          "doc_count" : 100,
-          "support" : 0.0213903743315508
-        },
-        {
-          "key" : {
-            "category.keyword" : [
-              "Women's Clothing",
-              "Women's Shoes"
-            ],
-            "price_range" : [
-              "50-100"
-            ],
-            "geoip.city_name" : [
-              "Dubai"
-            ]
-          },
-          "doc_count" : 59,
-          "support" : 0.012620320855614974
-        },
-        {
-          "key" : {
-            "category.keyword" : [
-              "Men's Clothing",
-              "Men's Shoes"
-            ],
-            "price_range" : [
-              "50-100"
-            ],
-            "geoip.city_name" : [
-              "Marrakesh"
-            ]
-          },
-          "doc_count" : 53,
-          "support" : 0.011336898395721925
-        }
-      ],
-    (...)
-    }
-  }
--------------------------------------------------
-// TEST[skip:setup kibana sample data]
-
-The response shows the categories that customers purchase from most frequently 
-together, the location of the customers who tend to buy items from these 
-categories, and the most frequent price ranges of these purchases.
\ No newline at end of file
diff --git a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc
index 1b47a2cf2cd4..ce0fea4f4ea9 100644
--- a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc
@@ -5,8 +5,8 @@
 <titleabbrev>Geohex grid</titleabbrev>
 ++++
 
-A multi-bucket aggregation that groups <<geo-point,`geo_point`>>
-values into buckets that represent a grid.
+A multi-bucket aggregation that groups <<geo-point,`geo_point`>> and
+<<geo-shape,`geo_shape`>> values into buckets that represent a grid.
 The resulting grid can be sparse and only
 contains cells that have matching data. Each cell corresponds to a
 https://h3geo.org/docs/core-library/h3Indexing#h3-cell-indexp[H3 cell index] and is
@@ -18,7 +18,7 @@ Precision for this aggregation can be between 0 and 15, inclusive.
 
 WARNING: High-precision requests can be very expensive in terms of RAM and
 result sizes. For example, the highest-precision geohex with a precision of 15
-produces cells that cover less than 10cm by 10cm. We recommend you use a
+produces cells that cover less than one square meter. We recommend you use a
 filter to limit high-precision requests to a smaller geographic area. For an example,
 refer to <<geohexgrid-high-precision>>.
 
@@ -220,21 +220,45 @@ Response:
 --------------------------------------------------
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/]
 
-[[geohexgrid-options]]
+[discrete]
+[role="xpack"]
+[[geohexgrid-aggregating-geo-shape]]
+==== Aggregating `geo_shape` fields
+
+Aggregating on <<geo-shape>> fields works almost as it does for points. There are two key differences:
+
+* When aggregating over `geo_point` data, points are considered within a hexagonal tile if they lie
+within the edges defined by great circles. In other words the calculation is done using spherical coordinates.
+However, when aggregating over `geo_shape` data, the shapes are considered within a hexagon if they lie
+within the edges defined as straight lines on an equirectangular projection.
+The reason is that Elasticsearch and Lucene treat edges using the equirectangular projection at index and search time.
+In order to ensure that search results and aggregation results are aligned, we therefore also use equirectangular
+projection in aggregations.
+For most data, the difference is subtle or not noticed.
+However, for low zoom levels (low precision), especially far from the equator,  this can be noticeable.
+For example, if the same point data is indexed as `geo_point` and `geo_shape`, it is possible  to get
+different results when aggregating at lower resolutions.
+* As is the case with <<geotilegrid-aggregating-geo-shape,`geotile_grid`>>,
+a single shape can be counted for in multiple tiles. A shape will contribute to the count of matching values
+if any part of its shape intersects with that tile. Below is an image that demonstrates this:
+
+
+image:images/spatial/geoshape_hexgrid.png[]
+
 ==== Options
 
 [horizontal]
 field::
-(Required, string) Field containing indexed geo-point values. Must be explicitly
-mapped as a <<geo-point,`geo_point`>> field. If the field contains an array,
-`geohex_grid` aggregates all array values.
+(Required, string) Field containing indexed geo-point or geo-shape values.
+Must be explicitly mapped as a <<geo-point,`geo_point`>> or a <<geo-shape,`geo_shape`>> field.
+If the field contains an array, `geohex_grid` aggregates all array values.
 
 precision::
 (Optional, integer) Integer zoom of the key used to define cells/buckets in
 the results. Defaults to `6`. Values outside of [`0`,`15`] will be rejected.
 
 bounds::
-(Optional, object) Bounding box used to filter the geo-points in each bucket.
+(Optional, object) Bounding box used to filter the geo-points or geo-shapes in each bucket.
 Accepts the same bounding box formats as the
 <<query-dsl-geo-bounding-box-query-accepted-formats,geo-bounding box query>>.
 
@@ -245,5 +269,6 @@ documents they contain.
 
 shard_size::
 (Optional, integer) Number of buckets returned from each shard. Defaults to
-`max(10,(size x number-of-shards))` to allow for more a accurate count of the
-top cells in the final result.
+`max(10,(size x number-of-shards))` to allow for a more accurate count of the
+top cells in the final result. Since each shard could have a different top result order,
+using a larger number here reduces the risk of inaccurate counts, but incurs a performance cost.
diff --git a/docs/reference/aggregations/bucket/geotilegrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geotilegrid-aggregation.asciidoc
index db5dc58f23cc..413cc622d5d0 100644
--- a/docs/reference/aggregations/bucket/geotilegrid-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/geotilegrid-aggregation.asciidoc
@@ -17,7 +17,7 @@ cover only a small area.
 * Low precision keys have a smaller range for x and y, and represent tiles that
 each cover a large area.
 
-See https://wiki.openstreetmap.org/wiki/Zoom_levels[Zoom level documentation]
+See https://wiki.openstreetmap.org/wiki/Zoom_levels[zoom level documentation]
 on how precision (zoom) correlates to size on the ground. Precision for this
 aggregation can be between 0 and 29, inclusive.
 
@@ -102,14 +102,15 @@ Response:
 --------------------------------------------------
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/]
 
+[[geotilegrid-high-precision]]
 ==== High-precision requests
 
-When requesting detailed buckets (typically for displaying a "zoomed in" map)
+When requesting detailed buckets (typically for displaying a "zoomed in" map),
 a filter like <<query-dsl-geo-bounding-box-query,geo_bounding_box>> should be
-applied to narrow the subject area otherwise potentially millions of buckets
+applied to narrow the subject area. Otherwise, potentially millions of buckets
 will be created and returned.
 
-[source,console]
+[source,console,id=geotilegrid-high-precision-ex]
 --------------------------------------------------
 POST /museums/_search?size=0
 {
@@ -137,6 +138,8 @@ POST /museums/_search?size=0
 --------------------------------------------------
 // TEST[continued]
 
+Response:
+
 [source,console-result]
 --------------------------------------------------
 {
@@ -166,13 +169,14 @@ POST /museums/_search?size=0
 --------------------------------------------------
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/]
 
+[[geotilegrid-addtl-bounding-box-filtering]]
 ==== Requests with additional bounding box filtering
 
 The `geotile_grid` aggregation supports an optional `bounds` parameter
-that restricts the cells considered to those that intersects the
-bounds provided. The `bounds` parameter accepts the bounding box in
-all the same <<query-dsl-geo-bounding-box-query-accepted-formats,accepted formats>> of the
-bounds specified in the Geo Bounding Box Query. This bounding box can be used with or
+that restricts the cells considered to those that intersect the
+provided bounds. The `bounds` parameter accepts the same
+<<query-dsl-geo-bounding-box-query-accepted-formats,bounding box formats>>
+as the geo-bounding box query. This bounding box can be used with or
 without an additional `geo_bounding_box` query for filtering the points prior to aggregating.
 It is an independent bounding box that can intersect with, be equal to, or be disjoint
 to any additional `geo_bounding_box` queries defined in the context of the aggregation.
@@ -197,6 +201,8 @@ POST /museums/_search?size=0
 --------------------------------------------------
 // TEST[continued]
 
+Response:
+
 [source,console-result]
 --------------------------------------------------
 {
@@ -225,9 +231,10 @@ POST /museums/_search?size=0
 
 [discrete]
 [role="xpack"]
+[[geotilegrid-aggregating-geo-shape]]
 ==== Aggregating `geo_shape` fields
 
-Aggregating on <<geo-shape>> fields works just as it does for points, except that a single
+Aggregating on <<geo-shape>> fields works almost as it does for points, except that a single
 shape can be counted for in multiple tiles. A shape will contribute to the count of matching values
 if any part of its shape intersects with that tile. Below is an image that demonstrates this:
 
@@ -237,20 +244,27 @@ image:images/spatial/geoshape_grid.png[]
 ==== Options
 
 [horizontal]
-field::         Mandatory. The name of the field indexed with GeoPoints.
+field::
+(Required, string) Field containing indexed geo-point or geo-shape values.
+Must be explicitly mapped as a <<geo-point,`geo_point`>> or a <<geo-shape,`geo_shape`>> field.
+If the field contains an array, `geotile_grid` aggregates all array values.
 
-precision::     Optional. The integer zoom of the key used to define
-                cells/buckets in the results. Defaults to 7.
-                Values outside of [0,29] will be rejected.
+precision::
+(Optional, integer) Integer zoom of the key used to define cells/buckets in
+the results. Defaults to `7`. Values outside of [`0`,`29`] will be rejected.
 
-bounds:         Optional. The bounding box to filter the points in the bucket.
+bounds::
+(Optional, object) Bounding box used to filter the geo-points or geo-shapes in each bucket.
+Accepts the same bounding box formats as the
+<<query-dsl-geo-bounding-box-query-accepted-formats,geo-bounding box query>>.
 
-size::          Optional. The maximum number of geohash buckets to return
-                (defaults to 10,000). When results are trimmed, buckets are
-                prioritised based on the volumes of documents they contain.
+size::
+(Optional, integer) Maximum number of buckets to return. Defaults to 10,000.
+When results are trimmed, buckets are prioritized based on the volume of
+documents they contain.
 
-shard_size::    Optional. To allow for more accurate counting of the top cells
-                returned in the final result the aggregation defaults to
-                returning `max(10,(size x number-of-shards))` buckets from each
-                shard. If this heuristic is undesirable, the number considered
-                from each shard can be over-ridden using this parameter.
+shard_size::
+(Optional, integer) Number of buckets returned from each shard. Defaults to
+`max(10,(size x number-of-shards))` to allow for a more accurate count of the
+top cells in the final result. Since each shard could have a different top result order,
+using a larger number here reduces the risk of inaccurate counts, but incurs a performance cost.
diff --git a/docs/reference/analysis/token-graphs.asciidoc b/docs/reference/analysis/token-graphs.asciidoc
index 9881afe908eb..55d69695bd62 100644
--- a/docs/reference/analysis/token-graphs.asciidoc
+++ b/docs/reference/analysis/token-graphs.asciidoc
@@ -34,7 +34,7 @@ include tokens for multi-word synonyms, such as using "atm" as a synonym for
 "automatic teller machine."
 
 However, only some token filters, known as _graph token filters_, accurately
-record the `positionLength` for multi-position tokens. This filters include:
+record the `positionLength` for multi-position tokens. These filters include:
 
 * <<analysis-synonym-graph-tokenfilter,`synonym_graph`>>
 * <<analysis-word-delimiter-graph-tokenfilter,`word_delimiter_graph`>>
@@ -105,4 +105,4 @@ in an invalid graph.
 image::images/analysis/token-graph-dns-invalid-ex.svg[align="center"]
 
 Avoid using invalid token graphs for search. Invalid graphs can cause unexpected
-search results.
\ No newline at end of file
+search results.
diff --git a/docs/reference/autoscaling/deciders/machine-learning-decider.asciidoc b/docs/reference/autoscaling/deciders/machine-learning-decider.asciidoc
index 1a2b230cf520..26ced6ad7bb2 100644
--- a/docs/reference/autoscaling/deciders/machine-learning-decider.asciidoc
+++ b/docs/reference/autoscaling/deciders/machine-learning-decider.asciidoc
@@ -2,25 +2,26 @@
 [[autoscaling-machine-learning-decider]]
 === Machine learning decider
 
-The {ml} decider (`ml`) calculates the memory required to run {ml} jobs.
+The {ml} decider (`ml`) calculates the memory and CPU requirements to run {ml} 
+jobs and trained models.
 
 The {ml} decider is enabled for policies governing `ml` nodes.
 
-NOTE: For {ml} jobs to open when the cluster is not appropriately
-scaled, set `xpack.ml.max_lazy_ml_nodes` to the largest number of possible {ml} 
-jobs (refer to <<advanced-ml-settings>> for more information). In {ess}, this is 
+NOTE: For {ml} jobs to open when the cluster is not appropriately scaled, set 
+`xpack.ml.max_lazy_ml_nodes` to the largest number of possible {ml} nodes (refer 
+to <<advanced-ml-settings>> for more information). In {ess}, this is 
 automatically set.
 
 [[autoscaling-machine-learning-decider-settings]]
 ==== Configuration settings
 
 Both `num_anomaly_jobs_in_queue` and `num_analytics_jobs_in_queue` are designed 
-to delay a scale-up event. If the cluster is too small, these settings indicate how many jobs of each type can be 
-unassigned from a node. Both settings are 
-only considered for jobs that can be opened given the current scale. If a job is 
-too large for any node size or if a job can't be assigned without user 
-intervention (for example, a user calling `_stop` against a real-time 
-{anomaly-job}), the numbers are ignored for that particular job.
+to delay a scale-up event. If the cluster is too small, these settings indicate 
+how many jobs of each type can be unassigned from a node. Both settings are only 
+considered for jobs that can be opened given the current scale. If a job is too 
+large for any node size or if a job can't be assigned without user intervention 
+(for example, a user calling `_stop` against a real-time {anomaly-job}), the 
+numbers are ignored for that particular job.
 
 `num_anomaly_jobs_in_queue`::
 (Optional, integer)
diff --git a/docs/reference/cluster/get-desired-balance.asciidoc b/docs/reference/cluster/get-desired-balance.asciidoc
index caa77561a08c..a9db8838ee27 100644
--- a/docs/reference/cluster/get-desired-balance.asciidoc
+++ b/docs/reference/cluster/get-desired-balance.asciidoc
@@ -32,6 +32,90 @@ The API returns the following result:
     "computation_time_in_millis": 0,
     "reconciliation_time_in_millis": 0
   },
+  "cluster_balance_stats" : {
+    "tiers": {
+      "data_hot" : {
+        "shard_count" : {
+          "total" : 7.0,
+          "min" : 2.0,
+          "max" : 3.0,
+          "average" : 2.3333333333333335,
+          "std_dev" : 0.4714045207910317
+        },
+        "forecast_write_load" : {
+          "total" : 21.0,
+          "min" : 6.0,
+          "max" : 8.5,
+          "average" : 7.0,
+          "std_dev" : 1.0801234497346435
+        },
+        "forecast_disk_usage" : {
+          "total" : 36.0,
+          "min" : 10.0,
+          "max" : 16.0,
+          "average" : 12.0,
+          "std_dev" : 2.8284271247461903
+        },
+        "actual_disk_usage" : {
+          "total" : 36.0,
+          "min" : 10.0,
+          "max" : 16.0,
+          "average" : 12.0,
+          "std_dev" : 2.8284271247461903
+        }
+      },
+      "data_warm" : {
+        "shard_count" : {
+          "total" : 3.0,
+          "min" : 1.0,
+          "max" : 1.0,
+          "average" : 1.0,
+          "std_dev" : 0.0
+        },
+        "forecast_write_load" : {
+          "total" : 0.0,
+          "min" : 0.0,
+          "max" : 0.0,
+          "average" : 0.0,
+          "std_dev" : 0.0
+        },
+        "forecast_disk_usage" : {
+          "total" : 42.0,
+          "min" : 12.0,
+          "max" : 18.0,
+          "average" : 14.0,
+          "std_dev" : 2.8284271247461903
+        },
+        "actual_disk_usage" : {
+          "total" : 42.0,
+          "min" : 12.0,
+          "max" : 18.0,
+          "average" : 14.0,
+          "std_dev" : 2.8284271247461903
+        }
+      }
+    },
+    "nodes": {
+      "node-1": {
+        "shard_count": 10,
+        "forecast_write_load": 8.5,
+        "forecast_disk_usage_bytes": 498435,
+        "actual_disk_usage_bytes": 498435
+      },
+      "node-2": {
+        "shard_count": 15,
+        "forecast_write_load": 3.25,
+        "forecast_disk_usage_bytes": 384935,
+        "actual_disk_usage_bytes": 384935
+      },
+      "node-3": {
+        "shard_count": 12,
+        "forecast_write_load": 6.0,
+        "forecast_disk_usage_bytes": 648766,
+        "actual_disk_usage_bytes": 648766
+      }
+    }
+  },
   "routing_table": {
     "test": {
       "0": {
@@ -44,7 +128,9 @@ The API returns the following result:
             "relocating_node": null,
             "relocating_node_is_desired": false,
             "shard_id": 0,
-            "index": "test"
+            "index": "test",
+            "forecast_write_load": 8.0,
+            "forecast_shard_size_in_bytes": 1024
           }
         ],
         "desired": {
@@ -66,7 +152,9 @@ The API returns the following result:
             "relocating_node": null,
             "relocating_node_is_desired": false,
             "shard_id": 1,
-            "index": "test"
+            "index": "test",
+            "forecast_write_load": null,
+            "forecast_shard_size_in_bytes": null
           }
         ],
         "desired": {
diff --git a/docs/reference/cluster/get-settings.asciidoc b/docs/reference/cluster/get-settings.asciidoc
index 16b88f605010..931ebc9759a8 100644
--- a/docs/reference/cluster/get-settings.asciidoc
+++ b/docs/reference/cluster/get-settings.asciidoc
@@ -37,7 +37,7 @@ defined, but can also include the default settings by calling the
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=flat-settings]
 
 `include_defaults`::
-    (Optional, Boolean) If `true`, returns all default cluster settings. 
+    (Optional, Boolean) If `true`, returns default cluster settings from the local node. 
     Defaults to `false`.
 
-include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms]
\ No newline at end of file
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms]
diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc
index da0402cac171..98e22eaafd84 100644
--- a/docs/reference/cluster/nodes-stats.asciidoc
+++ b/docs/reference/cluster/nodes-stats.asciidoc
@@ -2679,8 +2679,8 @@ requests.
 [[cluster-nodes-stats-api-example]]
 ==== {api-examples-title}
 
-[source,console]
---------------------------------------------------
+[source,console,id=nodes-stats-limit]
+----
 # return just indices
 GET /_nodes/stats/indices
 
@@ -2689,8 +2689,7 @@ GET /_nodes/stats/os,process
 
 # return just process for node with IP address 10.0.0.1
 GET /_nodes/10.0.0.1/stats/process
---------------------------------------------------
-// TESTRESPONSE[skip:"AwaitsFix https://github.com/elastic/elasticsearch/issues/91081"]
+----
 
 All stats can be explicitly requested via `/_nodes/stats/_all` or
 `/_nodes/stats?metric=_all`.
@@ -2698,8 +2697,8 @@ All stats can be explicitly requested via `/_nodes/stats/_all` or
 You can get information about indices stats on `node`, `indices`, or `shards`
 level.
 
-[source,console]
---------------------------------------------------
+[source,console,id=nodes-stats-indices]
+----
 # Fielddata summarized by node
 GET /_nodes/stats/indices/fielddata?fields=field1,field2
 
@@ -2711,21 +2710,19 @@ GET /_nodes/stats/indices/fielddata?level=shards&fields=field1,field2
 
 # You can use wildcards for field names
 GET /_nodes/stats/indices/fielddata?fields=field*
---------------------------------------------------
-// TESTRESPONSE[skip:"AwaitsFix https://github.com/elastic/elasticsearch/issues/91081"]
+----
 
 You can get statistics about search groups for searches executed
 on this node.
 
-[source,console]
---------------------------------------------------
+[source,console,id=nodes-stats-groups]
+----
 # All groups with all stats
 GET /_nodes/stats?groups=_all
 
 # Some groups from just the indices stats
 GET /_nodes/stats/indices?groups=foo,bar
---------------------------------------------------
-// TESTRESPONSE[skip:"AwaitsFix https://github.com/elastic/elasticsearch/issues/91081"]
+----
 
 [[cluster-nodes-stats-ingest-ex]]
 ===== Retrieve ingest statistics only
@@ -2734,25 +2731,23 @@ To return only ingest-related node statistics, set the `<metric>` path
 parameter to `ingest` and use the
 <<common-options-response-filtering,`filter_path`>> query parameter.
 
-[source,console]
---------------------------------------------------
+[source,console,id=nodes-stats-filter-path]
+----
 GET /_nodes/stats/ingest?filter_path=nodes.*.ingest
---------------------------------------------------
-// TESTRESPONSE[skip:"AwaitsFix https://github.com/elastic/elasticsearch/issues/91081"]
+----
 
 You can use the `metric` and `filter_path` query parameters to get the same
 response.
 
-[source,console]
---------------------------------------------------
+[source,console,id=nodes-stats-metric-filter-path]
+----
 GET /_nodes/stats?metric=ingest&filter_path=nodes.*.ingest
---------------------------------------------------
+----
 
 To further refine the response, change the `filter_path` value.
 For example, the following request only returns ingest pipeline statistics.
 
-[source,console]
---------------------------------------------------
+[source,console,id=nodes-stats-metric-filter-path-refined]
+----
 GET /_nodes/stats?metric=ingest&filter_path=nodes.*.ingest.pipelines
---------------------------------------------------
-// TESTRESPONSE[skip:"AwaitsFix https://github.com/elastic/elasticsearch/issues/91081"]
+----
diff --git a/docs/reference/cluster/prevalidate-node-removal.asciidoc b/docs/reference/cluster/prevalidate-node-removal.asciidoc
index b6312e37f025..d7f0ed64d6c0 100644
--- a/docs/reference/cluster/prevalidate-node-removal.asciidoc
+++ b/docs/reference/cluster/prevalidate-node-removal.asciidoc
@@ -21,7 +21,9 @@ Prevalidate node removal.
 [[prevalidate-node-removal-api-desc]]
 ==== {api-description-title}
 
-This API checks whether attempting to remove the specified node(s) from the cluster is likely to succeed or not. For a cluster with no unassigned shards, removal of any node is considered safe which means the removal of the nodes is likely to succeed. In case the cluster has a <<cluster-health,`red` cluster health status>>, it verifies that the removal of the node(s) would not risk removing the last remaining copy of an unassigned shard.
+This API checks whether attempting to remove the specified node(s) from the cluster is likely to succeed or not. For a cluster with no unassigned shards, removal of any node is considered safe which means the removal of the nodes is likely to succeed.
+
+In case the cluster has a <<cluster-health,`red` cluster health status>>, it verifies that the removal of the node(s) would not risk removing the last remaining copy of an unassigned shard. If there are red indices in the cluster, the API checks whether the red indices are <<searchable-snapshots, Searchable Snapshot>> indices, and if not, it sends a request to each of nodes specified in the API call to verify whether the nodes might contain local shard copies of the red indices that are not Searchable Snapshot indices. This request is processed on each receiving node, by checking whether the node has a shard directory for any of the red index shards.
 
 The response includes the overall safety of the removal of the specified nodes, and a detailed response for each node. The node-specific part of the response also includes more details on why removal of that node might not succeed.
 
@@ -32,7 +34,7 @@ Note that if the prevalidation result for a set of nodes returns `true` (i.e. it
 [[prevalidate-node-removal-api-query-params]]
 ==== {api-query-parms-title}
 
-include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout]
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms]
 
 `names`::
 (Optional, string) Comma-separated list of node names.
@@ -54,11 +56,50 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout]
 
 `nodes`::
     (object) Prevalidation result for the removal of each of the provided nodes.
++
+.Properties of `nodes`
+[%collapsible%open]
+====
+`<node>`::
+    (object) Contains information about the removal prevalidation of a specific node.
++
+.Properties of `<node>`
+[%collapsible%open]
+=======
+`id`::
+    (string) node ID
+`name`::
+    (string) node name
+`external_id`::
+    (string) node external ID
+`result`::
+    (object) Contains removal prevalidation result of the node.
++
+.Properties of `result`
+[%collapsible%open]
+========
+`is_safe`::
+    (boolean) Whether the removal of the node is considered safe or not.
+`reason`::
+    (string) A string that specifies the reason why the prevalidation result is considered safe or not. It can be one of the following values:
++
+--
+    * `no_problems`: The prevalidation did not find any issues that could prevent the node from being safely removed.
+    * `no_red_shards_except_searchable_snapshots`: The node can be safely removed as all red indices are searchable snapshot indices and therefore removing a node does not risk removing the last copy of that index from the cluster.
+    * `no_red_shards_on_node`: The node does not contain any copies of the red non-searchable-snapshot index shards.
+    * `red_shards_on_node`: The node might contain shard copies of some non-searchable-snapshot red indices. The list of the shards that might be on the node are specified in the `message` field.
+    * `unable_to_verify_red_shards`: Contacting the node failed or timed out. More details is provided in the `message` field.
+--
+`message`::
+    (Optional, string) Detailed information about the removal prevalidation result.
+========
+=======
+====
 
 [[prevalidate-node-removal-api-example]]
 ==== {api-examples-title}
 
-This example validates whether it is safe to remove the nodes `node1` and `node2`. The response indicates that it is safe to remove `node1`, but it might not be safe to remove `node2`. Therefore, the overall prevalidation of the removal of the two nodes returns `false`.
+This example validates whether it is safe to remove the nodes `node1` and `node2`. The response indicates that it is safe to remove `node1`, but it might not be safe to remove `node2` as it might contain copies of the specified red shards. Therefore, the overall prevalidation of the removal of the two nodes returns `false`.
 
 [source,console]
 --------------------------------------------------
@@ -72,7 +113,7 @@ The API returns the following response:
 --------------------------------------------------
 {
   "is_safe": false,
-  "message": "cluster health is RED",
+  "message": "removal of the following nodes might not be safe: [node2-id]",
   "nodes": [
     {
       "id": "node1-id",
@@ -80,6 +121,7 @@ The API returns the following response:
       "external_id" : "node1-externalId",
       "result" : {
         "is_safe": true,
+        "reason": "no_red_shards_on_node",
         "message": ""
       }
     },
@@ -89,7 +131,8 @@ The API returns the following response:
       "external_id" : "node2-externalId",
       "result" : {
         "is_safe": false,
-        "message": "node may contain a copy of a red index shard"
+        "reason": "red_shards_on_node",
+        "message": "node contains copies of the following red shards: [[indexName][0]]"
       }
     }
   ]
diff --git a/docs/reference/commands/cli-jvm-options.asciidoc b/docs/reference/commands/cli-jvm-options.asciidoc
new file mode 100644
index 000000000000..74af2c81d8d4
--- /dev/null
+++ b/docs/reference/commands/cli-jvm-options.asciidoc
@@ -0,0 +1,13 @@
+[[cli-tool-jvm-options-{tool-name}]]
+[float]
+==== JVM options
+
+CLI tools run with 64MB of heap. For most tools, this value is fine. However, if needed
+this can be overriden by setting the CLI_JAVA_OPTS environment variable. For example,
+the following increases the heap size used by the `pass:a[{tool-name}]` tool to 1GB.
+
+[source,shell,subs=attributes+]
+--------------------------------------------------
+export CLI_JAVA_OPTS="-Xmx1g"
+bin/elasticsearch-{tool-name} ...
+--------------------------------------------------
diff --git a/docs/reference/commands/node-tool.asciidoc b/docs/reference/commands/node-tool.asciidoc
index 4582d9b2b7ca..e36ed42f5199 100644
--- a/docs/reference/commands/node-tool.asciidoc
+++ b/docs/reference/commands/node-tool.asciidoc
@@ -50,6 +50,10 @@ This tool has a number of modes:
   {es}. This may sometimes allow you to downgrade to an earlier version of
   {es}.
 
+:tool-name: node
+include::cli-jvm-options.asciidoc[]
+:!tool-name:
+
 [[node-tool-repurpose]]
 [discrete]
 ==== Changing the role of a node
@@ -281,11 +285,13 @@ Unsafe cluster bootstrapping is only possible if there is at least one
 surviving master-eligible node. If there are no remaining master-eligible nodes
 then the cluster metadata is completely lost. However, the individual data
 nodes also contain a copy of the index metadata corresponding with their
-shards. This sometimes allows a new cluster to import these shards as
-<<modules-gateway-dangling-indices,dangling indices>>. You can sometimes
-recover some indices after the loss of all master-eligible nodes in a cluster
-by creating a new cluster and then using the `elasticsearch-node
-detach-cluster` command to move any surviving nodes into this new cluster.
+shards. It is therefore sometimes possible to manually import these shards as
+<<dangling-indices,dangling indices>>. For example you can sometimes recover some
+indices after the loss of all master-eligible nodes in a cluster by creating a new
+cluster and then using the `elasticsearch-node detach-cluster` command to move any
+surviving nodes into this new cluster. Once the new cluster is fully formed,
+use the <<dangling-indices-api,Dangling indices API>> to list, import or delete
+any dangling indices.
 
 There is a risk of data loss when importing a dangling index because data nodes
 may not have the most recent copy of the index metadata and do not have any
@@ -314,7 +320,9 @@ cluster`.
 <<modules-discovery-hosts-providers,discover the new cluster>>.
 6. Start each data node and verify that it has joined the new cluster.
 7. Wait for all recoveries to have completed, and investigate the data in the
-cluster to discover if any was lost during this process.
+cluster to discover if any was lost during this process. Use the
+<<dangling-indices-api,Dangling indices API>> to list, import or delete any
+dangling indices.
 
 The message `Node was successfully detached from the cluster` does not mean
 that there has been no data loss, it just means that tool was able to complete
diff --git a/docs/reference/commands/reconfigure-node.asciidoc b/docs/reference/commands/reconfigure-node.asciidoc
index f06aee9f94db..838de8909bda 100644
--- a/docs/reference/commands/reconfigure-node.asciidoc
+++ b/docs/reference/commands/reconfigure-node.asciidoc
@@ -28,7 +28,7 @@ cluster where security features are already enabled and configured. Before
 starting your new node, run the 
 <<create-enrollment-token,`elasticsearch-create-enrollment-token`>> tool
 with the `-s node` option to generate an enrollment token on any node in your
-existing cluster. On your new node, run the the
+existing cluster. On your new node, run the
 `elasticsearch-reconfigure-node` tool and pass the enrollment token as a
 parameter.
 
@@ -54,6 +54,11 @@ nodes in an existing, secured cluster.
 
 `-v, --verbose`:: Shows verbose output.
 
+
+:tool-name: reconfigure-node
+include::cli-jvm-options.asciidoc[]
+:!tool-name:
+
 [discrete]
 === Examples
 
diff --git a/docs/reference/commands/shard-tool.asciidoc b/docs/reference/commands/shard-tool.asciidoc
index 7dfdadbf4e2a..a2d9d557adf5 100644
--- a/docs/reference/commands/shard-tool.asciidoc
+++ b/docs/reference/commands/shard-tool.asciidoc
@@ -44,6 +44,10 @@ There are two ways to specify the path:
 * Use the `--dir` option to specify the full path to the corrupted index or
   translog files.
 
+:tool-name: shard
+include::cli-jvm-options.asciidoc[]
+:!tool-name:
+
 [discrete]
 ==== Removing corrupted data
 
diff --git a/docs/reference/data-streams/downsampling-ilm.asciidoc b/docs/reference/data-streams/downsampling-ilm.asciidoc
index 938d773bdec2..32499fcad0d6 100644
--- a/docs/reference/data-streams/downsampling-ilm.asciidoc
+++ b/docs/reference/data-streams/downsampling-ilm.asciidoc
@@ -4,8 +4,6 @@
 <titleabbrev>Run downsampling with ILM</titleabbrev>
 ++++
 
-preview::[]
-
 This is a simplified example that allows you to see quickly how
 <<downsampling,downsampling>> works as part of an ILM policy to reduce the
 storage size of a sampled set of metrics. The example uses typical Kubernetes
@@ -38,7 +36,7 @@ To enable downsampling, add a <<ilm-downsample,Downsample action>> and set
 <<ilm-downsample-options,`fixed_interval`>> to the downsampling interval at
 which you want to aggregate the original time series data.
 
-In this example, an ILM policy is configired for the `hot` phase. The downsample
+In this example, an ILM policy is configured for the `hot` phase. The downsample
 takes place after the initial index rollover, which for demonstration
 purposes is set to run after five minutes.
 
@@ -292,7 +290,8 @@ GET _data_stream
 If the ILM policy has not yet been applied, your results will be like the
 following. Note the original `index_name`: `.ds-datastream-<timestamp>-000001`.
 
-```
+[source,console-result]
+----
 {
   "data_streams": [
     {
@@ -329,7 +328,9 @@ following. Note the original `index_name`: `.ds-datastream-<timestamp>-000001`.
     }
   ]
 }
-```
+----
+// TEST[skip:todo]
+// TEST[continued]
 
 Next, run a search query:
 
@@ -341,7 +342,8 @@ GET datastream/_search
 
 The query returns your ten newly added documents.
 
-```
+[source,console-result]
+----
 {
   "took": 17,
   "timed_out": false,
@@ -357,7 +359,9 @@ The query returns your ten newly added documents.
       "relation": "eq"
     },
 ...
-```
+----
+// TEST[skip:todo]
+// TEST[continued]
 
 By default, index lifecycle management checks every ten minutes for indices that
 meet policy criteria. Wait for about ten minutes (maybe brew up a quick coffee
@@ -373,7 +377,8 @@ After the ILM policy has taken effect, the original
 `.ds-datastream-2022.08.26-000001` index is replaced with a new, downsampled
 index, in this case `downsample-6tkn-.ds-datastream-2022.08.26-000001`.
 
-```
+[source,console-result]
+----
 {
   "data_streams": [
     {
@@ -392,7 +397,9 @@ index, in this case `downsample-6tkn-.ds-datastream-2022.08.26-000001`.
         }
       ],
 ...
-```
+----
+// TEST[skip:todo]
+// TEST[continued]
 
 Run a search query on the datastream.
 
@@ -400,13 +407,14 @@ Run a search query on the datastream.
 ----
 GET datastream/_search
 ----
-// TEST[skip: The @timestamp value won't match an accepted range in the TSDS]
+// TEST[continued]
 
 The new downsampled index contains just one document that includes the `min`,
 `max`, `sum`, and `value_count` statistics based off of the original sampled
 metrics.
 
-```
+[source,console-result]
+----
 {
   "took": 6,
   "timed_out": false,
@@ -483,7 +491,9 @@ metrics.
     ]
   }
 }
-```
+----
+// TEST[skip:todo]
+// TEST[continued]
 
 Use the <<data-stream-stats-api,data stream stats API>> to get statistics for
 the data stream, including the storage size.
@@ -492,9 +502,10 @@ the data stream, including the storage size.
 ----
 GET /_data_stream/datastream/_stats?human=true
 ----
-// TEST[skip: The @timestamp value won't match an accepted range in the TSDS]
+// TEST[continued]
 
-```
+[source,console-result]
+----
 {
   "_shards": {
     "total": 4,
@@ -515,7 +526,9 @@ GET /_data_stream/datastream/_stats?human=true
     }
   ]
 }
-```
+----
+// TEST[skip:todo]
+// TEST[continued]
 
 This example demonstrates how downsampling works as part of an ILM policy to
 reduce the storage size of metrics data as it becomes less current and less
diff --git a/docs/reference/data-streams/downsampling-manual.asciidoc b/docs/reference/data-streams/downsampling-manual.asciidoc
index 311a02eb4868..475c57895fcc 100644
--- a/docs/reference/data-streams/downsampling-manual.asciidoc
+++ b/docs/reference/data-streams/downsampling-manual.asciidoc
@@ -4,8 +4,6 @@
 <titleabbrev>Run downsampling manually</titleabbrev>
 ++++
 
-preview::[]
-
 This is a simplified example that allows you to see quickly how
 <<downsampling,downsampling>> works to reduce the storage size of a time series
 index. The example uses typical Kubernetes cluster monitoring data. To test out
@@ -200,7 +198,6 @@ PUT /sample-01
 }
 
 ----
-// TEST
 
 [discrete]
 [[downsampling-manual-ingest-data]]
@@ -209,14 +206,13 @@ PUT /sample-01
 In a terminal window with {es} running, run the following curl command to load
 the documents from the downloaded sample data file:
 
-//[source,console]
-//----
-```
+[source,sh]
+----
 curl -s -H "Content-Type: application/json" \
    -XPOST http://<elasticsearch-node>/sample-01/_bulk?pretty \
    --data-binary @sample-k8s-metrics.json
-```
-//----
+----
+// NOTCONSOLE
 
 Approximately 18,000 documents are added. Check the search results for the newly
 ingested data:
@@ -227,11 +223,12 @@ GET /sample-01*/_search
 ----
 // TEST[continued]
 
-The query should return the first 10,000 hits. In each document you can see the
-time series dimensions (`host`, `node`, `pod` and `container`) as well as the
-various CPU and memory time series metrics.
+The query has at least 10,000 hits and returns the first 10. In each document
+you can see the time series dimensions (`host`, `node`, `pod` and `container`)
+as well as the various CPU and memory time series metrics.
 
-```
+[source,console-result]
+----
   "hits": {
     "total": {
       "value": 10000,
@@ -294,7 +291,9 @@ various CPU and memory time series metrics.
         }
       }
 ...
-```
+----
+// TEST[skip:todo]
+// TEST[continued]
 
 Next, run a terms aggregation on the set of time series dimensions (`_tsid`) to
 create a date histogram on a fixed interval of one day.
@@ -393,11 +392,12 @@ GET /sample-01*/_search
 ----
 // TEST[continued]
 
-In the query results, notice that the numer of hits has been reduced to only 288
+In the query results, notice that the number of hits has been reduced to only 288
 documents. As well, for each time series metric statistical representations have
 been calculated: `min`, `max`, `sum`, and `value_count`.
 
-```
+[source,console-result]
+----
   "hits": {
     "total": {
       "value": 288,
@@ -455,7 +455,8 @@ been calculated: `min`, `max`, `sum`, and `value_count`.
         }
       },
 ...
-```
+----
+// TEST[skip:todo]
 
 This example demonstrates how downsampling can dramatically reduce the number of
 records stored for time series data, within whatever time boundaries you choose.
diff --git a/docs/reference/data-streams/downsampling.asciidoc b/docs/reference/data-streams/downsampling.asciidoc
index d70cbb1da6ba..99273e1a6854 100644
--- a/docs/reference/data-streams/downsampling.asciidoc
+++ b/docs/reference/data-streams/downsampling.asciidoc
@@ -1,8 +1,6 @@
 [[downsampling]]
 === Downsampling a time series data stream
 
-preview::[]
-
 Downsampling provides a method to reduce the footprint of your <<tsds,time
 series data>> by storing it at reduced granularity.
 
@@ -72,18 +70,14 @@ To downsample a time series index, use the
 <<indices-downsample-data-stream,Downsample API>> and set `fixed_interval` to
 the level of granularity that you'd like:
 
-```
-POST /<source_index>/_downsample/<new_index>
-{
-    "fixed_interval": "1d"
-}
-```
+include::../indices/downsample-data-stream.asciidoc[tag=downsample-example]
 
 To downsample time series data as part of ILM, include a
 <<ilm-downsample,Downsample action>> in your ILM policy and set `fixed_interval`
 to the level of granularity that you'd like:
 
-```
+[source,console]
+----
 PUT _ilm/policy/my_policy
 {
   "policy": {
@@ -98,7 +92,7 @@ PUT _ilm/policy/my_policy
     }
   }
 }
-```
+----
 
 [discrete]
 [[querying-downsampled-indices]]
diff --git a/docs/reference/data-streams/set-up-tsds.asciidoc b/docs/reference/data-streams/set-up-tsds.asciidoc
index bee0d40ece4a..3c15011871f8 100644
--- a/docs/reference/data-streams/set-up-tsds.asciidoc
+++ b/docs/reference/data-streams/set-up-tsds.asciidoc
@@ -4,8 +4,6 @@
 <titleabbrev>Set up a TSDS</titleabbrev>
 ++++
 
-preview::[]
-
 To set up a <<tsds,time series data stream (TSDS)>>, follow these steps:
 
 . Check the <<tsds-prereqs,prerequisites>>.
diff --git a/docs/reference/data-streams/tsds-index-settings.asciidoc b/docs/reference/data-streams/tsds-index-settings.asciidoc
index dea828091c8f..8512a3841b37 100644
--- a/docs/reference/data-streams/tsds-index-settings.asciidoc
+++ b/docs/reference/data-streams/tsds-index-settings.asciidoc
@@ -1,8 +1,6 @@
 [[tsds-index-settings]]
 === Time series index settings
 
-preview::[]
-
 Backing indices in a <<tsds,time series data stream (TSDS)>> support the
 following index settings.
 
diff --git a/docs/reference/data-streams/tsds.asciidoc b/docs/reference/data-streams/tsds.asciidoc
index a454e1652e39..d94754d1f0d8 100644
--- a/docs/reference/data-streams/tsds.asciidoc
+++ b/docs/reference/data-streams/tsds.asciidoc
@@ -1,12 +1,9 @@
 [[tsds]]
 == Time series data stream (TSDS)
 
-preview::[]
-
 A time series data stream (TSDS) models timestamped metrics data as one or
 more time series.
 
-// TODO: Replace XX% with actual percentage
 You can use a TSDS to store metrics data more efficiently. In our benchmarks,
 metrics data stored in a TSDS used 44% less disk space than a regular data
 stream.
@@ -114,7 +111,8 @@ always a short encoded hash. To prevent the `_tsid` value from being overly
 large, {es} limits the number of dimensions for an index using the
 <<index-mapping-dimension-fields-limit,`index.mapping.dimension_fields.limit`>>
 index setting. While you can increase this limit, the resulting document `_tsid`
-value can't exceed 32KB.
+value can't exceed 32KB. Additionally the field name of a dimension cannot be
+longer than 512 bytes and the each dimension value can't exceed 1kb.
 ****
 
 [discrete]
@@ -189,6 +187,9 @@ document's `_tsid` using a <<docs-get,get document>> request. However, you can
 use the `_tsid` field in aggregations and retrieve the `_tsid` value in searches
 using the <<search-fields-param,`fields` parameter>>.
 
+WARNING: The format of the `_tsid` field shouldn't be relied upon. It may change
+from version to version.
+
 [discrete]
 [[time-bound-indices]]
 ==== Time-bound indices
diff --git a/docs/reference/docs/bulk.asciidoc b/docs/reference/docs/bulk.asciidoc
index 15814efde309..d90026073990 100644
--- a/docs/reference/docs/bulk.asciidoc
+++ b/docs/reference/docs/bulk.asciidoc
@@ -49,6 +49,9 @@ privilege.
 ** To automatically create a data stream or index with a bulk API request, you
 must have the `auto_configure`, `create_index`, or `manage` index privilege.
 
+** To make the result of a bulk operation visible to search using the `refresh`
+parameter, you must have the `maintenance` or `manage` index privilege.
+
 * Automatic data stream creation requires a matching index template with data
 stream enabled. See <<set-up-a-data-stream>>.
 
diff --git a/docs/reference/docs/delete-by-query.asciidoc b/docs/reference/docs/delete-by-query.asciidoc
index fc8e67ea0465..d671cb9b5a63 100644
--- a/docs/reference/docs/delete-by-query.asciidoc
+++ b/docs/reference/docs/delete-by-query.asciidoc
@@ -195,8 +195,6 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards]
 +
 Defaults to `open`.
 
-include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=from]
-
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable]
 
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=lenient]
diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index 4927c84fff85..5f8b1fcc7f0c 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -141,8 +141,6 @@ If the request contains `wait_for_completion=false`, {es}
 performs some preflight checks, launches the request, and returns a
 <<tasks,`task`>> you can use to cancel or get the status of the task.
 {es} creates a record of this task as a document at `_tasks/<task_id>`.
-When you are done with a task, you should delete the task document so
-{es} can reclaim the space.
 
 [[docs-reindex-from-multiple-sources]]
 ===== Reindex from multiple sources
@@ -577,6 +575,9 @@ Valid values: `index`, `create`. Defaults to `index`.
 IMPORTANT: To reindex to a data stream destination, this argument must be
 `create`.
 
+`pipeline`:::
+(Optional, string) the name of the <<reindex-with-an-ingest-pipeline,pipeline>> to use.
+
 `script`::
 `source`:::
 (Optional, string) The script to run to update the document source or metadata when reindexing.
diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc
index cd27ec369d25..5ef9d288623c 100644
--- a/docs/reference/docs/update-by-query.asciidoc
+++ b/docs/reference/docs/update-by-query.asciidoc
@@ -99,8 +99,6 @@ If the request contains `wait_for_completion=false`, {es}
 performs some preflight checks, launches the request, and returns a
 <<tasks,`task`>> you can use to cancel or get the status of the task.
 {es} creates a record of this task as a document at `.tasks/task/${taskId}`.
-When you are done with a task, you should delete the task document so
-{es} can reclaim the space.
 
 ===== Waiting for active shards
 
@@ -189,8 +187,6 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards]
 +
 Defaults to `open`.
 
-include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=from]
-
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable]
 
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=lenient]
diff --git a/docs/reference/eql/eql.asciidoc b/docs/reference/eql/eql.asciidoc
index 7b3234fe2464..f2191b8402c1 100644
--- a/docs/reference/eql/eql.asciidoc
+++ b/docs/reference/eql/eql.asciidoc
@@ -30,10 +30,16 @@ describe activity that goes beyond IOCs.
 [[eql-required-fields]]
 === Required fields
 
-To run an EQL search, the searched data stream or index must contain a
-_timestamp_ and _event category_ field. By default, EQL uses the `@timestamp`
-and `event.category` fields from the {ecs-ref}[Elastic Common Schema
-(ECS)]. To use a different timestamp or event category field, see
+With the exception of sample queries, EQL searches require that the searched
+data stream or index  contains a _timestamp_ field. By default, EQL uses the
+`@timestamp` field from the {ecs-ref}[Elastic Common Schema (ECS)]. 
+
+EQL searches also require an _event category_ field, unless you use the
+<<eql-syntax-match-any-event-category,`any` keyword>> to search for  documents
+without an event category field. By default, EQL uses the ECS `event.category`
+field.
+
+To use a different timestamp or event category field, see
 <<specify-a-timestamp-or-event-category-field>>.
 
 TIP: While no schema is required to use EQL, we recommend using the
@@ -314,6 +320,561 @@ GET /my-data-stream/_eql/search
 ----
 // TEST[setup:sec_logs]
 
+[discrete]
+[[eql-search-sample]]
+=== Sample chronologically unordered events
+
+Use EQL's <<eql-samples,sample syntax>> to search for events that match one or
+more join keys and a set of filters. Samples are similar to sequences, but do
+not return events in chronological order. In fact, sample queries can run on
+data without a timestamp. Sample queries can be useful to find correlations in
+events that don't always occur in the same sequence, or that occur across long
+time spans.
+
+.Click to show the sample data used in the examples below
+[%collapsible]
+====
+[source,console]
+----
+PUT /my-index-000001
+{
+    "mappings": {
+        "properties": {
+            "ip": {
+                "type":"ip"
+            },
+            "version": {
+                "type": "version"
+            },
+            "missing_keyword": {
+                "type": "keyword"
+            },
+            "@timestamp": {
+              "type": "date"
+            },
+            "type_test": {
+                "type": "keyword"
+            },
+            "@timestamp_pretty": {
+              "type": "date",
+              "format": "dd-MM-yyyy"
+            },
+            "event_type": {
+              "type": "keyword"
+            },
+            "event": {
+              "properties": {
+                "category": {
+                  "type": "alias",
+                  "path": "event_type"
+                }
+              }
+            },
+            "host": {
+              "type": "keyword"
+            },
+            "os": {
+              "type": "keyword"
+            },
+            "bool": {
+              "type": "boolean"
+            },
+            "uptime" : {
+              "type" : "long"
+            },
+            "port" : {
+              "type" : "long"
+            }
+        }
+    }
+}
+
+PUT /my-index-000002
+{
+    "mappings": {
+        "properties": {
+            "ip": {
+                "type":"ip"
+            },
+            "@timestamp": {
+              "type": "date"
+            },
+            "@timestamp_pretty": {
+              "type": "date",
+              "format": "yyyy-MM-dd"
+            },
+            "type_test": {
+                "type": "keyword"
+            },
+            "event_type": {
+              "type": "keyword"
+            },
+            "event": {
+              "properties": {
+                "category": {
+                  "type": "alias",
+                  "path": "event_type"
+                }
+              }
+            },
+            "host": {
+              "type": "keyword"
+            },
+            "op_sys": {
+              "type": "keyword"
+            },
+            "bool": {
+              "type": "boolean"
+            },
+            "uptime" : {
+              "type" : "long"
+            },
+            "port" : {
+              "type" : "long"
+            }
+        }
+    }
+}
+
+PUT /my-index-000003
+{
+    "mappings": {
+        "properties": {
+            "host_ip": {
+                "type":"ip"
+            },
+            "@timestamp": {
+              "type": "date"
+            },
+            "date": {
+              "type": "date"
+            },
+            "event_type": {
+              "type": "keyword"
+            },
+            "event": {
+              "properties": {
+                "category": {
+                  "type": "alias",
+                  "path": "event_type"
+                }
+              }
+            },
+            "missing_keyword": {
+                "type": "keyword"
+            },
+            "host": {
+              "type": "keyword"
+            },
+            "os": {
+              "type": "keyword"
+            },
+            "bool": {
+              "type": "boolean"
+            },
+            "uptime" : {
+              "type" : "long"
+            },
+            "port" : {
+              "type" : "long"
+            }
+        }
+    }
+}
+
+POST /my-index-000001/_bulk?refresh
+{"index":{"_id":1}}
+{"@timestamp":"1234567891","@timestamp_pretty":"12-12-2022","missing_keyword":"test","type_test":"abc","ip":"10.0.0.1","event_type":"alert","host":"doom","uptime":0,"port":1234,"os":"win10","version":"1.0.0","id":11}
+{"index":{"_id":2}}
+{"@timestamp":"1234567892","@timestamp_pretty":"13-12-2022","event_type":"alert","type_test":"abc","host":"CS","uptime":5,"port":1,"os":"win10","version":"1.2.0","id":12}
+{"index":{"_id":3}}
+{"@timestamp":"1234567893","@timestamp_pretty":"12-12-2022","event_type":"alert","type_test":"abc","host":"farcry","uptime":1,"port":1234,"bool":false,"os":"win10","version":"2.0.0","id":13}
+{"index":{"_id":4}}
+{"@timestamp":"1234567894","@timestamp_pretty":"13-12-2022","event_type":"alert","type_test":"abc","host":"GTA","uptime":3,"port":12,"os":"slack","version":"10.0.0","id":14}
+{"index":{"_id":5}}
+{"@timestamp":"1234567895","@timestamp_pretty":"17-12-2022","event_type":"alert","host":"sniper 3d","uptime":6,"port":1234,"os":"fedora","version":"20.1.0","id":15}
+{"index":{"_id":6}}
+{"@timestamp":"1234568896","@timestamp_pretty":"17-12-2022","event_type":"alert","host":"doom","port":65123,"bool":true,"os":"redhat","version":"20.10.0","id":16}
+{"index":{"_id":7}}
+{"@timestamp":"1234567897","@timestamp_pretty":"17-12-2022","missing_keyword":"yyy","event_type":"failure","host":"doom","uptime":15,"port":1234,"bool":true,"os":"redhat","version":"20.2.0","id":17}
+{"index":{"_id":8}}
+{"@timestamp":"1234567898","@timestamp_pretty":"12-12-2022","missing_keyword":"test","event_type":"success","host":"doom","uptime":16,"port":512,"os":"win10","version":"1.2.3","id":18}
+{"index":{"_id":9}}
+{"@timestamp":"1234567899","@timestamp_pretty":"15-12-2022","missing_keyword":"test","event_type":"success","host":"GTA","port":12,"bool":true,"os":"win10","version":"1.2.3","id":19}
+{"index":{"_id":10}}
+{"@timestamp":"1234567893","missing_keyword":null,"ip":"10.0.0.5","event_type":"alert","host":"farcry","uptime":1,"port":1234,"bool":true,"os":"win10","version":"1.2.3","id":110}
+
+POST /my-index-000002/_bulk?refresh
+{"index":{"_id":1}}
+{"@timestamp":"1234567991","type_test":"abc","ip":"10.0.0.1","event_type":"alert","host":"doom","uptime":0,"port":1234,"op_sys":"win10","id":21}
+{"index":{"_id":2}}
+{"@timestamp":"1234567992","type_test":"abc","event_type":"alert","host":"CS","uptime":5,"port":1,"op_sys":"win10","id":22}
+{"index":{"_id":3}}
+{"@timestamp":"1234567993","type_test":"abc","@timestamp_pretty":"2022-12-17","event_type":"alert","host":"farcry","uptime":1,"port":1234,"bool":false,"op_sys":"win10","id":23}
+{"index":{"_id":4}}
+{"@timestamp":"1234567994","event_type":"alert","host":"GTA","uptime":3,"port":12,"op_sys":"slack","id":24}
+{"index":{"_id":5}}
+{"@timestamp":"1234567995","event_type":"alert","host":"sniper 3d","uptime":6,"port":1234,"op_sys":"fedora","id":25}
+{"index":{"_id":6}}
+{"@timestamp":"1234568996","@timestamp_pretty":"2022-12-17","ip":"10.0.0.5","event_type":"alert","host":"doom","port":65123,"bool":true,"op_sys":"redhat","id":26}
+{"index":{"_id":7}}
+{"@timestamp":"1234567997","@timestamp_pretty":"2022-12-17","event_type":"failure","host":"doom","uptime":15,"port":1234,"bool":true,"op_sys":"redhat","id":27}
+{"index":{"_id":8}}
+{"@timestamp":"1234567998","ip":"10.0.0.1","event_type":"success","host":"doom","uptime":16,"port":512,"op_sys":"win10","id":28}
+{"index":{"_id":9}}
+{"@timestamp":"1234567999","ip":"10.0.0.1","event_type":"success","host":"GTA","port":12,"bool":false,"op_sys":"win10","id":29}
+
+POST /my-index-000003/_bulk?refresh
+{"index":{"_id":1}}
+{"@timestamp":"1334567891","host_ip":"10.0.0.1","event_type":"alert","host":"doom","uptime":0,"port":12,"os":"win10","id":31}
+{"index":{"_id":2}}
+{"@timestamp":"1334567892","event_type":"alert","host":"CS","os":"win10","id":32}
+{"index":{"_id":3}}
+{"@timestamp":"1334567893","event_type":"alert","host":"farcry","bool":true,"os":"win10","id":33}
+{"index":{"_id":4}}
+{"@timestamp":"1334567894","event_type":"alert","host":"GTA","os":"slack","bool":true,"id":34}
+{"index":{"_id":5}}
+{"@timestamp":"1234567895","event_type":"alert","host":"sniper 3d","os":"fedora","id":35}
+{"index":{"_id":6}}
+{"@timestamp":"1234578896","host_ip":"10.0.0.1","event_type":"alert","host":"doom","bool":true,"os":"redhat","id":36}
+{"index":{"_id":7}}
+{"@timestamp":"1234567897","event_type":"failure","missing_keyword":"test","host":"doom","bool":true,"os":"redhat","id":37}
+{"index":{"_id":8}}
+{"@timestamp":"1234577898","event_type":"success","host":"doom","os":"win10","id":38,"date":"1671235200000"}
+{"index":{"_id":9}}
+{"@timestamp":"1234577899","host_ip":"10.0.0.5","event_type":"success","host":"GTA","bool":true,"os":"win10","id":39}
+----
+====
+
+A sample query specifies at least one join key, using the <<eql-by-keyword,`by`
+keyword>>, and up to five filters:
+
+[source,console]
+----
+GET /my-index*/_eql/search
+{
+  "query": """
+    sample by host
+      [any where uptime > 0] 
+      [any where port > 100] 
+      [any where bool == true] 
+  """
+}
+----
+// TEST[continued]
+
+By default, the response’s `hits.sequences` property contains up to 10 samples.
+Each sample has a set of `join_keys` and an array with one matching event for
+each of the filters. Events are returned in the order of the filters they match:
+
+[source,console-result]
+----
+{
+  ...
+  "hits": {
+    "total": {
+      "value": 2,
+      "relation": "eq"
+    },
+    "sequences": [
+      {
+        "join_keys": [
+          "doom"                                      <1>
+        ],
+        "events": [
+          {                                           <2>
+            "_index": "my-index-000001",
+            "_id": "7",
+            "_source": {
+              "@timestamp": "1234567897",
+              "@timestamp_pretty": "17-12-2022",
+              "missing_keyword": "yyy",
+              "event_type": "failure",
+              "host": "doom",
+              "uptime": 15,
+              "port": 1234,
+              "bool": true,
+              "os": "redhat",
+              "version": "20.2.0",
+              "id": 17
+            }
+          },
+          {                                           <3>
+            "_index": "my-index-000001",
+            "_id": "1",
+            "_source": {
+              "@timestamp": "1234567891",
+              "@timestamp_pretty": "12-12-2022",
+              "missing_keyword": "test",
+              "type_test": "abc",
+              "ip": "10.0.0.1",
+              "event_type": "alert",
+              "host": "doom",
+              "uptime": 0,
+              "port": 1234,
+              "os": "win10",
+              "version": "1.0.0",
+              "id": 11
+            }
+          },
+          {                                           <4>
+            "_index": "my-index-000001",
+            "_id": "6",
+            "_source": {
+              "@timestamp": "1234568896",
+              "@timestamp_pretty": "17-12-2022",
+              "event_type": "alert",
+              "host": "doom",
+              "port": 65123,
+              "bool": true,
+              "os": "redhat",
+              "version": "20.10.0",
+              "id": 16
+            }
+          }
+        ]
+      },
+      {
+        "join_keys": [
+          "farcry"                                    <5>
+        ],
+        "events": [
+          {
+            "_index": "my-index-000001",
+            "_id": "3",
+            "_source": {
+              "@timestamp": "1234567893",
+              "@timestamp_pretty": "12-12-2022",
+              "event_type": "alert",
+              "type_test": "abc",
+              "host": "farcry",
+              "uptime": 1,
+              "port": 1234,
+              "bool": false,
+              "os": "win10",
+              "version": "2.0.0",
+              "id": 13
+            }
+          },
+          {
+            "_index": "my-index-000001",
+            "_id": "10",
+            "_source": {
+              "@timestamp": "1234567893",
+              "missing_keyword": null,
+              "ip": "10.0.0.5",
+              "event_type": "alert",
+              "host": "farcry",
+              "uptime": 1,
+              "port": 1234,
+              "bool": true,
+              "os": "win10",
+              "version": "1.2.3",
+              "id": 110
+            }
+          },
+          {
+            "_index": "my-index-000003",
+            "_id": "3",
+            "_source": {
+              "@timestamp": "1334567893",
+              "event_type": "alert",
+              "host": "farcry",
+              "bool": true,
+              "os": "win10",
+              "id": 33
+            }
+          }
+        ]
+      }
+    ]
+  }
+}
+----
+// TESTRESPONSE[s/  \.\.\.\n/"is_partial": false, "is_running": false, "took": $body.took, "timed_out": false,/]
+
+<1> The events in the first sample have a value of `doom` for `host`.
+<2> This event matches the first filter.
+<3> This event matches the second filter.
+<4> This event matches the third filter.
+<5> The events in the second sample have a value of `farcry` for `host`.
+
+You can specify multiple join keys:
+
+[source,console]
+----
+GET /my-index*/_eql/search
+{
+  "query": """
+    sample by host
+      [any where uptime > 0]   by os
+      [any where port > 100]   by op_sys
+      [any where bool == true] by os
+  """
+}
+----
+// TEST[continued]
+
+This query will return samples where each of the events shares the same value
+for `os` or `op_sys`, as well as for `host`. For example:
+
+[source,console-result]
+----
+{
+  ...
+  "hits": {
+    "total": {
+      "value": 2,
+      "relation": "eq"
+    },
+    "sequences": [
+      {
+        "join_keys": [
+          "doom",                                      <1>
+          "redhat"
+        ],
+        "events": [
+          {
+            "_index": "my-index-000001",
+            "_id": "7",
+            "_source": {
+              "@timestamp": "1234567897",
+              "@timestamp_pretty": "17-12-2022",
+              "missing_keyword": "yyy",
+              "event_type": "failure",
+              "host": "doom",
+              "uptime": 15,
+              "port": 1234,
+              "bool": true,
+              "os": "redhat",
+              "version": "20.2.0",
+              "id": 17
+            }
+          },
+          {
+            "_index": "my-index-000002",
+            "_id": "6",
+            "_source": {
+              "@timestamp": "1234568996",
+              "@timestamp_pretty": "2022-12-17",
+              "ip": "10.0.0.5",
+              "event_type": "alert",
+              "host": "doom",
+              "port": 65123,
+              "bool": true,
+              "op_sys": "redhat",
+              "id": 26
+            }
+          },
+          {
+            "_index": "my-index-000001",
+            "_id": "6",
+            "_source": {
+              "@timestamp": "1234568896",
+              "@timestamp_pretty": "17-12-2022",
+              "event_type": "alert",
+              "host": "doom",
+              "port": 65123,
+              "bool": true,
+              "os": "redhat",
+              "version": "20.10.0",
+              "id": 16
+            }
+          }
+        ]
+      },
+      {
+        "join_keys": [
+          "farcry",
+          "win10"
+        ],
+        "events": [
+          {
+            "_index": "my-index-000001",
+            "_id": "3",
+            "_source": {
+              "@timestamp": "1234567893",
+              "@timestamp_pretty": "12-12-2022",
+              "event_type": "alert",
+              "type_test": "abc",
+              "host": "farcry",
+              "uptime": 1,
+              "port": 1234,
+              "bool": false,
+              "os": "win10",
+              "version": "2.0.0",
+              "id": 13
+            }
+          },
+          {
+            "_index": "my-index-000002",
+            "_id": "3",
+            "_source": {
+              "@timestamp": "1234567993",
+              "type_test": "abc",
+              "@timestamp_pretty": "2022-12-17",
+              "event_type": "alert",
+              "host": "farcry",
+              "uptime": 1,
+              "port": 1234,
+              "bool": false,
+              "op_sys": "win10",
+              "id": 23
+            }
+          },
+          {
+            "_index": "my-index-000001",
+            "_id": "10",
+            "_source": {
+              "@timestamp": "1234567893",
+              "missing_keyword": null,
+              "ip": "10.0.0.5",
+              "event_type": "alert",
+              "host": "farcry",
+              "uptime": 1,
+              "port": 1234,
+              "bool": true,
+              "os": "win10",
+              "version": "1.2.3",
+              "id": 110
+            }
+          }
+        ]
+      }
+    ]
+  }
+}
+----
+// TESTRESPONSE[s/  \.\.\.\n/"is_partial": false, "is_running": false, "took": $body.took, "timed_out": false,/]
+
+<1> The events in this sample have a value of `doom` for `host` and a value of
+`redhat` for `os` or `op_sys`.
+
+By default, the response of a sample query contains up to 10 samples, with one
+sample per unique set of join keys. Use the `size` parameter to get a smaller or
+larger set of samples. To retrieve more than one sample per set of join keys,
+use the `max_samples_per_key` parameter. Pipes are not supported for sample
+queries. 
+
+[source,console]
+----
+GET /my-index*/_eql/search
+{
+  "max_samples_per_key": 2,     <1>
+  "size": 20,                   <2>
+  "query": """
+    sample
+      [any where uptime > 0]   by host,os
+      [any where port > 100]   by host,op_sys
+      [any where bool == true] by host,os
+  """
+}
+----
+// TEST[continued]
+
+<1> Retrieve up to 2 samples per set of join keys.
+<2> Retrieve up to 20 samples in total.
+
 [discrete]
 [[retrieve-selected-fields]]
 === Retrieve selected fields
diff --git a/docs/reference/eql/syntax.asciidoc b/docs/reference/eql/syntax.asciidoc
index 8fd06ad893c0..1893f6d82e6c 100644
--- a/docs/reference/eql/syntax.asciidoc
+++ b/docs/reference/eql/syntax.asciidoc
@@ -792,6 +792,47 @@ sequence
   [ library where process.name == "regsvr32.exe" ] by dll.path with runs=3
 ----
 
+[discrete]
+[[eql-samples]]
+=== Samples
+
+You can use EQL samples to describe and match a chronologically unordered series
+of events. All events in a sample share the same value for one or more fields
+that are specified using the <<eql-by-keyword,`by` keyword>> (join keys). Each
+item in a sample is an event category and event condition, surrounded by square
+brackets (`[ ]`). Events are listed in the order of the filters they match.
+
+[source,eql]
+----
+sample by join_key
+  [ event_category_1 where condition_1 ]
+  [ event_category_2 where condition_2 ]
+  ...
+----
+
+*Example* + 
+The following EQL sample query returns up to 10 samples with unique values for
+`host`. Each sample consists of two events:
+
+. Start with an event with:
++
+--
+* An event category of `file`
+* A `file.extension` of `exe`
+--
+. Followed by an event with an event category of `process`
+
+[source,eql]
+----
+sample by host
+  [ file where file.extension == "exe" ]
+  [ process where true ]
+----
+
+Sample queries do not take into account the chronological ordering of events.
+The `with maxspan` and `with runs` statements as well as the `until` keyword are
+not supported. 
+
 [discrete]
 [[eql-functions]]
 === Functions
diff --git a/docs/reference/features/apis/reset-features-api.asciidoc b/docs/reference/features/apis/reset-features-api.asciidoc
index 300ce166eaa1..d8ba0832cc2a 100644
--- a/docs/reference/features/apis/reset-features-api.asciidoc
+++ b/docs/reference/features/apis/reset-features-api.asciidoc
@@ -6,7 +6,7 @@
 
 experimental::[]
 
-Clears all of the the state information stored in system indices by {es} features, including the security and machine learning indices.
+Clears all of the state information stored in system indices by {es} features, including the security and machine learning indices.
 
 WARNING: Intended for development and testing use only. Do not reset features on a production cluster.
 
diff --git a/docs/reference/health/health.asciidoc b/docs/reference/health/health.asciidoc
index be310ca85ea0..bcbf06559b90 100644
--- a/docs/reference/health/health.asciidoc
+++ b/docs/reference/health/health.asciidoc
@@ -4,18 +4,14 @@
 <titleabbrev>Health</titleabbrev>
 ++++
 
-An experimental API that returns the health status of an {es} cluster.
-
-This API is currently experimental for internal use by Elastic software only.
-
-NOTE: {cloud-only}
+An API that reports the health status of an {es} cluster.
 
 [[health-api-request]]
 ==== {api-request-title}
 
-`GET /_internal/_health` +
+`GET /_health_report` +
 
-`GET /_internal/_health/<indicator>` +
+`GET /_health_report/<indicator>` +
 
 [[health-api-prereqs]]
 ==== {api-prereq-title}
@@ -26,8 +22,8 @@ NOTE: {cloud-only}
 [[health-api-desc]]
 ==== {api-description-title}
 
-The health API returns the health status of an Elasticsearch cluster. It
-returns a list of indicators that compose Elasticsearch functionality.
+The health API returns a report with the health status of an Elasticsearch cluster. The report
+contains a list of indicators that compose Elasticsearch functionality.
 
 Each indicator has a health status of: `green`, `unknown`, `yellow` or `red`. The indicator will
 provide an explanation and metadata describing the reason for its current health status.
@@ -44,7 +40,7 @@ steps that can be performed in order to improve the health of the system. The ro
 steps are encapsulated in a `diagnosis`.
 A diagnosis contains a cause detailing a root cause analysis, an action containing a brief description
 of the steps to take to fix the problem, the list of affected resources (if applicable), and a detailed
-step by step troubleshooting guide to fix the diagnosed problem.
+step-by-step troubleshooting guide to fix the diagnosed problem.
 
 NOTE: The health indicators perform root cause analysis of non-green health statuses. This can
 be computationally expensive when called frequently. When setting up automated polling of the API
@@ -92,6 +88,12 @@ for health status set `verbose` to `false` to disable the more expensive analysi
     These details include additional troubleshooting metrics and sometimes a root cause analysis of a health status.
     Defaults to `true`.
 
+`size`::
+    (Optional, integer) The maximum number of affected resources to return.
+    As a diagnosis can return multiple types of affected resources this parameter will limit the number of resources returned for each type to the configured value (e.g. a diagnosis could return
+    `1000` affected indices and `1000` affected nodes).
+    Defaults to `1000`.
+
 [role="child_attributes"]
 [[health-api-response-body]]
 ==== {api-response-body-title}
@@ -194,7 +196,7 @@ for health status set `verbose` to `false` to disable the more expensive analysi
 
 `action`::
     (string) A brief description the steps that should be taken to remediate the problem.
-    A more detailed step by step guide to remediate the problem is provided by the
+    A more detailed step-by-step guide to remediate the problem is provided by the
     `help_url` field.
 
 `affected_resources`::
@@ -203,7 +205,7 @@ for health status set `verbose` to `false` to disable the more expensive analysi
     diagnosis is applicable for.
 
 `help_url`::
-    (string) A link to the troubleshooting guide that'll fix the healh problem.
+    (string) A link to the troubleshooting guide that'll fix the health problem.
 ========
 =======
 ====
@@ -212,7 +214,7 @@ for health status set `verbose` to `false` to disable the more expensive analysi
 [[health-api-response-details]]
 ==== Indicator Details
 
-Each health indicator in the health api returns a set of details that further explains the state of the system. The
+Each health indicator in the health API returns a set of details that further explains the state of the system. The
 details have contents and a structure that is unique to each indicator.
 
 [[health-api-response-details-master-is-stable]]
@@ -380,7 +382,7 @@ watermark threshold>>.
 
 [source,console]
 --------------------------------------------------
-GET _internal/_health
+GET _health_report
 --------------------------------------------------
 
 The API returns a response with all the indicators regardless
@@ -388,14 +390,14 @@ of current status.
 
 [source,console]
 --------------------------------------------------
-GET _internal/_health/shards_availability
+GET _health_report/shards_availability
 --------------------------------------------------
 
 The API returns a response for just the shard availability indicator.
 
 [source,console]
 --------------------------------------------------
-GET _internal/_health?verbose=false
+GET _health_report?verbose=false
 --------------------------------------------------
 
 The API returns a response with all health indicators but will
diff --git a/docs/reference/how-to/search-speed.asciidoc b/docs/reference/how-to/search-speed.asciidoc
index 04b670c1a724..0db3ca04e99a 100644
--- a/docs/reference/how-to/search-speed.asciidoc
+++ b/docs/reference/how-to/search-speed.asciidoc
@@ -10,7 +10,7 @@ goes to the filesystem cache so that Elasticsearch can keep hot regions of the
 index in physical memory.
 
 [discrete]
-tag::readahead[]
+// tag::readahead[]
 === Avoid page cache thrashing by using modest readahead values on Linux
 
 Search can cause a lot of randomized read I/O. When the underlying block
@@ -35,7 +35,7 @@ as a transient setting). We recommend a value of `128KiB` for readahead.
 WARNING: `blockdev` expects values in 512 byte sectors whereas `lsblk` reports
 values in `KiB`. As an example, to temporarily set readahead to `128KiB`
 for `/dev/nvme0n1`, specify `blockdev --setra 256 /dev/nvme0n1`.
-end::readahead[]
+// end::readahead[]
 
 [discrete]
 === Use faster hardware
@@ -358,7 +358,7 @@ PUT index
 }
 --------------------------------------------------
 
-tag::warm-fs-cache[]
+// tag::warm-fs-cache[]
 [discrete]
 === Warm up the filesystem cache
 
@@ -372,7 +372,7 @@ depending on the file extension using the
 WARNING: Loading data into the filesystem cache eagerly on too many indices or
 too many files will make search _slower_ if the filesystem cache is not large
 enough to hold all the data. Use with caution.
-end::warm-fs-cache[]
+// end::warm-fs-cache[]
 
 [discrete]
 === Use index sorting to speed up conjunctions
@@ -424,6 +424,7 @@ be able to cope with `max_failures` node failures at once at most, then the
 right number of replicas for you is
 `max(max_failures, ceil(num_nodes / num_primaries) - 1)`.
 
+[discrete]
 === Tune your queries with the Search Profiler
 
 The {ref}/search-profile.html[Profile API] provides detailed information about
@@ -438,6 +439,7 @@ Because the Profile API itself adds significant overhead to the query,
 this information is best used to understand the relative cost of the various
 query components. It does not provide a reliable measure of actual processing time.
 
+[discrete]
 [[faster-phrase-queries]]
 === Faster phrase queries with `index_phrases`
 
@@ -446,6 +448,7 @@ indexes 2-shingles and is automatically leveraged by query parsers to run phrase
 queries that don't have a slop. If your use-case involves running lots of phrase
 queries, this can speed up queries significantly.
 
+[discrete]
 [[faster-prefix-queries]]
 === Faster prefix queries with `index_prefixes`
 
@@ -454,6 +457,7 @@ indexes prefixes of all terms and is automatically leveraged by query parsers to
 run prefix queries. If your use-case involves running lots of prefix queries,
 this can speed up queries significantly.
 
+[discrete]
 [[faster-filtering-with-constant-keyword]]
 === Use `constant_keyword` to speed up filtering
 
diff --git a/docs/reference/ilm/actions/ilm-downsample.asciidoc b/docs/reference/ilm/actions/ilm-downsample.asciidoc
index 6daf008cb5b9..7365e3b179d0 100644
--- a/docs/reference/ilm/actions/ilm-downsample.asciidoc
+++ b/docs/reference/ilm/actions/ilm-downsample.asciidoc
@@ -2,8 +2,6 @@
 [[ilm-downsample]]
 === Downsample
 
-preview::[]
-
 Phases allowed: hot, warm, cold.
 
 Aggregates a time series (TSDS) index and stores
diff --git a/docs/reference/ilm/apis/move-to-step.asciidoc b/docs/reference/ilm/apis/move-to-step.asciidoc
index 9b7c448649f7..f901b2402440 100644
--- a/docs/reference/ilm/apis/move-to-step.asciidoc
+++ b/docs/reference/ilm/apis/move-to-step.asciidoc
@@ -96,11 +96,11 @@ The name of the phase that contains the action you want to perform or resume.
 
 `action`::
 (Optional, string)
-The name action you want to perform or resume. 
+The name action you want to perform or resume. Required if `name` used.
 
 `name`::
 (Optional, string)
-The name of the step to move to and execute. 
+The name of the step to move to and execute. Required if `action` used.
 
 ====
 
@@ -176,6 +176,26 @@ If the request succeeds, you receive the following result:
   "acknowledged": true
 }
 --------------------------------------------------
+// TEST[continued]
 
 The request will fail if the index is not in the `new` phase as specified
 by the `current_step`.
+
+The following example pushes `my-index-000001` from the end of hot phase into
+the start of warm:
+
+[source,console]
+--------------------------------------------------
+POST _ilm/move/my-index-000001
+{
+  "current_step": {
+    "phase": "hot",
+    "action": "complete",
+    "name": "complete"
+  },
+  "next_step": {
+    "phase": "warm"
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE[skip: can't consistently get ILM in a completed hot phase during CI checks]
\ No newline at end of file
diff --git a/docs/reference/ilm/ilm-and-snapshots.asciidoc b/docs/reference/ilm/ilm-and-snapshots.asciidoc
index 4d50aeb11e1c..40fd5062164a 100644
--- a/docs/reference/ilm/ilm-and-snapshots.asciidoc
+++ b/docs/reference/ilm/ilm-and-snapshots.asciidoc
@@ -2,6 +2,11 @@
 [[index-lifecycle-and-snapshots]]
 == Restore a managed data stream or index
 
+To <<restore-snapshot-api,restore>> managed indices, ensure that the {ilm-init}
+policies referenced by the indices exist. If necessary, you can restore
+{ilm-init} policies by setting
+<<restore-snapshot-api-request-body,`include_global_state`>> to `true`. 
+
 When you restore a managed index or a data stream with managed backing indices, 
 {ilm-init} automatically resumes executing the restored indices' policies.
 A restored index's `min_age` is relative to when it was originally created or rolled over, 
diff --git a/docs/reference/ilm/ilm-index-lifecycle.asciidoc b/docs/reference/ilm/ilm-index-lifecycle.asciidoc
index aa97993054b6..80b5c6550421 100644
--- a/docs/reference/ilm/ilm-index-lifecycle.asciidoc
+++ b/docs/reference/ilm/ilm-index-lifecycle.asciidoc
@@ -87,27 +87,27 @@ actions in the order listed.
   - <<ilm-unfollow,Unfollow>>
   - <<ilm-rollover,Rollover>>
   - <<ilm-readonly,Read-Only>>
+  - <<ilm-downsample,Downsample>>
   - <<ilm-shrink,Shrink>>
   - <<ilm-forcemerge,Force Merge>>
   - <<ilm-searchable-snapshot, Searchable Snapshot>>
-  - <<ilm-downsample,Downsample>>  
 * Warm
   - <<ilm-set-priority,Set Priority>>
   - <<ilm-unfollow,Unfollow>>
   - <<ilm-readonly,Read-Only>>
+  - <<ilm-downsample,Downsample>>
   - <<ilm-allocate,Allocate>>
   - <<ilm-migrate,Migrate>>
   - <<ilm-shrink,Shrink>>
   - <<ilm-forcemerge,Force Merge>>
-  - <<ilm-downsample,Downsample>>  
 * Cold
   - <<ilm-set-priority,Set Priority>>
   - <<ilm-unfollow,Unfollow>>
   - <<ilm-readonly,Read-Only>>
+  - <<ilm-downsample,Downsample>>
   - <<ilm-searchable-snapshot, Searchable Snapshot>>
   - <<ilm-allocate,Allocate>>
   - <<ilm-migrate,Migrate>>
-  <<ilm-downsample,Downsample>>
 * Frozen
   - <<ilm-unfollow,Unfollow>>
   - <<ilm-searchable-snapshot, Searchable Snapshot>>
diff --git a/docs/reference/images/spatial/geogrid_h3.png b/docs/reference/images/spatial/geogrid_h3.png
new file mode 100644
index 000000000000..7552104044c2
Binary files /dev/null and b/docs/reference/images/spatial/geogrid_h3.png differ
diff --git a/docs/reference/images/spatial/geogrid_h3_child.png b/docs/reference/images/spatial/geogrid_h3_child.png
new file mode 100644
index 000000000000..02b6a8c7dcfe
Binary files /dev/null and b/docs/reference/images/spatial/geogrid_h3_child.png differ
diff --git a/docs/reference/images/spatial/geogrid_h3_children.png b/docs/reference/images/spatial/geogrid_h3_children.png
new file mode 100644
index 000000000000..0382bc22da7a
Binary files /dev/null and b/docs/reference/images/spatial/geogrid_h3_children.png differ
diff --git a/docs/reference/images/spatial/geogrid_tile.png b/docs/reference/images/spatial/geogrid_tile.png
new file mode 100644
index 000000000000..aa9d2a1ea08d
Binary files /dev/null and b/docs/reference/images/spatial/geogrid_tile.png differ
diff --git a/docs/reference/images/spatial/geogrid_tile_child.png b/docs/reference/images/spatial/geogrid_tile_child.png
new file mode 100644
index 000000000000..35284407c410
Binary files /dev/null and b/docs/reference/images/spatial/geogrid_tile_child.png differ
diff --git a/docs/reference/images/spatial/geoshape_hexgrid.png b/docs/reference/images/spatial/geoshape_hexgrid.png
new file mode 100644
index 000000000000..b669314555b6
Binary files /dev/null and b/docs/reference/images/spatial/geoshape_hexgrid.png differ
diff --git a/docs/reference/index-custom-title-page.html b/docs/reference/index-custom-title-page.html
index 0b4e05bf2742..5c006367ebd1 100644
--- a/docs/reference/index-custom-title-page.html
+++ b/docs/reference/index-custom-title-page.html
@@ -41,6 +41,14 @@
       -moz-columns: 2;
     }
   }
+
+  #guide h3.gtk {
+    margin-top: 0;
+  }
+
+  .mb-4, .my-4 {
+    margin-bottom: 0!important;
+  }
 </style>
 
 <div class="legalnotice"></div>
@@ -68,45 +76,7 @@ <h2>Search and analyze your data</h2>
   </div>
 </div>
 
-<h3>Explore by use case</h3>
-
-<div class="row my-4">
-  <div class="col-md-4 col-12 mb-2">
-    <a class="no-text-decoration" href="https://www.elastic.co/guide/en/enterprise-search/current/start.html">
-      <div class="card h-100">
-        <h4 class="mt-3">
-          <span class="inline-block float-left icon mr-2" style="background-image: url('https://images.contentstack.io/v3/assets/bltefdd0b53724fa2ce/blt11200907c1c033aa/634d9da119d8652169cf9b2b/enterprise-search-logo-color-32px.png');"></span>
-          Search my data
-        </h4>
-        <p>Create search experiences for your content, wherever it lives.</p>
-      </div>
-    </a>
-  </div>
-  <div class="col-md-4 col-12 mb-2">
-    <a class="no-text-decoration" href="https://www.elastic.co/guide/en/welcome-to-elastic/current/getting-started-observability.html">
-      <div class="card h-100">
-        <h4 class="mt-3">
-          <span class="inline-block float-left icon mr-2" style="background-image: url('https://images.contentstack.io/v3/assets/bltefdd0b53724fa2ce/bltaa08b370a00bbecc/634d9da14e565f1cdce27f7c/observability-logo-color-32px.png');"></span>
-          Observe my data
-        </h4>
-        <p>Follow our guides to monitor logs, metrics, and traces.</p>
-      </div>
-    </a>
-  </div>
-  <div class="col-md-4 col-12 mb-2">
-    <a class="no-text-decoration" href="https://www.elastic.co/guide/en/security/current/es-overview.html">
-      <div class="card h-100">
-        <h4 class="mt-3">
-          <span class="inline-block float-left icon mr-2" style="background-image: url('https://images.contentstack.io/v3/assets/bltefdd0b53724fa2ce/blt5e0e0ad9a13e6b8c/634d9da18473831f96bbdf1e/security-logo-color-32px.png');"></span>
-          Protect my environment
-        </h4>
-        <p>Learn how to defend against threats across your environment.</p>
-      </div>
-    </a>
-  </div>
-</div>
-
-<h3>Get to know Elasticsearch</h3>
+<h3 class="gtk">Get to know Elasticsearch</h3>
 
 <div class="my-5">
   <div class="d-flex align-items-center mb-3">
@@ -229,5 +199,42 @@ <h4 class="mt-3">
   </ul>
 </div>
 
+<h3 class="explore">Explore by use case</h3>
+
+<div class="row my-4">
+  <div class="col-md-4 col-12 mb-2">
+    <a class="no-text-decoration" href="https://www.elastic.co/guide/en/enterprise-search/current/start.html">
+      <div class="card h-100">
+        <h4 class="mt-3">
+          <span class="inline-block float-left icon mr-2" style="background-image: url('https://images.contentstack.io/v3/assets/bltefdd0b53724fa2ce/blt11200907c1c033aa/634d9da119d8652169cf9b2b/enterprise-search-logo-color-32px.png');"></span>
+          Search my data
+        </h4>
+        <p>Create search experiences for your content, wherever it lives.</p>
+      </div>
+    </a>
+  </div>
+  <div class="col-md-4 col-12 mb-2">
+    <a class="no-text-decoration" href="https://www.elastic.co/guide/en/welcome-to-elastic/current/getting-started-observability.html">
+      <div class="card h-100">
+        <h4 class="mt-3">
+          <span class="inline-block float-left icon mr-2" style="background-image: url('https://images.contentstack.io/v3/assets/bltefdd0b53724fa2ce/bltaa08b370a00bbecc/634d9da14e565f1cdce27f7c/observability-logo-color-32px.png');"></span>
+          Observe my data
+        </h4>
+        <p>Follow our guides to monitor logs, metrics, and traces.</p>
+      </div>
+    </a>
+  </div>
+  <div class="col-md-4 col-12 mb-2">
+    <a class="no-text-decoration" href="https://www.elastic.co/guide/en/security/current/es-overview.html">
+      <div class="card h-100">
+        <h4 class="mt-3">
+          <span class="inline-block float-left icon mr-2" style="background-image: url('https://images.contentstack.io/v3/assets/bltefdd0b53724fa2ce/blt5e0e0ad9a13e6b8c/634d9da18473831f96bbdf1e/security-logo-color-32px.png');"></span>
+          Protect my environment
+        </h4>
+        <p>Learn how to defend against threats across your environment.</p>
+      </div>
+    </a>
+  </div>
+</div>
 
 <p class="my-4"><a href="https://www.elastic.co/guide/index.html">View all Elastic docs</a></p>
diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc
index e258d8a74cc2..d5fc1fef7f33 100644
--- a/docs/reference/index-modules.asciidoc
+++ b/docs/reference/index-modules.asciidoc
@@ -320,7 +320,7 @@ Defaults to `*`, which matches all fields eligible for
     Default <<ingest,ingest pipeline>> for the index. Index requests will fail
     if the default pipeline is set and the pipeline does not exist. The default may be
     overridden using the `pipeline` parameter. The special pipeline name `_none` indicates
-    no ingest pipeline should be run.
+    no default ingest pipeline will run.
 
 [[index-final-pipeline]]
 `index.final_pipeline`::
@@ -328,7 +328,7 @@ Final <<ingest,ingest pipeline>> for the index. Indexing requests
 will fail if the final pipeline is set and the pipeline does not exist.
 The final pipeline always runs after the request pipeline (if specified) and
 the default pipeline (if it exists). The special pipeline name `_none`
-indicates no ingest pipeline will run.
+indicates no final ingest pipeline will run.
 +
 NOTE: You can't use a final pipeline to change the `_index` field. If the
 pipeline attempts to change the `_index` field, the indexing request will fail.
diff --git a/docs/reference/indices/downsample-data-stream.asciidoc b/docs/reference/indices/downsample-data-stream.asciidoc
index 425fd76472bf..8226c365dd50 100644
--- a/docs/reference/indices/downsample-data-stream.asciidoc
+++ b/docs/reference/indices/downsample-data-stream.asciidoc
@@ -5,8 +5,6 @@
 <titleabbrev>Downsample</titleabbrev>
 ++++
 
-preview::[]
-
 Aggregates a time series (TSDS) index and stores
 pre-computed statistical summaries (`min`, `max`, `sum`, `value_count` and
 `avg`) for each metric field grouped by a configured time interval. For example,
@@ -14,6 +12,7 @@ a TSDS index that contains metrics sampled every 10 seconds can be downsampled
 to an hourly index. All documents within an hour interval are summarized and
 stored as a single document in the downsample index.
 
+// tag::downsample-example[]
 ////
 [source,console]
 ----
@@ -74,6 +73,7 @@ DELETE _index_template/*
 ----
 // TEST[continued]
 ////
+// end::downsample-example[]
 
 [[downsample-api-request]]
 ==== {api-request-title}
diff --git a/docs/reference/indices/ignore-missing-component-templates.asciidoc b/docs/reference/indices/ignore-missing-component-templates.asciidoc
new file mode 100644
index 000000000000..8337be779c70
--- /dev/null
+++ b/docs/reference/indices/ignore-missing-component-templates.asciidoc
@@ -0,0 +1,95 @@
+[[ignore_missing_component_templates]]
+== Config ignore_missing_component_templates
+
+The configuration option `ignore_missing_component_templates` can be used when an index template references a component template that might not exist. Every time a data stream is created based on the index template, the existence of the component template will be checked. If it exists, it will used to form the index's composite settings. If it does not exist, it is ignored.
+
+=== Usage example
+
+In the following, one component template and an index template are created. The index template references two component templates, but only the `@package` one exists.
+
+
+Create the component template `logs-foo_component1`. This has to be created before the index template as it is not optional:
+
+[source,console]
+----
+PUT _component_template/logs-foo_component1
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "host.name": {
+          "type": "keyword"
+        }
+      }
+    }
+  }
+}
+----
+
+Next, the index template will be created and it references two component templates:
+
+[source,JSON]
+----
+  "composed_of": ["logs-foo_component1", "logs-foo_component2"]
+----
+
+Before, only the `logs-foo_component1` compontent template was created, meaning the `logs-foo_component2` is missing. Because of this the following entry was added to the config:
+
+[source,JSON]
+----
+  "ignore_missing_component_templates": ["logs-foo_component2"],
+----
+
+During creation of the template, it will not validate that `logs-foo_component2` exists:
+
+
+[source,console]
+----
+PUT _index_template/logs-foo
+{
+  "index_patterns": ["logs-foo-*"],
+  "data_stream": { },
+  "composed_of": ["logs-foo_component1", "logs-foo_component2"],
+  "ignore_missing_component_templates": ["logs-foo_component2"],
+  "priority": 500
+}
+----
+// TEST[continued]
+
+The index template `logs-foo` was successfully created. A data stream can be created based on this template:
+
+[source,console]
+----
+PUT _data_stream/logs-foo-bar
+----
+// TEST[continued]
+
+Looking at the mappings of the data stream, it will contain the `host.name` field.
+
+At a later stage, the missing component template might be added:
+
+[source,console]
+----
+PUT _component_template/logs-foo_component2
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "host.ip": {
+          "type": "ip"
+        }
+      }
+    }
+  }
+}
+----
+// TEST[continued]
+
+This will not have an immediate effect on the data stream. The mapping `host.ip` will only show up in the data stream mappings when the data stream is rolled over automatically next time or a manual rollover is triggered:
+
+[source,console]
+----
+POST logs-foo-bar/_rollover
+----
+// TEST[continued]
+// TEST[teardown:data_stream_cleanup]
diff --git a/docs/reference/indices/index-templates.asciidoc b/docs/reference/indices/index-templates.asciidoc
index 8a4c985970b2..6128ab48998f 100644
--- a/docs/reference/indices/index-templates.asciidoc
+++ b/docs/reference/indices/index-templates.asciidoc
@@ -161,3 +161,5 @@ DELETE _component_template/component_template1
 ////
 
 include::simulate-multi-component-templates.asciidoc[]
+
+include::ignore-missing-component-templates.asciidoc[]
diff --git a/docs/reference/indices/recovery.asciidoc b/docs/reference/indices/recovery.asciidoc
index 4a61155175d4..81b3aa13580c 100644
--- a/docs/reference/indices/recovery.asciidoc
+++ b/docs/reference/indices/recovery.asciidoc
@@ -4,10 +4,9 @@
 <titleabbrev>Index recovery</titleabbrev>
 ++++
 
-
-Returns information about ongoing and completed shard recoveries for one or more
-indices. For data streams, the API returns information for the stream's backing
-indices.
+Returns information about ongoing and completed shard recoveries for one or
+more indices. For data streams, the API returns information for the stream's
+backing indices.
 
 [source,console]
 ----
@@ -33,14 +32,14 @@ index, or alias.
 [[index-recovery-api-desc]]
 ==== {api-description-title}
 
-Use the index recovery API
-to get information about ongoing and completed shard recoveries.
+Use the index recovery API to get information about ongoing and completed shard
+recoveries.
 
 // tag::shard-recovery-desc[]
-Shard recovery is the process
-of syncing a replica shard from a primary shard.
-Upon completion,
-the replica shard is available for search.
+Shard recovery is the process of initializing a shard copy, such as restoring a
+primary shard from a snapshot or syncing a replica shard from a primary shard.
+When a shard recovery completes, the recovered shard is available for search
+and indexing.
 
 Recovery automatically occurs during the following processes:
 
@@ -52,6 +51,14 @@ Recovery automatically occurs during the following processes:
 <<indices-split-index,split>> operation.
 // end::shard-recovery-desc[]
 
+The index recovery API reports information about completed recoveries only for
+shard copies that currently exist in the cluster. It only reports the last
+recovery for each shard copy and does not report historical information about
+earlier recoveries, nor does it report information about the recoveries of
+shard copies that no longer exist. This means that if a shard copy completes a
+recovery and then {es} relocates it onto a different node then the information
+about the original recovery will not be shown in the recovery API.
+
 [[index-recovery-api-path-params]]
 ==== {api-path-parms-title}
 
diff --git a/docs/reference/indices/shard-stores.asciidoc b/docs/reference/indices/shard-stores.asciidoc
index ad50a218fa03..99277a8bf652 100644
--- a/docs/reference/indices/shard-stores.asciidoc
+++ b/docs/reference/indices/shard-stores.asciidoc
@@ -160,7 +160,8 @@ The API returns the following response:
                             "transport_address": "local[1]",
                             "external_id": "node_t0",
                             "attributes": {},
-                            "roles": [...]
+                            "roles": [...],
+                            "version": "8.7.0"
                         },
                         "allocation_id": "2iNySv_OQVePRX-yaRH_lQ", <4>
                         "allocation" : "primary|replica|unused" <5>
@@ -178,6 +179,7 @@ The API returns the following response:
 // TESTRESPONSE[s/: "[^"]*"/: $body.$_path/]
 // TESTRESPONSE[s/"attributes": \{[^}]*\}/"attributes": $body.$_path/]
 // TESTRESPONSE[s/"roles": \[[^]]*\]/"roles": $body.$_path/]
+// TESTRESPONSE[s/"8.7.0"/\$node_version/]
 
 
 
diff --git a/docs/reference/indices/simulate-index.asciidoc b/docs/reference/indices/simulate-index.asciidoc
index 54a9b36f83d1..e86b724332d7 100644
--- a/docs/reference/indices/simulate-index.asciidoc
+++ b/docs/reference/indices/simulate-index.asciidoc
@@ -4,9 +4,7 @@
 <titleabbrev>Simulate index</titleabbrev>
 ++++
 
-experimental[]
-
-Returns the index configuration that would be applied to the specified index from an 
+Returns the index configuration that would be applied to the specified index from an
 existing <<index-templates, index template>>.
 
 ////
diff --git a/docs/reference/ingest/processors.asciidoc b/docs/reference/ingest/processors.asciidoc
index 958dd0ec9d77..89fddc0dd78d 100644
--- a/docs/reference/ingest/processors.asciidoc
+++ b/docs/reference/ingest/processors.asciidoc
@@ -48,6 +48,7 @@ include::processors/enrich.asciidoc[]
 include::processors/fail.asciidoc[]
 include::processors/fingerprint.asciidoc[]
 include::processors/foreach.asciidoc[]
+include::processors/geo-grid.asciidoc[]
 include::processors/geoip.asciidoc[]
 include::processors/grok.asciidoc[]
 include::processors/gsub.asciidoc[]
diff --git a/docs/reference/ingest/processors/circle.asciidoc b/docs/reference/ingest/processors/circle.asciidoc
index f0ec60fcba90..2d46f797586b 100644
--- a/docs/reference/ingest/processors/circle.asciidoc
+++ b/docs/reference/ingest/processors/circle.asciidoc
@@ -12,11 +12,11 @@ Converts circle definitions of shapes to regular polygons which approximate them
 [options="header"]
 |======
 | Name                        | Required  | Default  | Description
-| `field`                     | yes       | -        | The string-valued field to trim whitespace from
+| `field`                     | yes       | -        | The field to interpret as a circle. Either a string in WKT format or a map for GeoJSON.
 | `target_field`              | no        | `field`  | The field to assign the polygon shape to, by default `field` is updated in-place
 | `ignore_missing`            | no        | `false`  | If `true` and `field` does not exist, the processor quietly exits without modifying the document
 | `error_distance`            | yes       | -        | The difference between the resulting inscribed distance from center to side and the circle's radius (measured in meters for `geo_shape`, unit-less for `shape`)
-| `shape_type`                | yes       | -        | which field mapping type is to be used when processing the circle: `geo_shape` or `shape`
+| `shape_type`                | yes       | -        | Which field mapping type is to be used when processing the circle: `geo_shape` or `shape`
 include::common-options.asciidoc[]
 |======
 
diff --git a/docs/reference/ingest/processors/geo-grid.asciidoc b/docs/reference/ingest/processors/geo-grid.asciidoc
new file mode 100644
index 000000000000..dc59ff09866a
--- /dev/null
+++ b/docs/reference/ingest/processors/geo-grid.asciidoc
@@ -0,0 +1,250 @@
+[role="xpack"]
+[[ingest-geo-grid-processor]]
+=== Geo-grid processor
+++++
+<titleabbrev>Geo-grid</titleabbrev>
+++++
+
+Converts geo-grid definitions of grid tiles or cells to regular bounding boxes or polygons which describe their shape.
+This is useful if there is a need to interact with the tile shapes as spatially indexable fields.
+For example the `geotile` field value `"4/8/3"` could be indexed as a string field, but that would not enable
+any spatial operations on it.
+Instead, convert it to the value
+`"POLYGON ((0.0 40.979898069620134, 22.5 40.979898069620134, 22.5 55.77657301866769, 0.0 55.77657301866769, 0.0 40.979898069620134))"`,
+which can be indexed as a <<geo-shape,`geo_shape`>> field.
+
+[[geo-grid-processor-options]]
+.geo_grid processor options
+[options="header"]
+|======
+| Name                        | Required  | Default   | Description
+| `field`                     | yes       | -         | The field to interpret as a geo-tile. The field format is determined by the `tile_type`.
+| `tile_type`                 | yes       | -         | Three tile formats are understood: `geohash`, `geotile` and `geohex`.
+| `target_field`              | no        | `field`   | The field to assign the polygon shape to, by default `field` is updated in-place.
+| `parent_field`              | no        | -         | If specified and a parent tile exists, save that tile address to this field.
+| `children_field`            | no        | -         | If specified and children tiles exist, save those tile addresses to this field as an array of strings.
+| `non_children_field`        | no        | -         | If specified and intersecting non-child tiles exist, save their addresses to this field as an array of strings.
+| `precision_field`           | no        | -         | If specified, save the tile precision (zoom) as an integer to this field.
+| `ignore_missing`            | no        | -         | If `true` and `field` does not exist, the processor quietly exits without modifying the document.
+| `target_format`             | no        | "GeoJSON" | Which format to save the generated polygon in. Either `WKT` or `GeoJSON`.
+include::common-options.asciidoc[]
+|======
+
+To demonstrate the usage of this ingest processor, consider an index called `geocells`
+with a mapping for a field `geocell` of type `geo_shape`.
+In order to populate that index using `geotile` and `geohex` fields, define
+two ingest processors:
+
+[source,console]
+--------------------------------------------------
+PUT geocells
+{
+  "mappings": {
+    "properties": {
+      "geocell": {
+        "type": "geo_shape"
+      }
+    }
+  }
+}
+
+PUT _ingest/pipeline/geotile2shape
+{
+  "description": "translate rectangular z/x/y geotile to bounding box",
+  "processors": [
+    {
+      "geo_grid": {
+        "field": "geocell",
+        "tile_type": "geotile"
+      }
+    }
+  ]
+}
+
+PUT _ingest/pipeline/geohex2shape
+{
+  "description": "translate H3 cell to polygon",
+  "processors": [
+    {
+      "geo_grid": {
+        "field": "geocell",
+        "tile_type": "geohex",
+        "target_format": "wkt"
+      }
+    }
+  ]
+}
+--------------------------------------------------
+
+These two pipelines can be used to index documents into the `geocells` index.
+The `geocell` field will be the string version of either a rectangular tile with format `z/x/y` or an H3 cell address,
+depending on which ingest processor we use when indexing the document.
+The resulting geometry will be represented and indexed as a <<geo-shape,`geo_shape`>> field in either
+http://geojson.org[GeoJSON] or the https://docs.opengeospatial.org/is/12-063r5/12-063r5.html[Well-Known Text] format.
+
+==== Example: Rectangular geotile with envelope in GeoJSON
+
+In this example a `geocell` field with a value defined in `z/x/y` format is indexed as a
+http://geojson.org[GeoJSON Envelope] since the ingest-processor above was defined with default `target_format`.
+
+[source,console]
+--------------------------------------------------
+PUT geocells/_doc/1?pipeline=geotile2shape
+{
+  "geocell": "4/8/5"
+}
+
+GET geocells/_doc/1
+--------------------------------------------------
+// TEST[continued]
+
+The response shows how the ingest-processor has replaced the `geocell` field with an indexable `geo_shape`:
+
+[source,console-result]
+--------------------------------------------------
+{
+ "_index": "geocells",
+  "_id": "1",
+  "_version": 1,
+  "_seq_no": 0,
+  "_primary_term": 1,
+  "found": true,
+  "_source": {
+    "geocell": {
+      "type": "Envelope",
+      "coordinates": [
+        [ 0.0, 55.77657301866769 ],
+        [ 22.5, 40.979898069620134 ]
+      ]
+    }
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"_version": \d+/"_version": $body._version/ s/"_seq_no": \d+/"_seq_no": $body._seq_no/ s/"_primary_term": 1/"_primary_term": $body._primary_term/]
+
+image:images/spatial/geogrid_tile.png[Kibana map with showing the geotile at 4/8/5 and its four child cells]
+
+==== Example: Hexagonal geohex with polygon in WKT format
+
+In this example a `geocell` field with an H3 string address is indexed as a
+https://docs.opengeospatial.org/is/12-063r5/12-063r5.html[WKT Polygon], since this ingest processor explicitly
+defined the `target_format`.
+
+[source,console]
+--------------------------------------------------
+PUT geocells/_doc/1?pipeline=geohex2shape
+{
+  "geocell": "811fbffffffffff"
+}
+
+GET geocells/_doc/1
+--------------------------------------------------
+// TEST[continued]
+
+The response shows how the ingest-processor has replaced the `geocell` field with an indexable `geo_shape`:
+
+[source,console-result]
+--------------------------------------------------
+{
+ "_index": "geocells",
+  "_id": "1",
+  "_version": 1,
+  "_seq_no": 0,
+  "_primary_term": 1,
+  "found": true,
+  "_source": {
+    "geocell": "POLYGON ((1.1885095294564962 49.470279179513454, 2.0265689212828875 45.18424864858389, 7.509948452934623 43.786609335802495, 12.6773177459836 46.40695743262768, 12.345747342333198 50.55427505169064, 6.259687012061477 51.964770150370896, 3.6300085578113794 50.610463307239115, 1.1885095294564962 49.470279179513454))"
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"_version": \d+/"_version": $body._version/ s/"_seq_no": \d+/"_seq_no": $body._seq_no/ s/"_primary_term": 1/"_primary_term": $body._primary_term/]
+
+image:images/spatial/geogrid_h3.png[Kibana map with showing an H3 cell, and its seven child cells]
+
+==== Example: Enriched tile details
+
+As described in <<geo-grid-processor-options,geo_grid processor options>>,
+there are many other fields that can be set, which will enrich the information available.
+For example, with H3 tiles there are 7 child tiles, but only the first is fully contained by the parent.
+The remaining six are only partially overlapping the parent, and there exist a further six non-child tiles
+that overlap the parent.
+This can be investigated by adding parent and child additional fields to the ingest-processor:
+
+[source,console]
+--------------------------------------------------
+PUT _ingest/pipeline/geohex2shape
+{
+  "description": "translate H3 cell to polygon with enriched fields",
+  "processors": [
+    {
+      "geo_grid": {
+        "description": "Ingest H3 cells like '811fbffffffffff' and create polygons",
+        "field": "geocell",
+        "tile_type": "geohex",
+        "target_format": "wkt",
+        "target_field": "shape",
+        "parent_field": "parent",
+        "children_field": "children",
+        "non_children_field": "nonChildren",
+        "precision_field": "precision"
+      }
+    }
+  ]
+}
+--------------------------------------------------
+
+Index the document to see a different result:
+
+[source,console]
+--------------------------------------------------
+PUT geocells/_doc/1?pipeline=geohex2shape
+{
+  "geocell": "811fbffffffffff"
+}
+
+GET geocells/_doc/1
+--------------------------------------------------
+// TEST[continued]
+
+The response from this index request:
+
+[source,console-result]
+--------------------------------------------------
+{
+  "_index": "geocells",
+  "_id": "1",
+  "_version": 1,
+  "_seq_no": 0,
+  "_primary_term": 1,
+  "found": true,
+  "_source": {
+    "parent": "801ffffffffffff",
+    "geocell": "811fbffffffffff",
+    "precision": 1,
+    "shape": "POLYGON ((1.1885095294564962 49.470279179513454, 2.0265689212828875 45.18424864858389, 7.509948452934623 43.786609335802495, 12.6773177459836 46.40695743262768, 12.345747342333198 50.55427505169064, 6.259687012061477 51.964770150370896, 3.6300085578113794 50.610463307239115, 1.1885095294564962 49.470279179513454))",
+    "children": [
+      "821f87fffffffff",
+      "821f8ffffffffff",
+      "821f97fffffffff",
+      "821f9ffffffffff",
+      "821fa7fffffffff",
+      "821faffffffffff",
+      "821fb7fffffffff"
+    ],
+    "nonChildren": [
+      "821ea7fffffffff",
+      "82186ffffffffff",
+      "82396ffffffffff",
+      "821f17fffffffff",
+      "821e37fffffffff",
+      "82194ffffffffff"
+    ]
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"_version": \d+/"_version": $body._version/ s/"_seq_no": \d+/"_seq_no": $body._seq_no/ s/"_primary_term": 1/"_primary_term": $body._primary_term/]
+
+This additional information will then enable, for example, creating a visualization of the H3 cell,
+its children and its intersecting non-children cells.
+
+image:images/spatial/geogrid_h3_children.png[Kibana map with three H3 layers: cell, children and intersecting non-children]
diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc
index fa31a3bbe254..d39f3be82d2b 100644
--- a/docs/reference/ingest/processors/geoip.asciidoc
+++ b/docs/reference/ingest/processors/geoip.asciidoc
@@ -11,7 +11,10 @@ IPv4 or IPv6 address.
 By default, the processor uses the GeoLite2 City, GeoLite2 Country, and GeoLite2
 ASN GeoIP2 databases from
 http://dev.maxmind.com/geoip/geoip2/geolite2/[MaxMind], shared under the
-CC BY-SA 4.0 license. {es} automatically downloads updates for
+CC BY-SA 4.0 license. It automatically downloads these databases if either
+`ingest.geoip.downloader.eager.download` is set to true, or your cluster
+has at least one pipeline with a `geoip` processor. {es}
+automatically downloads updates for
 these databases from the Elastic GeoIP endpoint:
 https://geoip.elastic.co/v1/database. To get download statistics for these
 updates, use the <<geoip-stats-api,GeoIP stats API>>.
@@ -412,6 +415,13 @@ If `true`, {es} automatically downloads and manages updates for GeoIP2 databases
 from the `ingest.geoip.downloader.endpoint`. If `false`, {es} does not download
 updates and deletes all downloaded databases. Defaults to `true`.
 
+[[ingest-geoip-downloader-eager-download]]
+(<<dynamic-cluster-setting,Dynamic>>, Boolean)
+If `true`, {es} downloads GeoIP2 databases immediately, regardless of whether a
+pipeline exists with a geoip processor. If `false`, {es} only begins downloading
+the databases if a pipeline with a geoip processor exists or is added. Defaults
+to `false`.
+
 [[ingest-geoip-downloader-endpoint]]
 `ingest.geoip.downloader.endpoint`::
 (<<static-cluster-setting,Static>>, string)
diff --git a/docs/reference/ingest/processors/json.asciidoc b/docs/reference/ingest/processors/json.asciidoc
index faf93f67b854..84df028ef074 100644
--- a/docs/reference/ingest/processors/json.asciidoc
+++ b/docs/reference/ingest/processors/json.asciidoc
@@ -16,6 +16,7 @@ Converts a JSON string into a structured JSON object.
 | `add_to_root`                   | no        | false     | Flag that forces the parsed JSON to be added at the top level of the document. `target_field` must not be set when this option is chosen.
 | `add_to_root_conflict_strategy` | no        | `replace` | When set to `replace`, root fields that conflict with fields from the parsed JSON will be overridden. When set to `merge`, conflicting fields will be merged. Only applicable if `add_to_root` is set to `true`.
 | `allow_duplicate_keys`          | no        | false     | When set to `true`, the JSON parser will not fail if the JSON contains duplicate keys. Instead, the last encountered value for any duplicate key wins.
+| `strict_json_parsing`           | no        | true      | When set to `true`, the JSON parser will strictly parse the field value. When set to `false`, the JSON parser will be more lenient but also more likely to drop parts of the field value. For example if `strict_json_parsing` is set to `true` and the field value is `123 "foo"` then the processor will throw an IllegalArgumentException. But if `strict_json_parsing` is set to `false` then the field value will be parsed as `123`.
 include::common-options.asciidoc[]
 |======
 
diff --git a/docs/reference/ingest/processors/remove.asciidoc b/docs/reference/ingest/processors/remove.asciidoc
index 6e9b4f24ff51..c3aa42a8f180 100644
--- a/docs/reference/ingest/processors/remove.asciidoc
+++ b/docs/reference/ingest/processors/remove.asciidoc
@@ -13,6 +13,7 @@ Removes existing fields. If one field doesn't exist, an exception will be thrown
 | Name             | Required  | Default  | Description
 | `field`          | yes       | -        | Fields to be removed. Supports <<template-snippets,template snippets>>.
 | `ignore_missing` | no        | `false`  | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
+| `keep`           | no        | -        | Fields to be kept. When set, all fields other than those specified are removed.
 include::common-options.asciidoc[]
 |======
 
@@ -39,3 +40,15 @@ To remove multiple fields, you can use the following query:
 }
 --------------------------------------------------
 // NOTCONSOLE
+
+You can also choose to remove all fields other than a specified list:
+
+[source,js]
+--------------------------------------------------
+{
+  "remove": {
+    "keep": ["url"]
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
diff --git a/docs/reference/mapping/fields/synthetic-source.asciidoc b/docs/reference/mapping/fields/synthetic-source.asciidoc
index 50a9ad315129..60b81c5f813a 100644
--- a/docs/reference/mapping/fields/synthetic-source.asciidoc
+++ b/docs/reference/mapping/fields/synthetic-source.asciidoc
@@ -1,5 +1,12 @@
 [[synthetic-source]]
-==== Synthetic `_source` preview:[]
+==== Synthetic `_source`
+
+IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices
+(indices that have `index.mode` set to `time_series`). For other indices
+synthetic `_source` is in technical preview. Features in technical preview may
+be changed or removed in a future release. Elastic will apply best effort to fix
+any issues, but features in technical preview are not subject to the support SLA
+of official GA features.
 
 Though very handy to have around, the source field takes up a significant amount
 of space on disk. Instead of storing source documents on disk exactly as you
@@ -25,6 +32,8 @@ space. There are a couple of restrictions to be aware of:
 
 * When you retrieve synthetic `_source` content it undergoes minor
 <<synthetic-source-modifications,modifications>> compared to the original JSON.
+* The `params._source` is unavailable in scripts. Instead use the
+{painless}/painless-field-context.html[`doc`] API or the <<script-fields-api, `field`>>.
 * Synthetic `_source` can be used with indices that contain only these field
 types:
 
diff --git a/docs/reference/mapping/params/index.asciidoc b/docs/reference/mapping/params/index.asciidoc
index fbfd6f11d839..08cc4ece23bb 100644
--- a/docs/reference/mapping/params/index.asciidoc
+++ b/docs/reference/mapping/params/index.asciidoc
@@ -2,6 +2,12 @@
 === `index`
 
 The `index` option controls whether field values are indexed. It accepts `true`
-or `false` and defaults to `true`. Fields that are not indexed are typically
-not queryable.
+or `false` and defaults to `true`. 
+
+Indexing a field creates data structures that enable the field to be queried 
+efficiently. <<number,Numeric types>>, <<date,date types>>,
+the <<boolean,boolean type>>, <<ip,ip type>>, <<geo-point,geo_point type>> and the
+<<keyword,keyword type>> can also be queried when they are not <<mapping-index,indexed>>
+ but only have doc values enabled. Queries on these fields are slow as a full scan
+ of the index has to be made. All other fields are not queryable.
 
diff --git a/docs/reference/mapping/runtime.asciidoc b/docs/reference/mapping/runtime.asciidoc
index fd5f1a01cb71..5e3a61277109 100644
--- a/docs/reference/mapping/runtime.asciidoc
+++ b/docs/reference/mapping/runtime.asciidoc
@@ -34,8 +34,8 @@ data into the Elastic Stack and access it right away. When you define a runtime
 field, you can immediately use it in search requests, aggregations, filtering,
 and sorting.
 
-If you make a runtime field an indexed field, you don't need to modify any
-queries that refer to the runtime field. Better yet, you can refer to some
+If you change a runtime field into an indexed field, you don't need to modify
+any queries that refer to the runtime field. Better yet, you can refer to some
 indices where the field is a runtime field, and other indices where the field
 is an indexed field. You have the flexibility to choose which fields to index
 and which ones to keep as runtime fields.
@@ -53,7 +53,7 @@ combined use less resources and reduce your operating costs.
 Runtime fields can replace many of the ways you can use scripting with the
 `_search` API. How you use a runtime field is impacted by the number of
 documents that the included script runs against. For example, if you're using
-the `fields` parameter on the `_search` API to 
+the `fields` parameter on the `_search` API to
 <<runtime-retrieving-fields,retrieve the values of a runtime field>>, the script
 runs only against the top hits just like script fields do.
 
@@ -77,7 +77,7 @@ If you move a script from any of these sections in a search request to a
 runtime field that is computing values from the same number of documents, the
 performance should be about the same. The performance for these features is
 largely dependent upon the calculations that the included script is running and
-how many documents the script runs against. 
+how many documents the script runs against.
 
 [discrete]
 [[runtime-compromises]]
@@ -88,9 +88,9 @@ the runtime script.
 
 To balance search performance and flexibility, index fields that you'll
 frequently search for and filter on, such as a timestamp. {es} automatically
-uses these indexed fields first when running a query, resulting in a fast 
-response time. You can then use runtime fields to limit the number of fields 
-that {es} needs to calculate values for. Using indexed fields in tandem with 
+uses these indexed fields first when running a query, resulting in a fast
+response time. You can then use runtime fields to limit the number of fields
+that {es} needs to calculate values for. Using indexed fields in tandem with
 runtime fields provides flexibility in the data that you index and how you
 define queries for other fields.
 
@@ -111,7 +111,7 @@ You map runtime fields by adding a `runtime` section under the mapping
 definition and defining
 <<modules-scripting-using,a Painless script>>. This script has access to the
 entire context of a document, including the original `_source` via `params._source`
-and any mapped fields plus their values. At query time, the script runs and 
+and any mapped fields plus their values. At query time, the script runs and
 generates values for each scripted field that is required for the query.
 
 .Emitting runtime field values
@@ -227,6 +227,16 @@ with `params._source` (such as `params._source.day_of_week`). For simplicity,
 defining a runtime field in the mapping definition without a script is the
 recommended option, whenever possible.
 
+[[runtime-errorhandling]]
+==== Ignoring script errors on runtime fields
+
+Scripts can throw errors at runtime, e.g. on accessing missing or invalid values
+in documents or because of performing invalid operations. The `on_script_error`
+parameter can be used to control error behaviour when this happens. Setting this
+parameter to `continue` will have the effect of silently ignoring all errors on
+this runtime field. The default `fail` value will cause a shard failure which
+gets reported in the search response.
+
 [[runtime-updating-scripts]]
 ==== Updating and removing runtime fields
 
@@ -932,14 +942,14 @@ can define runtime fields in the
 decide to index a runtime field for greater performance, just move the full
 runtime field definition (including the script) to the context of an index
 mapping. {es} automatically uses these indexed fields to drive queries,
-resulting in a fast response time. This capability means you can write a 
+resulting in a fast response time. This capability means you can write a
 script only once, and apply it to any context that supports runtime fields.
 
 NOTE: Indexing a `composite` runtime field is currently not supported.
 
-You can then use runtime fields to limit the number of fields that {es} needs 
-to calculate values for. Using indexed fields in tandem with runtime fields 
-provides flexibility in the data that you index and how you define queries for 
+You can then use runtime fields to limit the number of fields that {es} needs
+to calculate values for. Using indexed fields in tandem with runtime fields
+provides flexibility in the data that you index and how you define queries for
 other fields.
 
 IMPORTANT: After indexing a runtime field, you cannot update the included
@@ -1417,9 +1427,9 @@ GET my-index-000001/_search
 
 [[runtime-examples-grok-composite]]
 ==== Define a composite runtime field
-You can also define a _composite_ runtime field to emit multiple fields from a 
-single script. You can define a set of typed subfields and emit a map of 
-values. At search time, each subfield retrieves the value associated with 
+You can also define a _composite_ runtime field to emit multiple fields from a
+single script. You can define a set of typed subfields and emit a map of
+values. At search time, each subfield retrieves the value associated with
 their name in the map. This means that you only need to specify your grok
 pattern one time and can return multiple values:
 
@@ -1467,11 +1477,11 @@ GET my-index-000001/_search
 ----
 // TEST[continued]
 
-The API returns the following result. Because `http` is a `composite` runtime 
+The API returns the following result. Because `http` is a `composite` runtime
 field, the response includes each of the sub-fields under `fields`, including
-any associated values that match the query. Without building your data structure 
+any associated values that match the query. Without building your data structure
 in advance, you can search and explore your data in meaningful ways to
-experiment and determine which fields to index. 
+experiment and determine which fields to index.
 
 [source,console-result]
 ----
diff --git a/docs/reference/mapping/types/aggregate-metric-double.asciidoc b/docs/reference/mapping/types/aggregate-metric-double.asciidoc
index 9df58119e8a3..7cbfaff5bc5f 100644
--- a/docs/reference/mapping/types/aggregate-metric-double.asciidoc
+++ b/docs/reference/mapping/types/aggregate-metric-double.asciidoc
@@ -92,9 +92,11 @@ aggregation returns the sum of the values of all `value_count` sub-fields.
 and `value_count` metrics. To run an `avg` aggregation, the field must contain
 both `sum` and `value_count` metric sub-field.
 
-If you use an `aggregate_metric_double` field with other aggregations, the field
-uses the `default_metric` value, which behaves as a `double` field. The
-`default_metric` is also used in scripts and the following queries:
+Running any other aggregation on an `aggregate_metric_double` field will fail with
+an "unsupported aggregation" error.
+
+Finally, an `aggregate_metric_double` field supports the following queries for which
+it behaves as a `double` by delegating its behavior to its `default_metric` sub-field:
 
 * <<query-dsl-exists-query,`exists`>>
 * <<query-dsl-range-query,`range`>>
@@ -248,7 +250,15 @@ The search returns the following hit. The value of the `default_metric` field,
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,/]
 
 [[aggregate-metric-double-synthetic-source]]
-==== Synthetic `_source` preview:[]
+==== Synthetic `_source`
+
+IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices
+(indices that have `index.mode` set to `time_series`). For other indices
+synthetic `_source` is in technical preview. Features in technical preview may
+be changed or removed in a future release. Elastic will apply best effort to fix
+any issues, but features in technical preview are not subject to the support SLA
+of official GA features.
+
 `aggregate_metric-double` fields support <<synthetic-source,synthetic `_source`>> in their default
 configuration. Synthetic `_source` cannot be used together with <<ignore-malformed,`ignore_malformed`>>.
 
diff --git a/docs/reference/mapping/types/boolean.asciidoc b/docs/reference/mapping/types/boolean.asciidoc
index c1347866e406..8b62cecd9d52 100644
--- a/docs/reference/mapping/types/boolean.asciidoc
+++ b/docs/reference/mapping/types/boolean.asciidoc
@@ -216,7 +216,15 @@ The following parameters are accepted by `boolean` fields:
     Metadata about the field.
 
 [[boolean-synthetic-source]]
-==== Synthetic `_source` preview:[]
+==== Synthetic `_source`
+
+IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices
+(indices that have `index.mode` set to `time_series`). For other indices
+synthetic `_source` is in technical preview. Features in technical preview may
+be changed or removed in a future release. Elastic will apply best effort to fix
+any issues, but features in technical preview are not subject to the support SLA
+of official GA features.
+
 `boolean` fields support <<synthetic-source,synthetic `_source`>> in their
 default configuration. Synthetic `_source` cannot be used together with
 <<copy-to,`copy_to`>> or with <<doc-values,`doc_values`>> disabled.
diff --git a/docs/reference/mapping/types/date.asciidoc b/docs/reference/mapping/types/date.asciidoc
index eb5aeee9e8a1..4d69ddfd517c 100644
--- a/docs/reference/mapping/types/date.asciidoc
+++ b/docs/reference/mapping/types/date.asciidoc
@@ -10,9 +10,6 @@ JSON doesn't have a date data type, so dates in Elasticsearch can either be:
 * a number representing _milliseconds-since-the-epoch_.
 * a number representing _seconds-since-the-epoch_ (<<date-epoch-seconds, configuration>>).
 
-NOTE: Values for _milliseconds-since-the-epoch_ must be non-negative. Use a
-formatted date to represent dates before 1970.
-
 Internally, dates are converted to UTC (if the time-zone is specified) and
 stored as a long number representing milliseconds-since-the-epoch.
 
@@ -232,7 +229,15 @@ Which will reply with a date like:
 ----
 
 [[date-synthetic-source]]
-==== Synthetic `_source` preview:[]
+==== Synthetic `_source`
+
+IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices
+(indices that have `index.mode` set to `time_series`). For other indices
+synthetic `_source` is in technical preview. Features in technical preview may
+be changed or removed in a future release. Elastic will apply best effort to fix
+any issues, but features in technical preview are not subject to the support SLA
+of official GA features.
+
 `date` fields support <<synthetic-source,synthetic `_source`>> in their
 default configuration. Synthetic `_source` cannot be used together with
 <<copy-to,`copy_to`>>, <<ignore-malformed,`ignore_malformed`>> set to true
diff --git a/docs/reference/mapping/types/date_nanos.asciidoc b/docs/reference/mapping/types/date_nanos.asciidoc
index 493a08dda74f..8f3af831e2b7 100644
--- a/docs/reference/mapping/types/date_nanos.asciidoc
+++ b/docs/reference/mapping/types/date_nanos.asciidoc
@@ -141,7 +141,15 @@ field. This limitation also affects <<transforms,{transforms}>>.
 ---
 
 [[date-nanos-synthetic-source]]
-==== Synthetic `_source` preview:[]
+==== Synthetic `_source`
+
+IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices
+(indices that have `index.mode` set to `time_series`). For other indices
+synthetic `_source` is in technical preview. Features in technical preview may
+be changed or removed in a future release. Elastic will apply best effort to fix
+any issues, but features in technical preview are not subject to the support SLA
+of official GA features.
+
 `date_nanos` fields support <<synthetic-source,synthetic `_source`>> in their
 default configuration. Synthetic `_source` cannot be used together with
 <<copy-to,`copy_to`>>, <<ignore-malformed,`ignore_malformed`>> set to true
diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc
index 198fbf1f928e..1f9a872ca0bc 100644
--- a/docs/reference/mapping/types/dense-vector.asciidoc
+++ b/docs/reference/mapping/types/dense-vector.asciidoc
@@ -51,8 +51,6 @@ It is not possible to store multiple values in one `dense_vector` field.
 [[index-vectors-knn-search]]
 ==== Index vectors for kNN search
 
-experimental::[]
-
 include::{es-repo-dir}/search/search-your-data/knn-search.asciidoc[tag=knn-def]
 
 Dense vector fields can be used to rank documents in
@@ -195,5 +193,13 @@ neighbors for each new node. Defaults to `100`.
 ====
 
 [[dense-vector-synthetic-source]]
-==== Synthetic `_source` preview:[]
+==== Synthetic `_source`
+
+IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices
+(indices that have `index.mode` set to `time_series`). For other indices
+synthetic `_source` is in technical preview. Features in technical preview may
+be changed or removed in a future release. Elastic will apply best effort to fix
+any issues, but features in technical preview are not subject to the support SLA
+of official GA features.
+
 `dense_vector` fields support <<synthetic-source,synthetic `_source`>> .
diff --git a/docs/reference/mapping/types/geo-point.asciidoc b/docs/reference/mapping/types/geo-point.asciidoc
index 3f08234a98b0..47996b8e4822 100644
--- a/docs/reference/mapping/types/geo-point.asciidoc
+++ b/docs/reference/mapping/types/geo-point.asciidoc
@@ -208,7 +208,15 @@ def lon      = doc['location'].lon;
 --------------------------------------------------
 
 [[geo-point-synthetic-source]]
-==== Synthetic source preview:[]
+==== Synthetic source
+
+IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices
+(indices that have `index.mode` set to `time_series`). For other indices
+synthetic `_source` is in technical preview. Features in technical preview may
+be changed or removed in a future release. Elastic will apply best effort to fix
+any issues, but features in technical preview are not subject to the support SLA
+of official GA features.
+
 `geo_point` fields support <<synthetic-source,synthetic `_source`>> in their
 default configuration. Synthetic `_source` cannot be used together with
 <<ignore-malformed,`ignore_malformed`>>, <<copy-to,`copy_to`>>, or with
diff --git a/docs/reference/mapping/types/histogram.asciidoc b/docs/reference/mapping/types/histogram.asciidoc
index 703cce6c83d0..fa95c4e6328d 100644
--- a/docs/reference/mapping/types/histogram.asciidoc
+++ b/docs/reference/mapping/types/histogram.asciidoc
@@ -69,7 +69,15 @@ means the field can technically be aggregated with either algorithm, in practice
 index data in that manner (e.g. centroids for T-Digest or intervals for HDRHistogram) to ensure best accuracy.
 
 [[histogram-synthetic-source]]
-==== Synthetic `_source` preview:[]
+==== Synthetic `_source`
+
+IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices
+(indices that have `index.mode` set to `time_series`). For other indices
+synthetic `_source` is in technical preview. Features in technical preview may
+be changed or removed in a future release. Elastic will apply best effort to fix
+any issues, but features in technical preview are not subject to the support SLA
+of official GA features.
+
 `histogram` fields support <<synthetic-source,synthetic `_source`>> in their
 default configuration. Synthetic `_source` cannot be used together with
 <<ignore-malformed,`ignore_malformed`>> or <<copy-to,`copy_to`>>.
diff --git a/docs/reference/mapping/types/ip.asciidoc b/docs/reference/mapping/types/ip.asciidoc
index 1a7f90954a96..25e18a0c60a6 100644
--- a/docs/reference/mapping/types/ip.asciidoc
+++ b/docs/reference/mapping/types/ip.asciidoc
@@ -152,7 +152,15 @@ GET my-index-000001/_search
 --------------------------------------------------
 
 [[ip-synthetic-source]]
-==== Synthetic `_source` preview:[]
+==== Synthetic `_source`
+
+IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices
+(indices that have `index.mode` set to `time_series`). For other indices
+synthetic `_source` is in technical preview. Features in technical preview may
+be changed or removed in a future release. Elastic will apply best effort to fix
+any issues, but features in technical preview are not subject to the support SLA
+of official GA features.
+
 `ip` fields support <<synthetic-source,synthetic `_source`>> in their default
 configuration. Synthetic `_source` cannot be used together with
 <<copy-to,`copy_to`>> or with <<doc-values,`doc_values`>> disabled.
diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc
index c1bd5b49dfc9..a47e1d4549eb 100644
--- a/docs/reference/mapping/types/keyword.asciidoc
+++ b/docs/reference/mapping/types/keyword.asciidoc
@@ -170,7 +170,15 @@ Dimension fields have the following constraints:
 --
 
 [[keyword-synthetic-source]]
-==== Synthetic `_source` preview:[]
+==== Synthetic `_source`
+
+IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices
+(indices that have `index.mode` set to `time_series`). For other indices
+synthetic `_source` is in technical preview. Features in technical preview may
+be changed or removed in a future release. Elastic will apply best effort to fix
+any issues, but features in technical preview are not subject to the support SLA
+of official GA features.
+
 `keyword` fields support <<synthetic-source,synthetic `_source`>> in their
 default configuration. Synthetic `_source` cannot be used together with
 a <<normalizer,`normalizer`>> or <<copy-to,`copy_to`>>.
diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc
index 2e9805d2f178..bee79746806b 100644
--- a/docs/reference/mapping/types/numeric.asciidoc
+++ b/docs/reference/mapping/types/numeric.asciidoc
@@ -228,7 +228,15 @@ numeric field can't be both a time series dimension and a time series metric.
     This parameter is required.
 
 [[numeric-synthetic-source]]
-==== Synthetic `_source` preview:[]
+==== Synthetic `_source`
+
+IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices
+(indices that have `index.mode` set to `time_series`). For other indices
+synthetic `_source` is in technical preview. Features in technical preview may
+be changed or removed in a future release. Elastic will apply best effort to fix
+any issues, but features in technical preview are not subject to the support SLA
+of official GA features.
+
 All numeric fields except `unsigned_long` support <<synthetic-source,synthetic
 `_source`>> in their default configuration. Synthetic `_source` cannot be used
 together with <<ignore-malformed,`ignore_malformed`>>, <<copy-to,`copy_to`>>, or
diff --git a/docs/reference/mapping/types/rank-features.asciidoc b/docs/reference/mapping/types/rank-features.asciidoc
index 8be5e0018578..b54e99ede3fa 100644
--- a/docs/reference/mapping/types/rank-features.asciidoc
+++ b/docs/reference/mapping/types/rank-features.asciidoc
@@ -83,7 +83,10 @@ NOTE: `rank_features` fields only support single-valued features and strictly
 positive values. Multi-valued fields and zero or negative values will be rejected.
 
 NOTE: `rank_features` fields do not support sorting or aggregating and may
-only be queried using <<query-dsl-rank-feature-query,`rank_feature`>> queries.
+only be queried using <<query-dsl-rank-feature-query,`rank_feature`>> or <<query-dsl-term-query, `term`>> queries.
+
+NOTE: <<query-dsl-term-query, `term`>> queries on `rank_features` fields are scored by multiplying the matched
+stored feature value by the provided `boost`.
 
 NOTE: `rank_features` fields only preserve 9 significant bits for the
 precision, which translates to a relative error of about 0.4%.
diff --git a/docs/reference/mapping/types/text.asciidoc b/docs/reference/mapping/types/text.asciidoc
index 730ca12c1a3e..acc9c962add9 100644
--- a/docs/reference/mapping/types/text.asciidoc
+++ b/docs/reference/mapping/types/text.asciidoc
@@ -160,7 +160,15 @@ The following parameters are accepted by `text` fields:
     Metadata about the field.
 
 [[text-synthetic-source]]
-==== Synthetic `_source` preview:[]
+==== Synthetic `_source`
+
+IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices
+(indices that have `index.mode` set to `time_series`). For other indices
+synthetic `_source` is in technical preview. Features in technical preview may
+be changed or removed in a future release. Elastic will apply best effort to fix
+any issues, but features in technical preview are not subject to the support SLA
+of official GA features.
+
 `text` fields support <<synthetic-source,synthetic `_source`>> if they have
 a <<keyword-synthetic-source, `keyword`>> sub-field that supports synthetic
 `_source` or if the `text` field sets `store` to `true`. Either way, it may
@@ -258,11 +266,11 @@ Will become:
 
 `text` fields are searchable by default, but by default are not available for
 aggregations, sorting, or scripting. If you try to sort, aggregate, or access
-values from a script on a `text` field, you will see this exception:
+values from a `text` field using a script, you'll see an exception indicating
+that field data is disabled by default on text fields. To load field data in
+memory, set `fielddata=true` on your field.
 
-Fielddata is disabled on text fields by default. Set `fielddata=true` on
-`your_field_name` in order to load fielddata in memory by uninverting the
-inverted index. Note that this can however use significant memory.
+NOTE: Loading field data in memory can consume significant memory.
 
 Field data is the only way to access the analyzed tokens from a full text field
 in aggregations, sorting, or scripting. For example, a full text field like `New York`
diff --git a/docs/reference/mapping/types/version.asciidoc b/docs/reference/mapping/types/version.asciidoc
index b042ea02da7c..b208127ae270 100644
--- a/docs/reference/mapping/types/version.asciidoc
+++ b/docs/reference/mapping/types/version.asciidoc
@@ -69,7 +69,15 @@ you strongly rely on these kind of queries.
 
 
 [[version-synthetic-source]]
-==== Synthetic `_source` preview:[]
+==== Synthetic `_source`
+
+IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices
+(indices that have `index.mode` set to `time_series`). For other indices
+synthetic `_source` is in technical preview. Features in technical preview may
+be changed or removed in a future release. Elastic will apply best effort to fix
+any issues, but features in technical preview are not subject to the support SLA
+of official GA features.
+
 `version` fields support <<synthetic-source,synthetic `_source`>> so long as they don't
 declare <<copy-to,`copy_to`>>.
 
diff --git a/docs/reference/migration/index.asciidoc b/docs/reference/migration/index.asciidoc
index 23f031426ab1..a523ea2d47f7 100644
--- a/docs/reference/migration/index.asciidoc
+++ b/docs/reference/migration/index.asciidoc
@@ -1,6 +1,7 @@
 include::migration_intro.asciidoc[]
 
 * <<migrating-8.7,Migrating to 8.7>>
+* <<migrating-8.6,Migrating to 8.6>>
 * <<migrating-8.5,Migrating to 8.5>>
 * <<migrating-8.4,Migrating to 8.4>>
 * <<migrating-8.3,Migrating to 8.3>>
@@ -9,6 +10,7 @@ include::migration_intro.asciidoc[]
 * <<migrating-8.0,Migrating to 8.0>>
 
 include::migrate_8_7.asciidoc[]
+include::migrate_8_6.asciidoc[]
 include::migrate_8_5.asciidoc[]
 include::migrate_8_4.asciidoc[]
 include::migrate_8_3.asciidoc[]
diff --git a/docs/reference/migration/migrate_8_5.asciidoc b/docs/reference/migration/migrate_8_5.asciidoc
index 00b3fbd091b9..4a91c68afb1d 100644
--- a/docs/reference/migration/migrate_8_5.asciidoc
+++ b/docs/reference/migration/migrate_8_5.asciidoc
@@ -9,9 +9,6 @@ your application to {es} 8.5.
 
 See also <<release-highlights>> and <<es-release-notes>>.
 
-coming::[8.5.0]
-
-
 [discrete]
 [[breaking-changes-8.5]]
 === Breaking changes
diff --git a/docs/reference/migration/migrate_8_6.asciidoc b/docs/reference/migration/migrate_8_6.asciidoc
index c266e1784c09..60d2af03cbf6 100644
--- a/docs/reference/migration/migrate_8_6.asciidoc
+++ b/docs/reference/migration/migrate_8_6.asciidoc
@@ -9,9 +9,6 @@ your application to {es} 8.6.
 
 See also <<release-highlights>> and <<es-release-notes>>.
 
-coming::[8.6.0]
-
-
 [discrete]
 [[breaking-changes-8.6]]
 === Breaking changes
@@ -20,3 +17,79 @@ coming::[8.6.0]
 There are no breaking changes in {es} 8.6.
 // end::notable-breaking-changes[]
 
+
+[discrete]
+[[deprecated-8.6]]
+=== Deprecations
+
+The following functionality has been deprecated in {es} 8.6
+and will be removed in a future version.
+While this won't have an immediate impact on your applications,
+we strongly encourage you to take the described steps to update your code
+after upgrading to 8.6.
+
+To find out if you are using any deprecated functionality,
+enable <<deprecation-logging, deprecation logging>>.
+
+
+[discrete]
+[[deprecations_86_crud]]
+==== CRUD deprecations
+
+[[deprecate_remove_binary_default_of_false_for_ingest_attachment_processor]]
+.Deprecate 'remove_binary' default of false for ingest attachment processor
+[%collapsible]
+====
+*Details* +
+The default "remove_binary" option for the attachment processor will be changed from false to true in a later Elasticsearch release. This means that the binary file sent to Elasticsearch will not be retained.
+
+*Impact* +
+Users should update the "remove_binary" option to be explicitly true or false, instead of relying on the default value, so that no default value changes will affect Elasticsearch.
+====
+
+[discrete]
+[[deprecations_86_cluster_and_node_setting]]
+==== Cluster and node setting deprecations
+
+[[ensure_balance_threshold_at_least_1]]
+.Ensure balance threshold is at least 1
+[%collapsible]
+====
+*Details* +
+Values for `cluster.routing.allocation.balance.threshold` smaller than `1` are now ignored. Support for values less than `1` for this setting is deprecated and will be forbidden in a future version.
+
+*Impact* +
+Set `cluster.routing.allocation.balance.threshold` to be at least `1`.
+====
+
+[discrete]
+[[deprecations_86_mapping]]
+==== Mapping deprecations
+
+[[deprecate_silently_ignoring_type_fields_copy_to_boost_in_metadata_field_definition]]
+.Deprecate silently ignoring type, fields, copy_to and boost in metadata field definition
+[%collapsible]
+====
+*Details* +
+Unsupported parameters like type, fields, copy_to and boost are silently ignored when provided as part of the configuration of a metadata field in the index mappings. They will cause a deprecation warning when used in the mappings for indices that are created from 8.6 onwards.
+
+*Impact* +
+To resolve the deprecation warning, remove the mention of type, fields, copy_to or boost from any metadata field definition as part of index mappings. They take no effect so removing them won't have any impact besides resolving the deprecation warning.
+====
+
+[discrete]
+[[deprecations_86_rest_api]]
+==== REST API deprecations
+
+[[state_field_deprecated_in_cluster_reroute_response]]
+.state field is deprecated in /_cluster/reroute response
+[%collapsible]
+====
+*Details* +
+`state` field is deprecated in `/_cluster/reroute` response. Cluster state does not provide meaningful information
+about the result of reroute/commands execution. There are no guarantees that this exact state would be applied.
+
+*Impact* +
+Reroute API users should not rely on `state` field and instead use `explain` to request result of commands execution.
+====
+
diff --git a/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc
index a3a72bc39ab7..686632366315 100644
--- a/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc
@@ -135,6 +135,11 @@ reduced. If the bucket contains fewer samples than expected, the score is reduce
 `lower_confidence_bound`::::
 (Optional, double) Lower bound of the 95% confidence interval.
 
+`multimodal_distribution`::::
+(Optional, boolean) Indicates whether the bucket values' probability distribution has 
+several modes. When there are multiple modes, the typical value may not be the most 
+likely.
+
 `multi_bucket_impact`::::
 (Optional, integer) Impact of the deviation between actual and typical values in the 
 past 12 buckets.
diff --git a/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc
index dd9b2f16cb57..48893f1aadb8 100644
--- a/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc
+++ b/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc
@@ -25,8 +25,9 @@ Requires the `manage_ml` cluster privilege. This privilege is included in the
 [[ml-update-datafeed-desc]]
 == {api-description-title}
 
-If you update a {dfeed} property, you must stop and start the {dfeed} for the
-change to be applied.
+You can only update a {dfeed} property while the {dfeed} is stopped.
+However, it is possible to stop a {dfeed}, update one of its properties,
+and restart it without closing the associated job.
 
 IMPORTANT: When {es} {security-features} are enabled, your {dfeed} remembers
 which roles the user who updated it had at the time of update and runs the query
diff --git a/docs/reference/ml/anomaly-detection/functions/ml-count-functions.asciidoc b/docs/reference/ml/anomaly-detection/functions/ml-count-functions.asciidoc
index 2ef14025986f..aef8e13bd429 100644
--- a/docs/reference/ml/anomaly-detection/functions/ml-count-functions.asciidoc
+++ b/docs/reference/ml/anomaly-detection/functions/ml-count-functions.asciidoc
@@ -26,11 +26,11 @@ The {ml-features} include the following count functions:
 The `count` function detects anomalies when the number of events in a bucket is
 anomalous.
 
-The `high_count` function detects anomalies when the count of events in a
-bucket are unusually high.
+The `high_count` function detects anomalies when the count of events in a bucket 
+are unusually high.
 
-The `low_count` function detects anomalies when the count of events in a
-bucket are unusually low.
+The `low_count` function detects anomalies when the count of events in a bucket 
+are unusually low.
 
 These functions support the following properties:
 
@@ -111,8 +111,8 @@ PUT _ml/anomaly_detectors/example3
 --------------------------------------------------
 // TEST[skip:needs-licence]
 
-In this example, the function detects when the count of events for a
-status code is lower than usual.
+In this example, the function detects when the count of events for a status code 
+is lower than usual.
 
 When you use this function in a detector in your {anomaly-job}, it models the
 event rate for each status code and detects when a status code has an unusually
@@ -168,19 +168,19 @@ For more information about those properties, see the
 
 For example, if you have the following number of events per bucket:
 
-========================================
+====
 
 1,22,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,43,31,0,0,0,0,0,0,0,0,0,0,0,0,2,1
 
-========================================
+====
 
 The `non_zero_count` function models only the following data:
 
-========================================
+====
 
 1,22,2,43,31,2,1
 
-========================================
+====
 
 .Example 5: Analyzing signatures with the high_non_zero_count function
 [source,console]
diff --git a/docs/reference/ml/anomaly-detection/ml-configuring-aggregations.asciidoc b/docs/reference/ml/anomaly-detection/ml-configuring-aggregations.asciidoc
index 469f0bdb12b5..b9ae702d3ccb 100644
--- a/docs/reference/ml/anomaly-detection/ml-configuring-aggregations.asciidoc
+++ b/docs/reference/ml/anomaly-detection/ml-configuring-aggregations.asciidoc
@@ -34,6 +34,10 @@ to match. For example, if you use a `max` aggregation on a time field called
 descending order. Additional `composite` aggregation value sources are allowed, 
 such as `terms`.
 
+* The `size` parameter of the non-composite aggregations must match the 
+cardinality of your data. A greater value of the `size` parameter increases the 
+memory requirement of the aggregation.
+
 * If you set the `summary_count_field_name` property to a non-null value, the 
 {anomaly-job} expects to receive aggregated input. The property must be set to 
 the name of the field that contains the count of raw data points that have been 
@@ -439,3 +443,85 @@ the `error` field.
 ----------------------------------
 // NOTCONSOLE
 
+
+[discrete]
+[[aggs-amd-dfeeds]]
+== Using `aggregate_metric_double` field type in {dfeeds}
+
+
+NOTE: It is not currently possible to use `aggregate_metric_double` type fields 
+in {dfeeds} without aggregations. 
+
+You can use fields with the 
+{ref}/aggregate-metric-double.html[`aggregate_metric_double`] field type in a 
+{dfeed} with aggregations. It is required to retrieve the `value_count` of the 
+`aggregate_metric_double` filed in an aggregation and then use it as the 
+`summary_count_field_name` to provide the correct count that represents the 
+aggregation value.
+
+In the following example, `presum` is an `aggregate_metric_double` type field 
+that has all the possible metrics: `[ min, max, sum, value_count ]`. To use an 
+`avg` aggregation on this field, you need to perform a `value_count` aggregation 
+on `presum` and then set the field that contains the aggregated values 
+`my_count` as the `summary_count_field_name`: 
+
+
+[source,js]
+----------------------------------
+{
+  "analysis_config": {
+    "bucket_span": "1h",
+    "detectors": [
+      {
+        "function": "avg",
+        "field_name": "my_avg"
+      }
+    ],
+    "summary_count_field_name": "my_count" <1>
+  },
+  "data_description": {
+    "time_field": "timestamp"
+  },
+  "datafeed_config": {
+    "indices": [
+      "my_index"
+    ],
+    "datafeed_id": "datafeed-id",
+    "aggregations": {
+      "buckets": {
+        "date_histogram": {
+          "field": "time",
+          "fixed_interval": "360s",
+          "time_zone": "UTC"
+        },
+        "aggregations": {
+            "timestamp": {  
+                "max": {"field": "timestamp"}
+            },
+            "my_avg": {  <2>
+                "avg": {
+                    "field": "presum" 
+                }
+             },
+             "my_count": { <3>
+                 "value_count": {
+                     "field": "presum" 
+                 }
+             }
+          }
+        }
+     }
+  }
+}
+----------------------------------
+// NOTCONSOLE
+
+<1> The field `my_count` is set as the `summary_count_field_name`. This field 
+contains aggregated values from the `presum` `aggregate_metric_double` type 
+field (refer to footnote 3). 
+<2> The `avg` aggregation to use on the `presum` `aggregate_metric_double` type 
+field.
+<3> The `value_count` aggregation on the `presum` `aggregate_metric_double` type 
+field. This aggregated field must be set as the `summary_count_field_name` 
+(refer to footnote 1) to make it possible to use the `aggregate_metric_double` 
+type field in another aggregation.
\ No newline at end of file
diff --git a/docs/reference/ml/anomaly-detection/ml-configuring-alerts.asciidoc b/docs/reference/ml/anomaly-detection/ml-configuring-alerts.asciidoc
index 7afaf88081b2..265b94d1d69d 100644
--- a/docs/reference/ml/anomaly-detection/ml-configuring-alerts.asciidoc
+++ b/docs/reference/ml/anomaly-detection/ml-configuring-alerts.asciidoc
@@ -30,38 +30,38 @@ ideal for this purpose.
 [[creating-ml-rules]]
 == Creating a rule
 
-You can create {ml} rules in the {anomaly-job} wizard after you start the job, 
-from the job list, or under **{stack-manage-app} > {alerts-ui}**.
-
-On the *Create rule* window, give a name to the rule and optionally provide 
-tags. Specify the time interval for the rule to check detected anomalies or job 
-health changes. It is recommended to select an interval that is close to the 
-bucket span of the job. You can also select a notification option with the 
-_Notify_ selector. An alert remains active as long as the configured conditions 
-are met during the check interval. When there is no matching condition in the 
-next interval, the `Recovered` action group is invoked and the status of the 
-alert changes to `OK`. For more details, refer to the documentation of 
-{kibana-ref}/create-and-manage-rules.html#defining-rules-general-details[general rule details].
-
-Select the rule type you want to create under the {ml} section and continue to 
-configure it depending on whether it is an 
-<<creating-anomaly-alert-rules, {anomaly-detect} alert>> or an 
-<<creating-anomaly-jobs-health-rules, {anomaly-job} health>> rule.
+In *{stack-manage-app} > {rules-ui}*, you can create both types of {ml} rules:
 
 [role="screenshot"]
-image::images/ml-rule.jpg["Creating a new machine learning rule"]
+image::images/ml-rule.png["Creating a new machine learning rule",500]
+// NOTE: This is an autogenerated screenshot. Do not edit it directly.
 
+When you create a {ml} rule, you must provide a time interval for the rule to
+check detected anomalies or job health changes. It is recommended to select an
+interval that is close to the bucket span of the job.
+
+You must also select a notification option, which affects how often alerts
+generate actions. Options include running actions at each check interval, only
+when the alert status changes, or at a custom action interval. For more
+information about these options, refer to the
+{kibana-ref}/create-and-manage-rules.html#defining-rules-general-details[General rule details].
+
+In the *{ml-app}* app, you can create only {anomaly-detect} alert rules; create
+them from the {anomaly-job} wizard after you start the job or from the
+{anomaly-job} list.
 
 [[creating-anomaly-alert-rules]]
 === {anomaly-detect-cap} alert
 
-Select the job that the rule applies to.
+When you create an {anomaly-detect} alert rule, you must select the job that
+the rule applies to.
 
-You must select a type of {ml} result. In particular, you can create rules based 
-on bucket, record, or influencer results.
+You must also select a type of {ml} result. In particular, you can create rules
+based  on bucket, record, or influencer results.
 
 [role="screenshot"]
-image::images/ml-anomaly-alert-severity.jpg["Selecting result type, severity, and test interval", 500]
+image::images/ml-anomaly-alert-severity.png["Selecting result type, severity, and test interval", 500]
+// NOTE: This is an autogenerated screenshot. Do not edit it directly.
 
 For each rule, you can configure the `anomaly_score` that triggers the action. 
 The `anomaly_score` indicates the significance of a given anomaly compared to 
@@ -98,8 +98,9 @@ are met.
 [[creating-anomaly-jobs-health-rules]]
 === {anomaly-jobs-cap} health
 
-Select the job or group that the rule applies to. If you assign more jobs to the 
-group, they are included the next time the rule conditions are checked.
+When you create an {anomaly-jobs} health rule, you must select the job or group
+that the rule applies to. If you assign more jobs to the group, they are
+included the next time the rule conditions are checked.
 
 You can also use a special character (`*`) to apply the rule to all your jobs. 
 Jobs created after the rule are automatically included. You can exclude jobs 
@@ -131,7 +132,8 @@ _Errors in job messages_::
   that occur after the rule is created; it does not look at historic behavior.
 
 [role="screenshot"]
-image::images/ml-health-check-config.jpg["Selecting health checkers"]
+image::images/ml-health-check-config.png["Selecting health checkers",500]
+// NOTE: This is an autogenerated screenshot. Do not edit it directly.
 
 As the last step in the rule creation process, 
 <<defining-actions, define the actions>> that occur when the conditions
@@ -141,43 +143,35 @@ are met.
 [[defining-actions]]
 == Defining actions
 
-Connect your rule to actions that use supported built-in integrations by 
-selecting a connector type. Connectors are {kib} services or third-party 
-integrations that perform an action when the rule conditions are met or the 
-alert is recovered. You can select in which case the action will run.
-
-[role="screenshot"]
-image::images/ml-anomaly-alert-actions.jpg["Selecting connector type"]
-
-For example, you can choose _Slack_ as a connector type and configure it to send 
-a message to a channel you selected. You can also create an index connector that 
-writes the JSON object you configure to a specific index. It's also possible to 
-customize the notification messages. A list of variables is available to include 
-in the message, like job ID, anomaly score, time, top influencers, {dfeed} ID, 
-memory status and so on based on the selected rule type. Refer to 
-<<action-variables>> to see the full list of available variables by rule type.
+Your rule can use connectors, which are {kib} services or supported third-party
+integrations that run actions when the rule conditions are met or when the 
+alert is recovered. For details about creating connectors, refer to
+{kibana-ref}/action-types.html[Connectors].
 
+For example, you can use a Slack connector to send a message to a channel. Or
+you can use an index connector that writes an JSON object to a specific index.
+It's also possible to customize the notification messages. There is a set of
+variables that you can include in the message depending on the rule type; refer
+to <<action-variables>>.
 
 [role="screenshot"]
-image::images/ml-anomaly-alert-messages.jpg["Customizing your message"]
-
-After you save the configurations, the rule appears in the *{alerts-ui}* list 
-where you can check its status and see the overview of its configuration 
-information.
+image::images/ml-anomaly-alert-messages.png["Customizing your message",500]
+// NOTE: This is an autogenerated screenshot. Do not edit it directly.
 
-The name of an alert is always the same as the job ID of the associated 
-{anomaly-job} that triggered it. You can mute the notifications for a particular 
-{anomaly-job} on the page of the rule that lists the individual alerts. You can 
-open it via *{alerts-ui}* by selecting the rule name.
+After you save the configurations, the rule appears in the
+*{stack-manage-app} > {rules-ui}* list; you can check its status and see the
+overview of its configuration information.
 
+When an alert occurs, it is always the same name as the job ID of the associated 
+{anomaly-job} that triggered it. If necessary, you can snooze rules to prevent
+them from generating actions. For more details, refer to
+{kibana-ref}/create-and-manage-rules.html#controlling-rules[Snooze and disable rules].
 
 [[action-variables]]
 == Action variables
 
-You can add different variables to your action. The following variables are 
-specific to the {ml} rule types. An `*` marks the variables that can be used for 
-actions of recovered alerts.
-
+The following variables are specific to the {ml} rule types. An asterisk (`*`)
+marks the variables that you can use in actions related to recovered alerts.
 
 [[anomaly-alert-action-variables]]
 === {anomaly-detect-cap} alert action variables
diff --git a/docs/reference/ml/images/ml-anomaly-alert-actions.jpg b/docs/reference/ml/images/ml-anomaly-alert-actions.jpg
deleted file mode 100644
index a0b75152ca71..000000000000
Binary files a/docs/reference/ml/images/ml-anomaly-alert-actions.jpg and /dev/null differ
diff --git a/docs/reference/ml/images/ml-anomaly-alert-messages.jpg b/docs/reference/ml/images/ml-anomaly-alert-messages.jpg
deleted file mode 100644
index de5da557dd11..000000000000
Binary files a/docs/reference/ml/images/ml-anomaly-alert-messages.jpg and /dev/null differ
diff --git a/docs/reference/ml/images/ml-anomaly-alert-messages.png b/docs/reference/ml/images/ml-anomaly-alert-messages.png
new file mode 100644
index 000000000000..09216d566ab7
Binary files /dev/null and b/docs/reference/ml/images/ml-anomaly-alert-messages.png differ
diff --git a/docs/reference/ml/images/ml-anomaly-alert-severity.jpg b/docs/reference/ml/images/ml-anomaly-alert-severity.jpg
deleted file mode 100644
index dc6582ebbd84..000000000000
Binary files a/docs/reference/ml/images/ml-anomaly-alert-severity.jpg and /dev/null differ
diff --git a/docs/reference/ml/images/ml-anomaly-alert-severity.png b/docs/reference/ml/images/ml-anomaly-alert-severity.png
new file mode 100644
index 000000000000..9f15d464d2b6
Binary files /dev/null and b/docs/reference/ml/images/ml-anomaly-alert-severity.png differ
diff --git a/docs/reference/ml/images/ml-health-check-config.jpg b/docs/reference/ml/images/ml-health-check-config.jpg
deleted file mode 100644
index c235d7998452..000000000000
Binary files a/docs/reference/ml/images/ml-health-check-config.jpg and /dev/null differ
diff --git a/docs/reference/ml/images/ml-health-check-config.png b/docs/reference/ml/images/ml-health-check-config.png
new file mode 100644
index 000000000000..23a3833325bc
Binary files /dev/null and b/docs/reference/ml/images/ml-health-check-config.png differ
diff --git a/docs/reference/ml/images/ml-rule.jpg b/docs/reference/ml/images/ml-rule.jpg
deleted file mode 100644
index 44973e785401..000000000000
Binary files a/docs/reference/ml/images/ml-rule.jpg and /dev/null differ
diff --git a/docs/reference/ml/images/ml-rule.png b/docs/reference/ml/images/ml-rule.png
new file mode 100644
index 000000000000..1008dadf2b66
Binary files /dev/null and b/docs/reference/ml/images/ml-rule.png differ
diff --git a/docs/reference/ml/trained-models/apis/clear-trained-model-deployment-cache.asciidoc b/docs/reference/ml/trained-models/apis/clear-trained-model-deployment-cache.asciidoc
index 79d63848e66a..f0977430c9b0 100644
--- a/docs/reference/ml/trained-models/apis/clear-trained-model-deployment-cache.asciidoc
+++ b/docs/reference/ml/trained-models/apis/clear-trained-model-deployment-cache.asciidoc
@@ -8,8 +8,6 @@
 
 Clears a trained model deployment cache on all nodes where the trained model is assigned.
 
-beta::[]
-
 [[clear-trained-model-deployment-cache-request]]
 == {api-request-title}
 
diff --git a/docs/reference/ml/trained-models/apis/infer-trained-model-deployment.asciidoc b/docs/reference/ml/trained-models/apis/infer-trained-model-deployment.asciidoc
index 132d8e9de070..d92d74d894a3 100644
--- a/docs/reference/ml/trained-models/apis/infer-trained-model-deployment.asciidoc
+++ b/docs/reference/ml/trained-models/apis/infer-trained-model-deployment.asciidoc
@@ -46,8 +46,8 @@ Controls the amount of time to wait for {infer} results. Defaults to 10 seconds.
 `docs`::
 (Required, array)
 An array of objects to pass to the model for inference. The objects should
-contain a field matching your configured trained model input. Typically, the 
-field name is `text_field`. Currently, only a single value is allowed.
+contain a field matching your configured trained model input. Typically, the
+field name is `text_field`.
 
 ////
 [[infer-trained-model-deployment-results]]
@@ -62,7 +62,7 @@ field name is `text_field`. Currently, only a single value is allowed.
 [[infer-trained-model-deployment-example]]
 == {api-examples-title}
 
-The response depends on the task the model is trained for. If it is a text 
+The response depends on the task the model is trained for. If it is a text
 classification task, the response is the score. For example:
 
 [source,console]
@@ -123,7 +123,7 @@ The API returns in this case:
 ----
 // NOTCONSOLE
 
-Zero-shot classification tasks require extra configuration defining the class 
+Zero-shot classification tasks require extra configuration defining the class
 labels. These labels are passed in the zero-shot inference config.
 
 [source,console]
@@ -150,7 +150,7 @@ POST _ml/trained_models/model2/deployment/_infer
 --------------------------------------------------
 // TEST[skip:TBD]
 
-The API returns the predicted label and the confidence, as well as the top 
+The API returns the predicted label and the confidence, as well as the top
 classes:
 
 [source,console-result]
@@ -205,7 +205,7 @@ POST _ml/trained_models/model2/deployment/_infer
 --------------------------------------------------
 // TEST[skip:TBD]
 
-When the input has been truncated due to the limit imposed by the model's 
+When the input has been truncated due to the limit imposed by the model's
 `max_sequence_length` the `is_truncated` field appears in the response.
 
 [source,console-result]
diff --git a/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc b/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc
index 9f64a4a0e10d..ca68a1c78010 100644
--- a/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc
+++ b/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc
@@ -6,14 +6,12 @@
 <titleabbrev>Infer trained model</titleabbrev>
 ++++
 
-Evaluates a trained model. The model may be any supervised model either trained 
+Evaluates a trained model. The model may be any supervised model either trained
 by {dfanalytics} or imported.
 
-NOTE: For model deployments with caching enabled, results may be returned 
+NOTE: For model deployments with caching enabled, results may be returned
 directly from the {infer} cache.
 
-beta::[]
-
 [[infer-trained-model-request]]
 == {api-request-title}
 
@@ -51,9 +49,7 @@ Controls the amount of time to wait for {infer} results. Defaults to 10 seconds.
 (Required, array)
 An array of objects to pass to the model for inference. The objects should
 contain the fields matching your configured trained model input. Typically for
-NLP models, the field name is `text_field`. Currently for NLP models, only a
-single value is allowed. For {dfanalytics} or imported classification or
-regression models, more than one value is allowed.
+NLP models, the field name is `text_field`.
 
 //Begin inference_config
 `inference_config`::
@@ -106,7 +102,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-fill-mask]
 =====
 `num_top_classes`::::
 (Optional, integer)
-Number of top predicted tokens to return for replacing the mask token. Defaults 
+Number of top predicted tokens to return for replacing the mask token. Defaults
 to `0`.
 
 `results_field`::::
@@ -277,7 +273,7 @@ The maximum amount of words in the answer. Defaults to `15`.
 
 `num_top_classes`::::
 (Optional, integer)
-The number the top found answers to return. Defaults to `0`, meaning only the 
+The number the top found answers to return. Defaults to `0`, meaning only the
 best found answer is returned.
 
 `question`::::
@@ -374,7 +370,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-classific
 
 `num_top_classes`::::
 (Optional, integer)
-Specifies the number of top class predictions to return. Defaults to all classes 
+Specifies the number of top class predictions to return. Defaults to all classes
 (-1).
 
 `results_field`::::
@@ -886,7 +882,7 @@ POST _ml/trained_models/model2/_infer
 --------------------------------------------------
 // TEST[skip:TBD]
 
-When the input has been truncated due to the limit imposed by the model's 
+When the input has been truncated due to the limit imposed by the model's
 `max_sequence_length` the `is_truncated` field appears in the response.
 
 [source,console-result]
diff --git a/docs/reference/ml/trained-models/apis/put-trained-model-definition-part.asciidoc b/docs/reference/ml/trained-models/apis/put-trained-model-definition-part.asciidoc
index 323f6953061a..53cad5cffa37 100644
--- a/docs/reference/ml/trained-models/apis/put-trained-model-definition-part.asciidoc
+++ b/docs/reference/ml/trained-models/apis/put-trained-model-definition-part.asciidoc
@@ -8,8 +8,6 @@
 
 Creates part of a trained model definition.
 
-beta::[]
-
 [[ml-put-trained-model-definition-part-request]]
 == {api-request-title}
 
diff --git a/docs/reference/ml/trained-models/apis/put-trained-model-vocabulary.asciidoc b/docs/reference/ml/trained-models/apis/put-trained-model-vocabulary.asciidoc
index c252fcd8f7d5..154aad8350e0 100644
--- a/docs/reference/ml/trained-models/apis/put-trained-model-vocabulary.asciidoc
+++ b/docs/reference/ml/trained-models/apis/put-trained-model-vocabulary.asciidoc
@@ -9,8 +9,6 @@
 Creates a trained model vocabulary.
 This is supported only for natural language processing (NLP) models.
 
-beta::[]
-
 [[ml-put-trained-model-vocabulary-request]]
 == {api-request-title}
 
@@ -56,7 +54,7 @@ preference. Example: ["f o", "fo o"]. Must be provided for RoBERTa and BART styl
 The following example shows how to create a model vocabulary for a
 previously stored trained model configuration.
 
-[source,js]
+[source,console]
 --------------------------------------------------
 PUT _ml/trained_models/elastic__distilbert-base-uncased-finetuned-conll03-english/vocabulary
 {
@@ -67,7 +65,7 @@ PUT _ml/trained_models/elastic__distilbert-base-uncased-finetuned-conll03-englis
   ]
 }
 --------------------------------------------------
-// NOTCONSOLE
+// TEST[s/\.\.\./"[PAD]"/ skip:TBD]
 
 The API returns the following results:
 
diff --git a/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc b/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc
index 104d6f3f9e76..b3c1c2ddc488 100644
--- a/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc
+++ b/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc
@@ -8,8 +8,6 @@
 
 Starts a new trained model deployment.
 
-beta::[]
-
 [[start-trained-model-deployment-request]]
 == {api-request-title}
 
@@ -30,19 +28,19 @@ in an ingest pipeline or directly in the <<infer-trained-model>> API.
 Scaling inference performance can be achieved by setting the parameters
 `number_of_allocations` and `threads_per_allocation`.
 
-Increasing `threads_per_allocation` means more threads are used when an 
-inference request is processed on a node. This can improve inference speed for 
+Increasing `threads_per_allocation` means more threads are used when an
+inference request is processed on a node. This can improve inference speed for
 certain models. It may also result in improvement to throughput.
 
-Increasing `number_of_allocations` means more threads are used to process 
-multiple inference requests in parallel resulting in throughput improvement. 
-Each model allocation uses a number of threads defined by 
+Increasing `number_of_allocations` means more threads are used to process
+multiple inference requests in parallel resulting in throughput improvement.
+Each model allocation uses a number of threads defined by
 `threads_per_allocation`.
 
-Model allocations are distributed across {ml} nodes. All allocations assigned to 
-a node share the same copy of the model in memory. To avoid thread 
-oversubscription which is detrimental to performance, model allocations are 
-distributed in such a way that the total number of used threads does not surpass 
+Model allocations are distributed across {ml} nodes. All allocations assigned to
+a node share the same copy of the model in memory. To avoid thread
+oversubscription which is detrimental to performance, model allocations are
+distributed in such a way that the total number of used threads does not surpass
 the node's allocated processors.
 
 [[start-trained-model-deployment-path-params]]
@@ -57,9 +55,9 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-id]
 
 `cache_size`::
 (Optional, <<byte-units,byte value>>)
-The inference cache size (in memory outside the JVM heap) per node for the 
-model. The default value is the size of the model as reported by the 
-`model_size_bytes` field in the <<get-trained-models-stats>>. To disable the 
+The inference cache size (in memory outside the JVM heap) per node for the
+model. The default value is the size of the model as reported by the
+`model_size_bytes` field in the <<get-trained-models-stats>>. To disable the
 cache, `0b` can be provided.
 
 `number_of_allocations`::
@@ -73,17 +71,17 @@ The priority of the deployment. The default value is `normal`.
 There are two priority settings:
 +
 --
-* `normal`: Use this for deployments in production. The deployment allocations 
+* `normal`: Use this for deployments in production. The deployment allocations
 are distributed so that node processors are not oversubscribed.
-* `low`: Use this for testing model functionality. The intention is that these 
-deployments are not sent a high volume of input. The deployment is required to 
-have a single allocation with just one thread. Low priority deployments may be 
-assigned on nodes that already utilize all their processors but will be given a 
-lower CPU priority than normal deployments. Low priority deployments may be 
+* `low`: Use this for testing model functionality. The intention is that these
+deployments are not sent a high volume of input. The deployment is required to
+have a single allocation with just one thread. Low priority deployments may be
+assigned on nodes that already utilize all their processors but will be given a
+lower CPU priority than normal deployments. Low priority deployments may be
 unassigned in order to satisfy more allocations of normal priority deployments.
 --
 
-WARNING: Heavy usage of low priority deployments may impact performance of 
+WARNING: Heavy usage of low priority deployments may impact performance of
 normal priority deployments.
 
 `queue_capacity`::
@@ -91,20 +89,20 @@ normal priority deployments.
 Controls how many inference requests are allowed in the queue at a time.
 Every machine learning node in the cluster where the model can be allocated
 has a queue of this size; when the number of requests exceeds the total value,
-new requests are rejected with a 429 error. Defaults to 1024. Max allowed value 
+new requests are rejected with a 429 error. Defaults to 1024. Max allowed value
 is 1000000.
 
 `threads_per_allocation`::
 (Optional, integer)
-Sets the number of threads used by each model allocation during inference. This 
-generally increases the speed per inference request. The inference process is a 
-compute-bound process; `threads_per_allocations` must not exceed the number of 
-available allocated processors per node. Defaults to 1. Must be a power of 2. 
+Sets the number of threads used by each model allocation during inference. This
+generally increases the speed per inference request. The inference process is a
+compute-bound process; `threads_per_allocations` must not exceed the number of
+available allocated processors per node. Defaults to 1. Must be a power of 2.
 Max allowed value is 32.
 
 `timeout`::
 (Optional, time)
-Controls the amount of time to wait for the model to deploy. Defaults to 20 
+Controls the amount of time to wait for the model to deploy. Defaults to 30
 seconds.
 
 `wait_for`::
diff --git a/docs/reference/ml/trained-models/apis/stop-trained-model-deployment.asciidoc b/docs/reference/ml/trained-models/apis/stop-trained-model-deployment.asciidoc
index 6fb2b5bfa3c2..d5de2b61cfe2 100644
--- a/docs/reference/ml/trained-models/apis/stop-trained-model-deployment.asciidoc
+++ b/docs/reference/ml/trained-models/apis/stop-trained-model-deployment.asciidoc
@@ -8,8 +8,6 @@
 
 Stops a trained model deployment.
 
-beta::[]
-
 [[stop-trained-model-deployment-request]]
 == {api-request-title}
 
diff --git a/docs/reference/modules/cluster/misc.asciidoc b/docs/reference/modules/cluster/misc.asciidoc
index 83adaef9ec1a..024ca80bb5e3 100644
--- a/docs/reference/modules/cluster/misc.asciidoc
+++ b/docs/reference/modules/cluster/misc.asciidoc
@@ -1,8 +1,9 @@
 [[misc-cluster-settings]]
-==== Miscellaneous cluster settings
+=== Miscellaneous cluster settings
 
+[discrete]
 [[cluster-read-only]]
-===== Metadata
+==== Metadata
 
 An entire cluster may be set to read-only with the following setting:
 
@@ -21,9 +22,9 @@ WARNING: Don't rely on this setting to prevent changes to your cluster. Any
 user with access to the <<cluster-update-settings,cluster-update-settings>>
 API can make the cluster read-write again.
 
-
+[discrete]
 [[cluster-shard-limit]]
-===== Cluster shard limit
+==== Cluster shard limit
 
 There is a soft limit on the number of shards in a cluster, based on the number
 of nodes in the cluster. This is intended to prevent operations which may
@@ -101,8 +102,9 @@ number of shards for each node, use the
 setting.
 --
 
+[discrete]
 [[user-defined-data]]
-===== User-defined cluster metadata
+==== User-defined cluster metadata
 
 User-defined metadata can be stored and retrieved using the Cluster Settings API.
 This can be used to store arbitrary, infrequently-changing data about the cluster
@@ -127,8 +129,9 @@ metadata will be viewable by anyone with access to the
 <<cluster-get-settings,Cluster Get Settings>> API, and is recorded in the
 {es} logs.
 
+[discrete]
 [[cluster-max-tombstones]]
-===== Index tombstones
+==== Index tombstones
 
 The cluster state maintains index tombstones to explicitly denote indices that
 have been deleted. The number of tombstones maintained in the cluster state is
@@ -148,8 +151,9 @@ include::{es-repo-dir}/indices/dangling-indices-list.asciidoc[tag=dangling-index
 You can use the <<dangling-indices-api,Dangling indices API>> to manage
 this situation.
 
+[discrete]
 [[cluster-logger]]
-===== Logger
+==== Logger
 
 The settings which control logging can be updated <<dynamic-cluster-setting,dynamically>> with the
 `logger.` prefix. For instance, to increase the logging level of the
@@ -165,9 +169,9 @@ PUT /_cluster/settings
 }
 -------------------------------
 
-
+[discrete]
 [[persistent-tasks-allocation]]
-===== Persistent tasks allocation
+==== Persistent tasks allocation
 
 Plugins can create a kind of tasks called persistent tasks. Those tasks are
 usually long-lived tasks and are stored in the cluster state, allowing the
diff --git a/docs/reference/modules/cluster/shards_allocation.asciidoc b/docs/reference/modules/cluster/shards_allocation.asciidoc
index b9fd5cb868dd..42282e00e81d 100644
--- a/docs/reference/modules/cluster/shards_allocation.asciidoc
+++ b/docs/reference/modules/cluster/shards_allocation.asciidoc
@@ -35,7 +35,7 @@ one of the active allocation ids in the cluster state.
 `cluster.routing.allocation.node_concurrent_recoveries`::
      (<<dynamic-cluster-setting,Dynamic>>)
      A shortcut to set both `cluster.routing.allocation.node_concurrent_incoming_recoveries` and
-     `cluster.routing.allocation.node_concurrent_outgoing_recoveries`.
+     `cluster.routing.allocation.node_concurrent_outgoing_recoveries`. Defaults to 2.
 
 
 `cluster.routing.allocation.node_initial_primaries_recoveries`::
@@ -57,17 +57,18 @@ one of the active allocation ids in the cluster state.
 [[shards-rebalancing-settings]]
 ==== Shard rebalancing settings
 
-A cluster is _balanced_ when it has an equal number of shards on each node
-without having a concentration of shards from any index on any node. {es} runs
-an automatic process called _rebalancing_ which moves shards between the nodes
-in your cluster to improve its balance. Rebalancing obeys all other shard
-allocation rules such as <<cluster-shard-allocation-filtering,allocation
-filtering>> and <<forced-awareness,forced awareness>> which may prevent it from
-completely balancing the cluster. In that case, rebalancing strives to achieve
-the most balanced cluster possible within the rules you have configured. If you
-are using <<data-tiers,data tiers>> then {es} automatically applies allocation
-filtering rules to place each shard within the appropriate tier. These rules
-mean that the balancer works independently within each tier.
+A cluster is _balanced_ when it has an equal number of shards on each node, with
+all nodes needing equal resources, without having a concentration of shards from
+any index on any node. {es} runs an automatic process called _rebalancing_ which
+moves shards between the nodes in your cluster to improve its balance.
+Rebalancing obeys all other shard allocation rules such as
+<<cluster-shard-allocation-filtering,allocation filtering>> and
+<<forced-awareness,forced awareness>> which may prevent it from completely
+balancing the cluster. In that case, rebalancing strives to achieve the most
+balanced cluster possible within the rules you have configured. If you are using
+<<data-tiers,data tiers>> then {es} automatically applies allocation filtering
+rules to place each shard within the appropriate tier. These rules mean that the
+balancer works independently within each tier.
 
 You can use the following settings to control the rebalancing of shards across
 the cluster:
@@ -84,7 +85,6 @@ Enable or disable rebalancing for specific kinds of shards:
 * `none` -        No shard balancing of any kind are allowed for any indices.
 --
 
-
 `cluster.routing.allocation.allow_rebalance`::
 +
 --
@@ -98,13 +98,32 @@ Specify when shard rebalancing is allowed:
 --
 
 `cluster.routing.allocation.cluster_concurrent_rebalance`::
-      (<<dynamic-cluster-setting,Dynamic>>)
-      Allow to control how many concurrent shard rebalances are
-      allowed cluster wide. Defaults to `2`. Note that this setting
-      only controls the number of concurrent shard relocations due
-      to imbalances in the cluster. This setting does not limit shard
-      relocations due to <<cluster-shard-allocation-filtering,allocation
-      filtering>> or <<forced-awareness,forced awareness>>.
+(<<dynamic-cluster-setting,Dynamic>>)
+Defines the number of concurrent shard rebalances are allowed across the whole
+cluster. Defaults to `2`. Note that this setting only controls the number of
+concurrent shard relocations due to imbalances in the cluster. This setting does
+not limit shard relocations due to
+<<cluster-shard-allocation-filtering,allocation filtering>> or
+<<forced-awareness,forced awareness>>.
+
+`cluster.routing.allocation.type`::
++
+--
+Selects the algorithm used for computing the cluster balance. Defaults to
+`desired_balance` which selects the _desired balance allocator_. This allocator
+runs a background task which computes the desired balance of shards in the
+cluster. Once this background task completes, {es} moves shards to their
+desired locations.
+
+May also be set to `balanced` to select the legacy _balanced allocator_. This
+allocator was the default allocator in versions of {es} before 8.6.0. It runs
+in the foreground, preventing the master from doing other work in parallel. It
+works by selecting a small number of shard movements which immediately improve
+the balance of the cluster, and when those shard movements complete it runs
+again and selects another few shards to move. Since this allocator makes its
+decisions based only on the current state of the cluster, it will sometimes
+move a shard several times while balancing the cluster.
+--
 
 [[shards-rebalancing-heuristics]]
 ==== Shard balancing heuristics settings
@@ -114,28 +133,55 @@ of shards, and then moving shards between nodes to reduce the weight of the
 heavier nodes and increase the weight of the lighter ones. The cluster is
 balanced when there is no possible shard movement that can bring the weight of
 any node closer to the weight of any other node by more than a configurable
-threshold. The following settings allow you to control the details of these
-calculations.
+threshold.
+
+The weight of a node depends on the number of shards it holds and on the total
+estimated resource usage of those shards expressed in terms of the size of the
+shard on disk and the number of threads needed to support write traffic to the
+shard. {es} estimates the resource usage of shards belonging to data streams
+when they are created by a rollover. The estimated disk size of the new shard
+is the mean size of the other shards in the data stream. The estimated write
+load of the new shard is a weighted average of the actual write loads of recent
+shards in the data stream. Shards that do not belong to the write index of a
+data stream have an estimated write load of zero.
+
+The following settings control how {es} combines these values into an overall
+measure of each node's weight.
 
 `cluster.routing.allocation.balance.shard`::
-     (<<dynamic-cluster-setting,Dynamic>>)
-     Defines the weight factor for the total number of shards allocated on a node
-     (float). Defaults to `0.45f`. Raising this raises the tendency to
-     equalize the number of shards across all nodes in the cluster.
+(float, <<dynamic-cluster-setting,Dynamic>>)
+Defines the weight factor for the total number of shards allocated to each node.
+Defaults to `0.45f`. Raising this value increases the tendency of {es} to
+equalize the total number of shards across nodes ahead of the other balancing
+variables.
 
 `cluster.routing.allocation.balance.index`::
-     (<<dynamic-cluster-setting,Dynamic>>)
-     Defines the weight factor for the number of shards per index allocated
-      on a specific node (float). Defaults to `0.55f`. Raising this raises the
-      tendency to equalize the number of shards per index across all nodes in
-      the cluster.
+(float, <<dynamic-cluster-setting,Dynamic>>)
+Defines the weight factor for the number of shards per index allocated to each
+node. Defaults to `0.55f`. Raising this value increases the tendency of {es} to
+equalize the number of shards of each index across nodes ahead of the other
+balancing variables.
+
+`cluster.routing.allocation.balance.disk_usage`::
+(float, <<dynamic-cluster-setting,Dynamic>>)
+Defines the weight factor for balancing shards according to their predicted disk
+size in bytes. Defaults to `2e-11f`. Raising this value increases the tendency
+of {es} to equalize the total disk usage across nodes ahead of the other
+balancing variables.
+
+`cluster.routing.allocation.balance.write_load`::
+(float, <<dynamic-cluster-setting,Dynamic>>)
+Defines the weight factor for the write load of each shard, in terms of the
+estimated number of indexing threads needed by the shard. Defaults to `10.0f`.
+Raising this value increases the tendency of {es} to equalize the total write
+load across nodes ahead of the other balancing variables.
 
 `cluster.routing.allocation.balance.threshold`::
-     (<<dynamic-cluster-setting,Dynamic>>)
-     Minimal optimization value of operations that should be performed (non
-      negative float). Defaults to `1.0f`. Raising this will cause the cluster
-      to be less aggressive about optimizing the shard balance.
-
+(float, <<dynamic-cluster-setting,Dynamic>>)
+The minimum improvement in weight which triggers a rebalancing shard movement.
+Defaults to `1.0f`. Raising this value will cause {es} to stop rebalancing
+shards sooner, leaving the cluster in a more unbalanced state.
 
 NOTE: Regardless of the result of the balancing algorithm, rebalancing might
-not be allowed due to forced awareness or allocation filtering.
+not be allowed due to allocation rules such as forced awareness and allocation
+filtering.
diff --git a/docs/reference/modules/discovery/discovery-settings.asciidoc b/docs/reference/modules/discovery/discovery-settings.asciidoc
index c3410900896e..722b9382732f 100644
--- a/docs/reference/modules/discovery/discovery-settings.asciidoc
+++ b/docs/reference/modules/discovery/discovery-settings.asciidoc
@@ -201,7 +201,7 @@ Sets how long the master node waits for each cluster state update to be
 completely published to all nodes, unless `discovery.type` is set to
 `single-node`. The default value is `30s`. See <<cluster-state-publishing>>.
 
-`cluster.discovery_configuration_check.interval `::
+`cluster.discovery_configuration_check.interval`::
 (<<static-cluster-setting,Static>>)
 Sets the interval of some checks that will log warnings about an
 incorrect discovery configuration. The default value is `30s`.
diff --git a/docs/reference/modules/discovery/fault-detection.asciidoc b/docs/reference/modules/discovery/fault-detection.asciidoc
index f0cf3e4c18f0..66889e71d0ab 100644
--- a/docs/reference/modules/discovery/fault-detection.asciidoc
+++ b/docs/reference/modules/discovery/fault-detection.asciidoc
@@ -52,7 +52,7 @@ logs.
 * The master may appear busy due to frequent cluster state updates.
 
 To troubleshoot a cluster in this state, first ensure the cluster has a
-<<modules-discovery-troubleshooting,stable master>>. Next, focus on the nodes
+<<discovery-troubleshooting,stable master>>. Next, focus on the nodes
 unexpectedly leaving the cluster ahead of all other issues. It will not be
 possible to solve other issues until the cluster has a stable master node and
 stable node membership.
@@ -62,23 +62,33 @@ tools only offer a view of the state of the cluster at a single point in time.
 Instead, look at the cluster logs to see the pattern of behaviour over time.
 Focus particularly on logs from the elected master. When a node leaves the
 cluster, logs for the elected master include a message like this (with line
-breaks added for clarity):
+breaks added to make it easier to read):
 
 [source,text]
 ----
-[2022-03-21T11:02:35,513][INFO ][o.e.c.s.MasterService    ]
-    [instance-0000000000] node-left[
-        {instance-0000000004}{bfcMDTiDRkietFb9v_di7w}{aNlyORLASam1ammv2DzYXA}{172.27.47.21}{172.27.47.21:19054}{m}
-            reason: disconnected,
-        {tiebreaker-0000000003}{UNw_RuazQCSBskWZV8ID_w}{bltyVOQ-RNu20OQfTHSLtA}{172.27.161.154}{172.27.161.154:19251}{mv}
-            reason: disconnected
-        ], term: 14, version: 1653415, ...
+[2022-03-21T11:02:35,513][INFO ][o.e.c.c.NodeLeftExecutor] [instance-0000000000]
+    node-left: [{instance-0000000004}{bfcMDTiDRkietFb9v_di7w}{aNlyORLASam1ammv2DzYXA}{172.27.47.21}{172.27.47.21:19054}{m}]
+    with reason [disconnected]
 ----
 
-This message says that the `MasterService` on the elected master
-(`instance-0000000000`) is processing a `node-left` task. It lists the nodes
-that are being removed and the reasons for their removal. Other nodes may log
-similar messages, but report fewer details:
+This message says that the `NodeLeftExecutor` on the elected master
+(`instance-0000000000`) processed a `node-left` task, identifying the node that
+was removed and the reason for its removal. When the node joins the cluster
+again, logs for the elected master will include a message like this (with line
+breaks added to make it easier to read):
+
+[source,text]
+----
+[2022-03-21T11:02:59,892][INFO ][o.e.c.c.NodeJoinExecutor] [instance-0000000000]
+    node-join: [{instance-0000000004}{bfcMDTiDRkietFb9v_di7w}{UNw_RuazQCSBskWZV8ID_w}{172.27.47.21}{172.27.47.21:19054}{m}]
+    with reason [joining after restart, removed [24s] ago with reason [disconnected]]
+----
+
+This message says that the `NodeJoinExecutor` on the elected master
+(`instance-0000000000`) processed a `node-join` task, identifying the node that
+was added to the cluster and the reason for the task.
+
+Other nodes may log similar messages, but report fewer details:
 
 [source,text]
 ----
@@ -89,9 +99,10 @@ similar messages, but report fewer details:
     }, term: 14, version: 1653415, reason: Publication{term=14, version=1653415}
 ----
 
-Focus on the one from the `MasterService` which is only emitted on the elected
-master, since it contains more details. If you don't see the messages from the
-`MasterService`, check that:
+These messages are not especially useful for troubleshooting, so focus on the
+ones from the `NodeLeftExecutor` and `NodeJoinExecutor` which are only emitted
+on the elected master and which contain more details. If you don't see the
+messages from the `NodeLeftExecutor` and `NodeJoinExecutor`, check that:
 
 * You're looking at the logs for the elected master node.
 
@@ -104,18 +115,14 @@ start or stop following the elected master. You can use these messages to
 determine each node's view of the state of the master over time.
 
 If a node restarts, it will leave the cluster and then join the cluster again.
-When it rejoins, the `MasterService` will log that it is processing a
-`node-join` task. You can tell from the master logs that the node was restarted
-because the `node-join` message will indicate that it is
-`joining after restart`. In older {es} versions, you can also determine that a
-node restarted by looking at the second "ephemeral" ID in the `node-left` and
-subsequent `node-join` messages. This ephemeral ID is different each time the
-node starts up. If a node is unexpectedly restarting, you'll need to look at
-the node's logs to see why it is shutting down.
+When it rejoins, the `NodeJoinExecutor` will log that it processed a
+`node-join` task indicating that the node is `joining after restart`. If a node
+is unexpectedly restarting, look at the node's logs to see why it is shutting
+down.
 
 If the node did not restart then you should look at the reason for its
-departure in the `node-left` message, which is reported after each node. There
-are three possible reasons:
+departure more closely. Each reason has different troubleshooting steps,
+described below. There are three possible reasons:
 
 * `disconnected`: The connection from the master node to the removed node was
 closed.
@@ -134,6 +141,10 @@ control this mechanism.
 
 ===== Diagnosing `disconnected` nodes
 
+Nodes typically leave the cluster with reason `disconnected` when they shut
+down, but if they rejoin the cluster without restarting then there is some
+other problem.
+
 {es} is designed to run on a fairly reliable network. It opens a number of TCP
 connections between nodes and expects these connections to remain open forever.
 If a connection is closed then {es} will try and reconnect, so the occasional
@@ -190,10 +201,32 @@ logger.org.elasticsearch.cluster.coordination.LagDetector: DEBUG
 
 When this logger is enabled, {es} will attempt to run the
 <<cluster-nodes-hot-threads>> API on the faulty node and report the results in
-the logs on the elected master.
+the logs on the elected master. The results are compressed, encoded, and split
+into chunks to avoid truncation:
+
+[source,text]
+----
+[DEBUG][o.e.c.c.LagDetector      ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] [part 1]: H4sIAAAAAAAA/x...
+[DEBUG][o.e.c.c.LagDetector      ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] [part 2]: p7x3w1hmOQVtuV...
+[DEBUG][o.e.c.c.LagDetector      ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] [part 3]: v7uTboMGDbyOy+...
+[DEBUG][o.e.c.c.LagDetector      ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] [part 4]: 4tse0RnPnLeDNN...
+[DEBUG][o.e.c.c.LagDetector      ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] (gzip compressed, base64-encoded, and split into 4 parts on preceding log lines)
+----
+
+To reconstruct the output, base64-decode the data and decompress it using
+`gzip`. For instance, on Unix-like systems:
+
+[source,sh]
+----
+cat lagdetector.log | sed -e 's/.*://' | base64 --decode | gzip --decompress
+----
 
 ===== Diagnosing `follower check retry count exceeded` nodes
 
+Nodes sometimes leave the cluster with reason `follower check retry count
+exceeded` when they shut down, but if they rejoin the cluster without
+restarting then there is some other problem.
+
 {es} needs every node to respond to network messages successfully and
 reasonably quickly. If a node rejects requests or does not respond at all then
 it can be harmful to the cluster. If enough consecutive checks fail then the
@@ -340,3 +373,39 @@ checks are `transport_worker` and `cluster_coordination` threads, for which
 there should never be a long wait. There may also be evidence of long waits for
 threads in the {es} logs. Refer to <<modules-network-threading-model>> for more
 information.
+
+===== Diagnosing `ShardLockObtainFailedException` failures
+
+If a node leaves and rejoins the cluster then {es} will usually shut down and
+re-initialize its shards. If the shards do not shut down quickly enough then
+{es} may fail to re-initialize them due to a `ShardLockObtainFailedException`.
+
+To gather more information about the reason for shards shutting down slowly,
+configure the following logger:
+
+[source,yaml]
+----
+logger.org.elasticsearch.env.NodeEnvironment: DEBUG
+----
+
+When this logger is enabled, {es} will attempt to run the
+<<cluster-nodes-hot-threads>> API whenever it encounters a
+`ShardLockObtainFailedException`. The results are compressed, encoded, and
+split into chunks to avoid truncation:
+
+[source,text]
+----
+[DEBUG][o.e.e.NodeEnvironment    ] [master] hot threads while failing to obtain shard lock for [index][0] [part 1]: H4sIAAAAAAAA/x...
+[DEBUG][o.e.e.NodeEnvironment    ] [master] hot threads while failing to obtain shard lock for [index][0] [part 2]: p7x3w1hmOQVtuV...
+[DEBUG][o.e.e.NodeEnvironment    ] [master] hot threads while failing to obtain shard lock for [index][0] [part 3]: v7uTboMGDbyOy+...
+[DEBUG][o.e.e.NodeEnvironment    ] [master] hot threads while failing to obtain shard lock for [index][0] [part 4]: 4tse0RnPnLeDNN...
+[DEBUG][o.e.e.NodeEnvironment    ] [master] hot threads while failing to obtain shard lock for [index][0] (gzip compressed, base64-encoded, and split into 4 parts on preceding log lines)
+----
+
+To reconstruct the output, base64-decode the data and decompress it using
+`gzip`. For instance, on Unix-like systems:
+
+[source,sh]
+----
+cat shardlock.log | sed -e 's/.*://' | base64 --decode | gzip --decompress
+----
diff --git a/docs/reference/modules/indices/search-settings.asciidoc b/docs/reference/modules/indices/search-settings.asciidoc
index 17fe9ce08b54..e43ec076578d 100644
--- a/docs/reference/modules/indices/search-settings.asciidoc
+++ b/docs/reference/modules/indices/search-settings.asciidoc
@@ -6,23 +6,24 @@ limits.
 
 [[indices-query-bool-max-clause-count]]
 `indices.query.bool.max_clause_count`::
+deprecated:[8.0.0]
 (<<static-cluster-setting,Static>>, integer)
-Maximum number of clauses a query can contain. Defaults to `4096`.
+This deprecated setting has no effect.
 +
-This setting limits the total number of clauses that a query tree can have. The default of 4096
-is quite high and should normally be sufficient. This limit applies to the rewritten query, so
-not only `bool` queries can contribute high numbers of clauses, but also all queries that rewrite
-to `bool` queries internally such as `fuzzy` queries. The limit is in place to prevent searches
-from becoming too large, and taking up too much CPU and memory. In case you're considering
-increasing this setting, make sure you've exhausted all other options to avoid having to do this.
-Higher values can lead to performance degradations and memory issues, especially in clusters with
-a high load or few resources.
-
-Elasticsearch offers some tools to avoid running into issues with regards to the maximum number of
-clauses such as the <<query-dsl-terms-query,`terms`>> query, which allows querying many distinct
-values while still counting as a single clause, or the <<index-prefixes,`index_prefixes`>> option
-of <<text-field-type,`text`>> fields, which allows executing prefix queries that expand to a high
-number of terms as a single term query.
+{es} will now dynamically set the maximum number of allowed clauses in a query, using
+a heuristic based on the size of the search thread pool and the size of the heap allocated to
+the JVM. This limit has a minimum value of 1024 and will in most cases be larger (for example,
+a node with 30Gb RAM and 48 CPUs will have a maximum clause count of around 27,000). Larger
+heaps lead to higher values, and larger thread pools result in lower values.
++
+Queries with many clauses should be avoided whenever possible. If you previously bumped this
+setting to accommodate heavy queries, you might need to increase the amount of memory available
+to {es}, or to reduce the size of your search thread pool so that more memory is
+available to each concurrent search.
++
+In previous versions of Lucene you could get around this limit by nesting boolean queries
+within each other, but the limit is now based on the total number of leaf queries within the
+query as a whole and this workaround will no longer help.
 
 [[search-settings-max-buckets]]
 `search.max_buckets`::
diff --git a/docs/reference/monitoring/configuring-filebeat.asciidoc b/docs/reference/monitoring/configuring-filebeat.asciidoc
index 4288071c1ba4..196a9f080f2d 100644
--- a/docs/reference/monitoring/configuring-filebeat.asciidoc
+++ b/docs/reference/monitoring/configuring-filebeat.asciidoc
@@ -26,14 +26,6 @@ from impacting the performance of your production cluster. See
 
 --
 
-. Enable the collection of monitoring data on your cluster.
-+
---
-include::configuring-metricbeat.asciidoc[tag=enable-collection]
-
-For more information, see <<monitoring-settings>> and <<cluster-update-settings>>.
---
-
 . Identify which logs you want to monitor. 
 +
 --
diff --git a/docs/reference/monitoring/configuring-metricbeat.asciidoc b/docs/reference/monitoring/configuring-metricbeat.asciidoc
index 70c77c80bb34..3da47f584ee9 100644
--- a/docs/reference/monitoring/configuring-metricbeat.asciidoc
+++ b/docs/reference/monitoring/configuring-metricbeat.asciidoc
@@ -13,38 +13,6 @@ as described in <<collecting-monitoring-data>>.
 
 image::monitoring/images/metricbeat.png[Example monitoring architecture]
 
-. Enable the collection of monitoring data.
-+
---
-// tag::enable-collection[]
-Set `xpack.monitoring.collection.enabled` to `true` on the
-production cluster. By default, it is disabled (`false`). 
-
-You can use the following APIs to review and change this setting:
-
-[source,console]
-----------------------------------
-GET _cluster/settings
-----------------------------------
-
-[source,console]
-----------------------------------
-PUT _cluster/settings
-{
-  "persistent": {
-    "xpack.monitoring.collection.enabled": true
-  }
-}
-----------------------------------
-// TEST[warning:[xpack.monitoring.collection.enabled] setting was deprecated in Elasticsearch and will be removed in a future release.]
-
-If {es} {security-features} are enabled, you must have `monitor` cluster privileges to 
-view the cluster settings and `manage` cluster privileges to change them.
-// end::enable-collection[]
-
-For more information, see <<monitoring-settings>> and <<cluster-update-settings>>.
---
-
 . {metricbeat-ref}/metricbeat-installation-configuration.html[Install
 {metricbeat}]. Ideally install a single {metricbeat} instance configured with
 `scope: cluster` and configure `hosts` to point to an endpoint (e.g. a
@@ -187,28 +155,4 @@ For more information about these configuration options, see
 
 . {metricbeat-ref}/metricbeat-starting.html[Start {metricbeat}] on each node. 
 
-. Disable the default collection of {es} monitoring metrics.
-+
---
-Set `xpack.monitoring.elasticsearch.collection.enabled` to `false` on the 
-production cluster.
-
-You can use the following API to change this setting:
-
-[source,console]
-----------------------------------
-PUT _cluster/settings
-{
-  "persistent": {
-    "xpack.monitoring.elasticsearch.collection.enabled": false
-  }
-}
-----------------------------------
-// TEST[warning:[xpack.monitoring.elasticsearch.collection.enabled] setting was deprecated in Elasticsearch and will be removed in a future release.]
-
-If {es} {security-features} are enabled, you must have `monitor` cluster
-privileges to view the cluster settings and `manage` cluster privileges
-to change them.
---
-
 . {kibana-ref}/monitoring-data.html[View the monitoring data in {kib}]. 
diff --git a/docs/reference/query-dsl/script-score-query.asciidoc b/docs/reference/query-dsl/script-score-query.asciidoc
index b2843b92309e..08522708554f 100644
--- a/docs/reference/query-dsl/script-score-query.asciidoc
+++ b/docs/reference/query-dsl/script-score-query.asciidoc
@@ -145,7 +145,6 @@ A good default choice might be to use the `_seq_no`
 field, whose only drawback is that scores will change if the document is
 updated since update operations also update the value of the `_seq_no` field.
 
-
 [[decay-functions-numeric-fields]]
 ====== Decay functions for numeric fields
 You can read more about decay functions 
@@ -333,13 +332,13 @@ through a script:
 
 [[decay-functions]]
 ====== `decay` functions
-The `script_score` query has equivalent <<decay-functions, decay functions>>
-that can be used in script.
+The `script_score` query has equivalent <<decay-functions-numeric-fields, decay
+functions>> that can be used in scripts.
 
 include::{es-repo-dir}/vectors/vector-functions.asciidoc[]
 
 [[score-explanation]]
-====== Explain request
+===== Explain request
 Using an <<search-explain, explain request>> provides an explanation of how the parts of a score were computed. The `script_score` query can add its own explanation by setting the `explanation` parameter:
 
 [source,console]
diff --git a/docs/reference/query-dsl/wrapper-query.asciidoc b/docs/reference/query-dsl/wrapper-query.asciidoc
index b8b9626202e7..fe0d72684943 100644
--- a/docs/reference/query-dsl/wrapper-query.asciidoc
+++ b/docs/reference/query-dsl/wrapper-query.asciidoc
@@ -20,7 +20,4 @@ GET /_search
 
 <1> Base64 encoded string:  `{"term" : { "user.id" : "kimchy" }}`
 
-This query is more useful in the context of the Java high-level REST client or
-transport client to also accept queries as json formatted string.
-In these cases queries can be specified as a json or yaml formatted string or
-as a query builder (which is a available in the Java high-level REST client).
\ No newline at end of file
+This query is more useful in the context of Spring Data Elasticsearch. It's the way a user can add custom queries when using Spring Data repositories. The user can add a @Query() annotation to a repository method. When such a method is called we do a parameter replacement in the query argument of the annotation and then send this as the query part of a search request.
\ No newline at end of file
diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc
index 237d77d0a0c9..658a4f0a823f 100644
--- a/docs/reference/redirects.asciidoc
+++ b/docs/reference/redirects.asciidoc
@@ -1868,3 +1868,11 @@ For more information about reindexing from a remote cluster, refer to
 === Infer trained model deployment API
 
 See <<infer-trained-model>>.
+
+[role="exclude",id="watching-meetup-data"]
+=== Watching event data
+
+This example has been removed, as the Meetup.com streaming API has been
+discontinued.
+
+Refer to <<example-watches>> for other Watcher examples.
\ No newline at end of file
diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc
index b2af3c1db205..5b557a23c32b 100644
--- a/docs/reference/release-notes.asciidoc
+++ b/docs/reference/release-notes.asciidoc
@@ -6,7 +6,12 @@
 
 This section summarizes the changes in each release.
 
+
 * <<release-notes-8.7.0>>
+* <<release-notes-8.6.1>>
+* <<release-notes-8.6.0>>
+* <<release-notes-8.5.3>>
+* <<release-notes-8.5.2>>
 * <<release-notes-8.5.1>>
 * <<release-notes-8.5.0>>
 * <<release-notes-8.4.3>>
@@ -36,6 +41,10 @@ This section summarizes the changes in each release.
 --
 
 include::release-notes/8.7.0.asciidoc[]
+include::release-notes/8.6.1.asciidoc[]
+include::release-notes/8.6.0.asciidoc[]
+include::release-notes/8.5.3.asciidoc[]
+include::release-notes/8.5.2.asciidoc[]
 include::release-notes/8.5.1.asciidoc[]
 include::release-notes/8.5.0.asciidoc[]
 include::release-notes/8.4.3.asciidoc[]
diff --git a/docs/reference/release-notes/8.1.0.asciidoc b/docs/reference/release-notes/8.1.0.asciidoc
index 8cce7d90c03e..c1c44871d641 100644
--- a/docs/reference/release-notes/8.1.0.asciidoc
+++ b/docs/reference/release-notes/8.1.0.asciidoc
@@ -143,6 +143,9 @@ CRUD::
 Cluster Coordination::
 * Remove last few mentions of Zen discovery {es-pull}80410[#80410]
 
+Search::
+* Deprecate the `indices.query.bool.max_clause_count` node setting {es-pull}81525[#81525] (issue: {es-issue}46433[#46433])
+
 SQL::
 * Deprecate `index_include_frozen` request parameter {es-pull}83943[#83943] (issue: {es-issue}81939[#81939])
 
diff --git a/docs/reference/release-notes/8.4.0.asciidoc b/docs/reference/release-notes/8.4.0.asciidoc
index e83f9fb83472..daa6f623f771 100644
--- a/docs/reference/release-notes/8.4.0.asciidoc
+++ b/docs/reference/release-notes/8.4.0.asciidoc
@@ -3,6 +3,31 @@
 
 Also see <<breaking-changes-8.4,Breaking changes in 8.4>>.
 
+[[known-issues-8.4.0]]
+[float]
+=== Known issues
+
+// tag::ml-pre-7-datafeeds-known-issue[]
+* {ml-cap} {dfeeds} cannot be listed if any are not modified since version 6.x
++
+If you have a {dfeed} that was created in version 5.x or 6.x and has not
+been updated since 7.0, it is not possible to list {dfeeds} in 
+8.4 and 8.5. This means that {anomaly-jobs} cannot be managed using
+{kib}. This issue is fixed in 8.6.0.
++
+If you upgrade to 8.4 or 8.5 with such a {dfeed}, you need to
+work around the problem by updating each {dfeed}'s authorization information
+using https://support.elastic.dev/knowledge/view/b5a879db[these steps].
+// end::ml-pre-7-datafeeds-known-issue[]
+
+// tag::file-based-settings-deadlock-known-issue[]
+* Orchestrators which use a `settings.json` file to configure Elasticsearch may
+encounter deadlocks during master elections (issue: {es-issue}92812[#92812])
++
+To resolve the deadlock, remove the `settings.json` file and restart the
+affected node.
+// end::file-based-settings-deadlock-known-issue[]
+
 [[bug-8.4.0]]
 [float]
 === Bug fixes
diff --git a/docs/reference/release-notes/8.4.1.asciidoc b/docs/reference/release-notes/8.4.1.asciidoc
index c0ce679621fb..10f6f60010c3 100644
--- a/docs/reference/release-notes/8.4.1.asciidoc
+++ b/docs/reference/release-notes/8.4.1.asciidoc
@@ -3,14 +3,6 @@
 
 Also see <<breaking-changes-8.4,Breaking changes in 8.4>>.
 
-[[bug-8.4.1]]
-[float]
-=== Bug fixes
-
-Machine Learning::
-* [ML] Validate trained model deployment `queue_capacity` limit {es-pull}89611[#89611] (issue: {es-issue}89555[#89555])
-
-
 [[known-issues-8.4.1]]
 [float]
 === Known issues
@@ -18,3 +10,14 @@ Machine Learning::
 * When using date range search with format that does not have all date fields (missing month or day)
 an incorrectly parsed date could be used. The workaround is to use date pattern with all date fields (year, month, day)
 (issue: {es-issue}90187[#90187])
+
+include::8.4.0.asciidoc[tag=ml-pre-7-datafeeds-known-issue]
+
+include::8.4.0.asciidoc[tag=file-based-settings-deadlock-known-issue]
+
+[[bug-8.4.1]]
+[float]
+=== Bug fixes
+
+Machine Learning::
+* [ML] Validate trained model deployment `queue_capacity` limit {es-pull}89611[#89611] (issue: {es-issue}89555[#89555])
diff --git a/docs/reference/release-notes/8.4.2.asciidoc b/docs/reference/release-notes/8.4.2.asciidoc
index 2f6481876bd1..605c7d136179 100644
--- a/docs/reference/release-notes/8.4.2.asciidoc
+++ b/docs/reference/release-notes/8.4.2.asciidoc
@@ -20,6 +20,9 @@ This regression was fixed in version 8.4.3.
 an incorrectly parsed date could be used. The workaround is to use date pattern with all date fields (year, month, day)
 (issue: {es-issue}90187[#90187])
 
+include::8.4.0.asciidoc[tag=ml-pre-7-datafeeds-known-issue]
+
+include::8.4.0.asciidoc[tag=file-based-settings-deadlock-known-issue]
 
 [[bug-8.4.2]]
 [float]
diff --git a/docs/reference/release-notes/8.4.3.asciidoc b/docs/reference/release-notes/8.4.3.asciidoc
index e16c5006e7ff..7038091ab6be 100644
--- a/docs/reference/release-notes/8.4.3.asciidoc
+++ b/docs/reference/release-notes/8.4.3.asciidoc
@@ -3,6 +3,18 @@
 
 Also see <<breaking-changes-8.4,Breaking changes in 8.4>>.
 
+[[known-issues-8.4.3]]
+[float]
+=== Known issues
+
+* When using date range search with format that does not have all date fields (missing month or day)
+an incorrectly parsed date could be used. The workaround is to use date pattern with all date fields (year, month, day)
+(issue: {es-issue}90187[#90187])
+
+include::8.4.0.asciidoc[tag=ml-pre-7-datafeeds-known-issue]
+
+include::8.4.0.asciidoc[tag=file-based-settings-deadlock-known-issue]
+
 [[bug-8.4.3]]
 [float]
 === Bug fixes
@@ -19,12 +31,3 @@ Ingest Node::
 
 Ranking::
 * Ensure `cross_fields` always uses valid term statistics {es-pull}90314[#90314]
-
-
-[[known-issues-8.4.3]]
-[float]
-=== Known issues
-
-* When using date range search with format that does not have all date fields (missing month or day)
-an incorrectly parsed date could be used. The workaround is to use date pattern with all date fields (year, month, day)
-(issue: {es-issue}90187[#90187])
diff --git a/docs/reference/release-notes/8.5.0.asciidoc b/docs/reference/release-notes/8.5.0.asciidoc
index 1dbb44ca5f1d..50df64084340 100644
--- a/docs/reference/release-notes/8.5.0.asciidoc
+++ b/docs/reference/release-notes/8.5.0.asciidoc
@@ -3,6 +3,21 @@
 
 Also see <<breaking-changes-8.5,Breaking changes in 8.5>>.
 
+[[known-issues-8.5.0]]
+[float]
+=== Known issues
+
+* It is possible to inadvertently create an alias with the same name as an
+index in version 8.5.0. This action leaves the cluster in an invalid state in
+which several features will not work correctly, and it may not even be possible
+to restart nodes while in this state. Upgrade to 8.5.1 as soon as possible to
+avoid the risk of this occurring ({es-pull}91456[#91456]). If your cluster is
+affected by this issue, upgrade to 8.5.3 to repair it ({es-pull}91887[#91887]).
+
+include::8.4.0.asciidoc[tag=ml-pre-7-datafeeds-known-issue]
+
+include::8.4.0.asciidoc[tag=file-based-settings-deadlock-known-issue]
+
 [[breaking-8.5.0]]
 [float]
 === Breaking changes
@@ -319,12 +334,3 @@ Client::
 
 Packaging::
 * Upgrade bundled JDK to Java 19 {es-pull}90571[#90571]
-
-
-[[known-issues-8.5.0]]
-[float]
-=== Known issues
-
-* When using date range search with format that does not have all date fields (missing month or day)
-an incorrectly parsed date could be used. The workaround is to use date pattern with all date fields (year, month, day)
-(issue: {es-issue}90187[#90187])
diff --git a/docs/reference/release-notes/8.5.1.asciidoc b/docs/reference/release-notes/8.5.1.asciidoc
index 4bc3b4c54e41..9b326c760f7e 100644
--- a/docs/reference/release-notes/8.5.1.asciidoc
+++ b/docs/reference/release-notes/8.5.1.asciidoc
@@ -4,6 +4,14 @@
 
 Also see <<breaking-changes-8.5,Breaking changes in 8.5>>.
 
+[[known-issues-8.5.1]]
+[float]
+=== Known issues
+
+include::8.4.0.asciidoc[tag=ml-pre-7-datafeeds-known-issue]
+
+include::8.4.0.asciidoc[tag=file-based-settings-deadlock-known-issue]
+
 [[bug-8.5.1]]
 [float]
 === Bug fixes
diff --git a/docs/reference/release-notes/8.5.2.asciidoc b/docs/reference/release-notes/8.5.2.asciidoc
new file mode 100644
index 000000000000..147d7b9df890
--- /dev/null
+++ b/docs/reference/release-notes/8.5.2.asciidoc
@@ -0,0 +1,47 @@
+[[release-notes-8.5.2]]
+== {es} version 8.5.2
+
+
+Also see <<breaking-changes-8.5,Breaking changes in 8.5>>.
+
+[[known-issues-8.5.2]]
+[float]
+=== Known issues
+
+include::8.4.0.asciidoc[tag=ml-pre-7-datafeeds-known-issue]
+
+include::8.4.0.asciidoc[tag=file-based-settings-deadlock-known-issue]
+
+[[bug-8.5.2]]
+[float]
+=== Bug fixes
+
+Authorization::
+* Avoid potential unsupported operation exception in doc bitset cache {es-pull}91490[#91490]
+
+EQL::
+* Refine bwc version checks on `EqlSearchRequest` {es-pull}91510[#91510]
+
+Health::
+* SLM uneahlthy policies diagnosis recommends correct URL in action {es-pull}91506[#91506]
+
+Ingest Node::
+* Refactor `DatabaseNodeService` as a cluster state listener {es-pull}91567[#91567] (issue: {es-issue}86999[#86999])
+
+Stats::
+* Fix NPE in IndexService getNodeMappingStats {es-pull}91334[#91334] (issue: {es-issue}91259[#91259])
+
+Transform::
+* Fix failure when resolving indices from CCS {es-pull}91622[#91622] (issue: {es-issue}91550[#91550])
+
+[[enhancement-8.5.2]]
+[float]
+=== Enhancements
+
+EQL::
+* Remove version limitations for CCS {es-pull}91409[#91409]
+
+Ingest Node::
+* Refactor enrich maintenance coordination logic {es-pull}90931[#90931]
+
+
diff --git a/docs/reference/release-notes/8.5.3.asciidoc b/docs/reference/release-notes/8.5.3.asciidoc
new file mode 100644
index 000000000000..3e19cce9254b
--- /dev/null
+++ b/docs/reference/release-notes/8.5.3.asciidoc
@@ -0,0 +1,51 @@
+[[release-notes-8.5.3]]
+== {es} version 8.5.3
+
+Also see <<breaking-changes-8.5,Breaking changes in 8.5>>.
+
+[[known-issues-8.5.3]]
+[float]
+=== Known issues
+
+include::8.4.0.asciidoc[tag=ml-pre-7-datafeeds-known-issue]
+
+include::8.4.0.asciidoc[tag=file-based-settings-deadlock-known-issue]
+
+[[bug-8.5.3]]
+[float]
+=== Bug fixes
+
+Infra/Core::
+* Add `trace.id` to request trace logs {es-pull}91772[#91772] (issue: {es-issue}88174[#88174])
+* `DoPrivileged` in `ElasticsearchEncaughtExceptionHandler` and check modify thread {es-pull}91704[#91704] (issue: {es-issue}91650[#91650])
+
+Ingest Node::
+* Handle any exception thrown while generating source for an `IngestDocument` {es-pull}91981[#91981]
+
+Machine Learning::
+* ML stats failures should not stop the usage API working {es-pull}91917[#91917] (issue: {es-issue}91893[#91893])
+
+Stats::
+* Fix NPE in IndexService getNodeMappingStats {es-pull}91334[#91334] (issue: {es-issue}91259[#91259])
+
+Transform::
+* Fix failure when resolving indices from CCS {es-pull}91622[#91622] (issue: {es-issue}91550[#91550])
+
+[[enhancement-8.5.3]]
+[float]
+=== Enhancements
+
+Ingest Node::
+* Refactor enrich maintenance coordination logic {es-pull}90931[#90931]
+
+TLS::
+* Support SAN/dnsName for restricted trust {es-pull}91946[#91946]
+
+[[upgrade-8.5.3]]
+[float]
+=== Upgrades
+
+Engine::
+* Upgrade Lucene to version 9.4.2 {es-pull}91823[#91823]
+
+
diff --git a/docs/reference/release-notes/8.6.0.asciidoc b/docs/reference/release-notes/8.6.0.asciidoc
index dd1bfd4cd268..b268cd63534b 100644
--- a/docs/reference/release-notes/8.6.0.asciidoc
+++ b/docs/reference/release-notes/8.6.0.asciidoc
@@ -1,8 +1,275 @@
 [[release-notes-8.6.0]]
 == {es} version 8.6.0
 
-coming[8.6.0]
-
 Also see <<breaking-changes-8.6,Breaking changes in 8.6>>.
 
+[[known-issues-8.6.0]]
+[float]
+=== Known issues
+
+include::8.4.0.asciidoc[tag=file-based-settings-deadlock-known-issue]
+
+[[bug-8.6.0]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* GeoBoundsAggregations reject sub aggregations {es-pull}91073[#91073] (issue: {es-issue}91072[#91072])
+
+Allocation::
+* Avoid NPE when disassociateDeadNodes is executed for a node present in the desired balance {es-pull}91659[#91659]
+* Check `NodesShutdownMetadata` type before assuming restart {es-pull}90807[#90807]
+
+Authentication::
+* Fix time unit for connection request timeout of JWKs reload {es-pull}92080[#92080]
+* Improve performance for role mapping with DNs {es-pull}92074[#92074]
+* Improve robustness of `JwkSet` reloading {es-pull}92081[#92081]
+* Support stored authentication headers prior to version 6.7 {es-pull}92221[#92221]
+
+Authorization::
+* Make adding auth info to REST responses more robust {es-pull}92168[#92168]
+* Security remove datemath special handling {es-pull}91047[#91047]
+
+Cluster Coordination::
+* Fix `TransportMasterNodeAction` holding a CS reference needlessly {es-pull}90724[#90724] (issue: {es-issue}89220[#89220])
+* Include last-committed data in publication {es-pull}92259[#92259] (issue: {es-issue}90158[#90158])
+* Unsafe bootstrap memory optimization {es-pull}92493[#92493]
+
+EQL::
+* EQL sequences: support join on multi-values {es-pull}89965[#89965]
+
+Graph::
+* Fix potential issue with graph api's timed out field in response {es-pull}91006[#91006]
+
+Health::
+* Don't account for the unassigned reason when diagnosing NO_VALID_SHARD_COPY {es-pull}92416[#92416]
+* Fix NPE when evaluating the disk health for non-data nodes {es-pull}92643[#92643]
+* Use https in the short URLs for the `shards_availability` indicator {es-pull}92310[#92310]
+
+Indices APIs::
+* Trigger index settings providers when updating component templates {es-pull}91615[#91615] (issue: {es-issue}91592[#91592])
+
+Infra/Core::
+* Check reserved state in Metadata.isGlobalStateEquals {es-pull}92124[#92124]
+* Datastream unavailable exception metadata {es-pull}91461[#91461]
+* Fix `BytesRefArray` on append empty `BytesRef` {es-pull}91364[#91364]
+* Fix index expression options for requests with a single name or pattern {es-pull}91231[#91231]
+* Force init of Unbox in log4j {es-pull}92377[#92377] (issue: {es-issue}91964[#91964])
+* In file based settings, wait until security index is ready for role mappings {es-pull}92173[#92173] (issue: {es-issue}91939[#91939])
+* Index expression exclusions never trigger "not found" {es-pull}90902[#90902]
+* Update error states from inside the main state executor {es-pull}90346[#90346] (issue: {es-issue}90337[#90337])
+
+Infra/Scripting::
+* Fix compile with hex literals ending with d/f {es-pull}91501[#91501] (issue: {es-issue}88614[#88614])
+
+Ingest Node::
+* Fixing a race condition in `EnrichCoordinatorProxyAction` that can leave an item stuck in its queue {es-pull}90688[#90688] (issue: {es-issue}90598[#90598])
+
+Machine Learning::
+* Copy more settings when creating DF analytics destination index {es-pull}91546[#91546] (issue: {es-issue}89795[#89795])
+* Fix for 'No statistics' error message {ml-pull}2410[#2410]
+* Fix for 'No counts available' error message {ml-pull}2414[#2414]
+* Guard against input sequences that are too long for Question Answering models {es-pull}91924[#91924]
+* Improve performance of closing files before spawning {ml-pull}2424[#2424]
+* Skip remote clusters when performing up front privileges validation for datafeeds {es-pull}91895[#91895] (issue: {es-issue}87832[#87832])
+* Support fields with commas in data frame analytics `analyzed_fields` {es-pull}91710[#91710] (issue: {es-issue}72541[#72541])
+* Validate rule filters are present on open anomaly detection api {es-pull}92207[#92207]
+
+
+Mapping::
+* Consolidate field name validation when parsing mappings and documents {es-pull}91328[#91328]
+* Fix handling empty key case in the terms aggregation {es-pull}90822[#90822]
+
+Monitoring::
+* Fix logstash loadavg (xpack cases) {es-pull}90494[#90494]
+* [Stack Monitoring] Update ES module mappings {es-pull}90649[#90649]
+
+Network::
+* Clean up on exception while chunking XContent {es-pull}92024[#92024]
+* Fix Chunked APIs sending incorrect responses to HEAD requests {es-pull}92042[#92042] (issue: {es-issue}92032[#92032])
+* Reject connection attempts while closing {es-pull}92465[#92465]
+
+SQL::
+* Fix NPE on logging when not tracking total hits {es-pull}92425[#92425]
+
+Search::
+* Allow different decay values depending on the score function {es-pull}91195[#91195] (issue: {es-issue}78887[#78887])
+* Fix timing bug with DFS profiling {es-pull}92421[#92421]
+
+Snapshot/Restore::
+* Simplify and optimize deduplication of `RepositoryData` for a non-caching repository instance {es-pull}91851[#91851] (issue: {es-issue}89952[#89952])
+
+Store::
+* Fix numOpenOutputs and modCount in ByteSizeCachingDirectory {es-pull}92440[#92440] (issue: {es-issue}92434[#92434])
+
+Transform::
+* Skip remote clusters when performing up front privileges validation {es-pull}91788[#91788]
+
+Vector Search::
+* Make `knn` search requests fully cancellable {es-pull}90612[#90612]
+
+[[deprecation-8.6.0]]
+[float]
+=== Deprecations
+
+Allocation::
+* Deprecate state field in /_cluster/reroute response {es-pull}90399[#90399]
+* Ensure balance threshold is at least 1 {es-pull}92100[#92100]
+
+Ingest Node::
+* Deprecate 'remove_binary' default of false for ingest attachment processor {es-pull}90460[#90460]
+
+Mapping::
+* Deprecate silently ignoring type, fields, copy_to and boost in metadata field definition {es-pull}90989[#90989] (issue: {es-issue}35389[#35389])
+
+[[enhancement-8.6.0]]
+[float]
+=== Enhancements
+
+Allocation::
+* Clear up forecasted write load and shard size from previous write index during rollovers {es-pull}91590[#91590]
+* Forecast average shard size during rollovers {es-pull}91561[#91561]
+* Forecast write load during rollovers {es-pull}91425[#91425]
+* Improve shard balancing {es-pull}91603[#91603]
+* Introduce desired-balance allocator {es-pull}91343[#91343]
+* Limit shard realocation retries {es-pull}90296[#90296]
+* Prevalidate node removal API {es-pull}88952[#88952]
+* Set default `cluster.routing.allocation.balance.disk_usage` {es-pull}91951[#91951]
+* Store write load in the `IndexMetadata` during data streams rollovers {es-pull}91019[#91019]
+* Update the default `cluster.routing.allocation.balance.disk_usage` {es-pull}92065[#92065]
+* `DesiredBalance:` expose it via _internal/desired_balance {es-pull}91038[#91038] (issue: {es-issue}90583[#90583])
+
+Authorization::
+* [Fleet] Added logs-elastic_agent* read privileges to `kibana_system` {es-pull}91701[#91701]
+
+CRUD::
+* Keep track of average shard write load {es-pull}90768[#90768] (issue: {es-issue}90102[#90102])
+
+Geo::
+* Centroid aggregation for cartesian points and shapes {es-pull}89216[#89216] (issue: {es-issue}90156[#90156])
+* Improve H3#hexRing logic and add H3#areNeighborCells method {es-pull}91140[#91140]
+* Move SpatialUtils to geo library {es-pull}88088[#88088] (issue: {es-issue}86607[#86607])
+* Reduce number of object allocations in H3#geoToH3 and speed up computations {es-pull}91492[#91492]
+* Support `cartesian_bounds` aggregation on point and shape {es-pull}91298[#91298] (issue: {es-issue}90157[#90157])
+
+ILM+SLM::
+* ILM: Get policy support wildcard name {es-pull}89238[#89238]
+
+Infra/Core::
+* Handle APM global labels as affix setting {es-pull}91438[#91438] (issue: {es-issue}91278[#91278])
+* Improve date math exclusions in expressions {es-pull}90298[#90298]
+* Introduce a phase to use String.equals on constant strings, rather than def equality {es-pull}91362[#91362] (issue: {es-issue}91235[#91235])
+* More actionable error for ancient indices {es-pull}91243[#91243]
+* Operator/index templates {es-pull}90143[#90143]
+* Operator/ingest {es-pull}89735[#89735]
+* Transport threads and `_hot_threads` {es-pull}90482[#90482] (issue: {es-issue}90334[#90334])
+* Upgrade XContent to Jackson 2.14.0 and enable Fast Double Parser {es-pull}90553[#90553]
+
+Infra/Plugins::
+* Create placeholder plugin when loading stable plugins {es-pull}90870[#90870]
+* Example stable plugin {es-pull}90805[#90805]
+* Make `extendedPlugins,` `HasNativeController` and `moduleName` optional in plugin descriptor {es-pull}90835[#90835]
+* Rename `NamedComponent` name parameter to value {es-pull}91306[#91306]
+
+Infra/Scripting::
+* Use an explicit null check for null receivers in painless, rather than an NPE {es-pull}91347[#91347] (issue: {es-issue}91236[#91236])
+
+Machine Learning::
+* Add a filter parameter to frequent items {es-pull}91137[#91137]
+* Add a regex to the output of the `categorize_text` aggregation {es-pull}90723[#90723]
+* Add ability to filter and sort buckets by `change_point` numeric values {es-pull}91299[#91299]
+* Add api to update trained model deployment `number_of_allocations` {es-pull}90728[#90728]
+* Alias timestamp to @timestamp in anomaly detection results index {es-pull}90812[#90812]
+* Allow `model_aliases` to be used with Pytorch trained models {es-pull}91296[#91296]
+* Allow overriding timestamp field to null in file structure finder {es-pull}90764[#90764]
+* Audit a message every day the datafeed has seen no data {es-pull}91774[#91774]
+* Low priority trained model deployments {es-pull}91234[#91234] (issue: {es-issue}91024[#91024])
+* Provide additional information about anomaly score factors {es-pull}90675[#90675]
+
+Mapping::
+* Don't create IndexCaps objects when recording unmapped fields {es-pull}90806[#90806] (issue: {es-issue}90796[#90796])
+* aggregate metric double add a max min validation {es-pull}90381[#90381]
+
+Recovery::
+* Remove resize index settings once shards are started {es-pull}90391[#90391] (issue: {es-issue}90127[#90127])
+
+Rollup::
+* Test downsample runtime fields and security {es-pull}90593[#90593]
+
+Search::
+* Add LimitedOffsetsEnum to Limited offset token {es-pull}86110[#86110] (issue: {es-issue}86109[#86109])
+* Add profiling and documentation for dfs phase {es-pull}90536[#90536] (issue: {es-issue}89713[#89713])
+* Bulk merge field-caps responses using mapping hash {es-pull}86323[#86323]
+* Enhance nested depth tracking when parsing queries {es-pull}90425[#90425]
+* Expose telemetry about search usage {es-pull}91528[#91528]
+* Return docs when using nested mappings in archive indices {es-pull}90585[#90585] (issue: {es-issue}90523[#90523])
+* Use `IndexOrDocValues` query for IP range queries {es-pull}90303[#90303] (issue: {es-issue}83658[#83658])
+
+Snapshot/Restore::
+* Increase snaphot pool max size to 10 {es-pull}90282[#90282] (issue: {es-issue}89608[#89608])
+* Tie snapshot speed to node bandwidth settings {es-pull}91021[#91021] (issue: {es-issue}57023[#57023])
+
+Store::
+* Allow plugins to wrap Lucene directories created by the `IndexModule` {es-pull}91556[#91556]
+
+TLS::
+* Add certificate start/expiry dates to SSL Diagnostic message {es-pull}89461[#89461]
+
+TSDB::
+* Generate 'index.routing_path' from dynamic mapping templates {es-pull}90552[#90552] (issue: {es-issue}90528[#90528])
+* Support malformed numbers in synthetic `_source` {es-pull}90428[#90428]
+* Support synthetic `_source` for `_doc_count` field {es-pull}91465[#91465]
+* Synthetic _source: support `field` in many cases {es-pull}89950[#89950]
+* Synthetic `_source`: `ignore_malformed` for `ip` {es-pull}90038[#90038]
+* Synthetic `_source`: support `wildcard` field {es-pull}90196[#90196]
+
+Transform::
+* Add a health section to transform stats {es-pull}90760[#90760]
+* Support `aggregate_metric_double` field type in transform aggregations {es-pull}91045[#91045]
+
+Vector Search::
+* Add profiling information for knn vector queries {es-pull}90200[#90200]
+
+[[feature-8.6.0]]
+[float]
+=== New features
+
+Distributed::
+* Add "index" and "search" node roles with feature flag and setting {es-pull}90993[#90993]
+
+EQL::
+* EQL samples {es-pull}91312[#91312]
+
+Health::
+* Use chunked encoding for `RestGetHealthAction` {es-pull}91515[#91515] (issue: {es-issue}90223[#90223])
+* [HealthAPI] Use the `RestCancellableNodeClient` infrastructure {es-pull}91587[#91587]
+
+Machine Learning::
+* Make `categorize_text` aggregation GA {es-pull}88600[#88600]
+
+Vector Search::
+* Add fielddata and scripting support for byte-sized vectors {es-pull}91184[#91184]
+* Add support for indexing byte-sized knn vectors {es-pull}90774[#90774]
+
+[[regression-8.6.0]]
+[float]
+=== Regressions
+
+Infra/Core::
+* Revert "Remove `ImmutableOpenMap` from snapshot services" {es-pull}90287[#90287]
+
+[[upgrade-8.6.0]]
+[float]
+=== Upgrades
+
+Infra/Logging::
+* Upgrade to log4j 2.19.0 {es-pull}90589[#90589] (issue: {es-issue}90584[#90584])
+
+Network::
+* Upgrade to Netty 4.1.82.Final {es-pull}90604[#90604]
+* Upgrade to Netty 4.1.84 {es-pull}91271[#91271]
+
+Snapshot/Restore::
+* Upgrade GCS SDK to 2.13.1 {es-pull}92327[#92327]
+
 
diff --git a/docs/reference/release-notes/8.6.1.asciidoc b/docs/reference/release-notes/8.6.1.asciidoc
new file mode 100644
index 000000000000..a02142a5ee6a
--- /dev/null
+++ b/docs/reference/release-notes/8.6.1.asciidoc
@@ -0,0 +1,33 @@
+[[release-notes-8.6.1]]
+== {es} version 8.6.1
+
+Also see <<breaking-changes-8.6,Breaking changes in 8.6>>.
+
+[[bug-8.6.1]]
+[float]
+=== Bug fixes
+
+Data streams::
+* Fix wildcard expansion for delete-by-query on data streams {es-pull}92891[#92891]
+* Fix wildcard expansion for update-by-query on data streams {es-pull}92717[#92717] (issue: {es-issue}90272[#90272])
+
+Distributed::
+* Fix `ByteArrayIndexInput` with nonzero offset {es-pull}93205[#93205]
+
+ILM+SLM::
+* Get repository metadata from the cluster state doesn't throw an exception if a repo is missing {es-pull}92914[#92914]
+
+Infra/Core::
+* Don't announce ready until file settings are applied {es-pull}92856[#92856] (issue: {es-issue}92812[#92812])
+
+Machine Learning::
+* Utilise parallel allocations where the inference request contains multiple documents {es-pull}92359[#92359]
+
+Mapping::
+* Fix `_bulk` api `dynamic_templates` and explicit `op_type` {es-pull}92687[#92687]
+
+Search::
+* Avoid doing I/O when fetching min and max for keyword fields {es-pull}92026[#92026]
+* Reduce memory required for search responses when many shards are unavailable {es-pull}91365[#91365] (issue: {es-issue}90622[#90622])
+
+
diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc
index c514f86e4725..c0a06628631d 100644
--- a/docs/reference/release-notes/highlights.asciidoc
+++ b/docs/reference/release-notes/highlights.asciidoc
@@ -12,8 +12,8 @@ endif::[]
 // Add previous release to the list
 Other versions:
 
-//{ref-bare}/8.6/release-highlights.html[8.6]
-{ref-bare}/8.5/release-highlights.html[8.5]
+{ref-bare}/8.6/release-highlights.html[8.6]
+| {ref-bare}/8.5/release-highlights.html[8.5]
 | {ref-bare}/8.4/release-highlights.html[8.4]
 | {ref-bare}/8.3/release-highlights.html[8.3]
 | {ref-bare}/8.2/release-highlights.html[8.2]
diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc
index 39e8d52824bc..ca2d55c54333 100644
--- a/docs/reference/rest-api/common-parms.asciidoc
+++ b/docs/reference/rest-api/common-parms.asciidoc
@@ -126,17 +126,17 @@ tag::cluster-health-status[]
 Health status of the cluster, based on the state of its primary and replica
 shards. Statuses are:
 
-  `green`:::
-  All shards are assigned.
+  * `green`:
+    All shards are assigned.
 
-  `yellow`:::
-  All primary shards are assigned, but one or more replica shards are
-  unassigned. If a node in the cluster fails, some
-  data could be unavailable until that node is repaired.
+  * `yellow`:
+    All primary shards are assigned, but one or more replica shards are 
+    unassigned. If a node in the cluster fails, some data could be unavailable 
+    until that node is repaired.
 
-  `red`:::
-  One or more primary shards are unassigned, so some data is unavailable. This
-  can occur briefly during cluster startup as primary shards are assigned.
+  * `red`:
+    One or more primary shards are unassigned, so some data is unavailable. This
+    can occur briefly during cluster startup as primary shards are assigned.
 end::cluster-health-status[]
 
 tag::committed[]
@@ -713,7 +713,10 @@ end::payloads[]
 
 tag::pipeline[]
 `pipeline`::
-(Optional, string) ID of the pipeline to use to preprocess incoming documents.
+(Optional, string) ID of the pipeline to use to preprocess incoming documents. If the index has a
+default ingest pipeline specified, then setting the value to `_none` disables the default ingest
+pipeline for this request. If a final pipeline is configured it will always run, regardless of the
+value of this parameter.
 end::pipeline[]
 
 tag::pages-processed[]
diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc
index 13773b02fe41..8ae0ed5de5fd 100644
--- a/docs/reference/rest-api/usage.asciidoc
+++ b/docs/reference/rest-api/usage.asciidoc
@@ -400,6 +400,13 @@ GET /_xpack/usage
     "available" : true,
     "enabled" : true,
     "indices_count" : 0
+  },
+  "health_api" : {
+    "available" : true,
+    "enabled" : true,
+    "invocations": {
+      "total": 0
+    }
   }
 }
 ------------------------------------------------------------
@@ -408,6 +415,7 @@ GET /_xpack/usage
 // TESTRESPONSE[s/"eql" : \{[^\}]*\},/"eql" : $body.$_path,/]
 // TESTRESPONSE[s/"policy_stats" : \[[^\]]*\]/"policy_stats" : $body.$_path/]
 // TESTRESPONSE[s/"slm" : \{[^\}]*\},/"slm" : $body.$_path,/]
+// TESTRESPONSE[s/"health_api" : \{[^\}]*\}\s*\}/"health_api" : $body.$_path/]
 // TESTRESPONSE[s/ : true/ : $body.$_path/]
 // TESTRESPONSE[s/ : false/ : $body.$_path/]
 // TESTRESPONSE[s/ : (\-)?[0-9]+/ : $body.$_path/]
@@ -419,6 +427,7 @@ GET /_xpack/usage
 // 2. Handling eql, which is disabled by default on release builds and enabled
 //    everywhere else during the initial implementation phase until its release
 // 3. Ignore the contents of the `slm` object because it might contain policies
-// 4. All of the numbers and strings on the right hand side of *every* field in
+// 4. Ignore the contents of the `health_api` object because we do not know if the api has been called or not
+// 5. All of the numbers and strings on the right hand side of *every* field in
 //    the response are ignored. So we're really only asserting things about the
 //    the shape of this response, not the values in it.
diff --git a/docs/reference/search.asciidoc b/docs/reference/search.asciidoc
index 6da95f694728..3a147e6ad9bc 100644
--- a/docs/reference/search.asciidoc
+++ b/docs/reference/search.asciidoc
@@ -19,7 +19,6 @@ exception of the <<search-explain,explain API>>.
 * <<search-terms-enum>>
 * <<scroll-api>>
 * <<clear-scroll-api>>
-* <<semantic-search-api>>
 
 [discrete]
 [[search-testing-apis]]
@@ -53,8 +52,6 @@ include::search/point-in-time-api.asciidoc[]
 
 include::search/knn-search.asciidoc[]
 
-include::search/semantic-search.asciidoc[]
-
 include::search/scroll-api.asciidoc[]
 
 include::search/clear-scroll-api.asciidoc[]
diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc
index 30a554ec0b60..7369f5956ec4 100644
--- a/docs/reference/search/profile.asciidoc
+++ b/docs/reference/search/profile.asciidoc
@@ -97,7 +97,9 @@ The API returns the following result:
                   "create_weight": 4694895,
                   "shallow_advance": 0,
                   "create_weight_count": 1,
-                  "build_scorer": 7112295
+                  "build_scorer": 7112295,
+                  "count_weight": 0,
+                  "count_weight_count": 0
                 },
                 "children": [
                   {
@@ -122,7 +124,9 @@ The API returns the following result:
                       "create_weight": 2382719,
                       "shallow_advance": 9754,
                       "create_weight_count": 1,
-                      "build_scorer": 1355007
+                      "build_scorer": 1355007,
+                      "count_weight": 0,
+                      "count_weight_count": 0
                     }
                   },
                   {
@@ -147,7 +151,9 @@ The API returns the following result:
                       "create_weight": 130951,
                       "shallow_advance": 2512,
                       "create_weight_count": 1,
-                      "build_scorer": 46153
+                      "build_scorer": 46153,
+                      "count_weight": 0,
+                      "count_weight_count": 0
                     }
                   }
                 ]
@@ -386,7 +392,9 @@ Lucene execution:
   "create_weight": 4694895,
   "shallow_advance": 0,
   "create_weight_count": 1,
-  "build_scorer": 7112295
+  "build_scorer": 7112295,
+  "count_weight": 0,
+  "count_weight_count": 0
 }
 --------------------------------------------------
 // TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.$_path",\n"searches": [{\n"query": [{\n"type": "BooleanQuery",\n"description": "message:get message:search",\n"time_in_nanos": $body.$_path,/]
@@ -653,10 +661,12 @@ The API returns the following result:
                   "compute_max_score": 0,
                   "advance": 3942,
                   "advance_count": 4,
+                  "count_weight_count": 0,
                   "score": 0,
                   "build_scorer_count": 2,
                   "create_weight": 38380,
                   "shallow_advance": 0,
+                  "count_weight": 0,
                   "create_weight_count": 1,
                   "build_scorer": 99296
                 }
@@ -679,9 +689,11 @@ The API returns the following result:
                   "advance": 3552,
                   "advance_count": 1,
                   "score": 5027,
+                  "count_weight_count": 0,
                   "build_scorer_count": 2,
                   "create_weight": 107840,
                   "shallow_advance": 0,
+                  "count_weight": 0,
                   "create_weight_count": 1,
                   "build_scorer": 44215
                 }
@@ -967,7 +979,7 @@ overall time, the breakdown is inclusive of all children times.
 ===== Profiling Fetch
 
 All shards that fetched documents will have a `fetch` section in the profile.
-Let's execute a small search and have a look a the fetch profile:
+Let's execute a small search and have a look at the fetch profile:
 
 [source,console]
 ----
@@ -1236,7 +1248,8 @@ One of the `dfs.knn` sections for a shard looks like the following:
 [source,js]
 --------------------------------------------------
 "dfs" : {
-    "knn" : {
+    "knn" : [
+        {
         "query" : [
             {
                 "type" : "DocAndScoreQuery",
@@ -1260,7 +1273,9 @@ One of the `dfs.knn` sections for a shard looks like the following:
                   "create_weight" : 128879,
                   "shallow_advance" : 0,
                   "create_weight_count" : 1,
-                  "build_scorer" : 307595
+                  "build_scorer" : 307595,
+                  "count_weight": 0,
+                  "count_weight_count": 0
                 }
             }
         ],
@@ -1272,7 +1287,7 @@ One of the `dfs.knn` sections for a shard looks like the following:
                 "time_in_nanos" : 17163
             }
         ]
-    }
+    }   ]
 }
 --------------------------------------------------
 // TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.$_path",\n/]
diff --git a/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc b/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc
index 8206ec794eb0..4ca8acb1853b 100644
--- a/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc
+++ b/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc
@@ -1,16 +1,17 @@
-[cols="^,^,^,^,^,^,^,^,^,^,^"]
+[cols="^,^,^,^,^,^,^,^,^,^,^,^"]
 |====
-| 10+^h| Remote cluster version
+| 11+^h| Remote cluster version
 h| Local cluster version
-            |  6.8        | 7.1–7.16   | 7.17       | 8.0        | 8.1        | 8.2        | 8.3       | 8.4       | 8.5       |8.6
-| 6.8       |  {yes-icon} | {yes-icon} | {yes-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}
-| 7.1–7.16  |  {yes-icon} | {yes-icon} | {yes-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}
-| 7.17      |  {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon}
-| 8.0       |  {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon}
-| 8.1       |  {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon}
-| 8.2       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon}
-| 8.3       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon}|{yes-icon} | {yes-icon}| {yes-icon}
-| 8.4       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} |{yes-icon}| {yes-icon}| {yes-icon}
-| 8.5       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  |{yes-icon}| {yes-icon}| {yes-icon}
-| 8.6       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {yes-icon}| {yes-icon}
+            |  6.8        | 7.1–7.16   | 7.17       | 8.0        | 8.1        | 8.2        | 8.3       | 8.4       | 8.5       |8.6         |8.7
+| 6.8       |  {yes-icon} | {yes-icon} | {yes-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}
+| 7.1–7.16  |  {yes-icon} | {yes-icon} | {yes-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}
+| 7.17      |  {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon}
+| 8.0       |  {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon}
+| 8.1       |  {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon}
+| 8.2       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon}
+| 8.3       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon}|{yes-icon} | {yes-icon}| {yes-icon} | {yes-icon}
+| 8.4       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} |{yes-icon}| {yes-icon}| {yes-icon} | {yes-icon}
+| 8.5       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  |{yes-icon}| {yes-icon}| {yes-icon} | {yes-icon}
+| 8.6       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {yes-icon}| {yes-icon} | {yes-icon}
+| 8.7       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}| {no-icon} | {yes-icon} | {yes-icon}
 |====
\ No newline at end of file
diff --git a/docs/reference/search/search-your-data/highlighting.asciidoc b/docs/reference/search/search-your-data/highlighting.asciidoc
index 291ede6fcb44..8c2fa89bb341 100644
--- a/docs/reference/search/search-your-data/highlighting.asciidoc
+++ b/docs/reference/search/search-your-data/highlighting.asciidoc
@@ -948,7 +948,7 @@ highlighter is to find the best text fragments for the query, and highlight
 the query terms in the found fragments. For this, a highlighter needs to
 address several questions:
 
-- How break a text into fragments?
+- How to break a text into fragments?
 - How to find the best fragments among all fragments?
 - How to highlight the query terms in a fragment?
 
diff --git a/docs/reference/search/search-your-data/knn-search.asciidoc b/docs/reference/search/search-your-data/knn-search.asciidoc
index 6789ac4b9954..26ce7a12c115 100644
--- a/docs/reference/search/search-your-data/knn-search.asciidoc
+++ b/docs/reference/search/search-your-data/knn-search.asciidoc
@@ -151,7 +151,7 @@ page cache for it to be efficient. Please consult the
 configuration and sizing.
 
 To run an approximate kNN search, use the <<search-api-knn, `knn` option>>
-to search a `dense_vector` field with indexing enabled.
+to search one or more `dense_vector` fields with indexing enabled.
 
 . Explicitly map one or more `dense_vector` fields. Approximate kNN search
 requires the following mapping options:
@@ -176,6 +176,12 @@ PUT image-index
         "index": true,
         "similarity": "l2_norm"
       },
+      "title-vector": {
+        "type": "dense_vector",
+        "dims": 5,
+        "index": true,
+        "similarity": "l2_norm"
+      },
       "title": {
         "type": "text"
       },
@@ -194,11 +200,11 @@ PUT image-index
 ----
 POST image-index/_bulk?refresh=true
 { "index": { "_id": "1" } }
-{ "image-vector": [1, 5, -20], "title": "moose family", "file-type": "jpg" }
+{ "image-vector": [1, 5, -20], "title-vector": [12, 50, -10, 0, 1], "title": "moose family", "file-type": "jpg" }
 { "index": { "_id": "2" } }
-{ "image-vector": [42, 8, -15], "title": "alpine lake", "file-type": "png" }
+{ "image-vector": [42, 8, -15], "title-vector": [25, 1, 4, -12, 2], "title": "alpine lake", "file-type": "png" }
 { "index": { "_id": "3" } }
-{ "image-vector": [15, 11, 23], "title": "full moon", "file-type": "jpg" }
+{ "image-vector": [15, 11, 23], "title-vector": [1, 5, 25, 50, 20], "title": "full moon", "file-type": "jpg" }
 ...
 ----
 //TEST[continued]
@@ -406,6 +412,54 @@ over all documents that match the search. So for approximate kNN search, aggrega
 nearest documents. If the search also includes a `query`, then aggregations are calculated on the combined set of `knn`
 and `query` matches.
 
+[discrete]
+==== Search multiple kNN fields
+
+In addition to 'hybrid retrieval', you can search more than one kNN vector field at a time:
+
+[source,console]
+----
+POST image-index/_search
+{
+  "query": {
+    "match": {
+      "title": {
+        "query": "mountain lake",
+        "boost": 0.9
+      }
+    }
+  },
+  "knn": [ {
+    "field": "image-vector",
+    "query_vector": [54, 10, -2],
+    "k": 5,
+    "num_candidates": 50,
+    "boost": 0.1
+  },
+  {
+    "field": "title-vector",
+    "query_vector": [1, 20, -52, 23, 10],
+    "k": 10,
+    "num_candidates": 10,
+    "boost": 0.5
+  }],
+  "size": 10
+}
+----
+// TEST[continued]
+
+This search finds the global top `k = 5` vector matches for `image-vector` and the global `k = 10` for the `title-vector`.
+These top values are then combined with the matches from the `match` query and the top-10 documents are returned.
+The multiple `knn` entries and the `query` matches are combined through a disjunction,
+as if you took a boolean 'or' between them. The top `k` vector results represent the global nearest neighbors across
+all index shards.
+
+The scoring for a doc with the above configured boosts would be:
+
+```
+score = 0.9 * match_score + 0.1 * knn_score_image-vector + 0.5 * knn_score_title-vector
+```
+
 [discrete]
 [[knn-indexing-considerations]]
 ==== Indexing considerations
@@ -415,7 +469,7 @@ segment as an https://arxiv.org/abs/1603.09320[HNSW graph]. Indexing vectors for
 approximate kNN search can take substantial time because of how expensive it is
 to build these graphs. You may need to increase the client request timeout for
 index and bulk requests. The <<tune-knn-search, approximate kNN tuning guide>>
-contains important guidance around indexing performance, and how the the index
+contains important guidance around indexing performance, and how the index
 configuration can affect search performance.
 
 In addition to its search-time tuning parameters, the HNSW algorithm has
diff --git a/docs/reference/search/search.asciidoc b/docs/reference/search/search.asciidoc
index 67bf05b36f13..63eebb225e0e 100644
--- a/docs/reference/search/search.asciidoc
+++ b/docs/reference/search/search.asciidoc
@@ -483,14 +483,14 @@ A boost value greater than `1.0` increases the score. A boost value between
 experimental::[]
 [[search-api-knn]]
 `knn`::
-(Optional, object) 
+(Optional, object or array of objects)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn]
 +
 .Properties of `knn` object
 [%collapsible%open]
 ====
 `field`::
-(Required, string) 
+(Required, string)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-field]
 
 `filter`::
@@ -498,16 +498,22 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-field]
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-filter]
 
 `k`::
-(Required, integer) 
+(Required, integer)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-k]
 
 `num_candidates`::
-(Required, integer) 
+(Required, integer)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-num-candidates]
 
 `query_vector`::
-(Required, array of floats)
+(Optional, array of floats)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-query-vector]
+
+`query_vector_builder`::
+(Optional, object)
+A configuration object indicating how to build a query_vector before executing the request. You must provide
+a `query_vector_builder` or `query_vector`, but not both.
+
 ====
 
 [[search-api-min-score]]
diff --git a/docs/reference/search/semantic-search.asciidoc b/docs/reference/search/semantic-search.asciidoc
deleted file mode 100644
index 95a142d6d2a6..000000000000
--- a/docs/reference/search/semantic-search.asciidoc
+++ /dev/null
@@ -1,167 +0,0 @@
-[[semantic-search-api]]
-=== Semantic search API
-++++
-<titleabbrev>Semantic search</titleabbrev>
-++++
-
-experimental::[]
-Semantic search uses a text embedding NLP model to generate a dense vector from the input query string.
-The resulting dense vector is then used in a <<knn-search,k-nearest neighbor (knn) search>> against an index containing dense vectors
-created with the same text embedding model. The search results are semantically similar as learned
-by the model.
-
-[source,console]
-----
-GET my-index/_semantic_search
-{
-  "query_string": "A picture of a snow capped mountain",
-  "model_id": "my-text-embedding-model",
-  "knn": {
-    "field": "text_embedding",
-    "k": 10,
-    "num_candidates": 100
-  }
-}
-----
-// TEST[skip:TBD]
-
-
-[[semantic-search-api-request]]
-==== {api-request-title}
-
-`GET <target>/_semantic_search`
-
-`POST <target>/_semantic_search`
-
-[[semantic-search-api-prereqs]]
-==== {api-prereq-title}
-
-* If the {es} {security-features} are enabled, you must have the `read`
-<<privileges-list-indices,index privilege>> for the target data stream, index,
-or alias.
-
-[[semantic-search-api-desc]]
-==== {api-description-title}
-
-The semantic search API uses a text embedding model to create a dense vector
-representation of the query string.
-
-
-[[semantic-search-api-path-params]]
-==== {api-path-parms-title}
-
-`<target>`::
-(Optional, string) Comma-separated list of data streams, indices, and aliases
-to search. Supports wildcards (`*`). To search all data streams and indices,
-use `*` or `_all`.
-
-[role="child_attributes"]
-[[semantic-search-api-query-params]]
-==== {api-query-parms-title}
-
-include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing]
-
-[role="child_attributes"]
-[[semantic-search-api-request-body]]
-==== {api-request-body-title}
-
-`model_id`::
-(Required, string)
-include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-id]
-
-`query_string`::
-(Required, string) The input text to embed.
-
-`knn`::
-(Required, object)
-include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn]
-+
-.Properties of `knn` object
-[%collapsible%open]
-====
-`field`::
-(Required, string) 
-include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-field]
-
-`filter`::
-(Optional, <<query-dsl,Query DSL object>>)
-include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-filter]
-
-`k`::
-(Required, integer) 
-include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-k]
-
-`num_candidates`::
-(Required, integer) 
-include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-num-candidates]
-====
-
-`query`::
-(Optional, <<query-dsl,query object>>) Defines the search definition using the
-<<query-dsl,Query DSL>>.
-
-`text_embedding_config`::
-(Object, optional) Override certain setting of the text embedding model's 
-configuration.
-+
-.Properties of text_embedding inference
-[%collapsible%open]
-=====
-`results_field`::::
-(Optional, string)
-include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field]
-
-`tokenization`::::
-(Optional, object)
-include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization]
-+
-.Properties of tokenization
-[%collapsible%open]
-======
-`bert`::::
-(Optional, object)
-include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert]
-+
-.Properties of bert
-[%collapsible%open]
-=======
-`truncate`::::
-(Optional, string)
-include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate]
-=======
-`roberta`::::
-(Optional, object)
-include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta]
-+
-.Properties of roberta
-[%collapsible%open]
-=======
-`truncate`::::
-(Optional, string)
-include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate]
-=======
-`mpnet`::::
-(Optional, object)
-include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet]
-+
-.Properties of mpnet
-[%collapsible%open]
-=======
-`truncate`::::
-(Optional, string)
-include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate]
-=======
-======
-=====
-
-include::{es-repo-dir}/search/search.asciidoc[tag=docvalue-fields-def]
-include::{es-repo-dir}/search/search.asciidoc[tag=fields-param-def]
-include::{es-repo-dir}/search/search.asciidoc[tag=source-filtering-def]
-include::{es-repo-dir}/search/search.asciidoc[tag=stored-fields-def]
-
-[role="child_attributes"]
-[[semantic-search-api-response-body]]
-==== {api-response-body-title}
-
-The semantic search response has the same structure as a kNN search response.
-
diff --git a/docs/reference/search/suggesters/context-suggest.asciidoc b/docs/reference/search/suggesters/context-suggest.asciidoc
index 5baad38868ca..a925e532e50f 100644
--- a/docs/reference/search/suggesters/context-suggest.asciidoc
+++ b/docs/reference/search/suggesters/context-suggest.asciidoc
@@ -380,3 +380,9 @@ multiple context clauses. The following parameters are supported for a
     precision value can be a distance value (`5m`, `10km` etc.)
     or a raw geohash precision (`1`..`12`). Defaults to
     generating neighbours for index time precision level.
+
+NOTE: The precision field does not result in a distance match.
+Specifying a distance value like `10km` only results in a geohash precision value that represents tiles of that size.
+The precision will be used to encode the search geo point into a geohash tile for completion matching.
+A consequence of this is that points outside that tile, even if very close to the search point, will not be matched.
+Reducing the precision, or increasing the distance, can reduce the risk of this happening, but not entirely remove it.
diff --git a/docs/reference/searchable-snapshots/index.asciidoc b/docs/reference/searchable-snapshots/index.asciidoc
index 5f883535c49a..4a56961246c2 100644
--- a/docs/reference/searchable-snapshots/index.asciidoc
+++ b/docs/reference/searchable-snapshots/index.asciidoc
@@ -6,11 +6,11 @@ infrequently accessed and read-only data in a very cost-effective fashion. The
 <<cold-tier,cold>> and <<frozen-tier,frozen>> data tiers use {search-snaps} to
 reduce your storage and operating costs.
 
-{search-snaps-cap} eliminate the need for <<scalability,replica shards>>
-after rolling over from the hot tier, potentially halving the local storage needed to search 
-your data. {search-snaps-cap} rely on the same snapshot mechanism you already
-use for backups and have minimal impact on your snapshot repository storage
-costs.
+{search-snaps-cap} eliminate the need for <<scalability,replica shards>> after
+rolling over from the hot tier, potentially halving the local storage needed to
+search your data. {search-snaps-cap} rely on the same snapshot mechanism you
+already use for backups and have minimal impact on your snapshot repository
+storage costs.
 
 [discrete]
 [[using-searchable-snapshots]]
@@ -40,9 +40,9 @@ To mount an index from a snapshot that contains multiple indices, we recommend
 creating a <<clone-snapshot-api, clone>> of the snapshot that contains only the
 index you want to search, and mounting the clone. You should not delete a
 snapshot if it has any mounted indices, so creating a clone enables you to
-manage the lifecycle of the backup snapshot independently of any
-{search-snaps}. If you use {ilm-init} to manage your {search-snaps} then it
-will automatically look after cloning the snapshot as needed.
+manage the lifecycle of the backup snapshot independently of any {search-snaps}.
+If you use {ilm-init} to manage your {search-snaps} then it will automatically
+look after cloning the snapshot as needed.
 
 You can control the allocation of the shards of {search-snap} indices using the
 same mechanisms as for regular indices. For example, you could use
@@ -84,9 +84,9 @@ Use any of the following repository types with searchable snapshots:
 * <<snapshots-read-only-repository,Read-only HTTP and HTTPS repositories>>
 
 You can also use alternative implementations of these repository types, for
-instance <<repository-s3-client,MinIO>>,
-as long as they are fully compatible. Use the <<repo-analysis-api>> API
-to analyze your repository's suitability for use with searchable snapshots.
+instance <<repository-s3-client,MinIO>>, as long as they are fully compatible.
+Use the <<repo-analysis-api>> API to analyze your repository's suitability for
+use with searchable snapshots.
 // end::searchable-snapshot-repo-types[]
 
 [discrete]
@@ -122,40 +122,41 @@ performance characteristics and local storage footprints:
 
 [[fully-mounted]]
 Fully mounted index::
-Loads a full copy of the snapshotted index's shards onto node-local storage
-within the cluster. {ilm-init} uses this option in the `hot` and `cold` phases.
+Fully caches the snapshotted index's shards in the {es} cluster. {ilm-init} uses
+this option in the `hot` and `cold` phases.
 +
-Search performance for a fully mounted index is normally
-comparable to a regular index, since there is minimal need to access the
-snapshot repository. While recovery is ongoing, search performance may be
-slower than with a regular index because a search may need some data that has
-not yet been retrieved into the local copy. If that happens, {es} will eagerly
-retrieve the data needed to complete the search in parallel with the ongoing
-recovery. On-disk data is preserved across restarts, such that the node does
-not need to re-download data that is already stored on the node after a restart.
+Search performance for a fully mounted index is normally comparable to a regular
+index, since there is minimal need to access the snapshot repository. While
+recovery is ongoing, search performance may be slower than with a regular index
+because a search may need some data that has not yet been retrieved into the
+local cache. If that happens, {es} will eagerly retrieve the data needed to
+complete the search in parallel with the ongoing recovery. On-disk data is
+preserved across restarts, such that the node does not need to re-download data
+that is already stored on the node after a restart.
 +
 Indices managed by {ilm-init} are prefixed with `restored-` when fully mounted.
 
 [[partially-mounted]]
 Partially mounted index::
 Uses a local cache containing only recently searched parts of the snapshotted
-index's data. This cache has a fixed size and is shared across shards of partially
-mounted indices allocated on the same data node. {ilm-init} uses this option in the
-`frozen` phase.
+index's data. This cache has a fixed size and is shared across shards of
+partially mounted indices allocated on the same data node. {ilm-init} uses this
+option in the `frozen` phase.
 +
 If a search requires data that is not in the cache, {es} fetches the missing
 data from the snapshot repository. Searches that require these fetches are
-slower, but the fetched data is stored in the cache so that similar searches
-can be served more quickly in future. {es} will evict infrequently used data
-from the cache to free up space. The cache is cleared when a node is restarted.
+slower, but the fetched data is stored in the cache so that similar searches can
+be served more quickly in future. {es} will evict infrequently used data from
+the cache to free up space. The cache is cleared when a node is restarted.
 +
-Although slower than a fully mounted index or a regular index, a
-partially mounted index still returns search results quickly, even for
-large data sets, because the layout of data in the repository is heavily
-optimized for search. Many searches will need to retrieve only a small subset of
-the total shard data before returning results.
+Although slower than a fully mounted index or a regular index, a partially
+mounted index still returns search results quickly, even for large data sets,
+because the layout of data in the repository is heavily optimized for search.
+Many searches will need to retrieve only a small subset of the total shard data
+before returning results.
 +
-Indices managed by {ilm-init} are prefixed with `partial-` when partially mounted.
+Indices managed by {ilm-init} are prefixed with `partial-` when partially
+mounted.
 
 To partially mount an index, you must have one or more nodes with a shared cache
 available. By default, dedicated frozen data tier nodes (nodes with the
@@ -166,16 +167,16 @@ headroom of 100GB.
 Using a dedicated frozen tier is highly recommended for production use. If you
 do not have a dedicated frozen tier, you must configure the
 `xpack.searchable.snapshot.shared_cache.size` setting to reserve space for the
-cache on one or more nodes. Partially mounted indices
-are only allocated to nodes that have a shared cache.
+cache on one or more nodes. Partially mounted indices are only allocated to
+nodes that have a shared cache.
 
 [[searchable-snapshots-shared-cache]]
 `xpack.searchable.snapshot.shared_cache.size`::
 (<<static-cluster-setting,Static>>)
-Disk space reserved for the shared cache of partially mounted indices.
-Accepts a percentage of total disk space or an absolute <<byte-units,byte
-value>>. Defaults to `90%` of total disk space for dedicated frozen data tier
-nodes. Otherwise defaults to `0b`.
+Disk space reserved for the shared cache of partially mounted indices. Accepts a
+percentage of total disk space or an absolute <<byte-units,byte value>>.
+Defaults to `90%` of total disk space for dedicated frozen data tier nodes.
+Otherwise defaults to `0b`.
 
 `xpack.searchable.snapshot.shared_cache.size.max_headroom`::
 (<<static-cluster-setting,Static>>, <<byte-units,byte value>>)
@@ -189,8 +190,9 @@ To illustrate how these settings work in concert let us look at two examples
 when using the default values of the settings on a dedicated frozen node:
 
 * A 4000 GB disk will result in a shared cache sized at 3900 GB. 90% of 4000 GB
-is 3600 GB, leaving 400 GB headroom. The default `max_headroom` of 100 GB
-takes effect, and the result is therefore 3900 GB.
+is 3600 GB, leaving 400 GB headroom. The default `max_headroom` of 100 GB takes
+effect, and the result is therefore 3900 GB.
+
 * A 400 GB disk will result in a shared cache sized at 360 GB.
 
 You can configure the settings in `elasticsearch.yml`:
@@ -201,20 +203,20 @@ xpack.searchable.snapshot.shared_cache.size: 4TB
 ----
 
 IMPORTANT: You can only configure these settings on nodes with the
-<<data-frozen-node,`data_frozen`>> role. Additionally, nodes with a shared
-cache can only have a single <<path-settings,data path>>.
+<<data-frozen-node,`data_frozen`>> role. Additionally, nodes with a shared cache
+can only have a single <<path-settings,data path>>.
 
-{es} also uses a dedicated system index named `.snapshot-blob-cache` to speed
-up the recoveries of {search-snap} shards. This index is used as an additional
+{es} also uses a dedicated system index named `.snapshot-blob-cache` to speed up
+the recoveries of {search-snap} shards. This index is used as an additional
 caching layer on top of the partially or fully mounted data and contains the
 minimal required data to start the {search-snap} shards. {es} automatically
-deletes the documents that are no longer used in this index. This periodic
-clean up can be tuned using the following settings:
+deletes the documents that are no longer used in this index. This periodic clean
+up can be tuned using the following settings:
 
 `searchable_snapshots.blob_cache.periodic_cleanup.interval`::
 (<<dynamic-cluster-setting,Dynamic>>)
-The interval at which the periodic cleanup of the `.snapshot-blob-cache`
-index is scheduled. Defaults to every hour (`1h`).
+The interval at which the periodic cleanup of the `.snapshot-blob-cache` index
+is scheduled. Defaults to every hour (`1h`).
 
 `searchable_snapshots.blob_cache.periodic_cleanup.retention_period`::
 (<<dynamic-cluster-setting,Dynamic>>)
@@ -237,10 +239,10 @@ index. Defaults to `10m`.
 === Reduce costs with {search-snaps}
 
 In most cases, {search-snaps} reduce the costs of running a cluster by removing
-the need for replica shards  and for shard data to be copied between
-nodes. However, if it's particularly expensive to retrieve data from a snapshot
-repository in your environment, {search-snaps} may be more costly than
-regular indices. Ensure that the cost structure of your operating environment is
+the need for replica shards  and for shard data to be copied between nodes.
+However, if it's particularly expensive to retrieve data from a snapshot
+repository in your environment, {search-snaps} may be more costly than regular
+indices. Ensure that the cost structure of your operating environment is
 compatible with {search-snaps} before using them.
 
 [discrete]
@@ -250,7 +252,7 @@ compatible with {search-snaps} before using them.
 For resiliency, a regular index requires multiple redundant copies of each shard
 across multiple nodes. If a node fails, {es} uses the redundancy to rebuild any
 lost shard copies. A {search-snap} index doesn't require replicas. If a node
-containing a {search-snap} index fails, {es} can rebuild the lost shard copy
+containing a {search-snap} index fails, {es} can rebuild the lost shard cache
 from the snapshot repository.
 
 Without replicas, rarely-accessed {search-snap} indices require far fewer
@@ -264,11 +266,11 @@ only partially-mounted {search-snap} indices, requires even fewer resources.
 ==== Data transfer costs
 
 When a shard of a regular index is moved between nodes, its contents are copied
-from another node in your cluster. In many environments, the costs of moving data
-between nodes are significant, especially if running in a Cloud environment with
-nodes in different zones. In contrast, when mounting a {search-snap} index or
-moving one of its shards, the data is always copied from the snapshot repository.
-This is typically much cheaper.
+from another node in your cluster. In many environments, the costs of moving
+data between nodes are significant, especially if running in a Cloud environment
+with nodes in different zones. In contrast, when mounting a {search-snap} index
+or moving one of its shards, the data is always copied from the snapshot
+repository. This is typically much cheaper.
 
 WARNING: Most cloud providers charge significant fees for data transferred
 between regions and for data transferred out of their platforms. You should only
@@ -281,37 +283,49 @@ multiple clusters and use <<modules-cross-cluster-search,{ccs}>> or
 [[back-up-restore-searchable-snapshots]]
 === Back up and restore {search-snaps}
 
-You can use <<snapshots-take-snapshot,regular snapshots>> to back up a
-cluster containing {search-snap} indices. When you restore a snapshot
-containing {search-snap} indices, these indices are restored as {search-snap}
-indices again.
+You can use <<snapshots-take-snapshot,regular snapshots>> to back up a cluster
+containing {search-snap} indices. When you restore a snapshot containing
+{search-snap} indices, these indices are restored as {search-snap} indices
+again.
 
 Before you restore a snapshot containing a {search-snap} index, you must first
 <<snapshots-register-repository,register the repository>> containing the
 original index snapshot. When restored, the {search-snap} index mounts the
-original index snapshot from its original repository. If wanted, you
-can use separate repositories for regular snapshots and {search-snaps}.
+original index snapshot from its original repository. If wanted, you can use
+separate repositories for regular snapshots and {search-snaps}.
 
 A snapshot of a {search-snap} index contains only a small amount of metadata
 which identifies its original index snapshot. It does not contain any data from
 the original index. The restore of a backup will fail to restore any
 {search-snap} indices whose original index snapshot is unavailable.
 
-Because {search-snap} indices are not regular indices, it is not possible to
-use a <<snapshots-source-only-repository,source-only repository>> to take
-snapshots of {search-snap} indices.
+Because {search-snap} indices are not regular indices, it is not possible to use
+a <<snapshots-source-only-repository,source-only repository>> to take snapshots
+of {search-snap} indices.
 
 [discrete]
 [[searchable-snapshots-reliability]]
 === Reliability of {search-snaps}
 
 The sole copy of the data in a {search-snap} index is the underlying snapshot,
-stored in the repository. If the repository fails or corrupts the contents of
-the snapshot then the data is lost. Although {es} may have made copies of the
-data onto local storage, these copies may be incomplete and cannot be used to
-recover any data after a repository failure. You must make sure that your
-repository is reliable and protects against corruption of your data while it is
-at rest in the repository.
+stored in the repository. For example:
+
+* You cannot unregister a repository while any of the searchable snapshots it
+contains are mounted in {es}. You also cannot delete a snapshot if any of its
+indices are mounted as a searchable snapshot in the same cluster.
+
+* If you mount indices from snapshots held in a repository to which a different
+cluster has write access then you must make sure that the other cluster does not
+delete these snapshots.
+
+* If you delete a snapshot while it is mounted as a searchable snapshot then the
+data is lost. Similarly, if the repository fails or corrupts the contents of the
+snapshot then the data is lost.
+
+* Although {es} may have cached the data onto local storage, these caches may be
+incomplete and cannot be used to recover any data after a repository failure.
+You must make sure that your repository is reliable and protects against
+corruption of your data while it is at rest in the repository.
 
 The blob storage offered by all major public cloud providers typically offers
 very good protection against data loss or corruption. If you manage your own
diff --git a/docs/reference/settings/common-defs.asciidoc b/docs/reference/settings/common-defs.asciidoc
index 1fc9a8bfecf2..33e736c70046 100644
--- a/docs/reference/settings/common-defs.asciidoc
+++ b/docs/reference/settings/common-defs.asciidoc
@@ -149,20 +149,28 @@ Otherwise, it defaults to `jks`.
 end::ssl-truststore-type[]
 
 tag::ssl-verification-mode-values[]
-Controls the verification of certificates.
 +
-Valid values are:
-
- * `full`, which verifies that the provided certificate is signed by a trusted
-authority (CA) and also verifies that the server's hostname (or IP address)
-matches the names identified within the certificate.
- * `certificate`, which verifies that the provided certificate is signed by a
-trusted authority (CA), but does not perform any hostname verification.
- * `none`, which performs _no verification_ of the server's certificate. This
-mode disables many of the security benefits of SSL/TLS and should only be used
-after very careful consideration. It is primarily intended as a temporary
-diagnostic mechanism when attempting to resolve TLS errors; its use on
-production clusters is strongly discouraged.
+.Valid values
+[%collapsible%open]
+=====
+`full`::
+Validates that the provided certificate: has an issue date that's
+within the `not_before` and `not_after` dates; chains to a trusted Certificate
+Authority (CA); has a `hostname` or IP address that matches the names within
+the certificate.
+
+`certificate`::
+Validates the provided certificate and verifies that it's signed by a 
+trusted authority (CA), but doesn't check the certificate `hostname`.
+
+`none`::
+Performs no certificate validation.
 +
-The default value is `full`.
-end::ssl-verification-mode-values[]
+IMPORTANT: Setting certificate validation to `none` disables many security
+benefits of SSL/TLS, which is very dangerous. Only set this value if instructed
+by Elastic Support as a temporary diagnostic mechanism when attempting to
+resolve TLS errors.
+=====
++
+Defaults to `full`.
+end::ssl-verification-mode-values[]
\ No newline at end of file
diff --git a/docs/reference/settings/security-settings.asciidoc b/docs/reference/settings/security-settings.asciidoc
index 21d53aeebf89..a7c5bbcb1c40 100644
--- a/docs/reference/settings/security-settings.asciidoc
+++ b/docs/reference/settings/security-settings.asciidoc
@@ -45,6 +45,18 @@ starting {es} for the first time, which means that you must
 <<manually-configure-security,manually configure security>>.
 --
 
+`xpack.security.enrollment.enabled`::
+(<<static-cluster-setting,Static>>)
+Defaults to `false`. Controls enrollment (of nodes and {kib}) to a local node
+that's been <<configuring-stack-security, autoconfigured for security>>.
+When set to `true`, the local node can generate new enrollment tokens. Existing
+tokens can be used for enrollment if they are still valid.
++
+--
+The security autoconfiguration process will set this to `true` unless
+an administrator sets it to `false` before starting {es}.
+--
+
 `xpack.security.hide_settings`::
 (<<static-cluster-setting,Static>>)
 A comma-separated list of settings that are omitted from the results of the
@@ -1501,6 +1513,7 @@ This setting cannot be used with `ssl.truststore.password`.
 // tag::saml-ssl-verification-mode-tag[]
 `ssl.verification_mode` {ess-icon}::
 (<<static-cluster-setting,Static>>)
+Controls the verification of certificates.
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-verification-mode-values]
 // end::saml-ssl-verification-mode-tag[]
 
@@ -2010,13 +2023,24 @@ In addition to the <<ref-realm-settings,settings that are valid for all realms>>
 can specify the following settings.
 // end::jwt-description-tag[]
 
+// tag::jwt-token-type-tag[]
+`token_type` {ess-icon}::
+(<<static-cluster-setting,Static>>)
+The token type, `id_token` or `access_token`, that the JWT realm uses to verify
+incoming JWTs. Defaults to `id_token`.
+// end::jwt-token-type-tag[]
+
 // tag::jwt-allowed-audiences-tag[]
 `allowed_audiences` {ess-icon}::
 (<<static-cluster-setting,Static>>)
 A list of allowed JWT audiences that {es} should verify. {es} will only consume
 JWTs that were intended for any of these audiences, as denoted by the `aud`
-claim in the JWT). Examples of `aud` claim are `https://example.com/client1`
-and `other_service,elasticsearch`.
+claim in the JWT).
+The audiences are compared with exact string matches and do not support wildcards or regex.
+Examples of `aud` claim are `https://example.com/client1`
+and `other_service,elasticsearch`. When `token_type` is `access_token`, the audiences can
+be optionally denoted by a different claim in the JWT if `aud` does not exist.
+See also <<security-settings-jwt-fallback-claims-aud,`fallback_claims.aud`>>.
 // end::jwt-allowed-audiences-tag[]
 
 // tag::jwt-allowed-clock-skew-tag[]
@@ -2034,9 +2058,61 @@ Identifier is usually a case sensitive URL using the https scheme that contains
 scheme, host, and optionally, port number and path components and no query or
 fragment components. However, it can be any string. The value for this setting
 should be provided by your JWT Issuer.
+The issuer is compared with exact string matches and do not support wildcards or regex.
 Examples of `iss` claim are `https://example.com:8443/jwt` and `issuer123`.
 // end::jwt-allowed-issuer-tag[]
 
+// tag::jwt-allowed-subjects-tag[]
+`allowed_subjects` {ess-icon}::
+(<<static-cluster-setting,Static>>)
+A list of allowed JWT subjects that {es} should verify. {es} will only consume
+JWTs that were issued for any of these subjects, as denoted by the `sub`
+claim in the JWT.
+The subjects are compared with exact string matches and do not support wildcards or regex.
+Examples of `sub` claim are `https://example.com/user1`
+and `user_1,user2`.
+When `token_type` is `access_token`, this setting is mandatory and the subject can be
+optionally denoted by a different claim in the JWT if `sub` does not exist.
+See also <<security-settings-jwt-fallback-claims-sub,`fallback_claims.sub`>>.
+// end::jwt-allowed-subjects-tag[]
+
+// tag::jwt-fallback-claims-sub-tag[]
+[[security-settings-jwt-fallback-claims-sub]]
+`fallback_claims.sub` {ess-icon}::
+(<<static-cluster-setting,Static>>)
+The alternative claim to look for the subject information if the `sub` claim
+does not exist. It is configurable only when the `token_type` is `access_token`.
+The fallback is applied everywhere the `sub` claim is used.
+// end::jwt-fallback-claims-sub-tag[]
+
+// tag::jwt-fallback-claims-aud-tag[]
+[[security-settings-jwt-fallback-claims-aud]]
+`fallback_claims.aud` {ess-icon}::
+(<<static-cluster-setting,Static>>)
+The alternative claim to look for the audiences information if the `aud` claim
+does not exist. It is configurable only when the `token_type` is `access_token`.
+The fallback is applied everywhere the `aud` claim is used.
+// end::jwt-fallback-claims-aud-tag[]
+
+// tag::jwt-required-claims-tag[]
+`required_claims` {ess-icon}::
+(<<static-cluster-setting,Static>>)
+Additional claims and associated values that {es} should verify.
+This is a group setting that takes key/value pairs, where the key is a string
+and the value must be either a string or an array of strings.
+The values are compared with exact string matches and do not support wildcards or regex.
+
+For example:
+
+[source, yaml]
+------------------------------------------------------------
+xpack.security.authc.realms.jwt.jwt1:
+  required_claims:
+    token_use: "id"
+    versions: ["1.0", "2.0"]
+------------------------------------------------------------
+// end::jwt-required-claims-tag[]
+
 // tag::jwt-allowed-signature-algorithms-tag[]
 `allowed_signature_algorithms` {ess-icon}::
 (<<static-cluster-setting,Static>>)
diff --git a/docs/reference/settings/ssl-settings.asciidoc b/docs/reference/settings/ssl-settings.asciidoc
index 04aff7e7737b..8e139d836880 100644
--- a/docs/reference/settings/ssl-settings.asciidoc
+++ b/docs/reference/settings/ssl-settings.asciidoc
@@ -26,12 +26,16 @@ Defaults to +{client-auth-default}+.
 endif::client-auth-default[]
 endif::server[]
 
-ifdef::verifies[]
 +{ssl-prefix}.ssl.verification_mode+::
 (<<static-cluster-setting,Static>>)
-Controls the verification of certificates.
-include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-verification-mode-values]
+ifndef::verifies[]
+The SSL settings in `pass:a[{ssl-prefix}.ssl]` control a _server context_ for TLS, which
+defines the settings for the TLS connection. The use of `verification_mode` in
+a TLS _server_ is discouraged. 
 endif::verifies[]
+Defines how to verify the certificates presented by another party in the TLS 
+connection:
+include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-verification-mode-values]
 
 +{ssl-prefix}.ssl.cipher_suites+::
 (<<static-cluster-setting,Static>>)
diff --git a/docs/reference/setup/advanced-configuration.asciidoc b/docs/reference/setup/advanced-configuration.asciidoc
index 0fc562db2f2f..2a7ccc56742d 100644
--- a/docs/reference/setup/advanced-configuration.asciidoc
+++ b/docs/reference/setup/advanced-configuration.asciidoc
@@ -158,9 +158,7 @@ using the service manager. See <<windows-service>>.
 [[readiness-tcp-port]]
 ===== Enable the Elasticsearch TCP readiness port
 
-preview::["This functionality is in technical preview and may be changed or removed in a future release.
-It is intended for internal, experimental use. Features in technical preview are not subject to the support
-SLA of official GA features."]
+preview::[]
 
 If configured, a node can open a TCP port when the node is in a ready state. A node is deemed
 ready when it has successfully joined a cluster. In a single node configuration, the node is
diff --git a/docs/reference/setup/important-settings/gc-logging.asciidoc b/docs/reference/setup/important-settings/gc-logging.asciidoc
index 273ac3ca5bac..3534e1335c9f 100644
--- a/docs/reference/setup/important-settings/gc-logging.asciidoc
+++ b/docs/reference/setup/important-settings/gc-logging.asciidoc
@@ -37,7 +37,7 @@ Change the default GC log output location to `/opt/my-app/gc.log` by
 -Xlog:all=warning:stderr:utctime,level,tags
 
 # Enable GC logging to a custom location with a variety of options
--Xlog:gc*,gc+age=trace,safepoint:file=/opt/my-app/gc.log:utctime,pid,tags:filecount=32,filesize=64m
+-Xlog:gc*,gc+age=trace,safepoint:file=/opt/my-app/gc.log:utctime,level,pid,tags:filecount=32,filesize=64m
 ----
 
 Configure an {es} <<docker,Docker container>> to send GC debug logs to
diff --git a/docs/reference/setup/install/docker.asciidoc b/docs/reference/setup/install/docker.asciidoc
index 404e1f718c3b..52a62bb3aa89 100644
--- a/docs/reference/setup/install/docker.asciidoc
+++ b/docs/reference/setup/install/docker.asciidoc
@@ -96,7 +96,7 @@ endif::[]
 ifeval::["{release-state}"!="unreleased"]
 [source,sh,subs="attributes"]
 ----
-docker run --name es01 --net elastic -p 9200:9200 -p 9300:9300 -it {docker-image}
+docker run --name es01 --net elastic -p 9200:9200 -it {docker-image}
 ----
 
 endif::[]
@@ -403,14 +403,52 @@ sudo sysctl -w vm.max_map_count=262144
 
 ====== Windows with https://docs.docker.com/docker-for-windows/wsl[Docker Desktop WSL 2 backend]
 
-The `vm.max_map_count` setting must be set in the docker-desktop container:
+The `vm.max_map_count` setting must be set in the "docker-desktop" WSL instance before the
+ElasticSearch container will properly start. There are several ways to do this, depending
+on your version of Windows and your version of WSL.
+
+If you are on Windows 10 before version 22H2, or if you are on Windows 10 version 22H2 using the
+built-in version of WSL, you must either manually set it every time you restart Docker before starting
+your ElasticSearch container, or (if you do not wish to do so on every restart) you must globally set
+every WSL2 instance to have the `vm.max_map_count` changed. This is because these versions of WSL
+do not properly process the /etc/sysctl.conf file.
+
+To manually set it every time you reboot, you must run the following commands in a command prompt
+or PowerShell window every time you restart Docker:
 
 [source,sh]
 --------------------------------------------
-wsl -d docker-desktop
+wsl -d docker-desktop -u root
 sysctl -w vm.max_map_count=262144
 --------------------------------------------
 
+If you are on these versions of WSL and you do not want to have to run those commands every
+time you restart Docker, you can globally change every WSL distribution with this setting
+by modifying your %USERPROFILE%\.wslconfig as follows:
+
+[source,text]
+--------------------------------------------
+[wsl2]
+kernelCommandLine = "sysctl.vm.max_map_count=262144"
+--------------------------------------------
+
+This will cause all WSL2 VMs to have that setting assigned when they start.
+
+If you are on Windows 11, or Windows 10 version 22H2 and have installed the Microsoft Store
+version of WSL, you can modify the /etc/sysctl.conf within the "docker-desktop" WSL
+distribution, perhaps with commands like this:
+
+[source,sh]
+--------------------------------------------
+wsl -d docker-desktop -u root
+vi /etc/sysctl.conf
+--------------------------------------------
+
+and appending a line which reads:
+[source,text]
+--------------------------------------------
+vm.max_map_count = 262144
+--------------------------------------------
 
 ===== Configuration files must be readable by the `elasticsearch` user
 
diff --git a/docs/reference/setup/install/docker/docker-compose.yml b/docs/reference/setup/install/docker/docker-compose.yml
index 4c6ba48035b4..4b4ecf401b7d 100644
--- a/docs/reference/setup/install/docker/docker-compose.yml
+++ b/docs/reference/setup/install/docker/docker-compose.yml
@@ -84,7 +84,6 @@ services:
       - xpack.security.http.ssl.key=certs/es01/es01.key
       - xpack.security.http.ssl.certificate=certs/es01/es01.crt
       - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
-      - xpack.security.http.ssl.verification_mode=certificate
       - xpack.security.transport.ssl.enabled=true
       - xpack.security.transport.ssl.key=certs/es01/es01.key
       - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
@@ -124,7 +123,6 @@ services:
       - xpack.security.http.ssl.key=certs/es02/es02.key
       - xpack.security.http.ssl.certificate=certs/es02/es02.crt
       - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
-      - xpack.security.http.ssl.verification_mode=certificate
       - xpack.security.transport.ssl.enabled=true
       - xpack.security.transport.ssl.key=certs/es02/es02.key
       - xpack.security.transport.ssl.certificate=certs/es02/es02.crt
@@ -164,7 +162,6 @@ services:
       - xpack.security.http.ssl.key=certs/es03/es03.key
       - xpack.security.http.ssl.certificate=certs/es03/es03.crt
       - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
-      - xpack.security.http.ssl.verification_mode=certificate
       - xpack.security.transport.ssl.enabled=true
       - xpack.security.transport.ssl.key=certs/es03/es03.key
       - xpack.security.transport.ssl.certificate=certs/es03/es03.crt
diff --git a/docs/reference/setup/sysconfig/executable-jna-tmpdir.asciidoc b/docs/reference/setup/sysconfig/executable-jna-tmpdir.asciidoc
index 8aa41f2a9a94..0305a917da08 100644
--- a/docs/reference/setup/sysconfig/executable-jna-tmpdir.asciidoc
+++ b/docs/reference/setup/sysconfig/executable-jna-tmpdir.asciidoc
@@ -27,11 +27,21 @@ filesystem, or configure {es} to use a different location for its temporary
 directory by setting the <<es-tmpdir,`$ES_TMPDIR`>> environment variable. For
 instance:
 
+* If you are running {es} directly from a shell, set `$ES_TMPDIR` as follows:
++
 ["source","sh",subs="attributes"]
 --------------------------------------------
 export ES_TMPDIR=/usr/share/elasticsearch/tmp
 --------------------------------------------
 
+* If you are using `systemd` to run {es} as a service, add the following
+line to the `[Service]` section in a <<systemd,service override file>>:
++
+[source,text]
+--------------------------------------------
+Environment=ES_TMPDIR=/usr/share/elasticsearch/tmp
+--------------------------------------------
+
 If you need finer control over the location of these temporary files, you can
 also configure the path that JNA uses with the <<set-jvm-options,JVM flag>>
 `-Djna.tmpdir=<path>` and you can configure the path that `libffi` uses for its
diff --git a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc
index 3242fd2ee80f..649a904786f1 100644
--- a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc
+++ b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc
@@ -97,7 +97,7 @@ Don't expand wildcard patterns.
 `ignore_unavailable`::
 (Optional, Boolean)
 If `false`, the snapshot fails if any data stream or index in `indices` is
-missing or closed. If `true`, the snapshot ignores missing or closed data
+missing. If `true`, the snapshot ignores missing data
 streams and indices. Defaults to `false`.
 
 `include_global_state`::
diff --git a/docs/reference/snapshot-restore/index.asciidoc b/docs/reference/snapshot-restore/index.asciidoc
index aae149234de0..390f6664391b 100644
--- a/docs/reference/snapshot-restore/index.asciidoc
+++ b/docs/reference/snapshot-restore/index.asciidoc
@@ -199,8 +199,8 @@ contents of the repository then future snapshot or restore operations may fail,
 reporting corruption or other data inconsistencies, or may appear to succeed
 having silently lost some of your data.
 
-You may however safely <<snapshots-repository-backup,restore a repository from
-a backup>> as long as
+You may however safely <<snapshots-repository-backup,restore a repository from a
+backup>> as long as
 
 . The repository is not registered with {es} while you are restoring its
 contents.
@@ -208,6 +208,9 @@ contents.
 . When you have finished restoring the repository its contents are exactly as
 they were when you took the backup.
 
+If you no longer need any of the snapshots in a repository, unregister it from
+{es} before deleting its contents from the underlying storage.
+
 Additionally, snapshots may contain security-sensitive information, which you
 may wish to <<cluster-state-snapshots,store in a dedicated repository>>.
 
diff --git a/docs/reference/snapshot-restore/repository-s3.asciidoc b/docs/reference/snapshot-restore/repository-s3.asciidoc
index 097f60d36a5a..de72511010b5 100644
--- a/docs/reference/snapshot-restore/repository-s3.asciidoc
+++ b/docs/reference/snapshot-restore/repository-s3.asciidoc
@@ -143,9 +143,9 @@ settings belong in the `elasticsearch.yml` file.
 
 `read_timeout`::
 
-    The socket timeout for connecting to S3. The value should specify the unit.
-    For example, a value of `5s` specifies a 5 second timeout. The default value
-    is 50 seconds.
+    (<<time-units,time value>>) The maximum time {es} will wait to receive the next byte
+    of data over an established, open connection to the repository before it closes the
+    connection. The default value is 50 seconds.
 
 `max_retries`::
 
@@ -283,7 +283,7 @@ multiple deployments may share the same bucket.
 
 `chunk_size`::
 
-    Big files can be broken down into chunks during snapshotting if needed.
+    (<<byte-units,byte value>>) Big files can be broken down into chunks during snapshotting if needed.
     Specify the chunk size as a value and unit, for example:
     `1TB`, `1GB`, `10MB`. Defaults to the maximum size of a blob in the S3 which is `5TB`.
 
@@ -302,7 +302,8 @@ include::repository-shared-settings.asciidoc[]
 
 `buffer_size`::
 
-    Minimum threshold below which the chunk is uploaded using a single request.
+    (<<byte-units,byte value>>) Minimum threshold below which the chunk is
+    uploaded using a single request.
     Beyond this threshold, the S3 repository will use the
     https://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html[AWS
     Multipart Upload API] to split the chunk into several parts, each of
@@ -327,13 +328,14 @@ include::repository-shared-settings.asciidoc[]
 
     Sets the S3 storage class for objects stored in the snapshot repository.
     Values may be `standard`, `reduced_redundancy`, `standard_ia`, `onezone_ia`
-    and `intelligent_tiering`. Defaults to `standard`.
-    Changing this setting on an existing repository only affects the
-    storage class for newly created objects, resulting in a mixed usage of
-    storage classes. Additionally, S3 Lifecycle Policies can be used to manage
-    the storage class of existing objects. Due to the extra complexity with the
-    Glacier class lifecycle, it is not currently supported by this
-    repository type. For more information about the different classes, see
+    and `intelligent_tiering`. Defaults to `standard`. Changing this setting on
+    an existing repository only affects the storage class for newly created
+    objects, resulting in a mixed usage of storage classes. You may use an S3
+    Lifecycle Policy to adjust the storage class of existing objects in your
+    repository, but you must not transition objects to Glacier classes and you
+    must not expire objects. If you use Glacier storage classes or object
+    expiry then you may permanently lose access to your repository contents.
+    For more information about S3 storage classes, see
     https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html[AWS
     Storage Classes Guide]
 
diff --git a/docs/reference/snapshot-restore/restore-snapshot.asciidoc b/docs/reference/snapshot-restore/restore-snapshot.asciidoc
index 3aeac1c8556c..4fce5e99f449 100644
--- a/docs/reference/snapshot-restore/restore-snapshot.asciidoc
+++ b/docs/reference/snapshot-restore/restore-snapshot.asciidoc
@@ -289,14 +289,15 @@ the cluster.
 . Temporarily stop indexing and turn off the following features:
 +
 --
-* GeoIP database downloader
+* GeoIP database downloader and ILM history store
 +
 [source,console]
 ----
 PUT _cluster/settings
 {
   "persistent": {
-    "ingest.geoip.downloader.enabled": false
+    "ingest.geoip.downloader.enabled": false,
+    "indices.lifecycle.history_index_enabled": false
   }
 }
 ----
@@ -429,14 +430,15 @@ POST _snapshot/my_repository/my_snapshot_2099.05.06/_restore
 features you stopped:
 +
 --
-* GeoIP database downloader
+* GeoIP database downloader and ILM history store
 +
 [source,console]
 ----
 PUT _cluster/settings
 {
   "persistent": {
-    "ingest.geoip.downloader.enabled": true
+    "ingest.geoip.downloader.enabled": true,
+    "indices.lifecycle.history_index_enabled": true
   }
 }
 ----
diff --git a/docs/reference/sql/endpoints/odbc/installation.asciidoc b/docs/reference/sql/endpoints/odbc/installation.asciidoc
index ba5d99ecef19..3630e09ea160 100644
--- a/docs/reference/sql/endpoints/odbc/installation.asciidoc
+++ b/docs/reference/sql/endpoints/odbc/installation.asciidoc
@@ -11,8 +11,8 @@ The recommended installation platform is Windows 10 64 bit or Windows Server 201
 
 Before you install the {odbc} you need to meet the following prerequisites;
 
-* .NET Framework 4.0 full - https://www.microsoft.com/en-au/download/details.aspx?id=17718
-* Microsoft Visual C++ Redistributable for Visual Studio 2017 or later - https://support.microsoft.com/en-au/help/2977003/the-latest-supported-visual-c-downloads
+* .NET Framework 4.x full, latest - https://dotnet.microsoft.com/download/dotnet-framework
+* Microsoft Visual C++ Redistributable for Visual Studio 2017 or later - https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist
 - The 64 bit driver requires the x64 redistributable
 - The 32 bit driver requires the x86 or the x64 redistributable (the latter also installs the components needed for the 32 bit driver)
 * Elevated privileges (administrator) for the User performing the installation.
diff --git a/docs/reference/sql/security.asciidoc b/docs/reference/sql/security.asciidoc
index 7e75bfe08338..784778702591 100644
--- a/docs/reference/sql/security.asciidoc
+++ b/docs/reference/sql/security.asciidoc
@@ -24,13 +24,34 @@ PKI/X.509:: Use X.509 certificates to authenticate {es-sql} to {es}. For this, o
 [discrete]
 [[sql-security-permissions]]
 ==== Permissions (server-side)
-Lastly, one the server one need to add a few permissions to
-users so they can run SQL. To run SQL a user needs `read` and
+On the server, one needs to add a few permissions to
+users so they can run SQL. To run SQL, a user needs `read` and
 `indices:admin/get` permissions at minimum while some parts of 
-the API require `cluster:monitor/main`. 
+the API require `cluster:monitor/main`.
 
-The following example configures a role that can run SQL in JDBC querying the `test` and `bort`
-indices:
+You can add permissions by <<defining-roles,creating a role>>, and assigning
+that role to the user. Roles can be created using {kib}, an
+<<sql-role-api-example,API call>> or the <<sql-role-file-example,`roles.yml`
+configuration file>>. Using {kib} or the role management APIs is the preferred
+method for defining roles. File-based role management is useful if you want to
+define a role that doesn't need to change. You cannot use the role management
+APIs to view or edit a role defined in `roles.yml`. 
+
+[discrete]
+[[sql-role-api-example]]
+===== Add permissions with the role management APIs
+
+This example configures a role that can run SQL in JDBC querying the `test`
+index:
+
+include::{xes-repo-dir}/rest-api/security/create-roles.asciidoc[tag=sql-queries-permission]
+
+[discrete]
+[[sql-role-file-example]]
+===== Add permissions to `roles.yml`
+
+This example configures a role that can run SQL in JDBC querying the `test` and `bort`
+indices. Add the following to `roles.yml`:
 
 [source, yaml]
 --------------------------------------------------
diff --git a/docs/reference/tab-widgets/troubleshooting/data/restore-from-snapshot.asciidoc b/docs/reference/tab-widgets/troubleshooting/data/restore-from-snapshot.asciidoc
index 0a199bd6e48d..589965d8ab07 100644
--- a/docs/reference/tab-widgets/troubleshooting/data/restore-from-snapshot.asciidoc
+++ b/docs/reference/tab-widgets/troubleshooting/data/restore-from-snapshot.asciidoc
@@ -210,6 +210,21 @@ POST _snapshot/my_repository/snapshot-20200617/_restore
 <1> The indices to restore.
 +
 <2> We also want to restore the aliases.
++
+NOTE: If any <<feature-state,feature states>> need to be restored we'll need to specify them using the
+`feature_states` field and the indices that belong to the feature states we restore must not be specified under `indices`.
+The <<health-api, Health API>> returns both the `indices` and `feature_states` that need to be restored for the restore from snapshot diagnosis. e.g.:
++
+[source,console]
+----
+POST _snapshot/my_repository/snapshot-20200617/_restore
+{
+  "feature_states": [ "geoip" ],
+  "indices": "kibana_sample_data_flights,.ds-my-data-stream-2022.06.17-000001",
+  "include_aliases": true
+}
+----
+// TEST[skip:illustration purposes only]
 
 . Finally we can verify that the indices health is now `green` via the <<cat-indices,cat indices API>>.
 +
@@ -430,6 +445,21 @@ POST _snapshot/my_repository/snapshot-20200617/_restore
 <1> The indices to restore.
 +
 <2> We also want to restore the aliases.
++
+NOTE: If any <<feature-state,feature states>> need to be restored we'll need to specify them using the
+`feature_states` field and the indices that belong to the feature states we restore must not be specified under `indices`.
+The <<health-api, Health API>> returns both the `indices` and `feature_states` that need to be restored for the restore from snapshot diagnosis. e.g.:
++
+[source,console]
+----
+POST _snapshot/my_repository/snapshot-20200617/_restore
+{
+  "feature_states": [ "geoip" ],
+  "indices": "kibana_sample_data_flights,.ds-my-data-stream-2022.06.17-000001",
+  "include_aliases": true
+}
+----
+// TEST[skip:illustration purposes only]
 
 . Finally we can verify that the indices health is now `green` via the <<cat-indices,cat indices API>>.
 +
diff --git a/docs/reference/transform/api-quickref.asciidoc b/docs/reference/transform/api-quickref.asciidoc
index 2f46b11abcf2..e6a71f7e2b48 100644
--- a/docs/reference/transform/api-quickref.asciidoc
+++ b/docs/reference/transform/api-quickref.asciidoc
@@ -18,6 +18,7 @@ _transform/
 * <<reset-transform,Reset {transforms}>>
 * <<start-transform,Start {transforms}>>
 * <<stop-transform,Stop {transforms}>>
+* <<schedule-now-transform,Schedule Now {transforms}>>
 * <<update-transform,Update {transforms}>>
 
 For the full list, see <<transform-apis>>.
diff --git a/docs/reference/transform/apis/get-transform-stats.asciidoc b/docs/reference/transform/apis/get-transform-stats.asciidoc
index 5ae4cf4bd497..2a7ed1913546 100644
--- a/docs/reference/transform/apis/get-transform-stats.asciidoc
+++ b/docs/reference/transform/apis/get-transform-stats.asciidoc
@@ -152,18 +152,18 @@ that the {transform} is failing to keep up.
 `status`::
     (string) Health status of this transform. Statuses are:
 
-    `green`:::
-    The transform is healthy.
+   * `green`:
+      The transform is healthy.
 
-    `unknown`:::
-    The health of the transform could not be determined.
+    * `unknown`:
+      The health of the transform could not be determined.
 
-    `yellow`:::
-    The functionality of the transform is in a degraded state and may need remediation
-    to avoid the health becoming `red`.
+    * `yellow`:
+      The functionality of the transform is in a degraded state and may need 
+      remediation to avoid the health becoming `red`.
 
-    `red`:::
-    The transform is experiencing an outage or is unavailable for use.
+    * `red`:
+      The transform is experiencing an outage or is unavailable for use.
 
 `issues`::
     (Optional, array) If a non-healthy status is returned, contains a list of issues
diff --git a/docs/reference/transform/apis/index.asciidoc b/docs/reference/transform/apis/index.asciidoc
index e52daf0ad599..bd3a7588db44 100644
--- a/docs/reference/transform/apis/index.asciidoc
+++ b/docs/reference/transform/apis/index.asciidoc
@@ -15,6 +15,8 @@ include::reset-transform.asciidoc[leveloffset=+2]
 include::start-transform.asciidoc[leveloffset=+2]
 //STOP
 include::stop-transform.asciidoc[leveloffset=+2]
+//SCHEDULE_NOW
+include::schedule-now-transform.asciidoc[leveloffset=+2]
 //UPDATE-UPGRADE
 include::update-transform.asciidoc[leveloffset=+2]
 include::upgrade-transforms.asciidoc[leveloffset=+2]
diff --git a/docs/reference/transform/apis/schedule-now-transform.asciidoc b/docs/reference/transform/apis/schedule-now-transform.asciidoc
new file mode 100644
index 000000000000..9bc42b8b7d30
--- /dev/null
+++ b/docs/reference/transform/apis/schedule-now-transform.asciidoc
@@ -0,0 +1,63 @@
+[role="xpack"]
+[testenv="basic"]
+[[schedule-now-transform]]
+= Schedule Now {transform} API
+
+[subs="attributes"]
+++++
+<titleabbrev>Shedule Now {transform}</titleabbrev>
+++++
+
+Schedules now a {transform}.
+
+[[schedule-now-transform-request]]
+== {api-request-title}
+
+`POST _transform/<transform_id>/_schedule_now`
+
+[[schedule-now-transform-prereqs]]
+== {api-prereq-title}
+
+* Requires the `manage_transform` cluster privilege. This privilege is included
+in the `transform_admin` built-in role.
+
+[schedule-now-transform-desc]]
+== {api-description-title}
+
+If you _schedule_now a {transform}, it will process the new data instantly,
+without waiting for the configured `frequency` interval.
+After _schedule_now API is called, the transform will be processed again at
+`now + frequency` unless _schedule_now API is called again in the meantime.
+
+[[schedule-now-transform-path-parms]]
+== {api-path-parms-title}
+
+`<transform_id>`::
+(Required, string)
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id]
+
+[[schedule-now-transform-query-parms]]
+== {api-query-parms-title}
+
+`timeout`::
+(Optional, time)
+Period to wait for a response. If no response is received before the timeout
+expires, the request fails and returns an error. Defaults to `30s`.
+
+[[schedule-now-transform-examples]]
+== {api-examples-title}
+
+[source,console]
+--------------------------------------------------
+POST _transform/ecommerce_transform/_schedule_now
+--------------------------------------------------
+// TEST[skip:setup kibana sample data]
+
+When the {transform} is scheduled now, you receive the following results:
+
+[source,console-result]
+----
+{
+  "acknowledged" : true
+}
+----
diff --git a/docs/reference/transform/apis/start-transform.asciidoc b/docs/reference/transform/apis/start-transform.asciidoc
index 03f5bbe4d2f8..f4f99f0f3457 100644
--- a/docs/reference/transform/apis/start-transform.asciidoc
+++ b/docs/reference/transform/apis/start-transform.asciidoc
@@ -61,6 +61,11 @@ Identifier for the {transform}.
 [[start-transform-query-parms]]
 == {api-query-parms-title}
 
+`from`::
+(Optional, string) Restricts the set of transformed entities to those changed
+ after this time. Relative times like now-30d are supported.
+Only applicable for continuous transforms.
+
 `timeout`::
 (Optional, time)
 Period to wait for a response. If no response is received before the timeout
diff --git a/docs/reference/transform/apis/transform-apis.asciidoc b/docs/reference/transform/apis/transform-apis.asciidoc
index f98e192e495d..2912ffb1b17d 100644
--- a/docs/reference/transform/apis/transform-apis.asciidoc
+++ b/docs/reference/transform/apis/transform-apis.asciidoc
@@ -10,5 +10,6 @@
 * <<reset-transform>>
 * <<start-transform>>
 * <<stop-transform>>
+* <<schedule-now-transform>>
 * <<update-transform>>
 * <<upgrade-transforms>>
diff --git a/docs/reference/transform/transforms-at-scale.asciidoc b/docs/reference/transform/transforms-at-scale.asciidoc
index f4ad4fb112dc..f1d47c994324 100644
--- a/docs/reference/transform/transforms-at-scale.asciidoc
+++ b/docs/reference/transform/transforms-at-scale.asciidoc
@@ -116,6 +116,23 @@ example, greater than `2020-01-01T00:00:00`) to limit which historical indices
 are accessed. If you use a relative time value (for example, `now-30d`) then 
 this date range is re-evaluated at the point of each checkpoint execution.
 
+Consider using <<api-date-math-index-names,date math>> in your index names to 
+reduce the number of indices to resolve in your queries. Add a date pattern 
+- for example, `yyyy-MM-dd` - to your index names and use it to limit your query 
+to a specific date. The example below queries indices only from yesterday and 
+today: 
+
+[source,js]
+----------------------------------
+  "source": {
+    "index": [
+        "<mydata-{now/d-1d{yyyy-MM-dd}}*>", 
+        "<mydata-{now/d{yyyy-MM-dd}}*>"
+    ]
+  },
+----------------------------------
+// NOTCONSOLE
+
 
 [discrete]
 [[optimize-shading-strategy]]
diff --git a/docs/reference/troubleshooting/common-issues/circuit-breaker-errors.asciidoc b/docs/reference/troubleshooting/common-issues/circuit-breaker-errors.asciidoc
index ca815dd3c04d..fe79ef57b6ea 100644
--- a/docs/reference/troubleshooting/common-issues/circuit-breaker-errors.asciidoc
+++ b/docs/reference/troubleshooting/common-issues/circuit-breaker-errors.asciidoc
@@ -82,7 +82,7 @@ you've enabled fielddata and triggered the <<fielddata-circuit-breaker,fielddata
 circuit breaker>>, consider disabling it and using a `keyword` field instead.
 See <<fielddata>>.
 
-**Clear the fieldata cache**
+**Clear the fielddata cache**
 
 If you've triggered the fielddata circuit breaker and can't disable fielddata,
 use the <<indices-clearcache,clear cache API>> to clear the fielddata cache.
@@ -92,4 +92,4 @@ This may disrupt any in-flight searches that use fielddata.
 ----
 POST _cache/clear?fielddata=true
 ----
-// TEST[s/^/PUT my-index\n/]
\ No newline at end of file
+// TEST[s/^/PUT my-index\n/]
diff --git a/docs/reference/troubleshooting/snapshot/add-repository.asciidoc b/docs/reference/troubleshooting/snapshot/add-repository.asciidoc
index 63b84b5e91cb..dc2ce5a4bc25 100644
--- a/docs/reference/troubleshooting/snapshot/add-repository.asciidoc
+++ b/docs/reference/troubleshooting/snapshot/add-repository.asciidoc
@@ -3,7 +3,9 @@
 
 Multiple {es} deployments are writing to the same snapshot repository. {es} doesn't 
 support this configuration and only one cluster is allowed to write to the same 
-repository.
+repository. See <<snapshot-repository-contents>> for potential side-effects of
+corruption of the repository contents, which may not be resolved by the following
+guide.
 To remedy the situation mark the repository as read-only or remove it from all the
 other deployments, and re-add (recreate) the repository in the current deployment:
 
diff --git a/docs/reference/troubleshooting/troubleshooting-searches.asciidoc b/docs/reference/troubleshooting/troubleshooting-searches.asciidoc
index 3aca26671716..90a28058b6d6 100644
--- a/docs/reference/troubleshooting/troubleshooting-searches.asciidoc
+++ b/docs/reference/troubleshooting/troubleshooting-searches.asciidoc
@@ -297,4 +297,25 @@ used by the data stream.
 For static settings, you need to create a new index with the correct settings.
 Next, you can reindex the data into that index. For data streams, refer to
 <<change-static-index-setting-for-a-data-stream,Change a static index setting
-for a data stream>>.
\ No newline at end of file
+for a data stream>>.
+
+[discrete]
+[[troubleshooting-slow-searches]]
+=== Find slow queries
+
+<<index-modules-slowlog,Slow logs>> can help pinpoint slow performing search 
+requests. Enabling <<auditing-settings,audit logging>> on top can help determine 
+query source. Add the following settings to the `elasticsearch.yml` configuration file
+to trace queries. The resulting logging is verbose, so disable these settings when not 
+troubleshooting.
+
+[source,yaml]
+----
+xpack.security.audit.enabled: true
+xpack.security.audit.logfile.events.include: _all
+xpack.security.audit.logfile.events.emit_request_body: true
+----
+
+Refer to
+https://www.elastic.co/blog/advanced-tuning-finding-and-fixing-slow-elasticsearch-queries[Advanced
+tuning: finding and fixing slow Elasticsearch queries] for more information.
diff --git a/docs/reference/vectors/vector-functions.asciidoc b/docs/reference/vectors/vector-functions.asciidoc
index a492d8ad6ff2..e0ed85189c97 100644
--- a/docs/reference/vectors/vector-functions.asciidoc
+++ b/docs/reference/vectors/vector-functions.asciidoc
@@ -9,13 +9,20 @@ to limit the number of matched documents with a `query` parameter.
 
 This is the list of available vector functions and vector access methods:
 
-1. `cosineSimilarity` – calculates cosine similarity
-2. `dotProduct` – calculates dot product
-3. `l1norm` – calculates L^1^ distance
-4. `l2norm` - calculates L^2^ distance
-5. `doc[<field>].vectorValue` – returns a vector's value as an array of floats
-6. `doc[<field>].magnitude` – returns a vector's magnitude
+1. <<vector-functions-cosine,`cosineSimilarity`>> – calculates cosine similarity
+2. <<vector-functions-dot-product,`dotProduct`>> – calculates dot product
+3. <<vector-functions-l1,`l1norm`>> – calculates L^1^ distance
+4. <<vector-functions-l2,`l2norm`>> - calculates L^2^ distance
+5. <<vector-functions-accessing-vectors,`doc[<field>].vectorValue`>> – returns a vector's value as an array of floats
+6. <<vector-functions-accessing-vectors,`doc[<field>].magnitude`>> – returns a vector's magnitude
 
+NOTE: The recommended way to access dense vectors is through the
+`cosineSimilarity`, `dotProduct`, `l1norm` or `l2norm` functions. Please note
+however, that you should call these functions only once per script. For example,
+don’t use these functions in a loop to calculate the similarity between a
+document vector and multiple other vectors. If you need that functionality,
+reimplement these functions yourself by
+<<vector-functions-accessing-vectors,accessing vector values directly>>.
 
 Let's create an index with a `dense_vector` mapping and index a couple
 of documents into it.
@@ -54,6 +61,9 @@ POST my-index-000001/_refresh
 --------------------------------------------------
 // TESTSETUP
 
+[[vector-functions-cosine]]
+====== Cosine similarity
+
 The `cosineSimilarity` function calculates the measure of
 cosine similarity between a given query vector and document vectors.
 
@@ -90,6 +100,9 @@ GET my-index-000001/_search
 NOTE: If a document's dense vector field has a number of dimensions
 different from the query's vector, an error will be thrown.
 
+[[vector-functions-dot-product]]
+====== Dot product
+
 The `dotProduct` function calculates the measure of
 dot product between a given query vector and document vectors.
 
@@ -124,6 +137,9 @@ GET my-index-000001/_search
 
 <1> Using the standard sigmoid function prevents scores from being negative.
 
+[[vector-functions-l1]]
+====== L^1^ distance (Manhattan distance)
+
 The `l1norm` function calculates L^1^ distance
 (Manhattan distance) between a given query vector and
 document vectors.
@@ -163,6 +179,9 @@ we reversed the output from `l1norm` and `l2norm`. Also, to avoid
 division by 0 when a document vector matches the query exactly,
 we added `1` in the denominator.
 
+[[vector-functions-l2]]
+====== L^2^ distance (Euclidean distance)
+
 The `l2norm` function calculates L^2^ distance
 (Euclidean distance) between a given query vector and
 document vectors.
@@ -193,10 +212,13 @@ GET my-index-000001/_search
 }
 --------------------------------------------------
 
-NOTE: If a document doesn't have a value for a vector field on which
-a vector function is executed, an error will be thrown.
+[[vector-functions-missing-values]]
+====== Checking for missing values
+
+If a document doesn't have a value for a vector field on which a vector function
+is executed, an error will be thrown.
 
-You can check if a document has a value for the field `my_vector` by
+You can check if a document has a value for the field `my_vector` with
 `doc['my_vector'].size() == 0`. Your overall script can look like this:
 
 [source,js]
@@ -205,9 +227,10 @@ You can check if a document has a value for the field `my_vector` by
 --------------------------------------------------
 // NOTCONSOLE
 
-The recommended way to access dense vectors is through `cosineSimilarity`,
-`dotProduct`, `l1norm` or `l2norm` functions. But for custom use cases,
-you can access dense vectors's values directly through the following functions:
+[[vector-functions-accessing-vectors]]
+====== Accessing vectors directly
+
+You can access vector values directly through the following functions:
 
 - `doc[<field>].vectorValue` – returns a vector's value as an array of floats
 
diff --git a/docs/src/test/cluster/config/userdict_ko.txt b/docs/src/test/cluster/config/userdict_ko.txt
index 63c1c3a1e224..b70fb43a0092 100644
--- a/docs/src/test/cluster/config/userdict_ko.txt
+++ b/docs/src/test/cluster/config/userdict_ko.txt
@@ -1,5 +1,5 @@
 # Additional nouns
 c++
-C샤프
+C쁠쁠
 세종
-세종시 세종 시
\ No newline at end of file
+세종시 세종 시
diff --git a/docs/src/yamlRestTest/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java b/docs/src/yamlRestTest/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
index db2834f4abed..3754ca033998 100644
--- a/docs/src/yamlRestTest/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
+++ b/docs/src/yamlRestTest/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.yaml.ClientYamlDocsTestClient;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
@@ -43,7 +44,6 @@
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 
 import static java.util.Collections.emptyMap;
@@ -316,7 +316,7 @@ public void execute(ClientYamlTestExecutionContext executionContext) throws IOEx
             Object previousSecond = null;
             while (firstTokens.hasNext()) {
                 if (false == secondTokens.hasNext()) {
-                    fail(String.format(Locale.ROOT, """
+                    fail(Strings.format("""
                         %s has fewer tokens than %s. %s has [%s] but %s is out of tokens. \
                         %s's last token was [%s] and %s's last token was' [%s]
                         """, second, first, first, firstTokens.next(), second, first, previousFirst, second, previousSecond));
@@ -327,7 +327,7 @@ public void execute(ClientYamlTestExecutionContext executionContext) throws IOEx
                 String secondText = (String) secondToken.get("token");
                 // Check the text and produce an error message with the utf8 sequence if they don't match.
                 if (false == secondText.equals(firstText)) {
-                    fail(String.format(Locale.ROOT, """
+                    fail(Strings.format("""
                         text differs: %s was [%s] but %s was [%s]. In utf8 those are
                         %s and
                         %s
@@ -339,7 +339,7 @@ public void execute(ClientYamlTestExecutionContext executionContext) throws IOEx
                 previousSecond = secondToken;
             }
             if (secondTokens.hasNext()) {
-                fail(String.format(Locale.ROOT, """
+                fail(Strings.format("""
                     %s has more tokens than %s. %s has [%s] but %s is out of tokens. \
                     %s's last token was [%s] and %s's last token was [%s]
                     """, second, first, second, secondTokens.next(), first, first, previousFirst, second, previousSecond));
diff --git a/gradle/build.versions.toml b/gradle/build.versions.toml
index b158b94c8f8a..638c2ac14187 100644
--- a/gradle/build.versions.toml
+++ b/gradle/build.versions.toml
@@ -1,6 +1,6 @@
 [versions]
 asm = "9.3"
-jackson = "2.14.0"
+jackson = "2.14.2"
 junit5 = "5.8.1"
 spock = "2.1-groovy-3.0"
 
@@ -39,6 +39,6 @@ shadow-plugin = "gradle.plugin.com.github.johnrengelman:shadow:7.1.2"
 spock-core = { group = "org.spockframework", name="spock-core", version.ref="spock" }
 spock-junit4 = { group = "org.spockframework", name="spock-junit4", version.ref="spock" }
 spock-platform = { group = "org.spockframework", name="spock-bom", version.ref="spock" }
-spotless-plugin = "com.diffplug.spotless:spotless-plugin-gradle:6.7.2"
+spotless-plugin = "com.diffplug.spotless:spotless-plugin-gradle:6.11.0"
 wiremock = "com.github.tomakehurst:wiremock-jre8-standalone:2.23.2"
 xmlunit-core = "org.xmlunit:xmlunit-core:2.8.2"
diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
index f6f16e818873..1e220398dae3 100644
--- a/gradle/verification-metadata.xml
+++ b/gradle/verification-metadata.xml
@@ -114,21 +114,41 @@
             <sha256 value="1daf0ea4f9715e5821c04002f527729b5eb47e90e4180ae7cdd809748a4ee6b9" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.azure" name="azure-core" version="1.34.0">
+         <artifact name="azure-core-1.34.0.jar">
+            <sha256 value="74f02bbac2dd5485b4e3b3fb7b8cb70232039af27014a7d2938d0c186e051e40" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.azure" name="azure-core-http-netty" version="1.11.9">
          <artifact name="azure-core-http-netty-1.11.9.jar">
             <sha256 value="c22fbf3370ca43f7516fae8e1faf55b19ebd66ea3c339ce7352a698285d98514" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.azure" name="azure-core-http-netty" version="1.12.7">
+         <artifact name="azure-core-http-netty-1.12.7.jar">
+            <sha256 value="24d15d722bef5be7065b835ff413ab045136bd5a3927a00647cfa7413bea3bce" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.azure" name="azure-storage-blob" version="12.16.0">
          <artifact name="azure-storage-blob-12.16.0.jar">
             <sha256 value="45491c68099edd27680b3bd93e71bfb08ee9fe3781bc29667a508f27c0b8baec" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.azure" name="azure-storage-blob" version="12.20.1">
+         <artifact name="azure-storage-blob-12.20.1.jar">
+            <sha256 value="9a568f03f555cd3dc7c46fed557ea65a0aa00b5eaff1e47e3e21462a863504ba" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.azure" name="azure-storage-common" version="12.15.1">
          <artifact name="azure-storage-common-12.15.1.jar">
             <sha256 value="06ca93431ed2f5f22af2189dee10eccef94c229777596336b27d9c8f6bb5a5dd" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.azure" name="azure-storage-common" version="12.19.1">
+         <artifact name="azure-storage-common-12.19.1.jar">
+            <sha256 value="0e67a261c1088b9d688d7461e27da95f6830bdde713420b0d783ae1025c4d384" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.bettercloud" name="vault-java-driver" version="4.1.0">
          <artifact name="vault-java-driver-4.1.0.jar">
             <sha256 value="8878c47594fcfaa3e5b6fd75bb0e8f82cd8d4e7d947dd2a8e986f6d1f4b6c23e" origin="Generated by Gradle"/>
@@ -184,19 +204,19 @@
             <sha256 value="e983d13b1051f8dc49820970aa8da80364857cecf46d78d1b509366e523186bd" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.diffplug.spotless" name="spotless-lib" version="2.26.2">
-         <artifact name="spotless-lib-2.26.2.jar">
-            <sha256 value="ec82ca0395cbbdb52b259b0401cd984932f7a4869fad03a1c4264aba63b5a179" origin="Generated by Gradle"/>
+      <component group="com.diffplug.spotless" name="spotless-lib" version="2.30.0">
+         <artifact name="spotless-lib-2.30.0.jar">
+            <sha256 value="21f4d7fc032bfada42f29e629ce7d894e1a9bcf7dd5aa9b243fedd0c7d24f0a1" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.diffplug.spotless" name="spotless-lib-extra" version="2.26.2">
-         <artifact name="spotless-lib-extra-2.26.2.jar">
+      <component group="com.diffplug.spotless" name="spotless-lib-extra" version="2.30.0">
+         <artifact name="spotless-lib-extra-2.30.0.jar">
             <sha256 value="9b4ce98aa4f5dbc75850fe6d9a79e878d318be718f997e5ea4052c885621baab" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.diffplug.spotless" name="spotless-plugin-gradle" version="6.7.2">
-         <artifact name="spotless-plugin-gradle-6.7.2.jar">
-            <sha256 value="39627c87a2f589cbad5525509d7755a0a64ecc1efd9f28002c27ba45ea15c11d" origin="Generated by Gradle"/>
+      <component group="com.diffplug.spotless" name="spotless-plugin-gradle" version="6.11.0">
+         <artifact name="spotless-plugin-gradle-6.11.0.jar">
+            <sha256 value="54293b86170b821115772090bb90aeb466cda6bf9e08b06bacbd3bb94a714e1f" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.ethlo.time" name="itu" version="1.7.0">
@@ -204,49 +224,94 @@
             <sha256 value="55ceb418c9e8138c4fcf62e213c4c814d89e8a84c827d395407cbecba5d791e7" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.fasterxml.jackson.core" name="jackson-annotations" version="2.14.0">
-           <artifact name="jackson-annotations-2.14.0.jar">
-               <sha256 value="efaff8693acbae673468d251b5e5ea8fc7ce1b852327bccf1cce72244c2e5f1c" origin="Generated by Gradle"/>
-           </artifact>
+      <component group="com.fasterxml.jackson.core" name="jackson-annotations" version="2.10.5">
+         <artifact name="jackson-annotations-2.10.5.jar">
+            <sha256 value="5ad94fbb2642df695892c4d6e2ab4c319821e5f9bfb7b920f1378de4f611417c" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.fasterxml.jackson.core" name="jackson-annotations" version="2.13.4">
+         <artifact name="jackson-annotations-2.13.4.jar">
+            <sha256 value="ac5b27a634942391ca113850ee7db01df1499a240174021263501c05fc653b44" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.fasterxml.jackson.core" name="jackson-annotations" version="2.14.2">
+         <artifact name="jackson-annotations-2.14.2.jar">
+            <sha256 value="2c6869d505cf60dc066734b7d50339f975bd3adc635e26a78abb71acb4473c0d" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.fasterxml.jackson.core" name="jackson-core" version="2.10.5">
+         <artifact name="jackson-core-2.10.5.jar">
+            <sha256 value="2656010d1e921ac69b76fc7e0c0f5a6b14aca62fa9603e78831e6148eb7c77ba" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.fasterxml.jackson.core" name="jackson-core" version="2.13.4">
+         <artifact name="jackson-core-2.13.4.jar">
+            <sha256 value="4c2e043200edd9ee7ba6fc378bd5c17784a5bf2388e152d208068b51fd0839cf" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.fasterxml.jackson.core" name="jackson-core" version="2.14.2">
+         <artifact name="jackson-core-2.14.2.jar">
+            <sha256 value="b5d37a77c88277b97e3593c8740925216c06df8e4172bbde058528df04ad3e7a" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.fasterxml.jackson.core" name="jackson-databind" version="2.10.5.1">
+         <artifact name="jackson-databind-2.10.5.1.jar">
+            <sha256 value="f93db83891a53e8d268e2cc8fcd88ead2981edc2163e35c2a52c88d9ab57b4a0" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.fasterxml.jackson.core" name="jackson-databind" version="2.13.4.2">
+         <artifact name="jackson-databind-2.13.4.2.jar">
+            <sha256 value="ba1a368137f9b92f48dd07f299ff7a782a69487b853ef5578215426fcde8f08b" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.fasterxml.jackson.core" name="jackson-databind" version="2.14.2">
+         <artifact name="jackson-databind-2.14.2.jar">
+            <sha256 value="501d3abce4d18dcc381058ec593c5b94477906bba6efbac14dae40a642f77424" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.fasterxml.jackson.dataformat" name="jackson-dataformat-cbor" version="2.14.2">
+         <artifact name="jackson-dataformat-cbor-2.14.2.jar">
+            <sha256 value="c942726863a8b7e0483d30d9213f9dadc8b07eadf0767003a4fe6dc56daa7135" origin="Generated by Gradle"/>
+         </artifact>
       </component>
-      <component group="com.fasterxml.jackson.core" name="jackson-core" version="2.14.0">
-         <artifact name="jackson-core-2.14.0.jar">
-            <sha256 value="ab4793e5df4fbfae445ca55e9e1439311c80fa8b34fc13162c1260902b4dbea0" origin="Generated by Gradle"/>
+      <component group="com.fasterxml.jackson.dataformat" name="jackson-dataformat-smile" version="2.14.2">
+         <artifact name="jackson-dataformat-smile-2.14.2.jar">
+            <sha256 value="9c279bb29770de09289c14cf8862dd195112687cd7fde815919f54a9157ce213" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.fasterxml.jackson.core" name="jackson-databind" version="2.14.0">
-          <artifact name="jackson-databind-2.14.0.jar">
-              <sha256 value="54377fa855f52ed87e8f689b35249971840b16870dee76806d5d200cbcd66f27" origin="Generated by Gradle"/>
-          </artifact>
+      <component group="com.fasterxml.jackson.dataformat" name="jackson-dataformat-xml" version="2.13.4">
+         <artifact name="jackson-dataformat-xml-2.13.4.jar">
+            <sha256 value="76fbf0cedd51af6a13aba39c27c8c29a5a280dc24ee66577d559e4660d8709ce" origin="Generated by Gradle"/>
+         </artifact>
       </component>
-      <component group="com.fasterxml.jackson.dataformat" name="jackson-dataformat-cbor" version="2.14.0">
-         <artifact name="jackson-dataformat-cbor-2.14.0.jar">
-            <sha256 value="f93c35d8b48ebc8bfe96a9d9e6d7e473f4744eb74819213e1d3af6ba40c940d2" origin="Generated by Gradle"/>
+      <component group="com.fasterxml.jackson.dataformat" name="jackson-dataformat-yaml" version="2.14.2">
+         <artifact name="jackson-dataformat-yaml-2.14.2.jar">
+            <sha256 value="5c3a0a71d0339529c80ae771497b20fdc0fa7cb67c772f99af5935927560006a" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.fasterxml.jackson.dataformat" name="jackson-dataformat-smile" version="2.14.0">
-         <artifact name="jackson-dataformat-smile-2.14.0.jar">
-            <sha256 value="5cc75dadecccbd4f74bd27304ee3fd5c642adcad0fd1df97dcc670c04e15b367" origin="Generated by Gradle"/>
+      <component group="com.fasterxml.jackson.datatype" name="jackson-datatype-jsr310" version="2.13.4">
+         <artifact name="jackson-datatype-jsr310-2.13.4.jar">
+            <sha256 value="5ad75d210bacc17271925da28e1f393aaf8c83f6c92fbe5b2ed61954b84decf7" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.fasterxml.jackson.dataformat" name="jackson-dataformat-xml" version="2.14.0">
-         <artifact name="jackson-dataformat-xml-2.14.0.jar">
-            <sha256 value="2bb81f3cf2e10e10190e7ed55471b58249efe580941f907c0bce8443c6164334" origin="Generated by Gradle"/>
+      <component group="com.fasterxml.jackson.jaxrs" name="jackson-jaxrs-base" version="2.10.5">
+         <artifact name="jackson-jaxrs-base-2.10.5.jar">
+            <sha256 value="98f27188fa2a72ef5d3f85fab6e6ca0e76bde1a58c9396cb1cf91028080435d6" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.fasterxml.jackson.dataformat" name="jackson-dataformat-yaml" version="2.14.0">
-         <artifact name="jackson-dataformat-yaml-2.14.0.jar">
-            <sha256 value="76e8a33ef1f5f8cce9668ebaf8999626846ccacb36dea81bcdaf79e32443de33" origin="Generated by Gradle"/>
+      <component group="com.fasterxml.jackson.jaxrs" name="jackson-jaxrs-json-provider" version="2.10.5">
+         <artifact name="jackson-jaxrs-json-provider-2.10.5.jar">
+            <sha256 value="f0817100df27ded044dc9ac6effdb9961a3c37327c6c9262ed344218db048c7b" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.fasterxml.jackson.datatype" name="jackson-datatype-jsr310" version="2.14.0">
-         <artifact name="jackson-datatype-jsr310-2.14.0.jar">
-            <sha256 value="1e2ed10509b17e3c67d1724540f7a446d5bea6c4c7596c396dc76413309e4ed2" origin="Generated by Gradle"/>
+      <component group="com.fasterxml.jackson.module" name="jackson-module-jaxb-annotations" version="2.10.5">
+         <artifact name="jackson-module-jaxb-annotations-2.10.5.jar">
+            <sha256 value="994a0a510a35d55a869567807075736597da97e9d36ad1ebaff5e37def5a55d3" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.fasterxml.jackson.module" name="jackson-module-jaxb-annotations" version="2.14.0">
-         <artifact name="jackson-module-jaxb-annotations-2.14.0.jar">
-            <sha256 value="5ac9a0f78af0fdac22f5a4e25494bee2ed54bf1c760af63aa78a0147eb7f41d0" origin="Generated by Gradle"/>
+      <component group="com.fasterxml.jackson.module" name="jackson-module-jaxb-annotations" version="2.13.4">
+         <artifact name="jackson-module-jaxb-annotations-2.13.4.jar">
+            <sha256 value="b23725fd92b783e3ddc149d23f565b9da5bdfc98361be81488ff4e45e6735ba1" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.fasterxml.woodstox" name="woodstox-core" version="5.3.0">
@@ -259,6 +324,11 @@
             <sha256 value="dda9dd5cf9fbf5b69f58b4c16ab068b7bca1f428158d29722e00e8005d457037" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.fasterxml.woodstox" name="woodstox-core" version="6.4.0">
+         <artifact name="woodstox-core-6.4.0.jar">
+            <sha256 value="d6eb9f2f40049a7a808baf11ffba0737648e62ff52fde9271d808e5d57a27279" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.github.javaparser" name="javaparser-core" version="3.18.0">
          <artifact name="javaparser-core-3.18.0.jar">
             <sha256 value="df7e2d5b8319efd51d015ab5071a271f8463158bc2f0979d05393fe028d856a0" origin="Generated by Gradle"/>
@@ -284,19 +354,19 @@
             <sha256 value="896280c8bd0a81a020272af6bfabd855810c4e4fd5ff5605e109f3bb3d2a30e3" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.api" name="api-common" version="2.2.1">
-         <artifact name="api-common-2.2.1.jar">
-            <sha256 value="197b23f2d22324a923e5d643d1c71c51442ae7b9b166fe6ecd5ad9b6916689b5" origin="Generated by Gradle"/>
+      <component group="com.google.api" name="api-common" version="2.3.1">
+         <artifact name="api-common-2.3.1.jar">
+            <sha256 value="9a8c0f3a638eac791496709b7770c39bda9e56de50728f122237c813bb21f1ee" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.api" name="gax" version="2.0.0">
-         <artifact name="gax-2.0.0.jar">
-            <sha256 value="37974e31bf9ef5119b0411f83aaa209d4edb4fb10646eb6eaff68e7771a64e50" origin="Generated by Gradle"/>
+      <component group="com.google.api" name="gax" version="2.20.1">
+         <artifact name="gax-2.20.1.jar">
+            <sha256 value="e6588fc1786557fda95024a040b32106d58ffe72fab0933200c4d3c6197a4fe1" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.api" name="gax-httpjson" version="0.85.0">
-         <artifact name="gax-httpjson-0.85.0.jar">
-            <sha256 value="e0b0411e42c859e0fab4b092a3429aea2a71b2c2ebbbc24d719b9e762be97c35" origin="Generated by Gradle"/>
+      <component group="com.google.api" name="gax-httpjson" version="0.105.1">
+         <artifact name="gax-httpjson-0.105.1.jar">
+            <sha256 value="4b7e1135eb4a97bce9d9d8c56128c5c30594dc2bebf26c9851ac582d2b43b2db" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.google.api-client" name="google-api-client" version="1.33.1">
@@ -304,19 +374,19 @@
             <sha256 value="d471ac82f1b285b3fde1e5b7ff265dad7ab9cd73a052bcb01ef35054f7d5db02" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.api-client" name="google-api-client" version="1.35.1">
-         <artifact name="google-api-client-1.35.1.jar">
-            <sha256 value="24714add0c0248730a0e705448bbc91a5c68dbc0dc9f7fa0c97366bc7b275e23" origin="Generated by Gradle"/>
+      <component group="com.google.api-client" name="google-api-client" version="2.1.1">
+         <artifact name="google-api-client-2.1.1.jar">
+            <sha256 value="05d846e83a2924c2a6facbd72817228fee68bb522b818b9804b26a712f678bf3" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.api.grpc" name="proto-google-common-protos" version="2.3.2">
-         <artifact name="proto-google-common-protos-2.3.2.jar">
-            <sha256 value="99aa86a5e52dff58be410b5c30905b2ea9a2c995a87e237f160590953c661b6d" origin="Generated by Gradle"/>
+      <component group="com.google.api.grpc" name="proto-google-common-protos" version="2.9.6">
+         <artifact name="proto-google-common-protos-2.9.6.jar">
+            <sha256 value="84e715d92500fd0d44408f58713d5d70ae857d022de3d66d500fbf27fae88103" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.api.grpc" name="proto-google-iam-v1" version="1.0.14">
-         <artifact name="proto-google-iam-v1-1.0.14.jar">
-            <sha256 value="e722e8d67172a87ad1b5bd3ba02c678484958190c49c059494453b07e7ea5dca" origin="Generated by Gradle"/>
+      <component group="com.google.api.grpc" name="proto-google-iam-v1" version="1.6.2">
+         <artifact name="proto-google-iam-v1-1.6.2.jar">
+            <sha256 value="4fce59633b8a9871614231d213992e60e9b50e12f822c33d7e82a79c90addf25" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.google.apis" name="google-api-services-compute" version="v1-rev20220322-1.32.1">
@@ -324,34 +394,39 @@
             <sha256 value="05d913bb9b2baa8e4dfd99a1aee4864240feebf6647986e37aea88f3020313fe" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.apis" name="google-api-services-storage" version="v1-rev20210127-1.32.1">
-         <artifact name="google-api-services-storage-v1-rev20210127-1.32.1.jar">
-            <sha256 value="c23beb05bb842abed14c8fd75a3e6b7a2f51b0d3ece1903a9c5fd3ab770ad93e" origin="Generated by Gradle"/>
+      <component group="com.google.apis" name="google-api-services-storage" version="v1-rev20220705-2.0.0">
+         <artifact name="google-api-services-storage-v1-rev20220705-2.0.0.jar">
+            <sha256 value="c6027d6c58089402620ce338351ca2efd414f813007844bc252ceb5f47789f4f" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.auth" name="google-auth-library-credentials" version="1.0.0">
-         <artifact name="google-auth-library-credentials-1.0.0.jar">
-            <sha256 value="8b4e4bf2c89f72e9fdcbbd442c931e17cba01e84f058fd61e470bd19f8a92cc1" origin="Generated by Gradle"/>
+      <component group="com.google.auth" name="google-auth-library-credentials" version="1.11.0">
+         <artifact name="google-auth-library-credentials-1.11.0.jar">
+            <sha256 value="2f47d7d2126576f13c22b7442e97593e9e6285adbd6d4c8dd779d87a8cbb8270" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.auth" name="google-auth-library-oauth2-http" version="1.0.0">
-         <artifact name="google-auth-library-oauth2-http-1.0.0.jar">
-            <sha256 value="7cae6c73368659ac7a6b46018070f3d7980acd5bfaaa91b840f2e869cef1f07e" origin="Generated by Gradle"/>
+      <component group="com.google.auth" name="google-auth-library-oauth2-http" version="1.11.0">
+         <artifact name="google-auth-library-oauth2-http-1.11.0.jar">
+            <sha256 value="9fc1548aacc781367b5ca8da8e053018929d71af7303f6752952ddcb6f9b50c4" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.cloud" name="google-cloud-core" version="2.0.2">
-         <artifact name="google-cloud-core-2.0.2.jar">
-            <sha256 value="364be3e5617c973a4fa8ebe2a56aa8eaec9277f5dba7d033cc37b71b00025e45" origin="Generated by Gradle"/>
+      <component group="com.google.cloud" name="google-cloud-core" version="2.8.28">
+         <artifact name="google-cloud-core-2.8.28.jar">
+            <sha256 value="f775b5dc6b6478d07be13c64b3aa8ee0eded5e2571ff737e86ed05958cce4e92" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.cloud" name="google-cloud-core-http" version="2.0.2">
-         <artifact name="google-cloud-core-http-2.0.2.jar">
-            <sha256 value="087f0bc4d8de1daed73acb631b68c254dcf7993379110c6edf7bc7438d0e52ca" origin="Generated by Gradle"/>
+      <component group="com.google.cloud" name="google-cloud-core-http" version="2.8.28">
+         <artifact name="google-cloud-core-http-2.8.28.jar">
+            <sha256 value="96cd89a23a40b64baa28cba92676fba67d16cfbb1a7085827a1852a71c2dce46" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.cloud" name="google-cloud-storage" version="1.118.1">
-         <artifact name="google-cloud-storage-1.118.1.jar">
-            <sha256 value="21d675edc7caa02426290ed631b453004e7ece91bc811209855491263ac35f69" origin="Generated by Gradle"/>
+      <component group="com.google.cloud" name="google-cloud-storage" version="2.13.1">
+         <artifact name="google-cloud-storage-2.13.1.jar">
+            <sha256 value="5fee796ca3c328aabe1b96adfbb264714e82b3ceee2971a83732c8cada12a9b6" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.google.cloud" name="google-cloud-storage" version="2.15.1">
+         <artifact name="google-cloud-storage-2.15.1.jar">
+            <sha256 value="467dd38d9d028e4645473e8f0271e7a8b2f1c3f3439e4849eb34479cc6f54e66" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.google.code.findbugs" name="jsr305" version="3.0.0">
@@ -364,6 +439,11 @@
             <sha256 value="766ad2a0783f2687962c8ad74ceecc38a28b9f72a2d085ee438b7813e928d0c7" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.code.gson" name="gson" version="2.10">
+         <artifact name="gson-2.10.jar">
+            <sha256 value="0cdd163ce3598a20fc04eee71b140b24f6f2a3b35f0a499dbbdd9852e83fbfaf" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.code.gson" name="gson" version="2.2.4">
          <artifact name="gson-2.2.4.jar">
             <sha256 value="c0328cd07ca9e363a5acd00c1cf4afe8cf554bd6d373834981ba05cebec687fb" origin="Generated by Gradle"/>
@@ -469,9 +549,14 @@
             <sha256 value="93bf43e033c7441f66f163f2cb1244d89765c492b844031258ae92b7081dde6e" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.http-client" name="google-http-client-appengine" version="1.39.2">
-         <artifact name="google-http-client-appengine-1.39.2.jar">
-            <sha256 value="f6320dc0db035aff6eab2e28748560da718ab465fbc3481b653ea3f27aa5cac9" origin="Generated by Gradle"/>
+      <component group="com.google.http-client" name="google-http-client" version="1.42.3">
+         <artifact name="google-http-client-1.42.3.jar">
+            <sha256 value="e395dd1765e3e6bceb0c610706bcf4128de84bd6e65cf1d4adbf998b4114161c" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="com.google.http-client" name="google-http-client-appengine" version="1.42.3">
+         <artifact name="google-http-client-appengine-1.42.3.jar">
+            <sha256 value="3a876b1f704ac6d8500b8430b288dbce78c52dc18c43c008a587a7377bb81e2e" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.google.http-client" name="google-http-client-gson" version="1.39.2">
@@ -484,6 +569,11 @@
             <sha256 value="29deba06c722e00dd31fe7e3b526bdfec0577660b868a1efaceeddb5e04208a8" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.http-client" name="google-http-client-gson" version="1.42.3">
+         <artifact name="google-http-client-gson-1.42.3.jar">
+            <sha256 value="8196efaa89c5f73b00b2b48edad02fcd78524259407c37ab1860737988545cee" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.http-client" name="google-http-client-jackson2" version="1.39.2">
          <artifact name="google-http-client-jackson2-1.39.2.jar">
             <sha256 value="74a493a7ba25e1dfb3d3a4ddf713306d5a3786db00a3489d59061dc8cff321b1" origin="Generated by Gradle"/>
@@ -494,6 +584,11 @@
             <sha256 value="3b9add5859979519236f45cc9c9f53da975e0b5b88e0cdfd4bb009b112b4d435" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.google.http-client" name="google-http-client-jackson2" version="1.42.3">
+         <artifact name="google-http-client-jackson2-1.42.3.jar">
+            <sha256 value="b03a79f38e924e2eeaee673acf314626825284cb39948d06c2948ad083b4d089" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.google.inject" name="guice" version="3.0">
          <artifact name="guice-3.0.jar">
             <sha256 value="1a59d0421ffd355cc0b70b42df1c2e9af744c8a2d0c92da379f5fca2f07f1d22" origin="Generated by Gradle"/>
@@ -529,11 +624,6 @@
             <sha256 value="de16d5c8489729a8512f1a02fbd81f58f89249b72066987da4cc5c87ecb9f72d" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.oauth-client" name="google-oauth-client" version="1.33.0">
-         <artifact name="google-oauth-client-1.33.0.jar">
-            <sha256 value="ce615114bb3a7ed9729825ee4c21a22db9ccdeb31e2c5429e4a28949afa9c94b" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
       <component group="com.google.oauth-client" name="google-oauth-client" version="1.34.1">
          <artifact name="google-oauth-client-1.34.1.jar">
             <sha256 value="193edf97aefa28b93c5892bdc598bac34fa4c396588030084f290b1440e8b98a" origin="Generated by Gradle"/>
@@ -544,24 +634,14 @@
             <sha256 value="e0c1c64575c005601725e7c6a02cebf9e1285e888f756b2a1d73ffa8d725cc74" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.protobuf" name="protobuf-java" version="3.16.1">
-         <artifact name="protobuf-java-3.16.1.jar">
-            <sha256 value="7b845a34210acde78b7f77977b3724988b9c60b2dce7a93a9afbbb1fee7978c4" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="com.google.protobuf" name="protobuf-java" version="3.21.1">
-         <artifact name="protobuf-java-3.21.1.jar">
-            <sha256 value="ad275e75ee79e6c6388198dcb9acf0db2edee64782e11b508f379c3a2a17d168" origin="Generated by Gradle"/>
+      <component group="com.google.protobuf" name="protobuf-java" version="3.21.9">
+         <artifact name="protobuf-java-3.21.9.jar">
+            <sha256 value="1b78b4a76a71512debfdff8f8fc5aef6bfd459f65758fecf7aff245e6e6301e4" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.google.protobuf" name="protobuf-java" version="3.4.0">
-         <artifact name="protobuf-java-3.4.0.jar">
-            <sha256 value="dce7e66b32456a1b1198da0caff3a8acb71548658391e798c79369241e6490a4" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="com.google.protobuf" name="protobuf-java-util" version="3.17.3">
-         <artifact name="protobuf-java-util-3.17.3.jar">
-            <sha256 value="bf320ed076000e1d8c7cbf7601b056acaecab80f75b9a659b9f6398d0d7e3f79" origin="Generated by Gradle"/>
+      <component group="com.google.protobuf" name="protobuf-java-util" version="3.21.9">
+         <artifact name="protobuf-java-util-3.21.9.jar">
+            <sha256 value="a78226d02551ca2cb34b24b4168a75b4a84041cc3333f3725bb5ad9806a80b77" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.google.re2j" name="re2j" version="1.1">
@@ -574,9 +654,9 @@
             <sha256 value="04e3724984e2a5cbf55606cfa372a5bd3d3c5d2a21533a7004e3cde539761fa5" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.googlecode.javaewah" name="JavaEWAH" version="1.1.12">
-         <artifact name="JavaEWAH-1.1.12.jar">
-            <sha256 value="b19204331ac6e204befaf508a68f52ec6b4638d9d9bacddbebd435f9c4d500f2" origin="Generated by Gradle"/>
+      <component group="com.googlecode.javaewah" name="JavaEWAH" version="1.1.13">
+         <artifact name="JavaEWAH-1.1.13.jar">
+            <sha256 value="4c0fda2b1d317750d7ea324e36c70b2bc48310c0aaae67b98df0915d696d7111" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.googlecode.juniversalchardet" name="juniversalchardet" version="1.0.3">
@@ -609,6 +689,11 @@
             <sha256 value="9bd7bf869a44ba8aeb0cddd7e6616e88cd4795ba5bfce2230447cb0e185a646c" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.ibm.icu" name="icu4j" version="71.1">
+         <artifact name="icu4j-71.1.jar">
+            <sha256 value="91c4f8ebf0ceb489547098fe9d5c09a65eb419caea6ed714867f5280800bcf1a" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.jamesmurty.utils" name="java-xmlbuilder" version="0.4">
          <artifact name="java-xmlbuilder-0.4.jar">
             <sha256 value="681e53c4ffd59fa12068803b259e3a83d43f07a47c112e748a187dee179eb31f" origin="Generated by Gradle"/>
@@ -664,14 +749,14 @@
             <sha256 value="f8f5c726c8c615a94d9d6cd1073787f9c554841ae24c8bab5cd73481ea2021a8" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.maxmind.db" name="maxmind-db" version="2.0.0">
-         <artifact name="maxmind-db-2.0.0.jar">
-            <sha256 value="635e07839eb0cebd9caa26dd27a75b14008c2e31b589688d757114122ac8dac2" origin="Generated by Gradle"/>
+      <component group="com.maxmind.db" name="maxmind-db" version="3.0.0">
+         <artifact name="maxmind-db-3.0.0.jar">
+            <sha256 value="4d783d30abc7c8adb3580faa3f76c33af9ecb5b0fc1c3d83b204df3fbb6725c2" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.maxmind.geoip2" name="geoip2" version="3.0.0">
-         <artifact name="geoip2-3.0.0.jar">
-            <sha256 value="773eb6f61dfaaa64ddf8afd65ab7165f0eec18687224132c914f02cc570eb312" origin="Generated by Gradle"/>
+      <component group="com.maxmind.geoip2" name="geoip2" version="4.0.0">
+         <artifact name="geoip2-4.0.0.jar">
+            <sha256 value="b73aa72fbaf6fa6376a6c12f8976631b5bb183027695fac665eab5082b1a67f7" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.microsoft.azure" name="azure-core" version="0.9.3">
@@ -1069,9 +1154,9 @@
             <sha256 value="bcbf9055dff453fd6508bd7cca2a0aa2d5f059a9c94beed1f5fda1dc015607b8" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.grpc" name="grpc-context" version="1.39.0">
-         <artifact name="grpc-context-1.39.0.jar">
-            <sha256 value="97fe70b4e7da86ed4fe3deb16d83fc89c7e5b764ebac8c9dd5d51dd4b5aa920c" origin="Generated by Gradle"/>
+      <component group="io.grpc" name="grpc-context" version="1.49.2">
+         <artifact name="grpc-context-1.49.2.jar">
+            <sha256 value="7aaf3ad732a281be51fe0895acd4538b96e1d7e3b527bce54b651770b8ded174" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty" version="3.10.6.Final">
@@ -1099,9 +1184,9 @@
             <sha256 value="7b0171a4e8bcd573e08d9f2bba053c67b557ab5012106a5982ccbae5743814c0" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-buffer" version="4.1.84.Final">
-         <artifact name="netty-buffer-4.1.84.Final.jar">
-            <sha256 value="5341cbdb7ed9b23d008f14cca36f910b309446723b4f41bb866f522c6cf699c2" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-buffer" version="4.1.86.Final">
+         <artifact name="netty-buffer-4.1.86.Final.jar">
+            <sha256 value="e42e15f47c865266b1faa6e038ebfd7ddadcf9f4ae9e6617edd4881dbd4abe88" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-codec" version="4.1.42.Final">
@@ -1109,29 +1194,29 @@
             <sha256 value="e96ced697fb7df589da7c20c995e01f75a9cb246be242bbc4cd3b4af424ff189" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-codec" version="4.1.84.Final">
-         <artifact name="netty-codec-4.1.84.Final.jar">
-            <sha256 value="b659961e1e066d37d6e23c31973e616704b277fdbc6b18489ecac26f1ef801dd" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-codec" version="4.1.86.Final">
+         <artifact name="netty-codec-4.1.86.Final.jar">
+            <sha256 value="0456840b5c851dad6cab881cd1a9ad5d916db65d81048145df1d9a6d03325bea" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-codec-dns" version="4.1.84.Final">
-         <artifact name="netty-codec-dns-4.1.84.Final.jar">
-            <sha256 value="94a56a1c83a2261e76e9383e970d6f8eafa7e0fcd87ba616b770ab5244151b2a" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-codec-dns" version="4.1.86.Final">
+         <artifact name="netty-codec-dns-4.1.86.Final.jar">
+            <sha256 value="a6d5247960b1a81604506a752c997e8817c6a73c64a0c30fc8e72a1afb1e1502" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-codec-http" version="4.1.84.Final">
-         <artifact name="netty-codec-http-4.1.84.Final.jar">
-            <sha256 value="2585e2457de9db9d9195ae36cbbfb2432d85123672ec4a6831c6bb41a217c1b8" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-codec-http" version="4.1.86.Final">
+         <artifact name="netty-codec-http-4.1.86.Final.jar">
+            <sha256 value="3f6ceb3112cfcf7b70545eb5111220ce57db54d593f23f64c38333bb22c40b84" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-codec-http2" version="4.1.84.Final">
-         <artifact name="netty-codec-http2-4.1.84.Final.jar">
-            <sha256 value="15e1a1911708883c2e3cc50f7e38f535902353dcb8d274b0ec21087cc9fcbdcf" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-codec-http2" version="4.1.86.Final">
+         <artifact name="netty-codec-http2-4.1.86.Final.jar">
+            <sha256 value="e8e8e28e6ab6bb989aed904778922045f388cfb420bc1eb37abf4df8801db167" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-codec-socks" version="4.1.84.Final">
-         <artifact name="netty-codec-socks-4.1.84.Final.jar">
-            <sha256 value="676061ec6065b704864049c1801d3e4ac695841a1152f2a6dfe41b59fdeee1c3" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-codec-socks" version="4.1.86.Final">
+         <artifact name="netty-codec-socks-4.1.86.Final.jar">
+            <sha256 value="838a2f46678e84abb7ed766247d038cb85dfafc480ff8c2c5e42d91d99c5ff4d" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-common" version="4.1.42.Final">
@@ -1139,9 +1224,9 @@
             <sha256 value="3d0a918d78292eeca02a7bb2188daa4e5053b6e29b71e6308309033e121242b5" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-common" version="4.1.84.Final">
-         <artifact name="netty-common-4.1.84.Final.jar">
-            <sha256 value="50da7383e882acbd8bca9d823797bff62fa91c91f74f0da575816e3214a25560" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-common" version="4.1.86.Final">
+         <artifact name="netty-common-4.1.86.Final.jar">
+            <sha256 value="a35a3f16e7cd45c5d8529aa3e7702d4ef3b36213ea332db59744ea348fc2ae99" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-handler" version="4.1.42.Final">
@@ -1149,14 +1234,14 @@
             <sha256 value="11eda86500c33b9d386719b5419f513fd9c097d13894f25dd0c75b610d636e03" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-handler" version="4.1.84.Final">
-         <artifact name="netty-handler-4.1.84.Final.jar">
-            <sha256 value="0b6814893c88374e6bf474d5a2df04ee912c89dbc885b8115ac393238382e3f5" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-handler" version="4.1.86.Final">
+         <artifact name="netty-handler-4.1.86.Final.jar">
+            <sha256 value="e69b42292929b278dc522e25177ddf7c54025484b55879f8227349adfbe1c04d" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-handler-proxy" version="4.1.84.Final">
-         <artifact name="netty-handler-proxy-4.1.84.Final.jar">
-            <sha256 value="f385da33bc1df4cfe2f8162ae9d62c2be838cc30ae55f749ecdaff958188cbfa" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-handler-proxy" version="4.1.86.Final">
+         <artifact name="netty-handler-proxy-4.1.86.Final.jar">
+            <sha256 value="b812fe7d72c84fbc84e53b4f26764d82f76b06a65f331f0fbd9543aa73e18f91" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-resolver" version="4.1.42.Final">
@@ -1164,14 +1249,14 @@
             <sha256 value="89768242b6b7cce9bd9f5945ad21d1b4bae515c6b1bf03a8af5d1899779cebc9" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-resolver" version="4.1.84.Final">
-         <artifact name="netty-resolver-4.1.84.Final.jar">
-            <sha256 value="0876172e3cf4bbcecdce812ed145b17bc3fb5c645f3ed1873d9c56da1aa150b1" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-resolver" version="4.1.86.Final">
+         <artifact name="netty-resolver-4.1.86.Final.jar">
+            <sha256 value="7628a1309d7f2443dc41d8923a7f269e2981b9616f80a999eb7264ae6bcbfdba" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-resolver-dns" version="4.1.84.Final">
-         <artifact name="netty-resolver-dns-4.1.84.Final.jar">
-            <sha256 value="d7a90c6443c7a1b0189ac0cd8dac1a559b79de9cc4f465c2c6797260561fd4b6" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-resolver-dns" version="4.1.86.Final">
+         <artifact name="netty-resolver-dns-4.1.86.Final.jar">
+            <sha256 value="49b05d90233abb62ce8ffeb5147db54719be1ccec530f70e4d62b1f3727c1285" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-transport" version="4.1.42.Final">
@@ -1179,9 +1264,9 @@
             <sha256 value="dfa817a156ea263aa9ad8364a2e226527665c9722aca40a7945f228c2c14f1da" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-transport" version="4.1.84.Final">
-         <artifact name="netty-transport-4.1.84.Final.jar">
-            <sha256 value="ccfd4168ec83b1949b10e8927211dd8f9f89df984dec477cc63eb49a1209ee75" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-transport" version="4.1.86.Final">
+         <artifact name="netty-transport-4.1.86.Final.jar">
+            <sha256 value="f6726dcd54e4922b46b3b4f4467b443a70a30eb08a62620c8fe502d8cb802c9f" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-transport-native-epoll" version="4.1.42.Final">
@@ -1194,9 +1279,9 @@
             <sha256 value="508fba9128da78bd775ba854d71917ceb2b00b95a7600254f54a277a06761a86" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-transport-native-unix-common" version="4.1.84.Final">
-         <artifact name="netty-transport-native-unix-common-4.1.84.Final.jar">
-            <sha256 value="28c17f526eeb7ac23530f75b5ec28ca55c76c5466923ea7e0b93b085c6e1d6f3" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-transport-native-unix-common" version="4.1.86.Final">
+         <artifact name="netty-transport-native-unix-common-4.1.86.Final.jar">
+            <sha256 value="ec26d03a06565791d57e997f793677ee4d3fc47b290b7951898c2ecd0232f115" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.opencensus" name="opencensus-api" version="0.28.0">
@@ -1209,6 +1294,11 @@
             <sha256 value="421cbc06d9fb56fcf07332553677ddea04a47079b30955567c8a80b34689ed32" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="io.opencensus" name="opencensus-api" version="0.31.1">
+         <artifact name="opencensus-api-0.31.1.jar">
+            <sha256 value="f1474d47f4b6b001558ad27b952e35eda5cc7146788877fc52938c6eba24b382" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="io.opencensus" name="opencensus-contrib-http-util" version="0.28.0">
          <artifact name="opencensus-contrib-http-util-0.28.0.jar">
             <sha256 value="49c3db2a29f1fdb2f73928cbea969bd1d40fab7cc5bb6273022babd96f7a789b" origin="Generated by Gradle"/>
@@ -1219,6 +1309,11 @@
             <sha256 value="5432d2e9a1bdc8a0e835c6bce52b9d767fbf688cc8892432cff97c3389894b46" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="io.opencensus" name="opencensus-contrib-http-util" version="0.31.1">
+         <artifact name="opencensus-contrib-http-util-0.31.1.jar">
+            <sha256 value="3ea995b55a4068be22989b70cc29a4d788c2d328d1d50613a7a9afd13fdd2d0a" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="io.opentelemetry" name="opentelemetry-api" version="1.17.0">
          <artifact name="opentelemetry-api-1.17.0.jar">
             <sha256 value="dce58a21caef0c6be082ce5ec86ad75f3017744a029a454e068871c110ae029c" origin="Generated by Gradle"/>
@@ -1244,14 +1339,19 @@
             <sha256 value="5c82576a5f6142f084331e8b7526bf6cc9415ff013fbdb1b6810955fa2cd62ad" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.projectreactor.netty" name="reactor-netty-core" version="1.0.15">
-         <artifact name="reactor-netty-core-1.0.15.jar">
-            <sha256 value="04c51adfdb9d285de56b53f3eb5127483284e5128798c2ec2b673ef21f1a15d0" origin="Generated by Gradle"/>
+      <component group="io.projectreactor" name="reactor-core" version="3.4.23">
+         <artifact name="reactor-core-3.4.23.jar">
+            <sha256 value="d03b2f2eb5da79d0283358cba6ff36456982800f6cd2666857af46a43b6a9eb2" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="io.projectreactor.netty" name="reactor-netty-core" version="1.0.24">
+         <artifact name="reactor-netty-core-1.0.24.jar">
+            <sha256 value="53b01c19f7a9c5ddaa4eaf0d22a6be7a8f888d34d07054eeff1deec8869485b2" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.projectreactor.netty" name="reactor-netty-http" version="1.0.15">
-         <artifact name="reactor-netty-http-1.0.15.jar">
-            <sha256 value="0425ea219b1248fc43bfa83f4c7e1f49fcddad8af1fa53b531d02d4842ea30e4" origin="Generated by Gradle"/>
+      <component group="io.projectreactor.netty" name="reactor-netty-http" version="1.0.24">
+         <artifact name="reactor-netty-http-1.0.24.jar">
+            <sha256 value="22b6bd810444738d883a19e5f129535a393e84f13247c99bdd09c610bad34a7e" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.sgr" name="s2-geometry-library-java" version="1.0.1">
@@ -1482,25 +1582,25 @@
             <sha256 value="0a4aa0f2606bd6292a7ccd7d67a0db914bf5874dfb8a6184df3e6d63cdc93702" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="openjdk_19" name="linux" version="19.0.1">
-         <artifact name="linux-19.0.1-aarch64.tar.gz">
-            <sha256 value="88cadc91d5c7c540ea9df5d23678bb65dc2092fe4e00650b39d87f24f2328e17" origin="Generated by Gradle"/>
+      <component group="openjdk_19" name="linux" version="19.0.2">
+         <artifact name="linux-19.0.2-aarch64.tar.gz">
+            <sha256 value="95728187b4b5607c49de751a209ecda6e04d9ed7cee603cf36f454239106527b" origin="Generated by Gradle"/>
          </artifact>
-         <artifact name="linux-19.0.1-x64.tar.gz">
-            <sha256 value="7a466882c7adfa369319fe4adeb197ee5d7f79e75d641e9ef94abee1fc22b1fa" origin="Generated by Gradle"/>
+         <artifact name="linux-19.0.2-x64.tar.gz">
+            <sha256 value="34cf8d095cc071e9e10165f5c45023f96ec68397fdaabf6c64bfec1ffeee6198" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="openjdk_19" name="macos" version="19.0.1">
-         <artifact name="macos-19.0.1-aarch64.tar.gz">
-            <sha256 value="915054b18fc17216410cea7aba2321c55b82bd414e1ef3c7e1bafc7beb6856c8" origin="Generated by Gradle"/>
+      <component group="openjdk_19" name="macos" version="19.0.2">
+         <artifact name="macos-19.0.2-aarch64.tar.gz">
+            <sha256 value="4317442e14c5c2f4f698db0e41347df99d050a32137b2a02dfec28ed856577cc" origin="Generated by Gradle"/>
          </artifact>
-         <artifact name="macos-19.0.1-x64.tar.gz">
-            <sha256 value="469af195906979f96c1dc862c2f539a5e280d0daece493a95ebeb91962512161" origin="Generated by Gradle"/>
+         <artifact name="macos-19.0.2-x64.tar.gz">
+            <sha256 value="c57c7c511706738fff6540945e0159e97b8b328777e6460977dd64e00f4c2c0b" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="openjdk_19" name="windows" version="19.0.1">
-         <artifact name="windows-19.0.1-x64.zip">
-            <sha256 value="adb1a33c07b45c39b926bdeeadf800f701be9c3d04e0deb543069e5f09856185" origin="Generated by Gradle"/>
+      <component group="openjdk_19" name="windows" version="19.0.2">
+         <artifact name="windows-19.0.2-x64.zip">
+            <sha256 value="9f70eba3f2631674a2d7d3aa01150d697f68be16ad76662ff948d7fe1b4985d8" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.abego.treelayout" name="org.abego.treelayout.core" version="1.0.3">
@@ -1513,6 +1613,11 @@
             <sha256 value="28547dba48cfceb77b6efbfe069aebe9ed3324ae60dbd52093d13a1d636ed069" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.antlr" name="ST4" version="4.3.4">
+         <artifact name="ST4-4.3.4.jar">
+            <sha256 value="f927ac384c46d749f8b5ec68972a53aed21e00313509299616edb73bfa15ff33" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.antlr" name="antlr-runtime" version="3.4">
          <artifact name="antlr-runtime-3.4.jar">
             <sha256 value="5b7cf53b7b30b034023f58030c8147c433f2bee0fe7dec8fae6bebf3708c5a63" origin="Generated by Gradle"/>
@@ -1523,6 +1628,16 @@
             <sha256 value="ce3fc8ecb10f39e9a3cddcbb2ce350d272d9cd3d0b1e18e6fe73c3b9389c8734" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.antlr" name="antlr-runtime" version="3.5.3">
+         <artifact name="antlr-runtime-3.5.3.jar">
+            <sha256 value="68bf9f5a33dfcb34033495c587e6236bef4e37aa6612919f5b1e843b90669fb9" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.antlr" name="antlr4" version="4.11.1">
+         <artifact name="antlr4-4.11.1.jar">
+            <sha256 value="e9686e8a663ca512afe3a2eeb6f6ad3f303abb46188991f19ebc6a0fd9c1c14f" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.antlr" name="antlr4" version="4.5.3">
          <artifact name="antlr4-4.5.3.jar">
             <sha256 value="a32de739cfdf515774e696f91aa9697d2e7731e5cb5045ca8a4b657f8b1b4fb4" origin="Generated by Gradle"/>
@@ -1538,6 +1653,11 @@
             <sha256 value="da66be0c98acfb29bc708300d05f1a3269c40f9984a4cb9251cf2ba1898d1334" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.antlr" name="antlr4-runtime" version="4.11.1">
+         <artifact name="antlr4-runtime-4.11.1.jar">
+            <sha256 value="e06c6553c1ccc14d36052ec4b0fc6f13b808cf957b5b1dc3f61bf401996ada59" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.antlr" name="antlr4-runtime" version="4.5.1-1">
          <artifact name="antlr4-runtime-4.5.1-1.jar">
             <sha256 value="ffca72bc2a25bb2b0c80a58cee60530a78be17da739bb6c91a8c2e3584ca099e" origin="Generated by Gradle"/>
@@ -2431,121 +2551,481 @@
             <sha256 value="22b845d28c39e5fa3478db276ac0a75e6285ea00d30a938146686bfa75f44b8f" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-analysis-common" version="9.4.2">
+         <artifact name="lucene-analysis-common-9.4.2.jar">
+            <sha256 value="cafa2c66ac81894c860e2213c6ba71d4c5a3ae6a26be698318e7de88efdd4db5" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-analysis-common" version="9.5.0">
+         <artifact name="lucene-analysis-common-9.5.0.jar">
+            <sha256 value="1dd59135fc0e141788f5a8eb3d8a5ba40eddb18cefad4e3dbc14f3c55198dfc1" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-analysis-common" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-analysis-common-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="73f276384d65dc01697617112d960a0f01fcc197c4925f2144f386fcb21d9445" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-analysis-icu" version="9.4.1">
          <artifact name="lucene-analysis-icu-9.4.1.jar">
             <sha256 value="a5f35d7294ef475e9e3145e0a191551f5841da738212c4923b0a403a027db200" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-analysis-icu" version="9.4.2">
+         <artifact name="lucene-analysis-icu-9.4.2.jar">
+            <sha256 value="6fc669da198bea83f2c0616a4eb765f30b7dcb729725aa098528151bb36be9d6" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-analysis-icu" version="9.5.0">
+         <artifact name="lucene-analysis-icu-9.5.0.jar">
+            <sha256 value="bc44433bd25b2f1f38412dd062dde0318f51fc1ff551af60d1b325bfc4ff7649" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-analysis-icu" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-analysis-icu-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="946c7756ee4320d09b5d55e7895b6285a5ad29e99c9de54d2aec620c4378fbac" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-analysis-kuromoji" version="9.4.1">
          <artifact name="lucene-analysis-kuromoji-9.4.1.jar">
             <sha256 value="daaf3c6939a9b234682d45102b8c26872f0b2d33631feefb4795a54a6dddc75a" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-analysis-kuromoji" version="9.4.2">
+         <artifact name="lucene-analysis-kuromoji-9.4.2.jar">
+            <sha256 value="13928e760951491b66e1b37f3396a0ee4b150f02a4cbe57743b68dbd0d09ab61" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-analysis-kuromoji" version="9.5.0">
+         <artifact name="lucene-analysis-kuromoji-9.5.0.jar">
+            <sha256 value="ddfe972a178e638a40ec57c94e8d1ca4cde9f671cd98938d0444aa91bcbe9972" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-analysis-kuromoji" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-analysis-kuromoji-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="0a1a38c2e357c67e3583ef8894f336b389a5a5337795b1ef698d7cb0b826d8e6" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-analysis-morfologik" version="9.4.1">
          <artifact name="lucene-analysis-morfologik-9.4.1.jar">
             <sha256 value="55cc0c58ef51d62ad0db64d130bbf163ec2bb9431acb898446e9039165c97f5b" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-analysis-morfologik" version="9.4.2">
+         <artifact name="lucene-analysis-morfologik-9.4.2.jar">
+            <sha256 value="2835fb73f8d58ded27e9201a8d9546e0bdc5d028f5ae5f112f6f74cf71a291ee" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-analysis-morfologik" version="9.5.0">
+         <artifact name="lucene-analysis-morfologik-9.5.0.jar">
+            <sha256 value="d9242040bbd354dcb90680310c14648c52799585523f44d105c93c36a6a99317" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-analysis-morfologik" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-analysis-morfologik-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="7e2cf5326302d55962944d3ba7a18b8a76b9a49d87e15c4c1904a0b4ae6afdac" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-analysis-nori" version="9.4.1">
          <artifact name="lucene-analysis-nori-9.4.1.jar">
             <sha256 value="f3e3119787e002606181078e2d2d28715ea80222dd82d5f34013ab11a097e786" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-analysis-nori" version="9.4.2">
+         <artifact name="lucene-analysis-nori-9.4.2.jar">
+            <sha256 value="316776f69a9e493c36c9a1b9b18b1f10de3914a45537e7f324784ab11d748267" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-analysis-nori" version="9.5.0">
+         <artifact name="lucene-analysis-nori-9.5.0.jar">
+            <sha256 value="f91d9b8f08d9db3c4cc044a6cfc457284f0575dad585403fa0f17f6396143bad" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-analysis-nori" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-analysis-nori-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="4de35194eebdfb226cfe497058cec2dfd5834f7da78251bd5bc5362e0b36f807" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-analysis-phonetic" version="9.4.1">
          <artifact name="lucene-analysis-phonetic-9.4.1.jar">
             <sha256 value="57b21cf3c13a128d5817c369b43bb843a332ada8569bee99300b2439024daf1a" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-analysis-phonetic" version="9.4.2">
+         <artifact name="lucene-analysis-phonetic-9.4.2.jar">
+            <sha256 value="30c2af3312daae2facbc43c49e4635b8f6426641d2870f7c1750ba2496c889fe" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-analysis-phonetic" version="9.5.0">
+         <artifact name="lucene-analysis-phonetic-9.5.0.jar">
+            <sha256 value="6b2b2d4314b36546f70c36e9c8481acd7d532a2ec9587a64318bb544303d4cd4" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-analysis-phonetic" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-analysis-phonetic-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="e65ce76f0ca6dd3ff3befde0a8a04951af46e195284ee6e75eab32ad25aabce3" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-analysis-smartcn" version="9.4.1">
          <artifact name="lucene-analysis-smartcn-9.4.1.jar">
             <sha256 value="ff65b08a942c4e9b4050f4d4a697cb550fe04e12464af14869c09f02fc75bab5" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-analysis-smartcn" version="9.4.2">
+         <artifact name="lucene-analysis-smartcn-9.4.2.jar">
+            <sha256 value="ac101ee6815499d04a29693f089b141a2dd207a063e47968475bd829caa9bcf2" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-analysis-smartcn" version="9.5.0">
+         <artifact name="lucene-analysis-smartcn-9.5.0.jar">
+            <sha256 value="5fda65dd7975f30a7ab3157af4082f6687d736c3d39ac0e5b9299acd22d0b3ac" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-analysis-smartcn" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-analysis-smartcn-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="ca83467b4d20f00a4dbbf7c0ec35b977ed9dfc74498d563335e5a4d4c7d9474e" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-analysis-stempel" version="9.4.1">
          <artifact name="lucene-analysis-stempel-9.4.1.jar">
             <sha256 value="d32007019f35bef2f0ff47b192f4d2faac457bafa448336c905e0c1433e929e7" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-analysis-stempel" version="9.4.2">
+         <artifact name="lucene-analysis-stempel-9.4.2.jar">
+            <sha256 value="7b8848fa29b4506f87e03c86699feb243cd258fe3ce92f0db8201417689e530a" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-analysis-stempel" version="9.5.0">
+         <artifact name="lucene-analysis-stempel-9.5.0.jar">
+            <sha256 value="d6c19af6a6848660fcdad29f7046f2113715c7c50f38904751c46f7054e78e15" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-analysis-stempel" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-analysis-stempel-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="d0ebadf4fee6e727197981f2295dc043ddd47077b58624e76af9d4b84e65464b" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-backward-codecs" version="9.4.1">
          <artifact name="lucene-backward-codecs-9.4.1.jar">
             <sha256 value="abdbf830fae812b27ddc98d8287e60582c8f9cffd8292a064f1f9aa8ab5e0ff0" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-backward-codecs" version="9.4.2">
+         <artifact name="lucene-backward-codecs-9.4.2.jar">
+            <sha256 value="8a6dc9181693d61b25ddb057cec31abdb4d6573b12b4a096b9b5739093a45c91" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-backward-codecs" version="9.5.0">
+         <artifact name="lucene-backward-codecs-9.5.0.jar">
+            <sha256 value="407f16e131e13f626608b82185ef0ed70ecfb1356d0d265042101934795578bb" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-backward-codecs" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-backward-codecs-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="e1ec8f11b9c83c0249eb145729921a08227bbcfe883c409a17309b5ab6a3824e" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-codecs" version="9.4.1">
          <artifact name="lucene-codecs-9.4.1.jar">
             <sha256 value="e69bcce29b43e9e09c85eb9e8a17128aa90a631c8a1b014b4efc7b8487222f4e" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-codecs" version="9.4.2">
+         <artifact name="lucene-codecs-9.4.2.jar">
+            <sha256 value="a8bdb8e14bb6813837e5db84b4d163cd5bc6761bb370d5e6ade28561522eb587" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-codecs" version="9.5.0">
+         <artifact name="lucene-codecs-9.5.0.jar">
+            <sha256 value="4e6ab97b47af716a1f2d6afb4355ee4e0109df7381f278837cf10481ee4c6785" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-codecs" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-codecs-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="3f57ada190b4561df53d0521df67a64204b315720ccc4cc04a5f621cdac21a42" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-core" version="9.4.1">
          <artifact name="lucene-core-9.4.1.jar">
             <sha256 value="992f847526c953f734a18295901b6fbb8977a1d99275e01ed219e7e57cc0ae34" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-core" version="9.4.2">
+         <artifact name="lucene-core-9.4.2.jar">
+            <sha256 value="fd6db1bfe92c51439332230c1e95d2ac63633a9c3067d0355985594ff415431a" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-core" version="9.5.0">
+         <artifact name="lucene-core-9.5.0.jar">
+            <sha256 value="b1f37bf3573d8221d659ad97a239fcdbd49809937564d49d2ccf5c3dc26ba0ea" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-core" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-core-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="481c30915303e79477779b21fdf2769cd944dc5b5acef4ea2eac0bae5a5a59da" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-expressions" version="9.4.1">
          <artifact name="lucene-expressions-9.4.1.jar">
             <sha256 value="b037fbd8131443cf8a2f35f9c07eb68bf4d95cab01be67733a9225e7629a94e7" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-expressions" version="9.4.2">
+         <artifact name="lucene-expressions-9.4.2.jar">
+            <sha256 value="ed0366731de0a57c77e34113df8ea34c365f93d1e60556ec20dc253a10f516f5" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-expressions" version="9.5.0">
+         <artifact name="lucene-expressions-9.5.0.jar">
+            <sha256 value="153e1fee71e4deeb6e0a7d82bdafa7e17d64ee418c61068cc2657b496c447db2" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-expressions" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-expressions-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="edc4e4d8398bf91bbda2717f835f0c94d816dcd26d7a7e2173a3563e8d25a719" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-grouping" version="9.4.1">
          <artifact name="lucene-grouping-9.4.1.jar">
             <sha256 value="dd88245285b240246280a4f922780a68831ed79df40a4884650d17902a33e361" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-grouping" version="9.4.2">
+         <artifact name="lucene-grouping-9.4.2.jar">
+            <sha256 value="0312ba56249fb252ae8842fd148bc231bf4f3c20569a66f92105f1739815381d" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-grouping" version="9.5.0">
+         <artifact name="lucene-grouping-9.5.0.jar">
+            <sha256 value="8bbda497572ba3c13154be176156203ac46325c02c741ff1eaba118b852f8623" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-grouping" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-grouping-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="91ac493aae11d211f2ffbe8457cf2f0c7aba68fcf2ddddb0d4da53b4d6b471cf" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-highlighter" version="9.4.1">
          <artifact name="lucene-highlighter-9.4.1.jar">
             <sha256 value="241af5dd27834d0a37e9ab2ef235aa125ca680181a78c51248511ee80c053240" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-highlighter" version="9.4.2">
+         <artifact name="lucene-highlighter-9.4.2.jar">
+            <sha256 value="8512892073eb1c5f13a3fb1bf1b245cd66aeb1b45f7acfb47a83968a4141318d" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-highlighter" version="9.5.0">
+         <artifact name="lucene-highlighter-9.5.0.jar">
+            <sha256 value="be15113ba91aa033ae8c79abd44df43f22de36428a2849a52a244183553bb253" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-highlighter" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-highlighter-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="ff7eda5d69ef6aaad97fa36472252fc992d0151a6c8c5aad920ac05540be8a19" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-join" version="9.4.1">
          <artifact name="lucene-join-9.4.1.jar">
             <sha256 value="1dfd06b72ce8ac45e1747a1781bf7f75681de366e17e88756c171e506f0719bb" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-join" version="9.4.2">
+         <artifact name="lucene-join-9.4.2.jar">
+            <sha256 value="ae04eaa43bde57407a19aa9ab86038b8ee3b0d5c9d9333ecf63d283c306e6b0e" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-join" version="9.5.0">
+         <artifact name="lucene-join-9.5.0.jar">
+            <sha256 value="a723b89c619052bc353111ac7afe08249d30f232a6efb7d8c8bd3a37f76d1f16" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-join" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-join-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="c29e7b0e2de53408c3d04b6562577679a3932a5f03c45b3d26ad22b1147f2132" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-memory" version="9.4.1">
          <artifact name="lucene-memory-9.4.1.jar">
             <sha256 value="2426d5a2475c9163dcc28b2e6f2624381abc2b83ede3f6140ab3833724ac60e6" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-memory" version="9.4.2">
+         <artifact name="lucene-memory-9.4.2.jar">
+            <sha256 value="a0c9dee125bed499005893dcd9737a02b3aebe3cfa01ce6eb8d28a4e3b5dc5d4" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-memory" version="9.5.0">
+         <artifact name="lucene-memory-9.5.0.jar">
+            <sha256 value="b24a9b4abc743c1c762aa47e0d37bc4fbaad784625b3c7c8c5beb310a6714b68" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-memory" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-memory-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="f9ee01b34d8fa37e9b48426a00cf7e7fa947dd0356330ff9d14cefee345b2255" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-misc" version="9.4.1">
          <artifact name="lucene-misc-9.4.1.jar">
             <sha256 value="de8335139c1622234fc3c0d4973fcee9a7a7d53b26bc1b9ac72c3ba2c2f59b58" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-misc" version="9.4.2">
+         <artifact name="lucene-misc-9.4.2.jar">
+            <sha256 value="c3a3f2a487cf62bad685527f9ace8d4728aa173ab56054c328e6e397584e1992" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-misc" version="9.5.0">
+         <artifact name="lucene-misc-9.5.0.jar">
+            <sha256 value="7f80c756c88e8ac8bcb92cbf1d0c55deb9eb3186ab74d39cb8ed571460dcbbb6" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-misc" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-misc-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="43c258c6016e776d70a248301f4586ab643491800ed1a887bacf1240aaad92a6" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-queries" version="9.4.1">
          <artifact name="lucene-queries-9.4.1.jar">
             <sha256 value="0fce03ae6070868fb7369763ea4a29aecdcc13c6342c7fc94c90d3ae29860a40" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-queries" version="9.4.2">
+         <artifact name="lucene-queries-9.4.2.jar">
+            <sha256 value="1924525c4c706706baecbac7f9c0e434630b9ce595e55ae6aa63ca6d465fb453" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-queries" version="9.5.0">
+         <artifact name="lucene-queries-9.5.0.jar">
+            <sha256 value="4a9f394050ad2ae7d6917dcd7402f8d0c27c95f303f3996dd71c418d63a76f71" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-queries" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-queries-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="7b1534514f1766fc6b6851fcd18e24cbece06373a03410cf71e2b76f23c99bc1" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-queryparser" version="9.4.1">
          <artifact name="lucene-queryparser-9.4.1.jar">
             <sha256 value="c7bf66dea0ae9e7dc840347fd7eaefa9250a91f018bc2503888afed08010adea" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-queryparser" version="9.4.2">
+         <artifact name="lucene-queryparser-9.4.2.jar">
+            <sha256 value="dfc681a69bc60dd89e6bbf964a9b310b11e96de212da8597309b7caeb1cf591d" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-queryparser" version="9.5.0">
+         <artifact name="lucene-queryparser-9.5.0.jar">
+            <sha256 value="a75727f6c3d902e2df095b8cab7f276318babae957931c1d12722865e9af865c" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-queryparser" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-queryparser-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="b4542f0109a7a53c366ff0f4b2b22c573ab4a2400e928d9f0d065c1f731d6e8a" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-sandbox" version="9.4.1">
          <artifact name="lucene-sandbox-9.4.1.jar">
             <sha256 value="ddc756a446986101ab17ac15aedfeb14588832afff38b86c07a13cd93e6e4890" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-sandbox" version="9.4.2">
+         <artifact name="lucene-sandbox-9.4.2.jar">
+            <sha256 value="c381fe2797766d91b663a7cbee60e952a1c81dc37551445086f48f0dc5d47216" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-sandbox" version="9.5.0">
+         <artifact name="lucene-sandbox-9.5.0.jar">
+            <sha256 value="913a025d8c6f7e5dc42bf3a9cded88f2e05ac702be9c7edc359f3db6c149e065" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-sandbox" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-sandbox-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="6c246106bd4ad6ac2a99ee8e87b76e25c7b17ffad333fbeed6206e3d23fba5ba" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-spatial-extras" version="9.4.1">
          <artifact name="lucene-spatial-extras-9.4.1.jar">
             <sha256 value="2fe68c8a9141af9f7a7862b576c88b74ad4509f5a2427cb423463d4233e76d4c" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-spatial-extras" version="9.4.2">
+         <artifact name="lucene-spatial-extras-9.4.2.jar">
+            <sha256 value="c13713f828a03d47d3634e59abc7a53df13c23531d1b6b70c72d190ff64c5cd4" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-spatial-extras" version="9.5.0">
+         <artifact name="lucene-spatial-extras-9.5.0.jar">
+            <sha256 value="9bcc1a8e21b21fbdb49250d4d5ccda46c20d275bca33abc99136cae3aa4fac53" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-spatial-extras" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-spatial-extras-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="2ac4f5172e74a3772b68e86d5346f1edd88f4627d2fe3ce63e2b3d7717f36ee3" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-spatial3d" version="9.4.1">
          <artifact name="lucene-spatial3d-9.4.1.jar">
             <sha256 value="5911614baa3f6e75ee55d0ad63aa16286887766b2fda0654507ce6412d16353b" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-spatial3d" version="9.4.2">
+         <artifact name="lucene-spatial3d-9.4.2.jar">
+            <sha256 value="c9688a9ab51d5a827260892de6435595a0d524ff9426288af806eb595a000dbb" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-spatial3d" version="9.5.0">
+         <artifact name="lucene-spatial3d-9.5.0.jar">
+            <sha256 value="40968f818eb38fd6c15057f68b665325396d89e8ebb8523532fa0326544c06b6" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-spatial3d" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-spatial3d-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="4f4d96e385ddb5187714f5ab274d0e8cb74cab56dedbc5fa1363389c98a617a1" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-suggest" version="9.4.1">
          <artifact name="lucene-suggest-9.4.1.jar">
             <sha256 value="0bd1662bfaa60d6cba12e69b74f90e162900ed077320fe37fbf9e8f5cd78e59f" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-suggest" version="9.4.2">
+         <artifact name="lucene-suggest-9.4.2.jar">
+            <sha256 value="0d56f92c5414149562ea50c6863ac50fcbbb51239d4a72c2bafbe852fb7e0a27" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-suggest" version="9.5.0">
+         <artifact name="lucene-suggest-9.5.0.jar">
+            <sha256 value="f8fe43e8db331060ad58df9a44a8f1fa4937ad62b718cdde484d308c4bb5c83d" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-suggest" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-suggest-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="94878bf9c520702c0d5293922dd06866e25bedbd6f9bb823e28eb34dac907197" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-test-framework" version="9.4.1">
          <artifact name="lucene-test-framework-9.4.1.jar">
             <sha256 value="63850af6b753e0802cba7697db7821727b9b2e23eef11177c1ddbfb37ef3eaf2" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.lucene" name="lucene-test-framework" version="9.4.2">
+         <artifact name="lucene-test-framework-9.4.2.jar">
+            <sha256 value="be318e426e3ce93d704fff41abc8ccd09bb1def69a40615ecc549c21895dd79d" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-test-framework" version="9.5.0">
+         <artifact name="lucene-test-framework-9.5.0.jar">
+            <sha256 value="e22ce1393c4bfb80cce5cac81c766ce2c4588e18128903c94b5e3942e29f5509" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.apache.lucene" name="lucene-test-framework" version="9.5.0-snapshot-d19c3e2e0ed">
+         <artifact name="lucene-test-framework-9.5.0-snapshot-d19c3e2e0ed.jar">
+            <sha256 value="561369612e4602cf99bcc53201b4bf9b95ed09096ba593e755416f1c96ff4bfb" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.maven" name="maven-model" version="3.6.2">
          <artifact name="maven-model-3.6.2.jar">
             <sha256 value="f4ada31d7217efc11d2264dec3716623cefa3440cfb2b6b1dcc640a825159a7d" origin="Generated by Gradle"/>
@@ -2556,19 +3036,19 @@
             <sha256 value="08316826fa2b9357b061e52fa8f19ccae75420c949ebe29e28759d2bddd9b39b" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.pdfbox" name="fontbox" version="2.0.26">
-         <artifact name="fontbox-2.0.26.jar">
-            <sha256 value="94674c7d10ea45d9b0dc10387892637fc8efb5c69d2e72694f1c66feb9bb6ed7" origin="Generated by Gradle"/>
+      <component group="org.apache.pdfbox" name="fontbox" version="2.0.27">
+         <artifact name="fontbox-2.0.27.jar">
+            <sha256 value="dc7429868aaf3d313c524b9aab846a405e89ca4927f35762ca4d1a60bce1d7f4" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.pdfbox" name="jempbox" version="1.8.16">
-         <artifact name="jempbox-1.8.16.jar">
-            <sha256 value="ebef7cca5a5a77768e686972b4a89f0ffce7b46907fd96ac3d4f6ce2fa038055" origin="Generated by Gradle"/>
+      <component group="org.apache.pdfbox" name="jempbox" version="1.8.17">
+         <artifact name="jempbox-1.8.17.jar">
+            <sha256 value="ded9c81038dd1bbcba18f07e1028d70c9ceaf0b48ac56cea8ab6ec2c255fc1b3" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.pdfbox" name="pdfbox" version="2.0.26">
-         <artifact name="pdfbox-2.0.26.jar">
-            <sha256 value="b46ebb4140384b45a39e91e23b595ce9dcd8ba3a9ee914f15f40a42a9f805373" origin="Generated by Gradle"/>
+      <component group="org.apache.pdfbox" name="pdfbox" version="2.0.27">
+         <artifact name="pdfbox-2.0.27.jar">
+            <sha256 value="a25ad2a0be6b0bf9eb0e972abd09c34c0e797a3ce2a980d5ff035ff4cf078037" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.apache.poi" name="poi" version="5.2.2">
@@ -2616,59 +3096,59 @@
             <sha256 value="3902794d36d9b81da1b7e697f21ed04ccae276cc116eecc640a4cd0fff2691f2" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.tika" name="tika-core" version="2.4.0">
-         <artifact name="tika-core-2.4.0.jar">
-            <sha256 value="e6e25f52f3b542d39454d931b02aa4b01df99b384d6adf5bfaaf4566bf9ad3f2" origin="Generated by Gradle"/>
+      <component group="org.apache.tika" name="tika-core" version="2.6.0">
+         <artifact name="tika-core-2.6.0.jar">
+            <sha256 value="efc588724490efc77126c1295d0a6c73ddb812d8bf214ae347fab75e0aa4dc81" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.tika" name="tika-langdetect-tika" version="2.4.0">
-         <artifact name="tika-langdetect-tika-2.4.0.jar">
-            <sha256 value="ad952474c23a44700476c55db7dea95ef239023618d83a8d5bba8e22e2a8b2fb" origin="Generated by Gradle"/>
+      <component group="org.apache.tika" name="tika-langdetect-tika" version="2.6.0">
+         <artifact name="tika-langdetect-tika-2.6.0.jar">
+            <sha256 value="1431c1f18211798672f76749e5118afbbb72f121802ca1613c9801e62186c54f" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.tika" name="tika-parser-apple-module" version="2.4.0">
-         <artifact name="tika-parser-apple-module-2.4.0.jar">
-            <sha256 value="65a66604bb2dce6bd11757d6a09f4d81bb1ac9172a81dba38e12cbe8681ff073" origin="Generated by Gradle"/>
+      <component group="org.apache.tika" name="tika-parser-apple-module" version="2.6.0">
+         <artifact name="tika-parser-apple-module-2.6.0.jar">
+            <sha256 value="c1fbb6e29350e2a8aa48bd4ae5fc8e6bd8904f5e70cf016b60898c5cf559ace8" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.tika" name="tika-parser-html-module" version="2.4.0">
-         <artifact name="tika-parser-html-module-2.4.0.jar">
-            <sha256 value="2538806ef933edf3c18c6ca28077e18d30cfe9233500d0c5ec197bc9ac6f5bcb" origin="Generated by Gradle"/>
+      <component group="org.apache.tika" name="tika-parser-html-module" version="2.6.0">
+         <artifact name="tika-parser-html-module-2.6.0.jar">
+            <sha256 value="b89064745539ca01739d5990ac322aefc91b2103cea3633dfd215823f4847248" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.tika" name="tika-parser-microsoft-module" version="2.4.0">
-         <artifact name="tika-parser-microsoft-module-2.4.0.jar">
-            <sha256 value="760648b2bb5fa6ca99b03eed03c85a4310a20ae7c303fa263575534088b815cc" origin="Generated by Gradle"/>
+      <component group="org.apache.tika" name="tika-parser-microsoft-module" version="2.6.0">
+         <artifact name="tika-parser-microsoft-module-2.6.0.jar">
+            <sha256 value="deb973b247112c17477d96bb4929fb65a28fa6900953732b82998d4c5ec48386" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.tika" name="tika-parser-miscoffice-module" version="2.4.0">
-         <artifact name="tika-parser-miscoffice-module-2.4.0.jar">
-            <sha256 value="074cc6c6bb1e37599d2a9c7dbd8a87414b362ac5e89d7823d87b34dffef33eef" origin="Generated by Gradle"/>
+      <component group="org.apache.tika" name="tika-parser-miscoffice-module" version="2.6.0">
+         <artifact name="tika-parser-miscoffice-module-2.6.0.jar">
+            <sha256 value="901c8d77a4b054ae95203325883649f9acff2411529770554d9f64a4b61d83eb" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.tika" name="tika-parser-pdf-module" version="2.4.0">
-         <artifact name="tika-parser-pdf-module-2.4.0.jar">
-            <sha256 value="46b1a66b14c802b62227456d41027766835772a142d8974923b0f16dd2a7f000" origin="Generated by Gradle"/>
+      <component group="org.apache.tika" name="tika-parser-pdf-module" version="2.6.0">
+         <artifact name="tika-parser-pdf-module-2.6.0.jar">
+            <sha256 value="7900c08e952407b1754c1e07232d8d72a01e51e3ba602fcbe9e6310b2a476f3e" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.tika" name="tika-parser-text-module" version="2.4.0">
-         <artifact name="tika-parser-text-module-2.4.0.jar">
-            <sha256 value="bea22bb1ea5b6a3a18d7b5a3bbf0689f2becf7568799f0cd2965cb39348959e2" origin="Generated by Gradle"/>
+      <component group="org.apache.tika" name="tika-parser-text-module" version="2.6.0">
+         <artifact name="tika-parser-text-module-2.6.0.jar">
+            <sha256 value="15b7e20c51ed14d1dac25ba9b7d1179603a36f93204e7f445c0ab98e41b14d83" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.tika" name="tika-parser-xml-module" version="2.4.0">
-         <artifact name="tika-parser-xml-module-2.4.0.jar">
-            <sha256 value="53e14c28b6b0b1d76b8b901cc3c8f4f04f12e5e5adfec4aee73a53fa82b83980" origin="Generated by Gradle"/>
+      <component group="org.apache.tika" name="tika-parser-xml-module" version="2.6.0">
+         <artifact name="tika-parser-xml-module-2.6.0.jar">
+            <sha256 value="124d3ef1a35fc368bc3a49d616821bfe997e95e50f1c991b7d7076b7e9546462" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.tika" name="tika-parser-xmp-commons" version="2.4.0">
-         <artifact name="tika-parser-xmp-commons-2.4.0.jar">
-            <sha256 value="361c615f0c5322ba786d2b06274f9946f5ec8de7784edf4fa26375f755299a45" origin="Generated by Gradle"/>
+      <component group="org.apache.tika" name="tika-parser-xmp-commons" version="2.6.0">
+         <artifact name="tika-parser-xmp-commons-2.6.0.jar">
+            <sha256 value="b96bb7d066ced0a88cf3223363642dcba3d581592d5b0770ed197e7520f5bbf0" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.tika" name="tika-parser-zip-commons" version="2.4.0">
-         <artifact name="tika-parser-zip-commons-2.4.0.jar">
-            <sha256 value="9b6fddbaf712e9f8810faa1e0eabd97dd051c9ca33f021d812fda4d75c83b60c" origin="Generated by Gradle"/>
+      <component group="org.apache.tika" name="tika-parser-zip-commons" version="2.6.0">
+         <artifact name="tika-parser-zip-commons-2.6.0.jar">
+            <sha256 value="84a81033ab310540176d480d15f7409d42d33453e474f7a2feaa906a4431cea2" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.apache.xmlbeans" name="xmlbeans" version="5.0.3">
@@ -2959,9 +3439,9 @@
             <sha256 value="f74df330614a191adabffceed5952b79b43c6935df20f9046ae92d716df5ada5" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.eclipse.jgit" name="org.eclipse.jgit" version="5.13.0.202109080827-r">
-         <artifact name="org.eclipse.jgit-5.13.0.202109080827-r.jar">
-            <sha256 value="dabf8369f0cdf8cf17b7f7da4bda9320c570bb754c88e0beb7b852812cf5cd25" origin="Generated by Gradle"/>
+      <component group="org.eclipse.jgit" name="org.eclipse.jgit" version="5.13.1.202206130422-r">
+         <artifact name="org.eclipse.jgit-5.13.1.202206130422-r.jar">
+            <sha256 value="d679365a6c8e55c1496701099f5e5765433f68dcb659759416ba5b222eb5055c" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.eclipse.platform" name="org.eclipse.core.commands" version="3.10.100">
@@ -3044,6 +3524,11 @@
             <sha256 value="0e1dec40a1ede965941251eda968aeee052cc4f50378bc316cc48e8159bdbeb4" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.glassfish" name="javax.json" version="1.1.4">
+         <artifact name="javax.json-1.1.4.jar">
+            <sha256 value="17fdeb7e22375a7fb40bb0551306f6dcf2b5743078668adcdf6c642c9a9ec955" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.hamcrest" name="hamcrest" version="2.1">
          <artifact name="hamcrest-2.1.jar">
             <sha256 value="ba93b2e3a562322ba432f0a1b53addcc55cb188253319a020ed77f824e692050" origin="Generated by Gradle"/>
@@ -3399,6 +3884,11 @@
             <sha256 value="1263369b59e29c943918de11d6d6152e2ec6085ce63e5710516f8c67d368e4bc" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.ow2.asm" name="asm" version="9.4">
+         <artifact name="asm-9.4.jar">
+            <sha256 value="39d0e2b3dc45af65a09b097945750a94a126e052e124f93468443a1d0e15f381" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.ow2.asm" name="asm-analysis" version="7.2">
          <artifact name="asm-analysis-7.2.jar">
             <sha256 value="be922aae60ff1ff1768e8e6544a38a7f92bd0a6d6b0b9791f94955d1bd453de2" origin="Generated by Gradle"/>
@@ -3449,6 +3939,11 @@
             <sha256 value="ae629c2609f39681ef8d140a42a23800464a94f2d23e36d8f25cd10d5e4caff4" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.ow2.asm" name="asm-tree" version="9.4">
+         <artifact name="asm-tree-9.4.jar">
+            <sha256 value="c42d479cf24566a21eb20af7eeaeef4e86bdb4a886306cf72f483b65e75b2acf" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.ow2.asm" name="asm-util" version="7.2">
          <artifact name="asm-util-7.2.jar">
             <sha256 value="6e24913b021ffacfe8e7e053d6e0ccc731941148cfa078d4f1ed3d96904530f8" origin="Generated by Gradle"/>
@@ -3459,6 +3954,11 @@
             <sha256 value="1dee0481072d19c929b623e155e14d2f6085dc011529a0a0dbefc84cf571d865" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.reactivestreams" name="reactive-streams" version="1.0.4">
+         <artifact name="reactive-streams-1.0.4.jar">
+            <sha256 value="f75ca597789b3dac58f61857b9ac2e1034a68fa672db35055a8fb4509e325f28" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.redline-rpm" name="redline" version="1.2.10">
          <artifact name="redline-1.2.10.jar">
             <sha256 value="2e074e7bd42174eaeac98cebc782f0d7bcc8dc2a41617932ac9c79d56bc86ecd" origin="Generated by Gradle"/>
@@ -3539,9 +4039,9 @@
             <sha256 value="5196a0da2c5a33d1a04e88fc7a9cc109501bc265b5bac8edd9984a1885070ad4" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.threeten" name="threetenbp" version="1.5.1">
-         <artifact name="threetenbp-1.5.1.jar">
-            <sha256 value="4342ee04d87040f71b0aa9188ee960780ef2da734e32a8d43a522a580b5e0f3b" origin="Generated by Gradle"/>
+      <component group="org.threeten" name="threetenbp" version="1.6.5">
+         <artifact name="threetenbp-1.6.5.jar">
+            <sha256 value="b2551604c0d6516428e3065213b74461240378a07622201a37ad32638ecb417c" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.tmatesoft.sqljet" name="sqljet" version="1.1.10">
diff --git a/libs/build.gradle b/libs/build.gradle
index f2b7e247fabd..85c331149fc5 100644
--- a/libs/build.gradle
+++ b/libs/build.gradle
@@ -26,6 +26,7 @@ configure(subprojects - project('elasticsearch-log4j')) {
           && false == isPluginApi(project, depProject)
           && false == depProject.path.equals(':libs:elasticsearch-x-content')
           && false == depProject.path.equals(':libs:elasticsearch-core')
+          && false == depProject.path.equals(':libs:elasticsearch-plugin-api')
           && depProject.path.startsWith(':libs')
           && depProject.name.startsWith('elasticsearch-')) {
           throw new InvalidUserDataException("projects in :libs "
diff --git a/libs/core/src/main/java/org/elasticsearch/core/AbstractRefCounted.java b/libs/core/src/main/java/org/elasticsearch/core/AbstractRefCounted.java
index 5f5331c55549..42073489faf8 100644
--- a/libs/core/src/main/java/org/elasticsearch/core/AbstractRefCounted.java
+++ b/libs/core/src/main/java/org/elasticsearch/core/AbstractRefCounted.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.core;
 
+import java.util.Objects;
 import java.util.concurrent.atomic.AtomicInteger;
 
 /**
@@ -47,7 +48,7 @@ public final boolean tryIncRef() {
     public final boolean decRef() {
         touch();
         int i = refCount.decrementAndGet();
-        assert i >= 0;
+        assert i >= 0 : "invalid decRef call: already closed";
         if (i == 0) {
             try {
                 closeInternal();
@@ -94,11 +95,18 @@ public final int refCount() {
      * Construct an {@link AbstractRefCounted} which runs the given {@link Runnable} when all references are released.
      */
     public static AbstractRefCounted of(Runnable onClose) {
+        Objects.requireNonNull(onClose);
         return new AbstractRefCounted() {
             @Override
             protected void closeInternal() {
                 onClose.run();
             }
+
+            @Override
+            public String toString() {
+                return "refCounted[" + onClose + "]";
+            }
         };
     }
+
 }
diff --git a/libs/core/src/main/java/org/elasticsearch/core/Releasables.java b/libs/core/src/main/java/org/elasticsearch/core/Releasables.java
index 05e3d154929a..2d6781326a17 100644
--- a/libs/core/src/main/java/org/elasticsearch/core/Releasables.java
+++ b/libs/core/src/main/java/org/elasticsearch/core/Releasables.java
@@ -10,7 +10,7 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 
 /** Utility methods to work with {@link Releasable}s. */
 public enum Releasables {
@@ -98,13 +98,22 @@ public static Releasable wrap(final Releasable... releasables) {
     }
 
     /**
-     * Wraps a {@link Releasable} such that its {@link Releasable#close()} method can be called multiple times without double releasing.
+     * Wraps a {@link Releasable} such that its {@link Releasable#close()} method can be called multiple times without double-releasing.
      */
     public static Releasable releaseOnce(final Releasable releasable) {
-        final AtomicBoolean released = new AtomicBoolean(false);
-        return () -> {
-            if (released.compareAndSet(false, true)) {
-                releasable.close();
+        final var ref = new AtomicReference<>(releasable);
+        return new Releasable() {
+            @Override
+            public void close() {
+                final var acquired = ref.getAndSet(null);
+                if (acquired != null) {
+                    acquired.close();
+                }
+            }
+
+            @Override
+            public String toString() {
+                return "releaseOnce[" + ref.get() + "]";
             }
         };
     }
diff --git a/libs/core/src/main/java/org/elasticsearch/core/Streams.java b/libs/core/src/main/java/org/elasticsearch/core/Streams.java
index c32c7ddb8b1f..f28a8615c82f 100644
--- a/libs/core/src/main/java/org/elasticsearch/core/Streams.java
+++ b/libs/core/src/main/java/org/elasticsearch/core/Streams.java
@@ -12,6 +12,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.nio.ByteBuffer;
 
 /**
  * Simple utility methods for file and stream copying. All copy methods close all affected streams when done.
@@ -78,6 +79,62 @@ public static long copy(final InputStream in, final OutputStream out) throws IOE
         return copy(in, out, LOCAL_BUFFER.get(), true);
     }
 
+    /**
+     * Read up to {code count} bytes from {@code input} and store them into {@code buffer}.
+     * The buffers position will be incremented by the number of bytes read from the stream.
+     * @param input stream to read from
+     * @param buffer buffer to read into
+     * @param count maximum number of bytes to read
+     * @return number of bytes read from the stream
+     * @throws IOException in case of I/O errors
+     */
+    public static int read(InputStream input, ByteBuffer buffer, int count) throws IOException {
+        if (buffer.hasArray()) {
+            return readToHeapBuffer(input, buffer, count);
+        }
+        return readToDirectBuffer(input, buffer, count);
+    }
+
+    private static int readToHeapBuffer(InputStream input, ByteBuffer buffer, int count) throws IOException {
+        final int pos = buffer.position();
+        int read = readFully(input, buffer.array(), buffer.arrayOffset() + pos, count);
+        if (read > 0) {
+            buffer.position(pos + read);
+        }
+        return read;
+    }
+
+    private static int readToDirectBuffer(InputStream input, ByteBuffer b, int count) throws IOException {
+        int totalRead = 0;
+        final byte[] buffer = LOCAL_BUFFER.get();
+        while (totalRead < count) {
+            final int len = Math.min(count - totalRead, buffer.length);
+            final int read = input.read(buffer, 0, len);
+            if (read == -1) {
+                break;
+            }
+            b.put(buffer, 0, read);
+            totalRead += read;
+        }
+        return totalRead;
+    }
+
+    public static int readFully(InputStream reader, byte[] dest) throws IOException {
+        return readFully(reader, dest, 0, dest.length);
+    }
+
+    public static int readFully(InputStream reader, byte[] dest, int offset, int len) throws IOException {
+        int read = 0;
+        while (read < len) {
+            final int r = reader.read(dest, offset + read, len - read);
+            if (r == -1) {
+                break;
+            }
+            read += r;
+        }
+        return read;
+    }
+
     /**
      * Wraps an {@link OutputStream} such that it's {@code close} method becomes a noop
      *
diff --git a/libs/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java b/libs/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java
deleted file mode 100644
index a373161c71d5..000000000000
--- a/libs/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.common.util.concurrent;
-
-import org.elasticsearch.core.AbstractRefCounted;
-import org.elasticsearch.test.ESTestCase;
-import org.hamcrest.Matchers;
-
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-
-public class RefCountedTests extends ESTestCase {
-
-    public void testRefCount() {
-        MyRefCounted counted = new MyRefCounted();
-
-        int incs = randomIntBetween(1, 100);
-        for (int i = 0; i < incs; i++) {
-            if (randomBoolean()) {
-                counted.incRef();
-            } else {
-                assertTrue(counted.tryIncRef());
-            }
-            counted.ensureOpen();
-        }
-
-        for (int i = 0; i < incs; i++) {
-            counted.decRef();
-            counted.ensureOpen();
-        }
-
-        counted.incRef();
-        counted.decRef();
-        for (int i = 0; i < incs; i++) {
-            if (randomBoolean()) {
-                counted.incRef();
-            } else {
-                assertTrue(counted.tryIncRef());
-            }
-            counted.ensureOpen();
-        }
-
-        for (int i = 0; i < incs; i++) {
-            counted.decRef();
-            counted.ensureOpen();
-        }
-
-        counted.decRef();
-        assertFalse(counted.tryIncRef());
-        assertThat(
-            expectThrows(IllegalStateException.class, counted::incRef).getMessage(),
-            equalTo(AbstractRefCounted.ALREADY_CLOSED_MESSAGE)
-        );
-        assertThat(expectThrows(IllegalStateException.class, counted::ensureOpen).getMessage(), equalTo("closed"));
-    }
-
-    public void testMultiThreaded() throws InterruptedException {
-        final MyRefCounted counted = new MyRefCounted();
-        Thread[] threads = new Thread[randomIntBetween(2, 5)];
-        final CountDownLatch latch = new CountDownLatch(1);
-        final CopyOnWriteArrayList<Exception> exceptions = new CopyOnWriteArrayList<>();
-        for (int i = 0; i < threads.length; i++) {
-            threads[i] = new Thread(() -> {
-                try {
-                    latch.await();
-                    for (int j = 0; j < 10000; j++) {
-                        counted.incRef();
-                        assertTrue(counted.hasReferences());
-                        try {
-                            counted.ensureOpen();
-                        } finally {
-                            counted.decRef();
-                        }
-                    }
-                } catch (Exception e) {
-                    exceptions.add(e);
-                }
-            });
-            threads[i].start();
-        }
-        latch.countDown();
-        for (Thread thread : threads) {
-            thread.join();
-        }
-        counted.decRef();
-        assertThat(expectThrows(IllegalStateException.class, counted::ensureOpen).getMessage(), equalTo("closed"));
-        assertThat(
-            expectThrows(IllegalStateException.class, counted::incRef).getMessage(),
-            equalTo(AbstractRefCounted.ALREADY_CLOSED_MESSAGE)
-        );
-        assertThat(counted.refCount(), is(0));
-        assertFalse(counted.hasReferences());
-        assertThat(exceptions, Matchers.emptyIterable());
-    }
-
-    private static final class MyRefCounted extends AbstractRefCounted {
-
-        private final AtomicBoolean closed = new AtomicBoolean(false);
-
-        @Override
-        protected void closeInternal() {
-            this.closed.set(true);
-        }
-
-        public void ensureOpen() {
-            if (closed.get()) {
-                assertEquals(0, this.refCount());
-                assertFalse(hasReferences());
-                throw new IllegalStateException("closed");
-            }
-        }
-    }
-}
diff --git a/libs/core/src/test/java/org/elasticsearch/core/AbstractRefCountedTests.java b/libs/core/src/test/java/org/elasticsearch/core/AbstractRefCountedTests.java
new file mode 100644
index 000000000000..9610bae32a77
--- /dev/null
+++ b/libs/core/src/test/java/org/elasticsearch/core/AbstractRefCountedTests.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.core;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+public class AbstractRefCountedTests extends ESTestCase {
+
+    public void testRefCount() {
+        final RefCounted counted = createRefCounted();
+
+        int incs = randomIntBetween(1, 100);
+        for (int i = 0; i < incs; i++) {
+            if (randomBoolean()) {
+                counted.incRef();
+            } else {
+                assertTrue(counted.tryIncRef());
+            }
+            assertTrue(counted.hasReferences());
+        }
+
+        for (int i = 0; i < incs; i++) {
+            counted.decRef();
+            assertTrue(counted.hasReferences());
+        }
+
+        counted.incRef();
+        counted.decRef();
+        for (int i = 0; i < incs; i++) {
+            if (randomBoolean()) {
+                counted.incRef();
+            } else {
+                assertTrue(counted.tryIncRef());
+            }
+            assertTrue(counted.hasReferences());
+        }
+
+        for (int i = 0; i < incs; i++) {
+            counted.decRef();
+            assertTrue(counted.hasReferences());
+        }
+
+        counted.decRef();
+        assertFalse(counted.tryIncRef());
+        assertThat(
+            expectThrows(IllegalStateException.class, counted::incRef).getMessage(),
+            equalTo(AbstractRefCounted.ALREADY_CLOSED_MESSAGE)
+        );
+        assertFalse(counted.hasReferences());
+    }
+
+    public void testMultiThreaded() throws InterruptedException {
+        final AbstractRefCounted counted = createRefCounted();
+        final Thread[] threads = new Thread[randomIntBetween(2, 5)];
+        final CountDownLatch latch = new CountDownLatch(1);
+        for (int i = 0; i < threads.length; i++) {
+            threads[i] = new Thread(() -> {
+                try {
+                    latch.await();
+                    for (int j = 0; j < 10000; j++) {
+                        assertTrue(counted.hasReferences());
+                        if (randomBoolean()) {
+                            counted.incRef();
+                        } else {
+                            assertTrue(counted.tryIncRef());
+                        }
+                        assertTrue(counted.hasReferences());
+                        counted.decRef();
+                    }
+                } catch (Exception e) {
+                    throw new AssertionError(e);
+                }
+            });
+            threads[i].start();
+        }
+        latch.countDown();
+        for (Thread thread : threads) {
+            thread.join();
+        }
+        counted.decRef();
+        assertFalse(counted.hasReferences());
+        assertThat(
+            expectThrows(IllegalStateException.class, counted::incRef).getMessage(),
+            equalTo(AbstractRefCounted.ALREADY_CLOSED_MESSAGE)
+        );
+        assertThat(counted.refCount(), is(0));
+        assertFalse(counted.hasReferences());
+    }
+
+    public void testToString() {
+        assertEquals("refCounted[runnable description]", createRefCounted().toString());
+    }
+
+    public void testNullCheck() {
+        expectThrows(NullPointerException.class, () -> AbstractRefCounted.of(null));
+    }
+
+    private static AbstractRefCounted createRefCounted() {
+        final var closed = new AtomicBoolean();
+        return AbstractRefCounted.of(new Runnable() {
+            @Override
+            public void run() {
+                assertTrue(closed.compareAndSet(false, true));
+            }
+
+            @Override
+            public String toString() {
+                return "runnable description";
+            }
+        });
+    }
+}
diff --git a/libs/core/src/test/java/org/elasticsearch/core/internal/provider/EmbeddedImplClassLoaderTests.java b/libs/core/src/test/java/org/elasticsearch/core/internal/provider/EmbeddedImplClassLoaderTests.java
index 9cabf7a1ab97..db517b9cfabd 100644
--- a/libs/core/src/test/java/org/elasticsearch/core/internal/provider/EmbeddedImplClassLoaderTests.java
+++ b/libs/core/src/test/java/org/elasticsearch/core/internal/provider/EmbeddedImplClassLoaderTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.core.internal.provider;
 
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.internal.provider.EmbeddedImplClassLoader.CompoundEnumeration;
 import org.elasticsearch.test.ESTestCase;
@@ -452,7 +453,7 @@ private void testResourcesVersioned(String resourcePath, boolean enableMulti, in
             // getResources
             var urls1 = Collections.list(urlcLoader.getResources(resourcePath)).stream().map(URL::toString).toList();
             var urls2 = Collections.list(embedLoader.getResources(resourcePath)).stream().map(URL::toString).toList();
-            assertThat(String.format(Locale.ROOT, "urls1=%s, urls2=%s", urls1, urls2), urls2, hasSize(1));
+            assertThat(Strings.format("urls1=%s, urls2=%s", urls1, urls2), urls2, hasSize(1));
             assertThat(urls1.get(0), endsWith("!/" + expectedURLSuffix));
             assertThat(urls2.get(0), endsWith("impl.jar!/IMPL-JARS/res/res-impl.jar/" + expectedURLSuffix));
 
diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/CircleTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/CircleTests.java
index bae91638bd80..321dd5ccd4e8 100644
--- a/libs/geo/src/test/java/org/elasticsearch/geometry/CircleTests.java
+++ b/libs/geo/src/test/java/org/elasticsearch/geometry/CircleTests.java
@@ -59,4 +59,9 @@ public void testInitValidation() {
 
         StandardValidator.instance(true).validate(new Circle(200, 10, 1, 20));
     }
+
+    @Override
+    protected Circle mutateInstance(Circle instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/GeometryCollectionTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/GeometryCollectionTests.java
index 45842505ce5f..6a7bda7f9e0b 100644
--- a/libs/geo/src/test/java/org/elasticsearch/geometry/GeometryCollectionTests.java
+++ b/libs/geo/src/test/java/org/elasticsearch/geometry/GeometryCollectionTests.java
@@ -64,4 +64,9 @@ public void testInitValidation() {
 
         StandardValidator.instance(true).validate(new GeometryCollection<Geometry>(Collections.singletonList(new Point(20, 10, 30))));
     }
+
+    @Override
+    protected GeometryCollection<Geometry> mutateInstance(GeometryCollection<Geometry> instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/LineTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/LineTests.java
index 3003a243c96a..d3c386883fe0 100644
--- a/libs/geo/src/test/java/org/elasticsearch/geometry/LineTests.java
+++ b/libs/geo/src/test/java/org/elasticsearch/geometry/LineTests.java
@@ -80,4 +80,9 @@ public void testWKTValidation() {
         );
         assertEquals("found Z value [6.0] but [ignore_z_value] parameter is [false]", ex.getMessage());
     }
+
+    @Override
+    protected Line mutateInstance(Line instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/MultiLineTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/MultiLineTests.java
index b7d47784026d..68acba559626 100644
--- a/libs/geo/src/test/java/org/elasticsearch/geometry/MultiLineTests.java
+++ b/libs/geo/src/test/java/org/elasticsearch/geometry/MultiLineTests.java
@@ -62,4 +62,9 @@ public void testValidation() {
                 new MultiLine(Collections.singletonList(new Line(new double[] { 3, 4 }, new double[] { 1, 2 }, new double[] { 6, 5 })))
             );
     }
+
+    @Override
+    protected MultiLine mutateInstance(MultiLine instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/MultiPointTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/MultiPointTests.java
index 011d29322c9d..b25057182e95 100644
--- a/libs/geo/src/test/java/org/elasticsearch/geometry/MultiPointTests.java
+++ b/libs/geo/src/test/java/org/elasticsearch/geometry/MultiPointTests.java
@@ -69,4 +69,9 @@ public void testValidation() {
 
         StandardValidator.instance(true).validate(new MultiPoint(Collections.singletonList(new Point(2, 1, 3))));
     }
+
+    @Override
+    protected MultiPoint mutateInstance(MultiPoint instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/MultiPolygonTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/MultiPolygonTests.java
index bb6046d60960..ff4a6047f263 100644
--- a/libs/geo/src/test/java/org/elasticsearch/geometry/MultiPolygonTests.java
+++ b/libs/geo/src/test/java/org/elasticsearch/geometry/MultiPolygonTests.java
@@ -78,4 +78,9 @@ public void testValidation() {
                 )
             );
     }
+
+    @Override
+    protected MultiPolygon mutateInstance(MultiPolygon instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/PointTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/PointTests.java
index 40ae05802ae8..136c6b8154b1 100644
--- a/libs/geo/src/test/java/org/elasticsearch/geometry/PointTests.java
+++ b/libs/geo/src/test/java/org/elasticsearch/geometry/PointTests.java
@@ -56,4 +56,9 @@ public void testWKTValidation() {
         );
         assertEquals("found Z value [100.0] but [ignore_z_value] parameter is [false]", ex.getMessage());
     }
+
+    @Override
+    protected Point mutateInstance(Point instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/PolygonTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/PolygonTests.java
index 5445de367c21..7c9fdcf7bf16 100644
--- a/libs/geo/src/test/java/org/elasticsearch/geometry/PolygonTests.java
+++ b/libs/geo/src/test/java/org/elasticsearch/geometry/PolygonTests.java
@@ -132,4 +132,9 @@ public void testWKTValidation() {
             ex.getMessage()
         );
     }
+
+    @Override
+    protected Polygon mutateInstance(Polygon instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/RectangleTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/RectangleTests.java
index b89ea7fe279c..88315412524b 100644
--- a/libs/geo/src/test/java/org/elasticsearch/geometry/RectangleTests.java
+++ b/libs/geo/src/test/java/org/elasticsearch/geometry/RectangleTests.java
@@ -55,4 +55,9 @@ public void testInitValidation() {
 
         StandardValidator.instance(true).validate(new Rectangle(50, 10, 40, 30, 20, 60));
     }
+
+    @Override
+    protected Rectangle mutateInstance(Rectangle instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/libs/grok/src/main/java/org/elasticsearch/grok/GrokCaptureExtracter.java b/libs/grok/src/main/java/org/elasticsearch/grok/GrokCaptureExtracter.java
index a4acf0921d36..8a75d125646f 100644
--- a/libs/grok/src/main/java/org/elasticsearch/grok/GrokCaptureExtracter.java
+++ b/libs/grok/src/main/java/org/elasticsearch/grok/GrokCaptureExtracter.java
@@ -20,28 +20,49 @@
 /**
  * How to extract matches.
  */
-public abstract class GrokCaptureExtracter {
+public interface GrokCaptureExtracter {
+
     /**
      * Extract {@link Map} results. This implementation of {@link GrokCaptureExtracter}
      * is mutable and should be discarded after collecting a single result.
      */
-    static class MapExtracter extends GrokCaptureExtracter {
+    class MapExtracter implements GrokCaptureExtracter {
         private final Map<String, Object> result;
         private final List<GrokCaptureExtracter> fieldExtracters;
 
+        @SuppressWarnings("unchecked")
         MapExtracter(List<GrokCaptureConfig> captureConfig) {
             result = captureConfig.isEmpty() ? emptyMap() : new HashMap<>();
             fieldExtracters = new ArrayList<>(captureConfig.size());
             for (GrokCaptureConfig config : captureConfig) {
-                fieldExtracters.add(config.objectExtracter(v -> result.put(config.name(), v)));
+                fieldExtracters.add(config.objectExtracter(value -> {
+                    var key = config.name();
+
+                    // Logstash's Grok processor flattens the list of values to a single value in case there's only 1 match,
+                    // so we have to do the same to be compatible.
+                    // e.g.:
+                    // pattern = `%{SINGLEDIGIT:name}(%{SINGLEDIGIT:name})?`
+                    // - GROK(pattern, "1") => { name: 1 }
+                    // - GROK(pattern, "12") => { name: [1, 2] }
+                    if (result.containsKey(key)) {
+                        if (result.get(key)instanceof List<?> values) {
+                            ((ArrayList<Object>) values).add(value);
+                        } else {
+                            var values = new ArrayList<>();
+                            values.add(result.get(key));
+                            values.add(value);
+                            result.put(key, values);
+                        }
+                    } else {
+                        result.put(key, value);
+                    }
+                }));
             }
         }
 
         @Override
-        void extract(byte[] utf8Bytes, int offset, Region region) {
-            for (GrokCaptureExtracter extracter : fieldExtracters) {
-                extracter.extract(utf8Bytes, offset, region);
-            }
+        public void extract(byte[] utf8Bytes, int offset, Region region) {
+            fieldExtracters.forEach(extracter -> extracter.extract(utf8Bytes, offset, region));
         }
 
         Map<String, Object> result() {
@@ -49,5 +70,5 @@ Map<String, Object> result() {
         }
     }
 
-    abstract void extract(byte[] utf8Bytes, int offset, Region region);
+    void extract(byte[] utf8Bytes, int offset, Region region);
 }
diff --git a/libs/grok/src/main/java/org/elasticsearch/grok/GrokCaptureType.java b/libs/grok/src/main/java/org/elasticsearch/grok/GrokCaptureType.java
index d301febb261e..50ac44c7e13b 100644
--- a/libs/grok/src/main/java/org/elasticsearch/grok/GrokCaptureType.java
+++ b/libs/grok/src/main/java/org/elasticsearch/grok/GrokCaptureType.java
@@ -9,7 +9,6 @@
 package org.elasticsearch.grok;
 
 import org.elasticsearch.grok.GrokCaptureConfig.NativeExtracterMap;
-import org.joni.Region;
 
 import java.nio.charset.StandardCharsets;
 import java.util.function.Consumer;
@@ -70,16 +69,12 @@ static GrokCaptureType fromString(String str) {
     }
 
     protected final GrokCaptureExtracter rawExtracter(int[] backRefs, Consumer<? super String> emit) {
-        return new GrokCaptureExtracter() {
-            @Override
-            void extract(byte[] utf8Bytes, int offset, Region region) {
-                for (int number : backRefs) {
-                    if (region.beg[number] >= 0) {
-                        int matchOffset = offset + region.beg[number];
-                        int matchLength = region.end[number] - region.beg[number];
-                        emit.accept(new String(utf8Bytes, matchOffset, matchLength, StandardCharsets.UTF_8));
-                        return; // Capture only the first value.
-                    }
+        return (utf8Bytes, offset, region) -> {
+            for (int number : backRefs) {
+                if (region.beg[number] >= 0) {
+                    int matchOffset = offset + region.beg[number];
+                    int matchLength = region.end[number] - region.beg[number];
+                    emit.accept(new String(utf8Bytes, matchOffset, matchLength, StandardCharsets.UTF_8));
                 }
             }
         };
diff --git a/libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java b/libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java
index b668b73443e6..0a134f7ce56b 100644
--- a/libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java
+++ b/libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java
@@ -801,9 +801,15 @@ public void testMultipleNamedCapturesWithSameName() {
         Grok grok = new Grok(bank, "%{SINGLEDIGIT:num}%{SINGLEDIGIT:num}", logger::warn);
         assertCaptureConfig(grok, Map.of("num", STRING));
 
-        Map<String, Object> expected = new HashMap<>();
-        expected.put("num", "1");
-        assertThat(grok.captures("12"), equalTo(expected));
+        assertThat(grok.captures("12"), equalTo(Map.of("num", List.of("1", "2"))));
+
+        grok = new Grok(bank, "%{SINGLEDIGIT:num:int}(%{SINGLEDIGIT:num:int})?", logger::warn);
+        assertCaptureConfig(grok, Map.of("num", INTEGER));
+        assertEquals(grok.captures("1"), Map.of("num", 1));
+        assertEquals(grok.captures("1a"), Map.of("num", 1));
+        assertEquals(grok.captures("a1"), Map.of("num", 1));
+        assertEquals(grok.captures("12"), Map.of("num", List.of(1, 2)));
+        assertEquals(grok.captures("123"), Map.of("num", List.of(1, 2)));
     }
 
     public void testExponentialExpressions() {
diff --git a/libs/h3/NOTICE.txt b/libs/h3/NOTICE.txt
index a8e5e1c94c87..7074527ca3fb 100644
--- a/libs/h3/NOTICE.txt
+++ b/libs/h3/NOTICE.txt
@@ -19,3 +19,33 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
+
+--
+This project contains code sourced from https://github.com/jeffhain/jafama which is licensed under the Apache 2.0 License.
+
+Copyright 2012 Jeff Hain
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+=============================================================================
+Notice of fdlibm package this program is partially derived from:
+
+Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.
+
+Developed at SunSoft, a Sun Microsystems, Inc. business.
+Permission to use, copy, modify, and distribute this
+software is freely granted, provided that this notice
+is preserved.
+=============================================================================
+
+
diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java b/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java
index e4aa9876ddfa..5192fe836e73 100644
--- a/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java
+++ b/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java
@@ -30,6 +30,10 @@ final class Constants {
      * sqrt(3) / 2.0
      */
     public static double M_SQRT3_2 = 0.8660254037844386467637231707529361834714;
+    /**
+     * 2.0 * PI
+     */
+    public static final double M_2PI = 6.28318530717958647692528676655900576839433;
     /**
      * max H3 resolution; H3 version 1 has 16 resolutions, numbered 0 through 15
      */
diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/CoordIJK.java b/libs/h3/src/main/java/org/elasticsearch/h3/CoordIJK.java
index 8ef22379492c..e57f681fc2ea 100644
--- a/libs/h3/src/main/java/org/elasticsearch/h3/CoordIJK.java
+++ b/libs/h3/src/main/java/org/elasticsearch/h3/CoordIJK.java
@@ -106,11 +106,20 @@ void reset(int i, int j, int k) {
      * Find the center point in 2D cartesian coordinates of a hex.
      */
     public Vec2d ijkToHex2d() {
-        int i = this.i - this.k;
-        int j = this.j - this.k;
+        final int i = Math.subtractExact(this.i, this.k);
+        final int j = Math.subtractExact(this.j, this.k);
         return new Vec2d(i - 0.5 * j, j * Constants.M_SQRT3_2);
     }
 
+    /**
+     * Find the center point in spherical coordinates of a hex on a particular icosahedral face.
+     */
+    public LatLng ijkToGeo(int face, int res, boolean substrate) {
+        final int i = Math.subtractExact(this.i, this.k);
+        final int j = Math.subtractExact(this.j, this.k);
+        return Vec2d.hex2dToGeo(i - 0.5 * j, j * Constants.M_SQRT3_2, face, res, substrate);
+    }
+
     /**
      * Add ijk coordinates.
      *
@@ -120,9 +129,9 @@ public Vec2d ijkToHex2d() {
      */
 
     public void ijkAdd(int i, int j, int k) {
-        this.i += i;
-        this.j += j;
-        this.k += k;
+        this.i = Math.addExact(this.i, i);
+        this.j = Math.addExact(this.j, j);
+        this.k = Math.addExact(this.k, k);
     }
 
     /**
@@ -133,9 +142,9 @@ public void ijkAdd(int i, int j, int k) {
      * @param k the k coordinate
      */
     public void ijkSub(int i, int j, int k) {
-        this.i -= i;
-        this.j -= j;
-        this.k -= k;
+        this.i = Math.subtractExact(this.i, i);
+        this.j = Math.subtractExact(this.j, j);
+        this.k = Math.subtractExact(this.k, k);
     }
 
     /**
@@ -144,9 +153,7 @@ public void ijkSub(int i, int j, int k) {
      */
     public void ijkNormalize() {
         final int min = Math.min(i, Math.min(j, k));
-        i -= min;
-        j -= min;
-        k -= min;
+        ijkSub(min, min, min);
     }
 
     /**
@@ -158,9 +165,9 @@ public void downAp7() {
         // iVec (3, 0, 1)
         // jVec (1, 3, 0)
         // kVec (0, 1, 3)
-        final int i = this.i * 3 + this.j * 1 + this.k * 0;
-        final int j = this.i * 0 + this.j * 3 + this.k * 1;
-        final int k = this.i * 1 + this.j * 0 + this.k * 3;
+        final int i = Math.addExact(Math.multiplyExact(this.i, 3), this.j);
+        final int j = Math.addExact(Math.multiplyExact(this.j, 3), this.k);
+        final int k = Math.addExact(Math.multiplyExact(this.k, 3), this.i);
         this.i = i;
         this.j = j;
         this.k = k;
@@ -175,9 +182,9 @@ public void downAp7r() {
         // iVec (3, 1, 0)
         // jVec (0, 3, 1)
         // kVec (1, 0, 3)
-        final int i = this.i * 3 + this.j * 0 + this.k * 1;
-        final int j = this.i * 1 + this.j * 3 + this.k * 0;
-        final int k = this.i * 0 + this.j * 1 + this.k * 3;
+        final int i = Math.addExact(Math.multiplyExact(this.i, 3), this.k);
+        final int j = Math.addExact(Math.multiplyExact(this.j, 3), this.i);
+        final int k = Math.addExact(Math.multiplyExact(this.k, 3), this.j);
         this.i = i;
         this.j = j;
         this.k = k;
@@ -193,9 +200,9 @@ public void downAp3() {
         // iVec (2, 0, 1)
         // jVec (1, 2, 0)
         // kVec (0, 1, 2)
-        final int i = this.i * 2 + this.j * 1 + this.k * 0;
-        final int j = this.i * 0 + this.j * 2 + this.k * 1;
-        final int k = this.i * 1 + this.j * 0 + this.k * 2;
+        final int i = Math.addExact(Math.multiplyExact(this.i, 2), this.j);
+        final int j = Math.addExact(Math.multiplyExact(this.j, 2), this.k);
+        final int k = Math.addExact(Math.multiplyExact(this.k, 2), this.i);
         this.i = i;
         this.j = j;
         this.k = k;
@@ -211,9 +218,9 @@ public void downAp3r() {
         // iVec (2, 1, 0)
         // jVec (0, 2, 1)
         // kVec (1, 0, 2)
-        final int i = this.i * 2 + this.j * 0 + this.k * 1;
-        final int j = this.i * 1 + this.j * 2 + this.k * 0;
-        final int k = this.i * 0 + this.j * 1 + this.k * 2;
+        final int i = Math.addExact(Math.multiplyExact(this.i, 2), this.k);
+        final int j = Math.addExact(Math.multiplyExact(this.j, 2), this.i);
+        final int k = Math.addExact(Math.multiplyExact(this.k, 2), this.j);
         this.i = i;
         this.j = j;
         this.k = k;
@@ -229,9 +236,9 @@ public void ijkRotate60cw() {
         // iVec (1, 0, 1)
         // jVec (1, 1, 0)
         // kVec (0, 1, 1)
-        final int i = this.i * 1 + this.j * 1 + this.k * 0;
-        final int j = this.i * 0 + this.j * 1 + this.k * 1;
-        final int k = this.i * 1 + this.j * 0 + this.k * 1;
+        final int i = Math.addExact(this.i, this.j);
+        final int j = Math.addExact(this.j, this.k);
+        final int k = Math.addExact(this.i, this.k);
         this.i = i;
         this.j = j;
         this.k = k;
@@ -246,9 +253,9 @@ public void ijkRotate60ccw() {
         // iVec (1, 1, 0)
         // jVec (0, 1, 1)
         // kVec (1, 0, 1)
-        final int i = this.i * 1 + this.j * 0 + this.k * 1;
-        final int j = this.i * 1 + this.j * 1 + this.k * 0;
-        final int k = this.i * 0 + this.j * 1 + this.k * 1;
+        final int i = Math.addExact(this.i, this.k);
+        final int j = Math.addExact(this.i, this.j);
+        final int k = Math.addExact(this.j, this.k);
         this.i = i;
         this.j = j;
         this.k = k;
@@ -272,12 +279,10 @@ public void neighbor(int digit) {
      * clockwise aperture 7 grid.
      */
     public void upAp7r() {
-        i = this.i - this.k;
-        j = this.j - this.k;
-        int i = (int) Math.round((2 * this.i + this.j) / 7.0);
-        int j = (int) Math.round((3 * this.j - this.i) / 7.0);
-        this.i = i;
-        this.j = j;
+        final int i = Math.subtractExact(this.i, this.k);
+        final int j = Math.subtractExact(this.j, this.k);
+        this.i = (int) Math.round((Math.addExact(Math.multiplyExact(2, i), j)) / 7.0);
+        this.j = (int) Math.round((Math.subtractExact(Math.multiplyExact(3, j), i)) / 7.0);
         this.k = 0;
         ijkNormalize();
     }
@@ -288,12 +293,10 @@ public void upAp7r() {
      *
      */
     public void upAp7() {
-        i = this.i - this.k;
-        j = this.j - this.k;
-        int i = (int) Math.round((3 * this.i - this.j) / 7.0);
-        int j = (int) Math.round((this.i + 2 * this.j) / 7.0);
-        this.i = i;
-        this.j = j;
+        final int i = Math.subtractExact(this.i, this.k);
+        final int j = Math.subtractExact(this.j, this.k);
+        this.i = (int) Math.round((Math.subtractExact(Math.multiplyExact(3, i), j)) / 7.0);
+        this.j = (int) Math.round((Math.addExact(Math.multiplyExact(2, j), i)) / 7.0);
         this.k = 0;
         ijkNormalize();
     }
diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java b/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java
index 24eddd6cf365..df2ab26ca068 100644
--- a/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java
+++ b/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java
@@ -421,8 +421,7 @@ public Overage adjustOverageClassII(int res, boolean pentLeading4, boolean subst
      * @param res The H3 resolution of the cell.
      */
     public LatLng faceIjkToGeo(int res) {
-        Vec2d v = coord.ijkToHex2d();
-        return v.hex2dToGeo(face, res, false);
+        return coord.ijkToGeo(face, res, false);
     }
 
     /**
@@ -487,7 +486,11 @@ public CellBoundary faceIjkPentToCellBoundary(int res, int start, int length) {
                 }
 
                 final int unitScale = unitScaleByCIIres[adjRes] * 3;
-                lastCoord.ijkAdd(fijkOrient.translateI * unitScale, fijkOrient.translateJ * unitScale, fijkOrient.translateK * unitScale);
+                lastCoord.ijkAdd(
+                    Math.multiplyExact(fijkOrient.translateI, unitScale),
+                    Math.multiplyExact(fijkOrient.translateJ, unitScale),
+                    Math.multiplyExact(fijkOrient.translateK, unitScale)
+                );
                 lastCoord.ijkNormalize();
 
                 final Vec2d orig2d1 = lastCoord.ijkToHex2d();
@@ -522,8 +525,7 @@ public CellBoundary faceIjkPentToCellBoundary(int res, int start, int length) {
             // vert == start + NUM_PENT_VERTS is only used to test for possible
             // intersection on last edge
             if (vert < start + Constants.NUM_PENT_VERTS) {
-                final Vec2d vec = fijk.coord.ijkToHex2d();
-                final LatLng point = vec.hex2dToGeo(fijk.face, adjRes, true);
+                final LatLng point = fijk.coord.ijkToGeo(fijk.face, adjRes, true);
                 boundary.add(point);
             }
             lastFace = fijk.face;
@@ -594,10 +596,18 @@ public CellBoundary faceIjkToCellBoundary(final int res, final int start, final
                 // to each vertex to translate the vertices to that cell.
                 final int[] vertexLast = verts[lastV];
                 final int[] vertexV = verts[v];
-                scratch2.reset(vertexLast[0] + coord.i, vertexLast[1] + coord.j, vertexLast[2] + coord.k);
+                scratch2.reset(
+                    Math.addExact(vertexLast[0], this.coord.i),
+                    Math.addExact(vertexLast[1], this.coord.j),
+                    Math.addExact(vertexLast[2], this.coord.k)
+                );
                 scratch2.ijkNormalize();
                 final Vec2d orig2d0 = scratch2.ijkToHex2d();
-                scratch2.reset(vertexV[0] + coord.i, vertexV[1] + coord.j, vertexV[2] + coord.k);
+                scratch2.reset(
+                    Math.addExact(vertexV[0], this.coord.i),
+                    Math.addExact(vertexV[1], this.coord.j),
+                    Math.addExact(vertexV[2], this.coord.k)
+                );
                 scratch2.ijkNormalize();
                 final Vec2d orig2d1 = scratch2.ijkToHex2d();
 
@@ -628,7 +638,7 @@ public CellBoundary faceIjkToCellBoundary(final int res, final int start, final
                 adjacent hexagon edge will lie completely on a single icosahedron
                 face, and no additional vertex is required.
                 */
-                final boolean isIntersectionAtVertex = orig2d0.equals(inter) || orig2d1.equals(inter);
+                final boolean isIntersectionAtVertex = orig2d0.numericallyIdentical(inter) || orig2d1.numericallyIdentical(inter);
                 if (isIntersectionAtVertex == false) {
                     final LatLng point = inter.hex2dToGeo(this.face, adjRes, true);
                     boundary.add(point);
@@ -639,8 +649,7 @@ public CellBoundary faceIjkToCellBoundary(final int res, final int start, final
             // vert == start + NUM_HEX_VERTS is only used to test for possible
             // intersection on last edge
             if (vert < start + Constants.NUM_HEX_VERTS) {
-                final Vec2d vec = fijk.coord.ijkToHex2d();
-                final LatLng point = vec.hex2dToGeo(fijk.face, adjRes, true);
+                final LatLng point = fijk.coord.ijkToGeo(fijk.face, adjRes, true);
                 boundary.add(point);
             }
             lastFace = fijk.face;
@@ -695,7 +704,7 @@ static long faceIjkToH3(int res, int face, CoordIJK coord) {
                 scratch.reset(coord.i, coord.j, coord.k);
                 scratch.downAp7r();
             }
-            scratch.reset(lastI - scratch.i, lastJ - scratch.j, lastK - scratch.k);
+            scratch.reset(Math.subtractExact(lastI, scratch.i), Math.subtractExact(lastJ, scratch.j), Math.subtractExact(lastK, scratch.k));
             scratch.ijkNormalize();
             h = H3Index.H3_set_index_digit(h, r, scratch.unitIjkToDigit());
         }
diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/FastMath.java b/libs/h3/src/main/java/org/elasticsearch/h3/FastMath.java
new file mode 100644
index 000000000000..61d767901ae0
--- /dev/null
+++ b/libs/h3/src/main/java/org/elasticsearch/h3/FastMath.java
@@ -0,0 +1,1055 @@
+/*
+ * @notice
+ * Copyright 2012 Jeff Hain
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * =============================================================================
+ * Notice of fdlibm package this program is partially derived from:
+ *
+ * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.
+ *
+ * Developed at SunSoft, a Sun Microsystems, Inc. business.
+ * Permission to use, copy, modify, and distribute this
+ * software is freely granted, provided that this notice
+ * is preserved.
+ * =============================================================================
+ *
+ * This code sourced from:
+ * https://github.com/jeffhain/jafama/blob/d7d2a7659e96e148d827acc24cf385b872cda365/src/main/java/net/jafama/FastMath.java
+ */
+
+package org.elasticsearch.h3;
+
+/**
+ * This file is forked from https://github.com/jeffhain/jafama. In particular, it forks the following file:
+ * https://github.com/jeffhain/jafama/blob/master/src/main/java/net/jafama/FastMath.java
+ *
+ * It modifies the original implementation by removing not needed methods leaving the following trigonometric function:
+ * <ul>
+ * <li>{@link #cos(double)}</li>
+ * <li>{@link #sin(double)}</li>
+ * <li>{@link #tan(double)}</li>
+ * <li>{@link #acos(double)}</li>
+ * <li>{@link #asin(double)}</li>
+ * <li>{@link #atan(double)}</li>
+ * <li>{@link #atan2(double, double)}</li>
+ * </ul>
+ */
+final class FastMath {
+
+    /*
+     * For trigonometric functions, use of look-up tables and Taylor-Lagrange formula
+     * with 4 derivatives (more take longer to compute and don't add much accuracy,
+     * less require larger tables (which use more memory, take more time to initialize,
+     * and are slower to access (at least on the machine they were developed on))).
+     *
+     * For angles reduction of cos/sin/tan functions:
+     * - for small values, instead of reducing angles, and then computing the best index
+     *   for look-up tables, we compute this index right away, and use it for reduction,
+     * - for large values, treatments derived from fdlibm package are used, as done in
+     *   java.lang.Math. They are faster but still "slow", so if you work with
+     *   large numbers and need speed over accuracy for them, you might want to use
+     *   normalizeXXXFast treatments before your function, or modify cos/sin/tan
+     *   so that they call the fast normalization treatments instead of the accurate ones.
+     *   NB: If an angle is huge (like PI*1e20), in double precision format its last digits
+     *       are zeros, which most likely is not the case for the intended value, and doing
+     *       an accurate reduction on a very inaccurate value is most likely pointless.
+     *       But it gives some sort of coherence that could be needed in some cases.
+     *
+     * Multiplication on double appears to be about as fast (or not much slower) than call
+     * to <double_array>[<index>], and regrouping some doubles in a private class, to use
+     * index only once, does not seem to speed things up, so:
+     * - for uniformly tabulated values, to retrieve the parameter corresponding to
+     *   an index, we recompute it rather than using an array to store it,
+     * - for cos/sin, we recompute derivatives divided by (multiplied by inverse of)
+     *   factorial each time, rather than storing them in arrays.
+     *
+     * Lengths of look-up tables are usually of the form 2^n+1, for their values to be
+     * of the form (<a_constant> * k/2^n, k in 0 .. 2^n), so that particular values
+     * (PI/2, etc.) are "exactly" computed, as well as for other reasons.
+     *
+     * Most math treatments I could find on the web, including "fast" ones,
+     * usually take care of special cases (NaN, etc.) at the beginning, and
+     * then deal with the general case, which adds a useless overhead for the
+     * general (and common) case. In this class, special cases are only dealt
+     * with when needed, and if the general case does not already handle them.
+     */
+
+    // --------------------------------------------------------------------------
+    // GENERAL CONSTANTS
+    // --------------------------------------------------------------------------
+
+    private static final double ONE_DIV_F2 = 1 / 2.0;
+    private static final double ONE_DIV_F3 = 1 / 6.0;
+    private static final double ONE_DIV_F4 = 1 / 24.0;
+
+    private static final double TWO_POW_24 = Double.longBitsToDouble(0x4170000000000000L);
+    private static final double TWO_POW_N24 = Double.longBitsToDouble(0x3E70000000000000L);
+
+    private static final double TWO_POW_66 = Double.longBitsToDouble(0x4410000000000000L);
+
+    private static final int MIN_DOUBLE_EXPONENT = -1074;
+    private static final int MAX_DOUBLE_EXPONENT = 1023;
+
+    // --------------------------------------------------------------------------
+    // CONSTANTS FOR NORMALIZATIONS
+    // --------------------------------------------------------------------------
+
+    /*
+     * Table of constants for 1/(2*PI), 282 Hex digits (enough for normalizing doubles).
+     * 1/(2*PI) approximation = sum of ONE_OVER_TWOPI_TAB[i]*2^(-24*(i+1)).
+     */
+    private static final double[] ONE_OVER_TWOPI_TAB = {
+        0x28BE60,
+        0xDB9391,
+        0x054A7F,
+        0x09D5F4,
+        0x7D4D37,
+        0x7036D8,
+        0xA5664F,
+        0x10E410,
+        0x7F9458,
+        0xEAF7AE,
+        0xF1586D,
+        0xC91B8E,
+        0x909374,
+        0xB80192,
+        0x4BBA82,
+        0x746487,
+        0x3F877A,
+        0xC72C4A,
+        0x69CFBA,
+        0x208D7D,
+        0x4BAED1,
+        0x213A67,
+        0x1C09AD,
+        0x17DF90,
+        0x4E6475,
+        0x8E60D4,
+        0xCE7D27,
+        0x2117E2,
+        0xEF7E4A,
+        0x0EC7FE,
+        0x25FFF7,
+        0x816603,
+        0xFBCBC4,
+        0x62D682,
+        0x9B47DB,
+        0x4D9FB3,
+        0xC9F2C2,
+        0x6DD3D1,
+        0x8FD9A7,
+        0x97FA8B,
+        0x5D49EE,
+        0xB1FAF9,
+        0x7C5ECF,
+        0x41CE7D,
+        0xE294A4,
+        0xBA9AFE,
+        0xD7EC47 };
+
+    /*
+     * Constants for 2*PI. Only the 23 most significant bits of each mantissa are used.
+     * 2*PI approximation = sum of TWOPI_TAB<i>.
+     */
+    private static final double TWOPI_TAB0 = Double.longBitsToDouble(0x401921FB40000000L);
+    private static final double TWOPI_TAB1 = Double.longBitsToDouble(0x3E94442D00000000L);
+    private static final double TWOPI_TAB2 = Double.longBitsToDouble(0x3D18469880000000L);
+    private static final double TWOPI_TAB3 = Double.longBitsToDouble(0x3B98CC5160000000L);
+    private static final double TWOPI_TAB4 = Double.longBitsToDouble(0x3A101B8380000000L);
+
+    private static final double INVPIO2 = Double.longBitsToDouble(0x3FE45F306DC9C883L); // 6.36619772367581382433e-01 53 bits of 2/pi
+    private static final double PIO2_HI = Double.longBitsToDouble(0x3FF921FB54400000L); // 1.57079632673412561417e+00 first 33 bits of pi/2
+    private static final double PIO2_LO = Double.longBitsToDouble(0x3DD0B4611A626331L); // 6.07710050650619224932e-11 pi/2 - PIO2_HI
+    private static final double INVTWOPI = INVPIO2 / 4;
+    private static final double TWOPI_HI = 4 * PIO2_HI;
+    private static final double TWOPI_LO = 4 * PIO2_LO;
+
+    // fdlibm uses 2^19*PI/2 here, but we normalize with % 2*PI instead of % PI/2,
+    // and we can bear some more error.
+    private static final double NORMALIZE_ANGLE_MAX_MEDIUM_DOUBLE = StrictMath.pow(2, 20) * (2 * Math.PI);
+
+    // --------------------------------------------------------------------------
+    // CONSTANTS AND TABLES FOR COS, SIN
+    // --------------------------------------------------------------------------
+
+    private static final int SIN_COS_TABS_SIZE = (1 << getTabSizePower(11)) + 1;
+    private static final double SIN_COS_DELTA_HI = TWOPI_HI / (SIN_COS_TABS_SIZE - 1);
+    private static final double SIN_COS_DELTA_LO = TWOPI_LO / (SIN_COS_TABS_SIZE - 1);
+    private static final double SIN_COS_INDEXER = 1 / (SIN_COS_DELTA_HI + SIN_COS_DELTA_LO);
+    private static final double[] sinTab = new double[SIN_COS_TABS_SIZE];
+    private static final double[] cosTab = new double[SIN_COS_TABS_SIZE];
+
+    // Max abs value for fast modulo, above which we use regular angle normalization.
+    // This value must be < (Integer.MAX_VALUE / SIN_COS_INDEXER), to stay in range of int type.
+    // The higher it is, the higher the error, but also the faster it is for lower values.
+    // If you set it to ((Integer.MAX_VALUE / SIN_COS_INDEXER) * 0.99), worse accuracy on double range is about 1e-10.
+    private static final double SIN_COS_MAX_VALUE_FOR_INT_MODULO = ((Integer.MAX_VALUE >> 9) / SIN_COS_INDEXER) * 0.99;
+
+    // --------------------------------------------------------------------------
+    // CONSTANTS AND TABLES FOR TAN
+    // --------------------------------------------------------------------------
+
+    // We use the following formula:
+    // 1) tan(-x) = -tan(x)
+    // 2) tan(x) = 1/tan(PI/2-x)
+    // ---> we only have to compute tan(x) on [0,A] with PI/4<=A<PI/2.
+
+    // We use indexing past look-up tables, so that indexing information
+    // allows for fast recomputation of angle in [0,PI/2] range.
+    private static final int TAN_VIRTUAL_TABS_SIZE = (1 << getTabSizePower(12)) + 1;
+
+    // Must be >= 45deg, and supposed to be >= 51.4deg, as fdlibm code is not
+    // supposed to work with values inferior to that (51.4deg is about
+    // (PI/2-Double.longBitsToDouble(0x3FE5942800000000L))).
+    private static final double TAN_MAX_VALUE_FOR_TABS = Math.toRadians(77.0);
+
+    private static final int TAN_TABS_SIZE = (int) ((TAN_MAX_VALUE_FOR_TABS / (Math.PI / 2)) * (TAN_VIRTUAL_TABS_SIZE - 1)) + 1;
+    private static final double TAN_DELTA_HI = PIO2_HI / (TAN_VIRTUAL_TABS_SIZE - 1);
+    private static final double TAN_DELTA_LO = PIO2_LO / (TAN_VIRTUAL_TABS_SIZE - 1);
+    private static final double TAN_INDEXER = 1 / (TAN_DELTA_HI + TAN_DELTA_LO);
+    private static final double[] tanTab = new double[TAN_TABS_SIZE];
+    private static final double[] tanDer1DivF1Tab = new double[TAN_TABS_SIZE];
+    private static final double[] tanDer2DivF2Tab = new double[TAN_TABS_SIZE];
+    private static final double[] tanDer3DivF3Tab = new double[TAN_TABS_SIZE];
+    private static final double[] tanDer4DivF4Tab = new double[TAN_TABS_SIZE];
+
+    // Max abs value for fast modulo, above which we use regular angle normalization.
+    // This value must be < (Integer.MAX_VALUE / TAN_INDEXER), to stay in range of int type.
+    // The higher it is, the higher the error, but also the faster it is for lower values.
+    private static final double TAN_MAX_VALUE_FOR_INT_MODULO = (((Integer.MAX_VALUE >> 9) / TAN_INDEXER) * 0.99);
+
+    // --------------------------------------------------------------------------
+    // CONSTANTS AND TABLES FOR ACOS, ASIN
+    // --------------------------------------------------------------------------
+
+    // We use the following formula:
+    // 1) acos(x) = PI/2 - asin(x)
+    // 2) asin(-x) = -asin(x)
+    // ---> we only have to compute asin(x) on [0,1].
+    // For values not close to +-1, we use look-up tables;
+    // for values near +-1, we use code derived from fdlibm.
+
+    // Supposed to be >= sin(77.2deg), as fdlibm code is supposed to work with values > 0.975,
+    // but seems to work well enough as long as value >= sin(25deg).
+    private static final double ASIN_MAX_VALUE_FOR_TABS = StrictMath.sin(Math.toRadians(73.0));
+
+    private static final int ASIN_TABS_SIZE = (1 << getTabSizePower(13)) + 1;
+    private static final double ASIN_DELTA = ASIN_MAX_VALUE_FOR_TABS / (ASIN_TABS_SIZE - 1);
+    private static final double ASIN_INDEXER = 1 / ASIN_DELTA;
+    private static final double[] asinTab = new double[ASIN_TABS_SIZE];
+    private static final double[] asinDer1DivF1Tab = new double[ASIN_TABS_SIZE];
+    private static final double[] asinDer2DivF2Tab = new double[ASIN_TABS_SIZE];
+    private static final double[] asinDer3DivF3Tab = new double[ASIN_TABS_SIZE];
+    private static final double[] asinDer4DivF4Tab = new double[ASIN_TABS_SIZE];
+
+    private static final double ASIN_MAX_VALUE_FOR_POWTABS = StrictMath.sin(Math.toRadians(88.6));
+    private static final int ASIN_POWTABS_POWER = 84;
+
+    private static final double ASIN_POWTABS_ONE_DIV_MAX_VALUE = 1 / ASIN_MAX_VALUE_FOR_POWTABS;
+    private static final int ASIN_POWTABS_SIZE = (1 << getTabSizePower(12)) + 1;
+    private static final int ASIN_POWTABS_SIZE_MINUS_ONE = ASIN_POWTABS_SIZE - 1;
+    private static final double[] asinParamPowTab = new double[ASIN_POWTABS_SIZE];
+    private static final double[] asinPowTab = new double[ASIN_POWTABS_SIZE];
+    private static final double[] asinDer1DivF1PowTab = new double[ASIN_POWTABS_SIZE];
+    private static final double[] asinDer2DivF2PowTab = new double[ASIN_POWTABS_SIZE];
+    private static final double[] asinDer3DivF3PowTab = new double[ASIN_POWTABS_SIZE];
+    private static final double[] asinDer4DivF4PowTab = new double[ASIN_POWTABS_SIZE];
+
+    private static final double ASIN_PIO2_HI = Double.longBitsToDouble(0x3FF921FB54442D18L); // 1.57079632679489655800e+00
+    private static final double ASIN_PIO2_LO = Double.longBitsToDouble(0x3C91A62633145C07L); // 6.12323399573676603587e-17
+    private static final double ASIN_PS0 = Double.longBitsToDouble(0x3fc5555555555555L); // 1.66666666666666657415e-01
+    private static final double ASIN_PS1 = Double.longBitsToDouble(0xbfd4d61203eb6f7dL); // -3.25565818622400915405e-01
+    private static final double ASIN_PS2 = Double.longBitsToDouble(0x3fc9c1550e884455L); // 2.01212532134862925881e-01
+    private static final double ASIN_PS3 = Double.longBitsToDouble(0xbfa48228b5688f3bL); // -4.00555345006794114027e-02
+    private static final double ASIN_PS4 = Double.longBitsToDouble(0x3f49efe07501b288L); // 7.91534994289814532176e-04
+    private static final double ASIN_PS5 = Double.longBitsToDouble(0x3f023de10dfdf709L); // 3.47933107596021167570e-05
+    private static final double ASIN_QS1 = Double.longBitsToDouble(0xc0033a271c8a2d4bL); // -2.40339491173441421878e+00
+    private static final double ASIN_QS2 = Double.longBitsToDouble(0x40002ae59c598ac8L); // 2.02094576023350569471e+00
+    private static final double ASIN_QS3 = Double.longBitsToDouble(0xbfe6066c1b8d0159L); // -6.88283971605453293030e-01
+    private static final double ASIN_QS4 = Double.longBitsToDouble(0x3fb3b8c5b12e9282L); // 7.70381505559019352791e-02
+
+    // --------------------------------------------------------------------------
+    // CONSTANTS AND TABLES FOR ATAN
+    // --------------------------------------------------------------------------
+
+    // We use the formula atan(-x) = -atan(x)
+    // ---> we only have to compute atan(x) on [0,+infinity[.
+    // For values corresponding to angles not close to +-PI/2, we use look-up tables;
+    // for values corresponding to angles near +-PI/2, we use code derived from fdlibm.
+
+    // Supposed to be >= tan(67.7deg), as fdlibm code is supposed to work with values > 2.4375.
+    private static final double ATAN_MAX_VALUE_FOR_TABS = StrictMath.tan(Math.toRadians(74.0));
+
+    private static final int ATAN_TABS_SIZE = (1 << getTabSizePower(12)) + 1;
+    private static final double ATAN_DELTA = ATAN_MAX_VALUE_FOR_TABS / (ATAN_TABS_SIZE - 1);
+    private static final double ATAN_INDEXER = 1 / ATAN_DELTA;
+    private static final double[] atanTab = new double[ATAN_TABS_SIZE];
+    private static final double[] atanDer1DivF1Tab = new double[ATAN_TABS_SIZE];
+    private static final double[] atanDer2DivF2Tab = new double[ATAN_TABS_SIZE];
+    private static final double[] atanDer3DivF3Tab = new double[ATAN_TABS_SIZE];
+    private static final double[] atanDer4DivF4Tab = new double[ATAN_TABS_SIZE];
+
+    private static final double ATAN_HI3 = Double.longBitsToDouble(0x3ff921fb54442d18L); // 1.57079632679489655800e+00 atan(inf)hi
+    private static final double ATAN_LO3 = Double.longBitsToDouble(0x3c91a62633145c07L); // 6.12323399573676603587e-17 atan(inf)lo
+    private static final double ATAN_AT0 = Double.longBitsToDouble(0x3fd555555555550dL); // 3.33333333333329318027e-01
+    private static final double ATAN_AT1 = Double.longBitsToDouble(0xbfc999999998ebc4L); // -1.99999999998764832476e-01
+    private static final double ATAN_AT2 = Double.longBitsToDouble(0x3fc24924920083ffL); // 1.42857142725034663711e-01
+    private static final double ATAN_AT3 = Double.longBitsToDouble(0xbfbc71c6fe231671L); // -1.11111104054623557880e-01
+    private static final double ATAN_AT4 = Double.longBitsToDouble(0x3fb745cdc54c206eL); // 9.09088713343650656196e-02
+    private static final double ATAN_AT5 = Double.longBitsToDouble(0xbfb3b0f2af749a6dL); // -7.69187620504482999495e-02
+    private static final double ATAN_AT6 = Double.longBitsToDouble(0x3fb10d66a0d03d51L); // 6.66107313738753120669e-02
+    private static final double ATAN_AT7 = Double.longBitsToDouble(0xbfadde2d52defd9aL); // -5.83357013379057348645e-02
+    private static final double ATAN_AT8 = Double.longBitsToDouble(0x3fa97b4b24760debL); // 4.97687799461593236017e-02
+    private static final double ATAN_AT9 = Double.longBitsToDouble(0xbfa2b4442c6a6c2fL); // -3.65315727442169155270e-02
+    private static final double ATAN_AT10 = Double.longBitsToDouble(0x3f90ad3ae322da11L); // 1.62858201153657823623e-02
+
+    // --------------------------------------------------------------------------
+    // TABLE FOR POWERS OF TWO
+    // --------------------------------------------------------------------------
+
+    private static final double[] twoPowTab = new double[(MAX_DOUBLE_EXPONENT - MIN_DOUBLE_EXPONENT) + 1];
+
+    // --------------------------------------------------------------------------
+    // PUBLIC TREATMENTS
+    // --------------------------------------------------------------------------
+
+    /**
+     * @param angle Angle in radians.
+     * @return Angle cosine.
+     */
+    public static double cos(double angle) {
+        angle = Math.abs(angle);
+        if (angle > SIN_COS_MAX_VALUE_FOR_INT_MODULO) {
+            // Faster than using normalizeZeroTwoPi.
+            angle = remainderTwoPi(angle);
+            if (angle < 0.0) {
+                angle += 2 * Math.PI;
+            }
+        }
+        // index: possibly outside tables range.
+        int index = (int) (angle * SIN_COS_INDEXER + 0.5);
+        double delta = (angle - index * SIN_COS_DELTA_HI) - index * SIN_COS_DELTA_LO;
+        // Making sure index is within tables range.
+        // Last value of each table is the same than first, so we ignore it (tabs size minus one) for modulo.
+        index &= (SIN_COS_TABS_SIZE - 2); // index % (SIN_COS_TABS_SIZE-1)
+        double indexCos = cosTab[index];
+        double indexSin = sinTab[index];
+        return indexCos + delta * (-indexSin + delta * (-indexCos * ONE_DIV_F2 + delta * (indexSin * ONE_DIV_F3 + delta * indexCos
+            * ONE_DIV_F4)));
+    }
+
+    /**
+     * @param angle Angle in radians.
+     * @return Angle sine.
+     */
+    public static double sin(double angle) {
+        boolean negateResult;
+        if (angle < 0.0) {
+            angle = -angle;
+            negateResult = true;
+        } else {
+            negateResult = false;
+        }
+        if (angle > SIN_COS_MAX_VALUE_FOR_INT_MODULO) {
+            // Faster than using normalizeZeroTwoPi.
+            angle = remainderTwoPi(angle);
+            if (angle < 0.0) {
+                angle += 2 * Math.PI;
+            }
+        }
+        int index = (int) (angle * SIN_COS_INDEXER + 0.5);
+        double delta = (angle - index * SIN_COS_DELTA_HI) - index * SIN_COS_DELTA_LO;
+        index &= (SIN_COS_TABS_SIZE - 2); // index % (SIN_COS_TABS_SIZE-1)
+        double indexSin = sinTab[index];
+        double indexCos = cosTab[index];
+        double result = indexSin + delta * (indexCos + delta * (-indexSin * ONE_DIV_F2 + delta * (-indexCos * ONE_DIV_F3 + delta * indexSin
+            * ONE_DIV_F4)));
+        return negateResult ? -result : result;
+    }
+
+    /**
+     * @param angle Angle in radians.
+     * @return Angle tangent.
+     */
+    public static double tan(double angle) {
+        if (Math.abs(angle) > TAN_MAX_VALUE_FOR_INT_MODULO) {
+            // Faster than using normalizeMinusHalfPiHalfPi.
+            angle = remainderTwoPi(angle);
+            if (angle < -Math.PI / 2) {
+                angle += Math.PI;
+            } else if (angle > Math.PI / 2) {
+                angle -= Math.PI;
+            }
+        }
+        boolean negateResult;
+        if (angle < 0.0) {
+            angle = -angle;
+            negateResult = true;
+        } else {
+            negateResult = false;
+        }
+        int index = (int) (angle * TAN_INDEXER + 0.5);
+        double delta = (angle - index * TAN_DELTA_HI) - index * TAN_DELTA_LO;
+        // index modulo PI, i.e. 2*(virtual tab size minus one).
+        index &= (2 * (TAN_VIRTUAL_TABS_SIZE - 1) - 1); // index % (2*(TAN_VIRTUAL_TABS_SIZE-1))
+        // Here, index is in [0,2*(TAN_VIRTUAL_TABS_SIZE-1)-1], i.e. indicates an angle in [0,PI[.
+        if (index > (TAN_VIRTUAL_TABS_SIZE - 1)) {
+            index = (2 * (TAN_VIRTUAL_TABS_SIZE - 1)) - index;
+            delta = -delta;
+            negateResult = negateResult == false;
+        }
+        double result;
+        if (index < TAN_TABS_SIZE) {
+            result = tanTab[index] + delta * (tanDer1DivF1Tab[index] + delta * (tanDer2DivF2Tab[index] + delta * (tanDer3DivF3Tab[index]
+                + delta * tanDer4DivF4Tab[index])));
+        } else { // angle in ]TAN_MAX_VALUE_FOR_TABS,TAN_MAX_VALUE_FOR_INT_MODULO], or angle is NaN
+            // Using tan(angle) == 1/tan(PI/2-angle) formula: changing angle (index and delta), and inverting.
+            index = (TAN_VIRTUAL_TABS_SIZE - 1) - index;
+            result = 1 / (tanTab[index] - delta * (tanDer1DivF1Tab[index] - delta * (tanDer2DivF2Tab[index] - delta
+                * (tanDer3DivF3Tab[index] - delta * tanDer4DivF4Tab[index]))));
+        }
+        return negateResult ? -result : result;
+    }
+
+    /**
+     * @param value Value in [-1,1].
+     * @return Value arccosine, in radians, in [0,PI].
+     */
+    public static double acos(double value) {
+        return Math.PI / 2 - FastMath.asin(value);
+    }
+
+    /**
+     * @param value Value in [-1,1].
+     * @return Value arcsine, in radians, in [-PI/2,PI/2].
+     */
+    public static double asin(double value) {
+        boolean negateResult;
+        if (value < 0.0) {
+            value = -value;
+            negateResult = true;
+        } else {
+            negateResult = false;
+        }
+        if (value <= ASIN_MAX_VALUE_FOR_TABS) {
+            int index = (int) (value * ASIN_INDEXER + 0.5);
+            double delta = value - index * ASIN_DELTA;
+            double result = asinTab[index] + delta * (asinDer1DivF1Tab[index] + delta * (asinDer2DivF2Tab[index] + delta
+                * (asinDer3DivF3Tab[index] + delta * asinDer4DivF4Tab[index])));
+            return negateResult ? -result : result;
+        } else if (value <= ASIN_MAX_VALUE_FOR_POWTABS) {
+            int index = (int) (FastMath.powFast(value * ASIN_POWTABS_ONE_DIV_MAX_VALUE, ASIN_POWTABS_POWER) * ASIN_POWTABS_SIZE_MINUS_ONE
+                + 0.5);
+            double delta = value - asinParamPowTab[index];
+            double result = asinPowTab[index] + delta * (asinDer1DivF1PowTab[index] + delta * (asinDer2DivF2PowTab[index] + delta
+                * (asinDer3DivF3PowTab[index] + delta * asinDer4DivF4PowTab[index])));
+            return negateResult ? -result : result;
+        } else { // value > ASIN_MAX_VALUE_FOR_TABS, or value is NaN
+            // This part is derived from fdlibm.
+            if (value < 1.0) {
+                double t = (1.0 - value) * 0.5;
+                double p = t * (ASIN_PS0 + t * (ASIN_PS1 + t * (ASIN_PS2 + t * (ASIN_PS3 + t * (ASIN_PS4 + t * ASIN_PS5)))));
+                double q = 1.0 + t * (ASIN_QS1 + t * (ASIN_QS2 + t * (ASIN_QS3 + t * ASIN_QS4)));
+                double s = Math.sqrt(t);
+                double z = s + s * (p / q);
+                double result = ASIN_PIO2_HI - ((z + z) - ASIN_PIO2_LO);
+                return negateResult ? -result : result;
+            } else { // value >= 1.0, or value is NaN
+                if (value == 1.0) {
+                    return negateResult ? -Math.PI / 2 : Math.PI / 2;
+                } else {
+                    return Double.NaN;
+                }
+            }
+        }
+    }
+
+    /**
+     * @param value A double value.
+     * @return Value arctangent, in radians, in [-PI/2,PI/2].
+     */
+    public static double atan(double value) {
+        boolean negateResult;
+        if (value < 0.0) {
+            value = -value;
+            negateResult = true;
+        } else {
+            negateResult = false;
+        }
+        if (value == 1.0) {
+            // We want "exact" result for 1.0.
+            return negateResult ? -Math.PI / 4 : Math.PI / 4;
+        } else if (value <= ATAN_MAX_VALUE_FOR_TABS) {
+            int index = (int) (value * ATAN_INDEXER + 0.5);
+            double delta = value - index * ATAN_DELTA;
+            double result = atanTab[index] + delta * (atanDer1DivF1Tab[index] + delta * (atanDer2DivF2Tab[index] + delta
+                * (atanDer3DivF3Tab[index] + delta * atanDer4DivF4Tab[index])));
+            return negateResult ? -result : result;
+        } else { // value > ATAN_MAX_VALUE_FOR_TABS, or value is NaN
+            // This part is derived from fdlibm.
+            if (value < TWO_POW_66) {
+                double x = -1 / value;
+                double x2 = x * x;
+                double x4 = x2 * x2;
+                double s1 = x2 * (ATAN_AT0 + x4 * (ATAN_AT2 + x4 * (ATAN_AT4 + x4 * (ATAN_AT6 + x4 * (ATAN_AT8 + x4 * ATAN_AT10)))));
+                double s2 = x4 * (ATAN_AT1 + x4 * (ATAN_AT3 + x4 * (ATAN_AT5 + x4 * (ATAN_AT7 + x4 * ATAN_AT9))));
+                double result = ATAN_HI3 - ((x * (s1 + s2) - ATAN_LO3) - x);
+                return negateResult ? -result : result;
+            } else { // value >= 2^66, or value is NaN
+                if (Double.isNaN(value)) {
+                    return Double.NaN;
+                } else {
+                    return negateResult ? -Math.PI / 2 : Math.PI / 2;
+                }
+            }
+        }
+    }
+
+    /**
+     * For special values for which multiple conventions could be adopted, behaves like Math.atan2(double,double).
+     *
+     * @param y Coordinate on y axis.
+     * @param x Coordinate on x axis.
+     * @return Angle from x axis positive side to (x,y) position, in radians, in [-PI,PI].
+     *         Angle measure is positive when going from x axis to y axis (positive sides).
+     */
+    public static double atan2(double y, double x) {
+        if (x > 0.0) {
+            if (y == 0.0) {
+                return (1 / y == Double.NEGATIVE_INFINITY) ? -0.0 : 0.0;
+            }
+            if (x == Double.POSITIVE_INFINITY) {
+                if (y == Double.POSITIVE_INFINITY) {
+                    return Math.PI / 4;
+                } else if (y == Double.NEGATIVE_INFINITY) {
+                    return -Math.PI / 4;
+                } else if (y > 0.0) {
+                    return 0.0;
+                } else if (y < 0.0) {
+                    return -0.0;
+                } else {
+                    return Double.NaN;
+                }
+            } else {
+                return FastMath.atan(y / x);
+            }
+        } else if (x < 0.0) {
+            if (y == 0.0) {
+                return (1 / y == Double.NEGATIVE_INFINITY) ? -Math.PI : Math.PI;
+            }
+            if (x == Double.NEGATIVE_INFINITY) {
+                if (y == Double.POSITIVE_INFINITY) {
+                    return 3 * Math.PI / 4;
+                } else if (y == Double.NEGATIVE_INFINITY) {
+                    return -3 * Math.PI / 4;
+                } else if (y > 0.0) {
+                    return Math.PI;
+                } else if (y < 0.0) {
+                    return -Math.PI;
+                } else {
+                    return Double.NaN;
+                }
+            } else if (y > 0.0) {
+                return Math.PI / 2 + FastMath.atan(-x / y);
+            } else if (y < 0.0) {
+                return -Math.PI / 2 - FastMath.atan(x / y);
+            } else {
+                return Double.NaN;
+            }
+        } else if (x == 0.0) {
+            if (y == 0.0) {
+                if (1 / x == Double.NEGATIVE_INFINITY) {
+                    return (1 / y == Double.NEGATIVE_INFINITY) ? -Math.PI : Math.PI;
+                } else {
+                    return (1 / y == Double.NEGATIVE_INFINITY) ? -0.0 : 0.0;
+                }
+            }
+            if (y > 0.0) {
+                return Math.PI / 2;
+            } else if (y < 0.0) {
+                return -Math.PI / 2;
+            } else {
+                return Double.NaN;
+            }
+        } else {
+            return Double.NaN;
+        }
+    }
+
+    /**
+     * This treatment is somehow accurate for low values of |power|,
+     * and for |power*getExponent(value)| &lt; 1023 or so (to stay away
+     * from double extreme magnitudes (large and small)).
+     *
+     * @param value A double value.
+     * @param power A power.
+     * @return value^power.
+     */
+    private static double powFast(double value, int power) {
+        if (power > 5) { // Most common case first.
+            double oddRemains = 1.0;
+            do {
+                // Test if power is odd.
+                if ((power & 1) != 0) {
+                    oddRemains *= value;
+                }
+                value *= value;
+                power >>= 1; // power = power / 2
+            } while (power > 5);
+            // Here, power is in [3,5]: faster to finish outside the loop.
+            if (power == 3) {
+                return oddRemains * value * value * value;
+            } else {
+                double v2 = value * value;
+                if (power == 4) {
+                    return oddRemains * v2 * v2;
+                } else { // power == 5
+                    return oddRemains * v2 * v2 * value;
+                }
+            }
+        } else if (power >= 0) { // power in [0,5]
+            if (power < 3) { // power in [0,2]
+                if (power == 2) { // Most common case first.
+                    return value * value;
+                } else if (power != 0) { // faster than == 1
+                    return value;
+                } else { // power == 0
+                    return 1.0;
+                }
+            } else { // power in [3,5]
+                if (power == 3) {
+                    return value * value * value;
+                } else { // power in [4,5]
+                    double v2 = value * value;
+                    if (power == 4) {
+                        return v2 * v2;
+                    } else { // power == 5
+                        return v2 * v2 * value;
+                    }
+                }
+            }
+        } else { // power < 0
+            // Opposite of Integer.MIN_VALUE does not exist as int.
+            if (power == Integer.MIN_VALUE) {
+                // Integer.MAX_VALUE = -(power+1)
+                return 1.0 / (FastMath.powFast(value, Integer.MAX_VALUE) * value);
+            } else {
+                return 1.0 / FastMath.powFast(value, -power);
+            }
+        }
+    }
+
+    // --------------------------------------------------------------------------
+    // PRIVATE TREATMENTS
+    // --------------------------------------------------------------------------
+
+    /**
+     * FastMath is non-instantiable.
+     */
+    private FastMath() {}
+
+    /**
+     * Use look-up tables size power through this method,
+     * to make sure is it small in case java.lang.Math
+     * is directly used.
+     */
+    private static int getTabSizePower(int tabSizePower) {
+        return tabSizePower;
+    }
+
+    /**
+     * Remainder using an accurate definition of PI.
+     * Derived from a fdlibm treatment called __ieee754_rem_pio2.
+     *
+     * This method can return values slightly (like one ULP or so) outside [-Math.PI,Math.PI] range.
+     *
+     * @param angle Angle in radians.
+     * @return Remainder of (angle % (2*PI)), which is in [-PI,PI] range.
+     */
+    private static double remainderTwoPi(double angle) {
+        boolean negateResult;
+        if (angle < 0.0) {
+            negateResult = true;
+            angle = -angle;
+        } else {
+            negateResult = false;
+        }
+        if (angle <= NORMALIZE_ANGLE_MAX_MEDIUM_DOUBLE) {
+            double fn = (double) (int) (angle * INVTWOPI + 0.5);
+            double result = (angle - fn * TWOPI_HI) - fn * TWOPI_LO;
+            return negateResult ? -result : result;
+        } else if (angle < Double.POSITIVE_INFINITY) {
+            // Reworking exponent to have a value < 2^24.
+            long lx = Double.doubleToRawLongBits(angle);
+            long exp = ((lx >> 52) & 0x7FF) - 1046;
+            double z = Double.longBitsToDouble(lx - (exp << 52));
+
+            double x0 = (double) ((int) z);
+            z = (z - x0) * TWO_POW_24;
+            double x1 = (double) ((int) z);
+            double x2 = (z - x1) * TWO_POW_24;
+
+            double result = subRemainderTwoPi(x0, x1, x2, (int) exp, (x2 == 0) ? 2 : 3);
+            return negateResult ? -result : result;
+        } else { // angle is +infinity or NaN
+            return Double.NaN;
+        }
+    }
+
+    /**
+     * Remainder using an accurate definition of PI.
+     * Derived from a fdlibm treatment called __kernel_rem_pio2.
+     *
+     * @param x0 Most significant part of the value, as an integer &lt; 2^24, in double precision format. Must be >= 0.
+     * @param x1 Following significant part of the value, as an integer &lt; 2^24, in double precision format.
+     * @param x2 Least significant part of the value, as an integer &lt; 2^24, in double precision format.
+     * @param e0 Exponent of x0 (value is (2^e0)*(x0+(2^-24)*(x1+(2^-24)*x2))). Must be &ge; -20.
+     * @param nx Number of significant parts to take into account. Must be 2 or 3.
+     * @return Remainder of (value % (2*PI)), which is in [-PI,PI] range.
+     */
+    private static double subRemainderTwoPi(double x0, double x1, double x2, int e0, int nx) {
+        int ih;
+        double z, fw;
+        double f0, f1, f2, f3, f4, f5, f6 = 0.0, f7;
+        double q0, q1, q2, q3, q4, q5;
+        int iq0, iq1, iq2, iq3, iq4;
+
+        final int jx = nx - 1; // jx in [1,2] (nx in [2,3])
+        // Could use a table to avoid division, but the gain isn't worth it most likely...
+        final int jv = (e0 - 3) / 24; // We do not handle the case (e0-3 < -23).
+        int q = e0 - ((jv << 4) + (jv << 3)) - 24; // e0-24*(jv+1)
+
+        final int j = jv + 4;
+        if (jx == 1) {
+            f5 = (j >= 0) ? ONE_OVER_TWOPI_TAB[j] : 0.0;
+            f4 = (j >= 1) ? ONE_OVER_TWOPI_TAB[j - 1] : 0.0;
+            f3 = (j >= 2) ? ONE_OVER_TWOPI_TAB[j - 2] : 0.0;
+            f2 = (j >= 3) ? ONE_OVER_TWOPI_TAB[j - 3] : 0.0;
+            f1 = (j >= 4) ? ONE_OVER_TWOPI_TAB[j - 4] : 0.0;
+            f0 = (j >= 5) ? ONE_OVER_TWOPI_TAB[j - 5] : 0.0;
+
+            q0 = x0 * f1 + x1 * f0;
+            q1 = x0 * f2 + x1 * f1;
+            q2 = x0 * f3 + x1 * f2;
+            q3 = x0 * f4 + x1 * f3;
+            q4 = x0 * f5 + x1 * f4;
+        } else { // jx == 2
+            f6 = (j >= 0) ? ONE_OVER_TWOPI_TAB[j] : 0.0;
+            f5 = (j >= 1) ? ONE_OVER_TWOPI_TAB[j - 1] : 0.0;
+            f4 = (j >= 2) ? ONE_OVER_TWOPI_TAB[j - 2] : 0.0;
+            f3 = (j >= 3) ? ONE_OVER_TWOPI_TAB[j - 3] : 0.0;
+            f2 = (j >= 4) ? ONE_OVER_TWOPI_TAB[j - 4] : 0.0;
+            f1 = (j >= 5) ? ONE_OVER_TWOPI_TAB[j - 5] : 0.0;
+            f0 = (j >= 6) ? ONE_OVER_TWOPI_TAB[j - 6] : 0.0;
+
+            q0 = x0 * f2 + x1 * f1 + x2 * f0;
+            q1 = x0 * f3 + x1 * f2 + x2 * f1;
+            q2 = x0 * f4 + x1 * f3 + x2 * f2;
+            q3 = x0 * f5 + x1 * f4 + x2 * f3;
+            q4 = x0 * f6 + x1 * f5 + x2 * f4;
+        }
+
+        z = q4;
+        fw = (double) ((int) (TWO_POW_N24 * z));
+        iq0 = (int) (z - TWO_POW_24 * fw);
+        z = q3 + fw;
+        fw = (double) ((int) (TWO_POW_N24 * z));
+        iq1 = (int) (z - TWO_POW_24 * fw);
+        z = q2 + fw;
+        fw = (double) ((int) (TWO_POW_N24 * z));
+        iq2 = (int) (z - TWO_POW_24 * fw);
+        z = q1 + fw;
+        fw = (double) ((int) (TWO_POW_N24 * z));
+        iq3 = (int) (z - TWO_POW_24 * fw);
+        z = q0 + fw;
+
+        // Here, q is in [-25,2] range or so, so we can use the table right away.
+        double twoPowQ = twoPowTab[q - MIN_DOUBLE_EXPONENT];
+
+        z = (z * twoPowQ) % 8.0;
+        z -= (double) ((int) z);
+        if (q > 0) {
+            iq3 &= 0xFFFFFF >> q;
+            ih = iq3 >> (23 - q);
+        } else if (q == 0) {
+            ih = iq3 >> 23;
+        } else if (z >= 0.5) {
+            ih = 2;
+        } else {
+            ih = 0;
+        }
+        if (ih > 0) {
+            int carry;
+            if (iq0 != 0) {
+                carry = 1;
+                iq0 = 0x1000000 - iq0;
+                iq1 = 0x0FFFFFF - iq1;
+                iq2 = 0x0FFFFFF - iq2;
+                iq3 = 0x0FFFFFF - iq3;
+            } else {
+                if (iq1 != 0) {
+                    carry = 1;
+                    iq1 = 0x1000000 - iq1;
+                    iq2 = 0x0FFFFFF - iq2;
+                    iq3 = 0x0FFFFFF - iq3;
+                } else {
+                    if (iq2 != 0) {
+                        carry = 1;
+                        iq2 = 0x1000000 - iq2;
+                        iq3 = 0x0FFFFFF - iq3;
+                    } else {
+                        if (iq3 != 0) {
+                            carry = 1;
+                            iq3 = 0x1000000 - iq3;
+                        } else {
+                            carry = 0;
+                        }
+                    }
+                }
+            }
+            if (q > 0) {
+                switch (q) {
+                    case 1 -> iq3 &= 0x7FFFFF;
+                    case 2 -> iq3 &= 0x3FFFFF;
+                }
+            }
+            if (ih == 2) {
+                z = 1.0 - z;
+                if (carry != 0) {
+                    z -= twoPowQ;
+                }
+            }
+        }
+
+        if (z == 0.0) {
+            if (jx == 1) {
+                f6 = ONE_OVER_TWOPI_TAB[jv + 5];
+                q5 = x0 * f6 + x1 * f5;
+            } else { // jx == 2
+                f7 = ONE_OVER_TWOPI_TAB[jv + 5];
+                q5 = x0 * f7 + x1 * f6 + x2 * f5;
+            }
+
+            z = q5;
+            fw = (double) ((int) (TWO_POW_N24 * z));
+            iq0 = (int) (z - TWO_POW_24 * fw);
+            z = q4 + fw;
+            fw = (double) ((int) (TWO_POW_N24 * z));
+            iq1 = (int) (z - TWO_POW_24 * fw);
+            z = q3 + fw;
+            fw = (double) ((int) (TWO_POW_N24 * z));
+            iq2 = (int) (z - TWO_POW_24 * fw);
+            z = q2 + fw;
+            fw = (double) ((int) (TWO_POW_N24 * z));
+            iq3 = (int) (z - TWO_POW_24 * fw);
+            z = q1 + fw;
+            fw = (double) ((int) (TWO_POW_N24 * z));
+            iq4 = (int) (z - TWO_POW_24 * fw);
+            z = q0 + fw;
+
+            z = (z * twoPowQ) % 8.0;
+            z -= (double) ((int) z);
+            if (q > 0) {
+                // some parentheses for Eclipse formatter's weaknesses with bits shifts
+                iq4 &= (0xFFFFFF >> q);
+                ih = (iq4 >> (23 - q));
+            } else if (q == 0) {
+                ih = iq4 >> 23;
+            } else if (z >= 0.5) {
+                ih = 2;
+            } else {
+                ih = 0;
+            }
+            if (ih > 0) {
+                if (iq0 != 0) {
+                    iq0 = 0x1000000 - iq0;
+                    iq1 = 0x0FFFFFF - iq1;
+                    iq2 = 0x0FFFFFF - iq2;
+                    iq3 = 0x0FFFFFF - iq3;
+                    iq4 = 0x0FFFFFF - iq4;
+                } else {
+                    if (iq1 != 0) {
+                        iq1 = 0x1000000 - iq1;
+                        iq2 = 0x0FFFFFF - iq2;
+                        iq3 = 0x0FFFFFF - iq3;
+                        iq4 = 0x0FFFFFF - iq4;
+                    } else {
+                        if (iq2 != 0) {
+                            iq2 = 0x1000000 - iq2;
+                            iq3 = 0x0FFFFFF - iq3;
+                            iq4 = 0x0FFFFFF - iq4;
+                        } else {
+                            if (iq3 != 0) {
+                                iq3 = 0x1000000 - iq3;
+                                iq4 = 0x0FFFFFF - iq4;
+                            } else {
+                                if (iq4 != 0) {
+                                    iq4 = 0x1000000 - iq4;
+                                }
+                            }
+                        }
+                    }
+                }
+                if (q > 0) {
+                    switch (q) {
+                        case 1 -> iq4 &= 0x7FFFFF;
+                        case 2 -> iq4 &= 0x3FFFFF;
+                    }
+                }
+            }
+            fw = twoPowQ * TWO_POW_N24; // q -= 24, so initializing fw with ((2^q)*(2^-24)=2^(q-24))
+        } else {
+            // Here, q is in [-25,-2] range or so, so we could use twoPow's table right away with
+            // iq4 = (int)(z*twoPowTab[-q-TWO_POW_TAB_MIN_POW]);
+            // but tests show using division is faster...
+            iq4 = (int) (z / twoPowQ);
+            fw = twoPowQ;
+        }
+
+        q4 = fw * (double) iq4;
+        fw *= TWO_POW_N24;
+        q3 = fw * (double) iq3;
+        fw *= TWO_POW_N24;
+        q2 = fw * (double) iq2;
+        fw *= TWO_POW_N24;
+        q1 = fw * (double) iq1;
+        fw *= TWO_POW_N24;
+        q0 = fw * (double) iq0;
+        fw *= TWO_POW_N24;
+
+        fw = TWOPI_TAB0 * q4;
+        fw += TWOPI_TAB0 * q3 + TWOPI_TAB1 * q4;
+        fw += TWOPI_TAB0 * q2 + TWOPI_TAB1 * q3 + TWOPI_TAB2 * q4;
+        fw += TWOPI_TAB0 * q1 + TWOPI_TAB1 * q2 + TWOPI_TAB2 * q3 + TWOPI_TAB3 * q4;
+        fw += TWOPI_TAB0 * q0 + TWOPI_TAB1 * q1 + TWOPI_TAB2 * q2 + TWOPI_TAB3 * q3 + TWOPI_TAB4 * q4;
+
+        return (ih == 0) ? fw : -fw;
+    }
+
+    // --------------------------------------------------------------------------
+    // STATIC INITIALIZATIONS
+    // --------------------------------------------------------------------------
+
+    /**
+     * Initializes look-up tables.
+     *
+     * Might use some FastMath methods in there, not to spend
+     * an hour in it, but must take care not to use methods
+     * which look-up tables have not yet been initialized,
+     * or that are not accurate enough.
+     */
+    static {
+
+        // sin and cos
+
+        final int SIN_COS_PI_INDEX = (SIN_COS_TABS_SIZE - 1) / 2;
+        final int SIN_COS_PI_MUL_2_INDEX = 2 * SIN_COS_PI_INDEX;
+        final int SIN_COS_PI_MUL_0_5_INDEX = SIN_COS_PI_INDEX / 2;
+        final int SIN_COS_PI_MUL_1_5_INDEX = 3 * SIN_COS_PI_INDEX / 2;
+        for (int i = 0; i < SIN_COS_TABS_SIZE; i++) {
+            // angle: in [0,2*PI].
+            double angle = i * SIN_COS_DELTA_HI + i * SIN_COS_DELTA_LO;
+            double sinAngle = StrictMath.sin(angle);
+            double cosAngle = StrictMath.cos(angle);
+            // For indexes corresponding to null cosine or sine, we make sure the value is zero
+            // and not an epsilon. This allows for a much better accuracy for results close to zero.
+            if (i == SIN_COS_PI_INDEX) {
+                sinAngle = 0.0;
+            } else if (i == SIN_COS_PI_MUL_2_INDEX) {
+                sinAngle = 0.0;
+            } else if (i == SIN_COS_PI_MUL_0_5_INDEX) {
+                cosAngle = 0.0;
+            } else if (i == SIN_COS_PI_MUL_1_5_INDEX) {
+                cosAngle = 0.0;
+            }
+            sinTab[i] = sinAngle;
+            cosTab[i] = cosAngle;
+        }
+
+        // tan
+
+        for (int i = 0; i < TAN_TABS_SIZE; i++) {
+            // angle: in [0,TAN_MAX_VALUE_FOR_TABS].
+            double angle = i * TAN_DELTA_HI + i * TAN_DELTA_LO;
+            tanTab[i] = StrictMath.tan(angle);
+            double cosAngle = StrictMath.cos(angle);
+            double sinAngle = StrictMath.sin(angle);
+            double cosAngleInv = 1 / cosAngle;
+            double cosAngleInv2 = cosAngleInv * cosAngleInv;
+            double cosAngleInv3 = cosAngleInv2 * cosAngleInv;
+            double cosAngleInv4 = cosAngleInv2 * cosAngleInv2;
+            double cosAngleInv5 = cosAngleInv3 * cosAngleInv2;
+            tanDer1DivF1Tab[i] = cosAngleInv2;
+            tanDer2DivF2Tab[i] = ((2 * sinAngle) * cosAngleInv3) * ONE_DIV_F2;
+            tanDer3DivF3Tab[i] = ((2 * (1 + 2 * sinAngle * sinAngle)) * cosAngleInv4) * ONE_DIV_F3;
+            tanDer4DivF4Tab[i] = ((8 * sinAngle * (2 + sinAngle * sinAngle)) * cosAngleInv5) * ONE_DIV_F4;
+        }
+
+        // asin
+
+        for (int i = 0; i < ASIN_TABS_SIZE; i++) {
+            // x: in [0,ASIN_MAX_VALUE_FOR_TABS].
+            double x = i * ASIN_DELTA;
+            asinTab[i] = StrictMath.asin(x);
+            double oneMinusXSqInv = 1.0 / (1 - x * x);
+            double oneMinusXSqInv0_5 = StrictMath.sqrt(oneMinusXSqInv);
+            double oneMinusXSqInv1_5 = oneMinusXSqInv0_5 * oneMinusXSqInv;
+            double oneMinusXSqInv2_5 = oneMinusXSqInv1_5 * oneMinusXSqInv;
+            double oneMinusXSqInv3_5 = oneMinusXSqInv2_5 * oneMinusXSqInv;
+            asinDer1DivF1Tab[i] = oneMinusXSqInv0_5;
+            asinDer2DivF2Tab[i] = (x * oneMinusXSqInv1_5) * ONE_DIV_F2;
+            asinDer3DivF3Tab[i] = ((1 + 2 * x * x) * oneMinusXSqInv2_5) * ONE_DIV_F3;
+            asinDer4DivF4Tab[i] = ((5 + 2 * x * (2 + x * (5 - 2 * x))) * oneMinusXSqInv3_5) * ONE_DIV_F4;
+        }
+
+        for (int i = 0; i < ASIN_POWTABS_SIZE; i++) {
+            // x: in [0,ASIN_MAX_VALUE_FOR_POWTABS].
+            double x = StrictMath.pow(i * (1.0 / ASIN_POWTABS_SIZE_MINUS_ONE), 1.0 / ASIN_POWTABS_POWER) * ASIN_MAX_VALUE_FOR_POWTABS;
+            asinParamPowTab[i] = x;
+            asinPowTab[i] = StrictMath.asin(x);
+            double oneMinusXSqInv = 1.0 / (1 - x * x);
+            double oneMinusXSqInv0_5 = StrictMath.sqrt(oneMinusXSqInv);
+            double oneMinusXSqInv1_5 = oneMinusXSqInv0_5 * oneMinusXSqInv;
+            double oneMinusXSqInv2_5 = oneMinusXSqInv1_5 * oneMinusXSqInv;
+            double oneMinusXSqInv3_5 = oneMinusXSqInv2_5 * oneMinusXSqInv;
+            asinDer1DivF1PowTab[i] = oneMinusXSqInv0_5;
+            asinDer2DivF2PowTab[i] = (x * oneMinusXSqInv1_5) * ONE_DIV_F2;
+            asinDer3DivF3PowTab[i] = ((1 + 2 * x * x) * oneMinusXSqInv2_5) * ONE_DIV_F3;
+            asinDer4DivF4PowTab[i] = ((5 + 2 * x * (2 + x * (5 - 2 * x))) * oneMinusXSqInv3_5) * ONE_DIV_F4;
+        }
+
+        // atan
+
+        for (int i = 0; i < ATAN_TABS_SIZE; i++) {
+            // x: in [0,ATAN_MAX_VALUE_FOR_TABS].
+            double x = i * ATAN_DELTA;
+            double onePlusXSqInv = 1.0 / (1 + x * x);
+            double onePlusXSqInv2 = onePlusXSqInv * onePlusXSqInv;
+            double onePlusXSqInv3 = onePlusXSqInv2 * onePlusXSqInv;
+            double onePlusXSqInv4 = onePlusXSqInv2 * onePlusXSqInv2;
+            atanTab[i] = StrictMath.atan(x);
+            atanDer1DivF1Tab[i] = onePlusXSqInv;
+            atanDer2DivF2Tab[i] = (-2 * x * onePlusXSqInv2) * ONE_DIV_F2;
+            atanDer3DivF3Tab[i] = ((-2 + 6 * x * x) * onePlusXSqInv3) * ONE_DIV_F3;
+            atanDer4DivF4Tab[i] = ((24 * x * (1 - x * x)) * onePlusXSqInv4) * ONE_DIV_F4;
+        }
+
+        // twoPow
+
+        for (int i = MIN_DOUBLE_EXPONENT; i <= MAX_DOUBLE_EXPONENT; i++) {
+            twoPowTab[i - MIN_DOUBLE_EXPONENT] = StrictMath.pow(2.0, i);
+        }
+    }
+}
diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/H3.java b/libs/h3/src/main/java/org/elasticsearch/h3/H3.java
index a27799855239..46bcc3f141dd 100644
--- a/libs/h3/src/main/java/org/elasticsearch/h3/H3.java
+++ b/libs/h3/src/main/java/org/elasticsearch/h3/H3.java
@@ -33,6 +33,15 @@ public final class H3 {
 
     public static int MAX_H3_RES = Constants.MAX_H3_RES;
 
+    private static final long[] NORTH = new long[MAX_H3_RES + 1];
+    private static final long[] SOUTH = new long[MAX_H3_RES + 1];
+    static {
+        for (int res = 0; res <= H3.MAX_H3_RES; res++) {
+            NORTH[res] = H3.geoToH3(90, 0, res);
+            SOUTH[res] = H3.geoToH3(-90, 0, res);
+        }
+    }
+
     /**
      * Converts from <code>long</code> representation of an index to <code>String</code> representation.
      */
@@ -229,13 +238,9 @@ public static String h3ToParent(String h3Address) {
      * Returns the children of the given index.
      */
     public static long[] h3ToChildren(long h3) {
-        long[] children = new long[cellToChildrenSize(h3)];
-        int res = H3Index.H3_get_resolution(h3);
-        Iterator.IterCellsChildren it = Iterator.iterInitParent(h3, res + 1);
-        int pos = 0;
-        while (it.h != Iterator.H3_NULL) {
-            children[pos++] = it.h;
-            Iterator.iterStepChild(it);
+        final long[] children = new long[h3ToChildrenSize(h3)];
+        for (int i = 0; i < children.length; i++) {
+            children[i] = childPosToH3(h3, i);
         }
         return children;
     }
@@ -248,6 +253,96 @@ public static String[] h3ToChildren(String h3Address) {
         return h3ToStringList(h3ToChildren(stringToH3(h3Address)));
     }
 
+    /**
+     * Returns the child cell at the given position
+     */
+    public static long childPosToH3(long h3, int childPos) {
+        final int childrenRes = H3Index.H3_get_resolution(h3) + 1;
+        if (childrenRes > MAX_H3_RES) {
+            throw new IllegalArgumentException("Resolution overflow");
+        }
+        final long childH = H3Index.H3_set_resolution(h3, childrenRes);
+        if (childPos == 0) {
+            return H3Index.H3_set_index_digit(childH, childrenRes, CoordIJK.Direction.CENTER_DIGIT.digit());
+        }
+        final boolean isPentagon = isPentagon(h3);
+        final int maxPos = isPentagon ? 5 : 6;
+        if (childPos < 0 || childPos > maxPos) {
+            throw new IllegalArgumentException("invalid child position");
+        }
+        if (isPentagon) {
+            // Pentagon skip digit (position) is the number 1, therefore we add one
+            // to the current position.
+            return H3Index.H3_set_index_digit(childH, childrenRes, childPos + 1);
+        } else {
+            return H3Index.H3_set_index_digit(childH, childrenRes, childPos);
+        }
+    }
+
+    /**
+     * Returns the child address at the given position
+     */
+    public static String childPosToH3(String h3Address, int childPos) {
+        return h3ToString(childPosToH3(stringToH3(h3Address), childPos));
+    }
+
+    private static final int[] PEN_INTERSECTING_CHILDREN_DIRECTIONS = new int[] { 3, 1, 6, 4, 2 };
+    private static final int[] HEX_INTERSECTING_CHILDREN_DIRECTIONS = new int[] { 3, 6, 2, 5, 1, 4 };
+
+    /**
+     * Returns the h3 bins on the level below which are not children of the given H3 index but
+     * intersects with it.
+     */
+    public static long[] h3ToNoChildrenIntersecting(long h3) {
+        final boolean isPentagon = isPentagon(h3);
+        final long[] noChildren = new long[isPentagon ? 5 : 6];
+        for (int i = 0; i < noChildren.length; i++) {
+            noChildren[i] = noChildIntersectingPosToH3(h3, i);
+        }
+        return noChildren;
+    }
+
+    /**
+     * Returns the h3 addresses on the level below which are not children of the given H3 address but
+     * intersects with it.
+     */
+    public static String[] h3ToNoChildrenIntersecting(String h3Address) {
+        return h3ToStringList(h3ToNoChildrenIntersecting(stringToH3(h3Address)));
+    }
+
+    /**
+     * Returns the no child intersecting cell at the given position
+     */
+    public static long noChildIntersectingPosToH3(long h3, int childPos) {
+        final int childrenRes = H3Index.H3_get_resolution(h3) + 1;
+        if (childrenRes > MAX_H3_RES) {
+            throw new IllegalArgumentException("Resolution overflow");
+        }
+        final boolean isPentagon = isPentagon(h3);
+        final int maxPos = isPentagon ? 4 : 5;
+        if (childPos < 0 || childPos > maxPos) {
+            throw new IllegalArgumentException("invalid child position");
+        }
+        final long childH = H3Index.H3_set_resolution(h3, childrenRes);
+        if (isPentagon) {
+            // Pentagon skip digit (position) is the number 1, therefore we add one
+            // for the skip digit and one for the 0 (center) digit.
+            final long child = H3Index.H3_set_index_digit(childH, childrenRes, childPos + 2);
+            return HexRing.h3NeighborInDirection(child, PEN_INTERSECTING_CHILDREN_DIRECTIONS[childPos]);
+        } else {
+            // we add one for the 0 (center) digit.
+            final long child = H3Index.H3_set_index_digit(childH, childrenRes, childPos + 1);
+            return HexRing.h3NeighborInDirection(child, HEX_INTERSECTING_CHILDREN_DIRECTIONS[childPos]);
+        }
+    }
+
+    /**
+     * Returns the no child intersecting cell at the given position
+     */
+    public static String noChildIntersectingPosToH3(String h3Address, int childPos) {
+        return h3ToString(noChildIntersectingPosToH3(stringToH3(h3Address), childPos));
+    }
+
     /**
      * Returns the neighbor indexes.
      *
@@ -265,7 +360,59 @@ public static String[] hexRing(String h3Address) {
      * @return All neighbor indexes from the origin
      */
     public static long[] hexRing(long h3) {
-        return HexRing.hexRing(h3);
+        final long[] ring = new long[hexRingSize(h3)];
+        for (int i = 0; i < ring.length; i++) {
+            ring[i] = hexRingPosToH3(h3, i);
+            assert ring[i] >= 0;
+        }
+        return ring;
+    }
+
+    /**
+     * Returns the number of neighbor indexes.
+     *
+     * @param h3 Origin index
+     * @return the number of neighbor indexes from the origin
+     */
+    public static int hexRingSize(long h3) {
+        return H3Index.H3_is_pentagon(h3) ? 5 : 6;
+    }
+
+    /**
+     * Returns the number of neighbor indexes.
+     *
+     * @param h3Address Origin index
+     * @return the number of neighbor indexes from the origin
+     */
+    public static int hexRingSize(String h3Address) {
+        return hexRingSize(stringToH3(h3Address));
+    }
+
+    /**
+     * Returns the neighbor index at the given position.
+     *
+     * @param h3 Origin index
+     * @param ringPos position of the neighbour index
+     * @return the actual neighbour at the given position
+     */
+    public static long hexRingPosToH3(long h3, int ringPos) {
+        // for pentagons, we skip direction at position 2
+        final int pos = H3Index.H3_is_pentagon(h3) && ringPos >= 2 ? ringPos + 1 : ringPos;
+        if (pos < 0 || pos > 5) {
+            throw new IllegalArgumentException("invalid ring position");
+        }
+        return HexRing.h3NeighborInDirection(h3, HexRing.DIRECTIONS[pos].digit());
+    }
+
+    /**
+     * Returns the neighbor index at the given position.
+     *
+     * @param h3Address Origin index
+     * @param ringPos position of the neighbour index
+     * @return the actual neighbour at the given position
+     */
+    public static String hexRingPosToH3(String h3Address, int ringPos) {
+        return h3ToString(hexRingPosToH3(stringToH3(h3Address), ringPos));
     }
 
     /**
@@ -291,23 +438,140 @@ public static boolean areNeighborCells(long origin, long destination) {
     }
 
     /**
-     * cellToChildrenSize returns the exact number of children for a cell at a
+     * h3ToChildrenSize returns the exact number of children for a cell at a
      * given child resolution.
      *
-     * @param h         H3Index to find the number of children of
+     * @param h3         H3Index to find the number of children of
+     * @param childRes  The child resolution you're interested in
      *
-     * @return int      Exact number of children (handles hexagons and pentagons
+     * @return long      Exact number of children (handles hexagons and pentagons
      *                  correctly)
      */
-    private static int cellToChildrenSize(long h) {
-        int n = 1;
-        if (H3Index.H3_is_pentagon(h)) {
-            return (1 + 5 * (_ipow(7, n) - 1) / 6);
+    public static long h3ToChildrenSize(long h3, int childRes) {
+        final int parentRes = H3Index.H3_get_resolution(h3);
+        if (childRes <= parentRes || childRes > MAX_H3_RES) {
+            throw new IllegalArgumentException("Invalid child resolution [" + childRes + "]");
+        }
+        final int n = childRes - parentRes;
+        if (H3Index.H3_is_pentagon(h3)) {
+            return (1L + 5L * (_ipow(7, n) - 1L) / 6L);
         } else {
             return _ipow(7, n);
         }
     }
 
+    /**
+     * h3ToChildrenSize returns the exact number of children for a h3 affress at a
+     * given child resolution.
+     *
+     * @param h3Address  H3 address to find the number of children of
+     * @param childRes  The child resolution you're interested in
+     *
+     * @return int      Exact number of children (handles hexagons and pentagons
+     *                  correctly)
+     */
+    public static long h3ToChildrenSize(String h3Address, int childRes) {
+        return h3ToChildrenSize(stringToH3(h3Address), childRes);
+    }
+
+    /**
+     * h3ToChildrenSize returns the exact number of children
+     *
+     * @param h3         H3Index to find the number of children.
+     *
+     * @return int      Exact number of children, 6 for Pentagons and 7 for hexagons,
+     */
+    public static int h3ToChildrenSize(long h3) {
+        if (H3Index.H3_get_resolution(h3) == MAX_H3_RES) {
+            throw new IllegalArgumentException("Invalid child resolution [" + MAX_H3_RES + "]");
+        }
+        return isPentagon(h3) ? 6 : 7;
+    }
+
+    /**
+     * h3ToChildrenSize returns the exact number of children
+     *
+     * @param h3Address H3 address to find the number of children.
+     *
+     * @return int      Exact number of children, 6 for Pentagons and 7 for hexagons,
+     */
+    public static int h3ToChildrenSize(String h3Address) {
+        return h3ToChildrenSize(stringToH3(h3Address));
+    }
+
+    /**
+     * h3ToNotIntersectingChildrenSize returns the exact number of children intersecting
+     * the given parent but not part of the children set.
+     *
+     * @param h3         H3Index to find the number of children.
+     *
+     * @return int      Exact number of children, 5 for Pentagons and 6 for hexagons,
+     */
+    public static int h3ToNotIntersectingChildrenSize(long h3) {
+        if (H3Index.H3_get_resolution(h3) == MAX_H3_RES) {
+            throw new IllegalArgumentException("Invalid child resolution [" + MAX_H3_RES + "]");
+        }
+        return isPentagon(h3) ? 5 : 6;
+    }
+
+    /**
+     * h3ToNotIntersectingChildrenSize returns the exact number of children intersecting
+     * the given parent but not part of the children set.
+     *
+     * @param h3Address H3 address to find the number of children.
+     *
+     * @return int      Exact number of children, 5 for Pentagons and 6 for hexagons,
+     */
+    public static int h3ToNotIntersectingChildrenSize(String h3Address) {
+        return h3ToNotIntersectingChildrenSize(stringToH3(h3Address));
+    }
+
+    /**
+     * Find the h3 index containing the North Pole at the given resolution.
+     *
+     * @param res the provided resolution.
+     *
+     * @return the h3 index containing the North Pole.
+     */
+    public static long northPolarH3(int res) {
+        checkResolution(res);
+        return NORTH[res];
+    }
+
+    /**
+     * Find the h3 address containing the North Pole at the given resolution.
+     *
+     * @param res the provided resolution.
+     *
+     * @return the h3 address containing the North Pole.
+     */
+    public static String northPolarH3Address(int res) {
+        return h3ToString(northPolarH3(res));
+    }
+
+    /**
+     * Find the h3 index containing the South Pole at the given resolution.
+     *
+     * @param res the provided resolution.
+     *
+     * @return the h3 index containing the South Pole.
+     */
+    public static long southPolarH3(int res) {
+        checkResolution(res);
+        return SOUTH[res];
+    }
+
+    /**
+     * Find the h3 address containing the South Pole at the given resolution.
+     *
+     * @param res the provided resolution.
+     *
+     * @return the h3 address containing the South Pole.
+     */
+    public static String southPolarH3Address(int res) {
+        return h3ToString(southPolarH3(res));
+    }
+
     /**
      * _ipow does integer exponentiation efficiently. Taken from StackOverflow.
      *
@@ -316,8 +580,8 @@ private static int cellToChildrenSize(long h) {
      *
      * @return the exponentiated value
      */
-    private static int _ipow(int base, int exp) {
-        int result = 1;
+    private static long _ipow(int base, int exp) {
+        long result = 1;
         while (exp != 0) {
             if ((exp & 1) != 0) {
                 result *= base;
diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/HexRing.java b/libs/h3/src/main/java/org/elasticsearch/h3/HexRing.java
index 3dfe2417be06..6ee6bd73fb66 100644
--- a/libs/h3/src/main/java/org/elasticsearch/h3/HexRing.java
+++ b/libs/h3/src/main/java/org/elasticsearch/h3/HexRing.java
@@ -309,7 +309,7 @@ final class HexRing {
      *     \\2/
      * </pre>
      */
-    private static final CoordIJK.Direction[] DIRECTIONS = new CoordIJK.Direction[] {
+    static final CoordIJK.Direction[] DIRECTIONS = new CoordIJK.Direction[] {
         CoordIJK.Direction.J_AXES_DIGIT,
         CoordIJK.Direction.JK_AXES_DIGIT,
         CoordIJK.Direction.K_AXES_DIGIT,
@@ -587,31 +587,6 @@ final class HexRing {
         CoordIJK.Direction.I_AXES_DIGIT,
         CoordIJK.Direction.J_AXES_DIGIT };
 
-    /**
-     * Produce all neighboring cells. For Hexagons there will be 6 neighbors while
-     * for pentagon just 5.
-     * Output is placed in the provided array in no particular order.
-     *
-     * @param  origin   origin cell
-     */
-    public static long[] hexRing(long origin) {
-        final long[] out = H3Index.H3_is_pentagon(origin) ? new long[5] : new long[6];
-        int idx = 0;
-        long previous = -1;
-        for (int i = 0; i < 6; i++) {
-            long neighbor = h3NeighborInDirection(origin, DIRECTIONS[i].digit());
-            if (neighbor != -1) {
-                // -1 is an expected case when trying to traverse off of pentagons.
-                if (previous != neighbor) {
-                    out[idx++] = neighbor;
-                    previous = neighbor;
-                }
-            }
-        }
-        assert idx == out.length;
-        return out;
-    }
-
     /**
      * Returns whether or not the provided H3Indexes are neighbors.
      * @param origin The origin H3 index.
@@ -700,7 +675,7 @@ public static boolean areNeighbours(long origin, long destination) {
      * @param dir Direction to move in
      * @return H3Index of the specified neighbor or -1 if there is no more neighbor
      */
-    private static long h3NeighborInDirection(long origin, int dir) {
+    static long h3NeighborInDirection(long origin, int dir) {
         long current = origin;
 
         int newRotations = 0;
diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/Iterator.java b/libs/h3/src/main/java/org/elasticsearch/h3/Iterator.java
deleted file mode 100644
index d711c3e4224a..000000000000
--- a/libs/h3/src/main/java/org/elasticsearch/h3/Iterator.java
+++ /dev/null
@@ -1,306 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- * This project is based on a modification of https://github.com/uber/h3 which is licensed under the Apache 2.0 License.
- *
- * Copyright 2021 Uber Technologies, Inc.
- */
-package org.elasticsearch.h3;
-
-/**
- * Iterator structures and functions for the children of a cell.
- */
-final class Iterator {
-    /**
-     * Invalid index used to indicate an error from latLngToCell and related
-     * functions or missing data in arrays of H3 indices. Analogous to NaN in
-     * floating point.
-     */
-    public static final long H3_NULL = 0;
-
-    /**
-     * The number of bits in a single H3 resolution digit.
-     */
-    private static final int H3_PER_DIGIT_OFFSET = 3;
-
-    /**
-     * IterCellsChildren: struct for iterating through the descendants of
-     * a given cell.
-     * <p>
-     * Constructors:
-     * <p>
-     * Initialize with either `iterInitParent` or `iterInitBaseCellNum`.
-     * `iterInitParent` sets up an iterator for all the children of a given
-     * parent cell at a given resolution.
-     * <p>
-     * `iterInitBaseCellNum` sets up an iterator for children cells, given
-     * a base cell number (0--121).
-     * <p>
-     * Iteration:
-     * <p>
-     * Step iterator with `iterStepChild`.
-     * During the lifetime of the `IterCellsChildren`, the current iterate
-     * is accessed via the `IterCellsChildren.h` member.
-     * When the iterator is exhausted or if there was an error in initialization,
-     * `IterCellsChildren.h` will be `H3_NULL` even after calling `iterStepChild`.
-     */
-    static class IterCellsChildren {
-        long h;
-        int _parentRes;  // parent resolution
-        int _skipDigit;  // this digit skips `1` for pentagons
-
-        IterCellsChildren(long h, int _parentRes, int _skipDigit) {
-            this.h = h;
-            this._parentRes = _parentRes;
-            this._skipDigit = _skipDigit;
-        }
-    }
-
-    /**
-     * Create a fully nulled-out child iterator for when an iterator is exhausted.
-     * This helps minimize the chance that a user will depend on the iterator
-     * internal state after it's exhausted, like the child resolution, for
-     * example.
-     */
-    private static IterCellsChildren nullIter() {
-        return new IterCellsChildren(H3_NULL, -1, -1);
-    }
-
-    /**
-     ## Logic for iterating through the children of a cell
-     We'll describe the logic for ....
-     - normal (non pentagon iteration)
-     - pentagon iteration. define "pentagon digit"
-     ### Cell Index Component Diagrams
-     The lower 56 bits of an H3 Cell Index describe the following index components:
-     - the cell resolution (4 bits)
-     - the base cell number (7 bits)
-     - the child cell digit for each resolution from 1 to 15 (3*15 = 45 bits)
-     These are the bits we'll be focused on when iterating through child cells.
-     To help describe the iteration logic, we'll use diagrams displaying the
-     (decimal) values for each component like:
-     child digit for resolution 2
-     /
-     | res | base cell # | 1 | 2 | 3 | 4 | 5 | 6 | ... |
-     |-----|-------------|---|---|---|---|---|---|-----|
-     |   9 |          17 | 5 | 3 | 0 | 6 | 2 | 1 | ... |
-     ### Iteration through children of a hexagon (but not a pentagon)
-     Iteration through the children of a *hexagon* (but not a pentagon)
-     simply involves iterating through all the children values (0--6)
-     for each child digit (up to the child's resolution).
-     For example, suppose a resolution 3 hexagon index has the following
-     components:
-     parent resolution
-     /
-     | res | base cell # | 1 | 2 | 3 | 4 | 5 | 6 | ... |
-     |-----|-------------|---|---|---|---|---|---|-----|
-     |   3 |          17 | 3 | 5 | 1 | 7 | 7 | 7 | ... |
-     The iteration through all children of resolution 6 would look like:
-     parent res  child res
-     /           /
-     | res | base cell # | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | ... |
-     |-----|-------------|---|---|---|---|---|---|---|---|-----|
-     | 6   |          17 | 3 | 5 | 1 | 0 | 0 | 0 | 7 | 7 | ... |
-     | 6   |          17 | 3 | 5 | 1 | 0 | 0 | 1 | 7 | 7 | ... |
-     | ... |             |   |   |   |   |   |   |   |   |     |
-     | 6   |          17 | 3 | 5 | 1 | 0 | 0 | 6 | 7 | 7 | ... |
-     | 6   |          17 | 3 | 5 | 1 | 0 | 1 | 0 | 7 | 7 | ... |
-     | 6   |          17 | 3 | 5 | 1 | 0 | 1 | 1 | 7 | 7 | ... |
-     | ... |             |   |   |   |   |   |   |   |   |     |
-     | 6   |          17 | 3 | 5 | 1 | 6 | 6 | 6 | 7 | 7 | ... |
-     ### Step sequence on a *pentagon* cell
-     Pentagon cells have a base cell number (e.g., 97) corresponding to a
-     resolution 0 pentagon, and have all zeros from digit 1 to the digit
-     corresponding to the cell's resolution.
-     (We'll drop the ellipses from now on, knowing that digits should contain
-     7's beyond the cell resolution.)
-     parent res      child res
-     /               /
-     | res | base cell # | 1 | 2 | 3 | 4 | 5 | 6 |
-     |-----|-------------|---|---|---|---|---|---|
-     |   6 |          97 | 0 | 0 | 0 | 0 | 0 | 0 |
-     Iteration through children of a *pentagon* is almost the same
-     as *hexagon* iteration, except that we skip the *first* 1 value
-     that appears in the "skip digit". This corresponds to the fact
-     that a pentagon only has 6 children, which are denoted with
-     the numbers {0,2,3,4,5,6}.
-     The skip digit starts at the child resolution position.
-     When iterating through children more than one resolution below
-     the parent, we move the skip digit to the left
-     (up to the next coarser resolution) each time we skip the 1 value
-     in that digit.
-     Iteration would start like:
-     parent res      child res
-     /               /
-     | res | base cell # | 1 | 2 | 3 | 4 | 5 | 6 |
-     |-----|-------------|---|---|---|---|---|---|
-     |   6 |          97 | 0 | 0 | 0 | 0 | 0 | 0 |
-     \
-     skip digit
-     Noticing we skip the 1 value and move the skip digit,
-     the next iterate would be:
-     | res | base cell # | 1 | 2 | 3 | 4 | 5 | 6 |
-     |-----|-------------|---|---|---|---|---|---|
-     |   6 |          97 | 0 | 0 | 0 | 0 | 0 | 2 |
-     \
-     skip digit
-     Iteration continues normally until we get to:
-     | res | base cell # | 1 | 2 | 3 | 4 | 5 | 6 |
-     |-----|-------------|---|---|---|---|---|---|
-     |   6 |          97 | 0 | 0 | 0 | 0 | 0 | 6 |
-     \
-     skip digit
-     which is followed by (skipping the 1):
-     | res | base cell # | 1 | 2 | 3 | 4 | 5 | 6 |
-     |-----|-------------|---|---|---|---|---|---|
-     |   6 |          97 | 0 | 0 | 0 | 0 | 2 | 0 |
-     \
-     skip digit
-     For the next iterate, we won't skip the `1` in the previous digit
-     because it is no longer the skip digit:
-     | res | base cell # | 1 | 2 | 3 | 4 | 5 | 6 |
-     |-----|-------------|---|---|---|---|---|---|
-     |   6 |          97 | 0 | 0 | 0 | 0 | 2 | 1 |
-     \
-     skip digit
-     Iteration continues normally until we're right before the next skip
-     digit:
-     | res | base cell # | 1 | 2 | 3 | 4 | 5 | 6 |
-     |-----|-------------|---|---|---|---|---|---|
-     |   6 |          97 | 0 | 0 | 0 | 0 | 6 | 6 |
-     \
-     skip digit
-     Which is followed by
-     | res | base cell # | 1 | 2 | 3 | 4 | 5 | 6 |
-     |-----|-------------|---|---|---|---|---|---|
-     |   6 |          97 | 0 | 0 | 0 | 2 | 0 | 0 |
-     \
-     skip digit
-     and so on.
-     */
-
-    /**
-     * Initialize a IterCellsChildren struct representing the sequence giving
-     * the children of cell `h` at resolution `childRes`.
-     * <p>
-     * At any point in the iteration, starting once
-     * the struct is initialized, IterCellsChildren.h gives the current child.
-     * <p>
-     * Also, IterCellsChildren.h == H3_NULL when all the children have been iterated
-     * through, or if the input to `iterInitParent` was invalid.
-     */
-    public static IterCellsChildren iterInitParent(long h, int childRes) {
-
-        int parentRes = H3Index.H3_get_resolution(h);
-
-        if (childRes < parentRes || childRes > Constants.MAX_H3_RES || h == H3_NULL) {
-            return nullIter();
-        }
-
-        long newH = zeroIndexDigits(h, parentRes + 1, childRes);
-        newH = H3Index.H3_set_resolution(newH, childRes);
-
-        int _skipDigit;
-        if (H3Index.H3_is_pentagon(newH)) {
-            // The skip digit skips `1` for pentagons.
-            // The "_skipDigit" moves to the left as we count up from the
-            // child resolution to the parent resolution.
-            _skipDigit = childRes;
-        } else {
-            // if not a pentagon, we can ignore "skip digit" logic
-            _skipDigit = -1;
-        }
-
-        return new IterCellsChildren(newH, parentRes, _skipDigit);
-    }
-
-    /**
-     * Step a IterCellsChildren to the next child cell.
-     * When the iteration is over, IterCellsChildren.h will be H3_NULL.
-     * Handles iterating through hexagon and pentagon cells.
-     */
-    public static void iterStepChild(IterCellsChildren it) {
-        // once h == H3_NULL, the iterator returns an infinite sequence of H3_NULL
-        if (it.h == H3_NULL) return;
-
-        int childRes = H3Index.H3_get_resolution(it.h);
-
-        incrementResDigit(it, childRes);
-
-        for (int i = childRes; i >= it._parentRes; i--) {
-            if (i == it._parentRes) {
-                // if we're modifying the parent resolution digit, then we're done
-                // *it = _null_iter();
-                it.h = H3_NULL;
-                return;
-            }
-
-            // PENTAGON_SKIPPED_DIGIT == 1
-            if (i == it._skipDigit && getResDigit(it, i) == CoordIJK.Direction.PENTAGON_SKIPPED_DIGIT.digit()) {
-                // Then we are iterating through the children of a pentagon cell.
-                // All children of a pentagon have the property that the first
-                // nonzero digit between the parent and child resolutions is
-                // not 1.
-                // I.e., we never see a sequence like 00001.
-                // Thus, we skip the `1` in this digit.
-                incrementResDigit(it, i);
-                it._skipDigit -= 1;
-                return;
-            }
-
-            // INVALID_DIGIT == 7
-            if (getResDigit(it, i) == CoordIJK.Direction.INVALID_DIGIT.digit()) {
-                incrementResDigit(it, i);  // zeros out it[i] and increments it[i-1] by 1
-            } else {
-                break;
-            }
-        }
-    }
-
-    // extract the `res` digit (0--7) of the current cell
-    private static int getResDigit(IterCellsChildren it, int res) {
-        return H3Index.H3_get_index_digit(it.h, res);
-    }
-
-    /**
-     * Zero out index digits from start to end, inclusive.
-     * No-op if start > end.
-     */
-    private static long zeroIndexDigits(long h, int start, int end) {
-        if (start > end) {
-            return h;
-        }
-
-        long m = 0;
-
-        m = ~m;
-        m <<= H3_PER_DIGIT_OFFSET * (end - start + 1);
-        m = ~m;
-        m <<= H3_PER_DIGIT_OFFSET * (Constants.MAX_H3_RES - end);
-        m = ~m;
-
-        return h & m;
-    }
-
-    // increment the digit (0--7) at location `res`
-    private static void incrementResDigit(IterCellsChildren it, int res) {
-        long val = 1;
-        val <<= H3_PER_DIGIT_OFFSET * (Constants.MAX_H3_RES - res);
-        it.h += val;
-    }
-}
diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/LatLng.java b/libs/h3/src/main/java/org/elasticsearch/h3/LatLng.java
index 638498da72a1..713a09df8863 100644
--- a/libs/h3/src/main/java/org/elasticsearch/h3/LatLng.java
+++ b/libs/h3/src/main/java/org/elasticsearch/h3/LatLng.java
@@ -22,9 +22,19 @@
  */
 package org.elasticsearch.h3;
 
+import java.util.Objects;
+
 /** pair of latitude/longitude */
 public final class LatLng {
 
+    /** Minimum Angular resolution. */
+    private static final double MINIMUM_ANGULAR_RESOLUTION = Math.PI * 1.0e-12; // taken from lucene's spatial3d
+
+    /**
+     * pi / 2.0
+     */
+    private static final double M_PI_2 = 1.5707963267948966;
+
     // lat / lon in radians
     private final double lon;
     private final double lat;
@@ -62,9 +72,126 @@ public double getLonDeg() {
      * @return The azimuth in radians.
      */
     double geoAzimuthRads(double lat, double lon) {
-        return Math.atan2(
-            Math.cos(lat) * Math.sin(lon - this.lon),
-            Math.cos(this.lat) * Math.sin(lat) - Math.sin(this.lat) * Math.cos(lat) * Math.cos(lon - this.lon)
+        // algorithm from the original H3 library
+        final double cosLat = FastMath.cos(lat);
+        return FastMath.atan2(
+            cosLat * FastMath.sin(lon - this.lon),
+            FastMath.cos(this.lat) * FastMath.sin(lat) - FastMath.sin(this.lat) * cosLat * FastMath.cos(lon - this.lon)
         );
     }
+
+    /**
+     * Computes the point on the sphere with a specified azimuth and distance from
+     * this point.
+     *
+     * @param az       The desired azimuth.
+     * @param distance The desired distance.
+     * @return The LatLng point.
+     */
+    LatLng geoAzDistanceRads(double az, double distance) {
+        // algorithm from the original H3 library
+        az = Vec2d.posAngleRads(az);
+        final double sinDistance = FastMath.sin(distance);
+        final double cosDistance = FastMath.cos(distance);
+        final double sinP1Lat = FastMath.sin(getLatRad());
+        final double cosP1Lat = FastMath.cos(getLatRad());
+        final double sinlat = Math.max(-1.0, Math.min(1.0, sinP1Lat * cosDistance + cosP1Lat * sinDistance * FastMath.cos(az)));
+        final double lat = FastMath.asin(sinlat);
+        if (Math.abs(lat - M_PI_2) < Constants.EPSILON) { // north pole
+            return new LatLng(M_PI_2, 0.0);
+        } else if (Math.abs(lat + M_PI_2) < Constants.EPSILON) { // south pole
+            return new LatLng(-M_PI_2, 0.0);
+        } else {
+            final double cosLat = FastMath.cos(lat);
+            final double sinlng = Math.max(-1.0, Math.min(1.0, FastMath.sin(az) * sinDistance / cosLat));
+            final double coslng = Math.max(-1.0, Math.min(1.0, (cosDistance - sinP1Lat * FastMath.sin(lat)) / cosP1Lat / cosLat));
+            return new LatLng(lat, constrainLng(getLonRad() + FastMath.atan2(sinlng, coslng)));
+        }
+    }
+
+    /**
+     * constrainLng makes sure longitudes are in the proper bounds
+     *
+     * @param lng The origin lng value
+     * @return The corrected lng value
+     */
+    private static double constrainLng(double lng) {
+        while (lng > Math.PI) {
+            lng = lng - Constants.M_2PI;
+        }
+        while (lng < -Math.PI) {
+            lng = lng + Constants.M_2PI;
+        }
+        return lng;
+    }
+
+    /**
+     * Determines the maximum latitude of the great circle defined by this LatLng to the provided LatLng.
+     *
+     * @param latLng The LatLng.
+     * @return The maximum latitude of the great circle in radians.
+     */
+    public double greatCircleMaxLatitude(LatLng latLng) {
+        if (isNumericallyIdentical(latLng)) {
+            return latLng.lat;
+        }
+        return latLng.lat > this.lat ? greatCircleMaxLatitude(latLng, this) : greatCircleMaxLatitude(this, latLng);
+    }
+
+    private static double greatCircleMaxLatitude(LatLng latLng1, LatLng latLng2) {
+        // we compute the max latitude using Clairaut's formula (https://streckenflug.at/download/formeln.pdf)
+        assert latLng1.lat >= latLng2.lat;
+        final double az = latLng1.geoAzimuthRads(latLng2.lat, latLng2.lon);
+        // the great circle contains the maximum latitude only if the azimuth is between -90 and 90 degrees.
+        if (Math.abs(az) < M_PI_2) {
+            return FastMath.acos(Math.abs(FastMath.sin(az) * FastMath.cos(latLng1.lat)));
+        }
+        return latLng1.lat;
+    }
+
+    /**
+     * Determines the minimum latitude of the great circle defined by this LatLng to the provided LatLng.
+     *
+     * @param latLng The LatLng.
+     * @return The minimum latitude of the great circle in radians.
+     */
+    public double greatCircleMinLatitude(LatLng latLng) {
+        if (isNumericallyIdentical(latLng)) {
+            return latLng.lat;
+        }
+        return latLng.lat < this.lat ? greatCircleMinLatitude(latLng, this) : greatCircleMinLatitude(this, latLng);
+    }
+
+    private static double greatCircleMinLatitude(LatLng latLng1, LatLng latLng2) {
+        assert latLng1.lat <= latLng2.lat;
+        // we compute the min latitude using Clairaut's formula (https://streckenflug.at/download/formeln.pdf)
+        final double az = latLng1.geoAzimuthRads(latLng2.lat, latLng2.lon);
+        // the great circle contains the minimum latitude only if the azimuth is not between -90 and 90 degrees.
+        if (Math.abs(az) > M_PI_2) {
+            // note the sign
+            return -FastMath.acos(Math.abs(FastMath.sin(az) * FastMath.cos(latLng1.lat)));
+        }
+        return latLng1.lat;
+    }
+
+    boolean isNumericallyIdentical(LatLng latLng) {
+        return Math.abs(this.lat - latLng.lat) < MINIMUM_ANGULAR_RESOLUTION && Math.abs(this.lon - latLng.lon) < MINIMUM_ANGULAR_RESOLUTION;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        final LatLng latLng = (LatLng) o;
+        return Double.compare(latLng.lon, lon) == 0 && Double.compare(latLng.lat, lat) == 0;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(lon, lat);
+    }
 }
diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/Vec2d.java b/libs/h3/src/main/java/org/elasticsearch/h3/Vec2d.java
index 5aa59b534b96..12ce728a9996 100644
--- a/libs/h3/src/main/java/org/elasticsearch/h3/Vec2d.java
+++ b/libs/h3/src/main/java/org/elasticsearch/h3/Vec2d.java
@@ -32,6 +32,8 @@ final class Vec2d {
     /** sin(60') */
     private static final double M_SIN60 = Constants.M_SQRT3_2;
 
+    private static final double VEC2D_RESOLUTION = 1e-7;
+
     /**
      * icosahedron face centers in lat/lng radians
      */
@@ -85,19 +87,6 @@ final class Vec2d {
         { 2.361378999196363184, 0.266983896803167583, 4.455774101589558636 },  // face 19
     };
 
-    /**
-     * pi
-     */
-    private static final double M_PI = 3.14159265358979323846;
-    /**
-     * pi / 2.0
-     */
-    private static final double M_PI_2 = 1.5707963267948966;
-    /**
-     * 2.0 * PI
-     */
-    public static final double M_2PI = 6.28318530717958647692528676655900576839433;
-
     private final double x;  /// < x component
     private final double y;  /// < y component
 
@@ -107,7 +96,7 @@ final class Vec2d {
     }
 
     /**
-     * Determines the center point in spherical coordinates of a cell given by 2D
+     * Determines the center point in spherical coordinates of a cell given by this 2D
      * hex coordinates on a particular icosahedral face.
      *
      * @param face      The icosahedral face upon which the 2D hex coordinate system is
@@ -117,14 +106,30 @@ final class Vec2d {
      *                  grid relative to the specified resolution.
      */
     public LatLng hex2dToGeo(int face, int res, boolean substrate) {
+        return hex2dToGeo(this.x, this.y, face, res, substrate);
+    }
+
+    /**
+     * Determines the center point in spherical coordinates of a cell given by the provided 2D
+     * hex coordinates on a particular icosahedral face.
+     *
+     * @param x         The x component of the 2D hex coordinates.
+     * @param y         The y component of the 2D hex coordinates.
+     * @param face      The icosahedral face upon which the 2D hex coordinate system is
+     *                  centered.
+     * @param res       The H3 resolution of the cell.
+     * @param substrate Indicates whether or not this grid is actually a substrate
+     *                  grid relative to the specified resolution.
+     */
+    static LatLng hex2dToGeo(double x, double y, int face, int res, boolean substrate) {
         // calculate (r, theta) in hex2d
-        double r = v2dMag();
+        double r = Math.sqrt(x * x + y * y);
 
         if (r < Constants.EPSILON) {
             return faceCenterGeo[face];
         }
 
-        double theta = Math.atan2(y, x);
+        double theta = FastMath.atan2(y, x);
 
         // scale for current resolution length u
         for (int i = 0; i < res; i++) {
@@ -142,7 +147,7 @@ public LatLng hex2dToGeo(int face, int res, boolean substrate) {
         r *= Constants.RES0_U_GNOMONIC;
 
         // perform inverse gnomonic scaling of r
-        r = Math.atan(r);
+        r = FastMath.atan(r);
 
         // adjust theta for Class III
         // if a substrate grid, then it's already been adjusted for Class III
@@ -153,7 +158,7 @@ public LatLng hex2dToGeo(int face, int res, boolean substrate) {
         // find theta as an azimuth
         theta = posAngleRads(faceAxesAzRadsCII[face][0] - theta);
         // now find the point at (r,theta) from the face center
-        return geoAzDistanceRads(faceCenterGeo[face], theta, r);
+        return Vec3d.faceCenterPoint[face].geoAzDistanceRads(theta, r);
     }
 
     /**
@@ -162,13 +167,13 @@ public LatLng hex2dToGeo(int face, int res, boolean substrate) {
      *
      */
     static CoordIJK hex2dToCoordIJK(double x, double y) {
-        double a1, a2;
-        double x1, x2;
-        int m1, m2;
-        double r1, r2;
+        final double a1, a2;
+        final double x1, x2;
+        final int m1, m2;
+        final double r1, r2;
 
         // quantize into the ij system and then normalize
-        int k = 0;
+        final int k = 0;
         int i;
         int j;
 
@@ -194,17 +199,17 @@ static CoordIJK hex2dToCoordIJK(double x, double y) {
                     j = m2;
                 } else {
                     i = m1;
-                    j = m2 + 1;
+                    j = Math.incrementExact(m2);
                 }
             } else {
                 if (r2 < (1.0 - r1)) {
                     j = m2;
                 } else {
-                    j = m2 + 1;
+                    j = Math.incrementExact(m2);
                 }
 
                 if ((1.0 - r1) <= r2 && r2 < (2.0 * r1)) {
-                    i = m1 + 1;
+                    i = Math.incrementExact(m1);
                 } else {
                     i = m1;
                 }
@@ -214,21 +219,21 @@ static CoordIJK hex2dToCoordIJK(double x, double y) {
                 if (r2 < (1.0 - r1)) {
                     j = m2;
                 } else {
-                    j = m2 + 1;
+                    j = Math.addExact(m2, 1);
                 }
 
                 if ((2.0 * r1 - 1.0) < r2 && r2 < (1.0 - r1)) {
                     i = m1;
                 } else {
-                    i = m1 + 1;
+                    i = Math.incrementExact(m1);
                 }
             } else {
                 if (r2 < (r1 / 2.0)) {
-                    i = m1 + 1;
+                    i = Math.incrementExact(m1);
                     j = m2;
                 } else {
-                    i = m1 + 1;
-                    j = m2 + 1;
+                    i = Math.incrementExact(m1);
+                    j = Math.incrementExact(m2);
                 }
             }
         }
@@ -238,25 +243,29 @@ static CoordIJK hex2dToCoordIJK(double x, double y) {
         if (x < 0.0) {
             if ((j % 2) == 0)  // even
             {
-                int axisi = j / 2;
-                int diff = i - axisi;
-                i = i - 2 * diff;
+                final int axisi = j / 2;
+                final int diff = Math.subtractExact(i, axisi);
+                i = Math.subtractExact(i, Math.multiplyExact(2, diff));
             } else {
-                int axisi = (j + 1) / 2;
-                int diff = i - axisi;
-                i = i - (2 * diff + 1);
+                final int axisi = Math.addExact(j, 1) / 2;
+                final int diff = Math.subtractExact(i, axisi);
+                i = Math.subtractExact(i, Math.addExact(Math.multiplyExact(2, diff), 1));
             }
         }
 
         if (y < 0.0) {
-            i = i - (2 * j + 1) / 2;
-            j = -1 * j;
+            i = Math.subtractExact(i, Math.addExact(Math.multiplyExact(2, j), 1) / 2);
+            j = Math.multiplyExact(-1, j);
         }
         final CoordIJK coordIJK = new CoordIJK(i, j, k);
         coordIJK.ijkNormalize();
         return coordIJK;
     }
 
+    public boolean numericallyIdentical(Vec2d vec2d) {
+        return Math.abs(vec2d.x - x) < VEC2D_RESOLUTION && Math.abs(vec2d.y - y) < VEC2D_RESOLUTION;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -280,25 +289,14 @@ public int hashCode() {
      * @param p3 The second endpoint of the second line.
      */
     public static Vec2d v2dIntersect(Vec2d p0, Vec2d p1, Vec2d p2, Vec2d p3) {
-        double[] s1 = new double[2], s2 = new double[2];
-        s1[0] = p1.x - p0.x;
-        s1[1] = p1.y - p0.y;
-        s2[0] = p3.x - p2.x;
-        s2[1] = p3.y - p2.y;
-
-        float t;
-        t = (float) ((s2[0] * (p0.y - p2.y) - s2[1] * (p0.x - p2.x)) / (-s2[0] * s1[1] + s1[0] * s2[1]));
+        final double s1x = p1.x - p0.x;
+        final double s1y = p1.y - p0.y;
+        final double s2x = p3.x - p2.x;
+        final double s2y = p3.y - p2.y;
 
-        return new Vec2d(p0.x + (t * s1[0]), p0.y + (t * s1[1]));
-    }
+        final double t = ((s2x * (p0.y - p2.y) - s2y * (p0.x - p2.x)) / (-s2x * s1y + s1x * s2y));
 
-    /**
-     * Calculates the magnitude of a 2D cartesian vector.
-     *
-     * @return The magnitude of the vector.
-     */
-    private double v2dMag() {
-        return Math.sqrt(x * x + y * y);
+        return new Vec2d(p0.x + (t * s1x), p0.y + (t * s1y));
     }
 
     /**
@@ -309,106 +307,11 @@ private double v2dMag() {
      */
     static double posAngleRads(double rads) {
         if (rads < 0.0) {
-            return rads + M_2PI;
-        } else if (rads >= M_2PI) {
-            return rads - M_2PI;
+            return rads + Constants.M_2PI;
+        } else if (rads >= Constants.M_2PI) {
+            return rads - Constants.M_2PI;
         } else {
             return rads;
         }
     }
-
-    /**
-     * Computes the point on the sphere a specified azimuth and distance from
-     * another point.
-     *
-     * @param p1       The first spherical coordinates.
-     * @param az       The desired azimuth from p1.
-     * @param distance The desired distance from p1, must be non-negative.
-     *                 p1.
-     */
-    private static LatLng geoAzDistanceRads(LatLng p1, double az, double distance) {
-        if (distance < Constants.EPSILON) {
-            return p1;
-        }
-
-        double sinlat, sinlng, coslng;
-
-        az = posAngleRads(az);
-
-        double lat, lon;
-
-        // check for due north/south azimuth
-        if (az < Constants.EPSILON || Math.abs(az - M_PI) < Constants.EPSILON) {
-            if (az < Constants.EPSILON) {// due north
-                lat = p1.getLatRad() + distance;
-            } else { // due south
-                lat = p1.getLatRad() - distance;
-            }
-            if (Math.abs(lat - M_PI_2) < Constants.EPSILON) { // north pole
-                lat = M_PI_2;
-                lon = 0.0;
-            } else if (Math.abs(lat + M_PI_2) < Constants.EPSILON) { // south pole
-                lat = -M_PI_2;
-                lon = 0.0;
-            } else {
-                lon = constrainLng(p1.getLonRad());
-            }
-        } else { // not due north or south
-            final double sinDistance = Math.sin(distance);
-            final double cosDistance = Math.cos(distance);
-            final double sinP1Lat = Math.sin(p1.getLatRad());
-            final double cosP1Lat = Math.cos(p1.getLatRad());
-            sinlat = sinP1Lat * cosDistance + cosP1Lat * sinDistance * Math.cos(az);
-            if (sinlat > 1.0) {
-                sinlat = 1.0;
-            }
-            if (sinlat < -1.0) {
-                sinlat = -1.0;
-            }
-            lat = Math.asin(sinlat);
-            if (Math.abs(lat - M_PI_2) < Constants.EPSILON)  // north pole
-            {
-                lat = M_PI_2;
-                lon = 0.0;
-            } else if (Math.abs(lat + M_PI_2) < Constants.EPSILON)  // south pole
-            {
-                lat = -M_PI_2;
-                lon = 0.0;
-            } else {
-                final double cosLat = Math.cos(lat);
-                sinlng = Math.sin(az) * sinDistance / cosLat;
-                coslng = (cosDistance - sinP1Lat * Math.sin(lat)) / cosP1Lat / cosLat;
-                if (sinlng > 1.0) {
-                    sinlng = 1.0;
-                }
-                if (sinlng < -1.0) {
-                    sinlng = -1.0;
-                }
-                if (coslng > 1.0) {
-                    coslng = 1.0;
-                }
-                if (coslng < -1.0) {
-                    coslng = -1.0;
-                }
-                lon = constrainLng(p1.getLonRad() + Math.atan2(sinlng, coslng));
-            }
-        }
-        return new LatLng(lat, lon);
-    }
-
-    /**
-     * constrainLng makes sure longitudes are in the proper bounds
-     *
-     * @param lng The origin lng value
-     * @return The corrected lng value
-     */
-    private static double constrainLng(double lng) {
-        while (lng > M_PI) {
-            lng = lng - (2 * M_PI);
-        }
-        while (lng < -M_PI) {
-            lng = lng + (2 * M_PI);
-        }
-        return lng;
-    }
 }
diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/Vec3d.java b/libs/h3/src/main/java/org/elasticsearch/h3/Vec3d.java
index 814f0afa8d78..c5c4f8975597 100644
--- a/libs/h3/src/main/java/org/elasticsearch/h3/Vec3d.java
+++ b/libs/h3/src/main/java/org/elasticsearch/h3/Vec3d.java
@@ -81,10 +81,10 @@ private double pointSquareDist(double x, double y, double z) {
      * @return The H3 index.
      */
     static long geoToH3(int res, double lat, double lon) {
-        final double cosLat = Math.cos(lat);
-        final double z = Math.sin(lat);
-        final double x = Math.cos(lon) * cosLat;
-        final double y = Math.sin(lon) * cosLat;
+        final double cosLat = FastMath.cos(lat);
+        final double z = FastMath.sin(lat);
+        final double x = FastMath.cos(lon) * cosLat;
+        final double y = FastMath.sin(lon) * cosLat;
         // determine the icosahedron face
         int face = 0;
         double sqd = Vec3d.faceCenterPoint[0].pointSquareDist(x, y, z);
@@ -96,7 +96,7 @@ static long geoToH3(int res, double lat, double lon) {
             }
         }
         // cos(r) = 1 - 2 * sin^2(r/2) = 1 - 2 * (sqd / 4) = 1 - sqd/2
-        double r = Math.acos(1 - sqd / 2);
+        double r = FastMath.acos(1 - sqd / 2);
 
         if (r < Constants.EPSILON) {
             return FaceIJK.faceIjkToH3(res, face, new CoordIJK(0, 0, 0));
@@ -113,7 +113,7 @@ static long geoToH3(int res, double lat, double lon) {
         }
 
         // perform gnomonic scaling of r
-        r = Math.tan(r);
+        r = FastMath.tan(r);
 
         // scale for current resolution length u
         r /= Constants.RES0_U_GNOMONIC;
@@ -122,7 +122,7 @@ static long geoToH3(int res, double lat, double lon) {
         }
         // we now have (r, theta) in hex2d with theta ccw from x-axes
         // convert to face and centered IJK coordinates
-        return FaceIJK.faceIjkToH3(res, face, Vec2d.hex2dToCoordIJK(r * Math.cos(theta), r * Math.sin(theta)));
+        return FaceIJK.faceIjkToH3(res, face, Vec2d.hex2dToCoordIJK(r * FastMath.cos(theta), r * FastMath.sin(theta)));
     }
 
     /**
@@ -164,7 +164,52 @@ private static double square(double x) {
         final double c1c2Z = c1X * c2Y - c1Y * c2X;
 
         final double sign = Math.signum(dotProduct(this.x, this.y, this.z, c1c2X, c1c2Y, c1c2Z));
-        return Math.atan2(sign * magnitude(c1c2X, c1c2Y, c1c2Z), dotProduct(c1X, c1Y, c1Z, c2X, c2Y, c2Z));
+        return FastMath.atan2(sign * magnitude(c1c2X, c1c2Y, c1c2Z), dotProduct(c1X, c1Y, c1Z, c2X, c2Y, c2Z));
+    }
+
+    /**
+     * Computes the point on the sphere with a specified azimuth and distance from
+     * this point.
+     *
+     * @param az       The desired azimuth.
+     * @param distance The desired distance.
+     * @return The LatLng point.
+     */
+    LatLng geoAzDistanceRads(double az, double distance) {
+        az = Vec2d.posAngleRads(az);
+        // from https://www.movable-type.co.uk/scripts/latlong-vectors.html
+        // N = {0,0,1} – vector representing north pole
+        // d̂e = N×a – east vector at a
+        // dn = a×de – north vector at a
+        // d = dn·cosθ + de·sinθ – direction vector in dir’n of θ
+        // b = a·cosδ + d·sinδ
+
+        // east direction vector @ n1 (Gade's k_e_E)
+        final double magnitude = magnitude(this.x, this.y, 0);
+        final double deX = -this.y / magnitude;
+        final double deY = this.x / magnitude;
+
+        // north direction vector @ n1 (Gade's (k_n_E)
+        final double dnX = -this.z * deY;
+        final double dnY = this.z * deX;
+        final double dnZ = this.x * deY - this.y * deX;
+
+        final double sinAz = FastMath.sin(az);
+        final double cosAz = FastMath.cos(az);
+        final double sinDistance = FastMath.sin(distance);
+        final double cosDistance = FastMath.cos(distance);
+
+        // direction vector @ n1 (≡ C×n1; C = great circle)
+        final double dX = dnX * cosAz + deX * sinAz;
+        final double dY = dnY * cosAz + deY * sinAz;
+        final double dZ = dnZ * cosAz;
+
+        // Gade's n_EB_E = component of n2 parallel to n1 + component of n2 perpendicular to n1
+        final double n2X = this.x * cosDistance + dX * sinDistance;
+        final double n2Y = this.y * cosDistance + dY * sinDistance;
+        final double n2Z = this.z * cosDistance + dZ * sinDistance;
+
+        return new LatLng(FastMath.asin(n2Z), FastMath.atan2(n2Y, n2X));
     }
 
     /**
diff --git a/libs/h3/src/test/java/org/elasticsearch/h3/AzimuthTests.java b/libs/h3/src/test/java/org/elasticsearch/h3/AzimuthTests.java
deleted file mode 100644
index 1f342d0cc43d..000000000000
--- a/libs/h3/src/test/java/org/elasticsearch/h3/AzimuthTests.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.h3;
-
-import org.apache.lucene.spatial3d.geom.GeoPoint;
-import org.apache.lucene.spatial3d.geom.PlanetModel;
-import org.apache.lucene.tests.geo.GeoTestUtil;
-import org.elasticsearch.test.ESTestCase;
-
-public class AzimuthTests extends ESTestCase {
-
-    public void testLatLonVec3d() {
-        final double lat = Math.toRadians(GeoTestUtil.nextLatitude());
-        final double lon = Math.toRadians(GeoTestUtil.nextLongitude());
-        final GeoPoint point = new GeoPoint(PlanetModel.SPHERE, lat, lon);
-        for (int i = 0; i < Vec3d.faceCenterPoint.length; i++) {
-            final double azVec3d = Vec3d.faceCenterPoint[i].geoAzimuthRads(point.x, point.y, point.z);
-            final double azVec2d = Vec2d.faceCenterGeo[i].geoAzimuthRads(point.getLatitude(), point.getLongitude());
-            assertEquals(azVec2d, azVec3d, 1e-14);
-        }
-    }
-}
diff --git a/libs/h3/src/test/java/org/elasticsearch/h3/CellBoundaryTests.java b/libs/h3/src/test/java/org/elasticsearch/h3/CellBoundaryTests.java
index 7101b0ced03b..903e4ed40ec1 100644
--- a/libs/h3/src/test/java/org/elasticsearch/h3/CellBoundaryTests.java
+++ b/libs/h3/src/test/java/org/elasticsearch/h3/CellBoundaryTests.java
@@ -18,6 +18,8 @@
  */
 package org.elasticsearch.h3;
 
+import org.apache.lucene.geo.GeoEncodingUtils;
+import org.apache.lucene.tests.geo.GeoTestUtil;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.BufferedReader;
@@ -30,6 +32,9 @@
 import java.util.StringTokenizer;
 import java.util.zip.GZIPInputStream;
 
+import static org.hamcrest.Matchers.either;
+import static org.hamcrest.Matchers.equalTo;
+
 public class CellBoundaryTests extends ESTestCase {
 
     public void testRes0() throws Exception {
@@ -171,8 +176,46 @@ private void processOne(String h3Address, BufferedReader reader) throws IOExcept
         CellBoundary boundary = H3.h3ToGeoBoundary(h3Address);
         assert boundary.numPoints() == points.size();
         for (int i = 0; i < boundary.numPoints(); i++) {
-            assertEquals(h3Address, points.get(i)[0], boundary.getLatLon(i).getLatDeg(), 1e-8);
-            assertEquals(h3Address, points.get(i)[1], boundary.getLatLon(i).getLonDeg(), 1e-8);
+            assertEquals(h3Address, points.get(i)[0], boundary.getLatLon(i).getLatDeg(), 5e-7);
+            assertEquals(h3Address, points.get(i)[1], boundary.getLatLon(i).getLonDeg(), 5e-7);
+        }
+    }
+
+    public void testNumericEquivalentSharedBoundary() {
+        // we consider boundaries numerical equivalent if after encoded them using lucene, they resolve to the same number.
+        long h3 = H3.geoToH3(GeoTestUtil.nextLatitude(), GeoTestUtil.nextLongitude(), randomIntBetween(0, 15));
+        CellBoundary boundary = H3.h3ToGeoBoundary(h3);
+        for (long r : H3.hexRing(h3)) {
+            int count = 0;
+            CellBoundary ringBoundary = H3.h3ToGeoBoundary(r);
+            for (int i = 0; i < boundary.numPoints(); i++) {
+                LatLng latLng1 = boundary.getLatLon(i % boundary.numPoints());
+                LatLng latLng2 = boundary.getLatLon((i + 1) % boundary.numPoints());
+                int lon1 = GeoEncodingUtils.encodeLongitude(latLng1.getLonDeg());
+                int lat1 = GeoEncodingUtils.encodeLatitude(latLng1.getLatDeg());
+                int lon2 = GeoEncodingUtils.encodeLongitude(latLng2.getLonDeg());
+                int lat2 = GeoEncodingUtils.encodeLatitude(latLng2.getLatDeg());
+                if (isSharedBoundary(lon1, lat1, lon2, lat2, ringBoundary)) {
+                    count++;
+                }
+            }
+            assertThat("For cell " + H3.h3ToString(h3), count, either(equalTo(1)).or(equalTo(2)));
+        }
+    }
+
+    private boolean isSharedBoundary(int clon1, int clat1, int clon2, int clat2, CellBoundary boundary) {
+        for (int i = 0; i < boundary.numPoints(); i++) {
+            LatLng latLng1 = boundary.getLatLon(i % boundary.numPoints());
+            LatLng latLng2 = boundary.getLatLon((i + 1) % boundary.numPoints());
+            int lon1 = GeoEncodingUtils.encodeLongitude(latLng1.getLonDeg());
+            int lat1 = GeoEncodingUtils.encodeLatitude(latLng1.getLatDeg());
+            int lon2 = GeoEncodingUtils.encodeLongitude(latLng2.getLonDeg());
+            int lat2 = GeoEncodingUtils.encodeLatitude(latLng2.getLatDeg());
+            // edges are in opposite directions.
+            if (clon1 == lon2 & clat1 == lat2 && clon2 == lon1 && clat2 == lat1) {
+                return true;
+            }
         }
+        return false;
     }
 }
diff --git a/libs/h3/src/test/java/org/elasticsearch/h3/FastMathTests.java b/libs/h3/src/test/java/org/elasticsearch/h3/FastMathTests.java
new file mode 100644
index 000000000000..76b15188a182
--- /dev/null
+++ b/libs/h3/src/test/java/org/elasticsearch/h3/FastMathTests.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.h3;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.function.DoubleSupplier;
+import java.util.function.DoubleUnaryOperator;
+
+public class FastMathTests extends ESTestCase {
+
+    // accuracy for cos(x)
+    static double COS_DELTA = 1E-15;
+    // accuracy for sin(x)
+    static double SIN_DELTA = 1E-15;
+    // accuracy for asin(x)
+    static double ASIN_DELTA = 1E-14;
+    // accuracy for acos(x)
+    static double ACOS_DELTA = 1E-14;
+    // accuracy for tan(x)
+    static double TAN_DELTA = 1E-14;
+    // accuracy for atan(x)
+    static double ATAN_DELTA = 1E-14;
+    // accuracy for atan2(x)
+    static double ATAN2_DELTA = 1E-14;
+
+    public void testSin() {
+        doTest(Math::sin, FastMath::sin, d -> SIN_DELTA, () -> randomDoubleBetween(-2 * Math.PI, 2 * Math.PI, true));
+    }
+
+    public void testCos() {
+        doTest(Math::cos, FastMath::cos, d -> COS_DELTA, () -> randomDoubleBetween(-2 * Math.PI, 2 * Math.PI, true));
+    }
+
+    public void testTan() {
+        doTest(
+            Math::tan,
+            FastMath::tan,
+            d -> Math.max(TAN_DELTA, Math.abs(Math.tan(d)) * TAN_DELTA),
+            () -> randomDoubleBetween(-2 * Math.PI, 2 * Math.PI, true)
+        );
+    }
+
+    public void testAsin() {
+        doTest(Math::asin, FastMath::asin, d -> ASIN_DELTA, () -> randomDoubleBetween(-2, 2, true));
+    }
+
+    public void testAcos() {
+        doTest(Math::acos, FastMath::acos, d -> ACOS_DELTA, () -> randomDoubleBetween(-2, 2, true));
+    }
+
+    public void testAtan() {
+        doTest(
+            Math::atan,
+            FastMath::atan,
+            d -> ATAN_DELTA,
+            () -> randomDoubleBetween(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, true)
+        );
+    }
+
+    private void doTest(DoubleUnaryOperator expected, DoubleUnaryOperator actual, DoubleUnaryOperator delta, DoubleSupplier supplier) {
+        assertEquals(expected.applyAsDouble(Double.NaN), actual.applyAsDouble(Double.NaN), delta.applyAsDouble(Double.NaN));
+        assertEquals(
+            expected.applyAsDouble(Double.NEGATIVE_INFINITY),
+            actual.applyAsDouble(Double.NEGATIVE_INFINITY),
+            delta.applyAsDouble(Double.POSITIVE_INFINITY)
+        );
+        assertEquals(
+            expected.applyAsDouble(Double.POSITIVE_INFINITY),
+            actual.applyAsDouble(Double.POSITIVE_INFINITY),
+            delta.applyAsDouble(Double.POSITIVE_INFINITY)
+        );
+        assertEquals(
+            expected.applyAsDouble(Double.MAX_VALUE),
+            actual.applyAsDouble(Double.MAX_VALUE),
+            delta.applyAsDouble(Double.MAX_VALUE)
+        );
+        assertEquals(
+            expected.applyAsDouble(Double.MIN_VALUE),
+            actual.applyAsDouble(Double.MIN_VALUE),
+            delta.applyAsDouble(Double.MIN_VALUE)
+        );
+        assertEquals(expected.applyAsDouble(0), actual.applyAsDouble(0), delta.applyAsDouble(0));
+        for (int i = 0; i < 10000; i++) {
+            double d = supplier.getAsDouble();
+            assertEquals(expected.applyAsDouble(d), actual.applyAsDouble(d), delta.applyAsDouble(d));
+        }
+    }
+
+    public void testAtan2() {
+        assertEquals(Math.atan2(Double.NaN, Double.NaN), FastMath.atan2(Double.NaN, Double.NaN), ATAN2_DELTA);
+        assertEquals(
+            Math.atan2(Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY),
+            FastMath.atan2(Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY),
+            ATAN2_DELTA
+        );
+        assertEquals(
+            Math.atan2(Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY),
+            FastMath.atan2(Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY),
+            ATAN2_DELTA
+        );
+        assertEquals(Math.atan2(Double.MAX_VALUE, Double.MAX_VALUE), FastMath.atan2(Double.MAX_VALUE, Double.MAX_VALUE), ATAN2_DELTA);
+        assertEquals(Math.atan2(Double.MIN_VALUE, Double.MIN_VALUE), FastMath.atan2(Double.MIN_VALUE, Double.MIN_VALUE), ATAN2_DELTA);
+        assertEquals(Math.atan2(0, 0), FastMath.atan2(0, 0), ATAN2_DELTA);
+        for (int i = 0; i < 10000; i++) {
+            double x = randomDoubleBetween(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, true);
+            double y = randomDoubleBetween(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, true);
+            assertEquals(Math.atan2(x, y), FastMath.atan2(x, y), ATAN2_DELTA);
+        }
+    }
+}
diff --git a/libs/h3/src/test/java/org/elasticsearch/h3/GeoToH3Tests.java b/libs/h3/src/test/java/org/elasticsearch/h3/GeoToH3Tests.java
index 61289c50dbb0..cb7d416a5a9d 100644
--- a/libs/h3/src/test/java/org/elasticsearch/h3/GeoToH3Tests.java
+++ b/libs/h3/src/test/java/org/elasticsearch/h3/GeoToH3Tests.java
@@ -55,4 +55,16 @@ private GeoPolygon getGeoPolygon(String h3Address) {
         }
         return GeoPolygonFactory.makeGeoPolygon(PlanetModel.SPHERE, points);
     }
+
+    public void testNorthPoleCells() {
+        for (int res = 0; res <= H3.MAX_H3_RES; res++) {
+            assertEquals(H3.northPolarH3Address(res), H3.geoToH3Address(90, GeoTestUtil.nextLongitude(), res));
+        }
+    }
+
+    public void testSouthPoleCells() {
+        for (int res = 0; res <= H3.MAX_H3_RES; res++) {
+            assertEquals(H3.southPolarH3Address(res), H3.geoToH3Address(-90, GeoTestUtil.nextLongitude(), res));
+        }
+    }
 }
diff --git a/libs/h3/src/test/java/org/elasticsearch/h3/HexRingTests.java b/libs/h3/src/test/java/org/elasticsearch/h3/HexRingTests.java
index 486a5be8d621..8fe5c6206fff 100644
--- a/libs/h3/src/test/java/org/elasticsearch/h3/HexRingTests.java
+++ b/libs/h3/src/test/java/org/elasticsearch/h3/HexRingTests.java
@@ -25,6 +25,15 @@
 
 public class HexRingTests extends ESTestCase {
 
+    public void testInvalidHexRingPos() {
+        long h3 = H3.geoToH3(GeoTestUtil.nextLatitude(), GeoTestUtil.nextLongitude(), randomIntBetween(0, H3.MAX_H3_RES));
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> H3.hexRingPosToH3(h3, -1));
+        assertEquals(ex.getMessage(), "invalid ring position");
+        int pos = H3.isPentagon(h3) ? 5 : 6;
+        ex = expectThrows(IllegalArgumentException.class, () -> H3.hexRingPosToH3(h3, pos));
+        assertEquals(ex.getMessage(), "invalid ring position");
+    }
+
     public void testHexRing() {
         for (int i = 0; i < 500; i++) {
             double lat = GeoTestUtil.nextLatitude();
diff --git a/libs/h3/src/test/java/org/elasticsearch/h3/LatLngTests.java b/libs/h3/src/test/java/org/elasticsearch/h3/LatLngTests.java
new file mode 100644
index 000000000000..12767af2127b
--- /dev/null
+++ b/libs/h3/src/test/java/org/elasticsearch/h3/LatLngTests.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.h3;
+
+import org.apache.lucene.spatial3d.geom.GeoPoint;
+import org.apache.lucene.spatial3d.geom.LatLonBounds;
+import org.apache.lucene.spatial3d.geom.Plane;
+import org.apache.lucene.spatial3d.geom.PlanetModel;
+import org.apache.lucene.spatial3d.geom.SidedPlane;
+import org.apache.lucene.tests.geo.GeoTestUtil;
+import org.elasticsearch.test.ESTestCase;
+
+public class LatLngTests extends ESTestCase {
+
+    public void testLatLonAzimuthRads() {
+        final GeoPoint point = safePoint();
+        for (int i = 0; i < Vec3d.faceCenterPoint.length; i++) {
+            final double azVec3d = Vec3d.faceCenterPoint[i].geoAzimuthRads(point.x, point.y, point.z);
+            final double azVec2d = Vec2d.faceCenterGeo[i].geoAzimuthRads(point.getLatitude(), point.getLongitude());
+            assertEquals("Face " + i, azVec2d, azVec3d, 1e-12);
+
+        }
+    }
+
+    public void testLatLonAzDistanceRads() {
+        final double az = randomDoubleBetween(-2 * Math.PI, 2 * Math.PI, true);
+        final double distance = randomDoubleBetween(-Math.PI, Math.PI / 2, true);
+        for (int i = 0; i < Vec3d.faceCenterPoint.length; i++) {
+            final LatLng latLng3d = Vec3d.faceCenterPoint[i].geoAzDistanceRads(az, distance);
+            final LatLng latLng2d = Vec2d.faceCenterGeo[i].geoAzDistanceRads(az, distance);
+            assertTrue("Face " + i, latLng2d.isNumericallyIdentical(latLng3d));
+        }
+    }
+
+    /**
+     * Face 19 gives -180 vs 180 for azimuth when Latitude=-90 and Longitude is between 98 and 102.
+     * So we just exclude lat=-90 from the test to avoid this odd edge case.
+     */
+    private GeoPoint safePoint() {
+        GeoPoint point;
+        do {
+            final double lat = Math.toRadians(GeoTestUtil.nextLatitude());
+            final double lon = Math.toRadians(GeoTestUtil.nextLongitude());
+            point = new GeoPoint(PlanetModel.SPHERE, lat, lon);
+        } while (point.getLatitude() == -Math.PI / 2);
+        return point;
+    }
+
+    public void testGreatCircleMaxMinLatitude() {
+        for (int i = 0; i < 10; i++) {
+            final GeoPoint point1 = safePoint();
+            final GeoPoint point2 = safePoint();
+            final LatLng latLng1 = new LatLng(point1.getLatitude(), point1.getLongitude());
+            final LatLng latLng2 = new LatLng(point2.getLatitude(), point2.getLongitude());
+            final LatLonBounds bounds = getBounds(point1, point2);
+            assertEquals(bounds.getMaxLatitude(), latLng1.greatCircleMaxLatitude(latLng2), 1e-7);
+            assertEquals(bounds.getMinLatitude(), latLng1.greatCircleMinLatitude(latLng2), 1e-7);
+        }
+    }
+
+    private LatLonBounds getBounds(final GeoPoint point1, final GeoPoint point2) {
+        final LatLonBounds bounds = new LatLonBounds();
+        bounds.addPoint(point1);
+        bounds.addPoint(point2);
+        if (point1.isNumericallyIdentical(point2) == false) {
+            final Plane plane = new Plane(point1, point2);
+            bounds.addPlane(PlanetModel.SPHERE, plane, new SidedPlane(point1, plane, point2), new SidedPlane(point2, point1, plane));
+        }
+        return bounds;
+    }
+
+    public void testEqualsAndHashCode() {
+        final LatLng latLng = new LatLng(0, 0);
+        {
+            LatLng otherLatLng = new LatLng(1, 1);
+            assertNotEquals(latLng, otherLatLng);
+            assertNotEquals(latLng.hashCode(), otherLatLng.hashCode());
+        }
+        {
+            LatLng otherLatLng = new LatLng(0, 0);
+            assertEquals(latLng, otherLatLng);
+            assertEquals(latLng.hashCode(), otherLatLng.hashCode());
+        }
+    }
+}
diff --git a/libs/h3/src/test/java/org/elasticsearch/h3/ParentChildNavigationTests.java b/libs/h3/src/test/java/org/elasticsearch/h3/ParentChildNavigationTests.java
index 4e8ed3e4b646..11d5a01b2d4e 100644
--- a/libs/h3/src/test/java/org/elasticsearch/h3/ParentChildNavigationTests.java
+++ b/libs/h3/src/test/java/org/elasticsearch/h3/ParentChildNavigationTests.java
@@ -20,10 +20,70 @@
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 
+import org.apache.lucene.geo.Point;
+import org.apache.lucene.spatial3d.geom.GeoPoint;
+import org.apache.lucene.spatial3d.geom.GeoPolygon;
+import org.apache.lucene.spatial3d.geom.GeoPolygonFactory;
+import org.apache.lucene.spatial3d.geom.PlanetModel;
+import org.apache.lucene.tests.geo.GeoTestUtil;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matchers;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
 
 public class ParentChildNavigationTests extends ESTestCase {
 
+    public void testChildrenSize() {
+        Point point = GeoTestUtil.nextPoint();
+        int res = randomInt(H3.MAX_H3_RES - 1);
+        String h3Address = H3.geoToH3Address(point.getLat(), point.getLon(), res);
+        // check invalid resolutions
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> H3.h3ToChildrenSize(h3Address, res));
+        assertThat(ex.getMessage(), Matchers.containsString("Invalid child resolution"));
+        ex = expectThrows(IllegalArgumentException.class, () -> H3.h3ToChildrenSize(h3Address, H3.MAX_H3_RES + 1));
+        assertThat(ex.getMessage(), Matchers.containsString("Invalid child resolution"));
+        ex = expectThrows(
+            IllegalArgumentException.class,
+            () -> H3.h3ToChildrenSize(H3.geoToH3(point.getLat(), point.getLon(), H3.MAX_H3_RES))
+        );
+        assertThat(ex.getMessage(), Matchers.containsString("Invalid child resolution"));
+        // check methods gives same answer
+        assertEquals(H3.h3ToChildrenSize(h3Address), H3.h3ToChildrenSize(h3Address, res + 1));
+        // check against brute force counting
+        int childrenRes = Math.min(H3.MAX_H3_RES, res + randomIntBetween(2, 7));
+        long numChildren = H3.h3ToChildrenSize(h3Address, childrenRes);
+        assertEquals(numChildren(h3Address, childrenRes), numChildren);
+    }
+
+    private long numChildren(String h3Address, int finalRes) {
+        if (H3.getResolution(h3Address) == finalRes) {
+            return 1;
+        }
+        long result = 0;
+        for (int i = 0; i < H3.h3ToChildrenSize(h3Address); i++) {
+            result += numChildren(H3.childPosToH3(h3Address, i), finalRes);
+        }
+        return result;
+    }
+
+    public void testNoChildrenIntersectingSize() {
+        Point point = GeoTestUtil.nextPoint();
+        int res = randomInt(H3.MAX_H3_RES - 1);
+        String h3Address = H3.geoToH3Address(point.getLat(), point.getLon(), res);
+        // check invalid resolutions
+        IllegalArgumentException ex = expectThrows(
+            IllegalArgumentException.class,
+            () -> H3.h3ToNotIntersectingChildrenSize(H3.geoToH3(point.getLat(), point.getLon(), H3.MAX_H3_RES))
+        );
+        assertThat(ex.getMessage(), Matchers.containsString("Invalid child resolution"));
+        // check against brute force counting
+        long numChildren = H3.h3ToNotIntersectingChildrenSize(h3Address);
+        assertEquals(H3.h3ToNoChildrenIntersecting(h3Address).length, numChildren);
+    }
+
     public void testParentChild() {
         String[] h3Addresses = H3.getStringRes0Cells();
         String h3Address = RandomPicks.randomFrom(random(), h3Addresses);
@@ -31,6 +91,9 @@ public void testParentChild() {
         values[0] = h3Address;
         for (int i = 1; i < H3.MAX_H3_RES; i++) {
             h3Addresses = H3.h3ToChildren(h3Address);
+            // check all elements are unique
+            Set<String> mySet = Sets.newHashSet(h3Addresses);
+            assertEquals(mySet.size(), h3Addresses.length);
             h3Address = RandomPicks.randomFrom(random(), h3Addresses);
             values[i] = h3Address;
         }
@@ -44,11 +107,11 @@ public void testParentChild() {
 
     public void testHexRing() {
         String[] h3Addresses = H3.getStringRes0Cells();
-        String h3Address = RandomPicks.randomFrom(random(), h3Addresses);
         for (int i = 1; i < H3.MAX_H3_RES; i++) {
+            String h3Address = RandomPicks.randomFrom(random(), h3Addresses);
+            assertEquals(i - 1, H3.getResolution(h3Address));
             h3Addresses = H3.h3ToChildren(h3Address);
             assertHexRing(i, h3Address, h3Addresses);
-            h3Address = RandomPicks.randomFrom(random(), h3Addresses);
         }
     }
 
@@ -65,4 +128,55 @@ private void assertHexRing(int res, String h3Address, String[] children) {
             assertEquals(children[i], ring[positions[i - 1]]);
         }
     }
+
+    public void testNoChildrenIntersecting() {
+        String[] h3Addresses = H3.getStringRes0Cells();
+        String h3Address = RandomPicks.randomFrom(random(), h3Addresses);
+        for (int i = 1; i <= H3.MAX_H3_RES; i++) {
+            h3Addresses = H3.h3ToChildren(h3Address);
+            assertIntersectingChildren(h3Address, h3Addresses);
+            h3Address = RandomPicks.randomFrom(random(), h3Addresses);
+        }
+    }
+
+    private void assertIntersectingChildren(String h3Address, String[] children) {
+        int size = H3.h3ToNotIntersectingChildrenSize(h3Address);
+        for (int i = 0; i < size; i++) {
+            GeoPolygon p = getGeoPolygon(H3.noChildIntersectingPosToH3(h3Address, i));
+            int intersections = 0;
+            for (String o : children) {
+                if (p.intersects(getGeoPolygon(o))) {
+                    intersections++;
+                }
+            }
+            assertEquals(2, intersections);
+        }
+    }
+
+    private GeoPolygon getGeoPolygon(String h3Address) {
+        CellBoundary cellBoundary = H3.h3ToGeoBoundary(h3Address);
+        List<GeoPoint> points = new ArrayList<>(cellBoundary.numPoints());
+        for (int i = 0; i < cellBoundary.numPoints(); i++) {
+            LatLng latLng = cellBoundary.getLatLon(i);
+            points.add(new GeoPoint(PlanetModel.SPHERE, latLng.getLatRad(), latLng.getLonRad()));
+        }
+        return GeoPolygonFactory.makeGeoPolygon(PlanetModel.SPHERE, points);
+    }
+
+    public void testHexRingPos() {
+        String[] h3Addresses = H3.getStringRes0Cells();
+        for (int i = 0; i < H3.MAX_H3_RES; i++) {
+            String h3Address = RandomPicks.randomFrom(random(), h3Addresses);
+            assertHexRing3(h3Address);
+            h3Addresses = H3.h3ToChildren(h3Address);
+        }
+    }
+
+    private void assertHexRing3(String h3Address) {
+        String[] ring = H3.hexRing(h3Address);
+        assertEquals(ring.length, H3.hexRingSize(h3Address));
+        for (int i = 0; i < H3.hexRingSize(h3Address); i++) {
+            assertEquals(ring[i], H3.hexRingPosToH3(h3Address, i));
+        }
+    }
 }
diff --git a/libs/logging/build.gradle b/libs/logging/build.gradle
index 3859c0b12ced..3004af029cb5 100644
--- a/libs/logging/build.gradle
+++ b/libs/logging/build.gradle
@@ -9,7 +9,6 @@
 apply plugin: 'elasticsearch.publish'
 apply plugin: 'elasticsearch.build'
 
-
 tasks.named("loggerUsageCheck").configure {enabled = false }
 
 dependencies {
diff --git a/libs/plugin-analysis-api/build.gradle b/libs/plugin-analysis-api/build.gradle
index 184295ebdcc5..bc9b305aa5a4 100644
--- a/libs/plugin-analysis-api/build.gradle
+++ b/libs/plugin-analysis-api/build.gradle
@@ -1,3 +1,5 @@
+import org.elasticsearch.gradle.internal.info.BuildParams
+
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
@@ -9,6 +11,7 @@
 apply plugin: 'elasticsearch.publish'
 apply plugin: 'elasticsearch.build'
 
+group = "org.elasticsearch.plugin"
 
 tasks.named("loggerUsageCheck").configure {enabled = false }
 
@@ -26,3 +29,13 @@ tasks.named('forbiddenApisMain').configure {
 tasks.named("dependencyLicenses").configure {
   mapping from: /lucene-.*/, to: 'lucene'
 }
+
+tasks.named("thirdPartyAudit").configure {
+  if (BuildParams.runtimeJavaVersion == JavaVersion.VERSION_20) {
+    ignoreMissingClasses(
+      // This class was removed in Java 20 but is only referenced by a class that requires preview features anyhow
+      // See: https://github.com/apache/lucene/pull/12042
+      'java.lang.foreign.MemorySession',
+    )
+  }
+}
diff --git a/libs/plugin-analysis-api/src/main/java/module-info.java b/libs/plugin-analysis-api/src/main/java/module-info.java
index 899a5cd58a14..4652e6b77f7c 100644
--- a/libs/plugin-analysis-api/src/main/java/module-info.java
+++ b/libs/plugin-analysis-api/src/main/java/module-info.java
@@ -6,9 +6,9 @@
  * Side Public License, v 1.
  */
 
-module org.elasticsearch.plugin.analysis.api {
+module org.elasticsearch.plugin.analysis {
     requires org.apache.lucene.core;
-    requires org.elasticsearch.plugin.api;
+    requires org.elasticsearch.plugin;
 
-    exports org.elasticsearch.plugin.analysis.api;
+    exports org.elasticsearch.plugin.analysis;
 }
diff --git a/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/api/AnalysisMode.java b/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/AnalysisMode.java
similarity index 98%
rename from libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/api/AnalysisMode.java
rename to libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/AnalysisMode.java
index f6a1e4ba70ca..5180b3168fcb 100644
--- a/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/api/AnalysisMode.java
+++ b/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/AnalysisMode.java
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.plugin.analysis.api;
+package org.elasticsearch.plugin.analysis;
 
 /**
  * Enum representing the mode in which token filters and analyzers are allowed to operate.
diff --git a/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/api/AnalyzerFactory.java b/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/AnalyzerFactory.java
similarity index 82%
rename from libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/api/AnalyzerFactory.java
rename to libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/AnalyzerFactory.java
index 625f753a46d2..dfda8f292910 100644
--- a/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/api/AnalyzerFactory.java
+++ b/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/AnalyzerFactory.java
@@ -6,11 +6,11 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.plugin.analysis.api;
+package org.elasticsearch.plugin.analysis;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.elasticsearch.plugin.api.Extensible;
-import org.elasticsearch.plugin.api.Nameable;
+import org.elasticsearch.plugin.Extensible;
+import org.elasticsearch.plugin.Nameable;
 
 /**
  * An analysis component used to create Analyzers.
diff --git a/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/api/CharFilterFactory.java b/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/CharFilterFactory.java
similarity index 86%
rename from libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/api/CharFilterFactory.java
rename to libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/CharFilterFactory.java
index 7095e32050cf..cb8cbeba01b2 100644
--- a/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/api/CharFilterFactory.java
+++ b/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/CharFilterFactory.java
@@ -6,10 +6,10 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.plugin.analysis.api;
+package org.elasticsearch.plugin.analysis;
 
-import org.elasticsearch.plugin.api.Extensible;
-import org.elasticsearch.plugin.api.Nameable;
+import org.elasticsearch.plugin.Extensible;
+import org.elasticsearch.plugin.Nameable;
 
 import java.io.Reader;
 
diff --git a/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/api/TokenFilterFactory.java b/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/TokenFilterFactory.java
similarity index 90%
rename from libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/api/TokenFilterFactory.java
rename to libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/TokenFilterFactory.java
index 119918bd5b21..671390f99539 100644
--- a/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/api/TokenFilterFactory.java
+++ b/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/TokenFilterFactory.java
@@ -6,11 +6,11 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.plugin.analysis.api;
+package org.elasticsearch.plugin.analysis;
 
 import org.apache.lucene.analysis.TokenStream;
-import org.elasticsearch.plugin.api.Extensible;
-import org.elasticsearch.plugin.api.Nameable;
+import org.elasticsearch.plugin.Extensible;
+import org.elasticsearch.plugin.Nameable;
 
 /**
  * An analysis component used to create token filters.
diff --git a/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/api/TokenizerFactory.java b/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/TokenizerFactory.java
similarity index 81%
rename from libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/api/TokenizerFactory.java
rename to libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/TokenizerFactory.java
index b6a0e69d5223..9222c296ebd9 100644
--- a/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/api/TokenizerFactory.java
+++ b/libs/plugin-analysis-api/src/main/java/org/elasticsearch/plugin/analysis/TokenizerFactory.java
@@ -6,11 +6,11 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.plugin.analysis.api;
+package org.elasticsearch.plugin.analysis;
 
 import org.apache.lucene.analysis.Tokenizer;
-import org.elasticsearch.plugin.api.Extensible;
-import org.elasticsearch.plugin.api.Nameable;
+import org.elasticsearch.plugin.Extensible;
+import org.elasticsearch.plugin.Nameable;
 
 /**
  * An analysis component used to create tokenizers.
diff --git a/libs/plugin-api/build.gradle b/libs/plugin-api/build.gradle
index 8ca0f53e7547..8dfe7ce9c793 100644
--- a/libs/plugin-api/build.gradle
+++ b/libs/plugin-api/build.gradle
@@ -9,6 +9,7 @@
 apply plugin: 'elasticsearch.publish'
 apply plugin: 'elasticsearch.build'
 
+group = "org.elasticsearch.plugin"
 
 tasks.named("loggerUsageCheck").configure {enabled = false }
 
diff --git a/libs/plugin-api/src/main/java/module-info.java b/libs/plugin-api/src/main/java/module-info.java
index 28a6c9319cef..03946a839f7b 100644
--- a/libs/plugin-api/src/main/java/module-info.java
+++ b/libs/plugin-api/src/main/java/module-info.java
@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
-module org.elasticsearch.plugin.api {
-    exports org.elasticsearch.plugin.api;
+module org.elasticsearch.plugin {
+    exports org.elasticsearch.plugin;
+    exports org.elasticsearch.plugin.settings;
 }
diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/Extensible.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/Extensible.java
new file mode 100644
index 000000000000..feb65d625e48
--- /dev/null
+++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/Extensible.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugin;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.TYPE;
+
+/**
+ * Marker for things that can be loaded by component loader.
+ *
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(value = { TYPE })
+public @interface Extensible {
+}
diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/Inject.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/Inject.java
new file mode 100644
index 000000000000..c9b1f4b3c50a
--- /dev/null
+++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/Inject.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugin;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * An annotation used to mark constructor to inject plugin dependencies iee. settings.
+ * A constructor parameter has to be an interface marked with appropriate annotation (i.e AnalysisSetting)
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.CONSTRUCTOR)
+public @interface Inject {
+}
diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/api/Nameable.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/Nameable.java
similarity index 96%
rename from libs/plugin-api/src/main/java/org/elasticsearch/plugin/api/Nameable.java
rename to libs/plugin-api/src/main/java/org/elasticsearch/plugin/Nameable.java
index 79b9c5298ffc..373aa94b7b8c 100644
--- a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/api/Nameable.java
+++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/Nameable.java
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.plugin.api;
+package org.elasticsearch.plugin;
 
 /**
  * A named plugin component. Components with a name can be registered and fetched under a name given in
diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/NamedComponent.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/NamedComponent.java
new file mode 100644
index 000000000000..44f8e6dd1317
--- /dev/null
+++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/NamedComponent.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugin;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * An annotation used on plugin components which will be registered under a given name by Elasticsearch server.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ ElementType.TYPE })
+public @interface NamedComponent {
+    /**
+     * The name used for registration and lookup
+     * @return a name
+     */
+    String value();
+}
diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/api/Extensible.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/api/Extensible.java
deleted file mode 100644
index 0f9bfa3ea8ef..000000000000
--- a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/api/Extensible.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.plugin.api;
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-import static java.lang.annotation.ElementType.TYPE;
-
-/**
- * Marker for things that can be loaded by component loader.
- *
- */
-@Retention(RetentionPolicy.RUNTIME)
-@Target(value = { TYPE })
-public @interface Extensible {
-}
diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/api/NamedComponent.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/api/NamedComponent.java
deleted file mode 100644
index ca62f8dd5aab..000000000000
--- a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/api/NamedComponent.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.plugin.api;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-/**
- * An annotation used on plugin components which will be registered under a given name by Elasticsearch server.
- */
-@Retention(RetentionPolicy.RUNTIME)
-@Target({ ElementType.TYPE })
-public @interface NamedComponent {
-    /**
-     * The name used for registration and lookup
-     * @return a name
-     */
-    String value();
-}
diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/api/package-info.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/api/package-info.java
deleted file mode 100644
index b8ecb3165f32..000000000000
--- a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/api/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-/**
- * An api used by plugin developers to implement custom Elasticsearch plugins.
- * The package convention in plugin apis is as follows:
- * <ul>
- *      <li> The root package is org.elasticsearch.plugin</li>
- *      <li> Specialised API jars have their name following the root package.
- *           i.e. org.elasticsearch.plugin.analysis
- *      </li>
- *      <li> Interfaces and annotations used by plugin developers are in `api` package
- *           i.e org.elasticsearch.plugin.analysis.api or org.elasticsearch.plugin.api
- *      <li> packages which are not meant to be used by plugin developers should not be subpackages of api
- *           i.e org.elasticsearch.plugin.analysis.internal
- *      </li>
- * </ul>
- */
-package org.elasticsearch.plugin.api;
diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/package-info.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/package-info.java
new file mode 100644
index 000000000000..9e8ad40ef028
--- /dev/null
+++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * An api used by plugin developers to implement custom Elasticsearch plugins.
+ * The package convention in plugin apis is as follows:
+ * <ul>
+ *      <li> The root package is org.elasticsearch.plugin</li>
+ *      <li> Specialised API jars have their name following the root package.
+ *           Interfaces and annotations used by plugin developers should be placed under it.
+ *           i.e. org.elasticsearch.plugin.analysis
+ *      </li>
+ *      <li> packages which are not meant to be used by plugin developers should be under internal package suffix
+ *           i.e org.elasticsearch.plugin.analysis.internal
+ *      </li>
+ * </ul>
+ */
+package org.elasticsearch.plugin;
diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/AnalysisSettings.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/AnalysisSettings.java
new file mode 100644
index 000000000000..c0e6a59e0425
--- /dev/null
+++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/AnalysisSettings.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugin.settings;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.TYPE;
+
+/**
+ * An annotation used to mark analysis setting interface
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(value = { TYPE })
+public @interface AnalysisSettings {
+}
diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/BooleanSetting.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/BooleanSetting.java
new file mode 100644
index 000000000000..58749bea7d57
--- /dev/null
+++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/BooleanSetting.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugin.settings;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * An annotation used to mark a setting of type Boolean
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+public @interface BooleanSetting {
+    /**
+     * A name of a setting
+     */
+    String path();
+
+    /**
+     * A default value of a boolean setting
+     */
+    boolean defaultValue();
+}
diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/IntSetting.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/IntSetting.java
new file mode 100644
index 000000000000..83f4ef40b423
--- /dev/null
+++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/IntSetting.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugin.settings;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * An annotation used to mark a setting of type integer
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+public @interface IntSetting {
+    /**
+     * A name of a setting
+     */
+    String path();
+
+    /**
+     * A default value of an int setting
+     */
+    int defaultValue();
+}
diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/ListSetting.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/ListSetting.java
new file mode 100644
index 000000000000..5d561164bfb4
--- /dev/null
+++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/ListSetting.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugin.settings;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * An annotation used to mark a setting of type list.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+public @interface ListSetting {
+    /**
+     * A name of a setting
+     */
+    String path();
+}
diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/LongSetting.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/LongSetting.java
new file mode 100644
index 000000000000..67af5baf9448
--- /dev/null
+++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/LongSetting.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugin.settings;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * An annotation used to mark a setting of type Long
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+public @interface LongSetting {
+    /**
+     * A name of a setting
+     */
+    String path();
+
+    /**
+     * A default value of a long setting
+     */
+    long defaultValue();
+}
diff --git a/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/StringSetting.java b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/StringSetting.java
new file mode 100644
index 000000000000..e16c3af32877
--- /dev/null
+++ b/libs/plugin-api/src/main/java/org/elasticsearch/plugin/settings/StringSetting.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugin.settings;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * An annotation used to mark a setting of type String
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+public @interface StringSetting {
+    /**
+     * A name of a setting
+     */
+    String path();
+
+    /**
+     * A default value of a String setting
+     */
+    String defaultValue();
+}
diff --git a/libs/plugin-scanner/build.gradle b/libs/plugin-scanner/build.gradle
new file mode 100644
index 000000000000..a612533bc76c
--- /dev/null
+++ b/libs/plugin-scanner/build.gradle
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+apply plugin: 'elasticsearch.publish'
+
+tasks.named("jarHell").configure { enabled = false }
+
+
+tasks.named("dependencyLicenses").configure {
+  mapping from: /asm-.*/, to: 'asm'
+}
+
+dependencies {
+  api project(':libs:elasticsearch-core')
+  api project(':libs:elasticsearch-plugin-api')
+  api project(":libs:elasticsearch-x-content")
+
+  api 'org.ow2.asm:asm:9.4'
+  api 'org.ow2.asm:asm-tree:9.4'
+
+  testImplementation "junit:junit:${versions.junit}"
+  testImplementation(project(":test:framework")) {
+    exclude group: 'org.elasticsearch', module: 'elasticsearch-plugin-scanner'
+  }
+}
+tasks.named('forbiddenApisMain').configure {
+  // TODO: Need to decide how we want to handle for forbidden signatures with the changes to core
+  replaceSignatureFiles 'jdk-signatures'
+}
+
diff --git a/libs/plugin-scanner/licenses/asm-LICENSE.txt b/libs/plugin-scanner/licenses/asm-LICENSE.txt
new file mode 100644
index 000000000000..afb064f2f266
--- /dev/null
+++ b/libs/plugin-scanner/licenses/asm-LICENSE.txt
@@ -0,0 +1,26 @@
+Copyright (c) 2012 France Télécom
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. Neither the name of the copyright holders nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/libs/plugin-scanner/licenses/asm-NOTICE.txt b/libs/plugin-scanner/licenses/asm-NOTICE.txt
new file mode 100644
index 000000000000..8d1c8b69c3fc
--- /dev/null
+++ b/libs/plugin-scanner/licenses/asm-NOTICE.txt
@@ -0,0 +1 @@
+ 
diff --git a/libs/plugin-scanner/src/main/java/module-info.java b/libs/plugin-scanner/src/main/java/module-info.java
new file mode 100644
index 000000000000..31f078289214
--- /dev/null
+++ b/libs/plugin-scanner/src/main/java/module-info.java
@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module org.elasticsearch.plugin.scanner {
+    requires org.elasticsearch.base;
+    requires org.objectweb.asm;
+    requires org.elasticsearch.plugin;
+    requires org.elasticsearch.xcontent;
+
+    exports org.elasticsearch.plugin.scanner;
+}
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/scanner/AnnotatedHierarchyVisitor.java b/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/AnnotatedHierarchyVisitor.java
similarity index 98%
rename from build-tools/src/main/java/org/elasticsearch/gradle/plugin/scanner/AnnotatedHierarchyVisitor.java
rename to libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/AnnotatedHierarchyVisitor.java
index 77cb6c97ddb6..a372ff10683a 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/scanner/AnnotatedHierarchyVisitor.java
+++ b/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/AnnotatedHierarchyVisitor.java
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.gradle.plugin.scanner;
+package org.elasticsearch.plugin.scanner;
 
 import org.objectweb.asm.AnnotationVisitor;
 import org.objectweb.asm.ClassVisitor;
diff --git a/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/ClassReaders.java b/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/ClassReaders.java
new file mode 100644
index 000000000000..c4490b0c27a3
--- /dev/null
+++ b/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/ClassReaders.java
@@ -0,0 +1,122 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugin.scanner;
+
+import org.elasticsearch.core.PathUtils;
+import org.objectweb.asm.ClassReader;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * A utility class containing methods to create streams of ASM's ClassReader
+ *
+ * @see ClassReader
+ */
+public class ClassReaders {
+    private static final String MODULE_INFO = "module-info.class";
+
+    /**
+     * This method must be used within a try-with-resources statement or similar
+     * control structure.
+     */
+    public static List<ClassReader> ofDirWithJars(Path dir) {
+        if (dir == null) {
+            return Collections.emptyList();
+        }
+        try (var stream = Files.list(dir)) {
+            return ofPaths(stream);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public static List<ClassReader> ofPaths(Set<URL> classpathFiles) {
+        return ofPaths(classpathFiles.stream().map(ClassReaders::toPath));
+    }
+
+    private static Path toPath(URL url) {
+        try {
+            return PathUtils.get(url.toURI());
+        } catch (URISyntaxException e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    /**
+     * This method must be used within a try-with-resources statement or similar
+     * control structure.
+     */
+    public static List<ClassReader> ofPaths(Stream<Path> list) {
+        return list.filter(Files::exists).flatMap(p -> {
+            if (p.toString().endsWith(".jar")) {
+                return classesInJar(p).stream();
+            } else {
+                return classesInPath(p).stream();
+            }
+        }).toList();
+    }
+
+    private static List<ClassReader> classesInJar(Path jar) {
+        try (FileSystem jarFs = FileSystems.newFileSystem(jar)) {
+            Path root = jarFs.getPath("/");
+            return classesInPath(root);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private static List<ClassReader> classesInPath(Path root) {
+        try (var stream = Files.walk(root)) {
+            return stream.filter(p -> p.toString().endsWith(".class"))
+                .filter(p -> p.toString().endsWith(MODULE_INFO) == false)
+                .filter(p -> p.toString().startsWith("/META-INF") == false)// skip multi-release files
+                .map(p -> {
+                    try (InputStream is = Files.newInputStream(p)) {
+                        byte[] classBytes = is.readAllBytes();
+                        return new ClassReader(classBytes);
+                    } catch (IOException ex) {
+                        throw new UncheckedIOException(ex);
+                    }
+                })
+                .collect(Collectors.toList());
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public static List<ClassReader> ofClassPath() throws IOException {
+        String classpath = System.getProperty("java.class.path");
+        return ofClassPath(classpath);
+    }
+
+    public static List<ClassReader> ofClassPath(String classpath) {
+        if (classpath != null && classpath.equals("") == false) {// todo when do we set cp to "" ?
+            var classpathSeparator = System.getProperty("path.separator");
+
+            String[] pathelements = classpath.split(classpathSeparator);
+            return ofPaths(Arrays.stream(pathelements).map(Paths::get));
+        }
+        return Collections.emptyList();
+    }
+}
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/scanner/ClassScanner.java b/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/ClassScanner.java
similarity index 87%
rename from build-tools/src/main/java/org/elasticsearch/gradle/plugin/scanner/ClassScanner.java
rename to libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/ClassScanner.java
index e63ee65e4a1c..4341ac729cba 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/scanner/ClassScanner.java
+++ b/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/ClassScanner.java
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.gradle.plugin.scanner;
+package org.elasticsearch.plugin.scanner;
 
 import org.objectweb.asm.AnnotationVisitor;
 import org.objectweb.asm.ClassReader;
@@ -15,10 +15,10 @@
 import java.util.Deque;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.BiFunction;
-import java.util.stream.Stream;
 
 public class ClassScanner {
     private final Map<String, String> foundClasses;
@@ -32,10 +32,8 @@ public ClassScanner(String targetAnnotation, BiFunction<String, Map<String, Stri
         );
     }
 
-    public void visit(Stream<ClassReader> classReaderStream) {
-        try (classReaderStream) {
-            classReaderStream.forEach(classReader -> classReader.accept(annotatedHierarchyVisitor, ClassReader.SKIP_CODE));
-        }
+    public void visit(List<ClassReader> classReaders) {
+        classReaders.forEach(classReader -> classReader.accept(annotatedHierarchyVisitor, ClassReader.SKIP_CODE));
         addExtensibleDescendants(annotatedHierarchyVisitor.getClassHierarchy());
     }
 
diff --git a/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/NamedComponentScanner.java b/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/NamedComponentScanner.java
new file mode 100644
index 000000000000..e306cf93bba4
--- /dev/null
+++ b/libs/plugin-scanner/src/main/java/org/elasticsearch/plugin/scanner/NamedComponentScanner.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugin.scanner;
+
+import org.elasticsearch.plugin.Extensible;
+import org.elasticsearch.plugin.NamedComponent;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.objectweb.asm.AnnotationVisitor;
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.Type;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class NamedComponentScanner {
+
+    // main method to be used by gradle build plugin
+    public static void main(String[] args) throws IOException {
+        List<ClassReader> classReaders = ClassReaders.ofClassPath();
+
+        NamedComponentScanner scanner = new NamedComponentScanner();
+        Map<String, Map<String, String>> namedComponentsMap = scanner.scanForNamedClasses(classReaders);
+        Path outputFile = Path.of(args[0]);
+        scanner.writeToFile(namedComponentsMap, outputFile);
+    }
+
+    // scope for testing
+    public void writeToFile(Map<String, Map<String, String>> namedComponentsMap, Path outputFile) throws IOException {
+        Files.createDirectories(outputFile.getParent());
+
+        try (OutputStream outputStream = Files.newOutputStream(outputFile)) {
+            try (XContentBuilder namedComponents = XContentFactory.jsonBuilder(outputStream)) {
+                namedComponents.startObject();
+                for (Map.Entry<String, Map<String, String>> extensibleToComponents : namedComponentsMap.entrySet()) {
+                    namedComponents.startObject(extensibleToComponents.getKey());// extensible class name
+                    for (Map.Entry<String, String> components : extensibleToComponents.getValue().entrySet()) {
+                        namedComponents.field(components.getKey(), components.getValue());// component name : component class
+                    }
+                    namedComponents.endObject();
+                }
+                namedComponents.endObject();
+            }
+        }
+
+    }
+
+    // returns a Map<String, Map<String,String> - extensible interface -> map{ namedName -> className }
+    public Map<String, Map<String, String>> scanForNamedClasses(List<ClassReader> classReaders) {
+        ClassScanner extensibleClassScanner = new ClassScanner(Type.getDescriptor(Extensible.class), (classname, map) -> {
+            map.put(classname, classname);
+            return null;
+        });
+        extensibleClassScanner.visit(classReaders);
+
+        ClassScanner namedComponentsScanner = new ClassScanner(
+            Type.getDescriptor(NamedComponent.class),
+            (classname, map) -> new AnnotationVisitor(Opcodes.ASM9) {
+                @Override
+                public void visit(String key, Object value) {
+                    assert key.equals("value");
+                    assert value instanceof String;
+                    map.put(value.toString(), classname);
+                }
+            }
+        );
+
+        namedComponentsScanner.visit(classReaders);
+
+        Map<String, Map<String, String>> componentInfo = new HashMap<>();
+        for (var e : namedComponentsScanner.getFoundClasses().entrySet()) {
+            String name = e.getKey();
+            String classnameWithSlashes = e.getValue();
+            String extensibleClassnameWithSlashes = extensibleClassScanner.getFoundClasses().get(classnameWithSlashes);
+            if (extensibleClassnameWithSlashes == null) {
+                throw new RuntimeException(
+                    "Named component " + name + "(" + pathToClassName(classnameWithSlashes) + ") does not extend from an extensible class"
+                );
+            }
+            var named = componentInfo.computeIfAbsent(pathToClassName(extensibleClassnameWithSlashes), k -> new HashMap<>());
+            named.put(name, pathToClassName(classnameWithSlashes));
+        }
+        return componentInfo;
+    }
+
+    private String pathToClassName(String classWithSlashes) {
+        return classWithSlashes.replace('/', '.');
+    }
+
+}
diff --git a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/AnnotatedHierarchyVisitorTests.java b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/AnnotatedHierarchyVisitorTests.java
new file mode 100644
index 000000000000..8aaa36b7f9a0
--- /dev/null
+++ b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/AnnotatedHierarchyVisitorTests.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugin.scanner;
+
+import org.elasticsearch.plugin.Extensible;
+import org.elasticsearch.plugin.NamedComponent;
+import org.elasticsearch.plugin.scanner.test_model.ExtensibleClass;
+import org.elasticsearch.plugin.scanner.test_model.ExtensibleInterface;
+import org.elasticsearch.plugin.scanner.test_model.ImplementingExtensible;
+import org.elasticsearch.plugin.scanner.test_model.SubClass;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.Type;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URISyntaxException;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+
+public class AnnotatedHierarchyVisitorTests extends ESTestCase {
+    Set<String> foundClasses;
+    AnnotatedHierarchyVisitor visitor;
+
+    @Before
+    public void init() {
+        foundClasses = new HashSet<>();
+        visitor = new AnnotatedHierarchyVisitor(Type.getDescriptor(Extensible.class), (className) -> {
+            foundClasses.add(className);
+            return null;
+        });
+    }
+
+    public void testNoClassesAnnotated() throws IOException, URISyntaxException {
+        performScan(visitor, NamedComponent.class);
+
+        assertTrue(foundClasses.isEmpty());
+    }
+
+    public void testSingleAnnotatedClass() throws IOException, URISyntaxException {
+        performScan(visitor, ExtensibleClass.class);
+
+        assertThat(foundClasses, equalTo(Set.of(classNameToPath(ExtensibleClass.class))));
+    }
+
+    public void testSubClassofExtensible() throws IOException, URISyntaxException {
+        performScan(visitor, ExtensibleClass.class, SubClass.class);
+
+        assertThat(foundClasses, equalTo(Set.of(classNameToPath(ExtensibleClass.class))));
+        assertThat(
+            visitor.getClassHierarchy(),
+            equalTo(Map.of(classNameToPath(ExtensibleClass.class), Set.of(classNameToPath(SubClass.class))))
+        );
+    }
+
+    public void testSubInterfaceOfExtensible() throws IOException, URISyntaxException {
+        performScan(visitor, ImplementingExtensible.class, ExtensibleInterface.class);
+
+        assertThat(foundClasses, equalTo(Set.of(classNameToPath(ExtensibleInterface.class))));
+        assertThat(
+            visitor.getClassHierarchy(),
+            equalTo(Map.of(classNameToPath(ExtensibleInterface.class), Set.of(classNameToPath(ImplementingExtensible.class))))
+        );
+    }
+
+    private String classNameToPath(Class<?> clazz) {
+        return clazz.getCanonicalName().replace(".", "/");
+    }
+
+    private void performScan(AnnotatedHierarchyVisitor classVisitor, Class<?>... classes) throws IOException, URISyntaxException {
+        for (Class<?> clazz : classes) {
+            String className = classNameToPath(clazz) + ".class";
+            var stream = this.getClass().getClassLoader().getResourceAsStream(className);
+            try (InputStream fileInputStream = stream) {
+                ClassReader cr = new ClassReader(fileInputStream);
+                cr.accept(classVisitor, 0);
+            }
+        }
+    }
+
+}
diff --git a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/ClassReadersTests.java b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/ClassReadersTests.java
new file mode 100644
index 000000000000..764065bc462b
--- /dev/null
+++ b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/ClassReadersTests.java
@@ -0,0 +1,129 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugin.scanner;
+
+import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.compiler.InMemoryJavaCompiler;
+import org.elasticsearch.test.jar.JarUtils;
+import org.hamcrest.Matchers;
+import org.objectweb.asm.ClassReader;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class ClassReadersTests extends ESTestCase {
+
+    public void testModuleInfoIsNotReturnedAsAClassFromJar() throws IOException {
+        final Path tmp = createTempDir(getTestName());
+        final Path dirWithJar = tmp.resolve("jars-dir");
+        Files.createDirectories(dirWithJar);
+        Path jar = dirWithJar.resolve("api.jar");
+        JarUtils.createJarWithEntries(jar, Map.of("module-info.class", InMemoryJavaCompiler.compile("module-info", """
+            module p {}
+            """)));
+
+        List<ClassReader> classReaders = ClassReaders.ofPaths(Stream.of(jar));
+        org.hamcrest.MatcherAssert.assertThat(classReaders, Matchers.empty());
+
+        // aggressively delete the jar dir, so that any leaked filed handles fail this specific test on windows
+        IOUtils.rm(tmp);
+    }
+
+    public void testTwoClassesInAStreamFromJar() throws IOException {
+        final Path tmp = createTempDir(getTestName());
+        final Path dirWithJar = tmp.resolve("jars-dir");
+        Files.createDirectories(dirWithJar);
+        Path jar = dirWithJar.resolve("api.jar");
+        JarUtils.createJarWithEntries(jar, Map.of("p/A.class", InMemoryJavaCompiler.compile("p.A", """
+            package p;
+            public class A {}
+            """), "p/B.class", InMemoryJavaCompiler.compile("p.B", """
+            package p;
+            public class B {}
+            """)));
+
+        List<ClassReader> classReaders = ClassReaders.ofPaths(Stream.of(jar));
+        List<String> collect = classReaders.stream().map(cr -> cr.getClassName()).collect(Collectors.toList());
+        org.hamcrest.MatcherAssert.assertThat(collect, Matchers.containsInAnyOrder("p/A", "p/B"));
+
+        // aggressively delete the jar dir, so that any leaked filed handles fail this specific test on windows
+        IOUtils.rm(tmp);
+    }
+
+    public void testStreamOfJarsAndIndividualClasses() throws IOException {
+        final Path tmp = createTempDir(getTestName());
+        final Path dirWithJar = tmp.resolve("jars-dir");
+        Files.createDirectories(dirWithJar);
+
+        Path jar = dirWithJar.resolve("a_b.jar");
+        JarUtils.createJarWithEntries(jar, Map.of("p/A.class", InMemoryJavaCompiler.compile("p.A", """
+            package p;
+            public class A {}
+            """), "p/B.class", InMemoryJavaCompiler.compile("p.B", """
+            package p;
+            public class B {}
+            """)));
+
+        Path jar2 = dirWithJar.resolve("c_d.jar");
+        JarUtils.createJarWithEntries(jar2, Map.of("p/C.class", InMemoryJavaCompiler.compile("p.C", """
+            package p;
+            public class C {}
+            """), "p/D.class", InMemoryJavaCompiler.compile("p.D", """
+            package p;
+            public class D {}
+            """)));
+
+        InMemoryJavaCompiler.compile("p.E", """
+            package p;
+            public class E {}
+            """);
+        Files.write(tmp.resolve("E.class"), InMemoryJavaCompiler.compile("p.E", """
+            package p;
+            public class E {}
+            """));
+
+        List<ClassReader> classReaders = ClassReaders.ofPaths(Stream.of(tmp, jar, jar2));
+        List<String> collect = classReaders.stream().map(cr -> cr.getClassName()).collect(Collectors.toList());
+        org.hamcrest.MatcherAssert.assertThat(collect, Matchers.containsInAnyOrder("p/A", "p/B", "p/C", "p/D", "p/E"));
+    }
+
+    public void testMultipleJarsInADir() throws IOException {
+        final Path tmp = createTempDir(getTestName());
+        final Path dirWithJar = tmp.resolve("jars-dir");
+        Files.createDirectories(dirWithJar);
+
+        Path jar = dirWithJar.resolve("a_b.jar");
+        JarUtils.createJarWithEntries(jar, Map.of("p/A.class", InMemoryJavaCompiler.compile("p.A", """
+            package p;
+            public class A {}
+            """), "p/B.class", InMemoryJavaCompiler.compile("p.B", """
+            package p;
+            public class B {}
+            """)));
+
+        Path jar2 = dirWithJar.resolve("c_d.jar");
+        JarUtils.createJarWithEntries(jar2, Map.of("p/C.class", InMemoryJavaCompiler.compile("p.C", """
+            package p;
+            public class C {}
+            """), "p/D.class", InMemoryJavaCompiler.compile("p.D", """
+            package p;
+            public class D {}
+            """)));
+
+        List<ClassReader> classReaders = ClassReaders.ofDirWithJars(dirWithJar);
+        List<String> collect = classReaders.stream().map(cr -> cr.getClassName()).collect(Collectors.toList());
+        org.hamcrest.MatcherAssert.assertThat(collect, Matchers.containsInAnyOrder("p/A", "p/B", "p/C", "p/D"));
+    }
+}
diff --git a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/ClassScannerTests.java b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/ClassScannerTests.java
new file mode 100644
index 000000000000..b8b35dfb568a
--- /dev/null
+++ b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/ClassScannerTests.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugin.scanner;
+
+import org.elasticsearch.plugin.Extensible;
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matchers;
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.Type;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class ClassScannerTests extends ESTestCase {
+    static final System.Logger logger = System.getLogger(ClassScannerTests.class.getName());
+
+    public void testClassAndInterfaceHierarchy() throws IOException {
+        var reader = new ClassScanner(Type.getDescriptor(Extensible.class), (classname, map) -> {
+            map.put(classname, classname);
+            return null;
+        });
+        List<ClassReader> classReaders = ClassReaders.ofClassPath();
+        logger.log(System.Logger.Level.INFO, "classReaderStream size " + classReaders.size());
+
+        reader.visit(classReaders);
+        Map<String, String> extensibleClasses = reader.getFoundClasses();
+
+        org.hamcrest.MatcherAssert.assertThat(
+            extensibleClasses,
+            Matchers.allOf(
+                Matchers.hasEntry(
+                    "org/elasticsearch/plugin/scanner/test_model/ExtensibleClass",
+                    "org/elasticsearch/plugin/scanner/test_model/ExtensibleClass"
+                ),
+                Matchers.hasEntry(
+                    "org/elasticsearch/plugin/scanner/test_model/ImplementingExtensible",
+                    "org/elasticsearch/plugin/scanner/test_model/ExtensibleInterface"
+                ),
+                Matchers.hasEntry(
+                    "org/elasticsearch/plugin/scanner/test_model/SubClass",
+                    "org/elasticsearch/plugin/scanner/test_model/ExtensibleClass"
+                )
+            )
+        );
+    }
+
+}
diff --git a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/NamedComponentScannerTests.java b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/NamedComponentScannerTests.java
new file mode 100644
index 000000000000..cf81b81963c7
--- /dev/null
+++ b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/NamedComponentScannerTests.java
@@ -0,0 +1,227 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugin.scanner;
+
+import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.plugin.scanner.test_model.ExtensibleClass;
+import org.elasticsearch.plugin.scanner.test_model.ExtensibleInterface;
+import org.elasticsearch.plugin.scanner.test_model.TestNamedComponent;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.compiler.InMemoryJavaCompiler;
+import org.elasticsearch.test.jar.JarUtils;
+import org.objectweb.asm.ClassReader;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class NamedComponentScannerTests extends ESTestCase {
+
+    private Path tmpDir() throws IOException {
+        return createTempDir();
+    }
+
+    NamedComponentScanner namedComponentScanner = new NamedComponentScanner();
+
+    public void testFindNamedComponentInSingleClass() throws URISyntaxException {
+        Map<String, Map<String, String>> namedComponents = namedComponentScanner.scanForNamedClasses(
+            classReaderStream(TestNamedComponent.class, ExtensibleInterface.class)
+        );
+
+        org.hamcrest.MatcherAssert.assertThat(
+            namedComponents,
+            equalTo(
+                Map.of(
+                    ExtensibleInterface.class.getCanonicalName(),
+                    Map.of("test_named_component", TestNamedComponent.class.getCanonicalName())
+                )
+            )
+        );
+
+    }
+
+    public void testNamedComponentsAreFoundWhenSingleJarProvided() throws IOException {
+        final Path tmp = tmpDir();
+        final Path dirWithJar = tmp.resolve("jars-dir");
+        Files.createDirectories(dirWithJar);
+        Path jar = dirWithJar.resolve("plugin.jar");
+        JarUtils.createJarWithEntries(jar, Map.of("p/A.class", InMemoryJavaCompiler.compile("p.A", """
+            package p;
+            import org.elasticsearch.plugin.*;
+            import org.elasticsearch.plugin.scanner.test_model.*;
+            @NamedComponent("a_component")
+            public class A extends ExtensibleClass {}
+            """), "p/B.class", InMemoryJavaCompiler.compile("p.B", """
+            package p;
+            import org.elasticsearch.plugin.*;
+            import org.elasticsearch.plugin.scanner.test_model.*;
+            @NamedComponent("b_component")
+            public class B implements ExtensibleInterface{}
+            """)));
+        List<ClassReader> classReaderStream = Stream.concat(
+            ClassReaders.ofDirWithJars(dirWithJar).stream(),
+            ClassReaders.ofClassPath().stream()
+        )// contains plugin-api
+            .toList();
+
+        Map<String, Map<String, String>> namedComponents = namedComponentScanner.scanForNamedClasses(classReaderStream);
+
+        org.hamcrest.MatcherAssert.assertThat(
+            namedComponents,
+            equalTo(
+                Map.of(
+                    ExtensibleClass.class.getCanonicalName(),
+                    Map.of("a_component", "p.A"),
+                    ExtensibleInterface.class.getCanonicalName(),
+                    Map.of(
+                        "b_component",
+                        "p.B",
+                        // noise from classpath
+                        "test_named_component",
+                        "org.elasticsearch.plugin.scanner.test_model.TestNamedComponent"
+                    )
+                )
+            )
+        );
+
+        // aggressively delete the jar dir, so that any leaked filed handles fail this specific test on windows
+        IOUtils.rm(tmp);
+    }
+
+    public void testNamedComponentsCanExtednCommonSuperClass() throws IOException {
+        Map<String, CharSequence> sources = Map.of(
+            "p.CustomExtensibleInterface",
+            """
+                package p;
+                import org.elasticsearch.plugin.*;
+                import org.elasticsearch.plugin.scanner.test_model.*;
+                public interface CustomExtensibleInterface extends ExtensibleInterface {}
+                """,
+            // note that this class implements a custom interface
+            "p.CustomExtensibleClass",
+            """
+                package p;
+                import org.elasticsearch.plugin.*;
+                import org.elasticsearch.plugin.scanner.test_model.*;
+                public class CustomExtensibleClass implements CustomExtensibleInterface {}
+                """,
+            "p.A",
+            """
+                package p;
+                import org.elasticsearch.plugin.*;
+                import org.elasticsearch.plugin.scanner.test_model.*;
+                @NamedComponent("a_component")
+                public class A extends CustomExtensibleClass {}
+                """,
+            "p.B",
+            """
+                package p;
+                import org.elasticsearch.plugin.*;
+                import org.elasticsearch.plugin.scanner.test_model.*;
+                @NamedComponent("b_component")
+                public class B implements CustomExtensibleInterface{}
+                """
+        );
+        var classToBytes = InMemoryJavaCompiler.compile(sources);
+
+        Map<String, byte[]> jarEntries = new HashMap<>();
+        jarEntries.put("p/CustomExtensibleInterface.class", classToBytes.get("p.CustomExtensibleInterface"));
+        jarEntries.put("p/CustomExtensibleClass.class", classToBytes.get("p.CustomExtensibleClass"));
+        jarEntries.put("p/A.class", classToBytes.get("p.A"));
+        jarEntries.put("p/B.class", classToBytes.get("p.B"));
+
+        final Path tmp = tmpDir();
+        final Path dirWithJar = tmp.resolve("jars-dir");
+        Files.createDirectories(dirWithJar);
+        Path jar = dirWithJar.resolve("plugin.jar");
+        JarUtils.createJarWithEntries(jar, jarEntries);
+
+        Stream<ClassReader> classPath = ClassReaders.ofClassPath().stream();
+        List<ClassReader> classReaders = Stream.concat(ClassReaders.ofDirWithJars(dirWithJar).stream(), classPath)// contains plugin-api
+            .toList();
+
+        Map<String, Map<String, String>> namedComponents = namedComponentScanner.scanForNamedClasses(classReaders);
+
+        org.hamcrest.MatcherAssert.assertThat(
+            namedComponents,
+            equalTo(
+                Map.of(
+                    ExtensibleInterface.class.getCanonicalName(),
+                    Map.of(
+                        "a_component",
+                        "p.A",
+                        "b_component",
+                        "p.B",
+                        "test_named_component",
+                        "org.elasticsearch.plugin.scanner.test_model.TestNamedComponent"// noise from classpath
+                    )
+                )
+            )
+        );
+
+        // aggressively delete the jar dir, so that any leaked filed handles fail this specific test on windows
+        IOUtils.rm(tmp);
+    }
+
+    public void testWriteToFile() throws IOException {
+        Map<String, String> extensibleInterfaceComponents = new LinkedHashMap<>();
+        extensibleInterfaceComponents.put("a_component", "p.A");
+        extensibleInterfaceComponents.put("b_component", "p.B");
+        Map<String, Map<String, String>> mapToWrite = new LinkedHashMap<>();
+        mapToWrite.put(ExtensibleInterface.class.getCanonicalName(), extensibleInterfaceComponents);
+
+        Path path = tmpDir().resolve("file.json");
+        namedComponentScanner.writeToFile(mapToWrite, path);
+
+        String jsonMap = Files.readString(path);
+        assertThat(jsonMap, equalTo("""
+            {
+              "org.elasticsearch.plugin.scanner.test_model.ExtensibleInterface": {
+                "a_component": "p.A",
+                "b_component": "p.B"
+              }
+            }
+            """.replaceAll("[\n\r\s]", "")));
+    }
+
+    private List<ClassReader> classReaderStream(Class<?>... classes) {
+        try {
+            return Arrays.stream(classes).map(clazz -> {
+                String className = classNameToPath(clazz) + ".class";
+                var stream = this.getClass().getClassLoader().getResourceAsStream(className);
+                try (InputStream is = stream) {
+                    byte[] classBytes = is.readAllBytes();
+                    ClassReader classReader = new ClassReader(classBytes);
+                    return classReader;
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+            }).collect(Collectors.toList());
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private String classNameToPath(Class<?> clazz) {
+        return clazz.getCanonicalName().replace(".", "/");
+    }
+
+}
diff --git a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/ExtensibleClass.java b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/ExtensibleClass.java
new file mode 100644
index 000000000000..ad055a47629f
--- /dev/null
+++ b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/ExtensibleClass.java
@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugin.scanner.test_model;
+
+import org.elasticsearch.plugin.Extensible;
+
+@Extensible
+public class ExtensibleClass {}
diff --git a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/ExtensibleInterface.java b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/ExtensibleInterface.java
new file mode 100644
index 000000000000..a99f0a1c8da4
--- /dev/null
+++ b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/ExtensibleInterface.java
@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugin.scanner.test_model;
+
+import org.elasticsearch.plugin.Extensible;
+
+@Extensible
+public interface ExtensibleInterface {}
diff --git a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/ImplementingExtensible.java b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/ImplementingExtensible.java
new file mode 100644
index 000000000000..2b82972624f5
--- /dev/null
+++ b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/ImplementingExtensible.java
@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugin.scanner.test_model;
+
+public class ImplementingExtensible implements ExtensibleInterface {}
diff --git a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/SubClass.java b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/SubClass.java
new file mode 100644
index 000000000000..b11709023594
--- /dev/null
+++ b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/SubClass.java
@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugin.scanner.test_model;
+
+public class SubClass extends ExtensibleClass {}
diff --git a/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/TestNamedComponent.java b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/TestNamedComponent.java
new file mode 100644
index 000000000000..71db778ca4e1
--- /dev/null
+++ b/libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/test_model/TestNamedComponent.java
@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.plugin.scanner.test_model;
+
+import org.elasticsearch.plugin.NamedComponent;
+
+@NamedComponent("test_named_component")
+public class TestNamedComponent implements ExtensibleInterface {
+
+}
diff --git a/libs/secure-sm/src/main/java/org/elasticsearch/secure_sm/SecureSM.java b/libs/secure-sm/src/main/java/org/elasticsearch/secure_sm/SecureSM.java
index fa98bc1bacc6..68d2f45386c5 100644
--- a/libs/secure-sm/src/main/java/org/elasticsearch/secure_sm/SecureSM.java
+++ b/libs/secure-sm/src/main/java/org/elasticsearch/secure_sm/SecureSM.java
@@ -162,8 +162,12 @@ private static boolean isInnocuousThread(Thread t) {
     protected void checkThreadAccess(Thread t) {
         Objects.requireNonNull(t);
 
-        // first, check if we can modify threads at all.
-        checkPermission(MODIFY_THREAD_PERMISSION);
+        boolean targetThreadIsInnocuous = isInnocuousThread(t);
+        // we don't need to check if innocuous thread is modifying itself (like changes its name)
+        if (Thread.currentThread() != t || targetThreadIsInnocuous == false) {
+            // first, check if we can modify threads at all.
+            checkPermission(MODIFY_THREAD_PERMISSION);
+        }
 
         // check the threadgroup, if its our thread group or an ancestor, its fine.
         final ThreadGroup source = Thread.currentThread().getThreadGroup();
@@ -171,7 +175,7 @@ protected void checkThreadAccess(Thread t) {
 
         if (target == null) {
             return;    // its a dead thread, do nothing.
-        } else if (source.parentOf(target) == false && isInnocuousThread(t) == false) {
+        } else if (source.parentOf(target) == false && targetThreadIsInnocuous == false) {
             checkPermission(MODIFY_ARBITRARY_THREAD_PERMISSION);
         }
     }
diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationKeys.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationKeys.java
index 7884a417c39c..83dbd919a60e 100644
--- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationKeys.java
+++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationKeys.java
@@ -71,6 +71,12 @@ public class SslConfigurationKeys {
      */
     public static final String TRUSTSTORE_ALGORITHM = "truststore.algorithm";
 
+    /**
+     * The fields from the X509 certificate used for restricted trust. Internationally omitted from the list of setting returned by methods
+     * in this class. This is not a general purpose ssl configuration.
+     */
+    public static final String TRUST_RESTRICTIONS_X509_FIELDS = "trust_restrictions.x509_fields";
+
     // Key Management
     // -- Keystore
     /**
diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationLoader.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationLoader.java
index 4acee2dfcec0..f0f1bbfd9ea7 100644
--- a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationLoader.java
+++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationLoader.java
@@ -8,12 +8,15 @@
 
 package org.elasticsearch.common.ssl;
 
+import org.elasticsearch.core.Nullable;
+
 import java.nio.file.Path;
 import java.security.KeyStore;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -42,6 +45,7 @@
 import static org.elasticsearch.common.ssl.SslConfigurationKeys.TRUSTSTORE_PATH;
 import static org.elasticsearch.common.ssl.SslConfigurationKeys.TRUSTSTORE_SECURE_PASSWORD;
 import static org.elasticsearch.common.ssl.SslConfigurationKeys.TRUSTSTORE_TYPE;
+import static org.elasticsearch.common.ssl.SslConfigurationKeys.TRUST_RESTRICTIONS_X509_FIELDS;
 import static org.elasticsearch.common.ssl.SslConfigurationKeys.VERIFICATION_MODE;
 
 /**
@@ -115,6 +119,7 @@ public abstract class SslConfigurationLoader {
 
     static final List<String> DEFAULT_CIPHERS = JDK12_CIPHERS;
     private static final char[] EMPTY_PASSWORD = new char[0];
+    public static final List<X509Field> GLOBAL_DEFAULT_RESTRICTED_TRUST_FIELDS = List.of(X509Field.SAN_OTHERNAME_COMMONNAME);
 
     private final String settingPrefix;
 
@@ -124,6 +129,7 @@ public abstract class SslConfigurationLoader {
     private SslClientAuthenticationMode defaultClientAuth;
     private List<String> defaultCiphers;
     private List<String> defaultProtocols;
+    private List<X509Field> defaultRestrictedTrustFields;
 
     private Function<KeyStore, KeyStore> keyStoreFilter;
 
@@ -147,6 +153,7 @@ public SslConfigurationLoader(String settingPrefix) {
         this.defaultClientAuth = SslClientAuthenticationMode.OPTIONAL;
         this.defaultProtocols = DEFAULT_PROTOCOLS;
         this.defaultCiphers = DEFAULT_CIPHERS;
+        this.defaultRestrictedTrustFields = GLOBAL_DEFAULT_RESTRICTED_TRUST_FIELDS;
     }
 
     /**
@@ -204,6 +211,10 @@ public void setKeyStoreFilter(Function<KeyStore, KeyStore> keyStoreFilter) {
         this.keyStoreFilter = keyStoreFilter;
     }
 
+    public void setDefaultRestrictedTrustFields(List<X509Field> x509Fields) {
+        this.defaultRestrictedTrustFields = x509Fields;
+    }
+
     /**
      * Clients of this class should implement this method to determine whether there are any settings for a given prefix.
      * This is used to populate {@link SslConfiguration#explicitlyConfigured()}.
@@ -255,9 +266,14 @@ public SslConfiguration load(Path basePath) {
         final List<String> ciphers = resolveListSetting(CIPHERS, Function.identity(), defaultCiphers);
         final SslVerificationMode verificationMode = resolveSetting(VERIFICATION_MODE, SslVerificationMode::parse, defaultVerificationMode);
         final SslClientAuthenticationMode clientAuth = resolveSetting(CLIENT_AUTH, SslClientAuthenticationMode::parse, defaultClientAuth);
+        final List<X509Field> trustRestrictionsX509Fields = resolveListSetting(
+            TRUST_RESTRICTIONS_X509_FIELDS,
+            X509Field::parseForRestrictedTrust,
+            defaultRestrictedTrustFields
+        );
 
         final SslKeyConfig keyConfig = buildKeyConfig(basePath);
-        final SslTrustConfig trustConfig = buildTrustConfig(basePath, verificationMode, keyConfig);
+        final SslTrustConfig trustConfig = buildTrustConfig(basePath, verificationMode, keyConfig, Set.copyOf(trustRestrictionsX509Fields));
 
         if (protocols == null || protocols.isEmpty()) {
             throw new SslConfigException("no protocols configured in [" + settingPrefix + PROTOCOLS + "]");
@@ -278,7 +294,12 @@ public SslConfiguration load(Path basePath) {
         );
     }
 
-    protected SslTrustConfig buildTrustConfig(Path basePath, SslVerificationMode verificationMode, SslKeyConfig keyConfig) {
+    protected SslTrustConfig buildTrustConfig(
+        Path basePath,
+        SslVerificationMode verificationMode,
+        SslKeyConfig keyConfig,
+        @Nullable Set<X509Field> restrictedTrustFields
+    ) {
         final List<String> certificateAuthorities = resolveListSetting(CERTIFICATE_AUTHORITIES, Function.identity(), null);
         final String trustStorePath = resolveSetting(TRUSTSTORE_PATH, Function.identity(), null);
 
@@ -330,7 +351,7 @@ public SslKeyConfig buildKeyConfig(Path basePath) {
             }
             if (certificatePath == null) {
                 throw new SslConfigException(
-                    "cannot specify [" + settingPrefix + KEYSTORE_PATH + "] without also setting [" + settingPrefix + CERTIFICATE + "]"
+                    "cannot specify [" + settingPrefix + KEY + "] without also setting [" + settingPrefix + CERTIFICATE + "]"
                 );
             }
             final char[] password = resolvePasswordSetting(KEY_SECURE_PASSPHRASE, KEY_LEGACY_PASSPHRASE);
diff --git a/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/X509Field.java b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/X509Field.java
new file mode 100644
index 000000000000..64de9adb8c22
--- /dev/null
+++ b/libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/X509Field.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.common.ssl;
+
+import java.util.EnumSet;
+import java.util.stream.Collectors;
+
+/**
+ * An enumeration for referencing parts of an X509 certificate by a canonical string value.
+ */
+public enum X509Field {
+
+    SAN_OTHERNAME_COMMONNAME("subjectAltName.otherName.commonName", true),
+    SAN_DNS("subjectAltName.dnsName", true);
+
+    private final String configValue;
+    private final boolean supportedForRestrictedTrust;
+
+    X509Field(String configValue, boolean supportedForRestrictedTrust) {
+        this.configValue = configValue;
+        this.supportedForRestrictedTrust = supportedForRestrictedTrust;
+    }
+
+    @Override
+    public String toString() {
+        return configValue;
+    }
+
+    public static X509Field parseForRestrictedTrust(String s) {
+        return EnumSet.allOf(X509Field.class)
+            .stream()
+            .filter(v -> v.supportedForRestrictedTrust)
+            .filter(v -> v.configValue.equalsIgnoreCase(s))
+            .findFirst()
+            .orElseThrow(() -> {
+                throw new SslConfigException(
+                    s
+                        + " is not a supported x509 field for trust restrictions. "
+                        + "Recognised values are ["
+                        + EnumSet.allOf(X509Field.class).stream().map(e -> e.configValue).collect(Collectors.toSet())
+                        + "]"
+                );
+            });
+    }
+}
diff --git a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/SslConfigurationLoaderTests.java b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/SslConfigurationLoaderTests.java
index bf5f3d0696cb..5980aca16a78 100644
--- a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/SslConfigurationLoaderTests.java
+++ b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/SslConfigurationLoaderTests.java
@@ -22,6 +22,7 @@
 import javax.net.ssl.TrustManagerFactory;
 
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.instanceOf;
@@ -225,4 +226,20 @@ public void testChaCha20InCiphersOnJdk12Plus() {
         assertThat(SslConfigurationLoader.DEFAULT_CIPHERS, hasItem("TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256"));
         assertThat(SslConfigurationLoader.DEFAULT_CIPHERS, hasItem("TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256"));
     }
+
+    public void testErrorMessageForUnpairedKeyAndCertificateSettings() {
+        final boolean withKey = randomBoolean();
+        if (withKey) {
+            settings = Settings.builder().put("test.ssl.key", "certs/cert1.key").build();
+        } else {
+            settings = Settings.builder().put("test.ssl.certificate", "certs/cert1.cert").build();
+        }
+
+        final SslConfigException e = expectThrows(SslConfigException.class, () -> loader.load(certRoot));
+        if (withKey) {
+            assertThat(e.getMessage(), containsString("cannot specify [test.ssl.key] without also setting [test.ssl.certificate]"));
+        } else {
+            assertThat(e.getMessage(), containsString("cannot specify [test.ssl.certificate] without also setting [test.ssl.key]"));
+        }
+    }
 }
diff --git a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/X509FieldTests.java b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/X509FieldTests.java
new file mode 100644
index 000000000000..a0de14aec887
--- /dev/null
+++ b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/X509FieldTests.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common.ssl;
+
+import org.elasticsearch.test.ESTestCase;
+
+import static org.elasticsearch.common.ssl.X509Field.SAN_DNS;
+import static org.elasticsearch.common.ssl.X509Field.SAN_OTHERNAME_COMMONNAME;
+import static org.hamcrest.Matchers.containsString;
+
+public class X509FieldTests extends ESTestCase {
+
+    public void testParseForRestrictedTrust() {
+        assertEquals(SAN_OTHERNAME_COMMONNAME, X509Field.parseForRestrictedTrust("subjectAltName.otherName.commonName"));
+        assertEquals(SAN_DNS, X509Field.parseForRestrictedTrust("subjectAltName.dnsName"));
+        SslConfigException exception = expectThrows(SslConfigException.class, () -> X509Field.parseForRestrictedTrust("foo.bar"));
+        assertThat(exception.getMessage(), containsString("foo.bar"));
+        assertThat(exception.getMessage(), containsString("is not a supported x509 field for trust restrictions"));
+        assertThat(exception.getMessage(), containsString(SAN_OTHERNAME_COMMONNAME.toString()));
+        assertThat(exception.getMessage(), containsString(SAN_DNS.toString()));
+    }
+}
diff --git a/libs/x-content/build.gradle b/libs/x-content/build.gradle
index bf110b9adb53..958e818e019d 100644
--- a/libs/x-content/build.gradle
+++ b/libs/x-content/build.gradle
@@ -74,7 +74,7 @@ def generateProviderManifest = tasks.register("generateProviderManifest") {
   doLast {
     manifestFile.parentFile.mkdirs()
     manifestFile.setText(configurations.providerImpl.files.stream()
-      .map(f -> f.name).collect(Collectors.joining('\n')), 'UTF-8')
+      .map(f -> f.name).sorted().collect(Collectors.joining('\n')), 'UTF-8')
   }
 }
 
diff --git a/libs/x-content/impl/build.gradle b/libs/x-content/impl/build.gradle
index a703bd0b44f9..670d6c8896a4 100644
--- a/libs/x-content/impl/build.gradle
+++ b/libs/x-content/impl/build.gradle
@@ -10,13 +10,15 @@ apply plugin: 'elasticsearch.java'
 
 archivesBaseName = "x-content-impl"
 
+String jacksonVersion = "2.14.2"
+
 dependencies {
   compileOnly project(':libs:elasticsearch-core')
   compileOnly project(':libs:elasticsearch-x-content')
-  implementation "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${versions.jackson}"
-  implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}"
-  implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
+  implementation "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}"
+  implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${jacksonVersion}"
+  implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jacksonVersion}"
+  implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jacksonVersion}"
   implementation "org.yaml:snakeyaml:${versions.snakeyaml}"
 
   testImplementation(project(":test:framework")) {
diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java
index 32417bc86142..5208cbfb2952 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java
@@ -435,6 +435,22 @@ public XContentBuilder value(byte value) throws IOException {
         return this;
     }
 
+    public XContentBuilder array(String name, byte[] values) throws IOException {
+        return field(name).values(values);
+    }
+
+    private XContentBuilder values(byte[] values) throws IOException {
+        if (values == null) {
+            return nullValue();
+        }
+        startArray();
+        for (byte b : values) {
+            value(b);
+        }
+        endArray();
+        return this;
+    }
+
     ////////////////////////////////////////////////////////////////////////////
     // Double
     //////////////////////////////////
diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java
index 0cc95f546937..90555c6fed45 100644
--- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java
+++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java
@@ -305,7 +305,7 @@ public void testDotsAndDoubleWildcardInIncludedFieldName() throws IOException {
         );
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/80160")
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/92632")
     public void testDotsAndDoubleWildcardInExcludedFieldName() throws IOException {
         testFilter(
             builder -> builder.startObject().endObject(),
@@ -314,7 +314,6 @@ public void testDotsAndDoubleWildcardInExcludedFieldName() throws IOException {
             singleton("**.baz"),
             true
         );
-        // bug of double wildcard in excludes report in https://github.com/FasterXML/jackson-core/issues/700
         testFilter(
             builder -> builder.startObject().startObject("foo").field("baz", "test").endObject().endObject(),
             builder -> builder.startObject().startObject("foo").field("bar", "test").field("baz", "test").endObject().endObject(),
@@ -335,6 +334,61 @@ private void testFilter(Builder expected, Builder sample, Set<String> includes,
         assertFilterResult(expected.apply(createBuilder()), filter(sample, includes, excludes, matchFieldNamesWithDots));
     }
 
+    public void testArrayWithEmptyObjectInInclude() throws IOException {
+        testFilter(
+            builder -> builder.startObject().startArray("foo").startObject().field("bar", "baz").endObject().endArray().endObject(),
+            builder -> builder.startObject()
+                .startArray("foo")
+                .startObject()
+                .field("bar", "baz")
+                .endObject()
+                .startObject()
+                .endObject()
+                .endArray()
+                .endObject(),
+            singleton("foo.bar"),
+            emptySet(),
+            true
+        );
+    }
+
+    public void testArrayWithEmptyArrayInInclude() throws IOException {
+        testFilter(
+            builder -> builder.startObject().startArray("foo").startObject().field("bar", "baz").endObject().endArray().endObject(),
+            builder -> builder.startObject()
+                .startArray("foo")
+                .startObject()
+                .field("bar", "baz")
+                .endObject()
+                .startArray()
+                .endArray()
+                .endArray()
+                .endObject(),
+            singleton("foo.bar"),
+            emptySet(),
+            true
+        );
+    }
+
+    public void testArrayWithLastObjectSkipped() throws IOException {
+        testFilter(
+            builder -> builder.startObject().startArray("foo").startObject().field("bar", "baz").endObject().endArray().endObject(),
+            builder -> builder.startObject()
+                .startArray("foo")
+                .startObject()
+                .field("bar", "baz")
+                .endObject()
+                .startObject()
+                .field("skipped", "value")
+                .endObject()
+                .endArray()
+                .endObject(),
+            singleton("foo.bar"),
+            emptySet(),
+            true
+        );
+    }
+
     protected abstract void assertFilterResult(XContentBuilder expected, XContentBuilder actual);
 
     protected abstract XContentType getXContentType();
@@ -355,12 +409,6 @@ private XContentBuilder filter(Builder sample, Set<String> includes, Set<String>
         }
         FilterPath[] excludesFilter = FilterPath.compile(excludes);
         if (excludesFilter != null && Arrays.stream(excludesFilter).anyMatch(FilterPath::hasDoubleWildcard)) {
-            /*
-             * If there are any double wildcard filters the parser based
-             * filtering produced weird invalid json. Just field names
-             * and no objects?! Weird. Anyway, we can't use it.
-             */
-            assertFalse("can't filter on builder with dotted wildcards in exclude", matchFieldNamesWithDots);
             return filterOnBuilder(sample, includes, excludes);
         }
         return filterOnParser(sample, includes, excludes, matchFieldNamesWithDots);
diff --git a/modules/aggregations/build.gradle b/modules/aggregations/build.gradle
index b3cc698a5542..c4e4e5e0b401 100644
--- a/modules/aggregations/build.gradle
+++ b/modules/aggregations/build.gradle
@@ -1,4 +1,3 @@
-import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.internal.info.BuildParams
 
 /*
@@ -52,11 +51,7 @@ artifacts {
   restTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test"))
 }
 
-testClusters.configureEach {
-  module ':modules:lang-painless'
-  requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
-}
-
 dependencies {
   compileOnly(project(':modules:lang-painless:spi'))
+  clusterModules(project(':modules:lang-painless'))
 }
diff --git a/modules/aggregations/src/main/java/module-info.java b/modules/aggregations/src/main/java/module-info.java
index 3e378d108255..b9f2cb834736 100644
--- a/modules/aggregations/src/main/java/module-info.java
+++ b/modules/aggregations/src/main/java/module-info.java
@@ -17,8 +17,13 @@
     exports org.elasticsearch.aggregations.bucket.adjacency;
     exports org.elasticsearch.aggregations.bucket.timeseries;
     exports org.elasticsearch.aggregations.pipeline;
+    exports org.elasticsearch.aggregations.metric;
 
     opens org.elasticsearch.aggregations to org.elasticsearch.painless.spi; // whitelist resource access
 
     provides org.elasticsearch.painless.spi.PainlessExtension with org.elasticsearch.aggregations.AggregationsPainlessExtension;
+
+    provides org.elasticsearch.plugins.spi.NamedXContentProvider
+        with
+            org.elasticsearch.aggregations.metric.MatrixStatsNamedXContentProvider;
 }
diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/AggregationsPlugin.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/AggregationsPlugin.java
index c46f4b444dcf..8bf4785b96b4 100644
--- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/AggregationsPlugin.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/AggregationsPlugin.java
@@ -14,6 +14,9 @@
 import org.elasticsearch.aggregations.bucket.histogram.InternalAutoDateHistogram;
 import org.elasticsearch.aggregations.bucket.timeseries.InternalTimeSeries;
 import org.elasticsearch.aggregations.bucket.timeseries.TimeSeriesAggregationBuilder;
+import org.elasticsearch.aggregations.metric.InternalMatrixStats;
+import org.elasticsearch.aggregations.metric.MatrixStatsAggregationBuilder;
+import org.elasticsearch.aggregations.metric.MatrixStatsParser;
 import org.elasticsearch.aggregations.pipeline.BucketSelectorPipelineAggregationBuilder;
 import org.elasticsearch.aggregations.pipeline.BucketSortPipelineAggregationBuilder;
 import org.elasticsearch.aggregations.pipeline.Derivative;
@@ -48,6 +51,10 @@ public List<AggregationSpec> getAggregations() {
             ).addResultReader(InternalAutoDateHistogram::new)
                 .setAggregatorRegistrar(AutoDateHistogramAggregationBuilder::registerAggregators)
         );
+        specs.add(
+            new AggregationSpec(MatrixStatsAggregationBuilder.NAME, MatrixStatsAggregationBuilder::new, new MatrixStatsParser())
+                .addResultReader(InternalMatrixStats::new)
+        );
         if (IndexSettings.isTimeSeriesModeEnabled()) {
             specs.add(
                 new AggregationSpec(
diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java
index 8867ba732df1..db88685b5fbc 100644
--- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.aggregations.bucket.adjacency;
 
 import org.apache.lucene.search.IndexSearcher;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.aggregations.bucket.adjacency.AdjacencyMatrixAggregator.KeyedFilter;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -261,7 +261,7 @@ public String getType() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java
index 8002d1c34f60..cac35ce644bf 100644
--- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.aggregations.bucket.histogram;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Rounding;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -122,7 +122,7 @@ public AutoDateHistogramAggregationBuilder(String name) {
     public AutoDateHistogramAggregationBuilder(StreamInput in) throws IOException {
         super(in);
         numBuckets = in.readVInt();
-        if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_3_0)) {
             minimumIntervalExpression = in.readOptionalString();
         }
     }
@@ -130,7 +130,7 @@ public AutoDateHistogramAggregationBuilder(StreamInput in) throws IOException {
     @Override
     protected void innerWriteTo(StreamOutput out) throws IOException {
         out.writeVInt(numBuckets);
-        if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_3_0)) {
             out.writeOptionalString(minimumIntervalExpression);
         }
     }
@@ -267,8 +267,8 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     public static class RoundingInfo implements Writeable {
diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogram.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogram.java
index 4860bedaee61..c91a6bed8a71 100644
--- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogram.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogram.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.aggregations.bucket.histogram;
 
 import org.apache.lucene.util.PriorityQueue;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder.RoundingInfo;
 import org.elasticsearch.common.Rounding;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -226,7 +226,7 @@ public InternalAutoDateHistogram(StreamInput in) throws IOException {
         format = in.readNamedWriteable(DocValueFormat.class);
         buckets = in.readList(stream -> new Bucket(stream, format));
         this.targetBuckets = in.readVInt();
-        if (in.getVersion().onOrAfter(Version.V_8_3_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0)) {
             bucketInnerInterval = in.readVLong();
         } else {
             bucketInnerInterval = 1; // Calculated on merge.
@@ -239,7 +239,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeNamedWriteable(format);
         out.writeList(buckets);
         out.writeVInt(targetBuckets);
-        if (out.getVersion().onOrAfter(Version.V_8_3_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0)) {
             out.writeVLong(bucketInnerInterval);
         }
     }
diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java
index 088568fefc03..44fa62fef3fa 100644
--- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java
@@ -217,6 +217,9 @@ protected boolean lessThan(IteratorAndCurrent<InternalBucket> a, IteratorAndCurr
         }
 
         InternalTimeSeries reduced = new InternalTimeSeries(name, new ArrayList<>(initialCapacity), keyed, getMetadata());
+        Integer size = reduceContext.builder() instanceof TimeSeriesAggregationBuilder
+            ? ((TimeSeriesAggregationBuilder) reduceContext.builder()).getSize()
+            : null; // tests may use a fake builder
         List<InternalBucket> bucketsWithSameKey = new ArrayList<>(aggregations.size());
         BytesRef prevTsid = null;
         while (pq.size() > 0) {
@@ -240,12 +243,16 @@ protected boolean lessThan(IteratorAndCurrent<InternalBucket> a, IteratorAndCurr
             InternalBucket reducedBucket;
             if (bucketsWithSameKey.size() == 1) {
                 reducedBucket = bucketsWithSameKey.get(0);
+                reducedBucket.aggregations = InternalAggregations.reduce(List.of(reducedBucket.aggregations), reduceContext);
             } else {
                 reducedBucket = reduceBucket(bucketsWithSameKey, reduceContext);
             }
             BytesRef tsid = reducedBucket.key;
             assert prevTsid == null || tsid.compareTo(prevTsid) > 0;
             reduced.buckets.add(reducedBucket);
+            if (size != null && reduced.buckets.size() >= size) {
+                break;
+            }
             prevTsid = tsid;
         }
         return reduced;
diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationBuilder.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationBuilder.java
index 5815b0e49967..405a38237375 100644
--- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationBuilder.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationBuilder.java
@@ -8,13 +8,14 @@
 
 package org.elasticsearch.aggregations.bucket.timeseries;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.xcontent.InstantiatingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -30,9 +31,13 @@
 public class TimeSeriesAggregationBuilder extends AbstractAggregationBuilder<TimeSeriesAggregationBuilder> {
     public static final String NAME = "time_series";
     public static final ParseField KEYED_FIELD = new ParseField("keyed");
+    public static final ParseField SIZE_FIELD = new ParseField("size");
     public static final InstantiatingObjectParser<TimeSeriesAggregationBuilder, String> PARSER;
 
     private boolean keyed;
+    private int size;
+
+    private static final int DEFAULT_SIZE = MultiBucketConsumerService.DEFAULT_MAX_BUCKETS;
 
     static {
         InstantiatingObjectParser.Builder<TimeSeriesAggregationBuilder, String> parser = InstantiatingObjectParser.builder(
@@ -41,17 +46,23 @@ public class TimeSeriesAggregationBuilder extends AbstractAggregationBuilder<Tim
             TimeSeriesAggregationBuilder.class
         );
         parser.declareBoolean(optionalConstructorArg(), KEYED_FIELD);
+        parser.declareInt(optionalConstructorArg(), SIZE_FIELD);
         PARSER = parser.build();
     }
 
     public TimeSeriesAggregationBuilder(String name) {
-        this(name, true);
+        this(name, true, DEFAULT_SIZE);
     }
 
-    @ParserConstructor
     public TimeSeriesAggregationBuilder(String name, Boolean keyed) {
+        this(name, keyed, DEFAULT_SIZE);
+    }
+
+    @ParserConstructor
+    public TimeSeriesAggregationBuilder(String name, Boolean keyed, Integer size) {
         super(name);
         this.keyed = keyed != null ? keyed : true;
+        this.size = size != null ? size : DEFAULT_SIZE;
     }
 
     protected TimeSeriesAggregationBuilder(
@@ -61,16 +72,19 @@ protected TimeSeriesAggregationBuilder(
     ) {
         super(clone, factoriesBuilder, metadata);
         this.keyed = clone.keyed;
+        this.size = clone.size;
     }
 
     public TimeSeriesAggregationBuilder(StreamInput in) throws IOException {
         super(in);
         keyed = in.readBoolean();
+        size = in.readVInt();
     }
 
     @Override
     protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeBoolean(keyed);
+        out.writeVInt(size);
     }
 
     @Override
@@ -79,13 +93,14 @@ protected AggregatorFactory doBuild(
         AggregatorFactory parent,
         AggregatorFactories.Builder subFactoriesBuilder
     ) throws IOException {
-        return new TimeSeriesAggregationFactory(name, keyed, context, parent, subFactoriesBuilder, metadata);
+        return new TimeSeriesAggregationFactory(name, keyed, size, context, parent, subFactoriesBuilder, metadata);
     }
 
     @Override
     protected XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.field(KEYED_FIELD.getPreferredName(), keyed);
+        builder.field(SIZE_FIELD.getPreferredName(), size);
         builder.endObject();
         return builder;
     }
@@ -118,6 +133,18 @@ public void setKeyed(boolean keyed) {
         this.keyed = keyed;
     }
 
+    public TimeSeriesAggregationBuilder setSize(int size) {
+        if (size <= 0) {
+            throw new IllegalArgumentException("[size] must be greater than 0. Found [" + size + "] in [" + name + "]");
+        }
+        this.size = size;
+        return this;
+    }
+
+    public int getSize() {
+        return size;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -133,7 +160,7 @@ public int hashCode() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_1_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_1_0;
     }
 }
diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationFactory.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationFactory.java
index 881d23cd8c4f..72e390ffd124 100644
--- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationFactory.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationFactory.java
@@ -21,9 +21,12 @@ public class TimeSeriesAggregationFactory extends AggregatorFactory {
 
     private final boolean keyed;
 
+    private final int size;
+
     public TimeSeriesAggregationFactory(
         String name,
         boolean keyed,
+        int size,
         AggregationContext context,
         AggregatorFactory parent,
         AggregatorFactories.Builder subFactoriesBuilder,
@@ -31,11 +34,12 @@ public TimeSeriesAggregationFactory(
     ) throws IOException {
         super(name, context, parent, subFactoriesBuilder, metadata);
         this.keyed = keyed;
+        this.size = size;
     }
 
     @Override
     protected Aggregator createInternal(Aggregator parent, CardinalityUpperBound cardinality, Map<String, Object> metadata)
         throws IOException {
-        return new TimeSeriesAggregator(name, factories, keyed, context, parent, cardinality, metadata);
+        return new TimeSeriesAggregator(name, factories, keyed, context, parent, cardinality, metadata, size);
     }
 }
diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java
index 2d1e45183986..6930b0579a89 100644
--- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java
@@ -30,6 +30,7 @@ public class TimeSeriesAggregator extends BucketsAggregator {
 
     protected final BytesKeyedBucketOrds bucketOrds;
     private final boolean keyed;
+    private final int size;
 
     public TimeSeriesAggregator(
         String name,
@@ -38,11 +39,13 @@ public TimeSeriesAggregator(
         AggregationContext context,
         Aggregator parent,
         CardinalityUpperBound bucketCardinality,
-        Map<String, Object> metadata
+        Map<String, Object> metadata,
+        int size
     ) throws IOException {
         super(name, factories, context, parent, CardinalityUpperBound.MANY, metadata);
         this.keyed = keyed;
         bucketOrds = BytesKeyedBucketOrds.build(bigArrays(), bucketCardinality);
+        this.size = size;
     }
 
     @Override
@@ -66,6 +69,9 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I
                 );
                 bucket.bucketOrd = ordsEnum.ord();
                 buckets.add(bucket);
+                if (buckets.size() >= size) {
+                    break;
+                }
             }
             allBucketsPerOrd[ordIdx] = buckets.toArray(new InternalTimeSeries.InternalBucket[0]);
         }
@@ -92,8 +98,23 @@ protected void doClose() {
     protected LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCtx, LeafBucketCollector sub) throws IOException {
         return new LeafBucketCollectorBase(sub, null) {
 
+            // Keeping track of these fields helps to reduce time spent attempting to add bucket + tsid combos that already were added.
+            long currentTsidOrd = -1;
+            long currentBucket = -1;
+            long currentBucketOrdinal;
+
             @Override
             public void collect(int doc, long bucket) throws IOException {
+                // Naively comparing bucket against currentBucket and tsid ord to currentBucket can work really well.
+                // TimeSeriesIndexSearcher ensures that docs are emitted in tsid and timestamp order, so if tsid ordinal
+                // changes to what is stored in currentTsidOrd then that ordinal well never occur again. Same applies
+                // currentBucket if there is no parent aggregation or the immediate parent aggregation creates buckets
+                // based on @timestamp field or dimension fields (fields that make up the tsid).
+                if (currentBucket == bucket && currentTsidOrd == aggCtx.getTsidOrd()) {
+                    collectExistingBucket(sub, doc, currentBucketOrdinal);
+                    return;
+                }
+
                 long bucketOrdinal = bucketOrds.add(bucket, aggCtx.getTsid());
                 if (bucketOrdinal < 0) { // already seen
                     bucketOrdinal = -1 - bucketOrdinal;
@@ -101,6 +122,10 @@ public void collect(int doc, long bucket) throws IOException {
                 } else {
                     collectBucket(sub, doc, bucketOrdinal);
                 }
+
+                currentBucketOrdinal = bucketOrdinal;
+                currentTsidOrd = aggCtx.getTsidOrd();
+                currentBucket = bucket;
             }
         };
     }
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/ArrayValuesSource.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSource.java
similarity index 98%
rename from modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/ArrayValuesSource.java
rename to modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSource.java
index d853c06ea3e7..9a46d7120501 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/ArrayValuesSource.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSource.java
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-package org.elasticsearch.search.aggregations.matrix;
+package org.elasticsearch.aggregations.metric;
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.index.fielddata.NumericDoubleValues;
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/ArrayValuesSourceAggregationBuilder.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceAggregationBuilder.java
similarity index 99%
rename from modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/ArrayValuesSourceAggregationBuilder.java
rename to modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceAggregationBuilder.java
index 03094ad8690e..b78707bbb1f8 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/ArrayValuesSourceAggregationBuilder.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceAggregationBuilder.java
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-package org.elasticsearch.search.aggregations.matrix;
+package org.elasticsearch.aggregations.metric;
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/ArrayValuesSourceAggregatorFactory.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceAggregatorFactory.java
similarity index 98%
rename from modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/ArrayValuesSourceAggregatorFactory.java
rename to modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceAggregatorFactory.java
index 9249c31e156a..26e2110ecbb8 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/ArrayValuesSourceAggregatorFactory.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceAggregatorFactory.java
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.search.aggregations.matrix;
+package org.elasticsearch.aggregations.metric;
 
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/ArrayValuesSourceParser.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceParser.java
similarity index 89%
rename from modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/ArrayValuesSourceParser.java
rename to modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceParser.java
index 68daf3e8dbb6..eab1c2482028 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/ArrayValuesSourceParser.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceParser.java
@@ -6,14 +6,13 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.search.aggregations.matrix;
+package org.elasticsearch.aggregations.metric;
 
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.AggregationBuilder.CommonFields;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
-import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
@@ -27,36 +26,34 @@
 import java.util.List;
 import java.util.Map;
 
-public abstract class ArrayValuesSourceParser<VS extends ValuesSource> implements Aggregator.Parser {
+public abstract class ArrayValuesSourceParser implements Aggregator.Parser {
 
-    public abstract static class NumericValuesSourceParser extends ArrayValuesSourceParser<ValuesSource.Numeric> {
+    public abstract static class NumericValuesSourceParser extends ArrayValuesSourceParser {
 
         protected NumericValuesSourceParser(boolean formattable) {
-            super(formattable, CoreValuesSourceType.NUMERIC, ValueType.NUMERIC);
+            super(formattable, CoreValuesSourceType.NUMERIC);
         }
     }
 
-    public abstract static class BytesValuesSourceParser extends ArrayValuesSourceParser<ValuesSource.Bytes> {
+    public abstract static class BytesValuesSourceParser extends ArrayValuesSourceParser {
 
         protected BytesValuesSourceParser(boolean formattable) {
-            super(formattable, CoreValuesSourceType.KEYWORD, ValueType.STRING);
+            super(formattable, CoreValuesSourceType.KEYWORD);
         }
     }
 
-    public abstract static class GeoPointValuesSourceParser extends ArrayValuesSourceParser<ValuesSource.GeoPoint> {
+    public abstract static class GeoPointValuesSourceParser extends ArrayValuesSourceParser {
 
         protected GeoPointValuesSourceParser(boolean formattable) {
-            super(formattable, CoreValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
+            super(formattable, CoreValuesSourceType.GEOPOINT);
         }
     }
 
-    private boolean formattable = false;
-    private ValuesSourceType valuesSourceType = null;
-    private ValueType targetValueType = null;
+    private final boolean formattable;
+    private final ValuesSourceType valuesSourceType;
 
-    private ArrayValuesSourceParser(boolean formattable, ValuesSourceType valuesSourceType, ValueType targetValueType) {
+    private ArrayValuesSourceParser(boolean formattable, ValuesSourceType valuesSourceType) {
         this.valuesSourceType = valuesSourceType;
-        this.targetValueType = targetValueType;
         this.formattable = formattable;
     }
 
@@ -159,12 +156,7 @@ public final ArrayValuesSourceAggregationBuilder<?> parse(String aggregationName
             }
         }
 
-        ArrayValuesSourceAggregationBuilder<?> factory = createFactory(
-            aggregationName,
-            this.valuesSourceType,
-            this.targetValueType,
-            otherOptions
-        );
+        ArrayValuesSourceAggregationBuilder<?> factory = createFactory(aggregationName, this.valuesSourceType, otherOptions);
         if (fields != null) {
             factory.fields(fields);
         }
@@ -216,8 +208,6 @@ private void parseMissingAndAdd(
      *            the name of the aggregation
      * @param valuesSourceType
      *            the type of the {@link ValuesSource}
-     * @param targetValueType
-     *            the target type of the final value output by the aggregation
      * @param otherOptions
      *            a {@link Map} containing the extra options parsed by the
      *            {@link #token(String, String, XContentParser.Token, XContentParser, Map)}
@@ -227,14 +217,13 @@ private void parseMissingAndAdd(
     protected abstract ArrayValuesSourceAggregationBuilder<?> createFactory(
         String aggregationName,
         ValuesSourceType valuesSourceType,
-        ValueType targetValueType,
         Map<ParseField, Object> otherOptions
     );
 
     /**
      * Allows subclasses of {@link ArrayValuesSourceParser} to parse extra
      * parameters and store them in a {@link Map} which will later be passed to
-     * {@link #createFactory(String, ValuesSourceType, ValueType, Map)}.
+     * {@link #createFactory(String, ValuesSourceType, Map)}.
      *
      * @param aggregationName
      *            the name of the aggregation
@@ -247,7 +236,7 @@ protected abstract ArrayValuesSourceAggregationBuilder<?> createFactory(
      * @param otherOptions
      *            a {@link Map} of options to be populated by successive calls
      *            to this method which will then be passed to the
-     *            {@link #createFactory(String, ValuesSourceType, ValueType, Map)}
+     *            {@link #createFactory(String, ValuesSourceType, Map)}
      *            method
      * @return <code>true</code> if the current token was correctly parsed,
      *         <code>false</code> otherwise
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/InternalMatrixStats.java
similarity index 93%
rename from modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java
rename to modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/InternalMatrixStats.java
index f3b1db353544..c7428393b275 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/InternalMatrixStats.java
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-package org.elasticsearch.search.aggregations.matrix.stats;
+package org.elasticsearch.aggregations.metric;
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -27,7 +27,7 @@
 /**
  * Computes distribution statistics over multiple fields
  */
-public class InternalMatrixStats extends InternalAggregation implements MatrixStats {
+public class InternalMatrixStats extends InternalAggregation {
     /** per shard stats needed to compute stats */
     private final RunningStats stats;
     /** final result */
@@ -67,8 +67,7 @@ public String getWriteableName() {
         return MatrixStatsAggregationBuilder.NAME;
     }
 
-    /** get the number of documents */
-    @Override
+    /** return the total document count */
     public long getDocCount() {
         if (results != null) {
             return results.getDocCount();
@@ -79,8 +78,7 @@ public long getDocCount() {
         return stats.docCount;
     }
 
-    /** get the number of samples for the given field. == docCount - numMissing */
-    @Override
+    /** return total field count (differs from docCount if there are missing values) */
     public long getFieldCount(String field) {
         if (results == null) {
             return 0;
@@ -88,8 +86,7 @@ public long getFieldCount(String field) {
         return results.getFieldCount(field);
     }
 
-    /** get the mean for the given field */
-    @Override
+    /** return the field mean */
     public double getMean(String field) {
         if (results == null) {
             return Double.NaN;
@@ -97,8 +94,7 @@ public double getMean(String field) {
         return results.getMean(field);
     }
 
-    /** get the variance for the given field */
-    @Override
+    /** return the field variance */
     public double getVariance(String field) {
         if (results == null) {
             return Double.NaN;
@@ -106,8 +102,7 @@ public double getVariance(String field) {
         return results.getVariance(field);
     }
 
-    /** get the distribution skewness for the given field */
-    @Override
+    /** return the skewness of the distribution */
     public double getSkewness(String field) {
         if (results == null) {
             return Double.NaN;
@@ -115,8 +110,7 @@ public double getSkewness(String field) {
         return results.getSkewness(field);
     }
 
-    /** get the distribution shape for the given field */
-    @Override
+    /** return the kurtosis of the distribution */
     public double getKurtosis(String field) {
         if (results == null) {
             return Double.NaN;
@@ -124,8 +118,7 @@ public double getKurtosis(String field) {
         return results.getKurtosis(field);
     }
 
-    /** get the covariance between the two fields */
-    @Override
+    /** return the covariance between field x and field y */
     public double getCovariance(String fieldX, String fieldY) {
         if (results == null) {
             return Double.NaN;
@@ -133,8 +126,7 @@ public double getCovariance(String fieldX, String fieldY) {
         return results.getCovariance(fieldX, fieldY);
     }
 
-    /** get the correlation between the two fields */
-    @Override
+    /** return the correlation coefficient of field x and field y */
     public double getCorrelation(String fieldX, String fieldY) {
         if (results == null) {
             return Double.NaN;
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixAggregationInspectionHelper.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixAggregationInspectionHelper.java
similarity index 91%
rename from modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixAggregationInspectionHelper.java
rename to modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixAggregationInspectionHelper.java
index de1aff8913b3..6befc7e1c87e 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixAggregationInspectionHelper.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixAggregationInspectionHelper.java
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-package org.elasticsearch.search.aggregations.matrix.stats;
+package org.elasticsearch.aggregations.metric;
 
 /**
  * Counterpart to {@link org.elasticsearch.search.aggregations.support.AggregationInspectionHelper}, providing
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregationBuilder.java
similarity index 85%
rename from modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java
rename to modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregationBuilder.java
index c4c476ff65e9..d8edb19c2782 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregationBuilder.java
@@ -5,16 +5,15 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-package org.elasticsearch.search.aggregations.matrix.stats;
+package org.elasticsearch.aggregations.metric;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.matrix.ArrayValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.xcontent.ToXContent;
@@ -56,11 +55,16 @@ public boolean supportsSampling() {
      */
     public MatrixStatsAggregationBuilder(StreamInput in) throws IOException {
         super(in);
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            multiValueMode = MultiValueMode.readMultiValueModeFrom(in);
+        }
     }
 
     @Override
-    protected void innerWriteTo(StreamOutput out) {
-        // Do nothing, no extra state to write to stream
+    protected void innerWriteTo(StreamOutput out) throws IOException {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            multiValueMode.writeTo(out);
+        }
     }
 
     public MatrixStatsAggregationBuilder multiValueMode(MultiValueMode multiValueMode) {
@@ -94,7 +98,7 @@ public String getType() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregator.java
similarity index 94%
rename from modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java
rename to modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregator.java
index 6c3d41138ebc..c6ac2818f824 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregator.java
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-package org.elasticsearch.search.aggregations.matrix.stats;
+package org.elasticsearch.aggregations.metric;
 
 import org.apache.lucene.search.ScoreMode;
 import org.elasticsearch.common.util.ObjectArray;
@@ -17,7 +17,6 @@
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
-import org.elasticsearch.search.aggregations.matrix.ArrayValuesSource.NumericArrayValuesSource;
 import org.elasticsearch.search.aggregations.metrics.MetricsAggregator;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
@@ -30,7 +29,7 @@
  **/
 final class MatrixStatsAggregator extends MetricsAggregator {
     /** Multiple ValuesSource with field names */
-    private final NumericArrayValuesSource valuesSources;
+    private final ArrayValuesSource.NumericArrayValuesSource valuesSources;
 
     /** array of descriptive stats, per shard, needed to compute the correlation */
     ObjectArray<RunningStats> stats;
@@ -45,7 +44,7 @@ final class MatrixStatsAggregator extends MetricsAggregator {
     ) throws IOException {
         super(name, context, parent, metadata);
         if (valuesSources != null && valuesSources.isEmpty() == false) {
-            this.valuesSources = new NumericArrayValuesSource(valuesSources, multiValueMode);
+            this.valuesSources = new ArrayValuesSource.NumericArrayValuesSource(valuesSources, multiValueMode);
             stats = context.bigArrays().newObjectArray(1);
         } else {
             this.valuesSources = null;
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregatorFactory.java
similarity index 95%
rename from modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java
rename to modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregatorFactory.java
index 0e14bad7a909..2418aada6c26 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregatorFactory.java
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-package org.elasticsearch.search.aggregations.matrix.stats;
+package org.elasticsearch.aggregations.metric;
 
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.search.MultiValueMode;
@@ -14,7 +14,6 @@
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.CardinalityUpperBound;
-import org.elasticsearch.search.aggregations.matrix.ArrayValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/spi/MatrixStatsNamedXContentProvider.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsNamedXContentProvider.java
similarity index 84%
rename from modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/spi/MatrixStatsNamedXContentProvider.java
rename to modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsNamedXContentProvider.java
index ff3e6e06e22b..66670a3f6902 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/spi/MatrixStatsNamedXContentProvider.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsNamedXContentProvider.java
@@ -6,12 +6,10 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.search.aggregations.matrix.spi;
+package org.elasticsearch.aggregations.metric;
 
 import org.elasticsearch.plugins.spi.NamedXContentProvider;
 import org.elasticsearch.search.aggregations.Aggregation;
-import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder;
-import org.elasticsearch.search.aggregations.matrix.stats.ParsedMatrixStats;
 import org.elasticsearch.xcontent.ContextParser;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ParseField;
diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsParser.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsParser.java
new file mode 100644
index 000000000000..edf35947f07c
--- /dev/null
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsParser.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.aggregations.metric;
+
+import org.elasticsearch.search.MultiValueMode;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class MatrixStatsParser extends ArrayValuesSourceParser.NumericValuesSourceParser {
+
+    public MatrixStatsParser() {
+        super(true);
+    }
+
+    @Override
+    protected boolean token(
+        String aggregationName,
+        String currentFieldName,
+        XContentParser.Token token,
+        XContentParser parser,
+        Map<ParseField, Object> otherOptions
+    ) throws IOException {
+        if (ArrayValuesSourceAggregationBuilder.MULTIVALUE_MODE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+            if (token == XContentParser.Token.VALUE_STRING) {
+                otherOptions.put(ArrayValuesSourceAggregationBuilder.MULTIVALUE_MODE_FIELD, parser.text());
+                return true;
+            }
+        }
+        return false;
+    }
+
+    @Override
+    protected MatrixStatsAggregationBuilder createFactory(
+        String aggregationName,
+        ValuesSourceType valuesSourceType,
+        Map<ParseField, Object> otherOptions
+    ) {
+        MatrixStatsAggregationBuilder builder = new MatrixStatsAggregationBuilder(aggregationName);
+        String mode = (String) otherOptions.get(ArrayValuesSourceAggregationBuilder.MULTIVALUE_MODE_FIELD);
+        if (mode != null) {
+            builder.multiValueMode(MultiValueMode.fromString(mode));
+        }
+        return builder;
+    }
+}
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsResults.java
similarity index 99%
rename from modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java
rename to modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsResults.java
index 45dbe94e9ce8..b5b3fad8253e 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/MatrixStatsResults.java
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-package org.elasticsearch.search.aggregations.matrix.stats;
+package org.elasticsearch.aggregations.metric;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.io.stream.StreamInput;
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/ParsedMatrixStats.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ParsedMatrixStats.java
similarity index 97%
rename from modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/ParsedMatrixStats.java
rename to modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ParsedMatrixStats.java
index a89e6c7bbc30..0c44946ac96a 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/ParsedMatrixStats.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ParsedMatrixStats.java
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.search.aggregations.matrix.stats;
+package org.elasticsearch.aggregations.metric;
 
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.search.aggregations.ParsedAggregation;
@@ -21,7 +21,7 @@
 import java.util.Map;
 import java.util.Objects;
 
-public class ParsedMatrixStats extends ParsedAggregation implements MatrixStats {
+public class ParsedMatrixStats extends ParsedAggregation {
 
     private final Map<String, Long> counts = new LinkedHashMap<>();
     private final Map<String, Double> means = new HashMap<>();
@@ -42,12 +42,10 @@ private void setDocCount(long docCount) {
         this.docCount = docCount;
     }
 
-    @Override
     public long getDocCount() {
         return docCount;
     }
 
-    @Override
     public long getFieldCount(String field) {
         if (counts.containsKey(field) == false) {
             return 0;
@@ -55,27 +53,22 @@ public long getFieldCount(String field) {
         return counts.get(field);
     }
 
-    @Override
     public double getMean(String field) {
         return checkedGet(means, field);
     }
 
-    @Override
     public double getVariance(String field) {
         return checkedGet(variances, field);
     }
 
-    @Override
     public double getSkewness(String field) {
         return checkedGet(skewness, field);
     }
 
-    @Override
     public double getKurtosis(String field) {
         return checkedGet(kurtosis, field);
     }
 
-    @Override
     public double getCovariance(String fieldX, String fieldY) {
         if (fieldX.equals(fieldY)) {
             return checkedGet(variances, fieldX);
@@ -83,7 +76,6 @@ public double getCovariance(String fieldX, String fieldY) {
         return MatrixStatsResults.getValFromUpperTriangularMatrix(covariances, fieldX, fieldY);
     }
 
-    @Override
     public double getCorrelation(String fieldX, String fieldY) {
         if (fieldX.equals(fieldY)) {
             return 1.0;
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStats.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/RunningStats.java
similarity index 99%
rename from modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStats.java
rename to modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/RunningStats.java
index 5ec08c68ed98..e4b8e15cd5e1 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStats.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/RunningStats.java
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-package org.elasticsearch.search.aggregations.matrix.stats;
+package org.elasticsearch.aggregations.metric;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.io.stream.StreamInput;
diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java
index 15bd818ca98b..2271bbe44d3f 100644
--- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.aggregations.pipeline;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -214,7 +214,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java
index 2bd65c6b0a99..80f9c66814f6 100644
--- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.aggregations.pipeline;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
@@ -193,7 +193,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/DerivativePipelineAggregationBuilder.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/DerivativePipelineAggregationBuilder.java
index f4838441938e..32a8b5897ae8 100644
--- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/DerivativePipelineAggregationBuilder.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/DerivativePipelineAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.aggregations.pipeline;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Rounding;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -256,7 +256,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_4_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_4_0;
     }
 }
diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/MovFnPipelineAggregationBuilder.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/MovFnPipelineAggregationBuilder.java
index 8c721ae2e179..821869424c44 100644
--- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/MovFnPipelineAggregationBuilder.java
+++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/pipeline/MovFnPipelineAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.aggregations.pipeline;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -239,7 +239,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/modules/aggregations/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider b/modules/aggregations/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider
new file mode 100644
index 000000000000..c44951e05c94
--- /dev/null
+++ b/modules/aggregations/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider
@@ -0,0 +1,9 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+org.elasticsearch.aggregations.metric.MatrixStatsNamedXContentProvider
diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java
index a64d52b1ed7c..c44242556c4d 100644
--- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java
+++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/AutoDateHistogramAggregatorTests.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.BooleanFieldMapper;
 import org.elasticsearch.index.mapper.DateFieldMapper;
@@ -859,7 +860,7 @@ public void testIntervalMinute() throws IOException {
         fullDocCount.clear();
         fullDocCount.putAll(skeletonDocCount);
         for (int minute = 3; minute < 15; minute++) {
-            fullDocCount.put(formatted("2017-02-01T09:%02d:00.000Z", minute), 0);
+            fullDocCount.put(Strings.format("2017-02-01T09:%02d:00.000Z", minute), 0);
         }
         testSearchCase(
             DEFAULT_QUERY,
diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java
index 943e7f43c754..1ed6fa058e64 100644
--- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java
+++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java
@@ -171,4 +171,9 @@ public void testReduceSimple() {
         assertThat(result.getBuckets().get(4).key.utf8ToString(), equalTo("9"));
         assertThat(result.getBuckets().get(4).getDocCount(), equalTo(9L));
     }
+
+    @Override
+    protected InternalTimeSeries mutateInstance(InternalTimeSeries instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationBuilderTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationBuilderTests.java
index 2c87c9f298dc..eb738dc3fe82 100644
--- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationBuilderTests.java
+++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregationBuilderTests.java
@@ -14,7 +14,8 @@ public class TimeSeriesAggregationBuilderTests extends AggregationBuilderTestCas
 
     @Override
     protected TimeSeriesAggregationBuilder createTestAggregatorBuilder() {
-        return new TimeSeriesAggregationBuilder(randomAlphaOfLength(10), randomBoolean());
+        // Size set large enough tests not intending to hit the size limit shouldn't see it.
+        return new TimeSeriesAggregationBuilder(randomAlphaOfLength(10), randomBoolean(), randomIntBetween(1000, 100_000));
     }
 
 }
diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java
index b98e464daefb..20d2b45158b3 100644
--- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java
+++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java
@@ -10,6 +10,7 @@
 
 import org.apache.lucene.document.DoubleDocValuesField;
 import org.apache.lucene.document.FloatDocValuesField;
+import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.SortedNumericDocValuesField;
@@ -26,6 +27,10 @@
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper;
 import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper.TimeSeriesIdBuilder;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
+import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogram;
 import org.elasticsearch.search.aggregations.metrics.Sum;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
@@ -35,6 +40,7 @@
 import java.util.function.Consumer;
 
 import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
+import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 
@@ -77,6 +83,7 @@ public void testStandAloneTimeSeriesWithSum() throws IOException {
     public static void writeTS(RandomIndexWriter iw, long timestamp, Object[] dimensions, Object[] metrics) throws IOException {
         final List<IndexableField> fields = new ArrayList<>();
         fields.add(new SortedNumericDocValuesField(DataStreamTimestampFieldMapper.DEFAULT_PATH, timestamp));
+        fields.add(new LongPoint(DataStreamTimestampFieldMapper.DEFAULT_PATH, timestamp));
         final TimeSeriesIdBuilder builder = new TimeSeriesIdBuilder(null);
         for (int i = 0; i < dimensions.length; i += 2) {
             if (dimensions[i + 1]instanceof Number n) {
@@ -99,6 +106,119 @@ public static void writeTS(RandomIndexWriter iw, long timestamp, Object[] dimens
         iw.addDocument(fields);
     }
 
+    public void testWithDateHistogramExecutedAsFilterByFilterWithTimeSeriesIndexSearcher() throws IOException {
+        DateHistogramAggregationBuilder aggregationBuilder = new DateHistogramAggregationBuilder("by_timestamp").field("@timestamp")
+            .fixedInterval(DateHistogramInterval.HOUR)
+            .subAggregation(new TimeSeriesAggregationBuilder("ts").subAggregation(sum("sum").field("val1")));
+
+        // Before this threw a CollectionTerminatedException because FilterByFilterAggregation#getLeafCollector() always returns a
+        // LeafBucketCollector.NO_OP_COLLECTOR instance. And TimeSeriesIndexSearcher can't deal with this when initializing the
+        // leaf walkers.
+        testCase(iw -> {
+            long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z");
+            for (int i = 1; i <= 5000; i++) {
+                writeTS(iw, startTime++, new Object[] { "dim1", "aaa" }, new Object[] { "val1", 1 });
+            }
+        }, internalAggregation -> {
+            InternalDateHistogram dateHistogram = (InternalDateHistogram) internalAggregation;
+            assertThat(dateHistogram.getBuckets(), hasSize(1));
+            InternalTimeSeries timeSeries = dateHistogram.getBuckets().get(0).getAggregations().get("ts");
+            assertThat(timeSeries.getBuckets(), hasSize(1));
+            Sum sum = timeSeries.getBuckets().get(0).getAggregations().get("sum");
+            assertThat(sum.value(), equalTo(5000.0));
+        },
+            new AggTestConfig(
+                aggregationBuilder,
+                TimeSeriesIdFieldMapper.FIELD_TYPE,
+                new DateFieldMapper.DateFieldType("@timestamp"),
+                new KeywordFieldMapper.KeywordFieldType("dim1"),
+                new NumberFieldMapper.NumberFieldType("val1", NumberFieldMapper.NumberType.INTEGER)
+            ).withQuery(new MatchAllDocsQuery())
+        );
+    }
+
+    public void testMultiBucketAggregationAsSubAggregation() throws IOException {
+        long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z");
+        CheckedConsumer<RandomIndexWriter, IOException> buildIndex = iw -> {
+            writeTS(iw, startTime + 1, new Object[] { "dim1", "aaa", "dim2", "xxx" }, new Object[] {});
+            writeTS(iw, startTime + 2, new Object[] { "dim1", "aaa", "dim2", "yyy" }, new Object[] {});
+            writeTS(iw, startTime + 3, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] {});
+            writeTS(iw, startTime + 4, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] {});
+            writeTS(iw, startTime + 5, new Object[] { "dim1", "aaa", "dim2", "xxx" }, new Object[] {});
+            writeTS(iw, startTime + 6, new Object[] { "dim1", "aaa", "dim2", "yyy" }, new Object[] {});
+            writeTS(iw, startTime + 7, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] {});
+            writeTS(iw, startTime + 8, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] {});
+        };
+        Consumer<InternalTimeSeries> verifier = ts -> {
+            assertThat(ts.getBuckets(), hasSize(3));
+
+            assertThat(ts.getBucketByKey("{dim1=aaa, dim2=xxx}").docCount, equalTo(2L));
+            InternalDateHistogram byTimeStampBucket = ts.getBucketByKey("{dim1=aaa, dim2=xxx}").getAggregations().get("by_timestamp");
+            assertThat(
+                byTimeStampBucket.getBuckets(),
+                contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY))
+            );
+            assertThat(ts.getBucketByKey("{dim1=aaa, dim2=yyy}").docCount, equalTo(2L));
+            byTimeStampBucket = ts.getBucketByKey("{dim1=aaa, dim2=yyy}").getAggregations().get("by_timestamp");
+            assertThat(
+                byTimeStampBucket.getBuckets(),
+                contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY))
+            );
+            assertThat(ts.getBucketByKey("{dim1=bbb, dim2=zzz}").docCount, equalTo(4L));
+            byTimeStampBucket = ts.getBucketByKey("{dim1=bbb, dim2=zzz}").getAggregations().get("by_timestamp");
+            assertThat(
+                byTimeStampBucket.getBuckets(),
+                contains(new InternalDateHistogram.Bucket(startTime, 4, false, null, InternalAggregations.EMPTY))
+            );
+        };
+
+        DateHistogramAggregationBuilder dateBuilder = new DateHistogramAggregationBuilder("by_timestamp");
+        dateBuilder.field("@timestamp");
+        dateBuilder.fixedInterval(DateHistogramInterval.seconds(1));
+        TimeSeriesAggregationBuilder tsBuilder = new TimeSeriesAggregationBuilder("by_tsid");
+        tsBuilder.subAggregation(dateBuilder);
+        timeSeriesTestCase(tsBuilder, new MatchAllDocsQuery(), buildIndex, verifier);
+    }
+
+    public void testAggregationSize() throws IOException {
+        CheckedConsumer<RandomIndexWriter, IOException> buildIndex = multiTsWriter();
+
+        List<Consumer<InternalTimeSeries>> verifiers = new ArrayList<Consumer<InternalTimeSeries>>();
+
+        verifiers.add(ts -> assertThat(ts.getBucketByKey("{dim1=aaa, dim2=xxx}").docCount, equalTo(2L)));
+        verifiers.add(ts -> assertThat(ts.getBucketByKey("{dim1=aaa, dim2=yyy}").docCount, equalTo(2L)));
+        verifiers.add(ts -> assertThat(ts.getBucketByKey("{dim1=bbb, dim2=zzz}").docCount, equalTo(2L)));
+
+        for (int i = 1; i < 3; i++) {
+            int size = i;
+            Consumer<InternalTimeSeries> limitedVerifier = ts -> {
+                assertThat(ts.getBuckets(), hasSize(size));
+
+                for (int j = 0; j < size; j++) {
+                    verifiers.get(j).accept(ts);
+                }
+            };
+
+            TimeSeriesAggregationBuilder limitedTsBuilder = new TimeSeriesAggregationBuilder("by_tsid");
+            limitedTsBuilder.setSize(i);
+            timeSeriesTestCase(limitedTsBuilder, new MatchAllDocsQuery(), buildIndex, limitedVerifier);
+        }
+    }
+
+    private CheckedConsumer<RandomIndexWriter, IOException> multiTsWriter() {
+        long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z");
+        return iw -> {
+            writeTS(iw, startTime + 1, new Object[] { "dim1", "aaa", "dim2", "xxx" }, new Object[] { "val1", 1 });
+            writeTS(iw, startTime + 2, new Object[] { "dim1", "aaa", "dim2", "yyy" }, new Object[] { "val1", 2 });
+            writeTS(iw, startTime + 3, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] { "val1", 3 });
+            writeTS(iw, startTime + 4, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] { "val1", 4 });
+            writeTS(iw, startTime + 5, new Object[] { "dim1", "aaa", "dim2", "xxx" }, new Object[] { "val1", 5 });
+            writeTS(iw, startTime + 6, new Object[] { "dim1", "aaa", "dim2", "yyy" }, new Object[] { "val1", 6 });
+            writeTS(iw, startTime + 7, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] { "val1", 7 });
+            writeTS(iw, startTime + 8, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] { "val1", 8 });
+        };
+    }
+
     private void timeSeriesTestCase(
         TimeSeriesAggregationBuilder builder,
         Query query,
diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/BaseMatrixStatsTestCase.java
similarity index 95%
rename from modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java
rename to modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/BaseMatrixStatsTestCase.java
index 2e46dd436fa5..9cbebb4dab1e 100644
--- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java
+++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/BaseMatrixStatsTestCase.java
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-package org.elasticsearch.search.aggregations.matrix.stats;
+package org.elasticsearch.aggregations.metric;
 
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/InternalMatrixStatsTests.java
similarity index 83%
rename from modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java
rename to modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/InternalMatrixStatsTests.java
index 0dfbf854bc00..9ca315b5a7b7 100644
--- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStatsTests.java
+++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/InternalMatrixStatsTests.java
@@ -5,8 +5,10 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-package org.elasticsearch.search.aggregations.matrix.stats;
+package org.elasticsearch.aggregations.metric;
 
+import org.elasticsearch.aggregations.AggregationsPlugin;
+import org.elasticsearch.aggregations.metric.InternalMatrixStats.Fields;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.util.Maps;
@@ -20,8 +22,6 @@
 import org.elasticsearch.search.aggregations.AggregationReduceContext;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.ParsedAggregation;
-import org.elasticsearch.search.aggregations.matrix.MatrixAggregationPlugin;
-import org.elasticsearch.search.aggregations.matrix.stats.InternalMatrixStats.Fields;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
 import org.elasticsearch.test.InternalAggregationTestCase;
 import org.elasticsearch.xcontent.ContextParser;
@@ -46,7 +46,7 @@ public class InternalMatrixStatsTests extends InternalAggregationTestCase<Intern
 
     @Override
     protected SearchPlugin registerPlugin() {
-        return new MatrixAggregationPlugin();
+        return new AggregationsPlugin();
     }
 
     @Override
@@ -195,27 +195,23 @@ protected void assertFromXContent(InternalMatrixStats expected, ParsedAggregatio
             }
         }
 
-        final String unknownField = randomAlphaOfLength(3);
-        final String other = randomValueOtherThan(unknownField, () -> randomAlphaOfLength(3));
+        String unknownField = randomAlphaOfLength(3);
+        String other = randomValueOtherThan(unknownField, () -> randomAlphaOfLength(3));
+        // getFieldCount returns 0 for unknown fields
+        assertEquals(0.0, actual.getFieldCount(unknownField), 0.0);
 
-        for (MatrixStats matrix : Arrays.asList(actual)) {
+        expectThrows(IllegalArgumentException.class, () -> actual.getMean(unknownField));
+        expectThrows(IllegalArgumentException.class, () -> actual.getVariance(unknownField));
+        expectThrows(IllegalArgumentException.class, () -> actual.getSkewness(unknownField));
+        expectThrows(IllegalArgumentException.class, () -> actual.getKurtosis(unknownField));
 
-            // getFieldCount returns 0 for unknown fields
-            assertEquals(0.0, matrix.getFieldCount(unknownField), 0.0);
+        expectThrows(IllegalArgumentException.class, () -> actual.getCovariance(unknownField, unknownField));
+        expectThrows(IllegalArgumentException.class, () -> actual.getCovariance(unknownField, other));
+        expectThrows(IllegalArgumentException.class, () -> actual.getCovariance(other, unknownField));
 
-            expectThrows(IllegalArgumentException.class, () -> matrix.getMean(unknownField));
-            expectThrows(IllegalArgumentException.class, () -> matrix.getVariance(unknownField));
-            expectThrows(IllegalArgumentException.class, () -> matrix.getSkewness(unknownField));
-            expectThrows(IllegalArgumentException.class, () -> matrix.getKurtosis(unknownField));
-
-            expectThrows(IllegalArgumentException.class, () -> matrix.getCovariance(unknownField, unknownField));
-            expectThrows(IllegalArgumentException.class, () -> matrix.getCovariance(unknownField, other));
-            expectThrows(IllegalArgumentException.class, () -> matrix.getCovariance(other, unknownField));
-
-            assertEquals(1.0, matrix.getCorrelation(unknownField, unknownField), 0.0);
-            expectThrows(IllegalArgumentException.class, () -> matrix.getCorrelation(unknownField, other));
-            expectThrows(IllegalArgumentException.class, () -> matrix.getCorrelation(other, unknownField));
-        }
+        assertEquals(1.0, actual.getCorrelation(unknownField, unknownField), 0.0);
+        expectThrows(IllegalArgumentException.class, () -> actual.getCorrelation(unknownField, other));
+        expectThrows(IllegalArgumentException.class, () -> actual.getCorrelation(other, unknownField));
     }
 
     @Override
diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregatorTests.java
similarity index 91%
rename from modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorTests.java
rename to modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregatorTests.java
index d40aa1de5385..92dfb03ff301 100644
--- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorTests.java
+++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/MatrixStatsAggregatorTests.java
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-package org.elasticsearch.search.aggregations.matrix.stats;
+package org.elasticsearch.aggregations.metric;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
@@ -16,17 +16,14 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.apache.lucene.util.NumericUtils;
+import org.elasticsearch.aggregations.bucket.AggregationTestCase;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
-import org.elasticsearch.plugins.SearchPlugin;
-import org.elasticsearch.search.aggregations.AggregatorTestCase;
-import org.elasticsearch.search.aggregations.matrix.MatrixAggregationPlugin;
 
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.List;
 
-public class MatrixStatsAggregatorTests extends AggregatorTestCase {
+public class MatrixStatsAggregatorTests extends AggregationTestCase {
 
     public void testNoData() throws Exception {
         MappedFieldType ft = new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.DOUBLE);
@@ -100,9 +97,4 @@ public void testTwoFields() throws Exception {
             }
         }
     }
-
-    @Override
-    protected List<SearchPlugin> getSearchPlugins() {
-        return Collections.singletonList(new MatrixAggregationPlugin());
-    }
 }
diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MultiPassStats.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/MultiPassStats.java
similarity index 99%
rename from modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MultiPassStats.java
rename to modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/MultiPassStats.java
index 0bc3685f8cf8..6a43f02697e2 100644
--- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MultiPassStats.java
+++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/MultiPassStats.java
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-package org.elasticsearch.search.aggregations.matrix.stats;
+package org.elasticsearch.aggregations.metric;
 
 import org.elasticsearch.common.util.Maps;
 
diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStatsTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/RunningStatsTests.java
similarity index 98%
rename from modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStatsTests.java
rename to modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/RunningStatsTests.java
index 36d29be98c35..76eda2a8dca1 100644
--- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStatsTests.java
+++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/metric/RunningStatsTests.java
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-package org.elasticsearch.search.aggregations.matrix.stats;
+package org.elasticsearch.aggregations.metric;
 
 import java.util.Arrays;
 import java.util.HashSet;
diff --git a/modules/aggregations/src/yamlRestTest/java/org/elasticsearch/aggregations/AggregationsClientYamlTestSuiteIT.java b/modules/aggregations/src/yamlRestTest/java/org/elasticsearch/aggregations/AggregationsClientYamlTestSuiteIT.java
index a0df2af9d59e..a3c737e2795d 100644
--- a/modules/aggregations/src/yamlRestTest/java/org/elasticsearch/aggregations/AggregationsClientYamlTestSuiteIT.java
+++ b/modules/aggregations/src/yamlRestTest/java/org/elasticsearch/aggregations/AggregationsClientYamlTestSuiteIT.java
@@ -10,10 +10,20 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.ClassRule;
 
 public class AggregationsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .module("aggregations")
+        .module("lang-painless")
+        .feature(FeatureFlag.TIME_SERIES_MODE)
+        .build();
+
     public AggregationsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
@@ -22,4 +32,9 @@ public AggregationsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate t
     public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
 }
diff --git a/modules/aggs-matrix-stats/src/yamlRestTest/resources/rest-api-spec/test/stats/20_empty_bucket.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/matrix_stats.yml
similarity index 100%
rename from modules/aggs-matrix-stats/src/yamlRestTest/resources/rest-api-spec/test/stats/20_empty_bucket.yml
rename to modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/matrix_stats.yml
diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/matrix_stats_multi_value_field.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/matrix_stats_multi_value_field.yml
new file mode 100644
index 000000000000..2e2b9a94ee46
--- /dev/null
+++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/matrix_stats_multi_value_field.yml
@@ -0,0 +1,200 @@
+---
+setup:
+  - skip:
+      version: " - 8.6.99"
+      reason: serialization bug fixed in 8.7.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 3
+            number_of_routing_shards: 3
+          mappings:
+            "properties":
+              "val1":
+                "type": "double"
+              "val2":
+                "type": "double"
+              "val3":
+                "type": "double"
+
+  - do:
+      indices.create:
+        index: unmapped
+        body:
+          settings:
+            number_of_shards: 3
+
+  - do:
+      index:
+        index:  test
+        id:     "1"
+        body:   { "val1": 1.9, "val2": 3.1, "val3": 2.3, "vals" : [1.9, 16.143] }
+  - do:
+      index:
+        index:  test
+        id:     "2"
+        body:   { "val1": -5.2, "val2": -3.4, "val3": 2.3, "vals" : [155, 16.23]}
+  - do:
+      index:
+        index:  test
+        id:     "3"
+        body:   { "val1": -5.2, "val3": 2.3, "vals" : [-455, -32.32]}
+  - do:
+      index:
+        index:  test
+        id:     "4"
+        body:   { "val1": 18.3, "val2": 104.4, "val3": 2.3, "vals" : [0.14, 92.1]}
+  - do:
+      index:
+        index:  test
+        id:     "5"
+        body:   { "val1": -53.2, "val2": -322.4, "val3": 2.3, "vals" : [16, 16]}
+  - do:
+      index:
+        index:  test
+        id:     "6"
+        body:   { "val1": -578.9, "val2": 69.9, "val3": 2.3}
+  - do:
+      index:
+        index:  test
+        id:     "7"
+        body:   { "val1": 16.2, "val2": 17.2, "val3": 2.3, "vals" : [1234.3, -3433]}
+  - do:
+      index:
+        index:  test
+        id:     "8"
+        body:   { "val1": -4222.63, "val2": 316.44, "val3": 2.3, "vals" : [177.2, -93.333]}
+  - do:
+      index:
+        index:  test
+        id:     "9"
+        body:   { "val1": -59999.55, "val2": -3163.4, "val3": 2.3, "vals" : [-29.9, 163.0]}
+  - do:
+      index:
+        index:  test
+        id:     "10"
+        body:   { "val1": 782.7, "val2": 789.7, "val3": 2.3, "vals" : [-0.2, 1343.3]}
+  - do:
+      index:
+        index:  test
+        id:     "11"
+        body:   { "val1": -1.2, "val2": 6.3, "val3": 2.3, "vals" : [15.3, 16.9]}
+  - do:
+      index:
+        index:  test
+        id:     "12"
+        body:   { "val1": 0, "val2": 1.11, "val3": 2.3, "vals" : [-644.4, -644.4]}
+  - do:
+      index:
+        index:  test
+        id:     "13"
+        body:   { "val1": 0.1, "val2": 0.92, "val3": 2.3, "vals" : [73.2, 0.12]}
+  - do:
+      index:
+        index:  test
+        id:     "14"
+        body:   { "val1": 0.12, "val2": -82.4, "val3": 2.3, "vals" : [-0.001, 1295.3]}
+  - do:
+      index:
+        index:  test
+        id:     "15"
+        body:   { "val1": 98.2, "val2": 32.4, "val3": 2.3, "vals" : [15.5, 16.5]}
+
+  - do:
+      indices.refresh:
+        index: [test, unmapped]
+
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+
+---
+"Unmapped":
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: unmapped
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2", "vals"]} } } }
+
+  - match: {hits.total: 0}
+  - match: {aggregations.mfs.doc_count: 0}
+
+---
+"Multi value field Max":
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "vals"], "mode" : "max"} } } }
+
+  - match: {hits.total: 15}
+  - match: {aggregations.mfs.doc_count: 14}
+  - match: {aggregations.mfs.fields.0.count: 14}
+  - match: {aggregations.mfs.fields.0.correlation.val1: 0.06838646533369998}
+
+---
+"Multi value field Min":
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "vals"], "mode" : "min"} } } }
+
+  - match: {hits.total: 15}
+  - match: {aggregations.mfs.doc_count: 14}
+  - match: {aggregations.mfs.fields.0.count: 14}
+  - match: {aggregations.mfs.fields.0.correlation.val1: -0.09777682707831963}
+
+---
+"Partially unmapped":
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: [test, unmapped]
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2", "vals"]} } } }
+
+  - match: {hits.total: 15}
+  - match: {aggregations.mfs.doc_count: 13}
+  - match: {aggregations.mfs.fields.0.count: 13}
+  - match: {aggregations.mfs.fields.0.correlation.val1: -0.044997535185684244}
+
+---
+"Partially unmapped with missing defaults":
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: [test, unmapped]
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2", "vals"], "missing" : {"val2" : 10, "vals" : 5 } } } } }
+
+  - match: {hits.total: 15}
+  - match: {aggregations.mfs.doc_count: 15}
+  - match: {aggregations.mfs.fields.0.count: 15}
+  - match: {aggregations.mfs.fields.0.correlation.val2: 0.04028024709708195}
+
+---
+"With script":
+
+  - do:
+      catch: /parsing_exception/
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["vals", "val3"], "script" : { "my_script" : {"source" : "1 + doc['val1'].value", "lang" : "js"} } } } } }
+
+---
+"With script params":
+
+  - do:
+      catch: /parsing_exception/
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val3", "vals"], "script" : { "my_script" : {"source" : "my_var + doc['val1'].value", "params" : { "my_var" : 1 }, "lang" : "js" } } } } } }
diff --git a/modules/aggs-matrix-stats/src/yamlRestTest/resources/rest-api-spec/test/stats/30_single_value_field.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/matrix_stats_single_value_field.yml
similarity index 100%
rename from modules/aggs-matrix-stats/src/yamlRestTest/resources/rest-api-spec/test/stats/30_single_value_field.yml
rename to modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/matrix_stats_single_value_field.yml
diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/max_metric.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/max_metric.yml
index 551e0c01d128..0163df5bbaeb 100644
--- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/max_metric.yml
+++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/max_metric.yml
@@ -221,3 +221,45 @@ setup:
 
   - match: { aggregations.date_field_max.value: 1619827200000 }
   - match: { aggregations.date_field_max.value_as_string: "2021-05-01T00:00:00.000Z" }
+
+---
+"Counter field":
+  - skip:
+      version: " - 8.6.99"
+      reason: "counter field support added in 8.7"
+
+  - do:
+      indices.create:
+        index: myindex
+        body:
+          mappings:
+            properties:
+              counter_field:
+                type : long
+                time_series_metric: counter
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: myindex
+              _id:    "1"
+          - counter_field: 2
+          - index:
+              _index: myindex
+              _id:    "2"
+          - counter_field: 4
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: myindex
+        body:
+          aggs:
+            the_counter_max:
+              max:
+                field: counter_field
+
+  - match: { hits.total: 2 }
+  - length: { hits.hits: 2 }
+  - match: { aggregations.the_counter_max.value: 4 }
diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/min_metric.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/min_metric.yml
index 30c226f554c3..690b3d240096 100644
--- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/min_metric.yml
+++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/min_metric.yml
@@ -175,3 +175,45 @@ setup:
             the_string_min:
               min:
                 field: string_field
+
+---
+"Counter field":
+  - skip:
+      version: " - 8.6.99"
+      reason: "counter field support added in 8.7"
+
+  - do:
+      indices.create:
+        index: myindex
+        body:
+          mappings:
+            properties:
+              counter_field:
+                type : long
+                time_series_metric: counter
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: myindex
+              _id:    "1"
+          - counter_field: 2
+          - index:
+              _index: myindex
+              _id:    "2"
+          - counter_field: 4
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: myindex
+        body:
+          aggs:
+            the_counter_min:
+              min:
+                field: counter_field
+
+  - match: { hits.total: 2 }
+  - length: { hits.hits: 2 }
+  - match: { aggregations.the_counter_min.value: 2 }
diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml
index 8a8ee0a0f0e0..01829a5c1204 100644
--- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml
+++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml
@@ -54,7 +54,7 @@ setup:
 ---
 "Basic test":
   - skip:
-      version: " - 8.5.99"
+      version: " - 8.6.99"
       reason: Time series result serialization changed in 8.6.0
 
   - do:
@@ -74,8 +74,140 @@ setup:
 
 
   - match: { hits.total.value: 1 }
-  - length: { aggregations.ts.buckets: 1 }
+  - length: { aggregations: 1 }
 
   - match: { aggregations.ts.buckets.0.key: { "key": "foo" } }
   - match: { aggregations.ts.buckets.0.doc_count: 1 }
 
+---
+"Size test":
+  - skip:
+      version: " - 8.6.99"
+      reason: Size added in 8.7.0
+
+  - do:
+      search:
+        index: tsdb
+        body:
+          query:
+            range:
+              "@timestamp":
+                gte: "2019-01-01T00:10:00Z"
+          size: 0
+          aggs:
+            ts:
+              time_series:
+                keyed: false
+                size: 1
+
+  - length: { aggregations.ts.buckets: 1 }
+  - match: { aggregations.ts.buckets.0.key: { "key": "bar" } }
+
+  - do:
+      search:
+        index: tsdb
+        body:
+          query:
+            range:
+              "@timestamp":
+                gte: "2019-01-01T00:10:00Z"
+          size: 0
+          aggs:
+            ts:
+              time_series:
+                keyed: false
+                size: 3
+
+  - length: { aggregations.ts.buckets: 3 }
+  - match: { aggregations.ts.buckets.0.key: { "key": "bar" } }
+  - match: { aggregations.ts.buckets.1.key: { "key": "baz" } }
+  - match: { aggregations.ts.buckets.2.key: { "key": "foo" } }
+
+---
+"Sampler aggregation with nested time series aggregation failure":
+  - skip:
+      version: " - 8.6.99"
+      reason: "Handling for time series aggregation failures introduced in 8.7.0"
+
+  - do:
+      catch: '/\[random_sampler\] aggregation \[sample\] does not support sampling \[time_series\] aggregation \[ts\]/'
+      search:
+        index: tsdb
+        body:
+          aggs:
+            sample:
+              random_sampler:
+                probability: 1.0
+              aggs:
+                by_timestamp:
+                  date_histogram:
+                    field: "@timestamp"
+                    fixed_interval: 1h
+                  aggs:
+                    ts:
+                      time_series:
+                        keyed: false
+                      aggs:
+                        sum:
+                          sum:
+                            field: val
+
+---
+"Composite aggregation with nested time series aggregation failure":
+  - skip:
+      version: " - 8.6.99"
+      reason: "Handling for time series aggregation failures introduced in 8.7.0"
+
+  - do:
+      catch: '/\[composite\] aggregation is incompatible with time series execution mode/'
+      search:
+        index: tsdb
+        body:
+          aggs:
+            by_key:
+              composite:
+                sources: [
+                  {
+                    "key": {
+                      "terms": {
+                        "field": "key"
+                      }
+                    }
+                  }
+                ]
+              aggs:
+                date:
+                  date_histogram:
+                    field: "@timestamp"
+                    fixed_interval: "1h"
+                  aggs:
+                    ts:
+                      time_series:
+                        keyed: false
+                      aggs:
+                        sum:
+                          sum:
+                            field: val
+
+---
+"Global aggregation with nested time series aggregation failure":
+  - skip:
+      version: " - 8.6.99"
+      reason: "Handling for time series aggregation failures introduced in 8.7.0"
+
+  - do:
+      catch: '/Time series aggregations cannot be used inside global aggregation./'
+      search:
+        index: tsdb
+        body:
+          aggs:
+            global:
+              global: {}
+              aggs:
+                ts:
+                  time_series:
+                    keyed: false
+                  aggs:
+                    sum:
+                      sum:
+                        field: val
diff --git a/modules/aggs-matrix-stats/build.gradle b/modules/aggs-matrix-stats/build.gradle
deleted file mode 100644
index e0621544c446..000000000000
--- a/modules/aggs-matrix-stats/build.gradle
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
-
-esplugin {
-  description 'Adds aggregations whose input are a list of numeric fields and output includes a matrix.'
-  classname 'org.elasticsearch.search.aggregations.matrix.MatrixAggregationPlugin'
-}
-
-restResources {
-  restApi {
-    include '_common', 'indices', 'cluster', 'index', 'search', 'nodes'
-  }
-}
diff --git a/modules/aggs-matrix-stats/src/main/java/module-info.java b/modules/aggs-matrix-stats/src/main/java/module-info.java
deleted file mode 100644
index fa711a3b2b05..000000000000
--- a/modules/aggs-matrix-stats/src/main/java/module-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-module org.elasticsearch.aggs.matrix {
-    requires org.elasticsearch.base;
-    requires org.elasticsearch.server;
-    requires org.elasticsearch.xcontent;
-    requires org.apache.lucene.core;
-
-    exports org.elasticsearch.search.aggregations.matrix;
-    exports org.elasticsearch.search.aggregations.matrix.stats;
-
-    provides org.elasticsearch.plugins.spi.NamedXContentProvider
-        with
-            org.elasticsearch.search.aggregations.matrix.spi.MatrixStatsNamedXContentProvider;
-}
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationPlugin.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationPlugin.java
deleted file mode 100644
index 57ff7d227cee..000000000000
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationPlugin.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.search.aggregations.matrix;
-
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.SearchPlugin;
-import org.elasticsearch.search.aggregations.matrix.stats.InternalMatrixStats;
-import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder;
-import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsParser;
-
-import java.util.List;
-
-import static java.util.Collections.singletonList;
-
-public class MatrixAggregationPlugin extends Plugin implements SearchPlugin {
-    @Override
-    public List<AggregationSpec> getAggregations() {
-        return singletonList(
-            new AggregationSpec(MatrixStatsAggregationBuilder.NAME, MatrixStatsAggregationBuilder::new, new MatrixStatsParser())
-                .addResultReader(InternalMatrixStats::new)
-        );
-    }
-}
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsAggregationBuilders.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsAggregationBuilders.java
deleted file mode 100644
index 735a839bf357..000000000000
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsAggregationBuilders.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.search.aggregations.matrix;
-
-import org.elasticsearch.search.aggregations.matrix.stats.MatrixStats;
-import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder;
-
-public class MatrixStatsAggregationBuilders {
-    /**
-     * Create a new {@link MatrixStats} aggregation with the given name.
-     */
-    public static MatrixStatsAggregationBuilder matrixStats(String name) {
-        return new MatrixStatsAggregationBuilder(name);
-    }
-}
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStats.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStats.java
deleted file mode 100644
index b423fa2e5caf..000000000000
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStats.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.search.aggregations.matrix.stats;
-
-import org.elasticsearch.search.aggregations.Aggregation;
-
-/**
- * Interface for MatrixStats Metric Aggregation
- */
-public interface MatrixStats extends Aggregation {
-    /** return the total document count */
-    long getDocCount();
-
-    /** return total field count (differs from docCount if there are missing values) */
-    long getFieldCount(String field);
-
-    /** return the field mean */
-    double getMean(String field);
-
-    /** return the field variance */
-    double getVariance(String field);
-
-    /** return the skewness of the distribution */
-    double getSkewness(String field);
-
-    /** return the kurtosis of the distribution */
-    double getKurtosis(String field);
-
-    /** return the covariance between field x and field y */
-    double getCovariance(String fieldX, String fieldY);
-
-    /** return the correlation coefficient of field x and field y */
-    double getCorrelation(String fieldX, String fieldY);
-}
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java
deleted file mode 100644
index c42cea2778b0..000000000000
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.search.aggregations.matrix.stats;
-
-import org.elasticsearch.search.MultiValueMode;
-import org.elasticsearch.search.aggregations.matrix.ArrayValuesSourceParser.NumericValuesSourceParser;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentParser;
-
-import java.io.IOException;
-import java.util.Map;
-
-import static org.elasticsearch.search.aggregations.matrix.ArrayValuesSourceAggregationBuilder.MULTIVALUE_MODE_FIELD;
-
-public class MatrixStatsParser extends NumericValuesSourceParser {
-
-    public MatrixStatsParser() {
-        super(true);
-    }
-
-    @Override
-    protected boolean token(
-        String aggregationName,
-        String currentFieldName,
-        XContentParser.Token token,
-        XContentParser parser,
-        Map<ParseField, Object> otherOptions
-    ) throws IOException {
-        if (MULTIVALUE_MODE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-            if (token == XContentParser.Token.VALUE_STRING) {
-                otherOptions.put(MULTIVALUE_MODE_FIELD, parser.text());
-                return true;
-            }
-        }
-        return false;
-    }
-
-    @Override
-    protected MatrixStatsAggregationBuilder createFactory(
-        String aggregationName,
-        ValuesSourceType valuesSourceType,
-        ValueType targetValueType,
-        Map<ParseField, Object> otherOptions
-    ) {
-        MatrixStatsAggregationBuilder builder = new MatrixStatsAggregationBuilder(aggregationName);
-        String mode = (String) otherOptions.get(MULTIVALUE_MODE_FIELD);
-        if (mode != null) {
-            builder.multiValueMode(MultiValueMode.fromString(mode));
-        }
-        return builder;
-    }
-}
diff --git a/modules/aggs-matrix-stats/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider b/modules/aggs-matrix-stats/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider
deleted file mode 100644
index a2d706a39a60..000000000000
--- a/modules/aggs-matrix-stats/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider
+++ /dev/null
@@ -1 +0,0 @@
-org.elasticsearch.search.aggregations.matrix.spi.MatrixStatsNamedXContentProvider
\ No newline at end of file
diff --git a/modules/aggs-matrix-stats/src/yamlRestTest/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java b/modules/aggs-matrix-stats/src/yamlRestTest/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java
deleted file mode 100644
index 6f29a3fb765f..000000000000
--- a/modules/aggs-matrix-stats/src/yamlRestTest/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.search.aggregations.matrix;
-
-import com.carrotsearch.randomizedtesting.annotations.Name;
-import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
-import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
-import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-
-public class MatrixStatsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
-    public MatrixStatsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
-        super(testCandidate);
-    }
-
-    @ParametersFactory
-    public static Iterable<Object[]> parameters() throws Exception {
-        return ESClientYamlSuiteTestCase.createParameters();
-    }
-}
diff --git a/modules/aggs-matrix-stats/src/yamlRestTest/resources/rest-api-spec/test/stats/10_basic.yml b/modules/aggs-matrix-stats/src/yamlRestTest/resources/rest-api-spec/test/stats/10_basic.yml
deleted file mode 100644
index 2416d2b2b314..000000000000
--- a/modules/aggs-matrix-stats/src/yamlRestTest/resources/rest-api-spec/test/stats/10_basic.yml
+++ /dev/null
@@ -1,16 +0,0 @@
-# Integration tests for Matrix Aggs Plugin
-#
-"Matrix stats aggs loaded":
-    - skip:
-          reason: "contains is a newly added assertion"
-          features: contains
-    - do:
-        cluster.state: {}
-
-    # Get master node id
-    - set: { master_node: master }
-
-    - do:
-        nodes.info: {}
-
-    - contains:  { nodes.$master.modules: { name: aggs-matrix-stats } }
diff --git a/modules/aggs-matrix-stats/src/yamlRestTest/resources/rest-api-spec/test/stats/40_multi_value_field.yml b/modules/aggs-matrix-stats/src/yamlRestTest/resources/rest-api-spec/test/stats/40_multi_value_field.yml
deleted file mode 100644
index 295ac2160f23..000000000000
--- a/modules/aggs-matrix-stats/src/yamlRestTest/resources/rest-api-spec/test/stats/40_multi_value_field.yml
+++ /dev/null
@@ -1,197 +0,0 @@
----
-setup:
-
-  - do:
-      indices.create:
-        index: test
-        body:
-          settings:
-            number_of_shards: 3
-            number_of_routing_shards: 3
-          mappings:
-            "properties":
-              "val1":
-                "type": "double"
-              "val2":
-                "type": "double"
-              "val3":
-                "type": "double"
-
-  - do:
-      indices.create:
-        index: unmapped
-        body:
-          settings:
-            number_of_shards: 3
-
-  - do:
-      index:
-        index:  test
-        id:     "1"
-        body:   { "val1": 1.9, "val2": 3.1, "val3": 2.3, "vals" : [1.9, 16.143] }
-  - do:
-      index:
-        index:  test
-        id:     "2"
-        body:   { "val1": -5.2, "val2": -3.4, "val3": 2.3, "vals" : [155, 16.23]}
-  - do:
-      index:
-        index:  test
-        id:     "3"
-        body:   { "val1": -5.2, "val3": 2.3, "vals" : [-455, -32.32]}
-  - do:
-      index:
-        index:  test
-        id:     "4"
-        body:   { "val1": 18.3, "val2": 104.4, "val3": 2.3, "vals" : [0.14, 92.1]}
-  - do:
-      index:
-        index:  test
-        id:     "5"
-        body:   { "val1": -53.2, "val2": -322.4, "val3": 2.3, "vals" : [16, 16]}
-  - do:
-      index:
-        index:  test
-        id:     "6"
-        body:   { "val1": -578.9, "val2": 69.9, "val3": 2.3}
-  - do:
-      index:
-        index:  test
-        id:     "7"
-        body:   { "val1": 16.2, "val2": 17.2, "val3": 2.3, "vals" : [1234.3, -3433]}
-  - do:
-      index:
-        index:  test
-        id:     "8"
-        body:   { "val1": -4222.63, "val2": 316.44, "val3": 2.3, "vals" : [177.2, -93.333]}
-  - do:
-      index:
-        index:  test
-        id:     "9"
-        body:   { "val1": -59999.55, "val2": -3163.4, "val3": 2.3, "vals" : [-29.9, 163.0]}
-  - do:
-      index:
-        index:  test
-        id:     "10"
-        body:   { "val1": 782.7, "val2": 789.7, "val3": 2.3, "vals" : [-0.2, 1343.3]}
-  - do:
-      index:
-        index:  test
-        id:     "11"
-        body:   { "val1": -1.2, "val2": 6.3, "val3": 2.3, "vals" : [15.3, 16.9]}
-  - do:
-      index:
-        index:  test
-        id:     "12"
-        body:   { "val1": 0, "val2": 1.11, "val3": 2.3, "vals" : [-644.4, -644.4]}
-  - do:
-      index:
-        index:  test
-        id:     "13"
-        body:   { "val1": 0.1, "val2": 0.92, "val3": 2.3, "vals" : [73.2, 0.12]}
-  - do:
-      index:
-        index:  test
-        id:     "14"
-        body:   { "val1": 0.12, "val2": -82.4, "val3": 2.3, "vals" : [-0.001, 1295.3]}
-  - do:
-      index:
-        index:  test
-        id:     "15"
-        body:   { "val1": 98.2, "val2": 32.4, "val3": 2.3, "vals" : [15.5, 16.5]}
-
-  - do:
-      indices.refresh:
-        index: [test, unmapped]
-
-  - do:
-      cluster.health:
-        wait_for_status: yellow
-
----
-"Unmapped":
-
-  - do:
-      search:
-        rest_total_hits_as_int: true
-        index: unmapped
-        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2", "vals"]} } } }
-
-  - match: {hits.total: 0}
-  - match: {aggregations.mfs.doc_count: 0}
-
----
-"Multi value field Max":
-
-  - do:
-      search:
-        rest_total_hits_as_int: true
-        index: test
-        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "vals"], "mode" : "max"} } } }
-
-  - match: {hits.total: 15}
-  - match: {aggregations.mfs.doc_count: 14}
-  - match: {aggregations.mfs.fields.0.count: 14}
-  - match: {aggregations.mfs.fields.0.correlation.val1: 0.06838646533369998}
-
----
-"Multi value field Min":
-
-  - do:
-      search:
-        rest_total_hits_as_int: true
-        index: test
-        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "vals"], "mode" : "min"} } } }
-
-  - match: {hits.total: 15}
-  - match: {aggregations.mfs.doc_count: 14}
-  - match: {aggregations.mfs.fields.0.count: 14}
-  - match: {aggregations.mfs.fields.0.correlation.val1: -0.09777682707831963}
-
----
-"Partially unmapped":
-
-  - do:
-      search:
-        rest_total_hits_as_int: true
-        index: [test, unmapped]
-        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2", "vals"]} } } }
-
-  - match: {hits.total: 15}
-  - match: {aggregations.mfs.doc_count: 13}
-  - match: {aggregations.mfs.fields.0.count: 13}
-  - match: {aggregations.mfs.fields.0.correlation.val1: -0.044997535185684244}
-
----
-"Partially unmapped with missing defaults":
-
-  - do:
-      search:
-        rest_total_hits_as_int: true
-        index: [test, unmapped]
-        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2", "vals"], "missing" : {"val2" : 10, "vals" : 5 } } } } }
-
-  - match: {hits.total: 15}
-  - match: {aggregations.mfs.doc_count: 15}
-  - match: {aggregations.mfs.fields.0.count: 15}
-  - match: {aggregations.mfs.fields.0.correlation.val2: 0.04028024709708195}
-
----
-"With script":
-
-  - do:
-      catch: /parsing_exception/
-      search:
-        rest_total_hits_as_int: true
-        index: test
-        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["vals", "val3"], "script" : { "my_script" : {"source" : "1 + doc['val1'].value", "lang" : "js"} } } } } }
-
----
-"With script params":
-
-  - do:
-      catch: /parsing_exception/
-      search:
-        rest_total_hits_as_int: true
-        index: test
-        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val3", "vals"], "script" : { "my_script" : {"source" : "my_var + doc['val1'].value", "params" : { "my_var" : 1 }, "lang" : "js" } } } } } }
diff --git a/modules/analysis-common/build.gradle b/modules/analysis-common/build.gradle
index d1345695d8a1..9988d70b6365 100644
--- a/modules/analysis-common/build.gradle
+++ b/modules/analysis-common/build.gradle
@@ -5,8 +5,8 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java
index 3ccc257fc33b..a047f2ee90ad 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java
@@ -102,7 +102,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.logging.DeprecationCategory;
@@ -166,7 +166,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver expressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         this.scriptServiceHolder.set(scriptService);
         return Collections.emptyList();
diff --git a/modules/apm/src/main/java/org/elasticsearch/tracing/apm/APM.java b/modules/apm/src/main/java/org/elasticsearch/tracing/apm/APM.java
index 9f10612e116c..05926db14d97 100644
--- a/modules/apm/src/main/java/org/elasticsearch/tracing/apm/APM.java
+++ b/modules/apm/src/main/java/org/elasticsearch/tracing/apm/APM.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Setting;
@@ -81,7 +81,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer unused,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         final APMTracer apmTracer = tracer.get();
 
@@ -101,6 +101,7 @@ public List<Setting<?>> getSettings() {
             APMAgentSettings.APM_ENABLED_SETTING,
             APMAgentSettings.APM_TRACING_NAMES_INCLUDE_SETTING,
             APMAgentSettings.APM_TRACING_NAMES_EXCLUDE_SETTING,
+            APMAgentSettings.APM_TRACING_SANITIZE_FIELD_NAMES,
             APMAgentSettings.APM_AGENT_SETTINGS,
             APMAgentSettings.APM_SECRET_TOKEN_SETTING,
             APMAgentSettings.APM_API_KEY_SETTING
diff --git a/modules/apm/src/main/java/org/elasticsearch/tracing/apm/APMAgentSettings.java b/modules/apm/src/main/java/org/elasticsearch/tracing/apm/APMAgentSettings.java
index 140d48027a59..1b432c9d3ad3 100644
--- a/modules/apm/src/main/java/org/elasticsearch/tracing/apm/APMAgentSettings.java
+++ b/modules/apm/src/main/java/org/elasticsearch/tracing/apm/APMAgentSettings.java
@@ -57,6 +57,7 @@ void addClusterSettingsListeners(ClusterService clusterService, APMTracer apmTra
         });
         clusterSettings.addSettingsUpdateConsumer(APM_TRACING_NAMES_INCLUDE_SETTING, apmTracer::setIncludeNames);
         clusterSettings.addSettingsUpdateConsumer(APM_TRACING_NAMES_EXCLUDE_SETTING, apmTracer::setExcludeNames);
+        clusterSettings.addSettingsUpdateConsumer(APM_TRACING_SANITIZE_FIELD_NAMES, apmTracer::setLabelFilters);
         clusterSettings.addAffixMapUpdateConsumer(APM_AGENT_SETTINGS, map -> map.forEach(this::setAgentSetting), (x, y) -> {});
     }
 
@@ -143,6 +144,27 @@ void setAgentSetting(String key, String value) {
         NodeScope
     );
 
+    static final Setting<List<String>> APM_TRACING_SANITIZE_FIELD_NAMES = Setting.listSetting(
+        APM_SETTING_PREFIX + "sanitize_field_names",
+        List.of(
+            "password",
+            "passwd",
+            "pwd",
+            "secret",
+            "*key",
+            "*token*",
+            "*session*",
+            "*credit*",
+            "*card*",
+            "*auth*",
+            "*principal*",
+            "set-cookie"
+        ),
+        Function.identity(),
+        OperatorDynamic,
+        NodeScope
+    );
+
     static final Setting<Boolean> APM_ENABLED_SETTING = Setting.boolSetting(
         APM_SETTING_PREFIX + "enabled",
         false,
diff --git a/modules/apm/src/main/java/org/elasticsearch/tracing/apm/APMTracer.java b/modules/apm/src/main/java/org/elasticsearch/tracing/apm/APMTracer.java
index fad42b2768c3..5e8375e067fe 100644
--- a/modules/apm/src/main/java/org/elasticsearch/tracing/apm/APMTracer.java
+++ b/modules/apm/src/main/java/org/elasticsearch/tracing/apm/APMTracer.java
@@ -44,6 +44,7 @@
 import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_ENABLED_SETTING;
 import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_TRACING_NAMES_EXCLUDE_SETTING;
 import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_TRACING_NAMES_INCLUDE_SETTING;
+import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_TRACING_SANITIZE_FIELD_NAMES;
 
 /**
  * This is an implementation of the {@link org.elasticsearch.tracing.Tracer} interface, which uses
@@ -65,8 +66,10 @@ public class APMTracer extends AbstractLifecycleComponent implements org.elastic
 
     private List<String> includeNames;
     private List<String> excludeNames;
+    private List<String> labelFilters;
     /** Built using {@link #includeNames} and {@link #excludeNames}, and filters out spans based on their name. */
     private volatile CharacterRunAutomaton filterAutomaton;
+    private volatile CharacterRunAutomaton labelFilterAutomaton;
     private String clusterName;
     private String nodeName;
 
@@ -86,7 +89,10 @@ record APMServices(Tracer tracer, OpenTelemetry openTelemetry) {}
     public APMTracer(Settings settings) {
         this.includeNames = APM_TRACING_NAMES_INCLUDE_SETTING.get(settings);
         this.excludeNames = APM_TRACING_NAMES_EXCLUDE_SETTING.get(settings);
+        this.labelFilters = APM_TRACING_SANITIZE_FIELD_NAMES.get(settings);
+
         this.filterAutomaton = buildAutomaton(includeNames, excludeNames);
+        this.labelFilterAutomaton = buildAutomaton(labelFilters, List.of());
         this.enabled = APM_ENABLED_SETTING.get(settings);
     }
 
@@ -109,6 +115,16 @@ void setExcludeNames(List<String> excludeNames) {
         this.filterAutomaton = buildAutomaton(includeNames, excludeNames);
     }
 
+    void setLabelFilters(List<String> labelFilters) {
+        this.labelFilters = labelFilters;
+        this.labelFilterAutomaton = buildAutomaton(labelFilters, List.of());
+    }
+
+    // package-private for testing
+    CharacterRunAutomaton getLabelFilterAutomaton() {
+        return labelFilterAutomaton;
+    }
+
     @Override
     protected void doStart() {
         if (enabled) {
@@ -271,6 +287,12 @@ private void setSpanAttributes(@Nullable Map<String, Object> spanAttributes, Spa
             for (Map.Entry<String, Object> entry : spanAttributes.entrySet()) {
                 final String key = entry.getKey();
                 final Object value = entry.getValue();
+
+                if (this.labelFilterAutomaton.run(key)) {
+                    spanBuilder.setAttribute(key, "[REDACTED]");
+                    continue;
+                }
+
                 if (value instanceof String) {
                     spanBuilder.setAttribute(key, (String) value);
                 } else if (value instanceof Long) {
@@ -394,9 +416,9 @@ Map<String, Context> getSpans() {
         return spans;
     }
 
-    private static CharacterRunAutomaton buildAutomaton(List<String> includeNames, List<String> excludeNames) {
-        Automaton includeAutomaton = patternsToAutomaton(includeNames);
-        Automaton excludeAutomaton = patternsToAutomaton(excludeNames);
+    private static CharacterRunAutomaton buildAutomaton(List<String> includePatterns, List<String> excludePatterns) {
+        Automaton includeAutomaton = patternsToAutomaton(includePatterns);
+        Automaton excludeAutomaton = patternsToAutomaton(excludePatterns);
 
         if (includeAutomaton == null) {
             includeAutomaton = Automata.makeAnyString();
diff --git a/modules/apm/src/test/java/org/elasticsearch/tracing/apm/APMTracerTests.java b/modules/apm/src/test/java/org/elasticsearch/tracing/apm/APMTracerTests.java
index f4ab36d43aa4..9c6582fcc118 100644
--- a/modules/apm/src/test/java/org/elasticsearch/tracing/apm/APMTracerTests.java
+++ b/modules/apm/src/test/java/org/elasticsearch/tracing/apm/APMTracerTests.java
@@ -8,12 +8,14 @@
 
 package org.elasticsearch.tracing.apm;
 
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.List;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_ENABLED_SETTING;
 import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_TRACING_NAMES_EXCLUDE_SETTING;
@@ -166,6 +168,43 @@ public void test_whenTraceStarted_andSpanNameExcluded_thenSpanIsNotStarted() {
         assertThat(apmTracer.getSpans(), hasKey("id3"));
     }
 
+    /**
+     * Check that sensitive attributes are not added verbatim to a span, but instead the value is redacted.
+     */
+    public void test_whenAddingAttributes_thenSensitiveValuesAreRedacted() {
+        Settings settings = Settings.builder().put(APM_ENABLED_SETTING.getKey(), false).build();
+        APMTracer apmTracer = buildTracer(settings);
+        CharacterRunAutomaton labelFilterAutomaton = apmTracer.getLabelFilterAutomaton();
+
+        Stream.of(
+            "auth",
+            "auth-header",
+            "authValue",
+            "card",
+            "card-details",
+            "card-number",
+            "credit",
+            "credit-card",
+            "key",
+            "my-credit-number",
+            "my_session_id",
+            "passwd",
+            "password",
+            "principal",
+            "principal-value",
+            "pwd",
+            "secret",
+            "secure-key",
+            "sensitive-token*",
+            "session",
+            "session_id",
+            "set-cookie",
+            "some-auth",
+            "some-principal",
+            "token-for login"
+        ).forEach(key -> assertTrue("Expected label filter automaton to redact [" + key + "]", labelFilterAutomaton.run(key)));
+    }
+
     private APMTracer buildTracer(Settings settings) {
         APMTracer tracer = new APMTracer(settings);
         tracer.doStart();
diff --git a/modules/data-streams/build.gradle b/modules/data-streams/build.gradle
index 49136712b855..fa3346e26ab2 100644
--- a/modules/data-streams/build.gradle
+++ b/modules/data-streams/build.gradle
@@ -2,9 +2,9 @@ import org.elasticsearch.gradle.internal.info.BuildParams
 
 apply plugin: 'elasticsearch.test-with-dependencies'
 apply plugin: 'elasticsearch.internal-cluster-test'
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.internal-java-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
   description 'Elasticsearch Expanded Pack Plugin - Data Streams'
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java
index de30aeaa641e..dc073efd925e 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java
@@ -100,9 +100,11 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.OptionalLong;
 import java.util.Set;
@@ -746,7 +748,14 @@ public void testDataSteamAliasWithFilter() throws Exception {
             equalTo(
                 Map.of(
                     "logs-foobar",
-                    List.of(new DataStreamAlias("foo", List.of("logs-foobar"), null, Map.of("term", Map.of("type", Map.of("value", "y")))))
+                    List.of(
+                        new DataStreamAlias(
+                            "foo",
+                            List.of("logs-foobar"),
+                            null,
+                            Map.of("logs-foobar", Map.of("term", Map.of("type", Map.of("value", "y"))))
+                        )
+                    )
                 )
             )
         );
@@ -771,7 +780,14 @@ public void testDataSteamAliasWithFilter() throws Exception {
             equalTo(
                 Map.of(
                     "logs-foobar",
-                    List.of(new DataStreamAlias("foo", List.of("logs-foobar"), null, Map.of("term", Map.of("type", Map.of("value", "x")))))
+                    List.of(
+                        new DataStreamAlias(
+                            "foo",
+                            List.of("logs-foobar"),
+                            null,
+                            Map.of("logs-foobar", Map.of("term", Map.of("type", Map.of("value", "x"))))
+                        )
+                    )
                 )
             )
         );
@@ -790,17 +806,17 @@ public void testRandomDataSteamAliasesUpdate() throws Exception {
         String alias = randomAlphaOfLength(4);
         String[] dataStreams = Arrays.stream(generateRandomStringArray(16, 4, false, false))
             .map(s -> "log-" + s.toLowerCase(Locale.ROOT))
+            .distinct()
             .toArray(String[]::new);
         for (String dataStream : dataStreams) {
             CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStream);
             client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
         }
-        AliasActions addAction = new AliasActions(AliasActions.Type.ADD).aliases(alias)
-            .indices(dataStreams)
-            .filter(Map.of("term", Map.of("type", Map.of("value", "y"))));
+        Map<String, Object> indexFilters = Map.of("term", Map.of("type", Map.of("value", "y")));
+        AliasActions addAction = new AliasActions(AliasActions.Type.ADD).aliases(alias).indices(dataStreams).filter(indexFilters);
         assertAcked(client().admin().indices().aliases(new IndicesAliasesRequest().addAliasAction(addAction)).actionGet());
 
-        addAction = new AliasActions(AliasActions.Type.ADD).aliases(alias).indices(dataStreams[0]).writeIndex(true);
+        addAction = new AliasActions(AliasActions.Type.ADD).aliases(alias).indices(dataStreams[0]).filter(indexFilters).writeIndex(true);
         assertAcked(client().admin().indices().aliases(new IndicesAliasesRequest().addAliasAction(addAction)).actionGet());
 
         GetAliasesResponse response = client().admin().indices().getAliases(new GetAliasesRequest()).actionGet();
@@ -815,7 +831,16 @@ public void testRandomDataSteamAliasesUpdate() throws Exception {
         assertThat(result.get(0).getName(), equalTo(alias));
         assertThat(result.get(0).getDataStreams(), containsInAnyOrder(dataStreams));
         assertThat(result.get(0).getWriteDataStream(), equalTo(dataStreams[0]));
-        assertThat(result.get(0).getFilter().string(), equalTo("{\"term\":{\"type\":{\"value\":\"y\"}}}"));
+        for (String dataStream : dataStreams) {
+            assertThat(
+                result.stream()
+                    .map(resultAlias -> resultAlias.getFilter(dataStream))
+                    .filter(Objects::nonNull)
+                    .map(CompressedXContent::string)
+                    .collect(Collectors.toSet()),
+                containsInAnyOrder("{\"term\":{\"type\":{\"value\":\"y\"}}}")
+            );
+        }
     }
 
     public void testDataSteamAliasWithMalformedFilter() throws Exception {
@@ -2024,24 +2049,7 @@ public void testWriteIndexWriteLoadAndAvgShardSizeIsStoredAfterRollover() throws
         final var request = new CreateDataStreamAction.Request(dataStreamName);
         assertAcked(client().execute(CreateDataStreamAction.INSTANCE, request).actionGet());
 
-        assertBusy(() -> {
-            for (int i = 0; i < 10; i++) {
-                indexDocs(dataStreamName, randomIntBetween(100, 200));
-            }
-
-            final ClusterState clusterState = internalCluster().getCurrentMasterNodeInstance(ClusterService.class).state();
-            final DataStream dataStream = clusterState.getMetadata().dataStreams().get(dataStreamName);
-            final String writeIndex = dataStream.getWriteIndex().getName();
-            final IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats(writeIndex).get();
-            for (IndexShardStats indexShardStats : indicesStatsResponse.getIndex(writeIndex).getIndexShards().values()) {
-                for (ShardStats shard : indexShardStats.getShards()) {
-                    final IndexingStats.Stats shardIndexingStats = shard.getStats().getIndexing().getTotal();
-                    // Ensure that we have enough clock granularity before rolling over to ensure that we capture _some_ write load
-                    assertThat(shardIndexingStats.getTotalActiveTimeInMillis(), is(greaterThan(0L)));
-                    assertThat(shardIndexingStats.getWriteLoad(), is(greaterThan(0.0)));
-                }
-            }
-        });
+        indexDocsAndEnsureThereIsCapturedWriteLoad(dataStreamName);
 
         assertAcked(client().admin().indices().rolloverIndex(new RolloverRequest(dataStreamName, null)).actionGet());
         final ClusterState clusterState = internalCluster().getCurrentMasterNodeInstance(ClusterService.class).state();
@@ -2078,6 +2086,7 @@ public void testWriteLoadAndAvgShardSizeIsStoredInABestEffort() throws Exception
         // - We want to simulate two possible cases here:
         // - All the assigned nodes for shard 0 will fail to respond to the IndicesStatsRequest
         // - Only the shard 1 replica will respond successfully to the IndicesStatsRequest ensuring that we fall back in that case
+        // (only if it's not co-located with some other shard copies)
 
         final List<String> dataOnlyNodes = internalCluster().startDataOnlyNodes(4);
         final String dataStreamName = "logs-es";
@@ -2090,21 +2099,22 @@ public void testWriteLoadAndAvgShardSizeIsStoredInABestEffort() throws Exception
         DataStreamIT.putComposableIndexTemplate("my-template", null, List.of("logs-*"), indexSettings, null);
         final var createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
         assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet());
+        ensureGreen(dataStreamName);
 
-        for (int i = 0; i < 10; i++) {
-            indexDocs(dataStreamName, randomIntBetween(100, 200));
-        }
+        indexDocsAndEnsureThereIsCapturedWriteLoad(dataStreamName);
 
         final ClusterState clusterStateBeforeRollover = internalCluster().getCurrentMasterNodeInstance(ClusterService.class).state();
         final DataStream dataStreamBeforeRollover = clusterStateBeforeRollover.getMetadata().dataStreams().get(dataStreamName);
         final IndexRoutingTable currentDataStreamWriteIndexRoutingTable = clusterStateBeforeRollover.routingTable()
             .index(dataStreamBeforeRollover.getWriteIndex());
 
-        final List<String> failingIndicesStatsNodeIds = new ArrayList<>();
+        final Set<String> failingIndicesStatsNodeIds = new HashSet<>();
         for (ShardRouting shardRouting : currentDataStreamWriteIndexRoutingTable.shard(0).assignedShards()) {
             failingIndicesStatsNodeIds.add(shardRouting.currentNodeId());
         }
         failingIndicesStatsNodeIds.add(currentDataStreamWriteIndexRoutingTable.shard(1).primaryShard().currentNodeId());
+        final String shard1ReplicaNodeId = currentDataStreamWriteIndexRoutingTable.shard(1).replicaShards().get(0).currentNodeId();
+        final boolean shard1ReplicaIsAllocatedInAReachableNode = failingIndicesStatsNodeIds.contains(shard1ReplicaNodeId) == false;
 
         for (String nodeId : failingIndicesStatsNodeIds) {
             String nodeName = clusterStateBeforeRollover.nodes().resolveNode(nodeId).getName();
@@ -2114,7 +2124,12 @@ public void testWriteLoadAndAvgShardSizeIsStoredInABestEffort() throws Exception
                 (handler, request, channel, task) -> channel.sendResponse(new RuntimeException("Unable to get stats"))
             );
         }
-        assertThat(failingIndicesStatsNodeIds.size(), is(equalTo(3)));
+
+        logger.info(
+            "--> Node IDs failing to respond to stats requests {}, shard 1 replica routing {}",
+            failingIndicesStatsNodeIds,
+            currentDataStreamWriteIndexRoutingTable.shard(1).replicaShards().get(0)
+        );
 
         assertAcked(client().admin().indices().rolloverIndex(new RolloverRequest(dataStreamName, null)).actionGet());
         final ClusterState clusterState = internalCluster().getCurrentMasterNodeInstance(ClusterService.class).state();
@@ -2124,8 +2139,9 @@ public void testWriteLoadAndAvgShardSizeIsStoredInABestEffort() throws Exception
             final IndexMetadata indexMetadata = clusterState.metadata().index(index);
             final IndexMetadataStats metadataStats = indexMetadata.getStats();
 
-            if (index.equals(dataStream.getWriteIndex()) == false) {
-                assertThat(metadataStats, is(notNullValue()));
+            // If all the shards are co-located within the failing nodes, no stats will be stored during rollover
+            if (index.equals(dataStream.getWriteIndex()) == false && shard1ReplicaIsAllocatedInAReachableNode) {
+                assertThat("Expected stats for index " + index, metadataStats, is(notNullValue()));
 
                 final IndexWriteLoad indexWriteLoad = metadataStats.writeLoad();
                 // All stats request performed against nodes holding the shard 0 failed
@@ -2247,6 +2263,27 @@ public void testShardSizeIsForecastedDuringRollover() throws Exception {
         assertThat(forecastedShardSizeInBytes.getAsLong(), is(equalTo(expectedTotalSizeInBytes / shardCount)));
     }
 
+    private void indexDocsAndEnsureThereIsCapturedWriteLoad(String dataStreamName) throws Exception {
+        assertBusy(() -> {
+            for (int i = 0; i < 10; i++) {
+                indexDocs(dataStreamName, randomIntBetween(100, 200));
+            }
+
+            final ClusterState clusterState = internalCluster().getCurrentMasterNodeInstance(ClusterService.class).state();
+            final DataStream dataStream = clusterState.getMetadata().dataStreams().get(dataStreamName);
+            final String writeIndex = dataStream.getWriteIndex().getName();
+            final IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats(writeIndex).get();
+            for (IndexShardStats indexShardStats : indicesStatsResponse.getIndex(writeIndex).getIndexShards().values()) {
+                for (ShardStats shard : indexShardStats.getShards()) {
+                    final IndexingStats.Stats shardIndexingStats = shard.getStats().getIndexing().getTotal();
+                    // Ensure that we have enough clock granularity before rolling over to ensure that we capture _some_ write load
+                    assertThat(shardIndexingStats.getTotalActiveTimeInMillis(), is(greaterThan(0L)));
+                    assertThat(shardIndexingStats.getWriteLoad(), is(greaterThan(0.0)));
+                }
+            }
+        });
+    }
+
     static void putComposableIndexTemplate(
         String id,
         @Nullable String mappings,
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java
index bf7d9ecc5fc4..094a3f08f492 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java
@@ -180,7 +180,7 @@ public void testSnapshotAndRestore() throws Exception {
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds").size(), equalTo(1));
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds").get(0).getName(), equalTo("my-alias"));
         assertThat(
-            getAliasesResponse.getDataStreamAliases().get("ds").get(0).getFilter().string(),
+            getAliasesResponse.getDataStreamAliases().get("ds").get(0).getFilter("ds").string(),
             equalTo("{\"match_all\":{\"boost\":1.0}}")
         );
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds").get(0).getWriteDataStream(), equalTo("other-ds"));
@@ -188,7 +188,7 @@ public void testSnapshotAndRestore() throws Exception {
         assertThat(getAliasesResponse.getDataStreamAliases().get("other-ds").get(0).getName(), equalTo("my-alias"));
         assertThat(getAliasesResponse.getDataStreamAliases().get("other-ds").get(0).getWriteDataStream(), equalTo("other-ds"));
         assertThat(
-            getAliasesResponse.getDataStreamAliases().get("other-ds").get(0).getFilter().string(),
+            getAliasesResponse.getDataStreamAliases().get("other-ds").get(0).getFilter("ds").string(),
             equalTo("{\"match_all\":{\"boost\":1.0}}")
         );
     }
@@ -371,7 +371,7 @@ public void testSnapshotAndRestoreAllIncludeSpecificDataStream() throws Exceptio
                         "my-alias",
                         List.of(dataStreamToSnapshot),
                         "other-ds".equals(dataStreamToSnapshot) ? "other-ds" : null,
-                        Map.of("match_all", Map.of("boost", 1f))
+                        Map.of("other-ds", Map.of("match_all", Map.of("boost", 1f)), "ds", Map.of("match_all", Map.of("boost", 1f)))
                     )
                 )
             )
@@ -419,7 +419,7 @@ public void testSnapshotAndRestoreReplaceAll() throws Exception {
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds").get(0).getDataStreams(), containsInAnyOrder("ds", "other-ds"));
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds").get(0).getWriteDataStream(), equalTo("other-ds"));
         assertThat(
-            getAliasesResponse.getDataStreamAliases().get("ds").get(0).getFilter().string(),
+            getAliasesResponse.getDataStreamAliases().get("ds").get(0).getFilter("ds").string(),
             equalTo("{\"match_all\":{\"boost\":1.0}}")
         );
         assertThat(getAliasesResponse.getDataStreamAliases().get("other-ds").size(), equalTo(1));
@@ -427,7 +427,7 @@ public void testSnapshotAndRestoreReplaceAll() throws Exception {
         assertThat(getAliasesResponse.getDataStreamAliases().get("other-ds").get(0).getDataStreams(), containsInAnyOrder("ds", "other-ds"));
         assertThat(getAliasesResponse.getDataStreamAliases().get("other-ds").get(0).getWriteDataStream(), equalTo("other-ds"));
         assertThat(
-            getAliasesResponse.getDataStreamAliases().get("other-ds").get(0).getFilter().string(),
+            getAliasesResponse.getDataStreamAliases().get("other-ds").get(0).getFilter("ds").string(),
             equalTo("{\"match_all\":{\"boost\":1.0}}")
         );
 
@@ -473,7 +473,7 @@ public void testSnapshotAndRestoreAll() throws Exception {
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds").get(0).getWriteDataStream(), equalTo("other-ds"));
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds").get(0).getDataStreams(), containsInAnyOrder("ds", "other-ds"));
         assertThat(
-            getAliasesResponse.getDataStreamAliases().get("ds").get(0).getFilter().string(),
+            getAliasesResponse.getDataStreamAliases().get("ds").get(0).getFilter("ds").string(),
             equalTo("{\"match_all\":{\"boost\":1.0}}")
         );
 
@@ -482,7 +482,7 @@ public void testSnapshotAndRestoreAll() throws Exception {
         assertThat(getAliasesResponse.getDataStreamAliases().get("other-ds").get(0).getWriteDataStream(), equalTo("other-ds"));
         assertThat(getAliasesResponse.getDataStreamAliases().get("other-ds").get(0).getDataStreams(), containsInAnyOrder("ds", "other-ds"));
         assertThat(
-            getAliasesResponse.getDataStreamAliases().get("other-ds").get(0).getFilter().string(),
+            getAliasesResponse.getDataStreamAliases().get("other-ds").get(0).getFilter("ds").string(),
             equalTo("{\"match_all\":{\"boost\":1.0}}")
         );
 
@@ -567,21 +567,21 @@ public void testRename() throws Exception {
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds2").size(), equalTo(1));
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds2").get(0).getName(), equalTo("my-alias"));
         assertThat(
-            getAliasesResponse.getDataStreamAliases().get("ds2").get(0).getFilter().string(),
+            getAliasesResponse.getDataStreamAliases().get("ds2").get(0).getFilter("ds").string(),
             equalTo("{\"match_all\":{\"boost\":1.0}}")
         );
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds").size(), equalTo(1));
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds").get(0).getName(), equalTo("my-alias"));
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds").get(0).getWriteDataStream(), equalTo("other-ds"));
         assertThat(
-            getAliasesResponse.getDataStreamAliases().get("ds").get(0).getFilter().string(),
+            getAliasesResponse.getDataStreamAliases().get("ds").get(0).getFilter("ds").string(),
             equalTo("{\"match_all\":{\"boost\":1.0}}")
         );
         assertThat(getAliasesResponse.getDataStreamAliases().get("other-ds").size(), equalTo(1));
         assertThat(getAliasesResponse.getDataStreamAliases().get("other-ds").get(0).getName(), equalTo("my-alias"));
         assertThat(getAliasesResponse.getDataStreamAliases().get("other-ds").get(0).getWriteDataStream(), equalTo("other-ds"));
         assertThat(
-            getAliasesResponse.getDataStreamAliases().get("other-ds").get(0).getFilter().string(),
+            getAliasesResponse.getDataStreamAliases().get("other-ds").get(0).getFilter("ds").string(),
             equalTo("{\"match_all\":{\"boost\":1.0}}")
         );
     }
@@ -621,21 +621,21 @@ public void testRenameWriteDataStream() throws Exception {
         assertThat(getAliasesResponse.getDataStreamAliases().get("other-ds2").get(0).getName(), equalTo("my-alias"));
         assertThat(getAliasesResponse.getDataStreamAliases().get("other-ds2").get(0).getWriteDataStream(), equalTo("other-ds2"));
         assertThat(
-            getAliasesResponse.getDataStreamAliases().get("other-ds2").get(0).getFilter().string(),
+            getAliasesResponse.getDataStreamAliases().get("other-ds2").get(0).getFilter("ds").string(),
             equalTo("{\"match_all\":{\"boost\":1.0}}")
         );
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds").size(), equalTo(1));
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds").get(0).getName(), equalTo("my-alias"));
         assertThat(getAliasesResponse.getDataStreamAliases().get("ds").get(0).getWriteDataStream(), equalTo("other-ds2"));
         assertThat(
-            getAliasesResponse.getDataStreamAliases().get("ds").get(0).getFilter().string(),
+            getAliasesResponse.getDataStreamAliases().get("ds").get(0).getFilter("ds").string(),
             equalTo("{\"match_all\":{\"boost\":1.0}}")
         );
         assertThat(getAliasesResponse.getDataStreamAliases().get("other-ds").size(), equalTo(1));
         assertThat(getAliasesResponse.getDataStreamAliases().get("other-ds").get(0).getName(), equalTo("my-alias"));
         assertThat(getAliasesResponse.getDataStreamAliases().get("other-ds").get(0).getWriteDataStream(), equalTo("other-ds2"));
         assertThat(
-            getAliasesResponse.getDataStreamAliases().get("other-ds").get(0).getFilter().string(),
+            getAliasesResponse.getDataStreamAliases().get("other-ds").get(0).getFilter("ds").string(),
             equalTo("{\"match_all\":{\"boost\":1.0}}")
         );
     }
diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataTierDataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataTierDataStreamIT.java
index 0043df507733..f6af27cfa8f1 100644
--- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataTierDataStreamIT.java
+++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataTierDataStreamIT.java
@@ -9,7 +9,6 @@
 
 import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
-import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
@@ -48,29 +47,27 @@ public void testDefaultDataStreamAllocateToHot() {
             PutComposableIndexTemplateAction.INSTANCE,
             new PutComposableIndexTemplateAction.Request("template").indexTemplate(template)
         ).actionGet();
-        client().prepareIndex(index).setCreate(true).setId("1").setSource("@timestamp", "2020-09-09").setWaitForActiveShards(0).get();
 
-        Settings idxSettings = client().admin()
-            .indices()
-            .prepareGetIndex()
-            .addIndices(index)
+        var dsIndexName = client().prepareIndex(index)
+            .setCreate(true)
+            .setId("1")
+            .setSource("@timestamp", "2020-09-09")
+            .setWaitForActiveShards(0)
             .get()
-            .getSettings()
-            .get(DataStream.getDefaultBackingIndexName(index, 1));
+            .getIndex();
+        var idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(dsIndexName);
         assertThat(DataTier.TIER_PREFERENCE_SETTING.get(idxSettings), equalTo(DataTier.DATA_HOT));
 
         logger.info("--> waiting for {} to be yellow", index);
         ensureYellow(index);
 
         // Roll over index and ensure the second index also went to the "hot" tier
-        client().admin().indices().prepareRolloverIndex(index).get();
-        idxSettings = client().admin()
-            .indices()
-            .prepareGetIndex()
-            .addIndices(index)
-            .get()
-            .getSettings()
-            .get(DataStream.getDefaultBackingIndexName(index, 2));
+        var rolledOverIndexName = client().admin().indices().prepareRolloverIndex(index).get().getNewIndex();
+
+        // new index name should have the rolled over name
+        assertNotEquals(dsIndexName, rolledOverIndexName);
+
+        idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(rolledOverIndexName);
         assertThat(DataTier.TIER_PREFERENCE_SETTING.get(idxSettings), equalTo(DataTier.DATA_HOT));
     }
 
diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java
index 13d0c067b0cb..9ae76394011c 100644
--- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java
+++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.time.FormatNames;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.ObjectPath;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.time.Instant;
@@ -35,6 +36,14 @@
 
 public class TsdbDataStreamRestIT extends ESRestTestCase {
 
+    private static final String COMPONENT_TEMPLATE = """
+        {
+            "template": {
+                "settings": {}
+            }
+        }
+        """;
+
     private static final String TEMPLATE = """
         {
             "index_patterns": ["k8s*"],
@@ -97,6 +106,7 @@ public class TsdbDataStreamRestIT extends ESRestTestCase {
                     }
                 }
             },
+            "composed_of": ["custom_template"],
             "data_stream": {
             }
         }""";
@@ -190,12 +200,19 @@ public class TsdbDataStreamRestIT extends ESRestTestCase {
             {"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "elephant", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876eb4", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}
             """;
 
-    public void testTsdbDataStreams() throws Exception {
-        // Create a template
-        var putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1");
-        putComposableIndexTemplateRequest.setJsonEntity(TEMPLATE);
-        assertOK(client().performRequest(putComposableIndexTemplateRequest));
+    @Before
+    public void setup() throws IOException {
+        // Add component template:
+        var request = new Request("POST", "/_component_template/custom_template");
+        request.setJsonEntity(COMPONENT_TEMPLATE);
+        assertOK(client().performRequest(request));
+        // Add composable index template
+        request = new Request("POST", "/_index_template/1");
+        request.setJsonEntity(TEMPLATE);
+        assertOK(client().performRequest(request));
+    }
 
+    public void testTsdbDataStreams() throws Exception {
         var bulkRequest = new Request("POST", "/k8s/_bulk");
         bulkRequest.setJsonEntity(BULK.replace("$now", formatInstant(Instant.now())));
         bulkRequest.addParameter("refresh", "true");
@@ -331,10 +348,6 @@ public void testTsdbDataStreamsNanos() throws Exception {
     }
 
     public void testSimulateTsdbDataStreamTemplate() throws Exception {
-        var putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1");
-        putComposableIndexTemplateRequest.setJsonEntity(TEMPLATE);
-        assertOK(client().performRequest(putComposableIndexTemplateRequest));
-
         var simulateIndexTemplateRequest = new Request("POST", "/_index_template/_simulate_index/k8s");
         var response = client().performRequest(simulateIndexTemplateRequest);
         assertOK(response);
@@ -353,11 +366,6 @@ public void testSimulateTsdbDataStreamTemplate() throws Exception {
     }
 
     public void testSubsequentRollovers() throws Exception {
-        // Create a template
-        var putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1");
-        putComposableIndexTemplateRequest.setJsonEntity(TEMPLATE);
-        assertOK(client().performRequest(putComposableIndexTemplateRequest));
-
         var createDataStreamRequest = new Request("PUT", "/_data_stream/k8s");
         assertOK(client().performRequest(createDataStreamRequest));
 
@@ -460,16 +468,10 @@ public void testMigrateRegularDataStreamToTsdbDataStream() throws Exception {
         assertThat(e.getMessage(), containsString("is outside of ranges of currently writable indices"));
     }
 
-    public void testChangeTemplateIndexMode() throws Exception {
-        // Create a template
-        {
-            var putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1");
-            putComposableIndexTemplateRequest.setJsonEntity(TEMPLATE);
-            assertOK(client().performRequest(putComposableIndexTemplateRequest));
-        }
+    public void testDowngradeTsdbDataStreamToRegularDataStream() throws Exception {
+        var time = Instant.now();
         {
             var indexRequest = new Request("POST", "/k8s/_doc");
-            var time = Instant.now();
             indexRequest.setJsonEntity(DOC.replace("$time", formatInstant(time)));
             var response = client().performRequest(indexRequest);
             assertOK(response);
@@ -477,16 +479,91 @@ public void testChangeTemplateIndexMode() throws Exception {
         {
             var putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1");
             putComposableIndexTemplateRequest.setJsonEntity(NON_TSDB_TEMPLATE);
-            var e = expectThrows(ResponseException.class, () -> client().performRequest(putComposableIndexTemplateRequest));
-            assertThat(
-                e.getMessage(),
-                containsString(
-                    "composable template [1] with index patterns [k8s*], priority [null],"
-                        + " index.routing_path [] would cause tsdb data streams [k8s] to no longer match a data stream template"
-                        + " with a time_series index_mode"
-                )
-            );
+            client().performRequest(putComposableIndexTemplateRequest);
         }
+        {
+            {
+                // check prior to rollover
+                var getDataStreamsRequest = new Request("GET", "/_data_stream");
+                var getDataStreamResponse = client().performRequest(getDataStreamsRequest);
+                assertOK(getDataStreamResponse);
+                var dataStreams = entityAsMap(getDataStreamResponse);
+                assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.name"), equalTo("k8s"));
+                assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.indices"), hasSize(1));
+                assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.time_series"), notNullValue());
+            }
+            var rolloverRequest = new Request("POST", "/k8s/_rollover");
+            var rolloverResponse = client().performRequest(rolloverRequest);
+            assertOK(rolloverResponse);
+            var rolloverResponseBody = entityAsMap(rolloverResponse);
+            assertThat(rolloverResponseBody.get("rolled_over"), is(true));
+            {
+                // Data stream is no longer a tsdb data stream
+                var getDataStreamsRequest = new Request("GET", "/_data_stream");
+                var getDataStreamResponse = client().performRequest(getDataStreamsRequest);
+                assertOK(getDataStreamResponse);
+                var dataStreams = entityAsMap(getDataStreamResponse);
+                assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.name"), equalTo("k8s"));
+                assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.indices"), hasSize(2));
+                assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.time_series"), nullValue());
+            }
+            {
+                // old index remains a tsdb index
+                var oldIndex = (String) rolloverResponseBody.get("old_index");
+                assertThat(oldIndex, backingIndexEqualTo("k8s", 1));
+                var indices = getIndex(oldIndex);
+                var escapedBackingIndex = oldIndex.replace(".", "\\.");
+                assertThat(ObjectPath.evaluate(indices, escapedBackingIndex + ".data_stream"), equalTo("k8s"));
+                assertThat(ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.mode"), equalTo("time_series"));
+                assertThat(ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.time_series.start_time"), notNullValue());
+                assertThat(ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.time_series.end_time"), notNullValue());
+            }
+            {
+                // new index is a regular index
+                var newIndex = (String) rolloverResponseBody.get("new_index");
+                assertThat(newIndex, backingIndexEqualTo("k8s", 2));
+                var indices = getIndex(newIndex);
+                var escapedBackingIndex = newIndex.replace(".", "\\.");
+                assertThat(ObjectPath.evaluate(indices, escapedBackingIndex + ".data_stream"), equalTo("k8s"));
+                assertThat(ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.mode"), nullValue());
+                assertThat(ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.time_series.start_time"), nullValue());
+                assertThat(ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.time_series.end_time"), nullValue());
+            }
+        }
+        {
+            // All documents should be ingested into the most recent backing index:
+            // (since the data stream is no longer a tsdb data stream)
+            Instant[] timestamps = new Instant[] {
+                time,
+                time.plusSeconds(1),
+                time.plusSeconds(5),
+                time.minus(30, ChronoUnit.DAYS),
+                time.plus(30, ChronoUnit.DAYS) };
+            for (Instant timestamp : timestamps) {
+                var indexRequest = new Request("POST", "/k8s/_doc");
+                indexRequest.setJsonEntity(DOC.replace("$time", formatInstant(timestamp)));
+                var response = client().performRequest(indexRequest);
+                assertOK(response);
+                var responseBody = entityAsMap(response);
+                assertThat((String) responseBody.get("_index"), backingIndexEqualTo("k8s", 2));
+            }
+        }
+    }
+
+    public void testUpdateComponentTemplateDoesNotFailIndexTemplateValidation() throws IOException {
+        var request = new Request("POST", "/_component_template/custom_template");
+        request.setJsonEntity("""
+            {
+                "template": {
+                    "settings": {
+                        "index": {
+                            "number_of_replicas": 1
+                        }
+                    }
+                }
+            }
+            """);
+        client().performRequest(request);
     }
 
     private static Map<?, ?> getIndex(String indexName) throws IOException {
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java
index 7b932b943257..ee085aab09ec 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java
@@ -74,7 +74,7 @@ public Settings getAdditionalIndexSettings(
             if (migrating) {
                 indexMode = IndexMode.TIME_SERIES;
             } else if (dataStream != null) {
-                indexMode = dataStream.getIndexMode();
+                indexMode = timeSeries ? dataStream.getIndexMode() : null;
             } else if (timeSeries) {
                 indexMode = IndexMode.TIME_SERIES;
             } else {
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java
index 1a90db516dde..b07ab4c5d5c0 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java
@@ -20,7 +20,7 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -117,7 +117,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         var service = new UpdateTimeSeriesRangeService(environment.settings(), threadPool, clusterService);
         this.service.set(service);
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java
index a2c9959f62b8..4e51f605fc3c 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java
@@ -168,18 +168,11 @@ protected DataStreamsStatsAction.DataStreamShardStats readShardResult(StreamInpu
     }
 
     @Override
-    protected DataStreamsStatsAction.Response newResponse(
-        DataStreamsStatsAction.Request request,
-        int totalShards,
-        int successfulShards,
-        int failedShards,
-        List<DataStreamsStatsAction.DataStreamShardStats> dataStreamShardStats,
-        List<DefaultShardOperationFailedException> shardFailures,
-        ClusterState clusterState
-    ) {
+    protected
+        TransportBroadcastByNodeAction.ResponseFactory<DataStreamsStatsAction.Response, DataStreamsStatsAction.DataStreamShardStats>
+        getResponseFactory(DataStreamsStatsAction.Request request, ClusterState clusterState) {
         Map<String, AggregatedStats> aggregatedDataStreamsStats = new HashMap<>();
         Set<String> allBackingIndices = new HashSet<>();
-        long totalStoreSizeBytes = 0L;
         SortedMap<String, IndexAbstraction> indicesLookup = clusterState.getMetadata().getIndicesLookup();
 
         // Collect the number of backing indices from the cluster state. If every shard operation for an index fails,
@@ -203,43 +196,75 @@ protected DataStreamsStatsAction.Response newResponse(
             }
         }
 
-        for (DataStreamsStatsAction.DataStreamShardStats shardStat : dataStreamShardStats) {
-            String indexName = shardStat.getShardRouting().getIndexName();
-            IndexAbstraction indexAbstraction = indicesLookup.get(indexName);
-            IndexAbstraction.DataStream dataStream = indexAbstraction.getParentDataStream();
-            assert dataStream != null;
+        return new ResponseFactory(indicesLookup, allBackingIndices, aggregatedDataStreamsStats);
+    }
+
+    private class ResponseFactory
+        implements
+            TransportBroadcastByNodeAction.ResponseFactory<DataStreamsStatsAction.Response, DataStreamsStatsAction.DataStreamShardStats> {
 
-            // Aggregate global stats
-            totalStoreSizeBytes += shardStat.getStoreStats().sizeInBytes();
+        private final SortedMap<String, IndexAbstraction> indicesLookup;
+        private final Set<String> allBackingIndices;
+        private final Map<String, AggregatedStats> aggregatedDataStreamsStats;
 
-            // Aggregate data stream stats
-            AggregatedStats stats = aggregatedDataStreamsStats.computeIfAbsent(dataStream.getName(), s -> new AggregatedStats());
-            stats.storageBytes += shardStat.getStoreStats().sizeInBytes();
-            stats.maxTimestamp = Math.max(stats.maxTimestamp, shardStat.getMaxTimestamp());
+        ResponseFactory(
+            SortedMap<String, IndexAbstraction> indicesLookup,
+            Set<String> allBackingIndices,
+            Map<String, AggregatedStats> aggregatedDataStreamsStats
+        ) {
+            this.indicesLookup = indicesLookup;
+            this.allBackingIndices = allBackingIndices;
+            this.aggregatedDataStreamsStats = aggregatedDataStreamsStats;
         }
 
-        DataStreamsStatsAction.DataStreamStats[] dataStreamStats = aggregatedDataStreamsStats.entrySet()
-            .stream()
-            .map(
-                entry -> new DataStreamsStatsAction.DataStreamStats(
-                    entry.getKey(),
-                    entry.getValue().backingIndices.size(),
-                    ByteSizeValue.ofBytes(entry.getValue().storageBytes),
-                    entry.getValue().maxTimestamp
+        @Override
+        public DataStreamsStatsAction.Response newResponse(
+            int totalShards,
+            int successfulShards,
+            int failedShards,
+            List<DataStreamsStatsAction.DataStreamShardStats> dataStreamShardStats,
+            List<DefaultShardOperationFailedException> shardFailures
+        ) {
+            long totalStoreSizeBytes = 0L;
+
+            for (DataStreamsStatsAction.DataStreamShardStats shardStat : dataStreamShardStats) {
+                String indexName = shardStat.getShardRouting().getIndexName();
+                IndexAbstraction indexAbstraction = indicesLookup.get(indexName);
+                IndexAbstraction.DataStream dataStream = indexAbstraction.getParentDataStream();
+                assert dataStream != null;
+
+                // Aggregate global stats
+                totalStoreSizeBytes += shardStat.getStoreStats().sizeInBytes();
+
+                // Aggregate data stream stats
+                AggregatedStats stats = aggregatedDataStreamsStats.computeIfAbsent(dataStream.getName(), s -> new AggregatedStats());
+                stats.storageBytes += shardStat.getStoreStats().sizeInBytes();
+                stats.maxTimestamp = Math.max(stats.maxTimestamp, shardStat.getMaxTimestamp());
+            }
+
+            DataStreamsStatsAction.DataStreamStats[] dataStreamStats = aggregatedDataStreamsStats.entrySet()
+                .stream()
+                .map(
+                    entry -> new DataStreamsStatsAction.DataStreamStats(
+                        entry.getKey(),
+                        entry.getValue().backingIndices.size(),
+                        ByteSizeValue.ofBytes(entry.getValue().storageBytes),
+                        entry.getValue().maxTimestamp
+                    )
                 )
-            )
-            .toArray(DataStreamsStatsAction.DataStreamStats[]::new);
-
-        return new DataStreamsStatsAction.Response(
-            totalShards,
-            successfulShards,
-            failedShards,
-            shardFailures,
-            aggregatedDataStreamsStats.size(),
-            allBackingIndices.size(),
-            ByteSizeValue.ofBytes(totalStoreSizeBytes),
-            dataStreamStats
-        );
+                .toArray(DataStreamsStatsAction.DataStreamStats[]::new);
+
+            return new DataStreamsStatsAction.Response(
+                totalShards,
+                successfulShards,
+                failedShards,
+                shardFailures,
+                aggregatedDataStreamsStats.size(),
+                allBackingIndices.size(),
+                ByteSizeValue.ofBytes(totalStoreSizeBytes),
+                dataStreamStats
+            );
+        }
     }
 
     private static class AggregatedStats {
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java
index b7aceb6b0b46..b60d5af917cb 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.action.admin.indices.rollover.MetadataRolloverService;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
@@ -248,6 +249,7 @@ public void setup() throws Exception {
             when(env.sharedDataFile()).thenReturn(null);
             AllocationService allocationService = mock(AllocationService.class);
             when(allocationService.reroute(any(ClusterState.class), any(String.class), any())).then(i -> i.getArguments()[0]);
+            when(allocationService.getShardRoutingRoleStrategy()).thenReturn(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
             ShardLimitValidator shardLimitValidator = new ShardLimitValidator(Settings.EMPTY, clusterService);
             createIndexService = new MetadataCreateIndexService(
                 Settings.EMPTY,
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java
index 31c702cc4f88..6cc32996b493 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.IndexMode;
@@ -25,7 +26,10 @@
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.List;
+import java.util.Map;
 
+import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createFirstBackingIndex;
+import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance;
 import static org.elasticsearch.common.settings.Settings.builder;
 import static org.elasticsearch.datastreams.DataStreamIndexSettingsProvider.FORMATTER;
 import static org.hamcrest.Matchers.contains;
@@ -34,6 +38,8 @@
 
 public class DataStreamIndexSettingsProviderTests extends ESTestCase {
 
+    private static final TimeValue DEFAULT_LOOK_AHEAD_TIME = TimeValue.timeValueHours(2); // default
+
     DataStreamIndexSettingsProvider provider;
 
     @Before
@@ -48,7 +54,6 @@ public void testGetAdditionalIndexSettings() throws Exception {
         String dataStreamName = "logs-app1";
 
         Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
-        TimeValue lookAheadTime = TimeValue.timeValueHours(2); // default
         Settings settings = Settings.EMPTY;
         String mapping = """
             {
@@ -78,8 +83,8 @@ public void testGetAdditionalIndexSettings() throws Exception {
             List.of(new CompressedXContent(mapping))
         );
         assertThat(result.size(), equalTo(3));
-        assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(lookAheadTime.getMillis())));
-        assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(lookAheadTime.getMillis())));
+        assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
+        assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
         assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), contains("field3"));
     }
 
@@ -309,7 +314,7 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreatedTimeSettingsMi
         assertThat(
             e.getMessage(),
             equalTo(
-                formatted(
+                Strings.format(
                     "backing index [%s] in tsdb mode doesn't have the [index.time_series.end_time] index setting",
                     DataStream.getDefaultBackingIndexName(dataStreamName, 1, twoHoursAgo.toEpochMilli())
                 )
@@ -334,6 +339,55 @@ public void testGetAdditionalIndexSettingsNonTsdbTemplate() {
         assertThat(result.size(), equalTo(0));
     }
 
+    public void testGetAdditionalIndexSettingsMigrateToTsdb() {
+        Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
+        String dataStreamName = "logs-app1";
+        IndexMetadata idx = createFirstBackingIndex(dataStreamName).build();
+        DataStream existingDataStream = newInstance(dataStreamName, List.of(idx.getIndex()));
+        Metadata metadata = Metadata.builder().dataStreams(Map.of(dataStreamName, existingDataStream), Map.of()).build();
+
+        Settings settings = Settings.EMPTY;
+        Settings result = provider.getAdditionalIndexSettings(
+            DataStream.getDefaultBackingIndexName(dataStreamName, 2),
+            dataStreamName,
+            true,
+            metadata,
+            now,
+            settings,
+            List.of()
+        );
+        assertThat(result.size(), equalTo(2));
+        assertThat(IndexSettings.TIME_SERIES_START_TIME.get(result), equalTo(now.minusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
+        assertThat(IndexSettings.TIME_SERIES_END_TIME.get(result), equalTo(now.plusMillis(DEFAULT_LOOK_AHEAD_TIME.getMillis())));
+    }
+
+    public void testGetAdditionalIndexSettingsDowngradeFromTsdb() {
+        String dataStreamName = "logs-app1";
+        Instant twoHoursAgo = Instant.now().minus(4, ChronoUnit.HOURS).truncatedTo(ChronoUnit.MILLIS);
+        Metadata.Builder mb = Metadata.builder(
+            DataStreamTestHelper.getClusterStateWithDataStreams(
+                List.of(Tuple.tuple(dataStreamName, 1)),
+                List.of(),
+                twoHoursAgo.toEpochMilli(),
+                builder().build(),
+                1
+            ).getMetadata()
+        );
+        Metadata metadata = mb.build();
+
+        Settings settings = Settings.EMPTY;
+        Settings result = provider.getAdditionalIndexSettings(
+            DataStream.getDefaultBackingIndexName(dataStreamName, 2),
+            dataStreamName,
+            false,
+            metadata,
+            Instant.ofEpochMilli(1L),
+            settings,
+            List.of()
+        );
+        assertThat(result.size(), equalTo(0));
+    }
+
     public void testGenerateRoutingPathFromDynamicTemplate() throws Exception {
         Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
         TimeValue lookAheadTime = TimeValue.timeValueHours(2); // default
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java
index aebaf1a6c017..ff3013d6f545 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java
@@ -11,8 +11,6 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.ComponentTemplate;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
-import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
 import org.elasticsearch.cluster.metadata.Template;
@@ -21,7 +19,6 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettingProviders;
 import org.elasticsearch.indices.EmptySystemIndices;
@@ -30,8 +27,6 @@
 import org.elasticsearch.indices.ShardLimitValidator;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
-import java.time.Instant;
-import java.time.temporal.ChronoUnit;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
@@ -50,61 +45,6 @@
  */
 public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
 
-    public void testValidateTsdbDataStreamsReferringTsdbTemplate() throws Exception {
-        var state = ClusterState.EMPTY_STATE;
-        final var service = getMetadataIndexTemplateService();
-        var template = new ComposableIndexTemplate(
-            Collections.singletonList("logs-*-*"),
-            new Template(
-                builder().put("index.mode", "time_series").put("index.routing_path", "uid").build(),
-                new CompressedXContent(generateTsdbMapping()),
-                null
-            ),
-            null,
-            100L,
-            null,
-            null,
-            new ComposableIndexTemplate.DataStreamTemplate(false, false),
-            null
-        );
-        state = service.addIndexTemplateV2(state, false, "logs", template);
-
-        var now = Instant.now();
-        var mBuilder = Metadata.builder(state.getMetadata());
-        DataStreamTestHelper.getClusterStateWithDataStream(
-            mBuilder,
-            "unreferenced",
-            List.of(Tuple.tuple(now.minus(2, ChronoUnit.HOURS), now))
-        );
-        DataStreamTestHelper.getClusterStateWithDataStream(
-            mBuilder,
-            "logs-mysql-default",
-            List.of(Tuple.tuple(now.minus(2, ChronoUnit.HOURS), now))
-        );
-        var stateWithDS = ClusterState.builder(state).metadata(mBuilder).build();
-
-        var e = expectThrows(IllegalArgumentException.class, () -> {
-            ComposableIndexTemplate nonDSTemplate = new ComposableIndexTemplate(
-                Collections.singletonList("logs-*-*"),
-                null,
-                null,
-                100L,
-                null,
-                null,
-                new ComposableIndexTemplate.DataStreamTemplate(false, false),
-                null
-            );
-            service.addIndexTemplateV2(stateWithDS, false, "logs", nonDSTemplate);
-        });
-
-        assertThat(
-            e.getMessage(),
-            containsString(
-                "would cause tsdb data streams [logs-mysql-default] to no longer match a data stream template with a time_series index_mode"
-            )
-        );
-    }
-
     public void testRequireRoutingPath() throws Exception {
         final var service = getMetadataIndexTemplateService();
         {
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/CreateDataStreamRequestTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/CreateDataStreamRequestTests.java
index 5ade2fa4d63b..bff238d7f16c 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/CreateDataStreamRequestTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/CreateDataStreamRequestTests.java
@@ -28,6 +28,11 @@ protected Request createTestInstance() {
         return new Request(randomAlphaOfLength(8));
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testValidateRequest() {
         CreateDataStreamAction.Request req = new CreateDataStreamAction.Request("my-data-stream");
         ActionRequestValidationException e = req.validate();
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DataStreamsStatsResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DataStreamsStatsResponseTests.java
index 0c60f667db5a..9874c19a7824 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DataStreamsStatsResponseTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DataStreamsStatsResponseTests.java
@@ -30,6 +30,11 @@ protected DataStreamsStatsAction.Response createTestInstance() {
         return randomStatsResponse();
     }
 
+    @Override
+    protected DataStreamsStatsAction.Response mutateInstance(DataStreamsStatsAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static DataStreamsStatsAction.Response randomStatsResponse() {
         int dataStreamCount = randomInt(10);
         int backingIndicesTotal = 0;
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamRequestTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamRequestTests.java
index 583cc44a5598..d1b3bde33152 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamRequestTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamRequestTests.java
@@ -28,6 +28,11 @@ protected Request createTestInstance() {
         return new Request(randomArray(1, 3, String[]::new, () -> randomAlphaOfLength(6)));
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testValidateRequest() {
         DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[] { "my-data-stream" });
         ActionRequestValidationException e = req.validate();
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsRequestTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsRequestTests.java
index 58e87b07a6b1..5e6a1ee4bc9d 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsRequestTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsRequestTests.java
@@ -34,4 +34,9 @@ protected Request createTestInstance() {
         });
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
 }
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java
index 0e44aa87ae30..37c0d26f9d83 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java
@@ -51,4 +51,9 @@ protected Response createTestInstance() {
         }
         return new Response(dataStreams);
     }
+
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java
index 863a7dc39310..741122b888c9 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java
@@ -79,7 +79,7 @@ public void testPostParse() throws IOException {
         }));
 
         ParsedDocument doc = docMapper.parse(source(b -> b.field("@timestamp", "2020-12-12")));
-        assertThat(doc.rootDoc().getFields("@timestamp").length, equalTo(2));
+        assertThat(doc.rootDoc().getFields("@timestamp").length, equalTo(1));
 
         Exception e = expectThrows(MapperException.class, () -> docMapper.parse(source(b -> b.field("@timestamp1", "2020-12-12"))));
         assertThat(e.getCause().getMessage(), equalTo("data stream timestamp field [@timestamp] is missing"));
diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/MetadataCreateDataStreamServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/MetadataCreateDataStreamServiceTests.java
index 81d7e4766689..9e006034b580 100644
--- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/MetadataCreateDataStreamServiceTests.java
+++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/MetadataCreateDataStreamServiceTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.MapperTestUtils;
 import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
@@ -36,7 +37,7 @@ public void testValidateTimestampFieldMapping() throws Exception {
     public void testValidateTimestampFieldMappingNoFieldMapping() {
         Exception e = expectThrows(IllegalStateException.class, () -> validateTimestampFieldMapping(createMappingLookup("{}")));
         assertThat(e.getMessage(), equalTo("[" + DataStreamTimestampFieldMapper.NAME + "] meta field has been disabled"));
-        String mapping1 = formatted("""
+        String mapping1 = Strings.format("""
             {
               "%s": {
                 "enabled": false
diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/100_delete_by_query.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/100_delete_by_query.yml
index 8b76faf6c44c..7f1ec1f2c685 100644
--- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/100_delete_by_query.yml
+++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/100_delete_by_query.yml
@@ -65,3 +65,88 @@
       indices.delete_data_stream:
         name: simple-data-stream1
   - is_true: acknowledged
+
+---
+"Delete by query for multiple data streams":
+  - skip:
+      features: allowed_warnings
+      version: " - 7.8.99"
+      reason: "data streams available in 7.9+"
+
+  - do:
+      allowed_warnings:
+        - "index template [my-template2] has index patterns [simple-stream*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-template2
+        body:
+          index_patterns: [simple-stream*]
+          data_stream: {}
+
+  - do:
+      indices.create_data_stream:
+        name: simple-stream1
+  - is_true: acknowledged
+
+  - do:
+      index:
+        index:   simple-stream1
+        id:      "1"
+        op_type: create
+        body:    { "number": 4, '@timestamp': '2020-12-12' }
+
+  - do:
+      index:
+        index:   simple-stream2
+        id:      "2"
+        op_type: create
+        body:    { "number": 4, '@timestamp': '2020-12-12' }
+
+  - do:
+      index:
+        index:   simple-stream2
+        id:      "3"
+        op_type: create
+        body:    { "number": 6, '@timestamp': '2020-12-12' }
+
+  - do:
+      indices.refresh:
+        index: simple-stream1,simple-stream2
+
+  # delete any docs with number <= 4
+  - do:
+      delete_by_query:
+        index: simple-stream*
+        body:
+          query:
+            range:
+              number:
+                lte: 4
+
+  - match: {deleted: 2}
+  - match: {version_conflicts: 0}
+  - match: {batches: 1}
+  - match: {noops: 0}
+  - match: {failures: []}
+  - match: {throttled_millis: 0}
+  - gte: { took: 0 }
+
+  - do:
+      indices.refresh:
+        index: simple-stream1,simple-stream2
+
+  # verify that both documents with number originally <= 4 have been deleted
+  - do:
+      search:
+        index: simple-stream*
+        body: { query: { range: { number: { lte: 5 } } } }
+  - length:   { hits.hits: 0 }
+
+  - do:
+      indices.delete_data_stream:
+        name: simple-stream1
+  - is_true: acknowledged
+
+  - do:
+      indices.delete_data_stream:
+        name: simple-stream2
+  - is_true: acknowledged
diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/110_update_by_query.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/110_update_by_query.yml
index ef39fe124cfa..ccf7cead1d70 100644
--- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/110_update_by_query.yml
+++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/110_update_by_query.yml
@@ -104,3 +104,91 @@
       indices.delete_data_stream:
         name: simple-data-stream1
   - is_true: acknowledged
+
+---
+"Update by query for multiple data streams":
+  - skip:
+      features: allowed_warnings
+      version: " - 7.8.99"
+      reason: "data streams available in 7.9+"
+
+  - do:
+      allowed_warnings:
+        - "index template [my-template2] has index patterns [simple-stream*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-template2
+        body:
+          index_patterns: [simple-stream*]
+          data_stream: {}
+
+  - do:
+      indices.create_data_stream:
+        name: simple-stream1
+  - is_true: acknowledged
+
+  - do:
+      index:
+        index:   simple-stream1
+        id:      "1"
+        op_type: create
+        body:    { "number": 4, '@timestamp': '2020-12-12' }
+
+  - do:
+      index:
+        index:   simple-stream2
+        id:      "2"
+        op_type: create
+        body:    { "number": 4, '@timestamp': '2020-12-12' }
+
+  - do:
+      index:
+        index:   simple-stream2
+        id:      "3"
+        op_type: create
+        body:    { "number": 6, '@timestamp': '2020-12-12' }
+
+  - do:
+      indices.refresh:
+        index: simple-stream1,simple-stream2
+
+  # increment by one any docs with number <= 4
+  - do:
+      update_by_query:
+        index: simple-stream*
+        body:
+          script:
+            source: "ctx._source.number++"
+            lang: "painless"
+          query:
+            range:
+              number:
+                lte: 4
+
+  - match: {updated: 2}
+  - match: {version_conflicts: 0}
+  - match: {batches: 1}
+  - match: {noops: 0}
+  - match: {failures: []}
+  - match: {throttled_millis: 0}
+  - gte: { took: 0 }
+
+  - do:
+      indices.refresh:
+        index: simple-stream1,simple-stream2
+
+  # verify that both numbers originally <= 4 have been incremented by one
+  - do:
+      search:
+        index: simple-stream*
+        body: { query: { range: { number: { lte: 5 } } } }
+  - length:   { hits.hits: 2  }
+
+  - do:
+      indices.delete_data_stream:
+        name: simple-stream1
+  - is_true: acknowledged
+
+  - do:
+      indices.delete_data_stream:
+        name: simple-stream2
+  - is_true: acknowledged
diff --git a/modules/ingest-attachment/build.gradle b/modules/ingest-attachment/build.gradle
index 64d74ef8b356..36dfd027cd81 100644
--- a/modules/ingest-attachment/build.gradle
+++ b/modules/ingest-attachment/build.gradle
@@ -7,8 +7,8 @@ import org.elasticsearch.gradle.internal.info.BuildParams
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
   description 'Ingest processor that uses Apache Tika to extract contents'
@@ -16,8 +16,8 @@ esplugin {
 }
 
 versions << [
-  'tika'  : '2.4.0',
-  'pdfbox': '2.0.26',
+  'tika'  : '2.6.0',
+  'pdfbox': '2.0.27',
   'poi'   : '5.2.2',
   'mime4j': '0.8.5'
 ]
@@ -49,7 +49,7 @@ dependencies {
   // Adobe PDF
   api "org.apache.pdfbox:pdfbox:${versions.pdfbox}"
   api "org.apache.pdfbox:fontbox:${versions.pdfbox}"
-  api "org.apache.pdfbox:jempbox:1.8.16"
+  api "org.apache.pdfbox:jempbox:1.8.17"
   api "commons-logging:commons-logging:${versions.commonslogging}"
   // OpenOffice
   api "org.apache.poi:poi-ooxml:${versions.poi}"
diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle
index 958e64d2ac4b..e48156ef98da 100644
--- a/modules/ingest-common/build.gradle
+++ b/modules/ingest-common/build.gradle
@@ -5,8 +5,8 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
diff --git a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java
index 5ac11cd9963c..152ec847a490 100644
--- a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java
+++ b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.ingest.IngestStats;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptEngine;
@@ -61,7 +62,7 @@ public void testFailureInConditionalProcessor() {
         internalCluster().ensureAtLeastNumDataNodes(1);
         internalCluster().startMasterOnlyNode();
         final String pipelineId = "foo";
-        client().admin().cluster().preparePutPipeline(pipelineId, new BytesArray(formatted("""
+        client().admin().cluster().preparePutPipeline(pipelineId, new BytesArray(Strings.format("""
             {
               "processors": [
                 {
@@ -109,7 +110,7 @@ public void testScriptDisabled() throws Exception {
         String pipelineIdWithScript = pipelineIdWithoutScript + "_script";
         internalCluster().startNode();
 
-        BytesReference pipelineWithScript = new BytesArray(formatted("""
+        BytesReference pipelineWithScript = new BytesArray(Strings.format("""
             {
               "processors": [ { "script": { "lang": "%s", "source": "my_script" } } ]
             }""", MockScriptEngine.NAME));
@@ -179,7 +180,7 @@ public Settings onNodeStopped(String nodeName) {
     public void testPipelineWithScriptProcessorThatHasStoredScript() throws Exception {
         internalCluster().startNode();
 
-        client().admin().cluster().preparePutStoredScript().setId("1").setContent(new BytesArray(formatted("""
+        client().admin().cluster().preparePutStoredScript().setId("1").setContent(new BytesArray(Strings.format("""
             {"script": {"lang": "%s", "source": "my_script"} }
             """, MockScriptEngine.NAME)), XContentType.JSON).get();
         BytesReference pipeline = new BytesArray("""
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java
index 4a9fa93662bd..84c4315a6901 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java
@@ -36,11 +36,10 @@ enum DateFormat {
         @Override
         Function<String, ZonedDateTime> getFunction(String format, ZoneId timezone, Locale locale) {
             return (date) -> {
-                TemporalAccessor accessor = DateFormatter.forPattern("iso8601").parse(date);
+                TemporalAccessor accessor = ISO_8601.parse(date);
                 // even though locale could be set to en-us, Locale.ROOT (following iso8601 calendar data rules) should be used
                 return DateFormatters.from(accessor, Locale.ROOT, timezone).withZoneSameInstant(timezone);
             };
-
         }
     },
     Unix {
@@ -115,6 +114,14 @@ Function<String, ZonedDateTime> getFunction(String format, ZoneId zoneId, Locale
         }
     };
 
+    /** It's important to keep this variable as a constant because {@link DateFormatter#forPattern(String)} is an expensive method and,
+     * in this case, it's a never changing value.
+     * <br>
+     * Also, we shouldn't inline it in the {@link DateFormat#Iso8601}'s enum because it'd make useless the cache used
+     * at {@link DateProcessor}).
+     */
+    private static final DateFormatter ISO_8601 = DateFormatter.forPattern("iso8601");
+
     abstract Function<String, ZonedDateTime> getFunction(String format, ZoneId timezone, Locale locale);
 
     static DateFormat fromString(String format) {
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
index 71a73f605a4c..8ef870f77377 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
@@ -9,8 +9,10 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.util.LocaleUtils;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
@@ -19,6 +21,7 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.TemplateScript;
 
+import java.lang.ref.SoftReference;
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
@@ -26,7 +29,9 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 public final class DateProcessor extends AbstractProcessor {
 
@@ -72,9 +77,17 @@ public final class DateProcessor extends AbstractProcessor {
         this.targetField = targetField;
         this.formats = formats;
         this.dateParsers = new ArrayList<>(this.formats.size());
+
         for (String format : formats) {
             DateFormat dateFormat = DateFormat.fromString(format);
-            dateParsers.add((params) -> dateFormat.getFunction(format, newDateTimeZone(params), newLocale(params)));
+            dateParsers.add((params) -> {
+                var documentZoneId = newDateTimeZone(params);
+                var documentLocale = newLocale(params);
+                return Cache.INSTANCE.getOrCompute(
+                    new Cache.Key(format, documentZoneId, documentLocale),
+                    () -> dateFormat.getFunction(format, documentZoneId, documentLocale)
+                );
+            });
         }
         this.outputFormat = outputFormat;
         formatter = DateFormatter.forPattern(this.outputFormat);
@@ -198,4 +211,50 @@ public DateProcessor create(
             );
         }
     }
+
+    /**
+     * An ad-hoc cache class that just throws away the cached values once it's full because we don't want to affect the performance
+     * while applying eviction policies when adding new values or retrieving them.
+     */
+    static final class Cache {
+
+        private static final String CACHE_CAPACITY_SETTING = "es.ingest.date_processor.cache_capacity";
+        static final Cache INSTANCE;
+
+        static {
+            var cacheSizeStr = System.getProperty(CACHE_CAPACITY_SETTING, "256");
+            try {
+                INSTANCE = new Cache(Integer.parseInt(cacheSizeStr));
+            } catch (NumberFormatException e) {
+                throw new SettingsException("{} must be a valid number but was [{}]", CACHE_CAPACITY_SETTING, cacheSizeStr);
+            }
+        }
+        private final ConcurrentMap<Key, SoftReference<Function<String, ZonedDateTime>>> map;
+        private final int capacity;
+
+        Cache(int capacity) {
+            if (capacity <= 0) {
+                throw new IllegalArgumentException("cache capacity must be a value greater than 0 but was " + capacity);
+            }
+            this.capacity = capacity;
+            this.map = ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency(this.capacity);
+        }
+
+        Function<String, ZonedDateTime> getOrCompute(Key key, Supplier<Function<String, ZonedDateTime>> supplier) {
+            Function<String, ZonedDateTime> fn;
+            var element = map.get(key);
+            // element exist and wasn't GCed
+            if (element != null && (fn = element.get()) != null) {
+                return fn;
+            }
+            if (map.size() >= capacity) {
+                map.clear();
+            }
+            fn = supplier.get();
+            map.put(key, new SoftReference<>(fn));
+            return fn;
+        }
+
+        record Key(String format, ZoneId zoneId, Locale locale) {}
+    }
 }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java
index 5dcf944eaa2a..a13b7d21bc11 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
@@ -57,7 +57,7 @@ public Request(boolean sorted, String ecsCompatibility) {
         Request(StreamInput in) throws IOException {
             super(in);
             this.sorted = in.readBoolean();
-            this.ecsCompatibility = in.getVersion().onOrAfter(Version.V_8_0_0)
+            this.ecsCompatibility = in.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)
                 ? in.readString()
                 : GrokProcessor.DEFAULT_ECS_COMPATIBILITY_MODE;
         }
@@ -71,7 +71,7 @@ public ActionRequestValidationException validate() {
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             out.writeBoolean(sorted);
-            if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
                 out.writeString(ecsCompatibility);
             }
         }
@@ -144,18 +144,15 @@ public TransportAction(TransportService transportService, ActionFilters actionFi
 
         @Override
         protected void doExecute(Task task, Request request, ActionListener<Response> listener) {
-            try {
-                listener.onResponse(
-                    new Response(
-                        request.getEcsCompatibility().equals(Grok.ECS_COMPATIBILITY_MODES[0])
-                            ? request.sorted() ? sortedLegacyGrokPatterns : legacyGrokPatterns
-                            : request.sorted() ? sortedEcsV1GrokPatterns
-                            : ecsV1GrokPatterns
-                    )
-                );
-            } catch (Exception e) {
-                listener.onFailure(e);
-            }
+            ActionListener.completeWith(
+                listener,
+                () -> new Response(
+                    request.getEcsCompatibility().equals(Grok.ECS_COMPATIBILITY_MODES[0])
+                        ? request.sorted() ? sortedLegacyGrokPatterns : legacyGrokPatterns
+                        : request.sorted() ? sortedEcsV1GrokPatterns
+                        : ecsV1GrokPatterns
+                )
+            );
         }
     }
 
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java
index a79954de0f35..7b20cfbf0b39 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java
@@ -10,6 +10,7 @@
 
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
@@ -32,12 +33,14 @@
 public final class JsonProcessor extends AbstractProcessor {
 
     public static final String TYPE = "json";
+    private static final String STRICT_JSON_PARSING_PARAMETER = "strict_json_parsing";
 
     private final String field;
     private final String targetField;
     private final boolean addToRoot;
     private final ConflictStrategy addToRootConflictStrategy;
     private final boolean allowDuplicateKeys;
+    private final boolean strictJsonParsing;
 
     JsonProcessor(
         String tag,
@@ -47,6 +50,19 @@ public final class JsonProcessor extends AbstractProcessor {
         boolean addToRoot,
         ConflictStrategy addToRootConflictStrategy,
         boolean allowDuplicateKeys
+    ) {
+        this(tag, description, field, targetField, addToRoot, addToRootConflictStrategy, allowDuplicateKeys, true);
+    }
+
+    JsonProcessor(
+        String tag,
+        String description,
+        String field,
+        String targetField,
+        boolean addToRoot,
+        ConflictStrategy addToRootConflictStrategy,
+        boolean allowDuplicateKeys,
+        boolean strictJsonParsing
     ) {
         super(tag, description);
         this.field = field;
@@ -54,6 +70,7 @@ public final class JsonProcessor extends AbstractProcessor {
         this.addToRoot = addToRoot;
         this.addToRootConflictStrategy = addToRootConflictStrategy;
         this.allowDuplicateKeys = allowDuplicateKeys;
+        this.strictJsonParsing = strictJsonParsing;
     }
 
     public String getField() {
@@ -72,7 +89,7 @@ public ConflictStrategy getAddToRootConflictStrategy() {
         return addToRootConflictStrategy;
     }
 
-    public static Object apply(Object fieldValue, boolean allowDuplicateKeys) {
+    public static Object apply(Object fieldValue, boolean allowDuplicateKeys, boolean strictJsonParsing) {
         BytesReference bytesRef = fieldValue == null ? new BytesArray("null") : new BytesArray(fieldValue.toString());
         try (
             InputStream stream = bytesRef.streamInput();
@@ -96,14 +113,42 @@ public static Object apply(Object fieldValue, boolean allowDuplicateKeys) {
             } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
                 throw new IllegalArgumentException("cannot read binary value");
             }
+            if (strictJsonParsing) {
+                String errorMessage = Strings.format(
+                    "The input %s is not valid JSON and the %s parameter is true",
+                    fieldValue,
+                    STRICT_JSON_PARSING_PARAMETER
+                );
+                /*
+                 * If strict JSON parsing is disabled, then once we've found the first token then we move on. For example for the string
+                 * "123 \"foo\"" we would just return the first token, 123. However, if strict parsing is enabled (which it is by default),
+                 * then we check to see whether there are any more tokens at this point. We expect the next token to be null. If there is
+                 * another token or if the parser blows up, then we know we had invalid JSON and we alert the user with an
+                 * IllegalArgumentException.
+                 */
+                try {
+                    token = parser.nextToken();
+                } catch (IllegalArgumentException e) {
+                    throw new IllegalArgumentException(errorMessage, e);
+                }
+                if (token != null) {
+                    throw new IllegalArgumentException(errorMessage);
+                }
+            }
             return value;
         } catch (IOException e) {
             throw new IllegalArgumentException(e);
         }
     }
 
-    public static void apply(Map<String, Object> ctx, String fieldName, boolean allowDuplicateKeys, ConflictStrategy conflictStrategy) {
-        Object value = apply(ctx.get(fieldName), allowDuplicateKeys);
+    public static void apply(
+        Map<String, Object> ctx,
+        String fieldName,
+        boolean allowDuplicateKeys,
+        ConflictStrategy conflictStrategy,
+        boolean strictJsonParsing
+    ) {
+        Object value = apply(ctx.get(fieldName), allowDuplicateKeys, strictJsonParsing);
         if (value instanceof Map) {
             @SuppressWarnings("unchecked")
             Map<String, Object> map = (Map<String, Object>) value;
@@ -140,9 +185,9 @@ public static void recursiveMerge(Map<String, Object> target, Map<String, Object
     @Override
     public IngestDocument execute(IngestDocument document) throws Exception {
         if (addToRoot) {
-            apply(document.getSourceAndMetadata(), field, allowDuplicateKeys, addToRootConflictStrategy);
+            apply(document.getSourceAndMetadata(), field, allowDuplicateKeys, addToRootConflictStrategy, strictJsonParsing);
         } else {
-            document.setFieldValue(targetField, apply(document.getFieldValue(field, Object.class), allowDuplicateKeys));
+            document.setFieldValue(targetField, apply(document.getFieldValue(field, Object.class), allowDuplicateKeys, strictJsonParsing));
         }
         return document;
     }
@@ -217,6 +262,7 @@ public JsonProcessor create(
                     "Cannot set `add_to_root_conflict_strategy` if `add_to_root` is false"
                 );
             }
+            boolean strictParsing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, STRICT_JSON_PARSING_PARAMETER, true);
 
             if (targetField == null) {
                 targetField = field;
@@ -229,7 +275,8 @@ public JsonProcessor create(
                 targetField,
                 addToRoot,
                 addToRootConflictStrategy,
-                allowDuplicateKeys
+                allowDuplicateKeys,
+                strictParsing
             );
         }
     }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java
index 5a5e1b5144c0..7285983782e6 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java
@@ -59,7 +59,19 @@ public static String uppercase(String value) {
      * @return structured JSON object
      */
     public static Object json(Object fieldValue) {
-        return JsonProcessor.apply(fieldValue, false);
+        return JsonProcessor.apply(fieldValue, false, true);
+    }
+
+    /**
+     * Uses {@link JsonProcessor} to convert a JSON string to a structured JSON
+     * object. This method is a more lenient version of {@link #json(Object)}. For example if given fieldValue "123 foo",
+     * this method will return 123 rather than throwing an IllegalArgumentException.
+     *
+     * @param fieldValue JSON string
+     * @return structured JSON object
+     */
+    public static Object jsonLenient(Object fieldValue) {
+        return JsonProcessor.apply(fieldValue, false, false);
     }
 
     /**
@@ -72,7 +84,22 @@ public static Object json(Object fieldValue) {
      *             contains the JSON string
      */
     public static void json(Map<String, Object> map, String field) {
-        JsonProcessor.apply(map, field, false, JsonProcessor.ConflictStrategy.REPLACE);
+        JsonProcessor.apply(map, field, false, JsonProcessor.ConflictStrategy.REPLACE, true);
+    }
+
+    /**
+     * Uses {@link JsonProcessor} to convert a JSON string to a structured JSON
+     * object. This method is a more lenient version of {@link #json(Map, String)}. For example if given fieldValue
+     * "{"foo":"bar"} 123",
+     * this method will return a map with key-vale pair "foo" and "bar" rather than throwing an IllegalArgumentException.
+     *
+     * @param map map that contains the JSON string and will receive the
+     *            structured JSON content
+     * @param field key that identifies the entry in <code>map</code> that
+     *             contains the JSON string
+     */
+    public static void jsonLenient(Map<String, Object> map, String field) {
+        JsonProcessor.apply(map, field, false, JsonProcessor.ConflictStrategy.REPLACE, false);
     }
 
     /**
diff --git a/modules/ingest-common/src/main/resources/org/elasticsearch/ingest/common/processors_whitelist.txt b/modules/ingest-common/src/main/resources/org/elasticsearch/ingest/common/processors_whitelist.txt
index 4d05bdfc283b..e1779aade519 100644
--- a/modules/ingest-common/src/main/resources/org/elasticsearch/ingest/common/processors_whitelist.txt
+++ b/modules/ingest-common/src/main/resources/org/elasticsearch/ingest/common/processors_whitelist.txt
@@ -13,7 +13,9 @@ class org.elasticsearch.ingest.common.Processors {
   String lowercase(String)
   String uppercase(String)
   Object json(Object)
+  Object jsonLenient(Object)
   void json(Map, String)
+  void jsonLenient(Map, String)
   String urlDecode(String)
   String communityId(String, String, Object, Object, Object, Object, Object, Object, int)
   String communityId(String, String, Object, Object, Object, Object, Object, Object)
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorFactoryTestCase.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorFactoryTestCase.java
index b0e74000da57..209375ac12ed 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorFactoryTestCase.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorFactoryTestCase.java
@@ -15,7 +15,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
 public abstract class AbstractStringProcessorFactoryTestCase extends ESTestCase {
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java
index 54f11932bfe5..daccbc9559d4 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java
@@ -15,13 +15,12 @@
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
-import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 
 public class AppendProcessorFactoryTests extends ESTestCase {
 
@@ -39,14 +38,14 @@ public void testCreate() throws Exception {
         if (randomBoolean()) {
             value = "value1";
         } else {
-            value = Arrays.asList("value1", "value2", "value3");
+            value = List.of("value1", "value2", "value3");
         }
         config.put("value", value);
         String processorTag = randomAlphaOfLength(10);
         AppendProcessor appendProcessor = factory.create(null, processorTag, null, config);
         assertThat(appendProcessor.getTag(), equalTo(processorTag));
-        assertThat(appendProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("field1"));
-        assertThat(appendProcessor.getValue().copyAndResolve(Collections.emptyMap()), equalTo(value));
+        assertThat(appendProcessor.getField().newInstance(Map.of()).execute(), equalTo("field1"));
+        assertThat(appendProcessor.getValue().copyAndResolve(Map.of()), equalTo(value));
     }
 
     public void testCreateNoFieldPresent() throws Exception {
@@ -110,7 +109,7 @@ public void testMediaType() throws Exception {
 
         // invalid media type
         expectedMediaType = randomValueOtherThanMany(
-            m -> Arrays.asList(ConfigurationUtils.VALID_MEDIA_TYPES).contains(m),
+            m -> List.of(ConfigurationUtils.VALID_MEDIA_TYPES).contains(m),
             () -> randomAlphaOfLengthBetween(5, 9)
         );
         final Map<String, Object> config2 = new HashMap<>();
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorTests.java
index d789f1bd79c4..3b5c2d39a3ca 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorTests.java
@@ -27,11 +27,11 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.hamcrest.CoreMatchers.not;
-import static org.hamcrest.CoreMatchers.sameInstance;
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.sameInstance;
 
 public class AppendProcessorTests extends ESTestCase {
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java
index 1b35ea427c59..8ca6e71b4a51 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.RandomDocumentPicks;
-import org.hamcrest.CoreMatchers;
 
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
 public class BytesProcessorTests extends AbstractStringProcessorTestCase<Long> {
@@ -51,14 +51,8 @@ public void testTooLarge() {
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "8912pb");
         Processor processor = newProcessor(fieldName, randomBoolean(), fieldName);
         ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> processor.execute(ingestDocument));
-        assertThat(
-            exception.getMessage(),
-            CoreMatchers.equalTo("failed to parse setting [Ingest Field] with value [8912pb] as a size in bytes")
-        );
-        assertThat(
-            exception.getCause().getMessage(),
-            CoreMatchers.containsString("Values greater than 9223372036854775807 bytes are not supported")
-        );
+        assertThat(exception.getMessage(), equalTo("failed to parse setting [Ingest Field] with value [8912pb] as a size in bytes"));
+        assertThat(exception.getCause().getMessage(), containsString("Values greater than 9223372036854775807 bytes are not supported"));
     }
 
     public void testNotBytes() {
@@ -66,7 +60,7 @@ public void testNotBytes() {
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "junk");
         Processor processor = newProcessor(fieldName, randomBoolean(), fieldName);
         ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> processor.execute(ingestDocument));
-        assertThat(exception.getMessage(), CoreMatchers.equalTo("failed to parse setting [Ingest Field] with value [junk]"));
+        assertThat(exception.getMessage(), equalTo("failed to parse setting [Ingest Field] with value [junk]"));
     }
 
     public void testMissingUnits() {
@@ -74,7 +68,7 @@ public void testMissingUnits() {
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "1");
         Processor processor = newProcessor(fieldName, randomBoolean(), fieldName);
         ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> processor.execute(ingestDocument));
-        assertThat(exception.getMessage(), CoreMatchers.containsString("unit is missing or unrecognized"));
+        assertThat(exception.getMessage(), containsString("unit is missing or unrecognized"));
     }
 
     public void testFractional() throws Exception {
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorFactoryTests.java
index 3ded531131f3..8d08040a4058 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorFactoryTests.java
@@ -25,8 +25,8 @@
 import static org.elasticsearch.ingest.common.CommunityIdProcessor.Factory.DEFAULT_TARGET;
 import static org.elasticsearch.ingest.common.CommunityIdProcessor.Factory.DEFAULT_TRANSPORT;
 import static org.elasticsearch.ingest.common.CommunityIdProcessor.toUint16;
-import static org.hamcrest.CoreMatchers.containsString;
-import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 
 public class CommunityIdProcessorFactoryTests extends ESTestCase {
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java
index 60ae12dd1aa0..df31f0ece6f6 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java
@@ -10,12 +10,11 @@
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.test.ESTestCase;
-import org.hamcrest.Matchers;
 
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -46,7 +45,7 @@ public void testCreateUnsupportedType() throws Exception {
             factory.create(null, null, null, config);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), Matchers.equalTo("[type] type [" + type + "] not supported, cannot convert field."));
+            assertThat(e.getMessage(), equalTo("[type] type [" + type + "] not supported, cannot convert field."));
             assertThat(e.getMetadata("es.processor_type").get(0), equalTo(ConvertProcessor.TYPE));
             assertThat(e.getMetadata("es.property_name").get(0), equalTo("type"));
             assertThat(e.getMetadata("es.processor_tag"), nullValue());
@@ -62,7 +61,7 @@ public void testCreateNoFieldPresent() throws Exception {
             factory.create(null, null, null, config);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), Matchers.equalTo("[field] required property is missing"));
+            assertThat(e.getMessage(), equalTo("[field] required property is missing"));
         }
     }
 
@@ -74,7 +73,7 @@ public void testCreateNoTypePresent() throws Exception {
             factory.create(null, null, null, config);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), Matchers.equalTo("[type] required property is missing"));
+            assertThat(e.getMessage(), equalTo("[type] required property is missing"));
         }
     }
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
index 0590934ccd36..f1b7433e10a0 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
@@ -514,7 +514,7 @@ public void testAutoConvertNotString() throws Exception {
             }
             default -> throw new UnsupportedOperationException();
         }
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomValue));
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of("field", randomValue));
         Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
@@ -523,7 +523,7 @@ public void testAutoConvertNotString() throws Exception {
 
     public void testAutoConvertStringNotMatched() throws Exception {
         String value = "notAnIntFloatOrBool";
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", value));
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of("field", value));
         Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
@@ -533,10 +533,7 @@ public void testAutoConvertStringNotMatched() throws Exception {
     public void testAutoConvertMatchBoolean() throws Exception {
         boolean randomBoolean = randomBoolean();
         String booleanString = Boolean.toString(randomBoolean);
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(
-            random(),
-            Collections.singletonMap("field", booleanString)
-        );
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of("field", booleanString));
         Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
@@ -546,7 +543,7 @@ public void testAutoConvertMatchBoolean() throws Exception {
     public void testAutoConvertMatchInteger() throws Exception {
         int randomInt = randomInt();
         String randomString = Integer.toString(randomInt);
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of("field", randomString));
         Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
@@ -556,7 +553,7 @@ public void testAutoConvertMatchInteger() throws Exception {
     public void testAutoConvertMatchLong() throws Exception {
         long randomLong = randomLong();
         String randomString = Long.toString(randomLong);
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of("field", randomString));
         Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
@@ -567,7 +564,7 @@ public void testAutoConvertDoubleNotMatched() throws Exception {
         double randomDouble = randomDouble();
         String randomString = Double.toString(randomDouble);
         float randomFloat = Float.parseFloat(randomString);
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of("field", randomString));
         Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
@@ -578,7 +575,7 @@ public void testAutoConvertDoubleNotMatched() throws Exception {
     public void testAutoConvertMatchFloat() throws Exception {
         float randomFloat = randomFloat();
         String randomString = Float.toString(randomFloat);
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", randomString));
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of("field", randomString));
         Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, "field", "field", Type.AUTO, false);
         processor.execute(ingestDocument);
         Object convertedValue = ingestDocument.getFieldValue("field", Object.class);
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorFactoryTests.java
index c4a631d05c35..7053e079a87d 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorFactoryTests.java
@@ -10,10 +10,10 @@
 
 import org.elasticsearch.test.ESTestCase;
 
-import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
-import static java.util.Collections.emptyMap;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
@@ -24,7 +24,7 @@ public void testProcessorIsCreated() {
         CsvProcessor.Factory factory = new CsvProcessor.Factory();
         HashMap<String, Object> properties = new HashMap<>();
         properties.put("field", "field");
-        properties.put("target_fields", Collections.singletonList("target"));
+        properties.put("target_fields", List.of("target"));
         properties.put("quote", "|");
         properties.put("separator", "/");
         properties.put("empty_value", "empty");
@@ -39,6 +39,6 @@ public void testProcessorIsCreated() {
         assertThat(csv.emptyValue, equalTo("empty"));
         assertThat(csv.trim, equalTo(true));
         assertThat(csv.ignoreMissing, equalTo(true));
-        assertThat(properties, is(emptyMap()));
+        assertThat(properties, is(Map.of()));
     }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java
index 2d3b3b9a4a9d..f7c94e24881d 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java
@@ -19,8 +19,8 @@
 import java.util.Locale;
 import java.util.function.Function;
 
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.core.IsEqual.equalTo;
 
 public class DateFormatTests extends ESTestCase {
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java
index 32b181050102..1e247655c875 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java
@@ -11,14 +11,14 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.test.ESTestCase;
-import org.hamcrest.Matchers;
 
 import java.time.ZoneOffset;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.equalTo;
+
 public class DateIndexNameFactoryTests extends ESTestCase {
 
     public void testDefaults() throws Exception {
@@ -28,12 +28,12 @@ public void testDefaults() throws Exception {
         config.put("date_rounding", "y");
 
         DateIndexNameProcessor processor = factory.create(null, null, null, config);
-        assertThat(processor.getDateFormats().size(), Matchers.equalTo(1));
-        assertThat(processor.getField(), Matchers.equalTo("_field"));
-        assertThat(processor.getIndexNamePrefixTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo(""));
-        assertThat(processor.getDateRoundingTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("y"));
-        assertThat(processor.getIndexNameFormatTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("yyyy-MM-dd"));
-        assertThat(processor.getTimezone(), Matchers.equalTo(ZoneOffset.UTC));
+        assertThat(processor.getDateFormats().size(), equalTo(1));
+        assertThat(processor.getField(), equalTo("_field"));
+        assertThat(processor.getIndexNamePrefixTemplate().newInstance(Map.of()).execute(), equalTo(""));
+        assertThat(processor.getDateRoundingTemplate().newInstance(Map.of()).execute(), equalTo("y"));
+        assertThat(processor.getIndexNameFormatTemplate().newInstance(Map.of()).execute(), equalTo("yyyy-MM-dd"));
+        assertThat(processor.getTimezone(), equalTo(ZoneOffset.UTC));
     }
 
     public void testSpecifyOptionalSettings() throws Exception {
@@ -42,10 +42,10 @@ public void testSpecifyOptionalSettings() throws Exception {
         config.put("field", "_field");
         config.put("index_name_prefix", "_prefix");
         config.put("date_rounding", "y");
-        config.put("date_formats", Arrays.asList("UNIX", "UNIX_MS"));
+        config.put("date_formats", List.of("UNIX", "UNIX_MS"));
 
         DateIndexNameProcessor processor = factory.create(null, null, null, config);
-        assertThat(processor.getDateFormats().size(), Matchers.equalTo(2));
+        assertThat(processor.getDateFormats().size(), equalTo(2));
 
         config = new HashMap<>();
         config.put("field", "_field");
@@ -54,7 +54,7 @@ public void testSpecifyOptionalSettings() throws Exception {
         config.put("index_name_format", "yyyyMMdd");
 
         processor = factory.create(null, null, null, config);
-        assertThat(processor.getIndexNameFormatTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("yyyyMMdd"));
+        assertThat(processor.getIndexNameFormatTemplate().newInstance(Map.of()).execute(), equalTo("yyyyMMdd"));
 
         config = new HashMap<>();
         config.put("field", "_field");
@@ -63,7 +63,7 @@ public void testSpecifyOptionalSettings() throws Exception {
         config.put("timezone", "+02:00");
 
         processor = factory.create(null, null, null, config);
-        assertThat(processor.getTimezone(), Matchers.equalTo(ZoneOffset.ofHours(2)));
+        assertThat(processor.getTimezone(), equalTo(ZoneOffset.ofHours(2)));
 
         config = new HashMap<>();
         config.put("field", "_field");
@@ -71,7 +71,7 @@ public void testSpecifyOptionalSettings() throws Exception {
         config.put("date_rounding", "y");
 
         processor = factory.create(null, null, null, config);
-        assertThat(processor.getIndexNamePrefixTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("_prefix"));
+        assertThat(processor.getIndexNamePrefixTemplate().newInstance(Map.of()).execute(), equalTo("_prefix"));
     }
 
     public void testRequiredFields() throws Exception {
@@ -79,11 +79,11 @@ public void testRequiredFields() throws Exception {
         Map<String, Object> config = new HashMap<>();
         config.put("date_rounding", "y");
         ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
-        assertThat(e.getMessage(), Matchers.equalTo("[field] required property is missing"));
+        assertThat(e.getMessage(), equalTo("[field] required property is missing"));
 
         config.clear();
         config.put("field", "_field");
         e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
-        assertThat(e.getMessage(), Matchers.equalTo("[date_rounding] required property is missing"));
+        assertThat(e.getMessage(), equalTo("[date_rounding] required property is missing"));
     }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java
index a79580d743a0..0257bbcd46a7 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java
@@ -15,54 +15,33 @@
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
-import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.function.Function;
 
-import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.equalTo;
 
 public class DateIndexNameProcessorTests extends ESTestCase {
 
     public void testJavaPattern() throws Exception {
         Function<String, ZonedDateTime> function = DateFormat.Java.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSXX", ZoneOffset.UTC, Locale.ROOT);
-        DateIndexNameProcessor processor = createProcessor(
-            "_field",
-            Collections.singletonList(function),
-            ZoneOffset.UTC,
-            "events-",
-            "y",
-            "yyyyMMdd"
-        );
-        IngestDocument document = new IngestDocument(
-            "_index",
-            "_id",
-            1,
-            null,
-            null,
-            Collections.singletonMap("_field", "2016-04-25T12:24:20.101Z")
-        );
+        DateIndexNameProcessor processor = createProcessor("_field", List.of(function), ZoneOffset.UTC, "events-", "y", "yyyyMMdd");
+        IngestDocument document = new IngestDocument("_index", "_id", 1, null, null, Map.of("_field", "2016-04-25T12:24:20.101Z"));
         processor.execute(document);
         assertThat(document.getSourceAndMetadata().get("_index"), equalTo("<events-{20160425||/y{yyyyMMdd|UTC}}>"));
     }
 
     public void testTAI64N() throws Exception {
         Function<String, ZonedDateTime> function = DateFormat.Tai64n.getFunction(null, ZoneOffset.UTC, null);
-        DateIndexNameProcessor dateProcessor = createProcessor(
-            "_field",
-            Collections.singletonList(function),
-            ZoneOffset.UTC,
-            "events-",
-            "m",
-            "yyyyMMdd"
-        );
+        DateIndexNameProcessor dateProcessor = createProcessor("_field", List.of(function), ZoneOffset.UTC, "events-", "m", "yyyyMMdd");
         IngestDocument document = new IngestDocument(
             "_index",
             "_id",
             1,
             null,
             null,
-            Collections.singletonMap("_field", (randomBoolean() ? "@" : "") + "4000000050d506482dbdf024")
+            Map.of("_field", (randomBoolean() ? "@" : "") + "4000000050d506482dbdf024")
         );
         dateProcessor.execute(document);
         assertThat(document.getSourceAndMetadata().get("_index"), equalTo("<events-{20121222||/m{yyyyMMdd|UTC}}>"));
@@ -70,34 +49,20 @@ public void testTAI64N() throws Exception {
 
     public void testUnixMs() throws Exception {
         Function<String, ZonedDateTime> function = DateFormat.UnixMs.getFunction(null, ZoneOffset.UTC, null);
-        DateIndexNameProcessor dateProcessor = createProcessor(
-            "_field",
-            Collections.singletonList(function),
-            ZoneOffset.UTC,
-            "events-",
-            "m",
-            "yyyyMMdd"
-        );
-        IngestDocument document = new IngestDocument("_index", "_id", 1, null, null, Collections.singletonMap("_field", "1000500"));
+        DateIndexNameProcessor dateProcessor = createProcessor("_field", List.of(function), ZoneOffset.UTC, "events-", "m", "yyyyMMdd");
+        IngestDocument document = new IngestDocument("_index", "_id", 1, null, null, Map.of("_field", "1000500"));
         dateProcessor.execute(document);
         assertThat(document.getSourceAndMetadata().get("_index"), equalTo("<events-{19700101||/m{yyyyMMdd|UTC}}>"));
 
-        document = new IngestDocument("_index", "_id", 1, null, null, Collections.singletonMap("_field", 1000500L));
+        document = new IngestDocument("_index", "_id", 1, null, null, Map.of("_field", 1000500L));
         dateProcessor.execute(document);
         assertThat(document.getSourceAndMetadata().get("_index"), equalTo("<events-{19700101||/m{yyyyMMdd|UTC}}>"));
     }
 
     public void testUnix() throws Exception {
         Function<String, ZonedDateTime> function = DateFormat.Unix.getFunction(null, ZoneOffset.UTC, null);
-        DateIndexNameProcessor dateProcessor = createProcessor(
-            "_field",
-            Collections.singletonList(function),
-            ZoneOffset.UTC,
-            "events-",
-            "m",
-            "yyyyMMdd"
-        );
-        IngestDocument document = new IngestDocument("_index", "_id", 1, null, null, Collections.singletonMap("_field", "1000.5"));
+        DateIndexNameProcessor dateProcessor = createProcessor("_field", List.of(function), ZoneOffset.UTC, "events-", "m", "yyyyMMdd");
+        IngestDocument document = new IngestDocument("_index", "_id", 1, null, null, Map.of("_field", "1000.5"));
         dateProcessor.execute(document);
         assertThat(document.getSourceAndMetadata().get("_index"), equalTo("<events-{19700101||/m{yyyyMMdd|UTC}}>"));
     }
@@ -111,14 +76,14 @@ public void testTemplatedFields() throws Exception {
 
         DateIndexNameProcessor dateProcessor = createProcessor(
             "_field",
-            Collections.singletonList(dateTimeFunction),
+            List.of(dateTimeFunction),
             ZoneOffset.UTC,
             indexNamePrefix,
             dateRounding,
             indexNameFormat
         );
 
-        IngestDocument document = new IngestDocument("_index", "_id", 1, null, null, Collections.singletonMap("_field", date));
+        IngestDocument document = new IngestDocument("_index", "_id", 1, null, null, Map.of("_field", date));
         dateProcessor.execute(document);
 
         assertThat(
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java
index 5245a336acf4..52aa66c37d6a 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java
@@ -14,9 +14,8 @@
 import org.junit.Before;
 
 import java.time.ZoneId;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
@@ -36,13 +35,13 @@ public void testBuildDefaults() throws Exception {
         Map<String, Object> config = new HashMap<>();
         String sourceField = randomAlphaOfLengthBetween(1, 10);
         config.put("field", sourceField);
-        config.put("formats", Collections.singletonList("dd/MM/yyyyy"));
+        config.put("formats", List.of("dd/MM/yyyyy"));
         String processorTag = randomAlphaOfLength(10);
         DateProcessor processor = factory.create(null, processorTag, null, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo(sourceField));
         assertThat(processor.getTargetField(), equalTo(DateProcessor.DEFAULT_TARGET_FIELD));
-        assertThat(processor.getFormats(), equalTo(Collections.singletonList("dd/MM/yyyyy")));
+        assertThat(processor.getFormats(), equalTo(List.of("dd/MM/yyyyy")));
         assertNull(processor.getLocale());
         assertNull(processor.getTimezone());
     }
@@ -51,7 +50,7 @@ public void testMatchFieldIsMandatory() throws Exception {
         Map<String, Object> config = new HashMap<>();
         String targetField = randomAlphaOfLengthBetween(1, 10);
         config.put("target_field", targetField);
-        config.put("formats", Collections.singletonList("dd/MM/yyyyy"));
+        config.put("formats", List.of("dd/MM/yyyyy"));
 
         try {
             factory.create(null, null, null, config);
@@ -80,34 +79,34 @@ public void testParseLocale() throws Exception {
         Map<String, Object> config = new HashMap<>();
         String sourceField = randomAlphaOfLengthBetween(1, 10);
         config.put("field", sourceField);
-        config.put("formats", Collections.singletonList("dd/MM/yyyyy"));
+        config.put("formats", List.of("dd/MM/yyyyy"));
         Locale locale = randomFrom(Locale.GERMANY, Locale.FRENCH, Locale.ROOT);
         config.put("locale", locale.toLanguageTag());
 
         DateProcessor processor = factory.create(null, null, null, config);
-        assertThat(processor.getLocale().newInstance(Collections.emptyMap()).execute(), equalTo(locale.toLanguageTag()));
+        assertThat(processor.getLocale().newInstance(Map.of()).execute(), equalTo(locale.toLanguageTag()));
     }
 
     public void testParseTimezone() throws Exception {
         Map<String, Object> config = new HashMap<>();
         String sourceField = randomAlphaOfLengthBetween(1, 10);
         config.put("field", sourceField);
-        config.put("formats", Collections.singletonList("dd/MM/yyyyy"));
+        config.put("formats", List.of("dd/MM/yyyyy"));
 
         ZoneId timezone = randomZone();
         config.put("timezone", timezone.getId());
         DateProcessor processor = factory.create(null, null, null, config);
-        assertThat(processor.getTimezone().newInstance(Collections.emptyMap()).execute(), equalTo(timezone.getId()));
+        assertThat(processor.getTimezone().newInstance(Map.of()).execute(), equalTo(timezone.getId()));
     }
 
     public void testParseMatchFormats() throws Exception {
         Map<String, Object> config = new HashMap<>();
         String sourceField = randomAlphaOfLengthBetween(1, 10);
         config.put("field", sourceField);
-        config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
+        config.put("formats", List.of("dd/MM/yyyy", "dd-MM-yyyy"));
 
         DateProcessor processor = factory.create(null, null, null, config);
-        assertThat(processor.getFormats(), equalTo(Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy")));
+        assertThat(processor.getFormats(), equalTo(List.of("dd/MM/yyyy", "dd-MM-yyyy")));
     }
 
     public void testParseMatchFormatsFailure() throws Exception {
@@ -130,7 +129,7 @@ public void testParseTargetField() throws Exception {
         String targetField = randomAlphaOfLengthBetween(1, 10);
         config.put("field", sourceField);
         config.put("target_field", targetField);
-        config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
+        config.put("formats", List.of("dd/MM/yyyy", "dd-MM-yyyy"));
 
         DateProcessor processor = factory.create(null, null, null, config);
         assertThat(processor.getTargetField(), equalTo(targetField));
@@ -143,7 +142,7 @@ public void testParseOutputFormat() throws Exception {
         String targetField = randomAlphaOfLengthBetween(1, 10);
         config.put("field", sourceField);
         config.put("target_field", targetField);
-        config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
+        config.put("formats", List.of("dd/MM/yyyy", "dd-MM-yyyy"));
         config.put("output_format", outputFormat);
         DateProcessor processor = factory.create(null, null, null, config);
         assertThat(processor.getOutputFormat(), equalTo(outputFormat));
@@ -155,7 +154,7 @@ public void testDefaultOutputFormat() throws Exception {
         String targetField = randomAlphaOfLengthBetween(1, 10);
         config.put("field", sourceField);
         config.put("target_field", targetField);
-        config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
+        config.put("formats", List.of("dd/MM/yyyy", "dd-MM-yyyy"));
         DateProcessor processor = factory.create(null, null, null, config);
         assertThat(processor.getOutputFormat(), equalTo(DateProcessor.DEFAULT_OUTPUT_FORMAT));
     }
@@ -167,7 +166,7 @@ public void testInvalidOutputFormatRejected() throws Exception {
         String targetField = randomAlphaOfLengthBetween(1, 10);
         config.put("field", sourceField);
         config.put("target_field", targetField);
-        config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
+        config.put("formats", List.of("dd/MM/yyyy", "dd-MM-yyyy"));
         config.put("output_format", outputFormat);
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> factory.create(null, null, null, config));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
index be6ff3c5fdef..18454c866cb2 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.ingest.TestTemplateService;
@@ -19,15 +20,19 @@
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.function.Function;
+import java.util.function.Supplier;
 
-import static org.hamcrest.CoreMatchers.containsString;
-import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 public class DateProcessorTests extends ESTestCase {
 
@@ -46,7 +51,7 @@ public void testJavaPattern() {
             templatize(ZoneId.of("Europe/Amsterdam")),
             templatize(Locale.ENGLISH),
             "date_as_string",
-            Collections.singletonList("yyyy dd MM HH:mm:ss"),
+            List.of("yyyy dd MM HH:mm:ss"),
             "date_as_date"
         );
         Map<String, Object> document = new HashMap<>();
@@ -129,7 +134,7 @@ public void testJavaPatternNoTimezone() {
             null,
             null,
             "date_as_string",
-            Arrays.asList("yyyy dd MM HH:mm:ss XXX"),
+            List.of("yyyy dd MM HH:mm:ss XXX"),
             "date_as_date"
         );
 
@@ -148,7 +153,7 @@ public void testInvalidJavaPattern() {
                 templatize(ZoneOffset.UTC),
                 templatize(randomLocale(random())),
                 "date_as_string",
-                Collections.singletonList("invalid pattern"),
+                List.of("invalid pattern"),
                 "date_as_date"
             );
             Map<String, Object> document = new HashMap<>();
@@ -169,7 +174,7 @@ public void testJavaPatternLocale() {
             templatize(ZoneId.of("Europe/Amsterdam")),
             templatize(Locale.ITALIAN),
             "date_as_string",
-            Collections.singletonList("yyyy dd MMMM"),
+            List.of("yyyy dd MMMM"),
             "date_as_date"
         );
         Map<String, Object> document = new HashMap<>();
@@ -187,7 +192,7 @@ public void testJavaPatternEnglishLocale() {
             templatize(ZoneId.of("Europe/Amsterdam")),
             templatize(Locale.ENGLISH),
             "date_as_string",
-            Collections.singletonList("yyyy dd MMMM"),
+            List.of("yyyy dd MMMM"),
             "date_as_date"
         );
         Map<String, Object> document = new HashMap<>();
@@ -205,7 +210,7 @@ public void testJavaPatternDefaultYear() {
             templatize(ZoneId.of("Europe/Amsterdam")),
             templatize(Locale.ENGLISH),
             "date_as_string",
-            Collections.singletonList(format),
+            List.of(format),
             "date_as_date"
         );
         Map<String, Object> document = new HashMap<>();
@@ -225,7 +230,7 @@ public void testTAI64N() {
             templatize(ZoneOffset.ofHours(2)),
             templatize(randomLocale(random())),
             "date_as_string",
-            Collections.singletonList("TAI64N"),
+            List.of("TAI64N"),
             "date_as_date"
         );
         Map<String, Object> document = new HashMap<>();
@@ -243,7 +248,7 @@ public void testUnixMs() {
             templatize(ZoneOffset.UTC),
             templatize(randomLocale(random())),
             "date_as_string",
-            Collections.singletonList("UNIX_MS"),
+            List.of("UNIX_MS"),
             "date_as_date"
         );
         Map<String, Object> document = new HashMap<>();
@@ -266,7 +271,7 @@ public void testUnix() {
             templatize(ZoneOffset.UTC),
             templatize(randomLocale(random())),
             "date_as_string",
-            Collections.singletonList("UNIX"),
+            List.of("UNIX"),
             "date_as_date"
         );
         Map<String, Object> document = new HashMap<>();
@@ -283,7 +288,7 @@ public void testInvalidTimezone() {
             new TestTemplateService.MockTemplateScript.Factory("invalid_timezone"),
             templatize(randomLocale(random())),
             "date_as_string",
-            Collections.singletonList("yyyy"),
+            List.of("yyyy"),
             "date_as_date"
         );
         Map<String, Object> document = new HashMap<>();
@@ -303,7 +308,7 @@ public void testInvalidLocale() {
             templatize(ZoneOffset.UTC),
             new TestTemplateService.MockTemplateScript.Factory("invalid_locale"),
             "date_as_string",
-            Collections.singletonList("yyyy"),
+            List.of("yyyy"),
             "date_as_date"
         );
         Map<String, Object> document = new HashMap<>();
@@ -324,7 +329,7 @@ public void testOutputFormat() {
             null,
             null,
             "date_as_string",
-            Collections.singletonList("iso8601"),
+            List.of("iso8601"),
             "date_as_date",
             "HH:mm:ss.SSSSSSSSS"
         );
@@ -333,7 +338,34 @@ public void testOutputFormat() {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
         processor.execute(ingestDocument);
         // output format is time only with nanosecond precision
-        String expectedDate = "00:00:00." + formatted("%09d", nanosAfterEpoch);
+        String expectedDate = "00:00:00." + Strings.format("%09d", nanosAfterEpoch);
         assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo(expectedDate));
     }
+
+    @SuppressWarnings("unchecked")
+    public void testCacheIsEvictedAfterReachMaxCapacity() {
+        Supplier<Function<String, ZonedDateTime>> supplier1 = mock(Supplier.class);
+        Supplier<Function<String, ZonedDateTime>> supplier2 = mock(Supplier.class);
+        Function<String, ZonedDateTime> zonedDateTimeFunction1 = str -> ZonedDateTime.now();
+        Function<String, ZonedDateTime> zonedDateTimeFunction2 = str -> ZonedDateTime.now();
+        var cache = new DateProcessor.Cache(1);
+        var key1 = new DateProcessor.Cache.Key("format-1", ZoneId.systemDefault(), Locale.ROOT);
+        var key2 = new DateProcessor.Cache.Key("format-2", ZoneId.systemDefault(), Locale.ROOT);
+
+        when(supplier1.get()).thenReturn(zonedDateTimeFunction1);
+        when(supplier2.get()).thenReturn(zonedDateTimeFunction2);
+
+        assertEquals(cache.getOrCompute(key1, supplier1), zonedDateTimeFunction1); // 1 call to supplier1
+        assertEquals(cache.getOrCompute(key2, supplier2), zonedDateTimeFunction2); // 1 call to supplier2
+        assertEquals(cache.getOrCompute(key1, supplier1), zonedDateTimeFunction1); // 1 more call to supplier1
+        assertEquals(cache.getOrCompute(key1, supplier1), zonedDateTimeFunction1); // should use cached value
+        assertEquals(cache.getOrCompute(key2, supplier2), zonedDateTimeFunction2); // 1 more call to supplier2
+        assertEquals(cache.getOrCompute(key2, supplier2), zonedDateTimeFunction2); // should use cached value
+        assertEquals(cache.getOrCompute(key2, supplier2), zonedDateTimeFunction2); // should use cached value
+        assertEquals(cache.getOrCompute(key2, supplier2), zonedDateTimeFunction2); // should use cached value
+        assertEquals(cache.getOrCompute(key1, supplier1), zonedDateTimeFunction1); // 1 more to call to supplier1
+
+        verify(supplier1, times(3)).get();
+        verify(supplier2, times(2)).get();
+    }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorFactoryTests.java
index c426ee1ef8be..b4bd9fea704b 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorFactoryTests.java
@@ -12,14 +12,13 @@
 import org.elasticsearch.dissect.DissectException;
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
-import org.hamcrest.Matchers;
 
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
 
 public class DissectProcessorFactoryTests extends ESTestCase {
 
@@ -50,7 +49,7 @@ public void testCreateMissingField() {
         Map<String, Object> config = new HashMap<>();
         config.put("pattern", "%{a},%{b},%{c}");
         Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config));
-        assertThat(e.getMessage(), Matchers.equalTo("[field] required property is missing"));
+        assertThat(e.getMessage(), equalTo("[field] required property is missing"));
     }
 
     public void testCreateMissingPattern() {
@@ -58,7 +57,7 @@ public void testCreateMissingPattern() {
         Map<String, Object> config = new HashMap<>();
         config.put("field", randomAlphaOfLength(10));
         Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config));
-        assertThat(e.getMessage(), Matchers.equalTo("[pattern] required property is missing"));
+        assertThat(e.getMessage(), equalTo("[pattern] required property is missing"));
     }
 
     public void testCreateMissingOptionals() {
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorTests.java
index ec2762ebfe53..f57d82a7c0d1 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorTests.java
@@ -14,12 +14,13 @@
 import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
-import org.hamcrest.CoreMatchers;
 
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Map;
 
 import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
 /**
@@ -29,14 +30,7 @@
 public class DissectProcessorTests extends ESTestCase {
 
     public void testMatch() {
-        IngestDocument ingestDocument = new IngestDocument(
-            "_index",
-            "_id",
-            1,
-            null,
-            null,
-            Collections.singletonMap("message", "foo,bar,baz")
-        );
+        IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Map.of("message", "foo,bar,baz"));
         DissectProcessor dissectProcessor = new DissectProcessor("", null, "message", "%{a},%{b},%{c}", "", true);
         dissectProcessor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue("a", String.class), equalTo("foo"));
@@ -68,7 +62,7 @@ public void testAdvancedMatch() {
             1,
             null,
             null,
-            Collections.singletonMap("message", "foo       bar,,,,,,,baz nope:notagain 😊 🐇 🙃")
+            Map.of("message", "foo       bar,,,,,,,baz nope:notagain 😊 🐇 🙃")
         );
         DissectProcessor dissectProcessor = new DissectProcessor(
             "",
@@ -87,17 +81,10 @@ public void testAdvancedMatch() {
     }
 
     public void testMiss() {
-        IngestDocument ingestDocument = new IngestDocument(
-            "_index",
-            "_id",
-            1,
-            null,
-            null,
-            Collections.singletonMap("message", "foo:bar,baz")
-        );
+        IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Map.of("message", "foo:bar,baz"));
         DissectProcessor dissectProcessor = new DissectProcessor("", null, "message", "%{a},%{b},%{c}", "", true);
         DissectException e = expectThrows(DissectException.class, () -> dissectProcessor.execute(ingestDocument));
-        assertThat(e.getMessage(), CoreMatchers.containsString("Unable to find match for dissect pattern"));
+        assertThat(e.getMessage(), containsString("Unable to find match for dissect pattern"));
     }
 
     public void testNonStringValueWithIgnoreMissing() {
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java
index fc17506555ed..3e3c7af96486 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.test.ESTestCase;
 
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -43,7 +42,7 @@ public void testEscapeFields() throws Exception {
 
         source = new HashMap<>();
         source.put("foo.bar", "baz1");
-        source.put("foo", new HashMap<>(Collections.singletonMap("bar", "baz2")));
+        source.put("foo", new HashMap<>(Map.of("bar", "baz2")));
         document = TestIngestDocument.withDefaultVersion(source);
         processor = new DotExpanderProcessor("_tag", null, null, "foo.bar");
         processor.execute(document);
@@ -55,7 +54,7 @@ public void testEscapeFields() throws Exception {
 
         source = new HashMap<>();
         source.put("foo.bar", "2");
-        source.put("foo", new HashMap<>(Collections.singletonMap("bar", 1)));
+        source.put("foo", new HashMap<>(Map.of("bar", 1)));
         document = TestIngestDocument.withDefaultVersion(source);
         processor = new DotExpanderProcessor("_tag", null, null, "foo.bar");
         processor.execute(document);
@@ -103,7 +102,7 @@ public void testEscapeFields_valueField() throws Exception {
 
         source = new HashMap<>();
         source.put("foo.bar.baz", "baz1");
-        source.put("foo", new HashMap<>(Collections.singletonMap("bar", new HashMap<>())));
+        source.put("foo", new HashMap<>(Map.of("bar", new HashMap<>())));
         document = TestIngestDocument.withDefaultVersion(source);
         processor = new DotExpanderProcessor("_tag", null, null, "foo.bar.baz");
         processor.execute(document);
@@ -113,7 +112,7 @@ public void testEscapeFields_valueField() throws Exception {
 
         source = new HashMap<>();
         source.put("foo.bar.baz", "baz1");
-        source.put("foo", new HashMap<>(Collections.singletonMap("bar", "baz2")));
+        source.put("foo", new HashMap<>(Map.of("bar", "baz2")));
         IngestDocument document2 = TestIngestDocument.withDefaultVersion(source);
         Processor processor2 = new DotExpanderProcessor("_tag", null, null, "foo.bar.baz");
         e = expectThrows(IllegalArgumentException.class, () -> processor2.execute(document2));
@@ -122,7 +121,7 @@ public void testEscapeFields_valueField() throws Exception {
 
     public void testEscapeFields_path() throws Exception {
         Map<String, Object> source = new HashMap<>();
-        source.put("foo", new HashMap<>(Collections.singletonMap("bar.baz", "value")));
+        source.put("foo", new HashMap<>(Map.of("bar.baz", "value")));
         IngestDocument document = TestIngestDocument.withDefaultVersion(source);
         DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, "foo", "bar.baz");
         processor.execute(document);
@@ -131,7 +130,7 @@ public void testEscapeFields_path() throws Exception {
         assertThat(document.getFieldValue("foo.bar.baz", String.class), equalTo("value"));
 
         source = new HashMap<>();
-        source.put("field", new HashMap<>(Collections.singletonMap("foo.bar.baz", "value")));
+        source.put("field", new HashMap<>(Map.of("foo.bar.baz", "value")));
         document = TestIngestDocument.withDefaultVersion(source);
         processor = new DotExpanderProcessor("_tag", null, "field", "foo.bar.baz");
         processor.execute(document);
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java
index 4cf6ce1df10e..ad73ef65c6c9 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java
@@ -14,11 +14,10 @@
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.equalTo;
 
 public class FailProcessorFactoryTests extends ESTestCase {
 
@@ -35,7 +34,7 @@ public void testCreate() throws Exception {
         String processorTag = randomAlphaOfLength(10);
         FailProcessor failProcessor = factory.create(null, processorTag, null, config);
         assertThat(failProcessor.getTag(), equalTo(processorTag));
-        assertThat(failProcessor.getMessage().newInstance(Collections.emptyMap()).execute(), equalTo("error"));
+        assertThat(failProcessor.getMessage().newInstance(Map.of()).execute(), equalTo("error"));
     }
 
     public void testCreateMissingMessageField() throws Exception {
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorFactoryTests.java
index e7ec74520046..f9f808a0fe84 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorFactoryTests.java
@@ -14,7 +14,6 @@
 
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
@@ -75,7 +74,7 @@ public void testMethod() throws Exception {
 
         // invalid method
         String invalidMethod = randomValueOtherThanMany(
-            m -> Arrays.asList(FingerprintProcessor.Factory.SUPPORTED_DIGESTS).contains(m),
+            m -> List.of(FingerprintProcessor.Factory.SUPPORTED_DIGESTS).contains(m),
             () -> randomAlphaOfLengthBetween(5, 9)
         );
         config.put("fields", fieldList);
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java
index e5bd1910aa92..233d8fb2b860 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java
@@ -13,20 +13,20 @@
 import org.elasticsearch.ingest.TestProcessor;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
-import org.hamcrest.Matchers;
 
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
-import java.util.function.Consumer;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.sameInstance;
 import static org.mockito.Mockito.mock;
 
 public class ForEachProcessorFactoryTests extends ESTestCase {
 
     private final ScriptService scriptService = mock(ScriptService.class);
-    private final Consumer<Runnable> genericExecutor = Runnable::run;
 
     public void testCreate() throws Exception {
         Processor processor = new TestProcessor(ingestDocument -> {});
@@ -36,11 +36,11 @@ public void testCreate() throws Exception {
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        config.put("processor", Collections.singletonMap("_name", Collections.emptyMap()));
+        config.put("processor", Map.of("_name", Collections.emptyMap()));
         ForEachProcessor forEachProcessor = forEachFactory.create(registry, null, null, config);
-        assertThat(forEachProcessor, Matchers.notNullValue());
+        assertThat(forEachProcessor, notNullValue());
         assertThat(forEachProcessor.getField(), equalTo("_field"));
-        assertThat(forEachProcessor.getInnerProcessor(), Matchers.sameInstance(processor));
+        assertThat(forEachProcessor.getInnerProcessor(), sameInstance(processor));
         assertFalse(forEachProcessor.isIgnoreMissing());
     }
 
@@ -52,12 +52,12 @@ public void testSetIgnoreMissing() throws Exception {
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        config.put("processor", Collections.singletonMap("_name", Collections.emptyMap()));
+        config.put("processor", Map.of("_name", Collections.emptyMap()));
         config.put("ignore_missing", true);
         ForEachProcessor forEachProcessor = forEachFactory.create(registry, null, null, config);
-        assertThat(forEachProcessor, Matchers.notNullValue());
+        assertThat(forEachProcessor, notNullValue());
         assertThat(forEachProcessor.getField(), equalTo("_field"));
-        assertThat(forEachProcessor.getInnerProcessor(), Matchers.sameInstance(processor));
+        assertThat(forEachProcessor.getInnerProcessor(), sameInstance(processor));
         assertTrue(forEachProcessor.isIgnoreMissing());
     }
 
@@ -71,8 +71,8 @@ public void testCreateWithTooManyProcessorTypes() throws Exception {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         Map<String, Object> processorTypes = new HashMap<>();
-        processorTypes.put("_first", Collections.emptyMap());
-        processorTypes.put("_second", Collections.emptyMap());
+        processorTypes.put("_first", Map.of());
+        processorTypes.put("_second", Map.of());
         config.put("processor", processorTypes);
         Exception exception = expectThrows(ElasticsearchParseException.class, () -> forEachFactory.create(registry, null, null, config));
         assertThat(exception.getMessage(), equalTo("[processor] Must specify exactly one processor type"));
@@ -82,10 +82,10 @@ public void testCreateWithNonExistingProcessorType() throws Exception {
         ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        config.put("processor", Collections.singletonMap("_name", Collections.emptyMap()));
+        config.put("processor", Map.of("_name", Collections.emptyMap()));
         Exception expectedException = expectThrows(
             ElasticsearchParseException.class,
-            () -> forEachFactory.create(Collections.emptyMap(), null, null, config)
+            () -> forEachFactory.create(Map.of(), null, null, config)
         );
         assertThat(expectedException.getMessage(), equalTo("No processor type exists with name [_name]"));
     }
@@ -96,7 +96,7 @@ public void testCreateWithMissingField() throws Exception {
         registry.put("_name", (r, t, description, c) -> processor);
         ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
         Map<String, Object> config = new HashMap<>();
-        config.put("processor", Collections.singletonList(Collections.singletonMap("_name", Collections.emptyMap())));
+        config.put("processor", List.of(Map.of("_name", Map.of())));
         Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(registry, null, null, config));
         assertThat(exception.getMessage(), equalTo("[field] required property is missing"));
     }
@@ -105,7 +105,7 @@ public void testCreateWithMissingProcessor() {
         ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(Collections.emptyMap(), null, null, config));
+        Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(Map.of(), null, null, config));
         assertThat(exception.getMessage(), equalTo("[processor] required property is missing"));
     }
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java
index 0d96e4c6680b..5f9936bd0c96 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java
@@ -17,8 +17,6 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
@@ -27,7 +25,6 @@
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
-import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
 import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
@@ -41,7 +38,7 @@ public void testExecuteWithAsyncProcessor() throws Exception {
         values.add("foo");
         values.add("bar");
         values.add("baz");
-        IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Collections.singletonMap("values", values));
+        IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Map.of("values", values));
 
         ForEachProcessor processor = new ForEachProcessor("_tag", null, "values", new AsyncUpperCaseProcessor("_ingest._value"), false);
         execProcessor(processor, ingestDocument, (result, e) -> {});
@@ -57,14 +54,7 @@ public void testExecuteWithAsyncProcessor() throws Exception {
     }
 
     public void testExecuteWithFailure() {
-        IngestDocument ingestDocument = new IngestDocument(
-            "_index",
-            "_id",
-            1,
-            null,
-            null,
-            Collections.singletonMap("values", Arrays.asList("a", "b", "c"))
-        );
+        IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Map.of("values", List.of("a", "b", "c")));
 
         TestProcessor testProcessor = new TestProcessor(id -> {
             if ("c".equals(id.getFieldValue("_ingest._value", String.class))) {
@@ -76,7 +66,7 @@ public void testExecuteWithFailure() {
         execProcessor(processor, ingestDocument, (result, e) -> exceptions[0] = e);
         assertThat(exceptions[0].getMessage(), equalTo("failure"));
         assertThat(testProcessor.getInvokedCounter(), equalTo(3));
-        assertThat(ingestDocument.getFieldValue("values", List.class), equalTo(Arrays.asList("a", "b", "c")));
+        assertThat(ingestDocument.getFieldValue("values", List.class), equalTo(List.of("a", "b", "c")));
 
         testProcessor = new TestProcessor(id -> {
             String value = id.getFieldValue("_ingest._value", String.class);
@@ -91,12 +81,12 @@ public void testExecuteWithFailure() {
             "_tag",
             null,
             "values",
-            new CompoundProcessor(false, Arrays.asList(testProcessor), Arrays.asList(onFailureProcessor)),
+            new CompoundProcessor(false, List.of(testProcessor), List.of(onFailureProcessor)),
             false
         );
         execProcessor(processor, ingestDocument, (result, e) -> {});
         assertThat(testProcessor.getInvokedCounter(), equalTo(3));
-        assertThat(ingestDocument.getFieldValue("values", List.class), equalTo(Arrays.asList("A", "B", "c")));
+        assertThat(ingestDocument.getFieldValue("values", List.class), equalTo(List.of("A", "B", "c")));
         assertThat(ingestDocument.getFieldValue("error", String.class), equalTo("foo"));
     }
 
@@ -104,7 +94,7 @@ public void testMetadataAvailable() {
         List<Map<String, Object>> values = new ArrayList<>();
         values.add(new HashMap<>());
         values.add(new HashMap<>());
-        IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Collections.singletonMap("values", values));
+        IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Map.of("values", values));
 
         TestProcessor innerProcessor = new TestProcessor(id -> {
             id.setFieldValue("_ingest._value.index", id.getSourceAndMetadata().get("_index"));
@@ -180,9 +170,9 @@ public String getDescription() {
             }
         };
         int numValues = randomIntBetween(1, 10000);
-        List<String> values = IntStream.range(0, numValues).mapToObj(i -> "").collect(Collectors.toList());
+        List<String> values = IntStream.range(0, numValues).mapToObj(i -> "").toList();
 
-        IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Collections.singletonMap("values", values));
+        IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Map.of("values", values));
 
         ForEachProcessor processor = new ForEachProcessor("_tag", null, "values", innerProcessor, false);
         execProcessor(processor, ingestDocument, (result, e) -> {});
@@ -198,7 +188,7 @@ public void testModifyFieldsOutsideArray() {
         values.add("string");
         values.add(1);
         values.add(null);
-        IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Collections.singletonMap("values", values));
+        IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Map.of("values", values));
 
         TemplateScript.Factory template = new TestTemplateService.MockTemplateScript.Factory("errors");
 
@@ -209,7 +199,7 @@ public void testModifyFieldsOutsideArray() {
             new CompoundProcessor(
                 false,
                 List.of(new UppercaseProcessor("_tag_upper", null, "_ingest._value", false, "_ingest._value")),
-                List.of(new AppendProcessor("_tag", null, template, (model) -> (Collections.singletonList("added")), true))
+                List.of(new AppendProcessor("_tag", null, template, (model) -> (List.of("added")), true))
             ),
             false
         );
@@ -262,7 +252,7 @@ public void testNestedForEach() {
         value.put("values2", innerValues);
         values.add(value);
 
-        IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Collections.singletonMap("values1", values));
+        IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Map.of("values1", values));
 
         TestProcessor testProcessor = new TestProcessor(
             doc -> doc.setFieldValue("_ingest._value", doc.getFieldValue("_ingest._value", String.class).toUpperCase(Locale.ENGLISH))
@@ -337,7 +327,7 @@ public void testNestedForEachWithMapIteration() {
     }
 
     public void testIgnoreMissing() {
-        IngestDocument originalIngestDocument = new IngestDocument("_index", "_id", 1, null, null, Collections.emptyMap());
+        IngestDocument originalIngestDocument = new IngestDocument("_index", "_id", 1, null, null, Map.of());
         IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
         TestProcessor testProcessor = new TestProcessor(doc -> {});
         ForEachProcessor processor = new ForEachProcessor("_tag", null, "_ingest._value", testProcessor, true);
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java
index 300f58de884e..41154634b43e 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java
@@ -12,8 +12,8 @@
 import org.elasticsearch.grok.MatcherWatchdog;
 import org.elasticsearch.test.ESTestCase;
 
-import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -27,7 +27,7 @@ public void testBuild() throws Exception {
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        config.put("patterns", Collections.singletonList("(?<foo>\\w+)"));
+        config.put("patterns", List.of("(?<foo>\\w+)"));
         String processorTag = randomAlphaOfLength(10);
         GrokProcessor processor = factory.create(null, processorTag, null, config);
         assertThat(processor.getTag(), equalTo(processorTag));
@@ -41,7 +41,7 @@ public void testBuildWithIgnoreMissing() throws Exception {
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        config.put("patterns", Collections.singletonList("(?<foo>\\w+)"));
+        config.put("patterns", List.of("(?<foo>\\w+)"));
         config.put("ignore_missing", true);
         String processorTag = randomAlphaOfLength(10);
         GrokProcessor processor = factory.create(null, processorTag, null, config);
@@ -54,7 +54,7 @@ public void testBuildWithIgnoreMissing() throws Exception {
     public void testBuildMissingField() throws Exception {
         GrokProcessor.Factory factory = new GrokProcessor.Factory(MatcherWatchdog.noop());
         Map<String, Object> config = new HashMap<>();
-        config.put("patterns", Collections.singletonList("(?<foo>\\w+)"));
+        config.put("patterns", List.of("(?<foo>\\w+)"));
         ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
         assertThat(e.getMessage(), equalTo("[field] required property is missing"));
     }
@@ -71,7 +71,7 @@ public void testBuildEmptyPatternsList() throws Exception {
         GrokProcessor.Factory factory = new GrokProcessor.Factory(MatcherWatchdog.noop());
         Map<String, Object> config = new HashMap<>();
         config.put("field", "foo");
-        config.put("patterns", Collections.emptyList());
+        config.put("patterns", List.of());
         ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
         assertThat(e.getMessage(), equalTo("[patterns] List of patterns must not be empty"));
     }
@@ -81,8 +81,8 @@ public void testCreateWithCustomPatterns() throws Exception {
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        config.put("patterns", Collections.singletonList("%{MY_PATTERN:name}!"));
-        config.put("pattern_definitions", Collections.singletonMap("MY_PATTERN", "foo"));
+        config.put("patterns", List.of("%{MY_PATTERN:name}!"));
+        config.put("pattern_definitions", Map.of("MY_PATTERN", "foo"));
         GrokProcessor processor = factory.create(null, null, null, config);
         assertThat(processor.getMatchField(), equalTo("_field"));
         assertThat(processor.getGrok(), notNullValue());
@@ -93,7 +93,7 @@ public void testCreateWithInvalidPattern() throws Exception {
         GrokProcessor.Factory factory = new GrokProcessor.Factory(MatcherWatchdog.noop());
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        config.put("patterns", Collections.singletonList("["));
+        config.put("patterns", List.of("["));
         ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
         assertThat(e.getMessage(), equalTo("[patterns] Invalid regex pattern found in: [[]. premature end of char-class"));
     }
@@ -102,8 +102,8 @@ public void testCreateWithInvalidPatternDefinition() throws Exception {
         GrokProcessor.Factory factory = new GrokProcessor.Factory(MatcherWatchdog.noop());
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        config.put("patterns", Collections.singletonList("%{MY_PATTERN:name}!"));
-        config.put("pattern_definitions", Collections.singletonMap("MY_PATTERN", "["));
+        config.put("patterns", List.of("%{MY_PATTERN:name}!"));
+        config.put("pattern_definitions", Map.of("MY_PATTERN", "["));
         ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
         assertThat(
             e.getMessage(),
@@ -115,7 +115,7 @@ public void testCreateWithInvalidEcsCompatibilityMode() throws Exception {
         GrokProcessor.Factory factory = new GrokProcessor.Factory(MatcherWatchdog.noop());
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        config.put("patterns", Collections.singletonList("(?<foo>\\w+)"));
+        config.put("patterns", List.of("(?<foo>\\w+)"));
         String invalidEcsMode = randomAlphaOfLength(3);
         config.put("ecs_compatibility", invalidEcsMode);
         ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java
index bf7b18814ca4..75f6d7a61735 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java
@@ -26,11 +26,11 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.hamcrest.CoreMatchers.not;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
-import static org.hamcrest.core.IsNull.notNullValue;
-import static org.hamcrest.core.IsNull.nullValue;
 import static org.mockito.Mockito.mock;
 
 public class GrokProcessorGetActionTests extends ESTestCase {
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java
index bc8efeb32b56..5d568b7976fa 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java
@@ -14,9 +14,8 @@
 import org.elasticsearch.ingest.TestIngestDocument;
 import org.elasticsearch.test.ESTestCase;
 
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
@@ -31,8 +30,8 @@ public void testMatch() throws Exception {
         GrokProcessor processor = new GrokProcessor(
             randomAlphaOfLength(10),
             null,
-            Collections.singletonMap("ONE", "1"),
-            Collections.singletonList("%{ONE:one}"),
+            Map.of("ONE", "1"),
+            List.of("%{ONE:one}"),
             fieldName,
             false,
             false,
@@ -49,8 +48,8 @@ public void testIgnoreCase() throws Exception {
         GrokProcessor processor = new GrokProcessor(
             randomAlphaOfLength(10),
             null,
-            Collections.emptyMap(),
-            Collections.singletonList("(?<a>(?i)A)"),
+            Map.of(),
+            List.of("(?<a>(?i)A)"),
             fieldName,
             false,
             false,
@@ -67,8 +66,8 @@ public void testNoMatch() {
         GrokProcessor processor = new GrokProcessor(
             randomAlphaOfLength(10),
             null,
-            Collections.singletonMap("ONE", "1"),
-            Collections.singletonList("%{ONE:one}"),
+            Map.of("ONE", "1"),
+            List.of("%{ONE:one}"),
             fieldName,
             false,
             false,
@@ -87,8 +86,8 @@ public void testNoMatchingPatternName() {
             () -> new GrokProcessor(
                 randomAlphaOfLength(10),
                 null,
-                Collections.singletonMap("ONE", "1"),
-                Collections.singletonList("%{NOTONE:not_one}"),
+                Map.of("ONE", "1"),
+                List.of("%{NOTONE:not_one}"),
                 fieldName,
                 false,
                 false,
@@ -106,8 +105,8 @@ public void testMatchWithoutCaptures() throws Exception {
         GrokProcessor processor = new GrokProcessor(
             randomAlphaOfLength(10),
             null,
-            Collections.emptyMap(),
-            Collections.singletonList(fieldName),
+            Map.of(),
+            List.of(fieldName),
             fieldName,
             false,
             false,
@@ -124,8 +123,8 @@ public void testNullField() {
         GrokProcessor processor = new GrokProcessor(
             randomAlphaOfLength(10),
             null,
-            Collections.singletonMap("ONE", "1"),
-            Collections.singletonList("%{ONE:one}"),
+            Map.of("ONE", "1"),
+            List.of("%{ONE:one}"),
             fieldName,
             false,
             false,
@@ -143,8 +142,8 @@ public void testNullFieldWithIgnoreMissing() throws Exception {
         GrokProcessor processor = new GrokProcessor(
             randomAlphaOfLength(10),
             null,
-            Collections.singletonMap("ONE", "1"),
-            Collections.singletonList("%{ONE:one}"),
+            Map.of("ONE", "1"),
+            List.of("%{ONE:one}"),
             fieldName,
             false,
             true,
@@ -161,8 +160,8 @@ public void testNotStringField() {
         GrokProcessor processor = new GrokProcessor(
             randomAlphaOfLength(10),
             null,
-            Collections.singletonMap("ONE", "1"),
-            Collections.singletonList("%{ONE:one}"),
+            Map.of("ONE", "1"),
+            List.of("%{ONE:one}"),
             fieldName,
             false,
             false,
@@ -179,8 +178,8 @@ public void testNotStringFieldWithIgnoreMissing() {
         GrokProcessor processor = new GrokProcessor(
             randomAlphaOfLength(10),
             null,
-            Collections.singletonMap("ONE", "1"),
-            Collections.singletonList("%{ONE:one}"),
+            Map.of("ONE", "1"),
+            List.of("%{ONE:one}"),
             fieldName,
             false,
             true,
@@ -196,8 +195,8 @@ public void testMissingField() {
         GrokProcessor processor = new GrokProcessor(
             randomAlphaOfLength(10),
             null,
-            Collections.singletonMap("ONE", "1"),
-            Collections.singletonList("%{ONE:one}"),
+            Map.of("ONE", "1"),
+            List.of("%{ONE:one}"),
             fieldName,
             false,
             false,
@@ -214,8 +213,8 @@ public void testMissingFieldWithIgnoreMissing() throws Exception {
         GrokProcessor processor = new GrokProcessor(
             randomAlphaOfLength(10),
             null,
-            Collections.singletonMap("ONE", "1"),
-            Collections.singletonList("%{ONE:one}"),
+            Map.of("ONE", "1"),
+            List.of("%{ONE:one}"),
             fieldName,
             false,
             true,
@@ -237,7 +236,7 @@ public void testMultiplePatternsWithMatchReturn() throws Exception {
             randomAlphaOfLength(10),
             null,
             patternBank,
-            Arrays.asList("%{ONE:one}", "%{TWO:two}", "%{THREE:three}"),
+            List.of("%{ONE:one}", "%{TWO:two}", "%{THREE:three}"),
             fieldName,
             false,
             false,
@@ -261,7 +260,7 @@ public void testSetMetadata() throws Exception {
             randomAlphaOfLength(10),
             null,
             patternBank,
-            Arrays.asList("%{ONE:one}", "%{TWO:two}", "%{THREE:three}"),
+            List.of("%{ONE:one}", "%{TWO:two}", "%{THREE:three}"),
             fieldName,
             true,
             false,
@@ -284,7 +283,7 @@ public void testTraceWithOnePattern() throws Exception {
             randomAlphaOfLength(10),
             null,
             patternBank,
-            Arrays.asList("%{ONE:one}"),
+            List.of("%{ONE:one}"),
             fieldName,
             true,
             false,
@@ -297,17 +296,17 @@ public void testTraceWithOnePattern() throws Exception {
 
     public void testCombinedPatterns() {
         String combined;
-        combined = GrokProcessor.combinePatterns(Arrays.asList(""), false);
+        combined = GrokProcessor.combinePatterns(List.of(""), false);
         assertThat(combined, equalTo(""));
-        combined = GrokProcessor.combinePatterns(Arrays.asList(""), true);
+        combined = GrokProcessor.combinePatterns(List.of(""), true);
         assertThat(combined, equalTo(""));
-        combined = GrokProcessor.combinePatterns(Arrays.asList("foo"), false);
+        combined = GrokProcessor.combinePatterns(List.of("foo"), false);
         assertThat(combined, equalTo("foo"));
-        combined = GrokProcessor.combinePatterns(Arrays.asList("foo"), true);
+        combined = GrokProcessor.combinePatterns(List.of("foo"), true);
         assertThat(combined, equalTo("foo"));
-        combined = GrokProcessor.combinePatterns(Arrays.asList("foo", "bar"), false);
+        combined = GrokProcessor.combinePatterns(List.of("foo", "bar"), false);
         assertThat(combined, equalTo("(?:foo)|(?:bar)"));
-        combined = GrokProcessor.combinePatterns(Arrays.asList("foo", "bar"), true);
+        combined = GrokProcessor.combinePatterns(List.of("foo", "bar"), true);
         assertThat(combined, equalTo("(?<_ingest._grok_match_index.0>foo)|(?<_ingest._grok_match_index.1>bar)"));
     }
 
@@ -323,7 +322,7 @@ public void testCombineSamePatternNameAcrossPatterns() throws Exception {
             randomAlphaOfLength(10),
             null,
             patternBank,
-            Arrays.asList("%{ONE:first}-%{TWO:second}", "%{ONE:first}-%{THREE:second}"),
+            List.of("%{ONE:first}-%{TWO:second}", "%{ONE:first}-%{THREE:second}"),
             fieldName,
             randomBoolean(),
             randomBoolean(),
@@ -334,7 +333,7 @@ public void testCombineSamePatternNameAcrossPatterns() throws Exception {
         assertThat(doc.getFieldValue("second", String.class), equalTo("3"));
     }
 
-    public void testFirstWinNamedCapture() throws Exception {
+    public void testShouldCaptureAllSameNameGroupsAsList() throws Exception {
         String fieldName = RandomDocumentPicks.randomFieldName(random());
         IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         doc.setFieldValue(fieldName, "12");
@@ -344,14 +343,14 @@ public void testFirstWinNamedCapture() throws Exception {
             randomAlphaOfLength(10),
             null,
             patternBank,
-            Collections.singletonList("%{ONETWO:first}%{ONETWO:first}"),
+            List.of("%{ONETWO:first}%{ONETWO:first}"),
             fieldName,
             randomBoolean(),
             randomBoolean(),
             MatcherWatchdog.noop()
         );
         processor.execute(doc);
-        assertThat(doc.getFieldValue("first", String.class), equalTo("1"));
+        assertEquals(doc.getFieldValue("first", List.class), List.of("1", "2"));
     }
 
     public void testUnmatchedNamesNotIncludedInDocument() throws Exception {
@@ -365,7 +364,7 @@ public void testUnmatchedNamesNotIncludedInDocument() throws Exception {
             randomAlphaOfLength(10),
             null,
             patternBank,
-            Collections.singletonList("%{ONETWO:first}|%{THREE:second}"),
+            List.of("%{ONETWO:first}|%{THREE:second}"),
             fieldName,
             randomBoolean(),
             randomBoolean(),
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java
index 0e0d13cb9269..ce5a9e798741 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java
@@ -13,8 +13,8 @@
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 
 public class GsubProcessorFactoryTests extends AbstractStringProcessorFactoryTestCase {
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java
index 25303a42cf24..8f768f9c01bb 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java
@@ -14,7 +14,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.equalTo;
 
 public class JoinProcessorFactoryTests extends ESTestCase {
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java
index c6d7d9deff80..02c936bb4b62 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java
@@ -10,12 +10,14 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 
 public class JsonProcessorFactoryTests extends ESTestCase {
 
@@ -68,6 +70,29 @@ public void testCreateWithMissingField() throws Exception {
         assertThat(exception.getMessage(), equalTo("[field] required property is missing"));
     }
 
+    public void testCreateWithStrictParsingParameter() throws Exception {
+        String fieldName = randomAlphaOfLength(10);
+        String processorTag = randomAlphaOfLength(10);
+        IngestDocument document = new IngestDocument("_index", "_id", 1, null, null, Map.of(fieldName, "123 \"foo\""));
+
+        {
+            Map<String, Object> strictConfig = new HashMap<>();
+            strictConfig.put("field", fieldName);
+            JsonProcessor strictProcessor = FACTORY.create(null, processorTag, null, strictConfig);
+            IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> strictProcessor.execute(document));
+            assertThat(exception.getMessage(), containsString("is not valid JSON and the strict_json_parsing parameter is true"));
+        }
+
+        {
+            Map<String, Object> lenientConfig = new HashMap<>();
+            lenientConfig.put("field", fieldName);
+            lenientConfig.put("strict_json_parsing", false);
+            JsonProcessor lenientProcessor = FACTORY.create(null, processorTag, null, lenientConfig);
+            IngestDocument result = lenientProcessor.execute(document);
+            assertThat(result.getSource().get(fieldName), equalTo(123));
+        }
+    }
+
     public void testCreateWithBothTargetFieldAndAddToRoot() throws Exception {
         String randomField = randomAlphaOfLength(10);
         String randomTargetField = randomAlphaOfLength(5);
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
index a9596fb0083a..068a93d438a1 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
@@ -17,15 +17,17 @@
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xcontent.json.JsonXContent;
 
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.ingest.common.JsonProcessor.ConflictStrategy.MERGE;
 import static org.elasticsearch.ingest.common.JsonProcessor.ConflictStrategy.REPLACE;
+import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 
 public class JsonProcessorTests extends ESTestCase {
 
@@ -128,7 +130,7 @@ public void testString() throws Exception {
     public void testArray() throws Exception {
         JsonProcessor jsonProcessor = new JsonProcessor("tag", null, "field", "target_field", false, REPLACE, false);
         Map<String, Object> document = new HashMap<>();
-        List<Boolean> value = Arrays.asList(true, true, false);
+        List<Boolean> value = List.of(true, true, false);
         document.put("field", value.toString());
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
         jsonProcessor.execute(ingestDocument);
@@ -235,4 +237,86 @@ public void testAddBoolToRoot() {
         Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
         assertThat(exception.getMessage(), containsString("cannot add non-map fields to root of document"));
     }
+
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    public void testApply() {
+        {
+            Object result = JsonProcessor.apply("{\"foo\":\"bar\"}", true, true);
+            assertThat(result, instanceOf(Map.class));
+            Map resultMap = (Map) result;
+            assertThat(resultMap.size(), equalTo(1));
+            assertThat(resultMap.get("foo"), equalTo("bar"));
+        }
+        {
+            Object result = JsonProcessor.apply("\"foo\"", true, true);
+            assertThat(result, instanceOf(String.class));
+            assertThat(result, equalTo("foo"));
+        }
+        {
+            boolean boolValue = randomBoolean();
+            Object result = JsonProcessor.apply(Boolean.toString(boolValue), true, true);
+            assertThat(result, instanceOf(Boolean.class));
+            assertThat(result, equalTo(boolValue));
+        }
+        {
+            double value = randomDouble();
+            Object result = JsonProcessor.apply(Double.toString(value), true, true);
+            assertThat(result, instanceOf(Double.class));
+            assertThat((double) result, closeTo(value, .001));
+        }
+        {
+            List<Double> list = randomList(10, ESTestCase::randomDouble);
+            String value = list.stream().map(val -> Double.toString(val)).collect(Collectors.joining(",", "[", "]"));
+            Object result = JsonProcessor.apply(value, true, true);
+            assertThat(result, instanceOf(List.class));
+            List<Double> resultList = (List<Double>) result;
+            assertThat(resultList.size(), equalTo(list.size()));
+            for (int i = 0; i < list.size(); i++) {
+                assertThat(resultList.get(i), closeTo(list.get(i), .001));
+            }
+        }
+    }
+
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    public void testApplyWithInvalidJson() {
+        /*
+         * The following fail whether strictJsonParsing is set to true or false. The reason is that even the first token cannot be parsed
+         *  as JSON (since the first token is a not a primitive or an object -- just characters not in quotes).
+         */
+        expectThrows(IllegalArgumentException.class, () -> JsonProcessor.apply("foo", true, true));
+        expectThrows(IllegalArgumentException.class, () -> JsonProcessor.apply("foo", true, false));
+        expectThrows(IllegalArgumentException.class, () -> JsonProcessor.apply("foo [360113.865822] wbrdg-0afe001ce", true, true));
+        expectThrows(IllegalArgumentException.class, () -> JsonProcessor.apply("foo [360113.865822] wbrdg-0afe001ce", true, false));
+
+        /*
+         * The following are examples of malformed json but the first part of each is valid json. Previously apply parsed just the first
+         * token and ignored the rest, but it now throw an IllegalArgumentException unless strictJsonParsing is set to false. See
+         * https://github.com/elastic/elasticsearch/issues/92898.
+         */
+        expectThrows(IllegalArgumentException.class, () -> JsonProcessor.apply("123 foo", true, true));
+        expectThrows(IllegalArgumentException.class, () -> JsonProcessor.apply("45 this is {\"a\": \"json\"}", true, true));
+
+        {
+            expectThrows(IllegalArgumentException.class, () -> JsonProcessor.apply("[360113.865822] wbrdg-0afe001ce", true, true));
+            Object result = JsonProcessor.apply("[360113.865822] wbrdg-0afe001ce", true, false);
+            assertThat(result, instanceOf(List.class));
+            List<Double> resultList = (List<Double>) result;
+            assertThat(resultList.size(), equalTo(1));
+            assertThat(resultList.get(0), closeTo(360113.865822, .001));
+        }
+        {
+            expectThrows(IllegalArgumentException.class, () -> JsonProcessor.apply("{\"foo\":\"bar\"} wbrdg-0afe00e", true, true));
+            Object result = JsonProcessor.apply("{\"foo\":\"bar\"} wbrdg-0afe00e", true, false);
+            assertThat(result, instanceOf(Map.class));
+            Map resultMap = (Map) result;
+            assertThat(resultMap.size(), equalTo(1));
+            assertThat(resultMap.get("foo"), equalTo("bar"));
+        }
+        {
+            expectThrows(IllegalArgumentException.class, () -> JsonProcessor.apply(" 1268 : TimeOut = 123 : a", true, true));
+            Object result = JsonProcessor.apply(" 1268 : TimeOut = 123 : a", true, false);
+            assertThat(result, instanceOf(Integer.class));
+            assertThat(result, equalTo(1268));
+        }
+    }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java
index 3495428c9567..5d1669a33b7b 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java
@@ -15,12 +15,12 @@
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
-import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -41,7 +41,7 @@ public void testCreateWithDefaults() throws Exception {
         String processorTag = randomAlphaOfLength(10);
         KeyValueProcessor processor = factory.create(null, processorTag, null, config);
         assertThat(processor.getTag(), equalTo(processorTag));
-        assertThat(processor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("field1"));
+        assertThat(processor.getField().newInstance(Map.of()).execute(), equalTo("field1"));
         assertThat(processor.getFieldSplit(), equalTo("&"));
         assertThat(processor.getValueSplit(), equalTo("="));
         assertThat(processor.getIncludeKeys(), is(nullValue()));
@@ -55,18 +55,18 @@ public void testCreateWithAllFieldsSet() throws Exception {
         config.put("field_split", "&");
         config.put("value_split", "=");
         config.put("target_field", "target");
-        config.put("include_keys", Arrays.asList("a", "b"));
-        config.put("exclude_keys", Collections.emptyList());
+        config.put("include_keys", List.of("a", "b"));
+        config.put("exclude_keys", List.of());
         config.put("ignore_missing", true);
         String processorTag = randomAlphaOfLength(10);
         KeyValueProcessor processor = factory.create(null, processorTag, null, config);
         assertThat(processor.getTag(), equalTo(processorTag));
-        assertThat(processor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("field1"));
+        assertThat(processor.getField().newInstance(Map.of()).execute(), equalTo("field1"));
         assertThat(processor.getFieldSplit(), equalTo("&"));
         assertThat(processor.getValueSplit(), equalTo("="));
         assertThat(processor.getIncludeKeys(), equalTo(Sets.newHashSet("a", "b")));
-        assertThat(processor.getExcludeKeys(), equalTo(Collections.emptySet()));
-        assertThat(processor.getTargetField().newInstance(Collections.emptyMap()).execute(), equalTo("target"));
+        assertThat(processor.getExcludeKeys(), equalTo(Set.of()));
+        assertThat(processor.getTargetField().newInstance(Map.of()).execute(), equalTo("target"));
         assertTrue(processor.isIgnoreMissing());
     }
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java
index 44eed2611723..af9d7d952f33 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -36,24 +35,24 @@ public void test() throws Exception {
         Processor processor = createKvProcessor(fieldName, "&", "=", null, null, "target", false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue("target.first", String.class), equalTo("hello"));
-        assertThat(ingestDocument.getFieldValue("target.second", List.class), equalTo(Arrays.asList("world", "universe")));
+        assertThat(ingestDocument.getFieldValue("target.second", List.class), equalTo(List.of("world", "universe")));
     }
 
     public void testRootTarget() throws Exception {
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of());
         ingestDocument.setFieldValue("myField", "first=hello&second=world&second=universe");
         Processor processor = createKvProcessor("myField", "&", "=", null, null, null, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue("first", String.class), equalTo("hello"));
-        assertThat(ingestDocument.getFieldValue("second", List.class), equalTo(Arrays.asList("world", "universe")));
+        assertThat(ingestDocument.getFieldValue("second", List.class), equalTo(List.of("world", "universe")));
     }
 
     public void testKeySameAsSourceField() throws Exception {
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of());
         ingestDocument.setFieldValue("first", "first=hello");
         Processor processor = createKvProcessor("first", "&", "=", null, null, null, false);
         processor.execute(ingestDocument);
-        assertThat(ingestDocument.getFieldValue("first", List.class), equalTo(Arrays.asList("first=hello", "hello")));
+        assertThat(ingestDocument.getFieldValue("first", List.class), equalTo(List.of("first=hello", "hello")));
     }
 
     public void testIncludeKeys() throws Exception {
@@ -97,7 +96,7 @@ public void testIncludeAndExcludeKeys() throws Exception {
     }
 
     public void testMissingField() throws Exception {
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of());
         Processor processor = createKvProcessor("unknown", "&", "=", null, null, "target", false);
         IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
         assertThat(exception.getMessage(), equalTo("field [unknown] doesn't exist"));
@@ -116,7 +115,7 @@ public void testNullValueWithIgnoreMissing() throws Exception {
     }
 
     public void testNonExistentWithIgnoreMissing() throws Exception {
-        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of());
         IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
         Processor processor = createKvProcessor("unknown", "", "", null, null, "target", true);
         processor.execute(ingestDocument);
@@ -133,7 +132,7 @@ public void testFailFieldSplitMatch() throws Exception {
     }
 
     public void testFailValueSplitMatch() throws Exception {
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("foo", "bar"));
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of("foo", "bar"));
         Processor processor = createKvProcessor("foo", "&", "=", null, null, "target", false);
         Exception exception = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
         assertThat(exception.getMessage(), equalTo("field [foo] does not contain value_split [=]"));
@@ -145,7 +144,7 @@ public void testTrimKeyAndValue() throws Exception {
         Processor processor = createKvProcessor(fieldName, "&", "=", null, null, "target", false, " ", " ", false, null);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue("target.first", String.class), equalTo("hello"));
-        assertThat(ingestDocument.getFieldValue("target.second", List.class), equalTo(Arrays.asList("world", "universe")));
+        assertThat(ingestDocument.getFieldValue("target.second", List.class), equalTo(List.of("world", "universe")));
     }
 
     public void testTrimMultiCharSequence() throws Exception {
@@ -177,7 +176,7 @@ public void testStripBrackets() throws Exception {
         Processor processor = createKvProcessor(fieldName, "&", "=", null, null, "target", false, null, null, true, null);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue("target.first", String.class), equalTo("hello"));
-        assertThat(ingestDocument.getFieldValue("target.second", List.class), equalTo(Arrays.asList("world", "universe")));
+        assertThat(ingestDocument.getFieldValue("target.second", List.class), equalTo(List.of("world", "universe")));
         assertThat(ingestDocument.getFieldValue("target.third", String.class), equalTo("foo"));
         assertThat(ingestDocument.getFieldValue("target.fourth", String.class), equalTo("bar"));
         assertThat(ingestDocument.getFieldValue("target.fifth", String.class), equalTo("last"));
@@ -189,7 +188,7 @@ public void testAddPrefix() throws Exception {
         Processor processor = createKvProcessor(fieldName, "&", "=", null, null, "target", false, null, null, false, "arg_");
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue("target.arg_first", String.class), equalTo("hello"));
-        assertThat(ingestDocument.getFieldValue("target.arg_second", List.class), equalTo(Arrays.asList("world", "universe")));
+        assertThat(ingestDocument.getFieldValue("target.arg_second", List.class), equalTo(List.of("world", "universe")));
     }
 
     private static KeyValueProcessor createKvProcessor(
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java
index deece8362678..59787e106307 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java
@@ -14,7 +14,6 @@
 import org.junit.Before;
 
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -22,7 +21,7 @@
 import static org.elasticsearch.ingest.common.NetworkDirectionProcessor.Factory.DEFAULT_DEST_IP;
 import static org.elasticsearch.ingest.common.NetworkDirectionProcessor.Factory.DEFAULT_SOURCE_IP;
 import static org.elasticsearch.ingest.common.NetworkDirectionProcessor.Factory.DEFAULT_TARGET;
-import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 
 public class NetworkDirectionProcessorFactoryTests extends ESTestCase {
@@ -56,7 +55,7 @@ public void testCreate() throws Exception {
         assertThat(networkProcessor.getDestinationIpField(), equalTo(destIpField));
         assertThat(networkProcessor.getTargetField(), equalTo(targetField));
         assertThat(networkProcessor.getInternalNetworks().size(), greaterThan(0));
-        assertThat(networkProcessor.getInternalNetworks().get(0).newInstance(Collections.emptyMap()).execute(), equalTo("10.0.0.0/8"));
+        assertThat(networkProcessor.getInternalNetworks().get(0).newInstance(Map.of()).execute(), equalTo("10.0.0.0/8"));
         assertThat(networkProcessor.getIgnoreMissing(), equalTo(ignoreMissing));
     }
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java
index 4ae543afc393..7c53df0ca3f4 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -183,7 +182,7 @@ private void testNetworkDirectionProcessor(
     ) throws Exception {
         List<String> networks = null;
 
-        if (internalNetworks != null) networks = Arrays.asList(internalNetworks);
+        if (internalNetworks != null) networks = List.of(internalNetworks);
 
         String processorTag = randomAlphaOfLength(10);
         Map<String, Object> config = new HashMap<>();
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java
index f5100298c63d..5dc20dd9e07b 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java
@@ -15,7 +15,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.equalTo;
 
 public class RegisteredDomainProcessorFactoryTests extends ESTestCase {
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java
index 876f7823839f..49a9dd065184 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java
@@ -14,13 +14,11 @@
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 
-import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.equalTo;
 
 public class RemoveProcessorFactoryTests extends ESTestCase {
 
@@ -37,31 +35,28 @@ public void testCreate() throws Exception {
         String processorTag = randomAlphaOfLength(10);
         RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config);
         assertThat(removeProcessor.getTag(), equalTo(processorTag));
-        assertThat(removeProcessor.getFieldsToRemove().get(0).newInstance(Collections.emptyMap()).execute(), equalTo("field1"));
+        assertThat(removeProcessor.getFieldsToRemove().get(0).newInstance(Map.of()).execute(), equalTo("field1"));
     }
 
     public void testCreateKeepField() throws Exception {
         Map<String, Object> config = new HashMap<>();
-        config.put("keep", Arrays.asList("field1", "field2"));
+        config.put("keep", List.of("field1", "field2"));
         String processorTag = randomAlphaOfLength(10);
         RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config);
         assertThat(removeProcessor.getTag(), equalTo(processorTag));
-        assertThat(removeProcessor.getFieldsToKeep().get(0).newInstance(Collections.emptyMap()).execute(), equalTo("field1"));
-        assertThat(removeProcessor.getFieldsToKeep().get(1).newInstance(Collections.emptyMap()).execute(), equalTo("field2"));
+        assertThat(removeProcessor.getFieldsToKeep().get(0).newInstance(Map.of()).execute(), equalTo("field1"));
+        assertThat(removeProcessor.getFieldsToKeep().get(1).newInstance(Map.of()).execute(), equalTo("field2"));
     }
 
     public void testCreateMultipleFields() throws Exception {
         Map<String, Object> config = new HashMap<>();
-        config.put("field", Arrays.asList("field1", "field2"));
+        config.put("field", List.of("field1", "field2"));
         String processorTag = randomAlphaOfLength(10);
         RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config);
         assertThat(removeProcessor.getTag(), equalTo(processorTag));
         assertThat(
-            removeProcessor.getFieldsToRemove()
-                .stream()
-                .map(template -> template.newInstance(Collections.emptyMap()).execute())
-                .collect(Collectors.toList()),
-            equalTo(Arrays.asList("field1", "field2"))
+            removeProcessor.getFieldsToRemove().stream().map(template -> template.newInstance(Map.of()).execute()).toList(),
+            equalTo(List.of("field1", "field2"))
         );
     }
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java
index f17f019dc36c..5bda0972401e 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -32,8 +31,8 @@ public void testRemoveFields() throws Exception {
         Processor processor = new RemoveProcessor(
             randomAlphaOfLength(10),
             null,
-            Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory(field)),
-            Collections.emptyList(),
+            List.of(new TestTemplateService.MockTemplateScript.Factory(field)),
+            List.of(),
             false
         );
         processor.execute(ingestDocument);
@@ -107,34 +106,16 @@ public void testShouldKeep(String a, String b) {
 
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), source);
 
-        assertTrue(
-            RemoveProcessor.shouldKeep(
-                "name",
-                Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory("name")),
-                ingestDocument
-            )
-        );
+        assertTrue(RemoveProcessor.shouldKeep("name", List.of(new TestTemplateService.MockTemplateScript.Factory("name")), ingestDocument));
 
-        assertTrue(
-            RemoveProcessor.shouldKeep(
-                "age",
-                Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory("age")),
-                ingestDocument
-            )
-        );
+        assertTrue(RemoveProcessor.shouldKeep("age", List.of(new TestTemplateService.MockTemplateScript.Factory("age")), ingestDocument));
 
-        assertFalse(
-            RemoveProcessor.shouldKeep(
-                "name",
-                Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory("age")),
-                ingestDocument
-            )
-        );
+        assertFalse(RemoveProcessor.shouldKeep("name", List.of(new TestTemplateService.MockTemplateScript.Factory("age")), ingestDocument));
 
         assertTrue(
             RemoveProcessor.shouldKeep(
                 "address",
-                Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory("address.street")),
+                List.of(new TestTemplateService.MockTemplateScript.Factory("address.street")),
                 ingestDocument
             )
         );
@@ -142,7 +123,7 @@ public void testShouldKeep(String a, String b) {
         assertTrue(
             RemoveProcessor.shouldKeep(
                 "address",
-                Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory("address.number")),
+                List.of(new TestTemplateService.MockTemplateScript.Factory("address.number")),
                 ingestDocument
             )
         );
@@ -150,7 +131,7 @@ public void testShouldKeep(String a, String b) {
         assertTrue(
             RemoveProcessor.shouldKeep(
                 "address.street",
-                Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory("address")),
+                List.of(new TestTemplateService.MockTemplateScript.Factory("address")),
                 ingestDocument
             )
         );
@@ -158,23 +139,19 @@ public void testShouldKeep(String a, String b) {
         assertTrue(
             RemoveProcessor.shouldKeep(
                 "address.number",
-                Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory("address")),
+                List.of(new TestTemplateService.MockTemplateScript.Factory("address")),
                 ingestDocument
             )
         );
 
         assertTrue(
-            RemoveProcessor.shouldKeep(
-                "address",
-                Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory("address")),
-                ingestDocument
-            )
+            RemoveProcessor.shouldKeep("address", List.of(new TestTemplateService.MockTemplateScript.Factory("address")), ingestDocument)
         );
 
         assertFalse(
             RemoveProcessor.shouldKeep(
                 "address.street",
-                Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory("address.number")),
+                List.of(new TestTemplateService.MockTemplateScript.Factory("address.number")),
                 ingestDocument
             )
         );
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java
index ad1f6f0962de..2299081eb22c 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java
@@ -13,11 +13,10 @@
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.equalTo;
 
 public class RenameProcessorFactoryTests extends ESTestCase {
 
@@ -35,8 +34,8 @@ public void testCreate() throws Exception {
         String processorTag = randomAlphaOfLength(10);
         RenameProcessor renameProcessor = factory.create(null, processorTag, null, config);
         assertThat(renameProcessor.getTag(), equalTo(processorTag));
-        assertThat(renameProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("old_field"));
-        assertThat(renameProcessor.getTargetField().newInstance(Collections.emptyMap()).execute(), equalTo("new_field"));
+        assertThat(renameProcessor.getField().newInstance(Map.of()).execute(), equalTo("old_field"));
+        assertThat(renameProcessor.getTargetField().newInstance(Map.of()).execute(), equalTo("new_field"));
         assertThat(renameProcessor.isIgnoreMissing(), equalTo(false));
     }
 
@@ -48,8 +47,8 @@ public void testCreateWithIgnoreMissing() throws Exception {
         String processorTag = randomAlphaOfLength(10);
         RenameProcessor renameProcessor = factory.create(null, processorTag, null, config);
         assertThat(renameProcessor.getTag(), equalTo(processorTag));
-        assertThat(renameProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("old_field"));
-        assertThat(renameProcessor.getTargetField().newInstance(Collections.emptyMap()).execute(), equalTo("new_field"));
+        assertThat(renameProcessor.getField().newInstance(Map.of()).execute(), equalTo("old_field"));
+        assertThat(renameProcessor.getTargetField().newInstance(Map.of()).execute(), equalTo("new_field"));
         assertThat(renameProcessor.isIgnoreMissing(), equalTo(true));
     }
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java
index 5908fc8784d8..1d10c3090990 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -50,8 +49,8 @@ public void testRenameArrayElement() throws Exception {
         list.add("item3");
         document.put("list", list);
         List<Map<String, String>> one = new ArrayList<>();
-        one.add(Collections.singletonMap("one", "one"));
-        one.add(Collections.singletonMap("two", "two"));
+        one.add(Map.of("one", "one"));
+        one.add(Map.of("two", "two"));
         document.put("one", one);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
 
@@ -139,7 +138,7 @@ public void testRenameExistingFieldNullValue() throws Exception {
 
     public void testRenameAtomicOperationSetFails() throws Exception {
         Map<String, Object> metadata = new HashMap<>();
-        metadata.put("list", Collections.singletonList("item"));
+        metadata.put("list", List.of("item"));
 
         IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator(
             metadata,
@@ -162,7 +161,7 @@ public void testRenameAtomicOperationSetFails() throws Exception {
 
     public void testRenameAtomicOperationRemoveFails() throws Exception {
         Map<String, Object> metadata = new HashMap<>();
-        metadata.put("list", Collections.singletonList("item"));
+        metadata.put("list", List.of("item"));
 
         IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator(
             metadata,
@@ -185,16 +184,13 @@ public void testRenameLeafIntoBranch() throws Exception {
         IngestDocument ingestDocument = TestIngestDocument.withDefaultVersion(source);
         Processor processor1 = createRenameProcessor("foo", "foo.bar", false);
         processor1.execute(ingestDocument);
-        assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Collections.singletonMap("bar", "bar")));
+        assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Map.of("bar", "bar")));
         assertThat(ingestDocument.getFieldValue("foo.bar", String.class), equalTo("bar"));
 
         Processor processor2 = createRenameProcessor("foo.bar", "foo.bar.baz", false);
         processor2.execute(ingestDocument);
-        assertThat(
-            ingestDocument.getFieldValue("foo", Map.class),
-            equalTo(Collections.singletonMap("bar", Collections.singletonMap("baz", "bar")))
-        );
-        assertThat(ingestDocument.getFieldValue("foo.bar", Map.class), equalTo(Collections.singletonMap("baz", "bar")));
+        assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Map.of("bar", Map.of("baz", "bar"))));
+        assertThat(ingestDocument.getFieldValue("foo.bar", Map.class), equalTo(Map.of("baz", "bar")));
         assertThat(ingestDocument.getFieldValue("foo.bar.baz", String.class), equalTo("bar"));
 
         // for fun lets try to restore it (which don't allow today)
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
index 4a8a01218b52..b3e4a870177b 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
@@ -23,14 +23,14 @@
 import org.elasticsearch.xcontent.XContentParseException;
 import org.junit.Before;
 
-import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
-import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -55,7 +55,7 @@ public void testFactoryValidationWithDefaultLang() throws Exception {
         ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), null, configMap);
         assertThat(processor.getScript().getLang(), equalTo(randomType.equals("id") ? null : Script.DEFAULT_SCRIPT_LANG));
         assertThat(processor.getScript().getType().toString(), equalTo(INGEST_SCRIPT_PARAM_TO_TYPE.get(randomType)));
-        assertThat(processor.getScript().getParams(), equalTo(Collections.emptyMap()));
+        assertThat(processor.getScript().getParams(), equalTo(Map.of()));
     }
 
     public void testFactoryValidationWithParams() throws Exception {
@@ -65,7 +65,7 @@ public void testFactoryValidationWithParams() throws Exception {
 
         Map<String, Object> configMap = new HashMap<>();
         String randomType = randomFrom("id", "source");
-        Map<String, Object> randomParams = Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10));
+        Map<String, Object> randomParams = Map.of(randomAlphaOfLength(10), randomAlphaOfLength(10));
         configMap.put(randomType, "foo");
         configMap.put("params", randomParams);
         ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), null, configMap);
@@ -117,7 +117,7 @@ public void testFactoryInvalidateWithInvalidCompiledScript() throws Exception {
         ScriptException thrownException = new ScriptException(
             "compile-time exception",
             new RuntimeException(),
-            Collections.emptyList(),
+            List.of(),
             "script",
             "mockscript"
         );
@@ -139,13 +139,10 @@ public void testInlineIsCompiled() throws Exception {
         String scriptName = "foo";
         ScriptService scriptService = new ScriptService(
             Settings.builder().build(),
-            Collections.singletonMap(
-                Script.DEFAULT_SCRIPT_LANG,
-                new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> {
-                    ctx.put("foo", "bar");
-                    return null;
-                }), Collections.emptyMap())
-            ),
+            Map.of(Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Map.of(scriptName, ctx -> {
+                ctx.put("foo", "bar");
+                return null;
+            }), Map.of())),
             new HashMap<>(ScriptModule.CORE_CONTEXTS),
             () -> 1L
         );
@@ -156,7 +153,7 @@ public void testInlineIsCompiled() throws Exception {
         ScriptProcessor processor = factory.create(null, null, randomAlphaOfLength(10), configMap);
         assertThat(processor.getScript().getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG));
         assertThat(processor.getScript().getType(), equalTo(ScriptType.INLINE));
-        assertThat(processor.getScript().getParams(), equalTo(Collections.emptyMap()));
+        assertThat(processor.getScript().getParams(), equalTo(Map.of()));
         assertNotNull(processor.getPrecompiledIngestScriptFactory());
         CtxMap<?> ctx = TestIngestDocument.emptyIngestDocument().getCtxMap();
         processor.getPrecompiledIngestScriptFactory().newInstance(null, ctx).execute();
@@ -172,7 +169,7 @@ public void testStoredIsNotCompiled() throws Exception {
         ScriptProcessor processor = factory.create(null, null, randomAlphaOfLength(10), configMap);
         assertNull(processor.getScript().getLang());
         assertThat(processor.getScript().getType(), equalTo(ScriptType.STORED));
-        assertThat(processor.getScript().getParams(), equalTo(Collections.emptyMap()));
+        assertThat(processor.getScript().getParams(), equalTo(Map.of()));
         assertNull(processor.getPrecompiledIngestScriptFactory());
     }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java
index 972ca029b7b0..91dd0c861c02 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java
@@ -20,13 +20,12 @@
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasKey;
-import static org.hamcrest.core.Is.is;
+import static org.hamcrest.Matchers.is;
 
 public class ScriptProcessorTests extends ESTestCase {
 
@@ -39,20 +38,17 @@ public void setupScripting() {
         String scriptName = "script";
         scriptService = new ScriptService(
             Settings.builder().build(),
-            Collections.singletonMap(
-                Script.DEFAULT_SCRIPT_LANG,
-                new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> {
-                    Integer bytesIn = (Integer) ctx.get("bytes_in");
-                    Integer bytesOut = (Integer) ctx.get("bytes_out");
-                    ctx.put("bytes_total", bytesIn + bytesOut);
-                    ctx.put("_dynamic_templates", Map.of("foo", "bar"));
-                    return null;
-                }), Collections.emptyMap())
-            ),
+            Map.of(Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Map.of(scriptName, ctx -> {
+                Integer bytesIn = (Integer) ctx.get("bytes_in");
+                Integer bytesOut = (Integer) ctx.get("bytes_out");
+                ctx.put("bytes_total", bytesIn + bytesOut);
+                ctx.put("_dynamic_templates", Map.of("foo", "bar"));
+                return null;
+            }), Map.of())),
             new HashMap<>(ScriptModule.CORE_CONTEXTS),
             () -> 1L
         );
-        script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap());
+        script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Map.of());
         ingestScriptFactory = scriptService.compile(script, IngestScript.CONTEXT);
     }
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java
index ac1af5f3dcdd..085cc68fe363 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java
@@ -15,13 +15,12 @@
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
-import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 
 public class SetProcessorFactoryTests extends ESTestCase {
 
@@ -39,8 +38,8 @@ public void testCreate() throws Exception {
         String processorTag = randomAlphaOfLength(10);
         SetProcessor setProcessor = factory.create(null, processorTag, null, config);
         assertThat(setProcessor.getTag(), equalTo(processorTag));
-        assertThat(setProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("field1"));
-        assertThat(setProcessor.getValue().copyAndResolve(Collections.emptyMap()), equalTo("value1"));
+        assertThat(setProcessor.getField().newInstance(Map.of()).execute(), equalTo("field1"));
+        assertThat(setProcessor.getValue().copyAndResolve(Map.of()), equalTo("value1"));
         assertThat(setProcessor.isOverrideEnabled(), equalTo(true));
     }
 
@@ -53,8 +52,8 @@ public void testCreateWithOverride() throws Exception {
         String processorTag = randomAlphaOfLength(10);
         SetProcessor setProcessor = factory.create(null, processorTag, null, config);
         assertThat(setProcessor.getTag(), equalTo(processorTag));
-        assertThat(setProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("field1"));
-        assertThat(setProcessor.getValue().copyAndResolve(Collections.emptyMap()), equalTo("value1"));
+        assertThat(setProcessor.getField().newInstance(Map.of()).execute(), equalTo("field1"));
+        assertThat(setProcessor.getValue().copyAndResolve(Map.of()), equalTo("value1"));
         assertThat(setProcessor.isOverrideEnabled(), equalTo(overrideEnabled));
     }
 
@@ -113,7 +112,7 @@ public void testCreateWithCopyFrom() throws Exception {
         String processorTag = randomAlphaOfLength(10);
         SetProcessor setProcessor = factory.create(null, processorTag, null, config);
         assertThat(setProcessor.getTag(), equalTo(processorTag));
-        assertThat(setProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("field1"));
+        assertThat(setProcessor.getField().newInstance(Map.of()).execute(), equalTo("field1"));
         assertThat(setProcessor.getCopyFrom(), equalTo("field2"));
     }
 
@@ -143,7 +142,7 @@ public void testMediaType() throws Exception {
 
         // invalid media type
         expectedMediaType = randomValueOtherThanMany(
-            m -> Arrays.asList(ConfigurationUtils.VALID_MEDIA_TYPES).contains(m),
+            m -> List.of(ConfigurationUtils.VALID_MEDIA_TYPES).contains(m),
             () -> randomAlphaOfLengthBetween(5, 9)
         );
         final Map<String, Object> config2 = new HashMap<>();
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java
index e6477b8940a8..5973e4fe5741 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.ingest.ValueSource;
 import org.elasticsearch.test.ESTestCase;
-import org.hamcrest.Matchers;
 
 import java.util.ArrayList;
 import java.util.Date;
@@ -109,7 +108,7 @@ public void testSetMetadataExceptVersion() throws Exception {
         Processor processor = createSetProcessor(randomMetadata.getFieldName(), "_value", null, true, false);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         processor.execute(ingestDocument);
-        assertThat(ingestDocument.getFieldValue(randomMetadata.getFieldName(), String.class), Matchers.equalTo("_value"));
+        assertThat(ingestDocument.getFieldValue(randomMetadata.getFieldName(), String.class), equalTo("_value"));
     }
 
     public void testSetMetadataVersion() throws Exception {
@@ -117,7 +116,7 @@ public void testSetMetadataVersion() throws Exception {
         Processor processor = createSetProcessor(Metadata.VERSION.getFieldName(), version, null, true, false);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         processor.execute(ingestDocument);
-        assertThat(ingestDocument.getFieldValue(Metadata.VERSION.getFieldName(), Long.class), Matchers.equalTo(version));
+        assertThat(ingestDocument.getFieldValue(Metadata.VERSION.getFieldName(), Long.class), equalTo(version));
     }
 
     public void testSetMetadataVersionType() throws Exception {
@@ -125,7 +124,7 @@ public void testSetMetadataVersionType() throws Exception {
         Processor processor = createSetProcessor(Metadata.VERSION_TYPE.getFieldName(), versionType, null, true, false);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         processor.execute(ingestDocument);
-        assertThat(ingestDocument.getFieldValue(Metadata.VERSION_TYPE.getFieldName(), String.class), Matchers.equalTo(versionType));
+        assertThat(ingestDocument.getFieldValue(Metadata.VERSION_TYPE.getFieldName(), String.class), equalTo(versionType));
     }
 
     public void testSetMetadataIfSeqNo() throws Exception {
@@ -133,7 +132,7 @@ public void testSetMetadataIfSeqNo() throws Exception {
         Processor processor = createSetProcessor(Metadata.IF_SEQ_NO.getFieldName(), ifSeqNo, null, true, false);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         processor.execute(ingestDocument);
-        assertThat(ingestDocument.getFieldValue(Metadata.IF_SEQ_NO.getFieldName(), Long.class), Matchers.equalTo(ifSeqNo));
+        assertThat(ingestDocument.getFieldValue(Metadata.IF_SEQ_NO.getFieldName(), Long.class), equalTo(ifSeqNo));
     }
 
     public void testSetMetadataIfPrimaryTerm() throws Exception {
@@ -141,7 +140,7 @@ public void testSetMetadataIfPrimaryTerm() throws Exception {
         Processor processor = createSetProcessor(Metadata.IF_PRIMARY_TERM.getFieldName(), ifPrimaryTerm, null, true, false);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         processor.execute(ingestDocument);
-        assertThat(ingestDocument.getFieldValue(Metadata.IF_PRIMARY_TERM.getFieldName(), Long.class), Matchers.equalTo(ifPrimaryTerm));
+        assertThat(ingestDocument.getFieldValue(Metadata.IF_PRIMARY_TERM.getFieldName(), Long.class), equalTo(ifPrimaryTerm));
     }
 
     public void testSetDynamicTemplates() throws Exception {
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorFactoryTests.java
index 51abbf305837..0aeae9e6eed4 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorFactoryTests.java
@@ -15,7 +15,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.equalTo;
 
 public class SortProcessorFactoryTests extends ESTestCase {
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorTests.java
index 7d5a64216654..f638f8c12315 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorTests.java
@@ -15,10 +15,10 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -53,7 +53,7 @@ public void testSortIntegersNonRandom() throws Exception {
 
         Integer[] expectedResult = new Integer[] { 1, 2, 3, 4, 5, 10, 20, 21, 22, 50, 100 };
         List<Integer> fieldValue = new ArrayList<>(expectedResult.length);
-        fieldValue.addAll(Arrays.asList(expectedResult).subList(0, expectedResult.length));
+        fieldValue.addAll(List.of(expectedResult).subList(0, expectedResult.length));
         Collections.shuffle(fieldValue, random());
 
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
@@ -265,7 +265,7 @@ public void testSortNullValue() throws Exception {
     }
 
     public void testDescendingSortWithTargetField() throws Exception {
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of());
         int numItems = randomIntBetween(1, 10);
         List<String> fieldValue = new ArrayList<>(numItems);
         List<String> expectedResult = new ArrayList<>(numItems);
@@ -285,7 +285,7 @@ public void testDescendingSortWithTargetField() throws Exception {
     }
 
     public void testAscendingSortWithTargetField() throws Exception {
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of());
         int numItems = randomIntBetween(1, 10);
         List<String> fieldValue = new ArrayList<>(numItems);
         List<String> expectedResult = new ArrayList<>(numItems);
@@ -305,8 +305,8 @@ public void testAscendingSortWithTargetField() throws Exception {
     }
 
     public void testSortWithTargetFieldLeavesOriginalUntouched() throws Exception {
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
-        List<Integer> fieldValue = Arrays.asList(1, 5, 4);
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of());
+        List<Integer> fieldValue = List.of(1, 5, 4);
         List<Integer> expectedResult = new ArrayList<>(fieldValue);
         Collections.sort(expectedResult);
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java
index 891f67bd45a6..10d96b8add96 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java
@@ -14,7 +14,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.equalTo;
 
 public class SplitProcessorFactoryTests extends ESTestCase {
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java
index 73c94efdb985..500debd79b96 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.ingest.TestIngestDocument;
 import org.elasticsearch.test.ESTestCase;
 
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -31,7 +30,7 @@ public void testSplit() throws Exception {
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "127.0.0.1");
         Processor processor = new SplitProcessor(randomAlphaOfLength(10), null, fieldName, "\\.", false, false, fieldName);
         processor.execute(ingestDocument);
-        assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(Arrays.asList("127", "0", "0", "1")));
+        assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(List.of("127", "0", "0", "1")));
     }
 
     public void testSplitFieldNotFound() throws Exception {
@@ -70,7 +69,7 @@ public void testSplitNullValueWithIgnoreMissing() throws Exception {
     }
 
     public void testSplitNonExistentWithIgnoreMissing() throws Exception {
-        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of());
         IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
         Processor processor = new SplitProcessor(randomAlphaOfLength(10), null, "field", "\\.", true, false, "field");
         processor.execute(ingestDocument);
@@ -104,11 +103,11 @@ public void testSplitAppendable() throws Exception {
         splitProcessor.execute(ingestDocument);
         @SuppressWarnings("unchecked")
         List<String> flags = (List<String>) ingestDocument.getFieldValue("flags", List.class);
-        assertThat(flags, equalTo(Arrays.asList("new", "hot", "super", "fun", "interesting")));
+        assertThat(flags, equalTo(List.of("new", "hot", "super", "fun", "interesting")));
         ingestDocument.appendFieldValue("flags", "additional_flag");
         assertThat(
             ingestDocument.getFieldValue("flags", List.class),
-            equalTo(Arrays.asList("new", "hot", "super", "fun", "interesting", "additional_flag"))
+            equalTo(List.of("new", "hot", "super", "fun", "interesting", "additional_flag"))
         );
     }
 
@@ -118,15 +117,15 @@ public void testSplitWithTargetField() throws Exception {
         String targetFieldName = fieldName + randomAlphaOfLength(5);
         Processor processor = new SplitProcessor(randomAlphaOfLength(10), null, fieldName, "\\.", false, false, targetFieldName);
         processor.execute(ingestDocument);
-        assertThat(ingestDocument.getFieldValue(targetFieldName, List.class), equalTo(Arrays.asList("127", "0", "0", "1")));
+        assertThat(ingestDocument.getFieldValue(targetFieldName, List.class), equalTo(List.of("127", "0", "0", "1")));
     }
 
     public void testSplitWithPreserveTrailing() throws Exception {
-        doTestSplitWithPreserveTrailing(true, "foo|bar|baz||", Arrays.asList("foo", "bar", "baz", "", ""));
+        doTestSplitWithPreserveTrailing(true, "foo|bar|baz||", List.of("foo", "bar", "baz", "", ""));
     }
 
     public void testSplitWithoutPreserveTrailing() throws Exception {
-        doTestSplitWithPreserveTrailing(false, "foo|bar|baz||", Arrays.asList("foo", "bar", "baz"));
+        doTestSplitWithPreserveTrailing(false, "foo|bar|baz||", List.of("foo", "bar", "baz"));
     }
 
     private void doTestSplitWithPreserveTrailing(boolean preserveTrailing, String fieldValue, List<String> expected) throws Exception {
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorFactoryTests.java
index 08e99326330a..a6ba0a32b7bb 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorFactoryTests.java
@@ -15,7 +15,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.equalTo;
 
 public class UriPartsProcessorFactoryTests extends ESTestCase {
 
diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/140_json.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/140_json.yml
index 60208a52aba3..9ef22ff2ebef 100644
--- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/140_json.yml
+++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/140_json.yml
@@ -12,6 +12,10 @@ teardown:
       ingest.delete_pipeline:
         id: "3"
         ignore: 404
+  - do:
+      ingest.delete_pipeline:
+        id: "4"
+        ignore: 404
 
 ---
 "Test JSON Processor":
@@ -150,3 +154,35 @@ teardown:
         id: "3"
   - match: { _source.foo.bar: "baz" }
   - match: { _source.foo.qux: "quux" }
+
+---
+"Test JSON Processor lenient parsing":
+  - do:
+      ingest.put_pipeline:
+        id: "4"
+        body: {
+          "processors": [
+            {
+              "json" : {
+                "field" : "message",
+                "strict_json_parsing": false
+              }
+            }
+          ]
+        }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        id: "4"
+        pipeline: "4"
+        body: {
+          message: "123 foo"
+        }
+
+  - do:
+      get:
+        index: test
+        id: "4"
+  - match: { _source.message: 123 }
diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml
index 614b329c699f..d833e0111f83 100644
--- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml
+++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml
@@ -313,7 +313,7 @@ teardown:
               "foo": "bar"
             }
           }
-  - match: { error.root_cause.0.reason: "Failed to generate the source document for ingest pipeline [my_pipeline]" }
+  - match: { error.root_cause.0.reason: "Failed to generate the source document for ingest pipeline [my_pipeline] for document [test/1]" }
 
 ---
 "Test metadata":
diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/240_required_pipeline.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/240_final_pipeline.yml
similarity index 100%
rename from modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/240_required_pipeline.yml
rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/240_final_pipeline.yml
diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/270_set_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/270_set_processor.yml
index 61fc876d8180..594ff52c2b27 100644
--- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/270_set_processor.yml
+++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/270_set_processor.yml
@@ -177,3 +177,53 @@ teardown:
   - match: { _source.copied_foo_number: 3 }
   - is_true:  _source.copied_foo_boolean
   - match: { _source.foo_nochange: "no change" }
+
+---
+"Test set processor with reflection attempts":
+  - do:
+      ingest.put_pipeline:
+        id: "1"
+        body:  >
+          {
+            "processors" : [
+              {
+                "script": {
+                  "description": "Set a reference to a proper java object so we can attempt reflection",
+                  "lang": "painless",
+                  "source": "ctx.t = metadata().now"
+                }
+              },
+              {
+                "set": {
+                  "description": "Attempting to call a method (ZonedDateTime#getHour()) is ignored",
+                  "field": "method_call_is_ignored",
+                  "value": "{{t.hour}}"
+                }
+              },
+              {
+                "set": {
+                  "description": "Attempting to call a method that doesn't exist is ignored",
+                  "field": "missing_method_is_ignored",
+                  "value": "{{t.nothing}}"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        pipeline: "1"
+        body: {
+          foo: "hello"
+        }
+
+  - do:
+      get:
+        index: test
+        id: "1"
+  - match: { _source.foo: "hello" }
+  - match: { _source.method_call_is_ignored: "" }
+  - match: { _source.missing_method_is_ignored: "" }
diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/70_bulk.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/70_bulk.yml
index d7738987de18..c71e13429f72 100644
--- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/70_bulk.yml
+++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/70_bulk.yml
@@ -145,6 +145,45 @@ teardown:
   - is_false: _source.field1
   - match: {_source.field2: value2}
 
+---
+"Test bulk request with _none request pipeline and default pipeline":
+
+  - do:
+      bulk:
+        pipeline: pipeline1
+        body:
+          - index:
+              _index: test_index
+              _id:    test_id1
+          - f1: v1
+          - index:
+              _index: test_index
+              _id:    test_id2
+              pipeline: _none
+          - f1: v2
+  - gte: { ingest_took: 0 }
+
+  - do:
+      cluster.state: {}
+  # Get master node id
+  - set: { master_node: master }
+
+  - do:
+      get:
+        index: test_index
+        id: test_id1
+
+  - match: {_source.field1: value1}
+  - is_false: _source.field2
+
+  - do:
+      get:
+        index: test_index
+        id: test_id2
+
+  - is_false: _source.field1
+  - is_false: _source.field2
+
 ---
 "Update with pipeline":
   - skip:
diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle
index 86f59753e4f2..62a78e4ef033 100644
--- a/modules/ingest-geoip/build.gradle
+++ b/modules/ingest-geoip/build.gradle
@@ -8,8 +8,8 @@
 
 import org.apache.tools.ant.taskdefs.condition.Os
 
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
@@ -27,12 +27,12 @@ tasks.named('internalClusterTestTestingConventions').configure {
 }
 
 dependencies {
-  implementation('com.maxmind.geoip2:geoip2:3.0.0')
+  implementation('com.maxmind.geoip2:geoip2:4.0.0')
   // geoip2 dependencies:
   runtimeOnly("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
   runtimeOnly("com.fasterxml.jackson.core:jackson-databind:${versions.jackson}")
   runtimeOnly("com.fasterxml.jackson.core:jackson-core:${versions.jackson}")
-  implementation('com.maxmind.db:maxmind-db:2.0.0')
+  implementation('com.maxmind.db:maxmind-db:3.0.0')
 
   testImplementation 'org.elasticsearch:geolite2-databases:20191119'
   internalClusterTestImplementation project(path: ":modules:reindex")
diff --git a/modules/ingest-geoip/qa/file-based-update/build.gradle b/modules/ingest-geoip/qa/file-based-update/build.gradle
index f465172a55b7..0e396d62eb16 100644
--- a/modules/ingest-geoip/qa/file-based-update/build.gradle
+++ b/modules/ingest-geoip/qa/file-based-update/build.gradle
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 testClusters.configureEach {
   testDistribution = 'DEFAULT'
diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java
index 0e164cab818b..63c14ac4df96 100644
--- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java
+++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java
@@ -27,10 +27,13 @@
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.MatchQueryBuilder;
 import org.elasticsearch.index.query.RangeQueryBuilder;
+import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.geoip.stats.GeoIpDownloaderStatsAction;
 import org.elasticsearch.persistent.PersistentTaskParams;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
+import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.reindex.ReindexPlugin;
 import org.elasticsearch.search.SearchHit;
@@ -51,11 +54,13 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
@@ -82,7 +87,12 @@ public class GeoIpDownloaderIT extends AbstractGeoIpIT {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(ReindexPlugin.class, IngestGeoIpPlugin.class, GeoIpProcessorNonIngestNodeIT.IngestGeoIpSettingsPlugin.class);
+        return Arrays.asList(
+            ReindexPlugin.class,
+            IngestGeoIpPlugin.class,
+            GeoIpProcessorNonIngestNodeIT.IngestGeoIpSettingsPlugin.class,
+            NonGeoProcessorsPlugin.class
+        );
     }
 
     @Override
@@ -104,7 +114,7 @@ public void cleanUp() throws Exception {
             .setPersistentSettings(
                 Settings.builder()
                     .putNull(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey())
-                    .putNull(GeoIpDownloader.POLL_INTERVAL_SETTING.getKey())
+                    .putNull(GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getKey())
                     .putNull("ingest.geoip.database_validity")
             )
             .get();
@@ -149,6 +159,7 @@ public void cleanUp() throws Exception {
     @TestLogging(value = "org.elasticsearch.ingest.geoip:TRACE", reason = "https://github.com/elastic/elasticsearch/issues/75221")
     public void testInvalidTimestamp() throws Exception {
         assumeTrue("only test with fixture to have stable results", ENDPOINT != null);
+        putGeoIpPipeline();
         ClusterUpdateSettingsResponse settingsResponse = client().admin()
             .cluster()
             .prepareUpdateSettings()
@@ -160,7 +171,7 @@ public void testInvalidTimestamp() throws Exception {
             assertEquals(Set.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb"), state.getDatabases().keySet());
         }, 2, TimeUnit.MINUTES);
 
-        putPipeline();
+        putGeoIpPipeline();
         verifyUpdatedDatabase();
 
         settingsResponse = client().admin()
@@ -172,7 +183,9 @@ public void testInvalidTimestamp() throws Exception {
         settingsResponse = client().admin()
             .cluster()
             .prepareUpdateSettings()
-            .setPersistentSettings(Settings.builder().put(GeoIpDownloader.POLL_INTERVAL_SETTING.getKey(), TimeValue.timeValueDays(2)))
+            .setPersistentSettings(
+                Settings.builder().put(GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getKey(), TimeValue.timeValueDays(2))
+            )
             .get();
         assertTrue(settingsResponse.isAcknowledged());
         List<Path> geoIpTmpDirs = getGeoIpTmpDirs();
@@ -186,7 +199,7 @@ public void testInvalidTimestamp() throws Exception {
                 }
             }
         });
-        putPipeline();
+        putGeoIpPipeline();
         assertBusy(() -> {
             SimulateDocumentBaseResult result = simulatePipeline();
             assertThat(result.getFailure(), nullValue());
@@ -221,7 +234,9 @@ public void testUpdatedTimestamp() throws Exception {
         ClusterUpdateSettingsResponse settingsResponse = client().admin()
             .cluster()
             .prepareUpdateSettings()
-            .setPersistentSettings(Settings.builder().put(GeoIpDownloader.POLL_INTERVAL_SETTING.getKey(), TimeValue.timeValueDays(2)))
+            .setPersistentSettings(
+                Settings.builder().put(GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getKey(), TimeValue.timeValueDays(2))
+            )
             .get();
         assertTrue(settingsResponse.isAcknowledged());
         assertBusy(() -> assertNotEquals(lastCheck, getGeoIpTaskState().getDatabases().get("GeoLite2-ASN.mmdb").lastCheck()));
@@ -229,6 +244,7 @@ public void testUpdatedTimestamp() throws Exception {
     }
 
     public void testGeoIpDatabasesDownload() throws Exception {
+        putGeoIpPipeline();
         ClusterUpdateSettingsResponse settingsResponse = client().admin()
             .cluster()
             .prepareUpdateSettings()
@@ -238,6 +254,7 @@ public void testGeoIpDatabasesDownload() throws Exception {
         assertBusy(() -> {
             GeoIpTaskState state = getGeoIpTaskState();
             assertEquals(Set.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb"), state.getDatabases().keySet());
+            putGeoIpPipeline(); // This is to work around the race condition described in #92888
         }, 2, TimeUnit.MINUTES);
 
         for (String id : List.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb")) {
@@ -283,12 +300,39 @@ public void testGeoIpDatabasesDownload() throws Exception {
         }
     }
 
+    public void testGeoIpDatabasesDownloadNoGeoipProcessors() throws Exception {
+        assumeTrue("only test with fixture to have stable results", ENDPOINT != null);
+        String pipelineId = randomAlphaOfLength(10);
+        putGeoIpPipeline(pipelineId);
+        ClusterUpdateSettingsResponse settingsResponse = client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setPersistentSettings(Settings.builder().put(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey(), true))
+            .get();
+        assertTrue(settingsResponse.isAcknowledged());
+        assertBusy(() -> {
+            PersistentTasksCustomMetadata.PersistentTask<PersistentTaskParams> task = getTask();
+            assertNotNull(task);
+            assertNull(task.getState());
+            putGeoIpPipeline(); // This is to work around the race condition described in #92888
+        });
+        putNonGeoipPipeline(pipelineId);
+        assertBusy(() -> { assertNull(getTask().getState()); });
+        putNonGeoipPipeline(pipelineId);
+        assertNull(getTask().getState());
+        putGeoIpPipeline();
+        assertBusy(() -> {
+            GeoIpTaskState state = getGeoIpTaskState();
+            assertEquals(Set.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb"), state.getDatabases().keySet());
+        }, 2, TimeUnit.MINUTES);
+    }
+
     @TestLogging(value = "org.elasticsearch.ingest.geoip:TRACE", reason = "https://github.com/elastic/elasticsearch/issues/69972")
     public void testUseGeoIpProcessorWithDownloadedDBs() throws Exception {
         assumeTrue("only test with fixture to have stable results", ENDPOINT != null);
         setupDatabasesInConfigDirectory();
         // setup:
-        putPipeline();
+        putGeoIpPipeline();
 
         // verify before updating dbs
         {
@@ -355,7 +399,7 @@ public void testUseGeoIpProcessorWithDownloadedDBs() throws Exception {
     @TestLogging(value = "org.elasticsearch.ingest.geoip:TRACE", reason = "https://github.com/elastic/elasticsearch/issues/79074")
     public void testStartWithNoDatabases() throws Exception {
         assumeTrue("only test with fixture to have stable results", ENDPOINT != null);
-        putPipeline();
+        putGeoIpPipeline();
 
         // Behaviour without any databases loaded:
         {
@@ -438,7 +482,21 @@ private SimulateDocumentBaseResult simulatePipeline() throws IOException {
         return (SimulateDocumentBaseResult) simulateResponse.getResults().get(0);
     }
 
-    private void putPipeline() throws IOException {
+    /**
+     * This creates a pipeline with a geoip processor, which ought to cause the geoip downloader to begin (assuming it is enabled).
+     * @throws IOException
+     */
+    private void putGeoIpPipeline() throws IOException {
+        putGeoIpPipeline("_id");
+    }
+
+    /**
+     * This creates a pipeline named pipelineId with a geoip processor, which ought to cause the geoip downloader to begin (assuming it is
+     * enabled).
+     * @param pipelineId The name of the new pipeline with a geoip processor
+     * @throws IOException
+     */
+    private void putGeoIpPipeline(String pipelineId) throws IOException {
         BytesReference bytes;
         try (XContentBuilder builder = JsonXContent.contentBuilder()) {
             builder.startObject();
@@ -484,7 +542,45 @@ private void putPipeline() throws IOException {
             builder.endObject();
             bytes = BytesReference.bytes(builder);
         }
-        assertAcked(client().admin().cluster().preparePutPipeline("_id", bytes, XContentType.JSON).get());
+        assertAcked(client().admin().cluster().preparePutPipeline(pipelineId, bytes, XContentType.JSON).get());
+    }
+
+    /**
+     * This creates a pipeline named pipelineId that does _not_ have a geoip processor.
+     * @throws IOException
+     */
+    private void putNonGeoipPipeline(String pipelineId) throws IOException {
+        BytesReference bytes;
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            builder.startObject();
+            {
+                builder.startArray("processors");
+                {
+                    builder.startObject();
+                    {
+                        builder.startObject(NonGeoProcessorsPlugin.NON_GEO_PROCESSOR_TYPE);
+                        builder.endObject();
+                    }
+                    builder.endObject();
+                    builder.startObject();
+                    {
+                        builder.startObject(NonGeoProcessorsPlugin.NON_GEO_PROCESSOR_TYPE);
+                        builder.endObject();
+                    }
+                    builder.endObject();
+                    builder.startObject();
+                    {
+                        builder.startObject(NonGeoProcessorsPlugin.NON_GEO_PROCESSOR_TYPE);
+                        builder.endObject();
+                    }
+                    builder.endObject();
+                }
+                builder.endArray();
+            }
+            builder.endObject();
+            bytes = BytesReference.bytes(builder);
+        }
+        assertAcked(client().admin().cluster().preparePutPipeline(pipelineId, bytes, XContentType.JSON).get());
     }
 
     private List<Path> getGeoIpTmpDirs() throws IOException {
@@ -624,4 +720,32 @@ public int read(byte[] b, int off, int len) throws IOException {
             return read;
         }
     }
+
+    /**
+     * This class defines a processor of type "test".
+     */
+    public static final class NonGeoProcessorsPlugin extends Plugin implements IngestPlugin {
+        public static final String NON_GEO_PROCESSOR_TYPE = "test";
+
+        @Override
+        public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
+            Map<String, Processor.Factory> procMap = new HashMap<>();
+            procMap.put(NON_GEO_PROCESSOR_TYPE, (factories, tag, description, config) -> new AbstractProcessor(tag, description) {
+                @Override
+                public void execute(IngestDocument ingestDocument, BiConsumer<IngestDocument, Exception> handler) {}
+
+                @Override
+                public String getType() {
+                    return NON_GEO_PROCESSOR_TYPE;
+                }
+
+                @Override
+                public boolean isAsync() {
+                    return false;
+                }
+
+            });
+            return procMap;
+        }
+    }
 }
diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderStatsIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderStatsIT.java
index 288547b6a72d..eea763351dd0 100644
--- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderStatsIT.java
+++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderStatsIT.java
@@ -20,6 +20,8 @@
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xcontent.json.JsonXContent;
 import org.junit.After;
 
 import java.io.IOException;
@@ -29,6 +31,7 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
@@ -66,6 +69,11 @@ public void disableDownloader() {
     }
 
     public void testStats() throws Exception {
+        /*
+         * Testing without the geoip endpoint fixture falls back to https://storage.googleapis.com/, which can cause this test to run too
+         * slowly to pass.
+         */
+        assumeTrue("only test with fixture to have stable results", ENDPOINT != null);
         GeoIpDownloaderStatsAction.Request req = new GeoIpDownloaderStatsAction.Request();
         GeoIpDownloaderStatsAction.Response response = client().execute(GeoIpDownloaderStatsAction.INSTANCE, req).actionGet();
         XContentTestUtils.JsonMapView jsonMapView = new XContentTestUtils.JsonMapView(convertToMap(response));
@@ -75,7 +83,7 @@ public void testStats() throws Exception {
         assertThat(jsonMapView.get("stats.databases_count"), equalTo(0));
         assertThat(jsonMapView.get("stats.total_download_time"), equalTo(0));
         assertEquals(0, jsonMapView.<Map<String, Object>>get("nodes").size());
-
+        putPipeline();
         ClusterUpdateSettingsResponse settingsResponse = client().admin()
             .cluster()
             .prepareUpdateSettings()
@@ -103,6 +111,33 @@ public void testStats() throws Exception {
         });
     }
 
+    private void putPipeline() throws IOException {
+        BytesReference bytes;
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            builder.startObject();
+            {
+                builder.startArray("processors");
+                {
+                    builder.startObject();
+                    {
+                        builder.startObject("geoip");
+                        {
+                            builder.field("field", "ip");
+                            builder.field("target_field", "ip-city");
+                            builder.field("database_file", "GeoLite2-City.mmdb");
+                        }
+                        builder.endObject();
+                    }
+                    builder.endObject();
+                }
+                builder.endArray();
+            }
+            builder.endObject();
+            bytes = BytesReference.bytes(builder);
+        }
+        assertAcked(client().admin().cluster().preparePutPipeline("_id", bytes, XContentType.JSON).get());
+    }
+
     public static Map<String, Object> convertToMap(ToXContent part) throws IOException {
         XContentBuilder builder = XContentFactory.jsonBuilder();
         part.toXContent(builder, EMPTY_PARAMS);
diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskIT.java
index 60be668272b2..83fde48b39f3 100644
--- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskIT.java
+++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskIT.java
@@ -48,7 +48,7 @@ public void cleanUp() throws Exception {
                 .setPersistentSettings(
                     Settings.builder()
                         .putNull(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey())
-                        .putNull(GeoIpDownloader.POLL_INTERVAL_SETTING.getKey())
+                        .putNull(GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getKey())
                         .putNull("ingest.geoip.database_validity")
                 )
                 .get()
diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java
index 75b9e1eb9fce..178cfe0a20c4 100644
--- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java
+++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java
@@ -112,9 +112,8 @@ public void testLazyLoading() throws IOException {
     }
 
     private void assertDatabaseLoadStatus(final String node, final boolean loaded) {
-        final IngestService ingestService = internalCluster().getInstance(IngestService.class, node);
-        final GeoIpProcessor.Factory factory = (GeoIpProcessor.Factory) ingestService.getProcessorFactories().get("geoip");
-        for (final DatabaseReaderLazyLoader loader : factory.getAllDatabases()) {
+        final DatabaseNodeService databaseNodeService = internalCluster().getInstance(DatabaseNodeService.class, node);
+        for (final DatabaseReaderLazyLoader loader : databaseNodeService.getAllDatabases()) {
             if (loaded) {
                 assertNotNull(loader.databaseReader.get());
             } else {
diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java
index 56abe71f63c0..8d8b0b4215b3 100644
--- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java
+++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java
@@ -64,10 +64,10 @@ public class ReloadingDatabasesWhilePerformingGeoLookupsIT extends ESTestCase {
     public void test() throws Exception {
         Path geoIpConfigDir = createTempDir();
         Path geoIpTmpDir = createTempDir();
-        DatabaseNodeService databaseNodeService = createRegistry(geoIpConfigDir, geoIpTmpDir);
         ClusterService clusterService = mock(ClusterService.class);
         when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE);
-        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService, clusterService);
+        DatabaseNodeService databaseNodeService = createRegistry(geoIpConfigDir, geoIpTmpDir, clusterService);
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
         Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
         Files.copy(ConfigDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"));
         databaseNodeService.updateDatabase("GeoLite2-City.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City.mmdb"));
@@ -190,7 +190,8 @@ public void test() throws Exception {
         IOUtils.rm(geoIpConfigDir, geoIpTmpDir);
     }
 
-    private static DatabaseNodeService createRegistry(Path geoIpConfigDir, Path geoIpTmpDir) throws IOException {
+    private static DatabaseNodeService createRegistry(Path geoIpConfigDir, Path geoIpTmpDir, ClusterService clusterService)
+        throws IOException {
         GeoIpCache cache = new GeoIpCache(0);
         ConfigDatabases configDatabases = new ConfigDatabases(geoIpConfigDir, cache);
         copyDatabaseFiles(geoIpConfigDir, configDatabases);
@@ -199,9 +200,10 @@ private static DatabaseNodeService createRegistry(Path geoIpConfigDir, Path geoI
             mock(Client.class),
             cache,
             configDatabases,
-            Runnable::run
+            Runnable::run,
+            clusterService
         );
-        databaseNodeService.initialize("nodeId", mock(ResourceWatcherService.class), mock(IngestService.class), mock(ClusterService.class));
+        databaseNodeService.initialize("nodeId", mock(ResourceWatcherService.class), mock(IngestService.class));
         return databaseNodeService;
     }
 
diff --git a/modules/ingest-geoip/src/main/java/module-info.java b/modules/ingest-geoip/src/main/java/module-info.java
index 08f12becdf88..fa0b0266414f 100644
--- a/modules/ingest-geoip/src/main/java/module-info.java
+++ b/modules/ingest-geoip/src/main/java/module-info.java
@@ -12,8 +12,8 @@
     requires org.elasticsearch.xcontent;
     requires org.apache.logging.log4j;
     requires org.apache.lucene.core;
-    requires geoip2;
-    requires maxmind.db;
+    requires com.maxmind.geoip2;
+    requires com.maxmind.db;
 
     exports org.elasticsearch.ingest.geoip.stats to org.elasticsearch.server;
 }
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java
index 492b329e1976..e96197ab59ff 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.common.logging.HeaderWarning;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.CheckedRunnable;
 import org.elasticsearch.core.IOUtils;
@@ -63,6 +64,7 @@
 import java.util.zip.GZIPInputStream;
 
 import static org.elasticsearch.core.Strings.format;
+import static org.elasticsearch.persistent.PersistentTasksCustomMetadata.getTaskWithId;
 
 /**
  * A component that is responsible for making the databases maintained by {@link GeoIpDownloader}
@@ -77,13 +79,13 @@
  * 2) For each database check whether the databases have changed
  * by comparing the local and remote md5 hash or are locally missing.
  * 3) For each database identified in step 2 start downloading the database
- * chunks. Each chunks is appended to a tmp file (inside geoip tmp dir) and
+ * chunks. Each chunk is appended to a tmp file (inside geoip tmp dir) and
  * after all chunks have been downloaded, the database is uncompressed and
  * renamed to the final filename.After this the database is loaded and
  * if there is an old instance of this database then that is closed.
  * 4) Cleanup locally loaded databases that are no longer mentioned in {@link GeoIpTaskState}.
  */
-public final class DatabaseNodeService implements Closeable {
+public final class DatabaseNodeService implements GeoIpDatabaseProvider, Closeable {
 
     private static final Logger LOGGER = LogManager.getLogger(DatabaseNodeService.class);
 
@@ -93,34 +95,45 @@ public final class DatabaseNodeService implements Closeable {
     private Path geoipTmpDirectory;
     private final ConfigDatabases configDatabases;
     private final Consumer<Runnable> genericExecutor;
+    private final ClusterService clusterService;
     private IngestService ingestService;
 
     private final ConcurrentMap<String, DatabaseReaderLazyLoader> databases = new ConcurrentHashMap<>();
 
-    DatabaseNodeService(Environment environment, Client client, GeoIpCache cache, Consumer<Runnable> genericExecutor) {
+    DatabaseNodeService(
+        Environment environment,
+        Client client,
+        GeoIpCache cache,
+        Consumer<Runnable> genericExecutor,
+        ClusterService clusterService
+    ) {
         this(
             environment.tmpFile(),
             new OriginSettingClient(client, IngestService.INGEST_ORIGIN),
             cache,
             new ConfigDatabases(environment, cache),
-            genericExecutor
+            genericExecutor,
+            clusterService
         );
     }
 
-    DatabaseNodeService(Path tmpDir, Client client, GeoIpCache cache, ConfigDatabases configDatabases, Consumer<Runnable> genericExecutor) {
+    DatabaseNodeService(
+        Path tmpDir,
+        Client client,
+        GeoIpCache cache,
+        ConfigDatabases configDatabases,
+        Consumer<Runnable> genericExecutor,
+        ClusterService clusterService
+    ) {
         this.client = client;
         this.cache = cache;
         this.geoipTmpBaseDirectory = tmpDir.resolve("geoip-databases");
         this.configDatabases = configDatabases;
         this.genericExecutor = genericExecutor;
+        this.clusterService = clusterService;
     }
 
-    public void initialize(
-        String nodeId,
-        ResourceWatcherService resourceWatcher,
-        IngestService ingestServiceArg,
-        ClusterService clusterService
-    ) throws IOException {
+    public void initialize(String nodeId, ResourceWatcherService resourceWatcher, IngestService ingestServiceArg) throws IOException {
         configDatabases.initialize(resourceWatcher);
         geoipTmpDirectory = geoipTmpBaseDirectory.resolve(nodeId);
         Files.walkFileTree(geoipTmpDirectory, new FileVisitor<>() {
@@ -161,7 +174,35 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
         clusterService.addListener(event -> checkDatabases(event.state()));
     }
 
-    public DatabaseReaderLazyLoader getDatabase(String name) {
+    @Override
+    public Boolean isValid(String databaseFile) {
+        ClusterState currentState = clusterService.state();
+        assert currentState != null;
+
+        PersistentTasksCustomMetadata.PersistentTask<?> task = getTaskWithId(currentState, GeoIpDownloader.GEOIP_DOWNLOADER);
+        if (task == null || task.getState() == null) {
+            return true;
+        }
+        GeoIpTaskState state = (GeoIpTaskState) task.getState();
+        GeoIpTaskState.Metadata metadata = state.getDatabases().get(databaseFile);
+        // we never remove metadata from cluster state, if metadata is null we deal with built-in database, which is always valid
+        if (metadata == null) {
+            return true;
+        }
+
+        boolean valid = metadata.isValid(currentState.metadata().settings());
+        if (valid && metadata.isCloseToExpiration()) {
+            HeaderWarning.addWarning(
+                "database [{}] was not updated for over 25 days, geoip processor" + " will stop working if there is no update for 30 days",
+                databaseFile
+            );
+        }
+
+        return valid;
+    }
+
+    // for testing only:
+    DatabaseReaderLazyLoader getDatabaseReaderLazyLoader(String name) {
         // There is a need for reference counting in order to avoid using an instance
         // that gets closed while using it. (this can happen during a database update)
         while (true) {
@@ -174,6 +215,11 @@ public DatabaseReaderLazyLoader getDatabase(String name) {
         }
     }
 
+    @Override
+    public GeoIpDatabase getDatabase(String name) {
+        return getDatabaseReaderLazyLoader(name);
+    }
+
     List<DatabaseReaderLazyLoader> getAllDatabases() {
         List<DatabaseReaderLazyLoader> all = new ArrayList<>(configDatabases.getConfigDatabases().values());
         this.databases.forEach((key, value) -> all.add(value));
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java
index 13181e1f96bb..3519113033b4 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java
@@ -19,7 +19,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.SetOnce;
-import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.CheckedBiFunction;
 import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.core.Booleans;
@@ -34,8 +33,6 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -44,7 +41,7 @@
  * Facilitates lazy loading of the database reader, so that when the geoip plugin is installed, but not used,
  * no memory is being wasted on the database reader.
  */
-class DatabaseReaderLazyLoader implements Closeable {
+class DatabaseReaderLazyLoader implements GeoIpDatabase, Closeable {
 
     private static final boolean LOAD_DATABASE_ON_HEAP = Booleans.parseBoolean(System.getProperty("es.geoip.load_db_on_heap", "false"));
 
@@ -84,7 +81,8 @@ class DatabaseReaderLazyLoader implements Closeable {
      * @return the database type
      * @throws IOException if an I/O exception occurs reading the database type
      */
-    final String getDatabaseType() throws IOException {
+    @Override
+    public final String getDatabaseType() throws IOException {
         if (databaseType.get() == null) {
             synchronized (databaseType) {
                 if (databaseType.get() == null) {
@@ -154,17 +152,20 @@ InputStream databaseInputStream() throws IOException {
     }
 
     @Nullable
-    CityResponse getCity(InetAddress ipAddress) {
+    @Override
+    public CityResponse getCity(InetAddress ipAddress) {
         return getResponse(ipAddress, DatabaseReader::tryCity);
     }
 
     @Nullable
-    CountryResponse getCountry(InetAddress ipAddress) {
+    @Override
+    public CountryResponse getCountry(InetAddress ipAddress) {
         return getResponse(ipAddress, DatabaseReader::tryCountry);
     }
 
     @Nullable
-    AsnResponse getAsn(InetAddress ipAddress) {
+    @Override
+    public AsnResponse getAsn(InetAddress ipAddress) {
         return getResponse(ipAddress, DatabaseReader::tryAsn);
     }
 
@@ -172,7 +173,8 @@ boolean preLookup() {
         return currentUsages.updateAndGet(current -> current < 0 ? current : current + 1) > 0;
     }
 
-    void postLookup() throws IOException {
+    @Override
+    public void release() throws IOException {
         if (currentUsages.updateAndGet(current -> current > 0 ? current - 1 : current + 1) == -1) {
             doClose();
         }
@@ -187,14 +189,13 @@ private <T extends AbstractResponse> T getResponse(
         InetAddress ipAddress,
         CheckedBiFunction<DatabaseReader, InetAddress, Optional<T>, Exception> responseProvider
     ) {
-        SpecialPermission.check();
-        return AccessController.doPrivileged((PrivilegedAction<T>) () -> cache.putIfAbsent(ipAddress, databasePath.toString(), ip -> {
+        return cache.putIfAbsent(ipAddress, databasePath.toString(), ip -> {
             try {
                 return responseProvider.apply(get(), ipAddress).orElse(null);
             } catch (Exception e) {
                 throw new RuntimeException(e);
             }
-        }));
+        });
     }
 
     DatabaseReader get() throws IOException {
@@ -225,7 +226,8 @@ public void close() throws IOException {
         }
     }
 
-    private void doClose() throws IOException {
+    // Visible for Testing
+    protected void doClose() throws IOException {
         IOUtils.close(databaseReader.get());
         int numEntriesEvicted = cache.purgeCacheEntriesForDatabase(databasePath);
         LOGGER.info("evicted [{}] entries from cache after reloading database [{}]", numEntriesEvicted, databasePath);
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java
index 01938c617db6..30c0fcb74833 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java
@@ -15,7 +15,6 @@
 
 import java.net.InetAddress;
 import java.nio.file.Path;
-import java.util.Objects;
 import java.util.function.Function;
 
 /**
@@ -82,29 +81,5 @@ public int count() {
      * path is needed to be included in the cache key. For example, if we only used the IP address as the key the City and ASN the same
      * IP may be in both with different values and we need to cache both.
      */
-    private static class CacheKey {
-
-        private final InetAddress ip;
-        private final String databasePath;
-
-        private CacheKey(InetAddress ip, String databasePath) {
-            this.ip = ip;
-            this.databasePath = databasePath;
-        }
-
-        // generated
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            CacheKey cacheKey = (CacheKey) o;
-            return Objects.equals(ip, cacheKey.ip) && Objects.equals(databasePath, cacheKey.databasePath);
-        }
-
-        // generated
-        @Override
-        public int hashCode() {
-            return Objects.hash(ip, databasePath);
-        }
-    }
+    private record CacheKey(InetAddress ip, String databasePath) {}
 }
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabase.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabase.java
new file mode 100644
index 000000000000..0bcaab6e6915
--- /dev/null
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabase.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.ingest.geoip;
+
+import com.maxmind.geoip2.model.AsnResponse;
+import com.maxmind.geoip2.model.CityResponse;
+import com.maxmind.geoip2.model.CountryResponse;
+
+import org.elasticsearch.core.Nullable;
+
+import java.io.IOException;
+import java.net.InetAddress;
+
+/**
+ * Provides a uniform interface for interacting with various GeoIP databases.
+ */
+public interface GeoIpDatabase {
+
+    /**
+     * @return the database type as it is detailed in the database file metadata
+     * @throws IOException if the database file could not be read or the data could not be accessed
+     */
+    String getDatabaseType() throws IOException;
+
+    /**
+     * @param ipAddress the IP address to look up
+     * @return a response containing the city data for the given address if it exists, or <code>null</code> if it could not be found
+     * @throws UnsupportedOperationException may be thrown if the implementation does not support retrieving city data
+     */
+    @Nullable
+    CityResponse getCity(InetAddress ipAddress);
+
+    /**
+     * @param ipAddress the IP address to look up
+     * @return a response containing the country data for the given address if it exists, or <code>null</code> if it could not be found
+     * @throws UnsupportedOperationException may be thrown if the implementation does not support retrieving country data
+     */
+    @Nullable
+    CountryResponse getCountry(InetAddress ipAddress);
+
+    /**
+     * @param ipAddress the IP address to look up
+     * @return a response containing the Autonomous System Number for the given address if it exists, or <code>null</code> if it could not
+     *         be found
+     * @throws UnsupportedOperationException may be thrown if the implementation does not support retrieving ASN data
+     */
+    @Nullable
+    AsnResponse getAsn(InetAddress ipAddress);
+
+    /**
+     * Releases the current database object. Called after processing a single document. Databases should be closed or returned to a
+     * resource pool. No further interactions should be expected.
+     * @throws IOException if the implementation encounters any problem while cleaning up
+     */
+    void release() throws IOException;
+}
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabaseProvider.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabaseProvider.java
new file mode 100644
index 000000000000..6baa95904577
--- /dev/null
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabaseProvider.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.ingest.geoip;
+
+/**
+ * Provides construction and initialization logic for {@link GeoIpDatabase} instances.
+ */
+public interface GeoIpDatabaseProvider {
+
+    /**
+     * Determines if the given database name corresponds to an expired database. Expired databases will not be loaded.
+     * <br/><br/>
+     * Verifying database expiration is left to each provider implementation to determine. A return value of <code>false</code> does not
+     * preclude the possibility of a provider returning <code>true</code> in the future.
+     *
+     * @param name the name of the database to provide.
+     * @return <code>false</code> IFF the requested database file is expired,
+     *         <code>true</code> for all other cases (including unknown file name, file missing, wrong database type, etc).
+     */
+    Boolean isValid(String name);
+
+    /**
+     * @param name the name of the database to provide. Default database names that should always be supported are listed in
+     *             {@link IngestGeoIpPlugin#DEFAULT_DATABASE_FILENAMES}.
+     * @return a ready-to-use database instance, or <code>null</code> if no database could be loaded.
+     */
+    GeoIpDatabase getDatabase(String name);
+}
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java
index 6776ab9d629a..0732674632b3 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java
@@ -10,7 +10,6 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
@@ -48,6 +47,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.function.Supplier;
 
 /**
  * Main component responsible for downloading new GeoIP databases.
@@ -59,14 +59,6 @@ public class GeoIpDownloader extends AllocatedPersistentTask {
 
     private static final Logger logger = LogManager.getLogger(GeoIpDownloader.class);
 
-    public static final Setting<TimeValue> POLL_INTERVAL_SETTING = Setting.timeSetting(
-        "ingest.geoip.downloader.poll.interval",
-        TimeValue.timeValueDays(3),
-        TimeValue.timeValueDays(1),
-        Property.Dynamic,
-        Property.NodeScope
-    );
-
     // for overriding in tests
     private static final String DEFAULT_ENDPOINT = System.getProperty(
         "ingest.geoip.downloader.endpoint.default",
@@ -91,9 +83,16 @@ public class GeoIpDownloader extends AllocatedPersistentTask {
 
     // visible for testing
     protected volatile GeoIpTaskState state;
-    private volatile TimeValue pollInterval;
     private volatile Scheduler.ScheduledCancellable scheduled;
     private volatile GeoIpDownloaderStats stats = GeoIpDownloaderStats.EMPTY;
+    private final Supplier<TimeValue> pollIntervalSupplier;
+    private final Supplier<Boolean> eagerDownloadSupplier;
+    /*
+     * This variable tells us whether we have at least one pipeline with a geoip processor. If there are no geoip processors then we do
+     * not download geoip databases (unless configured to eagerly download). Access is not protected because it is set in the constructor
+     * and then only ever updated on the cluster state update thread (it is also read on the generic thread). Non-private for unit testing.
+     */
+    private final Supplier<Boolean> atLeastOneGeoipProcessorSupplier;
 
     GeoIpDownloader(
         Client client,
@@ -106,7 +105,10 @@ public class GeoIpDownloader extends AllocatedPersistentTask {
         String action,
         String description,
         TaskId parentTask,
-        Map<String, String> headers
+        Map<String, String> headers,
+        Supplier<TimeValue> pollIntervalSupplier,
+        Supplier<Boolean> eagerDownloadSupplier,
+        Supplier<Boolean> atLeastOneGeoipProcessorSupplier
     ) {
         super(id, type, action, description, parentTask, headers);
         this.httpClient = httpClient;
@@ -114,15 +116,9 @@ public class GeoIpDownloader extends AllocatedPersistentTask {
         this.clusterService = clusterService;
         this.threadPool = threadPool;
         endpoint = ENDPOINT_SETTING.get(settings);
-        pollInterval = POLL_INTERVAL_SETTING.get(settings);
-        clusterService.getClusterSettings().addSettingsUpdateConsumer(POLL_INTERVAL_SETTING, this::setPollInterval);
-    }
-
-    public void setPollInterval(TimeValue pollInterval) {
-        this.pollInterval = pollInterval;
-        if (scheduled != null && scheduled.cancel()) {
-            scheduleNextRun(TimeValue.ZERO);
-        }
+        this.pollIntervalSupplier = pollIntervalSupplier;
+        this.eagerDownloadSupplier = eagerDownloadSupplier;
+        this.atLeastOneGeoipProcessorSupplier = atLeastOneGeoipProcessorSupplier;
     }
 
     // visible for testing
@@ -130,6 +126,7 @@ void updateDatabases() throws IOException {
         var clusterState = clusterService.state();
         var geoipIndex = clusterState.getMetadata().getIndicesLookup().get(GeoIpDownloader.DATABASES_INDEX);
         if (geoipIndex != null) {
+            logger.trace("The {} index is not null", GeoIpDownloader.DATABASES_INDEX);
             if (clusterState.getRoutingTable().index(geoipIndex.getWriteIndex()).allPrimaryShardsActive() == false) {
                 throw new ElasticsearchException("not all primary shards of [" + DATABASES_INDEX + "] index are active");
             }
@@ -138,13 +135,18 @@ void updateDatabases() throws IOException {
                 throw blockException;
             }
         }
-
-        logger.debug("updating geoip databases");
-        List<Map<String, Object>> response = fetchDatabasesOverview();
-        for (Map<String, Object> res : response) {
-            if (res.get("name").toString().endsWith(".tgz")) {
-                processDatabase(res);
+        if (eagerDownloadSupplier.get() || atLeastOneGeoipProcessorSupplier.get()) {
+            logger.trace("Updating geoip databases");
+            List<Map<String, Object>> response = fetchDatabasesOverview();
+            for (Map<String, Object> res : response) {
+                if (res.get("name").toString().endsWith(".tgz")) {
+                    processDatabase(res);
+                }
             }
+        } else {
+            logger.trace(
+                "Not updating geoip databases because no geoip processors exist in the cluster and eager downloading is not configured"
+            );
         }
     }
 
@@ -186,7 +188,7 @@ void processDatabase(Map<String, Object> databaseInfo) {
             }
         } catch (Exception e) {
             stats = stats.failedDownload();
-            logger.error((Supplier<?>) () -> "error downloading geoip database [" + name + "]", e);
+            logger.error((org.apache.logging.log4j.util.Supplier<?>) () -> "error downloading geoip database [" + name + "]", e);
         }
     }
 
@@ -266,6 +268,9 @@ void setState(GeoIpTaskState state) {
         this.state = state;
     }
 
+    /**
+     * Downloads the geoip databases now, and schedules them to be downloaded again after pollInterval.
+     */
     void runDownloader() {
         if (isCancelled() || isCompleted()) {
             return;
@@ -281,7 +286,22 @@ void runDownloader() {
         } catch (Exception e) {
             logger.error("exception during geoip databases cleanup", e);
         }
-        scheduleNextRun(pollInterval);
+        scheduleNextRun(pollIntervalSupplier.get());
+    }
+
+    /**
+     * This method requests that the downloader be rescheduled to run immediately (presumably because a dynamic property supplied by
+     * pollIntervalSupplier or eagerDownloadSupplier has changed, or a pipeline with a geoip processor has been added). This method does
+     * nothing if this task is cancelled, completed, or has not yet been scheduled to run for the first time. It cancels any existing
+     * scheduled run.
+     */
+    public void requestReschedule() {
+        if (isCancelled() || isCompleted()) {
+            return;
+        }
+        if (scheduled != null && scheduled.cancel()) {
+            scheduleNextRun(TimeValue.ZERO);
+        }
     }
 
     private void cleanDatabases() {
@@ -321,4 +341,5 @@ private void scheduleNextRun(TimeValue time) {
             scheduled = threadPool.schedule(this::runDownloader, time, ThreadPool.Names.GENERIC);
         }
     }
+
 }
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java
index c56fd9c2d0c5..7457738b7530 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java
@@ -17,15 +17,20 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.ingest.IngestMetadata;
 import org.elasticsearch.ingest.IngestService;
+import org.elasticsearch.ingest.Pipeline;
+import org.elasticsearch.ingest.PipelineConfiguration;
 import org.elasticsearch.persistent.AllocatedPersistentTask;
 import org.elasticsearch.persistent.PersistentTaskState;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
@@ -35,7 +40,10 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.RemoteTransportException;
 
+import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.ingest.geoip.GeoIpDownloader.DATABASES_INDEX;
@@ -56,6 +64,20 @@ public final class GeoIpDownloaderTaskExecutor extends PersistentTasksExecutor<G
         Setting.Property.Dynamic,
         Setting.Property.NodeScope
     );
+    public static final Setting<TimeValue> POLL_INTERVAL_SETTING = Setting.timeSetting(
+        "ingest.geoip.downloader.poll.interval",
+        TimeValue.timeValueDays(3),
+        TimeValue.timeValueDays(1),
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting<Boolean> EAGER_DOWNLOAD_SETTING = Setting.boolSetting(
+        "ingest.geoip.downloader.eager.download",
+        false,
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
 
     private static final Logger logger = LogManager.getLogger(GeoIpDownloader.class);
 
@@ -66,6 +88,10 @@ public final class GeoIpDownloaderTaskExecutor extends PersistentTasksExecutor<G
     private final Settings settings;
     private final PersistentTasksService persistentTasksService;
     private final AtomicReference<GeoIpDownloader> currentTask = new AtomicReference<>();
+    private volatile TimeValue pollInterval;
+    private volatile boolean eagerDownload;
+    private volatile boolean atLeastOneGeoipProcessor;
+    private final AtomicBoolean taskIsBootstrapped = new AtomicBoolean(false);
 
     GeoIpDownloaderTaskExecutor(Client client, HttpClient httpClient, ClusterService clusterService, ThreadPool threadPool) {
         super(GEOIP_DOWNLOADER, ThreadPool.Names.GENERIC);
@@ -75,9 +101,18 @@ public final class GeoIpDownloaderTaskExecutor extends PersistentTasksExecutor<G
         this.threadPool = threadPool;
         this.settings = clusterService.getSettings();
         persistentTasksService = new PersistentTasksService(clusterService, threadPool, client);
-        clusterService.addListener(this);
+        this.pollInterval = POLL_INTERVAL_SETTING.get(settings);
+        this.eagerDownload = EAGER_DOWNLOAD_SETTING.get(settings);
+    }
 
+    /**
+     * This method completes the initialization of the GeoIpDownloaderTaskExecutor by registering several listeners.
+     */
+    public void init() {
+        clusterService.addListener(this);
         clusterService.getClusterSettings().addSettingsUpdateConsumer(ENABLED_SETTING, this::setEnabled);
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(EAGER_DOWNLOAD_SETTING, this::setEagerDownload);
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(POLL_INTERVAL_SETTING, this::setPollInterval);
     }
 
     private void setEnabled(boolean enabled) {
@@ -92,12 +127,32 @@ private void setEnabled(boolean enabled) {
         }
     }
 
+    private void setEagerDownload(Boolean eagerDownload) {
+        if (Objects.equals(this.eagerDownload, eagerDownload) == false) {
+            this.eagerDownload = eagerDownload;
+            GeoIpDownloader currentDownloader = getCurrentTask();
+            if (currentDownloader != null && Objects.equals(eagerDownload, Boolean.TRUE)) {
+                currentDownloader.requestReschedule();
+            }
+        }
+    }
+
+    private void setPollInterval(TimeValue pollInterval) {
+        if (Objects.equals(this.pollInterval, pollInterval) == false) {
+            this.pollInterval = pollInterval;
+            GeoIpDownloader currentDownloader = getCurrentTask();
+            if (currentDownloader != null) {
+                currentDownloader.requestReschedule();
+            }
+        }
+    }
+
     @Override
     protected void nodeOperation(AllocatedPersistentTask task, GeoIpTaskParams params, PersistentTaskState state) {
         GeoIpDownloader downloader = (GeoIpDownloader) task;
-        currentTask.set(downloader);
         GeoIpTaskState geoIpTaskState = state == null ? GeoIpTaskState.EMPTY : (GeoIpTaskState) state;
         downloader.setState(geoIpTaskState);
+        currentTask.set(downloader);
         if (ENABLED_SETTING.get(clusterService.state().metadata().settings(), settings)) {
             downloader.runDownloader();
         }
@@ -123,7 +178,10 @@ protected GeoIpDownloader createTask(
             action,
             getDescription(taskInProgress),
             parentTaskId,
-            headers
+            headers,
+            () -> pollInterval,
+            () -> eagerDownload,
+            () -> atLeastOneGeoipProcessor
         );
     }
 
@@ -140,12 +198,65 @@ public void clusterChanged(ClusterChangedEvent event) {
             return;
         }
 
-        clusterService.removeListener(this);
-        if (ENABLED_SETTING.get(event.state().getMetadata().settings(), settings)) {
-            startTask(() -> clusterService.addListener(this));
-        } else {
-            stopTask(() -> clusterService.addListener(this));
+        if (taskIsBootstrapped.getAndSet(true) == false) {
+            this.atLeastOneGeoipProcessor = hasAtLeastOneGeoipProcessor(event.state());
+            if (ENABLED_SETTING.get(event.state().getMetadata().settings(), settings)) {
+                startTask(() -> taskIsBootstrapped.set(false));
+            } else {
+                stopTask(() -> taskIsBootstrapped.set(false));
+            }
         }
+
+        if (event.metadataChanged() && event.changedCustomMetadataSet().contains(IngestMetadata.TYPE)) {
+            boolean newAtLeastOneGeoipProcessor = hasAtLeastOneGeoipProcessor(event.state());
+            if (newAtLeastOneGeoipProcessor && atLeastOneGeoipProcessor == false) {
+                atLeastOneGeoipProcessor = true;
+                logger.trace("Scheduling runDownloader because a geoip processor has been added");
+                GeoIpDownloader currentDownloader = getCurrentTask();
+                if (currentDownloader != null) {
+                    currentDownloader.requestReschedule();
+                }
+            } else {
+                atLeastOneGeoipProcessor = newAtLeastOneGeoipProcessor;
+            }
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    static boolean hasAtLeastOneGeoipProcessor(ClusterState clusterState) {
+        List<PipelineConfiguration> pipelineDefinitions = IngestService.getPipelines(clusterState);
+        return pipelineDefinitions.stream().anyMatch(pipelineDefinition -> {
+            Map<String, Object> pipelineMap = pipelineDefinition.getConfigAsMap();
+            return hasAtLeastOneGeoipProcessor((List<Map<String, Object>>) pipelineMap.get(Pipeline.PROCESSORS_KEY));
+        });
+    }
+
+    private static boolean hasAtLeastOneGeoipProcessor(List<Map<String, Object>> processors) {
+        return processors != null && processors.stream().anyMatch(GeoIpDownloaderTaskExecutor::hasAtLeastOneGeoipProcessor);
+    }
+
+    private static boolean hasAtLeastOneGeoipProcessor(Map<String, Object> processor) {
+        return processor != null
+            && (processor.containsKey(GeoIpProcessor.TYPE)
+                || isProcessorWithOnFailureGeoIpProcessor(processor)
+                || isForeachProcessorWithGeoipProcessor(processor));
+    }
+
+    @SuppressWarnings("unchecked")
+    private static boolean isProcessorWithOnFailureGeoIpProcessor(Map<String, Object> processor) {
+        return processor != null
+            && processor.values()
+                .stream()
+                .anyMatch(
+                    value -> value instanceof Map
+                        && hasAtLeastOneGeoipProcessor(((Map<String, List<Map<String, Object>>>) value).get("on_failure"))
+                );
+    }
+
+    @SuppressWarnings("unchecked")
+    private static boolean isForeachProcessorWithGeoipProcessor(Map<String, Object> processor) {
+        return processor.containsKey("foreach")
+            && hasAtLeastOneGeoipProcessor(((Map<String, Map<String, Object>>) processor.get("foreach")).get("processor"));
     }
 
     private void startTask(Runnable onFailure) {
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
index 1e9185f17594..eadad3673eba 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
@@ -20,18 +20,14 @@
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.logging.HeaderWarning;
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
-import org.elasticsearch.persistent.PersistentTasksCustomMetadata.PersistentTask;
 
 import java.io.IOException;
 import java.net.InetAddress;
@@ -50,7 +46,6 @@
 import static org.elasticsearch.ingest.ConfigurationUtils.readBooleanProperty;
 import static org.elasticsearch.ingest.ConfigurationUtils.readOptionalList;
 import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
-import static org.elasticsearch.persistent.PersistentTasksCustomMetadata.getTaskWithId;
 
 public final class GeoIpProcessor extends AbstractProcessor {
 
@@ -66,7 +61,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
     private final String field;
     private final Supplier<Boolean> isValid;
     private final String targetField;
-    private final CheckedSupplier<DatabaseReaderLazyLoader, IOException> supplier;
+    private final CheckedSupplier<GeoIpDatabase, IOException> supplier;
     private final Set<Property> properties;
     private final boolean ignoreMissing;
     private final boolean firstOnly;
@@ -74,22 +69,22 @@ public final class GeoIpProcessor extends AbstractProcessor {
 
     /**
      * Construct a geo-IP processor.
-     *  @param tag           the processor tag
+     * @param tag           the processor tag
      * @param description   the processor description
      * @param field         the source field to geo-IP map
      * @param supplier      a supplier of a geo-IP database reader; ideally this is lazily-loaded once on first use
-     * @param isValid
+     * @param isValid       a supplier that determines if the available database files are up-to-date and license compliant
      * @param targetField   the target field
      * @param properties    the properties; ideally this is lazily-loaded once on first use
      * @param ignoreMissing true if documents with a missing value for the field should be ignored
      * @param firstOnly     true if only first result should be returned in case of array
-     * @param databaseFile
+     * @param databaseFile  the name of the database file being queried; used only for tagging documents if the database is unavailable
      */
     GeoIpProcessor(
         final String tag,
         final String description,
         final String field,
-        final CheckedSupplier<DatabaseReaderLazyLoader, IOException> supplier,
+        final CheckedSupplier<GeoIpDatabase, IOException> supplier,
         final Supplier<Boolean> isValid,
         final String targetField,
         final Set<Property> properties,
@@ -125,70 +120,70 @@ public IngestDocument execute(IngestDocument ingestDocument) throws IOException
             throw new IllegalArgumentException("field [" + field + "] is null, cannot extract geoip information.");
         }
 
-        DatabaseReaderLazyLoader lazyLoader = this.supplier.get();
-        if (lazyLoader == null) {
+        GeoIpDatabase geoIpDatabase = this.supplier.get();
+        if (geoIpDatabase == null) {
             if (ignoreMissing == false) {
                 tag(ingestDocument, databaseFile);
             }
             return ingestDocument;
         }
 
-        if (ip instanceof String ipString) {
-            Map<String, Object> geoData = getGeoData(lazyLoader, ipString);
-            if (geoData.isEmpty() == false) {
-                ingestDocument.setFieldValue(targetField, geoData);
-            }
-        } else if (ip instanceof List<?> ipList) {
-            boolean match = false;
-            List<Map<String, Object>> geoDataList = new ArrayList<>(ipList.size());
-            for (Object ipAddr : ipList) {
-                if (ipAddr instanceof String == false) {
-                    throw new IllegalArgumentException("array in field [" + field + "] should only contain strings");
+        try {
+            if (ip instanceof String ipString) {
+                Map<String, Object> geoData = getGeoData(geoIpDatabase, ipString);
+                if (geoData.isEmpty() == false) {
+                    ingestDocument.setFieldValue(targetField, geoData);
                 }
-                Map<String, Object> geoData = getGeoData(lazyLoader, (String) ipAddr);
-                if (geoData.isEmpty()) {
-                    geoDataList.add(null);
-                    continue;
+            } else if (ip instanceof List<?> ipList) {
+                boolean match = false;
+                List<Map<String, Object>> geoDataList = new ArrayList<>(ipList.size());
+                for (Object ipAddr : ipList) {
+                    if (ipAddr instanceof String == false) {
+                        throw new IllegalArgumentException("array in field [" + field + "] should only contain strings");
+                    }
+                    Map<String, Object> geoData = getGeoData(geoIpDatabase, (String) ipAddr);
+                    if (geoData.isEmpty()) {
+                        geoDataList.add(null);
+                        continue;
+                    }
+                    if (firstOnly) {
+                        ingestDocument.setFieldValue(targetField, geoData);
+                        return ingestDocument;
+                    }
+                    match = true;
+                    geoDataList.add(geoData);
                 }
-                if (firstOnly) {
-                    ingestDocument.setFieldValue(targetField, geoData);
-                    return ingestDocument;
+                if (match) {
+                    ingestDocument.setFieldValue(targetField, geoDataList);
                 }
-                match = true;
-                geoDataList.add(geoData);
-            }
-            if (match) {
-                ingestDocument.setFieldValue(targetField, geoDataList);
+            } else {
+                throw new IllegalArgumentException("field [" + field + "] should contain only string or array of strings");
             }
-        } else {
-            throw new IllegalArgumentException("field [" + field + "] should contain only string or array of strings");
+        } finally {
+            geoIpDatabase.release();
         }
         return ingestDocument;
     }
 
-    private Map<String, Object> getGeoData(DatabaseReaderLazyLoader lazyLoader, String ip) throws IOException {
-        try {
-            final String databaseType = lazyLoader.getDatabaseType();
-            final InetAddress ipAddress = InetAddresses.forString(ip);
-            Map<String, Object> geoData;
-            if (databaseType.endsWith(CITY_DB_SUFFIX)) {
-                geoData = retrieveCityGeoData(lazyLoader, ipAddress);
-            } else if (databaseType.endsWith(COUNTRY_DB_SUFFIX)) {
-                geoData = retrieveCountryGeoData(lazyLoader, ipAddress);
+    private Map<String, Object> getGeoData(GeoIpDatabase geoIpDatabase, String ip) throws IOException {
+        final String databaseType = geoIpDatabase.getDatabaseType();
+        final InetAddress ipAddress = InetAddresses.forString(ip);
+        Map<String, Object> geoData;
+        if (databaseType.endsWith(CITY_DB_SUFFIX)) {
+            geoData = retrieveCityGeoData(geoIpDatabase, ipAddress);
+        } else if (databaseType.endsWith(COUNTRY_DB_SUFFIX)) {
+            geoData = retrieveCountryGeoData(geoIpDatabase, ipAddress);
 
-            } else if (databaseType.endsWith(ASN_DB_SUFFIX)) {
-                geoData = retrieveAsnGeoData(lazyLoader, ipAddress);
+        } else if (databaseType.endsWith(ASN_DB_SUFFIX)) {
+            geoData = retrieveAsnGeoData(geoIpDatabase, ipAddress);
 
-            } else {
-                throw new ElasticsearchParseException(
-                    "Unsupported database type [" + lazyLoader.getDatabaseType() + "]",
-                    new IllegalStateException()
-                );
-            }
-            return geoData;
-        } finally {
-            lazyLoader.postLookup();
+        } else {
+            throw new ElasticsearchParseException(
+                "Unsupported database type [" + geoIpDatabase.getDatabaseType() + "]",
+                new IllegalStateException()
+            );
         }
+        return geoData;
     }
 
     @Override
@@ -212,8 +207,8 @@ Set<Property> getProperties() {
         return properties;
     }
 
-    private Map<String, Object> retrieveCityGeoData(DatabaseReaderLazyLoader lazyLoader, InetAddress ipAddress) {
-        CityResponse response = lazyLoader.getCity(ipAddress);
+    private Map<String, Object> retrieveCityGeoData(GeoIpDatabase geoIpDatabase, InetAddress ipAddress) {
+        CityResponse response = geoIpDatabase.getCity(ipAddress);
         if (response == null) {
             return Map.of();
         }
@@ -288,8 +283,8 @@ private Map<String, Object> retrieveCityGeoData(DatabaseReaderLazyLoader lazyLoa
         return geoData;
     }
 
-    private Map<String, Object> retrieveCountryGeoData(DatabaseReaderLazyLoader lazyLoader, InetAddress ipAddress) {
-        CountryResponse response = lazyLoader.getCountry(ipAddress);
+    private Map<String, Object> retrieveCountryGeoData(GeoIpDatabase geoIpDatabase, InetAddress ipAddress) {
+        CountryResponse response = geoIpDatabase.getCountry(ipAddress);
         if (response == null) {
             return Map.of();
         }
@@ -323,8 +318,8 @@ private Map<String, Object> retrieveCountryGeoData(DatabaseReaderLazyLoader lazy
         return geoData;
     }
 
-    private Map<String, Object> retrieveAsnGeoData(DatabaseReaderLazyLoader lazyLoader, InetAddress ipAddress) {
-        AsnResponse response = lazyLoader.getAsn(ipAddress);
+    private Map<String, Object> retrieveAsnGeoData(GeoIpDatabase geoIpDatabase, InetAddress ipAddress) {
+        AsnResponse response = geoIpDatabase.getAsn(ipAddress);
         if (response == null) {
             return Map.of();
         }
@@ -358,6 +353,40 @@ private Map<String, Object> retrieveAsnGeoData(DatabaseReaderLazyLoader lazyLoad
         return geoData;
     }
 
+    /**
+     * Retrieves and verifies a {@link GeoIpDatabase} instance for each execution of the {@link GeoIpProcessor}. Guards against missing
+     * custom databases, and ensures that database instances are of the proper type before use.
+     */
+    public static final class DatabaseVerifyingSupplier implements CheckedSupplier<GeoIpDatabase, IOException> {
+        private final GeoIpDatabaseProvider geoIpDatabaseProvider;
+        private final String databaseFile;
+        private final String databaseType;
+
+        public DatabaseVerifyingSupplier(GeoIpDatabaseProvider geoIpDatabaseProvider, String databaseFile, String databaseType) {
+            this.geoIpDatabaseProvider = geoIpDatabaseProvider;
+            this.databaseFile = databaseFile;
+            this.databaseType = databaseType;
+        }
+
+        @Override
+        public GeoIpDatabase get() throws IOException {
+            GeoIpDatabase loader = geoIpDatabaseProvider.getDatabase(databaseFile);
+            if (Factory.useDatabaseUnavailableProcessor(loader, databaseFile)) {
+                return null;
+            } else if (loader == null) {
+                throw new ResourceNotFoundException("database file [" + databaseFile + "] doesn't exist");
+            }
+            // Only check whether the suffix has changed and not the entire database type.
+            // To sanity check whether a city db isn't overwriting with a country or asn db.
+            // For example overwriting a geoip lite city db with geoip city db is a valid change, but the db type is slightly different,
+            // by checking just the suffix this assertion doesn't fail.
+            String expectedSuffix = databaseType.substring(databaseType.lastIndexOf('-'));
+            assert loader.getDatabaseType().endsWith(expectedSuffix)
+                : "database type [" + loader.getDatabaseType() + "] doesn't match with expected suffix [" + expectedSuffix + "]";
+            return loader;
+        }
+    }
+
     public static final class Factory implements Processor.Factory {
         static final Set<Property> DEFAULT_CITY_PROPERTIES = Collections.unmodifiableSet(
             EnumSet.of(
@@ -377,16 +406,10 @@ public static final class Factory implements Processor.Factory {
             EnumSet.of(Property.IP, Property.ASN, Property.ORGANIZATION_NAME, Property.NETWORK)
         );
 
-        private final DatabaseNodeService databaseNodeService;
-        private final ClusterService clusterService;
-
-        List<DatabaseReaderLazyLoader> getAllDatabases() {
-            return databaseNodeService.getAllDatabases();
-        }
+        private final GeoIpDatabaseProvider geoIpDatabaseProvider;
 
-        public Factory(DatabaseNodeService databaseNodeService, ClusterService clusterService) {
-            this.databaseNodeService = databaseNodeService;
-            this.clusterService = clusterService;
+        public Factory(GeoIpDatabaseProvider geoIpDatabaseProvider) {
+            this.geoIpDatabaseProvider = geoIpDatabaseProvider;
         }
 
         @Override
@@ -409,17 +432,17 @@ public Processor create(
                 DEPRECATION_LOGGER.warn(DeprecationCategory.OTHER, "default_databases_message", DEFAULT_DATABASES_DEPRECATION_MESSAGE);
             }
 
-            DatabaseReaderLazyLoader lazyLoader = databaseNodeService.getDatabase(databaseFile);
-            if (useDatabaseUnavailableProcessor(lazyLoader, databaseFile)) {
+            GeoIpDatabase geoIpDatabase = geoIpDatabaseProvider.getDatabase(databaseFile);
+            if (useDatabaseUnavailableProcessor(geoIpDatabase, databaseFile)) {
                 return new DatabaseUnavailableProcessor(processorTag, description, databaseFile);
-            } else if (lazyLoader == null) {
+            } else if (geoIpDatabase == null) {
                 throw newConfigurationException(TYPE, processorTag, "database_file", "database file [" + databaseFile + "] doesn't exist");
             }
             final String databaseType;
             try {
-                databaseType = lazyLoader.getDatabaseType();
+                databaseType = geoIpDatabase.getDatabaseType();
             } finally {
-                lazyLoader.postLookup();
+                geoIpDatabase.release();
             }
 
             final Set<Property> properties;
@@ -449,54 +472,12 @@ public Processor create(
                     );
                 }
             }
-            CheckedSupplier<DatabaseReaderLazyLoader, IOException> supplier = () -> {
-                DatabaseReaderLazyLoader loader = databaseNodeService.getDatabase(databaseFile);
-                if (useDatabaseUnavailableProcessor(loader, databaseFile)) {
-                    return null;
-                } else if (loader == null) {
-                    throw new ResourceNotFoundException("database file [" + databaseFile + "] doesn't exist");
-                }
-                // Only check whether the suffix has changed and not the entire database type.
-                // To sanity check whether a city db isn't overwriting with a country or asn db.
-                // For example overwriting a geoip lite city db with geoip city db is a valid change, but the db type is slightly different,
-                // by checking just the suffix this assertion doesn't fail.
-                String expectedSuffix = databaseType.substring(databaseType.lastIndexOf('-'));
-                assert loader.getDatabaseType().endsWith(expectedSuffix)
-                    : "database type [" + loader.getDatabaseType() + "] doesn't match with expected suffix [" + expectedSuffix + "]";
-                return loader;
-            };
-            Supplier<Boolean> isValid = () -> {
-                ClusterState currentState = clusterService.state();
-                assert currentState != null;
-
-                PersistentTask<?> task = getTaskWithId(currentState, GeoIpDownloader.GEOIP_DOWNLOADER);
-                if (task == null || task.getState() == null) {
-                    return true;
-                }
-                GeoIpTaskState state = (GeoIpTaskState) task.getState();
-                GeoIpTaskState.Metadata metadata = state.getDatabases().get(databaseFile);
-                // we never remove metadata from cluster state, if metadata is null we deal with built-in database, which is always valid
-                if (metadata == null) {
-                    return true;
-                }
-
-                boolean valid = metadata.isValid(currentState.metadata().settings());
-                if (valid && metadata.isCloseToExpiration()) {
-                    HeaderWarning.addWarning(
-                        "database [{}] was not updated for over 25 days, geoip processor"
-                            + " will stop working if there is no update for 30 days",
-                        databaseFile
-                    );
-                }
-
-                return valid;
-            };
             return new GeoIpProcessor(
                 processorTag,
                 description,
                 ipField,
-                supplier,
-                isValid,
+                new DatabaseVerifyingSupplier(geoIpDatabaseProvider, databaseFile, databaseType),
+                () -> geoIpDatabaseProvider.isValid(databaseFile),
                 targetField,
                 properties,
                 ignoreMissing,
@@ -505,12 +486,12 @@ public Processor create(
             );
         }
 
-        private static boolean useDatabaseUnavailableProcessor(DatabaseReaderLazyLoader loader, String databaseName) {
-            // If there is no loader for a database we should fail with a config error, but
-            // if there is no loader for a builtin database that we manage via GeoipDownloader then don't fail.
+        private static boolean useDatabaseUnavailableProcessor(GeoIpDatabase database, String databaseName) {
+            // If there is no instance for a database we should fail with a config error, but
+            // if there is no instance for a builtin database that we manage via GeoipDownloader then don't fail.
             // In the latter case the database should become available at a later moment, so a processor impl
             // is returned that tags documents instead.
-            return loader == null && IngestGeoIpPlugin.DEFAULT_DATABASE_FILENAMES.contains(databaseName);
+            return database == null && IngestGeoIpPlugin.DEFAULT_DATABASE_FILENAMES.contains(databaseName);
         }
 
     }
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskParams.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskParams.java
index 6997964ace89..a8fe538f954d 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskParams.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskParams.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.ingest.geoip;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.persistent.PersistentTaskParams;
@@ -41,8 +41,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_13_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_13_0;
     }
 
     @Override
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java
index 40f2b49254a4..c924c19210f4 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.ingest.geoip;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.VersionedNamedWriteable;
@@ -124,8 +124,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_13_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_13_0;
     }
 
     @Override
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
index 6be4b1bf6e67..7873519478a2 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
@@ -16,7 +16,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -85,9 +85,10 @@ public class IngestGeoIpPlugin extends Plugin implements IngestPlugin, SystemInd
     public List<Setting<?>> getSettings() {
         return Arrays.asList(
             CACHE_SIZE,
+            GeoIpDownloaderTaskExecutor.EAGER_DOWNLOAD_SETTING,
+            GeoIpDownloaderTaskExecutor.ENABLED_SETTING,
             GeoIpDownloader.ENDPOINT_SETTING,
-            GeoIpDownloader.POLL_INTERVAL_SETTING,
-            GeoIpDownloaderTaskExecutor.ENABLED_SETTING
+            GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING
         );
     }
 
@@ -97,9 +98,15 @@ public Map<String, Processor.Factory> getProcessors(Processor.Parameters paramet
 
         long cacheSize = CACHE_SIZE.get(parameters.env.settings());
         GeoIpCache geoIpCache = new GeoIpCache(cacheSize);
-        DatabaseNodeService registry = new DatabaseNodeService(parameters.env, parameters.client, geoIpCache, parameters.genericExecutor);
+        DatabaseNodeService registry = new DatabaseNodeService(
+            parameters.env,
+            parameters.client,
+            geoIpCache,
+            parameters.genericExecutor,
+            parameters.ingestService.getClusterService()
+        );
         databaseRegistry.set(registry);
-        return Map.of(GeoIpProcessor.TYPE, new GeoIpProcessor.Factory(registry, parameters.ingestService.getClusterService()));
+        return Map.of(GeoIpProcessor.TYPE, new GeoIpProcessor.Factory(registry));
     }
 
     @Override
@@ -116,16 +123,17 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         try {
             String nodeId = nodeEnvironment.nodeId();
-            databaseRegistry.get().initialize(nodeId, resourceWatcherService, ingestService.get(), clusterService);
+            databaseRegistry.get().initialize(nodeId, resourceWatcherService, ingestService.get());
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
 
         geoIpDownloaderTaskExecutor = new GeoIpDownloaderTaskExecutor(client, new HttpClient(), clusterService, threadPool);
+        geoIpDownloaderTaskExecutor.init();
         return List.of(databaseRegistry.get(), geoIpDownloaderTaskExecutor);
     }
 
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsAction.java
index a160dfeec9b4..228758e886c6 100644
--- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsAction.java
+++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsAction.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.ingest.geoip.stats;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.nodes.BaseNodeResponse;
@@ -166,7 +166,7 @@ protected NodeResponse(StreamInput in) throws IOException {
             stats = in.readBoolean() ? new GeoIpDownloaderStats(in) : null;
             databases = in.readSet(StreamInput::readString);
             filesInTemp = in.readSet(StreamInput::readString);
-            configDatabases = in.getVersion().onOrAfter(Version.V_8_0_0) ? in.readSet(StreamInput::readString) : null;
+            configDatabases = in.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0) ? in.readSet(StreamInput::readString) : null;
         }
 
         protected NodeResponse(
@@ -208,7 +208,7 @@ public void writeTo(StreamOutput out) throws IOException {
             }
             out.writeCollection(databases, StreamOutput::writeString);
             out.writeCollection(filesInTemp, StreamOutput::writeString);
-            if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
                 out.writeCollection(configDatabases, StreamOutput::writeString);
             }
         }
diff --git a/modules/ingest-geoip/src/main/plugin-metadata/plugin-security.policy b/modules/ingest-geoip/src/main/plugin-metadata/plugin-security.policy
index 2f1e80e8e557..7002fba5c0c4 100644
--- a/modules/ingest-geoip/src/main/plugin-metadata/plugin-security.policy
+++ b/modules/ingest-geoip/src/main/plugin-metadata/plugin-security.policy
@@ -7,13 +7,5 @@
  */
 
 grant {
-  // needed because jackson-databind is using Class#getDeclaredConstructors(), Class#getDeclaredMethods() and
-  // Class#getDeclaredAnnotations() to find all public, private, protected, package protected and
-  // private constructors, methods or annotations. Just locating all public constructors, methods and annotations
-  // should be enough, so this permission wouldn't then be needed. Unfortunately this is not what jackson-databind does
-  // or can be configured to do.
-  permission java.lang.RuntimePermission "accessDeclaredMembers";
-  // Also needed because of jackson-databind:
-  permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
   permission java.net.SocketPermission "*", "connect";
 };
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java
index b775ffdb1f06..f2f6bb343f87 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.CheckedRunnable;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.TermQueryBuilder;
@@ -124,8 +125,8 @@ public void setup() throws IOException {
         ingestService = mock(IngestService.class);
         clusterService = mock(ClusterService.class);
         geoIpTmpDir = createTempDir();
-        databaseNodeService = new DatabaseNodeService(geoIpTmpDir, client, cache, configDatabases, Runnable::run);
-        databaseNodeService.initialize("nodeId", resourceWatcherService, ingestService, clusterService);
+        databaseNodeService = new DatabaseNodeService(geoIpTmpDir, client, cache, configDatabases, Runnable::run, clusterService);
+        databaseNodeService.initialize("nodeId", resourceWatcherService, ingestService);
     }
 
     @After
@@ -150,7 +151,7 @@ public void testCheckDatabases() throws Exception {
         assertThat(databaseNodeService.getDatabase("GeoIP2-City.mmdb"), nullValue());
         // Nothing should be downloaded, since the database is no longer valid (older than 30 days)
         databaseNodeService.checkDatabases(state);
-        DatabaseReaderLazyLoader database = databaseNodeService.getDatabase("GeoIP2-City.mmdb");
+        DatabaseReaderLazyLoader database = databaseNodeService.getDatabaseReaderLazyLoader("GeoIP2-City.mmdb");
         assertThat(database, nullValue());
         verify(client, times(0)).search(any());
         verify(ingestService, times(0)).reloadPipeline(anyString());
@@ -168,7 +169,7 @@ public void testCheckDatabases() throws Exception {
 
         // Database should be downloaded
         databaseNodeService.checkDatabases(state);
-        database = databaseNodeService.getDatabase("GeoIP2-City.mmdb");
+        database = databaseNodeService.getDatabaseReaderLazyLoader("GeoIP2-City.mmdb");
         assertThat(database, notNullValue());
         verify(client, times(10)).search(any());
         try (Stream<Path> files = Files.list(geoIpTmpDir.resolve("geoip-databases").resolve("nodeId"))) {
@@ -382,7 +383,8 @@ static ClusterState createClusterState(PersistentTasksCustomMetadata tasksCustom
             new ShardId(index, 0),
             true,
             RecoverySource.ExistingStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""),
+            ShardRouting.Role.DEFAULT
         );
         String nodeId = ESTestCase.randomAlphaOfLength(8);
         shardRouting = shardRouting.initialize(nodeId, null, shardRouting.getExpectedShardSize());
@@ -410,7 +412,7 @@ private static List<byte[]> gzip(String name, String content, int chunks) throws
         byte[] header = new byte[512];
         byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8);
         byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8);
-        byte[] sizeBytes = formatted("%1$012o", contentBytes.length).getBytes(StandardCharsets.UTF_8);
+        byte[] sizeBytes = Strings.format("%1$012o", contentBytes.length).getBytes(StandardCharsets.UTF_8);
         System.arraycopy(nameBytes, 0, header, 0, nameBytes.length);
         System.arraycopy(sizeBytes, 0, header, 124, 12);
         gzipOutputStream.write(header);
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutorTests.java
new file mode 100644
index 000000000000..5cbe205f5c9c
--- /dev/null
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutorTests.java
@@ -0,0 +1,253 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.ingest.geoip;
+
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.ingest.IngestMetadata;
+import org.elasticsearch.ingest.PipelineConfiguration;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class GeoIpDownloaderTaskExecutorTests extends ESTestCase {
+    public void testHasAtLeastOneGeoipProcessor() {
+        Map<String, PipelineConfiguration> configs = new HashMap<>();
+        IngestMetadata ingestMetadata = new IngestMetadata(configs);
+        ClusterState clusterState = mock(ClusterState.class);
+        Metadata metadata = mock(Metadata.class);
+        when(metadata.custom(IngestMetadata.TYPE)).thenReturn(ingestMetadata);
+        when(clusterState.getMetadata()).thenReturn(metadata);
+        List<String> expectHitsInputs = getPipelinesWithGeoIpProcessors();
+        List<String> expectMissesInputs = getPipelinesWithoutGeoIpProcessors();
+        {
+            // Test that hasAtLeastOneGeoipProcessor returns true for any pipeline with a geoip processor:
+            for (String pipeline : expectHitsInputs) {
+                configs.clear();
+                configs.put("_id1", new PipelineConfiguration("_id1", new BytesArray(pipeline), XContentType.JSON));
+                assertTrue(GeoIpDownloaderTaskExecutor.hasAtLeastOneGeoipProcessor(clusterState));
+            }
+        }
+        {
+            // Test that hasAtLeastOneGeoipProcessor returns false for any pipeline without a geoip processor:
+            for (String pipeline : expectMissesInputs) {
+                configs.clear();
+                configs.put("_id1", new PipelineConfiguration("_id1", new BytesArray(pipeline), XContentType.JSON));
+                assertFalse(GeoIpDownloaderTaskExecutor.hasAtLeastOneGeoipProcessor(clusterState));
+            }
+        }
+        {
+            /*
+             * Now test that hasAtLeastOneGeoipProcessor returns true for a mix of pipelines, some which have geoip processors and some
+             * which do not:
+             */
+            configs.clear();
+            for (String pipeline : expectHitsInputs) {
+                String id = randomAlphaOfLength(20);
+                configs.put(id, new PipelineConfiguration(id, new BytesArray(pipeline), XContentType.JSON));
+            }
+            for (String pipeline : expectMissesInputs) {
+                String id = randomAlphaOfLength(20);
+                configs.put(id, new PipelineConfiguration(id, new BytesArray(pipeline), XContentType.JSON));
+            }
+            assertTrue(GeoIpDownloaderTaskExecutor.hasAtLeastOneGeoipProcessor(clusterState));
+        }
+    }
+
+    /*
+     * This method returns an assorted list of pipelines that have geoip processors -- ones that ought to cause hasAtLeastOneGeoipProcessor
+     *  to return true.
+     */
+    private List<String> getPipelinesWithGeoIpProcessors() {
+        String simpleGeoIpProcessor = """
+            {
+              "processors":[
+                {
+                  "geoip":{
+                    "field":"provider"
+                  }
+                }
+              ]
+            }
+            """;
+        String onFailureWithGeoIpProcessor = """
+            {
+              "processors":[
+                {
+                  "rename":{
+                    "field":"provider",
+                    "target_field":"cloud.provider",
+                    "on_failure":[
+                      {
+                        "geoip":{
+                          "field":"error.message"
+                        }
+                      }
+                    ]
+                  }
+                }
+              ]
+            }
+            """;
+        String foreachWithGeoIpProcessor = """
+            {
+              "processors":[
+                {
+                  "foreach":{
+                    "field":"values",
+                    "processor":
+                      {
+                        "geoip":{
+                          "field":"someField"
+                        }
+                      }
+                  }
+                }
+              ]
+            }
+            """;
+        String nestedForeachWithGeoIpProcessor = """
+            {
+              "processors":[
+                {
+                  "foreach":{
+                    "field":"values",
+                    "processor":
+                      {
+                        "foreach":{
+                          "field":"someField",
+                          "processor":
+                            {
+                              "geoip":{
+                                "field":"someField"
+                              }
+                            }
+                        }
+                      }
+                  }
+                }
+              ]
+            }
+            """;
+        String nestedForeachWithOnFailureWithGeoIpProcessor = """
+            {
+              "processors":[
+                {
+                  "foreach":{
+                    "field":"values",
+                    "processor":
+                      {
+                        "foreach":{
+                          "field":"someField",
+                          "processor":
+                            {
+                              "rename":{
+                                "field":"provider",
+                                "target_field":"cloud.provider",
+                                "on_failure":[
+                                  {
+                                    "geoip":{
+                                      "field":"error.message"
+                                    }
+                                  }
+                                ]
+                              }
+                            }
+                        }
+                      }
+                  }
+                }
+              ]
+            }
+            """;
+        String onFailureWithForeachWithGeoIp = """
+            {
+              "processors":[
+                {
+                  "rename":{
+                    "field":"provider",
+                    "target_field":"cloud.provider",
+                    "on_failure":[
+                      {
+                        "foreach":{
+                          "field":"values",
+                          "processor":
+                            {
+                              "geoip":{
+                                "field":"someField"
+                              }
+                            }
+                        }
+                      }
+                    ]
+                  }
+                }
+              ]
+            }
+            """;
+        return List.of(
+            simpleGeoIpProcessor,
+            onFailureWithGeoIpProcessor,
+            foreachWithGeoIpProcessor,
+            nestedForeachWithGeoIpProcessor,
+            nestedForeachWithOnFailureWithGeoIpProcessor,
+            onFailureWithForeachWithGeoIp
+        );
+    }
+
+    /*
+     * This method returns an assorted list of pipelines that _do not_ have geoip processors -- ones that ought to cause
+     * hasAtLeastOneGeoipProcessor to return false.
+     */
+    private List<String> getPipelinesWithoutGeoIpProcessors() {
+        String empty = """
+            {
+            }
+            """;
+        String noProcessors = """
+            {
+              "processors":[
+              ]
+            }
+            """;
+        String onFailureWithForeachWithSet = """
+            {
+              "processors":[
+                {
+                  "rename":{
+                    "field":"provider",
+                    "target_field":"cloud.provider",
+                    "on_failure":[
+                      {
+                        "foreach":{
+                          "field":"values",
+                          "processor":
+                            {
+                              "set":{
+                                "field":"someField"
+                              }
+                            }
+                        }
+                      }
+                    ]
+                  }
+                }
+              ]
+            }
+            """;
+        return List.of(empty, noProcessors, onFailureWithForeachWithSet);
+    }
+}
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java
index 16088deb86b3..9f3334a07d8f 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java
@@ -53,6 +53,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiConsumer;
 
@@ -81,7 +82,12 @@ public void setup() {
         when(clusterService.getClusterSettings()).thenReturn(
             new ClusterSettings(
                 Settings.EMPTY,
-                Set.of(GeoIpDownloader.ENDPOINT_SETTING, GeoIpDownloader.POLL_INTERVAL_SETTING, GeoIpDownloaderTaskExecutor.ENABLED_SETTING)
+                Set.of(
+                    GeoIpDownloaderTaskExecutor.EAGER_DOWNLOAD_SETTING,
+                    GeoIpDownloader.ENDPOINT_SETTING,
+                    GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING,
+                    GeoIpDownloaderTaskExecutor.ENABLED_SETTING
+                )
             )
         );
         ClusterState state = createClusterState(new PersistentTasksCustomMetadata(1L, Map.of()));
@@ -98,7 +104,10 @@ public void setup() {
             "",
             "",
             EMPTY_TASK_ID,
-            Collections.emptyMap()
+            Collections.emptyMap(),
+            () -> GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getDefault(Settings.EMPTY),
+            () -> GeoIpDownloaderTaskExecutor.EAGER_DOWNLOAD_SETTING.getDefault(Settings.EMPTY),
+            () -> true
         );
     }
 
@@ -252,7 +261,10 @@ public void testProcessDatabaseNew() throws IOException {
             "",
             "",
             EMPTY_TASK_ID,
-            Collections.emptyMap()
+            Collections.emptyMap(),
+            () -> GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getDefault(Settings.EMPTY),
+            () -> GeoIpDownloaderTaskExecutor.EAGER_DOWNLOAD_SETTING.getDefault(Settings.EMPTY),
+            () -> true
         ) {
             @Override
             void updateTaskState() {
@@ -298,7 +310,10 @@ public void testProcessDatabaseUpdate() throws IOException {
             "",
             "",
             EMPTY_TASK_ID,
-            Collections.emptyMap()
+            Collections.emptyMap(),
+            () -> GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getDefault(Settings.EMPTY),
+            () -> GeoIpDownloaderTaskExecutor.EAGER_DOWNLOAD_SETTING.getDefault(Settings.EMPTY),
+            () -> true
         ) {
             @Override
             void updateTaskState() {
@@ -346,7 +361,10 @@ public void testProcessDatabaseSame() throws IOException {
             "",
             "",
             EMPTY_TASK_ID,
-            Collections.emptyMap()
+            Collections.emptyMap(),
+            () -> GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getDefault(Settings.EMPTY),
+            () -> GeoIpDownloaderTaskExecutor.EAGER_DOWNLOAD_SETTING.getDefault(Settings.EMPTY),
+            () -> true
         ) {
             @Override
             void updateTaskState() {
@@ -387,7 +405,10 @@ public void testUpdateTaskState() {
             "",
             "",
             EMPTY_TASK_ID,
-            Collections.emptyMap()
+            Collections.emptyMap(),
+            () -> GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getDefault(Settings.EMPTY),
+            () -> GeoIpDownloaderTaskExecutor.EAGER_DOWNLOAD_SETTING.getDefault(Settings.EMPTY),
+            () -> true
         ) {
             @Override
             public void updatePersistentTaskState(PersistentTaskState state, ActionListener<PersistentTask<?>> listener) {
@@ -414,7 +435,10 @@ public void testUpdateTaskStateError() {
             "",
             "",
             EMPTY_TASK_ID,
-            Collections.emptyMap()
+            Collections.emptyMap(),
+            () -> GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getDefault(Settings.EMPTY),
+            () -> GeoIpDownloaderTaskExecutor.EAGER_DOWNLOAD_SETTING.getDefault(Settings.EMPTY),
+            () -> true
         ) {
             @Override
             public void updatePersistentTaskState(PersistentTaskState state, ActionListener<PersistentTask<?>> listener) {
@@ -440,6 +464,7 @@ public void testUpdateDatabases() throws IOException {
         builder.close();
         when(httpClient.getBytes("a.b?elastic_geoip_service_tos=agree")).thenReturn(baos.toByteArray());
         Iterator<Map<String, Object>> it = maps.iterator();
+        final AtomicBoolean atLeastOneGeoipProcessor = new AtomicBoolean(false);
         geoIpDownloader = new GeoIpDownloader(
             client,
             httpClient,
@@ -451,7 +476,10 @@ public void testUpdateDatabases() throws IOException {
             "",
             "",
             EMPTY_TASK_ID,
-            Collections.emptyMap()
+            Collections.emptyMap(),
+            () -> GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING.getDefault(Settings.EMPTY),
+            () -> GeoIpDownloaderTaskExecutor.EAGER_DOWNLOAD_SETTING.getDefault(Settings.EMPTY),
+            atLeastOneGeoipProcessor::get
         ) {
             @Override
             void processDatabase(Map<String, Object> databaseInfo) {
@@ -459,6 +487,9 @@ void processDatabase(Map<String, Object> databaseInfo) {
             }
         };
         geoIpDownloader.updateDatabases();
+        assertTrue(it.hasNext());
+        atLeastOneGeoipProcessor.set(true);
+        geoIpDownloader.updateDatabases();
         assertFalse(it.hasNext());
     }
 
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
index 809e3b833127..d9ee6cda883e 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
@@ -72,10 +72,10 @@ public void loadDatabaseReaders() throws IOException {
         configDatabases = new ConfigDatabases(geoIpConfigDir, new GeoIpCache(1000));
         copyDatabaseFiles(geoIpConfigDir, configDatabases);
         geoipTmpDir = createTempDir();
-        databaseNodeService = new DatabaseNodeService(geoipTmpDir, client, cache, configDatabases, Runnable::run);
-        databaseNodeService.initialize("nodeId", mock(ResourceWatcherService.class), mock(IngestService.class), mock(ClusterService.class));
         clusterService = mock(ClusterService.class);
         when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE);
+        databaseNodeService = new DatabaseNodeService(geoipTmpDir, client, cache, configDatabases, Runnable::run, clusterService);
+        databaseNodeService.initialize("nodeId", mock(ResourceWatcherService.class), mock(IngestService.class));
     }
 
     @After
@@ -85,7 +85,7 @@ public void closeDatabaseReaders() throws IOException {
     }
 
     public void testBuildDefaults() throws Exception {
-        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService, clusterService);
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
@@ -101,7 +101,7 @@ public void testBuildDefaults() throws Exception {
     }
 
     public void testSetIgnoreMissing() throws Exception {
-        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService, clusterService);
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
@@ -118,7 +118,7 @@ public void testSetIgnoreMissing() throws Exception {
     }
 
     public void testCountryBuildDefaults() throws Exception {
-        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService, clusterService);
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
@@ -136,7 +136,7 @@ public void testCountryBuildDefaults() throws Exception {
     }
 
     public void testAsnBuildDefaults() throws Exception {
-        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService, clusterService);
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
@@ -154,7 +154,7 @@ public void testAsnBuildDefaults() throws Exception {
     }
 
     public void testBuildTargetField() throws Exception {
-        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService, clusterService);
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("target_field", "_field");
@@ -165,7 +165,7 @@ public void testBuildTargetField() throws Exception {
     }
 
     public void testBuildDbFile() throws Exception {
-        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService, clusterService);
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("database_file", "GeoLite2-Country.mmdb");
@@ -178,7 +178,7 @@ public void testBuildDbFile() throws Exception {
     }
 
     public void testBuildWithCountryDbAndAsnFields() throws Exception {
-        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService, clusterService);
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("database_file", "GeoLite2-Country.mmdb");
@@ -198,7 +198,7 @@ public void testBuildWithCountryDbAndAsnFields() throws Exception {
     }
 
     public void testBuildWithAsnDbAndCityFields() throws Exception {
-        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService, clusterService);
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("database_file", "GeoLite2-ASN.mmdb");
@@ -219,7 +219,7 @@ public void testBuildNonExistingDbFile() throws Exception {
             geoipTmpDir.resolve("GeoLite2-City.mmdb")
         );
         databaseNodeService.updateDatabase("GeoLite2-City.mmdb", "md5", geoipTmpDir.resolve("GeoLite2-City.mmdb"));
-        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService, clusterService);
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
@@ -229,7 +229,7 @@ public void testBuildNonExistingDbFile() throws Exception {
     }
 
     public void testBuildBuiltinDatabaseMissing() throws Exception {
-        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService, clusterService);
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
         cleanDatabaseFiles(geoIpConfigDir, configDatabases);
 
         Map<String, Object> config = new HashMap<>();
@@ -240,7 +240,7 @@ public void testBuildBuiltinDatabaseMissing() throws Exception {
     }
 
     public void testBuildFields() throws Exception {
-        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService, clusterService);
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
 
         Set<GeoIpProcessor.Property> properties = EnumSet.noneOf(GeoIpProcessor.Property.class);
         List<String> fieldNames = new ArrayList<>();
@@ -264,7 +264,7 @@ public void testBuildFields() throws Exception {
     }
 
     public void testBuildIllegalFieldOption() throws Exception {
-        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService, clusterService);
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
 
         Map<String, Object> config1 = new HashMap<>();
         config1.put("field", "_field");
@@ -298,8 +298,15 @@ public void testLazyLoading() throws Exception {
         // database readers used at class level are reused between tests. (we want to keep that otherwise running this
         // test will take roughly 4 times more time)
         Client client = mock(Client.class);
-        DatabaseNodeService databaseNodeService = new DatabaseNodeService(createTempDir(), client, cache, configDatabases, Runnable::run);
-        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService, clusterService);
+        DatabaseNodeService databaseNodeService = new DatabaseNodeService(
+            createTempDir(),
+            client,
+            cache,
+            configDatabases,
+            Runnable::run,
+            clusterService
+        );
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
         for (DatabaseReaderLazyLoader lazyLoader : configDatabases.getConfigDatabases().values()) {
             assertNull(lazyLoader.databaseReader.get());
         }
@@ -313,10 +320,10 @@ public void testLazyLoading() throws Exception {
         final GeoIpProcessor city = (GeoIpProcessor) factory.create(null, "_tag", null, config);
 
         // these are lazy loaded until first use so we expect null here
-        assertNull(databaseNodeService.getDatabase("GeoLite2-City.mmdb").databaseReader.get());
+        assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-City.mmdb").databaseReader.get());
         city.execute(document);
         // the first ingest should trigger a database load
-        assertNotNull(databaseNodeService.getDatabase("GeoLite2-City.mmdb").databaseReader.get());
+        assertNotNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-City.mmdb").databaseReader.get());
 
         config = new HashMap<>();
         config.put("field", "_field");
@@ -324,10 +331,10 @@ public void testLazyLoading() throws Exception {
         final GeoIpProcessor country = (GeoIpProcessor) factory.create(null, "_tag", null, config);
 
         // these are lazy loaded until first use so we expect null here
-        assertNull(databaseNodeService.getDatabase("GeoLite2-Country.mmdb").databaseReader.get());
+        assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-Country.mmdb").databaseReader.get());
         country.execute(document);
         // the first ingest should trigger a database load
-        assertNotNull(databaseNodeService.getDatabase("GeoLite2-Country.mmdb").databaseReader.get());
+        assertNotNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-Country.mmdb").databaseReader.get());
 
         config = new HashMap<>();
         config.put("field", "_field");
@@ -335,10 +342,10 @@ public void testLazyLoading() throws Exception {
         final GeoIpProcessor asn = (GeoIpProcessor) factory.create(null, "_tag", null, config);
 
         // these are lazy loaded until first use so we expect null here
-        assertNull(databaseNodeService.getDatabase("GeoLite2-ASN.mmdb").databaseReader.get());
+        assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-ASN.mmdb").databaseReader.get());
         asn.execute(document);
         // the first ingest should trigger a database load
-        assertNotNull(databaseNodeService.getDatabase("GeoLite2-ASN.mmdb").databaseReader.get());
+        assertNotNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-ASN.mmdb").databaseReader.get());
     }
 
     @SuppressWarnings("HiddenField")
@@ -360,9 +367,16 @@ public void testLoadingCustomDatabase() throws IOException {
         ResourceWatcherService resourceWatcherService = new ResourceWatcherService(Settings.EMPTY, threadPool);
         Client client = mock(Client.class);
         GeoIpCache cache = new GeoIpCache(1000);
-        DatabaseNodeService databaseNodeService = new DatabaseNodeService(createTempDir(), client, cache, configDatabases, Runnable::run);
-        databaseNodeService.initialize("nodeId", resourceWatcherService, mock(IngestService.class), mock(ClusterService.class));
-        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService, clusterService);
+        DatabaseNodeService databaseNodeService = new DatabaseNodeService(
+            createTempDir(),
+            client,
+            cache,
+            configDatabases,
+            Runnable::run,
+            clusterService
+        );
+        databaseNodeService.initialize("nodeId", resourceWatcherService, mock(IngestService.class));
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
         for (DatabaseReaderLazyLoader lazyLoader : configDatabases.getConfigDatabases().values()) {
             assertNull(lazyLoader.databaseReader.get());
         }
@@ -376,16 +390,16 @@ public void testLoadingCustomDatabase() throws IOException {
         final GeoIpProcessor city = (GeoIpProcessor) factory.create(null, "_tag", null, config);
 
         // these are lazy loaded until first use so we expect null here
-        assertNull(databaseNodeService.getDatabase("GeoIP2-City.mmdb").databaseReader.get());
+        assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoIP2-City.mmdb").databaseReader.get());
         city.execute(document);
         // the first ingest should trigger a database load
-        assertNotNull(databaseNodeService.getDatabase("GeoIP2-City.mmdb").databaseReader.get());
+        assertNotNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoIP2-City.mmdb").databaseReader.get());
         resourceWatcherService.close();
         threadPool.shutdown();
     }
 
     public void testFallbackUsingDefaultDatabases() throws Exception {
-        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService, clusterService);
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
         Map<String, Object> config = new HashMap<>();
         config.put("field", "source_field");
         config.put("fallback_to_default_databases", randomBoolean());
@@ -402,7 +416,7 @@ public void testDefaultDatabaseWithTaskPresent() throws Exception {
             .metadata(Metadata.builder().putCustom(PersistentTasksCustomMetadata.TYPE, tasks))
             .build();
         when(clusterService.state()).thenReturn(clusterState);
-        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService, clusterService);
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
@@ -414,7 +428,7 @@ public void testDefaultDatabaseWithTaskPresent() throws Exception {
     }
 
     public void testUpdateDatabaseWhileIngesting() throws Exception {
-        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService, clusterService);
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
         Map<String, Object> config = new HashMap<>();
         config.put("field", "source_field");
         GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config);
@@ -453,7 +467,7 @@ public void testUpdateDatabaseWhileIngesting() throws Exception {
     }
 
     public void testDatabaseNotReadyYet() throws Exception {
-        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService, clusterService);
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseNodeService);
         cleanDatabaseFiles(geoIpConfigDir, configDatabases);
 
         {
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
index bbefc307e800..80c72238b967 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
@@ -24,6 +24,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Supplier;
 
 import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
@@ -427,6 +428,35 @@ public void testListNoMatches() throws Exception {
         assertFalse(ingestDocument.hasField("target_field"));
     }
 
+    public void testListDatabaseReferenceCounting() throws Exception {
+        AtomicBoolean closeCheck = new AtomicBoolean(false);
+        var loader = loader("/GeoLite2-City.mmdb", closeCheck);
+        GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", () -> {
+            loader.preLookup();
+            return loader;
+        }, () -> true, "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false, false, "filename");
+
+        Map<String, Object> document = new HashMap<>();
+        document.put("source_field", Arrays.asList("8.8.8.8", "82.171.64.0"));
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        processor.execute(ingestDocument);
+
+        @SuppressWarnings("unchecked")
+        List<Map<String, Object>> geoData = (List<Map<String, Object>>) ingestDocument.getSourceAndMetadata().get("target_field");
+
+        Map<String, Object> location = new HashMap<>();
+        location.put("lat", 37.751d);
+        location.put("lon", -97.822d);
+        assertThat(geoData.get(0).get("location"), equalTo(location));
+
+        assertThat(geoData.get(1).get("city_name"), equalTo("Hoensbroek"));
+
+        // Check the loader's reference count and attempt to close
+        assertThat(loader.current(), equalTo(0));
+        loader.close();
+        assertTrue(closeCheck.get());
+    }
+
     public void testListFirstOnly() throws Exception {
         GeoIpProcessor processor = new GeoIpProcessor(
             randomAlphaOfLength(10),
@@ -545,12 +575,17 @@ public void testNoDatabase_ignoreMissing() throws Exception {
         assertIngestDocument(originalIngestDocument, ingestDocument);
     }
 
-    private CheckedSupplier<DatabaseReaderLazyLoader, IOException> loader(final String path) {
+    private CheckedSupplier<GeoIpDatabase, IOException> loader(final String path) {
+        var loader = loader(path, null);
+        return () -> loader;
+    }
+
+    private DatabaseReaderLazyLoader loader(final String path, final AtomicBoolean closed) {
         final Supplier<InputStream> databaseInputStreamSupplier = () -> GeoIpProcessor.class.getResourceAsStream(path);
         final CheckedSupplier<DatabaseReader, IOException> loader = () -> new DatabaseReader.Builder(databaseInputStreamSupplier.get())
             .build();
         final GeoIpCache cache = new GeoIpCache(1000);
-        DatabaseReaderLazyLoader lazyLoader = new DatabaseReaderLazyLoader(cache, PathUtils.get(path), null, loader) {
+        return new DatabaseReaderLazyLoader(cache, PathUtils.get(path), null, loader) {
 
             @Override
             long databaseFileSize() throws IOException {
@@ -571,8 +606,14 @@ InputStream databaseInputStream() throws IOException {
                 return databaseInputStreamSupplier.get();
             }
 
+            @Override
+            protected void doClose() throws IOException {
+                if (closed != null) {
+                    closed.set(true);
+                }
+                super.doClose();
+            }
         };
-        return () -> lazyLoader;
     }
 
 }
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java
index 137a5160b439..80e4836605e2 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java
@@ -41,4 +41,9 @@ protected GeoIpTaskState createTestInstance() {
         }
         return state;
     }
+
+    @Override
+    protected GeoIpTaskState mutateInstance(GeoIpTaskState instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsActionNodeResponseSerializingTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsActionNodeResponseSerializingTests.java
index 9e49d33089ea..6f3d84c2ce30 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsActionNodeResponseSerializingTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsActionNodeResponseSerializingTests.java
@@ -28,6 +28,11 @@ protected GeoIpDownloaderStatsAction.NodeResponse createTestInstance() {
         return createRandomInstance();
     }
 
+    @Override
+    protected GeoIpDownloaderStatsAction.NodeResponse mutateInstance(GeoIpDownloaderStatsAction.NodeResponse instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     static GeoIpDownloaderStatsAction.NodeResponse createRandomInstance() {
         DiscoveryNode node = new DiscoveryNode("id", buildNewFakeTransportAddress(), Version.CURRENT);
         Set<String> databases = Set.copyOf(randomList(10, () -> randomAlphaOfLengthBetween(5, 10)));
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsActionResponseSerializingTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsActionResponseSerializingTests.java
index 756b012b996f..b9fe1ee976d6 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsActionResponseSerializingTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsActionResponseSerializingTests.java
@@ -31,4 +31,9 @@ protected GeoIpDownloaderStatsAction.Response createTestInstance() {
         );
         return new GeoIpDownloaderStatsAction.Response(ClusterName.DEFAULT, nodeResponses, Collections.emptyList());
     }
+
+    @Override
+    protected GeoIpDownloaderStatsAction.Response mutateInstance(GeoIpDownloaderStatsAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsSerializingTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsSerializingTests.java
index cc6fd1ca903a..68b1ac4b28ff 100644
--- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsSerializingTests.java
+++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsSerializingTests.java
@@ -31,6 +31,11 @@ protected GeoIpDownloaderStats createTestInstance() {
         return createRandomInstance();
     }
 
+    @Override
+    protected GeoIpDownloaderStats mutateInstance(GeoIpDownloaderStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     static GeoIpDownloaderStats createRandomInstance() {
         GeoIpDownloaderStats stats = GeoIpDownloaderStats.EMPTY.databasesCount(randomInt(1000));
         int successes = randomInt(20);
diff --git a/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java b/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java
index 5b40f4a6ada4..8584229ec171 100644
--- a/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java
+++ b/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java
@@ -11,11 +11,17 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.entity.ContentType;
 import org.elasticsearch.client.Request;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.json.JsonXContent;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -36,6 +42,7 @@ public static Iterable<Object[]> parameters() throws Exception {
 
     @Before
     public void waitForDatabases() throws Exception {
+        putGeoipPipeline();
         assertBusy(() -> {
             Request request = new Request("GET", "/_ingest/geoip/stats");
             Map<String, Object> response = entityAsMap(client().performRequest(request));
@@ -53,4 +60,37 @@ public void waitForDatabases() throws Exception {
         });
     }
 
+    /**
+     * This creates a pipeline with a geoip processor so that the GeoipDownloader will download its databases.
+     * @throws IOException
+     */
+    private void putGeoipPipeline() throws IOException {
+        final BytesReference bytes;
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            builder.startObject();
+            {
+                builder.startArray("processors");
+                {
+                    builder.startObject();
+                    {
+                        builder.startObject("geoip");
+                        {
+                            builder.field("field", "ip");
+                            builder.field("target_field", "ip-city");
+                            builder.field("database_file", "GeoLite2-City.mmdb");
+                        }
+                        builder.endObject();
+                    }
+                    builder.endObject();
+                }
+                builder.endArray();
+            }
+            builder.endObject();
+            bytes = BytesReference.bytes(builder);
+        }
+        Request putPipelineRequest = new Request("PUT", "/_ingest/pipeline/pipeline-with-geoip");
+        putPipelineRequest.setEntity(new ByteArrayEntity(bytes.array(), ContentType.APPLICATION_JSON));
+        client().performRequest(putPipelineRequest);
+    }
+
 }
diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml
index b09dac97eba2..84801cfb9ada 100644
--- a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml
+++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml
@@ -309,3 +309,39 @@
   - match: { _source.geoip.asn: 29518 }
   - match: { _source.geoip.organization_name: "Bredband2 AB" }
   - match: { _source.geoip.network: "89.160.0.0/17" }
+
+---
+"Test simulate with Geoip Processor":
+- do:
+    ingest.put_pipeline:
+      id: "pipeline1"
+      body:  >
+        {
+          "processors": [
+          {
+            "geoip": {
+              "field": "source.ip",
+              "target_field": "source.geo"
+            }
+          }
+          ]
+        }
+- match: { acknowledged: true }
+
+- do:
+    ingest.simulate:
+      id: "pipeline1"
+      body:  >
+        {
+          "docs": [
+            {
+              "_source": {
+                "source": {
+                  "ip": "89.160.20.128"
+                }
+              }
+            }
+          ]
+        }
+- length: { docs: 1 }
+- match: { docs.0.doc._source.source.geo.city_name: "Linköping" }
diff --git a/modules/ingest-user-agent/build.gradle b/modules/ingest-user-agent/build.gradle
index bdf7fae42d84..5d3ae968b787 100644
--- a/modules/ingest-user-agent/build.gradle
+++ b/modules/ingest-user-agent/build.gradle
@@ -5,8 +5,8 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
   description 'Ingest processor that extracts information from a user agent'
diff --git a/modules/kibana/build.gradle b/modules/kibana/build.gradle
index d334fa380d7b..e57e16409495 100644
--- a/modules/kibana/build.gradle
+++ b/modules/kibana/build.gradle
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 esplugin {
   description 'Plugin exposing APIs for Kibana system indices'
diff --git a/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java b/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java
index 08b42cc49b90..bb1e32671a84 100644
--- a/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java
+++ b/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xcontent.json.JsonXContent;
 
@@ -69,7 +70,7 @@ public void testAliases() throws IOException {
 
     public void testBulkToKibanaIndex() throws IOException {
         Request request = request("POST", "/_bulk");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             { "index" : { "_index" : "%s", "_id" : "1" } }
             { "foo" : "bar" }
             """, indexName));
@@ -79,7 +80,7 @@ public void testBulkToKibanaIndex() throws IOException {
 
     public void testRefresh() throws IOException {
         Request request = request("POST", "/_bulk");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             { "index" : { "_index" : "%s", "_id" : "1" } }
             { "foo" : "bar" }
             """, indexName));
@@ -100,7 +101,7 @@ public void testRefresh() throws IOException {
 
     public void testGetFromKibanaIndex() throws IOException {
         Request request = request("POST", "/_bulk");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             { "index" : { "_index" : "%s", "_id" : "1" } }
             { "foo" : "bar" }
             """, indexName));
@@ -119,7 +120,7 @@ public void testGetFromKibanaIndex() throws IOException {
 
     public void testMultiGetFromKibanaIndex() throws IOException {
         Request request = request("POST", "/_bulk");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             { "index" : { "_index" : "%s", "_id" : "1" } }
             { "foo" : "bar" }
             { "index" : { "_index" : "%s", "_id" : "2" } }
@@ -131,7 +132,7 @@ public void testMultiGetFromKibanaIndex() throws IOException {
         assertThat(response.getStatusLine().getStatusCode(), is(200));
 
         Request getRequest = request("GET", "/_mget");
-        getRequest.setJsonEntity(formatted("""
+        getRequest.setJsonEntity(Strings.format("""
             {
               "docs": [
                 {
@@ -155,7 +156,7 @@ public void testMultiGetFromKibanaIndex() throws IOException {
 
     public void testSearchFromKibanaIndex() throws IOException {
         Request request = request("POST", "/_bulk");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             { "index" : { "_index" : "%s", "_id" : "1" } }
             { "foo" : "bar" }
             { "index" : { "_index" : "%s", "_id" : "2" } }
@@ -181,7 +182,7 @@ public void testSearchFromKibanaIndex() throws IOException {
 
     public void testDeleteFromKibanaIndex() throws IOException {
         Request request = request("POST", "/_bulk");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             { "index" : { "_index" : "%s", "_id" : "1" } }
             { "foo" : "bar" }
             { "index" : { "_index" : "%s", "_id" : "2" } }
@@ -199,7 +200,7 @@ public void testDeleteFromKibanaIndex() throws IOException {
 
     public void testDeleteByQueryFromKibanaIndex() throws IOException {
         Request request = request("POST", "/_bulk");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             { "index" : { "_index" : "%s", "_id" : "1" } }
             { "foo" : "bar" }
             { "index" : { "_index" : "%s", "_id" : "2" } }
@@ -289,7 +290,7 @@ public void testIndexingAndUpdatingDocs() throws IOException {
 
     public void testScrollingDocs() throws IOException {
         Request request = request("POST", "/_bulk");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             { "index" : { "_index" : "%s", "_id" : "1" } }
             { "foo" : "bar" }
             { "index" : { "_index" : "%s", "_id" : "2" } }
diff --git a/modules/lang-expression/build.gradle b/modules/lang-expression/build.gradle
index 1581de127427..66c9dcf6a236 100644
--- a/modules/lang-expression/build.gradle
+++ b/modules/lang-expression/build.gradle
@@ -5,8 +5,8 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
@@ -17,7 +17,7 @@ esplugin {
 dependencies {
   api "org.apache.lucene:lucene-expressions:${versions.lucene}"
   runtimeOnly "org.apache.lucene:lucene-codecs:${versions.lucene}"
-  runtimeOnly 'org.antlr:antlr4-runtime:4.5.1-1'
+  runtimeOnly "org.antlr:antlr4-runtime:${versions.antlr4}"
   runtimeOnly 'org.ow2.asm:asm:7.2'
   runtimeOnly 'org.ow2.asm:asm-commons:7.2'
   runtimeOnly 'org.ow2.asm:asm-tree:7.2'
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java
index 6d843e7b8065..e573d32f7b6c 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java
@@ -15,8 +15,9 @@
 import org.elasticsearch.script.FieldScript;
 import org.elasticsearch.script.ScriptException;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
 import java.text.ParseException;
@@ -53,7 +54,7 @@ public void setUp() throws Exception {
         lookup = new SearchLookup(
             field -> field.equals("field") ? fieldType : null,
             (ignored, _lookup, fdt) -> fieldData,
-            new SourceLookup.ReaderSourceProvider()
+            (ctx, doc) -> Source.empty(XContentType.JSON)
         );
     }
 
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java
index 30e94ed37947..eedd80bf6d29 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java
@@ -17,8 +17,9 @@
 import org.elasticsearch.script.NumberSortScript;
 import org.elasticsearch.script.ScriptException;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
 import java.text.ParseException;
@@ -54,7 +55,7 @@ public void setUp() throws Exception {
         lookup = new SearchLookup(
             field -> field.equals("field") ? fieldType : null,
             (ignored, _lookup, fdt) -> fieldData,
-            new SourceLookup.ReaderSourceProvider()
+            (ctx, doc) -> Source.empty(XContentType.JSON)
         );
     }
 
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java
index ad547c656f4b..d88b2d67e663 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java
@@ -16,8 +16,9 @@
 import org.elasticsearch.script.ScriptException;
 import org.elasticsearch.script.TermsSetQueryScript;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
 import java.text.ParseException;
@@ -53,7 +54,7 @@ public void setUp() throws Exception {
         lookup = new SearchLookup(
             field -> field.equals("field") ? fieldType : null,
             (ignored, _lookup, fdt) -> fieldData,
-            new SourceLookup.ReaderSourceProvider()
+            (ctx, doc) -> Source.empty(XContentType.JSON)
         );
     }
 
diff --git a/modules/lang-mustache/build.gradle b/modules/lang-mustache/build.gradle
index 7536e2f76abf..c36275699e21 100644
--- a/modules/lang-mustache/build.gradle
+++ b/modules/lang-mustache/build.gradle
@@ -5,9 +5,9 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java
index 37abffed2fed..6c432c7306d3 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java
@@ -14,7 +14,9 @@
 import com.github.mustachejava.Mustache;
 import com.github.mustachejava.MustacheException;
 import com.github.mustachejava.MustacheVisitor;
+import com.github.mustachejava.SafeMustacheFactory;
 import com.github.mustachejava.TemplateContext;
+import com.github.mustachejava.TemplateFunction;
 import com.github.mustachejava.codes.DefaultMustache;
 import com.github.mustachejava.codes.IterableCode;
 import com.github.mustachejava.codes.WriteCode;
@@ -34,12 +36,11 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.StringJoiner;
-import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-public class CustomMustacheFactory extends DefaultMustacheFactory {
+public class CustomMustacheFactory extends SafeMustacheFactory {
     static final String V7_JSON_MEDIA_TYPE_WITH_CHARSET = "application/json; charset=UTF-8";
     static final String JSON_MEDIA_TYPE_WITH_CHARSET = "application/json;charset=utf-8";
     static final String JSON_MEDIA_TYPE = "application/json";
@@ -64,7 +65,7 @@ public class CustomMustacheFactory extends DefaultMustacheFactory {
     private final Encoder encoder;
 
     public CustomMustacheFactory(String mediaType) {
-        super();
+        super(Collections.emptySet(), ".");
         setObjectHandler(new CustomReflectionObjectHandler());
         this.encoder = createEncoder(mediaType);
     }
@@ -145,7 +146,7 @@ protected void tag(Writer writer, String tag) throws IOException {
             writer.write(tc.endChars());
         }
 
-        protected abstract Function<String, String> createFunction(Object resolved);
+        protected abstract TemplateFunction createFunction(Object resolved);
 
         /**
          * At compile time, this function extracts the name of the variable:
@@ -188,7 +189,7 @@ static class ToJsonCode extends CustomCode {
 
         @Override
         @SuppressWarnings("unchecked")
-        protected Function<String, String> createFunction(Object resolved) {
+        protected TemplateFunction createFunction(Object resolved) {
             return s -> {
                 if (resolved == null) {
                     return null;
@@ -238,7 +239,7 @@ static class JoinerCode extends CustomCode {
         }
 
         @Override
-        protected Function<String, String> createFunction(Object resolved) {
+        protected TemplateFunction createFunction(Object resolved) {
             return s -> {
                 if (s == null) {
                     return null;
@@ -260,7 +261,7 @@ static boolean match(String variable) {
 
     static class CustomJoinerCode extends JoinerCode {
 
-        private static final Pattern PATTERN = Pattern.compile("^(?:" + CODE + " delimiter='(.*)')$");
+        private static final Pattern PATTERN = Pattern.compile("^" + CODE + " delimiter='(.*)'$");
 
         CustomJoinerCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String variable) {
             super(tc, df, mustache, extractDelimiter(variable));
@@ -357,7 +358,7 @@ static class UrlEncoder implements Encoder {
 
         @Override
         public void encode(String s, Writer writer) throws IOException {
-            writer.write(URLEncoder.encode(s, StandardCharsets.UTF_8.name()));
+            writer.write(URLEncoder.encode(s, StandardCharsets.UTF_8));
         }
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java
index d3ec96f68af5..fd2ab43f348e 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java
@@ -10,10 +10,10 @@
 
 import com.github.mustachejava.reflect.ReflectionObjectHandler;
 
-import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.iterable.Iterables;
 
+import java.lang.reflect.AccessibleObject;
 import java.lang.reflect.Array;
 import java.util.AbstractMap;
 import java.util.Collection;
@@ -40,6 +40,24 @@ public Object coerce(Object object) {
         }
     }
 
+    @Override
+    @SuppressWarnings("rawtypes")
+    protected AccessibleObject findMember(Class sClass, String name) {
+        /*
+         * overriding findMember from BaseObjectHandler (our superclass's superclass) to always return null.
+         *
+         * if you trace findMember there, you'll see that it always either returns null or invokes the getMethod
+         * or getField methods of that class. the last thing that getMethod and getField do is call 'setAccessible'
+         * but we don't have java.lang.reflect.ReflectPermission/suppressAccessChecks so that will always throw an
+         * exception.
+         *
+         * that is, with the permissions we're running with, it would always return null ('not found!') or throw
+         * an exception ('found, but you cannot do this!') -- so by overriding to null we're effectively saying
+         * "you will never find success going down this path, so don't bother trying"
+         */
+        return null;
+    }
+
     static final class ArrayMap extends AbstractMap<Object, Object> implements Iterable<Object> {
 
         private final Object array;
@@ -144,10 +162,4 @@ public Iterator<Object> iterator() {
             return col.iterator();
         }
     }
-
-    @Override
-    public String stringify(Object object) {
-        CollectionUtils.ensureNoSelfReferences(object, "CustomReflectionObjectHandler stringify");
-        return super.stringify(object);
-    }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponse.java
index 5aa962973b6f..f42648015535 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponse.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponse.java
@@ -10,7 +10,7 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.common.Strings;
@@ -102,7 +102,7 @@ public String toString() {
     MultiSearchTemplateResponse(StreamInput in) throws IOException {
         super(in);
         items = in.readArray(Item::new, Item[]::new);
-        if (in.getVersion().onOrAfter(Version.V_7_0_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_0_0)) {
             tookInMillis = in.readVLong();
         } else {
             tookInMillis = -1L;
@@ -136,7 +136,7 @@ public TimeValue getTook() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeArray(items);
-        if (out.getVersion().onOrAfter(Version.V_7_0_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_0_0)) {
             out.writeVLong(tookInMillis);
         }
     }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java
index 8abd0d65d58d..243487981c5f 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java
@@ -13,7 +13,6 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.script.GeneralScriptException;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
@@ -24,8 +23,6 @@
 import java.io.Reader;
 import java.io.StringReader;
 import java.io.StringWriter;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
 import java.util.Collections;
 import java.util.Map;
 import java.util.Set;
@@ -108,12 +105,7 @@ private class MustacheExecutableScript extends TemplateScript {
         public String execute() {
             final StringWriter writer = new StringWriter();
             try {
-                // crazy reflection here
-                SpecialPermission.check();
-                AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
-                    template.execute(writer, params);
-                    return null;
-                });
+                template.execute(writer, params);
             } catch (Exception e) {
                 logger.error(() -> format("Error running %s", template), e);
                 throw new GeneralScriptException("Error running " + template, e);
diff --git a/modules/lang-mustache/src/main/plugin-metadata/plugin-security.policy b/modules/lang-mustache/src/main/plugin-metadata/plugin-security.policy
deleted file mode 100644
index 2fb8547e7b3f..000000000000
--- a/modules/lang-mustache/src/main/plugin-metadata/plugin-security.policy
+++ /dev/null
@@ -1,12 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-grant {
-  // needed to do crazy reflection
-  permission java.lang.RuntimePermission "accessDeclaredMembers";
-};
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java
index ea741e684c07..4845279c4dd8 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java
@@ -15,8 +15,6 @@
 
 import java.util.Map;
 
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.singletonMap;
 import static org.elasticsearch.script.mustache.CustomMustacheFactory.JSON_MEDIA_TYPE;
 import static org.elasticsearch.script.mustache.CustomMustacheFactory.PLAIN_TEXT_MEDIA_TYPE;
 import static org.elasticsearch.script.mustache.CustomMustacheFactory.X_WWW_FORM_URLENCODED_MEDIA_TYPE;
@@ -67,31 +65,31 @@ public void testCreateEncoder() {
 
     public void testJsonEscapeEncoder() {
         final ScriptEngine engine = new MustacheScriptEngine();
-        final Map<String, String> params = randomBoolean() ? singletonMap(Script.CONTENT_TYPE_OPTION, JSON_MEDIA_TYPE) : emptyMap();
+        final Map<String, String> params = randomBoolean() ? Map.of(Script.CONTENT_TYPE_OPTION, JSON_MEDIA_TYPE) : Map.of();
 
         TemplateScript.Factory compiled = engine.compile(null, "{\"field\": \"{{value}}\"}", TemplateScript.CONTEXT, params);
 
-        TemplateScript executable = compiled.newInstance(singletonMap("value", "a \"value\""));
+        TemplateScript executable = compiled.newInstance(Map.of("value", "a \"value\""));
         assertThat(executable.execute(), equalTo("{\"field\": \"a \\\"value\\\"\"}"));
     }
 
     public void testDefaultEncoder() {
         final ScriptEngine engine = new MustacheScriptEngine();
-        final Map<String, String> params = singletonMap(Script.CONTENT_TYPE_OPTION, PLAIN_TEXT_MEDIA_TYPE);
+        final Map<String, String> params = Map.of(Script.CONTENT_TYPE_OPTION, PLAIN_TEXT_MEDIA_TYPE);
 
         TemplateScript.Factory compiled = engine.compile(null, "{\"field\": \"{{value}}\"}", TemplateScript.CONTEXT, params);
 
-        TemplateScript executable = compiled.newInstance(singletonMap("value", "a \"value\""));
+        TemplateScript executable = compiled.newInstance(Map.of("value", "a \"value\""));
         assertThat(executable.execute(), equalTo("{\"field\": \"a \"value\"\"}"));
     }
 
     public void testUrlEncoder() {
         final ScriptEngine engine = new MustacheScriptEngine();
-        final Map<String, String> params = singletonMap(Script.CONTENT_TYPE_OPTION, X_WWW_FORM_URLENCODED_MEDIA_TYPE);
+        final Map<String, String> params = Map.of(Script.CONTENT_TYPE_OPTION, X_WWW_FORM_URLENCODED_MEDIA_TYPE);
 
         TemplateScript.Factory compiled = engine.compile(null, "{\"field\": \"{{value}}\"}", TemplateScript.CONTEXT, params);
 
-        TemplateScript executable = compiled.newInstance(singletonMap("value", "tilde~ AND date:[2016 FROM*]"));
+        TemplateScript executable = compiled.newInstance(Map.of("value", "tilde~ AND date:[2016 FROM*]"));
         assertThat(executable.execute(), equalTo("{\"field\": \"tilde%7E+AND+date%3A%5B2016+FROM*%5D\"}"));
     }
 }
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java
index 2420fed44451..812a281d77c1 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java
@@ -18,8 +18,7 @@
 
 import java.io.IOException;
 import java.io.StringWriter;
-import java.util.Collections;
-import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -38,7 +37,7 @@ public void setup() {
     }
 
     public void testSimpleParameterReplace() {
-        Map<String, String> compileParams = Collections.singletonMap("content_type", "application/json");
+        Map<String, String> compileParams = Map.of("content_type", "application/json");
         {
             String template = """
                 GET _search
@@ -61,8 +60,7 @@ public void testSimpleParameterReplace() {
                     }
                   }
                 }""";
-            Map<String, Object> vars = new HashMap<>();
-            vars.put("boost_val", "0.3");
+            Map<String, Object> vars = Map.of("boost_val", "0.3");
             String o = qe.compile(null, template, TemplateScript.CONTEXT, compileParams).newInstance(vars).execute();
             assertEquals("""
                 GET _search
@@ -108,9 +106,7 @@ public void testSimpleParameterReplace() {
                     }
                   }
                 }""";
-            Map<String, Object> vars = new HashMap<>();
-            vars.put("boost_val", "0.3");
-            vars.put("body_val", "\"quick brown\"");
+            Map<String, Object> vars = Map.of("boost_val", "0.3", "body_val", "\"quick brown\"");
             String o = qe.compile(null, template, TemplateScript.CONTEXT, compileParams).newInstance(vars).execute();
             assertEquals("""
                 GET _search
@@ -141,7 +137,7 @@ public void testSimple() throws IOException {
             {"source":{"match_{{template}}": {}},"params":{"template":"all"}}""";
         XContentParser parser = createParser(JsonXContent.jsonXContent, templateString);
         Script script = Script.parse(parser);
-        TemplateScript.Factory compiled = qe.compile(null, script.getIdOrCode(), TemplateScript.CONTEXT, Collections.emptyMap());
+        TemplateScript.Factory compiled = qe.compile(null, script.getIdOrCode(), TemplateScript.CONTEXT, Map.of());
         TemplateScript TemplateScript = compiled.newInstance(script.getParams());
         assertThat(TemplateScript.execute(), equalTo("{\"match_all\":{}}"));
     }
@@ -157,11 +153,79 @@ public void testParseTemplateAsSingleStringWithConditionalClause() throws IOExce
             }""";
         XContentParser parser = createParser(JsonXContent.jsonXContent, templateString);
         Script script = Script.parse(parser);
-        TemplateScript.Factory compiled = qe.compile(null, script.getIdOrCode(), TemplateScript.CONTEXT, Collections.emptyMap());
+        TemplateScript.Factory compiled = qe.compile(null, script.getIdOrCode(), TemplateScript.CONTEXT, Map.of());
         TemplateScript TemplateScript = compiled.newInstance(script.getParams());
         assertThat(TemplateScript.execute(), equalTo("{ \"match_all\":{} }"));
     }
 
+    private static class TestReflection {
+
+        private final int privateField = 1;
+
+        public final int publicField = 2;
+
+        private int getPrivateMethod() {
+            return 3;
+        }
+
+        public int getPublicMethod() {
+            return 4;
+        }
+
+        @Override
+        public String toString() {
+            return List.of(privateField, publicField, getPrivateMethod(), getPublicMethod()).toString();
+        }
+    }
+
+    /**
+     * BWC test for some odd reflection edge-cases. It's not really expected that customer code would be exercising this,
+     * but maybe it's out there! Who knows!?
+     *
+     * If we change this, we should *know* that we're changing it.
+     */
+    @SuppressWarnings({ "deprecation", "removal" })
+    public void testReflection() {
+        Map<String, Object> vars = Map.of("obj", new TestReflection());
+
+        {
+            // non-reflective access calls toString
+            String templateString = "{{obj}}";
+            String o = qe.compile(null, templateString, TemplateScript.CONTEXT, Map.of()).newInstance(vars).execute();
+            assertThat(o, equalTo("[1, 2, 3, 4]"));
+        }
+        {
+            // accessing a field/method that *doesn't* exist will give an empty result
+            String templateString = "{{obj.missing}}";
+            String o = qe.compile(null, templateString, TemplateScript.CONTEXT, Map.of()).newInstance(vars).execute();
+            assertThat(o, equalTo(""));
+        }
+        {
+            // accessing a private field that does exist will give an empty result
+            String templateString = "{{obj.privateField}}";
+            String o = qe.compile(null, templateString, TemplateScript.CONTEXT, Map.of()).newInstance(vars).execute();
+            assertThat(o, equalTo(""));
+        }
+        {
+            // accessing a private method that does exist will give an empty result
+            String templateString = "{{obj.privateMethod}}";
+            String o = qe.compile(null, templateString, TemplateScript.CONTEXT, Map.of()).newInstance(vars).execute();
+            assertThat(o, equalTo(""));
+        }
+        {
+            // accessing a public field that does exist will give an empty result
+            String templateString = "{{obj.publicField}}";
+            String o = qe.compile(null, templateString, TemplateScript.CONTEXT, Map.of()).newInstance(vars).execute();
+            assertThat(o, equalTo(""));
+        }
+        {
+            // accessing a public method that does exist will give an empty result
+            String templateString = "{{obj.publicMethod}}";
+            String o = qe.compile(null, templateString, TemplateScript.CONTEXT, Map.of()).newInstance(vars).execute();
+            assertThat(o, equalTo(""));
+        }
+    }
+
     public void testEscapeJson() throws IOException {
         {
             StringWriter writer = new StringWriter();
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java
index 2c49fe38c56f..e06e69692893 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.script.ScriptEngine;
 import org.elasticsearch.script.ScriptException;
 import org.elasticsearch.script.TemplateScript;
@@ -23,7 +24,6 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 
@@ -150,7 +150,7 @@ public void testSizeAccessForCollectionsAndArrays() throws Exception {
         data.put("list", randomList);
         Map<String, Object> vars = new HashMap<>();
         vars.put("data", data);
-        String expectedString = String.format(Locale.ROOT, "%s %s", randomArrayValues.length, randomList.size());
+        String expectedString = Strings.format("%s %s", randomArrayValues.length, randomList.size());
         assertThat(factory.newInstance(vars).execute(), equalTo(expectedString));
     }
 
diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle
index 17e43a755bed..2dab15f7ddb4 100644
--- a/modules/lang-painless/build.gradle
+++ b/modules/lang-painless/build.gradle
@@ -9,8 +9,8 @@
 import org.elasticsearch.gradle.testclusters.DefaultTestClustersTask;
 
 apply plugin: 'elasticsearch.validate-rest-spec'
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
     description 'An easy, safe and fast scripting language for Elasticsearch'
@@ -34,7 +34,7 @@ configurations {
 }
 
 dependencies {
-    api 'org.antlr:antlr4-runtime:4.5.3'
+    api "org.antlr:antlr4-runtime:${versions.antlr4}"
     api 'org.ow2.asm:asm-util:7.2'
     api 'org.ow2.asm:asm-tree:7.2'
     api 'org.ow2.asm:asm-commons:7.2'
@@ -174,7 +174,7 @@ configurations {
 }
 
 dependencies {
-    regenerate 'org.antlr:antlr4:4.5.3'
+  regenerate "org.antlr:antlr4:${versions.antlr4}"
 }
 
 String grammarPath = 'src/main/antlr'
@@ -274,7 +274,7 @@ configurations {
 }
 
 dependencies {
-    regenerate 'org.antlr:antlr4:4.5.3'
+  regenerate "org.antlr:antlr4:${versions.antlr4}"
 }
 
 String suggestGrammarPath = 'src/main/antlr'
diff --git a/modules/lang-painless/src/main/java/module-info.java b/modules/lang-painless/src/main/java/module-info.java
index 7196d87e8402..539c2919b81f 100644
--- a/modules/lang-painless/src/main/java/module-info.java
+++ b/modules/lang-painless/src/main/java/module-info.java
@@ -13,7 +13,7 @@
     requires org.elasticsearch.server;
     requires org.elasticsearch.xcontent;
 
-    requires antlr4.runtime;
+    requires org.antlr.antlr4.runtime;
     requires org.apache.lucene.core;
     requires org.objectweb.asm;
     requires org.objectweb.asm.commons;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index a1a4f7ea124e..8b45e7d0edd5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -8,6 +8,8 @@
 
 package org.elasticsearch.painless;
 
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.painless.api.ValueIterator;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
 import org.elasticsearch.painless.lookup.PainlessMethod;
@@ -24,6 +26,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -46,90 +49,6 @@
  */
 public final class Def {
 
-    // TODO: Once Java has a factory for those in java.lang.invoke.MethodHandles, use it:
-
-    /** Helper class for isolating MethodHandles and methods to get the length of arrays
-     * (to emulate a "arraystore" bytecode using MethodHandles).
-     * See: https://bugs.openjdk.java.net/browse/JDK-8156915
-     */
-    @SuppressWarnings("unused") // getArrayLength() methods are are actually used, javac just does not know :)
-    private static final class ArrayLengthHelper {
-        private static final MethodHandles.Lookup PRIVATE_METHOD_HANDLES_LOOKUP = MethodHandles.lookup();
-
-        private static final Map<Class<?>, MethodHandle> ARRAY_TYPE_MH_MAPPING = Collections.unmodifiableMap(
-            Stream.of(
-                boolean[].class,
-                byte[].class,
-                short[].class,
-                int[].class,
-                long[].class,
-                char[].class,
-                float[].class,
-                double[].class,
-                Object[].class
-            ).collect(Collectors.toMap(Function.identity(), type -> {
-                try {
-                    return PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(
-                        PRIVATE_METHOD_HANDLES_LOOKUP.lookupClass(),
-                        "getArrayLength",
-                        MethodType.methodType(int.class, type)
-                    );
-                } catch (ReflectiveOperationException e) {
-                    throw new AssertionError(e);
-                }
-            }))
-        );
-
-        private static final MethodHandle OBJECT_ARRAY_MH = ARRAY_TYPE_MH_MAPPING.get(Object[].class);
-
-        static int getArrayLength(final boolean[] array) {
-            return array.length;
-        }
-
-        static int getArrayLength(final byte[] array) {
-            return array.length;
-        }
-
-        static int getArrayLength(final short[] array) {
-            return array.length;
-        }
-
-        static int getArrayLength(final int[] array) {
-            return array.length;
-        }
-
-        static int getArrayLength(final long[] array) {
-            return array.length;
-        }
-
-        static int getArrayLength(final char[] array) {
-            return array.length;
-        }
-
-        static int getArrayLength(final float[] array) {
-            return array.length;
-        }
-
-        static int getArrayLength(final double[] array) {
-            return array.length;
-        }
-
-        static int getArrayLength(final Object[] array) {
-            return array.length;
-        }
-
-        static MethodHandle arrayLengthGetter(Class<?> arrayType) {
-            if (arrayType.isArray() == false) {
-                throw new IllegalArgumentException("type must be an array");
-            }
-            return (ARRAY_TYPE_MH_MAPPING.containsKey(arrayType))
-                ? ARRAY_TYPE_MH_MAPPING.get(arrayType)
-                : OBJECT_ARRAY_MH.asType(OBJECT_ARRAY_MH.type().changeParameterType(0, arrayType));
-        }
-
-        private ArrayLengthHelper() {}
-    }
-
     /** pointer to Map.get(Object) */
     private static final MethodHandle MAP_GET;
     /** pointer to Map.put(Object,Object) */
@@ -138,26 +57,29 @@ private ArrayLengthHelper() {}
     private static final MethodHandle LIST_GET;
     /** pointer to List.set(int,Object) */
     private static final MethodHandle LIST_SET;
-    /** pointer to Iterable.iterator() */
-    private static final MethodHandle ITERATOR;
+    /** pointer to new ObjectIterator(Iterable.iterator()) */
+    private static final MethodHandle OBJECT_ITERATOR;
     /** pointer to {@link Def#mapIndexNormalize}. */
     private static final MethodHandle MAP_INDEX_NORMALIZE;
     /** pointer to {@link Def#listIndexNormalize}. */
     private static final MethodHandle LIST_INDEX_NORMALIZE;
-    /** factory for arraylength MethodHandle (intrinsic) from Java 9 (pkg-private for tests) */
-    static final MethodHandle JAVA9_ARRAY_LENGTH_MH_FACTORY;
+    /** factory for arraylength MethodHandle (intrinsic) */
+    private static final MethodHandle ARRAY_LENGTH;
 
     public static final Map<Class<?>, MethodHandle> DEF_TO_BOXED_TYPE_IMPLICIT_CAST;
 
     static {
-        final MethodHandles.Lookup methodHandlesLookup = MethodHandles.publicLookup();
+        final MethodHandles.Lookup methodHandlesLookup = MethodHandles.lookup();
 
         try {
             MAP_GET = methodHandlesLookup.findVirtual(Map.class, "get", MethodType.methodType(Object.class, Object.class));
             MAP_PUT = methodHandlesLookup.findVirtual(Map.class, "put", MethodType.methodType(Object.class, Object.class, Object.class));
             LIST_GET = methodHandlesLookup.findVirtual(List.class, "get", MethodType.methodType(Object.class, int.class));
             LIST_SET = methodHandlesLookup.findVirtual(List.class, "set", MethodType.methodType(Object.class, int.class, Object.class));
-            ITERATOR = methodHandlesLookup.findVirtual(Iterable.class, "iterator", MethodType.methodType(Iterator.class));
+            OBJECT_ITERATOR = MethodHandles.filterReturnValue(
+                methodHandlesLookup.findVirtual(Iterable.class, "iterator", MethodType.methodType(Iterator.class)),
+                methodHandlesLookup.findConstructor(ObjectIterator.class, MethodType.methodType(void.class, Iterator.class))
+            );
             MAP_INDEX_NORMALIZE = methodHandlesLookup.findStatic(
                 Def.class,
                 "mapIndexNormalize",
@@ -168,23 +90,14 @@ private ArrayLengthHelper() {}
                 "listIndexNormalize",
                 MethodType.methodType(int.class, List.class, int.class)
             );
-        } catch (final ReflectiveOperationException roe) {
-            throw new AssertionError(roe);
-        }
-
-        // lookup up the factory for arraylength MethodHandle (intrinsic) from Java 9:
-        // https://bugs.openjdk.java.net/browse/JDK-8156915
-        MethodHandle arrayLengthMHFactory;
-        try {
-            arrayLengthMHFactory = methodHandlesLookup.findStatic(
+            ARRAY_LENGTH = methodHandlesLookup.findStatic(
                 MethodHandles.class,
                 "arrayLength",
                 MethodType.methodType(MethodHandle.class, Class.class)
             );
-        } catch (final ReflectiveOperationException roe) {
-            arrayLengthMHFactory = null;
+        } catch (ReflectiveOperationException roe) {
+            throw new AssertionError(roe);
         }
-        JAVA9_ARRAY_LENGTH_MH_FACTORY = arrayLengthMHFactory;
 
         Map<Class<?>, MethodHandle> defToBoxedTypeImplicitCast = new HashMap<>();
 
@@ -232,15 +145,11 @@ static <T extends Throwable> void rethrow(Throwable t) throws T {
 
     /** Returns an array length getter MethodHandle for the given array type */
     static MethodHandle arrayLengthGetter(Class<?> arrayType) {
-        if (JAVA9_ARRAY_LENGTH_MH_FACTORY != null) {
-            try {
-                return (MethodHandle) JAVA9_ARRAY_LENGTH_MH_FACTORY.invokeExact(arrayType);
-            } catch (Throwable t) {
-                rethrow(t);
-                throw new AssertionError(t);
-            }
-        } else {
-            return ArrayLengthHelper.arrayLengthGetter(arrayType);
+        try {
+            return (MethodHandle) ARRAY_LENGTH.invokeExact(arrayType);
+        } catch (Throwable t) {
+            rethrow(t);
+            throw new AssertionError(t);
         }
     }
 
@@ -658,9 +567,129 @@ static MethodHandle lookupArrayLoad(Class<?> receiverClass) {
         );
     }
 
+    private static ClassCastException castException(Class<?> sourceClass, Class<?> targetClass, Boolean implicit) {
+        return new ClassCastException(
+            Strings.format(
+                "cannot %scast def [%s] to %s",
+                implicit != null ? (implicit ? "implicitly " : "explicitly ") : "",
+                PainlessLookupUtility.typeToUnboxedType(sourceClass).getCanonicalName(),
+                targetClass.getCanonicalName()
+            )
+        );
+    }
+
+    private abstract static class BaseIterator<T> implements ValueIterator<T> {
+        @Override
+        public boolean nextBoolean() {
+            Object next = next();
+            try {
+                return (boolean) next;
+            } catch (ClassCastException e) {
+                throw castException(next.getClass(), boolean.class, null);
+            }
+        }
+
+        @Override
+        public byte nextByte() {
+            Object next = next();
+            try {
+                return ((Number) next).byteValue();
+            } catch (ClassCastException e) {
+                throw castException(next.getClass(), byte.class, null);
+            }
+        }
+
+        @Override
+        public short nextShort() {
+            Object next = next();
+            try {
+                return ((Number) next).shortValue();
+            } catch (ClassCastException e) {
+                throw castException(next.getClass(), short.class, null);
+            }
+        }
+
+        @Override
+        public char nextChar() {
+            Object next = next();
+            try {
+                return (char) next;
+            } catch (ClassCastException e) {
+                throw castException(next.getClass(), char.class, null);
+            }
+        }
+
+        @Override
+        public int nextInt() {
+            Object next = next();
+            try {
+                return ((Number) next).intValue();
+            } catch (ClassCastException e) {
+                throw castException(next.getClass(), int.class, null);
+            }
+        }
+
+        @Override
+        public long nextLong() {
+            Object next = next();
+            try {
+                return ((Number) next).longValue();
+            } catch (ClassCastException e) {
+                throw castException(next.getClass(), long.class, null);
+            }
+        }
+
+        @Override
+        public float nextFloat() {
+            Object next = next();
+            try {
+                return ((Number) next).floatValue();
+            } catch (ClassCastException e) {
+                throw castException(next.getClass(), float.class, null);
+            }
+        }
+
+        @Override
+        public double nextDouble() {
+            Object next = next();
+            try {
+                return ((Number) next).doubleValue();
+            } catch (ClassCastException e) {
+                throw castException(next.getClass(), double.class, null);
+            }
+        }
+    }
+
+    private static class ObjectIterator<T> extends BaseIterator<T> {
+        private final Iterator<T> iterator;
+
+        ObjectIterator(Iterator<T> iterator) {
+            this.iterator = iterator;
+        }
+
+        @Override
+        public boolean hasNext() {
+            return iterator.hasNext();
+        }
+
+        @Override
+        public T next() {
+            return iterator.next();
+        }
+
+        @Override
+        public void remove() {
+            iterator.remove();
+        }
+
+        @Override
+        public void forEachRemaining(Consumer<? super T> action) {
+            iterator.forEachRemaining(action);
+        }
+    }
+
     /** Helper class for isolating MethodHandles and methods to get iterators over arrays
-     * (to emulate "enhanced for loop" using MethodHandles). These cause boxing, and are not as efficient
-     * as they could be, but works.
+     * (to emulate "enhanced for loop" using MethodHandles).
      */
     @SuppressWarnings("unused") // iterator() methods are are actually used, javac just does not know :)
     private static final class ArrayIteratorHelper {
@@ -682,7 +711,7 @@ private static final class ArrayIteratorHelper {
                     return PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(
                         PRIVATE_METHOD_HANDLES_LOOKUP.lookupClass(),
                         "iterator",
-                        MethodType.methodType(Iterator.class, type)
+                        MethodType.methodType(ValueIterator.class, type)
                     );
                 } catch (ReflectiveOperationException e) {
                     throw new AssertionError(e);
@@ -692,8 +721,8 @@ private static final class ArrayIteratorHelper {
 
         private static final MethodHandle OBJECT_ARRAY_MH = ARRAY_TYPE_MH_MAPPING.get(Object[].class);
 
-        static Iterator<Boolean> iterator(final boolean[] array) {
-            return new Iterator<Boolean>() {
+        static ValueIterator<Boolean> iterator(final boolean[] array) {
+            return new BaseIterator<Boolean>() {
                 int index = 0;
 
                 @Override
@@ -702,14 +731,19 @@ public boolean hasNext() {
                 }
 
                 @Override
-                public Boolean next() {
+                public boolean nextBoolean() {
                     return array[index++];
                 }
+
+                @Override
+                public Boolean next() {
+                    return nextBoolean();
+                }
             };
         }
 
-        static Iterator<Byte> iterator(final byte[] array) {
-            return new Iterator<Byte>() {
+        static ValueIterator<Byte> iterator(final byte[] array) {
+            return new BaseIterator<Byte>() {
                 int index = 0;
 
                 @Override
@@ -718,14 +752,49 @@ public boolean hasNext() {
                 }
 
                 @Override
-                public Byte next() {
+                public byte nextByte() {
                     return array[index++];
                 }
+
+                @Override
+                public short nextShort() {
+                    return nextByte();
+                }
+
+                @Override
+                public char nextChar() {
+                    return (char) nextByte();
+                }
+
+                @Override
+                public int nextInt() {
+                    return nextByte();
+                }
+
+                @Override
+                public long nextLong() {
+                    return nextByte();
+                }
+
+                @Override
+                public float nextFloat() {
+                    return nextByte();
+                }
+
+                @Override
+                public double nextDouble() {
+                    return nextByte();
+                }
+
+                @Override
+                public Byte next() {
+                    return nextByte();
+                }
             };
         }
 
-        static Iterator<Short> iterator(final short[] array) {
-            return new Iterator<Short>() {
+        static ValueIterator<Short> iterator(final short[] array) {
+            return new BaseIterator<Short>() {
                 int index = 0;
 
                 @Override
@@ -734,14 +803,49 @@ public boolean hasNext() {
                 }
 
                 @Override
-                public Short next() {
+                public byte nextByte() {
+                    return (byte) nextShort();
+                }
+
+                @Override
+                public short nextShort() {
                     return array[index++];
                 }
+
+                @Override
+                public char nextChar() {
+                    return (char) nextShort();
+                }
+
+                @Override
+                public int nextInt() {
+                    return nextShort();
+                }
+
+                @Override
+                public long nextLong() {
+                    return nextShort();
+                }
+
+                @Override
+                public float nextFloat() {
+                    return nextShort();
+                }
+
+                @Override
+                public double nextDouble() {
+                    return nextShort();
+                }
+
+                @Override
+                public Short next() {
+                    return nextShort();
+                }
             };
         }
 
-        static Iterator<Integer> iterator(final int[] array) {
-            return new Iterator<Integer>() {
+        static ValueIterator<Integer> iterator(final int[] array) {
+            return new BaseIterator<Integer>() {
                 int index = 0;
 
                 @Override
@@ -750,14 +854,49 @@ public boolean hasNext() {
                 }
 
                 @Override
-                public Integer next() {
+                public byte nextByte() {
+                    return (byte) nextInt();
+                }
+
+                @Override
+                public short nextShort() {
+                    return (short) nextInt();
+                }
+
+                @Override
+                public char nextChar() {
+                    return (char) nextInt();
+                }
+
+                @Override
+                public int nextInt() {
                     return array[index++];
                 }
+
+                @Override
+                public long nextLong() {
+                    return nextInt();
+                }
+
+                @Override
+                public float nextFloat() {
+                    return nextInt();
+                }
+
+                @Override
+                public double nextDouble() {
+                    return nextInt();
+                }
+
+                @Override
+                public Integer next() {
+                    return nextInt();
+                }
             };
         }
 
-        static Iterator<Long> iterator(final long[] array) {
-            return new Iterator<Long>() {
+        static ValueIterator<Long> iterator(final long[] array) {
+            return new BaseIterator<Long>() {
                 int index = 0;
 
                 @Override
@@ -766,14 +905,49 @@ public boolean hasNext() {
                 }
 
                 @Override
-                public Long next() {
+                public byte nextByte() {
+                    return (byte) nextLong();
+                }
+
+                @Override
+                public short nextShort() {
+                    return (short) nextLong();
+                }
+
+                @Override
+                public char nextChar() {
+                    return (char) nextLong();
+                }
+
+                @Override
+                public int nextInt() {
+                    return (int) nextLong();
+                }
+
+                @Override
+                public long nextLong() {
                     return array[index++];
                 }
+
+                @Override
+                public float nextFloat() {
+                    return nextLong();
+                }
+
+                @Override
+                public double nextDouble() {
+                    return nextLong();
+                }
+
+                @Override
+                public Long next() {
+                    return nextLong();
+                }
             };
         }
 
-        static Iterator<Character> iterator(final char[] array) {
-            return new Iterator<Character>() {
+        static ValueIterator<Character> iterator(final char[] array) {
+            return new BaseIterator<Character>() {
                 int index = 0;
 
                 @Override
@@ -782,14 +956,49 @@ public boolean hasNext() {
                 }
 
                 @Override
-                public Character next() {
+                public byte nextByte() {
+                    return (byte) nextChar();
+                }
+
+                @Override
+                public short nextShort() {
+                    return (short) nextChar();
+                }
+
+                @Override
+                public char nextChar() {
                     return array[index++];
                 }
+
+                @Override
+                public int nextInt() {
+                    return nextChar();
+                }
+
+                @Override
+                public long nextLong() {
+                    return nextChar();
+                }
+
+                @Override
+                public float nextFloat() {
+                    return nextChar();
+                }
+
+                @Override
+                public double nextDouble() {
+                    return nextChar();
+                }
+
+                @Override
+                public Character next() {
+                    return nextChar();
+                }
             };
         }
 
-        static Iterator<Float> iterator(final float[] array) {
-            return new Iterator<Float>() {
+        static ValueIterator<Float> iterator(final float[] array) {
+            return new BaseIterator<Float>() {
                 int index = 0;
 
                 @Override
@@ -798,14 +1007,49 @@ public boolean hasNext() {
                 }
 
                 @Override
-                public Float next() {
+                public byte nextByte() {
+                    return (byte) nextFloat();
+                }
+
+                @Override
+                public short nextShort() {
+                    return (short) nextFloat();
+                }
+
+                @Override
+                public char nextChar() {
+                    return (char) nextFloat();
+                }
+
+                @Override
+                public int nextInt() {
+                    return (int) nextFloat();
+                }
+
+                @Override
+                public long nextLong() {
+                    return (long) nextFloat();
+                }
+
+                @Override
+                public float nextFloat() {
                     return array[index++];
                 }
+
+                @Override
+                public double nextDouble() {
+                    return nextFloat();
+                }
+
+                @Override
+                public Float next() {
+                    return nextFloat();
+                }
             };
         }
 
-        static Iterator<Double> iterator(final double[] array) {
-            return new Iterator<Double>() {
+        static ValueIterator<Double> iterator(final double[] array) {
+            return new BaseIterator<Double>() {
                 int index = 0;
 
                 @Override
@@ -814,14 +1058,49 @@ public boolean hasNext() {
                 }
 
                 @Override
-                public Double next() {
+                public byte nextByte() {
+                    return (byte) nextDouble();
+                }
+
+                @Override
+                public short nextShort() {
+                    return (short) nextDouble();
+                }
+
+                @Override
+                public char nextChar() {
+                    return (char) nextDouble();
+                }
+
+                @Override
+                public int nextInt() {
+                    return (int) nextDouble();
+                }
+
+                @Override
+                public long nextLong() {
+                    return (long) nextDouble();
+                }
+
+                @Override
+                public float nextFloat() {
+                    return (float) nextDouble();
+                }
+
+                @Override
+                public double nextDouble() {
                     return array[index++];
                 }
+
+                @Override
+                public Double next() {
+                    return nextDouble();
+                }
             };
         }
 
-        static Iterator<Object> iterator(final Object[] array) {
-            return new Iterator<Object>() {
+        static ValueIterator<Object> iterator(final Object[] array) {
+            return new BaseIterator<Object>() {
                 int index = 0;
 
                 @Override
@@ -840,9 +1119,8 @@ static MethodHandle newIterator(Class<?> arrayType) {
             if (arrayType.isArray() == false) {
                 throw new IllegalArgumentException("type must be an array");
             }
-            return (ARRAY_TYPE_MH_MAPPING.containsKey(arrayType))
-                ? ARRAY_TYPE_MH_MAPPING.get(arrayType)
-                : OBJECT_ARRAY_MH.asType(OBJECT_ARRAY_MH.type().changeParameterType(0, arrayType));
+            MethodHandle iterator = ARRAY_TYPE_MH_MAPPING.get(arrayType);
+            return iterator != null ? iterator : OBJECT_ARRAY_MH.asType(OBJECT_ARRAY_MH.type().changeParameterType(0, arrayType));
         }
 
         private ArrayIteratorHelper() {}
@@ -855,7 +1133,7 @@ private ArrayIteratorHelper() {}
      */
     static MethodHandle lookupIterator(Class<?> receiverClass) {
         if (Iterable.class.isAssignableFrom(receiverClass)) {
-            return ITERATOR;
+            return OBJECT_ITERATOR;
         } else if (receiverClass.isArray()) {
             return ArrayIteratorHelper.newIterator(receiverClass);
         } else {
@@ -869,13 +1147,7 @@ public static boolean defToboolean(final Object value) {
         if (value instanceof Boolean) {
             return (boolean) value;
         } else {
-            throw new ClassCastException(
-                "cannot cast "
-                    + "def ["
-                    + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                    + "] to "
-                    + boolean.class.getCanonicalName()
-            );
+            throw castException(value.getClass(), boolean.class, null);
         }
     }
 
@@ -883,13 +1155,7 @@ public static byte defTobyteImplicit(final Object value) {
         if (value instanceof Byte) {
             return (byte) value;
         } else {
-            throw new ClassCastException(
-                "cannot implicitly cast "
-                    + "def ["
-                    + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                    + "] to "
-                    + byte.class.getCanonicalName()
-            );
+            throw castException(value.getClass(), byte.class, true);
         }
     }
 
@@ -899,13 +1165,7 @@ public static short defToshortImplicit(final Object value) {
         } else if (value instanceof Short) {
             return (short) value;
         } else {
-            throw new ClassCastException(
-                "cannot implicitly cast "
-                    + "def ["
-                    + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                    + "] to "
-                    + short.class.getCanonicalName()
-            );
+            throw castException(value.getClass(), short.class, true);
         }
     }
 
@@ -913,13 +1173,7 @@ public static char defTocharImplicit(final Object value) {
         if (value instanceof Character) {
             return (char) value;
         } else {
-            throw new ClassCastException(
-                "cannot implicitly cast "
-                    + "def ["
-                    + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                    + "] to "
-                    + char.class.getCanonicalName()
-            );
+            throw castException(value.getClass(), char.class, true);
         }
     }
 
@@ -933,13 +1187,7 @@ public static int defTointImplicit(final Object value) {
         } else if (value instanceof Integer) {
             return (int) value;
         } else {
-            throw new ClassCastException(
-                "cannot implicitly cast "
-                    + "def ["
-                    + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                    + "] to "
-                    + int.class.getCanonicalName()
-            );
+            throw castException(value.getClass(), int.class, true);
         }
     }
 
@@ -955,13 +1203,7 @@ public static long defTolongImplicit(final Object value) {
         } else if (value instanceof Long) {
             return (long) value;
         } else {
-            throw new ClassCastException(
-                "cannot implicitly cast "
-                    + "def ["
-                    + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                    + "] to "
-                    + long.class.getCanonicalName()
-            );
+            throw castException(value.getClass(), long.class, true);
         }
     }
 
@@ -979,13 +1221,7 @@ public static float defTofloatImplicit(final Object value) {
         } else if (value instanceof Float) {
             return (float) value;
         } else {
-            throw new ClassCastException(
-                "cannot implicitly cast "
-                    + "def ["
-                    + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                    + "] to "
-                    + float.class.getCanonicalName()
-            );
+            throw castException(value.getClass(), float.class, true);
         }
     }
 
@@ -1005,13 +1241,7 @@ public static double defTodoubleImplicit(final Object value) {
         } else if (value instanceof Double) {
             return (double) value;
         } else {
-            throw new ClassCastException(
-                "cannot implicitly cast "
-                    + "def ["
-                    + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                    + "] to "
-                    + double.class.getCanonicalName()
-            );
+            throw castException(value.getClass(), double.class, true);
         }
     }
 
@@ -1026,13 +1256,7 @@ public static byte defTobyteExplicit(final Object value) {
             || value instanceof Double) {
                 return ((Number) value).byteValue();
             } else {
-                throw new ClassCastException(
-                    "cannot explicitly cast "
-                        + "def ["
-                        + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                        + "] to "
-                        + byte.class.getCanonicalName()
-                );
+                throw castException(value.getClass(), byte.class, false);
             }
     }
 
@@ -1047,13 +1271,7 @@ public static short defToshortExplicit(final Object value) {
             || value instanceof Double) {
                 return ((Number) value).shortValue();
             } else {
-                throw new ClassCastException(
-                    "cannot explicitly cast "
-                        + "def ["
-                        + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                        + "] to "
-                        + short.class.getCanonicalName()
-                );
+                throw castException(value.getClass(), short.class, false);
             }
     }
 
@@ -1070,13 +1288,7 @@ public static char defTocharExplicit(final Object value) {
             || value instanceof Double) {
                 return (char) ((Number) value).intValue();
             } else {
-                throw new ClassCastException(
-                    "cannot explicitly cast "
-                        + "def ["
-                        + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                        + "] to "
-                        + char.class.getCanonicalName()
-                );
+                throw castException(value.getClass(), char.class, false);
             }
     }
 
@@ -1091,13 +1303,7 @@ public static int defTointExplicit(final Object value) {
             || value instanceof Double) {
                 return ((Number) value).intValue();
             } else {
-                throw new ClassCastException(
-                    "cannot explicitly cast "
-                        + "def ["
-                        + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                        + "] to "
-                        + int.class.getCanonicalName()
-                );
+                throw castException(value.getClass(), int.class, false);
             }
     }
 
@@ -1112,13 +1318,7 @@ public static long defTolongExplicit(final Object value) {
             || value instanceof Double) {
                 return ((Number) value).longValue();
             } else {
-                throw new ClassCastException(
-                    "cannot explicitly cast "
-                        + "def ["
-                        + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                        + "] to "
-                        + long.class.getCanonicalName()
-                );
+                throw castException(value.getClass(), long.class, false);
             }
     }
 
@@ -1133,13 +1333,7 @@ public static float defTofloatExplicit(final Object value) {
             || value instanceof Double) {
                 return ((Number) value).floatValue();
             } else {
-                throw new ClassCastException(
-                    "cannot explicitly cast "
-                        + "float ["
-                        + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                        + "] to "
-                        + byte.class.getCanonicalName()
-                );
+                throw castException(value.getClass(), float.class, false);
             }
     }
 
@@ -1154,13 +1348,7 @@ public static double defTodoubleExplicit(final Object value) {
             || value instanceof Double) {
                 return ((Number) value).doubleValue();
             } else {
-                throw new ClassCastException(
-                    "cannot explicitly cast "
-                        + "def ["
-                        + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                        + "] to "
-                        + byte.class.getCanonicalName()
-                );
+                throw castException(value.getClass(), byte.class, false);
             }
     }
 
@@ -1172,13 +1360,7 @@ public static Boolean defToBoolean(final Object value) {
         } else if (value instanceof Boolean) {
             return (Boolean) value;
         } else {
-            throw new ClassCastException(
-                "cannot implicitly cast "
-                    + "def ["
-                    + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                    + "] to "
-                    + Boolean.class.getCanonicalName()
-            );
+            throw castException(value.getClass(), Boolean.class, false);
         }
     }
 
@@ -1188,13 +1370,7 @@ public static Byte defToByteImplicit(final Object value) {
         } else if (value instanceof Byte) {
             return (Byte) value;
         } else {
-            throw new ClassCastException(
-                "cannot implicitly cast "
-                    + "def ["
-                    + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                    + "] to "
-                    + Byte.class.getCanonicalName()
-            );
+            throw castException(value.getClass(), Byte.class, false);
         }
     }
 
@@ -1206,13 +1382,7 @@ public static Short defToShortImplicit(final Object value) {
         } else if (value instanceof Short) {
             return (Short) value;
         } else {
-            throw new ClassCastException(
-                "cannot implicitly cast "
-                    + "def ["
-                    + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                    + "] to "
-                    + Short.class.getCanonicalName()
-            );
+            throw castException(value.getClass(), Short.class, false);
         }
     }
 
@@ -1222,13 +1392,7 @@ public static Character defToCharacterImplicit(final Object value) {
         } else if (value instanceof Character) {
             return (Character) value;
         } else {
-            throw new ClassCastException(
-                "cannot implicitly cast "
-                    + "def ["
-                    + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                    + "] to "
-                    + Character.class.getCanonicalName()
-            );
+            throw castException(value.getClass(), Character.class, false);
         }
     }
 
@@ -1244,13 +1408,7 @@ public static Integer defToIntegerImplicit(final Object value) {
         } else if (value instanceof Integer) {
             return (Integer) value;
         } else {
-            throw new ClassCastException(
-                "cannot implicitly cast "
-                    + "def ["
-                    + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                    + "] to "
-                    + Integer.class.getCanonicalName()
-            );
+            throw castException(value.getClass(), Integer.class, false);
         }
     }
 
@@ -1268,13 +1426,7 @@ public static Long defToLongImplicit(final Object value) {
         } else if (value instanceof Long) {
             return (Long) value;
         } else {
-            throw new ClassCastException(
-                "cannot implicitly cast "
-                    + "def ["
-                    + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                    + "] to "
-                    + Long.class.getCanonicalName()
-            );
+            throw castException(value.getClass(), Long.class, false);
         }
     }
 
@@ -1294,13 +1446,7 @@ public static Float defToFloatImplicit(final Object value) {
         } else if (value instanceof Float) {
             return (Float) value;
         } else {
-            throw new ClassCastException(
-                "cannot implicitly cast "
-                    + "def ["
-                    + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                    + "] to "
-                    + Float.class.getCanonicalName()
-            );
+            throw castException(value.getClass(), Float.class, false);
         }
     }
 
@@ -1322,13 +1468,7 @@ public static Double defToDoubleImplicit(final Object value) {
         } else if (value instanceof Double) {
             return (Double) value;
         } else {
-            throw new ClassCastException(
-                "cannot implicitly cast "
-                    + "def ["
-                    + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                    + "] to "
-                    + Double.class.getCanonicalName()
-            );
+            throw castException(value.getClass(), Double.class, false);
         }
     }
 
@@ -1345,13 +1485,7 @@ public static Byte defToByteExplicit(final Object value) {
             || value instanceof Double) {
                 return ((Number) value).byteValue();
             } else {
-                throw new ClassCastException(
-                    "cannot explicitly cast "
-                        + "def ["
-                        + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                        + "] to "
-                        + Byte.class.getCanonicalName()
-                );
+                throw castException(value.getClass(), Byte.class, false);
             }
     }
 
@@ -1368,13 +1502,7 @@ public static Short defToShortExplicit(final Object value) {
             || value instanceof Double) {
                 return ((Number) value).shortValue();
             } else {
-                throw new ClassCastException(
-                    "cannot explicitly cast "
-                        + "def ["
-                        + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                        + "] to "
-                        + Short.class.getCanonicalName()
-                );
+                throw castException(value.getClass(), Short.class, false);
             }
     }
 
@@ -1393,13 +1521,7 @@ public static Character defToCharacterExplicit(final Object value) {
             || value instanceof Double) {
                 return (char) ((Number) value).intValue();
             } else {
-                throw new ClassCastException(
-                    "cannot explicitly cast "
-                        + "def ["
-                        + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                        + "] to "
-                        + Character.class.getCanonicalName()
-                );
+                throw castException(value.getClass(), Character.class, false);
             }
     }
 
@@ -1416,13 +1538,7 @@ public static Integer defToIntegerExplicit(final Object value) {
             || value instanceof Double) {
                 return ((Number) value).intValue();
             } else {
-                throw new ClassCastException(
-                    "cannot explicitly cast "
-                        + "def ["
-                        + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                        + "] to "
-                        + Integer.class.getCanonicalName()
-                );
+                throw castException(value.getClass(), Integer.class, false);
             }
     }
 
@@ -1439,13 +1555,7 @@ public static Long defToLongExplicit(final Object value) {
             || value instanceof Double) {
                 return ((Number) value).longValue();
             } else {
-                throw new ClassCastException(
-                    "cannot explicitly cast "
-                        + "def ["
-                        + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                        + "] to "
-                        + Long.class.getCanonicalName()
-                );
+                throw castException(value.getClass(), Long.class, false);
             }
     }
 
@@ -1462,13 +1572,7 @@ public static Float defToFloatExplicit(final Object value) {
             || value instanceof Double) {
                 return ((Number) value).floatValue();
             } else {
-                throw new ClassCastException(
-                    "cannot explicitly cast "
-                        + "def ["
-                        + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                        + "] to "
-                        + Float.class.getCanonicalName()
-                );
+                throw castException(value.getClass(), Float.class, false);
             }
     }
 
@@ -1485,13 +1589,7 @@ public static Double defToDoubleExplicit(final Object value) {
             || value instanceof Double) {
                 return ((Number) value).doubleValue();
             } else {
-                throw new ClassCastException(
-                    "cannot explicitly cast "
-                        + "def ["
-                        + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                        + "] to "
-                        + Double.class.getCanonicalName()
-                );
+                throw castException(value.getClass(), Double.class, false);
             }
     }
 
@@ -1501,13 +1599,7 @@ public static String defToStringImplicit(final Object value) {
         } else if (value instanceof String) {
             return (String) value;
         } else {
-            throw new ClassCastException(
-                "cannot implicitly cast "
-                    + "def ["
-                    + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                    + "] to "
-                    + String.class.getCanonicalName()
-            );
+            throw castException(value.getClass(), String.class, true);
         }
     }
 
@@ -1519,13 +1611,7 @@ public static String defToStringExplicit(final Object value) {
         } else if (value instanceof String) {
             return (String) value;
         } else {
-            throw new ClassCastException(
-                "cannot explicitly cast "
-                    + "def ["
-                    + PainlessLookupUtility.typeToUnboxedType(value.getClass()).getCanonicalName()
-                    + "] to "
-                    + String.class.getCanonicalName()
-            );
+            throw castException(value.getClass(), String.class, false);
         }
     }
 
@@ -1616,9 +1702,8 @@ static MethodHandle arrayIndexNormalizer(Class<?> arrayType) {
             if (arrayType.isArray() == false) {
                 throw new IllegalArgumentException("type must be an array");
             }
-            return (ARRAY_TYPE_MH_MAPPING.containsKey(arrayType))
-                ? ARRAY_TYPE_MH_MAPPING.get(arrayType)
-                : OBJECT_ARRAY_MH.asType(OBJECT_ARRAY_MH.type().changeParameterType(0, arrayType));
+            MethodHandle handle = ARRAY_TYPE_MH_MAPPING.get(arrayType);
+            return handle != null ? handle : OBJECT_ARRAY_MH.asType(OBJECT_ARRAY_MH.type().changeParameterType(0, arrayType));
         }
 
         private ArrayIndexNormalizeHelper() {}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 135673010c6f..3d35c5d0ab5e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -382,13 +382,13 @@ Object fallback(Object[] args) throws Throwable {
                 } else if (type.parameterType(0) != Object.class) {
                     // case 2: only the argument is unknown, just check that
                     MethodType testType = MethodType.methodType(boolean.class, type);
-                    MethodHandle unaryTest = CHECK_RHS.bindTo(clazz0).bindTo(clazz1);
-                    test = unaryTest.asType(testType);
+                    MethodHandle unaryTest = CHECK_RHS.bindTo(clazz1);
+                    test = MethodHandles.dropArguments(unaryTest, 0, Object.class).asType(testType);
                     nullCheck = MethodHandles.dropArguments(NON_NULL, 0, clazz0).asType(testType);
                 } else {
                     // case 3: check both receiver and argument
                     MethodType testType = MethodType.methodType(boolean.class, type);
-                    MethodHandle binaryTest = CHECK_BOTH.bindTo(clazz0).bindTo(clazz1);
+                    MethodHandle binaryTest = MethodHandles.insertArguments(CHECK_BOTH, 0, clazz0, clazz1);
                     test = binaryTest.asType(testType);
                     nullCheck = BOTH_NON_NULL.asType(testType);
                 }
@@ -423,7 +423,7 @@ static boolean checkLHS(Class<?> clazz, Object leftObject) {
          * guard method for inline caching: checks the first argument is the same
          * as the cached first argument.
          */
-        static boolean checkRHS(Class<?> left, Class<?> right, Object leftObject, Object rightObject) {
+        static boolean checkRHS(Class<?> right, Object rightObject) {
             return rightObject.getClass() == right;
         }
 
@@ -460,7 +460,7 @@ static boolean bothNonNull(Object leftObject, Object rightObject) {
                 CHECK_RHS = methodHandlesLookup.findStatic(
                     methodHandlesLookup.lookupClass(),
                     "checkRHS",
-                    MethodType.methodType(boolean.class, Class.class, Class.class, Object.class, Object.class)
+                    MethodType.methodType(boolean.class, Class.class, Object.class)
                 );
                 CHECK_BOTH = methodHandlesLookup.findStatic(
                     methodHandlesLookup.lookupClass(),
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java
index 0a29ff80a45b..1132b1163565 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java
@@ -526,8 +526,12 @@ private static boolean eq(boolean a, boolean b) {
     }
 
     private static boolean eq(Object left, Object right) {
-        if (left != null && right != null) {
-            if (left instanceof Double) {
+        if (left == right) {
+            return true;
+        } else if (left != null && right != null) {
+            if (left.getClass() == right.getClass()) {
+                return left.equals(right);
+            } else if (left instanceof Double) {
                 if (right instanceof Number) {
                     return (double) left == ((Number) right).doubleValue();
                 } else if (right instanceof Character) {
@@ -537,7 +541,7 @@ private static boolean eq(Object left, Object right) {
                 if (left instanceof Number) {
                     return ((Number) left).doubleValue() == (double) right;
                 } else if (left instanceof Character) {
-                    return (char) left == ((Number) right).doubleValue();
+                    return (char) left == (double) right;
                 }
             } else if (left instanceof Float) {
                 if (right instanceof Number) {
@@ -549,7 +553,7 @@ private static boolean eq(Object left, Object right) {
                 if (left instanceof Number) {
                     return ((Number) left).floatValue() == (float) right;
                 } else if (left instanceof Character) {
-                    return (char) left == ((Number) right).floatValue();
+                    return (char) left == (float) right;
                 }
             } else if (left instanceof Long) {
                 if (right instanceof Number) {
@@ -561,7 +565,7 @@ private static boolean eq(Object left, Object right) {
                 if (left instanceof Number) {
                     return ((Number) left).longValue() == (long) right;
                 } else if (left instanceof Character) {
-                    return (char) left == ((Number) right).longValue();
+                    return (char) left == (long) right;
                 }
             } else if (left instanceof Number) {
                 if (right instanceof Number) {
@@ -578,7 +582,7 @@ private static boolean eq(Object left, Object right) {
             return left.equals(right);
         }
 
-        return left == null && right == null;
+        return false;
     }
 
     // comparison operators: applicable for any numeric type
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
index 816825a7700d..4da523bce634 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.painless;
 
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.painless.lookup.PainlessConstructor;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupUtility;
@@ -22,7 +23,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.painless.WriterConstants.CLASS_NAME;
 import static org.objectweb.asm.Opcodes.H_INVOKEINTERFACE;
@@ -73,14 +74,12 @@ public static FunctionRef create(
 
             if (interfaceMethod == null) {
                 throw new IllegalArgumentException(
-                    "cannot convert function reference ["
-                        + typeName
-                        + "::"
-                        + methodName
-                        + "] "
-                        + "to a non-functional interface ["
-                        + targetClassName
-                        + "]"
+                    Strings.format(
+                        "cannot convert function reference [%s::%s] to a non-functional interface [%s]",
+                        typeName,
+                        methodName,
+                        targetClassName
+                    )
                 );
             }
 
@@ -110,18 +109,14 @@ public static FunctionRef create(
 
                 if (localFunction == null) {
                     throw new IllegalArgumentException(
-                        "function reference [this::"
-                            + localFunctionKey
-                            + "] "
-                            + "matching ["
-                            + targetClassName
-                            + ", "
-                            + interfaceMethodName
-                            + "/"
-                            + interfaceTypeParametersSize
-                            + "] "
-                            + "not found"
-                            + (localFunctionKey.contains("$") ? " due to an incorrect number of arguments" : "")
+                        Strings.format(
+                            "function reference [this::%s] matching [%s, %s/%d] not found%s",
+                            localFunctionKey,
+                            targetClassName,
+                            interfaceMethodName,
+                            interfaceTypeParametersSize,
+                            localFunctionKey.contains("$") ? " due to an incorrect number of arguments" : ""
+                        )
                     );
                 }
 
@@ -144,19 +139,14 @@ public static FunctionRef create(
 
                 if (painlessConstructor == null) {
                     throw new IllegalArgumentException(
-                        "function reference ["
-                            + typeName
-                            + "::new/"
-                            + interfaceTypeParametersSize
-                            + "] "
-                            + "matching ["
-                            + targetClassName
-                            + ", "
-                            + interfaceMethodName
-                            + "/"
-                            + interfaceTypeParametersSize
-                            + "] "
-                            + "not found"
+                        Strings.format(
+                            "function reference [%s::new/%d] matching [%s, %s/%d] not found",
+                            typeName,
+                            interfaceTypeParametersSize,
+                            targetClassName,
+                            interfaceMethodName,
+                            interfaceTypeParametersSize
+                        )
                     );
                 }
 
@@ -193,35 +183,25 @@ public static FunctionRef create(
 
                     if (painlessMethod == null) {
                         throw new IllegalArgumentException(
-                            "function reference "
-                                + "["
-                                + typeName
-                                + "::"
-                                + methodName
-                                + "/"
-                                + interfaceTypeParametersSize
-                                + "] "
-                                + "matching ["
-                                + targetClassName
-                                + ", "
-                                + interfaceMethodName
-                                + "/"
-                                + interfaceTypeParametersSize
-                                + "] "
-                                + "not found"
+                            Strings.format(
+                                "function reference [%s::%s/%d] matching [%s, %s/%d] not found",
+                                typeName,
+                                methodName,
+                                interfaceTypeParametersSize,
+                                targetClassName,
+                                interfaceMethodName,
+                                interfaceTypeParametersSize
+                            )
                         );
                     }
                 } else if (captured) {
                     throw new IllegalArgumentException(
-                        "cannot use a static method as a function reference "
-                            + "["
-                            + typeName
-                            + "::"
-                            + methodName
-                            + "/"
-                            + interfaceTypeParametersSize
-                            + "] "
-                            + "with a non-static captured variable"
+                        Strings.format(
+                            "cannot use a static method as a function reference [%s::%s/%d] with a non-static captured variable",
+                            typeName,
+                            methodName,
+                            interfaceTypeParametersSize
+                        )
                     );
                 }
 
@@ -360,19 +340,20 @@ public String getFactoryMethodDescriptor() {
         if (factoryMethodReceiver == null) {
             return factoryMethodType.toMethodDescriptorString();
         }
-        List<Type> arguments = factoryMethodType.parameterList().stream().map(Type::getType).collect(Collectors.toList());
-        arguments.add(0, factoryMethodReceiver);
-        Type[] argArray = new Type[arguments.size()];
-        arguments.toArray(argArray);
-        return Type.getMethodDescriptor(Type.getType(factoryMethodType.returnType()), argArray);
+        Type[] arguments = Stream.concat(Stream.of(factoryMethodReceiver), factoryMethodType.parameterList().stream().map(Type::getType))
+            .toArray(Type[]::new);
+        return Type.getMethodDescriptor(Type.getType(factoryMethodType.returnType()), arguments);
     }
 
     /** Get the factory method type, updating the receiver if {@code factoryMethodReceiverClass} is non-null */
     public Class<?>[] factoryMethodParameters(Class<?> factoryMethodReceiverClass) {
-        List<Class<?>> parameters = new ArrayList<>(factoryMethodType.parameterList());
+        Class<?>[] parameters = factoryMethodType.parameterList().toArray(Class<?>[]::new);
         if (factoryMethodReceiverClass != null) {
-            parameters.add(0, factoryMethodReceiverClass);
+            Class<?>[] withReceiver = new Class<?>[parameters.length + 1];
+            withReceiver[0] = factoryMethodReceiverClass;
+            System.arraycopy(parameters, 0, withReceiver, 1, parameters.length);
+            parameters = withReceiver;
         }
-        return parameters.toArray(new Class<?>[0]);
+        return parameters;
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java
index 54bfa3c86c71..a2fa5dea229d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java
@@ -24,7 +24,6 @@
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.List;
-import java.util.stream.Collectors;
 
 import static java.lang.invoke.MethodHandles.Lookup;
 import static org.elasticsearch.painless.WriterConstants.CLASS_VERSION;
@@ -400,9 +399,9 @@ private static void generateInterfaceMethod(
         iface.visitCode();
 
         // Loads any captured variables onto the stack.
-        for (int captureCount = 0; captureCount < captures.length; ++captureCount) {
+        for (Capture capture : captures) {
             iface.loadThis();
-            iface.getField(lambdaClassType, captures[captureCount].name, captures[captureCount].type);
+            iface.getField(lambdaClassType, capture.name, capture.type);
         }
 
         // Loads any passed in arguments onto the stack.
@@ -442,13 +441,16 @@ private static void generateInterfaceMethod(
                 delegateClassType = Type.getType(clazz);
 
                 // functionalInterfaceWithCaptures needs to add the receiver and other captures
-                List<Type> parameters = interfaceMethodType.parameterList().stream().map(Type::getType).collect(Collectors.toList());
-                parameters.add(0, delegateClassType);
+                Type[] parameters = new Type[captures.length + interfaceMethodType.parameterList().size()];
+                int p = 0;
+                parameters[p++] = delegateClassType;
                 for (int i = 1; i < captures.length; i++) {
-                    parameters.add(i, captures[i].type);
+                    parameters[p++] = captures[i].type;
+                }
+                for (Class<?> pCls : interfaceMethodType.parameterList()) {
+                    parameters[p++] = Type.getType(pCls);
                 }
-                Type[] parametersArray = parameters.toArray(new Type[0]);
-                functionalInterfaceWithCaptures = Type.getMethodDescriptor(Type.getType(interfaceMethodType.returnType()), parametersArray);
+                functionalInterfaceWithCaptures = Type.getMethodDescriptor(Type.getType(interfaceMethodType.returnType()), parameters);
 
                 // delegateMethod does not need the receiver
                 List<Class<?>> factoryParameters = factoryMethodType.parameterList();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 878ff0e78b61..1e88edc788c6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -61,21 +61,10 @@
 import static org.elasticsearch.painless.WriterConstants.DEF_TO_STRING_EXPLICIT;
 import static org.elasticsearch.painless.WriterConstants.DEF_TO_STRING_IMPLICIT;
 import static org.elasticsearch.painless.WriterConstants.DEF_UTIL_TYPE;
-import static org.elasticsearch.painless.WriterConstants.INDY_STRING_CONCAT_BOOTSTRAP_HANDLE;
 import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE;
-import static org.elasticsearch.painless.WriterConstants.MAX_INDY_STRING_CONCAT_ARGS;
+import static org.elasticsearch.painless.WriterConstants.MAX_STRING_CONCAT_ARGS;
 import static org.elasticsearch.painless.WriterConstants.PAINLESS_ERROR_TYPE;
-import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_BOOLEAN;
-import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_CHAR;
-import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_DOUBLE;
-import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_FLOAT;
-import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_INT;
-import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_LONG;
-import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_OBJECT;
-import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_STRING;
-import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_CONSTRUCTOR;
-import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_TOSTRING;
-import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_TYPE;
+import static org.elasticsearch.painless.WriterConstants.STRING_CONCAT_BOOTSTRAP_HANDLE;
 import static org.elasticsearch.painless.WriterConstants.STRING_TO_CHAR;
 import static org.elasticsearch.painless.WriterConstants.STRING_TYPE;
 import static org.elasticsearch.painless.WriterConstants.UTILITY_TYPE;
@@ -90,7 +79,7 @@ public final class MethodWriter extends GeneratorAdapter {
     private final BitSet statements;
     private final CompilerSettings settings;
 
-    private final Deque<List<Type>> stringConcatArgs = (INDY_STRING_CONCAT_BOOTSTRAP_HANDLE == null) ? null : new ArrayDeque<>();
+    private final Deque<List<Type>> stringConcatArgs = new ArrayDeque<>();
 
     public MethodWriter(int access, Method method, ClassVisitor cw, BitSet statements, CompilerSettings settings) {
         super(
@@ -266,57 +255,28 @@ public static Type getType(Class<?> clazz) {
     /** Starts a new string concat.
      * @return the size of arguments pushed to stack (the object that does string concats, e.g. a StringBuilder)
      */
-    public int writeNewStrings() {
-        if (INDY_STRING_CONCAT_BOOTSTRAP_HANDLE != null) {
-            // Java 9+: we just push our argument collector onto deque
-            stringConcatArgs.push(new ArrayList<>());
-            return 0; // nothing added to stack
-        } else {
-            // Java 8: create a StringBuilder in bytecode
-            newInstance(STRINGBUILDER_TYPE);
-            dup();
-            invokeConstructor(STRINGBUILDER_TYPE, STRINGBUILDER_CONSTRUCTOR);
-            return 1; // StringBuilder on stack
-        }
+    public List<Type> writeNewStrings() {
+        List<Type> list = new ArrayList<>();
+        stringConcatArgs.push(list);
+        return list;
     }
 
     public void writeAppendStrings(Class<?> clazz) {
-        if (INDY_STRING_CONCAT_BOOTSTRAP_HANDLE != null) {
-            // Java 9+: record type information
-            stringConcatArgs.peek().add(getType(clazz));
-            // prevent too many concat args.
-            // If there are too many, do the actual concat:
-            if (stringConcatArgs.peek().size() >= MAX_INDY_STRING_CONCAT_ARGS) {
-                writeToStrings();
-                writeNewStrings();
-                // add the return value type as new first param for next concat:
-                stringConcatArgs.peek().add(STRING_TYPE);
-            }
-        } else {
-            // Java 8: push a StringBuilder append
-            if (clazz == boolean.class) invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_BOOLEAN);
-            else if (clazz == char.class) invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_CHAR);
-            else if (clazz == byte.class || clazz == short.class || clazz == int.class) invokeVirtual(
-                STRINGBUILDER_TYPE,
-                STRINGBUILDER_APPEND_INT
-            );
-            else if (clazz == long.class) invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_LONG);
-            else if (clazz == float.class) invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_FLOAT);
-            else if (clazz == double.class) invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_DOUBLE);
-            else if (clazz == String.class) invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_STRING);
-            else invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_OBJECT);
+        List<Type> currentConcat = stringConcatArgs.peek();
+        currentConcat.add(getType(clazz));
+        // prevent too many concat args.
+        // If there are too many, do the actual concat:
+        if (currentConcat.size() >= MAX_STRING_CONCAT_ARGS) {
+            writeToStrings();
+            currentConcat = writeNewStrings();
+            // add the return value type as new first param for next concat:
+            currentConcat.add(STRING_TYPE);
         }
     }
 
     public void writeToStrings() {
-        if (INDY_STRING_CONCAT_BOOTSTRAP_HANDLE != null) {
-            // Java 9+: use type information and push invokeDynamic
-            final String desc = Type.getMethodDescriptor(STRING_TYPE, stringConcatArgs.pop().stream().toArray(Type[]::new));
-            invokeDynamic("concat", desc, INDY_STRING_CONCAT_BOOTSTRAP_HANDLE);
-        } else {
-            // Java 8: call toString() on StringBuilder
-            invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_TOSTRING);
-        }
+        final String desc = Type.getMethodDescriptor(STRING_TYPE, stringConcatArgs.pop().toArray(Type[]::new));
+        invokeDynamic("concat", desc, STRING_CONCAT_BOOTSTRAP_HANDLE);
     }
 
     /** Writes a dynamic binary instruction: returnType, lhs, and rhs can be different */
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
index dfb547cbff0c..f3b7803bf2c5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -147,7 +147,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver expressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         // this is a hack to bind the painless script engine in guice (all components are added to guice), so that
         // the painless context api. this is a temporary measure until transport actions do no require guice
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index b30d45733cec..9207e0a55673 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.painless;
 
+import org.elasticsearch.painless.api.ValueIterator;
 import org.objectweb.asm.Handle;
 import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.Type;
@@ -17,10 +18,9 @@
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodType;
-import java.util.ArrayList;
+import java.lang.invoke.StringConcatFactory;
 import java.util.Collection;
 import java.util.Iterator;
-import java.util.List;
 import java.util.Objects;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -49,8 +49,17 @@ public final class WriterConstants {
     public static final MethodType NEEDS_PARAMETER_METHOD_TYPE = MethodType.methodType(boolean.class);
 
     public static final Type ITERATOR_TYPE = Type.getType(Iterator.class);
+    public static final Type VALUE_ITERATOR_TYPE = Type.getType(ValueIterator.class);
     public static final Method ITERATOR_HASNEXT = getAsmMethod(boolean.class, "hasNext");
     public static final Method ITERATOR_NEXT = getAsmMethod(Object.class, "next");
+    public static final Method VALUE_ITERATOR_NEXT_BOOLEAN = getAsmMethod(boolean.class, "nextBoolean");
+    public static final Method VALUE_ITERATOR_NEXT_BYTE = getAsmMethod(byte.class, "nextByte");
+    public static final Method VALUE_ITERATOR_NEXT_SHORT = getAsmMethod(short.class, "nextShort");
+    public static final Method VALUE_ITERATOR_NEXT_CHAR = getAsmMethod(char.class, "nextChar");
+    public static final Method VALUE_ITERATOR_NEXT_INT = getAsmMethod(int.class, "nextInt");
+    public static final Method VALUE_ITERATOR_NEXT_LONG = getAsmMethod(long.class, "nextLong");
+    public static final Method VALUE_ITERATOR_NEXT_FLOAT = getAsmMethod(float.class, "nextFloat");
+    public static final Method VALUE_ITERATOR_NEXT_DOUBLE = getAsmMethod(double.class, "nextDouble");
 
     public static final Type UTILITY_TYPE = Type.getType(Utility.class);
     public static final Method STRING_TO_CHAR = getAsmMethod(char.class, "StringTochar", String.class);
@@ -163,39 +172,23 @@ public final class WriterConstants {
         false
     );
 
-    /** dynamic invokedynamic bootstrap for indy string concats (Java 9+) */
-    public static final Handle INDY_STRING_CONCAT_BOOTSTRAP_HANDLE;
-    static {
-        Handle bs;
-        try {
-            final Class<?> factory = Class.forName("java.lang.invoke.StringConcatFactory");
-            final String methodName = "makeConcat";
-            final MethodType type = MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class);
-            // ensure it is there:
-            MethodHandles.publicLookup().findStatic(factory, methodName, type);
-            bs = new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(factory), methodName, type.toMethodDescriptorString(), false);
-        } catch (ReflectiveOperationException e) {
-            // not Java 9 - we set it null, so MethodWriter uses StringBuilder:
-            bs = null;
-        }
-        INDY_STRING_CONCAT_BOOTSTRAP_HANDLE = bs;
-    }
+    public static final MethodType MAKE_CONCAT_TYPE = MethodType.methodType(
+        CallSite.class,
+        MethodHandles.Lookup.class,
+        String.class,
+        MethodType.class
+    );
+    public static final Handle STRING_CONCAT_BOOTSTRAP_HANDLE = new Handle(
+        Opcodes.H_INVOKESTATIC,
+        Type.getInternalName(StringConcatFactory.class),
+        "makeConcat",
+        MAKE_CONCAT_TYPE.toMethodDescriptorString(),
+        false
+    );
 
-    public static final int MAX_INDY_STRING_CONCAT_ARGS = 200;
+    public static final int MAX_STRING_CONCAT_ARGS = 200;
 
     public static final Type STRING_TYPE = Type.getType(String.class);
-    public static final Type STRINGBUILDER_TYPE = Type.getType(StringBuilder.class);
-
-    public static final Method STRINGBUILDER_CONSTRUCTOR = getAsmMethod(void.class, CTOR_METHOD_NAME);
-    public static final Method STRINGBUILDER_APPEND_BOOLEAN = getAsmMethod(StringBuilder.class, "append", boolean.class);
-    public static final Method STRINGBUILDER_APPEND_CHAR = getAsmMethod(StringBuilder.class, "append", char.class);
-    public static final Method STRINGBUILDER_APPEND_INT = getAsmMethod(StringBuilder.class, "append", int.class);
-    public static final Method STRINGBUILDER_APPEND_LONG = getAsmMethod(StringBuilder.class, "append", long.class);
-    public static final Method STRINGBUILDER_APPEND_FLOAT = getAsmMethod(StringBuilder.class, "append", float.class);
-    public static final Method STRINGBUILDER_APPEND_DOUBLE = getAsmMethod(StringBuilder.class, "append", double.class);
-    public static final Method STRINGBUILDER_APPEND_STRING = getAsmMethod(StringBuilder.class, "append", String.class);
-    public static final Method STRINGBUILDER_APPEND_OBJECT = getAsmMethod(StringBuilder.class, "append", Object.class);
-    public static final Method STRINGBUILDER_TOSTRING = getAsmMethod(String.class, "toString");
 
     public static final Type OBJECTS_TYPE = Type.getType(Objects.class);
     public static final Method EQUALS = getAsmMethod(boolean.class, "equals", Object.class, Object.class);
@@ -203,12 +196,6 @@ public final class WriterConstants {
     public static final Type COLLECTION_TYPE = Type.getType(Collection.class);
     public static final Method COLLECTION_SIZE = getAsmMethod(int.class, "size");
 
-    public static final Type LIST_TYPE = Type.getType(List.class);
-    public static final Method LIST_ADD = getAsmMethod(boolean.class, "add", Object.class);
-
-    public static final Type ARRAY_LIST_TYPE = Type.getType(ArrayList.class);
-    public static final Method ARRAY_LIST_CTOR_WITH_SIZE = getAsmMethod(void.class, CTOR_METHOD_NAME, int.class);
-
     private static Method getAsmMethod(final Class<?> rtype, final String name, final Class<?>... ptypes) {
         return new Method(name, MethodType.methodType(rtype, ptypes).toMethodDescriptorString());
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java
index 6a3d54e73497..1c50b3558dcd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java
@@ -49,6 +49,7 @@
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
@@ -557,7 +558,8 @@ static Response innerShardOperation(Request request, ScriptService scriptService
                     BooleanFieldScript.LeafFactory leafFactory = factory.newFactory(
                         BooleanFieldScript.CONTEXT.name,
                         request.getScript().getParams(),
-                        context.lookup()
+                        context.lookup(),
+                        OnScriptError.FAIL
                     );
                     BooleanFieldScript booleanFieldScript = leafFactory.newInstance(leafReaderContext);
                     List<Boolean> booleans = new ArrayList<>();
@@ -571,7 +573,8 @@ static Response innerShardOperation(Request request, ScriptService scriptService
                         DateFieldScript.CONTEXT.name,
                         request.getScript().getParams(),
                         context.lookup(),
-                        DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER
+                        DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
+                        OnScriptError.FAIL
                     );
                     DateFieldScript dateFieldScript = leafFactory.newInstance(leafReaderContext);
                     List<String> dates = new ArrayList<>();
@@ -584,7 +587,8 @@ static Response innerShardOperation(Request request, ScriptService scriptService
                     DoubleFieldScript.LeafFactory leafFactory = factory.newFactory(
                         DoubleFieldScript.CONTEXT.name,
                         request.getScript().getParams(),
-                        context.lookup()
+                        context.lookup(),
+                        OnScriptError.FAIL
                     );
                     DoubleFieldScript doubleFieldScript = leafFactory.newInstance(leafReaderContext);
                     List<Double> doubles = new ArrayList<>();
@@ -597,7 +601,8 @@ static Response innerShardOperation(Request request, ScriptService scriptService
                     GeoPointFieldScript.LeafFactory leafFactory = factory.newFactory(
                         GeoPointFieldScript.CONTEXT.name,
                         request.getScript().getParams(),
-                        context.lookup()
+                        context.lookup(),
+                        OnScriptError.FAIL
                     );
                     GeoPointFieldScript geoPointFieldScript = leafFactory.newInstance(leafReaderContext);
                     List<GeoPoint> points = new ArrayList<>();
@@ -615,7 +620,8 @@ static Response innerShardOperation(Request request, ScriptService scriptService
                     IpFieldScript.LeafFactory leafFactory = factory.newFactory(
                         IpFieldScript.CONTEXT.name,
                         request.getScript().getParams(),
-                        context.lookup()
+                        context.lookup(),
+                        OnScriptError.FAIL
                     );
                     IpFieldScript ipFieldScript = leafFactory.newInstance(leafReaderContext);
                     List<String> ips = new ArrayList<>();
@@ -634,7 +640,8 @@ static Response innerShardOperation(Request request, ScriptService scriptService
                     LongFieldScript.LeafFactory leafFactory = factory.newFactory(
                         LongFieldScript.CONTEXT.name,
                         request.getScript().getParams(),
-                        context.lookup()
+                        context.lookup(),
+                        OnScriptError.FAIL
                     );
                     LongFieldScript longFieldScript = leafFactory.newInstance(leafReaderContext);
                     List<Long> longs = new ArrayList<>();
@@ -647,7 +654,8 @@ static Response innerShardOperation(Request request, ScriptService scriptService
                     StringFieldScript.LeafFactory leafFactory = factory.newFactory(
                         StringFieldScript.CONTEXT.name,
                         request.getScript().getParams(),
-                        context.lookup()
+                        context.lookup(),
+                        OnScriptError.FAIL
                     );
                     StringFieldScript stringFieldScript = leafFactory.newInstance(leafReaderContext);
                     List<String> keywords = new ArrayList<>();
@@ -660,10 +668,12 @@ static Response innerShardOperation(Request request, ScriptService scriptService
                     CompositeFieldScript.LeafFactory leafFactory = factory.newFactory(
                         CompositeFieldScript.CONTEXT.name,
                         request.getScript().getParams(),
-                        context.lookup()
+                        context.lookup(),
+                        OnScriptError.FAIL
                     );
                     CompositeFieldScript compositeFieldScript = leafFactory.newInstance(leafReaderContext);
-                    return new Response(compositeFieldScript.runForDoc(0));
+                    compositeFieldScript.runForDoc(0);
+                    return new Response(compositeFieldScript.getFieldValues());
                 }, indexService);
             } else {
                 throw new UnsupportedOperationException("unsupported context [" + scriptContext.name + "]");
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
index 4e810af6b239..983327eac2d0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
@@ -1,17 +1,22 @@
 // ANTLR GENERATED CODE: DO NOT EDIT
 package org.elasticsearch.painless.antlr;
 
-import org.antlr.v4.runtime.*;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.Lexer;
-import org.antlr.v4.runtime.atn.*;
+import org.antlr.v4.runtime.RuleContext;
+import org.antlr.v4.runtime.RuntimeMetaData;
+import org.antlr.v4.runtime.Vocabulary;
+import org.antlr.v4.runtime.VocabularyImpl;
+import org.antlr.v4.runtime.atn.ATN;
+import org.antlr.v4.runtime.atn.ATNDeserializer;
+import org.antlr.v4.runtime.atn.LexerATNSimulator;
+import org.antlr.v4.runtime.atn.PredictionContextCache;
 import org.antlr.v4.runtime.dfa.DFA;
-import org.antlr.v4.runtime.misc.*;
 
-@SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast" })
+@SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue" })
 abstract class PainlessLexer extends Lexer {
     static {
-        RuntimeMetaData.checkVersion("4.5.3", RuntimeMetaData.VERSION);
+        RuntimeMetaData.checkVersion("4.11.1", RuntimeMetaData.VERSION);
     }
 
     protected static final DFA[] _decisionToDFA;
@@ -25,269 +30,284 @@ abstract class PainlessLexer extends Lexer {
                     68, AOR = 69, ALSH = 70, ARSH = 71, AUSH = 72, OCTAL = 73, HEX = 74, INTEGER = 75, DECIMAL = 76, STRING = 77, REGEX =
                         78, TRUE = 79, FALSE = 80, NULL = 81, PRIMITIVE = 82, DEF = 83, ID = 84, DOTINTEGER = 85, DOTID = 86;
     public static final int AFTER_DOT = 1;
+    public static String[] channelNames = { "DEFAULT_TOKEN_CHANNEL", "HIDDEN" };
+
     public static String[] modeNames = { "DEFAULT_MODE", "AFTER_DOT" };
 
-    public static final String[] ruleNames = {
-        "WS",
-        "COMMENT",
-        "LBRACK",
-        "RBRACK",
-        "LBRACE",
-        "RBRACE",
-        "LP",
-        "RP",
-        "DOLLAR",
-        "DOT",
-        "NSDOT",
-        "COMMA",
-        "SEMICOLON",
-        "IF",
-        "IN",
-        "ELSE",
-        "WHILE",
-        "DO",
-        "FOR",
-        "CONTINUE",
-        "BREAK",
-        "RETURN",
-        "NEW",
-        "TRY",
-        "CATCH",
-        "THROW",
-        "THIS",
-        "INSTANCEOF",
-        "BOOLNOT",
-        "BWNOT",
-        "MUL",
-        "DIV",
-        "REM",
-        "ADD",
-        "SUB",
-        "LSH",
-        "RSH",
-        "USH",
-        "LT",
-        "LTE",
-        "GT",
-        "GTE",
-        "EQ",
-        "EQR",
-        "NE",
-        "NER",
-        "BWAND",
-        "XOR",
-        "BWOR",
-        "BOOLAND",
-        "BOOLOR",
-        "COND",
-        "COLON",
-        "ELVIS",
-        "REF",
-        "ARROW",
-        "FIND",
-        "MATCH",
-        "INCR",
-        "DECR",
-        "ASSIGN",
-        "AADD",
-        "ASUB",
-        "AMUL",
-        "ADIV",
-        "AREM",
-        "AAND",
-        "AXOR",
-        "AOR",
-        "ALSH",
-        "ARSH",
-        "AUSH",
-        "OCTAL",
-        "HEX",
-        "INTEGER",
-        "DECIMAL",
-        "STRING",
-        "REGEX",
-        "TRUE",
-        "FALSE",
-        "NULL",
-        "PRIMITIVE",
-        "DEF",
-        "ID",
-        "DOTINTEGER",
-        "DOTID" };
+    private static String[] makeRuleNames() {
+        return new String[] {
+            "WS",
+            "COMMENT",
+            "LBRACK",
+            "RBRACK",
+            "LBRACE",
+            "RBRACE",
+            "LP",
+            "RP",
+            "DOLLAR",
+            "DOT",
+            "NSDOT",
+            "COMMA",
+            "SEMICOLON",
+            "IF",
+            "IN",
+            "ELSE",
+            "WHILE",
+            "DO",
+            "FOR",
+            "CONTINUE",
+            "BREAK",
+            "RETURN",
+            "NEW",
+            "TRY",
+            "CATCH",
+            "THROW",
+            "THIS",
+            "INSTANCEOF",
+            "BOOLNOT",
+            "BWNOT",
+            "MUL",
+            "DIV",
+            "REM",
+            "ADD",
+            "SUB",
+            "LSH",
+            "RSH",
+            "USH",
+            "LT",
+            "LTE",
+            "GT",
+            "GTE",
+            "EQ",
+            "EQR",
+            "NE",
+            "NER",
+            "BWAND",
+            "XOR",
+            "BWOR",
+            "BOOLAND",
+            "BOOLOR",
+            "COND",
+            "COLON",
+            "ELVIS",
+            "REF",
+            "ARROW",
+            "FIND",
+            "MATCH",
+            "INCR",
+            "DECR",
+            "ASSIGN",
+            "AADD",
+            "ASUB",
+            "AMUL",
+            "ADIV",
+            "AREM",
+            "AAND",
+            "AXOR",
+            "AOR",
+            "ALSH",
+            "ARSH",
+            "AUSH",
+            "OCTAL",
+            "HEX",
+            "INTEGER",
+            "DECIMAL",
+            "STRING",
+            "REGEX",
+            "TRUE",
+            "FALSE",
+            "NULL",
+            "PRIMITIVE",
+            "DEF",
+            "ID",
+            "DOTINTEGER",
+            "DOTID" };
+    }
+
+    public static final String[] ruleNames = makeRuleNames();
+
+    private static String[] makeLiteralNames() {
+        return new String[] {
+            null,
+            null,
+            null,
+            "'{'",
+            "'}'",
+            "'['",
+            "']'",
+            "'('",
+            "')'",
+            "'$'",
+            "'.'",
+            "'?.'",
+            "','",
+            "';'",
+            "'if'",
+            "'in'",
+            "'else'",
+            "'while'",
+            "'do'",
+            "'for'",
+            "'continue'",
+            "'break'",
+            "'return'",
+            "'new'",
+            "'try'",
+            "'catch'",
+            "'throw'",
+            "'this'",
+            "'instanceof'",
+            "'!'",
+            "'~'",
+            "'*'",
+            "'/'",
+            "'%'",
+            "'+'",
+            "'-'",
+            "'<<'",
+            "'>>'",
+            "'>>>'",
+            "'<'",
+            "'<='",
+            "'>'",
+            "'>='",
+            "'=='",
+            "'==='",
+            "'!='",
+            "'!=='",
+            "'&'",
+            "'^'",
+            "'|'",
+            "'&&'",
+            "'||'",
+            "'?'",
+            "':'",
+            "'?:'",
+            "'::'",
+            "'->'",
+            "'=~'",
+            "'==~'",
+            "'++'",
+            "'--'",
+            "'='",
+            "'+='",
+            "'-='",
+            "'*='",
+            "'/='",
+            "'%='",
+            "'&='",
+            "'^='",
+            "'|='",
+            "'<<='",
+            "'>>='",
+            "'>>>='",
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            "'true'",
+            "'false'",
+            "'null'",
+            null,
+            "'def'" };
+    }
+
+    private static final String[] _LITERAL_NAMES = makeLiteralNames();
 
-    private static final String[] _LITERAL_NAMES = {
-        null,
-        null,
-        null,
-        "'{'",
-        "'}'",
-        "'['",
-        "']'",
-        "'('",
-        "')'",
-        "'$'",
-        "'.'",
-        "'?.'",
-        "','",
-        "';'",
-        "'if'",
-        "'in'",
-        "'else'",
-        "'while'",
-        "'do'",
-        "'for'",
-        "'continue'",
-        "'break'",
-        "'return'",
-        "'new'",
-        "'try'",
-        "'catch'",
-        "'throw'",
-        "'this'",
-        "'instanceof'",
-        "'!'",
-        "'~'",
-        "'*'",
-        "'/'",
-        "'%'",
-        "'+'",
-        "'-'",
-        "'<<'",
-        "'>>'",
-        "'>>>'",
-        "'<'",
-        "'<='",
-        "'>'",
-        "'>='",
-        "'=='",
-        "'==='",
-        "'!='",
-        "'!=='",
-        "'&'",
-        "'^'",
-        "'|'",
-        "'&&'",
-        "'||'",
-        "'?'",
-        "':'",
-        "'?:'",
-        "'::'",
-        "'->'",
-        "'=~'",
-        "'==~'",
-        "'++'",
-        "'--'",
-        "'='",
-        "'+='",
-        "'-='",
-        "'*='",
-        "'/='",
-        "'%='",
-        "'&='",
-        "'^='",
-        "'|='",
-        "'<<='",
-        "'>>='",
-        "'>>>='",
-        null,
-        null,
-        null,
-        null,
-        null,
-        null,
-        "'true'",
-        "'false'",
-        "'null'",
-        null,
-        "'def'" };
-    private static final String[] _SYMBOLIC_NAMES = {
-        null,
-        "WS",
-        "COMMENT",
-        "LBRACK",
-        "RBRACK",
-        "LBRACE",
-        "RBRACE",
-        "LP",
-        "RP",
-        "DOLLAR",
-        "DOT",
-        "NSDOT",
-        "COMMA",
-        "SEMICOLON",
-        "IF",
-        "IN",
-        "ELSE",
-        "WHILE",
-        "DO",
-        "FOR",
-        "CONTINUE",
-        "BREAK",
-        "RETURN",
-        "NEW",
-        "TRY",
-        "CATCH",
-        "THROW",
-        "THIS",
-        "INSTANCEOF",
-        "BOOLNOT",
-        "BWNOT",
-        "MUL",
-        "DIV",
-        "REM",
-        "ADD",
-        "SUB",
-        "LSH",
-        "RSH",
-        "USH",
-        "LT",
-        "LTE",
-        "GT",
-        "GTE",
-        "EQ",
-        "EQR",
-        "NE",
-        "NER",
-        "BWAND",
-        "XOR",
-        "BWOR",
-        "BOOLAND",
-        "BOOLOR",
-        "COND",
-        "COLON",
-        "ELVIS",
-        "REF",
-        "ARROW",
-        "FIND",
-        "MATCH",
-        "INCR",
-        "DECR",
-        "ASSIGN",
-        "AADD",
-        "ASUB",
-        "AMUL",
-        "ADIV",
-        "AREM",
-        "AAND",
-        "AXOR",
-        "AOR",
-        "ALSH",
-        "ARSH",
-        "AUSH",
-        "OCTAL",
-        "HEX",
-        "INTEGER",
-        "DECIMAL",
-        "STRING",
-        "REGEX",
-        "TRUE",
-        "FALSE",
-        "NULL",
-        "PRIMITIVE",
-        "DEF",
-        "ID",
-        "DOTINTEGER",
-        "DOTID" };
+    private static String[] makeSymbolicNames() {
+        return new String[] {
+            null,
+            "WS",
+            "COMMENT",
+            "LBRACK",
+            "RBRACK",
+            "LBRACE",
+            "RBRACE",
+            "LP",
+            "RP",
+            "DOLLAR",
+            "DOT",
+            "NSDOT",
+            "COMMA",
+            "SEMICOLON",
+            "IF",
+            "IN",
+            "ELSE",
+            "WHILE",
+            "DO",
+            "FOR",
+            "CONTINUE",
+            "BREAK",
+            "RETURN",
+            "NEW",
+            "TRY",
+            "CATCH",
+            "THROW",
+            "THIS",
+            "INSTANCEOF",
+            "BOOLNOT",
+            "BWNOT",
+            "MUL",
+            "DIV",
+            "REM",
+            "ADD",
+            "SUB",
+            "LSH",
+            "RSH",
+            "USH",
+            "LT",
+            "LTE",
+            "GT",
+            "GTE",
+            "EQ",
+            "EQR",
+            "NE",
+            "NER",
+            "BWAND",
+            "XOR",
+            "BWOR",
+            "BOOLAND",
+            "BOOLOR",
+            "COND",
+            "COLON",
+            "ELVIS",
+            "REF",
+            "ARROW",
+            "FIND",
+            "MATCH",
+            "INCR",
+            "DECR",
+            "ASSIGN",
+            "AADD",
+            "ASUB",
+            "AMUL",
+            "ADIV",
+            "AREM",
+            "AAND",
+            "AXOR",
+            "AOR",
+            "ALSH",
+            "ARSH",
+            "AUSH",
+            "OCTAL",
+            "HEX",
+            "INTEGER",
+            "DECIMAL",
+            "STRING",
+            "REGEX",
+            "TRUE",
+            "FALSE",
+            "NULL",
+            "PRIMITIVE",
+            "DEF",
+            "ID",
+            "DOTINTEGER",
+            "DOTID" };
+    }
+
+    private static final String[] _SYMBOLIC_NAMES = makeSymbolicNames();
     public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
     /**
@@ -344,6 +364,11 @@ public String getSerializedATN() {
         return _serializedATN;
     }
 
+    @Override
+    public String[] getChannelNames() {
+        return channelNames;
+    }
+
     @Override
     public String[] getModeNames() {
         return modeNames;
@@ -381,229 +406,408 @@ private boolean REGEX_sempred(RuleContext _localctx, int predIndex) {
         return true;
     }
 
-    public static final String _serializedATN = "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2X\u027e\b\1\b\1\4"
-        + "\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n"
-        + "\4\13\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22"
-        + "\t\22\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31"
-        + "\t\31\4\32\t\32\4\33\t\33\4\34\t\34\4\35\t\35\4\36\t\36\4\37\t\37\4 \t"
-        + " \4!\t!\4\"\t\"\4#\t#\4$\t$\4%\t%\4&\t&\4\'\t\'\4(\t(\4)\t)\4*\t*\4+\t"
-        + "+\4,\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\4\61\t\61\4\62\t\62\4\63\t\63\4\64"
-        + "\t\64\4\65\t\65\4\66\t\66\4\67\t\67\48\t8\49\t9\4:\t:\4;\t;\4<\t<\4=\t"
-        + "=\4>\t>\4?\t?\4@\t@\4A\tA\4B\tB\4C\tC\4D\tD\4E\tE\4F\tF\4G\tG\4H\tH\4"
-        + "I\tI\4J\tJ\4K\tK\4L\tL\4M\tM\4N\tN\4O\tO\4P\tP\4Q\tQ\4R\tR\4S\tS\4T\t"
-        + "T\4U\tU\4V\tV\4W\tW\3\2\6\2\u00b2\n\2\r\2\16\2\u00b3\3\2\3\2\3\3\3\3\3"
-        + "\3\3\3\7\3\u00bc\n\3\f\3\16\3\u00bf\13\3\3\3\3\3\3\3\3\3\3\3\7\3\u00c6"
-        + "\n\3\f\3\16\3\u00c9\13\3\3\3\3\3\5\3\u00cd\n\3\3\3\3\3\3\4\3\4\3\5\3\5"
-        + "\3\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3\13\3\13\3\13\3\13\3\f\3\f\3"
-        + "\f\3\f\3\f\3\r\3\r\3\16\3\16\3\17\3\17\3\17\3\20\3\20\3\20\3\21\3\21\3"
-        + "\21\3\21\3\21\3\22\3\22\3\22\3\22\3\22\3\22\3\23\3\23\3\23\3\24\3\24\3"
-        + "\24\3\24\3\25\3\25\3\25\3\25\3\25\3\25\3\25\3\25\3\25\3\26\3\26\3\26\3"
-        + "\26\3\26\3\26\3\27\3\27\3\27\3\27\3\27\3\27\3\27\3\30\3\30\3\30\3\30\3"
-        + "\31\3\31\3\31\3\31\3\32\3\32\3\32\3\32\3\32\3\32\3\33\3\33\3\33\3\33\3"
-        + "\33\3\33\3\34\3\34\3\34\3\34\3\34\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3"
-        + "\35\3\35\3\35\3\35\3\36\3\36\3\37\3\37\3 \3 \3!\3!\3!\3\"\3\"\3#\3#\3"
-        + "$\3$\3%\3%\3%\3&\3&\3&\3\'\3\'\3\'\3\'\3(\3(\3)\3)\3)\3*\3*\3+\3+\3+\3"
-        + ",\3,\3,\3-\3-\3-\3-\3.\3.\3.\3/\3/\3/\3/\3\60\3\60\3\61\3\61\3\62\3\62"
-        + "\3\63\3\63\3\63\3\64\3\64\3\64\3\65\3\65\3\66\3\66\3\67\3\67\3\67\38\3"
-        + "8\38\39\39\39\3:\3:\3:\3;\3;\3;\3;\3<\3<\3<\3=\3=\3=\3>\3>\3?\3?\3?\3"
-        + "@\3@\3@\3A\3A\3A\3B\3B\3B\3C\3C\3C\3D\3D\3D\3E\3E\3E\3F\3F\3F\3G\3G\3"
-        + "G\3G\3H\3H\3H\3H\3I\3I\3I\3I\3I\3J\3J\6J\u01be\nJ\rJ\16J\u01bf\3J\5J\u01c3"
-        + "\nJ\3K\3K\3K\6K\u01c8\nK\rK\16K\u01c9\3K\5K\u01cd\nK\3L\3L\3L\7L\u01d2"
-        + "\nL\fL\16L\u01d5\13L\5L\u01d7\nL\3L\5L\u01da\nL\3M\3M\3M\7M\u01df\nM\f"
-        + "M\16M\u01e2\13M\5M\u01e4\nM\3M\3M\6M\u01e8\nM\rM\16M\u01e9\5M\u01ec\n"
-        + "M\3M\3M\5M\u01f0\nM\3M\6M\u01f3\nM\rM\16M\u01f4\5M\u01f7\nM\3M\5M\u01fa"
-        + "\nM\3N\3N\3N\3N\3N\3N\7N\u0202\nN\fN\16N\u0205\13N\3N\3N\3N\3N\3N\3N\3"
-        + "N\7N\u020e\nN\fN\16N\u0211\13N\3N\5N\u0214\nN\3O\3O\3O\3O\6O\u021a\nO"
-        + "\rO\16O\u021b\3O\3O\7O\u0220\nO\fO\16O\u0223\13O\3O\3O\3P\3P\3P\3P\3P"
-        + "\3Q\3Q\3Q\3Q\3Q\3Q\3R\3R\3R\3R\3R\3S\3S\3S\3S\3S\3S\3S\3S\3S\3S\3S\3S"
-        + "\3S\3S\3S\3S\3S\3S\3S\3S\3S\3S\3S\3S\3S\3S\3S\3S\3S\3S\3S\3S\3S\3S\3S"
-        + "\3S\3S\3S\5S\u025d\nS\3T\3T\3T\3T\3U\3U\7U\u0265\nU\fU\16U\u0268\13U\3"
-        + "V\3V\3V\7V\u026d\nV\fV\16V\u0270\13V\5V\u0272\nV\3V\3V\3W\3W\7W\u0278"
-        + "\nW\fW\16W\u027b\13W\3W\3W\7\u00bd\u00c7\u0203\u020f\u021b\2X\4\3\6\4"
-        + "\b\5\n\6\f\7\16\b\20\t\22\n\24\13\26\f\30\r\32\16\34\17\36\20 \21\"\22"
-        + "$\23&\24(\25*\26,\27.\30\60\31\62\32\64\33\66\348\35:\36<\37> @!B\"D#"
-        + "F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60`\61b\62d\63f\64h\65j\66l\67n8p9r:t;v<x"
-        + "=z>|?~@\u0080A\u0082B\u0084C\u0086D\u0088E\u008aF\u008cG\u008eH\u0090"
-        + "I\u0092J\u0094K\u0096L\u0098M\u009aN\u009cO\u009eP\u00a0Q\u00a2R\u00a4"
-        + "S\u00a6T\u00a8U\u00aaV\u00acW\u00aeX\4\2\3\25\5\2\13\f\17\17\"\"\4\2\f"
-        + "\f\17\17\3\2\629\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2\63;\3\2\62;\b\2FFHHN"
-        + "Nffhhnn\4\2GGgg\4\2--//\6\2FFHHffhh\4\2$$^^\4\2))^^\3\2\f\f\4\2\f\f\61"
-        + "\61\t\2WWeekknouuwwzz\5\2C\\aac|\6\2\62;C\\aac|\u02a4\2\4\3\2\2\2\2\6"
-        + "\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2\2\2\2\16\3\2\2\2\2\20\3\2\2\2"
-        + "\2\22\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2\2\30\3\2\2\2\2\32\3\2\2\2\2\34"
-        + "\3\2\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3\2\2\2\2$\3\2\2\2\2&\3\2\2\2\2("
-        + "\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2\2\2\60\3\2\2\2\2\62\3\2\2\2\2"
-        + "\64\3\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3\2\2\2\2<\3\2\2\2\2>\3\2\2\2\2"
-        + "@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2\2\2H\3\2\2\2\2J\3\2\2\2\2L\3"
-        + "\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2T\3\2\2\2\2V\3\2\2\2\2X\3\2\2"
-        + "\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3\2\2\2\2b\3\2\2\2\2d\3\2\2\2"
-        + "\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2\2\2n\3\2\2\2\2p\3\2\2\2\2r"
-        + "\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2z\3\2\2\2\2|\3\2\2\2\2~\3\2"
-        + "\2\2\2\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084\3\2\2\2\2\u0086\3\2\2\2\2"
-        + "\u0088\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2\2\2\u008e\3\2\2\2\2\u0090"
-        + "\3\2\2\2\2\u0092\3\2\2\2\2\u0094\3\2\2\2\2\u0096\3\2\2\2\2\u0098\3\2\2"
-        + "\2\2\u009a\3\2\2\2\2\u009c\3\2\2\2\2\u009e\3\2\2\2\2\u00a0\3\2\2\2\2\u00a2"
-        + "\3\2\2\2\2\u00a4\3\2\2\2\2\u00a6\3\2\2\2\2\u00a8\3\2\2\2\2\u00aa\3\2\2"
-        + "\2\3\u00ac\3\2\2\2\3\u00ae\3\2\2\2\4\u00b1\3\2\2\2\6\u00cc\3\2\2\2\b\u00d0"
-        + "\3\2\2\2\n\u00d2\3\2\2\2\f\u00d4\3\2\2\2\16\u00d6\3\2\2\2\20\u00d8\3\2"
-        + "\2\2\22\u00da\3\2\2\2\24\u00dc\3\2\2\2\26\u00de\3\2\2\2\30\u00e2\3\2\2"
-        + "\2\32\u00e7\3\2\2\2\34\u00e9\3\2\2\2\36\u00eb\3\2\2\2 \u00ee\3\2\2\2\""
-        + "\u00f1\3\2\2\2$\u00f6\3\2\2\2&\u00fc\3\2\2\2(\u00ff\3\2\2\2*\u0103\3\2"
-        + "\2\2,\u010c\3\2\2\2.\u0112\3\2\2\2\60\u0119\3\2\2\2\62\u011d\3\2\2\2\64"
-        + "\u0121\3\2\2\2\66\u0127\3\2\2\28\u012d\3\2\2\2:\u0132\3\2\2\2<\u013d\3"
-        + "\2\2\2>\u013f\3\2\2\2@\u0141\3\2\2\2B\u0143\3\2\2\2D\u0146\3\2\2\2F\u0148"
-        + "\3\2\2\2H\u014a\3\2\2\2J\u014c\3\2\2\2L\u014f\3\2\2\2N\u0152\3\2\2\2P"
-        + "\u0156\3\2\2\2R\u0158\3\2\2\2T\u015b\3\2\2\2V\u015d\3\2\2\2X\u0160\3\2"
-        + "\2\2Z\u0163\3\2\2\2\\\u0167\3\2\2\2^\u016a\3\2\2\2`\u016e\3\2\2\2b\u0170"
-        + "\3\2\2\2d\u0172\3\2\2\2f\u0174\3\2\2\2h\u0177\3\2\2\2j\u017a\3\2\2\2l"
-        + "\u017c\3\2\2\2n\u017e\3\2\2\2p\u0181\3\2\2\2r\u0184\3\2\2\2t\u0187\3\2"
-        + "\2\2v\u018a\3\2\2\2x\u018e\3\2\2\2z\u0191\3\2\2\2|\u0194\3\2\2\2~\u0196"
-        + "\3\2\2\2\u0080\u0199\3\2\2\2\u0082\u019c\3\2\2\2\u0084\u019f\3\2\2\2\u0086"
-        + "\u01a2\3\2\2\2\u0088\u01a5\3\2\2\2\u008a\u01a8\3\2\2\2\u008c\u01ab\3\2"
-        + "\2\2\u008e\u01ae\3\2\2\2\u0090\u01b2\3\2\2\2\u0092\u01b6\3\2\2\2\u0094"
-        + "\u01bb\3\2\2\2\u0096\u01c4\3\2\2\2\u0098\u01d6\3\2\2\2\u009a\u01e3\3\2"
-        + "\2\2\u009c\u0213\3\2\2\2\u009e\u0215\3\2\2\2\u00a0\u0226\3\2\2\2\u00a2"
-        + "\u022b\3\2\2\2\u00a4\u0231\3\2\2\2\u00a6\u025c\3\2\2\2\u00a8\u025e\3\2"
-        + "\2\2\u00aa\u0262\3\2\2\2\u00ac\u0271\3\2\2\2\u00ae\u0275\3\2\2\2\u00b0"
-        + "\u00b2\t\2\2\2\u00b1\u00b0\3\2\2\2\u00b2\u00b3\3\2\2\2\u00b3\u00b1\3\2"
-        + "\2\2\u00b3\u00b4\3\2\2\2\u00b4\u00b5\3\2\2\2\u00b5\u00b6\b\2\2\2\u00b6"
-        + "\5\3\2\2\2\u00b7\u00b8\7\61\2\2\u00b8\u00b9\7\61\2\2\u00b9\u00bd\3\2\2"
-        + "\2\u00ba\u00bc\13\2\2\2\u00bb\u00ba\3\2\2\2\u00bc\u00bf\3\2\2\2\u00bd"
-        + "\u00be\3\2\2\2\u00bd\u00bb\3\2\2\2\u00be\u00c0\3\2\2\2\u00bf\u00bd\3\2"
-        + "\2\2\u00c0\u00cd\t\3\2\2\u00c1\u00c2\7\61\2\2\u00c2\u00c3\7,\2\2\u00c3"
-        + "\u00c7\3\2\2\2\u00c4\u00c6\13\2\2\2\u00c5\u00c4\3\2\2\2\u00c6\u00c9\3"
-        + "\2\2\2\u00c7\u00c8\3\2\2\2\u00c7\u00c5\3\2\2\2\u00c8\u00ca\3\2\2\2\u00c9"
-        + "\u00c7\3\2\2\2\u00ca\u00cb\7,\2\2\u00cb\u00cd\7\61\2\2\u00cc\u00b7\3\2"
-        + "\2\2\u00cc\u00c1\3\2\2\2\u00cd\u00ce\3\2\2\2\u00ce\u00cf\b\3\2\2\u00cf"
-        + "\7\3\2\2\2\u00d0\u00d1\7}\2\2\u00d1\t\3\2\2\2\u00d2\u00d3\7\177\2\2\u00d3"
-        + "\13\3\2\2\2\u00d4\u00d5\7]\2\2\u00d5\r\3\2\2\2\u00d6\u00d7\7_\2\2\u00d7"
-        + "\17\3\2\2\2\u00d8\u00d9\7*\2\2\u00d9\21\3\2\2\2\u00da\u00db\7+\2\2\u00db"
-        + "\23\3\2\2\2\u00dc\u00dd\7&\2\2\u00dd\25\3\2\2\2\u00de\u00df\7\60\2\2\u00df"
-        + "\u00e0\3\2\2\2\u00e0\u00e1\b\13\3\2\u00e1\27\3\2\2\2\u00e2\u00e3\7A\2"
-        + "\2\u00e3\u00e4\7\60\2\2\u00e4\u00e5\3\2\2\2\u00e5\u00e6\b\f\3\2\u00e6"
-        + "\31\3\2\2\2\u00e7\u00e8\7.\2\2\u00e8\33\3\2\2\2\u00e9\u00ea\7=\2\2\u00ea"
-        + "\35\3\2\2\2\u00eb\u00ec\7k\2\2\u00ec\u00ed\7h\2\2\u00ed\37\3\2\2\2\u00ee"
-        + "\u00ef\7k\2\2\u00ef\u00f0\7p\2\2\u00f0!\3\2\2\2\u00f1\u00f2\7g\2\2\u00f2"
-        + "\u00f3\7n\2\2\u00f3\u00f4\7u\2\2\u00f4\u00f5\7g\2\2\u00f5#\3\2\2\2\u00f6"
-        + "\u00f7\7y\2\2\u00f7\u00f8\7j\2\2\u00f8\u00f9\7k\2\2\u00f9\u00fa\7n\2\2"
-        + "\u00fa\u00fb\7g\2\2\u00fb%\3\2\2\2\u00fc\u00fd\7f\2\2\u00fd\u00fe\7q\2"
-        + "\2\u00fe\'\3\2\2\2\u00ff\u0100\7h\2\2\u0100\u0101\7q\2\2\u0101\u0102\7"
-        + "t\2\2\u0102)\3\2\2\2\u0103\u0104\7e\2\2\u0104\u0105\7q\2\2\u0105\u0106"
-        + "\7p\2\2\u0106\u0107\7v\2\2\u0107\u0108\7k\2\2\u0108\u0109\7p\2\2\u0109"
-        + "\u010a\7w\2\2\u010a\u010b\7g\2\2\u010b+\3\2\2\2\u010c\u010d\7d\2\2\u010d"
-        + "\u010e\7t\2\2\u010e\u010f\7g\2\2\u010f\u0110\7c\2\2\u0110\u0111\7m\2\2"
-        + "\u0111-\3\2\2\2\u0112\u0113\7t\2\2\u0113\u0114\7g\2\2\u0114\u0115\7v\2"
-        + "\2\u0115\u0116\7w\2\2\u0116\u0117\7t\2\2\u0117\u0118\7p\2\2\u0118/\3\2"
-        + "\2\2\u0119\u011a\7p\2\2\u011a\u011b\7g\2\2\u011b\u011c\7y\2\2\u011c\61"
-        + "\3\2\2\2\u011d\u011e\7v\2\2\u011e\u011f\7t\2\2\u011f\u0120\7{\2\2\u0120"
-        + "\63\3\2\2\2\u0121\u0122\7e\2\2\u0122\u0123\7c\2\2\u0123\u0124\7v\2\2\u0124"
-        + "\u0125\7e\2\2\u0125\u0126\7j\2\2\u0126\65\3\2\2\2\u0127\u0128\7v\2\2\u0128"
-        + "\u0129\7j\2\2\u0129\u012a\7t\2\2\u012a\u012b\7q\2\2\u012b\u012c\7y\2\2"
-        + "\u012c\67\3\2\2\2\u012d\u012e\7v\2\2\u012e\u012f\7j\2\2\u012f\u0130\7"
-        + "k\2\2\u0130\u0131\7u\2\2\u01319\3\2\2\2\u0132\u0133\7k\2\2\u0133\u0134"
-        + "\7p\2\2\u0134\u0135\7u\2\2\u0135\u0136\7v\2\2\u0136\u0137\7c\2\2\u0137"
-        + "\u0138\7p\2\2\u0138\u0139\7e\2\2\u0139\u013a\7g\2\2\u013a\u013b\7q\2\2"
-        + "\u013b\u013c\7h\2\2\u013c;\3\2\2\2\u013d\u013e\7#\2\2\u013e=\3\2\2\2\u013f"
-        + "\u0140\7\u0080\2\2\u0140?\3\2\2\2\u0141\u0142\7,\2\2\u0142A\3\2\2\2\u0143"
-        + "\u0144\7\61\2\2\u0144\u0145\6!\2\2\u0145C\3\2\2\2\u0146\u0147\7\'\2\2"
-        + "\u0147E\3\2\2\2\u0148\u0149\7-\2\2\u0149G\3\2\2\2\u014a\u014b\7/\2\2\u014b"
-        + "I\3\2\2\2\u014c\u014d\7>\2\2\u014d\u014e\7>\2\2\u014eK\3\2\2\2\u014f\u0150"
-        + "\7@\2\2\u0150\u0151\7@\2\2\u0151M\3\2\2\2\u0152\u0153\7@\2\2\u0153\u0154"
-        + "\7@\2\2\u0154\u0155\7@\2\2\u0155O\3\2\2\2\u0156\u0157\7>\2\2\u0157Q\3"
-        + "\2\2\2\u0158\u0159\7>\2\2\u0159\u015a\7?\2\2\u015aS\3\2\2\2\u015b\u015c"
-        + "\7@\2\2\u015cU\3\2\2\2\u015d\u015e\7@\2\2\u015e\u015f\7?\2\2\u015fW\3"
-        + "\2\2\2\u0160\u0161\7?\2\2\u0161\u0162\7?\2\2\u0162Y\3\2\2\2\u0163\u0164"
-        + "\7?\2\2\u0164\u0165\7?\2\2\u0165\u0166\7?\2\2\u0166[\3\2\2\2\u0167\u0168"
-        + "\7#\2\2\u0168\u0169\7?\2\2\u0169]\3\2\2\2\u016a\u016b\7#\2\2\u016b\u016c"
-        + "\7?\2\2\u016c\u016d\7?\2\2\u016d_\3\2\2\2\u016e\u016f\7(\2\2\u016fa\3"
-        + "\2\2\2\u0170\u0171\7`\2\2\u0171c\3\2\2\2\u0172\u0173\7~\2\2\u0173e\3\2"
-        + "\2\2\u0174\u0175\7(\2\2\u0175\u0176\7(\2\2\u0176g\3\2\2\2\u0177\u0178"
-        + "\7~\2\2\u0178\u0179\7~\2\2\u0179i\3\2\2\2\u017a\u017b\7A\2\2\u017bk\3"
-        + "\2\2\2\u017c\u017d\7<\2\2\u017dm\3\2\2\2\u017e\u017f\7A\2\2\u017f\u0180"
-        + "\7<\2\2\u0180o\3\2\2\2\u0181\u0182\7<\2\2\u0182\u0183\7<\2\2\u0183q\3"
-        + "\2\2\2\u0184\u0185\7/\2\2\u0185\u0186\7@\2\2\u0186s\3\2\2\2\u0187\u0188"
-        + "\7?\2\2\u0188\u0189\7\u0080\2\2\u0189u\3\2\2\2\u018a\u018b\7?\2\2\u018b"
-        + "\u018c\7?\2\2\u018c\u018d\7\u0080\2\2\u018dw\3\2\2\2\u018e\u018f\7-\2"
-        + "\2\u018f\u0190\7-\2\2\u0190y\3\2\2\2\u0191\u0192\7/\2\2\u0192\u0193\7"
-        + "/\2\2\u0193{\3\2\2\2\u0194\u0195\7?\2\2\u0195}\3\2\2\2\u0196\u0197\7-"
-        + "\2\2\u0197\u0198\7?\2\2\u0198\177\3\2\2\2\u0199\u019a\7/\2\2\u019a\u019b"
-        + "\7?\2\2\u019b\u0081\3\2\2\2\u019c\u019d\7,\2\2\u019d\u019e\7?\2\2\u019e"
-        + "\u0083\3\2\2\2\u019f\u01a0\7\61\2\2\u01a0\u01a1\7?\2\2\u01a1\u0085\3\2"
-        + "\2\2\u01a2\u01a3\7\'\2\2\u01a3\u01a4\7?\2\2\u01a4\u0087\3\2\2\2\u01a5"
-        + "\u01a6\7(\2\2\u01a6\u01a7\7?\2\2\u01a7\u0089\3\2\2\2\u01a8\u01a9\7`\2"
-        + "\2\u01a9\u01aa\7?\2\2\u01aa\u008b\3\2\2\2\u01ab\u01ac\7~\2\2\u01ac\u01ad"
-        + "\7?\2\2\u01ad\u008d\3\2\2\2\u01ae\u01af\7>\2\2\u01af\u01b0\7>\2\2\u01b0"
-        + "\u01b1\7?\2\2\u01b1\u008f\3\2\2\2\u01b2\u01b3\7@\2\2\u01b3\u01b4\7@\2"
-        + "\2\u01b4\u01b5\7?\2\2\u01b5\u0091\3\2\2\2\u01b6\u01b7\7@\2\2\u01b7\u01b8"
-        + "\7@\2\2\u01b8\u01b9\7@\2\2\u01b9\u01ba\7?\2\2\u01ba\u0093\3\2\2\2\u01bb"
-        + "\u01bd\7\62\2\2\u01bc\u01be\t\4\2\2\u01bd\u01bc\3\2\2\2\u01be\u01bf\3"
-        + "\2\2\2\u01bf\u01bd\3\2\2\2\u01bf\u01c0\3\2\2\2\u01c0\u01c2\3\2\2\2\u01c1"
-        + "\u01c3\t\5\2\2\u01c2\u01c1\3\2\2\2\u01c2\u01c3\3\2\2\2\u01c3\u0095\3\2"
-        + "\2\2\u01c4\u01c5\7\62\2\2\u01c5\u01c7\t\6\2\2\u01c6\u01c8\t\7\2\2\u01c7"
-        + "\u01c6\3\2\2\2\u01c8\u01c9\3\2\2\2\u01c9\u01c7\3\2\2\2\u01c9\u01ca\3\2"
-        + "\2\2\u01ca\u01cc\3\2\2\2\u01cb\u01cd\t\5\2\2\u01cc\u01cb\3\2\2\2\u01cc"
-        + "\u01cd\3\2\2\2\u01cd\u0097\3\2\2\2\u01ce\u01d7\7\62\2\2\u01cf\u01d3\t"
-        + "\b\2\2\u01d0\u01d2\t\t\2\2\u01d1\u01d0\3\2\2\2\u01d2\u01d5\3\2\2\2\u01d3"
-        + "\u01d1\3\2\2\2\u01d3\u01d4\3\2\2\2\u01d4\u01d7\3\2\2\2\u01d5\u01d3\3\2"
-        + "\2\2\u01d6\u01ce\3\2\2\2\u01d6\u01cf\3\2\2\2\u01d7\u01d9\3\2\2\2\u01d8"
-        + "\u01da\t\n\2\2\u01d9\u01d8\3\2\2\2\u01d9\u01da\3\2\2\2\u01da\u0099\3\2"
-        + "\2\2\u01db\u01e4\7\62\2\2\u01dc\u01e0\t\b\2\2\u01dd\u01df\t\t\2\2\u01de"
-        + "\u01dd\3\2\2\2\u01df\u01e2\3\2\2\2\u01e0\u01de\3\2\2\2\u01e0\u01e1\3\2"
-        + "\2\2\u01e1\u01e4\3\2\2\2\u01e2\u01e0\3\2\2\2\u01e3\u01db\3\2\2\2\u01e3"
-        + "\u01dc\3\2\2\2\u01e4\u01eb\3\2\2\2\u01e5\u01e7\5\26\13\2\u01e6\u01e8\t"
-        + "\t\2\2\u01e7\u01e6\3\2\2\2\u01e8\u01e9\3\2\2\2\u01e9\u01e7\3\2\2\2\u01e9"
-        + "\u01ea\3\2\2\2\u01ea\u01ec\3\2\2\2\u01eb\u01e5\3\2\2\2\u01eb\u01ec\3\2"
-        + "\2\2\u01ec\u01f6\3\2\2\2\u01ed\u01ef\t\13\2\2\u01ee\u01f0\t\f\2\2\u01ef"
-        + "\u01ee\3\2\2\2\u01ef\u01f0\3\2\2\2\u01f0\u01f2\3\2\2\2\u01f1\u01f3\t\t"
-        + "\2\2\u01f2\u01f1\3\2\2\2\u01f3\u01f4\3\2\2\2\u01f4\u01f2\3\2\2\2\u01f4"
-        + "\u01f5\3\2\2\2\u01f5\u01f7\3\2\2\2\u01f6\u01ed\3\2\2\2\u01f6\u01f7\3\2"
-        + "\2\2\u01f7\u01f9\3\2\2\2\u01f8\u01fa\t\r\2\2\u01f9\u01f8\3\2\2\2\u01f9"
-        + "\u01fa\3\2\2\2\u01fa\u009b\3\2\2\2\u01fb\u0203\7$\2\2\u01fc\u01fd\7^\2"
-        + "\2\u01fd\u0202\7$\2\2\u01fe\u01ff\7^\2\2\u01ff\u0202\7^\2\2\u0200\u0202"
-        + "\n\16\2\2\u0201\u01fc\3\2\2\2\u0201\u01fe\3\2\2\2\u0201\u0200\3\2\2\2"
-        + "\u0202\u0205\3\2\2\2\u0203\u0204\3\2\2\2\u0203\u0201\3\2\2\2\u0204\u0206"
-        + "\3\2\2\2\u0205\u0203\3\2\2\2\u0206\u0214\7$\2\2\u0207\u020f\7)\2\2\u0208"
-        + "\u0209\7^\2\2\u0209\u020e\7)\2\2\u020a\u020b\7^\2\2\u020b\u020e\7^\2\2"
-        + "\u020c\u020e\n\17\2\2\u020d\u0208\3\2\2\2\u020d\u020a\3\2\2\2\u020d\u020c"
-        + "\3\2\2\2\u020e\u0211\3\2\2\2\u020f\u0210\3\2\2\2\u020f\u020d\3\2\2\2\u0210"
-        + "\u0212\3\2\2\2\u0211\u020f\3\2\2\2\u0212\u0214\7)\2\2\u0213\u01fb\3\2"
-        + "\2\2\u0213\u0207\3\2\2\2\u0214\u009d\3\2\2\2\u0215\u0219\7\61\2\2\u0216"
-        + "\u0217\7^\2\2\u0217\u021a\n\20\2\2\u0218\u021a\n\21\2\2\u0219\u0216\3"
-        + "\2\2\2\u0219\u0218\3\2\2\2\u021a\u021b\3\2\2\2\u021b\u021c\3\2\2\2\u021b"
-        + "\u0219\3\2\2\2\u021c\u021d\3\2\2\2\u021d\u0221\7\61\2\2\u021e\u0220\t"
-        + "\22\2\2\u021f\u021e\3\2\2\2\u0220\u0223\3\2\2\2\u0221\u021f\3\2\2\2\u0221"
-        + "\u0222\3\2\2\2\u0222\u0224\3\2\2\2\u0223\u0221\3\2\2\2\u0224\u0225\6O"
-        + "\3\2\u0225\u009f\3\2\2\2\u0226\u0227\7v\2\2\u0227\u0228\7t\2\2\u0228\u0229"
-        + "\7w\2\2\u0229\u022a\7g\2\2\u022a\u00a1\3\2\2\2\u022b\u022c\7h\2\2\u022c"
-        + "\u022d\7c\2\2\u022d\u022e\7n\2\2\u022e\u022f\7u\2\2\u022f\u0230\7g\2\2"
-        + "\u0230\u00a3\3\2\2\2\u0231\u0232\7p\2\2\u0232\u0233\7w\2\2\u0233\u0234"
-        + "\7n\2\2\u0234\u0235\7n\2\2\u0235\u00a5\3\2\2\2\u0236\u0237\7d\2\2\u0237"
-        + "\u0238\7q\2\2\u0238\u0239\7q\2\2\u0239\u023a\7n\2\2\u023a\u023b\7g\2\2"
-        + "\u023b\u023c\7c\2\2\u023c\u025d\7p\2\2\u023d\u023e\7d\2\2\u023e\u023f"
-        + "\7{\2\2\u023f\u0240\7v\2\2\u0240\u025d\7g\2\2\u0241\u0242\7u\2\2\u0242"
-        + "\u0243\7j\2\2\u0243\u0244\7q\2\2\u0244\u0245\7t\2\2\u0245\u025d\7v\2\2"
-        + "\u0246\u0247\7e\2\2\u0247\u0248\7j\2\2\u0248\u0249\7c\2\2\u0249\u025d"
-        + "\7t\2\2\u024a\u024b\7k\2\2\u024b\u024c\7p\2\2\u024c\u025d\7v\2\2\u024d"
-        + "\u024e\7n\2\2\u024e\u024f\7q\2\2\u024f\u0250\7p\2\2\u0250\u025d\7i\2\2"
-        + "\u0251\u0252\7h\2\2\u0252\u0253\7n\2\2\u0253\u0254\7q\2\2\u0254\u0255"
-        + "\7c\2\2\u0255\u025d\7v\2\2\u0256\u0257\7f\2\2\u0257\u0258\7q\2\2\u0258"
-        + "\u0259\7w\2\2\u0259\u025a\7d\2\2\u025a\u025b\7n\2\2\u025b\u025d\7g\2\2"
-        + "\u025c\u0236\3\2\2\2\u025c\u023d\3\2\2\2\u025c\u0241\3\2\2\2\u025c\u0246"
-        + "\3\2\2\2\u025c\u024a\3\2\2\2\u025c\u024d\3\2\2\2\u025c\u0251\3\2\2\2\u025c"
-        + "\u0256\3\2\2\2\u025d\u00a7\3\2\2\2\u025e\u025f\7f\2\2\u025f\u0260\7g\2"
-        + "\2\u0260\u0261\7h\2\2\u0261\u00a9\3\2\2\2\u0262\u0266\t\23\2\2\u0263\u0265"
-        + "\t\24\2\2\u0264\u0263\3\2\2\2\u0265\u0268\3\2\2\2\u0266\u0264\3\2\2\2"
-        + "\u0266\u0267\3\2\2\2\u0267\u00ab\3\2\2\2\u0268\u0266\3\2\2\2\u0269\u0272"
-        + "\7\62\2\2\u026a\u026e\t\b\2\2\u026b\u026d\t\t\2\2\u026c\u026b\3\2\2\2"
-        + "\u026d\u0270\3\2\2\2\u026e\u026c\3\2\2\2\u026e\u026f\3\2\2\2\u026f\u0272"
-        + "\3\2\2\2\u0270\u026e\3\2\2\2\u0271\u0269\3\2\2\2\u0271\u026a\3\2\2\2\u0272"
-        + "\u0273\3\2\2\2\u0273\u0274\bV\4\2\u0274\u00ad\3\2\2\2\u0275\u0279\t\23"
-        + "\2\2\u0276\u0278\t\24\2\2\u0277\u0276\3\2\2\2\u0278\u027b\3\2\2\2\u0279"
-        + "\u0277\3\2\2\2\u0279\u027a\3\2\2\2\u027a\u027c\3\2\2\2\u027b\u0279\3\2"
-        + "\2\2\u027c\u027d\bW\4\2\u027d\u00af\3\2\2\2$\2\3\u00b3\u00bd\u00c7\u00cc"
-        + "\u01bf\u01c2\u01c9\u01cc\u01d3\u01d6\u01d9\u01e0\u01e3\u01e9\u01eb\u01ef"
-        + "\u01f4\u01f6\u01f9\u0201\u0203\u020d\u020f\u0213\u0219\u021b\u0221\u025c"
-        + "\u0266\u026e\u0271\u0279\5\b\2\2\4\3\2\4\2\2";
+    public static final String _serializedATN = "\u0004\u0000V\u027c\u0006\uffff\uffff\u0006\uffff\uffff\u0002\u0000\u0007"
+        + "\u0000\u0002\u0001\u0007\u0001\u0002\u0002\u0007\u0002\u0002\u0003\u0007"
+        + "\u0003\u0002\u0004\u0007\u0004\u0002\u0005\u0007\u0005\u0002\u0006\u0007"
+        + "\u0006\u0002\u0007\u0007\u0007\u0002\b\u0007\b\u0002\t\u0007\t\u0002\n"
+        + "\u0007\n\u0002\u000b\u0007\u000b\u0002\f\u0007\f\u0002\r\u0007\r\u0002"
+        + "\u000e\u0007\u000e\u0002\u000f\u0007\u000f\u0002\u0010\u0007\u0010\u0002"
+        + "\u0011\u0007\u0011\u0002\u0012\u0007\u0012\u0002\u0013\u0007\u0013\u0002"
+        + "\u0014\u0007\u0014\u0002\u0015\u0007\u0015\u0002\u0016\u0007\u0016\u0002"
+        + "\u0017\u0007\u0017\u0002\u0018\u0007\u0018\u0002\u0019\u0007\u0019\u0002"
+        + "\u001a\u0007\u001a\u0002\u001b\u0007\u001b\u0002\u001c\u0007\u001c\u0002"
+        + "\u001d\u0007\u001d\u0002\u001e\u0007\u001e\u0002\u001f\u0007\u001f\u0002"
+        + " \u0007 \u0002!\u0007!\u0002\"\u0007\"\u0002#\u0007#\u0002$\u0007$\u0002"
+        + "%\u0007%\u0002&\u0007&\u0002\'\u0007\'\u0002(\u0007(\u0002)\u0007)\u0002"
+        + "*\u0007*\u0002+\u0007+\u0002,\u0007,\u0002-\u0007-\u0002.\u0007.\u0002"
+        + "/\u0007/\u00020\u00070\u00021\u00071\u00022\u00072\u00023\u00073\u0002"
+        + "4\u00074\u00025\u00075\u00026\u00076\u00027\u00077\u00028\u00078\u0002"
+        + "9\u00079\u0002:\u0007:\u0002;\u0007;\u0002<\u0007<\u0002=\u0007=\u0002"
+        + ">\u0007>\u0002?\u0007?\u0002@\u0007@\u0002A\u0007A\u0002B\u0007B\u0002"
+        + "C\u0007C\u0002D\u0007D\u0002E\u0007E\u0002F\u0007F\u0002G\u0007G\u0002"
+        + "H\u0007H\u0002I\u0007I\u0002J\u0007J\u0002K\u0007K\u0002L\u0007L\u0002"
+        + "M\u0007M\u0002N\u0007N\u0002O\u0007O\u0002P\u0007P\u0002Q\u0007Q\u0002"
+        + "R\u0007R\u0002S\u0007S\u0002T\u0007T\u0002U\u0007U\u0001\u0000\u0004\u0000"
+        + "\u00b0\b\u0000\u000b\u0000\f\u0000\u00b1\u0001\u0000\u0001\u0000\u0001"
+        + "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0005\u0001\u00ba\b\u0001\n"
+        + "\u0001\f\u0001\u00bd\t\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"
+        + "\u0001\u0001\u0001\u0005\u0001\u00c4\b\u0001\n\u0001\f\u0001\u00c7\t\u0001"
+        + "\u0001\u0001\u0001\u0001\u0003\u0001\u00cb\b\u0001\u0001\u0001\u0001\u0001"
+        + "\u0001\u0002\u0001\u0002\u0001\u0003\u0001\u0003\u0001\u0004\u0001\u0004"
+        + "\u0001\u0005\u0001\u0005\u0001\u0006\u0001\u0006\u0001\u0007\u0001\u0007"
+        + "\u0001\b\u0001\b\u0001\t\u0001\t\u0001\t\u0001\t\u0001\n\u0001\n\u0001"
+        + "\n\u0001\n\u0001\n\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\r\u0001"
+        + "\r\u0001\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f"
+        + "\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u0010\u0001\u0010\u0001\u0010"
+        + "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0011\u0001\u0011\u0001\u0011"
+        + "\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0013\u0001\u0013"
+        + "\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013"
+        + "\u0001\u0013\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014"
+        + "\u0001\u0014\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015"
+        + "\u0001\u0015\u0001\u0015\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016"
+        + "\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0018\u0001\u0018"
+        + "\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0019\u0001\u0019"
+        + "\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u001a\u0001\u001a"
+        + "\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001b\u0001\u001b\u0001\u001b"
+        + "\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b"
+        + "\u0001\u001b\u0001\u001b\u0001\u001c\u0001\u001c\u0001\u001d\u0001\u001d"
+        + "\u0001\u001e\u0001\u001e\u0001\u001f\u0001\u001f\u0001\u001f\u0001 \u0001"
+        + " \u0001!\u0001!\u0001\"\u0001\"\u0001#\u0001#\u0001#\u0001$\u0001$\u0001"
+        + "$\u0001%\u0001%\u0001%\u0001%\u0001&\u0001&\u0001\'\u0001\'\u0001\'\u0001"
+        + "(\u0001(\u0001)\u0001)\u0001)\u0001*\u0001*\u0001*\u0001+\u0001+\u0001"
+        + "+\u0001+\u0001,\u0001,\u0001,\u0001-\u0001-\u0001-\u0001-\u0001.\u0001"
+        + ".\u0001/\u0001/\u00010\u00010\u00011\u00011\u00011\u00012\u00012\u0001"
+        + "2\u00013\u00013\u00014\u00014\u00015\u00015\u00015\u00016\u00016\u0001"
+        + "6\u00017\u00017\u00017\u00018\u00018\u00018\u00019\u00019\u00019\u0001"
+        + "9\u0001:\u0001:\u0001:\u0001;\u0001;\u0001;\u0001<\u0001<\u0001=\u0001"
+        + "=\u0001=\u0001>\u0001>\u0001>\u0001?\u0001?\u0001?\u0001@\u0001@\u0001"
+        + "@\u0001A\u0001A\u0001A\u0001B\u0001B\u0001B\u0001C\u0001C\u0001C\u0001"
+        + "D\u0001D\u0001D\u0001E\u0001E\u0001E\u0001E\u0001F\u0001F\u0001F\u0001"
+        + "F\u0001G\u0001G\u0001G\u0001G\u0001G\u0001H\u0001H\u0004H\u01bc\bH\u000b"
+        + "H\fH\u01bd\u0001H\u0003H\u01c1\bH\u0001I\u0001I\u0001I\u0004I\u01c6\b"
+        + "I\u000bI\fI\u01c7\u0001I\u0003I\u01cb\bI\u0001J\u0001J\u0001J\u0005J\u01d0"
+        + "\bJ\nJ\fJ\u01d3\tJ\u0003J\u01d5\bJ\u0001J\u0003J\u01d8\bJ\u0001K\u0001"
+        + "K\u0001K\u0005K\u01dd\bK\nK\fK\u01e0\tK\u0003K\u01e2\bK\u0001K\u0001K"
+        + "\u0004K\u01e6\bK\u000bK\fK\u01e7\u0003K\u01ea\bK\u0001K\u0001K\u0003K"
+        + "\u01ee\bK\u0001K\u0004K\u01f1\bK\u000bK\fK\u01f2\u0003K\u01f5\bK\u0001"
+        + "K\u0003K\u01f8\bK\u0001L\u0001L\u0001L\u0001L\u0001L\u0001L\u0005L\u0200"
+        + "\bL\nL\fL\u0203\tL\u0001L\u0001L\u0001L\u0001L\u0001L\u0001L\u0001L\u0005"
+        + "L\u020c\bL\nL\fL\u020f\tL\u0001L\u0003L\u0212\bL\u0001M\u0001M\u0001M"
+        + "\u0001M\u0004M\u0218\bM\u000bM\fM\u0219\u0001M\u0001M\u0005M\u021e\bM"
+        + "\nM\fM\u0221\tM\u0001M\u0001M\u0001N\u0001N\u0001N\u0001N\u0001N\u0001"
+        + "O\u0001O\u0001O\u0001O\u0001O\u0001O\u0001P\u0001P\u0001P\u0001P\u0001"
+        + "P\u0001Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001"
+        + "Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001"
+        + "Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001"
+        + "Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001Q\u0001Q\u0003Q\u025b"
+        + "\bQ\u0001R\u0001R\u0001R\u0001R\u0001S\u0001S\u0005S\u0263\bS\nS\fS\u0266"
+        + "\tS\u0001T\u0001T\u0001T\u0005T\u026b\bT\nT\fT\u026e\tT\u0003T\u0270\b"
+        + "T\u0001T\u0001T\u0001U\u0001U\u0005U\u0276\bU\nU\fU\u0279\tU\u0001U\u0001"
+        + "U\u0005\u00bb\u00c5\u0201\u020d\u0219\u0000V\u0002\u0001\u0004\u0002\u0006"
+        + "\u0003\b\u0004\n\u0005\f\u0006\u000e\u0007\u0010\b\u0012\t\u0014\n\u0016"
+        + "\u000b\u0018\f\u001a\r\u001c\u000e\u001e\u000f \u0010\"\u0011$\u0012&"
+        + "\u0013(\u0014*\u0015,\u0016.\u00170\u00182\u00194\u001a6\u001b8\u001c"
+        + ":\u001d<\u001e>\u001f@ B!D\"F#H$J%L&N\'P(R)T*V+X,Z-\\.^/`0b1d2f3h4j5l"
+        + "6n7p8r9t:v;x<z=|>~?\u0080@\u0082A\u0084B\u0086C\u0088D\u008aE\u008cF\u008e"
+        + "G\u0090H\u0092I\u0094J\u0096K\u0098L\u009aM\u009cN\u009eO\u00a0P\u00a2"
+        + "Q\u00a4R\u00a6S\u00a8T\u00aaU\u00acV\u0002\u0000\u0001\u0013\u0003\u0000"
+        + "\t\n\r\r  \u0002\u0000\n\n\r\r\u0001\u000007\u0002\u0000LLll\u0002\u0000"
+        + "XXxx\u0003\u000009AFaf\u0001\u000019\u0001\u000009\u0006\u0000DDFFLLd"
+        + "dffll\u0002\u0000EEee\u0002\u0000++--\u0004\u0000DDFFddff\u0002\u0000"
+        + "\"\"\\\\\u0002\u0000\'\'\\\\\u0001\u0000\n\n\u0002\u0000\n\n//\u0007\u0000"
+        + "UUcciilmssuuxx\u0003\u0000AZ__az\u0004\u000009AZ__az\u02a2\u0000\u0002"
+        + "\u0001\u0000\u0000\u0000\u0000\u0004\u0001\u0000\u0000\u0000\u0000\u0006"
+        + "\u0001\u0000\u0000\u0000\u0000\b\u0001\u0000\u0000\u0000\u0000\n\u0001"
+        + "\u0000\u0000\u0000\u0000\f\u0001\u0000\u0000\u0000\u0000\u000e\u0001\u0000"
+        + "\u0000\u0000\u0000\u0010\u0001\u0000\u0000\u0000\u0000\u0012\u0001\u0000"
+        + "\u0000\u0000\u0000\u0014\u0001\u0000\u0000\u0000\u0000\u0016\u0001\u0000"
+        + "\u0000\u0000\u0000\u0018\u0001\u0000\u0000\u0000\u0000\u001a\u0001\u0000"
+        + "\u0000\u0000\u0000\u001c\u0001\u0000\u0000\u0000\u0000\u001e\u0001\u0000"
+        + "\u0000\u0000\u0000 \u0001\u0000\u0000\u0000\u0000\"\u0001\u0000\u0000"
+        + "\u0000\u0000$\u0001\u0000\u0000\u0000\u0000&\u0001\u0000\u0000\u0000\u0000"
+        + "(\u0001\u0000\u0000\u0000\u0000*\u0001\u0000\u0000\u0000\u0000,\u0001"
+        + "\u0000\u0000\u0000\u0000.\u0001\u0000\u0000\u0000\u00000\u0001\u0000\u0000"
+        + "\u0000\u00002\u0001\u0000\u0000\u0000\u00004\u0001\u0000\u0000\u0000\u0000"
+        + "6\u0001\u0000\u0000\u0000\u00008\u0001\u0000\u0000\u0000\u0000:\u0001"
+        + "\u0000\u0000\u0000\u0000<\u0001\u0000\u0000\u0000\u0000>\u0001\u0000\u0000"
+        + "\u0000\u0000@\u0001\u0000\u0000\u0000\u0000B\u0001\u0000\u0000\u0000\u0000"
+        + "D\u0001\u0000\u0000\u0000\u0000F\u0001\u0000\u0000\u0000\u0000H\u0001"
+        + "\u0000\u0000\u0000\u0000J\u0001\u0000\u0000\u0000\u0000L\u0001\u0000\u0000"
+        + "\u0000\u0000N\u0001\u0000\u0000\u0000\u0000P\u0001\u0000\u0000\u0000\u0000"
+        + "R\u0001\u0000\u0000\u0000\u0000T\u0001\u0000\u0000\u0000\u0000V\u0001"
+        + "\u0000\u0000\u0000\u0000X\u0001\u0000\u0000\u0000\u0000Z\u0001\u0000\u0000"
+        + "\u0000\u0000\\\u0001\u0000\u0000\u0000\u0000^\u0001\u0000\u0000\u0000"
+        + "\u0000`\u0001\u0000\u0000\u0000\u0000b\u0001\u0000\u0000\u0000\u0000d"
+        + "\u0001\u0000\u0000\u0000\u0000f\u0001\u0000\u0000\u0000\u0000h\u0001\u0000"
+        + "\u0000\u0000\u0000j\u0001\u0000\u0000\u0000\u0000l\u0001\u0000\u0000\u0000"
+        + "\u0000n\u0001\u0000\u0000\u0000\u0000p\u0001\u0000\u0000\u0000\u0000r"
+        + "\u0001\u0000\u0000\u0000\u0000t\u0001\u0000\u0000\u0000\u0000v\u0001\u0000"
+        + "\u0000\u0000\u0000x\u0001\u0000\u0000\u0000\u0000z\u0001\u0000\u0000\u0000"
+        + "\u0000|\u0001\u0000\u0000\u0000\u0000~\u0001\u0000\u0000\u0000\u0000\u0080"
+        + "\u0001\u0000\u0000\u0000\u0000\u0082\u0001\u0000\u0000\u0000\u0000\u0084"
+        + "\u0001\u0000\u0000\u0000\u0000\u0086\u0001\u0000\u0000\u0000\u0000\u0088"
+        + "\u0001\u0000\u0000\u0000\u0000\u008a\u0001\u0000\u0000\u0000\u0000\u008c"
+        + "\u0001\u0000\u0000\u0000\u0000\u008e\u0001\u0000\u0000\u0000\u0000\u0090"
+        + "\u0001\u0000\u0000\u0000\u0000\u0092\u0001\u0000\u0000\u0000\u0000\u0094"
+        + "\u0001\u0000\u0000\u0000\u0000\u0096\u0001\u0000\u0000\u0000\u0000\u0098"
+        + "\u0001\u0000\u0000\u0000\u0000\u009a\u0001\u0000\u0000\u0000\u0000\u009c"
+        + "\u0001\u0000\u0000\u0000\u0000\u009e\u0001\u0000\u0000\u0000\u0000\u00a0"
+        + "\u0001\u0000\u0000\u0000\u0000\u00a2\u0001\u0000\u0000\u0000\u0000\u00a4"
+        + "\u0001\u0000\u0000\u0000\u0000\u00a6\u0001\u0000\u0000\u0000\u0000\u00a8"
+        + "\u0001\u0000\u0000\u0000\u0001\u00aa\u0001\u0000\u0000\u0000\u0001\u00ac"
+        + "\u0001\u0000\u0000\u0000\u0002\u00af\u0001\u0000\u0000\u0000\u0004\u00ca"
+        + "\u0001\u0000\u0000\u0000\u0006\u00ce\u0001\u0000\u0000\u0000\b\u00d0\u0001"
+        + "\u0000\u0000\u0000\n\u00d2\u0001\u0000\u0000\u0000\f\u00d4\u0001\u0000"
+        + "\u0000\u0000\u000e\u00d6\u0001\u0000\u0000\u0000\u0010\u00d8\u0001\u0000"
+        + "\u0000\u0000\u0012\u00da\u0001\u0000\u0000\u0000\u0014\u00dc\u0001\u0000"
+        + "\u0000\u0000\u0016\u00e0\u0001\u0000\u0000\u0000\u0018\u00e5\u0001\u0000"
+        + "\u0000\u0000\u001a\u00e7\u0001\u0000\u0000\u0000\u001c\u00e9\u0001\u0000"
+        + "\u0000\u0000\u001e\u00ec\u0001\u0000\u0000\u0000 \u00ef\u0001\u0000\u0000"
+        + "\u0000\"\u00f4\u0001\u0000\u0000\u0000$\u00fa\u0001\u0000\u0000\u0000"
+        + "&\u00fd\u0001\u0000\u0000\u0000(\u0101\u0001\u0000\u0000\u0000*\u010a"
+        + "\u0001\u0000\u0000\u0000,\u0110\u0001\u0000\u0000\u0000.\u0117\u0001\u0000"
+        + "\u0000\u00000\u011b\u0001\u0000\u0000\u00002\u011f\u0001\u0000\u0000\u0000"
+        + "4\u0125\u0001\u0000\u0000\u00006\u012b\u0001\u0000\u0000\u00008\u0130"
+        + "\u0001\u0000\u0000\u0000:\u013b\u0001\u0000\u0000\u0000<\u013d\u0001\u0000"
+        + "\u0000\u0000>\u013f\u0001\u0000\u0000\u0000@\u0141\u0001\u0000\u0000\u0000"
+        + "B\u0144\u0001\u0000\u0000\u0000D\u0146\u0001\u0000\u0000\u0000F\u0148"
+        + "\u0001\u0000\u0000\u0000H\u014a\u0001\u0000\u0000\u0000J\u014d\u0001\u0000"
+        + "\u0000\u0000L\u0150\u0001\u0000\u0000\u0000N\u0154\u0001\u0000\u0000\u0000"
+        + "P\u0156\u0001\u0000\u0000\u0000R\u0159\u0001\u0000\u0000\u0000T\u015b"
+        + "\u0001\u0000\u0000\u0000V\u015e\u0001\u0000\u0000\u0000X\u0161\u0001\u0000"
+        + "\u0000\u0000Z\u0165\u0001\u0000\u0000\u0000\\\u0168\u0001\u0000\u0000"
+        + "\u0000^\u016c\u0001\u0000\u0000\u0000`\u016e\u0001\u0000\u0000\u0000b"
+        + "\u0170\u0001\u0000\u0000\u0000d\u0172\u0001\u0000\u0000\u0000f\u0175\u0001"
+        + "\u0000\u0000\u0000h\u0178\u0001\u0000\u0000\u0000j\u017a\u0001\u0000\u0000"
+        + "\u0000l\u017c\u0001\u0000\u0000\u0000n\u017f\u0001\u0000\u0000\u0000p"
+        + "\u0182\u0001\u0000\u0000\u0000r\u0185\u0001\u0000\u0000\u0000t\u0188\u0001"
+        + "\u0000\u0000\u0000v\u018c\u0001\u0000\u0000\u0000x\u018f\u0001\u0000\u0000"
+        + "\u0000z\u0192\u0001\u0000\u0000\u0000|\u0194\u0001\u0000\u0000\u0000~"
+        + "\u0197\u0001\u0000\u0000\u0000\u0080\u019a\u0001\u0000\u0000\u0000\u0082"
+        + "\u019d\u0001\u0000\u0000\u0000\u0084\u01a0\u0001\u0000\u0000\u0000\u0086"
+        + "\u01a3\u0001\u0000\u0000\u0000\u0088\u01a6\u0001\u0000\u0000\u0000\u008a"
+        + "\u01a9\u0001\u0000\u0000\u0000\u008c\u01ac\u0001\u0000\u0000\u0000\u008e"
+        + "\u01b0\u0001\u0000\u0000\u0000\u0090\u01b4\u0001\u0000\u0000\u0000\u0092"
+        + "\u01b9\u0001\u0000\u0000\u0000\u0094\u01c2\u0001\u0000\u0000\u0000\u0096"
+        + "\u01d4\u0001\u0000\u0000\u0000\u0098\u01e1\u0001\u0000\u0000\u0000\u009a"
+        + "\u0211\u0001\u0000\u0000\u0000\u009c\u0213\u0001\u0000\u0000\u0000\u009e"
+        + "\u0224\u0001\u0000\u0000\u0000\u00a0\u0229\u0001\u0000\u0000\u0000\u00a2"
+        + "\u022f\u0001\u0000\u0000\u0000\u00a4\u025a\u0001\u0000\u0000\u0000\u00a6"
+        + "\u025c\u0001\u0000\u0000\u0000\u00a8\u0260\u0001\u0000\u0000\u0000\u00aa"
+        + "\u026f\u0001\u0000\u0000\u0000\u00ac\u0273\u0001\u0000\u0000\u0000\u00ae"
+        + "\u00b0\u0007\u0000\u0000\u0000\u00af\u00ae\u0001\u0000\u0000\u0000\u00b0"
+        + "\u00b1\u0001\u0000\u0000\u0000\u00b1\u00af\u0001\u0000\u0000\u0000\u00b1"
+        + "\u00b2\u0001\u0000\u0000\u0000\u00b2\u00b3\u0001\u0000\u0000\u0000\u00b3"
+        + "\u00b4\u0006\u0000\u0000\u0000\u00b4\u0003\u0001\u0000\u0000\u0000\u00b5"
+        + "\u00b6\u0005/\u0000\u0000\u00b6\u00b7\u0005/\u0000\u0000\u00b7\u00bb\u0001"
+        + "\u0000\u0000\u0000\u00b8\u00ba\t\u0000\u0000\u0000\u00b9\u00b8\u0001\u0000"
+        + "\u0000\u0000\u00ba\u00bd\u0001\u0000\u0000\u0000\u00bb\u00bc\u0001\u0000"
+        + "\u0000\u0000\u00bb\u00b9\u0001\u0000\u0000\u0000\u00bc\u00be\u0001\u0000"
+        + "\u0000\u0000\u00bd\u00bb\u0001\u0000\u0000\u0000\u00be\u00cb\u0007\u0001"
+        + "\u0000\u0000\u00bf\u00c0\u0005/\u0000\u0000\u00c0\u00c1\u0005*\u0000\u0000"
+        + "\u00c1\u00c5\u0001\u0000\u0000\u0000\u00c2\u00c4\t\u0000\u0000\u0000\u00c3"
+        + "\u00c2\u0001\u0000\u0000\u0000\u00c4\u00c7\u0001\u0000\u0000\u0000\u00c5"
+        + "\u00c6\u0001\u0000\u0000\u0000\u00c5\u00c3\u0001\u0000\u0000\u0000\u00c6"
+        + "\u00c8\u0001\u0000\u0000\u0000\u00c7\u00c5\u0001\u0000\u0000\u0000\u00c8"
+        + "\u00c9\u0005*\u0000\u0000\u00c9\u00cb\u0005/\u0000\u0000\u00ca\u00b5\u0001"
+        + "\u0000\u0000\u0000\u00ca\u00bf\u0001\u0000\u0000\u0000\u00cb\u00cc\u0001"
+        + "\u0000\u0000\u0000\u00cc\u00cd\u0006\u0001\u0000\u0000\u00cd\u0005\u0001"
+        + "\u0000\u0000\u0000\u00ce\u00cf\u0005{\u0000\u0000\u00cf\u0007\u0001\u0000"
+        + "\u0000\u0000\u00d0\u00d1\u0005}\u0000\u0000\u00d1\t\u0001\u0000\u0000"
+        + "\u0000\u00d2\u00d3\u0005[\u0000\u0000\u00d3\u000b\u0001\u0000\u0000\u0000"
+        + "\u00d4\u00d5\u0005]\u0000\u0000\u00d5\r\u0001\u0000\u0000\u0000\u00d6"
+        + "\u00d7\u0005(\u0000\u0000\u00d7\u000f\u0001\u0000\u0000\u0000\u00d8\u00d9"
+        + "\u0005)\u0000\u0000\u00d9\u0011\u0001\u0000\u0000\u0000\u00da\u00db\u0005"
+        + "$\u0000\u0000\u00db\u0013\u0001\u0000\u0000\u0000\u00dc\u00dd\u0005.\u0000"
+        + "\u0000\u00dd\u00de\u0001\u0000\u0000\u0000\u00de\u00df\u0006\t\u0001\u0000"
+        + "\u00df\u0015\u0001\u0000\u0000\u0000\u00e0\u00e1\u0005?\u0000\u0000\u00e1"
+        + "\u00e2\u0005.\u0000\u0000\u00e2\u00e3\u0001\u0000\u0000\u0000\u00e3\u00e4"
+        + "\u0006\n\u0001\u0000\u00e4\u0017\u0001\u0000\u0000\u0000\u00e5\u00e6\u0005"
+        + ",\u0000\u0000\u00e6\u0019\u0001\u0000\u0000\u0000\u00e7\u00e8\u0005;\u0000"
+        + "\u0000\u00e8\u001b\u0001\u0000\u0000\u0000\u00e9\u00ea\u0005i\u0000\u0000"
+        + "\u00ea\u00eb\u0005f\u0000\u0000\u00eb\u001d\u0001\u0000\u0000\u0000\u00ec"
+        + "\u00ed\u0005i\u0000\u0000\u00ed\u00ee\u0005n\u0000\u0000\u00ee\u001f\u0001"
+        + "\u0000\u0000\u0000\u00ef\u00f0\u0005e\u0000\u0000\u00f0\u00f1\u0005l\u0000"
+        + "\u0000\u00f1\u00f2\u0005s\u0000\u0000\u00f2\u00f3\u0005e\u0000\u0000\u00f3"
+        + "!\u0001\u0000\u0000\u0000\u00f4\u00f5\u0005w\u0000\u0000\u00f5\u00f6\u0005"
+        + "h\u0000\u0000\u00f6\u00f7\u0005i\u0000\u0000\u00f7\u00f8\u0005l\u0000"
+        + "\u0000\u00f8\u00f9\u0005e\u0000\u0000\u00f9#\u0001\u0000\u0000\u0000\u00fa"
+        + "\u00fb\u0005d\u0000\u0000\u00fb\u00fc\u0005o\u0000\u0000\u00fc%\u0001"
+        + "\u0000\u0000\u0000\u00fd\u00fe\u0005f\u0000\u0000\u00fe\u00ff\u0005o\u0000"
+        + "\u0000\u00ff\u0100\u0005r\u0000\u0000\u0100\'\u0001\u0000\u0000\u0000"
+        + "\u0101\u0102\u0005c\u0000\u0000\u0102\u0103\u0005o\u0000\u0000\u0103\u0104"
+        + "\u0005n\u0000\u0000\u0104\u0105\u0005t\u0000\u0000\u0105\u0106\u0005i"
+        + "\u0000\u0000\u0106\u0107\u0005n\u0000\u0000\u0107\u0108\u0005u\u0000\u0000"
+        + "\u0108\u0109\u0005e\u0000\u0000\u0109)\u0001\u0000\u0000\u0000\u010a\u010b"
+        + "\u0005b\u0000\u0000\u010b\u010c\u0005r\u0000\u0000\u010c\u010d\u0005e"
+        + "\u0000\u0000\u010d\u010e\u0005a\u0000\u0000\u010e\u010f\u0005k\u0000\u0000"
+        + "\u010f+\u0001\u0000\u0000\u0000\u0110\u0111\u0005r\u0000\u0000\u0111\u0112"
+        + "\u0005e\u0000\u0000\u0112\u0113\u0005t\u0000\u0000\u0113\u0114\u0005u"
+        + "\u0000\u0000\u0114\u0115\u0005r\u0000\u0000\u0115\u0116\u0005n\u0000\u0000"
+        + "\u0116-\u0001\u0000\u0000\u0000\u0117\u0118\u0005n\u0000\u0000\u0118\u0119"
+        + "\u0005e\u0000\u0000\u0119\u011a\u0005w\u0000\u0000\u011a/\u0001\u0000"
+        + "\u0000\u0000\u011b\u011c\u0005t\u0000\u0000\u011c\u011d\u0005r\u0000\u0000"
+        + "\u011d\u011e\u0005y\u0000\u0000\u011e1\u0001\u0000\u0000\u0000\u011f\u0120"
+        + "\u0005c\u0000\u0000\u0120\u0121\u0005a\u0000\u0000\u0121\u0122\u0005t"
+        + "\u0000\u0000\u0122\u0123\u0005c\u0000\u0000\u0123\u0124\u0005h\u0000\u0000"
+        + "\u01243\u0001\u0000\u0000\u0000\u0125\u0126\u0005t\u0000\u0000\u0126\u0127"
+        + "\u0005h\u0000\u0000\u0127\u0128\u0005r\u0000\u0000\u0128\u0129\u0005o"
+        + "\u0000\u0000\u0129\u012a\u0005w\u0000\u0000\u012a5\u0001\u0000\u0000\u0000"
+        + "\u012b\u012c\u0005t\u0000\u0000\u012c\u012d\u0005h\u0000\u0000\u012d\u012e"
+        + "\u0005i\u0000\u0000\u012e\u012f\u0005s\u0000\u0000\u012f7\u0001\u0000"
+        + "\u0000\u0000\u0130\u0131\u0005i\u0000\u0000\u0131\u0132\u0005n\u0000\u0000"
+        + "\u0132\u0133\u0005s\u0000\u0000\u0133\u0134\u0005t\u0000\u0000\u0134\u0135"
+        + "\u0005a\u0000\u0000\u0135\u0136\u0005n\u0000\u0000\u0136\u0137\u0005c"
+        + "\u0000\u0000\u0137\u0138\u0005e\u0000\u0000\u0138\u0139\u0005o\u0000\u0000"
+        + "\u0139\u013a\u0005f\u0000\u0000\u013a9\u0001\u0000\u0000\u0000\u013b\u013c"
+        + "\u0005!\u0000\u0000\u013c;\u0001\u0000\u0000\u0000\u013d\u013e\u0005~"
+        + "\u0000\u0000\u013e=\u0001\u0000\u0000\u0000\u013f\u0140\u0005*\u0000\u0000"
+        + "\u0140?\u0001\u0000\u0000\u0000\u0141\u0142\u0005/\u0000\u0000\u0142\u0143"
+        + "\u0004\u001f\u0000\u0000\u0143A\u0001\u0000\u0000\u0000\u0144\u0145\u0005"
+        + "%\u0000\u0000\u0145C\u0001\u0000\u0000\u0000\u0146\u0147\u0005+\u0000"
+        + "\u0000\u0147E\u0001\u0000\u0000\u0000\u0148\u0149\u0005-\u0000\u0000\u0149"
+        + "G\u0001\u0000\u0000\u0000\u014a\u014b\u0005<\u0000\u0000\u014b\u014c\u0005"
+        + "<\u0000\u0000\u014cI\u0001\u0000\u0000\u0000\u014d\u014e\u0005>\u0000"
+        + "\u0000\u014e\u014f\u0005>\u0000\u0000\u014fK\u0001\u0000\u0000\u0000\u0150"
+        + "\u0151\u0005>\u0000\u0000\u0151\u0152\u0005>\u0000\u0000\u0152\u0153\u0005"
+        + ">\u0000\u0000\u0153M\u0001\u0000\u0000\u0000\u0154\u0155\u0005<\u0000"
+        + "\u0000\u0155O\u0001\u0000\u0000\u0000\u0156\u0157\u0005<\u0000\u0000\u0157"
+        + "\u0158\u0005=\u0000\u0000\u0158Q\u0001\u0000\u0000\u0000\u0159\u015a\u0005"
+        + ">\u0000\u0000\u015aS\u0001\u0000\u0000\u0000\u015b\u015c\u0005>\u0000"
+        + "\u0000\u015c\u015d\u0005=\u0000\u0000\u015dU\u0001\u0000\u0000\u0000\u015e"
+        + "\u015f\u0005=\u0000\u0000\u015f\u0160\u0005=\u0000\u0000\u0160W\u0001"
+        + "\u0000\u0000\u0000\u0161\u0162\u0005=\u0000\u0000\u0162\u0163\u0005=\u0000"
+        + "\u0000\u0163\u0164\u0005=\u0000\u0000\u0164Y\u0001\u0000\u0000\u0000\u0165"
+        + "\u0166\u0005!\u0000\u0000\u0166\u0167\u0005=\u0000\u0000\u0167[\u0001"
+        + "\u0000\u0000\u0000\u0168\u0169\u0005!\u0000\u0000\u0169\u016a\u0005=\u0000"
+        + "\u0000\u016a\u016b\u0005=\u0000\u0000\u016b]\u0001\u0000\u0000\u0000\u016c"
+        + "\u016d\u0005&\u0000\u0000\u016d_\u0001\u0000\u0000\u0000\u016e\u016f\u0005"
+        + "^\u0000\u0000\u016fa\u0001\u0000\u0000\u0000\u0170\u0171\u0005|\u0000"
+        + "\u0000\u0171c\u0001\u0000\u0000\u0000\u0172\u0173\u0005&\u0000\u0000\u0173"
+        + "\u0174\u0005&\u0000\u0000\u0174e\u0001\u0000\u0000\u0000\u0175\u0176\u0005"
+        + "|\u0000\u0000\u0176\u0177\u0005|\u0000\u0000\u0177g\u0001\u0000\u0000"
+        + "\u0000\u0178\u0179\u0005?\u0000\u0000\u0179i\u0001\u0000\u0000\u0000\u017a"
+        + "\u017b\u0005:\u0000\u0000\u017bk\u0001\u0000\u0000\u0000\u017c\u017d\u0005"
+        + "?\u0000\u0000\u017d\u017e\u0005:\u0000\u0000\u017em\u0001\u0000\u0000"
+        + "\u0000\u017f\u0180\u0005:\u0000\u0000\u0180\u0181\u0005:\u0000\u0000\u0181"
+        + "o\u0001\u0000\u0000\u0000\u0182\u0183\u0005-\u0000\u0000\u0183\u0184\u0005"
+        + ">\u0000\u0000\u0184q\u0001\u0000\u0000\u0000\u0185\u0186\u0005=\u0000"
+        + "\u0000\u0186\u0187\u0005~\u0000\u0000\u0187s\u0001\u0000\u0000\u0000\u0188"
+        + "\u0189\u0005=\u0000\u0000\u0189\u018a\u0005=\u0000\u0000\u018a\u018b\u0005"
+        + "~\u0000\u0000\u018bu\u0001\u0000\u0000\u0000\u018c\u018d\u0005+\u0000"
+        + "\u0000\u018d\u018e\u0005+\u0000\u0000\u018ew\u0001\u0000\u0000\u0000\u018f"
+        + "\u0190\u0005-\u0000\u0000\u0190\u0191\u0005-\u0000\u0000\u0191y\u0001"
+        + "\u0000\u0000\u0000\u0192\u0193\u0005=\u0000\u0000\u0193{\u0001\u0000\u0000"
+        + "\u0000\u0194\u0195\u0005+\u0000\u0000\u0195\u0196\u0005=\u0000\u0000\u0196"
+        + "}\u0001\u0000\u0000\u0000\u0197\u0198\u0005-\u0000\u0000\u0198\u0199\u0005"
+        + "=\u0000\u0000\u0199\u007f\u0001\u0000\u0000\u0000\u019a\u019b\u0005*\u0000"
+        + "\u0000\u019b\u019c\u0005=\u0000\u0000\u019c\u0081\u0001\u0000\u0000\u0000"
+        + "\u019d\u019e\u0005/\u0000\u0000\u019e\u019f\u0005=\u0000\u0000\u019f\u0083"
+        + "\u0001\u0000\u0000\u0000\u01a0\u01a1\u0005%\u0000\u0000\u01a1\u01a2\u0005"
+        + "=\u0000\u0000\u01a2\u0085\u0001\u0000\u0000\u0000\u01a3\u01a4\u0005&\u0000"
+        + "\u0000\u01a4\u01a5\u0005=\u0000\u0000\u01a5\u0087\u0001\u0000\u0000\u0000"
+        + "\u01a6\u01a7\u0005^\u0000\u0000\u01a7\u01a8\u0005=\u0000\u0000\u01a8\u0089"
+        + "\u0001\u0000\u0000\u0000\u01a9\u01aa\u0005|\u0000\u0000\u01aa\u01ab\u0005"
+        + "=\u0000\u0000\u01ab\u008b\u0001\u0000\u0000\u0000\u01ac\u01ad\u0005<\u0000"
+        + "\u0000\u01ad\u01ae\u0005<\u0000\u0000\u01ae\u01af\u0005=\u0000\u0000\u01af"
+        + "\u008d\u0001\u0000\u0000\u0000\u01b0\u01b1\u0005>\u0000\u0000\u01b1\u01b2"
+        + "\u0005>\u0000\u0000\u01b2\u01b3\u0005=\u0000\u0000\u01b3\u008f\u0001\u0000"
+        + "\u0000\u0000\u01b4\u01b5\u0005>\u0000\u0000\u01b5\u01b6\u0005>\u0000\u0000"
+        + "\u01b6\u01b7\u0005>\u0000\u0000\u01b7\u01b8\u0005=\u0000\u0000\u01b8\u0091"
+        + "\u0001\u0000\u0000\u0000\u01b9\u01bb\u00050\u0000\u0000\u01ba\u01bc\u0007"
+        + "\u0002\u0000\u0000\u01bb\u01ba\u0001\u0000\u0000\u0000\u01bc\u01bd\u0001"
+        + "\u0000\u0000\u0000\u01bd\u01bb\u0001\u0000\u0000\u0000\u01bd\u01be\u0001"
+        + "\u0000\u0000\u0000\u01be\u01c0\u0001\u0000\u0000\u0000\u01bf\u01c1\u0007"
+        + "\u0003\u0000\u0000\u01c0\u01bf\u0001\u0000\u0000\u0000\u01c0\u01c1\u0001"
+        + "\u0000\u0000\u0000\u01c1\u0093\u0001\u0000\u0000\u0000\u01c2\u01c3\u0005"
+        + "0\u0000\u0000\u01c3\u01c5\u0007\u0004\u0000\u0000\u01c4\u01c6\u0007\u0005"
+        + "\u0000\u0000\u01c5\u01c4\u0001\u0000\u0000\u0000\u01c6\u01c7\u0001\u0000"
+        + "\u0000\u0000\u01c7\u01c5\u0001\u0000\u0000\u0000\u01c7\u01c8\u0001\u0000"
+        + "\u0000\u0000\u01c8\u01ca\u0001\u0000\u0000\u0000\u01c9\u01cb\u0007\u0003"
+        + "\u0000\u0000\u01ca\u01c9\u0001\u0000\u0000\u0000\u01ca\u01cb\u0001\u0000"
+        + "\u0000\u0000\u01cb\u0095\u0001\u0000\u0000\u0000\u01cc\u01d5\u00050\u0000"
+        + "\u0000\u01cd\u01d1\u0007\u0006\u0000\u0000\u01ce\u01d0\u0007\u0007\u0000"
+        + "\u0000\u01cf\u01ce\u0001\u0000\u0000\u0000\u01d0\u01d3\u0001\u0000\u0000"
+        + "\u0000\u01d1\u01cf\u0001\u0000\u0000\u0000\u01d1\u01d2\u0001\u0000\u0000"
+        + "\u0000\u01d2\u01d5\u0001\u0000\u0000\u0000\u01d3\u01d1\u0001\u0000\u0000"
+        + "\u0000\u01d4\u01cc\u0001\u0000\u0000\u0000\u01d4\u01cd\u0001\u0000\u0000"
+        + "\u0000\u01d5\u01d7\u0001\u0000\u0000\u0000\u01d6\u01d8\u0007\b\u0000\u0000"
+        + "\u01d7\u01d6\u0001\u0000\u0000\u0000\u01d7\u01d8\u0001\u0000\u0000\u0000"
+        + "\u01d8\u0097\u0001\u0000\u0000\u0000\u01d9\u01e2\u00050\u0000\u0000\u01da"
+        + "\u01de\u0007\u0006\u0000\u0000\u01db\u01dd\u0007\u0007\u0000\u0000\u01dc"
+        + "\u01db\u0001\u0000\u0000\u0000\u01dd\u01e0\u0001\u0000\u0000\u0000\u01de"
+        + "\u01dc\u0001\u0000\u0000\u0000\u01de\u01df\u0001\u0000\u0000\u0000\u01df"
+        + "\u01e2\u0001\u0000\u0000\u0000\u01e0\u01de\u0001\u0000\u0000\u0000\u01e1"
+        + "\u01d9\u0001\u0000\u0000\u0000\u01e1\u01da\u0001\u0000\u0000\u0000\u01e2"
+        + "\u01e9\u0001\u0000\u0000\u0000\u01e3\u01e5\u0003\u0014\t\u0000\u01e4\u01e6"
+        + "\u0007\u0007\u0000\u0000\u01e5\u01e4\u0001\u0000\u0000\u0000\u01e6\u01e7"
+        + "\u0001\u0000\u0000\u0000\u01e7\u01e5\u0001\u0000\u0000\u0000\u01e7\u01e8"
+        + "\u0001\u0000\u0000\u0000\u01e8\u01ea\u0001\u0000\u0000\u0000\u01e9\u01e3"
+        + "\u0001\u0000\u0000\u0000\u01e9\u01ea\u0001\u0000\u0000\u0000\u01ea\u01f4"
+        + "\u0001\u0000\u0000\u0000\u01eb\u01ed\u0007\t\u0000\u0000\u01ec\u01ee\u0007"
+        + "\n\u0000\u0000\u01ed\u01ec\u0001\u0000\u0000\u0000\u01ed\u01ee\u0001\u0000"
+        + "\u0000\u0000\u01ee\u01f0\u0001\u0000\u0000\u0000\u01ef\u01f1\u0007\u0007"
+        + "\u0000\u0000\u01f0\u01ef\u0001\u0000\u0000\u0000\u01f1\u01f2\u0001\u0000"
+        + "\u0000\u0000\u01f2\u01f0\u0001\u0000\u0000\u0000\u01f2\u01f3\u0001\u0000"
+        + "\u0000\u0000\u01f3\u01f5\u0001\u0000\u0000\u0000\u01f4\u01eb\u0001\u0000"
+        + "\u0000\u0000\u01f4\u01f5\u0001\u0000\u0000\u0000\u01f5\u01f7\u0001\u0000"
+        + "\u0000\u0000\u01f6\u01f8\u0007\u000b\u0000\u0000\u01f7\u01f6\u0001\u0000"
+        + "\u0000\u0000\u01f7\u01f8\u0001\u0000\u0000\u0000\u01f8\u0099\u0001\u0000"
+        + "\u0000\u0000\u01f9\u0201\u0005\"\u0000\u0000\u01fa\u01fb\u0005\\\u0000"
+        + "\u0000\u01fb\u0200\u0005\"\u0000\u0000\u01fc\u01fd\u0005\\\u0000\u0000"
+        + "\u01fd\u0200\u0005\\\u0000\u0000\u01fe\u0200\b\f\u0000\u0000\u01ff\u01fa"
+        + "\u0001\u0000\u0000\u0000\u01ff\u01fc\u0001\u0000\u0000\u0000\u01ff\u01fe"
+        + "\u0001\u0000\u0000\u0000\u0200\u0203\u0001\u0000\u0000\u0000\u0201\u0202"
+        + "\u0001\u0000\u0000\u0000\u0201\u01ff\u0001\u0000\u0000\u0000\u0202\u0204"
+        + "\u0001\u0000\u0000\u0000\u0203\u0201\u0001\u0000\u0000\u0000\u0204\u0212"
+        + "\u0005\"\u0000\u0000\u0205\u020d\u0005\'\u0000\u0000\u0206\u0207\u0005"
+        + "\\\u0000\u0000\u0207\u020c\u0005\'\u0000\u0000\u0208\u0209\u0005\\\u0000"
+        + "\u0000\u0209\u020c\u0005\\\u0000\u0000\u020a\u020c\b\r\u0000\u0000\u020b"
+        + "\u0206\u0001\u0000\u0000\u0000\u020b\u0208\u0001\u0000\u0000\u0000\u020b"
+        + "\u020a\u0001\u0000\u0000\u0000\u020c\u020f\u0001\u0000\u0000\u0000\u020d"
+        + "\u020e\u0001\u0000\u0000\u0000\u020d\u020b\u0001\u0000\u0000\u0000\u020e"
+        + "\u0210\u0001\u0000\u0000\u0000\u020f\u020d\u0001\u0000\u0000\u0000\u0210"
+        + "\u0212\u0005\'\u0000\u0000\u0211\u01f9\u0001\u0000\u0000\u0000\u0211\u0205"
+        + "\u0001\u0000\u0000\u0000\u0212\u009b\u0001\u0000\u0000\u0000\u0213\u0217"
+        + "\u0005/\u0000\u0000\u0214\u0215\u0005\\\u0000\u0000\u0215\u0218\b\u000e"
+        + "\u0000\u0000\u0216\u0218\b\u000f\u0000\u0000\u0217\u0214\u0001\u0000\u0000"
+        + "\u0000\u0217\u0216\u0001\u0000\u0000\u0000\u0218\u0219\u0001\u0000\u0000"
+        + "\u0000\u0219\u021a\u0001\u0000\u0000\u0000\u0219\u0217\u0001\u0000\u0000"
+        + "\u0000\u021a\u021b\u0001\u0000\u0000\u0000\u021b\u021f\u0005/\u0000\u0000"
+        + "\u021c\u021e\u0007\u0010\u0000\u0000\u021d\u021c\u0001\u0000\u0000\u0000"
+        + "\u021e\u0221\u0001\u0000\u0000\u0000\u021f\u021d\u0001\u0000\u0000\u0000"
+        + "\u021f\u0220\u0001\u0000\u0000\u0000\u0220\u0222\u0001\u0000\u0000\u0000"
+        + "\u0221\u021f\u0001\u0000\u0000\u0000\u0222\u0223\u0004M\u0001\u0000\u0223"
+        + "\u009d\u0001\u0000\u0000\u0000\u0224\u0225\u0005t\u0000\u0000\u0225\u0226"
+        + "\u0005r\u0000\u0000\u0226\u0227\u0005u\u0000\u0000\u0227\u0228\u0005e"
+        + "\u0000\u0000\u0228\u009f\u0001\u0000\u0000\u0000\u0229\u022a\u0005f\u0000"
+        + "\u0000\u022a\u022b\u0005a\u0000\u0000\u022b\u022c\u0005l\u0000\u0000\u022c"
+        + "\u022d\u0005s\u0000\u0000\u022d\u022e\u0005e\u0000\u0000\u022e\u00a1\u0001"
+        + "\u0000\u0000\u0000\u022f\u0230\u0005n\u0000\u0000\u0230\u0231\u0005u\u0000"
+        + "\u0000\u0231\u0232\u0005l\u0000\u0000\u0232\u0233\u0005l\u0000\u0000\u0233"
+        + "\u00a3\u0001\u0000\u0000\u0000\u0234\u0235\u0005b\u0000\u0000\u0235\u0236"
+        + "\u0005o\u0000\u0000\u0236\u0237\u0005o\u0000\u0000\u0237\u0238\u0005l"
+        + "\u0000\u0000\u0238\u0239\u0005e\u0000\u0000\u0239\u023a\u0005a\u0000\u0000"
+        + "\u023a\u025b\u0005n\u0000\u0000\u023b\u023c\u0005b\u0000\u0000\u023c\u023d"
+        + "\u0005y\u0000\u0000\u023d\u023e\u0005t\u0000\u0000\u023e\u025b\u0005e"
+        + "\u0000\u0000\u023f\u0240\u0005s\u0000\u0000\u0240\u0241\u0005h\u0000\u0000"
+        + "\u0241\u0242\u0005o\u0000\u0000\u0242\u0243\u0005r\u0000\u0000\u0243\u025b"
+        + "\u0005t\u0000\u0000\u0244\u0245\u0005c\u0000\u0000\u0245\u0246\u0005h"
+        + "\u0000\u0000\u0246\u0247\u0005a\u0000\u0000\u0247\u025b\u0005r\u0000\u0000"
+        + "\u0248\u0249\u0005i\u0000\u0000\u0249\u024a\u0005n\u0000\u0000\u024a\u025b"
+        + "\u0005t\u0000\u0000\u024b\u024c\u0005l\u0000\u0000\u024c\u024d\u0005o"
+        + "\u0000\u0000\u024d\u024e\u0005n\u0000\u0000\u024e\u025b\u0005g\u0000\u0000"
+        + "\u024f\u0250\u0005f\u0000\u0000\u0250\u0251\u0005l\u0000\u0000\u0251\u0252"
+        + "\u0005o\u0000\u0000\u0252\u0253\u0005a\u0000\u0000\u0253\u025b\u0005t"
+        + "\u0000\u0000\u0254\u0255\u0005d\u0000\u0000\u0255\u0256\u0005o\u0000\u0000"
+        + "\u0256\u0257\u0005u\u0000\u0000\u0257\u0258\u0005b\u0000\u0000\u0258\u0259"
+        + "\u0005l\u0000\u0000\u0259\u025b\u0005e\u0000\u0000\u025a\u0234\u0001\u0000"
+        + "\u0000\u0000\u025a\u023b\u0001\u0000\u0000\u0000\u025a\u023f\u0001\u0000"
+        + "\u0000\u0000\u025a\u0244\u0001\u0000\u0000\u0000\u025a\u0248\u0001\u0000"
+        + "\u0000\u0000\u025a\u024b\u0001\u0000\u0000\u0000\u025a\u024f\u0001\u0000"
+        + "\u0000\u0000\u025a\u0254\u0001\u0000\u0000\u0000\u025b\u00a5\u0001\u0000"
+        + "\u0000\u0000\u025c\u025d\u0005d\u0000\u0000\u025d\u025e\u0005e\u0000\u0000"
+        + "\u025e\u025f\u0005f\u0000\u0000\u025f\u00a7\u0001\u0000\u0000\u0000\u0260"
+        + "\u0264\u0007\u0011\u0000\u0000\u0261\u0263\u0007\u0012\u0000\u0000\u0262"
+        + "\u0261\u0001\u0000\u0000\u0000\u0263\u0266\u0001\u0000\u0000\u0000\u0264"
+        + "\u0262\u0001\u0000\u0000\u0000\u0264\u0265\u0001\u0000\u0000\u0000\u0265"
+        + "\u00a9\u0001\u0000\u0000\u0000\u0266\u0264\u0001\u0000\u0000\u0000\u0267"
+        + "\u0270\u00050\u0000\u0000\u0268\u026c\u0007\u0006\u0000\u0000\u0269\u026b"
+        + "\u0007\u0007\u0000\u0000\u026a\u0269\u0001\u0000\u0000\u0000\u026b\u026e"
+        + "\u0001\u0000\u0000\u0000\u026c\u026a\u0001\u0000\u0000\u0000\u026c\u026d"
+        + "\u0001\u0000\u0000\u0000\u026d\u0270\u0001\u0000\u0000\u0000\u026e\u026c"
+        + "\u0001\u0000\u0000\u0000\u026f\u0267\u0001\u0000\u0000\u0000\u026f\u0268"
+        + "\u0001\u0000\u0000\u0000\u0270\u0271\u0001\u0000\u0000\u0000\u0271\u0272"
+        + "\u0006T\u0002\u0000\u0272\u00ab\u0001\u0000\u0000\u0000\u0273\u0277\u0007"
+        + "\u0011\u0000\u0000\u0274\u0276\u0007\u0012\u0000\u0000\u0275\u0274\u0001"
+        + "\u0000\u0000\u0000\u0276\u0279\u0001\u0000\u0000\u0000\u0277\u0275\u0001"
+        + "\u0000\u0000\u0000\u0277\u0278\u0001\u0000\u0000\u0000\u0278\u027a\u0001"
+        + "\u0000\u0000\u0000\u0279\u0277\u0001\u0000\u0000\u0000\u027a\u027b\u0006"
+        + "U\u0002\u0000\u027b\u00ad\u0001\u0000\u0000\u0000\"\u0000\u0001\u00b1"
+        + "\u00bb\u00c5\u00ca\u01bd\u01c0\u01c7\u01ca\u01d1\u01d4\u01d7\u01de\u01e1"
+        + "\u01e7\u01e9\u01ed\u01f2\u01f4\u01f7\u01ff\u0201\u020b\u020d\u0211\u0217"
+        + "\u0219\u021f\u025a\u0264\u026c\u026f\u0277\u0003\u0006\u0000\u0000\u0002"
+        + "\u0001\u0000\u0002\u0000\u0000";
     public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray());
     static {
         _decisionToDFA = new DFA[_ATN.getNumberOfDecisions()];
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index 07ab6dd43890..45d35e87fc43 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -1,18 +1,31 @@
 // ANTLR GENERATED CODE: DO NOT EDIT
 package org.elasticsearch.painless.antlr;
 
-import org.antlr.v4.runtime.*;
-import org.antlr.v4.runtime.atn.*;
+import org.antlr.v4.runtime.FailedPredicateException;
+import org.antlr.v4.runtime.NoViableAltException;
+import org.antlr.v4.runtime.Parser;
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.antlr.v4.runtime.RecognitionException;
+import org.antlr.v4.runtime.RuleContext;
+import org.antlr.v4.runtime.RuntimeMetaData;
+import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.TokenStream;
+import org.antlr.v4.runtime.Vocabulary;
+import org.antlr.v4.runtime.VocabularyImpl;
+import org.antlr.v4.runtime.atn.ATN;
+import org.antlr.v4.runtime.atn.ATNDeserializer;
+import org.antlr.v4.runtime.atn.ParserATNSimulator;
+import org.antlr.v4.runtime.atn.PredictionContextCache;
 import org.antlr.v4.runtime.dfa.DFA;
-import org.antlr.v4.runtime.misc.*;
-import org.antlr.v4.runtime.tree.*;
+import org.antlr.v4.runtime.tree.ParseTreeVisitor;
+import org.antlr.v4.runtime.tree.TerminalNode;
 
 import java.util.List;
 
-@SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast" })
+@SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue" })
 class PainlessParser extends Parser {
     static {
-        RuntimeMetaData.checkVersion("4.5.3", RuntimeMetaData.VERSION);
+        RuntimeMetaData.checkVersion("4.11.1", RuntimeMetaData.VERSION);
     }
 
     protected static final DFA[] _decisionToDFA;
@@ -32,220 +45,234 @@ class PainlessParser extends Parser {
         RULE_refcasttype = 22, RULE_chain = 23, RULE_primary = 24, RULE_postfix = 25, RULE_postdot = 26, RULE_callinvoke = 27,
         RULE_fieldaccess = 28, RULE_braceaccess = 29, RULE_arrayinitializer = 30, RULE_listinitializer = 31, RULE_mapinitializer = 32,
         RULE_maptoken = 33, RULE_arguments = 34, RULE_argument = 35, RULE_lambda = 36, RULE_lamtype = 37, RULE_funcref = 38;
-    public static final String[] ruleNames = {
-        "source",
-        "function",
-        "parameters",
-        "statement",
-        "rstatement",
-        "dstatement",
-        "trailer",
-        "block",
-        "empty",
-        "initializer",
-        "afterthought",
-        "declaration",
-        "decltype",
-        "type",
-        "declvar",
-        "trap",
-        "noncondexpression",
-        "expression",
-        "unary",
-        "unarynotaddsub",
-        "castexpression",
-        "primordefcasttype",
-        "refcasttype",
-        "chain",
-        "primary",
-        "postfix",
-        "postdot",
-        "callinvoke",
-        "fieldaccess",
-        "braceaccess",
-        "arrayinitializer",
-        "listinitializer",
-        "mapinitializer",
-        "maptoken",
-        "arguments",
-        "argument",
-        "lambda",
-        "lamtype",
-        "funcref" };
-
-    private static final String[] _LITERAL_NAMES = {
-        null,
-        null,
-        null,
-        "'{'",
-        "'}'",
-        "'['",
-        "']'",
-        "'('",
-        "')'",
-        "'$'",
-        "'.'",
-        "'?.'",
-        "','",
-        "';'",
-        "'if'",
-        "'in'",
-        "'else'",
-        "'while'",
-        "'do'",
-        "'for'",
-        "'continue'",
-        "'break'",
-        "'return'",
-        "'new'",
-        "'try'",
-        "'catch'",
-        "'throw'",
-        "'this'",
-        "'instanceof'",
-        "'!'",
-        "'~'",
-        "'*'",
-        "'/'",
-        "'%'",
-        "'+'",
-        "'-'",
-        "'<<'",
-        "'>>'",
-        "'>>>'",
-        "'<'",
-        "'<='",
-        "'>'",
-        "'>='",
-        "'=='",
-        "'==='",
-        "'!='",
-        "'!=='",
-        "'&'",
-        "'^'",
-        "'|'",
-        "'&&'",
-        "'||'",
-        "'?'",
-        "':'",
-        "'?:'",
-        "'::'",
-        "'->'",
-        "'=~'",
-        "'==~'",
-        "'++'",
-        "'--'",
-        "'='",
-        "'+='",
-        "'-='",
-        "'*='",
-        "'/='",
-        "'%='",
-        "'&='",
-        "'^='",
-        "'|='",
-        "'<<='",
-        "'>>='",
-        "'>>>='",
-        null,
-        null,
-        null,
-        null,
-        null,
-        null,
-        "'true'",
-        "'false'",
-        "'null'",
-        null,
-        "'def'" };
-    private static final String[] _SYMBOLIC_NAMES = {
-        null,
-        "WS",
-        "COMMENT",
-        "LBRACK",
-        "RBRACK",
-        "LBRACE",
-        "RBRACE",
-        "LP",
-        "RP",
-        "DOLLAR",
-        "DOT",
-        "NSDOT",
-        "COMMA",
-        "SEMICOLON",
-        "IF",
-        "IN",
-        "ELSE",
-        "WHILE",
-        "DO",
-        "FOR",
-        "CONTINUE",
-        "BREAK",
-        "RETURN",
-        "NEW",
-        "TRY",
-        "CATCH",
-        "THROW",
-        "THIS",
-        "INSTANCEOF",
-        "BOOLNOT",
-        "BWNOT",
-        "MUL",
-        "DIV",
-        "REM",
-        "ADD",
-        "SUB",
-        "LSH",
-        "RSH",
-        "USH",
-        "LT",
-        "LTE",
-        "GT",
-        "GTE",
-        "EQ",
-        "EQR",
-        "NE",
-        "NER",
-        "BWAND",
-        "XOR",
-        "BWOR",
-        "BOOLAND",
-        "BOOLOR",
-        "COND",
-        "COLON",
-        "ELVIS",
-        "REF",
-        "ARROW",
-        "FIND",
-        "MATCH",
-        "INCR",
-        "DECR",
-        "ASSIGN",
-        "AADD",
-        "ASUB",
-        "AMUL",
-        "ADIV",
-        "AREM",
-        "AAND",
-        "AXOR",
-        "AOR",
-        "ALSH",
-        "ARSH",
-        "AUSH",
-        "OCTAL",
-        "HEX",
-        "INTEGER",
-        "DECIMAL",
-        "STRING",
-        "REGEX",
-        "TRUE",
-        "FALSE",
-        "NULL",
-        "PRIMITIVE",
-        "DEF",
-        "ID",
-        "DOTINTEGER",
-        "DOTID" };
+
+    private static String[] makeRuleNames() {
+        return new String[] {
+            "source",
+            "function",
+            "parameters",
+            "statement",
+            "rstatement",
+            "dstatement",
+            "trailer",
+            "block",
+            "empty",
+            "initializer",
+            "afterthought",
+            "declaration",
+            "decltype",
+            "type",
+            "declvar",
+            "trap",
+            "noncondexpression",
+            "expression",
+            "unary",
+            "unarynotaddsub",
+            "castexpression",
+            "primordefcasttype",
+            "refcasttype",
+            "chain",
+            "primary",
+            "postfix",
+            "postdot",
+            "callinvoke",
+            "fieldaccess",
+            "braceaccess",
+            "arrayinitializer",
+            "listinitializer",
+            "mapinitializer",
+            "maptoken",
+            "arguments",
+            "argument",
+            "lambda",
+            "lamtype",
+            "funcref" };
+    }
+
+    public static final String[] ruleNames = makeRuleNames();
+
+    private static String[] makeLiteralNames() {
+        return new String[] {
+            null,
+            null,
+            null,
+            "'{'",
+            "'}'",
+            "'['",
+            "']'",
+            "'('",
+            "')'",
+            "'$'",
+            "'.'",
+            "'?.'",
+            "','",
+            "';'",
+            "'if'",
+            "'in'",
+            "'else'",
+            "'while'",
+            "'do'",
+            "'for'",
+            "'continue'",
+            "'break'",
+            "'return'",
+            "'new'",
+            "'try'",
+            "'catch'",
+            "'throw'",
+            "'this'",
+            "'instanceof'",
+            "'!'",
+            "'~'",
+            "'*'",
+            "'/'",
+            "'%'",
+            "'+'",
+            "'-'",
+            "'<<'",
+            "'>>'",
+            "'>>>'",
+            "'<'",
+            "'<='",
+            "'>'",
+            "'>='",
+            "'=='",
+            "'==='",
+            "'!='",
+            "'!=='",
+            "'&'",
+            "'^'",
+            "'|'",
+            "'&&'",
+            "'||'",
+            "'?'",
+            "':'",
+            "'?:'",
+            "'::'",
+            "'->'",
+            "'=~'",
+            "'==~'",
+            "'++'",
+            "'--'",
+            "'='",
+            "'+='",
+            "'-='",
+            "'*='",
+            "'/='",
+            "'%='",
+            "'&='",
+            "'^='",
+            "'|='",
+            "'<<='",
+            "'>>='",
+            "'>>>='",
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            "'true'",
+            "'false'",
+            "'null'",
+            null,
+            "'def'" };
+    }
+
+    private static final String[] _LITERAL_NAMES = makeLiteralNames();
+
+    private static String[] makeSymbolicNames() {
+        return new String[] {
+            null,
+            "WS",
+            "COMMENT",
+            "LBRACK",
+            "RBRACK",
+            "LBRACE",
+            "RBRACE",
+            "LP",
+            "RP",
+            "DOLLAR",
+            "DOT",
+            "NSDOT",
+            "COMMA",
+            "SEMICOLON",
+            "IF",
+            "IN",
+            "ELSE",
+            "WHILE",
+            "DO",
+            "FOR",
+            "CONTINUE",
+            "BREAK",
+            "RETURN",
+            "NEW",
+            "TRY",
+            "CATCH",
+            "THROW",
+            "THIS",
+            "INSTANCEOF",
+            "BOOLNOT",
+            "BWNOT",
+            "MUL",
+            "DIV",
+            "REM",
+            "ADD",
+            "SUB",
+            "LSH",
+            "RSH",
+            "USH",
+            "LT",
+            "LTE",
+            "GT",
+            "GTE",
+            "EQ",
+            "EQR",
+            "NE",
+            "NER",
+            "BWAND",
+            "XOR",
+            "BWOR",
+            "BOOLAND",
+            "BOOLOR",
+            "COND",
+            "COLON",
+            "ELVIS",
+            "REF",
+            "ARROW",
+            "FIND",
+            "MATCH",
+            "INCR",
+            "DECR",
+            "ASSIGN",
+            "AADD",
+            "ASUB",
+            "AMUL",
+            "ADIV",
+            "AREM",
+            "AAND",
+            "AXOR",
+            "AOR",
+            "ALSH",
+            "ARSH",
+            "AUSH",
+            "OCTAL",
+            "HEX",
+            "INTEGER",
+            "DECIMAL",
+            "STRING",
+            "REGEX",
+            "TRUE",
+            "FALSE",
+            "NULL",
+            "PRIMITIVE",
+            "DEF",
+            "ID",
+            "DOTINTEGER",
+            "DOTID" };
+    }
+
+    private static final String[] _SYMBOLIC_NAMES = makeSymbolicNames();
     public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
     /**
@@ -281,7 +308,7 @@ public Vocabulary getVocabulary() {
 
     @Override
     public String getGrammarFileName() {
-        return "PainlessParser.g4";
+        return "java-escape";
     }
 
     @Override
@@ -304,6 +331,7 @@ public PainlessParser(TokenStream input) {
         _interp = new ParserATNSimulator(this, _ATN, _decisionToDFA, _sharedContextCache);
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class SourceContext extends ParserRuleContext {
         public TerminalNode EOF() {
             return getToken(PainlessParser.EOF, 0);
@@ -368,14 +396,8 @@ public final SourceContext source() throws RecognitionException {
                 setState(87);
                 _errHandler.sync(this);
                 _la = _input.LA(1);
-                while ((((_la) & ~0x3f) == 0
-                    && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << DOLLAR) | (1L << IF) | (1L << WHILE) | (1L << DO) | (1L << FOR)
-                        | (1L << CONTINUE) | (1L << BREAK) | (1L << RETURN) | (1L << NEW) | (1L << TRY) | (1L << THROW) | (1L << BOOLNOT)
-                        | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0)
-                    || ((((_la - 73)) & ~0x3f) == 0
-                        && ((1L << (_la - 73)) & ((1L << (OCTAL - 73)) | (1L << (HEX - 73)) | (1L << (INTEGER - 73)) | (1L << (DECIMAL
-                            - 73)) | (1L << (STRING - 73)) | (1L << (REGEX - 73)) | (1L << (TRUE - 73)) | (1L << (FALSE - 73)) | (1L
-                                << (NULL - 73)) | (1L << (PRIMITIVE - 73)) | (1L << (DEF - 73)) | (1L << (ID - 73)))) != 0)) {
+                while (((_la) & ~0x3f) == 0 && ((1L << _la) & 1729382310161040032L) != 0
+                    || (((_la - 73)) & ~0x3f) == 0 && ((1L << (_la - 73)) & 4095L) != 0) {
                     {
                         {
                             setState(84);
@@ -399,6 +421,7 @@ public final SourceContext source() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class FunctionContext extends ParserRuleContext {
         public DecltypeContext decltype() {
             return getRuleContext(DecltypeContext.class, 0);
@@ -457,6 +480,7 @@ public final FunctionContext function() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ParametersContext extends ParserRuleContext {
         public TerminalNode LP() {
             return getToken(PainlessParser.LP, 0);
@@ -516,9 +540,9 @@ public final ParametersContext parameters() throws RecognitionException {
                 setState(97);
                 match(LP);
                 setState(109);
+                _errHandler.sync(this);
                 _la = _input.LA(1);
-                if (((((_la - 82)) & ~0x3f) == 0
-                    && ((1L << (_la - 82)) & ((1L << (PRIMITIVE - 82)) | (1L << (DEF - 82)) | (1L << (ID - 82)))) != 0)) {
+                if ((((_la - 82)) & ~0x3f) == 0 && ((1L << (_la - 82)) & 7L) != 0) {
                     {
                         setState(98);
                         decltype();
@@ -558,6 +582,7 @@ public final ParametersContext parameters() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class StatementContext extends ParserRuleContext {
         public RstatementContext rstatement() {
             return getRuleContext(RstatementContext.class, 0);
@@ -597,6 +622,7 @@ public final StatementContext statement() throws RecognitionException {
         int _la;
         try {
             setState(117);
+            _errHandler.sync(this);
             switch (_input.LA(1)) {
                 case IF:
                 case WHILE:
@@ -642,6 +668,8 @@ public final StatementContext statement() throws RecognitionException {
                     if (!(_la == EOF || _la == SEMICOLON)) {
                         _errHandler.recoverInline(this);
                     } else {
+                        if (_input.LA(1) == Token.EOF) matchedEOF = true;
+                        _errHandler.reportMatch(this);
                         consume();
                     }
                 }
@@ -659,6 +687,7 @@ public final StatementContext statement() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class RstatementContext extends ParserRuleContext {
         public RstatementContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -676,6 +705,7 @@ public void copyFrom(RstatementContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ForContext extends RstatementContext {
         public TerminalNode FOR() {
             return getToken(PainlessParser.FOR, 0);
@@ -728,6 +758,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class TryContext extends RstatementContext {
         public TerminalNode TRY() {
             return getToken(PainlessParser.TRY, 0);
@@ -756,6 +787,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class WhileContext extends RstatementContext {
         public TerminalNode WHILE() {
             return getToken(PainlessParser.WHILE, 0);
@@ -792,6 +824,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class IneachContext extends RstatementContext {
         public TerminalNode FOR() {
             return getToken(PainlessParser.FOR, 0);
@@ -832,6 +865,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class IfContext extends RstatementContext {
         public TerminalNode IF() {
             return getToken(PainlessParser.IF, 0);
@@ -872,6 +906,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class EachContext extends RstatementContext {
         public TerminalNode FOR() {
             return getToken(PainlessParser.FOR, 0);
@@ -968,6 +1003,7 @@ public final RstatementContext rstatement() throws RecognitionException {
                     setState(132);
                     match(RP);
                     setState(135);
+                    _errHandler.sync(this);
                     switch (_input.LA(1)) {
                         case LBRACK:
                         case LBRACE:
@@ -1023,14 +1059,10 @@ public final RstatementContext rstatement() throws RecognitionException {
                     setState(138);
                     match(LP);
                     setState(140);
+                    _errHandler.sync(this);
                     _la = _input.LA(1);
-                    if ((((_la) & ~0x3f) == 0
-                        && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << DOLLAR) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT)
-                            | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0)
-                        || ((((_la - 73)) & ~0x3f) == 0
-                            && ((1L << (_la - 73)) & ((1L << (OCTAL - 73)) | (1L << (HEX - 73)) | (1L << (INTEGER - 73)) | (1L << (DECIMAL
-                                - 73)) | (1L << (STRING - 73)) | (1L << (REGEX - 73)) | (1L << (TRUE - 73)) | (1L << (FALSE - 73)) | (1L
-                                    << (NULL - 73)) | (1L << (PRIMITIVE - 73)) | (1L << (DEF - 73)) | (1L << (ID - 73)))) != 0)) {
+                    if (((_la) & ~0x3f) == 0 && ((1L << _la) & 1729382310068880032L) != 0
+                        || (((_la - 73)) & ~0x3f) == 0 && ((1L << (_la - 73)) & 4095L) != 0) {
                         {
                             setState(139);
                             initializer();
@@ -1040,14 +1072,10 @@ public final RstatementContext rstatement() throws RecognitionException {
                     setState(142);
                     match(SEMICOLON);
                     setState(144);
+                    _errHandler.sync(this);
                     _la = _input.LA(1);
-                    if ((((_la) & ~0x3f) == 0
-                        && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << DOLLAR) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT)
-                            | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0)
-                        || ((((_la - 73)) & ~0x3f) == 0
-                            && ((1L << (_la - 73)) & ((1L << (OCTAL - 73)) | (1L << (HEX - 73)) | (1L << (INTEGER - 73)) | (1L << (DECIMAL
-                                - 73)) | (1L << (STRING - 73)) | (1L << (REGEX - 73)) | (1L << (TRUE - 73)) | (1L << (FALSE - 73)) | (1L
-                                    << (NULL - 73)) | (1L << (ID - 73)))) != 0)) {
+                    if (((_la) & ~0x3f) == 0 && ((1L << _la) & 1729382310068880032L) != 0
+                        || (((_la - 73)) & ~0x3f) == 0 && ((1L << (_la - 73)) & 2559L) != 0) {
                         {
                             setState(143);
                             expression();
@@ -1057,14 +1085,10 @@ public final RstatementContext rstatement() throws RecognitionException {
                     setState(146);
                     match(SEMICOLON);
                     setState(148);
+                    _errHandler.sync(this);
                     _la = _input.LA(1);
-                    if ((((_la) & ~0x3f) == 0
-                        && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << DOLLAR) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT)
-                            | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0)
-                        || ((((_la - 73)) & ~0x3f) == 0
-                            && ((1L << (_la - 73)) & ((1L << (OCTAL - 73)) | (1L << (HEX - 73)) | (1L << (INTEGER - 73)) | (1L << (DECIMAL
-                                - 73)) | (1L << (STRING - 73)) | (1L << (REGEX - 73)) | (1L << (TRUE - 73)) | (1L << (FALSE - 73)) | (1L
-                                    << (NULL - 73)) | (1L << (ID - 73)))) != 0)) {
+                    if (((_la) & ~0x3f) == 0 && ((1L << _la) & 1729382310068880032L) != 0
+                        || (((_la - 73)) & ~0x3f) == 0 && ((1L << (_la - 73)) & 2559L) != 0) {
                         {
                             setState(147);
                             afterthought();
@@ -1074,6 +1098,7 @@ public final RstatementContext rstatement() throws RecognitionException {
                     setState(150);
                     match(RP);
                     setState(153);
+                    _errHandler.sync(this);
                     switch (_input.LA(1)) {
                         case LBRACK:
                         case LBRACE:
@@ -1200,6 +1225,7 @@ public final RstatementContext rstatement() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class DstatementContext extends ParserRuleContext {
         public DstatementContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -1217,6 +1243,7 @@ public void copyFrom(DstatementContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class DeclContext extends DstatementContext {
         public DeclarationContext declaration() {
             return getRuleContext(DeclarationContext.class, 0);
@@ -1233,6 +1260,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class BreakContext extends DstatementContext {
         public TerminalNode BREAK() {
             return getToken(PainlessParser.BREAK, 0);
@@ -1249,6 +1277,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ThrowContext extends DstatementContext {
         public TerminalNode THROW() {
             return getToken(PainlessParser.THROW, 0);
@@ -1269,6 +1298,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ContinueContext extends DstatementContext {
         public TerminalNode CONTINUE() {
             return getToken(PainlessParser.CONTINUE, 0);
@@ -1285,6 +1315,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ExprContext extends DstatementContext {
         public ExpressionContext expression() {
             return getRuleContext(ExpressionContext.class, 0);
@@ -1301,6 +1332,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class DoContext extends DstatementContext {
         public TerminalNode DO() {
             return getToken(PainlessParser.DO, 0);
@@ -1337,6 +1369,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ReturnContext extends DstatementContext {
         public TerminalNode RETURN() {
             return getToken(PainlessParser.RETURN, 0);
@@ -1409,14 +1442,10 @@ public final DstatementContext dstatement() throws RecognitionException {
                     setState(191);
                     match(RETURN);
                     setState(193);
+                    _errHandler.sync(this);
                     _la = _input.LA(1);
-                    if ((((_la) & ~0x3f) == 0
-                        && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << DOLLAR) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT)
-                            | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0)
-                        || ((((_la - 73)) & ~0x3f) == 0
-                            && ((1L << (_la - 73)) & ((1L << (OCTAL - 73)) | (1L << (HEX - 73)) | (1L << (INTEGER - 73)) | (1L << (DECIMAL
-                                - 73)) | (1L << (STRING - 73)) | (1L << (REGEX - 73)) | (1L << (TRUE - 73)) | (1L << (FALSE - 73)) | (1L
-                                    << (NULL - 73)) | (1L << (ID - 73)))) != 0)) {
+                    if (((_la) & ~0x3f) == 0 && ((1L << _la) & 1729382310068880032L) != 0
+                        || (((_la - 73)) & ~0x3f) == 0 && ((1L << (_la - 73)) & 2559L) != 0) {
                         {
                             setState(192);
                             expression();
@@ -1452,6 +1481,7 @@ public final DstatementContext dstatement() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class TrailerContext extends ParserRuleContext {
         public BlockContext block() {
             return getRuleContext(BlockContext.class, 0);
@@ -1482,6 +1512,7 @@ public final TrailerContext trailer() throws RecognitionException {
         enterRule(_localctx, 12, RULE_trailer);
         try {
             setState(202);
+            _errHandler.sync(this);
             switch (_input.LA(1)) {
                 case LBRACK:
                     enterOuterAlt(_localctx, 1); {
@@ -1538,6 +1569,7 @@ public final TrailerContext trailer() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class BlockContext extends ParserRuleContext {
         public TerminalNode LBRACK() {
             return getToken(PainlessParser.LBRACK, 0);
@@ -1602,15 +1634,10 @@ public final BlockContext block() throws RecognitionException {
                     _alt = getInterpreter().adaptivePredict(_input, 16, _ctx);
                 }
                 setState(212);
+                _errHandler.sync(this);
                 _la = _input.LA(1);
-                if ((((_la) & ~0x3f) == 0
-                    && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << DOLLAR) | (1L << DO) | (1L << CONTINUE) | (1L << BREAK) | (1L
-                        << RETURN) | (1L << NEW) | (1L << THROW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L
-                            << INCR) | (1L << DECR))) != 0)
-                    || ((((_la - 73)) & ~0x3f) == 0
-                        && ((1L << (_la - 73)) & ((1L << (OCTAL - 73)) | (1L << (HEX - 73)) | (1L << (INTEGER - 73)) | (1L << (DECIMAL
-                            - 73)) | (1L << (STRING - 73)) | (1L << (REGEX - 73)) | (1L << (TRUE - 73)) | (1L << (FALSE - 73)) | (1L
-                                << (NULL - 73)) | (1L << (PRIMITIVE - 73)) | (1L << (DEF - 73)) | (1L << (ID - 73)))) != 0)) {
+                if (((_la) & ~0x3f) == 0 && ((1L << _la) & 1729382310143591072L) != 0
+                    || (((_la - 73)) & ~0x3f) == 0 && ((1L << (_la - 73)) & 4095L) != 0) {
                     {
                         setState(211);
                         dstatement();
@@ -1630,6 +1657,7 @@ public final BlockContext block() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class EmptyContext extends ParserRuleContext {
         public TerminalNode SEMICOLON() {
             return getToken(PainlessParser.SEMICOLON, 0);
@@ -1670,6 +1698,7 @@ public final EmptyContext empty() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class InitializerContext extends ParserRuleContext {
         public DeclarationContext declaration() {
             return getRuleContext(DeclarationContext.class, 0);
@@ -1725,6 +1754,7 @@ public final InitializerContext initializer() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class AfterthoughtContext extends ParserRuleContext {
         public ExpressionContext expression() {
             return getRuleContext(ExpressionContext.class, 0);
@@ -1765,6 +1795,7 @@ public final AfterthoughtContext afterthought() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class DeclarationContext extends ParserRuleContext {
         public DecltypeContext decltype() {
             return getRuleContext(DecltypeContext.class, 0);
@@ -1840,6 +1871,7 @@ public final DeclarationContext declaration() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class DecltypeContext extends ParserRuleContext {
         public TypeContext type() {
             return getRuleContext(TypeContext.class, 0);
@@ -1915,6 +1947,7 @@ public final DecltypeContext decltype() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class TypeContext extends ParserRuleContext {
         public TerminalNode DEF() {
             return getToken(PainlessParser.DEF, 0);
@@ -1966,6 +1999,7 @@ public final TypeContext type() throws RecognitionException {
         try {
             int _alt;
             setState(251);
+            _errHandler.sync(this);
             switch (_input.LA(1)) {
                 case DEF:
                     enterOuterAlt(_localctx, 1); {
@@ -2016,6 +2050,7 @@ public final TypeContext type() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class DeclvarContext extends ParserRuleContext {
         public TerminalNode ID() {
             return getToken(PainlessParser.ID, 0);
@@ -2055,6 +2090,7 @@ public final DeclvarContext declvar() throws RecognitionException {
                 setState(253);
                 match(ID);
                 setState(256);
+                _errHandler.sync(this);
                 _la = _input.LA(1);
                 if (_la == ASSIGN) {
                     {
@@ -2076,6 +2112,7 @@ public final DeclvarContext declvar() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class TrapContext extends ParserRuleContext {
         public TerminalNode CATCH() {
             return getToken(PainlessParser.CATCH, 0);
@@ -2146,6 +2183,7 @@ public final TrapContext trap() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class NoncondexpressionContext extends ParserRuleContext {
         public NoncondexpressionContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -2163,6 +2201,7 @@ public void copyFrom(NoncondexpressionContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class SingleContext extends NoncondexpressionContext {
         public UnaryContext unary() {
             return getRuleContext(UnaryContext.class, 0);
@@ -2179,6 +2218,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class CompContext extends NoncondexpressionContext {
         public List<NoncondexpressionContext> noncondexpression() {
             return getRuleContexts(NoncondexpressionContext.class);
@@ -2231,6 +2271,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class BoolContext extends NoncondexpressionContext {
         public List<NoncondexpressionContext> noncondexpression() {
             return getRuleContexts(NoncondexpressionContext.class);
@@ -2259,6 +2300,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class BinaryContext extends NoncondexpressionContext {
         public List<NoncondexpressionContext> noncondexpression() {
             return getRuleContexts(NoncondexpressionContext.class);
@@ -2331,6 +2373,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ElvisContext extends NoncondexpressionContext {
         public List<NoncondexpressionContext> noncondexpression() {
             return getRuleContexts(NoncondexpressionContext.class);
@@ -2355,6 +2398,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class InstanceofContext extends NoncondexpressionContext {
         public NoncondexpressionContext noncondexpression() {
             return getRuleContext(NoncondexpressionContext.class, 0);
@@ -2422,9 +2466,11 @@ private NoncondexpressionContext noncondexpression(int _p) throws RecognitionExc
                                     if (!(precpred(_ctx, 13))) throw new FailedPredicateException(this, "precpred(_ctx, 13)");
                                     setState(269);
                                     _la = _input.LA(1);
-                                    if (!((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << MUL) | (1L << DIV) | (1L << REM))) != 0))) {
+                                    if (!(((_la) & ~0x3f) == 0 && ((1L << _la) & 15032385536L) != 0)) {
                                         _errHandler.recoverInline(this);
                                     } else {
+                                        if (_input.LA(1) == Token.EOF) matchedEOF = true;
+                                        _errHandler.reportMatch(this);
                                         consume();
                                     }
                                     setState(270);
@@ -2441,6 +2487,8 @@ private NoncondexpressionContext noncondexpression(int _p) throws RecognitionExc
                                     if (!(_la == ADD || _la == SUB)) {
                                         _errHandler.recoverInline(this);
                                     } else {
+                                        if (_input.LA(1) == Token.EOF) matchedEOF = true;
+                                        _errHandler.reportMatch(this);
                                         consume();
                                     }
                                     setState(273);
@@ -2457,6 +2505,8 @@ private NoncondexpressionContext noncondexpression(int _p) throws RecognitionExc
                                     if (!(_la == FIND || _la == MATCH)) {
                                         _errHandler.recoverInline(this);
                                     } else {
+                                        if (_input.LA(1) == Token.EOF) matchedEOF = true;
+                                        _errHandler.reportMatch(this);
                                         consume();
                                     }
                                     setState(276);
@@ -2470,9 +2520,11 @@ private NoncondexpressionContext noncondexpression(int _p) throws RecognitionExc
                                     if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)");
                                     setState(278);
                                     _la = _input.LA(1);
-                                    if (!((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LSH) | (1L << RSH) | (1L << USH))) != 0))) {
+                                    if (!(((_la) & ~0x3f) == 0 && ((1L << _la) & 481036337152L) != 0)) {
                                         _errHandler.recoverInline(this);
                                     } else {
+                                        if (_input.LA(1) == Token.EOF) matchedEOF = true;
+                                        _errHandler.reportMatch(this);
                                         consume();
                                     }
                                     setState(279);
@@ -2486,10 +2538,11 @@ private NoncondexpressionContext noncondexpression(int _p) throws RecognitionExc
                                     if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)");
                                     setState(281);
                                     _la = _input.LA(1);
-                                    if (!((((_la) & ~0x3f) == 0
-                                        && ((1L << _la) & ((1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0))) {
+                                    if (!(((_la) & ~0x3f) == 0 && ((1L << _la) & 8246337208320L) != 0)) {
                                         _errHandler.recoverInline(this);
                                     } else {
+                                        if (_input.LA(1) == Token.EOF) matchedEOF = true;
+                                        _errHandler.reportMatch(this);
                                         consume();
                                     }
                                     setState(282);
@@ -2503,10 +2556,11 @@ private NoncondexpressionContext noncondexpression(int _p) throws RecognitionExc
                                     if (!(precpred(_ctx, 7))) throw new FailedPredicateException(this, "precpred(_ctx, 7)");
                                     setState(284);
                                     _la = _input.LA(1);
-                                    if (!((((_la) & ~0x3f) == 0
-                                        && ((1L << _la) & ((1L << EQ) | (1L << EQR) | (1L << NE) | (1L << NER))) != 0))) {
+                                    if (!(((_la) & ~0x3f) == 0 && ((1L << _la) & 131941395333120L) != 0)) {
                                         _errHandler.recoverInline(this);
                                     } else {
+                                        if (_input.LA(1) == Token.EOF) matchedEOF = true;
+                                        _errHandler.reportMatch(this);
                                         consume();
                                     }
                                     setState(285);
@@ -2608,6 +2662,7 @@ private NoncondexpressionContext noncondexpression(int _p) throws RecognitionExc
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ExpressionContext extends ParserRuleContext {
         public ExpressionContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -2625,6 +2680,7 @@ public void copyFrom(ExpressionContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ConditionalContext extends ExpressionContext {
         public NoncondexpressionContext noncondexpression() {
             return getRuleContext(NoncondexpressionContext.class, 0);
@@ -2657,6 +2713,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class AssignmentContext extends ExpressionContext {
         public NoncondexpressionContext noncondexpression() {
             return getRuleContext(NoncondexpressionContext.class, 0);
@@ -2725,6 +2782,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class NonconditionalContext extends ExpressionContext {
         public NoncondexpressionContext noncondexpression() {
             return getRuleContext(NoncondexpressionContext.class, 0);
@@ -2778,12 +2836,11 @@ public final ExpressionContext expression() throws RecognitionException {
                     noncondexpression(0);
                     setState(320);
                     _la = _input.LA(1);
-                    if (!(((((_la - 61)) & ~0x3f) == 0
-                        && ((1L << (_la - 61)) & ((1L << (ASSIGN - 61)) | (1L << (AADD - 61)) | (1L << (ASUB - 61)) | (1L << (AMUL - 61))
-                            | (1L << (ADIV - 61)) | (1L << (AREM - 61)) | (1L << (AAND - 61)) | (1L << (AXOR - 61)) | (1L << (AOR - 61))
-                            | (1L << (ALSH - 61)) | (1L << (ARSH - 61)) | (1L << (AUSH - 61)))) != 0))) {
+                    if (!((((_la - 61)) & ~0x3f) == 0 && ((1L << (_la - 61)) & 4095L) != 0)) {
                         _errHandler.recoverInline(this);
                     } else {
+                        if (_input.LA(1) == Token.EOF) matchedEOF = true;
+                        _errHandler.reportMatch(this);
                         consume();
                     }
                     setState(321);
@@ -2801,6 +2858,7 @@ public final ExpressionContext expression() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class UnaryContext extends ParserRuleContext {
         public UnaryContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -2818,6 +2876,7 @@ public void copyFrom(UnaryContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class NotaddsubContext extends UnaryContext {
         public UnarynotaddsubContext unarynotaddsub() {
             return getRuleContext(UnarynotaddsubContext.class, 0);
@@ -2834,6 +2893,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class PreContext extends UnaryContext {
         public ChainContext chain() {
             return getRuleContext(ChainContext.class, 0);
@@ -2858,6 +2918,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class AddsubContext extends UnaryContext {
         public UnaryContext unary() {
             return getRuleContext(UnaryContext.class, 0);
@@ -2888,6 +2949,7 @@ public final UnaryContext unary() throws RecognitionException {
         int _la;
         try {
             setState(330);
+            _errHandler.sync(this);
             switch (_input.LA(1)) {
                 case INCR:
                 case DECR:
@@ -2898,6 +2960,8 @@ public final UnaryContext unary() throws RecognitionException {
                     if (!(_la == INCR || _la == DECR)) {
                         _errHandler.recoverInline(this);
                     } else {
+                        if (_input.LA(1) == Token.EOF) matchedEOF = true;
+                        _errHandler.reportMatch(this);
                         consume();
                     }
                     setState(326);
@@ -2913,6 +2977,8 @@ public final UnaryContext unary() throws RecognitionException {
                     if (!(_la == ADD || _la == SUB)) {
                         _errHandler.recoverInline(this);
                     } else {
+                        if (_input.LA(1) == Token.EOF) matchedEOF = true;
+                        _errHandler.reportMatch(this);
                         consume();
                     }
                     setState(328);
@@ -2954,6 +3020,7 @@ public final UnaryContext unary() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class UnarynotaddsubContext extends ParserRuleContext {
         public UnarynotaddsubContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -2971,6 +3038,7 @@ public void copyFrom(UnarynotaddsubContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class CastContext extends UnarynotaddsubContext {
         public CastexpressionContext castexpression() {
             return getRuleContext(CastexpressionContext.class, 0);
@@ -2987,6 +3055,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class NotContext extends UnarynotaddsubContext {
         public UnaryContext unary() {
             return getRuleContext(UnaryContext.class, 0);
@@ -3011,6 +3080,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ReadContext extends UnarynotaddsubContext {
         public ChainContext chain() {
             return getRuleContext(ChainContext.class, 0);
@@ -3027,6 +3097,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class PostContext extends UnarynotaddsubContext {
         public ChainContext chain() {
             return getRuleContext(ChainContext.class, 0);
@@ -3076,6 +3147,8 @@ public final UnarynotaddsubContext unarynotaddsub() throws RecognitionException
                     if (!(_la == INCR || _la == DECR)) {
                         _errHandler.recoverInline(this);
                     } else {
+                        if (_input.LA(1) == Token.EOF) matchedEOF = true;
+                        _errHandler.reportMatch(this);
                         consume();
                     }
                 }
@@ -3088,6 +3161,8 @@ public final UnarynotaddsubContext unarynotaddsub() throws RecognitionException
                     if (!(_la == BOOLNOT || _la == BWNOT)) {
                         _errHandler.recoverInline(this);
                     } else {
+                        if (_input.LA(1) == Token.EOF) matchedEOF = true;
+                        _errHandler.reportMatch(this);
                         consume();
                     }
                     setState(337);
@@ -3112,6 +3187,7 @@ public final UnarynotaddsubContext unarynotaddsub() throws RecognitionException
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class CastexpressionContext extends ParserRuleContext {
         public CastexpressionContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -3129,6 +3205,7 @@ public void copyFrom(CastexpressionContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class RefcastContext extends CastexpressionContext {
         public TerminalNode LP() {
             return getToken(PainlessParser.LP, 0);
@@ -3157,6 +3234,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class PrimordefcastContext extends CastexpressionContext {
         public TerminalNode LP() {
             return getToken(PainlessParser.LP, 0);
@@ -3229,6 +3307,7 @@ public final CastexpressionContext castexpression() throws RecognitionException
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class PrimordefcasttypeContext extends ParserRuleContext {
         public TerminalNode DEF() {
             return getToken(PainlessParser.DEF, 0);
@@ -3268,6 +3347,8 @@ public final PrimordefcasttypeContext primordefcasttype() throws RecognitionExce
                 if (!(_la == PRIMITIVE || _la == DEF)) {
                     _errHandler.recoverInline(this);
                 } else {
+                    if (_input.LA(1) == Token.EOF) matchedEOF = true;
+                    _errHandler.reportMatch(this);
                     consume();
                 }
             }
@@ -3281,6 +3362,7 @@ public final PrimordefcasttypeContext primordefcasttype() throws RecognitionExce
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class RefcasttypeContext extends ParserRuleContext {
         public TerminalNode DEF() {
             return getToken(PainlessParser.DEF, 0);
@@ -3348,6 +3430,7 @@ public final RefcasttypeContext refcasttype() throws RecognitionException {
         int _la;
         try {
             setState(384);
+            _errHandler.sync(this);
             switch (_input.LA(1)) {
                 case DEF:
                     enterOuterAlt(_localctx, 1); {
@@ -3444,6 +3527,7 @@ public final RefcasttypeContext refcasttype() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ChainContext extends ParserRuleContext {
         public ChainContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -3461,6 +3545,7 @@ public void copyFrom(ChainContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class DynamicContext extends ChainContext {
         public PrimaryContext primary() {
             return getRuleContext(PrimaryContext.class, 0);
@@ -3485,6 +3570,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class NewarrayContext extends ChainContext {
         public ArrayinitializerContext arrayinitializer() {
             return getRuleContext(ArrayinitializerContext.class, 0);
@@ -3550,6 +3636,7 @@ public final ChainContext chain() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class PrimaryContext extends ParserRuleContext {
         public PrimaryContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -3567,6 +3654,7 @@ public void copyFrom(PrimaryContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ListinitContext extends PrimaryContext {
         public ListinitializerContext listinitializer() {
             return getRuleContext(ListinitializerContext.class, 0);
@@ -3583,6 +3671,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class RegexContext extends PrimaryContext {
         public TerminalNode REGEX() {
             return getToken(PainlessParser.REGEX, 0);
@@ -3599,6 +3688,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class NullContext extends PrimaryContext {
         public TerminalNode NULL() {
             return getToken(PainlessParser.NULL, 0);
@@ -3615,6 +3705,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class StringContext extends PrimaryContext {
         public TerminalNode STRING() {
             return getToken(PainlessParser.STRING, 0);
@@ -3631,6 +3722,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class MapinitContext extends PrimaryContext {
         public MapinitializerContext mapinitializer() {
             return getRuleContext(MapinitializerContext.class, 0);
@@ -3647,6 +3739,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class CalllocalContext extends PrimaryContext {
         public ArgumentsContext arguments() {
             return getRuleContext(ArgumentsContext.class, 0);
@@ -3671,6 +3764,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class TrueContext extends PrimaryContext {
         public TerminalNode TRUE() {
             return getToken(PainlessParser.TRUE, 0);
@@ -3687,6 +3781,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class FalseContext extends PrimaryContext {
         public TerminalNode FALSE() {
             return getToken(PainlessParser.FALSE, 0);
@@ -3703,6 +3798,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class VariableContext extends PrimaryContext {
         public TerminalNode ID() {
             return getToken(PainlessParser.ID, 0);
@@ -3719,6 +3815,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class NumericContext extends PrimaryContext {
         public TerminalNode OCTAL() {
             return getToken(PainlessParser.OCTAL, 0);
@@ -3747,6 +3844,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class NewobjectContext extends PrimaryContext {
         public TerminalNode NEW() {
             return getToken(PainlessParser.NEW, 0);
@@ -3771,6 +3869,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class PrecedenceContext extends PrimaryContext {
         public TerminalNode LP() {
             return getToken(PainlessParser.LP, 0);
@@ -3819,11 +3918,11 @@ public final PrimaryContext primary() throws RecognitionException {
                     enterOuterAlt(_localctx, 2); {
                     setState(400);
                     _la = _input.LA(1);
-                    if (!(((((_la - 73)) & ~0x3f) == 0
-                        && ((1L << (_la - 73)) & ((1L << (OCTAL - 73)) | (1L << (HEX - 73)) | (1L << (INTEGER - 73)) | (1L << (DECIMAL
-                            - 73)))) != 0))) {
+                    if (!((((_la - 73)) & ~0x3f) == 0 && ((1L << (_la - 73)) & 15L) != 0)) {
                         _errHandler.recoverInline(this);
                     } else {
+                        if (_input.LA(1) == Token.EOF) matchedEOF = true;
+                        _errHandler.reportMatch(this);
                         consume();
                     }
                 }
@@ -3892,6 +3991,8 @@ public final PrimaryContext primary() throws RecognitionException {
                     if (!(_la == DOLLAR || _la == ID)) {
                         _errHandler.recoverInline(this);
                     } else {
+                        if (_input.LA(1) == Token.EOF) matchedEOF = true;
+                        _errHandler.reportMatch(this);
                         consume();
                     }
                     setState(410);
@@ -3920,6 +4021,7 @@ public final PrimaryContext primary() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class PostfixContext extends ParserRuleContext {
         public CallinvokeContext callinvoke() {
             return getRuleContext(CallinvokeContext.class, 0);
@@ -3985,6 +4087,7 @@ public final PostfixContext postfix() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class PostdotContext extends ParserRuleContext {
         public CallinvokeContext callinvoke() {
             return getRuleContext(CallinvokeContext.class, 0);
@@ -4040,6 +4143,7 @@ public final PostdotContext postdot() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class CallinvokeContext extends ParserRuleContext {
         public TerminalNode DOTID() {
             return getToken(PainlessParser.DOTID, 0);
@@ -4085,6 +4189,8 @@ public final CallinvokeContext callinvoke() throws RecognitionException {
                 if (!(_la == DOT || _la == NSDOT)) {
                     _errHandler.recoverInline(this);
                 } else {
+                    if (_input.LA(1) == Token.EOF) matchedEOF = true;
+                    _errHandler.reportMatch(this);
                     consume();
                 }
                 setState(427);
@@ -4102,6 +4208,7 @@ public final CallinvokeContext callinvoke() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class FieldaccessContext extends ParserRuleContext {
         public TerminalNode DOT() {
             return getToken(PainlessParser.DOT, 0);
@@ -4147,6 +4254,8 @@ public final FieldaccessContext fieldaccess() throws RecognitionException {
                 if (!(_la == DOT || _la == NSDOT)) {
                     _errHandler.recoverInline(this);
                 } else {
+                    if (_input.LA(1) == Token.EOF) matchedEOF = true;
+                    _errHandler.reportMatch(this);
                     consume();
                 }
                 setState(431);
@@ -4154,6 +4263,8 @@ public final FieldaccessContext fieldaccess() throws RecognitionException {
                 if (!(_la == DOTINTEGER || _la == DOTID)) {
                     _errHandler.recoverInline(this);
                 } else {
+                    if (_input.LA(1) == Token.EOF) matchedEOF = true;
+                    _errHandler.reportMatch(this);
                     consume();
                 }
             }
@@ -4167,6 +4278,7 @@ public final FieldaccessContext fieldaccess() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class BraceaccessContext extends ParserRuleContext {
         public TerminalNode LBRACE() {
             return getToken(PainlessParser.LBRACE, 0);
@@ -4219,6 +4331,7 @@ public final BraceaccessContext braceaccess() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ArrayinitializerContext extends ParserRuleContext {
         public ArrayinitializerContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -4236,6 +4349,7 @@ public void copyFrom(ArrayinitializerContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class NewstandardarrayContext extends ArrayinitializerContext {
         public TerminalNode NEW() {
             return getToken(PainlessParser.NEW, 0);
@@ -4292,6 +4406,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class NewinitializedarrayContext extends ArrayinitializerContext {
         public TerminalNode NEW() {
             return getToken(PainlessParser.NEW, 0);
@@ -4434,14 +4549,10 @@ public final ArrayinitializerContext arrayinitializer() throws RecognitionExcept
                     setState(460);
                     match(LBRACK);
                     setState(469);
+                    _errHandler.sync(this);
                     _la = _input.LA(1);
-                    if ((((_la) & ~0x3f) == 0
-                        && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << DOLLAR) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT)
-                            | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0)
-                        || ((((_la - 73)) & ~0x3f) == 0
-                            && ((1L << (_la - 73)) & ((1L << (OCTAL - 73)) | (1L << (HEX - 73)) | (1L << (INTEGER - 73)) | (1L << (DECIMAL
-                                - 73)) | (1L << (STRING - 73)) | (1L << (REGEX - 73)) | (1L << (TRUE - 73)) | (1L << (FALSE - 73)) | (1L
-                                    << (NULL - 73)) | (1L << (ID - 73)))) != 0)) {
+                    if (((_la) & ~0x3f) == 0 && ((1L << _la) & 1729382310068880032L) != 0
+                        || (((_la - 73)) & ~0x3f) == 0 && ((1L << (_la - 73)) & 2559L) != 0) {
                         {
                             setState(461);
                             expression();
@@ -4495,6 +4606,7 @@ public final ArrayinitializerContext arrayinitializer() throws RecognitionExcept
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ListinitializerContext extends ParserRuleContext {
         public TerminalNode LBRACE() {
             return getToken(PainlessParser.LBRACE, 0);
@@ -4589,6 +4701,7 @@ public final ListinitializerContext listinitializer() throws RecognitionExceptio
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class MapinitializerContext extends ParserRuleContext {
         public TerminalNode LBRACE() {
             return getToken(PainlessParser.LBRACE, 0);
@@ -4689,6 +4802,7 @@ public final MapinitializerContext mapinitializer() throws RecognitionException
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class MaptokenContext extends ParserRuleContext {
         public List<ExpressionContext> expression() {
             return getRuleContexts(ExpressionContext.class);
@@ -4741,6 +4855,7 @@ public final MaptokenContext maptoken() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ArgumentsContext extends ParserRuleContext {
         public TerminalNode LP() {
             return getToken(PainlessParser.LP, 0);
@@ -4793,14 +4908,10 @@ public final ArgumentsContext arguments() throws RecognitionException {
                     setState(515);
                     match(LP);
                     setState(524);
+                    _errHandler.sync(this);
                     _la = _input.LA(1);
-                    if ((((_la) & ~0x3f) == 0
-                        && ((1L << _la) & ((1L << LBRACE) | (1L << LP) | (1L << DOLLAR) | (1L << NEW) | (1L << THIS) | (1L << BOOLNOT) | (1L
-                            << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR))) != 0)
-                        || ((((_la - 73)) & ~0x3f) == 0
-                            && ((1L << (_la - 73)) & ((1L << (OCTAL - 73)) | (1L << (HEX - 73)) | (1L << (INTEGER - 73)) | (1L << (DECIMAL
-                                - 73)) | (1L << (STRING - 73)) | (1L << (REGEX - 73)) | (1L << (TRUE - 73)) | (1L << (FALSE - 73)) | (1L
-                                    << (NULL - 73)) | (1L << (PRIMITIVE - 73)) | (1L << (DEF - 73)) | (1L << (ID - 73)))) != 0)) {
+                    if (((_la) & ~0x3f) == 0 && ((1L << _la) & 1729382310203097760L) != 0
+                        || (((_la - 73)) & ~0x3f) == 0 && ((1L << (_la - 73)) & 4095L) != 0) {
                         {
                             setState(516);
                             argument();
@@ -4837,6 +4948,7 @@ public final ArgumentsContext arguments() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ArgumentContext extends ParserRuleContext {
         public ExpressionContext expression() {
             return getRuleContext(ExpressionContext.class, 0);
@@ -4902,6 +5014,7 @@ public final ArgumentContext argument() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class LambdaContext extends ParserRuleContext {
         public TerminalNode ARROW() {
             return getToken(PainlessParser.ARROW, 0);
@@ -4963,6 +5076,7 @@ public final LambdaContext lambda() throws RecognitionException {
             enterOuterAlt(_localctx, 1);
             {
                 setState(546);
+                _errHandler.sync(this);
                 switch (_input.LA(1)) {
                     case PRIMITIVE:
                     case DEF:
@@ -4975,9 +5089,9 @@ public final LambdaContext lambda() throws RecognitionException {
                         setState(534);
                         match(LP);
                         setState(543);
+                        _errHandler.sync(this);
                         _la = _input.LA(1);
-                        if (((((_la - 82)) & ~0x3f) == 0
-                            && ((1L << (_la - 82)) & ((1L << (PRIMITIVE - 82)) | (1L << (DEF - 82)) | (1L << (ID - 82)))) != 0)) {
+                        if ((((_la - 82)) & ~0x3f) == 0 && ((1L << (_la - 82)) & 7L) != 0) {
                             {
                                 setState(535);
                                 lamtype();
@@ -5010,6 +5124,7 @@ public final LambdaContext lambda() throws RecognitionException {
                 setState(548);
                 match(ARROW);
                 setState(551);
+                _errHandler.sync(this);
                 switch (_input.LA(1)) {
                     case LBRACK: {
                         setState(549);
@@ -5054,6 +5169,7 @@ public final LambdaContext lambda() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class LamtypeContext extends ParserRuleContext {
         public TerminalNode ID() {
             return getToken(PainlessParser.ID, 0);
@@ -5107,6 +5223,7 @@ public final LamtypeContext lamtype() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class FuncrefContext extends ParserRuleContext {
         public FuncrefContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -5124,6 +5241,7 @@ public void copyFrom(FuncrefContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ClassfuncrefContext extends FuncrefContext {
         public DecltypeContext decltype() {
             return getRuleContext(DecltypeContext.class, 0);
@@ -5148,6 +5266,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ConstructorfuncrefContext extends FuncrefContext {
         public DecltypeContext decltype() {
             return getRuleContext(DecltypeContext.class, 0);
@@ -5174,6 +5293,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class LocalfuncrefContext extends FuncrefContext {
         public TerminalNode THIS() {
             return getToken(PainlessParser.THIS, 0);
@@ -5299,224 +5419,395 @@ private boolean noncondexpression_sempred(NoncondexpressionContext _localctx, in
         return true;
     }
 
-    public static final String _serializedATN = "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3X\u023e\4\2\t\2\4"
-        + "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"
-        + "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"
-        + "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"
-        + "\4\32\t\32\4\33\t\33\4\34\t\34\4\35\t\35\4\36\t\36\4\37\t\37\4 \t \4!"
-        + "\t!\4\"\t\"\4#\t#\4$\t$\4%\t%\4&\t&\4\'\t\'\4(\t(\3\2\7\2R\n\2\f\2\16"
-        + "\2U\13\2\3\2\7\2X\n\2\f\2\16\2[\13\2\3\2\3\2\3\3\3\3\3\3\3\3\3\3\3\4\3"
-        + "\4\3\4\3\4\3\4\3\4\3\4\7\4k\n\4\f\4\16\4n\13\4\5\4p\n\4\3\4\3\4\3\5\3"
-        + "\5\3\5\3\5\5\5x\n\5\3\6\3\6\3\6\3\6\3\6\3\6\3\6\3\6\5\6\u0082\n\6\3\6"
-        + "\3\6\3\6\3\6\3\6\3\6\5\6\u008a\n\6\3\6\3\6\3\6\5\6\u008f\n\6\3\6\3\6\5"
-        + "\6\u0093\n\6\3\6\3\6\5\6\u0097\n\6\3\6\3\6\3\6\5\6\u009c\n\6\3\6\3\6\3"
-        + "\6\3\6\3\6\3\6\3\6\3\6\3\6\3\6\3\6\3\6\3\6\3\6\3\6\3\6\3\6\3\6\3\6\3\6"
-        + "\6\6\u00b2\n\6\r\6\16\6\u00b3\5\6\u00b6\n\6\3\7\3\7\3\7\3\7\3\7\3\7\3"
-        + "\7\3\7\3\7\3\7\3\7\3\7\5\7\u00c4\n\7\3\7\3\7\3\7\5\7\u00c9\n\7\3\b\3\b"
-        + "\5\b\u00cd\n\b\3\t\3\t\7\t\u00d1\n\t\f\t\16\t\u00d4\13\t\3\t\5\t\u00d7"
-        + "\n\t\3\t\3\t\3\n\3\n\3\13\3\13\5\13\u00df\n\13\3\f\3\f\3\r\3\r\3\r\3\r"
-        + "\7\r\u00e7\n\r\f\r\16\r\u00ea\13\r\3\16\3\16\3\16\7\16\u00ef\n\16\f\16"
-        + "\16\16\u00f2\13\16\3\17\3\17\3\17\3\17\3\17\7\17\u00f9\n\17\f\17\16\17"
-        + "\u00fc\13\17\5\17\u00fe\n\17\3\20\3\20\3\20\5\20\u0103\n\20\3\21\3\21"
-        + "\3\21\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22"
-        + "\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22"
-        + "\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22"
-        + "\3\22\3\22\3\22\3\22\3\22\7\22\u0136\n\22\f\22\16\22\u0139\13\22\3\23"
-        + "\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\5\23\u0146\n\23\3\24"
-        + "\3\24\3\24\3\24\3\24\5\24\u014d\n\24\3\25\3\25\3\25\3\25\3\25\3\25\3\25"
-        + "\5\25\u0156\n\25\3\26\3\26\3\26\3\26\3\26\3\26\3\26\3\26\3\26\3\26\5\26"
-        + "\u0162\n\26\3\27\3\27\3\30\3\30\3\30\6\30\u0169\n\30\r\30\16\30\u016a"
-        + "\3\30\3\30\3\30\6\30\u0170\n\30\r\30\16\30\u0171\3\30\3\30\3\30\7\30\u0177"
-        + "\n\30\f\30\16\30\u017a\13\30\3\30\3\30\7\30\u017e\n\30\f\30\16\30\u0181"
-        + "\13\30\5\30\u0183\n\30\3\31\3\31\7\31\u0187\n\31\f\31\16\31\u018a\13\31"
-        + "\3\31\5\31\u018d\n\31\3\32\3\32\3\32\3\32\3\32\3\32\3\32\3\32\3\32\3\32"
-        + "\3\32\3\32\3\32\3\32\3\32\3\32\3\32\3\32\3\32\5\32\u01a2\n\32\3\33\3\33"
-        + "\3\33\5\33\u01a7\n\33\3\34\3\34\5\34\u01ab\n\34\3\35\3\35\3\35\3\35\3"
-        + "\36\3\36\3\36\3\37\3\37\3\37\3\37\3 \3 \3 \3 \3 \3 \6 \u01be\n \r \16"
-        + " \u01bf\3 \3 \7 \u01c4\n \f \16 \u01c7\13 \5 \u01c9\n \3 \3 \3 \3 \3 "
-        + "\3 \3 \3 \7 \u01d3\n \f \16 \u01d6\13 \5 \u01d8\n \3 \3 \7 \u01dc\n \f"
-        + " \16 \u01df\13 \5 \u01e1\n \3!\3!\3!\3!\7!\u01e7\n!\f!\16!\u01ea\13!\3"
-        + "!\3!\3!\3!\5!\u01f0\n!\3\"\3\"\3\"\3\"\7\"\u01f6\n\"\f\"\16\"\u01f9\13"
-        + "\"\3\"\3\"\3\"\3\"\3\"\5\"\u0200\n\"\3#\3#\3#\3#\3$\3$\3$\3$\7$\u020a"
-        + "\n$\f$\16$\u020d\13$\5$\u020f\n$\3$\3$\3%\3%\3%\5%\u0216\n%\3&\3&\3&\3"
-        + "&\3&\7&\u021d\n&\f&\16&\u0220\13&\5&\u0222\n&\3&\5&\u0225\n&\3&\3&\3&"
-        + "\5&\u022a\n&\3\'\5\'\u022d\n\'\3\'\3\'\3(\3(\3(\3(\3(\3(\3(\3(\3(\3(\3"
-        + "(\5(\u023c\n(\3(\2\3\")\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*"
-        + ",.\60\62\64\668:<>@BDFHJLN\2\21\3\3\17\17\3\2!#\3\2$%\3\2;<\3\2&(\3\2"
-        + "),\3\2-\60\3\2?J\3\2=>\3\2\37 \3\2TU\3\2KN\4\2\13\13VV\3\2\f\r\3\2WX\u0279"
-        + "\2S\3\2\2\2\4^\3\2\2\2\6c\3\2\2\2\bw\3\2\2\2\n\u00b5\3\2\2\2\f\u00c8\3"
-        + "\2\2\2\16\u00cc\3\2\2\2\20\u00ce\3\2\2\2\22\u00da\3\2\2\2\24\u00de\3\2"
-        + "\2\2\26\u00e0\3\2\2\2\30\u00e2\3\2\2\2\32\u00eb\3\2\2\2\34\u00fd\3\2\2"
-        + "\2\36\u00ff\3\2\2\2 \u0104\3\2\2\2\"\u010b\3\2\2\2$\u0145\3\2\2\2&\u014c"
-        + "\3\2\2\2(\u0155\3\2\2\2*\u0161\3\2\2\2,\u0163\3\2\2\2.\u0182\3\2\2\2\60"
-        + "\u018c\3\2\2\2\62\u01a1\3\2\2\2\64\u01a6\3\2\2\2\66\u01aa\3\2\2\28\u01ac"
-        + "\3\2\2\2:\u01b0\3\2\2\2<\u01b3\3\2\2\2>\u01e0\3\2\2\2@\u01ef\3\2\2\2B"
-        + "\u01ff\3\2\2\2D\u0201\3\2\2\2F\u0205\3\2\2\2H\u0215\3\2\2\2J\u0224\3\2"
-        + "\2\2L\u022c\3\2\2\2N\u023b\3\2\2\2PR\5\4\3\2QP\3\2\2\2RU\3\2\2\2SQ\3\2"
-        + "\2\2ST\3\2\2\2TY\3\2\2\2US\3\2\2\2VX\5\b\5\2WV\3\2\2\2X[\3\2\2\2YW\3\2"
-        + "\2\2YZ\3\2\2\2Z\\\3\2\2\2[Y\3\2\2\2\\]\7\2\2\3]\3\3\2\2\2^_\5\32\16\2"
-        + "_`\7V\2\2`a\5\6\4\2ab\5\20\t\2b\5\3\2\2\2co\7\t\2\2de\5\32\16\2el\7V\2"
-        + "\2fg\7\16\2\2gh\5\32\16\2hi\7V\2\2ik\3\2\2\2jf\3\2\2\2kn\3\2\2\2lj\3\2"
-        + "\2\2lm\3\2\2\2mp\3\2\2\2nl\3\2\2\2od\3\2\2\2op\3\2\2\2pq\3\2\2\2qr\7\n"
-        + "\2\2r\7\3\2\2\2sx\5\n\6\2tu\5\f\7\2uv\t\2\2\2vx\3\2\2\2ws\3\2\2\2wt\3"
-        + "\2\2\2x\t\3\2\2\2yz\7\20\2\2z{\7\t\2\2{|\5$\23\2|}\7\n\2\2}\u0081\5\16"
-        + "\b\2~\177\7\22\2\2\177\u0082\5\16\b\2\u0080\u0082\6\6\2\2\u0081~\3\2\2"
-        + "\2\u0081\u0080\3\2\2\2\u0082\u00b6\3\2\2\2\u0083\u0084\7\23\2\2\u0084"
-        + "\u0085\7\t\2\2\u0085\u0086\5$\23\2\u0086\u0089\7\n\2\2\u0087\u008a\5\16"
-        + "\b\2\u0088\u008a\5\22\n\2\u0089\u0087\3\2\2\2\u0089\u0088\3\2\2\2\u008a"
-        + "\u00b6\3\2\2\2\u008b\u008c\7\25\2\2\u008c\u008e\7\t\2\2\u008d\u008f\5"
-        + "\24\13\2\u008e\u008d\3\2\2\2\u008e\u008f\3\2\2\2\u008f\u0090\3\2\2\2\u0090"
-        + "\u0092\7\17\2\2\u0091\u0093\5$\23\2\u0092\u0091\3\2\2\2\u0092\u0093\3"
-        + "\2\2\2\u0093\u0094\3\2\2\2\u0094\u0096\7\17\2\2\u0095\u0097\5\26\f\2\u0096"
-        + "\u0095\3\2\2\2\u0096\u0097\3\2\2\2\u0097\u0098\3\2\2\2\u0098\u009b\7\n"
-        + "\2\2\u0099\u009c\5\16\b\2\u009a\u009c\5\22\n\2\u009b\u0099\3\2\2\2\u009b"
-        + "\u009a\3\2\2\2\u009c\u00b6\3\2\2\2\u009d\u009e\7\25\2\2\u009e\u009f\7"
-        + "\t\2\2\u009f\u00a0\5\32\16\2\u00a0\u00a1\7V\2\2\u00a1\u00a2\7\67\2\2\u00a2"
-        + "\u00a3\5$\23\2\u00a3\u00a4\7\n\2\2\u00a4\u00a5\5\16\b\2\u00a5\u00b6\3"
-        + "\2\2\2\u00a6\u00a7\7\25\2\2\u00a7\u00a8\7\t\2\2\u00a8\u00a9\7V\2\2\u00a9"
-        + "\u00aa\7\21\2\2\u00aa\u00ab\5$\23\2\u00ab\u00ac\7\n\2\2\u00ac\u00ad\5"
-        + "\16\b\2\u00ad\u00b6\3\2\2\2\u00ae\u00af\7\32\2\2\u00af\u00b1\5\20\t\2"
-        + "\u00b0\u00b2\5 \21\2\u00b1\u00b0\3\2\2\2\u00b2\u00b3\3\2\2\2\u00b3\u00b1"
-        + "\3\2\2\2\u00b3\u00b4\3\2\2\2\u00b4\u00b6\3\2\2\2\u00b5y\3\2\2\2\u00b5"
-        + "\u0083\3\2\2\2\u00b5\u008b\3\2\2\2\u00b5\u009d\3\2\2\2\u00b5\u00a6\3\2"
-        + "\2\2\u00b5\u00ae\3\2\2\2\u00b6\13\3\2\2\2\u00b7\u00b8\7\24\2\2\u00b8\u00b9"
-        + "\5\20\t\2\u00b9\u00ba\7\23\2\2\u00ba\u00bb\7\t\2\2\u00bb\u00bc\5$\23\2"
-        + "\u00bc\u00bd\7\n\2\2\u00bd\u00c9\3\2\2\2\u00be\u00c9\5\30\r\2\u00bf\u00c9"
-        + "\7\26\2\2\u00c0\u00c9\7\27\2\2\u00c1\u00c3\7\30\2\2\u00c2\u00c4\5$\23"
-        + "\2\u00c3\u00c2\3\2\2\2\u00c3\u00c4\3\2\2\2\u00c4\u00c9\3\2\2\2\u00c5\u00c6"
-        + "\7\34\2\2\u00c6\u00c9\5$\23\2\u00c7\u00c9\5$\23\2\u00c8\u00b7\3\2\2\2"
-        + "\u00c8\u00be\3\2\2\2\u00c8\u00bf\3\2\2\2\u00c8\u00c0\3\2\2\2\u00c8\u00c1"
-        + "\3\2\2\2\u00c8\u00c5\3\2\2\2\u00c8\u00c7\3\2\2\2\u00c9\r\3\2\2\2\u00ca"
-        + "\u00cd\5\20\t\2\u00cb\u00cd\5\b\5\2\u00cc\u00ca\3\2\2\2\u00cc\u00cb\3"
-        + "\2\2\2\u00cd\17\3\2\2\2\u00ce\u00d2\7\5\2\2\u00cf\u00d1\5\b\5\2\u00d0"
-        + "\u00cf\3\2\2\2\u00d1\u00d4\3\2\2\2\u00d2\u00d0\3\2\2\2\u00d2\u00d3\3\2"
-        + "\2\2\u00d3\u00d6\3\2\2\2\u00d4\u00d2\3\2\2\2\u00d5\u00d7\5\f\7\2\u00d6"
-        + "\u00d5\3\2\2\2\u00d6\u00d7\3\2\2\2\u00d7\u00d8\3\2\2\2\u00d8\u00d9\7\6"
-        + "\2\2\u00d9\21\3\2\2\2\u00da\u00db\7\17\2\2\u00db\23\3\2\2\2\u00dc\u00df"
-        + "\5\30\r\2\u00dd\u00df\5$\23\2\u00de\u00dc\3\2\2\2\u00de\u00dd\3\2\2\2"
-        + "\u00df\25\3\2\2\2\u00e0\u00e1\5$\23\2\u00e1\27\3\2\2\2\u00e2\u00e3\5\32"
-        + "\16\2\u00e3\u00e8\5\36\20\2\u00e4\u00e5\7\16\2\2\u00e5\u00e7\5\36\20\2"
-        + "\u00e6\u00e4\3\2\2\2\u00e7\u00ea\3\2\2\2\u00e8\u00e6\3\2\2\2\u00e8\u00e9"
-        + "\3\2\2\2\u00e9\31\3\2\2\2\u00ea\u00e8\3\2\2\2\u00eb\u00f0\5\34\17\2\u00ec"
-        + "\u00ed\7\7\2\2\u00ed\u00ef\7\b\2\2\u00ee\u00ec\3\2\2\2\u00ef\u00f2\3\2"
-        + "\2\2\u00f0\u00ee\3\2\2\2\u00f0\u00f1\3\2\2\2\u00f1\33\3\2\2\2\u00f2\u00f0"
-        + "\3\2\2\2\u00f3\u00fe\7U\2\2\u00f4\u00fe\7T\2\2\u00f5\u00fa\7V\2\2\u00f6"
-        + "\u00f7\7\f\2\2\u00f7\u00f9\7X\2\2\u00f8\u00f6\3\2\2\2\u00f9\u00fc\3\2"
-        + "\2\2\u00fa\u00f8\3\2\2\2\u00fa\u00fb\3\2\2\2\u00fb\u00fe\3\2\2\2\u00fc"
-        + "\u00fa\3\2\2\2\u00fd\u00f3\3\2\2\2\u00fd\u00f4\3\2\2\2\u00fd\u00f5\3\2"
-        + "\2\2\u00fe\35\3\2\2\2\u00ff\u0102\7V\2\2\u0100\u0101\7?\2\2\u0101\u0103"
-        + "\5$\23\2\u0102\u0100\3\2\2\2\u0102\u0103\3\2\2\2\u0103\37\3\2\2\2\u0104"
-        + "\u0105\7\33\2\2\u0105\u0106\7\t\2\2\u0106\u0107\5\34\17\2\u0107\u0108"
-        + "\7V\2\2\u0108\u0109\7\n\2\2\u0109\u010a\5\20\t\2\u010a!\3\2\2\2\u010b"
-        + "\u010c\b\22\1\2\u010c\u010d\5&\24\2\u010d\u0137\3\2\2\2\u010e\u010f\f"
-        + "\17\2\2\u010f\u0110\t\3\2\2\u0110\u0136\5\"\22\20\u0111\u0112\f\16\2\2"
-        + "\u0112\u0113\t\4\2\2\u0113\u0136\5\"\22\17\u0114\u0115\f\r\2\2\u0115\u0116"
-        + "\t\5\2\2\u0116\u0136\5\"\22\16\u0117\u0118\f\f\2\2\u0118\u0119\t\6\2\2"
-        + "\u0119\u0136\5\"\22\r\u011a\u011b\f\13\2\2\u011b\u011c\t\7\2\2\u011c\u0136"
-        + "\5\"\22\f\u011d\u011e\f\t\2\2\u011e\u011f\t\b\2\2\u011f\u0136\5\"\22\n"
-        + "\u0120\u0121\f\b\2\2\u0121\u0122\7\61\2\2\u0122\u0136\5\"\22\t\u0123\u0124"
-        + "\f\7\2\2\u0124\u0125\7\62\2\2\u0125\u0136\5\"\22\b\u0126\u0127\f\6\2\2"
-        + "\u0127\u0128\7\63\2\2\u0128\u0136\5\"\22\7\u0129\u012a\f\5\2\2\u012a\u012b"
-        + "\7\64\2\2\u012b\u0136\5\"\22\6\u012c\u012d\f\4\2\2\u012d\u012e\7\65\2"
-        + "\2\u012e\u0136\5\"\22\5\u012f\u0130\f\3\2\2\u0130\u0131\78\2\2\u0131\u0136"
-        + "\5\"\22\3\u0132\u0133\f\n\2\2\u0133\u0134\7\36\2\2\u0134\u0136\5\32\16"
-        + "\2\u0135\u010e\3\2\2\2\u0135\u0111\3\2\2\2\u0135\u0114\3\2\2\2\u0135\u0117"
-        + "\3\2\2\2\u0135\u011a\3\2\2\2\u0135\u011d\3\2\2\2\u0135\u0120\3\2\2\2\u0135"
-        + "\u0123\3\2\2\2\u0135\u0126\3\2\2\2\u0135\u0129\3\2\2\2\u0135\u012c\3\2"
-        + "\2\2\u0135\u012f\3\2\2\2\u0135\u0132\3\2\2\2\u0136\u0139\3\2\2\2\u0137"
-        + "\u0135\3\2\2\2\u0137\u0138\3\2\2\2\u0138#\3\2\2\2\u0139\u0137\3\2\2\2"
-        + "\u013a\u0146\5\"\22\2\u013b\u013c\5\"\22\2\u013c\u013d\7\66\2\2\u013d"
-        + "\u013e\5$\23\2\u013e\u013f\7\67\2\2\u013f\u0140\5$\23\2\u0140\u0146\3"
-        + "\2\2\2\u0141\u0142\5\"\22\2\u0142\u0143\t\t\2\2\u0143\u0144\5$\23\2\u0144"
-        + "\u0146\3\2\2\2\u0145\u013a\3\2\2\2\u0145\u013b\3\2\2\2\u0145\u0141\3\2"
-        + "\2\2\u0146%\3\2\2\2\u0147\u0148\t\n\2\2\u0148\u014d\5\60\31\2\u0149\u014a"
-        + "\t\4\2\2\u014a\u014d\5&\24\2\u014b\u014d\5(\25\2\u014c\u0147\3\2\2\2\u014c"
-        + "\u0149\3\2\2\2\u014c\u014b\3\2\2\2\u014d\'\3\2\2\2\u014e\u0156\5\60\31"
-        + "\2\u014f\u0150\5\60\31\2\u0150\u0151\t\n\2\2\u0151\u0156\3\2\2\2\u0152"
-        + "\u0153\t\13\2\2\u0153\u0156\5&\24\2\u0154\u0156\5*\26\2\u0155\u014e\3"
-        + "\2\2\2\u0155\u014f\3\2\2\2\u0155\u0152\3\2\2\2\u0155\u0154\3\2\2\2\u0156"
-        + ")\3\2\2\2\u0157\u0158\7\t\2\2\u0158\u0159\5,\27\2\u0159\u015a\7\n\2\2"
-        + "\u015a\u015b\5&\24\2\u015b\u0162\3\2\2\2\u015c\u015d\7\t\2\2\u015d\u015e"
-        + "\5.\30\2\u015e\u015f\7\n\2\2\u015f\u0160\5(\25\2\u0160\u0162\3\2\2\2\u0161"
-        + "\u0157\3\2\2\2\u0161\u015c\3\2\2\2\u0162+\3\2\2\2\u0163\u0164\t\f\2\2"
-        + "\u0164-\3\2\2\2\u0165\u0168\7U\2\2\u0166\u0167\7\7\2\2\u0167\u0169\7\b"
-        + "\2\2\u0168\u0166\3\2\2\2\u0169\u016a\3\2\2\2\u016a\u0168\3\2\2\2\u016a"
-        + "\u016b\3\2\2\2\u016b\u0183\3\2\2\2\u016c\u016f\7T\2\2\u016d\u016e\7\7"
-        + "\2\2\u016e\u0170\7\b\2\2\u016f\u016d\3\2\2\2\u0170\u0171\3\2\2\2\u0171"
-        + "\u016f\3\2\2\2\u0171\u0172\3\2\2\2\u0172\u0183\3\2\2\2\u0173\u0178\7V"
-        + "\2\2\u0174\u0175\7\f\2\2\u0175\u0177\7X\2\2\u0176\u0174\3\2\2\2\u0177"
-        + "\u017a\3\2\2\2\u0178\u0176\3\2\2\2\u0178\u0179\3\2\2\2\u0179\u017f\3\2"
-        + "\2\2\u017a\u0178\3\2\2\2\u017b\u017c\7\7\2\2\u017c\u017e\7\b\2\2\u017d"
-        + "\u017b\3\2\2\2\u017e\u0181\3\2\2\2\u017f\u017d\3\2\2\2\u017f\u0180\3\2"
-        + "\2\2\u0180\u0183\3\2\2\2\u0181\u017f\3\2\2\2\u0182\u0165\3\2\2\2\u0182"
-        + "\u016c\3\2\2\2\u0182\u0173\3\2\2\2\u0183/\3\2\2\2\u0184\u0188\5\62\32"
-        + "\2\u0185\u0187\5\64\33\2\u0186\u0185\3\2\2\2\u0187\u018a\3\2\2\2\u0188"
-        + "\u0186\3\2\2\2\u0188\u0189\3\2\2\2\u0189\u018d\3\2\2\2\u018a\u0188\3\2"
-        + "\2\2\u018b\u018d\5> \2\u018c\u0184\3\2\2\2\u018c\u018b\3\2\2\2\u018d\61"
-        + "\3\2\2\2\u018e\u018f\7\t\2\2\u018f\u0190\5$\23\2\u0190\u0191\7\n\2\2\u0191"
-        + "\u01a2\3\2\2\2\u0192\u01a2\t\r\2\2\u0193\u01a2\7Q\2\2\u0194\u01a2\7R\2"
-        + "\2\u0195\u01a2\7S\2\2\u0196\u01a2\7O\2\2\u0197\u01a2\7P\2\2\u0198\u01a2"
-        + "\5@!\2\u0199\u01a2\5B\"\2\u019a\u01a2\7V\2\2\u019b\u019c\t\16\2\2\u019c"
-        + "\u01a2\5F$\2\u019d\u019e\7\31\2\2\u019e\u019f\5\34\17\2\u019f\u01a0\5"
-        + "F$\2\u01a0\u01a2\3\2\2\2\u01a1\u018e\3\2\2\2\u01a1\u0192\3\2\2\2\u01a1"
-        + "\u0193\3\2\2\2\u01a1\u0194\3\2\2\2\u01a1\u0195\3\2\2\2\u01a1\u0196\3\2"
-        + "\2\2\u01a1\u0197\3\2\2\2\u01a1\u0198\3\2\2\2\u01a1\u0199\3\2\2\2\u01a1"
-        + "\u019a\3\2\2\2\u01a1\u019b\3\2\2\2\u01a1\u019d\3\2\2\2\u01a2\63\3\2\2"
-        + "\2\u01a3\u01a7\58\35\2\u01a4\u01a7\5:\36\2\u01a5\u01a7\5<\37\2\u01a6\u01a3"
-        + "\3\2\2\2\u01a6\u01a4\3\2\2\2\u01a6\u01a5\3\2\2\2\u01a7\65\3\2\2\2\u01a8"
-        + "\u01ab\58\35\2\u01a9\u01ab\5:\36\2\u01aa\u01a8\3\2\2\2\u01aa\u01a9\3\2"
-        + "\2\2\u01ab\67\3\2\2\2\u01ac\u01ad\t\17\2\2\u01ad\u01ae\7X\2\2\u01ae\u01af"
-        + "\5F$\2\u01af9\3\2\2\2\u01b0\u01b1\t\17\2\2\u01b1\u01b2\t\20\2\2\u01b2"
-        + ";\3\2\2\2\u01b3\u01b4\7\7\2\2\u01b4\u01b5\5$\23\2\u01b5\u01b6\7\b\2\2"
-        + "\u01b6=\3\2\2\2\u01b7\u01b8\7\31\2\2\u01b8\u01bd\5\34\17\2\u01b9\u01ba"
-        + "\7\7\2\2\u01ba\u01bb\5$\23\2\u01bb\u01bc\7\b\2\2\u01bc\u01be\3\2\2\2\u01bd"
-        + "\u01b9\3\2\2\2\u01be\u01bf\3\2\2\2\u01bf\u01bd\3\2\2\2\u01bf\u01c0\3\2"
-        + "\2\2\u01c0\u01c8\3\2\2\2\u01c1\u01c5\5\66\34\2\u01c2\u01c4\5\64\33\2\u01c3"
-        + "\u01c2\3\2\2\2\u01c4\u01c7\3\2\2\2\u01c5\u01c3\3\2\2\2\u01c5\u01c6\3\2"
-        + "\2\2\u01c6\u01c9\3\2\2\2\u01c7\u01c5\3\2\2\2\u01c8\u01c1\3\2\2\2\u01c8"
-        + "\u01c9\3\2\2\2\u01c9\u01e1\3\2\2\2\u01ca\u01cb\7\31\2\2\u01cb\u01cc\5"
-        + "\34\17\2\u01cc\u01cd\7\7\2\2\u01cd\u01ce\7\b\2\2\u01ce\u01d7\7\5\2\2\u01cf"
-        + "\u01d4\5$\23\2\u01d0\u01d1\7\16\2\2\u01d1\u01d3\5$\23\2\u01d2\u01d0\3"
-        + "\2\2\2\u01d3\u01d6\3\2\2\2\u01d4\u01d2\3\2\2\2\u01d4\u01d5\3\2\2\2\u01d5"
-        + "\u01d8\3\2\2\2\u01d6\u01d4\3\2\2\2\u01d7\u01cf\3\2\2\2\u01d7\u01d8\3\2"
-        + "\2\2\u01d8\u01d9\3\2\2\2\u01d9\u01dd\7\6\2\2\u01da\u01dc\5\64\33\2\u01db"
-        + "\u01da\3\2\2\2\u01dc\u01df\3\2\2\2\u01dd\u01db\3\2\2\2\u01dd\u01de\3\2"
-        + "\2\2\u01de\u01e1\3\2\2\2\u01df\u01dd\3\2\2\2\u01e0\u01b7\3\2\2\2\u01e0"
-        + "\u01ca\3\2\2\2\u01e1?\3\2\2\2\u01e2\u01e3\7\7\2\2\u01e3\u01e8\5$\23\2"
-        + "\u01e4\u01e5\7\16\2\2\u01e5\u01e7\5$\23\2\u01e6\u01e4\3\2\2\2\u01e7\u01ea"
-        + "\3\2\2\2\u01e8\u01e6\3\2\2\2\u01e8\u01e9\3\2\2\2\u01e9\u01eb\3\2\2\2\u01ea"
-        + "\u01e8\3\2\2\2\u01eb\u01ec\7\b\2\2\u01ec\u01f0\3\2\2\2\u01ed\u01ee\7\7"
-        + "\2\2\u01ee\u01f0\7\b\2\2\u01ef\u01e2\3\2\2\2\u01ef\u01ed\3\2\2\2\u01f0"
-        + "A\3\2\2\2\u01f1\u01f2\7\7\2\2\u01f2\u01f7\5D#\2\u01f3\u01f4\7\16\2\2\u01f4"
-        + "\u01f6\5D#\2\u01f5\u01f3\3\2\2\2\u01f6\u01f9\3\2\2\2\u01f7\u01f5\3\2\2"
-        + "\2\u01f7\u01f8\3\2\2\2\u01f8\u01fa\3\2\2\2\u01f9\u01f7\3\2\2\2\u01fa\u01fb"
-        + "\7\b\2\2\u01fb\u0200\3\2\2\2\u01fc\u01fd\7\7\2\2\u01fd\u01fe\7\67\2\2"
-        + "\u01fe\u0200\7\b\2\2\u01ff\u01f1\3\2\2\2\u01ff\u01fc\3\2\2\2\u0200C\3"
-        + "\2\2\2\u0201\u0202\5$\23\2\u0202\u0203\7\67\2\2\u0203\u0204\5$\23\2\u0204"
-        + "E\3\2\2\2\u0205\u020e\7\t\2\2\u0206\u020b\5H%\2\u0207\u0208\7\16\2\2\u0208"
-        + "\u020a\5H%\2\u0209\u0207\3\2\2\2\u020a\u020d\3\2\2\2\u020b\u0209\3\2\2"
-        + "\2\u020b\u020c\3\2\2\2\u020c\u020f\3\2\2\2\u020d\u020b\3\2\2\2\u020e\u0206"
-        + "\3\2\2\2\u020e\u020f\3\2\2\2\u020f\u0210\3\2\2\2\u0210\u0211\7\n\2\2\u0211"
-        + "G\3\2\2\2\u0212\u0216\5$\23\2\u0213\u0216\5J&\2\u0214\u0216\5N(\2\u0215"
-        + "\u0212\3\2\2\2\u0215\u0213\3\2\2\2\u0215\u0214\3\2\2\2\u0216I\3\2\2\2"
-        + "\u0217\u0225\5L\'\2\u0218\u0221\7\t\2\2\u0219\u021e\5L\'\2\u021a\u021b"
-        + "\7\16\2\2\u021b\u021d\5L\'\2\u021c\u021a\3\2\2\2\u021d\u0220\3\2\2\2\u021e"
-        + "\u021c\3\2\2\2\u021e\u021f\3\2\2\2\u021f\u0222\3\2\2\2\u0220\u021e\3\2"
-        + "\2\2\u0221\u0219\3\2\2\2\u0221\u0222\3\2\2\2\u0222\u0223\3\2\2\2\u0223"
-        + "\u0225\7\n\2\2\u0224\u0217\3\2\2\2\u0224\u0218\3\2\2\2\u0225\u0226\3\2"
-        + "\2\2\u0226\u0229\7:\2\2\u0227\u022a\5\20\t\2\u0228\u022a\5$\23\2\u0229"
-        + "\u0227\3\2\2\2\u0229\u0228\3\2\2\2\u022aK\3\2\2\2\u022b\u022d\5\32\16"
-        + "\2\u022c\u022b\3\2\2\2\u022c\u022d\3\2\2\2\u022d\u022e\3\2\2\2\u022e\u022f"
-        + "\7V\2\2\u022fM\3\2\2\2\u0230\u0231\5\32\16\2\u0231\u0232\79\2\2\u0232"
-        + "\u0233\7V\2\2\u0233\u023c\3\2\2\2\u0234\u0235\5\32\16\2\u0235\u0236\7"
-        + "9\2\2\u0236\u0237\7\31\2\2\u0237\u023c\3\2\2\2\u0238\u0239\7\35\2\2\u0239"
-        + "\u023a\79\2\2\u023a\u023c\7V\2\2\u023b\u0230\3\2\2\2\u023b\u0234\3\2\2"
-        + "\2\u023b\u0238\3\2\2\2\u023cO\3\2\2\2>SYlow\u0081\u0089\u008e\u0092\u0096"
-        + "\u009b\u00b3\u00b5\u00c3\u00c8\u00cc\u00d2\u00d6\u00de\u00e8\u00f0\u00fa"
-        + "\u00fd\u0102\u0135\u0137\u0145\u014c\u0155\u0161\u016a\u0171\u0178\u017f"
-        + "\u0182\u0188\u018c\u01a1\u01a6\u01aa\u01bf\u01c5\u01c8\u01d4\u01d7\u01dd"
-        + "\u01e0\u01e8\u01ef\u01f7\u01ff\u020b\u020e\u0215\u021e\u0221\u0224\u0229"
-        + "\u022c\u023b";
+    public static final String _serializedATN = "\u0004\u0001V\u023c\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"
+        + "\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004\u0002"
+        + "\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007\u0002"
+        + "\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b\u0002"
+        + "\f\u0007\f\u0002\r\u0007\r\u0002\u000e\u0007\u000e\u0002\u000f\u0007\u000f"
+        + "\u0002\u0010\u0007\u0010\u0002\u0011\u0007\u0011\u0002\u0012\u0007\u0012"
+        + "\u0002\u0013\u0007\u0013\u0002\u0014\u0007\u0014\u0002\u0015\u0007\u0015"
+        + "\u0002\u0016\u0007\u0016\u0002\u0017\u0007\u0017\u0002\u0018\u0007\u0018"
+        + "\u0002\u0019\u0007\u0019\u0002\u001a\u0007\u001a\u0002\u001b\u0007\u001b"
+        + "\u0002\u001c\u0007\u001c\u0002\u001d\u0007\u001d\u0002\u001e\u0007\u001e"
+        + "\u0002\u001f\u0007\u001f\u0002 \u0007 \u0002!\u0007!\u0002\"\u0007\"\u0002"
+        + "#\u0007#\u0002$\u0007$\u0002%\u0007%\u0002&\u0007&\u0001\u0000\u0005\u0000"
+        + "P\b\u0000\n\u0000\f\u0000S\t\u0000\u0001\u0000\u0005\u0000V\b\u0000\n"
+        + "\u0000\f\u0000Y\t\u0000\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001"
+        + "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0002\u0001\u0002\u0001\u0002"
+        + "\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0005\u0002i\b\u0002"
+        + "\n\u0002\f\u0002l\t\u0002\u0003\u0002n\b\u0002\u0001\u0002\u0001\u0002"
+        + "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0003\u0003v\b\u0003"
+        + "\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004"
+        + "\u0001\u0004\u0001\u0004\u0003\u0004\u0080\b\u0004\u0001\u0004\u0001\u0004"
+        + "\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0003\u0004\u0088\b\u0004"
+        + "\u0001\u0004\u0001\u0004\u0001\u0004\u0003\u0004\u008d\b\u0004\u0001\u0004"
+        + "\u0001\u0004\u0003\u0004\u0091\b\u0004\u0001\u0004\u0001\u0004\u0003\u0004"
+        + "\u0095\b\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0003\u0004\u009a\b"
+        + "\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001"
+        + "\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001"
+        + "\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001"
+        + "\u0004\u0001\u0004\u0001\u0004\u0004\u0004\u00b0\b\u0004\u000b\u0004\f"
+        + "\u0004\u00b1\u0003\u0004\u00b4\b\u0004\u0001\u0005\u0001\u0005\u0001\u0005"
+        + "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"
+        + "\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00c2\b\u0005\u0001\u0005"
+        + "\u0001\u0005\u0001\u0005\u0003\u0005\u00c7\b\u0005\u0001\u0006\u0001\u0006"
+        + "\u0003\u0006\u00cb\b\u0006\u0001\u0007\u0001\u0007\u0005\u0007\u00cf\b"
+        + "\u0007\n\u0007\f\u0007\u00d2\t\u0007\u0001\u0007\u0003\u0007\u00d5\b\u0007"
+        + "\u0001\u0007\u0001\u0007\u0001\b\u0001\b\u0001\t\u0001\t\u0003\t\u00dd"
+        + "\b\t\u0001\n\u0001\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0005"
+        + "\u000b\u00e5\b\u000b\n\u000b\f\u000b\u00e8\t\u000b\u0001\f\u0001\f\u0001"
+        + "\f\u0005\f\u00ed\b\f\n\f\f\f\u00f0\t\f\u0001\r\u0001\r\u0001\r\u0001\r"
+        + "\u0001\r\u0005\r\u00f7\b\r\n\r\f\r\u00fa\t\r\u0003\r\u00fc\b\r\u0001\u000e"
+        + "\u0001\u000e\u0001\u000e\u0003\u000e\u0101\b\u000e\u0001\u000f\u0001\u000f"
+        + "\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u0010"
+        + "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010"
+        + "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010"
+        + "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010"
+        + "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010"
+        + "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010"
+        + "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010"
+        + "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0005\u0010"
+        + "\u0134\b\u0010\n\u0010\f\u0010\u0137\t\u0010\u0001\u0011\u0001\u0011\u0001"
+        + "\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001"
+        + "\u0011\u0001\u0011\u0001\u0011\u0003\u0011\u0144\b\u0011\u0001\u0012\u0001"
+        + "\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0003\u0012\u014b\b\u0012\u0001"
+        + "\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001"
+        + "\u0013\u0003\u0013\u0154\b\u0013\u0001\u0014\u0001\u0014\u0001\u0014\u0001"
+        + "\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001"
+        + "\u0014\u0003\u0014\u0160\b\u0014\u0001\u0015\u0001\u0015\u0001\u0016\u0001"
+        + "\u0016\u0001\u0016\u0004\u0016\u0167\b\u0016\u000b\u0016\f\u0016\u0168"
+        + "\u0001\u0016\u0001\u0016\u0001\u0016\u0004\u0016\u016e\b\u0016\u000b\u0016"
+        + "\f\u0016\u016f\u0001\u0016\u0001\u0016\u0001\u0016\u0005\u0016\u0175\b"
+        + "\u0016\n\u0016\f\u0016\u0178\t\u0016\u0001\u0016\u0001\u0016\u0005\u0016"
+        + "\u017c\b\u0016\n\u0016\f\u0016\u017f\t\u0016\u0003\u0016\u0181\b\u0016"
+        + "\u0001\u0017\u0001\u0017\u0005\u0017\u0185\b\u0017\n\u0017\f\u0017\u0188"
+        + "\t\u0017\u0001\u0017\u0003\u0017\u018b\b\u0017\u0001\u0018\u0001\u0018"
+        + "\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018"
+        + "\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018"
+        + "\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0003\u0018"
+        + "\u01a0\b\u0018\u0001\u0019\u0001\u0019\u0001\u0019\u0003\u0019\u01a5\b"
+        + "\u0019\u0001\u001a\u0001\u001a\u0003\u001a\u01a9\b\u001a\u0001\u001b\u0001"
+        + "\u001b\u0001\u001b\u0001\u001b\u0001\u001c\u0001\u001c\u0001\u001c\u0001"
+        + "\u001d\u0001\u001d\u0001\u001d\u0001\u001d\u0001\u001e\u0001\u001e\u0001"
+        + "\u001e\u0001\u001e\u0001\u001e\u0001\u001e\u0004\u001e\u01bc\b\u001e\u000b"
+        + "\u001e\f\u001e\u01bd\u0001\u001e\u0001\u001e\u0005\u001e\u01c2\b\u001e"
+        + "\n\u001e\f\u001e\u01c5\t\u001e\u0003\u001e\u01c7\b\u001e\u0001\u001e\u0001"
+        + "\u001e\u0001\u001e\u0001\u001e\u0001\u001e\u0001\u001e\u0001\u001e\u0001"
+        + "\u001e\u0005\u001e\u01d1\b\u001e\n\u001e\f\u001e\u01d4\t\u001e\u0003\u001e"
+        + "\u01d6\b\u001e\u0001\u001e\u0001\u001e\u0005\u001e\u01da\b\u001e\n\u001e"
+        + "\f\u001e\u01dd\t\u001e\u0003\u001e\u01df\b\u001e\u0001\u001f\u0001\u001f"
+        + "\u0001\u001f\u0001\u001f\u0005\u001f\u01e5\b\u001f\n\u001f\f\u001f\u01e8"
+        + "\t\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0003\u001f\u01ee"
+        + "\b\u001f\u0001 \u0001 \u0001 \u0001 \u0005 \u01f4\b \n \f \u01f7\t \u0001"
+        + " \u0001 \u0001 \u0001 \u0001 \u0003 \u01fe\b \u0001!\u0001!\u0001!\u0001"
+        + "!\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u0208\b\"\n\"\f\"\u020b\t\""
+        + "\u0003\"\u020d\b\"\u0001\"\u0001\"\u0001#\u0001#\u0001#\u0003#\u0214\b"
+        + "#\u0001$\u0001$\u0001$\u0001$\u0001$\u0005$\u021b\b$\n$\f$\u021e\t$\u0003"
+        + "$\u0220\b$\u0001$\u0003$\u0223\b$\u0001$\u0001$\u0001$\u0003$\u0228\b"
+        + "$\u0001%\u0003%\u022b\b%\u0001%\u0001%\u0001&\u0001&\u0001&\u0001&\u0001"
+        + "&\u0001&\u0001&\u0001&\u0001&\u0001&\u0001&\u0003&\u023a\b&\u0001&\u0000"
+        + "\u0001 \'\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010\u0012\u0014\u0016"
+        + "\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJL\u0000\u000f\u0001\u0001"
+        + "\r\r\u0001\u0000\u001f!\u0001\u0000\"#\u0001\u00009:\u0001\u0000$&\u0001"
+        + "\u0000\'*\u0001\u0000+.\u0001\u0000=H\u0001\u0000;<\u0001\u0000\u001d"
+        + "\u001e\u0001\u0000RS\u0001\u0000IL\u0002\u0000\t\tTT\u0001\u0000\n\u000b"
+        + "\u0001\u0000UV\u0277\u0000Q\u0001\u0000\u0000\u0000\u0002\\\u0001\u0000"
+        + "\u0000\u0000\u0004a\u0001\u0000\u0000\u0000\u0006u\u0001\u0000\u0000\u0000"
+        + "\b\u00b3\u0001\u0000\u0000\u0000\n\u00c6\u0001\u0000\u0000\u0000\f\u00ca"
+        + "\u0001\u0000\u0000\u0000\u000e\u00cc\u0001\u0000\u0000\u0000\u0010\u00d8"
+        + "\u0001\u0000\u0000\u0000\u0012\u00dc\u0001\u0000\u0000\u0000\u0014\u00de"
+        + "\u0001\u0000\u0000\u0000\u0016\u00e0\u0001\u0000\u0000\u0000\u0018\u00e9"
+        + "\u0001\u0000\u0000\u0000\u001a\u00fb\u0001\u0000\u0000\u0000\u001c\u00fd"
+        + "\u0001\u0000\u0000\u0000\u001e\u0102\u0001\u0000\u0000\u0000 \u0109\u0001"
+        + "\u0000\u0000\u0000\"\u0143\u0001\u0000\u0000\u0000$\u014a\u0001\u0000"
+        + "\u0000\u0000&\u0153\u0001\u0000\u0000\u0000(\u015f\u0001\u0000\u0000\u0000"
+        + "*\u0161\u0001\u0000\u0000\u0000,\u0180\u0001\u0000\u0000\u0000.\u018a"
+        + "\u0001\u0000\u0000\u00000\u019f\u0001\u0000\u0000\u00002\u01a4\u0001\u0000"
+        + "\u0000\u00004\u01a8\u0001\u0000\u0000\u00006\u01aa\u0001\u0000\u0000\u0000"
+        + "8\u01ae\u0001\u0000\u0000\u0000:\u01b1\u0001\u0000\u0000\u0000<\u01de"
+        + "\u0001\u0000\u0000\u0000>\u01ed\u0001\u0000\u0000\u0000@\u01fd\u0001\u0000"
+        + "\u0000\u0000B\u01ff\u0001\u0000\u0000\u0000D\u0203\u0001\u0000\u0000\u0000"
+        + "F\u0213\u0001\u0000\u0000\u0000H\u0222\u0001\u0000\u0000\u0000J\u022a"
+        + "\u0001\u0000\u0000\u0000L\u0239\u0001\u0000\u0000\u0000NP\u0003\u0002"
+        + "\u0001\u0000ON\u0001\u0000\u0000\u0000PS\u0001\u0000\u0000\u0000QO\u0001"
+        + "\u0000\u0000\u0000QR\u0001\u0000\u0000\u0000RW\u0001\u0000\u0000\u0000"
+        + "SQ\u0001\u0000\u0000\u0000TV\u0003\u0006\u0003\u0000UT\u0001\u0000\u0000"
+        + "\u0000VY\u0001\u0000\u0000\u0000WU\u0001\u0000\u0000\u0000WX\u0001\u0000"
+        + "\u0000\u0000XZ\u0001\u0000\u0000\u0000YW\u0001\u0000\u0000\u0000Z[\u0005"
+        + "\u0000\u0000\u0001[\u0001\u0001\u0000\u0000\u0000\\]\u0003\u0018\f\u0000"
+        + "]^\u0005T\u0000\u0000^_\u0003\u0004\u0002\u0000_`\u0003\u000e\u0007\u0000"
+        + "`\u0003\u0001\u0000\u0000\u0000am\u0005\u0007\u0000\u0000bc\u0003\u0018"
+        + "\f\u0000cj\u0005T\u0000\u0000de\u0005\f\u0000\u0000ef\u0003\u0018\f\u0000"
+        + "fg\u0005T\u0000\u0000gi\u0001\u0000\u0000\u0000hd\u0001\u0000\u0000\u0000"
+        + "il\u0001\u0000\u0000\u0000jh\u0001\u0000\u0000\u0000jk\u0001\u0000\u0000"
+        + "\u0000kn\u0001\u0000\u0000\u0000lj\u0001\u0000\u0000\u0000mb\u0001\u0000"
+        + "\u0000\u0000mn\u0001\u0000\u0000\u0000no\u0001\u0000\u0000\u0000op\u0005"
+        + "\b\u0000\u0000p\u0005\u0001\u0000\u0000\u0000qv\u0003\b\u0004\u0000rs"
+        + "\u0003\n\u0005\u0000st\u0007\u0000\u0000\u0000tv\u0001\u0000\u0000\u0000"
+        + "uq\u0001\u0000\u0000\u0000ur\u0001\u0000\u0000\u0000v\u0007\u0001\u0000"
+        + "\u0000\u0000wx\u0005\u000e\u0000\u0000xy\u0005\u0007\u0000\u0000yz\u0003"
+        + "\"\u0011\u0000z{\u0005\b\u0000\u0000{\u007f\u0003\f\u0006\u0000|}\u0005"
+        + "\u0010\u0000\u0000}\u0080\u0003\f\u0006\u0000~\u0080\u0004\u0004\u0000"
+        + "\u0000\u007f|\u0001\u0000\u0000\u0000\u007f~\u0001\u0000\u0000\u0000\u0080"
+        + "\u00b4\u0001\u0000\u0000\u0000\u0081\u0082\u0005\u0011\u0000\u0000\u0082"
+        + "\u0083\u0005\u0007\u0000\u0000\u0083\u0084\u0003\"\u0011\u0000\u0084\u0087"
+        + "\u0005\b\u0000\u0000\u0085\u0088\u0003\f\u0006\u0000\u0086\u0088\u0003"
+        + "\u0010\b\u0000\u0087\u0085\u0001\u0000\u0000\u0000\u0087\u0086\u0001\u0000"
+        + "\u0000\u0000\u0088\u00b4\u0001\u0000\u0000\u0000\u0089\u008a\u0005\u0013"
+        + "\u0000\u0000\u008a\u008c\u0005\u0007\u0000\u0000\u008b\u008d\u0003\u0012"
+        + "\t\u0000\u008c\u008b\u0001\u0000\u0000\u0000\u008c\u008d\u0001\u0000\u0000"
+        + "\u0000\u008d\u008e\u0001\u0000\u0000\u0000\u008e\u0090\u0005\r\u0000\u0000"
+        + "\u008f\u0091\u0003\"\u0011\u0000\u0090\u008f\u0001\u0000\u0000\u0000\u0090"
+        + "\u0091\u0001\u0000\u0000\u0000\u0091\u0092\u0001\u0000\u0000\u0000\u0092"
+        + "\u0094\u0005\r\u0000\u0000\u0093\u0095\u0003\u0014\n\u0000\u0094\u0093"
+        + "\u0001\u0000\u0000\u0000\u0094\u0095\u0001\u0000\u0000\u0000\u0095\u0096"
+        + "\u0001\u0000\u0000\u0000\u0096\u0099\u0005\b\u0000\u0000\u0097\u009a\u0003"
+        + "\f\u0006\u0000\u0098\u009a\u0003\u0010\b\u0000\u0099\u0097\u0001\u0000"
+        + "\u0000\u0000\u0099\u0098\u0001\u0000\u0000\u0000\u009a\u00b4\u0001\u0000"
+        + "\u0000\u0000\u009b\u009c\u0005\u0013\u0000\u0000\u009c\u009d\u0005\u0007"
+        + "\u0000\u0000\u009d\u009e\u0003\u0018\f\u0000\u009e\u009f\u0005T\u0000"
+        + "\u0000\u009f\u00a0\u00055\u0000\u0000\u00a0\u00a1\u0003\"\u0011\u0000"
+        + "\u00a1\u00a2\u0005\b\u0000\u0000\u00a2\u00a3\u0003\f\u0006\u0000\u00a3"
+        + "\u00b4\u0001\u0000\u0000\u0000\u00a4\u00a5\u0005\u0013\u0000\u0000\u00a5"
+        + "\u00a6\u0005\u0007\u0000\u0000\u00a6\u00a7\u0005T\u0000\u0000\u00a7\u00a8"
+        + "\u0005\u000f\u0000\u0000\u00a8\u00a9\u0003\"\u0011\u0000\u00a9\u00aa\u0005"
+        + "\b\u0000\u0000\u00aa\u00ab\u0003\f\u0006\u0000\u00ab\u00b4\u0001\u0000"
+        + "\u0000\u0000\u00ac\u00ad\u0005\u0018\u0000\u0000\u00ad\u00af\u0003\u000e"
+        + "\u0007\u0000\u00ae\u00b0\u0003\u001e\u000f\u0000\u00af\u00ae\u0001\u0000"
+        + "\u0000\u0000\u00b0\u00b1\u0001\u0000\u0000\u0000\u00b1\u00af\u0001\u0000"
+        + "\u0000\u0000\u00b1\u00b2\u0001\u0000\u0000\u0000\u00b2\u00b4\u0001\u0000"
+        + "\u0000\u0000\u00b3w\u0001\u0000\u0000\u0000\u00b3\u0081\u0001\u0000\u0000"
+        + "\u0000\u00b3\u0089\u0001\u0000\u0000\u0000\u00b3\u009b\u0001\u0000\u0000"
+        + "\u0000\u00b3\u00a4\u0001\u0000\u0000\u0000\u00b3\u00ac\u0001\u0000\u0000"
+        + "\u0000\u00b4\t\u0001\u0000\u0000\u0000\u00b5\u00b6\u0005\u0012\u0000\u0000"
+        + "\u00b6\u00b7\u0003\u000e\u0007\u0000\u00b7\u00b8\u0005\u0011\u0000\u0000"
+        + "\u00b8\u00b9\u0005\u0007\u0000\u0000\u00b9\u00ba\u0003\"\u0011\u0000\u00ba"
+        + "\u00bb\u0005\b\u0000\u0000\u00bb\u00c7\u0001\u0000\u0000\u0000\u00bc\u00c7"
+        + "\u0003\u0016\u000b\u0000\u00bd\u00c7\u0005\u0014\u0000\u0000\u00be\u00c7"
+        + "\u0005\u0015\u0000\u0000\u00bf\u00c1\u0005\u0016\u0000\u0000\u00c0\u00c2"
+        + "\u0003\"\u0011\u0000\u00c1\u00c0\u0001\u0000\u0000\u0000\u00c1\u00c2\u0001"
+        + "\u0000\u0000\u0000\u00c2\u00c7\u0001\u0000\u0000\u0000\u00c3\u00c4\u0005"
+        + "\u001a\u0000\u0000\u00c4\u00c7\u0003\"\u0011\u0000\u00c5\u00c7\u0003\""
+        + "\u0011\u0000\u00c6\u00b5\u0001\u0000\u0000\u0000\u00c6\u00bc\u0001\u0000"
+        + "\u0000\u0000\u00c6\u00bd\u0001\u0000\u0000\u0000\u00c6\u00be\u0001\u0000"
+        + "\u0000\u0000\u00c6\u00bf\u0001\u0000\u0000\u0000\u00c6\u00c3\u0001\u0000"
+        + "\u0000\u0000\u00c6\u00c5\u0001\u0000\u0000\u0000\u00c7\u000b\u0001\u0000"
+        + "\u0000\u0000\u00c8\u00cb\u0003\u000e\u0007\u0000\u00c9\u00cb\u0003\u0006"
+        + "\u0003\u0000\u00ca\u00c8\u0001\u0000\u0000\u0000\u00ca\u00c9\u0001\u0000"
+        + "\u0000\u0000\u00cb\r\u0001\u0000\u0000\u0000\u00cc\u00d0\u0005\u0003\u0000"
+        + "\u0000\u00cd\u00cf\u0003\u0006\u0003\u0000\u00ce\u00cd\u0001\u0000\u0000"
+        + "\u0000\u00cf\u00d2\u0001\u0000\u0000\u0000\u00d0\u00ce\u0001\u0000\u0000"
+        + "\u0000\u00d0\u00d1\u0001\u0000\u0000\u0000\u00d1\u00d4\u0001\u0000\u0000"
+        + "\u0000\u00d2\u00d0\u0001\u0000\u0000\u0000\u00d3\u00d5\u0003\n\u0005\u0000"
+        + "\u00d4\u00d3\u0001\u0000\u0000\u0000\u00d4\u00d5\u0001\u0000\u0000\u0000"
+        + "\u00d5\u00d6\u0001\u0000\u0000\u0000\u00d6\u00d7\u0005\u0004\u0000\u0000"
+        + "\u00d7\u000f\u0001\u0000\u0000\u0000\u00d8\u00d9\u0005\r\u0000\u0000\u00d9"
+        + "\u0011\u0001\u0000\u0000\u0000\u00da\u00dd\u0003\u0016\u000b\u0000\u00db"
+        + "\u00dd\u0003\"\u0011\u0000\u00dc\u00da\u0001\u0000\u0000\u0000\u00dc\u00db"
+        + "\u0001\u0000\u0000\u0000\u00dd\u0013\u0001\u0000\u0000\u0000\u00de\u00df"
+        + "\u0003\"\u0011\u0000\u00df\u0015\u0001\u0000\u0000\u0000\u00e0\u00e1\u0003"
+        + "\u0018\f\u0000\u00e1\u00e6\u0003\u001c\u000e\u0000\u00e2\u00e3\u0005\f"
+        + "\u0000\u0000\u00e3\u00e5\u0003\u001c\u000e\u0000\u00e4\u00e2\u0001\u0000"
+        + "\u0000\u0000\u00e5\u00e8\u0001\u0000\u0000\u0000\u00e6\u00e4\u0001\u0000"
+        + "\u0000\u0000\u00e6\u00e7\u0001\u0000\u0000\u0000\u00e7\u0017\u0001\u0000"
+        + "\u0000\u0000\u00e8\u00e6\u0001\u0000\u0000\u0000\u00e9\u00ee\u0003\u001a"
+        + "\r\u0000\u00ea\u00eb\u0005\u0005\u0000\u0000\u00eb\u00ed\u0005\u0006\u0000"
+        + "\u0000\u00ec\u00ea\u0001\u0000\u0000\u0000\u00ed\u00f0\u0001\u0000\u0000"
+        + "\u0000\u00ee\u00ec\u0001\u0000\u0000\u0000\u00ee\u00ef\u0001\u0000\u0000"
+        + "\u0000\u00ef\u0019\u0001\u0000\u0000\u0000\u00f0\u00ee\u0001\u0000\u0000"
+        + "\u0000\u00f1\u00fc\u0005S\u0000\u0000\u00f2\u00fc\u0005R\u0000\u0000\u00f3"
+        + "\u00f8\u0005T\u0000\u0000\u00f4\u00f5\u0005\n\u0000\u0000\u00f5\u00f7"
+        + "\u0005V\u0000\u0000\u00f6\u00f4\u0001\u0000\u0000\u0000\u00f7\u00fa\u0001"
+        + "\u0000\u0000\u0000\u00f8\u00f6\u0001\u0000\u0000\u0000\u00f8\u00f9\u0001"
+        + "\u0000\u0000\u0000\u00f9\u00fc\u0001\u0000\u0000\u0000\u00fa\u00f8\u0001"
+        + "\u0000\u0000\u0000\u00fb\u00f1\u0001\u0000\u0000\u0000\u00fb\u00f2\u0001"
+        + "\u0000\u0000\u0000\u00fb\u00f3\u0001\u0000\u0000\u0000\u00fc\u001b\u0001"
+        + "\u0000\u0000\u0000\u00fd\u0100\u0005T\u0000\u0000\u00fe\u00ff\u0005=\u0000"
+        + "\u0000\u00ff\u0101\u0003\"\u0011\u0000\u0100\u00fe\u0001\u0000\u0000\u0000"
+        + "\u0100\u0101\u0001\u0000\u0000\u0000\u0101\u001d\u0001\u0000\u0000\u0000"
+        + "\u0102\u0103\u0005\u0019\u0000\u0000\u0103\u0104\u0005\u0007\u0000\u0000"
+        + "\u0104\u0105\u0003\u001a\r\u0000\u0105\u0106\u0005T\u0000\u0000\u0106"
+        + "\u0107\u0005\b\u0000\u0000\u0107\u0108\u0003\u000e\u0007\u0000\u0108\u001f"
+        + "\u0001\u0000\u0000\u0000\u0109\u010a\u0006\u0010\uffff\uffff\u0000\u010a"
+        + "\u010b\u0003$\u0012\u0000\u010b\u0135\u0001\u0000\u0000\u0000\u010c\u010d"
+        + "\n\r\u0000\u0000\u010d\u010e\u0007\u0001\u0000\u0000\u010e\u0134\u0003"
+        + " \u0010\u000e\u010f\u0110\n\f\u0000\u0000\u0110\u0111\u0007\u0002\u0000"
+        + "\u0000\u0111\u0134\u0003 \u0010\r\u0112\u0113\n\u000b\u0000\u0000\u0113"
+        + "\u0114\u0007\u0003\u0000\u0000\u0114\u0134\u0003 \u0010\f\u0115\u0116"
+        + "\n\n\u0000\u0000\u0116\u0117\u0007\u0004\u0000\u0000\u0117\u0134\u0003"
+        + " \u0010\u000b\u0118\u0119\n\t\u0000\u0000\u0119\u011a\u0007\u0005\u0000"
+        + "\u0000\u011a\u0134\u0003 \u0010\n\u011b\u011c\n\u0007\u0000\u0000\u011c"
+        + "\u011d\u0007\u0006\u0000\u0000\u011d\u0134\u0003 \u0010\b\u011e\u011f"
+        + "\n\u0006\u0000\u0000\u011f\u0120\u0005/\u0000\u0000\u0120\u0134\u0003"
+        + " \u0010\u0007\u0121\u0122\n\u0005\u0000\u0000\u0122\u0123\u00050\u0000"
+        + "\u0000\u0123\u0134\u0003 \u0010\u0006\u0124\u0125\n\u0004\u0000\u0000"
+        + "\u0125\u0126\u00051\u0000\u0000\u0126\u0134\u0003 \u0010\u0005\u0127\u0128"
+        + "\n\u0003\u0000\u0000\u0128\u0129\u00052\u0000\u0000\u0129\u0134\u0003"
+        + " \u0010\u0004\u012a\u012b\n\u0002\u0000\u0000\u012b\u012c\u00053\u0000"
+        + "\u0000\u012c\u0134\u0003 \u0010\u0003\u012d\u012e\n\u0001\u0000\u0000"
+        + "\u012e\u012f\u00056\u0000\u0000\u012f\u0134\u0003 \u0010\u0001\u0130\u0131"
+        + "\n\b\u0000\u0000\u0131\u0132\u0005\u001c\u0000\u0000\u0132\u0134\u0003"
+        + "\u0018\f\u0000\u0133\u010c\u0001\u0000\u0000\u0000\u0133\u010f\u0001\u0000"
+        + "\u0000\u0000\u0133\u0112\u0001\u0000\u0000\u0000\u0133\u0115\u0001\u0000"
+        + "\u0000\u0000\u0133\u0118\u0001\u0000\u0000\u0000\u0133\u011b\u0001\u0000"
+        + "\u0000\u0000\u0133\u011e\u0001\u0000\u0000\u0000\u0133\u0121\u0001\u0000"
+        + "\u0000\u0000\u0133\u0124\u0001\u0000\u0000\u0000\u0133\u0127\u0001\u0000"
+        + "\u0000\u0000\u0133\u012a\u0001\u0000\u0000\u0000\u0133\u012d\u0001\u0000"
+        + "\u0000\u0000\u0133\u0130\u0001\u0000\u0000\u0000\u0134\u0137\u0001\u0000"
+        + "\u0000\u0000\u0135\u0133\u0001\u0000\u0000\u0000\u0135\u0136\u0001\u0000"
+        + "\u0000\u0000\u0136!\u0001\u0000\u0000\u0000\u0137\u0135\u0001\u0000\u0000"
+        + "\u0000\u0138\u0144\u0003 \u0010\u0000\u0139\u013a\u0003 \u0010\u0000\u013a"
+        + "\u013b\u00054\u0000\u0000\u013b\u013c\u0003\"\u0011\u0000\u013c\u013d"
+        + "\u00055\u0000\u0000\u013d\u013e\u0003\"\u0011\u0000\u013e\u0144\u0001"
+        + "\u0000\u0000\u0000\u013f\u0140\u0003 \u0010\u0000\u0140\u0141\u0007\u0007"
+        + "\u0000\u0000\u0141\u0142\u0003\"\u0011\u0000\u0142\u0144\u0001\u0000\u0000"
+        + "\u0000\u0143\u0138\u0001\u0000\u0000\u0000\u0143\u0139\u0001\u0000\u0000"
+        + "\u0000\u0143\u013f\u0001\u0000\u0000\u0000\u0144#\u0001\u0000\u0000\u0000"
+        + "\u0145\u0146\u0007\b\u0000\u0000\u0146\u014b\u0003.\u0017\u0000\u0147"
+        + "\u0148\u0007\u0002\u0000\u0000\u0148\u014b\u0003$\u0012\u0000\u0149\u014b"
+        + "\u0003&\u0013\u0000\u014a\u0145\u0001\u0000\u0000\u0000\u014a\u0147\u0001"
+        + "\u0000\u0000\u0000\u014a\u0149\u0001\u0000\u0000\u0000\u014b%\u0001\u0000"
+        + "\u0000\u0000\u014c\u0154\u0003.\u0017\u0000\u014d\u014e\u0003.\u0017\u0000"
+        + "\u014e\u014f\u0007\b\u0000\u0000\u014f\u0154\u0001\u0000\u0000\u0000\u0150"
+        + "\u0151\u0007\t\u0000\u0000\u0151\u0154\u0003$\u0012\u0000\u0152\u0154"
+        + "\u0003(\u0014\u0000\u0153\u014c\u0001\u0000\u0000\u0000\u0153\u014d\u0001"
+        + "\u0000\u0000\u0000\u0153\u0150\u0001\u0000\u0000\u0000\u0153\u0152\u0001"
+        + "\u0000\u0000\u0000\u0154\'\u0001\u0000\u0000\u0000\u0155\u0156\u0005\u0007"
+        + "\u0000\u0000\u0156\u0157\u0003*\u0015\u0000\u0157\u0158\u0005\b\u0000"
+        + "\u0000\u0158\u0159\u0003$\u0012\u0000\u0159\u0160\u0001\u0000\u0000\u0000"
+        + "\u015a\u015b\u0005\u0007\u0000\u0000\u015b\u015c\u0003,\u0016\u0000\u015c"
+        + "\u015d\u0005\b\u0000\u0000\u015d\u015e\u0003&\u0013\u0000\u015e\u0160"
+        + "\u0001\u0000\u0000\u0000\u015f\u0155\u0001\u0000\u0000\u0000\u015f\u015a"
+        + "\u0001\u0000\u0000\u0000\u0160)\u0001\u0000\u0000\u0000\u0161\u0162\u0007"
+        + "\n\u0000\u0000\u0162+\u0001\u0000\u0000\u0000\u0163\u0166\u0005S\u0000"
+        + "\u0000\u0164\u0165\u0005\u0005\u0000\u0000\u0165\u0167\u0005\u0006\u0000"
+        + "\u0000\u0166\u0164\u0001\u0000\u0000\u0000\u0167\u0168\u0001\u0000\u0000"
+        + "\u0000\u0168\u0166\u0001\u0000\u0000\u0000\u0168\u0169\u0001\u0000\u0000"
+        + "\u0000\u0169\u0181\u0001\u0000\u0000\u0000\u016a\u016d\u0005R\u0000\u0000"
+        + "\u016b\u016c\u0005\u0005\u0000\u0000\u016c\u016e\u0005\u0006\u0000\u0000"
+        + "\u016d\u016b\u0001\u0000\u0000\u0000\u016e\u016f\u0001\u0000\u0000\u0000"
+        + "\u016f\u016d\u0001\u0000\u0000\u0000\u016f\u0170\u0001\u0000\u0000\u0000"
+        + "\u0170\u0181\u0001\u0000\u0000\u0000\u0171\u0176\u0005T\u0000\u0000\u0172"
+        + "\u0173\u0005\n\u0000\u0000\u0173\u0175\u0005V\u0000\u0000\u0174\u0172"
+        + "\u0001\u0000\u0000\u0000\u0175\u0178\u0001\u0000\u0000\u0000\u0176\u0174"
+        + "\u0001\u0000\u0000\u0000\u0176\u0177\u0001\u0000\u0000\u0000\u0177\u017d"
+        + "\u0001\u0000\u0000\u0000\u0178\u0176\u0001\u0000\u0000\u0000\u0179\u017a"
+        + "\u0005\u0005\u0000\u0000\u017a\u017c\u0005\u0006\u0000\u0000\u017b\u0179"
+        + "\u0001\u0000\u0000\u0000\u017c\u017f\u0001\u0000\u0000\u0000\u017d\u017b"
+        + "\u0001\u0000\u0000\u0000\u017d\u017e\u0001\u0000\u0000\u0000\u017e\u0181"
+        + "\u0001\u0000\u0000\u0000\u017f\u017d\u0001\u0000\u0000\u0000\u0180\u0163"
+        + "\u0001\u0000\u0000\u0000\u0180\u016a\u0001\u0000\u0000\u0000\u0180\u0171"
+        + "\u0001\u0000\u0000\u0000\u0181-\u0001\u0000\u0000\u0000\u0182\u0186\u0003"
+        + "0\u0018\u0000\u0183\u0185\u00032\u0019\u0000\u0184\u0183\u0001\u0000\u0000"
+        + "\u0000\u0185\u0188\u0001\u0000\u0000\u0000\u0186\u0184\u0001\u0000\u0000"
+        + "\u0000\u0186\u0187\u0001\u0000\u0000\u0000\u0187\u018b\u0001\u0000\u0000"
+        + "\u0000\u0188\u0186\u0001\u0000\u0000\u0000\u0189\u018b\u0003<\u001e\u0000"
+        + "\u018a\u0182\u0001\u0000\u0000\u0000\u018a\u0189\u0001\u0000\u0000\u0000"
+        + "\u018b/\u0001\u0000\u0000\u0000\u018c\u018d\u0005\u0007\u0000\u0000\u018d"
+        + "\u018e\u0003\"\u0011\u0000\u018e\u018f\u0005\b\u0000\u0000\u018f\u01a0"
+        + "\u0001\u0000\u0000\u0000\u0190\u01a0\u0007\u000b\u0000\u0000\u0191\u01a0"
+        + "\u0005O\u0000\u0000\u0192\u01a0\u0005P\u0000\u0000\u0193\u01a0\u0005Q"
+        + "\u0000\u0000\u0194\u01a0\u0005M\u0000\u0000\u0195\u01a0\u0005N\u0000\u0000"
+        + "\u0196\u01a0\u0003>\u001f\u0000\u0197\u01a0\u0003@ \u0000\u0198\u01a0"
+        + "\u0005T\u0000\u0000\u0199\u019a\u0007\f\u0000\u0000\u019a\u01a0\u0003"
+        + "D\"\u0000\u019b\u019c\u0005\u0017\u0000\u0000\u019c\u019d\u0003\u001a"
+        + "\r\u0000\u019d\u019e\u0003D\"\u0000\u019e\u01a0\u0001\u0000\u0000\u0000"
+        + "\u019f\u018c\u0001\u0000\u0000\u0000\u019f\u0190\u0001\u0000\u0000\u0000"
+        + "\u019f\u0191\u0001\u0000\u0000\u0000\u019f\u0192\u0001\u0000\u0000\u0000"
+        + "\u019f\u0193\u0001\u0000\u0000\u0000\u019f\u0194\u0001\u0000\u0000\u0000"
+        + "\u019f\u0195\u0001\u0000\u0000\u0000\u019f\u0196\u0001\u0000\u0000\u0000"
+        + "\u019f\u0197\u0001\u0000\u0000\u0000\u019f\u0198\u0001\u0000\u0000\u0000"
+        + "\u019f\u0199\u0001\u0000\u0000\u0000\u019f\u019b\u0001\u0000\u0000\u0000"
+        + "\u01a01\u0001\u0000\u0000\u0000\u01a1\u01a5\u00036\u001b\u0000\u01a2\u01a5"
+        + "\u00038\u001c\u0000\u01a3\u01a5\u0003:\u001d\u0000\u01a4\u01a1\u0001\u0000"
+        + "\u0000\u0000\u01a4\u01a2\u0001\u0000\u0000\u0000\u01a4\u01a3\u0001\u0000"
+        + "\u0000\u0000\u01a53\u0001\u0000\u0000\u0000\u01a6\u01a9\u00036\u001b\u0000"
+        + "\u01a7\u01a9\u00038\u001c\u0000\u01a8\u01a6\u0001\u0000\u0000\u0000\u01a8"
+        + "\u01a7\u0001\u0000\u0000\u0000\u01a95\u0001\u0000\u0000\u0000\u01aa\u01ab"
+        + "\u0007\r\u0000\u0000\u01ab\u01ac\u0005V\u0000\u0000\u01ac\u01ad\u0003"
+        + "D\"\u0000\u01ad7\u0001\u0000\u0000\u0000\u01ae\u01af\u0007\r\u0000\u0000"
+        + "\u01af\u01b0\u0007\u000e\u0000\u0000\u01b09\u0001\u0000\u0000\u0000\u01b1"
+        + "\u01b2\u0005\u0005\u0000\u0000\u01b2\u01b3\u0003\"\u0011\u0000\u01b3\u01b4"
+        + "\u0005\u0006\u0000\u0000\u01b4;\u0001\u0000\u0000\u0000\u01b5\u01b6\u0005"
+        + "\u0017\u0000\u0000\u01b6\u01bb\u0003\u001a\r\u0000\u01b7\u01b8\u0005\u0005"
+        + "\u0000\u0000\u01b8\u01b9\u0003\"\u0011\u0000\u01b9\u01ba\u0005\u0006\u0000"
+        + "\u0000\u01ba\u01bc\u0001\u0000\u0000\u0000\u01bb\u01b7\u0001\u0000\u0000"
+        + "\u0000\u01bc\u01bd\u0001\u0000\u0000\u0000\u01bd\u01bb\u0001\u0000\u0000"
+        + "\u0000\u01bd\u01be\u0001\u0000\u0000\u0000\u01be\u01c6\u0001\u0000\u0000"
+        + "\u0000\u01bf\u01c3\u00034\u001a\u0000\u01c0\u01c2\u00032\u0019\u0000\u01c1"
+        + "\u01c0\u0001\u0000\u0000\u0000\u01c2\u01c5\u0001\u0000\u0000\u0000\u01c3"
+        + "\u01c1\u0001\u0000\u0000\u0000\u01c3\u01c4\u0001\u0000\u0000\u0000\u01c4"
+        + "\u01c7\u0001\u0000\u0000\u0000\u01c5\u01c3\u0001\u0000\u0000\u0000\u01c6"
+        + "\u01bf\u0001\u0000\u0000\u0000\u01c6\u01c7\u0001\u0000\u0000\u0000\u01c7"
+        + "\u01df\u0001\u0000\u0000\u0000\u01c8\u01c9\u0005\u0017\u0000\u0000\u01c9"
+        + "\u01ca\u0003\u001a\r\u0000\u01ca\u01cb\u0005\u0005\u0000\u0000\u01cb\u01cc"
+        + "\u0005\u0006\u0000\u0000\u01cc\u01d5\u0005\u0003\u0000\u0000\u01cd\u01d2"
+        + "\u0003\"\u0011\u0000\u01ce\u01cf\u0005\f\u0000\u0000\u01cf\u01d1\u0003"
+        + "\"\u0011\u0000\u01d0\u01ce\u0001\u0000\u0000\u0000\u01d1\u01d4\u0001\u0000"
+        + "\u0000\u0000\u01d2\u01d0\u0001\u0000\u0000\u0000\u01d2\u01d3\u0001\u0000"
+        + "\u0000\u0000\u01d3\u01d6\u0001\u0000\u0000\u0000\u01d4\u01d2\u0001\u0000"
+        + "\u0000\u0000\u01d5\u01cd\u0001\u0000\u0000\u0000\u01d5\u01d6\u0001\u0000"
+        + "\u0000\u0000\u01d6\u01d7\u0001\u0000\u0000\u0000\u01d7\u01db\u0005\u0004"
+        + "\u0000\u0000\u01d8\u01da\u00032\u0019\u0000\u01d9\u01d8\u0001\u0000\u0000"
+        + "\u0000\u01da\u01dd\u0001\u0000\u0000\u0000\u01db\u01d9\u0001\u0000\u0000"
+        + "\u0000\u01db\u01dc\u0001\u0000\u0000\u0000\u01dc\u01df\u0001\u0000\u0000"
+        + "\u0000\u01dd\u01db\u0001\u0000\u0000\u0000\u01de\u01b5\u0001\u0000\u0000"
+        + "\u0000\u01de\u01c8\u0001\u0000\u0000\u0000\u01df=\u0001\u0000\u0000\u0000"
+        + "\u01e0\u01e1\u0005\u0005\u0000\u0000\u01e1\u01e6\u0003\"\u0011\u0000\u01e2"
+        + "\u01e3\u0005\f\u0000\u0000\u01e3\u01e5\u0003\"\u0011\u0000\u01e4\u01e2"
+        + "\u0001\u0000\u0000\u0000\u01e5\u01e8\u0001\u0000\u0000\u0000\u01e6\u01e4"
+        + "\u0001\u0000\u0000\u0000\u01e6\u01e7\u0001\u0000\u0000\u0000\u01e7\u01e9"
+        + "\u0001\u0000\u0000\u0000\u01e8\u01e6\u0001\u0000\u0000\u0000\u01e9\u01ea"
+        + "\u0005\u0006\u0000\u0000\u01ea\u01ee\u0001\u0000\u0000\u0000\u01eb\u01ec"
+        + "\u0005\u0005\u0000\u0000\u01ec\u01ee\u0005\u0006\u0000\u0000\u01ed\u01e0"
+        + "\u0001\u0000\u0000\u0000\u01ed\u01eb\u0001\u0000\u0000\u0000\u01ee?\u0001"
+        + "\u0000\u0000\u0000\u01ef\u01f0\u0005\u0005\u0000\u0000\u01f0\u01f5\u0003"
+        + "B!\u0000\u01f1\u01f2\u0005\f\u0000\u0000\u01f2\u01f4\u0003B!\u0000\u01f3"
+        + "\u01f1\u0001\u0000\u0000\u0000\u01f4\u01f7\u0001\u0000\u0000\u0000\u01f5"
+        + "\u01f3\u0001\u0000\u0000\u0000\u01f5\u01f6\u0001\u0000\u0000\u0000\u01f6"
+        + "\u01f8\u0001\u0000\u0000\u0000\u01f7\u01f5\u0001\u0000\u0000\u0000\u01f8"
+        + "\u01f9\u0005\u0006\u0000\u0000\u01f9\u01fe\u0001\u0000\u0000\u0000\u01fa"
+        + "\u01fb\u0005\u0005\u0000\u0000\u01fb\u01fc\u00055\u0000\u0000\u01fc\u01fe"
+        + "\u0005\u0006\u0000\u0000\u01fd\u01ef\u0001\u0000\u0000\u0000\u01fd\u01fa"
+        + "\u0001\u0000\u0000\u0000\u01feA\u0001\u0000\u0000\u0000\u01ff\u0200\u0003"
+        + "\"\u0011\u0000\u0200\u0201\u00055\u0000\u0000\u0201\u0202\u0003\"\u0011"
+        + "\u0000\u0202C\u0001\u0000\u0000\u0000\u0203\u020c\u0005\u0007\u0000\u0000"
+        + "\u0204\u0209\u0003F#\u0000\u0205\u0206\u0005\f\u0000\u0000\u0206\u0208"
+        + "\u0003F#\u0000\u0207\u0205\u0001\u0000\u0000\u0000\u0208\u020b\u0001\u0000"
+        + "\u0000\u0000\u0209\u0207\u0001\u0000\u0000\u0000\u0209\u020a\u0001\u0000"
+        + "\u0000\u0000\u020a\u020d\u0001\u0000\u0000\u0000\u020b\u0209\u0001\u0000"
+        + "\u0000\u0000\u020c\u0204\u0001\u0000\u0000\u0000\u020c\u020d\u0001\u0000"
+        + "\u0000\u0000\u020d\u020e\u0001\u0000\u0000\u0000\u020e\u020f\u0005\b\u0000"
+        + "\u0000\u020fE\u0001\u0000\u0000\u0000\u0210\u0214\u0003\"\u0011\u0000"
+        + "\u0211\u0214\u0003H$\u0000\u0212\u0214\u0003L&\u0000\u0213\u0210\u0001"
+        + "\u0000\u0000\u0000\u0213\u0211\u0001\u0000\u0000\u0000\u0213\u0212\u0001"
+        + "\u0000\u0000\u0000\u0214G\u0001\u0000\u0000\u0000\u0215\u0223\u0003J%"
+        + "\u0000\u0216\u021f\u0005\u0007\u0000\u0000\u0217\u021c\u0003J%\u0000\u0218"
+        + "\u0219\u0005\f\u0000\u0000\u0219\u021b\u0003J%\u0000\u021a\u0218\u0001"
+        + "\u0000\u0000\u0000\u021b\u021e\u0001\u0000\u0000\u0000\u021c\u021a\u0001"
+        + "\u0000\u0000\u0000\u021c\u021d\u0001\u0000\u0000\u0000\u021d\u0220\u0001"
+        + "\u0000\u0000\u0000\u021e\u021c\u0001\u0000\u0000\u0000\u021f\u0217\u0001"
+        + "\u0000\u0000\u0000\u021f\u0220\u0001\u0000\u0000\u0000\u0220\u0221\u0001"
+        + "\u0000\u0000\u0000\u0221\u0223\u0005\b\u0000\u0000\u0222\u0215\u0001\u0000"
+        + "\u0000\u0000\u0222\u0216\u0001\u0000\u0000\u0000\u0223\u0224\u0001\u0000"
+        + "\u0000\u0000\u0224\u0227\u00058\u0000\u0000\u0225\u0228\u0003\u000e\u0007"
+        + "\u0000\u0226\u0228\u0003\"\u0011\u0000\u0227\u0225\u0001\u0000\u0000\u0000"
+        + "\u0227\u0226\u0001\u0000\u0000\u0000\u0228I\u0001\u0000\u0000\u0000\u0229"
+        + "\u022b\u0003\u0018\f\u0000\u022a\u0229\u0001\u0000\u0000\u0000\u022a\u022b"
+        + "\u0001\u0000\u0000\u0000\u022b\u022c\u0001\u0000\u0000\u0000\u022c\u022d"
+        + "\u0005T\u0000\u0000\u022dK\u0001\u0000\u0000\u0000\u022e\u022f\u0003\u0018"
+        + "\f\u0000\u022f\u0230\u00057\u0000\u0000\u0230\u0231\u0005T\u0000\u0000"
+        + "\u0231\u023a\u0001\u0000\u0000\u0000\u0232\u0233\u0003\u0018\f\u0000\u0233"
+        + "\u0234\u00057\u0000\u0000\u0234\u0235\u0005\u0017\u0000\u0000\u0235\u023a"
+        + "\u0001\u0000\u0000\u0000\u0236\u0237\u0005\u001b\u0000\u0000\u0237\u0238"
+        + "\u00057\u0000\u0000\u0238\u023a\u0005T\u0000\u0000\u0239\u022e\u0001\u0000"
+        + "\u0000\u0000\u0239\u0232\u0001\u0000\u0000\u0000\u0239\u0236\u0001\u0000"
+        + "\u0000\u0000\u023aM\u0001\u0000\u0000\u0000<QWjmu\u007f\u0087\u008c\u0090"
+        + "\u0094\u0099\u00b1\u00b3\u00c1\u00c6\u00ca\u00d0\u00d4\u00dc\u00e6\u00ee"
+        + "\u00f8\u00fb\u0100\u0133\u0135\u0143\u014a\u0153\u015f\u0168\u016f\u0176"
+        + "\u017d\u0180\u0186\u018a\u019f\u01a4\u01a8\u01bd\u01c3\u01c6\u01d2\u01d5"
+        + "\u01db\u01de\u01e6\u01ed\u01f5\u01fd\u0209\u020c\u0213\u021c\u021f\u0222"
+        + "\u0227\u022a\u0239";
     public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray());
     static {
         _decisionToDFA = new DFA[_ATN.getNumberOfDecisions()];
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
index a7203c710157..3c9a3698a78b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
@@ -11,6 +11,7 @@
  * @param <T> The return type of the visit operation. Use {@link Void} for
  * operations with no return type.
  */
+@SuppressWarnings("CheckReturnValue")
 class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implements PainlessParserVisitor<T> {
     /**
      * {@inheritDoc}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SuggestLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SuggestLexer.java
index 0bad35d925f2..e9d89d0a9667 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SuggestLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SuggestLexer.java
@@ -1,17 +1,22 @@
 // ANTLR GENERATED CODE: DO NOT EDIT
 package org.elasticsearch.painless.antlr;
 
-import org.antlr.v4.runtime.*;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.Lexer;
-import org.antlr.v4.runtime.atn.*;
+import org.antlr.v4.runtime.RuleContext;
+import org.antlr.v4.runtime.RuntimeMetaData;
+import org.antlr.v4.runtime.Vocabulary;
+import org.antlr.v4.runtime.VocabularyImpl;
+import org.antlr.v4.runtime.atn.ATN;
+import org.antlr.v4.runtime.atn.ATNDeserializer;
+import org.antlr.v4.runtime.atn.LexerATNSimulator;
+import org.antlr.v4.runtime.atn.PredictionContextCache;
 import org.antlr.v4.runtime.dfa.DFA;
-import org.antlr.v4.runtime.misc.*;
 
-@SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast" })
+@SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue" })
 public abstract class SuggestLexer extends Lexer {
     static {
-        RuntimeMetaData.checkVersion("4.5.3", RuntimeMetaData.VERSION);
+        RuntimeMetaData.checkVersion("4.11.1", RuntimeMetaData.VERSION);
     }
 
     protected static final DFA[] _decisionToDFA;
@@ -25,266 +30,281 @@ public abstract class SuggestLexer extends Lexer {
             68, ALSH = 69, ARSH = 70, AUSH = 71, OCTAL = 72, HEX = 73, INTEGER = 74, DECIMAL = 75, STRING = 76, REGEX = 77, TRUE = 78,
         FALSE = 79, NULL = 80, ATYPE = 81, TYPE = 82, ID = 83, UNKNOWN = 84, DOTINTEGER = 85, DOTID = 86;
     public static final int AFTER_DOT = 1;
+    public static String[] channelNames = { "DEFAULT_TOKEN_CHANNEL", "HIDDEN" };
+
     public static String[] modeNames = { "DEFAULT_MODE", "AFTER_DOT" };
 
-    public static final String[] ruleNames = {
-        "WS",
-        "COMMENT",
-        "LBRACK",
-        "RBRACK",
-        "LBRACE",
-        "RBRACE",
-        "LP",
-        "RP",
-        "DOT",
-        "NSDOT",
-        "COMMA",
-        "SEMICOLON",
-        "IF",
-        "IN",
-        "ELSE",
-        "WHILE",
-        "DO",
-        "FOR",
-        "CONTINUE",
-        "BREAK",
-        "RETURN",
-        "NEW",
-        "TRY",
-        "CATCH",
-        "THROW",
-        "THIS",
-        "INSTANCEOF",
-        "BOOLNOT",
-        "BWNOT",
-        "MUL",
-        "DIV",
-        "REM",
-        "ADD",
-        "SUB",
-        "LSH",
-        "RSH",
-        "USH",
-        "LT",
-        "LTE",
-        "GT",
-        "GTE",
-        "EQ",
-        "EQR",
-        "NE",
-        "NER",
-        "BWAND",
-        "XOR",
-        "BWOR",
-        "BOOLAND",
-        "BOOLOR",
-        "COND",
-        "COLON",
-        "ELVIS",
-        "REF",
-        "ARROW",
-        "FIND",
-        "MATCH",
-        "INCR",
-        "DECR",
-        "ASSIGN",
-        "AADD",
-        "ASUB",
-        "AMUL",
-        "ADIV",
-        "AREM",
-        "AAND",
-        "AXOR",
-        "AOR",
-        "ALSH",
-        "ARSH",
-        "AUSH",
-        "OCTAL",
-        "HEX",
-        "INTEGER",
-        "DECIMAL",
-        "STRING",
-        "REGEX",
-        "TRUE",
-        "FALSE",
-        "NULL",
-        "ATYPE",
-        "TYPE",
-        "ID",
-        "UNKNOWN",
-        "DOTINTEGER",
-        "DOTID" };
+    private static String[] makeRuleNames() {
+        return new String[] {
+            "WS",
+            "COMMENT",
+            "LBRACK",
+            "RBRACK",
+            "LBRACE",
+            "RBRACE",
+            "LP",
+            "RP",
+            "DOT",
+            "NSDOT",
+            "COMMA",
+            "SEMICOLON",
+            "IF",
+            "IN",
+            "ELSE",
+            "WHILE",
+            "DO",
+            "FOR",
+            "CONTINUE",
+            "BREAK",
+            "RETURN",
+            "NEW",
+            "TRY",
+            "CATCH",
+            "THROW",
+            "THIS",
+            "INSTANCEOF",
+            "BOOLNOT",
+            "BWNOT",
+            "MUL",
+            "DIV",
+            "REM",
+            "ADD",
+            "SUB",
+            "LSH",
+            "RSH",
+            "USH",
+            "LT",
+            "LTE",
+            "GT",
+            "GTE",
+            "EQ",
+            "EQR",
+            "NE",
+            "NER",
+            "BWAND",
+            "XOR",
+            "BWOR",
+            "BOOLAND",
+            "BOOLOR",
+            "COND",
+            "COLON",
+            "ELVIS",
+            "REF",
+            "ARROW",
+            "FIND",
+            "MATCH",
+            "INCR",
+            "DECR",
+            "ASSIGN",
+            "AADD",
+            "ASUB",
+            "AMUL",
+            "ADIV",
+            "AREM",
+            "AAND",
+            "AXOR",
+            "AOR",
+            "ALSH",
+            "ARSH",
+            "AUSH",
+            "OCTAL",
+            "HEX",
+            "INTEGER",
+            "DECIMAL",
+            "STRING",
+            "REGEX",
+            "TRUE",
+            "FALSE",
+            "NULL",
+            "ATYPE",
+            "TYPE",
+            "ID",
+            "UNKNOWN",
+            "DOTINTEGER",
+            "DOTID" };
+    }
+
+    public static final String[] ruleNames = makeRuleNames();
+
+    private static String[] makeLiteralNames() {
+        return new String[] {
+            null,
+            null,
+            null,
+            "'{'",
+            "'}'",
+            "'['",
+            "']'",
+            "'('",
+            "')'",
+            "'.'",
+            "'?.'",
+            "','",
+            "';'",
+            "'if'",
+            "'in'",
+            "'else'",
+            "'while'",
+            "'do'",
+            "'for'",
+            "'continue'",
+            "'break'",
+            "'return'",
+            "'new'",
+            "'try'",
+            "'catch'",
+            "'throw'",
+            "'this'",
+            "'instanceof'",
+            "'!'",
+            "'~'",
+            "'*'",
+            "'/'",
+            "'%'",
+            "'+'",
+            "'-'",
+            "'<<'",
+            "'>>'",
+            "'>>>'",
+            "'<'",
+            "'<='",
+            "'>'",
+            "'>='",
+            "'=='",
+            "'==='",
+            "'!='",
+            "'!=='",
+            "'&'",
+            "'^'",
+            "'|'",
+            "'&&'",
+            "'||'",
+            "'?'",
+            "':'",
+            "'?:'",
+            "'::'",
+            "'->'",
+            "'=~'",
+            "'==~'",
+            "'++'",
+            "'--'",
+            "'='",
+            "'+='",
+            "'-='",
+            "'*='",
+            "'/='",
+            "'%='",
+            "'&='",
+            "'^='",
+            "'|='",
+            "'<<='",
+            "'>>='",
+            "'>>>='",
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            "'true'",
+            "'false'",
+            "'null'" };
+    }
+
+    private static final String[] _LITERAL_NAMES = makeLiteralNames();
 
-    private static final String[] _LITERAL_NAMES = {
-        null,
-        null,
-        null,
-        "'{'",
-        "'}'",
-        "'['",
-        "']'",
-        "'('",
-        "')'",
-        "'.'",
-        "'?.'",
-        "','",
-        "';'",
-        "'if'",
-        "'in'",
-        "'else'",
-        "'while'",
-        "'do'",
-        "'for'",
-        "'continue'",
-        "'break'",
-        "'return'",
-        "'new'",
-        "'try'",
-        "'catch'",
-        "'throw'",
-        "'this'",
-        "'instanceof'",
-        "'!'",
-        "'~'",
-        "'*'",
-        "'/'",
-        "'%'",
-        "'+'",
-        "'-'",
-        "'<<'",
-        "'>>'",
-        "'>>>'",
-        "'<'",
-        "'<='",
-        "'>'",
-        "'>='",
-        "'=='",
-        "'==='",
-        "'!='",
-        "'!=='",
-        "'&'",
-        "'^'",
-        "'|'",
-        "'&&'",
-        "'||'",
-        "'?'",
-        "':'",
-        "'?:'",
-        "'::'",
-        "'->'",
-        "'=~'",
-        "'==~'",
-        "'++'",
-        "'--'",
-        "'='",
-        "'+='",
-        "'-='",
-        "'*='",
-        "'/='",
-        "'%='",
-        "'&='",
-        "'^='",
-        "'|='",
-        "'<<='",
-        "'>>='",
-        "'>>>='",
-        null,
-        null,
-        null,
-        null,
-        null,
-        null,
-        "'true'",
-        "'false'",
-        "'null'" };
-    private static final String[] _SYMBOLIC_NAMES = {
-        null,
-        "WS",
-        "COMMENT",
-        "LBRACK",
-        "RBRACK",
-        "LBRACE",
-        "RBRACE",
-        "LP",
-        "RP",
-        "DOT",
-        "NSDOT",
-        "COMMA",
-        "SEMICOLON",
-        "IF",
-        "IN",
-        "ELSE",
-        "WHILE",
-        "DO",
-        "FOR",
-        "CONTINUE",
-        "BREAK",
-        "RETURN",
-        "NEW",
-        "TRY",
-        "CATCH",
-        "THROW",
-        "THIS",
-        "INSTANCEOF",
-        "BOOLNOT",
-        "BWNOT",
-        "MUL",
-        "DIV",
-        "REM",
-        "ADD",
-        "SUB",
-        "LSH",
-        "RSH",
-        "USH",
-        "LT",
-        "LTE",
-        "GT",
-        "GTE",
-        "EQ",
-        "EQR",
-        "NE",
-        "NER",
-        "BWAND",
-        "XOR",
-        "BWOR",
-        "BOOLAND",
-        "BOOLOR",
-        "COND",
-        "COLON",
-        "ELVIS",
-        "REF",
-        "ARROW",
-        "FIND",
-        "MATCH",
-        "INCR",
-        "DECR",
-        "ASSIGN",
-        "AADD",
-        "ASUB",
-        "AMUL",
-        "ADIV",
-        "AREM",
-        "AAND",
-        "AXOR",
-        "AOR",
-        "ALSH",
-        "ARSH",
-        "AUSH",
-        "OCTAL",
-        "HEX",
-        "INTEGER",
-        "DECIMAL",
-        "STRING",
-        "REGEX",
-        "TRUE",
-        "FALSE",
-        "NULL",
-        "ATYPE",
-        "TYPE",
-        "ID",
-        "UNKNOWN",
-        "DOTINTEGER",
-        "DOTID" };
+    private static String[] makeSymbolicNames() {
+        return new String[] {
+            null,
+            "WS",
+            "COMMENT",
+            "LBRACK",
+            "RBRACK",
+            "LBRACE",
+            "RBRACE",
+            "LP",
+            "RP",
+            "DOT",
+            "NSDOT",
+            "COMMA",
+            "SEMICOLON",
+            "IF",
+            "IN",
+            "ELSE",
+            "WHILE",
+            "DO",
+            "FOR",
+            "CONTINUE",
+            "BREAK",
+            "RETURN",
+            "NEW",
+            "TRY",
+            "CATCH",
+            "THROW",
+            "THIS",
+            "INSTANCEOF",
+            "BOOLNOT",
+            "BWNOT",
+            "MUL",
+            "DIV",
+            "REM",
+            "ADD",
+            "SUB",
+            "LSH",
+            "RSH",
+            "USH",
+            "LT",
+            "LTE",
+            "GT",
+            "GTE",
+            "EQ",
+            "EQR",
+            "NE",
+            "NER",
+            "BWAND",
+            "XOR",
+            "BWOR",
+            "BOOLAND",
+            "BOOLOR",
+            "COND",
+            "COLON",
+            "ELVIS",
+            "REF",
+            "ARROW",
+            "FIND",
+            "MATCH",
+            "INCR",
+            "DECR",
+            "ASSIGN",
+            "AADD",
+            "ASUB",
+            "AMUL",
+            "ADIV",
+            "AREM",
+            "AAND",
+            "AXOR",
+            "AOR",
+            "ALSH",
+            "ARSH",
+            "AUSH",
+            "OCTAL",
+            "HEX",
+            "INTEGER",
+            "DECIMAL",
+            "STRING",
+            "REGEX",
+            "TRUE",
+            "FALSE",
+            "NULL",
+            "ATYPE",
+            "TYPE",
+            "ID",
+            "UNKNOWN",
+            "DOTINTEGER",
+            "DOTID" };
+    }
+
+    private static final String[] _SYMBOLIC_NAMES = makeSymbolicNames();
     public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
     /**
@@ -343,6 +363,11 @@ public String getSerializedATN() {
         return _serializedATN;
     }
 
+    @Override
+    public String[] getChannelNames() {
+        return channelNames;
+    }
+
     @Override
     public String[] getModeNames() {
         return modeNames;
@@ -390,221 +415,394 @@ private boolean TYPE_sempred(RuleContext _localctx, int predIndex) {
         return true;
     }
 
-    public static final String _serializedATN = "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2X\u0267\b\1\b\1\4"
-        + "\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n"
-        + "\4\13\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22"
-        + "\t\22\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31"
-        + "\t\31\4\32\t\32\4\33\t\33\4\34\t\34\4\35\t\35\4\36\t\36\4\37\t\37\4 \t"
-        + " \4!\t!\4\"\t\"\4#\t#\4$\t$\4%\t%\4&\t&\4\'\t\'\4(\t(\4)\t)\4*\t*\4+\t"
-        + "+\4,\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\4\61\t\61\4\62\t\62\4\63\t\63\4\64"
-        + "\t\64\4\65\t\65\4\66\t\66\4\67\t\67\48\t8\49\t9\4:\t:\4;\t;\4<\t<\4=\t"
-        + "=\4>\t>\4?\t?\4@\t@\4A\tA\4B\tB\4C\tC\4D\tD\4E\tE\4F\tF\4G\tG\4H\tH\4"
-        + "I\tI\4J\tJ\4K\tK\4L\tL\4M\tM\4N\tN\4O\tO\4P\tP\4Q\tQ\4R\tR\4S\tS\4T\t"
-        + "T\4U\tU\4V\tV\4W\tW\3\2\6\2\u00b2\n\2\r\2\16\2\u00b3\3\2\3\2\3\3\3\3\3"
-        + "\3\3\3\7\3\u00bc\n\3\f\3\16\3\u00bf\13\3\3\3\3\3\3\3\3\3\3\3\7\3\u00c6"
-        + "\n\3\f\3\16\3\u00c9\13\3\3\3\3\3\5\3\u00cd\n\3\3\3\3\3\3\4\3\4\3\5\3\5"
-        + "\3\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3\n\3\n\3\13\3\13\3\13\3\13\3"
-        + "\13\3\f\3\f\3\r\3\r\3\16\3\16\3\16\3\17\3\17\3\17\3\20\3\20\3\20\3\20"
-        + "\3\20\3\21\3\21\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\23\3\23\3\23\3\23"
-        + "\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\25\3\25\3\25\3\25\3\25"
-        + "\3\25\3\26\3\26\3\26\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\3\30\3\30"
-        + "\3\30\3\30\3\31\3\31\3\31\3\31\3\31\3\31\3\32\3\32\3\32\3\32\3\32\3\32"
-        + "\3\33\3\33\3\33\3\33\3\33\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3\34"
-        + "\3\34\3\34\3\35\3\35\3\36\3\36\3\37\3\37\3 \3 \3 \3!\3!\3\"\3\"\3#\3#"
-        + "\3$\3$\3$\3%\3%\3%\3&\3&\3&\3&\3\'\3\'\3(\3(\3(\3)\3)\3*\3*\3*\3+\3+\3"
-        + "+\3,\3,\3,\3,\3-\3-\3-\3.\3.\3.\3.\3/\3/\3\60\3\60\3\61\3\61\3\62\3\62"
-        + "\3\62\3\63\3\63\3\63\3\64\3\64\3\65\3\65\3\66\3\66\3\66\3\67\3\67\3\67"
-        + "\38\38\38\39\39\39\3:\3:\3:\3:\3;\3;\3;\3<\3<\3<\3=\3=\3>\3>\3>\3?\3?"
-        + "\3?\3@\3@\3@\3A\3A\3A\3B\3B\3B\3C\3C\3C\3D\3D\3D\3E\3E\3E\3F\3F\3F\3F"
-        + "\3G\3G\3G\3G\3H\3H\3H\3H\3H\3I\3I\6I\u01bc\nI\rI\16I\u01bd\3I\5I\u01c1"
-        + "\nI\3J\3J\3J\6J\u01c6\nJ\rJ\16J\u01c7\3J\5J\u01cb\nJ\3K\3K\3K\7K\u01d0"
-        + "\nK\fK\16K\u01d3\13K\5K\u01d5\nK\3K\5K\u01d8\nK\3L\3L\3L\7L\u01dd\nL\f"
-        + "L\16L\u01e0\13L\5L\u01e2\nL\3L\3L\6L\u01e6\nL\rL\16L\u01e7\5L\u01ea\n"
-        + "L\3L\3L\5L\u01ee\nL\3L\6L\u01f1\nL\rL\16L\u01f2\5L\u01f5\nL\3L\5L\u01f8"
-        + "\nL\3M\3M\3M\3M\3M\3M\7M\u0200\nM\fM\16M\u0203\13M\3M\3M\3M\3M\3M\3M\3"
-        + "M\7M\u020c\nM\fM\16M\u020f\13M\3M\5M\u0212\nM\3N\3N\3N\3N\6N\u0218\nN"
-        + "\rN\16N\u0219\3N\3N\7N\u021e\nN\fN\16N\u0221\13N\3N\3N\3O\3O\3O\3O\3O"
-        + "\3P\3P\3P\3P\3P\3P\3Q\3Q\3Q\3Q\3Q\3R\3R\3R\3R\6R\u0239\nR\rR\16R\u023a"
-        + "\3S\3S\3S\3S\7S\u0241\nS\fS\16S\u0244\13S\3S\3S\3T\3T\7T\u024a\nT\fT\16"
-        + "T\u024d\13T\3U\3U\3U\3U\3V\3V\3V\7V\u0256\nV\fV\16V\u0259\13V\5V\u025b"
-        + "\nV\3V\3V\3W\3W\7W\u0261\nW\fW\16W\u0264\13W\3W\3W\7\u00bd\u00c7\u0201"
-        + "\u020d\u0219\2X\4\3\6\4\b\5\n\6\f\7\16\b\20\t\22\n\24\13\26\f\30\r\32"
-        + "\16\34\17\36\20 \21\"\22$\23&\24(\25*\26,\27.\30\60\31\62\32\64\33\66"
-        + "\348\35:\36<\37> @!B\"D#F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60`\61b\62d\63f\64"
-        + "h\65j\66l\67n8p9r:t;v<x=z>|?~@\u0080A\u0082B\u0084C\u0086D\u0088E\u008a"
-        + "F\u008cG\u008eH\u0090I\u0092J\u0094K\u0096L\u0098M\u009aN\u009cO\u009e"
-        + "P\u00a0Q\u00a2R\u00a4S\u00a6T\u00a8U\u00aaV\u00acW\u00aeX\4\2\3\25\5\2"
-        + "\13\f\17\17\"\"\4\2\f\f\17\17\3\2\629\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2"
-        + "\63;\3\2\62;\b\2FFHHNNffhhnn\4\2GGgg\4\2--//\6\2FFHHffhh\4\2$$^^\4\2)"
-        + ")^^\3\2\f\f\4\2\f\f\61\61\t\2WWeekknouuwwzz\5\2C\\aac|\6\2\62;C\\aac|"
-        + "\u0288\2\4\3\2\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2\2\2\2\16"
-        + "\3\2\2\2\2\20\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2\2\30\3\2"
-        + "\2\2\2\32\3\2\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3\2\2\2\2$"
-        + "\3\2\2\2\2&\3\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2\2\2\60\3"
-        + "\2\2\2\2\62\3\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3\2\2\2\2"
-        + "<\3\2\2\2\2>\3\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2\2\2H\3"
-        + "\2\2\2\2J\3\2\2\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2T\3\2\2"
-        + "\2\2V\3\2\2\2\2X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3\2\2\2"
-        + "\2b\3\2\2\2\2d\3\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2\2\2n"
-        + "\3\2\2\2\2p\3\2\2\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2z\3\2"
-        + "\2\2\2|\3\2\2\2\2~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084\3\2"
-        + "\2\2\2\u0086\3\2\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2\2\2"
-        + "\u008e\3\2\2\2\2\u0090\3\2\2\2\2\u0092\3\2\2\2\2\u0094\3\2\2\2\2\u0096"
-        + "\3\2\2\2\2\u0098\3\2\2\2\2\u009a\3\2\2\2\2\u009c\3\2\2\2\2\u009e\3\2\2"
-        + "\2\2\u00a0\3\2\2\2\2\u00a2\3\2\2\2\2\u00a4\3\2\2\2\2\u00a6\3\2\2\2\2\u00a8"
-        + "\3\2\2\2\2\u00aa\3\2\2\2\3\u00ac\3\2\2\2\3\u00ae\3\2\2\2\4\u00b1\3\2\2"
-        + "\2\6\u00cc\3\2\2\2\b\u00d0\3\2\2\2\n\u00d2\3\2\2\2\f\u00d4\3\2\2\2\16"
-        + "\u00d6\3\2\2\2\20\u00d8\3\2\2\2\22\u00da\3\2\2\2\24\u00dc\3\2\2\2\26\u00e0"
-        + "\3\2\2\2\30\u00e5\3\2\2\2\32\u00e7\3\2\2\2\34\u00e9\3\2\2\2\36\u00ec\3"
-        + "\2\2\2 \u00ef\3\2\2\2\"\u00f4\3\2\2\2$\u00fa\3\2\2\2&\u00fd\3\2\2\2(\u0101"
-        + "\3\2\2\2*\u010a\3\2\2\2,\u0110\3\2\2\2.\u0117\3\2\2\2\60\u011b\3\2\2\2"
-        + "\62\u011f\3\2\2\2\64\u0125\3\2\2\2\66\u012b\3\2\2\28\u0130\3\2\2\2:\u013b"
-        + "\3\2\2\2<\u013d\3\2\2\2>\u013f\3\2\2\2@\u0141\3\2\2\2B\u0144\3\2\2\2D"
-        + "\u0146\3\2\2\2F\u0148\3\2\2\2H\u014a\3\2\2\2J\u014d\3\2\2\2L\u0150\3\2"
-        + "\2\2N\u0154\3\2\2\2P\u0156\3\2\2\2R\u0159\3\2\2\2T\u015b\3\2\2\2V\u015e"
-        + "\3\2\2\2X\u0161\3\2\2\2Z\u0165\3\2\2\2\\\u0168\3\2\2\2^\u016c\3\2\2\2"
-        + "`\u016e\3\2\2\2b\u0170\3\2\2\2d\u0172\3\2\2\2f\u0175\3\2\2\2h\u0178\3"
-        + "\2\2\2j\u017a\3\2\2\2l\u017c\3\2\2\2n\u017f\3\2\2\2p\u0182\3\2\2\2r\u0185"
-        + "\3\2\2\2t\u0188\3\2\2\2v\u018c\3\2\2\2x\u018f\3\2\2\2z\u0192\3\2\2\2|"
-        + "\u0194\3\2\2\2~\u0197\3\2\2\2\u0080\u019a\3\2\2\2\u0082\u019d\3\2\2\2"
-        + "\u0084\u01a0\3\2\2\2\u0086\u01a3\3\2\2\2\u0088\u01a6\3\2\2\2\u008a\u01a9"
-        + "\3\2\2\2\u008c\u01ac\3\2\2\2\u008e\u01b0\3\2\2\2\u0090\u01b4\3\2\2\2\u0092"
-        + "\u01b9\3\2\2\2\u0094\u01c2\3\2\2\2\u0096\u01d4\3\2\2\2\u0098\u01e1\3\2"
-        + "\2\2\u009a\u0211\3\2\2\2\u009c\u0213\3\2\2\2\u009e\u0224\3\2\2\2\u00a0"
-        + "\u0229\3\2\2\2\u00a2\u022f\3\2\2\2\u00a4\u0234\3\2\2\2\u00a6\u023c\3\2"
-        + "\2\2\u00a8\u0247\3\2\2\2\u00aa\u024e\3\2\2\2\u00ac\u025a\3\2\2\2\u00ae"
-        + "\u025e\3\2\2\2\u00b0\u00b2\t\2\2\2\u00b1\u00b0\3\2\2\2\u00b2\u00b3\3\2"
-        + "\2\2\u00b3\u00b1\3\2\2\2\u00b3\u00b4\3\2\2\2\u00b4\u00b5\3\2\2\2\u00b5"
-        + "\u00b6\b\2\2\2\u00b6\5\3\2\2\2\u00b7\u00b8\7\61\2\2\u00b8\u00b9\7\61\2"
-        + "\2\u00b9\u00bd\3\2\2\2\u00ba\u00bc\13\2\2\2\u00bb\u00ba\3\2\2\2\u00bc"
-        + "\u00bf\3\2\2\2\u00bd\u00be\3\2\2\2\u00bd\u00bb\3\2\2\2\u00be\u00c0\3\2"
-        + "\2\2\u00bf\u00bd\3\2\2\2\u00c0\u00cd\t\3\2\2\u00c1\u00c2\7\61\2\2\u00c2"
-        + "\u00c3\7,\2\2\u00c3\u00c7\3\2\2\2\u00c4\u00c6\13\2\2\2\u00c5\u00c4\3\2"
-        + "\2\2\u00c6\u00c9\3\2\2\2\u00c7\u00c8\3\2\2\2\u00c7\u00c5\3\2\2\2\u00c8"
-        + "\u00ca\3\2\2\2\u00c9\u00c7\3\2\2\2\u00ca\u00cb\7,\2\2\u00cb\u00cd\7\61"
-        + "\2\2\u00cc\u00b7\3\2\2\2\u00cc\u00c1\3\2\2\2\u00cd\u00ce\3\2\2\2\u00ce"
-        + "\u00cf\b\3\2\2\u00cf\7\3\2\2\2\u00d0\u00d1\7}\2\2\u00d1\t\3\2\2\2\u00d2"
-        + "\u00d3\7\177\2\2\u00d3\13\3\2\2\2\u00d4\u00d5\7]\2\2\u00d5\r\3\2\2\2\u00d6"
-        + "\u00d7\7_\2\2\u00d7\17\3\2\2\2\u00d8\u00d9\7*\2\2\u00d9\21\3\2\2\2\u00da"
-        + "\u00db\7+\2\2\u00db\23\3\2\2\2\u00dc\u00dd\7\60\2\2\u00dd\u00de\3\2\2"
-        + "\2\u00de\u00df\b\n\3\2\u00df\25\3\2\2\2\u00e0\u00e1\7A\2\2\u00e1\u00e2"
-        + "\7\60\2\2\u00e2\u00e3\3\2\2\2\u00e3\u00e4\b\13\3\2\u00e4\27\3\2\2\2\u00e5"
-        + "\u00e6\7.\2\2\u00e6\31\3\2\2\2\u00e7\u00e8\7=\2\2\u00e8\33\3\2\2\2\u00e9"
-        + "\u00ea\7k\2\2\u00ea\u00eb\7h\2\2\u00eb\35\3\2\2\2\u00ec\u00ed\7k\2\2\u00ed"
-        + "\u00ee\7p\2\2\u00ee\37\3\2\2\2\u00ef\u00f0\7g\2\2\u00f0\u00f1\7n\2\2\u00f1"
-        + "\u00f2\7u\2\2\u00f2\u00f3\7g\2\2\u00f3!\3\2\2\2\u00f4\u00f5\7y\2\2\u00f5"
-        + "\u00f6\7j\2\2\u00f6\u00f7\7k\2\2\u00f7\u00f8\7n\2\2\u00f8\u00f9\7g\2\2"
-        + "\u00f9#\3\2\2\2\u00fa\u00fb\7f\2\2\u00fb\u00fc\7q\2\2\u00fc%\3\2\2\2\u00fd"
-        + "\u00fe\7h\2\2\u00fe\u00ff\7q\2\2\u00ff\u0100\7t\2\2\u0100\'\3\2\2\2\u0101"
-        + "\u0102\7e\2\2\u0102\u0103\7q\2\2\u0103\u0104\7p\2\2\u0104\u0105\7v\2\2"
-        + "\u0105\u0106\7k\2\2\u0106\u0107\7p\2\2\u0107\u0108\7w\2\2\u0108\u0109"
-        + "\7g\2\2\u0109)\3\2\2\2\u010a\u010b\7d\2\2\u010b\u010c\7t\2\2\u010c\u010d"
-        + "\7g\2\2\u010d\u010e\7c\2\2\u010e\u010f\7m\2\2\u010f+\3\2\2\2\u0110\u0111"
-        + "\7t\2\2\u0111\u0112\7g\2\2\u0112\u0113\7v\2\2\u0113\u0114\7w\2\2\u0114"
-        + "\u0115\7t\2\2\u0115\u0116\7p\2\2\u0116-\3\2\2\2\u0117\u0118\7p\2\2\u0118"
-        + "\u0119\7g\2\2\u0119\u011a\7y\2\2\u011a/\3\2\2\2\u011b\u011c\7v\2\2\u011c"
-        + "\u011d\7t\2\2\u011d\u011e\7{\2\2\u011e\61\3\2\2\2\u011f\u0120\7e\2\2\u0120"
-        + "\u0121\7c\2\2\u0121\u0122\7v\2\2\u0122\u0123\7e\2\2\u0123\u0124\7j\2\2"
-        + "\u0124\63\3\2\2\2\u0125\u0126\7v\2\2\u0126\u0127\7j\2\2\u0127\u0128\7"
-        + "t\2\2\u0128\u0129\7q\2\2\u0129\u012a\7y\2\2\u012a\65\3\2\2\2\u012b\u012c"
-        + "\7v\2\2\u012c\u012d\7j\2\2\u012d\u012e\7k\2\2\u012e\u012f\7u\2\2\u012f"
-        + "\67\3\2\2\2\u0130\u0131\7k\2\2\u0131\u0132\7p\2\2\u0132\u0133\7u\2\2\u0133"
-        + "\u0134\7v\2\2\u0134\u0135\7c\2\2\u0135\u0136\7p\2\2\u0136\u0137\7e\2\2"
-        + "\u0137\u0138\7g\2\2\u0138\u0139\7q\2\2\u0139\u013a\7h\2\2\u013a9\3\2\2"
-        + "\2\u013b\u013c\7#\2\2\u013c;\3\2\2\2\u013d\u013e\7\u0080\2\2\u013e=\3"
-        + "\2\2\2\u013f\u0140\7,\2\2\u0140?\3\2\2\2\u0141\u0142\7\61\2\2\u0142\u0143"
-        + "\6 \2\2\u0143A\3\2\2\2\u0144\u0145\7\'\2\2\u0145C\3\2\2\2\u0146\u0147"
-        + "\7-\2\2\u0147E\3\2\2\2\u0148\u0149\7/\2\2\u0149G\3\2\2\2\u014a\u014b\7"
-        + ">\2\2\u014b\u014c\7>\2\2\u014cI\3\2\2\2\u014d\u014e\7@\2\2\u014e\u014f"
-        + "\7@\2\2\u014fK\3\2\2\2\u0150\u0151\7@\2\2\u0151\u0152\7@\2\2\u0152\u0153"
-        + "\7@\2\2\u0153M\3\2\2\2\u0154\u0155\7>\2\2\u0155O\3\2\2\2\u0156\u0157\7"
-        + ">\2\2\u0157\u0158\7?\2\2\u0158Q\3\2\2\2\u0159\u015a\7@\2\2\u015aS\3\2"
-        + "\2\2\u015b\u015c\7@\2\2\u015c\u015d\7?\2\2\u015dU\3\2\2\2\u015e\u015f"
-        + "\7?\2\2\u015f\u0160\7?\2\2\u0160W\3\2\2\2\u0161\u0162\7?\2\2\u0162\u0163"
-        + "\7?\2\2\u0163\u0164\7?\2\2\u0164Y\3\2\2\2\u0165\u0166\7#\2\2\u0166\u0167"
-        + "\7?\2\2\u0167[\3\2\2\2\u0168\u0169\7#\2\2\u0169\u016a\7?\2\2\u016a\u016b"
-        + "\7?\2\2\u016b]\3\2\2\2\u016c\u016d\7(\2\2\u016d_\3\2\2\2\u016e\u016f\7"
-        + "`\2\2\u016fa\3\2\2\2\u0170\u0171\7~\2\2\u0171c\3\2\2\2\u0172\u0173\7("
-        + "\2\2\u0173\u0174\7(\2\2\u0174e\3\2\2\2\u0175\u0176\7~\2\2\u0176\u0177"
-        + "\7~\2\2\u0177g\3\2\2\2\u0178\u0179\7A\2\2\u0179i\3\2\2\2\u017a\u017b\7"
-        + "<\2\2\u017bk\3\2\2\2\u017c\u017d\7A\2\2\u017d\u017e\7<\2\2\u017em\3\2"
-        + "\2\2\u017f\u0180\7<\2\2\u0180\u0181\7<\2\2\u0181o\3\2\2\2\u0182\u0183"
-        + "\7/\2\2\u0183\u0184\7@\2\2\u0184q\3\2\2\2\u0185\u0186\7?\2\2\u0186\u0187"
-        + "\7\u0080\2\2\u0187s\3\2\2\2\u0188\u0189\7?\2\2\u0189\u018a\7?\2\2\u018a"
-        + "\u018b\7\u0080\2\2\u018bu\3\2\2\2\u018c\u018d\7-\2\2\u018d\u018e\7-\2"
-        + "\2\u018ew\3\2\2\2\u018f\u0190\7/\2\2\u0190\u0191\7/\2\2\u0191y\3\2\2\2"
-        + "\u0192\u0193\7?\2\2\u0193{\3\2\2\2\u0194\u0195\7-\2\2\u0195\u0196\7?\2"
-        + "\2\u0196}\3\2\2\2\u0197\u0198\7/\2\2\u0198\u0199\7?\2\2\u0199\177\3\2"
-        + "\2\2\u019a\u019b\7,\2\2\u019b\u019c\7?\2\2\u019c\u0081\3\2\2\2\u019d\u019e"
-        + "\7\61\2\2\u019e\u019f\7?\2\2\u019f\u0083\3\2\2\2\u01a0\u01a1\7\'\2\2\u01a1"
-        + "\u01a2\7?\2\2\u01a2\u0085\3\2\2\2\u01a3\u01a4\7(\2\2\u01a4\u01a5\7?\2"
-        + "\2\u01a5\u0087\3\2\2\2\u01a6\u01a7\7`\2\2\u01a7\u01a8\7?\2\2\u01a8\u0089"
-        + "\3\2\2\2\u01a9\u01aa\7~\2\2\u01aa\u01ab\7?\2\2\u01ab\u008b\3\2\2\2\u01ac"
-        + "\u01ad\7>\2\2\u01ad\u01ae\7>\2\2\u01ae\u01af\7?\2\2\u01af\u008d\3\2\2"
-        + "\2\u01b0\u01b1\7@\2\2\u01b1\u01b2\7@\2\2\u01b2\u01b3\7?\2\2\u01b3\u008f"
-        + "\3\2\2\2\u01b4\u01b5\7@\2\2\u01b5\u01b6\7@\2\2\u01b6\u01b7\7@\2\2\u01b7"
-        + "\u01b8\7?\2\2\u01b8\u0091\3\2\2\2\u01b9\u01bb\7\62\2\2\u01ba\u01bc\t\4"
-        + "\2\2\u01bb\u01ba\3\2\2\2\u01bc\u01bd\3\2\2\2\u01bd\u01bb\3\2\2\2\u01bd"
-        + "\u01be\3\2\2\2\u01be\u01c0\3\2\2\2\u01bf\u01c1\t\5\2\2\u01c0\u01bf\3\2"
-        + "\2\2\u01c0\u01c1\3\2\2\2\u01c1\u0093\3\2\2\2\u01c2\u01c3\7\62\2\2\u01c3"
-        + "\u01c5\t\6\2\2\u01c4\u01c6\t\7\2\2\u01c5\u01c4\3\2\2\2\u01c6\u01c7\3\2"
-        + "\2\2\u01c7\u01c5\3\2\2\2\u01c7\u01c8\3\2\2\2\u01c8\u01ca\3\2\2\2\u01c9"
-        + "\u01cb\t\5\2\2\u01ca\u01c9\3\2\2\2\u01ca\u01cb\3\2\2\2\u01cb\u0095\3\2"
-        + "\2\2\u01cc\u01d5\7\62\2\2\u01cd\u01d1\t\b\2\2\u01ce\u01d0\t\t\2\2\u01cf"
-        + "\u01ce\3\2\2\2\u01d0\u01d3\3\2\2\2\u01d1\u01cf\3\2\2\2\u01d1\u01d2\3\2"
-        + "\2\2\u01d2\u01d5\3\2\2\2\u01d3\u01d1\3\2\2\2\u01d4\u01cc\3\2\2\2\u01d4"
-        + "\u01cd\3\2\2\2\u01d5\u01d7\3\2\2\2\u01d6\u01d8\t\n\2\2\u01d7\u01d6\3\2"
-        + "\2\2\u01d7\u01d8\3\2\2\2\u01d8\u0097\3\2\2\2\u01d9\u01e2\7\62\2\2\u01da"
-        + "\u01de\t\b\2\2\u01db\u01dd\t\t\2\2\u01dc\u01db\3\2\2\2\u01dd\u01e0\3\2"
-        + "\2\2\u01de\u01dc\3\2\2\2\u01de\u01df\3\2\2\2\u01df\u01e2\3\2\2\2\u01e0"
-        + "\u01de\3\2\2\2\u01e1\u01d9\3\2\2\2\u01e1\u01da\3\2\2\2\u01e2\u01e9\3\2"
-        + "\2\2\u01e3\u01e5\5\24\n\2\u01e4\u01e6\t\t\2\2\u01e5\u01e4\3\2\2\2\u01e6"
-        + "\u01e7\3\2\2\2\u01e7\u01e5\3\2\2\2\u01e7\u01e8\3\2\2\2\u01e8\u01ea\3\2"
-        + "\2\2\u01e9\u01e3\3\2\2\2\u01e9\u01ea\3\2\2\2\u01ea\u01f4\3\2\2\2\u01eb"
-        + "\u01ed\t\13\2\2\u01ec\u01ee\t\f\2\2\u01ed\u01ec\3\2\2\2\u01ed\u01ee\3"
-        + "\2\2\2\u01ee\u01f0\3\2\2\2\u01ef\u01f1\t\t\2\2\u01f0\u01ef\3\2\2\2\u01f1"
-        + "\u01f2\3\2\2\2\u01f2\u01f0\3\2\2\2\u01f2\u01f3\3\2\2\2\u01f3\u01f5\3\2"
-        + "\2\2\u01f4\u01eb\3\2\2\2\u01f4\u01f5\3\2\2\2\u01f5\u01f7\3\2\2\2\u01f6"
-        + "\u01f8\t\r\2\2\u01f7\u01f6\3\2\2\2\u01f7\u01f8\3\2\2\2\u01f8\u0099\3\2"
-        + "\2\2\u01f9\u0201\7$\2\2\u01fa\u01fb\7^\2\2\u01fb\u0200\7$\2\2\u01fc\u01fd"
-        + "\7^\2\2\u01fd\u0200\7^\2\2\u01fe\u0200\n\16\2\2\u01ff\u01fa\3\2\2\2\u01ff"
-        + "\u01fc\3\2\2\2\u01ff\u01fe\3\2\2\2\u0200\u0203\3\2\2\2\u0201\u0202\3\2"
-        + "\2\2\u0201\u01ff\3\2\2\2\u0202\u0204\3\2\2\2\u0203\u0201\3\2\2\2\u0204"
-        + "\u0212\7$\2\2\u0205\u020d\7)\2\2\u0206\u0207\7^\2\2\u0207\u020c\7)\2\2"
-        + "\u0208\u0209\7^\2\2\u0209\u020c\7^\2\2\u020a\u020c\n\17\2\2\u020b\u0206"
-        + "\3\2\2\2\u020b\u0208\3\2\2\2\u020b\u020a\3\2\2\2\u020c\u020f\3\2\2\2\u020d"
-        + "\u020e\3\2\2\2\u020d\u020b\3\2\2\2\u020e\u0210\3\2\2\2\u020f\u020d\3\2"
-        + "\2\2\u0210\u0212\7)\2\2\u0211\u01f9\3\2\2\2\u0211\u0205\3\2\2\2\u0212"
-        + "\u009b\3\2\2\2\u0213\u0217\7\61\2\2\u0214\u0215\7^\2\2\u0215\u0218\n\20"
-        + "\2\2\u0216\u0218\n\21\2\2\u0217\u0214\3\2\2\2\u0217\u0216\3\2\2\2\u0218"
-        + "\u0219\3\2\2\2\u0219\u021a\3\2\2\2\u0219\u0217\3\2\2\2\u021a\u021b\3\2"
-        + "\2\2\u021b\u021f\7\61\2\2\u021c\u021e\t\22\2\2\u021d\u021c\3\2\2\2\u021e"
-        + "\u0221\3\2\2\2\u021f\u021d\3\2\2\2\u021f\u0220\3\2\2\2\u0220\u0222\3\2"
-        + "\2\2\u0221\u021f\3\2\2\2\u0222\u0223\6N\3\2\u0223\u009d\3\2\2\2\u0224"
-        + "\u0225\7v\2\2\u0225\u0226\7t\2\2\u0226\u0227\7w\2\2\u0227\u0228\7g\2\2"
-        + "\u0228\u009f\3\2\2\2\u0229\u022a\7h\2\2\u022a\u022b\7c\2\2\u022b\u022c"
-        + "\7n\2\2\u022c\u022d\7u\2\2\u022d\u022e\7g\2\2\u022e\u00a1\3\2\2\2\u022f"
-        + "\u0230\7p\2\2\u0230\u0231\7w\2\2\u0231\u0232\7n\2\2\u0232\u0233\7n\2\2"
-        + "\u0233\u00a3\3\2\2\2\u0234\u0238\5\u00a6S\2\u0235\u0236\5\f\6\2\u0236"
-        + "\u0237\5\16\7\2\u0237\u0239\3\2\2\2\u0238\u0235\3\2\2\2\u0239\u023a\3"
-        + "\2\2\2\u023a\u0238\3\2\2\2\u023a\u023b\3\2\2\2\u023b\u00a5\3\2\2\2\u023c"
-        + "\u0242\5\u00a8T\2\u023d\u023e\5\24\n\2\u023e\u023f\5\u00a8T\2\u023f\u0241"
-        + "\3\2\2\2\u0240\u023d\3\2\2\2\u0241\u0244\3\2\2\2\u0242\u0240\3\2\2\2\u0242"
-        + "\u0243\3\2\2\2\u0243\u0245\3\2\2\2\u0244\u0242\3\2\2\2\u0245\u0246\6S"
-        + "\4\2\u0246\u00a7\3\2\2\2\u0247\u024b\t\23\2\2\u0248\u024a\t\24\2\2\u0249"
-        + "\u0248\3\2\2\2\u024a\u024d\3\2\2\2\u024b\u0249\3\2\2\2\u024b\u024c\3\2"
-        + "\2\2\u024c\u00a9\3\2\2\2\u024d\u024b\3\2\2\2\u024e\u024f\13\2\2\2\u024f"
-        + "\u0250\3\2\2\2\u0250\u0251\bU\2\2\u0251\u00ab\3\2\2\2\u0252\u025b\7\62"
-        + "\2\2\u0253\u0257\t\b\2\2\u0254\u0256\t\t\2\2\u0255\u0254\3\2\2\2\u0256"
-        + "\u0259\3\2\2\2\u0257\u0255\3\2\2\2\u0257\u0258\3\2\2\2\u0258\u025b\3\2"
-        + "\2\2\u0259\u0257\3\2\2\2\u025a\u0252\3\2\2\2\u025a\u0253\3\2\2\2\u025b"
-        + "\u025c\3\2\2\2\u025c\u025d\bV\4\2\u025d\u00ad\3\2\2\2\u025e\u0262\t\23"
-        + "\2\2\u025f\u0261\t\24\2\2\u0260\u025f\3\2\2\2\u0261\u0264\3\2\2\2\u0262"
-        + "\u0260\3\2\2\2\u0262\u0263\3\2\2\2\u0263\u0265\3\2\2\2\u0264\u0262\3\2"
-        + "\2\2\u0265\u0266\bW\4\2\u0266\u00af\3\2\2\2%\2\3\u00b3\u00bd\u00c7\u00cc"
-        + "\u01bd\u01c0\u01c7\u01ca\u01d1\u01d4\u01d7\u01de\u01e1\u01e7\u01e9\u01ed"
-        + "\u01f2\u01f4\u01f7\u01ff\u0201\u020b\u020d\u0211\u0217\u0219\u021f\u023a"
-        + "\u0242\u024b\u0257\u025a\u0262\5\b\2\2\4\3\2\4\2\2";
+    public static final String _serializedATN = "\u0004\u0000V\u0265\u0006\uffff\uffff\u0006\uffff\uffff\u0002\u0000\u0007"
+        + "\u0000\u0002\u0001\u0007\u0001\u0002\u0002\u0007\u0002\u0002\u0003\u0007"
+        + "\u0003\u0002\u0004\u0007\u0004\u0002\u0005\u0007\u0005\u0002\u0006\u0007"
+        + "\u0006\u0002\u0007\u0007\u0007\u0002\b\u0007\b\u0002\t\u0007\t\u0002\n"
+        + "\u0007\n\u0002\u000b\u0007\u000b\u0002\f\u0007\f\u0002\r\u0007\r\u0002"
+        + "\u000e\u0007\u000e\u0002\u000f\u0007\u000f\u0002\u0010\u0007\u0010\u0002"
+        + "\u0011\u0007\u0011\u0002\u0012\u0007\u0012\u0002\u0013\u0007\u0013\u0002"
+        + "\u0014\u0007\u0014\u0002\u0015\u0007\u0015\u0002\u0016\u0007\u0016\u0002"
+        + "\u0017\u0007\u0017\u0002\u0018\u0007\u0018\u0002\u0019\u0007\u0019\u0002"
+        + "\u001a\u0007\u001a\u0002\u001b\u0007\u001b\u0002\u001c\u0007\u001c\u0002"
+        + "\u001d\u0007\u001d\u0002\u001e\u0007\u001e\u0002\u001f\u0007\u001f\u0002"
+        + " \u0007 \u0002!\u0007!\u0002\"\u0007\"\u0002#\u0007#\u0002$\u0007$\u0002"
+        + "%\u0007%\u0002&\u0007&\u0002\'\u0007\'\u0002(\u0007(\u0002)\u0007)\u0002"
+        + "*\u0007*\u0002+\u0007+\u0002,\u0007,\u0002-\u0007-\u0002.\u0007.\u0002"
+        + "/\u0007/\u00020\u00070\u00021\u00071\u00022\u00072\u00023\u00073\u0002"
+        + "4\u00074\u00025\u00075\u00026\u00076\u00027\u00077\u00028\u00078\u0002"
+        + "9\u00079\u0002:\u0007:\u0002;\u0007;\u0002<\u0007<\u0002=\u0007=\u0002"
+        + ">\u0007>\u0002?\u0007?\u0002@\u0007@\u0002A\u0007A\u0002B\u0007B\u0002"
+        + "C\u0007C\u0002D\u0007D\u0002E\u0007E\u0002F\u0007F\u0002G\u0007G\u0002"
+        + "H\u0007H\u0002I\u0007I\u0002J\u0007J\u0002K\u0007K\u0002L\u0007L\u0002"
+        + "M\u0007M\u0002N\u0007N\u0002O\u0007O\u0002P\u0007P\u0002Q\u0007Q\u0002"
+        + "R\u0007R\u0002S\u0007S\u0002T\u0007T\u0002U\u0007U\u0001\u0000\u0004\u0000"
+        + "\u00b0\b\u0000\u000b\u0000\f\u0000\u00b1\u0001\u0000\u0001\u0000\u0001"
+        + "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0005\u0001\u00ba\b\u0001\n"
+        + "\u0001\f\u0001\u00bd\t\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"
+        + "\u0001\u0001\u0001\u0005\u0001\u00c4\b\u0001\n\u0001\f\u0001\u00c7\t\u0001"
+        + "\u0001\u0001\u0001\u0001\u0003\u0001\u00cb\b\u0001\u0001\u0001\u0001\u0001"
+        + "\u0001\u0002\u0001\u0002\u0001\u0003\u0001\u0003\u0001\u0004\u0001\u0004"
+        + "\u0001\u0005\u0001\u0005\u0001\u0006\u0001\u0006\u0001\u0007\u0001\u0007"
+        + "\u0001\b\u0001\b\u0001\b\u0001\b\u0001\t\u0001\t\u0001\t\u0001\t\u0001"
+        + "\t\u0001\n\u0001\n\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\f\u0001"
+        + "\r\u0001\r\u0001\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001"
+        + "\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001"
+        + "\u000f\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0011\u0001\u0011\u0001"
+        + "\u0011\u0001\u0011\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001"
+        + "\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0013\u0001"
+        + "\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0014\u0001"
+        + "\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001"
+        + "\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0016\u0001\u0016\u0001"
+        + "\u0016\u0001\u0016\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001"
+        + "\u0017\u0001\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0001"
+        + "\u0018\u0001\u0018\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0001"
+        + "\u0019\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001"
+        + "\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001"
+        + "\u001b\u0001\u001b\u0001\u001c\u0001\u001c\u0001\u001d\u0001\u001d\u0001"
+        + "\u001e\u0001\u001e\u0001\u001e\u0001\u001f\u0001\u001f\u0001 \u0001 \u0001"
+        + "!\u0001!\u0001\"\u0001\"\u0001\"\u0001#\u0001#\u0001#\u0001$\u0001$\u0001"
+        + "$\u0001$\u0001%\u0001%\u0001&\u0001&\u0001&\u0001\'\u0001\'\u0001(\u0001"
+        + "(\u0001(\u0001)\u0001)\u0001)\u0001*\u0001*\u0001*\u0001*\u0001+\u0001"
+        + "+\u0001+\u0001,\u0001,\u0001,\u0001,\u0001-\u0001-\u0001.\u0001.\u0001"
+        + "/\u0001/\u00010\u00010\u00010\u00011\u00011\u00011\u00012\u00012\u0001"
+        + "3\u00013\u00014\u00014\u00014\u00015\u00015\u00015\u00016\u00016\u0001"
+        + "6\u00017\u00017\u00017\u00018\u00018\u00018\u00018\u00019\u00019\u0001"
+        + "9\u0001:\u0001:\u0001:\u0001;\u0001;\u0001<\u0001<\u0001<\u0001=\u0001"
+        + "=\u0001=\u0001>\u0001>\u0001>\u0001?\u0001?\u0001?\u0001@\u0001@\u0001"
+        + "@\u0001A\u0001A\u0001A\u0001B\u0001B\u0001B\u0001C\u0001C\u0001C\u0001"
+        + "D\u0001D\u0001D\u0001D\u0001E\u0001E\u0001E\u0001E\u0001F\u0001F\u0001"
+        + "F\u0001F\u0001F\u0001G\u0001G\u0004G\u01ba\bG\u000bG\fG\u01bb\u0001G\u0003"
+        + "G\u01bf\bG\u0001H\u0001H\u0001H\u0004H\u01c4\bH\u000bH\fH\u01c5\u0001"
+        + "H\u0003H\u01c9\bH\u0001I\u0001I\u0001I\u0005I\u01ce\bI\nI\fI\u01d1\tI"
+        + "\u0003I\u01d3\bI\u0001I\u0003I\u01d6\bI\u0001J\u0001J\u0001J\u0005J\u01db"
+        + "\bJ\nJ\fJ\u01de\tJ\u0003J\u01e0\bJ\u0001J\u0001J\u0004J\u01e4\bJ\u000b"
+        + "J\fJ\u01e5\u0003J\u01e8\bJ\u0001J\u0001J\u0003J\u01ec\bJ\u0001J\u0004"
+        + "J\u01ef\bJ\u000bJ\fJ\u01f0\u0003J\u01f3\bJ\u0001J\u0003J\u01f6\bJ\u0001"
+        + "K\u0001K\u0001K\u0001K\u0001K\u0001K\u0005K\u01fe\bK\nK\fK\u0201\tK\u0001"
+        + "K\u0001K\u0001K\u0001K\u0001K\u0001K\u0001K\u0005K\u020a\bK\nK\fK\u020d"
+        + "\tK\u0001K\u0003K\u0210\bK\u0001L\u0001L\u0001L\u0001L\u0004L\u0216\b"
+        + "L\u000bL\fL\u0217\u0001L\u0001L\u0005L\u021c\bL\nL\fL\u021f\tL\u0001L"
+        + "\u0001L\u0001M\u0001M\u0001M\u0001M\u0001M\u0001N\u0001N\u0001N\u0001"
+        + "N\u0001N\u0001N\u0001O\u0001O\u0001O\u0001O\u0001O\u0001P\u0001P\u0001"
+        + "P\u0001P\u0004P\u0237\bP\u000bP\fP\u0238\u0001Q\u0001Q\u0001Q\u0001Q\u0005"
+        + "Q\u023f\bQ\nQ\fQ\u0242\tQ\u0001Q\u0001Q\u0001R\u0001R\u0005R\u0248\bR"
+        + "\nR\fR\u024b\tR\u0001S\u0001S\u0001S\u0001S\u0001T\u0001T\u0001T\u0005"
+        + "T\u0254\bT\nT\fT\u0257\tT\u0003T\u0259\bT\u0001T\u0001T\u0001U\u0001U"
+        + "\u0005U\u025f\bU\nU\fU\u0262\tU\u0001U\u0001U\u0005\u00bb\u00c5\u01ff"
+        + "\u020b\u0217\u0000V\u0002\u0001\u0004\u0002\u0006\u0003\b\u0004\n\u0005"
+        + "\f\u0006\u000e\u0007\u0010\b\u0012\t\u0014\n\u0016\u000b\u0018\f\u001a"
+        + "\r\u001c\u000e\u001e\u000f \u0010\"\u0011$\u0012&\u0013(\u0014*\u0015"
+        + ",\u0016.\u00170\u00182\u00194\u001a6\u001b8\u001c:\u001d<\u001e>\u001f"
+        + "@ B!D\"F#H$J%L&N\'P(R)T*V+X,Z-\\.^/`0b1d2f3h4j5l6n7p8r9t:v;x<z=|>~?\u0080"
+        + "@\u0082A\u0084B\u0086C\u0088D\u008aE\u008cF\u008eG\u0090H\u0092I\u0094"
+        + "J\u0096K\u0098L\u009aM\u009cN\u009eO\u00a0P\u00a2Q\u00a4R\u00a6S\u00a8"
+        + "T\u00aaU\u00acV\u0002\u0000\u0001\u0013\u0003\u0000\t\n\r\r  \u0002\u0000"
+        + "\n\n\r\r\u0001\u000007\u0002\u0000LLll\u0002\u0000XXxx\u0003\u000009A"
+        + "Faf\u0001\u000019\u0001\u000009\u0006\u0000DDFFLLddffll\u0002\u0000EE"
+        + "ee\u0002\u0000++--\u0004\u0000DDFFddff\u0002\u0000\"\"\\\\\u0002\u0000"
+        + "\'\'\\\\\u0001\u0000\n\n\u0002\u0000\n\n//\u0007\u0000UUcciilmssuuxx\u0003"
+        + "\u0000AZ__az\u0004\u000009AZ__az\u0286\u0000\u0002\u0001\u0000\u0000\u0000"
+        + "\u0000\u0004\u0001\u0000\u0000\u0000\u0000\u0006\u0001\u0000\u0000\u0000"
+        + "\u0000\b\u0001\u0000\u0000\u0000\u0000\n\u0001\u0000\u0000\u0000\u0000"
+        + "\f\u0001\u0000\u0000\u0000\u0000\u000e\u0001\u0000\u0000\u0000\u0000\u0010"
+        + "\u0001\u0000\u0000\u0000\u0000\u0012\u0001\u0000\u0000\u0000\u0000\u0014"
+        + "\u0001\u0000\u0000\u0000\u0000\u0016\u0001\u0000\u0000\u0000\u0000\u0018"
+        + "\u0001\u0000\u0000\u0000\u0000\u001a\u0001\u0000\u0000\u0000\u0000\u001c"
+        + "\u0001\u0000\u0000\u0000\u0000\u001e\u0001\u0000\u0000\u0000\u0000 \u0001"
+        + "\u0000\u0000\u0000\u0000\"\u0001\u0000\u0000\u0000\u0000$\u0001\u0000"
+        + "\u0000\u0000\u0000&\u0001\u0000\u0000\u0000\u0000(\u0001\u0000\u0000\u0000"
+        + "\u0000*\u0001\u0000\u0000\u0000\u0000,\u0001\u0000\u0000\u0000\u0000."
+        + "\u0001\u0000\u0000\u0000\u00000\u0001\u0000\u0000\u0000\u00002\u0001\u0000"
+        + "\u0000\u0000\u00004\u0001\u0000\u0000\u0000\u00006\u0001\u0000\u0000\u0000"
+        + "\u00008\u0001\u0000\u0000\u0000\u0000:\u0001\u0000\u0000\u0000\u0000<"
+        + "\u0001\u0000\u0000\u0000\u0000>\u0001\u0000\u0000\u0000\u0000@\u0001\u0000"
+        + "\u0000\u0000\u0000B\u0001\u0000\u0000\u0000\u0000D\u0001\u0000\u0000\u0000"
+        + "\u0000F\u0001\u0000\u0000\u0000\u0000H\u0001\u0000\u0000\u0000\u0000J"
+        + "\u0001\u0000\u0000\u0000\u0000L\u0001\u0000\u0000\u0000\u0000N\u0001\u0000"
+        + "\u0000\u0000\u0000P\u0001\u0000\u0000\u0000\u0000R\u0001\u0000\u0000\u0000"
+        + "\u0000T\u0001\u0000\u0000\u0000\u0000V\u0001\u0000\u0000\u0000\u0000X"
+        + "\u0001\u0000\u0000\u0000\u0000Z\u0001\u0000\u0000\u0000\u0000\\\u0001"
+        + "\u0000\u0000\u0000\u0000^\u0001\u0000\u0000\u0000\u0000`\u0001\u0000\u0000"
+        + "\u0000\u0000b\u0001\u0000\u0000\u0000\u0000d\u0001\u0000\u0000\u0000\u0000"
+        + "f\u0001\u0000\u0000\u0000\u0000h\u0001\u0000\u0000\u0000\u0000j\u0001"
+        + "\u0000\u0000\u0000\u0000l\u0001\u0000\u0000\u0000\u0000n\u0001\u0000\u0000"
+        + "\u0000\u0000p\u0001\u0000\u0000\u0000\u0000r\u0001\u0000\u0000\u0000\u0000"
+        + "t\u0001\u0000\u0000\u0000\u0000v\u0001\u0000\u0000\u0000\u0000x\u0001"
+        + "\u0000\u0000\u0000\u0000z\u0001\u0000\u0000\u0000\u0000|\u0001\u0000\u0000"
+        + "\u0000\u0000~\u0001\u0000\u0000\u0000\u0000\u0080\u0001\u0000\u0000\u0000"
+        + "\u0000\u0082\u0001\u0000\u0000\u0000\u0000\u0084\u0001\u0000\u0000\u0000"
+        + "\u0000\u0086\u0001\u0000\u0000\u0000\u0000\u0088\u0001\u0000\u0000\u0000"
+        + "\u0000\u008a\u0001\u0000\u0000\u0000\u0000\u008c\u0001\u0000\u0000\u0000"
+        + "\u0000\u008e\u0001\u0000\u0000\u0000\u0000\u0090\u0001\u0000\u0000\u0000"
+        + "\u0000\u0092\u0001\u0000\u0000\u0000\u0000\u0094\u0001\u0000\u0000\u0000"
+        + "\u0000\u0096\u0001\u0000\u0000\u0000\u0000\u0098\u0001\u0000\u0000\u0000"
+        + "\u0000\u009a\u0001\u0000\u0000\u0000\u0000\u009c\u0001\u0000\u0000\u0000"
+        + "\u0000\u009e\u0001\u0000\u0000\u0000\u0000\u00a0\u0001\u0000\u0000\u0000"
+        + "\u0000\u00a2\u0001\u0000\u0000\u0000\u0000\u00a4\u0001\u0000\u0000\u0000"
+        + "\u0000\u00a6\u0001\u0000\u0000\u0000\u0000\u00a8\u0001\u0000\u0000\u0000"
+        + "\u0001\u00aa\u0001\u0000\u0000\u0000\u0001\u00ac\u0001\u0000\u0000\u0000"
+        + "\u0002\u00af\u0001\u0000\u0000\u0000\u0004\u00ca\u0001\u0000\u0000\u0000"
+        + "\u0006\u00ce\u0001\u0000\u0000\u0000\b\u00d0\u0001\u0000\u0000\u0000\n"
+        + "\u00d2\u0001\u0000\u0000\u0000\f\u00d4\u0001\u0000\u0000\u0000\u000e\u00d6"
+        + "\u0001\u0000\u0000\u0000\u0010\u00d8\u0001\u0000\u0000\u0000\u0012\u00da"
+        + "\u0001\u0000\u0000\u0000\u0014\u00de\u0001\u0000\u0000\u0000\u0016\u00e3"
+        + "\u0001\u0000\u0000\u0000\u0018\u00e5\u0001\u0000\u0000\u0000\u001a\u00e7"
+        + "\u0001\u0000\u0000\u0000\u001c\u00ea\u0001\u0000\u0000\u0000\u001e\u00ed"
+        + "\u0001\u0000\u0000\u0000 \u00f2\u0001\u0000\u0000\u0000\"\u00f8\u0001"
+        + "\u0000\u0000\u0000$\u00fb\u0001\u0000\u0000\u0000&\u00ff\u0001\u0000\u0000"
+        + "\u0000(\u0108\u0001\u0000\u0000\u0000*\u010e\u0001\u0000\u0000\u0000,"
+        + "\u0115\u0001\u0000\u0000\u0000.\u0119\u0001\u0000\u0000\u00000\u011d\u0001"
+        + "\u0000\u0000\u00002\u0123\u0001\u0000\u0000\u00004\u0129\u0001\u0000\u0000"
+        + "\u00006\u012e\u0001\u0000\u0000\u00008\u0139\u0001\u0000\u0000\u0000:"
+        + "\u013b\u0001\u0000\u0000\u0000<\u013d\u0001\u0000\u0000\u0000>\u013f\u0001"
+        + "\u0000\u0000\u0000@\u0142\u0001\u0000\u0000\u0000B\u0144\u0001\u0000\u0000"
+        + "\u0000D\u0146\u0001\u0000\u0000\u0000F\u0148\u0001\u0000\u0000\u0000H"
+        + "\u014b\u0001\u0000\u0000\u0000J\u014e\u0001\u0000\u0000\u0000L\u0152\u0001"
+        + "\u0000\u0000\u0000N\u0154\u0001\u0000\u0000\u0000P\u0157\u0001\u0000\u0000"
+        + "\u0000R\u0159\u0001\u0000\u0000\u0000T\u015c\u0001\u0000\u0000\u0000V"
+        + "\u015f\u0001\u0000\u0000\u0000X\u0163\u0001\u0000\u0000\u0000Z\u0166\u0001"
+        + "\u0000\u0000\u0000\\\u016a\u0001\u0000\u0000\u0000^\u016c\u0001\u0000"
+        + "\u0000\u0000`\u016e\u0001\u0000\u0000\u0000b\u0170\u0001\u0000\u0000\u0000"
+        + "d\u0173\u0001\u0000\u0000\u0000f\u0176\u0001\u0000\u0000\u0000h\u0178"
+        + "\u0001\u0000\u0000\u0000j\u017a\u0001\u0000\u0000\u0000l\u017d\u0001\u0000"
+        + "\u0000\u0000n\u0180\u0001\u0000\u0000\u0000p\u0183\u0001\u0000\u0000\u0000"
+        + "r\u0186\u0001\u0000\u0000\u0000t\u018a\u0001\u0000\u0000\u0000v\u018d"
+        + "\u0001\u0000\u0000\u0000x\u0190\u0001\u0000\u0000\u0000z\u0192\u0001\u0000"
+        + "\u0000\u0000|\u0195\u0001\u0000\u0000\u0000~\u0198\u0001\u0000\u0000\u0000"
+        + "\u0080\u019b\u0001\u0000\u0000\u0000\u0082\u019e\u0001\u0000\u0000\u0000"
+        + "\u0084\u01a1\u0001\u0000\u0000\u0000\u0086\u01a4\u0001\u0000\u0000\u0000"
+        + "\u0088\u01a7\u0001\u0000\u0000\u0000\u008a\u01aa\u0001\u0000\u0000\u0000"
+        + "\u008c\u01ae\u0001\u0000\u0000\u0000\u008e\u01b2\u0001\u0000\u0000\u0000"
+        + "\u0090\u01b7\u0001\u0000\u0000\u0000\u0092\u01c0\u0001\u0000\u0000\u0000"
+        + "\u0094\u01d2\u0001\u0000\u0000\u0000\u0096\u01df\u0001\u0000\u0000\u0000"
+        + "\u0098\u020f\u0001\u0000\u0000\u0000\u009a\u0211\u0001\u0000\u0000\u0000"
+        + "\u009c\u0222\u0001\u0000\u0000\u0000\u009e\u0227\u0001\u0000\u0000\u0000"
+        + "\u00a0\u022d\u0001\u0000\u0000\u0000\u00a2\u0232\u0001\u0000\u0000\u0000"
+        + "\u00a4\u023a\u0001\u0000\u0000\u0000\u00a6\u0245\u0001\u0000\u0000\u0000"
+        + "\u00a8\u024c\u0001\u0000\u0000\u0000\u00aa\u0258\u0001\u0000\u0000\u0000"
+        + "\u00ac\u025c\u0001\u0000\u0000\u0000\u00ae\u00b0\u0007\u0000\u0000\u0000"
+        + "\u00af\u00ae\u0001\u0000\u0000\u0000\u00b0\u00b1\u0001\u0000\u0000\u0000"
+        + "\u00b1\u00af\u0001\u0000\u0000\u0000\u00b1\u00b2\u0001\u0000\u0000\u0000"
+        + "\u00b2\u00b3\u0001\u0000\u0000\u0000\u00b3\u00b4\u0006\u0000\u0000\u0000"
+        + "\u00b4\u0003\u0001\u0000\u0000\u0000\u00b5\u00b6\u0005/\u0000\u0000\u00b6"
+        + "\u00b7\u0005/\u0000\u0000\u00b7\u00bb\u0001\u0000\u0000\u0000\u00b8\u00ba"
+        + "\t\u0000\u0000\u0000\u00b9\u00b8\u0001\u0000\u0000\u0000\u00ba\u00bd\u0001"
+        + "\u0000\u0000\u0000\u00bb\u00bc\u0001\u0000\u0000\u0000\u00bb\u00b9\u0001"
+        + "\u0000\u0000\u0000\u00bc\u00be\u0001\u0000\u0000\u0000\u00bd\u00bb\u0001"
+        + "\u0000\u0000\u0000\u00be\u00cb\u0007\u0001\u0000\u0000\u00bf\u00c0\u0005"
+        + "/\u0000\u0000\u00c0\u00c1\u0005*\u0000\u0000\u00c1\u00c5\u0001\u0000\u0000"
+        + "\u0000\u00c2\u00c4\t\u0000\u0000\u0000\u00c3\u00c2\u0001\u0000\u0000\u0000"
+        + "\u00c4\u00c7\u0001\u0000\u0000\u0000\u00c5\u00c6\u0001\u0000\u0000\u0000"
+        + "\u00c5\u00c3\u0001\u0000\u0000\u0000\u00c6\u00c8\u0001\u0000\u0000\u0000"
+        + "\u00c7\u00c5\u0001\u0000\u0000\u0000\u00c8\u00c9\u0005*\u0000\u0000\u00c9"
+        + "\u00cb\u0005/\u0000\u0000\u00ca\u00b5\u0001\u0000\u0000\u0000\u00ca\u00bf"
+        + "\u0001\u0000\u0000\u0000\u00cb\u00cc\u0001\u0000\u0000\u0000\u00cc\u00cd"
+        + "\u0006\u0001\u0000\u0000\u00cd\u0005\u0001\u0000\u0000\u0000\u00ce\u00cf"
+        + "\u0005{\u0000\u0000\u00cf\u0007\u0001\u0000\u0000\u0000\u00d0\u00d1\u0005"
+        + "}\u0000\u0000\u00d1\t\u0001\u0000\u0000\u0000\u00d2\u00d3\u0005[\u0000"
+        + "\u0000\u00d3\u000b\u0001\u0000\u0000\u0000\u00d4\u00d5\u0005]\u0000\u0000"
+        + "\u00d5\r\u0001\u0000\u0000\u0000\u00d6\u00d7\u0005(\u0000\u0000\u00d7"
+        + "\u000f\u0001\u0000\u0000\u0000\u00d8\u00d9\u0005)\u0000\u0000\u00d9\u0011"
+        + "\u0001\u0000\u0000\u0000\u00da\u00db\u0005.\u0000\u0000\u00db\u00dc\u0001"
+        + "\u0000\u0000\u0000\u00dc\u00dd\u0006\b\u0001\u0000\u00dd\u0013\u0001\u0000"
+        + "\u0000\u0000\u00de\u00df\u0005?\u0000\u0000\u00df\u00e0\u0005.\u0000\u0000"
+        + "\u00e0\u00e1\u0001\u0000\u0000\u0000\u00e1\u00e2\u0006\t\u0001\u0000\u00e2"
+        + "\u0015\u0001\u0000\u0000\u0000\u00e3\u00e4\u0005,\u0000\u0000\u00e4\u0017"
+        + "\u0001\u0000\u0000\u0000\u00e5\u00e6\u0005;\u0000\u0000\u00e6\u0019\u0001"
+        + "\u0000\u0000\u0000\u00e7\u00e8\u0005i\u0000\u0000\u00e8\u00e9\u0005f\u0000"
+        + "\u0000\u00e9\u001b\u0001\u0000\u0000\u0000\u00ea\u00eb\u0005i\u0000\u0000"
+        + "\u00eb\u00ec\u0005n\u0000\u0000\u00ec\u001d\u0001\u0000\u0000\u0000\u00ed"
+        + "\u00ee\u0005e\u0000\u0000\u00ee\u00ef\u0005l\u0000\u0000\u00ef\u00f0\u0005"
+        + "s\u0000\u0000\u00f0\u00f1\u0005e\u0000\u0000\u00f1\u001f\u0001\u0000\u0000"
+        + "\u0000\u00f2\u00f3\u0005w\u0000\u0000\u00f3\u00f4\u0005h\u0000\u0000\u00f4"
+        + "\u00f5\u0005i\u0000\u0000\u00f5\u00f6\u0005l\u0000\u0000\u00f6\u00f7\u0005"
+        + "e\u0000\u0000\u00f7!\u0001\u0000\u0000\u0000\u00f8\u00f9\u0005d\u0000"
+        + "\u0000\u00f9\u00fa\u0005o\u0000\u0000\u00fa#\u0001\u0000\u0000\u0000\u00fb"
+        + "\u00fc\u0005f\u0000\u0000\u00fc\u00fd\u0005o\u0000\u0000\u00fd\u00fe\u0005"
+        + "r\u0000\u0000\u00fe%\u0001\u0000\u0000\u0000\u00ff\u0100\u0005c\u0000"
+        + "\u0000\u0100\u0101\u0005o\u0000\u0000\u0101\u0102\u0005n\u0000\u0000\u0102"
+        + "\u0103\u0005t\u0000\u0000\u0103\u0104\u0005i\u0000\u0000\u0104\u0105\u0005"
+        + "n\u0000\u0000\u0105\u0106\u0005u\u0000\u0000\u0106\u0107\u0005e\u0000"
+        + "\u0000\u0107\'\u0001\u0000\u0000\u0000\u0108\u0109\u0005b\u0000\u0000"
+        + "\u0109\u010a\u0005r\u0000\u0000\u010a\u010b\u0005e\u0000\u0000\u010b\u010c"
+        + "\u0005a\u0000\u0000\u010c\u010d\u0005k\u0000\u0000\u010d)\u0001\u0000"
+        + "\u0000\u0000\u010e\u010f\u0005r\u0000\u0000\u010f\u0110\u0005e\u0000\u0000"
+        + "\u0110\u0111\u0005t\u0000\u0000\u0111\u0112\u0005u\u0000\u0000\u0112\u0113"
+        + "\u0005r\u0000\u0000\u0113\u0114\u0005n\u0000\u0000\u0114+\u0001\u0000"
+        + "\u0000\u0000\u0115\u0116\u0005n\u0000\u0000\u0116\u0117\u0005e\u0000\u0000"
+        + "\u0117\u0118\u0005w\u0000\u0000\u0118-\u0001\u0000\u0000\u0000\u0119\u011a"
+        + "\u0005t\u0000\u0000\u011a\u011b\u0005r\u0000\u0000\u011b\u011c\u0005y"
+        + "\u0000\u0000\u011c/\u0001\u0000\u0000\u0000\u011d\u011e\u0005c\u0000\u0000"
+        + "\u011e\u011f\u0005a\u0000\u0000\u011f\u0120\u0005t\u0000\u0000\u0120\u0121"
+        + "\u0005c\u0000\u0000\u0121\u0122\u0005h\u0000\u0000\u01221\u0001\u0000"
+        + "\u0000\u0000\u0123\u0124\u0005t\u0000\u0000\u0124\u0125\u0005h\u0000\u0000"
+        + "\u0125\u0126\u0005r\u0000\u0000\u0126\u0127\u0005o\u0000\u0000\u0127\u0128"
+        + "\u0005w\u0000\u0000\u01283\u0001\u0000\u0000\u0000\u0129\u012a\u0005t"
+        + "\u0000\u0000\u012a\u012b\u0005h\u0000\u0000\u012b\u012c\u0005i\u0000\u0000"
+        + "\u012c\u012d\u0005s\u0000\u0000\u012d5\u0001\u0000\u0000\u0000\u012e\u012f"
+        + "\u0005i\u0000\u0000\u012f\u0130\u0005n\u0000\u0000\u0130\u0131\u0005s"
+        + "\u0000\u0000\u0131\u0132\u0005t\u0000\u0000\u0132\u0133\u0005a\u0000\u0000"
+        + "\u0133\u0134\u0005n\u0000\u0000\u0134\u0135\u0005c\u0000\u0000\u0135\u0136"
+        + "\u0005e\u0000\u0000\u0136\u0137\u0005o\u0000\u0000\u0137\u0138\u0005f"
+        + "\u0000\u0000\u01387\u0001\u0000\u0000\u0000\u0139\u013a\u0005!\u0000\u0000"
+        + "\u013a9\u0001\u0000\u0000\u0000\u013b\u013c\u0005~\u0000\u0000\u013c;"
+        + "\u0001\u0000\u0000\u0000\u013d\u013e\u0005*\u0000\u0000\u013e=\u0001\u0000"
+        + "\u0000\u0000\u013f\u0140\u0005/\u0000\u0000\u0140\u0141\u0004\u001e\u0000"
+        + "\u0000\u0141?\u0001\u0000\u0000\u0000\u0142\u0143\u0005%\u0000\u0000\u0143"
+        + "A\u0001\u0000\u0000\u0000\u0144\u0145\u0005+\u0000\u0000\u0145C\u0001"
+        + "\u0000\u0000\u0000\u0146\u0147\u0005-\u0000\u0000\u0147E\u0001\u0000\u0000"
+        + "\u0000\u0148\u0149\u0005<\u0000\u0000\u0149\u014a\u0005<\u0000\u0000\u014a"
+        + "G\u0001\u0000\u0000\u0000\u014b\u014c\u0005>\u0000\u0000\u014c\u014d\u0005"
+        + ">\u0000\u0000\u014dI\u0001\u0000\u0000\u0000\u014e\u014f\u0005>\u0000"
+        + "\u0000\u014f\u0150\u0005>\u0000\u0000\u0150\u0151\u0005>\u0000\u0000\u0151"
+        + "K\u0001\u0000\u0000\u0000\u0152\u0153\u0005<\u0000\u0000\u0153M\u0001"
+        + "\u0000\u0000\u0000\u0154\u0155\u0005<\u0000\u0000\u0155\u0156\u0005=\u0000"
+        + "\u0000\u0156O\u0001\u0000\u0000\u0000\u0157\u0158\u0005>\u0000\u0000\u0158"
+        + "Q\u0001\u0000\u0000\u0000\u0159\u015a\u0005>\u0000\u0000\u015a\u015b\u0005"
+        + "=\u0000\u0000\u015bS\u0001\u0000\u0000\u0000\u015c\u015d\u0005=\u0000"
+        + "\u0000\u015d\u015e\u0005=\u0000\u0000\u015eU\u0001\u0000\u0000\u0000\u015f"
+        + "\u0160\u0005=\u0000\u0000\u0160\u0161\u0005=\u0000\u0000\u0161\u0162\u0005"
+        + "=\u0000\u0000\u0162W\u0001\u0000\u0000\u0000\u0163\u0164\u0005!\u0000"
+        + "\u0000\u0164\u0165\u0005=\u0000\u0000\u0165Y\u0001\u0000\u0000\u0000\u0166"
+        + "\u0167\u0005!\u0000\u0000\u0167\u0168\u0005=\u0000\u0000\u0168\u0169\u0005"
+        + "=\u0000\u0000\u0169[\u0001\u0000\u0000\u0000\u016a\u016b\u0005&\u0000"
+        + "\u0000\u016b]\u0001\u0000\u0000\u0000\u016c\u016d\u0005^\u0000\u0000\u016d"
+        + "_\u0001\u0000\u0000\u0000\u016e\u016f\u0005|\u0000\u0000\u016fa\u0001"
+        + "\u0000\u0000\u0000\u0170\u0171\u0005&\u0000\u0000\u0171\u0172\u0005&\u0000"
+        + "\u0000\u0172c\u0001\u0000\u0000\u0000\u0173\u0174\u0005|\u0000\u0000\u0174"
+        + "\u0175\u0005|\u0000\u0000\u0175e\u0001\u0000\u0000\u0000\u0176\u0177\u0005"
+        + "?\u0000\u0000\u0177g\u0001\u0000\u0000\u0000\u0178\u0179\u0005:\u0000"
+        + "\u0000\u0179i\u0001\u0000\u0000\u0000\u017a\u017b\u0005?\u0000\u0000\u017b"
+        + "\u017c\u0005:\u0000\u0000\u017ck\u0001\u0000\u0000\u0000\u017d\u017e\u0005"
+        + ":\u0000\u0000\u017e\u017f\u0005:\u0000\u0000\u017fm\u0001\u0000\u0000"
+        + "\u0000\u0180\u0181\u0005-\u0000\u0000\u0181\u0182\u0005>\u0000\u0000\u0182"
+        + "o\u0001\u0000\u0000\u0000\u0183\u0184\u0005=\u0000\u0000\u0184\u0185\u0005"
+        + "~\u0000\u0000\u0185q\u0001\u0000\u0000\u0000\u0186\u0187\u0005=\u0000"
+        + "\u0000\u0187\u0188\u0005=\u0000\u0000\u0188\u0189\u0005~\u0000\u0000\u0189"
+        + "s\u0001\u0000\u0000\u0000\u018a\u018b\u0005+\u0000\u0000\u018b\u018c\u0005"
+        + "+\u0000\u0000\u018cu\u0001\u0000\u0000\u0000\u018d\u018e\u0005-\u0000"
+        + "\u0000\u018e\u018f\u0005-\u0000\u0000\u018fw\u0001\u0000\u0000\u0000\u0190"
+        + "\u0191\u0005=\u0000\u0000\u0191y\u0001\u0000\u0000\u0000\u0192\u0193\u0005"
+        + "+\u0000\u0000\u0193\u0194\u0005=\u0000\u0000\u0194{\u0001\u0000\u0000"
+        + "\u0000\u0195\u0196\u0005-\u0000\u0000\u0196\u0197\u0005=\u0000\u0000\u0197"
+        + "}\u0001\u0000\u0000\u0000\u0198\u0199\u0005*\u0000\u0000\u0199\u019a\u0005"
+        + "=\u0000\u0000\u019a\u007f\u0001\u0000\u0000\u0000\u019b\u019c\u0005/\u0000"
+        + "\u0000\u019c\u019d\u0005=\u0000\u0000\u019d\u0081\u0001\u0000\u0000\u0000"
+        + "\u019e\u019f\u0005%\u0000\u0000\u019f\u01a0\u0005=\u0000\u0000\u01a0\u0083"
+        + "\u0001\u0000\u0000\u0000\u01a1\u01a2\u0005&\u0000\u0000\u01a2\u01a3\u0005"
+        + "=\u0000\u0000\u01a3\u0085\u0001\u0000\u0000\u0000\u01a4\u01a5\u0005^\u0000"
+        + "\u0000\u01a5\u01a6\u0005=\u0000\u0000\u01a6\u0087\u0001\u0000\u0000\u0000"
+        + "\u01a7\u01a8\u0005|\u0000\u0000\u01a8\u01a9\u0005=\u0000\u0000\u01a9\u0089"
+        + "\u0001\u0000\u0000\u0000\u01aa\u01ab\u0005<\u0000\u0000\u01ab\u01ac\u0005"
+        + "<\u0000\u0000\u01ac\u01ad\u0005=\u0000\u0000\u01ad\u008b\u0001\u0000\u0000"
+        + "\u0000\u01ae\u01af\u0005>\u0000\u0000\u01af\u01b0\u0005>\u0000\u0000\u01b0"
+        + "\u01b1\u0005=\u0000\u0000\u01b1\u008d\u0001\u0000\u0000\u0000\u01b2\u01b3"
+        + "\u0005>\u0000\u0000\u01b3\u01b4\u0005>\u0000\u0000\u01b4\u01b5\u0005>"
+        + "\u0000\u0000\u01b5\u01b6\u0005=\u0000\u0000\u01b6\u008f\u0001\u0000\u0000"
+        + "\u0000\u01b7\u01b9\u00050\u0000\u0000\u01b8\u01ba\u0007\u0002\u0000\u0000"
+        + "\u01b9\u01b8\u0001\u0000\u0000\u0000\u01ba\u01bb\u0001\u0000\u0000\u0000"
+        + "\u01bb\u01b9\u0001\u0000\u0000\u0000\u01bb\u01bc\u0001\u0000\u0000\u0000"
+        + "\u01bc\u01be\u0001\u0000\u0000\u0000\u01bd\u01bf\u0007\u0003\u0000\u0000"
+        + "\u01be\u01bd\u0001\u0000\u0000\u0000\u01be\u01bf\u0001\u0000\u0000\u0000"
+        + "\u01bf\u0091\u0001\u0000\u0000\u0000\u01c0\u01c1\u00050\u0000\u0000\u01c1"
+        + "\u01c3\u0007\u0004\u0000\u0000\u01c2\u01c4\u0007\u0005\u0000\u0000\u01c3"
+        + "\u01c2\u0001\u0000\u0000\u0000\u01c4\u01c5\u0001\u0000\u0000\u0000\u01c5"
+        + "\u01c3\u0001\u0000\u0000\u0000\u01c5\u01c6\u0001\u0000\u0000\u0000\u01c6"
+        + "\u01c8\u0001\u0000\u0000\u0000\u01c7\u01c9\u0007\u0003\u0000\u0000\u01c8"
+        + "\u01c7\u0001\u0000\u0000\u0000\u01c8\u01c9\u0001\u0000\u0000\u0000\u01c9"
+        + "\u0093\u0001\u0000\u0000\u0000\u01ca\u01d3\u00050\u0000\u0000\u01cb\u01cf"
+        + "\u0007\u0006\u0000\u0000\u01cc\u01ce\u0007\u0007\u0000\u0000\u01cd\u01cc"
+        + "\u0001\u0000\u0000\u0000\u01ce\u01d1\u0001\u0000\u0000\u0000\u01cf\u01cd"
+        + "\u0001\u0000\u0000\u0000\u01cf\u01d0\u0001\u0000\u0000\u0000\u01d0\u01d3"
+        + "\u0001\u0000\u0000\u0000\u01d1\u01cf\u0001\u0000\u0000\u0000\u01d2\u01ca"
+        + "\u0001\u0000\u0000\u0000\u01d2\u01cb\u0001\u0000\u0000\u0000\u01d3\u01d5"
+        + "\u0001\u0000\u0000\u0000\u01d4\u01d6\u0007\b\u0000\u0000\u01d5\u01d4\u0001"
+        + "\u0000\u0000\u0000\u01d5\u01d6\u0001\u0000\u0000\u0000\u01d6\u0095\u0001"
+        + "\u0000\u0000\u0000\u01d7\u01e0\u00050\u0000\u0000\u01d8\u01dc\u0007\u0006"
+        + "\u0000\u0000\u01d9\u01db\u0007\u0007\u0000\u0000\u01da\u01d9\u0001\u0000"
+        + "\u0000\u0000\u01db\u01de\u0001\u0000\u0000\u0000\u01dc\u01da\u0001\u0000"
+        + "\u0000\u0000\u01dc\u01dd\u0001\u0000\u0000\u0000\u01dd\u01e0\u0001\u0000"
+        + "\u0000\u0000\u01de\u01dc\u0001\u0000\u0000\u0000\u01df\u01d7\u0001\u0000"
+        + "\u0000\u0000\u01df\u01d8\u0001\u0000\u0000\u0000\u01e0\u01e7\u0001\u0000"
+        + "\u0000\u0000\u01e1\u01e3\u0003\u0012\b\u0000\u01e2\u01e4\u0007\u0007\u0000"
+        + "\u0000\u01e3\u01e2\u0001\u0000\u0000\u0000\u01e4\u01e5\u0001\u0000\u0000"
+        + "\u0000\u01e5\u01e3\u0001\u0000\u0000\u0000\u01e5\u01e6\u0001\u0000\u0000"
+        + "\u0000\u01e6\u01e8\u0001\u0000\u0000\u0000\u01e7\u01e1\u0001\u0000\u0000"
+        + "\u0000\u01e7\u01e8\u0001\u0000\u0000\u0000\u01e8\u01f2\u0001\u0000\u0000"
+        + "\u0000\u01e9\u01eb\u0007\t\u0000\u0000\u01ea\u01ec\u0007\n\u0000\u0000"
+        + "\u01eb\u01ea\u0001\u0000\u0000\u0000\u01eb\u01ec\u0001\u0000\u0000\u0000"
+        + "\u01ec\u01ee\u0001\u0000\u0000\u0000\u01ed\u01ef\u0007\u0007\u0000\u0000"
+        + "\u01ee\u01ed\u0001\u0000\u0000\u0000\u01ef\u01f0\u0001\u0000\u0000\u0000"
+        + "\u01f0\u01ee\u0001\u0000\u0000\u0000\u01f0\u01f1\u0001\u0000\u0000\u0000"
+        + "\u01f1\u01f3\u0001\u0000\u0000\u0000\u01f2\u01e9\u0001\u0000\u0000\u0000"
+        + "\u01f2\u01f3\u0001\u0000\u0000\u0000\u01f3\u01f5\u0001\u0000\u0000\u0000"
+        + "\u01f4\u01f6\u0007\u000b\u0000\u0000\u01f5\u01f4\u0001\u0000\u0000\u0000"
+        + "\u01f5\u01f6\u0001\u0000\u0000\u0000\u01f6\u0097\u0001\u0000\u0000\u0000"
+        + "\u01f7\u01ff\u0005\"\u0000\u0000\u01f8\u01f9\u0005\\\u0000\u0000\u01f9"
+        + "\u01fe\u0005\"\u0000\u0000\u01fa\u01fb\u0005\\\u0000\u0000\u01fb\u01fe"
+        + "\u0005\\\u0000\u0000\u01fc\u01fe\b\f\u0000\u0000\u01fd\u01f8\u0001\u0000"
+        + "\u0000\u0000\u01fd\u01fa\u0001\u0000\u0000\u0000\u01fd\u01fc\u0001\u0000"
+        + "\u0000\u0000\u01fe\u0201\u0001\u0000\u0000\u0000\u01ff\u0200\u0001\u0000"
+        + "\u0000\u0000\u01ff\u01fd\u0001\u0000\u0000\u0000\u0200\u0202\u0001\u0000"
+        + "\u0000\u0000\u0201\u01ff\u0001\u0000\u0000\u0000\u0202\u0210\u0005\"\u0000"
+        + "\u0000\u0203\u020b\u0005\'\u0000\u0000\u0204\u0205\u0005\\\u0000\u0000"
+        + "\u0205\u020a\u0005\'\u0000\u0000\u0206\u0207\u0005\\\u0000\u0000\u0207"
+        + "\u020a\u0005\\\u0000\u0000\u0208\u020a\b\r\u0000\u0000\u0209\u0204\u0001"
+        + "\u0000\u0000\u0000\u0209\u0206\u0001\u0000\u0000\u0000\u0209\u0208\u0001"
+        + "\u0000\u0000\u0000\u020a\u020d\u0001\u0000\u0000\u0000\u020b\u020c\u0001"
+        + "\u0000\u0000\u0000\u020b\u0209\u0001\u0000\u0000\u0000\u020c\u020e\u0001"
+        + "\u0000\u0000\u0000\u020d\u020b\u0001\u0000\u0000\u0000\u020e\u0210\u0005"
+        + "\'\u0000\u0000\u020f\u01f7\u0001\u0000\u0000\u0000\u020f\u0203\u0001\u0000"
+        + "\u0000\u0000\u0210\u0099\u0001\u0000\u0000\u0000\u0211\u0215\u0005/\u0000"
+        + "\u0000\u0212\u0213\u0005\\\u0000\u0000\u0213\u0216\b\u000e\u0000\u0000"
+        + "\u0214\u0216\b\u000f\u0000\u0000\u0215\u0212\u0001\u0000\u0000\u0000\u0215"
+        + "\u0214\u0001\u0000\u0000\u0000\u0216\u0217\u0001\u0000\u0000\u0000\u0217"
+        + "\u0218\u0001\u0000\u0000\u0000\u0217\u0215\u0001\u0000\u0000\u0000\u0218"
+        + "\u0219\u0001\u0000\u0000\u0000\u0219\u021d\u0005/\u0000\u0000\u021a\u021c"
+        + "\u0007\u0010\u0000\u0000\u021b\u021a\u0001\u0000\u0000\u0000\u021c\u021f"
+        + "\u0001\u0000\u0000\u0000\u021d\u021b\u0001\u0000\u0000\u0000\u021d\u021e"
+        + "\u0001\u0000\u0000\u0000\u021e\u0220\u0001\u0000\u0000\u0000\u021f\u021d"
+        + "\u0001\u0000\u0000\u0000\u0220\u0221\u0004L\u0001\u0000\u0221\u009b\u0001"
+        + "\u0000\u0000\u0000\u0222\u0223\u0005t\u0000\u0000\u0223\u0224\u0005r\u0000"
+        + "\u0000\u0224\u0225\u0005u\u0000\u0000\u0225\u0226\u0005e\u0000\u0000\u0226"
+        + "\u009d\u0001\u0000\u0000\u0000\u0227\u0228\u0005f\u0000\u0000\u0228\u0229"
+        + "\u0005a\u0000\u0000\u0229\u022a\u0005l\u0000\u0000\u022a\u022b\u0005s"
+        + "\u0000\u0000\u022b\u022c\u0005e\u0000\u0000\u022c\u009f\u0001\u0000\u0000"
+        + "\u0000\u022d\u022e\u0005n\u0000\u0000\u022e\u022f\u0005u\u0000\u0000\u022f"
+        + "\u0230\u0005l\u0000\u0000\u0230\u0231\u0005l\u0000\u0000\u0231\u00a1\u0001"
+        + "\u0000\u0000\u0000\u0232\u0236\u0003\u00a4Q\u0000\u0233\u0234\u0003\n"
+        + "\u0004\u0000\u0234\u0235\u0003\f\u0005\u0000\u0235\u0237\u0001\u0000\u0000"
+        + "\u0000\u0236\u0233\u0001\u0000\u0000\u0000\u0237\u0238\u0001\u0000\u0000"
+        + "\u0000\u0238\u0236\u0001\u0000\u0000\u0000\u0238\u0239\u0001\u0000\u0000"
+        + "\u0000\u0239\u00a3\u0001\u0000\u0000\u0000\u023a\u0240\u0003\u00a6R\u0000"
+        + "\u023b\u023c\u0003\u0012\b\u0000\u023c\u023d\u0003\u00a6R\u0000\u023d"
+        + "\u023f\u0001\u0000\u0000\u0000\u023e\u023b\u0001\u0000\u0000\u0000\u023f"
+        + "\u0242\u0001\u0000\u0000\u0000\u0240\u023e\u0001\u0000\u0000\u0000\u0240"
+        + "\u0241\u0001\u0000\u0000\u0000\u0241\u0243\u0001\u0000\u0000\u0000\u0242"
+        + "\u0240\u0001\u0000\u0000\u0000\u0243\u0244\u0004Q\u0002\u0000\u0244\u00a5"
+        + "\u0001\u0000\u0000\u0000\u0245\u0249\u0007\u0011\u0000\u0000\u0246\u0248"
+        + "\u0007\u0012\u0000\u0000\u0247\u0246\u0001\u0000\u0000\u0000\u0248\u024b"
+        + "\u0001\u0000\u0000\u0000\u0249\u0247\u0001\u0000\u0000\u0000\u0249\u024a"
+        + "\u0001\u0000\u0000\u0000\u024a\u00a7\u0001\u0000\u0000\u0000\u024b\u0249"
+        + "\u0001\u0000\u0000\u0000\u024c\u024d\t\u0000\u0000\u0000\u024d\u024e\u0001"
+        + "\u0000\u0000\u0000\u024e\u024f\u0006S\u0000\u0000\u024f\u00a9\u0001\u0000"
+        + "\u0000\u0000\u0250\u0259\u00050\u0000\u0000\u0251\u0255\u0007\u0006\u0000"
+        + "\u0000\u0252\u0254\u0007\u0007\u0000\u0000\u0253\u0252\u0001\u0000\u0000"
+        + "\u0000\u0254\u0257\u0001\u0000\u0000\u0000\u0255\u0253\u0001\u0000\u0000"
+        + "\u0000\u0255\u0256\u0001\u0000\u0000\u0000\u0256\u0259\u0001\u0000\u0000"
+        + "\u0000\u0257\u0255\u0001\u0000\u0000\u0000\u0258\u0250\u0001\u0000\u0000"
+        + "\u0000\u0258\u0251\u0001\u0000\u0000\u0000\u0259\u025a\u0001\u0000\u0000"
+        + "\u0000\u025a\u025b\u0006T\u0002\u0000\u025b\u00ab\u0001\u0000\u0000\u0000"
+        + "\u025c\u0260\u0007\u0011\u0000\u0000\u025d\u025f\u0007\u0012\u0000\u0000"
+        + "\u025e\u025d\u0001\u0000\u0000\u0000\u025f\u0262\u0001\u0000\u0000\u0000"
+        + "\u0260\u025e\u0001\u0000\u0000\u0000\u0260\u0261\u0001\u0000\u0000\u0000"
+        + "\u0261\u0263\u0001\u0000\u0000\u0000\u0262\u0260\u0001\u0000\u0000\u0000"
+        + "\u0263\u0264\u0006U\u0002\u0000\u0264\u00ad\u0001\u0000\u0000\u0000#\u0000"
+        + "\u0001\u00b1\u00bb\u00c5\u00ca\u01bb\u01be\u01c5\u01c8\u01cf\u01d2\u01d5"
+        + "\u01dc\u01df\u01e5\u01e7\u01eb\u01f0\u01f2\u01f5\u01fd\u01ff\u0209\u020b"
+        + "\u020f\u0215\u0217\u021d\u0238\u0240\u0249\u0255\u0258\u0260\u0003\u0006"
+        + "\u0000\u0000\u0002\u0001\u0000\u0002\u0000\u0000";
     public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray());
     static {
         _decisionToDFA = new DFA[_ATN.getNumberOfDecisions()];
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/ValueIterator.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/ValueIterator.java
new file mode 100644
index 000000000000..7f366bc3284a
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/ValueIterator.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.painless.api;
+
+import java.util.Iterator;
+
+/**
+ * An {@link Iterator} that can return primitive values
+ */
+public interface ValueIterator<T> extends Iterator<T> {
+    boolean nextBoolean();
+
+    byte nextByte();
+
+    short nextShort();
+
+    char nextChar();
+
+    int nextInt();
+
+    long nextLong();
+
+    float nextFloat();
+
+    double nextDouble();
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
index 2d84c9d2baab..db380f69c7c0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.painless.lookup;
 
 import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.painless.Def;
 import org.elasticsearch.painless.spi.Whitelist;
 import org.elasticsearch.painless.spi.WhitelistClass;
@@ -260,6 +261,14 @@ public void addPainlessClass(ClassLoader classLoader, String javaClassName, Map<
         addPainlessClass(clazz, annotations);
     }
 
+    private static IllegalArgumentException lookupException(String formatText, Object... args) {
+        return new IllegalArgumentException(Strings.format(formatText, args));
+    }
+
+    private static IllegalArgumentException lookupException(Throwable cause, String formatText, Object... args) {
+        return new IllegalArgumentException(Strings.format(formatText, args), cause);
+    }
+
     public void addPainlessClass(Class<?> clazz, Map<Class<?>, Object> annotations) {
         Objects.requireNonNull(clazz);
         Objects.requireNonNull(annotations);
@@ -283,22 +292,18 @@ public void addPainlessClass(Class<?> clazz, Map<Class<?>, Object> annotations)
         if (existingClass == null) {
             javaClassNamesToClasses.put(clazz.getName().intern(), clazz);
         } else if (existingClass != clazz) {
-            throw new IllegalArgumentException(
-                "class ["
-                    + canonicalClassName
-                    + "] "
-                    + "cannot represent multiple java classes with the same name from different class loaders"
+            throw lookupException(
+                "class [%s] cannot represent multiple java classes with the same name from different class loaders",
+                canonicalClassName
             );
         }
 
         existingClass = canonicalClassNamesToClasses.get(canonicalClassName);
 
         if (existingClass != null && existingClass != clazz) {
-            throw new IllegalArgumentException(
-                "class ["
-                    + canonicalClassName
-                    + "] "
-                    + "cannot represent multiple java classes with the same name from different class loaders"
+            throw lookupException(
+                "class [%s] cannot represent multiple java classes with the same name from different class loaders",
+                canonicalClassName
             );
         }
 
@@ -333,22 +338,16 @@ public void addPainlessClass(Class<?> clazz, Map<Class<?>, Object> annotations)
                     if (annotations.get(AliasAnnotation.class)instanceof AliasAnnotation alias) {
                         Class<?> existing = canonicalClassNamesToClasses.put(alias.alias(), clazz);
                         if (existing != null) {
-                            throw new IllegalArgumentException(
-                                "Cannot add alias [" + alias.alias() + "] for [" + clazz + "] that shadows class [" + existing + "]"
-                            );
+                            throw lookupException("Cannot add alias [%s] for [%s] that shadows class [%s]", alias.alias(), clazz, existing);
                         }
                     }
                 }
             } else if (importedClass != clazz) {
-                throw new IllegalArgumentException(
-                    "imported class ["
-                        + importedCanonicalClassName
-                        + "] cannot represent multiple "
-                        + "classes ["
-                        + canonicalClassName
-                        + "] and ["
-                        + typeToCanonicalTypeName(importedClass)
-                        + "]"
+                throw lookupException(
+                    "imported class [%s] cannot represent multiple classes [%s] and [%s]",
+                    importedCanonicalClassName,
+                    canonicalClassName,
+                    typeToCanonicalTypeName(importedClass)
                 );
             } else if (importClassName == false) {
                 throw new IllegalArgumentException("inconsistent no_import parameter found for class [" + canonicalClassName + "]");
@@ -367,15 +366,11 @@ public void addPainlessConstructor(
         Class<?> targetClass = canonicalClassNamesToClasses.get(targetCanonicalClassName);
 
         if (targetClass == null) {
-            throw new IllegalArgumentException(
-                "target class ["
-                    + targetCanonicalClassName
-                    + "] not found"
-                    + "for constructor [["
-                    + targetCanonicalClassName
-                    + "], "
-                    + canonicalTypeNameParameters
-                    + "]"
+            throw lookupException(
+                "target class [%s] not found for constructor [[%s], %s]",
+                targetCanonicalClassName,
+                targetCanonicalClassName,
+                canonicalTypeNameParameters
             );
         }
 
@@ -385,15 +380,11 @@ public void addPainlessConstructor(
             Class<?> typeParameter = canonicalTypeNameToType(canonicalTypeNameParameter);
 
             if (typeParameter == null) {
-                throw new IllegalArgumentException(
-                    "type parameter ["
-                        + canonicalTypeNameParameter
-                        + "] not found "
-                        + "for constructor [["
-                        + targetCanonicalClassName
-                        + "], "
-                        + canonicalTypeNameParameters
-                        + "]"
+                throw lookupException(
+                    "type parameter [%s] not found for constructor [[%s], %s]",
+                    canonicalTypeNameParameter,
+                    targetCanonicalClassName,
+                    canonicalTypeNameParameters
                 );
             }
 
@@ -415,15 +406,11 @@ public void addPainlessConstructor(Class<?> targetClass, List<Class<?>> typePara
         PainlessClassBuilder painlessClassBuilder = classesToPainlessClassBuilders.get(targetClass);
 
         if (painlessClassBuilder == null) {
-            throw new IllegalArgumentException(
-                "target class ["
-                    + targetCanonicalClassName
-                    + "] not found"
-                    + "for constructor [["
-                    + targetCanonicalClassName
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "]"
+            throw lookupException(
+                "target class [%s] not found for constructor [[%s], %s]",
+                targetCanonicalClassName,
+                targetCanonicalClassName,
+                typesToCanonicalTypeNames(typeParameters)
             );
         }
 
@@ -432,15 +419,11 @@ public void addPainlessConstructor(Class<?> targetClass, List<Class<?>> typePara
 
         for (Class<?> typeParameter : typeParameters) {
             if (isValidType(typeParameter) == false) {
-                throw new IllegalArgumentException(
-                    "type parameter ["
-                        + typeToCanonicalTypeName(typeParameter)
-                        + "] not found "
-                        + "for constructor [["
-                        + targetCanonicalClassName
-                        + "], "
-                        + typesToCanonicalTypeNames(typeParameters)
-                        + "]"
+                throw lookupException(
+                    "type parameter [%s] not found for constructor [[%s], %s]",
+                    typeToCanonicalTypeName(typeParameter),
+                    targetCanonicalClassName,
+                    typesToCanonicalTypeNames(typeParameters)
                 );
             }
 
@@ -452,14 +435,11 @@ public void addPainlessConstructor(Class<?> targetClass, List<Class<?>> typePara
         try {
             javaConstructor = targetClass.getConstructor(javaTypeParameters.toArray(Class<?>[]::new));
         } catch (NoSuchMethodException nsme) {
-            throw new IllegalArgumentException(
-                "reflection object not found for constructor "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "]",
-                nsme
+            throw lookupException(
+                nsme,
+                "reflection object not found for constructor [[%s], %s]",
+                targetCanonicalClassName,
+                typesToCanonicalTypeNames(typeParameters)
             );
         }
 
@@ -468,14 +448,11 @@ public void addPainlessConstructor(Class<?> targetClass, List<Class<?>> typePara
         try {
             methodHandle = lookup(targetClass).unreflectConstructor(javaConstructor);
         } catch (IllegalAccessException iae) {
-            throw new IllegalArgumentException(
-                "method handle not found for constructor "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "]",
-                iae
+            throw lookupException(
+                iae,
+                "method handle not found for constructor [[%s], %s]",
+                targetCanonicalClassName,
+                typesToCanonicalTypeNames(typeParameters)
             );
         }
 
@@ -499,18 +476,12 @@ public void addPainlessConstructor(Class<?> targetClass, List<Class<?>> typePara
             newPainlessConstructor = painlessConstructorCache.computeIfAbsent(newPainlessConstructor, Function.identity());
             painlessClassBuilder.constructors.put(painlessConstructorKey.intern(), newPainlessConstructor);
         } else if (newPainlessConstructor.equals(existingPainlessConstructor) == false) {
-            throw new IllegalArgumentException(
-                "cannot add constructors with the same arity but are not equivalent for constructors "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "] and "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], "
-                    + typesToCanonicalTypeNames(existingPainlessConstructor.typeParameters())
-                    + "]"
+            throw lookupException(
+                "cannot add constructors with the same arity but are not equivalent for constructors [[%s], %s] and [[%s], %s]",
+                targetCanonicalClassName,
+                typesToCanonicalTypeNames(typeParameters),
+                targetCanonicalClassName,
+                typesToCanonicalTypeNames(existingPainlessConstructor.typeParameters())
             );
         }
     }
@@ -535,17 +506,12 @@ public void addPainlessMethod(
         Class<?> targetClass = canonicalClassNamesToClasses.get(targetCanonicalClassName);
 
         if (targetClass == null) {
-            throw new IllegalArgumentException(
-                "target class ["
-                    + targetCanonicalClassName
-                    + "] not found for method "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + canonicalTypeNameParameters
-                    + "]"
+            throw lookupException(
+                "target class [%s] not found for method [[%s], [%s], %s]",
+                targetCanonicalClassName,
+                targetCanonicalClassName,
+                methodName,
+                canonicalTypeNameParameters
             );
         }
 
@@ -555,16 +521,13 @@ public void addPainlessMethod(
             augmentedClass = loadClass(
                 classLoader,
                 augmentedCanonicalClassName,
-                () -> "augmented class ["
-                    + augmentedCanonicalClassName
-                    + "] not found for method "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + canonicalTypeNameParameters
-                    + "]"
+                () -> Strings.format(
+                    "augmented class [%s] not found for method [[%s], [%s], %s]",
+                    augmentedCanonicalClassName,
+                    targetCanonicalClassName,
+                    methodName,
+                    canonicalTypeNameParameters
+                )
             );
         }
 
@@ -574,17 +537,12 @@ public void addPainlessMethod(
             Class<?> typeParameter = canonicalTypeNameToType(canonicalTypeNameParameter);
 
             if (typeParameter == null) {
-                throw new IllegalArgumentException(
-                    "type parameter ["
-                        + canonicalTypeNameParameter
-                        + "] not found for method "
-                        + "[["
-                        + targetCanonicalClassName
-                        + "], ["
-                        + methodName
-                        + "], "
-                        + canonicalTypeNameParameters
-                        + "]"
+                throw lookupException(
+                    "type parameter [%s] not found for method [[%s], [%s], %s]",
+                    canonicalTypeNameParameter,
+                    targetCanonicalClassName,
+                    methodName,
+                    canonicalTypeNameParameters
                 );
             }
 
@@ -594,17 +552,12 @@ public void addPainlessMethod(
         Class<?> returnType = canonicalTypeNameToType(returnCanonicalTypeName);
 
         if (returnType == null) {
-            throw new IllegalArgumentException(
-                "return type ["
-                    + returnCanonicalTypeName
-                    + "] not found for method "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + canonicalTypeNameParameters
-                    + "]"
+            throw lookupException(
+                "return type [%s] not found for method [[%s], [%s], %s]",
+                returnCanonicalTypeName,
+                targetCanonicalClassName,
+                methodName,
+                canonicalTypeNameParameters
             );
         }
 
@@ -641,17 +594,12 @@ public void addPainlessMethod(
         PainlessClassBuilder painlessClassBuilder = classesToPainlessClassBuilders.get(targetClass);
 
         if (painlessClassBuilder == null) {
-            throw new IllegalArgumentException(
-                "target class ["
-                    + targetCanonicalClassName
-                    + "] not found for method "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "]"
+            throw lookupException(
+                "target class [%s] not found for method [[%s], [%s], %s]",
+                targetCanonicalClassName,
+                targetCanonicalClassName,
+                methodName,
+                typesToCanonicalTypeNames(typeParameters)
             );
         }
 
@@ -665,17 +613,12 @@ public void addPainlessMethod(
 
         for (Class<?> typeParameter : typeParameters) {
             if (isValidType(typeParameter) == false) {
-                throw new IllegalArgumentException(
-                    "type parameter ["
-                        + typeToCanonicalTypeName(typeParameter)
-                        + "] "
-                        + "not found for method [["
-                        + targetCanonicalClassName
-                        + "], ["
-                        + methodName
-                        + "], "
-                        + typesToCanonicalTypeNames(typeParameters)
-                        + "]"
+                throw lookupException(
+                    "type parameter [%s] not found for method [[%s], [%s], %s]",
+                    typeToCanonicalTypeName(typeParameter),
+                    targetCanonicalClassName,
+                    methodName,
+                    typesToCanonicalTypeNames(typeParameters)
                 );
             }
 
@@ -683,17 +626,12 @@ public void addPainlessMethod(
         }
 
         if (isValidType(returnType) == false) {
-            throw new IllegalArgumentException(
-                "return type ["
-                    + typeToCanonicalTypeName(returnType)
-                    + "] not found for method "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "]"
+            throw lookupException(
+                "return type [%s] not found for method [[%s], [%s], %s]",
+                typeToCanonicalTypeName(returnType),
+                targetCanonicalClassName,
+                methodName,
+                typesToCanonicalTypeNames(typeParameters)
             );
         }
 
@@ -703,16 +641,12 @@ public void addPainlessMethod(
             try {
                 javaMethod = targetClass.getMethod(methodName, javaTypeParameters.toArray(Class<?>[]::new));
             } catch (NoSuchMethodException nsme) {
-                throw new IllegalArgumentException(
-                    "reflection object not found for method [["
-                        + targetCanonicalClassName
-                        + "], "
-                        + "["
-                        + methodName
-                        + "], "
-                        + typesToCanonicalTypeNames(typeParameters)
-                        + "]",
-                    nsme
+                throw lookupException(
+                    nsme,
+                    "reflection object not found for method [[%s], [%s], %s]",
+                    targetCanonicalClassName,
+                    methodName,
+                    typesToCanonicalTypeNames(typeParameters)
                 );
             }
         } else {
@@ -720,33 +654,22 @@ public void addPainlessMethod(
                 javaMethod = augmentedClass.getMethod(methodName, javaTypeParameters.toArray(Class<?>[]::new));
 
                 if (Modifier.isStatic(javaMethod.getModifiers()) == false) {
-                    throw new IllegalArgumentException(
-                        "method [["
-                            + targetCanonicalClassName
-                            + "], ["
-                            + methodName
-                            + "], "
-                            + typesToCanonicalTypeNames(typeParameters)
-                            + "] with augmented class "
-                            + "["
-                            + typeToCanonicalTypeName(augmentedClass)
-                            + "] must be static"
+                    throw lookupException(
+                        "method [[%s], [%s], %s] with augmented class [%s] must be static",
+                        targetCanonicalClassName,
+                        methodName,
+                        typesToCanonicalTypeNames(typeParameters),
+                        typeToCanonicalTypeName(augmentedClass)
                     );
                 }
             } catch (NoSuchMethodException nsme) {
-                throw new IllegalArgumentException(
-                    "reflection object not found for method "
-                        + "[["
-                        + targetCanonicalClassName
-                        + "], ["
-                        + methodName
-                        + "], "
-                        + typesToCanonicalTypeNames(typeParameters)
-                        + "] "
-                        + "with augmented class ["
-                        + typeToCanonicalTypeName(augmentedClass)
-                        + "]",
-                    nsme
+                throw lookupException(
+                    nsme,
+                    "reflection object not found for method [[%s], [%s], %s] with augmented class [%s]",
+                    targetCanonicalClassName,
+                    methodName,
+                    typesToCanonicalTypeNames(typeParameters),
+                    typeToCanonicalTypeName(augmentedClass)
                 );
             }
         }
@@ -764,20 +687,13 @@ public void addPainlessMethod(
         }
 
         if (javaMethod.getReturnType() != typeToJavaType(returnType)) {
-            throw new IllegalArgumentException(
-                "return type ["
-                    + typeToCanonicalTypeName(javaMethod.getReturnType())
-                    + "] "
-                    + "does not match the specified returned type ["
-                    + typeToCanonicalTypeName(returnType)
-                    + "] "
-                    + "for method [["
-                    + targetClass.getCanonicalName()
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "]"
+            throw lookupException(
+                "return type [%s] does not match the specified returned type [%s] for method [[%s], [%s], %s]",
+                typeToCanonicalTypeName(javaMethod.getReturnType()),
+                typeToCanonicalTypeName(returnType),
+                targetClass.getCanonicalName(),
+                methodName,
+                typesToCanonicalTypeNames(typeParameters)
             );
         }
 
@@ -787,38 +703,26 @@ public void addPainlessMethod(
             try {
                 methodHandle = lookup(targetClass).unreflect(javaMethod);
             } catch (IllegalAccessException iae) {
-                throw new IllegalArgumentException(
-                    "method handle not found for method "
-                        + "[["
-                        + targetClass.getCanonicalName()
-                        + "], ["
-                        + methodName
-                        + "], "
-                        + typesToCanonicalTypeNames(typeParameters)
-                        + "], "
-                        + "with lookup ["
-                        + lookup(targetClass)
-                        + "]",
-                    iae
+                throw lookupException(
+                    iae,
+                    "method handle not found for method [[%s], [%s], %s], with lookup [%s]",
+                    targetClass.getCanonicalName(),
+                    methodName,
+                    typesToCanonicalTypeNames(typeParameters),
+                    lookup(targetClass)
                 );
             }
         } else {
             try {
                 methodHandle = lookup(augmentedClass).unreflect(javaMethod);
             } catch (IllegalAccessException iae) {
-                throw new IllegalArgumentException(
-                    "method handle not found for method "
-                        + "[["
-                        + targetClass.getCanonicalName()
-                        + "], ["
-                        + methodName
-                        + "], "
-                        + typesToCanonicalTypeNames(typeParameters)
-                        + "]"
-                        + "with augmented class ["
-                        + typeToCanonicalTypeName(augmentedClass)
-                        + "]",
-                    iae
+                throw lookupException(
+                    iae,
+                    "method handle not found for method [[%s], [%s], %s] with augmented class [%s]",
+                    targetClass.getCanonicalName(),
+                    methodName,
+                    typesToCanonicalTypeNames(typeParameters),
+                    typeToCanonicalTypeName(augmentedClass)
                 );
             }
         }
@@ -852,28 +756,17 @@ public void addPainlessMethod(
                 painlessClassBuilder.methods.put(painlessMethodKey.intern(), newPainlessMethod);
             }
         } else if (newPainlessMethod.equals(existingPainlessMethod) == false) {
-            throw new IllegalArgumentException(
+            throw lookupException(
                 "cannot add methods with the same name and arity but are not equivalent for methods "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + "["
-                    + typeToCanonicalTypeName(returnType)
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "] and "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + "["
-                    + typeToCanonicalTypeName(existingPainlessMethod.returnType())
-                    + "], "
-                    + typesToCanonicalTypeNames(existingPainlessMethod.typeParameters())
-                    + "]"
+                    + "[[%s], [%s], [%s], %s] and [[%s], [%s], [%s], %s]",
+                targetCanonicalClassName,
+                methodName,
+                typeToCanonicalTypeName(returnType),
+                typesToCanonicalTypeNames(typeParameters),
+                targetCanonicalClassName,
+                methodName,
+                typeToCanonicalTypeName(existingPainlessMethod.returnType()),
+                typesToCanonicalTypeNames(existingPainlessMethod.typeParameters())
             );
         }
     }
@@ -895,17 +788,12 @@ public void addPainlessField(
         Class<?> targetClass = canonicalClassNamesToClasses.get(targetCanonicalClassName);
 
         if (targetClass == null) {
-            throw new IllegalArgumentException(
-                "target class ["
-                    + targetCanonicalClassName
-                    + "] not found for field "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + fieldName
-                    + "], ["
-                    + canonicalTypeNameParameter
-                    + "]]"
+            throw lookupException(
+                "target class [%s] not found for field [[%s], [%s], [%s]]",
+                targetCanonicalClassName,
+                targetCanonicalClassName,
+                fieldName,
+                canonicalTypeNameParameter
             );
         }
 
@@ -919,29 +807,23 @@ public void addPainlessField(
             augmentedClass = loadClass(
                 classLoader,
                 augmentedCanonicalClassName,
-                () -> "augmented class ["
-                    + augmentedCanonicalClassName
-                    + "] not found for field "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + fieldName
-                    + "]"
+                () -> Strings.format(
+                    "augmented class [%s] not found for field [[%s], [%s]]",
+                    augmentedCanonicalClassName,
+                    targetCanonicalClassName,
+                    fieldName
+                )
             );
         }
 
         Class<?> typeParameter = canonicalTypeNameToType(canonicalTypeNameParameter);
 
         if (typeParameter == null) {
-            throw new IllegalArgumentException(
-                "type parameter ["
-                    + canonicalTypeNameParameter
-                    + "] not found "
-                    + "for field [["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + fieldName
-                    + "]"
+            throw lookupException(
+                "type parameter [%s] not found for field [[%s], [%s]]",
+                canonicalTypeNameParameter,
+                targetCanonicalClassName,
+                fieldName
             );
         }
 
@@ -976,32 +858,22 @@ public void addPainlessField(
         PainlessClassBuilder painlessClassBuilder = classesToPainlessClassBuilders.get(targetClass);
 
         if (painlessClassBuilder == null) {
-            throw new IllegalArgumentException(
-                "target class ["
-                    + targetCanonicalClassName
-                    + "] not found for field "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + fieldName
-                    + "], ["
-                    + typeToCanonicalTypeName(typeParameter)
-                    + "]]"
+            throw lookupException(
+                "target class [%s] not found for field [[%s], [%s], [%s]]",
+                targetCanonicalClassName,
+                targetCanonicalClassName,
+                fieldName,
+                typeToCanonicalTypeName(typeParameter)
             );
         }
 
         if (isValidType(typeParameter) == false) {
-            throw new IllegalArgumentException(
-                "type parameter ["
-                    + typeToCanonicalTypeName(typeParameter)
-                    + "] not found for field "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + fieldName
-                    + "], ["
-                    + typeToCanonicalTypeName(typeParameter)
-                    + "]]"
+            throw lookupException(
+                "type parameter [%s] not found for field [[%s], [%s], [%s]]",
+                typeToCanonicalTypeName(typeParameter),
+                targetCanonicalClassName,
+                fieldName,
+                typeToCanonicalTypeName(typeParameter)
             );
         }
 
@@ -1011,16 +883,12 @@ public void addPainlessField(
             try {
                 javaField = targetClass.getField(fieldName);
             } catch (NoSuchFieldException nsfe) {
-                throw new IllegalArgumentException(
-                    "reflection object not found for field "
-                        + "[["
-                        + targetCanonicalClassName
-                        + "], ["
-                        + fieldName
-                        + "], ["
-                        + typeToCanonicalTypeName(typeParameter)
-                        + "]]",
-                    nsfe
+                throw lookupException(
+                    nsfe,
+                    "reflection object not found for field [[%s], [%s], [%s]]",
+                    targetCanonicalClassName,
+                    fieldName,
+                    typeToCanonicalTypeName(typeParameter)
                 );
             }
         } else {
@@ -1028,48 +896,32 @@ public void addPainlessField(
                 javaField = augmentedClass.getField(fieldName);
 
                 if (Modifier.isStatic(javaField.getModifiers()) == false || Modifier.isFinal(javaField.getModifiers()) == false) {
-                    throw new IllegalArgumentException(
-                        "field [["
-                            + targetCanonicalClassName
-                            + "], ["
-                            + fieldName
-                            + "] "
-                            + "with augmented class ["
-                            + typeToCanonicalTypeName(augmentedClass)
-                            + "] must be static and final"
+                    throw lookupException(
+                        "field [[%s], [%s]] with augmented class [%s] must be static and final",
+                        targetCanonicalClassName,
+                        fieldName,
+                        typeToCanonicalTypeName(augmentedClass)
                     );
                 }
             } catch (NoSuchFieldException nsfe) {
-                throw new IllegalArgumentException(
-                    "reflection object not found for field "
-                        + "[["
-                        + targetCanonicalClassName
-                        + "], ["
-                        + fieldName
-                        + "], ["
-                        + typeToCanonicalTypeName(typeParameter)
-                        + "]]"
-                        + "with augmented class ["
-                        + typeToCanonicalTypeName(augmentedClass)
-                        + "]",
-                    nsfe
+                throw lookupException(
+                    nsfe,
+                    "reflection object not found for field [[%s], [%s], [%s]] with augmented class [%s]",
+                    targetCanonicalClassName,
+                    fieldName,
+                    typeToCanonicalTypeName(typeParameter),
+                    typeToCanonicalTypeName(augmentedClass)
                 );
             }
         }
 
         if (javaField.getType() != typeToJavaType(typeParameter)) {
-            throw new IllegalArgumentException(
-                "type parameter ["
-                    + typeToCanonicalTypeName(javaField.getType())
-                    + "] "
-                    + "does not match the specified type parameter ["
-                    + typeToCanonicalTypeName(typeParameter)
-                    + "] "
-                    + "for field [["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + fieldName
-                    + "]"
+            throw lookupException(
+                "type parameter [%s] does not match the specified type parameter [%s] for field [[%s], [%s]]",
+                typeToCanonicalTypeName(javaField.getType()),
+                typeToCanonicalTypeName(typeParameter),
+                targetCanonicalClassName,
+                fieldName
             );
         }
 
@@ -1094,26 +946,18 @@ public void addPainlessField(
             PainlessField newPainlessField = new PainlessField(javaField, typeParameter, annotations, methodHandleGetter, null);
 
             if (existingPainlessField == null) {
-                newPainlessField = painlessFieldCache.computeIfAbsent(newPainlessField, key -> key);
+                newPainlessField = painlessFieldCache.computeIfAbsent(newPainlessField, Function.identity());
                 painlessClassBuilder.staticFields.put(painlessFieldKey.intern(), newPainlessField);
             } else if (newPainlessField.equals(existingPainlessField) == false) {
-                throw new IllegalArgumentException(
-                    "cannot add fields with the same name but are not equivalent for fields "
-                        + "[["
-                        + targetCanonicalClassName
-                        + "], ["
-                        + fieldName
-                        + "], ["
-                        + typeToCanonicalTypeName(typeParameter)
-                        + "] and "
-                        + "[["
-                        + targetCanonicalClassName
-                        + "], ["
-                        + existingPainlessField.javaField().getName()
-                        + "], "
-                        + typeToCanonicalTypeName(existingPainlessField.typeParameter())
-                        + "] "
-                        + "with the same name and different type parameters"
+                throw lookupException(
+                    "cannot add fields with the same name but are not equivalent for fields [[%s], [%s], [%s]] and [[%s], [%s], [%s]]"
+                        + " with the same name and different type parameters",
+                    targetCanonicalClassName,
+                    fieldName,
+                    typeToCanonicalTypeName(typeParameter),
+                    targetCanonicalClassName,
+                    existingPainlessField.javaField().getName(),
+                    typeToCanonicalTypeName(existingPainlessField.typeParameter())
                 );
             }
         } else {
@@ -1140,23 +984,15 @@ public void addPainlessField(
                 newPainlessField = painlessFieldCache.computeIfAbsent(newPainlessField, key -> key);
                 painlessClassBuilder.fields.put(painlessFieldKey.intern(), newPainlessField);
             } else if (newPainlessField.equals(existingPainlessField) == false) {
-                throw new IllegalArgumentException(
-                    "cannot add fields with the same name but are not equivalent for fields "
-                        + "[["
-                        + targetCanonicalClassName
-                        + "], ["
-                        + fieldName
-                        + "], ["
-                        + typeToCanonicalTypeName(typeParameter)
-                        + "] and "
-                        + "[["
-                        + targetCanonicalClassName
-                        + "], ["
-                        + existingPainlessField.javaField().getName()
-                        + "], "
-                        + typeToCanonicalTypeName(existingPainlessField.typeParameter())
-                        + "] "
-                        + "with the same name and different type parameters"
+                throw lookupException(
+                    "cannot add fields with the same name but are not equivalent for fields [[%s], [%s], [%s]] and [[%s], [%s], [%s]]"
+                        + " with the same name and different type parameters",
+                    targetCanonicalClassName,
+                    fieldName,
+                    typeToCanonicalTypeName(typeParameter),
+                    targetCanonicalClassName,
+                    existingPainlessField.javaField().getName(),
+                    typeToCanonicalTypeName(existingPainlessField.typeParameter())
                 );
             }
         }
@@ -1180,38 +1016,18 @@ public void addImportedPainlessMethod(
         Class<?> targetClass = loadClass(classLoader, targetJavaClassName, () -> "class [" + targetJavaClassName + "] not found");
         String targetCanonicalClassName = typeToCanonicalTypeName(targetClass);
 
-        if (targetClass == null) {
-            throw new IllegalArgumentException(
-                "target class ["
-                    + targetCanonicalClassName
-                    + "] not found for imported method "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + canonicalTypeNameParameters
-                    + "]"
-            );
-        }
-
         List<Class<?>> typeParameters = new ArrayList<>(canonicalTypeNameParameters.size());
 
         for (String canonicalTypeNameParameter : canonicalTypeNameParameters) {
             Class<?> typeParameter = canonicalTypeNameToType(canonicalTypeNameParameter);
 
             if (typeParameter == null) {
-                throw new IllegalArgumentException(
-                    "type parameter ["
-                        + canonicalTypeNameParameter
-                        + "] not found for imported method "
-                        + "[["
-                        + targetCanonicalClassName
-                        + "], ["
-                        + methodName
-                        + "], "
-                        + canonicalTypeNameParameters
-                        + "]"
+                throw lookupException(
+                    "type parameter [%s] not found for imported method [[%s], [%s], %s]",
+                    canonicalTypeNameParameter,
+                    targetCanonicalClassName,
+                    methodName,
+                    canonicalTypeNameParameters
                 );
             }
 
@@ -1221,17 +1037,12 @@ public void addImportedPainlessMethod(
         Class<?> returnType = canonicalTypeNameToType(returnCanonicalTypeName);
 
         if (returnType == null) {
-            throw new IllegalArgumentException(
-                "return type ["
-                    + returnCanonicalTypeName
-                    + "] not found for imported method "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + canonicalTypeNameParameters
-                    + "]"
+            throw lookupException(
+                "return type [%s] not found for imported method [[%s], [%s], %s]",
+                returnCanonicalTypeName,
+                targetCanonicalClassName,
+                methodName,
+                canonicalTypeNameParameters
             );
         }
 
@@ -1260,11 +1071,9 @@ public void addImportedPainlessMethod(
         if (existingTargetClass == null) {
             javaClassNamesToClasses.put(targetClass.getName().intern(), targetClass);
         } else if (existingTargetClass != targetClass) {
-            throw new IllegalArgumentException(
-                "class ["
-                    + targetCanonicalClassName
-                    + "] "
-                    + "cannot represent multiple java classes with the same name from different class loaders"
+            throw lookupException(
+                "class [%s] cannot represent multiple java classes with the same name from different class loaders",
+                targetCanonicalClassName
             );
         }
 
@@ -1279,17 +1088,12 @@ public void addImportedPainlessMethod(
 
         for (Class<?> typeParameter : typeParameters) {
             if (isValidType(typeParameter) == false) {
-                throw new IllegalArgumentException(
-                    "type parameter ["
-                        + typeToCanonicalTypeName(typeParameter)
-                        + "] "
-                        + "not found for imported method [["
-                        + targetCanonicalClassName
-                        + "], ["
-                        + methodName
-                        + "], "
-                        + typesToCanonicalTypeNames(typeParameters)
-                        + "]"
+                throw lookupException(
+                    "type parameter [%s] not found for imported method [[%s], [%s], %s]",
+                    typeToCanonicalTypeName(typeParameter),
+                    targetCanonicalClassName,
+                    methodName,
+                    typesToCanonicalTypeNames(typeParameters)
                 );
             }
 
@@ -1297,17 +1101,12 @@ public void addImportedPainlessMethod(
         }
 
         if (isValidType(returnType) == false) {
-            throw new IllegalArgumentException(
-                "return type ["
-                    + typeToCanonicalTypeName(returnType)
-                    + "] not found for imported method "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "]"
+            throw lookupException(
+                "return type [%s] not found for imported method [[%s], [%s], %s]",
+                typeToCanonicalTypeName(returnType),
+                targetCanonicalClassName,
+                methodName,
+                typesToCanonicalTypeNames(typeParameters)
             );
         }
 
@@ -1316,46 +1115,32 @@ public void addImportedPainlessMethod(
         try {
             javaMethod = targetClass.getMethod(methodName, javaTypeParameters.toArray(new Class<?>[typeParametersSize]));
         } catch (NoSuchMethodException nsme) {
-            throw new IllegalArgumentException(
-                "imported method reflection object [["
-                    + targetCanonicalClassName
-                    + "], "
-                    + "["
-                    + methodName
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "] not found",
-                nsme
+            throw lookupException(
+                nsme,
+                "imported method reflection object [[%s], [%s], %s] not found",
+                targetCanonicalClassName,
+                methodName,
+                typesToCanonicalTypeNames(typeParameters)
             );
         }
 
         if (javaMethod.getReturnType() != typeToJavaType(returnType)) {
-            throw new IllegalArgumentException(
-                "return type ["
-                    + typeToCanonicalTypeName(javaMethod.getReturnType())
-                    + "] "
-                    + "does not match the specified returned type ["
-                    + typeToCanonicalTypeName(returnType)
-                    + "] "
-                    + "for imported method [["
-                    + targetClass.getCanonicalName()
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "]"
+            throw lookupException(
+                "return type [%s] does not match the specified returned type [%s] for imported method [[%s], [%s], %s]",
+                typeToCanonicalTypeName(javaMethod.getReturnType()),
+                typeToCanonicalTypeName(returnType),
+                targetClass.getCanonicalName(),
+                methodName,
+                typesToCanonicalTypeNames(typeParameters)
             );
         }
 
         if (Modifier.isStatic(javaMethod.getModifiers()) == false) {
-            throw new IllegalArgumentException(
-                "imported method [["
-                    + targetClass.getCanonicalName()
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "] must be static"
+            throw lookupException(
+                "imported method [[%s], [%s], %s] must be static",
+                targetClass.getCanonicalName(),
+                methodName,
+                typesToCanonicalTypeNames(typeParameters)
             );
         }
 
@@ -1374,16 +1159,12 @@ public void addImportedPainlessMethod(
         try {
             methodHandle = lookup(targetClass).unreflect(javaMethod);
         } catch (IllegalAccessException iae) {
-            throw new IllegalArgumentException(
-                "imported method handle [["
-                    + targetClass.getCanonicalName()
-                    + "], "
-                    + "["
-                    + methodName
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "] not found",
-                iae
+            throw lookupException(
+                iae,
+                "imported method handle [[%s], [%s], %s] not found",
+                targetClass.getCanonicalName(),
+                methodName,
+                typesToCanonicalTypeNames(typeParameters)
             );
         }
 
@@ -1404,29 +1185,17 @@ public void addImportedPainlessMethod(
             newImportedPainlessMethod = painlessMethodCache.computeIfAbsent(newImportedPainlessMethod, key -> key);
             painlessMethodKeysToImportedPainlessMethods.put(painlessMethodKey.intern(), newImportedPainlessMethod);
         } else if (newImportedPainlessMethod.equals(existingImportedPainlessMethod) == false) {
-            throw new IllegalArgumentException(
-                "cannot add imported methods with the same name and arity "
-                    + "but do not have equivalent methods "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + "["
-                    + typeToCanonicalTypeName(returnType)
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "] and "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + "["
-                    + typeToCanonicalTypeName(existingImportedPainlessMethod.returnType())
-                    + "], "
-                    + typesToCanonicalTypeNames(existingImportedPainlessMethod.typeParameters())
-                    + "]"
+            throw lookupException(
+                "cannot add imported methods with the same name and arity but do not have equivalent methods "
+                    + "[[%s], [%s], [%s], %s] and [[%s], [%s], [%s], %s]",
+                targetCanonicalClassName,
+                methodName,
+                typeToCanonicalTypeName(returnType),
+                typesToCanonicalTypeNames(typeParameters),
+                targetCanonicalClassName,
+                methodName,
+                typeToCanonicalTypeName(existingImportedPainlessMethod.returnType()),
+                typesToCanonicalTypeNames(existingImportedPainlessMethod.typeParameters())
             );
         }
     }
@@ -1454,17 +1223,12 @@ public void addPainlessClassBinding(
             Class<?> typeParameter = canonicalTypeNameToType(canonicalTypeNameParameter);
 
             if (typeParameter == null) {
-                throw new IllegalArgumentException(
-                    "type parameter ["
-                        + canonicalTypeNameParameter
-                        + "] not found for class binding "
-                        + "[["
-                        + targetCanonicalClassName
-                        + "], ["
-                        + methodName
-                        + "], "
-                        + canonicalTypeNameParameters
-                        + "]"
+                throw lookupException(
+                    "type parameter [%s] not found for class binding [[%s], [%s], %s]",
+                    canonicalTypeNameParameter,
+                    targetCanonicalClassName,
+                    methodName,
+                    canonicalTypeNameParameters
                 );
             }
 
@@ -1474,17 +1238,12 @@ public void addPainlessClassBinding(
         Class<?> returnType = canonicalTypeNameToType(returnCanonicalTypeName);
 
         if (returnType == null) {
-            throw new IllegalArgumentException(
-                "return type ["
-                    + returnCanonicalTypeName
-                    + "] not found for class binding "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + canonicalTypeNameParameters
-                    + "]"
+            throw lookupException(
+                "return type [%s] not found for class binding [[%s], [%s], %s]",
+                returnCanonicalTypeName,
+                targetCanonicalClassName,
+                methodName,
+                canonicalTypeNameParameters
             );
         }
 
@@ -1513,11 +1272,9 @@ public void addPainlessClassBinding(
         if (existingTargetClass == null) {
             javaClassNamesToClasses.put(targetClass.getName().intern(), targetClass);
         } else if (existingTargetClass != targetClass) {
-            throw new IllegalArgumentException(
-                "class ["
-                    + targetCanonicalClassName
-                    + "] "
-                    + "cannot represent multiple java classes with the same name from different class loaders"
+            throw lookupException(
+                "class [%s] cannot represent multiple java classes with the same name from different class loaders",
+                targetCanonicalClassName
             );
         }
 
@@ -1546,46 +1303,32 @@ public void addPainlessClassBinding(
             Class<?> typeParameter = typeParameters.get(typeParameterIndex);
 
             if (isValidType(typeParameter) == false) {
-                throw new IllegalArgumentException(
-                    "type parameter ["
-                        + typeToCanonicalTypeName(typeParameter)
-                        + "] not found "
-                        + "for class binding [["
-                        + targetCanonicalClassName
-                        + "], "
-                        + typesToCanonicalTypeNames(typeParameters)
-                        + "]"
+                throw lookupException(
+                    "type parameter [%s] not found for class binding [[%s], %s]",
+                    typeToCanonicalTypeName(typeParameter),
+                    targetCanonicalClassName,
+                    typesToCanonicalTypeNames(typeParameters)
                 );
             }
 
             Class<?> javaTypeParameter = constructorParameterTypes[typeParameterIndex];
 
             if (isValidType(javaTypeParameter) == false) {
-                throw new IllegalArgumentException(
-                    "type parameter ["
-                        + typeToCanonicalTypeName(typeParameter)
-                        + "] not found "
-                        + "for class binding [["
-                        + targetCanonicalClassName
-                        + "], "
-                        + typesToCanonicalTypeNames(typeParameters)
-                        + "]"
+                throw lookupException(
+                    "type parameter [%s] not found for class binding [[%s], %s]",
+                    typeToCanonicalTypeName(typeParameter),
+                    targetCanonicalClassName,
+                    typesToCanonicalTypeNames(typeParameters)
                 );
             }
 
             if (javaTypeParameter != typeToJavaType(typeParameter)) {
-                throw new IllegalArgumentException(
-                    "type parameter ["
-                        + typeToCanonicalTypeName(javaTypeParameter)
-                        + "] "
-                        + "does not match the specified type parameter ["
-                        + typeToCanonicalTypeName(typeParameter)
-                        + "] "
-                        + "for class binding [["
-                        + targetClass.getCanonicalName()
-                        + "], "
-                        + typesToCanonicalTypeNames(typeParameters)
-                        + "]"
+                throw lookupException(
+                    "type parameter [%s] does not match the specified type parameter [%s] for class binding [[%s], %s]",
+                    typeToCanonicalTypeName(javaTypeParameter),
+                    typeToCanonicalTypeName(typeParameter),
+                    targetClass.getCanonicalName(),
+                    typesToCanonicalTypeNames(typeParameters)
                 );
             }
         }
@@ -1623,80 +1366,54 @@ public void addPainlessClassBinding(
             Class<?> typeParameter = typeParameters.get(constructorParameterTypes.length + typeParameterIndex);
 
             if (isValidType(typeParameter) == false) {
-                throw new IllegalArgumentException(
-                    "type parameter ["
-                        + typeToCanonicalTypeName(typeParameter)
-                        + "] not found "
-                        + "for class binding [["
-                        + targetCanonicalClassName
-                        + "], "
-                        + typesToCanonicalTypeNames(typeParameters)
-                        + "]"
+                throw lookupException(
+                    "type parameter [%s] not found for class binding [[%s], %s]",
+                    typeToCanonicalTypeName(typeParameter),
+                    targetCanonicalClassName,
+                    typesToCanonicalTypeNames(typeParameters)
                 );
             }
 
             Class<?> javaTypeParameter = javaMethod.getParameterTypes()[typeParameterIndex];
 
             if (isValidType(javaTypeParameter) == false) {
-                throw new IllegalArgumentException(
-                    "type parameter ["
-                        + typeToCanonicalTypeName(typeParameter)
-                        + "] not found "
-                        + "for class binding [["
-                        + targetCanonicalClassName
-                        + "], "
-                        + typesToCanonicalTypeNames(typeParameters)
-                        + "]"
+                throw lookupException(
+                    "type parameter [%s] not found for class binding [[%s], %s]",
+                    typeToCanonicalTypeName(typeParameter),
+                    targetCanonicalClassName,
+                    typesToCanonicalTypeNames(typeParameters)
                 );
             }
 
             if (javaTypeParameter != typeToJavaType(typeParameter)) {
-                throw new IllegalArgumentException(
-                    "type parameter ["
-                        + typeToCanonicalTypeName(javaTypeParameter)
-                        + "] "
-                        + "does not match the specified type parameter ["
-                        + typeToCanonicalTypeName(typeParameter)
-                        + "] "
-                        + "for class binding [["
-                        + targetClass.getCanonicalName()
-                        + "], "
-                        + typesToCanonicalTypeNames(typeParameters)
-                        + "]"
+                throw lookupException(
+                    "type parameter [%s] does not match the specified type parameter [%s] for class binding [[%s], %s]",
+                    typeToCanonicalTypeName(javaTypeParameter),
+                    typeToCanonicalTypeName(typeParameter),
+                    targetClass.getCanonicalName(),
+                    typesToCanonicalTypeNames(typeParameters)
                 );
             }
         }
 
         if (isValidType(returnType) == false) {
-            throw new IllegalArgumentException(
-                "return type ["
-                    + typeToCanonicalTypeName(returnType)
-                    + "] not found for class binding "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "]"
+            throw lookupException(
+                "return type [%s] not found for class binding [[%s], [%s], %s]",
+                typeToCanonicalTypeName(returnType),
+                targetCanonicalClassName,
+                methodName,
+                typesToCanonicalTypeNames(typeParameters)
             );
         }
 
         if (javaMethod.getReturnType() != typeToJavaType(returnType)) {
-            throw new IllegalArgumentException(
-                "return type ["
-                    + typeToCanonicalTypeName(javaMethod.getReturnType())
-                    + "] "
-                    + "does not match the specified returned type ["
-                    + typeToCanonicalTypeName(returnType)
-                    + "] "
-                    + "for class binding [["
-                    + targetClass.getCanonicalName()
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "]"
+            throw lookupException(
+                "return type [%s] does not match the specified returned type [%s] for class binding [[%s], [%s], %s]",
+                typeToCanonicalTypeName(javaMethod.getReturnType()),
+                typeToCanonicalTypeName(returnType),
+                targetClass.getCanonicalName(),
+                methodName,
+                typesToCanonicalTypeNames(typeParameters)
             );
         }
 
@@ -1711,14 +1428,11 @@ public void addPainlessClassBinding(
         }
 
         if (Modifier.isStatic(javaMethod.getModifiers())) {
-            throw new IllegalArgumentException(
-                "class binding [["
-                    + targetClass.getCanonicalName()
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "] cannot be static"
+            throw lookupException(
+                "class binding [[%s], [%s], %s] cannot be static",
+                targetClass.getCanonicalName(),
+                methodName,
+                typesToCanonicalTypeNames(typeParameters)
             );
         }
 
@@ -1735,31 +1449,17 @@ public void addPainlessClassBinding(
             newPainlessClassBinding = painlessClassBindingCache.computeIfAbsent(newPainlessClassBinding, Function.identity());
             painlessMethodKeysToPainlessClassBindings.put(painlessMethodKey.intern(), newPainlessClassBinding);
         } else if (newPainlessClassBinding.equals(existingPainlessClassBinding) == false) {
-            throw new IllegalArgumentException(
-                "cannot add class bindings with the same name and arity "
-                    + "but do not have equivalent methods "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], "
-                    + "["
-                    + methodName
-                    + "], "
-                    + "["
-                    + typeToCanonicalTypeName(returnType)
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "] and "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], "
-                    + "["
-                    + methodName
-                    + "], "
-                    + "["
-                    + typeToCanonicalTypeName(existingPainlessClassBinding.returnType())
-                    + "], "
-                    + typesToCanonicalTypeNames(existingPainlessClassBinding.typeParameters())
-                    + "]"
+            throw lookupException(
+                "cannot add class bindings with the same name and arity but do not have equivalent methods "
+                    + "[[%s], [%s], [%s], %s] and [[%s], [%s], [%s], %s]",
+                targetCanonicalClassName,
+                methodName,
+                typeToCanonicalTypeName(returnType),
+                typesToCanonicalTypeNames(typeParameters),
+                targetCanonicalClassName,
+                methodName,
+                typeToCanonicalTypeName(existingPainlessClassBinding.returnType()),
+                typesToCanonicalTypeNames(existingPainlessClassBinding.typeParameters())
             );
         }
     }
@@ -1785,17 +1485,12 @@ public void addPainlessInstanceBinding(
             Class<?> typeParameter = canonicalTypeNameToType(canonicalTypeNameParameter);
 
             if (typeParameter == null) {
-                throw new IllegalArgumentException(
-                    "type parameter ["
-                        + canonicalTypeNameParameter
-                        + "] not found for instance binding "
-                        + "[["
-                        + targetCanonicalClassName
-                        + "], ["
-                        + methodName
-                        + "], "
-                        + canonicalTypeNameParameters
-                        + "]"
+                throw lookupException(
+                    "type parameter [%s] not found for instance binding [[%s], [%s], %s]",
+                    canonicalTypeNameParameter,
+                    targetCanonicalClassName,
+                    methodName,
+                    canonicalTypeNameParameters
                 );
             }
 
@@ -1805,17 +1500,12 @@ public void addPainlessInstanceBinding(
         Class<?> returnType = canonicalTypeNameToType(returnCanonicalTypeName);
 
         if (returnType == null) {
-            throw new IllegalArgumentException(
-                "return type ["
-                    + returnCanonicalTypeName
-                    + "] not found for class binding "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + canonicalTypeNameParameters
-                    + "]"
+            throw lookupException(
+                "return type [%s] not found for class binding [[%s], [%s], %s]",
+                returnCanonicalTypeName,
+                targetCanonicalClassName,
+                methodName,
+                canonicalTypeNameParameters
             );
         }
 
@@ -1846,11 +1536,9 @@ public void addPainlessInstanceBinding(
         if (existingTargetClass == null) {
             javaClassNamesToClasses.put(targetClass.getName().intern(), targetClass);
         } else if (existingTargetClass != targetClass) {
-            throw new IllegalArgumentException(
-                "class ["
-                    + targetCanonicalClassName
-                    + "] "
-                    + "cannot represent multiple java classes with the same name from different class loaders"
+            throw lookupException(
+                "class [%s] cannot represent multiple java classes with the same name from different class loaders",
+                targetCanonicalClassName
             );
         }
 
@@ -1865,17 +1553,12 @@ public void addPainlessInstanceBinding(
 
         for (Class<?> typeParameter : typeParameters) {
             if (isValidType(typeParameter) == false) {
-                throw new IllegalArgumentException(
-                    "type parameter ["
-                        + typeToCanonicalTypeName(typeParameter)
-                        + "] "
-                        + "not found for instance binding [["
-                        + targetCanonicalClassName
-                        + "], ["
-                        + methodName
-                        + "], "
-                        + typesToCanonicalTypeNames(typeParameters)
-                        + "]"
+                throw lookupException(
+                    "type parameter [%s] not found for instance binding [[%s], [%s], %s]",
+                    typeToCanonicalTypeName(typeParameter),
+                    targetCanonicalClassName,
+                    methodName,
+                    typesToCanonicalTypeNames(typeParameters)
                 );
             }
 
@@ -1883,17 +1566,12 @@ public void addPainlessInstanceBinding(
         }
 
         if (isValidType(returnType) == false) {
-            throw new IllegalArgumentException(
-                "return type ["
-                    + typeToCanonicalTypeName(returnType)
-                    + "] not found for imported method "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "]"
+            throw lookupException(
+                "return type [%s] not found for imported method [[%s], [%s], %s]",
+                typeToCanonicalTypeName(returnType),
+                targetCanonicalClassName,
+                methodName,
+                typesToCanonicalTypeNames(typeParameters)
             );
         }
 
@@ -1902,46 +1580,32 @@ public void addPainlessInstanceBinding(
         try {
             javaMethod = targetClass.getMethod(methodName, javaTypeParameters.toArray(new Class<?>[typeParametersSize]));
         } catch (NoSuchMethodException nsme) {
-            throw new IllegalArgumentException(
-                "instance binding reflection object [["
-                    + targetCanonicalClassName
-                    + "], "
-                    + "["
-                    + methodName
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "] not found",
-                nsme
+            throw lookupException(
+                nsme,
+                "instance binding reflection object [[%s], [%s], %s] not found",
+                targetCanonicalClassName,
+                methodName,
+                typesToCanonicalTypeNames(typeParameters)
             );
         }
 
         if (javaMethod.getReturnType() != typeToJavaType(returnType)) {
-            throw new IllegalArgumentException(
-                "return type ["
-                    + typeToCanonicalTypeName(javaMethod.getReturnType())
-                    + "] "
-                    + "does not match the specified returned type ["
-                    + typeToCanonicalTypeName(returnType)
-                    + "] "
-                    + "for instance binding [["
-                    + targetClass.getCanonicalName()
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "]"
+            throw lookupException(
+                "return type [%s] does not match the specified returned type [%s] for instance binding [[%s], [%s], %s]",
+                typeToCanonicalTypeName(javaMethod.getReturnType()),
+                typeToCanonicalTypeName(returnType),
+                targetClass.getCanonicalName(),
+                methodName,
+                typesToCanonicalTypeNames(typeParameters)
             );
         }
 
         if (Modifier.isStatic(javaMethod.getModifiers())) {
-            throw new IllegalArgumentException(
-                "instance binding [["
-                    + targetClass.getCanonicalName()
-                    + "], ["
-                    + methodName
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "] cannot be static"
+            throw lookupException(
+                "instance binding [[%s], [%s], %s] cannot be static",
+                targetClass.getCanonicalName(),
+                methodName,
+                typesToCanonicalTypeNames(typeParameters)
             );
         }
 
@@ -1968,34 +1632,19 @@ public void addPainlessInstanceBinding(
             newPainlessInstanceBinding = painlessInstanceBindingCache.computeIfAbsent(newPainlessInstanceBinding, key -> key);
             painlessMethodKeysToPainlessInstanceBindings.put(painlessMethodKey.intern(), newPainlessInstanceBinding);
         } else if (newPainlessInstanceBinding.equals(existingPainlessInstanceBinding) == false) {
-            throw new IllegalArgumentException(
-                "cannot add instances bindings with the same name and arity "
-                    + "but do not have equivalent methods "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], "
-                    + "["
-                    + methodName
-                    + "], "
-                    + "["
-                    + typeToCanonicalTypeName(returnType)
-                    + "], "
-                    + typesToCanonicalTypeNames(typeParameters)
-                    + "], "
-                    + painlessAnnotations
-                    + " and "
-                    + "[["
-                    + targetCanonicalClassName
-                    + "], "
-                    + "["
-                    + methodName
-                    + "], "
-                    + "["
-                    + typeToCanonicalTypeName(existingPainlessInstanceBinding.returnType())
-                    + "], "
-                    + typesToCanonicalTypeNames(existingPainlessInstanceBinding.typeParameters())
-                    + "], "
-                    + existingPainlessInstanceBinding.annotations()
+            throw lookupException(
+                "cannot add instances bindings with the same name and arity but do not have equivalent methods "
+                    + "[[%s], [%s], [%s], %s], %s and [[%s], [%s], [%s], %s], %s",
+                targetCanonicalClassName,
+                methodName,
+                typeToCanonicalTypeName(returnType),
+                typesToCanonicalTypeNames(typeParameters),
+                painlessAnnotations,
+                targetCanonicalClassName,
+                methodName,
+                typeToCanonicalTypeName(existingPainlessInstanceBinding.returnType()),
+                typesToCanonicalTypeNames(existingPainlessInstanceBinding.typeParameters()),
+                existingPainlessInstanceBinding.annotations()
             );
         }
     }
@@ -2014,20 +1663,20 @@ public PainlessLookup build() {
 
         if (javaClassNamesToClasses.values().containsAll(canonicalClassNamesToClasses.values()) == false) {
             throw new IllegalArgumentException(
-                "the values of java class names to classes " + "must be a superset of the values of canonical class names to classes"
+                "the values of java class names to classes must be a superset of the values of canonical class names to classes"
             );
         }
 
         if (javaClassNamesToClasses.values().containsAll(classesToPainlessClasses.keySet()) == false) {
             throw new IllegalArgumentException(
-                "the values of java class names to classes " + "must be a superset of the keys of classes to painless classes"
+                "the values of java class names to classes must be a superset of the keys of classes to painless classes"
             );
         }
 
         if (canonicalClassNamesToClasses.values().containsAll(classesToPainlessClasses.keySet()) == false
             || classesToPainlessClasses.keySet().containsAll(canonicalClassNamesToClasses.values()) == false) {
             throw new IllegalArgumentException(
-                "the values of canonical class names to classes " + "must have the same classes as the keys of classes to painless classes"
+                "the values of canonical class names to classes must have the same classes as the keys of classes to painless classes"
             );
         }
 
@@ -2048,7 +1697,7 @@ private void buildPainlessClassHierarchy() {
         }
 
         for (Class<?> subClass : classesToPainlessClassBuilders.keySet()) {
-            List<Class<?>> superInterfaces = new ArrayList<>(Arrays.asList(subClass.getInterfaces()));
+            Deque<Class<?>> superInterfaces = new ArrayDeque<>(Arrays.asList(subClass.getInterfaces()));
 
             // we check for Object.class as part of the allow listed classes because
             // it is possible for the compiler to work without Object
@@ -2087,7 +1736,7 @@ private void buildPainlessClassHierarchy() {
             Set<Class<?>> resolvedInterfaces = new HashSet<>();
 
             while (superInterfaces.isEmpty() == false) {
-                Class<?> superInterface = superInterfaces.remove(0);
+                Class<?> superInterface = superInterfaces.removeFirst();
 
                 if (resolvedInterfaces.add(superInterface)) {
                     if (classesToPainlessClassBuilders.containsKey(superInterface)) {
@@ -2119,12 +1768,10 @@ private void setFunctionalInterfaceMethod(Class<?> targetClass, PainlessClassBui
             }
 
             if (javaMethods.size() != 1 && targetClass.isAnnotationPresent(FunctionalInterface.class)) {
-                throw new IllegalArgumentException(
-                    "class ["
-                        + typeToCanonicalTypeName(targetClass)
-                        + "] "
-                        + "is illegally marked as a FunctionalInterface with java methods "
-                        + javaMethods
+                throw lookupException(
+                    "class [%s] is illegally marked as a FunctionalInterface with java methods %s",
+                    typeToCanonicalTypeName(targetClass),
+                    javaMethods
                 );
             } else if (javaMethods.size() == 1) {
                 java.lang.reflect.Method javaMethod = javaMethods.get(0);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
index be8bb9902a9a..766714a45469 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupUtility.java
@@ -340,11 +340,12 @@ public static String buildPainlessFieldKey(String fieldName) {
      * derived from an {@link org.elasticsearch.painless.spi.annotation.InjectConstantAnnotation}.
      */
     public static Object[] buildInjections(PainlessMethod painlessMethod, Map<String, Object> constants) {
-        if (painlessMethod.annotations().containsKey(InjectConstantAnnotation.class) == false) {
+        InjectConstantAnnotation injects = (InjectConstantAnnotation) painlessMethod.annotations().get(InjectConstantAnnotation.class);
+        if (injects == null) {
             return new Object[0];
         }
 
-        List<String> names = ((InjectConstantAnnotation) painlessMethod.annotations().get(InjectConstantAnnotation.class)).injects();
+        List<String> names = injects.injects();
         Object[] injections = new Object[names.size()];
 
         for (int i = 0; i < names.size(); i++) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultConstantFoldingOptimizationPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultConstantFoldingOptimizationPhase.java
index df57f9d3da65..2097a3e2995f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultConstantFoldingOptimizationPhase.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultConstantFoldingOptimizationPhase.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.painless.phase;
 
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.ir.BinaryMathNode;
@@ -45,6 +46,42 @@
  */
 public class DefaultConstantFoldingOptimizationPhase extends IRExpressionModifyingVisitor {
 
+    private static IllegalStateException unaryError(String type, String operation, String constant) {
+        return new IllegalStateException(
+            Strings.format(
+                "constant folding error: unexpected type [%s] for unary operation [%s] on constant [%s]",
+                type,
+                operation,
+                constant
+            )
+        );
+    }
+
+    private static IllegalStateException binaryError(String type, String operation, String constant1, String constant2) {
+        return error(type, "binary", operation, constant1, constant2);
+    }
+
+    private static IllegalStateException booleanError(String type, String operation, String constant1, String constant2) {
+        return error(type, "boolean", operation, constant1, constant2);
+    }
+
+    private static IllegalStateException comparisonError(String type, String operation, String constant1, String constant2) {
+        return error(type, "comparison", operation, constant1, constant2);
+    }
+
+    private static IllegalStateException error(String type, String opType, String operation, String constant1, String constant2) {
+        return new IllegalStateException(
+            Strings.format(
+                "constant folding error: unexpected type [%s] for %s operation [%s] on constants [%s] and [%s]",
+                type,
+                opType,
+                operation,
+                constant1,
+                constant2
+            )
+        );
+    }
+
     @Override
     public void visitUnaryMath(UnaryMathNode irUnaryMathNode, Consumer<ExpressionNode> scope) {
         irUnaryMathNode.getChildNode().visit(this, irUnaryMathNode::setChildNode);
@@ -67,17 +104,10 @@ public void visitUnaryMath(UnaryMathNode irUnaryMathNode, Consumer<ExpressionNod
                 } else {
                     throw irUnaryMathNode.getLocation()
                         .createError(
-                            new IllegalStateException(
-                                "constant folding error: "
-                                    + "unexpected type ["
-                                    + PainlessLookupUtility.typeToCanonicalTypeName(type)
-                                    + "] for "
-                                    + "unary operation ["
-                                    + operation.symbol
-                                    + "] on "
-                                    + "constant ["
-                                    + irConstantNode.getDecorationString(IRDConstant.class)
-                                    + "]"
+                            unaryError(
+                                PainlessLookupUtility.typeToCanonicalTypeName(type),
+                                operation.symbol,
+                                irConstantNode.getDecorationString(IRDConstant.class)
                             )
                         );
                 }
@@ -91,17 +121,10 @@ public void visitUnaryMath(UnaryMathNode irUnaryMathNode, Consumer<ExpressionNod
                 } else {
                     throw irUnaryMathNode.getLocation()
                         .createError(
-                            new IllegalStateException(
-                                "constant folding error: "
-                                    + "unexpected type ["
-                                    + PainlessLookupUtility.typeToCanonicalTypeName(type)
-                                    + "] for "
-                                    + "unary operation ["
-                                    + operation.symbol
-                                    + "] on "
-                                    + "constant ["
-                                    + irConstantNode.getDecorationString(IRDConstant.class)
-                                    + "]"
+                            unaryError(
+                                PainlessLookupUtility.typeToCanonicalTypeName(type),
+                                operation.symbol,
+                                irConstantNode.getDecorationString(IRDConstant.class)
                             )
                         );
                 }
@@ -113,17 +136,10 @@ public void visitUnaryMath(UnaryMathNode irUnaryMathNode, Consumer<ExpressionNod
                 } else {
                     throw irUnaryMathNode.getLocation()
                         .createError(
-                            new IllegalStateException(
-                                "constant folding error: "
-                                    + "unexpected type ["
-                                    + PainlessLookupUtility.typeToCanonicalTypeName(type)
-                                    + "] for "
-                                    + "unary operation ["
-                                    + operation.symbol
-                                    + "] on "
-                                    + "constant ["
-                                    + irConstantNode.getDecorationString(IRDConstant.class)
-                                    + "]"
+                            unaryError(
+                                PainlessLookupUtility.typeToCanonicalTypeName(type),
+                                operation.symbol,
+                                irConstantNode.getDecorationString(IRDConstant.class)
                             )
                         );
                 }
@@ -160,20 +176,11 @@ public void visitBinaryMath(BinaryMathNode irBinaryMathNode, Consumer<Expression
                 } else {
                     throw irBinaryMathNode.getLocation()
                         .createError(
-                            new IllegalStateException(
-                                "constant folding error: "
-                                    + "unexpected type ["
-                                    + PainlessLookupUtility.typeToCanonicalTypeName(type)
-                                    + "] for "
-                                    + "binary operation ["
-                                    + operation.symbol
-                                    + "] on "
-                                    + "constants ["
-                                    + irLeftConstantNode.getDecorationString(IRDConstant.class)
-                                    + "] "
-                                    + "and ["
-                                    + irRightConstantNode.getDecorationString(IRDConstant.class)
-                                    + "]"
+                            binaryError(
+                                PainlessLookupUtility.typeToCanonicalTypeName(type),
+                                operation.symbol,
+                                irLeftConstantNode.getDecorationString(IRDConstant.class),
+                                irRightConstantNode.getDecorationString(IRDConstant.class)
                             )
                         );
                 }
@@ -192,20 +199,11 @@ public void visitBinaryMath(BinaryMathNode irBinaryMathNode, Consumer<Expression
                     } else {
                         throw irBinaryMathNode.getLocation()
                             .createError(
-                                new IllegalStateException(
-                                    "constant folding error: "
-                                        + "unexpected type ["
-                                        + PainlessLookupUtility.typeToCanonicalTypeName(type)
-                                        + "] for "
-                                        + "binary operation ["
-                                        + operation.symbol
-                                        + "] on "
-                                        + "constants ["
-                                        + irLeftConstantNode.getDecorationString(IRDConstant.class)
-                                        + "] "
-                                        + "and ["
-                                        + irRightConstantNode.getDecorationString(IRDConstant.class)
-                                        + "]"
+                                binaryError(
+                                    PainlessLookupUtility.typeToCanonicalTypeName(type),
+                                    operation.symbol,
+                                    irLeftConstantNode.getDecorationString(IRDConstant.class),
+                                    irRightConstantNode.getDecorationString(IRDConstant.class)
                                 )
                             );
                     }
@@ -227,20 +225,11 @@ public void visitBinaryMath(BinaryMathNode irBinaryMathNode, Consumer<Expression
                     } else {
                         throw irBinaryMathNode.getLocation()
                             .createError(
-                                new IllegalStateException(
-                                    "constant folding error: "
-                                        + "unexpected type ["
-                                        + PainlessLookupUtility.typeToCanonicalTypeName(type)
-                                        + "] for "
-                                        + "binary operation ["
-                                        + operation.symbol
-                                        + "] on "
-                                        + "constants ["
-                                        + irLeftConstantNode.getDecorationString(IRDConstant.class)
-                                        + "] "
-                                        + "and ["
-                                        + irRightConstantNode.getDecorationString(IRDConstant.class)
-                                        + "]"
+                                binaryError(
+                                    PainlessLookupUtility.typeToCanonicalTypeName(type),
+                                    operation.symbol,
+                                    irLeftConstantNode.getDecorationString(IRDConstant.class),
+                                    irRightConstantNode.getDecorationString(IRDConstant.class)
                                 )
                             );
                     }
@@ -261,20 +250,11 @@ public void visitBinaryMath(BinaryMathNode irBinaryMathNode, Consumer<Expression
                 } else {
                     throw irBinaryMathNode.getLocation()
                         .createError(
-                            new IllegalStateException(
-                                "constant folding error: "
-                                    + "unexpected type ["
-                                    + PainlessLookupUtility.typeToCanonicalTypeName(type)
-                                    + "] for "
-                                    + "binary operation ["
-                                    + operation.symbol
-                                    + "] on "
-                                    + "constants ["
-                                    + irLeftConstantNode.getDecorationString(IRDConstant.class)
-                                    + "] "
-                                    + "and ["
-                                    + irRightConstantNode.getDecorationString(IRDConstant.class)
-                                    + "]"
+                            binaryError(
+                                PainlessLookupUtility.typeToCanonicalTypeName(type),
+                                operation.symbol,
+                                irLeftConstantNode.getDecorationString(IRDConstant.class),
+                                irRightConstantNode.getDecorationString(IRDConstant.class)
                             )
                         );
                 }
@@ -292,20 +272,11 @@ public void visitBinaryMath(BinaryMathNode irBinaryMathNode, Consumer<Expression
                 } else {
                     throw irBinaryMathNode.getLocation()
                         .createError(
-                            new IllegalStateException(
-                                "constant folding error: "
-                                    + "unexpected type ["
-                                    + PainlessLookupUtility.typeToCanonicalTypeName(type)
-                                    + "] for "
-                                    + "binary operation ["
-                                    + operation.symbol
-                                    + "] on "
-                                    + "constants ["
-                                    + irLeftConstantNode.getDecorationString(IRDConstant.class)
-                                    + "] "
-                                    + "and ["
-                                    + irRightConstantNode.getDecorationString(IRDConstant.class)
-                                    + "]"
+                            binaryError(
+                                PainlessLookupUtility.typeToCanonicalTypeName(type),
+                                operation.symbol,
+                                irLeftConstantNode.getDecorationString(IRDConstant.class),
+                                irRightConstantNode.getDecorationString(IRDConstant.class)
                             )
                         );
                 }
@@ -319,20 +290,11 @@ public void visitBinaryMath(BinaryMathNode irBinaryMathNode, Consumer<Expression
                 } else {
                     throw irBinaryMathNode.getLocation()
                         .createError(
-                            new IllegalStateException(
-                                "constant folding error: "
-                                    + "unexpected type ["
-                                    + PainlessLookupUtility.typeToCanonicalTypeName(type)
-                                    + "] for "
-                                    + "binary operation ["
-                                    + operation.symbol
-                                    + "] on "
-                                    + "constants ["
-                                    + irLeftConstantNode.getDecorationString(IRDConstant.class)
-                                    + "] "
-                                    + "and ["
-                                    + irRightConstantNode.getDecorationString(IRDConstant.class)
-                                    + "]"
+                            binaryError(
+                                PainlessLookupUtility.typeToCanonicalTypeName(type),
+                                operation.symbol,
+                                irLeftConstantNode.getDecorationString(IRDConstant.class),
+                                irRightConstantNode.getDecorationString(IRDConstant.class)
                             )
                         );
                 }
@@ -346,20 +308,11 @@ public void visitBinaryMath(BinaryMathNode irBinaryMathNode, Consumer<Expression
                 } else {
                     throw irBinaryMathNode.getLocation()
                         .createError(
-                            new IllegalStateException(
-                                "constant folding error: "
-                                    + "unexpected type ["
-                                    + PainlessLookupUtility.typeToCanonicalTypeName(type)
-                                    + "] for "
-                                    + "binary operation ["
-                                    + operation.symbol
-                                    + "] on "
-                                    + "constants ["
-                                    + irLeftConstantNode.getDecorationString(IRDConstant.class)
-                                    + "] "
-                                    + "and ["
-                                    + irRightConstantNode.getDecorationString(IRDConstant.class)
-                                    + "]"
+                            binaryError(
+                                PainlessLookupUtility.typeToCanonicalTypeName(type),
+                                operation.symbol,
+                                irLeftConstantNode.getDecorationString(IRDConstant.class),
+                                irRightConstantNode.getDecorationString(IRDConstant.class)
                             )
                         );
                 }
@@ -373,20 +326,11 @@ public void visitBinaryMath(BinaryMathNode irBinaryMathNode, Consumer<Expression
                 } else {
                     throw irBinaryMathNode.getLocation()
                         .createError(
-                            new IllegalStateException(
-                                "constant folding error: "
-                                    + "unexpected type ["
-                                    + PainlessLookupUtility.typeToCanonicalTypeName(type)
-                                    + "] for "
-                                    + "binary operation ["
-                                    + operation.symbol
-                                    + "] on "
-                                    + "constants ["
-                                    + irLeftConstantNode.getDecorationString(IRDConstant.class)
-                                    + "] and "
-                                    + "["
-                                    + irRightConstantNode.getDecorationString(IRDConstant.class)
-                                    + "]"
+                            binaryError(
+                                PainlessLookupUtility.typeToCanonicalTypeName(type),
+                                operation.symbol,
+                                irLeftConstantNode.getDecorationString(IRDConstant.class),
+                                irRightConstantNode.getDecorationString(IRDConstant.class)
                             )
                         );
                 }
@@ -400,20 +344,11 @@ public void visitBinaryMath(BinaryMathNode irBinaryMathNode, Consumer<Expression
                 } else {
                     throw irBinaryMathNode.getLocation()
                         .createError(
-                            new IllegalStateException(
-                                "constant folding error: "
-                                    + "unexpected type ["
-                                    + PainlessLookupUtility.typeToCanonicalTypeName(type)
-                                    + "] for "
-                                    + "binary operation ["
-                                    + operation.symbol
-                                    + "] on "
-                                    + "constants ["
-                                    + irLeftConstantNode.getDecorationString(IRDConstant.class)
-                                    + "] "
-                                    + "and ["
-                                    + irRightConstantNode.getDecorationString(IRDConstant.class)
-                                    + "]"
+                            binaryError(
+                                PainlessLookupUtility.typeToCanonicalTypeName(type),
+                                operation.symbol,
+                                irLeftConstantNode.getDecorationString(IRDConstant.class),
+                                irRightConstantNode.getDecorationString(IRDConstant.class)
                             )
                         );
                 }
@@ -429,20 +364,11 @@ public void visitBinaryMath(BinaryMathNode irBinaryMathNode, Consumer<Expression
                 } else {
                     throw irBinaryMathNode.getLocation()
                         .createError(
-                            new IllegalStateException(
-                                "constant folding error: "
-                                    + "unexpected type ["
-                                    + PainlessLookupUtility.typeToCanonicalTypeName(type)
-                                    + "] for "
-                                    + "binary operation ["
-                                    + operation.symbol
-                                    + "] on "
-                                    + "constants ["
-                                    + irLeftConstantNode.getDecorationString(IRDConstant.class)
-                                    + "] and "
-                                    + "["
-                                    + irRightConstantNode.getDecorationString(IRDConstant.class)
-                                    + "]"
+                            binaryError(
+                                PainlessLookupUtility.typeToCanonicalTypeName(type),
+                                operation.symbol,
+                                irLeftConstantNode.getDecorationString(IRDConstant.class),
+                                irRightConstantNode.getDecorationString(IRDConstant.class)
                             )
                         );
                 }
@@ -456,20 +382,11 @@ public void visitBinaryMath(BinaryMathNode irBinaryMathNode, Consumer<Expression
                 } else {
                     throw irBinaryMathNode.getLocation()
                         .createError(
-                            new IllegalStateException(
-                                "constant folding error: "
-                                    + "unexpected type ["
-                                    + PainlessLookupUtility.typeToCanonicalTypeName(type)
-                                    + "] for "
-                                    + "binary operation ["
-                                    + operation.symbol
-                                    + "] on "
-                                    + "constants ["
-                                    + irLeftConstantNode.getDecorationString(IRDConstant.class)
-                                    + "] "
-                                    + "and ["
-                                    + irRightConstantNode.getDecorationString(IRDConstant.class)
-                                    + "]"
+                            binaryError(
+                                PainlessLookupUtility.typeToCanonicalTypeName(type),
+                                operation.symbol,
+                                irLeftConstantNode.getDecorationString(IRDConstant.class),
+                                irRightConstantNode.getDecorationString(IRDConstant.class)
                             )
                         );
                 }
@@ -550,20 +467,11 @@ public void visitBoolean(BooleanNode irBooleanNode, Consumer<ExpressionNode> sco
                 } else {
                     throw irBooleanNode.getLocation()
                         .createError(
-                            new IllegalStateException(
-                                "constant folding error: "
-                                    + "unexpected type ["
-                                    + PainlessLookupUtility.typeToCanonicalTypeName(type)
-                                    + "] for "
-                                    + "binary operation ["
-                                    + operation.symbol
-                                    + "] on "
-                                    + "constants ["
-                                    + irLeftConstantNode.getDecorationString(IRDConstant.class)
-                                    + "] "
-                                    + "and ["
-                                    + irRightConstantNode.getDecorationString(IRDConstant.class)
-                                    + "]"
+                            binaryError(
+                                PainlessLookupUtility.typeToCanonicalTypeName(type),
+                                operation.symbol,
+                                irLeftConstantNode.getDecorationString(IRDConstant.class),
+                                irRightConstantNode.getDecorationString(IRDConstant.class)
                             )
                         );
                 }
@@ -580,20 +488,11 @@ public void visitBoolean(BooleanNode irBooleanNode, Consumer<ExpressionNode> sco
                 } else {
                     throw irBooleanNode.getLocation()
                         .createError(
-                            new IllegalStateException(
-                                "constant folding error: "
-                                    + "unexpected type ["
-                                    + PainlessLookupUtility.typeToCanonicalTypeName(type)
-                                    + "] for "
-                                    + "boolean operation ["
-                                    + operation.symbol
-                                    + "] on "
-                                    + "constants ["
-                                    + irLeftConstantNode.getDecorationString(IRDConstant.class)
-                                    + "] "
-                                    + "and ["
-                                    + irRightConstantNode.getDecorationString(IRDConstant.class)
-                                    + "]"
+                            booleanError(
+                                PainlessLookupUtility.typeToCanonicalTypeName(type),
+                                operation.symbol,
+                                irLeftConstantNode.getDecorationString(IRDConstant.class),
+                                irRightConstantNode.getDecorationString(IRDConstant.class)
                             )
                         );
                 }
@@ -687,20 +586,11 @@ public void visitComparison(ComparisonNode irComparisonNode, Consumer<Expression
                     } else {
                         throw irComparisonNode.getLocation()
                             .createError(
-                                new IllegalStateException(
-                                    "constant folding error: "
-                                        + "unexpected type ["
-                                        + PainlessLookupUtility.typeToCanonicalTypeName(type)
-                                        + "] for "
-                                        + "comparison operation ["
-                                        + operation.symbol
-                                        + "] on "
-                                        + "constants ["
-                                        + irLeftConstantNode.getDecorationString(IRDConstant.class)
-                                        + "] "
-                                        + "and ["
-                                        + irRightConstantNode.getDecorationString(IRDConstant.class)
-                                        + "]"
+                                comparisonError(
+                                    PainlessLookupUtility.typeToCanonicalTypeName(type),
+                                    operation.symbol,
+                                    irLeftConstantNode.getDecorationString(IRDConstant.class),
+                                    irRightConstantNode.getDecorationString(IRDConstant.class)
                                 )
                             );
                     }
@@ -719,20 +609,11 @@ public void visitComparison(ComparisonNode irComparisonNode, Consumer<Expression
                     } else {
                         throw irComparisonNode.getLocation()
                             .createError(
-                                new IllegalStateException(
-                                    "constant folding error: "
-                                        + "unexpected type ["
-                                        + PainlessLookupUtility.typeToCanonicalTypeName(type)
-                                        + "] for "
-                                        + "comparison operation ["
-                                        + operation.symbol
-                                        + "] on "
-                                        + "constants ["
-                                        + irLeftConstantNode.getDecorationString(IRDConstant.class)
-                                        + "] "
-                                        + "and ["
-                                        + irRightConstantNode.getDecorationString(IRDConstant.class)
-                                        + "]"
+                                comparisonError(
+                                    PainlessLookupUtility.typeToCanonicalTypeName(type),
+                                    operation.symbol,
+                                    irLeftConstantNode.getDecorationString(IRDConstant.class),
+                                    irRightConstantNode.getDecorationString(IRDConstant.class)
                                 )
                             );
                     }
@@ -751,20 +632,11 @@ public void visitComparison(ComparisonNode irComparisonNode, Consumer<Expression
                     } else {
                         throw irComparisonNode.getLocation()
                             .createError(
-                                new IllegalStateException(
-                                    "constant folding error: "
-                                        + "unexpected type ["
-                                        + PainlessLookupUtility.typeToCanonicalTypeName(type)
-                                        + "] for "
-                                        + "comparison operation ["
-                                        + operation.symbol
-                                        + "] on "
-                                        + "constants ["
-                                        + irLeftConstantNode.getDecorationString(IRDConstant.class)
-                                        + "] "
-                                        + "and ["
-                                        + irRightConstantNode.getDecorationString(IRDConstant.class)
-                                        + "]"
+                                comparisonError(
+                                    PainlessLookupUtility.typeToCanonicalTypeName(type),
+                                    operation.symbol,
+                                    irLeftConstantNode.getDecorationString(IRDConstant.class),
+                                    irRightConstantNode.getDecorationString(IRDConstant.class)
                                 )
                             );
                     }
@@ -783,20 +655,11 @@ public void visitComparison(ComparisonNode irComparisonNode, Consumer<Expression
                     } else {
                         throw irComparisonNode.getLocation()
                             .createError(
-                                new IllegalStateException(
-                                    "constant folding error: "
-                                        + "unexpected type ["
-                                        + PainlessLookupUtility.typeToCanonicalTypeName(type)
-                                        + "] for "
-                                        + "comparison operation ["
-                                        + operation.symbol
-                                        + "] on "
-                                        + "constants ["
-                                        + irLeftConstantNode.getDecorationString(IRDConstant.class)
-                                        + "] "
-                                        + "and ["
-                                        + irRightConstantNode.getDecorationString(IRDConstant.class)
-                                        + "]"
+                                comparisonError(
+                                    PainlessLookupUtility.typeToCanonicalTypeName(type),
+                                    operation.symbol,
+                                    irLeftConstantNode.getDecorationString(IRDConstant.class),
+                                    irRightConstantNode.getDecorationString(IRDConstant.class)
                                 )
                             );
                     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultIRTreeToASMBytesPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultIRTreeToASMBytesPhase.java
index d7ece3428149..084d5ee8e6c3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultIRTreeToASMBytesPhase.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultIRTreeToASMBytesPhase.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.painless.phase;
 
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.painless.ClassWriter;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Location;
@@ -16,6 +17,7 @@
 import org.elasticsearch.painless.ScriptClassInfo;
 import org.elasticsearch.painless.WriterConstants;
 import org.elasticsearch.painless.api.Augmentation;
+import org.elasticsearch.painless.api.ValueIterator;
 import org.elasticsearch.painless.ir.BinaryImplNode;
 import org.elasticsearch.painless.ir.BinaryMathNode;
 import org.elasticsearch.painless.ir.BlockNode;
@@ -163,7 +165,6 @@
 import java.util.ArrayList;
 import java.util.BitSet;
 import java.util.Collections;
-import java.util.Iterator;
 import java.util.List;
 import java.util.regex.Matcher;
 
@@ -174,6 +175,15 @@
 import static org.elasticsearch.painless.WriterConstants.ITERATOR_NEXT;
 import static org.elasticsearch.painless.WriterConstants.ITERATOR_TYPE;
 import static org.elasticsearch.painless.WriterConstants.OBJECTS_TYPE;
+import static org.elasticsearch.painless.WriterConstants.VALUE_ITERATOR_NEXT_BOOLEAN;
+import static org.elasticsearch.painless.WriterConstants.VALUE_ITERATOR_NEXT_BYTE;
+import static org.elasticsearch.painless.WriterConstants.VALUE_ITERATOR_NEXT_CHAR;
+import static org.elasticsearch.painless.WriterConstants.VALUE_ITERATOR_NEXT_DOUBLE;
+import static org.elasticsearch.painless.WriterConstants.VALUE_ITERATOR_NEXT_FLOAT;
+import static org.elasticsearch.painless.WriterConstants.VALUE_ITERATOR_NEXT_INT;
+import static org.elasticsearch.painless.WriterConstants.VALUE_ITERATOR_NEXT_LONG;
+import static org.elasticsearch.painless.WriterConstants.VALUE_ITERATOR_NEXT_SHORT;
+import static org.elasticsearch.painless.WriterConstants.VALUE_ITERATOR_TYPE;
 
 public class DefaultIRTreeToASMBytesPhase implements IRTreeVisitor<WriteScope> {
 
@@ -551,6 +561,8 @@ public void visitForEachSubIterableLoop(ForEachSubIterableNode irForEachSubItera
         MethodWriter methodWriter = writeScope.getMethodWriter();
         methodWriter.writeStatementOffset(irForEachSubIterableNode.getLocation());
 
+        PainlessMethod painlessMethod = irForEachSubIterableNode.getDecorationValue(IRDMethod.class);
+
         Variable variable = writeScope.defineVariable(
             irForEachSubIterableNode.getDecorationValue(IRDVariableType.class),
             irForEachSubIterableNode.getDecorationValue(IRDVariableName.class)
@@ -562,10 +574,8 @@ public void visitForEachSubIterableLoop(ForEachSubIterableNode irForEachSubItera
 
         visit(irForEachSubIterableNode.getConditionNode(), writeScope);
 
-        PainlessMethod painlessMethod = irForEachSubIterableNode.getDecorationValue(IRDMethod.class);
-
         if (painlessMethod == null) {
-            Type methodType = Type.getMethodType(Type.getType(Iterator.class), Type.getType(Object.class));
+            Type methodType = Type.getMethodType(Type.getType(ValueIterator.class), Type.getType(Object.class));
             methodWriter.invokeDefCall("iterator", methodType, DefBootstrap.ITERATOR);
         } else {
             methodWriter.invokeMethodCall(painlessMethod);
@@ -583,8 +593,22 @@ public void visitForEachSubIterableLoop(ForEachSubIterableNode irForEachSubItera
         methodWriter.ifZCmp(MethodWriter.EQ, end);
 
         methodWriter.visitVarInsn(iterator.getAsmType().getOpcode(Opcodes.ILOAD), iterator.getSlot());
-        methodWriter.invokeInterface(ITERATOR_TYPE, ITERATOR_NEXT);
-        methodWriter.writeCast(irForEachSubIterableNode.getDecorationValue(IRDCast.class));
+        if (painlessMethod != null || variable.getType().isPrimitive() == false) {
+            methodWriter.invokeInterface(ITERATOR_TYPE, ITERATOR_NEXT);
+            methodWriter.writeCast(irForEachSubIterableNode.getDecorationValue(IRDCast.class));
+        } else {
+            switch (variable.getAsmType().getSort()) {
+                case Type.BOOLEAN -> methodWriter.invokeInterface(VALUE_ITERATOR_TYPE, VALUE_ITERATOR_NEXT_BOOLEAN);
+                case Type.BYTE -> methodWriter.invokeInterface(VALUE_ITERATOR_TYPE, VALUE_ITERATOR_NEXT_BYTE);
+                case Type.SHORT -> methodWriter.invokeInterface(VALUE_ITERATOR_TYPE, VALUE_ITERATOR_NEXT_SHORT);
+                case Type.CHAR -> methodWriter.invokeInterface(VALUE_ITERATOR_TYPE, VALUE_ITERATOR_NEXT_CHAR);
+                case Type.INT -> methodWriter.invokeInterface(VALUE_ITERATOR_TYPE, VALUE_ITERATOR_NEXT_INT);
+                case Type.LONG -> methodWriter.invokeInterface(VALUE_ITERATOR_TYPE, VALUE_ITERATOR_NEXT_LONG);
+                case Type.FLOAT -> methodWriter.invokeInterface(VALUE_ITERATOR_TYPE, VALUE_ITERATOR_NEXT_FLOAT);
+                case Type.DOUBLE -> methodWriter.invokeInterface(VALUE_ITERATOR_TYPE, VALUE_ITERATOR_NEXT_DOUBLE);
+                default -> throw new IllegalArgumentException("Unknown primitive iteration variable type " + variable.getAsmType());
+            }
+        }
         methodWriter.visitVarInsn(variable.getAsmType().getOpcode(Opcodes.ISTORE), variable.getSlot());
 
         visit(irForEachSubIterableNode.getBlockNode(), writeScope.newLoopScope(begin, end));
@@ -788,12 +812,11 @@ public void visitUnaryMath(UnaryMathNode irUnaryMathNode, WriteScope writeScope)
                         methodWriter.push(-1L);
                     } else {
                         throw new IllegalStateException(
-                            "unexpected unary math operation ["
-                                + operation
-                                + "] "
-                                + "for type ["
-                                + irUnaryMathNode.getDecorationString(IRDExpressionType.class)
-                                + "]"
+                            Strings.format(
+                                "unexpected unary math operation [%s] for type [%s]",
+                                operation,
+                                irUnaryMathNode.getDecorationString(IRDExpressionType.class)
+                            )
                         );
                     }
 
@@ -813,12 +836,11 @@ public void visitUnaryMath(UnaryMathNode irUnaryMathNode, WriteScope writeScope)
                 }
             } else {
                 throw new IllegalStateException(
-                    "unexpected unary math operation ["
-                        + operation
-                        + "] "
-                        + "for type ["
-                        + irUnaryMathNode.getDecorationString(IRDExpressionType.class)
-                        + "]"
+                    Strings.format(
+                        "unexpected unary math operation [%s] for type [%s]",
+                        operation,
+                        irUnaryMathNode.getDecorationString(IRDExpressionType.class)
+                    )
                 );
             }
         }
@@ -845,12 +867,11 @@ public void visitBinaryMath(BinaryMathNode irBinaryMathNode, WriteScope writeSco
                 methodWriter.invokeVirtual(Type.getType(Matcher.class), WriterConstants.MATCHER_MATCHES);
             } else {
                 throw new IllegalStateException(
-                    "unexpected binary math operation ["
-                        + operation
-                        + "] "
-                        + "for type ["
-                        + irBinaryMathNode.getDecorationString(IRDExpressionType.class)
-                        + "]"
+                    Strings.format(
+                        "unexpected binary math operation [%s] for type [%s]",
+                        operation,
+                        irBinaryMathNode.getDecorationString(IRDExpressionType.class)
+                    )
                 );
             }
         } else {
@@ -973,24 +994,22 @@ public void visitComparison(ComparisonNode irComparisonNode, WriteScope writeSco
 
         if (comparisonType == void.class || comparisonType == byte.class || comparisonType == short.class || comparisonType == char.class) {
             throw new IllegalStateException(
-                "unexpected comparison operation ["
-                    + operation
-                    + "] "
-                    + "for type ["
-                    + irComparisonNode.getDecorationString(IRDExpressionType.class)
-                    + "]"
+                Strings.format(
+                    "unexpected comparison operation [%s] for type [%s]",
+                    operation,
+                    irComparisonNode.getDecorationString(IRDExpressionType.class)
+                )
             );
         } else if (comparisonType == boolean.class) {
             if (eq) methodWriter.ifCmp(type, MethodWriter.EQ, jump);
             else if (ne) methodWriter.ifCmp(type, MethodWriter.NE, jump);
             else {
                 throw new IllegalStateException(
-                    "unexpected comparison operation ["
-                        + operation
-                        + "] "
-                        + "for type ["
-                        + irComparisonNode.getDecorationString(IRDExpressionType.class)
-                        + "]"
+                    Strings.format(
+                        "unexpected comparison operation [%s] for type [%s]",
+                        operation,
+                        irComparisonNode.getDecorationString(IRDExpressionType.class)
+                    )
                 );
             }
         } else if (comparisonType == int.class
@@ -1005,12 +1024,11 @@ public void visitComparison(ComparisonNode irComparisonNode, WriteScope writeSco
                 else if (gte) methodWriter.ifCmp(type, MethodWriter.GE, jump);
                 else {
                     throw new IllegalStateException(
-                        "unexpected comparison operation ["
-                            + operation
-                            + "] "
-                            + "for type ["
-                            + irComparisonNode.getDecorationString(IRDExpressionType.class)
-                            + "]"
+                        Strings.format(
+                            "unexpected comparison operation [%s] for type [%s]",
+                            operation,
+                            irComparisonNode.getDecorationString(IRDExpressionType.class)
+                        )
                     );
                 }
 
@@ -1054,12 +1072,11 @@ public void visitComparison(ComparisonNode irComparisonNode, WriteScope writeSco
                     writejump = false;
                 } else {
                     throw new IllegalStateException(
-                        "unexpected comparison operation ["
-                            + operation
-                            + "] "
-                            + "for type ["
-                            + irComparisonNode.getDecorationString(IRDExpressionType.class)
-                            + "]"
+                        Strings.format(
+                            "unexpected comparison operation [%s] for type [%s]",
+                            operation,
+                            irComparisonNode.getDecorationString(IRDExpressionType.class)
+                        )
                     );
                 }
             } else {
@@ -1083,12 +1100,11 @@ public void visitComparison(ComparisonNode irComparisonNode, WriteScope writeSco
                     }
                 } else {
                     throw new IllegalStateException(
-                        "unexpected comparison operation ["
-                            + operation
-                            + "] "
-                            + "for type ["
-                            + irComparisonNode.getDecorationString(IRDExpressionType.class)
-                            + "]"
+                        Strings.format(
+                            "unexpected comparison operation [%s] for type [%s]",
+                            operation,
+                            irComparisonNode.getDecorationString(IRDExpressionType.class)
+                        )
                     );
                 }
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticAnalysisPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticAnalysisPhase.java
index 8c6ac55e8ffe..9e95734f0978 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticAnalysisPhase.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticAnalysisPhase.java
@@ -8,12 +8,14 @@
 
 package org.elasticsearch.painless.phase;
 
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.FunctionRef;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.lookup.PainlessCast;
+import org.elasticsearch.painless.lookup.PainlessClass;
 import org.elasticsearch.painless.lookup.PainlessClassBinding;
 import org.elasticsearch.painless.lookup.PainlessConstructor;
 import org.elasticsearch.painless.lookup.PainlessField;
@@ -157,6 +159,10 @@
  */
 public class DefaultSemanticAnalysisPhase extends UserTreeBaseVisitor<SemanticScope> {
 
+    private static ClassCastException castError(String formatText, Object... arguments) {
+        return new ClassCastException(Strings.format(formatText, arguments));
+    }
+
     /**
      * Decorates a user expression node with a PainlessCast.
      */
@@ -251,13 +257,11 @@ public void visitFunction(SFunction userFunctionNode, ScriptScope scriptScope) {
         if (userBlockNode.getStatementNodes().isEmpty()) {
             throw userFunctionNode.createError(
                 new IllegalArgumentException(
-                    "invalid function definition: "
-                        + "found no statements for function "
-                        + "["
-                        + functionName
-                        + "] with ["
-                        + typeParameters.size()
-                        + "] parameters"
+                    Strings.format(
+                        "invalid function definition: found no statements for function [%s] with [%d] parameters",
+                        functionName,
+                        typeParameters.size()
+                    )
                 )
             );
         }
@@ -270,13 +274,11 @@ public void visitFunction(SFunction userFunctionNode, ScriptScope scriptScope) {
         if (methodEscape == false && isAutoReturnEnabled == false && returnType != void.class) {
             throw userFunctionNode.createError(
                 new IllegalArgumentException(
-                    "invalid function definition: "
-                        + "not all paths provide a return value for function "
-                        + "["
-                        + functionName
-                        + "] with ["
-                        + typeParameters.size()
-                        + "] parameters"
+                    Strings.format(
+                        "invalid function definition: not all paths provide a return value for function [%s] with [%d] parameters",
+                        functionName,
+                        typeParameters.size()
+                    )
                 )
             );
         }
@@ -750,14 +752,10 @@ public void visitReturn(SReturn userReturnNode, SemanticScope semanticScope) {
         if (userValueNode == null) {
             if (semanticScope.getReturnType() != void.class) {
                 throw userReturnNode.createError(
-                    new ClassCastException(
-                        "cannot cast from "
-                            + "["
-                            + semanticScope.getReturnCanonicalTypeName()
-                            + "] to "
-                            + "["
-                            + PainlessLookupUtility.typeToCanonicalTypeName(void.class)
-                            + "]"
+                    castError(
+                        "cannot cast from [%s] to [%s]",
+                        semanticScope.getReturnCanonicalTypeName(),
+                        PainlessLookupUtility.typeToCanonicalTypeName(void.class)
                     )
                 );
             }
@@ -892,14 +890,12 @@ public void visitCatch(SCatch userCatchNode, SemanticScope semanticScope) {
 
         if (userCatchNode.getBaseException().isAssignableFrom(type) == false) {
             throw userCatchNode.createError(
-                new ClassCastException(
-                    "cannot cast from ["
-                        + PainlessLookupUtility.typeToCanonicalTypeName(type)
-                        + "] "
-                        + "to ["
-                        + PainlessLookupUtility.typeToCanonicalTypeName(baseException)
-                        + "]"
+                castError(
+                    "cannot cast from [%s] to [%s]",
+                    PainlessLookupUtility.typeToCanonicalTypeName(type),
+                    PainlessLookupUtility.typeToCanonicalTypeName(baseException)
                 )
+
             );
         }
 
@@ -1031,15 +1027,11 @@ public void visitAssignment(EAssignment userAssignmentNode, SemanticScope semant
 
             if (compoundType == null || (isShift && shiftType == null)) {
                 throw userAssignmentNode.createError(
-                    new ClassCastException(
-                        "invalid compound assignment: "
-                            + "cannot apply ["
-                            + operation.symbol
-                            + "=] to types ["
-                            + leftValueType
-                            + "] and ["
-                            + rightValueType
-                            + "]"
+                    castError(
+                        "invalid compound assignment: cannot apply [%s=] to types [%s] and [%s]",
+                        operation.symbol,
+                        leftValueType,
+                        rightValueType
                     )
                 );
             }
@@ -1162,17 +1154,13 @@ public void visitUnary(EUnary userUnaryNode, SemanticScope semanticScope) {
 
                 if (unaryType == null) {
                     throw userUnaryNode.createError(
-                        new ClassCastException(
-                            "cannot apply the "
-                                + operation.name
-                                + " operator "
-                                + "["
-                                + operation.symbol
-                                + "] to the type "
-                                + "["
-                                + PainlessLookupUtility.typeToCanonicalTypeName(childValueType)
-                                + "]"
+                        castError(
+                            "cannot apply the %s operator [%s] to the type [%s]",
+                            operation.name,
+                            operation.symbol,
+                            PainlessLookupUtility.typeToCanonicalTypeName(childValueType)
                         )
+
                     );
                 }
 
@@ -1267,20 +1255,14 @@ public void visitBinary(EBinary userBinaryNode, SemanticScope semanticScope) {
 
             if (binaryType == null) {
                 throw userBinaryNode.createError(
-                    new ClassCastException(
-                        "cannot apply the "
-                            + operation.name
-                            + " operator "
-                            + "["
-                            + operation.symbol
-                            + "] to the types "
-                            + "["
-                            + PainlessLookupUtility.typeToCanonicalTypeName(leftValueType)
-                            + "] and "
-                            + "["
-                            + PainlessLookupUtility.typeToCanonicalTypeName(rightValueType)
-                            + "]"
+                    castError(
+                        "cannot apply the %s operator [%s] to the types [%s] and [%s]",
+                        operation.name,
+                        operation.symbol,
+                        PainlessLookupUtility.typeToCanonicalTypeName(leftValueType),
+                        PainlessLookupUtility.typeToCanonicalTypeName(rightValueType)
                     )
+
                 );
             }
 
@@ -1404,19 +1386,12 @@ public void visitComp(EComp userCompNode, SemanticScope semanticScope) {
 
         if (promotedType == null) {
             throw userCompNode.createError(
-                new ClassCastException(
-                    "cannot apply the "
-                        + operation.name
-                        + " operator "
-                        + "["
-                        + operation.symbol
-                        + "] to the types "
-                        + "["
-                        + PainlessLookupUtility.typeToCanonicalTypeName(leftValueType)
-                        + "] and "
-                        + "["
-                        + PainlessLookupUtility.typeToCanonicalTypeName(rightValueType)
-                        + "]"
+                castError(
+                    "cannot apply the %s operator [%s] to the types [%s] and [%s]",
+                    operation.name,
+                    operation.symbol,
+                    PainlessLookupUtility.typeToCanonicalTypeName(leftValueType),
+                    PainlessLookupUtility.typeToCanonicalTypeName(rightValueType)
                 )
             );
         }
@@ -1565,13 +1540,11 @@ public void visitConditional(EConditional userConditionalNode, SemanticScope sem
             if (promote == null) {
                 throw userConditionalNode.createError(
                     new ClassCastException(
-                        "cannot apply the conditional operator [?:] to the types "
-                            + "["
-                            + PainlessLookupUtility.typeToCanonicalTypeName(leftValueType)
-                            + "] and "
-                            + "["
-                            + PainlessLookupUtility.typeToCanonicalTypeName(rightValueType)
-                            + "]"
+                        Strings.format(
+                            "cannot apply the conditional operator [?:] to the types [%s] and [%s]",
+                            PainlessLookupUtility.typeToCanonicalTypeName(leftValueType),
+                            PainlessLookupUtility.typeToCanonicalTypeName(rightValueType)
+                        )
                     )
                 );
             }
@@ -1821,12 +1794,11 @@ public void visitNewObj(ENewObj userNewObjNode, SemanticScope semanticScope) {
         if (semanticScope.getCondition(userNewObjNode, Write.class)) {
             throw userNewObjNode.createError(
                 new IllegalArgumentException(
-                    "invalid assignment cannot assign a value to new object with constructor "
-                        + "["
-                        + canonicalTypeName
-                        + "/"
-                        + userArgumentsSize
-                        + "]"
+                    Strings.format(
+                        "invalid assignment cannot assign a value to new object with constructor [%s/%d]",
+                        canonicalTypeName,
+                        userArgumentsSize
+                    )
                 )
             );
         }
@@ -1843,7 +1815,7 @@ public void visitNewObj(ENewObj userNewObjNode, SemanticScope semanticScope) {
         if (constructor == null) {
             throw userNewObjNode.createError(
                 new IllegalArgumentException(
-                    "constructor [" + typeToCanonicalTypeName(valueType) + ", <init>/" + userArgumentsSize + "] not found"
+                    Strings.format("constructor [%s, <init>/%d] not found", typeToCanonicalTypeName(valueType), userArgumentsSize)
                 )
             );
         }
@@ -1854,14 +1826,12 @@ public void visitNewObj(ENewObj userNewObjNode, SemanticScope semanticScope) {
         if (constructor.typeParameters().size() != userArgumentsSize) {
             throw userNewObjNode.createError(
                 new IllegalArgumentException(
-                    "When calling constructor on type ["
-                        + PainlessLookupUtility.typeToCanonicalTypeName(valueType)
-                        + "] "
-                        + "expected ["
-                        + constructor.typeParameters().size()
-                        + "] arguments, but found ["
-                        + userArgumentsSize
-                        + "]."
+                    Strings.format(
+                        "When calling constructor on type [%s] expected [%d] arguments, but found [%d].",
+                        PainlessLookupUtility.typeToCanonicalTypeName(valueType),
+                        constructor.typeParameters().size(),
+                        userArgumentsSize
+                    )
                 )
             );
         }
@@ -2294,13 +2264,12 @@ public void visitRegex(ERegex userRegexNode, SemanticScope semanticScope) {
         } catch (PatternSyntaxException pse) {
             throw new Location(location.getSourceName(), location.getOffset() + 1 + pse.getIndex()).createError(
                 new IllegalArgumentException(
-                    "invalid regular expression: "
-                        + "could not compile regex constant ["
-                        + pattern
-                        + "] with flags ["
-                        + flags
-                        + "]: "
-                        + pse.getDescription(),
+                    Strings.format(
+                        "invalid regular expression: could not compile regex constant [%s] with flags [%s]: %s",
+                        pattern,
+                        flags,
+                        pse.getDescription()
+                    ),
                     pse
                 )
             );
@@ -2363,11 +2332,11 @@ public void visitLambda(ELambda userLambdaNode, SemanticScope semanticScope) {
             }
             // check arity before we manipulate parameters
             if (interfaceMethod.typeParameters().size() != canonicalTypeNameParameters.size()) throw new IllegalArgumentException(
-                "Incorrect number of parameters for ["
-                    + interfaceMethod.javaMethod().getName()
-                    + "] in ["
-                    + targetType.getTargetCanonicalTypeName()
-                    + "]"
+                Strings.format(
+                    "Incorrect number of parameters for [%s] in [%s]",
+                    interfaceMethod.javaMethod().getName(),
+                    targetType.getTargetCanonicalTypeName()
+                )
             );
             // for method invocation, its allowed to ignore the return value
             if (interfaceMethod.returnType() == void.class) {
@@ -2724,13 +2693,11 @@ public void visitDot(EDot userDotNode, SemanticScope semanticScope) {
         if (prefixValueType != null && prefixStaticType != null) {
             throw userDotNode.createError(
                 new IllegalStateException(
-                    "cannot have both "
-                        + "value ["
-                        + prefixValueType.getValueCanonicalTypeName()
-                        + "] "
-                        + "and type ["
-                        + prefixStaticType.getStaticCanonicalTypeName()
-                        + "]"
+                    Strings.format(
+                        "cannot have both value [%s] and type [%s]",
+                        prefixValueType.getValueCanonicalTypeName(),
+                        prefixStaticType.getStaticCanonicalTypeName()
+                    )
                 )
             );
         }
@@ -3248,13 +3215,11 @@ public void visitCall(ECall userCallNode, SemanticScope semanticScope) {
         if (prefixValueType != null && prefixStaticType != null) {
             throw userCallNode.createError(
                 new IllegalStateException(
-                    "cannot have both "
-                        + "value ["
-                        + prefixValueType.getValueCanonicalTypeName()
-                        + "] "
-                        + "and type ["
-                        + prefixStaticType.getStaticCanonicalTypeName()
-                        + "]"
+                    Strings.format(
+                        "cannot have both value [%s] and type [%s]",
+                        prefixValueType.getValueCanonicalTypeName(),
+                        prefixStaticType.getStaticCanonicalTypeName()
+                    )
                 )
             );
         }
@@ -3283,21 +3248,20 @@ public void visitCall(ECall userCallNode, SemanticScope semanticScope) {
                 method = lookup.lookupPainlessMethod(type, false, methodName, userArgumentsSize);
 
                 if (method == null) {
-                    dynamic = lookup.lookupPainlessClass(type).annotations.containsKey(DynamicTypeAnnotation.class)
+                    PainlessClass pc = lookup.lookupPainlessClass(type);
+                    dynamic = pc != null
+                        && pc.annotations.containsKey(DynamicTypeAnnotation.class)
                         && lookup.lookupPainlessSubClassesMethod(type, methodName, userArgumentsSize) != null;
 
                     if (dynamic == false) {
                         throw userCallNode.createError(
                             new IllegalArgumentException(
-                                "member method "
-                                    + "["
-                                    + prefixValueType.getValueCanonicalTypeName()
-                                    + ", "
-                                    + methodName
-                                    + "/"
-                                    + userArgumentsSize
-                                    + "] "
-                                    + "not found"
+                                Strings.format(
+                                    "member method [%s, %s/%d] not found",
+                                    prefixValueType.getValueCanonicalTypeName(),
+                                    methodName,
+                                    userArgumentsSize
+                                )
                             )
                         );
                     }
@@ -3311,15 +3275,12 @@ public void visitCall(ECall userCallNode, SemanticScope semanticScope) {
             if (method == null) {
                 throw userCallNode.createError(
                     new IllegalArgumentException(
-                        "static method "
-                            + "["
-                            + prefixStaticType.getStaticCanonicalTypeName()
-                            + ", "
-                            + methodName
-                            + "/"
-                            + userArgumentsSize
-                            + "] "
-                            + "not found"
+                        Strings.format(
+                            "static method [%s, %s/%d] not found",
+                            prefixStaticType.getStaticCanonicalTypeName(),
+                            methodName,
+                            userArgumentsSize
+                        )
                     )
                 );
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticHeaderPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticHeaderPhase.java
index 24d5508c5de7..61be12039cbe 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticHeaderPhase.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticHeaderPhase.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.painless.phase;
 
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.node.SClass;
 import org.elasticsearch.painless.node.SFunction;
@@ -36,15 +37,13 @@ public void visitFunction(SFunction userFunctionNode, ScriptScope scriptScope) {
         if (parameterCount != parameterNames.size()) {
             throw userFunctionNode.createError(
                 new IllegalStateException(
-                    "invalid function definition: "
-                        + "parameter types size ["
-                        + canonicalTypeNameParameters.size()
-                        + "] is not equal to "
-                        + "parameter names size ["
-                        + parameterNames.size()
-                        + "] for function ["
-                        + functionName
-                        + "]"
+                    Strings.format(
+                        "invalid function definition: "
+                            + "parameter types size [%d] is not equal to parameter names size [%d] for function [%s]",
+                        canonicalTypeNameParameters.size(),
+                        parameterNames.size(),
+                        functionName
+                    )
                 )
             );
         }
@@ -65,12 +64,11 @@ public void visitFunction(SFunction userFunctionNode, ScriptScope scriptScope) {
         if (returnType == null) {
             throw userFunctionNode.createError(
                 new IllegalArgumentException(
-                    "invalid function definition: "
-                        + "return type ["
-                        + returnCanonicalTypeName
-                        + "] not found for function ["
-                        + functionKey
-                        + "]"
+                    Strings.format(
+                        "invalid function definition: return type [%s] not found for function [%s]",
+                        returnCanonicalTypeName,
+                        functionKey
+                    )
                 )
             );
         }
@@ -83,12 +81,11 @@ public void visitFunction(SFunction userFunctionNode, ScriptScope scriptScope) {
             if (paramType == null) {
                 throw userFunctionNode.createError(
                     new IllegalArgumentException(
-                        "invalid function definition: "
-                            + "parameter type ["
-                            + typeParameter
-                            + "] not found for function ["
-                            + functionKey
-                            + "]"
+                        Strings.format(
+                            "invalid function definition: parameter type [%s] not found for function [%s]",
+                            typeParameter,
+                            functionKey
+                        )
                     )
                 );
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java
index ddbdd2ce5ecb..4739f7682ef3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.WriterConstants;
+import org.elasticsearch.painless.api.ValueIterator;
 import org.elasticsearch.painless.ir.BinaryImplNode;
 import org.elasticsearch.painless.ir.BinaryMathNode;
 import org.elasticsearch.painless.ir.BlockNode;
@@ -759,17 +760,24 @@ public void visitEach(SEach userEachNode, ScriptScope scriptScope) {
             irForEachSubIterableNode.setBlockNode(irBlockNode);
             irForEachSubIterableNode.attachDecoration(new IRDVariableType(variable.type()));
             irForEachSubIterableNode.attachDecoration(new IRDVariableName(variable.name()));
-            irForEachSubIterableNode.attachDecoration(new IRDIterableType(Iterator.class));
             irForEachSubIterableNode.attachDecoration(new IRDIterableName("#itr" + userEachNode.getLocation().getOffset()));
 
             if (iterableValueType != def.class) {
+                irForEachSubIterableNode.attachDecoration(new IRDIterableType(Iterator.class));
                 irForEachSubIterableNode.attachDecoration(
                     new IRDMethod(scriptScope.getDecoration(userEachNode, IterablePainlessMethod.class).iterablePainlessMethod())
                 );
-            }
 
-            if (painlessCast != null) {
-                irForEachSubIterableNode.attachDecoration(new IRDCast(painlessCast));
+                if (painlessCast != null) {
+                    irForEachSubIterableNode.attachDecoration(new IRDCast(painlessCast));
+                }
+            } else {
+                // use ValueIterator as we could be iterating over an array directly
+                irForEachSubIterableNode.attachDecoration(new IRDIterableType(ValueIterator.class));
+
+                if (painlessCast != null && variable.type().isPrimitive() == false) {
+                    irForEachSubIterableNode.attachDecoration(new IRDCast(painlessCast));
+                }
             }
 
             irConditionNode = irForEachSubIterableNode;
@@ -905,7 +913,7 @@ public void visitAssignment(EAssignment userAssignmentNode, ScriptScope scriptSc
                 irCompoundNode = stringConcatenationNode;
 
                 // must handle the StringBuilder case for java version <= 8
-                if (irLoadNode instanceof BinaryImplNode bin && WriterConstants.INDY_STRING_CONCAT_BOOTSTRAP_HANDLE == null) {
+                if (irLoadNode instanceof BinaryImplNode bin && WriterConstants.STRING_CONCAT_BOOTSTRAP_HANDLE == null) {
                     bin.getLeftNode().attachDecoration(new IRDDepth(1));
                 }
                 // handles when the operation is mathematical
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticAnalysisPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticAnalysisPhase.java
index 89eeca52be4b..a7b4771a1fbf 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticAnalysisPhase.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticAnalysisPhase.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.painless.phase;
 
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.ScriptClassInfo;
@@ -77,13 +78,11 @@ public void visitFunction(SFunction userFunctionNode, ScriptScope scriptScope) {
             if (userBlockNode.getStatementNodes().isEmpty()) {
                 throw userFunctionNode.createError(
                     new IllegalArgumentException(
-                        "invalid function definition: "
-                            + "found no statements for function "
-                            + "["
-                            + functionName
-                            + "] with ["
-                            + typeParameters.size()
-                            + "] parameters"
+                        Strings.format(
+                            "invalid function definition: found no statements for function [%s] with [%d] parameters",
+                            functionName,
+                            typeParameters.size()
+                        )
                     )
                 );
             }
@@ -161,13 +160,11 @@ public void visitReturn(SReturn userReturnNode, SemanticScope semanticScope) {
             if (semanticScope.getReturnType() != void.class) {
                 throw userReturnNode.createError(
                     new ClassCastException(
-                        "cannot cast from "
-                            + "["
-                            + semanticScope.getReturnCanonicalTypeName()
-                            + "] to "
-                            + "["
-                            + PainlessLookupUtility.typeToCanonicalTypeName(void.class)
-                            + "]"
+                        Strings.format(
+                            "cannot cast from [%s] to [%s]",
+                            semanticScope.getReturnCanonicalTypeName(),
+                            PainlessLookupUtility.typeToCanonicalTypeName(void.class)
+                        )
                     )
                 );
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticHeaderPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticHeaderPhase.java
index ed5ca0a63f09..083f14d88b08 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticHeaderPhase.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessSemanticHeaderPhase.java
@@ -31,7 +31,7 @@ public void visitFunction(SFunction userFunctionNode, ScriptScope scriptScope) {
 
             if (functionTable.getFunction(functionKey) != null) {
                 throw userFunctionNode.createError(
-                    new IllegalArgumentException("invalid function definition: " + "found duplicate function [" + functionKey + "].")
+                    new IllegalArgumentException("invalid function definition: found duplicate function [" + functionKey + "].")
                 );
             }
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
index 4a93a4dc6866..5ab1a154ade5 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
@@ -8,7 +8,6 @@
 
 package org.elasticsearch.painless;
 
-import org.apache.lucene.util.Constants;
 import org.hamcrest.Matcher;
 
 import java.lang.invoke.MethodHandle;
@@ -34,7 +33,6 @@ protected Matcher<String> outOfBoundsExceptionMessageMatcher(int index, int size
     }
 
     public void testArrayLengthHelper() throws Throwable {
-        assertEquals(Constants.JRE_IS_MINIMUM_JAVA9, Def.JAVA9_ARRAY_LENGTH_MH_FACTORY != null);
         assertArrayLength(2, new int[2]);
         assertArrayLength(3, new long[3]);
         assertArrayLength(4, new byte[4]);
@@ -81,4 +79,129 @@ public void testForLoop() {
     public void testDivideArray() {
         assertEquals(1, exec("def[] x = new def[1]; x[0] = 2; return x[0] / 2"));
     }
+
+    public void testPrimitiveIteration() {
+        assertEquals(true, exec("def x = new boolean[] { true, false }; boolean s = false; for (boolean l : x) s |= l; return s"));
+        expectScriptThrows(
+            ClassCastException.class,
+            () -> exec("def x = new boolean[] { true, false }; short s = 0; for (short l : x) s += l; return s")
+        );
+        expectScriptThrows(
+            ClassCastException.class,
+            () -> exec("def x = new boolean[] { true, false }; char s = 0; for (char l : x) s = l; return s")
+        );
+        expectScriptThrows(
+            ClassCastException.class,
+            () -> exec("def x = new boolean[] { true, false }; int s = 0; for (int l : x) s += l; return s")
+        );
+        expectScriptThrows(
+            ClassCastException.class,
+            () -> exec("def x = new boolean[] { true, false }; long s = 0; for (long l : x) s += l; return s")
+        );
+        expectScriptThrows(
+            ClassCastException.class,
+            () -> exec("def x = new boolean[] { true, false }; float s = 0; for (float l : x) s += l; return s")
+        );
+        expectScriptThrows(
+            ClassCastException.class,
+            () -> exec("def x = new boolean[] { true, false }; double s = 0; for (double l : x) s += l; return s")
+        );
+        assertEquals(true, exec("def x = new boolean[] { true, false }; boolean s = false; for (def l : x) s |= l; return s"));
+
+        assertEquals((byte) 30, exec("def x = new byte[] { (byte)10, (byte)20 }; byte s = 0; for (byte l : x) s += l; return s"));
+        assertEquals((short) 30, exec("def x = new byte[] { (byte)10, (byte)20 }; short s = 0; for (short l : x) s += l; return s"));
+        assertEquals((char) 20, exec("def x = new byte[] { (byte)10, (byte)20 }; char s = 0; for (char l : x) s = l; return s"));
+        assertEquals(30, exec("def x = new byte[] { (byte)10, (byte)20 }; int s = 0; for (int l : x) s += l; return s"));
+        assertEquals(30L, exec("def x = new byte[] { (byte)10, (byte)20 }; long s = 0; for (long l : x) s += l; return s"));
+        assertEquals(30f, exec("def x = new byte[] { (byte)10, (byte)20 }; float s = 0; for (float l : x) s += l; return s"));
+        assertEquals(30d, exec("def x = new byte[] { (byte)10, (byte)20 }; double s = 0; for (double l : x) s += l; return s"));
+        assertEquals((byte) 30, exec("def x = new byte[] { (byte)10, (byte)20 }; byte s = 0; for (def l : x) s += l; return s"));
+
+        assertEquals((byte) 30, exec("def x = new short[] { (short)10, (short)20 }; byte s = 0; for (byte l : x) s += l; return s"));
+        assertEquals((short) 300, exec("def x = new short[] { (short)100, (short)200 }; short s = 0; for (short l : x) s += l; return s"));
+        assertEquals((char) 200, exec("def x = new short[] { (short)100, (short)200 }; char s = 0; for (char l : x) s = l; return s"));
+        assertEquals(300, exec("def x = new short[] { (short)100, (short)200 }; int s = 0; for (int l : x) s += l; return s"));
+        assertEquals(300L, exec("def x = new short[] { (short)100, (short)200 }; long s = 0; for (long l : x) s += l; return s"));
+        assertEquals(300f, exec("def x = new short[] { (short)100, (short)200 }; float s = 0; for (float l : x) s += l; return s"));
+        assertEquals(300d, exec("def x = new short[] { (short)100, (short)200 }; double s = 0; for (double l : x) s += l; return s"));
+        assertEquals((short) 300, exec("def x = new short[] { (short)100, (short)200 }; short s = 0; for (def l : x) s += l; return s"));
+
+        assertEquals((byte) 'b', exec("def x = new char[] { (char)'a', (char)'b' }; byte s = 0; for (byte l : x) s = l; return s"));
+        assertEquals((short) 'b', exec("def x = new char[] { (char)'a', (char)'b' }; short s = 0; for (short l : x) s = l; return s"));
+        assertEquals('b', exec("def x = new char[] { (char)'a', (char)'b' }; char s = 0; for (char l : x) s = l; return s"));
+        assertEquals((int) 'b', exec("def x = new char[] { (char)'a', (char)'b' }; int s = 0; for (int l : x) s = l; return s"));
+        assertEquals((long) 'b', exec("def x = new char[] { (char)'a', (char)'b' }; long s = 0; for (long l : x) s = l; return s"));
+        assertEquals((float) 'b', exec("def x = new char[] { (char)'a', (char)'b' }; float s = 0; for (float l : x) s = l; return s"));
+        assertEquals((double) 'b', exec("def x = new char[] { (char)'a', (char)'b' }; double s = 0; for (double l : x) s = l; return s"));
+        assertEquals('b', exec("def x = new char[] { (char)'a', (char)'b' }; char s = 0; for (def l : x) s = l; return s"));
+
+        assertEquals((byte) 30, exec("def x = new int[] { 10, 20 }; byte s = 0; for (byte l : x) s += l; return s"));
+        assertEquals((short) 300, exec("def x = new int[] { 100, 200 }; short s = 0; for (short l : x) s += l; return s"));
+        assertEquals((char) 200, exec("def x = new int[] { 100, 200 }; char s = 0; for (char l : x) s = l; return s"));
+        assertEquals(300, exec("def x = new int[] { 100, 200 }; int s = 0; for (int l : x) s += l; return s"));
+        assertEquals(300L, exec("def x = new int[] { 100, 200 }; long s = 0; for (long l : x) s += l; return s"));
+        assertEquals(300f, exec("def x = new int[] { 100, 200 }; float s = 0; for (float l : x) s += l; return s"));
+        assertEquals(300d, exec("def x = new int[] { 100, 200 }; double s = 0; for (double l : x) s += l; return s"));
+        assertEquals(300, exec("def x = new int[] { 100, 200 }; int s = 0; for (def l : x) s += l; return s"));
+
+        assertEquals((byte) 30, exec("def x = new long[] { 10, 20 }; byte s = 0; for (byte l : x) s += l; return s"));
+        assertEquals((short) 300, exec("def x = new long[] { 100, 200 }; short s = 0; for (short l : x) s += l; return s"));
+        assertEquals((char) 200, exec("def x = new long[] { 100, 200 }; char s = 0; for (char l : x) s = l; return s"));
+        assertEquals(300, exec("def x = new long[] { 100, 200 }; int s = 0; for (int l : x) s += l; return s"));
+        assertEquals(300L, exec("def x = new long[] { 100, 200 }; long s = 0; for (long l : x) s += l; return s"));
+        assertEquals(300f, exec("def x = new long[] { 100, 200 }; float s = 0; for (float l : x) s += l; return s"));
+        assertEquals(300d, exec("def x = new long[] { 100, 200 }; double s = 0; for (double l : x) s += l; return s"));
+        assertEquals(300L, exec("def x = new long[] { 100, 200 }; long s = 0; for (def l : x) s += l; return s"));
+
+        assertEquals((byte) 30, exec("def x = new float[] { 10, 20 }; byte s = 0; for (byte l : x) s += l; return s"));
+        assertEquals((short) 300, exec("def x = new float[] { 100, 200 }; short s = 0; for (short l : x) s += l; return s"));
+        assertEquals((char) 200, exec("def x = new float[] { 100, 200 }; char s = 0; for (char l : x) s = l; return s"));
+        assertEquals(300, exec("def x = new float[] { 100, 200 }; int s = 0; for (int l : x) s += l; return s"));
+        assertEquals(300L, exec("def x = new float[] { 100, 200 }; long s = 0; for (long l : x) s += l; return s"));
+        assertEquals(300f, exec("def x = new float[] { 100, 200 }; float s = 0; for (float l : x) s += l; return s"));
+        assertEquals(300d, exec("def x = new float[] { 100, 200 }; double s = 0; for (double l : x) s += l; return s"));
+        assertEquals(300f, exec("def x = new float[] { 100, 200 }; float s = 0; for (def l : x) s += l; return s"));
+
+        assertEquals((byte) 30, exec("def x = new double[] { 10, 20 }; byte s = 0; for (byte l : x) s += l; return s"));
+        assertEquals((short) 300, exec("def x = new double[] { 100, 200 }; short s = 0; for (short l : x) s += l; return s"));
+        assertEquals((char) 200, exec("def x = new double[] { 100, 200 }; char s = 0; for (char l : x) s = l; return s"));
+        assertEquals(300, exec("def x = new double[] { 100, 200 }; int s = 0; for (int l : x) s += l; return s"));
+        assertEquals(300L, exec("def x = new double[] { 100, 200 }; long s = 0; for (long l : x) s += l; return s"));
+        assertEquals(300f, exec("def x = new double[] { 100, 200 }; float s = 0; for (float l : x) s += l; return s"));
+        assertEquals(300d, exec("def x = new double[] { 100, 200 }; double s = 0; for (double l : x) s += l; return s"));
+        assertEquals(300d, exec("def x = new double[] { 100, 200 }; double s = 0; for (def l : x) s += l; return s"));
+
+        expectScriptThrows(
+            ClassCastException.class,
+            () -> exec("def x = new String[] { 'foo', 'bar' }; boolean s = false; for (boolean l : x) s |= l; return s")
+        );
+        expectScriptThrows(
+            ClassCastException.class,
+            () -> exec("def x = new String[] { 'foo', 'bar' }; byte s = 0; for (byte l : x) s += l; return s")
+        );
+        expectScriptThrows(
+            ClassCastException.class,
+            () -> exec("def x = new String[] { 'foo', 'bar' }; short s = 0; for (short l : x) s += l; return s")
+        );
+        expectScriptThrows(
+            ClassCastException.class,
+            () -> exec("def x = new String[] { 'foo', 'bar' }; char s = 0; for (char l : x) s = l; return s")
+        );
+        expectScriptThrows(
+            ClassCastException.class,
+            () -> exec("def x = new String[] { 'foo', 'bar' }; int s = 0; for (int l : x) s += l; return s")
+        );
+        expectScriptThrows(
+            ClassCastException.class,
+            () -> exec("def x = new String[] { 'foo', 'bar' }; long s = 0; for (long l : x) s += l; return s")
+        );
+        expectScriptThrows(
+            ClassCastException.class,
+            () -> exec("def x = new String[] { 'foo', 'bar' }; float s = 0; for (float l : x) s += l; return s")
+        );
+        expectScriptThrows(
+            ClassCastException.class,
+            () -> exec("def x = new String[] { 'foo', 'bar' }; double s = 0; for (double l : x) s += l; return s")
+        );
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java
index 89d1872e1ccc..b7bd562a3ee0 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java
@@ -11,6 +11,14 @@
 public class ComparisonTests extends ScriptTestCase {
 
     public void testDefEq() {
+        assertEquals(true, exec("def x = (byte)7; def y = (char)7; return x == y"));
+        assertEquals(true, exec("def x = (short)6; def y = (char)6; return x == y"));
+        assertEquals(true, exec("def x = (char)5; def y = (char)5; return x == y"));
+        assertEquals(true, exec("def x = (int)4; def y = (char)4; return x == y"));
+        assertEquals(false, exec("def x = (long)5; def y = (char)3; return x == y"));
+        assertEquals(false, exec("def x = (float)6; def y = (char)2; return x == y"));
+        assertEquals(false, exec("def x = (double)7; def y = (char)1; return x == y"));
+
         assertEquals(true, exec("def x = (byte)7; def y = (int)7; return x == y"));
         assertEquals(true, exec("def x = (short)6; def y = (int)6; return x == y"));
         assertEquals(true, exec("def x = (char)5; def y = (int)5; return x == y"));
@@ -19,6 +27,22 @@ public void testDefEq() {
         assertEquals(false, exec("def x = (float)6; def y = (int)2; return x == y"));
         assertEquals(false, exec("def x = (double)7; def y = (int)1; return x == y"));
 
+        assertEquals(true, exec("def x = (byte)7; def y = (long)7; return x == y"));
+        assertEquals(true, exec("def x = (short)6; def y = (long)6; return x == y"));
+        assertEquals(true, exec("def x = (char)5; def y = (long)5; return x == y"));
+        assertEquals(true, exec("def x = (int)4; def y = (long)4; return x == y"));
+        assertEquals(false, exec("def x = (long)5; def y = (long)3; return x == y"));
+        assertEquals(false, exec("def x = (float)6; def y = (long)2; return x == y"));
+        assertEquals(false, exec("def x = (double)7; def y = (long)1; return x == y"));
+
+        assertEquals(true, exec("def x = (byte)7; def y = (float)7; return x == y"));
+        assertEquals(true, exec("def x = (short)6; def y = (float)6; return x == y"));
+        assertEquals(true, exec("def x = (char)5; def y = (float)5; return x == y"));
+        assertEquals(true, exec("def x = (int)4; def y = (float)4; return x == y"));
+        assertEquals(false, exec("def x = (long)5; def y = (float)3; return x == y"));
+        assertEquals(false, exec("def x = (float)6; def y = (float)2; return x == y"));
+        assertEquals(false, exec("def x = (double)7; def y = (float)1; return x == y"));
+
         assertEquals(true, exec("def x = (byte)7; def y = (double)7; return x == y"));
         assertEquals(true, exec("def x = (short)6; def y = (double)6; return x == y"));
         assertEquals(true, exec("def x = (char)5; def y = (double)5; return x == y"));
@@ -41,6 +65,14 @@ public void testDefEq() {
     }
 
     public void testDefEqTypedLHS() {
+        assertEquals(true, exec("byte x = (byte)7; def y = (char)7; return x == y"));
+        assertEquals(true, exec("short x = (short)6; def y = (char)6; return x == y"));
+        assertEquals(true, exec("char x = (char)5; def y = (char)5; return x == y"));
+        assertEquals(true, exec("int x = (int)4; def y = (char)4; return x == y"));
+        assertEquals(false, exec("long x = (long)5; def y = (char)3; return x == y"));
+        assertEquals(false, exec("float x = (float)6; def y = (char)2; return x == y"));
+        assertEquals(false, exec("double x = (double)7; def y = (char)1; return x == y"));
+
         assertEquals(true, exec("byte x = (byte)7; def y = (int)7; return x == y"));
         assertEquals(true, exec("short x = (short)6; def y = (int)6; return x == y"));
         assertEquals(true, exec("char x = (char)5; def y = (int)5; return x == y"));
@@ -49,6 +81,22 @@ public void testDefEqTypedLHS() {
         assertEquals(false, exec("float x = (float)6; def y = (int)2; return x == y"));
         assertEquals(false, exec("double x = (double)7; def y = (int)1; return x == y"));
 
+        assertEquals(true, exec("byte x = (byte)7; def y = (long)7; return x == y"));
+        assertEquals(true, exec("short x = (short)6; def y = (long)6; return x == y"));
+        assertEquals(true, exec("char x = (char)5; def y = (long)5; return x == y"));
+        assertEquals(true, exec("int x = (int)4; def y = (long)4; return x == y"));
+        assertEquals(false, exec("long x = (long)5; def y = (long)3; return x == y"));
+        assertEquals(false, exec("float x = (float)6; def y = (long)2; return x == y"));
+        assertEquals(false, exec("double x = (double)7; def y = (long)1; return x == y"));
+
+        assertEquals(true, exec("byte x = (byte)7; def y = (float)7; return x == y"));
+        assertEquals(true, exec("short x = (short)6; def y = (float)6; return x == y"));
+        assertEquals(true, exec("char x = (char)5; def y = (float)5; return x == y"));
+        assertEquals(true, exec("int x = (int)4; def y = (float)4; return x == y"));
+        assertEquals(false, exec("long x = (long)5; def y = (float)3; return x == y"));
+        assertEquals(false, exec("float x = (float)6; def y = (float)2; return x == y"));
+        assertEquals(false, exec("double x = (double)7; def y = (float)1; return x == y"));
+
         assertEquals(true, exec("byte x = (byte)7; def y = (double)7; return x == y"));
         assertEquals(true, exec("short x = (short)6; def y = (double)6; return x == y"));
         assertEquals(true, exec("char x = (char)5; def y = (double)5; return x == y"));
@@ -70,6 +118,14 @@ public void testDefEqTypedLHS() {
     }
 
     public void testDefEqTypedRHS() {
+        assertEquals(true, exec("def x = (byte)7; char y = (char)7; return x == y"));
+        assertEquals(true, exec("def x = (short)6; char y = (char)6; return x == y"));
+        assertEquals(true, exec("def x = (char)5; char y = (char)5; return x == y"));
+        assertEquals(true, exec("def x = (int)4; char y = (char)4; return x == y"));
+        assertEquals(false, exec("def x = (long)5; char y = (char)3; return x == y"));
+        assertEquals(false, exec("def x = (float)6; char y = (char)2; return x == y"));
+        assertEquals(false, exec("def x = (double)7; char y = (char)1; return x == y"));
+
         assertEquals(true, exec("def x = (byte)7; int y = (int)7; return x == y"));
         assertEquals(true, exec("def x = (short)6; int y = (int)6; return x == y"));
         assertEquals(true, exec("def x = (char)5; int y = (int)5; return x == y"));
@@ -78,6 +134,22 @@ public void testDefEqTypedRHS() {
         assertEquals(false, exec("def x = (float)6; int y = (int)2; return x == y"));
         assertEquals(false, exec("def x = (double)7; int y = (int)1; return x == y"));
 
+        assertEquals(true, exec("def x = (byte)7; long y = (long)7; return x == y"));
+        assertEquals(true, exec("def x = (short)6; long y = (long)6; return x == y"));
+        assertEquals(true, exec("def x = (char)5; long y = (long)5; return x == y"));
+        assertEquals(true, exec("def x = (int)4; long y = (long)4; return x == y"));
+        assertEquals(false, exec("def x = (long)5; long y = (long)3; return x == y"));
+        assertEquals(false, exec("def x = (float)6; long y = (long)2; return x == y"));
+        assertEquals(false, exec("def x = (double)7; long y = (long)1; return x == y"));
+
+        assertEquals(true, exec("def x = (byte)7; float y = (float)7; return x == y"));
+        assertEquals(true, exec("def x = (short)6; float y = (float)6; return x == y"));
+        assertEquals(true, exec("def x = (char)5; float y = (float)5; return x == y"));
+        assertEquals(true, exec("def x = (int)4; float y = (float)4; return x == y"));
+        assertEquals(false, exec("def x = (long)5; float y = (float)3; return x == y"));
+        assertEquals(false, exec("def x = (float)6; float y = (float)2; return x == y"));
+        assertEquals(false, exec("def x = (double)7; float y = (float)1; return x == y"));
+
         assertEquals(true, exec("def x = (byte)7; double y = (double)7; return x == y"));
         assertEquals(true, exec("def x = (short)6; double y = (double)6; return x == y"));
         assertEquals(true, exec("def x = (char)5; double y = (double)5; return x == y"));
@@ -106,6 +178,7 @@ public void testDefEqr() {
         assertEquals(false, exec("def x = (long)5; def y = (int)3; return x === y"));
         assertEquals(false, exec("def x = (float)6; def y = (int)2; return x === y"));
         assertEquals(false, exec("def x = (double)7; def y = (int)1; return x === y"));
+
         assertEquals(false, exec("def x = false; def y = true; return x === y"));
 
         assertEquals(false, exec("def x = new HashMap(); def y = new HashMap(); return x === y"));
@@ -115,6 +188,14 @@ public void testDefEqr() {
     }
 
     public void testDefNe() {
+        assertEquals(false, exec("def x = (byte)7; def y = (char)7; return x != y"));
+        assertEquals(false, exec("def x = (short)6; def y = (char)6; return x != y"));
+        assertEquals(false, exec("def x = (char)5; def y = (char)5; return x != y"));
+        assertEquals(false, exec("def x = (int)4; def y = (char)4; return x != y"));
+        assertEquals(true, exec("def x = (long)5; def y = (char)3; return x != y"));
+        assertEquals(true, exec("def x = (float)6; def y = (char)2; return x != y"));
+        assertEquals(true, exec("def x = (double)7; def y = (char)1; return x != y"));
+
         assertEquals(false, exec("def x = (byte)7; def y = (int)7; return x != y"));
         assertEquals(false, exec("def x = (short)6; def y = (int)6; return x != y"));
         assertEquals(false, exec("def x = (char)5; def y = (int)5; return x != y"));
@@ -123,6 +204,22 @@ public void testDefNe() {
         assertEquals(true, exec("def x = (float)6; def y = (int)2; return x != y"));
         assertEquals(true, exec("def x = (double)7; def y = (int)1; return x != y"));
 
+        assertEquals(false, exec("def x = (byte)7; def y = (long)7; return x != y"));
+        assertEquals(false, exec("def x = (short)6; def y = (long)6; return x != y"));
+        assertEquals(false, exec("def x = (char)5; def y = (long)5; return x != y"));
+        assertEquals(false, exec("def x = (int)4; def y = (long)4; return x != y"));
+        assertEquals(true, exec("def x = (long)5; def y = (long)3; return x != y"));
+        assertEquals(true, exec("def x = (float)6; def y = (long)2; return x != y"));
+        assertEquals(true, exec("def x = (double)7; def y = (long)1; return x != y"));
+
+        assertEquals(false, exec("def x = (byte)7; def y = (float)7; return x != y"));
+        assertEquals(false, exec("def x = (short)6; def y = (float)6; return x != y"));
+        assertEquals(false, exec("def x = (char)5; def y = (float)5; return x != y"));
+        assertEquals(false, exec("def x = (int)4; def y = (float)4; return x != y"));
+        assertEquals(true, exec("def x = (long)5; def y = (float)3; return x != y"));
+        assertEquals(true, exec("def x = (float)6; def y = (float)2; return x != y"));
+        assertEquals(true, exec("def x = (double)7; def y = (float)1; return x != y"));
+
         assertEquals(false, exec("def x = (byte)7; def y = (double)7; return x != y"));
         assertEquals(false, exec("def x = (short)6; def y = (double)6; return x != y"));
         assertEquals(false, exec("def x = (char)5; def y = (double)5; return x != y"));
@@ -143,6 +240,14 @@ public void testDefNe() {
     }
 
     public void testDefNeTypedLHS() {
+        assertEquals(false, exec("byte x = (byte)7; def y = (char)7; return x != y"));
+        assertEquals(false, exec("short x = (short)6; def y = (char)6; return x != y"));
+        assertEquals(false, exec("char x = (char)5; def y = (char)5; return x != y"));
+        assertEquals(false, exec("int x = (int)4; def y = (char)4; return x != y"));
+        assertEquals(true, exec("long x = (long)5; def y = (char)3; return x != y"));
+        assertEquals(true, exec("float x = (float)6; def y = (char)2; return x != y"));
+        assertEquals(true, exec("double x = (double)7; def y = (char)1; return x != y"));
+
         assertEquals(false, exec("byte x = (byte)7; def y = (int)7; return x != y"));
         assertEquals(false, exec("short x = (short)6; def y = (int)6; return x != y"));
         assertEquals(false, exec("char x = (char)5; def y = (int)5; return x != y"));
@@ -151,6 +256,22 @@ public void testDefNeTypedLHS() {
         assertEquals(true, exec("float x = (float)6; def y = (int)2; return x != y"));
         assertEquals(true, exec("double x = (double)7; def y = (int)1; return x != y"));
 
+        assertEquals(false, exec("byte x = (byte)7; def y = (long)7; return x != y"));
+        assertEquals(false, exec("short x = (short)6; def y = (long)6; return x != y"));
+        assertEquals(false, exec("char x = (char)5; def y = (long)5; return x != y"));
+        assertEquals(false, exec("int x = (int)4; def y = (long)4; return x != y"));
+        assertEquals(true, exec("long x = (long)5; def y = (long)3; return x != y"));
+        assertEquals(true, exec("float x = (float)6; def y = (long)2; return x != y"));
+        assertEquals(true, exec("double x = (double)7; def y = (long)1; return x != y"));
+
+        assertEquals(false, exec("byte x = (byte)7; def y = (float)7; return x != y"));
+        assertEquals(false, exec("short x = (short)6; def y = (float)6; return x != y"));
+        assertEquals(false, exec("char x = (char)5; def y = (float)5; return x != y"));
+        assertEquals(false, exec("int x = (int)4; def y = (float)4; return x != y"));
+        assertEquals(true, exec("long x = (long)5; def y = (float)3; return x != y"));
+        assertEquals(true, exec("float x = (float)6; def y = (float)2; return x != y"));
+        assertEquals(true, exec("double x = (double)7; def y = (float)1; return x != y"));
+
         assertEquals(false, exec("byte x = (byte)7; def y = (double)7; return x != y"));
         assertEquals(false, exec("short x = (short)6; def y = (double)6; return x != y"));
         assertEquals(false, exec("char x = (char)5; def y = (double)5; return x != y"));
@@ -171,6 +292,14 @@ public void testDefNeTypedLHS() {
     }
 
     public void testDefNeTypedRHS() {
+        assertEquals(false, exec("def x = (byte)7; char y = (char)7; return x != y"));
+        assertEquals(false, exec("def x = (short)6; char y = (char)6; return x != y"));
+        assertEquals(false, exec("def x = (char)5; char y = (char)5; return x != y"));
+        assertEquals(false, exec("def x = (int)4; char y = (char)4; return x != y"));
+        assertEquals(true, exec("def x = (long)5; char y = (char)3; return x != y"));
+        assertEquals(true, exec("def x = (float)6; char y = (char)2; return x != y"));
+        assertEquals(true, exec("def x = (double)7; char y = (char)1; return x != y"));
+
         assertEquals(false, exec("def x = (byte)7; int y = (int)7; return x != y"));
         assertEquals(false, exec("def x = (short)6; int y = (int)6; return x != y"));
         assertEquals(false, exec("def x = (char)5; int y = (int)5; return x != y"));
@@ -179,6 +308,22 @@ public void testDefNeTypedRHS() {
         assertEquals(true, exec("def x = (float)6; int y = (int)2; return x != y"));
         assertEquals(true, exec("def x = (double)7; int y = (int)1; return x != y"));
 
+        assertEquals(false, exec("def x = (byte)7; long y = (long)7; return x != y"));
+        assertEquals(false, exec("def x = (short)6; long y = (long)6; return x != y"));
+        assertEquals(false, exec("def x = (char)5; long y = (long)5; return x != y"));
+        assertEquals(false, exec("def x = (int)4; long y = (long)4; return x != y"));
+        assertEquals(true, exec("def x = (long)5; long y = (long)3; return x != y"));
+        assertEquals(true, exec("def x = (float)6; long y = (long)2; return x != y"));
+        assertEquals(true, exec("def x = (double)7; long y = (long)1; return x != y"));
+
+        assertEquals(false, exec("def x = (byte)7; float y = (float)7; return x != y"));
+        assertEquals(false, exec("def x = (short)6; float y = (float)6; return x != y"));
+        assertEquals(false, exec("def x = (char)5; float y = (float)5; return x != y"));
+        assertEquals(false, exec("def x = (int)4; float y = (float)4; return x != y"));
+        assertEquals(true, exec("def x = (long)5; float y = (float)3; return x != y"));
+        assertEquals(true, exec("def x = (float)6; float y = (float)2; return x != y"));
+        assertEquals(true, exec("def x = (double)7; float y = (float)1; return x != y"));
+
         assertEquals(false, exec("def x = (byte)7; double y = (double)7; return x != y"));
         assertEquals(false, exec("def x = (short)6; double y = (double)6; return x != y"));
         assertEquals(false, exec("def x = (char)5; double y = (double)5; return x != y"));
@@ -222,6 +367,22 @@ public void testDefLt() {
         assertEquals(false, exec("def x = (float)6; def y = (int)2; return x < y"));
         assertEquals(false, exec("def x = (double)7; def y = (int)1; return x < y"));
 
+        assertEquals(true, exec("def x = (byte)1; def y = (long)7; return x < y"));
+        assertEquals(true, exec("def x = (short)2; def y = (long)6; return x < y"));
+        assertEquals(true, exec("def x = (char)3; def y = (long)5; return x < y"));
+        assertEquals(false, exec("def x = (int)4; def y = (long)4; return x < y"));
+        assertEquals(false, exec("def x = (long)5; def y = (long)3; return x < y"));
+        assertEquals(false, exec("def x = (float)6; def y = (long)2; return x < y"));
+        assertEquals(false, exec("def x = (double)7; def y = (long)1; return x < y"));
+
+        assertEquals(true, exec("def x = (byte)1; def y = (float)7; return x < y"));
+        assertEquals(true, exec("def x = (short)2; def y = (float)6; return x < y"));
+        assertEquals(true, exec("def x = (char)3; def y = (float)5; return x < y"));
+        assertEquals(false, exec("def x = (int)4; def y = (float)4; return x < y"));
+        assertEquals(false, exec("def x = (long)5; def y = (float)3; return x < y"));
+        assertEquals(false, exec("def x = (float)6; def y = (float)2; return x < y"));
+        assertEquals(false, exec("def x = (double)7; def y = (float)1; return x < y"));
+
         assertEquals(true, exec("def x = (byte)1; def y = (double)7; return x < y"));
         assertEquals(true, exec("def x = (short)2; def y = (double)6; return x < y"));
         assertEquals(true, exec("def x = (char)3; def y = (double)5; return x < y"));
@@ -240,6 +401,22 @@ public void testDefLtTypedLHS() {
         assertEquals(false, exec("float x = (float)6; def y = (int)2; return x < y"));
         assertEquals(false, exec("double x = (double)7; def y = (int)1; return x < y"));
 
+        assertEquals(true, exec("byte x = (byte)1; def y = (long)7; return x < y"));
+        assertEquals(true, exec("short x = (short)2; def y = (long)6; return x < y"));
+        assertEquals(true, exec("char x = (char)3; def y = (long)5; return x < y"));
+        assertEquals(false, exec("int x = (int)4; def y = (long)4; return x < y"));
+        assertEquals(false, exec("long x = (long)5; def y = (long)3; return x < y"));
+        assertEquals(false, exec("float x = (float)6; def y = (long)2; return x < y"));
+        assertEquals(false, exec("double x = (double)7; def y = (long)1; return x < y"));
+
+        assertEquals(true, exec("byte x = (byte)1; def y = (float)7; return x < y"));
+        assertEquals(true, exec("short x = (short)2; def y = (float)6; return x < y"));
+        assertEquals(true, exec("char x = (char)3; def y = (float)5; return x < y"));
+        assertEquals(false, exec("int x = (int)4; def y = (float)4; return x < y"));
+        assertEquals(false, exec("long x = (long)5; def y = (float)3; return x < y"));
+        assertEquals(false, exec("float x = (float)6; def y = (float)2; return x < y"));
+        assertEquals(false, exec("double x = (double)7; def y = (float)1; return x < y"));
+
         assertEquals(true, exec("byte x = (byte)1; def y = (double)7; return x < y"));
         assertEquals(true, exec("short x = (short)2; def y = (double)6; return x < y"));
         assertEquals(true, exec("char x = (char)3; def y = (double)5; return x < y"));
@@ -258,6 +435,22 @@ public void testDefLtTypedRHS() {
         assertEquals(false, exec("def x = (float)6; int y = (int)2; return x < y"));
         assertEquals(false, exec("def x = (double)7; int y = (int)1; return x < y"));
 
+        assertEquals(true, exec("def x = (byte)1; long y = (long)7; return x < y"));
+        assertEquals(true, exec("def x = (short)2; long y = (long)6; return x < y"));
+        assertEquals(true, exec("def x = (char)3; long y = (long)5; return x < y"));
+        assertEquals(false, exec("def x = (int)4; long y = (long)4; return x < y"));
+        assertEquals(false, exec("def x = (long)5; long y = (long)3; return x < y"));
+        assertEquals(false, exec("def x = (float)6; long y = (long)2; return x < y"));
+        assertEquals(false, exec("def x = (double)7; long y = (long)1; return x < y"));
+
+        assertEquals(true, exec("def x = (byte)1; float y = (float)7; return x < y"));
+        assertEquals(true, exec("def x = (short)2; float y = (float)6; return x < y"));
+        assertEquals(true, exec("def x = (char)3; float y = (float)5; return x < y"));
+        assertEquals(false, exec("def x = (int)4; float y = (float)4; return x < y"));
+        assertEquals(false, exec("def x = (long)5; float y = (float)3; return x < y"));
+        assertEquals(false, exec("def x = (float)6; float y = (float)2; return x < y"));
+        assertEquals(false, exec("def x = (double)7; float y = (float)1; return x < y"));
+
         assertEquals(true, exec("def x = (byte)1; double y = (double)7; return x < y"));
         assertEquals(true, exec("def x = (short)2; double y = (double)6; return x < y"));
         assertEquals(true, exec("def x = (char)3; double y = (double)5; return x < y"));
@@ -276,6 +469,22 @@ public void testDefLte() {
         assertEquals(false, exec("def x = (float)6; def y = (int)2; return x <= y"));
         assertEquals(false, exec("def x = (double)7; def y = (int)1; return x <= y"));
 
+        assertEquals(true, exec("def x = (byte)1; def y = (long)7; return x <= y"));
+        assertEquals(true, exec("def x = (short)2; def y = (long)6; return x <= y"));
+        assertEquals(true, exec("def x = (char)3; def y = (long)5; return x <= y"));
+        assertEquals(true, exec("def x = (int)4; def y = (long)4; return x <= y"));
+        assertEquals(false, exec("def x = (long)5; def y = (long)3; return x <= y"));
+        assertEquals(false, exec("def x = (float)6; def y = (long)2; return x <= y"));
+        assertEquals(false, exec("def x = (double)7; def y = (long)1; return x <= y"));
+
+        assertEquals(true, exec("def x = (byte)1; def y = (float)7; return x <= y"));
+        assertEquals(true, exec("def x = (short)2; def y = (float)6; return x <= y"));
+        assertEquals(true, exec("def x = (char)3; def y = (float)5; return x <= y"));
+        assertEquals(true, exec("def x = (int)4; def y = (float)4; return x <= y"));
+        assertEquals(false, exec("def x = (long)5; def y = (float)3; return x <= y"));
+        assertEquals(false, exec("def x = (float)6; def y = (float)2; return x <= y"));
+        assertEquals(false, exec("def x = (double)7; def y = (float)1; return x <= y"));
+
         assertEquals(true, exec("def x = (byte)1; def y = (double)7; return x <= y"));
         assertEquals(true, exec("def x = (short)2; def y = (double)6; return x <= y"));
         assertEquals(true, exec("def x = (char)3; def y = (double)5; return x <= y"));
@@ -294,6 +503,22 @@ public void testDefLteTypedLHS() {
         assertEquals(false, exec("float x = (float)6; def y = (int)2; return x <= y"));
         assertEquals(false, exec("double x = (double)7; def y = (int)1; return x <= y"));
 
+        assertEquals(true, exec("byte x = (byte)1; def y = (long)7; return x <= y"));
+        assertEquals(true, exec("short x = (short)2; def y = (long)6; return x <= y"));
+        assertEquals(true, exec("char x = (char)3; def y = (long)5; return x <= y"));
+        assertEquals(true, exec("int x = (int)4; def y = (long)4; return x <= y"));
+        assertEquals(false, exec("long x = (long)5; def y = (long)3; return x <= y"));
+        assertEquals(false, exec("float x = (float)6; def y = (long)2; return x <= y"));
+        assertEquals(false, exec("double x = (double)7; def y = (long)1; return x <= y"));
+
+        assertEquals(true, exec("byte x = (byte)1; def y = (float)7; return x <= y"));
+        assertEquals(true, exec("short x = (short)2; def y = (float)6; return x <= y"));
+        assertEquals(true, exec("char x = (char)3; def y = (float)5; return x <= y"));
+        assertEquals(true, exec("int x = (int)4; def y = (float)4; return x <= y"));
+        assertEquals(false, exec("long x = (long)5; def y = (float)3; return x <= y"));
+        assertEquals(false, exec("float x = (float)6; def y = (float)2; return x <= y"));
+        assertEquals(false, exec("double x = (double)7; def y = (float)1; return x <= y"));
+
         assertEquals(true, exec("byte x = (byte)1; def y = (double)7; return x <= y"));
         assertEquals(true, exec("short x = (short)2; def y = (double)6; return x <= y"));
         assertEquals(true, exec("char x = (char)3; def y = (double)5; return x <= y"));
@@ -312,6 +537,22 @@ public void testDefLteTypedRHS() {
         assertEquals(false, exec("def x = (float)6; int y = (int)2; return x <= y"));
         assertEquals(false, exec("def x = (double)7; int y = (int)1; return x <= y"));
 
+        assertEquals(true, exec("def x = (byte)1; long y = (long)7; return x <= y"));
+        assertEquals(true, exec("def x = (short)2; long y = (long)6; return x <= y"));
+        assertEquals(true, exec("def x = (char)3; long y = (long)5; return x <= y"));
+        assertEquals(true, exec("def x = (int)4; long y = (long)4; return x <= y"));
+        assertEquals(false, exec("def x = (long)5; long y = (long)3; return x <= y"));
+        assertEquals(false, exec("def x = (float)6; long y = (long)2; return x <= y"));
+        assertEquals(false, exec("def x = (double)7; long y = (long)1; return x <= y"));
+
+        assertEquals(true, exec("def x = (byte)1; float y = (float)7; return x <= y"));
+        assertEquals(true, exec("def x = (short)2; float y = (float)6; return x <= y"));
+        assertEquals(true, exec("def x = (char)3; float y = (float)5; return x <= y"));
+        assertEquals(true, exec("def x = (int)4; float y = (float)4; return x <= y"));
+        assertEquals(false, exec("def x = (long)5; float y = (float)3; return x <= y"));
+        assertEquals(false, exec("def x = (float)6; float y = (float)2; return x <= y"));
+        assertEquals(false, exec("def x = (double)7; float y = (float)1; return x <= y"));
+
         assertEquals(true, exec("def x = (byte)1; double y = (double)7; return x <= y"));
         assertEquals(true, exec("def x = (short)2; double y = (double)6; return x <= y"));
         assertEquals(true, exec("def x = (char)3; double y = (double)5; return x <= y"));
@@ -330,6 +571,22 @@ public void testDefGt() {
         assertEquals(true, exec("def x = (float)6; def y = (int)2; return x > y"));
         assertEquals(true, exec("def x = (double)7; def y = (int)1; return x > y"));
 
+        assertEquals(false, exec("def x = (byte)1; def y = (long)7; return x > y"));
+        assertEquals(false, exec("def x = (short)2; def y = (long)6; return x > y"));
+        assertEquals(false, exec("def x = (char)3; def y = (long)5; return x > y"));
+        assertEquals(false, exec("def x = (int)4; def y = (long)4; return x > y"));
+        assertEquals(true, exec("def x = (long)5; def y = (long)3; return x > y"));
+        assertEquals(true, exec("def x = (float)6; def y = (long)2; return x > y"));
+        assertEquals(true, exec("def x = (double)7; def y = (long)1; return x > y"));
+
+        assertEquals(false, exec("def x = (byte)1; def y = (float)7; return x > y"));
+        assertEquals(false, exec("def x = (short)2; def y = (float)6; return x > y"));
+        assertEquals(false, exec("def x = (char)3; def y = (float)5; return x > y"));
+        assertEquals(false, exec("def x = (int)4; def y = (float)4; return x > y"));
+        assertEquals(true, exec("def x = (long)5; def y = (float)3; return x > y"));
+        assertEquals(true, exec("def x = (float)6; def y = (float)2; return x > y"));
+        assertEquals(true, exec("def x = (double)7; def y = (float)1; return x > y"));
+
         assertEquals(false, exec("def x = (byte)1; def y = (double)7; return x > y"));
         assertEquals(false, exec("def x = (short)2; def y = (double)6; return x > y"));
         assertEquals(false, exec("def x = (char)3; def y = (double)5; return x > y"));
@@ -348,6 +605,22 @@ public void testDefGtTypedLHS() {
         assertEquals(true, exec("float x = (float)6; def y = (int)2; return x > y"));
         assertEquals(true, exec("double x = (double)7; def y = (int)1; return x > y"));
 
+        assertEquals(false, exec("byte x = (byte)1; def y = (long)7; return x > y"));
+        assertEquals(false, exec("short x = (short)2; def y = (long)6; return x > y"));
+        assertEquals(false, exec("char x = (char)3; def y = (long)5; return x > y"));
+        assertEquals(false, exec("int x = (int)4; def y = (long)4; return x > y"));
+        assertEquals(true, exec("long x = (long)5; def y = (long)3; return x > y"));
+        assertEquals(true, exec("float x = (float)6; def y = (long)2; return x > y"));
+        assertEquals(true, exec("double x = (double)7; def y = (long)1; return x > y"));
+
+        assertEquals(false, exec("byte x = (byte)1; def y = (float)7; return x > y"));
+        assertEquals(false, exec("short x = (short)2; def y = (float)6; return x > y"));
+        assertEquals(false, exec("char x = (char)3; def y = (float)5; return x > y"));
+        assertEquals(false, exec("int x = (int)4; def y = (float)4; return x > y"));
+        assertEquals(true, exec("long x = (long)5; def y = (float)3; return x > y"));
+        assertEquals(true, exec("float x = (float)6; def y = (float)2; return x > y"));
+        assertEquals(true, exec("double x = (double)7; def y = (float)1; return x > y"));
+
         assertEquals(false, exec("byte x = (byte)1; def y = (double)7; return x > y"));
         assertEquals(false, exec("short x = (short)2; def y = (double)6; return x > y"));
         assertEquals(false, exec("char x = (char)3; def y = (double)5; return x > y"));
@@ -366,6 +639,22 @@ public void testDefGtTypedRHS() {
         assertEquals(true, exec("def x = (float)6; int y = (int)2; return x > y"));
         assertEquals(true, exec("def x = (double)7; int y = (int)1; return x > y"));
 
+        assertEquals(false, exec("def x = (byte)1; long y = (long)7; return x > y"));
+        assertEquals(false, exec("def x = (short)2; long y = (long)6; return x > y"));
+        assertEquals(false, exec("def x = (char)3; long y = (long)5; return x > y"));
+        assertEquals(false, exec("def x = (int)4; long y = (long)4; return x > y"));
+        assertEquals(true, exec("def x = (long)5; long y = (long)3; return x > y"));
+        assertEquals(true, exec("def x = (float)6; long y = (long)2; return x > y"));
+        assertEquals(true, exec("def x = (double)7; long y = (long)1; return x > y"));
+
+        assertEquals(false, exec("def x = (byte)1; float y = (float)7; return x > y"));
+        assertEquals(false, exec("def x = (short)2; float y = (float)6; return x > y"));
+        assertEquals(false, exec("def x = (char)3; float y = (float)5; return x > y"));
+        assertEquals(false, exec("def x = (int)4; float y = (float)4; return x > y"));
+        assertEquals(true, exec("def x = (long)5; float y = (float)3; return x > y"));
+        assertEquals(true, exec("def x = (float)6; float y = (float)2; return x > y"));
+        assertEquals(true, exec("def x = (double)7; float y = (float)1; return x > y"));
+
         assertEquals(false, exec("def x = (byte)1; double y = (double)7; return x > y"));
         assertEquals(false, exec("def x = (short)2; double y = (double)6; return x > y"));
         assertEquals(false, exec("def x = (char)3; double y = (double)5; return x > y"));
@@ -384,6 +673,22 @@ public void testDefGte() {
         assertEquals(true, exec("def x = (float)6; def y = (int)2; return x >= y"));
         assertEquals(true, exec("def x = (double)7; def y = (int)1; return x >= y"));
 
+        assertEquals(false, exec("def x = (byte)1; def y = (long)7; return x >= y"));
+        assertEquals(false, exec("def x = (short)2; def y = (long)6; return x >= y"));
+        assertEquals(false, exec("def x = (char)3; def y = (long)5; return x >= y"));
+        assertEquals(true, exec("def x = (int)4; def y = (long)4; return x >= y"));
+        assertEquals(true, exec("def x = (long)5; def y = (long)3; return x >= y"));
+        assertEquals(true, exec("def x = (float)6; def y = (long)2; return x >= y"));
+        assertEquals(true, exec("def x = (double)7; def y = (long)1; return x >= y"));
+
+        assertEquals(false, exec("def x = (byte)1; def y = (float)7; return x >= y"));
+        assertEquals(false, exec("def x = (short)2; def y = (float)6; return x >= y"));
+        assertEquals(false, exec("def x = (char)3; def y = (float)5; return x >= y"));
+        assertEquals(true, exec("def x = (int)4; def y = (float)4; return x >= y"));
+        assertEquals(true, exec("def x = (long)5; def y = (float)3; return x >= y"));
+        assertEquals(true, exec("def x = (float)6; def y = (float)2; return x >= y"));
+        assertEquals(true, exec("def x = (double)7; def y = (float)1; return x >= y"));
+
         assertEquals(false, exec("def x = (byte)1; def y = (double)7; return x >= y"));
         assertEquals(false, exec("def x = (short)2; def y = (double)6; return x >= y"));
         assertEquals(false, exec("def x = (char)3; def y = (double)5; return x >= y"));
@@ -402,6 +707,22 @@ public void testDefGteTypedLHS() {
         assertEquals(true, exec("float x = (float)6; def y = (int)2; return x >= y"));
         assertEquals(true, exec("double x = (double)7; def y = (int)1; return x >= y"));
 
+        assertEquals(false, exec("byte x = (byte)1; def y = (long)7; return x >= y"));
+        assertEquals(false, exec("short x = (short)2; def y = (long)6; return x >= y"));
+        assertEquals(false, exec("char x = (char)3; def y = (long)5; return x >= y"));
+        assertEquals(true, exec("int x = (int)4; def y = (long)4; return x >= y"));
+        assertEquals(true, exec("long x = (long)5; def y = (long)3; return x >= y"));
+        assertEquals(true, exec("float x = (float)6; def y = (long)2; return x >= y"));
+        assertEquals(true, exec("double x = (double)7; def y = (long)1; return x >= y"));
+
+        assertEquals(false, exec("byte x = (byte)1; def y = (float)7; return x >= y"));
+        assertEquals(false, exec("short x = (short)2; def y = (float)6; return x >= y"));
+        assertEquals(false, exec("char x = (char)3; def y = (float)5; return x >= y"));
+        assertEquals(true, exec("int x = (int)4; def y = (float)4; return x >= y"));
+        assertEquals(true, exec("long x = (long)5; def y = (float)3; return x >= y"));
+        assertEquals(true, exec("float x = (float)6; def y = (float)2; return x >= y"));
+        assertEquals(true, exec("double x = (double)7; def y = (float)1; return x >= y"));
+
         assertEquals(false, exec("byte x = (byte)1; def y = (double)7; return x >= y"));
         assertEquals(false, exec("short x = (short)2; def y = (double)6; return x >= y"));
         assertEquals(false, exec("char x = (char)3; def y = (double)5; return x >= y"));
@@ -420,6 +741,22 @@ public void testDefGteTypedRHS() {
         assertEquals(true, exec("def x = (float)6; int y = (int)2; return x >= y"));
         assertEquals(true, exec("def x = (double)7; int y = (int)1; return x >= y"));
 
+        assertEquals(false, exec("def x = (byte)1; long y = (long)7; return x >= y"));
+        assertEquals(false, exec("def x = (short)2; long y = (long)6; return x >= y"));
+        assertEquals(false, exec("def x = (char)3; long y = (long)5; return x >= y"));
+        assertEquals(true, exec("def x = (int)4; long y = (long)4; return x >= y"));
+        assertEquals(true, exec("def x = (long)5; long y = (long)3; return x >= y"));
+        assertEquals(true, exec("def x = (float)6; long y = (long)2; return x >= y"));
+        assertEquals(true, exec("def x = (double)7; long y = (long)1; return x >= y"));
+
+        assertEquals(false, exec("def x = (byte)1; float y = (float)7; return x >= y"));
+        assertEquals(false, exec("def x = (short)2; float y = (float)6; return x >= y"));
+        assertEquals(false, exec("def x = (char)3; float y = (float)5; return x >= y"));
+        assertEquals(true, exec("def x = (int)4; float y = (float)4; return x >= y"));
+        assertEquals(true, exec("def x = (long)5; float y = (float)3; return x >= y"));
+        assertEquals(true, exec("def x = (float)6; float y = (float)2; return x >= y"));
+        assertEquals(true, exec("def x = (double)7; float y = (float)1; return x >= y"));
+
         assertEquals(false, exec("def x = (byte)1; double y = (double)7; return x >= y"));
         assertEquals(false, exec("def x = (short)2; double y = (double)6; return x >= y"));
         assertEquals(false, exec("def x = (char)3; double y = (double)5; return x >= y"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
index 5973cf8b3728..ffbe91bd4603 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
@@ -416,4 +416,39 @@ public void testLambdaArguments() {
             "synthetic lambda$synthetic$0(D)D"
         );
     }
+
+    public void testPrimitiveArrayIteration() {
+        assertBytecodeExists(
+            "def x = new boolean[] { true, false }; boolean s = false; for (boolean l : x) s |= l; return s",
+            "INVOKEINTERFACE org/elasticsearch/painless/api/ValueIterator.nextBoolean ()Z"
+        );
+        assertBytecodeExists(
+            "def x = new byte[] { (byte)10, (byte)20 }; byte s = 0; for (byte l : x) s += l; return s",
+            "INVOKEINTERFACE org/elasticsearch/painless/api/ValueIterator.nextByte ()B"
+        );
+        assertBytecodeExists(
+            "def x = new short[] { (short)100, (short)200 }; short s = 0; for (short l : x) s += l; return s",
+            "INVOKEINTERFACE org/elasticsearch/painless/api/ValueIterator.nextShort ()S"
+        );
+        assertBytecodeExists(
+            "def x = new char[] { (char)'a', (char)'b' }; char s = 0; for (char l : x) s = l; return s",
+            "INVOKEINTERFACE org/elasticsearch/painless/api/ValueIterator.nextChar ()C"
+        );
+        assertBytecodeExists(
+            "def x = new int[] { 100, 200 }; int s = 0; for (int l : x) s += l; return s",
+            "INVOKEINTERFACE org/elasticsearch/painless/api/ValueIterator.nextInt ()I"
+        );
+        assertBytecodeExists(
+            "def x = new long[] { 100, 200 }; long s = 0; for (long l : x) s += l; return s",
+            "INVOKEINTERFACE org/elasticsearch/painless/api/ValueIterator.nextLong ()J"
+        );
+        assertBytecodeExists(
+            "def x = new float[] { 100, 200 }; float s = 0; for (float l : x) s += l; return s",
+            "INVOKEINTERFACE org/elasticsearch/painless/api/ValueIterator.nextFloat ()F"
+        );
+        assertBytecodeExists(
+            "def x = new double[] { 100, 200 }; double s = 0; for (double l : x) s += l; return s",
+            "INVOKEINTERFACE org/elasticsearch/painless/api/ValueIterator.nextDouble ()D"
+        );
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/GetByPathAugmentationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/GetByPathAugmentationTests.java
index 8e942f500c68..3022eb0fccc8 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/GetByPathAugmentationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/GetByPathAugmentationTests.java
@@ -8,10 +8,11 @@
 
 package org.elasticsearch.painless;
 
+import org.elasticsearch.core.Strings;
+
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 
 public class GetByPathAugmentationTests extends ScriptTestCase {
@@ -38,20 +39,20 @@ public GetByPathAugmentationTests() {
     }
 
     private String toScript(String collection, String key) {
-        return String.format(Locale.ROOT, "return %s.getByPath('%s')", collection, key);
+        return Strings.format("return %s.getByPath('%s')", collection, key);
     }
 
     private String toScript(String collection, String key, String defaultValue) {
-        return String.format(Locale.ROOT, "return %s.getByPath('%s', %s)", collection, key, defaultValue);
+        return Strings.format("return %s.getByPath('%s', %s)", collection, key, defaultValue);
     }
 
     private String numberFormat(String unparsable, String path, int i) {
         String format = "Could not parse [%s] as a int index into list at path [%s] and index [%d]";
-        return String.format(Locale.ROOT, format, unparsable, path, i);
+        return Strings.format(format, unparsable, path, i);
     }
 
     private String missingValue(String path) {
-        return formatted("Could not find value at path [%s]", path);
+        return Strings.format("Could not find value at path [%s]", path);
     }
 
     private void assertPathValue(String collection, String key, Object value) {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptedMetricAggContextsTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptedMetricAggContextsTests.java
index 5177da3f6dd0..3aee371bec45 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptedMetricAggContextsTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptedMetricAggContextsTests.java
@@ -16,7 +16,8 @@
 import org.elasticsearch.script.ScriptedMetricAggContexts;
 import org.elasticsearch.search.lookup.LeafSearchLookup;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -24,6 +25,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Supplier;
 
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -110,9 +112,8 @@ public void testReturnSource() throws IOException {
         SearchLookup lookup = mock(SearchLookup.class);
         LeafSearchLookup leafLookup = mock(LeafSearchLookup.class);
         when(lookup.getLeafSearchLookup(leafReaderContext)).thenReturn(leafLookup);
-        SourceLookup sourceLookup = mock(SourceLookup.class);
-        when(leafLookup.asMap()).thenReturn(Collections.singletonMap("_source", sourceLookup));
-        when(sourceLookup.source()).thenReturn(Collections.singletonMap("test", 1));
+        Supplier<Source> source = () -> Source.fromMap(Map.of("test", 1), XContentType.JSON);
+        when(leafLookup.asMap()).thenReturn(Collections.singletonMap("_source", source));
         ScriptedMetricAggContexts.MapScript.LeafFactory leafFactory = factory.newFactory(params, state, lookup);
         ScriptedMetricAggContexts.MapScript script = leafFactory.newInstance(leafReaderContext);
 
@@ -141,9 +142,8 @@ public void testMapSourceAccess() throws IOException {
         SearchLookup lookup = mock(SearchLookup.class);
         LeafSearchLookup leafLookup = mock(LeafSearchLookup.class);
         when(lookup.getLeafSearchLookup(leafReaderContext)).thenReturn(leafLookup);
-        SourceLookup sourceLookup = mock(SourceLookup.class);
-        when(leafLookup.asMap()).thenReturn(Collections.singletonMap("_source", sourceLookup));
-        when(sourceLookup.source()).thenReturn(Collections.singletonMap("three", 3));
+        Supplier<Source> source = () -> Source.fromMap(Map.of("three", 3), XContentType.JSON);
+        when(leafLookup.asMap()).thenReturn(Collections.singletonMap("_source", source));
         ScriptedMetricAggContexts.MapScript.LeafFactory leafFactory = factory.newFactory(params, state, lookup);
         ScriptedMetricAggContexts.MapScript script = leafFactory.newInstance(leafReaderContext);
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
index 93ec854eb7e2..402947de3753 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
@@ -8,13 +8,13 @@
 
 package org.elasticsearch.painless;
 
-import org.apache.lucene.util.Constants;
+import org.elasticsearch.core.Strings;
 
 import java.util.HashMap;
 import java.util.Map;
 
 import static java.util.Collections.singletonMap;
-import static org.elasticsearch.painless.WriterConstants.MAX_INDY_STRING_CONCAT_ARGS;
+import static org.elasticsearch.painless.WriterConstants.MAX_STRING_CONCAT_ARGS;
 
 public class StringTests extends ScriptTestCase {
 
@@ -65,7 +65,7 @@ public void testAppendMultiple() {
     }
 
     public void testAppendMany() {
-        for (int i = MAX_INDY_STRING_CONCAT_ARGS - 5; i < MAX_INDY_STRING_CONCAT_ARGS + 5; i++) {
+        for (int i = MAX_STRING_CONCAT_ARGS - 5; i < MAX_STRING_CONCAT_ARGS + 5; i++) {
             doTestAppendMany(i);
         }
     }
@@ -74,14 +74,14 @@ private void doTestAppendMany(int count) {
         StringBuilder script = new StringBuilder("String s = \"cat\"; return s");
         StringBuilder result = new StringBuilder("cat");
         for (int i = 1; i < count; i++) {
-            final String s = formatted("%03d", i);
+            final String s = Strings.format("%03d", i);
             script.append(" + '").append(s).append("'.toString()");
             result.append(s);
         }
         final String s = script.toString();
         assertTrue(
             "every string part should be separately pushed to stack.",
-            Debugger.toString(s).contains(formatted("LDC \"%03d\"", count / 2))
+            Debugger.toString(s).contains(Strings.format("LDC \"%03d\"", count / 2))
         );
         assertEquals(result.toString(), exec(s));
     }
@@ -234,18 +234,14 @@ public void testBase64Augmentations() {
         assertEquals(rando, exec("params.rando.encodeBase64().decodeBase64()", singletonMap("rando", rando), true));
     }
 
-    public void testJava9ConstantStringConcatBytecode() {
-        assumeTrue("Needs Java 9 to test indified String concat", Constants.JRE_IS_MINIMUM_JAVA9);
-        assertNotNull(WriterConstants.INDY_STRING_CONCAT_BOOTSTRAP_HANDLE);
+    public void testConstantStringConcatBytecode() {
         assertBytecodeExists(
             "String s = \"cat\"; return s + true + 'abc' + null;",
             "INVOKEDYNAMIC concat(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;"
         );
     }
 
-    public void testJava9StringConcatBytecode() {
-        assumeTrue("Needs Java 9 to test indified String concat", Constants.JRE_IS_MINIMUM_JAVA9);
-        assertNotNull(WriterConstants.INDY_STRING_CONCAT_BOOTSTRAP_HANDLE);
+    public void testStringConcatBytecode() {
         assertBytecodeExists(
             "String s = \"cat\"; boolean t = true; Object u = null; return s + t + 'abc' + u;",
             "INVOKEDYNAMIC concat(Ljava/lang/String;ZLjava/lang/String;Ljava/lang/Object;)Ljava/lang/String;"
@@ -260,10 +256,4 @@ public void testNullStringConcat() {
         assertEquals("" + 2 + null, exec("2 + '' + null"));
         assertEquals("" + null + 2, exec("null + '' + 2"));
     }
-
-    public void testJava9NullStringConcatBytecode() {
-        assumeTrue("Needs Java 9 to test indified String concat", Constants.JRE_IS_MINIMUM_JAVA9);
-        assertNotNull(WriterConstants.INDY_STRING_CONCAT_BOOTSTRAP_HANDLE);
-        assertEquals("" + null + null, exec("'' + null + null"));
-    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index 77c2bb630c3d..7050bccffa06 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -840,4 +840,12 @@ public void testInstanceMethodNotFound() {
         iae = expectScriptThrows(IllegalArgumentException.class, () -> exec("doesNotExist(1, 'string', false)"));
         assertEquals(iae.getMessage(), "Unknown call [doesNotExist] with [3] arguments.");
     }
+
+    public void testArrayToArrayException() {
+        IllegalArgumentException iae = expectScriptThrows(
+            IllegalArgumentException.class,
+            () -> exec("return new String[] {'a'}.noMethod()")
+        );
+        assertTrue(iae.getMessage().contains("member method") && iae.getMessage().contains("not found"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/ContextInfoTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/ContextInfoTests.java
index babf2914343b..b7aa1c117233 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/ContextInfoTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/ContextInfoTests.java
@@ -172,6 +172,11 @@ protected PainlessContextInfo createTestInstance() {
         return new PainlessContextInfo(randomAlphaOfLength(20), classes, importedMethods, classBindings, instanceBindings);
     }
 
+    @Override
+    protected PainlessContextInfo mutateInstance(PainlessContextInfo instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<PainlessContextInfo> instanceReader() {
         return PainlessContextInfo::new;
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteRequestTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteRequestTests.java
index 7d957cca4d0b..7505f75293a3 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteRequestTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteRequestTests.java
@@ -75,6 +75,11 @@ protected PainlessExecuteAction.Request createTestInstance() {
         return new PainlessExecuteAction.Request(script, context != null ? context.name : null, contextSetup);
     }
 
+    @Override
+    protected PainlessExecuteAction.Request mutateInstance(PainlessExecuteAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<PainlessExecuteAction.Request> instanceReader() {
         return PainlessExecuteAction.Request::new;
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteResponseTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteResponseTests.java
index 6cb9f5a74ae0..20db938bdfa7 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteResponseTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteResponseTests.java
@@ -31,6 +31,11 @@ protected PainlessExecuteAction.Response createTestInstance() {
         return new PainlessExecuteAction.Response(result);
     }
 
+    @Override
+    protected PainlessExecuteAction.Response mutateInstance(PainlessExecuteAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected PainlessExecuteAction.Response doParseInstance(XContentParser parser) throws IOException {
         parser.nextToken(); // START-OBJECT
diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/130_metric_agg.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/130_metric_agg.yml
index 1c934b842171..161e4cee74b3 100644
--- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/130_metric_agg.yml
+++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/130_metric_agg.yml
@@ -9,6 +9,8 @@ setup:
             properties:
               double:
                 type: double
+              keyword:
+                type: keyword
 
   - do:
       cluster.health:
@@ -20,6 +22,7 @@ setup:
         id: "1"
         body:
           double: 1.0
+          keyword: "first"
 
   - do:
       index:
@@ -27,6 +30,7 @@ setup:
         id: "2"
         body:
           double: 1.0
+          keyword: "second"
 
   - do:
       index:
@@ -34,6 +38,7 @@ setup:
         id: "3"
         body:
           double: 2.0
+          keyword: "third"
 
   - do:
       indices.refresh: {}
@@ -61,3 +66,49 @@ setup:
   - match: { hits.total: 3 }
   - match: { aggregations.total.value: 4.0 }
 
+---
+"Scripted Metric Agg Non String Map Key":
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body: {
+          "size": 0,
+          "aggs": {
+            "total": {
+              "scripted_metric": {
+                "init_script": "state.transactions = [:]",
+                "map_script": "state.transactions[doc['double'].value] = doc['double'].value",
+                "combine_script": "return state.transactions",
+                "reduce_script": "double sum = 0; for (transactions in states) { for (entry in transactions.entrySet()) { sum += entry.getKey() - entry.getValue() } } return sum"
+              }
+            }
+          }
+        }
+
+  - match: { hits.total: 3 }
+  - match: { aggregations.total.value: 0.0 }
+
+---
+"Scripted Metric Agg String Map Key":
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body: {
+          "size": 0,
+          "aggs": {
+            "total": {
+              "scripted_metric": {
+                "init_script": "state.transactions = [:]",
+                "map_script": "state.transactions[doc['keyword'].value] = doc['double'].value",
+                "combine_script": "return state.transactions",
+                "reduce_script": "double sum = 0; for (transactions in states) { for (entry in transactions.entrySet()) { sum += entry.getValue() } } return sum"
+              }
+            }
+          }
+        }
+
+  - match: { hits.total: 3 }
+  - match: { aggregations.total.value: 4.0 }
+
diff --git a/modules/mapper-extras/build.gradle b/modules/mapper-extras/build.gradle
index a9db8325e1db..62e7744bfa7f 100644
--- a/modules/mapper-extras/build.gradle
+++ b/modules/mapper-extras/build.gradle
@@ -8,8 +8,8 @@
 
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
diff --git a/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/RankFeaturesMapperIntegrationIT.java b/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/RankFeaturesMapperIntegrationIT.java
new file mode 100644
index 000000000000..badc2dd568f5
--- /dev/null
+++ b/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/RankFeaturesMapperIntegrationIT.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+
+import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.equalTo;
+
+public class RankFeaturesMapperIntegrationIT extends ESIntegTestCase {
+
+    private static final String LOWER_RANKED_FEATURE = "ten";
+    private static final String HIGHER_RANKED_FEATURE = "twenty";
+    private static final String INDEX_NAME = "rank_feature_test";
+    private static final String FIELD_NAME = "all_rank_features";
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(MapperExtrasPlugin.class);
+    }
+
+    public void testRankFeaturesTermQuery() throws IOException {
+        init();
+        SearchResponse response = client().prepareSearch(INDEX_NAME)
+            .setQuery(QueryBuilders.termQuery(FIELD_NAME, HIGHER_RANKED_FEATURE))
+            .get();
+        assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+        for (SearchHit hit : response.getHits().getHits()) {
+            assertThat(hit.getScore(), equalTo(20f));
+        }
+
+        response = client().prepareSearch(INDEX_NAME)
+            .setQuery(QueryBuilders.termQuery(FIELD_NAME, HIGHER_RANKED_FEATURE).boost(100f))
+            .get();
+        assertThat(response.getHits().getTotalHits().value, equalTo(2L));
+        for (SearchHit hit : response.getHits().getHits()) {
+            assertThat(hit.getScore(), equalTo(2000f));
+        }
+
+        response = client().prepareSearch(INDEX_NAME)
+            .setQuery(
+                QueryBuilders.boolQuery()
+                    .should(QueryBuilders.termQuery(FIELD_NAME, HIGHER_RANKED_FEATURE))
+                    .should(QueryBuilders.termQuery(FIELD_NAME, LOWER_RANKED_FEATURE).boost(3f))
+                    .minimumShouldMatch(1)
+            )
+            .get();
+        assertThat(response.getHits().getTotalHits().value, equalTo(3L));
+        for (SearchHit hit : response.getHits().getHits()) {
+            if (hit.getId().equals("all")) {
+                assertThat(hit.getScore(), equalTo(50f));
+            }
+            if (hit.getId().equals("lower")) {
+                assertThat(hit.getScore(), equalTo(30f));
+            }
+            if (hit.getId().equals("higher")) {
+                assertThat(hit.getScore(), equalTo(20f));
+            }
+        }
+
+        response = client().prepareSearch(INDEX_NAME).setQuery(QueryBuilders.termQuery(FIELD_NAME, "missing_feature")).get();
+        assertThat(response.getHits().getTotalHits().value, equalTo(0L));
+    }
+
+    private void init() throws IOException {
+        Settings.Builder settings = Settings.builder();
+        settings.put(indexSettings());
+        prepareCreate(INDEX_NAME).setSettings(settings)
+            .setMapping(
+                jsonBuilder().startObject()
+                    .startObject("_doc")
+                    .startObject("properties")
+                    .startObject("all_rank_features")
+                    .field("type", "rank_features")
+                    .endObject()
+                    .endObject()
+                    .endObject()
+                    .endObject()
+            )
+            .get();
+        ensureGreen();
+
+        BulkResponse bulk = client().prepareBulk()
+            .add(
+                client().prepareIndex(INDEX_NAME)
+                    .setId("all")
+                    .setSource(Map.of("all_rank_features", Map.of(LOWER_RANKED_FEATURE, 10, HIGHER_RANKED_FEATURE, 20)))
+            )
+            .add(client().prepareIndex(INDEX_NAME).setId("lower").setSource(Map.of("all_rank_features", Map.of(LOWER_RANKED_FEATURE, 10))))
+            .add(
+                client().prepareIndex(INDEX_NAME).setId("higher").setSource(Map.of("all_rank_features", Map.of(HIGHER_RANKED_FEATURE, 20)))
+            )
+            .get();
+        assertFalse(bulk.buildFailureMessage(), bulk.hasFailures());
+        assertThat(refresh().getFailedShards(), equalTo(0));
+    }
+
+}
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java
index 08f39c583e2f..c6298f1eaf81 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java
+++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java
@@ -53,7 +53,7 @@
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.script.field.TextDocValuesField;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
@@ -211,14 +211,13 @@ private Function<LeafReaderContext, CheckedIntFunction<List<Object>, IOException
                     };
                 };
             }
-            SourceLookup sourceLookup = searchExecutionContext.lookup().source();
             ValueFetcher valueFetcher = valueFetcher(searchExecutionContext, null);
+            SourceProvider sourceProvider = searchExecutionContext.lookup();
             return context -> {
                 valueFetcher.setNextReader(context);
                 return docID -> {
                     try {
-                        sourceLookup.setSegmentAndDocument(context, docID);
-                        return valueFetcher.fetchValues(sourceLookup, docID, new ArrayList<>());
+                        return valueFetcher.fetchValues(sourceProvider.getSource(context, docID), docID, new ArrayList<>());
                     } catch (IOException e) {
                         throw new UncheckedIOException(e);
                     }
@@ -338,7 +337,7 @@ protected BytesRef storedToBytesRef(Object stored) {
                 name(),
                 CoreValuesSourceType.KEYWORD,
                 SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name())),
-                fieldDataContext.lookupSupplier().get().source(),
+                fieldDataContext.lookupSupplier().get(),
                 TextDocValuesField::new
             );
         }
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilder.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilder.java
index 25f078653411..28424a9e5289 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilder.java
+++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.document.FeatureField;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -413,7 +413,7 @@ protected int doHashCode() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java
index aea9fe9b2493..4187963e061c 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java
+++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java
@@ -10,6 +10,7 @@
 
 import org.apache.lucene.document.FeatureField;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.FieldDataContext;
@@ -27,6 +28,8 @@
 import java.io.IOException;
 import java.util.Map;
 
+import static org.elasticsearch.index.query.AbstractQueryBuilder.DEFAULT_BOOST;
+
 /**
  * A {@link FieldMapper} that exposes Lucene's {@link FeatureField} as a sparse
  * vector of features.
@@ -77,7 +80,7 @@ public static final class RankFeaturesFieldType extends MappedFieldType {
         private final boolean positiveScoreImpact;
 
         public RankFeaturesFieldType(String name, Map<String, String> meta, boolean positiveScoreImpact) {
-            super(name, false, false, false, TextSearchInfo.NONE, meta);
+            super(name, true, false, false, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
             this.positiveScoreImpact = positiveScoreImpact;
         }
 
@@ -86,10 +89,6 @@ public String typeName() {
             return CONTENT_TYPE;
         }
 
-        public boolean positiveScoreImpact() {
-            return positiveScoreImpact;
-        }
-
         @Override
         public Query existsQuery(SearchExecutionContext context) {
             throw new IllegalArgumentException("[rank_features] fields do not support [exists] queries");
@@ -107,7 +106,14 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
 
         @Override
         public Query termQuery(Object value, SearchExecutionContext context) {
-            throw new IllegalArgumentException("Queries on [rank_features] fields are not supported");
+            return FeatureField.newLinearQuery(name(), indexedValueForSearch(value), DEFAULT_BOOST);
+        }
+
+        private static String indexedValueForSearch(Object value) {
+            if (value instanceof BytesRef) {
+                return ((BytesRef) value).utf8ToString();
+            }
+            return value.toString();
         }
     }
 
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java
index ae088d5fed02..a88b4592448b 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java
+++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.fielddata.FieldData;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.FormattedDocValues;
@@ -78,7 +79,7 @@ private static ScaledFloatFieldMapper toType(FieldMapper in) {
 
     public static class Builder extends FieldMapper.Builder {
 
-        private final Parameter<Boolean> indexed = Parameter.indexParam(m -> toType(m).indexed, true);
+        private final Parameter<Boolean> indexed;
         private final Parameter<Boolean> hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true);
         private final Parameter<Boolean> stored = Parameter.storeParam(m -> toType(m).stored, false);
 
@@ -120,11 +121,13 @@ public static class Builder extends FieldMapper.Builder {
          */
         private final Parameter<TimeSeriesParams.MetricType> metric;
 
-        public Builder(String name, Settings settings) {
-            this(name, IGNORE_MALFORMED_SETTING.get(settings), COERCE_SETTING.get(settings));
+        private final IndexMode indexMode;
+
+        public Builder(String name, Settings settings, IndexMode indexMode) {
+            this(name, IGNORE_MALFORMED_SETTING.get(settings), COERCE_SETTING.get(settings), indexMode);
         }
 
-        public Builder(String name, boolean ignoreMalformedByDefault, boolean coerceByDefault) {
+        public Builder(String name, boolean ignoreMalformedByDefault, boolean coerceByDefault, IndexMode indexMode) {
             super(name);
             this.ignoreMalformed = Parameter.explicitBoolParam(
                 "ignore_malformed",
@@ -133,11 +136,19 @@ public Builder(String name, boolean ignoreMalformedByDefault, boolean coerceByDe
                 ignoreMalformedByDefault
             );
             this.coerce = Parameter.explicitBoolParam("coerce", true, m -> toType(m).coerce, coerceByDefault);
-
+            this.indexMode = indexMode;
+            this.indexed = Parameter.indexParam(m -> toType(m).indexed, () -> {
+                if (indexMode == IndexMode.TIME_SERIES) {
+                    var metricType = getMetric().getValue();
+                    return metricType != TimeSeriesParams.MetricType.COUNTER && metricType != TimeSeriesParams.MetricType.GAUGE;
+                } else {
+                    return true;
+                }
+            });
             this.metric = TimeSeriesParams.metricParam(
                 m -> toType(m).metricType,
-                TimeSeriesParams.MetricType.gauge,
-                TimeSeriesParams.MetricType.counter
+                TimeSeriesParams.MetricType.GAUGE,
+                TimeSeriesParams.MetricType.COUNTER
             ).addValidator(v -> {
                 if (v != null && hasDocValues.getValue() == false) {
                     throw new IllegalArgumentException(
@@ -162,6 +173,10 @@ public Builder metric(TimeSeriesParams.MetricType metric) {
             return this;
         }
 
+        private Parameter<TimeSeriesParams.MetricType> getMetric() {
+            return metric;
+        }
+
         @Override
         protected Parameter<?>[] getParameters() {
             return new Parameter<?>[] { indexed, hasDocValues, stored, ignoreMalformed, meta, scalingFactor, coerce, nullValue, metric };
@@ -183,7 +198,7 @@ public ScaledFloatFieldMapper build(MapperBuilderContext context) {
         }
     }
 
-    public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getSettings()));
+    public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getSettings(), c.getIndexSettings().getMode()));
 
     public static final class ScaledFloatFieldType extends SimpleMappedFieldType {
 
@@ -287,7 +302,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext
                 failIfNoDocValues();
             }
 
-            ValuesSourceType valuesSourceType = metricType == TimeSeriesParams.MetricType.counter
+            ValuesSourceType valuesSourceType = metricType == TimeSeriesParams.MetricType.COUNTER
                 ? TimeSeriesValuesSourceType.COUNTER
                 : IndexNumericFieldData.NumericType.LONG.getValuesSourceType();
             if ((operation == FielddataOperation.SEARCH || operation == FielddataOperation.SCRIPT) && hasDocValues()) {
@@ -310,7 +325,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext
                     name(),
                     valuesSourceType,
                     sourceValueFetcher(sourcePaths),
-                    searchLookup.source(),
+                    searchLookup,
                     ScaledFloatDocValuesField::new
                 );
             }
@@ -410,6 +425,7 @@ public String toString() {
     private final boolean ignoreMalformedByDefault;
     private final boolean coerceByDefault;
     private final TimeSeriesParams.MetricType metricType;
+    private final IndexMode indexMode;
 
     private ScaledFloatFieldMapper(
         String simpleName,
@@ -429,6 +445,7 @@ private ScaledFloatFieldMapper(
         this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue().value();
         this.coerceByDefault = builder.coerce.getDefaultValue().value();
         this.metricType = builder.metric.getValue();
+        this.indexMode = builder.indexMode;
     }
 
     boolean coerce() {
@@ -452,7 +469,7 @@ protected String contentType() {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(simpleName(), ignoreMalformedByDefault, coerceByDefault).metric(metricType).init(this);
+        return new Builder(simpleName(), ignoreMalformedByDefault, coerceByDefault, indexMode).metric(metricType).init(this);
     }
 
     @Override
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilderTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilderTests.java
index 9a03ecb7ec41..32446c3c2739 100644
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilderTests.java
+++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilderTests.java
@@ -111,7 +111,7 @@ public void testDefaultScoreFunction() throws IOException {
     }
 
     public void testIllegalField() {
-        String query = formatted("""
+        String query = Strings.format("""
             {
                 "rank_feature" : {
                     "field": "%s"
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java
index c30540d3106d..4e8e321a72af 100644
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java
+++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java
@@ -10,9 +10,12 @@
 
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -20,6 +23,7 @@
 import org.elasticsearch.index.mapper.MapperTestCase;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
+import org.elasticsearch.index.mapper.TimeSeriesParams;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -34,6 +38,7 @@
 import static java.util.Collections.singletonList;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
 
 public class ScaledFloatFieldMapperTests extends MapperTestCase {
 
@@ -89,15 +94,8 @@ public void testDefaults() throws Exception {
 
         ParsedDocument doc = mapper.parse(source(b -> b.field("field", 123)));
         IndexableField[] fields = doc.rootDoc().getFields("field");
-        assertEquals(2, fields.length);
-        IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
-        assertFalse(pointField.fieldType().stored());
-        assertEquals(1230, pointField.numericValue().longValue());
-        IndexableField dvField = fields[1];
-        assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
-        assertEquals(1230, dvField.numericValue().longValue());
-        assertFalse(dvField.fieldType().stored());
+        assertEquals(1, fields.length);
+        assertEquals("LongField <field:1230>", fields[0].toString());
     }
 
     public void testMissingScalingFactor() {
@@ -170,13 +168,9 @@ public void testStore() throws Exception {
         );
 
         IndexableField[] fields = doc.rootDoc().getFields("field");
-        assertEquals(3, fields.length);
-        IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
-        assertEquals(1230, pointField.numericValue().doubleValue(), 0d);
-        IndexableField dvField = fields[1];
-        assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
-        IndexableField storedField = fields[2];
+        assertEquals(2, fields.length);
+        assertEquals("LongField <field:1230>", fields[0].toString());
+        IndexableField storedField = fields[1];
         assertTrue(storedField.fieldType().stored());
         assertEquals(1230, storedField.numericValue().longValue());
     }
@@ -191,12 +185,8 @@ public void testCoerce() throws Exception {
             )
         );
         IndexableField[] fields = doc.rootDoc().getFields("field");
-        assertEquals(2, fields.length);
-        IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
-        assertEquals(1230, pointField.numericValue().longValue());
-        IndexableField dvField = fields[1];
-        assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
+        assertEquals(1, fields.length);
+        assertEquals("LongField <field:1230>", fields[0].toString());
 
         DocumentMapper mapper2 = createDocumentMapper(
             fieldMapping(b -> b.field("type", "scaled_float").field("scaling_factor", 10.0).field("coerce", false))
@@ -249,14 +239,8 @@ public void testNullValue() throws IOException {
             )
         );
         IndexableField[] fields = doc.rootDoc().getFields("field");
-        assertEquals(2, fields.length);
-        IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointDimensionCount());
-        assertFalse(pointField.fieldType().stored());
-        assertEquals(25, pointField.numericValue().longValue());
-        IndexableField dvField = fields[1];
-        assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
-        assertFalse(dvField.fieldType().stored());
+        assertEquals(1, fields.length);
+        assertEquals("LongField <field:25>", fields[0].toString());
     }
 
     /**
@@ -314,6 +298,19 @@ public void testMetricAndDocvalues() {
         assertThat(e.getCause().getMessage(), containsString("Field [time_series_metric] requires that [doc_values] is true"));
     }
 
+    public void testTimeSeriesIndexDefault() throws Exception {
+        var randomMetricType = randomFrom(TimeSeriesParams.MetricType.values());
+        var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName())
+            .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field");
+        var mapperService = createMapperService(indexSettings.build(), fieldMapping(b -> {
+            minimalMapping(b);
+            b.field("time_series_metric", randomMetricType.toString());
+        }));
+        var ft = (ScaledFloatFieldMapper.ScaledFloatFieldType) mapperService.fieldType("field");
+        assertThat(ft.getMetricType(), equalTo(randomMetricType));
+        assertThat(ft.isIndexed(), is(false));
+    }
+
     @Override
     protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException {
         // Large floats are a terrible idea but the round trip should still work no matter how badly you configure the field
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java
index b07ff9dafbf9..1493ece2671e 100644
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java
+++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java
@@ -216,14 +216,14 @@ public void testFieldData() throws IOException {
     }
 
     public void testFetchSourceValue() throws IOException {
-        MappedFieldType mapper = new ScaledFloatFieldMapper.Builder("field", false, false).scalingFactor(100)
+        MappedFieldType mapper = new ScaledFloatFieldMapper.Builder("field", false, false, null).scalingFactor(100)
             .build(MapperBuilderContext.root(false))
             .fieldType();
         assertEquals(List.of(3.14), fetchSourceValue(mapper, 3.1415926));
         assertEquals(List.of(3.14), fetchSourceValue(mapper, "3.1415"));
         assertEquals(List.of(), fetchSourceValue(mapper, ""));
 
-        MappedFieldType nullValueMapper = new ScaledFloatFieldMapper.Builder("field", false, false).scalingFactor(100)
+        MappedFieldType nullValueMapper = new ScaledFloatFieldMapper.Builder("field", false, false, null).scalingFactor(100)
             .nullValue(2.71)
             .build(MapperBuilderContext.root(false))
             .fieldType();
diff --git a/modules/parent-join/build.gradle b/modules/parent-join/build.gradle
index 66079cfd444c..903192e6ce25 100644
--- a/modules/parent-join/build.gradle
+++ b/modules/parent-join/build.gradle
@@ -5,8 +5,8 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
diff --git a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java
index 3e7cc6dead06..2821c38a6b24 100644
--- a/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java
+++ b/modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/ChildQuerySearchIT.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.common.lucene.search.function.CombineFunction;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.IdsQueryBuilder;
@@ -843,11 +844,11 @@ public void testSimpleQueryRewrite() throws Exception {
         // index simple data
         int childId = 0;
         for (int i = 0; i < 10; i++) {
-            String parentId = formatted("p%03d", i);
+            String parentId = Strings.format("p%03d", i);
             createIndexRequest("test", "parent", parentId, null, "p_field", parentId).get();
             int j = childId;
             for (; j < childId + 50; j++) {
-                String childUid = formatted("c%03d", j);
+                String childUid = Strings.format("c%03d", j);
                 createIndexRequest("test", "child", childUid, parentId, "c_field", childUid).get();
             }
             childId = j;
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java
index 1594e4e817b9..f191dfcda3bd 100644
--- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.join.aggregations;
 
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -180,7 +180,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java
index 83cc38636b10..fc797be04451 100644
--- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java
@@ -68,7 +68,7 @@ protected Aggregator doCreateInternal(Aggregator parent, CardinalityUpperBound c
             );
         }
         WithOrdinals valuesSource = (WithOrdinals) rawValuesSource;
-        long maxOrd = valuesSource.globalMaxOrd(context.searcher());
+        long maxOrd = valuesSource.globalMaxOrd(context.searcher().getIndexReader());
         return new ParentToChildrenAggregator(
             name,
             factories,
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregationBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregationBuilder.java
index 1b6cca403aba..c131be17c5e1 100644
--- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregationBuilder.java
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregationBuilder.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.join.aggregations;
 
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -178,7 +178,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregatorFactory.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregatorFactory.java
index 5ff008832622..f531ac67b98f 100644
--- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregatorFactory.java
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregatorFactory.java
@@ -68,7 +68,7 @@ protected Aggregator doCreateInternal(Aggregator children, CardinalityUpperBound
             );
         }
         WithOrdinals valuesSource = (WithOrdinals) rawValuesSource;
-        long maxOrd = valuesSource.globalMaxOrd(context.searcher());
+        long maxOrd = valuesSource.globalMaxOrd(context.searcher().getIndexReader());
         return new ChildrenToParentAggregator(
             name,
             factories,
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java
index a8845b2731ab..8f9b7bfd7e10 100644
--- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java
@@ -18,7 +18,7 @@
 import org.apache.lucene.search.join.ScoreMode;
 import org.apache.lucene.search.similarities.Similarity;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -540,7 +540,7 @@ protected void extractInnerHitBuilders(Map<String, InnerHitContextBuilder> inner
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java
index 9c297236a8e0..f3777c526994 100644
--- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -312,7 +312,7 @@ protected void extractInnerHitBuilders(Map<String, InnerHitContextBuilder> inner
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentIdQueryBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentIdQueryBuilder.java
index cfd39c7ee8f8..7d15bbcedf8f 100644
--- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentIdQueryBuilder.java
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentIdQueryBuilder.java
@@ -15,7 +15,7 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -192,7 +192,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/modules/percolator/build.gradle b/modules/percolator/build.gradle
index 785edc32c143..a871056539d3 100644
--- a/modules/percolator/build.gradle
+++ b/modules/percolator/build.gradle
@@ -5,8 +5,8 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
index de319f67b780..f2d06e2b72d4 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
@@ -33,6 +33,7 @@
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.get.GetRequest;
@@ -216,12 +217,12 @@ protected PercolateQueryBuilder(String field, Supplier<BytesReference> documentS
         super(in);
         field = in.readString();
         name = in.readOptionalString();
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             String documentType = in.readOptionalString();
             assert documentType == null;
         }
         indexedDocumentIndex = in.readOptionalString();
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             String indexedDocumentType = in.readOptionalString();
             assert indexedDocumentType == null;
         }
@@ -258,12 +259,12 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         }
         out.writeString(field);
         out.writeOptionalString(name);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             // In 7x, typeless percolate queries are represented by null documentType values
             out.writeOptionalString(null);
         }
         out.writeOptionalString(indexedDocumentIndex);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             // In 7x, typeless percolate queries are represented by null indexedDocumentType values
             out.writeOptionalString(null);
         }
@@ -660,7 +661,7 @@ public <IFD extends IndexFieldData<?>> IFD getForField(
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
index 722b47563559..f7c01bec3b2c 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
@@ -252,7 +252,7 @@ public void testRequiredParameters() {
 
     public void testFromJsonNoDocumentType() throws IOException {
         SearchExecutionContext searchExecutionContext = createSearchExecutionContext();
-        QueryBuilder queryBuilder = parseQuery(formatted("""
+        QueryBuilder queryBuilder = parseQuery(Strings.format("""
             {"percolate" : { "document": {}, "field":"%s"}}
             """, queryField));
         queryBuilder.toQuery(searchExecutionContext);
@@ -265,14 +265,14 @@ public void testFromJsonNoType() throws IOException {
         documentSource = Collections.singletonList(randomSource(new HashSet<>()));
 
         SearchExecutionContext searchExecutionContext = createSearchExecutionContext();
-        QueryBuilder queryBuilder = parseQuery(formatted("""
+        QueryBuilder queryBuilder = parseQuery(Strings.format("""
             {"percolate" : { "index": "%s", "id": "%s", "field":"%s"}}
             """, indexedDocumentIndex, indexedDocumentId, queryField));
         rewriteAndFetch(queryBuilder, searchExecutionContext).toQuery(searchExecutionContext);
     }
 
     public void testBothDocumentAndDocumentsSpecified() {
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(formatted("""
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(Strings.format("""
             {"percolate" : { "document": {}, "documents": [{}, {}], "field":"%s"}}
             """, queryField)));
         assertThat(e.getMessage(), containsString("The following fields are not allowed together: [document, documents]"));
@@ -382,7 +382,7 @@ public void testDisallowExpensiveQueries() {
 
     public void testFromJsonWithDocumentType() throws IOException {
         SearchExecutionContext searchExecutionContext = createSearchExecutionContext();
-        String queryAsString = formatted("""
+        String queryAsString = Strings.format("""
             {"percolate" : { "document": {}, "document_type":"%s", "field":"%s"}}
             """, docType, queryField);
         XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, queryAsString, RestApiVersion.V_7);
@@ -398,7 +398,7 @@ public void testFromJsonWithType() throws IOException {
         documentSource = Collections.singletonList(randomSource(new HashSet<>()));
         SearchExecutionContext searchExecutionContext = createSearchExecutionContext();
 
-        String queryAsString = formatted("""
+        String queryAsString = Strings.format("""
             {"percolate" : { "index": "%s", "type": "_doc", "id": "%s", "field":"%s"}}
             """, indexedDocumentIndex, indexedDocumentId, queryField);
         XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, queryAsString, RestApiVersion.V_7);
diff --git a/modules/rank-eval/build.gradle b/modules/rank-eval/build.gradle
index 5ac9922a1e07..1268a40dd5bd 100644
--- a/modules/rank-eval/build.gradle
+++ b/modules/rank-eval/build.gradle
@@ -5,8 +5,8 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java
index bea4d4222fa5..d6a9a25be51f 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.index.rankeval;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
@@ -45,7 +45,7 @@ public RankEvalRequest(RankEvalSpec rankingEvaluationSpec, String[] indices) {
         rankingEvaluationSpec = new RankEvalSpec(in);
         indices = in.readStringArray();
         indicesOptions = IndicesOptions.readIndicesOptions(in);
-        if (in.getVersion().onOrAfter(Version.V_7_6_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_6_0)) {
             searchType = SearchType.fromId(in.readByte());
         }
     }
@@ -126,7 +126,7 @@ public void writeTo(StreamOutput out) throws IOException {
         rankingEvaluationSpec.writeTo(out);
         out.writeStringArray(indices);
         indicesOptions.writeIndicesOptions(out);
-        if (out.getVersion().onOrAfter(Version.V_7_6_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_6_0)) {
             out.writeByte(searchType.id());
         }
     }
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java
index d761b8e28aa9..4436b9f3902f 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java
@@ -288,11 +288,11 @@ public void testMetricDetails() {
         assertEquals(expectedNdcg, detail.getNDCG(), 0.0);
         assertEquals(unratedDocs, detail.getUnratedDocs());
         if (idcg != 0) {
-            assertEquals(formatted("""
+            assertEquals(Strings.format("""
                 {"dcg":{"dcg":%s,"ideal_dcg":%s,"normalized_dcg":%s,"unrated_docs":%s}}\
                 """, dcg, idcg, expectedNdcg, unratedDocs), Strings.toString(detail));
         } else {
-            assertEquals(formatted("""
+            assertEquals(Strings.format("""
                 {"dcg":{"dcg":%s,"unrated_docs":%s}}\
                 """, dcg, unratedDocs), Strings.toString(detail));
         }
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java
index b602f2b0a119..d4ec7ba9b9ef 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.SearchParseException;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.ToXContent;
@@ -44,7 +43,6 @@
 
 import static java.util.Collections.singleton;
 import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
-import static org.elasticsearch.test.TestSearchContext.SHARD_TARGET;
 import static org.elasticsearch.test.XContentTestUtils.insertRandomFields;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
 import static org.hamcrest.Matchers.instanceOf;
@@ -54,7 +52,6 @@ public class RankEvalResponseTests extends ESTestCase {
     private static final Exception[] RANDOM_EXCEPTIONS = new Exception[] {
         new ClusterBlockException(singleton(NoMasterBlockService.NO_MASTER_BLOCK_WRITES)),
         new CircuitBreakingException("Data too large", 123, 456, CircuitBreaker.Durability.PERMANENT),
-        new SearchParseException(SHARD_TARGET, "Parse failure", new XContentLocation(12, 98)),
         new IllegalArgumentException("Closed resource", new RuntimeException("Resource")),
         new SearchPhaseExecutionException(
             "search",
diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle
index 1496e32927f9..4cd12b44aaaa 100644
--- a/modules/reindex/build.gradle
+++ b/modules/reindex/build.gradle
@@ -17,9 +17,9 @@ import org.gradle.api.internal.artifacts.ArtifactAttributes
 
 apply plugin: 'elasticsearch.test-with-dependencies'
 apply plugin: 'elasticsearch.jdk-download'
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.internal-java-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java
index 3155f49ed376..30c416ddadfe 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -100,7 +100,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver expressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         return Collections.singletonList(new ReindexSslConfig(environment.settings(), environment, resourceWatcherService));
     }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWithAuthTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWithAuthTests.java
index 974c4ead7d20..ed30fa5c4eeb 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWithAuthTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWithAuthTests.java
@@ -21,7 +21,7 @@
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -170,7 +170,7 @@ public Collection<Object> createComponents(
             IndexNameExpressionResolver expressionResolver,
             Supplier<RepositoriesService> repositoriesServiceSupplier,
             Tracer tracer,
-            AllocationDeciders allocationDeciders
+            AllocationService allocationService
         ) {
             testFilter.set(new ReindexFromRemoteWithAuthTests.TestFilter(threadPool));
             return Collections.emptyList();
diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteRequestBuildersTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteRequestBuildersTests.java
index f06d40afeb0a..e0cfe0088d9e 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteRequestBuildersTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/remote/RemoteRequestBuildersTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESTestCase;
@@ -228,7 +229,7 @@ public void testInitialSearchEntity() throws IOException {
         searchRequest.source().fetchSource(new String[] { "in1", "in2" }, new String[] { "out" });
         entity = initialSearch(searchRequest, new BytesArray(query), remoteVersion).getEntity();
         assertEquals(ContentType.APPLICATION_JSON.toString(), entity.getContentType().getValue());
-        assertEquals(XContentHelper.stripWhitespace(formatted("""
+        assertEquals(XContentHelper.stripWhitespace(Strings.format("""
             {
               "query": %s,
               "_source": {
diff --git a/modules/repository-azure/build.gradle b/modules/repository-azure/build.gradle
index 7f0b86f5db69..aa35dc85ea38 100644
--- a/modules/repository-azure/build.gradle
+++ b/modules/repository-azure/build.gradle
@@ -13,7 +13,7 @@ import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.internal-test-artifact-base'
 
@@ -23,19 +23,21 @@ esplugin {
 }
 
 versions << [
-  'azure': '12.16.0',
-  'azureCommon': '12.15.1',
-  'azureCore': '1.27.0',
-  'azureCoreHttpNetty': '1.11.9',
+  'azure': '12.20.1',
+  'azureCommon': '12.19.1',
+  'azureCore': '1.34.0',
+  'azureCoreHttpNetty': '1.12.7',
+  'azureJackson': '2.13.4',
+  'azureJacksonDatabind': '2.13.4.2',
 
   'jakartaActivation': '1.2.1',
   'jakartaXMLBind': '2.3.2',
   'stax2API': '4.2.1',
-  'woodstox': '6.2.7',
+  'woodstox': '6.4.0',
 
-  'reactorNetty': '1.0.15',
-  'reactorCore': '3.4.14',
-  'reactiveStreams': '1.0.3',
+  'reactorNetty': '1.0.24',
+  'reactorCore': '3.4.23',
+  'reactiveStreams': '1.0.4',
 ]
 
 dependencies {
@@ -45,14 +47,14 @@ dependencies {
   api "com.azure:azure-core:${versions.azureCore}"
 
   // jackson
-  api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
+  api "com.fasterxml.jackson.core:jackson-core:${versions.azureJackson}"
+  api "com.fasterxml.jackson.core:jackson-databind:${versions.azureJacksonDatabind}"
+  api "com.fasterxml.jackson.core:jackson-annotations:${versions.azureJackson}"
 
   // jackson xml
-  api "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${versions.jackson}"
-  api "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${versions.jackson}"
-  api "com.fasterxml.jackson.module:jackson-module-jaxb-annotations:${versions.jackson}"
+  api "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${versions.azureJackson}"
+  api "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${versions.azureJackson}"
+  api "com.fasterxml.jackson.module:jackson-module-jaxb-annotations:${versions.azureJackson}"
   api "jakarta.activation:jakarta.activation-api:${versions.jakartaActivation}"
   // The SDK uses javax.xml bindings
   api "jakarta.xml.bind:jakarta.xml.bind-api:${versions.jakartaXMLBind}"
@@ -117,7 +119,8 @@ tasks.named("thirdPartyAudit").configure {
     'com.aayushatharva.brotli4j.Brotli4jLoader',
     'com.aayushatharva.brotli4j.decoder.DecoderJNI$Status',
     'com.aayushatharva.brotli4j.decoder.DecoderJNI$Wrapper',
-    'com.aayushatharva.brotli4j.encoder.Encoders',
+    'com.aayushatharva.brotli4j.encoder.BrotliEncoderChannel',
+    'com.aayushatharva.brotli4j.encoder.Encoder',
     'com.aayushatharva.brotli4j.encoder.Encoder$Mode',
     'com.aayushatharva.brotli4j.encoder.Encoder$Parameters',
 
@@ -368,10 +371,9 @@ testClusters.matching { it.name == "yamlRestTest" }.configureEach {
     }
     if (useFixture) {
       setting 'azure.client.integration_test.endpoint_suffix', azureAddress
-      String firstPartOfSeed = BuildParams.testSeed.tokenize(':').get(0)
 
       def ignoreTestSeed = providers.systemProperty('ignore.tests.seed').isPresent() ? PropertyNormalization.IGNORE_VALUE : PropertyNormalization.DEFAULT
-      setting 'thread_pool.repository_azure.max', (Math.abs(Long.parseUnsignedLong(firstPartOfSeed, 16) % 10) + 1).toString(), ignoreTestSeed
+      setting 'thread_pool.repository_azure.max', (Math.abs(BuildParams.random.nextLong() % 10) + 1).toString(), ignoreTestSeed
     }
 }
 
diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java
index 169164cdce2b..48dce1a86b75 100644
--- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java
+++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Setting;
@@ -97,7 +97,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         AzureClientProvider azureClientProvider = AzureClientProvider.create(threadPool, settings);
         azureStoreService.set(createAzureStorageService(settings, azureClientProvider));
diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java
index f05fe4331b07..4c9a3b1e6991 100644
--- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java
+++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java
@@ -187,7 +187,10 @@ public void testWriteBlobWithRetries() throws Exception {
 
                 if (randomBoolean()) {
                     if (randomBoolean()) {
-                        Streams.readFully(exchange.getRequestBody(), new byte[randomIntBetween(1, Math.max(1, bytes.length - 1))]);
+                        org.elasticsearch.core.Streams.readFully(
+                            exchange.getRequestBody(),
+                            new byte[randomIntBetween(1, Math.max(1, bytes.length - 1))]
+                        );
                     } else {
                         Streams.readFully(exchange.getRequestBody());
                         AzureHttpHandler.sendError(exchange, randomFrom(RestStatus.INTERNAL_SERVER_ERROR, RestStatus.SERVICE_UNAVAILABLE));
diff --git a/modules/repository-gcs/build.gradle b/modules/repository-gcs/build.gradle
index dce2513fb3b8..4f878eb9d79d 100644
--- a/modules/repository-gcs/build.gradle
+++ b/modules/repository-gcs/build.gradle
@@ -1,7 +1,7 @@
 import org.apache.tools.ant.filters.ReplaceTokens
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.internal.test.RestIntegTestTask
-import org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin
+import org.elasticsearch.gradle.internal.test.rest.LegacyYamlRestTestPlugin
 import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin
 
 import java.nio.file.Files
@@ -16,7 +16,7 @@ import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.internal-test-artifact-base'
 
@@ -26,36 +26,36 @@ esplugin {
 }
 
 dependencies {
-  api 'com.google.cloud:google-cloud-storage:1.118.1'
-  api 'com.google.cloud:google-cloud-core:2.0.2'
-  api 'com.google.cloud:google-cloud-core-http:2.0.2'
-  runtimeOnly 'com.google.guava:guava:30.1.1-jre'
+  api 'com.google.cloud:google-cloud-storage:2.13.1'
+  api 'com.google.cloud:google-cloud-core:2.8.28'
+  api 'com.google.cloud:google-cloud-core-http:2.8.28'
+  runtimeOnly 'com.google.guava:guava:31.1-jre'
   runtimeOnly 'com.google.guava:failureaccess:1.0.1'
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
-  api 'com.google.api:api-common:2.2.1'
-  api 'com.google.api:gax:2.0.0'
-  api 'org.threeten:threetenbp:1.5.1'
-  api 'com.google.protobuf:protobuf-java-util:3.17.3'
-  api 'com.google.protobuf:protobuf-java:3.21.1'
-  api 'com.google.code.gson:gson:2.8.9'
-  api 'com.google.api.grpc:proto-google-common-protos:2.3.2'
-  api 'com.google.api.grpc:proto-google-iam-v1:1.0.14'
-  api 'com.google.auth:google-auth-library-credentials:1.0.0'
-  api 'com.google.auth:google-auth-library-oauth2-http:1.0.0'
-  api 'com.google.oauth-client:google-oauth-client:1.34.1'
-  api 'com.google.api-client:google-api-client:1.35.1'
-  api 'com.google.http-client:google-http-client:1.39.2'
-  api 'com.google.http-client:google-http-client-gson:1.39.2'
-  api 'com.google.http-client:google-http-client-appengine:1.39.2'
-  api 'com.google.http-client:google-http-client-jackson2:1.39.2'
+  api 'com.google.api:api-common:2.3.1'
+  api 'com.google.api:gax:2.20.1'
+  api 'org.threeten:threetenbp:1.6.5'
+  api "com.google.protobuf:protobuf-java-util:${versions.protobuf}"
+  api "com.google.protobuf:protobuf-java:${versions.protobuf}"
+  api 'com.google.code.gson:gson:2.10'
+  api 'com.google.api.grpc:proto-google-common-protos:2.9.6'
+  api 'com.google.api.grpc:proto-google-iam-v1:1.6.2'
+  api 'com.google.auth:google-auth-library-credentials:1.11.0'
+  api 'com.google.auth:google-auth-library-oauth2-http:1.11.0'
+  api "com.google.oauth-client:google-oauth-client:${versions.google_oauth_client}"
+  api 'com.google.api-client:google-api-client:2.1.1'
+  api 'com.google.http-client:google-http-client:1.42.3'
+  api 'com.google.http-client:google-http-client-gson:1.42.3'
+  api 'com.google.http-client:google-http-client-appengine:1.42.3'
+  api 'com.google.http-client:google-http-client-jackson2:1.42.3'
   api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  api 'com.google.api:gax-httpjson:0.85.0'
-  api 'io.grpc:grpc-context:1.39.0'
-  api 'io.opencensus:opencensus-api:0.28.0'
-  api 'io.opencensus:opencensus-contrib-http-util:0.28.0'
-  api 'com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1'
+  api 'com.google.api:gax-httpjson:0.105.1'
+  api 'io.grpc:grpc-context:1.49.2'
+  api 'io.opencensus:opencensus-api:0.31.1'
+  api 'io.opencensus:opencensus-contrib-http-util:0.31.1'
+  api 'com.google.apis:google-api-services-storage:v1-rev20220705-2.0.0'
 
   testImplementation project(':test:fixtures:gcs-fixture')
 }
@@ -180,10 +180,22 @@ tasks.named("thirdPartyAudit").configure {
     'org.apache.http.protocol.HttpContext',
     'org.apache.http.protocol.HttpProcessor',
     'org.apache.http.protocol.HttpRequestExecutor',
+
     // commons-logging provided dependencies
     'javax.servlet.ServletContextEvent',
     'javax.servlet.ServletContextListener'
   )
+
+
+  if(BuildParams.graalVmRuntime == false) {
+    ignoreMissingClasses(
+        'org.graalvm.nativeimage.hosted.Feature',
+        'org.graalvm.nativeimage.hosted.Feature$BeforeAnalysisAccess',
+        'org.graalvm.nativeimage.hosted.Feature$DuringAnalysisAccess',
+        'org.graalvm.nativeimage.hosted.Feature$FeatureAccess',
+        'org.graalvm.nativeimage.hosted.RuntimeReflection'
+    )
+  }
 }
 
 boolean useFixture = false
@@ -271,7 +283,7 @@ def largeBlobYamlRestTest = tasks.register("largeBlobYamlRestTest", RestIntegTes
     dependsOn "createServiceAccountFile"
   }
   SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-  SourceSet yamlRestTestSourceSet = sourceSets.getByName(InternalYamlRestTestPlugin.SOURCE_SET_NAME)
+  SourceSet yamlRestTestSourceSet = sourceSets.getByName(LegacyYamlRestTestPlugin.SOURCE_SET_NAME)
   setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
   setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
 
@@ -323,7 +335,7 @@ testClusters.matching {
 if (useFixture) {
   tasks.register("yamlRestTestApplicationDefaultCredentials", RestIntegTestTask.class) {
     SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-    SourceSet yamlRestTestSourceSet = sourceSets.getByName(InternalYamlRestTestPlugin.SOURCE_SET_NAME)
+    SourceSet yamlRestTestSourceSet = sourceSets.getByName(LegacyYamlRestTestPlugin.SOURCE_SET_NAME)
     setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
     setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
   }
diff --git a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
index d2cd3dc6a2b2..8a4fb8eb41bd 100644
--- a/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
+++ b/modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
@@ -10,10 +10,12 @@
 
 import fixture.gcs.FakeOAuth2HttpHandler;
 import fixture.gcs.GoogleCloudStorageHttpHandler;
+import fixture.gcs.TestUtils;
 
 import com.google.api.gax.retrying.RetrySettings;
 import com.google.cloud.http.HttpTransportOptions;
 import com.google.cloud.storage.StorageOptions;
+import com.google.cloud.storage.StorageRetryStrategy;
 import com.sun.net.httpserver.Headers;
 import com.sun.net.httpserver.HttpExchange;
 import com.sun.net.httpserver.HttpHandler;
@@ -226,6 +228,7 @@ StorageOptions createStorageOptions(
                 ) {
                     StorageOptions options = super.createStorageOptions(gcsClientSettings, httpTransportOptions);
                     return options.toBuilder()
+                        .setStorageRetryStrategy(StorageRetryStrategy.getLegacyStorageRetryStrategy())
                         .setHost(options.getHost())
                         .setCredentials(options.getCredentials())
                         .setRetrySettings(
diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java
index 74184fd0b275..4d29f14c7adb 100644
--- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java
+++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java
@@ -29,12 +29,12 @@
 import org.elasticsearch.common.blobstore.support.BlobMetadata;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.hash.MessageDigests;
-import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.core.Streams;
 import org.elasticsearch.core.SuppressForbidden;
 
 import java.io.ByteArrayInputStream;
diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
index 4743eb71f832..e65f391667d7 100644
--- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
+++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
@@ -19,6 +19,7 @@
 import com.google.cloud.http.HttpTransportOptions;
 import com.google.cloud.storage.Storage;
 import com.google.cloud.storage.StorageOptions;
+import com.google.cloud.storage.StorageRetryStrategy;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -176,6 +177,7 @@ StorageOptions createStorageOptions(
         final HttpTransportOptions httpTransportOptions
     ) {
         final StorageOptions.Builder storageOptionsBuilder = StorageOptions.newBuilder()
+            .setStorageRetryStrategy(StorageRetryStrategy.getLegacyStorageRetryStrategy())
             .setTransportOptions(httpTransportOptions)
             .setHeaderProvider(() -> {
                 return Strings.hasLength(gcsClientSettings.getApplicationName())
diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java
index 3a9a4fa99c57..ef38da7eb20e 100644
--- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java
+++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java
@@ -13,6 +13,7 @@
 import com.google.cloud.http.HttpTransportOptions;
 import com.google.cloud.storage.StorageException;
 import com.google.cloud.storage.StorageOptions;
+import com.google.cloud.storage.StorageRetryStrategy;
 import com.sun.net.httpserver.HttpHandler;
 
 import org.apache.http.HttpStatus;
@@ -60,6 +61,7 @@
 import static fixture.gcs.GoogleCloudStorageHttpHandler.getContentRangeLimit;
 import static fixture.gcs.GoogleCloudStorageHttpHandler.getContentRangeStart;
 import static fixture.gcs.GoogleCloudStorageHttpHandler.parseMultipartRequestBody;
+import static fixture.gcs.TestUtils.createServiceAccount;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.elasticsearch.repositories.blobstore.ESBlobStoreRepositoryIntegTestCase.randomBytes;
 import static org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStore.MAX_DELETES_PER_BATCH;
@@ -67,7 +69,6 @@
 import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.ENDPOINT_SETTING;
 import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.READ_TIMEOUT_SETTING;
 import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.TOKEN_URI_SETTING;
-import static org.elasticsearch.repositories.gcs.TestUtils.createServiceAccount;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -140,6 +141,7 @@ StorageOptions createStorageOptions(
                     retrySettingsBuilder.setMaxAttempts(maxRetries + 1);
                 }
                 return options.toBuilder()
+                    .setStorageRetryStrategy(StorageRetryStrategy.getLegacyStorageRetryStrategy())
                     .setHost(options.getHost())
                     .setCredentials(options.getCredentials())
                     .setRetrySettings(retrySettingsBuilder.build())
@@ -203,7 +205,7 @@ public void testWriteBlobWithRetries() throws Exception {
                 assertThat(content.isPresent(), is(true));
                 assertThat(content.get().v1(), equalTo(blobContainer.path().buildAsString() + "write_blob_max_retries"));
                 if (Objects.deepEquals(bytes, BytesReference.toBytes(content.get().v2()))) {
-                    byte[] response = formatted("""
+                    byte[] response = Strings.format("""
                         {"bucket":"bucket","name":"%s"}
                         """, content.get().v1()).getBytes(UTF_8);
                     exchange.getResponseHeaders().add("Content-Type", "application/json");
@@ -216,7 +218,10 @@ public void testWriteBlobWithRetries() throws Exception {
             }
             if (randomBoolean()) {
                 if (randomBoolean()) {
-                    Streams.readFully(exchange.getRequestBody(), new byte[randomIntBetween(1, Math.max(1, bytes.length - 1))]);
+                    org.elasticsearch.core.Streams.readFully(
+                        exchange.getRequestBody(),
+                        new byte[randomIntBetween(1, Math.max(1, bytes.length - 1))]
+                    );
                 } else {
                     Streams.readFully(exchange.getRequestBody());
                     exchange.sendResponseHeaders(HttpStatus.SC_INTERNAL_SERVER_ERROR, -1);
@@ -239,7 +244,7 @@ public void testWriteBlobWithReadTimeouts() {
         httpServer.createContext("/upload/storage/v1/b/bucket/o", exchange -> {
             if (randomBoolean()) {
                 if (randomBoolean()) {
-                    Streams.readFully(exchange.getRequestBody(), new byte[randomIntBetween(1, bytes.length - 1)]);
+                    org.elasticsearch.core.Streams.readFully(exchange.getRequestBody(), new byte[randomIntBetween(1, bytes.length - 1)]);
                 } else {
                     Streams.readFully(exchange.getRequestBody());
                 }
@@ -349,7 +354,7 @@ public void testWriteLargeBlob() throws IOException {
                 if (range.equals("bytes */*")) {
                     final int receivedSoFar = bytesReceived.get();
                     if (receivedSoFar > 0) {
-                        exchange.getResponseHeaders().add("Range", formatted("bytes=0-%d", receivedSoFar));
+                        exchange.getResponseHeaders().add("Range", Strings.format("bytes=0-%d", receivedSoFar));
                     }
                     exchange.getResponseHeaders().add("Content-Length", "0");
                     exchange.sendResponseHeaders(308 /* Resume Incomplete */, -1);
@@ -371,7 +376,7 @@ public void testWriteLargeBlob() throws IOException {
                         exchange.sendResponseHeaders(RestStatus.OK.getStatus(), -1);
                         return;
                     } else {
-                        exchange.getResponseHeaders().add("Range", String.format(Locale.ROOT, "bytes=%d/%d", rangeStart, rangeEnd));
+                        exchange.getResponseHeaders().add("Range", Strings.format("bytes=%d/%d", rangeStart, rangeEnd));
                         exchange.getResponseHeaders().add("Content-Length", "0");
                         exchange.sendResponseHeaders(308 /* Resume Incomplete */, -1);
                         return;
diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java
index f8fe2b5919d9..73eb4a48a21d 100644
--- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java
+++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.ESTestCase;
@@ -260,7 +261,7 @@ private static Tuple<ServiceAccountCredentials, byte[]> randomCredential(final S
         credentialBuilder.setPrivateKeyId("private_key_id_" + clientName);
         credentialBuilder.setScopes(Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL));
         final String encodedPrivateKey = Base64.getEncoder().encodeToString(keyPair.getPrivate().getEncoded());
-        final String serviceAccount = formatted("""
+        final String serviceAccount = Strings.format("""
             {
               "type": "service_account",
               "project_id": "project_id_%s",
diff --git a/modules/repository-s3/build.gradle b/modules/repository-s3/build.gradle
index a31bdf7ae9b7..892c8c42903e 100644
--- a/modules/repository-s3/build.gradle
+++ b/modules/repository-s3/build.gradle
@@ -1,7 +1,7 @@
 import org.apache.tools.ant.filters.ReplaceTokens
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.internal.test.RestIntegTestTask
-import org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin
+import org.elasticsearch.gradle.internal.test.rest.LegacyYamlRestTestPlugin
 import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin
 
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
@@ -13,7 +13,7 @@ import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.internal-test-artifact-base'
 
@@ -121,7 +121,7 @@ String s3ECSBasePath = System.getenv("amazon_s3_base_path_ecs")
 String s3STSBucket = System.getenv("amazon_s3_bucket_sts")
 String s3STSBasePath = System.getenv("amazon_s3_base_path_sts")
 
-boolean s3DisableChunkedEncoding = (new Random(Long.parseUnsignedLong(BuildParams.testSeed.tokenize(':').get(0), 16))).nextBoolean()
+boolean s3DisableChunkedEncoding = BuildParams.random.nextBoolean()
 
 // If all these variables are missing then we are testing against the internal fixture instead, which has the following
 // credentials hard-coded in.
@@ -244,7 +244,7 @@ if (useFixture) {
   tasks.register("yamlRestTestMinio", RestIntegTestTask) {
     description = "Runs REST tests using the Minio repository."
     SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-    SourceSet yamlRestTestSourceSet = sourceSets.getByName(InternalYamlRestTestPlugin.SOURCE_SET_NAME)
+    SourceSet yamlRestTestSourceSet = sourceSets.getByName(LegacyYamlRestTestPlugin.SOURCE_SET_NAME)
     setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
     setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
 
@@ -272,7 +272,7 @@ if (useFixture) {
   tasks.register("yamlRestTestECS", RestIntegTestTask.class) {
     description = "Runs tests using the ECS repository."
     SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-    SourceSet yamlRestTestSourceSet = sourceSets.getByName(InternalYamlRestTestPlugin.SOURCE_SET_NAME)
+    SourceSet yamlRestTestSourceSet = sourceSets.getByName(LegacyYamlRestTestPlugin.SOURCE_SET_NAME)
     setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
     setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
     systemProperty 'tests.rest.blacklist', [
@@ -298,7 +298,7 @@ if (useFixture) {
   tasks.register("yamlRestTestSTS", RestIntegTestTask.class) {
     description = "Runs tests with the STS (Secure Token Service)"
     SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-    SourceSet yamlRestTestSourceSet = sourceSets.getByName(InternalYamlRestTestPlugin.SOURCE_SET_NAME)
+    SourceSet yamlRestTestSourceSet = sourceSets.getByName(LegacyYamlRestTestPlugin.SOURCE_SET_NAME)
     setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
     setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
     systemProperty 'tests.rest.blacklist', [
diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java
index 79698eb4e282..3d4d426944db 100644
--- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java
+++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java
@@ -15,7 +15,7 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Setting;
@@ -102,7 +102,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         service.set(s3Service(environment));
         this.service.get().refreshAndClearCache(S3ClientSettings.load(settings));
diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/CustomWebIdentityTokenCredentialsProviderTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/CustomWebIdentityTokenCredentialsProviderTests.java
index f151b5f82785..04c47bb9b55e 100644
--- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/CustomWebIdentityTokenCredentialsProviderTests.java
+++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/CustomWebIdentityTokenCredentialsProviderTests.java
@@ -13,6 +13,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.mocksocket.MockHttpServer;
@@ -53,7 +54,7 @@ public void testCreateWebIdentityTokenCredentialsProvider() throws Exception {
                 assertEquals(ROLE_NAME, params.get("RoleSessionName"));
 
                 exchange.getResponseHeaders().add("Content-Type", "text/xml; charset=UTF-8");
-                byte[] response = formatted(
+                byte[] response = Strings.format(
                     """
                         <AssumeRoleWithWebIdentityResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
                           <AssumeRoleWithWebIdentityResult>
diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java
index de9842462d81..158af7b426ab 100644
--- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java
+++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java
@@ -190,7 +190,10 @@ public void testWriteBlobWithRetries() throws Exception {
 
                 if (randomBoolean()) {
                     if (randomBoolean()) {
-                        Streams.readFully(exchange.getRequestBody(), new byte[randomIntBetween(1, Math.max(1, bytes.length - 1))]);
+                        org.elasticsearch.core.Streams.readFully(
+                            exchange.getRequestBody(),
+                            new byte[randomIntBetween(1, Math.max(1, bytes.length - 1))]
+                        );
                     } else {
                         Streams.readFully(exchange.getRequestBody());
                         exchange.sendResponseHeaders(
@@ -222,7 +225,7 @@ public void testWriteBlobWithReadTimeouts() {
         httpServer.createContext(downloadStorageEndpoint(blobContainer, "write_blob_timeout"), exchange -> {
             if (randomBoolean()) {
                 if (randomBoolean()) {
-                    Streams.readFully(exchange.getRequestBody(), new byte[randomIntBetween(1, bytes.length - 1)]);
+                    org.elasticsearch.core.Streams.readFully(exchange.getRequestBody(), new byte[randomIntBetween(1, bytes.length - 1)]);
                 } else {
                     Streams.readFully(exchange.getRequestBody());
                 }
@@ -317,7 +320,10 @@ public void testWriteLargeBlob() throws Exception {
             // sends an error back or let the request time out
             if (useTimeout == false) {
                 if (randomBoolean() && contentLength > 0) {
-                    Streams.readFully(exchange.getRequestBody(), new byte[randomIntBetween(1, Math.toIntExact(contentLength - 1))]);
+                    org.elasticsearch.core.Streams.readFully(
+                        exchange.getRequestBody(),
+                        new byte[randomIntBetween(1, Math.toIntExact(contentLength - 1))]
+                    );
                 } else {
                     Streams.readFully(exchange.getRequestBody());
                     exchange.sendResponseHeaders(
@@ -412,7 +418,10 @@ public void testWriteLargeBlobStreaming() throws Exception {
             // sends an error back or let the request time out
             if (useTimeout == false) {
                 if (randomBoolean() && contentLength > 0) {
-                    Streams.readFully(exchange.getRequestBody(), new byte[randomIntBetween(1, Math.toIntExact(contentLength - 1))]);
+                    org.elasticsearch.core.Streams.readFully(
+                        exchange.getRequestBody(),
+                        new byte[randomIntBetween(1, Math.toIntExact(contentLength - 1))]
+                    );
                 } else {
                     Streams.readFully(exchange.getRequestBody());
                     exchange.sendResponseHeaders(
diff --git a/modules/repository-url/build.gradle b/modules/repository-url/build.gradle
index 164c64ad1f61..7b671802f3a2 100644
--- a/modules/repository-url/build.gradle
+++ b/modules/repository-url/build.gradle
@@ -8,8 +8,8 @@
 
 import org.elasticsearch.gradle.PropertyNormalization
 
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.test.fixtures'
 
diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClient.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClient.java
index 1d8f18390169..580348564b07 100644
--- a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClient.java
+++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClient.java
@@ -21,9 +21,9 @@
 import org.apache.http.ssl.SSLContexts;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.Streams;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.Closeable;
diff --git a/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java b/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java
index ff7218a9e57f..ace6555860f7 100644
--- a/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java
+++ b/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.blobstore.url.http.URLHttpClient;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -88,7 +88,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
 
         final URLHttpClient.Factory apacheURLHttpClientFactory = new URLHttpClient.Factory();
diff --git a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/AbstractURLBlobStoreTests.java b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/AbstractURLBlobStoreTests.java
index dfa4d310c96b..88a248453cc9 100644
--- a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/AbstractURLBlobStoreTests.java
+++ b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/AbstractURLBlobStoreTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -57,7 +58,7 @@ public void testNoBlobFound() throws IOException {
             ignored.read();
             fail("Should have thrown NoSuchFileException exception");
         } catch (NoSuchFileException e) {
-            assertEquals(formatted("blob object [%s] not found", incorrectBlobName), e.getMessage());
+            assertEquals(Strings.format("blob object [%s] not found", incorrectBlobName), e.getMessage());
         }
     }
 }
diff --git a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/http/RetryingHttpInputStreamTests.java b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/http/RetryingHttpInputStreamTests.java
index 2c8e7312def8..d1425e56072e 100644
--- a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/http/RetryingHttpInputStreamTests.java
+++ b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/http/RetryingHttpInputStreamTests.java
@@ -10,6 +10,7 @@
 
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
 
@@ -17,7 +18,6 @@
 import java.net.URI;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -44,7 +44,7 @@ public void testUnderlyingInputStreamIsAbortedAfterAFailureAndRetries() throws E
         when(secondHttpResponseInputStream.read(any(), anyInt(), anyInt())).thenReturn(blobSize - firstChunkSize).thenReturn(-1);
         final Map<String, String> secondResponseHeaders = Map.of(
             "Content-Range",
-            String.format(Locale.ROOT, "bytes %d-%d/%d", firstChunkSize, blobSize - 1, blobSize)
+            Strings.format("bytes %d-%d/%d", firstChunkSize, blobSize - 1, blobSize)
         );
 
         final List<MockHttpResponse> responses = List.of(
diff --git a/modules/runtime-fields-common/build.gradle b/modules/runtime-fields-common/build.gradle
index 5a2d268cf7a4..c4db67a89d36 100644
--- a/modules/runtime-fields-common/build.gradle
+++ b/modules/runtime-fields-common/build.gradle
@@ -7,8 +7,8 @@
  */
 
 apply plugin: 'elasticsearch.validate-rest-spec'
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
   description 'Module for runtime fields features and extensions that have large dependencies'
diff --git a/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsCommonPlugin.java b/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsCommonPlugin.java
index 5629406a662e..bc0b39e98999 100644
--- a/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsCommonPlugin.java
+++ b/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsCommonPlugin.java
@@ -10,7 +10,7 @@
 
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Setting;
@@ -76,7 +76,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         grokHelper.finishInitializing(threadPool);
         return List.of();
diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/114_composite_errors.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/114_composite_errors.yml
new file mode 100644
index 000000000000..cfaafedd943a
--- /dev/null
+++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/114_composite_errors.yml
@@ -0,0 +1,124 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            runtime:
+              rtf:
+                type: composite
+                on_script_error: continue
+                script:
+                  source: |
+                    if (doc["message"].value.equals("fail")) throw new Exception("error");
+                    emit('msg', doc['message'].value);
+                fields:
+                  msg:
+                    type: keyword
+              rtf_strict:
+                type: composite
+                on_script_error: fail
+                script:
+                  source: |
+                    if (doc["message"].value.equals("fail")) throw new Exception("error");
+                    emit('msg', doc['message'].value);
+                fields:
+                  msg:
+                    type: keyword
+            properties:
+              timestamp:
+                type: date
+              message:
+                type: keyword
+
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body: |
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:30:17-05:00", "message" : "this is okay"}
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:30:53-05:00", "message" : "fail"}
+
+---
+"query with continue on error":
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            term:
+              rtf.msg: "this is okay"
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._source.message: "this is okay"}
+
+---
+"query with fail on error":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: test
+        body:
+          query:
+            term:
+              rtf_strict.msg: "this is okay"
+
+---
+"query with search time field":
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            term:
+              rtf_search.msg: "this is okay"
+          runtime_mappings:
+            rtf_search:
+              type: composite
+              on_script_error: continue
+              script:
+                source: |
+                  if (doc["message"].value.equals("fail")) throw new Exception("error");
+                  emit('msg', doc['message'].value);
+              fields:
+                msg:
+                  type: keyword
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._source.message: "this is okay"}
+
+---
+fetch:
+  - do:
+      search:
+        index: test
+        body:
+          sort: timestamp
+          fields:
+            - message
+            - rtf.msg
+  - match: {hits.total.value: 2}
+  - match: {hits.hits.0.fields.message: ["this is okay"] }
+  - match: {hits.hits.0.fields.rtf\.msg: ["this is okay"] }
+  - match: {hits.hits.1.fields.message: ["fail"] }
+  - is_false: hits.hits.1.fields.rtf.msg
+
+---
+"terms agg":
+  - do:
+      search:
+        index: test
+        body:
+          aggs:
+            messages:
+              terms:
+                field: rtf.msg
+  - match: { hits.total.value: 2}
+  - length: { aggregations.messages.buckets: 1 }
+  - match: { aggregations.messages.buckets.0.key: "this is okay" }
+  - match: { aggregations.messages.buckets.0.doc_count: 1 }
diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/14_keyword_errors.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/14_keyword_errors.yml
new file mode 100644
index 000000000000..d21259ab551a
--- /dev/null
+++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/14_keyword_errors.yml
@@ -0,0 +1,216 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: testindex
+        body:
+          settings:
+            number_of_shards: 1
+          mappings:
+            runtime:
+              first_char:
+                type: keyword
+                script: |
+                  emit(doc['name'].value.substring(0,1));
+                on_script_error: continue
+              first_char_strict_error:
+                type: keyword
+                script: |
+                  emit(doc['name'].value.substring(0,1));
+                on_script_error: fail
+            properties:
+              name:
+                type: keyword
+
+  - do:
+      bulk:
+        index: testindex
+        refresh: true
+        body: |
+          {"index":{}}
+          {"name": "foo"}
+          {"index":{}}
+          {"name": ""}
+
+---
+"Query rtf with on_script_error continue":
+  - do:
+      search:
+        index: testindex
+        body:
+          query:
+            match:
+              first_char: "f"
+          fields: [ name, first_char ]
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0.fields.name: [ foo ] }
+  - match: { hits.hits.0.fields.first_char: [ f ] }
+
+---
+"Query rtf with on_script_error fail":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query:
+            match:
+              first_char_strict_error: "f"
+          fields: [ name, first_char_strict_error ]
+
+---
+"Aggregate on rtf with on_script_error continue":
+  - do:
+      search:
+        index: testindex
+        body:
+          aggs:
+            firstchar:
+              terms:
+                field: first_char
+  - length: { aggregations.firstchar.buckets: 1 }
+  - match: { aggregations.firstchar.buckets.0.key: "f" }
+
+---
+"Aggregate on rtf with on_script_error fail":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          aggs:
+            firstchar:
+              terms:
+                field: first_char_strict_error
+
+---
+"Fields retrieval with ignoring error":
+  - do:
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ name, first_char ]
+  - match: { hits.total.value: 2 }
+  - match: { hits.hits.0.fields.name: [ foo ] }
+  - match: { hits.hits.0.fields.first_char: [ f ] }
+  - match: { hits.hits.1.fields.name: [ "" ] }
+  - is_false: hits.hits.1.fields.first_char
+
+---
+"Fields retrieval with failing on error":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ name, first_char_strict_error ]
+
+---
+"Sorting with ignoring error":
+  - do:
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ name ]
+          sort: first_char
+  - match: { hits.total.value: 2 }
+  - match: { hits.hits.0.fields.name: [ foo ] }
+  - match: { hits.hits.1.fields.name: [ "" ] }
+
+---
+"Sorting with with failing on error":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ name ]
+          sort: first_char_strict_error
+
+---
+"Query search time rtf with on_script_error continue":
+  - do:
+      search:
+        index: testindex
+        body:
+          query:
+            match:
+              first_char_search: "f"
+          fields: [ name, first_char_search ]
+          runtime_mappings:
+            first_char_search:
+              type: keyword
+              script: |
+                emit(doc['name'].value.substring(0,1));
+              on_script_error: continue
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0.fields.name: [ foo ] }
+  - match: { hits.hits.0.fields.first_char_search: [ f ] }
+
+---
+"Query search time rtf with on_script_error fail":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query:
+            match:
+              first_char_search: "f"
+          fields: [ name, first_char_search ]
+          runtime_mappings:
+            first_char_search:
+              type: keyword
+              script: |
+                emit(doc['name'].value.substring(0,1));
+              on_script_error: fail
+
+---
+"Change error behaviour for lenient runtime field":
+
+  - do:
+      indices.put_mapping:
+        index: testindex
+        body:
+          runtime:
+            first_char_variant:
+              type: keyword
+              script: |
+                emit(doc['name'].value.substring(0,1));
+
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query:
+            match:
+              first_char_variant: "f"
+
+  - do:
+      indices.put_mapping:
+        index: testindex
+        body:
+          runtime:
+            first_char_variant:
+              type: keyword
+              script: |
+                emit(doc['name'].value.substring(0,1));
+              on_script_error: continue
+
+  - do:
+      search:
+        index: testindex
+        body:
+          query:
+            match:
+              first_char_variant: "f"
+          fields: [ name, first_char_variant ]
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0.fields.name: [ foo ] }
+  - match: { hits.hits.0.fields.first_char_variant: [ f ] }
diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/24_long_errors.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/24_long_errors.yml
new file mode 100644
index 000000000000..a832ff78bbed
--- /dev/null
+++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/24_long_errors.yml
@@ -0,0 +1,176 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: testindex
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            runtime:
+              rtf:
+                type: long
+                script: |
+                  if(doc['name'].value.equals("")) throw new Exception("empty");
+                  emit(doc['name'].value.length());
+                on_script_error: continue
+              rtf_strict_error:
+                type: long
+                script: |
+                  if(doc['name'].value.equals("")) throw new Exception("empty");
+                  emit(doc['name'].value.length());
+                on_script_error: fail
+            properties:
+              name:
+                type: keyword
+
+  - do:
+      bulk:
+        index: testindex
+        refresh: true
+        body: |
+          {"index":{}}
+          {"name": "foo"}
+          {"index":{}}
+          {"name": ""}
+
+---
+"Query rtf with on_script_error continue":
+  - do:
+      search:
+        index: testindex
+        body:
+          query:
+            match:
+              rtf: 3
+          fields: [ name, rtf ]
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0.fields.name: [ foo ] }
+  - match: { hits.hits.0.fields.rtf: [ 3 ] }
+
+---
+"Query rtf with on_script_error fail":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query:
+            match:
+              rtf_strict_error: 3
+          fields: [ name, rtf_strict_error ]
+
+---
+"Aggregate on rtf with on_script_error continue":
+  - do:
+      search:
+        index: testindex
+        body:
+          aggs:
+            firstchar:
+              terms:
+                field: rtf
+  - length: { aggregations.firstchar.buckets: 1 }
+  - match: { aggregations.firstchar.buckets.0.key: 3 }
+
+---
+"Aggregate on rtf with on_script_error fail":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          aggs:
+            firstchar:
+              terms:
+                field: rtf_strict_error
+
+---
+"Fields retrieval with ignoring error":
+  - do:
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ name, rtf ]
+  - match: { hits.total.value: 2 }
+  - match: { hits.hits.0.fields.name: [ foo ] }
+  - match: { hits.hits.0.fields.rtf: [ 3 ] }
+  - match: { hits.hits.1.fields.name: [ "" ] }
+  - is_false: hits.hits.1.fields.rtf
+
+---
+"Fields retrieval with failing on error":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ name, rtf_strict_error ]
+
+---
+"Sorting with ignoring error":
+  - do:
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ name ]
+          sort: rtf
+  - match: { hits.total.value: 2 }
+  - match: { hits.hits.0.fields.name: [ foo ] }
+  - match: { hits.hits.1.fields.name: [ "" ] }
+
+---
+"Sorting with with failing on error":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ name ]
+          sort: rtf_strict_error
+
+---
+"Query search time rtf with on_script_error continue":
+  - do:
+      search:
+        index: testindex
+        body:
+          query:
+            match:
+              rtf_search: 3
+          fields: [ name, rtf_search ]
+          runtime_mappings:
+            rtf_search:
+              type: long
+              script: |
+                if(doc['name'].value.equals("")) throw new Exception("empty");
+                emit(doc['name'].value.length());
+              on_script_error: continue
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0.fields.name: [ foo ] }
+  - match: { hits.hits.0.fields.rtf_search: [ 3 ] }
+
+---
+"Query search time rtf with on_script_error fail":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query:
+            match:
+              rtf_search: 3
+          fields: [ name, rtf_search ]
+          runtime_mappings:
+            rtf_search:
+              type: long
+              script: |
+                if(doc['name'].value.equals("")) throw new Exception("empty");
+                emit(doc['name'].value.length());
+              on_script_error: fail
diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/34_double_errors.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/34_double_errors.yml
new file mode 100644
index 000000000000..2371dd2f3f49
--- /dev/null
+++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/34_double_errors.yml
@@ -0,0 +1,176 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: testindex
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            runtime:
+              rtf:
+                type: double
+                script: |
+                  if(doc['name'].value.equals("")) throw new Exception("empty");
+                  emit(doc['name'].value.length());
+                on_script_error: continue
+              rtf_strict_error:
+                type: double
+                script: |
+                  if(doc['name'].value.equals("")) throw new Exception("empty");
+                  emit(doc['name'].value.length());
+                on_script_error: fail
+            properties:
+              name:
+                type: keyword
+
+  - do:
+      bulk:
+        index: testindex
+        refresh: true
+        body: |
+          {"index":{}}
+          {"name": "foo"}
+          {"index":{}}
+          {"name": ""}
+
+---
+"Query rtf with on_script_error continue":
+  - do:
+      search:
+        index: testindex
+        body:
+          query:
+            match:
+              rtf: 3
+          fields: [ name, rtf ]
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0.fields.name: [ foo ] }
+  - match: { hits.hits.0.fields.rtf: [ 3.0 ] }
+
+---
+"Query rtf with on_script_error fail":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query:
+            match:
+              rtf_strict_error: 3
+          fields: [ name, rtf_strict_error ]
+
+---
+"Aggregate on rtf with on_script_error continue":
+  - do:
+      search:
+        index: testindex
+        body:
+          aggs:
+            firstchar:
+              terms:
+                field: rtf
+  - length: { aggregations.firstchar.buckets: 1 }
+  - match: { aggregations.firstchar.buckets.0.key: 3.0 }
+
+---
+"Aggregate on rtf with on_script_error fail":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          aggs:
+            firstchar:
+              terms:
+                field: rtf_strict_error
+
+---
+"Fields retrieval with ignoring error":
+  - do:
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ name, rtf ]
+  - match: { hits.total.value: 2 }
+  - match: { hits.hits.0.fields.name: [ foo ] }
+  - match: { hits.hits.0.fields.rtf: [ 3.0 ] }
+  - match: { hits.hits.1.fields.name: [ "" ] }
+  - is_false: hits.hits.1.fields.rtf
+
+---
+"Fields retrieval with failing on error":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ name, rtf_strict_error ]
+
+---
+"Sorting with ignoring error":
+  - do:
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ name ]
+          sort: rtf
+  - match: { hits.total.value: 2 }
+  - match: { hits.hits.0.fields.name: [ foo ] }
+  - match: { hits.hits.1.fields.name: [ "" ] }
+
+---
+"Sorting with with failing on error":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ name ]
+          sort: rtf_strict_error
+
+---
+"Query search time rtf with on_script_error continue":
+  - do:
+      search:
+        index: testindex
+        body:
+          query:
+            match:
+              rtf_search: 3
+          fields: [ name, rtf_search ]
+          runtime_mappings:
+            rtf_search:
+              type: double
+              script: |
+                if(doc['name'].value.equals("")) throw new Exception("empty");
+                emit(doc['name'].value.length());
+              on_script_error: continue
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0.fields.name: [ foo ] }
+  - match: { hits.hits.0.fields.rtf_search: [ 3.0 ] }
+
+---
+"Query search time rtf with on_script_error fail":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query:
+            match:
+              rtf_search: 3
+          fields: [ name, rtf_search ]
+          runtime_mappings:
+            rtf_search:
+              type: double
+              script: |
+                if(doc['name'].value.equals("")) throw new Exception("empty");
+                emit(doc['name'].value.length());
+              on_script_error: fail
diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/44_date_errors.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/44_date_errors.yml
new file mode 100644
index 000000000000..e38de91c25d8
--- /dev/null
+++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/44_date_errors.yml
@@ -0,0 +1,184 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: testindex
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            runtime:
+              rtf:
+                type: date
+                format: yyyy-MM-dd
+                script: |
+                  if(doc['millis_since_epoch'].value < 0) throw new Exception("date before 1970");
+                  emit(doc['millis_since_epoch'].value);
+                on_script_error: continue
+              rtf_strict_error:
+                type: date
+                format: yyyy-MM-dd
+                script: |
+                  if(doc['millis_since_epoch'].value < 0) throw new Exception("date before 1970");
+                  emit(doc['millis_since_epoch'].value);
+                on_script_error: fail
+            properties:
+              millis_since_epoch:
+                type: long
+
+  - do:
+      bulk:
+        index: testindex
+        refresh: true
+        body: |
+          {"index":{}}
+          {"millis_since_epoch": 1671033474411}
+          {"index":{}}
+          {"millis_since_epoch": -1}
+
+---
+"Query rtf with on_script_error continue":
+  - do:
+      search:
+        index: testindex
+        body:
+          query:
+            range:
+              rtf:
+                gte: "2022-12-14"
+          fields: [ millis_since_epoch, rtf ]
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0.fields.millis_since_epoch: [ 1671033474411 ] }
+  - match: { hits.hits.0.fields.rtf: [ "2022-12-14" ] }
+
+---
+"Query rtf with on_script_error fail":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query:
+            range:
+              rtf_strict_error:
+                gte: "2022-12-14"
+          fields: [ millis_since_epoch, rtf_strict_error ]
+
+---
+"Aggregate on rtf with on_script_error continue":
+  - do:
+      search:
+        index: testindex
+        body:
+          aggs:
+            firstchar:
+              terms:
+                field: rtf
+  - length: { aggregations.firstchar.buckets: 1 }
+  - match: { aggregations.firstchar.buckets.0.key_as_string: "2022-12-14" }
+
+---
+"Aggregate on rtf with on_script_error fail":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          aggs:
+            firstchar:
+              terms:
+                field: rtf_strict_error
+
+---
+"Fields retrieval with ignoring error":
+  - do:
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ millis_since_epoch, rtf ]
+  - match: { hits.total.value: 2 }
+  - match: { hits.hits.0.fields.millis_since_epoch: [ 1671033474411 ] }
+  - match: { hits.hits.0.fields.rtf: [ "2022-12-14" ] }
+  - match: { hits.hits.1.fields.millis_since_epoch: [ -1 ] }
+  - is_false: hits.hits.1.fields.rtf
+
+---
+"Fields retrieval with failing on error":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ millis_since_epoch, rtf_strict_error ]
+
+---
+"Sorting with ignoring error":
+  - do:
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ millis_since_epoch ]
+          sort: rtf
+  - match: { hits.total.value: 2 }
+  - match: { hits.hits.0.fields.millis_since_epoch: [ 1671033474411 ] }
+  - match: { hits.hits.1.fields.millis_since_epoch: [ -1 ] }
+
+---
+"Sorting with with failing on error":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ millis_since_epoch ]
+          sort: rtf_strict_error
+
+---
+"Query search time rtf with on_script_error continue":
+  - do:
+      search:
+        index: testindex
+        body:
+          query:
+            range:
+              rtf_search:
+                gte: "2022-12-14"
+          fields: [ millis_since_epoch, rtf_search ]
+          runtime_mappings:
+            rtf_search:
+              type: date
+              format: yyyy-MM-dd
+              script: |
+                if(doc['millis_since_epoch'].value < 0) throw new Exception("date before 1970");
+                emit(doc['millis_since_epoch'].value);
+              on_script_error: continue
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0.fields.millis_since_epoch: [ 1671033474411 ] }
+  - match: { hits.hits.0.fields.rtf_search: [ "2022-12-14" ] }
+
+---
+"Query search time rtf with on_script_error fail":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query:
+            range:
+              rtf_search:
+                gte: "2022-12-14"
+          fields: [ millis_since_epoch, rtf_search ]
+          runtime_mappings:
+            rtf_search:
+              type: date
+              format: yyyy-MM-dd
+              script: |
+                if(doc['millis_since_epoch'].value < 0) throw new Exception("date before 1970");
+                emit(doc['millis_since_epoch'].value);
+              on_script_error: fail
diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/54_ip_errors.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/54_ip_errors.yml
new file mode 100644
index 000000000000..1dfbfa6117a6
--- /dev/null
+++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/54_ip_errors.yml
@@ -0,0 +1,177 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: testindex
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            runtime:
+              rtf:
+                type: ip
+                script: |
+                  if(doc['ip_string'].value.length() <= 0) throw new Exception("empty");
+                  emit(doc['ip_string'].value);
+                on_script_error: continue
+              rtf_strict_error:
+                type: ip
+                script: |
+                  if(doc['ip_string'].value.length() <= 0) throw new Exception("empty");
+                  emit(doc['ip_string'].value);
+                on_script_error: fail
+            properties:
+              ip_string:
+                type: keyword
+
+  - do:
+      bulk:
+        index: testindex
+        refresh: true
+        body: |
+          {"index":{}}
+          {"ip_string": "192.68.0.1"}
+          {"index":{}}
+          {"ip_string": ""}
+
+---
+"Query rtf with on_script_error continue":
+  - do:
+      search:
+        index: testindex
+        body:
+          query:
+            term:
+              rtf:
+                192.68.0.1
+          fields: [ ip_string, rtf ]
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0.fields.ip_string: [ "192.68.0.1" ] }
+  - match: { hits.hits.0.fields.rtf: [ 192.68.0.1 ] }
+
+---
+"Query rtf with on_script_error fail":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query:
+            term:
+              rtf_strict_error: 192.68.0.1
+          fields: [ ip_string, rtf_strict_error ]
+
+---
+"Aggregate on rtf with on_script_error continue":
+  - do:
+      search:
+        index: testindex
+        body:
+          aggs:
+            rtf:
+              terms:
+                field: rtf
+  - length: { aggregations.rtf.buckets: 1 }
+  - match: { aggregations.rtf.buckets.0.key: 192.68.0.1 }
+
+---
+"Aggregate on rtf with on_script_error fail":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          aggs:
+            rtf:
+              terms:
+                field: rtf_strict_error
+
+---
+"Fields retrieval with ignoring error":
+  - do:
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ ip_string, rtf ]
+  - match: { hits.total.value: 2 }
+  - match: { hits.hits.0.fields.ip_string: [ "192.68.0.1" ] }
+  - match: { hits.hits.0.fields.rtf: [ "192.68.0.1" ] }
+  - match: { hits.hits.1.fields.ip_string: [ "" ] }
+  - is_false: hits.hits.1.fields.rtf
+
+---
+"Fields retrieval with failing on error":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ ip_string, rtf_strict_error ]
+
+---
+"Sorting with ignoring error":
+  - do:
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ ip_string ]
+          sort: rtf
+  - match: { hits.total.value: 2 }
+  - match: { hits.hits.0.fields.ip_string: [ "192.68.0.1" ] }
+  - match: { hits.hits.1.fields.ip_string: [ "" ] }
+
+---
+"Sorting with with failing on error":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ ip_string ]
+          sort: rtf_strict_error
+
+---
+"Query search time rtf with on_script_error continue":
+  - do:
+      search:
+        index: testindex
+        body:
+          query:
+            match:
+              rtf_search: "192.68.0.1"
+          fields: [ ip_string, rtf_search ]
+          runtime_mappings:
+            rtf_search:
+              type: ip
+              script: |
+                if(doc['ip_string'].value.length() <= 0) throw new Exception("empty");
+                emit(doc['ip_string'].value);
+              on_script_error: continue
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0.fields.ip_string: [ "192.68.0.1" ] }
+  - match: { hits.hits.0.fields.rtf_search: [ "192.68.0.1" ] }
+
+---
+"Query search time rtf with on_script_error fail":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query:
+            match:
+              rtf_search: "192.68.0.1"
+          fields: [ ip_string, rtf_search ]
+          runtime_mappings:
+            rtf_search:
+              type: ip
+              script: |
+                if(doc['ip_string'].value.length() <= 0) throw new Exception("empty");
+                emit(doc['ip_string'].value);
+              on_script_error: fail
diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/64_boolean_errors.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/64_boolean_errors.yml
new file mode 100644
index 000000000000..bfdd15b26994
--- /dev/null
+++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/64_boolean_errors.yml
@@ -0,0 +1,184 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: testindex
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            runtime:
+              rtf:
+                type: boolean
+                script: |
+                  if(doc['age'].value < 0) throw new Exception("invalid age");
+                  emit(doc['age'].value >= 18);
+                on_script_error: continue
+              rtf_strict_error:
+                type: boolean
+                script: |
+                  if(doc['age'].value <= 0) throw new Exception("invalid age");
+                  emit(doc['age'].value >=18);
+                on_script_error: fail
+            properties:
+              age:
+                type: integer
+
+  - do:
+      bulk:
+        index: testindex
+        refresh: true
+        body: |
+          {"index":{}}
+          {"age": 14}
+          {"index":{}}
+          {"age": 20}
+          {"index":{}}
+          {"age": -1}
+
+---
+"Query rtf with on_script_error continue":
+  - do:
+      search:
+        index: testindex
+        body:
+          query:
+            match:
+              rtf:
+                true
+          fields: [ age, rtf ]
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0.fields.age: [ 20 ] }
+  - match: { hits.hits.0.fields.rtf: [ true ] }
+
+---
+"Query rtf with on_script_error fail":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query:
+            match:
+              rtf_strict_error: true
+          fields: [ age, rtf_strict_error ]
+
+---
+"Aggregate on rtf with on_script_error continue":
+  - do:
+      search:
+        index: testindex
+        body:
+          aggs:
+            rtf:
+              terms:
+                field: rtf
+                order: { "_key": "asc" }
+  - length: { aggregations.rtf.buckets: 2 }
+  - match: { aggregations.rtf.buckets.0.key_as_string: "false" }
+  - match: { aggregations.rtf.buckets.1.key_as_string: "true" }
+---
+"Aggregate on rtf with on_script_error fail":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          aggs:
+            rtf:
+              terms:
+                field: rtf_strict_error
+
+---
+"Fields retrieval with ignoring error":
+  - do:
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ age, rtf ]
+          sort: { "age": "desc" }
+  - match: { hits.total.value: 3 }
+  - match: { hits.hits.0.fields.age: [ 20 ] }
+  - match: { hits.hits.0.fields.rtf: [ true ] }
+  - match: { hits.hits.1.fields.age: [ 14 ] }
+  - match: { hits.hits.1.fields.rtf: [ false ] }
+  - match: { hits.hits.2.fields.age: [ -1 ] }
+  - is_false: hits.hits.2.fields.rtf
+
+---
+"Fields retrieval with failing on error":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ age, rtf_strict_error ]
+
+---
+"Sorting with ignoring error":
+  - do:
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ age ]
+          sort: rtf
+  - match: { hits.total.value: 3 }
+  - match: { hits.hits.0.fields.age: [ 14 ] }
+  - match: { hits.hits.1.fields.age: [ 20 ] }
+  - match: { hits.hits.2.fields.age: [ -1 ] }
+
+---
+"Sorting with with failing on error":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query: { match_all: { } }
+          fields: [ age ]
+          sort: rtf_strict_error
+
+---
+"Query search time rtf with on_script_error continue":
+  - do:
+      search:
+        index: testindex
+        body:
+          query:
+            match:
+              rtf_search: true
+          fields: [ age, rtf_search ]
+          runtime_mappings:
+            rtf_search:
+              type: boolean
+              script: |
+                if(doc['age'].value < 0) throw new Exception("invalid age");
+                emit(doc['age'].value >= 18);
+              on_script_error: continue
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0.fields.age: [ 20 ] }
+  - match: { hits.hits.0.fields.rtf_search: [ true ] }
+
+---
+"Query search time rtf with on_script_error fail":
+  - do:
+      catch: /type=script_exception, reason=runtime error/
+      search:
+        index: testindex
+        body:
+          query:
+            match:
+              rtf_search: true
+          fields: [ age, rtf_search ]
+          runtime_mappings:
+            rtf_search:
+              type: boolean
+              script: |
+                if(doc['age'].value < 0) throw new Exception("invalid age");
+                emit(doc['age'].value >= 18);
+              on_script_error: fail
diff --git a/modules/systemd/src/main/java/org/elasticsearch/systemd/SystemdPlugin.java b/modules/systemd/src/main/java/org/elasticsearch/systemd/SystemdPlugin.java
index f7690700d513..27bd1791dc4d 100644
--- a/modules/systemd/src/main/java/org/elasticsearch/systemd/SystemdPlugin.java
+++ b/modules/systemd/src/main/java/org/elasticsearch/systemd/SystemdPlugin.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.Build;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.core.TimeValue;
@@ -91,7 +91,7 @@ public Collection<Object> createComponents(
         final IndexNameExpressionResolver expressionResolver,
         final Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         if (enabled == false) {
             extender.set(null);
diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle
index bf43c4a88d91..5a6a13d5fd3f 100644
--- a/modules/transport-netty4/build.gradle
+++ b/modules/transport-netty4/build.gradle
@@ -7,12 +7,12 @@
  */
 
 import org.elasticsearch.gradle.internal.test.RestIntegTestTask
-import org.elasticsearch.gradle.internal.test.rest.InternalJavaRestTestPlugin
+import org.elasticsearch.gradle.internal.test.rest.LegacyJavaRestTestPlugin
 import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin
 
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.publish'
 
@@ -34,6 +34,8 @@ configurations {
 }
 
 dependencies {
+  api project(":libs:elasticsearch-ssl-config")
+
   // network stack
   api "io.netty:netty-buffer:${versions.netty}"
   api "io.netty:netty-codec:${versions.netty}"
@@ -71,7 +73,7 @@ TaskProvider<Test> pooledInternalClusterTest = tasks.register("pooledInternalClu
 
 TaskProvider<RestIntegTestTask> pooledJavaRestTest = tasks.register("pooledJavaRestTest", RestIntegTestTask) {
     SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-    SourceSet javaRestTestSourceSet = sourceSets.getByName(InternalJavaRestTestPlugin.SOURCE_SET_NAME)
+    SourceSet javaRestTestSourceSet = sourceSets.getByName(LegacyJavaRestTestPlugin.SOURCE_SET_NAME)
     setTestClassesDirs(javaRestTestSourceSet.getOutput().getClassesDirs())
     setClasspath(javaRestTestSourceSet.getRuntimeClasspath())
 
@@ -91,7 +93,8 @@ tasks.named("thirdPartyAudit").configure {
     'com.aayushatharva.brotli4j.Brotli4jLoader',
     'com.aayushatharva.brotli4j.decoder.DecoderJNI$Status',
     'com.aayushatharva.brotli4j.decoder.DecoderJNI$Wrapper',
-    'com.aayushatharva.brotli4j.encoder.Encoders',
+    'com.aayushatharva.brotli4j.encoder.BrotliEncoderChannel',
+    'com.aayushatharva.brotli4j.encoder.Encoder',
     'com.aayushatharva.brotli4j.encoder.Encoder$Mode',
     'com.aayushatharva.brotli4j.encoder.Encoder$Parameters',
 
diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java
index eaecc13a15d8..b381e0ea8bfb 100644
--- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java
+++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java
@@ -13,6 +13,7 @@
 
 import org.elasticsearch.ESNetty4IntegTestCase;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
@@ -53,7 +54,7 @@ public void testThatNettyHttpServerSupportsPipelining() throws Exception {
     private void assertOpaqueIdsInOrder(Collection<String> opaqueIds) {
         // check if opaque ids are monotonically increasing
         int i = 0;
-        String msg = formatted("Expected list of opaque ids to be monotonically increasing, got [%s]", opaqueIds);
+        String msg = Strings.format("Expected list of opaque ids to be monotonically increasing, got [%s]", opaqueIds);
         for (String opaqueId : opaqueIds) {
             assertThat(msg, opaqueId, is(String.valueOf(i++)));
         }
diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java
index 4566d2a56d95..3eccb8386066 100644
--- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java
+++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java
@@ -14,13 +14,12 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.junit.annotations.Network;
 import org.elasticsearch.transport.TransportInfo;
 
-import java.util.Locale;
-
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasKey;
@@ -39,7 +38,7 @@ public class Netty4TransportMultiPortIntegrationIT extends ESNetty4IntegTestCase
     protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
         if (randomPort == -1) {
             randomPort = randomIntBetween(49152, 65525);
-            randomPortRange = String.format(Locale.ROOT, "%s-%s", randomPort, randomPort + 10);
+            randomPortRange = Strings.format("%s-%s", randomPort, randomPort + 10);
         }
         Settings.Builder builder = Settings.builder()
             .put(super.nodeSettings(nodeOrdinal, otherSettings))
diff --git a/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java b/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java
index b1e2a6d7fdd1..e705c4303687 100644
--- a/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java
+++ b/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java
@@ -25,6 +25,7 @@
 import static org.elasticsearch.rest.RestStatus.OK;
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.nullValue;
 
 public class Netty4HeadBodyIsEmptyIT extends ESRestTestCase {
     public void testHeadRoot() throws IOException {
@@ -59,8 +60,8 @@ public void testDocumentExists() throws IOException {
 
     public void testIndexExists() throws IOException {
         createTestDoc();
-        headTestCase("/test", emptyMap(), greaterThan(0));
-        headTestCase("/test", singletonMap("pretty", "true"), greaterThan(0));
+        headTestCase("/test", emptyMap(), nullValue(Integer.class));
+        headTestCase("/test", singletonMap("pretty", "true"), nullValue(Integer.class));
     }
 
     public void testAliasExists() throws IOException {
@@ -177,7 +178,8 @@ private void headTestCase(
         request.setOptions(expectWarnings(expectedWarnings));
         Response response = client().performRequest(request);
         assertEquals(expectedStatusCode, response.getStatusLine().getStatusCode());
-        assertThat(Integer.valueOf(response.getHeader("Content-Length")), matcher);
+        final var contentLength = response.getHeader("Content-Length");
+        assertThat(contentLength == null ? null : Integer.valueOf(contentLength), matcher);
         assertNull("HEAD requests shouldn't have a response body but " + url + " did", response.getEntity());
     }
 
diff --git a/modules/transport-netty4/src/main/java/module-info.java b/modules/transport-netty4/src/main/java/module-info.java
index 92217b419c66..5f94b97be782 100644
--- a/modules/transport-netty4/src/main/java/module-info.java
+++ b/modules/transport-netty4/src/main/java/module-info.java
@@ -10,6 +10,7 @@
     requires jdk.net;
     requires org.elasticsearch.base;
     requires org.elasticsearch.server;
+    requires org.elasticsearch.sslconfig;
     requires org.elasticsearch.xcontent;
     requires org.apache.logging.log4j;
     requires org.apache.lucene.core;
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java
index 61c98788f17f..f5a32a0ec768 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java
@@ -216,7 +216,8 @@ private void doWrite(ChannelHandlerContext ctx, Netty4ChunkedHttpResponse readyR
         combiner.add((Future<Void>) first);
         currentChunkedWrite = new ChunkedWrite(combiner, promise, readyResponse);
         if (enqueueWrite(ctx, readyResponse, first)) {
-            // we were able to write out the first chunk directly, try writing out subsequent chunks until the channel becomes unwritable
+            // We were able to write out the first chunk directly, try writing out subsequent chunks until the channel becomes unwritable.
+            // NB "writable" means there's space in the downstream ChannelOutboundBuffer, we aren't trying to saturate the physical channel.
             while (ctx.channel().isWritable()) {
                 if (writeChunk(ctx, combiner, readyResponse.body())) {
                     finishChunkedWrite();
@@ -280,6 +281,7 @@ private boolean doFlush(ChannelHandlerContext ctx) throws IOException {
             return false;
         }
         while (channel.isWritable()) {
+            // NB "writable" means there's space in the downstream ChannelOutboundBuffer, we aren't trying to saturate the physical channel.
             WriteOperation currentWrite = queuedWrites.poll();
             if (currentWrite == null) {
                 doWriteQueued(ctx);
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java
index b2550d6ec42a..e741772eedb5 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java
@@ -24,7 +24,10 @@
 import io.netty.handler.codec.http.HttpContentDecompressor;
 import io.netty.handler.codec.http.HttpObjectAggregator;
 import io.netty.handler.codec.http.HttpRequestDecoder;
+import io.netty.handler.codec.http.HttpResponse;
 import io.netty.handler.codec.http.HttpResponseEncoder;
+import io.netty.handler.codec.http.HttpUtil;
+import io.netty.handler.ssl.SslHandler;
 import io.netty.handler.timeout.ReadTimeoutException;
 import io.netty.handler.timeout.ReadTimeoutHandler;
 import io.netty.util.AttributeKey;
@@ -32,6 +35,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.common.network.CloseableChannel;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
@@ -41,23 +45,29 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.http.AbstractHttpServerTransport;
 import org.elasticsearch.http.HttpChannel;
 import org.elasticsearch.http.HttpHandlingSettings;
 import org.elasticsearch.http.HttpReadTimeoutException;
 import org.elasticsearch.http.HttpServerChannel;
+import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.tracing.Tracer;
+import org.elasticsearch.transport.netty4.AcceptChannelHandler;
 import org.elasticsearch.transport.netty4.NetUtils;
 import org.elasticsearch.transport.netty4.Netty4Utils;
 import org.elasticsearch.transport.netty4.Netty4WriteThrottlingHandler;
 import org.elasticsearch.transport.netty4.NettyAllocator;
 import org.elasticsearch.transport.netty4.NettyByteBufSizer;
+import org.elasticsearch.transport.netty4.SSLExceptionHelper;
 import org.elasticsearch.transport.netty4.SharedGroupFactory;
+import org.elasticsearch.transport.netty4.TLSConfig;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
 import java.net.InetSocketAddress;
 import java.util.concurrent.TimeUnit;
+import java.util.function.BiPredicate;
 
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_CHUNK_SIZE;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH;
@@ -131,6 +141,8 @@ public class Netty4HttpServerTransport extends AbstractHttpServerTransport {
 
     private final SharedGroupFactory sharedGroupFactory;
     private final RecvByteBufAllocator recvByteBufAllocator;
+    private final TLSConfig tlsConfig;
+    private final AcceptChannelHandler.AcceptPredicate acceptChannelPredicate;
     private final int readTimeoutMillis;
 
     private final int maxCompositeBufferComponents;
@@ -146,7 +158,10 @@ public Netty4HttpServerTransport(
         Dispatcher dispatcher,
         ClusterSettings clusterSettings,
         SharedGroupFactory sharedGroupFactory,
-        Tracer tracer
+        Tracer tracer,
+        TLSConfig tlsConfig,
+        @Nullable AcceptChannelHandler.AcceptPredicate acceptChannelPredicate
+
     ) {
         super(
             settings,
@@ -161,6 +176,8 @@ public Netty4HttpServerTransport(
         Netty4Utils.setAvailableProcessors(EsExecutors.allocatedProcessors(settings));
         NettyAllocator.logAllocatorDescriptionIfNeeded();
         this.sharedGroupFactory = sharedGroupFactory;
+        this.tlsConfig = tlsConfig;
+        this.acceptChannelPredicate = acceptChannelPredicate;
 
         this.pipeliningMaxEvents = SETTING_PIPELINING_MAX_EVENTS.get(settings);
 
@@ -252,6 +269,9 @@ protected void doStart() {
             serverBootstrap.childOption(ChannelOption.SO_REUSEADDR, reuseAddress);
 
             bindServer();
+            if (acceptChannelPredicate != null) {
+                acceptChannelPredicate.setBoundAddress(boundAddress());
+            }
             success = true;
         } finally {
             if (success == false) {
@@ -279,7 +299,23 @@ protected void stopInternal() {
 
     @Override
     public void onException(HttpChannel channel, Exception cause) {
-        if (cause instanceof ReadTimeoutException) {
+        if (lifecycle.started() == false) {
+            return;
+        }
+
+        if (SSLExceptionHelper.isNotSslRecordException(cause)) {
+            logger.warn("received plaintext http traffic on an https channel, closing connection {}", channel);
+            CloseableChannel.closeChannel(channel);
+        } else if (SSLExceptionHelper.isCloseDuringHandshakeException(cause)) {
+            logger.debug("connection {} closed during ssl handshake", channel);
+            CloseableChannel.closeChannel(channel);
+        } else if (SSLExceptionHelper.isInsufficientBufferRemainingException(cause)) {
+            logger.debug("connection {} closed abruptly", channel);
+            CloseableChannel.closeChannel(channel);
+        } else if (SSLExceptionHelper.isReceivedCertificateUnknownException(cause)) {
+            logger.warn("http client did not trust this server's certificate, closing connection {}", channel);
+            CloseableChannel.closeChannel(channel);
+        } else if (cause instanceof ReadTimeoutException) {
             super.onException(channel, new HttpReadTimeoutException(readTimeoutMillis, cause));
         } else {
             super.onException(channel, cause);
@@ -287,7 +323,7 @@ public void onException(HttpChannel channel, Exception cause) {
     }
 
     public ChannelHandler configureServerChannelHandler() {
-        return new HttpChannelHandler(this, handlingSettings);
+        return new HttpChannelHandler(this, handlingSettings, tlsConfig, acceptChannelPredicate);
     }
 
     static final AttributeKey<Netty4HttpChannel> HTTP_CHANNEL_KEY = AttributeKey.newInstance("es-http-channel");
@@ -297,16 +333,35 @@ protected static class HttpChannelHandler extends ChannelInitializer<Channel> {
 
         private final Netty4HttpServerTransport transport;
         private final HttpHandlingSettings handlingSettings;
-
-        protected HttpChannelHandler(final Netty4HttpServerTransport transport, final HttpHandlingSettings handlingSettings) {
+        private final TLSConfig tlsConfig;
+        private final BiPredicate<String, InetSocketAddress> acceptChannelPredicate;
+
+        protected HttpChannelHandler(
+            final Netty4HttpServerTransport transport,
+            final HttpHandlingSettings handlingSettings,
+            final TLSConfig tlsConfig,
+            @Nullable final BiPredicate<String, InetSocketAddress> acceptChannelPredicate
+        ) {
             this.transport = transport;
             this.handlingSettings = handlingSettings;
+            this.tlsConfig = tlsConfig;
+            this.acceptChannelPredicate = acceptChannelPredicate;
         }
 
         @Override
         protected void initChannel(Channel ch) throws Exception {
             Netty4HttpChannel nettyHttpChannel = new Netty4HttpChannel(ch);
             ch.attr(HTTP_CHANNEL_KEY).set(nettyHttpChannel);
+            if (acceptChannelPredicate != null) {
+                ch.pipeline()
+                    .addLast(
+                        "accept_channel_handler",
+                        new AcceptChannelHandler(acceptChannelPredicate, HttpServerTransport.HTTP_PROFILE_NAME)
+                    );
+            }
+            if (tlsConfig.isTLSEnabled()) {
+                ch.pipeline().addLast("ssl", new SslHandler(tlsConfig.createServerSSLEngine()));
+            }
             ch.pipeline()
                 .addLast("chunked_writer", new Netty4WriteThrottlingHandler(transport.getThreadPool().getThreadContext()))
                 .addLast("byte_buf_sizer", NettyByteBufSizer.INSTANCE);
@@ -324,7 +379,18 @@ protected void initChannel(Channel ch) throws Exception {
             ch.pipeline()
                 .addLast("decoder", decoder)
                 .addLast("decoder_compress", new HttpContentDecompressor())
-                .addLast("encoder", new HttpResponseEncoder())
+                .addLast("encoder", new HttpResponseEncoder() {
+                    @Override
+                    protected boolean isContentAlwaysEmpty(HttpResponse msg) {
+                        // non-chunked responses (Netty4HttpResponse extends Netty's DefaultFullHttpResponse) with chunked transfer
+                        // encoding are only sent by us in response to HEAD requests and must always have an empty body
+                        if (msg instanceof Netty4HttpResponse netty4HttpResponse && HttpUtil.isTransferEncodingChunked(msg)) {
+                            assert netty4HttpResponse.content().isReadable() == false;
+                            return true;
+                        }
+                        return super.isContentAlwaysEmpty(msg);
+                    }
+                })
                 .addLast("aggregator", aggregator);
             if (handlingSettings.compression()) {
                 ch.pipeline().addLast("encoder_compress", new HttpContentCompressor(handlingSettings.compressionLevel()));
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/AcceptChannelHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/AcceptChannelHandler.java
new file mode 100644
index 000000000000..993d70c6e89f
--- /dev/null
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/AcceptChannelHandler.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.transport.netty4;
+
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.handler.ipfilter.AbstractRemoteAddressFilter;
+
+import org.elasticsearch.common.transport.BoundTransportAddress;
+
+import java.net.InetSocketAddress;
+import java.util.function.BiPredicate;
+
+@ChannelHandler.Sharable
+public class AcceptChannelHandler extends AbstractRemoteAddressFilter<InetSocketAddress> {
+
+    private final BiPredicate<String, InetSocketAddress> predicate;
+    private final String profile;
+
+    public AcceptChannelHandler(final BiPredicate<String, InetSocketAddress> predicate, final String profile) {
+        this.predicate = predicate;
+        this.profile = profile;
+    }
+
+    @Override
+    protected boolean accept(final ChannelHandlerContext ctx, final InetSocketAddress remoteAddress) throws Exception {
+        return predicate.test(profile, remoteAddress);
+    }
+
+    public interface AcceptPredicate extends BiPredicate<String, InetSocketAddress> {
+
+        void setBoundAddress(BoundTransportAddress boundHttpTransportAddress);
+    }
+}
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Plugin.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Plugin.java
index e39994c05a44..e5d6042382b7 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Plugin.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Plugin.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.transport.netty4;
 
 import org.apache.lucene.util.SetOnce;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.network.NetworkService;
@@ -78,7 +78,7 @@ public Map<String, Supplier<Transport>> getTransports(
             NETTY_TRANSPORT_NAME,
             () -> new Netty4Transport(
                 settings,
-                Version.CURRENT,
+                TransportVersion.CURRENT,
                 threadPool,
                 networkService,
                 pageCacheRecycler,
@@ -112,7 +112,9 @@ public Map<String, Supplier<HttpServerTransport>> getHttpTransports(
                 dispatcher,
                 clusterSettings,
                 getSharedGroupFactory(settings),
-                tracer
+                tracer,
+                TLSConfig.noTLS(),
+                null
             )
         );
     }
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
index 563dc8c77ac3..2913cc21d133 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
@@ -26,7 +26,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
@@ -42,6 +42,7 @@
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.ConnectionProfile;
 import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.TransportSettings;
 
@@ -98,7 +99,7 @@ public class Netty4Transport extends TcpTransport {
 
     public Netty4Transport(
         Settings settings,
-        Version version,
+        TransportVersion version,
         ThreadPool threadPool,
         NetworkService networkService,
         PageCacheRecycler pageCacheRecycler,
@@ -267,7 +268,7 @@ protected ChannelHandler getServerChannelInitializer(String name) {
         return new ServerChannelInitializer(name);
     }
 
-    protected ChannelHandler getClientChannelInitializer(DiscoveryNode node) {
+    protected ChannelHandler getClientChannelInitializer(DiscoveryNode node, ConnectionProfile connectionProfile) {
         return new ClientChannelInitializer();
     }
 
@@ -275,10 +276,10 @@ protected ChannelHandler getClientChannelInitializer(DiscoveryNode node) {
     static final AttributeKey<Netty4TcpServerChannel> SERVER_CHANNEL_KEY = AttributeKey.newInstance("es-server-channel");
 
     @Override
-    protected Netty4TcpChannel initiateChannel(DiscoveryNode node) throws IOException {
+    protected Netty4TcpChannel initiateChannel(DiscoveryNode node, ConnectionProfile connectionProfile) throws IOException {
         InetSocketAddress address = node.getAddress().address();
         Bootstrap bootstrapWithHandler = clientBootstrap.clone();
-        bootstrapWithHandler.handler(getClientChannelInitializer(node));
+        bootstrapWithHandler.handler(getClientChannelInitializer(node, connectionProfile));
         bootstrapWithHandler.remoteAddress(address);
         ChannelFuture connectFuture = bootstrapWithHandler.connect();
 
@@ -288,7 +289,13 @@ protected Netty4TcpChannel initiateChannel(DiscoveryNode node) throws IOExceptio
             throw new IOException(connectFuture.cause());
         }
 
-        Netty4TcpChannel nettyChannel = new Netty4TcpChannel(channel, false, "default", rstOnClose, connectFuture);
+        Netty4TcpChannel nettyChannel = new Netty4TcpChannel(
+            channel,
+            false,
+            connectionProfile.getTransportProfile(),
+            rstOnClose,
+            connectFuture
+        );
         channel.attr(CHANNEL_KEY).set(nettyChannel);
 
         return nettyChannel;
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/NettyAllocator.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/NettyAllocator.java
index 1a73c0d1a648..ea999ce0f471 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/NettyAllocator.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/NettyAllocator.java
@@ -98,7 +98,6 @@ public class NettyAllocator {
                         maxOrder = 5;
                     }
                 }
-                int tinyCacheSize = PooledByteBufAllocator.defaultTinyCacheSize();
                 int smallCacheSize = PooledByteBufAllocator.defaultSmallCacheSize();
                 int normalCacheSize = PooledByteBufAllocator.defaultNormalCacheSize();
                 boolean useCacheForAllThreads = PooledByteBufAllocator.defaultUseCacheForAllThreads();
@@ -108,7 +107,6 @@ public class NettyAllocator {
                     0,
                     pageSize,
                     maxOrder,
-                    tinyCacheSize,
                     smallCacheSize,
                     normalCacheSize,
                     useCacheForAllThreads
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/SSLExceptionHelper.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/SSLExceptionHelper.java
similarity index 87%
rename from x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/SSLExceptionHelper.java
rename to modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/SSLExceptionHelper.java
index 9bc63be18e04..19294c193605 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/SSLExceptionHelper.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/SSLExceptionHelper.java
@@ -1,10 +1,12 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
  */
-package org.elasticsearch.xpack.core.security.transport;
+
+package org.elasticsearch.transport.netty4;
 
 import io.netty.handler.codec.DecoderException;
 import io.netty.handler.ssl.NotSslRecordException;
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/TLSConfig.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/TLSConfig.java
new file mode 100644
index 000000000000..b78fcd1a5a9b
--- /dev/null
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/TLSConfig.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.transport.netty4;
+
+import org.elasticsearch.common.ssl.SslConfiguration;
+
+import javax.net.ssl.SSLEngine;
+
+public record TLSConfig(SslConfiguration sslConfiguration, EngineProvider engineProvider) {
+
+    public boolean isTLSEnabled() {
+        return sslConfiguration != null;
+    }
+
+    public SSLEngine createServerSSLEngine() {
+        assert isTLSEnabled();
+        SSLEngine sslEngine = engineProvider.create(sslConfiguration, null, -1);
+        sslEngine.setUseClientMode(false);
+        return sslEngine;
+    }
+
+    public static TLSConfig noTLS() {
+        return new TLSConfig(null, null);
+    }
+
+    @FunctionalInterface
+    public interface EngineProvider {
+
+        SSLEngine create(SslConfiguration configuration, String host, int port);
+    }
+}
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java
index 08721c68a34c..4b0757dd5144 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java
@@ -16,8 +16,6 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.common.util.MockPageCacheRecycler;
-import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.HttpTransportSettings;
@@ -30,6 +28,7 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.tracing.Tracer;
 import org.elasticsearch.transport.netty4.SharedGroupFactory;
+import org.elasticsearch.transport.netty4.TLSConfig;
 import org.junit.After;
 import org.junit.Before;
 
@@ -45,13 +44,11 @@
 public class Netty4BadRequestTests extends ESTestCase {
 
     private NetworkService networkService;
-    private PageCacheRecycler recycler;
     private ThreadPool threadPool;
 
     @Before
     public void setup() throws Exception {
         networkService = new NetworkService(Collections.emptyList());
-        recycler = new MockPageCacheRecycler(Settings.EMPTY);
         threadPool = new TestThreadPool("test");
     }
 
@@ -88,7 +85,9 @@ public void dispatchBadRequest(RestChannel channel, ThreadContext threadContext,
                 dispatcher,
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
                 new SharedGroupFactory(Settings.EMPTY),
-                Tracer.NOOP
+                Tracer.NOOP,
+                TLSConfig.noTLS(),
+                null
             )
         ) {
             httpServerTransport.start();
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java
index eb510c50159a..2524be154414 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java
@@ -21,15 +21,14 @@
 import io.netty.handler.codec.http.DefaultFullHttpRequest;
 import io.netty.handler.codec.http.FullHttpRequest;
 import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.handler.codec.http.HttpClientCodec;
 import io.netty.handler.codec.http.HttpContentDecompressor;
 import io.netty.handler.codec.http.HttpHeaderNames;
 import io.netty.handler.codec.http.HttpMethod;
 import io.netty.handler.codec.http.HttpObject;
 import io.netty.handler.codec.http.HttpObjectAggregator;
 import io.netty.handler.codec.http.HttpRequest;
-import io.netty.handler.codec.http.HttpRequestEncoder;
 import io.netty.handler.codec.http.HttpResponse;
-import io.netty.handler.codec.http.HttpResponseDecoder;
 import io.netty.handler.codec.http.HttpVersion;
 
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -175,8 +174,7 @@ private static class CountDownLatchHandler extends ChannelInitializer<SocketChan
         @Override
         protected void initChannel(SocketChannel ch) {
             final int maxContentLength = new ByteSizeValue(100, ByteSizeUnit.MB).bytesAsInt();
-            ch.pipeline().addLast(new HttpResponseDecoder());
-            ch.pipeline().addLast(new HttpRequestEncoder());
+            ch.pipeline().addLast(new HttpClientCodec());
             ch.pipeline().addLast(new HttpContentDecompressor());
             ch.pipeline().addLast(new HttpObjectAggregator(maxContentLength));
             ch.pipeline().addLast(new SimpleChannelInboundHandler<HttpObject>() {
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java
index 0813fd5afc99..42573ea4fe3f 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.tracing.Tracer;
 import org.elasticsearch.transport.netty4.SharedGroupFactory;
+import org.elasticsearch.transport.netty4.TLSConfig;
 import org.junit.After;
 import org.junit.Before;
 
@@ -104,7 +105,9 @@ class CustomNettyHttpServerTransport extends Netty4HttpServerTransport {
                 new NullDispatcher(),
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
                 new SharedGroupFactory(settings),
-                Tracer.NOOP
+                Tracer.NOOP,
+                TLSConfig.noTLS(),
+                null
             );
         }
 
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
index 087f277c819e..8fd764299cfe 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
@@ -40,6 +40,7 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -47,8 +48,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.util.MockPageCacheRecycler;
-import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.http.AbstractHttpServerTransportTestCase;
@@ -57,6 +56,7 @@
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.HttpTransportSettings;
 import org.elasticsearch.http.NullDispatcher;
+import org.elasticsearch.rest.ChunkedRestResponseBody;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -66,6 +66,8 @@
 import org.elasticsearch.tracing.Tracer;
 import org.elasticsearch.transport.netty4.NettyAllocator;
 import org.elasticsearch.transport.netty4.SharedGroupFactory;
+import org.elasticsearch.transport.netty4.TLSConfig;
+import org.elasticsearch.xcontent.ToXContent;
 import org.junit.After;
 import org.junit.Before;
 
@@ -94,14 +96,12 @@ public class Netty4HttpServerTransportTests extends AbstractHttpServerTransportT
 
     private NetworkService networkService;
     private ThreadPool threadPool;
-    private PageCacheRecycler recycler;
     private ClusterSettings clusterSettings;
 
     @Before
     public void setup() throws Exception {
         networkService = new NetworkService(Collections.emptyList());
         threadPool = new TestThreadPool("test");
-        recycler = new MockPageCacheRecycler(Settings.EMPTY);
         clusterSettings = randomClusterSettings();
     }
 
@@ -112,7 +112,6 @@ public void shutdown() throws Exception {
         }
         threadPool = null;
         networkService = null;
-        recycler = null;
         clusterSettings = null;
     }
 
@@ -176,7 +175,9 @@ public void dispatchBadRequest(RestChannel channel, ThreadContext threadContext,
                 dispatcher,
                 clusterSettings,
                 new SharedGroupFactory(settings),
-                Tracer.NOOP
+                Tracer.NOOP,
+                TLSConfig.noTLS(),
+                null
             )
         ) {
             transport.start();
@@ -225,7 +226,9 @@ public void testBindUnavailableAddress() {
                 new NullDispatcher(),
                 clusterSettings,
                 new SharedGroupFactory(Settings.EMPTY),
-                Tracer.NOOP
+                Tracer.NOOP,
+                TLSConfig.noTLS(),
+                null
             )
         ) {
             transport.start();
@@ -243,7 +246,9 @@ public void testBindUnavailableAddress() {
                     new NullDispatcher(),
                     clusterSettings,
                     new SharedGroupFactory(settings),
-                    Tracer.NOOP
+                    Tracer.NOOP,
+                    TLSConfig.noTLS(),
+                    null
                 )
             ) {
                 BindHttpException bindHttpException = expectThrows(BindHttpException.class, otherTransport::start);
@@ -295,7 +300,9 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th
                 dispatcher,
                 clusterSettings,
                 new SharedGroupFactory(settings),
-                Tracer.NOOP
+                Tracer.NOOP,
+                TLSConfig.noTLS(),
+                null
             )
         ) {
             transport.start();
@@ -363,11 +370,13 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th
                 dispatcher,
                 clusterSettings,
                 new SharedGroupFactory(Settings.EMPTY),
-                Tracer.NOOP
+                Tracer.NOOP,
+                TLSConfig.noTLS(),
+                null
             ) {
                 @Override
                 public ChannelHandler configureServerChannelHandler() {
-                    return new HttpChannelHandler(this, handlingSettings) {
+                    return new HttpChannelHandler(this, handlingSettings, TLSConfig.noTLS(), null) {
                         @Override
                         protected void initChannel(Channel ch) throws Exception {
                             super.initChannel(ch);
@@ -460,7 +469,9 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th
                 dispatcher,
                 randomClusterSettings(),
                 new SharedGroupFactory(settings),
-                Tracer.NOOP
+                Tracer.NOOP,
+                TLSConfig.noTLS(),
+                null
             )
         ) {
             transport.start();
@@ -530,7 +541,9 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th
                 dispatcher,
                 randomClusterSettings(),
                 new SharedGroupFactory(settings),
-                Tracer.NOOP
+                Tracer.NOOP,
+                TLSConfig.noTLS(),
+                null
             )
         ) {
             transport.start();
@@ -560,6 +573,69 @@ protected void initChannel(SocketChannel ch) {
         }
     }
 
+    public void testHeadRequestToChunkedApi() throws InterruptedException {
+        final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {
+
+            @Override
+            public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) {
+                try {
+                    channel.sendResponse(
+                        new RestResponse(
+                            OK,
+                            ChunkedRestResponseBody.fromXContent(
+                                ignored -> Iterators.single(
+                                    (builder, params) -> { throw new AssertionError("should not be called for HEAD REQUEST"); }
+                                ),
+                                ToXContent.EMPTY_PARAMS,
+                                channel
+                            )
+                        )
+                    );
+                } catch (IOException e) {
+                    throw new AssertionError(e);
+                }
+            }
+
+            @Override
+            public void dispatchBadRequest(final RestChannel channel, final ThreadContext threadContext, final Throwable cause) {
+                throw new AssertionError();
+            }
+
+        };
+
+        final Settings settings = createSettings();
+        try (
+            Netty4HttpServerTransport transport = new Netty4HttpServerTransport(
+                settings,
+                networkService,
+                threadPool,
+                xContentRegistry(),
+                dispatcher,
+                clusterSettings,
+                new SharedGroupFactory(settings),
+                Tracer.NOOP,
+                TLSConfig.noTLS(),
+                null
+            )
+        ) {
+            transport.start();
+            final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
+
+            try (Netty4HttpClient client = new Netty4HttpClient()) {
+                final String url = "/some-head-endpoint";
+                final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.HEAD, url);
+
+                final FullHttpResponse response = client.send(remoteAddress.address(), request);
+                try {
+                    assertThat(response.status(), equalTo(HttpResponseStatus.OK));
+                    assertFalse(response.content().isReadable());
+                } finally {
+                    response.release();
+                }
+            }
+        }
+    }
+
     private Settings createSettings() {
         return createBuilderWithPort().build();
     }
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java
index 2229e46e522a..28db025015b6 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.transport.netty4;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
@@ -56,7 +56,7 @@ public void startThreadPool() {
         PageCacheRecycler recycler = new MockPageCacheRecycler(Settings.EMPTY);
         nettyTransport = new Netty4Transport(
             settings,
-            Version.CURRENT,
+            TransportVersion.CURRENT,
             threadPool,
             networkService,
             recycler,
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java
index 9648f19ac8c9..1d35a6ac0146 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.transport.netty4;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
@@ -110,7 +110,7 @@ private TcpTransport startTransport(Settings settings, ThreadPool threadPool) {
         PageCacheRecycler recycler = new MockPageCacheRecycler(Settings.EMPTY);
         TcpTransport transport = new Netty4Transport(
             settings,
-            Version.CURRENT,
+            TransportVersion.CURRENT,
             threadPool,
             new NetworkService(Collections.emptyList()),
             recycler,
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java
index 91a31f19f0e3..157e81e55af3 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.transport.netty4;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -24,10 +25,13 @@
 import org.elasticsearch.transport.AbstractSimpleTransportTestCase;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.ConnectionProfile;
+import org.elasticsearch.transport.RemoteClusterPortSettings;
 import org.elasticsearch.transport.TcpChannel;
 import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.TestProfiles;
 import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportRequestOptions;
+import org.elasticsearch.transport.TransportSettings;
 
 import java.io.IOException;
 import java.net.InetAddress;
@@ -38,6 +42,7 @@
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
@@ -45,7 +50,7 @@
 public class SimpleNetty4TransportTests extends AbstractSimpleTransportTestCase {
 
     @Override
-    protected Transport build(Settings settings, final Version version, ClusterSettings clusterSettings, boolean doHandshake) {
+    protected Transport build(Settings settings, TransportVersion version, ClusterSettings clusterSettings, boolean doHandshake) {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
         return new Netty4Transport(
             settings,
@@ -63,12 +68,12 @@ public void executeHandshake(
                 DiscoveryNode node,
                 TcpChannel channel,
                 ConnectionProfile profile,
-                ActionListener<Version> listener
+                ActionListener<TransportVersion> listener
             ) {
                 if (doHandshake) {
                     super.executeHandshake(node, channel, profile, listener);
                 } else {
-                    listener.onResponse(version.minimumCompatibilityVersion());
+                    listener.onResponse(version.calculateMinimumCompatVersion());
                 }
             }
         };
@@ -119,6 +124,50 @@ public void testDefaultKeepAliveSettings() throws IOException {
         }
     }
 
+    public void testTransportProfile() {
+        final String transportProfile = randomFrom(
+            TransportSettings.DEFAULT_PROFILE,
+            RemoteClusterPortSettings.REMOTE_CLUSTER_PROFILE,
+            randomAlphaOfLengthBetween(5, 12)
+        );
+        final ConnectionProfile connectionProfile = ConnectionProfile.resolveConnectionProfile(
+            new ConnectionProfile.Builder().setTransportProfile(transportProfile)
+                .addConnections(
+                    1,
+                    TransportRequestOptions.Type.BULK,
+                    TransportRequestOptions.Type.PING,
+                    TransportRequestOptions.Type.RECOVERY,
+                    TransportRequestOptions.Type.REG,
+                    TransportRequestOptions.Type.STATE
+                )
+                .build(),
+            TestProfiles.LIGHT_PROFILE
+        );
+
+        try (
+            MockTransportService serviceC = buildService("TS_C", Version.CURRENT, Settings.EMPTY);
+            MockTransportService serviceD = buildService("TS_D", Version.CURRENT, Settings.EMPTY)
+        ) {
+
+            try (Transport.Connection connection = openConnection(serviceC, serviceD.getLocalDiscoNode(), connectionProfile)) {
+                assertThat(connection, instanceOf(StubbableTransport.WrappedConnection.class));
+                Transport.Connection conn = ((StubbableTransport.WrappedConnection) connection).getConnection();
+                assertThat(conn, instanceOf(TcpTransport.NodeChannels.class));
+                TcpTransport.NodeChannels nodeChannels = (TcpTransport.NodeChannels) conn;
+                for (TcpChannel channel : nodeChannels.getChannels()) {
+                    assertFalse(channel.isServerChannel());
+                    assertThat(channel.getProfile(), equalTo(transportProfile));
+                }
+
+                assertThat(serviceD.getOriginalTransport(), instanceOf(TcpTransport.class));
+                for (TcpChannel channel : getAcceptedChannels((TcpTransport) serviceD.getOriginalTransport())) {
+                    assertTrue(channel.isServerChannel());
+                    assertThat(channel.getProfile(), equalTo(TransportSettings.DEFAULT_PROFILE));
+                }
+            }
+        }
+    }
+
     private void checkDefaultKeepAliveOptions(TcpChannel channel) throws IOException {
         assertThat(channel, instanceOf(Netty4TcpChannel.class));
         Netty4TcpChannel nettyChannel = (Netty4TcpChannel) channel;
diff --git a/plugins/analysis-icu/build.gradle b/plugins/analysis-icu/build.gradle
index b10ca2b6dd49..1c7db6d040be 100644
--- a/plugins/analysis-icu/build.gradle
+++ b/plugins/analysis-icu/build.gradle
@@ -7,8 +7,8 @@ import org.elasticsearch.gradle.internal.info.BuildParams
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
diff --git a/plugins/analysis-kuromoji/build.gradle b/plugins/analysis-kuromoji/build.gradle
index 0a9cc629e7a4..a91e34018179 100644
--- a/plugins/analysis-kuromoji/build.gradle
+++ b/plugins/analysis-kuromoji/build.gradle
@@ -5,8 +5,8 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
   description 'The Japanese (kuromoji) Analysis plugin integrates Lucene kuromoji analysis module into elasticsearch.'
diff --git a/plugins/analysis-nori/build.gradle b/plugins/analysis-nori/build.gradle
index 4c545d4f72d4..51e93bf6cc2c 100644
--- a/plugins/analysis-nori/build.gradle
+++ b/plugins/analysis-nori/build.gradle
@@ -5,8 +5,8 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
   description 'The Korean (nori) Analysis plugin integrates Lucene nori analysis module into elasticsearch.'
diff --git a/plugins/analysis-nori/src/test/java/org/elasticsearch/plugin/analysis/nori/NoriAnalysisTests.java b/plugins/analysis-nori/src/test/java/org/elasticsearch/plugin/analysis/nori/NoriAnalysisTests.java
index 994ebffb9af6..c9fbef0d20ad 100644
--- a/plugins/analysis-nori/src/test/java/org/elasticsearch/plugin/analysis/nori/NoriAnalysisTests.java
+++ b/plugins/analysis-nori/src/test/java/org/elasticsearch/plugin/analysis/nori/NoriAnalysisTests.java
@@ -75,7 +75,7 @@ public void testNoriAnalyzer() throws Exception {
     public void testNoriAnalyzerUserDict() throws Exception {
         Settings settings = Settings.builder()
             .put("index.analysis.analyzer.my_analyzer.type", "nori")
-            .putList("index.analysis.analyzer.my_analyzer.user_dictionary_rules", "c++", "C샤프", "세종", "세종시 세종 시")
+            .putList("index.analysis.analyzer.my_analyzer.user_dictionary_rules", "c++", "C쁠쁠", "세종", "세종시 세종 시")
             .build();
         TestAnalysis analysis = createTestAnalysis(settings);
         Analyzer analyzer = analysis.indexAnalyzers.get("my_analyzer");
@@ -108,7 +108,7 @@ public void testNoriAnalyzerInvalidUserDictOption() throws Exception {
         Settings settings = Settings.builder()
             .put("index.analysis.analyzer.my_analyzer.type", "nori")
             .put("index.analysis.analyzer.my_analyzer.user_dictionary", "user_dict.txt")
-            .putList("index.analysis.analyzer.my_analyzer.user_dictionary_rules", "c++", "C샤프", "세종", "세종시 세종 시")
+            .putList("index.analysis.analyzer.my_analyzer.user_dictionary_rules", "c++", "C쁠쁠", "세종", "세종시 세종 시")
             .build();
         IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> createTestAnalysis(settings));
         assertThat(
diff --git a/plugins/analysis-nori/src/test/resources/org/elasticsearch/plugin/analysis/nori/user_dict.txt b/plugins/analysis-nori/src/test/resources/org/elasticsearch/plugin/analysis/nori/user_dict.txt
index 63c1c3a1e224..b70fb43a0092 100644
--- a/plugins/analysis-nori/src/test/resources/org/elasticsearch/plugin/analysis/nori/user_dict.txt
+++ b/plugins/analysis-nori/src/test/resources/org/elasticsearch/plugin/analysis/nori/user_dict.txt
@@ -1,5 +1,5 @@
 # Additional nouns
 c++
-C샤프
+C쁠쁠
 세종
-세종시 세종 시
\ No newline at end of file
+세종시 세종 시
diff --git a/plugins/analysis-phonetic/build.gradle b/plugins/analysis-phonetic/build.gradle
index 96149295b496..7646c0ee874d 100644
--- a/plugins/analysis-phonetic/build.gradle
+++ b/plugins/analysis-phonetic/build.gradle
@@ -5,8 +5,8 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
   description 'The Phonetic Analysis plugin integrates phonetic token filter analysis with elasticsearch.'
diff --git a/plugins/analysis-smartcn/build.gradle b/plugins/analysis-smartcn/build.gradle
index dd577a550e8f..c75c5c304048 100644
--- a/plugins/analysis-smartcn/build.gradle
+++ b/plugins/analysis-smartcn/build.gradle
@@ -5,8 +5,8 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
   description 'Smart Chinese Analysis plugin integrates Lucene Smart Chinese analysis module into elasticsearch.'
diff --git a/plugins/analysis-stempel/build.gradle b/plugins/analysis-stempel/build.gradle
index 363d923492e0..ca6d91df5cd1 100644
--- a/plugins/analysis-stempel/build.gradle
+++ b/plugins/analysis-stempel/build.gradle
@@ -5,8 +5,8 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
   description 'The Stempel (Polish) Analysis plugin integrates Lucene stempel (polish) analysis module into elasticsearch.'
diff --git a/plugins/analysis-ukrainian/build.gradle b/plugins/analysis-ukrainian/build.gradle
index 67aa47db8bbd..709b8831628b 100644
--- a/plugins/analysis-ukrainian/build.gradle
+++ b/plugins/analysis-ukrainian/build.gradle
@@ -5,8 +5,8 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
   description 'The Ukrainian Analysis plugin integrates the Lucene UkrainianMorfologikAnalyzer into elasticsearch.'
diff --git a/plugins/discovery-azure-classic/build.gradle b/plugins/discovery-azure-classic/build.gradle
index de47739d382b..3db7c21309b5 100644
--- a/plugins/discovery-azure-classic/build.gradle
+++ b/plugins/discovery-azure-classic/build.gradle
@@ -8,7 +8,7 @@ import org.elasticsearch.gradle.internal.info.BuildParams
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle
index 1e48fa537e43..b21f6224c9fc 100644
--- a/plugins/discovery-ec2/build.gradle
+++ b/plugins/discovery-ec2/build.gradle
@@ -7,7 +7,7 @@ import org.elasticsearch.gradle.internal.info.BuildParams
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
diff --git a/plugins/discovery-ec2/qa/amazon-ec2/build.gradle b/plugins/discovery-ec2/qa/amazon-ec2/build.gradle
index e8afba63970c..c7c4ecba40fe 100644
--- a/plugins/discovery-ec2/qa/amazon-ec2/build.gradle
+++ b/plugins/discovery-ec2/qa/amazon-ec2/build.gradle
@@ -10,11 +10,11 @@ import org.apache.tools.ant.filters.ReplaceTokens
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.internal.test.AntFixture
 import org.elasticsearch.gradle.internal.test.RestIntegTestTask
-import org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin
+import org.elasticsearch.gradle.internal.test.rest.LegacyYamlRestTestPlugin
 
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 
 dependencies {
   yamlRestTestImplementation project(':plugins:discovery-ec2')
@@ -62,7 +62,7 @@ tasks.named("yamlRestTest").configure { enabled = false }
   def yamlRestTestTask = tasks.register("yamlRestTest${action}", RestIntegTestTask) {
     dependsOn fixture
     SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-    SourceSet yamlRestTestSourceSet = sourceSets.getByName(InternalYamlRestTestPlugin.SOURCE_SET_NAME)
+    SourceSet yamlRestTestSourceSet = sourceSets.getByName(LegacyYamlRestTestPlugin.SOURCE_SET_NAME)
     testClassesDirs = yamlRestTestSourceSet.getOutput().getClassesDirs()
     classpath = yamlRestTestSourceSet.getRuntimeClasspath()
   }
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java
index 806ae2ff9d82..b0a8242a7ca5 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java
@@ -14,7 +14,7 @@
 import org.apache.http.HttpStatus;
 import org.apache.http.NameValuePair;
 import org.apache.http.client.utils.URLEncodedUtils;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
@@ -50,7 +50,7 @@ protected MockTransportService createTransportService() {
             Settings.EMPTY,
             new Netty4Transport(
                 Settings.EMPTY,
-                Version.CURRENT,
+                TransportVersion.CURRENT,
                 threadPool,
                 networkService,
                 PageCacheRecycler.NON_RECYCLING_INSTANCE,
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
index a06a76779516..87f9689f5a4d 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
@@ -17,7 +17,7 @@
 import org.apache.http.HttpStatus;
 import org.apache.http.NameValuePair;
 import org.apache.http.client.utils.URLEncodedUtils;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
@@ -63,7 +63,7 @@ public class Ec2DiscoveryTests extends AbstractEC2MockAPITestCase {
     protected MockTransportService createTransportService() {
         final Transport transport = new Netty4Transport(
             Settings.EMPTY,
-            Version.CURRENT,
+            TransportVersion.CURRENT,
             threadPool,
             new NetworkService(Collections.emptyList()),
             PageCacheRecycler.NON_RECYCLING_INSTANCE,
diff --git a/plugins/discovery-gce/build.gradle b/plugins/discovery-gce/build.gradle
index cfd1609078de..75a8095ef412 100644
--- a/plugins/discovery-gce/build.gradle
+++ b/plugins/discovery-gce/build.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
@@ -9,8 +9,7 @@ esplugin {
 versions << [
   'google'              : '1.41.1',
   'google_api_client'   : '1.33.1',
-  'api_services_compute': 'v1-rev20220322-1.32.1',
-  'google_oauth_client' : '1.33.0',
+  'api_services_compute': 'v1-rev20220322-1.32.1'
 ]
 
 dependencies {
diff --git a/plugins/discovery-gce/qa/gce/build.gradle b/plugins/discovery-gce/qa/gce/build.gradle
index c2cd26c2e928..7eb6e85903ba 100644
--- a/plugins/discovery-gce/qa/gce/build.gradle
+++ b/plugins/discovery-gce/qa/gce/build.gradle
@@ -13,7 +13,7 @@ import org.elasticsearch.gradle.internal.test.AntFixture
 
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 
 final int gceNumberOfNodes = 3
 
diff --git a/plugins/examples/build.gradle b/plugins/examples/build.gradle
index 263d73b128f0..e983292f7ab3 100644
--- a/plugins/examples/build.gradle
+++ b/plugins/examples/build.gradle
@@ -29,6 +29,14 @@ subprojects {
         }
       }
     }
+    if (gradle.includedBuilds.isEmpty()) {
+      maven {
+        url = "https://artifacts-snapshot.elastic.co/elasticsearch/${elasticsearchVersion}/maven"
+        mavenContent {
+          includeModule 'org.elasticsearch', 'elasticsearch'
+        }
+      }
+    }
 
     // Same for Lucene, add the snapshot repo based on the currently used Lucene version
     def luceneVersion = VersionProperties.getLucene()
diff --git a/plugins/examples/custom-significance-heuristic/src/test/java/org/elasticsearch/example/customsigheuristic/SimpleHeuristicWireTests.java b/plugins/examples/custom-significance-heuristic/src/test/java/org/elasticsearch/example/customsigheuristic/SimpleHeuristicWireTests.java
index 1c0d9601814d..d43374b3b5c6 100644
--- a/plugins/examples/custom-significance-heuristic/src/test/java/org/elasticsearch/example/customsigheuristic/SimpleHeuristicWireTests.java
+++ b/plugins/examples/custom-significance-heuristic/src/test/java/org/elasticsearch/example/customsigheuristic/SimpleHeuristicWireTests.java
@@ -43,4 +43,9 @@ protected Reader<SimpleHeuristic> instanceReader() {
     protected SimpleHeuristic createTestInstance() {
         return new SimpleHeuristic();
     }
+
+    @Override
+    protected SimpleHeuristic mutateInstance(SimpleHeuristic instance) throws IOException {
+        return null;
+    }
 }
diff --git a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionBuilder.java b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionBuilder.java
index fa4056515a5b..827f3e837bde 100644
--- a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionBuilder.java
+++ b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionBuilder.java
@@ -8,17 +8,17 @@
 
 package org.elasticsearch.example.customsuggester;
 
-import org.elasticsearch.Version;
-import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.BytesRefs;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.suggest.SuggestionBuilder;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -131,7 +131,7 @@ public SuggestionSearchContext.SuggestionContext build(SearchExecutionContext co
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/plugins/examples/rescore/src/main/java/org/elasticsearch/example/rescore/ExampleRescoreBuilder.java b/plugins/examples/rescore/src/main/java/org/elasticsearch/example/rescore/ExampleRescoreBuilder.java
index f599ddb63def..8691ab4e4f6e 100644
--- a/plugins/examples/rescore/src/main/java/org/elasticsearch/example/rescore/ExampleRescoreBuilder.java
+++ b/plugins/examples/rescore/src/main/java/org/elasticsearch/example/rescore/ExampleRescoreBuilder.java
@@ -13,24 +13,24 @@
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.LeafFieldData;
 import org.elasticsearch.index.fielddata.LeafNumericFieldData;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.SearchExecutionContext;
-import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.search.rescore.RescoreContext;
 import org.elasticsearch.search.rescore.Rescorer;
 import org.elasticsearch.search.rescore.RescorerBuilder;
-import org.elasticsearch.Version;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -217,7 +217,7 @@ public Explanation explain(int topLevelDocId, IndexSearcher searcher, RescoreCon
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java b/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java
index 998fe8894ee3..1eeb32ed1346 100644
--- a/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java
+++ b/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java
@@ -31,6 +31,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.function.Supplier;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -102,12 +103,30 @@ public void authorizeIndexAction(RequestInfo requestInfo, AuthorizationInfo auth
     }
 
     @Override
-    public void loadAuthorizedIndices(RequestInfo requestInfo, AuthorizationInfo authorizationInfo,
-                                      Map<String, IndexAbstraction> indicesLookup, ActionListener<Set<String>> listener) {
+    public void loadAuthorizedIndices(
+        RequestInfo requestInfo,
+        AuthorizationInfo authorizationInfo,
+        Map<String, IndexAbstraction> indicesLookup,
+        ActionListener<AuthorizationEngine.AuthorizedIndices> listener
+    ) {
         if (isSuperuser(requestInfo.getAuthentication().getEffectiveSubject().getUser())) {
-            listener.onResponse(indicesLookup.keySet());
+            listener.onResponse(new AuthorizedIndices() {
+                public Supplier<Set<String>> all() {
+                    return () -> indicesLookup.keySet();
+                }
+                public boolean check(String name) {
+                    return indicesLookup.containsKey(name);
+                }
+            });
         } else {
-            listener.onResponse(Collections.emptySet());
+            listener.onResponse(new AuthorizedIndices() {
+                public Supplier<Set<String>> all() {
+                    return () -> Set.of();
+                }
+                public boolean check(String name) {
+                    return false;
+                }
+            });
         }
     }
 
diff --git a/plugins/examples/stable-analysis/build.gradle b/plugins/examples/stable-analysis/build.gradle
index 69e44c46be9f..7686be80a2a1 100644
--- a/plugins/examples/stable-analysis/build.gradle
+++ b/plugins/examples/stable-analysis/build.gradle
@@ -10,18 +10,19 @@ esplugin {
 dependencies {
 
   //TODO transitive dependency off and plugin-api dependency?
-  compileOnly "org.elasticsearch:elasticsearch-plugin-api:${elasticsearchVersion}"
-  compileOnly "org.elasticsearch:elasticsearch-plugin-analysis-api:${elasticsearchVersion}"
+  compileOnly "org.elasticsearch.plugin:elasticsearch-plugin-api:${elasticsearchVersion}"
+  compileOnly "org.elasticsearch.plugin:elasticsearch-plugin-analysis-api:${elasticsearchVersion}"
   compileOnly "org.apache.lucene:lucene-analysis-common:${luceneVersion}"
 
   //TODO for testing this also have to be declared
-  testImplementation "org.elasticsearch:elasticsearch-plugin-api:${elasticsearchVersion}"
-  testImplementation "org.elasticsearch:elasticsearch-plugin-analysis-api:${elasticsearchVersion}"
+  testImplementation "org.elasticsearch.plugin:elasticsearch-plugin-api:${elasticsearchVersion}"
+  testImplementation "org.elasticsearch.plugin:elasticsearch-plugin-analysis-api:${elasticsearchVersion}"
   testImplementation "org.apache.lucene:lucene-analysis-common:${luceneVersion}"
 
   testImplementation ('junit:junit:4.13.2'){
     exclude group: 'org.hamcrest'
   }
+  testImplementation 'org.mockito:mockito-core:4.4.0'
   testImplementation 'org.hamcrest:hamcrest:2.2'
 
 }
diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/CharacterSkippingTokenizerFactory.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/CharacterSkippingTokenizerFactory.java
new file mode 100644
index 000000000000..570dc55c2538
--- /dev/null
+++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/CharacterSkippingTokenizerFactory.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.example.analysis;
+
+import org.apache.lucene.analysis.Tokenizer;
+import org.elasticsearch.example.analysis.lucene.CharSkippingTokenizer;
+import org.elasticsearch.plugin.analysis.TokenizerFactory;
+import org.elasticsearch.plugin.Inject;
+import org.elasticsearch.plugin.NamedComponent;
+
+import java.util.List;
+
+@NamedComponent("example_tokenizer_factory")
+public class CharacterSkippingTokenizerFactory implements TokenizerFactory {
+    private final List<String> tokenizerListOfChars;
+
+    @Inject
+    public CharacterSkippingTokenizerFactory(ExampleAnalysisSettings settings) {
+        this.tokenizerListOfChars = settings.singleCharsToSkipInTokenizer();
+    }
+
+    @Override
+    public Tokenizer create() {
+        return new CharSkippingTokenizer(tokenizerListOfChars);
+    }
+}
+
diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/CustomAnalyzerFactory.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/CustomAnalyzerFactory.java
new file mode 100644
index 000000000000..b546a1da8c6f
--- /dev/null
+++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/CustomAnalyzerFactory.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.example.analysis;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.elasticsearch.example.analysis.lucene.CharSkippingTokenizer;
+import org.elasticsearch.example.analysis.lucene.ReplaceCharToNumber;
+import org.elasticsearch.example.analysis.lucene.SkipStartingWithDigitTokenFilter;
+import org.elasticsearch.plugin.analysis.AnalyzerFactory;
+import org.elasticsearch.plugin.NamedComponent;
+import org.elasticsearch.plugin.Inject;
+
+import java.util.List;
+
+@NamedComponent( "example_analyzer_factory")
+public class CustomAnalyzerFactory implements AnalyzerFactory {
+    private final ExampleAnalysisSettings settings;
+
+    @Inject
+    public CustomAnalyzerFactory(ExampleAnalysisSettings settings) {
+        this.settings = settings;
+    }
+
+    @Override
+    public Analyzer create() {
+        return new CustomAnalyzer(settings);
+    }
+
+    static class CustomAnalyzer extends Analyzer {
+
+        private final ExampleAnalysisSettings settings;
+
+        public CustomAnalyzer(ExampleAnalysisSettings settings) {
+            this.settings = settings;
+        }
+
+        @Override
+        protected TokenStreamComponents createComponents(String fieldName) {
+            var tokenizerListOfChars = settings.singleCharsToSkipInTokenizer().isEmpty() ? List.of("_") : settings.singleCharsToSkipInTokenizer();
+            var tokenizer = new CharSkippingTokenizer(tokenizerListOfChars);
+
+            long tokenFilterNumber = settings.analyzerUseTokenListOfChars() ? settings.digitToSkipInTokenFilter() : -1;
+            var tokenFilter = new SkipStartingWithDigitTokenFilter(tokenizer, tokenFilterNumber);
+            return new TokenStreamComponents(
+                r -> tokenizer.setReader(new ReplaceCharToNumber(r, settings.oldCharToReplaceInCharFilter(), settings.newNumberToReplaceInCharFilter())),
+                tokenFilter
+            );
+        }
+    }
+}
+
diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleAnalysisSettings.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleAnalysisSettings.java
new file mode 100644
index 000000000000..024b28e468da
--- /dev/null
+++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleAnalysisSettings.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.example.analysis;
+
+import org.elasticsearch.plugin.settings.AnalysisSettings;
+import org.elasticsearch.plugin.settings.BooleanSetting;
+import org.elasticsearch.plugin.settings.IntSetting;
+import org.elasticsearch.plugin.settings.LongSetting;
+import org.elasticsearch.plugin.settings.StringSetting;
+import org.elasticsearch.plugin.settings.ListSetting;
+
+import java.util.List;
+
+/**
+ * This is a settings interface that will be injected into a plugins' constructors
+ * annotated with @Inject.
+ * The settings interface has to be annotated with @AnalysisSettings so that Elasticsearch can generate
+ * its dynamic implementation.
+ * Methods on the interface should be noarg and have a returned type.
+ * For types supported see the plugin-api/org.elasticsearch.plugin.api.settings classes
+ * @see CustomAnalyzerFactory an example injection of this interface
+ */
+@AnalysisSettings
+public interface ExampleAnalysisSettings {
+    /*
+     * This method presents the use of String typed setting.
+     * see the ReplaceCharWithNumberCharFilterFactory
+     */
+    @StringSetting(path = "old_char", defaultValue = " ")
+    String oldCharToReplaceInCharFilter();
+
+    /*
+     * This method presents the use of int typed setting.
+     * see the ReplaceCharWithNumberCharFilterFactory
+     */
+    @IntSetting(path = "new_number", defaultValue = 0)
+    int newNumberToReplaceInCharFilter();
+
+    /*
+     * This method presents the use of long typed setting.
+     * see the ExampleTokenFilterFactory
+     */
+    @LongSetting(path = "token_filter_number", defaultValue = 0L)
+    long digitToSkipInTokenFilter();
+
+    /*
+     * This method presents the use of boolean typed setting.
+     * see the ExampleAnalyzerFactory
+     */
+    @BooleanSetting(path = "analyzerUseTokenListOfChars", defaultValue = true)
+    boolean analyzerUseTokenListOfChars();
+    /*
+     * This method presents the use of boolean typed setting.
+     * see the ExampleTokenizerFactory
+     */
+    @ListSetting(path = "tokenizer_list_of_chars")
+    List<String> singleCharsToSkipInTokenizer();
+}
diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleAnalyzerFactory.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleAnalyzerFactory.java
deleted file mode 100644
index 053b8fffb451..000000000000
--- a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleAnalyzerFactory.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.example.analysis;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.elasticsearch.example.analysis.lucene.ReplaceHash;
-import org.elasticsearch.example.analysis.lucene.Skip1TokenFilter;
-import org.elasticsearch.example.analysis.lucene.UnderscoreTokenizer;
-import org.elasticsearch.plugin.api.NamedComponent;
-
-@NamedComponent( "example_analyzer_factory")
-public class ExampleAnalyzerFactory implements org.elasticsearch.plugin.analysis.api.AnalyzerFactory {
-
-    @Override
-    //TODO guide lucene
-    public Analyzer create() {
-        return new CustomAnalyzer();
-    }
-
-    static class CustomAnalyzer extends Analyzer {
-
-        @Override
-        protected TokenStreamComponents createComponents(String fieldName) {
-            var tokenizer = new UnderscoreTokenizer();
-            var tokenFilter = new Skip1TokenFilter(tokenizer);
-            return new TokenStreamComponents(r -> tokenizer.setReader(new ReplaceHash(r)), tokenFilter);
-        }
-    }
-}
-
diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleCharFilterFactory.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleCharFilterFactory.java
deleted file mode 100644
index e918aa77b6cc..000000000000
--- a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleCharFilterFactory.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.example.analysis;
-
-import org.apache.lucene.util.SuppressForbidden;
-import org.elasticsearch.example.analysis.lucene.ReplaceHash;
-import org.elasticsearch.plugin.analysis.api.CharFilterFactory;
-import org.elasticsearch.plugin.api.NamedComponent;
-
-import java.io.Reader;
-
-@NamedComponent( "example_char_filter")
-public class ExampleCharFilterFactory implements CharFilterFactory {
-    @Override
-    public Reader create(Reader reader) {
-        return new ReplaceHash(reader);
-    }
-}
-
-
diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleTokenFilterFactory.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleTokenFilterFactory.java
deleted file mode 100644
index 4f8e20455c36..000000000000
--- a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleTokenFilterFactory.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.example.analysis;
-
-import org.apache.lucene.analysis.TokenStream;
-import org.elasticsearch.example.analysis.lucene.AppendTokenFilter;
-import org.elasticsearch.example.analysis.lucene.Skip1TokenFilter;
-import org.elasticsearch.plugin.analysis.api.AnalysisMode;
-import org.elasticsearch.plugin.api.NamedComponent;
-
-@NamedComponent( "example_token_filter_factory")
-public class ExampleTokenFilterFactory implements org.elasticsearch.plugin.analysis.api.TokenFilterFactory {
-    @Override
-    public TokenStream create(TokenStream tokenStream) {
-        return new Skip1TokenFilter(tokenStream);
-    }
-
-    @Override
-    public TokenStream normalize(TokenStream tokenStream) {
-        return new AppendTokenFilter(tokenStream, "1");
-    }
-
-    @Override
-    public AnalysisMode getAnalysisMode() {
-        return org.elasticsearch.plugin.analysis.api.TokenFilterFactory.super.getAnalysisMode();
-    }
-
-}
-
diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleTokenizerFactory.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleTokenizerFactory.java
deleted file mode 100644
index dbd55fd9dfe5..000000000000
--- a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ExampleTokenizerFactory.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.example.analysis;
-
-import org.apache.lucene.analysis.Tokenizer;
-import org.elasticsearch.example.analysis.lucene.UnderscoreTokenizer;
-import org.elasticsearch.plugin.analysis.api.TokenizerFactory;
-import org.elasticsearch.plugin.api.NamedComponent;
-
-@NamedComponent( "example_tokenizer_factory")
-public class ExampleTokenizerFactory implements TokenizerFactory {
-    @Override
-    public Tokenizer create() {
-        return new UnderscoreTokenizer();
-    }
-
-}
-
diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ReplacingCharFilterFactory.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ReplacingCharFilterFactory.java
new file mode 100644
index 000000000000..91b509353b6d
--- /dev/null
+++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/ReplacingCharFilterFactory.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.example.analysis;
+
+import org.elasticsearch.example.analysis.lucene.ReplaceCharToNumber;
+import org.elasticsearch.plugin.NamedComponent;
+import org.elasticsearch.plugin.Inject;
+import org.elasticsearch.plugin.analysis.CharFilterFactory;
+
+import java.io.Reader;
+
+@NamedComponent( "example_char_filter")
+public class ReplacingCharFilterFactory implements CharFilterFactory {
+    private final String oldChar;
+    private final int newNumber;
+
+    @Inject
+    public ReplacingCharFilterFactory(ExampleAnalysisSettings settings) {
+        oldChar = settings.oldCharToReplaceInCharFilter();
+        newNumber = settings.newNumberToReplaceInCharFilter();
+    }
+
+    @Override
+    public Reader create(Reader reader) {
+        return new ReplaceCharToNumber(reader, oldChar, newNumber);
+    }
+}
+
+
diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/SkippingTokenFilterFactory.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/SkippingTokenFilterFactory.java
new file mode 100644
index 000000000000..75de12b5f6ca
--- /dev/null
+++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/SkippingTokenFilterFactory.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.example.analysis;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.elasticsearch.example.analysis.lucene.SkipStartingWithDigitTokenFilter;
+import org.elasticsearch.plugin.analysis.AnalysisMode;
+import org.elasticsearch.plugin.analysis.TokenFilterFactory;
+import org.elasticsearch.plugin.NamedComponent;
+import org.elasticsearch.plugin.Inject;
+
+@NamedComponent( "example_token_filter_factory")
+public class SkippingTokenFilterFactory implements TokenFilterFactory {
+    private final long tokenFilterNumber;
+
+    @Inject
+    public SkippingTokenFilterFactory(ExampleAnalysisSettings settings) {
+        this.tokenFilterNumber = settings.digitToSkipInTokenFilter();
+    }
+
+    @Override
+    public TokenStream create(TokenStream tokenStream) {
+        return new SkipStartingWithDigitTokenFilter(tokenStream, tokenFilterNumber);
+    }
+
+    @Override
+    public TokenStream normalize(TokenStream tokenStream) {
+        return new SkipStartingWithDigitTokenFilter(tokenStream, tokenFilterNumber);
+    }
+
+    @Override
+    public AnalysisMode getAnalysisMode() {
+        return TokenFilterFactory.super.getAnalysisMode();
+    }
+
+}
+
diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/CharSkippingTokenizer.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/CharSkippingTokenizer.java
new file mode 100644
index 000000000000..326ae870e5e9
--- /dev/null
+++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/CharSkippingTokenizer.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.example.analysis.lucene;
+
+import org.apache.lucene.analysis.util.CharTokenizer;
+
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+/*
+A tokenizer that skips configured tokens.
+configured tokens are passed as list of single char strings.
+ */
+public class CharSkippingTokenizer extends CharTokenizer {
+
+    private final Set<Integer> setOfChars;
+
+    public CharSkippingTokenizer(List<String> tokenizerListOfChars) {
+        this.setOfChars = tokenizerListOfChars.stream().map(s -> (int) s.charAt(0)).collect(Collectors.toSet());
+    }
+
+    @Override
+    protected boolean isTokenChar(int c) {
+        return setOfChars.contains(c) == false;
+    }
+}
diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/ReplaceCharToNumber.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/ReplaceCharToNumber.java
new file mode 100644
index 000000000000..262748a129ff
--- /dev/null
+++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/ReplaceCharToNumber.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.example.analysis.lucene;
+
+import org.apache.lucene.analysis.charfilter.MappingCharFilter;
+import org.apache.lucene.analysis.charfilter.NormalizeCharMap;
+
+import java.io.Reader;
+
+public class ReplaceCharToNumber extends MappingCharFilter {
+
+    public ReplaceCharToNumber(Reader in, String oldChar, int newNumber) {
+        super(charMap(oldChar, newNumber), in);
+    }
+
+    private static NormalizeCharMap charMap(String oldChar, int newNumber) {
+        NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
+        builder.add(oldChar, String.valueOf(newNumber));
+        return builder.build();
+    }
+}
diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/ReplaceHash.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/ReplaceHash.java
deleted file mode 100644
index 062ce694b76f..000000000000
--- a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/ReplaceHash.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.example.analysis.lucene;
-
-import org.apache.lucene.analysis.charfilter.MappingCharFilter;
-import org.apache.lucene.analysis.charfilter.NormalizeCharMap;
-
-import java.io.Reader;
-
-public class ReplaceHash extends MappingCharFilter {
-
-    public ReplaceHash(Reader in) {
-        super(charMap(), in);
-    }
-
-    private static NormalizeCharMap charMap() {
-        NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
-        builder.add("#", "3");
-        return builder.build();
-    }
-}
diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/Skip1TokenFilter.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/Skip1TokenFilter.java
deleted file mode 100644
index 7e84e580dfea..000000000000
--- a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/Skip1TokenFilter.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.example.analysis.lucene;
-
-import org.apache.lucene.analysis.FilteringTokenFilter;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-
-import java.io.IOException;
-
-public class Skip1TokenFilter extends FilteringTokenFilter {
-
-    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
-
-    public Skip1TokenFilter(TokenStream in) {
-        super(in);
-    }
-
-    @Override
-    protected boolean accept() throws IOException {
-        return termAtt.buffer()[0] != '1';
-    }
-}
diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/SkipStartingWithDigitTokenFilter.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/SkipStartingWithDigitTokenFilter.java
new file mode 100644
index 000000000000..0ddfa51af562
--- /dev/null
+++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/SkipStartingWithDigitTokenFilter.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.example.analysis.lucene;
+
+import org.apache.lucene.analysis.FilteringTokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+
+import java.io.IOException;
+
+public class SkipStartingWithDigitTokenFilter extends FilteringTokenFilter {
+
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private final long tokenFilterNumber;
+
+    public SkipStartingWithDigitTokenFilter(TokenStream in, long tokenFilterNumber) {
+        super(in);
+        this.tokenFilterNumber = tokenFilterNumber;
+    }
+
+    @Override
+    protected boolean accept() throws IOException {
+        return termAtt.buffer()[0] != (char) (tokenFilterNumber + '0');
+    }
+}
diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/UnderscoreTokenizer.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/UnderscoreTokenizer.java
deleted file mode 100644
index d3172cfb06aa..000000000000
--- a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/UnderscoreTokenizer.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.example.analysis.lucene;
-
-import org.apache.lucene.analysis.util.CharTokenizer;
-
-public class UnderscoreTokenizer extends CharTokenizer {
-
-    @Override
-    protected boolean isTokenChar(int c) {
-        return c != '_';
-    }
-}
diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/package-info.java b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/package-info.java
new file mode 100644
index 000000000000..5368b1f72d47
--- /dev/null
+++ b/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * Implementation notes to all components:
+ * <ul>
+ * <li>a @NamedComponent annotation with a name is required in order for the component to be found by Elasticsearch</li>
+ * <li>a constructor is annotated with @Inject and has a settings interface as an argument. See the javadoc for the</li>
+ * </ul>
+ * ExampleAnalysisSettings for more details:
+ * <ul>
+ * <li>a no/noarg constructor is also possible</li>
+ * <li>a methods from stable analysis api are to be implemented with Apache Lucene</li>
+ * </ul>
+ */
+package org.elasticsearch.example.analysis;
diff --git a/plugins/examples/stable-analysis/src/test/java/org/elasticsearch/example/analysis/ExampleCharFilterFactoryTests.java b/plugins/examples/stable-analysis/src/test/java/org/elasticsearch/example/analysis/ExampleCharFilterFactoryTests.java
deleted file mode 100644
index 4a81e2c66c6c..000000000000
--- a/plugins/examples/stable-analysis/src/test/java/org/elasticsearch/example/analysis/ExampleCharFilterFactoryTests.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.example.analysis;
-
-
-import org.elasticsearch.plugin.analysis.api.CharFilterFactory;
-import org.junit.Test;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.equalTo;
-
-public class ExampleCharFilterFactoryTests {
-    @Test
-    public void exampleCharFilterIsAnnotatedWithName() {
-        CharFilterFactory charFilterFactory = new ExampleCharFilterFactory();
-        assertThat(charFilterFactory.name(), equalTo("example_char_filter"));
-    }
-}
diff --git a/plugins/examples/stable-analysis/src/test/java/org/elasticsearch/example/analysis/ReplacingCharFilterFactoryTests.java b/plugins/examples/stable-analysis/src/test/java/org/elasticsearch/example/analysis/ReplacingCharFilterFactoryTests.java
new file mode 100644
index 000000000000..9859e9a59585
--- /dev/null
+++ b/plugins/examples/stable-analysis/src/test/java/org/elasticsearch/example/analysis/ReplacingCharFilterFactoryTests.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.example.analysis;
+
+
+import org.elasticsearch.plugin.analysis.CharFilterFactory;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+
+public class ReplacingCharFilterFactoryTests {
+    @Test
+    public void exampleCharFilterIsAnnotatedWithName() {
+        ExampleAnalysisSettings settings = Mockito.mock(ExampleAnalysisSettings.class);
+        CharFilterFactory charFilterFactory = new ReplacingCharFilterFactory(settings);
+        assertThat(charFilterFactory.name(), equalTo("example_char_filter"));
+    }
+}
diff --git a/plugins/examples/stable-analysis/src/yamlRestTest/resources/rest-api-spec/test/analysis/20_char_filter.yml b/plugins/examples/stable-analysis/src/yamlRestTest/resources/rest-api-spec/test/analysis/20_char_filter.yml
index ed9489b06abb..c88cf3ca7aae 100644
--- a/plugins/examples/stable-analysis/src/yamlRestTest/resources/rest-api-spec/test/analysis/20_char_filter.yml
+++ b/plugins/examples/stable-analysis/src/yamlRestTest/resources/rest-api-spec/test/analysis/20_char_filter.yml
@@ -1,8 +1,7 @@
 ## Smoke tests for char filters included in the analysis-common module
 
-#todo this can be expanded with parameters once settings support is added
 ---
-"Stable char_filter plugin. Replace # with 3":
+"Stable char_filter plugin. Replace char with number":
   - do:
       indices.analyze:
         body:
@@ -10,22 +9,65 @@
           tokenizer: standard
           char_filter:
             - type: example_char_filter
+              old_char: "#"
+              new_number: 3
   - length: { tokens: 1 }
   - match:  { tokens.0.token: "t3st" }
 
   - do:
       indices.analyze:
         body:
-          text: t#st
+          text: t st
           explain: true
           tokenizer: standard
           char_filter:
             - type: example_char_filter
   - match:  { detail.custom_analyzer: true }
   - length: { detail.charfilters.0.filtered_text: 1 }
-  - match:  { detail.charfilters.0.filtered_text.0: "t3st" }
+  - match:  { detail.charfilters.0.filtered_text.0: "t0st" }
   - length: { detail.tokenizer.tokens: 1 }
-  - match:  { detail.tokenizer.tokens.0.token: "t3st" }
+  - match:  { detail.tokenizer.tokens.0.token: "t0st" }
   - match:  { detail.tokenizer.tokens.0.start_offset: 0 }
   - match:  { detail.tokenizer.tokens.0.end_offset: 4 }
   - match:  { detail.tokenizer.tokens.0.position: 0 }
+
+---
+"Index and search with stable plugin char filter":
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            analysis:
+              char_filter:
+                my_char_filter:
+                  type: example_char_filter
+                  old_char: "#"
+                  new_number: 4
+              analyzer:
+                my_analyzer:
+                  tokenizer: standard
+                  char_filter: ["my_char_filter"
+                  ]
+          mappings:
+            properties:
+              text:
+                type: text
+                analyzer: my_analyzer
+  - do:
+      index:
+        index: test
+        id: "1"
+        body: { "text": "t#st" }
+  - do:
+      indices.refresh: { }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          query:
+            match:
+              text: t4st
+  - match: { hits.total: 1 }
diff --git a/plugins/examples/stable-analysis/src/yamlRestTest/resources/rest-api-spec/test/analysis/30_token_filter.yml b/plugins/examples/stable-analysis/src/yamlRestTest/resources/rest-api-spec/test/analysis/30_token_filter.yml
index 818649ff14ae..3d09633fbad0 100644
--- a/plugins/examples/stable-analysis/src/yamlRestTest/resources/rest-api-spec/test/analysis/30_token_filter.yml
+++ b/plugins/examples/stable-analysis/src/yamlRestTest/resources/rest-api-spec/test/analysis/30_token_filter.yml
@@ -10,9 +10,11 @@
           tokenizer: standard
           filter:
             - type: example_token_filter_factory
-  - length: { tokens: 2 }
-  - match:  { tokens.0.token: "2test" }
-  - match:  { tokens.1.token: "3test" }
+              token_filter_number: 2
+  - length: { tokens: 3 }
+  - match:  { tokens.0.token: "1test" }
+  - match:  { tokens.1.token: "1test" }
+  - match:  { tokens.2.token: "3test" }
 
   - do:
       indices.analyze:
@@ -22,7 +24,49 @@
           tokenizer: standard
           filter:
             - type: example_token_filter_factory
+              token_filter_number: 1
   - match:  { detail.custom_analyzer: true }
   - length: { detail.tokenfilters.0.tokens: 2 }
   - match:  { detail.tokenfilters.0.tokens.0.token: "2test" }
   - match:  { detail.tokenfilters.0.tokens.1.token: "3test" }
+
+
+---
+"Index and search with stable plugin token filter":
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            analysis:
+              filter:
+                my_token_filter:
+                  type: example_token_filter_factory
+                  token_filter_number: 1
+              analyzer:
+                my_analyzer:
+                  tokenizer: standard
+                  filter: ["my_token_filter"
+                  ]
+          mappings:
+            properties:
+              text:
+                type: text
+                analyzer: my_analyzer
+  - do:
+      index:
+        index: test
+        id: "1"
+        body: { "text": "1test 2test 1test 3test" }
+  - do:
+      indices.refresh: { }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          query:
+            match:
+              text: 2test
+  - match: { hits.total: 1 }
diff --git a/plugins/examples/stable-analysis/src/yamlRestTest/resources/rest-api-spec/test/analysis/40_tokenizer.yml b/plugins/examples/stable-analysis/src/yamlRestTest/resources/rest-api-spec/test/analysis/40_tokenizer.yml
index 0a76cbbae9af..7b15ccd3d4cd 100644
--- a/plugins/examples/stable-analysis/src/yamlRestTest/resources/rest-api-spec/test/analysis/40_tokenizer.yml
+++ b/plugins/examples/stable-analysis/src/yamlRestTest/resources/rest-api-spec/test/analysis/40_tokenizer.yml
@@ -6,8 +6,10 @@
   - do:
       indices.analyze:
         body:
-          text: x_y_z
-          tokenizer: example_tokenizer_factory
+          text: x y_z
+          tokenizer:
+            type: example_tokenizer_factory
+            tokenizer_list_of_chars: ["_", " "]
   - length: { tokens: 3 }
   - match:  { tokens.0.token: "x" }
   - match:  { tokens.1.token: "y" }
@@ -18,7 +20,9 @@
         body:
           text: x_y_z
           explain: true
-          tokenizer: example_tokenizer_factory
+          tokenizer:
+            type: example_tokenizer_factory
+            tokenizer_list_of_chars: ["_", " "]
   - match:  { detail.custom_analyzer: true }
   - length: { detail.tokenizer.tokens: 3 }
   - match:  { detail.tokenizer.tokens.0.token: "x" }
diff --git a/plugins/examples/stable-analysis/src/yamlRestTest/resources/rest-api-spec/test/analysis/50_analyzer.yml b/plugins/examples/stable-analysis/src/yamlRestTest/resources/rest-api-spec/test/analysis/50_analyzer.yml
index 8a536ffda14d..46ee6b8e15f1 100644
--- a/plugins/examples/stable-analysis/src/yamlRestTest/resources/rest-api-spec/test/analysis/50_analyzer.yml
+++ b/plugins/examples/stable-analysis/src/yamlRestTest/resources/rest-api-spec/test/analysis/50_analyzer.yml
@@ -2,23 +2,62 @@
 
 #todo this can be expanded with parameters once settings support is added
 ---
-"Stable analyzer provider plugin. It combines a underscore tokenizer, token filter which skips 1 and char filter which replaces # with 3":
+"Stable analyzer provider plugin. It combines a underscore tokenizer, token filter which skips 0 and char filter which replaces # with 3":
+#analyze request only allows analyzer field to be string (cannot be a NameOrDefinition)
+#tokenizes by _, replaces space with 0 and skips tokens staring with 0
   - do:
       indices.analyze:
         body:
-          text: 1x_y_#z
+          text: 0x_y_ z
           analyzer: example_analyzer_factory
-  - length: { tokens: 2 }
+  - length: { tokens: 1 }
   - match:  { tokens.0.token: "y" }
-  - match:  { tokens.1.token: "3z" }
 
   - do:
       indices.analyze:
         body:
-          text: 1x_y_#z
+          text: 0x_y_ z
           explain: true
           analyzer: example_analyzer_factory
   - match:  { detail.custom_analyzer: false } #?? why not custom?
-  - length: { detail.analyzer.tokens: 2 }
+  - length: { detail.analyzer.tokens: 1 }
   - match:  { detail.analyzer.tokens.0.token: "y" }
-  - match:  { detail.analyzer.tokens.1.token: "3z" }
+
+
+---
+"Index and search with stable plugin analyzer":
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            analysis:
+              analyzer:
+                my_analyzer:
+                  type: example_analyzer_factory
+                  old_char: "#"
+                  new_number: 3
+                  tokenizer_list_of_chars: [ "_", " " ]
+                  token_filter_number: 2
+          mappings:
+            properties:
+              text:
+                type: text
+                analyzer: my_analyzer
+  - do:
+      index:
+        index: test
+        id: "1"
+        body: { "text": "1x_y_#z" }
+  - do:
+      indices.refresh: { }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          query:
+            match:
+              text: 3z
+  - match: { hits.total: 1 }
diff --git a/plugins/mapper-annotated-text/build.gradle b/plugins/mapper-annotated-text/build.gradle
index 633cc3d3dcbb..275e7f5267e5 100644
--- a/plugins/mapper-annotated-text/build.gradle
+++ b/plugins/mapper-annotated-text/build.gradle
@@ -7,8 +7,8 @@ import org.elasticsearch.gradle.internal.info.BuildParams
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
   description 'The Mapper Annotated_text plugin adds support for text fields with markup used to inject annotation tokens into the index.'
diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedPassageFormatter.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedPassageFormatter.java
index 0ab917400c66..6a6e37883470 100644
--- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedPassageFormatter.java
+++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedPassageFormatter.java
@@ -19,6 +19,7 @@
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 /**
@@ -120,11 +121,27 @@ public String toString() {
     static MarkupPassage mergeAnnotations(AnnotationToken[] annotations, Passage passage) {
         MarkupPassage markupPassage = new MarkupPassage();
 
+        final Integer[] matches = new Integer[passage.getNumMatches()];
+        for (int i = 0; i < matches.length; ++i) {
+            matches[i] = i;
+        }
+        Arrays.sort(matches, (l, r) -> {
+            int lStart = passage.getMatchStarts()[l];
+            int lEnd = passage.getMatchEnds()[l];
+            int rStart = passage.getMatchStarts()[r];
+            int rEnd = passage.getMatchEnds()[r];
+            if (lStart == rStart) {
+                return rEnd - lEnd; // longest match first
+            } else {
+                return lStart - rStart;
+            }
+        });
+
         // Add search hits first - they take precedence over any other markup
-        for (int i = 0; i < passage.getNumMatches(); i++) {
-            int start = passage.getMatchStarts()[i];
-            int end = passage.getMatchEnds()[i];
-            String searchTerm = passage.getMatchTerms()[i].utf8ToString();
+        for (int matchId : matches) {
+            int start = passage.getMatchStarts()[matchId];
+            int end = passage.getMatchEnds()[matchId];
+            String searchTerm = passage.getMatchTerms()[matchId].utf8ToString();
             Markup markup = new Markup(start, end, SEARCH_HIT_TYPE + "=" + URLEncoder.encode(searchTerm, StandardCharsets.UTF_8));
             markupPassage.addUnlessOverlapping(markup);
         }
diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java
index 24a5e34a6eb2..f73bb7cbacbd 100644
--- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java
+++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java
@@ -18,6 +18,8 @@
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.PhraseQuery;
@@ -197,6 +199,16 @@ public void testAnnotatedTextOverlapsWithUnstructuredSearchTerms() throws Except
         assertHighlightOneDoc("text", markedUpInputs, query, Locale.ROOT, breakIterator, 0, expectedPassages);
     }
 
+    public void testAnnotatedTextHighlightQueryHasOverlappingTermAndAnnotation() throws Exception {
+        final String[] markedUpInputs = { "[Donald Trump](president) visited Singapore" };
+        String[] expectedPassages = { "[Donald Trump](_hit_term=president&president) visited Singapore" };
+        Query query = new BooleanQuery.Builder().add(new TermQuery(new Term("text", "donald")), BooleanClause.Occur.SHOULD)
+            .add(new TermQuery(new Term("text", "president")), BooleanClause.Occur.SHOULD)
+            .build();
+        BreakIterator breakIterator = new CustomSeparatorBreakIterator(MULTIVAL_SEP_CHAR);
+        assertHighlightOneDoc("text", markedUpInputs, query, Locale.ROOT, breakIterator, 0, expectedPassages);
+    }
+
     public void testAnnotatedTextMultiFieldWithBreakIterator() throws Exception {
         final String[] markedUpInputs = {
             "[Donald Trump](Donald+Trump) visited Singapore. Kim shook hands with Donald",
diff --git a/plugins/mapper-murmur3/build.gradle b/plugins/mapper-murmur3/build.gradle
index 6243871ef3bd..7e93e6d29539 100644
--- a/plugins/mapper-murmur3/build.gradle
+++ b/plugins/mapper-murmur3/build.gradle
@@ -7,8 +7,8 @@ import org.elasticsearch.gradle.internal.info.BuildParams
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
   description 'The Mapper Murmur3 plugin allows to compute hashes of a field\'s values at index-time and to store them in the index.'
diff --git a/plugins/mapper-size/src/yamlRestTest/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java b/plugins/mapper-size/src/yamlRestTest/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java
index a1bef57849f2..e9730418307e 100644
--- a/plugins/mapper-size/src/yamlRestTest/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java
+++ b/plugins/mapper-size/src/yamlRestTest/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java
@@ -11,11 +11,16 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.ClassRule;
 
 public class MapperSizeClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local().plugin("mapper-size").build();
+
     public MapperSizeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
@@ -24,4 +29,9 @@ public MapperSizeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate tes
     public static Iterable<Object[]> parameters() throws Exception {
         return createParameters();
     }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
 }
diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle
index 6cd9053f2d47..df8d3cdac1c4 100644
--- a/plugins/repository-hdfs/build.gradle
+++ b/plugins/repository-hdfs/build.gradle
@@ -15,8 +15,8 @@ import java.nio.file.Path
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 
 apply plugin: 'elasticsearch.test.fixtures'
-apply plugin: 'elasticsearch.internal-java-rest-test'
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 
 esplugin {
     description 'The HDFS repository plugin adds support for Hadoop Distributed File-System (HDFS) repositories.'
@@ -47,7 +47,7 @@ dependencies {
     }
     runtimeOnly "org.apache.hadoop:hadoop-client-runtime:${versions.hadoop}"
     implementation "org.apache.hadoop:hadoop-hdfs:${versions.hadoop}"
-    api 'com.google.protobuf:protobuf-java:3.4.0'
+    api "com.google.protobuf:protobuf-java:${versions.protobuf}"
     api "commons-logging:commons-logging:${versions.commonslogging}"
     api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
     api 'commons-cli:commons-cli:1.2'
@@ -304,8 +304,11 @@ tasks.named("thirdPartyAudit").configure {
     ignoreMissingClasses()
     ignoreViolations(
             // internal java api: sun.misc.Unsafe
+            'com.google.protobuf.MessageSchema',
             'com.google.protobuf.UnsafeUtil',
             'com.google.protobuf.UnsafeUtil$1',
+            'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
+            'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
             'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
             'com.google.protobuf.UnsafeUtil$MemoryAccessor',
             'org.apache.hadoop.hdfs.server.datanode.checker.AbstractFuture$UnsafeAtomicHelper',
diff --git a/plugins/repository-hdfs/src/javaRestTest/java/org/elasticsearch/repositories/hdfs/HaHdfsFailoverTestSuiteIT.java b/plugins/repository-hdfs/src/javaRestTest/java/org/elasticsearch/repositories/hdfs/HaHdfsFailoverTestSuiteIT.java
index 2d291cdce88f..0c0329e3cdb0 100644
--- a/plugins/repository-hdfs/src/javaRestTest/java/org/elasticsearch/repositories/hdfs/HaHdfsFailoverTestSuiteIT.java
+++ b/plugins/repository-hdfs/src/javaRestTest/java/org/elasticsearch/repositories/hdfs/HaHdfsFailoverTestSuiteIT.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.core.PathUtils;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.junit.Assert;
 
@@ -100,7 +101,7 @@ public void testHAFailoverWithRepository() throws Exception {
         // Create repository
         {
             Request request = new Request("PUT", "/_snapshot/hdfs_ha_repo_read");
-            request.setJsonEntity(formatted("""
+            request.setJsonEntity(Strings.format("""
                 {
                   "type": "hdfs",
                   "settings": {
diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java
index 624f327be1a0..f93d65098c1b 100644
--- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java
+++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java
@@ -18,7 +18,7 @@
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.core.Streams;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.CoreMatchers;
diff --git a/plugins/store-smb/build.gradle b/plugins/store-smb/build.gradle
index 91c7dcadf022..7147c7302733 100644
--- a/plugins/store-smb/build.gradle
+++ b/plugins/store-smb/build.gradle
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
diff --git a/qa/ccs-common-rest/build.gradle b/qa/ccs-common-rest/build.gradle
index 5e0d323f117b..c881e1728362 100644
--- a/qa/ccs-common-rest/build.gradle
+++ b/qa/ccs-common-rest/build.gradle
@@ -1,20 +1,4 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-
-import org.elasticsearch.gradle.Version
-import org.elasticsearch.gradle.internal.test.RestIntegTestTask
-import org.elasticsearch.gradle.testclusters.DefaultTestClustersTask
-
-
-apply plugin: 'elasticsearch.internal-testclusters'
-apply plugin: 'elasticsearch.standalone-rest-test'
-apply plugin: 'elasticsearch.rest-resources'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
 
 // This subproject copies a subset of the rest-api-spec rest tests and runs them in a slightly
 // modified way on two clusters connected via CCS. All operations except searches and a few other
@@ -35,42 +19,13 @@ restResources {
   }
 }
 
-def remoteCluster = testClusters.register("ccs-remote") {
-  numberOfNodes = 2
-  setting 'node.roles', '[data,ingest,master]'
-}
-
-def localCluster = testClusters.register("ccs-local") {
-  setting 'node.roles', '[data,ingest,master,remote_cluster_client]'
-  setting 'cluster.remote.connections_per_cluster', '1'
-  setting 'cluster.remote.remote_cluster.seeds',
-    { "\"${remoteCluster.get().getAllTransportPortURI().get(0)}\"" }
-}
-
-testClusters.configureEach {
-  setting 'xpack.security.enabled', 'false'
-  requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
+dependencies {
+  clusterModules project(':x-pack:plugin:async-search')
+  clusterModules project(':modules:mapper-extras')
+  clusterModules project(':modules:aggregations')
 }
 
-// the following task is needed to make sure the remote cluster is running before the local cluster
-// gets configured with the remotes cluster seed
-tasks.register('startRemoteCluster', DefaultTestClustersTask) {
-  useCluster remoteCluster
-  doLast {
-    clusters.each { c ->
-      print "Remote cluster transport uri for ccs configuration is: "
-      println c.getAllTransportPortURI().get(0)
-    }
-  }
-}
-
-tasks.register("ccs-remote", RestIntegTestTask) {
-  mustRunAfter("precommit")
-  dependsOn startRemoteCluster
-
-  useCluster remoteCluster
-  useCluster localCluster
-
+tasks.named("yamlRestTest") {
   systemProperty 'tests.rest.blacklist',
     [
       'search/150_rewrite_on_coordinator/Ensure that we fetch the document only once', // terms lookup query with index
@@ -86,13 +41,5 @@ tasks.register("ccs-remote", RestIntegTestTask) {
       'search.aggregation/70_adjacency_matrix/Terms lookup', // terms lookup by "index" doesn't seem to work correctly
       'async_search/20-with-poin-in-time/Async search with point in time' // [indices] cannot be used with point in time
     ].join(',')
-
-
-  doFirst {
-    println "Remote cluster endpoints are: ${-> remoteCluster.get().allHttpSocketURI.join(",")}"
-    println "Local cluster endpoints are: ${-> localCluster.get().allHttpSocketURI.join(",")}"
-    nonInputProperties.systemProperty('tests.rest.cluster', remoteCluster.map(c -> c.allHttpSocketURI.join(",")))
-    nonInputProperties.systemProperty('tests.rest.search_cluster', localCluster.map(c -> c.allHttpSocketURI.join(",")))
-  }
 }
 
diff --git a/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java
similarity index 89%
rename from qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java
rename to qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java
index 5a4df29b6488..9df00025d21b 100644
--- a/qa/ccs-common-rest/src/test/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java
+++ b/qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java
@@ -23,6 +23,9 @@
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.FeatureFlag;
+import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
 import org.elasticsearch.test.rest.ObjectPath;
 import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection;
 import org.elasticsearch.test.rest.yaml.section.DoSection;
@@ -32,6 +35,9 @@
 import org.elasticsearch.test.rest.yaml.section.MatchAssertion;
 import org.junit.AfterClass;
 import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -58,9 +64,34 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     private static RestClient adminSearchClient;
     private static List<HttpHost> clusterHosts;
     private static ClientYamlTestClient searchYamlTestClient;
-
     // the remote cluster is the one we write index operations etc... to
     private static final String REMOTE_CLUSTER_NAME = "remote_cluster";
+
+    private static LocalClusterConfigProvider commonClusterConfig = cluster -> cluster.module("x-pack-async-search")
+        .module("aggregations")
+        .module("mapper-extras")
+        .setting("xpack.security.enabled", "false")
+        .feature(FeatureFlag.TIME_SERIES_MODE);
+
+    private static ElasticsearchCluster remoteCluster = ElasticsearchCluster.local()
+        .name(REMOTE_CLUSTER_NAME)
+        .nodes(2)
+        .setting("node.roles", "[data,ingest,master]")
+        .apply(commonClusterConfig)
+        .build();
+
+    private static ElasticsearchCluster localCluster = ElasticsearchCluster.local()
+        .name("local_cluster")
+        .setting("node.roles", "[data,ingest,master,remote_cluster_client]")
+        .setting("cluster.remote.remote_cluster.seeds", () -> "\"" + remoteCluster.getTransportEndpoint(0) + "\"")
+        .setting("cluster.remote.connections_per_cluster", "1")
+        .apply(commonClusterConfig)
+        .build();
+
+    @ClassRule
+    // Use a RuleChain to ensure that remote cluster is started before local cluster
+    public static TestRule clusterRule = RuleChain.outerRule(remoteCluster).around(localCluster);
+
     // the CCS api calls that we run against the "search" cluster in this test setup
     private static final Set<String> CCS_APIS = Set.of(
         "search",
@@ -75,6 +106,11 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         "async_search.delete"
     );
 
+    @Override
+    protected String getTestRestCluster() {
+        return remoteCluster.getHttpAddresses();
+    }
+
     /**
      * initialize the search client and an additional administration client and check for an established connection
      */
@@ -84,9 +120,7 @@ public void initSearchClient() throws IOException {
             assert adminSearchClient == null;
             assert clusterHosts == null;
 
-            final String cluster = System.getProperty("tests.rest.search_cluster");
-            assertNotNull("[tests.rest.search_cluster] is not configured", cluster);
-            String[] stringUrls = cluster.split(",");
+            String[] stringUrls = localCluster.getHttpAddresses().split(",");
             List<HttpHost> hosts = new ArrayList<>(stringUrls.length);
             for (String stringUrl : stringUrls) {
                 int portSeparator = stringUrl.lastIndexOf(':');
diff --git a/qa/ccs-unavailable-clusters/build.gradle b/qa/ccs-unavailable-clusters/build.gradle
index 1b866385ecba..3b958bf9c8b9 100644
--- a/qa/ccs-unavailable-clusters/build.gradle
+++ b/qa/ccs-unavailable-clusters/build.gradle
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 testClusters.matching { it.name == "javaRestTest" }.configureEach {
   setting 'xpack.security.enabled', 'true'
diff --git a/qa/full-cluster-restart/build.gradle b/qa/full-cluster-restart/build.gradle
index a3af45b43363..b6f181809e0e 100644
--- a/qa/full-cluster-restart/build.gradle
+++ b/qa/full-cluster-restart/build.gradle
@@ -6,64 +6,20 @@
  * Side Public License, v 1.
  */
 
-
-import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
-apply plugin: 'elasticsearch.internal-testclusters'
-apply plugin: 'elasticsearch.standalone-rest-test'
-apply plugin: 'elasticsearch.internal-test-artifact'
+apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.internal-test-artifact-base'
 apply plugin: 'elasticsearch.bwc-test'
 
-BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName ->
-  def baseCluster = testClusters.register(baseName) {
-      if (bwcVersion.before(BuildParams.bwcVersions.minimumWireCompatibleVersion)) {
-        // When testing older versions we have to first upgrade to 7.last
-        versions = [bwcVersion.toString(), BuildParams.bwcVersions.minimumWireCompatibleVersion.toString(), project.version]
-      } else {
-        versions = [bwcVersion.toString(), project.version]
-      }
-      numberOfNodes = 2
-      // some tests rely on the translog not being flushed
-      setting 'indices.memory.shard_inactive_time', '60m'
-      setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
-      setting 'xpack.security.enabled', 'false'
-      requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
-  }
-
-  tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) {
-    useCluster baseCluster
-    mustRunAfter("precommit")
-    doFirst {
-      delete("${buildDir}/cluster/shared/repo/${baseName}")
-    }
-
-    systemProperty 'tests.is_old_cluster', 'true'
-  }
-
-  tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) {
-    useCluster baseCluster
-    dependsOn "${baseName}#oldClusterTest"
-    doFirst {
-      baseCluster.get().goToNextVersion()
-      if (bwcVersion.before(BuildParams.bwcVersions.minimumWireCompatibleVersion)) {
-        // When doing a full cluster restart of older versions we actually have to upgrade twice. First to 7.last, then to the current version.
-        baseCluster.get().goToNextVersion()
-      }
-    }
-    systemProperty 'tests.is_old_cluster', 'false'
-  }
-
-  String oldVersion = bwcVersion.toString().minus("-SNAPSHOT")
-  tasks.matching { it.name.startsWith(baseName) && it.name.endsWith("ClusterTest") }.configureEach {
-    it.systemProperty 'tests.old_cluster_version', oldVersion
-    it.systemProperty 'tests.path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
-    it.nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
-    it.nonInputProperties.systemProperty('tests.clustername', baseName)
-  }
+testArtifacts {
+  registerTestArtifactFromSourceSet(sourceSets.javaRestTest)
+}
 
-  tasks.register(bwcTaskName(bwcVersion)) {
-    dependsOn tasks.named("${baseName}#upgradedClusterTest")
+BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName ->
+  tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) {
+    usesBwcDistribution(bwcVersion)
+    systemProperty("tests.old_cluster_version", bwcVersion)
   }
 }
diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
new file mode 100644
index 000000000000..3f9a007e6bf4
--- /dev/null
+++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
@@ -0,0 +1,1902 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.cluster.settings.RestClusterGetSettingsResponse;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.MetadataIndexStateService;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.time.DateUtils;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Booleans;
+import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction;
+import org.elasticsearch.test.NotEqualMessageBuilder;
+import org.elasticsearch.test.XContentTestUtils;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.FeatureFlag;
+import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.ObjectPath;
+import org.elasticsearch.transport.Compression;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.json.JsonXContent;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestRule;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.IntStream;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
+import static java.util.stream.Collectors.toList;
+import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SYSTEM_INDEX_ENFORCEMENT_VERSION;
+import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING;
+import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY;
+import static org.elasticsearch.test.MapMatcher.assertMap;
+import static org.elasticsearch.test.MapMatcher.matchesMap;
+import static org.elasticsearch.transport.RemoteClusterService.REMOTE_CLUSTER_COMPRESS;
+import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+/**
+ * Tests to run before and after a full cluster restart. This is run twice,
+ * one with {@code tests.is_old_cluster} set to {@code true} against a cluster
+ * of an older version. The cluster is shutdown and a cluster of the new
+ * version is started with the same data directories and then this is rerun
+ * with {@code tests.is_old_cluster} set to {@code false}.
+ */
+public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCase {
+
+    private static TemporaryFolder repoDirectory = new TemporaryFolder();
+
+    protected static LocalClusterConfigProvider clusterConfig = c -> {};
+
+    private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .version(getOldClusterTestVersion())
+        .nodes(2)
+        .setting("path.repo", () -> repoDirectory.getRoot().getPath())
+        .setting("xpack.security.enabled", "false")
+        // some tests rely on the translog not being flushed
+        .setting("indices.memory.shard_inactive_time", "60m")
+        .apply(() -> clusterConfig)
+        .feature(FeatureFlag.TIME_SERIES_MODE)
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(cluster);
+
+    private String index;
+
+    public FullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) {
+        super(upgradeStatus);
+    }
+
+    @Override
+    protected ElasticsearchCluster getUpgradeCluster() {
+        return cluster;
+    }
+
+    @Before
+    public void setIndex() {
+        index = getRootTestName();
+    }
+
+    public void testSearch() throws Exception {
+        int count;
+        if (isRunningAgainstOldCluster()) {
+            XContentBuilder mappingsAndSettings = jsonBuilder();
+            mappingsAndSettings.startObject();
+            {
+                mappingsAndSettings.startObject("settings");
+                mappingsAndSettings.field("number_of_shards", 1);
+                mappingsAndSettings.field("number_of_replicas", 0);
+                mappingsAndSettings.endObject();
+            }
+            {
+                mappingsAndSettings.startObject("mappings");
+                mappingsAndSettings.startObject("properties");
+                {
+                    mappingsAndSettings.startObject("string");
+                    mappingsAndSettings.field("type", "text");
+                    mappingsAndSettings.endObject();
+                }
+                {
+                    mappingsAndSettings.startObject("dots_in_field_names");
+                    mappingsAndSettings.field("type", "text");
+                    mappingsAndSettings.endObject();
+                }
+                {
+                    mappingsAndSettings.startObject("binary");
+                    mappingsAndSettings.field("type", "binary");
+                    mappingsAndSettings.field("store", "true");
+                    mappingsAndSettings.endObject();
+                }
+                mappingsAndSettings.endObject();
+                mappingsAndSettings.endObject();
+            }
+            mappingsAndSettings.endObject();
+
+            Request createIndex = new Request("PUT", "/" + index);
+            createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
+            client().performRequest(createIndex);
+
+            count = randomIntBetween(2000, 3000);
+            byte[] randomByteArray = new byte[16];
+            random().nextBytes(randomByteArray);
+            indexRandomDocuments(
+                count,
+                true,
+                true,
+                randomBoolean(),
+                i -> JsonXContent.contentBuilder()
+                    .startObject()
+                    .field("string", randomAlphaOfLength(10))
+                    .field("int", randomInt(100))
+                    .field("float", randomFloat())
+                    // be sure to create a "proper" boolean (True, False) for the first document so that automapping is correct
+                    .field("bool", i > 0 && randomBoolean())
+                    .field("field.with.dots", randomAlphaOfLength(10))
+                    .field("binary", Base64.getEncoder().encodeToString(randomByteArray))
+                    .endObject()
+            );
+            refreshAllIndices();
+        } else {
+            count = countOfIndexedRandomDocuments();
+        }
+
+        ensureGreenLongWait(index);
+        assertBasicSearchWorks(count);
+        assertAllSearchWorks(count);
+        assertBasicAggregationWorks();
+        assertRealtimeGetWorks();
+        assertStoredBinaryFields(count);
+    }
+
+    public void testNewReplicas() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+            XContentBuilder mappingsAndSettings = jsonBuilder();
+            mappingsAndSettings.startObject();
+            {
+                mappingsAndSettings.startObject("settings");
+                mappingsAndSettings.field("number_of_shards", 1);
+                mappingsAndSettings.field("number_of_replicas", 0);
+                mappingsAndSettings.endObject();
+            }
+            {
+                mappingsAndSettings.startObject("mappings");
+                mappingsAndSettings.startObject("properties");
+                {
+                    mappingsAndSettings.startObject("field");
+                    mappingsAndSettings.field("type", "text");
+                    mappingsAndSettings.endObject();
+                }
+                mappingsAndSettings.endObject();
+                mappingsAndSettings.endObject();
+            }
+            mappingsAndSettings.endObject();
+
+            Request createIndex = new Request("PUT", "/" + index);
+            createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
+            client().performRequest(createIndex);
+
+            int numDocs = randomIntBetween(2000, 3000);
+            indexRandomDocuments(
+                numDocs,
+                true,
+                false,
+                randomBoolean(),
+                i -> JsonXContent.contentBuilder().startObject().field("field", "value").endObject()
+            );
+            logger.info("Refreshing [{}]", index);
+            client().performRequest(new Request("POST", "/" + index + "/_refresh"));
+        } else {
+            // The test runs with two nodes so this should still go green.
+            final int numReplicas = 1;
+            final long startTime = System.currentTimeMillis();
+            logger.debug("--> creating [{}] replicas for index [{}]", numReplicas, index);
+            Request setNumberOfReplicas = new Request("PUT", "/" + index + "/_settings");
+            setNumberOfReplicas.setJsonEntity("{ \"index\": { \"number_of_replicas\" : " + numReplicas + " }}");
+            client().performRequest(setNumberOfReplicas);
+
+            ensureGreenLongWait(index);
+
+            logger.debug("--> index [{}] is green, took [{}] ms", index, (System.currentTimeMillis() - startTime));
+            Map<String, Object> recoverRsp = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_recovery")));
+            logger.debug("--> recovery status:\n{}", recoverRsp);
+
+            Set<Integer> counts = new HashSet<>();
+            for (String node : dataNodes(index, client())) {
+                Request search = new Request("GET", "/" + index + "/_search");
+                search.addParameter("preference", "_only_nodes:" + node);
+                Map<String, Object> responseBody = entityAsMap(client().performRequest(search));
+                assertNoFailures(responseBody);
+                int hits = extractTotalHits(responseBody);
+                counts.add(hits);
+            }
+            assertEquals("All nodes should have a consistent number of documents", 1, counts.size());
+        }
+    }
+
+    public void testSearchTimeSeriesMode() throws Exception {
+        assumeTrue("indexing time series indices changed in 8.2.0", getOldClusterVersion().onOrAfter(Version.V_8_2_0));
+        int numDocs;
+        if (isRunningAgainstOldCluster()) {
+            numDocs = createTimeSeriesModeIndex(1);
+        } else {
+            numDocs = countOfIndexedRandomDocuments();
+        }
+        assertCountAll(numDocs);
+        Request request = new Request("GET", "/" + index + "/_search");
+        XContentBuilder body = jsonBuilder().startObject();
+        body.field("size", 0);
+        body.startObject("aggs").startObject("check").startObject("scripted_metric");
+        {
+            body.field("init_script", "state.timeSeries = new HashSet()");
+            body.field("map_script", "state.timeSeries.add(doc['dim'].value)");
+            body.field("combine_script", "return state.timeSeries");
+            StringBuilder reduceScript = new StringBuilder();
+            reduceScript.append("Set timeSeries = new TreeSet();");
+            reduceScript.append("for (s in states) {");
+            reduceScript.append("  for (ts in s) {");
+            reduceScript.append("    boolean newTs = timeSeries.add(ts);");
+            reduceScript.append("    if (false == newTs) {");
+            reduceScript.append("      throw new IllegalArgumentException(ts + ' appeared in two shards');");
+            reduceScript.append("    }");
+            reduceScript.append("  }");
+            reduceScript.append("}");
+            reduceScript.append("return timeSeries;");
+            body.field("reduce_script", reduceScript.toString());
+        }
+        body.endObject().endObject().endObject();
+        body.endObject();
+        request.setJsonEntity(Strings.toString(body));
+        Map<String, Object> response = entityAsMap(client().performRequest(request));
+        assertMap(
+            response,
+            matchesMap().extraOk()
+                .entry("hits", matchesMap().extraOk().entry("total", Map.of("value", numDocs, "relation", "eq")))
+                .entry("aggregations", Map.of("check", Map.of("value", IntStream.range(0, 10).mapToObj(i -> "dim" + i).collect(toList()))))
+        );
+    }
+
+    public void testNewReplicasTimeSeriesMode() throws Exception {
+        assumeTrue("indexing time series indices changed in 8.2.0", getOldClusterVersion().onOrAfter(Version.V_8_2_0));
+        if (isRunningAgainstOldCluster()) {
+            createTimeSeriesModeIndex(0);
+        } else {
+            // The test runs with two nodes so this should still go green.
+            final int numReplicas = 1;
+            final long startTime = System.currentTimeMillis();
+            logger.debug("--> creating [{}] replicas for index [{}]", numReplicas, index);
+            Request setNumberOfReplicas = new Request("PUT", "/" + index + "/_settings");
+            setNumberOfReplicas.setJsonEntity("{ \"index\": { \"number_of_replicas\" : " + numReplicas + " }}");
+            client().performRequest(setNumberOfReplicas);
+
+            ensureGreenLongWait(index);
+
+            logger.debug("--> index [{}] is green, took [{}] ms", index, (System.currentTimeMillis() - startTime));
+            Map<String, Object> recoverRsp = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_recovery")));
+            logger.debug("--> recovery status:\n{}", recoverRsp);
+
+            Set<Integer> counts = new HashSet<>();
+            for (String node : dataNodes(index, client())) {
+                Request search = new Request("GET", "/" + index + "/_search");
+                search.addParameter("preference", "_only_nodes:" + node);
+                Map<String, Object> responseBody = entityAsMap(client().performRequest(search));
+                assertNoFailures(responseBody);
+                int hits = extractTotalHits(responseBody);
+                counts.add(hits);
+            }
+            assertEquals("All nodes should have a consistent number of documents", 1, counts.size());
+        }
+    }
+
+    private int createTimeSeriesModeIndex(int replicas) throws IOException {
+        XContentBuilder mappingsAndSettings = jsonBuilder();
+        mappingsAndSettings.startObject();
+        {
+            mappingsAndSettings.startObject("settings");
+            mappingsAndSettings.field("number_of_shards", 1);
+            mappingsAndSettings.field("number_of_replicas", replicas);
+            mappingsAndSettings.field("mode", "time_series");
+            mappingsAndSettings.field("routing_path", "dim");
+            mappingsAndSettings.field("time_series.start_time", 1L);
+            mappingsAndSettings.field("time_series.end_time", DateUtils.MAX_MILLIS_BEFORE_9999 - 1);
+            mappingsAndSettings.endObject();
+        }
+        {
+            mappingsAndSettings.startObject("mappings");
+            mappingsAndSettings.startObject("properties");
+            {
+                mappingsAndSettings.startObject("@timestamp").field("type", "date").endObject();
+                mappingsAndSettings.startObject("dim").field("type", "keyword").field("time_series_dimension", true).endObject();
+            }
+            mappingsAndSettings.endObject();
+            mappingsAndSettings.endObject();
+        }
+        mappingsAndSettings.endObject();
+
+        Request createIndex = new Request("PUT", "/" + index);
+        createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
+        client().performRequest(createIndex);
+
+        int numDocs = randomIntBetween(2000, 3000);
+        long basetime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2021-01-01T00:00:00Z");
+        indexRandomDocuments(
+            numDocs,
+            true,
+            true,
+            false,
+            i -> JsonXContent.contentBuilder()
+                .startObject()
+                .field("@timestamp", basetime + TimeUnit.MINUTES.toMillis(i))
+                .field("dim", "dim" + (i % 10))
+                .endObject()
+        );
+        logger.info("Refreshing [{}]", index);
+        client().performRequest(new Request("POST", "/" + index + "/_refresh"));
+        return numDocs;
+    }
+
+    public void testClusterState() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+            XContentBuilder mappingsAndSettings = jsonBuilder();
+            mappingsAndSettings.startObject();
+            mappingsAndSettings.field("index_patterns", index);
+            mappingsAndSettings.field("order", "1000");
+            {
+                mappingsAndSettings.startObject("settings");
+                mappingsAndSettings.field("number_of_shards", 1);
+                mappingsAndSettings.field("number_of_replicas", 0);
+                mappingsAndSettings.endObject();
+            }
+            mappingsAndSettings.endObject();
+            Request createTemplate = new Request("PUT", "/_template/template_1");
+            createTemplate.setJsonEntity(Strings.toString(mappingsAndSettings));
+            createTemplate.setOptions(expectWarnings(RestPutIndexTemplateAction.DEPRECATION_WARNING));
+            client().performRequest(createTemplate);
+            client().performRequest(new Request("PUT", "/" + index));
+        }
+
+        // verifying if we can still read some properties from cluster state api:
+        Map<String, Object> clusterState = entityAsMap(client().performRequest(new Request("GET", "/_cluster/state")));
+
+        // Check some global properties:
+        String numberOfShards = (String) XContentMapValues.extractValue(
+            "metadata.templates.template_1.settings.index.number_of_shards",
+            clusterState
+        );
+        assertEquals("1", numberOfShards);
+        String numberOfReplicas = (String) XContentMapValues.extractValue(
+            "metadata.templates.template_1.settings.index.number_of_replicas",
+            clusterState
+        );
+        assertEquals("0", numberOfReplicas);
+
+        // Check some index properties:
+        numberOfShards = (String) XContentMapValues.extractValue(
+            "metadata.indices." + index + ".settings.index.number_of_shards",
+            clusterState
+        );
+        assertEquals("1", numberOfShards);
+        numberOfReplicas = (String) XContentMapValues.extractValue(
+            "metadata.indices." + index + ".settings.index.number_of_replicas",
+            clusterState
+        );
+        assertEquals("0", numberOfReplicas);
+        Version version = Version.fromId(
+            Integer.valueOf(
+                (String) XContentMapValues.extractValue("metadata.indices." + index + ".settings.index.version.created", clusterState)
+            )
+        );
+        assertEquals(getOldClusterVersion(), version);
+
+    }
+
+    public void testShrink() throws IOException {
+        String shrunkenIndex = index + "_shrunk";
+        int numDocs;
+        if (isRunningAgainstOldCluster()) {
+            XContentBuilder mappingsAndSettings = jsonBuilder();
+            mappingsAndSettings.startObject();
+            {
+                mappingsAndSettings.startObject("mappings");
+                {
+                    mappingsAndSettings.startObject("properties");
+                    {
+                        mappingsAndSettings.startObject("field");
+                        {
+                            mappingsAndSettings.field("type", "text");
+                        }
+                        mappingsAndSettings.endObject();
+                    }
+                    mappingsAndSettings.endObject();
+                }
+                mappingsAndSettings.endObject();
+
+                mappingsAndSettings.startObject("settings");
+                {
+                    mappingsAndSettings.field("index.number_of_shards", 5);
+                }
+                mappingsAndSettings.endObject();
+            }
+            mappingsAndSettings.endObject();
+
+            Request createIndex = new Request("PUT", "/" + index);
+            createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
+            client().performRequest(createIndex);
+
+            numDocs = randomIntBetween(512, 1024);
+            indexRandomDocuments(
+                numDocs,
+                true,
+                true,
+                randomBoolean(),
+                i -> JsonXContent.contentBuilder().startObject().field("field", "value").endObject()
+            );
+
+            ensureGreen(index); // wait for source index to be available on both nodes before starting shrink
+
+            Request updateSettingsRequest = new Request("PUT", "/" + index + "/_settings");
+            updateSettingsRequest.setJsonEntity("{\"settings\": {\"index.blocks.write\": true}}");
+            client().performRequest(updateSettingsRequest);
+
+            Request shrinkIndexRequest = new Request("PUT", "/" + index + "/_shrink/" + shrunkenIndex);
+
+            shrinkIndexRequest.setJsonEntity("{\"settings\": {\"index.number_of_shards\": 1}}");
+            client().performRequest(shrinkIndexRequest);
+
+            refreshAllIndices();
+        } else {
+            numDocs = countOfIndexedRandomDocuments();
+        }
+
+        Map<?, ?> response = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")));
+        assertNoFailures(response);
+        int totalShards = (int) XContentMapValues.extractValue("_shards.total", response);
+        assertThat(totalShards, greaterThan(1));
+        int successfulShards = (int) XContentMapValues.extractValue("_shards.successful", response);
+        assertEquals(totalShards, successfulShards);
+        int totalHits = extractTotalHits(response);
+        assertEquals(numDocs, totalHits);
+
+        response = entityAsMap(client().performRequest(new Request("GET", "/" + shrunkenIndex + "/_search")));
+        assertNoFailures(response);
+        totalShards = (int) XContentMapValues.extractValue("_shards.total", response);
+        assertEquals(1, totalShards);
+        successfulShards = (int) XContentMapValues.extractValue("_shards.successful", response);
+        assertEquals(1, successfulShards);
+        totalHits = extractTotalHits(response);
+        assertEquals(numDocs, totalHits);
+    }
+
+    public void testShrinkAfterUpgrade() throws IOException {
+        String shrunkenIndex = index + "_shrunk";
+        int numDocs;
+        if (isRunningAgainstOldCluster()) {
+            XContentBuilder mappingsAndSettings = jsonBuilder();
+            mappingsAndSettings.startObject();
+            {
+                mappingsAndSettings.startObject("mappings");
+                {
+                    mappingsAndSettings.startObject("properties");
+                    {
+                        mappingsAndSettings.startObject("field");
+                        {
+                            mappingsAndSettings.field("type", "text");
+                        }
+                        mappingsAndSettings.endObject();
+                    }
+                    mappingsAndSettings.endObject();
+                }
+                mappingsAndSettings.endObject();
+                // the default number of shards is now one so we have to set the number of shards to be more than one explicitly
+                mappingsAndSettings.startObject("settings");
+                mappingsAndSettings.field("index.number_of_shards", 5);
+                mappingsAndSettings.endObject();
+            }
+            mappingsAndSettings.endObject();
+
+            Request createIndex = new Request("PUT", "/" + index);
+            createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
+            client().performRequest(createIndex);
+
+            numDocs = randomIntBetween(512, 1024);
+            indexRandomDocuments(
+                numDocs,
+                true,
+                true,
+                randomBoolean(),
+                i -> JsonXContent.contentBuilder().startObject().field("field", "value").endObject()
+            );
+        } else {
+            ensureGreen(index); // wait for source index to be available on both nodes before starting shrink
+
+            Request updateSettingsRequest = new Request("PUT", "/" + index + "/_settings");
+            updateSettingsRequest.setJsonEntity("{\"settings\": {\"index.blocks.write\": true}}");
+            client().performRequest(updateSettingsRequest);
+
+            Request shrinkIndexRequest = new Request("PUT", "/" + index + "/_shrink/" + shrunkenIndex);
+            shrinkIndexRequest.setJsonEntity("{\"settings\": {\"index.number_of_shards\": 1}}");
+            client().performRequest(shrinkIndexRequest);
+
+            numDocs = countOfIndexedRandomDocuments();
+        }
+
+        refreshAllIndices();
+
+        Map<?, ?> response = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")));
+        assertNoFailures(response);
+        int totalShards = (int) XContentMapValues.extractValue("_shards.total", response);
+        assertThat(totalShards, greaterThan(1));
+        int successfulShards = (int) XContentMapValues.extractValue("_shards.successful", response);
+        assertEquals(totalShards, successfulShards);
+        int totalHits = extractTotalHits(response);
+        assertEquals(numDocs, totalHits);
+
+        if (isRunningAgainstOldCluster() == false) {
+            response = entityAsMap(client().performRequest(new Request("GET", "/" + shrunkenIndex + "/_search")));
+            assertNoFailures(response);
+            totalShards = (int) XContentMapValues.extractValue("_shards.total", response);
+            assertEquals(1, totalShards);
+            successfulShards = (int) XContentMapValues.extractValue("_shards.successful", response);
+            assertEquals(1, successfulShards);
+            totalHits = extractTotalHits(response);
+            assertEquals(numDocs, totalHits);
+        }
+    }
+
+    /**
+     * Test upgrading after a rollover. Specifically:
+     * <ol>
+     *  <li>Create an index with a write alias
+     *  <li>Write some documents to the write alias
+     *  <li>Roll over the index
+     *  <li>Make sure the document count is correct
+     *  <li>Upgrade
+     *  <li>Write some more documents to the write alias
+     *  <li>Make sure the document count is correct
+     * </ol>
+     */
+    public void testRollover() throws IOException {
+        if (isRunningAgainstOldCluster()) {
+            Request createIndex = new Request("PUT", "/" + index + "-000001");
+            createIndex.setJsonEntity(Strings.format("""
+                {
+                  "aliases": {
+                    "%s_write": {}
+                  }
+                }""", index));
+            client().performRequest(createIndex);
+        }
+
+        int bulkCount = 10;
+        String bulk = """
+            {"index":{}}
+            {"test":"test"}
+            """.repeat(bulkCount);
+
+        Request bulkRequest = new Request("POST", "/" + index + "_write/_bulk");
+
+        bulkRequest.setJsonEntity(bulk);
+        bulkRequest.addParameter("refresh", "");
+        assertThat(EntityUtils.toString(client().performRequest(bulkRequest).getEntity()), containsString("\"errors\":false"));
+
+        if (isRunningAgainstOldCluster()) {
+            Request rolloverRequest = new Request("POST", "/" + index + "_write/_rollover");
+            rolloverRequest.setJsonEntity("""
+                {  "conditions": {    "max_docs": 5  }}""");
+            client().performRequest(rolloverRequest);
+
+            assertThat(
+                EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices?v")).getEntity()),
+                containsString("testrollover-000002")
+            );
+        }
+
+        Request countRequest = new Request("POST", "/" + index + "-*/_search");
+        countRequest.addParameter("size", "0");
+        Map<String, Object> count = entityAsMap(client().performRequest(countRequest));
+        assertNoFailures(count);
+
+        int expectedCount = bulkCount + (isRunningAgainstOldCluster() ? 0 : bulkCount);
+        assertEquals(expectedCount, extractTotalHits(count));
+    }
+
+    void assertCountAll(int count) throws IOException {
+        Map<String, Object> response = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")));
+        assertNoFailures(response);
+        int numDocs = extractTotalHits(response);
+        logger.info("Found {} in old index", numDocs);
+        assertEquals(count, numDocs);
+    }
+
+    void assertBasicSearchWorks(int count) throws IOException {
+        logger.info("--> testing basic search");
+        {
+            assertCountAll(count);
+        }
+
+        logger.info("--> testing basic search with sort");
+        {
+            Request searchRequest = new Request("GET", "/" + index + "/_search");
+            searchRequest.setJsonEntity("""
+                { "sort": [{ "int" : "asc" }]}""");
+            Map<String, Object> response = entityAsMap(client().performRequest(searchRequest));
+            assertNoFailures(response);
+            assertTotalHits(count, response);
+        }
+
+        logger.info("--> testing exists filter");
+        {
+            Request searchRequest = new Request("GET", "/" + index + "/_search");
+            searchRequest.setJsonEntity("""
+                { "query": { "exists" : {"field": "string"} }}""");
+            Map<String, Object> response = entityAsMap(client().performRequest(searchRequest));
+            assertNoFailures(response);
+            assertTotalHits(count, response);
+        }
+
+        logger.info("--> testing field with dots in the name");
+        {
+            Request searchRequest = new Request("GET", "/" + index + "/_search");
+            searchRequest.setJsonEntity("""
+                { "query": { "exists" : {"field": "field.with.dots"} }}""");
+            Map<String, Object> response = entityAsMap(client().performRequest(searchRequest));
+            assertNoFailures(response);
+            assertTotalHits(count, response);
+        }
+    }
+
+    void assertAllSearchWorks(int count) throws IOException {
+        logger.info("--> testing _all search");
+        Map<String, Object> response = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")));
+        assertNoFailures(response);
+        assertTotalHits(count, response);
+        Map<?, ?> bestHit = (Map<?, ?>) ((List<?>) (XContentMapValues.extractValue("hits.hits", response))).get(0);
+
+        // Make sure there are payloads and they are taken into account for the score
+        // the 'string' field has a boost of 4 in the mappings so it should get a payload boost
+        String stringValue = (String) XContentMapValues.extractValue("_source.string", bestHit);
+        assertNotNull(stringValue);
+        String id = (String) bestHit.get("_id");
+
+        Request explainRequest = new Request("GET", "/" + index + "/_explain/" + id);
+        explainRequest.setJsonEntity("{ \"query\": { \"match_all\" : {} }}");
+        String explanation = toStr(client().performRequest(explainRequest));
+        assertFalse("Could not find payload boost in explanation\n" + explanation, explanation.contains("payloadBoost"));
+
+        // Make sure the query can run on the whole index
+        Request searchRequest = new Request("GET", "/" + index + "/_search");
+        searchRequest.setEntity(explainRequest.getEntity());
+        searchRequest.addParameter("explain", "true");
+        Map<?, ?> matchAllResponse = entityAsMap(client().performRequest(searchRequest));
+        assertNoFailures(matchAllResponse);
+        assertTotalHits(count, matchAllResponse);
+    }
+
+    void assertBasicAggregationWorks() throws IOException {
+        // histogram on a long
+        Request longHistogramRequest = new Request("GET", "/" + index + "/_search");
+        longHistogramRequest.setJsonEntity("""
+            {
+              "aggs": {
+                "histo": {
+                  "histogram": {
+                    "field": "int",
+                    "interval": 10
+                  }
+                }
+              }
+            }""");
+        Map<?, ?> longHistogram = entityAsMap(client().performRequest(longHistogramRequest));
+        assertNoFailures(longHistogram);
+        List<?> histoBuckets = (List<?>) XContentMapValues.extractValue("aggregations.histo.buckets", longHistogram);
+        int histoCount = 0;
+        for (Object entry : histoBuckets) {
+            Map<?, ?> bucket = (Map<?, ?>) entry;
+            histoCount += (Integer) bucket.get("doc_count");
+        }
+        assertTotalHits(histoCount, longHistogram);
+
+        // terms on a boolean
+        Request boolTermsRequest = new Request("GET", "/" + index + "/_search");
+        boolTermsRequest.setJsonEntity("""
+            {
+              "aggs": {
+                "bool_terms": {
+                  "terms": {
+                    "field": "bool"
+                  }
+                }
+              }
+            }""");
+        Map<?, ?> boolTerms = entityAsMap(client().performRequest(boolTermsRequest));
+        List<?> termsBuckets = (List<?>) XContentMapValues.extractValue("aggregations.bool_terms.buckets", boolTerms);
+        int termsCount = 0;
+        for (Object entry : termsBuckets) {
+            Map<?, ?> bucket = (Map<?, ?>) entry;
+            termsCount += (Integer) bucket.get("doc_count");
+        }
+        assertTotalHits(termsCount, boolTerms);
+    }
+
+    void assertRealtimeGetWorks() throws IOException {
+        Request disableAutoRefresh = new Request("PUT", "/" + index + "/_settings");
+        disableAutoRefresh.setJsonEntity("""
+            { "index": { "refresh_interval" : -1 }}""");
+        client().performRequest(disableAutoRefresh);
+
+        Request searchRequest = new Request("GET", "/" + index + "/_search");
+        searchRequest.setJsonEntity("""
+            { "query": { "match_all" : {} }}""");
+        Map<?, ?> searchResponse = entityAsMap(client().performRequest(searchRequest));
+        Map<?, ?> hit = (Map<?, ?>) ((List<?>) (XContentMapValues.extractValue("hits.hits", searchResponse))).get(0);
+        String docId = (String) hit.get("_id");
+
+        Request updateRequest = new Request("POST", "/" + index + "/_update/" + docId);
+        updateRequest.setJsonEntity("""
+            { "doc" : { "foo": "bar"}}""");
+        client().performRequest(updateRequest);
+
+        Request getRequest = new Request("GET", "/" + index + "/_doc/" + docId);
+
+        Map<String, Object> getRsp = entityAsMap(client().performRequest(getRequest));
+        Map<?, ?> source = (Map<?, ?>) getRsp.get("_source");
+        assertTrue("doc does not contain 'foo' key: " + source, source.containsKey("foo"));
+
+        Request enableAutoRefresh = new Request("PUT", "/" + index + "/_settings");
+        enableAutoRefresh.setJsonEntity("""
+            { "index": { "refresh_interval" : "1s" }}""");
+        client().performRequest(enableAutoRefresh);
+    }
+
+    void assertStoredBinaryFields(int count) throws Exception {
+        Request request = new Request("GET", "/" + index + "/_search");
+        request.setJsonEntity("""
+            {
+              "query": {
+                "match_all": {}
+              },
+              "size": 100,
+              "stored_fields": "binary"
+            }""");
+        Map<String, Object> rsp = entityAsMap(client().performRequest(request));
+
+        assertTotalHits(count, rsp);
+        List<?> hits = (List<?>) XContentMapValues.extractValue("hits.hits", rsp);
+        assertEquals(100, hits.size());
+        for (Object hit : hits) {
+            Map<?, ?> hitRsp = (Map<?, ?>) hit;
+            List<?> values = (List<?>) XContentMapValues.extractValue("fields.binary", hitRsp);
+            assertEquals(1, values.size());
+            String value = (String) values.get(0);
+            byte[] binaryValue = Base64.getDecoder().decode(value);
+            assertEquals("Unexpected string length [" + value + "]", 16, binaryValue.length);
+        }
+    }
+
+    static String toStr(Response response) throws IOException {
+        return EntityUtils.toString(response.getEntity());
+    }
+
+    static void assertNoFailures(Map<?, ?> response) {
+        int failed = (int) XContentMapValues.extractValue("_shards.failed", response);
+        assertEquals(0, failed);
+    }
+
+    void assertTotalHits(int expectedTotalHits, Map<?, ?> response) {
+        int actualTotalHits = extractTotalHits(response);
+        assertEquals(response.toString(), expectedTotalHits, actualTotalHits);
+    }
+
+    static int extractTotalHits(Map<?, ?> response) {
+        return (Integer) XContentMapValues.extractValue("hits.total.value", response);
+    }
+
+    /**
+     * Tests that a single document survives. Super basic smoke test.
+     */
+    public void testSingleDoc() throws IOException {
+        String docLocation = "/" + index + "/_doc/1";
+        String doc = "{\"test\": \"test\"}";
+
+        if (isRunningAgainstOldCluster()) {
+            Request createDoc = new Request("PUT", docLocation);
+            createDoc.setJsonEntity(doc);
+            client().performRequest(createDoc);
+        }
+
+        Request request = new Request("GET", docLocation);
+        assertThat(toStr(client().performRequest(request)), containsString(doc));
+    }
+
+    /**
+     * Tests that a single empty shard index is correctly recovered. Empty shards are often an edge case.
+     */
+    public void testEmptyShard() throws IOException {
+        final String indexName = "test_empty_shard";
+
+        if (isRunningAgainstOldCluster()) {
+            Settings.Builder settings = Settings.builder()
+                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
+                .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
+                // if the node with the replica is the first to be restarted, while a replica is still recovering
+                // then delayed allocation will kick in. When the node comes back, the master will search for a copy
+                // but the recovering copy will be seen as invalid and the cluster health won't return to GREEN
+                // before timing out
+                .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
+                .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
+            createIndex(indexName, settings.build());
+        }
+        ensureGreen(indexName);
+    }
+
+    /**
+     * Tests recovery of an index with or without a translog and the
+     * statistics we gather about that.
+     */
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/52031")
+    public void testRecovery() throws Exception {
+        int count;
+        boolean shouldHaveTranslog;
+        if (isRunningAgainstOldCluster()) {
+            count = between(200, 300);
+            /* We've had bugs in the past where we couldn't restore
+             * an index without a translog so we randomize whether
+             * or not we have one. */
+            shouldHaveTranslog = randomBoolean();
+            Settings.Builder settings = Settings.builder();
+            if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
+                settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
+            }
+            final String mappings = randomBoolean() ? "\"_source\": { \"enabled\": false}" : null;
+            createIndex(index, settings.build(), mappings);
+            indexRandomDocuments(count, true, true, true, i -> jsonBuilder().startObject().field("field", "value").endObject());
+
+            // make sure all recoveries are done
+            ensureGreen(index);
+
+            // Force flush so we're sure that all translog are committed
+            Request flushRequest = new Request("POST", "/" + index + "/_flush");
+            flushRequest.addParameter("force", "true");
+            flushRequest.addParameter("wait_if_ongoing", "true");
+            assertOK(client().performRequest(flushRequest));
+
+            if (shouldHaveTranslog) {
+                // Update a few documents so we are sure to have a translog
+                indexRandomDocuments(
+                    count / 10,
+                    false, // flushing here would invalidate the whole thing
+                    false,
+                    true,
+                    i -> jsonBuilder().startObject().field("field", "value").endObject()
+                );
+            }
+            saveInfoDocument(index + "_should_have_translog", Boolean.toString(shouldHaveTranslog));
+        } else {
+            count = countOfIndexedRandomDocuments();
+            shouldHaveTranslog = Booleans.parseBoolean(loadInfoDocument(index + "_should_have_translog"));
+        }
+
+        // Count the documents in the index to make sure we have as many as we put there
+        Request countRequest = new Request("GET", "/" + index + "/_search");
+        countRequest.addParameter("size", "0");
+        refreshAllIndices();
+        Map<String, Object> countResponse = entityAsMap(client().performRequest(countRequest));
+        assertTotalHits(count, countResponse);
+
+        if (false == isRunningAgainstOldCluster()) {
+            boolean restoredFromTranslog = false;
+            boolean foundPrimary = false;
+            Request recoveryRequest = new Request("GET", "/_cat/recovery/" + index);
+            recoveryRequest.addParameter("h", "index,shard,type,stage,translog_ops_recovered");
+            recoveryRequest.addParameter("s", "index,shard,type");
+            String recoveryResponse = toStr(client().performRequest(recoveryRequest));
+            for (String line : recoveryResponse.split("\n")) {
+                // Find the primaries
+                foundPrimary = true;
+                if (false == line.contains("done") && line.contains("existing_store")) {
+                    continue;
+                }
+                /* Mark if we see a primary that looked like it restored from the translog.
+                 * Not all primaries will look like this all the time because we modify
+                 * random documents when we want there to be a translog and they might
+                 * not be spread around all the shards. */
+                Matcher m = Pattern.compile("(\\d+)$").matcher(line);
+                assertTrue(line, m.find());
+                int translogOps = Integer.parseInt(m.group(1));
+                if (translogOps > 0) {
+                    restoredFromTranslog = true;
+                }
+            }
+            assertTrue("expected to find a primary but didn't\n" + recoveryResponse, foundPrimary);
+            assertEquals("mismatch while checking for translog recovery\n" + recoveryResponse, shouldHaveTranslog, restoredFromTranslog);
+
+            String currentLuceneVersion = Version.CURRENT.luceneVersion.toString();
+            String bwcLuceneVersion = getOldClusterVersion().luceneVersion.toString();
+            String minCompatibleBWCVersion = Version.CURRENT.minimumCompatibilityVersion().luceneVersion.toString();
+            if (shouldHaveTranslog && false == currentLuceneVersion.equals(bwcLuceneVersion)) {
+                int numCurrentVersion = 0;
+                int numBwcVersion = 0;
+                Request segmentsRequest = new Request("GET", "/_cat/segments/" + index);
+                segmentsRequest.addParameter("h", "prirep,shard,index,version");
+                segmentsRequest.addParameter("s", "prirep,shard,index");
+                String segmentsResponse = toStr(client().performRequest(segmentsRequest));
+                for (String line : segmentsResponse.split("\n")) {
+                    if (false == line.startsWith("p")) {
+                        continue;
+                    }
+                    Matcher m = Pattern.compile("(\\d+\\.\\d+\\.\\d+)$").matcher(line);
+                    assertTrue(line, m.find());
+                    String version = m.group(1);
+                    if (currentLuceneVersion.equals(version)) {
+                        numCurrentVersion++;
+                    } else if (bwcLuceneVersion.equals(version)) {
+                        numBwcVersion++;
+                    } else if (minCompatibleBWCVersion.equals(version) && minCompatibleBWCVersion.equals(bwcLuceneVersion) == false) {
+                        // Our upgrade path from 7.non-last always goes through 7.last, which depending on timing can create 7.last
+                        // index segment. We ignore those.
+                        continue;
+                    } else {
+                        fail("expected version to be one of [" + currentLuceneVersion + "," + bwcLuceneVersion + "] but was " + line);
+                    }
+                }
+                assertNotEquals(
+                    "expected at least 1 current segment after translog recovery. segments:\n" + segmentsResponse,
+                    0,
+                    numCurrentVersion
+                );
+                assertNotEquals("expected at least 1 old segment. segments:\n" + segmentsResponse, 0, numBwcVersion);
+            }
+        }
+    }
+
+    /**
+     * Tests snapshot/restore by creating a snapshot and restoring it. It takes
+     * a snapshot on the old cluster and restores it on the old cluster as a
+     * sanity check and on the new cluster as an upgrade test. It also takes a
+     * snapshot on the new cluster and restores that on the new cluster as a
+     * test that the repository is ok with containing snapshot from both the
+     * old and new versions. All of the snapshots include an index, a template,
+     * and some routing configuration.
+     */
+    public void testSnapshotRestore() throws IOException {
+        int count;
+        if (isRunningAgainstOldCluster()) {
+            // Create the index
+            count = between(200, 300);
+            Settings.Builder settings = Settings.builder();
+            if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
+                settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
+            }
+            createIndex(index, settings.build());
+            indexRandomDocuments(count, true, true, randomBoolean(), i -> jsonBuilder().startObject().field("field", "value").endObject());
+        } else {
+            count = countOfIndexedRandomDocuments();
+        }
+
+        // Refresh the index so the count doesn't fail
+        refreshAllIndices();
+
+        // Count the documents in the index to make sure we have as many as we put there
+        Request countRequest = new Request("GET", "/" + index + "/_search");
+        countRequest.addParameter("size", "0");
+        Map<String, Object> countResponse = entityAsMap(client().performRequest(countRequest));
+        assertTotalHits(count, countResponse);
+
+        // Stick a routing attribute into to cluster settings so we can see it after the restore
+        Request addRoutingSettings = new Request("PUT", "/_cluster/settings");
+        addRoutingSettings.setJsonEntity(Strings.format("""
+            {"persistent": {"cluster.routing.allocation.exclude.test_attr": "%s"}}
+            """, getOldClusterVersion()));
+        client().performRequest(addRoutingSettings);
+
+        // Stick a template into the cluster so we can see it after the restore
+        XContentBuilder templateBuilder = JsonXContent.contentBuilder().startObject();
+        templateBuilder.field("index_patterns", "evil_*"); // Don't confuse other tests by applying the template
+        templateBuilder.startObject("settings");
+        {
+            templateBuilder.field("number_of_shards", 1);
+        }
+        templateBuilder.endObject();
+        templateBuilder.startObject("mappings");
+        {
+            {
+                templateBuilder.startObject("_source");
+                {
+                    templateBuilder.field("enabled", true);
+                }
+                templateBuilder.endObject();
+            }
+        }
+        templateBuilder.endObject();
+        templateBuilder.startObject("aliases");
+        {
+            templateBuilder.startObject("alias1").endObject();
+            templateBuilder.startObject("alias2");
+            {
+                templateBuilder.startObject("filter");
+                {
+                    templateBuilder.startObject("term");
+                    {
+                        templateBuilder.field("version", isRunningAgainstOldCluster() ? getOldClusterVersion() : Version.CURRENT);
+                    }
+                    templateBuilder.endObject();
+                }
+                templateBuilder.endObject();
+            }
+            templateBuilder.endObject();
+        }
+        templateBuilder.endObject().endObject();
+        Request createTemplateRequest = new Request("PUT", "/_template/test_template");
+        createTemplateRequest.setJsonEntity(Strings.toString(templateBuilder));
+        createTemplateRequest.setOptions(expectWarnings(RestPutIndexTemplateAction.DEPRECATION_WARNING));
+
+        client().performRequest(createTemplateRequest);
+
+        if (isRunningAgainstOldCluster()) {
+            // Create the repo
+            XContentBuilder repoConfig = JsonXContent.contentBuilder().startObject();
+            {
+                repoConfig.field("type", "fs");
+                repoConfig.startObject("settings");
+                {
+                    repoConfig.field("compress", randomBoolean());
+                    repoConfig.field("location", repoDirectory.getRoot().getPath());
+                }
+                repoConfig.endObject();
+            }
+            repoConfig.endObject();
+            Request createRepoRequest = new Request("PUT", "/_snapshot/repo");
+            createRepoRequest.setJsonEntity(Strings.toString(repoConfig));
+            client().performRequest(createRepoRequest);
+        }
+
+        Request createSnapshot = new Request("PUT", "/_snapshot/repo/" + (isRunningAgainstOldCluster() ? "old_snap" : "new_snap"));
+        createSnapshot.addParameter("wait_for_completion", "true");
+        createSnapshot.setJsonEntity("{\"indices\": \"" + index + "\"}");
+        client().performRequest(createSnapshot);
+
+        checkSnapshot("old_snap", count, getOldClusterVersion());
+        if (false == isRunningAgainstOldCluster()) {
+            checkSnapshot("new_snap", count, Version.CURRENT);
+        }
+    }
+
+    public void testHistoryUUIDIsAdded() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+            XContentBuilder mappingsAndSettings = jsonBuilder();
+            mappingsAndSettings.startObject();
+            {
+                mappingsAndSettings.startObject("settings");
+                mappingsAndSettings.field("number_of_shards", 1);
+                mappingsAndSettings.field("number_of_replicas", 1);
+                mappingsAndSettings.endObject();
+            }
+            mappingsAndSettings.endObject();
+            Request createIndex = new Request("PUT", "/" + index);
+            createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
+            client().performRequest(createIndex);
+        } else {
+            ensureGreenLongWait(index);
+
+            Request statsRequest = new Request("GET", index + "/_stats");
+            statsRequest.addParameter("level", "shards");
+            Response response = client().performRequest(statsRequest);
+            List<Object> shardStats = ObjectPath.createFromResponse(response).evaluate("indices." + index + ".shards.0");
+            assertThat(shardStats, notNullValue());
+            assertThat("Expected stats for 2 shards", shardStats, hasSize(2));
+            String globalHistoryUUID = null;
+            for (Object shard : shardStats) {
+                final String nodeId = ObjectPath.evaluate(shard, "routing.node");
+                final Boolean primary = ObjectPath.evaluate(shard, "routing.primary");
+                logger.info("evaluating: {} , {}", ObjectPath.evaluate(shard, "routing"), ObjectPath.evaluate(shard, "commit"));
+                String historyUUID = ObjectPath.evaluate(shard, "commit.user_data.history_uuid");
+                assertThat("no history uuid found on " + nodeId + " (primary: " + primary + ")", historyUUID, notNullValue());
+                if (globalHistoryUUID == null) {
+                    globalHistoryUUID = historyUUID;
+                } else {
+                    assertThat(
+                        "history uuid mismatch on " + nodeId + " (primary: " + primary + ")",
+                        historyUUID,
+                        equalTo(globalHistoryUUID)
+                    );
+                }
+            }
+        }
+    }
+
+    public void testSoftDeletes() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+            XContentBuilder mappingsAndSettings = jsonBuilder();
+            mappingsAndSettings.startObject();
+            {
+                mappingsAndSettings.startObject("settings");
+                mappingsAndSettings.field("number_of_shards", 1);
+                mappingsAndSettings.field("number_of_replicas", 1);
+                if (randomBoolean()) {
+                    mappingsAndSettings.field("soft_deletes.enabled", true);
+                }
+                mappingsAndSettings.endObject();
+            }
+            mappingsAndSettings.endObject();
+            Request createIndex = new Request("PUT", "/" + index);
+            createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
+            client().performRequest(createIndex);
+            int numDocs = between(10, 100);
+            for (int i = 0; i < numDocs; i++) {
+                String doc = Strings.toString(JsonXContent.contentBuilder().startObject().field("field", "v1").endObject());
+                Request request = new Request("POST", "/" + index + "/_doc/" + i);
+                request.setJsonEntity(doc);
+                client().performRequest(request);
+                refreshAllIndices();
+            }
+            client().performRequest(new Request("POST", "/" + index + "/_flush"));
+            int liveDocs = numDocs;
+            assertTotalHits(liveDocs, entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))));
+            for (int i = 0; i < numDocs; i++) {
+                if (randomBoolean()) {
+                    String doc = Strings.toString(JsonXContent.contentBuilder().startObject().field("field", "v2").endObject());
+                    Request request = new Request("POST", "/" + index + "/_doc/" + i);
+                    request.setJsonEntity(doc);
+                    client().performRequest(request);
+                } else if (randomBoolean()) {
+                    client().performRequest(new Request("DELETE", "/" + index + "/_doc/" + i));
+                    liveDocs--;
+                }
+            }
+            refreshAllIndices();
+            assertTotalHits(liveDocs, entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))));
+            saveInfoDocument(index + "_doc_count", Integer.toString(liveDocs));
+        } else {
+            int liveDocs = Integer.parseInt(loadInfoDocument(index + "_doc_count"));
+            assertTotalHits(liveDocs, entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))));
+        }
+    }
+
+    /**
+     * This test creates an index in the old cluster and then closes it. When the cluster is fully restarted in a newer version,
+     * it verifies that the index exists and is replicated if the old version supports replication.
+     */
+    public void testClosedIndices() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+            createIndex(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build());
+            ensureGreen(index);
+
+            int numDocs = 0;
+            if (randomBoolean()) {
+                numDocs = between(1, 100);
+                for (int i = 0; i < numDocs; i++) {
+                    final Request request = new Request("POST", "/" + index + "/_doc/" + i);
+                    request.setJsonEntity(Strings.toString(JsonXContent.contentBuilder().startObject().field("field", "v1").endObject()));
+                    assertOK(client().performRequest(request));
+                    if (rarely()) {
+                        refreshAllIndices();
+                    }
+                }
+                refreshAllIndices();
+            }
+
+            assertTotalHits(numDocs, entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))));
+            saveInfoDocument(index + "_doc_count", Integer.toString(numDocs));
+            closeIndex(index);
+        }
+
+        if (getOldClusterVersion().onOrAfter(Version.V_7_2_0)) {
+            ensureGreenLongWait(index);
+            assertClosedIndex(index, true);
+        } else {
+            assertClosedIndex(index, false);
+        }
+
+        if (isRunningAgainstOldCluster() == false) {
+            openIndex(index);
+            ensureGreen(index);
+
+            final int expectedNumDocs = Integer.parseInt(loadInfoDocument(index + "_doc_count"));
+            assertTotalHits(expectedNumDocs, entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))));
+        }
+    }
+
+    /**
+     * Asserts that an index is closed in the cluster state. If `checkRoutingTable` is true, it also asserts
+     * that the index has started shards.
+     */
+    @SuppressWarnings("unchecked")
+    private void assertClosedIndex(final String indexName, final boolean checkRoutingTable) throws IOException {
+        final Map<String, ?> state = entityAsMap(client().performRequest(new Request("GET", "/_cluster/state")));
+
+        final Map<String, ?> metadata = (Map<String, Object>) XContentMapValues.extractValue("metadata.indices." + indexName, state);
+        assertThat(metadata, notNullValue());
+        assertThat(metadata.get("state"), equalTo("close"));
+
+        final Map<String, ?> blocks = (Map<String, Object>) XContentMapValues.extractValue("blocks.indices." + indexName, state);
+        assertThat(blocks, notNullValue());
+        assertThat(blocks.containsKey(String.valueOf(MetadataIndexStateService.INDEX_CLOSED_BLOCK_ID)), is(true));
+
+        final Map<String, ?> settings = (Map<String, Object>) XContentMapValues.extractValue("settings", metadata);
+        assertThat(settings, notNullValue());
+
+        final Map<String, ?> routingTable = (Map<String, Object>) XContentMapValues.extractValue(
+            "routing_table.indices." + indexName,
+            state
+        );
+        if (checkRoutingTable) {
+            assertThat(routingTable, notNullValue());
+            assertThat(Booleans.parseBoolean((String) XContentMapValues.extractValue("index.verified_before_close", settings)), is(true));
+            final String numberOfShards = (String) XContentMapValues.extractValue("index.number_of_shards", settings);
+            assertThat(numberOfShards, notNullValue());
+            final int nbShards = Integer.parseInt(numberOfShards);
+            assertThat(nbShards, greaterThanOrEqualTo(1));
+
+            for (int i = 0; i < nbShards; i++) {
+                final Collection<Map<String, ?>> shards = (Collection<Map<String, ?>>) XContentMapValues.extractValue(
+                    "shards." + i,
+                    routingTable
+                );
+                assertThat(shards, notNullValue());
+                assertThat(shards.size(), equalTo(2));
+                for (Map<String, ?> shard : shards) {
+                    assertThat(XContentMapValues.extractValue("shard", shard), equalTo(i));
+                    assertThat(XContentMapValues.extractValue("state", shard), equalTo("STARTED"));
+                    assertThat(XContentMapValues.extractValue("index", shard), equalTo(indexName));
+                }
+            }
+        } else {
+            assertThat(routingTable, nullValue());
+            assertThat(XContentMapValues.extractValue("index.verified_before_close", settings), nullValue());
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private void checkSnapshot(final String snapshotName, final int count, final Version tookOnVersion) throws IOException {
+        // Check the snapshot metadata, especially the version
+        Request listSnapshotRequest = new Request("GET", "/_snapshot/repo/" + snapshotName);
+        Map<String, Object> snapResponse = entityAsMap(client().performRequest(listSnapshotRequest));
+
+        assertEquals(singletonList(snapshotName), XContentMapValues.extractValue("snapshots.snapshot", snapResponse));
+        assertEquals(singletonList("SUCCESS"), XContentMapValues.extractValue("snapshots.state", snapResponse));
+        assertEquals(singletonList(tookOnVersion.toString()), XContentMapValues.extractValue("snapshots.version", snapResponse));
+
+        // Remove the routing setting and template so we can test restoring them.
+        Request clearRoutingFromSettings = new Request("PUT", "/_cluster/settings");
+        clearRoutingFromSettings.setJsonEntity("""
+            {"persistent":{"cluster.routing.allocation.exclude.test_attr": null}}""");
+        client().performRequest(clearRoutingFromSettings);
+        client().performRequest(new Request("DELETE", "/_template/test_template"));
+
+        // Restore
+        XContentBuilder restoreCommand = JsonXContent.contentBuilder().startObject();
+        restoreCommand.field("include_global_state", true);
+        restoreCommand.field("indices", index);
+        restoreCommand.field("rename_pattern", index);
+        restoreCommand.field("rename_replacement", "restored_" + index);
+        restoreCommand.endObject();
+        Request restoreRequest = new Request("POST", "/_snapshot/repo/" + snapshotName + "/_restore");
+        restoreRequest.addParameter("wait_for_completion", "true");
+        restoreRequest.setJsonEntity(Strings.toString(restoreCommand));
+        client().performRequest(restoreRequest);
+
+        // Make sure search finds all documents
+        Request countRequest = new Request("GET", "/restored_" + index + "/_search");
+        countRequest.addParameter("size", "0");
+        Map<String, Object> countResponse = entityAsMap(client().performRequest(countRequest));
+        assertTotalHits(count, countResponse);
+
+        // Add some extra documents to the index to be sure we can still write to it after restoring it
+        int extras = between(1, 100);
+        StringBuilder bulk = new StringBuilder();
+        for (int i = 0; i < extras; i++) {
+            bulk.append(Strings.format("""
+                {"index":{"_id":"%s"}}
+                {"test":"test"}
+                """, count + i));
+        }
+
+        Request writeToRestoredRequest = new Request("POST", "/restored_" + index + "/_bulk");
+
+        writeToRestoredRequest.addParameter("refresh", "true");
+        writeToRestoredRequest.setJsonEntity(bulk.toString());
+        assertThat(EntityUtils.toString(client().performRequest(writeToRestoredRequest).getEntity()), containsString("\"errors\":false"));
+
+        // And count to make sure the add worked
+        // Make sure search finds all documents
+        Request countAfterWriteRequest = new Request("GET", "/restored_" + index + "/_search");
+        countAfterWriteRequest.addParameter("size", "0");
+        Map<String, Object> countAfterResponse = entityAsMap(client().performRequest(countRequest));
+        assertTotalHits(count + extras, countAfterResponse);
+
+        // Clean up the index for the next iteration
+        client().performRequest(new Request("DELETE", "/restored_*"));
+
+        // Check settings added by the restore process
+        Request clusterSettingsRequest = new Request("GET", "/_cluster/settings");
+        clusterSettingsRequest.addParameter("flat_settings", "true");
+        Map<String, Object> clusterSettingsResponse = entityAsMap(client().performRequest(clusterSettingsRequest));
+        @SuppressWarnings("unchecked")
+        final Map<String, Object> persistentSettings = (Map<String, Object>) clusterSettingsResponse.get("persistent");
+        assertThat(persistentSettings.get("cluster.routing.allocation.exclude.test_attr"), equalTo(getOldClusterVersion().toString()));
+
+        // Check that the template was restored successfully
+        Request getTemplateRequest = new Request("GET", "/_template/test_template");
+
+        Map<String, Object> getTemplateResponse = entityAsMap(client().performRequest(getTemplateRequest));
+        Map<String, Object> expectedTemplate = new HashMap<>();
+        expectedTemplate.put("index_patterns", singletonList("evil_*"));
+
+        expectedTemplate.put("settings", singletonMap("index", singletonMap("number_of_shards", "1")));
+        expectedTemplate.put("mappings", singletonMap("_source", singletonMap("enabled", true)));
+
+        expectedTemplate.put("order", 0);
+        Map<String, Object> aliases = new HashMap<>();
+        aliases.put("alias1", emptyMap());
+        aliases.put("alias2", singletonMap("filter", singletonMap("term", singletonMap("version", tookOnVersion.toString()))));
+        expectedTemplate.put("aliases", aliases);
+        expectedTemplate = singletonMap("test_template", expectedTemplate);
+        if (false == expectedTemplate.equals(getTemplateResponse)) {
+            NotEqualMessageBuilder builder = new NotEqualMessageBuilder();
+            builder.compareMaps(getTemplateResponse, expectedTemplate);
+            logger.info("expected: {}\nactual:{}", expectedTemplate, getTemplateResponse);
+            fail("template doesn't match:\n" + builder.toString());
+        }
+    }
+
+    private void indexRandomDocuments(
+        final int count,
+        final boolean flushAllowed,
+        final boolean saveInfo,
+        final boolean specifyId,
+        final CheckedFunction<Integer, XContentBuilder, IOException> docSupplier
+    ) throws IOException {
+        logger.info("Indexing {} random documents", count);
+        for (int i = 0; i < count; i++) {
+            logger.debug("Indexing document [{}]", i);
+            Request createDocument = new Request("POST", "/" + index + "/_doc/" + (specifyId ? i : ""));
+            createDocument.setJsonEntity(Strings.toString(docSupplier.apply(i)));
+            client().performRequest(createDocument);
+            if (rarely()) {
+                refreshAllIndices();
+            }
+            if (flushAllowed && rarely()) {
+                logger.debug("Flushing [{}]", index);
+                client().performRequest(new Request("POST", "/" + index + "/_flush"));
+            }
+        }
+        if (saveInfo) {
+            saveInfoDocument(index + "_count", Integer.toString(count));
+        }
+    }
+
+    private void indexDocument(String id) throws IOException {
+        final Request indexRequest = new Request("POST", "/" + index + "/" + "_doc/" + id);
+        indexRequest.setJsonEntity(Strings.toString(JsonXContent.contentBuilder().startObject().field("f", "v").endObject()));
+        assertOK(client().performRequest(indexRequest));
+    }
+
+    private int countOfIndexedRandomDocuments() throws IOException {
+        return Integer.parseInt(loadInfoDocument(index + "_count"));
+    }
+
+    private void saveInfoDocument(String id, String value) throws IOException {
+        XContentBuilder infoDoc = JsonXContent.contentBuilder().startObject();
+        infoDoc.field("value", value);
+        infoDoc.endObject();
+        // Only create the first version so we know how many documents are created when the index is first created
+        Request request = new Request("PUT", "/info/_doc/" + id);
+        request.addParameter("op_type", "create");
+        request.setJsonEntity(Strings.toString(infoDoc));
+        client().performRequest(request);
+    }
+
+    private String loadInfoDocument(String id) throws IOException {
+        Request request = new Request("GET", "/info/_doc/" + id);
+        request.addParameter("filter_path", "_source");
+        String doc = toStr(client().performRequest(request));
+        Matcher m = Pattern.compile("\"value\":\"(.+)\"").matcher(doc);
+        assertTrue(doc, m.find());
+        return m.group(1);
+    }
+
+    private List<String> dataNodes(String indexName, RestClient client) throws IOException {
+        Request request = new Request("GET", indexName + "/_stats");
+        request.addParameter("level", "shards");
+        Response response = client.performRequest(request);
+        List<String> nodes = new ArrayList<>();
+        List<Object> shardStats = ObjectPath.createFromResponse(response).evaluate("indices." + indexName + ".shards.0");
+        for (Object shard : shardStats) {
+            final String nodeId = ObjectPath.evaluate(shard, "routing.node");
+            nodes.add(nodeId);
+        }
+        return nodes;
+    }
+
+    /**
+     * Wait for an index to have green health, waiting longer than
+     * {@link ESRestTestCase#ensureGreen}.
+     */
+    protected void ensureGreenLongWait(String indexName) throws IOException {
+        Request request = new Request("GET", "/_cluster/health/" + indexName);
+        request.addParameter("timeout", "2m");
+        request.addParameter("wait_for_status", "green");
+        request.addParameter("wait_for_no_relocating_shards", "true");
+        request.addParameter("wait_for_events", "languid");
+        request.addParameter("level", "shards");
+        Map<String, Object> healthRsp = entityAsMap(client().performRequest(request));
+        logger.info("health api response: {}", healthRsp);
+        assertEquals("green", healthRsp.get("status"));
+        assertFalse((Boolean) healthRsp.get("timed_out"));
+    }
+
+    public void testPeerRecoveryRetentionLeases() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+            XContentBuilder settings = jsonBuilder();
+            settings.startObject();
+            {
+                settings.startObject("settings");
+                settings.field("number_of_shards", between(1, 5));
+                settings.field("number_of_replicas", between(0, 1));
+                settings.endObject();
+            }
+            settings.endObject();
+
+            Request createIndex = new Request("PUT", "/" + index);
+            createIndex.setJsonEntity(Strings.toString(settings));
+            client().performRequest(createIndex);
+        }
+        ensureGreen(index);
+        ensurePeerRecoveryRetentionLeasesRenewedAndSynced(index);
+    }
+
+    /**
+     * Tests that with or without soft-deletes, we should perform an operation-based recovery if there were some
+     * but not too many uncommitted documents (i.e., less than 10% of committed documents or the extra translog)
+     * before we restart the cluster. This is important when we move from translog based to retention leases based
+     * peer recoveries.
+     */
+    public void testOperationBasedRecovery() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+            Settings.Builder settings = Settings.builder()
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1);
+            if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
+                settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
+            }
+            final String mappings = randomBoolean() ? "\"_source\": { \"enabled\": false}" : null;
+            createIndex(index, settings.build(), mappings);
+            ensureGreen(index);
+            int committedDocs = randomIntBetween(100, 200);
+            for (int i = 0; i < committedDocs; i++) {
+                indexDocument(Integer.toString(i));
+                if (rarely()) {
+                    flush(index, randomBoolean());
+                }
+            }
+            flush(index, true);
+            ensurePeerRecoveryRetentionLeasesRenewedAndSynced(index);
+            // less than 10% of the committed docs (see IndexSetting#FILE_BASED_RECOVERY_THRESHOLD_SETTING).
+            int uncommittedDocs = randomIntBetween(0, (int) (committedDocs * 0.1));
+            for (int i = 0; i < uncommittedDocs; i++) {
+                final String id = Integer.toString(randomIntBetween(1, 100));
+                indexDocument(id);
+            }
+        } else {
+            ensureGreen(index);
+            assertNoFileBasedRecovery(index, n -> true);
+            ensurePeerRecoveryRetentionLeasesRenewedAndSynced(index);
+        }
+    }
+
+    /**
+     * Verifies that once all shard copies on the new version, we should turn off the translog retention for indices with soft-deletes.
+     */
+    public void testTurnOffTranslogRetentionAfterUpgraded() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+            createIndex(
+                index,
+                Settings.builder()
+                    .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
+                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
+                    .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
+                    .build()
+            );
+            ensureGreen(index);
+            int numDocs = randomIntBetween(10, 100);
+            for (int i = 0; i < numDocs; i++) {
+                indexDocument(Integer.toString(randomIntBetween(1, 100)));
+                if (rarely()) {
+                    flush(index, randomBoolean());
+                }
+            }
+        } else {
+            ensureGreen(index);
+            flush(index, true);
+            assertEmptyTranslog(index);
+            ensurePeerRecoveryRetentionLeasesRenewedAndSynced(index);
+        }
+    }
+
+    public void testResize() throws Exception {
+        int numDocs;
+        if (isRunningAgainstOldCluster()) {
+            final Settings.Builder settings = Settings.builder()
+                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 3)
+                .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1);
+            if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
+                settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false);
+            }
+            final String mappings = randomBoolean() ? "\"_source\": { \"enabled\": false}" : null;
+            createIndex(index, settings.build(), mappings);
+            numDocs = randomIntBetween(10, 1000);
+            for (int i = 0; i < numDocs; i++) {
+                indexDocument(Integer.toString(i));
+                if (rarely()) {
+                    flush(index, randomBoolean());
+                }
+            }
+            saveInfoDocument("num_doc_" + index, Integer.toString(numDocs));
+            ensureGreen(index);
+        } else {
+            ensureGreen(index);
+            numDocs = Integer.parseInt(loadInfoDocument("num_doc_" + index));
+            int moreDocs = randomIntBetween(0, 100);
+            for (int i = 0; i < moreDocs; i++) {
+                indexDocument(Integer.toString(numDocs + i));
+                if (rarely()) {
+                    flush(index, randomBoolean());
+                }
+            }
+            Request updateSettingsRequest = new Request("PUT", "/" + index + "/_settings");
+            updateSettingsRequest.setJsonEntity("{\"settings\": {\"index.blocks.write\": true}}");
+            client().performRequest(updateSettingsRequest);
+            {
+                final String target = index + "_shrunken";
+                Request shrinkRequest = new Request("PUT", "/" + index + "/_shrink/" + target);
+                Settings.Builder settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1);
+                if (randomBoolean()) {
+                    settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
+                }
+                shrinkRequest.setJsonEntity("{\"settings\":" + Strings.toString(settings.build()) + "}");
+                client().performRequest(shrinkRequest);
+                ensureGreenLongWait(target);
+                assertNumHits(target, numDocs + moreDocs, 1);
+            }
+            {
+                final String target = index + "_split";
+                Settings.Builder settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 6);
+                if (randomBoolean()) {
+                    settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
+                }
+                Request splitRequest = new Request("PUT", "/" + index + "/_split/" + target);
+                splitRequest.setJsonEntity("{\"settings\":" + Strings.toString(settings.build()) + "}");
+                client().performRequest(splitRequest);
+                ensureGreenLongWait(target);
+                assertNumHits(target, numDocs + moreDocs, 6);
+            }
+            {
+                final String target = index + "_cloned";
+                client().performRequest(new Request("PUT", "/" + index + "/_clone/" + target));
+                ensureGreenLongWait(target);
+                assertNumHits(target, numDocs + moreDocs, 3);
+            }
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testSystemIndexMetadataIsUpgraded() throws Exception {
+        assumeTrue(".tasks became a system index in 7.10.0", getOldClusterVersion().onOrAfter(Version.V_7_10_0));
+        final String systemIndexWarning = "this request accesses system indices: [.tasks], but in a future major version, direct "
+            + "access to system indices will be prevented by default";
+        if (isRunningAgainstOldCluster()) {
+            // create index
+            Request createTestIndex = new Request("PUT", "/test_index_old");
+            createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0}}");
+            client().performRequest(createTestIndex);
+
+            Request bulk = new Request("POST", "/_bulk");
+            bulk.addParameter("refresh", "true");
+            bulk.setJsonEntity("""
+                {"index": {"_index": "test_index_old"}}
+                {"f1": "v1", "f2": "v2"}
+                """);
+            client().performRequest(bulk);
+
+            // start a async reindex job
+            Request reindex = new Request("POST", "/_reindex");
+            reindex.setJsonEntity("""
+                {
+                  "source":{
+                    "index":"test_index_old"
+                  },
+                  "dest":{
+                    "index":"test_index_reindex"
+                  }
+                }""");
+            reindex.addParameter("wait_for_completion", "false");
+            Map<String, Object> response = entityAsMap(client().performRequest(reindex));
+            String taskId = (String) response.get("task");
+
+            // wait for task
+            Request getTask = new Request("GET", "/_tasks/" + taskId);
+            getTask.addParameter("wait_for_completion", "true");
+            client().performRequest(getTask);
+
+            // make sure .tasks index exists
+            Request getTasksIndex = new Request("GET", "/.tasks");
+            getTasksIndex.setOptions(expectVersionSpecificWarnings(v -> {
+                v.current(systemIndexWarning);
+                v.compatible(systemIndexWarning);
+            }));
+            getTasksIndex.addParameter("allow_no_indices", "false");
+
+            getTasksIndex.setOptions(expectVersionSpecificWarnings(v -> {
+                v.current(systemIndexWarning);
+                v.compatible(systemIndexWarning);
+            }));
+            assertBusy(() -> {
+                try {
+                    assertThat(client().performRequest(getTasksIndex).getStatusLine().getStatusCode(), is(200));
+                } catch (ResponseException e) {
+                    throw new AssertionError(".tasks index does not exist yet");
+                }
+            });
+
+            // If we are on 7.x create an alias that includes both a system index and a non-system index so we can be sure it gets
+            // upgraded properly. If we're already on 8.x, skip this part of the test.
+            if (minimumNodeVersion().before(SYSTEM_INDEX_ENFORCEMENT_VERSION)) {
+                // Create an alias to make sure it gets upgraded properly
+                Request putAliasRequest = new Request("POST", "/_aliases");
+                putAliasRequest.setJsonEntity("""
+                    {
+                      "actions": [
+                        {"add":  {"index":  ".tasks", "alias": "test-system-alias"}},
+                        {"add":  {"index":  "test_index_reindex", "alias": "test-system-alias"}}
+                      ]
+                    }""");
+                putAliasRequest.setOptions(expectVersionSpecificWarnings(v -> {
+                    v.current(systemIndexWarning);
+                    v.compatible(systemIndexWarning);
+                }));
+                assertThat(client().performRequest(putAliasRequest).getStatusLine().getStatusCode(), is(200));
+            }
+        } else {
+            assertBusy(() -> {
+                Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata");
+                Map<String, Object> indices = new XContentTestUtils.JsonMapView(entityAsMap(client().performRequest(clusterStateRequest)))
+                    .get("metadata.indices");
+
+                // Make sure our non-system index is still non-system
+                assertThat(new XContentTestUtils.JsonMapView(indices).get("test_index_old.system"), is(false));
+
+                // Can't get the .tasks index via JsonMapView because it splits on `.`
+                assertThat(indices, hasKey(".tasks"));
+                XContentTestUtils.JsonMapView tasksIndex = new XContentTestUtils.JsonMapView((Map<String, Object>) indices.get(".tasks"));
+                assertThat(tasksIndex.get("system"), is(true));
+
+                // If .tasks was created in a 7.x version, it should have an alias on it that we need to make sure got upgraded properly.
+                final String tasksCreatedVersionString = tasksIndex.get("settings.index.version.created");
+                assertThat(tasksCreatedVersionString, notNullValue());
+                final Version tasksCreatedVersion = Version.fromId(Integer.parseInt(tasksCreatedVersionString));
+                if (tasksCreatedVersion.before(SYSTEM_INDEX_ENFORCEMENT_VERSION)) {
+                    // Verify that the alias survived the upgrade
+                    Request getAliasRequest = new Request("GET", "/_alias/test-system-alias");
+                    getAliasRequest.setOptions(expectVersionSpecificWarnings(v -> {
+                        v.current(systemIndexWarning);
+                        v.compatible(systemIndexWarning);
+                    }));
+                    Map<String, Object> aliasResponse = entityAsMap(client().performRequest(getAliasRequest));
+                    assertThat(aliasResponse, hasKey(".tasks"));
+                    assertThat(aliasResponse, hasKey("test_index_reindex"));
+                }
+            });
+        }
+    }
+
+    public void testEnableSoftDeletesOnRestore() throws Exception {
+        assumeTrue("soft deletes must be enabled on 8.0+", getOldClusterVersion().before(Version.V_8_0_0));
+        final String snapshot = "snapshot-" + index;
+        if (isRunningAgainstOldCluster()) {
+            final Settings.Builder settings = Settings.builder()
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1);
+            settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false);
+            createIndex(index, settings.build());
+            ensureGreen(index);
+            int numDocs = randomIntBetween(0, 100);
+            indexRandomDocuments(
+                numDocs,
+                true,
+                true,
+                randomBoolean(),
+                i -> jsonBuilder().startObject().field("field", "value").endObject()
+            );
+            // create repo
+            XContentBuilder repoConfig = JsonXContent.contentBuilder().startObject();
+            {
+                repoConfig.field("type", "fs");
+                repoConfig.startObject("settings");
+                {
+                    repoConfig.field("compress", randomBoolean());
+                    repoConfig.field("location", repoDirectory.getRoot().getPath());
+                }
+                repoConfig.endObject();
+            }
+            repoConfig.endObject();
+            Request createRepoRequest = new Request("PUT", "/_snapshot/repo");
+            createRepoRequest.setJsonEntity(Strings.toString(repoConfig));
+            client().performRequest(createRepoRequest);
+            // create snapshot
+            Request createSnapshot = new Request("PUT", "/_snapshot/repo/" + snapshot);
+            createSnapshot.addParameter("wait_for_completion", "true");
+            createSnapshot.setJsonEntity("{\"indices\": \"" + index + "\"}");
+            client().performRequest(createSnapshot);
+        } else {
+            String restoredIndex = "restored-" + index;
+            // Restore
+            XContentBuilder restoreCommand = JsonXContent.contentBuilder().startObject();
+            restoreCommand.field("indices", index);
+            restoreCommand.field("rename_pattern", index);
+            restoreCommand.field("rename_replacement", restoredIndex);
+            restoreCommand.startObject("index_settings");
+            {
+                restoreCommand.field("index.soft_deletes.enabled", true);
+            }
+            restoreCommand.endObject();
+            restoreCommand.endObject();
+            Request restoreRequest = new Request("POST", "/_snapshot/repo/" + snapshot + "/_restore");
+            restoreRequest.addParameter("wait_for_completion", "true");
+            restoreRequest.setJsonEntity(Strings.toString(restoreCommand));
+            client().performRequest(restoreRequest);
+            ensureGreen(restoredIndex);
+            int numDocs = countOfIndexedRandomDocuments();
+            assertTotalHits(numDocs, entityAsMap(client().performRequest(new Request("GET", "/" + restoredIndex + "/_search"))));
+        }
+    }
+
+    public void testForbidDisableSoftDeletesOnRestore() throws Exception {
+        final String snapshot = "snapshot-" + index;
+        if (isRunningAgainstOldCluster()) {
+            final Settings.Builder settings = Settings.builder()
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+                .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
+            createIndex(index, settings.build());
+            ensureGreen(index);
+            int numDocs = randomIntBetween(0, 100);
+            indexRandomDocuments(
+                numDocs,
+                true,
+                true,
+                randomBoolean(),
+                i -> jsonBuilder().startObject().field("field", "value").endObject()
+            );
+            // create repo
+            XContentBuilder repoConfig = JsonXContent.contentBuilder().startObject();
+            {
+                repoConfig.field("type", "fs");
+                repoConfig.startObject("settings");
+                {
+                    repoConfig.field("compress", randomBoolean());
+                    repoConfig.field("location", repoDirectory.getRoot().getPath());
+                }
+                repoConfig.endObject();
+            }
+            repoConfig.endObject();
+            Request createRepoRequest = new Request("PUT", "/_snapshot/repo");
+            createRepoRequest.setJsonEntity(Strings.toString(repoConfig));
+            client().performRequest(createRepoRequest);
+            // create snapshot
+            Request createSnapshot = new Request("PUT", "/_snapshot/repo/" + snapshot);
+            createSnapshot.addParameter("wait_for_completion", "true");
+            createSnapshot.setJsonEntity("{\"indices\": \"" + index + "\"}");
+            client().performRequest(createSnapshot);
+        } else {
+            // Restore
+            XContentBuilder restoreCommand = JsonXContent.contentBuilder().startObject();
+            restoreCommand.field("indices", index);
+            restoreCommand.field("rename_pattern", index);
+            restoreCommand.field("rename_replacement", "restored-" + index);
+            restoreCommand.startObject("index_settings");
+            {
+                restoreCommand.field("index.soft_deletes.enabled", false);
+            }
+            restoreCommand.endObject();
+            restoreCommand.endObject();
+            Request restoreRequest = new Request("POST", "/_snapshot/repo/" + snapshot + "/_restore");
+            restoreRequest.addParameter("wait_for_completion", "true");
+            restoreRequest.setJsonEntity(Strings.toString(restoreCommand));
+            final ResponseException error = expectThrows(ResponseException.class, () -> client().performRequest(restoreRequest));
+            assertThat(error.getMessage(), containsString("cannot disable setting [index.soft_deletes.enabled] on restore"));
+        }
+    }
+
+    /**
+     * In 7.14 the cluster.remote.*.transport.compress setting was change from a boolean to an enum setting
+     * with true/false as options. This test ensures that the old boolean setting in cluster state is
+     * translated properly. This test can be removed in 9.0.
+     */
+    public void testTransportCompressionSetting() throws IOException {
+        assumeTrue("the old transport.compress setting existed before 7.14", getOldClusterVersion().before(Version.V_7_14_0));
+        assumeTrue(
+            "Early versions of 6.x do not have cluster.remote* prefixed settings",
+            getOldClusterVersion().onOrAfter(Version.V_7_14_0.minimumCompatibilityVersion())
+        );
+        if (isRunningAgainstOldCluster()) {
+            final Request putSettingsRequest = new Request("PUT", "/_cluster/settings");
+            try (XContentBuilder builder = jsonBuilder()) {
+                builder.startObject();
+                {
+                    builder.startObject("persistent");
+                    {
+                        builder.field("cluster.remote.foo.seeds", Collections.singletonList("localhost:9200"));
+                        builder.field("cluster.remote.foo.transport.compress", "true");
+                    }
+                    builder.endObject();
+                }
+                builder.endObject();
+                putSettingsRequest.setJsonEntity(Strings.toString(builder));
+            }
+            client().performRequest(putSettingsRequest);
+        } else {
+            final Request getSettingsRequest = new Request("GET", "/_cluster/settings");
+            final Response getSettingsResponse = client().performRequest(getSettingsRequest);
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, getSettingsResponse.getEntity().getContent())) {
+                final Settings settings = RestClusterGetSettingsResponse.fromXContent(parser).getPersistentSettings();
+                assertThat(REMOTE_CLUSTER_COMPRESS.getConcreteSettingForNamespace("foo").get(settings), equalTo(Compression.Enabled.TRUE));
+            }
+        }
+    }
+
+    public static void assertNumHits(String index, int numHits, int totalShards) throws IOException {
+        Map<String, Object> resp = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")));
+        assertNoFailures(resp);
+        assertThat(XContentMapValues.extractValue("_shards.total", resp), equalTo(totalShards));
+        assertThat(XContentMapValues.extractValue("_shards.successful", resp), equalTo(totalShards));
+        assertThat(extractTotalHits(resp), equalTo(numHits));
+    }
+}
diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartTestOrdering.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartTestOrdering.java
new file mode 100644
index 000000000000..232619ee93bb
--- /dev/null
+++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartTestOrdering.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import com.carrotsearch.randomizedtesting.TestMethodAndParams;
+
+import java.util.Comparator;
+
+public class FullClusterRestartTestOrdering implements Comparator<TestMethodAndParams> {
+    @Override
+    public int compare(TestMethodAndParams o1, TestMethodAndParams o2) {
+        return Integer.compare(getOrdinal(o1), getOrdinal(o2));
+    }
+
+    private int getOrdinal(TestMethodAndParams t) {
+        return ((FullClusterRestartUpgradeStatus) t.getInstanceArguments().get(0)).ordinal();
+    }
+}
diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartUpgradeStatus.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartUpgradeStatus.java
new file mode 100644
index 000000000000..06048d020e2a
--- /dev/null
+++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartUpgradeStatus.java
@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+public enum FullClusterRestartUpgradeStatus {
+    OLD,
+    UPGRADED
+}
diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedFullClusterRestartTestCase.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedFullClusterRestartTestCase.java
new file mode 100644
index 000000000000..a064c8774380
--- /dev/null
+++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedFullClusterRestartTestCase.java
@@ -0,0 +1,100 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.util.Version;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.junit.AfterClass;
+import org.junit.Before;
+
+import java.util.Arrays;
+import java.util.Locale;
+
+import static org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus.OLD;
+import static org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus.UPGRADED;
+
+@TestCaseOrdering(FullClusterRestartTestOrdering.class)
+public abstract class ParameterizedFullClusterRestartTestCase extends ESRestTestCase {
+    private static final Version MINIMUM_WIRE_COMPATIBLE_VERSION = Version.fromString("7.17.0");
+    private static final Version OLD_CLUSTER_VERSION = Version.fromString(System.getProperty("tests.old_cluster_version"));
+    private static boolean upgradeFailed = false;
+    private static boolean upgraded = false;
+    private final FullClusterRestartUpgradeStatus requestedUpgradeStatus;
+
+    public ParameterizedFullClusterRestartTestCase(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) {
+        this.requestedUpgradeStatus = upgradeStatus;
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return Arrays.stream(FullClusterRestartUpgradeStatus.values()).map(v -> new Object[] { v }).toList();
+    }
+
+    @Before
+    public void maybeUpgrade() throws Exception {
+        if (upgraded == false && requestedUpgradeStatus == UPGRADED) {
+            try {
+                if (OLD_CLUSTER_VERSION.before(MINIMUM_WIRE_COMPATIBLE_VERSION)) {
+                    // First upgrade to latest wire compatible version
+                    getUpgradeCluster().upgradeToVersion(MINIMUM_WIRE_COMPATIBLE_VERSION);
+                }
+                getUpgradeCluster().upgradeToVersion(Version.CURRENT);
+                closeClients();
+                initClient();
+            } catch (Exception e) {
+                upgradeFailed = true;
+                throw e;
+            } finally {
+                upgraded = true;
+            }
+        }
+
+        // Skip remaining tests if upgrade failed
+        assumeFalse("Cluster upgrade failed", upgradeFailed);
+    }
+
+    @AfterClass
+    public static void resetUpgrade() {
+        upgraded = false;
+        upgradeFailed = false;
+    }
+
+    public boolean isRunningAgainstOldCluster() {
+        return requestedUpgradeStatus == OLD;
+    }
+
+    public static org.elasticsearch.Version getOldClusterVersion() {
+        return org.elasticsearch.Version.fromString(OLD_CLUSTER_VERSION.toString());
+    }
+
+    public static Version getOldClusterTestVersion() {
+        return Version.fromString(OLD_CLUSTER_VERSION.toString());
+    }
+
+    protected abstract ElasticsearchCluster getUpgradeCluster();
+
+    @Override
+    protected String getTestRestCluster() {
+        return getUpgradeCluster().getHttpAddresses();
+    }
+
+    @Override
+    protected boolean preserveClusterUponCompletion() {
+        return true;
+    }
+
+    protected String getRootTestName() {
+        return getTestName().split(" ")[0].toLowerCase(Locale.ROOT);
+    }
+}
diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java
new file mode 100644
index 000000000000..1636644409fc
--- /dev/null
+++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java
@@ -0,0 +1,256 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.InputStreamStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
+import org.elasticsearch.index.query.DisMaxQueryBuilder;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.MatchNoneQueryBuilder;
+import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
+import org.elasticsearch.index.query.MatchQueryBuilder;
+import org.elasticsearch.index.query.Operator;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.RangeQueryBuilder;
+import org.elasticsearch.index.query.SpanNearQueryBuilder;
+import org.elasticsearch.index.query.SpanTermQueryBuilder;
+import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
+import org.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.junit.ClassRule;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
+
+/**
+ * An integration test that tests whether percolator queries stored in older supported ES version can still be read by the
+ * current ES version. Percolator queries are stored in the binary format in a dedicated doc values field (see
+ * PercolatorFieldMapper#createQueryBuilderField(...) method). Using the query builders writable contract. This test
+ * does best effort verifying that we don't break bwc for query builders between the first previous major version and
+ * the latest current major release.
+ *
+ * The queries to test are specified in json format, which turns out to work because we tend break here rarely. If the
+ * json format of a query being tested here then feel free to change this.
+ */
+public class QueryBuilderBWCIT extends ParameterizedFullClusterRestartTestCase {
+    private static final List<Object[]> CANDIDATES = new ArrayList<>();
+
+    protected static LocalClusterConfigProvider clusterConfig = c -> {};
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .version(getOldClusterTestVersion())
+        .nodes(2)
+        .setting("xpack.security.enabled", "false")
+        .apply(() -> clusterConfig)
+        .build();
+
+    @Override
+    protected ElasticsearchCluster getUpgradeCluster() {
+        return cluster;
+    }
+
+    public QueryBuilderBWCIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) {
+        super(upgradeStatus);
+    }
+
+    static {
+        addCandidate("""
+            "match": { "text_field": "value"}
+            """, new MatchQueryBuilder("text_field", "value"));
+        addCandidate("""
+            "match": { "text_field": {"query": "value", "operator": "and"} }
+            """, new MatchQueryBuilder("text_field", "value").operator(Operator.AND));
+        addCandidate("""
+            "match": { "text_field": {"query": "value", "analyzer": "english"} }
+            """, new MatchQueryBuilder("text_field", "value").analyzer("english"));
+        addCandidate("""
+            "match": { "text_field": {"query": "value", "minimum_should_match": 3} }
+            """, new MatchQueryBuilder("text_field", "value").minimumShouldMatch("3"));
+        addCandidate("""
+            "match": { "text_field": {"query": "value", "fuzziness": "auto"} }
+            """, new MatchQueryBuilder("text_field", "value").fuzziness(Fuzziness.AUTO));
+        addCandidate("""
+            "match_phrase": { "text_field": "value"}
+            """, new MatchPhraseQueryBuilder("text_field", "value"));
+        addCandidate("""
+            "match_phrase": { "text_field": {"query": "value", "slop": 3}}
+            """, new MatchPhraseQueryBuilder("text_field", "value").slop(3));
+        addCandidate("""
+            "range": { "long_field": {"gte": 1, "lte": 9}}
+            """, new RangeQueryBuilder("long_field").from(1).to(9));
+        addCandidate(
+            """
+                "bool": { "must_not": [{"match_none": {}}], "must": [{"match_all": {}}], "filter": [{"match_all": {}}], \
+                "should": [{"match_all": {}}]}
+                """,
+            new BoolQueryBuilder().mustNot(new MatchNoneQueryBuilder())
+                .must(new MatchAllQueryBuilder())
+                .filter(new MatchAllQueryBuilder())
+                .should(new MatchAllQueryBuilder())
+        );
+        addCandidate(
+            """
+                "dis_max": {"queries": [{"match_all": {}},{"match_all": {}},{"match_all": {}}], "tie_breaker": 0.01}
+                """,
+            new DisMaxQueryBuilder().add(new MatchAllQueryBuilder())
+                .add(new MatchAllQueryBuilder())
+                .add(new MatchAllQueryBuilder())
+                .tieBreaker(0.01f)
+        );
+        addCandidate("""
+            "constant_score": {"filter": {"match_all": {}}, "boost": 0.1}
+            """, new ConstantScoreQueryBuilder(new MatchAllQueryBuilder()).boost(0.1f));
+        addCandidate(
+            """
+                "function_score": {"query": {"match_all": {}},"functions": [{"random_score": {}, "filter": {"match_all": {}}, \
+                "weight": 0.2}]}
+                """,
+            new FunctionScoreQueryBuilder(
+                new MatchAllQueryBuilder(),
+                new FunctionScoreQueryBuilder.FilterFunctionBuilder[] {
+                    new FunctionScoreQueryBuilder.FilterFunctionBuilder(
+                        new MatchAllQueryBuilder(),
+                        new RandomScoreFunctionBuilder().setWeight(0.2f)
+                    ) }
+            )
+        );
+        addCandidate(
+            """
+                "span_near": {"clauses": [{ "span_term": { "keyword_field": "value1" }}, \
+                { "span_term": { "keyword_field": "value2" }}]}
+                """,
+            new SpanNearQueryBuilder(new SpanTermQueryBuilder("keyword_field", "value1"), 0).addClause(
+                new SpanTermQueryBuilder("keyword_field", "value2")
+            )
+        );
+        addCandidate(
+            """
+                "span_near": {"clauses": [{ "span_term": { "keyword_field": "value1" }}, \
+                { "span_term": { "keyword_field": "value2" }}], "slop": 2}
+                """,
+            new SpanNearQueryBuilder(new SpanTermQueryBuilder("keyword_field", "value1"), 2).addClause(
+                new SpanTermQueryBuilder("keyword_field", "value2")
+            )
+        );
+        addCandidate(
+            """
+                "span_near": {"clauses": [{ "span_term": { "keyword_field": "value1" }}, \
+                { "span_term": { "keyword_field": "value2" }}], "slop": 2, "in_order": false}
+                """,
+            new SpanNearQueryBuilder(new SpanTermQueryBuilder("keyword_field", "value1"), 2).addClause(
+                new SpanTermQueryBuilder("keyword_field", "value2")
+            ).inOrder(false)
+        );
+    }
+
+    private static void addCandidate(String querySource, QueryBuilder expectedQb) {
+        CANDIDATES.add(new Object[] { "{\"query\": {" + querySource + "}}", expectedQb });
+    }
+
+    public void testQueryBuilderBWC() throws Exception {
+        String index = "queries";
+        if (isRunningAgainstOldCluster()) {
+            XContentBuilder mappingsAndSettings = jsonBuilder();
+            mappingsAndSettings.startObject();
+            {
+                mappingsAndSettings.startObject("settings");
+                mappingsAndSettings.field("number_of_shards", 1);
+                mappingsAndSettings.field("number_of_replicas", 0);
+                mappingsAndSettings.endObject();
+            }
+            {
+                mappingsAndSettings.startObject("mappings");
+                mappingsAndSettings.startObject("properties");
+                {
+                    mappingsAndSettings.startObject("query");
+                    mappingsAndSettings.field("type", "percolator");
+                    mappingsAndSettings.endObject();
+                }
+                {
+                    mappingsAndSettings.startObject("keyword_field");
+                    mappingsAndSettings.field("type", "keyword");
+                    mappingsAndSettings.endObject();
+                }
+                {
+                    mappingsAndSettings.startObject("text_field");
+                    mappingsAndSettings.field("type", "text");
+                    mappingsAndSettings.endObject();
+                }
+                {
+                    mappingsAndSettings.startObject("long_field");
+                    mappingsAndSettings.field("type", "long");
+                    mappingsAndSettings.endObject();
+                }
+                mappingsAndSettings.endObject();
+                mappingsAndSettings.endObject();
+            }
+            mappingsAndSettings.endObject();
+            Request request = new Request("PUT", "/" + index);
+            request.setJsonEntity(Strings.toString(mappingsAndSettings));
+            Response rsp = client().performRequest(request);
+            assertEquals(200, rsp.getStatusLine().getStatusCode());
+
+            for (int i = 0; i < CANDIDATES.size(); i++) {
+                request = new Request("PUT", "/" + index + "/_doc/" + Integer.toString(i));
+                request.setJsonEntity((String) CANDIDATES.get(i)[0]);
+                rsp = client().performRequest(request);
+                assertEquals(201, rsp.getStatusLine().getStatusCode());
+            }
+        } else {
+            NamedWriteableRegistry registry = new NamedWriteableRegistry(
+                new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedWriteables()
+            );
+
+            for (int i = 0; i < CANDIDATES.size(); i++) {
+                QueryBuilder expectedQueryBuilder = (QueryBuilder) CANDIDATES.get(i)[1];
+                Request request = new Request("GET", "/" + index + "/_search");
+                request.setJsonEntity(Strings.format("""
+                    {"query": {"ids": {"values": ["%s"]}}, "docvalue_fields": [{"field":"query.query_builder_field"}]}
+                    """, i));
+                Response rsp = client().performRequest(request);
+                assertEquals(200, rsp.getStatusLine().getStatusCode());
+                var hitRsp = (Map<?, ?>) ((List<?>) ((Map<?, ?>) responseAsMap(rsp).get("hits")).get("hits")).get(0);
+                String queryBuilderStr = (String) ((List<?>) ((Map<?, ?>) hitRsp.get("fields")).get("query.query_builder_field")).get(0);
+                byte[] qbSource = Base64.getDecoder().decode(queryBuilderStr);
+                try (InputStream in = new ByteArrayInputStream(qbSource, 0, qbSource.length)) {
+                    try (StreamInput input = new NamedWriteableAwareStreamInput(new InputStreamStreamInput(in), registry)) {
+                        input.setVersion(getOldClusterVersion());
+                        QueryBuilder queryBuilder = input.readNamedWriteable(QueryBuilder.class);
+                        assert in.read() == -1;
+                        assertEquals(expectedQueryBuilder, queryBuilder);
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
deleted file mode 100644
index 7e8b88131814..000000000000
--- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
+++ /dev/null
@@ -1,1865 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.upgrades;
-
-import org.apache.http.util.EntityUtils;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.cluster.settings.RestClusterGetSettingsResponse;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.client.ResponseException;
-import org.elasticsearch.client.RestClient;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.cluster.metadata.MetadataIndexStateService;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.time.DateUtils;
-import org.elasticsearch.common.xcontent.support.XContentMapValues;
-import org.elasticsearch.core.Booleans;
-import org.elasticsearch.core.CheckedFunction;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.mapper.DateFieldMapper;
-import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction;
-import org.elasticsearch.test.NotEqualMessageBuilder;
-import org.elasticsearch.test.XContentTestUtils;
-import org.elasticsearch.test.rest.ESRestTestCase;
-import org.elasticsearch.test.rest.ObjectPath;
-import org.elasticsearch.transport.Compression;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xcontent.json.JsonXContent;
-import org.junit.Before;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Base64;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.IntStream;
-
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.singletonList;
-import static java.util.Collections.singletonMap;
-import static java.util.stream.Collectors.toList;
-import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.SYSTEM_INDEX_ENFORCEMENT_VERSION;
-import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING;
-import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY;
-import static org.elasticsearch.test.MapMatcher.assertMap;
-import static org.elasticsearch.test.MapMatcher.matchesMap;
-import static org.elasticsearch.transport.RemoteClusterService.REMOTE_CLUSTER_COMPRESS;
-import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.hasKey;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-
-/**
- * Tests to run before and after a full cluster restart. This is run twice,
- * one with {@code tests.is_old_cluster} set to {@code true} against a cluster
- * of an older version. The cluster is shutdown and a cluster of the new
- * version is started with the same data directories and then this is rerun
- * with {@code tests.is_old_cluster} set to {@code false}.
- */
-public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
-
-    private String index;
-
-    @Before
-    public void setIndex() {
-        index = getTestName().toLowerCase(Locale.ROOT);
-    }
-
-    public void testSearch() throws Exception {
-        int count;
-        if (isRunningAgainstOldCluster()) {
-            XContentBuilder mappingsAndSettings = jsonBuilder();
-            mappingsAndSettings.startObject();
-            {
-                mappingsAndSettings.startObject("settings");
-                mappingsAndSettings.field("number_of_shards", 1);
-                mappingsAndSettings.field("number_of_replicas", 0);
-                mappingsAndSettings.endObject();
-            }
-            {
-                mappingsAndSettings.startObject("mappings");
-                mappingsAndSettings.startObject("properties");
-                {
-                    mappingsAndSettings.startObject("string");
-                    mappingsAndSettings.field("type", "text");
-                    mappingsAndSettings.endObject();
-                }
-                {
-                    mappingsAndSettings.startObject("dots_in_field_names");
-                    mappingsAndSettings.field("type", "text");
-                    mappingsAndSettings.endObject();
-                }
-                {
-                    mappingsAndSettings.startObject("binary");
-                    mappingsAndSettings.field("type", "binary");
-                    mappingsAndSettings.field("store", "true");
-                    mappingsAndSettings.endObject();
-                }
-                mappingsAndSettings.endObject();
-                mappingsAndSettings.endObject();
-            }
-            mappingsAndSettings.endObject();
-
-            Request createIndex = new Request("PUT", "/" + index);
-            createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
-            client().performRequest(createIndex);
-
-            count = randomIntBetween(2000, 3000);
-            byte[] randomByteArray = new byte[16];
-            random().nextBytes(randomByteArray);
-            indexRandomDocuments(
-                count,
-                true,
-                true,
-                randomBoolean(),
-                i -> JsonXContent.contentBuilder()
-                    .startObject()
-                    .field("string", randomAlphaOfLength(10))
-                    .field("int", randomInt(100))
-                    .field("float", randomFloat())
-                    // be sure to create a "proper" boolean (True, False) for the first document so that automapping is correct
-                    .field("bool", i > 0 && randomBoolean())
-                    .field("field.with.dots", randomAlphaOfLength(10))
-                    .field("binary", Base64.getEncoder().encodeToString(randomByteArray))
-                    .endObject()
-            );
-            refreshAllIndices();
-        } else {
-            count = countOfIndexedRandomDocuments();
-        }
-
-        ensureGreenLongWait(index);
-        assertBasicSearchWorks(count);
-        assertAllSearchWorks(count);
-        assertBasicAggregationWorks();
-        assertRealtimeGetWorks();
-        assertStoredBinaryFields(count);
-    }
-
-    public void testNewReplicas() throws Exception {
-        if (isRunningAgainstOldCluster()) {
-            XContentBuilder mappingsAndSettings = jsonBuilder();
-            mappingsAndSettings.startObject();
-            {
-                mappingsAndSettings.startObject("settings");
-                mappingsAndSettings.field("number_of_shards", 1);
-                mappingsAndSettings.field("number_of_replicas", 0);
-                mappingsAndSettings.endObject();
-            }
-            {
-                mappingsAndSettings.startObject("mappings");
-                mappingsAndSettings.startObject("properties");
-                {
-                    mappingsAndSettings.startObject("field");
-                    mappingsAndSettings.field("type", "text");
-                    mappingsAndSettings.endObject();
-                }
-                mappingsAndSettings.endObject();
-                mappingsAndSettings.endObject();
-            }
-            mappingsAndSettings.endObject();
-
-            Request createIndex = new Request("PUT", "/" + index);
-            createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
-            client().performRequest(createIndex);
-
-            int numDocs = randomIntBetween(2000, 3000);
-            indexRandomDocuments(
-                numDocs,
-                true,
-                false,
-                randomBoolean(),
-                i -> JsonXContent.contentBuilder().startObject().field("field", "value").endObject()
-            );
-            logger.info("Refreshing [{}]", index);
-            client().performRequest(new Request("POST", "/" + index + "/_refresh"));
-        } else {
-            // The test runs with two nodes so this should still go green.
-            final int numReplicas = 1;
-            final long startTime = System.currentTimeMillis();
-            logger.debug("--> creating [{}] replicas for index [{}]", numReplicas, index);
-            Request setNumberOfReplicas = new Request("PUT", "/" + index + "/_settings");
-            setNumberOfReplicas.setJsonEntity("{ \"index\": { \"number_of_replicas\" : " + numReplicas + " }}");
-            client().performRequest(setNumberOfReplicas);
-
-            ensureGreenLongWait(index);
-
-            logger.debug("--> index [{}] is green, took [{}] ms", index, (System.currentTimeMillis() - startTime));
-            Map<String, Object> recoverRsp = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_recovery")));
-            logger.debug("--> recovery status:\n{}", recoverRsp);
-
-            Set<Integer> counts = new HashSet<>();
-            for (String node : dataNodes(index, client())) {
-                Request search = new Request("GET", "/" + index + "/_search");
-                search.addParameter("preference", "_only_nodes:" + node);
-                Map<String, Object> responseBody = entityAsMap(client().performRequest(search));
-                assertNoFailures(responseBody);
-                int hits = extractTotalHits(responseBody);
-                counts.add(hits);
-            }
-            assertEquals("All nodes should have a consistent number of documents", 1, counts.size());
-        }
-    }
-
-    public void testSearchTimeSeriesMode() throws Exception {
-        assumeTrue("indexing time series indices changed in 8.2.0", getOldClusterVersion().onOrAfter(Version.V_8_2_0));
-        int numDocs;
-        if (isRunningAgainstOldCluster()) {
-            numDocs = createTimeSeriesModeIndex(1);
-        } else {
-            numDocs = countOfIndexedRandomDocuments();
-        }
-        assertCountAll(numDocs);
-        Request request = new Request("GET", "/" + index + "/_search");
-        XContentBuilder body = jsonBuilder().startObject();
-        body.field("size", 0);
-        body.startObject("aggs").startObject("check").startObject("scripted_metric");
-        {
-            body.field("init_script", "state.timeSeries = new HashSet()");
-            body.field("map_script", "state.timeSeries.add(doc['dim'].value)");
-            body.field("combine_script", "return state.timeSeries");
-            StringBuilder reduceScript = new StringBuilder();
-            reduceScript.append("Set timeSeries = new TreeSet();");
-            reduceScript.append("for (s in states) {");
-            reduceScript.append("  for (ts in s) {");
-            reduceScript.append("    boolean newTs = timeSeries.add(ts);");
-            reduceScript.append("    if (false == newTs) {");
-            reduceScript.append("      throw new IllegalArgumentException(ts + ' appeared in two shards');");
-            reduceScript.append("    }");
-            reduceScript.append("  }");
-            reduceScript.append("}");
-            reduceScript.append("return timeSeries;");
-            body.field("reduce_script", reduceScript.toString());
-        }
-        body.endObject().endObject().endObject();
-        body.endObject();
-        request.setJsonEntity(Strings.toString(body));
-        Map<String, Object> response = entityAsMap(client().performRequest(request));
-        assertMap(
-            response,
-            matchesMap().extraOk()
-                .entry("hits", matchesMap().extraOk().entry("total", Map.of("value", numDocs, "relation", "eq")))
-                .entry("aggregations", Map.of("check", Map.of("value", IntStream.range(0, 10).mapToObj(i -> "dim" + i).collect(toList()))))
-        );
-    }
-
-    public void testNewReplicasTimeSeriesMode() throws Exception {
-        assumeTrue("indexing time series indices changed in 8.2.0", getOldClusterVersion().onOrAfter(Version.V_8_2_0));
-        if (isRunningAgainstOldCluster()) {
-            createTimeSeriesModeIndex(0);
-        } else {
-            // The test runs with two nodes so this should still go green.
-            final int numReplicas = 1;
-            final long startTime = System.currentTimeMillis();
-            logger.debug("--> creating [{}] replicas for index [{}]", numReplicas, index);
-            Request setNumberOfReplicas = new Request("PUT", "/" + index + "/_settings");
-            setNumberOfReplicas.setJsonEntity("{ \"index\": { \"number_of_replicas\" : " + numReplicas + " }}");
-            client().performRequest(setNumberOfReplicas);
-
-            ensureGreenLongWait(index);
-
-            logger.debug("--> index [{}] is green, took [{}] ms", index, (System.currentTimeMillis() - startTime));
-            Map<String, Object> recoverRsp = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_recovery")));
-            logger.debug("--> recovery status:\n{}", recoverRsp);
-
-            Set<Integer> counts = new HashSet<>();
-            for (String node : dataNodes(index, client())) {
-                Request search = new Request("GET", "/" + index + "/_search");
-                search.addParameter("preference", "_only_nodes:" + node);
-                Map<String, Object> responseBody = entityAsMap(client().performRequest(search));
-                assertNoFailures(responseBody);
-                int hits = extractTotalHits(responseBody);
-                counts.add(hits);
-            }
-            assertEquals("All nodes should have a consistent number of documents", 1, counts.size());
-        }
-    }
-
-    private int createTimeSeriesModeIndex(int replicas) throws IOException {
-        XContentBuilder mappingsAndSettings = jsonBuilder();
-        mappingsAndSettings.startObject();
-        {
-            mappingsAndSettings.startObject("settings");
-            mappingsAndSettings.field("number_of_shards", 1);
-            mappingsAndSettings.field("number_of_replicas", replicas);
-            mappingsAndSettings.field("mode", "time_series");
-            mappingsAndSettings.field("routing_path", "dim");
-            mappingsAndSettings.field("time_series.start_time", 1L);
-            mappingsAndSettings.field("time_series.end_time", DateUtils.MAX_MILLIS_BEFORE_9999 - 1);
-            mappingsAndSettings.endObject();
-        }
-        {
-            mappingsAndSettings.startObject("mappings");
-            mappingsAndSettings.startObject("properties");
-            {
-                mappingsAndSettings.startObject("@timestamp").field("type", "date").endObject();
-                mappingsAndSettings.startObject("dim").field("type", "keyword").field("time_series_dimension", true).endObject();
-            }
-            mappingsAndSettings.endObject();
-            mappingsAndSettings.endObject();
-        }
-        mappingsAndSettings.endObject();
-
-        Request createIndex = new Request("PUT", "/" + index);
-        createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
-        client().performRequest(createIndex);
-
-        int numDocs = randomIntBetween(2000, 3000);
-        long basetime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2021-01-01T00:00:00Z");
-        indexRandomDocuments(
-            numDocs,
-            true,
-            true,
-            false,
-            i -> JsonXContent.contentBuilder()
-                .startObject()
-                .field("@timestamp", basetime + TimeUnit.MINUTES.toMillis(i))
-                .field("dim", "dim" + (i % 10))
-                .endObject()
-        );
-        logger.info("Refreshing [{}]", index);
-        client().performRequest(new Request("POST", "/" + index + "/_refresh"));
-        return numDocs;
-    }
-
-    public void testClusterState() throws Exception {
-        if (isRunningAgainstOldCluster()) {
-            XContentBuilder mappingsAndSettings = jsonBuilder();
-            mappingsAndSettings.startObject();
-            mappingsAndSettings.field("index_patterns", index);
-            mappingsAndSettings.field("order", "1000");
-            {
-                mappingsAndSettings.startObject("settings");
-                mappingsAndSettings.field("number_of_shards", 1);
-                mappingsAndSettings.field("number_of_replicas", 0);
-                mappingsAndSettings.endObject();
-            }
-            mappingsAndSettings.endObject();
-            Request createTemplate = new Request("PUT", "/_template/template_1");
-            createTemplate.setJsonEntity(Strings.toString(mappingsAndSettings));
-            createTemplate.setOptions(expectWarnings(RestPutIndexTemplateAction.DEPRECATION_WARNING));
-            client().performRequest(createTemplate);
-            client().performRequest(new Request("PUT", "/" + index));
-        }
-
-        // verifying if we can still read some properties from cluster state api:
-        Map<String, Object> clusterState = entityAsMap(client().performRequest(new Request("GET", "/_cluster/state")));
-
-        // Check some global properties:
-        String numberOfShards = (String) XContentMapValues.extractValue(
-            "metadata.templates.template_1.settings.index.number_of_shards",
-            clusterState
-        );
-        assertEquals("1", numberOfShards);
-        String numberOfReplicas = (String) XContentMapValues.extractValue(
-            "metadata.templates.template_1.settings.index.number_of_replicas",
-            clusterState
-        );
-        assertEquals("0", numberOfReplicas);
-
-        // Check some index properties:
-        numberOfShards = (String) XContentMapValues.extractValue(
-            "metadata.indices." + index + ".settings.index.number_of_shards",
-            clusterState
-        );
-        assertEquals("1", numberOfShards);
-        numberOfReplicas = (String) XContentMapValues.extractValue(
-            "metadata.indices." + index + ".settings.index.number_of_replicas",
-            clusterState
-        );
-        assertEquals("0", numberOfReplicas);
-        Version version = Version.fromId(
-            Integer.valueOf(
-                (String) XContentMapValues.extractValue("metadata.indices." + index + ".settings.index.version.created", clusterState)
-            )
-        );
-        assertEquals(getOldClusterVersion(), version);
-
-    }
-
-    public void testShrink() throws IOException {
-        String shrunkenIndex = index + "_shrunk";
-        int numDocs;
-        if (isRunningAgainstOldCluster()) {
-            XContentBuilder mappingsAndSettings = jsonBuilder();
-            mappingsAndSettings.startObject();
-            {
-                mappingsAndSettings.startObject("mappings");
-                {
-                    mappingsAndSettings.startObject("properties");
-                    {
-                        mappingsAndSettings.startObject("field");
-                        {
-                            mappingsAndSettings.field("type", "text");
-                        }
-                        mappingsAndSettings.endObject();
-                    }
-                    mappingsAndSettings.endObject();
-                }
-                mappingsAndSettings.endObject();
-
-                mappingsAndSettings.startObject("settings");
-                {
-                    mappingsAndSettings.field("index.number_of_shards", 5);
-                }
-                mappingsAndSettings.endObject();
-            }
-            mappingsAndSettings.endObject();
-
-            Request createIndex = new Request("PUT", "/" + index);
-            createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
-            client().performRequest(createIndex);
-
-            numDocs = randomIntBetween(512, 1024);
-            indexRandomDocuments(
-                numDocs,
-                true,
-                true,
-                randomBoolean(),
-                i -> JsonXContent.contentBuilder().startObject().field("field", "value").endObject()
-            );
-
-            ensureGreen(index); // wait for source index to be available on both nodes before starting shrink
-
-            Request updateSettingsRequest = new Request("PUT", "/" + index + "/_settings");
-            updateSettingsRequest.setJsonEntity("{\"settings\": {\"index.blocks.write\": true}}");
-            client().performRequest(updateSettingsRequest);
-
-            Request shrinkIndexRequest = new Request("PUT", "/" + index + "/_shrink/" + shrunkenIndex);
-
-            shrinkIndexRequest.setJsonEntity("{\"settings\": {\"index.number_of_shards\": 1}}");
-            client().performRequest(shrinkIndexRequest);
-
-            refreshAllIndices();
-        } else {
-            numDocs = countOfIndexedRandomDocuments();
-        }
-
-        Map<?, ?> response = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")));
-        assertNoFailures(response);
-        int totalShards = (int) XContentMapValues.extractValue("_shards.total", response);
-        assertThat(totalShards, greaterThan(1));
-        int successfulShards = (int) XContentMapValues.extractValue("_shards.successful", response);
-        assertEquals(totalShards, successfulShards);
-        int totalHits = extractTotalHits(response);
-        assertEquals(numDocs, totalHits);
-
-        response = entityAsMap(client().performRequest(new Request("GET", "/" + shrunkenIndex + "/_search")));
-        assertNoFailures(response);
-        totalShards = (int) XContentMapValues.extractValue("_shards.total", response);
-        assertEquals(1, totalShards);
-        successfulShards = (int) XContentMapValues.extractValue("_shards.successful", response);
-        assertEquals(1, successfulShards);
-        totalHits = extractTotalHits(response);
-        assertEquals(numDocs, totalHits);
-    }
-
-    public void testShrinkAfterUpgrade() throws IOException {
-        String shrunkenIndex = index + "_shrunk";
-        int numDocs;
-        if (isRunningAgainstOldCluster()) {
-            XContentBuilder mappingsAndSettings = jsonBuilder();
-            mappingsAndSettings.startObject();
-            {
-                mappingsAndSettings.startObject("mappings");
-                {
-                    mappingsAndSettings.startObject("properties");
-                    {
-                        mappingsAndSettings.startObject("field");
-                        {
-                            mappingsAndSettings.field("type", "text");
-                        }
-                        mappingsAndSettings.endObject();
-                    }
-                    mappingsAndSettings.endObject();
-                }
-                mappingsAndSettings.endObject();
-                // the default number of shards is now one so we have to set the number of shards to be more than one explicitly
-                mappingsAndSettings.startObject("settings");
-                mappingsAndSettings.field("index.number_of_shards", 5);
-                mappingsAndSettings.endObject();
-            }
-            mappingsAndSettings.endObject();
-
-            Request createIndex = new Request("PUT", "/" + index);
-            createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
-            client().performRequest(createIndex);
-
-            numDocs = randomIntBetween(512, 1024);
-            indexRandomDocuments(
-                numDocs,
-                true,
-                true,
-                randomBoolean(),
-                i -> JsonXContent.contentBuilder().startObject().field("field", "value").endObject()
-            );
-        } else {
-            ensureGreen(index); // wait for source index to be available on both nodes before starting shrink
-
-            Request updateSettingsRequest = new Request("PUT", "/" + index + "/_settings");
-            updateSettingsRequest.setJsonEntity("{\"settings\": {\"index.blocks.write\": true}}");
-            client().performRequest(updateSettingsRequest);
-
-            Request shrinkIndexRequest = new Request("PUT", "/" + index + "/_shrink/" + shrunkenIndex);
-            shrinkIndexRequest.setJsonEntity("{\"settings\": {\"index.number_of_shards\": 1}}");
-            client().performRequest(shrinkIndexRequest);
-
-            numDocs = countOfIndexedRandomDocuments();
-        }
-
-        refreshAllIndices();
-
-        Map<?, ?> response = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")));
-        assertNoFailures(response);
-        int totalShards = (int) XContentMapValues.extractValue("_shards.total", response);
-        assertThat(totalShards, greaterThan(1));
-        int successfulShards = (int) XContentMapValues.extractValue("_shards.successful", response);
-        assertEquals(totalShards, successfulShards);
-        int totalHits = extractTotalHits(response);
-        assertEquals(numDocs, totalHits);
-
-        if (isRunningAgainstOldCluster() == false) {
-            response = entityAsMap(client().performRequest(new Request("GET", "/" + shrunkenIndex + "/_search")));
-            assertNoFailures(response);
-            totalShards = (int) XContentMapValues.extractValue("_shards.total", response);
-            assertEquals(1, totalShards);
-            successfulShards = (int) XContentMapValues.extractValue("_shards.successful", response);
-            assertEquals(1, successfulShards);
-            totalHits = extractTotalHits(response);
-            assertEquals(numDocs, totalHits);
-        }
-    }
-
-    /**
-     * Test upgrading after a rollover. Specifically:
-     * <ol>
-     *  <li>Create an index with a write alias
-     *  <li>Write some documents to the write alias
-     *  <li>Roll over the index
-     *  <li>Make sure the document count is correct
-     *  <li>Upgrade
-     *  <li>Write some more documents to the write alias
-     *  <li>Make sure the document count is correct
-     * </ol>
-     */
-    public void testRollover() throws IOException {
-        if (isRunningAgainstOldCluster()) {
-            Request createIndex = new Request("PUT", "/" + index + "-000001");
-            createIndex.setJsonEntity(formatted("""
-                {
-                  "aliases": {
-                    "%s_write": {}
-                  }
-                }""", index));
-            client().performRequest(createIndex);
-        }
-
-        int bulkCount = 10;
-        String bulk = """
-            {"index":{}}
-            {"test":"test"}
-            """.repeat(bulkCount);
-
-        Request bulkRequest = new Request("POST", "/" + index + "_write/_bulk");
-
-        bulkRequest.setJsonEntity(bulk);
-        bulkRequest.addParameter("refresh", "");
-        assertThat(EntityUtils.toString(client().performRequest(bulkRequest).getEntity()), containsString("\"errors\":false"));
-
-        if (isRunningAgainstOldCluster()) {
-            Request rolloverRequest = new Request("POST", "/" + index + "_write/_rollover");
-            rolloverRequest.setJsonEntity("""
-                {  "conditions": {    "max_docs": 5  }}""");
-            client().performRequest(rolloverRequest);
-
-            assertThat(
-                EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices?v")).getEntity()),
-                containsString("testrollover-000002")
-            );
-        }
-
-        Request countRequest = new Request("POST", "/" + index + "-*/_search");
-        countRequest.addParameter("size", "0");
-        Map<String, Object> count = entityAsMap(client().performRequest(countRequest));
-        assertNoFailures(count);
-
-        int expectedCount = bulkCount + (isRunningAgainstOldCluster() ? 0 : bulkCount);
-        assertEquals(expectedCount, extractTotalHits(count));
-    }
-
-    void assertCountAll(int count) throws IOException {
-        Map<String, Object> response = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")));
-        assertNoFailures(response);
-        int numDocs = extractTotalHits(response);
-        logger.info("Found {} in old index", numDocs);
-        assertEquals(count, numDocs);
-    }
-
-    void assertBasicSearchWorks(int count) throws IOException {
-        logger.info("--> testing basic search");
-        {
-            assertCountAll(count);
-        }
-
-        logger.info("--> testing basic search with sort");
-        {
-            Request searchRequest = new Request("GET", "/" + index + "/_search");
-            searchRequest.setJsonEntity("""
-                { "sort": [{ "int" : "asc" }]}""");
-            Map<String, Object> response = entityAsMap(client().performRequest(searchRequest));
-            assertNoFailures(response);
-            assertTotalHits(count, response);
-        }
-
-        logger.info("--> testing exists filter");
-        {
-            Request searchRequest = new Request("GET", "/" + index + "/_search");
-            searchRequest.setJsonEntity("""
-                { "query": { "exists" : {"field": "string"} }}""");
-            Map<String, Object> response = entityAsMap(client().performRequest(searchRequest));
-            assertNoFailures(response);
-            assertTotalHits(count, response);
-        }
-
-        logger.info("--> testing field with dots in the name");
-        {
-            Request searchRequest = new Request("GET", "/" + index + "/_search");
-            searchRequest.setJsonEntity("""
-                { "query": { "exists" : {"field": "field.with.dots"} }}""");
-            Map<String, Object> response = entityAsMap(client().performRequest(searchRequest));
-            assertNoFailures(response);
-            assertTotalHits(count, response);
-        }
-    }
-
-    void assertAllSearchWorks(int count) throws IOException {
-        logger.info("--> testing _all search");
-        Map<String, Object> response = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")));
-        assertNoFailures(response);
-        assertTotalHits(count, response);
-        Map<?, ?> bestHit = (Map<?, ?>) ((List<?>) (XContentMapValues.extractValue("hits.hits", response))).get(0);
-
-        // Make sure there are payloads and they are taken into account for the score
-        // the 'string' field has a boost of 4 in the mappings so it should get a payload boost
-        String stringValue = (String) XContentMapValues.extractValue("_source.string", bestHit);
-        assertNotNull(stringValue);
-        String id = (String) bestHit.get("_id");
-
-        Request explainRequest = new Request("GET", "/" + index + "/_explain/" + id);
-        explainRequest.setJsonEntity("{ \"query\": { \"match_all\" : {} }}");
-        String explanation = toStr(client().performRequest(explainRequest));
-        assertFalse("Could not find payload boost in explanation\n" + explanation, explanation.contains("payloadBoost"));
-
-        // Make sure the query can run on the whole index
-        Request searchRequest = new Request("GET", "/" + index + "/_search");
-        searchRequest.setEntity(explainRequest.getEntity());
-        searchRequest.addParameter("explain", "true");
-        Map<?, ?> matchAllResponse = entityAsMap(client().performRequest(searchRequest));
-        assertNoFailures(matchAllResponse);
-        assertTotalHits(count, matchAllResponse);
-    }
-
-    void assertBasicAggregationWorks() throws IOException {
-        // histogram on a long
-        Request longHistogramRequest = new Request("GET", "/" + index + "/_search");
-        longHistogramRequest.setJsonEntity("""
-            {
-              "aggs": {
-                "histo": {
-                  "histogram": {
-                    "field": "int",
-                    "interval": 10
-                  }
-                }
-              }
-            }""");
-        Map<?, ?> longHistogram = entityAsMap(client().performRequest(longHistogramRequest));
-        assertNoFailures(longHistogram);
-        List<?> histoBuckets = (List<?>) XContentMapValues.extractValue("aggregations.histo.buckets", longHistogram);
-        int histoCount = 0;
-        for (Object entry : histoBuckets) {
-            Map<?, ?> bucket = (Map<?, ?>) entry;
-            histoCount += (Integer) bucket.get("doc_count");
-        }
-        assertTotalHits(histoCount, longHistogram);
-
-        // terms on a boolean
-        Request boolTermsRequest = new Request("GET", "/" + index + "/_search");
-        boolTermsRequest.setJsonEntity("""
-            {
-              "aggs": {
-                "bool_terms": {
-                  "terms": {
-                    "field": "bool"
-                  }
-                }
-              }
-            }""");
-        Map<?, ?> boolTerms = entityAsMap(client().performRequest(boolTermsRequest));
-        List<?> termsBuckets = (List<?>) XContentMapValues.extractValue("aggregations.bool_terms.buckets", boolTerms);
-        int termsCount = 0;
-        for (Object entry : termsBuckets) {
-            Map<?, ?> bucket = (Map<?, ?>) entry;
-            termsCount += (Integer) bucket.get("doc_count");
-        }
-        assertTotalHits(termsCount, boolTerms);
-    }
-
-    void assertRealtimeGetWorks() throws IOException {
-        Request disableAutoRefresh = new Request("PUT", "/" + index + "/_settings");
-        disableAutoRefresh.setJsonEntity("""
-            { "index": { "refresh_interval" : -1 }}""");
-        client().performRequest(disableAutoRefresh);
-
-        Request searchRequest = new Request("GET", "/" + index + "/_search");
-        searchRequest.setJsonEntity("""
-            { "query": { "match_all" : {} }}""");
-        Map<?, ?> searchResponse = entityAsMap(client().performRequest(searchRequest));
-        Map<?, ?> hit = (Map<?, ?>) ((List<?>) (XContentMapValues.extractValue("hits.hits", searchResponse))).get(0);
-        String docId = (String) hit.get("_id");
-
-        Request updateRequest = new Request("POST", "/" + index + "/_update/" + docId);
-        updateRequest.setJsonEntity("""
-            { "doc" : { "foo": "bar"}}""");
-        client().performRequest(updateRequest);
-
-        Request getRequest = new Request("GET", "/" + index + "/_doc/" + docId);
-
-        Map<String, Object> getRsp = entityAsMap(client().performRequest(getRequest));
-        Map<?, ?> source = (Map<?, ?>) getRsp.get("_source");
-        assertTrue("doc does not contain 'foo' key: " + source, source.containsKey("foo"));
-
-        Request enableAutoRefresh = new Request("PUT", "/" + index + "/_settings");
-        enableAutoRefresh.setJsonEntity("""
-            { "index": { "refresh_interval" : "1s" }}""");
-        client().performRequest(enableAutoRefresh);
-    }
-
-    void assertStoredBinaryFields(int count) throws Exception {
-        Request request = new Request("GET", "/" + index + "/_search");
-        request.setJsonEntity("""
-            {
-              "query": {
-                "match_all": {}
-              },
-              "size": 100,
-              "stored_fields": "binary"
-            }""");
-        Map<String, Object> rsp = entityAsMap(client().performRequest(request));
-
-        assertTotalHits(count, rsp);
-        List<?> hits = (List<?>) XContentMapValues.extractValue("hits.hits", rsp);
-        assertEquals(100, hits.size());
-        for (Object hit : hits) {
-            Map<?, ?> hitRsp = (Map<?, ?>) hit;
-            List<?> values = (List<?>) XContentMapValues.extractValue("fields.binary", hitRsp);
-            assertEquals(1, values.size());
-            String value = (String) values.get(0);
-            byte[] binaryValue = Base64.getDecoder().decode(value);
-            assertEquals("Unexpected string length [" + value + "]", 16, binaryValue.length);
-        }
-    }
-
-    static String toStr(Response response) throws IOException {
-        return EntityUtils.toString(response.getEntity());
-    }
-
-    static void assertNoFailures(Map<?, ?> response) {
-        int failed = (int) XContentMapValues.extractValue("_shards.failed", response);
-        assertEquals(0, failed);
-    }
-
-    void assertTotalHits(int expectedTotalHits, Map<?, ?> response) {
-        int actualTotalHits = extractTotalHits(response);
-        assertEquals(response.toString(), expectedTotalHits, actualTotalHits);
-    }
-
-    static int extractTotalHits(Map<?, ?> response) {
-        return (Integer) XContentMapValues.extractValue("hits.total.value", response);
-    }
-
-    /**
-     * Tests that a single document survives. Super basic smoke test.
-     */
-    public void testSingleDoc() throws IOException {
-        String docLocation = "/" + index + "/_doc/1";
-        String doc = "{\"test\": \"test\"}";
-
-        if (isRunningAgainstOldCluster()) {
-            Request createDoc = new Request("PUT", docLocation);
-            createDoc.setJsonEntity(doc);
-            client().performRequest(createDoc);
-        }
-
-        Request request = new Request("GET", docLocation);
-        assertThat(toStr(client().performRequest(request)), containsString(doc));
-    }
-
-    /**
-     * Tests that a single empty shard index is correctly recovered. Empty shards are often an edge case.
-     */
-    public void testEmptyShard() throws IOException {
-        final String indexName = "test_empty_shard";
-
-        if (isRunningAgainstOldCluster()) {
-            Settings.Builder settings = Settings.builder()
-                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
-                // if the node with the replica is the first to be restarted, while a replica is still recovering
-                // then delayed allocation will kick in. When the node comes back, the master will search for a copy
-                // but the recovering copy will be seen as invalid and the cluster health won't return to GREEN
-                // before timing out
-                .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
-                .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
-            createIndex(indexName, settings.build());
-        }
-        ensureGreen(indexName);
-    }
-
-    /**
-     * Tests recovery of an index with or without a translog and the
-     * statistics we gather about that.
-     */
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/52031")
-    public void testRecovery() throws Exception {
-        int count;
-        boolean shouldHaveTranslog;
-        if (isRunningAgainstOldCluster()) {
-            count = between(200, 300);
-            /* We've had bugs in the past where we couldn't restore
-             * an index without a translog so we randomize whether
-             * or not we have one. */
-            shouldHaveTranslog = randomBoolean();
-            Settings.Builder settings = Settings.builder();
-            if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
-                settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
-            }
-            final String mappings = randomBoolean() ? "\"_source\": { \"enabled\": false}" : null;
-            createIndex(index, settings.build(), mappings);
-            indexRandomDocuments(count, true, true, true, i -> jsonBuilder().startObject().field("field", "value").endObject());
-
-            // make sure all recoveries are done
-            ensureGreen(index);
-
-            // Force flush so we're sure that all translog are committed
-            Request flushRequest = new Request("POST", "/" + index + "/_flush");
-            flushRequest.addParameter("force", "true");
-            flushRequest.addParameter("wait_if_ongoing", "true");
-            assertOK(client().performRequest(flushRequest));
-
-            if (shouldHaveTranslog) {
-                // Update a few documents so we are sure to have a translog
-                indexRandomDocuments(
-                    count / 10,
-                    false, // flushing here would invalidate the whole thing
-                    false,
-                    true,
-                    i -> jsonBuilder().startObject().field("field", "value").endObject()
-                );
-            }
-            saveInfoDocument(index + "_should_have_translog", Boolean.toString(shouldHaveTranslog));
-        } else {
-            count = countOfIndexedRandomDocuments();
-            shouldHaveTranslog = Booleans.parseBoolean(loadInfoDocument(index + "_should_have_translog"));
-        }
-
-        // Count the documents in the index to make sure we have as many as we put there
-        Request countRequest = new Request("GET", "/" + index + "/_search");
-        countRequest.addParameter("size", "0");
-        refreshAllIndices();
-        Map<String, Object> countResponse = entityAsMap(client().performRequest(countRequest));
-        assertTotalHits(count, countResponse);
-
-        if (false == isRunningAgainstOldCluster()) {
-            boolean restoredFromTranslog = false;
-            boolean foundPrimary = false;
-            Request recoveryRequest = new Request("GET", "/_cat/recovery/" + index);
-            recoveryRequest.addParameter("h", "index,shard,type,stage,translog_ops_recovered");
-            recoveryRequest.addParameter("s", "index,shard,type");
-            String recoveryResponse = toStr(client().performRequest(recoveryRequest));
-            for (String line : recoveryResponse.split("\n")) {
-                // Find the primaries
-                foundPrimary = true;
-                if (false == line.contains("done") && line.contains("existing_store")) {
-                    continue;
-                }
-                /* Mark if we see a primary that looked like it restored from the translog.
-                 * Not all primaries will look like this all the time because we modify
-                 * random documents when we want there to be a translog and they might
-                 * not be spread around all the shards. */
-                Matcher m = Pattern.compile("(\\d+)$").matcher(line);
-                assertTrue(line, m.find());
-                int translogOps = Integer.parseInt(m.group(1));
-                if (translogOps > 0) {
-                    restoredFromTranslog = true;
-                }
-            }
-            assertTrue("expected to find a primary but didn't\n" + recoveryResponse, foundPrimary);
-            assertEquals("mismatch while checking for translog recovery\n" + recoveryResponse, shouldHaveTranslog, restoredFromTranslog);
-
-            String currentLuceneVersion = Version.CURRENT.luceneVersion.toString();
-            String bwcLuceneVersion = getOldClusterVersion().luceneVersion.toString();
-            String minCompatibleBWCVersion = Version.CURRENT.minimumCompatibilityVersion().luceneVersion.toString();
-            if (shouldHaveTranslog && false == currentLuceneVersion.equals(bwcLuceneVersion)) {
-                int numCurrentVersion = 0;
-                int numBwcVersion = 0;
-                Request segmentsRequest = new Request("GET", "/_cat/segments/" + index);
-                segmentsRequest.addParameter("h", "prirep,shard,index,version");
-                segmentsRequest.addParameter("s", "prirep,shard,index");
-                String segmentsResponse = toStr(client().performRequest(segmentsRequest));
-                for (String line : segmentsResponse.split("\n")) {
-                    if (false == line.startsWith("p")) {
-                        continue;
-                    }
-                    Matcher m = Pattern.compile("(\\d+\\.\\d+\\.\\d+)$").matcher(line);
-                    assertTrue(line, m.find());
-                    String version = m.group(1);
-                    if (currentLuceneVersion.equals(version)) {
-                        numCurrentVersion++;
-                    } else if (bwcLuceneVersion.equals(version)) {
-                        numBwcVersion++;
-                    } else if (minCompatibleBWCVersion.equals(version) && minCompatibleBWCVersion.equals(bwcLuceneVersion) == false) {
-                        // Our upgrade path from 7.non-last always goes through 7.last, which depending on timing can create 7.last
-                        // index segment. We ignore those.
-                        continue;
-                    } else {
-                        fail("expected version to be one of [" + currentLuceneVersion + "," + bwcLuceneVersion + "] but was " + line);
-                    }
-                }
-                assertNotEquals(
-                    "expected at least 1 current segment after translog recovery. segments:\n" + segmentsResponse,
-                    0,
-                    numCurrentVersion
-                );
-                assertNotEquals("expected at least 1 old segment. segments:\n" + segmentsResponse, 0, numBwcVersion);
-            }
-        }
-    }
-
-    /**
-     * Tests snapshot/restore by creating a snapshot and restoring it. It takes
-     * a snapshot on the old cluster and restores it on the old cluster as a
-     * sanity check and on the new cluster as an upgrade test. It also takes a
-     * snapshot on the new cluster and restores that on the new cluster as a
-     * test that the repository is ok with containing snapshot from both the
-     * old and new versions. All of the snapshots include an index, a template,
-     * and some routing configuration.
-     */
-    public void testSnapshotRestore() throws IOException {
-        int count;
-        if (isRunningAgainstOldCluster()) {
-            // Create the index
-            count = between(200, 300);
-            Settings.Builder settings = Settings.builder();
-            if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
-                settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
-            }
-            createIndex(index, settings.build());
-            indexRandomDocuments(count, true, true, randomBoolean(), i -> jsonBuilder().startObject().field("field", "value").endObject());
-        } else {
-            count = countOfIndexedRandomDocuments();
-        }
-
-        // Refresh the index so the count doesn't fail
-        refreshAllIndices();
-
-        // Count the documents in the index to make sure we have as many as we put there
-        Request countRequest = new Request("GET", "/" + index + "/_search");
-        countRequest.addParameter("size", "0");
-        Map<String, Object> countResponse = entityAsMap(client().performRequest(countRequest));
-        assertTotalHits(count, countResponse);
-
-        // Stick a routing attribute into to cluster settings so we can see it after the restore
-        Request addRoutingSettings = new Request("PUT", "/_cluster/settings");
-        addRoutingSettings.setJsonEntity(formatted("""
-            {"persistent": {"cluster.routing.allocation.exclude.test_attr": "%s"}}
-            """, getOldClusterVersion()));
-        client().performRequest(addRoutingSettings);
-
-        // Stick a template into the cluster so we can see it after the restore
-        XContentBuilder templateBuilder = JsonXContent.contentBuilder().startObject();
-        templateBuilder.field("index_patterns", "evil_*"); // Don't confuse other tests by applying the template
-        templateBuilder.startObject("settings");
-        {
-            templateBuilder.field("number_of_shards", 1);
-        }
-        templateBuilder.endObject();
-        templateBuilder.startObject("mappings");
-        {
-            {
-                templateBuilder.startObject("_source");
-                {
-                    templateBuilder.field("enabled", true);
-                }
-                templateBuilder.endObject();
-            }
-        }
-        templateBuilder.endObject();
-        templateBuilder.startObject("aliases");
-        {
-            templateBuilder.startObject("alias1").endObject();
-            templateBuilder.startObject("alias2");
-            {
-                templateBuilder.startObject("filter");
-                {
-                    templateBuilder.startObject("term");
-                    {
-                        templateBuilder.field("version", isRunningAgainstOldCluster() ? getOldClusterVersion() : Version.CURRENT);
-                    }
-                    templateBuilder.endObject();
-                }
-                templateBuilder.endObject();
-            }
-            templateBuilder.endObject();
-        }
-        templateBuilder.endObject().endObject();
-        Request createTemplateRequest = new Request("PUT", "/_template/test_template");
-        createTemplateRequest.setJsonEntity(Strings.toString(templateBuilder));
-        createTemplateRequest.setOptions(expectWarnings(RestPutIndexTemplateAction.DEPRECATION_WARNING));
-
-        client().performRequest(createTemplateRequest);
-
-        if (isRunningAgainstOldCluster()) {
-            // Create the repo
-            XContentBuilder repoConfig = JsonXContent.contentBuilder().startObject();
-            {
-                repoConfig.field("type", "fs");
-                repoConfig.startObject("settings");
-                {
-                    repoConfig.field("compress", randomBoolean());
-                    repoConfig.field("location", System.getProperty("tests.path.repo"));
-                }
-                repoConfig.endObject();
-            }
-            repoConfig.endObject();
-            Request createRepoRequest = new Request("PUT", "/_snapshot/repo");
-            createRepoRequest.setJsonEntity(Strings.toString(repoConfig));
-            client().performRequest(createRepoRequest);
-        }
-
-        Request createSnapshot = new Request("PUT", "/_snapshot/repo/" + (isRunningAgainstOldCluster() ? "old_snap" : "new_snap"));
-        createSnapshot.addParameter("wait_for_completion", "true");
-        createSnapshot.setJsonEntity("{\"indices\": \"" + index + "\"}");
-        client().performRequest(createSnapshot);
-
-        checkSnapshot("old_snap", count, getOldClusterVersion());
-        if (false == isRunningAgainstOldCluster()) {
-            checkSnapshot("new_snap", count, Version.CURRENT);
-        }
-    }
-
-    public void testHistoryUUIDIsAdded() throws Exception {
-        if (isRunningAgainstOldCluster()) {
-            XContentBuilder mappingsAndSettings = jsonBuilder();
-            mappingsAndSettings.startObject();
-            {
-                mappingsAndSettings.startObject("settings");
-                mappingsAndSettings.field("number_of_shards", 1);
-                mappingsAndSettings.field("number_of_replicas", 1);
-                mappingsAndSettings.endObject();
-            }
-            mappingsAndSettings.endObject();
-            Request createIndex = new Request("PUT", "/" + index);
-            createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
-            client().performRequest(createIndex);
-        } else {
-            ensureGreenLongWait(index);
-
-            Request statsRequest = new Request("GET", index + "/_stats");
-            statsRequest.addParameter("level", "shards");
-            Response response = client().performRequest(statsRequest);
-            List<Object> shardStats = ObjectPath.createFromResponse(response).evaluate("indices." + index + ".shards.0");
-            assertThat(shardStats, notNullValue());
-            assertThat("Expected stats for 2 shards", shardStats, hasSize(2));
-            String globalHistoryUUID = null;
-            for (Object shard : shardStats) {
-                final String nodeId = ObjectPath.evaluate(shard, "routing.node");
-                final Boolean primary = ObjectPath.evaluate(shard, "routing.primary");
-                logger.info("evaluating: {} , {}", ObjectPath.evaluate(shard, "routing"), ObjectPath.evaluate(shard, "commit"));
-                String historyUUID = ObjectPath.evaluate(shard, "commit.user_data.history_uuid");
-                assertThat("no history uuid found on " + nodeId + " (primary: " + primary + ")", historyUUID, notNullValue());
-                if (globalHistoryUUID == null) {
-                    globalHistoryUUID = historyUUID;
-                } else {
-                    assertThat(
-                        "history uuid mismatch on " + nodeId + " (primary: " + primary + ")",
-                        historyUUID,
-                        equalTo(globalHistoryUUID)
-                    );
-                }
-            }
-        }
-    }
-
-    public void testSoftDeletes() throws Exception {
-        if (isRunningAgainstOldCluster()) {
-            XContentBuilder mappingsAndSettings = jsonBuilder();
-            mappingsAndSettings.startObject();
-            {
-                mappingsAndSettings.startObject("settings");
-                mappingsAndSettings.field("number_of_shards", 1);
-                mappingsAndSettings.field("number_of_replicas", 1);
-                if (randomBoolean()) {
-                    mappingsAndSettings.field("soft_deletes.enabled", true);
-                }
-                mappingsAndSettings.endObject();
-            }
-            mappingsAndSettings.endObject();
-            Request createIndex = new Request("PUT", "/" + index);
-            createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
-            client().performRequest(createIndex);
-            int numDocs = between(10, 100);
-            for (int i = 0; i < numDocs; i++) {
-                String doc = Strings.toString(JsonXContent.contentBuilder().startObject().field("field", "v1").endObject());
-                Request request = new Request("POST", "/" + index + "/_doc/" + i);
-                request.setJsonEntity(doc);
-                client().performRequest(request);
-                refreshAllIndices();
-            }
-            client().performRequest(new Request("POST", "/" + index + "/_flush"));
-            int liveDocs = numDocs;
-            assertTotalHits(liveDocs, entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))));
-            for (int i = 0; i < numDocs; i++) {
-                if (randomBoolean()) {
-                    String doc = Strings.toString(JsonXContent.contentBuilder().startObject().field("field", "v2").endObject());
-                    Request request = new Request("POST", "/" + index + "/_doc/" + i);
-                    request.setJsonEntity(doc);
-                    client().performRequest(request);
-                } else if (randomBoolean()) {
-                    client().performRequest(new Request("DELETE", "/" + index + "/_doc/" + i));
-                    liveDocs--;
-                }
-            }
-            refreshAllIndices();
-            assertTotalHits(liveDocs, entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))));
-            saveInfoDocument(index + "_doc_count", Integer.toString(liveDocs));
-        } else {
-            int liveDocs = Integer.parseInt(loadInfoDocument(index + "_doc_count"));
-            assertTotalHits(liveDocs, entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))));
-        }
-    }
-
-    /**
-     * This test creates an index in the old cluster and then closes it. When the cluster is fully restarted in a newer version,
-     * it verifies that the index exists and is replicated if the old version supports replication.
-     */
-    public void testClosedIndices() throws Exception {
-        if (isRunningAgainstOldCluster()) {
-            createIndex(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build());
-            ensureGreen(index);
-
-            int numDocs = 0;
-            if (randomBoolean()) {
-                numDocs = between(1, 100);
-                for (int i = 0; i < numDocs; i++) {
-                    final Request request = new Request("POST", "/" + index + "/_doc/" + i);
-                    request.setJsonEntity(Strings.toString(JsonXContent.contentBuilder().startObject().field("field", "v1").endObject()));
-                    assertOK(client().performRequest(request));
-                    if (rarely()) {
-                        refreshAllIndices();
-                    }
-                }
-                refreshAllIndices();
-            }
-
-            assertTotalHits(numDocs, entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))));
-            saveInfoDocument(index + "_doc_count", Integer.toString(numDocs));
-            closeIndex(index);
-        }
-
-        if (getOldClusterVersion().onOrAfter(Version.V_7_2_0)) {
-            ensureGreenLongWait(index);
-            assertClosedIndex(index, true);
-        } else {
-            assertClosedIndex(index, false);
-        }
-
-        if (isRunningAgainstOldCluster() == false) {
-            openIndex(index);
-            ensureGreen(index);
-
-            final int expectedNumDocs = Integer.parseInt(loadInfoDocument(index + "_doc_count"));
-            assertTotalHits(expectedNumDocs, entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))));
-        }
-    }
-
-    /**
-     * Asserts that an index is closed in the cluster state. If `checkRoutingTable` is true, it also asserts
-     * that the index has started shards.
-     */
-    @SuppressWarnings("unchecked")
-    private void assertClosedIndex(final String indexName, final boolean checkRoutingTable) throws IOException {
-        final Map<String, ?> state = entityAsMap(client().performRequest(new Request("GET", "/_cluster/state")));
-
-        final Map<String, ?> metadata = (Map<String, Object>) XContentMapValues.extractValue("metadata.indices." + indexName, state);
-        assertThat(metadata, notNullValue());
-        assertThat(metadata.get("state"), equalTo("close"));
-
-        final Map<String, ?> blocks = (Map<String, Object>) XContentMapValues.extractValue("blocks.indices." + indexName, state);
-        assertThat(blocks, notNullValue());
-        assertThat(blocks.containsKey(String.valueOf(MetadataIndexStateService.INDEX_CLOSED_BLOCK_ID)), is(true));
-
-        final Map<String, ?> settings = (Map<String, Object>) XContentMapValues.extractValue("settings", metadata);
-        assertThat(settings, notNullValue());
-
-        final Map<String, ?> routingTable = (Map<String, Object>) XContentMapValues.extractValue(
-            "routing_table.indices." + indexName,
-            state
-        );
-        if (checkRoutingTable) {
-            assertThat(routingTable, notNullValue());
-            assertThat(Booleans.parseBoolean((String) XContentMapValues.extractValue("index.verified_before_close", settings)), is(true));
-            final String numberOfShards = (String) XContentMapValues.extractValue("index.number_of_shards", settings);
-            assertThat(numberOfShards, notNullValue());
-            final int nbShards = Integer.parseInt(numberOfShards);
-            assertThat(nbShards, greaterThanOrEqualTo(1));
-
-            for (int i = 0; i < nbShards; i++) {
-                final Collection<Map<String, ?>> shards = (Collection<Map<String, ?>>) XContentMapValues.extractValue(
-                    "shards." + i,
-                    routingTable
-                );
-                assertThat(shards, notNullValue());
-                assertThat(shards.size(), equalTo(2));
-                for (Map<String, ?> shard : shards) {
-                    assertThat(XContentMapValues.extractValue("shard", shard), equalTo(i));
-                    assertThat(XContentMapValues.extractValue("state", shard), equalTo("STARTED"));
-                    assertThat(XContentMapValues.extractValue("index", shard), equalTo(indexName));
-                }
-            }
-        } else {
-            assertThat(routingTable, nullValue());
-            assertThat(XContentMapValues.extractValue("index.verified_before_close", settings), nullValue());
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    private void checkSnapshot(final String snapshotName, final int count, final Version tookOnVersion) throws IOException {
-        // Check the snapshot metadata, especially the version
-        Request listSnapshotRequest = new Request("GET", "/_snapshot/repo/" + snapshotName);
-        Map<String, Object> snapResponse = entityAsMap(client().performRequest(listSnapshotRequest));
-
-        assertEquals(singletonList(snapshotName), XContentMapValues.extractValue("snapshots.snapshot", snapResponse));
-        assertEquals(singletonList("SUCCESS"), XContentMapValues.extractValue("snapshots.state", snapResponse));
-        assertEquals(singletonList(tookOnVersion.toString()), XContentMapValues.extractValue("snapshots.version", snapResponse));
-
-        // Remove the routing setting and template so we can test restoring them.
-        Request clearRoutingFromSettings = new Request("PUT", "/_cluster/settings");
-        clearRoutingFromSettings.setJsonEntity("""
-            {"persistent":{"cluster.routing.allocation.exclude.test_attr": null}}""");
-        client().performRequest(clearRoutingFromSettings);
-        client().performRequest(new Request("DELETE", "/_template/test_template"));
-
-        // Restore
-        XContentBuilder restoreCommand = JsonXContent.contentBuilder().startObject();
-        restoreCommand.field("include_global_state", true);
-        restoreCommand.field("indices", index);
-        restoreCommand.field("rename_pattern", index);
-        restoreCommand.field("rename_replacement", "restored_" + index);
-        restoreCommand.endObject();
-        Request restoreRequest = new Request("POST", "/_snapshot/repo/" + snapshotName + "/_restore");
-        restoreRequest.addParameter("wait_for_completion", "true");
-        restoreRequest.setJsonEntity(Strings.toString(restoreCommand));
-        client().performRequest(restoreRequest);
-
-        // Make sure search finds all documents
-        Request countRequest = new Request("GET", "/restored_" + index + "/_search");
-        countRequest.addParameter("size", "0");
-        Map<String, Object> countResponse = entityAsMap(client().performRequest(countRequest));
-        assertTotalHits(count, countResponse);
-
-        // Add some extra documents to the index to be sure we can still write to it after restoring it
-        int extras = between(1, 100);
-        StringBuilder bulk = new StringBuilder();
-        for (int i = 0; i < extras; i++) {
-            bulk.append(formatted("""
-                {"index":{"_id":"%s"}}
-                {"test":"test"}
-                """, count + i));
-        }
-
-        Request writeToRestoredRequest = new Request("POST", "/restored_" + index + "/_bulk");
-
-        writeToRestoredRequest.addParameter("refresh", "true");
-        writeToRestoredRequest.setJsonEntity(bulk.toString());
-        assertThat(EntityUtils.toString(client().performRequest(writeToRestoredRequest).getEntity()), containsString("\"errors\":false"));
-
-        // And count to make sure the add worked
-        // Make sure search finds all documents
-        Request countAfterWriteRequest = new Request("GET", "/restored_" + index + "/_search");
-        countAfterWriteRequest.addParameter("size", "0");
-        Map<String, Object> countAfterResponse = entityAsMap(client().performRequest(countRequest));
-        assertTotalHits(count + extras, countAfterResponse);
-
-        // Clean up the index for the next iteration
-        client().performRequest(new Request("DELETE", "/restored_*"));
-
-        // Check settings added by the restore process
-        Request clusterSettingsRequest = new Request("GET", "/_cluster/settings");
-        clusterSettingsRequest.addParameter("flat_settings", "true");
-        Map<String, Object> clusterSettingsResponse = entityAsMap(client().performRequest(clusterSettingsRequest));
-        @SuppressWarnings("unchecked")
-        final Map<String, Object> persistentSettings = (Map<String, Object>) clusterSettingsResponse.get("persistent");
-        assertThat(persistentSettings.get("cluster.routing.allocation.exclude.test_attr"), equalTo(getOldClusterVersion().toString()));
-
-        // Check that the template was restored successfully
-        Request getTemplateRequest = new Request("GET", "/_template/test_template");
-
-        Map<String, Object> getTemplateResponse = entityAsMap(client().performRequest(getTemplateRequest));
-        Map<String, Object> expectedTemplate = new HashMap<>();
-        expectedTemplate.put("index_patterns", singletonList("evil_*"));
-
-        expectedTemplate.put("settings", singletonMap("index", singletonMap("number_of_shards", "1")));
-        expectedTemplate.put("mappings", singletonMap("_source", singletonMap("enabled", true)));
-
-        expectedTemplate.put("order", 0);
-        Map<String, Object> aliases = new HashMap<>();
-        aliases.put("alias1", emptyMap());
-        aliases.put("alias2", singletonMap("filter", singletonMap("term", singletonMap("version", tookOnVersion.toString()))));
-        expectedTemplate.put("aliases", aliases);
-        expectedTemplate = singletonMap("test_template", expectedTemplate);
-        if (false == expectedTemplate.equals(getTemplateResponse)) {
-            NotEqualMessageBuilder builder = new NotEqualMessageBuilder();
-            builder.compareMaps(getTemplateResponse, expectedTemplate);
-            logger.info("expected: {}\nactual:{}", expectedTemplate, getTemplateResponse);
-            fail("template doesn't match:\n" + builder.toString());
-        }
-    }
-
-    private void indexRandomDocuments(
-        final int count,
-        final boolean flushAllowed,
-        final boolean saveInfo,
-        final boolean specifyId,
-        final CheckedFunction<Integer, XContentBuilder, IOException> docSupplier
-    ) throws IOException {
-        logger.info("Indexing {} random documents", count);
-        for (int i = 0; i < count; i++) {
-            logger.debug("Indexing document [{}]", i);
-            Request createDocument = new Request("POST", "/" + index + "/_doc/" + (specifyId ? i : ""));
-            createDocument.setJsonEntity(Strings.toString(docSupplier.apply(i)));
-            client().performRequest(createDocument);
-            if (rarely()) {
-                refreshAllIndices();
-            }
-            if (flushAllowed && rarely()) {
-                logger.debug("Flushing [{}]", index);
-                client().performRequest(new Request("POST", "/" + index + "/_flush"));
-            }
-        }
-        if (saveInfo) {
-            saveInfoDocument(index + "_count", Integer.toString(count));
-        }
-    }
-
-    private void indexDocument(String id) throws IOException {
-        final Request indexRequest = new Request("POST", "/" + index + "/" + "_doc/" + id);
-        indexRequest.setJsonEntity(Strings.toString(JsonXContent.contentBuilder().startObject().field("f", "v").endObject()));
-        assertOK(client().performRequest(indexRequest));
-    }
-
-    private int countOfIndexedRandomDocuments() throws IOException {
-        return Integer.parseInt(loadInfoDocument(index + "_count"));
-    }
-
-    private void saveInfoDocument(String id, String value) throws IOException {
-        XContentBuilder infoDoc = JsonXContent.contentBuilder().startObject();
-        infoDoc.field("value", value);
-        infoDoc.endObject();
-        // Only create the first version so we know how many documents are created when the index is first created
-        Request request = new Request("PUT", "/info/_doc/" + id);
-        request.addParameter("op_type", "create");
-        request.setJsonEntity(Strings.toString(infoDoc));
-        client().performRequest(request);
-    }
-
-    private String loadInfoDocument(String id) throws IOException {
-        Request request = new Request("GET", "/info/_doc/" + id);
-        request.addParameter("filter_path", "_source");
-        String doc = toStr(client().performRequest(request));
-        Matcher m = Pattern.compile("\"value\":\"(.+)\"").matcher(doc);
-        assertTrue(doc, m.find());
-        return m.group(1);
-    }
-
-    private List<String> dataNodes(String indexName, RestClient client) throws IOException {
-        Request request = new Request("GET", indexName + "/_stats");
-        request.addParameter("level", "shards");
-        Response response = client.performRequest(request);
-        List<String> nodes = new ArrayList<>();
-        List<Object> shardStats = ObjectPath.createFromResponse(response).evaluate("indices." + indexName + ".shards.0");
-        for (Object shard : shardStats) {
-            final String nodeId = ObjectPath.evaluate(shard, "routing.node");
-            nodes.add(nodeId);
-        }
-        return nodes;
-    }
-
-    /**
-     * Wait for an index to have green health, waiting longer than
-     * {@link ESRestTestCase#ensureGreen}.
-     */
-    protected void ensureGreenLongWait(String indexName) throws IOException {
-        Request request = new Request("GET", "/_cluster/health/" + indexName);
-        request.addParameter("timeout", "2m");
-        request.addParameter("wait_for_status", "green");
-        request.addParameter("wait_for_no_relocating_shards", "true");
-        request.addParameter("wait_for_events", "languid");
-        request.addParameter("level", "shards");
-        Map<String, Object> healthRsp = entityAsMap(client().performRequest(request));
-        logger.info("health api response: {}", healthRsp);
-        assertEquals("green", healthRsp.get("status"));
-        assertFalse((Boolean) healthRsp.get("timed_out"));
-    }
-
-    public void testPeerRecoveryRetentionLeases() throws Exception {
-        if (isRunningAgainstOldCluster()) {
-            XContentBuilder settings = jsonBuilder();
-            settings.startObject();
-            {
-                settings.startObject("settings");
-                settings.field("number_of_shards", between(1, 5));
-                settings.field("number_of_replicas", between(0, 1));
-                settings.endObject();
-            }
-            settings.endObject();
-
-            Request createIndex = new Request("PUT", "/" + index);
-            createIndex.setJsonEntity(Strings.toString(settings));
-            client().performRequest(createIndex);
-        }
-        ensureGreen(index);
-        ensurePeerRecoveryRetentionLeasesRenewedAndSynced(index);
-    }
-
-    /**
-     * Tests that with or without soft-deletes, we should perform an operation-based recovery if there were some
-     * but not too many uncommitted documents (i.e., less than 10% of committed documents or the extra translog)
-     * before we restart the cluster. This is important when we move from translog based to retention leases based
-     * peer recoveries.
-     */
-    public void testOperationBasedRecovery() throws Exception {
-        if (isRunningAgainstOldCluster()) {
-            Settings.Builder settings = Settings.builder()
-                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1);
-            if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
-                settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
-            }
-            final String mappings = randomBoolean() ? "\"_source\": { \"enabled\": false}" : null;
-            createIndex(index, settings.build(), mappings);
-            ensureGreen(index);
-            int committedDocs = randomIntBetween(100, 200);
-            for (int i = 0; i < committedDocs; i++) {
-                indexDocument(Integer.toString(i));
-                if (rarely()) {
-                    flush(index, randomBoolean());
-                }
-            }
-            flush(index, true);
-            ensurePeerRecoveryRetentionLeasesRenewedAndSynced(index);
-            // less than 10% of the committed docs (see IndexSetting#FILE_BASED_RECOVERY_THRESHOLD_SETTING).
-            int uncommittedDocs = randomIntBetween(0, (int) (committedDocs * 0.1));
-            for (int i = 0; i < uncommittedDocs; i++) {
-                final String id = Integer.toString(randomIntBetween(1, 100));
-                indexDocument(id);
-            }
-        } else {
-            ensureGreen(index);
-            assertNoFileBasedRecovery(index, n -> true);
-            ensurePeerRecoveryRetentionLeasesRenewedAndSynced(index);
-        }
-    }
-
-    /**
-     * Verifies that once all shard copies on the new version, we should turn off the translog retention for indices with soft-deletes.
-     */
-    public void testTurnOffTranslogRetentionAfterUpgraded() throws Exception {
-        if (isRunningAgainstOldCluster()) {
-            createIndex(
-                index,
-                Settings.builder()
-                    .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
-                    .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
-                    .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
-                    .build()
-            );
-            ensureGreen(index);
-            int numDocs = randomIntBetween(10, 100);
-            for (int i = 0; i < numDocs; i++) {
-                indexDocument(Integer.toString(randomIntBetween(1, 100)));
-                if (rarely()) {
-                    flush(index, randomBoolean());
-                }
-            }
-        } else {
-            ensureGreen(index);
-            flush(index, true);
-            assertEmptyTranslog(index);
-            ensurePeerRecoveryRetentionLeasesRenewedAndSynced(index);
-        }
-    }
-
-    public void testResize() throws Exception {
-        int numDocs;
-        if (isRunningAgainstOldCluster()) {
-            final Settings.Builder settings = Settings.builder()
-                .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 3)
-                .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1);
-            if (minimumNodeVersion().before(Version.V_8_0_0) && randomBoolean()) {
-                settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false);
-            }
-            final String mappings = randomBoolean() ? "\"_source\": { \"enabled\": false}" : null;
-            createIndex(index, settings.build(), mappings);
-            numDocs = randomIntBetween(10, 1000);
-            for (int i = 0; i < numDocs; i++) {
-                indexDocument(Integer.toString(i));
-                if (rarely()) {
-                    flush(index, randomBoolean());
-                }
-            }
-            saveInfoDocument("num_doc_" + index, Integer.toString(numDocs));
-            ensureGreen(index);
-        } else {
-            ensureGreen(index);
-            numDocs = Integer.parseInt(loadInfoDocument("num_doc_" + index));
-            int moreDocs = randomIntBetween(0, 100);
-            for (int i = 0; i < moreDocs; i++) {
-                indexDocument(Integer.toString(numDocs + i));
-                if (rarely()) {
-                    flush(index, randomBoolean());
-                }
-            }
-            Request updateSettingsRequest = new Request("PUT", "/" + index + "/_settings");
-            updateSettingsRequest.setJsonEntity("{\"settings\": {\"index.blocks.write\": true}}");
-            client().performRequest(updateSettingsRequest);
-            {
-                final String target = index + "_shrunken";
-                Request shrinkRequest = new Request("PUT", "/" + index + "/_shrink/" + target);
-                Settings.Builder settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1);
-                if (randomBoolean()) {
-                    settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
-                }
-                shrinkRequest.setJsonEntity("{\"settings\":" + Strings.toString(settings.build()) + "}");
-                client().performRequest(shrinkRequest);
-                ensureGreenLongWait(target);
-                assertNumHits(target, numDocs + moreDocs, 1);
-            }
-            {
-                final String target = index + "_split";
-                Settings.Builder settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 6);
-                if (randomBoolean()) {
-                    settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
-                }
-                Request splitRequest = new Request("PUT", "/" + index + "/_split/" + target);
-                splitRequest.setJsonEntity("{\"settings\":" + Strings.toString(settings.build()) + "}");
-                client().performRequest(splitRequest);
-                ensureGreenLongWait(target);
-                assertNumHits(target, numDocs + moreDocs, 6);
-            }
-            {
-                final String target = index + "_cloned";
-                client().performRequest(new Request("PUT", "/" + index + "/_clone/" + target));
-                ensureGreenLongWait(target);
-                assertNumHits(target, numDocs + moreDocs, 3);
-            }
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testSystemIndexMetadataIsUpgraded() throws Exception {
-        assumeTrue(".tasks became a system index in 7.10.0", getOldClusterVersion().onOrAfter(Version.V_7_10_0));
-        final String systemIndexWarning = "this request accesses system indices: [.tasks], but in a future major version, direct "
-            + "access to system indices will be prevented by default";
-        if (isRunningAgainstOldCluster()) {
-            // create index
-            Request createTestIndex = new Request("PUT", "/test_index_old");
-            createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0}}");
-            client().performRequest(createTestIndex);
-
-            Request bulk = new Request("POST", "/_bulk");
-            bulk.addParameter("refresh", "true");
-            bulk.setJsonEntity("""
-                {"index": {"_index": "test_index_old"}}
-                {"f1": "v1", "f2": "v2"}
-                """);
-            client().performRequest(bulk);
-
-            // start a async reindex job
-            Request reindex = new Request("POST", "/_reindex");
-            reindex.setJsonEntity("""
-                {
-                  "source":{
-                    "index":"test_index_old"
-                  },
-                  "dest":{
-                    "index":"test_index_reindex"
-                  }
-                }""");
-            reindex.addParameter("wait_for_completion", "false");
-            Map<String, Object> response = entityAsMap(client().performRequest(reindex));
-            String taskId = (String) response.get("task");
-
-            // wait for task
-            Request getTask = new Request("GET", "/_tasks/" + taskId);
-            getTask.addParameter("wait_for_completion", "true");
-            client().performRequest(getTask);
-
-            // make sure .tasks index exists
-            Request getTasksIndex = new Request("GET", "/.tasks");
-            getTasksIndex.setOptions(expectVersionSpecificWarnings(v -> {
-                v.current(systemIndexWarning);
-                v.compatible(systemIndexWarning);
-            }));
-            getTasksIndex.addParameter("allow_no_indices", "false");
-
-            getTasksIndex.setOptions(expectVersionSpecificWarnings(v -> {
-                v.current(systemIndexWarning);
-                v.compatible(systemIndexWarning);
-            }));
-            assertBusy(() -> {
-                try {
-                    assertThat(client().performRequest(getTasksIndex).getStatusLine().getStatusCode(), is(200));
-                } catch (ResponseException e) {
-                    throw new AssertionError(".tasks index does not exist yet");
-                }
-            });
-
-            // If we are on 7.x create an alias that includes both a system index and a non-system index so we can be sure it gets
-            // upgraded properly. If we're already on 8.x, skip this part of the test.
-            if (minimumNodeVersion().before(SYSTEM_INDEX_ENFORCEMENT_VERSION)) {
-                // Create an alias to make sure it gets upgraded properly
-                Request putAliasRequest = new Request("POST", "/_aliases");
-                putAliasRequest.setJsonEntity("""
-                    {
-                      "actions": [
-                        {"add":  {"index":  ".tasks", "alias": "test-system-alias"}},
-                        {"add":  {"index":  "test_index_reindex", "alias": "test-system-alias"}}
-                      ]
-                    }""");
-                putAliasRequest.setOptions(expectVersionSpecificWarnings(v -> {
-                    v.current(systemIndexWarning);
-                    v.compatible(systemIndexWarning);
-                }));
-                assertThat(client().performRequest(putAliasRequest).getStatusLine().getStatusCode(), is(200));
-            }
-        } else {
-            assertBusy(() -> {
-                Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata");
-                Map<String, Object> indices = new XContentTestUtils.JsonMapView(entityAsMap(client().performRequest(clusterStateRequest)))
-                    .get("metadata.indices");
-
-                // Make sure our non-system index is still non-system
-                assertThat(new XContentTestUtils.JsonMapView(indices).get("test_index_old.system"), is(false));
-
-                // Can't get the .tasks index via JsonMapView because it splits on `.`
-                assertThat(indices, hasKey(".tasks"));
-                XContentTestUtils.JsonMapView tasksIndex = new XContentTestUtils.JsonMapView((Map<String, Object>) indices.get(".tasks"));
-                assertThat(tasksIndex.get("system"), is(true));
-
-                // If .tasks was created in a 7.x version, it should have an alias on it that we need to make sure got upgraded properly.
-                final String tasksCreatedVersionString = tasksIndex.get("settings.index.version.created");
-                assertThat(tasksCreatedVersionString, notNullValue());
-                final Version tasksCreatedVersion = Version.fromId(Integer.parseInt(tasksCreatedVersionString));
-                if (tasksCreatedVersion.before(SYSTEM_INDEX_ENFORCEMENT_VERSION)) {
-                    // Verify that the alias survived the upgrade
-                    Request getAliasRequest = new Request("GET", "/_alias/test-system-alias");
-                    getAliasRequest.setOptions(expectVersionSpecificWarnings(v -> {
-                        v.current(systemIndexWarning);
-                        v.compatible(systemIndexWarning);
-                    }));
-                    Map<String, Object> aliasResponse = entityAsMap(client().performRequest(getAliasRequest));
-                    assertThat(aliasResponse, hasKey(".tasks"));
-                    assertThat(aliasResponse, hasKey("test_index_reindex"));
-                }
-            });
-        }
-    }
-
-    public void testEnableSoftDeletesOnRestore() throws Exception {
-        assumeTrue("soft deletes must be enabled on 8.0+", getOldClusterVersion().before(Version.V_8_0_0));
-        final String snapshot = "snapshot-" + index;
-        if (isRunningAgainstOldCluster()) {
-            final Settings.Builder settings = Settings.builder()
-                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1);
-            settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false);
-            createIndex(index, settings.build());
-            ensureGreen(index);
-            int numDocs = randomIntBetween(0, 100);
-            indexRandomDocuments(
-                numDocs,
-                true,
-                true,
-                randomBoolean(),
-                i -> jsonBuilder().startObject().field("field", "value").endObject()
-            );
-            // create repo
-            XContentBuilder repoConfig = JsonXContent.contentBuilder().startObject();
-            {
-                repoConfig.field("type", "fs");
-                repoConfig.startObject("settings");
-                {
-                    repoConfig.field("compress", randomBoolean());
-                    repoConfig.field("location", System.getProperty("tests.path.repo"));
-                }
-                repoConfig.endObject();
-            }
-            repoConfig.endObject();
-            Request createRepoRequest = new Request("PUT", "/_snapshot/repo");
-            createRepoRequest.setJsonEntity(Strings.toString(repoConfig));
-            client().performRequest(createRepoRequest);
-            // create snapshot
-            Request createSnapshot = new Request("PUT", "/_snapshot/repo/" + snapshot);
-            createSnapshot.addParameter("wait_for_completion", "true");
-            createSnapshot.setJsonEntity("{\"indices\": \"" + index + "\"}");
-            client().performRequest(createSnapshot);
-        } else {
-            String restoredIndex = "restored-" + index;
-            // Restore
-            XContentBuilder restoreCommand = JsonXContent.contentBuilder().startObject();
-            restoreCommand.field("indices", index);
-            restoreCommand.field("rename_pattern", index);
-            restoreCommand.field("rename_replacement", restoredIndex);
-            restoreCommand.startObject("index_settings");
-            {
-                restoreCommand.field("index.soft_deletes.enabled", true);
-            }
-            restoreCommand.endObject();
-            restoreCommand.endObject();
-            Request restoreRequest = new Request("POST", "/_snapshot/repo/" + snapshot + "/_restore");
-            restoreRequest.addParameter("wait_for_completion", "true");
-            restoreRequest.setJsonEntity(Strings.toString(restoreCommand));
-            client().performRequest(restoreRequest);
-            ensureGreen(restoredIndex);
-            int numDocs = countOfIndexedRandomDocuments();
-            assertTotalHits(numDocs, entityAsMap(client().performRequest(new Request("GET", "/" + restoredIndex + "/_search"))));
-        }
-    }
-
-    public void testForbidDisableSoftDeletesOnRestore() throws Exception {
-        final String snapshot = "snapshot-" + index;
-        if (isRunningAgainstOldCluster()) {
-            final Settings.Builder settings = Settings.builder()
-                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
-                .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
-            createIndex(index, settings.build());
-            ensureGreen(index);
-            int numDocs = randomIntBetween(0, 100);
-            indexRandomDocuments(
-                numDocs,
-                true,
-                true,
-                randomBoolean(),
-                i -> jsonBuilder().startObject().field("field", "value").endObject()
-            );
-            // create repo
-            XContentBuilder repoConfig = JsonXContent.contentBuilder().startObject();
-            {
-                repoConfig.field("type", "fs");
-                repoConfig.startObject("settings");
-                {
-                    repoConfig.field("compress", randomBoolean());
-                    repoConfig.field("location", System.getProperty("tests.path.repo"));
-                }
-                repoConfig.endObject();
-            }
-            repoConfig.endObject();
-            Request createRepoRequest = new Request("PUT", "/_snapshot/repo");
-            createRepoRequest.setJsonEntity(Strings.toString(repoConfig));
-            client().performRequest(createRepoRequest);
-            // create snapshot
-            Request createSnapshot = new Request("PUT", "/_snapshot/repo/" + snapshot);
-            createSnapshot.addParameter("wait_for_completion", "true");
-            createSnapshot.setJsonEntity("{\"indices\": \"" + index + "\"}");
-            client().performRequest(createSnapshot);
-        } else {
-            // Restore
-            XContentBuilder restoreCommand = JsonXContent.contentBuilder().startObject();
-            restoreCommand.field("indices", index);
-            restoreCommand.field("rename_pattern", index);
-            restoreCommand.field("rename_replacement", "restored-" + index);
-            restoreCommand.startObject("index_settings");
-            {
-                restoreCommand.field("index.soft_deletes.enabled", false);
-            }
-            restoreCommand.endObject();
-            restoreCommand.endObject();
-            Request restoreRequest = new Request("POST", "/_snapshot/repo/" + snapshot + "/_restore");
-            restoreRequest.addParameter("wait_for_completion", "true");
-            restoreRequest.setJsonEntity(Strings.toString(restoreCommand));
-            final ResponseException error = expectThrows(ResponseException.class, () -> client().performRequest(restoreRequest));
-            assertThat(error.getMessage(), containsString("cannot disable setting [index.soft_deletes.enabled] on restore"));
-        }
-    }
-
-    /**
-     * In 7.14 the cluster.remote.*.transport.compress setting was change from a boolean to an enum setting
-     * with true/false as options. This test ensures that the old boolean setting in cluster state is
-     * translated properly. This test can be removed in 9.0.
-     */
-    public void testTransportCompressionSetting() throws IOException {
-        assumeTrue("the old transport.compress setting existed before 7.14", getOldClusterVersion().before(Version.V_7_14_0));
-        assumeTrue(
-            "Early versions of 6.x do not have cluster.remote* prefixed settings",
-            getOldClusterVersion().onOrAfter(Version.V_7_14_0.minimumCompatibilityVersion())
-        );
-        if (isRunningAgainstOldCluster()) {
-            final Request putSettingsRequest = new Request("PUT", "/_cluster/settings");
-            try (XContentBuilder builder = jsonBuilder()) {
-                builder.startObject();
-                {
-                    builder.startObject("persistent");
-                    {
-                        builder.field("cluster.remote.foo.seeds", Collections.singletonList("localhost:9200"));
-                        builder.field("cluster.remote.foo.transport.compress", "true");
-                    }
-                    builder.endObject();
-                }
-                builder.endObject();
-                putSettingsRequest.setJsonEntity(Strings.toString(builder));
-            }
-            client().performRequest(putSettingsRequest);
-        } else {
-            final Request getSettingsRequest = new Request("GET", "/_cluster/settings");
-            final Response getSettingsResponse = client().performRequest(getSettingsRequest);
-            try (XContentParser parser = createParser(JsonXContent.jsonXContent, getSettingsResponse.getEntity().getContent())) {
-                final Settings settings = RestClusterGetSettingsResponse.fromXContent(parser).getPersistentSettings();
-                assertThat(REMOTE_CLUSTER_COMPRESS.getConcreteSettingForNamespace("foo").get(settings), equalTo(Compression.Enabled.TRUE));
-            }
-        }
-    }
-
-    public static void assertNumHits(String index, int numHits, int totalShards) throws IOException {
-        Map<String, Object> resp = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")));
-        assertNoFailures(resp);
-        assertThat(XContentMapValues.extractValue("_shards.total", resp), equalTo(totalShards));
-        assertThat(XContentMapValues.extractValue("_shards.successful", resp), equalTo(totalShards));
-        assertThat(extractTotalHits(resp), equalTo(numHits));
-    }
-}
diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java
deleted file mode 100644
index 34ae95d23b03..000000000000
--- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.upgrades;
-
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.stream.InputStreamStreamInput;
-import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.Fuzziness;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
-import org.elasticsearch.index.query.DisMaxQueryBuilder;
-import org.elasticsearch.index.query.MatchAllQueryBuilder;
-import org.elasticsearch.index.query.MatchNoneQueryBuilder;
-import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
-import org.elasticsearch.index.query.MatchQueryBuilder;
-import org.elasticsearch.index.query.Operator;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.RangeQueryBuilder;
-import org.elasticsearch.index.query.SpanNearQueryBuilder;
-import org.elasticsearch.index.query.SpanTermQueryBuilder;
-import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
-import org.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder;
-import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Base64;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
-
-/**
- * An integration test that tests whether percolator queries stored in older supported ES version can still be read by the
- * current ES version. Percolator queries are stored in the binary format in a dedicated doc values field (see
- * PercolatorFieldMapper#createQueryBuilderField(...) method). Using the query builders writable contract. This test
- * does best effort verifying that we don't break bwc for query builders between the first previous major version and
- * the latest current major release.
- *
- * The queries to test are specified in json format, which turns out to work because we tend break here rarely. If the
- * json format of a query being tested here then feel free to change this.
- */
-public class QueryBuilderBWCIT extends AbstractFullClusterRestartTestCase {
-
-    private static final List<Object[]> CANDIDATES = new ArrayList<>();
-
-    static {
-        addCandidate("""
-            "match": { "text_field": "value"}
-            """, new MatchQueryBuilder("text_field", "value"));
-        addCandidate("""
-            "match": { "text_field": {"query": "value", "operator": "and"} }
-            """, new MatchQueryBuilder("text_field", "value").operator(Operator.AND));
-        addCandidate("""
-            "match": { "text_field": {"query": "value", "analyzer": "english"} }
-            """, new MatchQueryBuilder("text_field", "value").analyzer("english"));
-        addCandidate("""
-            "match": { "text_field": {"query": "value", "minimum_should_match": 3} }
-            """, new MatchQueryBuilder("text_field", "value").minimumShouldMatch("3"));
-        addCandidate("""
-            "match": { "text_field": {"query": "value", "fuzziness": "auto"} }
-            """, new MatchQueryBuilder("text_field", "value").fuzziness(Fuzziness.AUTO));
-        addCandidate("""
-            "match_phrase": { "text_field": "value"}
-            """, new MatchPhraseQueryBuilder("text_field", "value"));
-        addCandidate("""
-            "match_phrase": { "text_field": {"query": "value", "slop": 3}}
-            """, new MatchPhraseQueryBuilder("text_field", "value").slop(3));
-        addCandidate("""
-            "range": { "long_field": {"gte": 1, "lte": 9}}
-            """, new RangeQueryBuilder("long_field").from(1).to(9));
-        addCandidate(
-            """
-                "bool": { "must_not": [{"match_none": {}}], "must": [{"match_all": {}}], "filter": [{"match_all": {}}], \
-                "should": [{"match_all": {}}]}
-                """,
-            new BoolQueryBuilder().mustNot(new MatchNoneQueryBuilder())
-                .must(new MatchAllQueryBuilder())
-                .filter(new MatchAllQueryBuilder())
-                .should(new MatchAllQueryBuilder())
-        );
-        addCandidate(
-            """
-                "dis_max": {"queries": [{"match_all": {}},{"match_all": {}},{"match_all": {}}], "tie_breaker": 0.01}
-                """,
-            new DisMaxQueryBuilder().add(new MatchAllQueryBuilder())
-                .add(new MatchAllQueryBuilder())
-                .add(new MatchAllQueryBuilder())
-                .tieBreaker(0.01f)
-        );
-        addCandidate("""
-            "constant_score": {"filter": {"match_all": {}}, "boost": 0.1}
-            """, new ConstantScoreQueryBuilder(new MatchAllQueryBuilder()).boost(0.1f));
-        addCandidate(
-            """
-                "function_score": {"query": {"match_all": {}},"functions": [{"random_score": {}, "filter": {"match_all": {}}, \
-                "weight": 0.2}]}
-                """,
-            new FunctionScoreQueryBuilder(
-                new MatchAllQueryBuilder(),
-                new FunctionScoreQueryBuilder.FilterFunctionBuilder[] {
-                    new FunctionScoreQueryBuilder.FilterFunctionBuilder(
-                        new MatchAllQueryBuilder(),
-                        new RandomScoreFunctionBuilder().setWeight(0.2f)
-                    ) }
-            )
-        );
-        addCandidate(
-            """
-                "span_near": {"clauses": [{ "span_term": { "keyword_field": "value1" }}, \
-                { "span_term": { "keyword_field": "value2" }}]}
-                """,
-            new SpanNearQueryBuilder(new SpanTermQueryBuilder("keyword_field", "value1"), 0).addClause(
-                new SpanTermQueryBuilder("keyword_field", "value2")
-            )
-        );
-        addCandidate(
-            """
-                "span_near": {"clauses": [{ "span_term": { "keyword_field": "value1" }}, \
-                { "span_term": { "keyword_field": "value2" }}], "slop": 2}
-                """,
-            new SpanNearQueryBuilder(new SpanTermQueryBuilder("keyword_field", "value1"), 2).addClause(
-                new SpanTermQueryBuilder("keyword_field", "value2")
-            )
-        );
-        addCandidate(
-            """
-                "span_near": {"clauses": [{ "span_term": { "keyword_field": "value1" }}, \
-                { "span_term": { "keyword_field": "value2" }}], "slop": 2, "in_order": false}
-                """,
-            new SpanNearQueryBuilder(new SpanTermQueryBuilder("keyword_field", "value1"), 2).addClause(
-                new SpanTermQueryBuilder("keyword_field", "value2")
-            ).inOrder(false)
-        );
-    }
-
-    private static void addCandidate(String querySource, QueryBuilder expectedQb) {
-        CANDIDATES.add(new Object[] { "{\"query\": {" + querySource + "}}", expectedQb });
-    }
-
-    public void testQueryBuilderBWC() throws Exception {
-        String index = "queries";
-        if (isRunningAgainstOldCluster()) {
-            XContentBuilder mappingsAndSettings = jsonBuilder();
-            mappingsAndSettings.startObject();
-            {
-                mappingsAndSettings.startObject("settings");
-                mappingsAndSettings.field("number_of_shards", 1);
-                mappingsAndSettings.field("number_of_replicas", 0);
-                mappingsAndSettings.endObject();
-            }
-            {
-                mappingsAndSettings.startObject("mappings");
-                mappingsAndSettings.startObject("properties");
-                {
-                    mappingsAndSettings.startObject("query");
-                    mappingsAndSettings.field("type", "percolator");
-                    mappingsAndSettings.endObject();
-                }
-                {
-                    mappingsAndSettings.startObject("keyword_field");
-                    mappingsAndSettings.field("type", "keyword");
-                    mappingsAndSettings.endObject();
-                }
-                {
-                    mappingsAndSettings.startObject("text_field");
-                    mappingsAndSettings.field("type", "text");
-                    mappingsAndSettings.endObject();
-                }
-                {
-                    mappingsAndSettings.startObject("long_field");
-                    mappingsAndSettings.field("type", "long");
-                    mappingsAndSettings.endObject();
-                }
-                mappingsAndSettings.endObject();
-                mappingsAndSettings.endObject();
-            }
-            mappingsAndSettings.endObject();
-            Request request = new Request("PUT", "/" + index);
-            request.setJsonEntity(Strings.toString(mappingsAndSettings));
-            Response rsp = client().performRequest(request);
-            assertEquals(200, rsp.getStatusLine().getStatusCode());
-
-            for (int i = 0; i < CANDIDATES.size(); i++) {
-                request = new Request("PUT", "/" + index + "/_doc/" + Integer.toString(i));
-                request.setJsonEntity((String) CANDIDATES.get(i)[0]);
-                rsp = client().performRequest(request);
-                assertEquals(201, rsp.getStatusLine().getStatusCode());
-            }
-        } else {
-            NamedWriteableRegistry registry = new NamedWriteableRegistry(
-                new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedWriteables()
-            );
-
-            for (int i = 0; i < CANDIDATES.size(); i++) {
-                QueryBuilder expectedQueryBuilder = (QueryBuilder) CANDIDATES.get(i)[1];
-                Request request = new Request("GET", "/" + index + "/_search");
-                request.setJsonEntity(formatted("""
-                    {"query": {"ids": {"values": ["%s"]}}, "docvalue_fields": [{"field":"query.query_builder_field"}]}
-                    """, i));
-                Response rsp = client().performRequest(request);
-                assertEquals(200, rsp.getStatusLine().getStatusCode());
-                var hitRsp = (Map<?, ?>) ((List<?>) ((Map<?, ?>) responseAsMap(rsp).get("hits")).get("hits")).get(0);
-                String queryBuilderStr = (String) ((List<?>) ((Map<?, ?>) hitRsp.get("fields")).get("query.query_builder_field")).get(0);
-                byte[] qbSource = Base64.getDecoder().decode(queryBuilderStr);
-                try (InputStream in = new ByteArrayInputStream(qbSource, 0, qbSource.length)) {
-                    try (StreamInput input = new NamedWriteableAwareStreamInput(new InputStreamStreamInput(in), registry)) {
-                        input.setVersion(getOldClusterVersion());
-                        QueryBuilder queryBuilder = input.readNamedWriteable(QueryBuilder.class);
-                        assert in.read() == -1;
-                        assertEquals(expectedQueryBuilder, queryBuilder);
-                    }
-                }
-            }
-        }
-    }
-}
diff --git a/qa/logging-config/src/test/java/org/elasticsearch/common/logging/ESJsonLayoutTests.java b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/ESJsonLayoutTests.java
index d7af475e15dc..e4fe90118bd1 100644
--- a/qa/logging-config/src/test/java/org/elasticsearch/common/logging/ESJsonLayoutTests.java
+++ b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/ESJsonLayoutTests.java
@@ -7,12 +7,11 @@
  */
 package org.elasticsearch.common.logging;
 
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
 import org.junit.BeforeClass;
 
-import java.util.Locale;
-
 public class ESJsonLayoutTests extends ESTestCase {
     @BeforeClass
     public static void initNodeName() {
@@ -27,7 +26,7 @@ public void testLayout() {
         ESJsonLayout server = ESJsonLayout.newBuilder().setType("server").build();
         String conversionPattern = server.getPatternLayout().getConversionPattern();
 
-        assertThat(conversionPattern, Matchers.equalTo(String.format(Locale.ROOT, """
+        assertThat(conversionPattern, Matchers.equalTo(Strings.format("""
             {\
             "type": "server", \
             "timestamp": "%%d{yyyy-MM-dd'T'HH:mm:ss,SSSZZ}", \
@@ -45,7 +44,7 @@ public void testLayoutWithAdditionalFieldOverride() {
         String conversionPattern = server.getPatternLayout().getConversionPattern();
 
         // message field is removed as is expected to be provided by a field from a message
-        assertThat(conversionPattern, Matchers.equalTo(String.format(Locale.ROOT, """
+        assertThat(conversionPattern, Matchers.equalTo(Strings.format("""
             {\
             "type": "server", \
             "timestamp": "%%d{yyyy-MM-dd'T'HH:mm:ss,SSSZZ}", \
diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/RareTermsIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/RareTermsIT.java
index 4ca8c8770adb..a33fc01d8446 100644
--- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/RareTermsIT.java
+++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/RareTermsIT.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.hamcrest.Matchers;
 
@@ -30,10 +31,11 @@ private int indexDocs(int numDocs, int id) throws Exception {
         final Request request = new Request("POST", "/_bulk");
         final StringBuilder builder = new StringBuilder();
         for (int i = 0; i < numDocs; ++i) {
-            builder.append(formatted("""
+            Object[] args = new Object[] { index, id++, i };
+            builder.append(Strings.format("""
                 { "index" : { "_index" : "%s", "_id": "%s" } }
                 {"str_value" : "s%s"}
-                """, index, id++, i));
+                """, args));
         }
         request.setJsonEntity(builder.toString());
         assertOK(client().performRequest(request));
@@ -62,7 +64,7 @@ public void testSingleValuedString() throws Exception {
 
     private void assertNumRareTerms(int maxDocs, int rareTerms) throws IOException {
         final Request request = new Request("POST", index + "/_search");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {
               "aggs": {
                 "rareTerms": {
diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java
index 9adec5df72ea..a0f0f1319b40 100644
--- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java
+++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/SearchWithMinCompatibleSearchNodeIT.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.CheckedRunnable;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.ObjectPath;
@@ -84,7 +85,7 @@ public void testMinVersionAsNewVersion() throws Exception {
                 );
                 assertThat(responseException.getMessage(), containsString("""
                     {"error":{"root_cause":[],"type":"search_phase_execution_exception\""""));
-                assertThat(responseException.getMessage(), containsString(formatted("""
+                assertThat(responseException.getMessage(), containsString(Strings.format("""
                     caused_by":{"type":"version_mismatch_exception",\
                     "reason":"One of the shards is incompatible with the required minimum version [%s]\"""", newVersion)));
             });
diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java
index 5703b1c5e2e6..a1b71e217594 100644
--- a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java
+++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java
@@ -1221,4 +1221,21 @@ public void test500Readiness() throws Exception {
         waitForElasticsearch(installation);
         assertTrue(readinessProbe(9399));
     }
+
+    public void test600Interrupt() {
+        waitForElasticsearch(installation, "elastic", PASSWORD);
+        final Result containerLogs = getContainerLogs();
+
+        assertThat("Container logs should contain starting ...", containerLogs.stdout(), containsString("starting ..."));
+
+        final List<ProcessInfo> infos = ProcessInfo.getProcessInfo(sh, "java");
+        final int maxPid = infos.stream().map(i -> i.pid()).max(Integer::compareTo).get();
+
+        sh.run("bash -c 'kill -int " + maxPid + "'"); // send ctrl+c to all java processes
+        final Result containerLogsAfter = getContainerLogs();
+
+        assertThat("Container logs should contain stopping ...", containerLogsAfter.stdout(), containsString("stopping ..."));
+        assertThat("No errors stdout", containerLogsAfter.stdout(), not(containsString("java.security.AccessControlException:")));
+        assertThat("No errors stderr", containerLogsAfter.stderr(), not(containsString("java.security.AccessControlException:")));
+    }
 }
diff --git a/qa/repository-multi-version/src/test/java/org/elasticsearch/upgrades/MultiVersionRepositoryAccessIT.java b/qa/repository-multi-version/src/test/java/org/elasticsearch/upgrades/MultiVersionRepositoryAccessIT.java
index 0a83c293dca0..59741fd105b6 100644
--- a/qa/repository-multi-version/src/test/java/org/elasticsearch/upgrades/MultiVersionRepositoryAccessIT.java
+++ b/qa/repository-multi-version/src/test/java/org/elasticsearch/upgrades/MultiVersionRepositoryAccessIT.java
@@ -280,7 +280,7 @@ private static void createSnapshot(String repoName, String name, String index) t
 
     private void createIndex(String name, int shards) throws IOException {
         final Request putIndexRequest = new Request("PUT", "/" + name);
-        putIndexRequest.setJsonEntity(formatted("""
+        putIndexRequest.setJsonEntity(Strings.format("""
             {
                 "settings" : {
                     "index" : {
diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
index 2de2786cfa92..d3078dd8c938 100644
--- a/qa/rolling-upgrade/build.gradle
+++ b/qa/rolling-upgrade/build.gradle
@@ -8,6 +8,7 @@
 
 
 import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.internal.BwcVersions
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
@@ -38,7 +39,9 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
     setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
     setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
     setting 'xpack.security.enabled', 'false'
+    setting 'logger.org.elasticsearch.cluster.service.MasterService', 'TRACE'
     setting 'logger.org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceShardsAllocator', 'TRACE'
+    setting 'logger.org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders', 'TRACE'
     requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
   }
 
@@ -51,10 +54,19 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
     doFirst {
       delete("${buildDir}/cluster/shared/repo/${baseName}")
     }
+    def excludeList = []
     systemProperty 'tests.rest.suite', 'old_cluster'
     systemProperty 'tests.upgrade_from_version', oldVersion
     nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
     nonInputProperties.systemProperty('tests.clustername', baseName)
+    if (bwcVersion.before("8.4.0")) {
+      excludeList.addAll(["old_cluster/30_vector_search/*"])
+    } else if (bwcVersion.before("8.6.0")) {
+      excludeList.addAll(["old_cluster/30_vector_search/Create indexed byte vectors and search"])
+    }
+    if (excludeList.isEmpty() == false) {
+      systemProperty 'tests.rest.blacklist', excludeList.join(',')
+    }
   }
 
   tasks.register("${baseName}#oneThirdUpgradedTest", StandaloneRestIntegTestTask) {
@@ -68,6 +80,15 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
     systemProperty 'tests.first_round', 'true'
     nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
     nonInputProperties.systemProperty('tests.clustername', baseName)
+    def excludeList = []
+    if (bwcVersion.before("8.4.0")) {
+      excludeList.addAll(["mixed_cluster/30_vector_search/*"])
+    } else if (bwcVersion.before("8.6.0")) {
+      excludeList.addAll(["mixed_cluster/30_vector_search/Search byte indices created in old cluster"])
+    }
+    if (excludeList.isEmpty() == false) {
+      systemProperty 'tests.rest.blacklist', excludeList.join(',')
+    }
   }
 
   tasks.register("${baseName}#twoThirdsUpgradedTest", StandaloneRestIntegTestTask) {
@@ -81,6 +102,15 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
     systemProperty 'tests.first_round', 'false'
     nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
     nonInputProperties.systemProperty('tests.clustername', baseName)
+    def excludeList = []
+    if (bwcVersion.before("8.4.0")) {
+      excludeList.addAll(["mixed_cluster/30_vector_search/*"])
+    } else if (bwcVersion.before("8.6.0")) {
+      excludeList.addAll(["mixed_cluster/30_vector_search/Search byte indices created in old cluster"])
+    }
+    if (excludeList.isEmpty() == false) {
+      systemProperty 'tests.rest.blacklist', excludeList.join(',')
+    }
   }
 
   tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) {
@@ -93,6 +123,15 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
     systemProperty 'tests.upgrade_from_version', oldVersion
     nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
     nonInputProperties.systemProperty('tests.clustername', baseName)
+    def excludeList = []
+    if (bwcVersion.before("8.4.0")) {
+      excludeList.addAll(["upgraded_cluster/30_vector_search/*"])
+    } else if (bwcVersion.before("8.6.0")) {
+      excludeList.addAll(["upgraded_cluster/30_vector_search/Search byte indices created in old cluster"])
+    }
+    if (excludeList.isEmpty() == false) {
+      systemProperty 'tests.rest.blacklist', excludeList.join(',')
+    }
   }
 
   tasks.register(bwcTaskName(bwcVersion)) {
diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java
index e0c42429f71c..2cbfe030be09 100644
--- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java
+++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java
@@ -43,9 +43,9 @@ public void testUpgradeDesiredNodes() throws Exception {
             return;
         }
 
-        if (UPGRADE_FROM_VERSION.onOrAfter(Processors.DOUBLE_PROCESSORS_SUPPORT_VERSION)) {
+        if (UPGRADE_FROM_VERSION.transportVersion.onOrAfter(Processors.DOUBLE_PROCESSORS_SUPPORT_VERSION)) {
             assertUpgradedNodesCanReadDesiredNodes();
-        } else if (UPGRADE_FROM_VERSION.onOrAfter(DesiredNode.RANGE_FLOAT_PROCESSORS_SUPPORT_VERSION)) {
+        } else if (UPGRADE_FROM_VERSION.transportVersion.onOrAfter(DesiredNode.RANGE_FLOAT_PROCESSORS_SUPPORT_VERSION)) {
             assertDesiredNodesUpdatedWithRoundedUpFloatsAreIdempotent();
         } else {
             assertDesiredNodesWithFloatProcessorsAreRejectedInOlderVersions();
diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java
index ee94757e3ab2..b860e53d447b 100644
--- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java
+++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java
@@ -135,7 +135,7 @@ public void testIndexing() throws IOException {
 
     public void testAutoIdWithOpTypeCreate() throws IOException {
         final String indexName = "auto_id_and_op_type_create_index";
-        String b = formatted("""
+        String b = Strings.format("""
             {"create": {"_index": "%s"}}
             {"f1": "v"}
             """, indexName);
@@ -325,7 +325,7 @@ private void tsdbBulk(StringBuilder bulk, String dim, long timeStart, long timeE
         long delta = TimeUnit.SECONDS.toMillis(20);
         double value = (timeStart - TSDB_TIMES[0]) / TimeUnit.SECONDS.toMillis(20) * rate;
         for (long t = timeStart; t < timeEnd; t += delta) {
-            bulk.append(formatted("""
+            bulk.append(Strings.format("""
                 {"index": {"_index": "tsdb"}}
                 {"@timestamp": %s, "dim": "%s", "value": %s}
                 """, t, dim, value));
diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
index 330b0e485d6b..5e0137e14308 100644
--- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
+++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
@@ -40,6 +40,7 @@
 import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING;
 import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING;
 import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY;
+import static org.elasticsearch.upgrades.UpgradeWithOldIndexSettingsIT.updateIndexSettingsPermittingSlowlogDeprecationWarning;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.in;
@@ -236,7 +237,7 @@ public void testRelocationWithConcurrentIndexing() throws Exception {
                 final String newNode = getNodeId(v -> v.equals(Version.CURRENT));
                 final String oldNode = getNodeId(v -> v.before(Version.CURRENT));
                 // remove the replica and guaranteed the primary is placed on the old node
-                updateIndexSettings(
+                updateIndexSettingsPermittingSlowlogDeprecationWarning(
                     index,
                     Settings.builder()
                         .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
@@ -246,7 +247,10 @@ public void testRelocationWithConcurrentIndexing() throws Exception {
                 );
                 ensureGreen(index); // wait for the primary to be assigned
                 ensureNoInitializingShards(); // wait for all other shard activity to finish
-                updateIndexSettings(index, Settings.builder().put("index.routing.allocation.include._id", newNode));
+                updateIndexSettingsPermittingSlowlogDeprecationWarning(
+                    index,
+                    Settings.builder().put("index.routing.allocation.include._id", newNode)
+                );
                 asyncIndexDocs(index, 10, 50).get();
                 // ensure the relocation from old node to new node has occurred; otherwise ensureGreen can
                 // return true even though shards haven't moved to the new node yet (allocation was throttled).
diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java
index ba61b703f12a..07afe0cc5693 100644
--- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java
+++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java
@@ -31,6 +31,7 @@
 
 import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING;
 import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY;
+import static org.elasticsearch.upgrades.UpgradeWithOldIndexSettingsIT.updateIndexSettingsPermittingSlowlogDeprecationWarning;
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.equalTo;
@@ -40,6 +41,8 @@
 import static org.hamcrest.Matchers.notNullValue;
 
 public class SnapshotBasedRecoveryIT extends AbstractRollingTestCase {
+
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/93271")
     public void testSnapshotBasedRecovery() throws Exception {
         final String indexName = "snapshot_based_recovery";
         final String repositoryName = "snapshot_based_recovery_repo";
@@ -76,8 +79,12 @@ public void testSnapshotBasedRecovery() throws Exception {
                     if (upgradedNodeIds.isEmpty() == false) {
                         assertThat(upgradedNodeIds.size(), is(equalTo(1)));
                         String upgradedNodeId = upgradedNodeIds.get(0);
+                        logger.info("--> excluding [{}] from node [{}]", indexName, upgradedNodeId);
                         updateIndexSettings(indexName, Settings.builder().put("index.routing.allocation.exclude._id", upgradedNodeId));
                         ensureGreen(indexName);
+                        logger.info("--> finished excluding [{}] from node [{}]", indexName, upgradedNodeId);
+                    } else {
+                        logger.info("--> no upgrading nodes, not adding any exclusions for [{}]", indexName);
                     }
 
                     String primaryNodeId = getPrimaryNodeIdOfShard(indexName, 0);
@@ -90,19 +97,37 @@ public void testSnapshotBasedRecovery() throws Exception {
                     // the primary to a node in the old version, this allows adding replicas in the first mixed round.
                     logger.info("--> Primary node in first mixed round {} / {}", primaryNodeId, primaryNodeVersion);
                     if (primaryNodeVersion.after(UPGRADE_FROM_VERSION)) {
+                        logger.info("--> cancelling primary shard on node [{}]", primaryNodeId);
                         cancelShard(indexName, 0, primaryNodeId);
+                        logger.info("--> done cancelling primary shard on node [{}]", primaryNodeId);
 
                         String currentPrimaryNodeId = getPrimaryNodeIdOfShard(indexName, 0);
                         assertThat(getNodeVersion(currentPrimaryNodeId), is(equalTo(UPGRADE_FROM_VERSION)));
                     }
                 } else {
+                    logger.info("--> not in first upgrade round, removing exclusions for [{}]", indexName);
                     updateIndexSettings(indexName, Settings.builder().putNull("index.routing.allocation.exclude._id"));
+                    logger.info("--> done removing exclusions for [{}]", indexName);
                 }
 
                 // Drop replicas
-                updateIndexSettings(indexName, Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0));
-                updateIndexSettings(indexName, Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1));
-                ensureGreen(indexName);
+                logger.info("--> dropping replicas from [{}]", indexName);
+                updateIndexSettingsPermittingSlowlogDeprecationWarning(
+                    indexName,
+                    Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
+                );
+                logger.info("--> finished dropping replicas from [{}], adding them back", indexName);
+                updateIndexSettingsPermittingSlowlogDeprecationWarning(
+                    indexName,
+                    Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
+                );
+                logger.info("--> finished adding replicas from [{}]", indexName);
+                try {
+                    ensureGreen(indexName);
+                } catch (AssertionError e) {
+                    logAllocationExplain();
+                    throw e;
+                }
                 assertMatchAllReturnsAllDocuments(indexName, numDocs);
                 assertMatchQueryReturnsAllDocuments(indexName, numDocs);
             }
@@ -110,6 +135,20 @@ public void testSnapshotBasedRecovery() throws Exception {
         }
     }
 
+    private void logAllocationExplain() throws Exception {
+        // Used to debug #91383
+        var request = new Request(HttpGet.METHOD_NAME, "_cluster/allocation/explain?include_disk_info=true&include_yes_decisions=true");
+        request.setJsonEntity("""
+                    {
+                      "index": "snapshot_based_recovery",
+                      "shard": 0,
+                      "primary": false
+                    }
+            """);
+        var response = client().performRequest(request);
+        logger.info("--> allocation explain {}", EntityUtils.toString(response.getEntity()));
+    }
+
     private List<String> getUpgradedNodeIds() throws IOException {
         Request request = new Request(HttpGet.METHOD_NAME, "_nodes/_all");
         Response response = client().performRequest(request);
@@ -178,7 +217,7 @@ private void cancelShard(String indexName, int shard, String nodeName) throws IO
             }
             builder.endObject();
 
-            Request request = new Request(HttpPost.METHOD_NAME, "/_cluster/reroute?pretty");
+            Request request = new Request(HttpPost.METHOD_NAME, "/_cluster/reroute?pretty&metric=none");
             request.setJsonEntity(Strings.toString(builder));
             Response response = client().performRequest(request);
             logger.info("--> Relocated primary to an older version {}", EntityUtils.toString(response.getEntity()));
diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java
index a261720e266d..35688e7c244c 100644
--- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java
+++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java
@@ -12,10 +12,11 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 
 import java.io.IOException;
-import java.util.Locale;
 import java.util.Map;
 
 import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM;
@@ -56,7 +57,7 @@ public void testOldIndexSettings() throws Exception {
                 if (UPGRADE_FROM_VERSION.before(Version.V_8_0_0)) {
                     bulk.setOptions(expectWarnings(EXPECTED_WARNING));
                 }
-                bulk.setJsonEntity(String.format(Locale.ROOT, """
+                bulk.setJsonEntity(Strings.format("""
                     {"index": {"_index": "%s"}}
                     {"f1": "v1", "f2": "v2"}
                     """, INDEX_NAME));
@@ -69,7 +70,7 @@ public void testOldIndexSettings() throws Exception {
                 if (UPGRADE_FROM_VERSION.before(Version.V_8_0_0)) {
                     bulk.setOptions(expectWarnings(EXPECTED_WARNING));
                 }
-                bulk.setJsonEntity(String.format(Locale.ROOT, """
+                bulk.setJsonEntity(Strings.format("""
                     {"index": {"_index": "%s"}}
                     {"f1": "v3", "f2": "v4"}
                     """, INDEX_NAME));
@@ -111,4 +112,20 @@ private void assertCount(String index, int countAtLeast) throws IOException {
 
         assertTrue(hitsTotal >= countAtLeast);
     }
+
+    public static void updateIndexSettingsPermittingSlowlogDeprecationWarning(String index, Settings.Builder settings) throws IOException {
+        Request request = new Request("PUT", "/" + index + "/_settings");
+        request.setJsonEntity(org.elasticsearch.common.Strings.toString(settings.build()));
+        if (UPGRADE_FROM_VERSION.before(Version.V_7_17_9)) {
+            // There is a bug (fixed in 7.17.9 and 8.7.0 where deprecation warnings could leak into ClusterApplierService#applyChanges)
+            // Below warnings are set (and leaking) from an index in this test case
+            request.setOptions(expectVersionSpecificWarnings(v -> {
+                v.compatible(
+                    "[index.indexing.slowlog.level] setting was deprecated in Elasticsearch and will be removed in a future release! "
+                        + "See the breaking changes documentation for the next major version."
+                );
+            }));
+        }
+        client().performRequest(request);
+    }
 }
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_vector_search.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_vector_search.yml
new file mode 100644
index 000000000000..969c4428c7c6
--- /dev/null
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_vector_search.yml
@@ -0,0 +1,142 @@
+---
+"Search float indices created in old cluster":
+  - do:
+      search:
+        index: test-float-index
+        body:
+          query:
+            script_score:
+              query: { "exists": { "field": "bdv" } }
+              script:
+                source: |
+                  field(params.field).get().dotProduct(params.query)
+                params:
+                  query: [4, 5, 6]
+                  field: bdv
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.0._score: 27 }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.1._score: 21 }
+  - match: { hits.hits.2._id: "1" }
+  - match: { hits.hits.2._score: 15 }
+
+  - do:
+      search:
+        index: test-float-index
+        body:
+          query:
+            script_score:
+              query: { "exists": { "field": "knn" } }
+              script:
+                source: |
+                  field(params.field).get().dotProduct(params.query)
+                params:
+                  query: [4, 5, 6]
+                  field: knn
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.0._score: 27 }
+  - match: { hits.hits.1._id: "5" }
+  - match: { hits.hits.1._score: 25 }
+  - match: { hits.hits.2._id: "7" }
+  - match: { hits.hits.2._score: 23 }
+  - do:
+      search:
+        index: test-float-index
+        body:
+          knn:
+            field: "knn"
+            query_vector: [4, 5, 6]
+            k: 3
+            num_candidates: 7
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "5" }
+  - match: { hits.hits.2._id: "2" }
+
+  - do:
+      search:
+        index: test-float-index
+        body:
+          knn:
+            field: "knn"
+            query_vector: [ 4, 5, 6 ]
+            k: 3
+            num_candidates: 6
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "5" }
+  - match: { hits.hits.2._id: "2" }
+
+---
+"Search byte indices created in old cluster":
+  - do:
+      search:
+        index: test-byte-index
+        body:
+          query:
+            script_score:
+              query: { "exists": { "field": "bdv" } }
+              script:
+                source: |
+                  field(params.field).get().dotProduct(params.query)
+                params:
+                  query: [4, 5, 6]
+                  field: bdv
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.0._score: 27 }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.1._score: 21 }
+  - match: { hits.hits.2._id: "1" }
+  - match: { hits.hits.2._score: 15 }
+
+  - do:
+      search:
+        index: test-byte-index
+        body:
+          query:
+            script_score:
+              query: { "exists": { "field": "knn" } }
+              script:
+                source: |
+                  field(params.field).get().dotProduct(params.query)
+                params:
+                  query: [4, 5, 6]
+                  field: knn
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.0._score: 27 }
+  - match: { hits.hits.1._id: "5" }
+  - match: { hits.hits.1._score: 25 }
+  - match: { hits.hits.2._id: "7" }
+  - match: { hits.hits.2._score: 23 }
+
+  - do:
+      search:
+        index: test-byte-index
+        body:
+          knn:
+            field: "knn"
+            query_vector: [4, 5, 6]
+            k: 3
+            num_candidates: 6
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "5" }
+  - match: { hits.hits.2._id: "2" }
+
+  - do:
+      search:
+        index: test-byte-index
+        body:
+          knn:
+            field: "knn"
+            query_vector: [4, 5, 6]
+            k: 3
+            num_candidates: 7
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "5" }
+  - match: { hits.hits.2._id: "2" }
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_vector_search.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_vector_search.yml
new file mode 100644
index 000000000000..b471fa56a47a
--- /dev/null
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_vector_search.yml
@@ -0,0 +1,228 @@
+---
+"Create indexed float vectors and search":
+  - do:
+      indices.create:
+        index: test-float-index
+        body:
+          settings:
+            number_of_shards: "1"
+          mappings:
+            properties:
+              bdv:
+                type: dense_vector
+                dims: 3
+              knn:
+                type: dense_vector
+                dims: 3
+                index: true
+                similarity: l2_norm
+  - do:
+      bulk:
+        index: test-float-index
+        refresh: true
+        body:
+          - '{"index": {"_id": "1"}}'
+          - '{"bdv": [1, 1, 1], "knn": [1, 1, 1]}'
+          - '{"index": {"_id": "2"}}'
+          - '{"bdv": [1, 1, 2], "knn": [1, 1, 2]}'
+          - '{"index": {"_id": "3"}}'
+          - '{"bdv": [1, 1, 3], "knn": [1, 1, 3]}'
+          - '{"index": {"_id": "4"}}'
+          - '{"knn": [1, 2, 1]}'
+          - '{"index": {"_id": "5"}}'
+          - '{"knn": [1, 3, 1]}'
+          - '{"index": {"_id": "6"}}'
+          - '{"knn": [2, 1, 1]}'
+          - '{"index": {"_id": "7"}}'
+          - '{"knn": [3, 1, 1]}'
+          - '{"index": {"_id": "missing_vector"}}'
+          - '{}'
+  - do:
+      indices.forcemerge:
+        index: test-float-index
+        max_num_segments: 1
+  - do:
+      search:
+        index: test-float-index
+        body:
+          query:
+            script_score:
+              query: { "exists": { "field": "bdv" } }
+              script:
+                source: |
+                  field(params.field).get().dotProduct(params.query)
+                params:
+                  query: [4, 5, 6]
+                  field: bdv
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.0._score: 27 }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.1._score: 21 }
+  - match: { hits.hits.2._id: "1" }
+  - match: { hits.hits.2._score: 15 }
+
+  - do:
+      search:
+        index: test-float-index
+        body:
+          query:
+            script_score:
+              query: { "exists": { "field": "knn" } }
+              script:
+                source: |
+                  field(params.field).get().dotProduct(params.query)
+                params:
+                  query: [4, 5, 6]
+                  field: knn
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.0._score: 27 }
+  - match: { hits.hits.1._id: "5" }
+  - match: { hits.hits.1._score: 25 }
+  - match: { hits.hits.2._id: "7" }
+  - match: { hits.hits.2._score: 23 }
+
+  - do:
+      search:
+        index: test-float-index
+        body:
+          knn:
+            field: "knn"
+            query_vector: [4, 5, 6]
+            k: 3
+            num_candidates: 7
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "5" }
+  - match: { hits.hits.2._id: "2" }
+
+  - do:
+      search:
+        index: test-float-index
+        body:
+          knn:
+            field: "knn"
+            query_vector: [ 4, 5, 6 ]
+            k: 3
+            num_candidates: 6
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "5" }
+  - match: { hits.hits.2._id: "2" }
+
+
+---
+"Create indexed byte vectors and search":
+  - do:
+      indices.create:
+        index: test-byte-index
+        body:
+          settings:
+            number_of_shards: "1"
+          mappings:
+            properties:
+              bdv:
+                type: dense_vector
+                element_type: byte
+                dims: 3
+              knn:
+                type: dense_vector
+                element_type: byte
+                dims: 3
+                index: true
+                similarity: l2_norm
+  - do:
+      bulk:
+        index: test-byte-index
+        refresh: true
+        body:
+          - '{"index": {"_id": "1"}}'
+          - '{"bdv": [1, 1, 1], "knn": [1, 1, 1]}'
+          - '{"index": {"_id": "2"}}'
+          - '{"bdv": [1, 1, 2], "knn": [1, 1, 2]}'
+          - '{"index": {"_id": "3"}}'
+          - '{"bdv": [1, 1, 3], "knn": [1, 1, 3]}'
+          - '{"index": {"_id": "4"}}'
+          - '{"knn": [1, 2, 1]}'
+          - '{"index": {"_id": "5"}}'
+          - '{"knn": [1, 3, 1]}'
+          - '{"index": {"_id": "6"}}'
+          - '{"knn": [2, 1, 1]}'
+          - '{"index": {"_id": "7"}}'
+          - '{"knn": [3, 1, 1]}'
+          - '{"index": {"_id": "missing_vector"}}'
+          - '{}'
+  - do:
+      indices.forcemerge:
+        index: test-byte-index
+        max_num_segments: 1
+  - do:
+      search:
+        index: test-byte-index
+        body:
+          query:
+            script_score:
+              query: { "exists": { "field": "bdv" } }
+              script:
+                source: |
+                  field(params.field).get().dotProduct(params.query)
+                params:
+                  query: [4, 5, 6]
+                  field: bdv
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.0._score: 27 }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.1._score: 21 }
+  - match: { hits.hits.2._id: "1" }
+  - match: { hits.hits.2._score: 15 }
+
+  - do:
+      search:
+        index: test-byte-index
+        body:
+          query:
+            script_score:
+              query: { "exists": { "field": "knn" } }
+              script:
+                source: |
+                  field(params.field).get().dotProduct(params.query)
+                params:
+                  query: [4, 5, 6]
+                  field: knn
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.0._score: 27 }
+  - match: { hits.hits.1._id: "5" }
+  - match: { hits.hits.1._score: 25 }
+  - match: { hits.hits.2._id: "7" }
+  - match: { hits.hits.2._score: 23 }
+
+  - do:
+      search:
+        index: test-byte-index
+        body:
+          knn:
+            field: "knn"
+            query_vector: [4, 5, 6]
+            k: 3
+            num_candidates: 6
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "5" }
+  - match: { hits.hits.2._id: "2" }
+
+  - do:
+      search:
+        index: test-byte-index
+        body:
+          knn:
+            field: "knn"
+            query_vector: [4, 5, 6]
+            k: 3
+            num_candidates: 7
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "5" }
+  - match: { hits.hits.2._id: "2" }
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_vector_search.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_vector_search.yml
new file mode 100644
index 000000000000..60304a0078ac
--- /dev/null
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_vector_search.yml
@@ -0,0 +1,143 @@
+---
+"Search float indices created in old cluster":
+  - do:
+      search:
+        index: test-float-index
+        body:
+          query:
+            script_score:
+              query: { "exists": { "field": "bdv" } }
+              script:
+                source: |
+                  field(params.field).get().dotProduct(params.query)
+                params:
+                  query: [4, 5, 6]
+                  field: bdv
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.0._score: 27 }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.1._score: 21 }
+  - match: { hits.hits.2._id: "1" }
+  - match: { hits.hits.2._score: 15 }
+
+  - do:
+      search:
+        index: test-float-index
+        body:
+          query:
+            script_score:
+              query: { "exists": { "field": "knn" } }
+              script:
+                source: |
+                  field(params.field).get().dotProduct(params.query)
+                params:
+                  query: [4, 5, 6]
+                  field: knn
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.0._score: 27 }
+  - match: { hits.hits.1._id: "5" }
+  - match: { hits.hits.1._score: 25 }
+  - match: { hits.hits.2._id: "7" }
+  - match: { hits.hits.2._score: 23 }
+
+  - do:
+      search:
+        index: test-float-index
+        body:
+          knn:
+            field: "knn"
+            query_vector: [4, 5, 6]
+            k: 3
+            num_candidates: 7
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "5" }
+  - match: { hits.hits.2._id: "2" }
+
+  - do:
+      search:
+        index: test-float-index
+        body:
+          knn:
+            field: "knn"
+            query_vector: [ 4, 5, 6 ]
+            k: 3
+            num_candidates: 6
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "5" }
+  - match: { hits.hits.2._id: "2" }
+
+---
+"Search byte indices created in old cluster":
+  - do:
+      search:
+        index: test-byte-index
+        body:
+          query:
+            script_score:
+              query: { "exists": { "field": "bdv" } }
+              script:
+                source: |
+                  field(params.field).get().dotProduct(params.query)
+                params:
+                  query: [4, 5, 6]
+                  field: bdv
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.0._score: 27 }
+  - match: { hits.hits.1._id: "2" }
+  - match: { hits.hits.1._score: 21 }
+  - match: { hits.hits.2._id: "1" }
+  - match: { hits.hits.2._score: 15 }
+
+  - do:
+      search:
+        index: test-byte-index
+        body:
+          query:
+            script_score:
+              query: { "exists": { "field": "knn" } }
+              script:
+                source: |
+                  field(params.field).get().dotProduct(params.query)
+                params:
+                  query: [4, 5, 6]
+                  field: knn
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.0._score: 27 }
+  - match: { hits.hits.1._id: "5" }
+  - match: { hits.hits.1._score: 25 }
+  - match: { hits.hits.2._id: "7" }
+  - match: { hits.hits.2._score: 23 }
+
+  - do:
+      search:
+        index: test-byte-index
+        body:
+          knn:
+            field: "knn"
+            query_vector: [4, 5, 6]
+            k: 3
+            num_candidates: 6
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "5" }
+  - match: { hits.hits.2._id: "2" }
+
+  - do:
+      search:
+        index: test-byte-index
+        body:
+          knn:
+            field: "knn"
+            query_vector: [4, 5, 6]
+            k: 3
+            num_candidates: 7
+
+  - match: { hits.hits.0._id: "3" }
+  - match: { hits.hits.1._id: "5" }
+  - match: { hits.hits.2._id: "2" }
diff --git a/qa/smoke-test-http/build.gradle b/qa/smoke-test-http/build.gradle
index c08194f7db87..6766823dc21e 100644
--- a/qa/smoke-test-http/build.gradle
+++ b/qa/smoke-test-http/build.gradle
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 //apply plugin: 'elasticsearch.test-with-dependencies'
 
 dependencies {
diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterStateRestCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterStateRestCancellationIT.java
index a0a4642d25ee..0830cc1f6640 100644
--- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterStateRestCancellationIT.java
+++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/ClusterStateRestCancellationIT.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.http;
 
 import org.apache.http.client.methods.HttpGet;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.Cancellable;
@@ -24,10 +24,11 @@
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.tasks.TaskInfo;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.CancellationException;
 import java.util.function.UnaryOperator;
@@ -106,8 +107,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT;
         }
 
         @Override
@@ -116,7 +117,7 @@ public void writeTo(StreamOutput out) {
         }
 
         @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) {
+        public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
             throw new AssertionError("task should have been cancelled before serializing this custom");
         }
     }
diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HealthRestCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HealthRestCancellationIT.java
index e0d6a75495de..cd570654b442 100644
--- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HealthRestCancellationIT.java
+++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HealthRestCancellationIT.java
@@ -47,7 +47,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     }
 
     public void testHealthRestCancellation() throws Exception {
-        runTest(new Request(HttpGet.METHOD_NAME, "/_internal/_health"));
+        runTest(new Request(HttpGet.METHOD_NAME, "/_health_report"));
     }
 
     private void runTest(Request request) throws Exception {
@@ -111,7 +111,7 @@ public String name() {
         }
 
         @Override
-        public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
+        public HealthIndicatorResult calculate(boolean verbose, int maxAffectedResourcesCount, HealthInfo healthInfo) {
             try {
                 operationBlock.acquire();
             } catch (InterruptedException e) {
diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java
index 0594191ed3c8..43d7630199bb 100644
--- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java
+++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestClusterInfoActionCancellationIT.java
@@ -26,7 +26,7 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.hamcrest.Matchers;
 
 import java.util.EnumSet;
 import java.util.concurrent.CancellationException;
@@ -35,11 +35,10 @@
 import static org.elasticsearch.action.support.ActionTestUtils.wrapAsRestResponseListener;
 import static org.elasticsearch.test.TaskAssertions.assertAllCancellableTasksAreCancelled;
 import static org.elasticsearch.test.TaskAssertions.assertAllTasksHaveFinished;
-import static org.elasticsearch.test.TaskAssertions.awaitTaskWithPrefix;
+import static org.elasticsearch.test.TaskAssertions.awaitTaskWithPrefixOnMaster;
 import static org.hamcrest.core.IsEqual.equalTo;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
-@TestLogging(value = "org.elasticsearch.tasks.TaskManager:TRACE,org.elasticsearch.test.TaskAssertions:TRACE", reason = "debugging")
 public class RestClusterInfoActionCancellationIT extends HttpSmokeTestCase {
 
     public void testGetMappingsCancellation() throws Exception {
@@ -77,8 +76,18 @@ private void runTest(String actionName, String endpoint) throws Exception {
         final Cancellable cancellable = getRestClient().performRequestAsync(request, wrapAsRestResponseListener(future));
 
         assertThat(future.isDone(), equalTo(false));
-        awaitTaskWithPrefix(actionName);
-
+        awaitTaskWithPrefixOnMaster(actionName);
+        // To ensure that the task is executing on master, we wait until the first blocked execution of the task registers its cluster state
+        // observer for further retries. This ensures that a task is not cancelled before we have started its execution, which could result
+        // in the task being unregistered and the test not being able to find any cancelled tasks.
+        assertBusy(
+            () -> assertThat(
+                internalCluster().getCurrentMasterNodeInstance(ClusterService.class)
+                    .getClusterApplierService()
+                    .getTimeoutClusterStateListenersSize(),
+                Matchers.greaterThan(0)
+            )
+        );
         cancellable.cancel();
         assertAllCancellableTasksAreCancelled(actionName);
 
diff --git a/qa/smoke-test-ingest-disabled/build.gradle b/qa/smoke-test-ingest-disabled/build.gradle
index ad539bb515e1..5ffefed52b71 100644
--- a/qa/smoke-test-ingest-disabled/build.gradle
+++ b/qa/smoke-test-ingest-disabled/build.gradle
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 
 dependencies {
   testImplementation project(':modules:ingest-common')
diff --git a/qa/smoke-test-ingest-disabled/src/yamlRestTest/resources/rest-api-spec/test/ingest_mustache/10_ingest_disabled.yml b/qa/smoke-test-ingest-disabled/src/yamlRestTest/resources/rest-api-spec/test/ingest_mustache/10_ingest_disabled.yml
index ed3c5f6f9228..0eb64954a45b 100644
--- a/qa/smoke-test-ingest-disabled/src/yamlRestTest/resources/rest-api-spec/test/ingest_mustache/10_ingest_disabled.yml
+++ b/qa/smoke-test-ingest-disabled/src/yamlRestTest/resources/rest-api-spec/test/ingest_mustache/10_ingest_disabled.yml
@@ -28,7 +28,7 @@
   - match: { acknowledged: true }
 
 ---
-"Test ingest simulate API works fine when node.ingest is set to false":
+"Test ingest simulate API fails when node.ingest is set to false":
   - do:
       ingest.put_pipeline:
         id: "my_pipeline"
@@ -47,6 +47,7 @@
   - match: { acknowledged: true }
 
   - do:
+      catch: /There are no ingest nodes in this cluster, unable to forward request to an ingest node./
       ingest.simulate:
         id: "my_pipeline"
         body: >
@@ -61,11 +62,6 @@
               }
             ]
           }
-  - length: { docs: 1 }
-  - match: { docs.0.doc._source.foo: "bar" }
-  - match: { docs.0.doc._source.field2: "_value" }
-  - length: { docs.0.doc._ingest: 1 }
-  - is_true: docs.0.doc._ingest.timestamp
 
 ---
 "Test index api with pipeline id fails when node.ingest is set to false":
diff --git a/qa/smoke-test-ingest-with-all-dependencies/build.gradle b/qa/smoke-test-ingest-with-all-dependencies/build.gradle
index 1f043272b417..6181548757c6 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/build.gradle
+++ b/qa/smoke-test-ingest-with-all-dependencies/build.gradle
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 
 dependencies {
   yamlRestTestImplementation project(':modules:lang-mustache')
@@ -24,4 +24,4 @@ tasks.named("yamlRestTestTestingConventions").configure {
 
 tasks.named("forbiddenPatterns").configure {
   exclude '**/*.mmdb'
-}
\ No newline at end of file
+}
diff --git a/qa/smoke-test-multinode/build.gradle b/qa/smoke-test-multinode/build.gradle
index 37615f938213..acead4425134 100644
--- a/qa/smoke-test-multinode/build.gradle
+++ b/qa/smoke-test-multinode/build.gradle
@@ -5,11 +5,6 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-
-
-import org.elasticsearch.gradle.Version
-import org.elasticsearch.gradle.internal.info.BuildParams
-
 apply plugin: 'elasticsearch.internal-yaml-rest-test'
 
 restResources {
@@ -18,22 +13,12 @@ restResources {
   }
 }
 
-File repo = file("$buildDir/testclusters/repo")
-testClusters.matching { it.name == "yamlRestTest" }.configureEach {
-  numberOfNodes = 2
-  setting 'path.repo', repo.absolutePath
-}
-
-testClusters.configureEach {
-  setting 'xpack.security.enabled', 'false'
-  requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
+dependencies {
+  clusterModules project(":modules:mapper-extras")
+  clusterModules project(":modules:ingest-common")
 }
 
 tasks.named("yamlRestTest").configure {
-  doFirst {
-    project.delete(repo)
-    repo.mkdirs()
-  }
   systemProperty 'tests.rest.blacklist', [
     'cat.templates/10_basic/No templates',
     'cat.templates/10_basic/Sort templates',
diff --git a/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java b/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java
index dd5e1a883d51..3368549c1d7a 100644
--- a/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java
+++ b/qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java
@@ -13,12 +13,25 @@
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 
 import org.apache.lucene.tests.util.TimeUnits;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.ClassRule;
 
 @TimeoutSuite(millis = 40 * TimeUnits.MINUTE) // some of the windows test VMs are slow as hell
 public class SmokeTestMultiNodeClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .nodes(2)
+        .module("mapper-extras")
+        .module("ingest-common")
+        // The first node does not have the ingest role so we're sure ingest requests are forwarded:
+        .node(0, n -> n.setting("node.roles", "[master,data,ml,remote_cluster_client,transform]"))
+        .feature(FeatureFlag.TIME_SERIES_MODE)
+        .build();
+
     public SmokeTestMultiNodeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
@@ -27,4 +40,9 @@ public SmokeTestMultiNodeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandi
     public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
 }
diff --git a/qa/smoke-test-multinode/src/yamlRestTest/resources/rest-api-spec/test/smoke_test_multinode/30_desired_balance.yml b/qa/smoke-test-multinode/src/yamlRestTest/resources/rest-api-spec/test/smoke_test_multinode/30_desired_balance.yml
index 867f4ad8d5d4..fc078adeda62 100644
--- a/qa/smoke-test-multinode/src/yamlRestTest/resources/rest-api-spec/test/smoke_test_multinode/30_desired_balance.yml
+++ b/qa/smoke-test-multinode/src/yamlRestTest/resources/rest-api-spec/test/smoke_test_multinode/30_desired_balance.yml
@@ -64,3 +64,45 @@ setup:
   - gte: { routing_table.test.1.desired.ignored: 0 }
   - is_true: 'routing_table.test.1.desired.node_ids'
 
+---
+"Test cluster_balance_stats":
+
+  - skip:
+      version: " - 8.6.99"
+      reason: "Field added in in 8.7.0"
+
+  - do:
+      _internal.get_desired_balance: { }
+
+  - is_true: 'cluster_balance_stats'
+  - is_true: 'cluster_balance_stats.tiers'
+  - is_true: 'cluster_balance_stats.tiers.data_content.shard_count'
+  - is_true: 'cluster_balance_stats.tiers.data_content.shard_count.total'
+  - is_true: 'cluster_balance_stats.tiers.data_content.shard_count.min'
+  - is_true: 'cluster_balance_stats.tiers.data_content.shard_count.max'
+  - is_true: 'cluster_balance_stats.tiers.data_content.shard_count.average'
+  - is_true: 'cluster_balance_stats.tiers.data_content.shard_count.std_dev'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_write_load'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_write_load.total'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_write_load.min'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_write_load.max'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_write_load.average'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_write_load.std_dev'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_disk_usage'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_disk_usage.total'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_disk_usage.min'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_disk_usage.max'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_disk_usage.average'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_disk_usage.std_dev'
+  - is_true: 'cluster_balance_stats.tiers.data_content.actual_disk_usage'
+  - is_true: 'cluster_balance_stats.tiers.data_content.actual_disk_usage.total'
+  - is_true: 'cluster_balance_stats.tiers.data_content.actual_disk_usage.min'
+  - is_true: 'cluster_balance_stats.tiers.data_content.actual_disk_usage.max'
+  - is_true: 'cluster_balance_stats.tiers.data_content.actual_disk_usage.average'
+  - is_true: 'cluster_balance_stats.tiers.data_content.actual_disk_usage.std_dev'
+  - is_true: 'cluster_balance_stats.nodes'
+  - is_true: 'cluster_balance_stats.nodes.test-cluster-0'
+  - gte: { 'cluster_balance_stats.nodes.test-cluster-0.shard_count' : 0 }
+  - gte: { 'cluster_balance_stats.nodes.test-cluster-0.forecast_write_load': 0.0 }
+  - gte: { 'cluster_balance_stats.nodes.test-cluster-0.forecast_disk_usage_bytes' : 0 }
+  - gte: { 'cluster_balance_stats.nodes.test-cluster-0.actual_disk_usage_bytes' : 0 }
diff --git a/qa/smoke-test-multinode/src/yamlRestTest/resources/rest-api-spec/test/smoke_test_multinode/40_simulate.yml b/qa/smoke-test-multinode/src/yamlRestTest/resources/rest-api-spec/test/smoke_test_multinode/40_simulate.yml
new file mode 100644
index 000000000000..eb27b021eb11
--- /dev/null
+++ b/qa/smoke-test-multinode/src/yamlRestTest/resources/rest-api-spec/test/smoke_test_multinode/40_simulate.yml
@@ -0,0 +1,978 @@
+---
+teardown:
+  - do:
+      ingest.delete_pipeline:
+        id: "my_pipeline"
+        ignore: 404
+
+---
+"Test simulate with stored ingest pipeline":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "set" : {
+                  "field" : "field2",
+                  "value" : "_value"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      ingest.simulate:
+        id: "my_pipeline"
+        body: >
+          {
+            "docs": [
+              {
+                "_index": "index",
+                "_id": "id",
+                "_source": {
+                  "foo": "bar"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._source.foo: "bar" }
+  - match: { docs.0.doc._source.field2: "_value" }
+  - length: { docs.0.doc._ingest: 1 }
+  - is_true: docs.0.doc._ingest.timestamp
+
+---
+"Test simulate with provided pipeline definition":
+  - do:
+      ingest.simulate:
+        body: >
+          {
+            "pipeline": {
+              "description": "_description",
+              "processors": [
+                {
+                  "set" : {
+                    "field" : "field2",
+                    "value" : "_value"
+                  }
+                }
+              ]
+            },
+            "docs": [
+              {
+                "_index": "index",
+                "_id": "id",
+                "_source": {
+                  "foo": "bar"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+
+---
+"Test simulate with provided invalid pipeline definition":
+  - do:
+      catch: bad_request
+      ingest.simulate:
+        body: >
+          {
+            "pipeline": {
+              "description": "_description",
+              "processors": [
+                {
+                  "set" : {
+                    "tag" : "fails",
+                    "value" : "_value"
+                  }
+                }
+              ]
+            },
+            "docs": [
+              {
+                "_index": "index",
+                "_id": "id",
+                "_source": {
+                  "foo": "bar"
+                }
+              }
+            ]
+          }
+  - match: { error.root_cause.0.type: "parse_exception" }
+  - match: { error.root_cause.0.reason: "[field] required property is missing" }
+  - match: { error.root_cause.0.processor_tag: "fails" }
+  - match: { error.root_cause.0.processor_type: "set" }
+  - match: { error.root_cause.0.property_name: "field" }
+
+---
+"Test simulate without id":
+  - do:
+      ingest.simulate:
+        body: >
+          {
+            "pipeline": {
+              "description": "_description",
+              "processors": [
+                {
+                  "set" : {
+                    "field" : "field2",
+                    "value" : "_value"
+                  }
+                }
+              ]
+            },
+            "docs": [
+              {
+                "_source": {
+                  "foo": "bar"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+
+---
+"Test simulate with provided pipeline definition with on_failure block":
+  - do:
+      ingest.simulate:
+        body: >
+          {
+            "pipeline": {
+              "description": "_description",
+              "processors": [
+                {
+                  "rename" : {
+                    "field" : "does_not_exist",
+                    "target_field" : "field2",
+                    "on_failure" : [
+                      {
+                        "set" : {
+                          "field" : "field2",
+                          "value" : "_value"
+                        }
+                      }
+                    ]
+                  }
+                }
+              ]
+            },
+            "docs": [
+              {
+                "_index": "index",
+                "_id": "id",
+                "_source": {
+                  "foo": "bar"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._source.foo: "bar" }
+  - match: { docs.0.doc._source.field2: "_value" }
+  - length: { docs.0.doc._ingest: 1 }
+  - is_true: docs.0.doc._ingest.timestamp
+
+---
+"Test simulate with no provided pipeline or pipeline_id":
+  - do:
+      catch: bad_request
+      ingest.simulate:
+        body: >
+          {
+            "docs": [
+              {
+                "_index": "index",
+                "_id": "id",
+                "_source": {
+                  "foo": "bar"
+                }
+              }
+            ]
+          }
+  - is_false: error.root_cause.0.processor_type
+  - is_false: error.root_cause.0.processor_tag
+  - match: { error.root_cause.0.property_name: "pipeline" }
+  - match: { error.reason: "[pipeline] required property is missing" }
+
+---
+"Test simulate with invalid processor config":
+  - do:
+      catch: bad_request
+      ingest.simulate:
+        body: >
+          {
+            "pipeline": {
+              "description": "_description",
+              "processors": [
+                {
+                  "set" : {
+                    "field" : "field2"
+                  }
+                }
+              ]
+            },
+            "docs": [
+              {
+                "_index": "index",
+                "_id": "id",
+                "_source": {
+                  "foo": "bar"
+                }
+              }
+            ]
+          }
+  - match: { error.root_cause.0.type: "parse_exception" }
+  - match: { error.root_cause.0.reason: "[value] required property is missing" }
+  - match: { error.root_cause.0.processor_type: "set" }
+  - match: { error.root_cause.0.property_name: "value" }
+  - is_false: error.root_cause.0.processor_tag
+
+---
+"Test simulate with verbose flag":
+  - do:
+      ingest.simulate:
+        verbose: true
+        body: >
+          {
+            "pipeline": {
+              "description": "_description",
+              "processors": [
+                {
+                  "set" : {
+                    "tag" : "processor[set]-0",
+                    "field" : "field2.value",
+                    "value" : "_value"
+                  }
+                },
+                {
+                  "set" : {
+                    "field" : "field3",
+                    "value" : "third_val"
+                  }
+                },
+                {
+                  "uppercase" : {
+                    "field" : "field2.value"
+                  }
+                },
+                {
+                  "lowercase" : {
+                    "field" : "foo.bar.0.item"
+                  }
+                }
+              ]
+            },
+            "docs": [
+              {
+                "_index": "index",
+                "_id": "id",
+                "_source": {
+                  "foo": {
+                    "bar" : [ {"item": "HELLO"} ]
+                  }
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+  - length: { docs.0.processor_results: 4 }
+  - match: { docs.0.processor_results.0.tag: "processor[set]-0" }
+  - length: { docs.0.processor_results.0.doc._source: 2 }
+  - match: { docs.0.processor_results.0.doc._source.foo.bar.0.item: "HELLO" }
+  - match: { docs.0.processor_results.0.doc._source.field2.value: "_value" }
+  - length: { docs.0.processor_results.0.doc._ingest: 2 }
+  - is_true: docs.0.processor_results.0.doc._ingest.timestamp
+  - is_true: docs.0.processor_results.0.doc._ingest.pipeline
+  - length: { docs.0.processor_results.1.doc._source: 3 }
+  - match: { docs.0.processor_results.1.doc._source.foo.bar.0.item: "HELLO" }
+  - match: { docs.0.processor_results.1.doc._source.field2.value: "_value" }
+  - match: { docs.0.processor_results.1.doc._source.field3: "third_val" }
+  - length: { docs.0.processor_results.1.doc._ingest: 2 }
+  - is_true: docs.0.processor_results.1.doc._ingest.timestamp
+  - is_true: docs.0.processor_results.1.doc._ingest.pipeline
+  - length: { docs.0.processor_results.2.doc._source: 3 }
+  - match: { docs.0.processor_results.2.doc._source.foo.bar.0.item: "HELLO" }
+  - match: { docs.0.processor_results.2.doc._source.field2.value: "_VALUE" }
+  - match: { docs.0.processor_results.2.doc._source.field3: "third_val" }
+  - length: { docs.0.processor_results.2.doc._ingest: 2 }
+  - is_true: docs.0.processor_results.2.doc._ingest.timestamp
+  - is_true: docs.0.processor_results.2.doc._ingest.pipeline
+  - length: { docs.0.processor_results.3.doc._source: 3 }
+  - match: { docs.0.processor_results.3.doc._source.foo.bar.0.item: "hello" }
+  - match: { docs.0.processor_results.3.doc._source.field2.value: "_VALUE" }
+  - match: { docs.0.processor_results.3.doc._source.field3: "third_val" }
+  - length: { docs.0.processor_results.3.doc._ingest: 2 }
+  - is_true: docs.0.processor_results.3.doc._ingest.timestamp
+  - is_true: docs.0.processor_results.3.doc._ingest.pipeline
+
+---
+"Test simulate with exception thrown":
+  - do:
+      ingest.simulate:
+        body: >
+          {
+            "pipeline": {
+              "description": "_description",
+              "processors": [
+                {
+                  "uppercase" : {
+                    "field" : "foo"
+                  }
+                }
+              ]
+            },
+            "docs": [
+              {
+                "_index": "index",
+                "_id": "id",
+                "_source": {
+                  "not_foo": "bar"
+                }
+              },
+              {
+                "_index": "index",
+                "_id": "id2",
+                "_source": {
+                  "foo": "bar"
+                }
+              }
+            ]
+          }
+  - length: { docs: 2 }
+  - match: { docs.0.error.type: "illegal_argument_exception" }
+  - match: { docs.1.doc._source.foo: "BAR" }
+  - length: { docs.1.doc._ingest: 1 }
+  - is_true: docs.1.doc._ingest.timestamp
+
+---
+"Test verbose simulate with exception thrown":
+  - do:
+      ingest.simulate:
+        verbose: true
+        body: >
+          {
+            "pipeline": {
+              "description": "_description",
+              "processors": [
+                {
+                  "convert" : {
+                    "field" : "foo",
+                    "type" : "integer"
+                  }
+                },
+                {
+                  "uppercase" : {
+                    "field" : "bar"
+                  }
+                }
+              ]
+            },
+            "docs": [
+              {
+                "_index": "index",
+                "_id": "id",
+                "_source": {
+                  "foo": "bar",
+                  "bar": "hello"
+                }
+              },
+              {
+                "_index": "index",
+                "_id": "id2",
+                "_source": {
+                  "foo": "5",
+                  "bar": "hello"
+                }
+              }
+            ]
+          }
+  - length: { docs: 2 }
+  - length: { docs.0.processor_results: 1 }
+  - match: { docs.0.processor_results.0.error.type: "illegal_argument_exception" }
+  - length: { docs.1.processor_results: 2 }
+  - match: { docs.1.processor_results.0.doc._index: "index" }
+  - match: { docs.1.processor_results.0.doc._source.foo: 5 }
+  - match: { docs.1.processor_results.0.doc._source.bar: "hello" }
+  - length: { docs.1.processor_results.0.doc._ingest: 2 }
+  - is_true: docs.1.processor_results.0.doc._ingest.timestamp
+  - is_true: docs.1.processor_results.0.doc._ingest.pipeline
+  - match: { docs.1.processor_results.1.doc._source.foo: 5 }
+  - match: { docs.1.processor_results.1.doc._source.bar: "HELLO" }
+  - length: { docs.1.processor_results.1.doc._ingest: 2 }
+  - is_true: docs.1.processor_results.1.doc._ingest.timestamp
+  - is_true: docs.1.processor_results.1.doc._ingest.pipeline
+
+---
+"Test verbose simulate with error in pipeline":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "rename" : {
+                  "field" : "does_not_exist",
+                  "target_field" : "_value"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      ingest.simulate:
+        verbose: true
+        body: >
+          {
+            "pipeline": {
+              "description": "_description",
+              "processors": [
+                {
+                  "pipeline" : {
+                    "name" : "my_pipeline"
+                  }
+                }
+              ]
+            },
+            "docs": [
+              {
+                "_index": "index",
+                "_id": "id",
+                "_source": {
+                  "foo": "bar",
+                  "bar": "hello"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+  - length: { docs.0.processor_results: 2 }
+  - match: { docs.0.processor_results.0.processor_type: "pipeline" }
+  - match: { docs.0.processor_results.0.status: "success" }
+  - match: { docs.0.processor_results.1.processor_type: "rename" }
+  - match: { docs.0.processor_results.1.status: "error" }
+  - match: { docs.0.processor_results.1.error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { docs.0.processor_results.1.error.root_cause.0.reason: "field [does_not_exist] doesn't exist" }
+  - match: { docs.0.processor_results.1.error.type: "illegal_argument_exception" }
+  - match: { docs.0.processor_results.1.error.reason: "field [does_not_exist] doesn't exist" }
+
+---
+"Test verbose simulate with on_failure":
+  - do:
+      ingest.simulate:
+        verbose: true
+        body: >
+          {
+            "pipeline" : {
+              "description": "_description",
+              "processors": [
+                {
+                  "set" : {
+                    "tag" : "setstatus-1",
+                    "field" : "status",
+                    "value" : 200
+                  }
+                },
+                {
+                  "rename" : {
+                    "tag" : "rename-1",
+                    "field" : "foofield",
+                    "target_field" : "field1",
+                    "on_failure" : [
+                      {
+                        "set" : {
+                          "tag" : "set on_failure rename",
+                          "field" : "foofield",
+                          "value" : "exists"
+                        }
+                      },
+                      {
+                        "rename" : {
+                          "field" : "foofield2",
+                          "target_field" : "field1",
+                          "on_failure" : [
+                            {
+                              "set" : {
+                                "field" : "foofield2",
+                                "value" : "ran"
+                              }
+                            }
+                          ]
+                        }
+                      }
+                    ]
+                  }
+                }
+              ]
+            },
+            "docs": [
+              {
+                "_index": "index",
+                "_id": "id",
+                "_source": {
+                  "field1": "123.42 400 <foo>"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+  - length: { docs.0.processor_results: 5 }
+  - match: { docs.0.processor_results.0.tag: "setstatus-1" }
+  - match: { docs.0.processor_results.0.doc._source.field1: "123.42 400 <foo>" }
+  - match: { docs.0.processor_results.0.doc._source.status: 200 }
+  - match: { docs.0.processor_results.1.tag: "rename-1" }
+  - match: { docs.0.processor_results.1.error.type: "illegal_argument_exception" }
+  - match: { docs.0.processor_results.1.error.reason: "field [foofield] doesn't exist" }
+  - match: { docs.0.processor_results.2.tag: "set on_failure rename" }
+  - is_false: docs.0.processor_results.3.tag
+  - is_false: docs.0.processor_results.4.tag
+  - match: { docs.0.processor_results.4.doc._source.foofield: "exists" }
+  - match: { docs.0.processor_results.4.doc._source.foofield2: "ran" }
+  - match: { docs.0.processor_results.4.doc._source.field1: "123.42 400 <foo>" }
+  - match: { docs.0.processor_results.4.doc._source.status: 200 }
+
+---
+"Test verbose simulate with ignore_failure and thrown exception":
+  - do:
+      ingest.simulate:
+        verbose: true
+        body: >
+          {
+            "pipeline" : {
+              "description": "_description",
+              "processors": [
+                {
+                  "set" : {
+                    "tag" : "setstatus-1",
+                    "field" : "status",
+                    "value" : 200
+                  }
+                },
+                {
+                  "rename" : {
+                    "tag" : "rename-1",
+                    "field" : "foofield",
+                    "target_field" : "field1",
+                    "ignore_failure": true,
+                    "on_failure" : [
+                      {
+                        "set" : {
+                          "tag" : "set on_failure rename",
+                          "field" : "foofield",
+                          "value" : "exists"
+                        }
+                      },
+                      {
+                        "rename" : {
+                          "field" : "foofield2",
+                          "target_field" : "field1",
+                          "on_failure" : [
+                            {
+                              "set" : {
+                                "field" : "foofield2",
+                                "value" : "ran"
+                              }
+                            }
+                          ]
+                        }
+                      }
+                    ]
+                  }
+                }
+              ]
+            },
+            "docs": [
+              {
+                "_index": "index",
+                "_id": "id",
+                "_source": {
+                  "field1": "123.42 400 <foo>"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+  - length: { docs.0.processor_results: 2 }
+  - match: { docs.0.processor_results.0.tag: "setstatus-1" }
+  - match: { docs.0.processor_results.0.doc._source.field1: "123.42 400 <foo>" }
+  - match: { docs.0.processor_results.0.doc._source.status: 200 }
+  - match: { docs.0.processor_results.0.status: "success" }
+  - match: { docs.0.processor_results.0.processor_type: "set" }
+  - match: { docs.0.processor_results.1.tag: "rename-1" }
+  - match: { docs.0.processor_results.1.ignored_error.error.type: "illegal_argument_exception" }
+  - match: { docs.0.processor_results.1.ignored_error.error.reason: "field [foofield] doesn't exist" }
+  - match: { docs.0.processor_results.1.doc._source.field1: "123.42 400 <foo>" }
+  - match: { docs.0.processor_results.1.doc._source.status: 200 }
+  - match: { docs.0.processor_results.1.status: "error_ignored" }
+  - match: { docs.0.processor_results.1.processor_type: "rename" }
+
+---
+"Test verbose simulate with ignore_failure and no exception thrown":
+  - do:
+      ingest.simulate:
+        verbose: true
+        body: >
+          {
+            "pipeline" : {
+              "description": "_description",
+              "processors": [
+                {
+                  "set" : {
+                    "tag" : "setstatus-1",
+                    "field" : "status",
+                    "value" : 200
+                  }
+                },
+                {
+                  "rename" : {
+                    "tag" : "rename-1",
+                    "field" : "status",
+                    "target_field" : "new_status",
+                    "ignore_failure": true
+                  }
+                }
+              ]
+            },
+            "docs": [
+              {
+                "_index": "index",
+                "_id": "id",
+                "_source": {
+                  "field1": "123.42 400 <foo>"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+  - length: { docs.0.processor_results: 2 }
+  - length: { docs.0.processor_results.0: 4 }
+  - match: { docs.0.processor_results.0.tag: "setstatus-1" }
+  - match: { docs.0.processor_results.0.status: "success" }
+  - match: { docs.0.processor_results.0.processor_type: "set" }
+  - match: { docs.0.processor_results.0.doc._source.field1: "123.42 400 <foo>" }
+  - match: { docs.0.processor_results.0.doc._source.status: 200 }
+  - length: { docs.0.processor_results.1: 4 }
+  - match: { docs.0.processor_results.1.tag: "rename-1" }
+  - match: { docs.0.processor_results.1.status: "success" }
+  - match: { docs.0.processor_results.1.processor_type: "rename" }
+  - match: { docs.0.processor_results.1.doc._source.new_status: 200 }
+
+---
+"Test verbose simulate with Pipeline Processor with Circular Pipelines":
+  - do:
+      ingest.put_pipeline:
+        id: "outer"
+        body:  >
+          {
+            "description" : "outer pipeline",
+            "processors" : [
+              {
+                "pipeline" : {
+                  "name": "inner"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      ingest.put_pipeline:
+        id: "inner"
+        body:  >
+          {
+            "description" : "inner pipeline",
+            "processors" : [
+              {
+                "pipeline" : {
+                  "name": "outer"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      ingest.simulate:
+        verbose: true
+        body: >
+          {
+            "pipeline": {
+              "processors" : [
+              {
+                "pipeline" : {
+                  "name": "outer"
+                }
+              }
+              ]
+            }
+            ,
+            "docs": [
+            {
+              "_index": "index",
+              "_id": "id",
+              "_source": {
+                "field1": "123.42 400 <foo>"
+              }
+            }
+            ]
+          }
+  - length: { docs: 1 }
+  - length: { docs.0.processor_results: 1 }
+  - match: { docs.0.processor_results.0.error.reason: "Cycle detected for pipeline: outer" }
+
+---
+"Test verbose simulate with Pipeline Processor with Multiple Pipelines":
+  - do:
+      ingest.put_pipeline:
+        id: "pipeline1"
+        body:  >
+          {
+            "processors": [
+            {
+              "set": {
+                "field": "pipeline1",
+                "value": true
+              }
+            },
+            {
+              "pipeline": {
+                "name": "pipeline2"
+              }
+            }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      ingest.put_pipeline:
+        id: "pipeline2"
+        body:  >
+          {
+            "processors": [
+            {
+              "set": {
+                "field": "pipeline2",
+                "value": true
+              }
+            }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      ingest.simulate:
+        verbose: true
+        body: >
+          {
+            "pipeline": {
+              "processors": [
+              {
+                "set": {
+                  "field": "pipeline0",
+                  "value": true,
+                  "description" : "first_set"
+                }
+              },
+              {
+                "pipeline": {
+                  "name": "pipeline1"
+                }
+              }
+              ]
+            },
+            "docs": [
+            {
+              "_index": "index",
+              "_id": "id",
+              "_source": {
+                "field1": "123.42 400 <foo>"
+              }
+            }
+            ]
+          }
+  - length: { docs: 1 }
+  - length: { docs.0.processor_results: 5 }
+  - match: { docs.0.processor_results.0.doc._source.pipeline0: true }
+  - match: { docs.0.processor_results.0.status: "success" }
+  - match: { docs.0.processor_results.0.processor_type: "set" }
+  - match: { docs.0.processor_results.0.description: "first_set" }
+  - is_false: docs.0.processor_results.0.doc._source.pipeline1
+  - is_false: docs.0.processor_results.0.doc._source.pipeline2
+  - match: { docs.0.processor_results.1.doc: null }
+  - match: { docs.0.processor_results.1.status: "success" }
+  - match: { docs.0.processor_results.1.processor_type: "pipeline" }
+  - match: { docs.0.processor_results.2.doc._source.pipeline0: true }
+  - match: { docs.0.processor_results.2.doc._source.pipeline1: true }
+  - is_false: docs.0.processor_results.2.doc._source.pipeline2
+  - match: { docs.0.processor_results.3.doc: null }
+  - match: { docs.0.processor_results.3.status: "success" }
+  - match: { docs.0.processor_results.3.processor_type: "pipeline" }
+  - match: { docs.0.processor_results.4.doc._source.pipeline0: true }
+  - match: { docs.0.processor_results.4.doc._source.pipeline1: true }
+  - match: { docs.0.processor_results.4.doc._source.pipeline2: true }
+
+---
+"Test verbose simulate with true conditional and on failure":
+  - do:
+      ingest.simulate:
+        verbose: true
+        body: >
+          {
+            "pipeline": {
+              "processors": [
+                {
+                  "rename": {
+                    "tag": "gunna_fail",
+                    "if": "true",
+                    "field": "foo1",
+                    "target_field": "fieldA",
+                    "on_failure": [
+                      {
+                        "set": {
+                          "field": "failed1",
+                          "value": "failed1",
+                          "tag": "failed1"
+                        }
+                      },
+                      {
+                        "rename": {
+                          "tag": "gunna_fail_again",
+                          "if": "true",
+                          "field": "foo2",
+                          "target_field": "fieldA",
+                          "on_failure": [
+                            {
+                              "set": {
+                                "field": "failed2",
+                                "value": "failed2",
+                                "tag": "failed2"
+                              }
+                            }
+                          ]
+                        }
+                      }
+                    ]
+                  }
+                }
+              ]
+            },
+            "docs": [
+              {
+                "_index": "index",
+                "_id": "id",
+                "_source": {
+                  "foo": "bar"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+  - length: { docs.0.processor_results: 4 }
+  - match: { docs.0.processor_results.0.tag: "gunna_fail" }
+  - match: { docs.0.processor_results.0.error.reason: "field [foo1] doesn't exist" }
+  - match: { docs.0.processor_results.0.status: "error" }
+  - match: { docs.0.processor_results.0.processor_type: "rename" }
+  - match: { docs.0.processor_results.1.tag: "failed1" }
+  - match: { docs.0.processor_results.1.doc._source.failed1: "failed1" }
+  - match: { docs.0.processor_results.1.doc._ingest.on_failure_processor_tag: "gunna_fail" }
+  - match: { docs.0.processor_results.1.status: "success" }
+  - match: { docs.0.processor_results.1.processor_type: "set" }
+  - match: { docs.0.processor_results.2.tag: "gunna_fail_again" }
+  - match: { docs.0.processor_results.2.error.reason: "field [foo2] doesn't exist" }
+  - match: { docs.0.processor_results.2.status: "error" }
+  - match: { docs.0.processor_results.2.processor_type: "rename" }
+  - match: { docs.0.processor_results.3.tag: "failed2" }
+  - match: { docs.0.processor_results.3.doc._source.failed1: "failed1" }
+  - match: { docs.0.processor_results.3.doc._source.failed2: "failed2" }
+  - match: { docs.0.processor_results.3.doc._ingest.on_failure_processor_tag: "gunna_fail_again" }
+  - match: { docs.0.processor_results.3.status: "success" }
+  - match: { docs.0.processor_results.3.processor_type: "set" }
+
+
+---
+"Test simulate with pipeline with conditional and skipped and dropped":
+  - do:
+      ingest.simulate:
+        verbose: true
+        body: >
+          {
+            "pipeline": {
+              "description": "_description",
+              "processors": [
+                {
+                  "set" : {
+                    "description": "processor_description",
+                    "tag": "processor_tag",
+                    "field" : "field2",
+                    "value" : "_value"
+                  }
+                },
+                {
+                  "drop" : {
+                    "if": "false"
+                  }
+                },
+                {
+                  "drop" : {
+                     "if": "true"
+                   }
+                }
+              ]
+            },
+            "docs": [
+              {
+                "_index": "index",
+                "_id": "id",
+                "_source": {
+                  "foo": "bar"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+  - length: { docs.0.processor_results: 3 }
+  - match: { docs.0.processor_results.0.doc._source.field2: "_value" }
+  - match: { docs.0.processor_results.0.description: "processor_description" }
+  - match: { docs.0.processor_results.0.tag: "processor_tag" }
+  - match: { docs.0.processor_results.0.status: "success" }
+  - match: { docs.0.processor_results.0.processor_type: "set" }
+  - match: { docs.0.processor_results.1.status: "skipped" }
+  - match: { docs.0.processor_results.1.processor_type: "drop" }
+  - match: { docs.0.processor_results.1.if.condition: "false" }
+  - match: { docs.0.processor_results.1.if.result: false }
+  - match: { docs.0.processor_results.2.status: "dropped" }
+  - match: { docs.0.processor_results.2.processor_type: "drop" }
+  - match: { docs.0.processor_results.2.if.condition: "true" }
+  - match: { docs.0.processor_results.2.if.result: true }
+---
+"Test simulate with provided pipeline that does not exist":
+  - do:
+      ingest.simulate:
+        verbose: true
+        body: >
+          {
+            "pipeline": {
+              "description": "_description",
+              "processors": [
+                {
+                  "pipeline": {
+                     "name": "____pipeline_doesnot_exist___"
+                  }
+                }
+              ]
+            },
+            "docs": [
+              {
+                "_source": {}
+              }
+            ]
+          }
+  - match: { docs.0.processor_results.0.status: "error" }
+  - match: { docs.0.processor_results.0.error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { docs.0.processor_results.0.error.root_cause.0.reason: "Pipeline processor configured for non-existent pipeline [____pipeline_doesnot_exist___]" }
diff --git a/qa/smoke-test-plugins/build.gradle b/qa/smoke-test-plugins/build.gradle
index 06a9d9a6613a..7ae46b8e0d68 100644
--- a/qa/smoke-test-plugins/build.gradle
+++ b/qa/smoke-test-plugins/build.gradle
@@ -9,7 +9,7 @@
 import org.apache.tools.ant.filters.ReplaceTokens
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 
 ext.pluginPaths = []
 project(':plugins').getChildProjects().each { pluginName, pluginProject ->
diff --git a/qa/stable-api/build.gradle b/qa/stable-api/build.gradle
new file mode 100644
index 000000000000..393c271967c6
--- /dev/null
+++ b/qa/stable-api/build.gradle
@@ -0,0 +1,6 @@
+subprojects {
+  apply plugin: 'elasticsearch.java'
+  apply plugin: 'elasticsearch.bwc-test'
+
+  group = 'org.elasticsearch.qa.stable-api'
+}
diff --git a/qa/stable-api/logging/build.gradle b/qa/stable-api/logging/build.gradle
new file mode 100644
index 000000000000..ada0dc1d169b
--- /dev/null
+++ b/qa/stable-api/logging/build.gradle
@@ -0,0 +1,4 @@
+ext.stableApiSince = "8.7.0"
+
+apply plugin: 'elasticsearch.stable-api'
+
diff --git a/qa/stable-api/plugin-analysis-api/build.gradle b/qa/stable-api/plugin-analysis-api/build.gradle
new file mode 100644
index 000000000000..c3fdc92c36bb
--- /dev/null
+++ b/qa/stable-api/plugin-analysis-api/build.gradle
@@ -0,0 +1,3 @@
+ext.stableApiSince = "8.7.0"
+
+apply plugin: 'elasticsearch.stable-api'
diff --git a/qa/stable-api/plugin-api/build.gradle b/qa/stable-api/plugin-api/build.gradle
new file mode 100644
index 000000000000..c3fdc92c36bb
--- /dev/null
+++ b/qa/stable-api/plugin-api/build.gradle
@@ -0,0 +1,3 @@
+ext.stableApiSince = "8.7.0"
+
+apply plugin: 'elasticsearch.stable-api'
diff --git a/qa/system-indices/build.gradle b/qa/system-indices/build.gradle
index a8be8e8374d5..4cb60860cca4 100644
--- a/qa/system-indices/build.gradle
+++ b/qa/system-indices/build.gradle
@@ -7,7 +7,7 @@
  */
 
 apply plugin: 'elasticsearch.base-internal-es-plugin'
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 esplugin {
   name 'system-indices-qa'
diff --git a/qa/unconfigured-node-name/build.gradle b/qa/unconfigured-node-name/build.gradle
index aae98def1db8..dd1dd0a2334f 100644
--- a/qa/unconfigured-node-name/build.gradle
+++ b/qa/unconfigured-node-name/build.gradle
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 testClusters.configureEach {
   setting 'xpack.security.enabled', 'false'
diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle
index e92d6767a803..e5049a117905 100644
--- a/rest-api-spec/build.gradle
+++ b/rest-api-spec/build.gradle
@@ -33,9 +33,8 @@ artifacts {
   restTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test"))
 }
 
-testClusters.configureEach {
-  module ':modules:mapper-extras'
-  requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
+dependencies {
+  clusterModules project(":modules:mapper-extras")
 }
 
 tasks.named("yamlRestTestV7CompatTransform").configure { task ->
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/_internal.health.json b/rest-api-spec/src/main/resources/rest-api-spec/api/_internal.health.json
deleted file mode 100644
index 4528c33b0853..000000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/_internal.health.json
+++ /dev/null
@@ -1,46 +0,0 @@
-{
-  "_internal.health":{
-    "documentation":{
-      "url": null,
-      "description":"Returns the health of the cluster."
-    },
-    "stability":"experimental",
-    "visibility":"private",
-    "headers":{
-      "accept": [ "application/json"]
-    },
-    "url":{
-      "paths":[
-        {
-          "path":"/_internal/_health",
-          "methods":[
-            "GET"
-          ]
-        },
-        {
-          "path":"/_internal/_health/{feature}",
-          "methods":[
-            "GET"
-          ],
-          "parts":{
-            "feature":{
-              "type":"string",
-              "description":"A feature of the cluster, as returned by the top-level health API"
-            }
-          }
-        }
-      ]
-    },
-    "params":{
-      "timeout":{
-        "type":"time",
-        "description":"Explicit operation timeout"
-      },
-      "verbose":{
-        "type":"boolean",
-        "description":"Opt in for more information about the health of the system",
-        "default":true
-      }
-    }
-  }
-}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/_internal.prevalidate_node_removal.json b/rest-api-spec/src/main/resources/rest-api-spec/api/_internal.prevalidate_node_removal.json
index 8563c17ceb53..8c945f2894f2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/_internal.prevalidate_node_removal.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/_internal.prevalidate_node_removal.json
@@ -35,6 +35,10 @@
       "master_timeout":{
         "type":"time",
         "description":"Explicit operation timeout for connection to master node"
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
       }
     }
   }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/health_report.json b/rest-api-spec/src/main/resources/rest-api-spec/api/health_report.json
new file mode 100644
index 000000000000..2cc9b8f00831
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/health_report.json
@@ -0,0 +1,53 @@
+{
+  "health_report": {
+    "documentation": {
+      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/health-api.html",
+      "description": "Returns the health of the cluster."
+    },
+    "stability": "stable",
+    "visibility": "public",
+    "headers": {
+      "accept": [
+        "application/json"
+      ]
+    },
+    "url": {
+      "paths": [
+        {
+          "path": "/_health_report",
+          "methods": [
+            "GET"
+          ]
+        },
+        {
+          "path": "/_health_report/{feature}",
+          "methods": [
+            "GET"
+          ],
+          "parts": {
+            "feature": {
+              "type": "string",
+              "description": "A feature of the cluster, as returned by the top-level health API"
+            }
+          }
+        }
+      ]
+    },
+    "params":{
+      "timeout":{
+        "type":"time",
+        "description":"Explicit operation timeout"
+      },
+      "verbose":{
+        "type":"boolean",
+        "description":"Opt in for more information about the health of the system",
+        "default":true
+      },
+      "size": {
+        "type": "int",
+        "description": "Limit the number of affected resources the health API returns",
+        "default": 1000
+      }
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ml.clear_trained_model_deployment_cache.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ml.clear_trained_model_deployment_cache.json
index ee92ab44ab8f..81f396a30b36 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ml.clear_trained_model_deployment_cache.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ml.clear_trained_model_deployment_cache.json
@@ -4,7 +4,7 @@
       "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/clear-trained-model-deployment-cache.html",
       "description":"Clear the cached results from a trained model deployment"
     },
-    "stability":"beta",
+    "stability":"stable",
     "visibility":"public",
     "headers":{
       "accept": [ "application/json"],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ml.infer_trained_model.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ml.infer_trained_model.json
index 539d46eb88dd..6041155b1ea6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ml.infer_trained_model.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ml.infer_trained_model.json
@@ -4,7 +4,7 @@
       "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/infer-trained-model.html",
       "description":"Evaluate a trained model."
     },
-    "stability":"beta",
+    "stability":"stable",
     "visibility":"public",
     "headers":{
       "accept": [ "application/json"],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ml.put_trained_model_definition_part.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ml.put_trained_model_definition_part.json
index 09e34a5c9eea..bf826f36e93c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ml.put_trained_model_definition_part.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ml.put_trained_model_definition_part.json
@@ -4,7 +4,7 @@
       "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/put-trained-model-definition-part.html",
       "description":"Creates part of a trained model definition"
     },
-    "stability":"beta",
+    "stability":"stable",
     "visibility":"public",
     "headers":{
       "accept": [ "application/json"],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ml.put_trained_model_vocabulary.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ml.put_trained_model_vocabulary.json
index 1d1c6005b369..f6fd70a75c00 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ml.put_trained_model_vocabulary.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ml.put_trained_model_vocabulary.json
@@ -4,7 +4,7 @@
       "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/put-trained-model-vocabulary.html",
       "description":"Creates a trained model vocabulary"
     },
-    "stability":"beta",
+    "stability":"stable",
     "visibility":"public",
     "headers":{
       "accept": [ "application/json"],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ml.start_trained_model_deployment.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ml.start_trained_model_deployment.json
index db3418759898..8f98bc03670a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ml.start_trained_model_deployment.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ml.start_trained_model_deployment.json
@@ -4,7 +4,7 @@
       "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/start-trained-model-deployment.html",
       "description":"Start a trained model deployment."
     },
-    "stability":"beta",
+    "stability":"stable",
     "visibility":"public",
     "headers":{
       "accept": [ "application/json"],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ml.stop_trained_model_deployment.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ml.stop_trained_model_deployment.json
index 5cb9d5764067..016d88c684e4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ml.stop_trained_model_deployment.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ml.stop_trained_model_deployment.json
@@ -4,7 +4,7 @@
       "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/stop-trained-model-deployment.html",
       "description":"Stop a trained model deployment."
     },
-    "stability":"beta",
+    "stability":"stable",
     "visibility":"public",
     "headers":{
       "accept": [ "application/json"],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ml.update_trained_model_deployment.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ml.update_trained_model_deployment.json
index df9ab6cd5e73..0d824ece4c8b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ml.update_trained_model_deployment.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ml.update_trained_model_deployment.json
@@ -1,7 +1,7 @@
 {
   "ml.update_trained_model_deployment":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-update-trained-model-deployment.html",
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/update-trained-model-deployment.html",
       "description":"Updates certain properties of trained model deployment."
     },
     "stability":"beta",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/semantic_search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/semantic_search.json
deleted file mode 100644
index 3af11e258227..000000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/semantic_search.json
+++ /dev/null
@@ -1,40 +0,0 @@
-{
-  "semantic_search":{
-    "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/semantic-search.html",
-      "description":"Semantic search API using dense vector similarity"
-    },
-    "stability":"experimental",
-    "visibility":"public",
-    "headers":{
-      "accept": [ "application/json"],
-      "content_type": ["application/json"]
-    },
-    "url":{
-      "paths":[
-        {
-          "path":"/{index}/_semantic_search",
-          "methods":[
-            "GET",
-            "POST"
-          ],
-          "parts":{
-            "index":{
-              "type":"list",
-              "description":"A comma-separated list of index names to search; use `_all` to perform the operation on all indices"
-            }
-          }
-        }
-      ]
-    },
-    "params": {
-      "routing":{
-        "type":"list",
-        "description":"A comma-separated list of specific routing values"
-      }
-    },
-    "body":{
-      "description":"The search definition"
-    }
-  }
-}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/transform.get_transform_stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/transform.get_transform_stats.json
index f425c41f0997..8139fac7a818 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/transform.get_transform_stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/transform.get_transform_stats.json
@@ -36,6 +36,11 @@
         "required":false,
         "description":"specifies a max number of transform stats to get, defaults to 100"
       },
+      "timeout":{
+        "type":"time",
+        "required":false,
+        "description":"Controls the time to wait for the stats"
+      },
       "allow_no_match":{
         "type":"boolean",
         "required":false,
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/transform.schedule_now_transform.json b/rest-api-spec/src/main/resources/rest-api-spec/api/transform.schedule_now_transform.json
new file mode 100644
index 000000000000..81ba9e071cfd
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/transform.schedule_now_transform.json
@@ -0,0 +1,38 @@
+{
+  "transform.schedule_now_transform":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/schedule-now-transform.html",
+      "description":"Schedules now a transform."
+    },
+    "stability":"stable",
+    "visibility":"public",
+    "headers":{
+      "accept":[ "application/json"],
+      "content_type":["application/json"]
+    },
+    "url":{
+      "paths":[
+        {
+          "path":"/_transform/{transform_id}/_schedule_now",
+          "methods":[
+            "POST"
+          ],
+          "parts":{
+            "transform_id":{
+              "type":"string",
+              "required":true,
+              "description":"The id of the transform."
+            }
+          }
+        }
+      ]
+    },
+    "params":{
+      "timeout":{
+        "type":"time",
+        "required":false,
+        "description":"Controls the time to wait for the scheduling to take place"
+      }
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/transform.start_transform.json b/rest-api-spec/src/main/resources/rest-api-spec/api/transform.start_transform.json
index 7c002957ea67..b7fb849987af 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/transform.start_transform.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/transform.start_transform.json
@@ -26,6 +26,11 @@
       ]
     },
     "params":{
+      "from":{
+        "type":"string",
+        "required":false,
+        "description":"Restricts the set of transformed entities to those changed after this time"
+      },
       "timeout":{
         "type":"time",
         "required":false,
diff --git a/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java b/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java
index 287c8a57627b..a10d53389af9 100644
--- a/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java
+++ b/rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java
@@ -13,8 +13,11 @@
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 
 import org.apache.lucene.tests.util.TimeUnits;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.ClassRule;
 
 /** Rest integration test. Runs against a cluster started by {@code gradle integTest} */
 
@@ -22,6 +25,12 @@
 @TimeoutSuite(millis = 40 * TimeUnits.MINUTE)
 public class ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .module("mapper-extras")
+        .feature(FeatureFlag.TIME_SERIES_MODE)
+        .build();
+
     public ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
@@ -30,4 +39,9 @@ public ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate
     public static Iterable<Object[]> parameters() throws Exception {
         return createParameters();
     }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
 }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/bulk/11_dynamic_templates.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/bulk/11_dynamic_templates.yml
index ef904b341deb..0784cffe3b84 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/bulk/11_dynamic_templates.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/bulk/11_dynamic_templates.yml
@@ -24,20 +24,20 @@
       bulk:
         refresh: true
         body:
-          - index:
+          - create:
               _index: test_index
               _id: id_1
               dynamic_templates:
                 location: location
-          - { "location": [ -71.34, 41.12 ]}
+          - { "location": "41.12,-71.34"}
           - index:
               _index: test_index
               _id: id_2
               dynamic_templates:
                 location: location
-          - { "location": "41.12,-71.34"}
+          - { "location": [ -71.34, 41.12 ]}
   - match: { errors: false }
-  - match: { items.0.index.result: created }
+  - match: { items.0.create.result: created }
   - match: { items.1.index.result: created }
 
   - do:
@@ -65,7 +65,7 @@
               _index: test_index
               _id: id_3
           - { "my_location": "41.12,-71.34" } # matches the field name defined in the `my_location` template
-          - index:
+          - create:
               _index: test_index
               _id: id_4
               dynamic_templates:
@@ -169,3 +169,64 @@
   - match: { items.0.index.error.reason: "failed to parse field [foo] of type [keyword] in document with id 'id_11'. Preview of field's value: '{bar=hello world}'"}
   - match: { items.1.index.status: 201 }
   - match: { items.1.index.result: created }
+
+---
+"Dynamic templates with op_type":
+  - skip:
+      version: " - 8.6.0"
+      reason: "bug fixed in 8.6.1"
+
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          mappings:
+            dynamic_templates:
+              - location:
+                  mapping:
+                    type: geo_point
+              - my_location:
+                  match: my*
+                  mapping:
+                    type: geo_point
+              - string:
+                  mapping:
+                    type: keyword
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: test_index
+              _id: id_1
+              op_type: create
+              dynamic_templates:
+                location: location
+          - { "location": "41.12,-71.34"}
+          - index:
+              _index: test_index
+              _id: id_2
+              op_type: index
+              dynamic_templates:
+                location: location
+          - { "location": [ -71.34, 41.12 ]}
+  - match: { errors: false }
+  - match: { items.0.create.result: created }
+  - match: { items.1.index.result: created }
+
+  - do:
+      search:
+        index: test_index
+        body:
+          query:
+            geo_bounding_box:
+              location:
+                top_left:
+                  lat: 42
+                  lon: -72
+                bottom_right:
+                  lat: 40
+                  lon: -74
+  - match: { hits.total.value: 2 }
+  - match: { hits.hits.0._id: id_1 }
+  - match: { hits.hits.1._id: id_2 }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.nodes/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.nodes/10_basic.yml
index eff06113c782..a81307bec2c7 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.nodes/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.nodes/10_basic.yml
@@ -7,7 +7,7 @@
   - match:
       $body: |
                /  #ip                          heap.percent        ram.percent     cpu         load_1m                load_5m                load_15m               node.role                   master          name
-               ^  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+            \s+  \d*         \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+  ((-)?\d*(\.\d+)?)? \s+ (-|[cdfhilmrstvw]{1,11}) \s+ [-*x]     \s+   .*     \n)+  $/
+               ^  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+            \s+  \d*         \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+  ((-)?\d*(\.\d+)?)? \s+ (-|[cdfhilmrstvwIS]{1,11}) \s+ [-*x]     \s+   .*     \n)+  $/
 
   - do:
       cat.nodes:
@@ -16,7 +16,7 @@
   - match:
       $body: |
                /^  ip                     \s+  heap\.percent   \s+  ram\.percent \s+ cpu      \s+ load_1m            \s+ load_5m            \s+ load_15m           \s+ node\.role              \s+  master   \s+   name  \n
-                  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+             \s+  \d*          \s+ (-)?\d* \s+  ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[cdfhilmrstvw]{1,11}) \s+  [-*x]    \s+   .*    \n)+  $/
+                  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+             \s+  \d*          \s+ (-)?\d* \s+  ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[cdfhilmrstvwIS]{1,11}) \s+  [-*x]    \s+   .*    \n)+  $/
 
   - do:
       cat.nodes:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_balance/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_balance/10_basic.yml
index 177b8adab19b..5969e946494c 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_balance/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_balance/10_basic.yml
@@ -19,6 +19,49 @@ setup:
   - gte: { stats.reconciliation_time_in_millis: 0 }
   - match: { routing_table: {} }
 
+---
+"Test cluster_balance_stats":
+
+  - skip:
+      version: " - 8.6.99"
+      reason: "Field added in in 8.7.0"
+
+  - do:
+      _internal.get_desired_balance: { }
+
+  - is_true: 'cluster_balance_stats'
+  - is_true: 'cluster_balance_stats.tiers'
+  - is_true: 'cluster_balance_stats.tiers.data_content.shard_count'
+  - is_true: 'cluster_balance_stats.tiers.data_content.shard_count.total'
+  - is_true: 'cluster_balance_stats.tiers.data_content.shard_count.min'
+  - is_true: 'cluster_balance_stats.tiers.data_content.shard_count.max'
+  - is_true: 'cluster_balance_stats.tiers.data_content.shard_count.average'
+  - is_true: 'cluster_balance_stats.tiers.data_content.shard_count.std_dev'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_write_load'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_write_load.total'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_write_load.min'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_write_load.max'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_write_load.average'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_write_load.std_dev'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_disk_usage'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_disk_usage.total'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_disk_usage.min'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_disk_usage.max'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_disk_usage.average'
+  - is_true: 'cluster_balance_stats.tiers.data_content.forecast_disk_usage.std_dev'
+  - is_true: 'cluster_balance_stats.tiers.data_content.actual_disk_usage'
+  - is_true: 'cluster_balance_stats.tiers.data_content.actual_disk_usage.total'
+  - is_true: 'cluster_balance_stats.tiers.data_content.actual_disk_usage.min'
+  - is_true: 'cluster_balance_stats.tiers.data_content.actual_disk_usage.max'
+  - is_true: 'cluster_balance_stats.tiers.data_content.actual_disk_usage.average'
+  - is_true: 'cluster_balance_stats.tiers.data_content.actual_disk_usage.std_dev'
+  - is_true: 'cluster_balance_stats.nodes'
+  - is_true: 'cluster_balance_stats.nodes.test-cluster-0'
+  - gte: { 'cluster_balance_stats.nodes.test-cluster-0.shard_count' : 0 }
+  - gte: { 'cluster_balance_stats.nodes.test-cluster-0.forecast_write_load': 0.0 }
+  - gte: { 'cluster_balance_stats.nodes.test-cluster-0.forecast_disk_usage_bytes' : 0 }
+  - gte: { 'cluster_balance_stats.nodes.test-cluster-0.actual_disk_usage_bytes' : 0 }
+
 ---
 "Test get desired balance for single shard":
   - do:
@@ -51,8 +94,9 @@ setup:
   - is_true: 'routing_table.test.0.current.0.node_is_desired'
   - is_false: 'routing_table.test.0.current.0.relocating_node'
   - is_false: 'routing_table.test.0.current.0.relocating_node_is_desired'
+  - is_false: 'routing_table.test.0.current.0.forecast_write_load'
+  - is_false: 'routing_table.test.0.current.0.forecast_shard_size_in_bytes'
   - match: { routing_table.test.0.desired.total: 1 }
   - gte: { routing_table.test.0.desired.unassigned: 0 }
   - gte: { routing_table.test.0.desired.ignored: 0 }
   - is_true: 'routing_table.test.0.desired.node_ids'
-
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/10_basic.yml
index 20760fef4b1d..a051b3626b21 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/10_basic.yml
@@ -211,7 +211,7 @@
   - skip:
       version: " - 7.1.99"
       reason:  "closed indices are replicated starting version 7.2.0"
-      features: ["allowed_warnings"]
+      features: ["allowed_warnings", "default_shards"]
 
   - do:
       indices.create:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.prevalidate_node_removal/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.prevalidate_node_removal/10_basic.yml
index 6e90d00ca595..740836efcdc4 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.prevalidate_node_removal/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.prevalidate_node_removal/10_basic.yml
@@ -2,8 +2,8 @@
 "Prevalidation basic test":
   - skip:
       features: contains
-      version: "- 8.5.99"
-      reason: "API added in 8.6.0"
+      version: "- 8.6.99"
+      reason: "The reason field was introduced in 8.7.0"
 
   # Fetch a node ID and stash it in node_id
   - do:
@@ -16,7 +16,7 @@
         ids: $node_id
 
   - match: { is_safe: true}
-  - contains: {nodes: {id: "$node_id", result: {is_safe: true, message: ""}}}
+  - contains: {nodes: {id: "$node_id", result: {is_safe: true, reason: no_problems, message: ""}}}
 ---
 "Prevalidation with no node specified":
   - skip:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml
index 566cc777faa4..cd83a7868528 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml
@@ -600,6 +600,7 @@ _doc_count:
             _source:
               mode: synthetic
 
+  # with _doc_count
   - do:
       index:
         index:   test
@@ -608,7 +609,6 @@ _doc_count:
         body:
           _doc_count: 3
           foo: bar
-
   - do:
       get:
         index: test
@@ -623,6 +623,98 @@ _doc_count:
         foo: bar
   - is_false: fields
 
+  # without _doc_count
+  - do:
+      index:
+        index:   test
+        id:      2
+        refresh: true
+        body:
+          foo: baz
+  - do:
+      get:
+        index: test
+        id:    2
+  - match: {_index: "test"}
+  - match: {_id: "2"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        foo: baz
+  - is_false: fields
+
+  # without immediately refreshing with _doc_count
+  - do:
+      index:
+        index:   test
+        id:      3
+        body:
+          _doc_count: 3
+          foo: qux
+  - do:
+      get:
+        index: test
+        id:    3
+  - match: {_index: "test"}
+  - match: {_id: "3"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        _doc_count: 3
+        foo: qux
+  - is_false: fields
+
+  # without immediately refreshing without _doc_count
+  - do:
+      index:
+        index:   test
+        id:      4
+        body:
+          foo: quux
+  - do:
+      get:
+        index: test
+        id:    4
+  - match: {_index: "test"}
+  - match: {_id: "4"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        foo: quux
+  - is_false: fields
+
+  # refresh all at once
+  - do:
+      indices.refresh: {}
+  - do:
+      get:
+        index: test
+        id:    3
+  - match: {_index: "test"}
+  - match: {_id: "3"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        _doc_count: 3
+        foo: qux
+  - is_false: fields
+  - do:
+      get:
+        index: test
+        id:    4
+  - match: {_index: "test"}
+  - match: {_id: "4"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match:
+      _source:
+        foo: quux
+  - is_false: fields
+
 ---
 ip with ignore_malformed:
   - skip:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/10_basic.yml
index 4f2c7113d877..5e6ca8247997 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/10_basic.yml
@@ -7,7 +7,7 @@
       # reason: "health was only added in 8.2.0, and master_is_stable in 8.4.0"
 
   - do:
-      _internal.health: {}
+      health_report: { }
 
   - is_true: cluster_name
   - match:   { status: "green" }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/30_feature.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/30_feature.yml
index 575e4c7ad41d..449954220a1e 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/30_feature.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/30_feature.yml
@@ -1,11 +1,11 @@
 ---
 "cluster health test drilling down into a feature":
   - skip:
-      version: "- 8.5.99"
-      reason: "the verbose parameter was only added in 8.6"
+      version: "- 8.6.99"
+      reason: "the API path changed in 8.7"
 
   - do:
-      _internal.health:
+      health_report:
         feature: master_is_stable
 
   - is_true: cluster_name
@@ -15,7 +15,7 @@
   - is_true: indicators.master_is_stable.details.recent_masters
 
   - do:
-      _internal.health:
+      health_report:
         feature: master_is_stable
         verbose: false
 
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/40_diagnosis.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/40_diagnosis.yml
index f475d3b0f1d0..76b81354b741 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/40_diagnosis.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/40_diagnosis.yml
@@ -1,12 +1,14 @@
 ---
 "Diagnosis":
   - skip:
-      version: "- 8.5.99"
-      reason: "diagnosis was redefined in 8.6.0"
+      version: "- 8.6.99"
+      reason: "the API path changed in 8.7"
 
   - do:
       indices.create:
         index: red_index
+        master_timeout: 1s
+        timeout: 1s
         body:
           settings:
             number_of_shards: 1
@@ -14,7 +16,7 @@
             index.routing.allocation.enable: none
 
   - do:
-      _internal.health:
+      health_report:
         feature: shards_availability
 
   - is_true: cluster_name
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.blocks/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.blocks/10_basic.yml
index f87553f222eb..b7e84d2fd526 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.blocks/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.blocks/10_basic.yml
@@ -6,9 +6,6 @@
   - do:
       indices.create:
         index: test_index
-        body:
-          settings:
-            number_of_replicas: 0
 
   - do:
       indices.add_block:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.open/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.open/10_basic.yml
index 3e55ef207a0d..93447612406b 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.open/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.open/10_basic.yml
@@ -6,13 +6,11 @@
   - do:
       indices.create:
         index: test_index
-        body:
-          settings:
-            number_of_replicas: 0
 
   - do:
       cluster.health:
-        wait_for_status: green
+        index: [test_index]
+        wait_for_no_initializing_shards: true
 
   - do:
       indices.close:
@@ -34,7 +32,8 @@
 
   - do:
       cluster.health:
-        wait_for_status: green
+        index: [test_index]
+        wait_for_no_initializing_shards: true
 
   - do:
       search:
@@ -88,6 +87,7 @@
   - is_true: acknowledged
   - match: { acknowledged: true }
   - match: { shards_acknowledged: true }
+
 ---
 "Close index response with result per index":
   - skip:
@@ -96,23 +96,19 @@
   - do:
       indices.create:
         index: index_1
-        body:
-          settings:
-            number_of_replicas: 0
 
   - do:
       indices.create:
         index: index_2
-        body:
-          settings:
-            number_of_replicas: 0
 
   - do:
       indices.create:
         index: index_3
-        body:
-          settings:
-            number_of_replicas: 0
+
+  - do:
+      cluster.health:
+        index: [index_1, index_2, index_3]
+        wait_for_no_initializing_shards: true
 
   - do:
       indices.close:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.open/20_multiple_indices.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.open/20_multiple_indices.yml
index f94a95c4fc98..ec71423bdc24 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.open/20_multiple_indices.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.open/20_multiple_indices.yml
@@ -2,24 +2,19 @@ setup:
   - do:
       indices.create:
         index: test_index1
-        body:
-          settings:
-            number_of_replicas: 0
+
   - do:
       indices.create:
         index: test_index2
-        body:
-          settings:
-            number_of_replicas: 0
+
   - do:
       indices.create:
         index: test_index3
-        body:
-          settings:
-            number_of_replicas: 0
+
   - do:
       cluster.health:
-        wait_for_status: green
+        index: [test_index1, test_index2, test_index3]
+        wait_for_no_initializing_shards: true
 
 ---
 "All indices":
@@ -46,7 +41,8 @@ setup:
 
   - do:
       cluster.health:
-        wait_for_status: green
+        index: [test_index1, test_index2, test_index3]
+        wait_for_no_initializing_shards: true
 
   - do:
       search:
@@ -78,7 +74,8 @@ setup:
 
   - do:
       cluster.health:
-        wait_for_status: green
+        index: [test_index1, test_index2, test_index3]
+        wait_for_no_initializing_shards: true
 
   - do:
       search:
@@ -110,7 +107,8 @@ setup:
 
   - do:
       cluster.health:
-        wait_for_status: green
+        index: [test_index1, test_index2, test_index3]
+        wait_for_no_initializing_shards: true
 
   - do:
       search:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml
index 5eef78a8c63b..2aaf492f0ff0 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml
@@ -286,3 +286,66 @@
 
   - is_false: purple-index.mappings.properties.nested.include_in_root
   - is_true: purple-index.mappings.properties.nested.include_in_parent
+
+---
+"Index template ignore_missing_component_template valid":
+  - skip:
+      version: " - 8.6.99"
+      reason: "index template v2 ignore_missing_component_template config not available before 8.7"
+      features: allowed_warnings
+
+  - do:
+      cluster.put_component_template:
+        name: red
+        body:
+          template:
+            mappings:
+              properties:
+                foo:
+                  type: keyword
+
+  - do:
+      allowed_warnings:
+        - "index template [blue] has index patterns [purple-index] matching patterns from existing older templates [global] with patterns (global => [*]); this template [blue] will take precedence during new index creation"
+      indices.put_index_template:
+        name: blue
+        body:
+          index_patterns: ["purple-index"]
+          composed_of: ["red", "blue"]
+          ignore_missing_component_templates: ["blue"]
+
+  - do:
+      indices.create:
+        index: purple-index
+
+  - do:
+      indices.get:
+        index: purple-index
+
+  - match: {purple-index.mappings.properties.foo: {type: keyword}}
+
+---
+"Index template ignore_missing_component_template invalid":
+  - skip:
+      version: " - 8.6.99"
+      reason: "index template v2 ignore_missing_component_template config not available before 8.7"
+      features: allowed_warnings
+
+  - do:
+      cluster.put_component_template:
+        name: red
+        body:
+          template:
+            mappings:
+              properties:
+                foo:
+                  type: keyword
+
+  - do:
+      catch: /index_template \[blue\] invalid, cause \[index template \[blue\] specifies a missing component templates \[blue\] that does not exist/
+      indices.put_index_template:
+        name: blue
+        body:
+          index_patterns: ["purple-index"]
+          composed_of: ["red", "blue"]
+          ignore_missing_component_templates: ["foo"]
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.sort/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.sort/10_basic.yml
index b9089689b0cf..8c39340a0af3 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.sort/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.sort/10_basic.yml
@@ -7,7 +7,6 @@
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index.sort.field: rank
           mappings:
             properties:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mlt/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mlt/10_basic.yml
index 47679b7f1058..7e05526eac9a 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mlt/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mlt/10_basic.yml
@@ -4,9 +4,6 @@
      indices.create:
        index:    test_1
        body:
-         settings:
-           index:
-             number_of_replicas: 0
          mappings:
            properties:
              foo:
@@ -23,10 +20,6 @@
   - do:
       indices.refresh: {}
 
-  - do:
-      cluster.health:
-           wait_for_status: green
-
   - do:
       search:
           rest_total_hits_as_int: true
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mlt/20_docs.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mlt/20_docs.yml
index 188e817ce759..c7477c5b538a 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mlt/20_docs.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mlt/20_docs.yml
@@ -3,9 +3,7 @@
   - do:
       indices.create:
         index: test_1
-        body:
-          settings:
-            number_of_replicas: 0
+
   - do:
       index:
           index:  test_1
@@ -27,10 +25,6 @@
   - do:
       indices.refresh: {}
 
-  - do:
-      cluster.health:
-           wait_for_status: green
-
   - do:
       search:
           rest_total_hits_as_int: true
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mlt/30_unlike.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mlt/30_unlike.yml
index c913268d807d..33fe1afce75b 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mlt/30_unlike.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mlt/30_unlike.yml
@@ -3,9 +3,7 @@
   - do:
       indices.create:
         index: test_1
-        body:
-          settings:
-            number_of_replicas: 0
+
   - do:
       index:
           index:  test_1
@@ -27,10 +25,6 @@
   - do:
       indices.refresh: {}
 
-  - do:
-      cluster.health:
-           wait_for_status: green
-
   - do:
       search:
           rest_total_hits_as_int: true
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/msearch/20_typed_keys.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/msearch/20_typed_keys.yml
index 89bc8bf53b8c..6e67dbd61f08 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/msearch/20_typed_keys.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/msearch/20_typed_keys.yml
@@ -4,8 +4,6 @@ setup:
       indices.create:
         index: test-0
         body:
-          settings:
-            number_of_replicas: 0
           mappings:
             properties:
               index_start_at:
@@ -23,8 +21,6 @@ setup:
       indices.create:
         index: test-1
         body:
-          settings:
-            number_of_replicas: 0
           mappings:
             properties:
               index_start_at:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/10_basic.yml
index 6dc62b24a39d..f3cb3f7810e3 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/10_basic.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/10_basic.yml
@@ -3,8 +3,6 @@ setup:
       indices.create:
           index: test
           body:
-            settings:
-              number_of_replicas: 0
             mappings:
               "properties":
                 "integer_range":
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/scroll/12_slices.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/scroll/12_slices.yml
index 6cc590a36d6c..2eadd3ec294a 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/scroll/12_slices.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/scroll/12_slices.yml
@@ -51,6 +51,7 @@ setup:
             match_all: {}
 
   - set: {_scroll_id: scroll_id}
+  - match: { _shards.successful: 5 }
   - match: {hits.total:      3    }
   - length: {hits.hits:      3    }
   - match: {hits.hits.0._id: "2" }
@@ -160,6 +161,7 @@ setup:
             match_all: {}
 
   - set: {_scroll_id: scroll_id}
+  - match: { _shards.successful: 5 }
   - match: { hits.total.value: 3 }
   - length: { hits.hits: 3 }
 
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/20_fvh.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/20_fvh.yml
index adb361098c94..a832ca222288 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/20_fvh.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/20_fvh.yml
@@ -22,7 +22,6 @@ setup:
   - do:
       index:
         index: test
-        refresh: true
         body:
           id : 1
           title : "The quick brown fox is brown"
@@ -31,7 +30,6 @@ setup:
   - do:
       index:
         index: test
-        refresh: true
         body:
           id : 2
           title : "The quick blue fox is blue"
@@ -39,6 +37,10 @@ setup:
             - title: "purple octopus"
             - title: "purple fish"
 
+  - do:
+      indices.refresh:
+        index: [test]
+
 ---
 "Highlight query":
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/50_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/50_synthetic_source.yml
index a9593ae5dae1..8f2333605b31 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/50_synthetic_source.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/50_synthetic_source.yml
@@ -30,7 +30,6 @@ setup:
       index:
         index:   test
         id:      1
-        refresh: true
         body:
           foo: the quick brown fox jumped over the lazy dog
 
@@ -38,7 +37,6 @@ setup:
       index:
         index:   test
         id:      2
-        refresh: true
         body:
           foo:
             - "To be, or not to be, that is the question:"
@@ -75,6 +73,10 @@ setup:
             - "With this regard their currents turn awry"
             - "And lose the name of action."
 
+  - do:
+      indices.refresh:
+        index: [test]
+
 ---
 keyword single plain:
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml
index 0ffec61788a7..26b8ab46bd4e 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml
@@ -6,8 +6,6 @@ setup:
       indices.create:
         index: test
         body:
-          settings:
-            number_of_replicas: 0
           mappings:
             properties:
               name:
@@ -17,6 +15,11 @@ setup:
                 dims: 5
                 index: true
                 similarity: l2_norm
+              another_vector:
+                type: dense_vector
+                dims: 5
+                index: true
+                similarity: l2_norm
 
   - do:
       index:
@@ -25,6 +28,7 @@ setup:
         body:
           name: cow.jpg
           vector: [230.0, 300.33, -34.8988, 15.555, -200.0]
+          another_vector: [130.0, 115.0, -1.02, 15.555, -100.0]
 
   - do:
       index:
@@ -33,6 +37,7 @@ setup:
         body:
           name: moose.jpg
           vector: [-0.5, 100.0, -13, 14.8, -156.0]
+          another_vector: [-0.5, 50.0, -1, 1, 120]
 
   - do:
       index:
@@ -41,6 +46,7 @@ setup:
         body:
           name: rabbit.jpg
           vector: [0.5, 111.3, -13.0, 14.8, -156.0]
+          another_vector: [-0.5, 11.0, 0, 12, 111.0]
 
   - do:
       indices.refresh: {}
@@ -66,7 +72,25 @@ setup:
 
   - match: {hits.hits.1._id: "3"}
   - match: {hits.hits.1.fields.name.0: "rabbit.jpg"}
+---
+"kNN multi-field search only":
+  - skip:
+      version: ' - 8.6.99'
+      reason: 'multi-field kNN search added to search endpoint in 8.7'
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "name" ]
+          knn:
+           - {field: vector, query_vector: [-0.5, 90.0, -10, 14.8, -156.0], k: 2, num_candidates: 3}
+           - {field: another_vector, query_vector: [-0.5, 11.0, 0, 12, 111.0], k: 2, num_candidates: 3}
+
+  - match: {hits.hits.0._id: "3"}
+  - match: {hits.hits.0.fields.name.0: "rabbit.jpg"}
 
+  - match: {hits.hits.1._id: "2"}
+  - match: {hits.hits.1.fields.name.0: "moose.jpg"}
 ---
 "kNN search plus query":
   - skip:
@@ -94,7 +118,31 @@ setup:
 
   - match: {hits.hits.2._id: "3"}
   - match: {hits.hits.2.fields.name.0: "rabbit.jpg"}
+---
+"kNN multi-field search with query":
+  - skip:
+      version: ' - 8.6.99'
+      reason: 'multi-field kNN search added to search endpoint in 8.7'
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "name" ]
+          knn:
+            - {field: vector, query_vector: [-0.5, 90.0, -10, 14.8, -156.0], k: 2, num_candidates: 3}
+            - {field: another_vector, query_vector: [-0.5, 11.0, 0, 12, 111.0], k: 2, num_candidates: 3}
+          query:
+            term:
+              name: cow.jpg
+
+  - match: {hits.hits.0._id: "3"}
+  - match: {hits.hits.0.fields.name.0: "rabbit.jpg"}
 
+  - match: {hits.hits.1._id: "1"}
+  - match: {hits.hits.1.fields.name.0: "cow.jpg"}
+
+  - match: {hits.hits.2._id: "2"}
+  - match: {hits.hits.2.fields.name.0: "moose.jpg"}
 ---
 "kNN search with filter":
   - skip:
@@ -110,7 +158,6 @@ setup:
             query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
             k: 2
             num_candidates: 3
-
             filter:
               term:
                 name: "rabbit.jpg"
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_byte.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_byte.yml
index a6177a0f586b..a9120547c886 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_byte.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_byte.yml
@@ -7,8 +7,6 @@ setup:
       indices.create:
         index: test
         body:
-          settings:
-            number_of_replicas: 0
           mappings:
             properties:
               name:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/60_knn_search_filter_alias.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/60_knn_search_filter_alias.yml
index b802d19bfe08..92065857bd57 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/60_knn_search_filter_alias.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/60_knn_search_filter_alias.yml
@@ -9,7 +9,6 @@ setup:
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
           mappings:
             dynamic: false
             properties:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/70_dense_vector_telemetry.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/70_dense_vector_telemetry.yml
index 165ec327c73f..136dc807a89d 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/70_dense_vector_telemetry.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/70_dense_vector_telemetry.yml
@@ -8,7 +8,6 @@ setup:
         body:
           settings:
             index.number_of_shards: 2
-            index.number_of_replicas: 0
           mappings:
             properties:
               vector1:
@@ -32,7 +31,6 @@ setup:
         body:
           settings:
             index.number_of_shards: 2
-            index.number_of_replicas: 0
           mappings:
             properties:
               vector1:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/240_date_nanos.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/240_date_nanos.yml
index 95f1ee6cd9f3..f35bf691fbdc 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/240_date_nanos.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/240_date_nanos.yml
@@ -7,7 +7,6 @@ setup:
           body:
             settings:
               number_of_shards: 3
-              number_of_replicas: 0
             mappings:
               properties:
                 date:
@@ -21,7 +20,6 @@ setup:
           body:
             settings:
               number_of_shards: 3
-              number_of_replicas: 0
             mappings:
               properties:
                 date:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/250_distance_feature.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/250_distance_feature.yml
index bafb7d52c718..e00d7bbb7236 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/250_distance_feature.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/250_distance_feature.yml
@@ -7,8 +7,6 @@ setup:
       indices.create:
         index: index1
         body:
-          settings:
-            number_of_replicas: 0
           mappings:
             properties:
               my_date:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml
index 415da2ad8767..c0b754c02275 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml
@@ -168,8 +168,8 @@ disabling stored fields removes fetch sub phases:
 ---
 dfs knn vector profiling:
   - skip:
-      version: ' - 8.5.99'
-      reason: dfs profiling implemented in 8.6.0
+      version: ' - 8.6.99'
+      reason: multi-knn dfs profiling implemented in 8.7.0
 
   - do:
       indices.create:
@@ -205,31 +205,31 @@ dfs knn vector profiling:
             num_candidates: 100
 
   - match: { hits.total.value: 1 }
-  - match: { profile.shards.0.dfs.knn.query.0.type: "DocAndScoreQuery" }
-  - match: { profile.shards.0.dfs.knn.query.0.description: "DocAndScore[100]" }
-  - gt: { profile.shards.0.dfs.knn.query.0.time_in_nanos: 0 }
-  - match: { profile.shards.0.dfs.knn.query.0.breakdown.set_min_competitive_score_count: 0 }
-  - match: { profile.shards.0.dfs.knn.query.0.breakdown.set_min_competitive_score: 0 }
-  - match: { profile.shards.0.dfs.knn.query.0.breakdown.match_count: 0 }
-  - match: { profile.shards.0.dfs.knn.query.0.breakdown.match: 0 }
-  - match: { profile.shards.0.dfs.knn.query.0.breakdown.shallow_advance_count: 0 }
-  - match: { profile.shards.0.dfs.knn.query.0.breakdown.shallow_advance: 0 }
-  - gt: { profile.shards.0.dfs.knn.query.0.breakdown.next_doc_count: 0 }
-  - gt: { profile.shards.0.dfs.knn.query.0.breakdown.next_doc: 0 }
-  - gt: { profile.shards.0.dfs.knn.query.0.breakdown.score_count: 0 }
-  - gt: { profile.shards.0.dfs.knn.query.0.breakdown.score: 0 }
-  - match: { profile.shards.0.dfs.knn.query.0.breakdown.compute_max_score_count: 0 }
-  - match: { profile.shards.0.dfs.knn.query.0.breakdown.compute_max_score: 0 }
-  - gt: { profile.shards.0.dfs.knn.query.0.breakdown.advance_count: 0 }
-  - gt: { profile.shards.0.dfs.knn.query.0.breakdown.advance: 0 }
-  - gt: { profile.shards.0.dfs.knn.query.0.breakdown.build_scorer_count: 0 }
-  - gt: { profile.shards.0.dfs.knn.query.0.breakdown.build_scorer: 0 }
-  - gt: { profile.shards.0.dfs.knn.query.0.breakdown.create_weight: 0 }
-  - gt: { profile.shards.0.dfs.knn.query.0.breakdown.create_weight_count: 0 }
-  - gt: { profile.shards.0.dfs.knn.rewrite_time: 0 }
-  - match: { profile.shards.0.dfs.knn.collector.0.name: "SimpleTopScoreDocCollector" }
-  - match: { profile.shards.0.dfs.knn.collector.0.reason: "search_top_hits" }
-  - gt: { profile.shards.0.dfs.knn.collector.0.time_in_nanos: 0 }
+  - match: { profile.shards.0.dfs.knn.0.query.0.type: "DocAndScoreQuery" }
+  - match: { profile.shards.0.dfs.knn.0.query.0.description: "DocAndScore[100]" }
+  - gt: { profile.shards.0.dfs.knn.0.query.0.time_in_nanos: 0 }
+  - match: { profile.shards.0.dfs.knn.0.query.0.breakdown.set_min_competitive_score_count: 0 }
+  - match: { profile.shards.0.dfs.knn.0.query.0.breakdown.set_min_competitive_score: 0 }
+  - match: { profile.shards.0.dfs.knn.0.query.0.breakdown.match_count: 0 }
+  - match: { profile.shards.0.dfs.knn.0.query.0.breakdown.match: 0 }
+  - match: { profile.shards.0.dfs.knn.0.query.0.breakdown.shallow_advance_count: 0 }
+  - match: { profile.shards.0.dfs.knn.0.query.0.breakdown.shallow_advance: 0 }
+  - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.next_doc_count: 0 }
+  - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.next_doc: 0 }
+  - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.score_count: 0 }
+  - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.score: 0 }
+  - match: { profile.shards.0.dfs.knn.0.query.0.breakdown.compute_max_score_count: 0 }
+  - match: { profile.shards.0.dfs.knn.0.query.0.breakdown.compute_max_score: 0 }
+  - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.advance_count: 0 }
+  - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.advance: 0 }
+  - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.build_scorer_count: 0 }
+  - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.build_scorer: 0 }
+  - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.create_weight: 0 }
+  - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.create_weight_count: 0 }
+  - gt: { profile.shards.0.dfs.knn.0.rewrite_time: 0 }
+  - match: { profile.shards.0.dfs.knn.0.collector.0.name: "SimpleTopScoreDocCollector" }
+  - match: { profile.shards.0.dfs.knn.0.collector.0.reason: "search_top_hits" }
+  - gt: { profile.shards.0.dfs.knn.0.collector.0.time_in_nanos: 0 }
 
 ---
 dfs profile for search with dfs_query_then_fetch:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/380_sort_segments_on_timestamp.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/380_sort_segments_on_timestamp.yml
index 8a1f6b03ef8b..fb45d63aad7f 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/380_sort_segments_on_timestamp.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/380_sort_segments_on_timestamp.yml
@@ -15,7 +15,6 @@
                 type: date
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
 
   # 1st segment
   - do:
@@ -54,7 +53,6 @@
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
 
   # 1st segment
   - do:
@@ -114,7 +112,6 @@
                 type: date
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
 
   # 1st segment missing @timestamp field
   - do:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/400_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/400_synthetic_source.yml
index 902f7086e94d..e013992e06c9 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/400_synthetic_source.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/400_synthetic_source.yml
@@ -436,3 +436,61 @@ _source filtering:
   - match:
       hits.hits.0._source:
         kwd: foo
+
+---
+_doc_count:
+  - skip:
+      version: " - 8.6.99"
+      reason: bug caused by many not having _doc_count fixed in 8.7.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+
+  - do:
+      index:
+        index:   test
+        id:      2
+        body:
+          foo: baz
+  - do:
+      index:
+        index:   test
+        id:      3
+        body:
+          foo: baz
+  - do:
+      index:
+        index:   test
+        id:      4
+        body:
+          foo: baz
+  - do:
+      index:
+        index:   test
+        id:      1
+        body:
+          _doc_count: 3
+          foo: bar
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        index: test
+        body:
+          sort: foo.keyword
+  - is_false: hits.hits.0.fields
+  - is_false: hits.hits.1.fields
+  - match:
+      hits.hits.0._source:
+        _doc_count: 3
+        foo: bar
+  - match:
+      hits.hits.1._source:
+        foo: baz
+
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/suggest/40_typed_keys.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/suggest/40_typed_keys.yml
index daac7d895611..89e26ec113f4 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/suggest/40_typed_keys.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/suggest/40_typed_keys.yml
@@ -4,8 +4,6 @@ setup:
       indices.create:
           index: test
           body:
-            settings:
-              number_of_replicas: 0
             mappings:
               properties:
                 title:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml
index 458b27149fd8..467002d2243e 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml
@@ -9,7 +9,6 @@ add time series mappings:
           body:
             settings:
               index:
-                number_of_replicas: 0
                 number_of_shards: 2
             mappings:
               properties:
@@ -60,7 +59,6 @@ can't shadow dimensions:
           body:
             settings:
               index:
-                number_of_replicas: 0
                 number_of_shards: 2
             mappings:
               properties:
@@ -126,7 +124,6 @@ can't shadow metrics:
           body:
             settings:
               index:
-                number_of_replicas: 0
                 number_of_shards: 2
             mappings:
               properties:
@@ -192,7 +189,6 @@ no _tsid in standard indices:
         body:
           settings:
             index:
-              number_of_replicas: 0
               number_of_shards: 2
           mappings:
             properties:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml
index 786df469954f..dbc8076c0a1a 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml
@@ -16,7 +16,6 @@ date:
                 time_series:
                   start_time: 2021-04-28T00:00:00Z
                   end_time: 2021-04-29T00:00:00Z
-                number_of_replicas: 0
                 number_of_shards: 2
             mappings:
               properties:
@@ -65,7 +64,6 @@ date_nanos:
                 time_series:
                   start_time: 2021-04-28T00:00:00Z
                   end_time: 2021-04-29T00:00:00Z
-                number_of_replicas: 0
                 number_of_shards: 2
             mappings:
               properties:
@@ -114,7 +112,6 @@ automatically add with date:
                 time_series:
                   start_time: 2021-04-28T00:00:00Z
                   end_time: 2021-04-29T00:00:00Z
-                number_of_replicas: 0
                 number_of_shards: 2
             mappings:
               properties:
@@ -162,7 +159,6 @@ reject @timestamp with wrong type:
                 time_series:
                   start_time: 2021-04-28T00:00:00Z
                   end_time: 2021-04-29T00:00:00Z
-                number_of_replicas: 0
                 number_of_shards: 2
             mappings:
               properties:
@@ -187,7 +183,6 @@ reject timestamp meta field with wrong type:
               time_series:
                 start_time: 2021-04-28T00:00:00Z
                 end_time: 2021-04-29T00:00:00Z
-              number_of_replicas: 0
               number_of_shards: 2
           mappings:
             _data_stream_timestamp:
@@ -210,7 +205,6 @@ enable timestamp meta field:
               time_series:
                 start_time: 2021-04-28T00:00:00Z
                 end_time: 2021-04-29T00:00:00Z
-              number_of_replicas: 0
               number_of_shards: 2
           mappings:
             _data_stream_timestamp:
@@ -240,7 +234,6 @@ reject bad timestamp meta field:
               time_series:
                 start_time: 2021-04-28T00:00:00Z
                 end_time: 2021-04-29T00:00:00Z
-              number_of_replicas: 0
               number_of_shards: 2
           mappings:
             _data_stream_timestamp: enabled
@@ -262,7 +255,6 @@ write without timestamp:
               time_series:
                 start_time: 2021-04-28T00:00:00Z
                 end_time: 2021-04-29T00:00:00Z
-              number_of_replicas: 0
               number_of_shards: 2
           mappings:
             properties:
@@ -302,7 +294,6 @@ explicitly enable timestamp meta field:
               time_series:
                 start_time: 2021-04-28T00:00:00Z
                 end_time: 2021-04-29T00:00:00Z
-              number_of_replicas: 0
               number_of_shards: 2
           mappings:
             properties:
@@ -353,7 +344,6 @@ unable to create a time series index with @timestamp runtime field:
         body:
           settings:
             number_of_shards: 1
-            number_of_replicas: 0
             index:
               mode: time_series
               routing_path: [metricset, k8s.pod.uid]
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml
index a5f0155f950c..a1c1c87ee687 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml
@@ -413,3 +413,124 @@ nested fields:
                           rx:
                             type: long
                             time_series_metric: gauge
+
+---
+regular source:
+  - skip:
+      version: " - 8.6.99"
+      reason: synthetic source
+
+  - do:
+      catch: '/time series indices only support synthetic source/'
+      indices.create:
+        index: tsdb_index
+        body:
+          settings:
+            index:
+              mode: time_series
+              routing_path: [k8s.pod.uid]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            _source:
+              mode: stored
+            properties:
+              "@timestamp":
+                type: date
+              k8s:
+                properties:
+                  pod:
+                    properties:
+                      uid:
+                        type: keyword
+                        time_series_dimension: true
+---
+disabled source:
+  - skip:
+      version: " - 8.6.99"
+      reason: synthetic source
+
+  - do:
+      catch: '/time series indices only support synthetic source/'
+      indices.create:
+        index: tsdb_index
+        body:
+          settings:
+            index:
+              mode: time_series
+              routing_path: [k8s.pod.uid]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            _source:
+              mode: disabled
+            properties:
+              "@timestamp":
+                type: date
+              k8s:
+                properties:
+                  pod:
+                    properties:
+                      uid:
+                        type: keyword
+                        time_series_dimension: true
+
+---
+source include/exclude:
+  - skip:
+      version: " - 8.6.99"
+      reason: synthetic source
+
+  - do:
+      catch: '/filtering the stored _source is incompatible with synthetic source/'
+      indices.create:
+        index: tsdb_index
+        body:
+          settings:
+            index:
+              mode: time_series
+              routing_path: [k8s.pod.uid]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            _source:
+              includes: [a]
+            properties:
+              "@timestamp":
+                type: date
+              k8s:
+                properties:
+                  pod:
+                    properties:
+                      uid:
+                        type: keyword
+                        time_series_dimension: true
+
+  - do:
+      catch: '/filtering the stored _source is incompatible with synthetic source/'
+      indices.create:
+        index: tsdb_index
+        body:
+          settings:
+            index:
+              mode: time_series
+              routing_path: [k8s.pod.uid]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            _source:
+              excludes: [b]
+            properties:
+              "@timestamp":
+                type: date
+              k8s:
+                properties:
+                  pod:
+                    properties:
+                      uid:
+                        type: keyword
+                        time_series_dimension: true
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml
index ee67c2fe33b6..a2aab4d8925f 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml
@@ -14,7 +14,6 @@ setup:
                 time_series:
                   start_time: 2021-04-28T00:00:00Z
                   end_time: 2021-04-29T00:00:00Z
-                number_of_replicas: 0
                 number_of_shards: 2
             mappings:
               properties:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml
index 03872f79a037..f78f16780cb4 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml
@@ -148,94 +148,3 @@ clone:
 
   - match: {hits.total.value: 1}
   - match: {hits.hits.0.fields._tsid: [{k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}]}
-
----
-clone no source index:
-  - skip:
-      version: " - 8.1.99"
-      reason: tsdb indexing changed in 8.2.0
-
-  - do:
-      indices.create:
-        index: test_no_source
-        body:
-          settings:
-            index:
-              mode: time_series
-              routing_path: [ metricset, k8s.pod.uid ]
-              time_series:
-                start_time: 2021-04-28T00:00:00Z
-                end_time: 2021-04-29T00:00:00Z
-              number_of_shards: 1
-              number_of_replicas: 0
-          mappings:
-            _source:
-              enabled: false
-            properties:
-              "@timestamp":
-                type: date
-              metricset:
-                type: keyword
-                time_series_dimension: true
-              k8s:
-                properties:
-                  pod:
-                    properties:
-                      uid:
-                        type: keyword
-                        time_series_dimension: true
-                      name:
-                        type: keyword
-                      ip:
-                        type: ip
-                      network:
-                        properties:
-                          tx:
-                            type: long
-                          rx:
-                            type: long
-
-  - do:
-      bulk:
-        refresh: true
-        index: test_no_source
-        body:
-          - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
-          - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2005177954, "rx": 801479970}}}}'
-          - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2006223737, "rx": 802337279}}}}'
-          - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.2", "network": {"tx": 2012916202, "rx": 803685721}}}}'
-          - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434521831, "rx": 530575198}}}}'
-          - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434577921, "rx": 530600088}}}}'
-          - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434587694, "rx": 530604797}}}}'
-          - '{"index": {}}'
-          - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}'
-
-  - do:
-      indices.put_settings:
-        index: test_no_source
-        body:
-          index.blocks.write: true
-
-  - do:
-      indices.clone:
-        index: test_no_source
-        target: test_no_source_clone
-
-  - do:
-      search:
-        index: test_no_source_clone
-        body:
-          docvalue_fields:
-            - field: _tsid
-          query:
-            query_string:
-              query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat'
-  - match: {hits.total.value: 1}
-  - match: {hits.hits.0.fields._tsid: [{k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}]}
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml
index 782e8226ebb3..adb71b912572 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml
@@ -14,7 +14,6 @@ setup:
                 time_series:
                   start_time: 2021-04-28T00:00:00Z
                   end_time: 2021-04-29T00:00:00Z
-                number_of_replicas: 0
                 number_of_shards: 2
             mappings:
               properties:
@@ -229,7 +228,7 @@ aggregate on _id:
         index: test
         body:
           size: 1
-          aggs: 
+          aggs:
             id:
               terms:
                 field: _id
diff --git a/server/build.gradle b/server/build.gradle
index 9e1985d5643c..b360afa1ad62 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -77,6 +77,12 @@ dependencies {
 
 }
 
+spotless {
+  java {
+    targetExclude "src/main/java/org/elasticsearch/Version.java"
+  }
+}
+
 tasks.named("forbiddenPatterns").configure {
     exclude '**/*.json'
     exclude '**/*.jmx'
@@ -229,6 +235,14 @@ tasks.named("thirdPartyAudit").configure {
             'org.zeromq.ZMQ',
     )
     ignoreMissingClasses 'javax.xml.bind.DatatypeConverter'
+
+    if (BuildParams.runtimeJavaVersion == JavaVersion.VERSION_20) {
+      ignoreMissingClasses(
+        // This class was removed in Java 20 but is only referenced by a class that requires preview features anyhow
+        // See: https://github.com/apache/lucene/pull/12042
+        'java.lang.foreign.MemorySession',
+      )
+    }
 }
 
 tasks.named("dependencyLicenses").configure {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
index 9ceeba1c7a59..76200604338d 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
@@ -366,7 +366,7 @@ public void testUnassignedReplicaWithPriorCopy() throws Exception {
                     assertThat(d.getExplanation(), startsWith("a copy of this shard is already allocated to this node ["));
                 } else if (d.label().equals("filter") && nodeHoldingPrimary == false) {
                     assertEquals(Decision.Type.NO, d.type());
-                    assertEquals(formatted("""
+                    assertEquals(Strings.format("""
                         node does not match index setting [index.routing.allocation.include] \
                         filters [_name:"%s"]\
                         """, primaryNodeName), d.getExplanation());
@@ -914,7 +914,7 @@ public void testBetterBalanceButCannotAllocate() throws Exception {
         for (Decision d : result.getCanAllocateDecision().getDecisions()) {
             if (d.label().equals("filter")) {
                 assertEquals(Decision.Type.NO, d.type());
-                assertEquals(formatted("""
+                assertEquals(Strings.format("""
                     node does not match index setting [index.routing.allocation.include] filters [_name:"%s"]\
                     """, primaryNodeName), d.getExplanation());
             } else {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java
index 2ce42b653c1f..3bcd7626a5f0 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksIT.java
@@ -508,8 +508,8 @@ protected void doExecute(Task task, TestRequest request, ActionListener<TestResp
             arrivedLatches.get(request).countDown();
             List<TestRequest> subRequests = request.subRequests;
             GroupedActionListener<TestResponse> groupedListener = new GroupedActionListener<>(
-                listener.map(r -> new TestResponse()),
-                subRequests.size() + 1
+                subRequests.size() + 1,
+                listener.map(r -> new TestResponse())
             );
             transportService.getThreadPool().generic().execute(ActionRunnable.supply(groupedListener, () -> {
                 assertTrue(beforeExecuteLatches.get(request).await(60, TimeUnit.SECONDS));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
index f019438efd92..36738d4f7681 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
@@ -43,6 +43,7 @@
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.tasks.RemovedTaskListener;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.tasks.TaskId;
@@ -264,7 +265,7 @@ public void testTransportBroadcastReplicationTasks() {
         // we will have as many [s][p] and [s][r] tasks as we have primary and replica shards
         assertEquals(numberOfShards.totalNumShards, numberOfEvents(RefreshAction.NAME + "[s][*]", Tuple::v1));
 
-        // we the [s][p] and [s][r] tasks should have a corresponding [s] task on the same node as a parent
+        // the [s][p] and [s][r] tasks should have a corresponding [s] task on the same node as a parent
         List<TaskInfo> spEvents = findEvents(RefreshAction.NAME + "[s][*]", Tuple::v1);
         for (TaskInfo taskInfo : spEvents) {
             List<TaskInfo> sTask;
@@ -272,17 +273,17 @@ public void testTransportBroadcastReplicationTasks() {
                 // A [s][p] level task should have a corresponding [s] level task on the same node
                 sTask = findEvents(
                     RefreshAction.NAME + "[s]",
-                    event -> event.v1()
-                        && taskInfo.taskId().getNodeId().equals(event.v2().taskId().getNodeId())
-                        && taskInfo.description().equals(event.v2().description())
+                    event -> event.v1() // task registration event
+                        && event.v2().taskId().equals(taskInfo.parentTaskId())
+                        && event.v2().taskId().getNodeId().equals(taskInfo.taskId().getNodeId())
                 );
             } else {
-                // A [s][r] level task should have a corresponding [s] level task on the a different node (where primary is located)
+                // A [s][r] level task should have a corresponding [s] level task on a different node (where primary is located)
                 sTask = findEvents(
                     RefreshAction.NAME + "[s]",
-                    event -> event.v1()
-                        && taskInfo.parentTaskId().getNodeId().equals(event.v2().taskId().getNodeId())
-                        && taskInfo.description().equals(event.v2().description())
+                    event -> event.v1() // task registration event
+                        && event.v2().taskId().equals(taskInfo.parentTaskId())
+                        && event.v2().taskId().getNodeId().equals(taskInfo.taskId().getNodeId()) == false
                 );
             }
             // There should be only one parent task
@@ -636,6 +637,11 @@ public void onTaskRegistered(Task task) {}
 
                     @Override
                     public void onTaskUnregistered(Task task) {}
+
+                    @Override
+                    public void subscribeForRemovedTasks(RemovedTaskListener removedTaskListener) {
+                        waitForWaitingToStart.countDown();
+                    }
                 });
             }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/ListTasksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/ListTasksIT.java
index f60ab6204cc2..f4b1903bba67 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/ListTasksIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/ListTasksIT.java
@@ -8,15 +8,38 @@
 
 package org.elasticsearch.action.admin.cluster.tasks;
 
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ActionRunnable;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 
-public class ListTasksIT extends ESIntegTestCase {
+public class ListTasksIT extends ESSingleNodeTestCase {
 
     public void testListTasksFilteredByDescription() {
 
@@ -40,4 +63,152 @@ public void testListTasksValidation() {
         assertThat(ex.getMessage(), containsString("matching on descriptions is not available when [detailed] is false"));
 
     }
+
+    public void testWaitForCompletion() throws Exception {
+        final var threadPool = getInstanceFromNode(ThreadPool.class);
+        final var threadContext = threadPool.getThreadContext();
+
+        final var barrier = new CyclicBarrier(2);
+        getInstanceFromNode(PluginsService.class).filterPlugins(TestPlugin.class).get(0).barrier = barrier;
+
+        final var testActionFuture = new PlainActionFuture<ActionResponse.Empty>();
+        client().execute(TEST_ACTION, new TestRequest(), testActionFuture.map(r -> {
+            assertThat(threadContext.getResponseHeaders().get(TestTransportAction.HEADER_NAME), hasItem(TestTransportAction.HEADER_VALUE));
+            return r;
+        }));
+
+        barrier.await(10, TimeUnit.SECONDS);
+
+        final var listTasksResponse = client().admin().cluster().prepareListTasks().setActions(TestTransportAction.NAME).get();
+        assertThat(listTasksResponse.getNodeFailures(), empty());
+        assertEquals(1, listTasksResponse.getTasks().size());
+        final var task = listTasksResponse.getTasks().get(0);
+        assertEquals(TestTransportAction.NAME, task.action());
+
+        final var listWaitFuture = new PlainActionFuture<Void>();
+        client().admin()
+            .cluster()
+            .prepareListTasks()
+            .setTargetTaskId(task.taskId())
+            .setWaitForCompletion(true)
+            .execute(listWaitFuture.delegateFailure((l, listResult) -> {
+                assertEquals(1, listResult.getTasks().size());
+                assertEquals(task.taskId(), listResult.getTasks().get(0).taskId());
+                // the task must now be complete:
+                client().admin().cluster().prepareListTasks().setActions(TestTransportAction.NAME).execute(l.map(listAfterWaitResult -> {
+                    assertThat(listAfterWaitResult.getTasks(), empty());
+                    assertThat(listAfterWaitResult.getNodeFailures(), empty());
+                    assertThat(listAfterWaitResult.getTaskFailures(), empty());
+                    return null;
+                }));
+                // and we must not see its header:
+                assertNull(threadContext.getResponseHeaders().get(TestTransportAction.HEADER_NAME));
+            }));
+
+        // briefly fill up the management pool so that (a) we know the wait has started and (b) we know it's not blocking
+        flushThreadPool(threadPool, ThreadPool.Names.MANAGEMENT);
+
+        final var getWaitFuture = new PlainActionFuture<Void>();
+        client().admin()
+            .cluster()
+            .prepareGetTask(task.taskId())
+            .setWaitForCompletion(true)
+            .execute(getWaitFuture.delegateFailure((l, getResult) -> {
+                assertTrue(getResult.getTask().isCompleted());
+                assertEquals(task.taskId(), getResult.getTask().getTask().taskId());
+                // the task must now be complete:
+                client().admin().cluster().prepareListTasks().setActions(TestTransportAction.NAME).execute(l.map(listAfterWaitResult -> {
+                    assertThat(listAfterWaitResult.getTasks(), empty());
+                    assertThat(listAfterWaitResult.getNodeFailures(), empty());
+                    assertThat(listAfterWaitResult.getTaskFailures(), empty());
+                    return null;
+                }));
+                // and we must not see its header:
+                assertNull(threadContext.getResponseHeaders().get(TestTransportAction.HEADER_NAME));
+            }));
+
+        assertFalse(listWaitFuture.isDone());
+        assertFalse(testActionFuture.isDone());
+        barrier.await(10, TimeUnit.SECONDS);
+        testActionFuture.get(10, TimeUnit.SECONDS);
+        listWaitFuture.get(10, TimeUnit.SECONDS);
+        getWaitFuture.get(10, TimeUnit.SECONDS);
+    }
+
+    private void flushThreadPool(ThreadPool threadPool, String executor) throws InterruptedException, BrokenBarrierException,
+        TimeoutException {
+        var maxThreads = threadPool.info(executor).getMax();
+        var barrier = new CyclicBarrier(maxThreads + 1);
+        for (int i = 0; i < maxThreads; i++) {
+            threadPool.executor(executor).execute(() -> {
+                try {
+                    barrier.await(10, TimeUnit.SECONDS);
+                } catch (Exception e) {
+                    throw new AssertionError(e);
+                }
+            });
+        }
+        barrier.await(10, TimeUnit.SECONDS);
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return List.of(TestPlugin.class);
+    }
+
+    private static final ActionType<ActionResponse.Empty> TEST_ACTION = new ActionType<>(
+        TestTransportAction.NAME,
+        in -> ActionResponse.Empty.INSTANCE
+    );
+
+    public static class TestPlugin extends Plugin implements ActionPlugin {
+        volatile CyclicBarrier barrier;
+
+        @Override
+        public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+            return List.of(new ActionHandler<>(TEST_ACTION, TestTransportAction.class));
+        }
+    }
+
+    public static class TestRequest extends ActionRequest {
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+    }
+
+    public static class TestTransportAction extends HandledTransportAction<TestRequest, ActionResponse.Empty> {
+
+        static final String NAME = "internal:test/action";
+
+        static final String HEADER_NAME = "HEADER_NAME";
+        static final String HEADER_VALUE = "HEADER_VALUE";
+
+        private final TestPlugin testPlugin;
+        private final ThreadPool threadPool;
+
+        @Inject
+        public TestTransportAction(
+            TransportService transportService,
+            ActionFilters actionFilters,
+            PluginsService pluginsService,
+            ThreadPool threadPool
+        ) {
+            super(NAME, transportService, actionFilters, in -> new TestRequest());
+            testPlugin = pluginsService.filterPlugins(TestPlugin.class).get(0);
+            this.threadPool = threadPool;
+        }
+
+        @Override
+        protected void doExecute(Task task, TestRequest request, ActionListener<ActionResponse.Empty> listener) {
+            final var barrier = testPlugin.barrier;
+            assertNotNull(barrier);
+            threadPool.generic().execute(ActionRunnable.run(listener, () -> {
+                barrier.await(10, TimeUnit.SECONDS);
+                threadPool.getThreadContext().addResponseHeader(HEADER_NAME, HEADER_VALUE);
+                barrier.await(10, TimeUnit.SECONDS);
+            }));
+        }
+    }
+
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java
index 8b9173ed9945..568353ab2f1b 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/PendingTasksBlocksIT.java
@@ -38,7 +38,7 @@ public void testPendingTasksWithIndexBlocks() {
             try {
                 enableIndexBlock("test", blockSetting);
                 PendingClusterTasksResponse response = client().admin().cluster().preparePendingClusterTasks().get();
-                assertNotNull(response.getPendingTasks());
+                assertNotNull(response.pendingTasks());
             } finally {
                 disableIndexBlock("test", blockSetting);
             }
@@ -54,7 +54,7 @@ public void testPendingTasksWithClusterReadOnlyBlock() {
         try {
             setClusterReadOnly(true);
             PendingClusterTasksResponse response = client().admin().cluster().preparePendingClusterTasks().get();
-            assertNotNull(response.getPendingTasks());
+            assertNotNull(response.pendingTasks());
         } finally {
             setClusterReadOnly(false);
         }
@@ -80,7 +80,7 @@ public boolean validateClusterForming() {
             }
         });
 
-        assertNotNull(client().admin().cluster().preparePendingClusterTasks().get().getPendingTasks());
+        assertNotNull(client().admin().cluster().preparePendingClusterTasks().get().pendingTasks());
 
         // starting one more node allows the cluster to recover
         internalCluster().startNode();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java
index eb86ecd83e78..d3ac559a2104 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java
@@ -73,14 +73,15 @@ protected boolean forbidPrivateIndexSettings() {
     }
 
     public void testCreateSplitIndexToN() throws IOException {
+        assumeFalse("https://github.com/elastic/elasticsearch/issues/33857", Constants.WINDOWS);
+
         int[][] possibleShardSplits = new int[][] { { 2, 4, 8 }, { 3, 6, 12 }, { 1, 2, 4 } };
         int[] shardSplits = randomFrom(possibleShardSplits);
         splitToN(shardSplits[0], shardSplits[1], shardSplits[2]);
     }
 
     public void testSplitFromOneToN() {
-
-        assumeFalse("https://github.com/elastic/elasticsearch/issues/34080", Constants.WINDOWS);
+        assumeFalse("https://github.com/elastic/elasticsearch/issues/33857", Constants.WINDOWS);
 
         splitToN(1, 5, 10);
         client().admin().indices().prepareDelete("*").get();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
index 5a24caabbd1a..ba1e060efd91 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
@@ -837,8 +837,8 @@ public void testRolloverConcurrently() throws Exception {
                         .prepareRolloverIndex(aliasName)
                         .waitForActiveShards(ActiveShardCount.NONE)
                         .get();
-                    assertThat(response.getOldIndex(), equalTo(aliasName + formatted("-%06d", j)));
-                    assertThat(response.getNewIndex(), equalTo(aliasName + formatted("-%06d", j + 1)));
+                    assertThat(response.getOldIndex(), equalTo(aliasName + Strings.format("-%06d", j)));
+                    assertThat(response.getNewIndex(), equalTo(aliasName + Strings.format("-%06d", j + 1)));
                     assertThat(response.isDryRun(), equalTo(false));
                     assertThat(response.isRolledOver(), equalTo(true));
                 }
@@ -859,7 +859,7 @@ public void testRolloverConcurrently() throws Exception {
             for (int j = 1; j <= numOfIndices; j++) {
                 AliasMetadata.Builder amBuilder = new AliasMetadata.Builder(aliasName);
                 amBuilder.writeIndex(j == numOfIndices);
-                expected.add(Map.entry(aliasName + formatted("-%06d", j), List.of(amBuilder.build())));
+                expected.add(Map.entry(aliasName + Strings.format("-%06d", j), List.of(amBuilder.build())));
             }
             assertThat(actual, containsInAnyOrder(expected.toArray(Object[]::new)));
         }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2IT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2IT.java
new file mode 100644
index 000000000000..a9e66a912504
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2IT.java
@@ -0,0 +1,302 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.bulk;
+
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+
+import org.elasticsearch.action.get.MultiGetItemResponse;
+import org.elasticsearch.action.get.MultiGetRequestBuilder;
+import org.elasticsearch.action.get.MultiGetResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.client.internal.Requests;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.both;
+import static org.hamcrest.Matchers.either;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+public class BulkProcessor2IT extends ESIntegTestCase {
+
+    public void testThatBulkProcessor2CountIsCorrect() throws Exception {
+        final CountDownLatch latch = new CountDownLatch(1);
+        BulkProcessor2TestListener listener = new BulkProcessor2TestListener(latch);
+
+        int numDocs = randomIntBetween(10, 100);
+        BulkProcessor2 processor = BulkProcessor2.builder(client()::bulk, listener, client().threadPool())
+            // let's make sure that the bulk action limit trips, one single execution will index all the documents
+            .setBulkActions(numDocs)
+            .setFlushInterval(TimeValue.timeValueHours(24))
+            .setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB))
+            .build();
+        try {
+
+            MultiGetRequestBuilder multiGetRequestBuilder = indexDocs(client(), processor, numDocs);
+
+            latch.await();
+
+            assertThat(listener.beforeCounts.get(), equalTo(1));
+            assertThat(listener.afterCounts.get(), equalTo(1));
+            assertThat(listener.bulkFailures.size(), equalTo(0));
+            assertResponseItems(listener.bulkItems, numDocs);
+            assertMultiGetResponse(multiGetRequestBuilder.get(), numDocs);
+            assertThat(processor.getTotalBytesInFlight(), equalTo(0L));
+        } finally {
+            processor.awaitClose(5, TimeUnit.SECONDS);
+        }
+    }
+
+    public void testBulkProcessor2ConcurrentRequests() throws Exception {
+        int bulkActions = randomIntBetween(10, 100);
+        int numDocs = randomIntBetween(bulkActions, bulkActions + 100);
+
+        int expectedBulkActions = numDocs / bulkActions;
+
+        final CountDownLatch latch = new CountDownLatch(expectedBulkActions);
+        int totalExpectedBulkActions = numDocs % bulkActions == 0 ? expectedBulkActions : expectedBulkActions + 1;
+        final CountDownLatch closeLatch = new CountDownLatch(totalExpectedBulkActions);
+
+        BulkProcessor2TestListener listener = new BulkProcessor2TestListener(latch, closeLatch);
+
+        MultiGetRequestBuilder multiGetRequestBuilder;
+        BulkProcessor2 processor = BulkProcessor2.builder(client()::bulk, listener, client().threadPool())
+            .setBulkActions(bulkActions)
+            // set interval and size to high values
+            .setFlushInterval(TimeValue.timeValueHours(24))
+            .setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB))
+            .build();
+        try {
+
+            multiGetRequestBuilder = indexDocs(client(), processor, numDocs);
+
+            latch.await();
+
+            assertThat(listener.beforeCounts.get(), equalTo(expectedBulkActions));
+            assertThat(listener.afterCounts.get(), equalTo(expectedBulkActions));
+            assertThat(listener.bulkFailures.size(), equalTo(0));
+            assertThat(listener.bulkItems.size(), equalTo(numDocs - numDocs % bulkActions));
+        } finally {
+            processor.awaitClose(5, TimeUnit.SECONDS);
+        }
+
+        closeLatch.await();
+
+        assertThat(listener.beforeCounts.get(), equalTo(totalExpectedBulkActions));
+        assertThat(listener.afterCounts.get(), equalTo(totalExpectedBulkActions));
+        assertThat(listener.bulkFailures.size(), equalTo(0));
+        assertThat(listener.bulkItems.size(), equalTo(numDocs));
+
+        Set<String> ids = new HashSet<>();
+        for (BulkItemResponse bulkItemResponse : listener.bulkItems) {
+            assertThat(bulkItemResponse.getFailureMessage(), bulkItemResponse.isFailed(), equalTo(false));
+            assertThat(bulkItemResponse.getIndex(), equalTo("test"));
+            // with concurrent requests > 1 we can't rely on the order of the bulk requests
+            assertThat(Integer.valueOf(bulkItemResponse.getId()), both(greaterThan(0)).and(lessThanOrEqualTo(numDocs)));
+            // we do want to check that we don't get duplicate ids back
+            assertThat(ids.add(bulkItemResponse.getId()), equalTo(true));
+        }
+
+        assertMultiGetResponse(multiGetRequestBuilder.get(), numDocs);
+        assertThat(processor.getTotalBytesInFlight(), equalTo(0L));
+    }
+
+    public void testBulkProcessor2WaitOnClose() throws Exception {
+        BulkProcessor2TestListener listener = new BulkProcessor2TestListener();
+
+        int numDocs = randomIntBetween(10, 100);
+        BulkProcessor2 processor = BulkProcessor2.builder(client()::bulk, listener, client().threadPool())
+            // let's make sure that the bulk action limit trips, one single execution will index all the documents
+            .setBulkActions(numDocs)
+            .setFlushInterval(TimeValue.timeValueHours(24))
+            .setBulkSize(new ByteSizeValue(randomIntBetween(1, 10), RandomPicks.randomFrom(random(), ByteSizeUnit.values())))
+            .build();
+
+        MultiGetRequestBuilder multiGetRequestBuilder = indexDocs(client(), processor, numDocs);
+        processor.awaitClose(1, TimeUnit.MINUTES);
+        assertThat(listener.beforeCounts.get(), greaterThanOrEqualTo(1));
+        assertThat(listener.afterCounts.get(), greaterThanOrEqualTo(1));
+        assertThat(listener.bulkFailures.size(), equalTo(0));
+        assertResponseItems(listener.bulkItems, numDocs);
+        assertMultiGetResponse(multiGetRequestBuilder.get(), numDocs);
+    }
+
+    public void testBulkProcessor2ConcurrentRequestsReadOnlyIndex() throws Exception {
+        createIndex("test-ro");
+        assertAcked(
+            client().admin()
+                .indices()
+                .prepareUpdateSettings("test-ro")
+                .setSettings(Settings.builder().put(IndexMetadata.SETTING_BLOCKS_WRITE, true))
+        );
+        ensureGreen();
+
+        int bulkActions = randomIntBetween(10, 100);
+        int numDocs = randomIntBetween(bulkActions, bulkActions + 100);
+
+        int expectedBulkActions = numDocs / bulkActions;
+
+        final CountDownLatch latch = new CountDownLatch(expectedBulkActions);
+        int totalExpectedBulkActions = numDocs % bulkActions == 0 ? expectedBulkActions : expectedBulkActions + 1;
+        final CountDownLatch closeLatch = new CountDownLatch(totalExpectedBulkActions);
+
+        int testDocs = 0;
+        int testReadOnlyDocs = 0;
+        MultiGetRequestBuilder multiGetRequestBuilder = client().prepareMultiGet();
+        BulkProcessor2TestListener listener = new BulkProcessor2TestListener(latch, closeLatch);
+
+        BulkProcessor2 processor = BulkProcessor2.builder(client()::bulk, listener, client().threadPool())
+            .setBulkActions(bulkActions)
+            // set interval and size to high values
+            .setFlushInterval(TimeValue.timeValueHours(24))
+            .setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB))
+            .build();
+        try {
+
+            for (int i = 1; i <= numDocs; i++) {
+                if (randomBoolean()) {
+                    testDocs++;
+                    processor.add(
+                        new IndexRequest("test").id(Integer.toString(testDocs)).source(Requests.INDEX_CONTENT_TYPE, "field", "value")
+                    );
+                    multiGetRequestBuilder.add("test", Integer.toString(testDocs));
+                } else {
+                    testReadOnlyDocs++;
+                    processor.add(
+                        new IndexRequest("test-ro").id(Integer.toString(testReadOnlyDocs))
+                            .source(Requests.INDEX_CONTENT_TYPE, "field", "value")
+                    );
+                }
+            }
+        } finally {
+            processor.awaitClose(5, TimeUnit.SECONDS);
+        }
+
+        closeLatch.await();
+
+        assertThat(listener.beforeCounts.get(), equalTo(totalExpectedBulkActions));
+        assertThat(listener.afterCounts.get(), equalTo(totalExpectedBulkActions));
+        assertThat(listener.bulkFailures.size(), equalTo(0));
+        assertThat(listener.bulkItems.size(), equalTo(testDocs + testReadOnlyDocs));
+        assertThat(processor.getTotalBytesInFlight(), equalTo(0L));
+        Set<String> ids = new HashSet<>();
+        Set<String> readOnlyIds = new HashSet<>();
+        for (BulkItemResponse bulkItemResponse : listener.bulkItems) {
+            assertThat(bulkItemResponse.getIndex(), either(equalTo("test")).or(equalTo("test-ro")));
+            if (bulkItemResponse.getIndex().equals("test")) {
+                assertThat(bulkItemResponse.isFailed(), equalTo(false));
+                // with concurrent requests > 1 we can't rely on the order of the bulk requests
+                assertThat(Integer.valueOf(bulkItemResponse.getId()), both(greaterThan(0)).and(lessThanOrEqualTo(testDocs)));
+                // we do want to check that we don't get duplicate ids back
+                assertThat(ids.add(bulkItemResponse.getId()), equalTo(true));
+            } else {
+                assertThat(bulkItemResponse.isFailed(), equalTo(true));
+                // with concurrent requests > 1 we can't rely on the order of the bulk requests
+                assertThat(Integer.valueOf(bulkItemResponse.getId()), both(greaterThan(0)).and(lessThanOrEqualTo(testReadOnlyDocs)));
+                // we do want to check that we don't get duplicate ids back
+                assertThat(readOnlyIds.add(bulkItemResponse.getId()), equalTo(true));
+            }
+        }
+
+        assertMultiGetResponse(multiGetRequestBuilder.get(), testDocs);
+    }
+
+    private static MultiGetRequestBuilder indexDocs(Client client, BulkProcessor2 processor, int numDocs) throws Exception {
+        MultiGetRequestBuilder multiGetRequestBuilder = client.prepareMultiGet();
+        for (int i = 1; i <= numDocs; i++) {
+            processor.add(
+                new IndexRequest("test").id(Integer.toString(i))
+                    .source(Requests.INDEX_CONTENT_TYPE, "field", randomRealisticUnicodeOfLengthBetween(1, 30))
+            );
+            multiGetRequestBuilder.add("test", Integer.toString(i));
+        }
+        return multiGetRequestBuilder;
+    }
+
+    private static void assertResponseItems(List<BulkItemResponse> bulkItemResponses, int numDocs) {
+        assertThat(bulkItemResponses.size(), is(numDocs));
+        int i = 1;
+        List<BulkItemResponse> sortedResponses = bulkItemResponses.stream()
+            .sorted(Comparator.comparing(o -> Integer.valueOf(o.getId())))
+            .toList();
+        for (BulkItemResponse bulkItemResponse : sortedResponses) {
+            assertThat(bulkItemResponse.getIndex(), equalTo("test"));
+            assertThat(bulkItemResponse.getId(), equalTo(Integer.toString(i++)));
+            assertThat(
+                "item " + i + " failed with cause: " + bulkItemResponse.getFailureMessage(),
+                bulkItemResponse.isFailed(),
+                equalTo(false)
+            );
+        }
+    }
+
+    private static void assertMultiGetResponse(MultiGetResponse multiGetResponse, int numDocs) {
+        assertThat(multiGetResponse.getResponses().length, equalTo(numDocs));
+        int i = 1;
+        for (MultiGetItemResponse multiGetItemResponse : multiGetResponse) {
+            assertThat(multiGetItemResponse.getIndex(), equalTo("test"));
+            assertThat(multiGetItemResponse.getId(), equalTo(Integer.toString(i++)));
+        }
+    }
+
+    private static class BulkProcessor2TestListener implements BulkProcessor2.Listener {
+
+        private final CountDownLatch[] latches;
+        private final AtomicInteger beforeCounts = new AtomicInteger();
+        private final AtomicInteger afterCounts = new AtomicInteger();
+        private final List<BulkItemResponse> bulkItems = new CopyOnWriteArrayList<>();
+        private final List<Throwable> bulkFailures = new CopyOnWriteArrayList<>();
+
+        private BulkProcessor2TestListener(CountDownLatch... latches) {
+            this.latches = latches;
+        }
+
+        @Override
+        public void beforeBulk(long executionId, BulkRequest request) {
+            beforeCounts.incrementAndGet();
+        }
+
+        @Override
+        public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
+            bulkItems.addAll(Arrays.asList(response.getItems()));
+            afterCounts.incrementAndGet();
+            for (CountDownLatch latch : latches) {
+                latch.countDown();
+            }
+        }
+
+        @Override
+        public void afterBulk(long executionId, BulkRequest request, Exception failure) {
+            bulkFailures.add(failure);
+            afterCounts.incrementAndGet();
+            for (CountDownLatch latch : latches) {
+                latch.countDown();
+            }
+        }
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2RetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2RetryIT.java
new file mode 100644
index 000000000000..bdf5ca3c4f52
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessor2RetryIT.java
@@ -0,0 +1,169 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.action.bulk;
+
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 2)
+public class BulkProcessor2RetryIT extends ESIntegTestCase {
+    private static final String INDEX_NAME = "test";
+    Map<String, Integer> requestToExecutionCountMap = new ConcurrentHashMap<>();
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        // Have very low pool and queue sizes to overwhelm internal pools easily
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal, otherSettings))
+            // don't mess with this one! It's quite sensitive to a low queue size
+            // (see also ThreadedActionListener which is happily spawning threads even when we already got rejected)
+            // .put("thread_pool.listener.queue_size", 1)
+            .put("thread_pool.get.queue_size", 1)
+            // default is 200
+            .put("thread_pool.write.queue_size", 30)
+            .build();
+    }
+
+    public void testBulkRejectionLoadWithoutBackoff() throws Throwable {
+        boolean rejectedExecutionExpected = true;
+        executeBulkRejectionLoad(0, rejectedExecutionExpected);
+    }
+
+    // @TestLogging(
+    // value = "org.elasticsearch.action.bulk.Retry2:trace",
+    // reason = "Logging information about locks useful for tracking down deadlock"
+    // )
+    public void testBulkRejectionLoadWithBackoff() throws Throwable {
+        boolean rejectedExecutionExpected = false;
+        executeBulkRejectionLoad(8, rejectedExecutionExpected);
+    }
+
+    @SuppressWarnings("unchecked")
+    private void executeBulkRejectionLoad(int maxRetries, boolean rejectedExecutionExpected) throws Throwable {
+        int numberOfAsyncOps = randomIntBetween(600, 700);
+        final CountDownLatch latch = new CountDownLatch(numberOfAsyncOps);
+        final Set<BulkResponse> successfulResponses = Collections.newSetFromMap(new ConcurrentHashMap<>());
+        final Set<Tuple<BulkRequest, Throwable>> failedResponses = Collections.newSetFromMap(new ConcurrentHashMap<>());
+
+        assertAcked(prepareCreate(INDEX_NAME));
+        ensureGreen();
+
+        BulkProcessor2 bulkProcessor = BulkProcessor2.builder(this::countAndBulk, new BulkProcessor2.Listener() {
+            @Override
+            public void beforeBulk(long executionId, BulkRequest request) {
+                // no op
+            }
+
+            @Override
+            public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
+                successfulResponses.add(response);
+                latch.countDown();
+            }
+
+            @Override
+            public void afterBulk(long executionId, BulkRequest request, Exception failure) {
+                failedResponses.add(Tuple.tuple(request, failure));
+                latch.countDown();
+            }
+        }, client().threadPool()).setBulkActions(1).setMaxNumberOfRetries(maxRetries).build();
+        indexDocs(bulkProcessor, numberOfAsyncOps);
+        latch.await(10, TimeUnit.SECONDS);
+        bulkProcessor.awaitClose(1, TimeUnit.SECONDS);
+        assertThat(successfulResponses.size() + failedResponses.size(), equalTo(numberOfAsyncOps));
+        // validate all responses
+        boolean rejectedAfterAllRetries = false;
+        for (BulkResponse bulkResponse : successfulResponses) {
+            for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
+                if (bulkItemResponse.isFailed()) {
+                    BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
+                    if (failure.getStatus() == RestStatus.TOO_MANY_REQUESTS) {
+                        if (rejectedExecutionExpected == false) {
+                            int count = requestToExecutionCountMap.get(bulkItemResponse.getId());
+                            if (count < maxRetries + 1) {
+                                throw new AssertionError("Got rejected although backoff policy would allow more retries");
+                            }
+                            rejectedAfterAllRetries = true;
+                        }
+                    } else {
+                        throw new AssertionError("Unexpected failure status: " + failure.getStatus());
+                    }
+                }
+            }
+        }
+        for (Tuple<BulkRequest, Throwable> failureTuple : failedResponses) {
+            if (ExceptionsHelper.status(failureTuple.v2()) == RestStatus.TOO_MANY_REQUESTS) {
+                if (rejectedExecutionExpected == false) {
+                    for (DocWriteRequest<?> request : failureTuple.v1().requests) {
+                        int count = requestToExecutionCountMap.get(request.id());
+                        if (count < maxRetries + 1) {
+                            throw new AssertionError("Got rejected although backoff policy would allow more retries");
+                        }
+                    }
+                    rejectedAfterAllRetries = true;
+                }
+                // ignored, we exceeded the write queue size when dispatching the initial bulk request
+            } else {
+                Throwable t = failureTuple.v2();
+                // we're not expecting any other errors
+                throw new AssertionError("Unexpected failure", t);
+            }
+        }
+
+        client().admin().indices().refresh(new RefreshRequest()).get();
+
+        SearchResponse results = client().prepareSearch(INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).setSize(0).get();
+        assertThat(bulkProcessor.getTotalBytesInFlight(), equalTo(0L));
+        if (rejectedExecutionExpected) {
+            assertThat((int) results.getHits().getTotalHits().value, lessThanOrEqualTo(numberOfAsyncOps));
+        } else if (rejectedAfterAllRetries) {
+            assertThat((int) results.getHits().getTotalHits().value, lessThan(numberOfAsyncOps));
+        } else {
+            assertThat((int) results.getHits().getTotalHits().value, equalTo(numberOfAsyncOps));
+        }
+    }
+
+    private static void indexDocs(BulkProcessor2 processor, int numDocs) {
+        for (int i = 1; i <= numDocs; i++) {
+            processor.add(
+                client().prepareIndex()
+                    .setIndex(INDEX_NAME)
+                    .setId(Integer.toString(i))
+                    .setSource("field", randomRealisticUnicodeOfLengthBetween(1, 30))
+                    .request()
+            );
+        }
+    }
+
+    void countAndBulk(BulkRequest request, ActionListener<BulkResponse> listener) {
+        for (DocWriteRequest<?> docWriteRequest : request.requests) {
+            requestToExecutionCountMap.compute(docWriteRequest.id(), (key, value) -> value == null ? 1 : value + 1);
+        }
+        client().bulk(request, listener);
+    }
+
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/ingest/AsyncIngestProcessorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/ingest/AsyncIngestProcessorIT.java
deleted file mode 100644
index 9bf600483a71..000000000000
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/ingest/AsyncIngestProcessorIT.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.action.ingest;
-
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.get.GetRequest;
-import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.NodeEnvironment;
-import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.Processor;
-import org.elasticsearch.plugins.IngestPlugin;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.repositories.RepositoriesService;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.test.ESSingleNodeTestCase;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.tracing.Tracer;
-import org.elasticsearch.watcher.ResourceWatcherService;
-import org.elasticsearch.xcontent.NamedXContentRegistry;
-import org.elasticsearch.xcontent.XContentType;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.function.BiConsumer;
-import java.util.function.Supplier;
-
-import static org.hamcrest.Matchers.equalTo;
-
-/**
- * The purpose of this test is to verify that when a processor executes an operation asynchronously that
- * the expected result is the same as if the same operation happens synchronously.
- *
- * In this test two test processor are defined that basically do the same operation, but a single processor
- * executes asynchronously. The result of the operation should be the same and also the order in which the
- * bulk responses are returned should be the same as how the corresponding index requests were defined.
- */
-public class AsyncIngestProcessorIT extends ESSingleNodeTestCase {
-
-    @Override
-    protected Collection<Class<? extends Plugin>> getPlugins() {
-        return List.of(TestPlugin.class);
-    }
-
-    public void testAsyncProcessorImplementation() {
-        // A pipeline with 2 processors: the test async processor and sync test processor.
-        BytesReference pipelineBody = new BytesArray("{\"processors\": [{\"test-async\": {}, \"test\": {}}]}");
-        client().admin().cluster().putPipeline(new PutPipelineRequest("_id", pipelineBody, XContentType.JSON)).actionGet();
-
-        BulkRequest bulkRequest = new BulkRequest();
-        int numDocs = randomIntBetween(8, 256);
-        for (int i = 0; i < numDocs; i++) {
-            bulkRequest.add(new IndexRequest("foobar").id(Integer.toString(i)).source("{}", XContentType.JSON).setPipeline("_id"));
-        }
-        BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet();
-        assertThat(bulkResponse.getItems().length, equalTo(numDocs));
-        for (int i = 0; i < numDocs; i++) {
-            String id = Integer.toString(i);
-            assertThat(bulkResponse.getItems()[i].getId(), equalTo(id));
-            GetResponse getResponse = client().get(new GetRequest("foobar", id)).actionGet();
-            // The expected result of async test processor:
-            assertThat(getResponse.getSource().get("foo"), equalTo("bar-" + id));
-            // The expected result of sync test processor:
-            assertThat(getResponse.getSource().get("bar"), equalTo("baz-" + id));
-        }
-    }
-
-    public static class TestPlugin extends Plugin implements IngestPlugin {
-
-        private ThreadPool threadPool;
-
-        @Override
-        public Collection<Object> createComponents(
-            Client client,
-            ClusterService clusterService,
-            ThreadPool threadPool,
-            ResourceWatcherService resourceWatcherService,
-            ScriptService scriptService,
-            NamedXContentRegistry xContentRegistry,
-            Environment environment,
-            NodeEnvironment nodeEnvironment,
-            NamedWriteableRegistry namedWriteableRegistry,
-            IndexNameExpressionResolver expressionResolver,
-            Supplier<RepositoriesService> repositoriesServiceSupplier,
-            Tracer tracer,
-            AllocationDeciders allocationDeciders
-        ) {
-            this.threadPool = threadPool;
-            return List.of();
-        }
-
-        @Override
-        public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
-            return Map.of("test-async", (factories, tag, description, config) -> new AbstractProcessor(tag, description) {
-
-                @Override
-                public void execute(IngestDocument ingestDocument, BiConsumer<IngestDocument, Exception> handler) {
-                    threadPool.generic().execute(() -> {
-                        String id = (String) ingestDocument.getSourceAndMetadata().get("_id");
-                        if (usually()) {
-                            try {
-                                Thread.sleep(10);
-                            } catch (InterruptedException e) {
-                                // ignore
-                            }
-                        }
-                        ingestDocument.setFieldValue("foo", "bar-" + id);
-                        handler.accept(ingestDocument, null);
-                    });
-                }
-
-                @Override
-                public String getType() {
-                    return "test-async";
-                }
-
-                @Override
-                public boolean isAsync() {
-                    return true;
-                }
-
-            }, "test", (processorFactories, tag, description, config) -> new AbstractProcessor(tag, description) {
-                @Override
-                public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
-                    String id = (String) ingestDocument.getSourceAndMetadata().get("_id");
-                    ingestDocument.setFieldValue("bar", "baz-" + id);
-                    return ingestDocument;
-                }
-
-                @Override
-                public String getType() {
-                    return "test";
-                }
-            });
-        }
-    }
-}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java
index 7dab09997618..3cd40347c8de 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
@@ -184,7 +185,7 @@ public SearchTask createTask(long id, String type, String action, TaskId parentT
     private static List<SearchShard> createRandomIndices(Client client) {
         int numIndices = randomIntBetween(3, 20);
         for (int i = 0; i < numIndices; i++) {
-            String indexName = formatted("index-%03d", i);
+            String indexName = Strings.format("index-%03d", i);
             assertAcked(client.admin().indices().prepareCreate(indexName).get());
             client.prepareIndex(indexName).setSource("number", i, "foo", "bar").get();
         }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java
index 2d772e25131c..72fa2c033ace 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.ScoreMode;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
@@ -654,8 +654,8 @@ public long bytesToPreallocate() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.V_EMPTY;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.ZERO;
         }
     }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java
index 0cfc783f1895..38cf9d57dba3 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.xcontent.XContentType;
@@ -126,7 +127,7 @@ public void testReplicationWaitsForActiveShardCount() throws Exception {
     }
 
     private String source(String id, String nameValue) {
-        return formatted("""
+        return Strings.format("""
             { "type1" : { "id" : "%s", "name" : "%s" } }
             """, id, nameValue);
     }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
index a85713b8fe75..b4c4a778620a 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
-import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -275,26 +274,16 @@ private IndexRoutingTable randomIndexRoutingTable(String index, String[] nodeIds
             int replicaCount = randomIntBetween(1, 10);
             Set<String> availableNodeIds = Sets.newHashSet(nodeIds);
             for (int j = 0; j < replicaCount; j++) {
-                UnassignedInfo unassignedInfo = null;
-                if (randomInt(5) == 1) {
-                    unassignedInfo = randomUnassignedInfo(randomAlphaOfLength(10));
-                }
+                var newState = j == 0 ? ShardRoutingState.STARTED : randomFrom(ShardRoutingState.INITIALIZING, ShardRoutingState.STARTED);
+                var unassignedInfo = randomInt(5) == 1 && newState == ShardRoutingState.INITIALIZING
+                    ? randomUnassignedInfo(randomAlphaOfLength(10))
+                    : null;
                 if (availableNodeIds.isEmpty()) {
                     break;
                 }
                 String nodeId = randomFrom(availableNodeIds);
                 availableNodeIds.remove(nodeId);
-                indexShard.addShard(
-                    TestShardRouting.newShardRouting(
-                        index,
-                        i,
-                        nodeId,
-                        null,
-                        j == 0,
-                        ShardRoutingState.fromValue((byte) randomIntBetween(2, 3)),
-                        unassignedInfo
-                    )
-                );
+                indexShard.addShard(TestShardRouting.newShardRouting(index, i, nodeId, null, j == 0, newState, unassignedInfo));
             }
             builder.addIndexShard(indexShard);
         }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateNodeRemovalIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateNodeRemovalIT.java
index 5f68717217a0..19f8a3c8bc87 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateNodeRemovalIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateNodeRemovalIT.java
@@ -13,17 +13,40 @@
 import org.elasticsearch.action.admin.cluster.node.shutdown.PrevalidateNodeRemovalAction;
 import org.elasticsearch.action.admin.cluster.node.shutdown.PrevalidateNodeRemovalRequest;
 import org.elasticsearch.action.admin.cluster.node.shutdown.PrevalidateNodeRemovalResponse;
+import org.elasticsearch.action.admin.cluster.node.shutdown.TransportPrevalidateShardPathAction;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.shard.ShardPath;
+import org.elasticsearch.indices.store.IndicesStore;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.oneOf;
+import static org.hamcrest.Matchers.startsWith;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class PrevalidateNodeRemovalIT extends ESIntegTestCase {
 
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(MockTransportService.TestPlugin.class);
+    }
+
     public void testNodeRemovalFromNonRedCluster() throws Exception {
         internalCluster().startMasterOnlyNode();
         String node1 = internalCluster().startDataOnlyNode();
@@ -45,10 +68,13 @@ public void testNodeRemovalFromNonRedCluster() throws Exception {
         }
         PrevalidateNodeRemovalResponse resp = client().execute(PrevalidateNodeRemovalAction.INSTANCE, req.build()).get();
         assertTrue(resp.getPrevalidation().isSafe());
+        assertThat(resp.getPrevalidation().message(), equalTo("cluster status is not RED"));
         assertThat(resp.getPrevalidation().nodes().size(), equalTo(1));
         NodesRemovalPrevalidation.NodeResult nodeResult = resp.getPrevalidation().nodes().get(0);
         assertNotNull(nodeResult);
         assertThat(nodeResult.name(), equalTo(nodeName));
+        assertThat(nodeResult.result().reason(), equalTo(NodesRemovalPrevalidation.Reason.NO_PROBLEMS));
+        assertThat(nodeResult.result().message(), equalTo(""));
         assertTrue(nodeResult.result().isSafe());
         // Enforce a replica to get unassigned
         updateIndexSettings(indexName, Settings.builder().put("index.routing.allocation.require._name", node1));
@@ -56,25 +82,145 @@ public void testNodeRemovalFromNonRedCluster() throws Exception {
         PrevalidateNodeRemovalRequest req2 = PrevalidateNodeRemovalRequest.builder().setNames(node2).build();
         PrevalidateNodeRemovalResponse resp2 = client().execute(PrevalidateNodeRemovalAction.INSTANCE, req2).get();
         assertTrue(resp2.getPrevalidation().isSafe());
+        assertThat(resp2.getPrevalidation().message(), equalTo("cluster status is not RED"));
         assertThat(resp2.getPrevalidation().nodes().size(), equalTo(1));
         NodesRemovalPrevalidation.NodeResult nodeResult2 = resp2.getPrevalidation().nodes().get(0);
         assertNotNull(nodeResult2);
         assertThat(nodeResult2.name(), equalTo(node2));
         assertTrue(nodeResult2.result().isSafe());
+        assertThat(nodeResult2.result().reason(), equalTo(NodesRemovalPrevalidation.Reason.NO_PROBLEMS));
+        assertThat(nodeResult2.result().message(), equalTo(""));
     }
 
-    public void testNodeRemovalFromRedCluster() throws Exception {
+    // Test that in case the nodes that are being prevalidated do not contain copies of any of the
+    // red shards, their removal is considered to be safe.
+    public void testNodeRemovalFromRedClusterWithNoLocalShardCopy() throws Exception {
         internalCluster().startMasterOnlyNode();
-        String node1 = internalCluster().startDataOnlyNode();
-        String node2 = internalCluster().startDataOnlyNode();
+        String nodeWithIndex = internalCluster().startDataOnlyNode();
+        List<String> otherNodes = internalCluster().startDataOnlyNodes(randomIntBetween(1, 3));
         // Create an index pinned to one node, and then stop that node so the index is RED.
         String indexName = "test-idx";
         createIndex(
             indexName,
-            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put("index.routing.allocation.require._name", node1).build()
+            Settings.builder()
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put("index.routing.allocation.require._name", nodeWithIndex)
+                .build()
         );
         ensureYellow(indexName);
+        internalCluster().stopNode(nodeWithIndex);
+        ensureRed(indexName);
+        String[] otherNodeNames = otherNodes.toArray(new String[otherNodes.size()]);
+        PrevalidateNodeRemovalRequest req = PrevalidateNodeRemovalRequest.builder().setNames(otherNodeNames).build();
+        PrevalidateNodeRemovalResponse resp = client().execute(PrevalidateNodeRemovalAction.INSTANCE, req).get();
+        assertTrue(resp.getPrevalidation().isSafe());
+        assertThat(resp.getPrevalidation().message(), equalTo(""));
+        assertThat(resp.getPrevalidation().nodes().size(), equalTo(otherNodes.size()));
+        for (NodesRemovalPrevalidation.NodeResult nodeResult : resp.getPrevalidation().nodes()) {
+            assertThat(nodeResult.name(), oneOf(otherNodeNames));
+            assertThat(nodeResult.result().reason(), equalTo(NodesRemovalPrevalidation.Reason.NO_RED_SHARDS_ON_NODE));
+            assertTrue(nodeResult.result().isSafe());
+        }
+    }
+
+    public void testNodeRemovalFromRedClusterWithLocalShardCopy() throws Exception {
+        internalCluster().startMasterOnlyNode();
+        String node1 = internalCluster().startDataOnlyNode();
+        String node2 = internalCluster().startDataOnlyNode();
+        String indexName = "test-idx";
+        createIndex(
+            indexName,
+            Settings.builder()
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put("index.routing.allocation.require._name", node1)
+                .build()
+        );
+        ensureGreen(indexName);
+        // Prevent node1 from removing its local index shard copies upon removal, by blocking
+        // its ACTION_SHARD_EXISTS requests since after a relocation, the source first waits
+        // until the shard exists somewhere else, then it removes it locally.
+        final CountDownLatch shardActiveRequestSent = new CountDownLatch(1);
+        MockTransportService node1transport = (MockTransportService) internalCluster().getInstance(TransportService.class, node1);
+        TransportService node2transport = internalCluster().getInstance(TransportService.class, node2);
+        node1transport.addSendBehavior(node2transport, (connection, requestId, action, request, options) -> {
+            if (action.equals(IndicesStore.ACTION_SHARD_EXISTS)) {
+                shardActiveRequestSent.countDown();
+                logger.info("prevent shard active request from being sent");
+                throw new ConnectTransportException(connection.getNode(), "DISCONNECT: simulated");
+            }
+            connection.sendRequest(requestId, action, request, options);
+        });
+        logger.info("--> move shard from {} to {}, and wait for relocation to finish", node1, node2);
+        updateIndexSettings(indexName, Settings.builder().put("index.routing.allocation.require._name", node2));
+        shardActiveRequestSent.await();
+        ensureGreen(indexName);
+        // To ensure that the index doesn't get relocated back to node1 after stopping node2, we
+        // index a doc to make the index copy on node1 (in case not deleted after the relocation) stale.
+        indexDoc(indexName, "some_id", "foo", "bar");
+        internalCluster().stopNode(node2);
+        ensureRed(indexName);
+        // Ensure that node1 still has data for the unassigned index
+        NodeEnvironment nodeEnv = internalCluster().getInstance(NodeEnvironment.class, node1);
+        Index index = internalCluster().clusterService().state().metadata().index(indexName).getIndex();
+        ShardPath shardPath = ShardPath.loadShardPath(logger, nodeEnv, new ShardId(index, 0), "");
+        assertNotNull("local index shards not found", shardPath);
+        // Prevalidate removal of node1
+        PrevalidateNodeRemovalRequest req = PrevalidateNodeRemovalRequest.builder().setNames(node1).build();
+        PrevalidateNodeRemovalResponse resp = client().execute(PrevalidateNodeRemovalAction.INSTANCE, req).get();
+        String node1Id = internalCluster().clusterService(node1).localNode().getId();
+        assertFalse(resp.getPrevalidation().isSafe());
+        assertThat(resp.getPrevalidation().message(), equalTo("removal of the following nodes might not be safe: [" + node1Id + "]"));
+        assertThat(resp.getPrevalidation().nodes().size(), equalTo(1));
+        NodesRemovalPrevalidation.NodeResult nodeResult = resp.getPrevalidation().nodes().get(0);
+        assertThat(nodeResult.name(), equalTo(node1));
+        assertFalse(nodeResult.result().isSafe());
+        assertThat(nodeResult.result().reason(), equalTo(NodesRemovalPrevalidation.Reason.RED_SHARDS_ON_NODE));
+        assertThat(nodeResult.result().message(), equalTo("node contains copies of the following red shards: [[" + indexName + "][0]]"));
+    }
+
+    public void testNodeRemovalFromRedClusterWithTimeout() throws Exception {
+        internalCluster().startMasterOnlyNode();
+        String node1 = internalCluster().startDataOnlyNode();
+        String node2 = internalCluster().startDataOnlyNode();
+        String indexName = "test-index";
+        createIndex(
+            indexName,
+            Settings.builder()
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put("index.routing.allocation.require._name", node1)
+                .build()
+        );
+        ensureGreen(indexName);
+        // make it red!
         internalCluster().stopNode(node1);
+        ensureRed(indexName);
+        MockTransportService node2TransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, node2);
+        node2TransportService.addRequestHandlingBehavior(
+            TransportPrevalidateShardPathAction.ACTION_NAME + "[n]",
+            (handler, request, channel, task) -> { logger.info("drop the check shards request"); }
+        );
+        PrevalidateNodeRemovalRequest req = PrevalidateNodeRemovalRequest.builder()
+            .setNames(node2)
+            .build()
+            .timeout(TimeValue.timeValueSeconds(1));
+        PrevalidateNodeRemovalResponse resp = client().execute(PrevalidateNodeRemovalAction.INSTANCE, req).get();
+        assertFalse("prevalidation result should return false", resp.getPrevalidation().isSafe());
+        String node2Id = internalCluster().clusterService(node2).localNode().getId();
+        assertThat(
+            resp.getPrevalidation().message(),
+            equalTo("cannot prevalidate removal of nodes with the following IDs: [" + node2Id + "]")
+        );
+        assertThat(resp.getPrevalidation().nodes().size(), equalTo(1));
+        NodesRemovalPrevalidation.NodeResult nodeResult = resp.getPrevalidation().nodes().get(0);
+        assertThat(nodeResult.name(), equalTo(node2));
+        assertFalse(nodeResult.result().isSafe());
+        assertThat(nodeResult.result().message(), startsWith("failed contacting the node"));
+        assertThat(nodeResult.result().reason(), equalTo(NodesRemovalPrevalidation.Reason.UNABLE_TO_VERIFY));
+    }
+
+    private void ensureRed(String indexName) throws Exception {
         assertBusy(() -> {
             ClusterHealthResponse healthResponse = client().admin()
                 .cluster()
@@ -85,15 +231,5 @@ public void testNodeRemovalFromRedCluster() throws Exception {
                 .actionGet();
             assertThat(healthResponse.getStatus(), equalTo(ClusterHealthStatus.RED));
         });
-        // With a RED non-searchable-snapshot index, node removal is potentially unsafe
-        // since that node might have the last copy of the unassigned index.
-        PrevalidateNodeRemovalRequest req = PrevalidateNodeRemovalRequest.builder().setNames(node2).build();
-        PrevalidateNodeRemovalResponse resp = client().execute(PrevalidateNodeRemovalAction.INSTANCE, req).get();
-        assertFalse(resp.getPrevalidation().isSafe());
-        assertThat(resp.getPrevalidation().message(), equalTo("cluster health is RED"));
-        assertThat(resp.getPrevalidation().nodes().size(), equalTo(1));
-        NodesRemovalPrevalidation.NodeResult nodeResult = resp.getPrevalidation().nodes().get(0);
-        assertThat(nodeResult.name(), equalTo(node2));
-        assertFalse(nodeResult.result().isSafe());
     }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateShardPathIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateShardPathIT.java
new file mode 100644
index 000000000000..66bcf3ab3242
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateShardPathIT.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.cluster;
+
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.elasticsearch.action.admin.cluster.node.shutdown.NodePrevalidateShardPathResponse;
+import org.elasticsearch.action.admin.cluster.node.shutdown.PrevalidateShardPathRequest;
+import org.elasticsearch.action.admin.cluster.node.shutdown.PrevalidateShardPathResponse;
+import org.elasticsearch.action.admin.cluster.node.shutdown.TransportPrevalidateShardPathAction;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.equalTo;
+
+/*
+ * We rely on the shard directory being deleted after the relocation. This removal sometimes fails
+ * with "java.io.IOException: access denied" when using WindowsFS which seems to be a known issue.
+ * See {@link FileSystemUtilsTests}.
+ */
+@LuceneTestCase.SuppressFileSystems(value = "WindowsFS")
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class PrevalidateShardPathIT extends ESIntegTestCase {
+
+    public void testCheckShards() throws Exception {
+        internalCluster().startMasterOnlyNode();
+        String node1 = internalCluster().startDataOnlyNode();
+        String node2 = internalCluster().startDataOnlyNode();
+        String indexName = "index1";
+        int index1shards = randomIntBetween(1, 5);
+        createIndex("index1", Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, index1shards).build());
+        ensureGreen(indexName);
+        var shardIds = clusterService().state()
+            .routingTable()
+            .allShards(indexName)
+            .stream()
+            .map(ShardRouting::shardId)
+            .collect(Collectors.toSet());
+        String node1Id = internalCluster().clusterService(node1).localNode().getId();
+        String node2Id = internalCluster().clusterService(node2).localNode().getId();
+        Set<ShardId> shardIdsToCheck = new HashSet<>(shardIds);
+        boolean includeUnknownShardId = randomBoolean();
+        if (includeUnknownShardId) {
+            shardIdsToCheck.add(new ShardId(randomAlphaOfLength(10), UUIDs.randomBase64UUID(), randomIntBetween(0, 10)));
+        }
+        PrevalidateShardPathRequest req = new PrevalidateShardPathRequest(shardIdsToCheck, node1Id, node2Id);
+        PrevalidateShardPathResponse resp = client().execute(TransportPrevalidateShardPathAction.TYPE, req).get();
+        var nodeResponses = resp.getNodes();
+        assertThat(nodeResponses.size(), equalTo(2));
+        assertThat(nodeResponses.stream().map(r -> r.getNode().getId()).collect(Collectors.toSet()), equalTo(Set.of(node1Id, node2Id)));
+        assertTrue(resp.failures().isEmpty());
+        for (NodePrevalidateShardPathResponse nodeResponse : nodeResponses) {
+            assertThat(nodeResponse.getShardIds(), equalTo(shardIds));
+        }
+        // Check that after relocation the source node doesn't have the shard path
+        String node3 = internalCluster().startDataOnlyNode();
+        updateIndexSettings(indexName, Settings.builder().put("index.routing.allocation.exclude._name", node2));
+        ensureGreen(indexName);
+        assertBusy(() -> {
+            try {
+                // The excluded node should eventually delete the shards
+                PrevalidateShardPathRequest req2 = new PrevalidateShardPathRequest(shardIdsToCheck, node2Id);
+                PrevalidateShardPathResponse resp2 = client().execute(TransportPrevalidateShardPathAction.TYPE, req2).get();
+                assertThat(resp2.getNodes().size(), equalTo(1));
+                assertThat(resp2.getNodes().get(0).getNode().getId(), equalTo(node2Id));
+                assertTrue("There should be no failures in the response", resp.failures().isEmpty());
+                assertTrue("The relocation source node should have removed the shard(s)", resp2.getNodes().get(0).getShardIds().isEmpty());
+            } catch (AssertionError e) {
+                // Removal of shards which are no longer allocated to the node is attempted on every cluster state change in IndicesStore.
+                // If for whatever reason the removal is not triggered (e.g. not enough nodes reported that the shards are active) or it
+                // temporarily failed to clean up the shard folder, we need to trigger another cluster state change for this removal to
+                // finally succeed.
+                updateIndexSettings(
+                    indexName,
+                    Settings.builder().put("index.routing.allocation.exclude.name", "non-existent" + randomAlphaOfLength(5))
+                );
+                throw e;
+            }
+        });
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
index 0be8f79a3e1c..8ed513a4b4b4 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.cluster;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -19,7 +19,7 @@
 import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.routing.RoutingTable;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
@@ -42,6 +42,7 @@
 import org.elasticsearch.tracing.Tracer;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.junit.Before;
@@ -50,6 +51,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -391,8 +393,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT;
         }
 
         @Override
@@ -401,8 +403,8 @@ public void writeTo(StreamOutput out) throws IOException {
         }
 
         @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            return builder;
+        public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+            return Collections.emptyIterator();
         }
 
         static NamedDiff<ClusterState.Custom> readDiffFrom(StreamInput in) throws IOException {
@@ -443,7 +445,7 @@ public Collection<Object> createComponents(
             final IndexNameExpressionResolver expressionResolver,
             final Supplier<RepositoriesService> repositoriesServiceSupplier,
             Tracer tracer,
-            AllocationDeciders allocationDeciders
+            AllocationService allocationService
         ) {
             clusterService.addListener(event -> {
                 final ClusterState state = event.state();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java
index 7f2c44faf912..a0afbf65300f 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java
@@ -35,7 +35,6 @@
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.io.FileSystemUtils;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.TimeValue;
@@ -402,7 +401,7 @@ public void testRerouteExplain() {
         assertThat(explanation.decisions().type(), equalTo(Decision.Type.YES));
     }
 
-    public void testMessageLogging() throws Exception {
+    public void testMessageLogging() {
         final Settings settings = Settings.builder()
             .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), Allocation.NONE.name())
             .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE.name())
@@ -431,10 +430,7 @@ public void testMessageLogging() throws Exception {
             .execute()
             .actionGet();
 
-        Logger actionLogger = LogManager.getLogger(TransportClusterRerouteAction.class);
-
         MockLogAppender dryRunMockLog = new MockLogAppender();
-        dryRunMockLog.start();
         dryRunMockLog.addExpectation(
             new MockLogAppender.UnseenEventExpectation(
                 "no completed message logged on dry run",
@@ -443,28 +439,26 @@ public void testMessageLogging() throws Exception {
                 "allocated an empty primary*"
             )
         );
-        Loggers.addAppender(actionLogger, dryRunMockLog);
-
-        AllocationCommand dryRunAllocation = new AllocateEmptyPrimaryAllocationCommand(indexName, 0, nodeName1, true);
-        ClusterRerouteResponse dryRunResponse = client().admin()
-            .cluster()
-            .prepareReroute()
-            .setExplain(randomBoolean())
-            .setDryRun(true)
-            .add(dryRunAllocation)
-            .execute()
-            .actionGet();
-
-        // during a dry run, messages exist but are not logged or exposed
-        assertThat(dryRunResponse.getExplanations().getYesDecisionMessages(), hasSize(1));
-        assertThat(dryRunResponse.getExplanations().getYesDecisionMessages().get(0), containsString("allocated an empty primary"));
 
-        dryRunMockLog.assertAllExpectationsMatched();
-        dryRunMockLog.stop();
-        Loggers.removeAppender(actionLogger, dryRunMockLog);
+        try (var ignored = dryRunMockLog.capturing(TransportClusterRerouteAction.class)) {
+            AllocationCommand dryRunAllocation = new AllocateEmptyPrimaryAllocationCommand(indexName, 0, nodeName1, true);
+            ClusterRerouteResponse dryRunResponse = client().admin()
+                .cluster()
+                .prepareReroute()
+                .setExplain(randomBoolean())
+                .setDryRun(true)
+                .add(dryRunAllocation)
+                .execute()
+                .actionGet();
+
+            // during a dry run, messages exist but are not logged or exposed
+            assertThat(dryRunResponse.getExplanations().getYesDecisionMessages(), hasSize(1));
+            assertThat(dryRunResponse.getExplanations().getYesDecisionMessages().get(0), containsString("allocated an empty primary"));
+
+            dryRunMockLog.assertAllExpectationsMatched();
+        }
 
         MockLogAppender allocateMockLog = new MockLogAppender();
-        allocateMockLog.start();
         allocateMockLog.addExpectation(
             new MockLogAppender.SeenEventExpectation(
                 "message for first allocate empty primary",
@@ -481,26 +475,25 @@ public void testMessageLogging() throws Exception {
                 "allocated an empty primary*" + nodeName2 + "*"
             )
         );
-        Loggers.addAppender(actionLogger, allocateMockLog);
-
-        AllocationCommand yesDecisionAllocation = new AllocateEmptyPrimaryAllocationCommand(indexName, 0, nodeName1, true);
-        AllocationCommand noDecisionAllocation = new AllocateEmptyPrimaryAllocationCommand("noexist", 1, nodeName2, true);
-        ClusterRerouteResponse response = client().admin()
-            .cluster()
-            .prepareReroute()
-            .setExplain(true) // so we get a NO decision back rather than an exception
-            .add(yesDecisionAllocation)
-            .add(noDecisionAllocation)
-            .execute()
-            .actionGet();
-
-        assertThat(response.getExplanations().getYesDecisionMessages(), hasSize(1));
-        assertThat(response.getExplanations().getYesDecisionMessages().get(0), containsString("allocated an empty primary"));
-        assertThat(response.getExplanations().getYesDecisionMessages().get(0), containsString(nodeName1));
-
-        allocateMockLog.assertAllExpectationsMatched();
-        allocateMockLog.stop();
-        Loggers.removeAppender(actionLogger, allocateMockLog);
+        try (var ignored = allocateMockLog.capturing(TransportClusterRerouteAction.class)) {
+
+            AllocationCommand yesDecisionAllocation = new AllocateEmptyPrimaryAllocationCommand(indexName, 0, nodeName1, true);
+            AllocationCommand noDecisionAllocation = new AllocateEmptyPrimaryAllocationCommand("noexist", 1, nodeName2, true);
+            ClusterRerouteResponse response = client().admin()
+                .cluster()
+                .prepareReroute()
+                .setExplain(true) // so we get a NO decision back rather than an exception
+                .add(yesDecisionAllocation)
+                .add(noDecisionAllocation)
+                .execute()
+                .actionGet();
+
+            assertThat(response.getExplanations().getYesDecisionMessages(), hasSize(1));
+            assertThat(response.getExplanations().getYesDecisionMessages().get(0), containsString("allocated an empty primary"));
+            assertThat(response.getExplanations().getYesDecisionMessages().get(0), containsString(nodeName1));
+
+            allocateMockLog.assertAllExpectationsMatched();
+        }
     }
 
     public void testClusterRerouteWithBlocks() {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/InitialClusterStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/InitialClusterStateIT.java
new file mode 100644
index 000000000000..6fddb755a6b2
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/InitialClusterStateIT.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.cluster.coordination;
+
+import org.elasticsearch.action.main.MainAction;
+import org.elasticsearch.action.main.MainRequest;
+import org.elasticsearch.action.main.MainResponse;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalTestCluster;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.node.Node.INITIAL_STATE_TIMEOUT_SETTING;
+
+@ESIntegTestCase.ClusterScope(numDataNodes = 0, autoManageMasterNodes = false)
+public class InitialClusterStateIT extends ESIntegTestCase {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal, otherSettings))
+            .put(INITIAL_STATE_TIMEOUT_SETTING.getKey(), TimeValue.ZERO)
+            .build();
+    }
+
+    private static void assertClusterUuid(boolean expectCommitted, String expectedValue) {
+        for (String nodeName : internalCluster().getNodeNames()) {
+            final Metadata metadata = client(nodeName).admin().cluster().prepareState().setLocal(true).get().getState().metadata();
+            assertEquals(expectCommitted, metadata.clusterUUIDCommitted());
+            assertEquals(expectedValue, metadata.clusterUUID());
+
+            final MainResponse mainResponse = PlainActionFuture.get(
+                fut -> client(nodeName).execute(MainAction.INSTANCE, new MainRequest(), fut),
+                10,
+                TimeUnit.SECONDS
+            );
+            assertEquals(expectedValue, mainResponse.getClusterUuid());
+        }
+    }
+
+    public void testClusterUuidInInitialClusterState() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+
+        try {
+            internalCluster().startDataOnlyNode();
+            assertClusterUuid(false, Metadata.UNKNOWN_CLUSTER_UUID);
+
+            internalCluster().startMasterOnlyNode();
+            internalCluster().validateClusterFormed();
+
+            final var clusterUUID = internalCluster().getCurrentMasterNodeInstance(ClusterService.class).state().metadata().clusterUUID();
+            assertNotEquals(Metadata.UNKNOWN_CLUSTER_UUID, clusterUUID);
+            assertClusterUuid(true, clusterUUID);
+
+            internalCluster().stopCurrentMasterNode();
+            assertClusterUuid(true, clusterUUID);
+
+            internalCluster().restartRandomDataNode(new InternalTestCluster.RestartCallback() {
+                @Override
+                public boolean validateClusterForming() {
+                    return false;
+                }
+            });
+            assertClusterUuid(true, clusterUUID);
+        } finally {
+            while (true) {
+                var node = internalCluster().getRandomNodeName();
+                if (node == null) {
+                    break;
+                }
+                assertTrue(internalCluster().stopNode(node));
+            }
+        }
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java
index 856bebba1f51..f1c816fe4a22 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.MappingMetadata;
@@ -102,7 +103,10 @@ public ClusterState execute(ClusterState currentState) {
                 builder.blocks(ClusterBlocks.builder().blocks(currentState.blocks()).removeIndexBlocks(index));
                 ClusterState updatedState = builder.build();
 
-                RoutingTable.Builder routingTable = RoutingTable.builder(updatedState.routingTable());
+                RoutingTable.Builder routingTable = RoutingTable.builder(
+                    TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
+                    updatedState.routingTable()
+                );
                 routingTable.addAsRecovery(updatedState.metadata().index(index));
                 updatedState = ClusterState.builder(updatedState).routingTable(routingTable.build()).build();
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java
index eae2c7d37e35..51348a4844ca 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java
@@ -11,7 +11,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Setting;
@@ -77,7 +77,7 @@ public Collection<Object> createComponents(
             IndexNameExpressionResolver expressionResolver,
             Supplier<RepositoriesService> repositoriesServiceSupplier,
             Tracer tracer,
-            AllocationDeciders allocationDeciders
+            AllocationService allocationService
         ) {
             clusterService.getClusterSettings()
                 .addSettingsUpdateConsumer(
@@ -97,7 +97,7 @@ public Collection<Object> createComponents(
                 expressionResolver,
                 repositoriesServiceSupplier,
                 tracer,
-                allocationDeciders
+                allocationService
             );
         }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java
index 7bf202d64fbe..8172316e8a7b 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESIntegTestCase;
 
-import java.util.Collections;
 import java.util.List;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -191,7 +190,6 @@ private void indexRandomData() throws Exception {
     private String findNodeWithShard() {
         ClusterState state = client().admin().cluster().prepareState().get().getState();
         List<ShardRouting> startedShards = RoutingNodesHelper.shardsWithState(state.getRoutingNodes(), ShardRoutingState.STARTED);
-        Collections.shuffle(startedShards, random());
-        return state.nodes().get(startedShards.get(0).currentNodeId()).getName();
+        return state.nodes().get(randomFrom(startedShards).currentNodeId()).getName();
     }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/ShardRoutingRoleIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/ShardRoutingRoleIT.java
new file mode 100644
index 000000000000..6eadcd9ee2cb
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/ShardRoutingRoleIT.java
@@ -0,0 +1,677 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.cluster.routing;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.admin.indices.refresh.TransportUnpromotableShardRefreshAction;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.command.AllocationCommand;
+import org.elasticsearch.cluster.routing.allocation.command.CancelAllocationCommand;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.engine.EngineFactory;
+import org.elasticsearch.index.engine.EngineTestCase;
+import org.elasticsearch.index.engine.InternalEngine;
+import org.elasticsearch.index.engine.NoOpEngine;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.plugins.ClusterPlugin;
+import org.elasticsearch.plugins.EnginePlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.snapshots.SnapshotState;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.XContentTestUtils;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.anEmptyMap;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.startsWith;
+
+@SuppressWarnings("resource")
+public class ShardRoutingRoleIT extends ESIntegTestCase {
+
+    private static final Logger logger = LogManager.getLogger(ShardRoutingRoleIT.class);
+
+    public static class TestPlugin extends Plugin implements ClusterPlugin, EnginePlugin {
+
+        volatile int numIndexingCopies = 1;
+        static final String NODE_ATTR_UNPROMOTABLE_ONLY = "unpromotableonly";
+
+        @Override
+        public ShardRoutingRoleStrategy getShardRoutingRoleStrategy() {
+            return new ShardRoutingRoleStrategy() {
+                @Override
+                public ShardRouting.Role newReplicaRole() {
+                    return ShardRouting.Role.SEARCH_ONLY;
+                }
+
+                @Override
+                public ShardRouting.Role newEmptyRole(int copyIndex) {
+                    assert 0 < numIndexingCopies;
+                    return copyIndex < numIndexingCopies ? ShardRouting.Role.INDEX_ONLY : ShardRouting.Role.SEARCH_ONLY;
+                }
+            };
+        }
+
+        @Override
+        public Collection<AllocationDecider> createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) {
+            return List.of(new AllocationDecider() {
+                @Override
+                public Decision canForceAllocatePrimary(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
+                    // once a primary is cancelled it _stays_ cancelled
+                    if (shardRouting.unassignedInfo().getReason() == UnassignedInfo.Reason.REROUTE_CANCELLED) {
+                        return Decision.NO;
+                    }
+                    return super.canForceAllocatePrimary(shardRouting, node, allocation);
+                }
+
+                @Override
+                public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
+                    var nodesWithUnpromotableOnly = allocation.getClusterState()
+                        .nodes()
+                        .stream()
+                        .filter(n -> Objects.equals("true", n.getAttributes().get(NODE_ATTR_UNPROMOTABLE_ONLY)))
+                        .map(DiscoveryNode::getName)
+                        .collect(Collectors.toUnmodifiableSet());
+                    if (nodesWithUnpromotableOnly.isEmpty() == false) {
+                        if (nodesWithUnpromotableOnly.contains(node.node().getName())) {
+                            if (shardRouting.isPromotableToPrimary()) {
+                                return allocation.decision(
+                                    Decision.NO,
+                                    "test",
+                                    "shard is promotable to primary so may not be assigned to [" + node.node().getName() + "]"
+                                );
+                            }
+                        } else {
+                            if (shardRouting.isPromotableToPrimary() == false) {
+                                return allocation.decision(
+                                    Decision.NO,
+                                    "test",
+                                    "shard is not promotable to primary so may not be assigned to [" + node.node().getName() + "]"
+                                );
+                            }
+                        }
+                    }
+                    return Decision.YES;
+                }
+            });
+        }
+
+        @Override
+        public Optional<EngineFactory> getEngineFactory(IndexSettings indexSettings) {
+            return Optional.of(config -> {
+                if (config.isPromotableToPrimary()) {
+                    return new InternalEngine(config);
+                } else {
+                    try {
+                        config.getStore().createEmpty();
+                    } catch (IOException e) {
+                        logger.error("Error creating empty store", e);
+                        throw new RuntimeException(e);
+                    }
+
+                    return new NoOpEngine(EngineTestCase.copy(config, () -> -1L));
+                }
+            });
+        }
+    }
+
+    @Override
+    protected boolean addMockInternalEngine() {
+        return false;
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return CollectionUtils.concatLists(List.of(MockTransportService.TestPlugin.class, TestPlugin.class), super.nodePlugins());
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal, otherSettings))
+            .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)
+            .build();
+    }
+
+    private static TestPlugin getMasterNodePlugin() {
+        return internalCluster().getCurrentMasterNodeInstance(PluginsService.class)
+            .filterPlugins(TestPlugin.class)
+            .stream()
+            .findFirst()
+            .orElseThrow(() -> new AssertionError("no plugin"));
+    }
+
+    private static final String INDEX_NAME = "test";
+
+    private static class RoutingTableWatcher implements ClusterStateListener {
+
+        final int numShards = between(1, 3);
+        int numIndexingCopies = between(1, 2);
+        int numReplicas = between(numIndexingCopies - 1, 3);
+
+        @Override
+        public void clusterChanged(ClusterChangedEvent event) {
+            assertRoles(event.state().getRoutingTable().index(INDEX_NAME));
+        }
+
+        private void assertRoles(IndexRoutingTable indexRoutingTable) {
+            if (indexRoutingTable == null) {
+                return;
+            }
+            var message = indexRoutingTable.prettyPrint();
+            assertEquals("number_of_shards: " + message, numShards, indexRoutingTable.size());
+            for (int shardId = 0; shardId < numShards; shardId++) {
+                final var indexShardRoutingTable = indexRoutingTable.shard(shardId);
+                assertEquals("number_of_replicas: " + message, numReplicas + 1, indexShardRoutingTable.size());
+                var indexingShards = 0;
+                for (int shardCopy = 0; shardCopy < numReplicas + 1; shardCopy++) {
+                    final var shardRouting = indexShardRoutingTable.shard(shardCopy);
+                    switch (shardRouting.role()) {
+                        case INDEX_ONLY -> indexingShards += 1;
+                        case SEARCH_ONLY -> assertFalse(shardRouting.primary());
+                        case DEFAULT -> fail("should not have any DEFAULT shards");
+                    }
+                    if (shardRouting.relocating()) {
+                        assertEquals("role on relocation: " + message, shardRouting.role(), shardRouting.getTargetRelocatingShard().role());
+                    }
+                }
+                assertEquals("number_of_indexing_shards: " + message, Math.min(numIndexingCopies, numReplicas + 1), indexingShards);
+            }
+        }
+
+        Settings getIndexSettings() {
+            logger.info("--> numShards={}, numReplicas={}", numShards, numReplicas);
+            return Settings.builder()
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas)
+                .build();
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void assertRolesInRoutingTableXContent(ClusterState state) {
+        try {
+            final var routingTable = (Map<String, Object>) XContentTestUtils.convertToMap(state).get("routing_table");
+            final var routingTableIndices = (Map<String, Object>) routingTable.get("indices");
+            final var routingTableIndex = (Map<String, Object>) routingTableIndices.get("test");
+            final var routingTableShards = (Map<String, Object>) routingTableIndex.get("shards");
+            for (final var routingTableShardValue : routingTableShards.values()) {
+                for (Object routingTableShardCopy : (List<Object>) routingTableShardValue) {
+                    final var routingTableShard = (Map<String, String>) routingTableShardCopy;
+                    assertNotNull(ShardRouting.Role.valueOf(routingTableShard.get("role")));
+                }
+            }
+        } catch (IOException e) {
+            throw new AssertionError("unexpected", e);
+        }
+    }
+
+    private static void installMockTransportVerifications(RoutingTableWatcher routingTableWatcher) {
+        for (var transportService : internalCluster().getInstances(TransportService.class)) {
+            MockTransportService mockTransportService = (MockTransportService) transportService;
+            mockTransportService.addSendBehavior((connection, requestId, action, request, options) -> {
+                if (routingTableWatcher.numIndexingCopies == 1) {
+                    assertThat("no recovery action should be exchanged", action, not(startsWith("internal:index/shard/recovery/")));
+                    assertThat("no replicated action should be exchanged", action, not(containsString("[r]")));
+                }
+                connection.sendRequest(requestId, action, request, options);
+            });
+            mockTransportService.addRequestHandlingBehavior(
+                TransportUnpromotableShardRefreshAction.NAME,
+                (handler, request, channel, task) -> {
+                    // Skip handling the request and send an immediate empty response
+                    channel.sendResponse(ActionResponse.Empty.INSTANCE);
+                }
+            );
+        }
+    }
+
+    public void testShardCreation() throws Exception {
+        var routingTableWatcher = new RoutingTableWatcher();
+
+        var numDataNodes = routingTableWatcher.numReplicas + 2;
+        internalCluster().ensureAtLeastNumDataNodes(numDataNodes);
+        installMockTransportVerifications(routingTableWatcher);
+        getMasterNodePlugin().numIndexingCopies = routingTableWatcher.numIndexingCopies;
+
+        final var masterClusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class);
+        try {
+            // verify the correct number of shard copies of each role as the routing table evolves
+            masterClusterService.addListener(routingTableWatcher);
+
+            createIndex(INDEX_NAME, routingTableWatcher.getIndexSettings());
+
+            final var clusterState = client().admin().cluster().prepareState().clear().setRoutingTable(true).get().getState();
+
+            // verify non-DEFAULT roles reported in cluster state XContent
+            assertRolesInRoutingTableXContent(clusterState);
+
+            // verify non-DEFAULT roles reported in cluster state string representation
+            var stateAsString = clusterState.toString();
+            assertThat(stateAsString, containsString("[" + ShardRouting.Role.INDEX_ONLY + "]"));
+            assertThat(stateAsString, not(containsString("[" + ShardRouting.Role.DEFAULT + "]")));
+            if (routingTableWatcher.numReplicas + 1 > routingTableWatcher.numIndexingCopies) {
+                assertThat(stateAsString, containsString("[" + ShardRouting.Role.SEARCH_ONLY + "]"));
+            }
+
+            ensureGreen(INDEX_NAME);
+            assertEngineTypes();
+
+            // new replicas get the SEARCH_ONLY role
+            routingTableWatcher.numReplicas += 1;
+            assertAcked(
+                client().admin()
+                    .indices()
+                    .prepareUpdateSettings("test")
+                    .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, routingTableWatcher.numReplicas))
+            );
+
+            ensureGreen(INDEX_NAME);
+            assertEngineTypes();
+            indexRandom(randomBoolean(), INDEX_NAME, randomIntBetween(50, 100));
+
+            // removing replicas drops SEARCH_ONLY copies first
+            while (routingTableWatcher.numReplicas > 0) {
+                routingTableWatcher.numReplicas -= 1;
+                assertAcked(
+                    client().admin()
+                        .indices()
+                        .prepareUpdateSettings("test")
+                        .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, routingTableWatcher.numReplicas))
+                );
+            }
+
+            // restoring the index from a snapshot may change the number of indexing replicas because the routing table is created afresh
+            var repoPath = randomRepoPath();
+            assertAcked(
+                client().admin()
+                    .cluster()
+                    .preparePutRepository("repo")
+                    .setType("fs")
+                    .setSettings(Settings.builder().put("location", repoPath))
+            );
+
+            assertEquals(
+                SnapshotState.SUCCESS,
+                client().admin().cluster().prepareCreateSnapshot("repo", "snap").setWaitForCompletion(true).get().getSnapshotInfo().state()
+            );
+
+            if (randomBoolean()) {
+                assertAcked(client().admin().indices().prepareDelete(INDEX_NAME));
+            } else {
+                assertAcked(client().admin().indices().prepareClose(INDEX_NAME));
+                ensureGreen(INDEX_NAME);
+            }
+
+            routingTableWatcher.numReplicas = between(0, numDataNodes - 1);
+            routingTableWatcher.numIndexingCopies = between(1, 2);
+            getMasterNodePlugin().numIndexingCopies = routingTableWatcher.numIndexingCopies;
+
+            assertEquals(
+                0,
+                client().admin()
+                    .cluster()
+                    .prepareRestoreSnapshot("repo", "snap")
+                    .setIndices(INDEX_NAME)
+                    .setIndexSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, routingTableWatcher.numReplicas))
+                    .setWaitForCompletion(true)
+                    .get()
+                    .getRestoreInfo()
+                    .failedShards()
+            );
+            ensureGreen(INDEX_NAME);
+            assertEngineTypes();
+        } finally {
+            masterClusterService.removeListener(routingTableWatcher);
+        }
+    }
+
+    private void assertEngineTypes() {
+        for (IndicesService indicesService : internalCluster().getInstances(IndicesService.class)) {
+            for (IndexService indexService : indicesService) {
+                for (IndexShard indexShard : indexService) {
+                    final var engine = indexShard.getEngineOrNull();
+                    assertNotNull(engine);
+                    if (indexShard.routingEntry().isPromotableToPrimary()
+                        && indexShard.indexSettings().getIndexMetadata().getState() == IndexMetadata.State.OPEN) {
+                        assertThat(engine, instanceOf(InternalEngine.class));
+                    } else {
+                        assertThat(engine, instanceOf(NoOpEngine.class));
+                    }
+                }
+            }
+        }
+    }
+
+    public void testRelocation() {
+        var routingTableWatcher = new RoutingTableWatcher();
+
+        var numDataNodes = routingTableWatcher.numReplicas + 2;
+        internalCluster().ensureAtLeastNumDataNodes(numDataNodes);
+        getMasterNodePlugin().numIndexingCopies = routingTableWatcher.numIndexingCopies;
+
+        final var masterClusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class);
+        try {
+            // verify the correct number of shard copies of each role as the routing table evolves
+            masterClusterService.addListener(routingTableWatcher);
+
+            createIndex(INDEX_NAME, routingTableWatcher.getIndexSettings());
+
+            for (String nodeName : internalCluster().getNodeNames()) {
+                assertAcked(
+                    client().admin()
+                        .indices()
+                        .prepareUpdateSettings("test")
+                        .setSettings(Settings.builder().put(IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + "._name", nodeName))
+                );
+                ensureGreen(INDEX_NAME);
+            }
+        } finally {
+            masterClusterService.removeListener(routingTableWatcher);
+        }
+    }
+
+    public void testPromotion() {
+        var routingTableWatcher = new RoutingTableWatcher();
+
+        var numDataNodes = routingTableWatcher.numReplicas + 2;
+        internalCluster().ensureAtLeastNumDataNodes(numDataNodes);
+        installMockTransportVerifications(routingTableWatcher);
+        getMasterNodePlugin().numIndexingCopies = routingTableWatcher.numIndexingCopies;
+
+        final var masterClusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class);
+        try {
+            // verify the correct number of shard copies of each role as the routing table evolves
+            masterClusterService.addListener(routingTableWatcher);
+
+            createIndex(INDEX_NAME, routingTableWatcher.getIndexSettings());
+            ensureGreen(INDEX_NAME);
+            assertEngineTypes();
+
+            assertAcked(
+                client().admin()
+                    .indices()
+                    .prepareUpdateSettings("test")
+                    .setSettings(Settings.builder().put(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._name", "not-a-node"))
+            );
+
+            AllocationCommand cancelPrimaryCommand;
+            while ((cancelPrimaryCommand = getCancelPrimaryCommand()) != null) {
+                client().admin().cluster().prepareReroute().add(cancelPrimaryCommand).get();
+            }
+        } finally {
+            masterClusterService.removeListener(routingTableWatcher);
+        }
+    }
+
+    @Nullable
+    public AllocationCommand getCancelPrimaryCommand() {
+        final var indexRoutingTable = client().admin()
+            .cluster()
+            .prepareState()
+            .clear()
+            .setRoutingTable(true)
+            .get()
+            .getState()
+            .routingTable()
+            .index(INDEX_NAME);
+        for (int shardId = 0; shardId < indexRoutingTable.size(); shardId++) {
+            final var indexShardRoutingTable = indexRoutingTable.shard(shardId);
+            if (indexShardRoutingTable.primaryShard().assignedToNode()) {
+                return new CancelAllocationCommand(INDEX_NAME, shardId, indexShardRoutingTable.primaryShard().currentNodeId(), true);
+            } else {
+                assertThat(indexShardRoutingTable.assignedShards(), empty());
+                for (int copy = 0; copy < indexShardRoutingTable.size(); copy++) {
+                    final var shardRouting = indexShardRoutingTable.shard(copy);
+                    assertEquals(
+                        shardRouting.role().isPromotableToPrimary()
+                            ? UnassignedInfo.Reason.REROUTE_CANCELLED
+                            : UnassignedInfo.Reason.UNPROMOTABLE_REPLICA,
+                        shardRouting.unassignedInfo().getReason()
+                    );
+                }
+            }
+        }
+        return null;
+    }
+
+    public void testSearchRouting() throws Exception {
+
+        var routingTableWatcher = new RoutingTableWatcher();
+        routingTableWatcher.numReplicas = Math.max(1, routingTableWatcher.numReplicas);
+        routingTableWatcher.numIndexingCopies = Math.min(routingTableWatcher.numIndexingCopies, routingTableWatcher.numReplicas);
+        getMasterNodePlugin().numIndexingCopies = routingTableWatcher.numIndexingCopies;
+
+        internalCluster().ensureAtLeastNumDataNodes(routingTableWatcher.numReplicas + 1);
+        installMockTransportVerifications(routingTableWatcher);
+
+        final var masterClusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class);
+        try {
+            // verify the correct number of shard copies of each role as the routing table evolves
+            masterClusterService.addListener(routingTableWatcher);
+
+            createIndex(INDEX_NAME, routingTableWatcher.getIndexSettings());
+            indexRandom(randomBoolean(), INDEX_NAME, randomIntBetween(50, 100));
+            ensureGreen(INDEX_NAME);
+            assertEngineTypes();
+
+            final var searchShardProfileKeys = new HashSet<String>();
+            final var indexRoutingTable = client().admin()
+                .cluster()
+                .prepareState()
+                .clear()
+                .setRoutingTable(true)
+                .get()
+                .getState()
+                .routingTable()
+                .index(INDEX_NAME);
+
+            for (int shardId = 0; shardId < indexRoutingTable.size(); shardId++) {
+                final var indexShardRoutingTable = indexRoutingTable.shard(shardId);
+                for (int shardCopy = 0; shardCopy < indexShardRoutingTable.size(); shardCopy++) {
+                    final var shardRouting = indexShardRoutingTable.shard(shardCopy);
+                    if (shardRouting.role() == ShardRouting.Role.SEARCH_ONLY) {
+                        searchShardProfileKeys.add("[" + shardRouting.currentNodeId() + "][" + INDEX_NAME + "][" + shardId + "]");
+                    }
+                }
+            }
+
+            for (int i = 0; i < 10; i++) {
+                final var search = client().prepareSearch(INDEX_NAME).setProfile(true);
+                switch (randomIntBetween(0, 2)) {
+                    case 0 -> search.setRouting(randomAlphaOfLength(10));
+                    case 1 -> search.setPreference(randomSearchPreference(routingTableWatcher.numShards, internalCluster().getNodeNames()));
+                    default -> {
+                        // do nothing
+                    }
+                }
+                final var profileResults = search.get().getProfileResults();
+                assertThat(profileResults, not(anEmptyMap()));
+                for (final var searchShardProfileKey : profileResults.keySet()) {
+                    assertThat(searchShardProfileKeys, hasItem(searchShardProfileKey));
+                }
+            }
+
+            // TODO also verify PIT routing
+            // TODO also verify the search-shards API
+        } finally {
+            masterClusterService.removeListener(routingTableWatcher);
+        }
+    }
+
+    private String randomSearchPreference(int numShards, String... nodeIds) {
+        final var preference = randomFrom(Preference.SHARDS, Preference.PREFER_NODES, Preference.LOCAL);
+        // ONLY_LOCAL and ONLY_NODES omitted here because they may yield no shard copies which causes the search to fail
+        // TODO add support for ONLY_LOCAL and ONLY_NODES too
+        return switch (preference) {
+            case LOCAL, ONLY_LOCAL -> preference.type();
+            case PREFER_NODES, ONLY_NODES -> preference.type() + ":" + String.join(",", randomNonEmptySubsetOf(Arrays.asList(nodeIds)));
+            case SHARDS -> preference.type()
+                + ":"
+                + String.join(
+                    ",",
+                    randomSubsetOf(between(1, numShards), IntStream.range(0, numShards).mapToObj(Integer::toString).toList())
+                );
+        };
+    }
+
+    public void testClosedIndex() {
+        var routingTableWatcher = new RoutingTableWatcher();
+
+        var numDataNodes = routingTableWatcher.numReplicas + 2;
+        internalCluster().ensureAtLeastNumDataNodes(numDataNodes);
+        installMockTransportVerifications(routingTableWatcher);
+        getMasterNodePlugin().numIndexingCopies = routingTableWatcher.numIndexingCopies;
+
+        final var masterClusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class);
+        try {
+            // verify the correct number of shard copies of each role as the routing table evolves
+            masterClusterService.addListener(routingTableWatcher);
+
+            createIndex(INDEX_NAME, routingTableWatcher.getIndexSettings());
+            ensureGreen(INDEX_NAME);
+            assertEngineTypes();
+
+            assertAcked(client().admin().indices().prepareClose(INDEX_NAME));
+            ensureGreen(INDEX_NAME);
+            assertEngineTypes();
+        } finally {
+            masterClusterService.removeListener(routingTableWatcher);
+        }
+    }
+
+    public void testRefreshOfUnpromotableShards() throws Exception {
+        var routingTableWatcher = new RoutingTableWatcher();
+
+        var numDataNodes = routingTableWatcher.numReplicas + 2;
+        internalCluster().ensureAtLeastNumDataNodes(numDataNodes);
+        installMockTransportVerifications(routingTableWatcher);
+        getMasterNodePlugin().numIndexingCopies = routingTableWatcher.numIndexingCopies;
+        final AtomicInteger unpromotableRefreshActions = new AtomicInteger(0);
+
+        for (var transportService : internalCluster().getInstances(TransportService.class)) {
+            MockTransportService mockTransportService = (MockTransportService) transportService;
+            mockTransportService.addSendBehavior((connection, requestId, action, request, options) -> {
+                if (action.startsWith(TransportUnpromotableShardRefreshAction.NAME)) {
+                    unpromotableRefreshActions.incrementAndGet();
+                }
+                connection.sendRequest(requestId, action, request, options);
+            });
+        }
+
+        final var masterClusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class);
+        try {
+            // verify the correct number of shard copies of each role as the routing table evolves
+            masterClusterService.addListener(routingTableWatcher);
+
+            createIndex(
+                INDEX_NAME,
+                Settings.builder()
+                    .put(routingTableWatcher.getIndexSettings())
+                    .put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), false)
+                    .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1)
+                    .build()
+            );
+            ensureGreen(INDEX_NAME);
+            assertEngineTypes();
+
+            indexRandom(true, INDEX_NAME, randomIntBetween(1, 10));
+
+            int singleRefreshExpectedUnpromotableActions = (routingTableWatcher.numReplicas - (routingTableWatcher.numIndexingCopies - 1))
+                * routingTableWatcher.numShards;
+            if (singleRefreshExpectedUnpromotableActions > 0) {
+                assertThat(
+                    "at least one refresh is expected where each primary sends an unpromotable refresh to each unpromotable replica shard.",
+                    unpromotableRefreshActions.get(),
+                    greaterThanOrEqualTo(singleRefreshExpectedUnpromotableActions)
+                );
+                assertThat(
+                    "the number of unpromotable refreshes seen is expected to be a multiple of the occurred refreshes",
+                    unpromotableRefreshActions.get() % singleRefreshExpectedUnpromotableActions,
+                    is(equalTo(0))
+                );
+            }
+        } finally {
+            masterClusterService.removeListener(routingTableWatcher);
+        }
+    }
+
+    public void testNodesWithUnpromotableShardsNeverGetReplicationActions() throws Exception {
+        var routingTableWatcher = new RoutingTableWatcher();
+        var additionalNumberOfNodesWithUnpromotableShards = randomIntBetween(1, 3);
+        routingTableWatcher.numReplicas = routingTableWatcher.numIndexingCopies + additionalNumberOfNodesWithUnpromotableShards - 1;
+        internalCluster().ensureAtLeastNumDataNodes(routingTableWatcher.numIndexingCopies + 1);
+        final List<String> nodesWithUnpromotableOnly = internalCluster().startDataOnlyNodes(
+            additionalNumberOfNodesWithUnpromotableShards,
+            Settings.builder().put("node.attr." + TestPlugin.NODE_ATTR_UNPROMOTABLE_ONLY, "true").build()
+        );
+        installMockTransportVerifications(routingTableWatcher);
+        getMasterNodePlugin().numIndexingCopies = routingTableWatcher.numIndexingCopies;
+
+        for (var transportService : internalCluster().getInstances(TransportService.class)) {
+            MockTransportService mockTransportService = (MockTransportService) transportService;
+            mockTransportService.addSendBehavior((connection, requestId, action, request, options) -> {
+                if (nodesWithUnpromotableOnly.contains(connection.getNode().getName())) {
+                    assertThat(action, not(containsString("[r]")));
+                }
+                connection.sendRequest(requestId, action, request, options);
+            });
+        }
+
+        final var masterClusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class);
+        try {
+            // verify the correct number of shard copies of each role as the routing table evolves
+            masterClusterService.addListener(routingTableWatcher);
+            createIndex(INDEX_NAME, routingTableWatcher.getIndexSettings());
+            ensureGreen(INDEX_NAME);
+            indexRandom(randomBoolean(), INDEX_NAME, randomIntBetween(50, 100));
+        } finally {
+            masterClusterService.removeListener(routingTableWatcher);
+        }
+    }
+
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/service/ClusterServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/service/ClusterServiceIT.java
index 064949bf4bae..24950fe160e8 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/service/ClusterServiceIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/service/ClusterServiceIT.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Arrays;
 import java.util.HashSet;
@@ -25,6 +26,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.StreamSupport;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -366,7 +368,7 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState)
         assertThat(response.pendingTasks().size(), greaterThanOrEqualTo(10));
         assertThat(response.pendingTasks().get(0).getSource().string(), equalTo("1"));
         assertThat(response.pendingTasks().get(0).isExecuting(), equalTo(true));
-        for (PendingClusterTask task : response) {
+        for (PendingClusterTask task : response.pendingTasks()) {
             controlSources.remove(task.getSource().string());
         }
         assertTrue(controlSources.isEmpty());
@@ -414,11 +416,7 @@ public void onFailure(Exception e) {
                 });
             }
 
-            final var startNanoTime = System.nanoTime();
-            while (TimeUnit.MILLISECONDS.convert(System.nanoTime() - startNanoTime, TimeUnit.NANOSECONDS) <= 0) {
-                // noinspection BusyWait
-                Thread.sleep(100);
-            }
+            waitForTimeToElapse();
 
             pendingClusterTasks = clusterService.getMasterService().pendingTasks();
             assertThat(pendingClusterTasks.size(), greaterThanOrEqualTo(5));
@@ -431,7 +429,7 @@ public void onFailure(Exception e) {
             response = internalCluster().coordOnlyNodeClient().admin().cluster().preparePendingClusterTasks().get();
             assertThat(response.pendingTasks().size(), greaterThanOrEqualTo(5));
             controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5"));
-            for (PendingClusterTask task : response) {
+            for (PendingClusterTask task : response.pendingTasks()) {
                 if (controlSources.remove(task.getSource().string())) {
                     assertThat(task.getTimeInQueueInMillis(), greaterThan(0L));
                 }
@@ -441,4 +439,28 @@ public void onFailure(Exception e) {
             block2.countDown();
         }
     }
+
+    private static void waitForTimeToElapse() throws InterruptedException {
+        final ThreadPool[] threadPools = StreamSupport.stream(internalCluster().getInstances(ClusterService.class).spliterator(), false)
+            .map(ClusterService::threadPool)
+            .toArray(ThreadPool[]::new);
+        final long[] startTimes = Arrays.stream(threadPools).mapToLong(ThreadPool::relativeTimeInMillis).toArray();
+
+        final var startNanoTime = System.nanoTime();
+        while (TimeUnit.MILLISECONDS.convert(System.nanoTime() - startNanoTime, TimeUnit.NANOSECONDS) <= 100) {
+            // noinspection BusyWait
+            Thread.sleep(100);
+        }
+
+        outer: do {
+            for (int i = 0; i < threadPools.length; i++) {
+                if (threadPools[i].relativeTimeInMillis() <= startTimes[i]) {
+                    // noinspection BusyWait
+                    Thread.sleep(100);
+                    continue outer;
+                }
+            }
+            return;
+        } while (true);
+    }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionIT.java
index 6f8f2eb11a80..4a823ed997ea 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/ClusterDisruptionIT.java
@@ -339,7 +339,7 @@ public void testSendingShardFailure() throws Exception {
 
         // fail a random shard
         ShardRouting failedShard = randomFrom(
-            clusterService().state().getRoutingNodes().node(nonMasterNodeId).shardsWithState(ShardRoutingState.STARTED)
+            clusterService().state().getRoutingNodes().node(nonMasterNodeId).shardsWithState(ShardRoutingState.STARTED).toList()
         );
         ShardStateAction service = internalCluster().getInstance(ShardStateAction.class, nonMasterNode);
         CountDownLatch latch = new CountDownLatch(1);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/MasterDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/MasterDisruptionIT.java
index e2866d669e06..d4a9467818fd 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/MasterDisruptionIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/MasterDisruptionIT.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.coordination.NoMasterBlockService;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.disruption.BlockMasterServiceOnMaster;
@@ -142,7 +143,7 @@ public void testIsolateMasterAndVerifyClusterStateConsensus() throws Exception {
                     assertEquals("different meta data version", state.metadata().version(), nodeState.metadata().version());
                     assertEquals("different routing", state.routingTable().toString(), nodeState.routingTable().toString());
                 } catch (AssertionError t) {
-                    fail(formatted("""
+                    fail(Strings.format("""
                         failed comparing cluster state: %s
                         --- cluster state of node [%s]: ---
                         %s
@@ -202,7 +203,7 @@ public void testVerifyApiBlocksDuringPartition() throws Exception {
                 success = false;
             }
             if (success == false) {
-                fail(formatted("""
+                fail(Strings.format("""
                     node [%s] has no master or has blocks, despite of being on the right side of the partition. State dump:
                     %s""", node, nodeState));
             }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/StableMasterDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/StableMasterDisruptionIT.java
index cf8c44119b22..941400f7ce48 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/StableMasterDisruptionIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/StableMasterDisruptionIT.java
@@ -135,7 +135,8 @@ private void assertGreenMasterStability(Client client) throws Exception {
 
     private void assertMasterStability(Client client, HealthStatus expectedStatus, Matcher<String> expectedMatcher) throws Exception {
         assertBusy(() -> {
-            GetHealthAction.Response healthResponse = client.execute(GetHealthAction.INSTANCE, new GetHealthAction.Request(true)).get();
+            GetHealthAction.Response healthResponse = client.execute(GetHealthAction.INSTANCE, new GetHealthAction.Request(true, 1000))
+                .get();
             String debugInformation = xContentToString(healthResponse);
             assertThat(debugInformation, healthResponse.findIndicator("master_is_stable").status(), equalTo(expectedStatus));
             assertThat(debugInformation, healthResponse.findIndicator("master_is_stable").symptom(), expectedMatcher);
@@ -144,7 +145,7 @@ private void assertMasterStability(Client client, HealthStatus expectedStatus, M
 
     private String xContentToString(ChunkedToXContent xContent) throws IOException {
         XContentBuilder builder = JsonXContent.contentBuilder();
-        xContent.toXContentChunked().forEachRemaining(xcontent -> {
+        xContent.toXContentChunked(ToXContent.EMPTY_PARAMS).forEachRemaining(xcontent -> {
             try {
                 xcontent.toXContent(builder, ToXContent.EMPTY_PARAMS);
             } catch (IOException e) {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java
index 155a40d6917b..ba086cb4e978 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java
@@ -9,13 +9,10 @@
 package org.elasticsearch.discovery.single;
 
 import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.core.LogEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.coordination.JoinHelper;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -105,7 +102,6 @@ public Path nodeConfigPath(int nodeOrdinal) {
 
     public void testCannotJoinNodeWithSingleNodeDiscovery() throws Exception {
         MockLogAppender mockAppender = new MockLogAppender();
-        mockAppender.start();
         mockAppender.addExpectation(
             new MockLogAppender.SeenEventExpectation("test", JoinHelper.class.getCanonicalName(), Level.INFO, "failed to join") {
 
@@ -158,20 +154,13 @@ public Path nodeConfigPath(int nodeOrdinal) {
                 "other",
                 Arrays.asList(getTestTransportPlugin(), MockHttpTransport.TestPlugin.class),
                 Function.identity()
-            )
+            );
+            var ignored = mockAppender.capturing(JoinHelper.class)
         ) {
-
-            Logger clusterLogger = LogManager.getLogger(JoinHelper.class);
-            Loggers.addAppender(clusterLogger, mockAppender);
-            try {
-                other.beforeTest(random());
-                final ClusterState first = internalCluster().getInstance(ClusterService.class).state();
-                assertThat(first.nodes().getSize(), equalTo(1));
-                assertBusy(() -> mockAppender.assertAllExpectationsMatched());
-            } finally {
-                Loggers.removeAppender(clusterLogger, mockAppender);
-                mockAppender.stop();
-            }
+            other.beforeTest(random());
+            final ClusterState first = internalCluster().getInstance(ClusterService.class).state();
+            assertThat(first.nodes().getSize(), equalTo(1));
+            assertBusy(mockAppender::assertAllExpectationsMatched);
         }
     }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java
index b04425f75df0..459b5cfeb491 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorIT.java
@@ -244,7 +244,6 @@ public void testRecentPrimaryInformation() throws Exception {
         transportServiceOnPrimary.clearAllRules();
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/91613")
     public void testFullClusterRestartPerformNoopRecovery() throws Exception {
         int numOfReplicas = randomIntBetween(1, 2);
         internalCluster().ensureAtLeastNumDataNodes(numOfReplicas + 2);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java
index cdb09dc31054..1008eabd6cc9 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java
@@ -225,7 +225,6 @@ public void testPreferCopyCanPerformNoopRecovery() throws Exception {
         transportServiceOnPrimary.clearAllRules();
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/91451")
     public void testFullClusterRestartPerformNoopRecovery() throws Exception {
         int numOfReplicas = randomIntBetween(1, 2);
         internalCluster().ensureAtLeastNumDataNodes(numOfReplicas + 2);
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java
index a49d38f5c2ea..c486cd5cfe7f 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/get/GetActionIT.java
@@ -789,7 +789,7 @@ public void testGeneratedStringFieldsStored() throws IOException {
     void indexSingleDocumentWithStringFieldsGeneratedFromText(boolean stored, boolean sourceEnabled) {
 
         String storedString = stored ? "true" : "false";
-        String createIndexSource = formatted("""
+        String createIndexSource = Strings.format("""
             {
               "settings": {
                 "index.translog.flush_threshold_size": "1pb",
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/health/GetHealthActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/health/GetHealthActionIT.java
index 858c783b0cb1..8d4789da7d1d 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/health/GetHealthActionIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/health/GetHealthActionIT.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.metrics.Counters;
@@ -35,11 +35,9 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
-import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.function.Supplier;
 import java.util.stream.Stream;
@@ -109,7 +107,7 @@ public Collection<Object> createComponents(
             IndexNameExpressionResolver indexNameExpressionResolver,
             Supplier<RepositoriesService> repositoriesServiceSupplier,
             Tracer tracer,
-            AllocationDeciders allocationDeciders
+            AllocationService allocationService
         ) {
             healthIndicatorServices.add(new IlmHealthIndicatorService(clusterService));
             healthIndicatorServices.add(new SlmHealthIndicatorService(clusterService));
@@ -144,7 +142,7 @@ public String name() {
         }
 
         @Override
-        public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
+        public HealthIndicatorResult calculate(boolean verbose, int maxAffectedResourcesCount, HealthInfo healthInfo) {
             var status = clusterService.getClusterSettings().get(statusSetting);
             return createIndicator(
                 status,
@@ -203,8 +201,10 @@ public void testGetHealth() throws Exception {
             {
                 ExecutionException exception = expectThrows(
                     ExecutionException.class,
-                    () -> client.execute(GetHealthAction.INSTANCE, new GetHealthAction.Request(NONEXISTENT_INDICATOR_NAME, randomBoolean()))
-                        .get()
+                    () -> client.execute(
+                        GetHealthAction.INSTANCE,
+                        new GetHealthAction.Request(NONEXISTENT_INDICATOR_NAME, randomBoolean(), 1000)
+                    ).get()
                 );
                 assertThat(exception.getCause(), instanceOf(ResourceNotFoundException.class));
             }
@@ -232,13 +232,6 @@ public void testGetHealth() throws Exception {
                 } else {
                     expectThrows(IllegalArgumentException.class, () -> stats.get(label));
                 }
-                Set<HealthStatus> expectedStatuses = new HashSet<>();
-                expectedStatuses.add(ilmIndicatorStatus);
-                expectedStatuses.add(mostSevereHealthStatus);
-                assertThat(response.getStatuses(), equalTo(expectedStatuses));
-                if (mostSevereHealthStatus != HealthStatus.GREEN || ilmIndicatorStatus != HealthStatus.GREEN) {
-                    assertThat(response.getIndicators().isEmpty(), equalTo(mostSevereHealthStatus == HealthStatus.GREEN));
-                }
             }
 
         } finally {
@@ -258,7 +251,7 @@ private void testRootLevel(
         HealthStatus clusterCoordinationIndicatorStatus,
         boolean verbose
     ) throws Exception {
-        var response = client.execute(GetHealthAction.INSTANCE, new GetHealthAction.Request(verbose)).get();
+        var response = client.execute(GetHealthAction.INSTANCE, new GetHealthAction.Request(verbose, 1000)).get();
 
         assertThat(
             response.getStatus(),
@@ -294,7 +287,7 @@ private void testRootLevel(
     }
 
     private void testIndicator(Client client, HealthStatus ilmIndicatorStatus, boolean verbose) throws Exception {
-        var response = client.execute(GetHealthAction.INSTANCE, new GetHealthAction.Request(ILM_INDICATOR_NAME, verbose)).get();
+        var response = client.execute(GetHealthAction.INSTANCE, new GetHealthAction.Request(ILM_INDICATOR_NAME, verbose, 1000)).get();
         assertNull(response.getStatus());
         assertThat(response.getClusterName(), equalTo(new ClusterName(cluster().getClusterName())));
         assertThat(
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/health/HealthServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/health/HealthServiceIT.java
index bc59dd27dc70..e987a6dafe6b 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/health/HealthServiceIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/health/HealthServiceIT.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
@@ -92,7 +92,7 @@ public void onFailure(Exception e) {
                         throw new RuntimeException(e);
                     }
                 };
-                healthService.getHealth(internalCluster.client(node), TestHealthIndicatorService.NAME, true, listener);
+                healthService.getHealth(internalCluster.client(node), TestHealthIndicatorService.NAME, true, 1000, listener);
                 assertBusy(() -> assertThat(onResponseCalled.get(), equalTo(true)));
             }
         }
@@ -136,7 +136,7 @@ public Collection<Object> createComponents(
             IndexNameExpressionResolver indexNameExpressionResolver,
             Supplier<RepositoriesService> repositoriesServiceSupplier,
             Tracer tracer,
-            AllocationDeciders allocationDeciders
+            AllocationService allocationService
         ) {
             healthIndicatorServices.add(new TestHealthIndicatorService());
             return new ArrayList<>(healthIndicatorServices);
@@ -158,7 +158,7 @@ public String name() {
         }
 
         @Override
-        public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
+        public HealthIndicatorResult calculate(boolean verbose, int maxAffectedResourcesCount, HealthInfo healthInfo) {
             assertThat(healthInfo.diskInfoByNode().size(), equalTo(internalCluster().getNodeNames().length));
             for (DiskHealthInfo diskHealthInfo : healthInfo.diskInfoByNode().values()) {
                 assertThat(diskHealthInfo.healthStatus(), equalTo(HealthStatus.GREEN));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/health/node/DiskHealthIndicatorServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/health/node/DiskHealthIndicatorServiceIT.java
index 9741f5c791ab..1cab207fda30 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/health/node/DiskHealthIndicatorServiceIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/health/node/DiskHealthIndicatorServiceIT.java
@@ -81,7 +81,7 @@ public void onFailure(Exception e) {
                 throw new RuntimeException(e);
             }
         };
-        healthService.getHealth(internalCluster().client(node), DiskHealthIndicatorService.NAME, true, listener);
+        healthService.getHealth(internalCluster().client(node), DiskHealthIndicatorService.NAME, true, 1000, listener);
         assertBusy(() -> assertNotNull(resultListReference.get()));
         return resultListReference.get();
     }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java
index d3333ce11640..98ce3933f9d3 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java
@@ -20,7 +20,7 @@
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -53,6 +53,7 @@
 import java.util.function.Supplier;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasToString;
@@ -89,7 +90,12 @@ public void testFinalPipelineCantChangeDestination() {
             IllegalStateException.class,
             () -> client().prepareIndex("index").setId("1").setSource(Map.of("field", "value")).get()
         );
-        assertThat(e, hasToString(containsString("final pipeline [final_pipeline] can't change the target index")));
+        assertThat(
+            e,
+            hasToString(
+                endsWith("final pipeline [final_pipeline] can't change the target index (from [index] to [target]) for document [1]")
+            )
+        );
     }
 
     public void testFinalPipelineOfOldDestinationIsNotInvoked() {
@@ -350,7 +356,7 @@ public Collection<Object> createComponents(
             final IndexNameExpressionResolver expressionResolver,
             final Supplier<RepositoriesService> repositoriesServiceSupplier,
             Tracer tracer,
-            AllocationDeciders allocationDeciders
+            AllocationService allocationService
         ) {
             return List.of();
         }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/SettingsListenerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/SettingsListenerIT.java
index dae4cdf4d9aa..a142a03a180f 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/index/SettingsListenerIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/SettingsListenerIT.java
@@ -9,7 +9,7 @@
 
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -75,7 +75,7 @@ public Collection<Object> createComponents(
             IndexNameExpressionResolver expressionResolver,
             Supplier<RepositoriesService> repositoriesServiceSupplier,
             Tracer tracer,
-            AllocationDeciders allocationDeciders
+            AllocationService allocationService
         ) {
             return Collections.singletonList(service);
         }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/engine/MaxDocsLimitIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/engine/MaxDocsLimitIT.java
index 9e392a8c5fa2..3c001a575250 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/index/engine/MaxDocsLimitIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/engine/MaxDocsLimitIT.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.gateway.PersistedClusterStateService;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.translog.Translog;
@@ -61,6 +62,18 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return CollectionUtils.appendToCopy(super.nodePlugins(), TestEnginePlugin.class);
     }
 
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        // Document page size should not be too small, else we can fail to write the cluster state for small max doc values
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal, otherSettings))
+            .put(
+                PersistedClusterStateService.DOCUMENT_PAGE_SIZE.getKey(),
+                PersistedClusterStateService.DOCUMENT_PAGE_SIZE.get(Settings.EMPTY)
+            )
+            .build();
+    }
+
     @Before
     public void setMaxDocs() {
         maxDocs.set(randomIntBetween(10, 100)); // Do not set this too low as we can fail to write the cluster state
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java
index 839ef9fe52a0..2ec1fff5a420 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java
@@ -51,6 +51,7 @@
 import org.elasticsearch.index.engine.NoOpEngine;
 import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.mapper.SourceToParse;
+import org.elasticsearch.index.seqno.ReplicationTracker;
 import org.elasticsearch.index.seqno.RetentionLeaseSyncer;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.translog.TestTranslog;
@@ -660,7 +661,9 @@ public static final IndexShard newIndexShard(
             RetentionLeaseSyncer.EMPTY,
             cbs,
             IndexModule.DEFAULT_SNAPSHOT_COMMIT_SUPPLIER,
-            System::nanoTime
+            System::nanoTime,
+            null,
+            ReplicationTracker.DEFAULT_FACTORY
         );
     }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndexingMemoryControllerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndexingMemoryControllerIT.java
index e7b24d49008f..feb2356b31a4 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndexingMemoryControllerIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndexingMemoryControllerIT.java
@@ -78,7 +78,9 @@ EngineConfig engineConfigWithLargerIndexingMemory(EngineConfig config) {
                 config.getPrimaryTermSupplier(),
                 config.getSnapshotCommitSupplier(),
                 config.getLeafSorter(),
-                config.getRelativeTimeInNanosSupplier()
+                config.getRelativeTimeInNanosSupplier(),
+                config.getIndexCommitListener(),
+                config.isPromotableToPrimary()
             );
         }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesLifecycleListenerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesLifecycleListenerIT.java
index 678efc090323..5b488b6873de 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesLifecycleListenerIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesLifecycleListenerIT.java
@@ -35,7 +35,6 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -272,7 +271,7 @@ private static void assertShardStatesMatch(
         try {
             assertBusy(waitPredicate, 1, TimeUnit.MINUTES);
         } catch (AssertionError ae) {
-            fail(String.format(Locale.ROOT, """
+            fail(Strings.format("""
                 failed to observe expect shard states
                 expected: [%d] shards with states: %s
                 observed:
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java
index e438f538151c..8649946308e8 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java
@@ -10,6 +10,7 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -28,7 +29,7 @@ public class ConcurrentDynamicTemplateIT extends ESIntegTestCase {
     // see #3544
     public void testConcurrentDynamicMapping() throws Exception {
         final String fieldName = "field";
-        final String mapping = formatted("""
+        final String mapping = Strings.format("""
             {
               "dynamic_templates": [
                 {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
index c86e9db4650c..31544cc63725 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
@@ -798,7 +798,6 @@ private void validateIndexRecoveryState(RecoveryState.Index indexState) {
         assertThat(indexState.recoveredBytesPercent(), lessThanOrEqualTo(100.0f));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/91087")
     public void testTransientErrorsDuringRecoveryAreRetried() throws Exception {
         final String recoveryActionToBlock = randomFrom(
             PeerRecoveryTargetService.Actions.PREPARE_TRANSLOG,
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/ReopenWhileClosingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/ReopenWhileClosingIT.java
index c907324a691c..c45e5343df45 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/state/ReopenWhileClosingIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/state/ReopenWhileClosingIT.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.indices.state;
 
 import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.close.CloseIndexResponse;
 import org.elasticsearch.action.admin.indices.close.TransportVerifyShardBeforeCloseAction;
 import org.elasticsearch.cluster.ClusterState;
@@ -16,6 +17,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ListenableFuture;
 import org.elasticsearch.core.Glob;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
@@ -24,6 +26,7 @@
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
@@ -128,8 +131,7 @@ private Releasable interceptVerifyShardBeforeCloseActions(final String indexPatt
             TransportService.class,
             internalCluster().getMasterName()
         );
-
-        final CountDownLatch release = new CountDownLatch(1);
+        final ListenableFuture<Void> release = new ListenableFuture<>();
         for (DiscoveryNode node : internalCluster().clusterService().state().getNodes()) {
             mockTransportService.addSendBehavior(
                 internalCluster().getInstance(TransportService.class, node.getName()),
@@ -140,21 +142,23 @@ private Releasable interceptVerifyShardBeforeCloseActions(final String indexPatt
                             if (Glob.globMatch(indexPattern, index)) {
                                 logger.info("request {} intercepted for index {}", requestId, index);
                                 onIntercept.run();
-                                try {
-                                    release.await();
+                                release.addListener(ActionListener.wrap(() -> {
                                     logger.info("request {} released for index {}", requestId, index);
-                                } catch (final InterruptedException e) {
-                                    throw new AssertionError(e);
-                                }
+                                    try {
+                                        connection.sendRequest(requestId, action, request, options);
+                                    } catch (IOException e) {
+                                        throw new AssertionError(e);
+                                    }
+                                }));
+                                return;
                             }
                         }
-
                     }
                     connection.sendRequest(requestId, action, request, options);
                 }
             );
         }
-        return Releasables.releaseOnce(release::countDown);
+        return Releasables.releaseOnce(() -> release.onResponse(null));
     }
 
     private static void assertIndexIsBlocked(final String... indices) {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestAsyncProcessorIT.java b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestAsyncProcessorIT.java
new file mode 100644
index 000000000000..eac5c092b7ef
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestAsyncProcessorIT.java
@@ -0,0 +1,151 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.ingest;
+
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.ingest.PutPipelineRequest;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.plugins.IngestPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.tracing.Tracer;
+import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.function.BiConsumer;
+import java.util.function.Supplier;
+
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * The purpose of this test is to verify that when a processor executes an operation asynchronously that
+ * the expected result is the same as if the same operation happens synchronously.
+ *
+ * In this test two test processor are defined that basically do the same operation, but a single processor
+ * executes asynchronously. The result of the operation should be the same and also the order in which the
+ * bulk responses are returned should be the same as how the corresponding index requests were defined.
+ */
+public class IngestAsyncProcessorIT extends ESSingleNodeTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return List.of(TestPlugin.class);
+    }
+
+    public void testAsyncProcessorImplementation() {
+        // A pipeline with 2 processors: the test async processor and sync test processor.
+        BytesReference pipelineBody = new BytesArray("{\"processors\": [{\"test-async\": {}, \"test\": {}}]}");
+        client().admin().cluster().putPipeline(new PutPipelineRequest("_id", pipelineBody, XContentType.JSON)).actionGet();
+
+        BulkRequest bulkRequest = new BulkRequest();
+        int numDocs = randomIntBetween(8, 256);
+        for (int i = 0; i < numDocs; i++) {
+            bulkRequest.add(new IndexRequest("foobar").id(Integer.toString(i)).source("{}", XContentType.JSON).setPipeline("_id"));
+        }
+        BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet();
+        assertThat(bulkResponse.getItems().length, equalTo(numDocs));
+        for (int i = 0; i < numDocs; i++) {
+            String id = Integer.toString(i);
+            assertThat(bulkResponse.getItems()[i].getId(), equalTo(id));
+            GetResponse getResponse = client().get(new GetRequest("foobar", id)).actionGet();
+            // The expected result of async test processor:
+            assertThat(getResponse.getSource().get("foo"), equalTo("bar-" + id));
+            // The expected result of sync test processor:
+            assertThat(getResponse.getSource().get("bar"), equalTo("baz-" + id));
+        }
+    }
+
+    public static class TestPlugin extends Plugin implements IngestPlugin {
+
+        private ThreadPool threadPool;
+
+        @Override
+        public Collection<Object> createComponents(
+            Client client,
+            ClusterService clusterService,
+            ThreadPool threadPool,
+            ResourceWatcherService resourceWatcherService,
+            ScriptService scriptService,
+            NamedXContentRegistry xContentRegistry,
+            Environment environment,
+            NodeEnvironment nodeEnvironment,
+            NamedWriteableRegistry namedWriteableRegistry,
+            IndexNameExpressionResolver expressionResolver,
+            Supplier<RepositoriesService> repositoriesServiceSupplier,
+            Tracer tracer,
+            AllocationService allocationService
+        ) {
+            this.threadPool = threadPool;
+            return List.of();
+        }
+
+        @Override
+        public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
+            return Map.of("test-async", (factories, tag, description, config) -> new AbstractProcessor(tag, description) {
+
+                @Override
+                public void execute(IngestDocument ingestDocument, BiConsumer<IngestDocument, Exception> handler) {
+                    threadPool.generic().execute(() -> {
+                        String id = (String) ingestDocument.getSourceAndMetadata().get("_id");
+                        if (usually()) {
+                            try {
+                                Thread.sleep(10);
+                            } catch (InterruptedException e) {
+                                // ignore
+                            }
+                        }
+                        ingestDocument.setFieldValue("foo", "bar-" + id);
+                        handler.accept(ingestDocument, null);
+                    });
+                }
+
+                @Override
+                public String getType() {
+                    return "test-async";
+                }
+
+                @Override
+                public boolean isAsync() {
+                    return true;
+                }
+
+            }, "test", (processorFactories, tag, description, config) -> new AbstractProcessor(tag, description) {
+                @Override
+                public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
+                    String id = (String) ingestDocument.getSourceAndMetadata().get("_id");
+                    ingestDocument.setFieldValue("bar", "baz-" + id);
+                    return ingestDocument;
+                }
+
+                @Override
+                public String getType() {
+                    return "test";
+                }
+            });
+        }
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestFileSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestFileSettingsIT.java
index 1fc92aa4ae68..aac7cd2ef22a 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestFileSettingsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestFileSettingsIT.java
@@ -101,7 +101,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
                  "ingest_pipelines": {
                    "my_ingest_pipeline": {
                        "description": "_description",
-                       "processors": [
+                       "processors":
                           {
                             "foo" : {
                               "field": "pipeline",
@@ -204,11 +204,11 @@ public void clusterChanged(ClusterChangedEvent event) {
                     clusterService.removeListener(this);
                     metadataVersion.set(event.state().metadata().version());
                     savedClusterState.countDown();
-                    assertEquals(ReservedStateErrorMetadata.ErrorKind.VALIDATION, reservedState.errorMetadata().errorKind());
+                    assertEquals(ReservedStateErrorMetadata.ErrorKind.PARSING, reservedState.errorMetadata().errorKind());
                     assertThat(reservedState.errorMetadata().errors(), allOf(notNullValue(), hasSize(1)));
                     assertThat(
                         reservedState.errorMetadata().errors().get(0),
-                        containsString("org.elasticsearch.ElasticsearchParseException: No processor type exists with name [foo]")
+                        containsString("org.elasticsearch.xcontent.XContentParseException: [17:16] [reserved_state_chunk] failed")
                     );
                 }
             }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java
new file mode 100644
index 000000000000..fea613e671b9
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java
@@ -0,0 +1,194 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.ingest;
+
+import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
+import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
+import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
+import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.ingest.PutPipelineRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.MockScriptEngine;
+import org.elasticsearch.script.MockScriptPlugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xcontent.json.JsonXContent;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+import static org.hamcrest.Matchers.equalTo;
+
+@ESIntegTestCase.ClusterScope(numDataNodes = 0, numClientNodes = 0, scope = ESIntegTestCase.Scope.TEST)
+public class IngestStatsNamesAndTypesIT extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(CustomIngestTestPlugin.class, CustomScriptPlugin.class);
+    }
+
+    @Override
+    protected boolean ignoreExternalCluster() {
+        return true;
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testIngestStatsNamesAndTypes() throws IOException {
+        String pipeline1 = org.elasticsearch.core.Strings.format("""
+            {
+              "processors": [
+                {
+                  "set": {
+                    "tag": "set-a",
+                    "field": "a",
+                    "value": "1"
+                  }
+                },
+                {
+                  "set": {
+                    "tag": "set-b",
+                    "field": "b",
+                    "value": "2",
+                    "if": {
+                      "lang": "%s",
+                      "source": "false_script"
+                    }
+                  }
+                },
+                {
+                  "set": {
+                    "tag": "set-c",
+                    "field": "c",
+                    "value": "3",
+                    "ignore_failure": true
+                  }
+                },
+                {
+                  "set": {
+                    "tag": "set-d",
+                    "field": "d",
+                    "value": "4",
+                    "if": {
+                      "lang": "%s",
+                      "source": "true_script"
+                    },
+                    "ignore_failure": true
+                  }
+                }
+              ]
+            }
+            """, MockScriptEngine.NAME, MockScriptEngine.NAME);
+        BytesReference pipeline1Reference = new BytesArray(pipeline1);
+        client().admin().cluster().putPipeline(new PutPipelineRequest("pipeline1", pipeline1Reference, XContentType.JSON)).actionGet();
+
+        // index a single document through the pipeline
+        BulkRequest bulkRequest = new BulkRequest();
+        bulkRequest.add(new IndexRequest("index1").id("1").source("{}", XContentType.JSON).setPipeline("pipeline1"));
+        client().bulk(bulkRequest).actionGet();
+
+        {
+            NodesStatsResponse nodesStatsResponse = client().admin()
+                .cluster()
+                .nodesStats(new NodesStatsRequest().addMetric("ingest"))
+                .actionGet();
+            assertThat(nodesStatsResponse.getNodes().size(), equalTo(1));
+
+            NodeStats stats = nodesStatsResponse.getNodes().get(0);
+            assertThat(stats.getIngestStats().getTotalStats().getIngestCount(), equalTo(1L));
+            assertThat(stats.getIngestStats().getPipelineStats().size(), equalTo(1));
+
+            IngestStats.PipelineStat pipelineStat = stats.getIngestStats().getPipelineStats().get(0);
+            assertThat(pipelineStat.getPipelineId(), equalTo("pipeline1"));
+            assertThat(pipelineStat.getStats().getIngestCount(), equalTo(1L));
+
+            List<IngestStats.ProcessorStat> processorStats = stats.getIngestStats().getProcessorStats().get("pipeline1");
+            assertThat(processorStats.size(), equalTo(4));
+
+            IngestStats.ProcessorStat setA = processorStats.get(0);
+            assertThat(setA.getName(), equalTo("set:set-a"));
+            assertThat(setA.getType(), equalTo("set"));
+            assertThat(setA.getStats().getIngestCount(), equalTo(1L));
+
+            IngestStats.ProcessorStat setB = processorStats.get(1);
+            assertThat(setB.getName(), equalTo("set:set-b"));
+            assertThat(setB.getType(), equalTo("conditional"));
+            assertThat(setB.getStats().getIngestCount(), equalTo(0L)); // see false_script above
+
+            IngestStats.ProcessorStat setC = processorStats.get(2);
+            assertThat(setC.getName(), equalTo("set:set-c"));
+            assertThat(setC.getType(), equalTo("set"));
+            assertThat(setC.getStats().getIngestCount(), equalTo(1L));
+
+            IngestStats.ProcessorStat setD = processorStats.get(3);
+            assertThat(setD.getName(), equalTo("compound:CompoundProcessor-set-d"));
+            assertThat(setD.getType(), equalTo("conditional"));
+            assertThat(setD.getStats().getIngestCount(), equalTo(1L));
+        }
+
+        {
+            // the bits that we want to read from the cluster stats response aren't visible in java code (no getters,
+            // non-public classes and methods), roundtrip through json so that we can read what we want
+            ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get();
+            XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
+            builder.startObject();
+            response.toXContent(builder, new ToXContent.MapParams(Map.of()));
+            builder.endObject();
+            Map<String, Object> stats = createParser(JsonXContent.jsonXContent, Strings.toString(builder)).map();
+
+            int setProcessorCount = path(stats, "nodes.ingest.processor_stats.set.count");
+            assertThat(setProcessorCount, equalTo(2));
+            int conditionalProcessorCount = path(stats, "nodes.ingest.processor_stats.conditional.count");
+            assertThat(conditionalProcessorCount, equalTo(1));
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static <T> T path(Map<String, Object> map, String path) {
+        String[] paths = path.split("\\.");
+        String[] leading = Arrays.copyOfRange(paths, 0, paths.length - 1);
+        String trailing = paths[paths.length - 1];
+        for (String key : leading) {
+            map = (Map<String, Object>) map.get(key);
+        }
+        return (T) map.get(trailing);
+    }
+
+    public static class CustomIngestTestPlugin extends IngestTestPlugin {
+        @Override
+        public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
+            Map<String, Processor.Factory> processors = new HashMap<>();
+            processors.put("set", (factories, tag, description, config) -> {
+                String field = (String) config.remove("field");
+                String value = (String) config.remove("value");
+                return new FakeProcessor("set", tag, description, (ingestDocument) -> ingestDocument.setFieldValue(field, value));
+            });
+
+            return processors;
+        }
+    }
+
+    public static class CustomScriptPlugin extends MockScriptPlugin {
+        @Override
+        protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
+            return Map.of("true_script", ctx -> true, "false_script", ctx -> false);
+        }
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTaskInitializationFailureIT.java b/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTaskInitializationFailureIT.java
index aa1a0d1fefd4..2002c36a0f86 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTaskInitializationFailureIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/persistent/PersistentTaskInitializationFailureIT.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.persistent;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -115,8 +115,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT;
         }
 
         @Override
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java
index 857ff11d5422..f40208c359ff 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -127,6 +128,14 @@ public void testListenersInvokedWhenIndexIsDeleted() throws Exception {
         }, 30L, TimeUnit.SECONDS);
     }
 
+    @TestLogging(
+        reason = "Debug #93226",
+        value = "org.elasticsearch.indices.cluster.IndicesClusterStateService:DEBUG,"
+            + "org.elasticsearch.indices.IndicesService:DEBUG,"
+            + "org.elasticsearch.index.IndexService:DEBUG,"
+            + "org.elasticsearch.env.NodeEnvironment:DEBUG,"
+            + "org.elasticsearch.cluster.service.MasterService:TRACE"
+    )
     public void testListenersInvokedWhenIndexIsRelocated() throws Exception {
         final String masterNode = internalCluster().startMasterOnlyNode();
         internalCluster().startDataOnlyNodes(4);
@@ -165,6 +174,7 @@ public void testListenersInvokedWhenIndexIsRelocated() throws Exception {
         }
 
         final List<String> excludedNodes = randomSubsetOf(2, shardsByNodes.keySet());
+        logger.info("--> excluding nodes {}", excludedNodes);
         assertAcked(
             client().admin()
                 .indices()
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java
index b25b59355b6f..2bb11d678e57 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java
@@ -7,25 +7,77 @@
  */
 package org.elasticsearch.readiness;
 
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.metadata.ReservedStateErrorMetadata;
+import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata;
+import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
+import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction;
+import org.elasticsearch.reservedstate.service.FileSettingsService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.readiness.ReadinessClientProbe;
 
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 import java.util.List;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 
+import static org.elasticsearch.node.Node.INITIAL_STATE_TIMEOUT_SETTING;
 import static org.elasticsearch.test.NodeRoles.dataOnlyNode;
 import static org.elasticsearch.test.NodeRoles.masterNode;
 import static org.elasticsearch.test.NodeRoles.nonDataNode;
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.notNullValue;
 
 @ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class ReadinessClusterIT extends ESIntegTestCase implements ReadinessClientProbe {
 
+    private static AtomicLong versionCounter = new AtomicLong(1);
+
+    private static String testErrorJSON = """
+        {
+             "metadata": {
+                 "version": "%s",
+                 "compatibility": "8.4.0"
+             },
+             "state": {
+                 "not_cluster_settings": {
+                     "search.allow_expensive_queries": "false"
+                 }
+             }
+        }""";
+
+    private static String testJSON = """
+        {
+             "metadata": {
+                 "version": "%s",
+                 "compatibility": "8.4.0"
+             },
+             "state": {
+                 "cluster_settings": {
+                     "indices.recovery.max_bytes_per_sec": "50mb"
+                 }
+             }
+        }""";
+
     @Override
     protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
         Settings.Builder settings = Settings.builder()
@@ -152,4 +204,148 @@ public Settings onNodeStopped(String nodeName) throws Exception {
             tcpReadinessProbeTrue(s);
         }
     }
+
+    private Tuple<CountDownLatch, AtomicLong> setupClusterStateListenerForError(String node) {
+        ClusterService clusterService = internalCluster().clusterService(node);
+        CountDownLatch savedClusterState = new CountDownLatch(1);
+        AtomicLong metadataVersion = new AtomicLong(-1);
+        clusterService.addListener(new ClusterStateListener() {
+            @Override
+            public void clusterChanged(ClusterChangedEvent event) {
+                ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
+                if (reservedState != null && reservedState.errorMetadata() != null) {
+                    assertEquals(ReservedStateErrorMetadata.ErrorKind.PARSING, reservedState.errorMetadata().errorKind());
+                    assertThat(reservedState.errorMetadata().errors(), allOf(notNullValue(), hasSize(1)));
+                    assertThat(
+                        reservedState.errorMetadata().errors().get(0),
+                        containsString("Missing handler definition for content key [not_cluster_settings]")
+                    );
+                    clusterService.removeListener(this);
+                    metadataVersion.set(event.state().metadata().version());
+                    savedClusterState.countDown();
+                }
+            }
+        });
+
+        return new Tuple<>(savedClusterState, metadataVersion);
+    }
+
+    private void writeJSONFile(String node, String json) throws Exception {
+        long version = versionCounter.incrementAndGet();
+
+        FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, node);
+
+        Files.createDirectories(fileSettingsService.operatorSettingsDir());
+        Path tempFilePath = createTempFile();
+
+        Files.write(tempFilePath, Strings.format(json, version).getBytes(StandardCharsets.UTF_8));
+        Files.move(tempFilePath, fileSettingsService.operatorSettingsFile(), StandardCopyOption.ATOMIC_MOVE);
+        logger.info("--> New file settings: [{}]", Strings.format(json, version));
+    }
+
+    public void testNotReadyOnBadFileSettings() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        logger.info("--> start data node / non master node");
+        String dataNode = internalCluster().startNode(Settings.builder().put(dataOnlyNode()).put("discovery.initial_state_timeout", "1s"));
+        FileSettingsService dataFileSettingsService = internalCluster().getInstance(FileSettingsService.class, dataNode);
+
+        assertFalse(dataFileSettingsService.watching());
+
+        logger.info("--> write bad file settings before we boot master node");
+        writeJSONFile(dataNode, testErrorJSON);
+
+        logger.info("--> start master node");
+        final String masterNode = internalCluster().startMasterOnlyNode(
+            Settings.builder().put(INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s").build()
+        );
+        assertMasterNode(internalCluster().nonMasterClient(), masterNode);
+        var savedClusterState = setupClusterStateListenerForError(masterNode);
+
+        // we need this after we setup the listener above, in case the node started and processed
+        // settings before we set our listener to cluster state changes.
+        causeClusterStateUpdate();
+
+        FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
+
+        assertTrue(masterFileSettingsService.watching());
+        assertFalse(dataFileSettingsService.watching());
+
+        boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
+        assertTrue(awaitSuccessful);
+
+        ReadinessService s = internalCluster().getInstance(ReadinessService.class, internalCluster().getMasterName());
+        assertNull(s.boundAddress());
+    }
+
+    private Tuple<CountDownLatch, AtomicLong> setupClusterStateListener(String node) {
+        ClusterService clusterService = internalCluster().clusterService(node);
+        CountDownLatch savedClusterState = new CountDownLatch(1);
+        AtomicLong metadataVersion = new AtomicLong(-1);
+        clusterService.addListener(new ClusterStateListener() {
+            @Override
+            public void clusterChanged(ClusterChangedEvent event) {
+                ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
+                if (reservedState != null) {
+                    ReservedStateHandlerMetadata handlerMetadata = reservedState.handlers().get(ReservedClusterSettingsAction.NAME);
+                    if (handlerMetadata != null && handlerMetadata.keys().contains("indices.recovery.max_bytes_per_sec")) {
+                        clusterService.removeListener(this);
+                        metadataVersion.set(event.state().metadata().version());
+                        savedClusterState.countDown();
+                    }
+                }
+            }
+        });
+
+        return new Tuple<>(savedClusterState, metadataVersion);
+    }
+
+    public void testReadyAfterCorrectFileSettings() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        logger.info("--> start data node / non master node");
+        String dataNode = internalCluster().startNode(Settings.builder().put(dataOnlyNode()).put("discovery.initial_state_timeout", "1s"));
+        FileSettingsService dataFileSettingsService = internalCluster().getInstance(FileSettingsService.class, dataNode);
+
+        assertFalse(dataFileSettingsService.watching());
+        var savedClusterState = setupClusterStateListener(dataNode);
+
+        logger.info("--> write correct file settings before we boot master node");
+        writeJSONFile(dataNode, testJSON);
+
+        logger.info("--> start master node");
+        final String masterNode = internalCluster().startMasterOnlyNode();
+        assertMasterNode(internalCluster().nonMasterClient(), masterNode);
+
+        FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
+
+        assertTrue(masterFileSettingsService.watching());
+        assertFalse(dataFileSettingsService.watching());
+
+        boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
+        assertTrue(awaitSuccessful);
+
+        ReadinessService s = internalCluster().getInstance(ReadinessService.class, internalCluster().getMasterName());
+        tcpReadinessProbeTrue(s);
+    }
+
+    private void causeClusterStateUpdate() {
+        PlainActionFuture.get(
+            fut -> internalCluster().getCurrentMasterNodeInstance(ClusterService.class)
+                .submitUnbatchedStateUpdateTask("poke", new ClusterStateUpdateTask() {
+                    @Override
+                    public ClusterState execute(ClusterState currentState) {
+                        return ClusterState.builder(currentState).build();
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        assert false : e;
+                    }
+
+                    @Override
+                    public void clusterStateProcessed(ClusterState initialState, ClusterState newState) {
+                        fut.onResponse(null);
+                    }
+                })
+        );
+    }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/IndexSnapshotsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/IndexSnapshotsServiceIT.java
index 69f88b1bc886..9f7c53d49407 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/IndexSnapshotsServiceIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/IndexSnapshotsServiceIT.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.repositories.fs.FsRepository;
@@ -31,7 +32,6 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 
@@ -59,7 +59,7 @@ public void testGetShardSnapshotFromUnknownRepoReturnsAnError() throws Exception
                 assertThat(repositoryException, is(notNullValue()));
                 assertThat(
                     repositoryException.getMessage(),
-                    equalTo(formatted("[%s] Unable to find the latest snapshot for shard [[idx][0]]", repository))
+                    equalTo(Strings.format("[%s] Unable to find the latest snapshot for shard [[idx][0]]", repository))
                 );
             }
         } else {
@@ -133,7 +133,7 @@ public void testGetShardSnapshotReturnsTheLatestSuccessfulSnapshot() throws Exce
                 indexRandomDocs(indexName2, 10);
             }
             final List<String> snapshotIndices = randomSubsetOf(indices);
-            final SnapshotInfo snapshotInfo = createSnapshot(repoName, formatted("snap-%03d", i), snapshotIndices);
+            final SnapshotInfo snapshotInfo = createSnapshot(repoName, Strings.format("snap-%03d", i), snapshotIndices);
             if (snapshotInfo.indices().contains(indexName)) {
                 lastSnapshot = snapshotInfo;
                 ClusterStateResponse clusterStateResponse = admin().cluster().prepareState().execute().actionGet();
@@ -206,10 +206,12 @@ public void testGetShardSnapshotFailureHandlingLetOtherRepositoriesRequestsMakeP
         createIndexWithContent(indexName);
 
         int snapshotIdx = 0;
-        createSnapshot(failingRepoName, formatted("snap-%03d", snapshotIdx++), Collections.singletonList(indexName));
+        Object[] args1 = new Object[] { snapshotIdx++ };
+        createSnapshot(failingRepoName, Strings.format("snap-%03d", args1), Collections.singletonList(indexName));
         SnapshotInfo latestSnapshot = null;
         for (String workingRepoName : workingRepoNames) {
-            String snapshot = formatted("snap-%03d", snapshotIdx++);
+            Object[] args = new Object[] { snapshotIdx++ };
+            String snapshot = Strings.format("snap-%03d", args);
             latestSnapshot = createSnapshot(workingRepoName, snapshot, Collections.singletonList(indexName));
         }
 
@@ -234,7 +236,7 @@ public void testGetShardSnapshotFailureHandlingLetOtherRepositoriesRequestsMakeP
         assertThat(error.isPresent(), is(equalTo(true)));
         assertThat(
             error.get().getMessage(),
-            equalTo(String.format(Locale.ROOT, "[%s] Unable to find the latest snapshot for shard [[%s][0]]", failingRepoName, indexName))
+            equalTo(Strings.format("[%s] Unable to find the latest snapshot for shard [[%s][0]]", failingRepoName, indexName))
         );
 
         for (String workingRepoName : workingRepoNames) {
@@ -264,7 +266,8 @@ public void testGetShardSnapshotInMultipleRepositoriesReturnsTheLatestSnapshot()
         int snapshotIdx = 0;
         SnapshotInfo expectedLatestSnapshot = null;
         for (String repository : repositories) {
-            String snapshot = formatted("snap-%03d", snapshotIdx++);
+            Object[] args = new Object[] { snapshotIdx++ };
+            String snapshot = Strings.format("snap-%03d", args);
             expectedLatestSnapshot = createSnapshot(repository, snapshot, Collections.singletonList(indexName));
         }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/ComponentTemplatesFileSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/ComponentTemplatesFileSettingsIT.java
index 8954bcd542cf..c7b2387b2fa6 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/ComponentTemplatesFileSettingsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/ComponentTemplatesFileSettingsIT.java
@@ -8,6 +8,8 @@
 
 package org.elasticsearch.reservedstate.service;
 
+import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.indices.template.get.GetComponentTemplateAction;
 import org.elasticsearch.action.admin.indices.template.get.GetComposableIndexTemplateAction;
 import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction;
@@ -33,6 +35,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
@@ -399,13 +402,15 @@ private void assertClusterStateSaveOK(CountDownLatch savedClusterState, AtomicLo
         boolean awaitSuccessful = savedClusterState.await(20, TimeUnit.SECONDS);
         assertTrue(awaitSuccessful);
 
-        final var response = client().execute(
-            GetComposableIndexTemplateAction.INSTANCE,
-            new GetComposableIndexTemplateAction.Request("template*")
-        ).get();
+        final ClusterStateResponse clusterStateResponse = client().admin()
+            .cluster()
+            .state(new ClusterStateRequest().waitForMetadataVersion(metadataVersion.get()))
+            .actionGet();
+
+        Map<String, ComposableIndexTemplate> allTemplates = clusterStateResponse.getState().metadata().templatesV2();
 
         assertThat(
-            response.indexTemplates().keySet().stream().collect(Collectors.toSet()),
+            allTemplates.keySet().stream().collect(Collectors.toSet()),
             containsInAnyOrder("template_1", "template_2", "template_other")
         );
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java
index cfde48d088db..10a14d027f9f 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java
@@ -34,7 +34,9 @@
 import java.util.concurrent.atomic.AtomicLong;
 
 import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING;
+import static org.elasticsearch.node.Node.INITIAL_STATE_TIMEOUT_SETTING;
 import static org.elasticsearch.test.NodeRoles.dataOnlyNode;
+import static org.elasticsearch.test.NodeRoles.masterNode;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -60,6 +62,30 @@ public class FileSettingsServiceIT extends ESIntegTestCase {
              }
         }""";
 
+    private static String testJSON43mb = """
+        {
+             "metadata": {
+                 "version": "%s",
+                 "compatibility": "8.4.0"
+             },
+             "state": {
+                 "cluster_settings": {
+                     "indices.recovery.max_bytes_per_sec": "43mb"
+                 }
+             }
+        }""";
+
+    private static String testCleanupJSON = """
+        {
+             "metadata": {
+                 "version": "%s",
+                 "compatibility": "8.4.0"
+             },
+             "state": {
+                 "cluster_settings": {}
+             }
+        }""";
+
     private static String testErrorJSON = """
         {
              "metadata": {
@@ -90,6 +116,29 @@ private void writeJSONFile(String node, String json) throws Exception {
 
         Files.write(tempFilePath, Strings.format(json, version).getBytes(StandardCharsets.UTF_8));
         Files.move(tempFilePath, fileSettingsService.operatorSettingsFile(), StandardCopyOption.ATOMIC_MOVE);
+        logger.info("--> New file settings: [{}]", Strings.format(json, version));
+    }
+
+    private Tuple<CountDownLatch, AtomicLong> setupCleanupClusterStateListener(String node) {
+        ClusterService clusterService = internalCluster().clusterService(node);
+        CountDownLatch savedClusterState = new CountDownLatch(1);
+        AtomicLong metadataVersion = new AtomicLong(-1);
+        clusterService.addListener(new ClusterStateListener() {
+            @Override
+            public void clusterChanged(ClusterChangedEvent event) {
+                ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
+                if (reservedState != null) {
+                    ReservedStateHandlerMetadata handlerMetadata = reservedState.handlers().get(ReservedClusterSettingsAction.NAME);
+                    if (handlerMetadata != null && handlerMetadata.keys().contains("indices.recovery.max_bytes_per_sec") == false) {
+                        clusterService.removeListener(this);
+                        metadataVersion.set(event.state().metadata().version());
+                        savedClusterState.countDown();
+                    }
+                }
+            }
+        });
+
+        return new Tuple<>(savedClusterState, metadataVersion);
     }
 
     private Tuple<CountDownLatch, AtomicLong> setupClusterStateListener(String node) {
@@ -114,7 +163,8 @@ public void clusterChanged(ClusterChangedEvent event) {
         return new Tuple<>(savedClusterState, metadataVersion);
     }
 
-    private void assertClusterStateSaveOK(CountDownLatch savedClusterState, AtomicLong metadataVersion) throws Exception {
+    private void assertClusterStateSaveOK(CountDownLatch savedClusterState, AtomicLong metadataVersion, String expectedBytesPerSec)
+        throws Exception {
         boolean awaitSuccessful = savedClusterState.await(20, TimeUnit.SECONDS);
         assertTrue(awaitSuccessful);
 
@@ -125,7 +175,7 @@ private void assertClusterStateSaveOK(CountDownLatch savedClusterState, AtomicLo
 
         assertThat(
             clusterStateResponse.getState().metadata().persistentSettings().get(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey()),
-            equalTo("50mb")
+            equalTo(expectedBytesPerSec)
         );
 
         ClusterUpdateSettingsRequest req = new ClusterUpdateSettingsRequest().persistentSettings(
@@ -158,7 +208,7 @@ public void testSettingsApplied() throws Exception {
         assertFalse(dataFileSettingsService.watching());
 
         writeJSONFile(masterNode, testJSON);
-        assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2());
+        assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "50mb");
     }
 
     public void testSettingsAppliedOnStart() throws Exception {
@@ -183,7 +233,41 @@ public void testSettingsAppliedOnStart() throws Exception {
         assertTrue(masterFileSettingsService.watching());
         assertFalse(dataFileSettingsService.watching());
 
-        assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2());
+        assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "50mb");
+    }
+
+    public void testReservedStatePersistsOnRestart() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        logger.info("--> start master node");
+        final String masterNode = internalCluster().startMasterOnlyNode(
+            Settings.builder().put(INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s").build()
+        );
+        assertMasterNode(internalCluster().masterClient(), masterNode);
+        var savedClusterState = setupClusterStateListener(masterNode);
+
+        FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
+
+        assertTrue(masterFileSettingsService.watching());
+
+        logger.info("--> write some settings");
+        writeJSONFile(masterNode, testJSON);
+        assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "50mb");
+
+        logger.info("--> restart master");
+        internalCluster().restartNode(masterNode);
+
+        final ClusterStateResponse clusterStateResponse = client().admin().cluster().state(new ClusterStateRequest()).actionGet();
+        assertEquals(
+            1,
+            clusterStateResponse.getState()
+                .metadata()
+                .reservedStateMetadata()
+                .get(FileSettingsService.NAMESPACE)
+                .handlers()
+                .get(ReservedClusterSettingsAction.NAME)
+                .keys()
+                .size()
+        );
     }
 
     private Tuple<CountDownLatch, AtomicLong> setupClusterStateListenerForError(String node) {
@@ -238,7 +322,9 @@ public void testErrorSaved() throws Exception {
         assertFalse(dataFileSettingsService.watching());
 
         logger.info("--> start master node");
-        final String masterNode = internalCluster().startMasterOnlyNode();
+        final String masterNode = internalCluster().startMasterOnlyNode(
+            Settings.builder().put(INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s").build()
+        );
         assertMasterNode(internalCluster().nonMasterClient(), masterNode);
         var savedClusterState = setupClusterStateListenerForError(masterNode);
 
@@ -250,4 +336,48 @@ public void testErrorSaved() throws Exception {
         writeJSONFile(masterNode, testErrorJSON);
         assertClusterStateNotSaved(savedClusterState.v1(), savedClusterState.v2());
     }
+
+    public void testSettingsAppliedOnMasterReElection() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        logger.info("--> start master node");
+        final String masterNode = internalCluster().startMasterOnlyNode();
+
+        logger.info("--> start master eligible nodes, 2 more for quorum");
+        String masterNode1 = internalCluster().startNode(Settings.builder().put(masterNode()).put("discovery.initial_state_timeout", "1s"));
+        String masterNode2 = internalCluster().startNode(Settings.builder().put(masterNode()).put("discovery.initial_state_timeout", "1s"));
+        FileSettingsService master1FS = internalCluster().getInstance(FileSettingsService.class, masterNode1);
+        FileSettingsService master2FS = internalCluster().getInstance(FileSettingsService.class, masterNode2);
+
+        assertFalse(master1FS.watching());
+        assertFalse(master2FS.watching());
+
+        var savedClusterState = setupClusterStateListener(masterNode);
+        FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
+
+        assertTrue(masterFileSettingsService.watching());
+
+        writeJSONFile(masterNode, testJSON);
+        assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "50mb");
+
+        internalCluster().stopCurrentMasterNode();
+        ensureStableCluster(2);
+
+        FileSettingsService masterFS = internalCluster().getCurrentMasterNodeInstance(FileSettingsService.class);
+        assertTrue(masterFS.watching());
+        logger.info("--> start another master eligible node to form a quorum");
+        internalCluster().startNode(Settings.builder().put(masterNode()).put("discovery.initial_state_timeout", "1s"));
+        ensureStableCluster(3);
+
+        savedClusterState = setupCleanupClusterStateListener(internalCluster().getMasterName());
+        writeJSONFile(internalCluster().getMasterName(), testCleanupJSON);
+
+        boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
+        assertTrue(awaitSuccessful);
+
+        savedClusterState = setupClusterStateListener(internalCluster().getMasterName());
+        writeJSONFile(internalCluster().getMasterName(), testJSON43mb);
+
+        assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "43mb");
+    }
+
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/RepositoriesFileSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/RepositoriesFileSettingsIT.java
index 81e28686ee95..577e9f275e03 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/RepositoriesFileSettingsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/RepositoriesFileSettingsIT.java
@@ -49,7 +49,6 @@
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class RepositoriesFileSettingsIT extends ESIntegTestCase {
-
     private static AtomicLong versionCounter = new AtomicLong(1);
 
     private static String testJSON = """
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/SnaphotsAndFileSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/SnaphotsAndFileSettingsIT.java
deleted file mode 100644
index 360dd4a68415..000000000000
--- a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/SnaphotsAndFileSettingsIT.java
+++ /dev/null
@@ -1,335 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.reservedstate.service;
-
-import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsAction;
-import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
-import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
-import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterStateListener;
-import org.elasticsearch.cluster.InternalClusterInfoService;
-import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata;
-import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.Strings;
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.core.Tuple;
-import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction;
-import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
-import org.elasticsearch.snapshots.SnapshotState;
-import org.junit.After;
-
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.hamcrest.Matchers.equalTo;
-
-/**
- * Tests that snapshot restore behaves correctly when we have file based settings that reserve part of the
- * cluster state
- */
-public class SnaphotsAndFileSettingsIT extends AbstractSnapshotIntegTestCase {
-    private static AtomicLong versionCounter = new AtomicLong(1);
-
-    private static String testFileSettingsJSON = """
-        {
-             "metadata": {
-                 "version": "%s",
-                 "compatibility": "8.4.0"
-             },
-             "state": {
-                 "cluster_settings": {
-                     "indices.recovery.max_bytes_per_sec": "50mb"
-                 }
-             }
-        }""";
-
-    private static String emptyFileSettingsJSON = """
-        {
-             "metadata": {
-                 "version": "%s",
-                 "compatibility": "8.4.0"
-             },
-             "state": {
-                 "cluster_settings": {}
-             }
-        }""";
-
-    @After
-    public void cleanUp() throws Exception {
-        awaitNoMoreRunningOperations();
-    }
-
-    private void writeJSONFile(String node, String json) throws Exception {
-        long version = versionCounter.incrementAndGet();
-
-        FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, node);
-
-        Files.createDirectories(fileSettingsService.operatorSettingsDir());
-        Path tempFilePath = createTempFile();
-
-        Files.write(tempFilePath, Strings.format(json, version).getBytes(StandardCharsets.UTF_8));
-        Files.move(tempFilePath, fileSettingsService.operatorSettingsFile(), StandardCopyOption.ATOMIC_MOVE);
-    }
-
-    private Tuple<CountDownLatch, AtomicLong> setupClusterStateListener(String node) {
-        ClusterService clusterService = internalCluster().clusterService(node);
-        CountDownLatch savedClusterState = new CountDownLatch(1);
-        AtomicLong metadataVersion = new AtomicLong(-1);
-        clusterService.addListener(new ClusterStateListener() {
-            @Override
-            public void clusterChanged(ClusterChangedEvent event) {
-                ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
-                if (reservedState != null && reservedState.version() != 0L) {
-                    ReservedStateHandlerMetadata handlerMetadata = reservedState.handlers().get(ReservedClusterSettingsAction.NAME);
-                    if (handlerMetadata != null && handlerMetadata.keys().contains("indices.recovery.max_bytes_per_sec")) {
-                        clusterService.removeListener(this);
-                        metadataVersion.set(event.state().metadata().version());
-                        savedClusterState.countDown();
-                    }
-                }
-            }
-        });
-
-        return new Tuple<>(savedClusterState, metadataVersion);
-    }
-
-    private ClusterStateResponse assertClusterStateSaveOK(CountDownLatch savedClusterState, AtomicLong metadataVersion) throws Exception {
-        boolean awaitSuccessful = savedClusterState.await(20, TimeUnit.SECONDS);
-        assertTrue(awaitSuccessful);
-
-        return clusterAdmin().state(new ClusterStateRequest().waitForMetadataVersion(metadataVersion.get())).get();
-    }
-
-    public void testRestoreWithRemovedFileSettings() throws Exception {
-        createRepository("test-repo", "fs");
-
-        logger.info("--> set some persistent cluster settings");
-        assertAcked(
-            clusterAdmin().prepareUpdateSettings()
-                .setPersistentSettings(
-                    Settings.builder()
-                        .put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey(), TimeValue.timeValueSeconds(25))
-                        .build()
-                )
-        );
-
-        ensureGreen();
-
-        String masterNode = internalCluster().getMasterName();
-
-        var savedClusterState = setupClusterStateListener(masterNode);
-        FileSettingsService fs = internalCluster().getInstance(FileSettingsService.class, masterNode);
-
-        logger.info("--> write some file based settings, putting some reserved state");
-        writeJSONFile(masterNode, testFileSettingsJSON);
-        final ClusterStateResponse savedStateResponse = assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2());
-        assertThat(
-            savedStateResponse.getState().metadata().persistentSettings().get(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey()),
-            equalTo("50mb")
-        );
-
-        logger.info("--> create full snapshot");
-        createFullSnapshot("test-repo", "test-snap");
-        assertThat(getSnapshot("test-repo", "test-snap").state(), equalTo(SnapshotState.SUCCESS));
-
-        assertAcked(
-            clusterAdmin().prepareUpdateSettings()
-                .setPersistentSettings(
-                    Settings.builder()
-                        .put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey(), TimeValue.timeValueSeconds(55))
-                        .build()
-                )
-        );
-
-        logger.info("--> deleting operator file, no file based settings");
-        Files.delete(fs.operatorSettingsFile());
-
-        logger.info("--> restore global state from the snapshot");
-        clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap").setRestoreGlobalState(true).setWaitForCompletion(true).get();
-
-        ensureGreen();
-
-        final ClusterStateResponse clusterStateResponse = clusterAdmin().state(new ClusterStateRequest().metadata(true)).actionGet();
-
-        // We expect no reserved metadata state for file based settings, the operator file was deleted.
-        assertNull(clusterStateResponse.getState().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE));
-
-        final ClusterGetSettingsAction.Response getSettingsResponse = clusterAdmin().execute(
-            ClusterGetSettingsAction.INSTANCE,
-            new ClusterGetSettingsAction.Request()
-        ).actionGet();
-
-        assertThat(
-            getSettingsResponse.persistentSettings().get(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey()),
-            equalTo("25s")
-        );
-        // We didn't remove the setting set by file settings, we simply removed the reserved (operator) section.
-        assertThat(getSettingsResponse.persistentSettings().get("indices.recovery.max_bytes_per_sec"), equalTo("50mb"));
-        // cleanup
-        assertAcked(
-            clusterAdmin().prepareUpdateSettings()
-                .setPersistentSettings(
-                    Settings.builder()
-                        .put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey(), (String) null)
-                        .put("indices.recovery.max_bytes_per_sec", (String) null)
-                        .build()
-                )
-        );
-    }
-
-    private Tuple<CountDownLatch, AtomicLong> removedReservedClusterStateListener(String node) {
-        ClusterService clusterService = internalCluster().clusterService(node);
-        CountDownLatch savedClusterState = new CountDownLatch(2);
-        AtomicLong metadataVersion = new AtomicLong(-1);
-        clusterService.addListener(new ClusterStateListener() {
-            @Override
-            public void clusterChanged(ClusterChangedEvent event) {
-                ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
-                // we first wait for reserved state version to become 0, then we expect to see it non-zero
-                if (reservedState != null && reservedState.version() == 0L) {
-                    // don't remove the state listener yet, we need it to see the version non-zero
-                    metadataVersion.set(event.state().metadata().version());
-                    savedClusterState.countDown();
-                } else if (reservedState != null && reservedState.version() != 0L && savedClusterState.getCount() < 2) {
-                    clusterService.removeListener(this);
-                    metadataVersion.set(event.state().metadata().version());
-                    savedClusterState.countDown();
-                }
-            }
-        });
-
-        return new Tuple<>(savedClusterState, metadataVersion);
-    }
-
-    private Tuple<CountDownLatch, AtomicLong> cleanedClusterStateListener(String node) {
-        ClusterService clusterService = internalCluster().clusterService(node);
-        CountDownLatch savedClusterState = new CountDownLatch(1);
-        AtomicLong metadataVersion = new AtomicLong(-1);
-        clusterService.addListener(new ClusterStateListener() {
-            @Override
-            public void clusterChanged(ClusterChangedEvent event) {
-                ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
-                if (reservedState != null) {
-                    ReservedStateHandlerMetadata handlerMetadata = reservedState.handlers().get(ReservedClusterSettingsAction.NAME);
-                    if (handlerMetadata == null) {
-                        fail("Should've found cluster settings in this metadata");
-                    }
-                    if (handlerMetadata.keys().isEmpty()) {
-                        clusterService.removeListener(this);
-                        metadataVersion.set(event.state().metadata().version());
-                        savedClusterState.countDown();
-                    }
-                }
-            }
-        });
-
-        return new Tuple<>(savedClusterState, metadataVersion);
-    }
-
-    public void testRestoreWithPersistedFileSettings() throws Exception {
-        createRepository("test-repo", "fs");
-
-        logger.info("--> set some persistent cluster settings");
-        assertAcked(
-            clusterAdmin().prepareUpdateSettings()
-                .setPersistentSettings(
-                    Settings.builder()
-                        .put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey(), TimeValue.timeValueSeconds(25))
-                        .build()
-                )
-        );
-
-        ensureGreen();
-
-        String masterNode = internalCluster().getMasterName();
-
-        var savedClusterState = setupClusterStateListener(masterNode);
-        FileSettingsService fs = internalCluster().getInstance(FileSettingsService.class, masterNode);
-
-        logger.info("--> write some file based settings, putting some reserved state");
-        writeJSONFile(masterNode, testFileSettingsJSON);
-        final ClusterStateResponse savedStateResponse = assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2());
-        assertThat(
-            savedStateResponse.getState().metadata().persistentSettings().get(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey()),
-            equalTo("50mb")
-        );
-
-        logger.info("--> create full snapshot");
-        createFullSnapshot("test-repo", "test-snap");
-        assertThat(getSnapshot("test-repo", "test-snap").state(), equalTo(SnapshotState.SUCCESS));
-
-        assertAcked(
-            clusterAdmin().prepareUpdateSettings()
-                .setPersistentSettings(
-                    Settings.builder()
-                        .put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey(), TimeValue.timeValueSeconds(55))
-                        .build()
-                )
-        );
-
-        logger.info("--> restore global state from the snapshot");
-        var removedReservedState = removedReservedClusterStateListener(masterNode);
-
-        clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap").setRestoreGlobalState(true).setWaitForCompletion(true).get();
-
-        ensureGreen();
-
-        // When the target cluster of a restore has an existing operator file, we don't un-reserve the reserved
-        // cluster state for file based settings, but instead we reset the version to 0 and 'touch' the operator file
-        // so that it gets re-processed.
-        logger.info("--> reserved state version will be reset to 0, because of snapshot restore");
-        // double timeout, we restore snapshot then apply the file
-        assertTrue(removedReservedState.v1().await(40, TimeUnit.SECONDS));
-
-        logger.info("--> reserved state would be restored to non-zero version");
-
-        final ClusterStateResponse clusterStateResponse = clusterAdmin().state(
-            new ClusterStateRequest().metadata(true).waitForMetadataVersion(removedReservedState.v2().get())
-        ).actionGet();
-
-        assertNotNull(clusterStateResponse.getState().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE));
-
-        final ClusterGetSettingsAction.Response getSettingsResponse = clusterAdmin().execute(
-            ClusterGetSettingsAction.INSTANCE,
-            new ClusterGetSettingsAction.Request()
-        ).actionGet();
-
-        assertThat(
-            getSettingsResponse.persistentSettings().get(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey()),
-            equalTo("25s")
-        );
-
-        // we need to remove the reserved state, so that clean-up can happen
-        var cleanupReservedState = cleanedClusterStateListener(masterNode);
-
-        logger.info("--> clear the file based settings");
-        writeJSONFile(masterNode, emptyFileSettingsJSON);
-        assertClusterStateSaveOK(cleanupReservedState.v1(), cleanupReservedState.v2());
-        // cleanup
-        assertAcked(
-            clusterAdmin().prepareUpdateSettings()
-                .setPersistentSettings(
-                    Settings.builder()
-                        .put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey(), (String) null)
-                        .put("indices.recovery.max_bytes_per_sec", (String) null)
-                        .build()
-                )
-        );
-    }
-
-}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/SnapshotsAndFileSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/SnapshotsAndFileSettingsIT.java
new file mode 100644
index 000000000000..82259968a98e
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/SnapshotsAndFileSettingsIT.java
@@ -0,0 +1,355 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.reservedstate.service;
+
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsAction;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.InternalClusterInfoService;
+import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata;
+import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction;
+import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
+import org.elasticsearch.snapshots.SnapshotState;
+import org.junit.After;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * Tests that snapshot restore behaves correctly when we have file based settings that reserve part of the
+ * cluster state
+ */
+@LuceneTestCase.SuppressFileSystems("*")
+public class SnapshotsAndFileSettingsIT extends AbstractSnapshotIntegTestCase {
+    private static AtomicLong versionCounter = new AtomicLong(1);
+
+    private static String testFileSettingsJSON = """
+        {
+             "metadata": {
+                 "version": "%s",
+                 "compatibility": "8.4.0"
+             },
+             "state": {
+                 "cluster_settings": {
+                     "indices.recovery.max_bytes_per_sec": "50mb"
+                 }
+             }
+        }""";
+
+    private static String emptyFileSettingsJSON = """
+        {
+             "metadata": {
+                 "version": "%s",
+                 "compatibility": "8.4.0"
+             },
+             "state": {
+                 "cluster_settings": {}
+             }
+        }""";
+
+    @After
+    public void cleanUp() throws Exception {
+        awaitNoMoreRunningOperations();
+    }
+
+    private long retryDelay(int retryCount) {
+        return 100 * (1 << retryCount) + Randomness.get().nextInt(10);
+    }
+
+    private void writeJSONFile(String node, String json) throws Exception {
+        long version = versionCounter.incrementAndGet();
+
+        FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, node);
+
+        Files.createDirectories(fileSettingsService.operatorSettingsDir());
+        Path tempFilePath = createTempFile();
+
+        Files.write(tempFilePath, Strings.format(json, version).getBytes(StandardCharsets.UTF_8));
+        int retryCount = 0;
+        do {
+            try {
+                // this can fail on Windows because of timing
+                Files.move(tempFilePath, fileSettingsService.operatorSettingsFile(), StandardCopyOption.ATOMIC_MOVE);
+                return;
+            } catch (IOException e) {
+                logger.info("--> retrying writing a settings file [" + retryCount + "]");
+                if (retryCount == 4) { // retry 5 times
+                    throw e;
+                }
+                Thread.sleep(retryDelay(retryCount));
+                retryCount++;
+            }
+        } while (true);
+    }
+
+    private Tuple<CountDownLatch, AtomicLong> setupClusterStateListener(String node) {
+        ClusterService clusterService = internalCluster().clusterService(node);
+        CountDownLatch savedClusterState = new CountDownLatch(1);
+        AtomicLong metadataVersion = new AtomicLong(-1);
+        clusterService.addListener(new ClusterStateListener() {
+            @Override
+            public void clusterChanged(ClusterChangedEvent event) {
+                ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
+                if (reservedState != null && reservedState.version() != 0L) {
+                    ReservedStateHandlerMetadata handlerMetadata = reservedState.handlers().get(ReservedClusterSettingsAction.NAME);
+                    if (handlerMetadata != null && handlerMetadata.keys().contains("indices.recovery.max_bytes_per_sec")) {
+                        clusterService.removeListener(this);
+                        metadataVersion.set(event.state().metadata().version());
+                        savedClusterState.countDown();
+                    }
+                }
+            }
+        });
+
+        return new Tuple<>(savedClusterState, metadataVersion);
+    }
+
+    private ClusterStateResponse assertClusterStateSaveOK(CountDownLatch savedClusterState, AtomicLong metadataVersion) throws Exception {
+        boolean awaitSuccessful = savedClusterState.await(20, TimeUnit.SECONDS);
+        assertTrue(awaitSuccessful);
+
+        return clusterAdmin().state(new ClusterStateRequest().waitForMetadataVersion(metadataVersion.get())).get();
+    }
+
+    public void testRestoreWithRemovedFileSettings() throws Exception {
+        createRepository("test-repo", "fs");
+
+        logger.info("--> set some persistent cluster settings");
+        assertAcked(
+            clusterAdmin().prepareUpdateSettings()
+                .setPersistentSettings(
+                    Settings.builder()
+                        .put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey(), TimeValue.timeValueSeconds(25))
+                        .build()
+                )
+        );
+
+        ensureGreen();
+
+        String masterNode = internalCluster().getMasterName();
+
+        var savedClusterState = setupClusterStateListener(masterNode);
+        FileSettingsService fs = internalCluster().getInstance(FileSettingsService.class, masterNode);
+
+        logger.info("--> write some file based settings, putting some reserved state");
+        writeJSONFile(masterNode, testFileSettingsJSON);
+        final ClusterStateResponse savedStateResponse = assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2());
+        assertThat(
+            savedStateResponse.getState().metadata().persistentSettings().get(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey()),
+            equalTo("50mb")
+        );
+
+        logger.info("--> create full snapshot");
+        createFullSnapshot("test-repo", "test-snap");
+        assertThat(getSnapshot("test-repo", "test-snap").state(), equalTo(SnapshotState.SUCCESS));
+
+        assertAcked(
+            clusterAdmin().prepareUpdateSettings()
+                .setPersistentSettings(
+                    Settings.builder()
+                        .put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey(), TimeValue.timeValueSeconds(55))
+                        .build()
+                )
+        );
+
+        logger.info("--> deleting operator file, no file based settings");
+        Files.delete(fs.operatorSettingsFile());
+
+        logger.info("--> restore global state from the snapshot");
+        clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap").setRestoreGlobalState(true).setWaitForCompletion(true).get();
+
+        ensureGreen();
+
+        final ClusterStateResponse clusterStateResponse = clusterAdmin().state(new ClusterStateRequest().metadata(true)).actionGet();
+
+        // We expect no reserved metadata state for file based settings, the operator file was deleted.
+        assertNull(clusterStateResponse.getState().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE));
+
+        final ClusterGetSettingsAction.Response getSettingsResponse = clusterAdmin().execute(
+            ClusterGetSettingsAction.INSTANCE,
+            new ClusterGetSettingsAction.Request()
+        ).actionGet();
+
+        assertThat(
+            getSettingsResponse.persistentSettings().get(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey()),
+            equalTo("25s")
+        );
+        // We didn't remove the setting set by file settings, we simply removed the reserved (operator) section.
+        assertThat(getSettingsResponse.persistentSettings().get("indices.recovery.max_bytes_per_sec"), equalTo("50mb"));
+        // cleanup
+        assertAcked(
+            clusterAdmin().prepareUpdateSettings()
+                .setPersistentSettings(
+                    Settings.builder()
+                        .put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey(), (String) null)
+                        .put("indices.recovery.max_bytes_per_sec", (String) null)
+                        .build()
+                )
+        );
+    }
+
+    private Tuple<CountDownLatch, AtomicLong> removedReservedClusterStateListener(String node) {
+        ClusterService clusterService = internalCluster().clusterService(node);
+        CountDownLatch savedClusterState = new CountDownLatch(2);
+        AtomicLong metadataVersion = new AtomicLong(-1);
+        clusterService.addListener(new ClusterStateListener() {
+            @Override
+            public void clusterChanged(ClusterChangedEvent event) {
+                ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
+                // we first wait for reserved state version to become 0, then we expect to see it non-zero
+                if (reservedState != null && reservedState.version() == 0L) {
+                    // don't remove the state listener yet, we need it to see the version non-zero
+                    metadataVersion.set(event.state().metadata().version());
+                    savedClusterState.countDown();
+                } else if (reservedState != null && reservedState.version() != 0L && savedClusterState.getCount() < 2) {
+                    clusterService.removeListener(this);
+                    metadataVersion.set(event.state().metadata().version());
+                    savedClusterState.countDown();
+                }
+            }
+        });
+
+        return new Tuple<>(savedClusterState, metadataVersion);
+    }
+
+    private Tuple<CountDownLatch, AtomicLong> cleanedClusterStateListener(String node) {
+        ClusterService clusterService = internalCluster().clusterService(node);
+        CountDownLatch savedClusterState = new CountDownLatch(1);
+        AtomicLong metadataVersion = new AtomicLong(-1);
+        clusterService.addListener(new ClusterStateListener() {
+            @Override
+            public void clusterChanged(ClusterChangedEvent event) {
+                ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
+                if (reservedState != null) {
+                    ReservedStateHandlerMetadata handlerMetadata = reservedState.handlers().get(ReservedClusterSettingsAction.NAME);
+                    if (handlerMetadata == null) {
+                        fail("Should've found cluster settings in this metadata");
+                    }
+                    if (handlerMetadata.keys().isEmpty()) {
+                        clusterService.removeListener(this);
+                        metadataVersion.set(event.state().metadata().version());
+                        savedClusterState.countDown();
+                    }
+                }
+            }
+        });
+
+        return new Tuple<>(savedClusterState, metadataVersion);
+    }
+
+    public void testRestoreWithPersistedFileSettings() throws Exception {
+        createRepository("test-repo", "fs");
+
+        logger.info("--> set some persistent cluster settings");
+        assertAcked(
+            clusterAdmin().prepareUpdateSettings()
+                .setPersistentSettings(
+                    Settings.builder()
+                        .put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey(), TimeValue.timeValueSeconds(25))
+                        .build()
+                )
+        );
+
+        ensureGreen();
+
+        String masterNode = internalCluster().getMasterName();
+
+        var savedClusterState = setupClusterStateListener(masterNode);
+
+        logger.info("--> write some file based settings, putting some reserved state");
+        writeJSONFile(masterNode, testFileSettingsJSON);
+        final ClusterStateResponse savedStateResponse = assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2());
+        assertThat(
+            savedStateResponse.getState().metadata().persistentSettings().get(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey()),
+            equalTo("50mb")
+        );
+
+        logger.info("--> create full snapshot");
+        createFullSnapshot("test-repo", "test-snap");
+        assertThat(getSnapshot("test-repo", "test-snap").state(), equalTo(SnapshotState.SUCCESS));
+
+        assertAcked(
+            clusterAdmin().prepareUpdateSettings()
+                .setPersistentSettings(
+                    Settings.builder()
+                        .put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey(), TimeValue.timeValueSeconds(55))
+                        .build()
+                )
+        );
+
+        logger.info("--> restore global state from the snapshot");
+        var removedReservedState = removedReservedClusterStateListener(masterNode);
+
+        clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap").setRestoreGlobalState(true).setWaitForCompletion(true).get();
+
+        ensureGreen();
+
+        // When the target cluster of a restore has an existing operator file, we don't un-reserve the reserved
+        // cluster state for file based settings, but instead we reset the version to 0 and 'touch' the operator file
+        // so that it gets re-processed.
+        logger.info("--> reserved state version will be reset to 0, because of snapshot restore");
+        // double timeout, we restore snapshot then apply the file
+        assertTrue(removedReservedState.v1().await(40, TimeUnit.SECONDS));
+
+        logger.info("--> reserved state would be restored to non-zero version");
+
+        final ClusterStateResponse clusterStateResponse = clusterAdmin().state(
+            new ClusterStateRequest().metadata(true).waitForMetadataVersion(removedReservedState.v2().get())
+        ).actionGet();
+
+        assertNotNull(clusterStateResponse.getState().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE));
+
+        final ClusterGetSettingsAction.Response getSettingsResponse = clusterAdmin().execute(
+            ClusterGetSettingsAction.INSTANCE,
+            new ClusterGetSettingsAction.Request()
+        ).actionGet();
+
+        assertThat(
+            getSettingsResponse.persistentSettings().get(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey()),
+            equalTo("25s")
+        );
+
+        // we need to remove the reserved state, so that clean-up can happen
+        var cleanupReservedState = cleanedClusterStateListener(masterNode);
+
+        logger.info("--> clear the file based settings");
+        writeJSONFile(masterNode, emptyFileSettingsJSON);
+        assertClusterStateSaveOK(cleanupReservedState.v1(), cleanupReservedState.v2());
+        // cleanup
+        assertAcked(
+            clusterAdmin().prepareUpdateSettings()
+                .setPersistentSettings(
+                    Settings.builder()
+                        .put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey(), (String) null)
+                        .put("indices.recovery.max_bytes_per_sec", (String) null)
+                        .build()
+                )
+        );
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/script/StoredScriptsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/script/StoredScriptsIT.java
index f9c245d20a34..0e3565b63d7f 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/script/StoredScriptsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/script/StoredScriptsIT.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.xcontent.XContentType;
@@ -40,7 +41,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     }
 
     public void testBasics() {
-        assertAcked(client().admin().cluster().preparePutStoredScript().setId("foobar").setContent(new BytesArray(formatted("""
+        assertAcked(client().admin().cluster().preparePutStoredScript().setId("foobar").setContent(new BytesArray(Strings.format("""
             {"script": {"lang": "%s", "source": "1"} }
             """, LANG)), XContentType.JSON));
         String script = client().admin().cluster().prepareGetStoredScript("foobar").get().getSource().getSource();
@@ -53,9 +54,9 @@ public void testBasics() {
 
         IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
-            () -> client().admin().cluster().preparePutStoredScript().setId("id#").setContent(new BytesArray(formatted("""
+            () -> { client().admin().cluster().preparePutStoredScript().setId("id#").setContent(new BytesArray(Strings.format("""
                 {"script": {"lang": "%s", "source": "1"} }
-                """, LANG)), XContentType.JSON).get()
+                """, LANG)), XContentType.JSON).get(); }
         );
         assertEquals("Validation Failed: 1: id cannot contain '#' for stored script;", e.getMessage());
     }
@@ -63,9 +64,9 @@ public void testBasics() {
     public void testMaxScriptSize() {
         IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
-            () -> client().admin().cluster().preparePutStoredScript().setId("foobar").setContent(new BytesArray(formatted("""
+            () -> { client().admin().cluster().preparePutStoredScript().setId("foobar").setContent(new BytesArray(Strings.format("""
                 {"script": { "lang": "%s", "source":"0123456789abcdef"} }\
-                """, LANG)), XContentType.JSON).get()
+                """, LANG)), XContentType.JSON).get(); }
         );
         assertEquals("exceeded max allowed stored script size in bytes [64] with size [65] for script [foobar]", e.getMessage());
     }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/MetadataIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/MetadataIT.java
index 95f85226c271..653a110f6116 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/MetadataIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/MetadataIT.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.search.aggregations.pipeline.InternalBucketMetricValue;
 import org.elasticsearch.test.ESIntegTestCase;
 
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -24,7 +23,6 @@
 import static org.elasticsearch.search.aggregations.PipelineAggregatorBuilders.maxBucket;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
-import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 
 public class MetadataIT extends ESIntegTestCase {
 
@@ -33,26 +31,13 @@ public void testMetadataSetOnAggregationResult() throws Exception {
         IndexRequestBuilder[] builders = new IndexRequestBuilder[randomInt(30)];
         for (int i = 0; i < builders.length; i++) {
             String name = "name_" + randomIntBetween(1, 10);
-            builders[i] = client().prepareIndex("idx")
-                .setSource(jsonBuilder().startObject().field("name", name).field("value", randomInt()).endObject());
+            builders[i] = client().prepareIndex("idx").setSource("name", name, "value", randomInt());
         }
         indexRandom(true, builders);
         ensureSearchable();
 
-        final Map<String, Object> nestedMetadata = new HashMap<String, Object>() {
-            {
-                put("nested", "value");
-            }
-        };
-
-        Map<String, Object> metadata = new HashMap<String, Object>() {
-            {
-                put("key", "value");
-                put("numeric", 1.2);
-                put("bool", true);
-                put("complex", nestedMetadata);
-            }
-        };
+        final var nestedMetadata = Map.of("nested", "value");
+        var metadata = Map.of("key", "value", "numeric", 1.2, "bool", true, "complex", nestedMetadata);
 
         SearchResponse response = client().prepareSearch("idx")
             .addAggregation(
@@ -95,8 +80,7 @@ private void assertMetadata(Map<String, Object> returnedMetadata) {
         Object nestedObject = returnedMetadata.get("complex");
         assertNotNull(nestedObject);
 
-        @SuppressWarnings("unchecked")
-        Map<String, Object> nestedMap = (Map<String, Object>) nestedObject;
+        Map<?, ?> nestedMap = (Map<?, ?>) nestedObject;
         assertEquals("value", nestedMap.get("nested"));
     }
 }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java
index 6272b29c068d..b17e12190405 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
@@ -520,7 +521,7 @@ public void testPartiallyUnmappedWithFormat() throws Exception {
         assertThat(terms.getBuckets().size(), equalTo(5));
 
         for (int i = 0; i < 5; i++) {
-            String key = formatted("%07.2f", (double) i);
+            String key = Strings.format("%07.2f", (double) i);
             DoubleTerms.Bucket bucket = terms.getBucketByKey(key);
             assertThat(bucket, notNullValue());
             assertThat(bucket.getKeyAsString(), equalTo(key));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java
index 81f416a48802..122168737c96 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
@@ -510,7 +511,7 @@ public void testPartiallyUnmappedWithFormat() throws Exception {
         assertThat(terms.getBuckets().size(), equalTo(5));
 
         for (int i = 0; i < 5; i++) {
-            String key = formatted("%04d", i);
+            String key = Strings.format("%04d", i);
             LongTerms.Bucket bucket = terms.getBucketByKey(key);
             assertThat(bucket, notNullValue());
             assertThat(bucket.getKeyAsString(), equalTo(key));
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
index c9009b848456..b03991c0af5e 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
@@ -150,7 +150,8 @@ public void testXContentResponse() throws Exception {
         classes.toXContent(responseBuilder, ToXContent.EMPTY_PARAMS);
         responseBuilder.endObject();
 
-        String result = formatted("""
+        Object[] args = new Object[] { type.equals("long") ? "0" : "\"0\"", type.equals("long") ? "1" : "\"1\"" };
+        String result = Strings.format("""
             {
               "class": {
                 "doc_count_error_upper_bound": 0,
@@ -191,7 +192,7 @@ public void testXContentResponse() throws Exception {
                 ]
               }
             }
-            """, type.equals("long") ? "0" : "\"0\"", type.equals("long") ? "1" : "\"1\"");
+            """, args);
         assertThat(Strings.toString(responseBuilder), equalTo(XContentHelper.stripWhitespace(result)));
 
     }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java
index a486f553d2bf..89df04d455cb 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.search.aggregations.metrics;
 
 import org.apache.logging.log4j.LogManager;
+import org.apache.lucene.tests.util.LuceneTestCase;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
@@ -49,6 +50,7 @@
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
+@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/92822")
 public class HDRPercentilesIT extends AbstractNumericTestCase {
 
     @Override
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java
index 11f020672bf0..6efda3f11d7f 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
@@ -303,23 +304,29 @@ public void setupSuiteScopeCluster() throws Exception {
         // When using the MockScriptPlugin we can map Stored scripts to inline scripts:
         // the id of the stored script is used in test method while the source of the stored script
         // must match a predefined script from CustomScriptPlugin.pluginScripts() method
-        assertAcked(client().admin().cluster().preparePutStoredScript().setId("initScript_stored").setContent(new BytesArray(formatted("""
-            {"script": {"lang": "%s", "source": "vars.multiplier = 3"} }
-            """, MockScriptPlugin.NAME)), XContentType.JSON));
+        assertAcked(
+            client().admin().cluster().preparePutStoredScript().setId("initScript_stored").setContent(new BytesArray(Strings.format("""
+                {"script": {"lang": "%s", "source": "vars.multiplier = 3"} }
+                """, MockScriptPlugin.NAME)), XContentType.JSON)
+        );
 
-        assertAcked(client().admin().cluster().preparePutStoredScript().setId("mapScript_stored").setContent(new BytesArray(formatted("""
-            {"script": {"lang": "%s", "source": "state.list.add(vars.multiplier)"} }
-            """, MockScriptPlugin.NAME)), XContentType.JSON));
+        assertAcked(
+            client().admin().cluster().preparePutStoredScript().setId("mapScript_stored").setContent(new BytesArray(Strings.format("""
+                {"script": {"lang": "%s", "source": "state.list.add(vars.multiplier)"} }
+                """, MockScriptPlugin.NAME)), XContentType.JSON)
+        );
 
         assertAcked(
-            client().admin().cluster().preparePutStoredScript().setId("combineScript_stored").setContent(new BytesArray(formatted("""
+            client().admin().cluster().preparePutStoredScript().setId("combineScript_stored").setContent(new BytesArray(Strings.format("""
                 {"script": {"lang": "%s", "source": "sum state values as a new aggregation"} }
                 """, MockScriptPlugin.NAME)), XContentType.JSON)
         );
 
-        assertAcked(client().admin().cluster().preparePutStoredScript().setId("reduceScript_stored").setContent(new BytesArray(formatted("""
-            {"script": {"lang": "%s", "source": "sum all states (lists) values as a new aggregation"} }
-            """, MockScriptPlugin.NAME)), XContentType.JSON));
+        assertAcked(
+            client().admin().cluster().preparePutStoredScript().setId("reduceScript_stored").setContent(new BytesArray(Strings.format("""
+                {"script": {"lang": "%s", "source": "sum all states (lists) values as a new aggregation"} }
+                """, MockScriptPlugin.NAME)), XContentType.JSON)
+        );
 
         indexRandom(true, builders);
         ensureSearchable();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchRedStateIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchRedStateIndexIT.java
index b17a5df319e6..33db3ac4103a 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchRedStateIndexIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/basic/SearchRedStateIndexIT.java
@@ -29,6 +29,7 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.emptyArray;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.instanceOf;
@@ -67,6 +68,9 @@ public void testClusterAllowPartialsWithRedState() throws Exception {
         assertThat("Expected total shards", searchResponse.getTotalShards(), equalTo(numShards));
         for (ShardSearchFailure failure : searchResponse.getShardFailures()) {
             assertThat(failure.getCause(), instanceOf(NoShardAvailableActionException.class));
+            assertThat(failure.getCause().getStackTrace(), emptyArray());
+            // We don't write out the entire, repetitive stacktrace in the reason
+            assertThat(failure.reason(), equalTo("org.elasticsearch.action.NoShardAvailableActionException" + System.lineSeparator()));
         }
     }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java
index 6656e0aa1f93..c492dfe60b6e 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java
@@ -8,6 +8,9 @@
 
 package org.elasticsearch.search.fieldcaps;
 
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.fieldcaps.FieldCapabilities;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesAction;
@@ -17,6 +20,10 @@
 import org.elasticsearch.action.fieldcaps.TransportFieldCapabilitiesAction;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.client.Cancellable;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
@@ -31,6 +38,7 @@
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.SourceLoader;
+import org.elasticsearch.index.mapper.StringStoredFieldFieldLoader;
 import org.elasticsearch.index.mapper.TimeSeriesParams;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -43,14 +51,18 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.search.DummyQueryBuilder;
+import org.elasticsearch.tasks.TaskInfo;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParser;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -58,6 +70,9 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -65,12 +80,14 @@
 import java.util.stream.IntStream;
 
 import static java.util.Collections.singletonList;
+import static org.elasticsearch.action.support.ActionTestUtils.wrapAsRestResponseListener;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.aMapWithSize;
 import static org.hamcrest.Matchers.array;
 import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.not;
@@ -103,6 +120,7 @@ public void setUp() throws Exception {
             .endObject()
             .startObject("playlist")
             .field("type", "text")
+            .field("store", true)
             .endObject()
             .startObject("some_dimension")
             .field("type", "keyword")
@@ -110,7 +128,7 @@ public void setUp() throws Exception {
             .endObject()
             .startObject("some_metric")
             .field("type", "long")
-            .field("time_series_metric", TimeSeriesParams.MetricType.counter)
+            .field("time_series_metric", TimeSeriesParams.MetricType.COUNTER)
             .endObject()
             .startObject("secret_soundtrack")
             .field("type", "alias")
@@ -153,7 +171,7 @@ public void setUp() throws Exception {
             .endObject()
             .startObject("some_metric")
             .field("type", "long")
-            .field("time_series_metric", TimeSeriesParams.MetricType.gauge)
+            .field("time_series_metric", TimeSeriesParams.MetricType.GAUGE)
             .endObject()
             .endObject()
             .endObject()
@@ -164,7 +182,17 @@ public void setUp() throws Exception {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return List.of(TestMapperPlugin.class, ExceptionOnRewriteQueryPlugin.class);
+        return List.of(TestMapperPlugin.class, ExceptionOnRewriteQueryPlugin.class, BlockingOnRewriteQueryPlugin.class);
+    }
+
+    @Override
+    protected boolean addMockHttpTransport() {
+        return false; // enable http
+    }
+
+    @Override
+    protected boolean ignoreExternalCluster() {
+        return true;
     }
 
     public void testFieldAlias() {
@@ -368,7 +396,7 @@ public void testFieldMetricsAndDimensions() {
         assertTrue(response.get().get("some_dimension").get("keyword").isDimension());
         assertNull(response.get().get("some_dimension").get("keyword").nonDimensionIndices());
         assertTrue(response.get().containsKey("some_metric"));
-        assertEquals(TimeSeriesParams.MetricType.counter, response.get().get("some_metric").get("long").getMetricType());
+        assertEquals(TimeSeriesParams.MetricType.COUNTER, response.get().get("some_metric").get("long").getMetricType());
         assertNull(response.get().get("some_metric").get("long").metricConflictsIndices());
 
         response = client().prepareFieldCaps("old_index", "new_index").setFields("some_dimension", "some_metric").get();
@@ -641,6 +669,52 @@ public void testManyIndicesWithSameMapping() {
         assertTrue(resp.getField("extra_field").get("integer").isAggregatable());
     }
 
+    public void testCancel() throws Exception {
+        BlockingOnRewriteQueryBuilder.blockOnRewrite();
+        PlainActionFuture<Response> future = PlainActionFuture.newFuture();
+        Request restRequest = new Request("POST", "/_field_caps?fields=*");
+        restRequest.setEntity(new StringEntity("""
+                  {
+                    "index_filter": {
+                        "blocking_query": {}
+                     }
+                  }
+            """, ContentType.APPLICATION_JSON.withCharset(StandardCharsets.UTF_8)));
+        Cancellable cancellable = getRestClient().performRequestAsync(restRequest, wrapAsRestResponseListener(future));
+        logger.info("--> waiting for field-caps tasks to be started");
+        assertBusy(() -> {
+            List<TaskInfo> tasks = client().admin()
+                .cluster()
+                .prepareListTasks()
+                .setActions("indices:data/read/field_caps", "indices:data/read/field_caps[n]")
+                .get()
+                .getTasks();
+            assertThat(tasks.size(), greaterThanOrEqualTo(2));
+            for (TaskInfo task : tasks) {
+                assertTrue(task.cancellable());
+                assertFalse(task.cancelled());
+            }
+        }, 30, TimeUnit.SECONDS);
+
+        cancellable.cancel();
+        logger.info("--> waiting for field-caps tasks to be cancelled");
+        assertBusy(() -> {
+            List<TaskInfo> tasks = client().admin()
+                .cluster()
+                .prepareListTasks()
+                .setActions("indices:data/read/field_caps", "indices:data/read/field_caps[n]")
+                .get()
+                .getTasks();
+            for (TaskInfo task : tasks) {
+                assertTrue(task.cancellable());
+                assertTrue(task.cancelled());
+            }
+        }, 30, TimeUnit.SECONDS);
+
+        BlockingOnRewriteQueryBuilder.unblockOnRewrite();
+        expectThrows(CancellationException.class, future::actionGet);
+    }
+
     private void assertIndices(FieldCapabilitiesResponse response, String... indices) {
         assertNotNull(response.getIndices());
         Arrays.sort(indices);
@@ -680,7 +754,6 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws
                 if (searchExecutionContext.indexMatches("*error*")) {
                     throw new IllegalArgumentException("I throw because I choose to.");
                 }
-                ;
             }
             return this;
         }
@@ -691,6 +764,60 @@ public String getWriteableName() {
         }
     }
 
+    public static class BlockingOnRewriteQueryPlugin extends Plugin implements SearchPlugin {
+
+        public BlockingOnRewriteQueryPlugin() {}
+
+        @Override
+        public List<QuerySpec<?>> getQueries() {
+            return List.of(
+                new QuerySpec<>("blocking_query", BlockingOnRewriteQueryBuilder::new, BlockingOnRewriteQueryBuilder::fromXContent)
+            );
+        }
+    }
+
+    static class BlockingOnRewriteQueryBuilder extends DummyQueryBuilder {
+        private static CountDownLatch blockingLatch = new CountDownLatch(1);
+        public static final String NAME = "blocking_query";
+
+        BlockingOnRewriteQueryBuilder() {
+
+        }
+
+        BlockingOnRewriteQueryBuilder(StreamInput in) throws IOException {
+            super(in);
+        }
+
+        static void blockOnRewrite() {
+            blockingLatch = new CountDownLatch(1);
+        }
+
+        static void unblockOnRewrite() {
+            blockingLatch.countDown();
+        }
+
+        @Override
+        protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+            try {
+                blockingLatch.await();
+            } catch (InterruptedException e) {
+                throw new AssertionError(e);
+            }
+            return this;
+        }
+
+        public static BlockingOnRewriteQueryBuilder fromXContent(XContentParser parser) {
+            ObjectParser<BlockingOnRewriteQueryBuilder, Void> objectParser = new ObjectParser<>(NAME, BlockingOnRewriteQueryBuilder::new);
+            declareStandardFields(objectParser);
+            return objectParser.apply(parser, null);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return NAME;
+        }
+    }
+
     public static final class TestMapperPlugin extends Plugin implements MapperPlugin {
         @Override
         public Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers() {
@@ -721,7 +848,13 @@ protected String contentType() {
 
         @Override
         public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
-            throw new UnsupportedOperationException();
+            return new StringStoredFieldFieldLoader(name(), simpleName(), null) {
+                @Override
+                protected void write(XContentBuilder b, Object value) throws IOException {
+                    BytesRef ref = (BytesRef) value;
+                    b.utf8Value(ref.bytes, ref.offset, ref.length);
+                }
+            };
         }
 
         private static final TypeParser PARSER = new FixedTypeParser(c -> new TestMetadataMapper());
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java
index 8a4761396d05..225d3218db23 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fields/SearchFieldsIT.java
@@ -28,7 +28,7 @@
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.lookup.FieldLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -53,6 +53,7 @@
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 import static java.util.Collections.singleton;
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
@@ -142,7 +143,7 @@ static Object fieldsScript(Map<String, Object> vars, String fieldName) {
 
         static Object sourceScript(Map<String, Object> vars, String path) {
             @SuppressWarnings("unchecked")
-            Map<String, Object> source = ((SourceLookup) vars.get("_source")).source();
+            Map<String, Object> source = ((Supplier<Source>) vars.get("_source")).get().source();
             return XContentMapValues.extractValue(path, source);
         }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
index e7e538e47fa3..85525e831da3 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery.ScoreMode;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
@@ -771,27 +772,27 @@ public void testDateWithoutOrigin() throws Exception {
         ZonedDateTime docDate = dt.minusDays(1);
         String docDateString = docDate.getYear()
             + "-"
-            + formatted("%02d", docDate.getMonthValue())
+            + Strings.format("%02d", docDate.getMonthValue())
             + "-"
-            + formatted("%02d", docDate.getDayOfMonth());
+            + Strings.format("%02d", docDate.getDayOfMonth());
         client().index(
             indexRequest("test").id("1").source(jsonBuilder().startObject().field("test", "value").field("num1", docDateString).endObject())
         ).actionGet();
         docDate = dt.minusDays(2);
         docDateString = docDate.getYear()
             + "-"
-            + formatted("%02d", docDate.getMonthValue())
+            + Strings.format("%02d", docDate.getMonthValue())
             + "-"
-            + formatted("%02d", docDate.getDayOfMonth());
+            + Strings.format("%02d", docDate.getDayOfMonth());
         client().index(
             indexRequest("test").id("2").source(jsonBuilder().startObject().field("test", "value").field("num1", docDateString).endObject())
         ).actionGet();
         docDate = dt.minusDays(3);
         docDateString = docDate.getYear()
             + "-"
-            + formatted("%02d", docDate.getMonthValue())
+            + Strings.format("%02d", docDate.getMonthValue())
             + "-"
-            + formatted("%02d", docDate.getDayOfMonth());
+            + Strings.format("%02d", docDate.getDayOfMonth());
         client().index(
             indexRequest("test").id("3").source(jsonBuilder().startObject().field("test", "value").field("num1", docDateString).endObject())
         ).actionGet();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java
index 2c443924d8b4..ea6f45a21caf 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.search.functionscore;
 
 import org.apache.lucene.search.Explanation;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
@@ -136,8 +136,8 @@ public DecayFunction getDecayFunction() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.V_EMPTY;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.ZERO;
         }
 
         private static final DecayFunction decayFunction = new LinearMultScoreFunction();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/msearch/MultiSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/msearch/MultiSearchIT.java
index ecca33f08d9d..47f3e0adf04f 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/msearch/MultiSearchIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/msearch/MultiSearchIT.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.msearch;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.search.MultiSearchRequest;
 import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.common.settings.Settings;
@@ -96,8 +96,8 @@ public void testCCSCheckCompatibility() throws Exception {
             .add(client().prepareSearch("test").setQuery(QueryBuilders.termQuery("field", "yyy")))
             .add(client().prepareSearch("test").setQuery(new DummyQueryBuilder() {
                 @Override
-                public Version getMinimalSupportedVersion() {
-                    return Version.CURRENT;
+                public TransportVersion getMinimalSupportedVersion() {
+                    return TransportVersion.CURRENT;
                 }
             }))
             .get();
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java
index cc14f36d02c9..f63ce82d3490 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java
@@ -46,6 +46,7 @@
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
 
 @ESIntegTestCase.SuiteScopeTestCase
@@ -690,7 +691,7 @@ public void testFilterByFilter() throws InterruptedException, IOException {
                             .entry("delegate", "FilterByFilterAggregator")
                             .entry(
                                 "delegate_debug",
-                                matchesMap().entry("segments_with_deleted_docs", 0)
+                                matchesMap().entry("segments_with_deleted_docs", greaterThanOrEqualTo(0))
                                     .entry("segments_with_doc_count_field", 0)
                                     .entry("segments_counted", 0)
                                     .entry("segments_collected", greaterThan(0))
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/profile/dfs/DfsProfilerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/dfs/DfsProfilerIT.java
new file mode 100644
index 000000000000..8b4d986f224d
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/profile/dfs/DfsProfilerIT.java
@@ -0,0 +1,139 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.profile.dfs;
+
+import org.apache.lucene.tests.util.English;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.search.profile.ProfileResult;
+import org.elasticsearch.search.profile.SearchProfileDfsPhaseResult;
+import org.elasticsearch.search.profile.SearchProfileShardResult;
+import org.elasticsearch.search.profile.query.CollectorResult;
+import org.elasticsearch.search.profile.query.QueryProfileShardResult;
+import org.elasticsearch.search.vectors.KnnSearchBuilder;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xcontent.XContentFactory;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.search.profile.query.RandomQueryGenerator.randomQueryBuilder;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.emptyOrNullString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class DfsProfilerIT extends ESIntegTestCase {
+
+    private static final int KNN_DIM = 3;
+
+    public void testProfileDfs() throws Exception {
+        String textField = "text_field";
+        String numericField = "number";
+        String vectorField = "vector";
+        String indexName = "text-dfs-profile";
+        createIndex(indexName, vectorField);
+        ensureGreen();
+
+        int numDocs = randomIntBetween(10, 50);
+        IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs];
+        for (int i = 0; i < numDocs; i++) {
+            docs[i] = client().prepareIndex(indexName)
+                .setId(String.valueOf(i))
+                .setSource(
+                    textField,
+                    English.intToEnglish(i),
+                    numericField,
+                    i,
+                    vectorField,
+                    new float[] { randomFloat(), randomFloat(), randomFloat() }
+                );
+        }
+        indexRandom(true, docs);
+        refresh();
+        int iters = between(5, 10);
+        for (int i = 0; i < iters; i++) {
+            QueryBuilder q = randomQueryBuilder(List.of(textField), List.of(numericField), numDocs, 3);
+            logger.info("Query: {}", q);
+            SearchResponse resp = client().prepareSearch()
+                .setQuery(q)
+                .setTrackTotalHits(true)
+                .setProfile(true)
+                .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
+                .setKnnSearch(
+                    randomList(
+                        2,
+                        5,
+                        () -> new KnnSearchBuilder(
+                            vectorField,
+                            new float[] { randomFloat(), randomFloat(), randomFloat() },
+                            randomIntBetween(5, 10),
+                            50
+                        )
+                    )
+                )
+                .get();
+
+            assertNotNull("Profile response element should not be null", resp.getProfileResults());
+            assertThat("Profile response should not be an empty array", resp.getProfileResults().size(), not(0));
+            for (Map.Entry<String, SearchProfileShardResult> shard : resp.getProfileResults().entrySet()) {
+                for (QueryProfileShardResult searchProfiles : shard.getValue().getQueryProfileResults()) {
+                    for (ProfileResult result : searchProfiles.getQueryResults()) {
+                        assertNotNull(result.getQueryName());
+                        assertNotNull(result.getLuceneDescription());
+                        assertThat(result.getTime(), greaterThan(0L));
+                    }
+                    CollectorResult result = searchProfiles.getCollectorResult();
+                    assertThat(result.getName(), is(not(emptyOrNullString())));
+                    assertThat(result.getTime(), greaterThan(0L));
+                }
+                SearchProfileDfsPhaseResult searchProfileDfsPhaseResult = shard.getValue().getSearchProfileDfsPhaseResult();
+                assertThat(searchProfileDfsPhaseResult, is(notNullValue()));
+                for (QueryProfileShardResult queryProfileShardResult : searchProfileDfsPhaseResult.getQueryProfileShardResult()) {
+                    for (ProfileResult result : queryProfileShardResult.getQueryResults()) {
+                        assertNotNull(result.getQueryName());
+                        assertNotNull(result.getLuceneDescription());
+                        assertThat(result.getTime(), greaterThan(0L));
+                    }
+                    CollectorResult result = queryProfileShardResult.getCollectorResult();
+                    assertThat(result.getName(), is(not(emptyOrNullString())));
+                    assertThat(result.getTime(), greaterThan(0L));
+                    assertThat(result.getTime(), greaterThan(0L));
+                }
+                ProfileResult statsResult = searchProfileDfsPhaseResult.getDfsShardResult();
+                assertThat(statsResult.getQueryName(), equalTo("statistics"));
+            }
+        }
+    }
+
+    private void createIndex(String name, String vectorField) throws IOException {
+        assertAcked(
+            prepareCreate(name).setMapping(
+                XContentFactory.jsonBuilder()
+                    .startObject()
+                    .startObject("properties")
+                    .startObject(vectorField)
+                    .field("type", "dense_vector")
+                    .field("dims", KNN_DIM)
+                    .field("index", true)
+                    .field("similarity", "cosine")
+                    .endObject()
+                    .endObject()
+                    .endObject()
+            )
+        );
+    }
+
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java
index 90b09bcec27e..c8d1eaf8425f 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
@@ -166,7 +167,7 @@ public void testIssue6614() throws ExecutionException, InterruptedException {
                             "foo",
                             "bar",
                             "timeUpdated",
-                            "2014/07/" + formatted("%02d", i + 1) + " " + formatted("%02d", j + 1) + ":00:00"
+                            "2014/07/" + Strings.format("%02d", i + 1) + " " + Strings.format("%02d", j + 1) + ":00:00"
                         )
                 );
             }
@@ -191,7 +192,7 @@ public void testIssue6614() throws ExecutionException, InterruptedException {
                 .setQuery(
                     QueryBuilders.boolQuery()
                         .must(QueryBuilders.termQuery("foo", "bar"))
-                        .must(QueryBuilders.rangeQuery("timeUpdated").gte("2014/" + formatted("%02d", randomIntBetween(1, 7)) + "/01"))
+                        .must(QueryBuilders.rangeQuery("timeUpdated").gte("2014/" + Strings.format("%02d", randomIntBetween(1, 7)) + "/01"))
                 )
                 .addSort(new FieldSortBuilder("timeUpdated").order(SortOrder.ASC).unmappedType("date"))
                 .setSize(scaledRandomIntBetween(1, docs))
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java
index 2273d46caf82..0b2ffae14377 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java
@@ -25,7 +25,7 @@
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.Collection;
@@ -34,6 +34,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
@@ -60,9 +61,10 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     public static class CustomScriptPlugin extends MockScriptPlugin {
 
         @Override
+        @SuppressWarnings("unchecked")
         protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
             return Collections.singletonMap("_source.field", vars -> {
-                Map<?, ?> src = ((SourceLookup) vars.get("_source")).source();
+                Map<?, ?> src = ((Supplier<Source>) vars.get("_source")).get().source();
                 return src.get("field");
             });
         }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
index 439401383e84..ed9e3ea9c30a 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.common.FieldMemoryStats;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
@@ -1248,7 +1249,7 @@ private static void assertSuggestions(
             }
         } else {
             for (String expectedSuggestion : suggestions) {
-                String errMsg = String.format(Locale.ROOT, "Expected elem %s to be in list %s", expectedSuggestion, suggestionList);
+                String errMsg = Strings.format("Expected elem %s to be in list %s", expectedSuggestion, suggestionList);
                 assertThat(errMsg, suggestionList, hasItem(expectedSuggestion));
             }
         }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java
index 9c656109dfd2..28c4b85df389 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java
@@ -219,7 +219,7 @@ public void testDeletesAreBatched() throws Exception {
 
         logger.info("--> waiting for batched deletes to finish");
         final PlainActionFuture<Collection<AcknowledgedResponse>> allDeletesDone = new PlainActionFuture<>();
-        final ActionListener<AcknowledgedResponse> deletesListener = new GroupedActionListener<>(allDeletesDone, deleteFutures.size());
+        final ActionListener<AcknowledgedResponse> deletesListener = new GroupedActionListener<>(deleteFutures.size(), allDeletesDone);
         for (StepListener<AcknowledgedResponse> deleteFuture : deleteFutures) {
             deleteFuture.addListener(deletesListener);
         }
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java
index 696e0de5d557..3b5df51f19d6 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.IndexMetaDataGenerations;
 import org.elasticsearch.repositories.Repository;
@@ -39,7 +40,6 @@
 import java.nio.file.StandardOpenOption;
 import java.util.Collections;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -289,7 +289,7 @@ public void testHandlingMissingRootLevelSnapshotMetadata() throws Exception {
 
         final SnapshotId snapshotToCorrupt = randomFrom(repositoryData.getSnapshotIds());
         logger.info("--> delete root level snapshot metadata blob for snapshot [{}]", snapshotToCorrupt);
-        Files.delete(repo.resolve(String.format(Locale.ROOT, BlobStoreRepository.SNAPSHOT_NAME_FORMAT, snapshotToCorrupt.getUUID())));
+        Files.delete(repo.resolve(Strings.format(BlobStoreRepository.SNAPSHOT_NAME_FORMAT, snapshotToCorrupt.getUUID())));
 
         logger.info("--> strip version information from index-N blob");
         final RepositoryData withoutVersions = new RepositoryData(
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CustomMetadataContextIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CustomMetadataContextIT.java
index 266ef7d74f54..5c075966499e 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CustomMetadataContextIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CustomMetadataContextIT.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.snapshots;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -215,8 +215,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT;
         }
 
         @Override
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
index 964b7ea97632..85c453ad6be3 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
@@ -51,6 +51,7 @@
 import org.elasticsearch.rest.AbstractRestChannel;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestResponseUtils;
 import org.elasticsearch.rest.action.admin.cluster.RestClusterStateAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetRepositoriesAction;
 import org.elasticsearch.snapshots.mockstore.MockRepository;
@@ -525,8 +526,9 @@ public void sendResponse(RestResponse response) {
             @Override
             public void sendResponse(RestResponse response) {
                 try {
-                    assertThat(response.content().utf8ToString(), containsString("notsecretusername"));
-                    assertThat(response.content().utf8ToString(), not(containsString("verysecretpassword")));
+                    final var responseBody = RestResponseUtils.getBodyContent(response).utf8ToString();
+                    assertThat(responseBody, containsString("notsecretusername"));
+                    assertThat(responseBody, not(containsString("verysecretpassword")));
                 } catch (AssertionError ex) {
                     clusterStateError.set(ex);
                 } finally {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java
index f36e8e8e6e90..91c6ea3c5b92 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
 import org.elasticsearch.cluster.SnapshotsInProgress;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.repositories.RepositoryMissingException;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -29,8 +30,10 @@
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.in;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 
 public class GetSnapshotsIT extends AbstractSnapshotIntegTestCase {
@@ -578,6 +581,23 @@ public void testSortAfter() throws Exception {
         assertThat(paginatedResponse2.totalCount(), is(3));
     }
 
+    public void testRetrievingSnapshotsWhenRepositoryIsMissing() throws Exception {
+        final String repoName = "test-repo";
+        final Path repoPath = randomRepoPath();
+        createRepository(repoName, "fs", repoPath);
+        final String missingRepoName = "missing";
+
+        final List<String> snapshotNames = createNSnapshots(repoName, randomIntBetween(1, 10));
+        snapshotNames.sort(String::compareTo);
+
+        final GetSnapshotsResponse response = clusterAdmin().prepareGetSnapshots(repoName, missingRepoName)
+            .setSort(GetSnapshotsRequest.SortBy.NAME)
+            .get();
+        assertThat(response.getSnapshots().stream().map(info -> info.snapshotId().getName()).toList(), equalTo(snapshotNames));
+        assertTrue(response.getFailures().containsKey(missingRepoName));
+        assertThat(response.getFailures().get(missingRepoName), instanceOf(RepositoryMissingException.class));
+    }
+
     // Create a snapshot that is guaranteed to have a unique start time and duration for tests around ordering by either.
     // Don't use this with more than 3 snapshots on platforms with low-resolution clocks as the durations could always collide there
     // causing an infinite loop
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java
index 771cb87e34f3..5bc943fcc8be 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
 import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotAction;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
@@ -274,6 +275,18 @@ public void testRepositoryConflict() throws Exception {
         logger.info("--> waiting for block to kick in on node [{}]", blockedNode);
         waitForBlock(blockedNode, repo);
 
+        assertTrue(
+            client().admin()
+                .cluster()
+                .prepareListTasks()
+                .setActions(DeleteSnapshotAction.NAME)
+                .setDetailed(true)
+                .get()
+                .getTasks()
+                .stream()
+                .anyMatch(ti -> ("[" + repo + "][" + snapshot1 + "]").equals(ti.description()))
+        );
+
         logger.info("--> try deleting the repository, should fail because the deletion of the snapshot is in progress");
         RepositoryConflictException e1 = expectThrows(
             RepositoryConflictException.class,
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java
deleted file mode 100644
index d89c38f323a8..000000000000
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryFilterUserMetadataIT.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.snapshots;
-
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.indices.recovery.RecoverySettings;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.RepositoryPlugin;
-import org.elasticsearch.repositories.FinalizeSnapshotContext;
-import org.elasticsearch.repositories.Repository;
-import org.elasticsearch.repositories.SnapshotShardContext;
-import org.elasticsearch.repositories.fs.FsRepository;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.xcontent.NamedXContentRegistry;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.hamcrest.Matchers.is;
-
-public class RepositoryFilterUserMetadataIT extends ESIntegTestCase {
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(MetadataFilteringPlugin.class);
-    }
-
-    public void testFilteredRepoMetadataIsUsed() {
-        final String masterName = internalCluster().getMasterName();
-        final String repoName = "test-repo";
-        assertAcked(
-            client().admin()
-                .cluster()
-                .preparePutRepository(repoName)
-                .setType(MetadataFilteringPlugin.TYPE)
-                .setSettings(
-                    Settings.builder().put("location", randomRepoPath()).put(MetadataFilteringPlugin.MASTER_SETTING_VALUE, masterName)
-                )
-        );
-        createIndex("test-idx");
-        final SnapshotInfo snapshotInfo = client().admin()
-            .cluster()
-            .prepareCreateSnapshot(repoName, "test-snap")
-            .setWaitForCompletion(true)
-            .get()
-            .getSnapshotInfo();
-        assertThat(snapshotInfo.userMetadata(), is(Collections.singletonMap(MetadataFilteringPlugin.MOCK_FILTERED_META, masterName)));
-    }
-
-    // Mock plugin that stores the name of the master node that started a snapshot in each snapshot's metadata
-    public static final class MetadataFilteringPlugin extends org.elasticsearch.plugins.Plugin implements RepositoryPlugin {
-
-        private static final String MOCK_FILTERED_META = "mock_filtered_meta";
-
-        private static final String MASTER_SETTING_VALUE = "initial_master";
-
-        private static final String TYPE = "mock_meta_filtering";
-
-        @Override
-        public Map<String, Repository.Factory> getRepositories(
-            Environment env,
-            NamedXContentRegistry namedXContentRegistry,
-            ClusterService clusterService,
-            BigArrays bigArrays,
-            RecoverySettings recoverySettings
-        ) {
-            return Collections.singletonMap(
-                "mock_meta_filtering",
-                metadata -> new FsRepository(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings) {
-
-                    // Storing the initially expected metadata value here to verify that #filterUserMetadata is only called once on the
-                    // initial master node starting the snapshot
-                    private final String initialMetaValue = metadata.settings().get(MASTER_SETTING_VALUE);
-
-                    @Override
-                    public void finalizeSnapshot(FinalizeSnapshotContext finalizeSnapshotContext) {
-                        super.finalizeSnapshot(finalizeSnapshotContext);
-                    }
-
-                    @Override
-                    public void snapshotShard(SnapshotShardContext context) {
-                        assertThat(context.userMetadata(), is(Collections.singletonMap(MOCK_FILTERED_META, initialMetaValue)));
-                        super.snapshotShard(context);
-                    }
-
-                    @Override
-                    public Map<String, Object> adaptUserMetadata(Map<String, Object> userMetadata) {
-                        return Collections.singletonMap(MOCK_FILTERED_META, clusterService.getNodeName());
-                    }
-                }
-            );
-        }
-    }
-}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceIT.java
index 36ecee336712..80b9c437a5dd 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceIT.java
@@ -67,7 +67,7 @@ public void testRepositoryIntegrityHealthIndicator() throws IOException, Interru
     }
 
     private void assertSnapshotRepositoryHealth(String message, Client client, HealthStatus status) {
-        var response = client.execute(GetHealthAction.INSTANCE, new GetHealthAction.Request(randomBoolean())).actionGet();
+        var response = client.execute(GetHealthAction.INSTANCE, new GetHealthAction.Request(randomBoolean(), 1000)).actionGet();
         assertThat(message, response.findIndicator(NAME).status(), equalTo(status));
     }
 
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java
index 131ace6a3364..f07057c496ae 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java
@@ -661,8 +661,8 @@ public void testConcurrentCreateAndStatusAPICalls() throws Exception {
         final var waitForCompletion = randomBoolean();
         final var createsListener = new PlainActionFuture<Void>();
         final var createsGroupedListener = new GroupedActionListener<CreateSnapshotResponse>(
-            createsListener.map(ignored -> null),
-            snapshotNames.length
+            snapshotNames.length,
+            createsListener.map(ignored -> null)
         );
         for (final var snapshotName : snapshotNames) {
             clusterAdmin().prepareCreateSnapshot(repoName, snapshotName)
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/transport/RemoteClusterPortSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/transport/RemoteClusterPortSettingsIT.java
new file mode 100644
index 000000000000..2543bcdfc5f6
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/elasticsearch/transport/RemoteClusterPortSettingsIT.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.transport;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import static org.elasticsearch.test.NodeRoles.dataOnlyNode;
+import static org.elasticsearch.test.NodeRoles.masterNode;
+import static org.hamcrest.Matchers.containsString;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
+public class RemoteClusterPortSettingsIT extends ESIntegTestCase {
+
+    public void testDirectlyConfiguringTransportProfileForRemoteClusterWillFailToStartTheNode() {
+        assumeTrue("untrusted remote cluster feature flag must be enabled", TcpTransport.isUntrustedRemoteClusterEnabled());
+
+        internalCluster().setBootstrapMasterNodeIndex(0);
+
+        final Settings.Builder builder = Settings.builder()
+            .put(randomBoolean() ? masterNode() : dataOnlyNode())
+            .put("discovery.initial_state_timeout", "1s")
+            .put("remote_cluster.enabled", true);
+
+        // Test that the same error message is always reported for direct usage of the _remote_cluster profile
+        switch (randomIntBetween(0, 2)) {
+            case 0 -> builder.put("transport.profiles._remote_cluster.tcp.keep_alive", true);
+            case 1 -> builder.put("transport.profiles._remote_cluster.port", 9900);
+            default -> builder.put("transport.profiles._remote_cluster.port", 9900)
+                .put("transport.profiles._remote_cluster.tcp.keep_alive", true);
+        }
+
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> internalCluster().startNode(builder));
+        assertThat(
+            e.getMessage(),
+            containsString(
+                "Remote Access settings should not be configured using the [_remote_cluster] profile. "
+                    + "Use the [remote_cluster.] settings instead."
+            )
+        );
+    }
+
+}
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/versioning/SimpleVersioningIT.java b/server/src/internalClusterTest/java/org/elasticsearch/versioning/SimpleVersioningIT.java
index 052c61635bab..7004e79c0866 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/versioning/SimpleVersioningIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/versioning/SimpleVersioningIT.java
@@ -19,13 +19,13 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
@@ -471,7 +471,7 @@ public String next() {
                 // zero-pad sequential
                 logger.info("--> use zero-padded sequential ids");
                 yield new IDSource() {
-                    final String zeroPad = String.format(Locale.ROOT, "%0" + TestUtil.nextInt(random, 4, 20) + "d", 0);
+                    final String zeroPad = Strings.format("%0" + TestUtil.nextInt(random, 4, 20) + "d", 0);
                     int upto;
 
                     @Override
diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java
index ca9f2c078871..51706347a38d 100644
--- a/server/src/main/java/module-info.java
+++ b/server/src/main/java/module-info.java
@@ -6,6 +6,8 @@
  * Side Public License, v 1.
  */
 
+import org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat;
+
 /** The Elasticsearch Server Module. */
 module org.elasticsearch.server {
     requires java.logging;
@@ -13,6 +15,7 @@
     requires java.sql;
     requires java.management;
     requires jdk.unsupported;
+    requires java.net.http; // required by ingest-geoip's dependency maxmind.geoip2 https://github.com/elastic/elasticsearch/issues/93553
 
     requires org.elasticsearch.cli;
     requires org.elasticsearch.base;
@@ -22,8 +25,8 @@
     requires org.elasticsearch.securesm;
     requires org.elasticsearch.xcontent;
     requires org.elasticsearch.logging;
-    requires org.elasticsearch.plugin.api;
-    requires org.elasticsearch.plugin.analysis.api;
+    requires org.elasticsearch.plugin;
+    requires org.elasticsearch.plugin.analysis;
 
     requires com.sun.jna;
     requires hppc;
@@ -53,6 +56,7 @@
     exports org.elasticsearch.action;
     exports org.elasticsearch.action.admin.cluster.allocation;
     exports org.elasticsearch.action.admin.cluster.configuration;
+    exports org.elasticsearch.action.admin.cluster.coordination;
     exports org.elasticsearch.action.admin.cluster.desirednodes;
     exports org.elasticsearch.action.admin.cluster.health;
     exports org.elasticsearch.action.admin.cluster.migration;
@@ -228,6 +232,7 @@
     exports org.elasticsearch.index.cache.query;
     exports org.elasticsearch.index.cache.request;
     exports org.elasticsearch.index.codec;
+    exports org.elasticsearch.index.codec.tsdb;
     exports org.elasticsearch.index.codec.bloomfilter;
     exports org.elasticsearch.index.engine;
     exports org.elasticsearch.index.fielddata;
@@ -281,7 +286,6 @@
     exports org.elasticsearch.monitor.os;
     exports org.elasticsearch.monitor.process;
     exports org.elasticsearch.node;
-    exports org.elasticsearch.reservedstate;
     exports org.elasticsearch.persistent;
     exports org.elasticsearch.persistent.decider;
     exports org.elasticsearch.plugins;
@@ -290,6 +294,7 @@
     exports org.elasticsearch.repositories;
     exports org.elasticsearch.repositories.blobstore;
     exports org.elasticsearch.repositories.fs;
+    exports org.elasticsearch.reservedstate;
     exports org.elasticsearch.rest;
     exports org.elasticsearch.rest.action;
     exports org.elasticsearch.rest.action.admin.cluster;
@@ -367,5 +372,9 @@
 
     uses org.elasticsearch.reservedstate.ReservedClusterStateHandlerProvider;
 
-    provides org.apache.lucene.codecs.PostingsFormat with org.elasticsearch.index.codec.bloomfilter.ES85BloomFilterPostingsFormat;
+    provides org.apache.lucene.codecs.PostingsFormat
+        with
+            org.elasticsearch.index.codec.bloomfilter.ES85BloomFilterPostingsFormat,
+            org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat;
+    provides org.apache.lucene.codecs.DocValuesFormat with ES87TSDBDocValuesFormat;
 }
diff --git a/server/src/main/java/org/elasticsearch/Build.java b/server/src/main/java/org/elasticsearch/Build.java
index 9cddfd504a0e..7279e0c4aa4c 100644
--- a/server/src/main/java/org/elasticsearch/Build.java
+++ b/server/src/main/java/org/elasticsearch/Build.java
@@ -151,7 +151,7 @@ static URL getElasticsearchCodeSourceLocation() {
     public static Build readBuild(StreamInput in) throws IOException {
         final Type type;
         // be lenient when reading on the wire, the enumeration values from other versions might be different than what we know
-        if (in.getVersion().before(Version.V_8_3_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_3_0)) {
             // this was the flavor, which is always the default distribution now
             in.readString();
         }
@@ -167,7 +167,7 @@ public static Build readBuild(StreamInput in) throws IOException {
     }
 
     public static void writeBuild(Build build, StreamOutput out) throws IOException {
-        if (out.getVersion().before(Version.V_8_3_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_3_0)) {
             // this was the flavor, which is always the default distribution now
             out.writeString("default");
         }
diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchException.java b/server/src/main/java/org/elasticsearch/ElasticsearchException.java
index 51718a226173..ded3862d47d2 100644
--- a/server/src/main/java/org/elasticsearch/ElasticsearchException.java
+++ b/server/src/main/java/org/elasticsearch/ElasticsearchException.java
@@ -275,7 +275,7 @@ public void writeTo(StreamOutput out) throws IOException {
     public static ElasticsearchException readException(StreamInput input, int id) throws IOException {
         CheckedFunction<StreamInput, ? extends ElasticsearchException, IOException> elasticsearchException = ID_TO_SUPPLIER.get(id);
         if (elasticsearchException == null) {
-            if (id == 127 && input.getVersion().before(Version.V_7_5_0)) {
+            if (id == 127 && input.getTransportVersion().before(TransportVersion.V_7_5_0)) {
                 // was SearchContextException
                 return new SearchException(input);
             }
@@ -287,10 +287,10 @@ public static ElasticsearchException readException(StreamInput input, int id) th
     /**
      * Returns <code>true</code> iff the given class is a registered for an exception to be read.
      */
-    public static boolean isRegistered(Class<? extends Throwable> exception, Version version) {
+    public static boolean isRegistered(Class<? extends Throwable> exception, TransportVersion version) {
         ElasticsearchExceptionHandle elasticsearchExceptionHandle = CLASS_TO_ELASTICSEARCH_EXCEPTION_HANDLE.get(exception);
         if (elasticsearchExceptionHandle != null) {
-            return version.onOrAfter(elasticsearchExceptionHandle.versionAdded);
+            return version.onOrAfter(elasticsearchExceptionHandle.versionAdded.transportVersion);
         }
         return false;
     }
@@ -1090,12 +1090,7 @@ private enum ElasticsearchExceptionHandle {
             71,
             UNKNOWN_VERSION_ADDED
         ),
-        SEARCH_PARSE_EXCEPTION(
-            org.elasticsearch.search.SearchParseException.class,
-            org.elasticsearch.search.SearchParseException::new,
-            72,
-            UNKNOWN_VERSION_ADDED
-        ),
+        // 72 was SearchParseException, only used in tests after 7.11
         CONCURRENT_SNAPSHOT_EXECUTION_EXCEPTION(
             org.elasticsearch.snapshots.ConcurrentSnapshotExecutionException.class,
             org.elasticsearch.snapshots.ConcurrentSnapshotExecutionException::new,
diff --git a/server/src/main/java/org/elasticsearch/TransportVersion.java b/server/src/main/java/org/elasticsearch/TransportVersion.java
new file mode 100644
index 000000000000..8f986efe2721
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/TransportVersion.java
@@ -0,0 +1,320 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+import java.util.TreeMap;
+
+/**
+ * Represents the version of the wire protocol used to communicate between ES nodes.
+ * <p>
+ * Prior to 8.7.0, the node {@link Version} was used everywhere. This class separates the wire protocol version
+ * from the running node version. Each node version has a reference to a specific transport version used by that node.
+ * <p>
+ * Each transport version constant has an id number, which for versions prior to 8.7.0 is the same as the node version
+ * for backwards compatibility.
+ * There is also a unique id string. This is not actually used in the protocol, but is there to ensure each protocol version
+ * is only added to the source file once. This string needs to be unique (here, a UUID, but can be any other unique nonempty string).
+ * If two concurrent PRs added the same protocol version, the unique string causes a git conflict, ensuring the second PR to be merged
+ * must be updated with the next free version. Without the unique id string, git will happily merge the two versions together,
+ * causing problems when you try to upgrade between those two PRs.
+ * <p>
+ * When adding new transport versions, it is recommended to leave a gap in the id number (say, 100)
+ * to leave space for any intermediate fixes that may be needed in the future.
+ * <p>
+ * The earliest compatible version is hardcoded at {@link #MINIMUM_COMPATIBLE}. Previously, this was dynamically calculated
+ * from the major/minor versions of {@link Version}, but {@code TransportVersion} does not have separate major/minor version numbers.
+ * So the minimum compatible version needs to be hard-coded as the transport version of the minimum compatible node version.
+ * That variable should be updated appropriately whenever we do a major version release.
+ */
+public class TransportVersion implements Comparable<TransportVersion> {
+    public static final TransportVersion ZERO = new TransportVersion(0, "00000000-0000-0000-0000-000000000000");
+    public static final TransportVersion V_7_0_0 = new TransportVersion(7_00_00_99, "7505fd05-d982-43ce-a63f-ff4c6c8bdeec");
+    public static final TransportVersion V_7_0_1 = new TransportVersion(7_00_01_99, "ae772780-e6f9-46a1-b0a0-20ed0cae37f7");
+    public static final TransportVersion V_7_1_0 = new TransportVersion(7_01_00_99, "fd09007c-1c54-450a-af99-9f941e1a53c2");
+    public static final TransportVersion V_7_1_1 = new TransportVersion(7_01_01_99, "f7ddb16c-3495-42ef-8d54-1461570ca68c");
+    public static final TransportVersion V_7_2_0 = new TransportVersion(7_02_00_99, "b74dbc52-e727-472c-af21-2156482e8796");
+    public static final TransportVersion V_7_2_1 = new TransportVersion(7_02_01_99, "a3217b94-f436-4aab-a020-162c83ba18f2");
+    public static final TransportVersion V_7_3_0 = new TransportVersion(7_03_00_99, "4f04e4c9-c5aa-49e4-8b99-abeb4e284a5a");
+    public static final TransportVersion V_7_3_1 = new TransportVersion(7_03_01_99, "532b9bc9-e11f-48a2-b997-67ca68ffb354");
+    public static final TransportVersion V_7_3_2 = new TransportVersion(7_03_02_99, "60da3953-8415-4d4f-a18d-853c3e68ebd6");
+    public static final TransportVersion V_7_4_0 = new TransportVersion(7_04_00_99, "ec7e58aa-55b4-4064-a9dd-fd723a2ba7a8");
+    public static final TransportVersion V_7_4_1 = new TransportVersion(7_04_01_99, "a316c26d-8e6a-4608-b1ec-062331552b98");
+    public static final TransportVersion V_7_4_2 = new TransportVersion(7_04_02_99, "031a77e1-3640-4c8a-80cf-28ded96bab48");
+    public static final TransportVersion V_7_5_0 = new TransportVersion(7_05_00_99, "cc6e14dc-9dc7-4b74-8e15-1f99a6cfbe03");
+    public static final TransportVersion V_7_5_1 = new TransportVersion(7_05_01_99, "9d12be44-16dc-44a8-a89a-45c9174ea596");
+    public static final TransportVersion V_7_5_2 = new TransportVersion(7_05_02_99, "484ed9de-7f5b-4e6b-a79a-0cb5e7570093");
+    public static final TransportVersion V_7_6_0 = new TransportVersion(7_06_00_99, "4637b8ae-f3df-43ae-a065-ad4c29f3373a");
+    public static final TransportVersion V_7_6_1 = new TransportVersion(7_06_01_99, "fe5b9f95-a311-4a92-943b-30ec256a331c");
+    public static final TransportVersion V_7_6_2 = new TransportVersion(7_06_02_99, "5396cb30-d91c-4789-85e8-77efd552c785");
+    public static final TransportVersion V_7_7_0 = new TransportVersion(7_07_00_99, "7bb73c48-ddb8-4437-b184-30371c35dd4b");
+    public static final TransportVersion V_7_7_1 = new TransportVersion(7_07_01_99, "85507b0f-0fca-4daf-a80b-451fe75e04a0");
+    public static final TransportVersion V_7_8_0 = new TransportVersion(7_08_00_99, "c3cc74af-d15e-494b-a907-6ad6dd2f4660");
+    public static final TransportVersion V_7_8_1 = new TransportVersion(7_08_01_99, "7acb9f6e-32f2-45ce-b87d-ca1f165b8e7a");
+    public static final TransportVersion V_7_9_0 = new TransportVersion(7_09_00_99, "9388fe76-192a-4053-b51c-d2a7b8eae545");
+    public static final TransportVersion V_7_9_1 = new TransportVersion(7_09_01_99, "30fa10fc-df6b-4435-bd9e-acdb9ae1b268");
+    public static final TransportVersion V_7_9_2 = new TransportVersion(7_09_02_99, "b58bb181-cecc-464e-b955-f6c1c1e7b4d0");
+    public static final TransportVersion V_7_9_3 = new TransportVersion(7_09_03_99, "4406926c-e2b6-4b9a-a72a-1bee8357ad3e");
+    public static final TransportVersion V_7_10_0 = new TransportVersion(7_10_00_99, "4efca195-38e4-4f74-b877-c26fb2a40733");
+    public static final TransportVersion V_7_10_1 = new TransportVersion(7_10_01_99, "0070260c-aa0b-4fc2-9c87-5cd5f23b005f");
+    public static final TransportVersion V_7_10_2 = new TransportVersion(7_10_02_99, "b369e2ed-261c-4b2f-8b42-0f0ba0549f8c");
+    public static final TransportVersion V_7_11_0 = new TransportVersion(7_11_00_99, "3b43bcbc-1c5e-4cc2-a3b4-8ac8b64239e8");
+    public static final TransportVersion V_7_11_1 = new TransportVersion(7_11_01_99, "2f75d13c-adde-4762-a46e-def8acce62b7");
+    public static final TransportVersion V_7_11_2 = new TransportVersion(7_11_02_99, "2c852a4b-236d-4e8b-9373-336c9b52685a");
+    public static final TransportVersion V_7_12_0 = new TransportVersion(7_12_00_99, "3be9ff6f-2d9f-4fc2-ba91-394dd5ebcf33");
+    public static final TransportVersion V_7_12_1 = new TransportVersion(7_12_01_99, "ee4fdfac-2039-4b00-b42d-579cbde7120c");
+    public static final TransportVersion V_7_13_0 = new TransportVersion(7_13_00_99, "e1fe494a-7c66-4571-8f8f-1d7e6d8df1b3");
+    public static final TransportVersion V_7_13_1 = new TransportVersion(7_13_01_99, "66bc8d82-36da-4d54-b22d-aca691dc3d70");
+    public static final TransportVersion V_7_13_2 = new TransportVersion(7_13_02_99, "2a6fc74c-4c44-4264-a619-37437cd2c5a0");
+    public static final TransportVersion V_7_13_3 = new TransportVersion(7_13_03_99, "a31592f5-f8d2-490c-a02e-da9501823d8d");
+    public static final TransportVersion V_7_13_4 = new TransportVersion(7_13_04_99, "3143240d-1831-4186-8a19-963336c4cea0");
+    public static final TransportVersion V_7_14_0 = new TransportVersion(7_14_00_99, "8cf0954c-b085-467f-b20b-3cb4b2e69e3e");
+    public static final TransportVersion V_7_14_1 = new TransportVersion(7_14_01_99, "3dbb62c3-cf73-4c76-8d5a-4ca70afe2c70");
+    public static final TransportVersion V_7_14_2 = new TransportVersion(7_14_02_99, "7943ae20-df60-45e5-97ba-82fc0dfc8b89");
+    public static final TransportVersion V_7_15_0 = new TransportVersion(7_15_00_99, "2273ac0e-00bb-4024-9e2e-ab78981623c6");
+    public static final TransportVersion V_7_15_1 = new TransportVersion(7_15_01_99, "a8c3503d-3452-45cf-b385-e855e16547fe");
+    public static final TransportVersion V_7_15_2 = new TransportVersion(7_15_02_99, "fbb8ad69-02e2-4c90-b2e4-23947107f8b4");
+    public static final TransportVersion V_7_16_0 = new TransportVersion(7_16_00_99, "59abadd2-25db-4547-a991-c92306a3934e");
+    public static final TransportVersion V_7_16_1 = new TransportVersion(7_16_01_99, "4ace6b6b-8bba-427f-8755-9e3b40092138");
+    public static final TransportVersion V_7_16_2 = new TransportVersion(7_16_02_99, "785567b9-b320-48ef-b538-1753228904cd");
+    public static final TransportVersion V_7_16_3 = new TransportVersion(7_16_03_99, "facf5ae7-3d4e-479c-9142-72529b784e30");
+    public static final TransportVersion V_7_17_0 = new TransportVersion(7_17_00_99, "322efe93-4c73-4e15-9274-bb76836c8fa8");
+    public static final TransportVersion V_7_17_1 = new TransportVersion(7_17_01_99, "51c72842-7974-4669-ad25-bf13ba307307");
+    public static final TransportVersion V_7_17_2 = new TransportVersion(7_17_02_99, "82bea8d0-bfea-47c2-b7d3-217d8feb67e3");
+    public static final TransportVersion V_7_17_3 = new TransportVersion(7_17_03_99, "a909c2f4-5cb8-46bf-af0f-cd18d1b7e9d2");
+    public static final TransportVersion V_7_17_4 = new TransportVersion(7_17_04_99, "5076e164-18a4-4373-8be7-15f1843c46db");
+    public static final TransportVersion V_7_17_5 = new TransportVersion(7_17_05_99, "da7e3509-7f61-4dd2-8d23-a61f628a62f6");
+    public static final TransportVersion V_7_17_6 = new TransportVersion(7_17_06_99, "a47ecf02-e457-474f-887d-ee15a7ebd969");
+    public static final TransportVersion V_7_17_7 = new TransportVersion(7_17_07_99, "108ba576-bb28-42f4-bcbf-845a0ce52560");
+    public static final TransportVersion V_7_17_8 = new TransportVersion(7_17_08_99, "82a3e70d-cf0e-4efb-ad16-6077ab9fe19f");
+    public static final TransportVersion V_7_17_9 = new TransportVersion(7_17_09_99, "afd50dda-735f-4eae-9309-3218ffec1b2d");
+    public static final TransportVersion V_7_17_10 = new TransportVersion(7_17_10_99, "18ae7108-6f7a-4205-adbb-cfcd6aa6ccc6");
+    public static final TransportVersion V_8_0_0 = new TransportVersion(8_00_00_99, "c7d2372c-9f01-4a79-8b11-227d862dfe4f");
+    public static final TransportVersion V_8_0_1 = new TransportVersion(8_00_01_99, "56e044c3-37e5-4f7e-bd38-f493927354ac");
+    public static final TransportVersion V_8_1_0 = new TransportVersion(8_01_00_99, "3dc49dce-9cef-492a-ac8d-3cc79f6b4280");
+    public static final TransportVersion V_8_1_1 = new TransportVersion(8_01_01_99, "40cf32e5-17b0-4187-9de1-022cdea69db9");
+    public static final TransportVersion V_8_1_2 = new TransportVersion(8_01_02_99, "54aa6394-08f3-4db7-b82e-314ae4b5b562");
+    public static final TransportVersion V_8_1_3 = new TransportVersion(8_01_03_99, "9772b54b-1e14-485f-92e8-8847b3a3d569");
+    public static final TransportVersion V_8_2_0 = new TransportVersion(8_02_00_99, "8ce6d555-202e-47db-ab7d-ade9dda1b7e8");
+    public static final TransportVersion V_8_2_1 = new TransportVersion(8_02_01_99, "ffbb67e8-cc33-4b02-a995-b461d9ee36c8");
+    public static final TransportVersion V_8_2_2 = new TransportVersion(8_02_02_99, "2499ee77-187d-4e10-8366-8e60d5f03676");
+    public static final TransportVersion V_8_2_3 = new TransportVersion(8_02_03_99, "046aae43-3090-4ece-8c27-8d489f097548");
+    public static final TransportVersion V_8_3_0 = new TransportVersion(8_03_00_99, "559ddb66-d857-4208-bed5-a995ccf478ea");
+    public static final TransportVersion V_8_3_1 = new TransportVersion(8_03_01_99, "31f9b136-dbbe-4fa1-b811-d6afa2a1b472");
+    public static final TransportVersion V_8_3_2 = new TransportVersion(8_03_02_99, "f6e9cd4c-2a71-4f9b-80d4-7ba97ebd18b2");
+    public static final TransportVersion V_8_3_3 = new TransportVersion(8_03_03_99, "a784de3e-533e-4844-8728-c55c6932dd8e");
+    public static final TransportVersion V_8_4_0 = new TransportVersion(8_04_00_99, "c0d12906-aa5b-45d4-94c7-cbcf4d9818ca");
+    public static final TransportVersion V_8_4_1 = new TransportVersion(8_04_01_99, "9a915f76-f259-4361-b53d-3f19c7797fd8");
+    public static final TransportVersion V_8_4_2 = new TransportVersion(8_04_02_99, "87c5b7b2-0f57-4172-8a81-b9f9a0198525");
+    public static final TransportVersion V_8_4_3 = new TransportVersion(8_04_03_99, "327cb1a0-9b5d-4be9-8033-285c2549f770");
+    public static final TransportVersion V_8_5_0 = new TransportVersion(8_05_00_99, "be3d7f23-7240-4904-9d7f-e25a0f766eca");
+    public static final TransportVersion V_8_5_1 = new TransportVersion(8_05_01_99, "d349d202-f01c-4dbb-85dd-947fb4267c99");
+    public static final TransportVersion V_8_5_2 = new TransportVersion(8_05_02_99, "b68b1331-fd64-44d9-9e71-f6796ec2024c");
+    public static final TransportVersion V_8_5_3 = new TransportVersion(8_05_03_99, "9ca3c835-e3b7-4622-a08e-d51e42403b06");
+    public static final TransportVersion V_8_5_4 = new TransportVersion(8_05_04_99, "97ee525c-555d-45ca-83dc-59cd592c8e86");
+    public static final TransportVersion V_8_6_0 = new TransportVersion(8_06_00_99, "e209c5ed-3488-4415-b561-33492ca3b789");
+    public static final TransportVersion V_8_6_1 = new TransportVersion(8_06_01_99, "9f113acb-1b21-4fda-bef9-2a3e669b5c7b");
+    public static final TransportVersion V_8_6_2 = new TransportVersion(8_06_02_99, "5a82fb68-b265-4a06-97c5-53496f823f51");
+    public static final TransportVersion V_8_7_0 = new TransportVersion(8_07_00_99, "f1ee7a85-4fa6-43f5-8679-33e2b750448b");
+    /*
+     * READ THE JAVADOC ABOVE BEFORE ADDING NEW TRANSPORT VERSIONS
+     * Detached transport versions added below here. Starts at ES major version 10 equivalent.
+     */
+    // NOTE: DO NOT UNCOMMENT until all transport code uses TransportVersion
+    // public static final TransportVersion V_10_000_000 = new TransportVersion(10_000_000, "dc3cbf06-3ed5-4e1b-9978-ee1d04d235bc");
+    /*
+     * When adding a new transport version, ensure there is a gap (say, 100) between versions
+     * This is to make it possible to add intermediate versions for any bug fixes that may be required.
+     *
+     * When adding versions for patch fixes, add numbers in the middle of the gap. This is to ensure there is always some space
+     * for patch fixes between any two versions.
+     */
+
+    /** Reference to the current transport version */
+    public static final TransportVersion CURRENT = V_8_7_0;
+
+    /** Reference to the earliest compatible transport version to this version of the codebase */
+    // TODO: can we programmatically calculate or check this? Don't want to introduce circular ref between Version/TransportVersion
+    public static final TransportVersion MINIMUM_COMPATIBLE = V_7_17_0;
+
+    static NavigableMap<Integer, TransportVersion> getAllVersionIds(Class<?> cls) {
+        NavigableMap<Integer, TransportVersion> builder = new TreeMap<>();
+        Map<String, TransportVersion> uniqueIds = new HashMap<>();
+
+        Set<String> ignore = Set.of("ZERO", "CURRENT", "MINIMUM_COMPATIBLE");
+        for (Field declaredField : cls.getFields()) {
+            if (declaredField.getType().equals(TransportVersion.class)) {
+                String fieldName = declaredField.getName();
+                if (ignore.contains(fieldName)) {
+                    continue;
+                }
+                try {
+                    TransportVersion version = (TransportVersion) declaredField.get(null);
+
+                    TransportVersion maybePrevious = builder.put(version.id, version);
+                    assert maybePrevious == null
+                        : "expected [" + version.id + "] to be uniquely mapped but saw [" + maybePrevious + "] and [" + version + "]";
+
+                    TransportVersion sameUniqueId = uniqueIds.put(version.uniqueId, version);
+                    assert sameUniqueId == null
+                        : "Versions "
+                            + version
+                            + " and "
+                            + sameUniqueId
+                            + " have the same unique id. Each TransportVersion should have a different unique id";
+                } catch (IllegalAccessException e) {
+                    assert false : "Version field [" + fieldName + "] should be public";
+                }
+            }
+        }
+
+        return Collections.unmodifiableNavigableMap(builder);
+    }
+
+    private static final NavigableMap<Integer, TransportVersion> VERSION_IDS;
+
+    static {
+        VERSION_IDS = getAllVersionIds(TransportVersion.class);
+    }
+
+    public static TransportVersion readVersion(StreamInput in) throws IOException {
+        return fromId(in.readVInt());
+    }
+
+    public static TransportVersion fromId(int id) {
+        TransportVersion known = VERSION_IDS.get(id);
+        if (known != null) {
+            return known;
+        }
+        // this is a version we don't otherwise know about - just create a placeholder
+        return new TransportVersion(id, "<unknown>");
+    }
+
+    public static void writeVersion(TransportVersion version, StreamOutput out) throws IOException {
+        out.writeVInt(version.id);
+    }
+
+    /**
+     * Returns the minimum version of {@code version1} and {@code version2}
+     */
+    public static TransportVersion min(TransportVersion version1, TransportVersion version2) {
+        return version1.id < version2.id ? version1 : version2;
+    }
+
+    /**
+     * Returns the maximum version of {@code version1} and {@code version2}
+     */
+    public static TransportVersion max(TransportVersion version1, TransportVersion version2) {
+        return version1.id > version2.id ? version1 : version2;
+    }
+
+    /**
+     * returns a sorted collection of declared transport version constants
+     */
+    public static Collection<TransportVersion> getAllVersions() {
+        return VERSION_IDS.values();
+    }
+
+    public final int id;
+    private final String uniqueId;
+
+    TransportVersion(int id, String uniqueId) {
+        this.id = id;
+        this.uniqueId = Strings.requireNonEmpty(uniqueId, "Each TransportVersion needs a unique string id");
+    }
+
+    /**
+     * Placeholder method for code compatibility with code calling {@code CURRENT.minimumCompatibilityVersion}.
+     */
+    @Deprecated(forRemoval = true)
+    public TransportVersion minimumCompatibilityVersion() {
+        assert this.equals(CURRENT) : "Should be CURRENT, but was: " + this;
+        return MINIMUM_COMPATIBLE;
+    }
+
+    @Deprecated(forRemoval = true)
+    public boolean isCompatible(TransportVersion version) {
+        return onOrAfter(version.calculateMinimumCompatVersion()) && version.onOrAfter(calculateMinimumCompatVersion());
+    }
+
+    private TransportVersion minimumCompatibleVersion;
+
+    /**
+     * Placeholder for code calling {@code minimumCompatibilityVersion} on arbitrary Version instances.
+     * Code calling this should be refactored to not do this.
+     */
+    @Deprecated(forRemoval = true)
+    public TransportVersion calculateMinimumCompatVersion() {
+        if (minimumCompatibleVersion == null) {
+            minimumCompatibleVersion = Version.findVersion(this).minimumCompatibilityVersion().transportVersion;
+        }
+        return minimumCompatibleVersion;
+    }
+
+    public boolean after(TransportVersion version) {
+        return version.id < id;
+    }
+
+    public boolean onOrAfter(TransportVersion version) {
+        return version.id <= id;
+    }
+
+    public boolean before(TransportVersion version) {
+        return version.id > id;
+    }
+
+    public boolean onOrBefore(TransportVersion version) {
+        return version.id >= id;
+    }
+
+    @Override
+    public int compareTo(TransportVersion other) {
+        return Integer.compare(this.id, other.id);
+    }
+
+    @Override
+    public String toString() {
+        return Integer.toString(id);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+
+        TransportVersion version = (TransportVersion) o;
+
+        if (id != version.id) {
+            return false;
+        }
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return id;
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java
index e43664f53d72..5c29f5ba97a2 100644
--- a/server/src/main/java/org/elasticsearch/Version.java
+++ b/server/src/main/java/org/elasticsearch/Version.java
@@ -26,8 +26,12 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.NavigableMap;
+import java.util.NoSuchElementException;
 import java.util.Objects;
+import java.util.TreeMap;
 
+@SuppressWarnings("checkstyle:linelength")
 public class Version implements Comparable<Version>, ToXContentFragment {
     /*
      * The logic for ID is: XXYYZZAA, where XX is major version, YY is minor version, ZZ is revision, and AA is alpha/beta/rc indicator AA
@@ -46,95 +50,101 @@ public class Version implements Comparable<Version>, ToXContentFragment {
      */
 
     public static final int V_EMPTY_ID = 0;
-    public static final Version V_EMPTY = new Version(V_EMPTY_ID, org.apache.lucene.util.Version.LATEST);
-    public static final Version V_7_0_0 = new Version(7_00_00_99, org.apache.lucene.util.Version.LUCENE_8_0_0);
-    public static final Version V_7_0_1 = new Version(7_00_01_99, org.apache.lucene.util.Version.LUCENE_8_0_0);
-    public static final Version V_7_1_0 = new Version(7_01_00_99, org.apache.lucene.util.Version.LUCENE_8_0_0);
-    public static final Version V_7_1_1 = new Version(7_01_01_99, org.apache.lucene.util.Version.LUCENE_8_0_0);
-    public static final Version V_7_2_0 = new Version(7_02_00_99, org.apache.lucene.util.Version.LUCENE_8_0_0);
-    public static final Version V_7_2_1 = new Version(7_02_01_99, org.apache.lucene.util.Version.LUCENE_8_0_0);
-    public static final Version V_7_3_0 = new Version(7_03_00_99, org.apache.lucene.util.Version.LUCENE_8_1_0);
-    public static final Version V_7_3_1 = new Version(7_03_01_99, org.apache.lucene.util.Version.LUCENE_8_1_0);
-    public static final Version V_7_3_2 = new Version(7_03_02_99, org.apache.lucene.util.Version.LUCENE_8_1_0);
-    public static final Version V_7_4_0 = new Version(7_04_00_99, org.apache.lucene.util.Version.LUCENE_8_2_0);
-    public static final Version V_7_4_1 = new Version(7_04_01_99, org.apache.lucene.util.Version.LUCENE_8_2_0);
-    public static final Version V_7_4_2 = new Version(7_04_02_99, org.apache.lucene.util.Version.LUCENE_8_2_0);
-    public static final Version V_7_5_0 = new Version(7_05_00_99, org.apache.lucene.util.Version.LUCENE_8_3_0);
-    public static final Version V_7_5_1 = new Version(7_05_01_99, org.apache.lucene.util.Version.LUCENE_8_3_0);
-    public static final Version V_7_5_2 = new Version(7_05_02_99, org.apache.lucene.util.Version.LUCENE_8_3_0);
-    public static final Version V_7_6_0 = new Version(7_06_00_99, org.apache.lucene.util.Version.LUCENE_8_4_0);
-    public static final Version V_7_6_1 = new Version(7_06_01_99, org.apache.lucene.util.Version.LUCENE_8_4_0);
-    public static final Version V_7_6_2 = new Version(7_06_02_99, org.apache.lucene.util.Version.LUCENE_8_4_0);
-    public static final Version V_7_7_0 = new Version(7_07_00_99, org.apache.lucene.util.Version.LUCENE_8_5_1);
-    public static final Version V_7_7_1 = new Version(7_07_01_99, org.apache.lucene.util.Version.LUCENE_8_5_1);
-    public static final Version V_7_8_0 = new Version(7_08_00_99, org.apache.lucene.util.Version.LUCENE_8_5_1);
-    public static final Version V_7_8_1 = new Version(7_08_01_99, org.apache.lucene.util.Version.LUCENE_8_5_1);
-    public static final Version V_7_9_0 = new Version(7_09_00_99, org.apache.lucene.util.Version.LUCENE_8_6_0);
-    public static final Version V_7_9_1 = new Version(7_09_01_99, org.apache.lucene.util.Version.LUCENE_8_6_2);
-    public static final Version V_7_9_2 = new Version(7_09_02_99, org.apache.lucene.util.Version.LUCENE_8_6_2);
-    public static final Version V_7_9_3 = new Version(7_09_03_99, org.apache.lucene.util.Version.LUCENE_8_6_2);
-    public static final Version V_7_10_0 = new Version(7_10_00_99, org.apache.lucene.util.Version.LUCENE_8_7_0);
-    public static final Version V_7_10_1 = new Version(7_10_01_99, org.apache.lucene.util.Version.LUCENE_8_7_0);
-    public static final Version V_7_10_2 = new Version(7_10_02_99, org.apache.lucene.util.Version.LUCENE_8_7_0);
-    public static final Version V_7_11_0 = new Version(7_11_00_99, org.apache.lucene.util.Version.LUCENE_8_7_0);
-    public static final Version V_7_11_1 = new Version(7_11_01_99, org.apache.lucene.util.Version.LUCENE_8_7_0);
-    public static final Version V_7_11_2 = new Version(7_11_02_99, org.apache.lucene.util.Version.LUCENE_8_7_0);
-    public static final Version V_7_12_0 = new Version(7_12_00_99, org.apache.lucene.util.Version.LUCENE_8_8_0);
-    public static final Version V_7_12_1 = new Version(7_12_01_99, org.apache.lucene.util.Version.LUCENE_8_8_0);
-    public static final Version V_7_13_0 = new Version(7_13_00_99, org.apache.lucene.util.Version.LUCENE_8_8_2);
-    public static final Version V_7_13_1 = new Version(7_13_01_99, org.apache.lucene.util.Version.LUCENE_8_8_2);
-    public static final Version V_7_13_2 = new Version(7_13_02_99, org.apache.lucene.util.Version.LUCENE_8_8_2);
-    public static final Version V_7_13_3 = new Version(7_13_03_99, org.apache.lucene.util.Version.LUCENE_8_8_2);
-    public static final Version V_7_13_4 = new Version(7_13_04_99, org.apache.lucene.util.Version.LUCENE_8_8_2);
-    public static final Version V_7_14_0 = new Version(7_14_00_99, org.apache.lucene.util.Version.LUCENE_8_9_0);
-    public static final Version V_7_14_1 = new Version(7_14_01_99, org.apache.lucene.util.Version.LUCENE_8_9_0);
-    public static final Version V_7_14_2 = new Version(7_14_02_99, org.apache.lucene.util.Version.LUCENE_8_9_0);
-    public static final Version V_7_15_0 = new Version(7_15_00_99, org.apache.lucene.util.Version.LUCENE_8_9_0);
-    public static final Version V_7_15_1 = new Version(7_15_01_99, org.apache.lucene.util.Version.LUCENE_8_9_0);
-    public static final Version V_7_15_2 = new Version(7_15_02_99, org.apache.lucene.util.Version.LUCENE_8_9_0);
-    public static final Version V_7_16_0 = new Version(7_16_00_99, org.apache.lucene.util.Version.LUCENE_8_10_1);
-    public static final Version V_7_16_1 = new Version(7_16_01_99, org.apache.lucene.util.Version.LUCENE_8_10_1);
-    public static final Version V_7_16_2 = new Version(7_16_02_99, org.apache.lucene.util.Version.LUCENE_8_10_1);
-    public static final Version V_7_16_3 = new Version(7_16_03_99, org.apache.lucene.util.Version.LUCENE_8_10_1);
-    public static final Version V_7_17_0 = new Version(7_17_00_99, org.apache.lucene.util.Version.LUCENE_8_11_1);
-    public static final Version V_7_17_1 = new Version(7_17_01_99, org.apache.lucene.util.Version.LUCENE_8_11_1);
-    public static final Version V_7_17_2 = new Version(7_17_02_99, org.apache.lucene.util.Version.LUCENE_8_11_1);
-    public static final Version V_7_17_3 = new Version(7_17_03_99, org.apache.lucene.util.Version.LUCENE_8_11_1);
-    public static final Version V_7_17_4 = new Version(7_17_04_99, org.apache.lucene.util.Version.LUCENE_8_11_1);
-    public static final Version V_7_17_5 = new Version(7_17_05_99, org.apache.lucene.util.Version.LUCENE_8_11_1);
-    public static final Version V_7_17_6 = new Version(7_17_06_99, org.apache.lucene.util.Version.LUCENE_8_11_1);
-    public static final Version V_7_17_7 = new Version(7_17_07_99, org.apache.lucene.util.Version.LUCENE_8_11_1);
-    public static final Version V_7_17_8 = new Version(7_17_08_99, org.apache.lucene.util.Version.LUCENE_8_11_1);
-    public static final Version V_8_0_0 = new Version(8_00_00_99, org.apache.lucene.util.Version.LUCENE_9_0_0);
-    public static final Version V_8_0_1 = new Version(8_00_01_99, org.apache.lucene.util.Version.LUCENE_9_0_0);
-    public static final Version V_8_1_0 = new Version(8_01_00_99, org.apache.lucene.util.Version.LUCENE_9_0_0);
-    public static final Version V_8_1_1 = new Version(8_01_01_99, org.apache.lucene.util.Version.LUCENE_9_0_0);
-    public static final Version V_8_1_2 = new Version(8_01_02_99, org.apache.lucene.util.Version.LUCENE_9_0_0);
-    public static final Version V_8_1_3 = new Version(8_01_03_99, org.apache.lucene.util.Version.LUCENE_9_0_0);
-    public static final Version V_8_2_0 = new Version(8_02_00_99, org.apache.lucene.util.Version.LUCENE_9_1_0);
-    public static final Version V_8_2_1 = new Version(8_02_01_99, org.apache.lucene.util.Version.LUCENE_9_1_0);
-    public static final Version V_8_2_2 = new Version(8_02_02_99, org.apache.lucene.util.Version.LUCENE_9_1_0);
-    public static final Version V_8_2_3 = new Version(8_02_03_99, org.apache.lucene.util.Version.LUCENE_9_1_0);
-    public static final Version V_8_3_0 = new Version(8_03_00_99, org.apache.lucene.util.Version.LUCENE_9_2_0);
-    public static final Version V_8_3_1 = new Version(8_03_01_99, org.apache.lucene.util.Version.LUCENE_9_2_0);
-    public static final Version V_8_3_2 = new Version(8_03_02_99, org.apache.lucene.util.Version.LUCENE_9_2_0);
-    public static final Version V_8_3_3 = new Version(8_03_03_99, org.apache.lucene.util.Version.LUCENE_9_2_0);
-    public static final Version V_8_4_0 = new Version(8_04_00_99, org.apache.lucene.util.Version.LUCENE_9_3_0);
-    public static final Version V_8_4_1 = new Version(8_04_01_99, org.apache.lucene.util.Version.LUCENE_9_3_0);
-    public static final Version V_8_4_2 = new Version(8_04_02_99, org.apache.lucene.util.Version.LUCENE_9_3_0);
-    public static final Version V_8_4_3 = new Version(8_04_03_99, org.apache.lucene.util.Version.LUCENE_9_3_0);
-    public static final Version V_8_5_0 = new Version(8_05_00_99, org.apache.lucene.util.Version.LUCENE_9_4_1);
-    public static final Version V_8_5_1 = new Version(8_05_01_99, org.apache.lucene.util.Version.LUCENE_9_4_1);
-    public static final Version V_8_5_2 = new Version(8_05_02_99, org.apache.lucene.util.Version.LUCENE_9_4_1);
-    public static final Version V_8_6_0 = new Version(8_06_00_99, org.apache.lucene.util.Version.LUCENE_9_4_1);
-    public static final Version V_8_7_0 = new Version(8_07_00_99, org.apache.lucene.util.Version.LUCENE_9_4_1);
+    public static final Version V_EMPTY = new Version(V_EMPTY_ID, TransportVersion.ZERO, org.apache.lucene.util.Version.LATEST);
+    public static final Version V_7_0_0 = new Version(7_00_00_99, TransportVersion.V_7_0_0, org.apache.lucene.util.Version.LUCENE_8_0_0);
+    public static final Version V_7_0_1 = new Version(7_00_01_99, TransportVersion.V_7_0_1, org.apache.lucene.util.Version.LUCENE_8_0_0);
+    public static final Version V_7_1_0 = new Version(7_01_00_99, TransportVersion.V_7_1_0, org.apache.lucene.util.Version.LUCENE_8_0_0);
+    public static final Version V_7_1_1 = new Version(7_01_01_99, TransportVersion.V_7_1_1, org.apache.lucene.util.Version.LUCENE_8_0_0);
+    public static final Version V_7_2_0 = new Version(7_02_00_99, TransportVersion.V_7_2_0, org.apache.lucene.util.Version.LUCENE_8_0_0);
+    public static final Version V_7_2_1 = new Version(7_02_01_99, TransportVersion.V_7_2_1, org.apache.lucene.util.Version.LUCENE_8_0_0);
+    public static final Version V_7_3_0 = new Version(7_03_00_99, TransportVersion.V_7_3_0, org.apache.lucene.util.Version.LUCENE_8_1_0);
+    public static final Version V_7_3_1 = new Version(7_03_01_99, TransportVersion.V_7_3_1, org.apache.lucene.util.Version.LUCENE_8_1_0);
+    public static final Version V_7_3_2 = new Version(7_03_02_99, TransportVersion.V_7_3_2, org.apache.lucene.util.Version.LUCENE_8_1_0);
+    public static final Version V_7_4_0 = new Version(7_04_00_99, TransportVersion.V_7_4_0, org.apache.lucene.util.Version.LUCENE_8_2_0);
+    public static final Version V_7_4_1 = new Version(7_04_01_99, TransportVersion.V_7_4_1, org.apache.lucene.util.Version.LUCENE_8_2_0);
+    public static final Version V_7_4_2 = new Version(7_04_02_99, TransportVersion.V_7_4_2, org.apache.lucene.util.Version.LUCENE_8_2_0);
+    public static final Version V_7_5_0 = new Version(7_05_00_99, TransportVersion.V_7_5_0, org.apache.lucene.util.Version.LUCENE_8_3_0);
+    public static final Version V_7_5_1 = new Version(7_05_01_99, TransportVersion.V_7_5_1, org.apache.lucene.util.Version.LUCENE_8_3_0);
+    public static final Version V_7_5_2 = new Version(7_05_02_99, TransportVersion.V_7_5_2, org.apache.lucene.util.Version.LUCENE_8_3_0);
+    public static final Version V_7_6_0 = new Version(7_06_00_99, TransportVersion.V_7_6_0, org.apache.lucene.util.Version.LUCENE_8_4_0);
+    public static final Version V_7_6_1 = new Version(7_06_01_99, TransportVersion.V_7_6_1, org.apache.lucene.util.Version.LUCENE_8_4_0);
+    public static final Version V_7_6_2 = new Version(7_06_02_99, TransportVersion.V_7_6_2, org.apache.lucene.util.Version.LUCENE_8_4_0);
+    public static final Version V_7_7_0 = new Version(7_07_00_99, TransportVersion.V_7_7_0, org.apache.lucene.util.Version.LUCENE_8_5_1);
+    public static final Version V_7_7_1 = new Version(7_07_01_99, TransportVersion.V_7_7_1, org.apache.lucene.util.Version.LUCENE_8_5_1);
+    public static final Version V_7_8_0 = new Version(7_08_00_99, TransportVersion.V_7_8_0, org.apache.lucene.util.Version.LUCENE_8_5_1);
+    public static final Version V_7_8_1 = new Version(7_08_01_99, TransportVersion.V_7_8_1, org.apache.lucene.util.Version.LUCENE_8_5_1);
+    public static final Version V_7_9_0 = new Version(7_09_00_99, TransportVersion.V_7_9_0, org.apache.lucene.util.Version.LUCENE_8_6_0);
+    public static final Version V_7_9_1 = new Version(7_09_01_99, TransportVersion.V_7_9_1, org.apache.lucene.util.Version.LUCENE_8_6_2);
+    public static final Version V_7_9_2 = new Version(7_09_02_99, TransportVersion.V_7_9_2, org.apache.lucene.util.Version.LUCENE_8_6_2);
+    public static final Version V_7_9_3 = new Version(7_09_03_99, TransportVersion.V_7_9_3, org.apache.lucene.util.Version.LUCENE_8_6_2);
+    public static final Version V_7_10_0 = new Version(7_10_00_99, TransportVersion.V_7_10_0, org.apache.lucene.util.Version.LUCENE_8_7_0);
+    public static final Version V_7_10_1 = new Version(7_10_01_99, TransportVersion.V_7_10_1, org.apache.lucene.util.Version.LUCENE_8_7_0);
+    public static final Version V_7_10_2 = new Version(7_10_02_99, TransportVersion.V_7_10_2, org.apache.lucene.util.Version.LUCENE_8_7_0);
+    public static final Version V_7_11_0 = new Version(7_11_00_99, TransportVersion.V_7_11_0, org.apache.lucene.util.Version.LUCENE_8_7_0);
+    public static final Version V_7_11_1 = new Version(7_11_01_99, TransportVersion.V_7_11_1, org.apache.lucene.util.Version.LUCENE_8_7_0);
+    public static final Version V_7_11_2 = new Version(7_11_02_99, TransportVersion.V_7_11_2, org.apache.lucene.util.Version.LUCENE_8_7_0);
+    public static final Version V_7_12_0 = new Version(7_12_00_99, TransportVersion.V_7_12_0, org.apache.lucene.util.Version.LUCENE_8_8_0);
+    public static final Version V_7_12_1 = new Version(7_12_01_99, TransportVersion.V_7_12_1, org.apache.lucene.util.Version.LUCENE_8_8_0);
+    public static final Version V_7_13_0 = new Version(7_13_00_99, TransportVersion.V_7_13_0, org.apache.lucene.util.Version.LUCENE_8_8_2);
+    public static final Version V_7_13_1 = new Version(7_13_01_99, TransportVersion.V_7_13_1, org.apache.lucene.util.Version.LUCENE_8_8_2);
+    public static final Version V_7_13_2 = new Version(7_13_02_99, TransportVersion.V_7_13_2, org.apache.lucene.util.Version.LUCENE_8_8_2);
+    public static final Version V_7_13_3 = new Version(7_13_03_99, TransportVersion.V_7_13_3, org.apache.lucene.util.Version.LUCENE_8_8_2);
+    public static final Version V_7_13_4 = new Version(7_13_04_99, TransportVersion.V_7_13_4, org.apache.lucene.util.Version.LUCENE_8_8_2);
+    public static final Version V_7_14_0 = new Version(7_14_00_99, TransportVersion.V_7_14_0, org.apache.lucene.util.Version.LUCENE_8_9_0);
+    public static final Version V_7_14_1 = new Version(7_14_01_99, TransportVersion.V_7_14_1, org.apache.lucene.util.Version.LUCENE_8_9_0);
+    public static final Version V_7_14_2 = new Version(7_14_02_99, TransportVersion.V_7_14_2, org.apache.lucene.util.Version.LUCENE_8_9_0);
+    public static final Version V_7_15_0 = new Version(7_15_00_99, TransportVersion.V_7_15_0, org.apache.lucene.util.Version.LUCENE_8_9_0);
+    public static final Version V_7_15_1 = new Version(7_15_01_99, TransportVersion.V_7_15_1, org.apache.lucene.util.Version.LUCENE_8_9_0);
+    public static final Version V_7_15_2 = new Version(7_15_02_99, TransportVersion.V_7_15_2, org.apache.lucene.util.Version.LUCENE_8_9_0);
+    public static final Version V_7_16_0 = new Version(7_16_00_99, TransportVersion.V_7_16_0, org.apache.lucene.util.Version.LUCENE_8_10_1);
+    public static final Version V_7_16_1 = new Version(7_16_01_99, TransportVersion.V_7_16_1, org.apache.lucene.util.Version.LUCENE_8_10_1);
+    public static final Version V_7_16_2 = new Version(7_16_02_99, TransportVersion.V_7_16_2, org.apache.lucene.util.Version.LUCENE_8_10_1);
+    public static final Version V_7_16_3 = new Version(7_16_03_99, TransportVersion.V_7_16_3, org.apache.lucene.util.Version.LUCENE_8_10_1);
+    public static final Version V_7_17_0 = new Version(7_17_00_99, TransportVersion.V_7_17_0, org.apache.lucene.util.Version.LUCENE_8_11_1);
+    public static final Version V_7_17_1 = new Version(7_17_01_99, TransportVersion.V_7_17_1, org.apache.lucene.util.Version.LUCENE_8_11_1);
+    public static final Version V_7_17_2 = new Version(7_17_02_99, TransportVersion.V_7_17_2, org.apache.lucene.util.Version.LUCENE_8_11_1);
+    public static final Version V_7_17_3 = new Version(7_17_03_99, TransportVersion.V_7_17_3, org.apache.lucene.util.Version.LUCENE_8_11_1);
+    public static final Version V_7_17_4 = new Version(7_17_04_99, TransportVersion.V_7_17_4, org.apache.lucene.util.Version.LUCENE_8_11_1);
+    public static final Version V_7_17_5 = new Version(7_17_05_99, TransportVersion.V_7_17_5, org.apache.lucene.util.Version.LUCENE_8_11_1);
+    public static final Version V_7_17_6 = new Version(7_17_06_99, TransportVersion.V_7_17_6, org.apache.lucene.util.Version.LUCENE_8_11_1);
+    public static final Version V_7_17_7 = new Version(7_17_07_99, TransportVersion.V_7_17_7, org.apache.lucene.util.Version.LUCENE_8_11_1);
+    public static final Version V_7_17_8 = new Version(7_17_08_99, TransportVersion.V_7_17_8, org.apache.lucene.util.Version.LUCENE_8_11_1);
+    public static final Version V_7_17_9 = new Version(7_17_09_99, TransportVersion.V_7_17_9, org.apache.lucene.util.Version.LUCENE_8_11_1);
+    public static final Version V_7_17_10 = new Version(7_17_10_99, TransportVersion.V_7_17_10, org.apache.lucene.util.Version.LUCENE_8_11_1);
+    public static final Version V_8_0_0 = new Version(8_00_00_99, TransportVersion.V_8_0_0, org.apache.lucene.util.Version.LUCENE_9_0_0);
+    public static final Version V_8_0_1 = new Version(8_00_01_99, TransportVersion.V_8_0_1, org.apache.lucene.util.Version.LUCENE_9_0_0);
+    public static final Version V_8_1_0 = new Version(8_01_00_99, TransportVersion.V_8_1_0, org.apache.lucene.util.Version.LUCENE_9_0_0);
+    public static final Version V_8_1_1 = new Version(8_01_01_99, TransportVersion.V_8_1_1, org.apache.lucene.util.Version.LUCENE_9_0_0);
+    public static final Version V_8_1_2 = new Version(8_01_02_99, TransportVersion.V_8_1_2, org.apache.lucene.util.Version.LUCENE_9_0_0);
+    public static final Version V_8_1_3 = new Version(8_01_03_99, TransportVersion.V_8_1_3, org.apache.lucene.util.Version.LUCENE_9_0_0);
+    public static final Version V_8_2_0 = new Version(8_02_00_99, TransportVersion.V_8_2_0, org.apache.lucene.util.Version.LUCENE_9_1_0);
+    public static final Version V_8_2_1 = new Version(8_02_01_99, TransportVersion.V_8_2_1, org.apache.lucene.util.Version.LUCENE_9_1_0);
+    public static final Version V_8_2_2 = new Version(8_02_02_99, TransportVersion.V_8_2_2, org.apache.lucene.util.Version.LUCENE_9_1_0);
+    public static final Version V_8_2_3 = new Version(8_02_03_99, TransportVersion.V_8_2_3, org.apache.lucene.util.Version.LUCENE_9_1_0);
+    public static final Version V_8_3_0 = new Version(8_03_00_99, TransportVersion.V_8_3_0, org.apache.lucene.util.Version.LUCENE_9_2_0);
+    public static final Version V_8_3_1 = new Version(8_03_01_99, TransportVersion.V_8_3_1, org.apache.lucene.util.Version.LUCENE_9_2_0);
+    public static final Version V_8_3_2 = new Version(8_03_02_99, TransportVersion.V_8_3_2, org.apache.lucene.util.Version.LUCENE_9_2_0);
+    public static final Version V_8_3_3 = new Version(8_03_03_99, TransportVersion.V_8_3_3, org.apache.lucene.util.Version.LUCENE_9_2_0);
+    public static final Version V_8_4_0 = new Version(8_04_00_99, TransportVersion.V_8_4_0, org.apache.lucene.util.Version.LUCENE_9_3_0);
+    public static final Version V_8_4_1 = new Version(8_04_01_99, TransportVersion.V_8_4_1, org.apache.lucene.util.Version.LUCENE_9_3_0);
+    public static final Version V_8_4_2 = new Version(8_04_02_99, TransportVersion.V_8_4_2, org.apache.lucene.util.Version.LUCENE_9_3_0);
+    public static final Version V_8_4_3 = new Version(8_04_03_99, TransportVersion.V_8_4_3, org.apache.lucene.util.Version.LUCENE_9_3_0);
+    public static final Version V_8_5_0 = new Version(8_05_00_99, TransportVersion.V_8_5_0, org.apache.lucene.util.Version.LUCENE_9_4_1);
+    public static final Version V_8_5_1 = new Version(8_05_01_99, TransportVersion.V_8_5_1, org.apache.lucene.util.Version.LUCENE_9_4_1);
+    public static final Version V_8_5_2 = new Version(8_05_02_99, TransportVersion.V_8_5_2, org.apache.lucene.util.Version.LUCENE_9_4_1);
+    public static final Version V_8_5_3 = new Version(8_05_03_99, TransportVersion.V_8_5_3, org.apache.lucene.util.Version.LUCENE_9_4_2);
+    public static final Version V_8_6_0 = new Version(8_06_00_99, TransportVersion.V_8_6_0, org.apache.lucene.util.Version.LUCENE_9_4_2);
+    public static final Version V_8_6_1 = new Version(8_06_01_99, TransportVersion.V_8_6_1, org.apache.lucene.util.Version.LUCENE_9_4_2);
+    public static final Version V_8_6_2 = new Version(8_06_02_99, TransportVersion.V_8_6_2, org.apache.lucene.util.Version.LUCENE_9_4_2);
+    public static final Version V_8_7_0 = new Version(8_07_00_99, TransportVersion.V_8_7_0, org.apache.lucene.util.Version.LUCENE_9_5_0);
+
     public static final Version CURRENT = V_8_7_0;
 
-    private static final Map<Integer, Version> idToVersion;
-    private static final Map<String, Version> stringToVersion;
+    private static final NavigableMap<Integer, Version> VERSION_IDS;
+    private static final Map<String, Version> VERSION_STRINGS;
 
     static {
-        final Map<Integer, Version> builder = new HashMap<>();
+        final NavigableMap<Integer, Version> builder = new TreeMap<>();
         final Map<String, Version> builderByString = new HashMap<>();
 
         for (final Field declaredField : Version.class.getFields()) {
@@ -176,16 +186,30 @@ public class Version implements Comparable<Version>, ToXContentFragment {
                 + "]";
         builder.put(V_EMPTY_ID, V_EMPTY);
         builderByString.put(V_EMPTY.toString(), V_EMPTY);
-        idToVersion = Map.copyOf(builder);
-        stringToVersion = Map.copyOf(builderByString);
+
+        VERSION_IDS = Collections.unmodifiableNavigableMap(builder);
+        VERSION_STRINGS = Map.copyOf(builderByString);
     }
 
     public static Version readVersion(StreamInput in) throws IOException {
         return fromId(in.readVInt());
     }
 
+    /**
+     * Returns the highest Version that has this or a lesser TransportVersion.
+     */
+    @Deprecated
+    static Version findVersion(TransportVersion transportVersion) {
+        return VERSION_IDS.descendingMap()
+            .values()
+            .stream()
+            .filter(v -> v.transportVersion.compareTo(transportVersion) <= 0)
+            .findFirst()
+            .orElseThrow(() -> new NoSuchElementException("No valid Version found")); // only if transportVersion < 0 ?????
+    }
+
     public static Version fromId(int id) {
-        final Version known = idToVersion.get(id);
+        final Version known = VERSION_IDS.get(id);
         if (known != null) {
             return known;
         }
@@ -197,7 +221,7 @@ private static Version fromIdSlow(int id) {
         // Our best guess is to use the same Lucene version as the previous
         // version in the list, assuming that it didn't change.
         List<Version> versions = DeclaredVersionsHolder.DECLARED_VERSIONS;
-        Version tmp = new Version(id, org.apache.lucene.util.Version.LATEST);
+        Version tmp = new Version(id, TransportVersion.CURRENT, org.apache.lucene.util.Version.LATEST);
         int index = Collections.binarySearch(versions, tmp);
         if (index < 0) {
             index = -2 - index;
@@ -213,7 +237,8 @@ private static Version fromIdSlow(int id) {
         } else {
             luceneVersion = versions.get(index).luceneVersion;
         }
-        return new Version(id, luceneVersion);
+        // TODO: assume this is an old version that has transport version == release version
+        return new Version(id, TransportVersion.fromId(id), luceneVersion);
     }
 
     public static void writeVersion(Version version, StreamOutput out) throws IOException {
@@ -221,14 +246,14 @@ public static void writeVersion(Version version, StreamOutput out) throws IOExce
     }
 
     /**
-     * Returns the minimum version between the 2.
+     * Returns the minimum version of {@code version1} and {@code version2}
      */
     public static Version min(Version version1, Version version2) {
         return version1.id < version2.id ? version1 : version2;
     }
 
     /**
-     * Returns the maximum version between the 2
+     * Returns the maximum version of {@code version1} and {@code version2}
      */
     public static Version max(Version version1, Version version2) {
         return version1.id > version2.id ? version1 : version2;
@@ -241,7 +266,7 @@ public static Version fromString(String version) {
         if (Strings.hasLength(version) == false) {
             return Version.CURRENT;
         }
-        final Version cached = stringToVersion.get(version);
+        final Version cached = VERSION_STRINGS.get(version);
         if (cached != null) {
             return cached;
         }
@@ -296,16 +321,18 @@ private static Version fromStringSlow(String version) {
     public final byte minor;
     public final byte revision;
     public final byte build;
+    public final TransportVersion transportVersion;
     public final org.apache.lucene.util.Version luceneVersion;
     private final String toString;
     private final int previousMajorId;
 
-    Version(int id, org.apache.lucene.util.Version luceneVersion) {
+    Version(int id, TransportVersion transportVersion, org.apache.lucene.util.Version luceneVersion) {
         this.id = id;
         this.major = (byte) ((id / 1000000) % 100);
         this.minor = (byte) ((id / 10000) % 100);
         this.revision = (byte) ((id / 100) % 100);
         this.build = (byte) (id % 100);
+        this.transportVersion = Objects.requireNonNull(transportVersion);
         this.luceneVersion = Objects.requireNonNull(luceneVersion);
         this.toString = major + "." + minor + "." + revision;
         this.previousMajorId = major > 0 ? (major - 1) * 1000000 + 99 : major;
diff --git a/server/src/main/java/org/elasticsearch/action/ActionListener.java b/server/src/main/java/org/elasticsearch/action/ActionListener.java
index 1b918186009c..dd8b629f208c 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionListener.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionListener.java
@@ -8,14 +8,19 @@
 
 package org.elasticsearch.action;
 
+import org.elasticsearch.Assertions;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.core.CheckedRunnable;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Releasables;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 
@@ -251,6 +256,13 @@ public String toString() {
         }
     }
 
+    /**
+     * Creates a listener which releases the given resource on completion (whether success or failure)
+     */
+    static <Response> ActionListener<Response> releasing(Releasable releasable) {
+        return assertOnce(wrap(runnableFromReleasable(releasable)));
+    }
+
     /**
      * Creates a listener that listens for a response (or failure) and executes the
      * corresponding runnable when the response (or failure) is received.
@@ -289,6 +301,29 @@ public String toString() {
         };
     }
 
+    /**
+     * Adds a wrapper around a listener which catches exceptions thrown by its {@link #onResponse} method and feeds them to its
+     * {@link #onFailure}.
+     */
+    static <DelegateResponse, Response extends DelegateResponse> ActionListener<Response> wrap(ActionListener<DelegateResponse> delegate) {
+        return new ActionListener<>() {
+            @Override
+            public void onResponse(Response response) {
+                ActionListener.run(delegate, l -> l.onResponse(response));
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                delegate.onFailure(e);
+            }
+
+            @Override
+            public String toString() {
+                return "wrapped{" + delegate + "}";
+            }
+        };
+    }
+
     /**
      * Notifies every given listener with the response passed to {@link #onResponse(Object)}. If a listener itself throws an exception
      * the exception is forwarded to {@link #onFailure(Exception)}. If in turn {@link #onFailure(Exception)} fails all remaining
@@ -331,7 +366,15 @@ static <Response> void onFailure(Iterable<ActionListener<Response>> listeners, E
      * callback when the listener is notified via either {@code #onResponse} or {@code #onFailure}.
      */
     static <Response> ActionListener<Response> runAfter(ActionListener<Response> delegate, Runnable runAfter) {
-        return new RunAfterActionListener<>(delegate, runAfter);
+        return assertOnce(new RunAfterActionListener<>(delegate, runAfter));
+    }
+
+    /**
+     * Wraps a given listener and returns a new listener which releases the provided {@code releaseAfter}
+     * resource when the listener is notified via either {@code #onResponse} or {@code #onFailure}.
+     */
+    static <Response> ActionListener<Response> releaseAfter(ActionListener<Response> delegate, Releasable releaseAfter) {
+        return assertOnce(new RunAfterActionListener<>(delegate, runnableFromReleasable(releaseAfter)));
     }
 
     final class RunAfterActionListener<T> extends Delegating<T, T> {
@@ -374,7 +417,7 @@ public String toString() {
      * not be executed.
      */
     static <Response> ActionListener<Response> runBefore(ActionListener<Response> delegate, CheckedRunnable<?> runBefore) {
-        return new RunBeforeActionListener<>(delegate, runBefore);
+        return assertOnce(new RunBeforeActionListener<>(delegate, runBefore));
     }
 
     final class RunBeforeActionListener<T> extends Delegating<T, T> {
@@ -418,15 +461,27 @@ public String toString() {
      * and {@link #onFailure(Exception)} of the provided listener will be called at most once.
      */
     static <Response> ActionListener<Response> notifyOnce(ActionListener<Response> delegate) {
-        return new NotifyOnceListener<Response>() {
+        final var delegateRef = new AtomicReference<>(delegate);
+        return new ActionListener<>() {
             @Override
-            protected void innerOnResponse(Response response) {
-                delegate.onResponse(response);
+            public void onResponse(Response response) {
+                final var acquired = delegateRef.getAndSet(null);
+                if (acquired != null) {
+                    acquired.onResponse(response);
+                }
             }
 
             @Override
-            protected void innerOnFailure(Exception e) {
-                delegate.onFailure(e);
+            public void onFailure(Exception e) {
+                final var acquired = delegateRef.getAndSet(null);
+                if (acquired != null) {
+                    acquired.onFailure(e);
+                }
+            }
+
+            @Override
+            public String toString() {
+                return "notifyOnce[" + delegateRef.get() + "]";
             }
         };
     }
@@ -458,4 +513,61 @@ static <Response> void completeWith(ActionListener<Response> listener, CheckedSu
             throw ex;
         }
     }
+
+    private static Runnable runnableFromReleasable(Releasable releasable) {
+        return new Runnable() {
+            @Override
+            public void run() {
+                Releasables.closeExpectNoException(releasable);
+            }
+
+            @Override
+            public String toString() {
+                return "release[" + releasable + "]";
+            }
+        };
+    }
+
+    static <Response> ActionListener<Response> assertOnce(ActionListener<Response> delegate) {
+        if (Assertions.ENABLED) {
+            return new ActionListener<>() {
+
+                // if complete, records the stack trace which first completed it
+                private final AtomicReference<ElasticsearchException> firstCompletion = new AtomicReference<>();
+
+                private void assertFirstRun() {
+                    var previousRun = firstCompletion.compareAndExchange(null, new ElasticsearchException(delegate.toString()));
+                    assert previousRun == null : previousRun; // reports the stack traces of both completions
+                }
+
+                @Override
+                public void onResponse(Response response) {
+                    assertFirstRun();
+                    delegate.onResponse(response);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    assertFirstRun();
+                    delegate.onFailure(e);
+                }
+
+                @Override
+                public String toString() {
+                    return delegate.toString();
+                }
+            };
+        } else {
+            return delegate;
+        }
+    }
+
+    static <T, L extends ActionListener<T>> void run(L listener, CheckedConsumer<L, Exception> action) {
+        try {
+            action.accept(listener);
+        } catch (Exception e) {
+            listener.onFailure(e);
+        }
+    }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java
index 172654fc915a..59d055e27415 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -41,6 +41,7 @@
 import org.elasticsearch.action.admin.cluster.node.reload.TransportNodesReloadSecureSettingsAction;
 import org.elasticsearch.action.admin.cluster.node.shutdown.PrevalidateNodeRemovalAction;
 import org.elasticsearch.action.admin.cluster.node.shutdown.TransportPrevalidateNodeRemovalAction;
+import org.elasticsearch.action.admin.cluster.node.shutdown.TransportPrevalidateShardPathAction;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsAction;
 import org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksAction;
@@ -707,6 +708,7 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
         actions.register(TransportNodesListShardStoreMetadata.TYPE, TransportNodesListShardStoreMetadata.class);
         actions.register(TransportShardFlushAction.TYPE, TransportShardFlushAction.class);
         actions.register(TransportShardRefreshAction.TYPE, TransportShardRefreshAction.class);
+        actions.register(TransportPrevalidateShardPathAction.TYPE, TransportPrevalidateShardPathAction.class);
 
         // desired nodes
         actions.register(GetDesiredNodesAction.INSTANCE, TransportGetDesiredNodesAction.class);
@@ -766,7 +768,7 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
         registerHandler.accept(new RestResetFeatureStateAction());
         registerHandler.accept(new RestGetFeatureUpgradeStatusAction());
         registerHandler.accept(new RestPostFeatureUpgradeAction());
-        registerHandler.accept(new RestGetIndicesAction(threadPool));
+        registerHandler.accept(new RestGetIndicesAction());
         registerHandler.accept(new RestIndicesStatsAction());
         registerHandler.accept(new RestIndicesSegmentsAction());
         registerHandler.accept(new RestIndicesShardStoresAction());
diff --git a/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java b/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java
index f2715710ed51..2226d3be323d 100644
--- a/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.action.support.WriteResponse;
@@ -118,7 +118,7 @@ public DocWriteResponse(ShardId shardId, String id, long seqNo, long primaryTerm
     protected DocWriteResponse(ShardId shardId, StreamInput in) throws IOException {
         super(in);
         this.shardId = shardId;
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             String type = in.readString();
             assert MapperService.SINGLE_MAPPING_NAME.equals(type) : "Expected [_doc] but received [" + type + "]";
         }
@@ -137,7 +137,7 @@ protected DocWriteResponse(ShardId shardId, StreamInput in) throws IOException {
     protected DocWriteResponse(StreamInput in) throws IOException {
         super(in);
         shardId = new ShardId(in);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             String type = in.readString();
             assert MapperService.SINGLE_MAPPING_NAME.equals(type) : "Expected [_doc] but received [" + type + "]";
         }
@@ -266,7 +266,7 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     private void writeWithoutShardId(StreamOutput out) throws IOException {
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeString(MapperService.SINGLE_MAPPING_NAME);
         }
         out.writeString(id);
diff --git a/server/src/main/java/org/elasticsearch/action/NoShardAvailableActionException.java b/server/src/main/java/org/elasticsearch/action/NoShardAvailableActionException.java
index 03690b632062..29df8ec55c9b 100644
--- a/server/src/main/java/org/elasticsearch/action/NoShardAvailableActionException.java
+++ b/server/src/main/java/org/elasticsearch/action/NoShardAvailableActionException.java
@@ -14,20 +14,36 @@
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
+import java.io.PrintWriter;
 
 public class NoShardAvailableActionException extends ElasticsearchException {
 
+    private static final StackTraceElement[] EMPTY_STACK_TRACE = new StackTraceElement[0];
+
+    // This is set so that no StackTrace is serialized in the scenario when we wrap other shard failures.
+    // It isn't necessary to serialize this field over the wire as the empty stack trace is serialized instead.
+    private final boolean onShardFailureWrapper;
+
+    public static NoShardAvailableActionException forOnShardFailureWrapper(String msg) {
+        return new NoShardAvailableActionException(null, msg, null, true);
+    }
+
     public NoShardAvailableActionException(ShardId shardId) {
-        this(shardId, null);
+        this(shardId, null, null, false);
     }
 
     public NoShardAvailableActionException(ShardId shardId, String msg) {
-        this(shardId, msg, null);
+        this(shardId, msg, null, false);
     }
 
     public NoShardAvailableActionException(ShardId shardId, String msg, Throwable cause) {
+        this(shardId, msg, cause, false);
+    }
+
+    private NoShardAvailableActionException(ShardId shardId, String msg, Throwable cause, boolean onShardFailureWrapper) {
         super(msg, cause);
         setShard(shardId);
+        this.onShardFailureWrapper = onShardFailureWrapper;
     }
 
     @Override
@@ -37,5 +53,22 @@ public RestStatus status() {
 
     public NoShardAvailableActionException(StreamInput in) throws IOException {
         super(in);
+        onShardFailureWrapper = false;
+    }
+
+    @Override
+    public StackTraceElement[] getStackTrace() {
+        return onShardFailureWrapper ? EMPTY_STACK_TRACE : super.getStackTrace();
+    }
+
+    @Override
+    public void printStackTrace(PrintWriter s) {
+        if (onShardFailureWrapper == false) {
+            super.printStackTrace(s);
+        } else {
+            // Override to simply print the first line of the trace, which is the current exception.
+            // Since we aren't serializing the repetitive stacktrace onShardFailureWrapper, we shouldn't print it out either
+            s.println(this);
+        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/NotifyOnceListener.java b/server/src/main/java/org/elasticsearch/action/NotifyOnceListener.java
deleted file mode 100644
index 582290f2a434..000000000000
--- a/server/src/main/java/org/elasticsearch/action/NotifyOnceListener.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.action;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * A listener that ensures that only one of onResponse or onFailure is called. And the method
- * the is called is only called once. Subclasses should implement notification logic with
- * innerOnResponse and innerOnFailure.
- */
-public abstract class NotifyOnceListener<Response> implements ActionListener<Response> {
-
-    private final AtomicBoolean hasBeenCalled = new AtomicBoolean(false);
-
-    protected abstract void innerOnResponse(Response response);
-
-    protected abstract void innerOnFailure(Exception e);
-
-    @Override
-    public final void onResponse(Response response) {
-        if (hasBeenCalled.compareAndSet(false, true)) {
-            innerOnResponse(response);
-        }
-    }
-
-    @Override
-    public final void onFailure(Exception e) {
-        if (hasBeenCalled.compareAndSet(false, true)) {
-            innerOnFailure(e);
-        }
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/action/RoutingMissingException.java b/server/src/main/java/org/elasticsearch/action/RoutingMissingException.java
index ab8a3f199230..000ba521b763 100644
--- a/server/src/main/java/org/elasticsearch/action/RoutingMissingException.java
+++ b/server/src/main/java/org/elasticsearch/action/RoutingMissingException.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.action;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.mapper.MapperService;
@@ -41,7 +41,7 @@ public RestStatus status() {
 
     public RoutingMissingException(StreamInput in) throws IOException {
         super(in);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             in.readString();
         }
         id = in.readString();
@@ -50,7 +50,7 @@ public RoutingMissingException(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeString(MapperService.SINGLE_MAPPING_NAME);
         }
         out.writeString(id);
diff --git a/server/src/main/java/org/elasticsearch/action/SingleResultDeduplicator.java b/server/src/main/java/org/elasticsearch/action/SingleResultDeduplicator.java
new file mode 100644
index 000000000000..273c542bc825
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/SingleResultDeduplicator.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action;
+
+import org.elasticsearch.action.support.ContextPreservingActionListener;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Consumer;
+
+/**
+ *
+ * Wraps an async action that consumes an {@link ActionListener} such that multiple invocations of {@link #execute(ActionListener)} can
+ * share the result from a single call to the wrapped action. This implementation is similar to {@link ResultDeduplicator} but offers
+ * stronger guarantees of not seeing a stale result ever. Concretely, every invocation of {@link #execute(ActionListener)} is guaranteed to
+ * be resolved with a response that has been computed at a time after the call to {@code execute} has been made. This allows this class to
+ * be used to deduplicate results from actions that produce results that change over time transparently.
+ *
+ * @param <T> Result type
+ */
+public final class SingleResultDeduplicator<T> {
+
+    private final ThreadContext threadContext;
+
+    /**
+     * List of listeners waiting for the execution after the current in-progress execution. If {@code null} then no execution is in
+     * progress currently, otherwise an execution is in progress and will trigger another execution that will resolve any listeners queued
+     * up here once done.
+     */
+    private List<ActionListener<T>> waitingListeners;
+
+    private final Consumer<ActionListener<T>> executeAction;
+
+    public SingleResultDeduplicator(ThreadContext threadContext, Consumer<ActionListener<T>> executeAction) {
+        this.threadContext = threadContext;
+        this.executeAction = executeAction;
+    }
+
+    /**
+     * Execute the action for the given {@code listener}.
+     * @param listener listener to resolve with execution result
+     */
+    public void execute(ActionListener<T> listener) {
+        synchronized (this) {
+            if (waitingListeners == null) {
+                // no queued up listeners, just execute this one directly without deduplication and instantiate the list so that
+                // subsequent executions will wait
+                waitingListeners = new ArrayList<>();
+            } else {
+                // already running an execution, queue this one up
+                waitingListeners.add(ContextPreservingActionListener.wrapPreservingContext(listener, threadContext));
+                return;
+            }
+        }
+        doExecute(listener);
+    }
+
+    private void doExecute(ActionListener<T> listener) {
+        final ActionListener<T> wrappedListener = ActionListener.runBefore(listener, () -> {
+            final List<ActionListener<T>> listeners;
+            synchronized (this) {
+                if (waitingListeners.isEmpty()) {
+                    // no listeners were queued up while this execution ran, so we just reset the state to not having a running execution
+                    waitingListeners = null;
+                    return;
+                } else {
+                    // we have queued up listeners, so we create a fresh list for the next execution and execute once to handle the
+                    // listeners currently queued up
+                    listeners = waitingListeners;
+                    waitingListeners = new ArrayList<>();
+                }
+            }
+            doExecute(new ActionListener<>() {
+                @Override
+                public void onResponse(T response) {
+                    ActionListener.onResponse(listeners, response);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    ActionListener.onFailure(listeners, e);
+                }
+            });
+        });
+        ActionListener.run(wrappedListener, executeAction::accept);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/StepListener.java b/server/src/main/java/org/elasticsearch/action/StepListener.java
index dab36040e3e4..e36b799b9290 100644
--- a/server/src/main/java/org/elasticsearch/action/StepListener.java
+++ b/server/src/main/java/org/elasticsearch/action/StepListener.java
@@ -14,6 +14,7 @@
 
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
 
@@ -40,7 +41,9 @@
  * }</pre>
  */
 
-public final class StepListener<Response> extends NotifyOnceListener<Response> {
+public final class StepListener<Response> implements ActionListener<Response> {
+
+    private final AtomicBoolean hasBeenCalled = new AtomicBoolean(false);
     private final ListenableFuture<Response> delegate;
 
     public StepListener() {
@@ -48,13 +51,17 @@ public StepListener() {
     }
 
     @Override
-    protected void innerOnResponse(Response response) {
-        delegate.onResponse(response);
+    public void onResponse(Response response) {
+        if (hasBeenCalled.compareAndSet(false, true)) {
+            delegate.onResponse(response);
+        }
     }
 
     @Override
-    protected void innerOnFailure(Exception e) {
-        delegate.onFailure(e);
+    public void onFailure(Exception e) {
+        if (hasBeenCalled.compareAndSet(false, true)) {
+            delegate.onFailure(e);
+        }
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
index dfb5c2ef02af..fd2721241fb9 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.cluster.allocation;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -67,7 +67,7 @@ public ClusterAllocationExplanation(
     }
 
     public ClusterAllocationExplanation(StreamInput in) throws IOException {
-        if (in.getVersion().onOrAfter(Version.V_7_15_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_15_0)) {
             this.specificShard = in.readBoolean();
         } else {
             this.specificShard = true; // suppress "this is a random shard" warning in BwC situations
@@ -81,7 +81,7 @@ public ClusterAllocationExplanation(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getVersion().onOrAfter(Version.V_7_15_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_15_0)) {
             out.writeBoolean(specificShard);
         } // else suppress "this is a random shard" warning in BwC situations
         shardRouting.writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponse.java
index a7695f5512fd..2b481d4cac3b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponse.java
@@ -7,36 +7,55 @@
  */
 package org.elasticsearch.action.admin.cluster.allocation;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.routing.AllocationId;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.allocation.allocator.ClusterBalanceStats;
 import org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceStats;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
-public class DesiredBalanceResponse extends ActionResponse implements ToXContentObject {
+import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.singleChunk;
+
+public class DesiredBalanceResponse extends ActionResponse implements ChunkedToXContentObject {
+
+    private static final TransportVersion CLUSTER_BALANCE_STATS_VERSION = TransportVersion.V_8_7_0;
 
     private final DesiredBalanceStats stats;
+    private final ClusterBalanceStats clusterBalanceStats;
     private final Map<String, Map<Integer, DesiredShards>> routingTable;
 
-    public DesiredBalanceResponse(DesiredBalanceStats stats, Map<String, Map<Integer, DesiredShards>> routingTable) {
+    public DesiredBalanceResponse(
+        DesiredBalanceStats stats,
+        ClusterBalanceStats clusterBalanceStats,
+        Map<String, Map<Integer, DesiredShards>> routingTable
+    ) {
         this.stats = stats;
+        this.clusterBalanceStats = clusterBalanceStats;
         this.routingTable = routingTable;
     }
 
     public static DesiredBalanceResponse from(StreamInput in) throws IOException {
         return new DesiredBalanceResponse(
             DesiredBalanceStats.readFrom(in),
+            in.getTransportVersion().onOrAfter(CLUSTER_BALANCE_STATS_VERSION)
+                ? ClusterBalanceStats.readFrom(in)
+                : ClusterBalanceStats.EMPTY,
             in.readImmutableMap(StreamInput::readString, v -> v.readImmutableMap(StreamInput::readVInt, DesiredShards::from))
         );
     }
@@ -44,6 +63,9 @@ public static DesiredBalanceResponse from(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         stats.writeTo(out);
+        if (out.getTransportVersion().onOrAfter(CLUSTER_BALANCE_STATS_VERSION)) {
+            clusterBalanceStats.writeTo(out);
+        }
         out.writeMap(
             routingTable,
             StreamOutput::writeString,
@@ -56,32 +78,38 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        {
-            builder.startObject("stats");
-            stats.toXContent(builder, params);
-            builder.endObject();
-        }
-        {
-            builder.startObject("routing_table");
-            for (Map.Entry<String, Map<Integer, DesiredShards>> indexEntry : routingTable.entrySet()) {
-                builder.startObject(indexEntry.getKey());
-                for (Map.Entry<Integer, DesiredShards> shardEntry : indexEntry.getValue().entrySet()) {
-                    builder.field(String.valueOf(shardEntry.getKey()));
-                    shardEntry.getValue().toXContent(builder, params);
-                }
-                builder.endObject();
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        return Iterators.concat(
+            singleChunk(
+                (builder, p) -> builder.startObject(),
+                (builder, p) -> builder.field("stats", stats),
+                (builder, p) -> builder.field("cluster_balance_stats", clusterBalanceStats),
+                (builder, p) -> builder.startObject("routing_table")
+            ),
+            routingTableToXContentChunked(),
+            singleChunk((builder, p) -> builder.endObject(), (builder, p) -> builder.endObject())
+        );
+    }
+
+    private Iterator<ToXContent> routingTableToXContentChunked() {
+        return routingTable.entrySet().stream().map(indexEntry -> (ToXContent) (builder, p) -> {
+            builder.startObject(indexEntry.getKey());
+            for (Map.Entry<Integer, DesiredShards> shardEntry : indexEntry.getValue().entrySet()) {
+                builder.field(String.valueOf(shardEntry.getKey()));
+                shardEntry.getValue().toXContent(builder, p);
             }
-            builder.endObject();
-        }
-        return builder.endObject();
+            return builder.endObject();
+        }).iterator();
     }
 
     public DesiredBalanceStats getStats() {
         return stats;
     }
 
+    public ClusterBalanceStats getClusterBalanceStats() {
+        return clusterBalanceStats;
+    }
+
     public Map<String, Map<Integer, DesiredShards>> getRoutingTable() {
         return routingTable;
     }
@@ -91,17 +119,24 @@ public boolean equals(Object o) {
         if (this == o) return true;
         return o instanceof DesiredBalanceResponse that
             && Objects.equals(stats, that.stats)
+            && Objects.equals(clusterBalanceStats, that.clusterBalanceStats)
             && Objects.equals(routingTable, that.routingTable);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(stats, routingTable);
+        return Objects.hash(stats, clusterBalanceStats, routingTable);
     }
 
     @Override
     public String toString() {
-        return "DesiredBalanceResponse{stats=" + stats + ", routingTable=" + routingTable + "}";
+        return "DesiredBalanceResponse{stats="
+            + stats
+            + ", clusterBalanceStats="
+            + clusterBalanceStats
+            + ", routingTable="
+            + routingTable
+            + "}";
     }
 
     public record DesiredShards(List<ShardView> current, ShardAssignmentView desired) implements Writeable, ToXContentObject {
@@ -139,21 +174,42 @@ public record ShardView(
         boolean relocatingNodeIsDesired,
         int shardId,
         String index,
-        AllocationId allocationId
+        @Nullable Double forecastWriteLoad,
+        @Nullable Long forecastShardSizeInBytes
     ) implements Writeable, ToXContentObject {
 
+        private static final TransportVersion ADD_FORECASTS_VERSION = TransportVersion.V_8_7_0;
+
         public static ShardView from(StreamInput in) throws IOException {
-            return new ShardView(
-                ShardRoutingState.fromValue(in.readByte()),
-                in.readBoolean(),
-                in.readOptionalString(),
-                in.readBoolean(),
-                in.readOptionalString(),
-                in.readBoolean(),
-                in.readVInt(),
-                in.readString(),
-                in.readOptionalWriteable(AllocationId::new)
-            );
+            if (in.getTransportVersion().onOrAfter(ADD_FORECASTS_VERSION)) {
+                return new ShardView(
+                    ShardRoutingState.fromValue(in.readByte()),
+                    in.readBoolean(),
+                    in.readOptionalString(),
+                    in.readBoolean(),
+                    in.readOptionalString(),
+                    in.readBoolean(),
+                    in.readVInt(),
+                    in.readString(),
+                    in.readOptionalDouble(),
+                    in.readOptionalLong()
+                );
+            } else {
+                var shardView = new ShardView(
+                    ShardRoutingState.fromValue(in.readByte()),
+                    in.readBoolean(),
+                    in.readOptionalString(),
+                    in.readBoolean(),
+                    in.readOptionalString(),
+                    in.readBoolean(),
+                    in.readVInt(),
+                    in.readString(),
+                    null,
+                    null
+                );
+                in.readOptionalWriteable(AllocationId::new);
+                return shardView;
+            }
         }
 
         @Override
@@ -166,7 +222,12 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(relocatingNodeIsDesired);
             out.writeVInt(shardId);
             out.writeString(index);
-            out.writeOptionalWriteable(allocationId);
+            if (out.getTransportVersion().onOrAfter(ADD_FORECASTS_VERSION)) {
+                out.writeOptionalDouble(forecastWriteLoad);
+                out.writeOptionalLong(forecastShardSizeInBytes);
+            } else {
+                out.writeMissingWriteable(AllocationId.class);
+            }
         }
 
         @Override
@@ -180,6 +241,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 .field("relocating_node_is_desired", relocatingNodeIsDesired)
                 .field("shard_id", shardId)
                 .field("index", index)
+                .field("forecast_write_load", forecastWriteLoad)
+                .field("forecast_shard_size_in_bytes", forecastShardSizeInBytes)
                 .endObject();
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java
index e2b40605798e..0ad58e9b987c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceAction.java
@@ -11,14 +11,17 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
-import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster;
+import org.elasticsearch.cluster.routing.allocation.allocator.ClusterBalanceStats;
 import org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalance;
 import org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.allocator.ShardAssignment;
@@ -34,11 +37,15 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.OptionalDouble;
+import java.util.OptionalLong;
 
 public class TransportGetDesiredBalanceAction extends TransportMasterNodeReadAction<DesiredBalanceRequest, DesiredBalanceResponse> {
 
     @Nullable
     private final DesiredBalanceShardsAllocator desiredBalanceShardsAllocator;
+    private final ClusterInfoService clusterInfoService;
+    private final WriteLoadForecaster writeLoadForecaster;
 
     @Inject
     public TransportGetDesiredBalanceAction(
@@ -47,7 +54,9 @@ public TransportGetDesiredBalanceAction(
         ThreadPool threadPool,
         ActionFilters actionFilters,
         IndexNameExpressionResolver indexNameExpressionResolver,
-        ShardsAllocator shardsAllocator
+        ShardsAllocator shardsAllocator,
+        ClusterInfoService clusterInfoService,
+        WriteLoadForecaster writeLoadForecaster
     ) {
         super(
             GetDesiredBalanceAction.NAME,
@@ -60,9 +69,9 @@ public TransportGetDesiredBalanceAction(
             DesiredBalanceResponse::from,
             ThreadPool.Names.MANAGEMENT
         );
-        this.desiredBalanceShardsAllocator = shardsAllocator instanceof DesiredBalanceShardsAllocator
-            ? (DesiredBalanceShardsAllocator) shardsAllocator
-            : null;
+        this.desiredBalanceShardsAllocator = shardsAllocator instanceof DesiredBalanceShardsAllocator allocator ? allocator : null;
+        this.clusterInfoService = clusterInfoService;
+        this.writeLoadForecaster = writeLoadForecaster;
     }
 
     @Override
@@ -72,13 +81,8 @@ protected void masterOperation(
         ClusterState state,
         ActionListener<DesiredBalanceResponse> listener
     ) throws Exception {
-        String allocatorName = ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING.get(state.metadata().settings());
-        if (allocatorName.equals(ClusterModule.DESIRED_BALANCE_ALLOCATOR) == false || desiredBalanceShardsAllocator == null) {
-            listener.onFailure(
-                new ResourceNotFoundException(
-                    "Expected the shard balance allocator to be `desired_balance`, but got `" + allocatorName + "`"
-                )
-            );
+        if (desiredBalanceShardsAllocator == null) {
+            listener.onFailure(new ResourceNotFoundException("Desired balance allocator is not in use, no desired balance found"));
             return;
         }
 
@@ -87,15 +91,31 @@ protected void masterOperation(
             listener.onFailure(new ResourceNotFoundException("Desired balance is not computed yet"));
             return;
         }
+        listener.onResponse(
+            new DesiredBalanceResponse(
+                desiredBalanceShardsAllocator.getStats(),
+                ClusterBalanceStats.createFrom(state, clusterInfoService.getClusterInfo(), writeLoadForecaster),
+                createRoutingTable(state, latestDesiredBalance)
+            )
+        );
+    }
+
+    private Map<String, Map<Integer, DesiredBalanceResponse.DesiredShards>> createRoutingTable(
+        ClusterState state,
+        DesiredBalance latestDesiredBalance
+    ) {
         Map<String, Map<Integer, DesiredBalanceResponse.DesiredShards>> routingTable = new HashMap<>();
         for (IndexRoutingTable indexRoutingTable : state.routingTable()) {
             Map<Integer, DesiredBalanceResponse.DesiredShards> indexDesiredShards = new HashMap<>();
+            IndexMetadata indexMetadata = state.metadata().index(indexRoutingTable.getIndex());
             for (int shardId = 0; shardId < indexRoutingTable.size(); shardId++) {
                 IndexShardRoutingTable shardRoutingTable = indexRoutingTable.shard(shardId);
                 ShardAssignment shardAssignment = latestDesiredBalance.assignments().get(shardRoutingTable.shardId());
                 List<DesiredBalanceResponse.ShardView> shardViews = new ArrayList<>();
                 for (int idx = 0; idx < shardRoutingTable.size(); idx++) {
                     ShardRouting shard = shardRoutingTable.shard(idx);
+                    OptionalDouble forecastedWriteLoad = writeLoadForecaster.getForecastedWriteLoad(indexMetadata);
+                    OptionalLong forecastedShardSizeInBytes = indexMetadata.getForecastedShardSizeInBytes();
                     shardViews.add(
                         new DesiredBalanceResponse.ShardView(
                             shard.state(),
@@ -110,7 +130,8 @@ protected void masterOperation(
                                 && shardAssignment.nodeIds().contains(shard.relocatingNodeId()),
                             shard.shardId().id(),
                             shard.getIndexName(),
-                            shard.allocationId()
+                            forecastedWriteLoad.isPresent() ? forecastedWriteLoad.getAsDouble() : null,
+                            forecastedShardSizeInBytes.isPresent() ? forecastedShardSizeInBytes.getAsLong() : null
                         )
                     );
                 }
@@ -131,7 +152,7 @@ protected void masterOperation(
             }
             routingTable.put(indexRoutingTable.getIndex().getName(), indexDesiredShards);
         }
-        listener.onResponse(new DesiredBalanceResponse(desiredBalanceShardsAllocator.getStats(), routingTable));
+        return routingTable;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java
index 75d512683e31..01bd97bdc5fb 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequest.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.action.admin.cluster.configuration;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.cluster.ClusterState;
@@ -72,7 +72,7 @@ public AddVotingConfigExclusionsRequest(String[] nodeIds, String[] nodeNames, Ti
 
     public AddVotingConfigExclusionsRequest(StreamInput in) throws IOException {
         super(in);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             final String[] legacyNodeDescriptions = in.readStringArray();
             if (legacyNodeDescriptions.length > 0) {
                 throw new IllegalArgumentException("legacy [node_name] field was deprecated and must be empty");
@@ -185,7 +185,7 @@ public ActionRequestValidationException validate() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeStringArray(Strings.EMPTY_ARRAY);
         }
         out.writeStringArray(nodeIds);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java
index b0d7000afa8a..e1fcf41de5eb 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java
@@ -79,9 +79,9 @@ protected void masterOperation(
         Task task,
         UpdateDesiredNodesRequest request,
         ClusterState state,
-        ActionListener<UpdateDesiredNodesResponse> listener
+        ActionListener<UpdateDesiredNodesResponse> responseListener
     ) throws Exception {
-        try {
+        ActionListener.run(responseListener, listener -> {
             settingsValidator.validate(request.getNodes());
             clusterService.submitStateUpdateTask(
                 "update-desired-nodes",
@@ -89,15 +89,13 @@ protected void masterOperation(
                 ClusterStateTaskConfig.build(Priority.URGENT, request.masterNodeTimeout()),
                 taskExecutor
             );
-        } catch (Exception e) {
-            listener.onFailure(e);
-        }
+        });
     }
 
     @Override
     protected void doExecute(Task task, UpdateDesiredNodesRequest request, ActionListener<UpdateDesiredNodesResponse> listener) {
         final var minNodeVersion = clusterService.state().nodes().getMinNodeVersion();
-        if (request.isCompatibleWithVersion(minNodeVersion) == false) {
+        if (request.isCompatibleWithVersion(minNodeVersion.transportVersion) == false) {
             listener.onFailure(
                 new IllegalArgumentException(
                     "Unable to use processor ranges, floating-point (with greater precision) processors "
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesRequest.java
index b4d7ffbc3d0d..825db3c31a99 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.cluster.desirednodes;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
@@ -24,7 +24,7 @@
 import java.util.Objects;
 
 public class UpdateDesiredNodesRequest extends AcknowledgedRequest<UpdateDesiredNodesRequest> {
-    private static final Version DRY_RUN_VERSION = Version.V_8_4_0;
+    private static final TransportVersion DRY_RUN_VERSION = TransportVersion.V_8_4_0;
 
     private final String historyID;
     private final long version;
@@ -58,7 +58,7 @@ public UpdateDesiredNodesRequest(StreamInput in) throws IOException {
         this.historyID = in.readString();
         this.version = in.readLong();
         this.nodes = in.readList(DesiredNode::readFrom);
-        if (in.getVersion().onOrAfter(DRY_RUN_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(DRY_RUN_VERSION)) {
             this.dryRun = in.readBoolean();
         } else {
             this.dryRun = false;
@@ -71,7 +71,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(historyID);
         out.writeLong(version);
         out.writeList(nodes);
-        if (out.getVersion().onOrAfter(DRY_RUN_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(DRY_RUN_VERSION)) {
             out.writeBoolean(dryRun);
         }
     }
@@ -98,7 +98,7 @@ public boolean isDryRun() {
         return dryRun;
     }
 
-    public boolean isCompatibleWithVersion(Version version) {
+    public boolean isCompatibleWithVersion(TransportVersion version) {
         if (version.onOrAfter(DesiredNode.RANGE_FLOAT_PROCESSORS_SUPPORT_VERSION)) {
             return true;
         }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesResponse.java
index 5cf4354099ea..0d99cc9d693e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesResponse.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.cluster.desirednodes;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -19,7 +19,7 @@
 import java.util.Objects;
 
 public class UpdateDesiredNodesResponse extends ActionResponse implements ToXContentObject {
-    private static final Version DRY_RUN_SUPPORTING_VERSION = Version.V_8_4_0;
+    private static final TransportVersion DRY_RUN_SUPPORTING_VERSION = TransportVersion.V_8_4_0;
 
     private final boolean replacedExistingHistoryId;
     private final boolean dryRun;
@@ -36,13 +36,13 @@ public UpdateDesiredNodesResponse(boolean replacedExistingHistoryId, boolean dry
     public UpdateDesiredNodesResponse(StreamInput in) throws IOException {
         super(in);
         this.replacedExistingHistoryId = in.readBoolean();
-        dryRun = in.getVersion().onOrAfter(DRY_RUN_SUPPORTING_VERSION) ? in.readBoolean() : false;
+        dryRun = in.getTransportVersion().onOrAfter(DRY_RUN_SUPPORTING_VERSION) ? in.readBoolean() : false;
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(replacedExistingHistoryId);
-        if (out.getVersion().onOrAfter(DRY_RUN_SUPPORTING_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(DRY_RUN_SUPPORTING_VERSION)) {
             out.writeBoolean(dryRun);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java
index 6fa578fbd05b..74ac542214a9 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java
@@ -335,7 +335,7 @@ public String toString() {
                 + version
                 + '\''
                 + ", exception='"
-                + exception.getMessage()
+                + (exception == null ? "null" : exception.getMessage())
                 + "'"
                 + '}';
         }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java
index 3d4b690c5bba..e87427c3fdef 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.cluster.node.hotthreads;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -36,7 +36,7 @@ public NodesHotThreadsRequest(StreamInput in) throws IOException {
         type = HotThreads.ReportType.of(in.readString());
         interval = in.readTimeValue();
         snapshots = in.readInt();
-        if (in.getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             sortOrder = HotThreads.SortOrder.of(in.readString());
         }
     }
@@ -118,7 +118,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(type.getTypeValue());
         out.writeTimeValue(interval);
         out.writeInt(snapshots);
-        if (out.getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             out.writeString(sortOrder.getOrderValue());
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
index ca9be0a65178..03f66b5e6d2f 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.action.admin.cluster.node.info;
 
 import org.elasticsearch.Build;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.support.nodes.BaseNodeResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -72,7 +73,7 @@ public NodeInfo(StreamInput in) throws IOException {
         addInfoIfNonNull(HttpInfo.class, in.readOptionalWriteable(HttpInfo::new));
         addInfoIfNonNull(PluginsAndModules.class, in.readOptionalWriteable(PluginsAndModules::new));
         addInfoIfNonNull(IngestInfo.class, in.readOptionalWriteable(IngestInfo::new));
-        if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             addInfoIfNonNull(AggregationInfo.class, in.readOptionalWriteable(AggregationInfo::new));
         }
     }
@@ -193,7 +194,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(getInfo(HttpInfo.class));
         out.writeOptionalWriteable(getInfo(PluginsAndModules.class));
         out.writeOptionalWriteable(getInfo(IngestInfo.class));
-        if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             out.writeOptionalWriteable(getInfo(AggregationInfo.class));
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java
index 974e90b11d8d..e6a91b152cad 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/PluginsAndModules.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.cluster.node.info;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.node.ReportingService;
@@ -41,7 +41,7 @@ public PluginsAndModules(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getVersion().onOrAfter(Version.V_8_3_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0)) {
             out.writeList(plugins);
         } else {
             out.writeList(plugins.stream().map(PluginRuntimeInfo::descriptor).toList());
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java
index 756e4312784a..be542d375aa4 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java
@@ -8,6 +8,8 @@
 
 package org.elasticsearch.action.admin.cluster.node.reload;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
@@ -37,6 +39,8 @@ public class TransportNodesReloadSecureSettingsAction extends TransportNodesActi
     NodesReloadSecureSettingsRequest.NodeRequest,
     NodesReloadSecureSettingsResponse.NodeResponse> {
 
+    private static final Logger logger = LogManager.getLogger(TransportNodesReloadSecureSettingsAction.class);
+
     private final Environment environment;
     private final PluginsService pluginsService;
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodePrevalidateShardPathRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodePrevalidateShardPathRequest.java
new file mode 100644
index 000000000000..c7964a8b4dfd
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodePrevalidateShardPathRequest.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.cluster.node.shutdown;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.transport.TransportRequest;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * A node-specific request derived from the corresponding {@link PrevalidateShardPathRequest}.
+*/
+public class NodePrevalidateShardPathRequest extends TransportRequest {
+
+    private final Set<ShardId> shardIds;
+
+    public NodePrevalidateShardPathRequest(Collection<ShardId> shardIds) {
+        this.shardIds = Set.copyOf(Objects.requireNonNull(shardIds));
+    }
+
+    public NodePrevalidateShardPathRequest(StreamInput in) throws IOException {
+        super(in);
+        this.shardIds = Set.copyOf(Objects.requireNonNull(in.readSet(ShardId::new)));
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeCollection(shardIds, (o, value) -> value.writeTo(o));
+    }
+
+    public Set<ShardId> getShardIds() {
+        return shardIds;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o instanceof NodePrevalidateShardPathRequest == false) return false;
+        NodePrevalidateShardPathRequest other = (NodePrevalidateShardPathRequest) o;
+        return Objects.equals(shardIds, other.shardIds);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(shardIds);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodePrevalidateShardPathResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodePrevalidateShardPathResponse.java
new file mode 100644
index 000000000000..9b23439f1c07
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodePrevalidateShardPathResponse.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.cluster.node.shutdown;
+
+import org.elasticsearch.action.support.nodes.BaseNodeResponse;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.shard.ShardId;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.Set;
+
+public class NodePrevalidateShardPathResponse extends BaseNodeResponse {
+
+    private final Set<ShardId> shardIds;
+
+    protected NodePrevalidateShardPathResponse(DiscoveryNode node, Set<ShardId> shardIds) {
+        super(node);
+        this.shardIds = Set.copyOf(Objects.requireNonNull(shardIds));
+    }
+
+    protected NodePrevalidateShardPathResponse(StreamInput in) throws IOException {
+        super(in);
+        shardIds = Set.copyOf(Objects.requireNonNull(in.readSet(ShardId::new)));
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeCollection(shardIds);
+    }
+
+    public Set<ShardId> getShardIds() {
+        return shardIds;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o instanceof NodePrevalidateShardPathResponse == false) return false;
+        NodePrevalidateShardPathResponse other = (NodePrevalidateShardPathResponse) o;
+        return Objects.equals(shardIds, other.shardIds) && Objects.equals(getNode(), other.getNode());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(shardIds, getNode());
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodesRemovalPrevalidation.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodesRemovalPrevalidation.java
index 0115ee973c1f..4f6ea0d5b83d 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodesRemovalPrevalidation.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodesRemovalPrevalidation.java
@@ -8,9 +8,11 @@
 
 package org.elasticsearch.action.admin.cluster.node.shutdown;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -118,15 +120,19 @@ public static NodeResult fromXContent(XContentParser parser) throws IOException
         }
     }
 
-    // The prevalidation result of a node
-    public record Result(boolean isSafe, String message) implements ToXContentObject, Writeable {
+    /**
+     *  The prevalidation result of a node.
+     * @param reason is nullable only for BWC between 8.6 and 8.7. In a fully-upgraded 8.7, it should always be non-null.
+     */
+    public record Result(boolean isSafe, @Nullable Reason reason, String message) implements ToXContentObject, Writeable {
 
         private static final ParseField IS_SAFE_FIELD = new ParseField("is_safe");
+        private static final ParseField REASON_FIELD = new ParseField("reason");
         private static final ParseField MESSAGE_FIELD = new ParseField("message");
 
         private static final ConstructingObjectParser<Result, Void> PARSER = new ConstructingObjectParser<>(
             "nodes_removal_prevalidation_result",
-            objects -> new Result((boolean) objects[0], (String) objects[1])
+            objects -> new Result((boolean) objects[0], Reason.fromString((String) objects[1]), (String) objects[2])
         );
 
         static {
@@ -135,23 +141,33 @@ public record Result(boolean isSafe, String message) implements ToXContentObject
 
         static <T> void configureParser(ConstructingObjectParser<T, Void> parser) {
             parser.declareBoolean(ConstructingObjectParser.constructorArg(), IS_SAFE_FIELD);
+            parser.declareString(ConstructingObjectParser.constructorArg(), REASON_FIELD);
             parser.declareString(ConstructingObjectParser.constructorArg(), MESSAGE_FIELD);
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(isSafe);
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+                reason.writeTo(out);
+            }
             out.writeString(message);
         }
 
         public static Result readFrom(final StreamInput in) throws IOException {
-            return new Result(in.readBoolean(), in.readString());
+            if (in.getTransportVersion().before(TransportVersion.V_8_7_0)) {
+                return new Result(in.readBoolean(), null, in.readString());
+            }
+            return new Result(in.readBoolean(), Reason.readFrom(in), in.readString());
         }
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
             builder.field(IS_SAFE_FIELD.getPreferredName(), isSafe);
+            if (reason != null) {
+                builder.field(REASON_FIELD.getPreferredName(), reason.reason);
+            }
             builder.field(MESSAGE_FIELD.getPreferredName(), message);
             builder.endObject();
             return builder;
@@ -161,4 +177,40 @@ public static Result fromXContent(XContentParser parser) throws IOException {
             return PARSER.parse(parser, null);
         }
     }
+
+    public enum Reason implements Writeable {
+        NO_PROBLEMS("no_problems"),
+        NO_RED_SHARDS_ON_NODE("no_red_shards_on_node"),
+        NO_RED_SHARDS_EXCEPT_SEARCHABLE_SNAPSHOTS("no_red_shards_except_searchable_snapshots"),
+        RED_SHARDS_ON_NODE("red_shards_on_node"),
+        UNABLE_TO_VERIFY("unable_to_verify_red_shards");
+
+        private final String reason;
+
+        Reason(String reason) {
+            this.reason = reason;
+        }
+
+        public String reason() {
+            return reason;
+        }
+
+        public static Reason readFrom(final StreamInput in) throws IOException {
+            return fromString(in.readString());
+        }
+
+        public static Reason fromString(String s) {
+            for (Reason r : values()) {
+                if (s.equalsIgnoreCase(r.reason)) {
+                    return r;
+                }
+            }
+            throw new IllegalArgumentException("unexpected Reason value [" + s + "]");
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(reason);
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalRequest.java
index 8f5dd9a0f83f..507309056793 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalRequest.java
@@ -8,11 +8,13 @@
 
 package org.elasticsearch.action.admin.cluster.node.shutdown;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -29,6 +31,7 @@ public class PrevalidateNodeRemovalRequest extends MasterNodeReadRequest<Prevali
     private final String[] names;
     private final String[] ids;
     private final String[] externalIds;
+    private TimeValue timeout = TimeValue.timeValueSeconds(30);
 
     private PrevalidateNodeRemovalRequest(Builder builder) {
         this.names = builder.names;
@@ -41,6 +44,9 @@ public PrevalidateNodeRemovalRequest(final StreamInput in) throws IOException {
         names = in.readStringArray();
         ids = in.readStringArray();
         externalIds = in.readStringArray();
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            timeout = in.readTimeValue();
+        }
     }
 
     @Override
@@ -49,6 +55,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeStringArray(names);
         out.writeStringArray(ids);
         out.writeStringArray(externalIds);
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            out.writeTimeValue(timeout);
+        }
     }
 
     @Override
@@ -79,6 +88,18 @@ public String[] getExternalIds() {
         return externalIds;
     }
 
+    public TimeValue timeout() {
+        return timeout;
+    }
+
+    public PrevalidateNodeRemovalRequest timeout(TimeValue timeout) {
+        this.timeout = timeout;
+        if (masterNodeTimeout == DEFAULT_MASTER_NODE_TIMEOUT) {
+            masterNodeTimeout = timeout;
+        }
+        return this;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateShardPathRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateShardPathRequest.java
new file mode 100644
index 000000000000..847ffd26d0ed
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateShardPathRequest.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.cluster.node.shutdown;
+
+import org.elasticsearch.action.support.nodes.BaseNodesRequest;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.shard.ShardId;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.Set;
+
+public class PrevalidateShardPathRequest extends BaseNodesRequest<PrevalidateShardPathRequest> {
+
+    private final Set<ShardId> shardIds;
+
+    public PrevalidateShardPathRequest(Set<ShardId> shardIds, String... nodeIds) {
+        super(nodeIds);
+        this.shardIds = Set.copyOf(Objects.requireNonNull(shardIds));
+    }
+
+    public PrevalidateShardPathRequest(StreamInput in) throws IOException {
+        super(in);
+        this.shardIds = Set.copyOf(Objects.requireNonNull(in.readSet(ShardId::new)));
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeCollection(shardIds);
+    }
+
+    public Set<ShardId> getShardIds() {
+        return shardIds;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o instanceof PrevalidateShardPathRequest == false) return false;
+        PrevalidateShardPathRequest other = (PrevalidateShardPathRequest) o;
+        return Objects.equals(shardIds, other.shardIds)
+            && Arrays.equals(nodesIds(), other.nodesIds())
+            && Objects.equals(timeout(), other.timeout());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(shardIds, Arrays.hashCode(nodesIds()), timeout());
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateShardPathResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateShardPathResponse.java
new file mode 100644
index 000000000000..2bbaa6eb2827
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateShardPathResponse.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.cluster.node.shutdown;
+
+import org.elasticsearch.action.FailedNodeException;
+import org.elasticsearch.action.support.nodes.BaseNodesResponse;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.List;
+
+public class PrevalidateShardPathResponse extends BaseNodesResponse<NodePrevalidateShardPathResponse> {
+
+    public PrevalidateShardPathResponse(
+        ClusterName clusterName,
+        List<NodePrevalidateShardPathResponse> nodes,
+        List<FailedNodeException> failures
+    ) {
+        super(clusterName, nodes, failures);
+    }
+
+    public PrevalidateShardPathResponse(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    protected List<NodePrevalidateShardPathResponse> readNodesFrom(StreamInput in) throws IOException {
+        return in.readList(NodePrevalidateShardPathResponse::new);
+    }
+
+    @Override
+    protected void writeNodesTo(StreamOutput out, List<NodePrevalidateShardPathResponse> nodes) throws IOException {
+        out.writeList(nodes);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateNodeRemovalAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateNodeRemovalAction.java
index 44a12dd54936..f3ce39a0cdbf 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateNodeRemovalAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateNodeRemovalAction.java
@@ -10,8 +10,10 @@
 
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
+import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
@@ -22,12 +24,16 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
@@ -46,13 +52,16 @@ public class TransportPrevalidateNodeRemovalAction extends TransportMasterNodeRe
 
     private static final Logger logger = LogManager.getLogger(TransportPrevalidateNodeRemovalAction.class);
 
+    private final NodeClient client;
+
     @Inject
     public TransportPrevalidateNodeRemovalAction(
         TransportService transportService,
         ClusterService clusterService,
         ThreadPool threadPool,
         ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        NodeClient client
     ) {
         super(
             PrevalidateNodeRemovalAction.NAME,
@@ -66,6 +75,7 @@ public TransportPrevalidateNodeRemovalAction(
             PrevalidateNodeRemovalResponse::new,
             ThreadPool.Names.SAME
         );
+        this.client = client;
     }
 
     @Override
@@ -73,14 +83,12 @@ protected void masterOperation(
         Task task,
         PrevalidateNodeRemovalRequest request,
         ClusterState state,
-        ActionListener<PrevalidateNodeRemovalResponse> listener
+        ActionListener<PrevalidateNodeRemovalResponse> responseListener
     ) {
-        try {
-            Set<DiscoveryNode> discoveryNodes = resolveNodes(request, state.nodes());
-            doPrevalidation(discoveryNodes, state, listener);
-        } catch (Exception e) {
-            listener.onFailure(e);
-        }
+        ActionListener.run(responseListener, listener -> {
+            Set<DiscoveryNode> requestNodes = resolveNodes(request, state.nodes());
+            doPrevalidation(request, requestNodes, state, listener);
+        });
     }
 
     public static Set<DiscoveryNode> resolveNodes(PrevalidateNodeRemovalRequest request, DiscoveryNodes discoveryNodes) {
@@ -141,62 +149,157 @@ protected ClusterBlockException checkBlock(PrevalidateNodeRemovalRequest request
     }
 
     private void doPrevalidation(
-        Set<DiscoveryNode> nodes,
+        PrevalidateNodeRemovalRequest request,
+        Set<DiscoveryNode> requestNodes,
         ClusterState clusterState,
         ActionListener<PrevalidateNodeRemovalResponse> listener
     ) {
-        assert nodes != null && nodes.isEmpty() == false;
+        assert requestNodes != null && requestNodes.isEmpty() == false;
 
-        logger.debug(() -> "prevalidate node removal for nodes " + nodes);
+        logger.debug(() -> "prevalidate node removal for nodes " + requestNodes);
         ClusterStateHealth clusterStateHealth = new ClusterStateHealth(clusterState);
         Metadata metadata = clusterState.metadata();
-        switch (clusterStateHealth.getStatus()) {
-            case GREEN, YELLOW -> {
-                List<NodeResult> nodesResults = nodes.stream()
-                    .map(dn -> new NodeResult(dn.getName(), dn.getId(), dn.getExternalId(), new Result(true, "")))
-                    .toList();
-                listener.onResponse(
-                    new PrevalidateNodeRemovalResponse(new NodesRemovalPrevalidation(true, "cluster status is not RED", nodesResults))
-                );
-            }
-            case RED -> {
-                Set<String> redIndices = clusterStateHealth.getIndices()
-                    .entrySet()
-                    .stream()
-                    .filter(entry -> entry.getValue().getStatus() == ClusterHealthStatus.RED)
-                    .map(Map.Entry::getKey)
-                    .collect(Collectors.toSet());
-                // If all red indices are searchable snapshot indices, it is safe to remove any node.
-                Set<String> redNonSSIndices = redIndices.stream()
-                    .map(metadata::index)
-                    .filter(i -> i.isSearchableSnapshot() == false)
-                    .map(im -> im.getIndex().getName())
-                    .collect(Collectors.toSet());
-                if (redNonSSIndices.isEmpty()) {
-                    List<NodeResult> nodeResults = nodes.stream()
-                        .map(dn -> new NodeResult(dn.getName(), dn.getId(), dn.getExternalId(), new Result(true, "")))
-                        .toList();
-                    listener.onResponse(
-                        new PrevalidateNodeRemovalResponse(
-                            new NodesRemovalPrevalidation(true, "all red indices are searchable snapshot indices", nodeResults)
-                        )
-                    );
-                } else {
-                    List<NodeResult> nodeResults = nodes.stream()
-                        .map(
-                            dn -> new NodeResult(
-                                dn.getName(),
-                                dn.getId(),
-                                dn.getExternalId(),
-                                new Result(false, "node may contain a copy of a red index shard")
-                            )
-                        )
-                        .toList();
-                    listener.onResponse(
-                        new PrevalidateNodeRemovalResponse(new NodesRemovalPrevalidation(false, "cluster health is RED", nodeResults))
-                    );
+        DiscoveryNodes clusterNodes = clusterState.getNodes();
+        if (clusterStateHealth.getStatus() == ClusterHealthStatus.GREEN || clusterStateHealth.getStatus() == ClusterHealthStatus.YELLOW) {
+            List<NodeResult> nodesResults = requestNodes.stream()
+                .map(
+                    dn -> new NodeResult(
+                        dn.getName(),
+                        dn.getId(),
+                        dn.getExternalId(),
+                        new Result(true, NodesRemovalPrevalidation.Reason.NO_PROBLEMS, "")
+                    )
+                )
+                .toList();
+            listener.onResponse(
+                new PrevalidateNodeRemovalResponse(new NodesRemovalPrevalidation(true, "cluster status is not RED", nodesResults))
+            );
+            return;
+        }
+        // RED cluster state
+        Set<String> redIndices = clusterStateHealth.getIndices()
+            .entrySet()
+            .stream()
+            .filter(entry -> entry.getValue().getStatus() == ClusterHealthStatus.RED)
+            .map(Map.Entry::getKey)
+            .collect(Collectors.toSet());
+        // If all red indices are searchable snapshot indices, it is safe to remove any node.
+        Set<String> redNonSSIndices = redIndices.stream()
+            .map(metadata::index)
+            .filter(i -> i.isSearchableSnapshot() == false)
+            .map(im -> im.getIndex().getName())
+            .collect(Collectors.toSet());
+        if (redNonSSIndices.isEmpty()) {
+            List<NodeResult> nodeResults = requestNodes.stream()
+                .map(
+                    dn -> new NodeResult(
+                        dn.getName(),
+                        dn.getId(),
+                        dn.getExternalId(),
+                        new Result(true, NodesRemovalPrevalidation.Reason.NO_RED_SHARDS_EXCEPT_SEARCHABLE_SNAPSHOTS, "")
+                    )
+                )
+                .toList();
+            listener.onResponse(
+                new PrevalidateNodeRemovalResponse(
+                    new NodesRemovalPrevalidation(true, "all red indices are searchable snapshot indices", nodeResults)
+                )
+            );
+        } else {
+            // Reach out to the nodes to find out whether they contain copies of the red non-searchable-snapshot indices
+            Set<ShardId> redShards = clusterStateHealth.getIndices()
+                .entrySet()
+                .stream()
+                .filter(indexHealthEntry -> redNonSSIndices.contains(indexHealthEntry.getKey()))
+                .map(Map.Entry::getValue) // ClusterHealthIndex of red non-searchable-snapshot indices
+                .flatMap(
+                    redIndexHealth -> redIndexHealth.getShards()
+                        .values()
+                        .stream()
+                        .filter(shardHealth -> shardHealth.getStatus() == ClusterHealthStatus.RED)
+                        .map(redShardHealth -> Tuple.tuple(redIndexHealth.getIndex(), redShardHealth))
+                ) // (Index, ClusterShardHealth) of all red shards
+                .map(
+                    redIndexShardHealthTuple -> new ShardId(
+                        metadata.index(redIndexShardHealthTuple.v1()).getIndex(),
+                        redIndexShardHealthTuple.v2().getShardId()
+                    )
+                ) // Convert to ShardId
+                .collect(Collectors.toSet());
+            var nodeIds = requestNodes.stream().map(DiscoveryNode::getId).toList().toArray(new String[0]);
+            var checkShardsRequest = new PrevalidateShardPathRequest(redShards, nodeIds).timeout(request.timeout());
+            client.execute(TransportPrevalidateShardPathAction.TYPE, checkShardsRequest, new ActionListener<>() {
+                @Override
+                public void onResponse(PrevalidateShardPathResponse response) {
+                    listener.onResponse(new PrevalidateNodeRemovalResponse(createPrevalidationResult(clusterNodes, response)));
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    listener.onFailure(e);
                 }
+            });
+        }
+    }
+
+    private NodesRemovalPrevalidation createPrevalidationResult(DiscoveryNodes nodes, PrevalidateShardPathResponse response) {
+        List<NodeResult> nodeResults = new ArrayList<>(response.getNodes().size() + response.failures().size());
+        for (NodePrevalidateShardPathResponse nodeResponse : response.getNodes()) {
+            Result result;
+            if (nodeResponse.getShardIds().isEmpty()) {
+                result = new Result(true, NodesRemovalPrevalidation.Reason.NO_RED_SHARDS_ON_NODE, "");
+            } else {
+                result = new Result(
+                    false,
+                    NodesRemovalPrevalidation.Reason.RED_SHARDS_ON_NODE,
+                    Strings.format("node contains copies of the following red shards: %s", nodeResponse.getShardIds())
+                );
             }
+            nodeResults.add(
+                new NodeResult(
+                    nodeResponse.getNode().getName(),
+                    nodeResponse.getNode().getId(),
+                    nodeResponse.getNode().getExternalId(),
+                    result
+                )
+            );
+        }
+        for (FailedNodeException failedResponse : response.failures()) {
+            DiscoveryNode node = nodes.get(failedResponse.nodeId());
+            nodeResults.add(
+                new NodeResult(
+                    node.getName(),
+                    node.getId(),
+                    node.getExternalId(),
+                    new Result(
+                        false,
+                        NodesRemovalPrevalidation.Reason.UNABLE_TO_VERIFY,
+                        Strings.format("failed contacting the node: %s", failedResponse.getDetailedMessage())
+                    )
+                )
+            );
+        }
+        // determine overall result from the node results.
+        Set<String> unsafeNodeRemovals = response.getNodes()
+            .stream()
+            .filter(r -> r.getShardIds().isEmpty() == false)
+            .map(r -> r.getNode().getId())
+            .collect(Collectors.toSet());
+        if (unsafeNodeRemovals.isEmpty() == false) {
+            return new NodesRemovalPrevalidation(
+                false,
+                Strings.format("removal of the following nodes might not be safe: %s", unsafeNodeRemovals),
+                nodeResults
+            );
+        }
+        if (response.failures().isEmpty() == false) {
+            Set<String> unknownNodeRemovals = response.failures().stream().map(FailedNodeException::nodeId).collect(Collectors.toSet());
+            return new NodesRemovalPrevalidation(
+                false,
+                Strings.format("cannot prevalidate removal of nodes with the following IDs: %s", unknownNodeRemovals),
+                nodeResults
+            );
         }
+        return new NodesRemovalPrevalidation(true, "", nodeResults);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateShardPathAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateShardPathAction.java
new file mode 100644
index 000000000000..251129795026
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/TransportPrevalidateShardPathAction.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.cluster.node.shutdown;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.FailedNodeException;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.nodes.TransportNodesAction;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.shard.ShardPath;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+
+/**
+ * Given a set of shard IDs, checks which of those shards have a matching directory in the local data path.
+ * This is used by {@link PrevalidateNodeRemovalAction} to find out whether a node may contain some copy
+ * of a specific shard. The response contains a subset of the request shard IDs which are in the cluster state
+ * of this node and have a matching shard path on the local data path.
+ */
+public class TransportPrevalidateShardPathAction extends TransportNodesAction<
+    PrevalidateShardPathRequest,
+    PrevalidateShardPathResponse,
+    NodePrevalidateShardPathRequest,
+    NodePrevalidateShardPathResponse> {
+
+    public static final String ACTION_NAME = "internal:admin/indices/prevalidate_shard_path";
+    public static final ActionType<PrevalidateShardPathResponse> TYPE = new ActionType<>(ACTION_NAME, PrevalidateShardPathResponse::new);
+    private static final Logger logger = LogManager.getLogger(TransportPrevalidateShardPathAction.class);
+
+    private final TransportService transportService;
+    private final NodeEnvironment nodeEnv;
+    private final Settings settings;
+
+    @Inject
+    public TransportPrevalidateShardPathAction(
+        ThreadPool threadPool,
+        ClusterService clusterService,
+        TransportService transportService,
+        ActionFilters actionFilters,
+        NodeEnvironment nodeEnv,
+        Settings settings
+    ) {
+        super(
+            ACTION_NAME,
+            threadPool,
+            clusterService,
+            transportService,
+            actionFilters,
+            PrevalidateShardPathRequest::new,
+            NodePrevalidateShardPathRequest::new,
+            ThreadPool.Names.MANAGEMENT,
+            NodePrevalidateShardPathResponse.class
+        );
+        this.transportService = transportService;
+        this.nodeEnv = nodeEnv;
+        this.settings = settings;
+    }
+
+    @Override
+    protected PrevalidateShardPathResponse newResponse(
+        PrevalidateShardPathRequest request,
+        List<NodePrevalidateShardPathResponse> nodeResponses,
+        List<FailedNodeException> failures
+    ) {
+        return new PrevalidateShardPathResponse(clusterService.getClusterName(), nodeResponses, failures);
+    }
+
+    @Override
+    protected NodePrevalidateShardPathRequest newNodeRequest(PrevalidateShardPathRequest request) {
+        return new NodePrevalidateShardPathRequest(request.getShardIds());
+    }
+
+    @Override
+    protected NodePrevalidateShardPathResponse newNodeResponse(StreamInput in, DiscoveryNode node) throws IOException {
+        return new NodePrevalidateShardPathResponse(in);
+    }
+
+    @Override
+    protected NodePrevalidateShardPathResponse nodeOperation(NodePrevalidateShardPathRequest request, Task task) {
+        Set<ShardId> localShards = new HashSet<>();
+        ShardPath shardPath = null;
+        // For each shard we only check whether the shard path exists, regardless of whether the content is a valid index or not.
+        for (ShardId shardId : request.getShardIds()) {
+            try {
+                var indexMetadata = clusterService.state().metadata().index(shardId.getIndex());
+                String customDataPath = null;
+                if (indexMetadata != null) {
+                    customDataPath = new IndexSettings(indexMetadata, settings).customDataPath();
+                } else {
+                    // The index is not known to this node. This shouldn't happen, but it can be safely ignored for this operation.
+                    logger.warn("node doesn't have metadata for the index [{}]", shardId.getIndex());
+                }
+                shardPath = ShardPath.loadShardPath(logger, nodeEnv, shardId, customDataPath);
+                if (shardPath != null) {
+                    localShards.add(shardId);
+                }
+            } catch (IOException e) {
+                final String path = shardPath != null ? shardPath.resolveIndex().toString() : "";
+                logger.error(() -> String.format(Locale.ROOT, "error loading shard path for shard [%s]", shardId), e);
+            }
+        }
+        return new NodePrevalidateShardPathResponse(transportService.getLocalNode(), localShards);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksResponse.java
index 51ad85f99328..a53ed8dacc36 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksResponse.java
@@ -11,11 +11,9 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.tasks.TaskInfo;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -43,17 +41,7 @@ public CancelTasksResponse(
         super(tasks, taskFailures, nodeFailures);
     }
 
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        return super.toXContent(builder, params);
-    }
-
     public static CancelTasksResponse fromXContent(XContentParser parser) {
         return PARSER.apply(parser, null);
     }
-
-    @Override
-    public String toString() {
-        return Strings.toString(this, true, true);
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
index 32ca0a59a0b1..cf623c37f721 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.action.admin.cluster.node.tasks.get;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
@@ -16,16 +17,19 @@
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.ContextPreservingActionListener;
 import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.support.ListenableActionFuture;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.tasks.RemovedTaskListener;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskInfo;
@@ -39,8 +43,9 @@
 
 import java.io.IOException;
 
+import static java.util.Objects.requireNonNullElse;
 import static org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction.TASKS_ORIGIN;
-import static org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction.waitForCompletionTimeout;
+import static org.elasticsearch.core.TimeValue.timeValueSeconds;
 
 /**
  * ActionType to get a single task. If the task isn't running then it'll try to request the status from request index.
@@ -53,6 +58,9 @@
  * </ul>
  */
 public class TransportGetTaskAction extends HandledTransportAction<GetTaskRequest, GetTaskResponse> {
+
+    private static final TimeValue DEFAULT_WAIT_FOR_COMPLETION_TIMEOUT = timeValueSeconds(30);
+
     private final ThreadPool threadPool;
     private final ClusterService clusterService;
     private final TransportService transportService;
@@ -130,19 +138,47 @@ void getRunningTaskFromNode(Task thisTask, GetTaskRequest request, ActionListene
             getFinishedTaskFromIndex(thisTask, request, listener);
         } else {
             if (request.getWaitForCompletion()) {
-                // Shift to the generic thread pool and let it wait for the task to complete so we don't block any important threads.
-                threadPool.generic().execute(new AbstractRunnable() {
-                    @Override
-                    protected void doRun() {
-                        taskManager.waitForTaskCompletion(runningTask, waitForCompletionTimeout(request.getTimeout()));
-                        waitedForCompletion(thisTask, request, runningTask.taskInfo(clusterService.localNode().getId(), true), listener);
+                final ListenableActionFuture<Void> future = new ListenableActionFuture<>();
+                RemovedTaskListener removedTaskListener = task -> {
+                    if (task.equals(runningTask)) {
+                        future.onResponse(null);
                     }
-
-                    @Override
-                    public void onFailure(Exception e) {
-                        listener.onFailure(e);
-                    }
-                });
+                };
+                taskManager.registerRemovedTaskListener(removedTaskListener);
+                // Check if the task had finished before we registered the listener, so we wouldn't wait
+                // for an event that would never come
+                if (taskManager.getTask(request.getTaskId().getId()) == null) {
+                    future.onResponse(null);
+                }
+                final ActionListener<Void> waitedForCompletionListener = ActionListener.runBefore(
+                    ActionListener.wrap(
+                        v -> waitedForCompletion(
+                            thisTask,
+                            request,
+                            runningTask.taskInfo(clusterService.localNode().getId(), true),
+                            listener
+                        ),
+                        listener::onFailure
+                    ),
+                    () -> taskManager.unregisterRemovedTaskListener(removedTaskListener)
+                );
+                if (future.isDone()) {
+                    // The task has already finished, we can run the completion listener in the same thread
+                    waitedForCompletionListener.onResponse(null);
+                } else {
+                    future.addListener(
+                        new ContextPreservingActionListener<>(
+                            threadPool.getThreadContext().newRestorableContext(false),
+                            waitedForCompletionListener
+                        )
+                    );
+                    var failByTimeout = threadPool.schedule(
+                        () -> future.onFailure(new ElasticsearchTimeoutException("Timed out waiting for completion of task")),
+                        requireNonNullElse(request.getTimeout(), DEFAULT_WAIT_FOR_COMPLETION_TIMEOUT),
+                        ThreadPool.Names.SAME
+                    );
+                    future.addListener(ActionListener.wrap(failByTimeout::cancel));
+                }
             } else {
                 TaskInfo info = runningTask.taskInfo(clusterService.localNode().getId(), true);
                 listener.onResponse(new GetTaskResponse(new TaskResult(false, info)));
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequest.java
index 3cd7c36fafc3..5b0194c81283 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.cluster.node.tasks.list;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.tasks.BaseTasksRequest;
 import org.elasticsearch.common.Strings;
@@ -40,7 +40,7 @@ public ListTasksRequest(StreamInput in) throws IOException {
         super(in);
         detailed = in.readBoolean();
         waitForCompletion = in.readBoolean();
-        if (in.getVersion().onOrAfter(Version.V_7_13_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0)) {
             descriptions = in.readStringArray();
         }
     }
@@ -50,7 +50,7 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeBoolean(detailed);
         out.writeBoolean(waitForCompletion);
-        if (out.getVersion().onOrAfter(Version.V_7_13_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0)) {
             out.writeStringArray(descriptions);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
index 28df33674ee7..695202630b47 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
@@ -16,22 +16,24 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.TriFunction;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskInfo;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
@@ -39,7 +41,7 @@
 /**
  * Returns the list of tasks currently running on the nodes
  */
-public class ListTasksResponse extends BaseTasksResponse implements ToXContentObject {
+public class ListTasksResponse extends BaseTasksResponse {
     private static final String TASKS = "tasks";
 
     private final List<TaskInfo> tasks;
@@ -142,7 +144,7 @@ private void buildTaskGroups() {
                 topLevelTasks.add(taskGroup);
             }
         }
-        this.groups = Collections.unmodifiableList(topLevelTasks.stream().map(TaskGroup.Builder::build).toList());
+        this.groups = topLevelTasks.stream().map(TaskGroup.Builder::build).toList();
     }
 
     /**
@@ -155,82 +157,98 @@ public List<TaskInfo> getTasks() {
     /**
      * Convert this task response to XContent grouping by executing nodes.
      */
-    public XContentBuilder toXContentGroupedByNode(XContentBuilder builder, Params params, DiscoveryNodes discoveryNodes)
-        throws IOException {
-        toXContentCommon(builder, params);
-        builder.startObject("nodes");
-        for (Map.Entry<String, List<TaskInfo>> entry : getPerNodeTasks().entrySet()) {
-            DiscoveryNode node = discoveryNodes.get(entry.getKey());
-            builder.startObject(entry.getKey());
-            if (node != null) {
-                // If the node is no longer part of the cluster, oh well, we'll just skip it's useful information.
-                builder.field("name", node.getName());
-                builder.field("transport_address", node.getAddress().toString());
-                builder.field("host", node.getHostName());
-                builder.field("ip", node.getAddress());
-
-                builder.startArray("roles");
-                for (DiscoveryNodeRole role : node.getRoles()) {
-                    builder.value(role.roleName());
-                }
-                builder.endArray();
+    public ChunkedToXContentObject groupedByNode(Supplier<DiscoveryNodes> nodesInCluster) {
+        return ignored -> {
+            final var discoveryNodes = nodesInCluster.get();
+            return Iterators.concat(Iterators.single((builder, params) -> {
+                builder.startObject();
+                toXContentCommon(builder, params);
+                builder.startObject("nodes");
+                return builder;
+            }), Iterators.flatMap(getPerNodeTasks().entrySet().iterator(), entry -> {
+                DiscoveryNode node = discoveryNodes.get(entry.getKey());
+                return Iterators.concat(Iterators.single((builder, params) -> {
+                    builder.startObject(entry.getKey());
+                    if (node != null) {
+                        // If the node is no longer part of the cluster, oh well, we'll just skip its useful information.
+                        builder.field("name", node.getName());
+                        builder.field("transport_address", node.getAddress().toString());
+                        builder.field("host", node.getHostName());
+                        builder.field("ip", node.getAddress());
+
+                        builder.startArray("roles");
+                        for (DiscoveryNodeRole role : node.getRoles()) {
+                            builder.value(role.roleName());
+                        }
+                        builder.endArray();
 
-                if (node.getAttributes().isEmpty() == false) {
-                    builder.startObject("attributes");
-                    for (Map.Entry<String, String> attrEntry : node.getAttributes().entrySet()) {
-                        builder.field(attrEntry.getKey(), attrEntry.getValue());
+                        if (node.getAttributes().isEmpty() == false) {
+                            builder.startObject("attributes");
+                            for (Map.Entry<String, String> attrEntry : node.getAttributes().entrySet()) {
+                                builder.field(attrEntry.getKey(), attrEntry.getValue());
+                            }
+                            builder.endObject();
+                        }
                     }
+                    builder.startObject(TASKS);
+                    return builder;
+                }), Iterators.flatMap(entry.getValue().iterator(), task -> Iterators.<ToXContent>single((builder, params) -> {
+                    builder.startObject(task.taskId().toString());
+                    task.toXContent(builder, params);
                     builder.endObject();
-                }
-            }
-            builder.startObject(TASKS);
-            for (TaskInfo task : entry.getValue()) {
-                builder.startObject(task.taskId().toString());
-                task.toXContent(builder, params);
+                    return builder;
+                })), Iterators.<ToXContent>single((builder, params) -> {
+                    builder.endObject();
+                    builder.endObject();
+                    return builder;
+                }));
+            }), Iterators.<ToXContent>single((builder, params) -> {
                 builder.endObject();
-            }
-            builder.endObject();
-            builder.endObject();
-        }
-        builder.endObject();
-        return builder;
+                builder.endObject();
+                return builder;
+            }));
+        };
     }
 
     /**
      * Convert this response to XContent grouping by parent tasks.
      */
-    public XContentBuilder toXContentGroupedByParents(XContentBuilder builder, Params params) throws IOException {
-        toXContentCommon(builder, params);
-        builder.startObject(TASKS);
-        for (TaskGroup group : getTaskGroups()) {
+    public ChunkedToXContentObject groupedByParent() {
+        return ignored -> Iterators.concat(Iterators.single((builder, params) -> {
+            builder.startObject();
+            toXContentCommon(builder, params);
+            builder.startObject(TASKS);
+            return builder;
+        }), getTaskGroups().stream().<ToXContent>map(group -> (builder, params) -> {
             builder.field(group.taskInfo().taskId().toString());
             group.toXContent(builder, params);
-        }
-        builder.endObject();
-        return builder;
+            return builder;
+        }).iterator(), Iterators.single((builder, params) -> {
+            builder.endObject();
+            builder.endObject();
+            return builder;
+        }));
     }
 
     /**
      * Presents a flat list of tasks
      */
-    public XContentBuilder toXContentGroupedByNone(XContentBuilder builder, Params params) throws IOException {
-        toXContentCommon(builder, params);
-        builder.startArray(TASKS);
-        for (TaskInfo taskInfo : getTasks()) {
+    public ChunkedToXContentObject groupedByNone() {
+        return ignored -> Iterators.concat(Iterators.single((builder, params) -> {
+            builder.startObject();
+            toXContentCommon(builder, params);
+            builder.startArray(TASKS);
+            return builder;
+        }), getTasks().stream().<ToXContent>map(taskInfo -> (builder, params) -> {
             builder.startObject();
             taskInfo.toXContent(builder, params);
             builder.endObject();
-        }
-        builder.endArray();
-        return builder;
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        toXContentGroupedByNone(builder, params);
-        builder.endObject();
-        return builder;
+            return builder;
+        }).iterator(), Iterators.single((builder, params) -> {
+            builder.endArray();
+            builder.endObject();
+            return builder;
+        }));
     }
 
     public static ListTasksResponse fromXContent(XContentParser parser) {
@@ -239,6 +257,7 @@ public static ListTasksResponse fromXContent(XContentParser parser) {
 
     @Override
     public String toString() {
-        return Strings.toString(this, true, true);
+        return Strings.toString(ChunkedToXContent.wrapAsToXContent(groupedByNone()), true, true);
     }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
index b81f7e3db3c7..5af717dfbb88 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
@@ -8,22 +8,33 @@
 
 package org.elasticsearch.action.admin.cluster.node.tasks.list;
 
+import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.ContextPreservingActionListener;
+import org.elasticsearch.action.support.ListenableActionFuture;
+import org.elasticsearch.action.support.ThreadedActionListener;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.AbstractRefCounted;
+import org.elasticsearch.core.RefCounted;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.tasks.RemovedTaskListener;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskInfo;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
 
+import static java.util.Objects.requireNonNullElse;
 import static org.elasticsearch.core.TimeValue.timeValueSeconds;
 
 public class TransportListTasksAction extends TransportTasksAction<Task, ListTasksRequest, ListTasksResponse, TaskInfo> {
@@ -66,19 +77,75 @@ protected void taskOperation(Task actionTask, ListTasksRequest request, Task tas
     }
 
     @Override
-    protected void processTasks(ListTasksRequest request, Consumer<Task> operation) {
+    protected void processTasks(ListTasksRequest request, Consumer<Task> operation, ActionListener<Void> nodeOperation) {
         if (request.getWaitForCompletion()) {
-            long timeoutNanos = waitForCompletionTimeout(request.getTimeout());
-            operation = operation.andThen(task -> {
-                if (task.getAction().startsWith(ListTasksAction.NAME)) {
-                    // It doesn't make sense to wait for List Tasks and it can cause an infinite loop of the task waiting
-                    // for itself or one of its child tasks
-                    return;
+            final ListenableActionFuture<Void> future = new ListenableActionFuture<>();
+            final Set<Task> removedTasks = Sets.newConcurrentHashSet();
+            final Set<Task> matchedTasks = Sets.newConcurrentHashSet();
+            final RefCounted removalRefs = AbstractRefCounted.of(() -> {
+                matchedTasks.removeAll(removedTasks);
+                removedTasks.clear();
+                if (matchedTasks.isEmpty()) {
+                    future.onResponse(null);
                 }
-                taskManager.waitForTaskCompletion(task, timeoutNanos);
             });
+
+            final AtomicBoolean collectionComplete = new AtomicBoolean();
+            final RemovedTaskListener removedTaskListener = task -> {
+                if (collectionComplete.get() == false && removalRefs.tryIncRef()) {
+                    removedTasks.add(task);
+                    removalRefs.decRef();
+                } else {
+                    matchedTasks.remove(task);
+                    if (matchedTasks.isEmpty()) {
+                        future.onResponse(null);
+                    }
+                }
+            };
+            taskManager.registerRemovedTaskListener(removedTaskListener);
+            final ActionListener<Void> allMatchedTasksRemovedListener = ActionListener.runBefore(
+                nodeOperation,
+                () -> taskManager.unregisterRemovedTaskListener(removedTaskListener)
+            );
+            try {
+                processTasks(request, task -> {
+                    if (task.getAction().startsWith(ListTasksAction.NAME) == false) {
+                        // It doesn't make sense to wait for List Tasks and it can cause an infinite loop of the task waiting
+                        // for itself or one of its child tasks
+                        matchedTasks.add(task);
+                    }
+                    operation.accept(task);
+                });
+            } catch (Exception e) {
+                allMatchedTasksRemovedListener.onFailure(e);
+                return;
+            }
+            removalRefs.decRef();
+            collectionComplete.set(true);
+
+            if (future.isDone()) {
+                // No tasks to wait, we can run nodeOperation in the management pool
+                allMatchedTasksRemovedListener.onResponse(null);
+            } else {
+                final var threadPool = clusterService.threadPool();
+                future.addListener(
+                    new ThreadedActionListener<>(
+                        threadPool.executor(ThreadPool.Names.MANAGEMENT),
+                        new ContextPreservingActionListener<>(
+                            threadPool.getThreadContext().newRestorableContext(false),
+                            allMatchedTasksRemovedListener
+                        )
+                    )
+                );
+                var cancellable = threadPool.schedule(
+                    () -> future.onFailure(new ElasticsearchTimeoutException("Timed out waiting for completion of tasks")),
+                    requireNonNullElse(request.getTimeout(), DEFAULT_WAIT_FOR_COMPLETION_TIMEOUT),
+                    ThreadPool.Names.SAME
+                );
+                future.addListener(ActionListener.wrap(cancellable::cancel));
+            }
+        } else {
+            super.processTasks(request, operation, nodeOperation);
         }
-        super.processTasks(request, operation);
     }
-
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java
index da8642241f99..4011d5365dd5 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
@@ -55,7 +56,8 @@ public void writeTo(StreamOutput out) throws IOException {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        repositories.toXContent(builder, new DelegatingMapParams(Map.of(RepositoriesMetadata.HIDE_GENERATIONS_PARAM, "true"), params));
+        ChunkedToXContent.wrapAsToXContent(repositories)
+            .toXContent(builder, new DelegatingMapParams(Map.of(RepositoriesMetadata.HIDE_GENERATIONS_PARAM, "true"), params));
         builder.endObject();
         return builder;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java
index 9b787a26134b..1964e0f468cd 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java
@@ -76,7 +76,12 @@ protected void masterOperation(
         ClusterState state,
         final ActionListener<GetRepositoriesResponse> listener
     ) {
-        listener.onResponse(new GetRepositoriesResponse(new RepositoriesMetadata(getRepositories(state, request.repositories()))));
+        RepositoriesResult result = getRepositories(state, request.repositories());
+        if (result.hasMissingRepositories()) {
+            listener.onFailure(new RepositoryMissingException(String.join(", ", result.missing())));
+        } else {
+            listener.onResponse(new GetRepositoriesResponse(new RepositoriesMetadata(result.metadata)));
+        }
     }
 
     /**
@@ -84,13 +89,14 @@ protected void masterOperation(
      *
      * @param state     Cluster state
      * @param repoNames Repository names or patterns to get metadata for
-     * @return list of repository metadata
+     * @return a result with the repository metadata that were found in the cluster state and the missing repositories
      */
-    public static List<RepositoryMetadata> getRepositories(ClusterState state, String[] repoNames) {
+    public static RepositoriesResult getRepositories(ClusterState state, String[] repoNames) {
         RepositoriesMetadata repositories = state.metadata().custom(RepositoriesMetadata.TYPE, RepositoriesMetadata.EMPTY);
         if (isMatchAll(repoNames)) {
-            return repositories.repositories();
+            return new RepositoriesResult(repositories.repositories());
         }
+        final List<String> missingRepositories = new ArrayList<>();
         final List<String> includePatterns = new ArrayList<>();
         final List<String> excludePatterns = new ArrayList<>();
         boolean seenWildcard = false;
@@ -102,7 +108,7 @@ public static List<RepositoryMetadata> getRepositories(ClusterState state, Strin
                     seenWildcard = true;
                 } else {
                     if (repositories.repository(repositoryOrPattern) == null) {
-                        throw new RepositoryMissingException(repositoryOrPattern);
+                        missingRepositories.add(repositoryOrPattern);
                     }
                 }
                 includePatterns.add(repositoryOrPattern);
@@ -119,6 +125,20 @@ public static List<RepositoryMetadata> getRepositories(ClusterState state, Strin
                 }
             }
         }
-        return List.copyOf(repositoryListBuilder);
+        return new RepositoriesResult(List.copyOf(repositoryListBuilder), missingRepositories);
+    }
+
+    /**
+     * A holder class that consists of the repository metadata and the names of the repositories that were not found in the cluster state.
+     */
+    public record RepositoriesResult(List<RepositoryMetadata> metadata, List<String> missing) {
+
+        RepositoriesResult(List<RepositoryMetadata> repositoryMetadata) {
+            this(repositoryMetadata, List.of());
+        }
+
+        boolean hasMissingRepositories() {
+            return missing.isEmpty() == false;
+        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
index 4e4743aea558..ba9ba10a80d0 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
@@ -8,25 +8,32 @@
 
 package org.elasticsearch.action.admin.cluster.reroute;
 
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.support.master.IsAcknowledgedSupplier;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.allocation.RoutingExplanations;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.action.search.RestSearchAction;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
 import java.util.Objects;
 
+import static org.elasticsearch.action.support.master.AcknowledgedResponse.ACKNOWLEDGED_KEY;
+
 /**
  * Response returned after a cluster reroute request
  */
-public class ClusterRerouteResponse extends AcknowledgedResponse implements ToXContentObject {
+public class ClusterRerouteResponse extends ActionResponse implements IsAcknowledgedSupplier, ChunkedToXContentObject {
 
     private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestSearchAction.class);
     public static final String STATE_FIELD_DEPRECATION_MESSAGE = "The [state] field in the response to the reroute API is deprecated "
@@ -37,15 +44,17 @@ public class ClusterRerouteResponse extends AcknowledgedResponse implements ToXC
      */
     private final ClusterState state;
     private final RoutingExplanations explanations;
+    private final boolean acknowledged;
 
     ClusterRerouteResponse(StreamInput in) throws IOException {
         super(in);
+        acknowledged = in.readBoolean();
         state = ClusterState.readFrom(in, null);
         explanations = RoutingExplanations.readFrom(in);
     }
 
     ClusterRerouteResponse(boolean acknowledged, ClusterState state, RoutingExplanations explanations) {
-        super(acknowledged);
+        this.acknowledged = acknowledged;
         this.state = state;
         this.explanations = explanations;
     }
@@ -61,26 +70,45 @@ public RoutingExplanations getExplanations() {
         return this.explanations;
     }
 
+    @Override
+    public final boolean isAcknowledged() {
+        return acknowledged;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
+        out.writeBoolean(acknowledged);
         state.writeTo(out);
         RoutingExplanations.writeTo(explanations, out);
     }
 
+    private boolean emitState(ToXContent.Params params) {
+        return Objects.equals(params.param("metric"), "none") == false;
+    }
+
     @Override
-    protected void addCustomFields(XContentBuilder builder, Params params) throws IOException {
-        if (Objects.equals(params.param("metric"), "none") == false) {
-            if (builder.getRestApiVersion() != RestApiVersion.V_7) {
-                deprecationLogger.critical(DeprecationCategory.API, "reroute_cluster_state", STATE_FIELD_DEPRECATION_MESSAGE);
-            }
-            builder.startObject("state");
-            state.toXContent(builder, params);
-            builder.endObject();
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
+        if (emitState(outerParams)) {
+            deprecationLogger.critical(DeprecationCategory.API, "reroute_cluster_state", STATE_FIELD_DEPRECATION_MESSAGE);
         }
+        return toXContentChunkedV7(outerParams);
+    }
 
-        if (params.paramAsBoolean("explain", false)) {
-            explanations.toXContent(builder, params);
-        }
+    @Override
+    public Iterator<? extends ToXContent> toXContentChunkedV7(ToXContent.Params outerParams) {
+        return Iterators.concat(
+            Iterators.single((builder, params) -> builder.startObject().field(ACKNOWLEDGED_KEY, isAcknowledged())),
+            emitState(outerParams)
+                ? ChunkedToXContentHelper.wrapWithObject("state", state.toXContentChunked(outerParams))
+                : Collections.emptyIterator(),
+            Iterators.single((builder, params) -> {
+                if (params.paramAsBoolean("explain", false)) {
+                    explanations.toXContent(builder, params);
+                }
+
+                builder.endObject();
+                return builder;
+            })
+        );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java
index 15af362fe4a4..76d59660da0b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.cluster.settings;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
@@ -37,12 +37,12 @@ public Request() {}
 
         public Request(StreamInput in) throws IOException {
             super(in);
-            assert in.getVersion().onOrAfter(Version.V_8_3_0);
+            assert in.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0);
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            assert out.getVersion().onOrAfter(Version.V_8_3_0);
+            assert out.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0);
             super.writeTo(out);
         }
 
@@ -77,7 +77,7 @@ public int hashCode() {
 
         public Response(StreamInput in) throws IOException {
             super(in);
-            assert in.getVersion().onOrAfter(Version.V_8_3_0);
+            assert in.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0);
             persistentSettings = Settings.readSettingsFromStream(in);
             transientSettings = Settings.readSettingsFromStream(in);
             settings = Settings.readSettingsFromStream(in);
@@ -91,7 +91,7 @@ public Response(Settings persistentSettings, Settings transientSettings, Setting
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            assert out.getVersion().onOrAfter(Version.V_8_3_0);
+            assert out.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0);
             persistentSettings.writeTo(out);
             transientSettings.writeTo(out);
             settings.writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
index 04acdb7a3fa4..e0085a45cd3c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.action.admin.cluster.snapshots.create;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -54,7 +54,7 @@ public class CreateSnapshotRequest extends MasterNodeRequest<CreateSnapshotReque
         IndicesRequest.Replaceable,
         ToXContentObject {
 
-    public static final Version SETTINGS_IN_REQUEST_VERSION = Version.V_8_0_0;
+    public static final TransportVersion SETTINGS_IN_REQUEST_VERSION = TransportVersion.V_8_0_0;
 
     public static int MAXIMUM_METADATA_BYTES = 1024; // chosen arbitrarily
 
@@ -96,7 +96,7 @@ public CreateSnapshotRequest(StreamInput in) throws IOException {
         repository = in.readString();
         indices = in.readStringArray();
         indicesOptions = IndicesOptions.readIndicesOptions(in);
-        if (in.getVersion().before(SETTINGS_IN_REQUEST_VERSION)) {
+        if (in.getTransportVersion().before(SETTINGS_IN_REQUEST_VERSION)) {
             readSettingsFromStream(in);
         }
         featureStates = in.readStringArray();
@@ -113,7 +113,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(repository);
         out.writeStringArray(indices);
         indicesOptions.writeIndicesOptions(out);
-        if (out.getVersion().before(SETTINGS_IN_REQUEST_VERSION)) {
+        if (out.getTransportVersion().before(SETTINGS_IN_REQUEST_VERSION)) {
             Settings.EMPTY.writeTo(out);
         }
         out.writeStringArray(featureStates);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java
index ce410efe79f4..99dcb33bc558 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java
@@ -10,10 +10,12 @@
 
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
 import java.io.IOException;
+import java.util.Arrays;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
@@ -111,4 +113,9 @@ public DeleteSnapshotRequest snapshots(String... snapshots) {
         this.snapshots = snapshots;
         return this;
     }
+
+    @Override
+    public String getDescription() {
+        return Strings.format("[%s]%s", repository, Arrays.toString(snapshots));
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateRequest.java
index db2df62af7e6..fff36f9bc687 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.cluster.snapshots.features;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -19,10 +19,10 @@
 /** Request for resetting feature state */
 public class ResetFeatureStateRequest extends MasterNodeRequest<ResetFeatureStateRequest> {
 
-    private static final Version FEATURE_RESET_ON_MASTER = Version.V_7_14_0;
+    private static final TransportVersion FEATURE_RESET_ON_MASTER = TransportVersion.V_7_14_0;
 
     public static ResetFeatureStateRequest fromStream(StreamInput in) throws IOException {
-        if (in.getVersion().before(FEATURE_RESET_ON_MASTER)) {
+        if (in.getTransportVersion().before(FEATURE_RESET_ON_MASTER)) {
             throw new IllegalStateException(
                 "feature reset is not available in a cluster that have nodes with version before " + FEATURE_RESET_ON_MASTER
             );
@@ -38,7 +38,7 @@ private ResetFeatureStateRequest(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getVersion().before(FEATURE_RESET_ON_MASTER)) {
+        if (out.getTransportVersion().before(FEATURE_RESET_ON_MASTER)) {
             throw new IllegalStateException(
                 "feature reset is not available in a cluster that have nodes with version before " + FEATURE_RESET_ON_MASTER
             );
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportResetFeatureStateAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportResetFeatureStateAction.java
index 6fa73bf2ea14..c0c928e2743d 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportResetFeatureStateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/features/TransportResetFeatureStateAction.java
@@ -73,11 +73,11 @@ protected void masterOperation(
 
         final int features = systemIndices.getFeatures().size();
         GroupedActionListener<ResetFeatureStateResponse.ResetFeatureStateStatus> groupedActionListener = new GroupedActionListener<>(
+            systemIndices.getFeatures().size(),
             listener.map(responses -> {
                 assert features == responses.size();
                 return new ResetFeatureStateResponse(new ArrayList<>(responses));
-            }),
-            systemIndices.getFeatures().size()
+            })
         );
 
         for (SystemIndices.Feature feature : systemIndices.getFeatures()) {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java
index 30938d91b29a..35b6abed2c2f 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.cluster.snapshots.get;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.Strings;
@@ -40,19 +40,19 @@ public class GetSnapshotsRequest extends MasterNodeRequest<GetSnapshotsRequest>
     public static final String NO_POLICY_PATTERN = "_none";
     public static final boolean DEFAULT_VERBOSE_MODE = true;
 
-    public static final Version SLM_POLICY_FILTERING_VERSION = Version.V_7_16_0;
+    public static final TransportVersion SLM_POLICY_FILTERING_VERSION = TransportVersion.V_7_16_0;
 
-    public static final Version FROM_SORT_VALUE_VERSION = Version.V_7_16_0;
+    public static final TransportVersion FROM_SORT_VALUE_VERSION = TransportVersion.V_7_16_0;
 
-    public static final Version MULTIPLE_REPOSITORIES_SUPPORT_ADDED = Version.V_7_14_0;
+    public static final TransportVersion MULTIPLE_REPOSITORIES_SUPPORT_ADDED = TransportVersion.V_7_14_0;
 
-    public static final Version PAGINATED_GET_SNAPSHOTS_VERSION = Version.V_7_14_0;
+    public static final TransportVersion PAGINATED_GET_SNAPSHOTS_VERSION = TransportVersion.V_7_14_0;
 
-    public static final Version NUMERIC_PAGINATION_VERSION = Version.V_7_15_0;
+    public static final TransportVersion NUMERIC_PAGINATION_VERSION = TransportVersion.V_7_15_0;
 
-    private static final Version SORT_BY_SHARDS_OR_REPO_VERSION = Version.V_7_16_0;
+    private static final TransportVersion SORT_BY_SHARDS_OR_REPO_VERSION = TransportVersion.V_7_16_0;
 
-    private static final Version INDICES_FLAG_VERSION = Version.V_8_3_0;
+    private static final TransportVersion INDICES_FLAG_VERSION = TransportVersion.V_8_3_0;
 
     public static final int NO_LIMIT = -1;
 
@@ -112,7 +112,7 @@ public GetSnapshotsRequest(String... repositories) {
 
     public GetSnapshotsRequest(StreamInput in) throws IOException {
         super(in);
-        if (in.getVersion().onOrAfter(MULTIPLE_REPOSITORIES_SUPPORT_ADDED)) {
+        if (in.getTransportVersion().onOrAfter(MULTIPLE_REPOSITORIES_SUPPORT_ADDED)) {
             repositories = in.readStringArray();
         } else {
             repositories = new String[] { in.readString() };
@@ -120,21 +120,21 @@ public GetSnapshotsRequest(StreamInput in) throws IOException {
         snapshots = in.readStringArray();
         ignoreUnavailable = in.readBoolean();
         verbose = in.readBoolean();
-        if (in.getVersion().onOrAfter(PAGINATED_GET_SNAPSHOTS_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(PAGINATED_GET_SNAPSHOTS_VERSION)) {
             after = in.readOptionalWriteable(After::new);
             sort = in.readEnum(SortBy.class);
             size = in.readVInt();
             order = SortOrder.readFromStream(in);
-            if (in.getVersion().onOrAfter(NUMERIC_PAGINATION_VERSION)) {
+            if (in.getTransportVersion().onOrAfter(NUMERIC_PAGINATION_VERSION)) {
                 offset = in.readVInt();
             }
-            if (in.getVersion().onOrAfter(SLM_POLICY_FILTERING_VERSION)) {
+            if (in.getTransportVersion().onOrAfter(SLM_POLICY_FILTERING_VERSION)) {
                 policies = in.readStringArray();
             }
-            if (in.getVersion().onOrAfter(FROM_SORT_VALUE_VERSION)) {
+            if (in.getTransportVersion().onOrAfter(FROM_SORT_VALUE_VERSION)) {
                 fromSortValue = in.readOptionalString();
             }
-            if (in.getVersion().onOrAfter(INDICES_FLAG_VERSION)) {
+            if (in.getTransportVersion().onOrAfter(INDICES_FLAG_VERSION)) {
                 includeIndexNames = in.readBoolean();
             }
         }
@@ -143,7 +143,7 @@ public GetSnapshotsRequest(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        if (out.getVersion().onOrAfter(MULTIPLE_REPOSITORIES_SUPPORT_ADDED)) {
+        if (out.getTransportVersion().onOrAfter(MULTIPLE_REPOSITORIES_SUPPORT_ADDED)) {
             out.writeStringArray(repositories);
         } else {
             if (repositories.length != 1) {
@@ -158,40 +158,44 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeStringArray(snapshots);
         out.writeBoolean(ignoreUnavailable);
         out.writeBoolean(verbose);
-        if (out.getVersion().onOrAfter(PAGINATED_GET_SNAPSHOTS_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(PAGINATED_GET_SNAPSHOTS_VERSION)) {
             out.writeOptionalWriteable(after);
             if ((sort == SortBy.SHARDS || sort == SortBy.FAILED_SHARDS || sort == SortBy.REPOSITORY)
-                && out.getVersion().before(SORT_BY_SHARDS_OR_REPO_VERSION)) {
+                && out.getTransportVersion().before(SORT_BY_SHARDS_OR_REPO_VERSION)) {
                 throw new IllegalArgumentException(
-                    "can't use sort by shard count or repository name with node version [" + out.getVersion() + "]"
+                    "can't use sort by shard count or repository name in transport version [" + out.getTransportVersion() + "]"
                 );
             }
             out.writeEnum(sort);
             out.writeVInt(size);
             order.writeTo(out);
-            if (out.getVersion().onOrAfter(NUMERIC_PAGINATION_VERSION)) {
+            if (out.getTransportVersion().onOrAfter(NUMERIC_PAGINATION_VERSION)) {
                 out.writeVInt(offset);
             } else if (offset != 0) {
                 throw new IllegalArgumentException(
-                    "can't use numeric offset in get snapshots request with node version [" + out.getVersion() + "]"
+                    "can't use numeric offset in get snapshots request in transport version [" + out.getTransportVersion() + "]"
                 );
             }
         } else if (sort != SortBy.START_TIME || size != NO_LIMIT || after != null || order != SortOrder.ASC) {
-            throw new IllegalArgumentException("can't use paginated get snapshots request with node version [" + out.getVersion() + "]");
+            throw new IllegalArgumentException(
+                "can't use paginated get snapshots request in transport version [" + out.getTransportVersion() + "]"
+            );
         }
-        if (out.getVersion().onOrAfter(SLM_POLICY_FILTERING_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(SLM_POLICY_FILTERING_VERSION)) {
             out.writeStringArray(policies);
         } else if (policies.length > 0) {
             throw new IllegalArgumentException(
-                "can't use slm policy filter in snapshots request with node version [" + out.getVersion() + "]"
+                "can't use slm policy filter in snapshots request in transport version [" + out.getTransportVersion() + "]"
             );
         }
-        if (out.getVersion().onOrAfter(FROM_SORT_VALUE_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(FROM_SORT_VALUE_VERSION)) {
             out.writeOptionalString(fromSortValue);
         } else if (fromSortValue != null) {
-            throw new IllegalArgumentException("can't use after-value in snapshot request with node version [" + out.getVersion() + "]");
+            throw new IllegalArgumentException(
+                "can't use after-value in snapshot request in transport version [" + out.getTransportVersion() + "]"
+            );
         }
-        if (out.getVersion().onOrAfter(INDICES_FLAG_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(INDICES_FLAG_VERSION)) {
             out.writeBoolean(includeIndexNames);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
index 026435f3a9b2..91b1740777d8 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
@@ -33,7 +33,7 @@
 /**
  * Get snapshots response
  */
-public class GetSnapshotsResponse extends ActionResponse implements ChunkedToXContent {
+public class GetSnapshotsResponse extends ActionResponse implements ChunkedToXContentObject {
 
     private static final int UNKNOWN_COUNT = -1;
 
@@ -93,7 +93,7 @@ public GetSnapshotsResponse(
 
     public GetSnapshotsResponse(StreamInput in) throws IOException {
         this.snapshots = in.readImmutableList(SnapshotInfo::readFrom);
-        if (in.getVersion().onOrAfter(GetSnapshotsRequest.MULTIPLE_REPOSITORIES_SUPPORT_ADDED)) {
+        if (in.getTransportVersion().onOrAfter(GetSnapshotsRequest.MULTIPLE_REPOSITORIES_SUPPORT_ADDED)) {
             final Map<String, ElasticsearchException> failedResponses = in.readMap(StreamInput::readString, StreamInput::readException);
             this.failures = Collections.unmodifiableMap(failedResponses);
             this.next = in.readOptionalString();
@@ -101,7 +101,7 @@ public GetSnapshotsResponse(StreamInput in) throws IOException {
             this.failures = Collections.emptyMap();
             this.next = null;
         }
-        if (in.getVersion().onOrAfter(GetSnapshotsRequest.NUMERIC_PAGINATION_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(GetSnapshotsRequest.NUMERIC_PAGINATION_VERSION)) {
             this.total = in.readVInt();
             this.remaining = in.readVInt();
         } else {
@@ -132,7 +132,7 @@ public String next() {
     }
 
     /**
-     * Returns true if there is a least one failed response.
+     * Returns true if there is at least one failed response.
      */
     public boolean isFailed() {
         return failures.isEmpty() == false;
@@ -149,7 +149,7 @@ public int remaining() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeList(snapshots);
-        if (out.getVersion().onOrAfter(GetSnapshotsRequest.MULTIPLE_REPOSITORIES_SUPPORT_ADDED)) {
+        if (out.getTransportVersion().onOrAfter(GetSnapshotsRequest.MULTIPLE_REPOSITORIES_SUPPORT_ADDED)) {
             out.writeMap(failures, StreamOutput::writeString, StreamOutput::writeException);
             out.writeOptionalString(next);
         } else {
@@ -158,14 +158,14 @@ public void writeTo(StreamOutput out) throws IOException {
                 throw failures.values().iterator().next();
             }
         }
-        if (out.getVersion().onOrAfter(GetSnapshotsRequest.NUMERIC_PAGINATION_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(GetSnapshotsRequest.NUMERIC_PAGINATION_VERSION)) {
             out.writeVInt(total);
             out.writeVInt(remaining);
         }
     }
 
     @Override
-    public Iterator<ToXContent> toXContentChunked() {
+    public Iterator<ToXContent> toXContentChunked(ToXContent.Params params) {
         return Iterators.concat(Iterators.single((b, p) -> {
             b.startObject();
             b.startArray("snapshots");
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
index 1235afc3b860..33fd38c9f9a4 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.action.StepListener;
 import org.elasticsearch.action.admin.cluster.repositories.get.TransportGetRepositoriesAction;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.GroupedActionListener;
+import org.elasticsearch.action.support.RefCountingListener;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.SnapshotsInProgress;
@@ -26,8 +26,8 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.repositories.GetSnapshotInfoContext;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.RepositoriesService;
@@ -53,12 +53,12 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
+import java.util.Queue;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiPredicate;
 import java.util.function.Predicate;
 import java.util.function.ToLongFunction;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 /**
@@ -112,12 +112,7 @@ protected void masterOperation(
         getMultipleReposSnapshotInfo(
             request.isSingleRepositoryRequest() == false,
             state.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY),
-            maybeFilterRepositories(
-                TransportGetRepositoriesAction.getRepositories(state, request.repositories()),
-                request.sort(),
-                request.order(),
-                request.fromSortValue()
-            ),
+            TransportGetRepositoriesAction.getRepositories(state, request.repositories()),
             request.snapshots(),
             request.ignoreUnavailable(),
             request.verbose(),
@@ -127,6 +122,7 @@ protected void masterOperation(
             request.offset(),
             request.size(),
             request.order(),
+            request.fromSortValue(),
             SnapshotPredicates.fromRequest(request),
             request.includeIndexNames(),
             listener
@@ -156,7 +152,7 @@ private static List<RepositoryMetadata> maybeFilterRepositories(
     private void getMultipleReposSnapshotInfo(
         boolean isMultiRepoRequest,
         SnapshotsInProgress snapshotsInProgress,
-        List<RepositoryMetadata> repos,
+        TransportGetRepositoriesAction.RepositoriesResult repositoriesResult,
         String[] snapshots,
         boolean ignoreUnavailable,
         boolean verbose,
@@ -166,66 +162,72 @@ private void getMultipleReposSnapshotInfo(
         int offset,
         int size,
         SortOrder order,
+        String fromSortValue,
         SnapshotPredicates predicates,
         boolean indices,
         ActionListener<GetSnapshotsResponse> listener
     ) {
-        // short-circuit if there are no repos, because we can not create GroupedActionListener of size 0
-        if (repos.isEmpty()) {
-            listener.onResponse(new GetSnapshotsResponse(Collections.emptyList(), Collections.emptyMap(), null, 0, 0));
-            return;
-        }
-        final GroupedActionListener<Tuple<Tuple<String, ElasticsearchException>, SnapshotsInRepo>> groupedActionListener =
-            new GroupedActionListener<>(listener.map(responses -> {
-                assert repos.size() == responses.size();
-                final List<SnapshotInfo> allSnapshots = responses.stream()
-                    .map(Tuple::v2)
-                    .filter(Objects::nonNull)
-                    .flatMap(snapshotsInRepo -> snapshotsInRepo.snapshotInfos.stream())
-                    .toList();
-                final Map<String, ElasticsearchException> failures = responses.stream()
-                    .map(Tuple::v1)
-                    .filter(Objects::nonNull)
-                    .collect(Collectors.toMap(Tuple::v1, Tuple::v2));
-                final SnapshotsInRepo snInfos = sortSnapshots(allSnapshots, sortBy, after, offset, size, order);
-                final List<SnapshotInfo> snapshotInfos = snInfos.snapshotInfos;
-                final int remaining = snInfos.remaining + responses.stream()
-                    .map(Tuple::v2)
-                    .filter(Objects::nonNull)
-                    .mapToInt(s -> s.remaining)
-                    .sum();
-                return new GetSnapshotsResponse(
-                    indices ? snapshotInfos : snapshotInfos.stream().map(SnapshotInfo::withoutIndices).toList(),
-                    failures,
-                    remaining > 0
-                        ? GetSnapshotsRequest.After.from(snapshotInfos.get(snapshotInfos.size() - 1), sortBy).asQueryParam()
-                        : null,
-                    responses.stream().map(Tuple::v2).filter(Objects::nonNull).mapToInt(s -> s.totalCount).sum(),
-                    remaining
-                );
-            }), repos.size());
-
-        for (final RepositoryMetadata repo : repos) {
-            final String repoName = repo.name();
-            getSingleRepoSnapshotInfo(
-                snapshotsInProgress,
-                repoName,
-                snapshots,
-                predicates,
-                ignoreUnavailable,
-                verbose,
-                cancellableTask,
+        // Process the missing repositories
+        final Map<String, ElasticsearchException> failures = ConcurrentCollections.newConcurrentMap();
+        for (String missingRepo : repositoriesResult.missing()) {
+            failures.put(missingRepo, new RepositoryMissingException(missingRepo));
+        }
+
+        final Queue<List<SnapshotInfo>> allSnapshotInfos = ConcurrentCollections.newQueue();
+        final var remaining = new AtomicInteger();
+        final var totalCount = new AtomicInteger();
+
+        List<RepositoryMetadata> repositories = maybeFilterRepositories(repositoriesResult.metadata(), sortBy, order, fromSortValue);
+        try (var listeners = new RefCountingListener(listener.map(ignored -> {
+            cancellableTask.ensureNotCancelled();
+            final var sortedSnapshotsInRepos = sortSnapshots(
+                allSnapshotInfos.stream().flatMap(Collection::stream),
+                totalCount.get(),
                 sortBy,
                 after,
-                order,
-                groupedActionListener.delegateResponse((groupedListener, e) -> {
-                    if (isMultiRepoRequest && e instanceof ElasticsearchException) {
-                        groupedListener.onResponse(Tuple.tuple(Tuple.tuple(repoName, (ElasticsearchException) e), null));
-                    } else {
-                        groupedListener.onFailure(e);
-                    }
-                }).map(snInfos -> Tuple.tuple(null, snInfos))
+                offset,
+                size,
+                order
+            );
+            final var snapshotInfos = sortedSnapshotsInRepos.snapshotInfos();
+            final int finalRemaining = sortedSnapshotsInRepos.remaining() + remaining.get();
+            return new GetSnapshotsResponse(
+                indices ? snapshotInfos : snapshotInfos.stream().map(SnapshotInfo::withoutIndices).toList(),
+                failures,
+                finalRemaining > 0
+                    ? GetSnapshotsRequest.After.from(snapshotInfos.get(snapshotInfos.size() - 1), sortBy).asQueryParam()
+                    : null,
+                totalCount.get(),
+                finalRemaining
             );
+        }))) {
+            for (final RepositoryMetadata repository : repositories) {
+                final String repoName = repository.name();
+                getSingleRepoSnapshotInfo(
+                    snapshotsInProgress,
+                    repoName,
+                    snapshots,
+                    predicates,
+                    ignoreUnavailable,
+                    verbose,
+                    cancellableTask,
+                    sortBy,
+                    after,
+                    order,
+                    listeners.acquire((SnapshotsInRepo snapshotsInRepo) -> {
+                        allSnapshotInfos.add(snapshotsInRepo.snapshotInfos());
+                        remaining.addAndGet(snapshotsInRepo.remaining());
+                        totalCount.addAndGet(snapshotsInRepo.totalCount());
+                    }).delegateResponse((l, e) -> {
+                        if (isMultiRepoRequest && e instanceof ElasticsearchException elasticsearchException) {
+                            failures.put(repoName, elasticsearchException);
+                            l.onResponse(SnapshotsInRepo.EMPTY);
+                        } else {
+                            l.onFailure(e);
+                        }
+                    })
+                );
+            }
         }
     }
 
@@ -582,6 +584,18 @@ private static SnapshotsInRepo sortSnapshots(
         int offset,
         int size,
         SortOrder order
+    ) {
+        return sortSnapshots(snapshotInfos.stream(), snapshotInfos.size(), sortBy, after, offset, size, order);
+    }
+
+    private static SnapshotsInRepo sortSnapshots(
+        Stream<SnapshotInfo> infos,
+        int totalCount,
+        GetSnapshotsRequest.SortBy sortBy,
+        @Nullable GetSnapshotsRequest.After after,
+        int offset,
+        int size,
+        SortOrder order
     ) {
         final Comparator<SnapshotInfo> comparator = switch (sortBy) {
             case START_TIME -> BY_START_TIME;
@@ -593,8 +607,6 @@ private static SnapshotsInRepo sortSnapshots(
             case REPOSITORY -> BY_REPOSITORY;
         };
 
-        Stream<SnapshotInfo> infos = snapshotInfos.stream();
-
         if (after != null) {
             assert offset == 0 : "can't combine after and offset but saw [" + after + "] and offset [" + offset + "]";
             infos = infos.filter(buildAfterPredicate(sortBy, after, order));
@@ -610,7 +622,7 @@ private static SnapshotsInRepo sortSnapshots(
         final List<SnapshotInfo> resultSet = size != GetSnapshotsRequest.NO_LIMIT && size < snapshots.size()
             ? snapshots.subList(0, size)
             : snapshots;
-        return new SnapshotsInRepo(resultSet, snapshotInfos.size(), allSnapshots.size() - resultSet.size());
+        return new SnapshotsInRepo(resultSet, totalCount, allSnapshots.size() - resultSet.size());
     }
 
     private static Predicate<SnapshotInfo> buildAfterPredicate(
@@ -861,6 +873,6 @@ private static Predicate<SnapshotInfo> filterByLongOffset(ToLongFunction<Snapsho
     }
 
     private record SnapshotsInRepo(List<SnapshotInfo> snapshotInfos, int totalCount, int remaining) {
-
+        private static final SnapshotsInRepo EMPTY = new SnapshotsInRepo(List.of(), 0, 0);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/TransportGetShardSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/TransportGetShardSnapshotAction.java
index 6cc255956f20..a58e1a778295 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/TransportGetShardSnapshotAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/TransportGetShardSnapshotAction.java
@@ -85,8 +85,8 @@ protected void masterOperation(
         }
 
         GroupedActionListener<Tuple<Optional<ShardSnapshotInfo>, RepositoryException>> groupedActionListener = new GroupedActionListener<>(
-            listener.map(TransportGetShardSnapshotAction::transformToResponse),
-            repositories.size()
+            repositories.size(),
+            listener.map(TransportGetShardSnapshotAction::transformToResponse)
         );
 
         BlockingQueue<String> repositoriesQueue = new LinkedBlockingQueue<>(repositories);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
index a561ac48ed79..259a70fa6ff3 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.cluster.snapshots.restore;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
@@ -48,7 +48,7 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq
     private boolean includeGlobalState = false;
     private boolean partial = false;
     private boolean includeAliases = true;
-    public static Version VERSION_SUPPORTING_QUIET_PARAMETER = Version.V_8_4_0;
+    public static TransportVersion VERSION_SUPPORTING_QUIET_PARAMETER = TransportVersion.V_8_4_0;
     private boolean quiet = false;
     private Settings indexSettings = Settings.EMPTY;
     private String[] ignoreIndexSettings = Strings.EMPTY_ARRAY;
@@ -85,7 +85,7 @@ public RestoreSnapshotRequest(StreamInput in) throws IOException {
         includeGlobalState = in.readBoolean();
         partial = in.readBoolean();
         includeAliases = in.readBoolean();
-        if (in.getVersion().onOrAfter(VERSION_SUPPORTING_QUIET_PARAMETER)) {
+        if (in.getTransportVersion().onOrAfter(VERSION_SUPPORTING_QUIET_PARAMETER)) {
             quiet = in.readBoolean();
         } else {
             quiet = true;
@@ -109,7 +109,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(includeGlobalState);
         out.writeBoolean(partial);
         out.writeBoolean(includeAliases);
-        if (out.getVersion().onOrAfter(VERSION_SUPPORTING_QUIET_PARAMETER)) {
+        if (out.getTransportVersion().onOrAfter(VERSION_SUPPORTING_QUIET_PARAMETER)) {
             out.writeBoolean(quiet);
         }
         indexSettings.writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java
index d9b47f7ecc23..788867123a88 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java
@@ -16,7 +16,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
@@ -43,7 +43,7 @@
 /**
  * Status of a snapshot
  */
-public class SnapshotStatus implements ChunkedToXContent, Writeable {
+public class SnapshotStatus implements ChunkedToXContentObject, Writeable {
 
     private final Snapshot snapshot;
 
@@ -190,7 +190,7 @@ public SnapshotStats getStats() {
     private static final String INCLUDE_GLOBAL_STATE = "include_global_state";
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked() {
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
         return Iterators.concat(Iterators.single((ToXContent) (b, p) -> {
             b.startObject()
                 .field(SNAPSHOT, snapshot.getSnapshotId().getName())
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java
index 13069119d3b9..a8dd1d54d3a0 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
@@ -22,16 +22,13 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
-import java.util.Spliterator;
-import java.util.Spliterators;
-import java.util.stream.StreamSupport;
 
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 
 /**
  * Snapshot status response
  */
-public class SnapshotsStatusResponse extends ActionResponse implements ChunkedToXContent {
+public class SnapshotsStatusResponse extends ActionResponse implements ChunkedToXContentObject {
 
     private final List<SnapshotStatus> snapshots;
 
@@ -89,12 +86,10 @@ public int hashCode() {
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked() {
-        return Iterators.concat(
-            Iterators.single((ToXContent) (b, p) -> b.startObject().startArray("snapshots")),
-            snapshots.stream()
-                .flatMap(s -> StreamSupport.stream(Spliterators.spliteratorUnknownSize(s.toXContentChunked(), Spliterator.ORDERED), false))
-                .iterator(),
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        return Iterators.<ToXContent>concat(
+            Iterators.single((b, p) -> b.startObject().startArray("snapshots")),
+            Iterators.flatMap(snapshots.iterator(), s -> s.toXContentChunked(params)),
             Iterators.single((b, p) -> b.endArray().endObject())
         );
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
index 55e98327a0a4..c0d69c0bdcf2 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
@@ -142,10 +142,9 @@ protected void masterOperation(
                 TransportNodesSnapshotsStatus.TYPE,
                 new TransportNodesSnapshotsStatus.Request(nodesIds.toArray(Strings.EMPTY_ARRAY)).snapshots(snapshots)
                     .timeout(request.masterNodeTimeout()),
+                // fork to snapshot meta since building the response is expensive for large snapshots
                 new ThreadedActionListener<>(
-                    logger,
-                    threadPool,
-                    ThreadPool.Names.SNAPSHOT_META, // fork to snapshot meta since building the response is expensive for large snapshots
+                    threadPool.executor(ThreadPool.Names.SNAPSHOT_META),
                     ActionListener.wrap(
                         nodeSnapshotStatuses -> buildResponse(
                             snapshotsInProgress,
@@ -156,8 +155,7 @@ protected void masterOperation(
                             listener
                         ),
                         listener::onFailure
-                    ),
-                    false
+                    )
                 )
             );
         } else {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java
index 48370ca2199c..c6af59caa7f1 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java
@@ -123,13 +123,11 @@ public void onClusterServiceClose() {
 
                     @Override
                     public void onTimeout(TimeValue timeout) {
-                        try {
-                            if (cancellableTask.notifyIfCancelled(listener) == false) {
-                                listener.onResponse(new ClusterStateResponse(state.getClusterName(), null, true));
+                        ActionListener.run(listener, l -> {
+                            if (cancellableTask.notifyIfCancelled(l) == false) {
+                                l.onResponse(new ClusterStateResponse(state.getClusterName(), null, true));
                             }
-                        } catch (Exception e) {
-                            listener.onFailure(e);
-                        }
+                        });
                     }
                 }, clusterState -> cancellableTask.isCancelled() || acceptableClusterStateOrFailedPredicate.test(clusterState));
         }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java
index 8c3634538d0e..f233cf57961c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.cluster.stats;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
@@ -38,7 +38,7 @@ public ClusterStatsNodeResponse(StreamInput in) throws IOException {
         this.nodeInfo = new NodeInfo(in);
         this.nodeStats = new NodeStats(in);
         shardsStats = in.readArray(ShardStats::new, ShardStats[]::new);
-        if (in.getVersion().onOrAfter(Version.V_8_6_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_6_0)) {
             searchUsageStats = new SearchUsageStats(in);
         } else {
             searchUsageStats = new SearchUsageStats();
@@ -101,7 +101,7 @@ public void writeTo(StreamOutput out) throws IOException {
         nodeInfo.writeTo(out);
         nodeStats.writeTo(out);
         out.writeArray(shardsStats);
-        if (out.getVersion().onOrAfter(Version.V_8_6_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_6_0)) {
             searchUsageStats.writeTo(out);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java
index f095c5d915cf..9d5df6ef1e4e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.cluster.stats;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.cluster.ClusterName;
@@ -41,7 +41,7 @@ public ClusterStatsResponse(StreamInput in) throws IOException {
         MappingStats mappingStats = in.readOptionalWriteable(MappingStats::new);
         AnalysisStats analysisStats = in.readOptionalWriteable(AnalysisStats::new);
         VersionStats versionStats = null;
-        if (in.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             versionStats = in.readOptionalWriteable(VersionStats::new);
         }
         this.clusterUUID = clusterUUID;
@@ -105,7 +105,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(clusterUUID);
         out.writeOptionalWriteable(indicesStats.getMappings());
         out.writeOptionalWriteable(indicesStats.getAnalysis());
-        if (out.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             out.writeOptionalWriteable(indicesStats.getVersions());
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java
index c509d7a3e88b..bc0ed5e70c59 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.cluster.stats;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
@@ -207,7 +207,7 @@ private static int countOccurrences(String script, Pattern pattern) {
     }
 
     MappingStats(StreamInput in) throws IOException {
-        if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             totalFieldCount = in.readOptionalVLong();
             totalDeduplicatedFieldCount = in.readOptionalVLong();
             totalMappingSizeBytes = in.readOptionalVLong();
@@ -222,7 +222,7 @@ private static int countOccurrences(String script, Pattern pattern) {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             out.writeOptionalVLong(totalFieldCount);
             out.writeOptionalVLong(totalDeduplicatedFieldCount);
             out.writeOptionalVLong(totalMappingSizeBytes);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java
index 04fc4075380a..a8fc72afb146 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java
@@ -10,16 +10,17 @@
 
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.service.PendingClusterTask;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
+import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.List;
 
-public class PendingClusterTasksResponse extends ActionResponse implements Iterable<PendingClusterTask>, ToXContentObject {
+public class PendingClusterTasksResponse extends ActionResponse implements ChunkedToXContentObject {
 
     private final List<PendingClusterTask> pendingTasks;
 
@@ -36,23 +37,11 @@ public List<PendingClusterTask> pendingTasks() {
         return pendingTasks;
     }
 
-    /**
-     * The pending cluster tasks
-     */
-    public List<PendingClusterTask> getPendingTasks() {
-        return pendingTasks();
-    }
-
-    @Override
-    public Iterator<PendingClusterTask> iterator() {
-        return pendingTasks.iterator();
-    }
-
     @Override
     public String toString() {
         StringBuilder sb = new StringBuilder();
         sb.append("tasks: (").append(pendingTasks.size()).append("):\n");
-        for (PendingClusterTask pendingClusterTask : this) {
+        for (PendingClusterTask pendingClusterTask : pendingTasks) {
             sb.append(pendingClusterTask.getInsertOrder())
                 .append("/")
                 .append(pendingClusterTask.getPriority())
@@ -66,10 +55,12 @@ public String toString() {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        builder.startArray(Fields.TASKS);
-        for (PendingClusterTask pendingClusterTask : this) {
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        return Iterators.concat(Iterators.single((builder, p) -> {
+            builder.startObject();
+            builder.startArray(Fields.TASKS);
+            return builder;
+        }), pendingTasks.stream().<ToXContent>map(pendingClusterTask -> (builder, p) -> {
             builder.startObject();
             builder.field(Fields.INSERT_ORDER, pendingClusterTask.getInsertOrder());
             builder.field(Fields.PRIORITY, pendingClusterTask.getPriority());
@@ -78,10 +69,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(Fields.TIME_IN_QUEUE_MILLIS, pendingClusterTask.getTimeInQueueInMillis());
             builder.field(Fields.TIME_IN_QUEUE, pendingClusterTask.getTimeInQueue());
             builder.endObject();
-        }
-        builder.endArray();
-        builder.endObject();
-        return builder;
+            return builder;
+        }).iterator(), Iterators.single((builder, p) -> {
+            builder.endArray();
+            builder.endObject();
+            return builder;
+        }));
     }
 
     static final class Fields {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java
index 098c0de05877..d594ab670106 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java
@@ -10,7 +10,6 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
@@ -27,7 +26,6 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
-import java.util.List;
 
 /**
  * Indices clear cache action.
@@ -62,20 +60,20 @@ public TransportClearIndicesCacheAction(
 
     @Override
     protected EmptyResult readShardResult(StreamInput in) throws IOException {
-        return EmptyResult.readEmptyResultFrom(in);
+        return EmptyResult.INSTANCE;
     }
 
     @Override
-    protected ClearIndicesCacheResponse newResponse(
+    protected ResponseFactory<ClearIndicesCacheResponse, TransportBroadcastByNodeAction.EmptyResult> getResponseFactory(
         ClearIndicesCacheRequest request,
-        int totalShards,
-        int successfulShards,
-        int failedShards,
-        List<EmptyResult> responses,
-        List<DefaultShardOperationFailedException> shardFailures,
         ClusterState clusterState
     ) {
-        return new ClearIndicesCacheResponse(totalShards, successfulShards, failedShards, shardFailures);
+        return (totalShards, successfulShards, failedShards, responses, shardFailures) -> new ClearIndicesCacheResponse(
+            totalShards,
+            successfulShards,
+            failedShards,
+            shardFailures
+        );
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java
index 9e3cb34e9a5d..8a46daa45e73 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java
@@ -190,7 +190,7 @@ public CreateIndexClusterStateUpdateRequest performReroute(boolean performRerout
     }
 
     /**
-     * @return The composable index template that matches with the index that will be cretaed by this request.
+     * @return The composable index template that matches with the index that will be created by this request.
      */
     public ComposableIndexTemplate matchingTemplate() {
         return matchingTemplate;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
index 985115cca377..5e75cc1c10be 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
@@ -10,7 +10,7 @@
 
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.admin.indices.alias.Alias;
@@ -82,7 +82,7 @@ public CreateIndexRequest(StreamInput in) throws IOException {
         cause = in.readString();
         index = in.readString();
         settings = readSettingsFromStream(in);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             int size = in.readVInt();
             assert size <= 1 : "Expected to read 0 or 1 mappings, but received " + size;
             if (size == 1) {
@@ -100,7 +100,7 @@ public CreateIndexRequest(StreamInput in) throws IOException {
             aliases.add(new Alias(in));
         }
         waitForActiveShards = ActiveShardCount.readFrom(in);
-        if (in.getVersion().onOrAfter(Version.V_7_12_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_12_0)) {
             origin = in.readString();
         }
     }
@@ -453,7 +453,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(cause);
         out.writeString(index);
         settings.writeTo(out);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             if ("{}".equals(mappings)) {
                 out.writeVInt(0);
             } else {
@@ -466,7 +466,7 @@ public void writeTo(StreamOutput out) throws IOException {
         }
         out.writeCollection(aliases);
         waitForActiveShards.writeTo(out);
-        if (out.getVersion().onOrAfter(Version.V_7_12_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_12_0)) {
             out.writeString(origin);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java
index ba47c28c869d..1b5d58491438 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java
@@ -20,11 +20,13 @@
 import org.apache.lucene.codecs.TermVectorsReader;
 import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
 import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.LeafReaderContext;
@@ -38,7 +40,6 @@
 import org.apache.lucene.index.TermState;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.VectorValues;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FilterDirectory;
@@ -137,7 +138,7 @@ void analyzeStoredFields(SegmentReader reader, IndexDiskUsageStats stats) throws
         final int skipMask = 0x1FF; // 511
         while (docID < reader.maxDoc()) {
             cancellationChecker.logEvent();
-            storedFieldsReader.visitDocument(docID, visitor);
+            storedFieldsReader.document(docID, visitor);
             // As we already estimate the size of stored fields, we can trade off the accuracy for the speed of the estimate.
             // Here we only visit 1/11 documents instead of all documents. Ideally, we should visit 1 doc then skip 10 docs
             // to avoid missing many skew documents. But, documents are stored in chunks in compressed format and a chunk can
@@ -525,23 +526,47 @@ void analyzeKnnVectors(SegmentReader reader, IndexDiskUsageStats stats) throws I
             cancellationChecker.checkForCancellation();
             directory.resetBytesRead();
             if (field.getVectorDimension() > 0) {
-                iterateDocValues(reader.maxDoc(), () -> vectorReader.getVectorValues(field.name), vectors -> {
-                    cancellationChecker.logEvent();
-                    vectors.vectorValue();
-                });
-
-                // do a couple of randomized searches to figure out min and max offsets of index file
-                VectorValues vectorValues = vectorReader.getVectorValues(field.name);
-                int numDocsToVisit = reader.maxDoc() < 10 ? reader.maxDoc() : 10 * (int) Math.log10(reader.maxDoc());
-                int skipFactor = Math.max(reader.maxDoc() / numDocsToVisit, 1);
-                for (int i = 0; i < reader.maxDoc(); i += skipFactor) {
-                    if ((i = vectorValues.advance(i)) == DocIdSetIterator.NO_MORE_DOCS) {
-                        break;
+                switch (field.getVectorEncoding()) {
+                    case BYTE -> {
+                        iterateDocValues(reader.maxDoc(), () -> vectorReader.getByteVectorValues(field.name), vectors -> {
+                            cancellationChecker.logEvent();
+                            vectors.vectorValue();
+                        });
+
+                        // do a couple of randomized searches to figure out min and max offsets of index file
+                        ByteVectorValues vectorValues = vectorReader.getByteVectorValues(field.name);
+                        int numDocsToVisit = reader.maxDoc() < 10 ? reader.maxDoc() : 10 * (int) Math.log10(reader.maxDoc());
+                        int skipFactor = Math.max(reader.maxDoc() / numDocsToVisit, 1);
+                        for (int i = 0; i < reader.maxDoc(); i += skipFactor) {
+                            if ((i = vectorValues.advance(i)) == DocIdSetIterator.NO_MORE_DOCS) {
+                                break;
+                            }
+                            cancellationChecker.checkForCancellation();
+                            vectorReader.search(field.name, vectorValues.vectorValue(), 100, null, Integer.MAX_VALUE);
+                        }
+                        stats.addKnnVectors(field.name, directory.getBytesRead());
+                    }
+                    case FLOAT32 -> {
+                        iterateDocValues(reader.maxDoc(), () -> vectorReader.getFloatVectorValues(field.name), vectors -> {
+                            cancellationChecker.logEvent();
+                            vectors.vectorValue();
+                        });
+
+                        // do a couple of randomized searches to figure out min and max offsets of index file
+                        FloatVectorValues vectorValues = vectorReader.getFloatVectorValues(field.name);
+                        int numDocsToVisit = reader.maxDoc() < 10 ? reader.maxDoc() : 10 * (int) Math.log10(reader.maxDoc());
+                        int skipFactor = Math.max(reader.maxDoc() / numDocsToVisit, 1);
+                        for (int i = 0; i < reader.maxDoc(); i += skipFactor) {
+                            if ((i = vectorValues.advance(i)) == DocIdSetIterator.NO_MORE_DOCS) {
+                                break;
+                            }
+                            cancellationChecker.checkForCancellation();
+                            vectorReader.search(field.name, vectorValues.vectorValue(), 100, null, Integer.MAX_VALUE);
+                        }
+                        stats.addKnnVectors(field.name, directory.getBytesRead());
                     }
-                    cancellationChecker.checkForCancellation();
-                    vectorReader.search(field.name, vectorValues.vectorValue(), 100, null, Integer.MAX_VALUE);
                 }
-                stats.addKnnVectors(field.name, directory.getBytesRead());
+
             }
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageStats.java
index 74b3240bdc2b..7534f6653e47 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageStats.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageStats.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.indices.diskusage;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -189,7 +189,7 @@ private PerFieldDiskUsage(StreamInput in) throws IOException {
             pointsBytes = in.readVLong();
             normsBytes = in.readVLong();
             termVectorsBytes = in.readVLong();
-            if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
                 knnVectorsBytes = in.readVLong();
             }
         }
@@ -202,7 +202,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeVLong(pointsBytes);
             out.writeVLong(normsBytes);
             out.writeVLong(termVectorsBytes);
-            if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
                 out.writeVLong(knnVectorsBytes);
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java
index a828f6e413d7..6fce79e31b91 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.util.List;
@@ -46,15 +47,11 @@ public TransportFlushAction(
             client,
             actionFilters,
             indexNameExpressionResolver,
-            TransportShardFlushAction.TYPE
+            TransportShardFlushAction.TYPE,
+            ThreadPool.Names.FLUSH
         );
     }
 
-    @Override
-    protected ReplicationResponse newShardResponse() {
-        return new ReplicationResponse();
-    }
-
     @Override
     protected ShardFlushRequest newShardRequest(FlushRequest request, ShardId shardId) {
         return new ShardFlushRequest(request, shardId);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java
index 32e67e95d193..6dca3285269e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.indices.flush;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.ActionFilters;
@@ -103,7 +103,7 @@ private static class PreShardSyncedFlushRequest extends TransportRequest {
 
         private PreShardSyncedFlushRequest(StreamInput in) throws IOException {
             super(in);
-            assert in.getVersion().before(Version.V_8_0_0) : "received pre_sync request from a new node";
+            assert in.getTransportVersion().before(TransportVersion.V_8_0_0) : "received pre_sync request from a new node";
             this.shardId = new ShardId(in);
         }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequest.java
index af416fa95a86..770b4a64ad24 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.indices.forcemerge;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.broadcast.BroadcastRequest;
 import org.elasticsearch.common.UUIDs;
@@ -48,7 +48,7 @@ public static final class Defaults {
      */
     private boolean shouldStoreResult;
 
-    private static final Version FORCE_MERGE_UUID_SIMPLE_VERSION = Version.V_8_0_0;
+    private static final TransportVersion FORCE_MERGE_UUID_SIMPLE_VERSION = TransportVersion.V_8_0_0;
 
     /**
      * Force merge UUID to store in the live commit data of a shard under
@@ -71,7 +71,7 @@ public ForceMergeRequest(StreamInput in) throws IOException {
         maxNumSegments = in.readInt();
         onlyExpungeDeletes = in.readBoolean();
         flush = in.readBoolean();
-        if (in.getVersion().onOrAfter(FORCE_MERGE_UUID_SIMPLE_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(FORCE_MERGE_UUID_SIMPLE_VERSION)) {
             forceMergeUUID = in.readString();
         } else {
             forceMergeUUID = in.readOptionalString();
@@ -167,7 +167,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeInt(maxNumSegments);
         out.writeBoolean(onlyExpungeDeletes);
         out.writeBoolean(flush);
-        if (out.getVersion().onOrAfter(FORCE_MERGE_UUID_SIMPLE_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(FORCE_MERGE_UUID_SIMPLE_VERSION)) {
             out.writeString(forceMergeUUID);
         } else {
             out.writeOptionalString(forceMergeUUID);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java
index cd5713e69bdd..8c97bd29ebea 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/TransportForceMergeAction.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
@@ -30,7 +29,6 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
-import java.util.List;
 
 /**
  * ForceMerge index/indices action.
@@ -66,20 +64,17 @@ public TransportForceMergeAction(
 
     @Override
     protected EmptyResult readShardResult(StreamInput in) throws IOException {
-        return EmptyResult.readEmptyResultFrom(in);
+        return EmptyResult.INSTANCE;
     }
 
     @Override
-    protected ForceMergeResponse newResponse(
-        ForceMergeRequest request,
-        int totalShards,
-        int successfulShards,
-        int failedShards,
-        List<EmptyResult> responses,
-        List<DefaultShardOperationFailedException> shardFailures,
-        ClusterState clusterState
-    ) {
-        return new ForceMergeResponse(totalShards, successfulShards, failedShards, shardFailures);
+    protected ResponseFactory<ForceMergeResponse, EmptyResult> getResponseFactory(ForceMergeRequest request, ClusterState clusterState) {
+        return (totalShards, successfulShards, failedShards, responses, shardFailures) -> new ForceMergeResponse(
+            totalShards,
+            successfulShards,
+            failedShards,
+            shardFailures
+        );
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java
index 4f8f24ea5b72..a36a7e606866 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java
@@ -8,21 +8,23 @@
 
 package org.elasticsearch.action.admin.indices.get;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.metadata.AliasMetadata;
 import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -33,7 +35,7 @@
 /**
  * A response for a get index action.
  */
-public class GetIndexResponse extends ActionResponse implements ToXContentObject {
+public class GetIndexResponse extends ActionResponse implements ChunkedToXContentObject {
 
     private Map<String, MappingMetadata> mappings = Map.of();
     private Map<String, List<AliasMetadata>> aliases = Map.of();
@@ -73,7 +75,7 @@ public GetIndexResponse(
     GetIndexResponse(StreamInput in) throws IOException {
         super(in);
         this.indices = in.readStringArray();
-        mappings = in.readImmutableOpenMap(StreamInput::readString, in.getVersion().before(Version.V_8_0_0) ? i -> {
+        mappings = in.readImmutableOpenMap(StreamInput::readString, in.getTransportVersion().before(TransportVersion.V_8_0_0) ? i -> {
             int numMappings = i.readVInt();
             assert numMappings == 0 || numMappings == 1 : "Expected 0 or 1 mappings but got " + numMappings;
             if (numMappings == 1) {
@@ -178,59 +180,58 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        {
-            for (final String index : indices) {
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return Iterators.concat(
+            Iterators.single((builder, params) -> builder.startObject()),
+            Arrays.stream(indices).<ToXContent>map(index -> (builder, params) -> {
                 builder.startObject(index);
-                {
-                    builder.startObject("aliases");
-                    List<AliasMetadata> indexAliases = aliases.get(index);
-                    if (indexAliases != null) {
-                        for (final AliasMetadata alias : indexAliases) {
-                            AliasMetadata.Builder.toXContent(alias, builder, params);
-                        }
-                    }
-                    builder.endObject();
-
-                    MappingMetadata indexMappings = mappings.get(index);
-                    if (indexMappings == null) {
-                        builder.startObject("mappings").endObject();
-                    } else {
-                        if (builder.getRestApiVersion() == RestApiVersion.V_7
-                            && params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY)) {
-                            builder.startObject("mappings");
-                            builder.field(MapperService.SINGLE_MAPPING_NAME, indexMappings.sourceAsMap());
-                            builder.endObject();
-                        } else {
-                            builder.field("mappings", indexMappings.sourceAsMap());
-                        }
-                    }
 
-                    builder.startObject("settings");
-                    Settings indexSettings = settings.get(index);
-                    if (indexSettings != null) {
-                        indexSettings.toXContent(builder, params);
+                builder.startObject("aliases");
+                List<AliasMetadata> indexAliases = aliases.get(index);
+                if (indexAliases != null) {
+                    for (final AliasMetadata alias : indexAliases) {
+                        AliasMetadata.Builder.toXContent(alias, builder, params);
                     }
-                    builder.endObject();
+                }
+                builder.endObject();
 
-                    Settings defaultIndexSettings = defaultSettings.get(index);
-                    if (defaultIndexSettings != null && defaultIndexSettings.isEmpty() == false) {
-                        builder.startObject("defaults");
-                        defaultIndexSettings.toXContent(builder, params);
+                MappingMetadata indexMappings = mappings.get(index);
+                if (indexMappings == null) {
+                    builder.startObject("mappings").endObject();
+                } else {
+                    if (builder.getRestApiVersion() == RestApiVersion.V_7
+                        && params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY)) {
+                        builder.startObject("mappings");
+                        builder.field(MapperService.SINGLE_MAPPING_NAME, indexMappings.sourceAsMap());
                         builder.endObject();
+                    } else {
+                        builder.field("mappings", indexMappings.sourceAsMap());
                     }
+                }
 
-                    String dataStream = dataStreams.get(index);
-                    if (dataStream != null) {
-                        builder.field("data_stream", dataStream);
-                    }
+                builder.startObject("settings");
+                Settings indexSettings = settings.get(index);
+                if (indexSettings != null) {
+                    indexSettings.toXContent(builder, params);
                 }
                 builder.endObject();
-            }
-        }
-        builder.endObject();
-        return builder;
+
+                Settings defaultIndexSettings = defaultSettings.get(index);
+                if (defaultIndexSettings != null && defaultIndexSettings.isEmpty() == false) {
+                    builder.startObject("defaults");
+                    defaultIndexSettings.toXContent(builder, params);
+                    builder.endObject();
+                }
+
+                String dataStream = dataStreams.get(index);
+                if (dataStream != null) {
+                    builder.field("data_stream", dataStream);
+                }
+
+                return builder.endObject();
+            }).iterator(),
+            Iterators.single((builder, params) -> builder.endObject())
+        );
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsIndexRequest.java
index f8fd09fcadfd..47a17006dc4b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsIndexRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsIndexRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.indices.mapping.get;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -28,12 +28,12 @@ public class GetFieldMappingsIndexRequest extends SingleShardRequest<GetFieldMap
 
     GetFieldMappingsIndexRequest(StreamInput in) throws IOException {
         super(in);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             in.readStringArray();   // former types array
         }
         fields = in.readStringArray();
         includeDefaults = in.readBoolean();
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             in.readBoolean();       // former probablySingleField boolean
         }
         originalIndices = OriginalIndices.readOriginalIndices(in);
@@ -73,12 +73,12 @@ public IndicesOptions indicesOptions() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeStringArray(Strings.EMPTY_ARRAY);
         }
         out.writeStringArray(fields);
         out.writeBoolean(includeDefaults);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeBoolean(false);
         }
         OriginalIndices.writeOriginalIndices(originalIndices, out);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java
index 91173223a6c0..fd6e6ec25649 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.indices.mapping.get;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
@@ -41,7 +41,7 @@ public GetFieldMappingsRequest() {}
     public GetFieldMappingsRequest(StreamInput in) throws IOException {
         super(in);
         indices = in.readStringArray();
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             String[] types = in.readStringArray();
             if (types != Strings.EMPTY_ARRAY) {
                 throw new IllegalArgumentException("Expected empty type array but received [" + Arrays.toString(types) + "]");
@@ -50,7 +50,7 @@ public GetFieldMappingsRequest(StreamInput in) throws IOException {
         }
         indicesOptions = IndicesOptions.readIndicesOptions(in);
         // Consume the deprecated local parameter
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             in.readBoolean();
         }
         fields = in.readStringArray();
@@ -112,11 +112,11 @@ public ActionRequestValidationException validate() {
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeStringArray(indices);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeStringArray(Strings.EMPTY_ARRAY);
         }
         indicesOptions.writeIndicesOptions(out);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeBoolean(true);
         }
         out.writeStringArray(fields);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
index 77de76ab9508..fd6c815a7462 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.indices.mapping.get;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -52,7 +52,7 @@ public class GetFieldMappingsResponse extends ActionResponse implements ToXConte
     GetFieldMappingsResponse(StreamInput in) throws IOException {
         super(in);
         mappings = in.readImmutableMap(StreamInput::readString, mapIn -> {
-            if (mapIn.getVersion().before(Version.V_8_0_0)) {
+            if (mapIn.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 int typesSize = mapIn.readVInt();
                 assert typesSize == 1 || typesSize == 0 : "Expected 0 or 1 types but got " + typesSize;
                 if (typesSize == 0) {
@@ -157,7 +157,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeMap(mappings, StreamOutput::writeString, (outpt, map) -> {
-            if (outpt.getVersion().before(Version.V_8_0_0)) {
+            if (outpt.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 outpt.writeVInt(1);
                 outpt.writeString(MapperService.SINGLE_MAPPING_NAME);
             }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java
index b2360de13238..22688f9f81a6 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java
@@ -8,14 +8,14 @@
 
 package org.elasticsearch.action.admin.indices.mapping.get;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.xcontent.ParseField;
@@ -28,7 +28,7 @@
 import static org.elasticsearch.rest.BaseRestHandler.DEFAULT_INCLUDE_TYPE_NAME_POLICY;
 import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER;
 
-public class GetMappingsResponse extends ActionResponse implements ChunkedToXContent {
+public class GetMappingsResponse extends ActionResponse implements ChunkedToXContentObject {
 
     private static final ParseField MAPPINGS = new ParseField("mappings");
 
@@ -40,7 +40,7 @@ public GetMappingsResponse(Map<String, MappingMetadata> mappings) {
 
     GetMappingsResponse(StreamInput in) throws IOException {
         super(in);
-        mappings = in.readImmutableMap(StreamInput::readString, in.getVersion().before(Version.V_8_0_0) ? i -> {
+        mappings = in.readImmutableMap(StreamInput::readString, in.getTransportVersion().before(TransportVersion.V_8_0_0) ? i -> {
             int mappingCount = i.readVInt();
             assert mappingCount == 1 || mappingCount == 0 : "Expected 0 or 1 mappings but got " + mappingCount;
             if (mappingCount == 1) {
@@ -67,7 +67,7 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public Iterator<ToXContent> toXContentChunked() {
+    public Iterator<ToXContent> toXContentChunked(ToXContent.Params outerParams) {
         return Iterators.concat(
             Iterators.single((b, p) -> b.startObject()),
             getMappings().entrySet().stream().map(indexEntry -> (ToXContent) (builder, params) -> {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
index 80a032c4542c..5635b39d0199 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.action.admin.indices.mapping.put;
 
 import org.elasticsearch.ElasticsearchGenerationException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -81,7 +81,7 @@ public PutMappingRequest(StreamInput in) throws IOException {
         super(in);
         indices = in.readStringArray();
         indicesOptions = IndicesOptions.readIndicesOptions(in);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             String type = in.readOptionalString();
             if (MapperService.SINGLE_MAPPING_NAME.equals(type) == false) {
                 throw new IllegalArgumentException("Expected type [_doc] but received [" + type + "]");
@@ -90,7 +90,7 @@ public PutMappingRequest(StreamInput in) throws IOException {
         source = in.readString();
         concreteIndex = in.readOptionalWriteable(Index::new);
         origin = in.readOptionalString();
-        if (in.getVersion().onOrAfter(Version.V_7_9_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_9_0)) {
             writeIndexOnly = in.readBoolean();
         }
     }
@@ -313,13 +313,13 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeStringArrayNullable(indices);
         indicesOptions.writeIndicesOptions(out);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeOptionalString(MapperService.SINGLE_MAPPING_NAME);
         }
         out.writeString(source);
         out.writeOptionalWriteable(concreteIndex);
         out.writeOptionalString(origin);
-        if (out.getVersion().onOrAfter(Version.V_7_9_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_9_0)) {
             out.writeBoolean(writeIndexOnly);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryResponse.java
index e53f17b68559..ea04ccf62763 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryResponse.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.xcontent.ToXContent;
 
@@ -26,7 +26,7 @@
 /**
  * Information regarding the recovery state of indices and their associated shards.
  */
-public class RecoveryResponse extends BaseBroadcastResponse implements ChunkedToXContent {
+public class RecoveryResponse extends BaseBroadcastResponse implements ChunkedToXContentObject {
 
     private final Map<String, List<RecoveryState>> shardRecoveryStates;
 
@@ -65,7 +65,7 @@ public Map<String, List<RecoveryState>> shardRecoveryStates() {
     }
 
     @Override
-    public Iterator<ToXContent> toXContentChunked() {
+    public Iterator<ToXContent> toXContentChunked(ToXContent.Params params) {
         return Iterators.concat(
             Iterators.single((b, p) -> b.startObject()),
             shardRecoveryStates.entrySet()
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java
index c5d39cdb5c19..e6b1758b1c28 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java
@@ -10,7 +10,6 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
@@ -71,33 +70,27 @@ protected RecoveryState readShardResult(StreamInput in) throws IOException {
     }
 
     @Override
-    protected RecoveryResponse newResponse(
-        RecoveryRequest request,
-        int totalShards,
-        int successfulShards,
-        int failedShards,
-        List<RecoveryState> responses,
-        List<DefaultShardOperationFailedException> shardFailures,
-        ClusterState clusterState
-    ) {
-        Map<String, List<RecoveryState>> shardResponses = new HashMap<>();
-        for (RecoveryState recoveryState : responses) {
-            if (recoveryState == null) {
-                continue;
-            }
-            String indexName = recoveryState.getShardId().getIndexName();
-            if (shardResponses.containsKey(indexName) == false) {
-                shardResponses.put(indexName, new ArrayList<>());
-            }
-            if (request.activeOnly()) {
-                if (recoveryState.getStage() != RecoveryState.Stage.DONE) {
+    protected ResponseFactory<RecoveryResponse, RecoveryState> getResponseFactory(RecoveryRequest request, ClusterState clusterState) {
+        return (totalShards, successfulShards, failedShards, responses, shardFailures) -> {
+            Map<String, List<RecoveryState>> shardResponses = new HashMap<>();
+            for (RecoveryState recoveryState : responses) {
+                if (recoveryState == null) {
+                    continue;
+                }
+                String indexName = recoveryState.getShardId().getIndexName();
+                if (shardResponses.containsKey(indexName) == false) {
+                    shardResponses.put(indexName, new ArrayList<>());
+                }
+                if (request.activeOnly()) {
+                    if (recoveryState.getStage() != RecoveryState.Stage.DONE) {
+                        shardResponses.get(indexName).add(recoveryState);
+                    }
+                } else {
                     shardResponses.get(indexName).add(recoveryState);
                 }
-            } else {
-                shardResponses.get(indexName).add(recoveryState);
             }
-        }
-        return new RecoveryResponse(totalShards, successfulShards, failedShards, shardResponses, shardFailures);
+            return new RecoveryResponse(totalShards, successfulShards, failedShards, shardResponses, shardFailures);
+        };
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java
index ff9f6640b412..ceb940502da5 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.util.List;
@@ -48,15 +49,11 @@ public TransportRefreshAction(
             client,
             actionFilters,
             indexNameExpressionResolver,
-            TransportShardRefreshAction.TYPE
+            TransportShardRefreshAction.TYPE,
+            ThreadPool.Names.REFRESH
         );
     }
 
-    @Override
-    protected ReplicationResponse newShardResponse() {
-        return new ReplicationResponse();
-    }
-
     @Override
     protected BasicReplicationRequest newShardRequest(RefreshRequest request, ShardId shardId) {
         BasicReplicationRequest replicationRequest = new BasicReplicationRequest(shardId);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java
index 27e185e98a9f..c7e7ab973382 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java
@@ -9,30 +9,45 @@
 package org.elasticsearch.action.admin.indices.refresh;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionListenerResponseHandler;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.RefCountingListener;
 import org.elasticsearch.action.support.replication.BasicReplicationRequest;
 import org.elasticsearch.action.support.replication.ReplicationResponse;
 import org.elasticsearch.action.support.replication.TransportReplicationAction;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportRequestOptions;
+import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 public class TransportShardRefreshAction extends TransportReplicationAction<
     BasicReplicationRequest,
     BasicReplicationRequest,
     ReplicationResponse> {
 
+    private static final Logger logger = LogManager.getLogger(TransportShardRefreshAction.class);
+
     public static final String NAME = RefreshAction.NAME + "[s]";
     public static final ActionType<ReplicationResponse> TYPE = new ActionType<>(NAME, ReplicationResponse::new);
+    public static final String SOURCE_API = "api";
 
     @Inject
     public TransportShardRefreshAction(
@@ -57,6 +72,7 @@ public TransportShardRefreshAction(
             BasicReplicationRequest::new,
             ThreadPool.Names.REFRESH
         );
+        new TransportUnpromotableShardRefreshAction(transportService, actionFilters, indicesService);
     }
 
     @Override
@@ -70,17 +86,49 @@ protected void shardOperationOnPrimary(
         IndexShard primary,
         ActionListener<PrimaryResult<BasicReplicationRequest, ReplicationResponse>> listener
     ) {
-        ActionListener.completeWith(listener, () -> {
-            primary.refresh("api");
+        try (var listeners = new RefCountingListener(listener.map(v -> new PrimaryResult<>(shardRequest, new ReplicationResponse())))) {
+            var refreshResult = primary.refresh(SOURCE_API);
             logger.trace("{} refresh request executed on primary", primary.shardId());
-            return new PrimaryResult<>(shardRequest, new ReplicationResponse());
-        });
+
+            // Forward the request to all nodes that hold unpromotable replica shards
+            final ClusterState clusterState = clusterService.state();
+            final Task parentTaskId = taskManager.getTask(shardRequest.getParentTask().getId());
+            clusterState.routingTable()
+                .shardRoutingTable(shardRequest.shardId())
+                .assignedShards()
+                .stream()
+                .filter(Predicate.not(ShardRouting::isPromotableToPrimary))
+                .map(ShardRouting::currentNodeId)
+                .collect(Collectors.toUnmodifiableSet())
+                .forEach(nodeId -> {
+                    final DiscoveryNode node = clusterState.nodes().get(nodeId);
+                    UnpromotableShardRefreshRequest request = new UnpromotableShardRefreshRequest(
+                        primary.shardId(),
+                        refreshResult.generation()
+                    );
+                    logger.trace("forwarding refresh request [{}] to node [{}]", request, node);
+                    transportService.sendChildRequest(
+                        node,
+                        TransportUnpromotableShardRefreshAction.NAME,
+                        request,
+                        parentTaskId,
+                        TransportRequestOptions.EMPTY,
+                        new ActionListenerResponseHandler<>(
+                            listeners.acquire(ignored -> {}),
+                            (in) -> TransportResponse.Empty.INSTANCE,
+                            ThreadPool.Names.REFRESH
+                        )
+                    );
+                });
+        } catch (Exception e) {
+            listener.onFailure(e);
+        }
     }
 
     @Override
     protected void shardOperationOnReplica(BasicReplicationRequest request, IndexShard replica, ActionListener<ReplicaResult> listener) {
         ActionListener.completeWith(listener, () -> {
-            replica.refresh("api");
+            replica.refresh(SOURCE_API);
             logger.trace("{} refresh request executed on replica", replica.shardId());
             return new ReplicaResult();
         });
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java
new file mode 100644
index 000000000000..500a53513a60
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.indices.refresh;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+public class TransportUnpromotableShardRefreshAction extends HandledTransportAction<UnpromotableShardRefreshRequest, ActionResponse.Empty> {
+    public static final String NAME = RefreshAction.NAME + "[u]";
+
+    private final IndicesService indicesService;
+
+    @Inject
+    public TransportUnpromotableShardRefreshAction(
+        TransportService transportService,
+        ActionFilters actionFilters,
+        IndicesService indicesService
+    ) {
+        super(NAME, transportService, actionFilters, UnpromotableShardRefreshRequest::new, ThreadPool.Names.REFRESH);
+        this.indicesService = indicesService;
+    }
+
+    @Override
+    protected void doExecute(Task task, UnpromotableShardRefreshRequest request, ActionListener<ActionResponse.Empty> responseListener) {
+        ActionListener.run(responseListener, listener -> {
+            assert request.getSegmentGeneration() != Engine.RefreshResult.UNKNOWN_GENERATION
+                : "The request segment is " + request.getSegmentGeneration();
+            IndexShard shard = indicesService.indexServiceSafe(request.getShardId().getIndex()).getShard(request.getShardId().id());
+            shard.waitForSegmentGeneration(request.getSegmentGeneration(), listener.map(l -> ActionResponse.Empty.INSTANCE));
+        });
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/UnpromotableShardRefreshRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/UnpromotableShardRefreshRequest.java
new file mode 100644
index 000000000000..52ef3917ce72
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/UnpromotableShardRefreshRequest.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.indices.refresh;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.shard.ShardId;
+
+import java.io.IOException;
+
+public class UnpromotableShardRefreshRequest extends ActionRequest {
+
+    private final ShardId shardId;
+    private final long segmentGeneration;
+
+    public UnpromotableShardRefreshRequest(final ShardId shardId, long segmentGeneration) {
+        this.shardId = shardId;
+        this.segmentGeneration = segmentGeneration;
+    }
+
+    public UnpromotableShardRefreshRequest(StreamInput in) throws IOException {
+        super(in);
+        shardId = new ShardId(in);
+        segmentGeneration = in.readVLong();
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        shardId.writeTo(out);
+        out.writeVLong(segmentGeneration);
+    }
+
+    public ShardId getShardId() {
+        return shardId;
+    }
+
+    public long getSegmentGeneration() {
+        return segmentGeneration;
+    }
+
+    @Override
+    public String toString() {
+        return "UnpromotableShardRefreshRequest{" + "shardId=" + shardId + ", segmentGeneration=" + segmentGeneration + '}';
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java
index c8c915029574..42f15c073f43 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
-import org.elasticsearch.cluster.metadata.IndexAbstractionResolver;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
@@ -31,6 +30,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.util.concurrent.CountDown;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.Index;
@@ -53,6 +53,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
@@ -137,6 +138,7 @@ public boolean allowsRemoteIndices() {
 
         @Override
         public boolean includeDataStreams() {
+            // request must allow data streams because the index name expression resolver for the action handler assumes it
             return true;
         }
     }
@@ -439,7 +441,7 @@ public static class TransportAction extends HandledTransportAction<Request, Resp
         private final ThreadPool threadPool;
         private final ClusterService clusterService;
         private final RemoteClusterService remoteClusterService;
-        private final IndexAbstractionResolver indexAbstractionResolver;
+        private final IndexNameExpressionResolver indexNameExpressionResolver;
         private final boolean ccsCheckCompatibility;
 
         @Inject
@@ -454,7 +456,7 @@ public TransportAction(
             this.threadPool = threadPool;
             this.clusterService = clusterService;
             this.remoteClusterService = transportService.getRemoteClusterService();
-            this.indexAbstractionResolver = new IndexAbstractionResolver(indexNameExpressionResolver);
+            this.indexNameExpressionResolver = indexNameExpressionResolver;
             this.ccsCheckCompatibility = SearchService.CCS_VERSION_CHECK_SETTING.get(clusterService.getSettings());
         }
 
@@ -469,22 +471,10 @@ protected void doExecute(Task task, Request request, final ActionListener<Respon
                 request.indices()
             );
             final OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
-            final Metadata metadata = clusterState.metadata();
             List<ResolvedIndex> indices = new ArrayList<>();
             List<ResolvedAlias> aliases = new ArrayList<>();
             List<ResolvedDataStream> dataStreams = new ArrayList<>();
-            if (localIndices != null) {
-                resolveIndices(
-                    localIndices.indices(),
-                    request.indicesOptions,
-                    metadata,
-                    indexAbstractionResolver,
-                    indices,
-                    aliases,
-                    dataStreams,
-                    request.includeDataStreams()
-                );
-            }
+            resolveIndices(localIndices, clusterState, indexNameExpressionResolver, indices, aliases, dataStreams);
 
             if (remoteClusterIndices.size() > 0) {
                 final int remoteRequests = remoteClusterIndices.size();
@@ -513,12 +503,36 @@ protected void doExecute(Task task, Request request, final ActionListener<Respon
             }
         }
 
+        /**
+         * Resolves the specified names and/or wildcard expressions to index abstractions. Returns results in the supplied lists.
+         *
+         * @param localIndices   The names and wildcard expressions to resolve
+         * @param clusterState   Cluster state
+         * @param resolver       Resolver instance for matching names
+         * @param indices        List containing any matching indices
+         * @param aliases        List containing any matching aliases
+         * @param dataStreams    List containing any matching data streams
+         */
+        static void resolveIndices(
+            @Nullable OriginalIndices localIndices,
+            ClusterState clusterState,
+            IndexNameExpressionResolver resolver,
+            List<ResolvedIndex> indices,
+            List<ResolvedAlias> aliases,
+            List<ResolvedDataStream> dataStreams
+        ) {
+            if (localIndices == null) {
+                return;
+            }
+            resolveIndices(localIndices.indices(), localIndices.indicesOptions(), clusterState, resolver, indices, aliases, dataStreams);
+        }
+
         /**
          * Resolves the specified names and/or wildcard expressions to index abstractions. Returns results in the supplied lists.
          *
          * @param names          The names and wildcard expressions to resolve
          * @param indicesOptions Options for expanding wildcards to indices with different states
-         * @param metadata       Cluster metadata
+         * @param clusterState   Cluster state
          * @param resolver       Resolver instance for matching names
          * @param indices        List containing any matching indices
          * @param aliases        List containing any matching aliases
@@ -528,22 +542,33 @@ protected void doExecute(Task task, Request request, final ActionListener<Respon
         static void resolveIndices(
             String[] names,
             IndicesOptions indicesOptions,
-            Metadata metadata,
-            IndexAbstractionResolver resolver,
+            ClusterState clusterState,
+            IndexNameExpressionResolver resolver,
             List<ResolvedIndex> indices,
             List<ResolvedAlias> aliases,
-            List<ResolvedDataStream> dataStreams,
-            boolean includeDataStreams
+            List<ResolvedDataStream> dataStreams
         ) {
-            List<String> resolvedIndexAbstractions = resolver.resolveIndexAbstractions(names, indicesOptions, metadata, includeDataStreams);
-            SortedMap<String, IndexAbstraction> lookup = metadata.getIndicesLookup();
+            // redundant check to ensure that we don't resolve the list of empty names to "all" in this context
+            if (names.length == 0) {
+                return;
+            }
+            // TODO This is a dirty hack around the IndexNameExpressionResolver optimisation for "*" as described in:
+            // https://github.com/elastic/elasticsearch/issues/92903.
+            // A standalone "*" expression is resolved slightly differently from a "*" embedded in another expression, eg "idx,*".
+            // The difference is only slight, and it usually doesn't cause problems (see
+            // https://github.com/elastic/elasticsearch/issues/92911 for a description of a problem).
+            // But in the case of the Resolve index API, the difference is observable, because resolving standalone "*" cannot show
+            // aliases (only indices and datastreams). The Resolve index API needs to show the aliases that match wildcards.
+            if (names.length == 1 && (Metadata.ALL.equals(names[0]) || Regex.isMatchAllPattern(names[0]))) {
+                names = new String[] { "**" };
+            }
+            Set<String> resolvedIndexAbstractions = resolver.resolveExpressions(clusterState, indicesOptions, true, names);
             for (String s : resolvedIndexAbstractions) {
-                enrichIndexAbstraction(metadata, s, lookup, indices, aliases, dataStreams);
+                enrichIndexAbstraction(clusterState, s, indices, aliases, dataStreams);
             }
             indices.sort(Comparator.comparing(ResolvedIndexAbstraction::getName));
             aliases.sort(Comparator.comparing(ResolvedIndexAbstraction::getName));
             dataStreams.sort(Comparator.comparing(ResolvedIndexAbstraction::getName));
-
         }
 
         private static void mergeResults(
@@ -568,18 +593,17 @@ private static void mergeResults(
         }
 
         private static void enrichIndexAbstraction(
-            Metadata metadata,
+            ClusterState clusterState,
             String indexAbstraction,
-            SortedMap<String, IndexAbstraction> lookup,
             List<ResolvedIndex> indices,
             List<ResolvedAlias> aliases,
             List<ResolvedDataStream> dataStreams
         ) {
-            IndexAbstraction ia = lookup.get(indexAbstraction);
+            IndexAbstraction ia = clusterState.metadata().getIndicesLookup().get(indexAbstraction);
             if (ia != null) {
                 switch (ia.getType()) {
                     case CONCRETE_INDEX -> {
-                        IndexMetadata writeIndex = metadata.index(ia.getWriteIndex());
+                        IndexMetadata writeIndex = clusterState.metadata().index(ia.getWriteIndex());
                         String[] aliasNames = writeIndex.getAliases().keySet().stream().sorted().toArray(String[]::new);
                         List<Attribute> attributes = new ArrayList<>();
                         attributes.add(writeIndex.getState() == IndexMetadata.State.OPEN ? Attribute.OPEN : Attribute.CLOSED);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
index b61e73bbfa26..ba7d6b03043c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.indices.rollover;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.xcontent.ToXContentFragment;
@@ -41,7 +41,7 @@ protected Condition(String name, Type type) {
      * Checks if this condition is available in a specific version.
      * This makes sure BWC when introducing a new condition which is not recognized by older versions.
      */
-    boolean includedInVersion(Version version) {
+    boolean includedInVersion(TransportVersion version) {
         return true;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxPrimaryShardDocsCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxPrimaryShardDocsCondition.java
index c27b4a7b7e73..678ec96c217c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxPrimaryShardDocsCondition.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxPrimaryShardDocsCondition.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.indices.rollover;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -62,7 +62,7 @@ public static MaxPrimaryShardDocsCondition fromXContent(XContentParser parser) t
     }
 
     @Override
-    boolean includedInVersion(Version version) {
-        return version.onOrAfter(Version.V_8_2_0);
+    boolean includedInVersion(TransportVersion version) {
+        return version.onOrAfter(TransportVersion.V_8_2_0);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinAgeCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinAgeCondition.java
index ddcfadd53dd7..98958d3b015c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinAgeCondition.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinAgeCondition.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.indices.rollover;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.TimeValue;
@@ -64,7 +64,7 @@ public static MinAgeCondition fromXContent(XContentParser parser) throws IOExcep
     }
 
     @Override
-    boolean includedInVersion(Version version) {
-        return version.onOrAfter(Version.V_8_4_0);
+    boolean includedInVersion(TransportVersion version) {
+        return version.onOrAfter(TransportVersion.V_8_4_0);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinDocsCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinDocsCondition.java
index 9a4fffc17018..8c6274cfadb8 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinDocsCondition.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinDocsCondition.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.indices.rollover;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -62,7 +62,7 @@ public static MinDocsCondition fromXContent(XContentParser parser) throws IOExce
     }
 
     @Override
-    boolean includedInVersion(Version version) {
-        return version.onOrAfter(Version.V_8_4_0);
+    boolean includedInVersion(TransportVersion version) {
+        return version.onOrAfter(TransportVersion.V_8_4_0);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinPrimaryShardDocsCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinPrimaryShardDocsCondition.java
index e1aee305742f..6aaea57e5b55 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinPrimaryShardDocsCondition.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinPrimaryShardDocsCondition.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.indices.rollover;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -62,7 +62,7 @@ public static MinPrimaryShardDocsCondition fromXContent(XContentParser parser) t
     }
 
     @Override
-    boolean includedInVersion(Version version) {
-        return version.onOrAfter(Version.V_8_4_0);
+    boolean includedInVersion(TransportVersion version) {
+        return version.onOrAfter(TransportVersion.V_8_4_0);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinPrimaryShardSizeCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinPrimaryShardSizeCondition.java
index 5ec8d26d9672..d7149e2a91be 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinPrimaryShardSizeCondition.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinPrimaryShardSizeCondition.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.indices.rollover;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -63,7 +63,7 @@ public static MinPrimaryShardSizeCondition fromXContent(XContentParser parser) t
     }
 
     @Override
-    boolean includedInVersion(Version version) {
-        return version.onOrAfter(Version.V_8_4_0);
+    boolean includedInVersion(TransportVersion version) {
+        return version.onOrAfter(TransportVersion.V_8_4_0);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinSizeCondition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinSizeCondition.java
index 82cf3c0daf30..52db7ff90cf2 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinSizeCondition.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MinSizeCondition.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.indices.rollover;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -63,7 +63,7 @@ public static MinSizeCondition fromXContent(XContentParser parser) throws IOExce
     }
 
     @Override
-    boolean includedInVersion(Version version) {
-        return version.onOrAfter(Version.V_8_4_0);
+    boolean includedInVersion(TransportVersion version) {
+        return version.onOrAfter(TransportVersion.V_8_4_0);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
index 9916acbef125..fd773a9e19b5 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
@@ -212,7 +212,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(newIndexName);
         out.writeBoolean(dryRun);
         out.writeCollection(
-            conditions.values().stream().filter(c -> c.includedInVersion(out.getVersion())).toList(),
+            conditions.values().stream().filter(c -> c.includedInVersion(out.getTransportVersion())).toList(),
             StreamOutput::writeNamedWriteable
         );
         createIndexRequest.writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java
index 0c5abb578880..5377a5af883f 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java
@@ -13,15 +13,13 @@
 import org.apache.lucene.search.SortedNumericSortField;
 import org.apache.lucene.search.SortedSetSortField;
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
-import org.elasticsearch.action.support.broadcast.BaseBroadcastResponse;
+import org.elasticsearch.action.support.broadcast.ChunkedBroadcastResponse;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.engine.Segment;
-import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -33,7 +31,7 @@
 import java.util.Locale;
 import java.util.Map;
 
-public class IndicesSegmentResponse extends BaseBroadcastResponse implements ChunkedToXContent {
+public class IndicesSegmentResponse extends ChunkedBroadcastResponse {
 
     private final ShardSegments[] shards;
 
@@ -79,72 +77,72 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked() {
-        return Iterators.concat(Iterators.single(((builder, params) -> {
-            builder.startObject();
-            RestActions.buildBroadcastShardsHeader(builder, params, this);
-            return builder.startObject(Fields.INDICES);
-        })), getIndices().values().stream().map(indexSegments -> (ToXContent) (builder, params) -> {
-            builder.startObject(indexSegments.getIndex());
-
-            builder.startObject(Fields.SHARDS);
-            for (IndexShardSegments indexSegment : indexSegments) {
-                builder.startArray(Integer.toString(indexSegment.shardId().id()));
-                for (ShardSegments shardSegments : indexSegment) {
-                    builder.startObject();
-
-                    builder.startObject(Fields.ROUTING);
-                    builder.field(Fields.STATE, shardSegments.getShardRouting().state());
-                    builder.field(Fields.PRIMARY, shardSegments.getShardRouting().primary());
-                    builder.field(Fields.NODE, shardSegments.getShardRouting().currentNodeId());
-                    if (shardSegments.getShardRouting().relocatingNodeId() != null) {
-                        builder.field(Fields.RELOCATING_NODE, shardSegments.getShardRouting().relocatingNodeId());
-                    }
-                    builder.endObject();
-
-                    builder.field(Fields.NUM_COMMITTED_SEGMENTS, shardSegments.getNumberOfCommitted());
-                    builder.field(Fields.NUM_SEARCH_SEGMENTS, shardSegments.getNumberOfSearch());
-
-                    builder.startObject(Fields.SEGMENTS);
-                    for (Segment segment : shardSegments) {
-                        builder.startObject(segment.getName());
-                        builder.field(Fields.GENERATION, segment.getGeneration());
-                        builder.field(Fields.NUM_DOCS, segment.getNumDocs());
-                        builder.field(Fields.DELETED_DOCS, segment.getDeletedDocs());
-                        builder.humanReadableField(Fields.SIZE_IN_BYTES, Fields.SIZE, segment.getSize());
-                        if (builder.getRestApiVersion() == RestApiVersion.V_7) {
-                            builder.humanReadableField(Fields.MEMORY_IN_BYTES, Fields.MEMORY, ByteSizeValue.ZERO);
-                        }
-                        builder.field(Fields.COMMITTED, segment.isCommitted());
-                        builder.field(Fields.SEARCH, segment.isSearch());
-                        if (segment.getVersion() != null) {
-                            builder.field(Fields.VERSION, segment.getVersion());
-                        }
-                        if (segment.isCompound() != null) {
-                            builder.field(Fields.COMPOUND, segment.isCompound());
+    protected Iterator<ToXContent> customXContentChunks(ToXContent.Params params) {
+        return Iterators.concat(
+            Iterators.single((builder, p) -> builder.startObject(Fields.INDICES)),
+            getIndices().values().stream().map(indexSegments -> (ToXContent) (builder, p) -> {
+                builder.startObject(indexSegments.getIndex());
+
+                builder.startObject(Fields.SHARDS);
+                for (IndexShardSegments indexSegment : indexSegments) {
+                    builder.startArray(Integer.toString(indexSegment.shardId().id()));
+                    for (ShardSegments shardSegments : indexSegment) {
+                        builder.startObject();
+
+                        builder.startObject(Fields.ROUTING);
+                        builder.field(Fields.STATE, shardSegments.getShardRouting().state());
+                        builder.field(Fields.PRIMARY, shardSegments.getShardRouting().primary());
+                        builder.field(Fields.NODE, shardSegments.getShardRouting().currentNodeId());
+                        if (shardSegments.getShardRouting().relocatingNodeId() != null) {
+                            builder.field(Fields.RELOCATING_NODE, shardSegments.getShardRouting().relocatingNodeId());
                         }
-                        if (segment.getMergeId() != null) {
-                            builder.field(Fields.MERGE_ID, segment.getMergeId());
-                        }
-                        if (segment.getSegmentSort() != null) {
-                            toXContent(builder, segment.getSegmentSort());
-                        }
-                        if (segment.attributes != null && segment.attributes.isEmpty() == false) {
-                            builder.field("attributes", segment.attributes);
+                        builder.endObject();
+
+                        builder.field(Fields.NUM_COMMITTED_SEGMENTS, shardSegments.getNumberOfCommitted());
+                        builder.field(Fields.NUM_SEARCH_SEGMENTS, shardSegments.getNumberOfSearch());
+
+                        builder.startObject(Fields.SEGMENTS);
+                        for (Segment segment : shardSegments) {
+                            builder.startObject(segment.getName());
+                            builder.field(Fields.GENERATION, segment.getGeneration());
+                            builder.field(Fields.NUM_DOCS, segment.getNumDocs());
+                            builder.field(Fields.DELETED_DOCS, segment.getDeletedDocs());
+                            builder.humanReadableField(Fields.SIZE_IN_BYTES, Fields.SIZE, segment.getSize());
+                            if (builder.getRestApiVersion() == RestApiVersion.V_7) {
+                                builder.humanReadableField(Fields.MEMORY_IN_BYTES, Fields.MEMORY, ByteSizeValue.ZERO);
+                            }
+                            builder.field(Fields.COMMITTED, segment.isCommitted());
+                            builder.field(Fields.SEARCH, segment.isSearch());
+                            if (segment.getVersion() != null) {
+                                builder.field(Fields.VERSION, segment.getVersion());
+                            }
+                            if (segment.isCompound() != null) {
+                                builder.field(Fields.COMPOUND, segment.isCompound());
+                            }
+                            if (segment.getMergeId() != null) {
+                                builder.field(Fields.MERGE_ID, segment.getMergeId());
+                            }
+                            if (segment.getSegmentSort() != null) {
+                                toXContent(builder, segment.getSegmentSort());
+                            }
+                            if (segment.attributes != null && segment.attributes.isEmpty() == false) {
+                                builder.field("attributes", segment.attributes);
+                            }
+                            builder.endObject();
                         }
                         builder.endObject();
-                    }
-                    builder.endObject();
 
-                    builder.endObject();
+                        builder.endObject();
+                    }
+                    builder.endArray();
                 }
-                builder.endArray();
-            }
-            builder.endObject();
+                builder.endObject();
 
-            builder.endObject();
-            return builder;
-        }).iterator(), Iterators.single((builder, params) -> builder.endObject().endObject()));
+                builder.endObject();
+                return builder;
+            }).iterator(),
+            Iterators.single((builder, p) -> builder.endObject())
+        );
     }
 
     private static void toXContent(XContentBuilder builder, Sort sort) throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequest.java
index 12ac41284588..204474654bb0 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.indices.segments;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.support.broadcast.BroadcastRequest;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -28,7 +28,7 @@ public IndicesSegmentsRequest() {
 
     public IndicesSegmentsRequest(StreamInput in) throws IOException {
         super(in);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             in.readBoolean();   // old 'verbose' option, since removed
         }
     }
@@ -40,7 +40,7 @@ public IndicesSegmentsRequest(String... indices) {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeBoolean(false);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java
index 470aedb2895f..0b035af943b5 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/TransportIndicesSegmentsAction.java
@@ -10,7 +10,6 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
@@ -30,7 +29,6 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
-import java.util.List;
 
 public class TransportIndicesSegmentsAction extends TransportBroadcastByNodeAction<
     IndicesSegmentsRequest,
@@ -83,17 +81,12 @@ protected ShardSegments readShardResult(StreamInput in) throws IOException {
     }
 
     @Override
-    protected IndicesSegmentResponse newResponse(
+    protected ResponseFactory<IndicesSegmentResponse, ShardSegments> getResponseFactory(
         IndicesSegmentsRequest request,
-        int totalShards,
-        int successfulShards,
-        int failedShards,
-        List<ShardSegments> results,
-        List<DefaultShardOperationFailedException> shardFailures,
         ClusterState clusterState
     ) {
-        return new IndicesSegmentResponse(
-            results.toArray(new ShardSegments[results.size()]),
+        return (totalShards, successfulShards, failedShards, results, shardFailures) -> new IndicesSegmentResponse(
+            results.toArray(new ShardSegments[0]),
             totalShards,
             successfulShards,
             failedShards,
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsResponse.java
index ad4c49c29a0e..fa95a93183c5 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsResponse.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -28,7 +28,7 @@
 import java.util.Map;
 import java.util.Objects;
 
-public class GetSettingsResponse extends ActionResponse implements ChunkedToXContent {
+public class GetSettingsResponse extends ActionResponse implements ChunkedToXContentObject {
 
     private final Map<String, Settings> indexToSettings;
     private final Map<String, Settings> indexToDefaultSettings;
@@ -154,7 +154,7 @@ public String toString() {
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked() {
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
         final boolean omitEmptySettings = indexToDefaultSettings.isEmpty();
         return toXContentChunked(omitEmptySettings);
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsClusterStateUpdateRequest.java
index b8353d5bc970..f52c659ea55f 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsClusterStateUpdateRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsClusterStateUpdateRequest.java
@@ -11,6 +11,8 @@
 import org.elasticsearch.cluster.ack.IndicesClusterStateUpdateRequest;
 import org.elasticsearch.common.settings.Settings;
 
+import java.util.Arrays;
+
 /**
  * Cluster state update request that allows to update settings for some indices
  */
@@ -51,4 +53,9 @@ public UpdateSettingsClusterStateUpdateRequest settings(Settings settings) {
         this.settings = settings;
         return this;
     }
+
+    @Override
+    public String toString() {
+        return Arrays.toString(indices()) + settings;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java
index 6e443744b883..ad9c4904ecc8 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.action.admin.indices.settings.put;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -54,7 +54,7 @@ public UpdateSettingsRequest(StreamInput in) throws IOException {
         indicesOptions = IndicesOptions.readIndicesOptions(in);
         settings = readSettingsFromStream(in);
         preserveExisting = in.readBoolean();
-        if (in.getVersion().onOrAfter(Version.V_7_12_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_12_0)) {
             origin = in.readString();
         }
     }
@@ -183,7 +183,7 @@ public void writeTo(StreamOutput out) throws IOException {
         indicesOptions.writeIndicesOptions(out);
         settings.writeTo(out);
         out.writeBoolean(preserveExisting);
-        if (out.getVersion().onOrAfter(Version.V_7_12_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_12_0)) {
             out.writeString(origin);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
index 8169860fd803..9547e5bd8f78 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -238,10 +237,6 @@ public IndicesShardStoresResponse(Map<String, Map<Integer, List<StoreStatus>>> s
         this.failures = failures;
     }
 
-    IndicesShardStoresResponse() {
-        this(Map.of(), Collections.emptyList());
-    }
-
     public IndicesShardStoresResponse(StreamInput in) throws IOException {
         super(in);
         storeStatuses = in.readImmutableMap(
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
index 29044fbe0af1..37cbdb9b0dab 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse.StoreStatus;
 import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse.StoreStatus.AllocationStatus;
 import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.client.internal.node.NodeClient;
@@ -35,10 +36,9 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.util.concurrent.CountDown;
+import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.gateway.AsyncShardFetch;
-import org.elasticsearch.gateway.AsyncShardFetch.Lister;
 import org.elasticsearch.gateway.TransportNodesListGatewayStartedShards;
 import org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.NodeGatewayStartedShards;
 import org.elasticsearch.index.shard.ShardId;
@@ -140,7 +140,7 @@ private class AsyncShardStoresInfoFetches {
         private final RoutingNodes routingNodes;
         private final Set<Tuple<ShardId, String>> shards;
         private final ActionListener<IndicesShardStoresResponse> listener;
-        private CountDown expectedOps;
+        private final RefCountingRunnable refs = new RefCountingRunnable(this::finish);
         private final Queue<InternalAsyncFetch.Response> fetchResponses;
 
         AsyncShardStoresInfoFetches(
@@ -154,19 +154,15 @@ private class AsyncShardStoresInfoFetches {
             this.shards = shards;
             this.listener = listener;
             this.fetchResponses = new ConcurrentLinkedQueue<>();
-            this.expectedOps = new CountDown(shards.size());
         }
 
         void start() {
-            if (shards.isEmpty()) {
-                listener.onResponse(new IndicesShardStoresResponse());
-            } else {
-                // explicitely type lister, some IDEs (Eclipse) are not able to correctly infer the function type
-                Lister<BaseNodesResponse<NodeGatewayStartedShards>, NodeGatewayStartedShards> lister = this::listStartedShards;
+            try {
                 for (Tuple<ShardId, String> shard : shards) {
-                    InternalAsyncFetch fetch = new InternalAsyncFetch(logger, "shard_stores", shard.v1(), shard.v2(), lister);
-                    fetch.fetchData(nodes, Collections.<String>emptySet());
+                    new InternalAsyncFetch(logger, "shard_stores", shard.v1(), shard.v2()).fetchData(nodes, Collections.emptySet());
                 }
+            } finally {
+                refs.close();
             }
         }
 
@@ -186,14 +182,10 @@ private void listStartedShards(
 
         private class InternalAsyncFetch extends AsyncShardFetch<NodeGatewayStartedShards> {
 
-            InternalAsyncFetch(
-                Logger logger,
-                String type,
-                ShardId shardId,
-                String customDataPath,
-                Lister<? extends BaseNodesResponse<NodeGatewayStartedShards>, NodeGatewayStartedShards> action
-            ) {
-                super(logger, type, shardId, customDataPath, action);
+            private final Releasable ref = refs.acquire();
+
+            InternalAsyncFetch(Logger logger, String type, ShardId shardId, String customDataPath) {
+                super(logger, type, shardId, customDataPath);
             }
 
             @Override
@@ -203,63 +195,17 @@ protected synchronized void processAsyncFetch(
                 long fetchingRound
             ) {
                 fetchResponses.add(new Response(shardId, responses, failures));
-                if (expectedOps.countDown()) {
-                    finish();
-                }
+                ref.close();
             }
 
-            void finish() {
-                Map<String, Map<Integer, List<StoreStatus>>> indicesStatuses = new HashMap<>();
-                List<Failure> failures = new ArrayList<>();
-                for (Response fetchResponse : fetchResponses) {
-                    var indexName = fetchResponse.shardId.getIndexName();
-                    var shardId = fetchResponse.shardId.id();
-                    var indexStatuses = indicesStatuses.computeIfAbsent(indexName, k -> new HashMap<>());
-                    var storeStatuses = indexStatuses.computeIfAbsent(shardId, k -> new ArrayList<>());
-
-                    for (NodeGatewayStartedShards r : fetchResponse.responses) {
-                        if (shardExistsInNode(r)) {
-                            var allocationStatus = getAllocationStatus(indexName, shardId, r.getNode());
-                            storeStatuses.add(new StoreStatus(r.getNode(), r.allocationId(), allocationStatus, r.storeException()));
-                        }
-                    }
-
-                    for (FailedNodeException failure : fetchResponse.failures) {
-                        failures.add(new Failure(failure.nodeId(), indexName, shardId, failure.getCause()));
-                    }
-                }
-                // make the status structure immutable
-                indicesStatuses.replaceAll((k, v) -> {
-                    v.replaceAll((s, l) -> {
-                        CollectionUtil.timSort(l);
-                        return List.copyOf(l);
-                    });
-                    return Map.copyOf(v);
-                });
-                listener.onResponse(new IndicesShardStoresResponse(Map.copyOf(indicesStatuses), List.copyOf(failures)));
-            }
-
-            private AllocationStatus getAllocationStatus(String index, int shardID, DiscoveryNode node) {
-                for (ShardRouting shardRouting : routingNodes.node(node.getId())) {
-                    ShardId shardId = shardRouting.shardId();
-                    if (shardId.id() == shardID && shardId.getIndexName().equals(index)) {
-                        if (shardRouting.primary()) {
-                            return AllocationStatus.PRIMARY;
-                        } else if (shardRouting.assignedToNode()) {
-                            return AllocationStatus.REPLICA;
-                        } else {
-                            return AllocationStatus.UNUSED;
-                        }
-                    }
-                }
-                return AllocationStatus.UNUSED;
-            }
-
-            /**
-             * A shard exists/existed in a node only if shard state file exists in the node
-             */
-            private static boolean shardExistsInNode(final NodeGatewayStartedShards response) {
-                return response.storeException() != null || response.allocationId() != null;
+            @Override
+            protected void list(
+                ShardId shardId,
+                String customDataPath,
+                DiscoveryNode[] nodes,
+                ActionListener<BaseNodesResponse<NodeGatewayStartedShards>> listener
+            ) {
+                listStartedShards(shardId, customDataPath, nodes, listener);
             }
 
             @Override
@@ -279,5 +225,59 @@ public class Response {
                 }
             }
         }
+
+        void finish() {
+            Map<String, Map<Integer, List<StoreStatus>>> indicesStatuses = new HashMap<>();
+            List<Failure> failures = new ArrayList<>();
+            for (InternalAsyncFetch.Response fetchResponse : fetchResponses) {
+                var indexName = fetchResponse.shardId.getIndexName();
+                var shardId = fetchResponse.shardId.id();
+                var indexStatuses = indicesStatuses.computeIfAbsent(indexName, k -> new HashMap<>());
+                var storeStatuses = indexStatuses.computeIfAbsent(shardId, k -> new ArrayList<>());
+
+                for (NodeGatewayStartedShards r : fetchResponse.responses) {
+                    if (shardExistsInNode(r)) {
+                        var allocationStatus = getAllocationStatus(indexName, shardId, r.getNode());
+                        storeStatuses.add(new StoreStatus(r.getNode(), r.allocationId(), allocationStatus, r.storeException()));
+                    }
+                }
+
+                for (FailedNodeException failure : fetchResponse.failures) {
+                    failures.add(new Failure(failure.nodeId(), indexName, shardId, failure.getCause()));
+                }
+            }
+            // make the status structure immutable
+            indicesStatuses.replaceAll((k, v) -> {
+                v.replaceAll((s, l) -> {
+                    CollectionUtil.timSort(l);
+                    return List.copyOf(l);
+                });
+                return Map.copyOf(v);
+            });
+            listener.onResponse(new IndicesShardStoresResponse(Map.copyOf(indicesStatuses), List.copyOf(failures)));
+        }
+
+        private AllocationStatus getAllocationStatus(String index, int shardID, DiscoveryNode node) {
+            for (ShardRouting shardRouting : routingNodes.node(node.getId())) {
+                ShardId shardId = shardRouting.shardId();
+                if (shardId.id() == shardID && shardId.getIndexName().equals(index)) {
+                    if (shardRouting.primary()) {
+                        return AllocationStatus.PRIMARY;
+                    } else if (shardRouting.assignedToNode()) {
+                        return AllocationStatus.REPLICA;
+                    } else {
+                        return AllocationStatus.UNUSED;
+                    }
+                }
+            }
+            return AllocationStatus.UNUSED;
+        }
+
+        /**
+         * A shard exists/existed in a node only if shard state file exists in the node
+         */
+        private static boolean shardExistsInNode(final NodeGatewayStartedShards response) {
+            return response.storeException() != null || response.allocationId() != null;
+        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
index 64af5d7dc3b2..abdc26981845 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.action.admin.indices.stats;
 
 import org.apache.lucene.store.AlreadyClosedException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -45,7 +45,7 @@
 
 public class CommonStats implements Writeable, ToXContentFragment {
 
-    private static final Version VERSION_SUPPORTING_NODE_MAPPINGS = Version.V_8_5_0;
+    private static final TransportVersion VERSION_SUPPORTING_NODE_MAPPINGS = TransportVersion.V_8_5_0;
 
     @Nullable
     public DocsStats docs;
@@ -201,11 +201,11 @@ public CommonStats(StreamInput in) throws IOException {
         translog = in.readOptionalWriteable(TranslogStats::new);
         requestCache = in.readOptionalWriteable(RequestCacheStats::new);
         recoveryStats = in.readOptionalWriteable(RecoveryStats::new);
-        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             bulk = in.readOptionalWriteable(BulkStats::new);
         }
         shards = in.readOptionalWriteable(ShardCountStats::new);
-        if (in.getVersion().onOrAfter(VERSION_SUPPORTING_NODE_MAPPINGS)) {
+        if (in.getTransportVersion().onOrAfter(VERSION_SUPPORTING_NODE_MAPPINGS)) {
             nodeMappings = in.readOptionalWriteable(NodeMappingStats::new);
         }
     }
@@ -228,11 +228,11 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(translog);
         out.writeOptionalWriteable(requestCache);
         out.writeOptionalWriteable(recoveryStats);
-        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             out.writeOptionalWriteable(bulk);
         }
         out.writeOptionalWriteable(shards);
-        if (out.getVersion().onOrAfter(VERSION_SUPPORTING_NODE_MAPPINGS)) {
+        if (out.getTransportVersion().onOrAfter(VERSION_SUPPORTING_NODE_MAPPINGS)) {
             out.writeOptionalWriteable(nodeMappings);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java
index d56c344a05c4..24fea14e5321 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.indices.stats;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -55,7 +55,7 @@ public CommonStatsFlags(StreamInput in) throws IOException {
                 flags.add(flag);
             }
         }
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             in.readStringArray();
         }
         groups = in.readStringArray();
@@ -73,7 +73,7 @@ public void writeTo(StreamOutput out) throws IOException {
         }
         out.writeLong(longFlags);
 
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeStringArrayNullable(Strings.EMPTY_ARRAY);
         }
         out.writeStringArrayNullable(groups);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java
index 7132449cee10..9b2efa83746e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java
@@ -9,16 +9,17 @@
 package org.elasticsearch.action.admin.indices.stats;
 
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
-import org.elasticsearch.action.support.broadcast.BroadcastResponse;
+import org.elasticsearch.action.support.broadcast.ChunkedBroadcastResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-public class FieldUsageStatsResponse extends BroadcastResponse {
+public class FieldUsageStatsResponse extends ChunkedBroadcastResponse {
     private final Map<String, List<FieldUsageShardResponse>> stats;
 
     FieldUsageStatsResponse(
@@ -48,19 +49,15 @@ public Map<String, List<FieldUsageShardResponse>> getStats() {
     }
 
     @Override
-    protected void addCustomXContentFields(XContentBuilder builder, Params params) throws IOException {
-        final List<Map.Entry<String, List<FieldUsageShardResponse>>> sortedEntries = stats.entrySet()
-            .stream()
-            .sorted(Map.Entry.comparingByKey())
-            .toList();
-        for (Map.Entry<String, List<FieldUsageShardResponse>> entry : sortedEntries) {
+    protected Iterator<ToXContent> customXContentChunks(ToXContent.Params params) {
+        return stats.entrySet().stream().sorted(Map.Entry.comparingByKey()).map(entry -> (ToXContent) (builder, p) -> {
             builder.startObject(entry.getKey());
             builder.startArray("shards");
             for (FieldUsageShardResponse resp : entry.getValue()) {
                 resp.toXContent(builder, params);
             }
             builder.endArray();
-            builder.endObject();
-        }
+            return builder.endObject();
+        }).iterator();
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java
index 0cc3c45b60d7..ca2f6f50b5d3 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java
@@ -149,4 +149,9 @@ public IndicesStatsRequestBuilder setIncludeSegmentFileSizes(boolean includeSegm
         request.includeSegmentFileSizes(includeSegmentFileSizes);
         return this;
     }
+
+    public IndicesStatsRequestBuilder setIncludeUnloadedSegments(boolean includeUnloadedSegments) {
+        request.includeUnloadedSegments(includeUnloadedSegments);
+        return this;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java
index 25c804a340a7..7c7add9262fd 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java
@@ -8,23 +8,27 @@
 
 package org.elasticsearch.action.admin.indices.stats;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.admin.indices.stats.IndexStats.IndexStatsBuilder;
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
-import org.elasticsearch.action.support.broadcast.BroadcastResponse;
-import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.action.support.broadcast.ChunkedBroadcastResponse;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.health.ClusterIndexHealth;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -33,7 +37,7 @@
 
 import static java.util.Collections.unmodifiableMap;
 
-public class IndicesStatsResponse extends BroadcastResponse {
+public class IndicesStatsResponse extends ChunkedBroadcastResponse {
 
     private final Map<String, ClusterHealthStatus> indexHealthMap;
 
@@ -46,7 +50,7 @@ public class IndicesStatsResponse extends BroadcastResponse {
     IndicesStatsResponse(StreamInput in) throws IOException {
         super(in);
         shards = in.readArray(ShardStats::new, ShardStats[]::new);
-        if (in.getVersion().onOrAfter(Version.V_8_1_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
             indexHealthMap = in.readMap(StreamInput::readString, ClusterHealthStatus::readFrom);
             indexStateMap = in.readMap(StreamInput::readString, IndexMetadata.State::readFrom);
         } else {
@@ -61,21 +65,23 @@ public class IndicesStatsResponse extends BroadcastResponse {
         int successfulShards,
         int failedShards,
         List<DefaultShardOperationFailedException> shardFailures,
-        ClusterState clusterState
+        Metadata metadata,
+        RoutingTable routingTable
     ) {
         super(totalShards, successfulShards, failedShards, shardFailures);
         this.shards = shards;
-        Objects.requireNonNull(clusterState);
+        Objects.requireNonNull(metadata);
+        Objects.requireNonNull(routingTable);
         Objects.requireNonNull(shards);
         Map<String, ClusterHealthStatus> indexHealthModifiableMap = new HashMap<>();
         Map<String, IndexMetadata.State> indexStateModifiableMap = new HashMap<>();
         for (ShardStats shard : shards) {
             Index index = shard.getShardRouting().index();
-            IndexMetadata indexMetadata = clusterState.getMetadata().index(index);
+            IndexMetadata indexMetadata = metadata.index(index);
             if (indexMetadata != null) {
                 indexHealthModifiableMap.computeIfAbsent(
                     index.getName(),
-                    ignored -> new ClusterIndexHealth(indexMetadata, clusterState.routingTable().index(index)).getStatus()
+                    ignored -> new ClusterIndexHealth(indexMetadata, routingTable.index(index)).getStatus()
                 );
                 indexStateModifiableMap.computeIfAbsent(index.getName(), ignored -> indexMetadata.getState());
             }
@@ -164,14 +170,14 @@ public CommonStats getPrimaries() {
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeArray(shards);
-        if (out.getVersion().onOrAfter(Version.V_8_1_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
             out.writeMap(indexHealthMap, StreamOutput::writeString, (o, s) -> s.writeTo(o));
             out.writeMap(indexStateMap, StreamOutput::writeString, (o, s) -> s.writeTo(o));
         }
     }
 
     @Override
-    protected void addCustomXContentFields(XContentBuilder builder, Params params) throws IOException {
+    protected Iterator<ToXContent> customXContentChunks(ToXContent.Params params) {
         final String level = params.param("level", "indices");
         final boolean isLevelValid = "cluster".equalsIgnoreCase(level)
             || "indices".equalsIgnoreCase(level)
@@ -179,22 +185,11 @@ protected void addCustomXContentFields(XContentBuilder builder, Params params) t
         if (isLevelValid == false) {
             throw new IllegalArgumentException("level parameter must be one of [cluster] or [indices] or [shards] but was [" + level + "]");
         }
-
-        builder.startObject("_all");
-
-        builder.startObject("primaries");
-        getPrimaries().toXContent(builder, params);
-        builder.endObject();
-
-        builder.startObject("total");
-        getTotal().toXContent(builder, params);
-        builder.endObject();
-
-        builder.endObject();
-
         if ("indices".equalsIgnoreCase(level) || "shards".equalsIgnoreCase(level)) {
-            builder.startObject(Fields.INDICES);
-            for (IndexStats indexStats : getIndices().values()) {
+            return Iterators.concat(Iterators.single(((builder, p) -> {
+                commonStats(builder, p);
+                return builder.startObject(Fields.INDICES);
+            })), getIndices().values().stream().<ToXContent>map(indexStats -> (builder, p) -> {
                 builder.startObject(indexStats.getIndex());
                 builder.field("uuid", indexStats.getUuid());
                 if (indexStats.getHealth() != null) {
@@ -204,11 +199,11 @@ protected void addCustomXContentFields(XContentBuilder builder, Params params) t
                     builder.field("status", indexStats.getState().toString().toLowerCase(Locale.ROOT));
                 }
                 builder.startObject("primaries");
-                indexStats.getPrimaries().toXContent(builder, params);
+                indexStats.getPrimaries().toXContent(builder, p);
                 builder.endObject();
 
                 builder.startObject("total");
-                indexStats.getTotal().toXContent(builder, params);
+                indexStats.getTotal().toXContent(builder, p);
                 builder.endObject();
 
                 if ("shards".equalsIgnoreCase(level)) {
@@ -217,17 +212,34 @@ protected void addCustomXContentFields(XContentBuilder builder, Params params) t
                         builder.startArray(Integer.toString(indexShardStats.getShardId().id()));
                         for (ShardStats shardStats : indexShardStats) {
                             builder.startObject();
-                            shardStats.toXContent(builder, params);
+                            shardStats.toXContent(builder, p);
                             builder.endObject();
                         }
                         builder.endArray();
                     }
                     builder.endObject();
                 }
-                builder.endObject();
-            }
-            builder.endObject();
+                return builder.endObject();
+            }).iterator(), Iterators.single((b, p) -> b.endObject()));
         }
+        return Iterators.single((b, p) -> {
+            commonStats(b, p);
+            return b;
+        });
+    }
+
+    private void commonStats(XContentBuilder builder, ToXContent.Params p) throws IOException {
+        builder.startObject("_all");
+
+        builder.startObject("primaries");
+        getPrimaries().toXContent(builder, p);
+        builder.endObject();
+
+        builder.startObject("total");
+        getTotal().toXContent(builder, p);
+        builder.endObject();
+
+        builder.endObject();
     }
 
     static final class Fields {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java
index 457f7219a569..d5a7d17f2a4b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.indices.stats;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -26,7 +26,7 @@
 
 public class ShardStats implements Writeable, ToXContentFragment {
 
-    private static final Version DEDUPLICATE_SHARD_PATH_VERSION = Version.V_8_4_0;
+    private static final TransportVersion DEDUPLICATE_SHARD_PATH_VERSION = TransportVersion.V_8_4_0;
 
     private final ShardRouting shardRouting;
     private final CommonStats commonStats;
@@ -46,7 +46,7 @@ public ShardStats(StreamInput in) throws IOException {
         commonStats = new CommonStats(in);
         commitStats = CommitStats.readOptionalCommitStatsFrom(in);
         statePath = in.readString();
-        if (in.getVersion().onOrAfter(DEDUPLICATE_SHARD_PATH_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(DEDUPLICATE_SHARD_PATH_VERSION)) {
             dataPath = Objects.requireNonNullElse(in.readOptionalString(), this.statePath);
         } else {
             dataPath = in.readString();
@@ -164,7 +164,7 @@ public void writeTo(StreamOutput out) throws IOException {
         commonStats.writeTo(out);
         out.writeOptionalWriteable(commitStats);
         out.writeString(statePath);
-        if (out.getVersion().onOrAfter(DEDUPLICATE_SHARD_PATH_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(DEDUPLICATE_SHARD_PATH_VERSION)) {
             out.writeOptionalString(statePath.equals(dataPath) ? null : dataPath);
         } else {
             out.writeString(dataPath);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportFieldUsageAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportFieldUsageAction.java
index b458b24c7b5e..9b3214787105 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportFieldUsageAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportFieldUsageAction.java
@@ -10,7 +10,6 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
@@ -67,20 +66,17 @@ protected FieldUsageShardResponse readShardResult(StreamInput in) throws IOExcep
     }
 
     @Override
-    protected FieldUsageStatsResponse newResponse(
+    protected ResponseFactory<FieldUsageStatsResponse, FieldUsageShardResponse> getResponseFactory(
         FieldUsageStatsRequest request,
-        int totalShards,
-        int successfulShards,
-        int failedShards,
-        List<FieldUsageShardResponse> fieldUsages,
-        List<DefaultShardOperationFailedException> shardFailures,
         ClusterState clusterState
     ) {
-        final Map<String, List<FieldUsageShardResponse>> combined = new HashMap<>();
-        for (FieldUsageShardResponse response : fieldUsages) {
-            combined.computeIfAbsent(response.shardRouting.shardId().getIndexName(), i -> new ArrayList<>()).add(response);
-        }
-        return new FieldUsageStatsResponse(totalShards, successfulShards, shardFailures.size(), shardFailures, combined);
+        return (totalShards, successfulShards, failedShards, fieldUsages, shardFailures) -> {
+            final Map<String, List<FieldUsageShardResponse>> combined = new HashMap<>();
+            for (FieldUsageShardResponse response : fieldUsages) {
+                combined.computeIfAbsent(response.shardRouting.shardId().getIndexName(), i -> new ArrayList<>()).add(response);
+            }
+            return new FieldUsageStatsResponse(totalShards, successfulShards, shardFailures.size(), shardFailures, combined);
+        };
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java
index 75e0d3c878ab..3d799e7efc42 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java
@@ -11,7 +11,6 @@
 import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
@@ -35,7 +34,6 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
-import java.util.List;
 
 public class TransportIndicesStatsAction extends TransportBroadcastByNodeAction<IndicesStatsRequest, IndicesStatsResponse, ShardStats> {
 
@@ -85,22 +83,19 @@ protected ShardStats readShardResult(StreamInput in) throws IOException {
     }
 
     @Override
-    protected IndicesStatsResponse newResponse(
-        IndicesStatsRequest request,
-        int totalShards,
-        int successfulShards,
-        int failedShards,
-        List<ShardStats> responses,
-        List<DefaultShardOperationFailedException> shardFailures,
-        ClusterState clusterState
-    ) {
-        return new IndicesStatsResponse(
-            responses.toArray(new ShardStats[responses.size()]),
+    protected ResponseFactory<IndicesStatsResponse, ShardStats> getResponseFactory(IndicesStatsRequest request, ClusterState clusterState) {
+        // NB avoid capture of full cluster state
+        final var metadata = clusterState.getMetadata();
+        final var routingTable = clusterState.routingTable();
+
+        return (totalShards, successfulShards, failedShards, responses, shardFailures) -> new IndicesStatsResponse(
+            responses.toArray(new ShardStats[0]),
             totalShards,
             successfulShards,
             failedShards,
             shardFailures,
-            clusterState
+            metadata,
+            routingTable
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java
index a332b57cd475..b2fb169b1f62 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java
@@ -9,7 +9,7 @@
 
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.admin.indices.alias.Alias;
@@ -76,7 +76,7 @@ public PutIndexTemplateRequest(StreamInput in) throws IOException {
         order = in.readInt();
         create = in.readBoolean();
         settings = readSettingsFromStream(in);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             int size = in.readVInt();
             for (int i = 0; i < size; i++) {
                 in.readString();    // type - cannot assert on _doc because 7x allows arbitrary type names
@@ -446,7 +446,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeInt(order);
         out.writeBoolean(create);
         settings.writeTo(out);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeVInt(mappings == null ? 0 : 1);
             if (mappings != null) {
                 out.writeString(MapperService.SINGLE_MAPPING_NAME);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java
index 0721f499d600..e5a399aed85f 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.indices.validate.query;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.support.broadcast.BroadcastShardRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -33,7 +33,7 @@ public class ShardValidateQueryRequest extends BroadcastShardRequest {
     public ShardValidateQueryRequest(StreamInput in) throws IOException {
         super(in);
         query = in.readNamedWriteable(QueryBuilder.class);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             int typesSize = in.readVInt();
             if (typesSize > 0) {
                 for (int i = 0; i < typesSize; i++) {
@@ -80,7 +80,7 @@ public long nowInMillis() {
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeNamedWriteable(query);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeVInt(0);   // no types to filter
         }
         filteringAliases.writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java
index 57c66605e315..6747b76f2aec 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.admin.indices.validate.query;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -48,7 +48,7 @@ public ValidateQueryRequest() {
     public ValidateQueryRequest(StreamInput in) throws IOException {
         super(in);
         query = in.readNamedWriteable(QueryBuilder.class);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             int typesSize = in.readVInt();
             if (typesSize > 0) {
                 for (int i = 0; i < typesSize; i++) {
@@ -137,7 +137,7 @@ public boolean allShards() {
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeNamedWriteable(query);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeVInt(0);   // no types to filter
         }
         out.writeBoolean(explain);
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
index 79c9dd2e6182..b5894d322b90 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
@@ -10,7 +10,7 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.DocWriteRequest.OpType;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.delete.DeleteResponse;
@@ -237,7 +237,7 @@ private Failure(String index, String id, Exception cause, RestStatus status, lon
          */
         public Failure(StreamInput in) throws IOException {
             index = in.readString();
-            if (in.getVersion().before(Version.V_8_0_0)) {
+            if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 in.readString();
                 // can't make an assertion about type names here because too many tests still set their own
                 // types bypassing various checks
@@ -253,7 +253,7 @@ public Failure(StreamInput in) throws IOException {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(index);
-            if (out.getVersion().before(Version.V_8_0_0)) {
+            if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 out.writeString(MapperService.SINGLE_MAPPING_NAME);
             }
             out.writeOptionalString(id);
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkProcessor2.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkProcessor2.java
new file mode 100644
index 000000000000..37b92d9287aa
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkProcessor2.java
@@ -0,0 +1,415 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.bulk;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.threadpool.Scheduler;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiConsumer;
+
+/**
+ * A bulk processor is a thread safe bulk processing class, allowing to easily set when to "flush" a new bulk request
+ * (either based on number of actions, based on the size, or time), and to easily control the number of concurrent bulk
+ * requests allowed to be executed in parallel.
+ * <p>
+ * In order to create a new bulk processor, use the {@link Builder}.
+ */
+public class BulkProcessor2 {
+
+    /**
+     * A listener for the execution.
+     */
+    public interface Listener {
+
+        /**
+         * Callback before the bulk is executed.
+         */
+        void beforeBulk(long executionId, BulkRequest request);
+
+        /**
+         * Callback after a successful execution of bulk request.
+         */
+        void afterBulk(long executionId, BulkRequest request, BulkResponse response);
+
+        /**
+         * Callback after a failed execution of bulk request.
+         * <p>
+         * Note that in case an instance of <code>InterruptedException</code> is passed, which means that request processing has been
+         * cancelled externally, the thread's interruption status has been restored prior to calling this method.
+         */
+        void afterBulk(long executionId, BulkRequest request, Exception failure);
+    }
+
+    /**
+     * A builder used to create a build an instance of a bulk processor.
+     */
+    public static class Builder {
+
+        private final BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer;
+        private final Listener listener;
+        private final ThreadPool threadPool;
+        private int maxRequestsInBulk = 1000;
+        private ByteSizeValue maxBulkSizeInBytes = new ByteSizeValue(5, ByteSizeUnit.MB);
+        private ByteSizeValue maxBytesInFlight = new ByteSizeValue(50, ByteSizeUnit.MB);
+        private TimeValue flushInterval = null;
+        private int maxNumberOfRetries = 3;
+
+        private Builder(BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer, Listener listener, ThreadPool threadPool) {
+            this.consumer = consumer;
+            this.listener = listener;
+            this.threadPool = threadPool;
+        }
+
+        /**
+         * Sets when to flush a new bulk request based on the number of actions currently added. Defaults to
+         * {@code 1000}. Can be set to {@code -1} to disable it.
+         */
+        public Builder setBulkActions(int bulkActions) {
+            this.maxRequestsInBulk = bulkActions;
+            return this;
+        }
+
+        /**
+         * Sets when to flush a new bulk request based on the size of actions currently added. Defaults to
+         * {@code 5mb}. Can be set to {@code -1} to disable it.
+         */
+        public Builder setBulkSize(ByteSizeValue maxBulkSizeInBytes) {
+            this.maxBulkSizeInBytes = maxBulkSizeInBytes;
+            return this;
+        }
+
+        /**
+         * Sets a flush interval flushing *any* bulk actions pending if the interval passes. Defaults to not set.
+         * <p>
+         * Note, both {@link #setBulkActions(int)} and {@link #setBulkSize(org.elasticsearch.common.unit.ByteSizeValue)}
+         * can be set to {@code -1} with the flush interval set allowing for complete async processing of bulk actions.
+         */
+        public Builder setFlushInterval(TimeValue flushInterval) {
+            this.flushInterval = flushInterval;
+            return this;
+        }
+
+        /**
+         * Sets the maximum number of times a BulkRequest will be retried if it fails.
+         */
+        public Builder setMaxNumberOfRetries(int maxNumberOfRetries) {
+            assert maxNumberOfRetries >= 0;
+            this.maxNumberOfRetries = maxNumberOfRetries;
+            return this;
+        }
+
+        /**
+         * Sets the maximum number of bytes allowed in in-flight requests (both the BulkRequest being built up by the BulkProcessor and
+         * any BulkRequests sent to Retry2 that have not yet completed) before subsequent calls to add()result in
+         * EsRejectedExecutionException. Defaults to 50mb.
+         */
+        public Builder setMaxBytesInFlight(ByteSizeValue maxBytesInFlight) {
+            this.maxBytesInFlight = maxBytesInFlight;
+            return this;
+        }
+
+        /**
+         * Builds a new bulk processor.
+         */
+        public BulkProcessor2 build() {
+            return new BulkProcessor2(
+                consumer,
+                maxNumberOfRetries,
+                listener,
+                maxRequestsInBulk,
+                maxBulkSizeInBytes,
+                maxBytesInFlight,
+                flushInterval,
+                threadPool
+            );
+        }
+    }
+
+    /**
+     * @param consumer The consumer that is called to fulfil bulk operations. This consumer _must_ operate either very fast or
+     *                 asynchronously.
+     * @param listener The BulkProcessor2 listener that gets called on bulk events
+     * @param threadPool The threadpool used to schedule the flush task for this bulk processor, if flushInterval is not null.
+     * @return the builder for BulkProcessor2
+     */
+    public static Builder builder(
+        BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer,
+        Listener listener,
+        ThreadPool threadPool
+    ) {
+        Objects.requireNonNull(consumer, "consumer");
+        Objects.requireNonNull(listener, "listener");
+        return new Builder(consumer, listener, threadPool);
+    }
+
+    private final int maxActionsPerBulkRequest;
+    private final long maxBulkSizeBytes;
+    private final ByteSizeValue maxBytesInFlight;
+    /*
+     * This is the approximate total number of bytes in in-flight requests, both in the BulkRequest that it is building up and in all of
+     * the BulkRequests that it has sent to Retry2 that have not completed yet. If this number would exceeds maxBytesInFlight, then calls
+     * to add() will throw EsRejectedExecutionExceptions.
+     */
+    private final AtomicLong totalBytesInFlight = new AtomicLong(0);
+
+    /**
+     * This is a task (which might be null) that is scheduled at some pont in the future to flush the bulk request and start a new bulk
+     * request. This variable is read and written to from multiple threads, and is protected by mutex.
+     */
+    private volatile Scheduler.Cancellable cancellableFlushTask = null;
+
+    private final AtomicLong executionIdGen = new AtomicLong();
+
+    private static final Logger logger = LogManager.getLogger(BulkProcessor2.class);
+
+    private final BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer;
+    private final Listener listener;
+
+    private final Retry2 retry;
+
+    private final TimeValue flushInterval;
+
+    private final ThreadPool threadPool;
+
+    /*
+     * This is the BulkRequest that is being built up by this class in calls to the various add methods.
+     */
+    private BulkRequest bulkRequestUnderConstruction;
+
+    private volatile boolean closed = false;
+    /*
+     * This mutex is used to protect two things related to the bulkRequest object: (1) it makes sure that two threads do not add requests
+     * to the BulkRequest at the same time since BulkRequest is not threadsafe and (2) it makes sure that no other thread is writing to
+     * the BulkRequest when we swap the bulkRequest variable over to a new BulkRequest object. It also protects access to
+     * cancellableFlushTask.
+     */
+    private final Object mutex = new Object();
+
+    BulkProcessor2(
+        BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer,
+        int maxNumberOfRetries,
+        Listener listener,
+        int maxActionsPerBulkRequest,
+        ByteSizeValue maxBulkSize,
+        ByteSizeValue maxBytesInFlight,
+        @Nullable TimeValue flushInterval,
+        ThreadPool threadPool
+    ) {
+        this.maxActionsPerBulkRequest = maxActionsPerBulkRequest;
+        this.maxBulkSizeBytes = maxBulkSize.getBytes();
+        this.maxBytesInFlight = maxBytesInFlight;
+        this.bulkRequestUnderConstruction = new BulkRequest();
+        this.consumer = consumer;
+        this.listener = listener;
+        this.retry = new Retry2(maxNumberOfRetries);
+        this.flushInterval = flushInterval;
+        this.threadPool = threadPool;
+    }
+
+    /**
+     * Closes the processor. Any remaining bulk actions are flushed if they can be flushed in the given time.
+     * <p>
+     * Waits for up to the specified timeout for all bulk requests to complete then returns
+     *
+     * @param timeout The maximum time to wait for the bulk requests to complete
+     * @param unit    The time unit of the {@code timeout} argument
+     * @throws InterruptedException If the current thread is interrupted
+     */
+    public void awaitClose(long timeout, TimeUnit unit) throws InterruptedException {
+        synchronized (mutex) {
+            if (closed) {
+                return;
+            }
+            closed = true;
+
+            if (cancellableFlushTask != null) {
+                cancellableFlushTask.cancel();
+            }
+
+            if (bulkRequestUnderConstruction.numberOfActions() > 0) {
+                execute();
+            }
+            this.retry.awaitClose(timeout, unit);
+        }
+    }
+
+    /**
+     * Adds an {@link IndexRequest} to the list of actions to execute. Follows the same behavior of {@link IndexRequest}
+     * (for example, if no id is provided, one will be generated, or usage of the create flag).
+     * @throws EsRejectedExecutionException if adding the approximate size in bytes of the request to totalBytesInFlight would exceed
+     * maxBytesInFlight
+     */
+    public BulkProcessor2 add(IndexRequest request) throws EsRejectedExecutionException {
+        return add((DocWriteRequest<?>) request);
+    }
+
+    /**
+     * Adds an {@link DeleteRequest} to the list of actions to execute.
+     * @throws EsRejectedExecutionException if adding the approximate size in bytes of the request to totalBytesInFlight would exceed
+     * maxBytesInFlight
+     */
+    public BulkProcessor2 add(DeleteRequest request) throws EsRejectedExecutionException {
+        return add((DocWriteRequest<?>) request);
+    }
+
+    /**
+     * Adds either a delete or an index request.
+     * @throws EsRejectedExecutionException if the total bytes already in flight exceeds maxBytesInFlight. In this case, the request will
+     * not be retried and it is on the client to decide whether to wait and try later.
+     */
+    private BulkProcessor2 add(DocWriteRequest<?> request) throws EsRejectedExecutionException {
+        internalAdd(request);
+        return this;
+    }
+
+    /*
+     * Exposed for unit testing
+     */
+    long getTotalBytesInFlight() {
+        return totalBytesInFlight.get();
+    }
+
+    protected void ensureOpen() {
+        if (closed) {
+            throw new IllegalStateException("bulk process already closed");
+        }
+    }
+
+    private void internalAdd(DocWriteRequest<?> request) throws EsRejectedExecutionException {
+        // bulkRequest and instance swapping is not threadsafe, so execute the mutations under a mutex.
+        // once the bulk request is ready to be shipped swap the instance reference unlock and send the local reference to the handler.
+        Tuple<BulkRequest, Long> bulkRequestToExecute;
+        synchronized (mutex) {
+            ensureOpen();
+            if (totalBytesInFlight.get() >= maxBytesInFlight.getBytes()) {
+                throw new EsRejectedExecutionException(
+                    "Cannot index request of size "
+                        + bulkRequestUnderConstruction.estimatedSizeInBytes()
+                        + " because "
+                        + totalBytesInFlight.get()
+                        + " bytes are already in flight and the max is "
+                        + maxBytesInFlight
+                );
+            }
+            long bytesBeforeNewRequest = bulkRequestUnderConstruction.estimatedSizeInBytes();
+            bulkRequestUnderConstruction.add(request);
+            totalBytesInFlight.addAndGet(bulkRequestUnderConstruction.estimatedSizeInBytes() - bytesBeforeNewRequest);
+            bulkRequestToExecute = newBulkRequestIfNeeded();
+        }
+        // execute sending the local reference outside the lock to allow handler to control the concurrency via it's configuration.
+        if (bulkRequestToExecute != null) {
+            execute(bulkRequestToExecute.v1(), bulkRequestToExecute.v2());
+        }
+        /*
+         * We could have the flush task running nonstop, checking every flushInterval whether there was data to flush. But there is
+         * likely to not be data almost all of the time, so this would waste a thread's time. So instead we schedule a flush task
+         * whenever we add data. If a task is already scheduled, it does nothing. Since both the cancellableFlushTask and the
+         * bulkRequestUnderConstruction are protected by the same mutex, there is no risk that a request will be left hanging.
+         */
+        scheduleFlushTask();
+    }
+
+    /**
+     * This method schedules a flush task to run flushInterval in the future if flushInterval is not null and if there is not already a
+     * flush task scheduled.
+     */
+    private void scheduleFlushTask() {
+        if (flushInterval == null) {
+            return;
+        }
+        /*
+         * This method is called from multiple threads. We synchronize on mutex here so that we are sure that cancellableFlushTask is not
+         * changed between when we check it and when we set it (whether that is a transition from null -> not null from another thread
+         * in this method or a change from not null -> null from the scheduled task).
+         */
+        synchronized (mutex) {
+            if (cancellableFlushTask == null) {
+                cancellableFlushTask = threadPool.schedule(() -> {
+                    synchronized (mutex) {
+                        if (closed == false && bulkRequestUnderConstruction.numberOfActions() > 0) {
+                            execute();
+                        }
+                        cancellableFlushTask = null;
+                    }
+                }, flushInterval, ThreadPool.Names.GENERIC);
+            }
+        }
+    }
+
+    private Tuple<BulkRequest, Long> newBulkRequestIfNeeded() {
+        assert Thread.holdsLock(mutex);
+        ensureOpen();
+        if (bulkRequestExceedsLimits() || totalBytesInFlight.get() >= maxBytesInFlight.getBytes()) {
+            final BulkRequest bulkRequest = this.bulkRequestUnderConstruction;
+            this.bulkRequestUnderConstruction = new BulkRequest();
+            return new Tuple<>(bulkRequest, executionIdGen.incrementAndGet());
+        }
+        return null;
+    }
+
+    /**
+     * This method sends the bulkRequest to the consumer up to maxNumberOfRetries times. The executionId is used to notify the listener
+     * both before and after the request.
+     * @param bulkRequest
+     * @param executionId
+     */
+    private void execute(BulkRequest bulkRequest, long executionId) {
+        try {
+            listener.beforeBulk(executionId, bulkRequest);
+            retry.consumeRequestWithRetries(consumer, bulkRequest, new ActionListener<>() {
+                @Override
+                public void onResponse(BulkResponse response) {
+                    totalBytesInFlight.addAndGet(-1 * bulkRequest.estimatedSizeInBytes());
+                    listener.afterBulk(executionId, bulkRequest, response);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    totalBytesInFlight.addAndGet(-1 * bulkRequest.estimatedSizeInBytes());
+                    listener.afterBulk(executionId, bulkRequest, e);
+                }
+            });
+        } catch (Exception e) {
+            logger.warn(() -> "Failed to execute bulk request " + executionId + ".", e);
+            totalBytesInFlight.addAndGet(-1 * bulkRequest.estimatedSizeInBytes());
+            listener.afterBulk(executionId, bulkRequest, e);
+        }
+    }
+
+    private void execute() {
+        assert Thread.holdsLock(mutex);
+        final BulkRequest bulkRequest = this.bulkRequestUnderConstruction;
+        final long executionId = executionIdGen.incrementAndGet();
+        this.bulkRequestUnderConstruction = new BulkRequest();
+        execute(bulkRequest, executionId);
+    }
+
+    private boolean bulkRequestExceedsLimits() {
+        assert Thread.holdsLock(mutex);
+        if (maxActionsPerBulkRequest != -1 && bulkRequestUnderConstruction.numberOfActions() >= maxActionsPerBulkRequest) {
+            return true;
+        }
+        return maxBulkSizeBytes != -1 && bulkRequestUnderConstruction.estimatedSizeInBytes() >= maxBulkSizeBytes;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java
index 4f5d54ac58e5..d00585c98933 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java
@@ -357,6 +357,7 @@ public void parse(
                                     .setIfSeqNo(ifSeqNo)
                                     .setIfPrimaryTerm(ifPrimaryTerm)
                                     .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType)
+                                    .setDynamicTemplates(dynamicTemplates)
                                     .setRequireAlias(requireAlias),
                                 type
                             );
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/Retry2.java b/server/src/main/java/org/elasticsearch/action/bulk/Retry2.java
new file mode 100644
index 000000000000..6c60d07f6e88
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/bulk/Retry2.java
@@ -0,0 +1,274 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.action.bulk;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.rest.RestStatus;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Phaser;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.BiConsumer;
+import java.util.function.Predicate;
+import java.util.stream.StreamSupport;
+
+/**
+ * Encapsulates asynchronous retry logic. This class will attempt to load a BulkRequest up to numberOfRetries times. If that number of
+ * times is exhausted, it sends the listener an EsRejectedExecutionException.
+ */
+class Retry2 {
+    private static final Logger logger = LogManager.getLogger(Retry2.class);
+    private final int maxNumberOfRetries;
+    /**
+     * Once awaitClose() has been called this is set to true. Any new requests that come in (whether via consumeRequestWithRetries() or a
+     * retry) will be rejected by sending EsRejectedExecutionExceptions to their listeners.
+     */
+    private boolean isClosing = false;
+    /*
+     * We register in-flight calls with this Phaser so that we know whether there are any still in flight when we call awaitClose(). The
+     * phaser is initialized with 1 party intentionally. This is because if the number of parties goes over 0 and then back down to 0 the
+     * phaser is automatically terminated. Since we're tracking the number of in flight calls to Elasticsearch we expect this to happen
+     * often. Putting an initial party in here makes sure that the phaser is never terminated before we're ready for it.
+     */
+    private final Phaser inFlightRequestsPhaser = new Phaser(1);
+
+    /**
+     * Creates a Retry2.
+     * @param maxNumberOfRetries This is the maximum number of times a BulkRequest will be retried
+     */
+    Retry2(int maxNumberOfRetries) {
+        this.maxNumberOfRetries = maxNumberOfRetries;
+    }
+
+    /**
+     * This method attempts to load the given BulkRequest (via the given BiConsumer). If the initial load fails with a retry-able reason,
+     * this class will retry the load up to maxNumberOfRetries times. The given ActionListener will be notified of the result, either on
+     * success or after failure when no retries are left. The listener is not notified of failures if it is still possible to retry.
+     * @param consumer The consumer to which apply the request and listener. This consumer is expected to perform its work asynchronously
+     *                (that is, not block the thread from which it is called).
+     * @param bulkRequest The bulk request that should be executed.
+     * @param listener A listener that is invoked when the bulk request finishes or completes with an exception.
+     */
+    public void consumeRequestWithRetries(
+        BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer,
+        BulkRequest bulkRequest,
+        ActionListener<BulkResponse> listener
+    ) {
+        if (isClosing) {
+            listener.onFailure(new EsRejectedExecutionException("The bulk processor is closing"));
+            return;
+        }
+        List<BulkItemResponse> responsesAccumulator = new ArrayList<>();
+        logger.trace("Sending a bulk request with {} bytes in {} items", bulkRequest.estimatedSizeInBytes(), bulkRequest.requests.size());
+        inFlightRequestsPhaser.register();
+        consumer.accept(bulkRequest, new RetryHandler(bulkRequest, responsesAccumulator, consumer, listener, maxNumberOfRetries));
+    }
+
+    /**
+     * Retries the bulkRequestForRetry if retriesRemaining is greater than 0, otherwise notifies the listener of failure
+     * @param bulkRequestForRetry The bulk request for retry. This should only include the items that have not previously succeeded
+     * @param responsesAccumulator An accumulator for all BulkItemResponses for the original bulkRequest across all retries
+     * @param consumer
+     * @param listener The listener to be notified of success or failure on this retry or subsequent retries
+     * @param retriesRemaining The number of times remaining that this BulkRequest can be retried
+     */
+    private void retry(
+        BulkRequest bulkRequestForRetry,
+        List<BulkItemResponse> responsesAccumulator,
+        BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer,
+        ActionListener<BulkResponse> listener,
+        int retriesRemaining
+    ) {
+        if (isClosing) {
+            listener.onFailure(new EsRejectedExecutionException("The bulk processor is closing"));
+            return;
+        }
+        if (retriesRemaining > 0) {
+            inFlightRequestsPhaser.register();
+            consumer.accept(
+                bulkRequestForRetry,
+                new RetryHandler(bulkRequestForRetry, responsesAccumulator, consumer, listener, retriesRemaining - 1)
+            );
+        } else {
+            listener.onFailure(
+                new EsRejectedExecutionException(
+                    "Could not retry the bulk request because the backoff policy does not allow any more retries"
+                )
+            );
+        }
+    }
+
+    /**
+     * This method makes an attempt to wait for any outstanding requests to complete. Any new requests that come in after this method has
+     * been called (whether via consumeRequestWithRetries() or a retry) will be rejected by sending EsRejectedExecutionExceptions to their
+     * listeners.
+     * @param timeout
+     * @param unit
+     */
+    void awaitClose(long timeout, TimeUnit unit) throws InterruptedException {
+        isClosing = true;
+        /*
+         * This removes the party that was placed in the phaser at initialization so that the phaser will terminate once all in-flight
+         * requests have been completed (i.e. this makes it possible that the number of parties can become 0).
+         */
+        inFlightRequestsPhaser.arriveAndDeregister();
+        try {
+            inFlightRequestsPhaser.awaitAdvanceInterruptibly(0, timeout, unit);
+        } catch (TimeoutException e) {
+            logger.debug("Timed out waiting for all requests to complete during awaitClose");
+        }
+    }
+
+    /**
+     * This listener will retry any failed requests within a bulk request if possible. It only delegates to the underlying listener once
+     * either all requests have succeeded or all retry attempts have been exhausted.
+     */
+    private final class RetryHandler implements ActionListener<BulkResponse> {
+        private static final RestStatus RETRY_STATUS = RestStatus.TOO_MANY_REQUESTS;
+        private final BulkRequest bulkRequest;
+        private final BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer;
+        private final ActionListener<BulkResponse> listener;
+        private final List<BulkItemResponse> responsesAccumulator;
+        private final long startTimestampNanos;
+        private final int retriesRemaining;
+
+        /**
+         * Creates a RetryHandler listener
+         * @param bulkRequest The BulkRequest to be sent, a subset of the original BulkRequest.
+         * @param responsesAccumulator The accumulator of all BulkItemResponses for the original BulkRequest. These are completed
+         *                             responses, meaning responses for successes, or responses for failures only if no more retries are
+         *                             allowed.
+         * @param consumer
+         * @param listener The delegate listener
+         * @param retriesRemaining The number of retry attempts remaining for the bulkRequestForRetry
+         */
+        RetryHandler(
+            BulkRequest bulkRequest,
+            List<BulkItemResponse> responsesAccumulator,
+            BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer,
+            ActionListener<BulkResponse> listener,
+            int retriesRemaining
+        ) {
+            this.bulkRequest = bulkRequest;
+            this.responsesAccumulator = responsesAccumulator;
+            this.consumer = consumer;
+            this.listener = listener;
+            this.startTimestampNanos = System.nanoTime();
+            this.retriesRemaining = retriesRemaining;
+        }
+
+        @Override
+        public void onResponse(BulkResponse bulkItemResponses) {
+            if (bulkItemResponses.hasFailures() == false) {
+                logger.trace(
+                    "Got a response in {} with {} items, no failures",
+                    bulkItemResponses.getTook(),
+                    bulkItemResponses.getItems().length
+                );
+                // we're done here, include all responses
+                addResponses(bulkItemResponses, (r -> true));
+                listener.onResponse(getAccumulatedResponse());
+            } else {
+                if (canRetry(bulkItemResponses)) {
+                    logger.trace(
+                        "Got a response in {} with {} items including failures, can retry",
+                        bulkItemResponses.getTook(),
+                        bulkItemResponses.getItems().length
+                    );
+                    addResponses(bulkItemResponses, (r -> r.isFailed() == false));
+                    BulkRequest retryRequest = createBulkRequestForRetry(bulkItemResponses);
+                    retry(retryRequest, responsesAccumulator, consumer, listener, retriesRemaining);
+                } else {
+                    logger.trace(
+                        "Got a response in {} with {} items including failures, cannot retry",
+                        bulkItemResponses.getTook(),
+                        bulkItemResponses.getItems().length
+                    );
+                    addResponses(bulkItemResponses, (r -> true));
+                    listener.onResponse(getAccumulatedResponse());
+                }
+            }
+            inFlightRequestsPhaser.arriveAndDeregister();
+        }
+
+        @Override
+        public void onFailure(Exception e) {
+            boolean canRetry = ExceptionsHelper.status(e) == RETRY_STATUS && retriesRemaining > 0;
+            if (canRetry) {
+                inFlightRequestsPhaser.arriveAndDeregister();
+                retry(bulkRequest, responsesAccumulator, consumer, listener, retriesRemaining);
+            } else {
+                listener.onFailure(e);
+                inFlightRequestsPhaser.arriveAndDeregister();
+            }
+        }
+
+        /**
+         * This creates a new BulkRequest from only those items in the bulkItemsResponses that failed.
+         * @param bulkItemResponses The latest response (including any successes and failures)
+         * @return
+         */
+        private BulkRequest createBulkRequestForRetry(BulkResponse bulkItemResponses) {
+            BulkRequest requestToReissue = new BulkRequest();
+            int index = 0;
+            for (BulkItemResponse bulkItemResponse : bulkItemResponses.getItems()) {
+                if (bulkItemResponse.isFailed()) {
+                    DocWriteRequest<?> originalBulkItemRequest = bulkRequest.requests().get(index);
+                    if (originalBulkItemRequest instanceof IndexRequest item) {
+                        item.reset();
+                    }
+                    requestToReissue.add(originalBulkItemRequest);
+                }
+                index++;
+            }
+            return requestToReissue;
+        }
+
+        /**
+         * Returns true if the given bulkItemResponses can be retried.
+         * @param bulkItemResponses
+         * @return
+         */
+        private boolean canRetry(BulkResponse bulkItemResponses) {
+            if (retriesRemaining == 0) {
+                return false;
+            }
+            for (BulkItemResponse bulkItemResponse : bulkItemResponses) {
+                if (bulkItemResponse.isFailed()) {
+                    final RestStatus status = bulkItemResponse.status();
+                    if (status != RETRY_STATUS) {
+                        return false;
+                    }
+                }
+            }
+            return true;
+        }
+
+        private void addResponses(BulkResponse response, Predicate<BulkItemResponse> filter) {
+            List<BulkItemResponse> bulkItemResponses = StreamSupport.stream(response.spliterator(), false).filter(filter).toList();
+            responsesAccumulator.addAll(bulkItemResponses);
+        }
+
+        private BulkResponse getAccumulatedResponse() {
+            BulkItemResponse[] itemResponses = responsesAccumulator.toArray(new BulkItemResponse[0]);
+            long stopTimestamp = System.nanoTime();
+            long totalLatencyMs = TimeValue.timeValueNanos(stopTimestamp - startTimestampNanos).millis();
+            logger.trace("Accumulated response includes {} items", itemResponses.length);
+            return new BulkResponse(itemResponses, totalLatencyMs);
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
index 827cdc57d21a..c942cb6aa391 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
@@ -30,6 +30,8 @@
 import org.elasticsearch.action.ingest.IngestActionForwarder;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.support.WriteResponse;
 import org.elasticsearch.action.support.replication.ReplicationResponse;
 import org.elasticsearch.action.update.UpdateRequest;
@@ -44,8 +46,10 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.IndexRouting;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.core.Releasable;
@@ -80,6 +84,7 @@
 import java.util.function.LongSupplier;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.EXCLUDED_DATA_STREAMS_KEY;
 import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM;
 import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
 
@@ -188,7 +193,32 @@ public static <Response extends ReplicationResponse & WriteResponse> ActionListe
     }
 
     @Override
-    protected void doExecute(Task task, BulkRequest bulkRequest, ActionListener<BulkResponse> listener) {
+    protected void doExecute(Task task, BulkRequest bulkRequest, ActionListener<BulkResponse> outerListener) {
+        // As a work-around to support `?refresh`, explicitly replace the refresh policy with a call to the Refresh API,
+        // and always set forced_refresh to true.
+        // TODO: Replace with a less hacky approach.
+        ActionListener<BulkResponse> listener = outerListener;
+        if (DiscoveryNode.isStateless(clusterService.getSettings()) && bulkRequest.getRefreshPolicy() != WriteRequest.RefreshPolicy.NONE) {
+            listener = outerListener.delegateFailure((l, r) -> {
+                final Set<String> indices = new HashSet<>();
+                for (BulkItemResponse response : r.getItems()) {
+                    if (response.isFailed() == false) {
+                        indices.add(response.getIndex());
+                    }
+                    DocWriteResponse docWriteResponse = response.getResponse();
+                    if (docWriteResponse != null) {
+                        docWriteResponse.setForcedRefresh(true);
+                    }
+                }
+                client.admin()
+                    .indices()
+                    .prepareRefresh()
+                    .setIndices(indices.toArray(Strings.EMPTY_ARRAY))
+                    .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_HIDDEN)
+                    .execute(l.map(ignored -> r));
+            });
+            bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE);
+        }
         /*
          * This is called on the Transport tread so we can check the indexing
          * memory pressure *quickly* but we don't want to keep the transport
@@ -234,7 +264,7 @@ protected void doInternalExecute(Task task, BulkRequest bulkRequest, String exec
             }
 
             if (actionRequest instanceof IndexRequest ir) {
-                ir.checkAutoIdWithOpTypeCreateSupportedByVersion(minNodeVersion);
+                ir.checkAutoIdWithOpTypeCreateSupportedByVersion(minNodeVersion.transportVersion);
                 if (ir.getAutoGeneratedTimestamp() != IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP) {
                     throw new IllegalArgumentException("autoGeneratedTimestamp should not be set externally");
                 }
@@ -245,7 +275,7 @@ protected void doInternalExecute(Task task, BulkRequest bulkRequest, String exec
             // this method (doExecute) will be called again, but with the bulk requests updated from the ingest node processing but
             // also with IngestService.NOOP_PIPELINE_NAME on each request. This ensures that this on the second time through this method,
             // this path is never taken.
-            try {
+            ActionListener.run(listener, l -> {
                 if (Assertions.ENABLED) {
                     final boolean arePipelinesResolved = bulkRequest.requests()
                         .stream()
@@ -255,13 +285,11 @@ protected void doInternalExecute(Task task, BulkRequest bulkRequest, String exec
                     assert arePipelinesResolved : bulkRequest;
                 }
                 if (clusterService.localNode().isIngestNode()) {
-                    processBulkIndexIngestRequest(task, bulkRequest, executorName, listener);
+                    processBulkIndexIngestRequest(task, bulkRequest, executorName, l);
                 } else {
-                    ingestForwarder.forwardIngestRequest(BulkAction.INSTANCE, bulkRequest, listener);
+                    ingestForwarder.forwardIngestRequest(BulkAction.INSTANCE, bulkRequest, l);
                 }
-            } catch (Exception e) {
-                listener.onFailure(e);
-            }
+            });
             return;
         }
 
@@ -754,10 +782,18 @@ private static class ConcreteIndices {
         }
 
         IndexAbstraction resolveIfAbsent(DocWriteRequest<?> request) {
-            return indexAbstractions.computeIfAbsent(
-                request.index(),
-                key -> indexNameExpressionResolver.resolveWriteIndexAbstraction(state, request)
-            );
+            try {
+                return indexAbstractions.computeIfAbsent(
+                    request.index(),
+                    key -> indexNameExpressionResolver.resolveWriteIndexAbstraction(state, request)
+                );
+            } catch (IndexNotFoundException e) {
+                if (e.getMetadataKeys().contains(EXCLUDED_DATA_STREAMS_KEY)) {
+                    throw new IllegalArgumentException("only write ops with an op_type of create are allowed in data streams", e);
+                } else {
+                    throw e;
+                }
+            }
         }
 
         IndexRouting routing(Index index) {
@@ -780,6 +816,7 @@ private void processBulkIndexIngestRequest(
         ingestService.executeBulkRequest(
             original.numberOfActions(),
             () -> bulkRequestModifier,
+            bulkRequestModifier::markItemAsDropped,
             bulkRequestModifier::markItemAsFailed,
             (originalThread, exception) -> {
                 if (exception != null) {
@@ -823,7 +860,6 @@ public boolean isForceExecution() {
                     }
                 }
             },
-            bulkRequestModifier::markItemAsDropped,
             executorName
         );
     }
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
index 44037a9dce6a..e7a7573899df 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
@@ -273,7 +273,6 @@ private void finishRequest() {
                         context.getBulkShardRequest(),
                         context.buildShardResponse(),
                         context.getLocationToSync(),
-                        null,
                         context.getPrimary(),
                         logger,
                         postWriteAction
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/CreateDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/CreateDataStreamAction.java
index 49bcf38ac0e5..f4eae07cc291 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/CreateDataStreamAction.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/CreateDataStreamAction.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.action.datastreams;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.IndicesRequest;
@@ -66,7 +66,7 @@ public ActionRequestValidationException validate() {
         public Request(StreamInput in) throws IOException {
             super(in);
             this.name = in.readString();
-            if (in.getVersion().onOrAfter(Version.V_7_16_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
                 this.startTime = in.readVLong();
             } else {
                 this.startTime = System.currentTimeMillis();
@@ -77,7 +77,7 @@ public Request(StreamInput in) throws IOException {
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             out.writeString(name);
-            if (out.getVersion().onOrAfter(Version.V_7_16_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
                 out.writeVLong(startTime);
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java
index 00bd66fc91b3..6ab2d390be37 100644
--- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java
+++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.action.datastreams;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
@@ -161,7 +161,7 @@ public DataStreamInfo(
                     ClusterHealthStatus.readFrom(in),
                     in.readOptionalString(),
                     in.readOptionalString(),
-                    in.getVersion().onOrAfter(Version.V_8_3_0) ? in.readOptionalWriteable(TimeSeries::new) : null
+                    in.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0) ? in.readOptionalWriteable(TimeSeries::new) : null
                 );
             }
 
@@ -194,7 +194,7 @@ public void writeTo(StreamOutput out) throws IOException {
                 dataStreamStatus.writeTo(out);
                 out.writeOptionalString(indexTemplate);
                 out.writeOptionalString(ilmPolicyName);
-                if (out.getVersion().onOrAfter(Version.V_8_3_0)) {
+                if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0)) {
                     out.writeOptionalWriteable(timeSeries);
                 }
             }
diff --git a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java
index 42914d85bdeb..d944ada2f424 100644
--- a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.action.delete;
 
 import org.apache.lucene.util.RamUsageEstimator;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.CompositeIndicesRequest;
 import org.elasticsearch.action.DocWriteRequest;
@@ -64,7 +64,7 @@ public DeleteRequest(StreamInput in) throws IOException {
 
     public DeleteRequest(@Nullable ShardId shardId, StreamInput in) throws IOException {
         super(shardId, in);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             String type = in.readString();
             assert MapperService.SINGLE_MAPPING_NAME.equals(type) : "Expected [_doc] but received [" + type + "]";
         }
@@ -256,7 +256,7 @@ public void writeThin(StreamOutput out) throws IOException {
     }
 
     private void writeBody(StreamOutput out) throws IOException {
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeString(MapperService.SINGLE_MAPPING_NAME);
         }
         out.writeString(id);
diff --git a/server/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java b/server/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java
index aceeb3bde53a..685eb0b8a199 100644
--- a/server/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.explain;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.action.support.single.shard.SingleShardRequest;
@@ -54,7 +54,7 @@ public ExplainRequest(String index, String id) {
 
     ExplainRequest(StreamInput in) throws IOException {
         super(in);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             String type = in.readString();
             assert MapperService.SINGLE_MAPPING_NAME.equals(type);
         }
@@ -160,7 +160,7 @@ public ActionRequestValidationException validate() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeString(MapperService.SINGLE_MAPPING_NAME);
         }
         out.writeString(id);
diff --git a/server/src/main/java/org/elasticsearch/action/explain/ExplainResponse.java b/server/src/main/java/org/elasticsearch/action/explain/ExplainResponse.java
index 0b0f340cf61e..97c56069fa76 100644
--- a/server/src/main/java/org/elasticsearch/action/explain/ExplainResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/explain/ExplainResponse.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.action.explain;
 
 import org.apache.lucene.search.Explanation;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -69,7 +69,7 @@ public ExplainResponse(String index, String id, boolean exists, Explanation expl
     public ExplainResponse(StreamInput in) throws IOException {
         super(in);
         index = in.readString();
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             in.readString();
         }
         id = in.readString();
@@ -118,7 +118,7 @@ public RestStatus status() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(index);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeString(MapperService.SINGLE_MAPPING_NAME);
         }
         out.writeString(id);
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java
index eec23999dedc..daa9ee30bf9a 100644
--- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java
+++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.fieldcaps;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -211,7 +211,7 @@ public FieldCapabilities(
             isSearchable,
             isAggregatable,
             isDimension == null ? false : isDimension,
-            metricType != null ? Enum.valueOf(TimeSeriesParams.MetricType.class, metricType) : null,
+            metricType != null ? TimeSeriesParams.MetricType.fromString(metricType) : null,
             indices != null ? indices.toArray(new String[0]) : null,
             nonSearchableIndices != null ? nonSearchableIndices.toArray(new String[0]) : null,
             nonAggregatableIndices != null ? nonAggregatableIndices.toArray(new String[0]) : null,
@@ -227,7 +227,7 @@ public FieldCapabilities(
         this.isMetadataField = in.readBoolean();
         this.isSearchable = in.readBoolean();
         this.isAggregatable = in.readBoolean();
-        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             this.isDimension = in.readBoolean();
             this.metricType = in.readOptionalEnum(TimeSeriesParams.MetricType.class);
         } else {
@@ -237,7 +237,7 @@ public FieldCapabilities(
         this.indices = in.readOptionalStringArray();
         this.nonSearchableIndices = in.readOptionalStringArray();
         this.nonAggregatableIndices = in.readOptionalStringArray();
-        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             this.nonDimensionIndices = in.readOptionalStringArray();
             this.metricConflictsIndices = in.readOptionalStringArray();
         } else {
@@ -254,14 +254,14 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(isMetadataField);
         out.writeBoolean(isSearchable);
         out.writeBoolean(isAggregatable);
-        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             out.writeBoolean(isDimension);
             out.writeOptionalEnum(metricType);
         }
         out.writeOptionalStringArray(indices);
         out.writeOptionalStringArray(nonSearchableIndices);
         out.writeOptionalStringArray(nonAggregatableIndices);
-        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             out.writeOptionalStringArray(nonDimensionIndices);
             out.writeOptionalStringArray(metricConflictsIndices);
         }
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java
index 9a44a188b85b..ec03a4465eea 100644
--- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java
+++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.ShardSearchRequest;
+import org.elasticsearch.tasks.CancellableTask;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -45,6 +46,7 @@ class FieldCapabilitiesFetcher {
     }
 
     FieldCapabilitiesIndexResponse fetch(
+        CancellableTask task,
         ShardId shardId,
         String[] fieldPatterns,
         String[] filters,
@@ -78,7 +80,7 @@ FieldCapabilitiesIndexResponse fetch(
                     return new FieldCapabilitiesIndexResponse(shardId.getIndexName(), indexMappingHash, existing, true);
                 }
             }
-
+            task.ensureNotCancelled();
             Predicate<String> fieldPredicate = indicesService.getFieldFilter().apply(shardId.getIndexName());
             final Map<String, IndexFieldCapabilities> responseMap = retrieveFieldCaps(
                 searchExecutionContext,
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java
index 3d03cfc92e1e..ab13525014cc 100644
--- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.fieldcaps;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -23,14 +23,14 @@
 import java.util.stream.Stream;
 
 final class FieldCapabilitiesIndexResponse implements Writeable {
-    private static final Version MAPPING_HASH_VERSION = Version.V_8_2_0;
+    private static final TransportVersion MAPPING_HASH_VERSION = TransportVersion.V_8_2_0;
 
     private final String indexName;
     @Nullable
     private final String indexMappingHash;
     private final Map<String, IndexFieldCapabilities> responseMap;
     private final boolean canMatch;
-    private final transient Version originVersion;
+    private final transient TransportVersion originVersion;
 
     FieldCapabilitiesIndexResponse(
         String indexName,
@@ -42,15 +42,15 @@ final class FieldCapabilitiesIndexResponse implements Writeable {
         this.indexMappingHash = indexMappingHash;
         this.responseMap = responseMap;
         this.canMatch = canMatch;
-        this.originVersion = Version.CURRENT;
+        this.originVersion = TransportVersion.CURRENT;
     }
 
     FieldCapabilitiesIndexResponse(StreamInput in) throws IOException {
         this.indexName = in.readString();
         this.responseMap = in.readMap(StreamInput::readString, IndexFieldCapabilities::new);
         this.canMatch = in.readBoolean();
-        this.originVersion = in.getVersion();
-        if (in.getVersion().onOrAfter(MAPPING_HASH_VERSION)) {
+        this.originVersion = in.getTransportVersion();
+        if (in.getTransportVersion().onOrAfter(MAPPING_HASH_VERSION)) {
             this.indexMappingHash = in.readOptionalString();
         } else {
             this.indexMappingHash = null;
@@ -62,7 +62,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(indexName);
         out.writeMap(responseMap, StreamOutput::writeString, (valueOut, fc) -> fc.writeTo(valueOut));
         out.writeBoolean(canMatch);
-        if (out.getVersion().onOrAfter(MAPPING_HASH_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(MAPPING_HASH_VERSION)) {
             out.writeOptionalString(indexMappingHash);
         }
     }
@@ -87,7 +87,7 @@ List<FieldCapabilitiesIndexResponse> getResponses() {
     }
 
     static List<FieldCapabilitiesIndexResponse> readList(StreamInput input) throws IOException {
-        if (input.getVersion().before(MAPPING_HASH_VERSION)) {
+        if (input.getTransportVersion().before(MAPPING_HASH_VERSION)) {
             return input.readList(FieldCapabilitiesIndexResponse::new);
         }
         final List<FieldCapabilitiesIndexResponse> ungroupedList = input.readList(FieldCapabilitiesIndexResponse::new);
@@ -96,7 +96,7 @@ static List<FieldCapabilitiesIndexResponse> readList(StreamInput input) throws I
     }
 
     static void writeList(StreamOutput output, List<FieldCapabilitiesIndexResponse> responses) throws IOException {
-        if (output.getVersion().before(MAPPING_HASH_VERSION)) {
+        if (output.getTransportVersion().before(MAPPING_HASH_VERSION)) {
             output.writeCollection(responses);
             return;
         }
@@ -152,7 +152,7 @@ public IndexFieldCapabilities getField(String field) {
         return responseMap.get(field);
     }
 
-    Version getOriginVersion() {
+    TransportVersion getOriginVersion() {
         return originVersion;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeRequest.java
index 12bd84952f04..ef1c068bdc3e 100644
--- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.fieldcaps;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
@@ -19,6 +19,9 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -41,7 +44,7 @@ class FieldCapabilitiesNodeRequest extends ActionRequest implements IndicesReque
         super(in);
         shardIds = in.readList(ShardId::new);
         fields = in.readStringArray();
-        if (in.getVersion().onOrAfter(Version.V_8_2_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
             filters = in.readStringArray();
             allowedTypes = in.readStringArray();
         } else {
@@ -121,7 +124,7 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeList(shardIds);
         out.writeStringArray(fields);
-        if (out.getVersion().onOrAfter(Version.V_8_2_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
             out.writeStringArray(filters);
             out.writeStringArray(allowedTypes);
         }
@@ -136,6 +139,30 @@ public ActionRequestValidationException validate() {
         return null;
     }
 
+    @Override
+    public String getDescription() {
+        final StringBuilder stringBuilder = new StringBuilder("shards[");
+        Strings.collectionToDelimitedStringWithLimit(shardIds, ",", "", "", 1024, stringBuilder);
+        stringBuilder.append("], fields[");
+        Strings.collectionToDelimitedStringWithLimit(Arrays.asList(fields), ",", "", "", 1024, stringBuilder);
+        stringBuilder.append("], filters[");
+        stringBuilder.append(Strings.collectionToDelimitedString(Arrays.asList(filters), ","));
+        stringBuilder.append("], types[");
+        stringBuilder.append(Strings.collectionToDelimitedString(Arrays.asList(allowedTypes), ","));
+        stringBuilder.append("]");
+        return stringBuilder.toString();
+    }
+
+    @Override
+    public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+        return new CancellableTask(id, type, action, "", parentTaskId, headers) {
+            @Override
+            public String getDescription() {
+                return FieldCapabilitiesNodeRequest.this.getDescription();
+            }
+        };
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java
index b1b51de5c069..a19767002623 100644
--- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.fieldcaps;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
@@ -18,6 +18,9 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -55,7 +58,7 @@ public FieldCapabilitiesRequest(StreamInput in) throws IOException {
         indexFilter = in.readOptionalNamedWriteable(QueryBuilder.class);
         nowInMillis = in.readOptionalLong();
         runtimeFields = in.readMap();
-        if (in.getVersion().onOrAfter(Version.V_8_2_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
             filters = in.readStringArray();
             types = in.readStringArray();
         }
@@ -93,7 +96,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalNamedWriteable(indexFilter);
         out.writeOptionalLong(nowInMillis);
         out.writeGenericMap(runtimeFields);
-        if (out.getVersion().onOrAfter(Version.V_8_2_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
             out.writeStringArray(filters);
             out.writeStringArray(types);
         }
@@ -271,4 +274,13 @@ public String getDescription() {
         return stringBuilder.toString();
     }
 
+    @Override
+    public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+        return new CancellableTask(id, type, action, "", parentTaskId, headers) {
+            @Override
+            public String getDescription() {
+                return FieldCapabilitiesRequest.this.getDescription();
+            }
+        };
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java
index f1bbc144cb4a..44ab201eb368 100644
--- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
@@ -34,7 +34,7 @@
 /**
  * Response for {@link FieldCapabilitiesRequest} requests.
  */
-public class FieldCapabilitiesResponse extends ActionResponse implements ChunkedToXContent {
+public class FieldCapabilitiesResponse extends ActionResponse implements ChunkedToXContentObject {
     private static final ParseField INDICES_FIELD = new ParseField("indices");
     private static final ParseField FIELDS_FIELD = new ParseField("fields");
     private static final ParseField FAILED_INDICES_FIELD = new ParseField("failed_indices");
@@ -152,7 +152,7 @@ private static void writeField(StreamOutput out, Map<String, FieldCapabilities>
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked() {
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
         if (indexResponses.size() > 0) {
             throw new IllegalStateException("cannot serialize non-merged response");
         }
@@ -241,6 +241,9 @@ public int hashCode() {
 
     @Override
     public String toString() {
+        if (indexResponses.size() > 0) {
+            return "FieldCapabilitiesResponse{unmerged}";
+        }
         return Strings.toString(this);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java
index 383f5f2a0d4f..9b3b7cd08c6c 100644
--- a/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java
+++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.fieldcaps;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -68,7 +68,7 @@ public class IndexFieldCapabilities implements Writeable {
         this.isMetadatafield = in.readBoolean();
         this.isSearchable = in.readBoolean();
         this.isAggregatable = in.readBoolean();
-        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             this.isDimension = in.readBoolean();
             this.metricType = in.readOptionalEnum(TimeSeriesParams.MetricType.class);
         } else {
@@ -85,7 +85,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(isMetadatafield);
         out.writeBoolean(isSearchable);
         out.writeBoolean(isAggregatable);
-        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             out.writeBoolean(isDimension);
             out.writeOptionalEnum(metricType);
         }
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/ResponseRewriter.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/ResponseRewriter.java
index aad38a5bf48a..d7d923ec089a 100644
--- a/server/src/main/java/org/elasticsearch/action/fieldcaps/ResponseRewriter.java
+++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/ResponseRewriter.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.fieldcaps;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -24,12 +24,12 @@
 final class ResponseRewriter {
 
     public static Map<String, IndexFieldCapabilities> rewriteOldResponses(
-        Version version,
+        TransportVersion version,
         Map<String, IndexFieldCapabilities> input,
         String[] filters,
         String[] allowedTypes
     ) {
-        if (version.onOrAfter(Version.V_8_2_0)) {
+        if (version.onOrAfter(TransportVersion.V_8_2_0)) {
             return input;   // nothing needs to be done
         }
         Function<IndexFieldCapabilities, IndexFieldCapabilities> transformer = buildTransformer(input, filters, allowedTypes);
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java
index c0dae35971cd..746c97b6e247 100644
--- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.search.SearchService;
+import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.RemoteClusterAware;
@@ -93,6 +94,8 @@ protected void doExecute(Task task, FieldCapabilitiesRequest request, final Acti
         if (ccsCheckCompatibility) {
             checkCCSVersionCompatibility(request);
         }
+        assert task instanceof CancellableTask;
+        final CancellableTask fieldCapTask = (CancellableTask) task;
         // retrieve the initial timestamp in case the action is a cross cluster search
         long nowInMillis = request.nowInMillis() == null ? System.currentTimeMillis() : request.nowInMillis();
         final ClusterState clusterState = clusterService.state();
@@ -129,7 +132,7 @@ protected void doExecute(Task task, FieldCapabilitiesRequest request, final Acti
         final FailureCollector indexFailures = new FailureCollector();
         // One for each cluster including the local cluster
         final CountDown completionCounter = new CountDown(1 + remoteClusterIndices.size());
-        final Runnable countDown = createResponseMerger(request, completionCounter, indexResponses, indexFailures, listener);
+        final Runnable countDown = createResponseMerger(request, fieldCapTask, completionCounter, indexResponses, indexFailures, listener);
         final RequestDispatcher requestDispatcher = new RequestDispatcher(
             clusterService,
             transportService,
@@ -180,6 +183,7 @@ private static void checkIndexBlocks(ClusterState clusterState, String[] concret
 
     private Runnable createResponseMerger(
         FieldCapabilitiesRequest request,
+        CancellableTask task,
         CountDown completionCounter,
         Map<String, FieldCapabilitiesIndexResponse> indexResponses,
         FailureCollector indexFailures,
@@ -193,7 +197,7 @@ private Runnable createResponseMerger(
                         // fork off to the management pool for merging the responses as the operation can run for longer than is acceptable
                         // on a transport thread in case of large numbers of indices and/or fields
                         threadPool.executor(ThreadPool.Names.SEARCH_COORDINATION)
-                            .submit(ActionRunnable.supply(listener, () -> merge(indexResponses, request, new ArrayList<>(failures))));
+                            .submit(ActionRunnable.supply(listener, () -> merge(indexResponses, task, request, new ArrayList<>(failures))));
                     } else {
                         listener.onResponse(
                             new FieldCapabilitiesResponse(new ArrayList<>(indexResponses.values()), new ArrayList<>(failures))
@@ -238,9 +242,11 @@ private static boolean hasSameMappingHash(FieldCapabilitiesIndexResponse r1, Fie
 
     private FieldCapabilitiesResponse merge(
         Map<String, FieldCapabilitiesIndexResponse> indexResponsesMap,
+        CancellableTask task,
         FieldCapabilitiesRequest request,
         List<FieldCapabilitiesFailure> failures
     ) {
+        task.ensureNotCancelled();
         final FieldCapabilitiesIndexResponse[] indexResponses = indexResponsesMap.values()
             .stream()
             .sorted(Comparator.comparing(FieldCapabilitiesIndexResponse::getIndexName))
@@ -261,6 +267,7 @@ private FieldCapabilitiesResponse merge(
             }
         }
 
+        task.ensureNotCancelled();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
         for (Map.Entry<String, Map<String, FieldCapabilities.Builder>> entry : responseMapBuilder.entrySet()) {
             Map<String, FieldCapabilities.Builder> typeMapBuilder = entry.getValue();
@@ -387,6 +394,7 @@ boolean isEmpty() {
     private class NodeTransportHandler implements TransportRequestHandler<FieldCapabilitiesNodeRequest> {
         @Override
         public void messageReceived(FieldCapabilitiesNodeRequest request, TransportChannel channel, Task task) throws Exception {
+            assert task instanceof CancellableTask;
             final ActionListener<FieldCapabilitiesNodeResponse> listener = new ChannelActionListener<>(channel, ACTION_NODE_NAME, request);
             ActionListener.completeWith(listener, () -> {
                 final List<FieldCapabilitiesIndexResponse> allResponses = new ArrayList<>();
@@ -404,6 +412,7 @@ public void messageReceived(FieldCapabilitiesNodeRequest request, TransportChann
                     for (ShardId shardId : shardIds) {
                         try {
                             final FieldCapabilitiesIndexResponse response = fetcher.fetch(
+                                (CancellableTask) task,
                                 shardId,
                                 request.fields(),
                                 request.filters(),
diff --git a/server/src/main/java/org/elasticsearch/action/get/GetRequest.java b/server/src/main/java/org/elasticsearch/action/get/GetRequest.java
index 4f1a8d5f9d54..9ed35e303b4e 100644
--- a/server/src/main/java/org/elasticsearch/action/get/GetRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/get/GetRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.get;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.RealtimeRequest;
 import org.elasticsearch.action.ValidateActions;
@@ -68,7 +68,7 @@ public GetRequest() {}
 
     GetRequest(StreamInput in) throws IOException {
         super(in);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             in.readString();
         }
         id = in.readString();
@@ -81,7 +81,7 @@ public GetRequest() {}
         this.versionType = VersionType.fromValue(in.readByte());
         this.version = in.readLong();
         fetchSourceContext = in.readOptionalWriteable(FetchSourceContext::readFrom);
-        if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             forceSyntheticSource = in.readBoolean();
         } else {
             forceSyntheticSource = false;
@@ -91,7 +91,7 @@ public GetRequest() {}
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeString(MapperService.SINGLE_MAPPING_NAME);
         }
         out.writeString(id);
@@ -104,7 +104,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeByte(versionType.getValue());
         out.writeLong(version);
         out.writeOptionalWriteable(fetchSourceContext);
-        if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             out.writeBoolean(forceSyntheticSource);
         } else {
             if (forceSyntheticSource) {
diff --git a/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java b/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java
index 09f24eec5296..5774fc58d999 100644
--- a/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.action.get;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.CompositeIndicesRequest;
@@ -85,7 +85,7 @@ public Item() {
 
         public Item(StreamInput in) throws IOException {
             index = in.readString();
-            if (in.getVersion().before(Version.V_8_0_0)) {
+            if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 in.readOptionalString();
             }
             id = in.readString();
@@ -179,7 +179,7 @@ public Item fetchSourceContext(FetchSourceContext fetchSourceContext) {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(index);
-            if (out.getVersion().before(Version.V_8_0_0)) {
+            if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 out.writeOptionalString(MapperService.SINGLE_MAPPING_NAME);
             }
             out.writeString(id);
@@ -263,7 +263,7 @@ public MultiGetRequest(StreamInput in) throws IOException {
         refresh = in.readBoolean();
         realtime = in.readBoolean();
         items = in.readList(Item::new);
-        if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             forceSyntheticSource = in.readBoolean();
         } else {
             forceSyntheticSource = false;
@@ -277,7 +277,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(refresh);
         out.writeBoolean(realtime);
         out.writeList(items);
-        if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             out.writeBoolean(forceSyntheticSource);
         } else {
             if (forceSyntheticSource) {
diff --git a/server/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java b/server/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java
index eb979bc57855..b7c68a51ed60 100644
--- a/server/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.action.get;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -57,7 +57,7 @@ public Failure(String index, String id, Exception exception) {
 
         Failure(StreamInput in) throws IOException {
             index = in.readString();
-            if (in.getVersion().before(Version.V_8_0_0)) {
+            if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 in.readOptionalString();
             }
             id = in.readString();
@@ -88,7 +88,7 @@ public String getMessage() {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(index);
-            if (out.getVersion().before(Version.V_8_0_0)) {
+            if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 out.writeOptionalString(MapperService.SINGLE_MAPPING_NAME);
             }
             out.writeString(id);
diff --git a/server/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java b/server/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java
index fc641c31512a..9b63fcc469a8 100644
--- a/server/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.get;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.single.shard.SingleShardRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -62,7 +62,7 @@ public class MultiGetShardRequest extends SingleShardRequest<MultiGetShardReques
         preference = in.readOptionalString();
         refresh = in.readBoolean();
         realtime = in.readBoolean();
-        if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             forceSyntheticSource = in.readBoolean();
         } else {
             forceSyntheticSource = false;
@@ -82,7 +82,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(preference);
         out.writeBoolean(refresh);
         out.writeBoolean(realtime);
-        if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             out.writeBoolean(forceSyntheticSource);
         } else {
             if (forceSyntheticSource) {
diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java
index 70d0cd40b5a3..a6d614798a94 100644
--- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java
@@ -10,6 +10,7 @@
 
 import org.apache.lucene.util.RamUsageEstimator;
 import org.elasticsearch.ElasticsearchGenerationException;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.CompositeIndicesRequest;
@@ -128,7 +129,7 @@ public IndexRequest(StreamInput in) throws IOException {
 
     public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOException {
         super(shardId, in);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             String type = in.readOptionalString();
             assert MapperService.SINGLE_MAPPING_NAME.equals(type) : "Expected [_doc] but received [" + type + "]";
         }
@@ -139,10 +140,10 @@ public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOExceptio
         version = in.readLong();
         versionType = VersionType.fromValue(in.readByte());
         pipeline = in.readOptionalString();
-        if (in.getVersion().onOrAfter(Version.V_7_5_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_5_0)) {
             finalPipeline = in.readOptionalString();
         }
-        if (in.getVersion().onOrAfter(Version.V_7_5_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_5_0)) {
             isPipelineResolved = in.readBoolean();
         }
         isRetry = in.readBoolean();
@@ -154,12 +155,12 @@ public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOExceptio
         }
         ifSeqNo = in.readZLong();
         ifPrimaryTerm = in.readVLong();
-        if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             requireAlias = in.readBoolean();
         } else {
             requireAlias = false;
         }
-        if (in.getVersion().onOrAfter(Version.V_7_13_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0)) {
             dynamicTemplates = in.readMap(StreamInput::readString, StreamInput::readString);
         }
     }
@@ -650,8 +651,8 @@ public void reset() {
         autoGeneratedTimestamp = UNSET_AUTO_GENERATED_TIMESTAMP;
     }
 
-    public void checkAutoIdWithOpTypeCreateSupportedByVersion(Version version) {
-        if (id == null && opType == OpType.CREATE && version.before(Version.V_7_5_0)) {
+    public void checkAutoIdWithOpTypeCreateSupportedByVersion(TransportVersion version) {
+        if (id == null && opType == OpType.CREATE && version.before(TransportVersion.V_7_5_0)) {
             throw new IllegalArgumentException(
                 "optype create not supported for indexing requests without explicit id until all nodes " + "are on version 7.5.0 or higher"
             );
@@ -660,20 +661,20 @@ public void checkAutoIdWithOpTypeCreateSupportedByVersion(Version version) {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        checkAutoIdWithOpTypeCreateSupportedByVersion(out.getVersion());
+        checkAutoIdWithOpTypeCreateSupportedByVersion(out.getTransportVersion());
         super.writeTo(out);
         writeBody(out);
     }
 
     @Override
     public void writeThin(StreamOutput out) throws IOException {
-        checkAutoIdWithOpTypeCreateSupportedByVersion(out.getVersion());
+        checkAutoIdWithOpTypeCreateSupportedByVersion(out.getTransportVersion());
         super.writeThin(out);
         writeBody(out);
     }
 
     private void writeBody(StreamOutput out) throws IOException {
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeOptionalString(MapperService.SINGLE_MAPPING_NAME);
         }
         out.writeOptionalString(id);
@@ -683,10 +684,10 @@ private void writeBody(StreamOutput out) throws IOException {
         out.writeLong(version);
         out.writeByte(versionType.getValue());
         out.writeOptionalString(pipeline);
-        if (out.getVersion().onOrAfter(Version.V_7_5_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_5_0)) {
             out.writeOptionalString(finalPipeline);
         }
-        if (out.getVersion().onOrAfter(Version.V_7_5_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_5_0)) {
             out.writeBoolean(isPipelineResolved);
         }
         out.writeBoolean(isRetry);
@@ -699,10 +700,10 @@ private void writeBody(StreamOutput out) throws IOException {
         }
         out.writeZLong(ifSeqNo);
         out.writeVLong(ifPrimaryTerm);
-        if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             out.writeBoolean(requireAlias);
         }
-        if (out.getVersion().onOrAfter(Version.V_7_13_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0)) {
             out.writeMap(dynamicTemplates, StreamOutput::writeString, StreamOutput::writeString);
         } else {
             if (dynamicTemplates.isEmpty() == false) {
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/ReservedPipelineAction.java b/server/src/main/java/org/elasticsearch/action/ingest/ReservedPipelineAction.java
index 05dfbc563689..4e796a638f48 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/ReservedPipelineAction.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/ReservedPipelineAction.java
@@ -9,14 +9,12 @@
 package org.elasticsearch.action.ingest;
 
 import org.elasticsearch.ElasticsearchGenerationException;
-import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.ingest.IngestMetadata;
 import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
 import org.elasticsearch.reservedstate.TransformState;
-import org.elasticsearch.reservedstate.service.FileSettingsService;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentParser;
@@ -40,19 +38,11 @@
 public class ReservedPipelineAction implements ReservedClusterStateHandler<List<PutPipelineRequest>> {
     public static final String NAME = "ingest_pipelines";
 
-    private final IngestService ingestService;
-    private final FileSettingsService fileSettingsService;
-
     /**
      * Creates a ReservedPipelineAction
      *
-     * @param ingestService requires {@link IngestService} for storing/deleting the pipelines
-     * @param fileSettingsService required for supplying the latest node infos
      */
-    public ReservedPipelineAction(IngestService ingestService, FileSettingsService fileSettingsService) {
-        this.ingestService = ingestService;
-        this.fileSettingsService = fileSettingsService;
-    }
+    public ReservedPipelineAction() {}
 
     @Override
     public String name() {
@@ -61,11 +51,9 @@ public String name() {
 
     private Collection<PutPipelineRequest> prepare(List<PutPipelineRequest> requests) {
         var exceptions = new ArrayList<Exception>();
-        NodesInfoResponse nodeInfos = fileSettingsService.nodeInfos();
-        assert nodeInfos != null;
         for (var pipeline : requests) {
             try {
-                ingestService.validatePipelineRequest(pipeline, nodeInfos);
+                validate(pipeline);
             } catch (Exception e) {
                 exceptions.add(e);
             }
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java
index 2a75ec3b7681..30d13ab93489 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java
@@ -8,22 +8,47 @@
 
 package org.elasticsearch.action.ingest;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionListenerResponseHandler;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.Collection;
 import java.util.Map;
+import java.util.Random;
 
 public class SimulatePipelineTransportAction extends HandledTransportAction<SimulatePipelineRequest, SimulatePipelineResponse> {
-
+    private static final Logger logger = LogManager.getLogger(SimulatePipelineTransportAction.class);
+    /**
+     * This is the amount of time given as the timeout for transport requests to the ingest node.
+     */
+    public static final Setting<TimeValue> INGEST_NODE_TRANSPORT_ACTION_TIMEOUT = Setting.timeSetting(
+        "ingest_node.transport_action_timeout",
+        TimeValue.timeValueSeconds(20),
+        TimeValue.timeValueMillis(1),
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
     private final IngestService ingestService;
     private final SimulateExecutionService executionService;
+    private final TransportService transportService;
+    private volatile TimeValue ingestNodeTransportActionTimeout;
+    // ThreadLocal because our unit testing framework does not like sharing Randoms across threads
+    private final ThreadLocal<Random> random = ThreadLocal.withInitial(Randomness::get);
 
     @Inject
     public SimulatePipelineTransportAction(
@@ -35,30 +60,77 @@ public SimulatePipelineTransportAction(
         super(SimulatePipelineAction.NAME, transportService, actionFilters, SimulatePipelineRequest::new);
         this.ingestService = ingestService;
         this.executionService = new SimulateExecutionService(threadPool);
+        this.transportService = transportService;
+        this.ingestNodeTransportActionTimeout = INGEST_NODE_TRANSPORT_ACTION_TIMEOUT.get(ingestService.getClusterService().getSettings());
+        ingestService.getClusterService()
+            .getClusterSettings()
+            .addSettingsUpdateConsumer(
+                INGEST_NODE_TRANSPORT_ACTION_TIMEOUT,
+                newTimeout -> this.ingestNodeTransportActionTimeout = newTimeout
+            );
     }
 
     @Override
     protected void doExecute(Task task, SimulatePipelineRequest request, ActionListener<SimulatePipelineResponse> listener) {
         final Map<String, Object> source = XContentHelper.convertToMap(request.getSource(), false, request.getXContentType()).v2();
-
-        final SimulatePipelineRequest.Parsed simulateRequest;
+        DiscoveryNodes discoveryNodes = ingestService.getClusterService().state().nodes();
+        Map<String, DiscoveryNode> ingestNodes = discoveryNodes.getIngestNodes();
+        if (ingestNodes.isEmpty()) {
+            /*
+             * Some resources used by pipelines, such as the geoip database, only exist on ingest nodes. Since we only run pipelines on
+             * nodes with the ingest role, we ought to only simulate a pipeline on nodes with the ingest role.
+             */
+            listener.onFailure(
+                new IllegalStateException("There are no ingest nodes in this cluster, unable to forward request to an ingest node.")
+            );
+            return;
+        }
         try {
-            if (request.getId() != null) {
-                simulateRequest = SimulatePipelineRequest.parseWithPipelineId(
-                    request.getId(),
-                    source,
-                    request.isVerbose(),
-                    ingestService,
-                    request.getRestApiVersion()
-                );
+            if (discoveryNodes.getLocalNode().isIngestNode()) {
+                final SimulatePipelineRequest.Parsed simulateRequest;
+                if (request.getId() != null) {
+                    simulateRequest = SimulatePipelineRequest.parseWithPipelineId(
+                        request.getId(),
+                        source,
+                        request.isVerbose(),
+                        ingestService,
+                        request.getRestApiVersion()
+                    );
+                } else {
+                    simulateRequest = SimulatePipelineRequest.parse(
+                        source,
+                        request.isVerbose(),
+                        ingestService,
+                        request.getRestApiVersion()
+                    );
+                }
+                executionService.execute(simulateRequest, listener);
             } else {
-                simulateRequest = SimulatePipelineRequest.parse(source, request.isVerbose(), ingestService, request.getRestApiVersion());
+                DiscoveryNode ingestNode = getRandomIngestNode(ingestNodes.values());
+                logger.trace("forwarding request [{}] to ingest node [{}]", actionName, ingestNode);
+                ActionListenerResponseHandler<SimulatePipelineResponse> handler = new ActionListenerResponseHandler<>(
+                    listener,
+                    SimulatePipelineAction.INSTANCE.getResponseReader()
+                );
+                if (task == null) {
+                    transportService.sendRequest(ingestNode, actionName, request, handler);
+                } else {
+                    transportService.sendChildRequest(
+                        ingestNode,
+                        actionName,
+                        request,
+                        task,
+                        TransportRequestOptions.timeout(ingestNodeTransportActionTimeout),
+                        handler
+                    );
+                }
             }
         } catch (Exception e) {
             listener.onFailure(e);
-            return;
         }
+    }
 
-        executionService.execute(simulateRequest, listener);
+    private DiscoveryNode getRandomIngestNode(Collection<DiscoveryNode> ingestNodes) {
+        return ingestNodes.toArray(new DiscoveryNode[0])[random.get().nextInt(ingestNodes.size())];
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
index a40ff5fa955f..0271afb9a9b7 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.action.ingest;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -159,7 +160,11 @@ public SimulateProcessorResult(String type, String processorTag, String descript
      * Read from a stream.
      */
     SimulateProcessorResult(StreamInput in) throws IOException {
-        this.processorTag = in.readString();
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            this.processorTag = in.readOptionalString();
+        } else {
+            this.processorTag = in.readString();
+        }
         this.ingestDocument = in.readOptionalWriteable(WriteableIngestDocument::new);
         this.failure = in.readException();
         this.description = in.readOptionalString();
@@ -174,7 +179,11 @@ public SimulateProcessorResult(String type, String processorTag, String descript
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(processorTag);
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            out.writeOptionalString(processorTag);
+        } else {
+            out.writeString(processorTag);
+        }
         out.writeOptionalWriteable(ingestDocument);
         out.writeException(failure);
         out.writeOptionalString(description);
diff --git a/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java b/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java
index fa7bef5f5392..fda6393cd50a 100644
--- a/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java
+++ b/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java
@@ -111,7 +111,7 @@ protected void dispatchedShardOperationOnPrimary(
     ) {
         ActionListener.completeWith(
             listener,
-            () -> new WritePrimaryResult<>(performOnPrimary(request), new ResyncReplicationResponse(), null, null, primary, logger)
+            () -> new WritePrimaryResult<>(performOnPrimary(request), new ResyncReplicationResponse(), null, primary, logger)
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
index b5fc113ee6e7..0f2caa7bbe49 100644
--- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
@@ -544,7 +544,7 @@ public final void onShardFailure(final int shardIndex, SearchShardTarget shardTa
         if (TransportActions.isShardNotAvailableException(e)) {
             // Groups shard not available exceptions under a generic exception that returns a SERVICE_UNAVAILABLE(503)
             // temporary error.
-            e = new NoShardAvailableActionException(shardTarget.getShardId(), e.getMessage());
+            e = NoShardAvailableActionException.forOnShardFailureWrapper(e.getMessage());
         }
         // we don't aggregate shard on failures due to the internal cancellation,
         // but do keep the header counts right
diff --git a/server/src/main/java/org/elasticsearch/action/search/CanMatchNodeRequest.java b/server/src/main/java/org/elasticsearch/action/search/CanMatchNodeRequest.java
index e811d838122b..8c5d828fff01 100644
--- a/server/src/main/java/org/elasticsearch/action/search/CanMatchNodeRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/CanMatchNodeRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.search;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -151,7 +151,7 @@ public CanMatchNodeRequest(StreamInput in) throws IOException {
         source = in.readOptionalWriteable(SearchSourceBuilder::new);
         indicesOptions = IndicesOptions.readIndicesOptions(in);
         searchType = SearchType.fromId(in.readByte());
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             // types no longer relevant so ignore
             String[] types = in.readStringArray();
             if (types.length > 0) {
@@ -177,7 +177,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(source);
         indicesOptions.writeIndicesOptions(out);
         out.writeByte(searchType.id());
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             // types not supported so send an empty array to previous versions
             out.writeStringArray(Strings.EMPTY_ARRAY);
         }
diff --git a/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java b/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java
index 1a0f0ad6621d..4c3e29fdda2f 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java
@@ -10,11 +10,10 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.StepListener;
-import org.elasticsearch.action.support.GroupedActionListener;
+import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.util.concurrent.CountDown;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportResponse;
 
@@ -33,7 +32,7 @@
 public final class ClearScrollController implements Runnable {
     private final DiscoveryNodes nodes;
     private final SearchTransportService searchTransportService;
-    private final CountDown expectedOps;
+    private final RefCountingRunnable refs = new RefCountingRunnable(this::finish);
     private final ActionListener<ClearScrollResponse> listener;
     private final AtomicBoolean hasFailed = new AtomicBoolean(false);
     private final AtomicInteger freedSearchContexts = new AtomicInteger(0);
@@ -52,9 +51,7 @@ public final class ClearScrollController implements Runnable {
         this.searchTransportService = searchTransportService;
         this.listener = listener;
         List<String> scrollIds = request.getScrollIds();
-        final int expectedOps;
         if (scrollIds.size() == 1 && "_all".equals(scrollIds.get(0))) {
-            expectedOps = nodes.getSize();
             runner = this::cleanAllScrolls;
         } else {
             // TODO: replace this with #closeContexts
@@ -63,16 +60,8 @@ public final class ClearScrollController implements Runnable {
                 SearchContextIdForNode[] context = parseScrollId(scrollId).getContext();
                 Collections.addAll(contexts, context);
             }
-            if (contexts.isEmpty()) {
-                expectedOps = 0;
-                runner = () -> listener.onResponse(new ClearScrollResponse(true, 0));
-            } else {
-                expectedOps = contexts.size();
-                runner = () -> cleanScrollIds(contexts);
-            }
+            runner = () -> cleanScrollIds(contexts);
         }
-        this.expectedOps = new CountDown(expectedOps);
-
     }
 
     @Override
@@ -81,44 +70,55 @@ public void run() {
     }
 
     void cleanAllScrolls() {
-        for (final DiscoveryNode node : nodes) {
-            try {
-                Transport.Connection connection = searchTransportService.getConnection(null, node);
-                searchTransportService.sendClearAllScrollContexts(connection, new ActionListener<TransportResponse>() {
-                    @Override
-                    public void onResponse(TransportResponse response) {
-                        onFreedContext(true);
-                    }
+        try {
+            for (final DiscoveryNode node : nodes) {
+                try {
+                    Transport.Connection connection = searchTransportService.getConnection(null, node);
+                    searchTransportService.sendClearAllScrollContexts(connection, ActionListener.releaseAfter(new ActionListener<>() {
+                        @Override
+                        public void onResponse(TransportResponse response) {
+                            onFreedContext(true);
+                        }
 
-                    @Override
-                    public void onFailure(Exception e) {
-                        onFailedFreedContext(e, node);
-                    }
-                });
-            } catch (Exception e) {
-                onFailedFreedContext(e, node);
+                        @Override
+                        public void onFailure(Exception e) {
+                            onFailedFreedContext(e, node);
+                        }
+                    }, refs.acquire()));
+                } catch (Exception e) {
+                    onFailedFreedContext(e, node);
+                }
             }
+        } finally {
+            refs.close();
         }
     }
 
     void cleanScrollIds(List<SearchContextIdForNode> contextIds) {
         SearchScrollAsyncAction.collectNodesAndRun(contextIds, nodes, searchTransportService, ActionListener.wrap(lookup -> {
-            for (SearchContextIdForNode target : contextIds) {
-                final DiscoveryNode node = lookup.apply(target.getClusterAlias(), target.getNode());
-                if (node == null) {
-                    onFreedContext(false);
-                } else {
-                    try {
-                        Transport.Connection connection = searchTransportService.getConnection(target.getClusterAlias(), node);
-                        searchTransportService.sendFreeContext(
-                            connection,
-                            target.getSearchContextId(),
-                            ActionListener.wrap(freed -> onFreedContext(freed.isFreed()), e -> onFailedFreedContext(e, node))
-                        );
-                    } catch (Exception e) {
-                        onFailedFreedContext(e, node);
+            try {
+                for (SearchContextIdForNode target : contextIds) {
+                    final DiscoveryNode node = lookup.apply(target.getClusterAlias(), target.getNode());
+                    if (node == null) {
+                        onFreedContext(false);
+                    } else {
+                        try {
+                            Transport.Connection connection = searchTransportService.getConnection(target.getClusterAlias(), node);
+                            searchTransportService.sendFreeContext(
+                                connection,
+                                target.getSearchContextId(),
+                                ActionListener.releaseAfter(
+                                    ActionListener.wrap(freed -> onFreedContext(freed.isFreed()), e -> onFailedFreedContext(e, node)),
+                                    refs.acquire()
+                                )
+                            );
+                        } catch (Exception e) {
+                            onFailedFreedContext(e, node);
+                        }
                     }
                 }
+            } finally {
+                refs.close();
             }
         }, listener::onFailure));
     }
@@ -127,22 +127,15 @@ private void onFreedContext(boolean freed) {
         if (freed) {
             freedSearchContexts.incrementAndGet();
         }
-        if (expectedOps.countDown()) {
-            boolean succeeded = hasFailed.get() == false;
-            listener.onResponse(new ClearScrollResponse(succeeded, freedSearchContexts.get()));
-        }
     }
 
     private void onFailedFreedContext(Throwable e, DiscoveryNode node) {
         logger.warn(() -> "Clear SC failed on node[" + node + "]", e);
-        /*
-         * We have to set the failure marker before we count down otherwise we can expose the failure marker before we have set it to a
-         * racing thread successfully freeing a context. This would lead to that thread responding that the clear scroll succeeded.
-         */
         hasFailed.set(true);
-        if (expectedOps.countDown()) {
-            listener.onResponse(new ClearScrollResponse(false, freedSearchContexts.get()));
-        }
+    }
+
+    private void finish() {
+        listener.onResponse(new ClearScrollResponse(hasFailed.get() == false, freedSearchContexts.get()));
     }
 
     /**
@@ -154,42 +147,39 @@ public static void closeContexts(
         Collection<SearchContextIdForNode> contextIds,
         ActionListener<Integer> listener
     ) {
-        if (contextIds.isEmpty()) {
-            listener.onResponse(0);
-            return;
-        }
         final Set<String> clusters = contextIds.stream()
-            .filter(ctx -> Strings.isEmpty(ctx.getClusterAlias()) == false)
             .map(SearchContextIdForNode::getClusterAlias)
+            .filter(clusterAlias -> Strings.isEmpty(clusterAlias) == false)
             .collect(Collectors.toSet());
         final StepListener<BiFunction<String, String, DiscoveryNode>> lookupListener = new StepListener<>();
-        if (clusters.isEmpty() == false) {
-            searchTransportService.getRemoteClusterService().collectNodes(clusters, lookupListener);
-        } else {
+        if (clusters.isEmpty()) {
             lookupListener.onResponse((cluster, nodeId) -> nodes.get(nodeId));
+        } else {
+            searchTransportService.getRemoteClusterService().collectNodes(clusters, lookupListener);
         }
-        lookupListener.whenComplete(nodeLookup -> {
-            final GroupedActionListener<Boolean> groupedListener = new GroupedActionListener<>(
-                listener.map(rs -> Math.toIntExact(rs.stream().filter(r -> r).count())),
-                contextIds.size()
-            );
-            for (SearchContextIdForNode contextId : contextIds) {
-                final DiscoveryNode node = nodeLookup.apply(contextId.getClusterAlias(), contextId.getNode());
-                if (node == null) {
-                    groupedListener.onResponse(false);
-                } else {
-                    try {
-                        final Transport.Connection connection = searchTransportService.getConnection(contextId.getClusterAlias(), node);
-                        searchTransportService.sendFreeContext(
-                            connection,
-                            contextId.getSearchContextId(),
-                            ActionListener.wrap(r -> groupedListener.onResponse(r.isFreed()), e -> groupedListener.onResponse(false))
-                        );
-                    } catch (Exception e) {
-                        groupedListener.onResponse(false);
+        lookupListener.addListener(listener.delegateFailure((l, nodeLookup) -> {
+            final var successes = new AtomicInteger();
+            try (RefCountingRunnable refs = new RefCountingRunnable(() -> l.onResponse(successes.get()))) {
+                for (SearchContextIdForNode contextId : contextIds) {
+                    final DiscoveryNode node = nodeLookup.apply(contextId.getClusterAlias(), contextId.getNode());
+                    if (node != null) {
+                        try {
+                            searchTransportService.sendFreeContext(
+                                searchTransportService.getConnection(contextId.getClusterAlias(), node),
+                                contextId.getSearchContextId(),
+                                refs.acquireListener().map(r -> {
+                                    if (r.isFreed()) {
+                                        successes.incrementAndGet();
+                                    }
+                                    return null;
+                                })
+                            );
+                        } catch (Exception e) {
+                            // ignored
+                        }
                     }
                 }
             }
-        }, listener::onFailure);
+        }));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
index 7927779c70b9..df81ce606ffa 100644
--- a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java
@@ -38,7 +38,7 @@ final class DfsQueryPhase extends SearchPhase {
     private final QueryPhaseResultConsumer queryResult;
     private final List<DfsSearchResult> searchResults;
     private final AggregatedDfs dfs;
-    private final DfsKnnResults knnResults;
+    private final List<DfsKnnResults> knnResults;
     private final Function<ArraySearchPhaseResults<SearchPhaseResult>, SearchPhase> nextPhaseFactory;
     private final SearchPhaseContext context;
     private final SearchTransportService searchTransportService;
@@ -47,7 +47,7 @@ final class DfsQueryPhase extends SearchPhase {
     DfsQueryPhase(
         List<DfsSearchResult> searchResults,
         AggregatedDfs dfs,
-        DfsKnnResults knnResults,
+        List<DfsKnnResults> knnResults,
         QueryPhaseResultConsumer queryResult,
         Function<ArraySearchPhaseResults<SearchPhaseResult>, SearchPhase> nextPhaseFactory,
         SearchPhaseContext context
@@ -132,20 +132,37 @@ public void onFailure(Exception exception) {
 
     private ShardSearchRequest rewriteShardSearchRequest(ShardSearchRequest request) {
         SearchSourceBuilder source = request.source();
-        if (source == null || source.knnSearch() == null) {
+        if (source == null || source.knnSearch().isEmpty()) {
             return request;
         }
 
         List<ScoreDoc> scoreDocs = new ArrayList<>();
-        for (ScoreDoc scoreDoc : knnResults.scoreDocs()) {
-            if (scoreDoc.shardIndex == request.shardRequestIndex()) {
-                scoreDocs.add(scoreDoc);
+        for (DfsKnnResults dfsKnnResults : knnResults) {
+            for (ScoreDoc scoreDoc : dfsKnnResults.scoreDocs()) {
+                if (scoreDoc.shardIndex == request.shardRequestIndex()) {
+                    scoreDocs.add(scoreDoc);
+                }
             }
         }
         scoreDocs.sort(Comparator.comparingInt(scoreDoc -> scoreDoc.doc));
+        // It is possible that the different results refer to the same doc.
+        for (int i = 0; i < scoreDocs.size() - 1; i++) {
+            ScoreDoc scoreDoc = scoreDocs.get(i);
+            int j = i + 1;
+            for (; j < scoreDocs.size(); j++) {
+                ScoreDoc otherScoreDoc = scoreDocs.get(j);
+                if (otherScoreDoc.doc != scoreDoc.doc) {
+                    break;
+                }
+                scoreDoc.score += otherScoreDoc.score;
+            }
+            if (j > i + 1) {
+                scoreDocs.subList(i + 1, j).clear();
+            }
+        }
         KnnScoreDocQueryBuilder knnQuery = new KnnScoreDocQueryBuilder(scoreDocs.toArray(new ScoreDoc[0]));
 
-        SearchSourceBuilder newSource = source.shallowCopy().knnSearch(null);
+        SearchSourceBuilder newSource = source.shallowCopy().knnSearch(List.of());
         if (source.query() == null) {
             newSource.query(knnQuery);
         } else {
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
index 9e04c26b42eb..662b72e114a4 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
@@ -87,7 +87,7 @@ protected void executePhaseOnShard(
     protected SearchPhase getNextPhase(final SearchPhaseResults<DfsSearchResult> results, SearchPhaseContext context) {
         final List<DfsSearchResult> dfsSearchResults = results.getAtomicArray().asList();
         final AggregatedDfs aggregatedDfs = SearchPhaseController.aggregateDfs(dfsSearchResults);
-        final DfsKnnResults mergedKnnResults = SearchPhaseController.mergeKnnResults(getRequest(), dfsSearchResults);
+        final List<DfsKnnResults> mergedKnnResults = SearchPhaseController.mergeKnnResults(getRequest(), dfsSearchResults);
 
         return new DfsQueryPhase(
             dfsSearchResults,
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
index 7dc1ddf0bba0..087957368bb1 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
@@ -46,7 +46,6 @@
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.search.suggest.Suggest.Suggestion;
 import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
-import org.elasticsearch.search.vectors.KnnSearchBuilder;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -128,26 +127,34 @@ public static AggregatedDfs aggregateDfs(Collection<DfsSearchResult> results) {
         return new AggregatedDfs(termStatistics, fieldStatistics, aggMaxDoc);
     }
 
-    public static DfsKnnResults mergeKnnResults(SearchRequest request, List<DfsSearchResult> dfsSearchResults) {
+    public static List<DfsKnnResults> mergeKnnResults(SearchRequest request, List<DfsSearchResult> dfsSearchResults) {
         if (request.hasKnnSearch() == false) {
             return null;
         }
 
-        List<TopDocs> topDocs = new ArrayList<>();
+        List<List<TopDocs>> topDocsLists = new ArrayList<>(request.source().knnSearch().size());
+        for (int i = 0; i < request.source().knnSearch().size(); i++) {
+            topDocsLists.add(new ArrayList<>());
+        }
+
         for (DfsSearchResult dfsSearchResult : dfsSearchResults) {
             if (dfsSearchResult.knnResults() != null) {
-                ScoreDoc[] scoreDocs = dfsSearchResult.knnResults().scoreDocs();
-                TotalHits totalHits = new TotalHits(scoreDocs.length, Relation.EQUAL_TO);
-
-                TopDocs shardTopDocs = new TopDocs(totalHits, scoreDocs);
-                setShardIndex(shardTopDocs, dfsSearchResult.getShardIndex());
-                topDocs.add(shardTopDocs);
+                for (int i = 0; i < dfsSearchResult.knnResults().size(); i++) {
+                    DfsKnnResults knnResults = dfsSearchResult.knnResults().get(i);
+                    ScoreDoc[] scoreDocs = knnResults.scoreDocs();
+                    TotalHits totalHits = new TotalHits(scoreDocs.length, Relation.EQUAL_TO);
+                    TopDocs shardTopDocs = new TopDocs(totalHits, scoreDocs);
+                    setShardIndex(shardTopDocs, dfsSearchResult.getShardIndex());
+                    topDocsLists.get(i).add(shardTopDocs);
+                }
             }
         }
-
-        KnnSearchBuilder knnSearch = request.source().knnSearch();
-        TopDocs mergedTopDocs = TopDocs.merge(knnSearch.k(), topDocs.toArray(new TopDocs[0]));
-        return new DfsKnnResults(mergedTopDocs.scoreDocs);
+        List<DfsKnnResults> mergedResults = new ArrayList<>(request.source().knnSearch().size());
+        for (int i = 0; i < request.source().knnSearch().size(); i++) {
+            TopDocs mergedTopDocs = TopDocs.merge(request.source().knnSearch().get(i).k(), topDocsLists.get(i).toArray(new TopDocs[0]));
+            mergedResults.add(new DfsKnnResults(mergedTopDocs.scoreDocs));
+        }
+        return mergedResults;
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index 6eb1b8adc7ad..412f90db7908 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.action.search;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
@@ -238,7 +239,7 @@ public SearchRequest(StreamInput in) throws IOException {
         preference = in.readOptionalString();
         scroll = in.readOptionalWriteable(Scroll::new);
         source = in.readOptionalWriteable(SearchSourceBuilder::new);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             // types no longer relevant so ignore
             String[] types = in.readStringArray();
             if (types.length > 0) {
@@ -262,16 +263,16 @@ public SearchRequest(StreamInput in) throws IOException {
             finalReduce = true;
         }
         ccsMinimizeRoundtrips = in.readBoolean();
-        if (in.getVersion().onOrAfter(Version.V_7_12_0) && in.readBoolean()) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_12_0) && in.readBoolean()) {
             minCompatibleShardNode = Version.readVersion(in);
         } else {
             minCompatibleShardNode = null;
         }
-        if (in.getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             waitForCheckpoints = in.readMap(StreamInput::readString, StreamInput::readLongArray);
             waitForCheckpointsTimeout = in.readTimeValue();
         }
-        if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             forceSyntheticSource = in.readBoolean();
         } else {
             forceSyntheticSource = false;
@@ -287,7 +288,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(preference);
         out.writeOptionalWriteable(scroll);
         out.writeOptionalWriteable(source);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             // types not supported so send an empty array to previous versions
             out.writeStringArray(Strings.EMPTY_ARRAY);
         }
@@ -303,27 +304,27 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(finalReduce);
         }
         out.writeBoolean(ccsMinimizeRoundtrips);
-        if (out.getVersion().onOrAfter(Version.V_7_12_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_12_0)) {
             out.writeBoolean(minCompatibleShardNode != null);
             if (minCompatibleShardNode != null) {
                 Version.writeVersion(minCompatibleShardNode, out);
             }
         }
-        Version waitForCheckpointsVersion = Version.V_7_16_0;
-        if (out.getVersion().onOrAfter(waitForCheckpointsVersion)) {
+        TransportVersion waitForCheckpointsVersion = TransportVersion.V_7_16_0;
+        if (out.getTransportVersion().onOrAfter(waitForCheckpointsVersion)) {
             out.writeMap(waitForCheckpoints, StreamOutput::writeString, StreamOutput::writeLongArray);
             out.writeTimeValue(waitForCheckpointsTimeout);
         } else if (waitForCheckpoints.isEmpty() == false) {
             throw new IllegalArgumentException(
-                "Remote node version ["
-                    + out.getVersion()
+                "Remote transport version ["
+                    + out.getTransportVersion()
                     + " incompatible with "
-                    + "wait_for_checkpoints. All nodes must be version ["
+                    + "wait_for_checkpoints. All nodes must use transport version ["
                     + waitForCheckpointsVersion
                     + "] or greater."
             );
         }
-        if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             out.writeBoolean(forceSyntheticSource);
         } else {
             if (forceSyntheticSource) {
@@ -743,7 +744,7 @@ public boolean isSuggestOnly() {
      * @return true if the request contains kNN search
      */
     public boolean hasKnnSearch() {
-        return source != null && source.knnSearch() != null;
+        return source != null && source.knnSearch().isEmpty() == false;
     }
 
     public int resolveTrackTotalHitsUpTo() {
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index 79a2d0d811ec..f40e8ccb84ec 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -177,7 +177,7 @@ public SearchRequestBuilder setPostFilter(QueryBuilder postFilter) {
      * Defines a kNN search. If a query is also provided, the kNN hits
      * are combined with the query hits.
      */
-    public SearchRequestBuilder setKnnSearch(KnnSearchBuilder knnSearch) {
+    public SearchRequestBuilder setKnnSearch(List<KnnSearchBuilder> knnSearch) {
         sourceBuilder().knnSearch(knnSearch);
         return this;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
index bfef22dd12ec..5f42c1543da8 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.action.search;
 
 import org.apache.lucene.search.TotalHits;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -84,7 +84,7 @@ public SearchResponse(StreamInput in) throws IOException {
         scrollId = in.readOptionalString();
         tookInMillis = in.readVLong();
         skippedShards = in.readVInt();
-        if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             pointInTimeId = in.readOptionalString();
         } else {
             pointInTimeId = null;
@@ -443,7 +443,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(scrollId);
         out.writeVLong(tookInMillis);
         out.writeVInt(skippedShards);
-        if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             out.writeOptionalString(pointInTimeId);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java b/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java
index ed185577c84c..2e3472d08227 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java
@@ -50,7 +50,14 @@ public final class SearchShardIterator implements Comparable<SearchShardIterator
      * @param originalIndices the indices that the search request originally related to (before any rewriting happened)
      */
     public SearchShardIterator(@Nullable String clusterAlias, ShardId shardId, List<ShardRouting> shards, OriginalIndices originalIndices) {
-        this(clusterAlias, shardId, shards.stream().map(ShardRouting::currentNodeId).toList(), originalIndices, null, null);
+        this(
+            clusterAlias,
+            shardId,
+            shards.stream().filter(ShardRouting::isSearchable).map(ShardRouting::currentNodeId).toList(),
+            originalIndices,
+            null,
+            null
+        );
     }
 
     public SearchShardIterator(
@@ -61,6 +68,7 @@ public SearchShardIterator(
         ShardSearchContextId searchContextId,
         TimeValue searchContextKeepAlive
     ) {
+        // TODO ensure all target nodes hold shards with a searchable ShardRoutingRole - at the moment, PIT searches don't check this
         this.shardId = shardId;
         this.targetNodesIterator = new PlainIterator<>(targetNodeIds);
         this.originalIndices = originalIndices;
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
index ccf58bfeadd6..92f30eef2bd8 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.action.search;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionListenerResponseHandler;
@@ -149,7 +150,8 @@ public void sendCanMatch(
         SearchTask task,
         final ActionListener<CanMatchNodeResponse> listener
     ) {
-        if (connection.getVersion().onOrAfter(Version.V_7_16_0) && connection.getNode().getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (connection.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)
+            && connection.getNode().getVersion().onOrAfter(Version.V_7_16_0)) {
             transportService.sendChildRequest(
                 connection,
                 QUERY_CAN_MATCH_NODE_NAME,
diff --git a/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java b/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
index 8da88798e488..5fad89a24118 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
+++ b/server/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
@@ -76,6 +76,7 @@ public ShardSearchFailure(Exception e, @Nullable SearchShardTarget shardTarget)
 
     /**
      * The search shard target the failure occurred on.
+     * @return The shardTarget, may be null
      */
     @Nullable
     public SearchShardTarget shard() {
@@ -95,7 +96,6 @@ public String toString() {
 
     public static ShardSearchFailure readShardSearchFailure(StreamInput in) throws IOException {
         return new ShardSearchFailure(in);
-
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
index 648f8e6c8ee0..a368cecd2677 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
@@ -108,7 +108,7 @@ protected void doExecute(Task task, MultiSearchRequest request, ActionListener<M
     static int defaultMaxConcurrentSearches(final int allocatedProcessors, final ClusterState state) {
         int numDateNodes = state.getNodes().getDataNodes().size();
         // we bound the default concurrency to preserve some search thread pool capacity for other searches
-        final int defaultSearchThreadPoolSize = Math.min(ThreadPool.searchThreadPoolSize(allocatedProcessors), 10);
+        final int defaultSearchThreadPoolSize = Math.min(ThreadPool.searchOrGetThreadPoolSize(allocatedProcessors), 10);
         return Math.max(1, numDateNodes * defaultSearchThreadPoolSize);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/support/ActiveShardCount.java b/server/src/main/java/org/elasticsearch/action/support/ActiveShardCount.java
index 3ed83fb1c4e5..34f0ce1a2fe2 100644
--- a/server/src/main/java/org/elasticsearch/action/support/ActiveShardCount.java
+++ b/server/src/main/java/org/elasticsearch/action/support/ActiveShardCount.java
@@ -153,8 +153,7 @@ public boolean enoughShardsActive(final ClusterState clusterState, final String.
                 waitForActiveShards = SETTING_WAIT_FOR_ACTIVE_SHARDS.get(indexMetadata.getSettings());
             }
             for (int i = 0; i < indexRoutingTable.size(); i++) {
-                IndexShardRoutingTable shardRouting = indexRoutingTable.shard(i);
-                if (waitForActiveShards.enoughShardsActive(shardRouting) == false) {
+                if (waitForActiveShards.enoughShardsActive(indexRoutingTable.shard(i)) == false) {
                     // not enough active shard copies yet
                     return false;
                 }
@@ -171,13 +170,13 @@ public boolean enoughShardsActive(final ClusterState clusterState, final String.
     public boolean enoughShardsActive(final IndexShardRoutingTable shardRoutingTable) {
         final int activeShardCount = shardRoutingTable.activeShards().size();
         if (this == ActiveShardCount.ALL) {
-            // adding 1 for the primary in addition to the total number of replicas,
-            // which gives us the total number of shard copies
-            return activeShardCount == shardRoutingTable.replicaShards().size() + 1;
-        } else if (this == ActiveShardCount.DEFAULT) {
-            return activeShardCount >= 1;
+            return activeShardCount == shardRoutingTable.size();
+        } else if (value == 0) {
+            return true;
+        } else if (value == 1) {
+            return shardRoutingTable.hasSearchShards() ? shardRoutingTable.getActiveSearchShardCount() >= 1 : activeShardCount >= 1;
         } else {
-            return activeShardCount >= value;
+            return shardRoutingTable.getActiveSearchShardCount() >= value;
         }
     }
 
@@ -189,5 +188,4 @@ public String toString() {
             default -> Integer.toString(value);
         };
     }
-
 }
diff --git a/server/src/main/java/org/elasticsearch/action/support/AdapterActionFuture.java b/server/src/main/java/org/elasticsearch/action/support/AdapterActionFuture.java
deleted file mode 100644
index 7f1ada5d6246..000000000000
--- a/server/src/main/java/org/elasticsearch/action/support/AdapterActionFuture.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.action.support;
-
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.common.util.concurrent.BaseFuture;
-import org.elasticsearch.common.util.concurrent.FutureUtils;
-import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException;
-import org.elasticsearch.core.TimeValue;
-
-import java.util.concurrent.TimeUnit;
-
-public abstract class AdapterActionFuture<T, L> extends BaseFuture<T> implements ActionFuture<T>, ActionListener<L> {
-
-    @Override
-    public T actionGet() {
-        try {
-            return FutureUtils.get(this);
-        } catch (ElasticsearchException e) {
-            throw unwrapEsException(e);
-        }
-    }
-
-    @Override
-    public T actionGet(String timeout) {
-        return actionGet(TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".actionGet.timeout"));
-    }
-
-    @Override
-    public T actionGet(long timeoutMillis) {
-        return actionGet(timeoutMillis, TimeUnit.MILLISECONDS);
-    }
-
-    @Override
-    public T actionGet(TimeValue timeout) {
-        return actionGet(timeout.millis(), TimeUnit.MILLISECONDS);
-    }
-
-    @Override
-    public T actionGet(long timeout, TimeUnit unit) {
-        try {
-            return FutureUtils.get(this, timeout, unit);
-        } catch (ElasticsearchException e) {
-            throw unwrapEsException(e);
-        }
-    }
-
-    @Override
-    public void onResponse(L result) {
-        set(convert(result));
-    }
-
-    @Override
-    public void onFailure(Exception e) {
-        setException(e);
-    }
-
-    protected abstract T convert(L listenerResponse);
-
-    private static RuntimeException unwrapEsException(ElasticsearchException esEx) {
-        Throwable root = esEx.unwrapCause();
-        if (root instanceof RuntimeException) {
-            return (RuntimeException) root;
-        }
-        return new UncategorizedExecutionException("Failed execution", root);
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/action/support/ChannelActionListener.java b/server/src/main/java/org/elasticsearch/action/support/ChannelActionListener.java
index 53674d8ac4ba..7e778f8a5fd8 100644
--- a/server/src/main/java/org/elasticsearch/action/support/ChannelActionListener.java
+++ b/server/src/main/java/org/elasticsearch/action/support/ChannelActionListener.java
@@ -35,11 +35,7 @@ public ChannelActionListener(TransportChannel channel, String actionName, Reques
 
     @Override
     public void onResponse(Response response) {
-        try {
-            channel.sendResponse(response);
-        } catch (Exception e) {
-            onFailure(e);
-        }
+        ActionListener.run(this, l -> l.channel.sendResponse(response));
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/support/CountDownActionListener.java b/server/src/main/java/org/elasticsearch/action/support/CountDownActionListener.java
new file mode 100644
index 000000000000..2dac0f4c8cb5
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/support/CountDownActionListener.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.action.support;
+
+import org.elasticsearch.action.ActionListener;
+
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Wraps another listener and adds a counter -- each invocation of this listener will decrement the counter, and when the counter has been
+ * exhausted the final invocation of this listener will delegate to the wrapped listener. Similar to {@link GroupedActionListener}, but for
+ * the cases where tracking individual results is not useful.
+ */
+public final class CountDownActionListener extends ActionListener.Delegating<Void, Void> {
+
+    private final AtomicInteger countDown;
+    private final AtomicReference<Exception> failure = new AtomicReference<>();
+
+    /**
+     * Creates a new listener
+     * @param groupSize the group size
+     * @param delegate the delegate listener
+     */
+    public CountDownActionListener(int groupSize, ActionListener<Void> delegate) {
+        super(Objects.requireNonNull(delegate));
+        if (groupSize <= 0) {
+            assert false : "illegal group size [" + groupSize + "]";
+            throw new IllegalArgumentException("groupSize must be greater than 0 but was " + groupSize);
+        }
+        countDown = new AtomicInteger(groupSize);
+    }
+
+    private boolean countDown() {
+        final var result = countDown.getAndUpdate(current -> Math.max(0, current - 1));
+        assert result > 0;
+        return result == 1;
+    }
+
+    @Override
+    public void onResponse(Void element) {
+        if (countDown()) {
+            if (failure.get() != null) {
+                super.onFailure(failure.get());
+            } else {
+                delegate.onResponse(element);
+            }
+        }
+    }
+
+    @Override
+    public void onFailure(Exception e) {
+        if (failure.compareAndSet(null, e) == false) {
+            failure.accumulateAndGet(e, (current, update) -> {
+                // we have to avoid self-suppression!
+                if (update != current) {
+                    current.addSuppressed(update);
+                }
+                return current;
+            });
+        }
+        if (countDown()) {
+            super.onFailure(failure.get());
+        }
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/action/support/GroupedActionListener.java b/server/src/main/java/org/elasticsearch/action/support/GroupedActionListener.java
index 7a65f100456a..b3d7e74bf88a 100644
--- a/server/src/main/java/org/elasticsearch/action/support/GroupedActionListener.java
+++ b/server/src/main/java/org/elasticsearch/action/support/GroupedActionListener.java
@@ -31,10 +31,10 @@ public final class GroupedActionListener<T> extends ActionListener.Delegating<T,
 
     /**
      * Creates a new listener
-     * @param delegate the delegate listener
      * @param groupSize the group size
+     * @param delegate the delegate listener
      */
-    public GroupedActionListener(ActionListener<Collection<T>> delegate, int groupSize) {
+    public GroupedActionListener(int groupSize, ActionListener<Collection<T>> delegate) {
         super(delegate);
         if (groupSize <= 0) {
             assert false : "illegal group size [" + groupSize + "]";
diff --git a/server/src/main/java/org/elasticsearch/action/support/ListenableActionFuture.java b/server/src/main/java/org/elasticsearch/action/support/ListenableActionFuture.java
index bbac7cba72b4..fcb079ef1fd8 100644
--- a/server/src/main/java/org/elasticsearch/action/support/ListenableActionFuture.java
+++ b/server/src/main/java/org/elasticsearch/action/support/ListenableActionFuture.java
@@ -17,7 +17,7 @@
  * A {@code Future} and {@link ActionListener} against which other {@link ActionListener}s can be registered later, to support
  * fanning-out a result to a dynamic collection of listeners.
  */
-public class ListenableActionFuture<T> extends AdapterActionFuture<T, T> {
+public class ListenableActionFuture<T> extends PlainActionFuture<T> {
 
     private Object listeners;
     private boolean executedListeners = false;
@@ -77,19 +77,10 @@ protected void done(boolean success) {
         }
     }
 
-    @Override
-    protected T convert(T listenerResponse) {
-        return listenerResponse;
-    }
-
     private void executeListener(final ActionListener<T> listener) {
-        try {
-            // we use a timeout of 0 to by pass assertion forbidding to call actionGet() (blocking) on a network thread.
-            // here we know we will never block
-            listener.onResponse(actionGet(0));
-        } catch (Exception e) {
-            listener.onFailure(e);
-        }
+        // we use a timeout of 0 to by pass assertion forbidding to call actionGet() (blocking) on a network thread.
+        // here we know we will never block
+        ActionListener.completeWith(listener, () -> actionGet(0));
     }
 
 }
diff --git a/server/src/main/java/org/elasticsearch/action/support/NodeResponseTracker.java b/server/src/main/java/org/elasticsearch/action/support/NodeResponseTracker.java
deleted file mode 100644
index aafd6166cb36..000000000000
--- a/server/src/main/java/org/elasticsearch/action/support/NodeResponseTracker.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.action.support;
-
-import java.util.Collection;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReferenceArray;
-
-/**
- * This class tracks the intermediate responses that will be used to create aggregated cluster response to a request. It also gives the
- * possibility to discard the intermediate results when asked, for example when the initial request is cancelled, in order to release the
- * resources.
- */
-public class NodeResponseTracker {
-
-    private final AtomicInteger counter = new AtomicInteger();
-    private final int expectedResponsesCount;
-    private volatile AtomicReferenceArray<Object> responses;
-    private volatile Exception causeOfDiscarding;
-
-    public NodeResponseTracker(int size) {
-        this.expectedResponsesCount = size;
-        this.responses = new AtomicReferenceArray<>(size);
-    }
-
-    public NodeResponseTracker(Collection<Object> array) {
-        this.expectedResponsesCount = array.size();
-        this.responses = new AtomicReferenceArray<>(array.toArray());
-    }
-
-    /**
-     * This method discards the results collected so far to free up the resources.
-     * @param cause the discarding, this will be communicated if they try to access the discarded results
-     */
-    public void discardIntermediateResponses(Exception cause) {
-        if (responses != null) {
-            this.causeOfDiscarding = cause;
-            responses = null;
-        }
-    }
-
-    public boolean responsesDiscarded() {
-        return responses == null;
-    }
-
-    /**
-     * This method stores a new node response if the intermediate responses haven't been discarded yet. If the responses are not discarded
-     * the method asserts that this is the first response encountered from this node to protect from miscounting the responses in case of a
-     * double invocation. If the responses have been discarded we accept this risk for simplicity.
-     * @param nodeIndex, the index that represents a single node of the cluster
-     * @param response, a response can be either a NodeResponse or an error
-     * @return true if all the nodes' responses have been received, else false
-     */
-    public boolean trackResponseAndCheckIfLast(int nodeIndex, Object response) {
-        AtomicReferenceArray<Object> responses = this.responses;
-
-        if (responsesDiscarded() == false) {
-            boolean firstEncounter = responses.compareAndSet(nodeIndex, null, response);
-            assert firstEncounter : "a response should be tracked only once";
-        }
-        return counter.incrementAndGet() == getExpectedResponseCount();
-    }
-
-    /**
-     * Returns the tracked response or null if the response hasn't been received yet for a specific index that represents a node of the
-     * cluster.
-     * @throws DiscardedResponsesException if the responses have been discarded
-     */
-    public Object getResponse(int nodeIndex) throws DiscardedResponsesException {
-        AtomicReferenceArray<Object> responses = this.responses;
-        if (responsesDiscarded()) {
-            throw new DiscardedResponsesException(causeOfDiscarding);
-        }
-        return responses.get(nodeIndex);
-    }
-
-    public int getExpectedResponseCount() {
-        return expectedResponsesCount;
-    }
-
-    /**
-     * This exception is thrown when the {@link NodeResponseTracker} is asked to give information about the responses after they have been
-     * discarded.
-     */
-    public static class DiscardedResponsesException extends Exception {
-
-        public DiscardedResponsesException(Exception cause) {
-            super(cause);
-        }
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java b/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java
index 20483b8ff440..4239608ea3d7 100644
--- a/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java
+++ b/server/src/main/java/org/elasticsearch/action/support/PlainActionFuture.java
@@ -8,16 +8,74 @@
 
 package org.elasticsearch.action.support;
 
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.util.concurrent.BaseFuture;
+import org.elasticsearch.common.util.concurrent.FutureUtils;
+import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException;
 import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.core.TimeValue;
 
 import java.util.concurrent.TimeUnit;
 
-public class PlainActionFuture<T> extends AdapterActionFuture<T, T> {
+public class PlainActionFuture<T> extends BaseFuture<T> implements ActionFuture<T>, ActionListener<T> {
 
     public static <T> PlainActionFuture<T> newFuture() {
         return new PlainActionFuture<>();
     }
 
+    @Override
+    public T actionGet() {
+        try {
+            return FutureUtils.get(this);
+        } catch (ElasticsearchException e) {
+            throw unwrapEsException(e);
+        }
+    }
+
+    @Override
+    public T actionGet(String timeout) {
+        return actionGet(TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".actionGet.timeout"));
+    }
+
+    @Override
+    public T actionGet(long timeoutMillis) {
+        return actionGet(timeoutMillis, TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public T actionGet(TimeValue timeout) {
+        return actionGet(timeout.millis(), TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public T actionGet(long timeout, TimeUnit unit) {
+        try {
+            return FutureUtils.get(this, timeout, unit);
+        } catch (ElasticsearchException e) {
+            throw unwrapEsException(e);
+        }
+    }
+
+    @Override
+    public void onResponse(T result) {
+        set(result);
+    }
+
+    @Override
+    public void onFailure(Exception e) {
+        setException(e);
+    }
+
+    private static RuntimeException unwrapEsException(ElasticsearchException esEx) {
+        Throwable root = esEx.unwrapCause();
+        if (root instanceof RuntimeException) {
+            return (RuntimeException) root;
+        }
+        return new UncategorizedExecutionException("Failed execution", root);
+    }
+
     public static <T, E extends Exception> T get(CheckedConsumer<PlainActionFuture<T>, E> e) throws E {
         PlainActionFuture<T> fut = newFuture();
         e.accept(fut);
@@ -29,9 +87,4 @@ public static <T, E extends Exception> T get(CheckedConsumer<PlainActionFuture<T
         e.accept(fut);
         return fut.actionGet(timeout, unit);
     }
-
-    @Override
-    protected T convert(T listenerResponse) {
-        return listenerResponse;
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/support/RefCountingListener.java b/server/src/main/java/org/elasticsearch/action/support/RefCountingListener.java
new file mode 100644
index 000000000000..b9c64768bb6b
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/support/RefCountingListener.java
@@ -0,0 +1,228 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.support;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.core.Releasable;
+
+import java.util.Objects;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+
+/**
+ * A mechanism to complete a listener on the completion of some (dynamic) collection of other actions. Basic usage is as follows:
+ *
+ * <pre>
+ * try (var refs = new RefCountingListener(finalListener)) {
+ *     for (var item : collection) {
+ *         runAsyncAction(item, refs.acquire()); // completes the acquired listener on completion
+ *     }
+ * }
+ * </pre>
+ *
+ * The delegate listener is completed when execution leaves the try-with-resources block and every acquired reference is released. The
+ * {@link RefCountingListener} collects (a bounded number of) exceptions received by its subsidiary listeners, and completes the delegate
+ * listener with an exception if (and only if) any subsidiary listener fails. However, unlike a {@link GroupedActionListener} it leaves it
+ * to the caller to collect the results of successful completions by accumulating them in a data structure of its choice. Also unlike a
+ * {@link GroupedActionListener} there is no need to declare the number of subsidiary listeners up front: listeners can be acquired
+ * dynamically as needed. Finally, you can continue to acquire additional listeners even outside the try-with-resources block, perhaps in a
+ * separate thread, as long as there's at least one listener outstanding:
+ *
+ * <pre>
+ * try (var refs = new RefCountingListener(finalListener)) {
+ *     for (var item : collection) {
+ *         if (condition(item)) {
+ *             runAsyncAction(item, refs.acquire(results::add));
+ *         }
+ *     }
+ *     if (flag) {
+ *         runOneOffAsyncAction(refs.acquire(results::add));
+ *         return;
+ *     }
+ *     for (var item : otherCollection) {
+ *         var itemRef = refs.acquire(); // delays completion while the background action is pending
+ *         executorService.execute(() -> {
+ *             try {
+ *                 if (condition(item)) {
+ *                     runOtherAsyncAction(item, refs.acquire(results::add));
+ *                 }
+ *             } finally {
+ *                 itemRef.onResponse(null);
+ *             }
+ *         });
+ *     }
+ * }
+ * </pre>
+ *
+ * In particular (and also unlike a {@link GroupedActionListener}) this works even if you don't acquire any extra refs at all: in that case,
+ * the delegate listener is completed at the end of the try-with-resources block.
+ */
+public final class RefCountingListener implements Releasable {
+
+    private final ActionListener<Void> delegate;
+    private final RefCountingRunnable refs = new RefCountingRunnable(this::finish);
+
+    private final AtomicReference<Exception> exceptionRef = new AtomicReference<>();
+    private final Semaphore exceptionPermits;
+    private final AtomicInteger droppedExceptionsRef = new AtomicInteger();
+
+    /**
+     * Construct a {@link RefCountingListener} which completes {@code delegate} when all refs are released.
+     * @param delegate The listener to complete when all refs are released. This listener must not throw any exception on completion. If all
+     *                 the acquired listeners completed successfully then so is the delegate. If any of the acquired listeners completed
+     *                 with failure then the delegate is completed with the first exception received, with other exceptions added to its
+     *                 collection of suppressed exceptions.
+     */
+    public RefCountingListener(ActionListener<Void> delegate) {
+        this(10, delegate);
+    }
+
+    /**
+     * Construct a {@link RefCountingListener} which completes {@code delegate} when all refs are released.
+     * @param delegate The listener to complete when all refs are released. This listener must not throw any exception on completion. If all
+     *                 the acquired listeners completed successfully then so is the delegate. If any of the acquired listeners completed
+     *                 with failure then the delegate is completed with the first exception received, with other exceptions added to its
+     *                 collection of suppressed exceptions.
+     * @param maxExceptions The maximum number of exceptions to accumulate on failure.
+     */
+    public RefCountingListener(int maxExceptions, ActionListener<Void> delegate) {
+        if (maxExceptions <= 0) {
+            assert false : maxExceptions;
+            throw new IllegalArgumentException("maxExceptions must be positive");
+        }
+        this.delegate = ActionListener.assertOnce(Objects.requireNonNull(delegate));
+        this.exceptionPermits = new Semaphore(maxExceptions);
+    }
+
+    /**
+     * Release the original reference to this object, which commpletes the delegate {@link ActionListener} if there are no other references.
+     *
+     * It is invalid to call this method more than once. Doing so will trip an assertion if assertions are enabled, but will be ignored
+     * otherwise. This deviates from the contract of {@link java.io.Closeable}.
+     */
+    @Override
+    public void close() {
+        refs.close();
+    }
+
+    private void finish() {
+        try {
+            var exception = exceptionRef.get();
+            if (exception == null) {
+                delegate.onResponse(null);
+            } else {
+                final var droppedExceptions = droppedExceptionsRef.getAndSet(0);
+                if (droppedExceptions > 0) {
+                    exception.addSuppressed(new ElasticsearchException(droppedExceptions + " further exceptions were dropped"));
+                }
+                delegate.onFailure(exception);
+            }
+        } catch (Exception e) {
+            assert false : e;
+            throw e;
+        }
+    }
+
+    /**
+     * Acquire a reference to this object and return a listener which releases it. The delegate {@link ActionListener} is called when all
+     * its references have been released.
+     *
+     * It is invalid to call this method once all references are released. Doing so will trip an assertion if assertions are enabled, and
+     * will throw an {@link IllegalStateException} otherwise.
+     *
+     * It is also invalid to complete the returned listener more than once. Doing so will trip an assertion if assertions are enabled, but
+     * will be ignored otherwise.
+     */
+    public ActionListener<Void> acquire() {
+        return new ActionListener<>() {
+            private final Releasable ref = refs.acquire();
+
+            @Override
+            public void onResponse(Void unused) {
+                ref.close();
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                try (ref) {
+                    addException(e);
+                }
+            }
+
+            @Override
+            public String toString() {
+                return RefCountingListener.this.toString();
+            }
+        };
+    }
+
+    /**
+     * Acquire a reference to this object and return a listener which consumes a response and releases the reference. The delegate {@link
+     * ActionListener} is called when all its references have been released. The consumer must not throw any exception.
+     *
+     * It is invalid to call this method once all references are released. Doing so will trip an assertion if assertions are enabled, and
+     * will throw an {@link IllegalStateException} otherwise.
+     *
+     * It is also invalid to complete the returned listener more than once. Doing so will trip an assertion if assertions are enabled, but
+     * will be ignored otherwise.
+     */
+    public <Response> ActionListener<Response> acquire(Consumer<Response> consumer) {
+        final var ref = refs.acquire();
+        final var consumerRef = new AtomicReference<>(Objects.requireNonNull(consumer));
+        return new ActionListener<>() {
+            @Override
+            public void onResponse(Response response) {
+                try (ref) {
+                    var acquiredConsumer = consumerRef.getAndSet(null);
+                    if (acquiredConsumer == null) {
+                        assert false : "already closed";
+                    } else {
+                        acquiredConsumer.accept(response);
+                    }
+                } catch (Exception e) {
+                    assert false : e;
+                    throw e;
+                }
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                try (ref) {
+                    var acquiredConsumer = consumerRef.getAndSet(null);
+                    assert acquiredConsumer != null : "already closed";
+                    addException(e);
+                }
+            }
+
+            @Override
+            public String toString() {
+                return RefCountingListener.this + "[" + consumerRef.get() + "]";
+            }
+        };
+    }
+
+    private void addException(Exception e) {
+        if (exceptionPermits.tryAcquire()) {
+            final var firstException = exceptionRef.compareAndExchange(null, e);
+            if (firstException != null && firstException != e) {
+                firstException.addSuppressed(e);
+            }
+        } else {
+            droppedExceptionsRef.incrementAndGet();
+        }
+    }
+
+    @Override
+    public String toString() {
+        return "refCounting[" + delegate + "]";
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/support/RefCountingRunnable.java b/server/src/main/java/org/elasticsearch/action/support/RefCountingRunnable.java
new file mode 100644
index 000000000000..c3a5bb398972
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/support/RefCountingRunnable.java
@@ -0,0 +1,149 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.support;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.util.concurrent.CountDown;
+import org.elasticsearch.core.AbstractRefCounted;
+import org.elasticsearch.core.RefCounted;
+import org.elasticsearch.core.Releasable;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * A mechanism to trigger an action on the completion of some (dynamic) collection of other actions. Basic usage is as follows:
+ *
+ * <pre>
+ * try (var refs = new RefCountingRunnable(finalRunnable)) {
+ *     for (var item : collection) {
+ *         runAsyncAction(item, refs.acquire()); // releases the acquired ref on completion
+ *     }
+ * }
+ * </pre>
+ *
+ * The delegate action is completed when execution leaves the try-with-resources block and every acquired reference is released. Unlike a
+ * {@link CountDown} there is no need to declare the number of subsidiary actions up front (refs can be acquired dynamically as needed) nor
+ * does the caller need to check for completion each time a reference is released. Moreover even outside the try-with-resources block you
+ * can continue to acquire additional references, even in a separate thread, as long as there's at least one reference outstanding:
+ *
+ * <pre>
+ * try (var refs = new RefCountingRunnable(finalRunnable)) {
+ *     for (var item : collection) {
+ *         if (condition(item)) {
+ *             runAsyncAction(item, refs.acquire());
+ *         }
+ *     }
+ *     if (flag) {
+ *         runOneOffAsyncAction(refs.acquire());
+ *         return;
+ *     }
+ *     for (var item : otherCollection) {
+ *         var itemRef = refs.acquire(); // delays completion while the background action is pending
+ *         executorService.execute(() -> {
+ *             try (var ignored = itemRef) {
+ *                 if (condition(item)) {
+ *                     runOtherAsyncAction(item, refs.acquire());
+ *                 }
+ *             }
+ *         });
+ *     }
+ * }
+ * </pre>
+ *
+ * In particular (and also unlike a {@link CountDown}) this works even if you don't acquire any extra refs at all: in that case, the
+ * delegate action executes at the end of the try-with-resources block.
+ */
+public final class RefCountingRunnable implements Releasable {
+
+    private static final Logger logger = LogManager.getLogger(RefCountingRunnable.class);
+    static final String ALREADY_CLOSED_MESSAGE = "already closed, cannot acquire or release any further refs";
+
+    private final RefCounted refCounted;
+    private final AtomicBoolean originalRefReleased = new AtomicBoolean();
+
+    private class AcquiredRef implements Releasable {
+        private final AtomicBoolean released = new AtomicBoolean();
+
+        @Override
+        public void close() {
+            releaseRef(released);
+        }
+
+        @Override
+        public String toString() {
+            return RefCountingRunnable.this.toString();
+        }
+    }
+
+    /**
+     * Construct a {@link RefCountingRunnable} which executes {@code delegate} when all refs are released.
+     * @param delegate The action to execute when all refs are released. This action must not throw any exception.
+     */
+    public RefCountingRunnable(Runnable delegate) {
+        this.refCounted = AbstractRefCounted.of(delegate);
+    }
+
+    /**
+     * Acquire a reference to this object and return an action which releases it. The delegate {@link Runnable} is called when all its
+     * references have been released.
+     *
+     * It is invalid to call this method once all references are released. Doing so will trip an assertion if assertions are enabled, and
+     * will throw an {@link IllegalStateException} otherwise.
+     *
+     * It is also invalid to release the acquired resource more than once. Doing so will trip an assertion if assertions are enabled, but
+     * will be ignored otherwise. This deviates from the contract of {@link java.io.Closeable}.
+     */
+    public Releasable acquire() {
+        if (refCounted.tryIncRef()) {
+            return new AcquiredRef();
+        }
+        assert false : ALREADY_CLOSED_MESSAGE;
+        throw new IllegalStateException(ALREADY_CLOSED_MESSAGE);
+    }
+
+    /**
+     * Acquire a reference to this object and return a listener which releases it when notified. The delegate {@link Runnable} is called
+     * when all its references have been released.
+     */
+    public ActionListener<Void> acquireListener() {
+        return ActionListener.releasing(acquire());
+    }
+
+    /**
+     * Release the original reference to this object, which executes the delegate {@link Runnable} if there are no other references.
+     *
+     * It is invalid to call this method more than once. Doing so will trip an assertion if assertions are enabled, but will be ignored
+     * otherwise. This deviates from the contract of {@link java.io.Closeable}.
+     */
+    @Override
+    public void close() {
+        releaseRef(originalRefReleased);
+    }
+
+    private void releaseRef(AtomicBoolean released) {
+        if (released.compareAndSet(false, true)) {
+            try {
+                refCounted.decRef();
+            } catch (Exception e) {
+                logger.error("exception in delegate", e);
+                assert false : e;
+            }
+        } else {
+            assert false : "already closed";
+        }
+    }
+
+    @Override
+    public String toString() {
+        return refCounted.toString();
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/action/support/ThreadedActionListener.java b/server/src/main/java/org/elasticsearch/action/support/ThreadedActionListener.java
index f85a365a1500..6e575a7f7dd9 100644
--- a/server/src/main/java/org/elasticsearch/action/support/ThreadedActionListener.java
+++ b/server/src/main/java/org/elasticsearch/action/support/ThreadedActionListener.java
@@ -8,39 +8,37 @@
 
 package org.elasticsearch.action.support;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.threadpool.ThreadPool;
+
+import java.util.concurrent.ExecutorService;
 
 /**
- * An action listener that wraps another action listener and threading its execution.
+ * An action listener that wraps another action listener and dispatches its completion to an executor.
  */
 public final class ThreadedActionListener<Response> extends ActionListener.Delegating<Response, Response> {
 
-    private final Logger logger;
-    private final ThreadPool threadPool;
-    private final String executor;
+    private static final Logger logger = LogManager.getLogger(ThreadedActionListener.class);
+
+    private final ExecutorService executor;
     private final boolean forceExecution;
 
-    public ThreadedActionListener(
-        Logger logger,
-        ThreadPool threadPool,
-        String executor,
-        ActionListener<Response> listener,
-        boolean forceExecution
-    ) {
+    public ThreadedActionListener(ExecutorService executor, ActionListener<Response> listener) {
+        this(executor, false, listener);
+    }
+
+    public ThreadedActionListener(ExecutorService executor, boolean forceExecution, ActionListener<Response> listener) {
         super(listener);
-        this.logger = logger;
-        this.threadPool = threadPool;
-        this.executor = executor;
         this.forceExecution = forceExecution;
+        this.executor = executor;
     }
 
     @Override
     public void onResponse(final Response response) {
-        threadPool.executor(executor).execute(new ActionRunnable<>(delegate) {
+        executor.execute(new ActionRunnable<>(delegate) {
             @Override
             public boolean isForceExecution() {
                 return forceExecution;
@@ -60,7 +58,7 @@ public String toString() {
 
     @Override
     public void onFailure(final Exception e) {
-        threadPool.executor(executor).execute(new AbstractRunnable() {
+        executor.execute(new AbstractRunnable() {
             @Override
             public boolean isForceExecution() {
                 return forceExecution;
@@ -84,8 +82,8 @@ public void onRejection(Exception e2) {
 
             @Override
             public void onFailure(Exception e) {
+                logger.error(() -> "failed to execute failure callback on [" + ThreadedActionListener.this + "]", e);
                 assert false : e;
-                logger.error(() -> "failed to execute failure callback on [" + delegate + "]", e);
             }
 
             @Override
diff --git a/server/src/main/java/org/elasticsearch/action/support/TransportAction.java b/server/src/main/java/org/elasticsearch/action/support/TransportAction.java
index de3911752964..4d3b9b0c15ff 100644
--- a/server/src/main/java/org/elasticsearch/action/support/TransportAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/TransportAction.java
@@ -111,11 +111,7 @@ private TaskResultStoringActionListener(TaskManager taskManager, Task task, Acti
 
         @Override
         public void onResponse(Response response) {
-            try {
-                taskManager.storeResult(task, response, delegate);
-            } catch (Exception e) {
-                delegate.onFailure(e);
-            }
+            ActionListener.run(delegate, l -> taskManager.storeResult(task, response, l));
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/ChunkedBroadcastResponse.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/ChunkedBroadcastResponse.java
new file mode 100644
index 000000000000..6564e30799ce
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/ChunkedBroadcastResponse.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.action.support.broadcast;
+
+import org.elasticsearch.action.support.DefaultShardOperationFailedException;
+import org.elasticsearch.common.collect.Iterators;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
+import org.elasticsearch.rest.action.RestActions;
+import org.elasticsearch.xcontent.ToXContent;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+
+public abstract class ChunkedBroadcastResponse extends BaseBroadcastResponse implements ChunkedToXContentObject {
+    public ChunkedBroadcastResponse(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    public ChunkedBroadcastResponse(
+        int totalShards,
+        int successfulShards,
+        int failedShards,
+        List<DefaultShardOperationFailedException> shardFailures
+    ) {
+        super(totalShards, successfulShards, failedShards, shardFailures);
+    }
+
+    @Override
+    public final Iterator<ToXContent> toXContentChunked(ToXContent.Params params) {
+        return Iterators.concat(Iterators.single((b, p) -> {
+            b.startObject();
+            RestActions.buildBroadcastShardsHeader(b, p, this);
+            return b;
+        }), customXContentChunks(params), Iterators.single((builder, p) -> builder.endObject()));
+    }
+
+    protected abstract Iterator<ToXContent> customXContentChunks(ToXContent.Params params);
+}
diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java
index 7e0b636d0056..6132b61a304c 100644
--- a/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java
@@ -137,11 +137,7 @@ protected AsyncBroadcastAction(Task task, Request request, ActionListener<Respon
         public void start() {
             if (shardsIts.size() == 0) {
                 // no shards
-                try {
-                    listener.onResponse(newResponse(request, new AtomicReferenceArray<ShardResponse>(0), clusterState));
-                } catch (Exception e) {
-                    listener.onFailure(e);
-                }
+                ActionListener.completeWith(listener, () -> newResponse(request, new AtomicReferenceArray<ShardResponse>(0), clusterState));
                 return;
             }
             // count the local operations, and perform the non local ones
@@ -247,11 +243,7 @@ protected AtomicReferenceArray<Object> shardsResponses() {
         }
 
         protected void finishHim() {
-            try {
-                listener.onResponse(newResponse(request, shardsResponses, clusterState));
-            } catch (Exception e) {
-                listener.onFailure(e);
-            }
+            ActionListener.completeWith(listener, () -> newResponse(request, shardsResponses, clusterState));
         }
 
         void setFailure(ShardIterator shardIt, int shardIndex, Exception e) {
diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
index 18e8b8b0f07c..f057d3e671a4 100644
--- a/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
@@ -8,14 +8,21 @@
 
 package org.elasticsearch.action.support.broadcast.node;
 
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionListenerResponseHandler;
+import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.ChannelActionListener;
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.action.support.NodeResponseTracker;
+import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.action.support.TransportActions;
 import org.elasticsearch.action.support.broadcast.BaseBroadcastResponse;
 import org.elasticsearch.action.support.broadcast.BroadcastRequest;
@@ -23,7 +30,6 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
@@ -31,18 +37,16 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.common.util.concurrent.ListenableFuture;
+import org.elasticsearch.common.util.concurrent.RunOnce;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
-import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.transport.TransportChannel;
-import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestHandler;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportResponse;
-import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
@@ -51,7 +55,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Consumer;
 
 import static org.elasticsearch.core.Strings.format;
 
@@ -71,6 +74,8 @@ public abstract class TransportBroadcastByNodeAction<
     Response extends BaseBroadcastResponse,
     ShardOperationResult extends Writeable> extends HandledTransportAction<Request, Response> {
 
+    private static final Logger logger = LogManager.getLogger(TransportBroadcastByNodeAction.class);
+
     private final ClusterService clusterService;
     private final TransportService transportService;
     private final IndexNameExpressionResolver indexNameExpressionResolver;
@@ -117,48 +122,6 @@ public TransportBroadcastByNodeAction(
         );
     }
 
-    private Response newResponse(
-        Request request,
-        NodeResponseTracker nodeResponseTracker,
-        int unavailableShardCount,
-        Map<String, List<ShardRouting>> nodes,
-        ClusterState clusterState
-    ) throws NodeResponseTracker.DiscardedResponsesException {
-        int totalShards = 0;
-        int successfulShards = 0;
-        List<ShardOperationResult> broadcastByNodeResponses = new ArrayList<>();
-        List<DefaultShardOperationFailedException> exceptions = new ArrayList<>();
-        for (int i = 0; i < nodeResponseTracker.getExpectedResponseCount(); i++) {
-            Object response = nodeResponseTracker.getResponse(i);
-            if (response instanceof FailedNodeException exception) {
-                totalShards += nodes.get(exception.nodeId()).size();
-                for (ShardRouting shard : nodes.get(exception.nodeId())) {
-                    exceptions.add(new DefaultShardOperationFailedException(shard.getIndexName(), shard.getId(), exception));
-                }
-            } else {
-                @SuppressWarnings("unchecked")
-                NodeResponse nodeResponse = (NodeResponse) response;
-                broadcastByNodeResponses.addAll(nodeResponse.results);
-                totalShards += nodeResponse.getTotalShards();
-                successfulShards += nodeResponse.getSuccessfulShards();
-                for (BroadcastShardOperationFailedException throwable : nodeResponse.getExceptions()) {
-                    if (TransportActions.isShardNotAvailableException(throwable) == false) {
-                        exceptions.add(
-                            new DefaultShardOperationFailedException(
-                                throwable.getShardId().getIndexName(),
-                                throwable.getShardId().getId(),
-                                throwable
-                            )
-                        );
-                    }
-                }
-            }
-        }
-        totalShards += unavailableShardCount;
-        int failedShards = exceptions.size();
-        return newResponse(request, totalShards, successfulShards, failedShards, broadcastByNodeResponses, exceptions, clusterState);
-    }
-
     /**
      * Deserialize a shard-level result from an input stream
      *
@@ -167,27 +130,31 @@ private Response newResponse(
      */
     protected abstract ShardOperationResult readShardResult(StreamInput in) throws IOException;
 
+    public interface ResponseFactory<Response, ShardOperationResult> {
+        /**
+         * Creates a new response to the underlying request.
+         *
+         * @param totalShards      the total number of shards considered for execution of the operation
+         * @param successfulShards the total number of shards for which execution of the operation was successful
+         * @param failedShards     the total number of shards for which execution of the operation failed
+         * @param results          the per-node aggregated shard-level results
+         * @param shardFailures    the exceptions corresponding to shard operation failures
+         * @return the response
+         */
+        Response newResponse(
+            int totalShards,
+            int successfulShards,
+            int failedShards,
+            List<ShardOperationResult> results,
+            List<DefaultShardOperationFailedException> shardFailures
+        );
+    }
+
     /**
-     * Creates a new response to the underlying request.
-     *
-     * @param request          the underlying request
-     * @param totalShards      the total number of shards considered for execution of the operation
-     * @param successfulShards the total number of shards for which execution of the operation was successful
-     * @param failedShards     the total number of shards for which execution of the operation failed
-     * @param results          the per-node aggregated shard-level results
-     * @param shardFailures    the exceptions corresponding to shard operation failures
-     * @param clusterState     the cluster state
-     * @return the response
+     * Create a response factory based on the requst and the cluster state captured at the time the request was handled. Implementations
+     * must avoid capturing the full cluster state if possible.
      */
-    protected abstract Response newResponse(
-        Request request,
-        int totalShards,
-        int successfulShards,
-        int failedShards,
-        List<ShardOperationResult> results,
-        List<DefaultShardOperationFailedException> shardFailures,
-        ClusterState clusterState
-    );
+    protected abstract ResponseFactory<Response, ShardOperationResult> getResponseFactory(Request request, ClusterState clusterState);
 
     /**
      * Deserialize a request from an input stream
@@ -255,341 +222,312 @@ protected String[] resolveConcreteIndexNames(ClusterState clusterState, Request
 
     @Override
     protected void doExecute(Task task, Request request, ActionListener<Response> listener) {
-        new AsyncAction(task, request, listener).start();
-    }
-
-    protected class AsyncAction implements CancellableTask.CancellationListener {
-        private final Task task;
-        private final Request request;
-        private final ActionListener<Response> listener;
-        private final ClusterState clusterState;
-        private final DiscoveryNodes nodes;
-        private final Map<String, List<ShardRouting>> nodeIds;
-        private final int unavailableShardCount;
-        private final NodeResponseTracker nodeResponseTracker;
-
-        protected AsyncAction(Task task, Request request, ActionListener<Response> listener) {
-            this.task = task;
-            this.request = request;
-            this.listener = listener;
-
-            clusterState = clusterService.state();
-            nodes = clusterState.nodes();
-
-            ClusterBlockException globalBlockException = checkGlobalBlock(clusterState, request);
-            if (globalBlockException != null) {
-                throw globalBlockException;
-            }
+        final var clusterState = clusterService.state();
 
-            String[] concreteIndices = resolveConcreteIndexNames(clusterState, request);
-            ClusterBlockException requestBlockException = checkRequestBlock(clusterState, request, concreteIndices);
-            if (requestBlockException != null) {
-                throw requestBlockException;
-            }
+        final var globalBlockException = checkGlobalBlock(clusterState, request);
+        if (globalBlockException != null) {
+            throw globalBlockException;
+        }
 
-            if (logger.isTraceEnabled()) {
-                logger.trace("resolving shards for [{}] based on cluster state version [{}]", actionName, clusterState.version());
-            }
-            ShardsIterator shardIt = shards(clusterState, request, concreteIndices);
-            nodeIds = new HashMap<>();
-
-            int unavailableShardCount = 0;
-            for (ShardRouting shard : shardIt) {
-                // send a request to the shard only if it is assigned to a node that is in the local node's cluster state
-                // a scenario in which a shard can be assigned but to a node that is not in the local node's cluster state
-                // is when the shard is assigned to the master node, the local node has detected the master as failed
-                // and a new master has not yet been elected; in this situation the local node will have removed the
-                // master node from the local cluster state, but the shards assigned to the master will still be in the
-                // routing table as such
-                if (shard.assignedToNode() && nodes.get(shard.currentNodeId()) != null) {
-                    String nodeId = shard.currentNodeId();
-                    if (nodeIds.containsKey(nodeId) == false) {
-                        nodeIds.put(nodeId, new ArrayList<>());
-                    }
-                    nodeIds.get(nodeId).add(shard);
-                } else {
-                    unavailableShardCount++;
-                }
+        final var concreteIndices = resolveConcreteIndexNames(clusterState, request);
+        final var requestBlockException = checkRequestBlock(clusterState, request, concreteIndices);
+        if (requestBlockException != null) {
+            throw requestBlockException;
+        }
 
+        logger.trace(() -> format("resolving shards for [%s] based on cluster state version [%s]", actionName, clusterState.version()));
+        final ShardsIterator shardIt = shards(clusterState, request, concreteIndices);
+        final Map<String, List<ShardRouting>> shardsByNodeId = new HashMap<>();
+
+        final var nodes = clusterState.nodes();
+        int unavailableShardCount = 0;
+        int availableShardCount = 0;
+        for (final var shard : shardIt) {
+            // send a request to the shard only if it is assigned to a node that is in the local node's cluster state
+            // a scenario in which a shard can be assigned but to a node that is not in the local node's cluster state
+            // is when the shard is assigned to the master node, the local node has detected the master as failed
+            // and a new master has not yet been elected; in this situation the local node will have removed the
+            // master node from the local cluster state, but the shards assigned to the master will still be in the
+            // routing table as such
+            final var nodeId = shard.currentNodeId();
+            if (nodeId != null && nodes.get(nodeId) != null) {
+                shardsByNodeId.computeIfAbsent(nodeId, n -> new ArrayList<>()).add(shard);
+                availableShardCount += 1;
+            } else {
+                unavailableShardCount++;
             }
-            this.unavailableShardCount = unavailableShardCount;
-            nodeResponseTracker = new NodeResponseTracker(nodeIds.size());
         }
 
-        public void start() {
+        executeAsCoordinatingNode(
+            task,
+            request,
+            shardsByNodeId,
+            unavailableShardCount,
+            availableShardCount,
+            nodes,
+            getResponseFactory(request, clusterState),
+            listener
+        );
+    }
+
+    private void executeAsCoordinatingNode(
+        Task task,
+        Request request,
+        Map<String, List<ShardRouting>> shardsByNodeId,
+        int unavailableShardCount,
+        int availableShardCount,
+        DiscoveryNodes nodes,
+        ResponseFactory<Response, ShardOperationResult> responseFactory,
+        ActionListener<Response> listener
+    ) {
+        final var mutex = new Object();
+        final var shardResponses = new ArrayList<ShardOperationResult>(availableShardCount);
+        final var exceptions = new ArrayList<DefaultShardOperationFailedException>(0);
+        final var totalShards = new AtomicInteger(unavailableShardCount);
+        final var successfulShards = new AtomicInteger(0);
+
+        final var resultListener = new ListenableFuture<Response>();
+        final var resultListenerCompleter = new RunOnce(() -> {
             if (task instanceof CancellableTask cancellableTask) {
-                cancellableTask.addListener(this);
-            }
-            if (nodeIds.size() == 0) {
-                try {
-                    onCompletion();
-                } catch (Exception e) {
-                    listener.onFailure(e);
-                }
-            } else {
-                int nodeIndex = -1;
-                for (Map.Entry<String, List<ShardRouting>> entry : nodeIds.entrySet()) {
-                    nodeIndex++;
-                    DiscoveryNode node = nodes.get(entry.getKey());
-                    sendNodeRequest(node, entry.getValue(), nodeIndex);
+                if (cancellableTask.notifyIfCancelled(resultListener)) {
+                    return;
                 }
             }
+            // ref releases all happen-before here so no need to be synchronized
+            resultListener.onResponse(
+                responseFactory.newResponse(totalShards.get(), successfulShards.get(), exceptions.size(), shardResponses, exceptions)
+            );
+        });
+
+        final var nodeFailureListeners = new ListenableFuture<NodeResponse>();
+        if (task instanceof CancellableTask cancellableTask) {
+            cancellableTask.addListener(() -> {
+                assert cancellableTask.isCancelled();
+                resultListenerCompleter.run();
+                cancellableTask.notifyIfCancelled(nodeFailureListeners);
+            });
         }
 
-        private void sendNodeRequest(final DiscoveryNode node, List<ShardRouting> shards, final int nodeIndex) {
-            try {
-                final NodeRequest nodeRequest = new NodeRequest(node.getId(), request, shards);
-                if (task != null) {
-                    nodeRequest.setParentTask(clusterService.localNode().getId(), task.getId());
-                }
+        final var transportRequestOptions = TransportRequestOptions.timeout(request.timeout());
 
-                final TransportRequestOptions transportRequestOptions = TransportRequestOptions.timeout(request.timeout());
+        try (var refs = new RefCountingRunnable(() -> {
+            resultListener.addListener(listener);
+            resultListenerCompleter.run();
+        })) {
+            for (final var entry : shardsByNodeId.entrySet()) {
+                final var node = nodes.get(entry.getKey());
+                final var shards = entry.getValue();
 
-                transportService.sendRequest(
-                    node,
-                    transportNodeBroadcastAction,
-                    nodeRequest,
-                    transportRequestOptions,
-                    new TransportResponseHandler<NodeResponse>() {
-                        @Override
-                        public NodeResponse read(StreamInput in) throws IOException {
-                            return new NodeResponse(in);
+                final ActionListener<NodeResponse> nodeResponseListener = ActionListener.notifyOnce(new ActionListener<NodeResponse>() {
+                    @Override
+                    public void onResponse(NodeResponse nodeResponse) {
+                        synchronized (mutex) {
+                            shardResponses.addAll(nodeResponse.getResults());
                         }
-
-                        @Override
-                        public void handleResponse(NodeResponse response) {
-                            onNodeResponse(node, nodeIndex, response);
+                        totalShards.addAndGet(nodeResponse.getTotalShards());
+                        successfulShards.addAndGet(nodeResponse.getSuccessfulShards());
+
+                        for (BroadcastShardOperationFailedException exception : nodeResponse.getExceptions()) {
+                            if (TransportActions.isShardNotAvailableException(exception)) {
+                                assert node.getVersion().before(Version.V_8_7_0) : node; // we stopped sending these ignored exceptions
+                            } else {
+                                synchronized (mutex) {
+                                    exceptions.add(
+                                        new DefaultShardOperationFailedException(
+                                            exception.getShardId().getIndexName(),
+                                            exception.getShardId().getId(),
+                                            exception
+                                        )
+                                    );
+                                }
+                            }
                         }
+                    }
 
-                        @Override
-                        public void handleException(TransportException exp) {
-                            onNodeFailure(node, nodeIndex, exp);
+                    @Override
+                    public void onFailure(Exception e) {
+                        if (task instanceof CancellableTask cancellableTask && cancellableTask.isCancelled()) {
+                            return;
                         }
-                    }
-                );
-            } catch (Exception e) {
-                onNodeFailure(node, nodeIndex, e);
-            }
-        }
 
-        protected void onNodeResponse(DiscoveryNode node, int nodeIndex, NodeResponse response) {
-            if (logger.isTraceEnabled()) {
-                logger.trace("received response for [{}] from node [{}]", actionName, node.getId());
-            }
+                        logger.debug(() -> format("failed to execute [%s] on node [%s]", actionName, node), e);
 
-            if (nodeResponseTracker.trackResponseAndCheckIfLast(nodeIndex, response)) {
-                onCompletion();
-            }
-        }
+                        final var failedNodeException = new FailedNodeException(node.getId(), "Failed node [" + node.getId() + "]", e);
+                        synchronized (mutex) {
+                            for (ShardRouting shard : shards) {
+                                exceptions.add(
+                                    new DefaultShardOperationFailedException(shard.getIndexName(), shard.getId(), failedNodeException)
+                                );
+                            }
+                        }
 
-        protected void onNodeFailure(DiscoveryNode node, int nodeIndex, Throwable t) {
-            String nodeId = node.getId();
-            logger.debug(() -> format("failed to execute [%s] on node [%s]", actionName, nodeId), t);
-            if (nodeResponseTracker.trackResponseAndCheckIfLast(
-                nodeIndex,
-                new FailedNodeException(nodeId, "Failed node [" + nodeId + "]", t)
-            )) {
-                onCompletion();
-            }
-        }
+                        totalShards.addAndGet(shards.size());
+                    }
 
-        protected void onCompletion() {
-            if ((task instanceof CancellableTask t) && t.notifyIfCancelled(listener)) {
-                return;
-            }
+                    @Override
+                    public String toString() {
+                        return "[" + actionName + "][" + node.descriptionWithoutAttributes() + "]";
+                    }
+                });
 
-            Response response = null;
-            try {
-                response = newResponse(request, nodeResponseTracker, unavailableShardCount, nodeIds, clusterState);
-            } catch (NodeResponseTracker.DiscardedResponsesException e) {
-                // We propagate the reason that the results, in this case the task cancellation, in case the listener needs to take
-                // follow-up actions
-                listener.onFailure((Exception) e.getCause());
-            } catch (Exception e) {
-                logger.debug("failed to combine responses from nodes", e);
-                listener.onFailure(e);
-            }
-            if (response != null) {
-                try {
-                    listener.onResponse(response);
-                } catch (Exception e) {
-                    listener.onFailure(e);
+                if (task instanceof CancellableTask) {
+                    nodeFailureListeners.addListener(nodeResponseListener);
                 }
-            }
-        }
 
-        @Override
-        public void onCancelled() {
-            assert task instanceof CancellableTask : "task must be cancellable";
-            try {
-                ((CancellableTask) task).ensureNotCancelled();
-            } catch (TaskCancelledException e) {
-                nodeResponseTracker.discardIntermediateResponses(e);
-            }
-        }
+                final var nodeRequest = new NodeRequest(request, shards, node.getId());
+                if (task != null) {
+                    nodeRequest.setParentTask(clusterService.localNode().getId(), task.getId());
+                }
 
-        // For testing purposes
-        public NodeResponseTracker getNodeResponseTracker() {
-            return nodeResponseTracker;
+                transportService.sendRequest(
+                    node,
+                    transportNodeBroadcastAction,
+                    nodeRequest,
+                    transportRequestOptions,
+                    new ActionListenerResponseHandler<>(
+                        ActionListener.releaseAfter(nodeResponseListener, refs.acquire()),
+                        NodeResponse::new
+                    )
+                );
+            }
         }
     }
 
     class BroadcastByNodeTransportRequestHandler implements TransportRequestHandler<NodeRequest> {
         @Override
         public void messageReceived(final NodeRequest request, TransportChannel channel, Task task) throws Exception {
-            List<ShardRouting> shards = request.getShards();
-            final int totalShards = shards.size();
-            if (logger.isTraceEnabled()) {
-                logger.trace("[{}] executing operation on [{}] shards", actionName, totalShards);
+            executeAsDataNode(
+                task,
+                request.getIndicesLevelRequest(),
+                request.getShards(),
+                request.getNodeId(),
+                new ChannelActionListener<>(channel, transportNodeBroadcastAction, request)
+            );
+        }
+    }
+
+    private void executeAsDataNode(
+        Task task,
+        Request request,
+        List<ShardRouting> shards,
+        String nodeId,
+        ActionListener<NodeResponse> listener
+    ) {
+        logger.trace("[{}] executing operation on [{}] shards", actionName, shards.size());
+
+        final var results = new ArrayList<ShardOperationResult>(shards.size());
+        final var exceptions = new ArrayList<BroadcastShardOperationFailedException>(0);
+
+        final var resultListener = new ListenableFuture<NodeResponse>();
+        final var resultListenerCompleter = new RunOnce(() -> {
+            if (task instanceof CancellableTask cancellableTask) {
+                if (cancellableTask.notifyIfCancelled(resultListener)) {
+                    return;
+                }
             }
-            final AtomicArray<Object> shardResultOrExceptions = new AtomicArray<>(totalShards);
+            // ref releases all happen-before here so no need to be synchronized
+            resultListener.onResponse(new NodeResponse(nodeId, shards.size(), results, exceptions));
+        });
+
+        final var shardFailureListeners = new ListenableFuture<ShardOperationResult>();
+        if (task instanceof CancellableTask cancellableTask) {
+            cancellableTask.addListener(() -> {
+                assert cancellableTask.isCancelled();
+                resultListenerCompleter.run();
+                cancellableTask.notifyIfCancelled(shardFailureListeners);
+            });
+        }
 
-            final AtomicInteger counter = new AtomicInteger(shards.size());
-            int shardIndex = -1;
-            for (final ShardRouting shardRouting : shards) {
-                shardIndex++;
-                final int finalShardIndex = shardIndex;
-                onShardOperation(request, shardRouting, task, ActionListener.notifyOnce(new ActionListener<ShardOperationResult>() {
+        try (var refs = new RefCountingRunnable(() -> {
+            resultListener.addListener(listener);
+            resultListenerCompleter.run();
+        })) {
+            for (final var shardRouting : shards) {
+                if (task instanceof CancellableTask cancellableTask && cancellableTask.isCancelled()) {
+                    return;
+                }
 
+                final ActionListener<ShardOperationResult> shardListener = ActionListener.notifyOnce(new ActionListener<>() {
                     @Override
                     public void onResponse(ShardOperationResult shardOperationResult) {
-                        shardResultOrExceptions.setOnce(finalShardIndex, shardOperationResult);
-                        if (counter.decrementAndGet() == 0) {
-                            finishHim(request, channel, task, shardResultOrExceptions);
+                        logger.trace(() -> format("[%s] completed operation for shard [%s]", actionName, shardRouting.shortSummary()));
+                        synchronized (results) {
+                            results.add(shardOperationResult);
                         }
                     }
 
                     @Override
                     public void onFailure(Exception e) {
-                        shardResultOrExceptions.setOnce(finalShardIndex, e);
-                        if (counter.decrementAndGet() == 0) {
-                            finishHim(request, channel, task, shardResultOrExceptions);
+                        if (task instanceof CancellableTask cancellableTask && cancellableTask.isCancelled()) {
+                            return;
                         }
-                    }
-                }));
-            }
-        }
-
-        @SuppressWarnings("unchecked")
-        private void finishHim(NodeRequest request, TransportChannel channel, Task task, AtomicArray<Object> shardResultOrExceptions) {
-            if (task instanceof CancellableTask) {
-                try {
-                    ((CancellableTask) task).ensureNotCancelled();
-                } catch (TaskCancelledException e) {
-                    try {
-                        channel.sendResponse(e);
-                    } catch (IOException ioException) {
-                        e.addSuppressed(ioException);
-                        logger.warn("failed to send response", e);
-                    }
-                    return;
-                }
-            }
-            List<BroadcastShardOperationFailedException> accumulatedExceptions = new ArrayList<>();
-            List<ShardOperationResult> results = new ArrayList<>();
-            for (int i = 0; i < shardResultOrExceptions.length(); i++) {
-                if (shardResultOrExceptions.get(i) instanceof BroadcastShardOperationFailedException) {
-                    accumulatedExceptions.add((BroadcastShardOperationFailedException) shardResultOrExceptions.get(i));
-                } else {
-                    results.add((ShardOperationResult) shardResultOrExceptions.get(i));
-                }
-            }
-
-            try {
-                channel.sendResponse(
-                    new NodeResponse(request.getNodeId(), shardResultOrExceptions.length(), results, accumulatedExceptions)
-                );
-            } catch (IOException e) {
-                logger.warn("failed to send response", e);
-            }
-        }
-
-        private void onShardOperation(
-            final NodeRequest request,
-            final ShardRouting shardRouting,
-            final Task task,
-            final ActionListener<ShardOperationResult> listener
-        ) {
-            if (task instanceof CancellableTask && ((CancellableTask) task).notifyIfCancelled(listener)) {
-                return;
-            }
-            if (logger.isTraceEnabled()) {
-                logger.trace("[{}]  executing operation for shard [{}]", actionName, shardRouting.shortSummary());
-            }
-            final Consumer<Exception> failureHandler = e -> {
-                BroadcastShardOperationFailedException failure = new BroadcastShardOperationFailedException(
-                    shardRouting.shardId(),
-                    "operation " + actionName + " failed",
-                    e
-                );
-                failure.setShard(shardRouting.shardId());
-                if (TransportActions.isShardNotAvailableException(e)) {
-                    if (logger.isTraceEnabled()) {
-                        logger.trace(
-                            () -> format("[%s] failed to execute operation for shard [%s]", actionName, shardRouting.shortSummary()),
-                            e
-                        );
-                    }
-                } else {
-                    if (logger.isDebugEnabled()) {
-                        logger.debug(
+                        logger.log(
+                            TransportActions.isShardNotAvailableException(e) ? Level.TRACE : Level.DEBUG,
                             () -> format("[%s] failed to execute operation for shard [%s]", actionName, shardRouting.shortSummary()),
                             e
                         );
-                    }
-                }
-                listener.onFailure(failure);
-            };
-            try {
-                shardOperation(request.indicesLevelRequest, shardRouting, task, new ActionListener<>() {
-                    @Override
-                    public void onResponse(ShardOperationResult shardOperationResult) {
-                        if (logger.isTraceEnabled()) {
-                            logger.trace("[{}]  completed operation for shard [{}]", actionName, shardRouting.shortSummary());
+                        if (TransportActions.isShardNotAvailableException(e) == false) {
+                            synchronized (exceptions) {
+                                exceptions.add(
+                                    new BroadcastShardOperationFailedException(
+                                        shardRouting.shardId(),
+                                        "operation " + actionName + " failed",
+                                        e
+                                    )
+                                );
+                            }
                         }
-                        listener.onResponse(shardOperationResult);
                     }
 
                     @Override
-                    public void onFailure(Exception e) {
-                        failureHandler.accept(e);
+                    public String toString() {
+                        return "[" + actionName + "][" + shardRouting + "]";
                     }
                 });
-            } catch (Exception e) {
-                assert false : "shardOperation should not throw an exception, but delegate to listener instead";
-                failureHandler.accept(e);
+
+                if (task instanceof CancellableTask) {
+                    shardFailureListeners.addListener(shardListener);
+                }
+
+                logger.trace(() -> format("[%s] executing operation for shard [%s]", actionName, shardRouting.shortSummary()));
+                ActionRunnable.wrap(
+                    ActionListener.releaseAfter(shardListener, refs.acquire()),
+                    l -> shardOperation(request, shardRouting, task, l)
+                ).run();
             }
         }
     }
 
-    public class NodeRequest extends TransportRequest implements IndicesRequest {
-        private String nodeId;
-
-        private List<ShardRouting> shards;
+    class NodeRequest extends TransportRequest implements IndicesRequest {
+        private final Request indicesLevelRequest;
+        private final List<ShardRouting> shards;
+        private final String nodeId;
 
-        protected Request indicesLevelRequest;
-
-        public NodeRequest(StreamInput in) throws IOException {
+        NodeRequest(StreamInput in) throws IOException {
             super(in);
             indicesLevelRequest = readRequestFrom(in);
             shards = in.readList(ShardRouting::new);
             nodeId = in.readString();
         }
 
-        public NodeRequest(String nodeId, Request request, List<ShardRouting> shards) {
-            this.indicesLevelRequest = request;
+        NodeRequest(Request indicesLevelRequest, List<ShardRouting> shards, String nodeId) {
+            this.indicesLevelRequest = indicesLevelRequest;
             this.shards = shards;
             this.nodeId = nodeId;
         }
 
-        public List<ShardRouting> getShards() {
+        List<ShardRouting> getShards() {
             return shards;
         }
 
-        public String getNodeId() {
+        String getNodeId() {
             return nodeId;
         }
 
+        Request getIndicesLevelRequest() {
+            return indicesLevelRequest;
+        }
+
         @Override
         public String[] indices() {
             return indicesLevelRequest.indices();
@@ -644,19 +582,23 @@ class NodeResponse extends TransportResponse {
             this.exceptions = exceptions;
         }
 
-        public String getNodeId() {
+        String getNodeId() {
             return nodeId;
         }
 
-        public int getTotalShards() {
+        int getTotalShards() {
             return totalShards;
         }
 
-        public int getSuccessfulShards() {
+        int getSuccessfulShards() {
             return results.size();
         }
 
-        public List<BroadcastShardOperationFailedException> getExceptions() {
+        List<ShardOperationResult> getResults() {
+            return results;
+        }
+
+        List<BroadcastShardOperationFailedException> getExceptions() {
             return exceptions;
         }
 
@@ -681,13 +623,7 @@ public static final class EmptyResult implements Writeable {
 
         private EmptyResult() {}
 
-        private EmptyResult(StreamInput in) {}
-
         @Override
         public void writeTo(StreamOutput out) {}
-
-        public static EmptyResult readEmptyResultFrom(StreamInput in) {
-            return INSTANCE;
-        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequestBuilder.java
index 013a922e2ef8..b26f6faa0881 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequestBuilder.java
@@ -7,6 +7,7 @@
  */
 package org.elasticsearch.action.support.master;
 
+import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.client.internal.ElasticsearchClient;
 import org.elasticsearch.core.TimeValue;
@@ -16,7 +17,7 @@
  */
 public abstract class AcknowledgedRequestBuilder<
     Request extends AcknowledgedRequest<Request>,
-    Response extends AcknowledgedResponse,
+    Response extends ActionResponse & IsAcknowledgedSupplier,
     RequestBuilder extends AcknowledgedRequestBuilder<Request, Response, RequestBuilder>> extends MasterNodeOperationRequestBuilder<
         Request,
         Response,
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java
index b019782b27d7..e76d85f825ff 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java
@@ -25,13 +25,14 @@
 /**
  * A response that indicates that a request has been acknowledged
  */
-public class AcknowledgedResponse extends ActionResponse implements ToXContentObject {
+public class AcknowledgedResponse extends ActionResponse implements IsAcknowledgedSupplier, ToXContentObject {
 
     public static final AcknowledgedResponse TRUE = new AcknowledgedResponse(true);
 
     public static final AcknowledgedResponse FALSE = new AcknowledgedResponse(false);
 
-    private static final ParseField ACKNOWLEDGED = new ParseField("acknowledged");
+    public static final String ACKNOWLEDGED_KEY = "acknowledged";
+    private static final ParseField ACKNOWLEDGED = new ParseField(ACKNOWLEDGED_KEY);
 
     protected static <T extends AcknowledgedResponse> void declareAcknowledgedField(ConstructingObjectParser<T, Void> objectParser) {
         objectParser.declareField(
@@ -65,6 +66,7 @@ protected AcknowledgedResponse(boolean acknowledged) {
      * Returns whether the response is acknowledged or not
      * @return true if the response is acknowledged, false otherwise
      */
+    @Override
     public final boolean isAcknowledged() {
         return acknowledged;
     }
@@ -77,7 +79,7 @@ public void writeTo(StreamOutput out) throws IOException {
     @Override
     public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(ACKNOWLEDGED.getPreferredName(), isAcknowledged());
+        builder.field(ACKNOWLEDGED_KEY, isAcknowledged());
         addCustomFields(builder, params);
         builder.endObject();
         return builder;
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/IsAcknowledgedSupplier.java b/server/src/main/java/org/elasticsearch/action/support/master/IsAcknowledgedSupplier.java
new file mode 100644
index 000000000000..ae5b1a93e235
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/support/master/IsAcknowledgedSupplier.java
@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.support.master;
+
+public interface IsAcknowledgedSupplier {
+    boolean isAcknowledged();
+}
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java b/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
index 67268712c49c..d404212fc834 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
@@ -245,7 +245,7 @@ protected void doStart(ClusterState clusterState) {
                             masterNode,
                             actionName,
                             request,
-                            new ActionListenerResponseHandler<Response>(listener, responseReader) {
+                            new ActionListenerResponseHandler<>(listener, responseReader, executor) {
                                 @Override
                                 public void handleException(final TransportException exp) {
                                     Throwable cause = exp.unwrapCause();
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java
index d07cee07a0d0..8e80444cc125 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.support.master.info;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
@@ -31,7 +31,7 @@ public ClusterInfoRequest() {}
     public ClusterInfoRequest(StreamInput in) throws IOException {
         super(in);
         indices = in.readStringArray();
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             in.readStringArray();
         }
         indicesOptions = IndicesOptions.readIndicesOptions(in);
@@ -41,7 +41,7 @@ public ClusterInfoRequest(StreamInput in) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeStringArray(indices);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeStringArray(Strings.EMPTY_ARRAY);
         }
         indicesOptions.writeIndicesOptions(out);
diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponse.java b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponse.java
index 2922d33a85d4..4113788aafca 100644
--- a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponse.java
@@ -33,7 +33,7 @@ protected BaseNodesXContentResponse(StreamInput in) throws IOException {
     }
 
     @Override
-    public final Iterator<? extends ToXContent> toXContentChunked() {
+    public final Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
         return Iterators.concat(Iterators.single((b, p) -> {
             b.startObject();
             RestActions.buildNodesHeader(b, p, this);
diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
index 02337b389430..5f805efe0c17 100644
--- a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
@@ -8,26 +8,29 @@
 
 package org.elasticsearch.action.support.nodes;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionListenerResponseHandler;
+import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.action.support.NodeResponseTracker;
+import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.util.concurrent.ListenableFuture;
+import org.elasticsearch.common.util.concurrent.RunOnce;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
-import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportChannel;
-import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestHandler;
 import org.elasticsearch.transport.TransportRequestOptions;
-import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
@@ -36,12 +39,16 @@
 import java.util.List;
 import java.util.Objects;
 
+import static org.elasticsearch.core.Strings.format;
+
 public abstract class TransportNodesAction<
     NodesRequest extends BaseNodesRequest<NodesRequest>,
     NodesResponse extends BaseNodesResponse<?>,
     NodeRequest extends TransportRequest,
     NodeResponse extends BaseNodeResponse> extends HandledTransportAction<NodesRequest, NodesResponse> {
 
+    private static final Logger logger = LogManager.getLogger(TransportNodesAction.class);
+
     protected final ThreadPool threadPool;
     protected final ClusterService clusterService;
     protected final TransportService transportService;
@@ -81,7 +88,7 @@ protected TransportNodesAction(
         this.nodeResponseClass = Objects.requireNonNull(nodeResponseClass);
 
         this.transportNodeAction = actionName + "[n]";
-        this.finalExecutor = finalExecutor;
+        this.finalExecutor = finalExecutor.equals(ThreadPool.Names.SAME) ? ThreadPool.Names.GENERIC : finalExecutor;
         transportService.registerRequestHandler(transportNodeAction, nodeExecutor, nodeRequest, new NodeTransportHandler());
     }
 
@@ -119,40 +126,89 @@ protected TransportNodesAction(
 
     @Override
     protected void doExecute(Task task, NodesRequest request, ActionListener<NodesResponse> listener) {
-        new AsyncAction(task, request, listener).start();
-    }
+        if (request.concreteNodes() == null) {
+            resolveRequest(request, clusterService.state());
+            assert request.concreteNodes() != null;
+        }
 
-    /**
-     * Map the responses into {@code nodeResponseClass} responses and {@link FailedNodeException}s, convert to a {@link NodesResponse} and
-     * pass it to the listener. Fails the listener with a {@link NullPointerException} if {@code nodesResponses} is null.
-     *
-     * @param request The associated request.
-     * @param nodeResponseTracker All node-level responses collected so far
-     * @throws NodeResponseTracker.DiscardedResponsesException if {@code nodeResponseTracker} has already discarded the intermediate results
-     * @see #newResponseAsync(Task, BaseNodesRequest, List, List, ActionListener)
-     */
-    // exposed for tests
-    void newResponse(Task task, NodesRequest request, NodeResponseTracker nodeResponseTracker, ActionListener<NodesResponse> listener)
-        throws NodeResponseTracker.DiscardedResponsesException {
+        final var responses = new ArrayList<NodeResponse>(request.concreteNodes().length);
+        final var exceptions = new ArrayList<FailedNodeException>(0);
 
-        if (nodeResponseTracker == null) {
-            listener.onFailure(new NullPointerException("nodesResponses"));
-            return;
+        final var resultListener = new ListenableFuture<NodesResponse>();
+        final var resultListenerCompleter = new RunOnce(() -> {
+            if (task instanceof CancellableTask cancellableTask) {
+                if (cancellableTask.notifyIfCancelled(resultListener)) {
+                    return;
+                }
+            }
+            // ref releases all happen-before here so no need to be synchronized
+            threadPool.executor(finalExecutor)
+                .execute(ActionRunnable.wrap(resultListener, l -> newResponseAsync(task, request, responses, exceptions, l)));
+        });
+
+        final var nodeCancellationListener = new ListenableFuture<NodeResponse>(); // collects node listeners & completes them if cancelled
+        if (task instanceof CancellableTask cancellableTask) {
+            cancellableTask.addListener(() -> {
+                assert cancellableTask.isCancelled();
+                resultListenerCompleter.run();
+                cancellableTask.notifyIfCancelled(nodeCancellationListener);
+            });
         }
 
-        final List<NodeResponse> responses = new ArrayList<>();
-        final List<FailedNodeException> failures = new ArrayList<>();
+        final var transportRequestOptions = TransportRequestOptions.timeout(request.timeout());
+
+        try (var refs = new RefCountingRunnable(() -> {
+            resultListener.addListener(listener);
+            resultListenerCompleter.run();
+        })) {
+            for (final var node : request.concreteNodes()) {
+                final ActionListener<NodeResponse> nodeResponseListener = ActionListener.notifyOnce(new ActionListener<>() {
+                    @Override
+                    public void onResponse(NodeResponse nodeResponse) {
+                        synchronized (responses) {
+                            responses.add(nodeResponse);
+                        }
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        if (task instanceof CancellableTask cancellableTask && cancellableTask.isCancelled()) {
+                            return;
+                        }
+
+                        logger.debug(() -> format("failed to execute [%s] on node [%s]", actionName, node), e);
+                        synchronized (exceptions) {
+                            exceptions.add(new FailedNodeException(node.getId(), "Failed node [" + node.getId() + "]", e));
+                        }
+                    }
+
+                    @Override
+                    public String toString() {
+                        return "[" + actionName + "][" + node.descriptionWithoutAttributes() + "]";
+                    }
+                });
+
+                if (task instanceof CancellableTask) {
+                    nodeCancellationListener.addListener(nodeResponseListener);
+                }
 
-        for (int i = 0; i < nodeResponseTracker.getExpectedResponseCount(); ++i) {
-            Object response = nodeResponseTracker.getResponse(i);
-            if (nodeResponseTracker.getResponse(i)instanceof FailedNodeException failedNodeException) {
-                failures.add(failedNodeException);
-            } else {
-                responses.add(nodeResponseClass.cast(response));
+                final var nodeRequest = newNodeRequest(request);
+                if (task != null) {
+                    nodeRequest.setParentTask(clusterService.localNode().getId(), task.getId());
+                }
+
+                transportService.sendRequest(
+                    node,
+                    transportNodeAction,
+                    nodeRequest,
+                    transportRequestOptions,
+                    new ActionListenerResponseHandler<>(
+                        ActionListener.releaseAfter(nodeResponseListener, refs.acquire()),
+                        in -> newNodeResponse(in, node)
+                    )
+                );
             }
         }
-
-        newResponseAsync(task, request, responses, failures, listener);
     }
 
     /**
@@ -195,141 +251,9 @@ protected void resolveRequest(NodesRequest request, ClusterState clusterState) {
         request.setConcreteNodes(Arrays.stream(nodesIds).map(clusterState.nodes()::get).toArray(DiscoveryNode[]::new));
     }
 
-    /**
-     * Get a backwards compatible transport action name
-     */
-    protected String getTransportNodeAction(DiscoveryNode node) {
-        return transportNodeAction;
-    }
-
-    class AsyncAction implements CancellableTask.CancellationListener {
-
-        private final NodesRequest request;
-        private final ActionListener<NodesResponse> listener;
-        private final NodeResponseTracker nodeResponseTracker;
-        private final Task task;
-
-        AsyncAction(Task task, NodesRequest request, ActionListener<NodesResponse> listener) {
-            this.task = task;
-            this.request = request;
-            this.listener = listener;
-            if (request.concreteNodes() == null) {
-                resolveRequest(request, clusterService.state());
-                assert request.concreteNodes() != null;
-            }
-            this.nodeResponseTracker = new NodeResponseTracker(request.concreteNodes().length);
-        }
-
-        void start() {
-            if (task instanceof CancellableTask cancellableTask) {
-                cancellableTask.addListener(this);
-            }
-            final DiscoveryNode[] nodes = request.concreteNodes();
-            if (nodes.length == 0) {
-                finishHim();
-                return;
-            }
-            final TransportRequestOptions transportRequestOptions = TransportRequestOptions.timeout(request.timeout());
-            for (int i = 0; i < nodes.length; i++) {
-                final int idx = i;
-                final DiscoveryNode node = nodes[i];
-                final String nodeId = node.getId();
-                try {
-                    TransportRequest nodeRequest = newNodeRequest(request);
-                    if (task != null) {
-                        nodeRequest.setParentTask(clusterService.localNode().getId(), task.getId());
-                    }
-
-                    transportService.sendRequest(
-                        node,
-                        getTransportNodeAction(node),
-                        nodeRequest,
-                        transportRequestOptions,
-                        new TransportResponseHandler<NodeResponse>() {
-                            @Override
-                            public NodeResponse read(StreamInput in) throws IOException {
-                                return newNodeResponse(in, node);
-                            }
-
-                            @Override
-                            public void handleResponse(NodeResponse response) {
-                                onOperation(idx, response);
-                            }
-
-                            @Override
-                            public void handleException(TransportException exp) {
-                                onFailure(idx, node.getId(), exp);
-                            }
-
-                            @Override
-                            public String toString() {
-                                return "AsyncActionNodeResponseHandler{node=" + node + ", action=" + AsyncAction.this + '}';
-                            }
-                        }
-                    );
-                } catch (Exception e) {
-                    onFailure(idx, nodeId, e);
-                }
-            }
-        }
-
-        // For testing purposes
-        NodeResponseTracker getNodeResponseTracker() {
-            return nodeResponseTracker;
-        }
-
-        private void onOperation(int idx, NodeResponse nodeResponse) {
-            if (nodeResponseTracker.trackResponseAndCheckIfLast(idx, nodeResponse)) {
-                finishHim();
-            }
-        }
-
-        private void onFailure(int idx, String nodeId, Throwable t) {
-            logger.debug(() -> "failed to execute on node [" + nodeId + "]", t);
-            if (nodeResponseTracker.trackResponseAndCheckIfLast(idx, new FailedNodeException(nodeId, "Failed node [" + nodeId + "]", t))) {
-                finishHim();
-            }
-        }
-
-        private void finishHim() {
-            if ((task instanceof CancellableTask t) && t.notifyIfCancelled(listener)) {
-                return;
-            }
-
-            final String executor = finalExecutor.equals(ThreadPool.Names.SAME) ? ThreadPool.Names.GENERIC : finalExecutor;
-            threadPool.executor(executor).execute(() -> {
-                try {
-                    newResponse(task, request, nodeResponseTracker, listener);
-                } catch (NodeResponseTracker.DiscardedResponsesException e) {
-                    // We propagate the reason that the results, in this case the task cancellation, in case the listener needs to take
-                    // follow-up actions
-                    listener.onFailure((Exception) e.getCause());
-                }
-            });
-        }
-
-        @Override
-        public void onCancelled() {
-            assert task instanceof CancellableTask : "task must be cancellable";
-            try {
-                ((CancellableTask) task).ensureNotCancelled();
-            } catch (TaskCancelledException e) {
-                nodeResponseTracker.discardIntermediateResponses(e);
-            }
-        }
-
-        @Override
-        public String toString() {
-            return "AsyncAction{request=" + request + ", listener=" + listener + '}';
-        }
-    }
-
     class NodeTransportHandler implements TransportRequestHandler<NodeRequest> {
         @Override
         public void messageReceived(NodeRequest request, TransportChannel channel, Task task) throws Exception {
-            if (task instanceof CancellableTask) {
-                ((CancellableTask) task).ensureNotCancelled();
-            }
             channel.sendResponse(nodeOperation(request, task));
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
index 8ec274bc410f..6b1916b4ec84 100644
--- a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
+++ b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
@@ -213,6 +213,7 @@ private void performOnReplica(
         final long maxSeqNoOfUpdatesOrDeletes,
         final PendingReplicationActions pendingReplicationActions
     ) {
+        assert shard.isPromotableToPrimary() : "only promotable shards should receive replication requests";
         if (logger.isTraceEnabled()) {
             logger.trace("[{}] sending op [{}] to replica {} for request [{}]", shard.shardId(), opType, shard, replicaRequest);
         }
diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java
index 62a2d3d38e06..94c90a0efcd8 100644
--- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java
@@ -8,12 +8,13 @@
 
 package org.elasticsearch.action.support.replication;
 
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.action.support.TransportActions;
 import org.elasticsearch.action.support.broadcast.BaseBroadcastResponse;
 import org.elasticsearch.action.support.broadcast.BroadcastRequest;
@@ -25,15 +26,16 @@
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.util.concurrent.CountDown;
+import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.transport.Transports;
 
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.Map;
 
 /**
  * Base class for requests that should be executed on all shards of an index or several indices.
@@ -49,6 +51,7 @@ public abstract class TransportBroadcastReplicationAction<
     private final ClusterService clusterService;
     private final IndexNameExpressionResolver indexNameExpressionResolver;
     private final NodeClient client;
+    private final String executor;
 
     public TransportBroadcastReplicationAction(
         String name,
@@ -58,66 +61,112 @@ public TransportBroadcastReplicationAction(
         NodeClient client,
         ActionFilters actionFilters,
         IndexNameExpressionResolver indexNameExpressionResolver,
-        ActionType<ShardResponse> replicatedBroadcastShardAction
+        ActionType<ShardResponse> replicatedBroadcastShardAction,
+        String executor
     ) {
         super(name, transportService, actionFilters, requestReader);
         this.client = client;
         this.replicatedBroadcastShardAction = replicatedBroadcastShardAction;
         this.clusterService = clusterService;
         this.indexNameExpressionResolver = indexNameExpressionResolver;
+        this.executor = executor;
     }
 
     @Override
     protected void doExecute(Task task, Request request, ActionListener<Response> listener) {
-        final ClusterState clusterState = clusterService.state();
-        List<ShardId> shards = shards(request, clusterState);
-        final CopyOnWriteArrayList<ShardResponse> shardsResponses = new CopyOnWriteArrayList<>();
-        if (shards.size() == 0) {
-            finishAndNotifyListener(listener, shardsResponses);
-        }
-        final CountDown responsesCountDown = new CountDown(shards.size());
-        for (final ShardId shardId : shards) {
-            ActionListener<ShardResponse> shardActionListener = new ActionListener<ShardResponse>() {
+        clusterService.threadPool().executor(executor).execute(ActionRunnable.wrap(listener, createAsyncAction(task, request)));
+    }
+
+    private CheckedConsumer<ActionListener<Response>, Exception> createAsyncAction(Task task, Request request) {
+        return new CheckedConsumer<ActionListener<Response>, Exception>() {
+
+            private int totalShardCopyCount;
+            private int successShardCopyCount;
+            private final List<DefaultShardOperationFailedException> allFailures = new ArrayList<>();
+
+            @Override
+            public void accept(ActionListener<Response> listener) {
+                assert totalShardCopyCount == 0 && successShardCopyCount == 0 && allFailures.isEmpty() : "shouldn't call this twice";
+
+                final ClusterState clusterState = clusterService.state();
+                final List<ShardId> shards = shards(request, clusterState);
+                final Map<String, IndexMetadata> indexMetadataByName = clusterState.getMetadata().indices();
+
+                try (var refs = new RefCountingRunnable(() -> finish(listener))) {
+                    for (final ShardId shardId : shards) {
+                        // NB This sends O(#shards) requests in a tight loop; TODO add some throttling here?
+                        shardExecute(
+                            task,
+                            request,
+                            shardId,
+                            ActionListener.releaseAfter(new ReplicationResponseActionListener(shardId, indexMetadataByName), refs.acquire())
+                        );
+                    }
+                }
+            }
+
+            private synchronized void addShardResponse(int numCopies, int successful, List<DefaultShardOperationFailedException> failures) {
+                totalShardCopyCount += numCopies;
+                successShardCopyCount += successful;
+                allFailures.addAll(failures);
+            }
+
+            void finish(ActionListener<Response> listener) {
+                // no need for synchronized here, the RefCountingRunnable guarantees that all the addShardResponse calls happen-before here
+                logger.trace("{}: got all shard responses", actionName);
+                listener.onResponse(newResponse(successShardCopyCount, allFailures.size(), totalShardCopyCount, allFailures));
+            }
+
+            class ReplicationResponseActionListener implements ActionListener<ShardResponse> {
+                private final ShardId shardId;
+                private final Map<String, IndexMetadata> indexMetadataByName;
+
+                ReplicationResponseActionListener(ShardId shardId, Map<String, IndexMetadata> indexMetadataByName) {
+                    this.shardId = shardId;
+                    this.indexMetadataByName = indexMetadataByName;
+                }
+
                 @Override
                 public void onResponse(ShardResponse shardResponse) {
-                    shardsResponses.add(shardResponse);
+                    assert shardResponse != null;
                     logger.trace("{}: got response from {}", actionName, shardId);
-                    if (responsesCountDown.countDown()) {
-                        finishAndNotifyListener(listener, shardsResponses);
-                    }
+                    addShardResponse(
+                        shardResponse.getShardInfo().getTotal(),
+                        shardResponse.getShardInfo().getSuccessful(),
+                        Arrays.stream(shardResponse.getShardInfo().getFailures())
+                            .map(
+                                f -> new DefaultShardOperationFailedException(
+                                    new BroadcastShardOperationFailedException(shardId, f.getCause())
+                                )
+                            )
+                            .toList()
+                    );
                 }
 
                 @Override
                 public void onFailure(Exception e) {
                     logger.trace("{}: got failure from {}", actionName, shardId);
-                    int totalNumCopies = clusterState.getMetadata().getIndexSafe(shardId.getIndex()).getNumberOfReplicas() + 1;
-                    ShardResponse shardResponse = newShardResponse();
-                    ReplicationResponse.ShardInfo.Failure[] failures;
+                    final int numCopies = indexMetadataByName.get(shardId.getIndexName()).getNumberOfReplicas() + 1;
+                    final List<DefaultShardOperationFailedException> result;
                     if (TransportActions.isShardNotAvailableException(e)) {
-                        failures = new ReplicationResponse.ShardInfo.Failure[0];
+                        result = List.of();
                     } else {
-                        ReplicationResponse.ShardInfo.Failure failure = new ReplicationResponse.ShardInfo.Failure(
-                            shardId,
-                            null,
-                            e,
-                            ExceptionsHelper.status(e),
-                            true
+                        final var failures = new DefaultShardOperationFailedException[numCopies];
+                        Arrays.fill(
+                            failures,
+                            new DefaultShardOperationFailedException(new BroadcastShardOperationFailedException(shardId, e))
                         );
-                        failures = new ReplicationResponse.ShardInfo.Failure[totalNumCopies];
-                        Arrays.fill(failures, failure);
-                    }
-                    shardResponse.setShardInfo(new ReplicationResponse.ShardInfo(totalNumCopies, 0, failures));
-                    shardsResponses.add(shardResponse);
-                    if (responsesCountDown.countDown()) {
-                        finishAndNotifyListener(listener, shardsResponses);
+                        result = Arrays.asList(failures);
                     }
+                    addShardResponse(numCopies, 0, result);
                 }
-            };
-            shardExecute(task, request, shardId, shardActionListener);
-        }
+            }
+
+        };
     }
 
     protected void shardExecute(Task task, Request request, ShardId shardId, ActionListener<ShardResponse> shardActionListener) {
+        assert Transports.assertNotTransportThread("may hit all the shards");
         ShardRequest shardRequest = newShardRequest(request, shardId);
         shardRequest.setParentTask(clusterService.localNode().getId(), task.getId());
         client.executeLocally(replicatedBroadcastShardAction, shardRequest, shardActionListener);
@@ -127,6 +176,7 @@ protected void shardExecute(Task task, Request request, ShardId shardId, ActionL
      * @return all shard ids the request should run on
      */
     protected List<ShardId> shards(Request request, ClusterState clusterState) {
+        assert Transports.assertNotTransportThread("may hit all the shards");
         List<ShardId> shardIds = new ArrayList<>();
         String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request);
         for (String index : concreteIndices) {
@@ -141,43 +191,13 @@ protected List<ShardId> shards(Request request, ClusterState clusterState) {
         return shardIds;
     }
 
-    protected abstract ShardResponse newShardResponse();
-
     protected abstract ShardRequest newShardRequest(Request request, ShardId shardId);
 
-    private void finishAndNotifyListener(ActionListener<Response> listener, CopyOnWriteArrayList<ShardResponse> shardsResponses) {
-        logger.trace("{}: got all shard responses", actionName);
-        int successfulShards = 0;
-        int failedShards = 0;
-        int totalNumCopies = 0;
-        List<DefaultShardOperationFailedException> shardFailures = null;
-        for (int i = 0; i < shardsResponses.size(); i++) {
-            ReplicationResponse shardResponse = shardsResponses.get(i);
-            if (shardResponse == null) {
-                // non active shard, ignore
-            } else {
-                failedShards += shardResponse.getShardInfo().getFailed();
-                successfulShards += shardResponse.getShardInfo().getSuccessful();
-                totalNumCopies += shardResponse.getShardInfo().getTotal();
-                if (shardFailures == null) {
-                    shardFailures = new ArrayList<>();
-                }
-                for (ReplicationResponse.ShardInfo.Failure failure : shardResponse.getShardInfo().getFailures()) {
-                    shardFailures.add(
-                        new DefaultShardOperationFailedException(
-                            new BroadcastShardOperationFailedException(failure.fullShardId(), failure.getCause())
-                        )
-                    );
-                }
-            }
-        }
-        listener.onResponse(newResponse(successfulShards, failedShards, totalNumCopies, shardFailures));
-    }
-
     protected abstract Response newResponse(
         int successfulShards,
         int failedShards,
         int totalNumCopies,
         List<DefaultShardOperationFailedException> shardFailures
     );
+
 }
diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
index 7e299f5baa25..1414aa5bbdb7 100644
--- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
@@ -497,7 +497,7 @@ public void handleException(TransportException exp) {
                     new ReplicationOperation<>(
                         primaryRequest.getRequest(),
                         primaryShardReference,
-                        responseListener.map(result -> result.finalResponseIfSuccessful),
+                        responseListener.map(result -> result.replicationResponse),
                         newReplicasProxy(),
                         logger,
                         threadPool,
@@ -533,29 +533,18 @@ protected void adaptResponse(Response response, IndexShard indexShard) {
     public static class PrimaryResult<ReplicaRequest extends ReplicationRequest<ReplicaRequest>, Response extends ReplicationResponse>
         implements
             ReplicationOperation.PrimaryResult<ReplicaRequest> {
-        protected final ReplicaRequest replicaRequest;
-        public final Response finalResponseIfSuccessful;
-        public final Exception finalFailure;
+        private final ReplicaRequest replicaRequest;
+        public final Response replicationResponse;
 
         /**
          * Result of executing a primary operation
-         * expects <code>finalResponseIfSuccessful</code> or <code>finalFailure</code> to be not-null
+         * expects <code>replicationResponse</code> to be not-null
          */
-        public PrimaryResult(ReplicaRequest replicaRequest, Response finalResponseIfSuccessful, Exception finalFailure) {
-            assert finalFailure != null ^ finalResponseIfSuccessful != null
-                : "either a response or a failure has to be not null, "
-                    + "found ["
-                    + finalFailure
-                    + "] failure and ["
-                    + finalResponseIfSuccessful
-                    + "] response";
-            this.replicaRequest = replicaRequest;
-            this.finalResponseIfSuccessful = finalResponseIfSuccessful;
-            this.finalFailure = finalFailure;
-        }
-
         public PrimaryResult(ReplicaRequest replicaRequest, Response replicationResponse) {
-            this(replicaRequest, replicationResponse, null);
+            assert replicaRequest != null : "request is required";
+            assert replicationResponse != null : "response is required";
+            this.replicaRequest = replicaRequest;
+            this.replicationResponse = replicationResponse;
         }
 
         @Override
@@ -565,18 +554,12 @@ public ReplicaRequest replicaRequest() {
 
         @Override
         public void setShardInfo(ReplicationResponse.ShardInfo shardInfo) {
-            if (finalResponseIfSuccessful != null) {
-                finalResponseIfSuccessful.setShardInfo(shardInfo);
-            }
+            replicationResponse.setShardInfo(shardInfo);
         }
 
         @Override
         public void runPostReplicationActions(ActionListener<Void> listener) {
-            if (finalFailure != null) {
-                listener.onFailure(finalFailure);
-            } else {
-                listener.onResponse(null);
-            }
+            listener.onResponse(null);
         }
     }
 
@@ -1140,8 +1123,8 @@ public void failShard(String reason, Exception e) {
         public void perform(Request request, ActionListener<PrimaryResult<ReplicaRequest, Response>> listener) {
             if (Assertions.ENABLED) {
                 listener = listener.map(result -> {
-                    assert result.replicaRequest() == null || result.finalFailure == null
-                        : "a replica request [" + result.replicaRequest() + "] with a primary failure [" + result.finalFailure + "]";
+                    assert result.replicaRequest().getParentTask().equals(request.getParentTask())
+                        : "a replica request [" + result.replicaRequest() + "] with a different parent task from the primary request";
                     return result;
                 });
             }
diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java
index ffcd530ae418..fb81739d14f9 100644
--- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java
@@ -270,58 +270,45 @@ public WritePrimaryResult(
             ReplicaRequest request,
             @Nullable Response finalResponse,
             @Nullable Location location,
-            @Nullable Exception operationFailure,
             IndexShard primary,
             Logger logger
         ) {
-            this(request, finalResponse, location, operationFailure, primary, logger, null);
+            this(request, finalResponse, location, primary, logger, null);
         }
 
         public WritePrimaryResult(
             ReplicaRequest request,
             @Nullable Response finalResponse,
             @Nullable Location location,
-            @Nullable Exception operationFailure,
             IndexShard primary,
             Logger logger,
             @Nullable Consumer<Runnable> postWriteAction
         ) {
-            super(request, finalResponse, operationFailure);
+            super(request, finalResponse);
             this.location = location;
             this.primary = primary;
             this.logger = logger;
             this.postWriteAction = postWriteAction;
-            assert location == null || operationFailure == null
-                : "expected either failure to be null or translog location to be null, "
-                    + "but found: ["
-                    + location
-                    + "] translog location and ["
-                    + operationFailure
-                    + "] failure";
         }
 
         @Override
         public void runPostReplicationActions(ActionListener<Void> listener) {
-            if (finalFailure != null) {
-                listener.onFailure(finalFailure);
-            } else {
-                /*
-                 * We call this after replication because this might wait for a refresh and that can take a while.
-                 * This way we wait for the refresh in parallel on the primary and on the replica.
-                 */
-                new AsyncAfterWriteAction(primary, replicaRequest, location, new RespondingWriteResult() {
-                    @Override
-                    public void onSuccess(boolean forcedRefresh) {
-                        finalResponseIfSuccessful.setForcedRefresh(forcedRefresh);
-                        listener.onResponse(null);
-                    }
+            /*
+             * We call this after replication because this might wait for a refresh and that can take a while.
+             * This way we wait for the refresh in parallel on the primary and on the replica.
+             */
+            new AsyncAfterWriteAction(primary, replicaRequest(), location, new RespondingWriteResult() {
+                @Override
+                public void onSuccess(boolean forcedRefresh) {
+                    replicationResponse.setForcedRefresh(forcedRefresh);
+                    listener.onResponse(null);
+                }
 
-                    @Override
-                    public void onFailure(Exception ex) {
-                        listener.onFailure(ex);
-                    }
-                }, logger, postWriteAction).run();
-            }
+                @Override
+                public void onFailure(Exception ex) {
+                    listener.onFailure(ex);
+                }
+            }, logger, postWriteAction).run();
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java b/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
index e56d1568d20e..30a9c92d7c04 100644
--- a/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
@@ -92,7 +92,10 @@ protected void doExecute(Task task, TasksRequest request, ActionListener<TasksRe
     private void nodeOperation(Task task, NodeTaskRequest nodeTaskRequest, ActionListener<NodeTasksResponse> listener) {
         TasksRequest request = nodeTaskRequest.tasksRequest;
         List<OperationTask> tasks = new ArrayList<>();
-        processTasks(request, tasks::add);
+        processTasks(request, tasks::add, ActionListener.wrap(noop -> nodeOperation(task, listener, request, tasks), listener::onFailure));
+    }
+
+    private void nodeOperation(Task task, ActionListener<NodeTasksResponse> listener, TasksRequest request, List<OperationTask> tasks) {
         if (tasks.isEmpty()) {
             listener.onResponse(new NodeTasksResponse(clusterService.localNode().getId(), emptyList(), emptyList()));
             return;
@@ -154,6 +157,11 @@ protected String[] resolveNodes(TasksRequest request, ClusterState clusterState)
         }
     }
 
+    protected void processTasks(TasksRequest request, Consumer<OperationTask> operation, ActionListener<Void> nodeOperation) {
+        processTasks(request, operation);
+        nodeOperation.onResponse(null);
+    }
+
     @SuppressWarnings("unchecked")
     protected void processTasks(TasksRequest request, Consumer<OperationTask> operation) {
         if (request.getTargetTaskId().isSet()) {
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
index bfa07fa55d3f..150545f51b14 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.action.termvectors;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -41,7 +41,7 @@ public Failure(String index, String id, Exception cause) {
 
         public Failure(StreamInput in) throws IOException {
             index = in.readString();
-            if (in.getVersion().before(Version.V_8_0_0)) {
+            if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 // types no longer relevant so ignore
                 String type = in.readOptionalString();
                 if (type != null) {
@@ -76,7 +76,7 @@ public Exception getCause() {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(index);
-            if (out.getVersion().before(Version.V_8_0_0)) {
+            if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 // types not supported so send an empty array to previous versions
                 out.writeOptionalString(null);
             }
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java
index 6a77388bcd5f..fb5f1f6ee134 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.action.termvectors;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.RealtimeRequest;
 import org.elasticsearch.action.ValidateActions;
@@ -128,7 +128,7 @@ public TermVectorsRequest() {}
 
     TermVectorsRequest(StreamInput in) throws IOException {
         super(in);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             // types no longer relevant so ignore
             in.readString();
         }
@@ -477,7 +477,7 @@ public ActionRequestValidationException validate() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             // types not supported so send an empty array to previous versions
             out.writeString("_doc");
         }
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java
index b30cb8a7a7f8..82174485d829 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java
@@ -16,7 +16,7 @@
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CharsRefBuilder;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.termvectors.TermVectorsRequest.Flag;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -91,7 +91,7 @@ public TermVectorsResponse(String index, String id) {
 
     TermVectorsResponse(StreamInput in) throws IOException {
         index = in.readString();
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             // types no longer relevant so ignore
             in.readString();
         }
@@ -109,7 +109,7 @@ public TermVectorsResponse(String index, String id) {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(index);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             // types not supported so send an empty array to previous versions
             out.writeString(MapperService.SINGLE_MAPPING_NAME);
         }
diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java
index 1c222590597b..6801bf2c0fd1 100644
--- a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.action.update;
 
 import org.apache.lucene.util.RamUsageEstimator;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.index.IndexRequest;
@@ -136,7 +136,7 @@ public UpdateRequest(StreamInput in) throws IOException {
     public UpdateRequest(@Nullable ShardId shardId, StreamInput in) throws IOException {
         super(shardId, in);
         waitForActiveShards = ActiveShardCount.readFrom(in);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             String type = in.readString();
             assert MapperService.SINGLE_MAPPING_NAME.equals(type) : "Expected [_doc] but received [" + type + "]";
         }
@@ -159,7 +159,7 @@ public UpdateRequest(@Nullable ShardId shardId, StreamInput in) throws IOExcepti
         ifPrimaryTerm = in.readVLong();
         detectNoop = in.readBoolean();
         scriptedUpsert = in.readBoolean();
-        if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             requireAlias = in.readBoolean();
         } else {
             requireAlias = false;
@@ -860,7 +860,7 @@ public void writeThin(StreamOutput out) throws IOException {
 
     private void doWrite(StreamOutput out, boolean thin) throws IOException {
         waitForActiveShards.writeTo(out);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeString(MapperService.SINGLE_MAPPING_NAME);
         }
         out.writeString(id);
@@ -905,7 +905,7 @@ private void doWrite(StreamOutput out, boolean thin) throws IOException {
         out.writeVLong(ifPrimaryTerm);
         out.writeBoolean(detectNoop);
         out.writeBoolean(scriptedUpsert);
-        if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             out.writeBoolean(requireAlias);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
index e56c53e16d28..a9ef2e68a676 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
@@ -11,14 +11,14 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.NodeValidationException;
 
 import java.io.PrintStream;
 
-import static org.elasticsearch.bootstrap.BootstrapInfo.USER_EXCEPTION_MARKER;
-
 /**
  * A container for transient state during bootstrap of the Elasticsearch process.
  */
@@ -71,10 +71,10 @@ Environment environment() {
         return nodeEnv.get();
     }
 
-    void exitWithUserException(int exitCode, Exception e) {
-        err.print(USER_EXCEPTION_MARKER);
-        err.println(e.getMessage());
-        gracefullyExit(exitCode);
+    void exitWithNodeValidationException(NodeValidationException e) {
+        Logger logger = LogManager.getLogger(Elasticsearch.class);
+        logger.error("node validation exception\n{}", e.getMessage());
+        gracefullyExit(ExitCodes.CONFIG);
     }
 
     void exitWithUnknownException(Throwable e) {
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
index 0399624609c4..b9610c689f92 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
@@ -25,6 +25,7 @@
 
 import java.io.BufferedReader;
 import java.io.IOException;
+import java.nio.ByteOrder;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.AllPermission;
@@ -205,6 +206,7 @@ static List<BootstrapCheck> checks() {
         checks.add(new EarlyAccessCheck());
         checks.add(new AllPermissionCheck());
         checks.add(new DiscoveryConfiguredCheck());
+        checks.add(new ByteOrderCheck());
         return Collections.unmodifiableList(checks);
     }
 
@@ -702,4 +704,19 @@ public BootstrapCheckResult check(BootstrapContext context) {
             );
         }
     }
+
+    static class ByteOrderCheck implements BootstrapCheck {
+
+        @Override
+        public BootstrapCheckResult check(BootstrapContext context) {
+            if (nativeByteOrder() != ByteOrder.LITTLE_ENDIAN) {
+                return BootstrapCheckResult.failure("Little-endian native byte order is required to run Elasticsearch");
+            }
+            return BootstrapCheckResult.success();
+        }
+
+        ByteOrder nativeByteOrder() {
+            return ByteOrder.nativeOrder();
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java
index 0e565f22ba76..740833dffc7c 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java
@@ -66,13 +66,6 @@ public static ConsoleLoader.Console getConsole() {
      */
     public static final String UNTRUSTED_CODEBASE = "/untrusted";
 
-    /**
-     * A non-printable character denoting a UserException has occurred.
-     *
-     * This is sent over stderr to the controlling CLI process.
-     */
-    public static final char USER_EXCEPTION_MARKER = '\u0015';
-
     /**
      * A non-printable character denoting the server is ready to process requests.
      *
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/ConsoleLoader.java b/server/src/main/java/org/elasticsearch/bootstrap/ConsoleLoader.java
index 8b0d914e2da3..955e4439d1f7 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/ConsoleLoader.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/ConsoleLoader.java
@@ -53,9 +53,8 @@ static Supplier<Console> buildConsoleLoader(ClassLoader classLoader) {
     private static ClassLoader buildClassLoader(Environment env) {
         final Path libDir = env.libFile().resolve("tools").resolve("ansi-console");
 
-        try {
-            final URL[] urls = Files.list(libDir)
-                .filter(each -> each.getFileName().toString().endsWith(".jar"))
+        try (var libDirFilesStream = Files.list(libDir)) {
+            final URL[] urls = libDirFilesStream.filter(each -> each.getFileName().toString().endsWith(".jar"))
                 .map(ConsoleLoader::pathToURL)
                 .toArray(URL[]::new);
 
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
index 8d30feb41452..bfaf712d4872 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
@@ -16,12 +16,11 @@
 import org.apache.lucene.util.StringHelper;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
-import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.filesystem.FileSystemNatives;
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.network.IfConfig;
-import org.elasticsearch.common.settings.KeyStoreWrapper;
 import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
@@ -39,6 +38,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PrintStream;
+import java.lang.invoke.MethodHandles;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.Permission;
@@ -66,7 +66,7 @@ public static void main(final String[] args) {
             initPhase2(bootstrap);
             initPhase3(bootstrap);
         } catch (NodeValidationException e) {
-            bootstrap.exitWithUserException(ExitCodes.CONFIG, e);
+            bootstrap.exitWithNodeValidationException(e);
         } catch (Throwable t) {
             bootstrap.exitWithUnknownException(t);
         }
@@ -143,14 +143,9 @@ public void checkPermission(Permission perm) {
      */
     private static void initPhase2(Bootstrap bootstrap) throws IOException {
         final ServerArgs args = bootstrap.args();
-        final SecureSettings keystore;
-        try {
-            keystore = KeyStoreWrapper.bootstrap(args.configDir(), args::keystorePassword);
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-        bootstrap.setSecureSettings(keystore);
-        Environment nodeEnv = createEnvironment(args.configDir(), args.nodeSettings(), keystore);
+        final SecureSettings secrets = args.secrets();
+        bootstrap.setSecureSettings(secrets);
+        Environment nodeEnv = createEnvironment(args.configDir(), args.nodeSettings(), secrets);
         bootstrap.setEnvironment(nodeEnv);
 
         initPidFile(args.pidFile());
@@ -182,6 +177,13 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException {
         // Log ifconfig output before SecurityManager is installed
         IfConfig.logIfNecessary();
 
+        try {
+            // ReferenceDocs class does nontrivial static initialization which should always succeed but load it now (before SM) to be sure
+            MethodHandles.publicLookup().ensureInitialized(ReferenceDocs.class);
+        } catch (IllegalAccessException unexpected) {
+            throw new AssertionError(unexpected);
+        }
+
         // install SM after natives, shutdown hooks, etc.
         org.elasticsearch.bootstrap.Security.configure(
             nodeEnv,
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java b/server/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java
index a8c732b6fa02..cefcd4bfb511 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java
@@ -52,12 +52,19 @@ static boolean isFatalUncaught(Throwable e) {
 
     void onFatalUncaught(final String threadName, final Throwable t) {
         final String message = "fatal error in thread [" + threadName + "], exiting";
-        logger.error(message, t);
+        logErrorMessage(t, message);
     }
 
     void onNonFatalUncaught(final String threadName, final Throwable t) {
         final String message = "uncaught exception in thread [" + threadName + "]";
-        logger.error(message, t);
+        logErrorMessage(t, message);
+    }
+
+    private void logErrorMessage(Throwable t, String message) {
+        AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
+            logger.error(message, t);
+            return null;
+        });
     }
 
     void halt(int status) {
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/ServerArgs.java b/server/src/main/java/org/elasticsearch/bootstrap/ServerArgs.java
index c324370573fc..bc7f1928d170 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/ServerArgs.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/ServerArgs.java
@@ -11,12 +11,13 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.SuppressForbidden;
 
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
 import java.nio.file.Path;
 
 /**
@@ -25,18 +26,13 @@
  * @param daemonize {@code true} if Elasticsearch should run as a daemon process, or {@code false} otherwise
  * @param quiet {@code false} if Elasticsearch should print log output to the console, {@code true} otherwise
  * @param pidFile absolute path to a file Elasticsearch should write its process id to, or {@code null} if no pid file should be written
- * @param keystorePassword the password for the Elasticsearch keystore
+ * @param secrets the provided secure settings implementation
  * @param nodeSettings the node settings read from {@code elasticsearch.yml}, the cli and the process environment
  * @param configDir the directory where {@code elasticsearch.yml} and other config exists
  */
-public record ServerArgs(
-    boolean daemonize,
-    boolean quiet,
-    Path pidFile,
-    SecureString keystorePassword,
-    Settings nodeSettings,
-    Path configDir
-) implements Writeable {
+public record ServerArgs(boolean daemonize, boolean quiet, Path pidFile, SecureSettings secrets, Settings nodeSettings, Path configDir)
+    implements
+        Writeable {
 
     /**
      * Arguments for running Elasticsearch.
@@ -44,12 +40,13 @@ public record ServerArgs(
      * @param daemonize {@code true} if Elasticsearch should run as a daemon process, or {@code false} otherwise
      * @param quiet {@code false} if Elasticsearch should print log output to the console, {@code true} otherwise
      * @param pidFile absolute path to a file Elasticsearch should write its process id to, or {@code null} if no pid file should be written
-     * @param keystorePassword the password for the Elasticsearch keystore
+     * @param secrets the provided secure settings implementation
      * @param nodeSettings the node settings read from {@code elasticsearch.yml}, the cli and the process environment
      * @param configDir the directory where {@code elasticsearch.yml} and other config exists
      */
     public ServerArgs {
         assert pidFile == null || pidFile.isAbsolute();
+        assert secrets != null;
     }
 
     /**
@@ -60,7 +57,7 @@ public ServerArgs(StreamInput in) throws IOException {
             in.readBoolean(),
             in.readBoolean(),
             readPidFile(in),
-            in.readSecureString(),
+            readSecureSettingsFromStream(in),
             Settings.readSettingsFromStream(in),
             resolvePath(in.readString())
         );
@@ -81,8 +78,19 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(daemonize);
         out.writeBoolean(quiet);
         out.writeOptionalString(pidFile == null ? null : pidFile.toString());
-        out.writeSecureString(keystorePassword);
+        out.writeString(secrets.getClass().getName());
+        secrets.writeTo(out);
         nodeSettings.writeTo(out);
         out.writeString(configDir.toString());
     }
+
+    private static SecureSettings readSecureSettingsFromStream(StreamInput in) throws IOException {
+        String className = in.readString();
+        try {
+            return (SecureSettings) Class.forName(className).getConstructor(StreamInput.class).newInstance(in);
+        } catch (NoSuchMethodException | ClassNotFoundException | InstantiationException | IllegalAccessException
+            | InvocationTargetException cfe) {
+            throw new IllegalArgumentException("Invalid secrets implementation [" + className + "]", cfe);
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/AbstractNamedDiffable.java b/server/src/main/java/org/elasticsearch/cluster/AbstractNamedDiffable.java
index f330bb037c12..d2ccf3d6a2e0 100644
--- a/server/src/main/java/org/elasticsearch/cluster/AbstractNamedDiffable.java
+++ b/server/src/main/java/org/elasticsearch/cluster/AbstractNamedDiffable.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.cluster;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -48,7 +48,7 @@ private static class CompleteNamedDiff<T extends NamedDiffable<T>> implements Na
          * is unnecessary.
          */
         @Nullable
-        private final Version minimalSupportedVersion;
+        private final TransportVersion minimalSupportedVersion;
 
         /**
          * Creates simple diff with changes
@@ -62,7 +62,7 @@ private static class CompleteNamedDiff<T extends NamedDiffable<T>> implements Na
         /**
          * Creates simple diff without changes
          */
-        CompleteNamedDiff(String name, Version minimalSupportedVersion) {
+        CompleteNamedDiff(String name, TransportVersion minimalSupportedVersion) {
             this.part = null;
             this.name = name;
             this.minimalSupportedVersion = minimalSupportedVersion;
@@ -108,7 +108,7 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
+        public TransportVersion getMinimalSupportedVersion() {
             assert minimalSupportedVersion != null : "shouldn't be called on the diff that was de-serialized from the stream";
             return minimalSupportedVersion;
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java
index bd26743690bb..c576389d72e2 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.cluster;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
@@ -42,8 +42,8 @@ public class ClusterInfo implements ToXContentFragment, Writeable {
 
     public static final ClusterInfo EMPTY = new ClusterInfo();
 
-    public static final Version DATA_SET_SIZE_SIZE_VERSION = Version.V_7_13_0;
-    public static final Version DATA_PATH_NEW_KEY_VERSION = Version.V_8_6_0;
+    public static final TransportVersion DATA_SET_SIZE_SIZE_VERSION = TransportVersion.V_7_13_0;
+    public static final TransportVersion DATA_PATH_NEW_KEY_VERSION = TransportVersion.V_8_6_0;
 
     private final Map<String, DiskUsage> leastAvailableSpaceUsage;
     private final Map<String, DiskUsage> mostAvailableSpaceUsage;
@@ -76,9 +76,9 @@ public ClusterInfo(
         Map<NodeAndPath, ReservedSpace> reservedSpace
     ) {
         this.leastAvailableSpaceUsage = Map.copyOf(leastAvailableSpaceUsage);
+        this.mostAvailableSpaceUsage = Map.copyOf(mostAvailableSpaceUsage);
         this.shardSizes = Map.copyOf(shardSizes);
         this.shardDataSetSizes = Map.copyOf(shardDataSetSizes);
-        this.mostAvailableSpaceUsage = Map.copyOf(mostAvailableSpaceUsage);
         this.dataPath = Map.copyOf(dataPath);
         this.reservedSpace = Map.copyOf(reservedSpace);
     }
@@ -87,17 +87,17 @@ public ClusterInfo(StreamInput in) throws IOException {
         this.leastAvailableSpaceUsage = in.readImmutableMap(StreamInput::readString, DiskUsage::new);
         this.mostAvailableSpaceUsage = in.readImmutableMap(StreamInput::readString, DiskUsage::new);
         this.shardSizes = in.readImmutableMap(StreamInput::readString, StreamInput::readLong);
-        if (in.getVersion().onOrAfter(DATA_SET_SIZE_SIZE_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(DATA_SET_SIZE_SIZE_VERSION)) {
             this.shardDataSetSizes = in.readImmutableMap(ShardId::new, StreamInput::readLong);
         } else {
             this.shardDataSetSizes = Map.of();
         }
-        if (in.getVersion().onOrAfter(DATA_PATH_NEW_KEY_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(DATA_PATH_NEW_KEY_VERSION)) {
             this.dataPath = in.readImmutableMap(NodeAndShard::new, StreamInput::readString);
         } else {
             this.dataPath = in.readImmutableMap(nested -> NodeAndShard.from(new ShardRouting(nested)), StreamInput::readString);
         }
-        if (in.getVersion().onOrAfter(StoreStats.RESERVED_BYTES_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(StoreStats.RESERVED_BYTES_VERSION)) {
             this.reservedSpace = in.readImmutableMap(NodeAndPath::new, ReservedSpace::new);
         } else {
             this.reservedSpace = Map.of();
@@ -109,15 +109,15 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeMap(this.leastAvailableSpaceUsage, StreamOutput::writeString, (o, v) -> v.writeTo(o));
         out.writeMap(this.mostAvailableSpaceUsage, StreamOutput::writeString, (o, v) -> v.writeTo(o));
         out.writeMap(this.shardSizes, StreamOutput::writeString, (o, v) -> o.writeLong(v == null ? -1 : v));
-        if (out.getVersion().onOrAfter(DATA_SET_SIZE_SIZE_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(DATA_SET_SIZE_SIZE_VERSION)) {
             out.writeMap(this.shardDataSetSizes, (o, s) -> s.writeTo(o), StreamOutput::writeLong);
         }
-        if (out.getVersion().onOrAfter(DATA_PATH_NEW_KEY_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(DATA_PATH_NEW_KEY_VERSION)) {
             out.writeMap(this.dataPath, (o, k) -> k.writeTo(o), StreamOutput::writeString);
         } else {
             out.writeMap(this.dataPath, (o, k) -> createFakeShardRoutingFromNodeAndShard(k).writeTo(o), StreamOutput::writeString);
         }
-        if (out.getVersion().onOrAfter(StoreStats.RESERVED_BYTES_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(StoreStats.RESERVED_BYTES_VERSION)) {
             out.writeMap(this.reservedSpace);
         }
     }
@@ -133,7 +133,8 @@ private static ShardRouting createFakeShardRoutingFromNodeAndShard(NodeAndShard
             nodeAndShard.shardId,
             true,
             RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(REINITIALIZED, "fake")
+            new UnassignedInfo(REINITIALIZED, "fake"),
+            ShardRouting.Role.DEFAULT // ok, this is only used prior to DATA_PATH_NEW_KEY_VERSION which has no other roles
         ).initialize(nodeAndShard.nodeId, null, 0L).moveToStarted(0L);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterInfoSimulator.java b/server/src/main/java/org/elasticsearch/cluster/ClusterInfoSimulator.java
index 0485e06f6663..0420611b8edf 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterInfoSimulator.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterInfoSimulator.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.cluster;
 
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.common.util.CopyOnFirstWriteMap;
 import org.elasticsearch.index.shard.ShardId;
 
 import java.util.HashMap;
@@ -19,14 +20,14 @@ public class ClusterInfoSimulator {
 
     private final Map<String, DiskUsage> leastAvailableSpaceUsage;
     private final Map<String, DiskUsage> mostAvailableSpaceUsage;
-    private final Map<String, Long> shardSizes;
+    private final CopyOnFirstWriteMap<String, Long> shardSizes;
     private final Map<ShardId, Long> shardDataSetSizes;
     private final Map<ClusterInfo.NodeAndShard, String> dataPath;
 
     public ClusterInfoSimulator(ClusterInfo clusterInfo) {
         this.leastAvailableSpaceUsage = new HashMap<>(clusterInfo.getNodeLeastAvailableDiskUsages());
         this.mostAvailableSpaceUsage = new HashMap<>(clusterInfo.getNodeMostAvailableDiskUsages());
-        this.shardSizes = new HashMap<>(clusterInfo.shardSizes);
+        this.shardSizes = new CopyOnFirstWriteMap<>(clusterInfo.shardSizes);
         this.shardDataSetSizes = Map.copyOf(clusterInfo.shardDataSetSizes);
         this.dataPath = Map.copyOf(clusterInfo.dataPath);
     }
@@ -101,6 +102,13 @@ private static long withinRange(long min, long max, long value) {
     }
 
     public ClusterInfo getClusterInfo() {
-        return new ClusterInfo(leastAvailableSpaceUsage, mostAvailableSpaceUsage, shardSizes, shardDataSetSizes, dataPath, Map.of());
+        return new ClusterInfo(
+            leastAvailableSpaceUsage,
+            mostAvailableSpaceUsage,
+            shardSizes.toImmutableMap(),
+            shardDataSetSizes,
+            dataPath,
+            Map.of()
+        );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index ffe8000480a2..d6e8c2ed8b9c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -25,7 +25,8 @@
 import org.elasticsearch.cluster.metadata.NodesShutdownMetadata;
 import org.elasticsearch.cluster.metadata.RepositoriesMetadata;
 import org.elasticsearch.cluster.routing.DelayedAllocationService;
-import org.elasticsearch.cluster.routing.RerouteService;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingRoleStrategy;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
@@ -114,6 +115,7 @@ public class ClusterModule extends AbstractModule {
     // pkg private for tests
     final Collection<AllocationDecider> deciderList;
     final ShardsAllocator shardsAllocator;
+    private final ShardRoutingRoleStrategy shardRoutingRoleStrategy;
 
     public ClusterModule(
         Settings settings,
@@ -123,7 +125,6 @@ public ClusterModule(
         SnapshotsInfoService snapshotsInfoService,
         ThreadPool threadPool,
         SystemIndices systemIndices,
-        Supplier<RerouteService> rerouteServiceSupplier,
         WriteLoadForecaster writeLoadForecaster
     ) {
         this.clusterPlugins = clusterPlugins;
@@ -136,15 +137,44 @@ public ClusterModule(
             clusterPlugins,
             clusterService,
             this::reconcile,
-            writeLoadForecaster,
-            clusterInfoService
+            writeLoadForecaster
         );
         this.clusterService = clusterService;
         this.indexNameExpressionResolver = new IndexNameExpressionResolver(threadPool.getThreadContext(), systemIndices);
-        this.allocationService = new AllocationService(allocationDeciders, shardsAllocator, clusterInfoService, snapshotsInfoService);
+        this.shardRoutingRoleStrategy = getShardRoutingRoleStrategy(clusterPlugins);
+        this.allocationService = new AllocationService(
+            allocationDeciders,
+            shardsAllocator,
+            clusterInfoService,
+            snapshotsInfoService,
+            shardRoutingRoleStrategy
+        );
         this.metadataDeleteIndexService = new MetadataDeleteIndexService(settings, clusterService, allocationService);
     }
 
+    static ShardRoutingRoleStrategy getShardRoutingRoleStrategy(List<ClusterPlugin> clusterPlugins) {
+        final var strategies = clusterPlugins.stream().map(ClusterPlugin::getShardRoutingRoleStrategy).filter(Objects::nonNull).toList();
+        return switch (strategies.size()) {
+            case 0 -> new ShardRoutingRoleStrategy() {
+
+                // NOTE: this is deliberately an anonymous class to avoid any possibility of using this DEFAULT-only strategy when a plugin
+                // has injected a different strategy.
+
+                @Override
+                public ShardRouting.Role newReplicaRole() {
+                    return ShardRouting.Role.DEFAULT;
+                }
+
+                @Override
+                public ShardRouting.Role newEmptyRole(int copyIndex) {
+                    return ShardRouting.Role.DEFAULT;
+                }
+            };
+            case 1 -> strategies.get(0);
+            default -> throw new IllegalArgumentException("multiple plugins define shard role strategies, which is not permitted");
+        };
+    }
+
     private ClusterState reconcile(ClusterState clusterState, Consumer<RoutingAllocation> routingAllocationConsumer) {
         return allocationService.executeWithRoutingAllocation(clusterState, "reconcile-desired-balance", routingAllocationConsumer);
     }
@@ -313,19 +343,19 @@ public static Collection<AllocationDecider> createAllocationDeciders(
         addAllocationDecider(deciders, new ResizeAllocationDecider());
         addAllocationDecider(deciders, new ReplicaAfterPrimaryActiveAllocationDecider());
         addAllocationDecider(deciders, new RebalanceOnlyWhenActiveAllocationDecider());
-        addAllocationDecider(deciders, new ClusterRebalanceAllocationDecider(settings, clusterSettings));
-        addAllocationDecider(deciders, new ConcurrentRebalanceAllocationDecider(settings, clusterSettings));
-        addAllocationDecider(deciders, new EnableAllocationDecider(settings, clusterSettings));
+        addAllocationDecider(deciders, new ClusterRebalanceAllocationDecider(clusterSettings));
+        addAllocationDecider(deciders, new ConcurrentRebalanceAllocationDecider(clusterSettings));
+        addAllocationDecider(deciders, new EnableAllocationDecider(clusterSettings));
         addAllocationDecider(deciders, new NodeVersionAllocationDecider());
         addAllocationDecider(deciders, new SnapshotInProgressAllocationDecider());
         addAllocationDecider(deciders, new RestoreInProgressAllocationDecider());
         addAllocationDecider(deciders, new NodeShutdownAllocationDecider());
         addAllocationDecider(deciders, new NodeReplacementAllocationDecider());
         addAllocationDecider(deciders, new FilterAllocationDecider(settings, clusterSettings));
-        addAllocationDecider(deciders, new SameShardAllocationDecider(settings, clusterSettings));
+        addAllocationDecider(deciders, new SameShardAllocationDecider(clusterSettings));
         addAllocationDecider(deciders, new DiskThresholdDecider(settings, clusterSettings));
-        addAllocationDecider(deciders, new ThrottlingAllocationDecider(settings, clusterSettings));
-        addAllocationDecider(deciders, new ShardsLimitAllocationDecider(settings, clusterSettings));
+        addAllocationDecider(deciders, new ThrottlingAllocationDecider(clusterSettings));
+        addAllocationDecider(deciders, new ShardsLimitAllocationDecider(clusterSettings));
         addAllocationDecider(deciders, new AwarenessAllocationDecider(settings, clusterSettings));
 
         clusterPlugins.stream()
@@ -349,15 +379,15 @@ private static ShardsAllocator createShardsAllocator(
         List<ClusterPlugin> clusterPlugins,
         ClusterService clusterService,
         DesiredBalanceReconcilerAction reconciler,
-        WriteLoadForecaster writeLoadForecaster,
-        ClusterInfoService clusterInfoService
+        WriteLoadForecaster writeLoadForecaster
     ) {
         Map<String, Supplier<ShardsAllocator>> allocators = new HashMap<>();
-        allocators.put(BALANCED_ALLOCATOR, () -> new BalancedShardsAllocator(settings, clusterSettings, writeLoadForecaster));
+        allocators.put(BALANCED_ALLOCATOR, () -> new BalancedShardsAllocator(clusterSettings, writeLoadForecaster));
         allocators.put(
             DESIRED_BALANCE_ALLOCATOR,
             () -> new DesiredBalanceShardsAllocator(
-                new BalancedShardsAllocator(settings, clusterSettings, writeLoadForecaster),
+                clusterSettings,
+                new BalancedShardsAllocator(clusterSettings, writeLoadForecaster),
                 threadPool,
                 clusterService,
                 reconciler
@@ -405,6 +435,7 @@ protected void configure() {
         bind(TaskResultsService.class).asEagerSingleton();
         bind(AllocationDeciders.class).toInstance(allocationDeciders);
         bind(ShardsAllocator.class).toInstance(shardsAllocator);
+        bind(ShardRoutingRoleStrategy.class).toInstance(shardRoutingRoleStrategy);
     }
 
     public void setExistingShardsAllocators(GatewayAllocator gatewayAllocator) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
index 3ece7e345ffd..165f2413f5f9 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.cluster;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlocks;
@@ -21,12 +21,9 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
-import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
-import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterApplierService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.MasterService;
@@ -35,6 +32,7 @@
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -43,19 +41,23 @@
 import org.elasticsearch.common.io.stream.VersionedNamedWriteable;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.xcontent.ToXContentFragment;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContent;
-import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.Executor;
 import java.util.function.Consumer;
+import java.util.function.Function;
 
 import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK;
 
@@ -92,7 +94,7 @@
  * <p>
  * Cluster state updates can be used to trigger various actions via a {@link ClusterStateListener} rather than using a timer.
  * <p>
- * Implements {@link ToXContentFragment} to be exposed in REST APIs (e.g. {@code GET _cluster/state} and {@code POST _cluster/reroute}) and
+ * Implements {@link ChunkedToXContent} to be exposed in REST APIs (e.g. {@code GET _cluster/state} and {@code POST _cluster/reroute}) and
  * to be indexed by monitoring, mostly just for diagnostics purposes. The {@link XContent} representation does not need to be 100% faithful
  * since we never reconstruct a cluster state from its XContent representation, but the more faithful it is the more useful it is for
  * diagnostics. Note that the {@link XContent} representation of the {@link Metadata} portion does have to be faithful (in {@link
@@ -101,11 +103,11 @@
  * Security-sensitive data such as passwords or private keys should not be stored in the cluster state, since the contents of the cluster
  * state are exposed in various APIs.
  */
-public class ClusterState implements ToXContentFragment, Diffable<ClusterState> {
+public class ClusterState implements ChunkedToXContent, Diffable<ClusterState> {
 
     public static final ClusterState EMPTY_STATE = builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).build();
 
-    public interface Custom extends NamedDiffable<Custom>, ToXContentFragment {
+    public interface Custom extends NamedDiffable<Custom>, ChunkedToXContent {
 
         /**
          * Returns <code>true</code> iff this {@link Custom} is private to the cluster and should never be send to a client.
@@ -121,7 +123,7 @@ default boolean isPrivate() {
          * the more faithful it is the more useful it is for diagnostics.
          */
         @Override
-        XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException;
+        Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params);
     }
 
     private static final NamedDiffableValueSerializer<Custom> CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class);
@@ -517,114 +519,131 @@ public String toString() {
         }
     }
 
+    private static <T> Iterator<ToXContent> chunkedSection(
+        boolean condition,
+        ToXContent before,
+        Iterator<T> items,
+        Function<T, Iterator<ToXContent>> fn,
+        ToXContent after
+    ) {
+        return condition
+            ? Iterators.concat(Iterators.single(before), Iterators.flatMap(items, fn::apply), Iterators.single(after))
+            : Collections.emptyIterator();
+    }
+
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        EnumSet<Metric> metrics = Metric.parseString(params.param("metric", "_all"), true);
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
+        final var metrics = Metric.parseString(outerParams.param("metric", "_all"), true);
 
-        // always provide the cluster_uuid as part of the top-level response (also part of the metadata response)
-        builder.field("cluster_uuid", metadata().clusterUUID());
+        return Iterators.concat(
 
-        if (metrics.contains(Metric.VERSION)) {
-            builder.field("version", version);
-            builder.field("state_uuid", stateUUID);
-        }
+            // header chunk
+            Iterators.single(((builder, params) -> {
+                // always provide the cluster_uuid as part of the top-level response (also part of the metadata response)
+                builder.field("cluster_uuid", metadata().clusterUUID());
 
-        if (metrics.contains(Metric.MASTER_NODE)) {
-            builder.field("master_node", nodes().getMasterNodeId());
-        }
-
-        if (metrics.contains(Metric.BLOCKS)) {
-            builder.startObject("blocks");
+                // state version info
+                if (metrics.contains(Metric.VERSION)) {
+                    builder.field("version", version);
+                    builder.field("state_uuid", stateUUID);
+                }
 
-            if (blocks().global().isEmpty() == false) {
-                builder.startObject("global");
-                for (ClusterBlock block : blocks().global()) {
-                    block.toXContent(builder, params);
+                // master node
+                if (metrics.contains(Metric.MASTER_NODE)) {
+                    builder.field("master_node", nodes().getMasterNodeId());
                 }
-                builder.endObject();
-            }
 
-            if (blocks().indices().isEmpty() == false) {
-                builder.startObject("indices");
-                for (Map.Entry<String, Set<ClusterBlock>> entry : blocks().indices().entrySet()) {
-                    builder.startObject(entry.getKey());
-                    for (ClusterBlock block : entry.getValue()) {
+                return builder;
+            })),
+
+            // blocks
+            chunkedSection(metrics.contains(Metric.BLOCKS), (builder, params) -> {
+                builder.startObject("blocks");
+                if (blocks().global().isEmpty() == false) {
+                    builder.startObject("global");
+                    for (ClusterBlock block : blocks().global()) {
                         block.toXContent(builder, params);
                     }
                     builder.endObject();
                 }
-                builder.endObject();
-            }
-
-            builder.endObject();
-        }
-
-        // nodes
-        if (metrics.contains(Metric.NODES)) {
-            builder.startObject("nodes");
-            for (DiscoveryNode node : nodes) {
-                node.toXContent(builder, params);
-            }
-            builder.endObject();
-        }
-
-        // meta data
-        if (metrics.contains(Metric.METADATA)) {
-            metadata.toXContent(builder, params);
-        }
-
-        // routing table
-        if (metrics.contains(Metric.ROUTING_TABLE)) {
-            builder.startObject("routing_table");
-            builder.startObject("indices");
-            for (IndexRoutingTable indexRoutingTable : routingTable()) {
-                builder.startObject(indexRoutingTable.getIndex().getName());
-                builder.startObject("shards");
-                for (int shardId = 0; shardId < indexRoutingTable.size(); shardId++) {
-                    IndexShardRoutingTable indexShardRoutingTable = indexRoutingTable.shard(shardId);
-                    builder.startArray(Integer.toString(indexShardRoutingTable.shardId().id()));
-                    for (int copy = 0; copy < indexShardRoutingTable.size(); copy++) {
-                        indexShardRoutingTable.shard(copy).toXContent(builder, params);
-                    }
-                    builder.endArray();
+                if (blocks().indices().isEmpty() == false) {
+                    builder.startObject("indices");
                 }
-                builder.endObject();
-                builder.endObject();
-            }
-            builder.endObject();
-            builder.endObject();
-        }
-
-        // routing nodes
-        if (metrics.contains(Metric.ROUTING_NODES)) {
-            builder.startObject("routing_nodes");
-            builder.startArray("unassigned");
-            for (ShardRouting shardRouting : getRoutingNodes().unassigned()) {
-                shardRouting.toXContent(builder, params);
-            }
-            builder.endArray();
-
-            builder.startObject("nodes");
-            for (RoutingNode routingNode : getRoutingNodes()) {
-                builder.startArray(routingNode.nodeId() == null ? "null" : routingNode.nodeId());
-                for (ShardRouting shardRouting : routingNode) {
-                    shardRouting.toXContent(builder, params);
+                return builder;
+            }, blocks.indices().entrySet().iterator(), entry -> Iterators.single((builder, params) -> {
+                builder.startObject(entry.getKey());
+                for (ClusterBlock block : entry.getValue()) {
+                    block.toXContent(builder, params);
                 }
-                builder.endArray();
-            }
-            builder.endObject();
-
-            builder.endObject();
-        }
-        if (metrics.contains(Metric.CUSTOMS)) {
-            for (Map.Entry<String, Custom> cursor : customs.entrySet()) {
-                builder.startObject(cursor.getKey());
-                cursor.getValue().toXContent(builder, params);
-                builder.endObject();
-            }
-        }
-
-        return builder;
+                return builder.endObject();
+            }), (builder, params) -> {
+                if (blocks().indices().isEmpty() == false) {
+                    builder.endObject();
+                }
+                return builder.endObject();
+            }),
+
+            // nodes
+            chunkedSection(
+                metrics.contains(Metric.NODES),
+                (builder, params) -> builder.startObject("nodes"),
+                nodes.iterator(),
+                Iterators::single,
+                (builder, params) -> builder.endObject()
+            ),
+
+            // metadata
+            metrics.contains(Metric.METADATA) ? metadata.toXContentChunked(outerParams) : Collections.emptyIterator(),
+
+            // routing table
+            chunkedSection(
+                metrics.contains(Metric.ROUTING_TABLE),
+                (builder, params) -> builder.startObject("routing_table").startObject("indices"),
+                routingTable().iterator(),
+                indexRoutingTable -> Iterators.single((builder, params) -> {
+                    builder.startObject(indexRoutingTable.getIndex().getName());
+                    builder.startObject("shards");
+                    for (int shardId = 0; shardId < indexRoutingTable.size(); shardId++) {
+                        IndexShardRoutingTable indexShardRoutingTable = indexRoutingTable.shard(shardId);
+                        builder.startArray(Integer.toString(indexShardRoutingTable.shardId().id()));
+                        for (int copy = 0; copy < indexShardRoutingTable.size(); copy++) {
+                            indexShardRoutingTable.shard(copy).toXContent(builder, params);
+                        }
+                        builder.endArray();
+                    }
+                    return builder.endObject().endObject();
+                }),
+                (builder, params) -> builder.endObject().endObject()
+            ),
+
+            // routing nodes
+            chunkedSection(
+                metrics.contains(Metric.ROUTING_NODES),
+                (builder, params) -> builder.startObject("routing_nodes").startArray("unassigned"),
+                getRoutingNodes().unassigned().iterator(),
+                Iterators::single,
+                (builder, params) -> builder.endArray() // no endObject() here, continued in next chunkedSection()
+            ),
+            chunkedSection(
+                metrics.contains(Metric.ROUTING_NODES),
+                (builder, params) -> builder.startObject("nodes"),
+                getRoutingNodes().iterator(),
+                routingNode -> Iterators.concat(
+                    ChunkedToXContentHelper.startArray(routingNode.nodeId() == null ? "null" : routingNode.nodeId()),
+                    routingNode.iterator(),
+                    ChunkedToXContentHelper.endArray()
+                ),
+                (builder, params) -> builder.endObject().endObject()
+            ),
+
+            // customs
+            metrics.contains(Metric.CUSTOMS)
+                ? Iterators.flatMap(
+                    customs.entrySet().iterator(),
+                    cursor -> ChunkedToXContentHelper.wrapWithObject(cursor.getKey(), cursor.getValue().toXContentChunked(outerParams))
+                )
+                : Collections.emptyIterator()
+        );
     }
 
     public static Builder builder(ClusterName clusterName) {
@@ -822,7 +841,7 @@ public static ClusterState readFrom(StreamInput in, DiscoveryNode localNode) thr
             Custom customIndexMetadata = in.readNamedWriteable(Custom.class);
             builder.putCustom(customIndexMetadata.getWriteableName(), customIndexMetadata);
         }
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             in.readVInt(); // used to be minimumMasterNodesOnPublishingMaster, which was used in 7.x for BWC with 6.x
         }
         return builder.build();
@@ -838,7 +857,7 @@ public void writeTo(StreamOutput out) throws IOException {
         nodes.writeTo(out);
         blocks.writeTo(out);
         VersionedNamedWriteable.writeVersionedWritables(out, customs);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeVInt(-1); // used to be minimumMasterNodesOnPublishingMaster, which was used in 7.x for BWC with 6.x
         }
     }
@@ -885,7 +904,7 @@ private static class ClusterStateDiff implements Diff<ClusterState> {
             metadata = Metadata.readDiffFrom(in);
             blocks = ClusterBlocks.readDiffFrom(in);
             customs = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER);
-            if (in.getVersion().before(Version.V_8_0_0)) {
+            if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 in.readVInt(); // used to be minimumMasterNodesOnPublishingMaster, which was used in 7.x for BWC with 6.x
             }
         }
@@ -901,7 +920,7 @@ public void writeTo(StreamOutput out) throws IOException {
             metadata.writeTo(out);
             blocks.writeTo(out);
             customs.writeTo(out);
-            if (out.getVersion().before(Version.V_8_0_0)) {
+            if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 out.writeVInt(-1); // used to be minimumMasterNodesOnPublishingMaster, which was used in 7.x for BWC with 6.x
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java b/server/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
index 02192b01991c..27de00e87966 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
@@ -21,7 +21,8 @@
  */
 public interface ClusterStateTaskExecutor<T extends ClusterStateTaskListener> {
     /**
-     * Update the cluster state based on the current state and the given tasks. Return the *same instance* if no update should be published.
+     * Update the cluster state based on the current state and the given tasks. Return {@code batchExecutionContext.initialState()} to avoid
+     * publishing any update.
      * <p>
      * If this method throws an exception then the cluster state is unchanged and every task's {@link ClusterStateTaskListener#onFailure}
      * method is called.
@@ -30,8 +31,15 @@ public interface ClusterStateTaskExecutor<T extends ClusterStateTaskListener> {
      * This works ok but beware that constructing a whole new {@link ClusterState} can be somewhat expensive, and there may sometimes be
      * surprisingly many tasks to process in the batch. If it's possible to accumulate the effects of the tasks at a lower level then you
      * should do that instead.
+     * <p>
+     * Returning {@code batchExecutionContext.initialState()} is an important and useful optimisation in most cases, but note that this
+     * fast-path exposes APIs to the risk of stale reads in the vicinity of a master failover: a node {@code N} that handles such a no-op
+     * task batch does not verify with its peers that it's still the master, and if it's not the master then another node {@code M} may
+     * already have become master and updated the state in a way that would be inconsistent with the response that {@code N} sends back to
+     * clients.
      *
-     * @return The resulting cluster state after executing all the tasks. If {code initialState} is returned then no update is published.
+     * @return The resulting cluster state after executing all the tasks. If {code batchExecutionContext.initialState()} is returned then no
+     * update is published.
      */
     ClusterState execute(BatchExecutionContext<T> batchExecutionContext) throws Exception;
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/DiffableUtils.java b/server/src/main/java/org/elasticsearch/cluster/DiffableUtils.java
index fc23db6015fa..2bd7f971c9a5 100644
--- a/server/src/main/java/org/elasticsearch/cluster/DiffableUtils.java
+++ b/server/src/main/java/org/elasticsearch/cluster/DiffableUtils.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.cluster;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -371,7 +371,7 @@ public List<Map.Entry<K, T>> getUpserts() {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeCollection(deletes, (o, v) -> keySerializer.writeKey(v, o));
-            Version version = out.getVersion();
+            TransportVersion version = out.getTransportVersion();
             // filter out custom states not supported by the other node
             int diffCount = 0;
             for (Map.Entry<K, Diff<T>> diff : diffs) {
@@ -499,14 +499,14 @@ public interface ValueSerializer<K, V> {
         /**
          * Whether this serializer supports the version of the output stream
          */
-        default boolean supportsVersion(Diff<V> value, Version version) {
+        default boolean supportsVersion(Diff<V> value, TransportVersion version) {
             return true;
         }
 
         /**
          * Whether this serializer supports the version of the output stream
          */
-        default boolean supportsVersion(V value, Version version) {
+        default boolean supportsVersion(V value, TransportVersion version) {
             return true;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
index c36ee576923a..40f3f344c46d 100644
--- a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.action.support.ThreadedActionListener;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.block.ClusterBlockException;
@@ -32,7 +33,6 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.CountDown;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.StoreStats;
@@ -161,7 +161,7 @@ public void clusterChanged(ClusterChangedEvent event) {
     private class AsyncRefresh {
 
         private final List<ActionListener<ClusterInfo>> thisRefreshListeners;
-        private final CountDown countDown = new CountDown(2);
+        private final RefCountingRunnable fetchRefs = new RefCountingRunnable(this::callListeners);
 
         AsyncRefresh(List<ActionListener<ClusterInfo>> thisRefreshListeners) {
             this.thisRefreshListeners = thisRefreshListeners;
@@ -177,15 +177,15 @@ void execute() {
                 return;
             }
 
-            assert countDown.isCountedDown() == false;
             logger.trace("starting async refresh");
 
-            try (var ignored = threadPool.getThreadContext().clearTraceContext()) {
-                fetchNodeStats();
-            }
-
-            try (var ignored = threadPool.getThreadContext().clearTraceContext()) {
-                fetchIndicesStats();
+            try (var ignoredRefs = fetchRefs) {
+                try (var ignored = threadPool.getThreadContext().clearTraceContext()) {
+                    fetchNodeStats();
+                }
+                try (var ignored = threadPool.getThreadContext().clearTraceContext()) {
+                    fetchIndicesStats();
+                }
             }
         }
 
@@ -200,10 +200,8 @@ private void fetchIndicesStats() {
                 .stats(
                     indicesStatsRequest,
                     new ThreadedActionListener<>(
-                        logger,
-                        threadPool,
-                        ThreadPool.Names.MANAGEMENT,
-                        ActionListener.runAfter(new ActionListener<>() {
+                        threadPool.executor(ThreadPool.Names.MANAGEMENT),
+                        ActionListener.releaseAfter(new ActionListener<>() {
                             @Override
                             public void onResponse(IndicesStatsResponse indicesStatsResponse) {
                                 logger.trace("received indices stats response");
@@ -277,8 +275,7 @@ public void onFailure(Exception e) {
                                 }
                                 indicesStatsSummary = IndicesStatsSummary.EMPTY;
                             }
-                        }, this::onStatsProcessed),
-                        false
+                        }, fetchRefs.acquire())
                     )
                 );
         }
@@ -288,7 +285,7 @@ private void fetchNodeStats() {
             nodesStatsRequest.clear();
             nodesStatsRequest.addMetric(NodesStatsRequest.Metric.FS.metricName());
             nodesStatsRequest.timeout(fetchTimeout);
-            client.admin().cluster().nodesStats(nodesStatsRequest, ActionListener.runAfter(new ActionListener<>() {
+            client.admin().cluster().nodesStats(nodesStatsRequest, ActionListener.releaseAfter(new ActionListener<>() {
                 @Override
                 public void onResponse(NodesStatsResponse nodesStatsResponse) {
                     logger.trace("received node stats response");
@@ -318,18 +315,12 @@ public void onFailure(Exception e) {
                     leastAvailableSpaceUsages = Map.of();
                     mostAvailableSpaceUsages = Map.of();
                 }
-            }, this::onStatsProcessed));
-        }
-
-        private void onStatsProcessed() {
-            if (countDown.countDown()) {
-                logger.trace("stats all received, computing cluster info and notifying listeners");
-                callListeners();
-            }
+            }, fetchRefs.acquire()));
         }
 
         private void callListeners() {
             try {
+                logger.trace("stats all received, computing cluster info and notifying listeners");
                 final ClusterInfo clusterInfo = getClusterInfo();
                 boolean anyListeners = false;
                 for (final Consumer<ClusterInfo> listener : listeners) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/NamedDiff.java b/server/src/main/java/org/elasticsearch/cluster/NamedDiff.java
index 598f43112f23..bda313b8bb6f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/NamedDiff.java
+++ b/server/src/main/java/org/elasticsearch/cluster/NamedDiff.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.cluster;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 
 /**
@@ -18,6 +18,6 @@ public interface NamedDiff<T extends Diffable<T>> extends Diff<T>, NamedWriteabl
     /**
      * The minimal version of the recipient this custom object can be sent to
      */
-    Version getMinimalSupportedVersion();
+    TransportVersion getMinimalSupportedVersion();
 
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/NamedDiffableValueSerializer.java b/server/src/main/java/org/elasticsearch/cluster/NamedDiffableValueSerializer.java
index 50c1d068aadf..d77afcd86a19 100644
--- a/server/src/main/java/org/elasticsearch/cluster/NamedDiffableValueSerializer.java
+++ b/server/src/main/java/org/elasticsearch/cluster/NamedDiffableValueSerializer.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.cluster;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 
 import java.io.IOException;
@@ -30,12 +30,12 @@ public T read(StreamInput in, String key) throws IOException {
     }
 
     @Override
-    public boolean supportsVersion(Diff<T> value, Version version) {
+    public boolean supportsVersion(Diff<T> value, TransportVersion version) {
         return version.onOrAfter(((NamedDiff<?>) value).getMinimalSupportedVersion());
     }
 
     @Override
-    public boolean supportsVersion(T value, Version version) {
+    public boolean supportsVersion(T value, TransportVersion version) {
         return version.onOrAfter(value.getMinimalSupportedVersion());
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java b/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
index f88ada956582..8809a883bd97 100644
--- a/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
@@ -11,7 +11,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.GroupedActionListener;
+import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.cluster.coordination.FollowersChecker;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -28,7 +28,6 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -97,31 +96,25 @@ public void connectToNodes(DiscoveryNodes discoveryNodes, Runnable onCompletion)
             return;
         }
 
-        final GroupedActionListener<Void> listener = new GroupedActionListener<>(
-            ActionListener.wrap(onCompletion),
-            discoveryNodes.getSize()
-        );
-
         final List<Runnable> runnables = new ArrayList<>(discoveryNodes.getSize());
-        synchronized (mutex) {
-            for (final DiscoveryNode discoveryNode : discoveryNodes) {
-                ConnectionTarget connectionTarget = targetsByNode.get(discoveryNode);
-                final boolean isNewNode = connectionTarget == null;
-                if (isNewNode) {
-                    connectionTarget = new ConnectionTarget(discoveryNode);
-                    targetsByNode.put(discoveryNode, connectionTarget);
-                }
+        try (var refs = new RefCountingRunnable(onCompletion)) {
+            synchronized (mutex) {
+                for (final DiscoveryNode discoveryNode : discoveryNodes) {
+                    ConnectionTarget connectionTarget = targetsByNode.get(discoveryNode);
+                    final boolean isNewNode = connectionTarget == null;
+                    if (isNewNode) {
+                        connectionTarget = new ConnectionTarget(discoveryNode);
+                        targetsByNode.put(discoveryNode, connectionTarget);
+                    }
 
-                if (isNewNode) {
-                    logger.debug("connecting to {}", discoveryNode);
-                    runnables.add(
-                        connectionTarget.connect(ActionListener.runAfter(listener, () -> logger.debug("connected to {}", discoveryNode)))
-                    );
-                } else {
-                    // known node, try and ensure it's connected but do not wait
-                    logger.trace("checking connection to existing node [{}]", discoveryNode);
-                    runnables.add(connectionTarget.connect(null));
-                    runnables.add(() -> listener.onResponse(null));
+                    if (isNewNode) {
+                        logger.debug("connecting to {}", discoveryNode);
+                        runnables.add(connectionTarget.connect(refs.acquire()));
+                    } else {
+                        // known node, try and ensure it's connected but do not wait
+                        logger.trace("checking connection to existing node [{}]", discoveryNode);
+                        runnables.add(connectionTarget.connect(null));
+                    }
                 }
             }
         }
@@ -153,18 +146,11 @@ public void disconnectFromNodesExcept(DiscoveryNodes discoveryNodes) {
      */
     void ensureConnections(Runnable onCompletion) {
         final List<Runnable> runnables = new ArrayList<>();
-        synchronized (mutex) {
-            final Collection<ConnectionTarget> connectionTargets = targetsByNode.values();
-            if (connectionTargets.isEmpty()) {
-                runnables.add(onCompletion);
-            } else {
+        try (var refs = new RefCountingRunnable(onCompletion)) {
+            synchronized (mutex) {
                 logger.trace("ensureConnections: {}", targetsByNode);
-                final GroupedActionListener<Void> listener = new GroupedActionListener<>(
-                    ActionListener.wrap(onCompletion),
-                    connectionTargets.size()
-                );
-                for (final ConnectionTarget connectionTarget : connectionTargets) {
-                    runnables.add(connectionTarget.connect(listener));
+                for (ConnectionTarget connectionTarget : targetsByNode.values()) {
+                    runnables.add(connectionTarget.connect(refs.acquire()));
                 }
             }
         }
@@ -225,6 +211,13 @@ private class ConnectionTarget {
         private final AtomicInteger consecutiveFailureCount = new AtomicInteger();
         private final AtomicReference<Releasable> connectionRef = new AtomicReference<>();
 
+        // all access to these fields is synchronized
+        private List<Releasable> pendingRefs;
+        private boolean connectionInProgress;
+
+        // placeholder listener for a fire-and-forget connection attempt
+        private static final List<Releasable> NOOP = List.of();
+
         ConnectionTarget(DiscoveryNode discoveryNode) {
             this.discoveryNode = discoveryNode;
         }
@@ -233,59 +226,97 @@ private void setConnectionRef(Releasable connectionReleasable) {
             Releasables.close(connectionRef.getAndSet(connectionReleasable));
         }
 
-        Runnable connect(ActionListener<Void> listener) {
+        Runnable connect(Releasable onCompletion) {
             return () -> {
-                final boolean alreadyConnected = transportService.nodeConnected(discoveryNode);
+                registerRef(onCompletion);
+                doConnect();
+            };
+        }
+
+        private synchronized void registerRef(Releasable ref) {
+            if (ref == null) {
+                pendingRefs = pendingRefs == null ? NOOP : pendingRefs;
+                return;
+            }
+
+            if (pendingRefs == null || pendingRefs == NOOP) {
+                pendingRefs = new ArrayList<>();
+            }
+            pendingRefs.add(ref);
+        }
 
-                if (alreadyConnected) {
-                    logger.trace("refreshing connection to {}", discoveryNode);
-                } else {
-                    logger.debug("connecting to {}", discoveryNode);
+        private synchronized Releasable acquireRefs() {
+            // Avoid concurrent connection attempts because they don't necessarily complete in order otherwise, and out-of-order completion
+            // might mean we end up disconnected from a node even though we triggered a call to connect() after all close() calls had
+            // finished.
+            if (connectionInProgress == false) {
+                var refs = pendingRefs;
+                if (refs != null) {
+                    pendingRefs = null;
+                    connectionInProgress = true;
+                    return Releasables.wrap(refs);
                 }
+            }
+            return null;
+        }
+
+        private synchronized void releaseListener() {
+            assert connectionInProgress;
+            connectionInProgress = false;
+        }
+
+        private void doConnect() {
+            // noinspection resource
+            var refs = acquireRefs();
+            if (refs == null) {
+                return;
+            }
+
+            final boolean alreadyConnected = transportService.nodeConnected(discoveryNode);
 
-                // It's possible that connectionRef is a reference to an older connection that closed out from under us, but that something
-                // else has opened a fresh connection to the node. Therefore we always call connectToNode() and update connectionRef.
-                transportService.connectToNode(discoveryNode, new ActionListener<>() {
-                    @Override
-                    public void onResponse(Releasable connectionReleasable) {
-                        if (alreadyConnected) {
-                            logger.trace("refreshed connection to {}", discoveryNode);
-                        } else {
-                            logger.debug("connected to {}", discoveryNode);
-                        }
-                        consecutiveFailureCount.set(0);
-                        setConnectionRef(connectionReleasable);
-
-                        final boolean isActive;
-                        synchronized (mutex) {
-                            isActive = targetsByNode.get(discoveryNode) == ConnectionTarget.this;
-                        }
-                        if (isActive == false) {
-                            logger.debug("connected to stale {} - releasing stale connection", discoveryNode);
-                            setConnectionRef(null);
-                        }
-                        if (listener != null) {
-                            listener.onResponse(null);
-                        }
+            if (alreadyConnected) {
+                logger.trace("refreshing connection to {}", discoveryNode);
+            } else {
+                logger.debug("connecting to {}", discoveryNode);
+            }
+
+            // It's possible that connectionRef is a reference to an older connection that closed out from under us, but that something else
+            // has opened a fresh connection to the node. Therefore we always call connectToNode() and update connectionRef.
+            transportService.connectToNode(discoveryNode, ActionListener.runAfter(new ActionListener<>() {
+                @Override
+                public void onResponse(Releasable connectionReleasable) {
+                    if (alreadyConnected) {
+                        logger.trace("refreshed connection to {}", discoveryNode);
+                    } else {
+                        logger.debug("connected to {}", discoveryNode);
                     }
+                    consecutiveFailureCount.set(0);
+                    setConnectionRef(connectionReleasable);
 
-                    @Override
-                    public void onFailure(Exception e) {
-                        final int currentFailureCount = consecutiveFailureCount.incrementAndGet();
-                        // only warn every 6th failure
-                        final Level level = currentFailureCount % 6 == 1 ? Level.WARN : Level.DEBUG;
-                        logger.log(
-                            level,
-                            () -> format("failed to connect to %s (tried [%s] times)", discoveryNode, currentFailureCount),
-                            e
-                        );
+                    final boolean isActive;
+                    synchronized (mutex) {
+                        isActive = targetsByNode.get(discoveryNode) == ConnectionTarget.this;
+                    }
+                    if (isActive == false) {
+                        logger.debug("connected to stale {} - releasing stale connection", discoveryNode);
                         setConnectionRef(null);
-                        if (listener != null) {
-                            listener.onFailure(e);
-                        }
                     }
-                });
-            };
+                    Releasables.closeExpectNoException(refs);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    final int currentFailureCount = consecutiveFailureCount.incrementAndGet();
+                    // only warn every 6th failure
+                    final Level level = currentFailureCount % 6 == 1 ? Level.WARN : Level.DEBUG;
+                    logger.log(level, () -> format("failed to connect to %s (tried [%s] times)", discoveryNode, currentFailureCount), e);
+                    setConnectionRef(null);
+                    Releasables.closeExpectNoException(refs);
+                }
+            }, () -> {
+                releaseListener();
+                transportService.getThreadPool().generic().execute(this::doConnect);
+            }));
         }
 
         void disconnect() {
diff --git a/server/src/main/java/org/elasticsearch/cluster/RepositoryCleanupInProgress.java b/server/src/main/java/org/elasticsearch/cluster/RepositoryCleanupInProgress.java
index 948e055e06ee..40f4d390059b 100644
--- a/server/src/main/java/org/elasticsearch/cluster/RepositoryCleanupInProgress.java
+++ b/server/src/main/java/org/elasticsearch/cluster/RepositoryCleanupInProgress.java
@@ -7,15 +7,17 @@
  */
 package org.elasticsearch.cluster;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.repositories.RepositoryOperation;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.List;
 
 public final class RepositoryCleanupInProgress extends AbstractNamedDiffable<ClusterState.Custom> implements ClusterState.Custom {
@@ -62,17 +64,17 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startArray(TYPE);
-        for (Entry entry : entries) {
-            builder.startObject();
-            {
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return Iterators.concat(
+            Iterators.single((builder, params) -> builder.startArray(TYPE)),
+            entries.stream().<ToXContent>map(entry -> (builder, params) -> {
+                builder.startObject();
                 builder.field("repository", entry.repository);
-            }
-            builder.endObject();
-        }
-        builder.endArray();
-        return builder;
+                builder.endObject();
+                return builder;
+            }).iterator(),
+            Iterators.single((builder, params) -> builder.endArray())
+        );
     }
 
     @Override
@@ -81,8 +83,8 @@ public String toString() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_4_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_4_0;
     }
 
     public static final class Entry implements Writeable, RepositoryOperation {
diff --git a/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java b/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java
index 3b19fee0210b..4d41473b2d0c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java
+++ b/server/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java
@@ -8,9 +8,10 @@
 
 package org.elasticsearch.cluster;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
 import org.elasticsearch.cluster.ClusterState.Custom;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -18,7 +19,6 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -344,8 +344,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.CURRENT.minimumCompatibilityVersion();
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.CURRENT.minimumCompatibilityVersion();
     }
 
     public static NamedDiff<Custom> readDiffFrom(StreamInput in) throws IOException {
@@ -361,7 +361,7 @@ public RestoreInProgress(StreamInput in) throws IOException {
             Snapshot snapshot = new Snapshot(in);
             State state = State.fromValue(in.readByte());
             boolean quiet;
-            if (in.getVersion().onOrAfter(RestoreSnapshotRequest.VERSION_SUPPORTING_QUIET_PARAMETER)) {
+            if (in.getTransportVersion().onOrAfter(RestoreSnapshotRequest.VERSION_SUPPORTING_QUIET_PARAMETER)) {
                 quiet = in.readBoolean();
             } else {
                 // Backwards compatibility: previously there was no logging of the start or completion of a snapshot restore
@@ -389,7 +389,7 @@ public void writeTo(StreamOutput out) throws IOException {
             o.writeString(entry.uuid);
             entry.snapshot().writeTo(o);
             o.writeByte(entry.state().value());
-            if (out.getVersion().onOrAfter(RestoreSnapshotRequest.VERSION_SUPPORTING_QUIET_PARAMETER)) {
+            if (out.getTransportVersion().onOrAfter(RestoreSnapshotRequest.VERSION_SUPPORTING_QUIET_PARAMETER)) {
                 o.writeBoolean(entry.quiet());
             }
             o.writeStringCollection(entry.indices);
@@ -398,49 +398,41 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
-        builder.startArray("snapshots");
-        for (Entry entry : entries.values()) {
-            toXContent(entry, builder);
-        }
-        builder.endArray();
-        return builder;
-    }
-
-    /**
-     * Serializes single restore operation
-     *
-     * @param entry   restore operation metadata
-     * @param builder XContent builder
-     */
-    public static void toXContent(Entry entry, XContentBuilder builder) throws IOException {
-        builder.startObject();
-        builder.field("snapshot", entry.snapshot().getSnapshotId().getName());
-        builder.field("repository", entry.snapshot().getRepository());
-        builder.field("state", entry.state());
-        builder.startArray("indices");
-        {
-            for (String index : entry.indices()) {
-                builder.value(index);
-            }
-        }
-        builder.endArray();
-        builder.startArray("shards");
-        {
-            for (Map.Entry<ShardId, ShardRestoreStatus> shardEntry : entry.shards.entrySet()) {
-                ShardId shardId = shardEntry.getKey();
-                ShardRestoreStatus status = shardEntry.getValue();
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return Iterators.concat(
+            Iterators.single((builder, params) -> builder.startArray("snapshots")),
+            entries.values().stream().<ToXContent>map(entry -> (builder, params) -> {
                 builder.startObject();
+                builder.field("snapshot", entry.snapshot().getSnapshotId().getName());
+                builder.field("repository", entry.snapshot().getRepository());
+                builder.field("state", entry.state());
+                builder.startArray("indices");
                 {
-                    builder.field("index", shardId.getIndex());
-                    builder.field("shard", shardId.getId());
-                    builder.field("state", status.state());
+                    for (String index : entry.indices()) {
+                        builder.value(index);
+                    }
+                }
+                builder.endArray();
+                builder.startArray("shards");
+                {
+                    for (Map.Entry<ShardId, ShardRestoreStatus> shardEntry : entry.shards.entrySet()) {
+                        ShardId shardId = shardEntry.getKey();
+                        ShardRestoreStatus status = shardEntry.getValue();
+                        builder.startObject();
+                        {
+                            builder.field("index", shardId.getIndex());
+                            builder.field("shard", shardId.getId());
+                            builder.field("state", status.state());
+                        }
+                        builder.endObject();
+                    }
                 }
-                builder.endObject();
-            }
-        }
 
-        builder.endArray();
-        builder.endObject();
+                builder.endArray();
+                builder.endObject();
+                return builder;
+            }).iterator(),
+            Iterators.single((builder, params) -> builder.endArray())
+        );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java
index ba9531fbe297..4f38256590bc 100644
--- a/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java
+++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java
@@ -8,22 +8,24 @@
 
 package org.elasticsearch.cluster;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.ClusterState.Custom;
 import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.repositories.RepositoryOperation;
 import org.elasticsearch.snapshots.SnapshotId;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
@@ -155,30 +157,32 @@ public static NamedDiff<Custom> readDiffFrom(StreamInput in) throws IOException
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.CURRENT.minimumCompatibilityVersion();
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.CURRENT.minimumCompatibilityVersion();
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startArray(TYPE);
-        for (Entry entry : entries) {
-            builder.startObject();
-            {
-                builder.field("repository", entry.repository());
-                builder.startArray("snapshots");
-                for (SnapshotId snapshot : entry.snapshots) {
-                    builder.value(snapshot.getName());
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return Iterators.concat(
+            Iterators.single((builder, params) -> builder.startArray(TYPE)),
+            entries.stream().<ToXContent>map(entry -> (builder, params) -> {
+                builder.startObject();
+                {
+                    builder.field("repository", entry.repository());
+                    builder.startArray("snapshots");
+                    for (SnapshotId snapshot : entry.snapshots) {
+                        builder.value(snapshot.getName());
+                    }
+                    builder.endArray();
+                    builder.timeField("start_time_millis", "start_time", entry.startTime);
+                    builder.field("repository_state_id", entry.repositoryStateId);
+                    builder.field("state", entry.state);
                 }
-                builder.endArray();
-                builder.timeField("start_time_millis", "start_time", entry.startTime);
-                builder.field("repository_state_id", entry.repositoryStateId);
-                builder.field("state", entry.state);
-            }
-            builder.endObject();
-        }
-        builder.endArray();
-        return builder;
+                builder.endObject();
+                return builder;
+            }).iterator(),
+            Iterators.single((builder, params) -> builder.endArray())
+        );
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
index 279e7ba4a6b9..e9736fcff52a 100644
--- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
+++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
@@ -8,10 +8,12 @@
 
 package org.elasticsearch.cluster;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState.Custom;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -184,14 +186,14 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.CURRENT.minimumCompatibilityVersion();
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.CURRENT.minimumCompatibilityVersion();
     }
 
-    private static final Version DIFFABLE_VERSION = Version.V_8_5_0;
+    private static final TransportVersion DIFFABLE_VERSION = TransportVersion.V_8_5_0;
 
     public static NamedDiff<Custom> readDiffFrom(StreamInput in) throws IOException {
-        if (in.getVersion().onOrAfter(DIFFABLE_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(DIFFABLE_VERSION)) {
             return new SnapshotInProgressDiff(in);
         }
         return readDiffFrom(Custom.class, TYPE, in);
@@ -212,14 +214,12 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
-        builder.startArray("snapshots");
-        final Iterator<Entry> iterator = asStream().iterator();
-        while (iterator.hasNext()) {
-            iterator.next().toXContent(builder, params);
-        }
-        builder.endArray();
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return Iterators.<ToXContent>concat(
+            Iterators.single((builder, params) -> builder.startArray("snapshots")),
+            asStream().iterator(),
+            Iterators.single((builder, params) -> builder.endArray())
+        );
     }
 
     @Override
@@ -1613,8 +1613,8 @@ public SnapshotsInProgress apply(Custom part) {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT.minimumCompatibilityVersion();
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT.minimumCompatibilityVersion();
         }
 
         @Override
@@ -1625,7 +1625,7 @@ public String getWriteableName() {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             assert after != null : "should only write instances that were diffed from this node's state";
-            if (out.getVersion().onOrAfter(DIFFABLE_VERSION)) {
+            if (out.getTransportVersion().onOrAfter(DIFFABLE_VERSION)) {
                 mapDiff.writeTo(out);
             } else {
                 new SimpleDiffable.CompleteDiff<>(after).writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/CleanableResponseHandler.java b/server/src/main/java/org/elasticsearch/cluster/coordination/CleanableResponseHandler.java
new file mode 100644
index 000000000000..ff1dafb99c4f
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/CleanableResponseHandler.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.cluster.coordination;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionListenerResponseHandler;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportResponse;
+
+/**
+ * Combines an ActionListenerResponseHandler with an ActionListener.runAfter action, but with an explicit type so that tests that simulate
+ * reboots can release resources without invoking the listener.
+ */
+public class CleanableResponseHandler<T extends TransportResponse> extends ActionListenerResponseHandler<T> {
+    private final Runnable cleanup;
+
+    public CleanableResponseHandler(ActionListener<? super T> listener, Writeable.Reader<T> reader, String executor, Runnable cleanup) {
+        super(ActionListener.runAfter(listener, cleanup), reader, executor);
+        this.cleanup = cleanup;
+    }
+
+    public void runCleanup() {
+        assert ThreadPool.assertCurrentThreadPool(); // should only be called from tests which simulate abrupt node restarts
+        cleanup.run();
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterBootstrapService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterBootstrapService.java
index 9e1772c7d844..028275ab1fa3 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterBootstrapService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterBootstrapService.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.cluster.coordination.CoordinationMetadata.VotingConfiguration;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -129,14 +130,9 @@ void logBootstrapState(Metadata metadata) {
             if (bootstrapRequirements.isEmpty()) {
                 logger.info("this node is locked into cluster UUID [{}] and will not attempt further cluster bootstrapping", clusterUUID);
             } else {
-                logger.warn(
-                    """
-                        this node is locked into cluster UUID [{}] but [{}] is set to {}; \
-                        remove this setting to avoid possible data loss caused by subsequent cluster bootstrap attempts""",
-                    clusterUUID,
-                    INITIAL_MASTER_NODES_SETTING.getKey(),
-                    bootstrapRequirements
-                );
+                transportService.getThreadPool()
+                    .scheduleWithFixedDelay(() -> logRemovalWarning(clusterUUID), TimeValue.timeValueHours(12), Names.SAME);
+                logRemovalWarning(clusterUUID);
             }
         } else {
             logger.info(
@@ -147,6 +143,19 @@ void logBootstrapState(Metadata metadata) {
         }
     }
 
+    private void logRemovalWarning(String clusterUUID) {
+        logger.warn(
+            """
+                this node is locked into cluster UUID [{}] but [{}] is set to {}; \
+                remove this setting to avoid possible data loss caused by subsequent cluster bootstrap attempts; \
+                for further information see {}""",
+            clusterUUID,
+            INITIAL_MASTER_NODES_SETTING.getKey(),
+            bootstrapRequirements,
+            ReferenceDocs.INITIAL_MASTER_NODES
+        );
+    }
+
     @Override
     public void onFoundPeersUpdated() {
         final Set<DiscoveryNode> nodes = getDiscoveredNodes();
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java
index 11488584b7ed..be82c8955a18 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.coordination.CoordinationState.VoteCollection;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -100,7 +101,11 @@ public void onFailure(Exception e) {
                 protected void doRun() {
                     if (isActive()) {
                         logLastFailedJoinAttempt.run();
-                        logger.warn(clusterFormationStateSupplier.get().getDescription());
+                        logger.warn(
+                            "{}; for troubleshooting guidance, see {}",
+                            clusterFormationStateSupplier.get().getDescription(),
+                            ReferenceDocs.DISCOVERY_TROUBLESHOOTING
+                        );
                     }
                 }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java
index cb3e5ac8edfd..1bb83fc80a3f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java
@@ -556,27 +556,18 @@ public interface PersistedState extends Closeable {
          * marked as committed.
          */
         default void markLastAcceptedStateAsCommitted() {
-            final ClusterState lastAcceptedState = getLastAcceptedState();
-            Metadata.Builder metadataBuilder = null;
-            if (lastAcceptedState.getLastAcceptedConfiguration().equals(lastAcceptedState.getLastCommittedConfiguration()) == false) {
-                final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder(lastAcceptedState.coordinationMetadata())
-                    .lastCommittedConfiguration(lastAcceptedState.getLastAcceptedConfiguration())
-                    .build();
-                metadataBuilder = Metadata.builder(lastAcceptedState.metadata());
-                metadataBuilder.coordinationMetadata(coordinationMetadata);
-            }
+            final var lastAcceptedState = getLastAcceptedState();
             assert lastAcceptedState.metadata().clusterUUID().equals(Metadata.UNKNOWN_CLUSTER_UUID) == false
                 : "received cluster state with empty cluster uuid: " + lastAcceptedState;
-            if (lastAcceptedState.metadata().clusterUUID().equals(Metadata.UNKNOWN_CLUSTER_UUID) == false
-                && lastAcceptedState.metadata().clusterUUIDCommitted() == false) {
-                if (metadataBuilder == null) {
-                    metadataBuilder = Metadata.builder(lastAcceptedState.metadata());
-                }
-                metadataBuilder.clusterUUIDCommitted(true);
+
+            if (lastAcceptedState.metadata().clusterUUIDCommitted() == false) {
                 logger.info("cluster UUID set to [{}]", lastAcceptedState.metadata().clusterUUID());
             }
-            if (metadataBuilder != null) {
-                setLastAcceptedState(ClusterState.builder(lastAcceptedState).metadata(metadataBuilder).build());
+
+            final var adjustedMetadata = lastAcceptedState.metadata()
+                .withLastCommittedValues(true, lastAcceptedState.getLastAcceptedConfiguration());
+            if (adjustedMetadata != lastAcceptedState.metadata()) {
+                setLastAcceptedState(ClusterState.builder(lastAcceptedState).metadata(adjustedMetadata).build());
             }
         }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java
index 33e3528246d1..b9a0a6950f88 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java
@@ -137,7 +137,7 @@ public class Coordinator extends AbstractLifecycleComponent implements ClusterSt
     private final AllocationService allocationService;
     private final JoinHelper joinHelper;
     private final JoinValidationService joinValidationService;
-    private final NodeRemovalClusterStateTaskExecutor nodeRemovalExecutor;
+    private final NodeLeftExecutor nodeLeftExecutor;
     private final Supplier<CoordinationState.PersistedState> persistedStateSupplier;
     private final NoMasterBlockService noMasterBlockService;
     final Object mutex = new Object(); // package-private to allow tests to call methods that assert that the mutex is held
@@ -205,7 +205,7 @@ public Coordinator(
         this.transportService = transportService;
         this.masterService = masterService;
         this.allocationService = allocationService;
-        this.onJoinValidators = JoinTaskExecutor.addBuiltInJoinValidators(onJoinValidators);
+        this.onJoinValidators = NodeJoinExecutor.addBuiltInJoinValidators(onJoinValidators);
         this.singleNodeDiscovery = DiscoveryModule.isSingleNodeDiscovery(settings);
         this.electionStrategy = electionStrategy;
         this.joinReasonService = new JoinReasonService(transportService.getThreadPool()::relativeTimeInMillis);
@@ -272,7 +272,7 @@ public Coordinator(
             this::removeNode,
             nodeHealthService
         );
-        this.nodeRemovalExecutor = new NodeRemovalClusterStateTaskExecutor(allocationService);
+        this.nodeLeftExecutor = new NodeLeftExecutor(allocationService);
         this.clusterApplier = clusterApplier;
         masterService.setClusterStateSupplier(this::getStateForMasterService);
         this.reconfigurator = new Reconfigurator(settings, clusterSettings);
@@ -339,16 +339,11 @@ private void onLeaderFailure(Supplier<String> message, Exception e) {
     private void removeNode(DiscoveryNode discoveryNode, String reason) {
         synchronized (mutex) {
             if (mode == Mode.LEADER) {
-                var task = new NodeRemovalClusterStateTaskExecutor.Task(
-                    discoveryNode,
-                    reason,
-                    () -> joinReasonService.onNodeRemoved(discoveryNode, reason)
-                );
                 masterService.submitStateUpdateTask(
                     "node-left",
-                    task,
+                    new NodeLeftExecutor.Task(discoveryNode, reason, () -> joinReasonService.onNodeRemoved(discoveryNode, reason)),
                     ClusterStateTaskConfig.build(Priority.IMMEDIATE),
-                    nodeRemovalExecutor
+                    nodeLeftExecutor
                 );
             }
         }
@@ -664,7 +659,7 @@ private void validateJoinRequest(JoinRequest joinRequest, ActionListener<Void> v
             if (stateForJoinValidation.getBlocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK) == false) {
                 // We do this in a couple of places including the cluster update thread. This one here is really just best effort to ensure
                 // we fail as fast as possible.
-                JoinTaskExecutor.ensureVersionBarrier(
+                NodeJoinExecutor.ensureVersionBarrier(
                     joinRequest.getSourceNode().getVersion(),
                     stateForJoinValidation.getNodes().getMinNodeVersion()
                 );
@@ -1002,6 +997,10 @@ protected void doStart() {
                         + votingConfiguration
                 );
             }
+            final Metadata.Builder metadata = Metadata.builder();
+            if (lastAcceptedState.metadata().clusterUUIDCommitted()) {
+                metadata.clusterUUID(lastAcceptedState.metadata().clusterUUID()).clusterUUIDCommitted(true);
+            }
             ClusterState initialState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(settings))
                 .blocks(
                     ClusterBlocks.builder()
@@ -1009,6 +1008,7 @@ protected void doStart() {
                         .addGlobalBlock(noMasterBlockService.getNoMasterBlock())
                 )
                 .nodes(DiscoveryNodes.builder().add(getLocalNode()).localNodeId(getLocalNode().getId()))
+                .metadata(metadata)
                 .build();
             applierState = initialState;
             clusterApplier.setInitialState(initialState);
@@ -1235,15 +1235,16 @@ ClusterState improveConfiguration(ClusterState clusterState) {
             )
             .map(DiscoveryNode::getId);
 
+        DiscoveryNode localNode = getLocalNode();
         final Set<DiscoveryNode> liveNodes = clusterState.nodes()
             .stream()
             .filter(DiscoveryNode::isMasterNode)
-            .filter(coordinationState.get()::containsJoinVoteFor)
+            .filter((n) -> coordinationState.get().containsJoinVoteFor(n) || n.equals(localNode))
             .collect(Collectors.toSet());
         final VotingConfiguration newConfig = reconfigurator.reconfigure(
             liveNodes,
             Stream.concat(masterIneligibleNodeIdsInVotingConfig, excludedNodeIds).collect(Collectors.toSet()),
-            getLocalNode(),
+            localNode,
             clusterState.getLastAcceptedConfiguration()
         );
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java
index 6e996bc29a52..82abf4b4c7d5 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java
@@ -14,7 +14,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.admin.indices.rollover.Condition;
 import org.elasticsearch.cli.ProcessInfo;
 import org.elasticsearch.cli.Terminal;
@@ -28,6 +28,7 @@
 import org.elasticsearch.cluster.metadata.DataStreamMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.cli.EnvironmentAwareCommand;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -37,6 +38,7 @@
 import org.elasticsearch.env.NodeMetadata;
 import org.elasticsearch.gateway.PersistedClusterStateService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 
@@ -45,6 +47,7 @@
 import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.EnumSet;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Objects;
 
@@ -217,7 +220,7 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
+        public TransportVersion getMinimalSupportedVersion() {
             assert false;
             throw new UnsupportedOperationException();
         }
@@ -229,8 +232,8 @@ public void writeTo(StreamOutput out) throws IOException {
         }
 
         @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            return builder.mapContents(contents);
+        public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+            return Iterators.single(((builder, params) -> builder.mapContents(contents)));
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java
index 48d6ba553cbb..ed5d28d69d79 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java
@@ -67,7 +67,7 @@ public class JoinHelper {
     private final MasterService masterService;
     private final ClusterApplier clusterApplier;
     private final TransportService transportService;
-    private final JoinTaskExecutor joinTaskExecutor;
+    private final NodeJoinExecutor nodeJoinExecutor;
     private final LongSupplier currentTermSupplier;
     private final NodeHealthService nodeHealthService;
     private final JoinReasonService joinReasonService;
@@ -94,7 +94,7 @@ public class JoinHelper {
         this.clusterApplier = clusterApplier;
         this.transportService = transportService;
         this.circuitBreakerService = circuitBreakerService;
-        this.joinTaskExecutor = new JoinTaskExecutor(allocationService, rerouteService);
+        this.nodeJoinExecutor = new NodeJoinExecutor(allocationService, rerouteService);
         this.currentTermSupplier = currentTermSupplier;
         this.nodeHealthService = nodeHealthService;
         this.joinReasonService = joinReasonService;
@@ -389,13 +389,17 @@ default void close(Mode newMode) {}
     class LeaderJoinAccumulator implements JoinAccumulator {
         @Override
         public void handleJoinRequest(DiscoveryNode sender, ActionListener<Void> joinListener) {
-            final JoinTask task = JoinTask.singleNode(
-                sender,
-                joinReasonService.getJoinReason(sender, Mode.LEADER),
-                joinListener,
-                currentTermSupplier.getAsLong()
+            masterService.submitStateUpdateTask(
+                "node-join",
+                JoinTask.singleNode(
+                    sender,
+                    joinReasonService.getJoinReason(sender, Mode.LEADER),
+                    joinListener,
+                    currentTermSupplier.getAsLong()
+                ),
+                ClusterStateTaskConfig.build(Priority.URGENT),
+                nodeJoinExecutor
             );
-            masterService.submitStateUpdateTask("node-join", task, ClusterStateTaskConfig.build(Priority.URGENT), joinTaskExecutor);
         }
 
         @Override
@@ -461,7 +465,7 @@ public void close(Mode newMode) {
                     "elected-as-master ([" + joinTask.nodeCount() + "] nodes joined)",
                     joinTask,
                     ClusterStateTaskConfig.build(Priority.URGENT),
-                    joinTaskExecutor
+                    nodeJoinExecutor
 
                 );
             } else {
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinReason.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinReason.java
new file mode 100644
index 000000000000..356a17f1e110
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinReason.java
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.cluster.coordination;
+
+import org.elasticsearch.common.ReferenceDocs;
+import org.elasticsearch.core.Nullable;
+
+/**
+ * @param message      Message describing the reason for the node joining
+ * @param guidanceDocs An optional link to troubleshooting guidance docs
+ */
+public record JoinReason(String message, @Nullable ReferenceDocs guidanceDocs) {}
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinReasonService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinReasonService.java
index b4aab2ea57ac..cf5ff1ff44c6 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinReasonService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinReasonService.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterApplierService;
 import org.elasticsearch.cluster.service.MasterService;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.core.Nullable;
@@ -107,9 +108,9 @@ public void onNodeRemoved(DiscoveryNode discoveryNode, String reason) {
      * @param currentMode   The current mode of the master that the node is joining.
      * @return              A description of the reason for the join, possibly including some details of its earlier removal.
      */
-    public String getJoinReason(DiscoveryNode discoveryNode, Coordinator.Mode currentMode) {
+    public JoinReason getJoinReason(DiscoveryNode discoveryNode, Coordinator.Mode currentMode) {
         return trackedNodes.getOrDefault(discoveryNode.getId(), UNKNOWN_NODE)
-            .getDescription(relativeTimeInMillisSupplier.getAsLong(), discoveryNode.getEphemeralId(), currentMode);
+            .getJoinReason(relativeTimeInMillisSupplier.getAsLong(), discoveryNode.getEphemeralId(), currentMode);
     }
 
     /**
@@ -134,7 +135,7 @@ private interface TrackedNode {
 
         TrackedNode withRemovalReason(String removalReason);
 
-        String getDescription(long currentTimeMillis, String joiningNodeEphemeralId, Coordinator.Mode currentMode);
+        JoinReason getJoinReason(long currentTimeMillis, String joiningNodeEphemeralId, Coordinator.Mode currentMode);
 
         long getRemovalAgeMillis(long currentTimeMillis);
     }
@@ -161,11 +162,11 @@ public TrackedNode withRemovalReason(String removalReason) {
         }
 
         @Override
-        public String getDescription(long currentTimeMillis, String joiningNodeEphemeralId, Coordinator.Mode currentMode) {
+        public JoinReason getJoinReason(long currentTimeMillis, String joiningNodeEphemeralId, Coordinator.Mode currentMode) {
             if (currentMode == CANDIDATE) {
-                return "completing election";
+                return COMPLETING_ELECTION;
             } else {
-                return "joining";
+                return NEW_NODE_JOINING;
             }
         }
 
@@ -193,11 +194,11 @@ public TrackedNode withRemovalReason(String removalReason) {
         }
 
         @Override
-        public String getDescription(long currentTimeMillis, String joiningNodeEphemeralId, Coordinator.Mode currentMode) {
+        public JoinReason getJoinReason(long currentTimeMillis, String joiningNodeEphemeralId, Coordinator.Mode currentMode) {
             if (currentMode == CANDIDATE) {
-                return "completing election";
+                return COMPLETING_ELECTION;
             } else {
-                return "rejoining";
+                return KNOWN_NODE_REJOINING;
             }
         }
 
@@ -231,7 +232,7 @@ public TrackedNode withRemovalReason(String removalReason) {
         }
 
         @Override
-        public String getDescription(long currentTimeMillis, String joiningNodeEphemeralId, Coordinator.Mode currentMode) {
+        public JoinReason getJoinReason(long currentTimeMillis, String joiningNodeEphemeralId, Coordinator.Mode currentMode) {
             final StringBuilder description = new StringBuilder();
             if (currentMode == CANDIDATE) {
                 description.append("completing election");
@@ -261,7 +262,7 @@ public String getDescription(long currentTimeMillis, String joiningNodeEphemeral
                 description.append(", [").append(removalCount).append("] total removals");
             }
 
-            return description.toString();
+            return new JoinReason(description.toString(), isRestarted ? null : ReferenceDocs.UNSTABLE_CLUSTER_TROUBLESHOOTING);
         }
 
         @Override
@@ -270,4 +271,7 @@ public long getRemovalAgeMillis(long currentTimeMillis) {
         }
     }
 
+    private static final JoinReason COMPLETING_ELECTION = new JoinReason("completing election", null);
+    private static final JoinReason NEW_NODE_JOINING = new JoinReason("joining", null);
+    private static final JoinReason KNOWN_NODE_REJOINING = new JoinReason("rejoining", null);
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTask.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTask.java
index 10121337a85c..38f5b48e4d1b 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTask.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTask.java
@@ -18,7 +18,7 @@
 
 public record JoinTask(List<NodeJoinTask> nodeJoinTasks, boolean isBecomingMaster, long term) implements ClusterStateTaskListener {
 
-    public static JoinTask singleNode(DiscoveryNode node, String reason, ActionListener<Void> listener, long term) {
+    public static JoinTask singleNode(DiscoveryNode node, JoinReason reason, ActionListener<Void> listener, long term) {
         return new JoinTask(List.of(new NodeJoinTask(node, reason, listener)), false, term);
     }
 
@@ -59,9 +59,9 @@ public Iterable<DiscoveryNode> nodes() {
         return () -> nodeJoinTasks.stream().map(j -> j.node).iterator();
     }
 
-    public record NodeJoinTask(DiscoveryNode node, String reason, ActionListener<Void> listener) {
+    public record NodeJoinTask(DiscoveryNode node, JoinReason reason, ActionListener<Void> listener) {
 
-        public NodeJoinTask(DiscoveryNode node, String reason, ActionListener<Void> listener) {
+        public NodeJoinTask(DiscoveryNode node, JoinReason reason, ActionListener<Void> listener) {
             this.node = Objects.requireNonNull(node);
             this.reason = reason;
             this.listener = listener;
@@ -76,7 +76,7 @@ public String toString() {
 
         public void appendDescription(StringBuilder stringBuilder) {
             node.appendDescriptionWithoutAttributes(stringBuilder);
-            stringBuilder.append(' ').append(reason);
+            stringBuilder.append(' ').append(reason.message());
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java
deleted file mode 100644
index c1c4133dcc41..000000000000
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java
+++ /dev/null
@@ -1,346 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.cluster.coordination;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateTaskExecutor;
-import org.elasticsearch.cluster.NotMasterException;
-import org.elasticsearch.cluster.block.ClusterBlocks;
-import org.elasticsearch.cluster.metadata.DesiredNodes;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.cluster.metadata.Metadata;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.RerouteService;
-import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.common.Priority;
-import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.function.BiConsumer;
-import java.util.stream.Collectors;
-
-import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK;
-
-public class JoinTaskExecutor implements ClusterStateTaskExecutor<JoinTask> {
-
-    private static final Logger logger = LogManager.getLogger(JoinTaskExecutor.class);
-
-    private final AllocationService allocationService;
-    private final RerouteService rerouteService;
-
-    public JoinTaskExecutor(AllocationService allocationService, RerouteService rerouteService) {
-        this.allocationService = allocationService;
-        this.rerouteService = rerouteService;
-    }
-
-    @Override
-    public ClusterState execute(BatchExecutionContext<JoinTask> batchExecutionContext) throws Exception {
-        // The current state that MasterService uses might have been updated by a (different) master in a higher term already. If so, stop
-        // processing the current cluster state update, there's no point in continuing to compute it as it will later be rejected by
-        // Coordinator#publish anyhow.
-        assert batchExecutionContext.taskContexts().isEmpty() == false : "Expected to have non empty join tasks list";
-
-        var term = batchExecutionContext.taskContexts().stream().mapToLong(t -> t.getTask().term()).max().getAsLong();
-
-        var split = batchExecutionContext.taskContexts().stream().collect(Collectors.partitioningBy(t -> t.getTask().term() == term));
-        for (TaskContext<JoinTask> outdated : split.get(false)) {
-            outdated.onFailure(
-                new NotMasterException("Higher term encountered (encountered: " + term + " > used: " + outdated.getTask().term() + ")")
-            );
-        }
-
-        final var joinTaskContexts = split.get(true);
-        final var initialState = batchExecutionContext.initialState();
-
-        if (initialState.term() > term) {
-            logger.trace("encountered higher term {} than current {}, there is a newer master", initialState.term(), term);
-            throw new NotMasterException(
-                "Higher term encountered (current: " + initialState.term() + " > used: " + term + "), there is a newer master"
-            );
-        }
-
-        final boolean isBecomingMaster = joinTaskContexts.stream().anyMatch(t -> t.getTask().isBecomingMaster());
-        final DiscoveryNodes currentNodes = initialState.nodes();
-        boolean nodesChanged = false;
-        ClusterState.Builder newState;
-
-        if (currentNodes.getMasterNode() == null && isBecomingMaster) {
-            assert initialState.term() < term : "there should be at most one become master task per election (= by term)";
-            // use these joins to try and become the master.
-            // Note that we don't have to do any validation of the amount of joining nodes - the commit
-            // during the cluster state publishing guarantees that we have enough
-            try (var ignored = batchExecutionContext.dropHeadersContext()) {
-                // suppress deprecation warnings e.g. from reroute()
-                newState = becomeMasterAndTrimConflictingNodes(initialState, joinTaskContexts, term);
-            }
-            nodesChanged = true;
-        } else if (currentNodes.isLocalNodeElectedMaster()) {
-            assert initialState.term() == term : "term should be stable for the same master";
-            newState = ClusterState.builder(initialState);
-        } else {
-            logger.trace("processing node joins, but we are not the master. current master: {}", currentNodes.getMasterNode());
-            throw new NotMasterException("Node [" + currentNodes.getLocalNode() + "] not master for join request");
-        }
-
-        DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(newState.nodes());
-
-        assert nodesBuilder.isLocalNodeElectedMaster();
-
-        Version minClusterNodeVersion = newState.nodes().getMinNodeVersion();
-        Version maxClusterNodeVersion = newState.nodes().getMaxNodeVersion();
-        // if the cluster is not fully-formed then the min version is not meaningful
-        final boolean enforceVersionBarrier = initialState.getBlocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK) == false;
-        // processing any joins
-        Map<String, String> joinedNodeIdsByNodeName = new HashMap<>();
-        for (final var joinTaskContext : joinTaskContexts) {
-            final var joinTask = joinTaskContext.getTask();
-            final List<Runnable> onTaskSuccess = new ArrayList<>(joinTask.nodeCount());
-            for (final JoinTask.NodeJoinTask nodeJoinTask : joinTask.nodeJoinTasks()) {
-                final DiscoveryNode node = nodeJoinTask.node();
-                if (currentNodes.nodeExistsWithSameRoles(node)) {
-                    logger.debug("received a join request for an existing node [{}]", node);
-                } else {
-                    try {
-                        if (enforceVersionBarrier) {
-                            ensureVersionBarrier(node.getVersion(), minClusterNodeVersion);
-                        }
-                        ensureNodesCompatibility(node.getVersion(), minClusterNodeVersion, maxClusterNodeVersion);
-                        // we do this validation quite late to prevent race conditions between nodes joining and importing dangling indices
-                        // we have to reject nodes that don't support all indices we have in this cluster
-                        ensureIndexCompatibility(node.getVersion(), initialState.getMetadata());
-                        nodesBuilder.add(node);
-                        nodesChanged = true;
-                        minClusterNodeVersion = Version.min(minClusterNodeVersion, node.getVersion());
-                        maxClusterNodeVersion = Version.max(maxClusterNodeVersion, node.getVersion());
-                        if (node.isMasterNode()) {
-                            joinedNodeIdsByNodeName.put(node.getName(), node.getId());
-                        }
-                    } catch (IllegalArgumentException | IllegalStateException e) {
-                        onTaskSuccess.add(() -> nodeJoinTask.listener().onFailure(e));
-                        continue;
-                    }
-                }
-                onTaskSuccess.add(() -> nodeJoinTask.listener().onResponse(null));
-            }
-            joinTaskContext.success(() -> {
-                for (Runnable joinCompleter : onTaskSuccess) {
-                    joinCompleter.run();
-                }
-            });
-        }
-
-        if (nodesChanged) {
-            rerouteService.reroute(
-                "post-join reroute",
-                Priority.HIGH,
-                ActionListener.wrap(r -> logger.trace("post-join reroute completed"), e -> logger.debug("post-join reroute failed", e))
-            );
-
-            if (joinedNodeIdsByNodeName.isEmpty() == false) {
-                final var currentVotingConfigExclusions = initialState.getVotingConfigExclusions();
-                final var newVotingConfigExclusions = currentVotingConfigExclusions.stream().map(e -> {
-                    // Update nodeId in VotingConfigExclusion when a new node with excluded node name joins
-                    if (CoordinationMetadata.VotingConfigExclusion.MISSING_VALUE_MARKER.equals(e.getNodeId())
-                        && joinedNodeIdsByNodeName.containsKey(e.getNodeName())) {
-                        return new CoordinationMetadata.VotingConfigExclusion(
-                            joinedNodeIdsByNodeName.get(e.getNodeName()),
-                            e.getNodeName()
-                        );
-                    } else {
-                        return e;
-                    }
-                }).collect(Collectors.toSet());
-
-                // if VotingConfigExclusions did get updated
-                if (newVotingConfigExclusions.equals(currentVotingConfigExclusions) == false) {
-                    final var coordMetadataBuilder = CoordinationMetadata.builder(initialState.coordinationMetadata())
-                        .term(term)
-                        .clearVotingConfigExclusions();
-                    newVotingConfigExclusions.forEach(coordMetadataBuilder::addVotingConfigExclusion);
-                    newState.metadata(Metadata.builder(initialState.metadata()).coordinationMetadata(coordMetadataBuilder.build()).build());
-                }
-            }
-
-            final ClusterState clusterStateWithNewNodesAndDesiredNodes = DesiredNodes.updateDesiredNodesStatusIfNeeded(
-                newState.nodes(nodesBuilder).build()
-            );
-            final ClusterState updatedState = allocationService.adaptAutoExpandReplicas(clusterStateWithNewNodesAndDesiredNodes);
-            assert enforceVersionBarrier == false
-                || updatedState.nodes().getMinNodeVersion().onOrAfter(initialState.nodes().getMinNodeVersion())
-                : "min node version decreased from ["
-                    + initialState.nodes().getMinNodeVersion()
-                    + "] to ["
-                    + updatedState.nodes().getMinNodeVersion()
-                    + "]";
-            return updatedState;
-        } else {
-            // we must return a new cluster state instance to force publishing. This is important
-            // for the joining node to finalize its join and set us as a master
-            return newState.build();
-        }
-    }
-
-    protected ClusterState.Builder becomeMasterAndTrimConflictingNodes(
-        ClusterState currentState,
-        List<TaskContext<JoinTask>> taskContexts,
-        long term
-    ) {
-        assert currentState.nodes().getMasterNodeId() == null : currentState;
-        assert currentState.term() < term : term + " vs " + currentState;
-        DiscoveryNodes currentNodes = currentState.nodes();
-        DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(currentNodes);
-        nodesBuilder.masterNodeId(currentState.nodes().getLocalNodeId());
-
-        for (final var taskContext : taskContexts) {
-            for (final var joiningNode : taskContext.getTask().nodes()) {
-                final DiscoveryNode nodeWithSameId = nodesBuilder.get(joiningNode.getId());
-                if (nodeWithSameId != null && nodeWithSameId.equals(joiningNode) == false) {
-                    logger.debug("removing existing node [{}], which conflicts with incoming join from [{}]", nodeWithSameId, joiningNode);
-                    nodesBuilder.remove(nodeWithSameId.getId());
-                }
-                final DiscoveryNode nodeWithSameAddress = currentNodes.findByAddress(joiningNode.getAddress());
-                if (nodeWithSameAddress != null && nodeWithSameAddress.equals(joiningNode) == false) {
-                    logger.debug(
-                        "removing existing node [{}], which conflicts with incoming join from [{}]",
-                        nodeWithSameAddress,
-                        joiningNode
-                    );
-                    nodesBuilder.remove(nodeWithSameAddress.getId());
-                }
-            }
-        }
-
-        // now trim any left over dead nodes - either left there when the previous master stepped down
-        // or removed by us above
-        ClusterState tmpState = ClusterState.builder(currentState)
-            .nodes(nodesBuilder)
-            .blocks(ClusterBlocks.builder().blocks(currentState.blocks()).removeGlobalBlock(NoMasterBlockService.NO_MASTER_BLOCK_ID))
-            .metadata(
-                Metadata.builder(currentState.metadata())
-                    .coordinationMetadata(CoordinationMetadata.builder(currentState.coordinationMetadata()).term(term).build())
-                    .build()
-            )
-            .build();
-        logger.trace("becomeMasterAndTrimConflictingNodes: {}", tmpState.nodes());
-        allocationService.cleanCaches();
-        tmpState = PersistentTasksCustomMetadata.disassociateDeadNodes(tmpState);
-        return ClusterState.builder(allocationService.disassociateDeadNodes(tmpState, false, "removed dead nodes on election"));
-    }
-
-    @Override
-    public boolean runOnlyOnMaster() {
-        // we validate that we are allowed to change the cluster state during cluster state processing
-        return false;
-    }
-
-    /**
-     * Ensures that all indices are compatible with the given node version. This will ensure that all indices in the given metadata
-     * will not be created with a newer version of elasticsearch as well as that all indices are newer or equal to the minimum index
-     * compatibility version.
-     * @see Version#minimumIndexCompatibilityVersion()
-     * @throws IllegalStateException if any index is incompatible with the given version
-     */
-    public static void ensureIndexCompatibility(final Version nodeVersion, Metadata metadata) {
-        Version supportedIndexVersion = nodeVersion.minimumIndexCompatibilityVersion();
-        // we ensure that all indices in the cluster we join are compatible with us no matter if they are
-        // closed or not we can't read mappings of these indices so we need to reject the join...
-        for (IndexMetadata idxMetadata : metadata) {
-            if (idxMetadata.getCompatibilityVersion().after(nodeVersion)) {
-                throw new IllegalStateException(
-                    "index "
-                        + idxMetadata.getIndex()
-                        + " version not supported: "
-                        + idxMetadata.getCompatibilityVersion()
-                        + " the node version is: "
-                        + nodeVersion
-                );
-            }
-            if (idxMetadata.getCompatibilityVersion().before(supportedIndexVersion)) {
-                throw new IllegalStateException(
-                    "index "
-                        + idxMetadata.getIndex()
-                        + " version not supported: "
-                        + idxMetadata.getCompatibilityVersion()
-                        + " minimum compatible index version is: "
-                        + supportedIndexVersion
-                );
-            }
-        }
-    }
-
-    /** ensures that the joining node has a version that's compatible with all current nodes*/
-    public static void ensureNodesCompatibility(final Version joiningNodeVersion, DiscoveryNodes currentNodes) {
-        final Version minNodeVersion = currentNodes.getMinNodeVersion();
-        final Version maxNodeVersion = currentNodes.getMaxNodeVersion();
-        ensureNodesCompatibility(joiningNodeVersion, minNodeVersion, maxNodeVersion);
-    }
-
-    /** ensures that the joining node has a version that's compatible with a given version range */
-    public static void ensureNodesCompatibility(Version joiningNodeVersion, Version minClusterNodeVersion, Version maxClusterNodeVersion) {
-        assert minClusterNodeVersion.onOrBefore(maxClusterNodeVersion) : minClusterNodeVersion + " > " + maxClusterNodeVersion;
-        if (joiningNodeVersion.isCompatible(maxClusterNodeVersion) == false) {
-            throw new IllegalStateException(
-                "node version ["
-                    + joiningNodeVersion
-                    + "] is not supported. "
-                    + "The cluster contains nodes with version ["
-                    + maxClusterNodeVersion
-                    + "], which is incompatible."
-            );
-        }
-        if (joiningNodeVersion.isCompatible(minClusterNodeVersion) == false) {
-            throw new IllegalStateException(
-                "node version ["
-                    + joiningNodeVersion
-                    + "] is not supported."
-                    + "The cluster contains nodes with version ["
-                    + minClusterNodeVersion
-                    + "], which is incompatible."
-            );
-        }
-    }
-
-    /**
-     * ensures that the joining node's version is equal or higher to the minClusterNodeVersion. This is needed
-     * to ensure that if the master is already fully operating under the new version, it doesn't go back to mixed
-     * version mode
-     **/
-    public static void ensureVersionBarrier(Version joiningNodeVersion, Version minClusterNodeVersion) {
-        if (joiningNodeVersion.before(minClusterNodeVersion)) {
-            throw new IllegalStateException(
-                "node version ["
-                    + joiningNodeVersion
-                    + "] may not join a cluster comprising only nodes of version ["
-                    + minClusterNodeVersion
-                    + "] or greater"
-            );
-        }
-    }
-
-    public static Collection<BiConsumer<DiscoveryNode, ClusterState>> addBuiltInJoinValidators(
-        Collection<BiConsumer<DiscoveryNode, ClusterState>> onJoinValidators
-    ) {
-        final Collection<BiConsumer<DiscoveryNode, ClusterState>> validators = new ArrayList<>();
-        validators.add((node, state) -> {
-            ensureNodesCompatibility(node.getVersion(), state.getNodes());
-            ensureIndexCompatibility(node.getVersion(), state.getMetadata());
-        });
-        validators.addAll(onJoinValidators);
-        return Collections.unmodifiableCollection(validators);
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java
index dee938d5e0bb..ecadc771ebda 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java
@@ -308,12 +308,13 @@ protected void doRun() throws Exception {
             transportService.sendRequest(
                 discoveryNode,
                 JOIN_VALIDATE_ACTION_NAME,
-                new BytesTransportRequest(bytes, discoveryNode.getVersion()),
+                new BytesTransportRequest(bytes, discoveryNode.getVersion().transportVersion),
                 REQUEST_OPTIONS,
-                new ActionListenerResponseHandler<>(
-                    ActionListener.runAfter(listener, bytes::decRef),
+                new CleanableResponseHandler<>(
+                    listener,
                     in -> TransportResponse.Empty.INSTANCE,
-                    ThreadPool.Names.CLUSTER_COORDINATION
+                    ThreadPool.Names.CLUSTER_COORDINATION,
+                    bytes::decRef
                 )
             );
             if (cachedBytes == null) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/LagDetector.java b/server/src/main/java/org/elasticsearch/cluster/coordination/LagDetector.java
index 6748fa9980d1..5e21f7c69341 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/LagDetector.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/LagDetector.java
@@ -7,6 +7,7 @@
  */
 package org.elasticsearch.cluster.coordination;
 
+import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
@@ -15,8 +16,13 @@
 import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.ReferenceDocs;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.logging.ChunkedLoggingStream;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.PrioritizedThrottledTaskRunner;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
@@ -25,6 +31,8 @@
 import org.elasticsearch.threadpool.ThreadPool.Names;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -196,11 +204,13 @@ static class HotThreadsLoggingLagListener implements LagListener {
         private final TransportService transportService;
         private final Client client;
         private final LagListener delegate;
+        private final PrioritizedThrottledTaskRunner<HotThreadsLoggingTask> loggingTaskRunner;
 
         HotThreadsLoggingLagListener(TransportService transportService, Client client, LagListener delegate) {
             this.transportService = transportService;
             this.client = client;
             this.delegate = delegate;
+            this.loggingTaskRunner = new PrioritizedThrottledTaskRunner<>("hot_threads", 1, transportService.getThreadPool().generic());
         }
 
         @Override
@@ -224,12 +234,13 @@ public void onResponse(NodesHotThreadsResponse nodesHotThreadsResponse) {
                             return;
                         }
 
-                        logger.debug(
-                            "hot threads from node [{}] lagging at version [{}] despite commit of cluster state version [{}]:\n{}",
-                            discoveryNode.descriptionWithoutAttributes(),
-                            appliedVersion,
-                            expectedVersion,
-                            nodesHotThreadsResponse.getNodes().get(0).getHotThreads()
+                        loggingTaskRunner.enqueueTask(
+                            new HotThreadsLoggingTask(
+                                discoveryNode,
+                                appliedVersion,
+                                expectedVersion,
+                                nodesHotThreadsResponse.getNodes().get(0).getHotThreads()
+                            )
                         );
                     }
 
@@ -281,4 +292,42 @@ public void onFailure(Exception e) {
         }
     }
 
+    static class HotThreadsLoggingTask extends AbstractRunnable implements Comparable<HotThreadsLoggingTask> {
+
+        private final String nodeHotThreads;
+        private final String prefix;
+
+        HotThreadsLoggingTask(DiscoveryNode discoveryNode, long appliedVersion, long expectedVersion, String nodeHotThreads) {
+            this.nodeHotThreads = nodeHotThreads;
+            this.prefix = Strings.format(
+                "hot threads from node [%s] lagging at version [%d] despite commit of cluster state version [%d]",
+                discoveryNode.descriptionWithoutAttributes(),
+                appliedVersion,
+                expectedVersion
+            );
+        }
+
+        @Override
+        public void onFailure(Exception e) {
+            logger.error(Strings.format("unexpected exception reporting %s", prefix), e);
+        }
+
+        @Override
+        protected void doRun() throws Exception {
+            try (
+                var writer = new OutputStreamWriter(
+                    ChunkedLoggingStream.create(logger, Level.DEBUG, prefix, ReferenceDocs.LAGGING_NODE_TROUBLESHOOTING),
+                    StandardCharsets.UTF_8
+                )
+            ) {
+                writer.write(nodeHotThreads);
+            }
+        }
+
+        @Override
+        public int compareTo(HotThreadsLoggingTask o) {
+            return 0;
+        }
+    }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java
new file mode 100644
index 000000000000..8a63fe027696
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java
@@ -0,0 +1,363 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.cluster.coordination;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateTaskExecutor;
+import org.elasticsearch.cluster.NotMasterException;
+import org.elasticsearch.cluster.block.ClusterBlocks;
+import org.elasticsearch.cluster.metadata.DesiredNodes;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RerouteService;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK;
+
+public class NodeJoinExecutor implements ClusterStateTaskExecutor<JoinTask> {
+
+    private static final Logger logger = LogManager.getLogger(NodeJoinExecutor.class);
+
+    private final AllocationService allocationService;
+    private final RerouteService rerouteService;
+
+    public NodeJoinExecutor(AllocationService allocationService, RerouteService rerouteService) {
+        this.allocationService = allocationService;
+        this.rerouteService = rerouteService;
+    }
+
+    @Override
+    public ClusterState execute(BatchExecutionContext<JoinTask> batchExecutionContext) throws Exception {
+        // The current state that MasterService uses might have been updated by a (different) master in a higher term already. If so, stop
+        // processing the current cluster state update, there's no point in continuing to compute it as it will later be rejected by
+        // Coordinator#publish anyhow.
+        assert batchExecutionContext.taskContexts().isEmpty() == false : "Expected to have non empty join tasks list";
+
+        var term = batchExecutionContext.taskContexts().stream().mapToLong(t -> t.getTask().term()).max().getAsLong();
+
+        var split = batchExecutionContext.taskContexts().stream().collect(Collectors.partitioningBy(t -> t.getTask().term() == term));
+        for (TaskContext<JoinTask> outdated : split.get(false)) {
+            outdated.onFailure(
+                new NotMasterException("Higher term encountered (encountered: " + term + " > used: " + outdated.getTask().term() + ")")
+            );
+        }
+
+        final var joinTaskContexts = split.get(true);
+        final var initialState = batchExecutionContext.initialState();
+
+        if (initialState.term() > term) {
+            logger.trace("encountered higher term {} than current {}, there is a newer master", initialState.term(), term);
+            throw new NotMasterException(
+                "Higher term encountered (current: " + initialState.term() + " > used: " + term + "), there is a newer master"
+            );
+        }
+
+        final boolean isBecomingMaster = joinTaskContexts.stream().anyMatch(t -> t.getTask().isBecomingMaster());
+        final DiscoveryNodes currentNodes = initialState.nodes();
+        boolean nodesChanged = false;
+        ClusterState.Builder newState;
+
+        if (currentNodes.getMasterNode() == null && isBecomingMaster) {
+            assert initialState.term() < term : "there should be at most one become master task per election (= by term)";
+            // use these joins to try and become the master.
+            // Note that we don't have to do any validation of the amount of joining nodes - the commit
+            // during the cluster state publishing guarantees that we have enough
+            try (var ignored = batchExecutionContext.dropHeadersContext()) {
+                // suppress deprecation warnings e.g. from reroute()
+                newState = becomeMasterAndTrimConflictingNodes(initialState, joinTaskContexts, term);
+            }
+            nodesChanged = true;
+        } else if (currentNodes.isLocalNodeElectedMaster()) {
+            assert initialState.term() == term : "term should be stable for the same master";
+            newState = ClusterState.builder(initialState);
+        } else {
+            logger.trace("processing node joins, but we are not the master. current master: {}", currentNodes.getMasterNode());
+            throw new NotMasterException("Node [" + currentNodes.getLocalNode() + "] not master for join request");
+        }
+
+        DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(newState.nodes());
+
+        assert nodesBuilder.isLocalNodeElectedMaster();
+
+        Version minClusterNodeVersion = newState.nodes().getMinNodeVersion();
+        Version maxClusterNodeVersion = newState.nodes().getMaxNodeVersion();
+        // if the cluster is not fully-formed then the min version is not meaningful
+        final boolean enforceVersionBarrier = initialState.getBlocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK) == false;
+        // processing any joins
+        Map<String, String> joinedNodeIdsByNodeName = new HashMap<>();
+        for (final var joinTaskContext : joinTaskContexts) {
+            final var joinTask = joinTaskContext.getTask();
+            final List<Runnable> onTaskSuccess = new ArrayList<>(joinTask.nodeCount());
+            for (final JoinTask.NodeJoinTask nodeJoinTask : joinTask.nodeJoinTasks()) {
+                final DiscoveryNode node = nodeJoinTask.node();
+                if (currentNodes.nodeExistsWithSameRoles(node)) {
+                    logger.debug("received a join request for an existing node [{}]", node);
+                } else {
+                    try {
+                        if (enforceVersionBarrier) {
+                            ensureVersionBarrier(node.getVersion(), minClusterNodeVersion);
+                        }
+                        ensureNodesCompatibility(node.getVersion(), minClusterNodeVersion, maxClusterNodeVersion);
+                        // we do this validation quite late to prevent race conditions between nodes joining and importing dangling indices
+                        // we have to reject nodes that don't support all indices we have in this cluster
+                        ensureIndexCompatibility(node.getVersion(), initialState.getMetadata());
+                        nodesBuilder.add(node);
+                        nodesChanged = true;
+                        minClusterNodeVersion = Version.min(minClusterNodeVersion, node.getVersion());
+                        maxClusterNodeVersion = Version.max(maxClusterNodeVersion, node.getVersion());
+                        if (node.isMasterNode()) {
+                            joinedNodeIdsByNodeName.put(node.getName(), node.getId());
+                        }
+                    } catch (IllegalArgumentException | IllegalStateException e) {
+                        onTaskSuccess.add(() -> nodeJoinTask.listener().onFailure(e));
+                        continue;
+                    }
+                }
+                onTaskSuccess.add(() -> {
+                    final var reason = nodeJoinTask.reason();
+                    if (reason.guidanceDocs() == null) {
+                        logger.info(
+                            "node-join: [{}] with reason [{}]",
+                            nodeJoinTask.node().descriptionWithoutAttributes(),
+                            reason.message()
+                        );
+                    } else {
+                        logger.warn(
+                            "node-join: [{}] with reason [{}]; for troubleshooting guidance, see {}",
+                            nodeJoinTask.node().descriptionWithoutAttributes(),
+                            reason.message(),
+                            reason.guidanceDocs()
+                        );
+                    }
+                    nodeJoinTask.listener().onResponse(null);
+                });
+            }
+            joinTaskContext.success(() -> {
+                for (Runnable joinCompleter : onTaskSuccess) {
+                    joinCompleter.run();
+                }
+            });
+        }
+
+        if (nodesChanged) {
+            rerouteService.reroute(
+                "post-join reroute",
+                Priority.HIGH,
+                ActionListener.wrap(r -> logger.trace("post-join reroute completed"), e -> logger.debug("post-join reroute failed", e))
+            );
+
+            if (joinedNodeIdsByNodeName.isEmpty() == false) {
+                final var currentVotingConfigExclusions = initialState.getVotingConfigExclusions();
+                final var newVotingConfigExclusions = currentVotingConfigExclusions.stream().map(e -> {
+                    // Update nodeId in VotingConfigExclusion when a new node with excluded node name joins
+                    if (CoordinationMetadata.VotingConfigExclusion.MISSING_VALUE_MARKER.equals(e.getNodeId())
+                        && joinedNodeIdsByNodeName.containsKey(e.getNodeName())) {
+                        return new CoordinationMetadata.VotingConfigExclusion(
+                            joinedNodeIdsByNodeName.get(e.getNodeName()),
+                            e.getNodeName()
+                        );
+                    } else {
+                        return e;
+                    }
+                }).collect(Collectors.toSet());
+
+                // if VotingConfigExclusions did get updated
+                if (newVotingConfigExclusions.equals(currentVotingConfigExclusions) == false) {
+                    final var coordMetadataBuilder = CoordinationMetadata.builder(initialState.coordinationMetadata())
+                        .term(term)
+                        .clearVotingConfigExclusions();
+                    newVotingConfigExclusions.forEach(coordMetadataBuilder::addVotingConfigExclusion);
+                    newState.metadata(Metadata.builder(initialState.metadata()).coordinationMetadata(coordMetadataBuilder.build()).build());
+                }
+            }
+
+            final ClusterState clusterStateWithNewNodesAndDesiredNodes = DesiredNodes.updateDesiredNodesStatusIfNeeded(
+                newState.nodes(nodesBuilder).build()
+            );
+            final ClusterState updatedState = allocationService.adaptAutoExpandReplicas(clusterStateWithNewNodesAndDesiredNodes);
+            assert enforceVersionBarrier == false
+                || updatedState.nodes().getMinNodeVersion().onOrAfter(initialState.nodes().getMinNodeVersion())
+                : "min node version decreased from ["
+                    + initialState.nodes().getMinNodeVersion()
+                    + "] to ["
+                    + updatedState.nodes().getMinNodeVersion()
+                    + "]";
+            return updatedState;
+        } else {
+            // we must return a new cluster state instance to force publishing. This is important
+            // for the joining node to finalize its join and set us as a master
+            return newState.build();
+        }
+    }
+
+    protected ClusterState.Builder becomeMasterAndTrimConflictingNodes(
+        ClusterState currentState,
+        List<TaskContext<JoinTask>> taskContexts,
+        long term
+    ) {
+        assert currentState.nodes().getMasterNodeId() == null : currentState;
+        assert currentState.term() < term : term + " vs " + currentState;
+        DiscoveryNodes currentNodes = currentState.nodes();
+        DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(currentNodes);
+        nodesBuilder.masterNodeId(currentState.nodes().getLocalNodeId());
+
+        for (final var taskContext : taskContexts) {
+            for (final var joiningNode : taskContext.getTask().nodes()) {
+                final DiscoveryNode nodeWithSameId = nodesBuilder.get(joiningNode.getId());
+                if (nodeWithSameId != null && nodeWithSameId.equals(joiningNode) == false) {
+                    logger.debug("removing existing node [{}], which conflicts with incoming join from [{}]", nodeWithSameId, joiningNode);
+                    nodesBuilder.remove(nodeWithSameId.getId());
+                }
+                final DiscoveryNode nodeWithSameAddress = currentNodes.findByAddress(joiningNode.getAddress());
+                if (nodeWithSameAddress != null && nodeWithSameAddress.equals(joiningNode) == false) {
+                    logger.debug(
+                        "removing existing node [{}], which conflicts with incoming join from [{}]",
+                        nodeWithSameAddress,
+                        joiningNode
+                    );
+                    nodesBuilder.remove(nodeWithSameAddress.getId());
+                }
+            }
+        }
+
+        // now trim any left over dead nodes - either left there when the previous master stepped down
+        // or removed by us above
+        ClusterState tmpState = ClusterState.builder(currentState)
+            .nodes(nodesBuilder)
+            .blocks(ClusterBlocks.builder().blocks(currentState.blocks()).removeGlobalBlock(NoMasterBlockService.NO_MASTER_BLOCK_ID))
+            .metadata(
+                Metadata.builder(currentState.metadata())
+                    .coordinationMetadata(CoordinationMetadata.builder(currentState.coordinationMetadata()).term(term).build())
+                    .build()
+            )
+            .build();
+        logger.trace("becomeMasterAndTrimConflictingNodes: {}", tmpState.nodes());
+        allocationService.cleanCaches();
+        tmpState = PersistentTasksCustomMetadata.disassociateDeadNodes(tmpState);
+        return ClusterState.builder(allocationService.disassociateDeadNodes(tmpState, false, "removed dead nodes on election"));
+    }
+
+    @Override
+    public boolean runOnlyOnMaster() {
+        // we validate that we are allowed to change the cluster state during cluster state processing
+        return false;
+    }
+
+    /**
+     * Ensures that all indices are compatible with the given node version. This will ensure that all indices in the given metadata
+     * will not be created with a newer version of elasticsearch as well as that all indices are newer or equal to the minimum index
+     * compatibility version.
+     * @see Version#minimumIndexCompatibilityVersion()
+     * @throws IllegalStateException if any index is incompatible with the given version
+     */
+    public static void ensureIndexCompatibility(final Version nodeVersion, Metadata metadata) {
+        Version supportedIndexVersion = nodeVersion.minimumIndexCompatibilityVersion();
+        // we ensure that all indices in the cluster we join are compatible with us no matter if they are
+        // closed or not we can't read mappings of these indices so we need to reject the join...
+        for (IndexMetadata idxMetadata : metadata) {
+            if (idxMetadata.getCompatibilityVersion().after(nodeVersion)) {
+                throw new IllegalStateException(
+                    "index "
+                        + idxMetadata.getIndex()
+                        + " version not supported: "
+                        + idxMetadata.getCompatibilityVersion()
+                        + " the node version is: "
+                        + nodeVersion
+                );
+            }
+            if (idxMetadata.getCompatibilityVersion().before(supportedIndexVersion)) {
+                throw new IllegalStateException(
+                    "index "
+                        + idxMetadata.getIndex()
+                        + " version not supported: "
+                        + idxMetadata.getCompatibilityVersion()
+                        + " minimum compatible index version is: "
+                        + supportedIndexVersion
+                );
+            }
+        }
+    }
+
+    /** ensures that the joining node has a version that's compatible with all current nodes*/
+    public static void ensureNodesCompatibility(final Version joiningNodeVersion, DiscoveryNodes currentNodes) {
+        final Version minNodeVersion = currentNodes.getMinNodeVersion();
+        final Version maxNodeVersion = currentNodes.getMaxNodeVersion();
+        ensureNodesCompatibility(joiningNodeVersion, minNodeVersion, maxNodeVersion);
+    }
+
+    /** ensures that the joining node has a version that's compatible with a given version range */
+    public static void ensureNodesCompatibility(Version joiningNodeVersion, Version minClusterNodeVersion, Version maxClusterNodeVersion) {
+        assert minClusterNodeVersion.onOrBefore(maxClusterNodeVersion) : minClusterNodeVersion + " > " + maxClusterNodeVersion;
+        if (joiningNodeVersion.isCompatible(maxClusterNodeVersion) == false) {
+            throw new IllegalStateException(
+                "node version ["
+                    + joiningNodeVersion
+                    + "] is not supported. "
+                    + "The cluster contains nodes with version ["
+                    + maxClusterNodeVersion
+                    + "], which is incompatible."
+            );
+        }
+        if (joiningNodeVersion.isCompatible(minClusterNodeVersion) == false) {
+            throw new IllegalStateException(
+                "node version ["
+                    + joiningNodeVersion
+                    + "] is not supported."
+                    + "The cluster contains nodes with version ["
+                    + minClusterNodeVersion
+                    + "], which is incompatible."
+            );
+        }
+    }
+
+    /**
+     * ensures that the joining node's version is equal or higher to the minClusterNodeVersion. This is needed
+     * to ensure that if the master is already fully operating under the new version, it doesn't go back to mixed
+     * version mode
+     **/
+    public static void ensureVersionBarrier(Version joiningNodeVersion, Version minClusterNodeVersion) {
+        if (joiningNodeVersion.before(minClusterNodeVersion)) {
+            throw new IllegalStateException(
+                "node version ["
+                    + joiningNodeVersion
+                    + "] may not join a cluster comprising only nodes of version ["
+                    + minClusterNodeVersion
+                    + "] or greater"
+            );
+        }
+    }
+
+    public static Collection<BiConsumer<DiscoveryNode, ClusterState>> addBuiltInJoinValidators(
+        Collection<BiConsumer<DiscoveryNode, ClusterState>> onJoinValidators
+    ) {
+        final Collection<BiConsumer<DiscoveryNode, ClusterState>> validators = new ArrayList<>();
+        validators.add((node, state) -> {
+            ensureNodesCompatibility(node.getVersion(), state.getNodes());
+            ensureIndexCompatibility(node.getVersion(), state.getMetadata());
+        });
+        validators.addAll(onJoinValidators);
+        return Collections.unmodifiableCollection(validators);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeLeftExecutor.java b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeLeftExecutor.java
new file mode 100644
index 000000000000..aad175c97098
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeLeftExecutor.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.cluster.coordination;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateTaskExecutor;
+import org.elasticsearch.cluster.ClusterStateTaskListener;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.service.MasterService;
+import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
+
+public class NodeLeftExecutor implements ClusterStateTaskExecutor<NodeLeftExecutor.Task> {
+
+    private static final Logger logger = LogManager.getLogger(NodeLeftExecutor.class);
+
+    private final AllocationService allocationService;
+
+    public record Task(DiscoveryNode node, String reason, Runnable onClusterStateProcessed) implements ClusterStateTaskListener {
+
+        @Override
+        public void onFailure(final Exception e) {
+            logger.log(MasterService.isPublishFailureException(e) ? Level.DEBUG : Level.ERROR, "unexpected failure during [node-left]", e);
+        }
+
+        @Override
+        public String toString() {
+            final StringBuilder stringBuilder = new StringBuilder();
+            node.appendDescriptionWithoutAttributes(stringBuilder);
+            stringBuilder.append(" reason: ").append(reason);
+            return stringBuilder.toString();
+        }
+    }
+
+    public NodeLeftExecutor(AllocationService allocationService) {
+        this.allocationService = allocationService;
+    }
+
+    @Override
+    public ClusterState execute(BatchExecutionContext<Task> batchExecutionContext) throws Exception {
+        final ClusterState initialState = batchExecutionContext.initialState();
+        final DiscoveryNodes.Builder remainingNodesBuilder = DiscoveryNodes.builder(initialState.nodes());
+        boolean removed = false;
+        for (final var taskContext : batchExecutionContext.taskContexts()) {
+            final var task = taskContext.getTask();
+            final String reason;
+            if (initialState.nodes().nodeExists(task.node())) {
+                remainingNodesBuilder.remove(task.node());
+                removed = true;
+                reason = task.reason();
+            } else {
+                logger.debug("node [{}] does not exist in cluster state, ignoring", task);
+                reason = null;
+            }
+            taskContext.success(() -> {
+                if (reason != null) {
+                    logger.info("node-left: [{}] with reason [{}]", task.node().descriptionWithoutAttributes(), reason);
+                }
+                task.onClusterStateProcessed.run();
+            });
+        }
+
+        if (removed == false) {
+            // no nodes to remove, keep the current cluster state
+            return initialState;
+        }
+
+        try (var ignored = batchExecutionContext.dropHeadersContext()) {
+            // suppress deprecation warnings e.g. from reroute()
+
+            final var remainingNodesClusterState = remainingNodesClusterState(initialState, remainingNodesBuilder);
+            final var ptasksDisassociatedState = PersistentTasksCustomMetadata.disassociateDeadNodes(remainingNodesClusterState);
+            return allocationService.disassociateDeadNodes(
+                ptasksDisassociatedState,
+                true,
+                describeTasks(batchExecutionContext.taskContexts().stream().map(TaskContext::getTask).toList())
+            );
+        }
+    }
+
+    // visible for testing
+    // hook is used in testing to ensure that correct cluster state is used to test whether a
+    // rejoin or reroute is needed
+    protected ClusterState remainingNodesClusterState(final ClusterState currentState, DiscoveryNodes.Builder remainingNodesBuilder) {
+        return ClusterState.builder(currentState).nodes(remainingNodesBuilder).build();
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeRemovalClusterStateTaskExecutor.java b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeRemovalClusterStateTaskExecutor.java
deleted file mode 100644
index 7cf929d6da87..000000000000
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeRemovalClusterStateTaskExecutor.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.cluster.coordination;
-
-import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateTaskExecutor;
-import org.elasticsearch.cluster.ClusterStateTaskListener;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.service.MasterService;
-import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
-
-public class NodeRemovalClusterStateTaskExecutor implements ClusterStateTaskExecutor<NodeRemovalClusterStateTaskExecutor.Task> {
-
-    private static final Logger logger = LogManager.getLogger(NodeRemovalClusterStateTaskExecutor.class);
-
-    private final AllocationService allocationService;
-
-    public record Task(DiscoveryNode node, String reason, Runnable onClusterStateProcessed) implements ClusterStateTaskListener {
-
-        @Override
-        public void onFailure(final Exception e) {
-            logger.log(MasterService.isPublishFailureException(e) ? Level.DEBUG : Level.ERROR, "unexpected failure during [node-left]", e);
-        }
-
-        @Override
-        public String toString() {
-            final StringBuilder stringBuilder = new StringBuilder();
-            node.appendDescriptionWithoutAttributes(stringBuilder);
-            stringBuilder.append(" reason: ").append(reason);
-            return stringBuilder.toString();
-        }
-    }
-
-    public NodeRemovalClusterStateTaskExecutor(AllocationService allocationService) {
-        this.allocationService = allocationService;
-    }
-
-    @Override
-    public ClusterState execute(BatchExecutionContext<Task> batchExecutionContext) throws Exception {
-        final ClusterState initialState = batchExecutionContext.initialState();
-        final DiscoveryNodes.Builder remainingNodesBuilder = DiscoveryNodes.builder(initialState.nodes());
-        boolean removed = false;
-        for (final var taskContext : batchExecutionContext.taskContexts()) {
-            final var task = taskContext.getTask();
-            if (initialState.nodes().nodeExists(task.node())) {
-                remainingNodesBuilder.remove(task.node());
-                removed = true;
-            } else {
-                logger.debug("node [{}] does not exist in cluster state, ignoring", task);
-            }
-            taskContext.success(task.onClusterStateProcessed::run);
-        }
-
-        if (removed == false) {
-            // no nodes to remove, keep the current cluster state
-            return initialState;
-        }
-
-        try (var ignored = batchExecutionContext.dropHeadersContext()) {
-            // suppress deprecation warnings e.g. from reroute()
-
-            final var remainingNodesClusterState = remainingNodesClusterState(initialState, remainingNodesBuilder);
-            final var ptasksDisassociatedState = PersistentTasksCustomMetadata.disassociateDeadNodes(remainingNodesClusterState);
-            return allocationService.disassociateDeadNodes(
-                ptasksDisassociatedState,
-                true,
-                describeTasks(batchExecutionContext.taskContexts().stream().map(TaskContext::getTask).toList())
-            );
-        }
-    }
-
-    // visible for testing
-    // hook is used in testing to ensure that correct cluster state is used to test whether a
-    // rejoin or reroute is needed
-    protected ClusterState remainingNodesClusterState(final ClusterState currentState, DiscoveryNodes.Builder remainingNodesBuilder) {
-        return ClusterState.builder(currentState).nodes(remainingNodesBuilder).build();
-    }
-
-}
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
index 5dd308640cc9..4d6b4ce1edd0 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java
@@ -10,9 +10,9 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionListenerResponseHandler;
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStatePublicationEvent;
@@ -86,6 +86,8 @@ public class PublicationTransportHandler {
         TransportRequestOptions.Type.STATE
     );
 
+    public static final TransportVersion INCLUDES_LAST_COMMITTED_DATA_VERSION = TransportVersion.V_8_6_0;
+
     private final SerializationStatsTracker serializationStatsTracker = new SerializationStatsTracker();
 
     public PublicationTransportHandler(
@@ -124,13 +126,14 @@ private PublishWithJoinResponse handleIncomingPublishRequest(BytesTransportReque
                 in = new InputStreamStreamInput(compressor.threadLocalInputStream(in));
             }
             in = new NamedWriteableAwareStreamInput(in, namedWriteableRegistry);
-            in.setVersion(request.version());
+            in.setTransportVersion(request.version());
             // If true we received full cluster state - otherwise diffs
             if (in.readBoolean()) {
                 final ClusterState incomingState;
                 // Close early to release resources used by the de-compression as early as possible
                 try (StreamInput input = in) {
                     incomingState = ClusterState.readFrom(input, transportService.getLocalNode());
+                    assert input.read() == -1;
                 } catch (Exception e) {
                     logger.warn("unexpected error while deserializing an incoming cluster state", e);
                     assert false : e;
@@ -151,11 +154,30 @@ private PublishWithJoinResponse handleIncomingPublishRequest(BytesTransportReque
                     ClusterState incomingState;
                     try {
                         final Diff<ClusterState> diff;
+                        final boolean includesLastCommittedData = request.version().onOrAfter(INCLUDES_LAST_COMMITTED_DATA_VERSION);
+                        final boolean clusterUuidCommitted;
+                        final CoordinationMetadata.VotingConfiguration lastCommittedConfiguration;
+
                         // Close stream early to release resources used by the de-compression as early as possible
                         try (StreamInput input = in) {
                             diff = ClusterState.readDiffFrom(input, lastSeen.nodes().getLocalNode());
+                            if (includesLastCommittedData) {
+                                clusterUuidCommitted = in.readBoolean();
+                                lastCommittedConfiguration = new CoordinationMetadata.VotingConfiguration(in);
+                            } else {
+                                clusterUuidCommitted = false;
+                                lastCommittedConfiguration = null;
+                            }
+                            assert input.read() == -1;
                         }
                         incomingState = diff.apply(lastSeen); // might throw IncompatibleClusterStateVersionException
+                        if (includesLastCommittedData) {
+                            final var adjustedMetadata = incomingState.metadata()
+                                .withLastCommittedValues(clusterUuidCommitted, lastCommittedConfiguration);
+                            if (adjustedMetadata != incomingState.metadata()) {
+                                incomingState = ClusterState.builder(incomingState).metadata(adjustedMetadata).build();
+                            }
+                        }
                     } catch (IncompatibleClusterStateVersionException e) {
                         incompatibleClusterStateDiffReceivedCount.incrementAndGet();
                         throw e;
@@ -205,7 +227,7 @@ public PublicationContext newPublicationContext(ClusterStatePublicationEvent clu
     }
 
     private ReleasableBytesReference serializeFullClusterState(ClusterState clusterState, DiscoveryNode node) {
-        final Version nodeVersion = node.getVersion();
+        final TransportVersion serializeVersion = node.getVersion().transportVersion;
         final RecyclerBytesStreamOutput bytesStream = transportService.newNetworkBytesStream();
         boolean success = false;
         try {
@@ -215,7 +237,7 @@ private ReleasableBytesReference serializeFullClusterState(ClusterState clusterS
                     CompressorFactory.COMPRESSOR.threadLocalOutputStream(Streams.flushOnCloseStream(bytesStream))
                 )
             ) {
-                stream.setVersion(nodeVersion);
+                stream.setTransportVersion(serializeVersion);
                 stream.writeBoolean(true);
                 clusterState.writeTo(stream);
                 uncompressedBytes = stream.position();
@@ -225,9 +247,9 @@ private ReleasableBytesReference serializeFullClusterState(ClusterState clusterS
             final ReleasableBytesReference result = new ReleasableBytesReference(bytesStream.bytes(), bytesStream);
             serializationStatsTracker.serializedFullState(uncompressedBytes, result.length());
             logger.trace(
-                "serialized full cluster state version [{}] for node version [{}] with size [{}]",
+                "serialized full cluster state version [{}] using transport version [{}] with size [{}]",
                 clusterState.version(),
-                nodeVersion,
+                serializeVersion,
                 result.length()
             );
             success = true;
@@ -239,8 +261,9 @@ private ReleasableBytesReference serializeFullClusterState(ClusterState clusterS
         }
     }
 
-    private ReleasableBytesReference serializeDiffClusterState(long clusterStateVersion, Diff<ClusterState> diff, DiscoveryNode node) {
-        final Version nodeVersion = node.getVersion();
+    private ReleasableBytesReference serializeDiffClusterState(ClusterState newState, Diff<ClusterState> diff, DiscoveryNode node) {
+        final long clusterStateVersion = newState.version();
+        final TransportVersion serializeVersion = node.getVersion().transportVersion;
         final RecyclerBytesStreamOutput bytesStream = transportService.newNetworkBytesStream();
         boolean success = false;
         try {
@@ -250,9 +273,13 @@ private ReleasableBytesReference serializeDiffClusterState(long clusterStateVers
                     CompressorFactory.COMPRESSOR.threadLocalOutputStream(Streams.flushOnCloseStream(bytesStream))
                 )
             ) {
-                stream.setVersion(nodeVersion);
+                stream.setTransportVersion(serializeVersion);
                 stream.writeBoolean(false);
                 diff.writeTo(stream);
+                if (serializeVersion.onOrAfter(INCLUDES_LAST_COMMITTED_DATA_VERSION)) {
+                    stream.writeBoolean(newState.metadata().clusterUUIDCommitted());
+                    newState.getLastCommittedConfiguration().writeTo(stream);
+                }
                 uncompressedBytes = stream.position();
             } catch (IOException e) {
                 throw new ElasticsearchException("failed to serialize cluster state diff for publishing to node {}", e, node);
@@ -260,9 +287,9 @@ private ReleasableBytesReference serializeDiffClusterState(long clusterStateVers
             final ReleasableBytesReference result = new ReleasableBytesReference(bytesStream.bytes(), bytesStream);
             serializationStatsTracker.serializedDiff(uncompressedBytes, result.length());
             logger.trace(
-                "serialized cluster state diff for version [{}] for node version [{}] with size [{}]",
+                "serialized cluster state diff for version [{}] using transport version [{}] with size [{}]",
                 clusterStateVersion,
-                nodeVersion,
+                serializeVersion,
                 result.length()
             );
             success = true;
@@ -316,7 +343,7 @@ void buildDiffAndSerializeStates() {
                 } else {
                     serializedDiffs.computeIfAbsent(
                         node.getVersion(),
-                        v -> serializeDiffClusterState(newState.version(), diffSupplier.getOrCompute(), node)
+                        v -> serializeDiffClusterState(newState, diffSupplier.getOrCompute(), node)
                     );
                 }
             }
@@ -440,14 +467,10 @@ private void sendClusterState(
             transportService.sendChildRequest(
                 destination,
                 PUBLISH_STATE_ACTION_NAME,
-                new BytesTransportRequest(bytes, destination.getVersion()),
+                new BytesTransportRequest(bytes, destination.getVersion().transportVersion),
                 task,
                 STATE_REQUEST_OPTIONS,
-                new ActionListenerResponseHandler<>(
-                    ActionListener.runAfter(listener, bytes::decRef),
-                    PublishWithJoinResponse::new,
-                    ThreadPool.Names.CLUSTER_COORDINATION
-                )
+                new CleanableResponseHandler<>(listener, PublishWithJoinResponse::new, ThreadPool.Names.CLUSTER_COORDINATION, bytes::decRef)
             );
         }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorService.java
index 2ed8dca833e5..2fd9b6304e2f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorService.java
@@ -41,7 +41,7 @@ public class StableMasterHealthIndicatorService implements HealthIndicatorServic
 
     public static final String NAME = "master_is_stable";
     public static final String GET_HELP_GUIDE = "https://ela.st/getting-help";
-    public static final Diagnosis CONTACT_SUPPORT_USER_ACTION = new Diagnosis(
+    public static final Diagnosis CONTACT_SUPPORT = new Diagnosis(
         new Diagnosis.Definition(
             NAME,
             "contact_support",
@@ -104,7 +104,7 @@ public String name() {
     }
 
     @Override
-    public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
+    public HealthIndicatorResult calculate(boolean verbose, int maxAffectedResourcesCount, HealthInfo healthInfo) {
         CoordinationDiagnosticsService.CoordinationDiagnosticsResult coordinationDiagnosticsResult = coordinationDiagnosticsService
             .diagnoseMasterStability(verbose);
         return getHealthIndicatorResult(coordinationDiagnosticsResult, verbose);
@@ -218,7 +218,7 @@ private String getNameForNodeId(String nodeId) {
      */
     private List<Diagnosis> getContactSupportUserActions(boolean explain) {
         if (explain) {
-            return List.of(CONTACT_SUPPORT_USER_ACTION);
+            return List.of(CONTACT_SUPPORT);
         } else {
             return List.of();
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java
index c4683aaabbae..a094fbee07e1 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java
@@ -116,10 +116,12 @@ protected void processDataPaths(Terminal terminal, Path[] dataPaths, OptionSet o
 
         final ClusterState newClusterState = ClusterState.builder(oldClusterState).metadata(newMetadata).build();
 
-        terminal.println(
-            Terminal.Verbosity.VERBOSE,
-            "[old cluster state = " + oldClusterState + ", new cluster state = " + newClusterState + "]"
-        );
+        if (terminal.isPrintable(Terminal.Verbosity.VERBOSE)) {
+            terminal.println(
+                Terminal.Verbosity.VERBOSE,
+                "[old cluster state = " + oldClusterState + ", new cluster state = " + newClusterState + "]"
+            );
+        }
 
         confirm(terminal, CONFIRMATION_MSG);
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ValidateJoinRequest.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ValidateJoinRequest.java
index 464be6284d21..d2f9f18466aa 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/ValidateJoinRequest.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ValidateJoinRequest.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.cluster.coordination;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -28,10 +28,10 @@ public class ValidateJoinRequest extends TransportRequest {
 
     public ValidateJoinRequest(StreamInput in) throws IOException {
         super(in);
-        if (in.getVersion().onOrAfter(Version.V_8_3_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0)) {
             // recent versions send a BytesTransportRequest containing a compressed representation of the state
             final var bytes = in.readReleasableBytesReference();
-            final var version = in.getVersion();
+            final var version = in.getTransportVersion();
             final var namedWriteableRegistry = in.namedWriteableRegistry();
             this.stateSupplier = () -> readCompressed(version, bytes, namedWriteableRegistry);
             this.refCounted = bytes;
@@ -43,8 +43,11 @@ public ValidateJoinRequest(StreamInput in) throws IOException {
         }
     }
 
-    private static ClusterState readCompressed(Version version, BytesReference bytes, NamedWriteableRegistry namedWriteableRegistry)
-        throws IOException {
+    private static ClusterState readCompressed(
+        TransportVersion version,
+        BytesReference bytes,
+        NamedWriteableRegistry namedWriteableRegistry
+    ) throws IOException {
         try (
             var bytesStreamInput = bytes.streamInput();
             var in = new NamedWriteableAwareStreamInput(
@@ -52,7 +55,7 @@ private static ClusterState readCompressed(Version version, BytesReference bytes
                 namedWriteableRegistry
             )
         ) {
-            in.setVersion(version);
+            in.setTransportVersion(version);
             return ClusterState.readFrom(in, null);
         }
     }
@@ -64,7 +67,7 @@ public ValidateJoinRequest(ClusterState state) {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        assert out.getVersion().before(Version.V_8_3_0);
+        assert out.getTransportVersion().before(TransportVersion.V_8_3_0);
         super.writeTo(out);
         stateSupplier.get().writeTo(out);
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
index 92fe6f18f9b5..26548f7b824c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java
@@ -8,21 +8,23 @@
 
 package org.elasticsearch.cluster.metadata;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.DiffableUtils;
 import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Objects;
 
@@ -84,8 +86,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_7_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_7_0;
     }
 
     @Override
@@ -98,13 +100,8 @@ public static ComponentTemplateMetadata fromXContent(XContentParser parser) thro
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(COMPONENT_TEMPLATE.getPreferredName());
-        for (Map.Entry<String, ComponentTemplate> template : componentTemplates.entrySet()) {
-            builder.field(template.getKey(), template.getValue(), params);
-        }
-        builder.endObject();
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return ChunkedToXContentHelper.xContentValuesMap(COMPONENT_TEMPLATE.getPreferredName(), componentTemplates);
     }
 
     @Override
@@ -166,8 +163,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.V_7_7_0;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.V_7_7_0;
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java
index 2a8ea2dc1b6b..377d91d60a99 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.SimpleDiffable;
@@ -46,6 +47,7 @@ public class ComposableIndexTemplate implements SimpleDiffable<ComposableIndexTe
     private static final ParseField METADATA = new ParseField("_meta");
     private static final ParseField DATA_STREAM = new ParseField("data_stream");
     private static final ParseField ALLOW_AUTO_CREATE = new ParseField("allow_auto_create");
+    private static final ParseField IGNORE_MISSING_COMPONENT_TEMPLATES = new ParseField("ignore_missing_component_templates");
 
     @SuppressWarnings("unchecked")
     public static final ConstructingObjectParser<ComposableIndexTemplate, Void> PARSER = new ConstructingObjectParser<>(
@@ -59,7 +61,8 @@ public class ComposableIndexTemplate implements SimpleDiffable<ComposableIndexTe
             (Long) a[4],
             (Map<String, Object>) a[5],
             (DataStreamTemplate) a[6],
-            (Boolean) a[7]
+            (Boolean) a[7],
+            (List<String>) a[8]
         )
     );
 
@@ -72,6 +75,7 @@ public class ComposableIndexTemplate implements SimpleDiffable<ComposableIndexTe
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA);
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), DataStreamTemplate.PARSER, DATA_STREAM);
         PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), ALLOW_AUTO_CREATE);
+        PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), IGNORE_MISSING_COMPONENT_TEMPLATES);
     }
 
     private final List<String> indexPatterns;
@@ -89,6 +93,8 @@ public class ComposableIndexTemplate implements SimpleDiffable<ComposableIndexTe
     private final DataStreamTemplate dataStreamTemplate;
     @Nullable
     private final Boolean allowAutoCreate;
+    @Nullable
+    private final List<String> ignoreMissingComponentTemplates;
 
     static Diff<ComposableIndexTemplate> readITV2DiffFrom(StreamInput in) throws IOException {
         return SimpleDiffable.readDiffFrom(ComposableIndexTemplate::new, in);
@@ -106,7 +112,7 @@ public ComposableIndexTemplate(
         @Nullable Long version,
         @Nullable Map<String, Object> metadata
     ) {
-        this(indexPatterns, template, componentTemplates, priority, version, metadata, null, null);
+        this(indexPatterns, template, componentTemplates, priority, version, metadata, null, null, null);
     }
 
     public ComposableIndexTemplate(
@@ -118,7 +124,7 @@ public ComposableIndexTemplate(
         @Nullable Map<String, Object> metadata,
         @Nullable DataStreamTemplate dataStreamTemplate
     ) {
-        this(indexPatterns, template, componentTemplates, priority, version, metadata, dataStreamTemplate, null);
+        this(indexPatterns, template, componentTemplates, priority, version, metadata, dataStreamTemplate, null, null);
     }
 
     public ComposableIndexTemplate(
@@ -130,6 +136,20 @@ public ComposableIndexTemplate(
         @Nullable Map<String, Object> metadata,
         @Nullable DataStreamTemplate dataStreamTemplate,
         @Nullable Boolean allowAutoCreate
+    ) {
+        this(indexPatterns, template, componentTemplates, priority, version, metadata, dataStreamTemplate, null, null);
+    }
+
+    public ComposableIndexTemplate(
+        List<String> indexPatterns,
+        @Nullable Template template,
+        @Nullable List<String> componentTemplates,
+        @Nullable Long priority,
+        @Nullable Long version,
+        @Nullable Map<String, Object> metadata,
+        @Nullable DataStreamTemplate dataStreamTemplate,
+        @Nullable Boolean allowAutoCreate,
+        @Nullable List<String> ignoreMissingComponentTemplates
     ) {
         this.indexPatterns = indexPatterns;
         this.template = template;
@@ -139,6 +159,7 @@ public ComposableIndexTemplate(
         this.metadata = metadata;
         this.dataStreamTemplate = dataStreamTemplate;
         this.allowAutoCreate = allowAutoCreate;
+        this.ignoreMissingComponentTemplates = ignoreMissingComponentTemplates;
     }
 
     public ComposableIndexTemplate(StreamInput in) throws IOException {
@@ -154,6 +175,11 @@ public ComposableIndexTemplate(StreamInput in) throws IOException {
         this.metadata = in.readMap();
         this.dataStreamTemplate = in.readOptionalWriteable(DataStreamTemplate::new);
         this.allowAutoCreate = in.readOptionalBoolean();
+        if (in.getVersion().onOrAfter(Version.V_8_7_0)) {
+            this.ignoreMissingComponentTemplates = in.readOptionalStringList();
+        } else {
+            this.ignoreMissingComponentTemplates = null;
+        }
     }
 
     public List<String> indexPatterns() {
@@ -204,6 +230,11 @@ public Boolean getAllowAutoCreate() {
         return this.allowAutoCreate;
     }
 
+    @Nullable
+    public List<String> getIgnoreMissingComponentTemplates() {
+        return ignoreMissingComponentTemplates;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeStringCollection(this.indexPatterns);
@@ -219,6 +250,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeGenericMap(this.metadata);
         out.writeOptionalWriteable(dataStreamTemplate);
         out.writeOptionalBoolean(allowAutoCreate);
+        if (out.getVersion().onOrAfter(Version.V_8_7_0)) {
+            out.writeOptionalStringCollection(ignoreMissingComponentTemplates);
+        }
     }
 
     @Override
@@ -246,6 +280,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (this.allowAutoCreate != null) {
             builder.field(ALLOW_AUTO_CREATE.getPreferredName(), allowAutoCreate);
         }
+        if (this.ignoreMissingComponentTemplates != null) {
+            builder.stringListField(IGNORE_MISSING_COMPONENT_TEMPLATES.getPreferredName(), ignoreMissingComponentTemplates);
+        }
         builder.endObject();
         return builder;
     }
@@ -260,7 +297,8 @@ public int hashCode() {
             this.version,
             this.metadata,
             this.dataStreamTemplate,
-            this.allowAutoCreate
+            this.allowAutoCreate,
+            this.ignoreMissingComponentTemplates
         );
     }
 
@@ -280,7 +318,8 @@ && componentTemplatesEquals(this.componentTemplates, other.componentTemplates)
             && Objects.equals(this.version, other.version)
             && Objects.equals(this.metadata, other.metadata)
             && Objects.equals(this.dataStreamTemplate, other.dataStreamTemplate)
-            && Objects.equals(this.allowAutoCreate, other.allowAutoCreate);
+            && Objects.equals(this.allowAutoCreate, other.allowAutoCreate)
+            && Objects.equals(this.ignoreMissingComponentTemplates, other.ignoreMissingComponentTemplates);
     }
 
     static boolean componentTemplatesEquals(List<String> c1, List<String> c2) {
@@ -331,12 +370,12 @@ public DataStreamTemplate(boolean hidden, boolean allowCustomRouting) {
 
         DataStreamTemplate(StreamInput in) throws IOException {
             hidden = in.readBoolean();
-            if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
                 allowCustomRouting = in.readBoolean();
             } else {
                 allowCustomRouting = false;
             }
-            if (in.getVersion().onOrAfter(Version.V_8_1_0) && in.getVersion().before(Version.V_8_3_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0) && in.getTransportVersion().before(TransportVersion.V_8_3_0)) {
                 // Accidentally included index_mode to binary node to node protocol in previous releases.
                 // (index_mode is removed and was part of code based when tsdb was behind a feature flag)
                 // (index_mode was behind a feature in the xcontent parser, so it could never actually used)
@@ -379,10 +418,11 @@ public boolean isAllowCustomRouting() {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(hidden);
-            if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
                 out.writeBoolean(allowCustomRouting);
             }
-            if (out.getVersion().onOrAfter(Version.V_8_1_0) && out.getVersion().before(Version.V_8_3_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)
+                && out.getTransportVersion().before(TransportVersion.V_8_3_0)) {
                 // See comment in constructor.
                 out.writeBoolean(false);
             }
@@ -420,6 +460,7 @@ public static class Builder {
         private Map<String, Object> metadata;
         private DataStreamTemplate dataStreamTemplate;
         private Boolean allowAutoCreate;
+        private List<String> ignoreMissingComponentTemplates;
 
         public Builder() {}
 
@@ -463,6 +504,11 @@ public Builder allowAutoCreate(Boolean allowAutoCreate) {
             return this;
         }
 
+        public Builder ignoreMissingComponentTemplates(List<String> ignoreMissingComponentTemplates) {
+            this.ignoreMissingComponentTemplates = ignoreMissingComponentTemplates;
+            return this;
+        }
+
         public ComposableIndexTemplate build() {
             return new ComposableIndexTemplate(
                 this.indexPatterns,
@@ -472,7 +518,8 @@ public ComposableIndexTemplate build() {
                 this.version,
                 this.metadata,
                 this.dataStreamTemplate,
-                this.allowAutoCreate
+                this.allowAutoCreate,
+                this.ignoreMissingComponentTemplates
             );
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java
index 1b5193510aa6..53c73c7906b9 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java
@@ -8,21 +8,23 @@
 
 package org.elasticsearch.cluster.metadata;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.DiffableUtils;
 import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Objects;
 
@@ -89,8 +91,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_7_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_7_0;
     }
 
     @Override
@@ -99,13 +101,8 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(INDEX_TEMPLATE.getPreferredName());
-        for (Map.Entry<String, ComposableIndexTemplate> template : indexTemplates.entrySet()) {
-            builder.field(template.getKey(), template.getValue(), params);
-        }
-        builder.endObject();
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return ChunkedToXContentHelper.xContentValuesMap(INDEX_TEMPLATE.getPreferredName(), indexTemplates);
     }
 
     @Override
@@ -167,8 +164,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.V_7_7_0;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.V_7_7_0;
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
index 297219107b13..8e66cecd19c4 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.PointValues;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.SimpleDiffable;
 import org.elasticsearch.common.Strings;
@@ -294,10 +294,13 @@ public DataStream rollover(Index writeIndex, long generation, boolean timeSeries
      */
     public DataStream unsafeRollover(Index writeIndex, long generation, boolean timeSeries) {
         IndexMode indexMode = this.indexMode;
-        // This allows for migrating a data stream to be a tsdb data stream:
-        // (only if index_mode=null|standard then allow it to be set to time_series)
         if ((indexMode == null || indexMode == IndexMode.STANDARD) && timeSeries) {
+            // This allows for migrating a data stream to be a tsdb data stream:
+            // (only if index_mode=null|standard then allow it to be set to time_series)
             indexMode = IndexMode.TIME_SERIES;
+        } else if (indexMode == IndexMode.TIME_SERIES && timeSeries == false) {
+            // Allow downgrading a time series data stream to a regular data stream
+            indexMode = null;
         }
 
         List<Index> backingIndices = new ArrayList<>(indices);
@@ -526,8 +529,8 @@ public DataStream(StreamInput in) throws IOException {
             in.readBoolean(),
             in.readBoolean(),
             in.readBoolean(),
-            in.getVersion().onOrAfter(Version.V_8_0_0) ? in.readBoolean() : false,
-            in.getVersion().onOrAfter(Version.V_8_1_0) ? in.readOptionalEnum(IndexMode.class) : null
+            in.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0) ? in.readBoolean() : false,
+            in.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0) ? in.readOptionalEnum(IndexMode.class) : null
         );
     }
 
@@ -550,10 +553,10 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(hidden);
         out.writeBoolean(replicated);
         out.writeBoolean(system);
-        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             out.writeBoolean(allowCustomRouting);
         }
-        if (out.getVersion().onOrAfter(Version.V_8_1_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
             out.writeOptionalEnum(indexMode);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAlias.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAlias.java
index cac3e342c1dd..7efba02dfe34 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAlias.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamAlias.java
@@ -7,6 +7,7 @@
  */
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.SimpleDiffable;
 import org.elasticsearch.common.ParsingException;
@@ -15,6 +16,8 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -25,29 +28,62 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 public class DataStreamAlias implements SimpleDiffable<DataStreamAlias>, ToXContentFragment {
 
     public static final ParseField DATA_STREAMS_FIELD = new ParseField("data_streams");
     public static final ParseField WRITE_DATA_STREAM_FIELD = new ParseField("write_data_stream");
-    public static final ParseField FILTER_FIELD = new ParseField("filter");
+    /*
+     * Before 8.7.0, we incorrectly only kept one filter for all DataStreams in the DataStreamAlias. This field remains here so that we can
+     * read old cluster states during an upgrade. We never write XContent with this field as of 8.7.0.
+     */
+    public static final ParseField OLD_FILTER_FIELD = new ParseField("filter");
+    public static final ParseField FILTERS_FIELD = new ParseField("filters");
+    private static final Logger logger = LogManager.getLogger(DataStreamAlias.class);
 
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<DataStreamAlias, String> PARSER = new ConstructingObjectParser<>(
         "data_stream_alias",
         false,
-        (args, name) -> new DataStreamAlias(name, (List<String>) args[0], (String) args[1], (CompressedXContent) args[2])
+        (args, name) -> {
+            /*
+             * If we are reading an older cluster state from disk we have to support reading in the single filter that was used before
+             * 8.7.0. In this case the new dataStreamsToFilters map will be null. So we write a new dataStreamsToFilters using the existing
+             * filter value for all DataStreams in order to carry forward the previously-existing behavior.
+             */
+            Map<String, CompressedXContent> dataStreamsToFilters = (Map<String, CompressedXContent>) args[3];
+            CompressedXContent oldFilter = (CompressedXContent) args[2];
+            List<String> dataStreamNames = (List<String>) args[0];
+            if (dataStreamsToFilters == null && oldFilter != null && dataStreamNames != null) {
+                logger.info(
+                    "Reading in data stream alias [{}] with a pre-8.7.0-style data stream filter and using it for all data streams in "
+                        + "the data stream alias",
+                    name
+                );
+                dataStreamsToFilters = new HashMap<>();
+                for (String dataStreamName : dataStreamNames) {
+                    dataStreamsToFilters.put(dataStreamName, oldFilter);
+                }
+            }
+            if (dataStreamsToFilters == null) {
+                dataStreamsToFilters = Map.of();
+            }
+            return new DataStreamAlias(name, dataStreamNames, dataStreamsToFilters, (String) args[1]);
+        }
     );
 
     static {
         PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), DATA_STREAMS_FIELD);
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), WRITE_DATA_STREAM_FIELD);
+        // Note: This field is not used in 8.7.0 and higher:
         PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
             if (p.currentToken() == XContentParser.Token.VALUE_EMBEDDED_OBJECT || p.currentToken() == XContentParser.Token.VALUE_STRING) {
                 return new CompressedXContent(p.binaryValue());
@@ -58,24 +94,55 @@ public class DataStreamAlias implements SimpleDiffable<DataStreamAlias>, ToXCont
                 assert false : "unexpected token [" + p.currentToken() + " ]";
                 return null;
             }
-        }, FILTER_FIELD, ObjectParser.ValueType.VALUE_OBJECT_ARRAY);
+        }, OLD_FILTER_FIELD, ObjectParser.ValueType.VALUE_OBJECT_ARRAY);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(HashMap::new, xContentParser -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_EMBEDDED_OBJECT || p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return new CompressedXContent(p.binaryValue());
+            } else if (p.currentToken() == XContentParser.Token.START_OBJECT) {
+                XContentBuilder builder = XContentFactory.jsonBuilder().map(p.mapOrdered());
+                return new CompressedXContent(BytesReference.bytes(builder));
+            } else {
+                assert false : "unexpected token [" + p.currentToken() + " ]";
+                return null;
+            }
+        }), FILTERS_FIELD);
     }
 
     private final String name;
     private final List<String> dataStreams;
     private final String writeDataStream;
-    private final CompressedXContent filter;
-
-    private DataStreamAlias(String name, List<String> dataStreams, String writeDataStream, CompressedXContent filter) {
+    // package-private for testing
+    final Map<String, CompressedXContent> dataStreamToFilterMap;
+
+    private DataStreamAlias(
+        String name,
+        List<String> dataStreams,
+        Map<String, CompressedXContent> dataStreamsToFilters,
+        String writeDataStream
+    ) {
         this.name = Objects.requireNonNull(name);
         this.dataStreams = List.copyOf(dataStreams);
         this.writeDataStream = writeDataStream;
-        this.filter = filter;
+        this.dataStreamToFilterMap = new HashMap<>(dataStreamsToFilters);
         assert writeDataStream == null || dataStreams.contains(writeDataStream);
     }
 
-    public DataStreamAlias(String name, List<String> dataStreams, String writeDataStream, Map<String, Object> filter) {
-        this(name, dataStreams, writeDataStream, compress(filter));
+    public DataStreamAlias(
+        String name,
+        List<String> dataStreams,
+        String writeDataStream,
+        Map<String, Map<String, Object>> dataStreamsToFilters
+    ) {
+        this(name, dataStreams, compressFiltersMap(dataStreamsToFilters), writeDataStream);
+    }
+
+    private static Map<String, CompressedXContent> compressFiltersMap(Map<String, Map<String, Object>> dataStreamToUncompressedFilterMap) {
+        if (dataStreamToUncompressedFilterMap == null) {
+            return Map.of();
+        }
+        return dataStreamToUncompressedFilterMap.entrySet()
+            .stream()
+            .collect(Collectors.toMap(Map.Entry::getKey, entry -> compress(entry.getValue())));
     }
 
     private static CompressedXContent compress(Map<String, Object> filterAsMap) {
@@ -100,7 +167,21 @@ public DataStreamAlias(StreamInput in) throws IOException {
         this.name = in.readString();
         this.dataStreams = in.readStringList();
         this.writeDataStream = in.readOptionalString();
-        this.filter = in.readBoolean() ? CompressedXContent.readCompressedString(in) : null;
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            this.dataStreamToFilterMap = in.readMap(StreamInput::readString, CompressedXContent::readCompressedString);
+        } else {
+            this.dataStreamToFilterMap = new HashMap<>();
+            CompressedXContent filter = in.readBoolean() ? CompressedXContent.readCompressedString(in) : null;
+            if (filter != null) {
+                /*
+                 * Here we're reading in a DataStreamAlias from before 8.7.0, which did not correctly associate filters with DataStreams.
+                 * So we associated the same filter with all DataStreams in the alias to replicate the old behavior.
+                 */
+                for (String dataStream : dataStreams) {
+                    dataStreamToFilterMap.put(dataStream, filter);
+                }
+            }
+        }
     }
 
     /**
@@ -128,12 +209,12 @@ public String getWriteDataStream() {
         return writeDataStream;
     }
 
-    public CompressedXContent getFilter() {
-        return filter;
+    public CompressedXContent getFilter(String dataStreamName) {
+        return dataStreamToFilterMap.get(dataStreamName);
     }
 
     public boolean filteringRequired() {
-        return filter != null;
+        return dataStreamToFilterMap.isEmpty() == false;
     }
 
     /**
@@ -158,23 +239,25 @@ public DataStreamAlias update(String dataStream, Boolean isWriteDataStream, Map<
         }
 
         boolean filterUpdated;
-        CompressedXContent filter;
         if (filterAsMap != null) {
-            filter = compress(filterAsMap);
-            if (this.filter == null) {
+            CompressedXContent previousFilter = dataStreamToFilterMap.get(dataStream);
+            if (previousFilter == null) {
                 filterUpdated = true;
             } else {
-                filterUpdated = filterAsMap.equals(decompress(this.filter)) == false;
+                filterUpdated = filterAsMap.equals(decompress(previousFilter)) == false;
             }
         } else {
-            filter = this.filter;
             filterUpdated = false;
         }
 
         Set<String> dataStreams = new HashSet<>(this.dataStreams);
         boolean added = dataStreams.add(dataStream);
         if (added || Objects.equals(this.writeDataStream, writeDataStream) == false || filterUpdated) {
-            return new DataStreamAlias(name, List.copyOf(dataStreams), writeDataStream, filter);
+            Map<String, CompressedXContent> newDataStreamToFilterMap = new HashMap<>(dataStreamToFilterMap);
+            if (filterAsMap != null) {
+                newDataStreamToFilterMap.put(dataStream, compress(filterAsMap));
+            }
+            return new DataStreamAlias(name, List.copyOf(dataStreams), newDataStreamToFilterMap, writeDataStream);
         } else {
             return this;
         }
@@ -199,7 +282,7 @@ public DataStreamAlias removeDataStream(String dataStream) {
             if (dataStream.equals(writeDataStream)) {
                 writeDataStream = null;
             }
-            return new DataStreamAlias(name, List.copyOf(dataStreams), writeDataStream, filter);
+            return new DataStreamAlias(name, List.copyOf(dataStreams), dataStreamToFilterMap, writeDataStream);
         }
     }
 
@@ -216,7 +299,7 @@ public DataStreamAlias intersect(Predicate<String> filter) {
         if (intersectingDataStreams.contains(writeDataStream) == false) {
             writeDataStream = null;
         }
-        return new DataStreamAlias(this.name, intersectingDataStreams, writeDataStream, this.filter);
+        return new DataStreamAlias(this.name, intersectingDataStreams, this.dataStreamToFilterMap, writeDataStream);
     }
 
     /**
@@ -273,7 +356,7 @@ public DataStreamAlias restore(DataStreamAlias previous, String renamePattern, S
             }
         }
 
-        return new DataStreamAlias(this.name, List.copyOf(mergedDataStreams), writeDataStream, filter);
+        return new DataStreamAlias(this.name, List.copyOf(mergedDataStreams), dataStreamToFilterMap, writeDataStream);
     }
 
     public static Diff<DataStreamAlias> readDiffFrom(StreamInput in) throws IOException {
@@ -296,15 +379,17 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (writeDataStream != null) {
             builder.field(WRITE_DATA_STREAM_FIELD.getPreferredName(), writeDataStream);
         }
-        if (filter != null) {
-            boolean binary = params.paramAsBoolean("binary", false);
+        boolean binary = params.paramAsBoolean("binary", false);
+        builder.startObject("filters");
+        for (Map.Entry<String, CompressedXContent> entry : dataStreamToFilterMap.entrySet()) {
             if (binary) {
-                builder.field("filter", filter.compressed());
+                builder.field(entry.getKey(), entry.getValue().compressed());
             } else {
-                builder.field("filter", XContentHelper.convertToMap(filter.uncompressed(), true).v2());
+                builder.field(entry.getKey(), XContentHelper.convertToMap(entry.getValue().uncompressed(), true).v2());
             }
         }
         builder.endObject();
+        builder.endObject();
         return builder;
     }
 
@@ -313,11 +398,19 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(name);
         out.writeStringCollection(dataStreams);
         out.writeOptionalString(writeDataStream);
-        if (filter != null) {
-            out.writeBoolean(true);
-            filter.writeTo(out);
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            out.writeMap(dataStreamToFilterMap, StreamOutput::writeString, (out1, filter) -> filter.writeTo(out1));
         } else {
-            out.writeBoolean(false);
+            if (dataStreamToFilterMap.isEmpty()) {
+                out.writeBoolean(false);
+            } else {
+                /*
+                 * TransportVersions before 8.7 incorrectly only allowed a single filter for all datastreams,
+                 * and randomly dropped all others. We replicate that buggy behavior here if we have to write
+                 * to an older node because there is no way to send multipole filters to an older node.
+                 */
+                dataStreamToFilterMap.values().iterator().next().writeTo(out);
+            }
         }
     }
 
@@ -329,12 +422,12 @@ public boolean equals(Object o) {
         return Objects.equals(name, that.name)
             && Objects.equals(dataStreams, that.dataStreams)
             && Objects.equals(writeDataStream, that.writeDataStream)
-            && Objects.equals(filter, that.filter);
+            && Objects.equals(dataStreamToFilterMap, that.dataStreamToFilterMap);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(name, dataStreams, writeDataStream, filter);
+        return Objects.hash(name, dataStreams, writeDataStream, dataStreamToFilterMap);
     }
 
     @Override
@@ -348,8 +441,11 @@ public String toString() {
             + ", writeDataStream='"
             + writeDataStream
             + '\''
-            + ", filter="
-            + (filter != null ? filter.string() : "null")
+            + ", dataStreamToFilterMap="
+            + dataStreamToFilterMap.keySet()
+                .stream()
+                .map(key -> key + "=" + dataStreamToFilterMap.get(key))
+                .collect(Collectors.joining(", ", "{", "}"))
             + '}';
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java
index ff05aab210f6..17d2d2d1109c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java
@@ -9,18 +9,20 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.DiffableUtils;
 import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentParser;
 
@@ -28,6 +30,7 @@
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -114,7 +117,12 @@ public DataStreamMetadata withAlias(String aliasName, String dataStream, Boolean
         DataStreamAlias alias = dataStreamAliases.get(aliasName);
         if (alias == null) {
             String writeDataStream = isWriteDataStream != null && isWriteDataStream ? dataStream : null;
-            alias = new DataStreamAlias(aliasName, List.of(dataStream), writeDataStream, filterAsMap);
+            alias = new DataStreamAlias(
+                aliasName,
+                List.of(dataStream),
+                writeDataStream,
+                filterAsMap == null ? null : Map.of(dataStream, filterAsMap)
+            );
         } else {
             DataStreamAlias copy = alias.update(dataStream, isWriteDataStream, filterAsMap);
             if (copy == alias) {
@@ -207,8 +215,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_7_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_7_0;
     }
 
     @Override
@@ -222,14 +230,13 @@ public static DataStreamMetadata fromXContent(XContentParser parser) throws IOEx
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.xContentValuesMap(DATA_STREAM.getPreferredName(), dataStreams);
-        builder.startObject(DATA_STREAM_ALIASES.getPreferredName());
-        for (Map.Entry<String, DataStreamAlias> dataStream : dataStreamAliases.entrySet()) {
-            dataStream.getValue().toXContent(builder, params);
-        }
-        builder.endObject();
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return Iterators.concat(
+            ChunkedToXContentHelper.xContentValuesMap(DATA_STREAM.getPreferredName(), dataStreams),
+            ChunkedToXContentHelper.startObject(DATA_STREAM_ALIASES.getPreferredName()),
+            dataStreamAliases.values().iterator(),
+            ChunkedToXContentHelper.endObject()
+        );
     }
 
     @Override
@@ -305,8 +312,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.V_7_7_0;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.V_7_7_0;
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java
index 8437ad0efcf3..2c29df2e661c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
@@ -38,7 +39,7 @@
 import static org.elasticsearch.node.NodeRoleSettings.NODE_ROLES_SETTING;
 
 public final class DesiredNode implements Writeable, ToXContentObject, Comparable<DesiredNode> {
-    public static final Version RANGE_FLOAT_PROCESSORS_SUPPORT_VERSION = Version.V_8_3_0;
+    public static final TransportVersion RANGE_FLOAT_PROCESSORS_SUPPORT_VERSION = TransportVersion.V_8_3_0;
 
     private static final ParseField SETTINGS_FIELD = new ParseField("settings");
     private static final ParseField PROCESSORS_FIELD = new ParseField("processors");
@@ -173,7 +174,7 @@ public static DesiredNode readFrom(StreamInput in) throws IOException {
         final var settings = Settings.readSettingsFromStream(in);
         final Processors processors;
         final ProcessorsRange processorsRange;
-        if (in.getVersion().onOrAfter(RANGE_FLOAT_PROCESSORS_SUPPORT_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(RANGE_FLOAT_PROCESSORS_SUPPORT_VERSION)) {
             processors = in.readOptionalWriteable(Processors::readFrom);
             processorsRange = in.readOptionalWriteable(ProcessorsRange::readFrom);
         } else {
@@ -189,7 +190,7 @@ public static DesiredNode readFrom(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         settings.writeTo(out);
-        if (out.getVersion().onOrAfter(RANGE_FLOAT_PROCESSORS_SUPPORT_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(RANGE_FLOAT_PROCESSORS_SUPPORT_VERSION)) {
             out.writeOptionalWriteable(processors);
             out.writeOptionalWriteable(processorsRange);
         } else {
@@ -296,7 +297,7 @@ public Set<DiscoveryNodeRole> getRoles() {
         return roles;
     }
 
-    public boolean isCompatibleWithVersion(Version version) {
+    public boolean isCompatibleWithVersion(TransportVersion version) {
         if (version.onOrAfter(RANGE_FLOAT_PROCESSORS_SUPPORT_VERSION)) {
             return true;
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodeWithStatus.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodeWithStatus.java
index 13a10213b8ed..036b31fa65be 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodeWithStatus.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodeWithStatus.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -30,7 +31,7 @@ public record DesiredNodeWithStatus(DesiredNode desiredNode, Status status)
         ToXContentObject,
         Comparable<DesiredNodeWithStatus> {
 
-    private static final Version STATUS_TRACKING_SUPPORT_VERSION = Version.V_8_4_0;
+    private static final TransportVersion STATUS_TRACKING_SUPPORT_VERSION = TransportVersion.V_8_4_0;
     private static final ParseField STATUS_FIELD = new ParseField("status");
 
     public static final ConstructingObjectParser<DesiredNodeWithStatus, Void> PARSER = new ConstructingObjectParser<>(
@@ -47,7 +48,7 @@ public record DesiredNodeWithStatus(DesiredNode desiredNode, Status status)
             ),
             // An unknown status is expected during upgrades to versions >= STATUS_TRACKING_SUPPORT_VERSION
             // the desired node status would be populated when a node in the newer version is elected as
-            // master, the desired nodes status update happens in JoinTaskExecutor.
+            // master, the desired nodes status update happens in NodeJoinExecutor.
             args[6] == null ? Status.PENDING : (Status) args[6]
         )
     );
@@ -77,14 +78,14 @@ public String externalId() {
     public static DesiredNodeWithStatus readFrom(StreamInput in) throws IOException {
         final var desiredNode = DesiredNode.readFrom(in);
         final Status status;
-        if (in.getVersion().onOrAfter(STATUS_TRACKING_SUPPORT_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(STATUS_TRACKING_SUPPORT_VERSION)) {
             status = Status.fromValue(in.readShort());
         } else {
             // During upgrades, we consider all desired nodes as PENDING
             // since it's impossible to know if a node that was supposed to
             // join the cluster, it joined. The status will be updated
             // once the master node is upgraded to a version >= STATUS_TRACKING_SUPPORT_VERSION
-            // in JoinTaskExecutor or when the desired nodes are upgraded to a new version.
+            // in NodeJoinExecutor or when the desired nodes are upgraded to a new version.
             status = Status.PENDING;
         }
         return new DesiredNodeWithStatus(desiredNode, status);
@@ -93,7 +94,7 @@ public static DesiredNodeWithStatus readFrom(StreamInput in) throws IOException
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         desiredNode.writeTo(out);
-        if (out.getVersion().onOrAfter(STATUS_TRACKING_SUPPORT_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(STATUS_TRACKING_SUPPORT_VERSION)) {
             out.writeShort(status.value);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodes.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodes.java
index b720c90a0673..cadc57c0c2a2 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodes.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodes.java
@@ -8,7 +8,9 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.action.admin.cluster.desirednodes.TransportUpdateDesiredNodesAction;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.coordination.NodeJoinExecutor;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -97,8 +99,7 @@
  *  </ul>
  *
  * <p>
- *  See {@code JoinTaskExecutor} and {@code TransportUpdateDesiredNodesAction} for more details about
- *  desired nodes status tracking.
+ *  See {@link NodeJoinExecutor} and {@link TransportUpdateDesiredNodesAction} for more details about desired nodes status tracking.
  * </p>
  *
  * <p>
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodesMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodesMetadata.java
index 64a384f77e12..f307d060f449 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodesMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodesMetadata.java
@@ -8,24 +8,26 @@
 
 package org.elasticsearch.cluster.metadata;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.AbstractNamedDiffable;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.NamedDiff;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.EnumSet;
+import java.util.Iterator;
 import java.util.Objects;
 
 public class DesiredNodesMetadata extends AbstractNamedDiffable<Metadata.Custom> implements Metadata.Custom {
-    private static final Version MIN_SUPPORTED_VERSION = Version.V_8_1_0;
+    private static final TransportVersion MIN_SUPPORTED_VERSION = TransportVersion.V_8_1_0;
     public static final String TYPE = "desired_nodes";
 
     public static final DesiredNodesMetadata EMPTY = new DesiredNodesMetadata((DesiredNodes) null);
@@ -67,9 +69,8 @@ public static DesiredNodesMetadata fromXContent(XContentParser parser) throws IO
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(LATEST_FIELD.getPreferredName(), latestDesiredNodes, params);
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return Iterators.single((builder, params) -> builder.field(LATEST_FIELD.getPreferredName(), latestDesiredNodes, params));
     }
 
     public static DesiredNodesMetadata fromClusterState(ClusterState clusterState) {
@@ -92,7 +93,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
+    public TransportVersion getMinimalSupportedVersion() {
         return MIN_SUPPORTED_VERSION;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java
index 76f9643b0fb4..e9405f061e99 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java
@@ -14,11 +14,11 @@
 import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel;
 
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
 
 public class IndexAbstractionResolver {
 
@@ -28,30 +28,12 @@ public IndexAbstractionResolver(IndexNameExpressionResolver indexNameExpressionR
         this.indexNameExpressionResolver = indexNameExpressionResolver;
     }
 
-    public List<String> resolveIndexAbstractions(
-        String[] indices,
-        IndicesOptions indicesOptions,
-        Metadata metadata,
-        boolean includeDataStreams
-    ) {
-        return resolveIndexAbstractions(Arrays.asList(indices), indicesOptions, metadata, includeDataStreams);
-    }
-
-    public List<String> resolveIndexAbstractions(
-        Iterable<String> indices,
-        IndicesOptions indicesOptions,
-        Metadata metadata,
-        boolean includeDataStreams
-    ) {
-        Set<String> availableIndexAbstractions = metadata.getIndicesLookup().keySet();
-        return resolveIndexAbstractions(indices, indicesOptions, metadata, availableIndexAbstractions, includeDataStreams);
-    }
-
     public List<String> resolveIndexAbstractions(
         Iterable<String> indices,
         IndicesOptions indicesOptions,
         Metadata metadata,
-        Collection<String> availableIndexAbstractions,
+        Supplier<Set<String>> allAuthorizedAndAvailable,
+        Predicate<String> isAuthorized,
         boolean includeDataStreams
     ) {
         List<String> finalIndices = new ArrayList<>();
@@ -72,7 +54,7 @@ public List<String> resolveIndexAbstractions(
             if (indicesOptions.expandWildcardExpressions() && Regex.isSimpleMatchPattern(indexAbstraction)) {
                 wildcardSeen = true;
                 Set<String> resolvedIndices = new HashSet<>();
-                for (String authorizedIndex : availableIndexAbstractions) {
+                for (String authorizedIndex : allAuthorizedAndAvailable.get()) {
                     if (Regex.simpleMatch(indexAbstraction, authorizedIndex)
                         && isIndexVisible(
                             indexAbstraction,
@@ -100,7 +82,10 @@ && isIndexVisible(
             } else {
                 if (minus) {
                     finalIndices.remove(indexAbstraction);
-                } else if (indicesOptions.ignoreUnavailable() == false || availableIndexAbstractions.contains(indexAbstraction)) {
+                } else if (indicesOptions.ignoreUnavailable() == false || isAuthorized.test(indexAbstraction)) {
+                    // Unauthorized names are considered unavailable, so if `ignoreUnavailable` is `true` they should be silently
+                    // discarded from the `finalIndices` list. Other "ways of unavailable" must be handled by the action
+                    // handler, see: https://github.com/elastic/elasticsearch/issues/90215
                     finalIndices.add(indexAbstraction);
                 }
             }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
index 7b600d875c79..ec8f659913e8 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.cluster.metadata;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -17,10 +17,12 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.xcontent.ContextParser;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
@@ -32,6 +34,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.EnumSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
 
@@ -84,8 +87,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.CURRENT.minimumCompatibilityVersion();
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.CURRENT.minimumCompatibilityVersion();
     }
 
     @Override
@@ -123,12 +126,8 @@ public boolean containsIndex(final Index index) {
     }
 
     @Override
-    public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
-        builder.startArray(TOMBSTONES_FIELD.getPreferredName());
-        for (Tombstone tombstone : tombstones) {
-            tombstone.toXContent(builder, params);
-        }
-        return builder.endArray();
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return ChunkedToXContentHelper.array(TOMBSTONES_FIELD.getPreferredName(), tombstones.iterator());
     }
 
     public static IndexGraveyard fromXContent(final XContentParser parser) throws IOException {
@@ -334,8 +333,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT.minimumCompatibilityVersion();
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT.minimumCompatibilityVersion();
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
index 77ea513099e4..436470b3ebda 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
@@ -8,6 +8,9 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.rollover.RolloverInfo;
 import org.elasticsearch.action.support.ActiveShardCount;
@@ -83,6 +86,8 @@
 
 public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragment {
 
+    private static final Logger logger = LogManager.getLogger(IndexMetadata.class);
+
     public static final ClusterBlock INDEX_READ_ONLY_BLOCK = new ClusterBlock(
         5,
         "index read-only (api)",
@@ -530,9 +535,9 @@ public Iterator<Setting<?>> settings() {
 
     public static final String INDEX_STATE_FILE_PREFIX = "state-";
 
-    static final Version SYSTEM_INDEX_FLAG_ADDED = Version.V_7_10_0;
+    static final TransportVersion SYSTEM_INDEX_FLAG_ADDED = TransportVersion.V_7_10_0;
 
-    static final Version STATS_AND_FORECAST_ADDED = Version.V_8_6_0;
+    static final TransportVersion STATS_AND_FORECAST_ADDED = TransportVersion.V_8_6_0;
 
     private final int routingNumShards;
     private final int routingFactor;
@@ -1402,7 +1407,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    private static final Version SETTING_DIFF_VERSION = Version.V_8_5_0;
+    private static final TransportVersion SETTING_DIFF_VERSION = TransportVersion.V_8_5_0;
 
     private static class IndexMetadataDiff implements Diff<IndexMetadata> {
 
@@ -1483,13 +1488,13 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
             version = in.readLong();
             mappingVersion = in.readVLong();
             settingsVersion = in.readVLong();
-            if (in.getVersion().onOrAfter(Version.V_7_2_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_2_0)) {
                 aliasesVersion = in.readVLong();
             } else {
                 aliasesVersion = 1;
             }
             state = State.fromId(in.readByte());
-            if (in.getVersion().onOrAfter(SETTING_DIFF_VERSION)) {
+            if (in.getTransportVersion().onOrAfter(SETTING_DIFF_VERSION)) {
                 settings = null;
                 settingsDiff = Settings.readSettingsDiffFromStream(in);
             } else {
@@ -1510,13 +1515,13 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
                 DiffableUtils.getStringKeySerializer(),
                 ROLLOVER_INFO_DIFF_VALUE_READER
             );
-            if (in.getVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) {
+            if (in.getTransportVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) {
                 isSystem = in.readBoolean();
             } else {
                 isSystem = false;
             }
             timestampRange = IndexLongFieldRange.readFrom(in);
-            if (in.getVersion().onOrAfter(STATS_AND_FORECAST_ADDED)) {
+            if (in.getTransportVersion().onOrAfter(STATS_AND_FORECAST_ADDED)) {
                 stats = in.readOptionalWriteable(IndexMetadataStats::new);
                 indexWriteLoadForecast = in.readOptionalDouble();
                 shardSizeInBytesForecast = in.readOptionalLong();
@@ -1534,13 +1539,13 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeLong(version);
             out.writeVLong(mappingVersion);
             out.writeVLong(settingsVersion);
-            if (out.getVersion().onOrAfter(Version.V_7_2_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_2_0)) {
                 out.writeVLong(aliasesVersion);
             }
             out.writeByte(state.id);
             assert settings != null
                 : "settings should always be non-null since this instance is not expected to have been read from another node";
-            if (out.getVersion().onOrAfter(SETTING_DIFF_VERSION)) {
+            if (out.getTransportVersion().onOrAfter(SETTING_DIFF_VERSION)) {
                 settingsDiff.writeTo(out);
             } else {
                 settings.writeTo(out);
@@ -1551,11 +1556,11 @@ public void writeTo(StreamOutput out) throws IOException {
             customData.writeTo(out);
             inSyncAllocationIds.writeTo(out);
             rolloverInfos.writeTo(out);
-            if (out.getVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) {
+            if (out.getTransportVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) {
                 out.writeBoolean(isSystem);
             }
             timestampRange.writeTo(out);
-            if (out.getVersion().onOrAfter(STATS_AND_FORECAST_ADDED)) {
+            if (out.getTransportVersion().onOrAfter(STATS_AND_FORECAST_ADDED)) {
                 out.writeOptionalWriteable(stats);
                 out.writeOptionalDouble(indexWriteLoadForecast);
                 out.writeOptionalLong(shardSizeInBytesForecast);
@@ -1589,7 +1594,7 @@ public IndexMetadata apply(IndexMetadata part) {
             builder.stats(stats);
             builder.indexWriteLoadForecast(indexWriteLoadForecast);
             builder.shardSizeInBytesForecast(shardSizeInBytesForecast);
-            return builder.build();
+            return builder.build(true);
         }
     }
 
@@ -1607,7 +1612,7 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function<String,
         builder.version(in.readLong());
         builder.mappingVersion(in.readVLong());
         builder.settingsVersion(in.readVLong());
-        if (in.getVersion().onOrAfter(Version.V_7_2_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_2_0)) {
             builder.aliasesVersion(in.readVLong());
         }
         builder.setRoutingNumShards(in.readInt());
@@ -1616,7 +1621,7 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function<String,
         builder.primaryTerms(in.readVLongArray());
         int mappingsSize = in.readVInt();
         if (mappingsSize == 1) {
-            if (mappingLookup != null && in.getVersion().onOrAfter(Metadata.MAPPINGS_AS_HASH_VERSION)) {
+            if (mappingLookup != null && in.getTransportVersion().onOrAfter(Metadata.MAPPINGS_AS_HASH_VERSION)) {
                 final String mappingHash = in.readString();
                 final MappingMetadata metadata = mappingLookup.apply(mappingHash);
                 assert metadata != null : "failed to find mapping [" + mappingHash + "] for [" + builder.index + "]";
@@ -1646,17 +1651,17 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function<String,
         for (int i = 0; i < rolloverAliasesSize; i++) {
             builder.putRolloverInfo(new RolloverInfo(in));
         }
-        if (in.getVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) {
+        if (in.getTransportVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) {
             builder.system(in.readBoolean());
         }
         builder.timestampRange(IndexLongFieldRange.readFrom(in));
 
-        if (in.getVersion().onOrAfter(STATS_AND_FORECAST_ADDED)) {
+        if (in.getTransportVersion().onOrAfter(STATS_AND_FORECAST_ADDED)) {
             builder.stats(in.readOptionalWriteable(IndexMetadataStats::new));
             builder.indexWriteLoadForecast(in.readOptionalDouble());
             builder.shardSizeInBytesForecast(in.readOptionalLong());
         }
-        return builder.build();
+        return builder.build(true);
     }
 
     /**
@@ -1667,7 +1672,7 @@ public void writeTo(StreamOutput out, boolean mappingsAsHash) throws IOException
         out.writeLong(version);
         out.writeVLong(mappingVersion);
         out.writeVLong(settingsVersion);
-        if (out.getVersion().onOrAfter(Version.V_7_2_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_2_0)) {
             out.writeVLong(aliasesVersion);
         }
         out.writeInt(routingNumShards);
@@ -1679,7 +1684,7 @@ public void writeTo(StreamOutput out, boolean mappingsAsHash) throws IOException
             out.writeVInt(0);
         } else {
             out.writeVInt(1);
-            if (mappingsAsHash && out.getVersion().onOrAfter(Metadata.MAPPINGS_AS_HASH_VERSION)) {
+            if (mappingsAsHash && out.getTransportVersion().onOrAfter(Metadata.MAPPINGS_AS_HASH_VERSION)) {
                 out.writeString(mapping.getSha256());
             } else {
                 mapping.writeTo(out);
@@ -1693,11 +1698,11 @@ public void writeTo(StreamOutput out, boolean mappingsAsHash) throws IOException
             (o, v) -> DiffableUtils.StringSetValueSerializer.getInstance().write(v, o)
         );
         out.writeCollection(rolloverInfos.values());
-        if (out.getVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) {
+        if (out.getTransportVersion().onOrAfter(SYSTEM_INDEX_FLAG_ADDED)) {
             out.writeBoolean(isSystem);
         }
         timestampRange.writeTo(out);
-        if (out.getVersion().onOrAfter(STATS_AND_FORECAST_ADDED)) {
+        if (out.getTransportVersion().onOrAfter(STATS_AND_FORECAST_ADDED)) {
             out.writeOptionalWriteable(stats);
             out.writeOptionalDouble(writeLoadForecast);
             out.writeOptionalLong(shardSizeInBytesForecast);
@@ -2012,6 +2017,11 @@ public Builder shardSizeInBytesForecast(Long shardSizeInBytesForecast) {
         }
 
         public IndexMetadata build() {
+            return build(false);
+        }
+
+        // package private for testing
+        IndexMetadata build(boolean repair) {
             /*
              * We expect that the metadata has been properly built to set the number of shards and the number of replicas, and do not rely
              * on the default values here. Those must have been set upstream.
@@ -2140,7 +2150,16 @@ public IndexMetadata build() {
             var aliasesMap = aliases.build();
             for (AliasMetadata alias : aliasesMap.values()) {
                 if (alias.alias().equals(index)) {
-                    throw new IllegalArgumentException("alias name [" + index + "] self-conflicts with index name");
+                    if (repair && indexCreatedVersion.equals(Version.V_8_5_0)) {
+                        var updatedBuilder = ImmutableOpenMap.builder(aliasesMap);
+                        final var brokenAlias = updatedBuilder.remove(index);
+                        final var fixedAlias = AliasMetadata.newAliasMetadata(brokenAlias, index + "-alias-corrupted-by-8-5");
+                        aliasesMap = updatedBuilder.fPut(fixedAlias.getAlias(), fixedAlias).build();
+                        logger.warn("Repaired corrupted alias with the same name as its index for [{}]", index);
+                        break;
+                    } else {
+                        throw new IllegalArgumentException("alias name [" + index + "] self-conflicts with index name");
+                    }
                 }
             }
 
@@ -2463,7 +2482,7 @@ public static IndexMetadata fromXContent(XContentParser parser, Map<String, Mapp
             assert settingsVersion : "settings version should be present for indices created on or after 6.5.0";
             assert indexCreatedVersion(builder.settings).before(Version.V_7_2_0) || aliasesVersion
                 : "aliases version should be present for indices created on or after 7.2.0";
-            return builder.build();
+            return builder.build(true);
         }
 
         /**
@@ -2574,7 +2593,7 @@ public static IndexMetadata legacyFromXContent(XContentParser parser) throws IOE
                 builder.putMapping(MappingMetadata.EMPTY_MAPPINGS); // just make sure it's not empty so that _source can be read
             }
 
-            IndexMetadata indexMetadata = builder.build();
+            IndexMetadata indexMetadata = builder.build(true);
             assert indexMetadata.getCreationVersion().isLegacyIndexVersion();
             assert indexMetadata.getCompatibilityVersion().isLegacyIndexVersion();
             return indexMetadata;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
index 440e2c2c7aad..6ffb54a57d8b 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
@@ -43,6 +43,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -185,11 +186,7 @@ public List<String> dataStreamNames(ClusterState state, IndicesOptions options,
             getSystemIndexAccessPredicate(),
             getNetNewSystemIndexPredicate()
         );
-        if (indexExpressions == null || indexExpressions.length == 0) {
-            indexExpressions = new String[] { "*" };
-        }
-
-        final Collection<String> expressions = resolveExpressions(Arrays.asList(indexExpressions), context);
+        final Collection<String> expressions = resolveExpressions(context, indexExpressions);
         return expressions.stream()
             .map(x -> state.metadata().getIndicesLookup().get(x))
             .filter(Objects::nonNull)
@@ -219,13 +216,10 @@ public IndexAbstraction resolveWriteIndexAbstraction(ClusterState state, DocWrit
             getNetNewSystemIndexPredicate()
         );
 
-        final Collection<String> expressions = resolveExpressions(List.of(request.index()), context);
+        final Collection<String> expressions = resolveExpressions(context, request.index());
 
         if (expressions.size() == 1) {
             IndexAbstraction ia = state.metadata().getIndicesLookup().get(expressions.iterator().next());
-            if (ia == null) {
-                throw new IndexNotFoundException(expressions.iterator().next());
-            }
             if (ia.getType() == Type.ALIAS) {
                 Index writeIndex = ia.getWriteIndex();
                 if (writeIndex == null) {
@@ -247,8 +241,28 @@ public IndexAbstraction resolveWriteIndexAbstraction(ClusterState state, DocWrit
         }
     }
 
-    private static Collection<String> resolveExpressions(List<String> expressions, Context context) {
-        return WildcardExpressionResolver.resolve(context, DateMathExpressionResolver.resolve(context, expressions));
+    protected static Collection<String> resolveExpressions(Context context, String... expressions) {
+        if (context.getOptions().expandWildcardExpressions() == false) {
+            if (expressions == null || expressions.length == 0 || expressions.length == 1 && Metadata.ALL.equals(expressions[0])) {
+                return List.of();
+            } else {
+                return ExplicitResourceNameFilter.filterUnavailable(
+                    context,
+                    DateMathExpressionResolver.resolve(context, List.of(expressions))
+                );
+            }
+        } else {
+            if (expressions == null
+                || expressions.length == 0
+                || expressions.length == 1 && (Metadata.ALL.equals(expressions[0]) || Regex.isMatchAllPattern(expressions[0]))) {
+                return WildcardExpressionResolver.resolveAll(context);
+            } else {
+                return WildcardExpressionResolver.resolve(
+                    context,
+                    ExplicitResourceNameFilter.filterUnavailable(context, DateMathExpressionResolver.resolve(context, List.of(expressions)))
+                );
+            }
+        }
     }
 
     /**
@@ -320,47 +334,13 @@ String[] concreteIndexNames(Context context, String... indexExpressions) {
     }
 
     Index[] concreteIndices(Context context, String... indexExpressions) {
-        IndicesOptions options = context.getOptions();
-        if (indexExpressions == null || indexExpressions.length == 0) {
-            indexExpressions = new String[] { Metadata.ALL };
-        } else {
-            if (options.ignoreUnavailable() == false) {
-                List<String> crossClusterIndices = Arrays.stream(indexExpressions).filter(index -> index.contains(":")).toList();
-                if (crossClusterIndices.size() > 0) {
-                    throw new IllegalArgumentException(
-                        "Cross-cluster calls are not supported in this context but remote indices "
-                            + "were requested: "
-                            + crossClusterIndices
-                    );
-                }
-            }
-        }
-
-        final Collection<String> expressions = resolveExpressions(Arrays.asList(indexExpressions), context);
+        final Collection<String> expressions = resolveExpressions(context, indexExpressions);
 
-        if (expressions.isEmpty() || (expressions.size() == 1 && expressions.iterator().next().equals(Metadata.ALL))) {
-            if (options.allowNoIndices() == false) {
-                throw notFoundException(indexExpressions);
-            } else {
-                return Index.EMPTY_ARRAY;
-            }
-        }
-
-        final Set<Index> concreteIndices = Sets.newLinkedHashSetWithExpectedSize(expressions.size());
-        final SortedMap<String, IndexAbstraction> indicesLookup = context.state.metadata().getIndicesLookup();
+        final Set<Index> concreteIndicesResult = Sets.newLinkedHashSetWithExpectedSize(expressions.size());
+        final Map<String, IndexAbstraction> indicesLookup = context.getState().metadata().getIndicesLookup();
         for (String expression : expressions) {
-            if (options.ignoreUnavailable() == false) {
-                ensureAliasOrIndexExists(context, expression);
-            }
-            IndexAbstraction indexAbstraction = indicesLookup.get(expression);
-            if (indexAbstraction == null) {
-                continue;
-            } else if (indexAbstraction.getType() == Type.ALIAS && context.getOptions().ignoreAliases()) {
-                continue;
-            } else if (indexAbstraction.isDataStreamRelated() && context.includeDataStreams() == false) {
-                continue;
-            }
-
+            final IndexAbstraction indexAbstraction = indicesLookup.get(expression);
+            assert indexAbstraction != null;
             if (indexAbstraction.getType() == Type.ALIAS && context.isResolveToWriteIndex()) {
                 Index writeIndex = indexAbstraction.getWriteIndex();
                 if (writeIndex == null) {
@@ -373,15 +353,15 @@ Index[] concreteIndices(Context context, String... indexExpressions) {
                     );
                 }
                 if (addIndex(writeIndex, null, context)) {
-                    concreteIndices.add(writeIndex);
+                    concreteIndicesResult.add(writeIndex);
                 }
             } else if (indexAbstraction.getType() == Type.DATA_STREAM && context.isResolveToWriteIndex()) {
                 Index writeIndex = indexAbstraction.getWriteIndex();
                 if (addIndex(writeIndex, null, context)) {
-                    concreteIndices.add(writeIndex);
+                    concreteIndicesResult.add(writeIndex);
                 }
             } else {
-                if (indexAbstraction.getIndices().size() > 1 && options.allowAliasesToMultipleIndices() == false) {
+                if (indexAbstraction.getIndices().size() > 1 && context.getOptions().allowAliasesToMultipleIndices() == false) {
                     String[] indexNames = new String[indexAbstraction.getIndices().size()];
                     int i = 0;
                     for (Index indexName : indexAbstraction.getIndices()) {
@@ -398,18 +378,18 @@ Index[] concreteIndices(Context context, String... indexExpressions) {
                 }
 
                 for (Index index : indexAbstraction.getIndices()) {
-                    if (shouldTrackConcreteIndex(context, options, index)) {
-                        concreteIndices.add(index);
+                    if (shouldTrackConcreteIndex(context, context.getOptions(), index)) {
+                        concreteIndicesResult.add(index);
                     }
                 }
             }
         }
 
-        if (options.allowNoIndices() == false && concreteIndices.isEmpty()) {
+        if (context.getOptions().allowNoIndices() == false && concreteIndicesResult.isEmpty()) {
             throw notFoundException(indexExpressions);
         }
-        checkSystemIndexAccess(context, concreteIndices);
-        return concreteIndices.toArray(Index.EMPTY_ARRAY);
+        checkSystemIndexAccess(context, concreteIndicesResult);
+        return concreteIndicesResult.toArray(Index.EMPTY_ARRAY);
     }
 
     private void checkSystemIndexAccess(Context context, Set<Index> concreteIndices) {
@@ -459,13 +439,14 @@ private void checkSystemIndexAccess(Context context, Set<Index> concreteIndices)
     }
 
     private static IndexNotFoundException notFoundException(String... indexExpressions) {
-        IndexNotFoundException infe;
-        if (indexExpressions != null && indexExpressions.length == 1) {
-            if (Metadata.ALL.equals(indexExpressions[0])) {
-                infe = new IndexNotFoundException("no indices exist", indexExpressions[0]);
-            } else {
-                infe = new IndexNotFoundException(indexExpressions[0]);
-            }
+        final IndexNotFoundException infe;
+        if (indexExpressions == null
+            || indexExpressions.length == 0
+            || (indexExpressions.length == 1 && Metadata.ALL.equals(indexExpressions[0]))) {
+            infe = new IndexNotFoundException("no indices exist", Metadata.ALL);
+            infe.setResources("index_or_alias", Metadata.ALL);
+        } else if (indexExpressions.length == 1) {
+            infe = new IndexNotFoundException(indexExpressions[0]);
             infe.setResources("index_or_alias", indexExpressions[0]);
         } else {
             infe = new IndexNotFoundException((String) null);
@@ -474,24 +455,6 @@ private static IndexNotFoundException notFoundException(String... indexExpressio
         return infe;
     }
 
-    @Nullable
-    private static void ensureAliasOrIndexExists(Context context, String expression) {
-        IndexAbstraction indexAbstraction = context.getState().getMetadata().getIndicesLookup().get(expression);
-        if (indexAbstraction == null) {
-            throw notFoundException(expression);
-        }
-        // treat aliases as unavailable indices when ignoreAliases is set to true (e.g. delete index and update aliases api)
-        if (indexAbstraction.getType() == Type.ALIAS && context.getOptions().ignoreAliases()) {
-            throw aliasesNotSupportedException(expression);
-        }
-        if (indexAbstraction.isDataStreamRelated() && context.includeDataStreams() == false) {
-            IndexNotFoundException infe = notFoundException(expression);
-            // Allows callers to handle IndexNotFoundException differently based on whether data streams were excluded.
-            infe.addMetadata(EXCLUDED_DATA_STREAMS_KEY, "true");
-            throw infe;
-        }
-    }
-
     private static boolean shouldTrackConcreteIndex(Context context, IndicesOptions options, Index index) {
         if (context.systemIndexAccessLevel == SystemIndexAccessLevel.BACKWARDS_COMPATIBLE_ONLY
             && context.netNewSystemIndexPredicate.test(index.getName())) {
@@ -652,17 +615,32 @@ public static String resolveDateMathExpression(String dateExpression, long time)
      * Resolve an array of expressions to the set of indices and aliases that these expressions match.
      */
     public Set<String> resolveExpressions(ClusterState state, String... expressions) {
+        return resolveExpressions(state, IndicesOptions.lenientExpandOpen(), false, expressions);
+    }
+
+    /**
+     * Resolve the expression to the set of indices, aliases, and, optionally, datastreams that the expression matches.
+     * If {@param preserveDataStreams} is {@code true}, datastreams that are covered by the wildcards from the
+     * {@param expressions} are returned as-is, without expanding them further to their respective backing indices.
+     */
+    public Set<String> resolveExpressions(
+        ClusterState state,
+        IndicesOptions indicesOptions,
+        boolean preserveDataStreams,
+        String... expressions
+    ) {
         Context context = new Context(
             state,
-            IndicesOptions.lenientExpandOpen(),
+            indicesOptions,
             true,
             false,
             true,
+            preserveDataStreams,
             getSystemIndexAccessLevel(),
             getSystemIndexAccessPredicate(),
             getNetNewSystemIndexPredicate()
         );
-        Collection<String> resolved = resolveExpressions(Arrays.asList(expressions), context);
+        Collection<String> resolved = resolveExpressions(context, expressions);
         if (resolved instanceof Set<String>) {
             // unmodifiable without creating a new collection as it might contain many items
             return Collections.unmodifiableSet((Set<String>) resolved);
@@ -789,10 +767,7 @@ public Map<String, Set<String>> resolveSearchRouting(ClusterState state, @Nullab
             getSystemIndexAccessPredicate(),
             getNetNewSystemIndexPredicate()
         );
-        final Collection<String> resolvedExpressions = resolveExpressions(
-            expressions != null ? Arrays.asList(expressions) : Collections.emptyList(),
-            context
-        );
+        final Collection<String> resolvedExpressions = resolveExpressions(context, expressions);
 
         // TODO: it appears that this can never be true?
         if (isAllIndices(resolvedExpressions)) {
@@ -1128,25 +1103,11 @@ private WildcardExpressionResolver() {
             // Utility class
         }
 
-        /**
-         * Returns a collection of resource names given the {@param expressions} which contains wildcards and exclusions.
-         */
-        public static Collection<String> resolve(Context context, List<String> expressions) {
-            Objects.requireNonNull(expressions);
-            if (context.getOptions().expandWildcardExpressions() == false) {
-                return expressions;
-            } else if (isEmptyOrTrivialWildcard(expressions)) {
-                return innerResolveAll(context);
-            } else {
-                return innerResolve(context, expressions);
-            }
-        }
-
         /**
          * Returns all the indices and all the datastreams, considering the open/closed, system, and hidden context parameters.
          * Depending on the context, returns the names of the datastreams themselves or their backing indices.
          */
-        private static Collection<String> innerResolveAll(Context context) {
+        public static Collection<String> resolveAll(Context context) {
             List<String> resolvedExpressions = resolveEmptyOrTrivialWildcard(context);
             if (context.includeDataStreams() == false) {
                 return resolvedExpressions;
@@ -1173,8 +1134,7 @@ private static Collection<String> innerResolveAll(Context context) {
 
         /**
          * Returns all the existing resource (index, alias and datastream) names that the {@param expressions} list resolves to.
-         * The passed-in {@param expressions} can contain wildcards and exclusions, as well as plain resource names,
-         * but it mustn't be empty.
+         * The passed-in {@param expressions} can contain wildcards and exclusions, as well as plain resource names.
          * <br>
          * The return is a {@code Collection} (usually a {@code Set} but can also be a {@code List}, for performance reasons) of plain
          * resource names only. All the returned resources are "accessible", in the given context, i.e. the resources exist
@@ -1187,80 +1147,39 @@ private static Collection<String> innerResolveAll(Context context) {
          * ultimately returned, instead of the alias or datastream name</li>
          * </ol>
          */
-        private static Collection<String> innerResolve(Context context, List<String> expressions) {
-            if (Objects.requireNonNull(expressions).isEmpty()) {
-                throw new IllegalStateException("Cannot resolve empty index expression");
+        public static Collection<String> resolve(Context context, List<String> expressions) {
+            ExpressionList expressionList = new ExpressionList(context, expressions);
+            // fast exit if there are no wildcards to evaluate
+            if (expressionList.hasWildcard() == false) {
+                return expressions;
             }
-            Collection<String> result = null;
-            boolean wildcardSeen = false;
-            for (int i = 0; i < expressions.size(); i++) {
-                String expression = validateAliasOrIndex(expressions.get(i));
-                boolean isExclusion = false;
-                if (expression.charAt(0) == '-' && wildcardSeen) {
-                    isExclusion = true;
-                    expression = expression.substring(1);
-                }
-                if (Regex.isSimpleMatchPattern(expression)) {
-                    wildcardSeen = true;
-                    Stream<IndexAbstraction> matchingResources = matchResourcesToWildcard(context, expression);
+            Set<String> result = new HashSet<>();
+            for (ExpressionList.Expression expression : expressionList) {
+                if (expression.isWildcard()) {
+                    Stream<IndexAbstraction> matchingResources = matchResourcesToWildcard(context, expression.get());
                     Stream<String> matchingOpenClosedNames = expandToOpenClosed(context, matchingResources);
                     AtomicBoolean emptyWildcardExpansion = new AtomicBoolean(false);
                     if (context.getOptions().allowNoIndices() == false) {
                         emptyWildcardExpansion.set(true);
                         matchingOpenClosedNames = matchingOpenClosedNames.peek(x -> emptyWildcardExpansion.set(false));
                     }
-                    if (result == null) {
-                        // add all the previous expressions because they exist but were not added, as an optimisation
-                        result = new HashSet<>(expressions.subList(0, i));
-                    }
-                    if (isExclusion) {
+                    if (expression.isExclusion()) {
                         matchingOpenClosedNames.forEachOrdered(result::remove);
                     } else {
                         matchingOpenClosedNames.forEachOrdered(result::add);
                     }
                     if (emptyWildcardExpansion.get()) {
-                        throw notFoundException(expression);
+                        throw notFoundException(expression.get());
                     }
                 } else {
-                    if (isExclusion) {
-                        if (result == null) {
-                            // add all the previous expressions because they exist but were not added, as an optimisation
-                            result = new HashSet<>(expressions.subList(0, i));
-                        }
-                        result.remove(expression);
+                    if (expression.isExclusion()) {
+                        result.remove(expression.get());
                     } else {
-                        // missing expression that is neither an exclusion nor a wildcard
-                        // TODO investigate if this check can be moved outside the wildcard resolver
-                        if (context.getOptions().ignoreUnavailable() == false) {
-                            ensureAliasOrIndexExists(context, expression);
-                        }
-                        if (result != null) {
-                            // skip adding the expression as an optimization
-                            result.add(expression);
-                        }
+                        result.add(expression.get());
                     }
                 }
             }
-            if (result == null) {
-                // optimisation that avoids allocating a new collection when all the argument expressions are explicit names
-                return expressions;
-            } else {
-                return result;
-            }
-        }
-
-        private static String validateAliasOrIndex(String expression) {
-            if (Strings.isEmpty(expression)) {
-                throw notFoundException(expression);
-            }
-            // Expressions can not start with an underscore. This is reserved for APIs. If the check gets here, the API
-            // does not exist and the path is interpreted as an expression. If the expression begins with an underscore,
-            // throw a specific error that is different from the [[IndexNotFoundException]], which is typically thrown
-            // if the expression can't be found.
-            if (expression.charAt(0) == '_') {
-                throw new InvalidIndexNameException(expression, "must not start with '_'.");
-            }
-            return expression;
+            return result;
         }
 
         private static IndexMetadata.State excludeState(IndicesOptions options) {
@@ -1287,7 +1206,7 @@ private static IndexMetadata.State excludeState(IndicesOptions options) {
          * It does NOT consider the open or closed status of index resources.
          */
         private static Stream<IndexAbstraction> matchResourcesToWildcard(Context context, String wildcardExpression) {
-            assert Regex.isSimpleMatchPattern(wildcardExpression);
+            assert isWildcard(wildcardExpression);
             final SortedMap<String, IndexAbstraction> indicesLookup = context.getState().getMetadata().getIndicesLookup();
             Stream<IndexAbstraction> matchesStream;
             if (Regex.isSuffixMatchPattern(wildcardExpression)) {
@@ -1364,11 +1283,6 @@ private static Stream<String> expandToOpenClosed(Context context, Stream<IndexAb
             });
         }
 
-        private static boolean isEmptyOrTrivialWildcard(List<String> expressions) {
-            return expressions.isEmpty()
-                || (expressions.size() == 1 && (Metadata.ALL.equals(expressions.get(0)) || Regex.isMatchAllPattern(expressions.get(0))));
-        }
-
         private static List<String> resolveEmptyOrTrivialWildcard(Context context) {
             final String[] allIndices = resolveEmptyOrTrivialWildcardToAllIndices(context.getOptions(), context.getState().metadata());
             if (context.systemIndexAccessLevel == SystemIndexAccessLevel.ALL) {
@@ -1434,19 +1348,10 @@ private DateMathExpressionResolver() {
             // utility class
         }
 
-        public static List<String> resolve(final Context context, List<String> expressions) {
+        public static List<String> resolve(Context context, List<String> expressions) {
             List<String> result = new ArrayList<>(expressions.size());
-            boolean wildcardSeen = false;
-            for (String expression : expressions) {
-                // accepts date-math exclusions that are of the form "-<...{}>", i.e. the "-" is outside the "<>" date-math template
-                if (Strings.hasLength(expression) && expression.charAt(0) == '-' && wildcardSeen) {
-                    result.add("-" + resolveExpression(expression.substring(1), context::getStartTime));
-                } else {
-                    result.add(resolveExpression(expression, context::getStartTime));
-                }
-                if (context.getOptions().expandWildcardExpressions() && Regex.isSimpleMatchPattern(expression)) {
-                    wildcardSeen = true;
-                }
+            for (ExpressionList.Expression expression : new ExpressionList(context, expressions)) {
+                result.add(resolveExpression(expression, context::getStartTime));
             }
             return result;
         }
@@ -1455,6 +1360,15 @@ static String resolveExpression(String expression) {
             return resolveExpression(expression, System::currentTimeMillis);
         }
 
+        static String resolveExpression(ExpressionList.Expression expression, LongSupplier getTime) {
+            if (expression.isExclusion()) {
+                // accepts date-math exclusions that are of the form "-<...{}>", i.e. the "-" is outside the "<>" date-math template
+                return "-" + resolveExpression(expression.get(), getTime);
+            } else {
+                return resolveExpression(expression.get(), getTime);
+            }
+        }
+
         @SuppressWarnings("fallthrough")
         static String resolveExpression(String expression, LongSupplier getTime) {
             if (expression.startsWith(EXPRESSION_LEFT_BOUND) == false || expression.endsWith(EXPRESSION_RIGHT_BOUND) == false) {
@@ -1601,6 +1515,141 @@ static String resolveExpression(String expression, LongSupplier getTime) {
         }
     }
 
+    public static final class ExplicitResourceNameFilter {
+
+        private ExplicitResourceNameFilter() {
+            // Utility class
+        }
+
+        /**
+         * Returns an expression list with "unavailable" (missing or not acceptable) resource names filtered out.
+         * Only explicit resource names are considered for filtering. Wildcard and exclusion expressions are kept in.
+         */
+        public static List<String> filterUnavailable(Context context, List<String> expressions) {
+            ensureRemoteIndicesRequireIgnoreUnavailable(context.getOptions(), expressions);
+            List<String> result = new ArrayList<>(expressions.size());
+            for (ExpressionList.Expression expression : new ExpressionList(context, expressions)) {
+                validateAliasOrIndex(expression);
+                if (expression.isWildcard() || expression.isExclusion() || ensureAliasOrIndexExists(context, expression.get())) {
+                    result.add(expression.expression());
+                }
+            }
+            return result;
+        }
+
+        /**
+         * This returns `true` if the given {@param name} is of a resource that exists.
+         * Otherwise, it returns `false` if the `ignore_unvailable` option is `true`, or, if `false`, it throws a "not found" type of
+         * exception.
+         */
+        @Nullable
+        private static boolean ensureAliasOrIndexExists(Context context, String name) {
+            boolean ignoreUnavailable = context.getOptions().ignoreUnavailable();
+            IndexAbstraction indexAbstraction = context.getState().getMetadata().getIndicesLookup().get(name);
+            if (indexAbstraction == null) {
+                if (ignoreUnavailable) {
+                    return false;
+                } else {
+                    throw notFoundException(name);
+                }
+            }
+            // treat aliases as unavailable indices when ignoreAliases is set to true (e.g. delete index and update aliases api)
+            if (indexAbstraction.getType() == Type.ALIAS && context.getOptions().ignoreAliases()) {
+                if (ignoreUnavailable) {
+                    return false;
+                } else {
+                    throw aliasesNotSupportedException(name);
+                }
+            }
+            if (indexAbstraction.isDataStreamRelated() && context.includeDataStreams() == false) {
+                if (ignoreUnavailable) {
+                    return false;
+                } else {
+                    IndexNotFoundException infe = notFoundException(name);
+                    // Allows callers to handle IndexNotFoundException differently based on whether data streams were excluded.
+                    infe.addMetadata(EXCLUDED_DATA_STREAMS_KEY, "true");
+                    throw infe;
+                }
+            }
+            return true;
+        }
+
+        private static void validateAliasOrIndex(ExpressionList.Expression expression) {
+            if (Strings.isEmpty(expression.expression())) {
+                throw notFoundException(expression.expression());
+            }
+            // Expressions can not start with an underscore. This is reserved for APIs. If the check gets here, the API
+            // does not exist and the path is interpreted as an expression. If the expression begins with an underscore,
+            // throw a specific error that is different from the [[IndexNotFoundException]], which is typically thrown
+            // if the expression can't be found.
+            if (expression.expression().charAt(0) == '_') {
+                throw new InvalidIndexNameException(expression.expression(), "must not start with '_'.");
+            }
+        }
+
+        private static void ensureRemoteIndicesRequireIgnoreUnavailable(IndicesOptions options, List<String> indexExpressions) {
+            if (options.ignoreUnavailable() == false) {
+                List<String> crossClusterIndices = indexExpressions.stream().filter(index -> index.contains(":")).toList();
+                if (crossClusterIndices.size() > 0) {
+                    throw new IllegalArgumentException(
+                        "Cross-cluster calls are not supported in this context but remote indices were requested: " + crossClusterIndices
+                    );
+                }
+            }
+        }
+    }
+
+    /**
+     * Used to iterate expression lists and work out which expression item is a wildcard or an exclusion.
+     */
+    public static final class ExpressionList implements Iterable<ExpressionList.Expression> {
+        private final List<Expression> expressionsList;
+        private final boolean hasWildcard;
+
+        public record Expression(String expression, boolean isWildcard, boolean isExclusion) {
+            public String get() {
+                if (isExclusion()) {
+                    // drop the leading "-" if exclusion because it is easier for callers to handle it like this
+                    return expression().substring(1);
+                } else {
+                    return expression();
+                }
+            }
+        }
+
+        /**
+         * Creates the expression iterable that can be used to easily check which expression item is a wildcard or an exclusion (or both).
+         * The {@param context} is used to check if wildcards ought to be considered or not.
+         */
+        public ExpressionList(Context context, List<String> expressionStrings) {
+            List<Expression> expressionsList = new ArrayList<>(expressionStrings.size());
+            boolean wildcardSeen = false;
+            for (String expressionString : expressionStrings) {
+                boolean isExclusion = expressionString.startsWith("-") && wildcardSeen;
+                if (context.getOptions().expandWildcardExpressions() && isWildcard(expressionString)) {
+                    wildcardSeen = true;
+                    expressionsList.add(new Expression(expressionString, true, isExclusion));
+                } else {
+                    expressionsList.add(new Expression(expressionString, false, isExclusion));
+                }
+            }
+            this.expressionsList = expressionsList;
+            this.hasWildcard = wildcardSeen;
+        }
+
+        /**
+         * Returns {@code true} if the expression contains any wildcard and the options allow wildcard expansion
+         */
+        public boolean hasWildcard() {
+            return this.hasWildcard;
+        }
+
+        @Override
+        public Iterator<ExpressionList.Expression> iterator() {
+            return expressionsList.iterator();
+        }
+    }
+
     /**
      * This is a context for the DateMathExpressionResolver which does not require {@code IndicesOptions} or {@code ClusterState}
      * since it uses only the start time to resolve expressions.
@@ -1624,4 +1673,8 @@ public IndicesOptions getOptions() {
             throw new UnsupportedOperationException("should never be called");
         }
     }
+
+    private static boolean isWildcard(String expression) {
+        return Regex.isSimpleMatchPattern(expression);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java
index e5f37ab537ce..dfc05552d3e7 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java
@@ -315,14 +315,14 @@ public IndexTemplateMetadata build() {
          * This method is used for serializing templates before storing them in the cluster metadata,
          * and also in the REST layer when returning a deprecated typed response.
          */
-        public static void toXContentWithTypes(
+        public static XContentBuilder toXContentWithTypes(
             IndexTemplateMetadata indexTemplateMetadata,
             XContentBuilder builder,
             ToXContent.Params params
         ) throws IOException {
             builder.startObject(indexTemplateMetadata.name());
             toInnerXContent(indexTemplateMetadata, builder, params, true);
-            builder.endObject();
+            return builder.endObject();
         }
 
         /**
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MappingMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MappingMetadata.java
index 17f1b75c2e54..d102624ccb06 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MappingMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MappingMetadata.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.SimpleDiffable;
 import org.elasticsearch.common.compress.CompressedXContent;
@@ -75,7 +75,7 @@ public MappingMetadata(String type, Map<String, Object> mapping) {
     }
 
     public static void writeMappingMetadata(StreamOutput out, Map<String, MappingMetadata> mappings) throws IOException {
-        out.writeMap(mappings, StreamOutput::writeString, out.getVersion().before(Version.V_8_0_0) ? (o, v) -> {
+        out.writeMap(mappings, StreamOutput::writeString, out.getTransportVersion().before(TransportVersion.V_8_0_0) ? (o, v) -> {
             o.writeVInt(v == EMPTY_MAPPINGS ? 0 : 1);
             if (v != EMPTY_MAPPINGS) {
                 o.writeString(MapperService.SINGLE_MAPPING_NAME);
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java
index 21dd59e79199..4d19109738a8 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java
@@ -11,6 +11,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.Diff;
@@ -22,11 +23,13 @@
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.coordination.CoordinationMetadata;
+import org.elasticsearch.cluster.coordination.PublicationTransportHandler;
 import org.elasticsearch.cluster.metadata.IndexAbstraction.ConcreteIndex;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.VersionedNamedWriteable;
@@ -37,6 +40,8 @@
 import org.elasticsearch.common.util.ArrayUtils;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.core.Nullable;
@@ -51,7 +56,6 @@
 import org.elasticsearch.xcontent.NamedObjectNotFoundException;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 
@@ -91,7 +95,7 @@
  * The details of how this is persisted are covered in {@link org.elasticsearch.gateway.PersistedClusterStateService}.
  * </p>
  */
-public class Metadata extends AbstractCollection<IndexMetadata> implements Diffable<Metadata>, ToXContentFragment {
+public class Metadata extends AbstractCollection<IndexMetadata> implements Diffable<Metadata>, ChunkedToXContent {
 
     private static final Logger logger = LogManager.getLogger(Metadata.class);
 
@@ -137,7 +141,7 @@ public enum XContentContext {
      * Custom metadata that persists (via XContent) across restarts. The deserialization method for each implementation must be registered
      * with the {@link NamedXContentRegistry}.
      */
-    public interface Custom extends NamedDiffable<Custom>, ToXContentFragment {
+    public interface Custom extends NamedDiffable<Custom>, ChunkedToXContent {
 
         EnumSet<XContentContext> context();
 
@@ -279,6 +283,33 @@ private Metadata(
         this.mappingsByHash = mappingsByHash;
         this.oldestIndexVersion = oldestIndexVersion;
         this.reservedStateMetadata = reservedStateMetadata;
+        assert assertConsistent();
+    }
+
+    private boolean assertConsistent() {
+        final var lookup = this.indicesLookup;
+        final var dsMetadata = custom(DataStreamMetadata.TYPE, DataStreamMetadata.EMPTY);
+        assert lookup == null || lookup.equals(Builder.buildIndicesLookup(dsMetadata, indices));
+        try {
+            Builder.ensureNoNameCollisions(aliasedIndices.keySet(), indices, dsMetadata);
+        } catch (Exception e) {
+            assert false : e;
+        }
+        assert Builder.assertDataStreams(indices, dsMetadata);
+        assert Set.of(allIndices).equals(indices.keySet());
+        final Function<Predicate<IndexMetadata>, Set<String>> indicesByPredicate = predicate -> indices.entrySet()
+            .stream()
+            .filter(entry -> predicate.test(entry.getValue()))
+            .map(Map.Entry::getKey)
+            .collect(Collectors.toUnmodifiableSet());
+        assert Set.of(allOpenIndices).equals(indicesByPredicate.apply(idx -> idx.getState() == IndexMetadata.State.OPEN));
+        assert Set.of(allClosedIndices).equals(indicesByPredicate.apply(idx -> idx.getState() == IndexMetadata.State.CLOSE));
+        assert Set.of(visibleIndices).equals(indicesByPredicate.apply(idx -> idx.isHidden() == false));
+        assert Set.of(visibleOpenIndices)
+            .equals(indicesByPredicate.apply(idx -> idx.isHidden() == false && idx.getState() == IndexMetadata.State.OPEN));
+        assert Set.of(visibleClosedIndices)
+            .equals(indicesByPredicate.apply(idx -> idx.isHidden() == false && idx.getState() == IndexMetadata.State.CLOSE));
+        return true;
     }
 
     public Metadata withIncrementedVersion() {
@@ -440,6 +471,42 @@ public Metadata withCoordinationMetadata(CoordinationMetadata coordinationMetada
         );
     }
 
+    public Metadata withLastCommittedValues(
+        boolean clusterUUIDCommitted,
+        CoordinationMetadata.VotingConfiguration lastCommittedConfiguration
+    ) {
+        if (clusterUUIDCommitted == this.clusterUUIDCommitted
+            && lastCommittedConfiguration.equals(this.coordinationMetadata.getLastCommittedConfiguration())) {
+            return this;
+        }
+        return new Metadata(
+            clusterUUID,
+            clusterUUIDCommitted,
+            version,
+            CoordinationMetadata.builder(coordinationMetadata).lastCommittedConfiguration(lastCommittedConfiguration).build(),
+            transientSettings,
+            persistentSettings,
+            settings,
+            hashesOfConsistentSettings,
+            totalNumberOfShards,
+            totalOpenIndexShards,
+            indices,
+            aliasedIndices,
+            templates,
+            customs,
+            allIndices,
+            visibleIndices,
+            allOpenIndices,
+            visibleOpenIndices,
+            allClosedIndices,
+            visibleClosedIndices,
+            indicesLookup,
+            mappingsByHash,
+            oldestIndexVersion,
+            reservedStateMetadata
+        );
+    }
+
     /**
      * Creates a copy of this instance updated with the given {@link IndexMetadata} that must only contain changes to primary terms
      * and in-sync allocation ids relative to the existing entries. This method is only used by
@@ -1297,6 +1364,9 @@ public static boolean isGlobalStateEquals(Metadata metadata1, Metadata metadata2
         if (customCount1 != customCount2) {
             return false;
         }
+        if (Objects.equals(metadata1.reservedStateMetadata, metadata2.reservedStateMetadata) == false) {
+            return false;
+        }
         return true;
     }
 
@@ -1306,7 +1376,7 @@ public Diff<Metadata> diff(Metadata previousState) {
     }
 
     public static Diff<Metadata> readDiffFrom(StreamInput in) throws IOException {
-        if (in.getVersion().onOrAfter(MetadataDiff.NOOP_METADATA_DIFF_VERSION) && in.readBoolean()) {
+        if (in.getTransportVersion().onOrAfter(MetadataDiff.NOOP_METADATA_DIFF_VERSION) && in.readBoolean()) {
             return SimpleDiffable.empty();
         }
         return new MetadataDiff(in);
@@ -1317,9 +1387,55 @@ public static Metadata fromXContent(XContentParser parser) throws IOException {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        Builder.toXContent(this, builder, params);
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params p) {
+        XContentContext context = XContentContext.valueOf(p.param(CONTEXT_MODE_PARAM, CONTEXT_MODE_API));
+        final Iterator<? extends ToXContent> start = context == XContentContext.API
+            ? ChunkedToXContentHelper.startObject("metadata")
+            : Iterators.single((builder, params) -> builder.startObject("meta-data").field("version", version()));
+
+        final Iterator<? extends ToXContent> persistentSettings = context != XContentContext.API && persistentSettings().isEmpty() == false
+            ? Iterators.single((builder, params) -> {
+                builder.startObject("settings");
+                persistentSettings().toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true")));
+                return builder.endObject();
+            })
+            : Collections.emptyIterator();
+
+        final Iterator<? extends ToXContent> indices = context == XContentContext.API
+            ? ChunkedToXContentHelper.wrapWithObject("indices", indices().values().iterator())
+            : Collections.emptyIterator();
+
+        return Iterators.concat(start, Iterators.<ToXContent>single((builder, params) -> {
+            builder.field("cluster_uuid", clusterUUID);
+            builder.field("cluster_uuid_committed", clusterUUIDCommitted);
+            builder.startObject("cluster_coordination");
+            coordinationMetadata().toXContent(builder, params);
+            return builder.endObject();
+        }),
+            persistentSettings,
+            ChunkedToXContentHelper.wrapWithObject(
+                "templates",
+                templates().values()
+                    .stream()
+                    .map(
+                        template -> (ToXContent) (builder, params) -> IndexTemplateMetadata.Builder.toXContentWithTypes(
+                            template,
+                            builder,
+                            params
+                        )
+                    )
+                    .iterator()
+            ),
+            indices,
+            Iterators.flatMap(
+                customs.entrySet().iterator(),
+                entry -> entry.getValue().context().contains(context)
+                    ? ChunkedToXContentHelper.wrapWithObject(entry.getKey(), entry.getValue().toXContentChunked(p))
+                    : Collections.emptyIterator()
+            ),
+            ChunkedToXContentHelper.wrapWithObject("reserved_state", reservedStateMetadata().values().iterator()),
+            ChunkedToXContentHelper.endObject()
+        );
     }
 
     public Map<String, MappingMetadata> getMappingsByHash() {
@@ -1328,7 +1444,9 @@ public Map<String, MappingMetadata> getMappingsByHash() {
 
     private static class MetadataDiff implements Diff<Metadata> {
 
-        private static final Version NOOP_METADATA_DIFF_VERSION = Version.V_8_5_0;
+        private static final TransportVersion NOOP_METADATA_DIFF_VERSION = TransportVersion.V_8_5_0;
+        private static final TransportVersion NOOP_METADATA_DIFF_SAFE_VERSION =
+            PublicationTransportHandler.INCLUDES_LAST_COMMITTED_DATA_VERSION;
 
         private final long version;
         private final String clusterUUID;
@@ -1394,7 +1512,7 @@ private MetadataDiff(StreamInput in) throws IOException {
             coordinationMetadata = new CoordinationMetadata(in);
             transientSettings = Settings.readSettingsFromStream(in);
             persistentSettings = Settings.readSettingsFromStream(in);
-            if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_3_0)) {
                 hashesOfConsistentSettings = DiffableStringMap.readDiffFrom(in);
             } else {
                 hashesOfConsistentSettings = DiffableStringMap.DiffableStringMapDiff.EMPTY;
@@ -1402,7 +1520,7 @@ private MetadataDiff(StreamInput in) throws IOException {
             indices = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), INDEX_METADATA_DIFF_VALUE_READER);
             templates = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), TEMPLATES_DIFF_VALUE_READER);
             customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER);
-            if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
                 reservedStateMetadata = DiffableUtils.readJdkMapDiff(
                     in,
                     DiffableUtils.getStringKeySerializer(),
@@ -1415,12 +1533,15 @@ private MetadataDiff(StreamInput in) throws IOException {
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            if (out.getVersion().onOrAfter(NOOP_METADATA_DIFF_VERSION)) {
+            if (out.getTransportVersion().onOrAfter(NOOP_METADATA_DIFF_SAFE_VERSION)) {
                 out.writeBoolean(empty);
                 if (empty) {
                     // noop diff
                     return;
                 }
+            } else if (out.getTransportVersion().onOrAfter(NOOP_METADATA_DIFF_VERSION)) {
+                // noops are not safe with these versions, see #92259
+                out.writeBoolean(false);
             }
             out.writeString(clusterUUID);
             out.writeBoolean(clusterUUIDCommitted);
@@ -1428,13 +1549,13 @@ public void writeTo(StreamOutput out) throws IOException {
             coordinationMetadata.writeTo(out);
             transientSettings.writeTo(out);
             persistentSettings.writeTo(out);
-            if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_3_0)) {
                 hashesOfConsistentSettings.writeTo(out);
             }
             indices.writeTo(out);
             templates.writeTo(out);
             customs.writeTo(out);
-            if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
                 reservedStateMetadata.writeTo(out);
             }
         }
@@ -1467,7 +1588,7 @@ public Metadata apply(Metadata part) {
         }
     }
 
-    public static final Version MAPPINGS_AS_HASH_VERSION = Version.V_8_1_0;
+    public static final TransportVersion MAPPINGS_AS_HASH_VERSION = TransportVersion.V_8_1_0;
 
     public static Metadata readFrom(StreamInput in) throws IOException {
         Builder builder = new Builder();
@@ -1477,11 +1598,11 @@ public static Metadata readFrom(StreamInput in) throws IOException {
         builder.coordinationMetadata(new CoordinationMetadata(in));
         builder.transientSettings(readSettingsFromStream(in));
         builder.persistentSettings(readSettingsFromStream(in));
-        if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_3_0)) {
             builder.hashesOfConsistentSettings(DiffableStringMap.readFrom(in));
         }
         final Function<String, MappingMetadata> mappingLookup;
-        if (in.getVersion().onOrAfter(MAPPINGS_AS_HASH_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(MAPPINGS_AS_HASH_VERSION)) {
             final Map<String, MappingMetadata> mappingMetadataMap = in.readMapValues(MappingMetadata::new, MappingMetadata::getSha256);
             if (mappingMetadataMap.size() > 0) {
                 mappingLookup = mappingMetadataMap::get;
@@ -1504,7 +1625,7 @@ public static Metadata readFrom(StreamInput in) throws IOException {
             Custom customIndexMetadata = in.readNamedWriteable(Custom.class);
             builder.putCustom(customIndexMetadata.getWriteableName(), customIndexMetadata);
         }
-        if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             int reservedStateSize = in.readVInt();
             for (int i = 0; i < reservedStateSize; i++) {
                 builder.put(ReservedStateMetadata.readFrom(in));
@@ -1521,22 +1642,22 @@ public void writeTo(StreamOutput out) throws IOException {
         coordinationMetadata.writeTo(out);
         transientSettings.writeTo(out);
         persistentSettings.writeTo(out);
-        if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_3_0)) {
             hashesOfConsistentSettings.writeTo(out);
         }
         // Starting in #MAPPINGS_AS_HASH_VERSION we write the mapping metadata first and then write the indices without metadata so that
         // we avoid writing duplicate mappings twice
-        if (out.getVersion().onOrAfter(MAPPINGS_AS_HASH_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(MAPPINGS_AS_HASH_VERSION)) {
             out.writeMapValues(mappingsByHash);
         }
         out.writeVInt(indices.size());
-        final boolean writeMappingsHash = out.getVersion().onOrAfter(MAPPINGS_AS_HASH_VERSION);
+        final boolean writeMappingsHash = out.getTransportVersion().onOrAfter(MAPPINGS_AS_HASH_VERSION);
         for (IndexMetadata indexMetadata : this) {
             indexMetadata.writeTo(out, writeMappingsHash);
         }
         out.writeCollection(templates.values());
         VersionedNamedWriteable.writeVersionedWritables(out, customs);
-        if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             out.writeCollection(reservedStateMetadata.values());
         }
     }
@@ -2468,60 +2589,6 @@ static boolean assertDataStreams(Map<String, IndexMetadata> indices, DataStreamM
             return true;
         }
 
-        public static void toXContent(Metadata metadata, XContentBuilder builder, ToXContent.Params params) throws IOException {
-            XContentContext context = XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, CONTEXT_MODE_API));
-
-            if (context == XContentContext.API) {
-                builder.startObject("metadata");
-            } else {
-                builder.startObject("meta-data");
-                builder.field("version", metadata.version());
-            }
-
-            builder.field("cluster_uuid", metadata.clusterUUID);
-            builder.field("cluster_uuid_committed", metadata.clusterUUIDCommitted);
-
-            builder.startObject("cluster_coordination");
-            metadata.coordinationMetadata().toXContent(builder, params);
-            builder.endObject();
-
-            if (context != XContentContext.API && metadata.persistentSettings().isEmpty() == false) {
-                builder.startObject("settings");
-                metadata.persistentSettings().toXContent(builder, new MapParams(Collections.singletonMap("flat_settings", "true")));
-                builder.endObject();
-            }
-
-            builder.startObject("templates");
-            for (IndexTemplateMetadata template : metadata.templates().values()) {
-                IndexTemplateMetadata.Builder.toXContentWithTypes(template, builder, params);
-            }
-            builder.endObject();
-
-            if (context == XContentContext.API) {
-                builder.startObject("indices");
-                for (IndexMetadata indexMetadata : metadata) {
-                    IndexMetadata.Builder.toXContent(indexMetadata, builder, params);
-                }
-                builder.endObject();
-            }
-
-            for (Map.Entry<String, Custom> cursor : metadata.customs().entrySet()) {
-                if (cursor.getValue().context().contains(context)) {
-                    builder.startObject(cursor.getKey());
-                    cursor.getValue().toXContent(builder, params);
-                    builder.endObject();
-                }
-            }
-
-            builder.startObject("reserved_state");
-            for (ReservedStateMetadata ReservedStateMetadata : metadata.reservedStateMetadata().values()) {
-                ReservedStateMetadata.toXContent(builder, params);
-            }
-            builder.endObject();
-
-            builder.endObject();
-        }
-
         public static Metadata fromXContent(XContentParser parser) throws IOException {
             Builder builder = new Builder();
 
@@ -2639,7 +2706,7 @@ private void dedupeMapping(IndexMetadata.Builder indexMetadataBuilder) {
         Map<String, String> params = Maps.newMapWithExpectedSize(2);
         params.put("binary", "true");
         params.put(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY);
-        FORMAT_PARAMS = new MapParams(params);
+        FORMAT_PARAMS = new ToXContent.MapParams(params);
     }
 
     /**
@@ -2649,7 +2716,7 @@ private void dedupeMapping(IndexMetadata.Builder indexMetadataBuilder) {
 
         @Override
         public void toXContent(XContentBuilder builder, Metadata state) throws IOException {
-            Builder.toXContent(state, builder, FORMAT_PARAMS);
+            ChunkedToXContent.wrapAsToXContent(state).toXContent(builder, FORMAT_PARAMS);
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java
index 99585895005e..222ec3c68915 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingRoleStrategy;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.DataTier;
@@ -504,7 +505,13 @@ private ClusterState applyCreateIndexWithTemporaryService(
 
             indexService.getIndexEventListener().beforeIndexAddedToCluster(indexMetadata.getIndex(), indexMetadata.getSettings());
 
-            ClusterState updated = clusterStateCreateIndex(currentState, request.blocks(), indexMetadata, metadataTransformer);
+            ClusterState updated = clusterStateCreateIndex(
+                currentState,
+                request.blocks(),
+                indexMetadata,
+                metadataTransformer,
+                allocationService.getShardRoutingRoleStrategy()
+            );
             if (request.performReroute()) {
                 updated = allocationService.reroute(updated, "index [" + indexMetadata.getIndex().getName() + "] created", rerouteListener);
             }
@@ -1219,7 +1226,8 @@ static ClusterState clusterStateCreateIndex(
         ClusterState currentState,
         Set<ClusterBlock> clusterBlocks,
         IndexMetadata indexMetadata,
-        BiConsumer<Metadata.Builder, IndexMetadata> metadataTransformer
+        BiConsumer<Metadata.Builder, IndexMetadata> metadataTransformer,
+        ShardRoutingRoleStrategy shardRoutingRoleStrategy
     ) {
         final Metadata newMetadata;
         if (metadataTransformer != null) {
@@ -1236,7 +1244,7 @@ static ClusterState clusterStateCreateIndex(
 
         ClusterState updatedState = ClusterState.builder(currentState).blocks(blocks).metadata(newMetadata).build();
 
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(updatedState.routingTable())
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(shardRoutingRoleStrategy, updatedState.routingTable())
             .addAsNew(updatedState.metadata().index(indexName));
         return ClusterState.builder(updatedState).routingTable(routingTableBuilder.build()).build();
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java
index ed53313a95ed..0ec58787dfb6 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java
@@ -12,15 +12,20 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesClusterStateUpdateRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.ClusterStateAckListener;
+import org.elasticsearch.cluster.ClusterStateTaskConfig;
+import org.elasticsearch.cluster.ClusterStateTaskExecutor;
+import org.elasticsearch.cluster.ClusterStateTaskListener;
+import org.elasticsearch.cluster.SimpleBatchedAckListenerTaskExecutor;
 import org.elasticsearch.cluster.metadata.AliasAction.NewAliasValidator;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexService;
@@ -55,6 +60,8 @@ public class MetadataIndexAliasesService {
 
     private final NamedXContentRegistry xContentRegistry;
 
+    private final ClusterStateTaskExecutor<ApplyAliasesTask> executor;
+
     @Inject
     public MetadataIndexAliasesService(
         ClusterService clusterService,
@@ -66,20 +73,19 @@ public MetadataIndexAliasesService(
         this.indicesService = indicesService;
         this.deleteIndexService = deleteIndexService;
         this.xContentRegistry = xContentRegistry;
-    }
+        this.executor = new SimpleBatchedAckListenerTaskExecutor<>() {
 
-    public void indicesAliases(final IndicesAliasesClusterStateUpdateRequest request, final ActionListener<AcknowledgedResponse> listener) {
-        submitUnbatchedTask("index-aliases", new AckedClusterStateUpdateTask(Priority.URGENT, request, listener) {
             @Override
-            public ClusterState execute(ClusterState currentState) {
-                return applyAliasActions(currentState, request.actions());
+            public Tuple<ClusterState, ClusterStateAckListener> executeTask(ApplyAliasesTask applyAliasesTask, ClusterState clusterState) {
+                return new Tuple<>(applyAliasActions(clusterState, applyAliasesTask.request().actions()), applyAliasesTask);
             }
-        });
+        };
     }
 
-    @SuppressForbidden(reason = "legacy usage of unbatched task") // TODO add support for batching here
-    private void submitUnbatchedTask(@SuppressWarnings("SameParameterValue") String source, ClusterStateUpdateTask task) {
-        clusterService.submitUnbatchedStateUpdateTask(source, task);
+    public void indicesAliases(final IndicesAliasesClusterStateUpdateRequest request, final ActionListener<AcknowledgedResponse> listener) {
+        var task = new ApplyAliasesTask(request, listener);
+        var config = ClusterStateTaskConfig.build(Priority.URGENT);
+        clusterService.submitStateUpdateTask("index-aliases", task, config, executor);
     }
 
     /**
@@ -198,6 +204,11 @@ public ClusterState applyAliasActions(ClusterState currentState, Iterable<AliasA
         }
     }
 
+    // Visible for testing purposes
+    ClusterStateTaskExecutor<ApplyAliasesTask> getExecutor() {
+        return executor;
+    }
+
     private void validateFilter(
         List<Index> indicesToClose,
         Map<String, IndexService> indices,
@@ -244,4 +255,43 @@ private static void validateAliasTargetIsNotDSBackingIndex(ClusterState currentS
             );
         }
     }
+
+    /**
+     * A cluster state update task that consists of the cluster state request and the listeners that need to be notified upon completion.
+     */
+    record ApplyAliasesTask(IndicesAliasesClusterStateUpdateRequest request, ActionListener<AcknowledgedResponse> listener)
+        implements
+            ClusterStateTaskListener,
+            ClusterStateAckListener {
+
+        @Override
+        public void onFailure(Exception e) {
+            listener.onFailure(e);
+        }
+
+        @Override
+        public boolean mustAck(DiscoveryNode discoveryNode) {
+            return true;
+        }
+
+        @Override
+        public void onAllNodesAcked() {
+            listener.onResponse(AcknowledgedResponse.TRUE);
+        }
+
+        @Override
+        public void onAckFailure(Exception e) {
+            listener.onResponse(AcknowledgedResponse.FALSE);
+        }
+
+        @Override
+        public void onAckTimeout() {
+            listener.onResponse(AcknowledgedResponse.FALSE);
+        }
+
+        @Override
+        public TimeValue ackTimeout() {
+            return request.ackTimeout();
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java
index 592654ac8c12..322109b1a2d9 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
-import org.elasticsearch.action.NotifyOnceListener;
 import org.elasticsearch.action.admin.indices.close.CloseIndexClusterStateUpdateRequest;
 import org.elasticsearch.action.admin.indices.close.CloseIndexResponse;
 import org.elasticsearch.action.admin.indices.close.CloseIndexResponse.IndexResult;
@@ -45,6 +44,7 @@
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRoutingRoleStrategy;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.allocator.AllocationActionMultiListener;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -225,7 +225,8 @@ public ClusterState execute(BatchExecutionContext<CloseIndicesTask> batchExecuti
                     final Tuple<ClusterState, List<IndexResult>> closingResult = closeRoutingTable(
                         state,
                         task.blockedIndices,
-                        task.verifyResults
+                        task.verifyResults,
+                        allocationService.getShardRoutingRoleStrategy()
                     );
                     state = closingResult.v1();
                     final List<IndexResult> indices = closingResult.v2();
@@ -619,9 +620,9 @@ private void waitForShardsReadyForClosing(
             for (int i = 0; i < indexRoutingTable.size(); i++) {
                 IndexShardRoutingTable shardRoutingTable = indexRoutingTable.shard(i);
                 final int shardId = shardRoutingTable.shardId().id();
-                sendVerifyShardBeforeCloseRequest(shardRoutingTable, closingBlock, new NotifyOnceListener<>() {
+                sendVerifyShardBeforeCloseRequest(shardRoutingTable, closingBlock, ActionListener.notifyOnce(new ActionListener<>() {
                     @Override
-                    public void innerOnResponse(final ReplicationResponse replicationResponse) {
+                    public void onResponse(ReplicationResponse replicationResponse) {
                         ShardResult.Failure[] failures = Arrays.stream(replicationResponse.getShardInfo().getFailures())
                             .map(f -> new ShardResult.Failure(f.index(), f.shardId(), f.getCause(), f.nodeId()))
                             .toArray(ShardResult.Failure[]::new);
@@ -630,7 +631,7 @@ public void innerOnResponse(final ReplicationResponse replicationResponse) {
                     }
 
                     @Override
-                    public void innerOnFailure(final Exception e) {
+                    public void onFailure(Exception e) {
                         ShardResult.Failure failure = new ShardResult.Failure(index.getName(), shardId, e);
                         results.setOnce(shardId, new ShardResult(shardId, new ShardResult.Failure[] { failure }));
                         processIfFinished();
@@ -641,7 +642,7 @@ private void processIfFinished() {
                             onResponse.accept(new IndexResult(index, results.toArray(new ShardResult[results.length()])));
                         }
                     }
-                });
+                }));
             }
         }
 
@@ -749,9 +750,9 @@ private void waitForShardsReady(
             for (int i = 0; i < indexRoutingTable.size(); i++) {
                 IndexShardRoutingTable shardRoutingTable = indexRoutingTable.shard(i);
                 final int shardId = shardRoutingTable.shardId().id();
-                sendVerifyShardBlockRequest(shardRoutingTable, clusterBlock, new NotifyOnceListener<>() {
+                sendVerifyShardBlockRequest(shardRoutingTable, clusterBlock, ActionListener.notifyOnce(new ActionListener<>() {
                     @Override
-                    public void innerOnResponse(final ReplicationResponse replicationResponse) {
+                    public void onResponse(ReplicationResponse replicationResponse) {
                         AddBlockShardResult.Failure[] failures = Arrays.stream(replicationResponse.getShardInfo().getFailures())
                             .map(f -> new AddBlockShardResult.Failure(f.index(), f.shardId(), f.getCause(), f.nodeId()))
                             .toArray(AddBlockShardResult.Failure[]::new);
@@ -760,7 +761,7 @@ public void innerOnResponse(final ReplicationResponse replicationResponse) {
                     }
 
                     @Override
-                    public void innerOnFailure(final Exception e) {
+                    public void onFailure(Exception e) {
                         AddBlockShardResult.Failure failure = new AddBlockShardResult.Failure(index.getName(), shardId, e);
                         results.setOnce(shardId, new AddBlockShardResult(shardId, new AddBlockShardResult.Failure[] { failure }));
                         processIfFinished();
@@ -773,7 +774,7 @@ private void processIfFinished() {
                             onResponse.accept(result);
                         }
                     }
-                });
+                }));
             }
         }
 
@@ -809,11 +810,12 @@ private void sendVerifyShardBlockRequest(
     static Tuple<ClusterState, List<IndexResult>> closeRoutingTable(
         final ClusterState currentState,
         final Map<Index, ClusterBlock> blockedIndices,
-        final Map<Index, IndexResult> verifyResult
+        final Map<Index, IndexResult> verifyResult,
+        ShardRoutingRoleStrategy shardRoutingRoleStrategy
     ) {
         final Metadata.Builder metadata = Metadata.builder(currentState.metadata());
         final ClusterBlocks.Builder blocks = ClusterBlocks.builder(currentState.blocks());
-        final RoutingTable.Builder routingTable = RoutingTable.builder(currentState.routingTable());
+        final RoutingTable.Builder routingTable = RoutingTable.builder(shardRoutingRoleStrategy, currentState.routingTable());
 
         final Set<String> closedIndices = new HashSet<>();
         Map<Index, IndexResult> closingResults = new HashMap<>(verifyResult);
@@ -1151,7 +1153,10 @@ private ClusterState openIndices(final Index[] indices, final ClusterState curre
 
             ClusterState updatedState = ClusterState.builder(currentState).metadata(metadata).blocks(blocks).build();
 
-            final RoutingTable.Builder routingTable = RoutingTable.builder(updatedState.routingTable());
+            final RoutingTable.Builder routingTable = RoutingTable.builder(
+                allocationService.getShardRoutingRoleStrategy(),
+                updatedState.routingTable()
+            );
             for (IndexMetadata previousIndexMetadata : indicesToOpen) {
                 if (previousIndexMetadata.getState() != IndexMetadata.State.OPEN) {
                     routingTable.addAsFromCloseToOpen(updatedState.metadata().getIndexSafe(previousIndexMetadata.getIndex()));
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
index 2e5cdab353da..3dca87dcbde4 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
@@ -38,7 +38,6 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettingProvider;
 import org.elasticsearch.index.IndexSettingProviders;
@@ -309,14 +308,7 @@ public ClusterState addComponentTemplate(
                 final String composableTemplateName = entry.getKey();
                 final ComposableIndexTemplate composableTemplate = entry.getValue();
                 try {
-                    validateCompositeTemplate(
-                        tempStateWithComponentTemplateAdded,
-                        composableTemplateName,
-                        composableTemplate,
-                        indicesService,
-                        xContentRegistry,
-                        systemIndices
-                    );
+                    validateIndexTemplateV2(composableTemplateName, composableTemplate, tempStateWithComponentTemplateAdded);
                 } catch (Exception e) {
                     if (validationFailure == null) {
                         validationFailure = new IllegalArgumentException(
@@ -513,17 +505,34 @@ public static void validateV2TemplateRequest(Metadata metadata, String name, Com
         }
 
         final Map<String, ComponentTemplate> componentTemplates = metadata.componentTemplates();
+        final List<String> ignoreMissingComponentTemplates = (template.getIgnoreMissingComponentTemplates() == null
+            ? List.of()
+            : template.getIgnoreMissingComponentTemplates());
         final List<String> missingComponentTemplates = template.composedOf()
             .stream()
             .filter(componentTemplate -> componentTemplates.containsKey(componentTemplate) == false)
+            .filter(componentTemplate -> ignoreMissingComponentTemplates.contains(componentTemplate) == false)
             .toList();
 
-        if (missingComponentTemplates.size() > 0) {
+        if (missingComponentTemplates.size() > 0 && ignoreMissingComponentTemplates.size() == 0) {
             throw new InvalidIndexTemplateException(
                 name,
                 "index template [" + name + "] specifies component templates " + missingComponentTemplates + " that do not exist"
             );
         }
+
+        if (missingComponentTemplates.size() > 0 && ignoreMissingComponentTemplates.size() > 0) {
+
+            throw new InvalidIndexTemplateException(
+                name,
+                "index template ["
+                    + name
+                    + "] specifies a missing component templates "
+                    + missingComponentTemplates
+                    + " "
+                    + "that does not exist and is not part of 'ignore_missing_component_templates'"
+            );
+        }
     }
 
     public ClusterState addIndexTemplateV2(
@@ -587,7 +596,8 @@ public ClusterState addIndexTemplateV2(
                 template.version(),
                 template.metadata(),
                 template.getDataStreamTemplate(),
-                template.getAllowAutoCreate()
+                template.getAllowAutoCreate(),
+                template.getIgnoreMissingComponentTemplates()
             );
         }
 
@@ -687,12 +697,12 @@ private void validateIndexTemplateV2(String name, ComposableIndexTemplate indexT
             indexTemplate.version(),
             indexTemplate.metadata(),
             indexTemplate.getDataStreamTemplate(),
-            indexTemplate.getAllowAutoCreate()
+            indexTemplate.getAllowAutoCreate(),
+            indexTemplate.getIgnoreMissingComponentTemplates()
         );
 
         validate(name, templateToValidate);
         validateDataStreamsStillReferenced(currentState, name, templateToValidate);
-        validateTsdbDataStreamsReferringTsdbTemplate(currentState, name, templateToValidate);
 
         // Finally, right before adding the template, we need to ensure that the composite settings,
         // mappings, and aliases are valid after it's been composed with the component templates
@@ -765,56 +775,6 @@ private static void validateDataStreamsStillReferenced(ClusterState state, Strin
         }
     }
 
-    // This method should be invoked after validateDataStreamsStillReferenced(...)
-    private static void validateTsdbDataStreamsReferringTsdbTemplate(
-        ClusterState state,
-        String templateName,
-        ComposableIndexTemplate newTemplate
-    ) {
-        Metadata currentMetadata = state.getMetadata();
-        Metadata updatedMetadata = null;
-        Set<String> dataStreamsWithNonTsdbTemplate = null;
-
-        for (var dataStream : state.metadata().dataStreams().values()) {
-            if (dataStream.getIndexMode() != IndexMode.TIME_SERIES) {
-                continue;
-            }
-
-            if (updatedMetadata == null) {
-                updatedMetadata = Metadata.builder(state.metadata()).put(templateName, newTemplate).build();
-            }
-            var matchingTemplate = findV2Template(updatedMetadata, dataStream.getName(), false);
-            if (templateName.equals(matchingTemplate)) {
-                if (currentMetadata.isTimeSeriesTemplate(newTemplate) == false) {
-                    if (dataStreamsWithNonTsdbTemplate == null) {
-                        dataStreamsWithNonTsdbTemplate = new HashSet<>();
-                    }
-                    dataStreamsWithNonTsdbTemplate.add(dataStream.getName());
-                }
-            }
-        }
-
-        if (dataStreamsWithNonTsdbTemplate != null) {
-            var settings = MetadataIndexTemplateService.resolveSettings(newTemplate, currentMetadata.componentTemplates());
-            var routingPaths = IndexMetadata.INDEX_ROUTING_PATH.get(settings);
-            throw new IllegalArgumentException(
-                "composable template ["
-                    + templateName
-                    + "] with index patterns "
-                    + newTemplate.indexPatterns()
-                    + ", priority ["
-                    + newTemplate.priority()
-                    + "]"
-                    + ", index.routing_path "
-                    + routingPaths
-                    + " "
-                    + "would cause tsdb data streams "
-                    + dataStreamsWithNonTsdbTemplate
-                    + " to no longer match a data stream template with a time_series index_mode"
-            );
-        }
-    }
-
     /**
      * Return a map of v1 template names to their index patterns for v1 templates that would overlap
      * with the given v2 template's index patterns.
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java
index 0c138087e6e5..a85471d04ad7 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java
@@ -212,7 +212,10 @@ ClusterState execute(ClusterState currentState) {
                      *
                      * TODO: should we update the in-sync allocation IDs once the data is deleted by the node?
                      */
-                    routingTableBuilder = RoutingTable.builder(currentState.routingTable());
+                    routingTableBuilder = RoutingTable.builder(
+                        allocationService.getShardRoutingRoleStrategy(),
+                        currentState.routingTable()
+                    );
                     routingTableBuilder.updateNumberOfReplicas(updatedNumberOfReplicas, actualIndices);
                     metadataBuilder.updateNumberOfReplicas(updatedNumberOfReplicas, actualIndices);
                     logger.info("updating number_of_replicas to [{}] for indices {}", updatedNumberOfReplicas, actualIndices);
@@ -302,6 +305,11 @@ ClusterState execute(ClusterState currentState) {
 
             return updatedState;
         }
+
+        @Override
+        public String toString() {
+            return request.toString();
+        }
     }
 
     public void updateSettings(final UpdateSettingsClusterStateUpdateRequest request, final ActionListener<AcknowledgedResponse> listener) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java
index ab1f1a6812f5..2efbc1c372aa 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.cluster.metadata;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.DiffableUtils;
@@ -16,14 +16,16 @@
 import org.elasticsearch.cluster.SimpleDiffable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -39,7 +41,7 @@
  */
 public class NodesShutdownMetadata implements Metadata.Custom {
     public static final String TYPE = "node_shutdown";
-    public static final Version NODE_SHUTDOWN_VERSION = Version.V_7_13_0;
+    public static final TransportVersion NODE_SHUTDOWN_VERSION = TransportVersion.V_7_13_0;
     public static final NodesShutdownMetadata EMPTY = new NodesShutdownMetadata(Map.of());
 
     private static final ParseField NODES_FIELD = new ParseField("nodes");
@@ -149,7 +151,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
+    public TransportVersion getMinimalSupportedVersion() {
         return NODE_SHUTDOWN_VERSION;
     }
 
@@ -167,9 +169,8 @@ public int hashCode() {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(NODES_FIELD.getPreferredName(), nodes);
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return ChunkedToXContentHelper.xContentValuesMap(NODES_FIELD.getPreferredName(), nodes);
     }
 
     /**
@@ -213,7 +214,7 @@ static Diff<SingleNodeShutdownMetadata> readNodesDiffFrom(StreamInput in) throws
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
+        public TransportVersion getMinimalSupportedVersion() {
             return NODE_SHUTDOWN_VERSION;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetadata.java
index 720e57f7a194..69dc3f41ea71 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetadata.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.AbstractNamedDiffable;
 import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.Metadata.Custom;
@@ -27,6 +27,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.EnumSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.function.UnaryOperator;
 
@@ -166,8 +167,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.CURRENT.minimumCompatibilityVersion();
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.CURRENT.minimumCompatibilityVersion();
     }
 
     public RepositoriesMetadata(StreamInput in) throws IOException {
@@ -250,15 +251,11 @@ public static RepositoriesMetadata fromXContent(XContentParser parser) throws IO
         return new RepositoriesMetadata(repository);
     }
 
-    /**
-     * {@inheritDoc}
-     */
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
-        for (RepositoryMetadata repository : repositories) {
-            toXContent(repository, builder, params);
-        }
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return repositories.stream()
+            .map(repository -> (ToXContent) (builder, params) -> toXContent(repository, builder, params))
+            .iterator();
     }
 
     @Override
@@ -273,7 +270,8 @@ public EnumSet<Metadata.XContentContext> context() {
      * @param builder    XContent builder
      * @param params     serialization parameters
      */
-    public static void toXContent(RepositoryMetadata repository, XContentBuilder builder, ToXContent.Params params) throws IOException {
+    public static XContentBuilder toXContent(RepositoryMetadata repository, XContentBuilder builder, ToXContent.Params params)
+        throws IOException {
         builder.startObject(repository.name());
         builder.field("type", repository.type());
         if (repository.uuid().equals(RepositoryData.MISSING_UUID) == false) {
@@ -288,6 +286,7 @@ public static void toXContent(RepositoryMetadata repository, XContentBuilder bui
             builder.field("pending_generation", repository.pendingGeneration());
         }
         builder.endObject();
+        return builder;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetadata.java
index 7790e265220a..66e339a474ed 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetadata.java
@@ -129,7 +129,7 @@ public long pendingGeneration() {
 
     public RepositoryMetadata(StreamInput in) throws IOException {
         name = in.readString();
-        if (in.getVersion().onOrAfter(SnapshotsService.UUIDS_IN_REPO_DATA_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(SnapshotsService.UUIDS_IN_REPO_DATA_VERSION.transportVersion)) {
             uuid = in.readString();
         } else {
             uuid = RepositoryData.MISSING_UUID;
@@ -148,7 +148,7 @@ public RepositoryMetadata(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(name);
-        if (out.getVersion().onOrAfter(SnapshotsService.UUIDS_IN_REPO_DATA_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(SnapshotsService.UUIDS_IN_REPO_DATA_VERSION.transportVersion)) {
             out.writeString(uuid);
         }
         out.writeString(type);
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java
index dbde71aeef67..7f7f8a87cbfb 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.cluster.metadata;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -22,7 +22,7 @@
 import java.util.Objects;
 
 public class ShutdownShardMigrationStatus implements Writeable, ToXContentObject {
-    private static final Version ALLOCATION_DECISION_ADDED_VERSION = Version.V_7_16_0;
+    private static final TransportVersion ALLOCATION_DECISION_ADDED_VERSION = TransportVersion.V_7_16_0;
 
     public static final String NODE_ALLOCATION_DECISION_KEY = "node_allocation_decision";
 
@@ -57,7 +57,7 @@ public ShutdownShardMigrationStatus(StreamInput in) throws IOException {
         this.status = in.readEnum(SingleNodeShutdownMetadata.Status.class);
         this.shardsRemaining = in.readLong();
         this.explanation = in.readOptionalString();
-        if (in.getVersion().onOrAfter(ALLOCATION_DECISION_ADDED_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(ALLOCATION_DECISION_ADDED_VERSION)) {
             this.allocationDecision = in.readOptionalWriteable(ShardAllocationDecision::new);
         } else {
             this.allocationDecision = null;
@@ -100,7 +100,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeEnum(status);
         out.writeLong(shardsRemaining);
         out.writeOptionalString(explanation);
-        if (out.getVersion().onOrAfter(ALLOCATION_DECISION_ADDED_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(ALLOCATION_DECISION_ADDED_VERSION)) {
             out.writeOptionalWriteable(allocationDecision);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java
index 489389b24769..69500c2e9b90 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.cluster.metadata;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.SimpleDiffable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -33,7 +33,7 @@
  */
 public class SingleNodeShutdownMetadata implements SimpleDiffable<SingleNodeShutdownMetadata>, ToXContentObject {
 
-    public static final Version REPLACE_SHUTDOWN_TYPE_ADDED_VERSION = Version.V_7_16_0;
+    public static final TransportVersion REPLACE_SHUTDOWN_TYPE_ADDED_VERSION = TransportVersion.V_7_16_0;
 
     public static final ParseField NODE_ID_FIELD = new ParseField("node_id");
     public static final ParseField TYPE_FIELD = new ParseField("type");
@@ -133,7 +133,7 @@ public SingleNodeShutdownMetadata(StreamInput in) throws IOException {
         this.startedAtMillis = in.readVLong();
         this.nodeSeen = in.readBoolean();
         this.allocationDelay = in.readOptionalTimeValue();
-        if (in.getVersion().onOrAfter(REPLACE_SHUTDOWN_TYPE_ADDED_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(REPLACE_SHUTDOWN_TYPE_ADDED_VERSION)) {
             this.targetNodeName = in.readOptionalString();
         } else {
             this.targetNodeName = null;
@@ -199,7 +199,7 @@ public TimeValue getAllocationDelay() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(nodeId);
-        if (out.getVersion().before(REPLACE_SHUTDOWN_TYPE_ADDED_VERSION) && this.type == SingleNodeShutdownMetadata.Type.REPLACE) {
+        if (out.getTransportVersion().before(REPLACE_SHUTDOWN_TYPE_ADDED_VERSION) && this.type == SingleNodeShutdownMetadata.Type.REPLACE) {
             out.writeEnum(SingleNodeShutdownMetadata.Type.REMOVE);
         } else {
             out.writeEnum(type);
@@ -208,7 +208,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(startedAtMillis);
         out.writeBoolean(nodeSeen);
         out.writeOptionalTimeValue(allocationDelay);
-        if (out.getVersion().onOrAfter(REPLACE_SHUTDOWN_TYPE_ADDED_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(REPLACE_SHUTDOWN_TYPE_ADDED_VERSION)) {
             out.writeOptionalString(targetNodeName);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
index 73cf25fd3e8e..4db38a81c5f3 100644
--- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
+++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.cluster.node;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
@@ -63,7 +64,7 @@ public static boolean isStateless(final Settings settings) {
     }
 
     static final String COORDINATING_ONLY = "coordinating_only";
-    public static final Version EXTERNAL_ID_VERSION = Version.V_8_3_0;
+    public static final TransportVersion EXTERNAL_ID_VERSION = TransportVersion.V_8_3_0;
     public static final Comparator<DiscoveryNode> DISCOVERY_NODE_COMPARATOR = Comparator.comparing(DiscoveryNode::getName)
         .thenComparing(DiscoveryNode::getId);
 
@@ -138,7 +139,6 @@ public static boolean isDedicatedFrozenNode(final Settings settings) {
     private final Map<String, String> attributes;
     private final Version version;
     private final SortedSet<DiscoveryNodeRole> roles;
-
     private final Set<String> roleNames;
     private final String externalId;
 
@@ -412,7 +412,7 @@ public DiscoveryNode(StreamInput in) throws IOException {
         }
         this.roles = Collections.unmodifiableSortedSet(roles);
         this.version = Version.readVersion(in);
-        if (in.getVersion().onOrAfter(EXTERNAL_ID_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(EXTERNAL_ID_VERSION)) {
             this.externalId = readStringLiteral.read(in);
         } else {
             this.externalId = nodeName;
@@ -445,7 +445,7 @@ public void writeTo(StreamOutput out) throws IOException {
             o.writeBoolean(role.canContainData());
         });
         Version.writeVersion(version, out);
-        if (out.getVersion().onOrAfter(EXTERNAL_ID_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(EXTERNAL_ID_VERSION)) {
             out.writeString(externalId);
         }
     }
@@ -605,6 +605,7 @@ public void appendDescriptionWithoutAttributes(StringBuilder stringBuilder) {
             roles.stream().map(DiscoveryNodeRole::roleNameAbbreviation).sorted().forEach(stringBuilder::append);
             stringBuilder.append('}');
         }
+        stringBuilder.append('{').append(version).append('}');
     }
 
     public String descriptionWithoutAttributes() {
@@ -620,19 +621,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field("ephemeral_id", getEphemeralId());
         builder.field("transport_address", getAddress().toString());
         builder.field("external_id", getExternalId());
-
-        builder.startObject("attributes");
-        for (Map.Entry<String, String> entry : attributes.entrySet()) {
-            builder.field(entry.getKey(), entry.getValue());
-        }
-        builder.endObject();
-
+        builder.stringStringMap("attributes", attributes);
         builder.startArray("roles");
         for (DiscoveryNodeRole role : roles) {
             builder.value(role.roleName());
         }
         builder.endArray();
-
+        builder.field("version", version);
         builder.endObject();
         return builder;
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java
index 11440ffeca90..ceaa696f6e55 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java
@@ -36,6 +36,7 @@
 import java.util.Set;
 import java.util.function.IntConsumer;
 import java.util.function.IntSupplier;
+import java.util.function.Predicate;
 
 import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 
@@ -233,7 +234,7 @@ public void collectSearchShards(String routing, IntConsumer consumer) {
     }
 
     public static class ExtractFromSource extends IndexRouting {
-        private final List<String> routingPaths;
+        private final Predicate<String> isRoutingPath;
         private final XContentParserConfiguration parserConfig;
 
         ExtractFromSource(IndexMetadata metadata) {
@@ -241,7 +242,8 @@ public static class ExtractFromSource extends IndexRouting {
             if (metadata.isRoutingPartitionedIndex()) {
                 throw new IllegalArgumentException("routing_partition_size is incompatible with routing_path");
             }
-            this.routingPaths = metadata.getRoutingPaths();
+            List<String> routingPaths = metadata.getRoutingPaths();
+            isRoutingPath = Regex.simpleMatcher(routingPaths.toArray(String[]::new));
             this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.copyOf(routingPaths), null, true);
         }
 
@@ -262,7 +264,7 @@ public String createId(XContentType sourceType, BytesReference source, byte[] su
         public String createId(Map<String, Object> flat, byte[] suffix) {
             Builder b = builder();
             for (Map.Entry<String, Object> e : flat.entrySet()) {
-                if (Regex.simpleMatch(routingPaths, e.getKey())) {
+                if (isRoutingPath.test(e.getKey())) {
                     b.hashes.add(new NameAndHash(new BytesRef(e.getKey()), hash(new BytesRef(e.getValue().toString()))));
                 }
             }
@@ -299,7 +301,7 @@ public class Builder {
             private final List<NameAndHash> hashes = new ArrayList<>();
 
             public void addMatching(String fieldName, BytesRef string) {
-                if (Regex.simpleMatch(routingPaths, fieldName)) {
+                if (isRoutingPath.test(fieldName)) {
                     hashes.add(new NameAndHash(new BytesRef(fieldName), hash(string)));
                 }
             }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
index 8924a47b03bd..0c62dce1b220 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
@@ -32,6 +32,7 @@
 import java.util.List;
 import java.util.Set;
 import java.util.function.Predicate;
+import java.util.stream.Stream;
 
 /**
  * The {@link IndexRoutingTable} represents routing information for a single
@@ -50,11 +51,15 @@
  */
 public class IndexRoutingTable implements SimpleDiffable<IndexRoutingTable> {
 
-    private static final List<Predicate<ShardRouting>> PRIORITY_REMOVE_CLAUSES = List.of(
-        ShardRouting::unassigned,
-        ShardRouting::initializing,
+    private static final List<Predicate<ShardRouting>> PRIORITY_REMOVE_CLAUSES = Stream.<Predicate<ShardRouting>>of(
+        shardRouting -> shardRouting.isPromotableToPrimary() == false,
         shardRouting -> true
-    );
+    )
+        .flatMap(
+            p1 -> Stream.<Predicate<ShardRouting>>of(ShardRouting::unassigned, ShardRouting::initializing, shardRouting -> true)
+                .map(p1::and)
+        )
+        .toList();
     private final Index index;
     private final ShardShuffler shuffler;
 
@@ -131,7 +136,9 @@ boolean validate(Metadata metadata) {
                     );
                 }
                 final Set<String> inSyncAllocationIds = indexMetadata.inSyncAllocationIds(shardRouting.id());
-                if (shardRouting.active() && inSyncAllocationIds.contains(shardRouting.allocationId().getId()) == false) {
+                if (shardRouting.active()
+                    && shardRouting.isPromotableToPrimary()
+                    && inSyncAllocationIds.contains(shardRouting.allocationId().getId()) == false) {
                     throw new IllegalStateException(
                         "active shard routing "
                             + shardRouting
@@ -302,7 +309,7 @@ public int hashCode() {
 
     public static IndexRoutingTable readFrom(StreamInput in) throws IOException {
         Index index = new Index(in);
-        Builder builder = new Builder(index);
+        Builder builder = new Builder(ShardRoutingRoleStrategy.NO_SHARD_CREATION, index);
 
         int size = in.readVInt();
         builder.ensureShardArray(size);
@@ -324,15 +331,21 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     public static Builder builder(Index index) {
-        return new Builder(index);
+        return new Builder(ShardRoutingRoleStrategy.NO_SHARD_CREATION, index);
+    }
+
+    public static Builder builder(ShardRoutingRoleStrategy shardRoutingRoleStrategy, Index index) {
+        return new Builder(shardRoutingRoleStrategy, index);
     }
 
     public static class Builder {
 
+        private final ShardRoutingRoleStrategy shardRoutingRoleStrategy;
         private final Index index;
         private IndexShardRoutingTable.Builder[] shards;
 
-        public Builder(Index index) {
+        public Builder(ShardRoutingRoleStrategy shardRoutingRoleStrategy, Index index) {
+            this.shardRoutingRoleStrategy = shardRoutingRoleStrategy;
             this.index = index;
         }
 
@@ -438,7 +451,8 @@ private Builder initializeAsRestore(
                                 shardId,
                                 primary,
                                 primary ? EmptyStoreRecoverySource.INSTANCE : PeerRecoverySource.INSTANCE,
-                                unassignedInfo
+                                unassignedInfo,
+                                shardRoutingRoleStrategy.newRestoredRole(i)
                             )
                         );
                     } else {
@@ -447,7 +461,8 @@ private Builder initializeAsRestore(
                                 shardId,
                                 primary,
                                 primary ? recoverySource : PeerRecoverySource.INSTANCE,
-                                withLastAllocatedNodeId(unassignedInfo, previousNodes, i)
+                                withLastAllocatedNodeId(unassignedInfo, previousNodes, i),
+                                shardRoutingRoleStrategy.newRestoredRole(i)
                             )
                         );
                     }
@@ -493,7 +508,8 @@ private Builder initializeEmpty(
                             shardId,
                             primary,
                             primary ? primaryRecoverySource : PeerRecoverySource.INSTANCE,
-                            withLastAllocatedNodeId(unassignedInfo, previousNodes, i)
+                            withLastAllocatedNodeId(unassignedInfo, previousNodes, i),
+                            shardRoutingRoleStrategy.newEmptyRole(i)
                         )
                     );
                 }
@@ -546,7 +562,7 @@ private static UnassignedInfo withLastAllocatedNodeId(UnassignedInfo unassignedI
                 );
         }
 
-        public Builder addReplica() {
+        public Builder addReplica(ShardRouting.Role role) {
             assert shards != null;
             for (IndexShardRoutingTable.Builder existing : shards) {
                 assert existing != null;
@@ -556,7 +572,8 @@ public Builder addReplica() {
                         existing.shardId(),
                         false,
                         PeerRecoverySource.INSTANCE,
-                        new UnassignedInfo(UnassignedInfo.Reason.REPLICA_ADDED, null)
+                        new UnassignedInfo(UnassignedInfo.Reason.REPLICA_ADDED, null),
+                        role
                     )
                 );
             }
@@ -580,16 +597,12 @@ public Builder removeReplica() {
                     builder.addShard(shardRouting);
                 }
 
-                boolean removed = false;
-                for (Predicate<ShardRouting> removeClause : PRIORITY_REMOVE_CLAUSES) {
-                    if (removed == false) {
-                        for (int copy = 0; copy < indexShard.size(); copy++) {
-                            ShardRouting shardRouting = indexShard.shard(copy);
-                            if (shardRouting.primary() == false && removeClause.test(shardRouting)) {
-                                builder.removeShard(shardRouting);
-                                removed = true;
-                                break;
-                            }
+                findAndRemove: for (Predicate<ShardRouting> removeClause : PRIORITY_REMOVE_CLAUSES) {
+                    for (int copy = 0; copy < indexShard.size(); copy++) {
+                        ShardRouting shardRouting = indexShard.shard(copy);
+                        if (shardRouting.primary() == false && removeClause.test(shardRouting)) {
+                            builder.removeShard(shardRouting);
+                            break findAndRemove;
                         }
                     }
                 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
index 7cb809fc783e..3a5a369caa3f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
@@ -45,23 +45,23 @@ public class IndexShardRoutingTable {
 
     final ShardShuffler shuffler;
     final ShardId shardId;
-
+    final ShardRouting[] shards;
     final ShardRouting primary;
     final List<ShardRouting> replicas;
-    final ShardRouting[] shards;
     final List<ShardRouting> activeShards;
     final List<ShardRouting> assignedShards;
-    final boolean allShardsStarted;
-
     /**
      * The initializing list, including ones that are initializing on a target node because of relocation.
      * If we can come up with a better variable name, it would be nice...
      */
     final List<ShardRouting> allInitializingShards;
+    final boolean allShardsStarted;
+    final int activeSearchShardCount;
+    final int totalSearchShardCount;
 
     IndexShardRoutingTable(ShardId shardId, List<ShardRouting> shards) {
-        this.shardId = shardId;
         this.shuffler = new RotationShardShuffler(Randomness.get().nextInt());
+        this.shardId = shardId;
         this.shards = shards.toArray(ShardRouting[]::new);
 
         ShardRouting primary = null;
@@ -70,6 +70,8 @@ public class IndexShardRoutingTable {
         List<ShardRouting> assignedShards = new ArrayList<>();
         List<ShardRouting> allInitializingShards = new ArrayList<>();
         boolean allShardsStarted = true;
+        int activeSearchShardCount = 0;
+        int totalSearchShardCount = 0;
         for (ShardRouting shard : this.shards) {
             if (shard.primary()) {
                 assert primary == null : "duplicate primary: " + primary + " vs " + shard;
@@ -79,6 +81,12 @@ public class IndexShardRoutingTable {
             }
             if (shard.active()) {
                 activeShards.add(shard);
+                if (shard.role().isSearchable()) {
+                    activeSearchShardCount++;
+                }
+            }
+            if (shard.role().isSearchable()) {
+                totalSearchShardCount++;
             }
             if (shard.initializing()) {
                 allInitializingShards.add(shard);
@@ -97,12 +105,14 @@ public class IndexShardRoutingTable {
                 allShardsStarted = false;
             }
         }
-        this.allShardsStarted = allShardsStarted;
         this.primary = primary;
         this.replicas = CollectionUtils.wrapUnmodifiableOrEmptySingleton(replicas);
         this.activeShards = CollectionUtils.wrapUnmodifiableOrEmptySingleton(activeShards);
         this.assignedShards = CollectionUtils.wrapUnmodifiableOrEmptySingleton(assignedShards);
         this.allInitializingShards = CollectionUtils.wrapUnmodifiableOrEmptySingleton(allInitializingShards);
+        this.allShardsStarted = allShardsStarted;
+        this.activeSearchShardCount = activeSearchShardCount;
+        this.totalSearchShardCount = totalSearchShardCount;
     }
 
     /**
@@ -461,6 +471,24 @@ public boolean allShardsStarted() {
         return allShardsStarted;
     }
 
+    /**
+     * @return the count of active searchable shards
+     */
+    public int getActiveSearchShardCount() {
+        return activeSearchShardCount;
+    }
+
+    /**
+     * @return the total count of searchable shards
+     */
+    public int getTotalSearchShardCount() {
+        return totalSearchShardCount;
+    }
+
+    public boolean hasSearchShards() {
+        return totalSearchShardCount > 0;
+    }
+
     @Nullable
     public ShardRouting getByAllocationId(String allocationId) {
         for (ShardRouting shardRouting : assignedShards()) {
@@ -471,15 +499,17 @@ public ShardRouting getByAllocationId(String allocationId) {
         return null;
     }
 
-    public Set<String> getAllAllocationIds() {
+    public Set<String> getPromotableAllocationIds() {
         assert MasterService.assertNotMasterUpdateThread("not using this on the master thread so we don't have to pre-compute this");
         Set<String> allAllocationIds = new HashSet<>();
         for (ShardRouting shard : shards) {
-            if (shard.relocating()) {
-                allAllocationIds.add(shard.getTargetRelocatingShard().allocationId().getId());
-            }
-            if (shard.assignedToNode()) {
-                allAllocationIds.add(shard.allocationId().getId());
+            if (shard.isPromotableToPrimary()) {
+                if (shard.relocating()) {
+                    allAllocationIds.add(shard.getTargetRelocatingShard().allocationId().getId());
+                }
+                if (shard.assignedToNode()) {
+                    allAllocationIds.add(shard.allocationId().getId());
+                }
             }
         }
         return allAllocationIds;
@@ -559,6 +589,7 @@ public IndexShardRoutingTable build() {
             // don't allow more than one shard copy with same id to be allocated to same node
             assert distinctNodes(shards) : "more than one shard with same id assigned to same node (shards: " + shards + ")";
             assert noDuplicatePrimary(shards) : "expected but did not find unique primary in shard routing table: " + shards;
+            assert noAssignedReplicaWithoutActivePrimary(shards) : "unexpected assigned replica with no active primary: " + shards;
             return new IndexShardRoutingTable(shardId, shards);
         }
 
@@ -589,9 +620,24 @@ static boolean noDuplicatePrimary(List<ShardRouting> shards) {
                     seenPrimary = true;
                 }
             }
-            // We should be able to return seenPrimary here, but in tests there are many routing tables with no primary (e.g. empty) so for
-            // now we leniently allow there to be no primary as well. TODO fix those tests and stop being lenient here.
-            return true;
+            return seenPrimary;
+        }
+
+        static boolean noAssignedReplicaWithoutActivePrimary(List<ShardRouting> shards) {
+            boolean seenAssignedReplica = false;
+            for (final var shard : shards) {
+                if (shard.currentNodeId() != null) {
+                    if (shard.primary()) {
+                        if (shard.active()) {
+                            return true;
+                        }
+                    } else {
+                        seenAssignedReplica = true;
+                    }
+                }
+            }
+
+            return seenAssignedReplica == false;
         }
 
         public static IndexShardRoutingTable.Builder readFrom(StreamInput in) throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
index af6b27fef98f..53671458e0cf 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
@@ -20,11 +20,11 @@
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.function.Predicate;
+import java.util.stream.Stream;
 
 /**
  * A {@link RoutingNode} represents a cluster node associated with a single {@link DiscoveryNode} including all shards
@@ -209,55 +209,6 @@ void remove(ShardRouting shard) {
         assert invariant();
     }
 
-    /**
-     * Determine the number of shards with a specific state
-     * @param states set of states which should be counted
-     * @return number of shards
-     */
-    public int numberOfShardsWithState(ShardRoutingState... states) {
-        if (states.length == 1) {
-            if (states[0] == ShardRoutingState.INITIALIZING) {
-                return initializingShards.size();
-            } else if (states[0] == ShardRoutingState.RELOCATING) {
-                return relocatingShards.size();
-            } else if (states[0] == ShardRoutingState.STARTED) {
-                return startedShards.size();
-            }
-        }
-
-        int count = 0;
-        for (ShardRouting shardEntry : this) {
-            for (ShardRoutingState state : states) {
-                if (shardEntry.state() == state) {
-                    count++;
-                }
-            }
-        }
-        return count;
-    }
-
-    /**
-     * Determine the shards with a specific state
-     * @param state state which should be listed
-     * @return List of shards
-     */
-    public List<ShardRouting> shardsWithState(ShardRoutingState state) {
-        if (state == ShardRoutingState.INITIALIZING) {
-            return new ArrayList<>(initializingShards);
-        } else if (state == ShardRoutingState.RELOCATING) {
-            return new ArrayList<>(relocatingShards);
-        } else if (state == ShardRoutingState.STARTED) {
-            return new ArrayList<>(startedShards);
-        }
-        List<ShardRouting> shards = new ArrayList<>();
-        for (ShardRouting shardEntry : this) {
-            if (shardEntry.state() == state) {
-                shards.add(shardEntry);
-            }
-        }
-        return shards;
-    }
-
     private static final ShardRouting[] EMPTY_SHARD_ROUTING_ARRAY = new ShardRouting[0];
 
     public ShardRouting[] initializing() {
@@ -272,54 +223,45 @@ public ShardRouting[] started() {
         return startedShards.toArray(EMPTY_SHARD_ROUTING_ARRAY);
     }
 
+    /**
+     * Determine the number of shards with a specific state
+     * @param state which should be counted
+     * @return number of shards
+     */
+    public int numberOfShardsWithState(ShardRoutingState state) {
+        return internalGetShardsWithState(state).size();
+    }
+
+    /**
+     * Determine the shards with a specific state
+     * @param state state which should be listed
+     * @return List of shards
+     */
+    public Stream<ShardRouting> shardsWithState(ShardRoutingState state) {
+        return internalGetShardsWithState(state).stream();
+    }
+
     /**
      * Determine the shards of an index with a specific state
      * @param index id of the index
      * @param states set of states which should be listed
      * @return a list of shards
      */
-    public List<ShardRouting> shardsWithState(String index, ShardRoutingState... states) {
-        List<ShardRouting> shards = new ArrayList<>();
-
-        if (states.length == 1) {
-            if (states[0] == ShardRoutingState.INITIALIZING) {
-                for (ShardRouting shardEntry : initializingShards) {
-                    if (shardEntry.getIndexName().equals(index) == false) {
-                        continue;
-                    }
-                    shards.add(shardEntry);
-                }
-                return shards;
-            } else if (states[0] == ShardRoutingState.RELOCATING) {
-                for (ShardRouting shardEntry : relocatingShards) {
-                    if (shardEntry.getIndexName().equals(index) == false) {
-                        continue;
-                    }
-                    shards.add(shardEntry);
-                }
-                return shards;
-            } else if (states[0] == ShardRoutingState.STARTED) {
-                for (ShardRouting shardEntry : startedShards) {
-                    if (shardEntry.getIndexName().equals(index) == false) {
-                        continue;
-                    }
-                    shards.add(shardEntry);
-                }
-                return shards;
-            }
-        }
+    public Stream<ShardRouting> shardsWithState(String index, ShardRoutingState... states) {
+        return Stream.of(states).flatMap(state -> shardsWithState(index, state));
+    }
 
-        for (ShardRouting shardEntry : this) {
-            if (shardEntry.getIndexName().equals(index) == false) {
-                continue;
-            }
-            for (ShardRoutingState state : states) {
-                if (shardEntry.state() == state) {
-                    shards.add(shardEntry);
-                }
-            }
-        }
-        return shards;
+    public Stream<ShardRouting> shardsWithState(String index, ShardRoutingState state) {
+        return shardsWithState(state).filter(shardRouting -> Objects.equals(shardRouting.getIndexName(), index));
+    }
+
+    private LinkedHashSet<ShardRouting> internalGetShardsWithState(ShardRoutingState state) {
+        return switch (state) {
+            case UNASSIGNED -> throw new IllegalArgumentException("Unassigned shards are not linked to a routing node");
+            case INITIALIZING -> initializingShards;
+            case STARTED -> startedShards;
+            case RELOCATING -> relocatingShards;
+        };
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
index c5e90c2a0c11..5dda2a885a0b 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
@@ -345,14 +345,12 @@ public ShardRouting activePrimary(ShardId shardId) {
     }
 
     /**
-     * Returns one active replica shard for the given shard id or <code>null</code> if
-     * no active replica is found.
-     *
-     * Since replicas could possibly be on nodes with a older version of ES than
-     * the primary is, this will return replicas on the highest version of ES.
+     * Returns one active and promotable replica shard for the given shard id or <code>null</code> if no active replica is found.
      *
+     * Since replicas could possibly be on nodes with a older version of ES than the primary is, this will return replicas on the highest
+     * version of ES.
      */
-    public ShardRouting activeReplicaWithHighestVersion(ShardId shardId) {
+    public ShardRouting activePromotableReplicaWithHighestVersion(ShardId shardId) {
         // It's possible for replicaNodeVersion to be null, when disassociating dead nodes
         // that have been removed, the shards are failed, and part of the shard failing
         // calls this method with an out-of-date RoutingNodes, where the version might not
@@ -361,6 +359,7 @@ public ShardRouting activeReplicaWithHighestVersion(ShardId shardId) {
         return assignedShards(shardId).stream()
             .filter(shr -> shr.primary() == false && shr.active())
             .filter(shr -> node(shr.currentNodeId()) != null)
+            .filter(ShardRouting::isPromotableToPrimary)
             .max(
                 Comparator.comparing(
                     shr -> node(shr.currentNodeId()).node(),
@@ -655,17 +654,36 @@ assert node(failedShard.currentNodeId()).getByShardId(failedShard.shardId()) ==
     }
 
     private void unassignPrimaryAndPromoteActiveReplicaIfExists(
-        ShardRouting failedShard,
+        ShardRouting failedPrimary,
         UnassignedInfo unassignedInfo,
         RoutingChangesObserver routingChangesObserver
     ) {
-        assert failedShard.primary();
-        ShardRouting activeReplica = activeReplicaWithHighestVersion(failedShard.shardId());
-        if (activeReplica == null) {
-            moveToUnassigned(failedShard, unassignedInfo);
+        assert failedPrimary.primary();
+        ShardRouting replicaToPromote = activePromotableReplicaWithHighestVersion(failedPrimary.shardId());
+        if (replicaToPromote == null) {
+            moveToUnassigned(failedPrimary, unassignedInfo);
+            for (ShardRouting unpromotableReplica : List.copyOf(assignedShards(failedPrimary.shardId()))) {
+                assert unpromotableReplica.primary() == false : unpromotableReplica;
+                assert unpromotableReplica.isPromotableToPrimary() == false : unpromotableReplica;
+                moveToUnassigned(
+                    unpromotableReplica,
+                    new UnassignedInfo(
+                        UnassignedInfo.Reason.UNPROMOTABLE_REPLICA,
+                        unassignedInfo.getMessage(),
+                        unassignedInfo.getFailure(),
+                        0,
+                        unassignedInfo.getUnassignedTimeInNanos(),
+                        unassignedInfo.getUnassignedTimeInMillis(),
+                        false, // TODO debatable, but do we want to delay reassignment of unpromotable replicas tho?
+                        AllocationStatus.NO_ATTEMPT,
+                        Set.of(),
+                        unpromotableReplica.currentNodeId()
+                    )
+                );
+            }
         } else {
-            movePrimaryToUnassignedAndDemoteToReplica(failedShard, unassignedInfo);
-            promoteReplicaToPrimary(activeReplica, routingChangesObserver);
+            movePrimaryToUnassignedAndDemoteToReplica(failedPrimary, unassignedInfo);
+            promoteReplicaToPrimary(replicaToPromote, routingChangesObserver);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java
index 23d1528f72db..5596691fa3a3 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java
@@ -441,21 +441,40 @@ public static Builder builder(RoutingTable routingTable) {
         return new Builder(routingTable);
     }
 
+    public static Builder builder(ShardRoutingRoleStrategy shardRoutingRoleStrategy) {
+        return new Builder(shardRoutingRoleStrategy);
+    }
+
+    public static Builder builder(ShardRoutingRoleStrategy shardRoutingRoleStrategy, RoutingTable routingTable) {
+        return new Builder(shardRoutingRoleStrategy, routingTable);
+    }
+
     /**
      * Builder for the routing table. Note that build can only be called one time.
      */
     public static class Builder {
 
+        private final ShardRoutingRoleStrategy shardRoutingRoleStrategy;
         private long version;
         private ImmutableOpenMap.Builder<String, IndexRoutingTable> indicesRouting;
 
         public Builder() {
-            indicesRouting = ImmutableOpenMap.builder();
+            this(ShardRoutingRoleStrategy.NO_SHARD_CREATION);
         }
 
         public Builder(RoutingTable routingTable) {
-            version = routingTable.version;
-            indicesRouting = ImmutableOpenMap.builder(routingTable.indicesRouting);
+            this(ShardRoutingRoleStrategy.NO_SHARD_CREATION, routingTable);
+        }
+
+        public Builder(ShardRoutingRoleStrategy shardRoutingRoleStrategy) {
+            this.shardRoutingRoleStrategy = shardRoutingRoleStrategy;
+            this.indicesRouting = ImmutableOpenMap.builder();
+        }
+
+        public Builder(ShardRoutingRoleStrategy shardRoutingRoleStrategy, RoutingTable routingTable) {
+            this.shardRoutingRoleStrategy = shardRoutingRoleStrategy;
+            this.version = routingTable.version;
+            this.indicesRouting = ImmutableOpenMap.builder(routingTable.indicesRouting);
         }
 
         private static void addShard(
@@ -463,7 +482,7 @@ private static void addShard(
             final ShardRouting shardRoutingEntry
         ) {
             Index index = shardRoutingEntry.index();
-            indexRoutingTableBuilders.computeIfAbsent(index.getName(), idxName -> new IndexRoutingTable.Builder(index))
+            indexRoutingTableBuilders.computeIfAbsent(index.getName(), idxName -> IndexRoutingTable.builder(index))
                 .addShard(shardRoutingEntry);
         }
 
@@ -485,7 +504,7 @@ public Builder updateNumberOfReplicas(final int numberOfReplicas, final String[]
                     continue;
                 }
                 int currentNumberOfReplicas = indexRoutingTable.shard(0).size() - 1; // remove the required primary
-                IndexRoutingTable.Builder builder = new IndexRoutingTable.Builder(indexRoutingTable.getIndex());
+                IndexRoutingTable.Builder builder = new IndexRoutingTable.Builder(shardRoutingRoleStrategy, indexRoutingTable.getIndex());
                 // re-add all the shards
                 builder.ensureShardArray(indexRoutingTable.size());
                 for (int i = 0; i < indexRoutingTable.size(); i++) {
@@ -494,7 +513,7 @@ public Builder updateNumberOfReplicas(final int numberOfReplicas, final String[]
                 if (currentNumberOfReplicas < numberOfReplicas) {
                     // now, add "empty" ones
                     for (int i = 0; i < (numberOfReplicas - currentNumberOfReplicas); i++) {
-                        builder.addReplica();
+                        builder.addReplica(shardRoutingRoleStrategy.newReplicaRole());
                     }
                 } else if (currentNumberOfReplicas > numberOfReplicas) {
                     for (int i = 0; i < (currentNumberOfReplicas - numberOfReplicas); i++) {
@@ -508,9 +527,10 @@ public Builder updateNumberOfReplicas(final int numberOfReplicas, final String[]
 
         public Builder addAsNew(IndexMetadata indexMetadata) {
             if (indexMetadata.getState() == IndexMetadata.State.OPEN) {
-                IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetadata.getIndex()).initializeAsNew(
-                    indexMetadata
-                );
+                IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(
+                    shardRoutingRoleStrategy,
+                    indexMetadata.getIndex()
+                ).initializeAsNew(indexMetadata);
                 add(indexRoutingBuilder);
             }
             return this;
@@ -518,8 +538,10 @@ public Builder addAsNew(IndexMetadata indexMetadata) {
 
         public Builder addAsRecovery(IndexMetadata indexMetadata) {
             if (indexMetadata.getState() == IndexMetadata.State.OPEN || isIndexVerifiedBeforeClosed(indexMetadata)) {
-                IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetadata.getIndex())
-                    .initializeAsRecovery(indexMetadata);
+                IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(
+                    shardRoutingRoleStrategy,
+                    indexMetadata.getIndex()
+                ).initializeAsRecovery(indexMetadata);
                 add(indexRoutingBuilder);
             }
             return this;
@@ -527,8 +549,10 @@ public Builder addAsRecovery(IndexMetadata indexMetadata) {
 
         public Builder addAsFromDangling(IndexMetadata indexMetadata) {
             if (indexMetadata.getState() == IndexMetadata.State.OPEN || isIndexVerifiedBeforeClosed(indexMetadata)) {
-                IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetadata.getIndex())
-                    .initializeAsFromDangling(indexMetadata);
+                IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(
+                    shardRoutingRoleStrategy,
+                    indexMetadata.getIndex()
+                ).initializeAsFromDangling(indexMetadata);
                 add(indexRoutingBuilder);
             }
             return this;
@@ -536,8 +560,10 @@ public Builder addAsFromDangling(IndexMetadata indexMetadata) {
 
         public Builder addAsFromCloseToOpen(IndexMetadata indexMetadata) {
             if (indexMetadata.getState() == IndexMetadata.State.OPEN) {
-                IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetadata.getIndex())
-                    .initializeAsFromCloseToOpen(indexMetadata, indicesRouting.get(indexMetadata.getIndex().getName()));
+                IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(
+                    shardRoutingRoleStrategy,
+                    indexMetadata.getIndex()
+                ).initializeAsFromCloseToOpen(indexMetadata, indicesRouting.get(indexMetadata.getIndex().getName()));
                 add(indexRoutingBuilder);
             }
             return this;
@@ -545,27 +571,27 @@ public Builder addAsFromCloseToOpen(IndexMetadata indexMetadata) {
 
         public Builder addAsFromOpenToClose(IndexMetadata indexMetadata) {
             assert isIndexVerifiedBeforeClosed(indexMetadata);
-            IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetadata.getIndex())
-                .initializeAsFromOpenToClose(indexMetadata, indicesRouting.get(indexMetadata.getIndex().getName()));
+            IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(
+                shardRoutingRoleStrategy,
+                indexMetadata.getIndex()
+            ).initializeAsFromOpenToClose(indexMetadata, indicesRouting.get(indexMetadata.getIndex().getName()));
             return add(indexRoutingBuilder);
         }
 
         public Builder addAsRestore(IndexMetadata indexMetadata, SnapshotRecoverySource recoverySource) {
-            IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetadata.getIndex()).initializeAsRestore(
-                indexMetadata,
-                recoverySource,
-                indicesRouting.get(indexMetadata.getIndex().getName())
-            );
+            IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(
+                shardRoutingRoleStrategy,
+                indexMetadata.getIndex()
+            ).initializeAsRestore(indexMetadata, recoverySource, indicesRouting.get(indexMetadata.getIndex().getName()));
             add(indexRoutingBuilder);
             return this;
         }
 
         public Builder addAsNewRestore(IndexMetadata indexMetadata, SnapshotRecoverySource recoverySource, Set<Integer> ignoreShards) {
-            IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetadata.getIndex()).initializeAsNewRestore(
-                indexMetadata,
-                recoverySource,
-                ignoreShards
-            );
+            IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(
+                shardRoutingRoleStrategy,
+                indexMetadata.getIndex()
+            ).initializeAsNewRestore(indexMetadata, recoverySource, ignoreShards);
             add(indexRoutingBuilder);
             return this;
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
index ac6dd661f046..37ae9784d9cf 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
@@ -8,17 +8,19 @@
 
 package org.elasticsearch.cluster.routing;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RecoverySource.ExistingStoreRecoverySource;
 import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -36,8 +38,8 @@ public final class ShardRouting implements Writeable, ToXContentObject {
      * Used if shard size is not available
      */
     public static final long UNAVAILABLE_EXPECTED_SHARD_SIZE = -1;
-    private static final Version EXPECTED_SHARD_SIZE_FOR_STARTED_VERSION = Version.V_8_5_0;
-    private static final Version RELOCATION_FAILURE_INFO_VERSION = Version.V_8_6_0;
+    private static final TransportVersion EXPECTED_SHARD_SIZE_FOR_STARTED_VERSION = TransportVersion.V_8_5_0;
+    private static final TransportVersion RELOCATION_FAILURE_INFO_VERSION = TransportVersion.V_8_6_0;
 
     private final ShardId shardId;
     private final String currentNodeId;
@@ -60,6 +62,7 @@ public final class ShardRouting implements Writeable, ToXContentObject {
     private final long expectedShardSize;
     @Nullable
     private final ShardRouting targetRelocatingShard;
+    private final Role role;
 
     /**
      * A constructor to internally create shard routing instances, note, the internal flag should only be set to true
@@ -75,7 +78,8 @@ public final class ShardRouting implements Writeable, ToXContentObject {
         UnassignedInfo unassignedInfo,
         RelocationFailureInfo relocationFailureInfo,
         AllocationId allocationId,
-        long expectedShardSize
+        long expectedShardSize,
+        Role role
     ) {
         this.shardId = shardId;
         this.currentNodeId = currentNodeId;
@@ -87,17 +91,45 @@ public final class ShardRouting implements Writeable, ToXContentObject {
         this.relocationFailureInfo = relocationFailureInfo;
         this.allocationId = allocationId;
         this.expectedShardSize = expectedShardSize;
+        this.role = role;
         this.targetRelocatingShard = initializeTargetRelocatingShard();
-        assert (state == ShardRoutingState.UNASSIGNED && unassignedInfo == null) == false : "unassigned shard must be created with meta";
+
+        assert assertConsistent();
+    }
+
+    private boolean assertConsistent() {
         assert relocationFailureInfo != null : "relocation failure info must be always set";
-        assert (state == ShardRoutingState.UNASSIGNED || state == ShardRoutingState.INITIALIZING) == (recoverySource != null)
-            : "recovery source only available on unassigned or initializing shard but was " + state;
-        assert recoverySource == null || recoverySource == PeerRecoverySource.INSTANCE || primary
-            : "replica shards always recover from primary";
-        assert (currentNodeId == null) == (state == ShardRoutingState.UNASSIGNED)
-            : "unassigned shard must not be assigned to a node " + this;
-        assert relocatingNodeId == null || state == ShardRoutingState.RELOCATING || state == ShardRoutingState.INITIALIZING
-            : state + " shard must not have relocating node " + this;
+        assert role != null : "role must be always set";
+        assert primary == false || role.isPromotableToPrimary() : "shard with unpromotable role was promoted to primary: " + this;
+        switch (state) {
+            case UNASSIGNED -> {
+                assert currentNodeId == null : state + " shard must not be assigned to a node " + this;
+                assert relocatingNodeId == null : state + " shard must not be relocating to a node " + this;
+                assert unassignedInfo != null : state + " shard must be created with unassigned info " + this;
+                assert recoverySource != null : state + " shard must be created with a recovery source" + this;
+                assert primary ^ recoverySource == PeerRecoverySource.INSTANCE : "replica shards always recover from primary" + this;
+            }
+            case INITIALIZING -> {
+                assert currentNodeId != null : state + " shard must be assigned to a node " + this;
+                // relocatingNodeId is not set for initializing shard but set for relocating shard counterpart
+                // unassignedInfo is kept after starting unassigned shard but not present for relocating shard counterpart
+                assert recoverySource != null : state + "shard must be created with a recovery source" + this;
+                assert primary || recoverySource == PeerRecoverySource.INSTANCE : "replica shards always recover from primary" + this;
+            }
+            case STARTED -> {
+                assert currentNodeId != null : state + " shard must be assigned to a node " + this;
+                assert relocatingNodeId == null : state + " shard must not be relocating to a node " + this;
+                assert unassignedInfo == null : state + " shard must be created without unassigned info " + this;
+                assert recoverySource == null : state + " shard must be created without a recovery source" + this;
+            }
+            case RELOCATING -> {
+                assert currentNodeId != null : state + " shard must be assigned to a node " + this;
+                assert relocatingNodeId != null : state + " shard must be relocating to a node " + this;
+                assert unassignedInfo == null : state + " shard must be created without unassigned info " + this;
+                assert recoverySource == null : state + " shard must be created without a recovery source" + this;
+            }
+        }
+        return true;
     }
 
     @Nullable
@@ -113,7 +145,8 @@ private ShardRouting initializeTargetRelocatingShard() {
                 unassignedInfo,
                 RelocationFailureInfo.NO_FAILURES,
                 AllocationId.newTargetRelocation(allocationId),
-                expectedShardSize
+                expectedShardSize,
+                role
             );
         } else {
             return null;
@@ -127,7 +160,8 @@ public static ShardRouting newUnassigned(
         ShardId shardId,
         boolean primary,
         RecoverySource recoverySource,
-        UnassignedInfo unassignedInfo
+        UnassignedInfo unassignedInfo,
+        Role role
     ) {
         return new ShardRouting(
             shardId,
@@ -139,7 +173,8 @@ public static ShardRouting newUnassigned(
             unassignedInfo,
             RelocationFailureInfo.NO_FAILURES,
             null,
-            UNAVAILABLE_EXPECTED_SHARD_SIZE
+            UNAVAILABLE_EXPECTED_SHARD_SIZE,
+            role
         );
     }
 
@@ -304,7 +339,7 @@ public ShardRouting(ShardId shardId, StreamInput in) throws IOException {
             recoverySource = null;
         }
         unassignedInfo = in.readOptionalWriteable(UnassignedInfo::new);
-        if (in.getVersion().onOrAfter(RELOCATION_FAILURE_INFO_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(RELOCATION_FAILURE_INFO_VERSION)) {
             relocationFailureInfo = RelocationFailureInfo.readFrom(in);
         } else {
             relocationFailureInfo = RelocationFailureInfo.NO_FAILURES;
@@ -312,11 +347,16 @@ public ShardRouting(ShardId shardId, StreamInput in) throws IOException {
         allocationId = in.readOptionalWriteable(AllocationId::new);
         if (state == ShardRoutingState.RELOCATING
             || state == ShardRoutingState.INITIALIZING
-            || (state == ShardRoutingState.STARTED && in.getVersion().onOrAfter(EXPECTED_SHARD_SIZE_FOR_STARTED_VERSION))) {
+            || (state == ShardRoutingState.STARTED && in.getTransportVersion().onOrAfter(EXPECTED_SHARD_SIZE_FOR_STARTED_VERSION))) {
             expectedShardSize = in.readLong();
         } else {
             expectedShardSize = UNAVAILABLE_EXPECTED_SHARD_SIZE;
         }
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            role = Role.readFrom(in);
+        } else {
+            role = Role.DEFAULT;
+        }
         targetRelocatingShard = initializeTargetRelocatingShard();
     }
 
@@ -339,15 +379,23 @@ public void writeToThin(StreamOutput out) throws IOException {
             recoverySource.writeTo(out);
         }
         out.writeOptionalWriteable(unassignedInfo);
-        if (out.getVersion().onOrAfter(RELOCATION_FAILURE_INFO_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(RELOCATION_FAILURE_INFO_VERSION)) {
             relocationFailureInfo.writeTo(out);
         }
         out.writeOptionalWriteable(allocationId);
         if (state == ShardRoutingState.RELOCATING
             || state == ShardRoutingState.INITIALIZING
-            || (state == ShardRoutingState.STARTED && out.getVersion().onOrAfter(EXPECTED_SHARD_SIZE_FOR_STARTED_VERSION))) {
+            || (state == ShardRoutingState.STARTED && out.getTransportVersion().onOrAfter(EXPECTED_SHARD_SIZE_FOR_STARTED_VERSION))) {
             out.writeLong(expectedShardSize);
         }
+
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            role.writeTo(out);
+        } else if (role != Role.DEFAULT) {
+            throw new IllegalStateException(
+                Strings.format("cannot send role [%s] with transport version [%s]", role, out.getTransportVersion())
+            );
+        }
     }
 
     @Override
@@ -369,7 +417,8 @@ public ShardRouting updateUnassigned(UnassignedInfo unassignedInfo, RecoverySour
             unassignedInfo,
             relocationFailureInfo,
             allocationId,
-            expectedShardSize
+            expectedShardSize,
+            role
         );
     }
 
@@ -385,7 +434,8 @@ public ShardRouting updateRelocationFailure(RelocationFailureInfo relocationFail
             unassignedInfo,
             relocationFailureInfo,
             allocationId,
-            expectedShardSize
+            expectedShardSize,
+            role
         );
     }
 
@@ -414,7 +464,8 @@ public ShardRouting moveToUnassigned(UnassignedInfo unassignedInfo) {
             unassignedInfo,
             RelocationFailureInfo.NO_FAILURES,
             null,
-            UNAVAILABLE_EXPECTED_SHARD_SIZE
+            UNAVAILABLE_EXPECTED_SHARD_SIZE,
+            role
         );
     }
 
@@ -442,7 +493,8 @@ public ShardRouting initialize(String nodeId, @Nullable String existingAllocatio
             unassignedInfo,
             RelocationFailureInfo.NO_FAILURES,
             allocationId,
-            expectedShardSize
+            expectedShardSize,
+            role
         );
     }
 
@@ -463,7 +515,8 @@ public ShardRouting relocate(String relocatingNodeId, long expectedShardSize) {
             null,
             relocationFailureInfo,
             AllocationId.newRelocation(allocationId),
-            expectedShardSize
+            expectedShardSize,
+            role
         );
     }
 
@@ -485,7 +538,8 @@ public ShardRouting cancelRelocation() {
             null,
             relocationFailureInfo.incFailedRelocations(),
             AllocationId.cancelRelocation(allocationId),
-            UNAVAILABLE_EXPECTED_SHARD_SIZE
+            UNAVAILABLE_EXPECTED_SHARD_SIZE,
+            role
         );
     }
 
@@ -509,7 +563,8 @@ public ShardRouting removeRelocationSource() {
             unassignedInfo,
             relocationFailureInfo,
             AllocationId.finishRelocation(allocationId),
-            expectedShardSize
+            expectedShardSize,
+            role
         );
     }
 
@@ -530,7 +585,8 @@ public ShardRouting reinitializeReplicaShard() {
             unassignedInfo,
             relocationFailureInfo,
             AllocationId.newInitializing(),
-            expectedShardSize
+            expectedShardSize,
+            role
         );
     }
 
@@ -557,7 +613,8 @@ public ShardRouting moveToStarted(long expectedShardSize) {
             null,
             RelocationFailureInfo.NO_FAILURES,
             allocationId,
-            expectedShardSize
+            expectedShardSize,
+            role
         );
     }
 
@@ -581,7 +638,8 @@ public ShardRouting moveActiveReplicaToPrimary() {
             unassignedInfo,
             relocationFailureInfo,
             allocationId,
-            expectedShardSize
+            expectedShardSize,
+            role
         );
     }
 
@@ -605,7 +663,8 @@ public ShardRouting moveUnassignedFromPrimary() {
             unassignedInfo,
             relocationFailureInfo,
             allocationId,
-            expectedShardSize
+            expectedShardSize,
+            role
         );
     }
 
@@ -734,7 +793,8 @@ public boolean equalsIgnoringMetadata(ShardRouting other) {
             && Objects.equals(relocatingNodeId, other.relocatingNodeId)
             && Objects.equals(allocationId, other.allocationId)
             && state == other.state
-            && Objects.equals(recoverySource, other.recoverySource);
+            && Objects.equals(recoverySource, other.recoverySource)
+            && role == other.role;
     }
 
     @Override
@@ -770,6 +830,7 @@ public int hashCode() {
             h = 31 * h + (allocationId != null ? allocationId.hashCode() : 0);
             h = 31 * h + (unassignedInfo != null ? unassignedInfo.hashCode() : 0);
             h = 31 * h + (relocationFailureInfo != null ? relocationFailureInfo.hashCode() : 0);
+            h = 31 * h + role.hashCode();
             hashCode = h;
         }
         return h;
@@ -790,6 +851,9 @@ public String shortSummary() {
         if (relocatingNodeId != null) {
             sb.append("relocating [").append(relocatingNodeId).append("], ");
         }
+        if (role != Role.DEFAULT) {
+            sb.append("[").append(role).append("], ");
+        }
         if (primary) {
             sb.append("[P]");
         } else {
@@ -835,6 +899,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             unassignedInfo.toXContent(builder, params);
         }
         relocationFailureInfo.toXContent(builder, params);
+        role.toXContent(builder, params);
         return builder.endObject();
     }
 
@@ -855,4 +920,66 @@ public long getExpectedShardSize() {
     public RecoverySource recoverySource() {
         return recoverySource;
     }
+
+    public Role role() {
+        return role;
+    }
+
+    public boolean isPromotableToPrimary() {
+        return role.isPromotableToPrimary();
+    }
+
+    public boolean isSearchable() {
+        return role.isSearchable();
+    }
+
+    public enum Role implements Writeable, ToXContentFragment {
+        DEFAULT((byte) 0, true, true),
+        INDEX_ONLY((byte) 1, true, false),
+        SEARCH_ONLY((byte) 2, false, true);
+
+        private final byte code;
+        private final boolean promotable;
+        private final boolean searchable;
+
+        Role(byte code, boolean promotable, boolean searchable) {
+            this.code = code;
+            this.promotable = promotable;
+            this.searchable = searchable;
+        }
+
+        /**
+         * @return whether a shard copy with this role may be promoted from replica to primary. If {@code index.number_of_replicas} is
+         * reduced, unpromotable replicas are removed first.
+         */
+        public boolean isPromotableToPrimary() {
+            return promotable;
+        }
+
+        /**
+         * @return whether a shard copy with this role may be the target of a search.
+         */
+        public boolean isSearchable() {
+            return searchable;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return this == DEFAULT ? builder : builder.field("role", toString());
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeByte(code);
+        }
+
+        public static Role readFrom(StreamInput in) throws IOException {
+            return switch (in.readByte()) {
+                case 0 -> DEFAULT;
+                case 1 -> INDEX_ONLY;
+                case 2 -> SEARCH_ONLY;
+                default -> throw new IllegalStateException("unknown role");
+            };
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/ShardRoutingRoleStrategy.java b/server/src/main/java/org/elasticsearch/cluster/routing/ShardRoutingRoleStrategy.java
new file mode 100644
index 000000000000..e136333e7e39
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/ShardRoutingRoleStrategy.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.cluster.routing;
+
+public interface ShardRoutingRoleStrategy {
+
+    /**
+     * @return the role for a copy of a new empty shard, where {@code copyIndex} is the index of the copy ({@code 0} for the primary and
+     * {@code 1..N} for replicas).
+     */
+    ShardRouting.Role newEmptyRole(int copyIndex);
+
+    /**
+     * @return the role for a new replica copy of an existing shard.
+     */
+    ShardRouting.Role newReplicaRole();
+
+    /**
+     * @return the role for a copy of a new shard being restored from snapshot, where {@code copyIndex} is the index of the copy ({@code 0}
+     * for the primary and {@code 1..N} for replicas).
+     */
+    default ShardRouting.Role newRestoredRole(int copyIndex) {
+        return newEmptyRole(copyIndex);
+    }
+
+    /**
+     * A strategy that refuses to create any new shard copies, which is used (for instance) when reading shard copies from a remote node.
+     */
+    ShardRoutingRoleStrategy NO_SHARD_CREATION = new ShardRoutingRoleStrategy() {
+        @Override
+        public ShardRouting.Role newEmptyRole(int copyIndex) {
+            return newReplicaRole();
+        }
+
+        @Override
+        public ShardRouting.Role newReplicaRole() {
+            assert false : "no shard creation permitted";
+            throw new IllegalStateException("no shard creation permitted");
+        }
+    };
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java b/server/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
index 95e442b14055..f20ef4134243 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.cluster.routing;
 
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
@@ -47,7 +47,8 @@ public final class UnassignedInfo implements ToXContentFragment, Writeable {
      * The version that the {@code lastAllocatedNode} field was added in. Used to adapt streaming of this class as appropriate for the
      * version of the node sending/receiving it. Should be removed once wire compatibility with this version is no longer necessary.
      */
-    private static final Version VERSION_LAST_ALLOCATED_NODE_ADDED = Version.V_7_15_0;
+    private static final TransportVersion VERSION_LAST_ALLOCATED_NODE_ADDED = TransportVersion.V_7_15_0;
+    private static final TransportVersion VERSION_UNPROMOTABLE_REPLICA_ADDED = TransportVersion.V_8_7_0;
 
     public static final DateFormatter DATE_TIME_FORMATTER = DateFormatter.forPattern("date_optional_time").withZone(ZoneOffset.UTC);
 
@@ -133,7 +134,11 @@ public enum Reason {
          * Similar to NODE_LEFT, but at the time the node left, it had been registered for a restart via the Node Shutdown API. Note that
          * there is no verification that it was ready to be restarted, so this may be an intentional restart or a node crash.
          */
-        NODE_RESTARTING
+        NODE_RESTARTING,
+        /**
+         * Replica is unpromotable and the primary failed.
+         */
+        UNPROMOTABLE_REPLICA
     }
 
     /**
@@ -297,7 +302,7 @@ public UnassignedInfo(StreamInput in) throws IOException {
         this.failedAllocations = in.readVInt();
         this.lastAllocationStatus = AllocationStatus.readFrom(in);
         this.failedNodeIds = Collections.unmodifiableSet(in.readSet(StreamInput::readString));
-        if (in.getVersion().onOrAfter(VERSION_LAST_ALLOCATED_NODE_ADDED)) {
+        if (in.getTransportVersion().onOrAfter(VERSION_LAST_ALLOCATED_NODE_ADDED)) {
             this.lastAllocatedNodeId = in.readOptionalString();
         } else {
             this.lastAllocatedNodeId = null;
@@ -305,8 +310,10 @@ public UnassignedInfo(StreamInput in) throws IOException {
     }
 
     public void writeTo(StreamOutput out) throws IOException {
-        if (reason.equals(Reason.NODE_RESTARTING) && out.getVersion().before(VERSION_LAST_ALLOCATED_NODE_ADDED)) {
+        if (reason.equals(Reason.NODE_RESTARTING) && out.getTransportVersion().before(VERSION_LAST_ALLOCATED_NODE_ADDED)) {
             out.writeByte((byte) Reason.NODE_LEFT.ordinal());
+        } else if (reason.equals(Reason.UNPROMOTABLE_REPLICA) && out.getTransportVersion().before(VERSION_UNPROMOTABLE_REPLICA_ADDED)) {
+            out.writeByte((byte) Reason.PRIMARY_FAILED.ordinal());
         } else {
             out.writeByte((byte) reason.ordinal());
         }
@@ -318,7 +325,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVInt(failedAllocations);
         lastAllocationStatus.writeTo(out);
         out.writeCollection(failedNodeIds, StreamOutput::writeString);
-        if (out.getVersion().onOrAfter(VERSION_LAST_ALLOCATED_NODE_ADDED)) {
+        if (out.getTransportVersion().onOrAfter(VERSION_LAST_ALLOCATED_NODE_ADDED)) {
             out.writeOptionalString(lastAllocatedNodeId);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
index 6038bb25ca16..b9431045172e 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingRoleStrategy;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
@@ -76,6 +77,7 @@ public class AllocationService {
     private final ShardsAllocator shardsAllocator;
     private final ClusterInfoService clusterInfoService;
     private final SnapshotsInfoService snapshotsInfoService;
+    private final ShardRoutingRoleStrategy shardRoutingRoleStrategy;
 
     // only for tests that use the GatewayAllocator as the unique ExistingShardsAllocator
     public AllocationService(
@@ -83,9 +85,10 @@ public AllocationService(
         GatewayAllocator gatewayAllocator,
         ShardsAllocator shardsAllocator,
         ClusterInfoService clusterInfoService,
-        SnapshotsInfoService snapshotsInfoService
+        SnapshotsInfoService snapshotsInfoService,
+        ShardRoutingRoleStrategy shardRoutingRoleStrategy
     ) {
-        this(allocationDeciders, shardsAllocator, clusterInfoService, snapshotsInfoService);
+        this(allocationDeciders, shardsAllocator, clusterInfoService, snapshotsInfoService, shardRoutingRoleStrategy);
         setExistingShardsAllocators(Collections.singletonMap(GatewayAllocator.ALLOCATOR_NAME, gatewayAllocator));
     }
 
@@ -93,12 +96,14 @@ public AllocationService(
         AllocationDeciders allocationDeciders,
         ShardsAllocator shardsAllocator,
         ClusterInfoService clusterInfoService,
-        SnapshotsInfoService snapshotsInfoService
+        SnapshotsInfoService snapshotsInfoService,
+        ShardRoutingRoleStrategy shardRoutingRoleStrategy
     ) {
         this.allocationDeciders = allocationDeciders;
         this.shardsAllocator = shardsAllocator;
         this.clusterInfoService = clusterInfoService;
         this.snapshotsInfoService = snapshotsInfoService;
+        this.shardRoutingRoleStrategy = shardRoutingRoleStrategy;
     }
 
     /**
@@ -117,6 +122,10 @@ public AllocationDeciders getAllocationDeciders() {
         return allocationDeciders;
     }
 
+    public ShardRoutingRoleStrategy getShardRoutingRoleStrategy() {
+        return shardRoutingRoleStrategy;
+    }
+
     /**
      * Applies the started shards. Note, only initializing ShardRouting instances that exist in the routing table should be
      * provided as parameter and no duplicates should be contained.
@@ -297,7 +306,7 @@ public ClusterState adaptAutoExpandReplicas(ClusterState clusterState) {
         if (autoExpandReplicaChanges.isEmpty()) {
             return clusterState;
         } else {
-            final RoutingTable.Builder routingTableBuilder = RoutingTable.builder(clusterState.routingTable());
+            final RoutingTable.Builder routingTableBuilder = RoutingTable.builder(shardRoutingRoleStrategy, clusterState.routingTable());
             final Metadata.Builder metadataBuilder = Metadata.builder(clusterState.metadata());
             for (Map.Entry<Integer, List<String>> entry : autoExpandReplicaChanges.entrySet()) {
                 final int numberOfReplicas = entry.getKey();
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
index f5a4e1ff3a1e..86a8f20115c5 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
@@ -11,7 +11,8 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.GroupedActionListener;
+import org.elasticsearch.action.support.RefCountingRunnable;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.ClusterState;
@@ -31,6 +32,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.Releasable;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -301,123 +303,109 @@ public void onNewInfo(ClusterInfo info) {
             }
         }
 
-        final ActionListener<Void> listener = new GroupedActionListener<>(ActionListener.wrap(this::checkFinished), 3);
-
-        if (reroute) {
-            logger.debug("rerouting shards: [{}]", explanation);
-            rerouteService.reroute("disk threshold monitor", Priority.HIGH, ActionListener.wrap(reroutedClusterState -> {
-
-                for (DiskUsage diskUsage : usagesOverHighThreshold) {
-                    final RoutingNode routingNode = reroutedClusterState.getRoutingNodes().node(diskUsage.getNodeId());
-                    final DiskUsage usageIncludingRelocations;
-                    final long relocatingShardsSize;
-                    if (routingNode != null) { // might be temporarily null if the ClusterInfoService and the ClusterService are out of step
-                        relocatingShardsSize = sizeOfRelocatingShards(routingNode, diskUsage, info, reroutedClusterState);
-                        usageIncludingRelocations = new DiskUsage(
-                            diskUsage.getNodeId(),
-                            diskUsage.getNodeName(),
-                            diskUsage.getPath(),
-                            diskUsage.getTotalBytes(),
-                            diskUsage.getFreeBytes() - relocatingShardsSize
-                        );
-                    } else {
-                        usageIncludingRelocations = diskUsage;
-                        relocatingShardsSize = 0L;
-                    }
-                    final ByteSizeValue total = ByteSizeValue.ofBytes(usageIncludingRelocations.getTotalBytes());
-
-                    if (usageIncludingRelocations.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHighStage(total).getBytes()) {
-                        nodesOverHighThresholdAndRelocating.remove(diskUsage.getNodeId());
-                        logger.warn(
-                            "high disk watermark [{}] exceeded on {}, shards will be relocated away from this node; "
-                                + "currently relocating away shards totalling [{}] bytes; the node is expected to continue to exceed "
-                                + "the high disk watermark when these relocations are complete",
-                            diskThresholdSettings.describeHighThreshold(total, false),
-                            diskUsage,
-                            -relocatingShardsSize
-                        );
-                    } else if (nodesOverHighThresholdAndRelocating.add(diskUsage.getNodeId())) {
-                        logger.info(
-                            "high disk watermark [{}] exceeded on {}, shards will be relocated away from this node; "
-                                + "currently relocating away shards totalling [{}] bytes; the node is expected to be below the high "
-                                + "disk watermark when these relocations are complete",
-                            diskThresholdSettings.describeHighThreshold(total, false),
-                            diskUsage,
-                            -relocatingShardsSize
-                        );
-                    } else {
-                        logger.debug(
-                            "high disk watermark [{}] exceeded on {}, shards will be relocated away from this node; "
-                                + "currently relocating away shards totalling [{}] bytes",
-                            diskThresholdSettings.describeHighThreshold(total, false),
-                            diskUsage,
-                            -relocatingShardsSize
-                        );
-                    }
-                }
-
-                setLastRunTimeMillis();
-                listener.onResponse(null);
-            }, e -> {
-                logger.debug("reroute failed", e);
-                setLastRunTimeMillis();
-                listener.onFailure(e);
-            }));
-        } else {
-            logger.trace("no reroute required");
-            listener.onResponse(null);
-        }
-
-        // Generate a map of node name to ID so we can use it to look up node replacement targets
-        final Map<String, String> nodeNameToId = state.getRoutingNodes()
-            .stream()
-            .collect(Collectors.toMap(rn -> rn.node().getName(), RoutingNode::nodeId, (s1, s2) -> s2));
+        try (var asyncRefs = new RefCountingRunnable(this::checkFinished)) {
+
+            if (reroute) {
+                logger.debug("rerouting shards: [{}]", explanation);
+                rerouteService.reroute(
+                    "disk threshold monitor",
+                    Priority.HIGH,
+                    ActionListener.releaseAfter(ActionListener.runAfter(ActionListener.wrap(reroutedClusterState -> {
+
+                        for (DiskUsage diskUsage : usagesOverHighThreshold) {
+                            final RoutingNode routingNode = reroutedClusterState.getRoutingNodes().node(diskUsage.getNodeId());
+                            final DiskUsage usageIncludingRelocations;
+                            final long relocatingShardsSize;
+                            if (routingNode != null) { // might be temporarily null if ClusterInfoService and ClusterService are out of step
+                                relocatingShardsSize = sizeOfRelocatingShards(routingNode, diskUsage, info, reroutedClusterState);
+                                usageIncludingRelocations = new DiskUsage(
+                                    diskUsage.getNodeId(),
+                                    diskUsage.getNodeName(),
+                                    diskUsage.getPath(),
+                                    diskUsage.getTotalBytes(),
+                                    diskUsage.getFreeBytes() - relocatingShardsSize
+                                );
+                            } else {
+                                usageIncludingRelocations = diskUsage;
+                                relocatingShardsSize = 0L;
+                            }
+                            final ByteSizeValue total = ByteSizeValue.ofBytes(usageIncludingRelocations.getTotalBytes());
+
+                            if (usageIncludingRelocations.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHighStage(total)
+                                .getBytes()) {
+                                nodesOverHighThresholdAndRelocating.remove(diskUsage.getNodeId());
+                                logger.warn("""
+                                    high disk watermark [{}] exceeded on {}, shards will be relocated away from this node; currently \
+                                    relocating away shards totalling [{}] bytes; the node is expected to continue to exceed the high disk \
+                                    watermark when these relocations are complete\
+                                    """, diskThresholdSettings.describeHighThreshold(total, false), diskUsage, -relocatingShardsSize);
+                            } else if (nodesOverHighThresholdAndRelocating.add(diskUsage.getNodeId())) {
+                                logger.info("""
+                                    high disk watermark [{}] exceeded on {}, shards will be relocated away from this node; currently \
+                                    relocating away shards totalling [{}] bytes; the node is expected to be below the high disk watermark \
+                                    when these relocations are complete\
+                                    """, diskThresholdSettings.describeHighThreshold(total, false), diskUsage, -relocatingShardsSize);
+                            } else {
+                                logger.debug("""
+                                    high disk watermark [{}] exceeded on {}, shards will be relocated away from this node; currently \
+                                    relocating away shards totalling [{}] bytes\
+                                    """, diskThresholdSettings.describeHighThreshold(total, false), diskUsage, -relocatingShardsSize);
+                            }
+                        }
+                    }, e -> logger.debug("reroute failed", e)), this::setLastRunTimeMillis), asyncRefs.acquire())
+                );
+            } else {
+                logger.trace("no reroute required");
+            }
 
-        // Calculate both the source node id and the target node id of a "replace" type shutdown
-        final Set<String> nodesIdsPartOfReplacement = state.metadata()
-            .nodeShutdowns()
-            .values()
-            .stream()
-            .filter(meta -> meta.getType() == SingleNodeShutdownMetadata.Type.REPLACE)
-            .flatMap(meta -> Stream.of(meta.getNodeId(), nodeNameToId.get(meta.getTargetNodeName())))
-            .collect(Collectors.toSet());
-
-        // Generate a set of all the indices that exist on either the target or source of a node replacement
-        final Set<String> indicesOnReplaceSourceOrTarget = new HashSet<>();
-        for (String nodeId : nodesIdsPartOfReplacement) {
-            for (ShardRouting shardRouting : state.getRoutingNodes().node(nodeId)) {
-                indicesOnReplaceSourceOrTarget.add(shardRouting.index().getName());
+            // Generate a map of node name to ID so we can use it to look up node replacement targets
+            final Map<String, String> nodeNameToId = state.getRoutingNodes()
+                .stream()
+                .collect(Collectors.toMap(rn -> rn.node().getName(), RoutingNode::nodeId, (s1, s2) -> s2));
+
+            // Calculate both the source node id and the target node id of a "replace" type shutdown
+            final Set<String> nodesIdsPartOfReplacement = state.metadata()
+                .nodeShutdowns()
+                .values()
+                .stream()
+                .filter(meta -> meta.getType() == SingleNodeShutdownMetadata.Type.REPLACE)
+                .flatMap(meta -> Stream.of(meta.getNodeId(), nodeNameToId.get(meta.getTargetNodeName())))
+                .collect(Collectors.toSet());
+
+            // Generate a set of all the indices that exist on either the target or source of a node replacement
+            final Set<String> indicesOnReplaceSourceOrTarget = new HashSet<>();
+            for (String nodeId : nodesIdsPartOfReplacement) {
+                for (ShardRouting shardRouting : state.getRoutingNodes().node(nodeId)) {
+                    indicesOnReplaceSourceOrTarget.add(shardRouting.index().getName());
+                }
             }
-        }
 
-        final Set<String> indicesToAutoRelease = state.routingTable()
-            .indicesRouting()
-            .keySet()
-            .stream()
-            .filter(index -> indicesNotToAutoRelease.contains(index) == false)
-            .filter(index -> state.getBlocks().hasIndexBlock(index, IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK))
-            // Do not auto release indices that are on either the source or the target of a node replacement
-            .filter(index -> indicesOnReplaceSourceOrTarget.contains(index) == false)
-            .collect(Collectors.toSet());
-
-        if (indicesToAutoRelease.isEmpty() == false) {
-            logger.info(
-                "releasing read-only block on indices "
-                    + indicesToAutoRelease
-                    + " since they are now allocated to nodes with sufficient disk space"
-            );
-            updateIndicesReadOnly(indicesToAutoRelease, listener, false);
-        } else {
-            logger.trace("no auto-release required");
-            listener.onResponse(null);
-        }
+            final Set<String> indicesToAutoRelease = state.routingTable()
+                .indicesRouting()
+                .keySet()
+                .stream()
+                .filter(index -> indicesNotToAutoRelease.contains(index) == false)
+                .filter(index -> state.getBlocks().hasIndexBlock(index, IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK))
+                // Do not auto release indices that are on either the source or the target of a node replacement
+                .filter(index -> indicesOnReplaceSourceOrTarget.contains(index) == false)
+                .collect(Collectors.toSet());
+
+            if (indicesToAutoRelease.isEmpty() == false) {
+                logger.info(
+                    "releasing read-only block on indices "
+                        + indicesToAutoRelease
+                        + " since they are now allocated to nodes with sufficient disk space"
+                );
+                updateIndicesReadOnly(indicesToAutoRelease, asyncRefs.acquire(), false);
+            } else {
+                logger.trace("no auto-release required");
+            }
 
-        indicesToMarkReadOnly.removeIf(index -> state.getBlocks().indexBlocked(ClusterBlockLevel.WRITE, index));
-        logger.trace("marking indices as read-only: [{}]", indicesToMarkReadOnly);
-        if (indicesToMarkReadOnly.isEmpty() == false) {
-            updateIndicesReadOnly(indicesToMarkReadOnly, listener, true);
-        } else {
-            listener.onResponse(null);
+            indicesToMarkReadOnly.removeIf(index -> state.getBlocks().indexBlocked(ClusterBlockLevel.WRITE, index));
+            logger.trace("marking indices as read-only: [{}]", indicesToMarkReadOnly);
+            if (indicesToMarkReadOnly.isEmpty() == false) {
+                updateIndicesReadOnly(indicesToMarkReadOnly, asyncRefs.acquire(), true);
+            }
         }
     }
 
@@ -453,23 +441,24 @@ private void setLastRunTimeMillis() {
         lastRunTimeMillis.getAndUpdate(l -> Math.max(l, currentTimeMillisSupplier.getAsLong()));
     }
 
-    protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener<Void> listener, boolean readOnly) {
+    protected void updateIndicesReadOnly(Set<String> indicesToUpdate, Releasable onCompletion, boolean readOnly) {
         // set read-only block but don't block on the response
-        ActionListener<Void> wrappedListener = ActionListener.wrap(r -> {
-            setLastRunTimeMillis();
-            listener.onResponse(r);
-        }, e -> {
-            logger.debug(() -> "setting indices [" + readOnly + "] read-only failed", e);
-            setLastRunTimeMillis();
-            listener.onFailure(e);
-        });
         Settings readOnlySettings = readOnly ? READ_ONLY_ALLOW_DELETE_SETTINGS : NOT_READ_ONLY_ALLOW_DELETE_SETTINGS;
         client.admin()
             .indices()
             .prepareUpdateSettings(indicesToUpdate.toArray(Strings.EMPTY_ARRAY))
             .setSettings(readOnlySettings)
             .origin("disk-threshold-monitor")
-            .execute(wrappedListener.map(r -> null));
+            .execute(
+                ActionListener.releaseAfter(
+                    ActionListener.runAfter(
+                        ActionListener.<AcknowledgedResponse>noop()
+                            .delegateResponse((l, e) -> logger.debug(() -> "setting indices [" + readOnly + "] read-only failed", e)),
+                        this::setLastRunTimeMillis
+                    ),
+                    onCompletion
+                )
+            );
     }
 
     private void removeExistingIndexBlocks() {
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java
index 469e7f7efe36..e0b53e312e40 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java
@@ -69,12 +69,14 @@ public void shardStarted(ShardRouting initializingShard, ShardRouting startedSha
                 + "] and startedShard.allocationId ["
                 + startedShard.allocationId().getId()
                 + "] have to have the same";
-        Updates updates = changes(startedShard.shardId());
-        updates.addedAllocationIds.add(startedShard.allocationId().getId());
-        if (startedShard.primary()
-            // started shard has to have null recoverySource; have to pick up recoverySource from its initializing state
-            && (initializingShard.recoverySource() == RecoverySource.ExistingStoreRecoverySource.FORCE_STALE_PRIMARY_INSTANCE)) {
-            updates.removedAllocationIds.add(RecoverySource.ExistingStoreRecoverySource.FORCED_ALLOCATION_ID);
+        if (startedShard.isPromotableToPrimary()) {
+            Updates updates = changes(startedShard.shardId());
+            updates.addedAllocationIds.add(startedShard.allocationId().getId());
+            if (startedShard.primary()
+                // started shard has to have null recoverySource; have to pick up recoverySource from its initializing state
+                && (initializingShard.recoverySource() == RecoverySource.ExistingStoreRecoverySource.FORCE_STALE_PRIMARY_INSTANCE)) {
+                updates.removedAllocationIds.add(RecoverySource.ExistingStoreRecoverySource.FORCED_ALLOCATION_ID);
+            }
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
index ecc6fb295fea..7846cbacb90d 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
@@ -407,7 +407,17 @@ public void setSimulatedClusterInfo(ClusterInfo clusterInfo) {
     }
 
     public RoutingAllocation immutableClone() {
-        return new RoutingAllocation(deciders, clusterState, clusterInfo, shardSizeInfo, currentNanoTime);
+        return new RoutingAllocation(
+            deciders,
+            routingNodesChanged()
+                ? ClusterState.builder(clusterState)
+                    .routingTable(RoutingTable.of(clusterState.routingTable().version(), routingNodes))
+                    .build()
+                : clusterState,
+            clusterInfo,
+            shardSizeInfo,
+            currentNanoTime
+        );
     }
 
     public RoutingAllocation mutableCloneForSimulation() {
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/ShardsAvailabilityHealthIndicatorService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/ShardsAvailabilityHealthIndicatorService.java
index 2efe60c65146..51e53d086739 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/ShardsAvailabilityHealthIndicatorService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/ShardsAvailabilityHealthIndicatorService.java
@@ -43,6 +43,7 @@
 import org.elasticsearch.health.ImpactArea;
 import org.elasticsearch.health.SimpleHealthIndicatorDetails;
 import org.elasticsearch.health.node.HealthInfo;
+import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.snapshots.SnapshotShardSizeInfo;
 
 import java.util.ArrayList;
@@ -59,6 +60,8 @@
 import java.util.stream.Stream;
 
 import static java.util.stream.Collectors.joining;
+import static java.util.stream.Collectors.toMap;
+import static java.util.stream.Collectors.toSet;
 import static org.elasticsearch.cluster.health.ClusterShardHealth.getInactivePrimaryHealth;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_PREFIX;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING;
@@ -67,6 +70,7 @@
 import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING;
 import static org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider.CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING;
 import static org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING;
+import static org.elasticsearch.health.Diagnosis.Resource.Type.FEATURE_STATE;
 import static org.elasticsearch.health.Diagnosis.Resource.Type.INDEX;
 import static org.elasticsearch.health.HealthStatus.GREEN;
 import static org.elasticsearch.health.HealthStatus.RED;
@@ -96,9 +100,16 @@ public class ShardsAvailabilityHealthIndicatorService implements HealthIndicator
     private final ClusterService clusterService;
     private final AllocationService allocationService;
 
-    public ShardsAvailabilityHealthIndicatorService(ClusterService clusterService, AllocationService allocationService) {
+    private final SystemIndices systemIndices;
+
+    public ShardsAvailabilityHealthIndicatorService(
+        ClusterService clusterService,
+        AllocationService allocationService,
+        SystemIndices systemIndices
+    ) {
         this.clusterService = clusterService;
         this.allocationService = allocationService;
+        this.systemIndices = systemIndices;
     }
 
     @Override
@@ -107,7 +118,7 @@ public String name() {
     }
 
     @Override
-    public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
+    public HealthIndicatorResult calculate(boolean verbose, int maxAffectedResourcesCount, HealthInfo healthInfo) {
         var state = clusterService.state();
         var shutdown = state.getMetadata().custom(NodesShutdownMetadata.TYPE, NodesShutdownMetadata.EMPTY);
         var status = new ShardAllocationStatus(state.getMetadata());
@@ -126,7 +137,7 @@ public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
             status.getSymptom(),
             status.getDetails(verbose),
             status.getImpacts(),
-            status.getDiagnosis(verbose)
+            status.getDiagnosis(verbose, maxAffectedResourcesCount)
         );
     }
 
@@ -134,7 +145,7 @@ public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
     public static final String PRIMARY_UNASSIGNED_IMPACT_ID = "primary_unassigned";
     public static final String REPLICA_UNASSIGNED_IMPACT_ID = "replica_unassigned";
 
-    public static final String RESTORE_FROM_SNAPSHOT_ACTION_GUIDE = "http://ela.st/restore-snapshot";
+    public static final String RESTORE_FROM_SNAPSHOT_ACTION_GUIDE = "https://ela.st/restore-snapshot";
     public static final Diagnosis.Definition ACTION_RESTORE_FROM_SNAPSHOT = new Diagnosis.Definition(
         NAME,
         "restore_from_snapshot",
@@ -144,7 +155,7 @@ public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
         RESTORE_FROM_SNAPSHOT_ACTION_GUIDE
     );
 
-    public static final String DIAGNOSE_SHARDS_ACTION_GUIDE = "http://ela.st/diagnose-shards";
+    public static final String DIAGNOSE_SHARDS_ACTION_GUIDE = "https://ela.st/diagnose-shards";
     public static final Diagnosis.Definition ACTION_CHECK_ALLOCATION_EXPLAIN_API = new Diagnosis.Definition(
         NAME,
         "explain_allocations",
@@ -155,7 +166,7 @@ public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
         DIAGNOSE_SHARDS_ACTION_GUIDE
     );
 
-    public static final String FIX_DELAYED_SHARDS_GUIDE = "http://ela.st/fix-delayed-shard-allocation";
+    public static final String FIX_DELAYED_SHARDS_GUIDE = "https://ela.st/fix-delayed-shard-allocation";
     public static final Diagnosis.Definition DIAGNOSIS_WAIT_FOR_OR_FIX_DELAYED_SHARDS = new Diagnosis.Definition(
         NAME,
         "delayed_shard_allocations",
@@ -166,7 +177,7 @@ public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
         FIX_DELAYED_SHARDS_GUIDE
     );
 
-    public static final String ENABLE_INDEX_ALLOCATION_GUIDE = "http://ela.st/fix-index-allocation";
+    public static final String ENABLE_INDEX_ALLOCATION_GUIDE = "https://ela.st/fix-index-allocation";
     public static final Diagnosis.Definition ACTION_ENABLE_INDEX_ROUTING_ALLOCATION = new Diagnosis.Definition(
         NAME,
         "enable_index_allocations",
@@ -179,7 +190,7 @@ public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
             + "].",
         ENABLE_INDEX_ALLOCATION_GUIDE
     );
-    public static final String ENABLE_CLUSTER_ALLOCATION_ACTION_GUIDE = "http://ela.st/fix-cluster-allocation";
+    public static final String ENABLE_CLUSTER_ALLOCATION_ACTION_GUIDE = "https://ela.st/fix-cluster-allocation";
     public static final Diagnosis.Definition ACTION_ENABLE_CLUSTER_ROUTING_ALLOCATION = new Diagnosis.Definition(
         NAME,
         "enable_cluster_allocations",
@@ -193,7 +204,7 @@ public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
         ENABLE_CLUSTER_ALLOCATION_ACTION_GUIDE
     );
 
-    public static final String ENABLE_TIER_ACTION_GUIDE = "http://ela.st/enable-tier";
+    public static final String ENABLE_TIER_ACTION_GUIDE = "https://ela.st/enable-tier";
     public static final Map<String, Diagnosis.Definition> ACTION_ENABLE_TIERS_LOOKUP = DataTier.ALL_DATA_TIERS.stream()
         .collect(
             Collectors.toUnmodifiableMap(
@@ -209,7 +220,7 @@ public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
             )
         );
 
-    public static final String INCREASE_SHARD_LIMIT_ACTION_GUIDE = "http://ela.st/index-total-shards";
+    public static final String INCREASE_SHARD_LIMIT_ACTION_GUIDE = "https://ela.st/index-total-shards";
     public static final Diagnosis.Definition ACTION_INCREASE_SHARD_LIMIT_INDEX_SETTING = new Diagnosis.Definition(
         NAME,
         "increase_shard_limit_index_setting",
@@ -240,7 +251,7 @@ public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
             )
         );
 
-    public static final String INCREASE_CLUSTER_SHARD_LIMIT_ACTION_GUIDE = "http://ela.st/cluster-total-shards";
+    public static final String INCREASE_CLUSTER_SHARD_LIMIT_ACTION_GUIDE = "https://ela.st/cluster-total-shards";
     public static final Diagnosis.Definition ACTION_INCREASE_SHARD_LIMIT_CLUSTER_SETTING = new Diagnosis.Definition(
         NAME,
         "increase_shard_limit_cluster_setting",
@@ -271,7 +282,7 @@ public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
             )
         );
 
-    public static final String MIGRATE_TO_TIERS_ACTION_GUIDE = "http://ela.st/migrate-to-tiers";
+    public static final String MIGRATE_TO_TIERS_ACTION_GUIDE = "https://ela.st/migrate-to-tiers";
     public static final Diagnosis.Definition ACTION_MIGRATE_TIERS_AWAY_FROM_REQUIRE_DATA = new Diagnosis.Definition(
         NAME,
         "migrate_data_tiers_require_data",
@@ -340,7 +351,7 @@ public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
             )
         );
 
-    public static final String TIER_CAPACITY_ACTION_GUIDE = "http://ela.st/tier-capacity";
+    public static final String TIER_CAPACITY_ACTION_GUIDE = "https://ela.st/tier-capacity";
     public static final Diagnosis.Definition ACTION_INCREASE_NODE_CAPACITY = new Diagnosis.Definition(
         NAME,
         "increase_node_capacity_for_allocations",
@@ -441,9 +452,7 @@ List<Diagnosis.Definition> diagnoseUnassignedShardRouting(ShardRouting shardRout
         LOGGER.trace("Diagnosing unassigned shard [{}] due to reason [{}]", shardRouting.shardId(), shardRouting.unassignedInfo());
         switch (shardRouting.unassignedInfo().getLastAllocationStatus()) {
             case NO_VALID_SHARD_COPY:
-                if (UnassignedInfo.Reason.NODE_LEFT == shardRouting.unassignedInfo().getReason()) {
-                    diagnosisDefs.add(ACTION_RESTORE_FROM_SNAPSHOT);
-                }
+                diagnosisDefs.add(ACTION_RESTORE_FROM_SNAPSHOT);
                 break;
             case NO_ATTEMPT:
                 if (shardRouting.unassignedInfo().isDelayed()) {
@@ -762,7 +771,7 @@ private Optional<Diagnosis.Definition> checkNotEnoughNodesInDataTier(
         }
     }
 
-    private class ShardAllocationStatus {
+    class ShardAllocationStatus {
         private final ShardAllocationCounts primaries = new ShardAllocationCounts();
         private final ShardAllocationCounts replicas = new ShardAllocationCounts();
         private final Metadata clusterMetadata;
@@ -893,9 +902,10 @@ public List<HealthIndicatorImpact> getImpacts() {
         /**
          * Returns the diagnosis for unassigned primary and replica shards.
          * @param verbose true if the diagnosis should be generated, false if they should be omitted.
+         * @param maxAffectedResourcesCount the max number of affected resources to be returned as part of the diagnosis
          * @return The diagnoses list the indicator identified. Alternatively, an empty list if none were found or verbose is false.
          */
-        public List<Diagnosis> getDiagnosis(boolean verbose) {
+        public List<Diagnosis> getDiagnosis(boolean verbose, int maxAffectedResourcesCount) {
             if (verbose) {
                 Map<Diagnosis.Definition, Set<String>> diagnosisToAffectedIndices = new HashMap<>(primaries.diagnosisDefinitions);
                 replicas.diagnosisDefinitions.forEach((diagnosisDef, indicesWithReplicasUnassigned) -> {
@@ -909,27 +919,108 @@ public List<Diagnosis> getDiagnosis(boolean verbose) {
                 if (diagnosisToAffectedIndices.isEmpty()) {
                     return List.of();
                 } else {
-                    return diagnosisToAffectedIndices.entrySet()
-                        .stream()
-                        .map(
-                            e -> new Diagnosis(
-                                e.getKey(),
-                                List.of(
-                                    new Diagnosis.Resource(
-                                        INDEX,
-                                        e.getValue()
-                                            .stream()
-                                            .sorted(indicesComparatorByPriorityAndName(clusterMetadata))
-                                            .collect(Collectors.toList())
-                                    )
+
+                    return diagnosisToAffectedIndices.entrySet().stream().map(e -> {
+                        List<Diagnosis.Resource> affectedResources = new ArrayList<>(1);
+                        if (e.getKey().equals(ACTION_RESTORE_FROM_SNAPSHOT)) {
+                            Set<String> restoreFromSnapshotIndices = e.getValue();
+                            if (restoreFromSnapshotIndices != null && restoreFromSnapshotIndices.isEmpty() == false) {
+                                affectedResources = getRestoreFromSnapshotAffectedResources(
+                                    clusterMetadata,
+                                    systemIndices,
+                                    restoreFromSnapshotIndices,
+                                    maxAffectedResourcesCount
+                                );
+                            }
+                        } else {
+                            affectedResources.add(
+                                new Diagnosis.Resource(
+                                    INDEX,
+                                    e.getValue()
+                                        .stream()
+                                        .sorted(indicesComparatorByPriorityAndName(clusterMetadata))
+                                        .limit(Math.min(e.getValue().size(), maxAffectedResourcesCount))
+                                        .collect(Collectors.toList())
                                 )
-                            )
-                        )
-                        .collect(Collectors.toList());
+                            );
+                        }
+                        return new Diagnosis(e.getKey(), affectedResources);
+                    }).collect(Collectors.toList());
                 }
             } else {
                 return List.of();
             }
         }
+
+        /**
+         * The restore from snapshot operation requires the user to specify indices and feature states.
+         * The indices that are part of the feature states must not be specified. This method loops through all the
+         * identified unassigned indices and returns the affected {@link Diagnosis.Resource}s of type `INDEX`
+         * and if applicable `FEATURE_STATE`
+         */
+        static List<Diagnosis.Resource> getRestoreFromSnapshotAffectedResources(
+            Metadata metadata,
+            SystemIndices systemIndices,
+            Set<String> restoreFromSnapshotIndices,
+            int maxAffectedResourcesCount
+        ) {
+            List<Diagnosis.Resource> affectedResources = new ArrayList<>(2);
+
+            Set<String> affectedIndices = new HashSet<>(restoreFromSnapshotIndices);
+            Set<String> affectedFeatureStates = new HashSet<>();
+            Map<String, Set<String>> featureToSystemIndices = systemIndices.getFeatures()
+                .stream()
+                .collect(
+                    toMap(
+                        SystemIndices.Feature::getName,
+                        feature -> feature.getIndexDescriptors()
+                            .stream()
+                            .flatMap(descriptor -> descriptor.getMatchingIndices(metadata).stream())
+                            .collect(toSet())
+                    )
+                );
+
+            for (Map.Entry<String, Set<String>> featureToIndices : featureToSystemIndices.entrySet()) {
+                for (String featureIndex : featureToIndices.getValue()) {
+                    if (restoreFromSnapshotIndices.contains(featureIndex)) {
+                        affectedFeatureStates.add(featureToIndices.getKey());
+                        affectedIndices.remove(featureIndex);
+                    }
+                }
+            }
+
+            Map<String, Set<String>> featureToDsBackingIndices = systemIndices.getFeatures()
+                .stream()
+                .collect(
+                    toMap(
+                        SystemIndices.Feature::getName,
+                        feature -> feature.getDataStreamDescriptors()
+                            .stream()
+                            .flatMap(descriptor -> descriptor.getBackingIndexNames(metadata).stream())
+                            .collect(toSet())
+                    )
+                );
+
+            // the shards_availability indicator works with indices so let's remove the feature states data streams backing indices from
+            // the list of affected indices (the feature state will cover the restore of these indices too)
+            for (Map.Entry<String, Set<String>> featureToBackingIndices : featureToDsBackingIndices.entrySet()) {
+                for (String featureIndex : featureToBackingIndices.getValue()) {
+                    if (restoreFromSnapshotIndices.contains(featureIndex)) {
+                        affectedFeatureStates.add(featureToBackingIndices.getKey());
+                        affectedIndices.remove(featureIndex);
+                    }
+                }
+            }
+
+            if (affectedIndices.isEmpty() == false) {
+                affectedResources.add(new Diagnosis.Resource(INDEX, affectedIndices.stream().limit(maxAffectedResourcesCount).toList()));
+            }
+            if (affectedFeatureStates.isEmpty() == false) {
+                affectedResources.add(
+                    new Diagnosis.Resource(FEATURE_STATE, affectedFeatureStates.stream().limit(maxAffectedResourcesCount).toList())
+                );
+            }
+            return affectedResources;
+        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
index aaaa4cd26b04..2aaecfdcda7a 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
@@ -34,10 +34,13 @@
 import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
 import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.logging.DeprecationCategory;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.gateway.PriorityComparator;
@@ -50,13 +53,12 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.OptionalLong;
 import java.util.Set;
 import java.util.function.BiFunction;
-import java.util.function.Consumer;
 import java.util.stream.StreamSupport;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
+import static org.elasticsearch.common.settings.ClusterSettings.createBuiltInClusterSettings;
 
 /**
  * The {@link BalancedShardsAllocator} re-balances the nodes allocations
@@ -100,7 +102,7 @@ public class BalancedShardsAllocator implements ShardsAllocator {
     );
     public static final Setting<Float> DISK_USAGE_BALANCE_FACTOR_SETTING = Setting.floatSetting(
         "cluster.routing.allocation.balance.disk_usage",
-        0.0f,
+        2e-11f,
         0.0f,
         Property.Dynamic,
         Property.NodeScope
@@ -121,23 +123,49 @@ public class BalancedShardsAllocator implements ShardsAllocator {
 
     private final WriteLoadForecaster writeLoadForecaster;
 
+    public BalancedShardsAllocator() {
+        this(Settings.EMPTY);
+    }
+
     public BalancedShardsAllocator(Settings settings) {
-        this(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), WriteLoadForecaster.DEFAULT);
+        this(createBuiltInClusterSettings(settings), WriteLoadForecaster.DEFAULT);
+    }
+
+    public BalancedShardsAllocator(ClusterSettings clusterSettings) {
+        this(clusterSettings, WriteLoadForecaster.DEFAULT);
     }
 
     @Inject
-    public BalancedShardsAllocator(Settings settings, ClusterSettings clusterSettings, WriteLoadForecaster writeLoadForecaster) {
-        watchSetting(settings, clusterSettings, INDEX_BALANCE_FACTOR_SETTING, value -> this.indexBalanceFactor = value);
-        watchSetting(settings, clusterSettings, SHARD_BALANCE_FACTOR_SETTING, value -> this.shardBalanceFactor = value);
-        watchSetting(settings, clusterSettings, WRITE_LOAD_BALANCE_FACTOR_SETTING, value -> this.writeLoadBalanceFactor = value);
-        watchSetting(settings, clusterSettings, DISK_USAGE_BALANCE_FACTOR_SETTING, value -> this.diskUsageBalanceFactor = value);
-        watchSetting(settings, clusterSettings, THRESHOLD_SETTING, value -> this.threshold = value);
+    public BalancedShardsAllocator(ClusterSettings clusterSettings, WriteLoadForecaster writeLoadForecaster) {
+        clusterSettings.initializeAndWatch(INDEX_BALANCE_FACTOR_SETTING, value -> this.indexBalanceFactor = value);
+        clusterSettings.initializeAndWatch(SHARD_BALANCE_FACTOR_SETTING, value -> this.shardBalanceFactor = value);
+        clusterSettings.initializeAndWatch(WRITE_LOAD_BALANCE_FACTOR_SETTING, value -> this.writeLoadBalanceFactor = value);
+        clusterSettings.initializeAndWatch(DISK_USAGE_BALANCE_FACTOR_SETTING, value -> this.diskUsageBalanceFactor = value);
+        clusterSettings.initializeAndWatch(THRESHOLD_SETTING, value -> this.threshold = ensureValidThreshold(value));
         this.writeLoadForecaster = writeLoadForecaster;
     }
 
-    private <T> void watchSetting(Settings settings, ClusterSettings clusterSettings, Setting<T> setting, Consumer<T> consumer) {
-        consumer.accept(setting.get(settings));
-        clusterSettings.addSettingsUpdateConsumer(setting, consumer);
+    /**
+     * Clamp threshold to be at least 1, and log a critical deprecation warning if smaller values are given.
+     *
+     * Once {@link org.elasticsearch.Version#V_7_17_0} goes out of scope, start to properly reject such bad values.
+     */
+    private static float ensureValidThreshold(float threshold) {
+        if (1.0f <= threshold) {
+            return threshold;
+        } else {
+            DeprecationLogger.getLogger(BalancedShardsAllocator.class)
+                .critical(
+                    DeprecationCategory.SETTINGS,
+                    "balance_threshold_too_small",
+                    "ignoring value [{}] for [{}] since it is smaller than 1.0; "
+                        + "setting [{}] to a value smaller than 1.0 will be forbidden in a future release",
+                    threshold,
+                    THRESHOLD_SETTING.getKey(),
+                    THRESHOLD_SETTING.getKey()
+                );
+            return 1.0f;
+        }
     }
 
     @Override
@@ -292,26 +320,26 @@ float minWeightDelta(Balancer balancer, String index) {
      * A {@link Balancer}
      */
     public static class Balancer {
-        private final Map<String, ModelNode> nodes;
         private final WriteLoadForecaster writeLoadForecaster;
         private final RoutingAllocation allocation;
         private final RoutingNodes routingNodes;
+        private final Metadata metadata;
         private final WeightFunction weight;
 
         private final float threshold;
-        private final Metadata metadata;
         private final float avgShardsPerNode;
         private final double avgWriteLoadPerNode;
         private final double avgDiskUsageInBytesPerNode;
+        private final Map<String, ModelNode> nodes;
         private final NodeSorter sorter;
 
         public Balancer(WriteLoadForecaster writeLoadForecaster, RoutingAllocation allocation, WeightFunction weight, float threshold) {
             this.writeLoadForecaster = writeLoadForecaster;
             this.allocation = allocation;
-            this.weight = weight;
-            this.threshold = threshold;
             this.routingNodes = allocation.routingNodes();
             this.metadata = allocation.metadata();
+            this.weight = weight;
+            this.threshold = threshold;
             avgShardsPerNode = ((float) metadata.getTotalNumberOfShards()) / routingNodes.size();
             avgWriteLoadPerNode = getTotalWriteLoad(writeLoadForecaster, metadata) / routingNodes.size();
             avgDiskUsageInBytesPerNode = ((double) getTotalDiskUsageInBytes(allocation.clusterInfo(), metadata) / routingNodes.size());
@@ -342,15 +370,10 @@ private static long getTotalDiskUsageInBytes(ClusterInfo clusterInfo, Metadata m
 
         // Visible for testing
         static long getIndexDiskUsageInBytes(ClusterInfo clusterInfo, IndexMetadata indexMetadata) {
-            OptionalLong forecastedShardSizeInBytes = indexMetadata.getForecastedShardSizeInBytes();
-            final long indexDiskUsageInBytes;
-            if (forecastedShardSizeInBytes.isPresent()) {
-                int i = numberOfCopies(indexMetadata);
-                indexDiskUsageInBytes = forecastedShardSizeInBytes.getAsLong() * i;
-            } else {
-                indexDiskUsageInBytes = getIndexDiskUsageInBytesFromClusterInfo(clusterInfo, indexMetadata);
-            }
-            return indexDiskUsageInBytes;
+            var forecastedShardSizeInBytes = indexMetadata.getForecastedShardSizeInBytes();
+            return forecastedShardSizeInBytes.isPresent()
+                ? forecastedShardSizeInBytes.getAsLong() * numberOfCopies(indexMetadata)
+                : getIndexDiskUsageInBytesFromClusterInfo(clusterInfo, indexMetadata);
         }
 
         private static long getIndexDiskUsageInBytesFromClusterInfo(ClusterInfo clusterInfo, IndexMetadata indexMetadata) {
@@ -379,6 +402,10 @@ private static long getIndexDiskUsageInBytesFromClusterInfo(ClusterInfo clusterI
             return shardCount == 0 ? 0 : (totalSizeInBytes / shardCount) * numberOfCopies(indexMetadata);
         }
 
+        private static long getShardDiskUsageInBytes(ShardRouting shardRouting, IndexMetadata indexMetadata, ClusterInfo clusterInfo) {
+            return indexMetadata.getForecastedShardSizeInBytes().orElseGet(() -> clusterInfo.getShardSize(shardRouting, 0L));
+        }
+
         private static int numberOfCopies(IndexMetadata indexMetadata) {
             return indexMetadata.getNumberOfShards() * (1 + indexMetadata.getNumberOfReplicas());
         }
@@ -387,6 +414,14 @@ private double getShardWriteLoad(String index) {
             return writeLoadForecaster.getForecastedWriteLoad(metadata.index(index)).orElse(0.0);
         }
 
+        private double diskUsageInBytesPerShard(String index) {
+            var indexMetadata = metadata.index(index);
+            var forecastedShardSizeInBytes = indexMetadata.getForecastedShardSizeInBytes();
+            return forecastedShardSizeInBytes.isPresent()
+                ? forecastedShardSizeInBytes.getAsLong()
+                : (double) getIndexDiskUsageInBytesFromClusterInfo(allocation.clusterInfo(), indexMetadata) / numberOfCopies(indexMetadata);
+        }
+
         /**
          * Returns an array view on the nodes in the balancer. Nodes should not be removed from this list.
          */
@@ -416,10 +451,6 @@ public double avgDiskUsageInBytesPerNode() {
             return avgDiskUsageInBytesPerNode;
         }
 
-        public double diskUsageInBytesPerShard(String index) {
-            return metadata.index(index).getForecastedShardSizeInBytes().orElse(0);
-        }
-
         /**
          * Returns a new {@link NodeSorter} that sorts the nodes based on their
          * current weight with respect to the index passed to the sorter. The
@@ -931,9 +962,9 @@ private Decision decideCanForceAllocateForVacate(ShardRouting shardRouting, Rout
          * process. In short, this method recreates the status-quo in the cluster.
          */
         private Map<String, ModelNode> buildModelFromAssigned() {
-            Map<String, ModelNode> nodes = new HashMap<>();
+            Map<String, ModelNode> nodes = Maps.newMapWithExpectedSize(routingNodes.size());
             for (RoutingNode rn : routingNodes) {
-                ModelNode node = new ModelNode(writeLoadForecaster, metadata, rn);
+                ModelNode node = new ModelNode(writeLoadForecaster, metadata, allocation.clusterInfo(), rn);
                 nodes.put(rn.nodeId(), node);
                 for (ShardRouting shard : rn) {
                     assert rn.nodeId().equals(shard.currentNodeId());
@@ -1220,18 +1251,21 @@ private boolean tryRelocateShard(ModelNode minNode, ModelNode maxNode, String id
     }
 
     static class ModelNode implements Iterable<ModelIndex> {
-        private final Map<String, ModelIndex> indices = new HashMap<>();
         private int numShards = 0;
         private double writeLoad = 0.0;
         private double diskUsageInBytes = 0.0;
         private final WriteLoadForecaster writeLoadForecaster;
         private final Metadata metadata;
+        private final ClusterInfo clusterInfo;
         private final RoutingNode routingNode;
+        private final Map<String, ModelIndex> indices;
 
-        ModelNode(WriteLoadForecaster writeLoadForecaster, Metadata metadata, RoutingNode routingNode) {
+        ModelNode(WriteLoadForecaster writeLoadForecaster, Metadata metadata, ClusterInfo clusterInfo, RoutingNode routingNode) {
             this.writeLoadForecaster = writeLoadForecaster;
             this.metadata = metadata;
+            this.clusterInfo = clusterInfo;
             this.routingNode = routingNode;
+            this.indices = Maps.newMapWithExpectedSize(routingNode.size() + 10);// some extra to account for shard movements
         }
 
         public ModelIndex getIndex(String indexName) {
@@ -1275,7 +1309,7 @@ public void addShard(ShardRouting shard) {
             indices.computeIfAbsent(shard.getIndexName(), t -> new ModelIndex()).addShard(shard);
             IndexMetadata indexMetadata = metadata.index(shard.index());
             writeLoad += writeLoadForecaster.getForecastedWriteLoad(indexMetadata).orElse(0.0);
-            diskUsageInBytes += indexMetadata.getForecastedShardSizeInBytes().orElse(0);
+            diskUsageInBytes += Balancer.getShardDiskUsageInBytes(shard, indexMetadata, clusterInfo);
             numShards++;
         }
 
@@ -1289,7 +1323,7 @@ public void removeShard(ShardRouting shard) {
             }
             IndexMetadata indexMetadata = metadata.index(shard.index());
             writeLoad -= writeLoadForecaster.getForecastedWriteLoad(indexMetadata).orElse(0.0);
-            diskUsageInBytes -= indexMetadata.getForecastedShardSizeInBytes().orElse(0);
+            diskUsageInBytes -= Balancer.getShardDiskUsageInBytes(shard, indexMetadata, clusterInfo);
             numShards--;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterBalanceStats.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterBalanceStats.java
new file mode 100644
index 000000000000..7021bd6554b4
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterBalanceStats.java
@@ -0,0 +1,222 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.cluster.routing.allocation.allocator;
+
+import org.elasticsearch.cluster.ClusterInfo;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.cluster.routing.RoutingNode;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.ToDoubleFunction;
+
+public record ClusterBalanceStats(Map<String, TierBalanceStats> tiers, Map<String, NodeBalanceStats> nodes)
+    implements
+        Writeable,
+        ToXContentObject {
+
+    public static ClusterBalanceStats EMPTY = new ClusterBalanceStats(Map.of(), Map.of());
+
+    public static ClusterBalanceStats createFrom(
+        ClusterState clusterState,
+        ClusterInfo clusterInfo,
+        WriteLoadForecaster writeLoadForecaster
+    ) {
+        var tierToNodeStats = new HashMap<String, List<NodeBalanceStats>>();
+        var nodes = new HashMap<String, NodeBalanceStats>();
+        for (RoutingNode routingNode : clusterState.getRoutingNodes()) {
+            var dataRoles = routingNode.node().getRoles().stream().filter(DiscoveryNodeRole::canContainData).toList();
+            if (dataRoles.isEmpty()) {
+                continue;
+            }
+            var nodeStats = NodeBalanceStats.createFrom(routingNode, clusterState.metadata(), clusterInfo, writeLoadForecaster);
+            nodes.put(routingNode.node().getName(), nodeStats);
+            for (DiscoveryNodeRole role : dataRoles) {
+                tierToNodeStats.computeIfAbsent(role.roleName(), ignored -> new ArrayList<>()).add(nodeStats);
+            }
+        }
+        return new ClusterBalanceStats(Maps.transformValues(tierToNodeStats, TierBalanceStats::createFrom), nodes);
+    }
+
+    public static ClusterBalanceStats readFrom(StreamInput in) throws IOException {
+        return new ClusterBalanceStats(
+            in.readImmutableMap(StreamInput::readString, TierBalanceStats::readFrom),
+            in.readImmutableMap(StreamInput::readString, NodeBalanceStats::readFrom)
+        );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeMap(tiers, StreamOutput::writeString, StreamOutput::writeWriteable);
+        out.writeMap(nodes, StreamOutput::writeString, StreamOutput::writeWriteable);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return builder.startObject().field("tiers").map(tiers).field("nodes").map(nodes).endObject();
+    }
+
+    public record TierBalanceStats(
+        MetricStats shardCount,
+        MetricStats forecastWriteLoad,
+        MetricStats forecastShardSize,
+        MetricStats actualShardSize
+    ) implements Writeable, ToXContentObject {
+
+        private static TierBalanceStats createFrom(List<NodeBalanceStats> nodes) {
+            return new TierBalanceStats(
+                MetricStats.createFrom(nodes, it -> it.shards),
+                MetricStats.createFrom(nodes, it -> it.forecastWriteLoad),
+                MetricStats.createFrom(nodes, it -> it.forecastShardSize),
+                MetricStats.createFrom(nodes, it -> it.actualShardSize)
+            );
+        }
+
+        public static TierBalanceStats readFrom(StreamInput in) throws IOException {
+            return new TierBalanceStats(
+                MetricStats.readFrom(in),
+                MetricStats.readFrom(in),
+                MetricStats.readFrom(in),
+                MetricStats.readFrom(in)
+            );
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            shardCount.writeTo(out);
+            forecastWriteLoad.writeTo(out);
+            forecastShardSize.writeTo(out);
+            actualShardSize.writeTo(out);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return builder.startObject()
+                .field("shard_count", shardCount)
+                .field("forecast_write_load", forecastWriteLoad)
+                .field("forecast_disk_usage", forecastShardSize)
+                .field("actual_disk_usage", actualShardSize)
+                .endObject();
+        }
+    }
+
+    public record MetricStats(double total, double min, double max, double average, double stdDev) implements Writeable, ToXContentObject {
+
+        private static MetricStats createFrom(List<NodeBalanceStats> nodes, ToDoubleFunction<NodeBalanceStats> metricExtractor) {
+            assert nodes.isEmpty() == false : "Stats must be created from non empty nodes";
+            double total = 0.0;
+            double total2 = 0.0;
+            double min = Double.POSITIVE_INFINITY;
+            double max = Double.NEGATIVE_INFINITY;
+            int count = 0;
+            for (NodeBalanceStats node : nodes) {
+                var metric = metricExtractor.applyAsDouble(node);
+                if (Double.isNaN(metric)) {
+                    continue;
+                }
+                total += metric;
+                total2 += Math.pow(metric, 2);
+                min = Math.min(min, metric);
+                max = Math.max(max, metric);
+                count++;
+            }
+            double average = count == 0 ? Double.NaN : total / count;
+            double stdDev = count == 0 ? Double.NaN : Math.sqrt(total2 / count - Math.pow(average, 2));
+            return new MetricStats(total, min, max, average, stdDev);
+        }
+
+        public static MetricStats readFrom(StreamInput in) throws IOException {
+            return new MetricStats(in.readDouble(), in.readDouble(), in.readDouble(), in.readDouble(), in.readDouble());
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeDouble(total);
+            out.writeDouble(min);
+            out.writeDouble(max);
+            out.writeDouble(average);
+            out.writeDouble(stdDev);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return builder.startObject()
+                .field("total", total)
+                .field("min", min)
+                .field("max", max)
+                .field("average", average)
+                .field("std_dev", stdDev)
+                .endObject();
+        }
+    }
+
+    public record NodeBalanceStats(int shards, double forecastWriteLoad, long forecastShardSize, long actualShardSize)
+        implements
+            Writeable,
+            ToXContentObject {
+
+        private static NodeBalanceStats createFrom(
+            RoutingNode routingNode,
+            Metadata metadata,
+            ClusterInfo clusterInfo,
+            WriteLoadForecaster writeLoadForecaster
+        ) {
+            double forecastWriteLoad = 0.0;
+            long forecastShardSize = 0L;
+            long actualShardSize = 0L;
+
+            for (ShardRouting shardRouting : routingNode) {
+                var indexMetadata = metadata.index(shardRouting.index());
+                var shardSize = clusterInfo.getShardSize(shardRouting, 0L);
+                assert indexMetadata != null;
+                forecastWriteLoad += writeLoadForecaster.getForecastedWriteLoad(indexMetadata).orElse(0.0);
+                forecastShardSize += indexMetadata.getForecastedShardSizeInBytes().orElse(shardSize);
+                actualShardSize += shardSize;
+            }
+
+            return new NodeBalanceStats(routingNode.size(), forecastWriteLoad, forecastShardSize, actualShardSize);
+        }
+
+        public static NodeBalanceStats readFrom(StreamInput in) throws IOException {
+            return new NodeBalanceStats(in.readInt(), in.readDouble(), in.readLong(), in.readLong());
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeInt(shards);
+            out.writeDouble(forecastWriteLoad);
+            out.writeLong(forecastShardSize);
+            out.writeLong(actualShardSize);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return builder.startObject()
+                .field("shard_count", shards)
+                .field("forecast_write_load", forecastWriteLoad)
+                .humanReadableField("forecast_disk_usage_bytes", "forecast_disk_usage", ByteSizeValue.ofBytes(forecastShardSize))
+                .humanReadableField("actual_disk_usage_bytes", "actual_disk_usage", ByteSizeValue.ofBytes(actualShardSize))
+                .endObject();
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java
index 97a39321fba3..7603659a9680 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java
@@ -16,10 +16,15 @@
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
 import org.elasticsearch.common.metrics.MeanMetric;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.core.Strings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -43,12 +48,25 @@ public class DesiredBalanceComputer {
 
     private static final Logger logger = LogManager.getLogger(DesiredBalanceComputer.class);
 
+    private final ThreadPool threadPool;
     private final ShardsAllocator delegateAllocator;
 
     protected final MeanMetric iterations = new MeanMetric();
 
-    public DesiredBalanceComputer(ShardsAllocator delegateAllocator) {
+    public static final Setting<TimeValue> PROGRESS_LOG_INTERVAL_SETTING = Setting.timeSetting(
+        "cluster.routing.allocation.desired_balance.progress_log_interval",
+        TimeValue.timeValueMinutes(1),
+        TimeValue.ZERO,
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
+    private TimeValue progressLogInterval;
+
+    public DesiredBalanceComputer(ClusterSettings clusterSettings, ThreadPool threadPool, ShardsAllocator delegateAllocator) {
+        this.threadPool = threadPool;
         this.delegateAllocator = delegateAllocator;
+        clusterSettings.initializeAndWatch(PROGRESS_LOG_INTERVAL_SETTING, value -> this.progressLogInterval = value);
     }
 
     public DesiredBalance compute(
@@ -211,6 +229,11 @@ public DesiredBalance compute(
             }
         }
 
+        final int iterationCountReportInterval = computeIterationCountReportInterval(routingAllocation);
+        final long timeWarningInterval = progressLogInterval.millis();
+        final long computationStartedTime = threadPool.relativeTimeInMillis();
+        long nextReportTime = computationStartedTime + timeWarningInterval;
+
         int i = 0;
         boolean hasChanges = false;
         while (true) {
@@ -246,30 +269,45 @@ public DesiredBalance compute(
             }
 
             i++;
+            final int iterations = i;
+            final long currentTime = threadPool.relativeTimeInMillis();
+            final boolean reportByTime = nextReportTime <= currentTime;
+            final boolean reportByIterationCount = i % iterationCountReportInterval == 0;
+            if (reportByTime || reportByIterationCount) {
+                nextReportTime = currentTime + timeWarningInterval;
+            }
+
             if (hasChanges == false) {
-                logger.debug("Desired balance computation for [{}] converged after [{}] iterations", desiredBalanceInput.index(), i);
+                logger.debug(
+                    "Desired balance computation for [{}] converged after [{}] and [{}] iterations",
+                    desiredBalanceInput.index(),
+                    TimeValue.timeValueMillis(currentTime - computationStartedTime).toString(),
+                    i
+                );
                 break;
             }
             if (isFresh.test(desiredBalanceInput) == false) {
                 // we run at least one iteration, but if another reroute happened meanwhile
                 // then publish the interim state and restart the calculation
-                logger.debug("""
-                    Newer cluster state received after [{}] iterations, publishing incomplete desired balance for [{}] and restarting \
-                    computation
-                    """, i, desiredBalanceInput.index());
-                break;
-            }
-            if (i % 100 == 0) {
-                // TODO this warning should be time based, iteration count should be proportional to the number of shards
-                logger.log(
-                    i % 1000000 == 0 ? Level.INFO : Level.DEBUG,
-                    Strings.format(
-                        "Desired balance computation for [%d] is still not converged after [%d] iterations",
-                        desiredBalanceInput.index(),
-                        i
-                    )
+                logger.debug(
+                    "Desired balance computation for [{}] interrupted after [{}] and [{}] iterations as newer cluster state received. "
+                        + "Publishing intermediate desired balance and restarting computation",
+                    desiredBalanceInput.index(),
+                    i,
+                    TimeValue.timeValueMillis(currentTime - computationStartedTime).toString()
                 );
+                break;
             }
+
+            logger.log(
+                reportByIterationCount || reportByTime ? Level.INFO : i % 100 == 0 ? Level.DEBUG : Level.TRACE,
+                () -> Strings.format(
+                    "Desired balance computation for [%d] is still not converged after [%s] and [%d] iterations",
+                    desiredBalanceInput.index(),
+                    TimeValue.timeValueMillis(currentTime - computationStartedTime).toString(),
+                    iterations
+                )
+            );
         }
         iterations.inc(i);
 
@@ -347,4 +385,13 @@ private static UnassignedInfo discardAllocationStatus(UnassignedInfo info) {
             info.getLastAllocatedNodeId()
         );
     }
+
+    private static int computeIterationCountReportInterval(RoutingAllocation allocation) {
+        final int relativeSize = allocation.metadata().getTotalNumberOfShards();
+        int iterations = 1000;
+        while (iterations < relativeSize && iterations < 1_000_000_000) {
+            iterations *= 10;
+        }
+        return iterations;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java
index 6bf0ed8eeba0..19a388964fea 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java
@@ -11,6 +11,9 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.ArrayUtil;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.MetadataIndexStateService;
 import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RoutingNode;
@@ -95,8 +98,11 @@ private boolean allocateUnassignedInvariant() {
 
         assert routingNodes.unassigned().isEmpty();
 
-        final var shardCounts = allocation.metadata()
-            .stream()
+        final var shardCounts = allocation.metadata().stream().filter(indexMetadata ->
+        // skip any pre-7.2 closed indices which have no routing table entries at all
+        indexMetadata.getCreationVersion().onOrAfter(Version.V_7_2_0)
+            || indexMetadata.getState() == IndexMetadata.State.OPEN
+            || MetadataIndexStateService.isIndexVerifiedBeforeClosed(indexMetadata))
             .flatMap(
                 indexMetadata -> IntStream.range(0, indexMetadata.getNumberOfShards())
                     .mapToObj(
@@ -151,7 +157,7 @@ private void failAllocationOfNewPrimaries(RoutingAllocation allocation) {
     private void allocateUnassigned() {
         RoutingNodes.UnassignedShards unassigned = routingNodes.unassigned();
         if (logger.isTraceEnabled()) {
-            logger.trace("Start allocating unassigned shards");
+            logger.trace("Start allocating unassigned shards: {}", routingNodes.toString());
         }
         if (unassigned.isEmpty()) {
             return;
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java
index 652fe0c468ec..f9e0a70555e8 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.cluster.service.MasterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.metrics.CounterMetric;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -77,12 +78,19 @@ public interface DesiredBalanceReconcilerAction {
     }
 
     public DesiredBalanceShardsAllocator(
+        ClusterSettings clusterSettings,
         ShardsAllocator delegateAllocator,
         ThreadPool threadPool,
         ClusterService clusterService,
         DesiredBalanceReconcilerAction reconciler
     ) {
-        this(delegateAllocator, threadPool, clusterService, new DesiredBalanceComputer(delegateAllocator), reconciler);
+        this(
+            delegateAllocator,
+            threadPool,
+            clusterService,
+            new DesiredBalanceComputer(clusterSettings, threadPool, delegateAllocator),
+            reconciler
+        );
     }
 
     public DesiredBalanceShardsAllocator(
@@ -211,6 +219,11 @@ protected void reconcile(DesiredBalance desiredBalance, RoutingAllocation alloca
         }
         allocationOrdering.retainNodes(getNodeIds(allocation.routingNodes()));
         recordTime(cumulativeReconciliationTime, new DesiredBalanceReconciler(desiredBalance, allocation, allocationOrdering)::run);
+        if (logger.isTraceEnabled()) {
+            logger.trace("Reconciled desired balance: {}", desiredBalance);
+        } else {
+            logger.debug("Reconciled desired balance for [{}]", desiredBalance.lastConvergedIndex());
+        }
     }
 
     public DesiredBalance getDesiredBalance() {
@@ -251,6 +264,11 @@ public void onFailure(Exception e) {
             assert MasterService.isPublishFailureException(e) : e;
             onNoLongerMaster();
         }
+
+        @Override
+        public String toString() {
+            return "ReconcileDesiredBalanceTask[lastConvergedIndex=" + desiredBalance.lastConvergedIndex() + "]";
+        }
     }
 
     private final class ReconcileDesiredBalanceExecutor implements ClusterStateTaskExecutor<ReconcileDesiredBalanceTask> {
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceStats.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceStats.java
index ce7b24d8e60c..f0634fa508ce 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceStats.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceStats.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.xcontent.ToXContentFragment;
+import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
@@ -26,7 +26,7 @@ public record DesiredBalanceStats(
     long computationIterations,
     long cumulativeComputationTime,
     long cumulativeReconciliationTime
-) implements Writeable, ToXContentFragment {
+) implements Writeable, ToXContentObject {
 
     public static DesiredBalanceStats readFrom(StreamInput in) throws IOException {
         return new DesiredBalanceStats(
@@ -55,7 +55,7 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-
+        builder.startObject();
         builder.field("computation_active", computationActive);
         builder.field("computation_submitted", computationSubmitted);
         builder.field("computation_executed", computationExecuted);
@@ -64,7 +64,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field("computation_converged_index", lastConvergedIndex);
         builder.humanReadableField("computation_time_in_millis", "computation_time", new TimeValue(cumulativeComputationTime));
         builder.humanReadableField("reconciliation_time_in_millis", "reconciliation_time", new TimeValue(cumulativeReconciliationTime));
-
+        builder.endObject();
         return builder;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java
index d80c51d9740f..3b8b30be9545 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java
@@ -142,6 +142,14 @@ public Decision canAllocate(IndexMetadata indexMetadata, RoutingNode node, Routi
             Decision decision = allocationDecider.canAllocate(indexMetadata, node, allocation);
             // short track if a NO is returned.
             if (decision.type() == Decision.Type.NO) {
+                if (logger.isTraceEnabled()) {
+                    logger.trace(
+                        "Can not allocate [{}] on node [{}] due to [{}]",
+                        indexMetadata.getIndex().getName(),
+                        node.node(),
+                        allocationDecider.getClass().getSimpleName()
+                    );
+                }
                 if (allocation.debugDecision() == false) {
                     return Decision.NO;
                 } else {
@@ -178,6 +186,13 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocat
             Decision decision = allocationDecider.canAllocate(shardRouting, allocation);
             // short track if a NO is returned.
             if (decision.type() == Decision.Type.NO) {
+                if (logger.isTraceEnabled()) {
+                    logger.trace(
+                        "Can not allocate [{}] on any node due to [{}]",
+                        shardRouting,
+                        allocationDecider.getClass().getSimpleName()
+                    );
+                }
                 if (allocation.debugDecision() == false) {
                     return Decision.NO;
                 } else {
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java
index 9d22bbd20385..88d4a652a5a3 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.settings.Settings;
 
 import java.util.Locale;
 
@@ -89,10 +88,9 @@ public String toString() {
 
     private volatile ClusterRebalanceType type;
 
-    public ClusterRebalanceAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
-        type = CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.get(settings);
+    public ClusterRebalanceAllocationDecider(ClusterSettings clusterSettings) {
+        clusterSettings.initializeAndWatch(CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING, this::setType);
         logger.debug("using [{}] with [{}]", CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, type);
-        clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING, this::setType);
     }
 
     private void setType(ClusterRebalanceType type) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java
index bd32f670e226..fd3ce510a105 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.settings.Settings;
 
 /**
  * Similar to the {@link ClusterRebalanceAllocationDecider} this
@@ -44,13 +43,12 @@ public class ConcurrentRebalanceAllocationDecider extends AllocationDecider {
     );
     private volatile int clusterConcurrentRebalance;
 
-    public ConcurrentRebalanceAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
-        this.clusterConcurrentRebalance = CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING.get(settings);
-        logger.debug("using [cluster_concurrent_rebalance] with [{}]", clusterConcurrentRebalance);
-        clusterSettings.addSettingsUpdateConsumer(
+    public ConcurrentRebalanceAllocationDecider(ClusterSettings clusterSettings) {
+        clusterSettings.initializeAndWatch(
             CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING,
             this::setClusterConcurrentRebalance
         );
+        logger.debug("using [cluster_concurrent_rebalance] with [{}]", clusterConcurrentRebalance);
     }
 
     private void setClusterConcurrentRebalance(int concurrentRebalance) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java
index 19b2e3d986eb..11624e39d176 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java
@@ -84,11 +84,9 @@ public class EnableAllocationDecider extends AllocationDecider {
     private volatile Rebalance enableRebalance;
     private volatile Allocation enableAllocation;
 
-    public EnableAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
-        this.enableAllocation = CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.get(settings);
-        this.enableRebalance = CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.get(settings);
-        clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING, this::setEnableAllocation);
-        clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING, this::setEnableRebalance);
+    public EnableAllocationDecider(ClusterSettings clusterSettings) {
+        clusterSettings.initializeAndWatch(CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING, this::setEnableAllocation);
+        clusterSettings.initializeAndWatch(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING, this::setEnableRebalance);
     }
 
     private void setEnableRebalance(Rebalance enableRebalance) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java
index d244c38b5913..f37039608d7b 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java
@@ -34,6 +34,8 @@ public class MaxRetryAllocationDecider extends AllocationDecider {
         Setting.Property.NotCopyableOnResize
     );
 
+    private static final String RETRY_FAILED_API = "POST /_cluster/reroute?retry_failed&metric=none";
+
     public static final String NAME = "max_retry";
 
     private static final Decision YES_NO_FAILURES = Decision.single(Decision.Type.YES, NAME, "shard has no previous failures");
@@ -69,9 +71,9 @@ private static Decision debugDecision(Decision decision, UnassignedInfo info, in
             return Decision.single(
                 Decision.Type.NO,
                 NAME,
-                "shard has exceeded the maximum number of retries [%d] on failed allocation attempts - "
-                    + "manually call [/_cluster/reroute?retry_failed=true] to retry, [%s]",
+                "shard has exceeded the maximum number of retries [%d] on failed allocation attempts - manually call [%s] to retry, [%s]",
                 maxRetries,
+                RETRY_FAILED_API,
                 info.toString()
             );
         } else {
@@ -90,9 +92,9 @@ private static Decision debugDecision(Decision decision, RelocationFailureInfo i
             return Decision.single(
                 Decision.Type.NO,
                 NAME,
-                "shard has exceeded the maximum number of retries [%d] on failed relocation attempts - "
-                    + "manually call [/_cluster/reroute?retry_failed=true] to retry, [%s]",
+                "shard has exceeded the maximum number of retries [%d] on failed relocation attempts - manually call [%s] to retry, [%s]",
                 maxRetries,
+                RETRY_FAILED_API,
                 info.toString()
             );
         } else {
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java
index 44ec04a3e1ae..7f34150f2674 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.settings.Settings;
 
 /**
  * An allocation decider that prevents multiple instances of the same shard to
@@ -46,9 +45,8 @@ public class SameShardAllocationDecider extends AllocationDecider {
 
     private volatile boolean sameHost;
 
-    public SameShardAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
-        this.sameHost = CLUSTER_ROUTING_ALLOCATION_SAME_HOST_SETTING.get(settings);
-        clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_SAME_HOST_SETTING, this::setSameHost);
+    public SameShardAllocationDecider(ClusterSettings clusterSettings) {
+        clusterSettings.initializeAndWatch(CLUSTER_ROUTING_ALLOCATION_SAME_HOST_SETTING, this::setSameHost);
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
index e53688654e64..c43fd599df13 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.settings.Settings;
 
 import java.util.function.BiPredicate;
 
@@ -70,9 +69,8 @@ public class ShardsLimitAllocationDecider extends AllocationDecider {
         Property.NodeScope
     );
 
-    public ShardsLimitAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
-        this.clusterShardLimit = CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING.get(settings);
-        clusterSettings.addSettingsUpdateConsumer(CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING, this::setClusterShardLimit);
+    public ShardsLimitAllocationDecider(ClusterSettings clusterSettings) {
+        clusterSettings.initializeAndWatch(CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING, this::setClusterShardLimit);
     }
 
     private void setClusterShardLimit(int clusterShardLimit) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
index 3952330ba829..7011258e61a6 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.settings.Settings;
 
 import static org.elasticsearch.cluster.routing.allocation.decider.Decision.THROTTLE;
 import static org.elasticsearch.cluster.routing.allocation.decider.Decision.YES;
@@ -81,24 +80,19 @@ public class ThrottlingAllocationDecider extends AllocationDecider {
     private volatile int concurrentIncomingRecoveries;
     private volatile int concurrentOutgoingRecoveries;
 
-    public ThrottlingAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
-        this.primariesInitialRecoveries = CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING.get(settings);
-        concurrentIncomingRecoveries = CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING.get(settings);
-        concurrentOutgoingRecoveries = CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.get(settings);
-
-        clusterSettings.addSettingsUpdateConsumer(
+    public ThrottlingAllocationDecider(ClusterSettings clusterSettings) {
+        clusterSettings.initializeAndWatch(
             CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING,
             this::setPrimariesInitialRecoveries
         );
-        clusterSettings.addSettingsUpdateConsumer(
+        clusterSettings.initializeAndWatch(
             CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING,
             this::setConcurrentIncomingRecoverries
         );
-        clusterSettings.addSettingsUpdateConsumer(
+        clusterSettings.initializeAndWatch(
             CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING,
             this::setConcurrentOutgoingRecoverries
         );
-
         logger.debug(
             "using node_concurrent_outgoing_recoveries [{}], node_concurrent_incoming_recoveries [{}], "
                 + "node_initial_primaries_recoveries [{}]",
diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
index 49935668ed47..699e559a19cc 100644
--- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
@@ -537,6 +537,7 @@ private static void callClusterStateAppliers(
             try (Releasable ignored = stopWatch.record(name)) {
                 applier.applyClusterState(clusterChangedEvent);
             }
+            // TODO assert "ClusterStateApplier must not set response headers in the ClusterApplierService"
         }
     }
 
@@ -560,6 +561,7 @@ private static void callClusterStateListener(
             } catch (Exception ex) {
                 logger.warn("failed to notify ClusterStateListener", ex);
             }
+            // TODO assert "ClusterStateApplier must not set response headers in the ClusterStateListener"
         }
     }
 
@@ -654,4 +656,9 @@ protected boolean applicationMayFail() {
     public ClusterApplierRecordingService.Stats getStats() {
         return recordingService.getStats();
     }
+
+    // Exposed only for testing
+    public int getTimeoutClusterStateListenersSize() {
+        return timeoutClusterStateListeners.size();
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterStateUpdateStats.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterStateUpdateStats.java
index 96f9bd5c084e..2efafb228bb5 100644
--- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterStateUpdateStats.java
+++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterStateUpdateStats.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.cluster.service;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -117,7 +117,7 @@ public ClusterStateUpdateStats(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        assert out.getVersion().onOrAfter(Version.V_7_16_0) : out.getVersion();
+        assert out.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0) : out.getTransportVersion();
         out.writeVLong(unchangedTaskCount);
         out.writeVLong(publicationSuccessCount);
         out.writeVLong(publicationFailureCount);
diff --git a/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java b/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java
index 06c4df9a87a3..17e78d6ca474 100644
--- a/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java
@@ -277,12 +277,6 @@ private void runTasks(
             previousClusterState,
             executeTasks(previousClusterState, executionResults, executor, summary, threadPool.getThreadContext())
         );
-        // fail all tasks that have failed
-        for (final var executionResult : executionResults) {
-            if (executionResult.failure != null) {
-                executionResult.updateTask.onFailure(executionResult.failure, executionResult::restoreResponseHeaders);
-            }
-        }
         final TimeValue computationTime = getTimeSince(computationStartTime);
         logExecutionTime(computationTime, "compute cluster state update", summary);
 
@@ -950,7 +944,7 @@ void onBatchFailure(Exception failure) {
 
         void onPublishSuccess(ClusterState newClusterState) {
             if (publishedStateConsumer == null && onPublicationSuccess == null) {
-                assert failure != null;
+                notifyFailure();
                 return;
             }
             try (ThreadContext.StoredContext ignored = updateTask.threadContextSupplier.get()) {
@@ -967,7 +961,7 @@ void onPublishSuccess(ClusterState newClusterState) {
 
         void onClusterStateUnchanged(ClusterState clusterState) {
             if (publishedStateConsumer == null && onPublicationSuccess == null) {
-                assert failure != null;
+                notifyFailure();
                 return;
             }
             try (ThreadContext.StoredContext ignored = updateTask.threadContextSupplier.get()) {
@@ -985,6 +979,10 @@ void onClusterStateUnchanged(ClusterState clusterState) {
         void onPublishFailure(FailedToCommitClusterStateException e) {
             if (publishedStateConsumer == null && onPublicationSuccess == null) {
                 assert failure != null;
+                var taskFailure = failure;
+                failure = new FailedToCommitClusterStateException(e.getMessage(), e);
+                failure.addSuppressed(taskFailure);
+                notifyFailure();
                 return;
             }
             try (ThreadContext.StoredContext ignored = updateTask.threadContextSupplier.get()) {
@@ -996,9 +994,14 @@ void onPublishFailure(FailedToCommitClusterStateException e) {
             }
         }
 
+        void notifyFailure() {
+            assert failure != null;
+            this.updateTask.onFailure(this.failure, this::restoreResponseHeaders);
+        }
+
         ContextPreservingAckListener getContextPreservingAckListener() {
             assert incomplete() == false;
-            return updateTask.wrapInTaskContext(clusterStateAckListener, this::restoreResponseHeaders);
+            return failure == null ? updateTask.wrapInTaskContext(clusterStateAckListener, this::restoreResponseHeaders) : null;
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java
new file mode 100644
index 000000000000..856ba30d6c4a
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common;
+
+import org.elasticsearch.Build;
+import org.elasticsearch.Version;
+import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Encapsulates links to pages in the reference docs, so that for example we can include URLs in logs and API outputs. Each instance's
+ * {@link #toString()} yields (a string representation of) a URL for the relevant docs. Links are defined in the resource file
+ * {@code reference-docs-links.json} which must include definitions for exactly the set of values of this enum.
+ */
+public enum ReferenceDocs {
+    INITIAL_MASTER_NODES,
+    DISCOVERY_TROUBLESHOOTING,
+    UNSTABLE_CLUSTER_TROUBLESHOOTING,
+    LAGGING_NODE_TROUBLESHOOTING,
+    SHARD_LOCK_TROUBLESHOOTING,
+    CONCURRENT_REPOSITORY_WRITERS,
+    ARCHIVE_INDICES,
+    // this comment keeps the ';' on the next line so every entry above has a trailing ',' which makes the diff for adding new links cleaner
+    ;
+
+    private static final Map<String, String> linksBySymbol;
+
+    static {
+        try (var resourceStream = readFromJarResourceUrl(ReferenceDocs.class.getResource("reference-docs-links.json"))) {
+            linksBySymbol = Map.copyOf(readLinksBySymbol(resourceStream));
+        } catch (Exception e) {
+            assert false : e;
+            throw new IllegalStateException("could not read links resource", e);
+        }
+    }
+
+    static final String UNRELEASED_VERSION_COMPONENT = "master";
+    static final String VERSION_COMPONENT = getVersionComponent(Version.CURRENT, Build.CURRENT.isSnapshot());
+
+    static Map<String, String> readLinksBySymbol(InputStream inputStream) throws Exception {
+        try (var parser = XContentFactory.xContent(XContentType.JSON).createParser(XContentParserConfiguration.EMPTY, inputStream)) {
+            final var result = parser.map(LinkedHashMap::new, XContentParser::text);
+            final var iterator = result.keySet().iterator();
+            for (int i = 0; i < values().length; i++) {
+                final var expected = values()[i].name();
+                if (iterator.hasNext() == false) {
+                    throw new IllegalStateException("ran out of values at index " + i + ": expecting " + expected);
+                }
+                final var actual = iterator.next();
+                if (actual.equals(expected) == false) {
+                    throw new IllegalStateException("mismatch at index " + i + ": found " + actual + " but expected " + expected);
+                }
+            }
+            if (iterator.hasNext()) {
+                throw new IllegalStateException("found unexpected extra value: " + iterator.next());
+            }
+            return result;
+        }
+    }
+
+    /**
+     * Compute the version component of the URL path (e.g. {@code 8.5} or {@code master}) for a particular version of Elasticsearch. Exposed
+     * for testing, but all items use {@link #VERSION_COMPONENT} ({@code getVersionComponent(Version.CURRENT, Build.CURRENT.isSnapshot())})
+     * which relates to the current version and build.
+     */
+    static String getVersionComponent(Version version, boolean isSnapshot) {
+        return isSnapshot && version.revision == 0 ? UNRELEASED_VERSION_COMPONENT : version.major + "." + version.minor;
+    }
+
+    @Override
+    public String toString() {
+        return "https://www.elastic.co/guide/en/elasticsearch/reference/" + VERSION_COMPONENT + "/" + linksBySymbol.get(name());
+    }
+
+    @SuppressForbidden(reason = "reads resource from jar")
+    private static InputStream readFromJarResourceUrl(URL source) throws IOException {
+        if (source == null) {
+            throw new FileNotFoundException("links resource not found at [" + source + "]");
+        }
+        return source.openStream();
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/common/Strings.java b/server/src/main/java/org/elasticsearch/common/Strings.java
index 0fed68bdf732..d0dab040558b 100644
--- a/server/src/main/java/org/elasticsearch/common/Strings.java
+++ b/server/src/main/java/org/elasticsearch/common/Strings.java
@@ -756,7 +756,7 @@ public static String toString(ToXContent toXContent) {
      */
     @Deprecated
     public static String toString(ChunkedToXContent chunkedToXContent) {
-        return toString(ChunkedToXContent.wrapAsXContentObject(chunkedToXContent));
+        return toString(chunkedToXContent, false, false);
     }
 
     /**
@@ -795,7 +795,7 @@ public static String toString(ToXContent toXContent, boolean pretty, boolean hum
      */
     @Deprecated
     public static String toString(ChunkedToXContent chunkedToXContent, boolean pretty, boolean human) {
-        return toString(ChunkedToXContent.wrapAsXContentObject(chunkedToXContent), pretty, human);
+        return toString(ChunkedToXContent.wrapAsToXContent(chunkedToXContent), pretty, human);
     }
 
     /**
@@ -917,4 +917,11 @@ public static String toLowercaseAscii(String in) {
             .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append)
             .toString();
     }
+
+    /**
+     * Alias for {@link org.elasticsearch.core.Strings#format}
+     */
+    public static String format(String format, Object... args) {
+        return org.elasticsearch.core.Strings.format(format, args);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/bytes/BytesArray.java b/server/src/main/java/org/elasticsearch/common/bytes/BytesArray.java
index 4ae67e81ded9..8b48e871fec6 100644
--- a/server/src/main/java/org/elasticsearch/common/bytes/BytesArray.java
+++ b/server/src/main/java/org/elasticsearch/common/bytes/BytesArray.java
@@ -137,6 +137,11 @@ public int getIntLE(int index) {
         return ByteUtils.readIntLE(bytes, offset + index);
     }
 
+    @Override
+    public long getLongLE(int index) {
+        return ByteUtils.readLongLE(bytes, offset + index);
+    }
+
     @Override
     public double getDoubleLE(int index) {
         return ByteUtils.readDoubleLE(bytes, offset + index);
diff --git a/server/src/main/java/org/elasticsearch/common/bytes/CompositeBytesReference.java b/server/src/main/java/org/elasticsearch/common/bytes/CompositeBytesReference.java
index aa4ead0b6b90..9a75ebd0e60e 100644
--- a/server/src/main/java/org/elasticsearch/common/bytes/CompositeBytesReference.java
+++ b/server/src/main/java/org/elasticsearch/common/bytes/CompositeBytesReference.java
@@ -238,6 +238,18 @@ public int getIntLE(int index) {
         return super.getIntLE(index);
     }
 
+    @Override
+    public long getLongLE(int index) {
+        int i = getOffsetIndex(index);
+        int idx = index - offsets[i];
+        int end = idx + 8;
+        BytesReference wholeLongsLivesHere = references[i];
+        if (end <= wholeLongsLivesHere.length()) {
+            return wholeLongsLivesHere.getLongLE(idx);
+        }
+        return super.getLongLE(index);
+    }
+
     @Override
     public double getDoubleLE(int index) {
         int i = getOffsetIndex(index);
diff --git a/server/src/main/java/org/elasticsearch/common/collect/Iterators.java b/server/src/main/java/org/elasticsearch/common/collect/Iterators.java
index a2629ffd0556..2f78ba70529c 100644
--- a/server/src/main/java/org/elasticsearch/common/collect/Iterators.java
+++ b/server/src/main/java/org/elasticsearch/common/collect/Iterators.java
@@ -8,9 +8,13 @@
 
 package org.elasticsearch.common.collect;
 
+import org.elasticsearch.core.Nullable;
+
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.NoSuchElementException;
 import java.util.Objects;
+import java.util.function.Function;
 
 public class Iterators {
 
@@ -43,36 +47,33 @@ public static <T> Iterator<T> concat(Iterator<? extends T>... iterators) {
             throw new NullPointerException("iterators");
         }
 
-        // explicit generic type argument needed for type inference
-        return new ConcatenatedIterator<T>(iterators);
+        for (int i = 0; i < iterators.length; i++) {
+            if (iterators[i].hasNext()) {
+                // explicit generic type argument needed for type inference
+                return new ConcatenatedIterator<T>(iterators, i);
+            }
+        }
+
+        return Collections.emptyIterator();
     }
 
-    static class ConcatenatedIterator<T> implements Iterator<T> {
+    private static class ConcatenatedIterator<T> implements Iterator<T> {
         private final Iterator<? extends T>[] iterators;
-        private int index = 0;
+        private int index;
 
-        @SafeVarargs
-        @SuppressWarnings("varargs")
-        ConcatenatedIterator(Iterator<? extends T>... iterators) {
-            if (iterators == null) {
-                throw new NullPointerException("iterators");
-            }
-            for (int i = 0; i < iterators.length; i++) {
+        ConcatenatedIterator(Iterator<? extends T>[] iterators, int startIndex) {
+            for (int i = startIndex; i < iterators.length; i++) {
                 if (iterators[i] == null) {
                     throw new NullPointerException("iterators[" + i + "]");
                 }
             }
             this.iterators = iterators;
+            this.index = startIndex;
         }
 
         @Override
         public boolean hasNext() {
-            boolean hasNext = false;
-            while (index < iterators.length && (hasNext = iterators[index].hasNext()) == false) {
-                index++;
-            }
-
-            return hasNext;
+            return index < iterators.length;
         }
 
         @Override
@@ -80,7 +81,11 @@ public T next() {
             if (hasNext() == false) {
                 throw new NoSuchElementException();
             }
-            return iterators[index].next();
+            final T value = iterators[index].next();
+            while (index < iterators.length && iterators[index].hasNext() == false) {
+                index++;
+            }
+            return value;
         }
     }
 
@@ -110,4 +115,54 @@ public T next() {
             return array[index++];
         }
     }
+
+    public static <T, U> Iterator<? extends U> flatMap(Iterator<? extends T> input, Function<T, Iterator<? extends U>> fn) {
+        while (input.hasNext()) {
+            final var value = fn.apply(input.next());
+            if (value.hasNext()) {
+                return new FlatMapIterator<>(input, fn, value);
+            }
+        }
+
+        return Collections.emptyIterator();
+    }
+
+    private static final class FlatMapIterator<T, U> implements Iterator<U> {
+
+        private final Iterator<? extends T> input;
+        private final Function<T, Iterator<? extends U>> fn;
+
+        @Nullable // if finished, otherwise currentOutput.hasNext() is true
+        private Iterator<? extends U> currentOutput;
+
+        FlatMapIterator(Iterator<? extends T> input, Function<T, Iterator<? extends U>> fn, Iterator<? extends U> firstOutput) {
+            this.input = input;
+            this.fn = fn;
+            this.currentOutput = firstOutput;
+        }
+
+        @Override
+        public boolean hasNext() {
+            return currentOutput != null;
+        }
+
+        @Override
+        public U next() {
+            if (hasNext() == false) {
+                throw new NoSuchElementException();
+            }
+            // noinspection ConstantConditions this is for documentation purposes
+            assert currentOutput != null && currentOutput.hasNext();
+            final U value = currentOutput.next();
+            while (currentOutput != null && currentOutput.hasNext() == false) {
+                if (input.hasNext()) {
+                    currentOutput = fn.apply(input.next());
+                } else {
+                    currentOutput = null;
+                }
+            }
+            return value;
+        }
+    }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java b/server/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java
index b6b69a9d4d22..2de39744e40a 100644
--- a/server/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java
+++ b/server/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.common.compress;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.hash.MessageDigests;
@@ -209,7 +209,7 @@ public String getSha256() {
     public static CompressedXContent readCompressedString(StreamInput in) throws IOException {
         final String sha256;
         final byte[] compressedData;
-        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             sha256 = in.readString();
             compressedData = in.readByteArray();
         } else {
@@ -221,7 +221,7 @@ public static CompressedXContent readCompressedString(StreamInput in) throws IOE
     }
 
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             out.writeString(sha256);
         } else {
             int crc32 = crc32FromCompressed(bytes);
diff --git a/server/src/main/java/org/elasticsearch/common/compress/DeflateCompressor.java b/server/src/main/java/org/elasticsearch/common/compress/DeflateCompressor.java
index b8879ec299b3..4ff54ded571b 100644
--- a/server/src/main/java/org/elasticsearch/common/compress/DeflateCompressor.java
+++ b/server/src/main/java/org/elasticsearch/common/compress/DeflateCompressor.java
@@ -10,9 +10,9 @@
 
 import org.elasticsearch.Assertions;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Streams;
 
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
diff --git a/server/src/main/java/org/elasticsearch/common/document/DocumentField.java b/server/src/main/java/org/elasticsearch/common/document/DocumentField.java
index ec6f378c4c07..5828b485ce36 100644
--- a/server/src/main/java/org/elasticsearch/common/document/DocumentField.java
+++ b/server/src/main/java/org/elasticsearch/common/document/DocumentField.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.common.document;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -44,12 +44,12 @@ public class DocumentField implements Writeable, Iterable<Object> {
     public DocumentField(StreamInput in) throws IOException {
         name = in.readString();
         values = in.readList(StreamInput::readGenericValue);
-        if (in.getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             ignoredValues = in.readList(StreamInput::readGenericValue);
         } else {
             ignoredValues = Collections.emptyList();
         }
-        if (in.getVersion().onOrAfter(Version.V_8_2_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
             lookupFields = in.readList(LookupField::new);
         } else {
             lookupFields = List.of();
@@ -114,10 +114,10 @@ public List<Object> getIgnoredValues() {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(name);
         out.writeCollection(values, StreamOutput::writeGenericValue);
-        if (out.getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             out.writeCollection(ignoredValues, StreamOutput::writeGenericValue);
         }
-        if (out.getVersion().onOrAfter(Version.V_8_2_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
             out.writeList(lookupFields);
         } else {
             if (lookupFields.isEmpty() == false) {
diff --git a/server/src/main/java/org/elasticsearch/common/inject/MembersInjectorStore.java b/server/src/main/java/org/elasticsearch/common/inject/MembersInjectorStore.java
index 93cb30eff3e2..9352c84db28f 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/MembersInjectorStore.java
+++ b/server/src/main/java/org/elasticsearch/common/inject/MembersInjectorStore.java
@@ -70,8 +70,6 @@ private <T> MembersInjectorImpl<T> createWithListeners(TypeLiteral<T> type, Erro
         List<SingleMemberInjector> injectors = getInjectors(injectionPoints, errors);
         errors.throwIfNewErrors(numErrorsBefore);
 
-        errors.throwIfNewErrors(numErrorsBefore);
-
         return new MembersInjectorImpl<>(injector, type, injectors);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/common/io/Streams.java b/server/src/main/java/org/elasticsearch/common/io/Streams.java
index 1089884ee0e0..fd16c80df1da 100644
--- a/server/src/main/java/org/elasticsearch/common/io/Streams.java
+++ b/server/src/main/java/org/elasticsearch/common/io/Streams.java
@@ -136,22 +136,6 @@ public static String copyToString(Reader in) throws IOException {
         return out.toString();
     }
 
-    public static int readFully(InputStream reader, byte[] dest) throws IOException {
-        return readFully(reader, dest, 0, dest.length);
-    }
-
-    public static int readFully(InputStream reader, byte[] dest, int offset, int len) throws IOException {
-        int read = 0;
-        while (read < len) {
-            final int r = reader.read(dest, offset + read, len - read);
-            if (r == -1) {
-                break;
-            }
-            read += r;
-        }
-        return read;
-    }
-
     /**
      * Fully consumes the input stream, throwing the bytes away. Returns the number of bytes consumed.
      */
diff --git a/server/src/main/java/org/elasticsearch/common/io/UTF8StreamWriter.java b/server/src/main/java/org/elasticsearch/common/io/UTF8StreamWriter.java
deleted file mode 100644
index d912db541f05..000000000000
--- a/server/src/main/java/org/elasticsearch/common/io/UTF8StreamWriter.java
+++ /dev/null
@@ -1,292 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.common.io;
-
-import java.io.CharConversionException;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.Writer;
-
-public final class UTF8StreamWriter extends Writer {
-
-    /**
-     * Holds the current output stream or <code>null</code> if closed.
-     */
-    private OutputStream _outputStream;
-
-    /**
-     * Holds the bytes' buffer.
-     */
-    private final byte[] _bytes;
-
-    /**
-     * Holds the bytes buffer index.
-     */
-    private int _index;
-
-    /**
-     * Creates a UTF-8 writer having a byte buffer of moderate capacity (2048).
-     */
-    public UTF8StreamWriter() {
-        _bytes = new byte[2048];
-    }
-
-    /**
-     * Creates a UTF-8 writer having a byte buffer of specified capacity.
-     *
-     * @param capacity the capacity of the byte buffer.
-     */
-    public UTF8StreamWriter(int capacity) {
-        _bytes = new byte[capacity];
-    }
-
-    /**
-     * Sets the output stream to use for writing until this writer is closed.
-     * For example:[code]
-     * Writer writer = new UTF8StreamWriter().setOutputStream(out);
-     * [/code] is equivalent but writes faster than [code]
-     * Writer writer = new java.io.OutputStreamWriter(out, "UTF-8");
-     * [/code]
-     *
-     * @param out the output stream.
-     * @return this UTF-8 writer.
-     * @throws IllegalStateException if this writer is being reused and
-     *                               it has not been {@link #close closed} or {@link #reset reset}.
-     */
-    public UTF8StreamWriter setOutput(OutputStream out) {
-        if (_outputStream != null) throw new IllegalStateException("Writer not closed or reset");
-        _outputStream = out;
-        return this;
-    }
-
-    /**
-     * Writes a single character. This method supports 16-bits
-     * character surrogates.
-     *
-     * @param c <code>char</code> the character to be written (possibly
-     *          a surrogate).
-     * @throws IOException if an I/O error occurs.
-     */
-    public void write(char c) throws IOException {
-        if ((c < 0xd800) || (c > 0xdfff)) {
-            write((int) c);
-        } else if (c < 0xdc00) { // High surrogate.
-            _highSurrogate = c;
-        } else { // Low surrogate.
-            int code = ((_highSurrogate - 0xd800) << 10) + (c - 0xdc00) + 0x10000;
-            write(code);
-        }
-    }
-
-    private char _highSurrogate;
-
-    /**
-     * Writes a character given its 31-bits Unicode.
-     *
-     * @param code the 31 bits Unicode of the character to be written.
-     * @throws IOException if an I/O error occurs.
-     */
-    @Override
-    public void write(int code) throws IOException {
-        if ((code & 0xffffff80) == 0) {
-            _bytes[_index] = (byte) code;
-            if (++_index >= _bytes.length) {
-                flushBuffer();
-            }
-        } else { // Writes more than one byte.
-            write2(code);
-        }
-    }
-
-    private void write2(int c) throws IOException {
-        if ((c & 0xfffff800) == 0) { // 2 bytes.
-            _bytes[_index] = (byte) (0xc0 | (c >> 6));
-            if (++_index >= _bytes.length) {
-                flushBuffer();
-            }
-            _bytes[_index] = (byte) (0x80 | (c & 0x3f));
-            if (++_index >= _bytes.length) {
-                flushBuffer();
-            }
-        } else if ((c & 0xffff0000) == 0) { // 3 bytes.
-            _bytes[_index] = (byte) (0xe0 | (c >> 12));
-            if (++_index >= _bytes.length) {
-                flushBuffer();
-            }
-            _bytes[_index] = (byte) (0x80 | ((c >> 6) & 0x3f));
-            if (++_index >= _bytes.length) {
-                flushBuffer();
-            }
-            _bytes[_index] = (byte) (0x80 | (c & 0x3f));
-            if (++_index >= _bytes.length) {
-                flushBuffer();
-            }
-        } else if ((c & 0xff200000) == 0) { // 4 bytes.
-            _bytes[_index] = (byte) (0xf0 | (c >> 18));
-            if (++_index >= _bytes.length) {
-                flushBuffer();
-            }
-            _bytes[_index] = (byte) (0x80 | ((c >> 12) & 0x3f));
-            if (++_index >= _bytes.length) {
-                flushBuffer();
-            }
-            _bytes[_index] = (byte) (0x80 | ((c >> 6) & 0x3f));
-            if (++_index >= _bytes.length) {
-                flushBuffer();
-            }
-            _bytes[_index] = (byte) (0x80 | (c & 0x3f));
-            if (++_index >= _bytes.length) {
-                flushBuffer();
-            }
-        } else if ((c & 0xf4000000) == 0) { // 5 bytes.
-            _bytes[_index] = (byte) (0xf8 | (c >> 24));
-            if (++_index >= _bytes.length) {
-                flushBuffer();
-            }
-            _bytes[_index] = (byte) (0x80 | ((c >> 18) & 0x3f));
-            if (++_index >= _bytes.length) {
-                flushBuffer();
-            }
-            _bytes[_index] = (byte) (0x80 | ((c >> 12) & 0x3f));
-            if (++_index >= _bytes.length) {
-                flushBuffer();
-            }
-            _bytes[_index] = (byte) (0x80 | ((c >> 6) & 0x3f));
-            if (++_index >= _bytes.length) {
-                flushBuffer();
-            }
-            _bytes[_index] = (byte) (0x80 | (c & 0x3f));
-            if (++_index >= _bytes.length) {
-                flushBuffer();
-            }
-        } else if ((c & 0x80000000) == 0) { // 6 bytes.
-            _bytes[_index] = (byte) (0xfc | (c >> 30));
-            if (++_index >= _bytes.length) {
-                flushBuffer();
-            }
-            _bytes[_index] = (byte) (0x80 | ((c >> 24) & 0x3f));
-            if (++_index >= _bytes.length) {
-                flushBuffer();
-            }
-            _bytes[_index] = (byte) (0x80 | ((c >> 18) & 0x3f));
-            if (++_index >= _bytes.length) {
-                flushBuffer();
-            }
-            _bytes[_index] = (byte) (0x80 | ((c >> 12) & 0x3F));
-            if (++_index >= _bytes.length) {
-                flushBuffer();
-            }
-            _bytes[_index] = (byte) (0x80 | ((c >> 6) & 0x3F));
-            if (++_index >= _bytes.length) {
-                flushBuffer();
-            }
-            _bytes[_index] = (byte) (0x80 | (c & 0x3F));
-            if (++_index >= _bytes.length) {
-                flushBuffer();
-            }
-        } else {
-            throw new CharConversionException("Illegal character U+" + Integer.toHexString(c));
-        }
-    }
-
-    /**
-     * Writes a portion of an array of characters.
-     *
-     * @param cbuf the array of characters.
-     * @param off  the offset from which to start writing characters.
-     * @param len  the number of characters to write.
-     * @throws IOException if an I/O error occurs.
-     */
-    @Override
-    public void write(char cbuf[], int off, int len) throws IOException {
-        final int off_plus_len = off + len;
-        for (int i = off; i < off_plus_len;) {
-            char c = cbuf[i++];
-            if (c < 0x80) {
-                _bytes[_index] = (byte) c;
-                if (++_index >= _bytes.length) {
-                    flushBuffer();
-                }
-            } else {
-                write(c);
-            }
-        }
-    }
-
-    /**
-     * Writes a portion of a string.
-     *
-     * @param str a String.
-     * @param off the offset from which to start writing characters.
-     * @param len the number of characters to write.
-     * @throws IOException if an I/O error occurs
-     */
-    @Override
-    public void write(String str, int off, int len) throws IOException {
-        final int off_plus_len = off + len;
-        for (int i = off; i < off_plus_len;) {
-            char c = str.charAt(i++);
-            if (c < 0x80) {
-                _bytes[_index] = (byte) c;
-                if (++_index >= _bytes.length) {
-                    flushBuffer();
-                }
-            } else {
-                write(c);
-            }
-        }
-    }
-
-    /**
-     * Flushes the stream.  If the stream has saved any characters from the
-     * various write() methods in a buffer, write them immediately to their
-     * intended destination.  Then, if that destination is another character or
-     * byte stream, flush it.  Thus one flush() invocation will flush all the
-     * buffers in a chain of Writers and OutputStreams.
-     *
-     * @throws IOException if an I/O error occurs.
-     */
-    @Override
-    public void flush() throws IOException {
-        flushBuffer();
-        _outputStream.flush();
-    }
-
-    /**
-     * Closes and {@link #reset resets} this writer for reuse.
-     *
-     * @throws IOException if an I/O error occurs
-     */
-    @Override
-    public void close() throws IOException {
-        if (_outputStream != null) {
-            flushBuffer();
-            _outputStream.close();
-            reset();
-        }
-    }
-
-    /**
-     * Flushes the internal bytes buffer.
-     *
-     * @throws IOException if an I/O error occurs
-     */
-    private void flushBuffer() throws IOException {
-        if (_outputStream == null) throw new IOException("Stream closed");
-        _outputStream.write(_bytes, 0, _index);
-        _index = 0;
-    }
-
-    // Implements Reusable.
-    public void reset() {
-        _highSurrogate = 0;
-        _index = 0;
-        _outputStream = null;
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java b/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java
index 5c637f27dfd1..8ec408c2b08c 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.common.io.stream;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.bytes.ReleasableBytesReference;
 import org.elasticsearch.core.Releasable;
@@ -49,12 +49,12 @@ public static <T extends Writeable> DelayableWriteable<T> referencing(T referenc
      * when {@link #expand()} is called.
      */
     public static <T extends Writeable> DelayableWriteable<T> delayed(Writeable.Reader<T> reader, StreamInput in) throws IOException {
-        return new Serialized<>(reader, in.getVersion(), in.namedWriteableRegistry(), in.readReleasableBytesReference());
+        return new Serialized<>(reader, in.getTransportVersion(), in.namedWriteableRegistry(), in.readReleasableBytesReference());
     }
 
     public static <T extends Writeable> DelayableWriteable<T> referencing(Writeable.Reader<T> reader, StreamInput in) throws IOException {
         try (ReleasableBytesReference serialized = in.readReleasableBytesReference()) {
-            return new Referencing<>(deserialize(reader, in.getVersion(), in.namedWriteableRegistry(), serialized));
+            return new Referencing<>(deserialize(reader, in.getTransportVersion(), in.namedWriteableRegistry(), serialized));
         }
     }
 
@@ -103,12 +103,12 @@ public T expand() {
         public Serialized<T> asSerialized(Reader<T> reader, NamedWriteableRegistry registry) {
             BytesStreamOutput buffer;
             try {
-                buffer = writeToBuffer(Version.CURRENT);
+                buffer = writeToBuffer(TransportVersion.CURRENT);
             } catch (IOException e) {
                 throw new RuntimeException("unexpected error writing writeable to buffer", e);
             }
             // TODO: this path is currently not used in production code, if it ever is this should start using pooled buffers
-            return new Serialized<>(reader, Version.CURRENT, registry, ReleasableBytesReference.wrap(buffer.bytes()));
+            return new Serialized<>(reader, TransportVersion.CURRENT, registry, ReleasableBytesReference.wrap(buffer.bytes()));
         }
 
         @Override
@@ -121,9 +121,9 @@ public long getSerializedSize() {
             return DelayableWriteable.getSerializedSize(reference);
         }
 
-        private BytesStreamOutput writeToBuffer(Version version) throws IOException {
+        private BytesStreamOutput writeToBuffer(TransportVersion version) throws IOException {
             try (BytesStreamOutput buffer = new BytesStreamOutput()) {
-                buffer.setVersion(version);
+                buffer.setTransportVersion(version);
                 reference.writeTo(buffer);
                 return buffer;
             }
@@ -141,13 +141,13 @@ public void close() {
      */
     public static class Serialized<T extends Writeable> extends DelayableWriteable<T> {
         private final Writeable.Reader<T> reader;
-        private final Version serializedAtVersion;
+        private final TransportVersion serializedAtVersion;
         private final NamedWriteableRegistry registry;
         private final ReleasableBytesReference serialized;
 
         private Serialized(
             Writeable.Reader<T> reader,
-            Version serializedAtVersion,
+            TransportVersion serializedAtVersion,
             NamedWriteableRegistry registry,
             ReleasableBytesReference serialized
         ) {
@@ -159,7 +159,7 @@ private Serialized(
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            if (out.getVersion() == serializedAtVersion) {
+            if (out.getTransportVersion() == serializedAtVersion) {
                 /*
                  * If the version *does* line up we can just copy the bytes
                  * which is good because this is how shard request caching
@@ -214,7 +214,7 @@ public void close() {
      */
     public static long getSerializedSize(Writeable ref) {
         try (CountingStreamOutput out = new CountingStreamOutput()) {
-            out.setVersion(Version.CURRENT);
+            out.setTransportVersion(TransportVersion.CURRENT);
             ref.writeTo(out);
             return out.size;
         } catch (IOException exc) {
@@ -224,7 +224,7 @@ public static long getSerializedSize(Writeable ref) {
 
     private static <T> T deserialize(
         Reader<T> reader,
-        Version serializedAtVersion,
+        TransportVersion serializedAtVersion,
         NamedWriteableRegistry registry,
         BytesReference serialized
     ) throws IOException {
@@ -233,7 +233,7 @@ private static <T> T deserialize(
                 ? serialized.streamInput()
                 : new NamedWriteableAwareStreamInput(serialized.streamInput(), registry)
         ) {
-            in.setVersion(serializedAtVersion);
+            in.setTransportVersion(serializedAtVersion);
             return reader.read(in);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java
index 79fa2995146b..beed68a6ab5b 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/FilterStreamInput.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.common.io.stream;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.ReleasableBytesReference;
 
@@ -86,13 +87,29 @@ public int available() throws IOException {
     }
 
     @Override
+    @Deprecated(forRemoval = true)
     public Version getVersion() {
         return delegate.getVersion();
     }
 
     @Override
+    public TransportVersion getTransportVersion() {
+        return delegate.getTransportVersion();
+    }
+
+    @Override
+    @Deprecated(forRemoval = true)
     public void setVersion(Version version) {
         delegate.setVersion(version);
+        // also set the version on this stream directly, so that any uses of this.version are still correct
+        super.setVersion(version);
+    }
+
+    @Override
+    public void setTransportVersion(TransportVersion version) {
+        delegate.setTransportVersion(version);
+        // also set the version on this stream directly, so that any uses of this.version are still correct
+        super.setTransportVersion(version);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java
index 402397e22ced..439900122c83 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/InputStreamStreamInput.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.common.io.stream;
 
-import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.core.Streams;
 
 import java.io.EOFException;
 import java.io.IOException;
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/RecyclerBytesStreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/RecyclerBytesStreamOutput.java
index 7c372f6b52bf..4ebebdbd8e9b 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/RecyclerBytesStreamOutput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/RecyclerBytesStreamOutput.java
@@ -127,7 +127,7 @@ public void writeWithSizePrefix(Writeable writeable) throws IOException {
         // manipulation of the offsets on the pages after writing to tmp. This will require adjustments to the places in this class
         // that make assumptions about the page size
         try (RecyclerBytesStreamOutput tmp = new RecyclerBytesStreamOutput(recycler)) {
-            tmp.setVersion(getVersion());
+            tmp.setTransportVersion(getTransportVersion());
             writeable.writeTo(tmp);
             int size = tmp.size();
             writeVInt(size);
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
index 05753b46fa12..faea2ad8bc86 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
@@ -17,6 +17,7 @@
 import org.apache.lucene.util.BitUtil;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -83,19 +84,35 @@
  */
 public abstract class StreamInput extends InputStream {
 
-    private Version version = Version.CURRENT;
+    private TransportVersion version = TransportVersion.CURRENT;
 
     /**
      * The version of the node on the other side of this stream.
      */
+    @Deprecated(forRemoval = true)
     public Version getVersion() {
+        return Version.fromId(this.version.id);
+    }
+
+    /**
+     * The transport version the data is serialized as.
+     */
+    public TransportVersion getTransportVersion() {
         return this.version;
     }
 
     /**
      * Set the version of the node on the other side of this stream.
      */
+    @Deprecated(forRemoval = true)
     public void setVersion(Version version) {
+        this.version = version.transportVersion;
+    }
+
+    /**
+     * Set the transport version of the data in this stream.
+     */
+    public void setTransportVersion(TransportVersion version) {
         this.version = version;
     }
 
@@ -599,6 +616,20 @@ public String[] readOptionalStringArray() throws IOException {
         return null;
     }
 
+    /**
+     * Reads an optional byte array. It's effectively the same as readByteArray, except
+     * it supports null.
+     * @return a byte array or null
+     * @throws IOException
+     */
+    @Nullable
+    public byte[] readOptionalByteArray() throws IOException {
+        if (readBoolean()) {
+            return readByteArray();
+        }
+        return null;
+    }
+
     /**
      * If the returned map contains any entries it will be mutable. If it is empty it might be immutable.
      */
@@ -738,8 +769,12 @@ public Object readGenericValue() throws IOException {
             case 6 -> readByteArray();
             case 7 -> readArrayList();
             case 8 -> readArray();
-            case 9 -> readLinkedHashMap();
-            case 10 -> readHashMap();
+            case 9 -> getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)
+                ? readOrderedMap(StreamInput::readGenericValue, StreamInput::readGenericValue)
+                : readOrderedMap(StreamInput::readString, StreamInput::readGenericValue);
+            case 10 -> getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)
+                ? readMap(StreamInput::readGenericValue, StreamInput::readGenericValue)
+                : readMap(StreamInput::readString, StreamInput::readGenericValue);
             case 11 -> readByte();
             case 12 -> readDate();
             case 13 ->
@@ -817,30 +852,6 @@ private Object[] readArray() throws IOException {
         return list8;
     }
 
-    private Map<String, Object> readLinkedHashMap() throws IOException {
-        int size9 = readArraySize();
-        if (size9 == 0) {
-            return Collections.emptyMap();
-        }
-        Map<String, Object> map9 = Maps.newLinkedHashMapWithExpectedSize(size9);
-        for (int i = 0; i < size9; i++) {
-            map9.put(readString(), readGenericValue());
-        }
-        return map9;
-    }
-
-    private Map<String, Object> readHashMap() throws IOException {
-        int size10 = readArraySize();
-        if (size10 == 0) {
-            return Collections.emptyMap();
-        }
-        Map<String, Object> map10 = Maps.newMapWithExpectedSize(size10);
-        for (int i = 0; i < size10; i++) {
-            map10.put(readString(), readGenericValue());
-        }
-        return map10;
-    }
-
     private Date readDate() throws IOException {
         return new Date(readLong());
     }
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
index 2465d211ef95..19b792f33254 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
@@ -17,6 +17,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -77,19 +78,35 @@ public abstract class StreamOutput extends OutputStream {
 
     private static final int MAX_NESTED_EXCEPTION_LEVEL = 100;
 
-    private Version version = Version.CURRENT;
+    private TransportVersion version = TransportVersion.CURRENT;
 
     /**
      * The version of the node on the other side of this stream.
      */
+    @Deprecated(forRemoval = true)
     public Version getVersion() {
+        return Version.fromId(this.version.id);
+    }
+
+    /**
+     * The transport version to serialize the data as.
+     */
+    public TransportVersion getTransportVersion() {
         return this.version;
     }
 
     /**
      * Set the version of the node on the other side of this stream.
      */
+    @Deprecated(forRemoval = true)
     public void setVersion(Version version) {
+        this.version = version.transportVersion;
+    }
+
+    /**
+     * Set the transport version of the data in this stream.
+     */
+    public void setTransportVersion(TransportVersion version) {
         this.version = version;
     }
 
@@ -147,7 +164,7 @@ public void writeByteArray(byte[] b) throws IOException {
      */
     public void writeWithSizePrefix(Writeable writeable) throws IOException {
         final BytesStreamOutput tmp = new BytesStreamOutput();
-        tmp.setVersion(version);
+        tmp.setTransportVersion(version);
         writeable.writeTo(tmp);
         writeBytesReference(tmp.bytes());
     }
@@ -537,6 +554,19 @@ public void writeOptionalStringArray(@Nullable String[] array) throws IOExceptio
         }
     }
 
+    /**
+     * Writes a byte array, for null arrays it writes false.
+     * @param array an array or null
+     */
+    public void writeOptionalByteArray(@Nullable byte[] array) throws IOException {
+        if (array == null) {
+            writeBoolean(false);
+        } else {
+            writeBoolean(true);
+            writeByteArray(array);
+        }
+    }
+
     public void writeGenericMap(@Nullable Map<String, Object> map) throws IOException {
         writeGenericValue(map);
     }
@@ -561,7 +591,11 @@ public void writeMapWithConsistentOrder(@Nullable Map<String, ? extends Object>
             .iterator();
         while (iterator.hasNext()) {
             Map.Entry<String, ?> next = iterator.next();
-            this.writeString(next.getKey());
+            if (this.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+                this.writeGenericValue(next.getKey());
+            } else {
+                this.writeString(next.getKey());
+            }
             this.writeGenericValue(next.getValue());
         }
     }
@@ -688,12 +722,13 @@ public final void writeOptionalInstant(@Nullable Instant instant) throws IOExcep
             } else {
                 o.writeByte((byte) 10);
             }
-            @SuppressWarnings("unchecked")
-            final Map<String, Object> map = (Map<String, Object>) v;
-            o.writeVInt(map.size());
-            for (Map.Entry<String, Object> entry : map.entrySet()) {
-                o.writeString(entry.getKey());
-                o.writeGenericValue(entry.getValue());
+            if (o.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+                final Map<?, ?> map = (Map<?, ?>) v;
+                o.writeMap(map, StreamOutput::writeGenericValue, StreamOutput::writeGenericValue);
+            } else {
+                @SuppressWarnings("unchecked")
+                final Map<String, ?> map = (Map<String, ?>) v;
+                o.writeMap(map, StreamOutput::writeString, StreamOutput::writeGenericValue);
             }
         }),
         entry(Byte.class, (o, v) -> {
@@ -956,6 +991,14 @@ public void writeOptionalWriteable(@Nullable Writeable writeable) throws IOExcep
         }
     }
 
+    /**
+     * This method allow to use a method reference when writing collection elements such as
+     * {@code out.writeMap(map, StreamOutput::writeString, StreamOutput::writeWriteable)}
+     */
+    public void writeWriteable(Writeable writeable) throws IOException {
+        writeable.writeTo(this);
+    }
+
     public void writeException(Throwable throwable) throws IOException {
         writeException(throwable, throwable, 0);
     }
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/VersionCheckingStreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/VersionCheckingStreamOutput.java
index a686f35f394a..78cb36e9a6c9 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/VersionCheckingStreamOutput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/VersionCheckingStreamOutput.java
@@ -62,14 +62,14 @@ public void writeOptionalNamedWriteable(@Nullable NamedWriteable namedWriteable)
     }
 
     private void checkVersionCompatibility(VersionedNamedWriteable namedWriteable) {
-        if (namedWriteable.getMinimalSupportedVersion().after(getVersion())) {
+        if (namedWriteable.getMinimalSupportedVersion().after(getTransportVersion())) {
             throw new IllegalArgumentException(
                 "["
                     + namedWriteable.getWriteableName()
                     + "] was released first in version "
                     + namedWriteable.getMinimalSupportedVersion()
                     + ", failed compatibility check trying to send it to node with version "
-                    + getVersion()
+                    + getTransportVersion()
             );
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/VersionedNamedWriteable.java b/server/src/main/java/org/elasticsearch/common/io/stream/VersionedNamedWriteable.java
index 3b0e971225aa..1b25b0969488 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/VersionedNamedWriteable.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/VersionedNamedWriteable.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.common.io.stream;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 
 import java.io.IOException;
 import java.util.Map;
@@ -26,7 +26,7 @@ public interface VersionedNamedWriteable extends NamedWriteable {
     /**
      * The minimal version of the recipient this object can be sent to
      */
-    Version getMinimalSupportedVersion();
+    TransportVersion getMinimalSupportedVersion();
 
     /**
      * Tests whether or not the custom should be serialized. The criteria is the output stream must be at least the minimum supported
@@ -39,7 +39,7 @@ public interface VersionedNamedWriteable extends NamedWriteable {
      * @return true if the custom should be serialized and false otherwise
      */
     static <T extends VersionedNamedWriteable> boolean shouldSerialize(final StreamOutput out, final T custom) {
-        return out.getVersion().onOrAfter(custom.getMinimalSupportedVersion());
+        return out.getTransportVersion().onOrAfter(custom.getMinimalSupportedVersion());
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/common/logging/ChunkedLoggingStream.java b/server/src/main/java/org/elasticsearch/common/logging/ChunkedLoggingStream.java
new file mode 100644
index 000000000000..941a6d1f9eb3
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/logging/ChunkedLoggingStream.java
@@ -0,0 +1,148 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common.logging;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.ReferenceDocs;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Objects;
+import java.util.zip.GZIPOutputStream;
+
+/**
+ * An {@link OutputStream} which Gzip-compresses the written data, Base64-encodes it, and writes it in fixed-size chunks to a logger. This
+ * is useful for debugging information that may be too large for a single log message and/or which may include data which cannot be
+ * recorded faithfully in plain-text (e.g. binary data or data with significant whitespace).
+ */
+public class ChunkedLoggingStream extends OutputStream {
+
+    static final int CHUNK_SIZE = ByteSizeUnit.KB.toIntBytes(2);
+
+    /**
+     * Create an {@link OutputStream} which Gzip-compresses the written data, Base64-encodes it, and writes it in fixed-size (2kiB) chunks
+     * to the given logger. If the data fits into a single chunk then the output looks like this:
+     *
+     * <pre>
+     * $PREFIX (gzip compressed and base64-encoded; for details see ...): H4sIAAAAA...
+     * </pre>
+     *
+     * If there are multiple chunks then they are written like this:
+     *
+     * <pre>
+     * $PREFIX [part 1]: H4sIAAAAA...
+     * $PREFIX [part 2]: r38c4MBHO...
+     * $PREFIX [part 3]: ECyRFONaL...
+     * $PREFIX [part 4]: kTgm+Qswm...
+     * $PREFIX (gzip compressed, base64-encoded, and split into 4 parts on preceding log lines; for details see ...)
+     * </pre>
+     *
+     * @param logger        The logger to receive the chunks of data.
+     * @param level         The log level to use for the logging.
+     * @param prefix        A prefix for each chunk, which should be reasonably unique to allow for reconstruction of the original message
+     *                      even if multiple such streams are used concurrently.
+     * @param referenceDocs A link to the relevant reference docs to help users interpret the output. Relevant reference docs are required
+     *                      because the output is rather human-unfriendly and we need somewhere to describe how to decode it.
+     */
+    public static OutputStream create(Logger logger, Level level, String prefix, ReferenceDocs referenceDocs) throws IOException {
+        return new GZIPOutputStream(Base64.getEncoder().wrap(new ChunkedLoggingStream(logger, level, prefix, referenceDocs)));
+    }
+
+    private final Logger logger;
+    private final Level level;
+    private final String prefix;
+    private final ReferenceDocs referenceDocs;
+
+    private int chunk;
+    private int offset;
+    private boolean closed;
+    private final byte[] buffer = new byte[CHUNK_SIZE];
+
+    ChunkedLoggingStream(Logger logger, Level level, String prefix, ReferenceDocs referenceDocs) {
+        this.logger = Objects.requireNonNull(logger);
+        this.level = Objects.requireNonNull(level);
+        this.prefix = Objects.requireNonNull(prefix);
+        this.referenceDocs = Objects.requireNonNull(referenceDocs);
+    }
+
+    private void flushBuffer() {
+        assert closed || offset == CHUNK_SIZE : offset;
+        assert offset >= 0 && offset <= CHUNK_SIZE : offset;
+        chunk += 1;
+
+        final var chunkString = new String(buffer, 0, offset, StandardCharsets.ISO_8859_1);
+        offset = 0;
+
+        if (closed && chunk == 1) {
+            logger.log(level, "{} (gzip compressed and base64-encoded; for details see {}): {}", prefix, referenceDocs, chunkString);
+        } else {
+            logger.log(level, "{} [part {}]: {}", prefix, chunk, chunkString);
+        }
+    }
+
+    @Override
+    public void write(int b) throws IOException {
+        assert closed == false;
+        if (offset == CHUNK_SIZE) {
+            flushBuffer();
+        }
+        buffer[offset] = (byte) b;
+        assert assertSafeByte(buffer[offset]);
+        offset += 1;
+    }
+
+    @Override
+    public void write(byte[] b, int off, int len) throws IOException {
+        assert closed == false;
+        assert assertSafeBytes(b, off, len);
+        while (len > 0) {
+            if (offset == CHUNK_SIZE) {
+                flushBuffer();
+            }
+            var copyLen = Math.min(len, CHUNK_SIZE - offset);
+            System.arraycopy(b, off, buffer, offset, copyLen);
+            offset += copyLen;
+            off += copyLen;
+            len -= copyLen;
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+        if (closed == false) {
+            closed = true;
+            flushBuffer();
+            if (chunk > 1) {
+                logger.log(
+                    level,
+                    "{} (gzip compressed, base64-encoded, and split into {} parts on preceding log lines; for details see {})",
+                    prefix,
+                    chunk,
+                    referenceDocs
+                );
+            }
+        }
+    }
+
+    private static boolean assertSafeBytes(byte[] b, int off, int len) {
+        for (int i = off; i < off + len; i++) {
+            assertSafeByte(b[i]);
+        }
+        return true;
+    }
+
+    private static boolean assertSafeByte(byte b) {
+        assert 0x20 <= b && b < 0x7f;
+        return true;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java b/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
index 2147b7a5531b..55c1f645a965 100644
--- a/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
+++ b/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
@@ -30,6 +30,7 @@
 import org.apache.logging.log4j.status.StatusData;
 import org.apache.logging.log4j.status.StatusListener;
 import org.apache.logging.log4j.status.StatusLogger;
+import org.apache.logging.log4j.util.Unbox;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.logging.internal.LoggerFactoryImpl;
 import org.elasticsearch.common.settings.Settings;
@@ -41,6 +42,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PrintStream;
+import java.lang.invoke.MethodHandles;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.FileVisitOption;
 import java.nio.file.FileVisitResult;
@@ -122,6 +124,7 @@ public static void configure(final Environment environment, boolean useConsole)
         }
         configureESLogging();
         configure(environment.settings(), environment.configFile(), environment.logsFile(), useConsole);
+        initializeStatics();
     }
 
     public static void configureESLogging() {
@@ -144,6 +147,17 @@ public static void setNodeName(String nodeName) {
         NodeNamePatternConverter.setNodeName(nodeName);
     }
 
+    // Some classes within log4j have static initializers that require security manager permissions.
+    // Here we aggressively initialize those classes during logging configuration so that
+    // actual logging calls at runtime do not trigger that initialization.
+    private static void initializeStatics() {
+        try {
+            MethodHandles.publicLookup().ensureInitialized(Unbox.class);
+        } catch (IllegalAccessException impossible) {
+            throw new AssertionError(impossible);
+        }
+    }
+
     private static void checkErrorListener() {
         assert errorListenerIsRegistered() : "expected error listener to be registered";
         if (error.get()) {
@@ -246,8 +260,22 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr
         // Redirect stdout/stderr to log4j. While we ensure Elasticsearch code does not write to those streams,
         // third party libraries may do that. Note that we do NOT close the streams because other code may have
         // grabbed a handle to the streams and intend to write to it, eg log4j for writing to the console
-        System.setOut(new PrintStream(new LoggingOutputStream(LogManager.getLogger("stdout"), Level.INFO), false, StandardCharsets.UTF_8));
-        System.setErr(new PrintStream(new LoggingOutputStream(LogManager.getLogger("stderr"), Level.WARN), false, StandardCharsets.UTF_8));
+        System.setOut(
+            new PrintStream(new LoggingOutputStream(LogManager.getLogger("stdout"), Level.INFO, List.of()), false, StandardCharsets.UTF_8)
+        );
+        System.setErr(
+            new PrintStream(
+                new LoggingOutputStream(
+                    LogManager.getLogger("stderr"),
+                    Level.WARN,
+                    // MMapDirectory messages come from Lucene, suggesting to users as a warning that they should enable preview features in
+                    // the JDK
+                    List.of("MMapDirectory")
+                ),
+                false,
+                StandardCharsets.UTF_8
+            )
+        );
 
         final Logger rootLogger = LogManager.getRootLogger();
         Appender appender = Loggers.findAppender(rootLogger, ConsoleAppender.class);
diff --git a/server/src/main/java/org/elasticsearch/common/logging/LoggingOutputStream.java b/server/src/main/java/org/elasticsearch/common/logging/LoggingOutputStream.java
index 2f38986716df..6fd98259d925 100644
--- a/server/src/main/java/org/elasticsearch/common/logging/LoggingOutputStream.java
+++ b/server/src/main/java/org/elasticsearch/common/logging/LoggingOutputStream.java
@@ -15,6 +15,7 @@
 import java.io.OutputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
+import java.util.List;
 
 /**
  * A stream whose output is sent to the configured logger, line by line.
@@ -42,9 +43,12 @@ static class Buffer {
 
     private final Level level;
 
-    LoggingOutputStream(Logger logger, Level level) {
+    private final List<String> messageFilters;
+
+    LoggingOutputStream(Logger logger, Level level, List<String> messageFilters) {
         this.logger = logger;
         this.level = level;
+        this.messageFilters = messageFilters;
     }
 
     @Override
@@ -103,8 +107,17 @@ public void close() {
         threadLocal = null;
     }
 
+    private void log(String msg) {
+        for (String filter : messageFilters) {
+            if (msg.contains(filter)) {
+                return;
+            }
+        }
+        this.log0(msg);
+    }
+
     // pkg private for testing
-    void log(String msg) {
+    protected void log0(String msg) {
         logger.log(level, msg);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
index cf31d3ff1005..a660730bfb02 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
@@ -86,7 +86,7 @@
 import java.util.Map;
 
 public class Lucene {
-    public static final String LATEST_CODEC = "Lucene94";
+    public static final String LATEST_CODEC = "Lucene95";
 
     public static final String SOFT_DELETES_FIELD = "__soft_deletes";
 
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java b/server/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java
index 5c411c7b397c..1293c59486e7 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java
@@ -299,7 +299,29 @@ private static boolean termArraysEquals(List<Term[]> termArrays1, List<Term[]> t
     @Override
     public void visit(QueryVisitor visitor) {
         if (visitor.acceptField(field)) {
-            visitor.visitLeaf(this);    // TODO implement term visiting
+            visitor = visitor.getSubVisitor(BooleanClause.Occur.MUST, this);
+            for (int i = 0; i < termArrays.size() - 1; i++) {
+                if (termArrays.get(i).length == 1) {
+                    visitor.consumeTerms(this, termArrays.get(i)[0]);
+                } else {
+                    QueryVisitor shouldVisitor = visitor.getSubVisitor(BooleanClause.Occur.SHOULD, this);
+                    shouldVisitor.consumeTerms(this, termArrays.get(i));
+                }
+            }
+            /* We don't report automata here because this breaks the unified highlighter,
+               which extracts automata separately from phrases. MPPQ gets rewritten to a
+               SpanMTQQuery by the PhraseHelper in any case, so highlighting is taken
+               care of there instead.  If we extract automata here then the trailing prefix
+               word will be highlighted wherever it appears in the document, instead of only
+               as part of a phrase. This can be re-instated once we switch to using Matches
+               to highlight.
+            for (Term prefixTerm : termArrays.get(termArrays.size() - 1)) {
+                visitor.consumeTermsMatching(this, field, () -> {
+                    CompiledAutomaton ca = new CompiledAutomaton(PrefixQuery.toAutomaton(prefixTerm.bytes()));
+                    return ca.runAutomaton;
+                });
+            }
+            */
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/store/ByteArrayIndexInput.java b/server/src/main/java/org/elasticsearch/common/lucene/store/ByteArrayIndexInput.java
index 24ac1b32bc3b..e0dc4c941187 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/store/ByteArrayIndexInput.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/store/ByteArrayIndexInput.java
@@ -9,6 +9,7 @@
 
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.BitUtil;
+import org.elasticsearch.common.Strings;
 
 import java.io.EOFException;
 import java.io.IOException;
@@ -65,16 +66,14 @@ public IndexInput slice(String sliceDescription, long offset, long length) throw
             return new ByteArrayIndexInput(sliceDescription, bytes, this.offset + (int) offset, (int) length);
         } else {
             throw new IllegalArgumentException(
-                "slice() "
-                    + sliceDescription
-                    + " out of bounds: offset="
-                    + offset
-                    + ",length="
-                    + length
-                    + ",fileLength="
-                    + this.length
-                    + ": "
-                    + this
+                Strings.format(
+                    "slice() %s out of bounds: offset=%d,length=%d,fileLength=%d: %s",
+                    sliceDescription,
+                    offset,
+                    length,
+                    this.length,
+                    this
+                )
             );
         }
     }
@@ -99,7 +98,7 @@ public void readBytes(final byte[] b, final int offset, int len) throws IOExcept
     @Override
     public short readShort() throws IOException {
         try {
-            return (short) BitUtil.VH_LE_SHORT.get(bytes, pos);
+            return (short) BitUtil.VH_LE_SHORT.get(bytes, pos + offset);
         } finally {
             pos += Short.BYTES;
         }
@@ -108,7 +107,7 @@ public short readShort() throws IOException {
     @Override
     public int readInt() throws IOException {
         try {
-            return (int) BitUtil.VH_LE_INT.get(bytes, pos);
+            return (int) BitUtil.VH_LE_INT.get(bytes, pos + offset);
         } finally {
             pos += Integer.BYTES;
         }
@@ -117,7 +116,7 @@ public int readInt() throws IOException {
     @Override
     public long readLong() throws IOException {
         try {
-            return (long) BitUtil.VH_LE_LONG.get(bytes, pos);
+            return (long) BitUtil.VH_LE_LONG.get(bytes, pos + offset);
         } finally {
             pos += Long.BYTES;
         }
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/store/BytesReferenceIndexInput.java b/server/src/main/java/org/elasticsearch/common/lucene/store/BytesReferenceIndexInput.java
new file mode 100644
index 000000000000..b40dc36e1111
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/lucene/store/BytesReferenceIndexInput.java
@@ -0,0 +1,139 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common.lucene.store;
+
+import org.apache.lucene.store.IndexInput;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+public class BytesReferenceIndexInput extends IndexInput {
+
+    private final BytesReference bytesReference;
+
+    private int filePointer;
+    private StreamInput streamInput;
+
+    public BytesReferenceIndexInput(String resourceDescription, BytesReference bytesReference) {
+        this(resourceDescription, bytesReference, 0);
+    }
+
+    private BytesReferenceIndexInput(String resourceDescription, BytesReference bytesReference, int filePointer) {
+        super(resourceDescription);
+        this.bytesReference = bytesReference;
+        this.filePointer = filePointer;
+    }
+
+    @Override
+    public void close() throws IOException {}
+
+    @Override
+    public long getFilePointer() {
+        return filePointer;
+    }
+
+    private StreamInput getOrOpenStreamInput() throws IOException {
+        if (streamInput == null) {
+            streamInput = bytesReference.slice(filePointer, bytesReference.length() - filePointer).streamInput();
+        }
+        return streamInput;
+    }
+
+    @Override
+    public void seek(long longPos) throws IOException {
+        if (longPos < 0) {
+            throw new IllegalArgumentException("Seeking to negative position: " + longPos);
+        } else if (longPos > bytesReference.length()) {
+            throw new EOFException("seek past EOF");
+        }
+        var pos = (int) longPos;
+        if (pos < filePointer) {
+            streamInput = null;
+        } else if (streamInput != null) {
+            final var toSkip = pos - filePointer;
+            final var skipped = streamInput.skip(toSkip);
+            assert skipped == toSkip;
+        }
+        filePointer = pos;
+    }
+
+    @Override
+    public long length() {
+        return bytesReference.length();
+    }
+
+    @Override
+    public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
+        if (offset >= 0L && length >= 0L && offset + length <= bytesReference.length()) {
+            return new BytesReferenceIndexInput(sliceDescription, bytesReference.slice((int) offset, (int) length));
+        } else {
+            throw new IllegalArgumentException(
+                Strings.format(
+                    "slice() %s out of bounds: offset=%d,length=%d,fileLength=%d: %s",
+                    sliceDescription,
+                    offset,
+                    length,
+                    bytesReference.length(),
+                    this
+                )
+            );
+        }
+    }
+
+    @Override
+    public byte readByte() throws IOException {
+        try {
+            return getOrOpenStreamInput().readByte();
+        } finally {
+            filePointer += 1;
+        }
+    }
+
+    @Override
+    public void readBytes(byte[] b, int offset, int len) throws IOException {
+        getOrOpenStreamInput().readBytes(b, offset, len);
+        filePointer += len;
+    }
+
+    @Override
+    public short readShort() throws IOException {
+        try {
+            return Short.reverseBytes(getOrOpenStreamInput().readShort());
+        } finally {
+            filePointer += Short.BYTES;
+        }
+    }
+
+    @Override
+    public int readInt() throws IOException {
+        try {
+            return Integer.reverseBytes(getOrOpenStreamInput().readInt());
+        } finally {
+            filePointer += Integer.BYTES;
+        }
+    }
+
+    @Override
+    public long readLong() throws IOException {
+        try {
+            return Long.reverseBytes(getOrOpenStreamInput().readLong());
+        } finally {
+            filePointer += Long.BYTES;
+        }
+    }
+
+    @SuppressWarnings("MethodDoesntCallSuperMethod")
+    @Override
+    public IndexInput clone() {
+        return new BytesReferenceIndexInput(toString(), bytesReference, filePointer);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/store/InputStreamIndexInput.java b/server/src/main/java/org/elasticsearch/common/lucene/store/InputStreamIndexInput.java
index 04fe48c01607..c7abaa781792 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/store/InputStreamIndexInput.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/store/InputStreamIndexInput.java
@@ -43,6 +43,9 @@ public int read(byte[] b, int off, int len) throws IOException {
         } else if (off < 0 || len < 0 || len > b.length - off) {
             throw new IndexOutOfBoundsException();
         }
+        if (len == 0) {
+            return 0;
+        }
         if (indexInput.getFilePointer() >= indexInput.length()) {
             return -1;
         }
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java b/server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java
index 50c6da7418d8..a60c3f10b215 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java
@@ -8,15 +8,18 @@
 
 package org.elasticsearch.common.lucene.uid;
 
+import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.PointValues;
 import org.apache.lucene.index.PostingsEnum;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndSeqNo;
 import org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndVersion;
@@ -52,10 +55,15 @@ final class PerThreadIDVersionAndSeqNoLookup {
     /** used for assertions to make sure class usage meets assumptions */
     private final Object readerKey;
 
+    final boolean loadedTimestampRange;
+    final long minTimestamp;
+    final long maxTimestamp;
+
     /**
      * Initialize lookup for the provided segment
      */
-    PerThreadIDVersionAndSeqNoLookup(LeafReader reader, String uidField, boolean trackReaderKey) throws IOException {
+    PerThreadIDVersionAndSeqNoLookup(LeafReader reader, String uidField, boolean trackReaderKey, boolean loadTimestampRange)
+        throws IOException {
         this.uidField = uidField;
         final Terms terms = reader.terms(uidField);
         if (terms == null) {
@@ -84,10 +92,20 @@ final class PerThreadIDVersionAndSeqNoLookup {
         Object readerKey = null;
         assert trackReaderKey ? (readerKey = reader.getCoreCacheHelper().getKey()) != null : readerKey == null;
         this.readerKey = readerKey;
+
+        this.loadedTimestampRange = loadTimestampRange;
+        if (loadTimestampRange) {
+            PointValues tsPointValues = reader.getPointValues(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD);
+            minTimestamp = LongPoint.decodeDimension(tsPointValues.getMinPackedValue(), 0);
+            maxTimestamp = LongPoint.decodeDimension(tsPointValues.getMaxPackedValue(), 0);
+        } else {
+            minTimestamp = 0;
+            maxTimestamp = Long.MAX_VALUE;
+        }
     }
 
-    PerThreadIDVersionAndSeqNoLookup(LeafReader reader, String uidField) throws IOException {
-        this(reader, uidField, true);
+    PerThreadIDVersionAndSeqNoLookup(LeafReader reader, String uidField, boolean loadTimestampRange) throws IOException {
+        this(reader, uidField, true, loadTimestampRange);
     }
 
     /** Return null if id is not found.
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java b/server/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java
index 7098e17bfeaa..b4433a36d498 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java
@@ -13,9 +13,12 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.util.CloseableThreadLocal;
+import org.elasticsearch.Assertions;
+import org.elasticsearch.common.util.ByteUtils;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 
 import java.io.IOException;
+import java.util.Base64;
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.ConcurrentMap;
@@ -34,7 +37,8 @@ public final class VersionsAndSeqNoResolver {
         }
     };
 
-    private static PerThreadIDVersionAndSeqNoLookup[] getLookupState(IndexReader reader, String uidField) throws IOException {
+    private static PerThreadIDVersionAndSeqNoLookup[] getLookupState(IndexReader reader, String uidField, boolean loadTimestampRange)
+        throws IOException {
         // We cache on the top level
         // This means cache entries have a shorter lifetime, maybe as low as 1s with the
         // default refresh interval and a steady indexing rate, but on the other hand it
@@ -59,9 +63,24 @@ private static PerThreadIDVersionAndSeqNoLookup[] getLookupState(IndexReader rea
         if (lookupState == null) {
             lookupState = new PerThreadIDVersionAndSeqNoLookup[reader.leaves().size()];
             for (LeafReaderContext leaf : reader.leaves()) {
-                lookupState[leaf.ord] = new PerThreadIDVersionAndSeqNoLookup(leaf.reader(), uidField);
+                lookupState[leaf.ord] = new PerThreadIDVersionAndSeqNoLookup(leaf.reader(), uidField, loadTimestampRange);
             }
             ctl.set(lookupState);
+        } else {
+            if (Assertions.ENABLED) {
+                // Ensure cached lookup instances have loaded timestamp range if that was requested
+                for (PerThreadIDVersionAndSeqNoLookup lookup : lookupState) {
+                    if (lookup.loadedTimestampRange != loadTimestampRange) {
+                        throw new AssertionError(
+                            "Mismatch between lookup.loadedTimestampRange ["
+                                + lookup.loadedTimestampRange
+                                + "] and loadTimestampRange ["
+                                + loadTimestampRange
+                                + "]"
+                        );
+                    }
+                }
+            }
         }
 
         if (lookupState.length != reader.leaves().size()) {
@@ -117,8 +136,8 @@ public static class DocIdAndSeqNo {
      * <li>a doc ID and a version otherwise
      * </ul>
      */
-    public static DocIdAndVersion loadDocIdAndVersion(IndexReader reader, Term term, boolean loadSeqNo) throws IOException {
-        PerThreadIDVersionAndSeqNoLookup[] lookups = getLookupState(reader, term.field());
+    public static DocIdAndVersion timeSeriesLoadDocIdAndVersion(IndexReader reader, Term term, boolean loadSeqNo) throws IOException {
+        PerThreadIDVersionAndSeqNoLookup[] lookups = getLookupState(reader, term.field(), false);
         List<LeafReaderContext> leaves = reader.leaves();
         // iterate backwards to optimize for the frequently updated documents
         // which are likely to be in the last segments
@@ -133,11 +152,58 @@ public static DocIdAndVersion loadDocIdAndVersion(IndexReader reader, Term term,
         return null;
     }
 
+    /**
+     * A special variant of loading docid and version in case of time series indices.
+     * <p>
+     * Makes use of the fact that timestamp is part of the id, the existence of @timestamp field and
+     * that segments are sorted by {@link org.elasticsearch.cluster.metadata.DataStream#TIMESERIES_LEAF_READERS_SORTER}.
+     * This allows this method to know whether there is no document with the specified id without loading the docid for
+     * the specified id.
+     *
+     * @param reader    The reader load docid, version and seqno from.
+     * @param uid       The term that describes the uid of the document to load docid, version and seqno for.
+     * @param id        The id that contains the encoded timestamp. The timestamp is used to skip checking the id for entire segments.
+     * @param loadSeqNo Whether to load sequence number from _seq_no doc values field.
+     * @return the internal doc ID and version for the specified term from the specified reader or
+     *         returning <code>null</code> if no document was found for the specified id
+     * @throws IOException In case of an i/o related failure
+     */
+    public static DocIdAndVersion timeSeriesLoadDocIdAndVersion(IndexReader reader, Term uid, String id, boolean loadSeqNo)
+        throws IOException {
+        byte[] idAsBytes = Base64.getUrlDecoder().decode(id);
+        assert idAsBytes.length == 20;
+        // id format: [4 bytes (basic hash routing fields), 8 bytes prefix of 128 murmurhash dimension fields, 8 bytes
+        // @timestamp)
+        long timestamp = ByteUtils.readLongBE(idAsBytes, 12);
+
+        PerThreadIDVersionAndSeqNoLookup[] lookups = getLookupState(reader, uid.field(), true);
+        List<LeafReaderContext> leaves = reader.leaves();
+        // iterate in default order, the segments should be sorted by DataStream#TIMESERIES_LEAF_READERS_SORTER
+        long prevMaxTimestamp = Long.MAX_VALUE;
+        for (final LeafReaderContext leaf : leaves) {
+            PerThreadIDVersionAndSeqNoLookup lookup = lookups[leaf.ord];
+            assert lookup.loadedTimestampRange;
+            assert prevMaxTimestamp >= lookup.maxTimestamp;
+            if (timestamp < lookup.minTimestamp) {
+                continue;
+            }
+            if (timestamp > lookup.maxTimestamp) {
+                return null;
+            }
+            DocIdAndVersion result = lookup.lookupVersion(uid.bytes(), loadSeqNo, leaf);
+            if (result != null) {
+                return result;
+            }
+            prevMaxTimestamp = lookup.maxTimestamp;
+        }
+        return null;
+    }
+
     public static DocIdAndVersion loadDocIdAndVersionUncached(IndexReader reader, Term term, boolean loadSeqNo) throws IOException {
         List<LeafReaderContext> leaves = reader.leaves();
         for (int i = leaves.size() - 1; i >= 0; i--) {
             final LeafReaderContext leaf = leaves.get(i);
-            PerThreadIDVersionAndSeqNoLookup lookup = new PerThreadIDVersionAndSeqNoLookup(leaf.reader(), term.field(), false);
+            PerThreadIDVersionAndSeqNoLookup lookup = new PerThreadIDVersionAndSeqNoLookup(leaf.reader(), term.field(), false, false);
             DocIdAndVersion result = lookup.lookupVersion(term.bytes(), loadSeqNo, leaf);
             if (result != null) {
                 return result;
@@ -151,7 +217,7 @@ public static DocIdAndVersion loadDocIdAndVersionUncached(IndexReader reader, Te
      * The result is either null or the live and latest version of the given uid.
      */
     public static DocIdAndSeqNo loadDocIdAndSeqNo(IndexReader reader, Term term) throws IOException {
-        final PerThreadIDVersionAndSeqNoLookup[] lookups = getLookupState(reader, term.field());
+        final PerThreadIDVersionAndSeqNoLookup[] lookups = getLookupState(reader, term.field(), false);
         final List<LeafReaderContext> leaves = reader.leaves();
         // iterate backwards to optimize for the frequently updated documents
         // which are likely to be in the last segments
diff --git a/server/src/main/java/org/elasticsearch/common/metrics/Counters.java b/server/src/main/java/org/elasticsearch/common/metrics/Counters.java
index 2496f741dde2..4d34e6489f20 100644
--- a/server/src/main/java/org/elasticsearch/common/metrics/Counters.java
+++ b/server/src/main/java/org/elasticsearch/common/metrics/Counters.java
@@ -87,11 +87,11 @@ public boolean hasCounters() {
      *     },
      *     "foobar": 5
      * }
-     * @return A nested map with all the current configured counters
+     * @return A mutable nested map with all the current configured counters. The map is mutable to allow the client to further enrich it.
      * @throws IllegalStateException if there is a conflict in a path of two counters for example `foo`: 1 and `foo.bar`: 1.
      */
     @SuppressWarnings("unchecked")
-    public Map<String, Object> toNestedMap() {
+    public Map<String, Object> toMutableNestedMap() {
         Map<String, Object> root = new HashMap<>();
         for (var counter : counters.entrySet()) {
             Map<String, Object> currentLevel = root;
diff --git a/server/src/main/java/org/elasticsearch/common/regex/Regex.java b/server/src/main/java/org/elasticsearch/common/regex/Regex.java
index ff2ac47ad209..73975e6c91e6 100644
--- a/server/src/main/java/org/elasticsearch/common/regex/Regex.java
+++ b/server/src/main/java/org/elasticsearch/common/regex/Regex.java
@@ -11,13 +11,17 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.common.Strings;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
+import java.util.Set;
+import java.util.function.Predicate;
 import java.util.regex.Pattern;
 
 public class Regex {
@@ -90,6 +94,40 @@ public static Automaton simpleMatchToAutomaton(String... patterns) {
         return Operations.union(automata);
     }
 
+    /**
+     * Create a {@link Predicate} that matches the given patterns. Evaluating
+     * the returned predicate against a {@link String} yields the same result as
+     * running {@link #simpleMatch(String[], String)} but may run faster,
+     * especially in the case when there are multiple patterns.
+     */
+    public static Predicate<String> simpleMatcher(String... patterns) {
+        if (patterns == null || patterns.length == 0) {
+            return str -> false;
+        }
+        boolean hasWildcard = false;
+        for (String pattern : patterns) {
+            if (isMatchAllPattern(pattern)) {
+                return str -> true;
+            }
+            if (isSimpleMatchPattern(pattern)) {
+                hasWildcard = true;
+                break;
+            }
+        }
+        if (patterns.length == 1) {
+            if (hasWildcard) {
+                return str -> simpleMatch(patterns[0], str);
+            } else {
+                return patterns[0]::equals;
+            }
+        } else if (hasWildcard == false) {
+            return Set.copyOf(Arrays.asList(patterns))::contains;
+        } else {
+            Automaton automaton = simpleMatchToAutomaton(patterns);
+            return new CharacterRunAutomaton(automaton)::run;
+        }
+    }
+
     /**
      * Match a String against the given pattern, supporting the following simple
      * pattern styles: "xxx*", "*xxx", "*xxx*" and "xxx*yyy" matches (with an
diff --git a/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
index 579d4a7c9eea..f35228354404 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
@@ -434,6 +434,20 @@ public synchronized <T> void addSettingsUpdateConsumer(Setting<T> setting, Consu
         addSettingsUpdateConsumer(setting, consumer, (s) -> {});
     }
 
+    /**
+     * This methods passes the setting value to a consumer during the initialization and on every setting change
+     * <p>
+     * Note: Only settings registered in {@link org.elasticsearch.cluster.ClusterModule} can be changed dynamically.
+     * </p>
+     */
+    public synchronized <T> void initializeAndWatch(Setting<T> setting, Consumer<T> consumer) {
+        assert setting.getProperties().contains(Setting.Property.Dynamic)
+            || setting.getProperties().contains(Setting.Property.OperatorDynamic) : "Can only watch dynamic settings";
+        assert setting.getProperties().contains(Setting.Property.NodeScope) : "Can only watch node settings";
+        consumer.accept(setting.get(settings));
+        addSettingsUpdateConsumer(setting, consumer);
+    }
+
     protected void validateDeprecatedAndRemovedSettingV7(Settings settings, Setting<?> setting) {}
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index f45758df42d8..c7a294b56c95 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction;
 import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction;
 import org.elasticsearch.action.bulk.WriteAckDelay;
+import org.elasticsearch.action.ingest.SimulatePipelineTransportAction;
 import org.elasticsearch.action.search.TransportSearchAction;
 import org.elasticsearch.action.support.AutoCreateIndex;
 import org.elasticsearch.action.support.DestructiveOperations;
@@ -40,6 +41,7 @@
 import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
+import org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceComputer;
 import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ConcurrentRebalanceAllocationDecider;
@@ -110,6 +112,7 @@
 import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.ProxyConnectionStrategy;
+import org.elasticsearch.transport.RemoteClusterPortSettings;
 import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.transport.RemoteConnectionStrategy;
 import org.elasticsearch.transport.SniffConnectionStrategy;
@@ -130,6 +133,14 @@
  */
 public final class ClusterSettings extends AbstractScopedSettings {
 
+    public static ClusterSettings createBuiltInClusterSettings() {
+        return createBuiltInClusterSettings(Settings.EMPTY);
+    }
+
+    public static ClusterSettings createBuiltInClusterSettings(Settings nodeSettings) {
+        return new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+    }
+
     public ClusterSettings(final Settings nodeSettings, final Set<Setting<?>> settingsSet) {
         this(nodeSettings, settingsSet, Collections.emptySet());
     }
@@ -198,6 +209,7 @@ public void apply(Settings value, Settings current, Settings previous) {
         BalancedShardsAllocator.WRITE_LOAD_BALANCE_FACTOR_SETTING,
         BalancedShardsAllocator.DISK_USAGE_BALANCE_FACTOR_SETTING,
         BalancedShardsAllocator.THRESHOLD_SETTING,
+        DesiredBalanceComputer.PROGRESS_LOG_INTERVAL_SETTING,
         BreakerSettings.CIRCUIT_BREAKER_LIMIT_SETTING,
         BreakerSettings.CIRCUIT_BREAKER_OVERHEAD_SETTING,
         BreakerSettings.CIRCUIT_BREAKER_TYPE,
@@ -316,6 +328,7 @@ public void apply(Settings value, Settings current, Settings previous) {
         HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING,
         HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING,
         IndexModule.NODE_STORE_ALLOW_MMAP,
+        IndexSettings.NODE_DEFAULT_REFRESH_INTERVAL_SETTING,
         ClusterApplierService.CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING,
         ClusterService.USER_DEFINED_METADATA,
         MasterService.MASTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING,
@@ -535,9 +548,24 @@ public void apply(Settings value, Settings current, Settings previous) {
         HealthNodeTaskExecutor.ENABLED_SETTING,
         LocalHealthMonitor.POLL_INTERVAL_SETTING,
         TransportHealthNodeAction.HEALTH_NODE_TRANSPORT_ACTION_TIMEOUT,
+        SimulatePipelineTransportAction.INGEST_NODE_TRANSPORT_ACTION_TIMEOUT,
         WriteAckDelay.WRITE_ACK_DELAY_INTERVAL,
         WriteAckDelay.WRITE_ACK_DELAY_RANDOMNESS_BOUND,
-        TcpTransport.isUntrustedRemoteClusterEnabled() ? RemoteClusterService.REMOTE_CLUSTER_AUTHORIZATION : null
+        TcpTransport.isUntrustedRemoteClusterEnabled() ? RemoteClusterService.REMOTE_CLUSTER_AUTHORIZATION : null,
+        TcpTransport.isUntrustedRemoteClusterEnabled() ? RemoteClusterPortSettings.REMOTE_CLUSTER_PORT_ENABLED : null,
+        TcpTransport.isUntrustedRemoteClusterEnabled() ? RemoteClusterPortSettings.HOST : null,
+        TcpTransport.isUntrustedRemoteClusterEnabled() ? RemoteClusterPortSettings.PUBLISH_HOST : null,
+        TcpTransport.isUntrustedRemoteClusterEnabled() ? RemoteClusterPortSettings.BIND_HOST : null,
+        TcpTransport.isUntrustedRemoteClusterEnabled() ? RemoteClusterPortSettings.PORT : null,
+        TcpTransport.isUntrustedRemoteClusterEnabled() ? RemoteClusterPortSettings.PUBLISH_PORT : null,
+        TcpTransport.isUntrustedRemoteClusterEnabled() ? RemoteClusterPortSettings.TCP_KEEP_ALIVE : null,
+        TcpTransport.isUntrustedRemoteClusterEnabled() ? RemoteClusterPortSettings.TCP_KEEP_IDLE : null,
+        TcpTransport.isUntrustedRemoteClusterEnabled() ? RemoteClusterPortSettings.TCP_KEEP_INTERVAL : null,
+        TcpTransport.isUntrustedRemoteClusterEnabled() ? RemoteClusterPortSettings.TCP_KEEP_COUNT : null,
+        TcpTransport.isUntrustedRemoteClusterEnabled() ? RemoteClusterPortSettings.TCP_NO_DELAY : null,
+        TcpTransport.isUntrustedRemoteClusterEnabled() ? RemoteClusterPortSettings.TCP_REUSE_ADDRESS : null,
+        TcpTransport.isUntrustedRemoteClusterEnabled() ? RemoteClusterPortSettings.TCP_SEND_BUFFER_SIZE : null,
+        StatelessSecureSettings.STATELESS_SECURE_SETTINGS
     ).filter(Objects::nonNull).collect(Collectors.toSet());
 
     static List<SettingUpgrader<?>> BUILT_IN_SETTING_UPGRADERS = Collections.emptyList();
diff --git a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
index c4a0477bcec6..8e5b06056bcc 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
@@ -25,6 +25,9 @@
 import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -80,7 +83,7 @@ private enum EntryType {
     }
 
     /** An entry in the keystore. The bytes are opaque and interpreted based on the entry type. */
-    private static class Entry {
+    private static class Entry implements Writeable {
         final byte[] bytes;
         final byte[] sha256Digest;
 
@@ -88,6 +91,17 @@ private static class Entry {
             this.bytes = bytes;
             this.sha256Digest = MessageDigests.sha256().digest(bytes);
         }
+
+        Entry(StreamInput input) throws IOException {
+            this.bytes = input.readByteArray();
+            this.sha256Digest = input.readByteArray();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeByteArray(bytes);
+            out.writeByteArray(sha256Digest);
+        }
     }
 
     /**
@@ -167,6 +181,14 @@ private KeyStoreWrapper(int formatVersion, boolean hasPassword, byte[] dataBytes
         this.dataBytes = dataBytes;
     }
 
+    public KeyStoreWrapper(StreamInput input) throws IOException {
+        formatVersion = input.readInt();
+        hasPassword = input.readBoolean();
+        dataBytes = input.readOptionalByteArray();
+        entries.set(input.readMap(StreamInput::readString, Entry::new));
+        closed = input.readBoolean();
+    }
+
     /**
      * Get the metadata format version for the keystore
      **/
@@ -614,4 +636,14 @@ public synchronized void close() {
             }
         }
     }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeInt(formatVersion);
+        out.writeBoolean(hasPassword);
+        out.writeOptionalByteArray(dataBytes);
+        var entriesMap = entries.get();
+        out.writeMap((entriesMap == null) ? Map.of() : entriesMap, StreamOutput::writeString, (o, v) -> v.writeTo(o));
+        out.writeBoolean(closed);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/settings/LocallyMountedSecrets.java b/server/src/main/java/org/elasticsearch/common/settings/LocallyMountedSecrets.java
new file mode 100644
index 000000000000..8157db24463c
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/settings/LocallyMountedSecrets.java
@@ -0,0 +1,186 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common.settings;
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.reservedstate.service.ReservedStateVersion;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.GeneralSecurityException;
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.xcontent.XContentType.JSON;
+
+/**
+ * An implementation of {@link SecureSettings} which loads the secrets from
+ * externally mounted local directory. It looks for the folder called 'secrets'
+ * under the config directory. All secure settings should be supplied in a single
+ * file called 'secrets.json' which sits inside the 'secrets' directory.
+ * <p>
+ * If the 'secrets' directory or the 'secrets.json' file don't exist, the
+ * SecureSettings implementation is loaded with empty settings map.
+ * <p>
+ * Example secrets.json format:
+ *         {
+ *              "metadata": {
+ *                  "version": "1",
+ *                  "compatibility": "8.7.0"
+ *              },
+ *              "secrets": {
+ *                  "secure.setting.key.one": "aaa",
+ *                  "secure.setting.key.two": "bbb"
+ *              }
+ *         }
+ */
+public class LocallyMountedSecrets implements SecureSettings {
+
+    public static final String SECRETS_FILE_NAME = "secrets.json";
+    public static final String SECRETS_DIRECTORY = "secrets";
+
+    public static final ParseField SECRETS_FIELD = new ParseField("secrets");
+    public static final ParseField METADATA_FIELD = new ParseField("metadata");
+
+    @SuppressWarnings("unchecked")
+    private final ConstructingObjectParser<LocalFileSecrets, Void> secretsParser = new ConstructingObjectParser<>(
+        "locally_mounted_secrets",
+        a -> new LocalFileSecrets((Map<String, String>) a[0], (ReservedStateVersion) a[1])
+    );
+
+    private final String secretsDir;
+    private final String secretsFile;
+    private final SetOnce<LocalFileSecrets> secrets = new SetOnce<>();
+
+    /**
+     * Direct constructor to be used by the CLI
+     */
+    public LocallyMountedSecrets(Environment environment) {
+        var secretsDirPath = environment.configFile().toAbsolutePath().resolve(SECRETS_DIRECTORY);
+        var secretsFilePath = secretsDirPath.resolve(SECRETS_FILE_NAME);
+        secretsParser.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> p.map(), SECRETS_FIELD);
+        secretsParser.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> ReservedStateVersion.parse(p), METADATA_FIELD);
+        if (Files.exists(secretsDirPath) && Files.exists(secretsFilePath)) {
+            try {
+                secrets.set(processSecretsFile(secretsFilePath));
+            } catch (IOException e) {
+                throw new IllegalStateException("Error processing secrets file", e);
+            }
+        } else {
+            secrets.set(new LocalFileSecrets(Map.of(), new ReservedStateVersion(-1L, Version.CURRENT)));
+        }
+        this.secretsDir = secretsDirPath.toString();
+        this.secretsFile = secretsFilePath.toString();
+    }
+
+    /**
+     * Used by {@link org.elasticsearch.bootstrap.ServerArgs} to deserialize the secrets
+     * when they are received by the Elasticsearch process. The ServerCli code serializes
+     * the secrets as part of ServerArgs.
+     */
+    public LocallyMountedSecrets(StreamInput in) throws IOException {
+        this.secretsDir = in.readString();
+        this.secretsFile = in.readString();
+        if (in.readBoolean()) {
+            secrets.set(LocalFileSecrets.readFrom(in));
+        }
+        // TODO: Add support for watching for file changes here.
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(secretsDir);
+        out.writeString(secretsFile);
+        if (secrets.get() == null) {
+            out.writeBoolean(false);
+        } else {
+            out.writeBoolean(true);
+            secrets.get().writeTo(out);
+        }
+    }
+
+    @Override
+    public boolean isLoaded() {
+        return secrets.get() != null;
+    }
+
+    @Override
+    public Set<String> getSettingNames() {
+        assert isLoaded();
+        return secrets.get().map().keySet();
+    }
+
+    @Override
+    public SecureString getString(String setting) {
+        assert isLoaded();
+        var value = secrets.get().map().get(setting);
+        if (value == null) {
+            return null;
+        }
+        return new SecureString(value.toCharArray());
+    }
+
+    @Override
+    public InputStream getFile(String setting) throws GeneralSecurityException {
+        assert isLoaded();
+        return new ByteArrayInputStream(getString(setting).toString().getBytes(StandardCharsets.UTF_8));
+    }
+
+    @Override
+    public byte[] getSHA256Digest(String setting) throws GeneralSecurityException {
+        assert isLoaded();
+        return MessageDigests.sha256().digest(getString(setting).toString().getBytes(StandardCharsets.UTF_8));
+    }
+
+    @Override
+    public void close() throws IOException {
+        if (null != secrets.get() && secrets.get().map().isEmpty() == false) {
+            for (var entry : secrets.get().map().entrySet()) {
+                entry.setValue(null);
+            }
+        }
+    }
+
+    // package private for testing
+    LocalFileSecrets processSecretsFile(Path path) throws IOException {
+        try (
+            var fis = Files.newInputStream(path);
+            var bis = new BufferedInputStream(fis);
+            var parser = JSON.xContent().createParser(XContentParserConfiguration.EMPTY, bis)
+        ) {
+            return secretsParser.apply(parser, null);
+        }
+    }
+
+    record LocalFileSecrets(Map<String, String> map, ReservedStateVersion metadata) implements Writeable {
+        public static LocalFileSecrets readFrom(StreamInput in) throws IOException {
+            return new LocalFileSecrets(in.readMap(StreamInput::readString, StreamInput::readString), ReservedStateVersion.readFrom(in));
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeMap((map == null) ? Map.of() : map, StreamOutput::writeString, StreamOutput::writeString);
+            metadata.writeTo(out);
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/common/settings/SecureSettings.java b/server/src/main/java/org/elasticsearch/common/settings/SecureSettings.java
index 25edbe6e846b..3b7d5c901877 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/SecureSettings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/SecureSettings.java
@@ -8,6 +8,8 @@
 
 package org.elasticsearch.common.settings;
 
+import org.elasticsearch.common.io.stream.Writeable;
+
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
@@ -17,7 +19,7 @@
 /**
  * An accessor for settings which are securely stored. See {@link SecureSetting}.
  */
-public interface SecureSettings extends Closeable {
+public interface SecureSettings extends Closeable, Writeable {
 
     /** Returns true iff the settings are loaded and retrievable. */
     boolean isLoaded();
@@ -26,7 +28,7 @@ public interface SecureSettings extends Closeable {
     Set<String> getSettingNames();
 
     /** Return a string setting. The {@link SecureString} should be closed once it is used. */
-    SecureString getString(String setting) throws GeneralSecurityException;
+    SecureString getString(String setting);
 
     /** Return a file setting. The {@link InputStream} should be closed once it is used. */
     InputStream getFile(String setting) throws GeneralSecurityException;
diff --git a/server/src/main/java/org/elasticsearch/common/settings/Settings.java b/server/src/main/java/org/elasticsearch/common/settings/Settings.java
index 1ebb189a358e..bbd43da1f83b 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/Settings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/Settings.java
@@ -1503,7 +1503,7 @@ public Set<String> getSettingNames() {
         }
 
         @Override
-        public SecureString getString(String setting) throws GeneralSecurityException {
+        public SecureString getString(String setting) {
             return delegate.getString(addPrefix.apply(setting));
         }
 
@@ -1521,6 +1521,11 @@ public byte[] getSHA256Digest(String setting) throws GeneralSecurityException {
         public void close() throws IOException {
             delegate.close();
         }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            throw new IllegalStateException("Unsupported operation");
+        }
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/common/settings/StatelessSecureSettings.java b/server/src/main/java/org/elasticsearch/common/settings/StatelessSecureSettings.java
new file mode 100644
index 000000000000..98c1f188de2f
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/settings/StatelessSecureSettings.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common.settings;
+
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * An implementation of secure settings from YML settings.
+ *
+ * WARNING: this is a temporary class only for Stateless. It applies only to YML settings with a predetermined prefix.
+ */
+public class StatelessSecureSettings implements SecureSettings {
+    static final String PREFIX = "insecure.";
+    static final Setting.AffixSetting<String> STATELESS_SECURE_SETTINGS = Setting.prefixKeySetting(
+        PREFIX,
+        (key) -> Setting.simpleString(key, Setting.Property.NodeScope)
+    );
+
+    private final Settings settings;
+    private final Set<String> names;
+
+    private StatelessSecureSettings(Settings settings) {
+        if (DiscoveryNode.isStateless(settings) == false) {
+            throw new IllegalArgumentException("StatelessSecureSettings are supported only in stateless");
+        }
+        this.settings = Settings.builder().put(settings, false).build();
+        this.names = settings.keySet()
+            .stream()
+            .filter(key -> (key.startsWith(PREFIX)))
+            .map(s -> s.replace(PREFIX, ""))
+            .collect(Collectors.toUnmodifiableSet());
+    }
+
+    public static Settings install(Settings settings) {
+        StatelessSecureSettings statelessSecureSettings = new StatelessSecureSettings(settings);
+        return Settings.builder().put(settings, false).setSecureSettings(statelessSecureSettings).build();
+    }
+
+    @Override
+    public boolean isLoaded() {
+        return true;
+    }
+
+    @Override
+    public Set<String> getSettingNames() {
+        return names;
+    }
+
+    @Override
+    public SecureString getString(String setting) {
+        return new SecureString(STATELESS_SECURE_SETTINGS.getConcreteSetting(PREFIX + setting).get(settings).toCharArray());
+    }
+
+    @Override
+    public InputStream getFile(String setting) throws GeneralSecurityException {
+        return new ByteArrayInputStream(
+            STATELESS_SECURE_SETTINGS.getConcreteSetting(PREFIX + setting).get(settings).getBytes(StandardCharsets.UTF_8)
+        );
+    }
+
+    @Override
+    public byte[] getSHA256Digest(String setting) {
+        return MessageDigests.sha256()
+            .digest(STATELESS_SECURE_SETTINGS.getConcreteSetting(PREFIX + setting).get(settings).getBytes(StandardCharsets.UTF_8));
+    }
+
+    @Override
+    public void close() throws IOException {}
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        throw new IllegalStateException("Unsupported operation");
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/common/unit/Processors.java b/server/src/main/java/org/elasticsearch/common/unit/Processors.java
index b95bdd3615f3..89db778266ea 100644
--- a/server/src/main/java/org/elasticsearch/common/unit/Processors.java
+++ b/server/src/main/java/org/elasticsearch/common/unit/Processors.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.common.unit;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -29,8 +29,8 @@ public class Processors implements Writeable, Comparable<Processors>, ToXContent
     public static final Processors ZERO = new Processors(0.0);
     public static final Processors MAX_PROCESSORS = new Processors(Double.MAX_VALUE);
 
-    public static final Version FLOAT_PROCESSORS_SUPPORT_VERSION = Version.V_8_3_0;
-    public static final Version DOUBLE_PROCESSORS_SUPPORT_VERSION = Version.V_8_5_0;
+    public static final TransportVersion FLOAT_PROCESSORS_SUPPORT_VERSION = TransportVersion.V_8_3_0;
+    public static final TransportVersion DOUBLE_PROCESSORS_SUPPORT_VERSION = TransportVersion.V_8_5_0;
     static final int NUMBER_OF_DECIMAL_PLACES = 5;
     private static final double MIN_REPRESENTABLE_PROCESSORS = 1E-5;
 
@@ -63,9 +63,9 @@ public static Processors of(Double count) {
 
     public static Processors readFrom(StreamInput in) throws IOException {
         final double processorCount;
-        if (in.getVersion().before(FLOAT_PROCESSORS_SUPPORT_VERSION)) {
+        if (in.getTransportVersion().before(FLOAT_PROCESSORS_SUPPORT_VERSION)) {
             processorCount = in.readInt();
-        } else if (in.getVersion().before(DOUBLE_PROCESSORS_SUPPORT_VERSION)) {
+        } else if (in.getTransportVersion().before(DOUBLE_PROCESSORS_SUPPORT_VERSION)) {
             processorCount = in.readFloat();
         } else {
             processorCount = in.readDouble();
@@ -75,10 +75,10 @@ public static Processors readFrom(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getVersion().before(FLOAT_PROCESSORS_SUPPORT_VERSION)) {
+        if (out.getTransportVersion().before(FLOAT_PROCESSORS_SUPPORT_VERSION)) {
             assert hasDecimals() == false;
             out.writeInt((int) count);
-        } else if (out.getVersion().before(DOUBLE_PROCESSORS_SUPPORT_VERSION)) {
+        } else if (out.getTransportVersion().before(DOUBLE_PROCESSORS_SUPPORT_VERSION)) {
             out.writeFloat((float) count);
         } else {
             out.writeDouble(count);
@@ -143,7 +143,7 @@ private boolean hasDecimals() {
         return ((int) count) != Math.ceil(count);
     }
 
-    public boolean isCompatibleWithVersion(Version version) {
+    public boolean isCompatibleWithVersion(TransportVersion version) {
         if (version.onOrAfter(FLOAT_PROCESSORS_SUPPORT_VERSION)) {
             return true;
         }
diff --git a/server/src/main/java/org/elasticsearch/common/util/ArrayUtils.java b/server/src/main/java/org/elasticsearch/common/util/ArrayUtils.java
index 37e9fc4049cb..16411d8dd1ff 100644
--- a/server/src/main/java/org/elasticsearch/common/util/ArrayUtils.java
+++ b/server/src/main/java/org/elasticsearch/common/util/ArrayUtils.java
@@ -84,4 +84,19 @@ public static String[] append(String[] array, String added) {
         updated[array.length] = added;
         return updated;
     }
+
+    /**
+     * Reverse the {@code length} values on the array starting from {@code offset}.
+     */
+    public static void reverseSubArray(double[] array, int offset, int length) {
+        int start = offset;
+        int end = offset + length;
+        while (end > start) {
+            final double scratch = array[start];
+            array[start] = array[end - 1];
+            array[end - 1] = scratch;
+            start++;
+            end--;
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/util/BigArrays.java b/server/src/main/java/org/elasticsearch/common/util/BigArrays.java
index 7fc329148181..edec336c2a02 100644
--- a/server/src/main/java/org/elasticsearch/common/util/BigArrays.java
+++ b/server/src/main/java/org/elasticsearch/common/util/BigArrays.java
@@ -152,6 +152,13 @@ public boolean hasArray() {
         public byte[] array() {
             return array;
         }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            int size = Math.toIntExact(size()) * Byte.BYTES;
+            out.writeVInt(size);
+            out.write(array, 0, size);
+        }
     }
 
     private static class ByteArrayAsIntArrayWrapper extends AbstractArrayWrapper implements IntArray {
@@ -261,6 +268,13 @@ public void set(long index, byte[] buf, int offset, int len) {
             assert index >= 0 && index < size();
             System.arraycopy(buf, offset << 3, array, (int) index << 3, len << 3);
         }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            int size = Math.toIntExact(size()) * Long.BYTES;
+            out.writeVInt(size);
+            out.write(array, 0, size);
+        }
     }
 
     private static class ByteArrayAsDoubleArrayWrapper extends AbstractArrayWrapper implements DoubleArray {
diff --git a/server/src/main/java/org/elasticsearch/common/util/BigByteArray.java b/server/src/main/java/org/elasticsearch/common/util/BigByteArray.java
index 108e5b1a4bfc..72a2fc41a9a1 100644
--- a/server/src/main/java/org/elasticsearch/common/util/BigByteArray.java
+++ b/server/src/main/java/org/elasticsearch/common/util/BigByteArray.java
@@ -11,9 +11,12 @@
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.RamUsageEstimator;
+import org.elasticsearch.common.io.stream.StreamOutput;
 
+import java.io.IOException;
 import java.util.Arrays;
 
+import static org.elasticsearch.common.util.BigLongArray.writePages;
 import static org.elasticsearch.common.util.PageCacheRecycler.BYTE_PAGE_SIZE;
 
 /**
@@ -36,6 +39,11 @@ final class BigByteArray extends AbstractBigArray implements ByteArray {
         }
     }
 
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        writePages(out, Math.toIntExact(size), pages, Byte.BYTES, BYTE_PAGE_SIZE);
+    }
+
     @Override
     public byte get(long index) {
         final int pageIndex = pageIndex(index);
diff --git a/server/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java b/server/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java
index ecfbfc5b9c6b..b2a55973cd44 100644
--- a/server/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java
+++ b/server/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java
@@ -18,6 +18,7 @@
 import java.nio.ByteOrder;
 import java.util.Arrays;
 
+import static org.elasticsearch.common.util.BigLongArray.writePages;
 import static org.elasticsearch.common.util.PageCacheRecycler.DOUBLE_PAGE_SIZE;
 
 /**
@@ -26,12 +27,6 @@
  */
 final class BigDoubleArray extends AbstractBigArray implements DoubleArray {
 
-    static {
-        if (ByteOrder.nativeOrder() != ByteOrder.LITTLE_ENDIAN) {
-            throw new Error("The deserialization assumes this class is written with little-endian numbers.");
-        }
-    }
-
     private static final BigDoubleArray ESTIMATOR = new BigDoubleArray(0, BigArrays.NON_RECYCLING_INSTANCE, false);
 
     static final VarHandle VH_PLATFORM_NATIVE_DOUBLE = MethodHandles.byteArrayViewVarHandle(double[].class, ByteOrder.nativeOrder());
@@ -134,18 +129,6 @@ public void set(long index, byte[] buf, int offset, int len) {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        int size = (int) this.size;
-        out.writeVInt(size * Double.BYTES);
-        int lastPageEnd = size % DOUBLE_PAGE_SIZE;
-        if (lastPageEnd == 0) {
-            for (byte[] page : pages) {
-                out.write(page);
-            }
-            return;
-        }
-        for (int i = 0; i < pages.length - 1; i++) {
-            out.write(pages[i]);
-        }
-        out.write(pages[pages.length - 1], 0, lastPageEnd * Double.BYTES);
+        writePages(out, Math.toIntExact(size), pages, Double.BYTES, DOUBLE_PAGE_SIZE);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/util/BigIntArray.java b/server/src/main/java/org/elasticsearch/common/util/BigIntArray.java
index e3cf7389f7ed..e053baea9aa5 100644
--- a/server/src/main/java/org/elasticsearch/common/util/BigIntArray.java
+++ b/server/src/main/java/org/elasticsearch/common/util/BigIntArray.java
@@ -18,6 +18,7 @@
 import java.nio.ByteOrder;
 import java.util.Arrays;
 
+import static org.elasticsearch.common.util.BigLongArray.writePages;
 import static org.elasticsearch.common.util.PageCacheRecycler.INT_PAGE_SIZE;
 
 /**
@@ -25,12 +26,6 @@
  * configurable length.
  */
 final class BigIntArray extends AbstractBigArray implements IntArray {
-    static {
-        if (ByteOrder.nativeOrder() != ByteOrder.LITTLE_ENDIAN) {
-            throw new Error("The deserialization assumes this class is written with little-endian ints.");
-        }
-    }
-
     private static final BigIntArray ESTIMATOR = new BigIntArray(0, BigArrays.NON_RECYCLING_INSTANCE, false);
 
     static final VarHandle VH_PLATFORM_NATIVE_INT = MethodHandles.byteArrayViewVarHandle(int[].class, ByteOrder.nativeOrder());
@@ -49,22 +44,7 @@ final class BigIntArray extends AbstractBigArray implements IntArray {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (size > Integer.MAX_VALUE / Integer.BYTES) {
-            throw new IllegalArgumentException();
-        }
-        int intSize = (int) size;
-        out.writeVInt(intSize * Integer.BYTES);
-        int lastPageEnd = intSize % INT_PAGE_SIZE;
-        if (lastPageEnd == 0) {
-            for (byte[] page : pages) {
-                out.write(page);
-            }
-            return;
-        }
-        for (int i = 0; i < pages.length - 1; i++) {
-            out.write(pages[i]);
-        }
-        out.write(pages[pages.length - 1], 0, lastPageEnd * Integer.BYTES);
+        writePages(out, (int) size, pages, Integer.BYTES, INT_PAGE_SIZE);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/common/util/BigLongArray.java b/server/src/main/java/org/elasticsearch/common/util/BigLongArray.java
index 8e7bb52f5a11..1044b5fb78ee 100644
--- a/server/src/main/java/org/elasticsearch/common/util/BigLongArray.java
+++ b/server/src/main/java/org/elasticsearch/common/util/BigLongArray.java
@@ -10,7 +10,9 @@
 
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.RamUsageEstimator;
+import org.elasticsearch.common.io.stream.StreamOutput;
 
+import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.VarHandle;
 import java.nio.ByteOrder;
@@ -126,4 +128,24 @@ public static long estimateRamBytes(final long size) {
     public void set(long index, byte[] buf, int offset, int len) {
         set(index, buf, offset, len, pages, 3);
     }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        writePages(out, Math.toIntExact(size), pages, Long.BYTES, LONG_PAGE_SIZE);
+    }
+
+    static void writePages(StreamOutput out, int size, byte[][] pages, int bytesPerValue, int pageSize) throws IOException {
+        out.writeVInt(size * bytesPerValue);
+        int lastPageEnd = size % pageSize;
+        if (lastPageEnd == 0) {
+            for (byte[] page : pages) {
+                out.write(page);
+            }
+            return;
+        }
+        for (int i = 0; i < pages.length - 1; i++) {
+            out.write(pages[i]);
+        }
+        out.write(pages[pages.length - 1], 0, lastPageEnd * bytesPerValue);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/util/ByteArray.java b/server/src/main/java/org/elasticsearch/common/util/ByteArray.java
index 61c218310396..e3b51ee7d2e3 100644
--- a/server/src/main/java/org/elasticsearch/common/util/ByteArray.java
+++ b/server/src/main/java/org/elasticsearch/common/util/ByteArray.java
@@ -9,13 +9,20 @@
 package org.elasticsearch.common.util;
 
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 
 /**
  * Abstraction of an array of byte values.
  */
-public interface ByteArray extends BigArray {
+public interface ByteArray extends BigArray, Writeable {
+
+    static ByteArray readFrom(StreamInput in) throws IOException {
+        return new ReleasableByteArray(in);
+    }
 
     /**
      * Get an element given its index.
diff --git a/server/src/main/java/org/elasticsearch/common/util/CollectionUtils.java b/server/src/main/java/org/elasticsearch/common/util/CollectionUtils.java
index 79a177b48869..1664b4690bc7 100644
--- a/server/src/main/java/org/elasticsearch/common/util/CollectionUtils.java
+++ b/server/src/main/java/org/elasticsearch/common/util/CollectionUtils.java
@@ -9,7 +9,6 @@
 package org.elasticsearch.common.util;
 
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.collect.Iterators;
 
 import java.nio.file.Path;
 import java.util.AbstractList;
@@ -21,7 +20,6 @@
 import java.util.IdentityHashMap;
 import java.util.List;
 import java.util.ListIterator;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.RandomAccess;
@@ -100,44 +98,56 @@ public static int[] toArray(Collection<Integer> ints) {
      * @param messageHint A string to be included in the exception message if the call fails, to provide
      *                    more context to the handler of the exception
      */
-    public static void ensureNoSelfReferences(Object value, String messageHint) {
-        Iterable<?> it = convert(value);
-        if (it != null) {
-            ensureNoSelfReferences(it, value, Collections.newSetFromMap(new IdentityHashMap<>()), messageHint);
-        }
+    public static void ensureNoSelfReferences(final Object value, final String messageHint) {
+        ensureNoSelfReferences(value, Collections.newSetFromMap(new IdentityHashMap<>()), messageHint);
     }
 
-    @SuppressWarnings("unchecked")
-    private static Iterable<?> convert(Object value) {
-        if (value == null) {
-            return null;
-        }
-        if (value instanceof Map<?, ?> map) {
-            return () -> Iterators.concat(map.keySet().iterator(), map.values().iterator());
-        } else if ((value instanceof Iterable) && (value instanceof Path == false)) {
-            return (Iterable<?>) value;
+    private static void ensureNoSelfReferences(final Object value, final Set<Object> ancestors, final String messageHint) {
+        // these instanceof checks are a bit on the ugly side, but it's important for performance that we have
+        // a separate dispatch point for Maps versus for Iterables. a polymorphic version of this code would
+        // be prettier, but it would also likely be quite a bit slower. this is a hot path for ingest pipelines,
+        // and performance here is important.
+        if (value == null || value instanceof String || value instanceof Number || value instanceof Boolean) {
+            // noop
+        } else if (value instanceof Map<?, ?> m && m.isEmpty() == false) {
+            ensureNoSelfReferences(m, ancestors, messageHint);
+        } else if ((value instanceof Iterable<?> i) && (value instanceof Path == false)) {
+            ensureNoSelfReferences(i, i, ancestors, messageHint);
         } else if (value instanceof Object[]) {
-            return Arrays.asList((Object[]) value);
-        } else {
-            return null;
+            // note: the iterable and reference arguments are different
+            ensureNoSelfReferences(Arrays.asList((Object[]) value), value, ancestors, messageHint);
         }
     }
 
+    private static void ensureNoSelfReferences(final Map<?, ?> reference, final Set<Object> ancestors, final String messageHint) {
+        addToAncestorsOrThrow(reference, ancestors, messageHint);
+        for (Map.Entry<?, ?> e : reference.entrySet()) {
+            ensureNoSelfReferences(e.getKey(), ancestors, messageHint);
+            ensureNoSelfReferences(e.getValue(), ancestors, messageHint);
+        }
+        ancestors.remove(reference);
+    }
+
     private static void ensureNoSelfReferences(
-        final Iterable<?> value,
-        Object originalReference,
+        final Iterable<?> iterable,
+        final Object reference,
         final Set<Object> ancestors,
-        String messageHint
+        final String messageHint
     ) {
-        if (value != null) {
-            if (ancestors.add(originalReference) == false) {
-                String suffix = Strings.isNullOrEmpty(messageHint) ? "" : String.format(Locale.ROOT, " (%s)", messageHint);
-                throw new IllegalArgumentException("Iterable object is self-referencing itself" + suffix);
-            }
-            for (Object o : value) {
-                ensureNoSelfReferences(convert(o), o, ancestors, messageHint);
+        addToAncestorsOrThrow(reference, ancestors, messageHint);
+        for (Object o : iterable) {
+            ensureNoSelfReferences(o, ancestors, messageHint);
+        }
+        ancestors.remove(reference);
+    }
+
+    private static void addToAncestorsOrThrow(Object reference, Set<Object> ancestors, String messageHint) {
+        if (ancestors.add(reference) == false) {
+            StringBuilder sb = new StringBuilder("Iterable object is self-referencing itself");
+            if (Strings.hasLength(messageHint)) {
+                sb.append(" (").append(messageHint).append(")");
             }
-            ancestors.remove(originalReference);
+            throw new IllegalArgumentException(sb.toString());
         }
     }
 
@@ -178,7 +188,7 @@ public static <E> ArrayList<E> iterableAsArrayList(Iterable<? extends E> element
             throw new NullPointerException("elements");
         }
         if (elements instanceof Collection) {
-            return new ArrayList<>((Collection) elements);
+            return new ArrayList<>((Collection<E>) elements);
         } else {
             ArrayList<E> list = new ArrayList<>();
             for (E element : elements) {
diff --git a/server/src/main/java/org/elasticsearch/common/util/CopyOnFirstWriteMap.java b/server/src/main/java/org/elasticsearch/common/util/CopyOnFirstWriteMap.java
new file mode 100644
index 000000000000..08d86c143fc1
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/util/CopyOnFirstWriteMap.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common.util;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * This map is designed to be constructed from an immutable map and be copied only if a (rare) mutation operation occurs.
+ * It should be converted back to an immutable map using `org.elasticsearch.common.util.LazyCopyOnWriteMap#toImmutableMap()`.
+ */
+public class CopyOnFirstWriteMap<K, V> implements Map<K, V> {
+
+    private Map<K, V> source;
+    private boolean wasCopied = false;
+
+    public CopyOnFirstWriteMap(Map<K, V> source) {
+        this.source = source;
+    }
+
+    private Map<K, V> getForUpdate() {
+        if (wasCopied == false) {
+            source = new HashMap<>(source);
+            wasCopied = true;
+        }
+        return source;
+    }
+
+    private Map<K, V> getForRead() {
+        return source;
+    }
+
+    public Map<K, V> toImmutableMap() {
+        return Map.copyOf(getForRead());
+    }
+
+    @Override
+    public int size() {
+        return getForRead().size();
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return getForRead().isEmpty();
+    }
+
+    @Override
+    public boolean containsKey(Object key) {
+        return getForRead().containsKey(key);
+    }
+
+    @Override
+    public boolean containsValue(Object value) {
+        return getForRead().containsValue(value);
+    }
+
+    @Override
+    public V get(Object key) {
+        return getForRead().get(key);
+    }
+
+    @Override
+    public V put(K key, V value) {
+        return getForUpdate().put(key, value);
+    }
+
+    @Override
+    public V remove(Object key) {
+        return getForUpdate().remove(key);
+    }
+
+    @Override
+    public void putAll(Map<? extends K, ? extends V> m) {
+        getForUpdate().putAll(m);
+    }
+
+    @Override
+    public void clear() {
+        getForUpdate().clear();
+    }
+
+    @Override
+    public Set<K> keySet() {
+        return Collections.unmodifiableSet(getForRead().keySet());
+    }
+
+    @Override
+    public Collection<V> values() {
+        return Collections.unmodifiableCollection(getForRead().values());
+    }
+
+    @Override
+    public Set<Entry<K, V>> entrySet() {
+        return Collections.unmodifiableSet(getForRead().entrySet());
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/common/util/LazyMap.java b/server/src/main/java/org/elasticsearch/common/util/LazyMap.java
deleted file mode 100644
index 7b6be425985e..000000000000
--- a/server/src/main/java/org/elasticsearch/common/util/LazyMap.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.common.util;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.function.BiConsumer;
-import java.util.function.BiFunction;
-import java.util.function.Function;
-import java.util.function.Supplier;
-
-public class LazyMap<K, V> implements Map<K, V> {
-
-    private final Supplier<Map<K, V>> mapSupplier;
-    private volatile Map<K, V> map;
-
-    public LazyMap(Supplier<Map<K, V>> mapSupplier) {
-        this.mapSupplier = mapSupplier;
-    }
-
-    private Map<K, V> get() {
-        if (map == null) {
-            synchronized (this) {
-                if (map == null) {
-                    map = Objects.requireNonNullElse(mapSupplier.get(), Collections.emptyMap());
-                }
-            }
-        }
-        return map;
-    }
-
-    @Override
-    public int size() {
-        return get().size();
-    }
-
-    @Override
-    public boolean isEmpty() {
-        return get().isEmpty();
-    }
-
-    @Override
-    public boolean containsKey(Object key) {
-        return get().containsKey(key);
-    }
-
-    @Override
-    public boolean containsValue(Object value) {
-        return get().containsValue(value);
-    }
-
-    @Override
-    public V get(Object key) {
-        return get().get(key);
-    }
-
-    @Override
-    public V put(K key, V value) {
-        return get().put(key, value);
-    }
-
-    @Override
-    public V remove(Object key) {
-        return get().remove(key);
-    }
-
-    @Override
-    public void putAll(Map<? extends K, ? extends V> m) {
-        get().putAll(m);
-    }
-
-    @Override
-    public void clear() {
-        get().clear();
-    }
-
-    @Override
-    public Set<K> keySet() {
-        return get().keySet();
-    }
-
-    @Override
-    public Collection<V> values() {
-        return get().values();
-    }
-
-    @Override
-    public Set<Entry<K, V>> entrySet() {
-        return get().entrySet();
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        return get().equals(o);
-    }
-
-    @Override
-    public int hashCode() {
-        return get().hashCode();
-    }
-
-    @Override
-    public String toString() {
-        return get().toString();
-    }
-
-    // Override default methods in Map
-    @Override
-    public V getOrDefault(Object k, V defaultValue) {
-        return get().getOrDefault(k, defaultValue);
-    }
-
-    @Override
-    public void forEach(BiConsumer<? super K, ? super V> action) {
-        get().forEach(action);
-    }
-
-    @Override
-    public void replaceAll(BiFunction<? super K, ? super V, ? extends V> function) {
-        get().replaceAll(function);
-    }
-
-    @Override
-    public V putIfAbsent(K key, V value) {
-        return get().putIfAbsent(key, value);
-    }
-
-    @Override
-    public boolean remove(Object key, Object value) {
-        return get().remove(key, value);
-    }
-
-    @Override
-    public boolean replace(K key, V oldValue, V newValue) {
-        return get().replace(key, oldValue, newValue);
-    }
-
-    @Override
-    public V replace(K key, V value) {
-        return get().replace(key, value);
-    }
-
-    @Override
-    public V computeIfAbsent(K key, Function<? super K, ? extends V> mappingFunction) {
-        return get().computeIfAbsent(key, mappingFunction);
-    }
-
-    @Override
-    public V computeIfPresent(K key, BiFunction<? super K, ? super V, ? extends V> remappingFunction) {
-        return get().computeIfPresent(key, remappingFunction);
-    }
-
-    @Override
-    public V compute(K key, BiFunction<? super K, ? super V, ? extends V> remappingFunction) {
-        return get().compute(key, remappingFunction);
-    }
-
-    @Override
-    public V merge(K key, V value, BiFunction<? super V, ? super V, ? extends V> remappingFunction) {
-        return get().merge(key, value, remappingFunction);
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/common/util/LongArray.java b/server/src/main/java/org/elasticsearch/common/util/LongArray.java
index 984f1df48701..bd293a135640 100644
--- a/server/src/main/java/org/elasticsearch/common/util/LongArray.java
+++ b/server/src/main/java/org/elasticsearch/common/util/LongArray.java
@@ -8,10 +8,19 @@
 
 package org.elasticsearch.common.util;
 
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+
 /**
  * Abstraction of an array of long values.
  */
-public interface LongArray extends BigArray {
+public interface LongArray extends BigArray, Writeable {
+
+    static LongArray readFrom(StreamInput in) throws IOException {
+        return new ReleasableLongArray(in);
+    }
 
     /**
      * Get an element given its index.
diff --git a/server/src/main/java/org/elasticsearch/common/util/Maps.java b/server/src/main/java/org/elasticsearch/common/util/Maps.java
index a0ff346da0d9..276c36f8ce95 100644
--- a/server/src/main/java/org/elasticsearch/common/util/Maps.java
+++ b/server/src/main/java/org/elasticsearch/common/util/Maps.java
@@ -295,7 +295,14 @@ static int capacity(int expectedSize) {
      * This method creates a copy of the {@code source} map using {@code copyValueFunction} to create a defensive copy of each value.
      */
     public static <K, V> Map<K, V> copyOf(Map<K, V> source, Function<V, V> copyValueFunction) {
-        var copy = Maps.<K, V>newHashMapWithExpectedSize(source.size());
+        return transformValues(source, copyValueFunction);
+    }
+
+    /**
+     * Copy a map and transform it values using supplied function
+     */
+    public static <K, V1, V2> Map<K, V2> transformValues(Map<K, V1> source, Function<V1, V2> copyValueFunction) {
+        var copy = Maps.<K, V2>newHashMapWithExpectedSize(source.size());
         for (var entry : source.entrySet()) {
             copy.put(entry.getKey(), copyValueFunction.apply(entry.getValue()));
         }
diff --git a/server/src/main/java/org/elasticsearch/common/util/ReleasableByteArray.java b/server/src/main/java/org/elasticsearch/common/util/ReleasableByteArray.java
new file mode 100644
index 000000000000..0102195f4e80
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/util/ReleasableByteArray.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common.util;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.bytes.ReleasableBytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+import static org.elasticsearch.common.util.BigArrays.indexIsInt;
+
+public class ReleasableByteArray implements ByteArray {
+
+    private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(ReleasableByteArray.class);
+
+    private final ReleasableBytesReference ref;
+
+    ReleasableByteArray(StreamInput in) throws IOException {
+        this.ref = in.readReleasableBytesReference();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeBytesReference(ref);
+    }
+
+    @Override
+    public long size() {
+        return ref.length() / Byte.BYTES;
+    }
+
+    @Override
+    public byte get(long index) {
+        assert indexIsInt(index);
+        return ref.get((int) index);
+    }
+
+    @Override
+    public boolean get(long index, int len, BytesRef ref) {
+        assert indexIsInt(index);
+        BytesReference sliced = this.ref.slice((int) index, len);
+        if (sliced.length() != 0) {
+            ref.offset = sliced.arrayOffset();
+            ref.length = sliced.length();
+            ref.bytes = sliced.array();
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    @Override
+    public byte set(long index, byte value) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void set(long index, byte[] buf, int offset, int len) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void fill(long fromIndex, long toIndex, byte value) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean hasArray() {
+        return ref.hasArray();
+    }
+
+    @Override
+    public byte[] array() {
+        // The assumption of this method is that the returned array has valid entries starting from slot 0 and
+        // this isn't case when just returning the array from ReleasableBytesReference#array().
+        // The interface that this class implements should have something like an arrayOffset() method,
+        // so that callers know from what array offset the first actual byte starts.
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public long ramBytesUsed() {
+        /*
+         * If we return the size of the buffer that we've sliced
+         * we're likely to double count things.
+         */
+        return SHALLOW_SIZE;
+    }
+
+    @Override
+    public void close() {
+        ref.decRef();
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/common/util/ReleasableLongArray.java b/server/src/main/java/org/elasticsearch/common/util/ReleasableLongArray.java
new file mode 100644
index 000000000000..44764ea1e171
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/util/ReleasableLongArray.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common.util;
+
+import org.apache.lucene.util.RamUsageEstimator;
+import org.elasticsearch.common.bytes.ReleasableBytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+public class ReleasableLongArray implements LongArray {
+
+    private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(ReleasableLongArray.class);
+
+    private final ReleasableBytesReference ref;
+
+    ReleasableLongArray(StreamInput in) throws IOException {
+        this.ref = in.readReleasableBytesReference();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeBytesReference(ref);
+    }
+
+    @Override
+    public long size() {
+        return ref.length() / Long.BYTES;
+    }
+
+    @Override
+    public long get(long index) {
+        if (index > Integer.MAX_VALUE / Long.BYTES) {
+            // We can't serialize messages longer than 2gb anyway
+            throw new ArrayIndexOutOfBoundsException();
+        }
+        return ref.getLongLE((int) index * Long.BYTES);
+    }
+
+    @Override
+    public long set(long index, long value) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public long increment(long index, long inc) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void fill(long fromIndex, long toIndex, long value) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void set(long index, byte[] buf, int offset, int len) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public long ramBytesUsed() {
+        /*
+         * If we return the size of the buffer that we've sliced
+         * we're likely to double count things.
+         */
+        return SHALLOW_SIZE;
+    }
+
+    @Override
+    public void close() {
+        ref.decRef();
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/AbstractThrottledTaskRunner.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/AbstractThrottledTaskRunner.java
new file mode 100644
index 000000000000..ea37dad5ba21
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/AbstractThrottledTaskRunner.java
@@ -0,0 +1,158 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common.util.concurrent;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.core.Strings;
+
+import java.util.Queue;
+import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * {@link AbstractThrottledTaskRunner} runs the enqueued tasks using the given executor, limiting the number of tasks that are submitted to
+ * the executor at once.
+ */
+public class AbstractThrottledTaskRunner<T extends ActionListener<Releasable>> {
+    private static final Logger logger = LogManager.getLogger(AbstractThrottledTaskRunner.class);
+
+    private final String taskRunnerName;
+    // The max number of tasks that this runner will schedule to concurrently run on the executor.
+    private final int maxRunningTasks;
+    // As we fork off dequeued tasks to the given executor, technically the following counter represents
+    // the number of the concurrent pollAndSpawn calls currently checking the queue for a task to run. This
+    // doesn't necessarily correspond to currently running tasks, since a pollAndSpawn could return without
+    // actually running a task when the queue is empty.
+    private final AtomicInteger runningTasks = new AtomicInteger();
+    private final Queue<T> tasks;
+    private final Executor executor;
+
+    public AbstractThrottledTaskRunner(final String name, final int maxRunningTasks, final Executor executor, final Queue<T> taskQueue) {
+        assert maxRunningTasks > 0;
+        this.taskRunnerName = name;
+        this.maxRunningTasks = maxRunningTasks;
+        this.executor = executor;
+        this.tasks = taskQueue;
+    }
+
+    /**
+     * Submits a task for execution. If there are fewer than {@code maxRunningTasks} tasks currently running then this task is immediately
+     * submitted to the executor. Otherwise this task is enqueued and will be submitted to the executor in turn on completion of some other
+     * task.
+     *
+     * Tasks are executed via their {@link ActionListener#onResponse} method, receiving a {@link Releasable} which must be closed on
+     * completion of the task. Task which are rejected from their executor are notified via their {@link ActionListener#onFailure} method.
+     * Neither of these methods may themselves throw exceptions.
+     */
+    public void enqueueTask(final T task) {
+        logger.trace("[{}] enqueuing task {}", taskRunnerName, task);
+        tasks.add(task);
+        // Try to run a task since now there is at least one in the queue. If the maxRunningTasks is
+        // reached, the task is just enqueued.
+        pollAndSpawn();
+    }
+
+    /**
+     * Allows certain tasks to force their execution, bypassing the queue-length limit on the executor. See also {@link
+     * AbstractRunnable#isForceExecution()}.
+     */
+    protected boolean isForceExecution(@SuppressWarnings("unused") /* TODO test this */ T task) {
+        return false;
+    }
+
+    private void pollAndSpawn() {
+        // A pollAndSpawn attempts to run a new task. There could be many concurrent pollAndSpawn calls competing
+        // to get a "free slot", since we attempt to run a new task on every enqueueTask call and every time an
+        // existing task is finished.
+        while (incrementRunningTasks()) {
+            T task = tasks.poll();
+            if (task == null) {
+                logger.trace("[{}] task queue is empty", taskRunnerName);
+                // We have taken up a "free slot", but there are no tasks in the queue! This could happen each time a worker
+                // sees an empty queue after running a task. Decrement to give competing pollAndSpawn calls a chance!
+                int decremented = runningTasks.decrementAndGet();
+                assert decremented >= 0;
+                // We might have blocked all competing pollAndSpawn calls. This could happen for example when
+                // maxRunningTasks=1 and a task got enqueued just after checking the queue but before decrementing.
+                // To be sure, return only if the queue is still empty. If the queue is not empty, this might be the
+                // only pollAndSpawn call in progress, and returning without peeking would risk ending up with a
+                // non-empty queue and no workers!
+                if (tasks.peek() == null) break;
+            } else {
+                final boolean isForceExecution = isForceExecution(task);
+                executor.execute(new AbstractRunnable() {
+                    private boolean rejected; // need not be volatile - if we're rejected then that happens-before calling onAfter
+
+                    private final Releasable releasable = Releasables.releaseOnce(() -> {
+                        // To avoid missing to run tasks that are enqueued and waiting, we check the queue again once running
+                        // a task is finished.
+                        int decremented = runningTasks.decrementAndGet();
+                        assert decremented >= 0;
+
+                        if (rejected == false) {
+                            pollAndSpawn();
+                        }
+                    });
+
+                    @Override
+                    public boolean isForceExecution() {
+                        return isForceExecution;
+                    }
+
+                    @Override
+                    public void onRejection(Exception e) {
+                        logger.trace("[{}] task {} rejected", taskRunnerName, task);
+                        rejected = true;
+                        try {
+                            task.onFailure(e);
+                        } finally {
+                            releasable.close();
+                        }
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        // should not happen
+                        logger.error(() -> Strings.format("[%s] task %s failed", taskRunnerName, task), e);
+                        assert false : e;
+                        task.onFailure(e);
+                    }
+
+                    @Override
+                    protected void doRun() {
+                        logger.trace("[{}] running task {}", taskRunnerName, task);
+                        task.onResponse(releasable);
+                    }
+
+                    @Override
+                    public String toString() {
+                        return task.toString();
+                    }
+                });
+            }
+        }
+    }
+
+    // Each worker thread that runs a task, first needs to get a "free slot" in order to respect maxRunningTasks.
+    private boolean incrementRunningTasks() {
+        int preUpdateValue = runningTasks.getAndAccumulate(maxRunningTasks, (v, maxRunning) -> v < maxRunning ? v + 1 : v);
+        assert preUpdateValue <= maxRunningTasks;
+        return preUpdateValue < maxRunningTasks;
+    }
+
+    // exposed for testing
+    int runningTasks() {
+        return runningTasks.get();
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/CountDown.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/CountDown.java
index bbbea0f31d8d..ba7b6fadb2ae 100644
--- a/server/src/main/java/org/elasticsearch/common/util/concurrent/CountDown.java
+++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/CountDown.java
@@ -12,51 +12,49 @@
 import java.util.concurrent.atomic.AtomicInteger;
 
 /**
- * A simple thread safe count-down class that in contrast to a {@link CountDownLatch}
- * never blocks. This class is useful if a certain action has to wait for N concurrent
- * tasks to return or a timeout to occur in order to proceed.
+ * A simple thread-safe count-down class that does not block, unlike a {@link CountDownLatch}. This class is useful if an action must wait
+ * for N concurrent tasks to succeed, or some other task to fail, in order to proceed. When called enough times, exactly one invocation of
+ * {@link #countDown()} or {@link #fastForward()} will return {@code true}.
  */
 public final class CountDown {
 
     private final AtomicInteger countDown;
-    private final int originalCount;
 
     public CountDown(int count) {
+        // A count of zero was permitted in older versions, but is trappy (requiring explicit special handling) because countDown() and
+        // fastForward() would never return true. This case is now forbidden in tests, forcing callers to handle the special case up-front,
+        // but the older behaviour is preserved when assertions are disabled. See #92196.
+        // TODO drop this leniency once we are confident nothing relies on the older behaviour any more.
+        assert count > 0 : "count must be positive";
         if (count < 0) {
             throw new IllegalArgumentException("count must be greater or equal to 0 but was: " + count);
         }
-        this.originalCount = count;
         this.countDown = new AtomicInteger(count);
     }
 
+    private static int assertValidCount(int count) {
+        assert count >= 0 : count;
+        return count;
+    }
+
     /**
-     * Decrements the count-down and returns <code>true</code> iff this call
-     * reached zero otherwise <code>false</code>
+     * Decrements the count and returns {@code true} if and only if the count reached zero with this call.
      */
     public boolean countDown() {
-        assert originalCount > 0;
-        return countDown.getAndUpdate(current -> {
-            assert current >= 0;
-            return current == 0 ? 0 : current - 1;
-        }) == 1;
+        return countDown.getAndUpdate(current -> assertValidCount(current) == 0 ? 0 : current - 1) == 1;
     }
 
     /**
-     * Fast forwards the count-down to zero and returns <code>true</code> iff
-     * the count down reached zero with this fast forward call otherwise
-     * <code>false</code>
+     * Fast-forwards the count to zero and returns {@code true} if and only if the count reached zero with this call.
      */
     public boolean fastForward() {
-        assert originalCount > 0;
-        assert countDown.get() >= 0;
-        return countDown.getAndSet(0) > 0;
+        return assertValidCount(countDown.getAndSet(0)) > 0;
     }
 
     /**
-     * Returns <code>true</code> iff the count-down has reached zero. Otherwise <code>false</code>
+     * Returns {@code true} if and only if the count has reached zero.
      */
     public boolean isCountedDown() {
-        assert countDown.get() >= 0;
-        return countDown.get() == 0;
+        return assertValidCount(countDown.get()) == 0;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/ListenableFuture.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/ListenableFuture.java
index a378d4020112..f2788d278c81 100644
--- a/server/src/main/java/org/elasticsearch/common/util/concurrent/ListenableFuture.java
+++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/ListenableFuture.java
@@ -105,39 +105,31 @@ protected void done(boolean ignored) {
     }
 
     private void notifyListenerDirectly(ActionListener<V> listener) {
-        try {
-            // call get in a non-blocking fashion as we could be on a network thread
-            // or another thread like the scheduler, which we should never block!
-            assert done;
-            V value = FutureUtils.get(ListenableFuture.this, 0L, TimeUnit.NANOSECONDS);
-            listener.onResponse(value);
-        } catch (Exception e) {
-            listener.onFailure(e);
-        }
+        // call get in a non-blocking fashion as we could be on a network thread
+        // or another thread like the scheduler, which we should never block!
+        assert done;
+        ActionListener.completeWith(listener, () -> FutureUtils.get(ListenableFuture.this, 0L, TimeUnit.NANOSECONDS));
     }
 
     private void notifyListener(ActionListener<V> listener, ExecutorService executorService) {
-        try {
-            executorService.execute(new Runnable() {
-                @Override
-                public void run() {
-                    notifyListenerDirectly(listener);
-                }
+        ActionListener.run(listener, l -> executorService.execute(new Runnable() {
+            @Override
+            public void run() {
+                notifyListenerDirectly(l);
+            }
 
-                @Override
-                public String toString() {
-                    return "ListenableFuture notification";
-                }
-            });
-        } catch (Exception e) {
-            listener.onFailure(e);
-        }
+            @Override
+            public String toString() {
+                return "ListenableFuture notification";
+            }
+        }));
     }
 
     @Override
     public void onResponse(V v) {
         final boolean set = set(v);
         if (set == false) {
+            assert false;
             throw new IllegalStateException("did not set value, value or exception already set?");
         }
     }
@@ -146,6 +138,7 @@ public void onResponse(V v) {
     public void onFailure(Exception e) {
         final boolean set = setException(e);
         if (set == false) {
+            assert false;
             throw new IllegalStateException("did not set exception, value already set or exception already set?");
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/PrioritizedThrottledTaskRunner.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/PrioritizedThrottledTaskRunner.java
index 9bc7617a4c79..96d6b0ed9471 100644
--- a/server/src/main/java/org/elasticsearch/common/util/concurrent/PrioritizedThrottledTaskRunner.java
+++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/PrioritizedThrottledTaskRunner.java
@@ -8,100 +8,82 @@
 
 package org.elasticsearch.common.util.concurrent;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.core.Releasable;
 
-import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.Executor;
 import java.util.concurrent.PriorityBlockingQueue;
-import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * {@link PrioritizedThrottledTaskRunner} performs the enqueued tasks in the order dictated by the
  * natural ordering of the tasks, limiting the max number of concurrently running tasks. Each new task
  * that is dequeued to be run, is forked off to the given executor.
  */
-public class PrioritizedThrottledTaskRunner<T extends Comparable<T> & Runnable> {
-    private static final Logger logger = LogManager.getLogger(PrioritizedThrottledTaskRunner.class);
-
-    private final String taskRunnerName;
-    // The max number of tasks that this runner will schedule to concurrently run on the executor.
-    private final int maxRunningTasks;
-    // As we fork off dequeued tasks to the given executor, technically the following counter represents
-    // the number of the concurrent pollAndSpawn calls currently checking the queue for a task to run. This
-    // doesn't necessarily correspond to currently running tasks, since a pollAndSpawn could return without
-    // actually running a task when the queue is empty.
-    private final AtomicInteger runningTasks = new AtomicInteger();
-    private final BlockingQueue<T> tasks = new PriorityBlockingQueue<>();
-    private final Executor executor;
+public class PrioritizedThrottledTaskRunner<T extends AbstractRunnable & Comparable<T>> {
 
-    public PrioritizedThrottledTaskRunner(final String name, final int maxRunningTasks, final Executor executor) {
-        assert maxRunningTasks > 0;
-        this.taskRunnerName = name;
-        this.maxRunningTasks = maxRunningTasks;
-        this.executor = executor;
-    }
+    private final AbstractThrottledTaskRunner<TaskWrapper<T>> runner;
+    private final PriorityBlockingQueue<TaskWrapper<T>> queue;
 
-    public void enqueueTask(final T task) {
-        logger.trace("[{}] enqueuing task {}", taskRunnerName, task);
-        tasks.add(task);
-        // Try to run a task since now there is at least one in the queue. If the maxRunningTasks is
-        // reached, the task is just enqueued.
-        pollAndSpawn();
-    }
+    private static class TaskWrapper<T extends AbstractRunnable & Comparable<T>>
+        implements
+            ActionListener<Releasable>,
+            Comparable<TaskWrapper<T>> {
+
+        private final T task;
+
+        TaskWrapper(T task) {
+            this.task = task;
+        }
+
+        @Override
+        public int compareTo(TaskWrapper<T> o) {
+            return task.compareTo(o.task);
+        }
+
+        @Override
+        public String toString() {
+            return task.toString();
+        }
 
-    // visible for testing
-    protected void pollAndSpawn() {
-        // A pollAndSpawn attempts to run a new task. There could be many concurrent pollAndSpawn calls competing
-        // to get a "free slot", since we attempt to run a new task on every enqueueTask call and every time an
-        // existing task is finished.
-        while (incrementRunningTasks()) {
-            T task = tasks.poll();
-            if (task == null) {
-                logger.trace("[{}] task queue is empty", taskRunnerName);
-                // We have taken up a "free slot", but there are no tasks in the queue! This could happen each time a worker
-                // sees an empty queue after running a task. Decrement to give competing pollAndSpawn calls a chance!
-                int decremented = runningTasks.decrementAndGet();
-                assert decremented >= 0;
-                // We might have blocked all competing pollAndSpawn calls. This could happen for example when
-                // maxRunningTasks=1 and a task got enqueued just after checking the queue but before decrementing.
-                // To be sure, return only if the queue is still empty. If the queue is not empty, this might be the
-                // only pollAndSpawn call in progress, and returning without peeking would risk ending up with a
-                // non-empty queue and no workers!
-                if (tasks.peek() == null) break;
-            } else {
-                executor.execute(() -> runTask(task));
+        @Override
+        public void onResponse(Releasable releasable) {
+            try (releasable) {
+                task.run();
             }
         }
+
+        @Override
+        public void onFailure(Exception e) {
+            assert e instanceof EsRejectedExecutionException : e;
+            try {
+                task.onRejection(e);
+            } finally {
+                task.onAfter();
+            }
+        }
+    }
+
+    public PrioritizedThrottledTaskRunner(final String name, final int maxRunningTasks, final Executor executor) {
+        this.queue = new PriorityBlockingQueue<>();
+        this.runner = new AbstractThrottledTaskRunner<>(name, maxRunningTasks, executor, queue);
     }
 
-    // Each worker thread that runs a task, first needs to get a "free slot" in order to respect maxRunningTasks.
-    private boolean incrementRunningTasks() {
-        int preUpdateValue = runningTasks.getAndAccumulate(maxRunningTasks, (v, maxRunning) -> v < maxRunning ? v + 1 : v);
-        assert preUpdateValue <= maxRunningTasks;
-        return preUpdateValue < maxRunningTasks;
+    /**
+     * Submits a task for execution. If there are fewer than {@code maxRunningTasks} tasks currently running then this task is immediately
+     * submitted to the executor. Otherwise this task is enqueued and will be submitted to the executor in turn on completion of some other
+     * task.
+     */
+    public void enqueueTask(final T task) {
+        runner.enqueueTask(new TaskWrapper<>(task));
     }
 
     // Only use for testing
     public int runningTasks() {
-        return runningTasks.get();
+        return runner.runningTasks();
     }
 
     // Only use for testing
     public int queueSize() {
-        return tasks.size();
-    }
-
-    private void runTask(final T task) {
-        try {
-            logger.trace("[{}] running task {}", taskRunnerName, task);
-            task.run();
-        } finally {
-            // To avoid missing to run tasks that are enqueued and waiting, we check the queue again once running
-            // a task is finished.
-            int decremented = runningTasks.decrementAndGet();
-            assert decremented >= 0;
-            pollAndSpawn();
-        }
+        return queue.size();
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/RunOnce.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/RunOnce.java
index 0b9334a03c44..e14f2d6463fa 100644
--- a/server/src/main/java/org/elasticsearch/common/util/concurrent/RunOnce.java
+++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/RunOnce.java
@@ -8,25 +8,24 @@
 package org.elasticsearch.common.util.concurrent;
 
 import java.util.Objects;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 
 /**
- * Runnable that can only be run one time.
+ * Runnable that prevents running its delegate more than once.
  */
 public class RunOnce implements Runnable {
 
-    private final Runnable delegate;
-    private final AtomicBoolean hasRun;
+    private final AtomicReference<Runnable> delegateRef;
 
     public RunOnce(final Runnable delegate) {
-        this.delegate = Objects.requireNonNull(delegate);
-        this.hasRun = new AtomicBoolean(false);
+        delegateRef = new AtomicReference<>(Objects.requireNonNull(delegate));
     }
 
     @Override
     public void run() {
-        if (hasRun.compareAndSet(false, true)) {
-            delegate.run();
+        var acquired = delegateRef.getAndSet(null);
+        if (acquired != null) {
+            acquired.run();
         }
     }
 
@@ -34,6 +33,11 @@ public void run() {
      * {@code true} if the {@link RunOnce} has been executed once.
      */
     public boolean hasRun() {
-        return hasRun.get();
+        return delegateRef.get() == null;
+    }
+
+    @Override
+    public String toString() {
+        return "RunOnce[" + delegateRef.get() + "]";
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
index 16d11a3cbc3c..ef004bb5d9e5 100644
--- a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
+++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
@@ -312,11 +312,29 @@ public StoredContext newStoredContext() {
 
     /**
      * Just like {@link #stashContext()} but no default context is set. Instead, the {@code transientHeadersToClear} argument can be used
-     * to clear specific transient headers in the new context. All headers (with the possible exception of {@code responseHeaders}) are
-     * restored by closing the returned {@link StoredContext}.
-     *
+     * to clear specific transient headers in the new context and {@code requestHeadersToClear} can be used to clear specific request
+     * headers. All original headers (without the {@code responseHeaders}) are restored by closing the returned {@link StoredContext}.
+     */
+    public StoredContext newStoredContext(Collection<String> transientHeadersToClear, Collection<String> requestHeadersToClear) {
+        return newStoredContext(false, transientHeadersToClear, requestHeadersToClear);
+    }
+
+    /**
+     * Just like {@link #newStoredContext(Collection, Collection)} but all headers are restored to original,
+     * except of {@code responseHeaders} which will be preserved from the restore thread.
      */
-    public StoredContext newStoredContext(Collection<String> transientHeadersToClear) {
+    public StoredContext newStoredContextPreservingResponseHeaders(
+        Collection<String> transientHeadersToClear,
+        Collection<String> requestHeadersToClear
+    ) {
+        return newStoredContext(true, transientHeadersToClear, requestHeadersToClear);
+    }
+
+    private StoredContext newStoredContext(
+        boolean preserveResponseHeaders,
+        Collection<String> transientHeadersToClear,
+        Collection<String> requestHeadersToClear
+    ) {
         final ThreadContextStruct originalContext = threadLocal.get();
         // clear specific transient headers from the current context
         Map<String, Object> newTransientHeaders = null;
@@ -328,18 +346,34 @@ public StoredContext newStoredContext(Collection<String> transientHeadersToClear
                 newTransientHeaders.remove(transientHeaderToClear);
             }
         }
-        // this is the context when this method returns
-        if (newTransientHeaders != null) {
+        Map<String, String> newRequestHeaders = null;
+        for (String requestHeaderToClear : requestHeadersToClear) {
+            if (originalContext.requestHeaders.containsKey(requestHeaderToClear)) {
+                if (newRequestHeaders == null) {
+                    newRequestHeaders = new HashMap<>(originalContext.requestHeaders);
+                }
+                newRequestHeaders.remove(requestHeaderToClear);
+            }
+        }
+        if (newTransientHeaders != null || newRequestHeaders != null) {
             ThreadContextStruct threadContextStruct = new ThreadContextStruct(
-                originalContext.requestHeaders,
+                newRequestHeaders != null ? newRequestHeaders : originalContext.requestHeaders,
                 originalContext.responseHeaders,
-                newTransientHeaders,
+                newTransientHeaders != null ? newTransientHeaders : originalContext.transientHeaders,
                 originalContext.isSystemContext,
                 originalContext.warningHeadersSize
             );
             threadLocal.set(threadContextStruct);
         }
-        return storedOriginalContext(originalContext);
+        // this is the context when this method returns
+        final ThreadContextStruct newContext = threadLocal.get();
+        return () -> {
+            if (preserveResponseHeaders && threadLocal.get() != newContext) {
+                threadLocal.set(originalContext.putResponseHeaders(threadLocal.get().responseHeaders));
+            } else {
+                threadLocal.set(originalContext);
+            }
+        };
     }
 
     /**
@@ -510,6 +544,13 @@ public <T> T getTransient(String key) {
         return (T) threadLocal.get().transientHeaders.get(key);
     }
 
+    /**
+     * Returns unmodifiable copy of all transient headers.
+     */
+    public Map<String, Object> getTransientHeaders() {
+        return Collections.unmodifiableMap(threadLocal.get().transientHeaders);
+    }
+
     /**
      * Add the {@code value} for the specified {@code key} Any duplicate {@code value} is ignored.
      *
diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThrottledTaskRunner.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThrottledTaskRunner.java
new file mode 100644
index 000000000000..674e58ee766d
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThrottledTaskRunner.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common.util.concurrent;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.core.Releasable;
+
+import java.util.concurrent.Executor;
+
+public class ThrottledTaskRunner extends AbstractThrottledTaskRunner<ActionListener<Releasable>> {
+    // a simple AbstractThrottledTaskRunner which fixes the task type and uses a regular FIFO blocking queue.
+    public ThrottledTaskRunner(String name, int maxRunningTasks, Executor executor) {
+        super(name, maxRunningTasks, executor, ConcurrentCollections.newBlockingQueue());
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/common/util/set/Sets.java b/server/src/main/java/org/elasticsearch/common/util/set/Sets.java
index 9fa9f6adc34f..207bc844e522 100644
--- a/server/src/main/java/org/elasticsearch/common/util/set/Sets.java
+++ b/server/src/main/java/org/elasticsearch/common/util/set/Sets.java
@@ -159,6 +159,14 @@ public static <T> Set<T> union(Set<T> left, Set<T> right) {
         return union;
     }
 
+    /**
+     * The intersection of two sets. Namely, the resulting set contains all the elements that are in both sets.
+     * Neither input is mutated by this operation, an entirely new set is returned.
+     *
+     * @param set1 the first set
+     * @param set2 the second set
+     * @return the unmodifiable intersection of the two sets
+     */
     public static <T> Set<T> intersection(Set<T> set1, Set<T> set2) {
         Objects.requireNonNull(set1);
         Objects.requireNonNull(set2);
@@ -171,7 +179,21 @@ public static <T> Set<T> intersection(Set<T> set1, Set<T> set2) {
             left = set2;
             right = set1;
         }
-        return left.stream().filter(right::contains).collect(Collectors.toSet());
+
+        final Set<T> empty = Set.of();
+        Set<T> result = empty;
+        for (T t : left) {
+            if (right.contains(t)) {
+                if (result == empty) {
+                    // delay allocation of a non-empty result set
+                    result = new HashSet<>();
+                }
+                result.add(t);
+            }
+        }
+
+        // the empty set is already unmodifiable
+        return result == empty ? result : Collections.unmodifiableSet(result);
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java
index f565df41a7c0..a519b518e4c8 100644
--- a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java
+++ b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContent.java
@@ -8,38 +8,80 @@
 
 package org.elasticsearch.common.xcontent;
 
+import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 
+import java.io.IOException;
 import java.util.Iterator;
 
 /**
- * An extension of {@link ToXContent} that can be serialized in chunks by creating an {@link Iterator<ToXContent>}.
- * This is used by the REST layer to implement flow control that does not rely on blocking the serializing thread when writing the
- * serialized bytes to a non-blocking channel.
+ * An alternative to {@link ToXContent} allowing for progressive serialization by creating an {@link Iterator} of {@link ToXContent} chunks.
+ * <p>
+ * The REST layer only serializes enough chunks at once to keep an outbound buffer full, rather than consuming all the time and memory
+ * needed to serialize the entire response as must be done with the regular {@link ToXContent} responses.
  */
 public interface ChunkedToXContent {
 
     /**
-     * Create an iterator of {@link ToXContent} chunks, that must be serialized individually with the same {@link XContentBuilder} and
-     * {@link ToXContent.Params} for each call until it is fully drained.
+     * Create an iterator of {@link ToXContent} chunks for a REST response. Each chunk is serialized with the same {@link XContentBuilder}
+     * and {@link ToXContent.Params}, which is also the same as the {@link ToXContent.Params} passed as the {@code params} argument. For
+     * best results, all chunks should be {@code O(1)} size. See also {@link ChunkedToXContentHelper} for some handy utilities.
+     * <p>
+     * Note that chunked response bodies cannot send deprecation warning headers once transmission has started, so implementations must
+     * check for deprecated feature use before returning.
+     *
      * @return iterator over chunks of {@link ToXContent}
      */
-    Iterator<? extends ToXContent> toXContentChunked();
+    Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params);
 
     /**
-     * Wraps the given instance in a {@link ToXContentObject} that will fully serialize the instance when serialized.
+     * Create an iterator of {@link ToXContent} chunks for a response to the {@link RestApiVersion#V_7} API. Each chunk is serialized with
+     * the same {@link XContentBuilder} and {@link ToXContent.Params}, which is also the same as the {@link ToXContent.Params} passed as the
+     * {@code params} argument. For best results, all chunks should be {@code O(1)} size. See also {@link ChunkedToXContentHelper} for some
+     * handy utilities.
+     * <p>
+     * Similar to {@link #toXContentChunked} but for the {@link RestApiVersion#V_7} API. By default this method delegates to {@link
+     * #toXContentChunked}.
+     * <p>
+     * Note that chunked response bodies cannot send deprecation warning headers once transmission has started, so implementations must
+     * check for deprecated feature use before returning.
+     *
+     * @return iterator over chunks of {@link ToXContent}
+     */
+    default Iterator<? extends ToXContent> toXContentChunkedV7(ToXContent.Params params) {
+        return toXContentChunked(params);
+    }
+
+    /**
+     * Wraps the given instance in a {@link ToXContent} that will fully serialize the instance when serialized.
+     *
      * @param chunkedToXContent instance to wrap
-     * @return x-content object
+     * @return x-content instance
      */
-    static ToXContentObject wrapAsXContentObject(ChunkedToXContent chunkedToXContent) {
-        return (builder, params) -> {
-            Iterator<? extends ToXContent> serialization = chunkedToXContent.toXContentChunked();
-            while (serialization.hasNext()) {
-                serialization.next().toXContent(builder, params);
+    static ToXContent wrapAsToXContent(ChunkedToXContent chunkedToXContent) {
+        return new ToXContent() {
+            @Override
+            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+                Iterator<? extends ToXContent> serialization = chunkedToXContent.toXContentChunked(params);
+                while (serialization.hasNext()) {
+                    serialization.next().toXContent(builder, params);
+                }
+                return builder;
+            }
+
+            @Override
+            public boolean isFragment() {
+                return chunkedToXContent.isFragment();
             }
-            return builder;
         };
     }
+
+    /**
+     * @return true iff this instance serializes as a fragment. See {@link ToXContentObject} for additional details.
+     */
+    default boolean isFragment() {
+        return true;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java
new file mode 100644
index 000000000000..6a51d708b57a
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common.xcontent;
+
+import org.elasticsearch.common.collect.Iterators;
+import org.elasticsearch.xcontent.ToXContent;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.function.Function;
+
+public enum ChunkedToXContentHelper {
+    ;
+
+    public static Iterator<ToXContent> startObject() {
+        return Iterators.single(((builder, params) -> builder.startObject()));
+    }
+
+    public static Iterator<ToXContent> startObject(String name) {
+        return Iterators.single(((builder, params) -> builder.startObject(name)));
+    }
+
+    public static Iterator<ToXContent> endObject() {
+        return Iterators.single(((builder, params) -> builder.endObject()));
+    }
+
+    public static Iterator<ToXContent> startArray(String name) {
+        return Iterators.single(((builder, params) -> builder.startArray(name)));
+    }
+
+    public static Iterator<ToXContent> endArray() {
+        return Iterators.single(((builder, params) -> builder.endArray()));
+    }
+
+    public static Iterator<ToXContent> map(String name, Map<String, ?> map) {
+        return map(name, map, entry -> (ToXContent) (builder, params) -> builder.field(entry.getKey(), entry.getValue()));
+    }
+
+    public static Iterator<ToXContent> xContentFragmentValuesMap(String name, Map<String, ? extends ToXContent> map) {
+        return map(
+            name,
+            map,
+            entry -> (ToXContent) (builder, params) -> entry.getValue().toXContent(builder.startObject(entry.getKey()), params).endObject()
+        );
+    }
+
+    public static Iterator<ToXContent> xContentValuesMap(String name, Map<String, ? extends ToXContent> map) {
+        return map(
+            name,
+            map,
+            entry -> (ToXContent) (builder, params) -> entry.getValue().toXContent(builder.field(entry.getKey()), params)
+        );
+    }
+
+    public static Iterator<ToXContent> field(String name, boolean value) {
+        return Iterators.single(((builder, params) -> builder.field(name, value)));
+    }
+
+    public static Iterator<ToXContent> array(String name, Iterator<? extends ToXContent> contents) {
+        return Iterators.concat(ChunkedToXContentHelper.startArray(name), contents, ChunkedToXContentHelper.endArray());
+    }
+
+    public static <T extends ToXContent> Iterator<ToXContent> wrapWithObject(String name, Iterator<T> iterator) {
+        return Iterators.concat(startObject(name), iterator, endObject());
+    }
+
+    private static <T> Iterator<ToXContent> map(String name, Map<String, T> map, Function<Map.Entry<String, T>, ToXContent> toXContent) {
+        return wrapWithObject(name, map.entrySet().stream().map(toXContent).iterator());
+    }
+
+    public static Iterator<ToXContent> singleChunk(ToXContent... contents) {
+        return Iterators.single((builder, params) -> {
+            for (ToXContent content : contents) {
+                content.toXContent(builder, params);
+            }
+            return builder;
+        });
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentObject.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentObject.java
new file mode 100644
index 000000000000..b0fad066cd82
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentObject.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.common.xcontent;
+
+/**
+ * Chunked equivalent of {@link org.elasticsearch.xcontent.ToXContentObject} that serializes as a full object.
+ */
+public interface ChunkedToXContentObject extends ChunkedToXContent {
+
+    @Override
+    default boolean isFragment() {
+        return false;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java
index a71f4fe70f32..fda5055e5585 100644
--- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java
+++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java
@@ -10,7 +10,7 @@
 
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -594,7 +594,7 @@ public static BytesReference childBytes(XContentParser parser) throws IOExceptio
      * @param xContentType an instance to serialize
      */
     public static void writeTo(StreamOutput out, XContentType xContentType) throws IOException {
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             // when sending an enumeration to <v8 node it does not have new VND_ XContentType instances
             out.writeVInt(xContentType.canonical().ordinal());
         } else {
diff --git a/server/src/main/java/org/elasticsearch/discovery/DiscoveryStats.java b/server/src/main/java/org/elasticsearch/discovery/DiscoveryStats.java
index c755e05af0aa..8d1368555a72 100644
--- a/server/src/main/java/org/elasticsearch/discovery/DiscoveryStats.java
+++ b/server/src/main/java/org/elasticsearch/discovery/DiscoveryStats.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.discovery;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.coordination.PendingClusterStateStats;
 import org.elasticsearch.cluster.coordination.PublishClusterStateStats;
 import org.elasticsearch.cluster.service.ClusterApplierRecordingService;
@@ -43,12 +43,12 @@ public DiscoveryStats(
     public DiscoveryStats(StreamInput in) throws IOException {
         queueStats = in.readOptionalWriteable(PendingClusterStateStats::new);
         publishStats = in.readOptionalWriteable(PublishClusterStateStats::new);
-        if (in.getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             clusterStateUpdateStats = in.readOptionalWriteable(ClusterStateUpdateStats::new);
         } else {
             clusterStateUpdateStats = null;
         }
-        if (in.getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             applierRecordingStats = in.readOptionalWriteable(ClusterApplierRecordingService.Stats::new);
         } else {
             applierRecordingStats = null;
@@ -59,10 +59,10 @@ public DiscoveryStats(StreamInput in) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(queueStats);
         out.writeOptionalWriteable(publishStats);
-        if (out.getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             out.writeOptionalWriteable(clusterStateUpdateStats);
         }
-        if (out.getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             out.writeOptionalWriteable(applierRecordingStats);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/discovery/HandshakingTransportAddressConnector.java b/server/src/main/java/org/elasticsearch/discovery/HandshakingTransportAddressConnector.java
index f852a4f584e6..74b1fda553ab 100644
--- a/server/src/main/java/org/elasticsearch/discovery/HandshakingTransportAddressConnector.java
+++ b/server/src/main/java/org/elasticsearch/discovery/HandshakingTransportAddressConnector.java
@@ -12,7 +12,6 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.NotifyOnceListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.UUIDs;
@@ -95,10 +94,10 @@ public void connectToRemoteMasterNode(TransportAddress transportAddress, ActionL
 
                     // use NotifyOnceListener to make sure the following line does not result in onFailure being called when
                     // the connection is closed in the onResponse handler
-                    transportService.handshake(connection, probeHandshakeTimeout, new NotifyOnceListener<>() {
+                    transportService.handshake(connection, probeHandshakeTimeout, ActionListener.notifyOnce(new ActionListener<>() {
 
                         @Override
-                        protected void innerOnResponse(DiscoveryNode remoteNode) {
+                        public void onResponse(DiscoveryNode remoteNode) {
                             try {
                                 // success means (amongst other things) that the cluster names match
                                 logger.trace("[{}] handshake successful: {}", transportAddress, remoteNode);
@@ -166,7 +165,7 @@ public void onFailure(Exception e) {
                         }
 
                         @Override
-                        protected void innerOnFailure(Exception e) {
+                        public void onFailure(Exception e) {
                             // we opened a connection and successfully performed a low-level handshake, so we were definitely
                             // talking to an Elasticsearch node, but the high-level handshake failed indicating some kind of
                             // mismatched configurations (e.g. cluster name) that the user should address
@@ -175,7 +174,7 @@ protected void innerOnFailure(Exception e) {
                             listener.onFailure(e);
                         }
 
-                    });
+                    }));
 
                 })
             );
diff --git a/server/src/main/java/org/elasticsearch/env/Environment.java b/server/src/main/java/org/elasticsearch/env/Environment.java
index 90d03fe1ce94..9db7f831a186 100644
--- a/server/src/main/java/org/elasticsearch/env/Environment.java
+++ b/server/src/main/java/org/elasticsearch/env/Environment.java
@@ -9,9 +9,11 @@
 package org.elasticsearch.env;
 
 import org.apache.lucene.util.Constants;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.StatelessSecureSettings;
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.SuppressForbidden;
 
@@ -156,7 +158,12 @@ public Environment(final Settings settings, final Path configPath) {
             assert sharedDataFile != null;
             finalSettings.put(Environment.PATH_SHARED_DATA_SETTING.getKey(), sharedDataFile.toString());
         }
-        this.settings = finalSettings.build();
+
+        if (DiscoveryNode.isStateless(settings)) {
+            this.settings = StatelessSecureSettings.install(finalSettings.build());
+        } else {
+            this.settings = finalSettings.build();
+        }
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java
index e4c42af85af4..c21f2c172795 100644
--- a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java
+++ b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.env;
 
+import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.util.Strings;
@@ -25,8 +26,10 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.FileSystemUtils;
+import org.elasticsearch.common.logging.ChunkedLoggingStream;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -49,12 +52,15 @@
 import org.elasticsearch.index.store.FsDirectoryFactory;
 import org.elasticsearch.monitor.fs.FsInfo;
 import org.elasticsearch.monitor.fs.FsProbe;
+import org.elasticsearch.monitor.jvm.HotThreads;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.io.OutputStreamWriter;
 import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.AtomicMoveNotSupportedException;
 import java.nio.file.DirectoryStream;
 import java.nio.file.FileStore;
@@ -926,6 +932,38 @@ public Set<ShardId> lockedShards() {
         }
     }
 
+    // throttle the hot-threads calls: no more than one per minute
+    private final Semaphore shardLockHotThreadsPermit = new Semaphore(1);
+    private long nextShardLockHotThreadsNanos = Long.MIN_VALUE;
+
+    private void maybeLogThreadDump(ShardId shardId, String message) {
+        if (logger.isDebugEnabled() == false) {
+            return;
+        }
+
+        final var prefix = format("hot threads while failing to obtain shard lock for %s: %s", shardId, message);
+        if (shardLockHotThreadsPermit.tryAcquire()) {
+            try {
+                final var now = System.nanoTime();
+                if (now <= nextShardLockHotThreadsNanos) {
+                    return;
+                }
+                nextShardLockHotThreadsNanos = now + TimeUnit.SECONDS.toNanos(60);
+                final var hotThreads = new HotThreads().busiestThreads(500).ignoreIdleThreads(false).detect();
+                try (
+                    var stream = ChunkedLoggingStream.create(logger, Level.DEBUG, prefix, ReferenceDocs.SHARD_LOCK_TROUBLESHOOTING);
+                    var writer = new OutputStreamWriter(stream, StandardCharsets.UTF_8)
+                ) {
+                    writer.write(hotThreads);
+                }
+            } catch (Exception e) {
+                logger.error(format("could not obtain %s", prefix), e);
+            } finally {
+                shardLockHotThreadsPermit.release();
+            }
+        }
+    }
+
     private final class InternalShardLock {
         /*
          * This class holds a mutex for exclusive access and timeout / wait semantics
@@ -975,18 +1013,15 @@ void acquire(long timeoutInMillis, final String details) throws ShardLockObtainF
                     setDetails(details);
                 } else {
                     final Tuple<Long, String> lockDetails = this.lockDetails; // single volatile read
-                    throw new ShardLockObtainFailedException(
-                        shardId,
-                        "obtaining shard lock for ["
-                            + details
-                            + "] timed out after ["
-                            + timeoutInMillis
-                            + "ms], lock already held for ["
-                            + lockDetails.v2()
-                            + "] with age ["
-                            + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - lockDetails.v1())
-                            + "ms]"
+                    final var message = format(
+                        "obtaining shard lock for [%s] timed out after [%dms], lock already held for [%s] with age [%dms]",
+                        details,
+                        timeoutInMillis,
+                        lockDetails.v2(),
+                        TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - lockDetails.v1())
                     );
+                    maybeLogThreadDump(shardId, message);
+                    throw new ShardLockObtainFailedException(shardId, message);
                 }
             } catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
diff --git a/server/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java b/server/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java
index 56f913984dfe..1249074bfabc 100644
--- a/server/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java
+++ b/server/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java
@@ -49,36 +49,21 @@
  */
 public abstract class AsyncShardFetch<T extends BaseNodeResponse> implements Releasable {
 
-    /**
-     * An action that lists the relevant shard data that needs to be fetched.
-     */
-    public interface Lister<NodesResponse extends BaseNodesResponse<NodeResponse>, NodeResponse extends BaseNodeResponse> {
-        void list(ShardId shardId, @Nullable String customDataPath, DiscoveryNode[] nodes, ActionListener<NodesResponse> listener);
-    }
-
     protected final Logger logger;
     protected final String type;
     protected final ShardId shardId;
     protected final String customDataPath;
-    private final Lister<BaseNodesResponse<T>, T> action;
     private final Map<String, NodeEntry<T>> cache = new HashMap<>();
     private final Set<String> nodesToIgnore = new HashSet<>();
     private final AtomicLong round = new AtomicLong();
     private boolean closed;
 
     @SuppressWarnings("unchecked")
-    protected AsyncShardFetch(
-        Logger logger,
-        String type,
-        ShardId shardId,
-        String customDataPath,
-        Lister<? extends BaseNodesResponse<T>, T> action
-    ) {
+    protected AsyncShardFetch(Logger logger, String type, ShardId shardId, String customDataPath) {
         this.logger = logger;
         this.type = type;
         this.shardId = Objects.requireNonNull(shardId);
         this.customDataPath = Objects.requireNonNull(customDataPath);
-        this.action = (Lister<BaseNodesResponse<T>, T>) action;
     }
 
     @Override
@@ -307,7 +292,7 @@ private boolean hasAnyNodeFetching(Map<String, NodeEntry<T>> shardCache) {
     // visible for testing
     void asyncFetch(final DiscoveryNode[] nodes, long fetchingRound) {
         logger.trace("{} fetching [{}] from {}", shardId, type, nodes);
-        action.list(shardId, customDataPath, nodes, new ActionListener<BaseNodesResponse<T>>() {
+        list(shardId, customDataPath, nodes, new ActionListener<BaseNodesResponse<T>>() {
             @Override
             public void onResponse(BaseNodesResponse<T> response) {
                 assert assertSameNodes(response);
@@ -336,6 +321,13 @@ private boolean assertSameNodes(BaseNodesResponse<T> response) {
         });
     }
 
+    protected abstract void list(
+        ShardId shardId,
+        @Nullable String customDataPath,
+        DiscoveryNode[] nodes,
+        ActionListener<BaseNodesResponse<T>> listener
+    );
+
     /**
      * The result of a fetch operation. Make sure to first check {@link #hasData()} before
      * fetching the actual data.
diff --git a/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java b/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java
index 914cd156643b..6c7c751491c3 100644
--- a/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java
+++ b/server/src/main/java/org/elasticsearch/gateway/ClusterStateUpdaters.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRoutingRoleStrategy;
 import org.elasticsearch.common.settings.ClusterSettings;
 
 import java.util.Map;
@@ -82,9 +83,9 @@ static ClusterState recoverClusterBlocks(final ClusterState state) {
         return ClusterState.builder(state).blocks(blocks).build();
     }
 
-    static ClusterState updateRoutingTable(final ClusterState state) {
+    static ClusterState updateRoutingTable(final ClusterState state, ShardRoutingRoleStrategy shardRoutingRoleStrategy) {
         // initialize all index routing tables as empty
-        final RoutingTable.Builder routingTableBuilder = RoutingTable.builder(state.routingTable());
+        final RoutingTable.Builder routingTableBuilder = RoutingTable.builder(shardRoutingRoleStrategy, state.routingTable());
         for (final IndexMetadata indexMetadata : state.metadata().indices().values()) {
             routingTableBuilder.addAsRecovery(indexMetadata);
         }
diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java b/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
index fdd2f6b0052b..a8dbda1b137d 100644
--- a/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
+++ b/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.core.Releasables;
-import org.elasticsearch.gateway.AsyncShardFetch.Lister;
 import org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.NodeGatewayStartedShards;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.store.TransportNodesListShardStoreMetadata;
@@ -212,16 +211,10 @@ private boolean hasNewNodes(DiscoveryNodes nodes) {
         return false;
     }
 
-    class InternalAsyncFetch<T extends BaseNodeResponse> extends AsyncShardFetch<T> {
+    abstract class InternalAsyncFetch<T extends BaseNodeResponse> extends AsyncShardFetch<T> {
 
-        InternalAsyncFetch(
-            Logger logger,
-            String type,
-            ShardId shardId,
-            String customDataPath,
-            Lister<? extends BaseNodesResponse<T>, T> action
-        ) {
-            super(logger, type, shardId, customDataPath, action);
+        InternalAsyncFetch(Logger logger, String type, ShardId shardId, String customDataPath) {
+            super(logger, type, shardId, customDataPath);
         }
 
         @Override
@@ -250,16 +243,28 @@ class InternalPrimaryShardAllocator extends PrimaryShardAllocator {
         @Override
         protected AsyncShardFetch.FetchResult<NodeGatewayStartedShards> fetchData(ShardRouting shard, RoutingAllocation allocation) {
             // explicitly type lister, some IDEs (Eclipse) are not able to correctly infer the function type
-            Lister<BaseNodesResponse<NodeGatewayStartedShards>, NodeGatewayStartedShards> lister = this::listStartedShards;
             AsyncShardFetch<NodeGatewayStartedShards> fetch = asyncFetchStarted.computeIfAbsent(
                 shard.shardId(),
                 shardId -> new InternalAsyncFetch<>(
                     logger,
                     "shard_started",
                     shardId,
-                    IndexMetadata.INDEX_DATA_PATH_SETTING.get(allocation.metadata().index(shard.index()).getSettings()),
-                    lister
-                )
+                    IndexMetadata.INDEX_DATA_PATH_SETTING.get(allocation.metadata().index(shard.index()).getSettings())
+                ) {
+                    @Override
+                    protected void list(
+                        ShardId shardId,
+                        String customDataPath,
+                        DiscoveryNode[] nodes,
+                        ActionListener<BaseNodesResponse<NodeGatewayStartedShards>> listener
+                    ) {
+                        client.executeLocally(
+                            TransportNodesListGatewayStartedShards.TYPE,
+                            new TransportNodesListGatewayStartedShards.Request(shardId, customDataPath, nodes),
+                            ActionListener.wrap(listener)
+                        );
+                    }
+                }
             );
             AsyncShardFetch.FetchResult<NodeGatewayStartedShards> shardState = fetch.fetchData(
                 allocation.nodes(),
@@ -272,19 +277,6 @@ protected AsyncShardFetch.FetchResult<NodeGatewayStartedShards> fetchData(ShardR
             return shardState;
         }
 
-        private void listStartedShards(
-            ShardId shardId,
-            String customDataPath,
-            DiscoveryNode[] nodes,
-            ActionListener<BaseNodesResponse<NodeGatewayStartedShards>> listener
-        ) {
-            var request = new TransportNodesListGatewayStartedShards.Request(shardId, customDataPath, nodes);
-            client.executeLocally(
-                TransportNodesListGatewayStartedShards.TYPE,
-                request,
-                ActionListener.wrap(listener::onResponse, listener::onFailure)
-            );
-        }
     }
 
     class InternalReplicaShardAllocator extends ReplicaShardAllocator {
@@ -297,17 +289,28 @@ class InternalReplicaShardAllocator extends ReplicaShardAllocator {
 
         @Override
         protected AsyncShardFetch.FetchResult<NodeStoreFilesMetadata> fetchData(ShardRouting shard, RoutingAllocation allocation) {
-            // explicitly type lister, some IDEs (Eclipse) are not able to correctly infer the function type
-            Lister<BaseNodesResponse<NodeStoreFilesMetadata>, NodeStoreFilesMetadata> lister = this::listStoreFilesMetadata;
             AsyncShardFetch<NodeStoreFilesMetadata> fetch = asyncFetchStore.computeIfAbsent(
                 shard.shardId(),
                 shardId -> new InternalAsyncFetch<>(
                     logger,
                     "shard_store",
                     shard.shardId(),
-                    IndexMetadata.INDEX_DATA_PATH_SETTING.get(allocation.metadata().index(shard.index()).getSettings()),
-                    lister
-                )
+                    IndexMetadata.INDEX_DATA_PATH_SETTING.get(allocation.metadata().index(shard.index()).getSettings())
+                ) {
+                    @Override
+                    protected void list(
+                        ShardId shardId,
+                        String customDataPath,
+                        DiscoveryNode[] nodes,
+                        ActionListener<BaseNodesResponse<NodeStoreFilesMetadata>> listener
+                    ) {
+                        client.executeLocally(
+                            TransportNodesListShardStoreMetadata.TYPE,
+                            new TransportNodesListShardStoreMetadata.Request(shardId, customDataPath, nodes),
+                            ActionListener.wrap(listener)
+                        );
+                    }
+                }
             );
             AsyncShardFetch.FetchResult<NodeStoreFilesMetadata> shardStores = fetch.fetchData(
                 allocation.nodes(),
@@ -319,20 +322,6 @@ protected AsyncShardFetch.FetchResult<NodeStoreFilesMetadata> fetchData(ShardRou
             return shardStores;
         }
 
-        private void listStoreFilesMetadata(
-            ShardId shardId,
-            String customDataPath,
-            DiscoveryNode[] nodes,
-            ActionListener<BaseNodesResponse<NodeStoreFilesMetadata>> listener
-        ) {
-            var request = new TransportNodesListShardStoreMetadata.Request(shardId, customDataPath, nodes);
-            client.executeLocally(
-                TransportNodesListShardStoreMetadata.TYPE,
-                request,
-                ActionListener.wrap(listener::onResponse, listener::onFailure)
-            );
-        }
-
         @Override
         protected boolean hasInitiatedFetching(ShardRouting shard) {
             return asyncFetchStore.get(shard.shardId()) != null;
diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayService.java b/server/src/main/java/org/elasticsearch/gateway/GatewayService.java
index 3162a3a4ddf0..10bcc2cae9b6 100644
--- a/server/src/main/java/org/elasticsearch/gateway/GatewayService.java
+++ b/server/src/main/java/org/elasticsearch/gateway/GatewayService.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RerouteService;
+import org.elasticsearch.cluster.routing.ShardRoutingRoleStrategy;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.MasterService;
 import org.elasticsearch.common.Priority;
@@ -70,6 +71,7 @@ public class GatewayService extends AbstractLifecycleComponent implements Cluste
 
     static final TimeValue DEFAULT_RECOVER_AFTER_TIME_IF_EXPECTED_NODES_IS_SET = TimeValue.timeValueMinutes(5);
 
+    private final ShardRoutingRoleStrategy shardRoutingRoleStrategy;
     private final ThreadPool threadPool;
 
     private final RerouteService rerouteService;
@@ -87,10 +89,12 @@ public GatewayService(
         final Settings settings,
         final RerouteService rerouteService,
         final ClusterService clusterService,
+        final ShardRoutingRoleStrategy shardRoutingRoleStrategy,
         final ThreadPool threadPool
     ) {
         this.rerouteService = rerouteService;
         this.clusterService = clusterService;
+        this.shardRoutingRoleStrategy = shardRoutingRoleStrategy;
         this.threadPool = threadPool;
         this.expectedDataNodes = EXPECTED_DATA_NODES_SETTING.get(settings);
 
@@ -214,7 +218,9 @@ public ClusterState execute(final ClusterState currentState) {
                 logger.debug("cluster is already recovered");
                 return currentState;
             }
-            return ClusterStateUpdaters.removeStateNotRecoveredBlock(ClusterStateUpdaters.updateRoutingTable(currentState));
+            return ClusterStateUpdaters.removeStateNotRecoveredBlock(
+                ClusterStateUpdaters.updateRoutingTable(currentState, shardRoutingRoleStrategy)
+            );
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java b/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
index 509d29f885b7..cf4e903ef652 100644
--- a/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
+++ b/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
@@ -10,6 +10,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionListenerResponseHandler;
@@ -118,7 +119,10 @@ public ClusterState execute(ClusterState currentState) {
                     }
                     Metadata.Builder metadata = Metadata.builder(currentState.metadata());
                     ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
-                    RoutingTable.Builder routingTableBuilder = RoutingTable.builder(currentState.routingTable());
+                    RoutingTable.Builder routingTableBuilder = RoutingTable.builder(
+                        allocationService.getShardRoutingRoleStrategy(),
+                        currentState.routingTable()
+                    );
                     final Version minIndexCompatibilityVersion = currentState.getNodes()
                         .getMaxNodeVersion()
                         .minimumIndexCompatibilityVersion();
@@ -269,7 +273,7 @@ public void writeTo(StreamOutput out) throws IOException {
     public static class AllocateDangledResponse extends TransportResponse {
 
         private AllocateDangledResponse(StreamInput in) throws IOException {
-            if (in.getVersion().before(Version.V_8_0_0)) {
+            if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 in.readBoolean();
             }
         }
@@ -278,7 +282,7 @@ private AllocateDangledResponse() {}
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            if (out.getVersion().before(Version.V_8_0_0)) {
+            if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 out.writeBoolean(true);
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java b/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java
index 752ef4345229..28e0a7a1db16 100644
--- a/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java
+++ b/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java
@@ -60,6 +60,7 @@
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.CheckedFunction;
@@ -71,7 +72,6 @@
 import org.elasticsearch.env.NodeMetadata;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentParser;
@@ -1082,7 +1082,7 @@ private void addIndexMetadataDocuments(IndexMetadata indexMetadata) throws IOExc
 
         private void addGlobalMetadataDocuments(Metadata metadata) throws IOException {
             logger.trace("updating global metadata doc");
-            writePages(metadata, (bytesRef, pageIndex, isLastPage) -> {
+            writePages(ChunkedToXContent.wrapAsToXContent(metadata), (bytesRef, pageIndex, isLastPage) -> {
                 final Document document = new Document();
                 document.add(new StringField(TYPE_FIELD_NAME, GLOBAL_TYPE_NAME, Field.Store.NO));
                 document.add(new StoredField(PAGE_FIELD_NAME, pageIndex));
@@ -1094,7 +1094,7 @@ private void addGlobalMetadataDocuments(Metadata metadata) throws IOException {
             });
         }
 
-        private void writePages(ToXContentFragment metadata, PageWriter pageWriter) throws IOException {
+        private void writePages(ToXContent metadata, PageWriter pageWriter) throws IOException {
             try (
                 PageWriterOutputStream paginatedStream = new PageWriterOutputStream(documentBuffer, pageWriter);
                 OutputStream compressedStream = CompressorFactory.COMPRESSOR.threadLocalOutputStream(paginatedStream);
diff --git a/server/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java b/server/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
index 393e4fd66c9d..5246d875692c 100644
--- a/server/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
+++ b/server/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
@@ -327,7 +327,7 @@ public static Tuple<Decision, Map<String, NodeAllocationResult>> canBeAllocatedT
      * Takes the store info for nodes that have a shard store and adds them to the node decisions,
      * leaving the node explanations untouched for those nodes that do not have any store information.
      */
-    private static List<NodeAllocationResult> augmentExplanationsWithStoreInfo(
+    public static List<NodeAllocationResult> augmentExplanationsWithStoreInfo(
         Map<String, NodeAllocationResult> nodeDecisions,
         Map<String, NodeAllocationResult> withShardStores
     ) {
diff --git a/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java b/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
index 0a40899762ed..8fc67c3a3195 100644
--- a/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
+++ b/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
@@ -8,8 +8,10 @@
 
 package org.elasticsearch.gateway;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.ActionFilters;
@@ -56,6 +58,8 @@ public class TransportNodesListGatewayStartedShards extends TransportNodesAction
     TransportNodesListGatewayStartedShards.NodeRequest,
     TransportNodesListGatewayStartedShards.NodeGatewayStartedShards> {
 
+    private static final Logger logger = LogManager.getLogger(TransportNodesListGatewayStartedShards.class);
+
     public static final String ACTION_NAME = "internal:gateway/local/started_shards";
     public static final ActionType<NodesGatewayStartedShards> TYPE = new ActionType<>(ACTION_NAME, NodesGatewayStartedShards::new);
 
@@ -187,7 +191,7 @@ public static class Request extends BaseNodesRequest<Request> {
         public Request(StreamInput in) throws IOException {
             super(in);
             shardId = new ShardId(in);
-            if (in.getVersion().onOrAfter(Version.V_7_6_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_6_0)) {
                 customDataPath = in.readString();
             } else {
                 customDataPath = null;
@@ -218,7 +222,7 @@ public String getCustomDataPath() {
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             shardId.writeTo(out);
-            if (out.getVersion().onOrAfter(Version.V_7_6_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_6_0)) {
                 out.writeString(customDataPath);
             }
         }
@@ -258,7 +262,7 @@ public static class NodeRequest extends TransportRequest {
         public NodeRequest(StreamInput in) throws IOException {
             super(in);
             shardId = new ShardId(in);
-            if (in.getVersion().onOrAfter(Version.V_7_6_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_6_0)) {
                 customDataPath = in.readString();
             } else {
                 customDataPath = null;
@@ -274,7 +278,7 @@ public NodeRequest(Request request) {
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             shardId.writeTo(out);
-            if (out.getVersion().onOrAfter(Version.V_7_6_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_6_0)) {
                 assert customDataPath != null;
                 out.writeString(customDataPath);
             }
diff --git a/server/src/main/java/org/elasticsearch/health/Diagnosis.java b/server/src/main/java/org/elasticsearch/health/Diagnosis.java
index 2f3d44b1f699..a190dd3d5df0 100644
--- a/server/src/main/java/org/elasticsearch/health/Diagnosis.java
+++ b/server/src/main/java/org/elasticsearch/health/Diagnosis.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.ToXContent;
 
@@ -19,9 +20,6 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
-import java.util.Spliterator;
-import java.util.Spliterators;
-import java.util.stream.StreamSupport;
 
 import static org.elasticsearch.health.HealthService.HEALTH_API_ID_PREFIX;
 
@@ -46,6 +44,7 @@ public enum Type {
             INDEX("indices"),
             NODE("nodes"),
             SLM_POLICY("slm_policies"),
+            FEATURE_STATE("feature_states"),
             SNAPSHOT_REPOSITORY("snapshot_repositories");
 
             private final String displayValue;
@@ -77,8 +76,8 @@ public Resource(Collection<DiscoveryNode> nodes) {
         }
 
         @Override
-        public Iterator<? extends ToXContent> toXContentChunked() {
-            Iterator<? extends ToXContent> valuesIterator;
+        public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
+            final Iterator<? extends ToXContent> valuesIterator;
             if (nodes != null) {
                 valuesIterator = nodes.stream().map(node -> (ToXContent) (builder, params) -> {
                     builder.startObject();
@@ -92,11 +91,7 @@ public Iterator<? extends ToXContent> toXContentChunked() {
             } else {
                 valuesIterator = values.stream().map(value -> (ToXContent) (builder, params) -> builder.value(value)).iterator();
             }
-            return Iterators.concat(
-                Iterators.single((ToXContent) (builder, params) -> builder.startArray(type.displayValue)),
-                valuesIterator,
-                Iterators.single((builder, params) -> builder.endArray())
-            );
+            return ChunkedToXContentHelper.array(type.displayValue, valuesIterator);
         }
 
         @Override
@@ -147,12 +142,12 @@ public String getUniqueId() {
     }
 
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked() {
-        Iterator<? extends ToXContent> resourcesIterator = Collections.emptyIterator();
-        if (affectedResources != null && affectedResources.size() > 0) {
-            resourcesIterator = affectedResources.stream()
-                .flatMap(s -> StreamSupport.stream(Spliterators.spliteratorUnknownSize(s.toXContentChunked(), Spliterator.ORDERED), false))
-                .iterator();
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
+        final Iterator<? extends ToXContent> resourcesIterator;
+        if (affectedResources == null) {
+            resourcesIterator = Collections.emptyIterator();
+        } else {
+            resourcesIterator = Iterators.flatMap(affectedResources.iterator(), s -> s.toXContentChunked(outerParams));
         }
         return Iterators.concat(Iterators.single((ToXContent) (builder, params) -> {
             builder.startObject();
diff --git a/server/src/main/java/org/elasticsearch/health/GetHealthAction.java b/server/src/main/java/org/elasticsearch/health/GetHealthAction.java
index e8a8a823308e..5e60b2b6c87b 100644
--- a/server/src/main/java/org/elasticsearch/health/GetHealthAction.java
+++ b/server/src/main/java/org/elasticsearch/health/GetHealthAction.java
@@ -37,6 +37,8 @@
 import java.util.NoSuchElementException;
 import java.util.Objects;
 
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
 public class GetHealthAction extends ActionType<GetHealthAction.Response> {
 
     public static final GetHealthAction INSTANCE = new GetHealthAction();
@@ -93,7 +95,7 @@ public void writeTo(StreamOutput out) throws IOException {
 
         @Override
         @SuppressWarnings("unchecked")
-        public Iterator<? extends ToXContent> toXContentChunked() {
+        public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
             return Iterators.concat(Iterators.single((ToXContent) (builder, params) -> {
                 builder.startObject();
                 if (status != null) {
@@ -111,7 +113,7 @@ public Iterator<? extends ToXContent> toXContentChunked() {
                                 // indicators however the affected resources which are the O(indices) fields are
                                 // flat mapped over all diagnoses within the indicator
                                 Iterators.single((ToXContent) (builder, params) -> builder.field(indicator.name())),
-                                indicator.toXContentChunked()
+                                indicator.toXContentChunked(outerParams)
                             )
                         )
                         .toArray(Iterator[]::new)
@@ -146,21 +148,25 @@ public String toString() {
     public static class Request extends ActionRequest {
         private final String indicatorName;
         private final boolean verbose;
+        private final int size;
 
-        public Request(boolean verbose) {
-            // We never compute details if no indicator name is given because of the runtime cost:
-            this.indicatorName = null;
-            this.verbose = verbose;
+        public Request(boolean verbose, int size) {
+            this(null, verbose, size);
         }
 
-        public Request(String indicatorName, boolean verbose) {
+        public Request(String indicatorName, boolean verbose, int size) {
             this.indicatorName = indicatorName;
             this.verbose = verbose;
+            this.size = size;
         }
 
         @Override
         public ActionRequestValidationException validate() {
-            return null;
+            ActionRequestValidationException validationException = null;
+            if (size < 0) {
+                validationException = addValidationError("The size parameter must be a positive integer", validationException);
+            }
+            return validationException;
         }
 
         @Override
@@ -195,11 +201,21 @@ public TransportAction(
         @Override
         protected void doExecute(Task task, Request request, ActionListener<Response> responseListener) {
             assert task instanceof CancellableTask;
-            healthService.getHealth(client, request.indicatorName, request.verbose, responseListener.map(healthIndicatorResults -> {
-                Response response = new Response(clusterService.getClusterName(), healthIndicatorResults, request.indicatorName == null);
-                healthApiStats.track(request.verbose, response);
-                return response;
-            }));
+            healthService.getHealth(
+                client,
+                request.indicatorName,
+                request.verbose,
+                request.size,
+                responseListener.map(healthIndicatorResults -> {
+                    Response response = new Response(
+                        clusterService.getClusterName(),
+                        healthIndicatorResults,
+                        request.indicatorName == null
+                    );
+                    healthApiStats.track(request.verbose, response);
+                    return response;
+                })
+            );
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/health/HealthIndicatorResult.java b/server/src/main/java/org/elasticsearch/health/HealthIndicatorResult.java
index 46ecabca1590..6e8e884a41d5 100644
--- a/server/src/main/java/org/elasticsearch/health/HealthIndicatorResult.java
+++ b/server/src/main/java/org/elasticsearch/health/HealthIndicatorResult.java
@@ -9,15 +9,12 @@
 package org.elasticsearch.health;
 
 import org.elasticsearch.common.collect.Iterators;
-import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.xcontent.ToXContent;
 
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Spliterator;
-import java.util.Spliterators;
-import java.util.stream.StreamSupport;
 
 public record HealthIndicatorResult(
     String name,
@@ -26,14 +23,14 @@ public record HealthIndicatorResult(
     HealthIndicatorDetails details,
     List<HealthIndicatorImpact> impacts,
     List<Diagnosis> diagnosisList
-) implements ChunkedToXContent {
+) implements ChunkedToXContentObject {
     @Override
-    public Iterator<? extends ToXContent> toXContentChunked() {
-        Iterator<? extends ToXContent> diagnosisIterator = Collections.emptyIterator();
-        if (diagnosisList != null && diagnosisList.isEmpty() == false) {
-            diagnosisIterator = diagnosisList.stream()
-                .flatMap(s -> StreamSupport.stream(Spliterators.spliteratorUnknownSize(s.toXContentChunked(), Spliterator.ORDERED), false))
-                .iterator();
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
+        final Iterator<? extends ToXContent> diagnosisIterator;
+        if (diagnosisList == null) {
+            diagnosisIterator = Collections.emptyIterator();
+        } else {
+            diagnosisIterator = Iterators.flatMap(diagnosisList.iterator(), s -> s.toXContentChunked(outerParams));
         }
         return Iterators.concat(Iterators.single((ToXContent) (builder, params) -> {
             builder.startObject();
diff --git a/server/src/main/java/org/elasticsearch/health/HealthIndicatorService.java b/server/src/main/java/org/elasticsearch/health/HealthIndicatorService.java
index 1c0972f8cb35..867df43aece3 100644
--- a/server/src/main/java/org/elasticsearch/health/HealthIndicatorService.java
+++ b/server/src/main/java/org/elasticsearch/health/HealthIndicatorService.java
@@ -17,12 +17,21 @@
 
 /**
  * This is a service interface used to calculate health indicator from the different modules or plugins.
+
+ * NOTE: if you are adding the name of an indicator or the id of a diagnosis you need to update the configuration
+ * of the health-api-indexer in the telemetry repository so the new/changed fields will be indexed properly.
  */
 public interface HealthIndicatorService {
 
+    int MAX_AFFECTED_RESOURCES_COUNT = 1000;
+
     String name();
 
-    HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo);
+    default HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
+        return calculate(verbose, MAX_AFFECTED_RESOURCES_COUNT, healthInfo);
+    }
+
+    HealthIndicatorResult calculate(boolean verbose, int maxAffectedResourcesCount, HealthInfo healthInfo);
 
     /**
      * This method creates a HealthIndicatorResult with the given information. Note that it sorts the impacts by severity (the lower the
diff --git a/server/src/main/java/org/elasticsearch/health/HealthService.java b/server/src/main/java/org/elasticsearch/health/HealthService.java
index c3079c6a7170..7828f12e7aba 100644
--- a/server/src/main/java/org/elasticsearch/health/HealthService.java
+++ b/server/src/main/java/org/elasticsearch/health/HealthService.java
@@ -76,20 +76,25 @@ public HealthService(
      *
      * @param client        A client to be used to fetch the health data from the health node
      * @param indicatorName If not null, the returned results will only have this indicator
-     * @param explain       Whether to compute the details portion of the results
+     * @param verbose       Whether to compute the details portion of the results
      * @param listener      A listener to be notified of the list of all HealthIndicatorResult if indicatorName is null, or one
      *                      HealthIndicatorResult if indicatorName is not null
+     * @param maxAffectedResourcesCount The maximum number of affected resources to return per each type.
      * @throws ResourceNotFoundException if an indicator name is given and the indicator is not found
      */
     public void getHealth(
         Client client,
         @Nullable String indicatorName,
-        boolean explain,
+        boolean verbose,
+        int maxAffectedResourcesCount,
         ActionListener<List<HealthIndicatorResult>> listener
     ) {
+        if (maxAffectedResourcesCount < 0) {
+            throw new IllegalArgumentException("The max number of resources must be a positive integer");
+        }
         // Determine if cluster is stable enough to calculate health before running other indicators
         List<HealthIndicatorResult> preflightResults = preflightHealthIndicatorServices.stream()
-            .map(service -> service.calculate(explain, HealthInfo.EMPTY_HEALTH_INFO))
+            .map(service -> service.calculate(verbose, maxAffectedResourcesCount, HealthInfo.EMPTY_HEALTH_INFO))
             .toList();
 
         // If any of these are not GREEN, then we cannot obtain health from other indicators
@@ -113,7 +118,7 @@ public void onResponse(FetchHealthInfoCacheAction.Response response) {
                     ActionRunnable<List<HealthIndicatorResult>> calculateFilteredIndicatorsRunnable = calculateFilteredIndicatorsRunnable(
                         indicatorName,
                         healthInfo,
-                        explain,
+                        verbose,
                         listener
                     );
 
@@ -131,7 +136,7 @@ public void onFailure(Exception e) {
                     ActionRunnable<List<HealthIndicatorResult>> calculateFilteredIndicatorsRunnable = calculateFilteredIndicatorsRunnable(
                         indicatorName,
                         HealthInfo.EMPTY_HEALTH_INFO,
-                        explain,
+                        verbose,
                         listener
                     );
                     try {
@@ -150,7 +155,7 @@ private ActionRunnable<List<HealthIndicatorResult>> calculateFilteredIndicatorsR
                     return ActionRunnable.wrap(listener, l -> {
                         List<HealthIndicatorResult> results = Stream.concat(
                             filteredPreflightResults,
-                            filteredIndicators.map(service -> service.calculate(explain, healthInfo))
+                            filteredIndicators.map(service -> service.calculate(explain, maxAffectedResourcesCount, healthInfo))
                         ).toList();
 
                         validateResultsAndNotifyListener(indicatorName, results, l);
@@ -160,7 +165,7 @@ private ActionRunnable<List<HealthIndicatorResult>> calculateFilteredIndicatorsR
 
         } else {
             // Mark remaining indicators as UNKNOWN
-            HealthIndicatorDetails unknownDetails = healthUnknownReason(preflightResults, explain);
+            HealthIndicatorDetails unknownDetails = healthUnknownReason(preflightResults, verbose);
             Stream<HealthIndicatorResult> filteredIndicatorResults = filteredIndicators.map(
                 service -> generateUnknownResult(service, UNKNOWN_RESULT_SUMMARY_PREFLIGHT_FAILED, unknownDetails)
             );
diff --git a/server/src/main/java/org/elasticsearch/health/RestGetHealthAction.java b/server/src/main/java/org/elasticsearch/health/RestGetHealthAction.java
index b1ab2e6dea74..c7068828de6a 100644
--- a/server/src/main/java/org/elasticsearch/health/RestGetHealthAction.java
+++ b/server/src/main/java/org/elasticsearch/health/RestGetHealthAction.java
@@ -23,6 +23,8 @@ public class RestGetHealthAction extends BaseRestHandler {
 
     private static final String VERBOSE_PARAM = "verbose";
 
+    private static final String SIZE_PARAM = "size";
+
     @Override
     public String getName() {
         // TODO: Existing - "cluster_health_action", "cat_health_action"
@@ -31,14 +33,15 @@ public String getName() {
 
     @Override
     public List<Route> routes() {
-        return List.of(new Route(GET, "/_internal/_health"), new Route(GET, "/_internal/_health/{indicator}"));
+        return List.of(new Route(GET, "/_health_report"), new Route(GET, "/_health_report/{indicator}"));
     }
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         String indicatorName = request.param("indicator");
         boolean verbose = request.paramAsBoolean(VERBOSE_PARAM, true);
-        GetHealthAction.Request getHealthRequest = new GetHealthAction.Request(indicatorName, verbose);
+        int size = request.paramAsInt(SIZE_PARAM, 1000);
+        GetHealthAction.Request getHealthRequest = new GetHealthAction.Request(indicatorName, verbose, size);
         return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute(
             GetHealthAction.INSTANCE,
             getHealthRequest,
diff --git a/server/src/main/java/org/elasticsearch/health/metadata/HealthMetadata.java b/server/src/main/java/org/elasticsearch/health/metadata/HealthMetadata.java
index 4c64e6f404a5..859a4fc2a8c1 100644
--- a/server/src/main/java/org/elasticsearch/health/metadata/HealthMetadata.java
+++ b/server/src/main/java/org/elasticsearch/health/metadata/HealthMetadata.java
@@ -8,20 +8,24 @@
 
 package org.elasticsearch.health.metadata;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.AbstractNamedDiffable;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.NamedDiff;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.RelativeByteSizeValue;
 import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.Objects;
 
 /**
@@ -49,8 +53,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_5_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_5_0;
     }
 
     @Override
@@ -63,22 +67,19 @@ public static NamedDiff<ClusterState.Custom> readDiffFrom(StreamInput in) throws
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(DISK_METADATA.getPreferredName());
-        diskMetadata.toXContent(builder, params);
-        builder.endObject();
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return Iterators.single((builder, params) -> {
+            builder.startObject(DISK_METADATA.getPreferredName());
+            diskMetadata.toXContent(builder, params);
+            builder.endObject();
+            return builder;
+        });
     }
 
     public static HealthMetadata getFromClusterState(ClusterState clusterState) {
         return clusterState.custom(HealthMetadata.TYPE);
     }
 
-    @Override
-    public boolean isFragment() {
-        return true;
-    }
-
     public Disk getDiskMetadata() {
         return diskMetadata;
     }
@@ -96,6 +97,11 @@ public int hashCode() {
         return Objects.hash(diskMetadata);
     }
 
+    @Override
+    public String toString() {
+        return "HealthMetadata{diskMetadata=" + Strings.toString(diskMetadata) + '}';
+    }
+
     /**
      * Contains the thresholds necessary to determine the health of the disk space of a node. The thresholds are determined by the elected
      * master.
@@ -110,7 +116,7 @@ public record Disk(
     ) implements ToXContentFragment, Writeable {
 
         public static final String TYPE = "disk";
-        public static Version VERSION_SUPPORTING_HEADROOM_FIELDS = Version.V_8_5_0;
+        public static final TransportVersion VERSION_SUPPORTING_HEADROOM_FIELDS = TransportVersion.V_8_5_0;
 
         private static final ParseField HIGH_WATERMARK_FIELD = new ParseField("high_watermark");
         private static final ParseField HIGH_MAX_HEADROOM_FIELD = new ParseField("high_max_headroom");
@@ -133,10 +139,10 @@ static Disk readFrom(StreamInput in) throws IOException {
                 FROZEN_FLOOD_STAGE_WATERMARK_FIELD.getPreferredName()
             );
             ByteSizeValue frozenFloodStageMaxHeadroom = ByteSizeValue.readFrom(in);
-            ByteSizeValue highMaxHeadroom = in.getVersion().onOrAfter(VERSION_SUPPORTING_HEADROOM_FIELDS)
+            ByteSizeValue highMaxHeadroom = in.getTransportVersion().onOrAfter(VERSION_SUPPORTING_HEADROOM_FIELDS)
                 ? ByteSizeValue.readFrom(in)
                 : ByteSizeValue.MINUS_ONE;
-            ByteSizeValue floodStageMaxHeadroom = in.getVersion().onOrAfter(VERSION_SUPPORTING_HEADROOM_FIELDS)
+            ByteSizeValue floodStageMaxHeadroom = in.getTransportVersion().onOrAfter(VERSION_SUPPORTING_HEADROOM_FIELDS)
                 ? ByteSizeValue.readFrom(in)
                 : ByteSizeValue.MINUS_ONE;
             return new Disk(
@@ -155,7 +161,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(describeFloodStageWatermark());
             out.writeString(describeFrozenFloodStageWatermark());
             frozenFloodStageMaxHeadroom.writeTo(out);
-            if (out.getVersion().onOrAfter(VERSION_SUPPORTING_HEADROOM_FIELDS)) {
+            if (out.getTransportVersion().onOrAfter(VERSION_SUPPORTING_HEADROOM_FIELDS)) {
                 highMaxHeadroom.writeTo(out);
                 floodStageMaxHeadroom.writeTo(out);
             }
@@ -185,11 +191,11 @@ private ByteSizeValue getFreeBytes(ByteSizeValue total, RelativeByteSizeValue wa
         }
 
         public ByteSizeValue getFreeBytesHighWatermark(ByteSizeValue total) {
-            return getFreeBytes(total, highWatermark, ByteSizeValue.MINUS_ONE);
+            return getFreeBytes(total, highWatermark, highMaxHeadroom);
         }
 
         public ByteSizeValue getFreeBytesFloodStageWatermark(ByteSizeValue total) {
-            return getFreeBytes(total, floodStageWatermark, ByteSizeValue.MINUS_ONE);
+            return getFreeBytes(total, floodStageWatermark, floodStageMaxHeadroom);
         }
 
         public ByteSizeValue getFreeBytesFrozenFloodStageWatermark(ByteSizeValue total) {
diff --git a/server/src/main/java/org/elasticsearch/health/node/DiskHealthIndicatorService.java b/server/src/main/java/org/elasticsearch/health/node/DiskHealthIndicatorService.java
index 0c96c6807a65..dafd5829ff4b 100644
--- a/server/src/main/java/org/elasticsearch/health/node/DiskHealthIndicatorService.java
+++ b/server/src/main/java/org/elasticsearch/health/node/DiskHealthIndicatorService.java
@@ -41,6 +41,7 @@
 import java.util.stream.Stream;
 
 import static org.elasticsearch.cluster.node.DiscoveryNode.DISCOVERY_NODE_COMPARATOR;
+import static org.elasticsearch.common.util.CollectionUtils.limitSize;
 import static org.elasticsearch.health.node.HealthIndicatorDisplayValues.are;
 import static org.elasticsearch.health.node.HealthIndicatorDisplayValues.getSortedUniqueValuesString;
 import static org.elasticsearch.health.node.HealthIndicatorDisplayValues.getTruncatedIndices;
@@ -68,7 +69,6 @@ public class DiskHealthIndicatorService implements HealthIndicatorService {
     private static final String IMPACT_INGEST_AT_RISK_ID = "ingest_capability_at_risk";
     private static final String IMPACT_CLUSTER_STABILITY_AT_RISK_ID = "cluster_stability_at_risk";
     private static final String IMPACT_CLUSTER_FUNCTIONALITY_UNAVAILABLE_ID = "cluster_functionality_unavailable";
-    private static final String IMPACT_DATA_NODE_WITHOUT_DISK_SPACE = "data_node_without_disk_space";
 
     private final ClusterService clusterService;
 
@@ -82,7 +82,7 @@ public String name() {
     }
 
     @Override
-    public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
+    public HealthIndicatorResult calculate(boolean verbose, int maxAffectedResourcesCount, HealthInfo healthInfo) {
         Map<String, DiskHealthInfo> diskHealthInfoMap = healthInfo.diskInfoByNode();
         if (diskHealthInfoMap == null || diskHealthInfoMap.isEmpty()) {
             /*
@@ -107,7 +107,7 @@ public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
             diskHealthAnalyzer.getSymptom(),
             diskHealthAnalyzer.getDetails(verbose),
             diskHealthAnalyzer.getImpacts(),
-            diskHealthAnalyzer.getDiagnoses()
+            diskHealthAnalyzer.getDiagnoses(maxAffectedResourcesCount)
         );
     }
 
@@ -344,7 +344,7 @@ List<HealthIndicatorImpact> getImpacts() {
             return impacts;
         }
 
-        private List<Diagnosis> getDiagnoses() {
+        private List<Diagnosis> getDiagnoses(int size) {
             if (healthStatus == HealthStatus.GREEN) {
                 return List.of();
             }
@@ -353,7 +353,7 @@ private List<Diagnosis> getDiagnoses() {
                 Set<String> affectedIndices = Sets.union(blockedIndices, indicesAtRisk);
                 List<Diagnosis.Resource> affectedResources = new ArrayList<>();
                 if (dataNodes.size() > 0) {
-                    Diagnosis.Resource nodeResources = new Diagnosis.Resource(dataNodes);
+                    Diagnosis.Resource nodeResources = new Diagnosis.Resource(limitSize(dataNodes, size));
                     affectedResources.add(nodeResources);
                 }
                 if (affectedIndices.size() > 0) {
@@ -361,60 +361,24 @@ private List<Diagnosis> getDiagnoses() {
                         Diagnosis.Resource.Type.INDEX,
                         affectedIndices.stream()
                             .sorted(indicesComparatorByPriorityAndName(clusterState.metadata()))
+                            .limit(Math.min(affectedIndices.size(), size))
                             .collect(Collectors.toList())
                     );
                     affectedResources.add(indexResources);
                 }
-                if (affectedIndices.size() > 0) {
-                    diagnosisList.add(
-                        new Diagnosis(
-                            new Diagnosis.Definition(
-                                NAME,
-                                "add_disk_capacity_data_nodes",
-                                String.format(
-                                    Locale.ROOT,
-                                    "%d %s %s on nodes that have run or are likely to run out of disk space, "
-                                        + "this can temporarily disable writing on %s %s.",
-                                    affectedIndices.size(),
-                                    indices(affectedIndices.size()),
-                                    regularVerb("reside", affectedIndices.size()),
-                                    these(affectedIndices.size()),
-                                    indices(affectedIndices.size())
-                                ),
-                                "Enable autoscaling (if applicable), add disk capacity or free up disk space to resolve "
-                                    + "this. If you have already taken action please wait for the rebalancing to complete.",
-                                "https://ela.st/fix-data-disk"
-                            ),
-                            affectedResources
-                        )
-                    );
-                } else {
-                    diagnosisList.add(
-                        new Diagnosis(
-                            new Diagnosis.Definition(
-                                NAME,
-                                "add_disk_capacity_data_nodes",
-                                "Disk is almost full.",
-                                "Enable autoscaling (if applicable), add disk capacity or free up disk space to resolve "
-                                    + "this. If you have already taken action please wait for the rebalancing to complete.",
-                                "https://ela.st/fix-data-disk"
-                            ),
-                            affectedResources
-                        )
-                    );
-                }
+                diagnosisList.add(createDataNodeDiagnosis(affectedIndices.size(), affectedResources));
             }
             if (masterNodes.containsKey(HealthStatus.RED)) {
-                diagnosisList.add(createNonDataNodeDiagnosis(HealthStatus.RED, masterNodes.get(HealthStatus.RED), true));
+                diagnosisList.add(createNonDataNodeDiagnosis(HealthStatus.RED, masterNodes.get(HealthStatus.RED), size, true));
             }
             if (masterNodes.containsKey(HealthStatus.YELLOW)) {
-                diagnosisList.add(createNonDataNodeDiagnosis(HealthStatus.YELLOW, masterNodes.get(HealthStatus.YELLOW), true));
+                diagnosisList.add(createNonDataNodeDiagnosis(HealthStatus.YELLOW, masterNodes.get(HealthStatus.YELLOW), size, true));
             }
             if (otherNodes.containsKey(HealthStatus.RED)) {
-                diagnosisList.add(createNonDataNodeDiagnosis(HealthStatus.RED, otherNodes.get(HealthStatus.RED), false));
+                diagnosisList.add(createNonDataNodeDiagnosis(HealthStatus.RED, otherNodes.get(HealthStatus.RED), size, false));
             }
             if (otherNodes.containsKey(HealthStatus.YELLOW)) {
-                diagnosisList.add(createNonDataNodeDiagnosis(HealthStatus.YELLOW, otherNodes.get(HealthStatus.YELLOW), false));
+                diagnosisList.add(createNonDataNodeDiagnosis(HealthStatus.YELLOW, otherNodes.get(HealthStatus.YELLOW), size, false));
             }
             return diagnosisList;
         }
@@ -487,7 +451,35 @@ static Set<String> getIndicesForNodes(List<DiscoveryNode> nodes, ClusterState cl
                 .collect(Collectors.toSet());
         }
 
-        private Diagnosis createNonDataNodeDiagnosis(HealthStatus healthStatus, Collection<DiscoveryNode> nodes, boolean isMaster) {
+        // Visible for testing
+        static Diagnosis createDataNodeDiagnosis(int numberOfAffectedIndices, List<Diagnosis.Resource> affectedResources) {
+            String message = numberOfAffectedIndices == 0
+                ? "Disk is almost full."
+                : String.format(
+                    Locale.ROOT,
+                    "%d %s %s on nodes that have run or are likely to run out of disk space, "
+                        + "this can temporarily disable writing on %s %s.",
+                    numberOfAffectedIndices,
+                    indices(numberOfAffectedIndices),
+                    regularVerb("reside", numberOfAffectedIndices),
+                    these(numberOfAffectedIndices),
+                    indices(numberOfAffectedIndices)
+                );
+            return new Diagnosis(
+                new Diagnosis.Definition(
+                    NAME,
+                    "add_disk_capacity_data_nodes",
+                    message,
+                    "Enable autoscaling (if applicable), add disk capacity or free up disk space to resolve "
+                        + "this. If you have already taken action please wait for the rebalancing to complete.",
+                    "https://ela.st/fix-data-disk"
+                ),
+                affectedResources
+            );
+        }
+
+        // Visible for testing
+        static Diagnosis createNonDataNodeDiagnosis(HealthStatus healthStatus, List<DiscoveryNode> nodes, int size, boolean isMaster) {
             return new Diagnosis(
                 new Diagnosis.Definition(
                     NAME,
@@ -496,7 +488,7 @@ private Diagnosis createNonDataNodeDiagnosis(HealthStatus healthStatus, Collecti
                     "Please add capacity to the current nodes, or replace them with ones with higher capacity.",
                     isMaster ? "https://ela.st/fix-master-disk" : "https://ela.st/fix-disk-space"
                 ),
-                List.of(new Diagnosis.Resource(nodes))
+                List.of(new Diagnosis.Resource(limitSize(nodes, size)))
             );
         }
 
diff --git a/server/src/main/java/org/elasticsearch/health/node/LocalHealthMonitor.java b/server/src/main/java/org/elasticsearch/health/node/LocalHealthMonitor.java
index 857dda9f0d46..9d034bb3c249 100644
--- a/server/src/main/java/org/elasticsearch/health/node/LocalHealthMonitor.java
+++ b/server/src/main/java/org/elasticsearch/health/node/LocalHealthMonitor.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.DiskUsage;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -159,7 +160,7 @@ private void startMonitoringIfNecessary() {
                 );
                 logger.debug("Local health monitoring started {}", monitoring);
             } else {
-                logger.debug("Local health monitoring already started {}, skipping", monitoring);
+                logger.trace("Local health monitoring already started {}, skipping", monitoring);
             }
         }
     }
@@ -433,9 +434,18 @@ DiskHealthInfo getHealth(HealthMetadata healthMetadata, ClusterState clusterStat
             }
 
             long highThreshold = diskMetadata.getFreeBytesHighWatermark(totalBytes).getBytes();
-            if (usage.getFreeBytes() < highThreshold && hasRelocatingShards(clusterState, node.getId()) == false) {
-                logger.debug("High disk watermark [{}] exceeded on {}", highThreshold, usage);
-                return new DiskHealthInfo(HealthStatus.YELLOW, DiskHealthInfo.Cause.NODE_OVER_HIGH_THRESHOLD);
+            if (usage.getFreeBytes() < highThreshold) {
+                if (node.canContainData()) {
+                    // for data nodes only report YELLOW if shards can't move away from the node
+                    if (DiskCheck.hasRelocatingShards(clusterState, node) == false) {
+                        logger.debug("High disk watermark [{}] exceeded on {}", highThreshold, usage);
+                        return new DiskHealthInfo(HealthStatus.YELLOW, DiskHealthInfo.Cause.NODE_OVER_HIGH_THRESHOLD);
+                    }
+                } else {
+                    // for non-data nodes report YELLOW when the disk high watermark is breached
+                    logger.debug("High disk watermark [{}] exceeded on {}", highThreshold, usage);
+                    return new DiskHealthInfo(HealthStatus.YELLOW, DiskHealthInfo.Cause.NODE_OVER_HIGH_THRESHOLD);
+                }
             }
             return new DiskHealthInfo(HealthStatus.GREEN);
         }
@@ -461,8 +471,13 @@ private DiskUsage getDiskUsage() {
             return DiskUsage.findLeastAvailablePath(nodeStats);
         }
 
-        private boolean hasRelocatingShards(ClusterState clusterState, String nodeId) {
-            return clusterState.getRoutingNodes().node(nodeId).shardsWithState(ShardRoutingState.RELOCATING).isEmpty() == false;
+        static boolean hasRelocatingShards(ClusterState clusterState, DiscoveryNode node) {
+            RoutingNode routingNode = clusterState.getRoutingNodes().node(node.getId());
+            if (routingNode == null) {
+                // routing node will be null for non-data nodes
+                return false;
+            }
+            return routingNode.numberOfShardsWithState(ShardRoutingState.RELOCATING) > 0;
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/health/node/selection/HealthNodeTaskParams.java b/server/src/main/java/org/elasticsearch/health/node/selection/HealthNodeTaskParams.java
index 888d1f2dae95..4a68115af634 100644
--- a/server/src/main/java/org/elasticsearch/health/node/selection/HealthNodeTaskParams.java
+++ b/server/src/main/java/org/elasticsearch/health/node/selection/HealthNodeTaskParams.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.health.node.selection;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.persistent.PersistentTaskParams;
@@ -46,8 +46,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_5_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_5_0;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/health/stats/HealthApiStats.java b/server/src/main/java/org/elasticsearch/health/stats/HealthApiStats.java
index e6b6fedec296..99a1a4dce4b5 100644
--- a/server/src/main/java/org/elasticsearch/health/stats/HealthApiStats.java
+++ b/server/src/main/java/org/elasticsearch/health/stats/HealthApiStats.java
@@ -15,10 +15,6 @@
 import org.elasticsearch.health.HealthIndicatorResult;
 import org.elasticsearch.health.HealthStatus;
 
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
 import java.util.function.BiFunction;
 import java.util.function.Function;
 
@@ -43,9 +39,6 @@ public class HealthApiStats {
         diagnosis
     );
 
-    private final Set<HealthStatus> statuses = ConcurrentHashMap.newKeySet();
-    private final ConcurrentMap<HealthStatus, Set<String>> indicators = new ConcurrentHashMap<>();
-    private final ConcurrentMap<HealthStatus, Set<String>> diagnoses = new ConcurrentHashMap<>();
     private final Counters stats = new Counters(TOTAL_INVOCATIONS);
 
     public HealthApiStats() {}
@@ -65,19 +58,15 @@ public void track(boolean verbose, GetHealthAction.Response response) {
             : response.getIndicatorResults().stream().map(HealthIndicatorResult::status).findFirst().orElse(null);
         if (status != null) {
             stats.inc(statusLabel.apply(status));
-            statuses.add(status);
         }
 
         if (status != HealthStatus.GREEN) {
             for (HealthIndicatorResult indicator : response.getIndicatorResults()) {
                 if (indicator.status() != HealthStatus.GREEN) {
                     stats.inc(indicatorLabel.apply(indicator.status(), indicator.name()));
-                    indicators.computeIfAbsent(indicator.status(), k -> ConcurrentHashMap.newKeySet()).add(indicator.name());
                     if (indicator.diagnosisList() != null) {
                         for (Diagnosis diagnosis : indicator.diagnosisList()) {
                             stats.inc(diagnosisLabel.apply(indicator.status(), diagnosis.definition().getUniqueId()));
-                            diagnoses.computeIfAbsent(indicator.status(), k -> ConcurrentHashMap.newKeySet())
-                                .add(diagnosis.definition().getUniqueId());
                         }
                     }
                 }
@@ -92,16 +81,4 @@ public boolean hasCounters() {
     public Counters getStats() {
         return stats;
     }
-
-    public Map<HealthStatus, Set<String>> getIndicators() {
-        return Map.copyOf(indicators);
-    }
-
-    public Map<HealthStatus, Set<String>> getDiagnoses() {
-        return Map.copyOf(diagnoses);
-    }
-
-    public Set<HealthStatus> getStatuses() {
-        return Set.copyOf(statuses);
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/health/stats/HealthApiStatsAction.java b/server/src/main/java/org/elasticsearch/health/stats/HealthApiStatsAction.java
index 237fd6bf6d5a..f2df53cd2327 100644
--- a/server/src/main/java/org/elasticsearch/health/stats/HealthApiStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/health/stats/HealthApiStatsAction.java
@@ -20,16 +20,11 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.metrics.Counters;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.health.HealthStatus;
 import org.elasticsearch.transport.TransportRequest;
 
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
 
 /**
  * This class collects the stats of the health API from every node
@@ -113,47 +108,13 @@ public Counters getStats() {
             return Counters.merge(counters);
         }
 
-        public Set<HealthStatus> getStatuses() {
-            Set<HealthStatus> statuses = new HashSet<>();
-            for (Node node : getNodes()) {
-                statuses.addAll(node.statuses);
-            }
-            return statuses;
-        }
-
-        public Map<HealthStatus, Set<String>> getIndicators() {
-            Map<HealthStatus, Set<String>> indicators = new HashMap<>();
-            for (Node node : getNodes()) {
-                for (HealthStatus status : node.indicators.keySet()) {
-                    indicators.computeIfAbsent(status, s -> new HashSet<>()).addAll(node.indicators.get(status));
-                }
-            }
-            return indicators;
-        }
-
-        public Map<HealthStatus, Set<String>> getDiagnoses() {
-            Map<HealthStatus, Set<String>> diagnoses = new HashMap<>();
-            for (Node node : getNodes()) {
-                for (HealthStatus status : node.diagnoses.keySet()) {
-                    diagnoses.computeIfAbsent(status, s -> new HashSet<>()).addAll(node.diagnoses.get(status));
-                }
-            }
-            return diagnoses;
-        }
-
         public static class Node extends BaseNodeResponse {
             @Nullable
             private Counters stats;
-            private Set<HealthStatus> statuses = Set.of();
-            private Map<HealthStatus, Set<String>> indicators = Map.of();
-            private Map<HealthStatus, Set<String>> diagnoses = Map.of();
 
             public Node(StreamInput in) throws IOException {
                 super(in);
                 stats = in.readOptionalWriteable(Counters::new);
-                statuses = in.readSet(HealthStatus::read);
-                indicators = in.readMap(HealthStatus::read, input -> input.readSet(StreamInput::readString));
-                diagnoses = in.readMap(HealthStatus::read, input -> input.readSet(StreamInput::readString));
             }
 
             public Node(DiscoveryNode node) {
@@ -168,25 +129,10 @@ public void setStats(Counters stats) {
                 this.stats = stats;
             }
 
-            public void setStatuses(Set<HealthStatus> statuses) {
-                this.statuses = statuses;
-            }
-
-            public void setIndicators(Map<HealthStatus, Set<String>> indicators) {
-                this.indicators = indicators;
-            }
-
-            public void setDiagnoses(Map<HealthStatus, Set<String>> diagnoses) {
-                this.diagnoses = diagnoses;
-            }
-
             @Override
             public void writeTo(StreamOutput out) throws IOException {
                 super.writeTo(out);
                 out.writeOptionalWriteable(stats);
-                out.writeCollection(statuses);
-                out.writeMap(indicators, StreamOutput::writeEnum, (o, v) -> o.writeCollection(v, StreamOutput::writeString));
-                out.writeMap(diagnoses, StreamOutput::writeEnum, (o, v) -> o.writeCollection(v, StreamOutput::writeString));
             }
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/health/stats/HealthApiStatsTransportAction.java b/server/src/main/java/org/elasticsearch/health/stats/HealthApiStatsTransportAction.java
index 946fcb1addbf..b5377e2a98cd 100644
--- a/server/src/main/java/org/elasticsearch/health/stats/HealthApiStatsTransportAction.java
+++ b/server/src/main/java/org/elasticsearch/health/stats/HealthApiStatsTransportAction.java
@@ -79,9 +79,6 @@ protected HealthApiStatsAction.Response.Node nodeOperation(HealthApiStatsAction.
         if (healthApiStats.hasCounters()) {
             statsResponse.setStats(healthApiStats.getStats());
         }
-        statsResponse.setStatuses(healthApiStats.getStatuses());
-        statsResponse.setIndicators(healthApiStats.getIndicators());
-        statsResponse.setDiagnoses(healthApiStats.getDiagnoses());
         return statsResponse;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/http/HttpServerTransport.java b/server/src/main/java/org/elasticsearch/http/HttpServerTransport.java
index 42eaa0a38445..b2528cf9f87a 100644
--- a/server/src/main/java/org/elasticsearch/http/HttpServerTransport.java
+++ b/server/src/main/java/org/elasticsearch/http/HttpServerTransport.java
@@ -17,6 +17,7 @@
 
 public interface HttpServerTransport extends LifecycleComponent, ReportingService<HttpInfo> {
 
+    String HTTP_PROFILE_NAME = ".http";
     String HTTP_SERVER_WORKER_THREAD_NAME_PREFIX = "http_server_worker";
 
     BoundTransportAddress boundAddress();
diff --git a/server/src/main/java/org/elasticsearch/http/HttpTracer.java b/server/src/main/java/org/elasticsearch/http/HttpTracer.java
index 2978f00d7a16..9e879c44aea6 100644
--- a/server/src/main/java/org/elasticsearch/http/HttpTracer.java
+++ b/server/src/main/java/org/elasticsearch/http/HttpTracer.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
@@ -33,6 +34,11 @@ class HttpTracer {
     private volatile String[] tracerLogInclude;
     private volatile String[] tracerLogExclude;
 
+    // for testing
+    HttpTracer() {
+        tracerLogInclude = tracerLogExclude = new String[0];
+    }
+
     HttpTracer(Settings settings, ClusterSettings clusterSettings) {
 
         setTracerLogInclude(HttpTransportSettings.SETTING_HTTP_TRACE_LOG_INCLUDE.get(settings));
@@ -55,14 +61,17 @@ class HttpTracer {
     @Nullable
     HttpTracer maybeLogRequest(RestRequest restRequest, @Nullable Exception e) {
         if (logger.isTraceEnabled() && TransportService.shouldTraceAction(restRequest.uri(), tracerLogInclude, tracerLogExclude)) {
+            // trace.id in the response log is included from threadcontext, which isn't set at request log time
+            // so include it here as part of the message
             logger.trace(
                 () -> format(
-                    "[%s][%s][%s][%s] received request from [%s]",
+                    "[%s][%s][%s][%s] received request from [%s]%s",
                     restRequest.getRequestId(),
                     restRequest.header(Task.X_OPAQUE_ID_HTTP_HEADER),
                     restRequest.method(),
                     restRequest.uri(),
-                    restRequest.getHttpChannel()
+                    restRequest.getHttpChannel(),
+                    RestUtils.extractTraceId(restRequest.header(Task.TRACE_PARENT_HTTP_HEADER)).map(t -> " trace.id: " + t).orElse("")
                 ),
                 e
             );
@@ -89,6 +98,7 @@ void logResponse(
         long requestId,
         boolean success
     ) {
+        // trace id is included in the ThreadContext for the response
         logger.trace(
             () -> format(
                 "[%s][%s][%s][%s][%s] sent response to [%s] success [%s]",
diff --git a/server/src/main/java/org/elasticsearch/index/CompositeIndexEventListener.java b/server/src/main/java/org/elasticsearch/index/CompositeIndexEventListener.java
index e6ec4b98f189..0c8248313469 100644
--- a/server/src/main/java/org/elasticsearch/index/CompositeIndexEventListener.java
+++ b/server/src/main/java/org/elasticsearch/index/CompositeIndexEventListener.java
@@ -9,17 +9,22 @@
 package org.elasticsearch.index;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ListenableFuture;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Collection;
+import java.util.Iterator;
 import java.util.List;
 
 import static org.elasticsearch.core.Strings.format;
@@ -238,16 +243,53 @@ public void onStoreClosed(ShardId shardId) {
         }
     }
 
-    @Override
-    public void beforeIndexShardRecovery(final IndexShard indexShard, final IndexSettings indexSettings) {
-        for (IndexEventListener listener : listeners) {
+    private void iterateBeforeIndexShardRecovery(
+        final IndexShard indexShard,
+        final IndexSettings indexSettings,
+        final Iterator<IndexEventListener> iterator,
+        final ActionListener<Void> outerListener
+    ) {
+        while (iterator.hasNext()) {
+            final var nextListener = iterator.next();
+            final var future = new ListenableFuture<Void>();
             try {
-                listener.beforeIndexShardRecovery(indexShard, indexSettings);
+                nextListener.beforeIndexShardRecovery(indexShard, indexSettings, future);
+                if (future.isDone()) {
+                    // common case, not actually async, so just check for an exception and continue on the same thread
+                    future.get();
+                    continue;
+                }
             } catch (Exception e) {
-                logger.warn(() -> format("failed to invoke the listener before the shard recovery starts for %s", indexShard.shardId()), e);
-                throw e;
+                outerListener.onFailure(e);
+                return;
             }
+
+            // future was not completed straight away, but might be done by now, so continue on a fresh thread to avoid stack overflow
+            future.addListener(
+                outerListener.delegateFailure(
+                    (delegate, v) -> indexShard.getThreadPool()
+                        .executor(ThreadPool.Names.GENERIC)
+                        .execute(
+                            ActionRunnable.wrap(delegate, l -> iterateBeforeIndexShardRecovery(indexShard, indexSettings, iterator, l))
+                        )
+                )
+            );
+            return;
         }
+
+        outerListener.onResponse(null);
+    }
+
+    @Override
+    public void beforeIndexShardRecovery(
+        final IndexShard indexShard,
+        final IndexSettings indexSettings,
+        final ActionListener<Void> outerListener
+    ) {
+        iterateBeforeIndexShardRecovery(indexShard, indexSettings, listeners.iterator(), outerListener.delegateResponse((l, e) -> {
+            logger.warn(() -> format("failed to invoke the listener before the shard recovery starts for %s", indexShard.shardId()), e);
+            l.onFailure(e);
+        }));
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java
index 71a824535392..b46b8334408f 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexMode.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.index.mapper.NestedLookup;
 import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
 import org.elasticsearch.index.mapper.RoutingFieldMapper;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper;
 import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper;
 
@@ -110,6 +111,9 @@ public DocumentDimensions buildDocumentDimensions(IndexSettings settings) {
         public boolean shouldValidateTimestamp() {
             return false;
         }
+
+        @Override
+        public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) {}
     },
     TIME_SERIES("time_series") {
         @Override
@@ -196,6 +200,13 @@ public DocumentDimensions buildDocumentDimensions(IndexSettings settings) {
         public boolean shouldValidateTimestamp() {
             return true;
         }
+
+        @Override
+        public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) {
+            if (sourceFieldMapper.isSynthetic() == false) {
+                throw new IllegalArgumentException("time series indices only support synthetic source");
+            }
+        }
     };
 
     protected static String tsdbMode() {
@@ -310,6 +321,11 @@ public String getName() {
      */
     public abstract boolean shouldValidateTimestamp();
 
+    /**
+     * Validates the source field mapper
+     */
+    public abstract void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper);
+
     /**
      * Parse a string into an {@link IndexMode}.
      */
diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java
index 8fcd4a29c6b0..9f2e1c76e24d 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexModule.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.TriFunction;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -33,6 +34,7 @@
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
@@ -44,9 +46,11 @@
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.seqno.ReplicationTracker;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexingOperationListener;
 import org.elasticsearch.index.shard.SearchOperationListener;
+import org.elasticsearch.index.shard.ShardPath;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.index.store.FsDirectoryFactory;
 import org.elasticsearch.indices.IndicesQueryCache;
@@ -147,7 +151,17 @@ public final class IndexModule {
      * created by {@link org.elasticsearch.plugins.IndexStorePlugin.DirectoryFactory}.
      */
     @FunctionalInterface
-    public interface DirectoryWrapper extends CheckedFunction<Directory, Directory, IOException> {}
+    public interface DirectoryWrapper {
+        /**
+         * Wrap a given {@link Directory}
+         *
+         * @param directory the {@link Directory} to wrap
+         * @param shardRouting the {@link ShardRouting} associated with the {@link Directory} or {@code null} is unknown
+         * @return a {@link Directory}
+         * @throws IOException
+         */
+        Directory wrap(Directory directory, @Nullable ShardRouting shardRouting) throws IOException;
+    }
 
     private final IndexSettings indexSettings;
     private final AnalysisRegistry analysisRegistry;
@@ -165,6 +179,9 @@ public interface DirectoryWrapper extends CheckedFunction<Directory, Directory,
     private final AtomicBoolean frozen = new AtomicBoolean(false);
     private final BooleanSupplier allowExpensiveQueries;
     private final Map<String, IndexStorePlugin.RecoveryStateFactory> recoveryStateFactories;
+    private final SetOnce<Engine.IndexCommitListener> indexCommitListener = new SetOnce<>();
+
+    private final SetOnce<ReplicationTracker.Factory> replicationTrackerFactory = new SetOnce<>();
 
     /**
      * Construct the index module for the index with the specified index settings. The index module contains extension points for plugins
@@ -370,6 +387,16 @@ public void setDirectoryWrapper(DirectoryWrapper wrapper) {
         this.indexDirectoryWrapper.set(Objects.requireNonNull(wrapper));
     }
 
+    public void setIndexCommitListener(Engine.IndexCommitListener listener) {
+        ensureNotFrozen();
+        this.indexCommitListener.set(Objects.requireNonNull(listener));
+    }
+
+    public void setReplicationTrackerFactory(ReplicationTracker.Factory factory) {
+        ensureNotFrozen();
+        this.replicationTrackerFactory.set(factory);
+    }
+
     IndexEventListener freeze() { // pkg private for testing
         if (this.frozen.compareAndSet(false, true)) {
             return new CompositeIndexEventListener(indexSettings, indexEventListeners);
@@ -517,7 +544,9 @@ public IndexService newIndexService(
                 valuesSourceRegistry,
                 recoveryStateFactory,
                 indexFoldersDeletionListener,
-                snapshotCommitSupplier
+                snapshotCommitSupplier,
+                indexCommitListener.get(),
+                Objects.requireNonNullElse(replicationTrackerFactory.get(), ReplicationTracker.DEFAULT_FACTORY)
             );
             success = true;
             return indexService;
@@ -559,7 +588,18 @@ private IndexStorePlugin.DirectoryFactory getDirectoryFactory(
         final DirectoryWrapper directoryWrapper = this.indexDirectoryWrapper.get();
         assert frozen.get() : "IndexModule configuration not frozen";
         if (directoryWrapper != null) {
-            return (idxSettings, shardPath) -> directoryWrapper.apply(factory.newDirectory(idxSettings, shardPath));
+            return new IndexStorePlugin.DirectoryFactory() {
+                @Override
+                public Directory newDirectory(IndexSettings indexSettings, ShardPath shardPath) throws IOException {
+                    return newDirectory(indexSettings, shardPath, null);
+                }
+
+                @Override
+                public Directory newDirectory(IndexSettings indexSettings, ShardPath shardPath, ShardRouting shardRouting)
+                    throws IOException {
+                    return directoryWrapper.wrap(factory.newDirectory(indexSettings, shardPath, shardRouting), shardRouting);
+                }
+            };
         }
         return factory;
     }
diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java
index a107efaa9764..f50979aa82d2 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexService.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexService.java
@@ -54,6 +54,7 @@
 import org.elasticsearch.index.mapper.NodeMappingStats;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.index.query.SearchIndexNameMatcher;
+import org.elasticsearch.index.seqno.ReplicationTracker;
 import org.elasticsearch.index.seqno.RetentionLeaseSyncer;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
@@ -112,6 +113,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
     private final IndexStorePlugin.RecoveryStateFactory recoveryStateFactory;
     private final IndexStorePlugin.SnapshotCommitSupplier snapshotCommitSupplier;
     private final CheckedFunction<DirectoryReader, DirectoryReader, IOException> readerWrapper;
+    private final Engine.IndexCommitListener indexCommitListener;
     private final IndexCache indexCache;
     private final MapperService mapperService;
     private final XContentParserConfiguration parserConfiguration;
@@ -145,6 +147,8 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
     private final Supplier<Sort> indexSortSupplier;
     private final ValuesSourceRegistry valuesSourceRegistry;
 
+    private final ReplicationTracker.Factory replicationTrackerFactory;
+
     public IndexService(
         IndexSettings indexSettings,
         IndexCreationContext indexCreationContext,
@@ -175,7 +179,9 @@ public IndexService(
         ValuesSourceRegistry valuesSourceRegistry,
         IndexStorePlugin.RecoveryStateFactory recoveryStateFactory,
         IndexStorePlugin.IndexFoldersDeletionListener indexFoldersDeletionListener,
-        IndexStorePlugin.SnapshotCommitSupplier snapshotCommitSupplier
+        IndexStorePlugin.SnapshotCommitSupplier snapshotCommitSupplier,
+        Engine.IndexCommitListener indexCommitListener,
+        ReplicationTracker.Factory replicationTrackerFactory
     ) {
         super(indexSettings);
         this.allowExpensiveQueries = allowExpensiveQueries;
@@ -242,6 +248,7 @@ public IndexService(
         this.readerWrapper = wrapperFactory.apply(this);
         this.searchOperationListeners = Collections.unmodifiableList(searchOperationListeners);
         this.indexingOperationListeners = Collections.unmodifiableList(indexingOperationListeners);
+        this.indexCommitListener = indexCommitListener;
         try (var ignored = threadPool.getThreadContext().clearTraceContext()) {
             // kick off async ops for the first shard in this index
             this.refreshTask = new AsyncRefreshTask(this);
@@ -249,6 +256,7 @@ public IndexService(
             this.globalCheckpointTask = new AsyncGlobalCheckpointTask(this);
             this.retentionLeaseSyncTask = new AsyncRetentionLeaseSyncTask(this);
         }
+        this.replicationTrackerFactory = replicationTrackerFactory;
         updateFsyncTaskIfNecessary();
     }
 
@@ -300,8 +308,10 @@ public IndexShard getShard(int shardId) {
     }
 
     public NodeMappingStats getNodeMappingStats() {
+        if (mapperService == null) {
+            return null;
+        }
         long totalCount = mapperService().mappingLookup().getTotalFieldsCount();
-        Index index = index();
         long totalEstimatedOverhead = totalCount * 1024L; // 1KiB estimated per mapping
         NodeMappingStats indexNodeMappingStats = new NodeMappingStats(totalCount, totalEstimatedOverhead);
         return indexNodeMappingStats;
@@ -484,7 +494,7 @@ public synchronized IndexShard createShard(
                     warmer.warm(reader, shard, IndexService.this.indexSettings);
                 }
             };
-            Directory directory = directoryFactory.newDirectory(this.indexSettings, path);
+            final Directory directory = directoryFactory.newDirectory(this.indexSettings, path, routing);
             store = new Store(
                 shardId,
                 this.indexSettings,
@@ -514,7 +524,9 @@ public synchronized IndexShard createShard(
                 retentionLeaseSyncer,
                 circuitBreakerService,
                 snapshotCommitSupplier,
-                System::nanoTime
+                System::nanoTime,
+                indexCommitListener,
+                replicationTrackerFactory
             );
             eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created");
             eventListener.afterIndexShardCreated(indexShard);
diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java
index dfbfb47937b2..8dd27a2344c1 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java
@@ -251,9 +251,15 @@ public final class IndexSettings {
         Property.IndexScope
     );
     public static final TimeValue DEFAULT_REFRESH_INTERVAL = new TimeValue(1, TimeUnit.SECONDS);
+    public static final Setting<TimeValue> NODE_DEFAULT_REFRESH_INTERVAL_SETTING = Setting.timeSetting(
+        "node._internal.default_refresh_interval",
+        DEFAULT_REFRESH_INTERVAL,
+        new TimeValue(-1, TimeUnit.MILLISECONDS),
+        Property.NodeScope
+    );
     public static final Setting<TimeValue> INDEX_REFRESH_INTERVAL_SETTING = Setting.timeSetting(
         "index.refresh_interval",
-        DEFAULT_REFRESH_INTERVAL,
+        NODE_DEFAULT_REFRESH_INTERVAL_SETTING,
         new TimeValue(-1, TimeUnit.MILLISECONDS),
         Property.Dynamic,
         Property.IndexScope
@@ -774,7 +780,10 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         mappingDimensionFieldsLimit = scopedSettings.get(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING);
         indexRouting = IndexRouting.fromIndexMetadata(indexMetadata);
 
-        scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, mergePolicyConfig::setNoCFSRatio);
+        scopedSettings.addSettingsUpdateConsumer(
+            MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING,
+            mergePolicyConfig::setCompoundFormatThreshold
+        );
         scopedSettings.addSettingsUpdateConsumer(
             MergePolicyConfig.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING,
             mergePolicyConfig::setDeletesPctAllowed
diff --git a/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java b/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
index 12d84a57b359..80ae79c0f2a5 100644
--- a/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
+++ b/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
@@ -71,8 +71,8 @@
  *
  *     Controls the maximum percentage of deleted documents that is tolerated in
  *     the index. Lower values make the index more space efficient at the
- *     expense of increased CPU and I/O activity. Values must be between <code>20</code> and
- *     <code>50</code>. Default value is <code>33</code>.
+ *     expense of increased CPU and I/O activity. Values must be between <code>5</code> and
+ *     <code>50</code>. Default value is <code>20</code>.
  * </ul>
  *
  * <p>
@@ -110,11 +110,12 @@ public final class MergePolicyConfig {
     public static final int DEFAULT_MAX_MERGE_AT_ONCE = 10;
     public static final ByteSizeValue DEFAULT_MAX_MERGED_SEGMENT = new ByteSizeValue(5, ByteSizeUnit.GB);
     public static final double DEFAULT_SEGMENTS_PER_TIER = 10.0d;
-    public static final double DEFAULT_DELETES_PCT_ALLOWED = 33.0d;
-    public static final Setting<Double> INDEX_COMPOUND_FORMAT_SETTING = new Setting<>(
-        "index.compound_format",
-        Double.toString(TieredMergePolicy.DEFAULT_NO_CFS_RATIO),
-        MergePolicyConfig::parseNoCFSRatio,
+    public static final double DEFAULT_DELETES_PCT_ALLOWED = 20.0d;
+    private static final String INDEX_COMPOUND_FORMAT_SETTING_KEY = "index.compound_format";
+    public static final Setting<CompoundFileThreshold> INDEX_COMPOUND_FORMAT_SETTING = new Setting<>(
+        INDEX_COMPOUND_FORMAT_SETTING_KEY,
+        "1gb",
+        MergePolicyConfig::parseCompoundFormat,
         Property.Dynamic,
         Property.IndexScope
     );
@@ -163,7 +164,7 @@ public final class MergePolicyConfig {
     public static final Setting<Double> INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING = Setting.doubleSetting(
         "index.merge.policy.deletes_pct_allowed",
         DEFAULT_DELETES_PCT_ALLOWED,
-        20.0d,
+        5.0d,
         50.0d,
         Property.Dynamic,
         Property.IndexScope
@@ -189,7 +190,7 @@ public final class MergePolicyConfig {
             );
         }
         maxMergeAtOnce = adjustMaxMergeAtOnceIfNeeded(maxMergeAtOnce, segmentsPerTier);
-        mergePolicy.setNoCFSRatio(indexSettings.getValue(INDEX_COMPOUND_FORMAT_SETTING));
+        indexSettings.getValue(INDEX_COMPOUND_FORMAT_SETTING).configure(mergePolicy);
         mergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed);
         mergePolicy.setFloorSegmentMB(floorSegment.getMbFrac());
         mergePolicy.setMaxMergeAtOnce(maxMergeAtOnce);
@@ -229,8 +230,8 @@ void setExpungeDeletesAllowed(Double value) {
         mergePolicy.setForceMergeDeletesPctAllowed(value);
     }
 
-    void setNoCFSRatio(Double noCFSRatio) {
-        mergePolicy.setNoCFSRatio(noCFSRatio);
+    void setCompoundFormatThreshold(CompoundFileThreshold compoundFileThreshold) {
+        compoundFileThreshold.configure(mergePolicy);
     }
 
     void setDeletesPctAllowed(Double deletesPctAllowed) {
@@ -261,25 +262,81 @@ MergePolicy getMergePolicy() {
         return mergesEnabled ? mergePolicy : NoMergePolicy.INSTANCE;
     }
 
-    private static double parseNoCFSRatio(String noCFSRatio) {
+    private static CompoundFileThreshold parseCompoundFormat(String noCFSRatio) {
         noCFSRatio = noCFSRatio.trim();
         if (noCFSRatio.equalsIgnoreCase("true")) {
-            return 1.0d;
+            return new CompoundFileThreshold(1.0d);
         } else if (noCFSRatio.equalsIgnoreCase("false")) {
-            return 0.0;
+            return new CompoundFileThreshold(0.0d);
         } else {
             try {
-                double value = Double.parseDouble(noCFSRatio);
-                if (value < 0.0 || value > 1.0) {
-                    throw new IllegalArgumentException("NoCFSRatio must be in the interval [0..1] but was: [" + value + "]");
+                try {
+                    return new CompoundFileThreshold(Double.parseDouble(noCFSRatio));
+                } catch (NumberFormatException ex) {
+                    throw new IllegalArgumentException(
+                        "index.compound_format must be a boolean, a non-negative byte size or a ratio in the interval [0..1] but was: ["
+                            + noCFSRatio
+                            + "]",
+                        ex
+                    );
                 }
-                return value;
-            } catch (NumberFormatException ex) {
+            } catch (IllegalArgumentException e) {
+                try {
+                    return new CompoundFileThreshold(ByteSizeValue.parseBytesSizeValue(noCFSRatio, INDEX_COMPOUND_FORMAT_SETTING_KEY));
+                } catch (RuntimeException e2) {
+                    e.addSuppressed(e2);
+                }
+                throw e;
+            }
+        }
+    }
+
+    public static class CompoundFileThreshold {
+        private Double noCFSRatio;
+        private ByteSizeValue noCFSSize;
+
+        private CompoundFileThreshold(double noCFSRatio) {
+            if (noCFSRatio < 0.0 || noCFSRatio > 1.0) {
                 throw new IllegalArgumentException(
-                    "Expected a boolean or a value in the interval [0..1] but was: " + "[" + noCFSRatio + "]",
-                    ex
+                    "index.compound_format must be a boolean, a non-negative byte size or a ratio in the interval [0..1] but was: ["
+                        + noCFSRatio
+                        + "]"
                 );
             }
+            this.noCFSRatio = noCFSRatio;
+            this.noCFSSize = null;
+        }
+
+        private CompoundFileThreshold(ByteSizeValue noCFSSize) {
+            if (noCFSSize.getBytes() < 0) {
+                throw new IllegalArgumentException(
+                    "index.compound_format must be a boolean, a non-negative byte size or a ratio in the interval [0..1] but was: ["
+                        + noCFSSize
+                        + "]"
+                );
+            }
+            this.noCFSRatio = null;
+            this.noCFSSize = noCFSSize;
+        }
+
+        void configure(MergePolicy mergePolicy) {
+            if (noCFSRatio != null) {
+                assert noCFSSize == null;
+                mergePolicy.setNoCFSRatio(noCFSRatio);
+                mergePolicy.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);
+            } else {
+                mergePolicy.setNoCFSRatio(1.0);
+                mergePolicy.setMaxCFSSegmentSizeMB(noCFSSize.getMbFrac());
+            }
+        }
+
+        @Override
+        public String toString() {
+            if (noCFSRatio != null) {
+                return "max CFS ratio: " + noCFSRatio;
+            } else {
+                return "max CFS size: " + noCFSSize;
+            }
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java b/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
index 266d84f394ac..f1f03eff88d0 100644
--- a/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
+++ b/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
@@ -43,6 +43,7 @@
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardUtils;
+import org.elasticsearch.lucene.util.BitSets;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.Closeable;
@@ -104,7 +105,7 @@ public static BitSet bitsetFromQuery(Query query, LeafReaderContext context) thr
         if (s == null) {
             return null;
         } else {
-            return BitSet.of(s.iterator(), context.reader().maxDoc());
+            return BitSets.of(s.iterator(), context.reader().maxDoc());
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java
index 0daf88610fbe..990d44f5baef 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.index.codec;
 
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.mapper.MapperService;
@@ -35,11 +35,11 @@ public class CodecService {
     public CodecService(@Nullable MapperService mapperService, BigArrays bigArrays) {
         final var codecs = new HashMap<String, Codec>();
         if (mapperService == null) {
-            codecs.put(DEFAULT_CODEC, new Lucene94Codec());
-            codecs.put(BEST_COMPRESSION_CODEC, new Lucene94Codec(Lucene94Codec.Mode.BEST_COMPRESSION));
+            codecs.put(DEFAULT_CODEC, new Lucene95Codec());
+            codecs.put(BEST_COMPRESSION_CODEC, new Lucene95Codec(Lucene95Codec.Mode.BEST_COMPRESSION));
         } else {
-            codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Lucene94Codec.Mode.BEST_SPEED, mapperService, bigArrays));
-            codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMapperCodec(Lucene94Codec.Mode.BEST_COMPRESSION, mapperService, bigArrays));
+            codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Lucene95Codec.Mode.BEST_SPEED, mapperService, bigArrays));
+            codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMapperCodec(Lucene95Codec.Mode.BEST_COMPRESSION, mapperService, bigArrays));
         }
         codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
         for (String codec : Codec.availableCodecs()) {
diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java
index 268ba302033f..117276858571 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java
@@ -13,14 +13,20 @@
 import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.codec.bloomfilter.ES85BloomFilterPostingsFormat;
+import org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat;
+import org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat;
 import org.elasticsearch.index.mapper.IdFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.MappingLookup;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.mapper.TimeSeriesParams;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 
 /**
@@ -31,11 +37,12 @@
  * per index in real time via the mapping API. If no specific postings format or vector format is
  * configured for a specific field the default postings or vector format is used.
  */
-public class PerFieldMapperCodec extends Lucene94Codec {
-    private final MapperService mapperService;
+public class PerFieldMapperCodec extends Lucene95Codec {
 
+    private final MapperService mapperService;
     private final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat();
-    private final ES85BloomFilterPostingsFormat bloomFilterPostingsFormat;
+    private final ES87BloomFilterPostingsFormat bloomFilterPostingsFormat;
+    private final ES87TSDBDocValuesFormat tsdbDocValuesFormat;
 
     static {
         assert Codec.forName(Lucene.LATEST_CODEC).getClass().isAssignableFrom(PerFieldMapperCodec.class)
@@ -45,7 +52,8 @@ public class PerFieldMapperCodec extends Lucene94Codec {
     public PerFieldMapperCodec(Mode compressionMode, MapperService mapperService, BigArrays bigArrays) {
         super(compressionMode);
         this.mapperService = mapperService;
-        this.bloomFilterPostingsFormat = new ES85BloomFilterPostingsFormat(bigArrays, this::internalGetPostingsFormatForField);
+        this.bloomFilterPostingsFormat = new ES87BloomFilterPostingsFormat(bigArrays, this::internalGetPostingsFormatForField);
+        this.tsdbDocValuesFormat = new ES87TSDBDocValuesFormat();
     }
 
     @Override
@@ -64,10 +72,19 @@ private PostingsFormat internalGetPostingsFormatForField(String field) {
         return super.getPostingsFormatForField(field);
     }
 
-    private boolean useBloomFilter(String field) {
-        return IdFieldMapper.NAME.equals(field)
-            && mapperService.mappingLookup().isDataStreamTimestampFieldEnabled() == false
-            && IndexSettings.BLOOM_FILTER_ID_FIELD_ENABLED_SETTING.get(mapperService.getIndexSettings().getSettings());
+    boolean useBloomFilter(String field) {
+        IndexSettings indexSettings = mapperService.getIndexSettings();
+        if (mapperService.mappingLookup().isDataStreamTimestampFieldEnabled()) {
+            // In case for time series indices, they _id isn't randomly generated,
+            // but based on dimension fields and timestamp field, so during indexing
+            // version/seq_no/term needs to be looked up and having a bloom filter
+            // can speed this up significantly.
+            return indexSettings.getMode() == IndexMode.TIME_SERIES
+                && IdFieldMapper.NAME.equals(field)
+                && IndexSettings.BLOOM_FILTER_ID_FIELD_ENABLED_SETTING.get(indexSettings.getSettings());
+        } else {
+            return IdFieldMapper.NAME.equals(field) && IndexSettings.BLOOM_FILTER_ID_FIELD_ENABLED_SETTING.get(indexSettings.getSettings());
+        }
     }
 
     @Override
@@ -84,6 +101,39 @@ public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
 
     @Override
     public DocValuesFormat getDocValuesFormatForField(String field) {
+        if (useTSDBDocValuesFormat(field)) {
+            return tsdbDocValuesFormat;
+        }
         return docValuesFormat;
     }
+
+    private boolean useTSDBDocValuesFormat(final String field) {
+        return IndexSettings.isTimeSeriesModeEnabled()
+            && isTimeSeriesModeIndex()
+            && isNotSpecialField(field)
+            && (isCounterMetricType(field) || isTimestampField(field));
+    }
+
+    private boolean isTimeSeriesModeIndex() {
+        return IndexMode.TIME_SERIES.equals(mapperService.getIndexSettings().getMode());
+    }
+
+    private boolean isCounterMetricType(String field) {
+        if (mapperService != null) {
+            final MappingLookup mappingLookup = mapperService.mappingLookup();
+            if (mappingLookup.getMapper(field) instanceof NumberFieldMapper) {
+                final MappedFieldType fieldType = mappingLookup.getFieldType(field);
+                return TimeSeriesParams.MetricType.COUNTER.equals(fieldType.getMetricType());
+            }
+        }
+        return false;
+    }
+
+    private boolean isTimestampField(String field) {
+        return "@timestamp".equals(field);
+    }
+
+    private boolean isNotSpecialField(String field) {
+        return field.startsWith("_") == false;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/codec/bloomfilter/ES85BloomFilterPostingsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/bloomfilter/ES85BloomFilterPostingsFormat.java
index 4b4a77f914f5..d26fb52a82bc 100644
--- a/server/src/main/java/org/elasticsearch/index/codec/bloomfilter/ES85BloomFilterPostingsFormat.java
+++ b/server/src/main/java/org/elasticsearch/index/codec/bloomfilter/ES85BloomFilterPostingsFormat.java
@@ -22,11 +22,9 @@
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.codecs.FieldsConsumer;
 import org.apache.lucene.codecs.FieldsProducer;
-import org.apache.lucene.codecs.NormsProducer;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.index.BaseTermsEnum;
 import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.FilterLeafReader;
 import org.apache.lucene.index.ImpactsEnum;
 import org.apache.lucene.index.IndexFileNames;
@@ -45,10 +43,6 @@
 import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.BitUtil;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.lucene.store.IndexOutputOutputStream;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.common.util.ByteArray;
 import org.elasticsearch.core.IOUtils;
 
 import java.io.Closeable;
@@ -60,9 +54,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Set;
-import java.util.function.Function;
 
 /**
  * This implementation is forked from Lucene's BloomFilterPosting to support on-disk bloom filters.
@@ -77,26 +69,13 @@ public class ES85BloomFilterPostingsFormat extends PostingsFormat {
     static final String BLOOM_FILTER_META_FILE = "bfm";
     static final String BLOOM_FILTER_INDEX_FILE = "bfi";
 
-    private Function<String, PostingsFormat> postingsFormats;
-    private BigArrays bigArrays;
-
-    public ES85BloomFilterPostingsFormat(BigArrays bigArrays, Function<String, PostingsFormat> postingsFormats) {
-        this();
-        this.bigArrays = Objects.requireNonNull(bigArrays);
-        this.postingsFormats = Objects.requireNonNull(postingsFormats);
-    }
-
     public ES85BloomFilterPostingsFormat() {
         super(BLOOM_CODEC_NAME);
     }
 
     @Override
     public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
-        if (postingsFormats == null || bigArrays == null) {
-            assert false : BLOOM_CODEC_NAME + " was initialized with a wrong constructor";
-            throw new UnsupportedOperationException(BLOOM_CODEC_NAME + " was initialized with a wrong constructor");
-        }
-        return new FieldsWriter(state);
+        throw new UnsupportedOperationException();
     }
 
     @Override
@@ -109,128 +88,15 @@ public String toString() {
         return BLOOM_CODEC_NAME;
     }
 
-    private static String metaFile(SegmentInfo si, String segmentSuffix) {
+    static String metaFile(SegmentInfo si, String segmentSuffix) {
         return IndexFileNames.segmentFileName(si.name, segmentSuffix, BLOOM_FILTER_META_FILE);
     }
 
-    private static String indexFile(SegmentInfo si, String segmentSuffix) {
+    static String indexFile(SegmentInfo si, String segmentSuffix) {
         return IndexFileNames.segmentFileName(si.name, segmentSuffix, BLOOM_FILTER_INDEX_FILE);
     }
 
-    final class FieldsWriter extends FieldsConsumer {
-        private final SegmentWriteState state;
-        private final IndexOutput indexOut;
-        private final List<BloomFilter> bloomFilters = new ArrayList<>();
-        private final List<FieldsGroup> fieldsGroups = new ArrayList<>();
-        private final List<Closeable> toCloses = new ArrayList<>();
-        private boolean closed;
-
-        FieldsWriter(SegmentWriteState state) throws IOException {
-            this.state = state;
-            boolean success = false;
-            try {
-                indexOut = state.directory.createOutput(indexFile(state.segmentInfo, state.segmentSuffix), state.context);
-                toCloses.add(indexOut);
-                CodecUtil.writeIndexHeader(indexOut, BLOOM_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
-                success = true;
-            } finally {
-                if (success == false) {
-                    IOUtils.closeWhileHandlingException(toCloses);
-                }
-            }
-        }
-
-        @Override
-        public void write(Fields fields, NormsProducer norms) throws IOException {
-            writePostings(fields, norms);
-            writeBloomFilters(fields);
-        }
-
-        private void writePostings(Fields fields, NormsProducer norms) throws IOException {
-            final Map<PostingsFormat, FieldsGroup> currentGroups = new HashMap<>();
-            for (String field : fields) {
-                final PostingsFormat postingsFormat = postingsFormats.apply(field);
-                if (postingsFormat == null) {
-                    throw new IllegalStateException("PostingsFormat for field [" + field + "] wasn't specified");
-                }
-                FieldsGroup group = currentGroups.get(postingsFormat);
-                if (group == null) {
-                    group = new FieldsGroup(postingsFormat, Integer.toString(fieldsGroups.size()), new ArrayList<>());
-                    currentGroups.put(postingsFormat, group);
-                    fieldsGroups.add(group);
-                }
-                group.fields.add(field);
-            }
-            for (FieldsGroup group : currentGroups.values()) {
-                final FieldsConsumer writer = group.postingsFormat.fieldsConsumer(new SegmentWriteState(state, group.suffix));
-                toCloses.add(writer);
-                final Fields maskedFields = new FilterLeafReader.FilterFields(fields) {
-                    @Override
-                    public Iterator<String> iterator() {
-                        return group.fields.iterator();
-                    }
-                };
-                writer.write(maskedFields, norms);
-            }
-        }
-
-        private void writeBloomFilters(Fields fields) throws IOException {
-            for (String field : fields) {
-                final Terms terms = fields.terms(field);
-                if (terms == null) {
-                    continue;
-                }
-                final int bloomFilterSize = bloomFilterSize(state.segmentInfo.maxDoc());
-                final int numBytes = numBytesForBloomFilter(bloomFilterSize);
-                try (ByteArray buffer = bigArrays.newByteArray(numBytes)) {
-                    final TermsEnum termsEnum = terms.iterator();
-                    while (true) {
-                        final BytesRef term = termsEnum.next();
-                        if (term == null) {
-                            break;
-                        }
-                        final int hash = hashTerm(term) % bloomFilterSize;
-                        final int pos = hash >> 3;
-                        final int mask = 1 << (hash & 0x7);
-                        final byte val = (byte) (buffer.get(pos) | mask);
-                        buffer.set(pos, val);
-                    }
-                    bloomFilters.add(new BloomFilter(field, indexOut.getFilePointer(), bloomFilterSize));
-                    final BytesReference bytes = BytesReference.fromByteArray(buffer, numBytes);
-                    bytes.writeTo(new IndexOutputOutputStream(indexOut));
-                }
-            }
-        }
-
-        @Override
-        public void close() throws IOException {
-            if (closed) {
-                return;
-            }
-            closed = true;
-            try {
-                CodecUtil.writeFooter(indexOut);
-            } finally {
-                IOUtils.close(toCloses);
-            }
-            try (IndexOutput metaOut = state.directory.createOutput(metaFile(state.segmentInfo, state.segmentSuffix), state.context)) {
-                CodecUtil.writeIndexHeader(metaOut, BLOOM_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
-                // write postings formats
-                metaOut.writeVInt(fieldsGroups.size());
-                for (FieldsGroup group : fieldsGroups) {
-                    group.writeTo(metaOut, state.fieldInfos);
-                }
-                // Write bloom filters
-                metaOut.writeVInt(bloomFilters.size());
-                for (BloomFilter bloomFilter : bloomFilters) {
-                    bloomFilter.writeTo(metaOut, state.fieldInfos);
-                }
-                CodecUtil.writeFooter(metaOut);
-            }
-        }
-    }
-
-    private record BloomFilter(String field, long startFilePointer, int bloomFilterSize) {
+    record BloomFilter(String field, long startFilePointer, int bloomFilterSize) {
         void writeTo(IndexOutput out, FieldInfos fieldInfos) throws IOException {
             out.writeVInt(fieldInfos.fieldInfo(field).number);
             out.writeVLong(startFilePointer);
@@ -245,7 +111,7 @@ static BloomFilter readFrom(IndexInput in, FieldInfos fieldInfos) throws IOExcep
         }
     }
 
-    private record FieldsGroup(PostingsFormat postingsFormat, String suffix, List<String> fields) {
+    record FieldsGroup(PostingsFormat postingsFormat, String suffix, List<String> fields) {
         void writeTo(IndexOutput out, FieldInfos fieldInfos) throws IOException {
             out.writeString(postingsFormat.getName());
             out.writeString(suffix);
diff --git a/server/src/main/java/org/elasticsearch/index/codec/bloomfilter/ES87BloomFilterPostingsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/bloomfilter/ES87BloomFilterPostingsFormat.java
new file mode 100644
index 000000000000..191fe8f75b2f
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/codec/bloomfilter/ES87BloomFilterPostingsFormat.java
@@ -0,0 +1,774 @@
+/*
+ * @notice
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Modifications copyright (C) 2023 Elasticsearch B.V.
+ */
+package org.elasticsearch.index.codec.bloomfilter;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.FieldsConsumer;
+import org.apache.lucene.codecs.FieldsProducer;
+import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.index.BaseTermsEnum;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.FilterLeafReader;
+import org.apache.lucene.index.ImpactsEnum;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.TermState;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.RandomAccessInput;
+import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.lucene.store.IndexOutputOutputStream;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.ByteArray;
+import org.elasticsearch.common.util.ByteUtils;
+import org.elasticsearch.core.IOUtils;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Function;
+
+/**
+ * This implementation is forked from Lucene's BloomFilterPosting to support on-disk bloom filters.
+ * <p>
+ * A {@link PostingsFormat} useful for low doc-frequency fields such as primary keys. Bloom filters
+ * offers "fast-fail" for reads in segments known to have no record of the key.
+ */
+public class ES87BloomFilterPostingsFormat extends PostingsFormat {
+    static final String BLOOM_CODEC_NAME = "ES87BloomFilter";
+    static final int VERSION_START = 0;
+    static final int VERSION_CURRENT = VERSION_START;
+    static final String BLOOM_FILTER_META_FILE = "bfm";
+    static final String BLOOM_FILTER_INDEX_FILE = "bfi";
+    /** Bloom filters target 10 bits per entry, which, along with 7 hash functions, yields about 1% false positives. */
+    private static final int BITS_PER_ENTRY = 10;
+    /** The optimal number of hash functions for a bloom filter is approximately 0.7 times the number of bits per entry. */
+    private static final int NUM_HASH_FUNCTIONS = 7;
+
+    private Function<String, PostingsFormat> postingsFormats;
+    private BigArrays bigArrays;
+
+    public ES87BloomFilterPostingsFormat(BigArrays bigArrays, Function<String, PostingsFormat> postingsFormats) {
+        this();
+        this.bigArrays = Objects.requireNonNull(bigArrays);
+        this.postingsFormats = Objects.requireNonNull(postingsFormats);
+    }
+
+    public ES87BloomFilterPostingsFormat() {
+        super(BLOOM_CODEC_NAME);
+    }
+
+    @Override
+    public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+        if (postingsFormats == null || bigArrays == null) {
+            assert false : BLOOM_CODEC_NAME + " was initialized with a wrong constructor";
+            throw new UnsupportedOperationException(BLOOM_CODEC_NAME + " was initialized with a wrong constructor");
+        }
+        return new FieldsWriter(state);
+    }
+
+    @Override
+    public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
+        return new FieldsReader(state);
+    }
+
+    @Override
+    public String toString() {
+        return BLOOM_CODEC_NAME;
+    }
+
+    private static String metaFile(SegmentInfo si, String segmentSuffix) {
+        return IndexFileNames.segmentFileName(si.name, segmentSuffix, BLOOM_FILTER_META_FILE);
+    }
+
+    private static String indexFile(SegmentInfo si, String segmentSuffix) {
+        return IndexFileNames.segmentFileName(si.name, segmentSuffix, BLOOM_FILTER_INDEX_FILE);
+    }
+
+    final class FieldsWriter extends FieldsConsumer {
+        private final SegmentWriteState state;
+        private final IndexOutput indexOut;
+        private final List<BloomFilter> bloomFilters = new ArrayList<>();
+        private final List<FieldsGroup> fieldsGroups = new ArrayList<>();
+        private final List<Closeable> toCloses = new ArrayList<>();
+        private boolean closed;
+        private final int[] hashes = new int[NUM_HASH_FUNCTIONS];
+
+        FieldsWriter(SegmentWriteState state) throws IOException {
+            this.state = state;
+            boolean success = false;
+            try {
+                indexOut = state.directory.createOutput(indexFile(state.segmentInfo, state.segmentSuffix), state.context);
+                toCloses.add(indexOut);
+                CodecUtil.writeIndexHeader(indexOut, BLOOM_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
+                success = true;
+            } finally {
+                if (success == false) {
+                    IOUtils.closeWhileHandlingException(toCloses);
+                }
+            }
+        }
+
+        @Override
+        public void write(Fields fields, NormsProducer norms) throws IOException {
+            writePostings(fields, norms);
+            writeBloomFilters(fields);
+        }
+
+        private void writePostings(Fields fields, NormsProducer norms) throws IOException {
+            final Map<PostingsFormat, FieldsGroup> currentGroups = new HashMap<>();
+            for (String field : fields) {
+                final PostingsFormat postingsFormat = postingsFormats.apply(field);
+                if (postingsFormat == null) {
+                    throw new IllegalStateException("PostingsFormat for field [" + field + "] wasn't specified");
+                }
+                FieldsGroup group = currentGroups.get(postingsFormat);
+                if (group == null) {
+                    group = new FieldsGroup(postingsFormat, Integer.toString(fieldsGroups.size()), new ArrayList<>());
+                    currentGroups.put(postingsFormat, group);
+                    fieldsGroups.add(group);
+                }
+                group.fields.add(field);
+            }
+            for (FieldsGroup group : currentGroups.values()) {
+                final FieldsConsumer writer = group.postingsFormat.fieldsConsumer(new SegmentWriteState(state, group.suffix));
+                toCloses.add(writer);
+                final Fields maskedFields = new FilterLeafReader.FilterFields(fields) {
+                    @Override
+                    public Iterator<String> iterator() {
+                        return group.fields.iterator();
+                    }
+                };
+                writer.write(maskedFields, norms);
+            }
+        }
+
+        private void writeBloomFilters(Fields fields) throws IOException {
+            for (String field : fields) {
+                final Terms terms = fields.terms(field);
+                if (terms == null) {
+                    continue;
+                }
+                final int bloomFilterSize = bloomFilterSize(state.segmentInfo.maxDoc());
+                final int numBytes = numBytesForBloomFilter(bloomFilterSize);
+                try (ByteArray buffer = bigArrays.newByteArray(numBytes)) {
+                    final TermsEnum termsEnum = terms.iterator();
+                    while (true) {
+                        final BytesRef term = termsEnum.next();
+                        if (term == null) {
+                            break;
+                        }
+
+                        hashTerm(term, hashes);
+                        for (int hash : hashes) {
+                            hash = hash % bloomFilterSize;
+                            final int pos = hash >> 3;
+                            final int mask = 1 << (hash & 7);
+                            final byte val = (byte) (buffer.get(pos) | mask);
+                            buffer.set(pos, val);
+                        }
+                    }
+                    bloomFilters.add(new BloomFilter(field, indexOut.getFilePointer(), bloomFilterSize));
+                    final BytesReference bytes = BytesReference.fromByteArray(buffer, numBytes);
+                    bytes.writeTo(new IndexOutputOutputStream(indexOut));
+                }
+            }
+        }
+
+        @Override
+        public void close() throws IOException {
+            if (closed) {
+                return;
+            }
+            final long indexFileLength;
+            closed = true;
+            try {
+                CodecUtil.writeFooter(indexOut);
+                indexFileLength = indexOut.getFilePointer();
+            } finally {
+                IOUtils.close(toCloses);
+            }
+            try (IndexOutput metaOut = state.directory.createOutput(metaFile(state.segmentInfo, state.segmentSuffix), state.context)) {
+                CodecUtil.writeIndexHeader(metaOut, BLOOM_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
+                // write postings formats
+                metaOut.writeVInt(fieldsGroups.size());
+                for (FieldsGroup group : fieldsGroups) {
+                    group.writeTo(metaOut, state.fieldInfos);
+                }
+                // Write bloom filters metadata
+                metaOut.writeVInt(bloomFilters.size());
+                for (BloomFilter bloomFilter : bloomFilters) {
+                    bloomFilter.writeTo(metaOut, state.fieldInfos);
+                }
+                metaOut.writeVLong(indexFileLength);
+                CodecUtil.writeFooter(metaOut);
+            }
+        }
+    }
+
+    private record BloomFilter(String field, long startFilePointer, int bloomFilterSize) {
+        void writeTo(IndexOutput out, FieldInfos fieldInfos) throws IOException {
+            out.writeVInt(fieldInfos.fieldInfo(field).number);
+            out.writeVLong(startFilePointer);
+            out.writeVInt(bloomFilterSize);
+        }
+
+        static BloomFilter readFrom(IndexInput in, FieldInfos fieldInfos) throws IOException {
+            final String fieldName = fieldInfos.fieldInfo(in.readVInt()).name;
+            final long startFilePointer = in.readVLong();
+            final int bloomFilterSize = in.readVInt();
+            return new BloomFilter(fieldName, startFilePointer, bloomFilterSize);
+        }
+    }
+
+    private record FieldsGroup(PostingsFormat postingsFormat, String suffix, List<String> fields) {
+        void writeTo(IndexOutput out, FieldInfos fieldInfos) throws IOException {
+            out.writeString(postingsFormat.getName());
+            out.writeString(suffix);
+            out.writeVInt(fields.size());
+            for (String field : fields) {
+                out.writeVInt(fieldInfos.fieldInfo(field).number);
+            }
+        }
+
+        static FieldsGroup readFrom(IndexInput in, FieldInfos fieldInfos) throws IOException {
+            final PostingsFormat postingsFormat = forName(in.readString());
+            final String suffix = in.readString();
+            final int numFields = in.readVInt();
+            final List<String> fields = new ArrayList<>();
+            for (int i = 0; i < numFields; i++) {
+                fields.add(fieldInfos.fieldInfo(in.readVInt()).name);
+            }
+            return new FieldsGroup(postingsFormat, suffix, fields);
+        }
+    }
+
+    static final class FieldsReader extends FieldsProducer {
+        private final Map<String, BloomFilter> bloomFilters;
+        private final List<Closeable> toCloses = new ArrayList<>();
+        private final Map<String, FieldsProducer> readerMap = new HashMap<>();
+        private final IndexInput indexIn;
+
+        FieldsReader(SegmentReadState state) throws IOException {
+            boolean success = false;
+            try (
+                ChecksumIndexInput metaIn = state.directory.openChecksumInput(
+                    metaFile(state.segmentInfo, state.segmentSuffix),
+                    IOContext.READONCE
+                )
+            ) {
+                Map<String, BloomFilter> bloomFilters = null;
+                Throwable priorE = null;
+                long indexFileLength = 0;
+                try {
+                    CodecUtil.checkIndexHeader(
+                        metaIn,
+                        BLOOM_CODEC_NAME,
+                        VERSION_START,
+                        VERSION_CURRENT,
+                        state.segmentInfo.getId(),
+                        state.segmentSuffix
+                    );
+                    // read postings formats
+                    final int numFieldsGroups = metaIn.readVInt();
+                    for (int i = 0; i < numFieldsGroups; i++) {
+                        final FieldsGroup group = FieldsGroup.readFrom(metaIn, state.fieldInfos);
+                        final FieldsProducer reader = group.postingsFormat.fieldsProducer(new SegmentReadState(state, group.suffix));
+                        toCloses.add(reader);
+                        for (String field : group.fields) {
+                            readerMap.put(field, reader);
+                        }
+                    }
+                    // read bloom filters
+                    final int numBloomFilters = metaIn.readVInt();
+                    bloomFilters = new HashMap<>(numBloomFilters);
+                    for (int i = 0; i < numBloomFilters; i++) {
+                        final BloomFilter bloomFilter = BloomFilter.readFrom(metaIn, state.fieldInfos);
+                        bloomFilters.put(bloomFilter.field, bloomFilter);
+                    }
+
+                    indexFileLength = metaIn.readVLong();
+                } catch (Throwable t) {
+                    priorE = t;
+                } finally {
+                    CodecUtil.checkFooter(metaIn, priorE);
+                }
+                this.bloomFilters = bloomFilters;
+                indexIn = state.directory.openInput(indexFile(state.segmentInfo, state.segmentSuffix), state.context);
+                toCloses.add(indexIn);
+                CodecUtil.checkIndexHeader(
+                    indexIn,
+                    BLOOM_CODEC_NAME,
+                    VERSION_START,
+                    VERSION_CURRENT,
+                    state.segmentInfo.getId(),
+                    state.segmentSuffix
+                );
+                CodecUtil.retrieveChecksum(indexIn, indexFileLength);
+                assert assertBloomFilterSizes(state.segmentInfo);
+                success = true;
+            } finally {
+                if (success == false) {
+                    IOUtils.closeWhileHandlingException(toCloses);
+                }
+            }
+        }
+
+        private boolean assertBloomFilterSizes(SegmentInfo segmentInfo) {
+            for (BloomFilter bloomFilter : bloomFilters.values()) {
+                assert bloomFilter.bloomFilterSize == bloomFilterSize(segmentInfo.maxDoc())
+                    : "bloom_filter=" + bloomFilter + ", max_docs=" + segmentInfo.maxDoc();
+            }
+            return true;
+        }
+
+        @Override
+        public Iterator<String> iterator() {
+            return readerMap.keySet().iterator();
+        }
+
+        @Override
+        public void close() throws IOException {
+            IOUtils.close(toCloses);
+        }
+
+        @Override
+        public Terms terms(String field) throws IOException {
+            final FieldsProducer reader = readerMap.get(field);
+            if (reader == null) {
+                return null;
+            }
+            final Terms terms = reader.terms(field);
+            if (terms == null) {
+                return null;
+            }
+            final BloomFilter bloomFilter = bloomFilters.get(field);
+            if (bloomFilter != null) {
+                final RandomAccessInput data = indexIn.randomAccessSlice(
+                    bloomFilter.startFilePointer(),
+                    numBytesForBloomFilter(bloomFilter.bloomFilterSize)
+                );
+                return new BloomFilterTerms(terms, data, bloomFilter.bloomFilterSize);
+            } else {
+                return terms;
+            }
+        }
+
+        @Override
+        public int size() {
+            return readerMap.size();
+        }
+
+        @Override
+        public void checkIntegrity() throws IOException {
+            // already fully checked the meta file; let's fully checked the index file.
+            CodecUtil.checksumEntireFile(indexIn);
+            // multiple fields can share the same reader
+            final Set<FieldsProducer> seenReaders = new HashSet<>();
+            for (FieldsProducer reader : readerMap.values()) {
+                if (seenReaders.add(reader)) {
+                    reader.checkIntegrity();
+                }
+            }
+        }
+    }
+
+    private static class BloomFilterTerms extends FilterLeafReader.FilterTerms {
+        private final RandomAccessInput data;
+        private final int bloomFilterSize;
+        private final int[] hashes = new int[NUM_HASH_FUNCTIONS];
+
+        BloomFilterTerms(Terms in, RandomAccessInput data, int bloomFilterSize) {
+            super(in);
+            this.data = data;
+            this.bloomFilterSize = bloomFilterSize;
+        }
+
+        private boolean mayContainTerm(BytesRef term) throws IOException {
+            hashTerm(term, hashes);
+            for (int hash : hashes) {
+                hash = hash % bloomFilterSize;
+                final int pos = hash >> 3;
+                final int mask = 1 << (hash & 7);
+                final byte bits = data.readByte(pos);
+                if ((bits & mask) == 0) {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        @Override
+        public TermsEnum iterator() throws IOException {
+            return new LazyFilterTermsEnum() {
+                private TermsEnum delegate;
+
+                @Override
+                TermsEnum getDelegate() throws IOException {
+                    if (delegate == null) {
+                        delegate = in.iterator();
+                    }
+                    return delegate;
+                }
+
+                @Override
+                public boolean seekExact(BytesRef term) throws IOException {
+                    if (mayContainTerm(term)) {
+                        return getDelegate().seekExact(term);
+                    } else {
+                        return false;
+                    }
+                }
+
+                @Override
+                public void seekExact(BytesRef term, TermState state) throws IOException {
+                    getDelegate().seekExact(term, state);
+                }
+
+                @Override
+                public TermState termState() throws IOException {
+                    // TODO: return TermState that includes BloomFilter and fix _disk_usage API
+                    return getDelegate().termState();
+                }
+            };
+        }
+    }
+
+    private abstract static class LazyFilterTermsEnum extends BaseTermsEnum {
+        abstract TermsEnum getDelegate() throws IOException;
+
+        @Override
+        public SeekStatus seekCeil(BytesRef text) throws IOException {
+            return getDelegate().seekCeil(text);
+        }
+
+        @Override
+        public void seekExact(long ord) throws IOException {
+            getDelegate().seekExact(ord);
+        }
+
+        @Override
+        public BytesRef term() throws IOException {
+            return getDelegate().term();
+        }
+
+        @Override
+        public long ord() throws IOException {
+            return getDelegate().ord();
+        }
+
+        @Override
+        public int docFreq() throws IOException {
+            return getDelegate().docFreq();
+        }
+
+        @Override
+        public long totalTermFreq() throws IOException {
+            return getDelegate().totalTermFreq();
+        }
+
+        @Override
+        public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException {
+            return getDelegate().postings(reuse, flags);
+        }
+
+        @Override
+        public ImpactsEnum impacts(int flags) throws IOException {
+            return getDelegate().impacts(flags);
+        }
+
+        @Override
+        public BytesRef next() throws IOException {
+            return getDelegate().next();
+        }
+
+        @Override
+        public AttributeSource attributes() {
+            try {
+                return getDelegate().attributes();
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        }
+    }
+
+    static int bloomFilterSize(int maxDocs) {
+        if (maxDocs < 1) {
+            throw new IllegalStateException("maxDocs must be greater than or equal to 1, got " + maxDocs);
+        }
+        // 10% saturation (i.e., 10 bits for each term)
+        long numBits = maxDocs * (long) BITS_PER_ENTRY;
+        // Round to the next multiple of 8 since we can only store whole bytes
+        numBits = ((numBits - 1) | 0x07L) + 1;
+        if (numBits > Integer.MAX_VALUE) {
+            return Integer.MAX_VALUE;
+        } else {
+            return (int) numBits;
+        }
+    }
+
+    static int numBytesForBloomFilter(int bloomFilterSize) {
+        return Math.toIntExact((bloomFilterSize + 7L) / 8L);
+    }
+
+    // Uses MurmurHash3-128 to generate a 64-bit hash value, then picks 7 subsets of 31 bits each and returns the values in the
+    // outputs array. This provides us with 7 reasonably independent hashes of the data for the cost of one MurmurHash3 calculation.
+    static int[] hashTerm(BytesRef br, int[] outputs) {
+        final long hash64 = MurmurHash3.hash64(br.bytes, br.offset, br.length);
+        final int upperHalf = (int) (hash64 >> 32);
+        final int lowerHalf = (int) hash64;
+        // Derive 7 hash outputs by combining the two 64-bit halves, adding the upper half multiplied with different small constants
+        // without common gcd.
+        outputs[0] = (lowerHalf + 2 * upperHalf) & 0x7FFF_FFFF;
+        outputs[1] = (lowerHalf + 3 * upperHalf) & 0x7FFF_FFFF;
+        outputs[2] = (lowerHalf + 5 * upperHalf) & 0x7FFF_FFFF;
+        outputs[3] = (lowerHalf + 7 * upperHalf) & 0x7FFF_FFFF;
+        outputs[4] = (lowerHalf + 11 * upperHalf) & 0x7FFF_FFFF;
+        outputs[5] = (lowerHalf + 13 * upperHalf) & 0x7FFF_FFFF;
+        outputs[6] = (lowerHalf + 17 * upperHalf) & 0x7FFF_FFFF;
+        return outputs;
+    }
+
+    //
+    // The following Murmur3 implementation is borrowed from commons-codec.
+    //
+
+    /**
+     * Implementation of the MurmurHash3 128-bit hash functions.
+     *
+     * <p>
+     * MurmurHash is a non-cryptographic hash function suitable for general hash-based lookup. The name comes from two basic
+     * operations, multiply (MU) and rotate (R), used in its inner loop. Unlike cryptographic hash functions, it is not
+     * specifically designed to be difficult to reverse by an adversary, making it unsuitable for cryptographic purposes.
+     * </p>
+     *
+     * <p>
+     * This contains a Java port of the 32-bit hash function {@code MurmurHash3_x86_32} and the 128-bit hash function
+     * {@code MurmurHash3_x64_128} from Austin Appleby's original {@code c++} code in SMHasher.
+     * </p>
+     *
+     * <p>
+     * This is public domain code with no copyrights. From home page of
+     * <a href="https://github.com/aappleby/smhasher">SMHasher</a>:
+     * </p>
+     *
+     * <blockquote> "All MurmurHash versions are public domain software, and the author disclaims all copyright to their
+     * code." </blockquote>
+     *
+     * <p>
+     * Original adaption from Apache Hive. That adaption contains a {@code hash64} method that is not part of the original
+     * MurmurHash3 code. It is not recommended to use these methods. They will be removed in a future release. To obtain a
+     * 64-bit hash use half of the bits from the {@code hash128x64} methods using the input data converted to bytes.
+     * </p>
+     *
+     * @see <a href="https://en.wikipedia.org/wiki/MurmurHash">MurmurHash</a>
+     * @see <a href="https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp"> Original MurmurHash3 c++
+     *      code</a>
+     * @see <a href=
+     *      "https://github.com/apache/hive/blob/master/storage-api/src/java/org/apache/hive/common/util/Murmur3.java">
+     *      Apache Hive Murmer3</a>
+     * @since 1.13
+     */
+    public static final class MurmurHash3 {
+        /**
+         * A default seed to use for the murmur hash algorithm.
+         * Has the value {@code 104729}.
+         */
+        public static final int DEFAULT_SEED = 104729;
+
+        // Constants for 128-bit variant
+        private static final long C1 = 0x87c37b91114253d5L;
+        private static final long C2 = 0x4cf5ad432745937fL;
+        private static final int R1 = 31;
+        private static final int R2 = 27;
+        private static final int R3 = 33;
+        private static final int M = 5;
+        private static final int N1 = 0x52dce729;
+        private static final int N2 = 0x38495ab5;
+
+        /** No instance methods. */
+        private MurmurHash3() {}
+
+        /**
+         * Generates 64-bit hash from the byte array with the given offset, length and seed by discarding the second value of the 128-bit
+         * hash.
+         *
+         * This version uses the default seed.
+         *
+         * @param data The input byte array
+         * @param offset The first element of array
+         * @param length The length of array
+         * @return The sum of the two 64-bit hashes that make up the hash128
+         */
+        public static long hash64(final byte[] data, final int offset, final int length) {
+            // We hope that the C2 escape analysis prevents ths allocation from creating GC pressure.
+            long[] hash128 = { 0, 0 };
+            hash128x64Internal(data, offset, length, DEFAULT_SEED, hash128);
+            return hash128[0];
+        }
+
+        /**
+         * Generates 128-bit hash from the byte array with the given offset, length and seed.
+         *
+         * <p>This is an implementation of the 128-bit hash function {@code MurmurHash3_x64_128}
+         * from Austin Appleby's original MurmurHash3 {@code c++} code in SMHasher.</p>
+         *
+         * @param data The input byte array
+         * @param offset The first element of array
+         * @param length The length of array
+         * @param seed The initial seed value
+         * @return The 128-bit hash (2 longs)
+         */
+        @SuppressWarnings("fallthrough")
+        private static long[] hash128x64Internal(
+            final byte[] data,
+            final int offset,
+            final int length,
+            final long seed,
+            final long[] result
+        ) {
+            long h1 = seed;
+            long h2 = seed;
+            final int nblocks = length >> 4;
+
+            // body
+            for (int i = 0; i < nblocks; i++) {
+                final int index = offset + (i << 4);
+                long k1 = ByteUtils.readLongLE(data, index);
+                long k2 = ByteUtils.readLongLE(data, index + 8);
+
+                // mix functions for k1
+                k1 *= C1;
+                k1 = Long.rotateLeft(k1, R1);
+                k1 *= C2;
+                h1 ^= k1;
+                h1 = Long.rotateLeft(h1, R2);
+                h1 += h2;
+                h1 = h1 * M + N1;
+
+                // mix functions for k2
+                k2 *= C2;
+                k2 = Long.rotateLeft(k2, R3);
+                k2 *= C1;
+                h2 ^= k2;
+                h2 = Long.rotateLeft(h2, R1);
+                h2 += h1;
+                h2 = h2 * M + N2;
+            }
+
+            // tail
+            long k1 = 0;
+            long k2 = 0;
+            final int index = offset + (nblocks << 4);
+            switch (offset + length - index) {
+                case 15:
+                    k2 ^= ((long) data[index + 14] & 0xff) << 48;
+                case 14:
+                    k2 ^= ((long) data[index + 13] & 0xff) << 40;
+                case 13:
+                    k2 ^= ((long) data[index + 12] & 0xff) << 32;
+                case 12:
+                    k2 ^= ((long) data[index + 11] & 0xff) << 24;
+                case 11:
+                    k2 ^= ((long) data[index + 10] & 0xff) << 16;
+                case 10:
+                    k2 ^= ((long) data[index + 9] & 0xff) << 8;
+                case 9:
+                    k2 ^= data[index + 8] & 0xff;
+                    k2 *= C2;
+                    k2 = Long.rotateLeft(k2, R3);
+                    k2 *= C1;
+                    h2 ^= k2;
+
+                case 8:
+                    k1 ^= ((long) data[index + 7] & 0xff) << 56;
+                case 7:
+                    k1 ^= ((long) data[index + 6] & 0xff) << 48;
+                case 6:
+                    k1 ^= ((long) data[index + 5] & 0xff) << 40;
+                case 5:
+                    k1 ^= ((long) data[index + 4] & 0xff) << 32;
+                case 4:
+                    k1 ^= ((long) data[index + 3] & 0xff) << 24;
+                case 3:
+                    k1 ^= ((long) data[index + 2] & 0xff) << 16;
+                case 2:
+                    k1 ^= ((long) data[index + 1] & 0xff) << 8;
+                case 1:
+                    k1 ^= data[index] & 0xff;
+                    k1 *= C1;
+                    k1 = Long.rotateLeft(k1, R1);
+                    k1 *= C2;
+                    h1 ^= k1;
+            }
+
+            // finalization
+            h1 ^= length;
+            h2 ^= length;
+
+            h1 += h2;
+            h2 += h1;
+
+            h1 = fmix64(h1);
+            h2 = fmix64(h2);
+
+            h1 += h2;
+            h2 += h1;
+
+            result[0] = h1;
+            result[1] = h2;
+            return result;
+        }
+
+        /**
+         * Performs the final avalanche mix step of the 64-bit hash function {@code MurmurHash3_x64_128}.
+         *
+         * @param hash The current hash
+         * @return The final hash
+         */
+        private static long fmix64(long hash) {
+            hash ^= (hash >>> 33);
+            hash *= 0xff51afd7ed558ccdL;
+            hash ^= (hash >>> 33);
+            hash *= 0xc4ceb9fe1a85ec53L;
+            hash ^= (hash >>> 33);
+            return hash;
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtil.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtil.java
new file mode 100644
index 000000000000..23ad4810d853
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtil.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.codec.tsdb;
+
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+
+import java.io.IOException;
+
+public class DocValuesForUtil {
+    private final ForUtil forUtil = new ForUtil();
+    private final int blockSize;
+
+    public DocValuesForUtil() {
+        this(ES87TSDBDocValuesFormat.DEFAULT_NUMERIC_BLOCK_SIZE);
+    }
+
+    public DocValuesForUtil(int blockSize) {
+        this.blockSize = blockSize;
+    }
+
+    void encode(long[] in, int bitsPerValue, DataOutput out) throws IOException {
+        if (bitsPerValue <= 24) { // these bpvs are handled efficiently by ForUtil
+            forUtil.encode(in, bitsPerValue, out);
+        } else if (bitsPerValue <= 32) {
+            collapse32(in);
+            for (int i = 0; i < blockSize / 2; ++i) {
+                out.writeLong(in[i]);
+            }
+        } else {
+            for (long l : in) {
+                out.writeLong(l);
+            }
+        }
+    }
+
+    void decode(int bitsPerValue, DataInput in, long[] out) throws IOException {
+        if (bitsPerValue <= 24) {
+            forUtil.decode(bitsPerValue, in, out);
+        } else if (bitsPerValue <= 32) {
+            in.readLongs(out, 0, blockSize / 2);
+            expand32(out);
+        } else {
+            in.readLongs(out, 0, blockSize);
+        }
+    }
+
+    private static void collapse32(long[] arr) {
+        for (int i = 0; i < 64; ++i) {
+            arr[i] = (arr[i] << 32) | arr[64 + i];
+        }
+    }
+
+    private static void expand32(long[] arr) {
+        for (int i = 0; i < 64; ++i) {
+            long l = arr[i];
+            arr[i] = l >>> 32;
+            arr[64 + i] = l & 0xFFFFFFFFL;
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesConsumer.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesConsumer.java
new file mode 100644
index 000000000000..dfc35fc8ab38
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesConsumer.java
@@ -0,0 +1,246 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.codec.tsdb;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.codecs.lucene90.IndexedDISI;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.EmptyDocValuesProducer;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.store.ByteBuffersDataOutput;
+import org.apache.lucene.store.ByteBuffersIndexOutput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.packed.DirectMonotonicWriter;
+import org.elasticsearch.core.IOUtils;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+final class ES87TSDBDocValuesConsumer extends DocValuesConsumer {
+
+    IndexOutput data, meta;
+    final int maxDoc;
+    private final int numericBlockShift;
+    private final int numericBlockSize;
+    private final int directMonotonicBlockShift;
+
+    ES87TSDBDocValuesConsumer(
+        SegmentWriteState state,
+        String dataCodec,
+        String dataExtension,
+        String metaCodec,
+        String metaExtension,
+        int numericBlockShift,
+        int numericBlockSize,
+        int directMonotonicBlockShift
+    ) throws IOException {
+        boolean success = false;
+        this.numericBlockShift = numericBlockShift;
+        this.numericBlockSize = numericBlockSize;
+        this.directMonotonicBlockShift = directMonotonicBlockShift;
+        try {
+            final String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
+            data = state.directory.createOutput(dataName, state.context);
+            CodecUtil.writeIndexHeader(
+                data,
+                dataCodec,
+                ES87TSDBDocValuesFormat.VERSION_CURRENT,
+                state.segmentInfo.getId(),
+                state.segmentSuffix
+            );
+            String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
+            meta = state.directory.createOutput(metaName, state.context);
+            CodecUtil.writeIndexHeader(
+                meta,
+                metaCodec,
+                ES87TSDBDocValuesFormat.VERSION_CURRENT,
+                state.segmentInfo.getId(),
+                state.segmentSuffix
+            );
+            maxDoc = state.segmentInfo.maxDoc();
+            success = true;
+        } finally {
+            if (success == false) {
+                IOUtils.closeWhileHandlingException(this);
+            }
+        }
+    }
+
+    @Override
+    public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+        meta.writeInt(field.number);
+        meta.writeByte(ES87TSDBDocValuesFormat.NUMERIC);
+        writeNumericField(field, new EmptyDocValuesProducer() {
+            @Override
+            public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
+                return DocValues.singleton(valuesProducer.getNumeric(field));
+            }
+        });
+    }
+
+    private long[] writeNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+        int numDocsWithValue = 0;
+        long numValues = 0;
+
+        SortedNumericDocValues values = valuesProducer.getSortedNumeric(field);
+        for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
+            numDocsWithValue++;
+            final int count = values.docValueCount();
+            numValues += count;
+        }
+
+        if (numDocsWithValue == 0) { // meta[-2, 0]: No documents with values
+            meta.writeLong(-2); // docsWithFieldOffset
+            meta.writeLong(0L); // docsWithFieldLength
+            meta.writeShort((short) -1); // jumpTableEntryCount
+            meta.writeByte((byte) -1); // denseRankPower
+        } else if (numDocsWithValue == maxDoc) { // meta[-1, 0]: All documents have values
+            meta.writeLong(-1); // docsWithFieldOffset
+            meta.writeLong(0L); // docsWithFieldLength
+            meta.writeShort((short) -1); // jumpTableEntryCount
+            meta.writeByte((byte) -1); // denseRankPower
+        } else { // meta[data.offset, data.length]: IndexedDISI structure for documents with values
+            long offset = data.getFilePointer();
+            meta.writeLong(offset); // docsWithFieldOffset
+            values = valuesProducer.getSortedNumeric(field);
+            final short jumpTableEntryCount = IndexedDISI.writeBitSet(values, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER);
+            meta.writeLong(data.getFilePointer() - offset); // docsWithFieldLength
+            meta.writeShort(jumpTableEntryCount);
+            meta.writeByte(IndexedDISI.DEFAULT_DENSE_RANK_POWER);
+        }
+        meta.writeLong(numValues);
+
+        if (numValues > 0) {
+            meta.writeInt(directMonotonicBlockShift);
+            final ByteBuffersDataOutput indexOut = new ByteBuffersDataOutput();
+            final DirectMonotonicWriter indexWriter = DirectMonotonicWriter.getInstance(
+                meta,
+                new ByteBuffersIndexOutput(indexOut, "temp-dv-index", "temp-dv-index"),
+                1L + ((numValues - 1) >>> numericBlockShift),
+                directMonotonicBlockShift
+            );
+
+            final long[] buffer = new long[numericBlockSize];
+            int bufferSize = 0;
+            final long valuesDataOffset = data.getFilePointer();
+            final ES87TSDBDocValuesEncoder encoder = new ES87TSDBDocValuesEncoder(numericBlockSize);
+
+            values = valuesProducer.getSortedNumeric(field);
+            for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
+                final int count = values.docValueCount();
+                for (int i = 0; i < count; ++i) {
+                    buffer[bufferSize++] = values.nextValue();
+                    if (bufferSize == numericBlockSize) {
+                        indexWriter.add(data.getFilePointer() - valuesDataOffset);
+                        encoder.encode(buffer, data);
+                        bufferSize = 0;
+                    }
+                }
+            }
+            if (bufferSize > 0) {
+                indexWriter.add(data.getFilePointer() - valuesDataOffset);
+                // Fill unused slots in the block with zeroes rather than junk
+                Arrays.fill(buffer, bufferSize, numericBlockSize, 0L);
+                encoder.encode(buffer, data);
+            }
+
+            final long valuesDataLength = data.getFilePointer() - valuesDataOffset;
+            indexWriter.finish();
+            final long indexDataOffset = data.getFilePointer();
+            data.copyBytes(indexOut.toDataInput(), indexOut.size());
+            meta.writeLong(indexDataOffset);
+            meta.writeLong(data.getFilePointer() - indexDataOffset);
+
+            meta.writeLong(valuesDataOffset);
+            meta.writeLong(valuesDataLength);
+        }
+
+        return new long[] { numDocsWithValue, numValues };
+    }
+
+    @Override
+    public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+        throw new UnsupportedOperationException("Unsupported binary doc values for field [" + field.name + "]");
+    }
+
+    @Override
+    public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+        throw new UnsupportedOperationException("Unsupported sorted doc values for field [" + field.name + "]");
+    }
+
+    @Override
+    public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+        meta.writeInt(field.number);
+        meta.writeByte(ES87TSDBDocValuesFormat.SORTED_NUMERIC);
+        writeSortedNumericField(field, valuesProducer);
+    }
+
+    private void writeSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+        long[] stats = writeNumericField(field, valuesProducer);
+        int numDocsWithField = Math.toIntExact(stats[0]);
+        long numValues = stats[1];
+        assert numValues >= numDocsWithField;
+
+        meta.writeInt(numDocsWithField);
+        if (numValues > numDocsWithField) {
+            long start = data.getFilePointer();
+            meta.writeLong(start);
+            meta.writeVInt(directMonotonicBlockShift);
+
+            final DirectMonotonicWriter addressesWriter = DirectMonotonicWriter.getInstance(
+                meta,
+                data,
+                numDocsWithField + 1L,
+                directMonotonicBlockShift
+            );
+            long addr = 0;
+            addressesWriter.add(addr);
+            SortedNumericDocValues values = valuesProducer.getSortedNumeric(field);
+            for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
+                addr += values.docValueCount();
+                addressesWriter.add(addr);
+            }
+            addressesWriter.finish();
+            meta.writeLong(data.getFilePointer() - start);
+        }
+    }
+
+    @Override
+    public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
+        throw new UnsupportedOperationException("Unsupported sorted set doc values for field [" + field.name + "]");
+    }
+
+    @Override
+    public void close() throws IOException {
+        boolean success = false;
+        try {
+            if (meta != null) {
+                meta.writeInt(-1); // write EOF marker
+                CodecUtil.writeFooter(meta); // write checksum
+            }
+            if (data != null) {
+                CodecUtil.writeFooter(data); // write checksum
+            }
+            success = true;
+        } finally {
+            if (success) {
+                IOUtils.close(data, meta);
+            } else {
+                IOUtils.closeWhileHandlingException(data, meta);
+            }
+            meta = data = null;
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesEncoder.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesEncoder.java
new file mode 100644
index 000000000000..5bae4857a5c8
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesEncoder.java
@@ -0,0 +1,212 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.codec.tsdb;
+
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.MathUtil;
+import org.apache.lucene.util.packed.PackedInts;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+public class ES87TSDBDocValuesEncoder {
+    private final DocValuesForUtil forUtil;
+    private final int blockSize;
+
+    public ES87TSDBDocValuesEncoder() {
+        this(ES87TSDBDocValuesFormat.DEFAULT_NUMERIC_BLOCK_SIZE);
+    }
+
+    public ES87TSDBDocValuesEncoder(int blockSize) {
+        this.blockSize = blockSize;
+        this.forUtil = new DocValuesForUtil(blockSize);
+    }
+
+    public int getBlockSize() {
+        return blockSize;
+    }
+
+    /**
+     * Delta-encode monotonic fields. This is typically helpful with near-primary sort fields or
+     * SORTED_NUMERIC/SORTED_SET doc values with many values per document.
+     */
+    private void deltaEncode(int token, int tokenBits, long[] in, DataOutput out) throws IOException {
+        int gts = 0;
+        int lts = 0;
+        for (int i = 1; i < blockSize; ++i) {
+            if (in[i] > in[i - 1]) {
+                gts++;
+            } else if (in[i] < in[i - 1]) {
+                lts++;
+            }
+        }
+
+        final boolean doDeltaCompression = (gts == 0 && lts >= 2) || (lts == 0 && gts >= 2);
+        long first = 0;
+        if (doDeltaCompression) {
+            for (int i = blockSize - 1; i > 0; --i) {
+                in[i] -= in[i - 1];
+            }
+            // Avoid setting in[0] to 0 in case there is a minimum interval between
+            // consecutive values. This might later help compress data using fewer
+            // bits per value.
+            first = in[0] - in[1];
+            in[0] = in[1];
+            token = (token << 1) | 0x01;
+        } else {
+            token <<= 1;
+        }
+        removeOffset(token, tokenBits + 1, in, out);
+        if (doDeltaCompression) {
+            out.writeZLong(first);
+        }
+    }
+
+    private void removeOffset(int token, int tokenBits, long[] in, DataOutput out) throws IOException {
+        long min = Long.MAX_VALUE;
+        long max = Long.MIN_VALUE;
+        for (long l : in) {
+            min = Math.min(l, min);
+            max = Math.max(l, max);
+        }
+
+        if (max - min < 0) {
+            // overflow
+            min = 0;
+        } else if (min > 0 && min < (max >>> 2)) {
+            // removing the offset is unlikely going to help save bits per value, yet it makes decoding
+            // slower
+            min = 0;
+        }
+
+        if (min != 0) {
+            for (int i = 0; i < blockSize; ++i) {
+                in[i] -= min;
+            }
+            token = (token << 1) | 0x01;
+        } else {
+            token <<= 1;
+        }
+
+        gcdEncode(token, tokenBits + 1, in, out);
+        if (min != 0) {
+            out.writeZLong(min);
+        }
+    }
+
+    /**
+     * See if numbers have a common divisor. This is typically helpful for integer values in
+     * floats/doubles or dates that don't have millisecond accuracy.
+     */
+    private void gcdEncode(int token, int tokenBits, long[] in, DataOutput out) throws IOException {
+        long gcd = 0;
+        for (long l : in) {
+            gcd = MathUtil.gcd(gcd, l);
+            if (gcd == 1) {
+                break;
+            }
+        }
+        final boolean doGcdCompression = Long.compareUnsigned(gcd, 1) > 0;
+        if (doGcdCompression) {
+            for (int i = 0; i < blockSize; ++i) {
+                in[i] /= gcd;
+            }
+            token = (token << 1) | 0x01;
+        } else {
+            token <<= 1;
+        }
+
+        forEncode(token, tokenBits + 1, in, out);
+        if (doGcdCompression) {
+            out.writeVLong(gcd - 2);
+        }
+    }
+
+    private void forEncode(int token, int tokenBits, long[] in, DataOutput out) throws IOException {
+        long or = 0;
+        for (long l : in) {
+            or |= l;
+        }
+
+        final int bitsPerValue = or == 0 ? 0 : PackedInts.unsignedBitsRequired(or);
+
+        out.writeVInt((bitsPerValue << tokenBits) | token);
+        if (bitsPerValue > 0) {
+            forUtil.encode(in, bitsPerValue, out);
+        }
+    }
+
+    /**
+     * Encode the given longs using a combination of delta-coding, GCD factorization and bit packing.
+     */
+    void encode(long[] in, DataOutput out) throws IOException {
+        assert in.length == blockSize;
+
+        deltaEncode(0, 0, in, out);
+    }
+
+    /** Decode longs that have been encoded with {@link #encode}. */
+    void decode(DataInput in, long[] out) throws IOException {
+        assert out.length == blockSize : out.length;
+
+        final int token = in.readVInt();
+        final int bitsPerValue = token >>> 3;
+
+        if (bitsPerValue != 0) {
+            forUtil.decode(bitsPerValue, in, out);
+        } else {
+            Arrays.fill(out, 0L);
+        }
+
+        // simple blocks that only perform bit packing exit early here
+        // this is typical for SORTED(_SET) ordinals
+        if ((token & 0x07) != 0) {
+
+            final boolean doGcdCompression = (token & 0x01) != 0;
+            if (doGcdCompression) {
+                final long gcd = 2 + in.readVLong();
+                mul(out, gcd);
+            }
+
+            final boolean hasOffset = (token & 0x02) != 0;
+            if (hasOffset) {
+                final long min = in.readZLong();
+                add(out, min);
+            }
+
+            final boolean doDeltaCompression = (token & 0x04) != 0;
+            if (doDeltaCompression) {
+                final long first = in.readZLong();
+                out[0] += first;
+                deltaDecode(out);
+            }
+        }
+    }
+
+    // this loop should auto-vectorize
+    private void mul(long[] arr, long m) {
+        for (int i = 0; i < blockSize; ++i) {
+            arr[i] *= m;
+        }
+    }
+
+    // this loop should auto-vectorize
+    private void add(long[] arr, long min) {
+        for (int i = 0; i < blockSize; ++i) {
+            arr[i] += min;
+        }
+    }
+
+    private void deltaDecode(long[] arr) {
+        for (int i = 1; i < blockSize; ++i) {
+            arr[i] += arr[i - 1];
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormat.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormat.java
new file mode 100644
index 000000000000..77e98489f79f
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormat.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.codec.tsdb;
+
+import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+
+import java.io.IOException;
+
+public class ES87TSDBDocValuesFormat extends org.apache.lucene.codecs.DocValuesFormat {
+
+    static final int DEFAULT_NUMERIC_BLOCK_SHIFT = 7;
+    static final int DEFAULT_NUMERIC_BLOCK_SIZE = 1 << DEFAULT_NUMERIC_BLOCK_SHIFT;
+    static final int DEFAULT_DIRECT_MONOTONIC_BLOCK_SHIFT = 16;
+    static final String CODEC_NAME = "ES87TSDB";
+    static final String DATA_CODEC = "ES87TSDBDocValuesData";
+    static final String DATA_EXTENSION = "dvd";
+    static final String META_CODEC = "ES87TSDBDocValuesMetadata";
+    static final String META_EXTENSION = "dvm";
+    static final int VERSION_START = 0;
+    static final int VERSION_CURRENT = VERSION_START;
+    static final byte NUMERIC = 0;
+    static final byte BINARY = 1;
+    static final byte SORTED = 2;
+    static final byte SORTED_SET = 3;
+    static final byte SORTED_NUMERIC = 4;
+
+    private final int numericBlockShift;
+    private final int numericBlockSize;
+    private final int numericBlockMask;
+    private final int directMonotonicBlockShift;
+
+    public ES87TSDBDocValuesFormat() {
+        this(DEFAULT_NUMERIC_BLOCK_SHIFT, DEFAULT_DIRECT_MONOTONIC_BLOCK_SHIFT);
+    }
+
+    public ES87TSDBDocValuesFormat(int numericBlockShift, int directMonotonicBlockShift) {
+        super(CODEC_NAME);
+        this.numericBlockShift = numericBlockShift;
+        this.numericBlockSize = 1 << numericBlockShift;
+        this.numericBlockMask = numericBlockSize - 1;
+        this.directMonotonicBlockShift = directMonotonicBlockShift;
+    }
+
+    @Override
+    public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+        return new ES87TSDBDocValuesConsumer(
+            state,
+            DATA_CODEC,
+            DATA_EXTENSION,
+            META_CODEC,
+            META_EXTENSION,
+            numericBlockShift,
+            numericBlockSize,
+            directMonotonicBlockShift
+        );
+    }
+
+    @Override
+    public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException {
+        return new ES87TSDBDocValuesProducer(
+            state,
+            DATA_CODEC,
+            DATA_EXTENSION,
+            META_CODEC,
+            META_EXTENSION,
+            numericBlockShift,
+            numericBlockSize,
+            numericBlockMask
+        );
+    }
+
+    public int getNumericBlockShift() {
+        return numericBlockShift;
+    }
+
+    public int getNumericBlockSize() {
+        return numericBlockSize;
+    }
+
+    public int getNumericBlockMask() {
+        return numericBlockMask;
+    }
+
+    public int getDirectMonotonicBlockShift() {
+        return directMonotonicBlockShift;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java
new file mode 100644
index 000000000000..5a6991850333
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java
@@ -0,0 +1,527 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.codec.tsdb;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.codecs.lucene90.IndexedDISI;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.RandomAccessInput;
+import org.apache.lucene.util.LongValues;
+import org.apache.lucene.util.packed.DirectMonotonicReader;
+import org.elasticsearch.core.IOUtils;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class ES87TSDBDocValuesProducer extends DocValuesProducer {
+    private final Map<String, NumericEntry> numerics = new HashMap<>();
+    private final Map<String, SortedNumericEntry> sortedNumerics = new HashMap<>();
+    private final IndexInput data;
+    private final int maxDoc;
+    private final int numericBlockShift;
+    private final int numericBlockSize;
+    private final int numericBlockMask;
+
+    ES87TSDBDocValuesProducer(
+        SegmentReadState state,
+        String dataCodec,
+        String dataExtension,
+        String metaCodec,
+        String metaExtension,
+        int numericBlockShift,
+        int numericBlockSize,
+        int numericBlockMask
+    ) throws IOException {
+        this.numericBlockShift = numericBlockShift;
+        this.numericBlockSize = numericBlockSize;
+        this.numericBlockMask = numericBlockMask;
+        String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
+        this.maxDoc = state.segmentInfo.maxDoc();
+
+        // read in the entries from the metadata file.
+        int version = -1;
+        try (ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context)) {
+            Throwable priorE = null;
+
+            try {
+                version = CodecUtil.checkIndexHeader(
+                    in,
+                    metaCodec,
+                    ES87TSDBDocValuesFormat.VERSION_START,
+                    ES87TSDBDocValuesFormat.VERSION_CURRENT,
+                    state.segmentInfo.getId(),
+                    state.segmentSuffix
+                );
+
+                readFields(in, state.fieldInfos);
+
+            } catch (Throwable exception) {
+                priorE = exception;
+            } finally {
+                CodecUtil.checkFooter(in, priorE);
+            }
+        }
+
+        String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
+        this.data = state.directory.openInput(dataName, state.context);
+        boolean success = false;
+        try {
+            final int version2 = CodecUtil.checkIndexHeader(
+                data,
+                dataCodec,
+                ES87TSDBDocValuesFormat.VERSION_START,
+                ES87TSDBDocValuesFormat.VERSION_CURRENT,
+                state.segmentInfo.getId(),
+                state.segmentSuffix
+            );
+            if (version != version2) {
+                throw new CorruptIndexException("Format versions mismatch: meta=" + version + ", data=" + version2, data);
+            }
+
+            // NOTE: data file is too costly to verify checksum against all the bytes on open,
+            // but for now we at least verify proper structure of the checksum footer: which looks
+            // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
+            // such as file truncation.
+            CodecUtil.retrieveChecksum(data);
+
+            success = true;
+        } finally {
+            if (success == false) {
+                IOUtils.closeWhileHandlingException(this.data);
+            }
+        }
+    }
+
+    @Override
+    public NumericDocValues getNumeric(FieldInfo field) throws IOException {
+        NumericEntry entry = numerics.get(field.name);
+        return getNumeric(entry);
+    }
+
+    @Override
+    public BinaryDocValues getBinary(FieldInfo field) throws IOException {
+        throw new UnsupportedOperationException("Unsupported binary doc values for field [" + field.name + "]");
+    }
+
+    @Override
+    public SortedDocValues getSorted(FieldInfo field) throws IOException {
+        throw new UnsupportedOperationException("Unsupported sorted doc values for field [" + field.name + "]");
+    }
+
+    @Override
+    public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
+        SortedNumericEntry entry = sortedNumerics.get(field.name);
+        return getSortedNumeric(entry);
+    }
+
+    @Override
+    public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
+        throw new UnsupportedOperationException("Unsupported sorted set doc values for field [" + field.name + "]");
+    }
+
+    @Override
+    public void checkIntegrity() throws IOException {
+        CodecUtil.checksumEntireFile(data);
+    }
+
+    @Override
+    public void close() throws IOException {
+        data.close();
+    }
+
+    private void readFields(IndexInput meta, FieldInfos infos) throws IOException {
+        for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) {
+            FieldInfo info = infos.fieldInfo(fieldNumber);
+            if (info == null) {
+                throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
+            }
+            byte type = meta.readByte();
+            if (type == ES87TSDBDocValuesFormat.NUMERIC) {
+                numerics.put(info.name, readNumeric(meta));
+            } else if (type == ES87TSDBDocValuesFormat.BINARY) {
+                throw new CorruptIndexException("unsupported type: " + type, meta);
+            } else if (type == ES87TSDBDocValuesFormat.SORTED) {
+                throw new CorruptIndexException("unsupported type: " + type, meta);
+            } else if (type == ES87TSDBDocValuesFormat.SORTED_SET) {
+                throw new CorruptIndexException("unsupported type: " + type, meta);
+            } else if (type == ES87TSDBDocValuesFormat.SORTED_NUMERIC) {
+                sortedNumerics.put(info.name, readSortedNumeric(meta));
+            } else {
+                throw new CorruptIndexException("invalid type: " + type, meta);
+            }
+        }
+    }
+
+    private NumericEntry readNumeric(IndexInput meta) throws IOException {
+        NumericEntry entry = new NumericEntry();
+        readNumeric(meta, entry);
+        return entry;
+    }
+
+    private void readNumeric(IndexInput meta, NumericEntry entry) throws IOException {
+        entry.docsWithFieldOffset = meta.readLong();
+        entry.docsWithFieldLength = meta.readLong();
+        entry.jumpTableEntryCount = meta.readShort();
+        entry.denseRankPower = meta.readByte();
+        entry.numValues = meta.readLong();
+        if (entry.numValues > 0) {
+            final int indexBlockShift = meta.readInt();
+            entry.indexMeta = DirectMonotonicReader.loadMeta(meta, 1 + ((entry.numValues - 1) >>> numericBlockShift), indexBlockShift);
+            entry.indexOffset = meta.readLong();
+            entry.indexLength = meta.readLong();
+            entry.valuesOffset = meta.readLong();
+            entry.valuesLength = meta.readLong();
+        }
+    }
+
+    private SortedNumericEntry readSortedNumeric(IndexInput meta) throws IOException {
+        SortedNumericEntry entry = new SortedNumericEntry();
+        readSortedNumeric(meta, entry);
+        return entry;
+    }
+
+    private SortedNumericEntry readSortedNumeric(IndexInput meta, SortedNumericEntry entry) throws IOException {
+        readNumeric(meta, entry);
+        entry.numDocsWithField = meta.readInt();
+        if (entry.numDocsWithField != entry.numValues) {
+            entry.addressesOffset = meta.readLong();
+            final int blockShift = meta.readVInt();
+            entry.addressesMeta = DirectMonotonicReader.loadMeta(meta, entry.numDocsWithField + 1, blockShift);
+            entry.addressesLength = meta.readLong();
+        }
+        return entry;
+    }
+
+    private abstract static class NumericValues {
+        abstract long advance(long index) throws IOException;
+    }
+
+    private NumericDocValues getNumeric(NumericEntry entry) throws IOException {
+        if (entry.docsWithFieldOffset == -2) {
+            // empty
+            return DocValues.emptyNumeric();
+        }
+
+        // NOTE: we could make this a bit simpler by reusing #getValues but this
+        // makes things slower.
+
+        final RandomAccessInput indexSlice = data.randomAccessSlice(entry.indexOffset, entry.indexLength);
+        final DirectMonotonicReader indexReader = DirectMonotonicReader.getInstance(entry.indexMeta, indexSlice);
+        final IndexInput valuesData = data.slice("values", entry.valuesOffset, entry.valuesLength);
+
+        if (entry.docsWithFieldOffset == -1) {
+            // dense
+            return new NumericDocValues() {
+
+                private final int maxDoc = ES87TSDBDocValuesProducer.this.maxDoc;
+                private int doc = -1;
+                private final ES87TSDBDocValuesEncoder decoder = new ES87TSDBDocValuesEncoder(numericBlockSize);
+                private long currentBlockIndex = -1;
+                private final long[] currentBlock = new long[numericBlockSize];
+
+                @Override
+                public int docID() {
+                    return doc;
+                }
+
+                @Override
+                public int nextDoc() throws IOException {
+                    return advance(doc + 1);
+                }
+
+                @Override
+                public int advance(int target) throws IOException {
+                    if (target >= maxDoc) {
+                        return doc = NO_MORE_DOCS;
+                    }
+                    return doc = target;
+                }
+
+                @Override
+                public boolean advanceExact(int target) {
+                    doc = target;
+                    return true;
+                }
+
+                @Override
+                public long cost() {
+                    return maxDoc;
+                }
+
+                @Override
+                public long longValue() throws IOException {
+                    final int index = doc;
+                    final int blockIndex = index >>> numericBlockShift;
+                    final int blockInIndex = index & numericBlockMask;
+                    if (blockIndex != currentBlockIndex) {
+                        assert blockIndex > currentBlockIndex;
+                        if (blockIndex - 1 > currentBlockIndex) {
+                            valuesData.seek(indexReader.get(blockIndex));
+                        }
+                        currentBlockIndex = blockIndex;
+                        decoder.decode(valuesData, currentBlock);
+                    }
+                    return currentBlock[blockInIndex];
+                }
+            };
+        } else {
+            final IndexedDISI disi = new IndexedDISI(
+                data,
+                entry.docsWithFieldOffset,
+                entry.docsWithFieldLength,
+                entry.jumpTableEntryCount,
+                entry.denseRankPower,
+                entry.numValues
+            );
+            return new NumericDocValues() {
+
+                private final ES87TSDBDocValuesEncoder decoder = new ES87TSDBDocValuesEncoder(numericBlockSize);
+                private long currentBlockIndex = -1;
+                private final long[] currentBlock = new long[numericBlockSize];
+
+                @Override
+                public int advance(int target) throws IOException {
+                    return disi.advance(target);
+                }
+
+                @Override
+                public boolean advanceExact(int target) throws IOException {
+                    return disi.advanceExact(target);
+                }
+
+                @Override
+                public int nextDoc() throws IOException {
+                    return disi.nextDoc();
+                }
+
+                @Override
+                public int docID() {
+                    return disi.docID();
+                }
+
+                @Override
+                public long cost() {
+                    return disi.cost();
+                }
+
+                @Override
+                public long longValue() throws IOException {
+                    final int index = disi.index();
+                    final int blockIndex = index >>> numericBlockShift;
+                    final int blockInIndex = index & numericBlockMask;
+                    if (blockIndex != currentBlockIndex) {
+                        assert blockIndex > currentBlockIndex;
+                        if (blockIndex - 1 > currentBlockIndex) {
+                            valuesData.seek(indexReader.get(blockIndex));
+                        }
+                        currentBlockIndex = blockIndex;
+                        decoder.decode(valuesData, currentBlock);
+                    }
+                    return currentBlock[blockInIndex];
+                }
+            };
+        }
+    }
+
+    private NumericValues getValues(NumericEntry entry) throws IOException {
+        assert entry.numValues > 0;
+        final RandomAccessInput indexSlice = data.randomAccessSlice(entry.indexOffset, entry.indexLength);
+        final DirectMonotonicReader indexReader = DirectMonotonicReader.getInstance(entry.indexMeta, indexSlice);
+
+        final IndexInput valuesData = data.slice("values", entry.valuesOffset, entry.valuesLength);
+        return new NumericValues() {
+
+            private final ES87TSDBDocValuesEncoder decoder = new ES87TSDBDocValuesEncoder();
+            private long currentBlockIndex = -1;
+            private final long[] currentBlock = new long[numericBlockSize];
+
+            @Override
+            long advance(long index) throws IOException {
+                final long blockIndex = index >>> numericBlockShift;
+                final int blockInIndex = (int) (index & numericBlockMask);
+                if (blockIndex != currentBlockIndex) {
+                    assert blockIndex > currentBlockIndex;
+                    if (blockIndex - 1 > currentBlockIndex) {
+                        valuesData.seek(indexReader.get(blockIndex));
+                    }
+                    currentBlockIndex = blockIndex;
+                    decoder.decode(valuesData, currentBlock);
+                }
+                return currentBlock[blockInIndex];
+            }
+        };
+    }
+
+    private SortedNumericDocValues getSortedNumeric(SortedNumericEntry entry) throws IOException {
+        if (entry.numValues == entry.numDocsWithField) {
+            return DocValues.singleton(getNumeric(entry));
+        }
+
+        final RandomAccessInput addressesInput = data.randomAccessSlice(entry.addressesOffset, entry.addressesLength);
+        final LongValues addresses = DirectMonotonicReader.getInstance(entry.addressesMeta, addressesInput);
+
+        final NumericValues values = getValues(entry);
+
+        if (entry.docsWithFieldOffset == -1) {
+            // dense
+            return new SortedNumericDocValues() {
+
+                int doc = -1;
+                long start, end;
+                int count;
+
+                @Override
+                public int nextDoc() throws IOException {
+                    return advance(doc + 1);
+                }
+
+                @Override
+                public int docID() {
+                    return doc;
+                }
+
+                @Override
+                public long cost() {
+                    return maxDoc;
+                }
+
+                @Override
+                public int advance(int target) throws IOException {
+                    if (target >= maxDoc) {
+                        return doc = NO_MORE_DOCS;
+                    }
+                    start = addresses.get(target);
+                    end = addresses.get(target + 1L);
+                    count = (int) (end - start);
+                    return doc = target;
+                }
+
+                @Override
+                public boolean advanceExact(int target) throws IOException {
+                    start = addresses.get(target);
+                    end = addresses.get(target + 1L);
+                    count = (int) (end - start);
+                    doc = target;
+                    return true;
+                }
+
+                @Override
+                public long nextValue() throws IOException {
+                    return values.advance(start++);
+                }
+
+                @Override
+                public int docValueCount() {
+                    return count;
+                }
+            };
+        } else {
+            // sparse
+            final IndexedDISI disi = new IndexedDISI(
+                data,
+                entry.docsWithFieldOffset,
+                entry.docsWithFieldLength,
+                entry.jumpTableEntryCount,
+                entry.denseRankPower,
+                entry.numDocsWithField
+            );
+            return new SortedNumericDocValues() {
+
+                boolean set;
+                long start, end;
+                int count;
+
+                @Override
+                public int nextDoc() throws IOException {
+                    set = false;
+                    return disi.nextDoc();
+                }
+
+                @Override
+                public int docID() {
+                    return disi.docID();
+                }
+
+                @Override
+                public long cost() {
+                    return disi.cost();
+                }
+
+                @Override
+                public int advance(int target) throws IOException {
+                    set = false;
+                    return disi.advance(target);
+                }
+
+                @Override
+                public boolean advanceExact(int target) throws IOException {
+                    set = false;
+                    return disi.advanceExact(target);
+                }
+
+                @Override
+                public long nextValue() throws IOException {
+                    set();
+                    return values.advance(start++);
+                }
+
+                @Override
+                public int docValueCount() {
+                    set();
+                    return count;
+                }
+
+                private void set() {
+                    if (set == false) {
+                        final int index = disi.index();
+                        start = addresses.get(index);
+                        end = addresses.get(index + 1L);
+                        count = (int) (end - start);
+                        set = true;
+                    }
+                }
+            };
+        }
+    }
+
+    private static class NumericEntry {
+        long docsWithFieldOffset;
+        long docsWithFieldLength;
+        short jumpTableEntryCount;
+        byte denseRankPower;
+        long numValues;
+        long indexOffset;
+        long indexLength;
+        DirectMonotonicReader.Meta indexMeta;
+        long valuesOffset;
+        long valuesLength;
+    }
+
+    private static class SortedNumericEntry extends NumericEntry {
+        int numDocsWithField;
+        DirectMonotonicReader.Meta addressesMeta;
+        long addressesOffset;
+        long addressesLength;
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ForUtil.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ForUtil.java
new file mode 100644
index 000000000000..2f20e0c8cee2
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ForUtil.java
@@ -0,0 +1,1047 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.codec.tsdb;
+
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+
+import java.io.IOException;
+
+// Inspired from https://fulmicoton.com/posts/bitpacking/
+// Encodes multiple integers in a long to get SIMD-like speedups.
+// If bitsPerValue <= 8 then we pack 8 ints per long
+// else if bitsPerValue <= 16 we pack 4 ints per long
+// else we pack 2 ints per long
+public final class ForUtil {
+
+    static final int DEFAULT_BLOCK_SIZE = 128;
+    private final int blockSize;
+    private final int blockSizeLog2;
+    private final long[] tmp;
+
+    public ForUtil() {
+        this(DEFAULT_BLOCK_SIZE);
+    }
+
+    public ForUtil(int blockSize) {
+        this.blockSize = blockSize;
+        this.blockSizeLog2 = (int) (Math.log(blockSize) / Math.log(2));
+        this.tmp = new long[blockSize / 2];
+    }
+
+    private static long expandMask32(long mask32) {
+        return mask32 | (mask32 << 32);
+    }
+
+    private static long expandMask16(long mask16) {
+        return expandMask32(mask16 | (mask16 << 16));
+    }
+
+    private static long expandMask8(long mask8) {
+        return expandMask16(mask8 | (mask8 << 8));
+    }
+
+    private static long mask32(int bitsPerValue) {
+        return expandMask32((1L << bitsPerValue) - 1);
+    }
+
+    private static long mask16(int bitsPerValue) {
+        return expandMask16((1L << bitsPerValue) - 1);
+    }
+
+    private static long mask8(int bitsPerValue) {
+        return expandMask8((1L << bitsPerValue) - 1);
+    }
+
+    private static void expand8(long[] arr) {
+        for (int i = 0; i < 16; ++i) {
+            long l = arr[i];
+            arr[i] = (l >>> 56) & 0xFFL;
+            arr[16 + i] = (l >>> 48) & 0xFFL;
+            arr[32 + i] = (l >>> 40) & 0xFFL;
+            arr[48 + i] = (l >>> 32) & 0xFFL;
+            arr[64 + i] = (l >>> 24) & 0xFFL;
+            arr[80 + i] = (l >>> 16) & 0xFFL;
+            arr[96 + i] = (l >>> 8) & 0xFFL;
+            arr[112 + i] = l & 0xFFL;
+        }
+    }
+
+    private static void expand8To32(long[] arr) {
+        for (int i = 0; i < 16; ++i) {
+            long l = arr[i];
+            arr[i] = (l >>> 24) & 0x000000FF000000FFL;
+            arr[16 + i] = (l >>> 16) & 0x000000FF000000FFL;
+            arr[32 + i] = (l >>> 8) & 0x000000FF000000FFL;
+            arr[48 + i] = l & 0x000000FF000000FFL;
+        }
+    }
+
+    private static void collapse8(long[] arr) {
+        for (int i = 0; i < 16; ++i) {
+            arr[i] = (arr[i] << 56) | (arr[16 + i] << 48) | (arr[32 + i] << 40) | (arr[48 + i] << 32) | (arr[64 + i] << 24) | (arr[80 + i]
+                << 16) | (arr[96 + i] << 8) | arr[112 + i];
+        }
+    }
+
+    private static void expand16(long[] arr) {
+        for (int i = 0; i < 32; ++i) {
+            long l = arr[i];
+            arr[i] = (l >>> 48) & 0xFFFFL;
+            arr[32 + i] = (l >>> 32) & 0xFFFFL;
+            arr[64 + i] = (l >>> 16) & 0xFFFFL;
+            arr[96 + i] = l & 0xFFFFL;
+        }
+    }
+
+    private static void expand16To32(long[] arr) {
+        for (int i = 0; i < 32; ++i) {
+            long l = arr[i];
+            arr[i] = (l >>> 16) & 0x0000FFFF0000FFFFL;
+            arr[32 + i] = l & 0x0000FFFF0000FFFFL;
+        }
+    }
+
+    private static void collapse16(long[] arr) {
+        for (int i = 0; i < 32; ++i) {
+            arr[i] = (arr[i] << 48) | (arr[32 + i] << 32) | (arr[64 + i] << 16) | arr[96 + i];
+        }
+    }
+
+    private static void expand32(long[] arr) {
+        for (int i = 0; i < 64; ++i) {
+            long l = arr[i];
+            arr[i] = l >>> 32;
+            arr[64 + i] = l & 0xFFFFFFFFL;
+        }
+    }
+
+    private static void collapse32(long[] arr) {
+        for (int i = 0; i < 64; ++i) {
+            arr[i] = (arr[i] << 32) | arr[64 + i];
+        }
+    }
+
+    /** Encode 128 integers from {@code longs} into {@code out}. */
+    void encode(long[] longs, int bitsPerValue, DataOutput out) throws IOException {
+        final int nextPrimitive;
+        final int numLongs;
+        if (bitsPerValue <= 8) {
+            nextPrimitive = 8;
+            numLongs = blockSize / 8;
+            collapse8(longs);
+        } else if (bitsPerValue <= 16) {
+            nextPrimitive = 16;
+            numLongs = blockSize / 4;
+            collapse16(longs);
+        } else {
+            nextPrimitive = 32;
+            numLongs = blockSize / 2;
+            collapse32(longs);
+        }
+
+        final int numLongsPerShift = bitsPerValue * 2;
+        int idx = 0;
+        int shift = nextPrimitive - bitsPerValue;
+        for (int i = 0; i < numLongsPerShift; ++i) {
+            tmp[i] = longs[idx++] << shift;
+        }
+        for (shift = shift - bitsPerValue; shift >= 0; shift -= bitsPerValue) {
+            for (int i = 0; i < numLongsPerShift; ++i) {
+                tmp[i] |= longs[idx++] << shift;
+            }
+        }
+
+        final int remainingBitsPerLong = shift + bitsPerValue;
+        final long maskRemainingBitsPerLong;
+        if (nextPrimitive == 8) {
+            maskRemainingBitsPerLong = MASKS8[remainingBitsPerLong];
+        } else if (nextPrimitive == 16) {
+            maskRemainingBitsPerLong = MASKS16[remainingBitsPerLong];
+        } else {
+            maskRemainingBitsPerLong = MASKS32[remainingBitsPerLong];
+        }
+
+        int tmpIdx = 0;
+        int remainingBitsPerValue = bitsPerValue;
+        while (idx < numLongs) {
+            if (remainingBitsPerValue >= remainingBitsPerLong) {
+                remainingBitsPerValue -= remainingBitsPerLong;
+                tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & maskRemainingBitsPerLong;
+                if (remainingBitsPerValue == 0) {
+                    idx++;
+                    remainingBitsPerValue = bitsPerValue;
+                }
+            } else {
+                final long mask1, mask2;
+                if (nextPrimitive == 8) {
+                    mask1 = MASKS8[remainingBitsPerValue];
+                    mask2 = MASKS8[remainingBitsPerLong - remainingBitsPerValue];
+                } else if (nextPrimitive == 16) {
+                    mask1 = MASKS16[remainingBitsPerValue];
+                    mask2 = MASKS16[remainingBitsPerLong - remainingBitsPerValue];
+                } else {
+                    mask1 = MASKS32[remainingBitsPerValue];
+                    mask2 = MASKS32[remainingBitsPerLong - remainingBitsPerValue];
+                }
+                tmp[tmpIdx] |= (longs[idx++] & mask1) << (remainingBitsPerLong - remainingBitsPerValue);
+                remainingBitsPerValue = bitsPerValue - remainingBitsPerLong + remainingBitsPerValue;
+                tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & mask2;
+            }
+        }
+
+        for (int i = 0; i < numLongsPerShift; ++i) {
+            out.writeLong(tmp[i]);
+        }
+    }
+
+    /** Number of bytes required to encode 128 integers of {@code bitsPerValue} bits per value. */
+    int numBytes(int bitsPerValue) {
+        return bitsPerValue << (blockSizeLog2 - 3);
+    }
+
+    private static void decodeSlow(int blockSize, int bitsPerValue, DataInput in, long[] tmp, long[] longs) throws IOException {
+        final int numLongs = bitsPerValue << 1;
+        in.readLongs(tmp, 0, numLongs);
+        final long mask = MASKS32[bitsPerValue];
+        int longsIdx = 0;
+        int shift = 32 - bitsPerValue;
+        for (; shift >= 0; shift -= bitsPerValue) {
+            shiftLongs(tmp, numLongs, longs, longsIdx, shift, mask);
+            longsIdx += numLongs;
+        }
+        final int remainingBitsPerLong = shift + bitsPerValue;
+        final long mask32RemainingBitsPerLong = MASKS32[remainingBitsPerLong];
+        int tmpIdx = 0;
+        int remainingBits = remainingBitsPerLong;
+        for (; longsIdx < blockSize / 2; ++longsIdx) {
+            int b = bitsPerValue - remainingBits;
+            long l = (tmp[tmpIdx++] & MASKS32[remainingBits]) << b;
+            while (b >= remainingBitsPerLong) {
+                b -= remainingBitsPerLong;
+                l |= (tmp[tmpIdx++] & mask32RemainingBitsPerLong) << b;
+            }
+            if (b > 0) {
+                l |= (tmp[tmpIdx] >>> (remainingBitsPerLong - b)) & MASKS32[b];
+                remainingBits = remainingBitsPerLong - b;
+            } else {
+                remainingBits = remainingBitsPerLong;
+            }
+            longs[longsIdx] = l;
+        }
+    }
+
+    /**
+     * The pattern that this shiftLongs method applies is recognized by the C2 compiler, which
+     * generates SIMD instructions for it in order to shift multiple longs at once.
+     */
+    private static void shiftLongs(long[] a, int count, long[] b, int bi, int shift, long mask) {
+        for (int i = 0; i < count; ++i) {
+            b[bi + i] = (a[i] >>> shift) & mask;
+        }
+    }
+
+    private static final long[] MASKS8 = new long[8];
+    private static final long[] MASKS16 = new long[16];
+    private static final long[] MASKS32 = new long[32];
+
+    static {
+        for (int i = 0; i < 8; ++i) {
+            MASKS8[i] = mask8(i);
+        }
+        for (int i = 0; i < 16; ++i) {
+            MASKS16[i] = mask16(i);
+        }
+        for (int i = 0; i < 32; ++i) {
+            MASKS32[i] = mask32(i);
+        }
+    }
+    // mark values in array as final longs to avoid the cost of reading array, arrays should only be
+    // used when the idx is a variable
+    private static final long MASK8_1 = MASKS8[1];
+    private static final long MASK8_2 = MASKS8[2];
+    private static final long MASK8_3 = MASKS8[3];
+    private static final long MASK8_4 = MASKS8[4];
+    private static final long MASK8_5 = MASKS8[5];
+    private static final long MASK8_6 = MASKS8[6];
+    private static final long MASK8_7 = MASKS8[7];
+    private static final long MASK16_1 = MASKS16[1];
+    private static final long MASK16_2 = MASKS16[2];
+    private static final long MASK16_3 = MASKS16[3];
+    private static final long MASK16_4 = MASKS16[4];
+    private static final long MASK16_5 = MASKS16[5];
+    private static final long MASK16_6 = MASKS16[6];
+    private static final long MASK16_7 = MASKS16[7];
+    private static final long MASK16_9 = MASKS16[9];
+    private static final long MASK16_10 = MASKS16[10];
+    private static final long MASK16_11 = MASKS16[11];
+    private static final long MASK16_12 = MASKS16[12];
+    private static final long MASK16_13 = MASKS16[13];
+    private static final long MASK16_14 = MASKS16[14];
+    private static final long MASK16_15 = MASKS16[15];
+    private static final long MASK32_1 = MASKS32[1];
+    private static final long MASK32_2 = MASKS32[2];
+    private static final long MASK32_3 = MASKS32[3];
+    private static final long MASK32_4 = MASKS32[4];
+    private static final long MASK32_5 = MASKS32[5];
+    private static final long MASK32_6 = MASKS32[6];
+    private static final long MASK32_7 = MASKS32[7];
+    private static final long MASK32_8 = MASKS32[8];
+    private static final long MASK32_9 = MASKS32[9];
+    private static final long MASK32_10 = MASKS32[10];
+    private static final long MASK32_11 = MASKS32[11];
+    private static final long MASK32_12 = MASKS32[12];
+    private static final long MASK32_13 = MASKS32[13];
+    private static final long MASK32_14 = MASKS32[14];
+    private static final long MASK32_15 = MASKS32[15];
+    private static final long MASK32_17 = MASKS32[17];
+    private static final long MASK32_18 = MASKS32[18];
+    private static final long MASK32_19 = MASKS32[19];
+    private static final long MASK32_20 = MASKS32[20];
+    private static final long MASK32_21 = MASKS32[21];
+    private static final long MASK32_22 = MASKS32[22];
+    private static final long MASK32_23 = MASKS32[23];
+    private static final long MASK32_24 = MASKS32[24];
+
+    /** Decode 128 integers into {@code longs}. */
+    void decode(int bitsPerValue, DataInput in, long[] longs) throws IOException {
+        switch (bitsPerValue) {
+            case 1:
+                decode1(in, tmp, longs);
+                expand8(longs);
+                break;
+            case 2:
+                decode2(in, tmp, longs);
+                expand8(longs);
+                break;
+            case 3:
+                decode3(in, tmp, longs);
+                expand8(longs);
+                break;
+            case 4:
+                decode4(in, tmp, longs);
+                expand8(longs);
+                break;
+            case 5:
+                decode5(in, tmp, longs);
+                expand8(longs);
+                break;
+            case 6:
+                decode6(in, tmp, longs);
+                expand8(longs);
+                break;
+            case 7:
+                decode7(in, tmp, longs);
+                expand8(longs);
+                break;
+            case 8:
+                decode8(in, tmp, longs);
+                expand8(longs);
+                break;
+            case 9:
+                decode9(in, tmp, longs);
+                expand16(longs);
+                break;
+            case 10:
+                decode10(in, tmp, longs);
+                expand16(longs);
+                break;
+            case 11:
+                decode11(in, tmp, longs);
+                expand16(longs);
+                break;
+            case 12:
+                decode12(in, tmp, longs);
+                expand16(longs);
+                break;
+            case 13:
+                decode13(in, tmp, longs);
+                expand16(longs);
+                break;
+            case 14:
+                decode14(in, tmp, longs);
+                expand16(longs);
+                break;
+            case 15:
+                decode15(in, tmp, longs);
+                expand16(longs);
+                break;
+            case 16:
+                decode16(in, tmp, longs);
+                expand16(longs);
+                break;
+            case 17:
+                decode17(in, tmp, longs);
+                expand32(longs);
+                break;
+            case 18:
+                decode18(in, tmp, longs);
+                expand32(longs);
+                break;
+            case 19:
+                decode19(in, tmp, longs);
+                expand32(longs);
+                break;
+            case 20:
+                decode20(in, tmp, longs);
+                expand32(longs);
+                break;
+            case 21:
+                decode21(in, tmp, longs);
+                expand32(longs);
+                break;
+            case 22:
+                decode22(in, tmp, longs);
+                expand32(longs);
+                break;
+            case 23:
+                decode23(in, tmp, longs);
+                expand32(longs);
+                break;
+            case 24:
+                decode24(in, tmp, longs);
+                expand32(longs);
+                break;
+            default:
+                decodeSlow(blockSize, bitsPerValue, in, tmp, longs);
+                expand32(longs);
+                break;
+        }
+    }
+
+    /**
+     * Decodes 128 integers into 64 {@code longs} such that each long contains two values, each
+     * represented with 32 bits. Values [0..63] are encoded in the high-order bits of {@code longs}
+     * [0..63], and values [64..127] are encoded in the low-order bits of {@code longs} [0..63]. This
+     * representation may allow subsequent operations to be performed on two values at a time.
+     */
+    void decodeTo32(int bitsPerValue, DataInput in, long[] longs) throws IOException {
+        switch (bitsPerValue) {
+            case 1:
+                decode1(in, tmp, longs);
+                expand8To32(longs);
+                break;
+            case 2:
+                decode2(in, tmp, longs);
+                expand8To32(longs);
+                break;
+            case 3:
+                decode3(in, tmp, longs);
+                expand8To32(longs);
+                break;
+            case 4:
+                decode4(in, tmp, longs);
+                expand8To32(longs);
+                break;
+            case 5:
+                decode5(in, tmp, longs);
+                expand8To32(longs);
+                break;
+            case 6:
+                decode6(in, tmp, longs);
+                expand8To32(longs);
+                break;
+            case 7:
+                decode7(in, tmp, longs);
+                expand8To32(longs);
+                break;
+            case 8:
+                decode8(in, tmp, longs);
+                expand8To32(longs);
+                break;
+            case 9:
+                decode9(in, tmp, longs);
+                expand16To32(longs);
+                break;
+            case 10:
+                decode10(in, tmp, longs);
+                expand16To32(longs);
+                break;
+            case 11:
+                decode11(in, tmp, longs);
+                expand16To32(longs);
+                break;
+            case 12:
+                decode12(in, tmp, longs);
+                expand16To32(longs);
+                break;
+            case 13:
+                decode13(in, tmp, longs);
+                expand16To32(longs);
+                break;
+            case 14:
+                decode14(in, tmp, longs);
+                expand16To32(longs);
+                break;
+            case 15:
+                decode15(in, tmp, longs);
+                expand16To32(longs);
+                break;
+            case 16:
+                decode16(in, tmp, longs);
+                expand16To32(longs);
+                break;
+            case 17:
+                decode17(in, tmp, longs);
+                break;
+            case 18:
+                decode18(in, tmp, longs);
+                break;
+            case 19:
+                decode19(in, tmp, longs);
+                break;
+            case 20:
+                decode20(in, tmp, longs);
+                break;
+            case 21:
+                decode21(in, tmp, longs);
+                break;
+            case 22:
+                decode22(in, tmp, longs);
+                break;
+            case 23:
+                decode23(in, tmp, longs);
+                break;
+            case 24:
+                decode24(in, tmp, longs);
+                break;
+            default:
+                decodeSlow(blockSize, bitsPerValue, in, tmp, longs);
+                break;
+        }
+    }
+
+    private static void decode1(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 2);
+        shiftLongs(tmp, 2, longs, 0, 7, MASK8_1);
+        shiftLongs(tmp, 2, longs, 2, 6, MASK8_1);
+        shiftLongs(tmp, 2, longs, 4, 5, MASK8_1);
+        shiftLongs(tmp, 2, longs, 6, 4, MASK8_1);
+        shiftLongs(tmp, 2, longs, 8, 3, MASK8_1);
+        shiftLongs(tmp, 2, longs, 10, 2, MASK8_1);
+        shiftLongs(tmp, 2, longs, 12, 1, MASK8_1);
+        shiftLongs(tmp, 2, longs, 14, 0, MASK8_1);
+    }
+
+    private static void decode2(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 4);
+        shiftLongs(tmp, 4, longs, 0, 6, MASK8_2);
+        shiftLongs(tmp, 4, longs, 4, 4, MASK8_2);
+        shiftLongs(tmp, 4, longs, 8, 2, MASK8_2);
+        shiftLongs(tmp, 4, longs, 12, 0, MASK8_2);
+    }
+
+    private static void decode3(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 6);
+        shiftLongs(tmp, 6, longs, 0, 5, MASK8_3);
+        shiftLongs(tmp, 6, longs, 6, 2, MASK8_3);
+        for (int iter = 0, tmpIdx = 0, longsIdx = 12; iter < 2; ++iter, tmpIdx += 3, longsIdx += 2) {
+            long l0 = (tmp[tmpIdx + 0] & MASK8_2) << 1;
+            l0 |= (tmp[tmpIdx + 1] >>> 1) & MASK8_1;
+            longs[longsIdx + 0] = l0;
+            long l1 = (tmp[tmpIdx + 1] & MASK8_1) << 2;
+            l1 |= (tmp[tmpIdx + 2] & MASK8_2) << 0;
+            longs[longsIdx + 1] = l1;
+        }
+    }
+
+    private static void decode4(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 8);
+        shiftLongs(tmp, 8, longs, 0, 4, MASK8_4);
+        shiftLongs(tmp, 8, longs, 8, 0, MASK8_4);
+    }
+
+    private static void decode5(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 10);
+        shiftLongs(tmp, 10, longs, 0, 3, MASK8_5);
+        for (int iter = 0, tmpIdx = 0, longsIdx = 10; iter < 2; ++iter, tmpIdx += 5, longsIdx += 3) {
+            long l0 = (tmp[tmpIdx + 0] & MASK8_3) << 2;
+            l0 |= (tmp[tmpIdx + 1] >>> 1) & MASK8_2;
+            longs[longsIdx + 0] = l0;
+            long l1 = (tmp[tmpIdx + 1] & MASK8_1) << 4;
+            l1 |= (tmp[tmpIdx + 2] & MASK8_3) << 1;
+            l1 |= (tmp[tmpIdx + 3] >>> 2) & MASK8_1;
+            longs[longsIdx + 1] = l1;
+            long l2 = (tmp[tmpIdx + 3] & MASK8_2) << 3;
+            l2 |= (tmp[tmpIdx + 4] & MASK8_3) << 0;
+            longs[longsIdx + 2] = l2;
+        }
+    }
+
+    private static void decode6(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 12);
+        shiftLongs(tmp, 12, longs, 0, 2, MASK8_6);
+        shiftLongs(tmp, 12, tmp, 0, 0, MASK8_2);
+        for (int iter = 0, tmpIdx = 0, longsIdx = 12; iter < 4; ++iter, tmpIdx += 3, longsIdx += 1) {
+            long l0 = tmp[tmpIdx + 0] << 4;
+            l0 |= tmp[tmpIdx + 1] << 2;
+            l0 |= tmp[tmpIdx + 2] << 0;
+            longs[longsIdx + 0] = l0;
+        }
+    }
+
+    private static void decode7(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 14);
+        shiftLongs(tmp, 14, longs, 0, 1, MASK8_7);
+        shiftLongs(tmp, 14, tmp, 0, 0, MASK8_1);
+        for (int iter = 0, tmpIdx = 0, longsIdx = 14; iter < 2; ++iter, tmpIdx += 7, longsIdx += 1) {
+            long l0 = tmp[tmpIdx + 0] << 6;
+            l0 |= tmp[tmpIdx + 1] << 5;
+            l0 |= tmp[tmpIdx + 2] << 4;
+            l0 |= tmp[tmpIdx + 3] << 3;
+            l0 |= tmp[tmpIdx + 4] << 2;
+            l0 |= tmp[tmpIdx + 5] << 1;
+            l0 |= tmp[tmpIdx + 6] << 0;
+            longs[longsIdx + 0] = l0;
+        }
+    }
+
+    private static void decode8(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(longs, 0, 16);
+    }
+
+    private static void decode9(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 18);
+        shiftLongs(tmp, 18, longs, 0, 7, MASK16_9);
+        for (int iter = 0, tmpIdx = 0, longsIdx = 18; iter < 2; ++iter, tmpIdx += 9, longsIdx += 7) {
+            long l0 = (tmp[tmpIdx + 0] & MASK16_7) << 2;
+            l0 |= (tmp[tmpIdx + 1] >>> 5) & MASK16_2;
+            longs[longsIdx + 0] = l0;
+            long l1 = (tmp[tmpIdx + 1] & MASK16_5) << 4;
+            l1 |= (tmp[tmpIdx + 2] >>> 3) & MASK16_4;
+            longs[longsIdx + 1] = l1;
+            long l2 = (tmp[tmpIdx + 2] & MASK16_3) << 6;
+            l2 |= (tmp[tmpIdx + 3] >>> 1) & MASK16_6;
+            longs[longsIdx + 2] = l2;
+            long l3 = (tmp[tmpIdx + 3] & MASK16_1) << 8;
+            l3 |= (tmp[tmpIdx + 4] & MASK16_7) << 1;
+            l3 |= (tmp[tmpIdx + 5] >>> 6) & MASK16_1;
+            longs[longsIdx + 3] = l3;
+            long l4 = (tmp[tmpIdx + 5] & MASK16_6) << 3;
+            l4 |= (tmp[tmpIdx + 6] >>> 4) & MASK16_3;
+            longs[longsIdx + 4] = l4;
+            long l5 = (tmp[tmpIdx + 6] & MASK16_4) << 5;
+            l5 |= (tmp[tmpIdx + 7] >>> 2) & MASK16_5;
+            longs[longsIdx + 5] = l5;
+            long l6 = (tmp[tmpIdx + 7] & MASK16_2) << 7;
+            l6 |= (tmp[tmpIdx + 8] & MASK16_7) << 0;
+            longs[longsIdx + 6] = l6;
+        }
+    }
+
+    private static void decode10(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 20);
+        shiftLongs(tmp, 20, longs, 0, 6, MASK16_10);
+        for (int iter = 0, tmpIdx = 0, longsIdx = 20; iter < 4; ++iter, tmpIdx += 5, longsIdx += 3) {
+            long l0 = (tmp[tmpIdx + 0] & MASK16_6) << 4;
+            l0 |= (tmp[tmpIdx + 1] >>> 2) & MASK16_4;
+            longs[longsIdx + 0] = l0;
+            long l1 = (tmp[tmpIdx + 1] & MASK16_2) << 8;
+            l1 |= (tmp[tmpIdx + 2] & MASK16_6) << 2;
+            l1 |= (tmp[tmpIdx + 3] >>> 4) & MASK16_2;
+            longs[longsIdx + 1] = l1;
+            long l2 = (tmp[tmpIdx + 3] & MASK16_4) << 6;
+            l2 |= (tmp[tmpIdx + 4] & MASK16_6) << 0;
+            longs[longsIdx + 2] = l2;
+        }
+    }
+
+    private static void decode11(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 22);
+        shiftLongs(tmp, 22, longs, 0, 5, MASK16_11);
+        for (int iter = 0, tmpIdx = 0, longsIdx = 22; iter < 2; ++iter, tmpIdx += 11, longsIdx += 5) {
+            long l0 = (tmp[tmpIdx + 0] & MASK16_5) << 6;
+            l0 |= (tmp[tmpIdx + 1] & MASK16_5) << 1;
+            l0 |= (tmp[tmpIdx + 2] >>> 4) & MASK16_1;
+            longs[longsIdx + 0] = l0;
+            long l1 = (tmp[tmpIdx + 2] & MASK16_4) << 7;
+            l1 |= (tmp[tmpIdx + 3] & MASK16_5) << 2;
+            l1 |= (tmp[tmpIdx + 4] >>> 3) & MASK16_2;
+            longs[longsIdx + 1] = l1;
+            long l2 = (tmp[tmpIdx + 4] & MASK16_3) << 8;
+            l2 |= (tmp[tmpIdx + 5] & MASK16_5) << 3;
+            l2 |= (tmp[tmpIdx + 6] >>> 2) & MASK16_3;
+            longs[longsIdx + 2] = l2;
+            long l3 = (tmp[tmpIdx + 6] & MASK16_2) << 9;
+            l3 |= (tmp[tmpIdx + 7] & MASK16_5) << 4;
+            l3 |= (tmp[tmpIdx + 8] >>> 1) & MASK16_4;
+            longs[longsIdx + 3] = l3;
+            long l4 = (tmp[tmpIdx + 8] & MASK16_1) << 10;
+            l4 |= (tmp[tmpIdx + 9] & MASK16_5) << 5;
+            l4 |= (tmp[tmpIdx + 10] & MASK16_5) << 0;
+            longs[longsIdx + 4] = l4;
+        }
+    }
+
+    private static void decode12(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 24);
+        shiftLongs(tmp, 24, longs, 0, 4, MASK16_12);
+        shiftLongs(tmp, 24, tmp, 0, 0, MASK16_4);
+        for (int iter = 0, tmpIdx = 0, longsIdx = 24; iter < 8; ++iter, tmpIdx += 3, longsIdx += 1) {
+            long l0 = tmp[tmpIdx + 0] << 8;
+            l0 |= tmp[tmpIdx + 1] << 4;
+            l0 |= tmp[tmpIdx + 2] << 0;
+            longs[longsIdx + 0] = l0;
+        }
+    }
+
+    private static void decode13(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 26);
+        shiftLongs(tmp, 26, longs, 0, 3, MASK16_13);
+        for (int iter = 0, tmpIdx = 0, longsIdx = 26; iter < 2; ++iter, tmpIdx += 13, longsIdx += 3) {
+            long l0 = (tmp[tmpIdx + 0] & MASK16_3) << 10;
+            l0 |= (tmp[tmpIdx + 1] & MASK16_3) << 7;
+            l0 |= (tmp[tmpIdx + 2] & MASK16_3) << 4;
+            l0 |= (tmp[tmpIdx + 3] & MASK16_3) << 1;
+            l0 |= (tmp[tmpIdx + 4] >>> 2) & MASK16_1;
+            longs[longsIdx + 0] = l0;
+            long l1 = (tmp[tmpIdx + 4] & MASK16_2) << 11;
+            l1 |= (tmp[tmpIdx + 5] & MASK16_3) << 8;
+            l1 |= (tmp[tmpIdx + 6] & MASK16_3) << 5;
+            l1 |= (tmp[tmpIdx + 7] & MASK16_3) << 2;
+            l1 |= (tmp[tmpIdx + 8] >>> 1) & MASK16_2;
+            longs[longsIdx + 1] = l1;
+            long l2 = (tmp[tmpIdx + 8] & MASK16_1) << 12;
+            l2 |= (tmp[tmpIdx + 9] & MASK16_3) << 9;
+            l2 |= (tmp[tmpIdx + 10] & MASK16_3) << 6;
+            l2 |= (tmp[tmpIdx + 11] & MASK16_3) << 3;
+            l2 |= (tmp[tmpIdx + 12] & MASK16_3) << 0;
+            longs[longsIdx + 2] = l2;
+        }
+    }
+
+    private static void decode14(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 28);
+        shiftLongs(tmp, 28, longs, 0, 2, MASK16_14);
+        shiftLongs(tmp, 28, tmp, 0, 0, MASK16_2);
+        for (int iter = 0, tmpIdx = 0, longsIdx = 28; iter < 4; ++iter, tmpIdx += 7, longsIdx += 1) {
+            long l0 = tmp[tmpIdx + 0] << 12;
+            l0 |= tmp[tmpIdx + 1] << 10;
+            l0 |= tmp[tmpIdx + 2] << 8;
+            l0 |= tmp[tmpIdx + 3] << 6;
+            l0 |= tmp[tmpIdx + 4] << 4;
+            l0 |= tmp[tmpIdx + 5] << 2;
+            l0 |= tmp[tmpIdx + 6] << 0;
+            longs[longsIdx + 0] = l0;
+        }
+    }
+
+    private static void decode15(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 30);
+        shiftLongs(tmp, 30, longs, 0, 1, MASK16_15);
+        shiftLongs(tmp, 30, tmp, 0, 0, MASK16_1);
+        for (int iter = 0, tmpIdx = 0, longsIdx = 30; iter < 2; ++iter, tmpIdx += 15, longsIdx += 1) {
+            long l0 = tmp[tmpIdx + 0] << 14;
+            l0 |= tmp[tmpIdx + 1] << 13;
+            l0 |= tmp[tmpIdx + 2] << 12;
+            l0 |= tmp[tmpIdx + 3] << 11;
+            l0 |= tmp[tmpIdx + 4] << 10;
+            l0 |= tmp[tmpIdx + 5] << 9;
+            l0 |= tmp[tmpIdx + 6] << 8;
+            l0 |= tmp[tmpIdx + 7] << 7;
+            l0 |= tmp[tmpIdx + 8] << 6;
+            l0 |= tmp[tmpIdx + 9] << 5;
+            l0 |= tmp[tmpIdx + 10] << 4;
+            l0 |= tmp[tmpIdx + 11] << 3;
+            l0 |= tmp[tmpIdx + 12] << 2;
+            l0 |= tmp[tmpIdx + 13] << 1;
+            l0 |= tmp[tmpIdx + 14] << 0;
+            longs[longsIdx + 0] = l0;
+        }
+    }
+
+    private static void decode16(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(longs, 0, 32);
+    }
+
+    private static void decode17(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 34);
+        shiftLongs(tmp, 34, longs, 0, 15, MASK32_17);
+        for (int iter = 0, tmpIdx = 0, longsIdx = 34; iter < 2; ++iter, tmpIdx += 17, longsIdx += 15) {
+            long l0 = (tmp[tmpIdx + 0] & MASK32_15) << 2;
+            l0 |= (tmp[tmpIdx + 1] >>> 13) & MASK32_2;
+            longs[longsIdx + 0] = l0;
+            long l1 = (tmp[tmpIdx + 1] & MASK32_13) << 4;
+            l1 |= (tmp[tmpIdx + 2] >>> 11) & MASK32_4;
+            longs[longsIdx + 1] = l1;
+            long l2 = (tmp[tmpIdx + 2] & MASK32_11) << 6;
+            l2 |= (tmp[tmpIdx + 3] >>> 9) & MASK32_6;
+            longs[longsIdx + 2] = l2;
+            long l3 = (tmp[tmpIdx + 3] & MASK32_9) << 8;
+            l3 |= (tmp[tmpIdx + 4] >>> 7) & MASK32_8;
+            longs[longsIdx + 3] = l3;
+            long l4 = (tmp[tmpIdx + 4] & MASK32_7) << 10;
+            l4 |= (tmp[tmpIdx + 5] >>> 5) & MASK32_10;
+            longs[longsIdx + 4] = l4;
+            long l5 = (tmp[tmpIdx + 5] & MASK32_5) << 12;
+            l5 |= (tmp[tmpIdx + 6] >>> 3) & MASK32_12;
+            longs[longsIdx + 5] = l5;
+            long l6 = (tmp[tmpIdx + 6] & MASK32_3) << 14;
+            l6 |= (tmp[tmpIdx + 7] >>> 1) & MASK32_14;
+            longs[longsIdx + 6] = l6;
+            long l7 = (tmp[tmpIdx + 7] & MASK32_1) << 16;
+            l7 |= (tmp[tmpIdx + 8] & MASK32_15) << 1;
+            l7 |= (tmp[tmpIdx + 9] >>> 14) & MASK32_1;
+            longs[longsIdx + 7] = l7;
+            long l8 = (tmp[tmpIdx + 9] & MASK32_14) << 3;
+            l8 |= (tmp[tmpIdx + 10] >>> 12) & MASK32_3;
+            longs[longsIdx + 8] = l8;
+            long l9 = (tmp[tmpIdx + 10] & MASK32_12) << 5;
+            l9 |= (tmp[tmpIdx + 11] >>> 10) & MASK32_5;
+            longs[longsIdx + 9] = l9;
+            long l10 = (tmp[tmpIdx + 11] & MASK32_10) << 7;
+            l10 |= (tmp[tmpIdx + 12] >>> 8) & MASK32_7;
+            longs[longsIdx + 10] = l10;
+            long l11 = (tmp[tmpIdx + 12] & MASK32_8) << 9;
+            l11 |= (tmp[tmpIdx + 13] >>> 6) & MASK32_9;
+            longs[longsIdx + 11] = l11;
+            long l12 = (tmp[tmpIdx + 13] & MASK32_6) << 11;
+            l12 |= (tmp[tmpIdx + 14] >>> 4) & MASK32_11;
+            longs[longsIdx + 12] = l12;
+            long l13 = (tmp[tmpIdx + 14] & MASK32_4) << 13;
+            l13 |= (tmp[tmpIdx + 15] >>> 2) & MASK32_13;
+            longs[longsIdx + 13] = l13;
+            long l14 = (tmp[tmpIdx + 15] & MASK32_2) << 15;
+            l14 |= (tmp[tmpIdx + 16] & MASK32_15) << 0;
+            longs[longsIdx + 14] = l14;
+        }
+    }
+
+    private static void decode18(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 36);
+        shiftLongs(tmp, 36, longs, 0, 14, MASK32_18);
+        for (int iter = 0, tmpIdx = 0, longsIdx = 36; iter < 4; ++iter, tmpIdx += 9, longsIdx += 7) {
+            long l0 = (tmp[tmpIdx + 0] & MASK32_14) << 4;
+            l0 |= (tmp[tmpIdx + 1] >>> 10) & MASK32_4;
+            longs[longsIdx + 0] = l0;
+            long l1 = (tmp[tmpIdx + 1] & MASK32_10) << 8;
+            l1 |= (tmp[tmpIdx + 2] >>> 6) & MASK32_8;
+            longs[longsIdx + 1] = l1;
+            long l2 = (tmp[tmpIdx + 2] & MASK32_6) << 12;
+            l2 |= (tmp[tmpIdx + 3] >>> 2) & MASK32_12;
+            longs[longsIdx + 2] = l2;
+            long l3 = (tmp[tmpIdx + 3] & MASK32_2) << 16;
+            l3 |= (tmp[tmpIdx + 4] & MASK32_14) << 2;
+            l3 |= (tmp[tmpIdx + 5] >>> 12) & MASK32_2;
+            longs[longsIdx + 3] = l3;
+            long l4 = (tmp[tmpIdx + 5] & MASK32_12) << 6;
+            l4 |= (tmp[tmpIdx + 6] >>> 8) & MASK32_6;
+            longs[longsIdx + 4] = l4;
+            long l5 = (tmp[tmpIdx + 6] & MASK32_8) << 10;
+            l5 |= (tmp[tmpIdx + 7] >>> 4) & MASK32_10;
+            longs[longsIdx + 5] = l5;
+            long l6 = (tmp[tmpIdx + 7] & MASK32_4) << 14;
+            l6 |= (tmp[tmpIdx + 8] & MASK32_14) << 0;
+            longs[longsIdx + 6] = l6;
+        }
+    }
+
+    private static void decode19(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 38);
+        shiftLongs(tmp, 38, longs, 0, 13, MASK32_19);
+        for (int iter = 0, tmpIdx = 0, longsIdx = 38; iter < 2; ++iter, tmpIdx += 19, longsIdx += 13) {
+            long l0 = (tmp[tmpIdx + 0] & MASK32_13) << 6;
+            l0 |= (tmp[tmpIdx + 1] >>> 7) & MASK32_6;
+            longs[longsIdx + 0] = l0;
+            long l1 = (tmp[tmpIdx + 1] & MASK32_7) << 12;
+            l1 |= (tmp[tmpIdx + 2] >>> 1) & MASK32_12;
+            longs[longsIdx + 1] = l1;
+            long l2 = (tmp[tmpIdx + 2] & MASK32_1) << 18;
+            l2 |= (tmp[tmpIdx + 3] & MASK32_13) << 5;
+            l2 |= (tmp[tmpIdx + 4] >>> 8) & MASK32_5;
+            longs[longsIdx + 2] = l2;
+            long l3 = (tmp[tmpIdx + 4] & MASK32_8) << 11;
+            l3 |= (tmp[tmpIdx + 5] >>> 2) & MASK32_11;
+            longs[longsIdx + 3] = l3;
+            long l4 = (tmp[tmpIdx + 5] & MASK32_2) << 17;
+            l4 |= (tmp[tmpIdx + 6] & MASK32_13) << 4;
+            l4 |= (tmp[tmpIdx + 7] >>> 9) & MASK32_4;
+            longs[longsIdx + 4] = l4;
+            long l5 = (tmp[tmpIdx + 7] & MASK32_9) << 10;
+            l5 |= (tmp[tmpIdx + 8] >>> 3) & MASK32_10;
+            longs[longsIdx + 5] = l5;
+            long l6 = (tmp[tmpIdx + 8] & MASK32_3) << 16;
+            l6 |= (tmp[tmpIdx + 9] & MASK32_13) << 3;
+            l6 |= (tmp[tmpIdx + 10] >>> 10) & MASK32_3;
+            longs[longsIdx + 6] = l6;
+            long l7 = (tmp[tmpIdx + 10] & MASK32_10) << 9;
+            l7 |= (tmp[tmpIdx + 11] >>> 4) & MASK32_9;
+            longs[longsIdx + 7] = l7;
+            long l8 = (tmp[tmpIdx + 11] & MASK32_4) << 15;
+            l8 |= (tmp[tmpIdx + 12] & MASK32_13) << 2;
+            l8 |= (tmp[tmpIdx + 13] >>> 11) & MASK32_2;
+            longs[longsIdx + 8] = l8;
+            long l9 = (tmp[tmpIdx + 13] & MASK32_11) << 8;
+            l9 |= (tmp[tmpIdx + 14] >>> 5) & MASK32_8;
+            longs[longsIdx + 9] = l9;
+            long l10 = (tmp[tmpIdx + 14] & MASK32_5) << 14;
+            l10 |= (tmp[tmpIdx + 15] & MASK32_13) << 1;
+            l10 |= (tmp[tmpIdx + 16] >>> 12) & MASK32_1;
+            longs[longsIdx + 10] = l10;
+            long l11 = (tmp[tmpIdx + 16] & MASK32_12) << 7;
+            l11 |= (tmp[tmpIdx + 17] >>> 6) & MASK32_7;
+            longs[longsIdx + 11] = l11;
+            long l12 = (tmp[tmpIdx + 17] & MASK32_6) << 13;
+            l12 |= (tmp[tmpIdx + 18] & MASK32_13) << 0;
+            longs[longsIdx + 12] = l12;
+        }
+    }
+
+    private static void decode20(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 40);
+        shiftLongs(tmp, 40, longs, 0, 12, MASK32_20);
+        for (int iter = 0, tmpIdx = 0, longsIdx = 40; iter < 8; ++iter, tmpIdx += 5, longsIdx += 3) {
+            long l0 = (tmp[tmpIdx + 0] & MASK32_12) << 8;
+            l0 |= (tmp[tmpIdx + 1] >>> 4) & MASK32_8;
+            longs[longsIdx + 0] = l0;
+            long l1 = (tmp[tmpIdx + 1] & MASK32_4) << 16;
+            l1 |= (tmp[tmpIdx + 2] & MASK32_12) << 4;
+            l1 |= (tmp[tmpIdx + 3] >>> 8) & MASK32_4;
+            longs[longsIdx + 1] = l1;
+            long l2 = (tmp[tmpIdx + 3] & MASK32_8) << 12;
+            l2 |= (tmp[tmpIdx + 4] & MASK32_12) << 0;
+            longs[longsIdx + 2] = l2;
+        }
+    }
+
+    private static void decode21(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 42);
+        shiftLongs(tmp, 42, longs, 0, 11, MASK32_21);
+        for (int iter = 0, tmpIdx = 0, longsIdx = 42; iter < 2; ++iter, tmpIdx += 21, longsIdx += 11) {
+            long l0 = (tmp[tmpIdx + 0] & MASK32_11) << 10;
+            l0 |= (tmp[tmpIdx + 1] >>> 1) & MASK32_10;
+            longs[longsIdx + 0] = l0;
+            long l1 = (tmp[tmpIdx + 1] & MASK32_1) << 20;
+            l1 |= (tmp[tmpIdx + 2] & MASK32_11) << 9;
+            l1 |= (tmp[tmpIdx + 3] >>> 2) & MASK32_9;
+            longs[longsIdx + 1] = l1;
+            long l2 = (tmp[tmpIdx + 3] & MASK32_2) << 19;
+            l2 |= (tmp[tmpIdx + 4] & MASK32_11) << 8;
+            l2 |= (tmp[tmpIdx + 5] >>> 3) & MASK32_8;
+            longs[longsIdx + 2] = l2;
+            long l3 = (tmp[tmpIdx + 5] & MASK32_3) << 18;
+            l3 |= (tmp[tmpIdx + 6] & MASK32_11) << 7;
+            l3 |= (tmp[tmpIdx + 7] >>> 4) & MASK32_7;
+            longs[longsIdx + 3] = l3;
+            long l4 = (tmp[tmpIdx + 7] & MASK32_4) << 17;
+            l4 |= (tmp[tmpIdx + 8] & MASK32_11) << 6;
+            l4 |= (tmp[tmpIdx + 9] >>> 5) & MASK32_6;
+            longs[longsIdx + 4] = l4;
+            long l5 = (tmp[tmpIdx + 9] & MASK32_5) << 16;
+            l5 |= (tmp[tmpIdx + 10] & MASK32_11) << 5;
+            l5 |= (tmp[tmpIdx + 11] >>> 6) & MASK32_5;
+            longs[longsIdx + 5] = l5;
+            long l6 = (tmp[tmpIdx + 11] & MASK32_6) << 15;
+            l6 |= (tmp[tmpIdx + 12] & MASK32_11) << 4;
+            l6 |= (tmp[tmpIdx + 13] >>> 7) & MASK32_4;
+            longs[longsIdx + 6] = l6;
+            long l7 = (tmp[tmpIdx + 13] & MASK32_7) << 14;
+            l7 |= (tmp[tmpIdx + 14] & MASK32_11) << 3;
+            l7 |= (tmp[tmpIdx + 15] >>> 8) & MASK32_3;
+            longs[longsIdx + 7] = l7;
+            long l8 = (tmp[tmpIdx + 15] & MASK32_8) << 13;
+            l8 |= (tmp[tmpIdx + 16] & MASK32_11) << 2;
+            l8 |= (tmp[tmpIdx + 17] >>> 9) & MASK32_2;
+            longs[longsIdx + 8] = l8;
+            long l9 = (tmp[tmpIdx + 17] & MASK32_9) << 12;
+            l9 |= (tmp[tmpIdx + 18] & MASK32_11) << 1;
+            l9 |= (tmp[tmpIdx + 19] >>> 10) & MASK32_1;
+            longs[longsIdx + 9] = l9;
+            long l10 = (tmp[tmpIdx + 19] & MASK32_10) << 11;
+            l10 |= (tmp[tmpIdx + 20] & MASK32_11) << 0;
+            longs[longsIdx + 10] = l10;
+        }
+    }
+
+    private static void decode22(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 44);
+        shiftLongs(tmp, 44, longs, 0, 10, MASK32_22);
+        for (int iter = 0, tmpIdx = 0, longsIdx = 44; iter < 4; ++iter, tmpIdx += 11, longsIdx += 5) {
+            long l0 = (tmp[tmpIdx + 0] & MASK32_10) << 12;
+            l0 |= (tmp[tmpIdx + 1] & MASK32_10) << 2;
+            l0 |= (tmp[tmpIdx + 2] >>> 8) & MASK32_2;
+            longs[longsIdx + 0] = l0;
+            long l1 = (tmp[tmpIdx + 2] & MASK32_8) << 14;
+            l1 |= (tmp[tmpIdx + 3] & MASK32_10) << 4;
+            l1 |= (tmp[tmpIdx + 4] >>> 6) & MASK32_4;
+            longs[longsIdx + 1] = l1;
+            long l2 = (tmp[tmpIdx + 4] & MASK32_6) << 16;
+            l2 |= (tmp[tmpIdx + 5] & MASK32_10) << 6;
+            l2 |= (tmp[tmpIdx + 6] >>> 4) & MASK32_6;
+            longs[longsIdx + 2] = l2;
+            long l3 = (tmp[tmpIdx + 6] & MASK32_4) << 18;
+            l3 |= (tmp[tmpIdx + 7] & MASK32_10) << 8;
+            l3 |= (tmp[tmpIdx + 8] >>> 2) & MASK32_8;
+            longs[longsIdx + 3] = l3;
+            long l4 = (tmp[tmpIdx + 8] & MASK32_2) << 20;
+            l4 |= (tmp[tmpIdx + 9] & MASK32_10) << 10;
+            l4 |= (tmp[tmpIdx + 10] & MASK32_10) << 0;
+            longs[longsIdx + 4] = l4;
+        }
+    }
+
+    private static void decode23(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 46);
+        shiftLongs(tmp, 46, longs, 0, 9, MASK32_23);
+        for (int iter = 0, tmpIdx = 0, longsIdx = 46; iter < 2; ++iter, tmpIdx += 23, longsIdx += 9) {
+            long l0 = (tmp[tmpIdx + 0] & MASK32_9) << 14;
+            l0 |= (tmp[tmpIdx + 1] & MASK32_9) << 5;
+            l0 |= (tmp[tmpIdx + 2] >>> 4) & MASK32_5;
+            longs[longsIdx + 0] = l0;
+            long l1 = (tmp[tmpIdx + 2] & MASK32_4) << 19;
+            l1 |= (tmp[tmpIdx + 3] & MASK32_9) << 10;
+            l1 |= (tmp[tmpIdx + 4] & MASK32_9) << 1;
+            l1 |= (tmp[tmpIdx + 5] >>> 8) & MASK32_1;
+            longs[longsIdx + 1] = l1;
+            long l2 = (tmp[tmpIdx + 5] & MASK32_8) << 15;
+            l2 |= (tmp[tmpIdx + 6] & MASK32_9) << 6;
+            l2 |= (tmp[tmpIdx + 7] >>> 3) & MASK32_6;
+            longs[longsIdx + 2] = l2;
+            long l3 = (tmp[tmpIdx + 7] & MASK32_3) << 20;
+            l3 |= (tmp[tmpIdx + 8] & MASK32_9) << 11;
+            l3 |= (tmp[tmpIdx + 9] & MASK32_9) << 2;
+            l3 |= (tmp[tmpIdx + 10] >>> 7) & MASK32_2;
+            longs[longsIdx + 3] = l3;
+            long l4 = (tmp[tmpIdx + 10] & MASK32_7) << 16;
+            l4 |= (tmp[tmpIdx + 11] & MASK32_9) << 7;
+            l4 |= (tmp[tmpIdx + 12] >>> 2) & MASK32_7;
+            longs[longsIdx + 4] = l4;
+            long l5 = (tmp[tmpIdx + 12] & MASK32_2) << 21;
+            l5 |= (tmp[tmpIdx + 13] & MASK32_9) << 12;
+            l5 |= (tmp[tmpIdx + 14] & MASK32_9) << 3;
+            l5 |= (tmp[tmpIdx + 15] >>> 6) & MASK32_3;
+            longs[longsIdx + 5] = l5;
+            long l6 = (tmp[tmpIdx + 15] & MASK32_6) << 17;
+            l6 |= (tmp[tmpIdx + 16] & MASK32_9) << 8;
+            l6 |= (tmp[tmpIdx + 17] >>> 1) & MASK32_8;
+            longs[longsIdx + 6] = l6;
+            long l7 = (tmp[tmpIdx + 17] & MASK32_1) << 22;
+            l7 |= (tmp[tmpIdx + 18] & MASK32_9) << 13;
+            l7 |= (tmp[tmpIdx + 19] & MASK32_9) << 4;
+            l7 |= (tmp[tmpIdx + 20] >>> 5) & MASK32_4;
+            longs[longsIdx + 7] = l7;
+            long l8 = (tmp[tmpIdx + 20] & MASK32_5) << 18;
+            l8 |= (tmp[tmpIdx + 21] & MASK32_9) << 9;
+            l8 |= (tmp[tmpIdx + 22] & MASK32_9) << 0;
+            longs[longsIdx + 8] = l8;
+        }
+    }
+
+    private static void decode24(DataInput in, long[] tmp, long[] longs) throws IOException {
+        in.readLongs(tmp, 0, 48);
+        shiftLongs(tmp, 48, longs, 0, 8, MASK32_24);
+        shiftLongs(tmp, 48, tmp, 0, 0, MASK32_8);
+        for (int iter = 0, tmpIdx = 0, longsIdx = 48; iter < 16; ++iter, tmpIdx += 3, longsIdx += 1) {
+            long l0 = tmp[tmpIdx + 0] << 16;
+            l0 |= tmp[tmpIdx + 1] << 8;
+            l0 |= tmp[tmpIdx + 2] << 0;
+            longs[longsIdx + 0] = l0;
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java b/server/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java
index dfdbafe6eb2a..6f5716c88031 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java
@@ -13,17 +13,22 @@
 import org.apache.lucene.index.IndexDeletionPolicy;
 import org.apache.lucene.index.SegmentInfos;
 import org.elasticsearch.common.lucene.FilterIndexCommit;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogDeletionPolicy;
 
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.LongSupplier;
+import java.util.stream.Collectors;
 
 /**
  * An {@link IndexDeletionPolicy} that coordinates between Lucene's commits and the retention of translog generation files,
@@ -38,6 +43,17 @@ public class CombinedDeletionPolicy extends IndexDeletionPolicy {
     private final SoftDeletesPolicy softDeletesPolicy;
     private final LongSupplier globalCheckpointSupplier;
     private final Map<IndexCommit, Integer> snapshottedCommits; // Number of snapshots held against each commit point.
+
+    interface CommitsListener {
+
+        void onNewAcquiredCommit(IndexCommit commit, Set<String> additionalFiles);
+
+        void onDeletedCommit(IndexCommit commit);
+    }
+
+    @Nullable
+    private final CommitsListener commitsListener;
+
     private volatile IndexCommit safeCommit; // the most recent safe commit point - its max_seqno at most the persisted global checkpoint.
     private volatile long maxSeqNoOfNextSafeCommit;
     private volatile IndexCommit lastCommit; // the most recent commit point
@@ -47,12 +63,14 @@ public class CombinedDeletionPolicy extends IndexDeletionPolicy {
         Logger logger,
         TranslogDeletionPolicy translogDeletionPolicy,
         SoftDeletesPolicy softDeletesPolicy,
-        LongSupplier globalCheckpointSupplier
+        LongSupplier globalCheckpointSupplier,
+        @Nullable CommitsListener commitsListener
     ) {
         this.logger = logger;
         this.translogDeletionPolicy = translogDeletionPolicy;
         this.softDeletesPolicy = softDeletesPolicy;
         this.globalCheckpointSupplier = globalCheckpointSupplier;
+        this.commitsListener = commitsListener;
         this.snapshottedCommits = new HashMap<>();
     }
 
@@ -86,11 +104,15 @@ public void onCommit(List<? extends IndexCommit> commits) throws IOException {
             logger.info("failed to get the total docs from the safe commit; use the total docs from the previous safe commit", ex);
             totalDocsOfSafeCommit = safeCommitInfo.docCount;
         }
+        IndexCommit newCommit = null;
+        IndexCommit previousLastCommit = null;
+        List<IndexCommit> deletedCommits = null;
         synchronized (this) {
             this.safeCommitInfo = new SafeCommitInfo(
                 Long.parseLong(safeCommit.getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)),
                 totalDocsOfSafeCommit
             );
+            previousLastCommit = this.lastCommit;
             this.lastCommit = commits.get(commits.size() - 1);
             this.safeCommit = safeCommit;
             updateRetentionPolicy();
@@ -99,13 +121,32 @@ public void onCommit(List<? extends IndexCommit> commits) throws IOException {
             } else {
                 this.maxSeqNoOfNextSafeCommit = Long.parseLong(commits.get(keptPosition + 1).getUserData().get(SequenceNumbers.MAX_SEQ_NO));
             }
+            if (commitsListener != null && previousLastCommit != this.lastCommit) {
+                newCommit = acquireIndexCommit(false);
+            }
             for (int i = 0; i < keptPosition; i++) {
-                if (snapshottedCommits.containsKey(commits.get(i)) == false) {
-                    deleteCommit(commits.get(i));
+                final IndexCommit commit = commits.get(i);
+                if (snapshottedCommits.containsKey(commit) == false) {
+                    deleteCommit(commit);
+                    if (deletedCommits == null) {
+                        deletedCommits = new ArrayList<>();
+                    }
+                    deletedCommits.add(commit);
                 }
             }
         }
         assert assertSafeCommitUnchanged(safeCommit);
+        if (commitsListener != null) {
+            if (newCommit != null) {
+                final Set<String> additionalFiles = listOfNewFileNames(previousLastCommit, newCommit);
+                commitsListener.onNewAcquiredCommit(newCommit, additionalFiles);
+            }
+            if (deletedCommits != null) {
+                for (IndexCommit deletedCommit : deletedCommits) {
+                    commitsListener.onDeletedCommit(deletedCommit);
+                }
+            }
+        }
     }
 
     private boolean assertSafeCommitUnchanged(IndexCommit safeCommit) {
@@ -154,7 +195,11 @@ synchronized IndexCommit acquireIndexCommit(boolean acquiringSafeCommit) {
         assert lastCommit != null : "Last commit is not initialized yet";
         final IndexCommit snapshotting = acquiringSafeCommit ? safeCommit : lastCommit;
         snapshottedCommits.merge(snapshotting, 1, Integer::sum); // increase refCount
-        return new SnapshotIndexCommit(snapshotting);
+        return wrapCommit(snapshotting);
+    }
+
+    protected IndexCommit wrapCommit(IndexCommit indexCommit) {
+        return new SnapshotIndexCommit(indexCommit);
     }
 
     /**
@@ -226,6 +271,11 @@ private static int indexOfKeptCommits(List<? extends IndexCommit> commits, long
         return 0;
     }
 
+    private Set<String> listOfNewFileNames(IndexCommit previous, IndexCommit current) throws IOException {
+        final Set<String> previousFiles = previous != null ? new HashSet<>(previous.getFileNames()) : Set.of();
+        return current.getFileNames().stream().filter(f -> previousFiles.contains(f) == false).collect(Collectors.toUnmodifiableSet());
+    }
+
     /**
      * Checks whether the deletion policy is holding on to snapshotted commits
      */
diff --git a/server/src/main/java/org/elasticsearch/index/engine/CompletionStatsCache.java b/server/src/main/java/org/elasticsearch/index/engine/CompletionStatsCache.java
index 03096c88f0c9..f66b85647189 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/CompletionStatsCache.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/CompletionStatsCache.java
@@ -25,7 +25,7 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
 
-class CompletionStatsCache implements ReferenceManager.RefreshListener {
+public class CompletionStatsCache implements ReferenceManager.RefreshListener {
 
     private final Supplier<Engine.Searcher> searcherSupplier;
 
@@ -37,11 +37,11 @@ class CompletionStatsCache implements ReferenceManager.RefreshListener {
      */
     private final AtomicReference<PlainActionFuture<CompletionStats>> completionStatsFutureRef = new AtomicReference<>();
 
-    CompletionStatsCache(Supplier<Engine.Searcher> searcherSupplier) {
+    public CompletionStatsCache(Supplier<Engine.Searcher> searcherSupplier) {
         this.searcherSupplier = searcherSupplier;
     }
 
-    CompletionStats get(String... fieldNamePatterns) {
+    public CompletionStats get(String... fieldNamePatterns) {
         final PlainActionFuture<CompletionStats> newFuture = new PlainActionFuture<>();
         final PlainActionFuture<CompletionStats> oldFuture = completionStatsFutureRef.compareAndExchange(null, newFuture);
 
diff --git a/server/src/main/java/org/elasticsearch/index/engine/ElasticsearchReaderManager.java b/server/src/main/java/org/elasticsearch/index/engine/ElasticsearchReaderManager.java
index f5041ec69ebd..ef3b402a74c8 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/ElasticsearchReaderManager.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/ElasticsearchReaderManager.java
@@ -25,7 +25,7 @@
  *
  */
 @SuppressForbidden(reason = "reference counting is required here")
-class ElasticsearchReaderManager extends ReferenceManager<ElasticsearchDirectoryReader> {
+public class ElasticsearchReaderManager extends ReferenceManager<ElasticsearchDirectoryReader> {
 
     /**
      * Creates and returns a new ElasticsearchReaderManager from the given
@@ -34,7 +34,7 @@ class ElasticsearchReaderManager extends ReferenceManager<ElasticsearchDirectory
      *
      * @param reader            the directoryReader to use for future reopens
      */
-    ElasticsearchReaderManager(ElasticsearchDirectoryReader reader) {
+    public ElasticsearchReaderManager(ElasticsearchDirectoryReader reader) {
         this.current = reader;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
index c05428e4abb2..da72115d3850 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -26,6 +26,7 @@
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.logging.Loggers;
@@ -214,6 +215,33 @@ public void verifyEngineBeforeIndexClosing() throws IllegalStateException {
         }
     }
 
+    public interface IndexCommitListener {
+
+        /**
+         * This method is invoked each time a new Lucene commit is created through this engine. There is no guarantee that a listener will
+         * be notified of the commits in order, ie newer commits may appear before older ones. The {@link IndexCommitRef} prevents the
+         * {@link IndexCommitRef} files to be deleted from disk until the reference is closed. As such, the listener must close the
+         * reference as soon as it is done with it.
+         *
+         * @param shardId         the {@link ShardId} of shard
+         * @param store           the index shard store
+         * @param primaryTerm     the shard's primary term value
+         * @param indexCommitRef  a reference on the newly created index commit
+         * @param additionalFiles the set of filenames that are added by the new commit
+         */
+        void onNewCommit(ShardId shardId, Store store, long primaryTerm, IndexCommitRef indexCommitRef, Set<String> additionalFiles);
+
+        /**
+         * This method is invoked after the policy deleted the given {@link IndexCommit}. A listener is never notified of a deleted commit
+         * until the corresponding {@link Engine.IndexCommitRef} received through {@link #onNewCommit} has been closed; closing which in
+         * turn can call this method directly.
+         *
+         * @param shardId the {@link ShardId} of shard
+         * @param deletedCommit the deleted {@link IndexCommit}
+         */
+        void onIndexCommitDelete(ShardId shardId, IndexCommit deletedCommit);
+    }
+
     /**
      * A throttling class that can be activated, causing the
      * {@code acquireThrottle} method to block on a lock when throttling
@@ -541,7 +569,7 @@ public boolean isFound() {
 
     public static class NoOpResult extends Result {
 
-        NoOpResult(long term, long seqNo) {
+        public NoOpResult(long term, long seqNo) {
             super(Operation.TYPE.NO_OP, 0, term, seqNo, null);
         }
 
@@ -557,7 +585,7 @@ protected final GetResult getFromSearcher(Get get, Engine.Searcher searcher, boo
             if (uncachedLookup) {
                 docIdAndVersion = VersionsAndSeqNoResolver.loadDocIdAndVersionUncached(searcher.getIndexReader(), get.uid(), true);
             } else {
-                docIdAndVersion = VersionsAndSeqNoResolver.loadDocIdAndVersion(searcher.getIndexReader(), get.uid(), true);
+                docIdAndVersion = VersionsAndSeqNoResolver.timeSeriesLoadDocIdAndVersion(searcher.getIndexReader(), get.uid(), true);
             }
         } catch (Exception e) {
             Releasables.closeWhileHandlingException(searcher);
@@ -986,16 +1014,14 @@ public boolean refreshNeeded() {
      * changes.
      */
     @Nullable
-    public abstract void refresh(String source) throws EngineException;
+    public abstract RefreshResult refresh(String source) throws EngineException;
 
     /**
      * Synchronously refreshes the engine for new search operations to reflect the latest
      * changes unless another thread is already refreshing the engine concurrently.
-     *
-     * @return <code>true</code> if the a refresh happened. Otherwise <code>false</code>
      */
     @Nullable
-    public abstract boolean maybeRefresh(String source) throws EngineException;
+    public abstract RefreshResult maybeRefresh(String source) throws EngineException;
 
     /**
      * Called when our engine is using too much heap and should move buffered indexed/deleted documents to disk.
@@ -1929,4 +1955,27 @@ public interface TranslogRecoveryRunner {
     public final EngineConfig getEngineConfig() {
         return engineConfig;
     }
+
+    /**
+     * Allows registering a listener for when the index shard is on a segment generation >= minGeneration.
+     */
+    public void addSegmentGenerationListener(long minGeneration, ActionListener<Long> listener) {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Captures the result of a refresh operation on the index shard.
+     * <p>
+     * <code>refreshed</code> is true if a refresh happened. If refreshed, <code>generation</code>
+     * contains the generation of the index commit that the reader has opened upon refresh.
+     */
+    public record RefreshResult(boolean refreshed, long generation) {
+
+        public static final long UNKNOWN_GENERATION = -1L;
+        public static final RefreshResult NO_REFRESH = new RefreshResult(false);
+
+        public RefreshResult(boolean refreshed) {
+            this(refreshed, UNKNOWN_GENERATION);
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java b/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
index e8a2bf0aa58d..940ddf1373e5 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
@@ -125,6 +125,11 @@ public Supplier<RetentionLeases> retentionLeasesSupplier() {
 
     private final LongSupplier relativeTimeInNanosSupplier;
 
+    @Nullable
+    private final Engine.IndexCommitListener indexCommitListener;
+
+    private final boolean promotableToPrimary;
+
     /**
      * Creates a new {@link org.elasticsearch.index.engine.EngineConfig}
      */
@@ -152,7 +157,9 @@ public EngineConfig(
         LongSupplier primaryTermSupplier,
         IndexStorePlugin.SnapshotCommitSupplier snapshotCommitSupplier,
         Comparator<LeafReader> leafSorter,
-        LongSupplier relativeTimeInNanosSupplier
+        LongSupplier relativeTimeInNanosSupplier,
+        Engine.IndexCommitListener indexCommitListener,
+        boolean promotableToPrimary
     ) {
         this.shardId = shardId;
         this.indexSettings = indexSettings;
@@ -193,6 +200,8 @@ public EngineConfig(
         this.snapshotCommitSupplier = snapshotCommitSupplier;
         this.leafSorter = leafSorter;
         this.relativeTimeInNanosSupplier = relativeTimeInNanosSupplier;
+        this.indexCommitListener = indexCommitListener;
+        this.promotableToPrimary = promotableToPrimary;
     }
 
     /**
@@ -236,6 +245,13 @@ public Codec getCodec() {
         return codecService.codec(codecName);
     }
 
+    /**
+     * @return the {@link CodecService}
+     */
+    public CodecService getCodecService() {
+        return codecService;
+    }
+
     /**
      * Returns a thread-pool mainly used to get estimated time stamps from
      * {@link org.elasticsearch.threadpool.ThreadPool#relativeTimeInMillis()} and to schedule
@@ -395,4 +411,16 @@ public Comparator<LeafReader> getLeafSorter() {
     public LongSupplier getRelativeTimeInNanosSupplier() {
         return relativeTimeInNanosSupplier;
     }
+
+    @Nullable
+    public Engine.IndexCommitListener getIndexCommitListener() {
+        return indexCommitListener;
+    }
+
+    /**
+     * @return whether the engine should be configured so that it can be promoted to primary in future
+     */
+    public boolean isPromotableToPrimary() {
+        return promotableToPrimary;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/engine/IdStoredFieldLoader.java b/server/src/main/java/org/elasticsearch/index/engine/IdStoredFieldLoader.java
index 734678417c6b..61f9f2c0f24d 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/IdStoredFieldLoader.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/IdStoredFieldLoader.java
@@ -35,7 +35,7 @@ final class IdStoredFieldLoader {
 
     private static CheckedBiConsumer<Integer, StoredFieldVisitor, IOException> getStoredFieldsReader(LeafReader in) {
         if (in instanceof SequentialStoredFieldsLeafReader) {
-            return (((SequentialStoredFieldsLeafReader) in).getSequentialStoredFieldsReader())::visitDocument;
+            return (((SequentialStoredFieldsLeafReader) in).getSequentialStoredFieldsReader())::document;
         }
         throw new IllegalArgumentException("Requires a SequentialStoredFieldsReader, got " + in.getClass());
     }
diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index a3cd492918ff..84696f339e8f 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -42,6 +42,7 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.lucene.LoggerInfoStream;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
@@ -59,6 +60,7 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.cache.query.TrivialQueryCachingPolicy;
@@ -105,6 +107,7 @@
 import java.util.function.Function;
 import java.util.function.LongConsumer;
 import java.util.function.LongSupplier;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -226,7 +229,8 @@ public InternalEngine(EngineConfig engineConfig) {
                     logger,
                     translogDeletionPolicy,
                     softDeletesPolicy,
-                    translog::getLastSyncedGlobalCheckpoint
+                    translog::getLastSyncedGlobalCheckpoint,
+                    newCommitsListener()
                 );
                 this.localCheckpointTracker = createLocalCheckpointTracker(localCheckpointTrackerSupplier);
                 writer = createWriter();
@@ -322,6 +326,28 @@ private SoftDeletesPolicy newSoftDeletesPolicy() throws IOException {
         );
     }
 
+    @Nullable
+    private CombinedDeletionPolicy.CommitsListener newCommitsListener() {
+        final Engine.IndexCommitListener listener = engineConfig.getIndexCommitListener();
+        if (listener != null) {
+            var primaryTerm = config().getPrimaryTermSupplier().getAsLong();
+            return new CombinedDeletionPolicy.CommitsListener() {
+                @Override
+                public void onNewAcquiredCommit(final IndexCommit commit, final Set<String> additionalFiles) {
+                    final IndexCommitRef indexCommitRef = acquireIndexCommitRef(() -> commit);
+                    assert indexCommitRef.getIndexCommit() == commit;
+                    listener.onNewCommit(shardId, store, primaryTerm, indexCommitRef, additionalFiles);
+                }
+
+                @Override
+                public void onDeletedCommit(IndexCommit commit) {
+                    listener.onIndexCommitDelete(shardId, commit);
+                }
+            };
+        }
+        return null;
+    }
+
     @Override
     public CompletionStats completionStats(String... fieldNamePatterns) {
         return completionStatsCache.get(fieldNamePatterns);
@@ -812,7 +838,21 @@ private VersionValue resolveDocVersion(final Operation op, boolean loadSeqNo) th
             assert incrementIndexVersionLookup(); // used for asserting in tests
             final VersionsAndSeqNoResolver.DocIdAndVersion docIdAndVersion;
             try (Searcher searcher = acquireSearcher("load_version", SearcherScope.INTERNAL)) {
-                docIdAndVersion = VersionsAndSeqNoResolver.loadDocIdAndVersion(searcher.getIndexReader(), op.uid(), loadSeqNo);
+                if (engineConfig.getIndexSettings().getMode() == IndexMode.TIME_SERIES) {
+                    assert engineConfig.getLeafSorter() == DataStream.TIMESERIES_LEAF_READERS_SORTER;
+                    docIdAndVersion = VersionsAndSeqNoResolver.timeSeriesLoadDocIdAndVersion(
+                        searcher.getIndexReader(),
+                        op.uid(),
+                        op.id(),
+                        loadSeqNo
+                    );
+                } else {
+                    docIdAndVersion = VersionsAndSeqNoResolver.timeSeriesLoadDocIdAndVersion(
+                        searcher.getIndexReader(),
+                        op.uid(),
+                        loadSeqNo
+                    );
+                }
             }
             if (docIdAndVersion != null) {
                 versionValue = new IndexVersionValue(null, docIdAndVersion.version, docIdAndVersion.seqNo, docIdAndVersion.primaryTerm);
@@ -1792,20 +1832,21 @@ protected Optional<Exception> preFlightCheckForNoOp(final NoOp noOp) throws IOEx
     }
 
     @Override
-    public void refresh(String source) throws EngineException {
-        refresh(source, SearcherScope.EXTERNAL, true);
+    public RefreshResult refresh(String source) throws EngineException {
+        return refresh(source, SearcherScope.EXTERNAL, true);
     }
 
     @Override
-    public boolean maybeRefresh(String source) throws EngineException {
+    public RefreshResult maybeRefresh(String source) throws EngineException {
         return refresh(source, SearcherScope.EXTERNAL, false);
     }
 
-    final boolean refresh(String source, SearcherScope scope, boolean block) throws EngineException {
+    final RefreshResult refresh(String source, SearcherScope scope, boolean block) throws EngineException {
         // both refresh types will result in an internal refresh but only the external will also
         // pass the new reader reference to the external reader manager.
         final long localCheckpointBeforeRefresh = localCheckpointTracker.getProcessedCheckpoint();
         boolean refreshed;
+        long segmentGeneration = RefreshResult.UNKNOWN_GENERATION;
         try {
             // refresh does not need to hold readLock as ReferenceManager can handle correctly if the engine is closed in mid-way.
             if (store.tryIncRef()) {
@@ -1821,6 +1862,14 @@ final boolean refresh(String source, SearcherScope scope, boolean block) throws
                     } else {
                         refreshed = referenceManager.maybeRefresh();
                     }
+                    if (refreshed) {
+                        final ElasticsearchDirectoryReader current = referenceManager.acquire();
+                        try {
+                            segmentGeneration = current.getIndexCommit().getGeneration();
+                        } finally {
+                            referenceManager.release(current);
+                        }
+                    }
                 } finally {
                     store.decRef();
                 }
@@ -1852,7 +1901,7 @@ final boolean refresh(String source, SearcherScope scope, boolean block) throws
         // for a long time:
         maybePruneDeletes();
         mergeScheduler.refreshConfig();
-        return refreshed;
+        return new RefreshResult(refreshed, segmentGeneration);
     }
 
     @Override
@@ -1928,10 +1977,9 @@ public boolean flush(boolean force, boolean waitIfOngoing) throws EngineExceptio
                 // newly created commit points to a different translog generation (can free translog),
                 // or (4) the local checkpoint information in the last commit is stale, which slows down future recoveries.
                 boolean hasUncommittedChanges = indexWriter.hasUncommittedChanges();
-                boolean shouldPeriodicallyFlush = shouldPeriodicallyFlush();
                 if (hasUncommittedChanges
                     || force
-                    || shouldPeriodicallyFlush
+                    || shouldPeriodicallyFlush()
                     || getProcessedLocalCheckpoint() > Long.parseLong(
                         lastCommittedSegmentInfos.userData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)
                     )) {
@@ -1941,15 +1989,6 @@ public boolean flush(boolean force, boolean waitIfOngoing) throws EngineExceptio
                         logger.trace("starting commit for flush; commitTranslog=true");
                         commitIndexWriter(indexWriter, translog);
                         logger.trace("finished commit for flush");
-
-                        // a temporary debugging to investigate test failure - issue#32827. Remove when the issue is resolved
-                        logger.debug(
-                            "new commit on flush, hasUncommittedChanges:{}, force:{}, shouldPeriodicallyFlush:{}",
-                            hasUncommittedChanges,
-                            force,
-                            shouldPeriodicallyFlush
-                        );
-
                         // we need to refresh in order to clear older version values
                         refresh("version_table_flush", SearcherScope.INTERNAL, true);
                         translog.trimUnreferencedReaders();
@@ -2158,22 +2197,14 @@ public void forceMerge(final boolean flush, int maxNumSegments, boolean onlyExpu
         }
     }
 
-    @Override
-    public IndexCommitRef acquireLastIndexCommit(final boolean flushFirst) throws EngineException {
-        // we have to flush outside of the readlock otherwise we might have a problem upgrading
-        // the to a write lock when we fail the engine in this operation
-        if (flushFirst) {
-            logger.trace("start flush for snapshot");
-            flush(false, true);
-            logger.trace("finish flush for snapshot");
-        }
+    private IndexCommitRef acquireIndexCommitRef(final Supplier<IndexCommit> indexCommitSupplier) {
         store.incRef();
         boolean success = false;
         try {
-            final IndexCommit lastCommit = combinedDeletionPolicy.acquireIndexCommit(false);
+            final IndexCommit indexCommit = indexCommitSupplier.get();
             final IndexCommitRef commitRef = new IndexCommitRef(
-                lastCommit,
-                () -> IOUtils.close(() -> releaseIndexCommit(lastCommit), store::decRef)
+                indexCommit,
+                () -> IOUtils.close(() -> releaseIndexCommit(indexCommit), store::decRef)
             );
             success = true;
             return commitRef;
@@ -2185,22 +2216,20 @@ public IndexCommitRef acquireLastIndexCommit(final boolean flushFirst) throws En
     }
 
     @Override
-    public IndexCommitRef acquireSafeIndexCommit() throws EngineException {
-        store.incRef();
-        boolean success = false;
-        try {
-            final IndexCommit safeCommit = combinedDeletionPolicy.acquireIndexCommit(true);
-            final IndexCommitRef commitRef = new IndexCommitRef(
-                safeCommit,
-                () -> IOUtils.close(() -> releaseIndexCommit(safeCommit), store::decRef)
-            );
-            success = true;
-            return commitRef;
-        } finally {
-            if (success == false) {
-                store.decRef();
-            }
+    public IndexCommitRef acquireLastIndexCommit(final boolean flushFirst) throws EngineException {
+        // we have to flush outside of the readlock otherwise we might have a problem upgrading
+        // the to a write lock when we fail the engine in this operation
+        if (flushFirst) {
+            logger.trace("start flush for snapshot");
+            flush(false, true);
+            logger.trace("finish flush for snapshot");
         }
+        return acquireIndexCommitRef(() -> combinedDeletionPolicy.acquireIndexCommit(false));
+    }
+
+    @Override
+    public IndexCommitRef acquireSafeIndexCommit() throws EngineException {
+        return acquireIndexCommitRef(() -> combinedDeletionPolicy.acquireIndexCommit(true));
     }
 
     private void releaseIndexCommit(IndexCommit snapshot) throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java b/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java
index f673b693df6c..b81709aae8cb 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java
@@ -338,7 +338,7 @@ private Translog.Operation readDocAsOp(int docIndex) throws IOException {
             assert singleConsumer : "Sequential access optimization must not be enabled for multiple consumers";
             assert parallelArray.useSequentialStoredFieldsReader;
             assert storedFieldsReaderOrd == leaf.ord : storedFieldsReaderOrd + " != " + leaf.ord;
-            storedFieldsReader.visitDocument(segmentDocID, fields);
+            storedFieldsReader.document(segmentDocID, fields);
         } else {
             leaf.reader().document(segmentDocID, fields);
         }
diff --git a/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java b/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java
index b4d6c5ebaa00..e8ae9d605b3f 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java
@@ -17,12 +17,15 @@
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.util.concurrent.ReleasableLock;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.seqno.SeqNoStats;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogConfig;
 import org.elasticsearch.index.translog.TranslogDeletionPolicy;
+import org.elasticsearch.index.translog.TranslogStats;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -43,7 +46,21 @@ public final class NoOpEngine extends ReadOnlyEngine {
     private final DocsStats docsStats;
 
     public NoOpEngine(EngineConfig config) {
-        super(config, null, null, true, Function.identity(), true, true);
+        this(
+            config,
+            config.isPromotableToPrimary() ? null : new TranslogStats(0, 0, 0, 0, 0),
+            config.isPromotableToPrimary()
+                ? null
+                : new SeqNoStats(
+                    config.getGlobalCheckpointSupplier().getAsLong(),
+                    config.getGlobalCheckpointSupplier().getAsLong(),
+                    config.getGlobalCheckpointSupplier().getAsLong()
+                )
+        );
+    }
+
+    public NoOpEngine(EngineConfig config, @Nullable TranslogStats translogStats, SeqNoStats seqNoStats) {
+        super(config, seqNoStats, translogStats, true, Function.identity(), true, true);
         this.segmentsStats = new SegmentsStats();
         Directory directory = store.directory();
         try (DirectoryReader reader = openDirectory(directory)) {
diff --git a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
index e47c3cf984c2..7d66c96bd4c1 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
@@ -418,14 +418,15 @@ public List<Segment> segments() {
     }
 
     @Override
-    public void refresh(String source) {
+    public RefreshResult refresh(String source) {
         // we could allow refreshes if we want down the road the reader manager will then reflect changes to a rw-engine
         // opened side-by-side
+        return RefreshResult.NO_REFRESH;
     }
 
     @Override
-    public boolean maybeRefresh(String source) throws EngineException {
-        return false;
+    public RefreshResult maybeRefresh(String source) throws EngineException {
+        return RefreshResult.NO_REFRESH;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java b/server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java
index 1a0b8531d7fb..05c5031edbdd 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/RecoverySourcePruneMergePolicy.java
@@ -195,8 +195,8 @@ public void close() throws IOException {
             }
 
             @Override
-            public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
-                in.visitDocument(docID, visitor);
+            public void document(int docID, StoredFieldVisitor visitor) throws IOException {
+                in.document(docID, visitor);
             }
 
             @Override
@@ -220,11 +220,11 @@ private static class RecoverySourcePruningStoredFieldsReader extends FilterStore
             }
 
             @Override
-            public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
+            public void document(int docID, StoredFieldVisitor visitor) throws IOException {
                 if (recoverySourceToKeep != null && recoverySourceToKeep.get(docID)) {
-                    super.visitDocument(docID, visitor);
+                    super.document(docID, visitor);
                 } else {
-                    super.visitDocument(docID, new FilterStoredFieldVisitor(visitor) {
+                    super.document(docID, new FilterStoredFieldVisitor(visitor) {
                         @Override
                         public Status needsField(FieldInfo fieldInfo) throws IOException {
                             if (recoverySourceField.equals(fieldInfo.name)) {
diff --git a/server/src/main/java/org/elasticsearch/index/engine/Segment.java b/server/src/main/java/org/elasticsearch/index/engine/Segment.java
index 8bff90cf5856..88dbe7d454a3 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/Segment.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/Segment.java
@@ -14,7 +14,7 @@
 import org.apache.lucene.search.SortedNumericSortField;
 import org.apache.lucene.search.SortedSetSelector;
 import org.apache.lucene.search.SortedSetSortField;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -52,7 +52,7 @@ public Segment(StreamInput in) throws IOException {
         version = Lucene.parseVersionLenient(in.readOptionalString(), null);
         compound = in.readOptionalBoolean();
         mergeId = in.readOptionalString();
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             in.readLong(); // memoryInBytes
         }
         if (in.readBoolean()) {
@@ -159,7 +159,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(version.toString());
         out.writeOptionalBoolean(compound);
         out.writeOptionalString(mergeId);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeLong(0); // memoryInBytes
         }
 
@@ -252,7 +252,7 @@ private static void writeSegmentSort(StreamOutput out, Sort sort) throws IOExcep
                 o.writeBoolean(((SortedNumericSortField) field).getSelector() == SortedNumericSelector.Type.MAX);
                 o.writeBoolean(field.getReverse());
             } else if (field.getType().equals(SortField.Type.STRING)) {
-                if (o.getVersion().before(Version.V_8_5_0)) {
+                if (o.getTransportVersion().before(TransportVersion.V_8_5_0)) {
                     // The closest supported version before 8.5.0 was SortedSet fields, so we mimic that
                     o.writeByte(SORT_STRING_SET);
                     o.writeOptionalBoolean(field.getMissingValue() == null ? null : field.getMissingValue() == SortField.STRING_FIRST);
diff --git a/server/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java b/server/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java
index 321effc55730..5a2edcb05978 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.index.engine;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -39,7 +39,7 @@ public SegmentsStats() {
 
     public SegmentsStats(StreamInput in) throws IOException {
         count = in.readVLong();
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             in.readLong(); // memoryInBytes
             in.readLong(); // termsMemoryInBytes
             in.readLong(); // storedFieldsMemoryInBytes
@@ -220,7 +220,7 @@ static final class Fields {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(count);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeLong(0L); // memoryInBytes
             out.writeLong(0L); // termsMemoryInBytes
             out.writeLong(0L); // storedFieldsMemoryInBytes
@@ -250,7 +250,7 @@ public static class FileStats implements Writeable, ToXContentFragment {
         private final long max;
 
         FileStats(StreamInput in) throws IOException {
-            if (in.getVersion().onOrAfter(Version.V_7_13_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0)) {
                 this.ext = in.readString();
                 this.total = in.readVLong();
                 this.count = in.readVLong();
@@ -295,7 +295,7 @@ public long getMax() {
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            if (out.getVersion().onOrAfter(Version.V_7_13_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0)) {
                 out.writeString(ext);
                 out.writeVLong(total);
                 out.writeVLong(count);
diff --git a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java
index c377fdd73f3d..c6ac89f1a8e5 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java
@@ -11,11 +11,13 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.index.BaseTermsEnum;
 import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.ImpactsEnum;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexOptions;
@@ -30,11 +32,12 @@
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.StoredFields;
+import org.apache.lucene.index.TermVectors;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.VectorEncoding;
 import org.apache.lucene.index.VectorSimilarityFunction;
-import org.apache.lucene.index.VectorValues;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.ByteBuffersDirectory;
@@ -345,8 +348,13 @@ public NumericDocValues getNormValues(String field) throws IOException {
         }
 
         @Override
-        public VectorValues getVectorValues(String field) throws IOException {
-            return getDelegate().getVectorValues(field);
+        public FloatVectorValues getFloatVectorValues(String field) throws IOException {
+            return getDelegate().getFloatVectorValues(field);
+        }
+
+        @Override
+        public ByteVectorValues getByteVectorValues(String field) throws IOException {
+            return getDelegate().getByteVectorValues(field);
         }
 
         @Override
@@ -354,6 +362,11 @@ public TopDocs searchNearestVectors(String field, float[] target, int k, Bits ac
             return getDelegate().searchNearestVectors(field, target, k, acceptDocs, visitedLimit);
         }
 
+        @Override
+        public TopDocs searchNearestVectors(String field, byte[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
+            return getDelegate().searchNearestVectors(field, target, k, acceptDocs, visitedLimit);
+        }
+
         @Override
         public FieldInfos getFieldInfos() {
             return getDelegate().getFieldInfos();
@@ -382,6 +395,16 @@ public Fields getTermVectors(int docID) throws IOException {
             return getDelegate().getTermVectors(docID);
         }
 
+        @Override
+        public TermVectors termVectors() throws IOException {
+            return getDelegate().termVectors();
+        }
+
+        @Override
+        public StoredFields storedFields() throws IOException {
+            return getDelegate().storedFields();
+        }
+
         @Override
         public int numDocs() {
             return 1;
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java
index 8d24977a9c22..47f47a763e5b 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java
@@ -17,7 +17,7 @@
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.search.sort.BucketedSort;
 import org.elasticsearch.search.sort.SortOrder;
 
@@ -30,20 +30,20 @@ public abstract static class Builder<T> implements IndexFieldData.Builder {
         protected final String fieldName;
         protected final ValuesSourceType valuesSourceType;
         protected final ValueFetcher valueFetcher;
-        protected final SourceLookup sourceLookup;
+        protected final SourceProvider sourceProvider;
         protected final ToScriptFieldFactory<T> toScriptFieldFactory;
 
         public Builder(
             String fieldName,
             ValuesSourceType valuesSourceType,
             ValueFetcher valueFetcher,
-            SourceLookup sourceLookup,
+            SourceProvider sourceProvider,
             ToScriptFieldFactory<T> toScriptFieldFactory
         ) {
             this.fieldName = fieldName;
             this.valuesSourceType = valuesSourceType;
             this.valueFetcher = valueFetcher;
-            this.sourceLookup = sourceLookup;
+            this.sourceProvider = sourceProvider;
             this.toScriptFieldFactory = toScriptFieldFactory;
         }
     }
@@ -51,20 +51,20 @@ public Builder(
     protected final String fieldName;
     protected final ValuesSourceType valuesSourceType;
     protected final ValueFetcher valueFetcher;
-    protected final SourceLookup sourceLookup;
+    protected final SourceProvider sourceProvider;
     protected final ToScriptFieldFactory<T> toScriptFieldFactory;
 
     protected SourceValueFetcherIndexFieldData(
         String fieldName,
         ValuesSourceType valuesSourceType,
         ValueFetcher valueFetcher,
-        SourceLookup sourceLookup,
+        SourceProvider sourceProvider,
         ToScriptFieldFactory<T> toScriptFieldFactory
     ) {
         this.fieldName = fieldName;
         this.valuesSourceType = valuesSourceType;
         this.valueFetcher = valueFetcher;
-        this.sourceLookup = sourceLookup;
+        this.sourceProvider = sourceProvider;
         this.toScriptFieldFactory = toScriptFieldFactory;
     }
 
@@ -112,18 +112,18 @@ public abstract static class SourceValueFetcherLeafFieldData<T> implements LeafF
         protected final LeafReaderContext leafReaderContext;
 
         protected final ValueFetcher valueFetcher;
-        protected final SourceLookup sourceLookup;
+        protected final SourceProvider sourceProvider;
 
         public SourceValueFetcherLeafFieldData(
             ToScriptFieldFactory<T> toScriptFieldFactory,
             LeafReaderContext leafReaderContext,
             ValueFetcher valueFetcher,
-            SourceLookup sourceLookup
+            SourceProvider sourceProvider
         ) {
             this.toScriptFieldFactory = toScriptFieldFactory;
             this.leafReaderContext = leafReaderContext;
             this.valueFetcher = valueFetcher;
-            this.sourceLookup = sourceLookup;
+            this.sourceProvider = sourceProvider;
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherMultiGeoPointIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherMultiGeoPointIndexFieldData.java
index 0d79eb51d043..efa99c5d3f9f 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherMultiGeoPointIndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherMultiGeoPointIndexFieldData.java
@@ -15,7 +15,8 @@
 import org.elasticsearch.script.field.DocValuesScriptFieldFactory;
 import org.elasticsearch.script.field.ToScriptFieldFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.search.lookup.SourceProvider;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -30,10 +31,10 @@ public Builder(
             String fieldName,
             ValuesSourceType valuesSourceType,
             ValueFetcher valueFetcher,
-            SourceLookup sourceLookup,
+            SourceProvider sourceProvider,
             ToScriptFieldFactory<MultiGeoPointValues> toScriptFieldFactory
         ) {
-            super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory);
+            super(fieldName, valuesSourceType, valueFetcher, sourceProvider, toScriptFieldFactory);
         }
 
         @Override
@@ -42,7 +43,7 @@ public SourceValueFetcherMultiGeoPointIndexFieldData build(IndexFieldDataCache c
                 fieldName,
                 valuesSourceType,
                 valueFetcher,
-                sourceLookup,
+                sourceProvider,
                 toScriptFieldFactory
             );
         }
@@ -52,15 +53,15 @@ protected SourceValueFetcherMultiGeoPointIndexFieldData(
         String fieldName,
         ValuesSourceType valuesSourceType,
         ValueFetcher valueFetcher,
-        SourceLookup sourceLookup,
+        SourceProvider sourceProvider,
         ToScriptFieldFactory<MultiGeoPointValues> toScriptFieldFactory
     ) {
-        super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory);
+        super(fieldName, valuesSourceType, valueFetcher, sourceProvider, toScriptFieldFactory);
     }
 
     @Override
     public SourceValueFetcherMultiGeoPointLeafFieldData loadDirect(LeafReaderContext context) throws Exception {
-        return new SourceValueFetcherMultiGeoPointLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceLookup);
+        return new SourceValueFetcherMultiGeoPointLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceProvider);
     }
 
     public static class SourceValueFetcherMultiGeoPointLeafFieldData extends
@@ -70,15 +71,15 @@ public SourceValueFetcherMultiGeoPointLeafFieldData(
             ToScriptFieldFactory<MultiGeoPointValues> toScriptFieldFactory,
             LeafReaderContext leafReaderContext,
             ValueFetcher valueFetcher,
-            SourceLookup sourceLookup
+            SourceProvider sourceProvider
         ) {
-            super(toScriptFieldFactory, leafReaderContext, valueFetcher, sourceLookup);
+            super(toScriptFieldFactory, leafReaderContext, valueFetcher, sourceProvider);
         }
 
         @Override
         public DocValuesScriptFieldFactory getScriptFieldFactory(String name) {
             return toScriptFieldFactory.getScriptFieldFactory(
-                new MultiGeoPointValues(new SourceValueFetcherMultiGeoPointDocValues(leafReaderContext, valueFetcher, sourceLookup)),
+                new MultiGeoPointValues(new SourceValueFetcherMultiGeoPointDocValues(leafReaderContext, valueFetcher, sourceProvider)),
                 name
             );
         }
@@ -90,18 +91,17 @@ public static class SourceValueFetcherMultiGeoPointDocValues extends
         public SourceValueFetcherMultiGeoPointDocValues(
             LeafReaderContext leafReaderContext,
             ValueFetcher valueFetcher,
-            SourceLookup sourceLookup
+            SourceProvider sourceProvider
         ) {
-            super(leafReaderContext, valueFetcher, sourceLookup);
+            super(leafReaderContext, valueFetcher, sourceProvider);
         }
 
         @Override
         @SuppressWarnings("unchecked")
         public boolean advanceExact(int doc) throws IOException {
-            sourceLookup.setSegmentAndDocument(leafReaderContext, doc);
             values.clear();
-
-            for (Object value : valueFetcher.fetchValues(sourceLookup, doc, Collections.emptyList())) {
+            Source source = sourceProvider.getSource(leafReaderContext, doc);
+            for (Object value : valueFetcher.fetchValues(source, doc, Collections.emptyList())) {
                 assert value instanceof Map && ((Map<Object, Object>) value).get("coordinates") instanceof List;
                 List<Object> coordinates = ((Map<String, List<Object>>) value).get("coordinates");
                 assert coordinates.size() == 2 && coordinates.get(1) instanceof Number && coordinates.get(0) instanceof Number;
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java
index 6ba6a15cde11..cc47e796008c 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java
@@ -15,7 +15,8 @@
 import org.elasticsearch.script.field.DocValuesScriptFieldFactory;
 import org.elasticsearch.script.field.ToScriptFieldFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.search.lookup.SourceProvider;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -31,10 +32,10 @@ public Builder(
             String fieldName,
             ValuesSourceType valuesSourceType,
             ValueFetcher valueFetcher,
-            SourceLookup sourceLookup,
+            SourceProvider sourceProvider,
             ToScriptFieldFactory<SortedBinaryDocValues> toScriptFieldFactory
         ) {
-            super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory);
+            super(fieldName, valuesSourceType, valueFetcher, sourceProvider, toScriptFieldFactory);
         }
 
         @Override
@@ -43,7 +44,7 @@ public SourceValueFetcherSortedBinaryIndexFieldData build(IndexFieldDataCache ca
                 fieldName,
                 valuesSourceType,
                 valueFetcher,
-                sourceLookup,
+                sourceProvider,
                 toScriptFieldFactory
             );
         }
@@ -53,15 +54,15 @@ protected SourceValueFetcherSortedBinaryIndexFieldData(
         String fieldName,
         ValuesSourceType valuesSourceType,
         ValueFetcher valueFetcher,
-        SourceLookup sourceLookup,
+        SourceProvider sourceProvider,
         ToScriptFieldFactory<SortedBinaryDocValues> toScriptFieldFactory
     ) {
-        super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory);
+        super(fieldName, valuesSourceType, valueFetcher, sourceProvider, toScriptFieldFactory);
     }
 
     @Override
     public SourceValueFetcherSortedBinaryLeafFieldData loadDirect(LeafReaderContext context) throws Exception {
-        return new SourceValueFetcherSortedBinaryLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceLookup);
+        return new SourceValueFetcherSortedBinaryLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceProvider);
     }
 
     public static class SourceValueFetcherSortedBinaryLeafFieldData extends SourceValueFetcherLeafFieldData<SortedBinaryDocValues> {
@@ -70,15 +71,15 @@ public SourceValueFetcherSortedBinaryLeafFieldData(
             ToScriptFieldFactory<SortedBinaryDocValues> toScriptFieldFactory,
             LeafReaderContext leafReaderContext,
             ValueFetcher valueFetcher,
-            SourceLookup sourceLookup
+            SourceProvider sourceProvider
         ) {
-            super(toScriptFieldFactory, leafReaderContext, valueFetcher, sourceLookup);
+            super(toScriptFieldFactory, leafReaderContext, valueFetcher, sourceProvider);
         }
 
         @Override
         public DocValuesScriptFieldFactory getScriptFieldFactory(String name) {
             return toScriptFieldFactory.getScriptFieldFactory(
-                new SourceValueFetcherSortedBinaryDocValues(leafReaderContext, valueFetcher, sourceLookup),
+                new SourceValueFetcherSortedBinaryDocValues(leafReaderContext, valueFetcher, sourceProvider),
                 name
             );
         }
@@ -89,7 +90,7 @@ public static class SourceValueFetcherSortedBinaryDocValues extends SortedBinary
         private final LeafReaderContext leafReaderContext;
 
         private final ValueFetcher valueFetcher;
-        private final SourceLookup sourceLookup;
+        private final SourceProvider sourceProvider;
 
         private final SortedSet<BytesRef> values;
         private Iterator<BytesRef> iterator;
@@ -97,21 +98,20 @@ public static class SourceValueFetcherSortedBinaryDocValues extends SortedBinary
         public SourceValueFetcherSortedBinaryDocValues(
             LeafReaderContext leafReaderContext,
             ValueFetcher valueFetcher,
-            SourceLookup sourceLookup
+            SourceProvider sourceProvider
         ) {
             this.leafReaderContext = leafReaderContext;
             this.valueFetcher = valueFetcher;
-            this.sourceLookup = sourceLookup;
+            this.sourceProvider = sourceProvider;
 
             values = new TreeSet<>();
         }
 
         @Override
         public boolean advanceExact(int doc) throws IOException {
-            sourceLookup.setSegmentAndDocument(leafReaderContext, doc);
             values.clear();
-
-            for (Object object : valueFetcher.fetchValues(sourceLookup, doc, Collections.emptyList())) {
+            Source source = sourceProvider.getSource(leafReaderContext, doc);
+            for (Object object : valueFetcher.fetchValues(source, doc, Collections.emptyList())) {
                 values.add(new BytesRef(object.toString()));
             }
 
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBooleanIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBooleanIndexFieldData.java
index 7fec75ce7b64..5cf486a35274 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBooleanIndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBooleanIndexFieldData.java
@@ -15,7 +15,8 @@
 import org.elasticsearch.script.field.DocValuesScriptFieldFactory;
 import org.elasticsearch.script.field.ToScriptFieldFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.search.lookup.SourceProvider;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -28,10 +29,10 @@ public Builder(
             String fieldName,
             ValuesSourceType valuesSourceType,
             ValueFetcher valueFetcher,
-            SourceLookup sourceLookup,
+            SourceProvider sourceProvider,
             ToScriptFieldFactory<SortedNumericDocValues> toScriptFieldFactory
         ) {
-            super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory);
+            super(fieldName, valuesSourceType, valueFetcher, sourceProvider, toScriptFieldFactory);
         }
 
         @Override
@@ -40,7 +41,7 @@ public SourceValueFetcherSortedBooleanIndexFieldData build(IndexFieldDataCache c
                 fieldName,
                 valuesSourceType,
                 valueFetcher,
-                sourceLookup,
+                sourceProvider,
                 toScriptFieldFactory
             );
         }
@@ -50,15 +51,15 @@ protected SourceValueFetcherSortedBooleanIndexFieldData(
         String fieldName,
         ValuesSourceType valuesSourceType,
         ValueFetcher valueFetcher,
-        SourceLookup sourceLookup,
+        SourceProvider sourceProvider,
         ToScriptFieldFactory<SortedNumericDocValues> toScriptFieldFactory
     ) {
-        super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory);
+        super(fieldName, valuesSourceType, valueFetcher, sourceProvider, toScriptFieldFactory);
     }
 
     @Override
     public SourceValueFetcherLeafFieldData<SortedNumericDocValues> loadDirect(LeafReaderContext context) throws Exception {
-        return new SourceValueFetcherSortedBooleanLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceLookup);
+        return new SourceValueFetcherSortedBooleanLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceProvider);
     }
 
     private static class SourceValueFetcherSortedBooleanLeafFieldData extends SourceValueFetcherLeafFieldData<SortedNumericDocValues> {
@@ -67,15 +68,15 @@ private SourceValueFetcherSortedBooleanLeafFieldData(
             ToScriptFieldFactory<SortedNumericDocValues> toScriptFieldFactory,
             LeafReaderContext leafReaderContext,
             ValueFetcher valueFetcher,
-            SourceLookup sourceLookup
+            SourceProvider sourceProvider
         ) {
-            super(toScriptFieldFactory, leafReaderContext, valueFetcher, sourceLookup);
+            super(toScriptFieldFactory, leafReaderContext, valueFetcher, sourceProvider);
         }
 
         @Override
         public DocValuesScriptFieldFactory getScriptFieldFactory(String name) {
             return toScriptFieldFactory.getScriptFieldFactory(
-                new SourceValueFetcherSortedBooleanDocValues(leafReaderContext, valueFetcher, sourceLookup),
+                new SourceValueFetcherSortedBooleanDocValues(leafReaderContext, valueFetcher, sourceProvider),
                 name
             );
         }
@@ -86,7 +87,7 @@ static class SourceValueFetcherSortedBooleanDocValues extends SortedNumericDocVa
         private final LeafReaderContext leafReaderContext;
 
         private final ValueFetcher valueFetcher;
-        private final SourceLookup sourceLookup;
+        private final SourceProvider sourceProvider;
 
         private int trueCount;
         private int falseCount;
@@ -95,21 +96,19 @@ static class SourceValueFetcherSortedBooleanDocValues extends SortedNumericDocVa
         SourceValueFetcherSortedBooleanDocValues(
             LeafReaderContext leafReaderContext,
             ValueFetcher valueFetcher,
-            SourceLookup sourceLookup
+            SourceProvider sourceProvider
         ) {
             this.leafReaderContext = leafReaderContext;
             this.valueFetcher = valueFetcher;
-            this.sourceLookup = sourceLookup;
+            this.sourceProvider = sourceProvider;
         }
 
         @Override
         public boolean advanceExact(int doc) throws IOException {
-            sourceLookup.setSegmentAndDocument(leafReaderContext, doc);
-
             trueCount = 0;
             falseCount = 0;
-
-            for (Object value : valueFetcher.fetchValues(sourceLookup, doc, Collections.emptyList())) {
+            Source source = sourceProvider.getSource(leafReaderContext, doc);
+            for (Object value : valueFetcher.fetchValues(source, doc, Collections.emptyList())) {
                 assert value instanceof Boolean;
                 if ((Boolean) value) {
                     ++trueCount;
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedDoubleIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedDoubleIndexFieldData.java
index ee3b28bf9641..c1659441f92d 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedDoubleIndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedDoubleIndexFieldData.java
@@ -14,7 +14,8 @@
 import org.elasticsearch.script.field.DocValuesScriptFieldFactory;
 import org.elasticsearch.script.field.ToScriptFieldFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.search.lookup.SourceProvider;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -30,10 +31,10 @@ public Builder(
             String fieldName,
             ValuesSourceType valuesSourceType,
             ValueFetcher valueFetcher,
-            SourceLookup sourceLookup,
+            SourceProvider sourceProvider,
             ToScriptFieldFactory<SortedNumericDoubleValues> toScriptFieldFactory
         ) {
-            super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory);
+            super(fieldName, valuesSourceType, valueFetcher, sourceProvider, toScriptFieldFactory);
         }
 
         @Override
@@ -42,7 +43,7 @@ public SourceValueFetcherSortedDoubleIndexFieldData build(IndexFieldDataCache ca
                 fieldName,
                 valuesSourceType,
                 valueFetcher,
-                sourceLookup,
+                sourceProvider,
                 toScriptFieldFactory
             );
         }
@@ -52,15 +53,15 @@ protected SourceValueFetcherSortedDoubleIndexFieldData(
         String fieldName,
         ValuesSourceType valuesSourceType,
         ValueFetcher valueFetcher,
-        SourceLookup sourceLookup,
+        SourceProvider sourceProvider,
         ToScriptFieldFactory<SortedNumericDoubleValues> toScriptFieldFactory
     ) {
-        super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory);
+        super(fieldName, valuesSourceType, valueFetcher, sourceProvider, toScriptFieldFactory);
     }
 
     @Override
     public SourceValueFetcherLeafFieldData<SortedNumericDoubleValues> loadDirect(LeafReaderContext context) throws Exception {
-        return new SourceValueFetcherSortedDoubleLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceLookup);
+        return new SourceValueFetcherSortedDoubleLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceProvider);
     }
 
     private static class SourceValueFetcherSortedDoubleLeafFieldData extends SourceValueFetcherLeafFieldData<SortedNumericDoubleValues> {
@@ -69,15 +70,15 @@ private SourceValueFetcherSortedDoubleLeafFieldData(
             ToScriptFieldFactory<SortedNumericDoubleValues> toScriptFieldFactory,
             LeafReaderContext leafReaderContext,
             ValueFetcher valueFetcher,
-            SourceLookup sourceLookup
+            SourceProvider sourceProvider
         ) {
-            super(toScriptFieldFactory, leafReaderContext, valueFetcher, sourceLookup);
+            super(toScriptFieldFactory, leafReaderContext, valueFetcher, sourceProvider);
         }
 
         @Override
         public DocValuesScriptFieldFactory getScriptFieldFactory(String name) {
             return toScriptFieldFactory.getScriptFieldFactory(
-                new SourceValueFetcherSortedNumericDoubleValues(leafReaderContext, valueFetcher, sourceLookup),
+                new SourceValueFetcherSortedNumericDoubleValues(leafReaderContext, valueFetcher, sourceProvider),
                 name
             );
         }
@@ -88,7 +89,7 @@ private static class SourceValueFetcherSortedNumericDoubleValues extends SortedN
         private final LeafReaderContext leafReaderContext;
 
         private final ValueFetcher valueFetcher;
-        private final SourceLookup sourceLookup;
+        private final SourceProvider sourceProvider;
 
         private final List<Double> values;
         private Iterator<Double> iterator;
@@ -96,21 +97,20 @@ private static class SourceValueFetcherSortedNumericDoubleValues extends SortedN
         private SourceValueFetcherSortedNumericDoubleValues(
             LeafReaderContext leafReaderContext,
             ValueFetcher valueFetcher,
-            SourceLookup sourceLookup
+            SourceProvider sourceProvider
         ) {
             this.leafReaderContext = leafReaderContext;
             this.valueFetcher = valueFetcher;
-            this.sourceLookup = sourceLookup;
+            this.sourceProvider = sourceProvider;
 
             values = new ArrayList<>();
         }
 
         @Override
         public boolean advanceExact(int doc) throws IOException {
-            sourceLookup.setSegmentAndDocument(leafReaderContext, doc);
             values.clear();
-
-            for (Object value : valueFetcher.fetchValues(sourceLookup, doc, Collections.emptyList())) {
+            Source source = sourceProvider.getSource(leafReaderContext, doc);
+            for (Object value : valueFetcher.fetchValues(source, doc, Collections.emptyList())) {
                 assert value instanceof Number;
                 values.add(((Number) value).doubleValue());
             }
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java
index 37c5198c52ac..f3f9446a42af 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java
@@ -15,7 +15,8 @@
 import org.elasticsearch.script.field.DocValuesScriptFieldFactory;
 import org.elasticsearch.script.field.ToScriptFieldFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.search.lookup.SourceProvider;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -31,10 +32,10 @@ public Builder(
             String fieldName,
             ValuesSourceType valuesSourceType,
             ValueFetcher valueFetcher,
-            SourceLookup sourceLookup,
+            SourceProvider sourceProvider,
             ToScriptFieldFactory<SortedNumericDocValues> toScriptFieldFactory
         ) {
-            super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory);
+            super(fieldName, valuesSourceType, valueFetcher, sourceProvider, toScriptFieldFactory);
         }
 
         @Override
@@ -43,7 +44,7 @@ public SourceValueFetcherSortedNumericIndexFieldData build(IndexFieldDataCache c
                 fieldName,
                 valuesSourceType,
                 valueFetcher,
-                sourceLookup,
+                sourceProvider,
                 toScriptFieldFactory
             );
         }
@@ -53,15 +54,15 @@ protected SourceValueFetcherSortedNumericIndexFieldData(
         String fieldName,
         ValuesSourceType valuesSourceType,
         ValueFetcher valueFetcher,
-        SourceLookup sourceLookup,
+        SourceProvider sourceProvider,
         ToScriptFieldFactory<SortedNumericDocValues> toScriptFieldFactory
     ) {
-        super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory);
+        super(fieldName, valuesSourceType, valueFetcher, sourceProvider, toScriptFieldFactory);
     }
 
     @Override
     public SourceValueFetcherSortedNumericLeafFieldData loadDirect(LeafReaderContext context) throws Exception {
-        return new SourceValueFetcherSortedNumericLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceLookup);
+        return new SourceValueFetcherSortedNumericLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceProvider);
     }
 
     public static class SourceValueFetcherSortedNumericLeafFieldData extends SourceValueFetcherLeafFieldData<SortedNumericDocValues> {
@@ -70,15 +71,15 @@ public SourceValueFetcherSortedNumericLeafFieldData(
             ToScriptFieldFactory<SortedNumericDocValues> toScriptFieldFactory,
             LeafReaderContext leafReaderContext,
             ValueFetcher valueFetcher,
-            SourceLookup sourceLookup
+            SourceProvider sourceProvider
         ) {
-            super(toScriptFieldFactory, leafReaderContext, valueFetcher, sourceLookup);
+            super(toScriptFieldFactory, leafReaderContext, valueFetcher, sourceProvider);
         }
 
         @Override
         public DocValuesScriptFieldFactory getScriptFieldFactory(String name) {
             return toScriptFieldFactory.getScriptFieldFactory(
-                new SourceValueFetcherSortedNumericDocValues(leafReaderContext, valueFetcher, sourceLookup),
+                new SourceValueFetcherSortedNumericDocValues(leafReaderContext, valueFetcher, sourceProvider),
                 name
             );
         }
@@ -89,7 +90,7 @@ public static class SourceValueFetcherSortedNumericDocValues extends SortedNumer
         protected final LeafReaderContext leafReaderContext;
 
         protected final ValueFetcher valueFetcher;
-        protected final SourceLookup sourceLookup;
+        protected final SourceProvider sourceProvider;
 
         protected final List<Long> values;
         protected Iterator<Long> iterator;
@@ -97,21 +98,20 @@ public static class SourceValueFetcherSortedNumericDocValues extends SortedNumer
         public SourceValueFetcherSortedNumericDocValues(
             LeafReaderContext leafReaderContext,
             ValueFetcher valueFetcher,
-            SourceLookup sourceLookup
+            SourceProvider sourceProvider
         ) {
             this.leafReaderContext = leafReaderContext;
             this.valueFetcher = valueFetcher;
-            this.sourceLookup = sourceLookup;
+            this.sourceProvider = sourceProvider;
 
             values = new ArrayList<>();
         }
 
         @Override
         public boolean advanceExact(int doc) throws IOException {
-            sourceLookup.setSegmentAndDocument(leafReaderContext, doc);
             values.clear();
-
-            for (Object value : valueFetcher.fetchValues(sourceLookup, doc, Collections.emptyList())) {
+            Source source = sourceProvider.getSource(leafReaderContext, doc);
+            for (Object value : valueFetcher.fetchValues(source, doc, Collections.emptyList())) {
                 assert value instanceof Number;
                 values.add(((Number) value).longValue());
             }
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/StringScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/StringScriptDocValues.java
index e42659743521..f5e236e9e864 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/StringScriptDocValues.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/StringScriptDocValues.java
@@ -21,7 +21,8 @@ public final class StringScriptDocValues extends SortingBinaryDocValues {
 
     @Override
     public boolean advanceExact(int docId) {
-        List<String> results = script.resultsForDoc(docId);
+        script.runForDoc(docId);
+        List<String> results = script.getValues();
         count = results.size();
         if (count == 0) {
             return false;
diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java
index 7e2283fec76d..bbb5f40431e6 100644
--- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java
+++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java
@@ -60,6 +60,23 @@ public List<String> fieldsToLoad() {
         };
     }
 
+    /**
+     * Creates a StoredFieldLoader tuned for sequential reads of _source
+     */
+    public static StoredFieldLoader sequentialSource() {
+        return new StoredFieldLoader() {
+            @Override
+            public LeafStoredFieldLoader getLoader(LeafReaderContext ctx, int[] docs) {
+                return new ReaderStoredFieldLoader(sequentialReader(ctx), true, Set.of());
+            }
+
+            @Override
+            public List<String> fieldsToLoad() {
+                return List.of();
+            }
+        };
+    }
+
     /**
      * Creates a no-op StoredFieldLoader that will not load any fields from disk
      */
@@ -82,8 +99,16 @@ private static CheckedBiConsumer<Integer, FieldsVisitor, IOException> reader(Lea
         if (docs == null) {
             return leafReader::document;
         }
-        if (leafReader instanceof SequentialStoredFieldsLeafReader lf && docs.length > 10 && hasSequentialDocs(docs)) {
-            return lf.getSequentialStoredFieldsReader()::visitDocument;
+        if (docs.length > 10 && hasSequentialDocs(docs)) {
+            return sequentialReader(ctx);
+        }
+        return leafReader::document;
+    }
+
+    private static CheckedBiConsumer<Integer, FieldsVisitor, IOException> sequentialReader(LeafReaderContext ctx) {
+        LeafReader leafReader = ctx.reader();
+        if (leafReader instanceof SequentialStoredFieldsLeafReader lf) {
+            return lf.getSequentialStoredFieldsReader()::document;
         }
         return leafReader::document;
     }
diff --git a/server/src/main/java/org/elasticsearch/index/get/GetResult.java b/server/src/main/java/org/elasticsearch/index/get/GetResult.java
index 2d8b6b0572bd..efedb141d246 100644
--- a/server/src/main/java/org/elasticsearch/index/get/GetResult.java
+++ b/server/src/main/java/org/elasticsearch/index/get/GetResult.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.index.get;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressorFactory;
@@ -24,7 +24,7 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.rest.action.document.RestMultiGetAction;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
@@ -66,7 +66,7 @@ public class GetResult implements Writeable, Iterable<DocumentField>, ToXContent
 
     public GetResult(StreamInput in) throws IOException {
         index = in.readString();
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             in.readOptionalString();
         }
         id = in.readString();
@@ -225,7 +225,7 @@ public Map<String, Object> sourceAsMap() throws ElasticsearchParseException {
             return sourceAsMap;
         }
 
-        sourceAsMap = SourceLookup.sourceAsMap(source);
+        sourceAsMap = Source.fromBytes(source).source();
         return sourceAsMap;
     }
 
@@ -386,7 +386,7 @@ public static GetResult fromXContent(XContentParser parser) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(index);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeOptionalString(MapperService.SINGLE_MAPPING_NAME);
         }
         out.writeString(id);
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java
index a6db9824c756..c481efeb3a23 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java
@@ -163,7 +163,7 @@ protected AbstractGeometryFieldMapper(
         MultiFields multiFields,
         CopyTo copyTo,
         Parser<T> parser,
-        String onScriptError
+        OnScriptError onScriptError
     ) {
         super(simpleName, mappedFieldType, multiFields, copyTo, true, onScriptError);
         this.ignoreMalformed = Explicit.EXPLICIT_FALSE;
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java
index 571f0d59a9f2..14db646d8684 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java
@@ -54,7 +54,7 @@ protected AbstractPointGeometryFieldMapper(
         MultiFields multiFields,
         CopyTo copyTo,
         Parser<T> parser,
-        String onScriptError
+        OnScriptError onScriptError
     ) {
         super(simpleName, mappedFieldType, multiFields, copyTo, parser, onScriptError);
         this.nullValue = null;
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java
index aa0e0c17a52b..b84294168397 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java
@@ -215,7 +215,7 @@ public void validateMatchedRoutingPath() {
     abstract static class Builder<Factory> extends RuntimeField.Builder {
         private final ScriptContext<Factory> scriptContext;
 
-        final FieldMapper.Parameter<Script> script = new FieldMapper.Parameter<>(
+        private final FieldMapper.Parameter<Script> script = new FieldMapper.Parameter<>(
             "script",
             true,
             () -> null,
@@ -225,6 +225,11 @@ abstract static class Builder<Factory> extends RuntimeField.Builder {
             Objects::toString
         ).setSerializerCheck((id, ic, v) -> ic);
 
+        private final FieldMapper.Parameter<OnScriptError> onScriptError = FieldMapper.Parameter.onScriptErrorParam(
+            m -> m.onScriptError,
+            script
+        );
+
         Builder(String name, ScriptContext<Factory> scriptContext) {
             super(name);
             this.scriptContext = scriptContext;
@@ -247,7 +252,8 @@ protected final RuntimeField createRuntimeField(MappingParserContext parserConte
         protected final RuntimeField createChildRuntimeField(
             MappingParserContext parserContext,
             String parent,
-            Function<SearchLookup, CompositeFieldScript.LeafFactory> parentScriptFactory
+            Function<SearchLookup, CompositeFieldScript.LeafFactory> parentScriptFactory,
+            OnScriptError onScriptError
         ) {
             if (script.isConfigured()) {
                 throw new IllegalArgumentException(
@@ -257,7 +263,7 @@ protected final RuntimeField createChildRuntimeField(
             String fullName = parent + "." + name;
             return new LeafRuntimeField(
                 name,
-                createFieldType(fullName, getCompositeLeafFactory(parentScriptFactory), getScript(), meta()),
+                createFieldType(fullName, getCompositeLeafFactory(parentScriptFactory), getScript(), meta(), onScriptError),
                 getParameters()
             );
         }
@@ -267,26 +273,34 @@ final RuntimeField createRuntimeField(Factory scriptFactory) {
         }
 
         final RuntimeField createRuntimeField(Factory scriptFactory, Version indexVersion) {
-            var fieldType = createFieldType(name, scriptFactory, getScript(), meta(), indexVersion);
+            var fieldType = createFieldType(name, scriptFactory, getScript(), meta(), indexVersion, onScriptError.get());
             return new LeafRuntimeField(name, fieldType, getParameters());
         }
 
-        abstract AbstractScriptFieldType<?> createFieldType(String name, Factory factory, Script script, Map<String, String> meta);
+        abstract AbstractScriptFieldType<?> createFieldType(
+            String name,
+            Factory factory,
+            Script script,
+            Map<String, String> meta,
+            OnScriptError onScriptError
+        );
 
         AbstractScriptFieldType<?> createFieldType(
             String name,
             Factory factory,
             Script script,
             Map<String, String> meta,
-            Version supportedVersion
+            Version supportedVersion,
+            OnScriptError onScriptError
         ) {
-            return createFieldType(name, factory, script, meta);
+            return createFieldType(name, factory, script, meta, onScriptError);
         }
 
         @Override
         protected List<FieldMapper.Parameter<?>> getParameters() {
             List<FieldMapper.Parameter<?>> parameters = new ArrayList<>(super.getParameters());
             parameters.add(script);
+            parameters.add(onScriptError);
             return Collections.unmodifiableList(parameters);
         }
 
@@ -296,5 +310,6 @@ protected final Script getScript() {
             }
             return script.get();
         }
+
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
index fc19cdddd457..e8709360a963 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
@@ -96,7 +96,7 @@ public static class Builder extends FieldMapper.Builder {
         ).acceptsNull();
 
         private final Parameter<Script> script = Parameter.scriptParam(m -> toType(m).script);
-        private final Parameter<String> onScriptError = Parameter.onScriptErrorParam(m -> toType(m).onScriptError, script);
+        private final Parameter<OnScriptError> onScriptError = Parameter.onScriptErrorParam(m -> toType(m).onScriptError, script);
 
         private final Parameter<Map<String, String>> meta = Parameter.metaParam();
 
@@ -139,7 +139,7 @@ private FieldValues<Boolean> scriptValues() {
             BooleanFieldScript.Factory scriptFactory = scriptCompiler.compile(script.get(), BooleanFieldScript.CONTEXT);
             return scriptFactory == null
                 ? null
-                : (lookup, ctx, doc, consumer) -> scriptFactory.newFactory(name, script.get().getParams(), lookup)
+                : (lookup, ctx, doc, consumer) -> scriptFactory.newFactory(name, script.get().getParams(), lookup, OnScriptError.FAIL)
                     .newInstance(ctx)
                     .runForDoc(doc, consumer);
         }
@@ -280,7 +280,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext
                     name(),
                     CoreValuesSourceType.BOOLEAN,
                     sourceValueFetcher(sourcePaths),
-                    searchLookup.source(),
+                    searchLookup,
                     BooleanDocValuesField::new
                 );
             }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java
index 76fd5aa59e03..ce1dc432a837 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java
@@ -46,9 +46,10 @@ AbstractScriptFieldType<?> createFieldType(
             String name,
             BooleanFieldScript.Factory factory,
             Script script,
-            Map<String, String> meta
+            Map<String, String> meta,
+            OnScriptError onScriptError
         ) {
-            return new BooleanScriptFieldType(name, factory, script, meta);
+            return new BooleanScriptFieldType(name, factory, script, meta, onScriptError);
         }
 
         @Override
@@ -67,10 +68,16 @@ public static RuntimeField sourceOnly(String name) {
         return new Builder(name).createRuntimeField(BooleanFieldScript.PARSE_FROM_SOURCE);
     }
 
-    BooleanScriptFieldType(String name, BooleanFieldScript.Factory scriptFactory, Script script, Map<String, String> meta) {
+    BooleanScriptFieldType(
+        String name,
+        BooleanFieldScript.Factory scriptFactory,
+        Script script,
+        Map<String, String> meta,
+        OnScriptError onScriptError
+    ) {
         super(
             name,
-            searchLookup -> scriptFactory.newFactory(name, script.getParams(), searchLookup),
+            searchLookup -> scriptFactory.newFactory(name, script.getParams(), searchLookup, onScriptError),
             script,
             scriptFactory.isResultDeterministic(),
             meta
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompositeRuntimeField.java b/server/src/main/java/org/elasticsearch/index/mapper/CompositeRuntimeField.java
index ea8fbb0dcd60..d7b039dde1c2 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/CompositeRuntimeField.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/CompositeRuntimeField.java
@@ -47,6 +47,11 @@ public class CompositeRuntimeField implements RuntimeField {
             }
         });
 
+        private final FieldMapper.Parameter<OnScriptError> onScriptError = FieldMapper.Parameter.onScriptErrorParam(
+            m -> m.onScriptError,
+            script
+        );
+
         private final FieldMapper.Parameter<Map<String, Object>> fields = new FieldMapper.Parameter<Map<String, Object>>(
             "fields",
             false,
@@ -66,6 +71,7 @@ protected List<FieldMapper.Parameter<?>> getParameters() {
             List<FieldMapper.Parameter<?>> parameters = new ArrayList<>(super.getParameters());
             parameters.add(script);
             parameters.add(fields);
+            parameters.add(onScriptError);
             return Collections.unmodifiableList(parameters);
         }
 
@@ -73,7 +79,8 @@ protected List<FieldMapper.Parameter<?>> getParameters() {
         protected RuntimeField createChildRuntimeField(
             MappingParserContext parserContext,
             String parent,
-            Function<SearchLookup, CompositeFieldScript.LeafFactory> parentScriptFactory
+            Function<SearchLookup, CompositeFieldScript.LeafFactory> parentScriptFactory,
+            OnScriptError onScriptError
         ) {
             throw new IllegalArgumentException("Composite field [" + name + "] cannot be a child of composite field [" + parent + "]");
         }
@@ -84,7 +91,8 @@ protected RuntimeField createRuntimeField(MappingParserContext parserContext) {
             Function<RuntimeField.Builder, RuntimeField> builder = b -> b.createChildRuntimeField(
                 parserContext,
                 name,
-                lookup -> factory.newFactory(name, script.get().getParams(), lookup)
+                lookup -> factory.newFactory(name, script.get().getParams(), lookup, onScriptError.get()),
+                onScriptError.get()
             );
             Map<String, RuntimeField> runtimeFields = RuntimeField.parseRuntimeFields(
                 new HashMap<>(fields.getValue()),
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
index 1ce591e15f0c..0b8f3855fdd2 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
@@ -10,6 +10,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.document.LongField;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StoredField;
@@ -17,8 +18,8 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.PointValues;
 import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
+import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
@@ -245,7 +246,7 @@ public static class Builder extends FieldMapper.Builder {
         private final Parameter<Boolean> ignoreMalformed;
 
         private final Parameter<Script> script = Parameter.scriptParam(m -> toType(m).script);
-        private final Parameter<String> onScriptError = Parameter.onScriptErrorParam(m -> toType(m).onScriptError, script);
+        private final Parameter<OnScriptError> onScriptError = Parameter.onScriptErrorParam(m -> toType(m).onScriptError, script);
 
         private final Resolution resolution;
         private final Version indexCreatedVersion;
@@ -303,9 +304,13 @@ private FieldValues<Long> scriptValues() {
             DateFieldScript.Factory factory = scriptCompiler.compile(script.get(), DateFieldScript.CONTEXT);
             return factory == null
                 ? null
-                : (lookup, ctx, doc, consumer) -> factory.newFactory(name, script.get().getParams(), lookup, buildFormatter())
-                    .newInstance(ctx)
-                    .runForDoc(doc, consumer::accept);
+                : (lookup, ctx, doc, consumer) -> factory.newFactory(
+                    name,
+                    script.get().getParams(),
+                    lookup,
+                    buildFormatter(),
+                    OnScriptError.FAIL
+                ).newInstance(ctx).runForDoc(doc, consumer::accept);
         }
 
         @Override
@@ -785,7 +790,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext
                     name(),
                     resolution.numericType().getValuesSourceType(),
                     sourceValueFetcher(sourcePaths),
-                    searchLookup.source(),
+                    searchLookup,
                     resolution.getDefaultToScriptFieldFactory()
                 );
             }
@@ -903,17 +908,20 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
     }
 
     private void indexValue(DocumentParserContext context, long timestamp) {
-        if (indexed) {
-            context.doc().add(new LongPoint(fieldType().name(), timestamp));
-        }
-        if (hasDocValues) {
+        if (indexed && hasDocValues) {
+            context.doc().add(new LongField(fieldType().name(), timestamp));
+        } else if (hasDocValues) {
             context.doc().add(new SortedNumericDocValuesField(fieldType().name(), timestamp));
-        } else if (store || indexed) {
-            context.addToFieldNames(fieldType().name());
+        } else if (indexed) {
+            context.doc().add(new LongPoint(fieldType().name(), timestamp));
         }
         if (store) {
             context.doc().add(new StoredField(fieldType().name(), timestamp));
         }
+        if (hasDocValues == false && (indexed || store)) {
+            // When the field doesn't have doc values so that we can run exists queries, we also need to index the field name separately.
+            context.addToFieldNames(fieldType().name());
+        }
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java
index f6da438283d9..c2371877ae6e 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java
@@ -87,23 +87,29 @@ protected List<FieldMapper.Parameter<?>> getParameters() {
             return Collections.unmodifiableList(parameters);
         }
 
-        @Override
         AbstractScriptFieldType<?> createFieldType(
             String name,
             DateFieldScript.Factory factory,
             Script script,
             Map<String, String> meta,
-            Version supportedVersion
+            Version supportedVersion,
+            OnScriptError onScriptError
         ) {
             String pattern = format.getValue() == null ? DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.pattern() : format.getValue();
             Locale locale = this.locale.getValue() == null ? Locale.ROOT : this.locale.getValue();
             DateFormatter dateTimeFormatter = DateFormatter.forPattern(pattern, supportedVersion).withLocale(locale);
-            return new DateScriptFieldType(name, factory, dateTimeFormatter, script, meta);
+            return new DateScriptFieldType(name, factory, dateTimeFormatter, script, meta, onScriptError);
         }
 
         @Override
-        AbstractScriptFieldType<?> createFieldType(String name, DateFieldScript.Factory factory, Script script, Map<String, String> meta) {
-            return createFieldType(name, factory, script, meta, Version.CURRENT);
+        AbstractScriptFieldType<?> createFieldType(
+            String name,
+            DateFieldScript.Factory factory,
+            Script script,
+            Map<String, String> meta,
+            OnScriptError onScriptError
+        ) {
+            return createFieldType(name, factory, script, meta, Version.CURRENT, onScriptError);
         }
 
         @Override
@@ -131,11 +137,12 @@ public static RuntimeField sourceOnly(String name, DateFormatter dateTimeFormatt
         DateFieldScript.Factory scriptFactory,
         DateFormatter dateTimeFormatter,
         Script script,
-        Map<String, String> meta
+        Map<String, String> meta,
+        OnScriptError onScriptError
     ) {
         super(
             name,
-            searchLookup -> scriptFactory.newFactory(name, script.getParams(), searchLookup, dateTimeFormatter),
+            searchLookup -> scriptFactory.newFactory(name, script.getParams(), searchLookup, dateTimeFormatter, onScriptError),
             script,
             scriptFactory.isResultDeterministic(),
             meta
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java
index fa5b01913be8..946a0a1aa171 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java
@@ -155,7 +155,15 @@ public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf
                 hasValue = false;
                 return null;
             }
-            return docId -> hasValue = docId == postings.advance(docId);
+            return docId -> {
+                if (docId < postings.docID()) {
+                    return hasValue = false;
+                }
+                if (docId == postings.docID()) {
+                    return hasValue = true;
+                }
+                return hasValue = docId == postings.advance(docId);
+            };
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java
index 6f5f0c336633..0b0cc5673a31 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java
@@ -8,6 +8,8 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.util.BytesRef;
+
 import java.net.InetAddress;
 import java.util.HashSet;
 import java.util.Set;
@@ -16,7 +18,17 @@
  * Collects dimensions from documents.
  */
 public interface DocumentDimensions {
-    void addString(String fieldName, String value);
+
+    /**
+     * This overloaded method tries to take advantage of the fact that the UTF-8
+     * value is already computed in some cases when we want to collect
+     * dimensions, so we can save re-computing the UTF-8 encoding.
+     */
+    void addString(String fieldName, BytesRef utf8Value);
+
+    default void addString(String fieldName, String value) {
+        addString(fieldName, new BytesRef(value));
+    }
 
     void addIp(String fieldName, InetAddress value);
 
@@ -30,6 +42,12 @@ public interface DocumentDimensions {
     class OnlySingleValueAllowed implements DocumentDimensions {
         private final Set<String> names = new HashSet<>();
 
+        @Override
+        public void addString(String fieldName, BytesRef value) {
+            add(fieldName);
+        }
+
+        // Override to skip the UTF-8 conversion that happens in the default implementation
         @Override
         public void addString(String fieldName, String value) {
             add(fieldName);
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java
index e2a8feb30fa7..401416c4e4b4 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java
@@ -9,10 +9,12 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.LeafMetaData;
@@ -24,10 +26,11 @@
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.StoredFields;
+import org.apache.lucene.index.TermVectors;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.VectorEncoding;
 import org.apache.lucene.index.VectorSimilarityFunction;
-import org.apache.lucene.index.VectorValues;
 import org.apache.lucene.index.memory.MemoryIndex;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.TopDocs;
@@ -46,10 +49,9 @@
 /**
  * A {@link LeafReader} over a lucene document that exposes doc values and stored fields.
  * Note that unlike lucene's {@link MemoryIndex} implementation, this holds no state and
- * does not attempt to do any analysis on text fields.  It also supports stored
- * fields where MemoryIndex does not.  It is used to back index-time scripts that
- * reference field data and stored fields from a document that has not yet been
- * indexed.
+ * does not attempt to do any analysis on text fields.  It is used to back index-time
+ * scripts that reference field data and stored fields from a document that has not yet
+ * been indexed.
  */
 class DocumentLeafReader extends LeafReader {
 
@@ -175,6 +177,11 @@ public void document(int docID, StoredFieldVisitor visitor) throws IOException {
         }
     }
 
+    @Override
+    public StoredFields storedFields() throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
     @Override
     public CacheHelper getCoreCacheHelper() {
         throw new UnsupportedOperationException();
@@ -191,7 +198,7 @@ public NumericDocValues getNormValues(String field) throws IOException {
     }
 
     @Override
-    public VectorValues getVectorValues(String field) throws IOException {
+    public FloatVectorValues getFloatVectorValues(String field) throws IOException {
         throw new UnsupportedOperationException();
     }
 
@@ -240,6 +247,21 @@ protected void doClose() throws IOException {
         throw new UnsupportedOperationException();
     }
 
+    @Override
+    public ByteVectorValues getByteVectorValues(String field) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public TopDocs searchNearestVectors(String field, byte[] target, int k, Bits acceptDocs, int visitedLimit) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public TermVectors termVectors() throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
     @Override
     public CacheHelper getReaderCacheHelper() {
         throw new UnsupportedOperationException();
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
index 938212aac04f..ea52660c40b1 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
@@ -93,13 +93,12 @@ public void validate(IndexSettings settings, boolean checkLimits) {
             }
         }
 
+        settings.getMode().validateMapping(mappingLookup);
         /*
          * Build an empty source loader to validate that the mapping is compatible
          * with the source loading strategy declared on the source field mapper.
          */
         sourceMapper().newSourceLoader(mapping());
-
-        settings.getMode().validateMapping(mappingLookup);
         if (settings.getIndexSortConfig().hasIndexSort() && mappers().nestedLookup() != NestedLookup.EMPTY) {
             throw new IllegalArgumentException("cannot have nested fields when index sort is activated");
         }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
index a41298014a19..7a33cc113a70 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -8,23 +8,19 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.regex.Regex;
-import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
@@ -39,7 +35,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.function.Consumer;
-import java.util.function.Function;
 
 /**
  * A parser for documents
@@ -47,20 +42,11 @@
 public final class DocumentParser {
 
     private final XContentParserConfiguration parserConfiguration;
-    private final Function<DateFormatter, MappingParserContext> dateParserContext;
-    private final IndexSettings indexSettings;
-    private final IndexAnalyzers indexAnalyzers;
-
-    DocumentParser(
-        XContentParserConfiguration parserConfiguration,
-        Function<DateFormatter, MappingParserContext> dateParserContext,
-        IndexSettings indexSettings,
-        IndexAnalyzers indexAnalyzers
-    ) {
-        this.dateParserContext = dateParserContext;
+    private final MappingParserContext mappingParserContext;
+
+    DocumentParser(XContentParserConfiguration parserConfiguration, MappingParserContext mappingParserContext) {
+        this.mappingParserContext = mappingParserContext;
         this.parserConfiguration = parserConfiguration;
-        this.indexSettings = indexSettings;
-        this.indexAnalyzers = indexAnalyzers;
     }
 
     /**
@@ -75,7 +61,7 @@ public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingL
         final InternalDocumentParserContext context;
         final XContentType xContentType = source.getXContentType();
         try (XContentParser parser = XContentHelper.createParser(parserConfiguration, source.source(), xContentType)) {
-            context = new InternalDocumentParserContext(mappingLookup, indexSettings, indexAnalyzers, dateParserContext, source, parser);
+            context = new InternalDocumentParserContext(mappingLookup, mappingParserContext, source, parser);
             validateStart(context.parser());
             MetadataFieldMapper[] metadataFieldsMappers = mappingLookup.getMapping().getSortedMetadataMappers();
             internalParseDocument(mappingLookup.getMapping().getRoot(), metadataFieldsMappers, context);
@@ -146,7 +132,7 @@ private static void executeIndexTimeScripts(DocumentParserContext context) {
             (ft, lookup, fto) -> ft.fielddataBuilder(
                 new FieldDataContext(context.indexSettings().getIndex().getName(), lookup, context.mappingLookup()::sourcePaths, fto)
             ).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()),
-            new SourceLookup.ReaderSourceProvider()
+            (ctx, doc) -> Source.fromBytes(context.sourceToParse().source())
         );
         // field scripts can be called both by the loop at the end of this method and via
         // the document reader, so to ensure that we don't run them multiple times we
@@ -245,7 +231,6 @@ static Mapping createDynamicUpdate(DocumentParserContext context) {
             rootBuilder.addRuntimeField(runtimeField);
         }
         RootObjectMapper root = rootBuilder.build(MapperBuilderContext.root(context.mappingLookup().isSourceSynthetic()));
-        root.fixRedundantIncludes();
         return context.mappingLookup().getMapping().mappingUpdate(root);
     }
 
@@ -267,7 +252,7 @@ static void parseObjectOrNested(DocumentParserContext context, ObjectMapper mapp
         }
 
         if (mapper.isNested()) {
-            context = nestedContext(context, (NestedObjectMapper) mapper);
+            context = context.createNestedContext((NestedObjectMapper) mapper);
         }
 
         // if we are at the end of the previous object, advance
@@ -384,32 +369,6 @@ private static void addFields(Version indexCreatedVersion, LuceneDocument nested
         }
     }
 
-    private static DocumentParserContext nestedContext(DocumentParserContext context, NestedObjectMapper mapper) {
-        context = context.createNestedContext(mapper.fullPath());
-        LuceneDocument nestedDoc = context.doc();
-        LuceneDocument parentDoc = nestedDoc.getParent();
-
-        // We need to add the uid or id to this nested Lucene document too,
-        // If we do not do this then when a document gets deleted only the root Lucene document gets deleted and
-        // not the nested Lucene documents! Besides the fact that we would have zombie Lucene documents, the ordering of
-        // documents inside the Lucene index (document blocks) will be incorrect, as nested documents of different root
-        // documents are then aligned with other root documents. This will lead tothe nested query, sorting, aggregations
-        // and inner hits to fail or yield incorrect results.
-        IndexableField idField = parentDoc.getField(IdFieldMapper.NAME);
-        if (idField != null) {
-            // We just need to store the id as indexed field, so that IndexWriter#deleteDocuments(term) can then
-            // delete it when the root document is deleted too.
-            // NOTE: we don't support nested fields in tsdb so it's safe to assume the standard id mapper.
-            nestedDoc.add(new Field(IdFieldMapper.NAME, idField.binaryValue(), ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE));
-        } else {
-            throw new IllegalStateException("The root document of a nested document should have an _id field");
-        }
-
-        Version version = context.indexSettings().getIndexVersionCreated();
-        nestedDoc.add(NestedPathFieldMapper.field(version, mapper.nestedTypePath()));
-        return context;
-    }
-
     static void parseObjectOrField(DocumentParserContext context, Mapper mapper) throws IOException {
         if (mapper instanceof ObjectMapper objectMapper) {
             parseObjectOrNested(context, objectMapper);
@@ -857,13 +816,11 @@ private static class InternalDocumentParserContext extends DocumentParserContext
 
         InternalDocumentParserContext(
             MappingLookup mappingLookup,
-            IndexSettings indexSettings,
-            IndexAnalyzers indexAnalyzers,
-            Function<DateFormatter, MappingParserContext> parserContext,
+            MappingParserContext mappingParserContext,
             SourceToParse source,
             XContentParser parser
         ) throws IOException {
-            super(mappingLookup, indexSettings, indexAnalyzers, parserContext, source);
+            super(mappingLookup, mappingParserContext, source);
             if (mappingLookup.getMapping().getRoot().subobjects()) {
                 this.parser = DotExpandingXContentParser.expandDots(parser, this.path::isWithinLeafObject);
             } else {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java
index 17ba0de2bd7b..d5ea689743ea 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
@@ -24,7 +25,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.function.Function;
 
 /**
  * Context used when parsing incoming documents. Holds everything that is needed to parse a document as well as
@@ -81,7 +81,7 @@ protected void addDoc(LuceneDocument doc) {
     private final IndexSettings indexSettings;
     private final IndexAnalyzers indexAnalyzers;
     private final MappingLookup mappingLookup;
-    private final Function<DateFormatter, MappingParserContext> parserContextFunction;
+    private final MappingParserContext mappingParserContext;
     private final SourceToParse sourceToParse;
     private final Set<String> ignoredFields;
     private final List<Mapper> dynamicMappers;
@@ -97,7 +97,7 @@ private DocumentParserContext(DocumentParserContext in) {
         this.mappingLookup = in.mappingLookup;
         this.indexSettings = in.indexSettings;
         this.indexAnalyzers = in.indexAnalyzers;
-        this.parserContextFunction = in.parserContextFunction;
+        this.mappingParserContext = in.mappingParserContext;
         this.sourceToParse = in.sourceToParse;
         this.ignoredFields = in.ignoredFields;
         this.dynamicMappers = in.dynamicMappers;
@@ -110,24 +110,18 @@ private DocumentParserContext(DocumentParserContext in) {
         this.dimensions = in.dimensions;
     }
 
-    protected DocumentParserContext(
-        MappingLookup mappingLookup,
-        IndexSettings indexSettings,
-        IndexAnalyzers indexAnalyzers,
-        Function<DateFormatter, MappingParserContext> parserContextFunction,
-        SourceToParse source
-    ) {
+    protected DocumentParserContext(MappingLookup mappingLookup, MappingParserContext mappingParserContext, SourceToParse source) {
         this.mappingLookup = mappingLookup;
-        this.indexSettings = indexSettings;
-        this.indexAnalyzers = indexAnalyzers;
-        this.parserContextFunction = parserContextFunction;
+        this.indexSettings = mappingParserContext.getIndexSettings();
+        this.indexAnalyzers = mappingParserContext.getIndexAnalyzers();
+        this.mappingParserContext = mappingParserContext;
         this.sourceToParse = source;
         this.ignoredFields = new HashSet<>();
         this.dynamicMappers = new ArrayList<>();
         this.newFieldsSeen = new HashSet<>();
         this.dynamicObjectMappers = new HashMap<>();
         this.dynamicRuntimeFields = new ArrayList<>();
-        this.dimensions = indexSettings.getMode().buildDocumentDimensions(indexSettings);
+        this.dimensions = this.indexSettings.getMode().buildDocumentDimensions(this.indexSettings);
     }
 
     public final IndexSettings indexSettings() {
@@ -151,7 +145,7 @@ public final MetadataFieldMapper getMetadataMapper(String mapperName) {
     }
 
     public final MappingParserContext dynamicTemplateParserContext(DateFormatter dateFormatter) {
-        return parserContextFunction.apply(dateFormatter);
+        return mappingParserContext.createDynamicTemplateContext(dateFormatter);
     }
 
     public final SourceToParse sourceToParse() {
@@ -321,12 +315,28 @@ public boolean isWithinCopyTo() {
     /**
      * Return a new context that will be used within a nested document.
      */
-    public final DocumentParserContext createNestedContext(String fullPath) {
+    public final DocumentParserContext createNestedContext(NestedObjectMapper nestedMapper) {
         if (isWithinCopyTo()) {
             // nested context will already have been set up for copy_to fields
             return this;
         }
-        final LuceneDocument doc = new LuceneDocument(fullPath, doc());
+        final LuceneDocument doc = new LuceneDocument(nestedMapper.fullPath(), doc());
+        // We need to add the uid or id to this nested Lucene document too,
+        // If we do not do this then when a document gets deleted only the root Lucene document gets deleted and
+        // not the nested Lucene documents! Besides the fact that we would have zombie Lucene documents, the ordering of
+        // documents inside the Lucene index (document blocks) will be incorrect, as nested documents of different root
+        // documents are then aligned with other root documents. This will lead to the nested query, sorting, aggregations
+        // and inner hits to fail or yield incorrect results.
+        IndexableField idField = doc.getParent().getField(IdFieldMapper.NAME);
+        if (idField != null) {
+            // We just need to store the id as indexed field, so that IndexWriter#deleteDocuments(term) can then
+            // delete it when the root document is deleted too.
+            // NOTE: we don't support nested fields in tsdb so it's safe to assume the standard id mapper.
+            doc.add(new Field(IdFieldMapper.NAME, idField.binaryValue(), ProvidedIdFieldMapper.Defaults.NESTED_FIELD_TYPE));
+        } else {
+            throw new IllegalStateException("The root document of a nested document should have an _id field");
+        }
+        doc.add(NestedPathFieldMapper.field(indexSettings().getIndexVersionCreated(), nestedMapper.nestedTypePath()));
         addDoc(doc);
         return switchDoc(doc);
     }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptFieldType.java
index bec343ed474e..6adf7a804450 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptFieldType.java
@@ -47,9 +47,10 @@ AbstractScriptFieldType<?> createFieldType(
             String name,
             DoubleFieldScript.Factory factory,
             Script script,
-            Map<String, String> meta
+            Map<String, String> meta,
+            OnScriptError onScriptError
         ) {
-            return new DoubleScriptFieldType(name, factory, script, meta);
+            return new DoubleScriptFieldType(name, factory, script, meta, onScriptError);
         }
 
         @Override
@@ -67,10 +68,16 @@ public static RuntimeField sourceOnly(String name) {
         return new Builder(name).createRuntimeField(DoubleFieldScript.PARSE_FROM_SOURCE);
     }
 
-    DoubleScriptFieldType(String name, DoubleFieldScript.Factory scriptFactory, Script script, Map<String, String> meta) {
+    DoubleScriptFieldType(
+        String name,
+        DoubleFieldScript.Factory scriptFactory,
+        Script script,
+        Map<String, String> meta,
+        OnScriptError onScriptError
+    ) {
         super(
             name,
-            searchLookup -> scriptFactory.newFactory(name, script.getParams(), searchLookup),
+            searchLookup -> scriptFactory.newFactory(name, script.getParams(), searchLookup, onScriptError),
             script,
             scriptFactory.isResultDeterministic(),
             meta
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java
index 6acd6128a9eb..4cd6f7f0ba88 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java
@@ -328,7 +328,8 @@ public void newDynamicLongField(DocumentParserContext context, String name) thro
                     NumberFieldMapper.NumberType.LONG,
                     ScriptCompiler.NONE,
                     context.indexSettings().getSettings(),
-                    context.indexSettings().getIndexVersionCreated()
+                    context.indexSettings().getIndexVersionCreated(),
+                    context.indexSettings().getMode()
                 ),
                 context
             );
@@ -345,7 +346,8 @@ public void newDynamicDoubleField(DocumentParserContext context, String name) th
                     NumberFieldMapper.NumberType.FLOAT,
                     ScriptCompiler.NONE,
                     context.indexSettings().getSettings(),
-                    context.indexSettings().getIndexVersionCreated()
+                    context.indexSettings().getIndexVersionCreated(),
+                    context.indexSettings().getMode()
                 ),
                 context
             );
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
index e3026baa536c..a51efd2fd4b3 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
@@ -70,7 +70,7 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
     protected final MultiFields multiFields;
     protected final CopyTo copyTo;
     protected final boolean hasScript;
-    protected final String onScriptError;
+    protected final OnScriptError onScriptError;
 
     /**
      * @param simpleName        the leaf name of the mapper
@@ -96,7 +96,7 @@ protected FieldMapper(
         MultiFields multiFields,
         CopyTo copyTo,
         boolean hasScript,
-        String onScriptError
+        OnScriptError onScriptError
     ) {
         super(simpleName);
         // could be blank but not empty on indices created < 8.6.0
@@ -247,7 +247,7 @@ public final void executeScript(
         try {
             indexScriptValues(searchLookup, readerContext, doc, documentParserContext);
         } catch (Exception e) {
-            if ("continue".equals(onScriptError)) {
+            if (onScriptError == OnScriptError.CONTINUE) {
                 documentParserContext.addIgnoredField(name());
             } else {
                 throw new MapperParsingException("Error executing script on field [" + name() + "]", e);
@@ -816,6 +816,23 @@ public static Parameter<Boolean> boolParam(
             );
         }
 
+        public static Parameter<Boolean> boolParam(
+            String name,
+            boolean updateable,
+            Function<FieldMapper, Boolean> initializer,
+            Supplier<Boolean> defaultValue
+        ) {
+            return new Parameter<>(
+                name,
+                updateable,
+                defaultValue,
+                (n, c, o) -> XContentMapValues.nodeBooleanValue(o),
+                initializer,
+                XContentBuilder::field,
+                Objects::toString
+            );
+        }
+
         /**
          * Defines a parameter that takes the values {@code true} or {@code false}, and will always serialize
          * its value if configured.
@@ -938,12 +955,12 @@ public static <T extends Enum<T>> Parameter<T> enumParam(
         /**
          * Defines a parameter that takes one of a restricted set of values from an enumeration.
          *
-         * @param name          the parameter name
-         * @param updateable    whether the parameter can be changed by a mapping update
-         * @param initializer   a function that reads the parameter value from an existing mapper
-         * @param defaultValue  the default value, to be used if the parameter is undefined in a mapping
-         * @param enumClass     the enumeration class the parameter takes values from
-         * @param values        the set of values that the parameter can take
+         * @param name            the parameter name
+         * @param updateable      whether the parameter can be changed by a mapping update
+         * @param initializer     a function that reads the parameter value from an existing mapper
+         * @param defaultValue    the default value, to be used if the parameter is undefined in a mapping
+         * @param enumClass       the enumeration class the parameter takes values from
+         * @param acceptedValues  the set of values that the parameter can take
          */
         public static <T extends Enum<T>> Parameter<T> restrictedEnumParam(
             String name,
@@ -951,23 +968,28 @@ public static <T extends Enum<T>> Parameter<T> restrictedEnumParam(
             Function<FieldMapper, T> initializer,
             T defaultValue,
             Class<T> enumClass,
-            Set<T> values
+            Set<T> acceptedValues
         ) {
-            assert values.size() > 0;
-            return new Parameter<T>(name, updateable, () -> defaultValue, (n, c, o) -> {
+            assert acceptedValues.size() > 0;
+            return new Parameter<>(name, updateable, () -> defaultValue, (n, c, o) -> {
                 if (o == null) {
                     return defaultValue;
                 }
-                try {
-                    @SuppressWarnings("unchecked")
-                    T enumValue = Enum.valueOf(enumClass, (String) o);
-                    return enumValue;
-                } catch (IllegalArgumentException e) {
-                    throw new MapperParsingException("Unknown value [" + o + "] for field [" + name + "] - accepted values are " + values);
+                EnumSet<T> enumSet = EnumSet.allOf(enumClass);
+                for (T t : enumSet) {
+                    // the string representation may differ from the actual name of the enum type (e.g. lowercase vs uppercase)
+                    if (t.toString().equals(o.toString())) {
+                        return t;
+                    }
                 }
+                throw new MapperParsingException(
+                    "Unknown value [" + o + "] for field [" + name + "] - accepted values are " + acceptedValues
+                );
             }, initializer, XContentBuilder::field, Objects::toString).addValidator(v -> {
-                if (v != null && values.contains(v) == false) {
-                    throw new MapperParsingException("Unknown value [" + v + "] for field [" + name + "] - accepted values are " + values);
+                if (v != null && acceptedValues.contains(v) == false) {
+                    throw new MapperParsingException(
+                        "Unknown value [" + v + "] for field [" + name + "] - accepted values are " + acceptedValues
+                    );
                 }
             });
         }
@@ -1037,6 +1059,10 @@ public static Parameter<Boolean> indexParam(Function<FieldMapper, Boolean> initi
             return Parameter.boolParam("index", false, initializer, defaultValue);
         }
 
+        public static Parameter<Boolean> indexParam(Function<FieldMapper, Boolean> initializer, Supplier<Boolean> defaultValue) {
+            return Parameter.boolParam("index", false, initializer, defaultValue);
+        }
+
         public static Parameter<Boolean> storeParam(Function<FieldMapper, Boolean> initializer, boolean defaultValue) {
             return Parameter.boolParam("store", false, initializer, defaultValue);
         }
@@ -1069,29 +1095,12 @@ public static Parameter<Script> scriptParam(Function<FieldMapper, Script> initia
          * @param dependentScriptParam the corresponding required script parameter
          * @return a new on_error_script parameter
          */
-        public static Parameter<String> onScriptErrorParam(
-            Function<FieldMapper, String> initializer,
+        public static Parameter<OnScriptError> onScriptErrorParam(
+            Function<FieldMapper, OnScriptError> initializer,
             Parameter<Script> dependentScriptParam
         ) {
-            return new Parameter<>(
-                "on_script_error",
-                true,
-                () -> "fail",
-                (n, c, o) -> XContentMapValues.nodeStringValue(o),
-                initializer,
-                XContentBuilder::field,
-                Function.identity()
-            ).addValidator(v -> {
-                switch (v) {
-                    case "fail":
-                    case "continue":
-                        return;
-                    default:
-                        throw new MapperParsingException(
-                            "Unknown value [" + v + "] for field [on_script_error] - accepted values are [fail, continue]"
-                        );
-                }
-            }).requiresParameter(dependentScriptParam);
+            return Parameter.enumParam("on_script_error", true, initializer, OnScriptError.FAIL, OnScriptError.class)
+                .requiresParameter(dependentScriptParam);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java
index 9d3f96a4ac0d..5601093b6de6 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java
@@ -78,7 +78,7 @@ public static class Builder extends FieldMapper.Builder {
         final Parameter<Boolean> hasDocValues = Parameter.docValuesParam(m -> builder(m).hasDocValues.get(), true);
         final Parameter<Boolean> stored = Parameter.storeParam(m -> builder(m).stored.get(), false);
         private final Parameter<Script> script = Parameter.scriptParam(m -> builder(m).script.get());
-        private final Parameter<String> onScriptError = Parameter.onScriptErrorParam(m -> builder(m).onScriptError.get(), script);
+        private final Parameter<OnScriptError> onScriptError = Parameter.onScriptErrorParam(m -> builder(m).onScriptError.get(), script);
         final Parameter<Map<String, String>> meta = Parameter.metaParam();
 
         private final ScriptCompiler scriptCompiler;
@@ -143,7 +143,7 @@ private FieldValues<GeoPoint> scriptValues() {
             GeoPointFieldScript.Factory factory = scriptCompiler.compile(this.script.get(), GeoPointFieldScript.CONTEXT);
             return factory == null
                 ? null
-                : (lookup, ctx, doc, consumer) -> factory.newFactory(name, script.get().getParams(), lookup)
+                : (lookup, ctx, doc, consumer) -> factory.newFactory(name, script.get().getParams(), lookup, OnScriptError.FAIL)
                     .newInstance(ctx)
                     .runForDoc(doc, consumer);
         }
@@ -389,7 +389,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext
                     name(),
                     CoreValuesSourceType.GEOPOINT,
                     valueFetcher(sourcePaths, null, null),
-                    searchLookup.source(),
+                    searchLookup,
                     GeoPointDocValuesField::new
                 );
             }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java
index bcb3cf313d3c..6e21c777a524 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java
@@ -50,9 +50,10 @@ AbstractScriptFieldType<?> createFieldType(
             String name,
             GeoPointFieldScript.Factory factory,
             Script script,
-            Map<String, String> meta
+            Map<String, String> meta,
+            OnScriptError onScriptError
         ) {
-            return new GeoPointScriptFieldType(name, factory, getScript(), meta());
+            return new GeoPointScriptFieldType(name, factory, getScript(), meta(), onScriptError);
         }
 
         @Override
@@ -66,10 +67,16 @@ GeoPointFieldScript.Factory getCompositeLeafFactory(Function<SearchLookup, Compo
         }
     });
 
-    GeoPointScriptFieldType(String name, GeoPointFieldScript.Factory scriptFactory, Script script, Map<String, String> meta) {
+    GeoPointScriptFieldType(
+        String name,
+        GeoPointFieldScript.Factory scriptFactory,
+        Script script,
+        Map<String, String> meta,
+        OnScriptError onScriptError
+    ) {
         super(
             name,
-            searchLookup -> scriptFactory.newFactory(name, script.getParams(), searchLookup),
+            searchLookup -> scriptFactory.newFactory(name, script.getParams(), searchLookup, onScriptError),
             script,
             scriptFactory.isResultDeterministic(),
             meta
@@ -154,7 +161,7 @@ public static Function<LeafReaderContext, AbstractLongFieldScript> valuesEncoded
     ) {
         return ctx -> {
             GeoPointFieldScript script = delegateLeafFactory.apply(ctx);
-            return new AbstractLongFieldScript(name, Map.of(), lookup, ctx) {
+            return new AbstractLongFieldScript(name, Map.of(), lookup, OnScriptError.FAIL, ctx) {
                 private int docId;
 
                 @Override
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
index e7cd1c2ed7dc..68b6fc55c738 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
@@ -71,7 +71,7 @@ public static class Builder extends FieldMapper.Builder {
             .acceptsNull();
 
         private final Parameter<Script> script = Parameter.scriptParam(m -> toType(m).script);
-        private final Parameter<String> onScriptError = Parameter.onScriptErrorParam(m -> toType(m).onScriptError, script);
+        private final Parameter<OnScriptError> onScriptError = Parameter.onScriptErrorParam(m -> toType(m).onScriptError, script);
 
         private final Parameter<Map<String, String>> meta = Parameter.metaParam();
         private final Parameter<Boolean> dimension;
@@ -145,7 +145,7 @@ private FieldValues<InetAddress> scriptValues() {
             IpFieldScript.Factory factory = scriptCompiler.compile(this.script.get(), IpFieldScript.CONTEXT);
             return factory == null
                 ? null
-                : (lookup, ctx, doc, consumer) -> factory.newFactory(name, script.get().getParams(), lookup)
+                : (lookup, ctx, doc, consumer) -> factory.newFactory(name, script.get().getParams(), lookup, OnScriptError.FAIL)
                     .newInstance(ctx)
                     .runForDoc(doc, consumer);
         }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java
index 3b706a574816..9ed1493eea14 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java
@@ -45,8 +45,14 @@ public final class IpScriptFieldType extends AbstractScriptFieldType<IpFieldScri
 
     public static final RuntimeField.Parser PARSER = new RuntimeField.Parser(name -> new Builder<>(name, IpFieldScript.CONTEXT) {
         @Override
-        AbstractScriptFieldType<?> createFieldType(String name, IpFieldScript.Factory factory, Script script, Map<String, String> meta) {
-            return new IpScriptFieldType(name, factory, getScript(), meta());
+        AbstractScriptFieldType<?> createFieldType(
+            String name,
+            IpFieldScript.Factory factory,
+            Script script,
+            Map<String, String> meta,
+            OnScriptError onScriptError
+        ) {
+            return new IpScriptFieldType(name, factory, getScript(), meta(), onScriptError);
         }
 
         @Override
@@ -60,10 +66,16 @@ IpFieldScript.Factory getCompositeLeafFactory(Function<SearchLookup, CompositeFi
         }
     });
 
-    IpScriptFieldType(String name, IpFieldScript.Factory scriptFactory, Script script, Map<String, String> meta) {
+    IpScriptFieldType(
+        String name,
+        IpFieldScript.Factory scriptFactory,
+        Script script,
+        Map<String, String> meta,
+        OnScriptError onScriptError
+    ) {
         super(
             name,
-            searchLookup -> scriptFactory.newFactory(name, script.getParams(), searchLookup),
+            searchLookup -> scriptFactory.newFactory(name, script.getParams(), searchLookup, onScriptError),
             script,
             scriptFactory.isResultDeterministic(),
             meta
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
index 8ebf6c5c5fc6..b6b6bfe5a855 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
@@ -173,7 +173,7 @@ public static class Builder extends FieldMapper.Builder {
         private final Parameter<Map<String, String>> meta = Parameter.metaParam();
 
         private final Parameter<Script> script = Parameter.scriptParam(m -> toType(m).script);
-        private final Parameter<String> onScriptError = Parameter.onScriptErrorParam(m -> toType(m).onScriptError, script);
+        private final Parameter<OnScriptError> onScriptError = Parameter.onScriptErrorParam(m -> toType(m).onScriptError, script);
         private final Parameter<Boolean> dimension;
 
         private final IndexAnalyzers indexAnalyzers;
@@ -245,7 +245,7 @@ private FieldValues<String> scriptValues() {
             StringFieldScript.Factory scriptFactory = scriptCompiler.compile(script.get(), StringFieldScript.CONTEXT);
             return scriptFactory == null
                 ? null
-                : (lookup, ctx, doc, consumer) -> scriptFactory.newFactory(name, script.get().getParams(), lookup)
+                : (lookup, ctx, doc, consumer) -> scriptFactory.newFactory(name, script.get().getParams(), lookup, OnScriptError.FAIL)
                     .newInstance(ctx)
                     .runForDoc(doc, consumer);
         }
@@ -735,7 +735,7 @@ protected BytesRef storedToBytesRef(Object stored) {
                 name(),
                 CoreValuesSourceType.KEYWORD,
                 sourceValueFetcher(sourcePaths),
-                fieldDataContext.lookupSupplier().get().source(),
+                fieldDataContext.lookupSupplier().get(),
                 KeywordDocValuesField::new
             );
         }
@@ -995,44 +995,47 @@ private void indexValue(DocumentParserContext context, String value) {
         }
 
         value = normalizeValue(fieldType().normalizer(), name(), value);
-        if (fieldType().isDimension()) {
-            context.getDimensions().addString(fieldType().name(), value);
-        }
 
         // convert to utf8 only once before feeding postings/dv/stored fields
         final BytesRef binaryValue = new BytesRef(value);
 
-        // If the UTF8 encoding of the field value is bigger than the max length 32766, Lucene fill fail the indexing request and, to roll
-        // back the changes, will mark the (possibly partially indexed) document as deleted. This results in deletes, even in an append-only
-        // workload, which in turn leads to slower merges, as these will potentially have to fall back to MergeStrategy.DOC instead of
-        // MergeStrategy.BULK. To avoid this, we do a preflight check here before indexing the document into Lucene.
-        if (binaryValue.length > MAX_TERM_LENGTH) {
-            byte[] prefix = new byte[30];
-            System.arraycopy(binaryValue.bytes, binaryValue.offset, prefix, 0, 30);
-            String msg = "Document contains at least one immense term in field=\""
-                + fieldType().name()
-                + "\" (whose "
-                + "UTF8 encoding is longer than the max length "
-                + MAX_TERM_LENGTH
-                + "), all of which were "
-                + "skipped. Please correct the analyzer to not produce such terms. The prefix of the first immense "
-                + "term is: '"
-                + Arrays.toString(prefix)
-                + "...'";
-            throw new IllegalArgumentException(msg);
+        if (fieldType().isDimension()) {
+            context.getDimensions().addString(fieldType().name(), binaryValue);
         }
 
-        if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
-            Field field = new KeywordField(fieldType().name(), binaryValue, fieldType);
-            context.doc().add(field);
+        if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored() || fieldType().hasDocValues()) {
+            // If the UTF8 encoding of the field value is bigger than the max length 32766, Lucene fill fail the indexing request and, to
+            // roll back the changes, will mark the (possibly partially indexed) document as deleted. This results in deletes, even in an
+            // append-only workload, which in turn leads to slower merges, as these will potentially have to fall back to MergeStrategy.DOC
+            // instead of MergeStrategy.BULK. To avoid this, we do a preflight check here before indexing the document into Lucene.
+            if (binaryValue.length > MAX_TERM_LENGTH) {
+                byte[] prefix = new byte[30];
+                System.arraycopy(binaryValue.bytes, binaryValue.offset, prefix, 0, 30);
+                String msg = "Document contains at least one immense term in field=\""
+                    + fieldType().name()
+                    + "\" (whose "
+                    + "UTF8 encoding is longer than the max length "
+                    + MAX_TERM_LENGTH
+                    + "), all of which were "
+                    + "skipped. Please correct the analyzer to not produce such terms. The prefix of the first immense "
+                    + "term is: '"
+                    + Arrays.toString(prefix)
+                    + "...'";
+                throw new IllegalArgumentException(msg);
+            }
+
+            if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
+                Field field = new KeywordField(fieldType().name(), binaryValue, fieldType);
+                context.doc().add(field);
 
-            if (fieldType().hasDocValues() == false && fieldType.omitNorms()) {
-                context.addToFieldNames(fieldType().name());
+                if (fieldType().hasDocValues() == false && fieldType.omitNorms()) {
+                    context.addToFieldNames(fieldType().name());
+                }
             }
-        }
 
-        if (fieldType().hasDocValues()) {
-            context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue));
+            if (fieldType().hasDocValues()) {
+                context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue));
+            }
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptFieldType.java
index 25d8fa04d023..8239ed3347c6 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptFieldType.java
@@ -54,9 +54,10 @@ AbstractScriptFieldType<?> createFieldType(
             String name,
             StringFieldScript.Factory factory,
             Script script,
-            Map<String, String> meta
+            Map<String, String> meta,
+            OnScriptError onScriptError
         ) {
-            return new KeywordScriptFieldType(name, factory, script, meta);
+            return new KeywordScriptFieldType(name, factory, script, meta, onScriptError);
         }
 
         @Override
@@ -74,10 +75,16 @@ public static RuntimeField sourceOnly(String name) {
         return new Builder(name).createRuntimeField(StringFieldScript.PARSE_FROM_SOURCE);
     }
 
-    public KeywordScriptFieldType(String name, StringFieldScript.Factory scriptFactory, Script script, Map<String, String> meta) {
+    public KeywordScriptFieldType(
+        String name,
+        StringFieldScript.Factory scriptFactory,
+        Script script,
+        Map<String, String> meta,
+        OnScriptError onScriptError
+    ) {
         super(
             name,
-            searchLookup -> scriptFactory.newFactory(name, script.getParams(), searchLookup),
+            searchLookup -> scriptFactory.newFactory(name, script.getParams(), searchLookup, onScriptError),
             script,
             scriptFactory.isResultDeterministic(),
             meta
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java
index 7314071b8550..ca1147c6796d 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java
@@ -43,8 +43,14 @@ private static class Builder extends AbstractScriptFieldType.Builder<LongFieldSc
         }
 
         @Override
-        AbstractScriptFieldType<?> createFieldType(String name, LongFieldScript.Factory factory, Script script, Map<String, String> meta) {
-            return new LongScriptFieldType(name, factory, script, meta);
+        AbstractScriptFieldType<?> createFieldType(
+            String name,
+            LongFieldScript.Factory factory,
+            Script script,
+            Map<String, String> meta,
+            OnScriptError onScriptError
+        ) {
+            return new LongScriptFieldType(name, factory, script, meta, onScriptError);
         }
 
         @Override
@@ -62,10 +68,16 @@ public static RuntimeField sourceOnly(String name) {
         return new Builder(name).createRuntimeField(LongFieldScript.PARSE_FROM_SOURCE);
     }
 
-    public LongScriptFieldType(String name, LongFieldScript.Factory scriptFactory, Script script, Map<String, String> meta) {
+    public LongScriptFieldType(
+        String name,
+        LongFieldScript.Factory scriptFactory,
+        Script script,
+        Map<String, String> meta,
+        OnScriptError onScriptError
+    ) {
         super(
             name,
-            searchLookup -> scriptFactory.newFactory(name, script.getParams(), searchLookup),
+            searchLookup -> scriptFactory.newFactory(name, script.getParams(), searchLookup, onScriptError),
             script,
             scriptFactory.isResultDeterministic(),
             meta
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LookupRuntimeFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/LookupRuntimeFieldType.java
index cc5dd40504da..beed384f4424 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/LookupRuntimeFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/LookupRuntimeFieldType.java
@@ -159,7 +159,8 @@ protected RuntimeField createRuntimeField(MappingParserContext parserContext) {
         protected RuntimeField createChildRuntimeField(
             MappingParserContext parserContext,
             String parentName,
-            Function<SearchLookup, CompositeFieldScript.LeafFactory> parentScriptFactory
+            Function<SearchLookup, CompositeFieldScript.LeafFactory> parentScriptFactory,
+            OnScriptError onScriptError
         ) {
             return createRuntimeField(parserContext);
         }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java
index 8ed6bf3d1db7..f1daf17f3ee6 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java
@@ -16,7 +16,7 @@
 /**
  * Holds context for building Mapper objects from their Builders
  */
-public final class MapperBuilderContext {
+public class MapperBuilderContext {
 
     /**
      * The root context, to be used when building a tree of mappers
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
index bdb4af089e43..7a96d7c8e7d4 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexSettings;
@@ -126,7 +125,7 @@ public enum MergeReason {
     private final DocumentParser documentParser;
     private final Version indexVersionCreated;
     private final MapperRegistry mapperRegistry;
-    private final Supplier<MappingParserContext> parserContextSupplier;
+    private final MappingParserContext mappingParserContext;
 
     private volatile DocumentMapper mapper;
 
@@ -144,30 +143,23 @@ public MapperService(
         this.indexVersionCreated = indexSettings.getIndexVersionCreated();
         this.indexAnalyzers = indexAnalyzers;
         this.mapperRegistry = mapperRegistry;
-        Function<DateFormatter, MappingParserContext> parserContextFunction = dateFormatter -> new MappingParserContext(
+        this.mappingParserContext = new MappingParserContext(
             similarityService::getSimilarity,
             type -> mapperRegistry.getMapperParser(type, indexVersionCreated),
             mapperRegistry.getRuntimeFieldParsers()::get,
             indexVersionCreated,
             searchExecutionContextSupplier,
-            dateFormatter,
             scriptCompiler,
             indexAnalyzers,
             indexSettings,
             idFieldMapper
         );
-        this.documentParser = new DocumentParser(
-            parserConfiguration,
-            dateFormatter -> new MappingParserContext.DynamicTemplateParserContext(parserContextFunction.apply(dateFormatter)),
-            indexSettings,
-            indexAnalyzers
-        );
+        this.documentParser = new DocumentParser(parserConfiguration, this.mappingParserContext);
         Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers = mapperRegistry.getMetadataMapperParsers(
             indexSettings.getIndexVersionCreated()
         );
-        this.parserContextSupplier = () -> parserContextFunction.apply(null);
         this.mappingParser = new MappingParser(
-            parserContextSupplier,
+            mappingParserContext,
             metadataMapperParsers,
             this::getMetadataMappers,
             this::resolveDocumentType
@@ -183,7 +175,7 @@ public IndexAnalyzers getIndexAnalyzers() {
     }
 
     public MappingParserContext parserContext() {
-        return parserContextSupplier.get();
+        return this.mappingParserContext;
     }
 
     /**
@@ -360,7 +352,6 @@ public DocumentMapper merge(String type, CompressedXContent mappingSource, Merge
 
     private DocumentMapper newDocumentMapper(Mapping mapping, MergeReason reason, CompressedXContent mappingSource) {
         DocumentMapper newMapper = new DocumentMapper(documentParser, mapping, mappingSource);
-        newMapper.mapping().getRoot().fixRedundantIncludes();
         newMapper.validate(indexSettings, reason != MergeReason.MAPPING_RECOVERY);
         return newMapper;
     }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java
index e112a676ad38..a7b18e1e6213 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
-import org.elasticsearch.search.lookup.SourceLookup;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -492,18 +491,4 @@ public void validateDoesNotShadow(String name) {
             throw new MapperParsingException("Field [" + name + "] attempted to shadow a time_series_metric");
         }
     }
-
-    /**
-     * Returns a SourceProvider describing how to read the source. If using synthetic source, returns the null source provider
-     * expecting the source to either be provided later by a fetch phase or not be accessed at all (as in scripts).
-     * @return
-     */
-    public SourceLookup.SourceProvider getSourceProvider() {
-        SourceFieldMapper sourceMapper = (SourceFieldMapper) getMapper("_source");
-        if (sourceMapper == null || sourceMapper.isSynthetic() == false) {
-            return new SourceLookup.ReaderSourceProvider();
-        } else {
-            return new SourceLookup.NullSourceProvider();
-        }
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java
index 4c12d6eb5c07..c6b780a94c55 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java
@@ -25,18 +25,18 @@
  * Parser for {@link Mapping} provided in {@link CompressedXContent} format
  */
 public final class MappingParser {
-    private final Supplier<MappingParserContext> parserContextSupplier;
+    private final MappingParserContext mappingParserContext;
     private final Supplier<Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper>> metadataMappersSupplier;
     private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers;
     private final Function<String, String> documentTypeResolver;
 
     MappingParser(
-        Supplier<MappingParserContext> parserContextSupplier,
+        MappingParserContext mappingParserContext,
         Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers,
         Supplier<Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper>> metadataMappersSupplier,
         Function<String, String> documentTypeResolver
     ) {
-        this.parserContextSupplier = parserContextSupplier;
+        this.mappingParserContext = mappingParserContext;
         this.metadataMapperParsers = metadataMapperParsers;
         this.metadataMappersSupplier = metadataMappersSupplier;
         this.documentTypeResolver = documentTypeResolver;
@@ -97,9 +97,8 @@ Mapping parse(@Nullable String type, CompressedXContent source) throws MapperPar
     }
 
     private Mapping parse(String type, Map<String, Object> mapping) throws MapperParsingException {
-        MappingParserContext parserContext = parserContextSupplier.get();
 
-        RootObjectMapper.Builder rootObjectMapper = RootObjectMapper.parse(type, mapping, parserContext);
+        RootObjectMapper.Builder rootObjectMapper = RootObjectMapper.parse(type, mapping, this.mappingParserContext);
 
         Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> metadataMappers = metadataMappersSupplier.get();
         Map<String, Object> meta = null;
@@ -119,7 +118,7 @@ private Mapping parse(String type, Map<String, Object> mapping) throws MapperPar
                 }
                 @SuppressWarnings("unchecked")
                 Map<String, Object> fieldNodeMap = (Map<String, Object>) fieldNode;
-                MetadataFieldMapper metadataFieldMapper = typeParser.parse(fieldName, fieldNodeMap, parserContext)
+                MetadataFieldMapper metadataFieldMapper = typeParser.parse(fieldName, fieldNodeMap, this.mappingParserContext)
                     .build(MapperBuilderContext.forMetadata());
                 metadataMappers.put(metadataFieldMapper.getClass(), metadataFieldMapper);
                 assert fieldNodeMap.isEmpty();
@@ -148,7 +147,7 @@ private Mapping parse(String type, Map<String, Object> mapping) throws MapperPar
              */
             meta = Collections.unmodifiableMap(new HashMap<>(removed));
         }
-        if (parserContext.indexVersionCreated().isLegacyIndexVersion() == false) {
+        if (this.mappingParserContext.indexVersionCreated().isLegacyIndexVersion() == false) {
             // legacy indices are allowed to have extra definitions that we ignore (we will drop them on import)
             checkNoRemainingFields(mapping, "Root mapping definition has unsupported parameters: ");
         }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java
index f2df66b64958..bd5377adb091 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java
@@ -31,7 +31,6 @@ public class MappingParserContext {
     private final Function<String, RuntimeField.Parser> runtimeFieldParsers;
     private final Version indexVersionCreated;
     private final Supplier<SearchExecutionContext> searchExecutionContextSupplier;
-    private final DateFormatter dateFormatter;
     private final ScriptCompiler scriptCompiler;
     private final IndexAnalyzers indexAnalyzers;
     private final IndexSettings indexSettings;
@@ -43,7 +42,6 @@ public MappingParserContext(
         Function<String, RuntimeField.Parser> runtimeFieldParsers,
         Version indexVersionCreated,
         Supplier<SearchExecutionContext> searchExecutionContextSupplier,
-        DateFormatter dateFormatter,
         ScriptCompiler scriptCompiler,
         IndexAnalyzers indexAnalyzers,
         IndexSettings indexSettings,
@@ -54,7 +52,6 @@ public MappingParserContext(
         this.runtimeFieldParsers = runtimeFieldParsers;
         this.indexVersionCreated = indexVersionCreated;
         this.searchExecutionContextSupplier = searchExecutionContextSupplier;
-        this.dateFormatter = dateFormatter;
         this.scriptCompiler = scriptCompiler;
         this.indexAnalyzers = indexAnalyzers;
         this.indexSettings = indexSettings;
@@ -103,7 +100,7 @@ public Supplier<SearchExecutionContext> searchExecutionContext() {
      * If {@code null}, then date fields will default to {@link DateFieldMapper#DEFAULT_DATE_TIME_FORMATTER}.
      */
     public DateFormatter getDateFormatter() {
-        return dateFormatter;
+        return null;
     }
 
     public boolean isWithinMultiField() {
@@ -124,8 +121,8 @@ public ScriptCompiler scriptCompiler() {
         return scriptCompiler;
     }
 
-    static MappingParserContext createMultiFieldContext(MappingParserContext in) {
-        return new MultiFieldParserContext(in);
+    public MappingParserContext createMultiFieldContext() {
+        return new MultiFieldParserContext(this);
     }
 
     private static class MultiFieldParserContext extends MappingParserContext {
@@ -136,7 +133,6 @@ private static class MultiFieldParserContext extends MappingParserContext {
                 in.runtimeFieldParsers,
                 in.indexVersionCreated,
                 in.searchExecutionContextSupplier,
-                in.dateFormatter,
                 in.scriptCompiler,
                 in.indexAnalyzers,
                 in.indexSettings,
@@ -150,20 +146,32 @@ public boolean isWithinMultiField() {
         }
     }
 
-    static class DynamicTemplateParserContext extends MappingParserContext {
-        DynamicTemplateParserContext(MappingParserContext in) {
+    public MappingParserContext createDynamicTemplateContext(DateFormatter dateFormatter) {
+        return new DynamicTemplateParserContext(this, dateFormatter);
+    }
+
+    private static class DynamicTemplateParserContext extends MappingParserContext {
+
+        private final DateFormatter dateFormatter;
+
+        DynamicTemplateParserContext(MappingParserContext in, DateFormatter dateFormatter) {
             super(
                 in.similarityLookupService,
                 in.typeParsers,
                 in.runtimeFieldParsers,
                 in.indexVersionCreated,
                 in.searchExecutionContextSupplier,
-                in.dateFormatter,
                 in.scriptCompiler,
                 in.indexAnalyzers,
                 in.indexSettings,
                 in.idFieldMapper
             );
+            this.dateFormatter = dateFormatter;
+        }
+
+        @Override
+        public DateFormatter getDateFormatter() {
+            return dateFormatter;
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java
index 45f1363fb1a3..b82c13574e14 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java
@@ -49,7 +49,21 @@ Builder includeInParent(boolean includeInParent) {
 
         @Override
         public NestedObjectMapper build(MapperBuilderContext context) {
-            return new NestedObjectMapper(name, context.buildFullName(name), buildMappers(context.createChildContext(name)), this);
+            boolean parentIncludedInRoot = this.includeInRoot.value();
+            if (context instanceof NestedMapperBuilderContext nc) {
+                // we're already inside a nested mapper, so adjust our includes
+                if (nc.parentIncludedInRoot && this.includeInParent.value()) {
+                    this.includeInRoot = Explicit.IMPLICIT_FALSE;
+                }
+            } else {
+                // this is a top-level nested mapper, so include_in_parent = include_in_root
+                parentIncludedInRoot |= this.includeInParent.value();
+                if (this.includeInParent.value()) {
+                    this.includeInRoot = Explicit.IMPLICIT_FALSE;
+                }
+            }
+            NestedMapperBuilderContext nestedContext = new NestedMapperBuilderContext(context.buildFullName(name), parentIncludedInRoot);
+            return new NestedObjectMapper(name, context.buildFullName(name), buildMappers(nestedContext), this);
         }
     }
 
@@ -89,6 +103,21 @@ protected static void parseNested(String name, Map<String, Object> node, NestedO
         }
     }
 
+    private static class NestedMapperBuilderContext extends MapperBuilderContext {
+
+        final boolean parentIncludedInRoot;
+
+        NestedMapperBuilderContext(String path, boolean parentIncludedInRoot) {
+            super(path, false);
+            this.parentIncludedInRoot = parentIncludedInRoot;
+        }
+
+        @Override
+        public MapperBuilderContext createChildContext(String name) {
+            return new NestedMapperBuilderContext(buildFullName(name), parentIncludedInRoot);
+        }
+    }
+
     private Explicit<Boolean> includeInRoot;
     private Explicit<Boolean> includeInParent;
     private final String nestedTypePath;
@@ -153,7 +182,7 @@ public ObjectMapper.Builder newBuilder(Version indexVersionCreated) {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(simpleName());
         builder.field("type", CONTENT_TYPE);
-        if (includeInParent.value()) {
+        if (includeInParent.explicit() && includeInParent.value()) {
             builder.field("include_in_parent", includeInParent.value());
         }
         if (includeInRoot.value()) {
@@ -191,10 +220,28 @@ public ObjectMapper merge(Mapper mergeWith, MapperService.MergeReason reason, Ma
                 throw new MapperException("the [include_in_root] parameter can't be updated on a nested object mapping");
             }
         }
+        if (parentBuilderContext instanceof NestedMapperBuilderContext nc) {
+            if (nc.parentIncludedInRoot && toMerge.includeInParent.value()) {
+                toMerge.includeInRoot = Explicit.IMPLICIT_FALSE;
+            }
+        } else {
+            if (toMerge.includeInParent.value()) {
+                toMerge.includeInRoot = Explicit.IMPLICIT_FALSE;
+            }
+        }
         toMerge.doMerge(mergeWithObject, reason, parentBuilderContext);
         return toMerge;
     }
 
+    @Override
+    protected MapperBuilderContext createChildContext(MapperBuilderContext mapperBuilderContext, String name) {
+        boolean parentIncludedInRoot = this.includeInRoot.value();
+        if (mapperBuilderContext instanceof NestedMapperBuilderContext == false) {
+            parentIncludedInRoot |= this.includeInParent.value();
+        }
+        return new NestedMapperBuilderContext(mapperBuilderContext.buildFullName(name), parentIncludedInRoot);
+    }
+
     @Override
     public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
         throw new IllegalArgumentException("field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source");
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
index d35582cd1d27..64350930f5da 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
@@ -8,17 +8,21 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.document.DoubleField;
 import org.apache.lucene.document.DoublePoint;
+import org.apache.lucene.document.FloatField;
 import org.apache.lucene.document.FloatPoint;
+import org.apache.lucene.document.IntField;
 import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.LongField;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.sandbox.document.HalfFloatPoint;
-import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
+import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
@@ -30,6 +34,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
@@ -55,7 +60,7 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import org.elasticsearch.search.lookup.FieldValues;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParser.Token;
@@ -86,7 +91,7 @@ private static NumberFieldMapper toType(FieldMapper in) {
 
     public static class Builder extends FieldMapper.Builder {
 
-        private final Parameter<Boolean> indexed = Parameter.indexParam(m -> toType(m).indexed, true);
+        private final Parameter<Boolean> indexed;
         private final Parameter<Boolean> hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true);
         private final Parameter<Boolean> stored = Parameter.storeParam(m -> toType(m).stored, false);
 
@@ -96,7 +101,7 @@ public static class Builder extends FieldMapper.Builder {
         private final Parameter<Number> nullValue;
 
         private final Parameter<Script> script = Parameter.scriptParam(m -> toType(m).script);
-        private final Parameter<String> onScriptError = Parameter.onScriptErrorParam(m -> toType(m).onScriptError, script);
+        private final Parameter<OnScriptError> onScriptError = Parameter.onScriptErrorParam(m -> toType(m).onScriptError, script);
 
         /**
          * Parameter that marks this field as a time series dimension.
@@ -118,12 +123,21 @@ public static class Builder extends FieldMapper.Builder {
         private boolean allowMultipleValues = true;
         private final Version indexCreatedVersion;
 
-        public Builder(String name, NumberType type, ScriptCompiler compiler, Settings settings, Version indexCreatedVersion) {
-            this(name, type, compiler, IGNORE_MALFORMED_SETTING.get(settings), COERCE_SETTING.get(settings), indexCreatedVersion);
+        private final IndexMode indexMode;
+
+        public Builder(
+            String name,
+            NumberType type,
+            ScriptCompiler compiler,
+            Settings settings,
+            Version indexCreatedVersion,
+            IndexMode mode
+        ) {
+            this(name, type, compiler, IGNORE_MALFORMED_SETTING.get(settings), COERCE_SETTING.get(settings), indexCreatedVersion, mode);
         }
 
         public static Builder docValuesOnly(String name, NumberType type, Version indexCreatedVersion) {
-            Builder builder = new Builder(name, type, ScriptCompiler.NONE, false, false, indexCreatedVersion);
+            Builder builder = new Builder(name, type, ScriptCompiler.NONE, false, false, indexCreatedVersion, null);
             builder.indexed.setValue(false);
             builder.dimension.setValue(false);
             return builder;
@@ -135,7 +149,8 @@ public Builder(
             ScriptCompiler compiler,
             boolean ignoreMalformedByDefault,
             boolean coerceByDefault,
-            Version indexCreatedVersion
+            Version indexCreatedVersion,
+            IndexMode mode
         ) {
             super(name);
             this.type = type;
@@ -158,7 +173,15 @@ public Builder(
                 XContentBuilder::field,
                 Objects::toString
             ).acceptsNull();
-
+            this.indexMode = mode;
+            this.indexed = Parameter.indexParam(m -> toType(m).indexed, () -> {
+                if (indexMode == IndexMode.TIME_SERIES) {
+                    var metricType = getMetric().getValue();
+                    return metricType != MetricType.COUNTER && metricType != MetricType.GAUGE;
+                } else {
+                    return true;
+                }
+            });
             this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension).addValidator(v -> {
                 if (v && EnumSet.of(NumberType.INTEGER, NumberType.LONG, NumberType.BYTE, NumberType.SHORT).contains(type) == false) {
                     throw new IllegalArgumentException(
@@ -178,7 +201,7 @@ public Builder(
                 }
             });
 
-            this.metric = TimeSeriesParams.metricParam(m -> toType(m).metricType, MetricType.gauge, MetricType.counter).addValidator(v -> {
+            this.metric = TimeSeriesParams.metricParam(m -> toType(m).metricType, MetricType.GAUGE, MetricType.COUNTER).addValidator(v -> {
                 if (v != null && hasDocValues.getValue() == false) {
                     throw new IllegalArgumentException(
                         "Field [" + TimeSeriesParams.TIME_SERIES_METRIC_PARAM + "] requires that [" + hasDocValues.name + "] is true"
@@ -217,6 +240,10 @@ public Builder metric(MetricType metric) {
             return this;
         }
 
+        private Parameter<MetricType> getMetric() {
+            return metric;
+        }
+
         public Builder allowMultipleValues(boolean allowMultipleValues) {
             this.allowMultipleValues = allowMultipleValues;
             return this;
@@ -390,14 +417,14 @@ public IndexFieldData.Builder getFieldDataBuilder(String name, ValuesSourceType
             public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
                 String name,
                 ValuesSourceType valuesSourceType,
-                SourceLookup sourceLookup,
+                SourceProvider sourceProvider,
                 ValueFetcher valueFetcher
             ) {
                 return new SourceValueFetcherSortedDoubleIndexFieldData.Builder(
                     name,
                     valuesSourceType,
                     valueFetcher,
-                    sourceLookup,
+                    sourceProvider,
                     HalfFloatDocValuesField::new
                 );
             }
@@ -521,11 +548,12 @@ public Query rangeQuery(
             @Override
             public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) {
                 final float f = value.floatValue();
-                if (indexed) {
-                    document.add(new FloatPoint(name, f));
-                }
-                if (docValued) {
+                if (indexed && docValued) {
+                    document.add(new FloatField(name, f));
+                } else if (docValued) {
                     document.add(new SortedNumericDocValuesField(name, NumericUtils.floatToSortableInt(f)));
+                } else if (indexed) {
+                    document.add(new FloatPoint(name, f));
                 }
                 if (stored) {
                     document.add(new StoredField(name, f));
@@ -541,14 +569,14 @@ public IndexFieldData.Builder getFieldDataBuilder(String name, ValuesSourceType
             public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
                 String name,
                 ValuesSourceType valuesSourceType,
-                SourceLookup sourceLookup,
+                SourceProvider sourceProvider,
                 ValueFetcher valueFetcher
             ) {
                 return new SourceValueFetcherSortedDoubleIndexFieldData.Builder(
                     name,
                     valuesSourceType,
                     valueFetcher,
-                    sourceLookup,
+                    sourceProvider,
                     FloatDocValuesField::new
                 );
             }
@@ -592,7 +620,7 @@ public Double parse(XContentParser parser, boolean coerce) throws IOException {
             @Override
             public FieldValues<Number> compile(String fieldName, Script script, ScriptCompiler compiler) {
                 DoubleFieldScript.Factory scriptFactory = compiler.compile(script, DoubleFieldScript.CONTEXT);
-                return (lookup, ctx, doc, consumer) -> scriptFactory.newFactory(fieldName, script.getParams(), lookup)
+                return (lookup, ctx, doc, consumer) -> scriptFactory.newFactory(fieldName, script.getParams(), lookup, OnScriptError.FAIL)
                     .newInstance(ctx)
                     .runForDoc(doc, consumer::accept);
             }
@@ -650,11 +678,12 @@ public Query rangeQuery(
             @Override
             public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) {
                 final double d = value.doubleValue();
-                if (indexed) {
-                    document.add(new DoublePoint(name, d));
-                }
-                if (docValued) {
+                if (indexed && docValued) {
+                    document.add(new DoubleField(name, d));
+                } else if (docValued) {
                     document.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong(d)));
+                } else if (indexed) {
+                    document.add(new DoublePoint(name, d));
                 }
                 if (stored) {
                     document.add(new StoredField(name, d));
@@ -670,14 +699,14 @@ public IndexFieldData.Builder getFieldDataBuilder(String name, ValuesSourceType
             public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
                 String name,
                 ValuesSourceType valuesSourceType,
-                SourceLookup sourceLookup,
+                SourceProvider sourceProvider,
                 ValueFetcher valueFetcher
             ) {
                 return new SourceValueFetcherSortedDoubleIndexFieldData.Builder(
                     name,
                     valuesSourceType,
                     valueFetcher,
-                    sourceLookup,
+                    sourceProvider,
                     DoubleDocValuesField::new
                 );
             }
@@ -774,14 +803,14 @@ public IndexFieldData.Builder getFieldDataBuilder(String name, ValuesSourceType
             public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
                 String name,
                 ValuesSourceType valuesSourceType,
-                SourceLookup sourceLookup,
+                SourceProvider sourceProvider,
                 ValueFetcher valueFetcher
             ) {
                 return new SourceValueFetcherSortedNumericIndexFieldData.Builder(
                     name,
                     valuesSourceType,
                     valueFetcher,
-                    sourceLookup,
+                    sourceProvider,
                     ByteDocValuesField::new
                 );
             }
@@ -863,14 +892,14 @@ public IndexFieldData.Builder getFieldDataBuilder(String name, ValuesSourceType
             public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
                 String name,
                 ValuesSourceType valuesSourceType,
-                SourceLookup sourceLookup,
+                SourceProvider sourceProvider,
                 ValueFetcher valueFetcher
             ) {
                 return new SourceValueFetcherSortedNumericIndexFieldData.Builder(
                     name,
                     valuesSourceType,
                     valueFetcher,
-                    sourceLookup,
+                    sourceProvider,
                     ShortDocValuesField::new
                 );
             }
@@ -999,11 +1028,12 @@ public Query rangeQuery(
             @Override
             public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) {
                 final int i = value.intValue();
-                if (indexed) {
-                    document.add(new IntPoint(name, i));
-                }
-                if (docValued) {
+                if (indexed && docValued) {
+                    document.add(new IntField(name, i));
+                } else if (docValued) {
                     document.add(new SortedNumericDocValuesField(name, i));
+                } else if (indexed) {
+                    document.add(new IntPoint(name, i));
                 }
                 if (stored) {
                     document.add(new StoredField(name, i));
@@ -1019,14 +1049,14 @@ public IndexFieldData.Builder getFieldDataBuilder(String name, ValuesSourceType
             public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
                 String name,
                 ValuesSourceType valuesSourceType,
-                SourceLookup sourceLookup,
+                SourceProvider sourceProvider,
                 ValueFetcher valueFetcher
             ) {
                 return new SourceValueFetcherSortedNumericIndexFieldData.Builder(
                     name,
                     valuesSourceType,
                     valueFetcher,
-                    sourceLookup,
+                    sourceProvider,
                     IntegerDocValuesField::new
                 );
             }
@@ -1055,7 +1085,7 @@ public Long parse(XContentParser parser, boolean coerce) throws IOException {
             @Override
             public FieldValues<Number> compile(String fieldName, Script script, ScriptCompiler compiler) {
                 final LongFieldScript.Factory scriptFactory = compiler.compile(script, LongFieldScript.CONTEXT);
-                return (lookup, ctx, doc, consumer) -> scriptFactory.newFactory(fieldName, script.getParams(), lookup)
+                return (lookup, ctx, doc, consumer) -> scriptFactory.newFactory(fieldName, script.getParams(), lookup, OnScriptError.FAIL)
                     .newInstance(ctx)
                     .runForDoc(doc, consumer::accept);
             }
@@ -1125,11 +1155,12 @@ public Query rangeQuery(
             @Override
             public void addFields(LuceneDocument document, String name, Number value, boolean indexed, boolean docValued, boolean stored) {
                 final long l = value.longValue();
-                if (indexed) {
-                    document.add(new LongPoint(name, l));
-                }
-                if (docValued) {
+                if (indexed && docValued) {
+                    document.add(new LongField(name, l));
+                } else if (docValued) {
                     document.add(new SortedNumericDocValuesField(name, l));
+                } else if (indexed) {
+                    document.add(new LongPoint(name, l));
                 }
                 if (stored) {
                     document.add(new StoredField(name, l));
@@ -1145,14 +1176,14 @@ public IndexFieldData.Builder getFieldDataBuilder(String name, ValuesSourceType
             public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
                 String name,
                 ValuesSourceType valuesSourceType,
-                SourceLookup sourceLookup,
+                SourceProvider sourceProvider,
                 ValueFetcher valueFetcher
             ) {
                 return new SourceValueFetcherSortedNumericIndexFieldData.Builder(
                     name,
                     valuesSourceType,
                     valueFetcher,
-                    sourceLookup,
+                    sourceProvider,
                     LongDocValuesField::new
                 );
             }
@@ -1171,7 +1202,14 @@ SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String
             this.name = name;
             this.numericType = numericType;
             this.parser = new TypeParser(
-                (n, c) -> new Builder(n, this, c.scriptCompiler(), c.getSettings(), c.indexVersionCreated()),
+                (n, c) -> new Builder(
+                    n,
+                    this,
+                    c.scriptCompiler(),
+                    c.getSettings(),
+                    c.indexVersionCreated(),
+                    c.getIndexSettings().getMode()
+                ),
                 MINIMUM_COMPATIBILITY_VERSION
             );
         }
@@ -1385,7 +1423,7 @@ public static Query longRangeQuery(
         public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
             String name,
             ValuesSourceType valuesSourceType,
-            SourceLookup sourceLookup,
+            SourceProvider sourceProvider,
             ValueFetcher valueFetcher
         ) {
             throw new UnsupportedOperationException("not supported for source fallback");
@@ -1551,7 +1589,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext
                 failIfNoDocValues();
             }
 
-            ValuesSourceType valuesSourceType = metricType == TimeSeriesParams.MetricType.counter
+            ValuesSourceType valuesSourceType = metricType == TimeSeriesParams.MetricType.COUNTER
                 ? TimeSeriesValuesSourceType.COUNTER
                 : type.numericType.getValuesSourceType();
 
@@ -1562,12 +1600,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext
             if (operation == FielddataOperation.SCRIPT) {
                 SearchLookup searchLookup = fieldDataContext.lookupSupplier().get();
                 Set<String> sourcePaths = fieldDataContext.sourcePathsLookup().apply(name());
-                return type.getValueFetcherFieldDataBuilder(
-                    name(),
-                    valuesSourceType,
-                    searchLookup.source(),
-                    sourceValueFetcher(sourcePaths)
-                );
+                return type.getValueFetcherFieldDataBuilder(name(), valuesSourceType, searchLookup, sourceValueFetcher(sourcePaths));
             }
 
             throw new IllegalStateException("unknown field data type [" + operation.name() + "]");
@@ -1639,7 +1672,6 @@ public MetricType getMetricType() {
     }
 
     private final NumberType type;
-
     private final boolean indexed;
     private final boolean hasDocValues;
     private final boolean stored;
@@ -1657,6 +1689,8 @@ public MetricType getMetricType() {
     private final Version indexCreatedVersion;
     private final boolean storeMalformedFields;
 
+    private final IndexMode indexMode;
+
     private NumberFieldMapper(
         String simpleName,
         MappedFieldType mappedFieldType,
@@ -1683,6 +1717,7 @@ private NumberFieldMapper(
         this.allowMultipleValues = builder.allowMultipleValues;
         this.indexCreatedVersion = builder.indexCreatedVersion;
         this.storeMalformedFields = storeMalformedFields;
+        this.indexMode = builder.indexMode;
     }
 
     boolean coerce() {
@@ -1784,9 +1819,11 @@ protected void indexScriptValues(
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(simpleName(), type, scriptCompiler, ignoreMalformedByDefault, coerceByDefault, indexCreatedVersion).dimension(
-            dimension
-        ).metric(metricType).allowMultipleValues(allowMultipleValues).init(this);
+        return new Builder(simpleName(), type, scriptCompiler, ignoreMalformedByDefault, coerceByDefault, indexCreatedVersion, indexMode)
+            .dimension(dimension)
+            .metric(metricType)
+            .allowMultipleValues(allowMultipleValues)
+            .init(this);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/OnScriptError.java b/server/src/main/java/org/elasticsearch/index/mapper/OnScriptError.java
new file mode 100644
index 000000000000..42c3d1386793
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/mapper/OnScriptError.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import java.util.Locale;
+
+/**
+ * Represents the behaviour when a runtime field or an index-time script fails: either fail and raise the error,
+ * or continue and ignore the error.
+ */
+public enum OnScriptError {
+    FAIL,
+    CONTINUE;
+
+    @Override
+    public final String toString() {
+        return name().toLowerCase(Locale.ROOT);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java
index 9e5516b978ee..96d494a44b8f 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java
@@ -120,36 +120,6 @@ public RootObjectMapper build(MapperBuilderContext context) {
         }
     }
 
-    /**
-     * Removes redundant root includes in {@link NestedObjectMapper} trees to avoid duplicate
-     * fields on the root mapper when {@code isIncludeInRoot} is {@code true} for a node that is
-     * itself included into a parent node, for which either {@code isIncludeInRoot} is
-     * {@code true} or which is transitively included in root by a chain of nodes with
-     * {@code isIncludeInParent} returning {@code true}.
-     */
-    // TODO it would be really nice to make this an implementation detail of NestedObjectMapper
-    // and run it as part of the builder, but this does not yet work because of the way that
-    // index templates are merged together. If merge() was run on Builder objects rather than
-    // on Mappers then we could move this.
-    public void fixRedundantIncludes() {
-        fixRedundantIncludes(this, true);
-    }
-
-    private static void fixRedundantIncludes(ObjectMapper objectMapper, boolean parentIncluded) {
-        for (Mapper mapper : objectMapper) {
-            if (mapper instanceof NestedObjectMapper child) {
-                boolean isNested = child.isNested();
-                boolean includeInRootViaParent = parentIncluded && isNested && child.isIncludeInParent();
-                boolean includedInRoot = isNested && child.isIncludeInRoot();
-                if (includeInRootViaParent && includedInRoot) {
-                    child.setIncludeInParent(true);
-                    child.setIncludeInRoot(false);
-                }
-                fixRedundantIncludes(child, includeInRootViaParent || includedInRoot);
-            }
-        }
-    }
-
     private Explicit<DateFormatter[]> dynamicDateTimeFormatters;
     private Explicit<Boolean> dateDetection;
     private Explicit<Boolean> numericDetection;
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RuntimeField.java b/server/src/main/java/org/elasticsearch/index/mapper/RuntimeField.java
index 9c9899b33482..24bc510549c8 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/RuntimeField.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/RuntimeField.java
@@ -63,7 +63,8 @@ protected List<Parameter<?>> getParameters() {
         protected abstract RuntimeField createChildRuntimeField(
             MappingParserContext parserContext,
             String parentName,
-            Function<SearchLookup, CompositeFieldScript.LeafFactory> parentScriptFactory
+            Function<SearchLookup, CompositeFieldScript.LeafFactory> parentScriptFactory,
+            OnScriptError onScriptError
         );
 
         public final void parse(String name, MappingParserContext parserContext, Map<String, Object> fieldNode) {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
index 54aaaa845c81..f8b92625ef5f 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.lookup.Source;
@@ -48,7 +49,16 @@ private enum Mode {
         null,
         Explicit.IMPLICIT_TRUE,
         Strings.EMPTY_ARRAY,
-        Strings.EMPTY_ARRAY
+        Strings.EMPTY_ARRAY,
+        null
+    );
+
+    private static final SourceFieldMapper TSDB_DEFAULT = new SourceFieldMapper(
+        Mode.SYNTHETIC,
+        Explicit.IMPLICIT_TRUE,
+        Strings.EMPTY_ARRAY,
+        Strings.EMPTY_ARRAY,
+        IndexMode.TIME_SERIES
     );
 
     public static class Defaults {
@@ -79,7 +89,7 @@ public static class Builder extends MetadataFieldMapper.Builder {
         private final Parameter<Mode> mode = new Parameter<>(
             "mode",
             true,
-            () -> null,
+            () -> getIndexMode() == IndexMode.TIME_SERIES ? Mode.SYNTHETIC : null,
             (n, c, o) -> Mode.valueOf(o.toString().toUpperCase(Locale.ROOT)),
             m -> toType(m).enabled.explicit() ? null : toType(m).mode,
             (b, n, v) -> b.field(n, v.toString().toLowerCase(Locale.ROOT)),
@@ -97,8 +107,16 @@ public static class Builder extends MetadataFieldMapper.Builder {
             m -> Arrays.asList(toType(m).excludes)
         );
 
-        public Builder() {
+        private final IndexMode indexMode;
+
+        public Builder(IndexMode indexMode) {
             super(Defaults.NAME);
+            this.indexMode = indexMode;
+        }
+
+        public Builder setSynthetic() {
+            this.mode.setValue(Mode.SYNTHETIC);
+            return this;
         }
 
         @Override
@@ -122,18 +140,30 @@ public SourceFieldMapper build() {
                 throw new MapperParsingException("Cannot set both [mode] and [enabled] parameters");
             }
             if (isDefault()) {
-                return DEFAULT;
+                return indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT : DEFAULT;
             }
-            return new SourceFieldMapper(
+            SourceFieldMapper sourceFieldMapper = new SourceFieldMapper(
                 mode.get(),
                 enabled.get(),
                 includes.getValue().toArray(String[]::new),
-                excludes.getValue().toArray(String[]::new)
+                excludes.getValue().toArray(String[]::new),
+                indexMode
             );
+            if (indexMode != null) {
+                indexMode.validateSourceFieldMapper(sourceFieldMapper);
+            }
+            return sourceFieldMapper;
+        }
+
+        private IndexMode getIndexMode() {
+            return indexMode;
         }
     }
 
-    public static final TypeParser PARSER = new ConfigurableTypeParser(c -> DEFAULT, c -> new Builder());
+    public static final TypeParser PARSER = new ConfigurableTypeParser(
+        c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES ? TSDB_DEFAULT : DEFAULT,
+        c -> new Builder(c.getIndexSettings().getMode())
+    );
 
     static final class SourceFieldType extends MappedFieldType {
 
@@ -173,7 +203,9 @@ public Query termQuery(Object value, SearchExecutionContext context) {
     private final String[] excludes;
     private final SourceFilter sourceFilter;
 
-    private SourceFieldMapper(Mode mode, Explicit<Boolean> enabled, String[] includes, String[] excludes) {
+    private final IndexMode indexMode;
+
+    private SourceFieldMapper(Mode mode, Explicit<Boolean> enabled, String[] includes, String[] excludes, IndexMode indexMode) {
         super(new SourceFieldType((enabled.explicit() && enabled.value()) || (enabled.explicit() == false && mode != Mode.DISABLED)));
         assert enabled.explicit() == false || mode == null;
         this.mode = mode;
@@ -185,6 +217,7 @@ private SourceFieldMapper(Mode mode, Explicit<Boolean> enabled, String[] include
             throw new IllegalArgumentException("filtering the stored _source is incompatible with synthetic source");
         }
         this.complete = stored() && sourceFilter == null;
+        this.indexMode = indexMode;
     }
 
     private static SourceFilter buildSourceFilter(String[] includes, String[] excludes) {
@@ -254,13 +287,13 @@ protected String contentType() {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder().init(this);
+        return new Builder(indexMode).init(this);
     }
 
     /**
      * Build something to load source {@code _source}.
      */
-    public <T> SourceLoader newSourceLoader(Mapping mapping) {
+    public SourceLoader newSourceLoader(Mapping mapping) {
         if (mode == Mode.SYNTHETIC) {
             return new SourceLoader.Synthetic(mapping);
         }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
index 2cbc2cc58433..51cf935654e8 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
@@ -1015,7 +1015,7 @@ protected BytesRef storedToBytesRef(Object stored) {
                 name(),
                 CoreValuesSourceType.KEYWORD,
                 SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name())),
-                fieldDataContext.lookupSupplier().get().source(),
+                fieldDataContext.lookupSupplier().get(),
                 TextDocValuesField::new
             );
         }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java
index 3032c4e696cd..a85ba6d0e9a4 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java
@@ -239,7 +239,7 @@ public BytesReference build() throws IOException {
         }
 
         @Override
-        public void addString(String fieldName, String value) {
+        public void addString(String fieldName, BytesRef utf8Value) {
             try (BytesStreamOutput out = new BytesStreamOutput()) {
                 out.write((byte) 's');
                 /*
@@ -247,17 +247,16 @@ public void addString(String fieldName, String value) {
                  * so it's easier for folks to reason about the space taken up. Mostly
                  * it'll be smaller too.
                  */
-                BytesRef bytes = new BytesRef(value);
-                if (bytes.length > DIMENSION_VALUE_LIMIT) {
+                if (utf8Value.length > DIMENSION_VALUE_LIMIT) {
                     throw new IllegalArgumentException(
-                        "Dimension fields must be less than [" + DIMENSION_VALUE_LIMIT + "] bytes but was [" + bytes.length + "]."
+                        "Dimension fields must be less than [" + DIMENSION_VALUE_LIMIT + "] bytes but was [" + utf8Value.length + "]."
                     );
                 }
-                out.writeBytesRef(bytes);
+                out.writeBytesRef(utf8Value);
                 add(fieldName, out.bytes());
 
                 if (routingBuilder != null) {
-                    routingBuilder.addMatching(fieldName, bytes);
+                    routingBuilder.addMatching(fieldName, utf8Value);
                 }
             } catch (IOException e) {
                 throw new IllegalArgumentException("Dimension field cannot be serialized.", e);
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java
index 2ab6a5ad6859..cb0b9ddd87f0 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java
@@ -10,6 +10,7 @@
 
 import java.util.Arrays;
 import java.util.EnumSet;
+import java.util.Locale;
 import java.util.function.Function;
 
 /**
@@ -23,8 +24,8 @@ public final class TimeSeriesParams {
     private TimeSeriesParams() {}
 
     public enum MetricType {
-        gauge(new String[] { "max", "min", "value_count", "sum" }),
-        counter(new String[] { "last_value" });
+        GAUGE(new String[] { "max", "min", "value_count", "sum" }),
+        COUNTER(new String[] { "last_value" });
 
         private final String[] supportedAggs;
 
@@ -35,6 +36,20 @@ public enum MetricType {
         public String[] supportedAggs() {
             return supportedAggs;
         }
+
+        @Override
+        public final String toString() {
+            return name().toLowerCase(Locale.ROOT);
+        }
+
+        public static MetricType fromString(String value) {
+            for (MetricType metricType : values()) {
+                if (metricType.toString().equals(value)) {
+                    return metricType;
+                }
+            }
+            throw new IllegalArgumentException("No enum constant MetricType." + value);
+        }
     }
 
     public static FieldMapper.Parameter<MetricType> metricParam(Function<FieldMapper, MetricType> initializer, MetricType... values) {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java
index d5e74a432cc1..b31e78ea49ce 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java
@@ -116,25 +116,8 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra
             );
         }
         long timestamp = timestampFields[0].numericValue().longValue();
-
-        Hash128 hash = new Hash128();
-        MurmurHash3.hash128(tsid.bytes, tsid.offset, tsid.length, SEED, hash);
-
         byte[] suffix = new byte[16];
-        ByteUtils.writeLongLE(hash.h1, suffix, 0);
-        ByteUtils.writeLongBE(timestamp, suffix, 8);   // Big Ending shrinks the inverted index by ~37%
-
-        IndexRouting.ExtractFromSource indexRouting = (IndexRouting.ExtractFromSource) context.indexSettings().getIndexRouting();
-        String id = routingBuilder.createId(suffix, () -> {
-            if (context.getDynamicMappers().isEmpty() == false) {
-                throw new IllegalStateException(
-                    "Didn't find any fields to include in the routing which would be fine if there are"
-                        + " dynamic mapping waiting but we couldn't find any of those either!"
-                );
-            }
-            return 0;
-        });
-        assert Uid.isURLBase64WithoutPadding(id); // Make sure we get to use Uid's nice optimizations
+        String id = createId(context.getDynamicMappers().isEmpty(), routingBuilder, tsid, timestamp, suffix);
         /*
          * Make sure that _id from extracting the tsid matches that _id
          * from extracting the _source. This should be true for all valid
@@ -144,6 +127,7 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra
          * at all we just skip the assertion because we can't be sure
          * it always must pass.
          */
+        IndexRouting.ExtractFromSource indexRouting = (IndexRouting.ExtractFromSource) context.indexSettings().getIndexRouting();
         assert context.getDynamicMappers().isEmpty() == false
             || context.getDynamicRuntimeFields().isEmpty() == false
             || id.equals(indexRouting.createId(TimeSeriesIdFieldMapper.decodeTsid(tsid), suffix));
@@ -168,6 +152,32 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra
         context.doc().add(new Field(NAME, uidEncoded, FIELD_TYPE));
     }
 
+    public static String createId(
+        boolean dynamicMappersExists,
+        IndexRouting.ExtractFromSource.Builder routingBuilder,
+        BytesRef tsid,
+        long timestamp,
+        byte[] suffix
+    ) {
+        Hash128 hash = new Hash128();
+        MurmurHash3.hash128(tsid.bytes, tsid.offset, tsid.length, SEED, hash);
+
+        ByteUtils.writeLongLE(hash.h1, suffix, 0);
+        ByteUtils.writeLongBE(timestamp, suffix, 8);   // Big Ending shrinks the inverted index by ~37%
+
+        String id = routingBuilder.createId(suffix, () -> {
+            if (dynamicMappersExists == false) {
+                throw new IllegalStateException(
+                    "Didn't find any fields to include in the routing which would be fine if there are"
+                        + " dynamic mapping waiting but we couldn't find any of those either!"
+                );
+            }
+            return 0;
+        });
+        assert Uid.isURLBase64WithoutPadding(id); // Make sure we get to use Uid's nice optimizations
+        return id;
+    }
+
     @Override
     public String documentDescription(DocumentParserContext context) {
         /*
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java
index 16b2bab0d49a..35399502a8fe 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java
@@ -110,7 +110,7 @@ public static boolean parseMultiField(
                 }
             }
 
-            parserContext = MappingParserContext.createMultiFieldContext(parserContext);
+            parserContext = parserContext.createMultiFieldContext();
 
             final Map<String, Object> multiFieldsPropNodes;
             if (propNode instanceof List && ((List<?>) propNode).isEmpty()) {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java
index 6bd9700feb90..20f2032698f4 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java
@@ -9,19 +9,21 @@
 package org.elasticsearch.index.mapper.vectors;
 
 import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat;
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.document.KnnVectorField;
+import org.apache.lucene.document.KnnByteVectorField;
+import org.apache.lucene.document.KnnFloatVectorField;
 import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.ByteVectorValues;
+import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.VectorSimilarityFunction;
-import org.apache.lucene.index.VectorValues;
 import org.apache.lucene.search.FieldExistsQuery;
-import org.apache.lucene.search.KnnVectorQuery;
+import org.apache.lucene.search.KnnByteVectorQuery;
+import org.apache.lucene.search.KnnFloatVectorQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.VectorUtil;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.fielddata.FieldDataContext;
@@ -43,13 +45,16 @@
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParser.Token;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.time.ZoneId;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.function.Function;
 import java.util.stream.Stream;
 
 import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
@@ -177,13 +182,18 @@ public void writeValue(ByteBuffer byteBuffer, float value) {
             }
 
             @Override
-            public float readValue(ByteBuffer byteBuffer) {
-                return byteBuffer.get();
+            public void readAndWriteValue(ByteBuffer byteBuffer, XContentBuilder b) throws IOException {
+                b.value(byteBuffer.get());
             }
 
             @Override
-            KnnVectorField createKnnVectorField(String name, float[] vector, VectorSimilarityFunction function) {
-                return new KnnVectorField(name, VectorUtil.toBytesRef(vector), function);
+            KnnByteVectorField createKnnVectorField(String name, byte[] vector, VectorSimilarityFunction function) {
+                return new KnnByteVectorField(name, vector, function);
+            }
+
+            @Override
+            KnnFloatVectorField createKnnVectorField(String name, float[] vector, VectorSimilarityFunction function) {
+                throw new IllegalArgumentException("cannot create a float vector field from byte");
             }
 
             @Override
@@ -244,18 +254,103 @@ public void checkVectorBounds(float[] vector) {
             }
 
             @Override
-            void checkVectorMagnitude(VectorSimilarity similarity, float[] vector, float squaredMagnitude) {
+            void checkVectorMagnitude(
+                VectorSimilarity similarity,
+                Function<StringBuilder, StringBuilder> appender,
+                float squaredMagnitude
+            ) {
                 StringBuilder errorBuilder = null;
 
-                if (similarity == VectorSimilarity.cosine && Math.sqrt(squaredMagnitude) == 0.0f) {
+                if (similarity == VectorSimilarity.COSINE && Math.sqrt(squaredMagnitude) == 0.0f) {
                     errorBuilder = new StringBuilder(
-                        "The [" + VectorSimilarity.cosine.name() + "] similarity does not support vectors with zero magnitude."
+                        "The [" + VectorSimilarity.COSINE + "] similarity does not support vectors with zero magnitude."
                     );
                 }
 
                 if (errorBuilder != null) {
-                    throw new IllegalArgumentException(appendErrorElements(errorBuilder, vector).toString());
+                    throw new IllegalArgumentException(appender.apply(errorBuilder).toString());
+                }
+            }
+
+            @Override
+            public Field parseKnnVector(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException {
+                int index = 0;
+                byte[] vector = new byte[fieldMapper.dims];
+                float squaredMagnitude = 0;
+                for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
+                    fieldMapper.checkDimensionExceeded(index, context);
+                    ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser());
+                    final int value;
+                    if (context.parser().numberType() != XContentParser.NumberType.INT) {
+                        float floatValue = context.parser().floatValue(true);
+                        if (floatValue % 1.0f != 0.0f) {
+                            throw new IllegalArgumentException(
+                                "element_type ["
+                                    + this
+                                    + "] vectors only support non-decimal values but found decimal value ["
+                                    + floatValue
+                                    + "] at dim ["
+                                    + index
+                                    + "];"
+                            );
+                        }
+                        value = (int) floatValue;
+                    } else {
+                        value = context.parser().intValue(true);
+                    }
+                    if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) {
+                        throw new IllegalArgumentException(
+                            "element_type ["
+                                + this
+                                + "] vectors only support integers between ["
+                                + Byte.MIN_VALUE
+                                + ", "
+                                + Byte.MAX_VALUE
+                                + "] but found ["
+                                + value
+                                + "] at dim ["
+                                + index
+                                + "];"
+                        );
+                    }
+                    vector[index++] = (byte) value;
+                    squaredMagnitude += value * value;
                 }
+                fieldMapper.checkDimensionMatches(index, context);
+                checkVectorMagnitude(fieldMapper.similarity, errorByteElementsAppender(vector), squaredMagnitude);
+                return createKnnVectorField(fieldMapper.fieldType().name(), vector, fieldMapper.similarity.function);
+            }
+
+            @Override
+            double parseKnnVectorToByteBuffer(DocumentParserContext context, DenseVectorFieldMapper fieldMapper, ByteBuffer byteBuffer)
+                throws IOException {
+                double dotProduct = 0f;
+                int index = 0;
+                for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
+                    fieldMapper.checkDimensionExceeded(index, context);
+                    ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser());
+                    int value = context.parser().intValue(true);
+                    if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) {
+                        throw new IllegalArgumentException(
+                            "element_type ["
+                                + this
+                                + "] vectors only support integers between ["
+                                + Byte.MIN_VALUE
+                                + ", "
+                                + Byte.MAX_VALUE
+                                + "] but found ["
+                                + value
+                                + "] at dim ["
+                                + index
+                                + "];"
+                        );
+                    }
+                    byteBuffer.put((byte) value);
+                    dotProduct += value * value;
+                    index++;
+                }
+                fieldMapper.checkDimensionMatches(index, context);
+                return dotProduct;
             }
         },
 
@@ -272,13 +367,18 @@ public void writeValue(ByteBuffer byteBuffer, float value) {
             }
 
             @Override
-            public float readValue(ByteBuffer byteBuffer) {
-                return byteBuffer.getFloat();
+            public void readAndWriteValue(ByteBuffer byteBuffer, XContentBuilder b) throws IOException {
+                b.value(byteBuffer.getFloat());
             }
 
             @Override
-            KnnVectorField createKnnVectorField(String name, float[] vector, VectorSimilarityFunction function) {
-                return new KnnVectorField(name, vector, function);
+            KnnFloatVectorField createKnnVectorField(String name, float[] vector, VectorSimilarityFunction function) {
+                return new KnnFloatVectorField(name, vector, function);
+            }
+
+            @Override
+            KnnByteVectorField createKnnVectorField(String name, byte[] vector, VectorSimilarityFunction function) {
+                throw new IllegalArgumentException("cannot create a byte vector field from float");
             }
 
             @Override
@@ -299,23 +399,66 @@ public void checkVectorBounds(float[] vector) {
             }
 
             @Override
-            void checkVectorMagnitude(VectorSimilarity similarity, float[] vector, float squaredMagnitude) {
+            void checkVectorMagnitude(
+                VectorSimilarity similarity,
+                Function<StringBuilder, StringBuilder> appender,
+                float squaredMagnitude
+            ) {
                 StringBuilder errorBuilder = null;
 
-                if (similarity == VectorSimilarity.dot_product && Math.abs(squaredMagnitude - 1.0f) > 1e-4f) {
+                if (similarity == VectorSimilarity.DOT_PRODUCT && Math.abs(squaredMagnitude - 1.0f) > 1e-4f) {
                     errorBuilder = new StringBuilder(
-                        "The [" + VectorSimilarity.dot_product.name() + "] similarity can only be used with unit-length vectors."
+                        "The [" + VectorSimilarity.DOT_PRODUCT + "] similarity can only be used with unit-length vectors."
                     );
-                } else if (similarity == VectorSimilarity.cosine && Math.sqrt(squaredMagnitude) == 0.0f) {
+                } else if (similarity == VectorSimilarity.COSINE && Math.sqrt(squaredMagnitude) == 0.0f) {
                     errorBuilder = new StringBuilder(
-                        "The [" + VectorSimilarity.cosine.name() + "] similarity does not support vectors with zero magnitude."
+                        "The [" + VectorSimilarity.COSINE + "] similarity does not support vectors with zero magnitude."
                     );
                 }
 
                 if (errorBuilder != null) {
-                    throw new IllegalArgumentException(appendErrorElements(errorBuilder, vector).toString());
+                    throw new IllegalArgumentException(appender.apply(errorBuilder).toString());
                 }
             }
+
+            @Override
+            public Field parseKnnVector(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException {
+                int index = 0;
+                float[] vector = new float[fieldMapper.dims];
+                float squaredMagnitude = 0;
+                for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
+                    fieldMapper.checkDimensionExceeded(index, context);
+                    ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser());
+
+                    float value = context.parser().floatValue(true);
+                    vector[index++] = value;
+                    squaredMagnitude += value * value;
+                }
+                fieldMapper.checkDimensionMatches(index, context);
+                checkVectorBounds(vector);
+                checkVectorMagnitude(fieldMapper.similarity, errorFloatElementsAppender(vector), squaredMagnitude);
+                return createKnnVectorField(fieldMapper.fieldType().name(), vector, fieldMapper.similarity.function);
+            }
+
+            @Override
+            double parseKnnVectorToByteBuffer(DocumentParserContext context, DenseVectorFieldMapper fieldMapper, ByteBuffer byteBuffer)
+                throws IOException {
+                double dotProduct = 0f;
+                int index = 0;
+                float[] vector = new float[fieldMapper.dims];
+                for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
+                    fieldMapper.checkDimensionExceeded(index, context);
+                    ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser());
+                    float value = context.parser().floatValue(true);
+                    vector[index] = value;
+                    byteBuffer.putFloat(value);
+                    dotProduct += value * value;
+                    index++;
+                }
+                fieldMapper.checkDimensionMatches(index, context);
+                checkVectorBounds(vector);
+                return dotProduct;
+            }
         };
 
         final int elementBytes;
@@ -326,15 +469,26 @@ void checkVectorMagnitude(VectorSimilarity similarity, float[] vector, float squ
 
         public abstract void writeValue(ByteBuffer byteBuffer, float value);
 
-        public abstract float readValue(ByteBuffer byteBuffer);
+        public abstract void readAndWriteValue(ByteBuffer byteBuffer, XContentBuilder b) throws IOException;
+
+        abstract KnnFloatVectorField createKnnVectorField(String name, float[] vector, VectorSimilarityFunction function);
 
-        abstract KnnVectorField createKnnVectorField(String name, float[] vector, VectorSimilarityFunction function);
+        abstract KnnByteVectorField createKnnVectorField(String name, byte[] vector, VectorSimilarityFunction function);
 
         abstract IndexFieldData.Builder fielddataBuilder(DenseVectorFieldType denseVectorFieldType, FieldDataContext fieldDataContext);
 
+        abstract Field parseKnnVector(DocumentParserContext context, DenseVectorFieldMapper fieldMapper) throws IOException;
+
+        abstract double parseKnnVectorToByteBuffer(DocumentParserContext context, DenseVectorFieldMapper fieldMapper, ByteBuffer byteBuffer)
+            throws IOException;
+
         public abstract void checkVectorBounds(float[] vector);
 
-        abstract void checkVectorMagnitude(VectorSimilarity similarity, float[] vector, float squaredMagnitude);
+        abstract void checkVectorMagnitude(
+            VectorSimilarity similarity,
+            Function<StringBuilder, StringBuilder> errorElementsAppender,
+            float squaredMagnitude
+        );
 
         void checkNanAndInfinite(float[] vector) {
             StringBuilder errorBuilder = null;
@@ -383,6 +537,30 @@ StringBuilder appendErrorElements(StringBuilder errorBuilder, float[] vector) {
             errorBuilder.append("]");
             return errorBuilder;
         }
+
+        StringBuilder appendErrorElements(StringBuilder errorBuilder, byte[] vector) {
+            // Include the first five elements of the invalid vector in the error message
+            errorBuilder.append(" Preview of invalid vector: [");
+            for (int i = 0; i < Math.min(5, vector.length); i++) {
+                if (i > 0) {
+                    errorBuilder.append(", ");
+                }
+                errorBuilder.append(vector[i]);
+            }
+            if (vector.length >= 5) {
+                errorBuilder.append(", ...");
+            }
+            errorBuilder.append("]");
+            return errorBuilder;
+        }
+
+        Function<StringBuilder, StringBuilder> errorFloatElementsAppender(float[] vector) {
+            return sb -> appendErrorElements(sb, vector);
+        }
+
+        Function<StringBuilder, StringBuilder> errorByteElementsAppender(byte[] vector) {
+            return sb -> appendErrorElements(sb, vector);
+        }
     }
 
     static final Map<String, ElementType> namesToElementType = Map.of(
@@ -393,15 +571,20 @@ StringBuilder appendErrorElements(StringBuilder errorBuilder, float[] vector) {
     );
 
     enum VectorSimilarity {
-        l2_norm(VectorSimilarityFunction.EUCLIDEAN),
-        cosine(VectorSimilarityFunction.COSINE),
-        dot_product(VectorSimilarityFunction.DOT_PRODUCT);
+        L2_NORM(VectorSimilarityFunction.EUCLIDEAN),
+        COSINE(VectorSimilarityFunction.COSINE),
+        DOT_PRODUCT(VectorSimilarityFunction.DOT_PRODUCT);
 
         public final VectorSimilarityFunction function;
 
         VectorSimilarity(VectorSimilarityFunction function) {
             this.function = function;
         }
+
+        @Override
+        public final String toString() {
+            return name().toLowerCase(Locale.ROOT);
+        }
     }
 
     private abstract static class IndexOptions implements ToXContent {
@@ -540,7 +723,7 @@ public Query termQuery(Object value, SearchExecutionContext context) {
             throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support term queries");
         }
 
-        public KnnVectorQuery createKnnQuery(float[] queryVector, int numCands, Query filter) {
+        public Query createKnnQuery(byte[] queryVector, int numCands, Query filter) {
             if (isIndexed() == false) {
                 throw new IllegalArgumentException(
                     "to perform knn search on field [" + name() + "], its mapping must have [index] set to [true]"
@@ -553,17 +736,54 @@ public KnnVectorQuery createKnnQuery(float[] queryVector, int numCands, Query fi
                 );
             }
 
+            if (elementType != ElementType.BYTE) {
+                throw new IllegalArgumentException(
+                    "only [" + ElementType.BYTE + "] elements are supported when querying field [" + name() + "]"
+                );
+            }
+
+            if (similarity == VectorSimilarity.DOT_PRODUCT || similarity == VectorSimilarity.COSINE) {
+                float squaredMagnitude = 0.0f;
+                for (byte b : queryVector) {
+                    squaredMagnitude += b * b;
+                }
+                elementType.checkVectorMagnitude(similarity, elementType.errorByteElementsAppender(queryVector), squaredMagnitude);
+            }
+
+            return new KnnByteVectorQuery(name(), queryVector, numCands, filter);
+        }
+
+        public Query createKnnQuery(float[] queryVector, int numCands, Query filter) {
+            if (isIndexed() == false) {
+                throw new IllegalArgumentException(
+                    "to perform knn search on field [" + name() + "], its mapping must have [index] set to [true]"
+                );
+            }
+
+            if (queryVector.length != dims) {
+                throw new IllegalArgumentException(
+                    "the query vector has a different dimension [" + queryVector.length + "] than the index vectors [" + dims + "]"
+                );
+            }
             elementType.checkVectorBounds(queryVector);
 
-            if (similarity == VectorSimilarity.dot_product || similarity == VectorSimilarity.cosine) {
+            if (similarity == VectorSimilarity.DOT_PRODUCT || similarity == VectorSimilarity.COSINE) {
                 float squaredMagnitude = 0.0f;
                 for (float e : queryVector) {
                     squaredMagnitude += e * e;
                 }
-                elementType.checkVectorMagnitude(similarity, queryVector, squaredMagnitude);
+                elementType.checkVectorMagnitude(similarity, elementType.errorFloatElementsAppender(queryVector), squaredMagnitude);
             }
-
-            return new KnnVectorQuery(name(), queryVector, numCands, filter);
+            return switch (elementType) {
+                case BYTE -> {
+                    byte[] bytes = new byte[queryVector.length];
+                    for (int i = 0; i < queryVector.length; i++) {
+                        bytes[i] = (byte) queryVector[i];
+                    }
+                    yield new KnnByteVectorQuery(name(), bytes, numCands, filter);
+                }
+                case FLOAT -> new KnnFloatVectorQuery(name(), queryVector, numCands, filter);
+            };
         }
     }
 
@@ -616,27 +836,15 @@ public void parse(DocumentParserContext context) throws IOException {
                     + "] doesn't not support indexing multiple values for the same field in the same document"
             );
         }
-
+        if (Token.VALUE_NULL == context.parser().currentToken()) {
+            return;
+        }
         Field field = fieldType().indexed ? parseKnnVector(context) : parseBinaryDocValuesVector(context);
         context.doc().addWithKey(fieldType().name(), field);
     }
 
     private Field parseKnnVector(DocumentParserContext context) throws IOException {
-        float[] vector = new float[dims];
-        float squaredMagnitude = 0.0f;
-        int index = 0;
-        for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
-            checkDimensionExceeded(index, context);
-            ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser());
-
-            float value = context.parser().floatValue(true);
-            vector[index++] = value;
-            squaredMagnitude += value * value;
-        }
-        checkDimensionMatches(index, context);
-        elementType.checkVectorBounds(vector);
-        elementType.checkVectorMagnitude(similarity, vector, squaredMagnitude);
-        return elementType.createKnnVectorField(fieldType().name(), vector, similarity.function);
+        return elementType.parseKnnVector(context, this);
     }
 
     private Field parseBinaryDocValuesVector(DocumentParserContext context) throws IOException {
@@ -647,22 +855,7 @@ private Field parseBinaryDocValuesVector(DocumentParserContext context) throws I
             : new byte[dims * elementType.elementBytes];
 
         ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
-        double dotProduct = 0f;
-
-        int index = 0;
-        float[] vector = new float[dims];
-        for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
-            checkDimensionExceeded(index, context);
-            ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser());
-            float value = context.parser().floatValue(true);
-            vector[index] = value;
-            elementType.writeValue(byteBuffer, value);
-            dotProduct += value * value;
-            index++;
-        }
-        checkDimensionMatches(index, context);
-        elementType.checkVectorBounds(vector);
-
+        double dotProduct = elementType.parseKnnVectorToByteBuffer(context, this, byteBuffer);
         if (indexCreatedVersion.onOrAfter(Version.V_7_5_0)) {
             // encode vector magnitude at the end
             float vectorMagnitude = (float) Math.sqrt(dotProduct);
@@ -753,7 +946,7 @@ public KnnVectorsFormat getKnnVectorsFormatForField() {
             return null; // use default format
         } else {
             HnswIndexOptions hnswIndexOptions = (HnswIndexOptions) indexOptions;
-            return new Lucene94HnswVectorsFormat(hnswIndexOptions.m, hnswIndexOptions.efConstruction);
+            return new Lucene95HnswVectorsFormat(hnswIndexOptions.m, hnswIndexOptions.efConstruction);
         }
     }
 
@@ -771,7 +964,8 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
     }
 
     private class IndexedSyntheticFieldLoader implements SourceLoader.SyntheticFieldLoader {
-        private VectorValues values;
+        private FloatVectorValues values;
+        private ByteVectorValues byteVectorValues;
         private boolean hasValue;
 
         @Override
@@ -781,14 +975,21 @@ public Stream<Map.Entry<String, StoredFieldLoader>> storedFieldLoaders() {
 
         @Override
         public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException {
-            values = leafReader.getVectorValues(name());
-            if (values == null) {
-                return null;
+            values = leafReader.getFloatVectorValues(name());
+            if (values != null) {
+                return docId -> {
+                    hasValue = docId == values.advance(docId);
+                    return hasValue;
+                };
             }
-            return docId -> {
-                hasValue = docId == values.advance(docId);
-                return hasValue;
-            };
+            byteVectorValues = leafReader.getByteVectorValues(name());
+            if (byteVectorValues != null) {
+                return docId -> {
+                    hasValue = docId == byteVectorValues.advance(docId);
+                    return hasValue;
+                };
+            }
+            return null;
         }
 
         @Override
@@ -802,8 +1003,15 @@ public void write(XContentBuilder b) throws IOException {
                 return;
             }
             b.startArray(simpleName());
-            for (float v : values.vectorValue()) {
-                b.value(v);
+            if (values != null) {
+                for (float v : values.vectorValue()) {
+                    b.value(v);
+                }
+            } else if (byteVectorValues != null) {
+                byte[] vectorValue = byteVectorValues.vectorValue();
+                for (byte value : vectorValue) {
+                    b.value(value);
+                }
             }
             b.endArray();
         }
@@ -844,7 +1052,7 @@ public void write(XContentBuilder b) throws IOException {
             BytesRef ref = values.binaryValue();
             ByteBuffer byteBuffer = ByteBuffer.wrap(ref.bytes, ref.offset, ref.length);
             for (int dim = 0; dim < dims; dim++) {
-                b.value(elementType.readValue(byteBuffer));
+                elementType.readAndWriteValue(byteBuffer, b);
             }
             b.endArray();
         }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java
index 736a5bdf275b..d8afdd8e2d06 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java
@@ -11,7 +11,6 @@
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.VectorValues;
 import org.elasticsearch.Version;
 import org.elasticsearch.index.fielddata.LeafFieldData;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
@@ -56,15 +55,9 @@ public SortedBinaryDocValues getBytesValues() {
     public DocValuesScriptFieldFactory getScriptFieldFactory(String name) {
         try {
             if (indexed) {
-                VectorValues values = reader.getVectorValues(field);
-                if (values == VectorValues.EMPTY) {
-                    // There's no way for KnnDenseVectorDocValuesField to reliably differentiate between VectorValues.EMPTY and
-                    // values that can be iterated through. Since VectorValues.EMPTY throws on docID(), pass a null instead.
-                    values = null;
-                }
                 return switch (elementType) {
-                    case BYTE -> new ByteKnnDenseVectorDocValuesField(values, name, elementType, dims);
-                    case FLOAT -> new KnnDenseVectorDocValuesField(values, name, elementType, dims);
+                    case BYTE -> new ByteKnnDenseVectorDocValuesField(reader.getByteVectorValues(field), name, dims);
+                    case FLOAT -> new KnnDenseVectorDocValuesField(reader.getFloatVectorValues(field), name, dims);
                 };
             } else {
                 BinaryDocValues values = DocValues.getBinary(reader, field);
diff --git a/server/src/main/java/org/elasticsearch/index/query/AbstractGeometryQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/AbstractGeometryQueryBuilder.java
index 525cc72d74c9..e2b566a88d09 100644
--- a/server/src/main/java/org/elasticsearch/index/query/AbstractGeometryQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/AbstractGeometryQueryBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.SetOnce;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.client.internal.Client;
@@ -141,7 +141,7 @@ protected AbstractGeometryQueryBuilder(StreamInput in) throws IOException {
         } else {
             shape = null;
             indexedShapeId = in.readOptionalString();
-            if (in.getVersion().before(Version.V_8_0_0)) {
+            if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 String type = in.readOptionalString();
                 assert MapperService.SINGLE_MAPPING_NAME.equals(type) : "Expected type [_doc], got [" + type + "]";
             }
@@ -166,7 +166,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
             GeometryIO.writeGeometry(out, shape);
         } else {
             out.writeOptionalString(indexedShapeId);
-            if (out.getVersion().before(Version.V_8_0_0)) {
+            if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 out.writeOptionalString(MapperService.SINGLE_MAPPING_NAME);
             }
             out.writeOptionalString(indexedShapeIndex);
diff --git a/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
index 24d4735759b8..4be406f5e924 100644
--- a/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -406,7 +406,7 @@ private static boolean rewriteClauses(
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java
index afc46972ef13..fc08aa1f23cc 100644
--- a/server/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.queries.function.FunctionScoreQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -229,7 +229,7 @@ protected void extractInnerHitBuilders(Map<String, InnerHitContextBuilder> inner
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java
index 52741a88e056..86b4320659a0 100644
--- a/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilder.java
@@ -21,7 +21,7 @@
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.QueryBuilder;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -407,11 +407,11 @@ public Query createPhraseQuery(String field, String queryText, int phraseSlop) {
         }
 
         @Override
-        protected Query newSynonymQuery(TermAndBoost[] terms) {
+        protected Query newSynonymQuery(String field, TermAndBoost[] terms) {
             CombinedFieldQuery.Builder query = new CombinedFieldQuery.Builder();
             for (TermAndBoost termAndBoost : terms) {
                 assert termAndBoost.boost == BoostAttribute.DEFAULT_BOOST;
-                BytesRef bytes = termAndBoost.term.bytes();
+                BytesRef bytes = termAndBoost.term;
                 query.addTerm(bytes);
             }
             for (FieldAndBoost fieldAndBoost : fields) {
@@ -424,8 +424,8 @@ protected Query newSynonymQuery(TermAndBoost[] terms) {
 
         @Override
         protected Query newTermQuery(Term term, float boost) {
-            TermAndBoost termAndBoost = new TermAndBoost(term, boost);
-            return newSynonymQuery(new TermAndBoost[] { termAndBoost });
+            TermAndBoost termAndBoost = new TermAndBoost(term.bytes(), boost);
+            return newSynonymQuery(term.field(), new TermAndBoost[] { termAndBoost });
         }
 
         @Override
@@ -458,7 +458,7 @@ protected boolean doEquals(CombinedFieldsQueryBuilder other) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_13_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_13_0;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
index 86dffdb9edbb..2fb9275e125f 100644
--- a/server/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.DeprecationLogger;
@@ -62,7 +62,7 @@ public static CommonTermsQueryBuilder fromXContent(XContentParser parser) throws
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java
index adc48c8e1cca..80fd568f7210 100644
--- a/server/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -160,7 +160,7 @@ protected void extractInnerHitBuilders(Map<String, InnerHitContextBuilder> inner
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java
index b282996bc7a9..8cfc43f92fb0 100644
--- a/server/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -223,7 +223,7 @@ protected void extractInnerHitBuilders(Map<String, InnerHitContextBuilder> inner
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/DistanceFeatureQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/DistanceFeatureQueryBuilder.java
index b3e9ccc7f787..3b67f59e81c8 100644
--- a/server/src/main/java/org/elasticsearch/index/query/DistanceFeatureQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/DistanceFeatureQueryBuilder.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
@@ -198,7 +198,7 @@ public String toString() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_2_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_2_0;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java
index 11c056974ec5..d7e54df00ed0 100644
--- a/server/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -178,7 +178,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java
index 5ff78c048d38..05723117ebea 100644
--- a/server/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.queries.spans.FieldMaskingSpanQuery;
 import org.apache.lucene.queries.spans.SpanQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -176,7 +176,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java
index 850c4b0809a7..3d28b89d45f7 100644
--- a/server/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -357,7 +357,7 @@ protected boolean doEquals(FuzzyQueryBuilder other) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
index 3b88d00b9769..31faad19e2e2 100644
--- a/server/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Numbers;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.geo.GeoBoundingBox;
@@ -76,7 +76,7 @@ public GeoBoundingBoxQueryBuilder(StreamInput in) throws IOException {
         super(in);
         fieldName = in.readString();
         geoBoundingBox = new GeoBoundingBox(in);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             in.readVInt(); // ignore value
         }
         validationMethod = GeoValidationMethod.readFromStream(in);
@@ -87,7 +87,7 @@ public GeoBoundingBoxQueryBuilder(StreamInput in) throws IOException {
     protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeString(fieldName);
         geoBoundingBox.writeTo(out);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeVInt(0);
         }
         validationMethod.writeTo(out);
@@ -395,7 +395,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java
index 87f8c79da6ad..5b35a9af9993 100644
--- a/server/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.geo.GeoDistance;
@@ -401,7 +401,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java
index 03358ea4835b..a662d08fdd47 100644
--- a/server/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java
@@ -14,7 +14,7 @@
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.geo.GeoPoint;
@@ -328,7 +328,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
index 0f80cc7cc538..fb342d6b71a9 100644
--- a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.geo.GeometryParser;
 import org.elasticsearch.common.geo.ShapeRelation;
@@ -270,7 +270,7 @@ public static GeoShapeQueryBuilder fromXContent(XContentParser parser) throws IO
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java
index 25fb35801cf4..ea4e37d61f77 100644
--- a/server/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -53,7 +53,7 @@ public IdsQueryBuilder() {
      */
     public IdsQueryBuilder(StreamInput in) throws IOException {
         super(in);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             // types no longer relevant so ignore
             String[] types = in.readStringArray();
             if (types.length > 0) {
@@ -65,7 +65,7 @@ public IdsQueryBuilder(StreamInput in) throws IOException {
 
     @Override
     protected void doWriteTo(StreamOutput out) throws IOException {
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             // types not supported so send an empty array to previous versions
             out.writeStringArray(Strings.EMPTY_ARRAY);
         }
@@ -155,7 +155,7 @@ protected boolean doEquals(IdsQueryBuilder other) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
index f3eaf053ce79..39b62b54bf08 100644
--- a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.index.query;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -189,7 +189,7 @@ public InnerHitBuilder(StreamInput in) throws IOException {
         highlightBuilder = in.readOptionalWriteable(HighlightBuilder::new);
         this.innerCollapseBuilder = in.readOptionalWriteable(CollapseBuilder::new);
 
-        if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             if (in.readBoolean()) {
                 fetchFields = in.readList(FieldAndFormat::new);
             }
@@ -229,7 +229,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(highlightBuilder);
         out.writeOptionalWriteable(innerCollapseBuilder);
 
-        if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             out.writeBoolean(fetchFields != null);
             if (fetchFields != null) {
                 out.writeList(fetchFields);
diff --git a/server/src/main/java/org/elasticsearch/index/query/IntervalQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/IntervalQueryBuilder.java
index 3b1f16ad6334..5d42a6eb750d 100644
--- a/server/src/main/java/org/elasticsearch/index/query/IntervalQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/IntervalQueryBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.queries.intervals.IntervalQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -155,7 +155,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java b/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java
index 01c5a749caf0..a0ace42c4088 100644
--- a/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java
+++ b/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.queries.intervals.Intervals;
 import org.apache.lucene.queries.intervals.IntervalsSource;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -120,7 +120,7 @@ public Match(StreamInput in) throws IOException {
             this.ordered = in.readBoolean();
             this.analyzer = in.readOptionalString();
             this.filter = in.readOptionalWriteable(IntervalFilter::new);
-            if (in.getVersion().onOrAfter(Version.V_7_2_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_2_0)) {
                 this.useField = in.readOptionalString();
             } else {
                 this.useField = null;
@@ -204,7 +204,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(ordered);
             out.writeOptionalString(analyzer);
             out.writeOptionalWriteable(filter);
-            if (out.getVersion().onOrAfter(Version.V_7_2_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_2_0)) {
                 out.writeOptionalString(useField);
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/index/query/MatchAllQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MatchAllQueryBuilder.java
index def6fcdeea2f..ff50fd1e8e8a 100644
--- a/server/src/main/java/org/elasticsearch/index/query/MatchAllQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/MatchAllQueryBuilder.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -82,7 +82,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/MatchBoolPrefixQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MatchBoolPrefixQueryBuilder.java
index f06e502ac4c1..5d6a5892e6ca 100644
--- a/server/src/main/java/org/elasticsearch/index/query/MatchBoolPrefixQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/MatchBoolPrefixQueryBuilder.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -404,7 +404,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_2_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_2_0;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java
index e83aabd16169..354cb4e060a1 100644
--- a/server/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -100,7 +100,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilder.java
index 23ef97e1cd59..d7df607a2c70 100644
--- a/server/src/main/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilder.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -64,7 +64,7 @@ public MatchPhrasePrefixQueryBuilder(StreamInput in) throws IOException {
         slop = in.readVInt();
         maxExpansions = in.readVInt();
         analyzer = in.readOptionalString();
-        if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             this.zeroTermsQuery = ZeroTermsQueryOption.readFromStream(in);
         }
     }
@@ -76,7 +76,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeVInt(slop);
         out.writeVInt(maxExpansions);
         out.writeOptionalString(analyzer);
-        if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             zeroTermsQuery.writeTo(out);
         }
     }
@@ -292,7 +292,7 @@ public static MatchPhrasePrefixQueryBuilder fromXContent(XContentParser parser)
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/MatchPhraseQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MatchPhraseQueryBuilder.java
index 3fbfc1ce0f69..74acd1a419e8 100644
--- a/server/src/main/java/org/elasticsearch/index/query/MatchPhraseQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/MatchPhraseQueryBuilder.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -288,7 +288,7 @@ public static MatchPhraseQueryBuilder fromXContent(XContentParser parser) throws
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
index 8353063a2cc2..b96085accf24 100644
--- a/server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -117,7 +117,7 @@ public MatchQueryBuilder(StreamInput in) throws IOException {
         fuzzyRewrite = in.readOptionalString();
         fuzziness = in.readOptionalWriteable(Fuzziness::new);
         // cutoff_frequency has been removed
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             in.readOptionalFloat();
         }
         autoGenerateSynonymsPhraseQuery = in.readBoolean();
@@ -139,7 +139,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeOptionalString(fuzzyRewrite);
         out.writeOptionalWriteable(fuzziness);
         // cutoff_frequency has been removed
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeOptionalFloat(null);
         }
         out.writeBoolean(autoGenerateSynonymsPhraseQuery);
@@ -577,7 +577,7 @@ public static MatchQueryBuilder fromXContent(XContentParser parser) throws IOExc
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
index 22913f181e1e..de168614ccdd 100644
--- a/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
@@ -15,7 +15,7 @@
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.RoutingMissingException;
 import org.elasticsearch.action.termvectors.MultiTermVectorsItemResponse;
 import org.elasticsearch.action.termvectors.MultiTermVectorsRequest;
@@ -206,7 +206,7 @@ public Item(@Nullable String index, XContentBuilder doc) {
         @SuppressWarnings("unchecked")
         Item(StreamInput in) throws IOException {
             index = in.readOptionalString();
-            if (in.getVersion().before(Version.V_8_0_0)) {
+            if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 // types no longer relevant so ignore
                 String type = in.readOptionalString();
                 if (type != null) {
@@ -229,7 +229,7 @@ public Item(@Nullable String index, XContentBuilder doc) {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalString(index);
-            if (out.getVersion().before(Version.V_8_0_0)) {
+            if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 // types not supported so send an empty array to previous versions
                 out.writeOptionalString(null);
             }
@@ -1185,7 +1185,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
index 27a1b696f156..dfc6a95bb7ff 100644
--- a/server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -223,7 +223,7 @@ public MultiMatchQueryBuilder(StreamInput in) throws IOException {
         fuzzyRewrite = in.readOptionalString();
         tieBreaker = in.readOptionalFloat();
         lenient = in.readOptionalBoolean();
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             in.readOptionalFloat();
         }
         zeroTermsQuery = ZeroTermsQueryOption.readFromStream(in);
@@ -246,7 +246,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeOptionalString(fuzzyRewrite);
         out.writeOptionalFloat(tieBreaker);
         out.writeOptionalBoolean(lenient);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeOptionalFloat(null);
         }
         zeroTermsQuery.writeTo(out);
@@ -826,7 +826,7 @@ protected boolean doEquals(MultiMatchQueryBuilder other) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
index 75ed4b551cd7..d36e2f7d7eef 100644
--- a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
@@ -24,7 +24,7 @@
 import org.apache.lucene.search.join.ParentChildrenBlockJoinQuery;
 import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.search.MaxScoreCollector;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -438,7 +438,7 @@ public TopDocsAndMaxScore topDocs(SearchHit hit) throws IOException {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java
index 8867d91902da..40d7737ebd7e 100644
--- a/server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java
@@ -12,7 +12,7 @@
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -72,7 +72,7 @@ public PrefixQueryBuilder(StreamInput in) throws IOException {
         fieldName = in.readString();
         value = in.readString();
         rewrite = in.readOptionalString();
-        if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             caseInsensitive = in.readBoolean();
         }
     }
@@ -82,7 +82,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeString(fieldName);
         out.writeString(value);
         out.writeOptionalString(rewrite);
-        if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             out.writeBoolean(caseInsensitive);
         }
     }
@@ -235,7 +235,7 @@ protected boolean doEquals(PrefixQueryBuilder other) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java
index a9f3027cf2c2..bd0af0ac8c12 100644
--- a/server/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java
@@ -12,7 +12,7 @@
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.automaton.Operations;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -967,7 +967,7 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
index addd5f1ceca7..04e228fab331 100644
--- a/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.geo.ShapeRelation;
@@ -551,7 +551,7 @@ protected boolean doEquals(RangeQueryBuilder other) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java
index f6ee1620a0ea..c802209ddbb7 100644
--- a/server/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java
@@ -14,7 +14,7 @@
 import org.apache.lucene.search.RegexpQuery;
 import org.apache.lucene.util.automaton.Operations;
 import org.apache.lucene.util.automaton.RegExp;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -86,7 +86,7 @@ public RegexpQueryBuilder(StreamInput in) throws IOException {
         syntaxFlagsValue = in.readVInt();
         maxDeterminizedStates = in.readVInt();
         rewrite = in.readOptionalString();
-        if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             caseInsensitive = in.readBoolean();
         }
     }
@@ -98,7 +98,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeVInt(syntaxFlagsValue);
         out.writeVInt(maxDeterminizedStates);
         out.writeOptionalString(rewrite);
-        if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             out.writeBoolean(caseInsensitive);
         }
     }
@@ -315,7 +315,7 @@ protected boolean doEquals(RegexpQueryBuilder other) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
index a4b57a31bb00..6200d660d565 100644
--- a/server/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
@@ -20,7 +20,7 @@
 import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -224,7 +224,7 @@ protected boolean doEquals(ScriptQueryBuilder other) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java
index 0aa390b13fe9..eb3c01c90c2f 100644
--- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java
+++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java
@@ -59,7 +59,7 @@
 import org.elasticsearch.search.NestedDocuments;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.transport.RemoteClusterAware;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
 
@@ -491,21 +491,33 @@ public boolean containsBrokenAnalysis(String field) {
      */
     public SearchLookup lookup() {
         if (this.lookup == null) {
-            SourceLookup.SourceProvider sourceProvider = mappingLookup == null
-                ? new SourceLookup.ReaderSourceProvider()
-                : mappingLookup.getSourceProvider();
-            this.lookup = new SearchLookup(
-                this::getFieldType,
-                (fieldType, searchLookup, fielddataOperation) -> indexFieldDataLookup.apply(
-                    fieldType,
-                    new FieldDataContext(fullyQualifiedIndex.getName(), searchLookup, this::sourcePath, fielddataOperation)
-                ),
-                sourceProvider
-            );
+            SourceProvider sourceProvider = isSourceSynthetic()
+                ? (ctx, doc) -> { throw new IllegalArgumentException("Cannot access source from scripts in synthetic mode"); }
+                : SourceProvider.fromStoredFields();
+            setSourceProvider(sourceProvider);
         }
         return this.lookup;
     }
 
+    /**
+     * Replace the standard source provider on the SearchLookup
+     *
+     * Note that this will replace the current SearchLookup with a new one, but will not update
+     * the source provider on previously build lookups. This method should only be called before
+     * IndexReader access by the current context
+     */
+    public void setSourceProvider(SourceProvider sourceProvider) {
+        // TODO can we assert that this is only called during FetchPhase?
+        this.lookup = new SearchLookup(
+            this::getFieldType,
+            (fieldType, searchLookup, fielddataOperation) -> indexFieldDataLookup.apply(
+                fieldType,
+                new FieldDataContext(fullyQualifiedIndex.getName(), searchLookup, this::sourcePath, fielddataOperation)
+            ),
+            sourceProvider
+        );
+    }
+
     public NestedScope nestedScope() {
         return nestedScope;
     }
diff --git a/server/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java
index c26675f50060..906db7cc6366 100644
--- a/server/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -599,7 +599,7 @@ protected boolean doEquals(SimpleQueryStringBuilder other) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java
index 72d94a60b2d3..e8eb0e5faf6a 100644
--- a/server/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.queries.spans.SpanContainingQuery;
 import org.apache.lucene.queries.spans.SpanQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -165,7 +165,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java
index 460ef50bd3dd..7be46f4807ec 100644
--- a/server/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.queries.spans.SpanFirstQuery;
 import org.apache.lucene.queries.spans.SpanQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -160,7 +160,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java
index e2bd8c88f967..967b8618a0ae 100644
--- a/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java
@@ -14,7 +14,7 @@
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TopTermsRewrite;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -187,7 +187,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
index 9b9a772af7a2..cf24a17d18a4 100644
--- a/server/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.queries.spans.SpanNearQuery;
 import org.apache.lucene.queries.spans.SpanQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -274,8 +274,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     /**
@@ -370,8 +370,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.V_EMPTY;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.ZERO;
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java
index befca7fd87df..8b81be1b8f0a 100644
--- a/server/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.queries.spans.SpanNotQuery;
 import org.apache.lucene.queries.spans.SpanQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -262,7 +262,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java
index cf1738dddecc..9847d4206bd9 100644
--- a/server/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.queries.spans.SpanOrQuery;
 import org.apache.lucene.queries.spans.SpanQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -165,7 +165,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java
index 87940e2f09f1..7b82e8ed72d5 100644
--- a/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java
@@ -12,7 +12,7 @@
 import org.apache.lucene.queries.spans.SpanQuery;
 import org.apache.lucene.queries.spans.SpanTermQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.lucene.BytesRefs;
@@ -132,7 +132,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java
index 1187b1771480..6e3ca76aabf4 100644
--- a/server/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.queries.spans.SpanQuery;
 import org.apache.lucene.queries.spans.SpanWithinQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -174,7 +174,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java
index 0e5084a576ee..6d6616f47531 100644
--- a/server/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -86,7 +86,7 @@ public boolean caseInsensitive() {
      */
     public TermQueryBuilder(StreamInput in) throws IOException {
         super(in);
-        if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             caseInsensitive = in.readBoolean();
         }
     }
@@ -94,7 +94,7 @@ public TermQueryBuilder(StreamInput in) throws IOException {
     @Override
     protected void doWriteTo(StreamOutput out) throws IOException {
         super.doWriteTo(out);
-        if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             out.writeBoolean(caseInsensitive);
         }
     }
@@ -225,7 +225,7 @@ protected final boolean doEquals(TermQueryBuilder other) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
index c88a0ee73da8..146e2cd4daf7 100644
--- a/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
@@ -12,7 +12,7 @@
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.SetOnce;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.client.internal.Client;
@@ -49,7 +49,7 @@
  */
 public class TermsQueryBuilder extends AbstractQueryBuilder<TermsQueryBuilder> {
     public static final String NAME = "terms";
-    private static final Version VERSION_STORE_VALUES_AS_BYTES_REFERENCE = Version.V_7_12_0;
+    private static final TransportVersion VERSION_STORE_VALUES_AS_BYTES_REFERENCE = TransportVersion.V_7_12_0;
 
     private final String fieldName;
     private final Values values;
@@ -415,7 +415,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) {
     private abstract static class Values extends AbstractCollection implements Writeable {
 
         private static Values readFrom(StreamInput in) throws IOException {
-            if (in.getVersion().onOrAfter(VERSION_STORE_VALUES_AS_BYTES_REFERENCE)) {
+            if (in.getTransportVersion().onOrAfter(VERSION_STORE_VALUES_AS_BYTES_REFERENCE)) {
                 return in.readOptionalWriteable(BinaryValues::new);
             } else {
                 List<?> list = (List<?>) in.readGenericValue();
@@ -424,7 +424,7 @@ private static Values readFrom(StreamInput in) throws IOException {
         }
 
         private static void writeTo(StreamOutput out, Values values) throws IOException {
-            if (out.getVersion().onOrAfter(VERSION_STORE_VALUES_AS_BYTES_REFERENCE)) {
+            if (out.getTransportVersion().onOrAfter(VERSION_STORE_VALUES_AS_BYTES_REFERENCE)) {
                 out.writeOptionalWriteable(values);
             } else {
                 if (values == null) {
@@ -561,7 +561,7 @@ public Object next() {
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            if (out.getVersion().onOrAfter(VERSION_STORE_VALUES_AS_BYTES_REFERENCE)) {
+            if (out.getTransportVersion().onOrAfter(VERSION_STORE_VALUES_AS_BYTES_REFERENCE)) {
                 out.writeBytesReference(valueRef);
             } else {
                 valueRef.writeTo(out);
@@ -642,7 +642,7 @@ public Object[] toArray(IntFunction generator) {
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            if (out.getVersion().onOrAfter(VERSION_STORE_VALUES_AS_BYTES_REFERENCE)) {
+            if (out.getTransportVersion().onOrAfter(VERSION_STORE_VALUES_AS_BYTES_REFERENCE)) {
                 BytesReference bytesRef = serialize(values, false);
                 out.writeBytesReference(bytesRef);
             } else {
@@ -665,7 +665,7 @@ public int hashCode() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java
index d6d31b03a8f9..0822dbf229f4 100644
--- a/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java
@@ -18,7 +18,7 @@
 import org.apache.lucene.search.LongValuesSource;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -438,7 +438,7 @@ public LongValuesSource rewrite(IndexSearcher searcher) throws IOException {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java b/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java
index a9b0fd9da018..dd293ac40c6a 100644
--- a/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -98,7 +98,7 @@ public void setValue(String value) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java
index f0a9d0495c0d..cfabd9fa7c56 100644
--- a/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java
@@ -12,7 +12,7 @@
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -83,7 +83,7 @@ public WildcardQueryBuilder(StreamInput in) throws IOException {
         fieldName = in.readString();
         value = in.readString();
         rewrite = in.readOptionalString();
-        if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             caseInsensitive = in.readBoolean();
         }
     }
@@ -93,7 +93,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeString(fieldName);
         out.writeString(value);
         out.writeOptionalString(rewrite);
-        if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             out.writeBoolean(caseInsensitive);
         }
     }
@@ -248,7 +248,7 @@ protected boolean doEquals(WildcardQueryBuilder other) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java
index 8781a98b8355..c151cca780ab 100644
--- a/server/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -161,7 +161,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext context) throws IOException
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/ExponentialDecayFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/ExponentialDecayFunctionBuilder.java
index 6d62e96ad711..b26efb90fb36 100644
--- a/server/src/main/java/org/elasticsearch/index/query/functionscore/ExponentialDecayFunctionBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/ExponentialDecayFunctionBuilder.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.index.query.functionscore;
 
 import org.apache.lucene.search.Explanation;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 
@@ -83,7 +83,7 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/FieldValueFactorFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/FieldValueFactorFunctionBuilder.java
index 50c765d4eba9..68f4ea71ea26 100644
--- a/server/src/main/java/org/elasticsearch/index/query/functionscore/FieldValueFactorFunctionBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/FieldValueFactorFunctionBuilder.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.index.query.functionscore;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -133,8 +133,8 @@ protected int doHashCode() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java
index 274d4373a816..57ce77eda5bd 100644
--- a/server/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -649,7 +649,7 @@ private static String parseFiltersAndFunctions(
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/GaussDecayFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/GaussDecayFunctionBuilder.java
index bba3fa22a994..c90c89c577b5 100644
--- a/server/src/main/java/org/elasticsearch/index/query/functionscore/GaussDecayFunctionBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/GaussDecayFunctionBuilder.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.index.query.functionscore;
 
 import org.apache.lucene.search.Explanation;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.xcontent.ParseField;
@@ -52,8 +52,8 @@ public DecayFunction getDecayFunction() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     private static final class GaussScoreFunction implements DecayFunction {
diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/LinearDecayFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/LinearDecayFunctionBuilder.java
index f4729a59cef3..5c4ec45aa0ac 100644
--- a/server/src/main/java/org/elasticsearch/index/query/functionscore/LinearDecayFunctionBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/LinearDecayFunctionBuilder.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.index.query.functionscore;
 
 import org.apache.lucene.search.Explanation;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 
@@ -57,8 +57,8 @@ public DecayFunction getDecayFunction() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     private static final class LinearDecayScoreFunction implements DecayFunction {
diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java
index 1cbe92ea4c45..8fe58ae80e0b 100644
--- a/server/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.index.query.functionscore;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -180,8 +180,8 @@ private static int hash(long value) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     public static RandomScoreFunctionBuilder fromXContent(XContentParser parser) throws IOException, ParsingException {
diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java
index f042132e1338..9ba4eaa9eed9 100644
--- a/server/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.index.query.functionscore;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -81,8 +81,8 @@ protected int doHashCode() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreQueryBuilder.java
index d658cb678bcf..44e517e36580 100644
--- a/server/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreQueryBuilder.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery;
@@ -201,7 +201,7 @@ protected void extractInnerHitBuilders(Map<String, InnerHitContextBuilder> inner
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/WeightBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/WeightBuilder.java
index 38ec58e68e76..2af55eaaf844 100644
--- a/server/src/main/java/org/elasticsearch/index/query/functionscore/WeightBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/WeightBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.index.query.functionscore;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.search.function.ScoreFunction;
@@ -57,8 +57,8 @@ protected int doHashCode() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/refresh/RefreshStats.java b/server/src/main/java/org/elasticsearch/index/refresh/RefreshStats.java
index 58aa34851fe8..5b627e4a77eb 100644
--- a/server/src/main/java/org/elasticsearch/index/refresh/RefreshStats.java
+++ b/server/src/main/java/org/elasticsearch/index/refresh/RefreshStats.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.index.refresh;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -39,7 +39,7 @@ public RefreshStats() {}
     public RefreshStats(StreamInput in) throws IOException {
         total = in.readVLong();
         totalTimeInMillis = in.readVLong();
-        if (in.getVersion().onOrAfter(Version.V_7_2_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_2_0)) {
             externalTotal = in.readVLong();
             externalTotalTimeInMillis = in.readVLong();
         }
@@ -50,7 +50,7 @@ public RefreshStats(StreamInput in) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(total);
         out.writeVLong(totalTimeInMillis);
-        if (out.getVersion().onOrAfter(Version.V_7_2_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_2_0)) {
             out.writeVLong(externalTotal);
             out.writeVLong(externalTotalTimeInMillis);
         }
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
index 32b8c67fbcf4..85424c2eef7d 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
@@ -123,6 +123,11 @@ protected DeleteByQueryRequest self() {
         return this;
     }
 
+    @Override
+    public boolean includeDataStreams() {
+        return true;
+    }
+
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException e = super.validate();
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/RemoteInfo.java b/server/src/main/java/org/elasticsearch/index/reindex/RemoteInfo.java
index f35558ad6a56..f168ef9ec2dc 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/RemoteInfo.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/RemoteInfo.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -99,7 +99,7 @@ public RemoteInfo(StreamInput in) throws IOException {
         port = in.readVInt();
         query = in.readBytesReference();
         username = in.readOptionalString();
-        if (in.getVersion().before(Version.V_8_2_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_2_0)) {
             password = new SecureString(in.readOptionalString().toCharArray());
         } else {
             password = in.readOptionalSecureString();
@@ -122,7 +122,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVInt(port);
         out.writeBytesReference(query);
         out.writeOptionalString(username);
-        if (out.getVersion().before(Version.V_8_2_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_2_0)) {
             out.writeOptionalString(password.toString());
         } else {
             out.writeOptionalSecureString(password);
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
index 23116135ea2f..d30b54fdafd4 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
@@ -126,6 +126,11 @@ protected UpdateByQueryRequest self() {
         return this;
     }
 
+    @Override
+    public boolean includeDataStreams() {
+        return true;
+    }
+
     @Override
     public UpdateByQueryRequest forSlice(TaskId slicingTask, SearchRequest slice, int totalSlices) {
         UpdateByQueryRequest request = doForSlice(new UpdateByQueryRequest(slice, false), slicingTask, totalSlices);
diff --git a/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java b/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java
index 42412dccfab5..524fe4a1a1be 100644
--- a/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java
+++ b/server/src/main/java/org/elasticsearch/index/search/MatchQueryParser.java
@@ -538,9 +538,9 @@ private void add(BooleanQuery.Builder q, String field, List<Term> current, Boole
             } else {
                 // We don't apply prefix on synonyms
                 final TermAndBoost[] termAndBoosts = current.stream()
-                    .map(t -> new TermAndBoost(t, BoostAttribute.DEFAULT_BOOST))
+                    .map(t -> new TermAndBoost(t.bytes(), BoostAttribute.DEFAULT_BOOST))
                     .toArray(TermAndBoost[]::new);
-                q.add(newSynonymQuery(termAndBoosts), operator);
+                q.add(newSynonymQuery(field, termAndBoosts), operator);
             }
         }
 
@@ -648,9 +648,9 @@ public Query next() {
                     } else {
                         // We don't apply prefix on synonyms
                         final TermAndBoost[] termAndBoosts = Arrays.stream(terms)
-                            .map(t -> new TermAndBoost(t, BoostAttribute.DEFAULT_BOOST))
+                            .map(t -> new TermAndBoost(t.bytes(), BoostAttribute.DEFAULT_BOOST))
                             .toArray(TermAndBoost[]::new);
-                        queryPos = newSynonymQuery(termAndBoosts);
+                        queryPos = newSynonymQuery(field, termAndBoosts);
                     }
                 }
                 if (queryPos != null) {
diff --git a/server/src/main/java/org/elasticsearch/index/search/MultiMatchQueryParser.java b/server/src/main/java/org/elasticsearch/index/search/MultiMatchQueryParser.java
index 1220aeb9e96a..4c3020bdcd9d 100644
--- a/server/src/main/java/org/elasticsearch/index/search/MultiMatchQueryParser.java
+++ b/server/src/main/java/org/elasticsearch/index/search/MultiMatchQueryParser.java
@@ -196,10 +196,10 @@ protected Query createBooleanPrefixQuery(String field, String queryText, Boolean
         }
 
         @Override
-        protected Query newSynonymQuery(TermAndBoost[] terms) {
+        protected Query newSynonymQuery(String field, TermAndBoost[] terms) {
             BytesRef[] values = new BytesRef[terms.length];
             for (int i = 0; i < terms.length; i++) {
-                values[i] = terms[i].term.bytes();
+                values[i] = terms[i].term;
             }
             return blendTerms(context, values, tieBreaker, lenient, blendedFields);
         }
diff --git a/server/src/main/java/org/elasticsearch/index/search/stats/FieldUsageStats.java b/server/src/main/java/org/elasticsearch/index/search/stats/FieldUsageStats.java
index 0e96fa38b38a..93d047a87c4d 100644
--- a/server/src/main/java/org/elasticsearch/index/search/stats/FieldUsageStats.java
+++ b/server/src/main/java/org/elasticsearch/index/search/stats/FieldUsageStats.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.index.search.stats;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -210,7 +210,7 @@ public PerFieldUsageStats(StreamInput in) throws IOException {
             payloads = in.readVLong();
             termVectors = in.readVLong();
             points = in.readVLong();
-            if (in.getVersion().onOrAfter(Version.V_8_1_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
                 knnVectors = in.readVLong();
             } else {
                 knnVectors = 0;
@@ -232,7 +232,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeVLong(payloads);
             out.writeVLong(termVectors);
             out.writeVLong(points);
-            if (out.getVersion().onOrAfter(Version.V_8_1_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
                 out.writeVLong(knnVectors);
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java
index f7140210cfd1..12ae735d16b5 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java
@@ -52,6 +52,8 @@
 import java.util.stream.LongStream;
 import java.util.stream.Stream;
 
+import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
+
 /**
  * This class is responsible for tracking the replication group with its progress and safety markers (local and global checkpoints).
  *
@@ -65,6 +67,28 @@
  */
 public class ReplicationTracker extends AbstractIndexShardComponent implements LongSupplier {
 
+    public static final ReplicationTracker.Factory DEFAULT_FACTORY = (
+        shardId,
+        allocationId,
+        indexSettings,
+        operationPrimaryTerm,
+        onGlobalCheckpointUpdated,
+        currentTimeMillisSupplier,
+        onSyncRetentionLeases,
+        safeCommitInfoSupplier,
+        onReplicationGroupUpdated) -> new ReplicationTracker(
+            shardId,
+            allocationId,
+            indexSettings,
+            operationPrimaryTerm,
+            UNASSIGNED_SEQ_NO,
+            onGlobalCheckpointUpdated,
+            currentTimeMillisSupplier,
+            onSyncRetentionLeases,
+            safeCommitInfoSupplier,
+            onReplicationGroupUpdated
+        );
+
     /**
      * The allocation ID for the shard to which this tracker is a component of.
      */
@@ -239,6 +263,7 @@ public synchronized RetentionLeases getRetentionLeases(final boolean expireLease
         final long retentionLeaseMillis = indexSettings.getRetentionLeaseMillis();
         final Set<String> leaseIdsForCurrentPeers = routingTable.assignedShards()
             .stream()
+            .filter(ShardRouting::isPromotableToPrimary)
             .map(ReplicationTracker::getPeerRecoveryRetentionLeaseId)
             .collect(Collectors.toSet());
         final boolean allShardsStarted = routingTable.allShardsStarted();
@@ -583,7 +608,7 @@ public synchronized void renewPeerRecoveryRetentionLeases() {
         boolean renewalNeeded = false;
         for (int copy = 0; copy < routingTable.size(); copy++) {
             final ShardRouting shardRouting = routingTable.shard(copy);
-            if (shardRouting.assignedToNode() == false) {
+            if (shardRouting.assignedToNode() == false || shardRouting.isPromotableToPrimary() == false) {
                 continue;
             }
             final RetentionLease retentionLease = retentionLeases.get(getPeerRecoveryRetentionLeaseId(shardRouting));
@@ -604,7 +629,7 @@ public synchronized void renewPeerRecoveryRetentionLeases() {
         if (renewalNeeded) {
             for (int copy = 0; copy < routingTable.size(); copy++) {
                 final ShardRouting shardRouting = routingTable.shard(copy);
-                if (shardRouting.assignedToNode()) {
+                if (shardRouting.assignedToNode() && shardRouting.isPromotableToPrimary()) {
                     final RetentionLease retentionLease = retentionLeases.get(getPeerRecoveryRetentionLeaseId(shardRouting));
                     if (retentionLease != null) {
                         final CheckpointState checkpointState = checkpoints.get(shardRouting.allocationId().getId());
@@ -850,8 +875,15 @@ private boolean invariant() {
         assert replicationGroup == null || replicationGroup.equals(calculateReplicationGroup())
             : "cached replication group out of sync: expected: " + calculateReplicationGroup() + " but was: " + replicationGroup;
 
+        if (replicationGroup != null) {
+            assert replicationGroup.getReplicationTargets().stream().allMatch(ShardRouting::isPromotableToPrimary)
+                : "expected all replication target shards of the replication group to be promotable to primary";
+            assert replicationGroup.getSkippedShards().stream().allMatch(ShardRouting::isPromotableToPrimary)
+                : "expected all skipped shards of the replication group to be promotable to primary";
+        }
+
         // all assigned shards from the routing table are tracked
-        assert routingTable == null || checkpoints.keySet().containsAll(routingTable.getAllAllocationIds())
+        assert routingTable == null || checkpoints.keySet().containsAll(routingTable.getPromotableAllocationIds())
             : "local checkpoints " + checkpoints + " not in-sync with routing table " + routingTable;
 
         for (Map.Entry<String, CheckpointState> entry : checkpoints.entrySet()) {
@@ -871,7 +903,7 @@ private boolean invariant() {
         if (primaryMode && indexSettings.isSoftDeleteEnabled() && hasAllPeerRecoveryRetentionLeases) {
             // all tracked shard copies have a corresponding peer-recovery retention lease
             for (final ShardRouting shardRouting : routingTable.assignedShards()) {
-                if (checkpoints.get(shardRouting.allocationId().getId()).tracked) {
+                if (shardRouting.isPromotableToPrimary() && checkpoints.get(shardRouting.allocationId().getId()).tracked) {
                     assert retentionLeases.contains(getPeerRecoveryRetentionLeaseId(shardRouting))
                         : "no retention lease for tracked shard [" + shardRouting + "] in " + retentionLeases;
                     assert PEER_RECOVERY_RETENTION_LEASE_SOURCE.equals(
@@ -1127,6 +1159,7 @@ private void addPeerRecoveryRetentionLeaseForSolePrimary() {
         } else if (hasAllPeerRecoveryRetentionLeases == false
             && routingTable.assignedShards()
                 .stream()
+                .filter(ShardRouting::isPromotableToPrimary)
                 .allMatch(
                     shardRouting -> retentionLeases.contains(getPeerRecoveryRetentionLeaseId(shardRouting))
                         || checkpoints.get(shardRouting.allocationId().getId()).tracked == false
@@ -1161,6 +1194,7 @@ public synchronized void updateFromMaster(
             // remove entries which don't exist on master
             Set<String> initializingAllocationIds = routingTable.getAllInitializingShards()
                 .stream()
+                .filter(ShardRouting::isPromotableToPrimary)
                 .map(ShardRouting::allocationId)
                 .map(AllocationId::getId)
                 .collect(Collectors.toSet());
@@ -1471,11 +1505,17 @@ public synchronized boolean hasAllPeerRecoveryRetentionLeases() {
      */
     public synchronized void createMissingPeerRecoveryRetentionLeases(ActionListener<Void> listener) {
         if (hasAllPeerRecoveryRetentionLeases == false) {
-            final List<ShardRouting> shardRoutings = routingTable.assignedShards();
-            final GroupedActionListener<ReplicationResponse> groupedActionListener = new GroupedActionListener<>(ActionListener.wrap(vs -> {
-                setHasAllPeerRecoveryRetentionLeases();
-                listener.onResponse(null);
-            }, listener::onFailure), shardRoutings.size());
+            final List<ShardRouting> shardRoutings = routingTable.assignedShards()
+                .stream()
+                .filter(ShardRouting::isPromotableToPrimary)
+                .toList();
+            final GroupedActionListener<ReplicationResponse> groupedActionListener = new GroupedActionListener<>(
+                shardRoutings.size(),
+                ActionListener.wrap(vs -> {
+                    setHasAllPeerRecoveryRetentionLeases();
+                    listener.onResponse(null);
+                }, listener::onFailure)
+            );
             for (ShardRouting shardRouting : shardRoutings) {
                 if (retentionLeases.contains(getPeerRecoveryRetentionLeaseId(shardRouting))) {
                     groupedActionListener.onResponse(null);
@@ -1624,4 +1664,22 @@ public int hashCode() {
             return result;
         }
     }
+
+    /**
+     * Factory interface used by {@link org.elasticsearch.index.IndexModule#setReplicationTrackerFactory(Factory)} to enable custom
+     * overrides of this class.
+     */
+    public interface Factory {
+        ReplicationTracker create(
+            ShardId shardId,
+            String allocationId,
+            IndexSettings indexSettings,
+            long operationPrimaryTerm,
+            LongConsumer onGlobalCheckpointUpdated,
+            LongSupplier currentTimeMillisSupplier,
+            BiConsumer<RetentionLeases, ActionListener<ReplicationResponse>> onSyncRetentionLeases,
+            Supplier<SafeCommitInfo> safeCommitInfoSupplier,
+            Consumer<ReplicationGroup> onReplicationGroupUpdated
+        );
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java
index c2c33bafc8df..8ac992221aba 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLease.java
@@ -99,7 +99,12 @@ public RetentionLease(final String id, final long retainingSequenceNumber, final
         this.id = id;
         this.retainingSequenceNumber = retainingSequenceNumber;
         this.timestamp = timestamp;
-        this.source = source;
+        // deduplicate the string instances to save memory for the known possible source values
+        this.source = switch (source) {
+            case "ccr" -> "ccr";
+            case ReplicationTracker.PEER_RECOVERY_RETENTION_LEASE_SOURCE -> ReplicationTracker.PEER_RECOVERY_RETENTION_LEASE_SOURCE;
+            default -> source;
+        };
     }
 
     /**
@@ -109,10 +114,7 @@ public RetentionLease(final String id, final long retainingSequenceNumber, final
      * @throws IOException if an I/O exception occurs reading from the stream
      */
     public RetentionLease(final StreamInput in) throws IOException {
-        id = in.readString();
-        retainingSequenceNumber = in.readZLong();
-        timestamp = in.readVLong();
-        source = in.readString();
+        this(in.readString(), in.readZLong(), in.readVLong(), in.readString());
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java
index 86cfa84b90b8..10d0508989fb 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java
@@ -163,7 +163,7 @@ protected void dispatchedShardOperationOnPrimary(
             Objects.requireNonNull(request);
             Objects.requireNonNull(primary);
             primary.persistRetentionLeases();
-            return new WritePrimaryResult<>(request, new Response(), null, null, primary, getLogger());
+            return new WritePrimaryResult<>(request, new Response(), null, primary, getLogger());
         });
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexEventListener.java b/server/src/main/java/org/elasticsearch/index/shard/IndexEventListener.java
index c9231f4022c3..beb79c55b12d 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexEventListener.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexEventListener.java
@@ -7,6 +7,7 @@
  */
 package org.elasticsearch.index.shard;
 
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Nullable;
@@ -166,8 +167,11 @@ default void onStoreClosed(ShardId shardId) {}
      *
      * @param indexShard    the shard that is about to recover
      * @param indexSettings the shard's index settings
+     * @param listener      listener notified when this step completes
      */
-    default void beforeIndexShardRecovery(IndexShard indexShard, IndexSettings indexSettings) {}
+    default void beforeIndexShardRecovery(IndexShard indexShard, IndexSettings indexSettings, ActionListener<Void> listener) {
+        listener.onResponse(null);
+    }
 
     /**
      * Called after the raw files have been restored from the repository but any other recovery processing has happened
diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 947349ecb97e..871928a96e4b 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -55,7 +55,6 @@
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.AsyncIOProcessor;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.CheckedConsumer;
@@ -222,6 +221,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     private final PendingReplicationActions pendingReplicationActions;
     private final ReplicationTracker replicationTracker;
     private final IndexStorePlugin.SnapshotCommitSupplier snapshotCommitSupplier;
+    private final Engine.IndexCommitListener indexCommitListener;
 
     protected volatile ShardRouting shardRouting;
     protected volatile IndexShardState state;
@@ -290,6 +290,9 @@ Runnable getGlobalCheckpointSyncer() {
     private volatile long startedRelativeTimeInNanos;
     private volatile long indexingTimeBeforeShardStartedInNanos;
 
+    // the translog keeps track of the GCP, but unpromotable shards have no translog so we need to track the GCP here instead
+    private volatile long globalCheckPointIfUnpromotable;
+
     public IndexShard(
         final ShardRouting shardRouting,
         final IndexSettings indexSettings,
@@ -311,7 +314,9 @@ public IndexShard(
         final RetentionLeaseSyncer retentionLeaseSyncer,
         final CircuitBreakerService circuitBreakerService,
         final IndexStorePlugin.SnapshotCommitSupplier snapshotCommitSupplier,
-        final LongSupplier relativeTimeInNanosSupplier
+        final LongSupplier relativeTimeInNanosSupplier,
+        final Engine.IndexCommitListener indexCommitListener,
+        final ReplicationTracker.Factory replicationTrackerFactory
     ) throws IOException {
         super(shardRouting.shardId(), indexSettings);
         assert shardRouting.initializing();
@@ -359,12 +364,11 @@ public IndexShard(
         this.pendingPrimaryTerm = primaryTerm;
         this.globalCheckpointListeners = new GlobalCheckpointListeners(shardId, threadPool.scheduler(), logger);
         this.pendingReplicationActions = new PendingReplicationActions(shardId, threadPool);
-        this.replicationTracker = new ReplicationTracker(
+        this.replicationTracker = replicationTrackerFactory.create(
             shardId,
             aId,
             indexSettings,
             primaryTerm,
-            UNASSIGNED_SEQ_NO,
             globalCheckpointListeners::globalCheckpointUpdated,
             threadPool::absoluteTimeInMillis,
             (retentionLeases, listener) -> retentionLeaseSyncer.sync(shardId, aId, getPendingPrimaryTerm(), retentionLeases, listener),
@@ -390,6 +394,7 @@ public IndexShard(
         this.refreshPendingLocationListener = new RefreshPendingLocationListener();
         this.isDataStreamIndex = mapperService == null ? false : mapperService.mappingLookup().isDataStreamTimestampFieldEnabled();
         this.relativeTimeInNanosSupplier = relativeTimeInNanosSupplier;
+        this.indexCommitListener = indexCommitListener;
     }
 
     public ThreadPool getThreadPool() {
@@ -703,17 +708,26 @@ public void onFailure(Exception e) {
 
         if (indexSettings.isSoftDeleteEnabled() && useRetentionLeasesInPeerRecovery == false) {
             final RetentionLeases retentionLeases = replicationTracker.getRetentionLeases();
-            final Set<ShardRouting> shardRoutings = Sets.newHashSetWithExpectedSize(routingTable.size());
+            boolean allShardsUseRetentionLeases = true;
             for (int copy = 0; copy < routingTable.size(); copy++) {
-                shardRoutings.add(routingTable.shard(copy));
-            }
-            shardRoutings.addAll(routingTable.assignedShards()); // include relocation targets
-            if (shardRoutings.stream()
-                .allMatch(
-                    shr -> shr.assignedToNode() && retentionLeases.contains(ReplicationTracker.getPeerRecoveryRetentionLeaseId(shr))
-                )) {
-                useRetentionLeasesInPeerRecovery = true;
+                ShardRouting shardRouting = routingTable.shard(copy);
+                if (shardRouting.isPromotableToPrimary()) {
+                    if (shardRouting.assignedToNode() == false
+                        || retentionLeases.contains(ReplicationTracker.getPeerRecoveryRetentionLeaseId(shardRouting)) == false) {
+                        allShardsUseRetentionLeases = false;
+                        break;
+                    }
+                    if (this.shardRouting.relocating()) {
+                        ShardRouting shardRoutingReloc = this.shardRouting.getTargetRelocatingShard();
+                        if (shardRoutingReloc.assignedToNode() == false
+                            || retentionLeases.contains(ReplicationTracker.getPeerRecoveryRetentionLeaseId(shardRoutingReloc)) == false) {
+                            allShardsUseRetentionLeases = false;
+                            break;
+                        }
+                    }
+                }
             }
+            useRetentionLeasesInPeerRecovery = allShardsUseRetentionLeases;
         }
     }
 
@@ -1214,12 +1228,12 @@ public Engine.GetResult get(Engine.Get get) {
     /**
      * Writes all indexing changes to disk and opens a new searcher reflecting all changes.  This can throw {@link AlreadyClosedException}.
      */
-    public void refresh(String source) {
+    public Engine.RefreshResult refresh(String source) {
         verifyNotClosed();
         if (logger.isTraceEnabled()) {
             logger.trace("refresh with source [{}]", source);
         }
-        getEngine().refresh(source);
+        return getEngine().refresh(source);
     }
 
     /**
@@ -1584,6 +1598,11 @@ static Engine.Searcher wrapSearcher(
         ); // completes stats recording
     }
 
+    public void setGlobalCheckpointIfUnpromotable(long globalCheckpoint) {
+        assert shardRouting.isPromotableToPrimary() == false : "must only call this on unpromotable shards";
+        globalCheckPointIfUnpromotable = globalCheckpoint;
+    }
+
     private static final class NonClosingReaderWrapper extends FilterDirectoryReader {
 
         private NonClosingReaderWrapper(DirectoryReader in) throws IOException {
@@ -1634,13 +1653,13 @@ public void close(String reason, boolean flushEngine) throws IOException {
         }
     }
 
-    public void preRecovery() {
+    public void preRecovery(ActionListener<Void> listener) {
         final IndexShardState currentState = this.state; // single volatile read
         if (currentState == IndexShardState.CLOSED) {
             throw new IndexShardNotRecoveringException(shardId, currentState);
         }
         assert currentState == IndexShardState.RECOVERING : "expected a recovering shard " + shardId + " but got " + currentState;
-        indexEventListener.beforeIndexShardRecovery(this, indexSettings);
+        indexEventListener.beforeIndexShardRecovery(this, indexSettings, listener);
     }
 
     public void postRecovery(String reason) throws IndexShardStartedException, IndexShardRelocatedException, IndexShardClosedException {
@@ -1882,12 +1901,16 @@ int runTranslogRecovery(Engine engine, Translog.Snapshot snapshot, Engine.Operat
     }
 
     private void loadGlobalCheckpointToReplicationTracker() throws IOException {
-        // we have to set it before we open an engine and recover from the translog because
-        // acquiring a snapshot from the translog causes a sync which causes the global checkpoint to be pulled in,
-        // and an engine can be forced to close in ctor which also causes the global checkpoint to be pulled in.
-        final String translogUUID = store.readLastCommittedSegmentsInfo().getUserData().get(Translog.TRANSLOG_UUID_KEY);
-        final long globalCheckpoint = Translog.readGlobalCheckpoint(translogConfig.getTranslogPath(), translogUUID);
-        replicationTracker.updateGlobalCheckpointOnReplica(globalCheckpoint, "read from translog checkpoint");
+        if (shardRouting.isPromotableToPrimary()) {
+            // we have to set it before we open an engine and recover from the translog because
+            // acquiring a snapshot from the translog causes a sync which causes the global checkpoint to be pulled in,
+            // and an engine can be forced to close in ctor which also causes the global checkpoint to be pulled in.
+            final String translogUUID = store.readLastCommittedSegmentsInfo().getUserData().get(Translog.TRANSLOG_UUID_KEY);
+            final long globalCheckpoint = Translog.readGlobalCheckpoint(translogConfig.getTranslogPath(), translogUUID);
+            replicationTracker.updateGlobalCheckpointOnReplica(globalCheckpoint, "read from translog checkpoint");
+        } else {
+            replicationTracker.updateGlobalCheckpointOnReplica(globalCheckPointIfUnpromotable, "from CleanFilesRequest");
+        }
     }
 
     /**
@@ -2052,6 +2075,7 @@ public ShardLongFieldRange getTimestampRange() {
         final ShardLongFieldRange rawTimestampFieldRange;
         try {
             rawTimestampFieldRange = getEngine().getRawFieldRange(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD);
+            assert rawTimestampFieldRange != null;
         } catch (IOException | AlreadyClosedException e) {
             logger.debug("exception obtaining range for timestamp field", e);
             return ShardLongFieldRange.UNKNOWN;
@@ -2997,7 +3021,7 @@ Engine getEngine() {
      * NOTE: returns null if engine is not yet started (e.g. recovery phase 1, copying over index files, is still running), or if engine is
      * closed.
      */
-    protected Engine getEngineOrNull() {
+    public Engine getEngineOrNull() {
         return this.currentEngineReference.get();
     }
 
@@ -3270,7 +3294,9 @@ private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) {
             this::getOperationPrimaryTerm,
             snapshotCommitSupplier,
             isTimeseriesIndex ? TIMESERIES_LEAF_READERS_SORTER : null,
-            relativeTimeInNanosSupplier
+            relativeTimeInNanosSupplier,
+            indexCommitListener,
+            routingEntry().isPromotableToPrimary()
         );
     }
 
@@ -3769,7 +3795,7 @@ EngineFactory getEngineFactory() {
     }
 
     // for tests
-    ReplicationTracker getReplicationTracker() {
+    public ReplicationTracker getReplicationTracker() {
         return replicationTracker;
     }
 
@@ -3797,7 +3823,7 @@ && isSearchIdle()
                 if (logger.isTraceEnabled()) {
                     logger.trace("refresh with source [schedule]");
                 }
-                return getEngine().maybeRefresh("schedule");
+                return getEngine().maybeRefresh("schedule").refreshed();
             }
         }
         final Engine engine = getEngine();
@@ -4114,4 +4140,8 @@ public long getRelativeTimeInNanos() {
     public String toString() {
         return "IndexShard(shardRouting=" + shardRouting + ")";
     }
+
+    public void waitForSegmentGeneration(long segmentGeneration, ActionListener<Long> listener) {
+        getEngine().addSegmentGenerationListener(segmentGeneration, listener);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java
index 3390988b21f6..5853b5d89d75 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShardOperationPermits.java
@@ -8,7 +8,10 @@
 
 package org.elasticsearch.index.shard;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Assertions;
+import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
@@ -18,6 +21,7 @@
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -29,7 +33,6 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Supplier;
 
@@ -41,6 +44,8 @@
  */
 final class IndexShardOperationPermits implements Closeable {
 
+    private static final Logger logger = LogManager.getLogger(IndexShardOperationPermits.class);
+
     private final ShardId shardId;
     private final ThreadPool threadPool;
 
@@ -81,13 +86,17 @@ public void close() {
      * started. Delayed operations are run once the {@link Releasable} is released or if a failure occurs while acquiring all permits; in
      * this case the {@code onFailure} handler will be invoked after delayed operations are released.
      *
-     * @param onAcquired {@link ActionListener} that is invoked once acquisition is successful or failed
+     * @param onAcquired {@link ActionListener} that is invoked once acquisition is successful or failed. This listener should not throw.
      * @param timeout    the maximum time to wait for the in-flight operations block
      * @param timeUnit   the time unit of the {@code timeout} argument
      * @param executor   executor on which to wait for in-flight operations to finish and acquire all permits
      */
     public void blockOperations(final ActionListener<Releasable> onAcquired, final long timeout, final TimeUnit timeUnit, String executor) {
         delayOperations();
+        waitUntilBlocked(ActionListener.assertOnce(onAcquired), timeout, timeUnit, executor);
+    }
+
+    private void waitUntilBlocked(ActionListener<Releasable> onAcquired, long timeout, TimeUnit timeUnit, String executor) {
         threadPool.executor(executor).execute(new AbstractRunnable() {
 
             final Releasable released = Releasables.releaseOnce(() -> releaseDelayedOperations());
@@ -102,9 +111,23 @@ public void onFailure(final Exception e) {
             }
 
             @Override
-            protected void doRun() throws Exception {
-                final Releasable releasable = acquireAll(timeout, timeUnit);
-                onAcquired.onResponse(() -> Releasables.close(releasable, released));
+            protected void doRun() {
+                final Releasable releasable;
+                try {
+                    releasable = acquireAll(timeout, timeUnit);
+                } catch (Exception e) {
+                    onFailure(e);
+                    return;
+                }
+
+                final Releasable combined = Releasables.wrap(releasable, released);
+                try {
+                    onAcquired.onResponse(combined);
+                } catch (Exception e) {
+                    logger.error("onAcquired#onResponse should not throw", e);
+                    assert false : e; // should not throw, we cannot do anything with this exception
+                    combined.close();
+                }
             }
         });
     }
@@ -119,7 +142,7 @@ private void delayOperations() {
         }
     }
 
-    private Releasable acquireAll(final long timeout, final TimeUnit timeUnit) throws InterruptedException, TimeoutException {
+    private Releasable acquireAll(final long timeout, final TimeUnit timeUnit) throws InterruptedException {
         if (Assertions.ENABLED) {
             // since delayed is not volatile, we have to synchronize even here for visibility
             synchronized (this) {
@@ -133,7 +156,7 @@ private Releasable acquireAll(final long timeout, final TimeUnit timeUnit) throw
             });
             return release;
         } else {
-            throw new TimeoutException("timeout while blocking operations");
+            throw new ElasticsearchTimeoutException("timeout while blocking operations after [" + new TimeValue(timeout, timeUnit) + "]");
         }
     }
 
@@ -195,7 +218,7 @@ public void acquire(
         } else {
             stackTrace = null;
         }
-        acquire(onAcquired, executorOnDelay, forceExecution, debugInfo, stackTrace);
+        acquire(ActionListener.assertOnce(onAcquired), executorOnDelay, forceExecution, debugInfo, stackTrace);
     }
 
     private void acquire(
diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java b/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java
index b998b214cef4..f416178a6b10 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexingStats.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.index.shard;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -27,7 +27,7 @@
 public class IndexingStats implements Writeable, ToXContentFragment {
 
     public static class Stats implements Writeable, ToXContentFragment {
-        private static final Version WRITE_LOAD_AVG_SUPPORTED_VERSION = Version.V_8_6_0;
+        private static final TransportVersion WRITE_LOAD_AVG_SUPPORTED_VERSION = TransportVersion.V_8_6_0;
 
         private long indexCount;
         private long indexTimeInMillis;
@@ -55,7 +55,7 @@ public Stats(StreamInput in) throws IOException {
             noopUpdateCount = in.readVLong();
             isThrottled = in.readBoolean();
             throttleTimeInMillis = in.readLong();
-            if (in.getVersion().onOrAfter(WRITE_LOAD_AVG_SUPPORTED_VERSION)) {
+            if (in.getTransportVersion().onOrAfter(WRITE_LOAD_AVG_SUPPORTED_VERSION)) {
                 totalIndexingTimeSinceShardStartedInNanos = in.readLong();
                 totalActiveTimeInNanos = in.readLong();
             }
@@ -196,7 +196,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeVLong(noopUpdateCount);
             out.writeBoolean(isThrottled);
             out.writeLong(throttleTimeInMillis);
-            if (out.getVersion().onOrAfter(WRITE_LOAD_AVG_SUPPORTED_VERSION)) {
+            if (out.getTransportVersion().onOrAfter(WRITE_LOAD_AVG_SUPPORTED_VERSION)) {
                 out.writeLong(totalIndexingTimeSinceShardStartedInNanos);
                 out.writeLong(totalActiveTimeInNanos);
             }
@@ -268,7 +268,7 @@ public IndexingStats() {
 
     public IndexingStats(StreamInput in) throws IOException {
         totalStats = new Stats(in);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             if (in.readBoolean()) {
                 Map<String, Stats> typeStats = in.readMap(StreamInput::readString, Stats::new);
                 assert typeStats.size() == 1;
@@ -349,7 +349,7 @@ static final class Fields {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         totalStats.writeTo(out);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeBoolean(false);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/index/shard/ReplicationGroup.java b/server/src/main/java/org/elasticsearch/index/shard/ReplicationGroup.java
index cf3b8fc0fbaf..53f932faf451 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/ReplicationGroup.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/ReplicationGroup.java
@@ -40,11 +40,14 @@ public ReplicationGroup(
         this.trackedAllocationIds = trackedAllocationIds;
         this.version = version;
 
-        this.unavailableInSyncShards = Sets.difference(inSyncAllocationIds, routingTable.getAllAllocationIds());
+        this.unavailableInSyncShards = Sets.difference(inSyncAllocationIds, routingTable.getPromotableAllocationIds());
         this.replicationTargets = new ArrayList<>();
         this.skippedShards = new ArrayList<>();
         for (int copy = 0; copy < routingTable.size(); copy++) {
             ShardRouting shard = routingTable.shard(copy);
+            if (shard.isPromotableToPrimary() == false) {
+                continue;
+            }
             if (shard.unassigned()) {
                 assert shard.primary() == false : "primary shard should not be unassigned in a replication group: " + shard;
                 skippedShards.add(shard);
diff --git a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
index 7f256e328b76..c3c875fc4c32 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.StepListener;
 import org.elasticsearch.action.support.ThreadedActionListener;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -85,11 +86,13 @@ void recoverFromStore(final IndexShard indexShard, ActionListener<Boolean> liste
             RecoverySource.Type recoveryType = indexShard.recoveryState().getRecoverySource().getType();
             assert recoveryType == RecoverySource.Type.EMPTY_STORE || recoveryType == RecoverySource.Type.EXISTING_STORE
                 : "expected store recovery type but was: " + recoveryType;
-            ActionListener.completeWith(recoveryListener(indexShard, listener), () -> {
-                logger.debug("starting recovery from store ...");
-                internalRecoverFromStore(indexShard);
-                return true;
-            });
+            logger.debug("starting recovery from store ...");
+            final var recoveryListener = recoveryListener(indexShard, listener);
+            try {
+                internalRecoverFromStore(indexShard, recoveryListener.map(ignored -> true));
+            } catch (Exception e) {
+                recoveryListener.onFailure(e);
+            }
         } else {
             listener.onResponse(false);
         }
@@ -120,37 +123,42 @@ void recoverFromLocalShards(
             Sort indexSort = indexShard.getIndexSort();
             final boolean hasNested = indexShard.mapperService().hasNested();
             final boolean isSplit = sourceMetadata.getNumberOfShards() < indexShard.indexSettings().getNumberOfShards();
-            ActionListener.completeWith(recoveryListener(indexShard, listener), () -> {
-                logger.debug("starting recovery from local shards {}", shards);
-                try {
-                    final Directory directory = indexShard.store().directory(); // don't close this directory!!
-                    final Directory[] sources = shards.stream().map(LocalShardSnapshot::getSnapshotDirectory).toArray(Directory[]::new);
-                    final long maxSeqNo = shards.stream().mapToLong(LocalShardSnapshot::maxSeqNo).max().getAsLong();
-                    final long maxUnsafeAutoIdTimestamp = shards.stream()
-                        .mapToLong(LocalShardSnapshot::maxUnsafeAutoIdTimestamp)
-                        .max()
-                        .getAsLong();
-                    addIndices(
-                        indexShard.recoveryState().getIndex(),
-                        directory,
-                        indexSort,
-                        sources,
-                        maxSeqNo,
-                        maxUnsafeAutoIdTimestamp,
-                        indexShard.indexSettings().getIndexMetadata(),
-                        indexShard.shardId().id(),
-                        isSplit,
-                        hasNested
-                    );
-                    internalRecoverFromStore(indexShard);
-                    // just trigger a merge to do housekeeping on the
-                    // copied segments - we will also see them in stats etc.
-                    indexShard.getEngine().forceMerge(false, -1, false, UUIDs.randomBase64UUID());
-                    return true;
-                } catch (IOException ex) {
-                    throw new IndexShardRecoveryException(indexShard.shardId(), "failed to recover from local shards", ex);
-                }
-            });
+
+            final var recoveryListener = recoveryListener(indexShard, listener);
+            logger.debug("starting recovery from local shards {}", shards);
+            try {
+                final Directory directory = indexShard.store().directory(); // don't close this directory!!
+                final Directory[] sources = shards.stream().map(LocalShardSnapshot::getSnapshotDirectory).toArray(Directory[]::new);
+                final long maxSeqNo = shards.stream().mapToLong(LocalShardSnapshot::maxSeqNo).max().getAsLong();
+                final long maxUnsafeAutoIdTimestamp = shards.stream()
+                    .mapToLong(LocalShardSnapshot::maxUnsafeAutoIdTimestamp)
+                    .max()
+                    .getAsLong();
+                addIndices(
+                    indexShard.recoveryState().getIndex(),
+                    directory,
+                    indexSort,
+                    sources,
+                    maxSeqNo,
+                    maxUnsafeAutoIdTimestamp,
+                    indexShard.indexSettings().getIndexMetadata(),
+                    indexShard.shardId().id(),
+                    isSplit,
+                    hasNested
+                );
+                internalRecoverFromStore(indexShard, recoveryListener.delegateFailure((delegate, v) -> {
+                    ActionListener.completeWith(delegate, () -> {
+                        // just trigger a merge to do housekeeping on the
+                        // copied segments - we will also see them in stats etc.
+                        indexShard.getEngine().forceMerge(false, -1, false, UUIDs.randomBase64UUID());
+                        return true;
+                    });
+                }));
+            } catch (IOException e) {
+                recoveryListener.onFailure(new IndexShardRecoveryException(indexShard.shardId(), "failed to recover from local shards", e));
+            } catch (Exception e) {
+                recoveryListener.onFailure(e);
+            }
         } else {
             listener.onResponse(false);
         }
@@ -391,84 +399,85 @@ private ActionListener<Boolean> recoveryListener(IndexShard indexShard, ActionLi
     /**
      * Recovers the state of the shard from the store.
      */
-    private void internalRecoverFromStore(IndexShard indexShard) throws IndexShardRecoveryException {
-        indexShard.preRecovery();
-        final RecoveryState recoveryState = indexShard.recoveryState();
-        final boolean indexShouldExists = recoveryState.getRecoverySource().getType() != RecoverySource.Type.EMPTY_STORE;
-        indexShard.prepareForIndexRecovery();
-        SegmentInfos si = null;
-        final Store store = indexShard.store();
-        store.incRef();
-        try {
+    private void internalRecoverFromStore(IndexShard indexShard, ActionListener<Void> outerListener) {
+        indexShard.preRecovery(outerListener.delegateFailure((listener, ignored) -> ActionRunnable.run(listener, () -> {
+            final RecoveryState recoveryState = indexShard.recoveryState();
+            final boolean indexShouldExists = recoveryState.getRecoverySource().getType() != RecoverySource.Type.EMPTY_STORE;
+            indexShard.prepareForIndexRecovery();
+            SegmentInfos si = null;
+            final Store store = indexShard.store();
+            store.incRef();
             try {
-                store.failIfCorrupted();
                 try {
-                    si = store.readLastCommittedSegmentsInfo();
-                } catch (Exception e) {
-                    String files = "_unknown_";
+                    store.failIfCorrupted();
                     try {
-                        files = Arrays.toString(store.directory().listAll());
-                    } catch (Exception inner) {
-                        files += " (failure=" + ExceptionsHelper.stackTrace(inner) + ")";
+                        si = store.readLastCommittedSegmentsInfo();
+                    } catch (Exception e) {
+                        String files = "_unknown_";
+                        try {
+                            files = Arrays.toString(store.directory().listAll());
+                        } catch (Exception inner) {
+                            files += " (failure=" + ExceptionsHelper.stackTrace(inner) + ")";
+                        }
+                        if (indexShouldExists) {
+                            throw new IndexShardRecoveryException(
+                                shardId,
+                                "shard allocated for local recovery (post api), should exist, but doesn't, current files: " + files,
+                                e
+                            );
+                        }
                     }
-                    if (indexShouldExists) {
-                        throw new IndexShardRecoveryException(
-                            shardId,
-                            "shard allocated for local recovery (post api), should exist, but doesn't, current files: " + files,
-                            e
-                        );
+                    if (si != null && indexShouldExists == false) {
+                        // it exists on the directory, but shouldn't exist on the FS, its a leftover (possibly dangling)
+                        // its a "new index create" API, we have to do something, so better to clean it than use same data
+                        logger.trace("cleaning existing shard, shouldn't exists");
+                        Lucene.cleanLuceneIndex(store.directory());
+                        si = null;
                     }
+                } catch (Exception e) {
+                    throw new IndexShardRecoveryException(shardId, "failed to fetch index version after copying it over", e);
                 }
-                if (si != null && indexShouldExists == false) {
-                    // it exists on the directory, but shouldn't exist on the FS, its a leftover (possibly dangling)
-                    // its a "new index create" API, we have to do something, so better to clean it than use same data
-                    logger.trace("cleaning existing shard, shouldn't exists");
-                    Lucene.cleanLuceneIndex(store.directory());
-                    si = null;
-                }
-            } catch (Exception e) {
-                throw new IndexShardRecoveryException(shardId, "failed to fetch index version after copying it over", e);
-            }
-            if (recoveryState.getRecoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS) {
-                assert indexShouldExists;
-                bootstrap(indexShard, store);
-                writeEmptyRetentionLeasesFile(indexShard);
-            } else if (indexShouldExists) {
-                if (recoveryState.getRecoverySource().shouldBootstrapNewHistoryUUID()) {
-                    store.bootstrapNewHistory();
+                if (recoveryState.getRecoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS) {
+                    assert indexShouldExists;
+                    bootstrap(indexShard, store);
                     writeEmptyRetentionLeasesFile(indexShard);
-                }
-                // since we recover from local, just fill the files and size
-                final RecoveryState.Index index = recoveryState.getIndex();
-                try {
-                    if (si != null) {
-                        addRecoveredFileDetails(si, store, index);
+                } else if (indexShouldExists) {
+                    if (recoveryState.getRecoverySource().shouldBootstrapNewHistoryUUID()) {
+                        store.bootstrapNewHistory();
+                        writeEmptyRetentionLeasesFile(indexShard);
                     }
-                } catch (IOException e) {
-                    logger.debug("failed to list file details", e);
+                    // since we recover from local, just fill the files and size
+                    final RecoveryState.Index index = recoveryState.getIndex();
+                    try {
+                        if (si != null) {
+                            addRecoveredFileDetails(si, store, index);
+                        }
+                    } catch (IOException e) {
+                        logger.debug("failed to list file details", e);
+                    }
+                    index.setFileDetailsComplete();
+                } else {
+                    store.createEmpty();
+                    final String translogUUID = Translog.createEmptyTranslog(
+                        indexShard.shardPath().resolveTranslog(),
+                        SequenceNumbers.NO_OPS_PERFORMED,
+                        shardId,
+                        indexShard.getPendingPrimaryTerm()
+                    );
+                    store.associateIndexWithNewTranslog(translogUUID);
+                    writeEmptyRetentionLeasesFile(indexShard);
+                    indexShard.recoveryState().getIndex().setFileDetailsComplete();
                 }
-                index.setFileDetailsComplete();
-            } else {
-                store.createEmpty();
-                final String translogUUID = Translog.createEmptyTranslog(
-                    indexShard.shardPath().resolveTranslog(),
-                    SequenceNumbers.NO_OPS_PERFORMED,
-                    shardId,
-                    indexShard.getPendingPrimaryTerm()
-                );
-                store.associateIndexWithNewTranslog(translogUUID);
-                writeEmptyRetentionLeasesFile(indexShard);
-                indexShard.recoveryState().getIndex().setFileDetailsComplete();
+                indexShard.openEngineAndRecoverFromTranslog();
+                indexShard.getEngine().fillSeqNoGaps(indexShard.getPendingPrimaryTerm());
+                indexShard.finalizeRecovery();
+                indexShard.postRecovery("post recovery from shard_store");
+            } catch (EngineException | IOException e) {
+                throw new IndexShardRecoveryException(shardId, "failed to recover from gateway", e);
+            } finally {
+                store.decRef();
             }
-            indexShard.openEngineAndRecoverFromTranslog();
-            indexShard.getEngine().fillSeqNoGaps(indexShard.getPendingPrimaryTerm());
-            indexShard.finalizeRecovery();
-            indexShard.postRecovery("post recovery from shard_store");
-        } catch (EngineException | IOException e) {
-            throw new IndexShardRecoveryException(shardId, "failed to recover from gateway", e);
-        } finally {
-            store.decRef();
-        }
+        }).run()));
     }
 
     private static void writeEmptyRetentionLeasesFile(IndexShard indexShard) throws IOException {
@@ -492,72 +501,71 @@ private void restore(
         IndexShard indexShard,
         Repository repository,
         SnapshotRecoverySource restoreSource,
-        ActionListener<Boolean> listener
+        ActionListener<Boolean> outerListener
     ) {
         logger.debug("restoring from {} ...", indexShard.recoveryState().getRecoverySource());
-        indexShard.preRecovery();
-        final RecoveryState.Translog translogState = indexShard.recoveryState().getTranslog();
-        if (restoreSource == null) {
-            listener.onFailure(new IndexShardRestoreFailedException(shardId, "empty restore source"));
-            return;
-        }
-        if (logger.isTraceEnabled()) {
-            logger.trace("[{}] restoring shard [{}]", restoreSource.snapshot(), shardId);
-        }
-        final ActionListener<Void> restoreListener = ActionListener.wrap(v -> {
-            indexShard.getIndexEventListener().afterFilesRestoredFromRepository(indexShard);
-            final Store store = indexShard.store();
-            bootstrap(indexShard, store);
-            assert indexShard.shardRouting.primary() : "only primary shards can recover from store";
-            writeEmptyRetentionLeasesFile(indexShard);
-            indexShard.openEngineAndRecoverFromTranslog();
-            indexShard.getEngine().fillSeqNoGaps(indexShard.getPendingPrimaryTerm());
-            indexShard.finalizeRecovery();
-            indexShard.postRecovery("restore done");
-            listener.onResponse(true);
-        }, e -> listener.onFailure(new IndexShardRestoreFailedException(shardId, "restore failed", e)));
-        try {
-            translogState.totalOperations(0);
-            translogState.totalOperationsOnStart(0);
-            indexShard.prepareForIndexRecovery();
-            final ShardId snapshotShardId;
-            final IndexId indexId = restoreSource.index();
-            if (shardId.getIndexName().equals(indexId.getName())) {
-                snapshotShardId = shardId;
-            } else {
-                snapshotShardId = new ShardId(indexId.getName(), IndexMetadata.INDEX_UUID_NA_VALUE, shardId.id());
+        indexShard.preRecovery(outerListener.delegateFailure((listener, ignored) -> {
+
+            final RecoveryState.Translog translogState = indexShard.recoveryState().getTranslog();
+            if (restoreSource == null) {
+                listener.onFailure(new IndexShardRestoreFailedException(shardId, "empty restore source"));
+                return;
             }
-            final StepListener<IndexId> indexIdListener = new StepListener<>();
-            // If the index UUID was not found in the recovery source we will have to load RepositoryData and resolve it by index name
-            if (indexId.getId().equals(IndexMetadata.INDEX_UUID_NA_VALUE)) {
-                // BwC path, running against an old version master that did not add the IndexId to the recovery source
-                repository.getRepositoryData(
-                    new ThreadedActionListener<>(
-                        logger,
-                        indexShard.getThreadPool(),
-                        ThreadPool.Names.GENERIC,
-                        indexIdListener.map(repositoryData -> repositoryData.resolveIndexId(indexId.getName())),
-                        false
-                    )
-                );
-            } else {
-                indexIdListener.onResponse(indexId);
+            if (logger.isTraceEnabled()) {
+                logger.trace("[{}] restoring shard [{}]", restoreSource.snapshot(), shardId);
             }
-            assert indexShard.getEngineOrNull() == null;
-            indexIdListener.whenComplete(idx -> {
-                assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.GENERIC, ThreadPool.Names.SNAPSHOT);
-                repository.restoreShard(
-                    indexShard.store(),
-                    restoreSource.snapshot().getSnapshotId(),
-                    idx,
-                    snapshotShardId,
-                    indexShard.recoveryState(),
-                    restoreListener
-                );
-            }, restoreListener::onFailure);
-        } catch (Exception e) {
-            restoreListener.onFailure(e);
-        }
+            final ActionListener<Void> restoreListener = ActionListener.wrap(v -> {
+                indexShard.getIndexEventListener().afterFilesRestoredFromRepository(indexShard);
+                final Store store = indexShard.store();
+                bootstrap(indexShard, store);
+                assert indexShard.shardRouting.primary() : "only primary shards can recover from store";
+                writeEmptyRetentionLeasesFile(indexShard);
+                indexShard.openEngineAndRecoverFromTranslog();
+                indexShard.getEngine().fillSeqNoGaps(indexShard.getPendingPrimaryTerm());
+                indexShard.finalizeRecovery();
+                indexShard.postRecovery("restore done");
+                listener.onResponse(true);
+            }, e -> listener.onFailure(new IndexShardRestoreFailedException(shardId, "restore failed", e)));
+            try {
+                translogState.totalOperations(0);
+                translogState.totalOperationsOnStart(0);
+                indexShard.prepareForIndexRecovery();
+                final ShardId snapshotShardId;
+                final IndexId indexId = restoreSource.index();
+                if (shardId.getIndexName().equals(indexId.getName())) {
+                    snapshotShardId = shardId;
+                } else {
+                    snapshotShardId = new ShardId(indexId.getName(), IndexMetadata.INDEX_UUID_NA_VALUE, shardId.id());
+                }
+                final StepListener<IndexId> indexIdListener = new StepListener<>();
+                // If the index UUID was not found in the recovery source we will have to load RepositoryData and resolve it by index name
+                if (indexId.getId().equals(IndexMetadata.INDEX_UUID_NA_VALUE)) {
+                    // BwC path, running against an old version master that did not add the IndexId to the recovery source
+                    repository.getRepositoryData(
+                        new ThreadedActionListener<>(
+                            indexShard.getThreadPool().generic(),
+                            indexIdListener.map(repositoryData -> repositoryData.resolveIndexId(indexId.getName()))
+                        )
+                    );
+                } else {
+                    indexIdListener.onResponse(indexId);
+                }
+                assert indexShard.getEngineOrNull() == null;
+                indexIdListener.whenComplete(idx -> {
+                    assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.GENERIC, ThreadPool.Names.SNAPSHOT);
+                    repository.restoreShard(
+                        indexShard.store(),
+                        restoreSource.snapshot().getSnapshotId(),
+                        idx,
+                        snapshotShardId,
+                        indexShard.recoveryState(),
+                        restoreListener
+                    );
+                }, restoreListener::onFailure);
+            } catch (Exception e) {
+                restoreListener.onFailure(e);
+            }
+        }));
     }
 
     public static void bootstrap(final IndexShard indexShard, final Store store) throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/index/stats/IndexingPressureStats.java b/server/src/main/java/org/elasticsearch/index/stats/IndexingPressureStats.java
index a2de41318b56..e4db6921a44d 100644
--- a/server/src/main/java/org/elasticsearch/index/stats/IndexingPressureStats.java
+++ b/server/src/main/java/org/elasticsearch/index/stats/IndexingPressureStats.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.index.stats;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -57,7 +57,7 @@ public IndexingPressureStats(StreamInput in) throws IOException {
         primaryRejections = in.readVLong();
         replicaRejections = in.readVLong();
 
-        if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             memoryLimit = in.readVLong();
         } else {
             memoryLimit = -1L;
@@ -129,7 +129,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(primaryRejections);
         out.writeVLong(replicaRejections);
 
-        if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             out.writeVLong(memoryLimit);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/index/store/ByteSizeCachingDirectory.java b/server/src/main/java/org/elasticsearch/index/store/ByteSizeCachingDirectory.java
index a97cd3f2e2d7..932cc80f8f63 100644
--- a/server/src/main/java/org/elasticsearch/index/store/ByteSizeCachingDirectory.java
+++ b/server/src/main/java/org/elasticsearch/index/store/ByteSizeCachingDirectory.java
@@ -128,6 +128,8 @@ private IndexOutput wrapIndexOutput(IndexOutput out) {
             numOpenOutputs++;
         }
         return new FilterIndexOutput(out.toString(), out) {
+            private boolean closed;
+
             @Override
             public void writeBytes(byte[] b, int length) throws IOException {
                 // Don't write to atomicXXX here since it might be called in
@@ -168,8 +170,11 @@ public void close() throws IOException {
                     super.close();
                 } finally {
                     synchronized (ByteSizeCachingDirectory.this) {
-                        numOpenOutputs--;
-                        modCount++;
+                        if (closed == false) {
+                            closed = true;
+                            numOpenOutputs--;
+                            modCount++;
+                        }
                     }
                 }
             }
diff --git a/server/src/main/java/org/elasticsearch/index/store/Store.java b/server/src/main/java/org/elasticsearch/index/store/Store.java
index c3e3389a86e7..6f056a45ddfd 100644
--- a/server/src/main/java/org/elasticsearch/index/store/Store.java
+++ b/server/src/main/java/org/elasticsearch/index/store/Store.java
@@ -40,7 +40,6 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -55,6 +54,7 @@
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.RefCounted;
+import org.elasticsearch.core.Streams;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.env.NodeEnvironment;
diff --git a/server/src/main/java/org/elasticsearch/index/store/StoreFileMetadata.java b/server/src/main/java/org/elasticsearch/index/store/StoreFileMetadata.java
index eadffeeecdca..d9422c648ef8 100644
--- a/server/src/main/java/org/elasticsearch/index/store/StoreFileMetadata.java
+++ b/server/src/main/java/org/elasticsearch/index/store/StoreFileMetadata.java
@@ -15,6 +15,7 @@
 import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -28,7 +29,7 @@
 public class StoreFileMetadata implements Writeable {
 
     public static final BytesRef UNAVAILABLE_WRITER_UUID = new BytesRef();
-    private static final org.elasticsearch.Version WRITER_UUID_MIN_VERSION = org.elasticsearch.Version.V_7_16_0;
+    private static final TransportVersion WRITER_UUID_MIN_VERSION = TransportVersion.V_7_16_0;
 
     private final String name;
 
@@ -68,7 +69,7 @@ public StoreFileMetadata(StreamInput in) throws IOException {
         checksum = in.readString();
         writtenBy = in.readString();
         hash = in.readBytesRef();
-        if (in.getVersion().onOrAfter(WRITER_UUID_MIN_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(WRITER_UUID_MIN_VERSION)) {
             writerUuid = StoreFileMetadata.toWriterUuid(in.readBytesRef());
         } else {
             writerUuid = UNAVAILABLE_WRITER_UUID;
@@ -82,7 +83,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(checksum);
         out.writeString(writtenBy);
         out.writeBytesRef(hash);
-        if (out.getVersion().onOrAfter(WRITER_UUID_MIN_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(WRITER_UUID_MIN_VERSION)) {
             out.writeBytesRef(writerUuid);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/index/store/StoreStats.java b/server/src/main/java/org/elasticsearch/index/store/StoreStats.java
index e73f8df40ffe..0820f859355a 100644
--- a/server/src/main/java/org/elasticsearch/index/store/StoreStats.java
+++ b/server/src/main/java/org/elasticsearch/index/store/StoreStats.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.index.store;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -27,8 +27,8 @@ public class StoreStats implements Writeable, ToXContentFragment {
      */
     public static final long UNKNOWN_RESERVED_BYTES = -1L;
 
-    public static final Version RESERVED_BYTES_VERSION = Version.V_7_9_0;
-    public static final Version TOTAL_DATA_SET_SIZE_SIZE_VERSION = Version.V_7_13_0;
+    public static final TransportVersion RESERVED_BYTES_VERSION = TransportVersion.V_7_9_0;
+    public static final TransportVersion TOTAL_DATA_SET_SIZE_SIZE_VERSION = TransportVersion.V_7_13_0;
 
     private long sizeInBytes;
     private long totalDataSetSizeInBytes;
@@ -40,12 +40,12 @@ public StoreStats() {
 
     public StoreStats(StreamInput in) throws IOException {
         sizeInBytes = in.readVLong();
-        if (in.getVersion().onOrAfter(TOTAL_DATA_SET_SIZE_SIZE_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(TOTAL_DATA_SET_SIZE_SIZE_VERSION)) {
             totalDataSetSizeInBytes = in.readVLong();
         } else {
             totalDataSetSizeInBytes = sizeInBytes;
         }
-        if (in.getVersion().onOrAfter(RESERVED_BYTES_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(RESERVED_BYTES_VERSION)) {
             reservedSize = in.readZLong();
         } else {
             reservedSize = UNKNOWN_RESERVED_BYTES;
@@ -118,10 +118,10 @@ public ByteSizeValue getReservedSize() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(sizeInBytes);
-        if (out.getVersion().onOrAfter(TOTAL_DATA_SET_SIZE_SIZE_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(TOTAL_DATA_SET_SIZE_SIZE_VERSION)) {
             out.writeVLong(totalDataSetSizeInBytes);
         }
-        if (out.getVersion().onOrAfter(RESERVED_BYTES_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(RESERVED_BYTES_VERSION)) {
             out.writeZLong(reservedSize);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/index/translog/OperationListener.java b/server/src/main/java/org/elasticsearch/index/translog/OperationListener.java
new file mode 100644
index 000000000000..ddca01292784
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/translog/OperationListener.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.translog;
+
+import org.elasticsearch.common.bytes.BytesReference;
+
+@FunctionalInterface
+public interface OperationListener {
+
+    /**
+     * This method is called when a new operation is added to the translog. The BytesReference is a releasable
+     * instance, so it should not be retained beyond the scope of this method.
+     *
+     * @param data a releasable bytes reference of the data add
+     * @param seqNo the sequence number of the operation
+     * @param location the location written
+     */
+    void operationAdded(BytesReference data, long seqNo, Translog.Location location);
+}
diff --git a/server/src/main/java/org/elasticsearch/index/translog/Translog.java b/server/src/main/java/org/elasticsearch/index/translog/Translog.java
index f15ea8beceb0..8eba045ee3cc 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/Translog.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/Translog.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.index.translog;
 
 import org.apache.lucene.store.AlreadyClosedException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -41,6 +41,7 @@
 import java.io.IOException;
 import java.nio.channels.FileChannel;
 import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.nio.file.StandardOpenOption;
@@ -124,6 +125,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
     private final String translogUUID;
     private final TranslogDeletionPolicy deletionPolicy;
     private final LongConsumer persistedSequenceNumberConsumer;
+    private final OperationListener operationListener;
 
     /**
      * Creates a new Translog instance. This method will create a new transaction log unless the given {@link TranslogGeneration} is
@@ -157,6 +159,7 @@ public Translog(
         this.globalCheckpointSupplier = globalCheckpointSupplier;
         this.primaryTermSupplier = primaryTermSupplier;
         this.persistedSequenceNumberConsumer = persistedSequenceNumberConsumer;
+        this.operationListener = config.getOperationListener();
         this.deletionPolicy = deletionPolicy;
         this.translogUUID = translogUUID;
         bigArrays = config.getBigArrays();
@@ -234,7 +237,13 @@ private ArrayList<TranslogReader> recoverFromFiles(Checkpoint checkpoint) throws
             // translog was found.
             for (long i = checkpoint.generation; i >= minGenerationToRecoverFrom; i--) {
                 Path committedTranslogFile = location.resolve(getFilename(i));
-                if (Files.exists(committedTranslogFile) == false) {
+                final Checkpoint readerCheckpoint = i == checkpoint.generation
+                    ? checkpoint
+                    : Checkpoint.read(location.resolve(getCommitCheckpointFileName(i)));
+                final TranslogReader reader;
+                try {
+                    reader = openReader(committedTranslogFile, readerCheckpoint);
+                } catch (NoSuchFileException fnfe) {
                     throw new TranslogCorruptedException(
                         committedTranslogFile.toString(),
                         "translog file doesn't exist with generation: "
@@ -246,10 +255,6 @@ private ArrayList<TranslogReader> recoverFromFiles(Checkpoint checkpoint) throws
                             + " - translog ids must be consecutive"
                     );
                 }
-                final Checkpoint readerCheckpoint = i == checkpoint.generation
-                    ? checkpoint
-                    : Checkpoint.read(location.resolve(getCommitCheckpointFileName(i)));
-                final TranslogReader reader = openReader(committedTranslogFile, readerCheckpoint);
                 assert reader.getPrimaryTerm() <= primaryTermSupplier.getAsLong()
                     : "Primary terms go backwards; current term ["
                         + primaryTermSupplier.getAsLong()
@@ -552,7 +557,8 @@ TranslogWriter createWriter(
                 tragedy,
                 persistedSequenceNumberConsumer,
                 bigArrays,
-                diskIoBufferPool
+                diskIoBufferPool,
+                operationListener
             );
         } catch (final IOException e) {
             throw new TranslogException(shardId, "failed to create new translog file", e);
@@ -1234,7 +1240,9 @@ public long version() {
         }
 
         private void write(final StreamOutput out) throws IOException {
-            final int format = out.getVersion().onOrAfter(Version.V_8_0_0) ? SERIALIZATION_FORMAT : FORMAT_NO_VERSION_TYPE;
+            final int format = out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)
+                ? SERIALIZATION_FORMAT
+                : FORMAT_NO_VERSION_TYPE;
             out.writeVInt(format);
             out.writeString(id);
             if (format < FORMAT_NO_DOC_TYPE) {
@@ -1395,7 +1403,9 @@ public BytesReference source() {
         }
 
         private void write(final StreamOutput out) throws IOException {
-            final int format = out.getVersion().onOrAfter(Version.V_8_0_0) ? SERIALIZATION_FORMAT : FORMAT_NO_VERSION_TYPE;
+            final int format = out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)
+                ? SERIALIZATION_FORMAT
+                : FORMAT_NO_VERSION_TYPE;
             out.writeVInt(format);
             if (format < FORMAT_NO_DOC_TYPE) {
                 out.writeString(MapperService.SINGLE_MAPPING_NAME);
@@ -1945,7 +1955,8 @@ public static String createEmptyTranslog(
             new TragicExceptionHolder(),
             seqNo -> { throw new UnsupportedOperationException(); },
             BigArrays.NON_RECYCLING_INSTANCE,
-            DiskIoBufferPool.INSTANCE
+            DiskIoBufferPool.INSTANCE,
+            (d, s, l) -> {}
         );
         writer.close();
         return uuid;
diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogConfig.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogConfig.java
index bf3a5b5f1be1..add0867846a4 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/TranslogConfig.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogConfig.java
@@ -32,6 +32,7 @@ public final class TranslogConfig {
     private final ShardId shardId;
     private final Path translogPath;
     private final ByteSizeValue bufferSize;
+    private final OperationListener operationListener;
 
     /**
      * Creates a new TranslogConfig instance
@@ -51,6 +52,18 @@ public TranslogConfig(ShardId shardId, Path translogPath, IndexSettings indexSet
         BigArrays bigArrays,
         ByteSizeValue bufferSize,
         DiskIoBufferPool diskIoBufferPool
+    ) {
+        this(shardId, translogPath, indexSettings, bigArrays, bufferSize, diskIoBufferPool, (d, s, l) -> {});
+    }
+
+    public TranslogConfig(
+        ShardId shardId,
+        Path translogPath,
+        IndexSettings indexSettings,
+        BigArrays bigArrays,
+        ByteSizeValue bufferSize,
+        DiskIoBufferPool diskIoBufferPool,
+        OperationListener operationListener
     ) {
         this.bufferSize = bufferSize;
         this.indexSettings = indexSettings;
@@ -58,6 +71,7 @@ public TranslogConfig(ShardId shardId, Path translogPath, IndexSettings indexSet
         this.translogPath = translogPath;
         this.bigArrays = bigArrays;
         this.diskIoBufferPool = diskIoBufferPool;
+        this.operationListener = operationListener;
     }
 
     /**
@@ -102,4 +116,8 @@ public ByteSizeValue getBufferSize() {
     public DiskIoBufferPool getDiskIoBufferPool() {
         return diskIoBufferPool;
     }
+
+    public OperationListener getOperationListener() {
+        return operationListener;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
index 4e67234f0b32..4114963fb4a4 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
@@ -67,6 +67,7 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
 
     // callback that's called whenever an operation with a given sequence number is successfully persisted.
     private final LongConsumer persistedSequenceNumberConsumer;
+    private final OperationListener operationListener;
 
     protected final AtomicBoolean closed = new AtomicBoolean(false);
     // lock order try(Releasable lock = writeLock.acquire()) -> synchronized(this)
@@ -97,7 +98,8 @@ private TranslogWriter(
         final TragicExceptionHolder tragedy,
         final LongConsumer persistedSequenceNumberConsumer,
         final BigArrays bigArrays,
-        final DiskIoBufferPool diskIoBufferPool
+        final DiskIoBufferPool diskIoBufferPool,
+        final OperationListener operationListener
     ) throws IOException {
         super(initialCheckpoint.generation, channel, path, header);
         assert initialCheckpoint.offset == channel.position()
@@ -124,6 +126,7 @@ private TranslogWriter(
         this.diskIoBufferPool = diskIoBufferPool;
         this.seenSequenceNumbers = Assertions.ENABLED ? new HashMap<>() : null;
         this.tragedy = tragedy;
+        this.operationListener = operationListener;
     }
 
     public static TranslogWriter create(
@@ -141,7 +144,9 @@ public static TranslogWriter create(
         TragicExceptionHolder tragedy,
         final LongConsumer persistedSequenceNumberConsumer,
         final BigArrays bigArrays,
-        DiskIoBufferPool diskIoBufferPool
+        DiskIoBufferPool diskIoBufferPool,
+        final OperationListener operationListener
+
     ) throws IOException {
         final Path checkpointFile = file.getParent().resolve(Translog.CHECKPOINT_FILE_NAME);
 
@@ -183,7 +188,8 @@ public static TranslogWriter create(
                 tragedy,
                 persistedSequenceNumberConsumer,
                 bigArrays,
-                diskIoBufferPool
+                diskIoBufferPool,
+                operationListener
             );
         } catch (Exception exception) {
             // if we fail to bake the file-generation into the checkpoint we stick with the file and once we recover and that
@@ -241,6 +247,7 @@ public Translog.Location add(final BytesReference data, final long seqNo) throws
             assert assertNoSeqNumberConflict(seqNo, data);
 
             location = new Translog.Location(generation, offset, data.length());
+            operationListener.operationAdded(data, seqNo, location);
             bufferedBytes = buffer.size();
         }
 
diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java
index 7cde9cdb07d3..318b54da653a 100644
--- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -619,14 +619,17 @@ public void onStoreClosed(ShardId shardId) {
         };
         finalListeners.add(onStoreClose);
         finalListeners.add(oldShardsStats);
-        final IndexService indexService = createIndexService(
-            CREATE_INDEX,
-            indexMetadata,
-            indicesQueryCache,
-            indicesFieldDataCache,
-            finalListeners,
-            indexingMemoryController
-        );
+        IndexService indexService;
+        try (var ignored = threadPool.getThreadContext().newStoredContext()) {
+            indexService = createIndexService(
+                CREATE_INDEX,
+                indexMetadata,
+                indicesQueryCache,
+                indicesFieldDataCache,
+                finalListeners,
+                indexingMemoryController
+            );
+        }
         boolean success = false;
         try {
             if (writeDanglingIndices && nodeWriteDanglingIndicesInfo) {
@@ -1648,14 +1651,20 @@ public AliasFilter buildAliasFilter(ClusterState state, String index, Set<String
 
         Metadata metadata = state.metadata();
         IndexAbstraction ia = state.metadata().getIndicesLookup().get(index);
-        if (ia.getParentDataStream() != null) {
-            List<QueryBuilder> filters = Arrays.stream(aliases).map(name -> metadata.dataStreamAliases().get(name)).map(dataStreamAlias -> {
-                try {
-                    return filterParser.apply(dataStreamAlias.getFilter().uncompressed());
-                } catch (IOException e) {
-                    throw new UncheckedIOException(e);
-                }
-            }).toList();
+        IndexAbstraction.DataStream dataStream = ia.getParentDataStream();
+        if (dataStream != null) {
+            String dataStreamName = dataStream.getName();
+            List<QueryBuilder> filters = Arrays.stream(aliases)
+                .map(name -> metadata.dataStreamAliases().get(name))
+                .filter(dataStreamAlias -> dataStreamAlias.getFilter(dataStreamName) != null)
+                .map(dataStreamAlias -> {
+                    try {
+                        return filterParser.apply(dataStreamAlias.getFilter(dataStreamName).uncompressed());
+                    } catch (IOException e) {
+                        throw new UncheckedIOException(e);
+                    }
+                })
+                .toList();
             if (filters.isEmpty()) {
                 return AliasFilter.of(null, aliases);
             } else {
diff --git a/server/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java b/server/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java
index cd3db9e4478b..43e32ba0b0e1 100644
--- a/server/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java
+++ b/server/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.indices;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
@@ -51,7 +51,7 @@
  */
 public class NodeIndicesStats implements Writeable, ToXContentFragment {
 
-    private static final Version VERSION_SUPPORTING_STATS_BY_INDEX = Version.V_8_5_0;
+    private static final TransportVersion VERSION_SUPPORTING_STATS_BY_INDEX = TransportVersion.V_8_5_0;
 
     private final CommonStats stats;
     private final Map<Index, List<IndexShardStats>> statsByShard;
@@ -72,7 +72,7 @@ public NodeIndicesStats(StreamInput in) throws IOException {
             statsByShard.put(index, indexShardStats);
         }
 
-        if (in.getVersion().onOrAfter(VERSION_SUPPORTING_STATS_BY_INDEX)) {
+        if (in.getTransportVersion().onOrAfter(VERSION_SUPPORTING_STATS_BY_INDEX)) {
             statsByIndex = in.readMap(Index::new, CommonStats::new);
         } else {
             statsByIndex = new HashMap<>();
@@ -196,7 +196,7 @@ public NodeMappingStats getNodeMappingStats() {
     public void writeTo(StreamOutput out) throws IOException {
         stats.writeTo(out);
         out.writeMap(statsByShard, (o, k) -> k.writeTo(o), StreamOutput::writeList);
-        if (out.getVersion().onOrAfter(VERSION_SUPPORTING_STATS_BY_INDEX)) {
+        if (out.getTransportVersion().onOrAfter(VERSION_SUPPORTING_STATS_BY_INDEX)) {
             out.writeMap(statsByIndex, (o, k) -> k.writeTo(o), (o, v) -> v.writeTo(o));
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexManager.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexManager.java
index 2b0243277519..d46d4a4baea0 100644
--- a/server/src/main/java/org/elasticsearch/indices/SystemIndexManager.java
+++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexManager.java
@@ -116,8 +116,8 @@ public void clusterChanged(ClusterChangedEvent event) {
                 // Use a GroupedActionListener so that we only release the lock once all upgrade attempts have succeeded or failed.
                 // The failures are logged in upgradeIndexMetadata(), so we don't actually care about them here.
                 ActionListener<AcknowledgedResponse> listener = new GroupedActionListener<>(
-                    ActionListener.wrap(() -> isUpgradeInProgress.set(false)),
-                    descriptors.size()
+                    descriptors.size(),
+                    ActionListener.wrap(() -> isUpgradeInProgress.set(false))
                 );
 
                 descriptors.forEach(descriptor -> upgradeIndexMetadata(descriptor, listener));
diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java
index 28750d663582..54e2f04877b6 100644
--- a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java
+++ b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java
@@ -922,6 +922,7 @@ public static void cleanUpFeature(
             }
 
             GroupedActionListener<ResetFeatureStateStatus> groupedListener = new GroupedActionListener<>(
+                taskCount,
                 ActionListener.wrap(listenerResults -> {
                     List<ResetFeatureStateStatus> errors = listenerResults.stream()
                         .filter(status -> status.getStatus() == ResetFeatureStateResponse.ResetFeatureStateStatus.Status.FAILURE)
@@ -936,8 +937,7 @@ public static void cleanUpFeature(
                         errors.forEach(e -> logger.warn(() -> "error while resetting feature [" + name + "]", e.getException()));
                         listener.onResponse(ResetFeatureStateStatus.failure(name, new Exception(exceptions.toString())));
                     }
-                }, listener::onFailure),
-                taskCount
+                }, listener::onFailure)
             );
 
             // Send cleanup for the associated indices, they don't need special origin since they are not protected
diff --git a/server/src/main/java/org/elasticsearch/indices/TermsLookup.java b/server/src/main/java/org/elasticsearch/indices/TermsLookup.java
index 99a1751dd58f..b18cf552c645 100644
--- a/server/src/main/java/org/elasticsearch/indices/TermsLookup.java
+++ b/server/src/main/java/org/elasticsearch/indices/TermsLookup.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.indices;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -56,7 +56,7 @@ public TermsLookup(String index, String id, String path) {
      * Read from a stream.
      */
     public TermsLookup(StreamInput in) throws IOException {
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             in.readOptionalString();
         }
         id = in.readString();
@@ -67,7 +67,7 @@ public TermsLookup(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeOptionalString(MapperService.SINGLE_MAPPING_NAME);
         }
         out.writeString(id);
diff --git a/server/src/main/java/org/elasticsearch/indices/analysis/wrappers/SettingsInvocationHandler.java b/server/src/main/java/org/elasticsearch/indices/analysis/wrappers/SettingsInvocationHandler.java
new file mode 100644
index 000000000000..82c1761dc6f6
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/indices/analysis/wrappers/SettingsInvocationHandler.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.indices.analysis.wrappers;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugin.settings.BooleanSetting;
+import org.elasticsearch.plugin.settings.IntSetting;
+import org.elasticsearch.plugin.settings.ListSetting;
+import org.elasticsearch.plugin.settings.LongSetting;
+import org.elasticsearch.plugin.settings.StringSetting;
+
+import java.lang.annotation.Annotation;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.util.Collections;
+import java.util.function.Function;
+
+public class SettingsInvocationHandler implements InvocationHandler {
+
+    private static Logger LOGGER = LogManager.getLogger(SettingsInvocationHandler.class);
+    private Settings settings;
+    private Environment environment;
+
+    public SettingsInvocationHandler(Settings settings, Environment environment) {
+        this.settings = settings;
+        this.environment = environment;
+    }
+
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    public static <T> T create(Settings settings, Class<T> parameterType, Environment environment) {
+        return (T) Proxy.newProxyInstance(
+            parameterType.getClassLoader(),
+            new Class[] { parameterType },
+            new SettingsInvocationHandler(settings, environment)
+        );
+    }
+
+    @Override
+    public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+        assert method.getAnnotations().length == 1;
+        Annotation annotation = method.getAnnotations()[0];
+
+        if (annotation instanceof IntSetting setting) {
+            return getValue(Integer::valueOf, setting.path(), setting.defaultValue());
+        } else if (annotation instanceof LongSetting setting) {
+            return getValue(Long::valueOf, setting.path(), setting.defaultValue());
+        } else if (annotation instanceof BooleanSetting setting) {
+            return getValue(Boolean::valueOf, setting.path(), setting.defaultValue());
+        } else if (annotation instanceof StringSetting setting) {
+            return getValue(String::valueOf, setting.path(), setting.defaultValue());
+        } else if (annotation instanceof ListSetting setting) {
+            return settings.getAsList(setting.path(), Collections.emptyList());
+        } else {
+            throw new IllegalArgumentException("Unrecognised annotation " + annotation);
+        }
+    }
+
+    private <T> T getValue(Function<String, T> parser, String path, T defaultValue) {
+        String key = path;
+        if (settings.get(key) != null) {
+            return parser.apply(settings.get(key));
+        }
+        return defaultValue;
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/indices/analysis/wrappers/StableApiWrappers.java b/server/src/main/java/org/elasticsearch/indices/analysis/wrappers/StableApiWrappers.java
index 771d3012a242..e3d4b02c52ee 100644
--- a/server/src/main/java/org/elasticsearch/indices/analysis/wrappers/StableApiWrappers.java
+++ b/server/src/main/java/org/elasticsearch/indices/analysis/wrappers/StableApiWrappers.java
@@ -15,6 +15,13 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.indices.analysis.AnalysisModule;
+import org.elasticsearch.plugin.Inject;
+import org.elasticsearch.plugin.analysis.AnalysisMode;
+import org.elasticsearch.plugin.analysis.AnalyzerFactory;
+import org.elasticsearch.plugin.analysis.CharFilterFactory;
+import org.elasticsearch.plugin.analysis.TokenFilterFactory;
+import org.elasticsearch.plugin.analysis.TokenizerFactory;
+import org.elasticsearch.plugin.settings.AnalysisSettings;
 import org.elasticsearch.plugins.scanners.PluginInfo;
 import org.elasticsearch.plugins.scanners.StablePluginsRegistry;
 
@@ -36,41 +43,25 @@ public class StableApiWrappers {
     public static
         Map<String, AnalysisModule.AnalysisProvider<org.elasticsearch.index.analysis.CharFilterFactory>>
         oldApiForStableCharFilterFactory(StablePluginsRegistry stablePluginRegistry) {
-        return mapStablePluginApiToOld(
-            stablePluginRegistry,
-            org.elasticsearch.plugin.analysis.api.CharFilterFactory.class,
-            StableApiWrappers::wrapCharFilterFactory
-        );
+        return mapStablePluginApiToOld(stablePluginRegistry, CharFilterFactory.class, StableApiWrappers::wrapCharFilterFactory);
     }
 
     public static
         Map<String, AnalysisModule.AnalysisProvider<org.elasticsearch.index.analysis.TokenFilterFactory>>
         oldApiForTokenFilterFactory(StablePluginsRegistry stablePluginRegistry) {
-        return mapStablePluginApiToOld(
-            stablePluginRegistry,
-            org.elasticsearch.plugin.analysis.api.TokenFilterFactory.class,
-            StableApiWrappers::wrapTokenFilterFactory
-        );
+        return mapStablePluginApiToOld(stablePluginRegistry, TokenFilterFactory.class, StableApiWrappers::wrapTokenFilterFactory);
     }
 
     public static Map<String, AnalysisModule.AnalysisProvider<org.elasticsearch.index.analysis.TokenizerFactory>> oldApiForTokenizerFactory(
         StablePluginsRegistry stablePluginRegistry
     ) {
-        return mapStablePluginApiToOld(
-            stablePluginRegistry,
-            org.elasticsearch.plugin.analysis.api.TokenizerFactory.class,
-            StableApiWrappers::wrapTokenizerFactory
-        );
+        return mapStablePluginApiToOld(stablePluginRegistry, TokenizerFactory.class, StableApiWrappers::wrapTokenizerFactory);
     }
 
     public static
         Map<String, AnalysisModule.AnalysisProvider<org.elasticsearch.index.analysis.AnalyzerProvider<?>>>
         oldApiForAnalyzerFactory(StablePluginsRegistry stablePluginRegistry) {
-        return mapStablePluginApiToOld(
-            stablePluginRegistry,
-            org.elasticsearch.plugin.analysis.api.AnalyzerFactory.class,
-            StableApiWrappers::wrapAnalyzerFactory
-        );
+        return mapStablePluginApiToOld(stablePluginRegistry, AnalyzerFactory.class, StableApiWrappers::wrapAnalyzerFactory);
     }
 
     private static <T, F> Map<String, AnalysisModule.AnalysisProvider<T>> mapStablePluginApiToOld(
@@ -104,9 +95,7 @@ public T get(IndexSettings indexSettings, Environment environment, String name,
         };
     }
 
-    private static org.elasticsearch.index.analysis.CharFilterFactory wrapCharFilterFactory(
-        org.elasticsearch.plugin.analysis.api.CharFilterFactory charFilterFactory
-    ) {
+    private static org.elasticsearch.index.analysis.CharFilterFactory wrapCharFilterFactory(CharFilterFactory charFilterFactory) {
         return new org.elasticsearch.index.analysis.CharFilterFactory() {
             @Override
             public String name() {
@@ -125,9 +114,7 @@ public Reader normalize(Reader reader) {
         };
     }
 
-    private static org.elasticsearch.index.analysis.TokenFilterFactory wrapTokenFilterFactory(
-        org.elasticsearch.plugin.analysis.api.TokenFilterFactory f
-    ) {
+    private static org.elasticsearch.index.analysis.TokenFilterFactory wrapTokenFilterFactory(TokenFilterFactory f) {
         return new org.elasticsearch.index.analysis.TokenFilterFactory() {
             @Override
             public String name() {
@@ -149,17 +136,13 @@ public org.elasticsearch.index.analysis.AnalysisMode getAnalysisMode() {
                 return mapAnalysisMode(f.getAnalysisMode());
             }
 
-            private org.elasticsearch.index.analysis.AnalysisMode mapAnalysisMode(
-                org.elasticsearch.plugin.analysis.api.AnalysisMode analysisMode
-            ) {
+            private org.elasticsearch.index.analysis.AnalysisMode mapAnalysisMode(AnalysisMode analysisMode) {
                 return org.elasticsearch.index.analysis.AnalysisMode.valueOf(analysisMode.name());
             }
         };
     }
 
-    private static org.elasticsearch.index.analysis.TokenizerFactory wrapTokenizerFactory(
-        org.elasticsearch.plugin.analysis.api.TokenizerFactory f
-    ) {
+    private static org.elasticsearch.index.analysis.TokenizerFactory wrapTokenizerFactory(TokenizerFactory f) {
         return new org.elasticsearch.index.analysis.TokenizerFactory() {
 
             @Override
@@ -174,9 +157,7 @@ public Tokenizer create() {
         };
     }
 
-    private static org.elasticsearch.index.analysis.AnalyzerProvider<?> wrapAnalyzerFactory(
-        org.elasticsearch.plugin.analysis.api.AnalyzerFactory f
-    ) {
+    private static org.elasticsearch.index.analysis.AnalyzerProvider<?> wrapAnalyzerFactory(AnalyzerFactory f) {
         return new org.elasticsearch.index.analysis.AnalyzerProvider<>() {
             @Override
             public String name() {
@@ -204,10 +185,47 @@ private static <T> T createInstance(
         Environment environment
     ) {
         try {
-            Constructor<T> constructor = clazz.getConstructor();
-            return constructor.newInstance();
-        } catch (NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e) {
-            throw new IllegalStateException("cannot create instance of " + clazz, e);
+
+            Constructor<?>[] constructors = clazz.getConstructors();
+            if (constructors.length > 1) {
+                throw new IllegalStateException("Plugin can only have one public constructor.");
+            }
+            Constructor<?> constructor = constructors[0];
+            if (constructor.getParameterCount() == 0) {
+                return (T) constructor.newInstance();
+            } else {
+                Inject inject = constructor.getAnnotation(Inject.class);
+                if (inject != null) {
+                    Class<?>[] parameterTypes = constructor.getParameterTypes();
+                    Object[] parameters = new Object[parameterTypes.length];
+                    for (int i = 0; i < parameterTypes.length; i++) {
+                        Object settings = createSettings(parameterTypes[i], indexSettings, nodeSettings, analysisSettings, environment);
+                        parameters[i] = settings;
+                    }
+                    return (T) constructor.newInstance(parameters);
+                } else {
+                    throw new IllegalStateException("Missing @Inject annotation for constructor with settings.");
+                }
+            }
+
+        } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) {
+            throw new IllegalStateException("Cannot create instance of " + clazz, e);
+        }
+
+    }
+
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    private static <T> T createSettings(
+        Class<T> settingsClass,
+        IndexSettings indexSettings,
+        Settings nodeSettings,
+        Settings analysisSettings,
+        Environment environment
+    ) {
+        if (settingsClass.getAnnotationsByType(AnalysisSettings.class).length > 0) {
+            return SettingsInvocationHandler.create(analysisSettings, settingsClass, environment);
         }
+
+        throw new IllegalArgumentException("Parameter is not instance of a class annotated with settings annotation.");
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/MultiFileWriter.java b/server/src/main/java/org/elasticsearch/indices/recovery/MultiFileWriter.java
index 954c2b616d24..cb1168caf6e9 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/MultiFileWriter.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/MultiFileWriter.java
@@ -37,26 +37,40 @@
 
 public class MultiFileWriter extends AbstractRefCounted implements Releasable {
 
-    public MultiFileWriter(Store store, RecoveryState.Index indexState, String tempFilePrefix, Logger logger, Runnable ensureOpen) {
-        this.store = store;
-        this.indexState = indexState;
-        this.tempFilePrefix = tempFilePrefix;
-        this.logger = logger;
-        this.ensureOpen = ensureOpen;
-    }
-
-    private final Runnable ensureOpen;
-    private final AtomicBoolean closed = new AtomicBoolean(false);
-    private final Logger logger;
     private final Store store;
     private final RecoveryState.Index indexState;
     private final String tempFilePrefix;
+    private final Logger logger;
+    private final Runnable ensureOpen;
+    private final boolean verifyOutput;
+
+    private final AtomicBoolean closed = new AtomicBoolean(false);
 
     private final ConcurrentMap<String, IndexOutput> openIndexOutputs = ConcurrentCollections.newConcurrentMap();
     private final ConcurrentMap<String, FileChunkWriter> fileChunkWriters = ConcurrentCollections.newConcurrentMap();
 
     final Map<String, String> tempFileNames = ConcurrentCollections.newConcurrentMap();
 
+    public MultiFileWriter(Store store, RecoveryState.Index indexState, String tempFilePrefix, Logger logger, Runnable ensureOpen) {
+        this(store, indexState, tempFilePrefix, logger, ensureOpen, true);
+    }
+
+    public MultiFileWriter(
+        Store store,
+        RecoveryState.Index indexState,
+        String tempFilePrefix,
+        Logger logger,
+        Runnable ensureOpen,
+        boolean verifyOutput
+    ) {
+        this.store = store;
+        this.indexState = indexState;
+        this.tempFilePrefix = tempFilePrefix;
+        this.logger = logger;
+        this.ensureOpen = ensureOpen;
+        this.verifyOutput = verifyOutput;
+    }
+
     public void writeFileChunk(StoreFileMetadata fileMetadata, long position, ReleasableBytesReference content, boolean lastChunk)
         throws IOException {
         assert Transports.assertNotTransportThread("multi_file_writer");
@@ -69,7 +83,7 @@ public void writeFileChunk(StoreFileMetadata fileMetadata, long position, Releas
         }
     }
 
-    public void writeFile(StoreFileMetadata fileMetadata, long readSnapshotFileBufferSize, InputStream stream) throws Exception {
+    public void writeFile(StoreFileMetadata fileMetadata, long readSnapshotFileBufferSize, InputStream stream) throws IOException {
         ensureOpen.run();
         assert Transports.assertNotTransportThread("multi_file_writer");
 
@@ -81,7 +95,7 @@ public void writeFile(StoreFileMetadata fileMetadata, long readSnapshotFileBuffe
         tempFileNames.put(tempFileName, fileName);
 
         incRef();
-        try (IndexOutput indexOutput = store.createVerifyingOutput(tempFileName, fileMetadata, IOContext.DEFAULT)) {
+        try (IndexOutput indexOutput = createIndexOutput(tempFileName, fileMetadata, IOContext.DEFAULT)) {
             int bufferSize = Math.toIntExact(Math.min(readSnapshotFileBufferSize, fileMetadata.length()));
             byte[] buffer = new byte[bufferSize];
             int length;
@@ -107,7 +121,7 @@ public void writeFile(StoreFileMetadata fileMetadata, long readSnapshotFileBuffe
             assert Arrays.asList(store.directory().listAll()).contains(tempFileName)
                 : "expected: [" + tempFileName + "] in " + Arrays.toString(store.directory().listAll());
             store.directory().sync(Collections.singleton(tempFileName));
-        } catch (Exception e) {
+        } catch (IOException e) {
             tempFileNames.remove(tempFileName);
             store.deleteQuiet(tempFileName);
             indexState.resetRecoveredBytesOfFile(fileName);
@@ -117,6 +131,12 @@ public void writeFile(StoreFileMetadata fileMetadata, long readSnapshotFileBuffe
         }
     }
 
+    private IndexOutput createIndexOutput(String tempFileName, StoreFileMetadata fileMetadata, IOContext context) throws IOException {
+        return verifyOutput
+            ? store.createVerifyingOutput(tempFileName, fileMetadata, context)
+            : store.directory().createOutput(tempFileName, context);
+    }
+
     /** Get a temporary name for the provided file name. */
     String getTempNameForFile(String origFile) {
         return tempFilePrefix + origFile;
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java
index 3b58a9d7b4ea..312669dce91b 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Releasables;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.engine.RecoveryEngineException;
@@ -209,65 +210,94 @@ protected void reestablishRecovery(final StartRecoveryRequest request, final Str
     }
 
     private void doRecovery(final long recoveryId, final StartRecoveryRequest preExistingRequest) {
-        final String actionName;
-        final TransportRequest requestToSend;
-        final StartRecoveryRequest startRequest;
-        final RecoveryState.Timer timer;
-        try (RecoveryRef recoveryRef = onGoingRecoveries.getRecovery(recoveryId)) {
-            if (recoveryRef == null) {
-                logger.trace("not running recovery with id [{}] - can not find it (probably finished)", recoveryId);
-                return;
-            }
-            final RecoveryTarget recoveryTarget = recoveryRef.target();
-            timer = recoveryTarget.state().getTimer();
-            if (preExistingRequest == null) {
-                try {
-                    final IndexShard indexShard = recoveryTarget.indexShard();
-                    indexShard.preRecovery();
-                    assert recoveryTarget.sourceNode() != null : "can not do a recovery without a source node";
-                    logger.trace("{} preparing shard for peer recovery", recoveryTarget.shardId());
-                    indexShard.prepareForIndexRecovery();
-                    if (indexShard.indexSettings().getIndexMetadata().isSearchableSnapshot()) {
-                        // for searchable snapshots, peer recovery is treated similarly to recovery from snapshot
-                        indexShard.getIndexEventListener().afterFilesRestoredFromRepository(indexShard);
-                        final Store store = indexShard.store();
-                        store.incRef();
-                        try {
-                            StoreRecovery.bootstrap(indexShard, store);
-                        } finally {
-                            store.decRef();
-                        }
+        final RecoveryRef recoveryRef = onGoingRecoveries.getRecovery(recoveryId);
+        if (recoveryRef == null) {
+            logger.trace("not running recovery with id [{}] - can not find it (probably finished)", recoveryId);
+            return;
+        }
+        final RecoveryTarget recoveryTarget = recoveryRef.target();
+        assert recoveryTarget.sourceNode() != null : "cannot do a recovery without a source node";
+        final RecoveryState recoveryState = recoveryTarget.state();
+        final RecoveryState.Timer timer = recoveryState.getTimer();
+        final IndexShard indexShard = recoveryTarget.indexShard();
+        final Releasable onCompletion = Releasables.wrap(recoveryTarget.disableRecoveryMonitor(), recoveryRef);
+
+        // async version of the catch/finally structure we need, but this does nothing with successes so needs further modification below
+        final var cleanupOnly = ActionListener.notifyOnce(ActionListener.runBefore(ActionListener.noop().delegateResponse((l, e) -> {
+            // this will be logged as warning later on...
+            logger.trace("unexpected error while preparing shard for peer recovery, failing recovery", e);
+            onGoingRecoveries.failRecovery(
+                recoveryId,
+                new RecoveryFailedException(recoveryTarget.state(), "failed to prepare shard for recovery", e),
+                true
+            );
+        }), onCompletion::close));
+
+        if (indexShard.routingEntry().isPromotableToPrimary() == false) {
+            assert preExistingRequest == null;
+            assert indexShard.indexSettings().getIndexMetadata().isSearchableSnapshot() == false;
+            ActionListener.run(cleanupOnly.map(v -> {
+                logger.trace("{} preparing shard for peer recovery", recoveryTarget.shardId());
+                indexShard.prepareForIndexRecovery();
+                // Skip unnecessary intermediate stages
+                recoveryState.setStage(RecoveryState.Stage.VERIFY_INDEX);
+                recoveryState.setStage(RecoveryState.Stage.TRANSLOG);
+                indexShard.openEngineAndSkipTranslogRecovery();
+                recoveryState.getIndex().setFileDetailsComplete();
+                recoveryState.setStage(RecoveryState.Stage.FINALIZE);
+                onGoingRecoveries.markRecoveryAsDone(recoveryId);
+                return null;
+            }), indexShard::preRecovery);
+            return;
+        }
+
+        record StartRecoveryRequestToSend(StartRecoveryRequest startRecoveryRequest, String actionName, TransportRequest requestToSend) {}
+        final ActionListener<StartRecoveryRequestToSend> toSendListener = cleanupOnly.map(r -> {
+            logger.trace(
+                "{} [{}]: recovery from {}",
+                r.startRecoveryRequest().shardId(),
+                r.actionName(),
+                r.startRecoveryRequest().sourceNode()
+            );
+            transportService.sendRequest(
+                r.startRecoveryRequest().sourceNode(),
+                r.actionName(),
+                r.requestToSend(),
+                new RecoveryResponseHandler(r.startRecoveryRequest(), timer)
+            );
+            return null;
+        });
+
+        if (preExistingRequest == null) {
+            ActionListener.run(toSendListener.map(v -> {
+                logger.trace("{} preparing shard for peer recovery", recoveryTarget.shardId());
+                indexShard.prepareForIndexRecovery();
+                if (indexShard.indexSettings().getIndexMetadata().isSearchableSnapshot()) {
+                    // for searchable snapshots, peer recovery is treated similarly to recovery from snapshot
+                    indexShard.getIndexEventListener().afterFilesRestoredFromRepository(indexShard);
+                    final Store store = indexShard.store();
+                    store.incRef();
+                    try {
+                        StoreRecovery.bootstrap(indexShard, store);
+                    } finally {
+                        store.decRef();
                     }
-                    final long startingSeqNo = indexShard.recoverLocallyUpToGlobalCheckpoint();
-                    assert startingSeqNo == UNASSIGNED_SEQ_NO || recoveryTarget.state().getStage() == RecoveryState.Stage.TRANSLOG
-                        : "unexpected recovery stage [" + recoveryTarget.state().getStage() + "] starting seqno [ " + startingSeqNo + "]";
-                    startRequest = getStartRecoveryRequest(logger, clusterService.localNode(), recoveryTarget, startingSeqNo);
-                    requestToSend = startRequest;
-                    actionName = PeerRecoverySourceService.Actions.START_RECOVERY;
-                } catch (final Exception e) {
-                    // this will be logged as warning later on...
-                    logger.trace("unexpected error while preparing shard for peer recovery, failing recovery", e);
-                    onGoingRecoveries.failRecovery(
-                        recoveryId,
-                        new RecoveryFailedException(recoveryTarget.state(), "failed to prepare shard for recovery", e),
-                        true
-                    );
-                    return;
                 }
-                logger.trace("{} starting recovery from {}", startRequest.shardId(), startRequest.sourceNode());
-            } else {
-                startRequest = preExistingRequest;
-                requestToSend = new ReestablishRecoveryRequest(recoveryId, startRequest.shardId(), startRequest.targetAllocationId());
-                actionName = PeerRecoverySourceService.Actions.REESTABLISH_RECOVERY;
-                logger.trace("{} reestablishing recovery from {}", startRequest.shardId(), startRequest.sourceNode());
-            }
+                final long startingSeqNo = indexShard.recoverLocallyUpToGlobalCheckpoint();
+                assert startingSeqNo == UNASSIGNED_SEQ_NO || recoveryTarget.state().getStage() == RecoveryState.Stage.TRANSLOG
+                    : "unexpected recovery stage [" + recoveryTarget.state().getStage() + "] starting seqno [ " + startingSeqNo + "]";
+                final var startRequest = getStartRecoveryRequest(logger, clusterService.localNode(), recoveryTarget, startingSeqNo);
+                return new StartRecoveryRequestToSend(startRequest, PeerRecoverySourceService.Actions.START_RECOVERY, startRequest);
+            }), indexShard::preRecovery);
+        } else {
+            toSendListener.onResponse(
+                new StartRecoveryRequestToSend(
+                    preExistingRequest,
+                    PeerRecoverySourceService.Actions.REESTABLISH_RECOVERY,
+                    new ReestablishRecoveryRequest(recoveryId, preExistingRequest.shardId(), preExistingRequest.targetAllocationId())
+                )
+            );
         }
-        transportService.sendRequest(
-            startRequest.sourceNode(),
-            actionName,
-            requestToSend,
-            new RecoveryResponseHandler(startRequest, timer)
-        );
     }
 
     // Visible for testing
@@ -296,14 +326,20 @@ public static StartRecoveryRequest getStartRecoveryRequest(
 
         Store.MetadataSnapshot metadataSnapshot;
         try {
-            metadataSnapshot = recoveryTarget.indexShard().snapshotStoreMetadata();
-            // Make sure that the current translog is consistent with the Lucene index; otherwise, we have to throw away the Lucene index.
-            try {
-                final String expectedTranslogUUID = metadataSnapshot.commitUserData().get(Translog.TRANSLOG_UUID_KEY);
-                final long globalCheckpoint = Translog.readGlobalCheckpoint(recoveryTarget.translogLocation(), expectedTranslogUUID);
-                assert globalCheckpoint + 1 >= startingSeqNo : "invalid startingSeqNo " + startingSeqNo + " >= " + globalCheckpoint;
-            } catch (IOException | TranslogCorruptedException e) {
-                logGlobalCheckpointWarning(logger, startingSeqNo, e);
+            if (recoveryTarget.indexShard().routingEntry().isPromotableToPrimary()) {
+                metadataSnapshot = recoveryTarget.indexShard().snapshotStoreMetadata();
+                // Make sure that the current translog is consistent with the Lucene index; otherwise, we have to throw away the Lucene
+                // index.
+                try {
+                    final String expectedTranslogUUID = metadataSnapshot.commitUserData().get(Translog.TRANSLOG_UUID_KEY);
+                    final long globalCheckpoint = Translog.readGlobalCheckpoint(recoveryTarget.translogLocation(), expectedTranslogUUID);
+                    assert globalCheckpoint + 1 >= startingSeqNo : "invalid startingSeqNo " + startingSeqNo + " >= " + globalCheckpoint;
+                } catch (IOException | TranslogCorruptedException e) {
+                    logGlobalCheckpointWarning(logger, startingSeqNo, e);
+                    metadataSnapshot = Store.MetadataSnapshot.EMPTY;
+                    startingSeqNo = UNASSIGNED_SEQ_NO;
+                }
+            } else {
                 metadataSnapshot = Store.MetadataSnapshot.EMPTY;
                 startingSeqNo = UNASSIGNED_SEQ_NO;
             }
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java
index d896425eef2c..54310659a39f 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java
@@ -164,6 +164,7 @@ public RecoveryRef getRecoverySafe(long id, ShardId shardId) {
             throw new IndexShardClosedException(shardId);
         }
         assert recoveryRef.target().shardId().equals(shardId);
+        assert recoveryRef.target().indexShard().routingEntry().isPromotableToPrimary();
         return recoveryRef;
     }
 
@@ -254,11 +255,11 @@ public boolean cancelRecoveriesForShard(ShardId shardId, String reason) {
     }
 
     /**
-     * a reference to {@link RecoveryTarget}, which implements {@link AutoCloseable}. closing the reference
+     * a reference to {@link RecoveryTarget}, which implements {@link Releasable}. closing the reference
      * causes {@link RecoveryTarget#decRef()} to be called. This makes sure that the underlying resources
      * will not be freed until {@link RecoveryRef#close()} is called.
      */
-    public static class RecoveryRef implements AutoCloseable {
+    public static class RecoveryRef implements Releasable {
 
         private final RecoveryTarget status;
         private final AtomicBoolean closed = new AtomicBoolean(false);
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java
index e433cd21b4c9..8915665ead84 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java
@@ -12,6 +12,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.RateLimiter;
 import org.apache.lucene.store.RateLimiter.SimpleRateLimiter;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
@@ -44,7 +45,7 @@
 public class RecoverySettings {
     public static final Version SNAPSHOT_RECOVERIES_SUPPORTED_VERSION = Version.V_7_15_0;
     public static final Version SEQ_NO_SNAPSHOT_RECOVERIES_SUPPORTED_VERSION = Version.V_7_16_0;
-    public static final Version SNAPSHOT_FILE_DOWNLOAD_THROTTLING_SUPPORTED_VERSION = Version.V_7_16_0;
+    public static final TransportVersion SNAPSHOT_FILE_DOWNLOAD_THROTTLING_SUPPORTED_VERSION = TransportVersion.V_7_16_0;
 
     private static final Logger logger = LogManager.getLogger(RecoverySettings.class);
 
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySnapshotFileRequest.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySnapshotFileRequest.java
index 2f2558eab3d8..47cf4ef4824e 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySnapshotFileRequest.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySnapshotFileRequest.java
@@ -50,8 +50,8 @@ public RecoverySnapshotFileRequest(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        assert out.getVersion().onOrAfter(RecoverySettings.SNAPSHOT_RECOVERIES_SUPPORTED_VERSION)
-            : "Unexpected serialization version " + out.getVersion();
+        assert out.getTransportVersion().onOrAfter(RecoverySettings.SNAPSHOT_RECOVERIES_SUPPORTED_VERSION.transportVersion)
+            : "Unexpected serialization version " + out.getTransportVersion();
         super.writeTo(out);
         out.writeLong(recoveryId);
         shardId.writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
index 02e4d13656a9..d8de6392cf8e 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.StepListener;
+import org.elasticsearch.action.support.ListenableActionFuture;
 import org.elasticsearch.action.support.ThreadedActionListener;
 import org.elasticsearch.action.support.replication.ReplicationResponse;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
@@ -121,7 +122,7 @@ public class RecoverySourceHandler {
     private final RecoveryPlannerService recoveryPlannerService;
     private final CancellableThreads cancellableThreads = new CancellableThreads();
     private final List<Closeable> resources = new CopyOnWriteArrayList<>();
-    private final ListenableFuture<RecoveryResponse> future = new ListenableFuture<>();
+    private final ListenableActionFuture<RecoveryResponse> future = new ListenableActionFuture<>();
 
     public RecoverySourceHandler(
         IndexShard shard,
@@ -285,13 +286,7 @@ && isTargetSameHistory()
                             // new one later on in the recovery.
                             shard.removePeerRecoveryRetentionLease(
                                 request.targetNode().getId(),
-                                new ThreadedActionListener<>(
-                                    logger,
-                                    shard.getThreadPool(),
-                                    ThreadPool.Names.GENERIC,
-                                    deleteRetentionLeaseStep,
-                                    false
-                                )
+                                new ThreadedActionListener<>(shard.getThreadPool().generic(), deleteRetentionLeaseStep)
                             );
                         } catch (RetentionLeaseNotFoundException e) {
                             logger.debug("no peer-recovery retention lease for " + request.targetAllocationId());
@@ -415,6 +410,8 @@ static void runUnderPrimaryPermit(
     ) {
         cancellableThreads.execute(() -> {
             CompletableFuture<Releasable> permit = new CompletableFuture<>();
+
+            // this wrapping looks unnecessary necessary, see #93290; TODO remove it
             final ActionListener<Releasable> onAcquired = new ActionListener<Releasable>() {
                 @Override
                 public void onResponse(Releasable releasable) {
@@ -574,6 +571,7 @@ void phase1(IndexCommit snapshot, long startingSeqNo, IntSupplier translogOps, A
                     translogOps.getAsInt(),
                     getRequest().targetNode().getVersion(),
                     canUseSnapshots,
+                    request.isPrimaryRelocation(),
                     ActionListener.wrap(plan -> recoverFilesFromSourceAndSnapshot(plan, store, stopWatch, listener), listener::onFailure)
                 );
             } else {
@@ -979,7 +977,7 @@ void createRetentionLease(final long startingSeqNo, ActionListener<RetentionLeas
                 final StepListener<ReplicationResponse> cloneRetentionLeaseStep = new StepListener<>();
                 final RetentionLease clonedLease = shard.cloneLocalPeerRecoveryRetentionLease(
                     request.targetNode().getId(),
-                    new ThreadedActionListener<>(logger, shard.getThreadPool(), ThreadPool.Names.GENERIC, cloneRetentionLeaseStep, false)
+                    new ThreadedActionListener<>(shard.getThreadPool().generic(), cloneRetentionLeaseStep)
                 );
                 logger.trace("cloned primary's retention lease as [{}]", clonedLease);
                 cloneRetentionLeaseStep.addListener(listener.map(rr -> clonedLease));
@@ -994,7 +992,7 @@ void createRetentionLease(final long startingSeqNo, ActionListener<RetentionLeas
                 final RetentionLease newLease = shard.addPeerRecoveryRetentionLease(
                     request.targetNode().getId(),
                     estimatedGlobalCheckpoint,
-                    new ThreadedActionListener<>(logger, shard.getThreadPool(), ThreadPool.Names.GENERIC, addRetentionLeaseStep, false)
+                    new ThreadedActionListener<>(shard.getThreadPool().generic(), addRetentionLeaseStep)
                 );
                 addRetentionLeaseStep.addListener(listener.map(rr -> newLease));
                 logger.trace("created retention lease with estimated checkpoint of [{}]", estimatedGlobalCheckpoint);
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java
index 6f7eee079e41..dfa726a42cb6 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java
@@ -631,7 +631,7 @@ public FileDetail(StreamInput in) throws IOException {
             length = in.readVLong();
             recovered = in.readVLong();
             reused = in.readBoolean();
-            if (in.getVersion().onOrAfter(RecoverySettings.SNAPSHOT_RECOVERIES_SUPPORTED_VERSION)) {
+            if (in.getTransportVersion().onOrAfter(RecoverySettings.SNAPSHOT_RECOVERIES_SUPPORTED_VERSION.transportVersion)) {
                 recoveredFromSnapshot = in.readLong();
             }
         }
@@ -642,7 +642,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeVLong(length);
             out.writeVLong(recovered);
             out.writeBoolean(reused);
-            if (out.getVersion().onOrAfter(RecoverySettings.SNAPSHOT_RECOVERIES_SUPPORTED_VERSION)) {
+            if (out.getTransportVersion().onOrAfter(RecoverySettings.SNAPSHOT_RECOVERIES_SUPPORTED_VERSION.transportVersion)) {
                 out.writeLong(recoveredFromSnapshot);
             }
         }
@@ -769,7 +769,7 @@ public RecoveryFilesDetails() {
 
         RecoveryFilesDetails(StreamInput in) throws IOException {
             fileDetails = in.readMapValues(FileDetail::new, FileDetail::name);
-            if (in.getVersion().onOrAfter(StoreStats.RESERVED_BYTES_VERSION)) {
+            if (in.getTransportVersion().onOrAfter(StoreStats.RESERVED_BYTES_VERSION)) {
                 complete = in.readBoolean();
             } else {
                 // This flag is used by disk-based allocation to decide whether the remaining bytes measurement is accurate or not; if not
@@ -783,7 +783,7 @@ public RecoveryFilesDetails() {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeCollection(values());
-            if (out.getVersion().onOrAfter(StoreStats.RESERVED_BYTES_VERSION)) {
+            if (out.getTransportVersion().onOrAfter(StoreStats.RESERVED_BYTES_VERSION)) {
                 out.writeBoolean(complete);
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
index aec8f06d5cf8..3189478f0a2e 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.core.AbstractRefCounted;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Releasables;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.mapper.MapperException;
 import org.elasticsearch.index.seqno.ReplicationTracker;
@@ -47,6 +48,7 @@
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
 import static org.elasticsearch.core.Strings.format;
@@ -80,7 +82,7 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget
     // last time this status was accessed
     private volatile long lastAccessTime = System.nanoTime();
 
-    private volatile boolean recoveryMonitorEnabled = true;
+    private final AtomicInteger recoveryMonitorBlocks = new AtomicInteger();
 
     @Nullable // if we're not downloading files from snapshots in this recovery or we're retrying
     private volatile Releasable snapshotFileDownloadsPermit;
@@ -185,7 +187,7 @@ public boolean hasPermitToDownloadSnapshotFiles() {
 
     /** return the last time this RecoveryStatus was used (based on System.nanoTime() */
     public long lastAccessTime() {
-        if (recoveryMonitorEnabled) {
+        if (recoveryMonitorBlocks.get() == 0) {
             return lastAccessTime;
         }
         return System.nanoTime();
@@ -204,12 +206,11 @@ public void setLastAccessTime() {
      * @return releasable that once closed will re-enable liveness checks by the recovery monitor
      */
     public Releasable disableRecoveryMonitor() {
-        assert recoveryMonitorEnabled : "recovery monitor already disabled";
-        recoveryMonitorEnabled = false;
-        return () -> {
+        recoveryMonitorBlocks.incrementAndGet();
+        return Releasables.releaseOnce(() -> {
             setLastAccessTime();
-            recoveryMonitorEnabled = true;
-        };
+            recoveryMonitorBlocks.decrementAndGet();
+        });
     }
 
     public Store store() {
@@ -316,6 +317,7 @@ public void markAsDone() {
 
     @Override
     protected void closeInternal() {
+        assert recoveryMonitorBlocks.get() == 0;
         try {
             multiFileWriter.close();
         } finally {
@@ -495,15 +497,18 @@ public void cleanFiles(
             final Store store = store();
             store.incRef();
             try {
-                store.cleanupAndVerify("recovery CleanFilesRequestHandler", sourceMetadata);
-                final String translogUUID = Translog.createEmptyTranslog(
-                    indexShard.shardPath().resolveTranslog(),
-                    globalCheckpoint,
-                    shardId,
-                    indexShard.getPendingPrimaryTerm()
-                );
-                store.associateIndexWithNewTranslog(translogUUID);
-
+                if (indexShard.routingEntry().isPromotableToPrimary()) {
+                    store.cleanupAndVerify("recovery CleanFilesRequestHandler", sourceMetadata);
+                    final String translogUUID = Translog.createEmptyTranslog(
+                        indexShard.shardPath().resolveTranslog(),
+                        globalCheckpoint,
+                        shardId,
+                        indexShard.getPendingPrimaryTerm()
+                    );
+                    store.associateIndexWithNewTranslog(translogUUID);
+                } else {
+                    indexShard.setGlobalCheckpointIfUnpromotable(globalCheckpoint);
+                }
                 if (indexShard.getRetentionLeases().leases().isEmpty()) {
                     // if empty, may be a fresh IndexShard, so write an empty leases file to disk
                     indexShard.persistRetentionLeases();
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RemoteRecoveryTargetHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RemoteRecoveryTargetHandler.java
index 4d6cdcaba63b..1078bfc03d27 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RemoteRecoveryTargetHandler.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RemoteRecoveryTargetHandler.java
@@ -294,7 +294,6 @@ public void writeFileChunk(
             totalTranslogOps,
             throttleTimeInNanos
         );
-        final Writeable.Reader<TransportResponse.Empty> reader = in -> TransportResponse.Empty.INSTANCE;
 
         // Fork the actual sending onto a separate thread so we can send them concurrently even if CPU-bound (e.g. using compression).
         // The AsyncIOProcessor and MultiFileWriter both concentrate their work onto fewer threads if possible, but once we have
@@ -302,14 +301,8 @@ public void writeFileChunk(
         threadPool.generic()
             .execute(
                 ActionRunnable.wrap(
-                    listener,
-                    l -> executeRetryableAction(
-                        action,
-                        request,
-                        fileChunkRequestOptions,
-                        ActionListener.runBefore(l.map(r -> null), request::decRef),
-                        reader
-                    )
+                    ActionListener.<TransportResponse.Empty>runBefore(listener.map(r -> null), request::decRef),
+                    l -> executeRetryableAction(action, request, fileChunkRequestOptions, l, in -> TransportResponse.Empty.INSTANCE)
                 )
             );
     }
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/StartRecoveryRequest.java b/server/src/main/java/org/elasticsearch/indices/recovery/StartRecoveryRequest.java
index a19388ca2612..a60ae14263a3 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/StartRecoveryRequest.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/StartRecoveryRequest.java
@@ -43,7 +43,7 @@ public StartRecoveryRequest(StreamInput in) throws IOException {
         metadataSnapshot = Store.MetadataSnapshot.readFrom(in);
         primaryRelocation = in.readBoolean();
         startingSeqNo = in.readLong();
-        if (in.getVersion().onOrAfter(RecoverySettings.SNAPSHOT_FILE_DOWNLOAD_THROTTLING_SUPPORTED_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(RecoverySettings.SNAPSHOT_FILE_DOWNLOAD_THROTTLING_SUPPORTED_VERSION)) {
             canDownloadSnapshotFiles = in.readBoolean();
         } else {
             canDownloadSnapshotFiles = true;
@@ -134,7 +134,7 @@ public void writeTo(StreamOutput out) throws IOException {
         metadataSnapshot.writeTo(out);
         out.writeBoolean(primaryRelocation);
         out.writeLong(startingSeqNo);
-        if (out.getVersion().onOrAfter(RecoverySettings.SNAPSHOT_FILE_DOWNLOAD_THROTTLING_SUPPORTED_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(RecoverySettings.SNAPSHOT_FILE_DOWNLOAD_THROTTLING_SUPPORTED_VERSION)) {
             out.writeBoolean(canDownloadSnapshotFiles);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/plan/PeerOnlyRecoveryPlannerService.java b/server/src/main/java/org/elasticsearch/indices/recovery/plan/PeerOnlyRecoveryPlannerService.java
index 9bff81f5b4ff..f24c612ebcc8 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/plan/PeerOnlyRecoveryPlannerService.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/plan/PeerOnlyRecoveryPlannerService.java
@@ -35,6 +35,7 @@ public void computeRecoveryPlan(
         int translogOps,
         Version targetVersion,
         boolean useSnapshots,
+        boolean primaryRelocation,
         ActionListener<ShardRecoveryPlan> listener
     ) {
         ActionListener.completeWith(listener, () -> {
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/plan/RecoveryPlannerService.java b/server/src/main/java/org/elasticsearch/indices/recovery/plan/RecoveryPlannerService.java
index 3318e08b3d7a..25f9a143b053 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/plan/RecoveryPlannerService.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/plan/RecoveryPlannerService.java
@@ -23,6 +23,7 @@ void computeRecoveryPlan(
         int translogOps,
         Version targetVersion,
         boolean useSnapshots,
+        boolean primaryRelocation,
         ActionListener<ShardRecoveryPlan> listener
     );
 }
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/plan/ShardSnapshotsService.java b/server/src/main/java/org/elasticsearch/indices/recovery/plan/ShardSnapshotsService.java
index 762b6af6586b..89a068d6d5aa 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/plan/ShardSnapshotsService.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/plan/ShardSnapshotsService.java
@@ -75,7 +75,7 @@ public ShardSnapshotsService(
     }
 
     public void fetchLatestSnapshotsForShard(ShardId shardId, ActionListener<Optional<ShardSnapshot>> listener) {
-        assert shardId != null : "SharId was null but a value was expected";
+        assert shardId != null : "ShardId was null but a value was expected";
 
         final RepositoriesMetadata currentReposMetadata = clusterService.state()
             .metadata()
@@ -104,24 +104,30 @@ public void fetchLatestSnapshotsForShard(ShardId shardId, ActionListener<Optiona
         client.execute(
             GetShardSnapshotAction.INSTANCE,
             request,
-            new ThreadedActionListener<>(logger, threadPool, ThreadPool.Names.GENERIC, listener.map(this::fetchSnapshotFiles), false)
+            new ThreadedActionListener<>(
+                threadPool.generic(),
+                listener.map(shardSnapshotResponse -> fetchSnapshotFiles(shardId, shardSnapshotResponse))
+            )
         );
     }
 
-    private Optional<ShardSnapshot> fetchSnapshotFiles(GetShardSnapshotResponse shardSnapshotResponse) {
+    private Optional<ShardSnapshot> fetchSnapshotFiles(ShardId shardId, GetShardSnapshotResponse shardSnapshotResponse) {
         assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.GENERIC);
 
         final Optional<ShardSnapshotInfo> latestShardSnapshotOpt = shardSnapshotResponse.getLatestShardSnapshot();
         if (latestShardSnapshotOpt.isEmpty()) {
+            logger.debug("{} no latest shard snapshot found", shardId);
             return Optional.empty();
         }
 
         final ShardSnapshotInfo latestShardSnapshot = latestShardSnapshotOpt.get();
         try {
             final Snapshot snapshot = latestShardSnapshot.getSnapshot();
+            logger.debug("{} considering recovery from [{}][{}]", shardId, snapshot.getRepository(), snapshot.getSnapshotId());
 
             final Repository repository = repositoriesService.repository(snapshot.getRepository());
             if (repository instanceof BlobStoreRepository == false) {
+                logger.debug("{} not recovering from snapshot in non-blobstore repository [{}]", shardId, snapshot.getRepository());
                 return Optional.empty();
             }
 
diff --git a/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java b/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
index 90c1ff9edfe5..4e099fb27f54 100644
--- a/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
+++ b/server/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
@@ -184,6 +184,8 @@ public void clusterChanged(ClusterChangedEvent event) {
     }
 
     static boolean shardCanBeDeleted(String localNodeId, IndexShardRoutingTable indexShardRoutingTable) {
+        assert indexShardRoutingTable.size() > 0;
+
         // a shard can be deleted if all its copies are active, and its not allocated on this node
         if (indexShardRoutingTable.size() == 0) {
             // should not really happen, there should always be at least 1 (primary) shard in a
diff --git a/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetadata.java b/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetadata.java
index 1da1aeae6484..50501c3dbc25 100644
--- a/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetadata.java
+++ b/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetadata.java
@@ -8,8 +8,10 @@
 
 package org.elasticsearch.indices.store;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.ActionFilters;
@@ -58,6 +60,8 @@ public class TransportNodesListShardStoreMetadata extends TransportNodesAction<
     TransportNodesListShardStoreMetadata.NodeRequest,
     TransportNodesListShardStoreMetadata.NodeStoreFilesMetadata> {
 
+    private static final Logger logger = LogManager.getLogger(TransportNodesListShardStoreMetadata.class);
+
     public static final String ACTION_NAME = "internal:cluster/nodes/indices/shard/store";
     public static final ActionType<NodesStoreFilesMetadata> TYPE = new ActionType<>(ACTION_NAME, NodesStoreFilesMetadata::new);
 
@@ -200,7 +204,7 @@ public record StoreFilesMetadata(Store.MetadataSnapshot metadataSnapshot, List<R
         public static final StoreFilesMetadata EMPTY = new StoreFilesMetadata(Store.MetadataSnapshot.EMPTY, emptyList());
 
         public static StoreFilesMetadata readFrom(StreamInput in) throws IOException {
-            if (in.getVersion().before(Version.V_8_2_0)) {
+            if (in.getTransportVersion().before(TransportVersion.V_8_2_0)) {
                 new ShardId(in);
             }
             final var metadataSnapshot = Store.MetadataSnapshot.readFrom(in);
@@ -214,12 +218,12 @@ public static StoreFilesMetadata readFrom(StreamInput in) throws IOException {
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            if (out.getVersion().before(Version.V_8_2_0)) {
+            if (out.getTransportVersion().before(TransportVersion.V_8_2_0)) {
                 // no compatible version cares about the shard ID, we can just make one up
                 FAKE_SHARD_ID.writeTo(out);
 
                 // NB only checked this for versions back to 7.17.0, we are assuming that we don't use this with earlier versions:
-                assert out.getVersion().onOrAfter(Version.V_7_17_0) : out.getVersion();
+                assert out.getTransportVersion().onOrAfter(TransportVersion.V_7_17_0) : out.getTransportVersion();
             }
             metadataSnapshot.writeTo(out);
             out.writeList(peerRecoveryRetentionLeases);
diff --git a/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java b/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java
index 8f1ad6ede6d3..2910ab11e8c9 100644
--- a/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java
+++ b/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java
@@ -8,15 +8,12 @@
 
 package org.elasticsearch.ingest;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.core.Tuple;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.BiConsumer;
 import java.util.function.LongSupplier;
 import java.util.stream.Collectors;
@@ -31,8 +28,6 @@ public class CompoundProcessor implements Processor {
     public static final String ON_FAILURE_PROCESSOR_TAG_FIELD = "on_failure_processor_tag";
     public static final String ON_FAILURE_PIPELINE_FIELD = "on_failure_pipeline";
 
-    private static final Logger logger = LogManager.getLogger(CompoundProcessor.class);
-
     private final boolean ignoreFailure;
     private final List<Processor> processors;
     private final List<Processor> onFailureProcessors;
@@ -40,10 +35,6 @@ public class CompoundProcessor implements Processor {
     private final LongSupplier relativeTimeProvider;
     private final boolean isAsync;
 
-    CompoundProcessor(LongSupplier relativeTimeProvider, boolean ignoreFailure, Processor... processors) {
-        this(ignoreFailure, List.of(processors), List.of(), relativeTimeProvider);
-    }
-
     public CompoundProcessor(Processor... processors) {
         this(false, List.of(processors), List.of());
     }
@@ -58,7 +49,6 @@ public CompoundProcessor(boolean ignoreFailure, List<Processor> processors, List
         List<Processor> onFailureProcessors,
         LongSupplier relativeTimeProvider
     ) {
-        super();
         this.ignoreFailure = ignoreFailure;
         this.processors = List.copyOf(processors);
         this.onFailureProcessors = List.copyOf(onFailureProcessors);
@@ -198,43 +188,24 @@ void innerExecute(int currentProcessor, IngestDocument ingestDocument, final BiC
         final IngestMetric finalMetric = processorsWithMetrics.get(currentProcessor).v2();
         final Processor finalProcessor = processorsWithMetrics.get(currentProcessor).v1();
         final IngestDocument finalIngestDocument = ingestDocument;
-        /*
-         * Our assumption is that the listener passed to the processor is only ever called once. However, there is no way to enforce
-         * that in all processors and all the code that they call. If the listener is called more than once it causes problems
-         * such as the metrics being wrong. The listenerHasBeenCalled variable is used to make sure that the code in the listener
-         * is only executed once.
-         */
-        final AtomicBoolean listenerHasBeenCalled = new AtomicBoolean(false);
         finalMetric.preIngest();
-        final AtomicBoolean postIngestHasBeenCalled = new AtomicBoolean(false);
         try {
             finalProcessor.execute(ingestDocument, (result, e) -> {
-                if (listenerHasBeenCalled.getAndSet(true)) {
-                    logger.warn("A listener was unexpectedly called more than once", new RuntimeException(e));
-                    assert false : "A listener was unexpectedly called more than once";
+                long ingestTimeInNanos = relativeTimeProvider.getAsLong() - startTimeInNanos;
+                finalMetric.postIngest(ingestTimeInNanos);
+                if (e != null) {
+                    executeOnFailureOuter(finalCurrentProcessor, finalIngestDocument, handler, finalProcessor, finalMetric, e);
                 } else {
-                    long ingestTimeInNanos = relativeTimeProvider.getAsLong() - startTimeInNanos;
-                    finalMetric.postIngest(ingestTimeInNanos);
-                    postIngestHasBeenCalled.set(true);
-                    if (e != null) {
-                        executeOnFailureOuter(finalCurrentProcessor, finalIngestDocument, handler, finalProcessor, finalMetric, e);
+                    if (result != null) {
+                        innerExecute(nextProcessor, result, handler);
                     } else {
-                        if (result != null) {
-                            innerExecute(nextProcessor, result, handler);
-                        } else {
-                            handler.accept(null, null);
-                        }
+                        handler.accept(null, null);
                     }
                 }
             });
         } catch (Exception e) {
             long ingestTimeInNanos = relativeTimeProvider.getAsLong() - startTimeInNanos;
-            if (postIngestHasBeenCalled.get()) {
-                logger.warn("Preventing postIngest from being called more than once", e);
-                assert false : "Attempt to call postIngest more than once";
-            } else {
-                finalMetric.postIngest(ingestTimeInNanos);
-            }
+            finalMetric.postIngest(ingestTimeInNanos);
             executeOnFailureOuter(finalCurrentProcessor, finalIngestDocument, handler, finalProcessor, finalMetric, e);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/ingest/ConditionalProcessor.java b/server/src/main/java/org/elasticsearch/ingest/ConditionalProcessor.java
index 597307d433d6..1fa0c0e78e4d 100644
--- a/server/src/main/java/org/elasticsearch/ingest/ConditionalProcessor.java
+++ b/server/src/main/java/org/elasticsearch/ingest/ConditionalProcessor.java
@@ -8,8 +8,6 @@
 
 package org.elasticsearch.ingest;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.script.DynamicMap;
@@ -28,7 +26,6 @@
 import java.util.ListIterator;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.BiConsumer;
 import java.util.function.Function;
 import java.util.function.LongSupplier;
@@ -36,6 +33,9 @@
 
 import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
 
+/**
+ * A wrapping processor that adds 'if' logic around the wrapped processor.
+ */
 public class ConditionalProcessor extends AbstractProcessor implements WrappingProcessor {
 
     private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(DynamicMap.class);
@@ -48,8 +48,6 @@ public class ConditionalProcessor extends AbstractProcessor implements WrappingP
         return value;
     });
 
-    private static final Logger logger = LogManager.getLogger(ConditionalProcessor.class);
-
     static final String TYPE = "conditional";
 
     private final Script condition;
@@ -125,27 +123,15 @@ public void execute(IngestDocument ingestDocument, BiConsumer<IngestDocument, Ex
 
         if (matches) {
             final long startTimeInNanos = relativeTimeProvider.getAsLong();
-            /*
-             * Our assumption is that the listener passed to the processor is only ever called once. However, there is no way to enforce
-             * that in all processors and all of the code that they call. If the listener is called more than once it causes problems
-             * such as the metrics being wrong. The listenerHasBeenCalled variable is used to make sure that the code in the listener
-             * is only executed once.
-             */
-            final AtomicBoolean listenerHasBeenCalled = new AtomicBoolean(false);
             metric.preIngest();
             processor.execute(ingestDocument, (result, e) -> {
-                if (listenerHasBeenCalled.getAndSet(true)) {
-                    logger.warn("A listener was unexpectedly called more than once", new RuntimeException(e));
-                    assert false : "A listener was unexpectedly called more than once";
+                long ingestTimeInNanos = relativeTimeProvider.getAsLong() - startTimeInNanos;
+                metric.postIngest(ingestTimeInNanos);
+                if (e != null) {
+                    metric.ingestFailed();
+                    handler.accept(null, e);
                 } else {
-                    long ingestTimeInNanos = relativeTimeProvider.getAsLong() - startTimeInNanos;
-                    metric.postIngest(ingestTimeInNanos);
-                    if (e != null) {
-                        metric.ingestFailed();
-                        handler.accept(null, e);
-                    } else {
-                        handler.accept(result, null);
-                    }
+                    handler.accept(result, null);
                 }
             });
         } else {
diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java
index 4d146eb96622..d1c2e392e273 100644
--- a/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java
+++ b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java
@@ -22,7 +22,7 @@
  * _index, _id and _routing must be a String or null
  * _version_type must be a lower case VersionType or null
  * _version must be representable as a long without loss of precision or null
- * _dyanmic_templates must be a map
+ * _dynamic_templates must be a map
  * _if_seq_no must be a long or null
  * _if_primary_term must be a long or null
  *
diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java
index dd5a5607c7f6..f471926087ae 100644
--- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java
+++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java
@@ -9,7 +9,6 @@
 package org.elasticsearch.ingest;
 
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.util.LazyMap;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.VersionType;
@@ -26,6 +25,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Base64;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
@@ -43,15 +43,23 @@
 public final class IngestDocument {
 
     public static final String INGEST_KEY = "_ingest";
-    public static final String PIPELINE_CYCLE_ERROR_MESSAGE = "Cycle detected for pipeline: ";
+    public static final String SOURCE_KEY = SourceFieldMapper.NAME; // "_source"
     private static final String INGEST_KEY_PREFIX = INGEST_KEY + ".";
-    private static final String SOURCE_PREFIX = SourceFieldMapper.NAME + ".";
+    private static final String SOURCE_PREFIX = SOURCE_KEY + ".";
 
+    public static final String PIPELINE_CYCLE_ERROR_MESSAGE = "Cycle detected for pipeline: ";
     static final String TIMESTAMP = "timestamp";
 
     private final IngestCtxMap ctxMap;
     private final Map<String, Object> ingestMetadata;
 
+    /**
+     * Shallowly read-only, very limited, map-like view of the ctxMap and ingestMetadata,
+     * for providing as a model to TemplateScript and ValueSource instances. This avoids the cost of
+     * constructing a purpose-built map on each template evaluation.
+     */
+    private final DelegatingMapView templateModel;
+
     // Contains all pipelines that have been executed for this document
     private final Set<String> executedPipelines = new LinkedHashSet<>();
 
@@ -61,6 +69,7 @@ public IngestDocument(String index, String id, long version, String routing, Ver
         this.ctxMap = new IngestCtxMap(index, id, version, routing, versionType, ZonedDateTime.now(ZoneOffset.UTC), source);
         this.ingestMetadata = new HashMap<>();
         this.ingestMetadata.put(TIMESTAMP, ctxMap.getMetadata().getNow());
+        this.templateModel = initializeTemplateModel();
     }
 
     /**
@@ -91,16 +100,22 @@ public IngestDocument(Map<String, Object> sourceAndMetadata, Map<String, Object>
                 }
             }
         }
-        this.ingestMetadata = new HashMap<>(ingestMetadata);
         this.ctxMap = new IngestCtxMap(source, new IngestDocMetadata(metadata, IngestCtxMap.getTimestamp(ingestMetadata)));
+        this.ingestMetadata = new HashMap<>(ingestMetadata);
+        this.templateModel = initializeTemplateModel();
     }
 
     /**
      * Constructor to create an IngestDocument from its constituent maps
      */
     IngestDocument(IngestCtxMap ctxMap, Map<String, Object> ingestMetadata) {
-        this.ctxMap = ctxMap;
-        this.ingestMetadata = ingestMetadata;
+        this.ctxMap = Objects.requireNonNull(ctxMap);
+        this.ingestMetadata = Objects.requireNonNull(ingestMetadata);
+        this.templateModel = initializeTemplateModel();
+    }
+
+    private DelegatingMapView initializeTemplateModel() {
+        return new DelegatingMapView(ctxMap, Map.of(SOURCE_KEY, ctxMap, INGEST_KEY, ingestMetadata));
     }
 
     /**
@@ -425,8 +440,7 @@ public void appendFieldValue(String path, Object value, boolean allowDuplicates)
      * @throws IllegalArgumentException if the path is null, empty or invalid.
      */
     public void appendFieldValue(TemplateScript.Factory fieldPathTemplate, ValueSource valueSource) {
-        Map<String, Object> model = createTemplateModel();
-        appendFieldValue(fieldPathTemplate.newInstance(model).execute(), valueSource.copyAndResolve(model));
+        appendFieldValue(fieldPathTemplate.newInstance(templateModel).execute(), valueSource.copyAndResolve(templateModel));
     }
 
     /**
@@ -443,8 +457,11 @@ public void appendFieldValue(TemplateScript.Factory fieldPathTemplate, ValueSour
      * @throws IllegalArgumentException if the path is null, empty or invalid.
      */
     public void appendFieldValue(TemplateScript.Factory fieldPathTemplate, ValueSource valueSource, boolean allowDuplicates) {
-        Map<String, Object> model = createTemplateModel();
-        appendFieldValue(fieldPathTemplate.newInstance(model).execute(), valueSource.copyAndResolve(model), allowDuplicates);
+        appendFieldValue(
+            fieldPathTemplate.newInstance(templateModel).execute(),
+            valueSource.copyAndResolve(templateModel),
+            allowDuplicates
+        );
     }
 
     /**
@@ -471,8 +488,7 @@ public void setFieldValue(String path, Object value) {
      * item identified by the provided path.
      */
     public void setFieldValue(TemplateScript.Factory fieldPathTemplate, ValueSource valueSource) {
-        Map<String, Object> model = createTemplateModel();
-        setFieldValue(fieldPathTemplate.newInstance(model).execute(), valueSource.copyAndResolve(model), false);
+        setFieldValue(fieldPathTemplate.newInstance(templateModel).execute(), valueSource.copyAndResolve(templateModel), false);
     }
 
     /**
@@ -486,8 +502,7 @@ public void setFieldValue(TemplateScript.Factory fieldPathTemplate, ValueSource
      * item identified by the provided path.
      */
     public void setFieldValue(TemplateScript.Factory fieldPathTemplate, ValueSource valueSource, boolean ignoreEmptyValue) {
-        Map<String, Object> model = createTemplateModel();
-        Object value = valueSource.copyAndResolve(model);
+        Object value = valueSource.copyAndResolve(templateModel);
         if (ignoreEmptyValue && valueSource instanceof ValueSource.TemplatedValue) {
             if (value == null) {
                 return;
@@ -498,7 +513,7 @@ public void setFieldValue(TemplateScript.Factory fieldPathTemplate, ValueSource
             }
         }
 
-        setFieldValue(fieldPathTemplate.newInstance(model).execute(), value, false);
+        setFieldValue(fieldPathTemplate.newInstance(templateModel).execute(), value, false);
     }
 
     /**
@@ -512,7 +527,6 @@ public void setFieldValue(TemplateScript.Factory fieldPathTemplate, ValueSource
      * item identified by the provided path.
      */
     public void setFieldValue(TemplateScript.Factory fieldPathTemplate, Object value, boolean ignoreEmptyValue) {
-        Map<String, Object> model = createTemplateModel();
         if (ignoreEmptyValue) {
             if (value == null) {
                 return;
@@ -524,7 +538,7 @@ public void setFieldValue(TemplateScript.Factory fieldPathTemplate, Object value
             }
         }
 
-        setFieldValue(fieldPathTemplate.newInstance(model).execute(), value, false);
+        setFieldValue(fieldPathTemplate.newInstance(templateModel).execute(), value, false);
     }
 
     private void setFieldValue(String path, Object value, boolean append) {
@@ -698,18 +712,7 @@ private static <T> T cast(String path, Object object, Class<T> clazz) {
     }
 
     public String renderTemplate(TemplateScript.Factory template) {
-        return template.newInstance(createTemplateModel()).execute();
-    }
-
-    private Map<String, Object> createTemplateModel() {
-        return new LazyMap<>(() -> {
-            Map<String, Object> model = new HashMap<>(ctxMap);
-            model.put(SourceFieldMapper.NAME, ctxMap);
-            // If there is a field in the source with the name '_ingest' it gets overwritten here,
-            // if access to that field is required then it get accessed via '_source._ingest'
-            model.put(INGEST_KEY, ingestMetadata);
-            return model;
-        });
+        return template.newInstance(templateModel).execute();
     }
 
     /**
@@ -826,6 +829,12 @@ private static Set<String> getAllFields(Map<String, Object> input, String prefix
      * @param handler handles the result or failure
      */
     public void executePipeline(Pipeline pipeline, BiConsumer<IngestDocument, Exception> handler) {
+        // shortcut if the pipeline is empty
+        if (pipeline.getProcessors().isEmpty()) {
+            handler.accept(this, null);
+            return;
+        }
+
         if (executedPipelines.add(pipeline.getId())) {
             Object previousPipeline = ingestMetadata.put("pipeline", pipeline.getId());
             pipeline.execute(this, (result, e) -> {
@@ -973,4 +982,82 @@ static ResolveResult error(String errorMessage) {
 
         }
     }
+
+    /**
+     * Provides a shallowly read-only, very limited, map-like view of two maps. The only methods that are implemented are
+     * {@link Map#get(Object)} and {@link Map#containsKey(Object)}, everything else throws UnsupportedOperationException.
+     *
+     * The overrides map has higher priority than the primary map -- values in that map under some key will take priority over values
+     * in the primary map under the same key.
+     *
+     * @param primary the primary map
+     * @param overrides the overrides map
+     */
+    private record DelegatingMapView(Map<String, Object> primary, Map<String, Object> overrides) implements Map<String, Object> {
+
+        @Override
+        public boolean containsKey(Object key) {
+            // most normal uses of this in practice will end up passing in keys that match the primary, rather than the overrides,
+            // in which case we can shortcut by checking the primary first
+            return primary.containsKey(key) || overrides.containsKey(key);
+        }
+
+        @Override
+        public Object get(Object key) {
+            // null values in the overrides map are treated as *key not present*, so we don't have to do a containsKey check here --
+            // if the overrides map returns null we can simply delegate to the primary
+            Object result = overrides.get(key);
+            return result != null ? result : primary.get(key);
+        }
+
+        @Override
+        public int size() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public boolean isEmpty() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public boolean containsValue(Object value) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public Object put(String key, Object value) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public Object remove(Object key) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void putAll(Map<? extends String, ?> m) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void clear() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public Set<String> keySet() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public Collection<Object> values() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public Set<Entry<String, Object>> entrySet() {
+            throw new UnsupportedOperationException();
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestMetadata.java b/server/src/main/java/org/elasticsearch/ingest/IngestMetadata.java
index 401301e9f0e6..29c2808c1a59 100644
--- a/server/src/main/java/org/elasticsearch/ingest/IngestMetadata.java
+++ b/server/src/main/java/org/elasticsearch/ingest/IngestMetadata.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.ingest;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.DiffableUtils;
 import org.elasticsearch.cluster.NamedDiff;
@@ -16,9 +16,10 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -26,6 +27,7 @@
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
@@ -59,8 +61,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.CURRENT.minimumCompatibilityVersion();
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.CURRENT.minimumCompatibilityVersion();
     }
 
     public Map<String, PipelineConfiguration> getPipelines() {
@@ -95,13 +97,8 @@ public static IngestMetadata fromXContent(XContentParser parser) throws IOExcept
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startArray(PIPELINES_FIELD.getPreferredName());
-        for (PipelineConfiguration pipeline : pipelines.values()) {
-            pipeline.toXContent(builder, params);
-        }
-        builder.endArray();
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return ChunkedToXContentHelper.array(PIPELINES_FIELD.getPreferredName(), pipelines.values().iterator());
     }
 
     @Override
@@ -151,8 +148,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT.minimumCompatibilityVersion();
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT.minimumCompatibilityVersion();
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java
index 11ce150cb923..12018ce90651 100644
--- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java
+++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.ingest.DeletePipelineRequest;
 import org.elasticsearch.action.ingest.PutPipelineRequest;
+import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterChangedEvent;
@@ -43,8 +44,10 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.env.Environment;
@@ -72,8 +75,6 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.IntConsumer;
@@ -212,7 +213,7 @@ public static boolean resolvePipelines(
             String defaultPipeline = null;
             String finalPipeline = null;
             IndexMetadata indexMetadata = null;
-            // start to look for default or final pipelines via settings found in the index meta data
+            // start to look for default or final pipelines via settings found in the index metadata
             if (originalRequest != null) {
                 indexMetadata = metadata.indices()
                     .get(IndexNameExpressionResolver.resolveDateMathExpression(originalRequest.index(), epochMillis));
@@ -252,11 +253,9 @@ public static boolean resolvePipelines(
                     Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template);
                     if (IndexSettings.DEFAULT_PIPELINE.exists(settings)) {
                         defaultPipeline = IndexSettings.DEFAULT_PIPELINE.get(settings);
-                        // we can not break in case a lower-order template has a final pipeline that we need to collect
                     }
                     if (IndexSettings.FINAL_PIPELINE.exists(settings)) {
                         finalPipeline = IndexSettings.FINAL_PIPELINE.get(settings);
-                        // we can not break in case a lower-order template has a default pipeline that we need to collect
                     }
                     indexRequest.setPipeline(Objects.requireNonNullElse(defaultPipeline, NOOP_PIPELINE_NAME));
                     indexRequest.setFinalPipeline(Objects.requireNonNullElse(finalPipeline, NOOP_PIPELINE_NAME));
@@ -669,13 +668,14 @@ void validatePipeline(Map<DiscoveryNode, IngestInfo> ingestInfos, String pipelin
     }
 
     public void executeBulkRequest(
-        int numberOfActionRequests,
-        Iterable<DocWriteRequest<?>> actionRequests,
-        BiConsumer<Integer, Exception> onFailure,
-        BiConsumer<Thread, Exception> onCompletion,
-        IntConsumer onDropped,
-        String executorName
+        final int numberOfActionRequests,
+        final Iterable<DocWriteRequest<?>> actionRequests,
+        final IntConsumer onDropped,
+        final BiConsumer<Integer, Exception> onFailure,
+        final BiConsumer<Thread, Exception> onCompletion,
+        final String executorName
     ) {
+        assert numberOfActionRequests > 0 : "numberOfActionRequests must be greater than 0 but was [" + numberOfActionRequests + "]";
 
         threadPool.executor(executorName).execute(new AbstractRunnable() {
 
@@ -687,81 +687,92 @@ public void onFailure(Exception e) {
             @Override
             protected void doRun() {
                 final Thread originalThread = Thread.currentThread();
-                final AtomicInteger counter = new AtomicInteger(numberOfActionRequests);
-                int i = 0;
-                for (DocWriteRequest<?> actionRequest : actionRequests) {
-                    IndexRequest indexRequest = TransportBulkAction.getIndexWriteRequest(actionRequest);
-                    if (indexRequest == null) {
-                        if (counter.decrementAndGet() == 0) {
-                            onCompletion.accept(originalThread, null);
+                try (var refs = new RefCountingRunnable(() -> onCompletion.accept(originalThread, null))) {
+                    int i = 0;
+                    for (DocWriteRequest<?> actionRequest : actionRequests) {
+                        IndexRequest indexRequest = TransportBulkAction.getIndexWriteRequest(actionRequest);
+                        if (indexRequest == null) {
+                            i++;
+                            continue;
                         }
-                        assert counter.get() >= 0;
-                        i++;
-                        continue;
-                    }
 
-                    final String pipelineId = indexRequest.getPipeline();
-                    indexRequest.setPipeline(NOOP_PIPELINE_NAME);
-                    final String finalPipelineId = indexRequest.getFinalPipeline();
-                    indexRequest.setFinalPipeline(NOOP_PIPELINE_NAME);
-                    boolean hasFinalPipeline = true;
-                    final List<String> pipelines;
-                    if (IngestService.NOOP_PIPELINE_NAME.equals(pipelineId) == false
-                        && IngestService.NOOP_PIPELINE_NAME.equals(finalPipelineId) == false) {
-                        pipelines = List.of(pipelineId, finalPipelineId);
-                    } else if (IngestService.NOOP_PIPELINE_NAME.equals(pipelineId) == false) {
-                        pipelines = List.of(pipelineId);
-                        hasFinalPipeline = false;
-                    } else if (IngestService.NOOP_PIPELINE_NAME.equals(finalPipelineId) == false) {
-                        pipelines = List.of(finalPipelineId);
-                    } else {
-                        if (counter.decrementAndGet() == 0) {
-                            onCompletion.accept(originalThread, null);
+                        final String pipelineId = indexRequest.getPipeline();
+                        indexRequest.setPipeline(NOOP_PIPELINE_NAME);
+                        final String finalPipelineId = indexRequest.getFinalPipeline();
+                        indexRequest.setFinalPipeline(NOOP_PIPELINE_NAME);
+                        boolean hasFinalPipeline = true;
+                        final List<String> pipelines;
+                        if (IngestService.NOOP_PIPELINE_NAME.equals(pipelineId) == false
+                            && IngestService.NOOP_PIPELINE_NAME.equals(finalPipelineId) == false) {
+                            pipelines = List.of(pipelineId, finalPipelineId);
+                        } else if (IngestService.NOOP_PIPELINE_NAME.equals(pipelineId) == false) {
+                            pipelines = List.of(pipelineId);
+                            hasFinalPipeline = false;
+                        } else if (IngestService.NOOP_PIPELINE_NAME.equals(finalPipelineId) == false) {
+                            pipelines = List.of(finalPipelineId);
+                        } else {
+                            i++;
+                            continue;
                         }
-                        assert counter.get() >= 0;
-                        i++;
-                        continue;
-                    }
 
-                    executePipelines(
-                        i,
-                        pipelines.iterator(),
-                        hasFinalPipeline,
-                        indexRequest,
-                        onDropped,
-                        onFailure,
-                        counter,
-                        onCompletion,
-                        originalThread
-                    );
+                        // start the stopwatch and acquire a ref to indicate that we're working on this document
+                        final long startTimeInNanos = System.nanoTime();
+                        totalMetrics.preIngest();
+                        final int slot = i;
+                        final Releasable ref = refs.acquire();
+                        // the document listener gives us three-way logic: a document can fail processing (1), or it can
+                        // be successfully processed. a successfully processed document can be kept (2) or dropped (3).
+                        final ActionListener<Boolean> documentListener = ActionListener.runAfter(new ActionListener<>() {
+                            @Override
+                            public void onResponse(Boolean kept) {
+                                assert kept != null;
+                                if (kept == false) {
+                                    onDropped.accept(slot);
+                                }
+                            }
+
+                            @Override
+                            public void onFailure(Exception e) {
+                                totalMetrics.ingestFailed();
+                                onFailure.accept(slot, e);
+                            }
+                        }, () -> {
+                            // regardless of success or failure, we always stop the ingest "stopwatch" and release the ref to indicate
+                            // that we're finished with this document
+                            final long ingestTimeInNanos = System.nanoTime() - startTimeInNanos;
+                            totalMetrics.postIngest(ingestTimeInNanos);
+                            ref.close();
+                        });
 
-                    i++;
+                        IngestDocument ingestDocument = newIngestDocument(indexRequest);
+                        executePipelines(pipelines.iterator(), hasFinalPipeline, indexRequest, ingestDocument, documentListener);
+
+                        i++;
+                    }
                 }
             }
         });
     }
 
     private void executePipelines(
-        final int slot,
-        final Iterator<String> it,
+        final Iterator<String> pipelineIds,
         final boolean hasFinalPipeline,
         final IndexRequest indexRequest,
-        final IntConsumer onDropped,
-        final BiConsumer<Integer, Exception> onFailure,
-        final AtomicInteger counter,
-        final BiConsumer<Thread, Exception> onCompletion,
-        final Thread originalThread
+        final IngestDocument ingestDocument,
+        final ActionListener<Boolean> listener
     ) {
-        assert it.hasNext();
-        final String pipelineId = it.next();
+        assert pipelineIds.hasNext();
+        final String pipelineId = pipelineIds.next();
         try {
-            PipelineHolder holder = pipelines.get(pipelineId);
+            final PipelineHolder holder = pipelines.get(pipelineId);
             if (holder == null) {
                 throw new IllegalArgumentException("pipeline with id [" + pipelineId + "] does not exist");
             }
-            Pipeline pipeline = holder.pipeline;
-            String originalIndex = indexRequest.indices()[0];
-            innerExecute(slot, indexRequest, pipeline, onDropped, e -> {
+            final Pipeline pipeline = holder.pipeline;
+            final String originalIndex = indexRequest.indices()[0];
+            executePipeline(ingestDocument, pipeline, (keep, e) -> {
+                assert keep != null;
+
                 if (e != null) {
                     logger.debug(
                         () -> format(
@@ -772,49 +783,77 @@ private void executePipelines(
                         ),
                         e
                     );
-                    onFailure.accept(slot, e);
+                    listener.onFailure(e);
+                    return; // document failed!
+                }
+
+                if (keep == false) {
+                    listener.onResponse(false);
+                    return; // document dropped!
+                }
+
+                // update the index request so that we can execute additional pipelines (if any), etc
+                updateIndexRequestMetadata(indexRequest, ingestDocument.getMetadata());
+                try {
+                    // check for self-references if necessary, (i.e. if a script processor has run), and clear the bit
+                    if (ingestDocument.doNoSelfReferencesCheck()) {
+                        CollectionUtils.ensureNoSelfReferences(ingestDocument.getSource(), null);
+                        ingestDocument.doNoSelfReferencesCheck(false);
+                    }
+                } catch (IllegalArgumentException ex) {
+                    // An IllegalArgumentException can be thrown when an ingest processor creates a source map that is self-referencing.
+                    // In that case, we catch and wrap the exception, so we can include more details
+                    listener.onFailure(
+                        new IllegalArgumentException(
+                            format(
+                                "Failed to generate the source document for ingest pipeline [%s] for document [%s/%s]",
+                                pipelineId,
+                                indexRequest.index(),
+                                indexRequest.id()
+                            ),
+                            ex
+                        )
+                    );
+                    return; // document failed!
                 }
 
-                Iterator<String> newIt = it;
+                Iterator<String> newPipelineIds = pipelineIds;
                 boolean newHasFinalPipeline = hasFinalPipeline;
-                String newIndex = indexRequest.indices()[0];
+                final String newIndex = indexRequest.indices()[0];
 
                 if (Objects.equals(originalIndex, newIndex) == false) {
-                    if (hasFinalPipeline && it.hasNext() == false) {
-                        totalMetrics.ingestFailed();
-                        onFailure.accept(
-                            slot,
-                            new IllegalStateException("final pipeline [" + pipelineId + "] can't change the target index")
+                    if (hasFinalPipeline && pipelineIds.hasNext() == false) {
+                        listener.onFailure(
+                            new IllegalStateException(
+                                format(
+                                    "final pipeline [%s] can't change the target index (from [%s] to [%s]) for document [%s]",
+                                    pipelineId,
+                                    originalIndex,
+                                    newIndex,
+                                    indexRequest.id()
+                                )
+                            )
                         );
+                        return; // document failed!
                     } else {
                         indexRequest.isPipelineResolved(false);
                         resolvePipelines(null, indexRequest, state.metadata());
                         if (IngestService.NOOP_PIPELINE_NAME.equals(indexRequest.getFinalPipeline()) == false) {
-                            newIt = Collections.singleton(indexRequest.getFinalPipeline()).iterator();
+                            newPipelineIds = Collections.singleton(indexRequest.getFinalPipeline()).iterator();
                             newHasFinalPipeline = true;
                         } else {
-                            newIt = Collections.emptyIterator();
+                            newPipelineIds = Collections.emptyIterator();
                         }
                     }
                 }
 
-                if (newIt.hasNext()) {
-                    executePipelines(
-                        slot,
-                        newIt,
-                        newHasFinalPipeline,
-                        indexRequest,
-                        onDropped,
-                        onFailure,
-                        counter,
-                        onCompletion,
-                        originalThread
-                    );
+                if (newPipelineIds.hasNext()) {
+                    executePipelines(newPipelineIds, newHasFinalPipeline, indexRequest, ingestDocument, listener);
                 } else {
-                    if (counter.decrementAndGet() == 0) {
-                        onCompletion.accept(originalThread, null);
-                    }
-                    assert counter.get() >= 0;
+                    // update the index request's source and (potentially) cache the timestamp for TSDB
+                    updateIndexRequestSource(indexRequest, ingestDocument);
+                    cacheRawTimestamp(indexRequest, ingestDocument);
+                    listener.onResponse(true); // document succeeded!
                 }
             });
         } catch (Exception e) {
@@ -822,14 +861,27 @@ private void executePipelines(
                 () -> format("failed to execute pipeline [%s] for document [%s/%s]", pipelineId, indexRequest.index(), indexRequest.id()),
                 e
             );
-            onFailure.accept(slot, e);
-            if (counter.decrementAndGet() == 0) {
-                onCompletion.accept(originalThread, null);
-            }
-            assert counter.get() >= 0;
+            listener.onFailure(e); // document failed!
         }
     }
 
+    private void executePipeline(
+        final IngestDocument ingestDocument,
+        final Pipeline pipeline,
+        final BiConsumer<Boolean, Exception> handler
+    ) {
+        // adapt our {@code BiConsumer<Boolean, Exception>} handler shape to the
+        // {@code BiConsumer<IngestDocument, Exception>} handler shape used internally
+        // by ingest pipelines and processors
+        ingestDocument.executePipeline(pipeline, (result, e) -> {
+            if (e != null) {
+                handler.accept(true, e);
+            } else {
+                handler.accept(result != null, null);
+            }
+        });
+    }
+
     public IngestStats stats() {
         IngestStats.Builder statsBuilder = new IngestStats.Builder();
         statsBuilder.addTotalMetrics(totalMetrics);
@@ -851,7 +903,7 @@ public IngestStats stats() {
     /**
      * Adds a listener that gets invoked with the current cluster state before processor factories
      * get invoked.
-     *
+     * <p>
      * This is useful for components that are used by ingest processors, so that they have the opportunity to update
      * before these components get used by the ingest processor factory.
      */
@@ -881,104 +933,70 @@ static String getProcessorName(Processor processor) {
         return sb.toString();
     }
 
-    private void innerExecute(
-        int slot,
-        IndexRequest indexRequest,
-        Pipeline pipeline,
-        IntConsumer itemDroppedHandler,
-        Consumer<Exception> handler
-    ) {
-        if (pipeline.getProcessors().isEmpty()) {
-            handler.accept(null);
-            return;
-        }
+    /**
+     * Builds a new ingest document from the passed-in index request.
+     */
+    private static IngestDocument newIngestDocument(final IndexRequest request) {
+        return new IngestDocument(
+            request.index(),
+            request.id(),
+            request.version(),
+            request.routing(),
+            request.versionType(),
+            request.sourceAsMap()
+        );
+    }
 
-        long startTimeInNanos = System.nanoTime();
-        // the pipeline specific stat holder may not exist and that is fine:
-        // (e.g. the pipeline may have been removed while we're ingesting a document
-        totalMetrics.preIngest();
-        String index = indexRequest.index();
-        String id = indexRequest.id();
-        String routing = indexRequest.routing();
-        long version = indexRequest.version();
-        VersionType versionType = indexRequest.versionType();
-        Map<String, Object> sourceAsMap = indexRequest.sourceAsMap();
-        IngestDocument ingestDocument = new IngestDocument(index, id, version, routing, versionType, sourceAsMap);
-        /*
-         * Our assumption is that the listener passed to the processor is only ever called once. However, there is no way to enforce
-         * that in all processors and all of the code that they call. If the listener is called more than once it causes problems
-         * such as the metrics being wrong. The listenerHasBeenCalled variable is used to make sure that the code in the listener
-         * is only executed once.
-         */
-        final AtomicBoolean listenerHasBeenCalled = new AtomicBoolean(false);
-        ingestDocument.executePipeline(pipeline, (result, e) -> {
-            if (listenerHasBeenCalled.getAndSet(true)) {
-                logger.warn("A listener was unexpectedly called more than once", new RuntimeException(e));
-                assert false : "A listener was unexpectedly called more than once";
-            } else {
-                long ingestTimeInNanos = System.nanoTime() - startTimeInNanos;
-                totalMetrics.postIngest(ingestTimeInNanos);
-                if (e != null) {
-                    totalMetrics.ingestFailed();
-                    handler.accept(e);
-                } else if (result == null) {
-                    itemDroppedHandler.accept(slot);
-                    handler.accept(null);
-                } else {
-                    org.elasticsearch.script.Metadata metadata = ingestDocument.getMetadata();
-
-                    // it's fine to set all metadata fields all the time, as ingest document holds their starting values
-                    // before ingestion, which might also get modified during ingestion.
-                    indexRequest.index(metadata.getIndex());
-                    indexRequest.id(metadata.getId());
-                    indexRequest.routing(metadata.getRouting());
-                    indexRequest.version(metadata.getVersion());
-                    if (metadata.getVersionType() != null) {
-                        indexRequest.versionType(VersionType.fromString(metadata.getVersionType()));
-                    }
-                    Number number;
-                    if ((number = metadata.getIfSeqNo()) != null) {
-                        indexRequest.setIfSeqNo(number.longValue());
-                    }
-                    if ((number = metadata.getIfPrimaryTerm()) != null) {
-                        indexRequest.setIfPrimaryTerm(number.longValue());
-                    }
-                    try {
-                        boolean ensureNoSelfReferences = ingestDocument.doNoSelfReferencesCheck();
-                        indexRequest.source(ingestDocument.getSource(), indexRequest.getContentType(), ensureNoSelfReferences);
-                    } catch (IllegalArgumentException ex) {
-                        // An IllegalArgumentException can be thrown when an ingest
-                        // processor creates a source map that is self-referencing.
-                        // In that case, we catch and wrap the exception so we can
-                        // include which pipeline failed.
-                        totalMetrics.ingestFailed();
-                        handler.accept(
-                            new IllegalArgumentException(
-                                "Failed to generate the source document for ingest pipeline [" + pipeline.getId() + "]",
-                                ex
-                            )
-                        );
-                        return;
-                    }
-                    Map<String, String> map;
-                    if ((map = metadata.getDynamicTemplates()) != null) {
-                        Map<String, String> mergedDynamicTemplates = new HashMap<>(indexRequest.getDynamicTemplates());
-                        mergedDynamicTemplates.putAll(map);
-                        indexRequest.setDynamicTemplates(mergedDynamicTemplates);
-                    }
-                    postIngest(ingestDocument, indexRequest);
+    /**
+     * Updates an index request based on the metadata of an ingest document.
+     */
+    private static void updateIndexRequestMetadata(final IndexRequest request, final org.elasticsearch.script.Metadata metadata) {
+        // it's fine to set all metadata fields all the time, as ingest document holds their starting values
+        // before ingestion, which might also get modified during ingestion.
+        request.index(metadata.getIndex());
+        request.id(metadata.getId());
+        request.routing(metadata.getRouting());
+        request.version(metadata.getVersion());
+        if (metadata.getVersionType() != null) {
+            request.versionType(VersionType.fromString(metadata.getVersionType()));
+        }
+        Number number;
+        if ((number = metadata.getIfSeqNo()) != null) {
+            request.setIfSeqNo(number.longValue());
+        }
+        if ((number = metadata.getIfPrimaryTerm()) != null) {
+            request.setIfPrimaryTerm(number.longValue());
+        }
+        Map<String, String> map;
+        if ((map = metadata.getDynamicTemplates()) != null) {
+            Map<String, String> mergedDynamicTemplates = new HashMap<>(request.getDynamicTemplates());
+            mergedDynamicTemplates.putAll(map);
+            request.setDynamicTemplates(mergedDynamicTemplates);
+        }
+    }
 
-                    handler.accept(null);
-                }
-            }
-        });
+    /**
+     * Updates an index request based on the source of an ingest document, guarding against self-references if necessary.
+     */
+    private static void updateIndexRequestSource(final IndexRequest request, final IngestDocument document) {
+        boolean ensureNoSelfReferences = document.doNoSelfReferencesCheck();
+        // we already check for self references elsewhere (and clear the bit), so this should always be false,
+        // keeping the check and assert as a guard against extraordinarily surprising circumstances
+        assert ensureNoSelfReferences == false;
+        request.source(document.getSource(), request.getContentType(), ensureNoSelfReferences);
     }
 
-    private void postIngest(IngestDocument ingestDocument, IndexRequest indexRequest) {
-        // cache timestamp from ingest source map
-        Object rawTimestamp = ingestDocument.getSourceAndMetadata().get(TimestampField.FIXED_TIMESTAMP_FIELD);
-        if (rawTimestamp != null && indexRequest.getRawTimestamp() == null) {
-            indexRequest.setRawTimestamp(rawTimestamp);
+    /**
+     * Grab the @timestamp and store it on the index request so that TSDB can use it without needing to parse
+     * the source for this document.
+     */
+    private static void cacheRawTimestamp(final IndexRequest request, final IngestDocument document) {
+        if (request.getRawTimestamp() == null) {
+            // cache the @timestamp from the ingest document's source map if there is one
+            Object rawTimestamp = document.getSource().get(TimestampField.FIXED_TIMESTAMP_FIELD);
+            if (rawTimestamp != null) {
+                request.setRawTimestamp(rawTimestamp);
+            }
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/ingest/Pipeline.java b/server/src/main/java/org/elasticsearch/ingest/Pipeline.java
index a18119bdffba..f4d60ecf5ce3 100644
--- a/server/src/main/java/org/elasticsearch/ingest/Pipeline.java
+++ b/server/src/main/java/org/elasticsearch/ingest/Pipeline.java
@@ -8,8 +8,6 @@
 
 package org.elasticsearch.ingest;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.script.ScriptService;
@@ -17,7 +15,6 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.BiConsumer;
 import java.util.function.LongSupplier;
 
@@ -32,8 +29,6 @@ public final class Pipeline {
     public static final String ON_FAILURE_KEY = "on_failure";
     public static final String META_KEY = "_meta";
 
-    private static final Logger logger = LogManager.getLogger(Pipeline.class);
-
     private final String id;
     @Nullable
     private final String description;
@@ -119,20 +114,14 @@ public void execute(IngestDocument ingestDocument, BiConsumer<IngestDocument, Ex
          * such as the metrics being wrong. The listenerHasBeenCalled variable is used to make sure that the code in the listener
          * is only executed once.
          */
-        final AtomicBoolean listenerHasBeenCalled = new AtomicBoolean(false);
         metrics.preIngest();
         compoundProcessor.execute(ingestDocument, (result, e) -> {
-            if (listenerHasBeenCalled.getAndSet(true)) {
-                logger.warn("A listener was unexpectedly called more than once", new RuntimeException(e));
-                assert false : "A listener was unexpectedly called more than once";
-            } else {
-                long ingestTimeInNanos = relativeTimeProvider.getAsLong() - startTimeInNanos;
-                metrics.postIngest(ingestTimeInNanos);
-                if (e != null) {
-                    metrics.ingestFailed();
-                }
-                handler.accept(result, e);
+            long ingestTimeInNanos = relativeTimeProvider.getAsLong() - startTimeInNanos;
+            metrics.postIngest(ingestTimeInNanos);
+            if (e != null) {
+                metrics.ingestFailed();
             }
+            handler.accept(result, e);
         });
     }
 
diff --git a/server/src/main/java/org/elasticsearch/ingest/WrappingProcessor.java b/server/src/main/java/org/elasticsearch/ingest/WrappingProcessor.java
index d84632edde26..d4096c01e26b 100644
--- a/server/src/main/java/org/elasticsearch/ingest/WrappingProcessor.java
+++ b/server/src/main/java/org/elasticsearch/ingest/WrappingProcessor.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.ingest;
 
 /**
- * A srapping processor is one that encapsulates an inner processor, or a processor that the wrapped processor acts upon. All processors
+ * A wrapping processor is one that encapsulates an inner processor, or a processor that the wrapped processor acts upon. All processors
  * that contain an "inner" processor should implement this interface, such that the actual processor can be obtained.
  */
 public interface WrappingProcessor extends Processor {
diff --git a/server/src/main/java/org/elasticsearch/lucene/util/BitSets.java b/server/src/main/java/org/elasticsearch/lucene/util/BitSets.java
new file mode 100644
index 000000000000..53b4d68ce749
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/lucene/util/BitSets.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.lucene.util;
+
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.BitSet;
+
+import java.io.IOException;
+
+public final class BitSets {
+    private BitSets() {}
+
+    /**
+     * Build a {@link BitSet} from the content of the provided {@link DocIdSetIterator}. If the iterator matches all documents,
+     * then this method will wrap the returned Bitset as {@link MatchAllBitSet} to reduce memory usage.
+     */
+    public static BitSet of(DocIdSetIterator iter, int maxDocs) throws IOException {
+        final BitSet bitSet = BitSet.of(iter, maxDocs);
+        if (bitSet.cardinality() == maxDocs) {
+            return new MatchAllBitSet(maxDocs);
+        } else {
+            return bitSet;
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/lucene/util/MatchAllBitSet.java b/server/src/main/java/org/elasticsearch/lucene/util/MatchAllBitSet.java
new file mode 100644
index 000000000000..7cce67199a21
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/lucene/util/MatchAllBitSet.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.lucene.util;
+
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.BitSet;
+import org.apache.lucene.util.RamUsageEstimator;
+
+import java.io.IOException;
+
+/**
+ * An optimized implementation of {@link BitSet} that matches all documents to reduce memory usage.
+ */
+public final class MatchAllBitSet extends BitSet {
+    private static final long RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(MatchAllBitSet.class);
+
+    private final int numBits;
+
+    public MatchAllBitSet(int numBits) {
+        this.numBits = numBits;
+    }
+
+    @Override
+    public void set(int i) {
+
+    }
+
+    @Override
+    public boolean getAndSet(int i) {
+        return true;
+    }
+
+    @Override
+    public void clear(int i) {
+        assert false : "MatchAllBitSet doesn't support clear";
+        throw new UnsupportedOperationException("MatchAllBitSet doesn't support clear");
+    }
+
+    @Override
+    public void clear(int startIndex, int endIndex) {
+        assert false : "MatchAllBitSet doesn't support clear";
+        throw new UnsupportedOperationException("MatchAllBitSet doesn't support clear");
+    }
+
+    @Override
+    public int cardinality() {
+        return numBits;
+    }
+
+    @Override
+    public int approximateCardinality() {
+        return numBits;
+    }
+
+    @Override
+    public int prevSetBit(int index) {
+        return index;
+    }
+
+    @Override
+    public int nextSetBit(int index) {
+        return index;
+    }
+
+    @Override
+    public long ramBytesUsed() {
+        return RAM_BYTES_USED;
+    }
+
+    @Override
+    public boolean get(int index) {
+        return true;
+    }
+
+    @Override
+    public int length() {
+        return numBits;
+    }
+
+    @Override
+    public void or(DocIdSetIterator iter) throws IOException {
+
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java b/server/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
index c4b450946adc..f928f83531aa 100644
--- a/server/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
+++ b/server/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.monitor.jvm;
 
 import org.apache.lucene.util.Constants;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -268,7 +268,7 @@ public JvmInfo(StreamInput in) throws IOException {
         vmName = in.readString();
         vmVersion = in.readString();
         vmVendor = in.readString();
-        if (in.getVersion().before(Version.V_8_3_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_3_0)) {
             // Before 8.0 the no-jdk distributions could have bundledJdk false, this is always true now.
             in.readBoolean();
         }
@@ -302,7 +302,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(vmName);
         out.writeString(vmVersion);
         out.writeString(vmVendor);
-        if (out.getVersion().before(Version.V_8_3_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_3_0)) {
             out.writeBoolean(true);
         }
         out.writeOptionalBoolean(usingBundledJdk);
diff --git a/server/src/main/java/org/elasticsearch/monitor/os/OsInfo.java b/server/src/main/java/org/elasticsearch/monitor/os/OsInfo.java
index a7be10d87308..d956ee4f6852 100644
--- a/server/src/main/java/org/elasticsearch/monitor/os/OsInfo.java
+++ b/server/src/main/java/org/elasticsearch/monitor/os/OsInfo.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.monitor.os;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.Processors;
@@ -19,7 +19,7 @@
 import java.io.IOException;
 
 public class OsInfo implements ReportingService.Info {
-    private static final Version DOUBLE_PRECISION_ALLOCATED_PROCESSORS_SUPPORT = Version.V_8_5_0;
+    private static final TransportVersion DOUBLE_PRECISION_ALLOCATED_PROCESSORS_SUPPORT = TransportVersion.V_8_5_0;
 
     private final long refreshInterval;
     private final int availableProcessors;
@@ -50,7 +50,7 @@ public OsInfo(
     public OsInfo(StreamInput in) throws IOException {
         this.refreshInterval = in.readLong();
         this.availableProcessors = in.readInt();
-        if (in.getVersion().onOrAfter(DOUBLE_PRECISION_ALLOCATED_PROCESSORS_SUPPORT)) {
+        if (in.getTransportVersion().onOrAfter(DOUBLE_PRECISION_ALLOCATED_PROCESSORS_SUPPORT)) {
             this.allocatedProcessors = Processors.readFrom(in);
         } else {
             this.allocatedProcessors = Processors.of((double) in.readInt());
@@ -65,7 +65,7 @@ public OsInfo(StreamInput in) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeLong(refreshInterval);
         out.writeInt(availableProcessors);
-        if (out.getVersion().onOrAfter(DOUBLE_PRECISION_ALLOCATED_PROCESSORS_SUPPORT)) {
+        if (out.getTransportVersion().onOrAfter(DOUBLE_PRECISION_ALLOCATED_PROCESSORS_SUPPORT)) {
             allocatedProcessors.writeTo(out);
         } else {
             out.writeInt(getAllocatedProcessors());
diff --git a/server/src/main/java/org/elasticsearch/monitor/os/OsStats.java b/server/src/main/java/org/elasticsearch/monitor/os/OsStats.java
index 1d56d2f1387f..dab3eb100e9f 100644
--- a/server/src/main/java/org/elasticsearch/monitor/os/OsStats.java
+++ b/server/src/main/java/org/elasticsearch/monitor/os/OsStats.java
@@ -10,7 +10,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -276,7 +276,7 @@ public Mem(StreamInput in) throws IOException {
                 total = 0;
             }
             this.total = total;
-            if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
                 long adjustedTotal = in.readLong();
                 assert adjustedTotal >= 0 : "expected adjusted total memory to be positive, got: " + adjustedTotal;
                 if (adjustedTotal < 0) {
@@ -299,7 +299,7 @@ public Mem(StreamInput in) throws IOException {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeLong(total);
-            if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
                 out.writeLong(adjustedTotal);
             }
             out.writeLong(free);
diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java
index 566204bb8530..815be41ed17e 100644
--- a/server/src/main/java/org/elasticsearch/node/Node.java
+++ b/server/src/main/java/org/elasticsearch/node/Node.java
@@ -202,6 +202,7 @@
 import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.tracing.Tracer;
+import org.elasticsearch.transport.RemoteClusterPortSettings;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportInterceptor;
 import org.elasticsearch.transport.TransportService;
@@ -581,7 +582,6 @@ protected Node(
                 snapshotsInfoService,
                 threadPool,
                 systemIndices,
-                rerouteServiceReference::get,
                 writeLoadForecaster
             );
             modules.add(clusterModule);
@@ -732,7 +732,7 @@ protected Node(
                     clusterModule.getIndexNameExpressionResolver(),
                     repositoriesServiceReference::get,
                     tracer,
-                    clusterModule.getAllocationService().getAllocationDeciders()
+                    clusterModule.getAllocationService()
                 )
             ).toList();
 
@@ -858,17 +858,14 @@ protected Node(
             );
 
             actionModule.getReservedClusterStateService().installStateHandler(new ReservedRepositoryAction(repositoryService));
+            actionModule.getReservedClusterStateService().installStateHandler(new ReservedPipelineAction());
 
             FileSettingsService fileSettingsService = new FileSettingsService(
                 clusterService,
                 actionModule.getReservedClusterStateService(),
-                environment,
-                client
+                environment
             );
 
-            actionModule.getReservedClusterStateService()
-                .installStateHandler(new ReservedPipelineAction(ingestService, fileSettingsService));
-
             RestoreService restoreService = new RestoreService(
                 clusterService,
                 repositoryService,
@@ -998,7 +995,13 @@ protected Node(
                 discoveryModule.getCoordinator(),
                 masterHistoryService
             );
-            HealthService healthService = createHealthService(clusterService, clusterModule, coordinationDiagnosticsService, threadPool);
+            HealthService healthService = createHealthService(
+                clusterService,
+                clusterModule,
+                coordinationDiagnosticsService,
+                threadPool,
+                systemIndices
+            );
             HealthMetadataService healthMetadataService = HealthMetadataService.create(clusterService, settings);
             LocalHealthMonitor localHealthMonitor = LocalHealthMonitor.create(settings, clusterService, nodeService, threadPool, client);
             HealthInfoCache nodeHealthOverview = HealthInfoCache.create(clusterService);
@@ -1200,7 +1203,8 @@ private HealthService createHealthService(
         ClusterService clusterService,
         ClusterModule clusterModule,
         CoordinationDiagnosticsService coordinationDiagnosticsService,
-        ThreadPool threadPool
+        ThreadPool threadPool,
+        SystemIndices systemIndices
     ) {
         List<HealthIndicatorService> preflightHealthIndicatorServices = Collections.singletonList(
             new StableMasterHealthIndicatorService(coordinationDiagnosticsService, clusterService)
@@ -1208,7 +1212,7 @@ private HealthService createHealthService(
         var serverHealthIndicatorServices = new ArrayList<>(
             List.of(
                 new RepositoryIntegrityHealthIndicatorService(clusterService),
-                new ShardsAvailabilityHealthIndicatorService(clusterService, clusterModule.getAllocationService())
+                new ShardsAvailabilityHealthIndicatorService(clusterService, clusterModule.getAllocationService(), systemIndices)
             )
         );
         serverHealthIndicatorServices.add(new DiskHealthIndicatorService(clusterService));
@@ -1228,22 +1232,22 @@ private RecoveryPlannerService getRecoveryPlannerService(
         ClusterService clusterService,
         RepositoriesService repositoryService
     ) {
-        final List<RecoveryPlannerPlugin> recoveryPlannerPlugins = pluginsService.filterPlugins(RecoveryPlannerPlugin.class);
-        if (recoveryPlannerPlugins.isEmpty()) {
+        final List<RecoveryPlannerService> recoveryPlannerServices = pluginsService.filterPlugins(RecoveryPlannerPlugin.class)
+            .stream()
+            .map(
+                plugin -> plugin.createRecoveryPlannerService(
+                    new ShardSnapshotsService(client, repositoryService, threadPool, clusterService)
+                )
+            )
+            .filter(Optional::isPresent)
+            .map(Optional::get)
+            .toList();
+        if (recoveryPlannerServices.isEmpty()) {
             return new PeerOnlyRecoveryPlannerService();
+        } else if (recoveryPlannerServices.size() > 1) {
+            throw new IllegalStateException("Expected a single RecoveryPlannerService but got: " + recoveryPlannerServices.size());
         }
-
-        if (recoveryPlannerPlugins.size() > 1) {
-            throw new IllegalStateException("A single RecoveryPlannerPlugin was expected but got: " + recoveryPlannerPlugins);
-        }
-
-        final ShardSnapshotsService shardSnapshotsService = new ShardSnapshotsService(
-            client,
-            repositoryService,
-            threadPool,
-            clusterService
-        );
-        return recoveryPlannerPlugins.get(0).createRecoveryPlannerService(shardSnapshotsService);
+        return recoveryPlannerServices.get(0);
     }
 
     private WriteLoadForecaster getWriteLoadForecaster(ThreadPool threadPool, Settings settings, ClusterSettings clusterSettings) {
@@ -1388,6 +1392,13 @@ public Node start() throws NodeValidationException {
             pluginsService.flatMap(Plugin::getBootstrapChecks).toList()
         );
 
+        final FileSettingsService fileSettingsService = injector.getInstance(FileSettingsService.class);
+        fileSettingsService.start();
+        // if we are using the readiness service, listen for the file settings being applied
+        if (ReadinessService.enabled(environment)) {
+            fileSettingsService.addFileSettingsChangedListener(injector.getInstance(ReadinessService.class));
+        }
+
         clusterService.addStateApplier(transportService.getTaskManager());
         // start after transport service so the local disco is known
         coordinator.start(); // start before cluster service so that it can set initial state on ClusterApplierService
@@ -1438,7 +1449,6 @@ public void onTimeout(TimeValue timeout) {
             }
         }
 
-        injector.getInstance(FileSettingsService.class).start();
         injector.getInstance(HttpServerTransport.class).start();
 
         if (WRITE_PORTS_FILE_SETTING.get(settings())) {
@@ -1451,6 +1461,10 @@ public void onTimeout(TimeValue timeout) {
                 ReadinessService readiness = injector.getInstance(ReadinessService.class);
                 readiness.addBoundAddressListener(address -> writePortsFile("readiness", address));
             }
+
+            if (RemoteClusterPortSettings.REMOTE_CLUSTER_PORT_ENABLED.get(environment.settings())) {
+                writePortsFile("remote_cluster", transport.boundRemoteAccessAddress());
+            }
         }
 
         logger.info("started {}", transportService.getLocalNode());
diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java
index d7d542c7117b..1da90ecc4014 100644
--- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java
+++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java
@@ -9,16 +9,18 @@
 
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.AbstractNamedDiffable;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.VersionedNamedWriteable;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ObjectParser;
@@ -34,6 +36,7 @@
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -195,8 +198,8 @@ public long getNumberOfTasksOnNode(String nodeId, String taskName) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.CURRENT.minimumCompatibilityVersion();
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.CURRENT.minimumCompatibilityVersion();
     }
 
     @Override
@@ -552,16 +555,11 @@ public static NamedDiff<Metadata.Custom> readDiffFrom(StreamInput in) throws IOE
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field("last_allocation_id", lastAllocationId);
-        builder.startArray("tasks");
-        {
-            for (PersistentTask<?> entry : tasks.values()) {
-                entry.toXContent(builder, params);
-            }
-        }
-        builder.endArray();
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return Iterators.concat(
+            Iterators.single((builder, params) -> builder.field("last_allocation_id", lastAllocationId)),
+            ChunkedToXContentHelper.array("tasks", tasks.values().iterator())
+        );
     }
 
     public static Builder builder() {
diff --git a/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java b/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java
index 3ca862d88731..e7981144d02f 100644
--- a/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java
+++ b/server/src/main/java/org/elasticsearch/plugins/ClusterPlugin.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.plugins;
 
+import org.elasticsearch.cluster.routing.ShardRoutingRoleStrategy;
 import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster;
 import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
@@ -68,6 +69,10 @@ default Collection<WriteLoadForecaster> createWriteLoadForecasters(
         return Collections.emptyList();
     }
 
+    default ShardRoutingRoleStrategy getShardRoutingRoleStrategy() {
+        return null;
+    }
+
     /**
      * Called when the node is started
      */
diff --git a/server/src/main/java/org/elasticsearch/plugins/IndexStorePlugin.java b/server/src/main/java/org/elasticsearch/plugins/IndexStorePlugin.java
index 39d7f7d1d91f..3730f9a65b99 100644
--- a/server/src/main/java/org/elasticsearch/plugins/IndexStorePlugin.java
+++ b/server/src/main/java/org/elasticsearch/plugins/IndexStorePlugin.java
@@ -44,6 +44,18 @@ interface DirectoryFactory {
          * @throws IOException if an IOException occurs while opening the directory
          */
         Directory newDirectory(IndexSettings indexSettings, ShardPath shardPath) throws IOException;
+
+        /**
+         * Creates a new directory per shard. This method is called once per shard on shard creation.
+         * @param indexSettings the shards index settings
+         * @param shardPath the path the shard is using
+         * @param shardRouting the {@link ShardRouting}
+         * @return a new lucene directory instance
+         * @throws IOException if an IOException occurs while opening the directory
+         */
+        default Directory newDirectory(IndexSettings indexSettings, ShardPath shardPath, ShardRouting shardRouting) throws IOException {
+            return newDirectory(indexSettings, shardPath);
+        }
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/plugins/Plugin.java b/server/src/main/java/org/elasticsearch/plugins/Plugin.java
index 916f637a04ca..faa5a54a3e16 100644
--- a/server/src/main/java/org/elasticsearch/plugins/Plugin.java
+++ b/server/src/main/java/org/elasticsearch/plugins/Plugin.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.io.stream.NamedWriteable;
@@ -82,6 +82,7 @@ public abstract class Plugin implements Closeable {
      * @param repositoriesServiceSupplier A supplier for the service that manages snapshot repositories; will return null when this method
      *                                    is called, but will return the repositories service once the node is initialized.
      * @param tracer                      An interface for distributed tracing
+     * @param allocationService           A service to manage shard allocation in the cluster
      */
     public Collection<Object> createComponents(
         Client client,
@@ -96,7 +97,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         return Collections.emptyList();
     }
diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginDescriptor.java b/server/src/main/java/org/elasticsearch/plugins/PluginDescriptor.java
index 455e1a3cfdee..dd56e1895731 100644
--- a/server/src/main/java/org/elasticsearch/plugins/PluginDescriptor.java
+++ b/server/src/main/java/org/elasticsearch/plugins/PluginDescriptor.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.plugins;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -41,11 +42,13 @@ public class PluginDescriptor implements Writeable, ToXContentObject {
 
     public static final String INTERNAL_DESCRIPTOR_FILENAME = "plugin-descriptor.properties";
     public static final String STABLE_DESCRIPTOR_FILENAME = "stable-plugin-descriptor.properties";
+    public static final String NAMED_COMPONENTS_FILENAME = "named_components.json";
+
     public static final String ES_PLUGIN_POLICY = "plugin-security.policy";
 
-    private static final Version LICENSED_PLUGINS_SUPPORT = Version.V_7_11_0;
-    private static final Version MODULE_NAME_SUPPORT = Version.V_8_3_0;
-    private static final Version BOOTSTRAP_SUPPORT_REMOVED = Version.V_8_4_0;
+    private static final TransportVersion LICENSED_PLUGINS_SUPPORT = TransportVersion.V_7_11_0;
+    private static final TransportVersion MODULE_NAME_SUPPORT = TransportVersion.V_8_3_0;
+    private static final TransportVersion BOOTSTRAP_SUPPORT_REMOVED = TransportVersion.V_8_4_0;
 
     private final String name;
     private final String description;
@@ -116,7 +119,7 @@ public PluginDescriptor(final StreamInput in) throws IOException {
         elasticsearchVersion = Version.readVersion(in);
         javaVersion = in.readString();
         this.classname = in.readString();
-        if (in.getVersion().onOrAfter(MODULE_NAME_SUPPORT)) {
+        if (in.getTransportVersion().onOrAfter(MODULE_NAME_SUPPORT)) {
             this.moduleName = in.readOptionalString();
         } else {
             this.moduleName = null;
@@ -124,8 +127,8 @@ public PluginDescriptor(final StreamInput in) throws IOException {
         extendedPlugins = in.readStringList();
         hasNativeController = in.readBoolean();
 
-        if (in.getVersion().onOrAfter(LICENSED_PLUGINS_SUPPORT)) {
-            if (in.getVersion().before(BOOTSTRAP_SUPPORT_REMOVED)) {
+        if (in.getTransportVersion().onOrAfter(LICENSED_PLUGINS_SUPPORT)) {
+            if (in.getTransportVersion().before(BOOTSTRAP_SUPPORT_REMOVED)) {
                 in.readString(); // plugin type
                 in.readOptionalString(); // java opts
             }
@@ -134,7 +137,7 @@ public PluginDescriptor(final StreamInput in) throws IOException {
             isLicensed = false;
         }
 
-        if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             isModular = in.readBoolean();
             isStable = in.readBoolean();
         } else {
@@ -151,20 +154,20 @@ public void writeTo(final StreamOutput out) throws IOException {
         Version.writeVersion(elasticsearchVersion, out);
         out.writeString(javaVersion);
         out.writeString(classname);
-        if (out.getVersion().onOrAfter(MODULE_NAME_SUPPORT)) {
+        if (out.getTransportVersion().onOrAfter(MODULE_NAME_SUPPORT)) {
             out.writeOptionalString(moduleName);
         }
         out.writeStringCollection(extendedPlugins);
         out.writeBoolean(hasNativeController);
 
-        if (out.getVersion().onOrAfter(LICENSED_PLUGINS_SUPPORT)) {
-            if (out.getVersion().before(BOOTSTRAP_SUPPORT_REMOVED)) {
+        if (out.getTransportVersion().onOrAfter(LICENSED_PLUGINS_SUPPORT)) {
+            if (out.getTransportVersion().before(BOOTSTRAP_SUPPORT_REMOVED)) {
                 out.writeString("ISOLATED");
                 out.writeOptionalString(null);
             }
             out.writeBoolean(isLicensed);
         }
-        if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             out.writeBoolean(isModular);
             out.writeBoolean(isStable);
         }
diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginRuntimeInfo.java b/server/src/main/java/org/elasticsearch/plugins/PluginRuntimeInfo.java
index bf6c077b4de4..f58f14bcd7a7 100644
--- a/server/src/main/java/org/elasticsearch/plugins/PluginRuntimeInfo.java
+++ b/server/src/main/java/org/elasticsearch/plugins/PluginRuntimeInfo.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.plugins;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -40,7 +40,7 @@ public PluginRuntimeInfo(StreamInput in) throws IOException {
     }
 
     private static Boolean readIsOfficial(StreamInput in) throws IOException {
-        if (in.getVersion().onOrAfter(Version.V_8_3_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0)) {
             return in.readBoolean();
         } else {
             return null;
@@ -48,7 +48,7 @@ private static Boolean readIsOfficial(StreamInput in) throws IOException {
     }
 
     private static PluginApiInfo readApiInfo(StreamInput in) throws IOException {
-        if (in.getVersion().onOrAfter(Version.V_8_3_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0)) {
             return in.readOptionalWriteable(PluginApiInfo::new);
         } else {
             return null;
@@ -72,7 +72,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         descriptor.writeTo(out);
-        if (out.getVersion().onOrAfter(Version.V_8_3_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0)) {
             out.writeBoolean(isOfficial);
             out.writeOptionalWriteable(pluginApiInfo);
         }
diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java
index 424f58e2d557..5bad422d8f06 100644
--- a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java
+++ b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java
@@ -523,12 +523,33 @@ static LayerAndLoader createPlugin(
                 extendedPlugins.stream().map(LoadedPlugin::layer)
             ).toList();
             return createPluginModuleLayer(bundle, pluginParentLoader, parentLayers);
+        } else if (plugin.isStable()) {
+            logger.debug(() -> "Loading bundle: " + plugin.getName() + ", non-modular as synthetic module");
+            return LayerAndLoader.ofLoader(
+                UberModuleClassLoader.getInstance(
+                    pluginParentLoader,
+                    ModuleLayer.boot(),
+                    "synthetic." + toModuleName(plugin.getName()),
+                    bundle.allUrls,
+                    Set.of("org.elasticsearch.server") // TODO: instead of denying server, allow only jvm + stable API modules
+                )
+            );
         } else {
             logger.debug(() -> "Loading bundle: " + plugin.getName() + ", non-modular");
             return LayerAndLoader.ofLoader(URLClassLoader.newInstance(bundle.urls.toArray(URL[]::new), pluginParentLoader));
         }
     }
 
+    // package-visible for testing
+    static String toModuleName(String name) {
+        String result = name.replaceAll("\\W+", ".") // replace non-alphanumeric character strings with dots
+            .replaceAll("(^[^A-Za-z_]*)", "") // trim non-alpha or underscore characters from start
+            .replaceAll("\\.$", "") // trim trailing dot
+            .toLowerCase(Locale.getDefault());
+        assert ModuleSupport.isPackageName(result);
+        return result;
+    }
+
     private static void checkDeprecations(
         PluginIntrospector inspector,
         List<PluginDescriptor> pluginDescriptors,
diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsUtils.java b/server/src/main/java/org/elasticsearch/plugins/PluginsUtils.java
index afe5f0b40e3e..e08331ee8fb9 100644
--- a/server/src/main/java/org/elasticsearch/plugins/PluginsUtils.java
+++ b/server/src/main/java/org/elasticsearch/plugins/PluginsUtils.java
@@ -77,7 +77,30 @@ public static List<Path> findPluginDirs(final Path rootPath) throws IOException
      * Verify the given plugin is compatible with the current Elasticsearch installation.
      */
     public static void verifyCompatibility(PluginDescriptor info) {
-        if (info.getElasticsearchVersion().equals(Version.CURRENT) == false) {
+        if (info.isStable()) {
+            if (info.getElasticsearchVersion().major != Version.CURRENT.major) {
+                throw new IllegalArgumentException(
+                    "Stable Plugin ["
+                        + info.getName()
+                        + "] was built for Elasticsearch major version "
+                        + info.getElasticsearchVersion().major
+                        + " but version "
+                        + Version.CURRENT
+                        + " is running"
+                );
+            }
+            if (info.getElasticsearchVersion().after(Version.CURRENT)) {
+                throw new IllegalArgumentException(
+                    "Stable Plugin ["
+                        + info.getName()
+                        + "] was built for Elasticsearch version "
+                        + info.getElasticsearchVersion()
+                        + " but earlier version "
+                        + Version.CURRENT
+                        + " is running"
+                );
+            }
+        } else if (info.getElasticsearchVersion().equals(Version.CURRENT) == false) {
             throw new IllegalArgumentException(
                 "Plugin ["
                     + info.getName()
diff --git a/server/src/main/java/org/elasticsearch/plugins/RecoveryPlannerPlugin.java b/server/src/main/java/org/elasticsearch/plugins/RecoveryPlannerPlugin.java
index ed0f7a0abcd0..8602d9104dcb 100644
--- a/server/src/main/java/org/elasticsearch/plugins/RecoveryPlannerPlugin.java
+++ b/server/src/main/java/org/elasticsearch/plugins/RecoveryPlannerPlugin.java
@@ -11,10 +11,12 @@
 import org.elasticsearch.indices.recovery.plan.RecoveryPlannerService;
 import org.elasticsearch.indices.recovery.plan.ShardSnapshotsService;
 
+import java.util.Optional;
+
 /**
- * A plugin that allows creating custom {@code RecoveryPlannerService}. Only one plugin of this type
- * is allowed to be installed at once.
+ * A plugin that allows creating custom {@code RecoveryPlannerService}. Only one {@code RecoveryPlannerService} is allowed to be active
+ * at once.
  */
 public interface RecoveryPlannerPlugin {
-    RecoveryPlannerService createRecoveryPlannerService(ShardSnapshotsService shardSnapshotsService);
+    Optional<RecoveryPlannerService> createRecoveryPlannerService(ShardSnapshotsService shardSnapshotsService);
 }
diff --git a/server/src/main/java/org/elasticsearch/plugins/SearchPlugin.java b/server/src/main/java/org/elasticsearch/plugins/SearchPlugin.java
index 6067768715aa..bcd82b18470c 100644
--- a/server/src/main/java/org/elasticsearch/plugins/SearchPlugin.java
+++ b/server/src/main/java/org/elasticsearch/plugins/SearchPlugin.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.search.suggest.Suggester;
 import org.elasticsearch.search.suggest.SuggestionBuilder;
+import org.elasticsearch.search.vectors.QueryVectorBuilder;
 import org.elasticsearch.xcontent.ContextParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContent;
@@ -73,6 +74,14 @@ default List<SignificanceHeuristicSpec<?>> getSignificanceHeuristics() {
         return emptyList();
     }
 
+    /**
+     * The new {@link QueryVectorBuilder}s defined by this plugin. {@linkplain QueryVectorBuilder}s can be used within a kNN
+     * search to build the query vector instead of having the user provide the vector directly
+     */
+    default List<QueryVectorBuilderSpec<?>> getQueryVectorBuilders() {
+        return emptyList();
+    }
+
     /**
      * The new {@link FetchSubPhase}s defined by this plugin.
      */
@@ -592,4 +601,37 @@ public Map<String, Highlighter> getHighlighters() {
             return highlighters;
         }
     }
+
+    /**
+     * Specification of custom {@link QueryVectorBuilder}.
+     */
+    class QueryVectorBuilderSpec<T extends QueryVectorBuilder> extends SearchExtensionSpec<T, BiFunction<XContentParser, Void, T>> {
+        /**
+         * Specification of custom {@link QueryVectorBuilder}.
+         *
+         * @param name holds the names by which this query vector builder might be parsed.
+         *             The {@link ParseField#getPreferredName()} is special as it
+         *             is the name by under which the reader is registered. So it is the name that the builder should use as its
+         *             {@link NamedWriteable#getWriteableName()} too.
+         * @param reader the reader registered for this query vector builder. Typically a reference to a constructor that takes a
+         *        {@link StreamInput}
+         * @param parser the parser the reads the query vector builder from xcontent
+         */
+        public QueryVectorBuilderSpec(ParseField name, Writeable.Reader<T> reader, BiFunction<XContentParser, Void, T> parser) {
+            super(name, reader, parser);
+        }
+
+        /**
+         * Specification of custom {@link QueryVectorBuilder}.
+         *
+         * @param name the name by which this query vector builder might be parsed or deserialized.
+         *             Make sure that the query builder returns this name for {@link NamedWriteable#getWriteableName()}.
+         * @param reader the reader registered for this query vector builder. Typically a reference to a constructor that takes a
+         *        {@link StreamInput}
+         * @param parser the parser the reads the query vector builder from xcontent
+         */
+        public QueryVectorBuilderSpec(String name, Writeable.Reader<T> reader, BiFunction<XContentParser, Void, T> parser) {
+            super(name, reader, parser);
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/plugins/UberModuleClassLoader.java b/server/src/main/java/org/elasticsearch/plugins/UberModuleClassLoader.java
index 0981d30a9225..dca3afb2ed74 100644
--- a/server/src/main/java/org/elasticsearch/plugins/UberModuleClassLoader.java
+++ b/server/src/main/java/org/elasticsearch/plugins/UberModuleClassLoader.java
@@ -280,6 +280,13 @@ protected Class<?> loadClass(String cn, boolean resolve) throws ClassNotFoundExc
         }
     }
 
+    // For testing in cases where code must be given access to an unnamed module
+    void addReadsSystemClassLoaderUnnamedModule() {
+        moduleController.layer()
+            .modules()
+            .forEach(module -> moduleController.addReads(module, ClassLoader.getSystemClassLoader().getUnnamedModule()));
+    }
+
     /**
      * Returns the package name for the given class name
      */
@@ -311,4 +318,8 @@ public void close() throws Exception {
         AccessController.doPrivileged(pa);
     }
 
+    // visible for testing
+    public URLClassLoader getInternalLoader() {
+        return internalLoader;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/plugins/scanners/ExtensiblesRegistry.java b/server/src/main/java/org/elasticsearch/plugins/scanners/ExtensiblesRegistry.java
index dc40bd6fcd8e..ec874c9a2c5c 100644
--- a/server/src/main/java/org/elasticsearch/plugins/scanners/ExtensiblesRegistry.java
+++ b/server/src/main/java/org/elasticsearch/plugins/scanners/ExtensiblesRegistry.java
@@ -10,6 +10,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.plugin.Extensible;
 
 import java.util.Map;
 
@@ -58,7 +59,7 @@
  *     "E2" : "E2"
  * }
  *
- * @see org.elasticsearch.plugin.api.Extensible
+ * @see Extensible
  */
 public class ExtensiblesRegistry {
 
diff --git a/server/src/main/java/org/elasticsearch/plugins/scanners/StablePluginsRegistry.java b/server/src/main/java/org/elasticsearch/plugins/scanners/StablePluginsRegistry.java
index cb5656477b62..6e2780a08251 100644
--- a/server/src/main/java/org/elasticsearch/plugins/scanners/StablePluginsRegistry.java
+++ b/server/src/main/java/org/elasticsearch/plugins/scanners/StablePluginsRegistry.java
@@ -27,7 +27,7 @@ public class StablePluginsRegistry {
     effectively means that an interface which extends another interface marked with @Extensible is also extensible
     NameToPluginInfo map is a map of Name to PluginInfo
     i.e.
-    org.elasticsearch.plugin.analysis.api.TokenFilterFactory ->
+    org.elasticsearch.plugin.analysis.TokenFilterFactory ->
         {"nori" -> {nori, org.elasticserach.plugin.analysis.new_nori.NoriReadingFormFilterFactory, classloaderInstance}
      */
     private final Map<String /*Extensible */, NameToPluginInfo> namedComponents;
diff --git a/server/src/main/java/org/elasticsearch/readiness/ReadinessService.java b/server/src/main/java/org/elasticsearch/readiness/ReadinessService.java
index de6a9e1f099b..8d9d57c2c919 100644
--- a/server/src/main/java/org/elasticsearch/readiness/ReadinessService.java
+++ b/server/src/main/java/org/elasticsearch/readiness/ReadinessService.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.reservedstate.service.FileSettingsChangedListener;
 import org.elasticsearch.shutdown.PluginShutdownService;
 import org.elasticsearch.transport.BindTransportException;
 
@@ -36,7 +37,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicReference;
 
-public class ReadinessService extends AbstractLifecycleComponent implements ClusterStateListener {
+public class ReadinessService extends AbstractLifecycleComponent implements ClusterStateListener, FileSettingsChangedListener {
     private static final Logger logger = LogManager.getLogger(ReadinessService.class);
 
     private final Environment environment;
@@ -47,6 +48,9 @@ public class ReadinessService extends AbstractLifecycleComponent implements Clus
     volatile CountDownLatch listenerThreadLatch = new CountDownLatch(0);
     final AtomicReference<InetSocketAddress> boundSocket = new AtomicReference<>();
     private final Collection<BoundAddressListener> boundAddressListeners = new CopyOnWriteArrayList<>();
+    private volatile boolean fileSettingsApplied = false;
+    private volatile boolean masterElected = false;
+    private volatile boolean shuttingDown = false;
 
     public static final Setting<Integer> PORT = Setting.intSetting("readiness.port", -1, Setting.Property.NodeScope);
 
@@ -190,6 +194,13 @@ protected void doStop() {
     // package private for testing
     synchronized void stopListener() {
         assert enabled(environment);
+
+        // Avoid unnecessary logging if stop is repeatedly called.
+        // This can happen because we call stop listener on cluster state updates.
+        if (ready() == false) {
+            return;
+        }
+
         try {
             logger.info(
                 "stopping readiness service on channel {}",
@@ -213,13 +224,20 @@ protected void doClose() {}
     @Override
     public void clusterChanged(ClusterChangedEvent event) {
         ClusterState clusterState = event.state();
-
         Set<String> shutdownNodeIds = PluginShutdownService.shutdownNodes(clusterState);
-        if (shutdownNodeIds.contains(clusterState.nodes().getLocalNodeId())) {
+
+        this.masterElected = clusterState.nodes().getMasterNodeId() != null;
+        this.shuttingDown = shutdownNodeIds.contains(clusterState.nodes().getLocalNodeId());
+
+        if (shuttingDown) {
             setReady(false);
             logger.info("marking node as not ready because it's shutting down");
         } else {
-            setReady(clusterState.nodes().getMasterNodeId() != null);
+            if (clusterState.nodes().getLocalNodeId().equals(clusterState.nodes().getMasterNodeId())) {
+                setReady(fileSettingsApplied);
+            } else {
+                setReady(masterElected);
+            }
         }
     }
 
@@ -239,6 +257,12 @@ public void addBoundAddressListener(BoundAddressListener listener) {
         boundAddressListeners.add(listener);
     }
 
+    @Override
+    public void settingsChanged() {
+        fileSettingsApplied = true;
+        setReady(masterElected && (shuttingDown == false));
+    }
+
     /**
      * A listener to be notified when the readiness service establishes the port it's listening on.
      * The {@link #addressBound(BoundTransportAddress)} method is called after the readiness service socket
diff --git a/server/src/main/java/org/elasticsearch/repositories/IndexMetaDataGenerations.java b/server/src/main/java/org/elasticsearch/repositories/IndexMetaDataGenerations.java
index 0fd5bf4944a0..7f2a09637dbf 100644
--- a/server/src/main/java/org/elasticsearch/repositories/IndexMetaDataGenerations.java
+++ b/server/src/main/java/org/elasticsearch/repositories/IndexMetaDataGenerations.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.snapshots.SnapshotId;
 
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -32,7 +31,7 @@
  */
 public final class IndexMetaDataGenerations {
 
-    public static final IndexMetaDataGenerations EMPTY = new IndexMetaDataGenerations(Collections.emptyMap(), Collections.emptyMap());
+    public static final IndexMetaDataGenerations EMPTY = new IndexMetaDataGenerations(Map.of(), Map.of());
 
     /**
      * Map of {@link SnapshotId} to a map of the indices in a snapshot mapping {@link IndexId} to metadata identifiers.
@@ -94,7 +93,7 @@ public String indexMetaBlobId(SnapshotId snapshotId, IndexId indexId) {
      */
     @Nullable
     public String snapshotIndexMetadataIdentifier(SnapshotId snapshotId, IndexId indexId) {
-        return lookup.getOrDefault(snapshotId, Collections.emptyMap()).get(indexId);
+        return lookup.getOrDefault(snapshotId, Map.of()).get(indexId);
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/repositories/IndexSnapshotsService.java b/server/src/main/java/org/elasticsearch/repositories/IndexSnapshotsService.java
index 565e0420e62a..e01631f936f7 100644
--- a/server/src/main/java/org/elasticsearch/repositories/IndexSnapshotsService.java
+++ b/server/src/main/java/org/elasticsearch/repositories/IndexSnapshotsService.java
@@ -8,6 +8,8 @@
 
 package org.elasticsearch.repositories;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.StepListener;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -28,6 +30,9 @@
 import java.util.Optional;
 
 public class IndexSnapshotsService {
+
+    private static final Logger logger = LogManager.getLogger(IndexSnapshotsService.class);
+
     private static final Comparator<Tuple<SnapshotId, RepositoryData.SnapshotDetails>> START_TIME_COMPARATOR = Comparator.<
         Tuple<SnapshotId, RepositoryData.SnapshotDetails>>comparingLong(pair -> pair.v2().getStartTimeMillis()).thenComparing(Tuple::v1);
 
@@ -62,6 +67,7 @@ public void getLatestSuccessfulSnapshotForShard(
 
         repositoryDataStepListener.whenComplete(repositoryData -> {
             if (repositoryData.hasIndex(indexName) == false) {
+                logger.debug("{} repository [{}] has no snapshots of this index", shardId, repositoryName);
                 listener.onResponse(Optional.empty());
                 return;
             }
@@ -81,11 +87,13 @@ public void getLatestSuccessfulSnapshotForShard(
                 // have the start/end date populated in RepositoryData. We could fetch all the backups and find out if there is
                 // a valid candidate, but for simplicity we just consider that we couldn't find any valid snapshot. Existing
                 // snapshots start/end timestamps should appear in the RepositoryData eventually.
+                logger.debug("{} could not determine latest snapshot of this shard in repository [{}]", shardId, repositoryName);
                 listener.onResponse(Optional.empty());
                 return;
             }
 
             final SnapshotId snapshotId = latestSnapshotId.get();
+            logger.debug("{} fetching details of [{}][{}]", shardId, repositoryName, snapshotId);
             repository.getSnapshotInfo(
                 snapshotId,
                 snapshotInfoStepListener.map(
@@ -100,6 +108,7 @@ public void getLatestSuccessfulSnapshotForShard(
 
             if (snapshotInfo == null || snapshotInfo.state() != SnapshotState.SUCCESS) {
                 // We couldn't find a valid candidate
+                logger.debug("{} failed to retrieve snapshot details from [{}]", shardId, repositoryName);
                 listener.onResponse(Optional.empty());
                 return;
             }
diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
index dcfcf09cd1b0..2f28adcebc98 100644
--- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
+++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
@@ -318,7 +318,7 @@ public void validateRepositoryCanBeCreated(final PutRepositoryRequest request) {
         final RepositoryMetadata newRepositoryMetadata = new RepositoryMetadata(request.name(), request.type(), request.settings());
 
         // Trying to create the new repository on master to make sure it works
-        closeRepository(createRepository(newRepositoryMetadata, typesRegistry, RepositoriesService::throwRepositoryTypeDoesNotExists));
+        closeRepository(createRepository(newRepositoryMetadata));
     }
 
     private void submitUnbatchedTask(@SuppressWarnings("SameParameterValue") String source, ClusterStateUpdateTask task) {
@@ -757,6 +757,25 @@ private static Repository createRepository(
         }
     }
 
+    /**
+     * Creates a repository holder.
+     *
+     * <p>WARNING: This method is intended for expert only usage mainly in plugins/modules. Please take note of the following:</p>
+     *
+     * <ul>
+     *     <li>This method does not register the repository (e.g., in the cluster state).</li>
+     *     <li>This method starts the repository. The repository should be closed after use.</li>
+     *     <li>The repository metadata should be associated to an already registered non-internal repository type and factory pair.</li>
+     * </ul>
+     *
+     * @param repositoryMetadata the repository metadata
+     * @return the started repository
+     * @throws RepositoryException if repository type is not registered
+     */
+    public Repository createRepository(RepositoryMetadata repositoryMetadata) {
+        return createRepository(repositoryMetadata, typesRegistry, RepositoriesService::throwRepositoryTypeDoesNotExists);
+    }
+
     private static Repository throwRepositoryTypeDoesNotExists(RepositoryMetadata repositoryMetadata) {
         throw new RepositoryException(repositoryMetadata.name(), "repository type [" + repositoryMetadata.type() + "] does not exist");
     }
diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java
index cc4cd2fc4499..d818476c68c2 100644
--- a/server/src/main/java/org/elasticsearch/repositories/Repository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
-import org.elasticsearch.cluster.SnapshotsInProgress;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
@@ -31,7 +30,6 @@
 import java.io.IOException;
 import java.util.Collection;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -299,14 +297,6 @@ void cloneShardSnapshot(
         ActionListener<ShardSnapshotResult> listener
     );
 
-    /**
-     * Hook that allows a repository to filter the user supplied snapshot metadata in {@link SnapshotsInProgress.Entry#userMetadata()}
-     * during snapshot initialization.
-     */
-    default Map<String, Object> adaptUserMetadata(Map<String, Object> userMetadata) {
-        return userMetadata;
-    }
-
     /**
      * Block until all in-flight operations for this repository have completed. Must only be called after this instance has been closed
      * by a call to stop {@link #close()}.
diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java
index c412bf7260ec..089b5a6e639b 100644
--- a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java
+++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java
@@ -875,7 +875,7 @@ private static IndexMetaDataGenerations buildIndexMetaGenerations(
             for (Map.Entry<String, String> generationEntry : val.entrySet()) {
                 forSnapshot.put(indexLookup.get(generationEntry.getKey()), generationEntry.getValue());
             }
-            indexGenerations.put(snapshotIdMapEntry.getKey(), forSnapshot);
+            indexGenerations.put(snapshotIdMapEntry.getKey(), Map.copyOf(forSnapshot));
         }
         return new IndexMetaDataGenerations(indexGenerations, indexMetaIdentifiers);
     }
diff --git a/server/src/main/java/org/elasticsearch/repositories/SnapshotShardContext.java b/server/src/main/java/org/elasticsearch/repositories/SnapshotShardContext.java
index 68cef0607151..52fea9deffb3 100644
--- a/server/src/main/java/org/elasticsearch/repositories/SnapshotShardContext.java
+++ b/server/src/main/java/org/elasticsearch/repositories/SnapshotShardContext.java
@@ -18,8 +18,6 @@
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.snapshots.SnapshotId;
 
-import java.util.Map;
-
 /**
  * Context holding the state for creating a shard snapshot via {@link Repository#snapshotShard(SnapshotShardContext)}.
  * Wraps a {@link org.elasticsearch.index.engine.Engine.IndexCommitRef} that is released once this instances is completed by invoking
@@ -36,7 +34,6 @@ public final class SnapshotShardContext extends ActionListener.Delegating<ShardS
     private final String shardStateIdentifier;
     private final IndexShardSnapshotStatus snapshotStatus;
     private final Version repositoryMetaVersion;
-    private final Map<String, Object> userMetadata;
     private final long snapshotStartTime;
 
     /**
@@ -50,8 +47,6 @@ public final class SnapshotShardContext extends ActionListener.Delegating<ShardS
      *                              snapshotting will be done by inspecting the physical files referenced by {@code snapshotIndexCommit}
      * @param snapshotStatus        snapshot status
      * @param repositoryMetaVersion version of the updated repository metadata to write
-     * @param userMetadata          user metadata of the snapshot found in
-     *                              {@link org.elasticsearch.cluster.SnapshotsInProgress.Entry#userMetadata()}
      * @param snapshotStartTime     start time of the snapshot found in
      *                              {@link org.elasticsearch.cluster.SnapshotsInProgress.Entry#startTime()}
      * @param listener              listener invoked on completion
@@ -65,7 +60,6 @@ public SnapshotShardContext(
         @Nullable String shardStateIdentifier,
         IndexShardSnapshotStatus snapshotStatus,
         Version repositoryMetaVersion,
-        Map<String, Object> userMetadata,
         final long snapshotStartTime,
         ActionListener<ShardSnapshotResult> listener
     ) {
@@ -78,7 +72,6 @@ public SnapshotShardContext(
         this.shardStateIdentifier = shardStateIdentifier;
         this.snapshotStatus = snapshotStatus;
         this.repositoryMetaVersion = repositoryMetaVersion;
-        this.userMetadata = userMetadata;
         this.snapshotStartTime = snapshotStartTime;
     }
 
@@ -115,10 +108,6 @@ public Version getRepositoryMetaVersion() {
         return repositoryMetaVersion;
     }
 
-    public Map<String, Object> userMetadata() {
-        return userMetadata;
-    }
-
     public long snapshotStartTime() {
         return snapshotStartTime;
     }
diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index d7df332673be..730244de5c47 100644
--- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -25,11 +25,13 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
-import org.elasticsearch.action.ResultDeduplicator;
+import org.elasticsearch.action.SingleResultDeduplicator;
 import org.elasticsearch.action.StepListener;
 import org.elasticsearch.action.support.GroupedActionListener;
 import org.elasticsearch.action.support.ListenableActionFuture;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.support.RefCountingListener;
+import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.action.support.ThreadedActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
@@ -44,6 +46,7 @@
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Numbers;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.blobstore.BlobContainer;
@@ -67,6 +70,7 @@
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.Nullable;
@@ -272,14 +276,16 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
     public static final ChecksumBlobStoreFormat<Metadata> GLOBAL_METADATA_FORMAT = new ChecksumBlobStoreFormat<>(
         "metadata",
         METADATA_NAME_FORMAT,
-        (repoName, parser) -> Metadata.fromXContent(parser)
+        (repoName, parser) -> Metadata.fromXContent(parser),
+        ChunkedToXContent::wrapAsToXContent
     );
 
     public static final ChecksumBlobStoreFormat<IndexMetadata> INDEX_METADATA_FORMAT = new ChecksumBlobStoreFormat<>(
         "index-metadata",
         METADATA_NAME_FORMAT,
         (repoName, parser) -> IndexMetadata.Builder.legacyFromXContent(parser),
-        (repoName, parser) -> IndexMetadata.fromXContent(parser)
+        (repoName, parser) -> IndexMetadata.fromXContent(parser),
+        Function.identity()
     );
 
     private static final String SNAPSHOT_CODEC = "snapshot";
@@ -287,19 +293,22 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
     public static final ChecksumBlobStoreFormat<SnapshotInfo> SNAPSHOT_FORMAT = new ChecksumBlobStoreFormat<>(
         SNAPSHOT_CODEC,
         SNAPSHOT_NAME_FORMAT,
-        SnapshotInfo::fromXContentInternal
+        SnapshotInfo::fromXContentInternal,
+        Function.identity()
     );
 
     public static final ChecksumBlobStoreFormat<BlobStoreIndexShardSnapshot> INDEX_SHARD_SNAPSHOT_FORMAT = new ChecksumBlobStoreFormat<>(
         SNAPSHOT_CODEC,
         SNAPSHOT_NAME_FORMAT,
-        (repoName, parser) -> BlobStoreIndexShardSnapshot.fromXContent(parser)
+        (repoName, parser) -> BlobStoreIndexShardSnapshot.fromXContent(parser),
+        Function.identity()
     );
 
     public static final ChecksumBlobStoreFormat<BlobStoreIndexShardSnapshots> INDEX_SHARD_SNAPSHOTS_FORMAT = new ChecksumBlobStoreFormat<>(
         "snapshots",
         SNAPSHOT_INDEX_NAME_FORMAT,
-        (repoName, parser) -> BlobStoreIndexShardSnapshots.fromXContent(parser)
+        (repoName, parser) -> BlobStoreIndexShardSnapshots.fromXContent(parser),
+        Function.identity()
     );
 
     public static final Setting<ByteSizeValue> MAX_SNAPSHOT_BYTES_PER_SEC = Setting.byteSizeSetting(
@@ -413,7 +422,11 @@ protected BlobStoreRepository(
         this.namedXContentRegistry = namedXContentRegistry;
         this.basePath = basePath;
         this.maxSnapshotCount = MAX_SNAPSHOTS_SETTING.get(metadata.settings());
-        this.repoDataDeduplicator = new ResultDeduplicator<>(threadPool.getThreadContext());
+        this.repoDataLoadDeduplicator = new SingleResultDeduplicator<>(
+            threadPool.getThreadContext(),
+            listener -> threadPool.executor(ThreadPool.Names.SNAPSHOT_META)
+                .execute(ActionRunnable.wrap(listener, this::doGetRepositoryData))
+        );
         shardSnapshotTaskRunner = new ShardSnapshotTaskRunner(
             threadPool.info(ThreadPool.Names.SNAPSHOT).getMax(),
             threadPool.executor(ThreadPool.Names.SNAPSHOT),
@@ -954,31 +967,36 @@ private void doDeleteShardSnapshots(
             writeUpdatedRepoDataStep.whenComplete(updatedRepoData -> {
                 listener.onRepositoryDataWritten(updatedRepoData);
                 // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion
-                final ActionListener<Void> afterCleanupsListener = new GroupedActionListener<>(ActionListener.wrap(listener::onDone), 2);
-                cleanupUnlinkedRootAndIndicesBlobs(snapshotIds, foundIndices, rootBlobs, updatedRepoData, afterCleanupsListener);
-                asyncCleanupUnlinkedShardLevelBlobs(
-                    repositoryData,
-                    snapshotIds,
-                    writeShardMetaDataAndComputeDeletesStep.result(),
-                    afterCleanupsListener
-                );
+                try (var refs = new RefCountingRunnable(listener::onDone)) {
+                    cleanupUnlinkedRootAndIndicesBlobs(snapshotIds, foundIndices, rootBlobs, updatedRepoData, refs.acquireListener());
+                    asyncCleanupUnlinkedShardLevelBlobs(
+                        repositoryData,
+                        snapshotIds,
+                        writeShardMetaDataAndComputeDeletesStep.result(),
+                        refs.acquireListener()
+                    );
+                }
             }, listener::onFailure);
         } else {
             // Write the new repository data first (with the removed snapshot), using no shard generations
             final RepositoryData updatedRepoData = repositoryData.removeSnapshots(snapshotIds, ShardGenerations.EMPTY);
             writeIndexGen(updatedRepoData, repositoryStateId, repoMetaVersion, Function.identity(), ActionListener.wrap(newRepoData -> {
-                // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion
-                final ActionListener<Void> afterCleanupsListener = new GroupedActionListener<>(ActionListener.wrap(() -> {
+                try (var refs = new RefCountingRunnable(() -> {
                     listener.onRepositoryDataWritten(newRepoData);
                     listener.onDone();
-                }), 2);
-                cleanupUnlinkedRootAndIndicesBlobs(snapshotIds, foundIndices, rootBlobs, newRepoData, afterCleanupsListener);
-                final StepListener<Collection<ShardSnapshotMetaDeleteResult>> writeMetaAndComputeDeletesStep = new StepListener<>();
-                writeUpdatedShardMetaDataAndComputeDeletes(snapshotIds, repositoryData, false, writeMetaAndComputeDeletesStep);
-                writeMetaAndComputeDeletesStep.whenComplete(
-                    deleteResults -> asyncCleanupUnlinkedShardLevelBlobs(repositoryData, snapshotIds, deleteResults, afterCleanupsListener),
-                    afterCleanupsListener::onFailure
-                );
+                })) {
+                    // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion
+                    cleanupUnlinkedRootAndIndicesBlobs(snapshotIds, foundIndices, rootBlobs, newRepoData, refs.acquireListener());
+                    writeUpdatedShardMetaDataAndComputeDeletes(
+                        snapshotIds,
+                        repositoryData,
+                        false,
+                        refs.acquireListener()
+                            .delegateFailure(
+                                (l, deleteResults) -> asyncCleanupUnlinkedShardLevelBlobs(repositoryData, snapshotIds, deleteResults, l)
+                            )
+                    );
+                }
             }, listener::onFailure));
         }
     }
@@ -1033,8 +1051,8 @@ private void writeUpdatedShardMetaDataAndComputeDeletes(
 
         // Listener that flattens out the delete results for each index
         final ActionListener<Collection<ShardSnapshotMetaDeleteResult>> deleteIndexMetadataListener = new GroupedActionListener<>(
-            onAllShardsCompleted.map(res -> res.stream().flatMap(Collection::stream).toList()),
-            indices.size()
+            indices.size(),
+            onAllShardsCompleted.map(res -> res.stream().flatMap(Collection::stream).toList())
         );
 
         for (IndexId indexId : indices) {
@@ -1048,8 +1066,8 @@ private void writeUpdatedShardMetaDataAndComputeDeletes(
                 .map(id -> oldRepositoryData.indexMetaDataGenerations().indexMetaBlobId(id, indexId))
                 .collect(Collectors.toSet());
             final ActionListener<Integer> allShardCountsListener = new GroupedActionListener<>(
-                shardCountListener,
-                indexMetaGenerations.size()
+                indexMetaGenerations.size(),
+                shardCountListener
             );
             final BlobContainer indexContainer = indexContainer(indexId);
             for (String indexMetaGeneration : indexMetaGenerations) {
@@ -1078,8 +1096,8 @@ private void writeUpdatedShardMetaDataAndComputeDeletes(
                 }
                 // Listener for collecting the results of removing the snapshot from each shard's metadata in the current index
                 final ActionListener<ShardSnapshotMetaDeleteResult> allShardsListener = new GroupedActionListener<>(
-                    deleteIndexMetadataListener,
-                    shardCount
+                    shardCount,
+                    deleteIndexMetadataListener
                 );
                 for (int shardId = 0; shardId < shardCount; shardId++) {
                     final int finalShardId = shardId;
@@ -1178,13 +1196,13 @@ private void cleanupStaleBlobs(
         RepositoryData newRepoData,
         ActionListener<DeleteResult> listener
     ) {
-        final GroupedActionListener<DeleteResult> groupedListener = new GroupedActionListener<>(ActionListener.wrap(deleteResults -> {
+        final GroupedActionListener<DeleteResult> groupedListener = new GroupedActionListener<>(2, ActionListener.wrap(deleteResults -> {
             DeleteResult deleteResult = DeleteResult.ZERO;
             for (DeleteResult result : deleteResults) {
                 deleteResult = deleteResult.add(result);
             }
             listener.onResponse(deleteResult);
-        }, listener::onFailure), 2);
+        }, listener::onFailure));
 
         final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
         final List<String> staleRootBlobs = staleRootBlobs(newRepoData, rootBlobs.keySet());
@@ -1404,7 +1422,7 @@ public void finalizeSnapshot(final FinalizeSnapshotContext finalizeSnapshotConte
                 indexMetaIdentifiers = null;
             }
 
-            final ActionListener<Void> allMetaListener = new GroupedActionListener<>(ActionListener.wrap(v -> {
+            try (var allMetaListeners = new RefCountingListener(ActionListener.wrap(v -> {
                 final String slmPolicy = slmPolicy(snapshotInfo);
                 final SnapshotDetails snapshotDetails = new SnapshotDetails(
                     snapshotInfo.state(),
@@ -1427,52 +1445,53 @@ public void finalizeSnapshot(final FinalizeSnapshotContext finalizeSnapshotConte
                         }
                     }, onUpdateFailure)
                 );
-            }, onUpdateFailure), 2 + indices.size());
-
-            // We ignore all FileAlreadyExistsException when writing metadata since otherwise a master failover while in this method will
-            // mean that no snap-${uuid}.dat blob is ever written for this snapshot. This is safe because any updated version of the
-            // index or global metadata will be compatible with the segments written in this snapshot as well.
-            // Failing on an already existing index-${repoGeneration} below ensures that the index.latest blob is not updated in a way
-            // that decrements the generation it points at
-            final Metadata clusterMetadata = finalizeSnapshotContext.clusterMetadata();
-            // Write Global MetaData
-            executor.execute(
-                ActionRunnable.run(
-                    allMetaListener,
-                    () -> GLOBAL_METADATA_FORMAT.write(clusterMetadata, blobContainer(), snapshotId.getUUID(), compress)
-                )
-            );
+            }, onUpdateFailure))) {
+
+                // We ignore all FileAlreadyExistsException when writing metadata since otherwise a master failover while in this method
+                // will mean that no snap-${uuid}.dat blob is ever written for this snapshot. This is safe because any updated version of
+                // the index or global metadata will be compatible with the segments written in this snapshot as well.
+                // Failing on an already existing index-${repoGeneration} below ensures that the index.latest blob is not updated in a way
+                // that decrements the generation it points at
+                final Metadata clusterMetadata = finalizeSnapshotContext.clusterMetadata();
+                // Write Global MetaData
+                executor.execute(
+                    ActionRunnable.run(
+                        allMetaListeners.acquire(),
+                        () -> GLOBAL_METADATA_FORMAT.write(clusterMetadata, blobContainer(), snapshotId.getUUID(), compress)
+                    )
+                );
 
-            // write the index metadata for each index in the snapshot
-            for (IndexId index : indices) {
-                executor.execute(ActionRunnable.run(allMetaListener, () -> {
-                    final IndexMetadata indexMetaData = clusterMetadata.index(index.getName());
-                    if (writeIndexGens) {
-                        final String identifiers = IndexMetaDataGenerations.buildUniqueIdentifier(indexMetaData);
-                        String metaUUID = existingRepositoryData.indexMetaDataGenerations().getIndexMetaBlobId(identifiers);
-                        if (metaUUID == null) {
-                            // We don't yet have this version of the metadata so we write it
-                            metaUUID = UUIDs.base64UUID();
-                            INDEX_METADATA_FORMAT.write(indexMetaData, indexContainer(index), metaUUID, compress);
-                            indexMetaIdentifiers.put(identifiers, metaUUID);
+                // write the index metadata for each index in the snapshot
+                for (IndexId index : indices) {
+                    executor.execute(ActionRunnable.run(allMetaListeners.acquire(), () -> {
+                        final IndexMetadata indexMetaData = clusterMetadata.index(index.getName());
+                        if (writeIndexGens) {
+                            final String identifiers = IndexMetaDataGenerations.buildUniqueIdentifier(indexMetaData);
+                            String metaUUID = existingRepositoryData.indexMetaDataGenerations().getIndexMetaBlobId(identifiers);
+                            if (metaUUID == null) {
+                                // We don't yet have this version of the metadata so we write it
+                                metaUUID = UUIDs.base64UUID();
+                                INDEX_METADATA_FORMAT.write(indexMetaData, indexContainer(index), metaUUID, compress);
+                                indexMetaIdentifiers.put(identifiers, metaUUID);
+                            }
+                            indexMetas.put(index, identifiers);
+                        } else {
+                            INDEX_METADATA_FORMAT.write(
+                                clusterMetadata.index(index.getName()),
+                                indexContainer(index),
+                                snapshotId.getUUID(),
+                                compress
+                            );
                         }
-                        indexMetas.put(index, identifiers);
-                    } else {
-                        INDEX_METADATA_FORMAT.write(
-                            clusterMetadata.index(index.getName()),
-                            indexContainer(index),
-                            snapshotId.getUUID(),
-                            compress
-                        );
-                    }
-                }));
+                    }));
+                }
+                executor.execute(
+                    ActionRunnable.run(
+                        allMetaListeners.acquire(),
+                        () -> SNAPSHOT_FORMAT.write(snapshotInfo, blobContainer(), snapshotId.getUUID(), compress)
+                    )
+                );
             }
-            executor.execute(
-                ActionRunnable.run(
-                    allMetaListener,
-                    () -> SNAPSHOT_FORMAT.write(snapshotInfo, blobContainer(), snapshotId.getUUID(), compress)
-                )
-            );
         }, onUpdateFailure);
     }
 
@@ -1787,19 +1806,7 @@ public void getRepositoryData(ActionListener<RepositoryData> listener) {
                 metadata.name(),
                 latestKnownRepoGen
             );
-            // Don't deduplicate repo data loading if we don't have strong consistency guarantees between the repo and the cluster state
-            // Also, if we are not caching repository data (for tests) we assume that the contents of the repository data at a given
-            // generation may change
-            final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT_META);
-            if (bestEffortConsistency || cacheRepositoryData == false) {
-                executor.execute(ActionRunnable.wrap(listener, this::doGetRepositoryData));
-            } else {
-                repoDataDeduplicator.executeOnce(
-                    metadata,
-                    listener,
-                    (metadata, l) -> executor.execute(ActionRunnable.wrap(l, this::doGetRepositoryData))
-                );
-            }
+            repoDataLoadDeduplicator.execute(listener);
         }
     }
 
@@ -1843,78 +1850,70 @@ private void initializeRepoGenerationTracking(ActionListener<RepositoryData> lis
                     }
                     existingListener.onFailure(e);
                 };
-                threadPool.generic()
-                    .execute(
-                        ActionRunnable.wrap(
-                            ActionListener.wrap(
-                                repoData -> submitUnbatchedTask(
-                                    "set initial safe repository generation [" + metadata.name() + "][" + repoData.getGenId() + "]",
-                                    new ClusterStateUpdateTask() {
-                                        @Override
-                                        public ClusterState execute(ClusterState currentState) {
-                                            RepositoryMetadata metadata = getRepoMetadata(currentState);
-                                            // No update to the repository generation should have occurred concurrently in general except
-                                            // for
-                                            // extreme corner cases like failing over to an older version master node and back to the
-                                            // current
-                                            // node concurrently
-                                            if (metadata.generation() != RepositoryData.UNKNOWN_REPO_GEN) {
-                                                throw new RepositoryException(
-                                                    metadata.name(),
-                                                    "Found unexpected initialized repo metadata [" + metadata + "]"
-                                                );
-                                            }
-                                            return ClusterState.builder(currentState)
-                                                .metadata(
-                                                    Metadata.builder(currentState.getMetadata())
-                                                        .putCustom(
-                                                            RepositoriesMetadata.TYPE,
-                                                            currentState.metadata()
-                                                                .<RepositoriesMetadata>custom(RepositoriesMetadata.TYPE)
-                                                                .withUpdatedGeneration(
-                                                                    metadata.name(),
-                                                                    repoData.getGenId(),
-                                                                    repoData.getGenId()
-                                                                )
-                                                        )
+                repoDataLoadDeduplicator.execute(
+                    ActionListener.wrap(
+                        repoData -> submitUnbatchedTask(
+                            "set initial safe repository generation [" + metadata.name() + "][" + repoData.getGenId() + "]",
+                            new ClusterStateUpdateTask() {
+                                @Override
+                                public ClusterState execute(ClusterState currentState) {
+                                    RepositoryMetadata metadata = getRepoMetadata(currentState);
+                                    // No update to the repository generation should have occurred concurrently in general except
+                                    // for
+                                    // extreme corner cases like failing over to an older version master node and back to the
+                                    // current
+                                    // node concurrently
+                                    if (metadata.generation() != RepositoryData.UNKNOWN_REPO_GEN) {
+                                        throw new RepositoryException(
+                                            metadata.name(),
+                                            "Found unexpected initialized repo metadata [" + metadata + "]"
+                                        );
+                                    }
+                                    return ClusterState.builder(currentState)
+                                        .metadata(
+                                            Metadata.builder(currentState.getMetadata())
+                                                .putCustom(
+                                                    RepositoriesMetadata.TYPE,
+                                                    currentState.metadata()
+                                                        .<RepositoriesMetadata>custom(RepositoriesMetadata.TYPE)
+                                                        .withUpdatedGeneration(metadata.name(), repoData.getGenId(), repoData.getGenId())
                                                 )
-                                                .build();
-                                        }
+                                        )
+                                        .build();
+                                }
 
-                                        @Override
-                                        public void onFailure(Exception e) {
-                                            onFailure.accept(e);
-                                        }
+                                @Override
+                                public void onFailure(Exception e) {
+                                    onFailure.accept(e);
+                                }
 
-                                        @Override
-                                        public void clusterStateProcessed(ClusterState oldState, ClusterState newState) {
-                                            logger.trace(
-                                                "[{}] initialized repository generation in cluster state to [{}]",
-                                                metadata.name(),
-                                                repoData.getGenId()
-                                            );
-                                            // Resolve listeners on generic pool since some callbacks for repository data do additional IO
-                                            threadPool.generic().execute(() -> {
-                                                final ActionListener<RepositoryData> existingListener;
-                                                synchronized (BlobStoreRepository.this) {
-                                                    existingListener = repoDataInitialized;
-                                                    repoDataInitialized = null;
-                                                }
-                                                existingListener.onResponse(repoData);
-                                                logger.trace(
-                                                    "[{}] called listeners after initializing repository to generation [{}]",
-                                                    metadata.name(),
-                                                    repoData.getGenId()
-                                                );
-                                            });
+                                @Override
+                                public void clusterStateProcessed(ClusterState oldState, ClusterState newState) {
+                                    logger.trace(
+                                        "[{}] initialized repository generation in cluster state to [{}]",
+                                        metadata.name(),
+                                        repoData.getGenId()
+                                    );
+                                    // Resolve listeners on generic pool since some callbacks for repository data do additional IO
+                                    threadPool.generic().execute(() -> {
+                                        final ActionListener<RepositoryData> existingListener;
+                                        synchronized (BlobStoreRepository.this) {
+                                            existingListener = repoDataInitialized;
+                                            repoDataInitialized = null;
                                         }
-                                    }
-                                ),
-                                onFailure
-                            ),
-                            this::doGetRepositoryData
-                        )
-                    );
+                                        existingListener.onResponse(repoData);
+                                        logger.trace(
+                                            "[{}] called listeners after initializing repository to generation [{}]",
+                                            metadata.name(),
+                                            repoData.getGenId()
+                                        );
+                                    });
+                                }
+                            }
+                        ),
+                        onFailure
+                    )
+                );
             } else {
                 logger.trace(
                     "[{}] waiting for existing initialization of repository metadata generation in cluster state",
@@ -1926,11 +1925,9 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState)
     }
 
     /**
-     * {@link RepositoryData} loading deduplicator. This may only be used with consistent generation repositories, meaning
-     * {@link #bestEffortConsistency} must be {@code false}, in which case we can assume that the {@link RepositoryData} loaded is
-     * unique for a given value of {@link #metadata} at any point in time.
+     * Deduplicator that deduplicates the physical loading of {@link RepositoryData} from the repositories' underlying storage.
      */
-    private final ResultDeduplicator<RepositoryMetadata, RepositoryData> repoDataDeduplicator;
+    private final SingleResultDeduplicator<RepositoryData> repoDataLoadDeduplicator;
 
     private void doGetRepositoryData(ActionListener<RepositoryData> listener) {
         // Retry loading RepositoryData in a loop in case we run into concurrent modifications of the repository.
@@ -1985,7 +1982,7 @@ private void doGetRepositoryData(ActionListener<RepositoryData> listener) {
                             clusterService,
                             metadata.name(),
                             loaded,
-                            new ThreadedActionListener<>(logger, threadPool, ThreadPool.Names.GENERIC, listener.map(v -> loaded), false)
+                            new ThreadedActionListener<>(threadPool.generic(), listener.map(v -> loaded))
                         );
                     }
                 }
@@ -2063,20 +2060,12 @@ private void cacheRepositoryData(RepositoryData repositoryData, Version version)
     }
 
     private RepositoryException corruptedStateException(@Nullable Exception cause, @Nullable Tuple<Long, String> previousWriterInfo) {
-        return new RepositoryException(
-            metadata.name(),
-            "The repository has been disabled to prevent data corruption because its contents were found not to match its expected state. "
-                + "This is either because something other than this cluster modified the repository contents, or because the repository's "
-                + "underlying storage behaves incorrectly. To re-enable this repository, first ensure that this cluster has exclusive "
-                + "write access to it, and then re-register the repository with this cluster. "
-                + "See https://www.elastic.co/guide/en/elasticsearch/reference/"
-                + Version.CURRENT.major
-                + "."
-                + Version.CURRENT.minor
-                + "/add-repository.html for further information."
-                + previousWriterMessage(previousWriterInfo),
-            cause
-        );
+        return new RepositoryException(metadata.name(), Strings.format("""
+            The repository has been disabled to prevent data corruption because its contents were found not to match its expected state. \
+            This is either because something other than this cluster modified the repository contents, or because the repository's \
+            underlying storage behaves incorrectly. To re-enable this repository, first ensure that this cluster has exclusive write \
+            access to it, and then re-register the repository with this cluster. See %s for further information.\
+            %s""", ReferenceDocs.CONCURRENT_REPOSITORY_WRITERS, previousWriterMessage(previousWriterInfo)), cause);
     }
 
     private static String previousWriterMessage(@Nullable Tuple<Long, String> previousWriterInfo) {
@@ -3164,7 +3153,7 @@ private static ActionListener<Void> fileQueueListener(
         int numberOfFiles,
         ActionListener<Collection<Void>> listener
     ) {
-        return new GroupedActionListener<>(listener, numberOfFiles).delegateResponse((l, e) -> {
+        return new GroupedActionListener<>(numberOfFiles, listener).delegateResponse((l, e) -> {
             files.clear(); // Stop uploading the remaining files if we run into any exception
             l.onFailure(e);
         });
diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java
index 53765f452657..5e4f429a9076 100644
--- a/server/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java
+++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java
@@ -41,12 +41,13 @@
 import java.util.Collections;
 import java.util.Locale;
 import java.util.Map;
+import java.util.function.Function;
 import java.util.zip.CRC32;
 
 /**
  * Snapshot metadata file format used in v2.0 and above
  */
-public final class ChecksumBlobStoreFormat<T extends ToXContent> {
+public final class ChecksumBlobStoreFormat<T> {
 
     // Serialization parameters to specify correct context for metadata serialization.
     // When metadata is serialized certain elements of the metadata shouldn't be included into snapshot
@@ -68,31 +69,42 @@ public final class ChecksumBlobStoreFormat<T extends ToXContent> {
 
     private final CheckedBiFunction<String, XContentParser, T, IOException> fallbackReader;
 
+    private final Function<T, ? extends ToXContent> writer;
+
     /**
      * @param codec          codec name
      * @param blobNameFormat format of the blobname in {@link String#format} format
      * @param reader         prototype object that can deserialize T from XContent
      * @param fallbackReader fallback prototype object that can deserialize T from XContent in case reader fails
+     * @param writer         function that maps the given type to a {@link ToXContent} for serialization
      */
     public ChecksumBlobStoreFormat(
         String codec,
         String blobNameFormat,
         CheckedBiFunction<String, XContentParser, T, IOException> reader,
-        @Nullable CheckedBiFunction<String, XContentParser, T, IOException> fallbackReader
+        @Nullable CheckedBiFunction<String, XContentParser, T, IOException> fallbackReader,
+        Function<T, ? extends ToXContent> writer
     ) {
         this.reader = reader;
         this.blobNameFormat = blobNameFormat;
         this.codec = codec;
         this.fallbackReader = fallbackReader;
+        this.writer = writer;
     }
 
     /**
      * @param codec          codec name
      * @param blobNameFormat format of the blobname in {@link String#format} format
      * @param reader         prototype object that can deserialize T from XContent
+     * @param writer         function that maps the given type to a {@link ToXContent} for serialization
      */
-    public ChecksumBlobStoreFormat(String codec, String blobNameFormat, CheckedBiFunction<String, XContentParser, T, IOException> reader) {
-        this(codec, blobNameFormat, reader, null);
+    public ChecksumBlobStoreFormat(
+        String codec,
+        String blobNameFormat,
+        CheckedBiFunction<String, XContentParser, T, IOException> reader,
+        Function<T, ? extends ToXContent> writer
+    ) {
+        this(codec, blobNameFormat, reader, null, writer);
     }
 
     /**
@@ -289,13 +301,13 @@ private int getAvailable() throws IOException {
             final int footerLen = CodecUtil.footerLength();
             if (bufferCount == 0) {
                 // first read, fill the buffer
-                bufferCount = Streams.readFully(in, buffer, 0, buffer.length);
+                bufferCount = org.elasticsearch.core.Streams.readFully(in, buffer, 0, buffer.length);
             } else if (bufferPos == bufferCount - footerLen) {
                 // crc and discard all but the last 16 bytes in the buffer that might be the footer bytes
                 assert bufferCount >= footerLen;
                 crc32.update(buffer, 0, bufferPos);
                 System.arraycopy(buffer, bufferPos, buffer, 0, footerLen);
-                bufferCount = footerLen + Streams.readFully(in, buffer, footerLen, buffer.length - footerLen);
+                bufferCount = footerLen + org.elasticsearch.core.Streams.readFully(in, buffer, footerLen, buffer.length - footerLen);
                 bufferPos = 0;
             }
             // bytes in the buffer minus 16 bytes that could be the footer
@@ -371,7 +383,7 @@ public void close() {
                     : new ToXContent.DelegatingMapParams(extraParams, SNAPSHOT_ONLY_FORMAT_PARAMS);
 
                 builder.startObject();
-                obj.toXContent(builder, params);
+                writer.apply(obj).toXContent(builder, params);
                 builder.endObject();
             }
             CodecUtil.writeFooter(indexOutput);
diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/ShardSnapshotTaskRunner.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/ShardSnapshotTaskRunner.java
index 7c0a2ed9c65a..bd4084f1a401 100644
--- a/server/src/main/java/org/elasticsearch/repositories/blobstore/ShardSnapshotTaskRunner.java
+++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/ShardSnapshotTaskRunner.java
@@ -8,10 +8,14 @@
 
 package org.elasticsearch.repositories.blobstore;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.common.CheckedBiConsumer;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.PrioritizedThrottledTaskRunner;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.repositories.SnapshotShardContext;
 
 import java.io.IOException;
@@ -28,11 +32,12 @@
  * and zero or more {@link FileSnapshotTask}s.
  */
 public class ShardSnapshotTaskRunner {
+    private static final Logger logger = LogManager.getLogger(ShardSnapshotTaskRunner.class);
     private final PrioritizedThrottledTaskRunner<SnapshotTask> taskRunner;
     private final Consumer<SnapshotShardContext> shardSnapshotter;
     private final CheckedBiConsumer<SnapshotShardContext, FileInfo, IOException> fileSnapshotter;
 
-    abstract static class SnapshotTask implements Comparable<SnapshotTask>, Runnable {
+    abstract static class SnapshotTask extends AbstractRunnable implements Comparable<SnapshotTask> {
 
         private static final Comparator<SnapshotTask> COMPARATOR = Comparator.comparingLong(
             (SnapshotTask t) -> t.context().snapshotStartTime()
@@ -54,6 +59,12 @@ public SnapshotShardContext context() {
         public final int compareTo(SnapshotTask other) {
             return COMPARATOR.compare(this, other);
         }
+
+        @Override
+        public void onFailure(Exception e) {
+            assert false : e;
+            logger.error(Strings.format("snapshot task [%s] unexpectedly failed", this), e);
+        }
     }
 
     class ShardSnapshotTask extends SnapshotTask {
@@ -62,7 +73,7 @@ class ShardSnapshotTask extends SnapshotTask {
         }
 
         @Override
-        public void run() {
+        public void doRun() {
             shardSnapshotter.accept(context);
         }
 
@@ -88,7 +99,7 @@ class FileSnapshotTask extends SnapshotTask {
         }
 
         @Override
-        public void run() {
+        public void doRun() {
             ActionRunnable.run(fileSnapshotListener, () -> {
                 FileInfo fileInfo = fileInfos.get();
                 if (fileInfo != null) {
diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsChangedListener.java b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsChangedListener.java
new file mode 100644
index 000000000000..1d6d2c85c55e
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsChangedListener.java
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.reservedstate.service;
+
+/**
+ * Listener interface for the file settings service. Listeners will get
+ * notified when the settings have been updated, or if there are no settings
+ * on initial start.
+ */
+public interface FileSettingsChangedListener {
+    void settingsChanged();
+}
diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java
index d65531650ea7..d158beb4aee2 100644
--- a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java
+++ b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java
@@ -10,16 +10,13 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
-import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
-import org.elasticsearch.client.internal.ClusterAdminClient;
-import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
@@ -37,7 +34,8 @@
 import java.nio.file.attribute.BasicFileAttributes;
 import java.nio.file.attribute.FileTime;
 import java.time.Instant;
-import java.util.concurrent.CompletableFuture;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.ExecutionException;
 
 import static org.elasticsearch.xcontent.XContentType.JSON;
@@ -63,7 +61,6 @@ public class FileSettingsService extends AbstractLifecycleComponent implements C
     private final ClusterService clusterService;
     private final ReservedClusterStateService stateService;
     private final Path operatorSettingsDir;
-    private final NodeClient nodeClient;
 
     private WatchService watchService; // null;
     private Thread watcherThread;
@@ -72,12 +69,10 @@ public class FileSettingsService extends AbstractLifecycleComponent implements C
     private WatchKey configDirWatchKey;
 
     private volatile boolean active = false;
-    private volatile boolean initialState = true;
 
     public static final String OPERATOR_DIRECTORY = "operator";
 
-    private volatile NodesInfoResponse nodesInfoResponse = null;
-    private volatile boolean nodeInfosRefreshRequired = true;
+    private final List<FileSettingsChangedListener> eventListeners;
 
     /**
      * Constructs the {@link FileSettingsService}
@@ -86,16 +81,11 @@ public class FileSettingsService extends AbstractLifecycleComponent implements C
      * @param stateService an instance of the immutable cluster state controller, so we can perform the cluster state changes
      * @param environment we need the environment to pull the location of the config and operator directories
      */
-    public FileSettingsService(
-        ClusterService clusterService,
-        ReservedClusterStateService stateService,
-        Environment environment,
-        NodeClient nodeClient
-    ) {
+    public FileSettingsService(ClusterService clusterService, ReservedClusterStateService stateService, Environment environment) {
         this.clusterService = clusterService;
         this.stateService = stateService;
         this.operatorSettingsDir = environment.configFile().toAbsolutePath().resolve(OPERATOR_DIRECTORY);
-        this.nodeClient = nodeClient;
+        this.eventListeners = new CopyOnWriteArrayList<>();
     }
 
     public Path operatorSettingsDir() {
@@ -127,8 +117,13 @@ protected void doStart() {
         // We need the additional active flag, since cluster state can change after we've shutdown the service
         // causing the watcher to start again.
         this.active = Files.exists(operatorSettingsDir().getParent());
-        startIfMaster(clusterService.state());
-        clusterService.addListener(this);
+        if (active == false) {
+            // we don't have a config directory, we can't possibly launch the file settings service
+            return;
+        }
+        if (DiscoveryNode.isMasterNode(clusterService.getSettings())) {
+            clusterService.addListener(this);
+        }
     }
 
     @Override
@@ -149,16 +144,14 @@ private boolean currentNodeMaster(ClusterState clusterState) {
     public void clusterChanged(ClusterChangedEvent event) {
         ClusterState clusterState = event.state();
         startIfMaster(clusterState);
-        checkForNodeChanges(event);
     }
 
     private void startIfMaster(ClusterState clusterState) {
         if (currentNodeMaster(clusterState)) {
-            startWatcher(clusterState, initialState);
+            startWatcher(clusterState);
         } else {
             stopWatcher();
         }
-        initialState = false;
     }
 
     /**
@@ -221,7 +214,7 @@ public boolean watching() {
         return watcherThread != null;
     }
 
-    synchronized void startWatcher(ClusterState clusterState, boolean onStartup) {
+    synchronized void startWatcher(ClusterState clusterState) {
         if (watching() || active == false) {
             refreshExistingFileStateIfNeeded(clusterState);
 
@@ -239,21 +232,6 @@ synchronized void startWatcher(ClusterState clusterState, boolean onStartup) {
             Path settingsDirPath = operatorSettingsDir();
             this.watchService = settingsDirPath.getParent().getFileSystem().newWatchService();
             if (Files.exists(settingsDirPath)) {
-                Path settingsFilePath = operatorSettingsFile();
-                if (Files.exists(settingsFilePath)) {
-                    logger.debug("found initial operator settings file [{}], applying...", settingsFilePath);
-                    // we make a distinction here for startup, so that if we had operator settings before the node started
-                    // we would fail startup.
-                    try {
-                        processFileSettings(settingsFilePath).get();
-                    } catch (ExecutionException e) {
-                        if (onStartup) {
-                            throw new FileSettingsStartupException("Error applying operator settings", e.getCause());
-                        } else {
-                            logger.error("Error processing operator settings json file", e.getCause());
-                        }
-                    }
-                }
                 settingsDirWatchKey = enableSettingsWatcher(settingsDirWatchKey, settingsDirPath);
             } else {
                 logger.debug("operator settings directory [{}] not found, will watch for its creation...", settingsDirPath);
@@ -285,6 +263,18 @@ private void watcherThread() {
         try {
             logger.info("file settings service up and running [tid={}]", Thread.currentThread().getId());
 
+            Path path = operatorSettingsFile();
+
+            if (Files.exists(path)) {
+                logger.debug("found initial operator settings file [{}], applying...", path);
+                processSettingsAndNotifyListeners();
+            } else {
+                // Notify everyone we don't have any initial file settings
+                for (var listener : eventListeners) {
+                    listener.settingsChanged();
+                }
+            }
+
             WatchKey key;
             while ((key = watchService.take()) != null) {
                 /*
@@ -304,8 +294,6 @@ private void watcherThread() {
                 Path settingsPath = operatorSettingsDir();
                 if (Files.exists(settingsPath)) {
                     try {
-                        Path path = operatorSettingsFile();
-
                         if (logger.isDebugEnabled()) {
                             key.pollEvents().forEach(e -> logger.debug("{}:{}", e.kind().toString(), e.context().toString()));
                         } else {
@@ -320,11 +308,7 @@ private void watcherThread() {
                         settingsDirWatchKey = enableSettingsWatcher(settingsDirWatchKey, settingsPath);
 
                         if (watchedFileChanged(path)) {
-                            try {
-                                processFileSettings(path).get();
-                            } catch (ExecutionException e) {
-                                logger.error("Error processing operator settings json file", e.getCause());
-                            }
+                            processSettingsAndNotifyListeners();
                         }
                     } catch (IOException e) {
                         logger.warn("encountered I/O error while watching file settings", e);
@@ -341,9 +325,21 @@ private void watcherThread() {
         }
     }
 
+    // package private for testing
+    void processSettingsAndNotifyListeners() throws InterruptedException {
+        try {
+            processFileSettings(operatorSettingsFile()).get();
+            for (var listener : eventListeners) {
+                listener.settingsChanged();
+            }
+        } catch (ExecutionException e) {
+            logger.error("Error processing operator settings json file", e.getCause());
+        }
+    }
+
     synchronized void stopWatcher() {
-        logger.debug("stopping watcher ...");
         if (watching()) {
+            logger.debug("stopping watcher ...");
             // make sure watch service is closed whatever
             // this will also close any outstanding keys
             try (var ws = watchService) {
@@ -369,7 +365,7 @@ synchronized void stopWatcher() {
                 logger.info("watcher service stopped");
             }
         } else {
-            logger.debug("file settings service already stopped");
+            logger.trace("file settings service already stopped");
         }
     }
 
@@ -404,52 +400,27 @@ WatchKey enableSettingsWatcher(WatchKey previousKey, Path settingsDir) throws IO
         } while (true);
     }
 
-    CompletableFuture<Void> processFileSettings(Path path) {
-        CompletableFuture<Void> completion = new CompletableFuture<>();
+    PlainActionFuture<Void> processFileSettings(Path path) {
+        PlainActionFuture<Void> completion = PlainActionFuture.newFuture();
         logger.info("processing path [{}] for [{}]", path, NAMESPACE);
         try (
             var fis = Files.newInputStream(path);
             var bis = new BufferedInputStream(fis);
             var parser = JSON.xContent().createParser(XContentParserConfiguration.EMPTY, bis)
         ) {
-            ReservedStateChunk parsedState = stateService.parse(NAMESPACE, parser);
-            if (nodeInfosRefreshRequired || nodesInfoResponse == null) {
-                var nodesInfoRequest = NodesInfoRequest.requestWithMetrics(NodesInfoRequest.Metric.INGEST);
-
-                clusterAdminClient().nodesInfo(nodesInfoRequest, new ActionListener<>() {
-                    @Override
-                    public void onResponse(NodesInfoResponse response) {
-                        // stash the latest node infos response and continue with processing the file
-                        nodesInfoResponse = response;
-                        nodeInfosRefreshRequired = false;
-                        stateService.process(NAMESPACE, parsedState, (e) -> completeProcessing(e, completion));
-                    }
-
-                    @Override
-                    public void onFailure(Exception e) {
-                        completion.completeExceptionally(e);
-                    }
-                });
-            } else {
-                stateService.process(NAMESPACE, parsedState, (e) -> completeProcessing(e, completion));
-            }
+            stateService.process(NAMESPACE, parser, (e) -> completeProcessing(e, completion));
         } catch (Exception e) {
-            completion.completeExceptionally(e);
+            completion.onFailure(e);
         }
 
         return completion;
     }
 
-    // package private for testing, separate method so that it can be mocked in tests
-    ClusterAdminClient clusterAdminClient() {
-        return nodeClient.admin().cluster();
-    }
-
-    private void completeProcessing(Exception e, CompletableFuture<Void> completion) {
+    private void completeProcessing(Exception e, PlainActionFuture<Void> completion) {
         if (e != null) {
-            completion.completeExceptionally(e);
+            completion.onFailure(e);
         } else {
-            completion.complete(null);
+            completion.onResponse(null);
         }
     }
 
@@ -459,23 +430,7 @@ private void completeProcessing(Exception e, CompletableFuture<Void> completion)
      */
     record FileUpdateState(long timestamp, String path, Object fileKey) {}
 
-    /**
-     * Error subclass that is thrown when we encounter a fatal error while applying
-     * the operator cluster state at Elasticsearch boot time.
-     */
-    public static class FileSettingsStartupException extends RuntimeException {
-        public FileSettingsStartupException(String message, Throwable t) {
-            super(message, t);
-        }
-    }
-
-    void checkForNodeChanges(ClusterChangedEvent event) {
-        if (currentNodeMaster(event.state()) && event.nodesChanged()) {
-            nodeInfosRefreshRequired = true;
-        }
-    }
-
-    public NodesInfoResponse nodeInfos() {
-        return nodesInfoResponse;
+    public void addFileSettingsChangedListener(FileSettingsChangedListener listener) {
+        eventListeners.add(listener);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java
index 0a3a9f447d0e..f5d9c663a358 100644
--- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java
+++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java
@@ -12,7 +12,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.action.support.GroupedActionListener;
+import org.elasticsearch.action.support.RefCountingListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateTaskConfig;
 import org.elasticsearch.cluster.metadata.ReservedStateErrorMetadata;
@@ -29,6 +29,7 @@
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -345,29 +346,14 @@ void executeNonStateTransformationSteps(
         List<Consumer<ActionListener<NonStateTransformResult>>> nonStateTransforms,
         ActionListener<Collection<NonStateTransformResult>> listener
     ) {
-        // Don't create grouped listener with 0 actions, just return
-        if (nonStateTransforms.isEmpty()) {
-            listener.onResponse(List.of());
-            return;
-        }
-
-        GroupedActionListener<NonStateTransformResult> postTasksListener = new GroupedActionListener<>(new ActionListener<>() {
-            @Override
-            public void onResponse(Collection<NonStateTransformResult> updateKeyTaskResult) {
-                listener.onResponse(updateKeyTaskResult);
+        final List<NonStateTransformResult> result = Collections.synchronizedList(new ArrayList<>(nonStateTransforms.size()));
+        try (var listeners = new RefCountingListener(listener.map(ignored -> result))) {
+            for (var transform : nonStateTransforms) {
+                // non cluster state transforms don't modify the cluster state, they however are given a chance to return a more
+                // up-to-date version of the modified keys we should save in the reserved state. These calls are
+                // async and report back when they are done through the postTasksListener.
+                transform.accept(listeners.acquire(result::add));
             }
-
-            @Override
-            public void onFailure(Exception e) {
-                listener.onFailure(e);
-            }
-        }, nonStateTransforms.size());
-
-        for (var transform : nonStateTransforms) {
-            // non cluster state transforms don't modify the cluster state, they however are given a chance to return a more
-            // up-to-date version of the modified keys we should save in the reserved state. These calls are
-            // async and report back when they are done through the postTasksListener.
-            transform.accept(postTasksListener);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateVersion.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateVersion.java
index c7c21b458482..8eb2e17684d9 100644
--- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateVersion.java
+++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateVersion.java
@@ -9,15 +9,20 @@
 package org.elasticsearch.reservedstate.service;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentParser;
 
+import java.io.IOException;
+
 /**
  * File settings metadata class that holds information about
  * versioning and Elasticsearch version compatibility
  */
-public record ReservedStateVersion(Long version, Version compatibleWith) {
+public record ReservedStateVersion(Long version, Version compatibleWith) implements Writeable {
 
     public static final ParseField VERSION = new ParseField("version");
     public static final ParseField COMPATIBILITY = new ParseField("compatibility");
@@ -44,4 +49,14 @@ public static ReservedStateVersion parse(XContentParser parser) {
     public Version minCompatibleVersion() {
         return compatibleWith;
     }
+
+    public static ReservedStateVersion readFrom(StreamInput input) throws IOException {
+        return new ReservedStateVersion(input.readLong(), Version.readVersion(input));
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeLong(version());
+        Version.writeVersion(compatibleWith(), out);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBody.java b/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBody.java
index 746727e29555..d6efaf43d2ec 100644
--- a/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBody.java
+++ b/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBody.java
@@ -14,6 +14,8 @@
 import org.elasticsearch.common.recycler.Recycler;
 import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.Streams;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -81,7 +83,9 @@ public void write(byte[] b, int off, int len) throws IOException {
                 Streams.noCloseStream(out)
             );
 
-            private final Iterator<? extends ToXContent> serialization = chunkedToXContent.toXContentChunked();
+            private final Iterator<? extends ToXContent> serialization = builder.getRestApiVersion() == RestApiVersion.V_7
+                ? chunkedToXContent.toXContentChunkedV7(params)
+                : chunkedToXContent.toXContentChunked(params);
 
             private BytesStream target;
 
@@ -92,20 +96,32 @@ public boolean isDone() {
 
             @Override
             public ReleasableBytesReference encodeChunk(int sizeHint, Recycler<BytesRef> recycler) throws IOException {
-                final RecyclerBytesStreamOutput chunkStream = new RecyclerBytesStreamOutput(recycler);
-                assert this.target == null;
-                this.target = chunkStream;
-                while (serialization.hasNext()) {
-                    serialization.next().toXContent(builder, params);
-                    if (chunkStream.size() >= sizeHint) {
-                        break;
+                try {
+                    final RecyclerBytesStreamOutput chunkStream = new RecyclerBytesStreamOutput(recycler);
+                    assert target == null;
+                    target = chunkStream;
+                    while (serialization.hasNext()) {
+                        serialization.next().toXContent(builder, params);
+                        if (chunkStream.size() >= sizeHint) {
+                            break;
+                        }
+                    }
+                    if (serialization.hasNext() == false) {
+                        builder.close();
+                    }
+                    final var result = new ReleasableBytesReference(
+                        chunkStream.bytes(),
+                        () -> Releasables.closeExpectNoException(chunkStream)
+                    );
+                    target = null;
+                    return result;
+                } finally {
+                    if (target != null) {
+                        assert false : "failure encoding chunk";
+                        IOUtils.closeWhileHandlingException(target);
+                        target = null;
                     }
                 }
-                if (serialization.hasNext() == false) {
-                    builder.close();
-                }
-                this.target = null;
-                return new ReleasableBytesReference(chunkStream.bytes(), () -> IOUtils.closeWhileHandlingException(chunkStream));
             }
 
             @Override
diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java
index 81825c790cc1..4b3223bd7704 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestController.java
@@ -44,6 +44,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Supplier;
@@ -70,6 +71,7 @@ public class RestController implements HttpServerTransport.Dispatcher {
 
     static final String ELASTIC_PRODUCT_HTTP_HEADER = "X-elastic-product";
     static final String ELASTIC_PRODUCT_HTTP_HEADER_VALUE = "Elasticsearch";
+    static final Set<String> RESERVED_PATHS = Set.of("/__elb_health__", "/__elb_health__/zk", "/_health", "/_health/zk");
 
     private static final BytesReference FAVICON_RESPONSE;
 
@@ -263,6 +265,9 @@ private void registerHandlerNoWrap(RestRequest.Method method, String path, RestA
         assert RestApiVersion.minimumSupported() == version || RestApiVersion.current() == version
             : "REST API compatibility is only supported for version " + RestApiVersion.minimumSupported().major;
 
+        if (RESERVED_PATHS.contains(path)) {
+            throw new IllegalArgumentException("path [" + path + "] is a reserved path and may not be registered");
+        }
         handlers.insertOrUpdate(
             path,
             new MethodHandlers(path).addMethod(method, version, handler),
@@ -460,14 +465,10 @@ private void startTrace(ThreadContext threadContext, RestChannel channel, String
             name = restPath;
         }
 
-        Map<String, Object> attributes = Maps.newMapWithExpectedSize(req.getHeaders().size() + 3);
+        final Map<String, Object> attributes = Maps.newMapWithExpectedSize(req.getHeaders().size() + 3);
         req.getHeaders().forEach((key, values) -> {
             final String lowerKey = key.toLowerCase(Locale.ROOT).replace('-', '_');
-            final String value = switch (lowerKey) {
-                case "authorization", "cookie", "secret", "session", "set_cookie", "token" -> "[REDACTED]";
-                default -> String.join("; ", values);
-            };
-            attributes.put("http.request.headers." + lowerKey, value);
+            attributes.put("http.request.headers." + lowerKey, values.size() == 1 ? values.get(0) : String.join("; ", values));
         });
         attributes.put("http.method", method);
         attributes.put("http.url", req.uri());
@@ -561,8 +562,9 @@ private void copyRestHeaders(RestRequest request, ThreadContext threadContext) {
                     throw new IllegalArgumentException("multiple values for single-valued header [" + name + "].");
                 } else if (name.equals(Task.TRACE_PARENT_HTTP_HEADER)) {
                     String traceparent = distinctHeaderValues.get(0);
-                    if (traceparent.length() >= 55) {
-                        threadContext.putHeader(Task.TRACE_ID, traceparent.substring(3, 35));
+                    Optional<String> traceId = RestUtils.extractTraceId(traceparent);
+                    if (traceId.isPresent()) {
+                        threadContext.putHeader(Task.TRACE_ID, traceId.get());
                         threadContext.putTransient("parent_" + Task.TRACE_PARENT_HTTP_HEADER, traceparent);
                     }
                 } else if (name.equals(Task.TRACE_STATE)) {
@@ -661,7 +663,7 @@ private static void handleOptionsRequest(RestChannel channel, Set<RestRequest.Me
      * Handle a requests with no candidate handlers (return a 400 Bad Request
      * error).
      */
-    private static void handleBadRequest(String uri, RestRequest.Method method, RestChannel channel) throws IOException {
+    public static void handleBadRequest(String uri, RestRequest.Method method, RestChannel channel) throws IOException {
         try (XContentBuilder builder = channel.newErrorBuilder()) {
             builder.startObject();
             {
diff --git a/server/src/main/java/org/elasticsearch/rest/RestResponse.java b/server/src/main/java/org/elasticsearch/rest/RestResponse.java
index 70618dad3df1..94f72687f077 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestResponse.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestResponse.java
@@ -122,7 +122,7 @@ public RestResponse(RestChannel channel, RestStatus status, Exception e) throws
         try (XContentBuilder builder = channel.newErrorBuilder()) {
             build(builder, params, status, channel.detailedErrorsEnabled(), e);
             this.content = BytesReference.bytes(builder);
-            this.responseMediaType = builder.contentType().mediaType();
+            this.responseMediaType = builder.getResponseContentTypeString();
         }
         if (e instanceof ElasticsearchException) {
             copyHeaders(((ElasticsearchException) e));
diff --git a/server/src/main/java/org/elasticsearch/rest/RestUtils.java b/server/src/main/java/org/elasticsearch/rest/RestUtils.java
index 80d68161243d..39ef8200424b 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestUtils.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestUtils.java
@@ -16,6 +16,7 @@
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Map;
+import java.util.Optional;
 import java.util.regex.Pattern;
 
 public class RestUtils {
@@ -236,6 +237,18 @@ public static String[] corsSettingAsArray(String corsSetting) {
         if (Strings.isNullOrEmpty(corsSetting)) {
             return new String[0];
         }
-        return Arrays.asList(corsSetting.split(",")).stream().map(String::trim).toArray(size -> new String[size]);
+        return Arrays.stream(corsSetting.split(",")).map(String::trim).toArray(String[]::new);
     }
+
+    /**
+     * Extract the trace id from the specified traceparent string.
+     * @see <a href="https://www.w3.org/TR/trace-context/#traceparent-header">W3 traceparent spec</a>
+     *
+     * @param traceparent   The value from the {@code traceparent} HTTP header
+     * @return  The trace id from the traceparent string, or {@code Optional.empty()} if it is not present.
+     */
+    public static Optional<String> extractTraceId(String traceparent) {
+        return traceparent != null && traceparent.length() >= 55 ? Optional.of(traceparent.substring(3, 35)) : Optional.empty();
+    }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/DispatchingRestToXContentListener.java b/server/src/main/java/org/elasticsearch/rest/action/DispatchingRestToXContentListener.java
deleted file mode 100644
index be4cbd120dcb..000000000000
--- a/server/src/main/java/org/elasticsearch/rest/action/DispatchingRestToXContentListener.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.rest.action;
-
-import org.elasticsearch.action.ActionRunnable;
-import org.elasticsearch.rest.RestChannel;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.util.concurrent.ExecutorService;
-
-/**
- * Response listener for REST requests which dispatches the serialization of the response off of the thread on which the response was
- * received, since that thread is often a transport thread and XContent serialization might be expensive.
- */
-public class DispatchingRestToXContentListener<Response extends ToXContentObject> extends RestActionListener<Response> {
-
-    private final ExecutorService executor;
-    private final RestRequest restRequest;
-
-    public DispatchingRestToXContentListener(ExecutorService executor, RestChannel channel, RestRequest restRequest) {
-        super(channel);
-        this.executor = executor;
-        this.restRequest = restRequest;
-    }
-
-    protected ToXContent.Params getParams() {
-        return restRequest;
-    }
-
-    @Override
-    protected void processResponse(Response response) {
-        executor.execute(ActionRunnable.wrap(this, l -> new RestBuilderListener<Response>(channel) {
-            @Override
-            public RestResponse buildResponse(final Response response, final XContentBuilder builder) throws Exception {
-                ensureOpen();
-                response.toXContent(builder, getParams());
-                return new RestResponse(RestStatus.OK, builder);
-            }
-        }.onResponse(response)));
-    }
-
-}
diff --git a/server/src/main/java/org/elasticsearch/rest/action/RestChunkedToXContentListener.java b/server/src/main/java/org/elasticsearch/rest/action/RestChunkedToXContentListener.java
index ece3a432f059..27e2b70feecf 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/RestChunkedToXContentListener.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/RestChunkedToXContentListener.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
 
@@ -22,12 +23,19 @@
  */
 public final class RestChunkedToXContentListener<Response extends ChunkedToXContent> extends RestActionListener<Response> {
 
+    private final ToXContent.Params params;
+
     public RestChunkedToXContentListener(RestChannel channel) {
+        this(channel, channel.request());
+    }
+
+    public RestChunkedToXContentListener(RestChannel channel, ToXContent.Params params) {
         super(channel);
+        this.params = params;
     }
 
     @Override
     protected void processResponse(Response response) throws IOException {
-        channel.sendResponse(new RestResponse(RestStatus.OK, ChunkedRestResponseBody.fromXContent(response, channel.request(), channel)));
+        channel.sendResponse(new RestResponse(RestStatus.OK, ChunkedRestResponseBody.fromXContent(response, params, channel)));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java
index d583d001e5c2..ac58014909a1 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java
@@ -67,7 +67,10 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
             }
             fieldRequest.fields(Strings.splitStringByCommaToArray(request.param("fields")));
         }
-        return channel -> client.fieldCaps(fieldRequest, new RestChunkedToXContentListener<>(channel));
+        return channel -> {
+            RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel());
+            cancelClient.fieldCaps(fieldRequest, new RestChunkedToXContentListener<>(channel));
+        };
     }
 
     private static final ParseField INDEX_FILTER_FIELD = new ParseField("index_filter");
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterRerouteAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterRerouteAction.java
index a04e89bba8d2..1dca904c5582 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterRerouteAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterRerouteAction.java
@@ -18,7 +18,7 @@
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.rest.action.RestChunkedToXContentListener;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.xcontent.ParseField;
@@ -82,7 +82,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         if (metric == null) {
             request.params().put("metric", DEFAULT_METRICS);
         }
-        return channel -> client.admin().cluster().reroute(clusterRerouteRequest, new RestToXContentListener<>(channel));
+        return channel -> client.admin().cluster().reroute(clusterRerouteRequest, new RestChunkedToXContentListener<>(channel));
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java
index c3a79c1cc42e..7928f3504930 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java
@@ -18,19 +18,22 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.DispatchingRestToXContentListener;
 import org.elasticsearch.rest.action.RestCancellableNodeClient;
+import org.elasticsearch.rest.action.RestChunkedToXContentListener;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.EnumSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 import java.util.function.LongSupplier;
@@ -108,21 +111,10 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute(
             ClusterStateAction.INSTANCE,
             clusterStateRequest,
-            new DispatchingRestToXContentListener<RestClusterStateResponse>(
-                // Process serialization on MANAGEMENT pool since the serialization of the cluster state to XContent
-                // can be too slow to execute on an IO thread
-                threadPool.executor(ThreadPool.Names.MANAGEMENT),
+            new RestChunkedToXContentListener<RestClusterStateResponse>(
                 channel,
-                request
-            ) {
-                @Override
-                protected ToXContent.Params getParams() {
-                    return new ToXContent.DelegatingMapParams(
-                        singletonMap(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API),
-                        request
-                    );
-                }
-            }.map(response -> new RestClusterStateResponse(clusterStateRequest, response, threadPool::relativeTimeInMillis))
+                new ToXContent.DelegatingMapParams(singletonMap(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API), request)
+            ).map(response -> new RestClusterStateResponse(clusterStateRequest, response, threadPool::relativeTimeInMillis))
         );
     }
 
@@ -141,7 +133,7 @@ static final class Fields {
         static final String CLUSTER_NAME = "cluster_name";
     }
 
-    private static class RestClusterStateResponse implements ToXContentObject {
+    private static class RestClusterStateResponse implements ChunkedToXContent {
 
         private final ClusterStateRequest request;
         private final ClusterStateResponse response;
@@ -156,22 +148,25 @@ private static class RestClusterStateResponse implements ToXContentObject {
         }
 
         @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
             if (request.local() == false
                 && currentTimeMillisSupplier.getAsLong() - startTimeMillis > request.masterNodeTimeout().millis()) {
                 throw new ElasticsearchTimeoutException("Timed out getting cluster state");
             }
-            builder.startObject();
-            if (request.waitForMetadataVersion() != null) {
-                builder.field(Fields.WAIT_FOR_TIMED_OUT, response.isWaitForTimedOut());
-            }
-            builder.field(Fields.CLUSTER_NAME, response.getClusterName().value());
+
             final ClusterState responseState = response.getState();
-            if (responseState != null) {
-                responseState.toXContent(builder, params);
-            }
-            builder.endObject();
-            return builder;
+
+            return Iterators.concat(Iterators.single((builder, params) -> {
+                builder.startObject();
+                if (request.waitForMetadataVersion() != null) {
+                    builder.field(Fields.WAIT_FOR_TIMED_OUT, response.isWaitForTimedOut());
+                }
+                builder.field(Fields.CLUSTER_NAME, response.getClusterName().value());
+                return builder;
+            }),
+                responseState == null ? Collections.emptyIterator() : responseState.toXContentChunked(outerParams),
+                ChunkedToXContentHelper.endObject()
+            );
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetDesiredBalanceAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetDesiredBalanceAction.java
index cdc21f2e7c44..11732747b2b3 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetDesiredBalanceAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetDesiredBalanceAction.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.rest.action.RestChunkedToXContentListener;
 
 import java.io.IOException;
 import java.util.List;
@@ -35,7 +35,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
         return restChannel -> client.execute(
             GetDesiredBalanceAction.INSTANCE,
             new DesiredBalanceRequest(),
-            new RestToXContentListener<>(restChannel)
+            new RestChunkedToXContentListener<>(restChannel)
         );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestListTasksAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestListTasksAction.java
index bbf2d86ce07d..99417fbc962b 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestListTasksAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestListTasksAction.java
@@ -18,12 +18,8 @@
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.rest.action.RestBuilderListener;
-import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.rest.action.RestChunkedToXContentListener;
 import org.elasticsearch.tasks.TaskId;
-import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.List;
@@ -80,33 +76,17 @@ public static ListTasksRequest generateListTasksRequest(RestRequest request) {
     public static <T extends ListTasksResponse> ActionListener<T> listTasksResponseListener(
         Supplier<DiscoveryNodes> nodesInCluster,
         String groupBy,
-        final RestChannel channel
+        RestChannel channel
     ) {
-        if ("nodes".equals(groupBy)) {
-            return new RestBuilderListener<T>(channel) {
-                @Override
-                public RestResponse buildResponse(T response, XContentBuilder builder) throws Exception {
-                    builder.startObject();
-                    response.toXContentGroupedByNode(builder, channel.request(), nodesInCluster.get());
-                    builder.endObject();
-                    return new RestResponse(RestStatus.OK, builder);
-                }
-            };
-        } else if ("parents".equals(groupBy)) {
-            return new RestBuilderListener<T>(channel) {
-                @Override
-                public RestResponse buildResponse(T response, XContentBuilder builder) throws Exception {
-                    builder.startObject();
-                    response.toXContentGroupedByParents(builder, channel.request());
-                    builder.endObject();
-                    return new RestResponse(RestStatus.OK, builder);
-                }
-            };
-        } else if ("none".equals(groupBy)) {
-            return new RestToXContentListener<>(channel);
-        } else {
-            throw new IllegalArgumentException("[group_by] must be one of [nodes], [parents] or [none] but was [" + groupBy + "]");
-        }
+        final var listener = new RestChunkedToXContentListener<>(channel);
+        return switch (groupBy) {
+            case "nodes" -> listener.map(response -> response.groupedByNode(nodesInCluster));
+            case "parents" -> listener.map(response -> response.groupedByParent());
+            case "none" -> listener.map(response -> response.groupedByNone());
+            default -> throw new IllegalArgumentException(
+                "[group_by] must be one of [nodes], [parents] or [none] but was [" + groupBy + "]"
+            );
+        };
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPendingClusterTasksAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPendingClusterTasksAction.java
index b150fe539255..b5dc4eb50b88 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPendingClusterTasksAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPendingClusterTasksAction.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.rest.action.RestChunkedToXContentListener;
 
 import java.io.IOException;
 import java.util.List;
@@ -36,6 +36,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         PendingClusterTasksRequest pendingClusterTasksRequest = new PendingClusterTasksRequest();
         pendingClusterTasksRequest.masterNodeTimeout(request.paramAsTime("master_timeout", pendingClusterTasksRequest.masterNodeTimeout()));
         pendingClusterTasksRequest.local(request.paramAsBoolean("local", pendingClusterTasksRequest.local()));
-        return channel -> client.admin().cluster().pendingClusterTasks(pendingClusterTasksRequest, new RestToXContentListener<>(channel));
+        return channel -> client.admin()
+            .cluster()
+            .pendingClusterTasks(pendingClusterTasksRequest, new RestChunkedToXContentListener<>(channel));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPrevalidateNodeRemovalAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPrevalidateNodeRemovalAction.java
index 6280abd2f048..01b404e02f0a 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPrevalidateNodeRemovalAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPrevalidateNodeRemovalAction.java
@@ -44,6 +44,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
             .setExternalIds(externalIds)
             .build();
         prevalidationRequest.masterNodeTimeout(request.paramAsTime("master_timeout", prevalidationRequest.masterNodeTimeout()));
+        prevalidationRequest.timeout(request.paramAsTime("timeout", prevalidationRequest.timeout()));
         return channel -> client.execute(
             PrevalidateNodeRemovalAction.INSTANCE,
             prevalidationRequest,
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestFieldUsageStatsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestFieldUsageStatsAction.java
index c1c20724a8c7..9a5f54013998 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestFieldUsageStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestFieldUsageStatsAction.java
@@ -17,7 +17,7 @@
 import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestCancellableNodeClient;
-import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.rest.action.RestChunkedToXContentListener;
 
 import java.io.IOException;
 import java.util.List;
@@ -42,7 +42,7 @@ public BaseRestHandler.RestChannelConsumer prepareRequest(final RestRequest requ
         fusRequest.fields(request.paramAsStringArray("fields", fusRequest.fields()));
         return channel -> {
             final RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel());
-            cancelClient.execute(FieldUsageStatsAction.INSTANCE, fusRequest, new RestToXContentListener<>(channel));
+            cancelClient.execute(FieldUsageStatsAction.INSTANCE, fusRequest, new RestChunkedToXContentListener<>(channel));
         };
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java
index 6765c9e504a5..f731e2b72223 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java
@@ -168,8 +168,11 @@ static RestResponse buildRestResponse(
                             if (entry.getKey().equals(alias.getWriteDataStream())) {
                                 builder.field("is_write_index", true);
                             }
-                            if (alias.getFilter() != null) {
-                                builder.field("filter", XContentHelper.convertToMap(alias.getFilter().uncompressed(), true).v2());
+                            if (alias.getFilter(entry.getKey()) != null) {
+                                builder.field(
+                                    "filter",
+                                    XContentHelper.convertToMap(alias.getFilter(entry.getKey()).uncompressed(), true).v2()
+                                );
                             }
                             builder.endObject();
                         }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java
index 11c725695aa1..46a686d6e74b 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java
@@ -17,9 +17,8 @@
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.DispatchingRestToXContentListener;
 import org.elasticsearch.rest.action.RestCancellableNodeClient;
-import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.rest.action.RestChunkedToXContentListener;
 
 import java.io.IOException;
 import java.util.List;
@@ -39,12 +38,6 @@ public class RestGetIndicesAction extends BaseRestHandler {
 
     private static final Set<String> COMPATIBLE_RESPONSE_PARAMS = addToCopy(Settings.FORMAT_PARAMS, INCLUDE_TYPE_NAME_PARAMETER);
 
-    private final ThreadPool threadPool;
-
-    public RestGetIndicesAction(ThreadPool threadPool) {
-        this.threadPool = threadPool;
-    }
-
     @Override
     public List<Route> routes() {
         return List.of(new Route(GET, "/{index}"), new Route(HEAD, "/{index}"));
@@ -76,10 +69,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         final var httpChannel = request.getHttpChannel();
         return channel -> new RestCancellableNodeClient(client, httpChannel).admin()
             .indices()
-            .getIndex(
-                getIndexRequest,
-                new DispatchingRestToXContentListener<>(threadPool.executor(ThreadPool.Names.MANAGEMENT), channel, request)
-            );
+            .getIndex(getIndexRequest, new RestChunkedToXContentListener<>(channel));
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java
index 737c1ccc4cba..74924e4bfaf7 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java
@@ -19,7 +19,7 @@
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestCancellableNodeClient;
-import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.rest.action.RestChunkedToXContentListener;
 import org.elasticsearch.rest.action.document.RestMultiTermVectorsAction;
 
 import java.io.IOException;
@@ -140,7 +140,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
 
         return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).admin()
             .indices()
-            .stats(indicesStatsRequest, new RestToXContentListener<>(channel));
+            .stats(indicesStatsRequest, new RestChunkedToXContentListener<>(channel));
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRecoveryAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRecoveryAction.java
index d8a9c9227fa5..a4a5df0674b8 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRecoveryAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRecoveryAction.java
@@ -9,17 +9,13 @@
 package org.elasticsearch.rest.action.admin.indices;
 
 import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest;
-import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.ChunkedRestResponseBody;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.rest.action.RestActionListener;
 import org.elasticsearch.rest.action.RestCancellableNodeClient;
+import org.elasticsearch.rest.action.RestChunkedToXContentListener;
 
 import java.io.IOException;
 import java.util.List;
@@ -55,14 +51,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         recoveryRequest.indicesOptions(IndicesOptions.fromRequest(request, recoveryRequest.indicesOptions()));
         return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).admin()
             .indices()
-            .recoveries(recoveryRequest, new RestActionListener<>(channel) {
-                @Override
-                protected void processResponse(RecoveryResponse recoveryResponse) throws IOException {
-                    ensureOpen();
-                    channel.sendResponse(
-                        new RestResponse(RestStatus.OK, ChunkedRestResponseBody.fromXContent(recoveryResponse, request, channel))
-                    );
-                }
-            });
+            .recoveries(recoveryRequest, new RestChunkedToXContentListener<>(channel));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/AbstractCatAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/AbstractCatAction.java
index 535753828712..ade43e624509 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/AbstractCatAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/AbstractCatAction.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.io.Streams;
-import org.elasticsearch.common.io.UTF8StreamWriter;
 import org.elasticsearch.common.io.stream.BytesStream;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -18,6 +17,8 @@
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
 import java.util.Set;
 
 import static org.elasticsearch.rest.action.cat.RestTable.buildHelpWidths;
@@ -39,17 +40,22 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
                 Table table = getTableWithHeader(request);
                 int[] width = buildHelpWidths(table, request);
                 BytesStream bytesOutput = Streams.flushOnCloseStream(channel.bytesOutput());
-                UTF8StreamWriter out = new UTF8StreamWriter().setOutput(bytesOutput);
-                for (Table.Cell cell : table.getHeaders()) {
-                    // need to do left-align always, so create new cells
-                    pad(new Table.Cell(cell.value), width[0], request, out);
-                    out.append(" | ");
-                    pad(new Table.Cell(cell.attr.containsKey("alias") ? cell.attr.get("alias") : ""), width[1], request, out);
-                    out.append(" | ");
-                    pad(new Table.Cell(cell.attr.containsKey("desc") ? cell.attr.get("desc") : "not available"), width[2], request, out);
-                    out.append("\n");
+                try (var out = new OutputStreamWriter(bytesOutput, StandardCharsets.UTF_8)) {
+                    for (Table.Cell cell : table.getHeaders()) {
+                        // need to do left-align always, so create new cells
+                        pad(new Table.Cell(cell.value), width[0], request, out);
+                        out.append(" | ");
+                        pad(new Table.Cell(cell.attr.containsKey("alias") ? cell.attr.get("alias") : ""), width[1], request, out);
+                        out.append(" | ");
+                        pad(
+                            new Table.Cell(cell.attr.containsKey("desc") ? cell.attr.get("desc") : "not available"),
+                            width[2],
+                            request,
+                            out
+                        );
+                        out.append("\n");
+                    }
                 }
-                out.close();
                 channel.sendResponse(new RestResponse(RestStatus.OK, RestResponse.TEXT_CONTENT_TYPE, bytesOutput.bytes()));
             };
         } else {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
index 2c63bcfb4810..bdb3fa20b89d 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
@@ -8,24 +8,19 @@
 
 package org.elasticsearch.rest.action.cat;
 
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
-import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
-import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
-import org.elasticsearch.action.support.GroupedActionListener;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.RefCountingListener;
 import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.health.ClusterIndexHealth;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.settings.Settings;
@@ -40,13 +35,11 @@
 import java.time.Instant;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
-import java.util.Collection;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
-import java.util.function.Function;
-import java.util.stream.Collectors;
+import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.action.support.master.MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT;
 import static org.elasticsearch.common.util.set.Sets.addToCopy;
@@ -86,163 +79,68 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli
         final boolean includeUnloadedSegments = request.paramAsBoolean("include_unloaded_segments", false);
 
         return channel -> {
-            final ActionListener<Table> listener = ActionListener.notifyOnce(new RestResponseListener<>(channel) {
-                @Override
-                public RestResponse buildResponse(final Table table) throws Exception {
-                    return RestTable.buildResponse(table, channel);
-                }
-            });
+            final var indexSettingsRef = new AtomicReference<GetSettingsResponse>();
+            final var indicesStatsRef = new AtomicReference<IndicesStatsResponse>();
+            final var clusterStateRef = new AtomicReference<ClusterStateResponse>();
 
-            sendGetSettingsRequest(indices, indicesOptions, masterNodeTimeout, client, new ActionListener.Delegating<>(listener) {
+            try (var listeners = new RefCountingListener(new RestResponseListener<>(channel) {
                 @Override
-                public void onResponse(final GetSettingsResponse getSettingsResponse) {
-                    final GroupedActionListener<ActionResponse> groupedListener = createGroupedListener(request, 4, delegate);
-                    groupedListener.onResponse(getSettingsResponse);
-
-                    // The list of indices that will be returned is determined by the indices returned from the Get Settings call.
-                    // All the other requests just provide additional detail, and wildcards may be resolved differently depending on the
-                    // type of request in the presence of security plugins (looking at you, ClusterHealthRequest), so
-                    // force the IndicesOptions for all the sub-requests to be as inclusive as possible.
-                    final IndicesOptions subRequestIndicesOptions = IndicesOptions.lenientExpandHidden();
-
-                    // Indices that were successfully resolved during the get settings request might be deleted when the subsequent cluster
-                    // state, cluster health and indices stats requests execute. We have to distinguish two cases:
-                    // 1) the deleted index was explicitly passed as parameter to the /_cat/indices request. In this case we want the
-                    // subsequent requests to fail.
-                    // 2) the deleted index was resolved as part of a wildcard or _all. In this case, we want the subsequent requests not to
-                    // fail on the deleted index (as we want to ignore wildcards that cannot be resolved).
-                    // This behavior can be ensured by letting the cluster state, cluster health and indices stats requests re-resolve the
-                    // index names with the same indices options that we used for the initial cluster state request (strictExpand).
-                    sendIndicesStatsRequest(
-                        indices,
-                        subRequestIndicesOptions,
-                        includeUnloadedSegments,
-                        client,
-                        ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure)
-                    );
-                    sendClusterStateRequest(
-                        indices,
-                        subRequestIndicesOptions,
-                        masterNodeTimeout,
-                        client,
-                        ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure)
-                    );
-                    sendClusterHealthRequest(
-                        indices,
-                        subRequestIndicesOptions,
-                        masterNodeTimeout,
-                        client,
-                        ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure)
+                public RestResponse buildResponse(Void ignored) throws Exception {
+                    return RestTable.buildResponse(
+                        buildTable(
+                            request,
+                            indexSettingsRef.get().getIndexToSettings(),
+                            clusterStateRef.get().getState(),
+                            indicesStatsRef.get().getIndices()
+                        ),
+                        channel
                     );
                 }
-            });
-        };
-    }
-
-    /**
-     * We're using the Get Settings API here to resolve the authorized indices for the user.
-     * This is because the Cluster State and Cluster Health APIs do not filter output based
-     * on index privileges, so they can't be used to determine which indices are authorized
-     * or not. On top of this, the Indices Stats API cannot be used either to resolve indices
-     * as it does not provide information for all existing indices (for example recovering
-     * indices or non replicated closed indices are not reported in indices stats response).
-     */
-    private static void sendGetSettingsRequest(
-        final String[] indices,
-        final IndicesOptions indicesOptions,
-        final TimeValue masterNodeTimeout,
-        final NodeClient client,
-        final ActionListener<GetSettingsResponse> listener
-    ) {
-        final GetSettingsRequest request = new GetSettingsRequest();
-        request.indices(indices);
-        request.indicesOptions(indicesOptions);
-        request.masterNodeTimeout(masterNodeTimeout);
-        request.names(IndexSettings.INDEX_SEARCH_THROTTLED.getKey());
-
-        client.admin().indices().getSettings(request, listener);
-    }
-
-    private static void sendClusterStateRequest(
-        final String[] indices,
-        final IndicesOptions indicesOptions,
-        final TimeValue masterNodeTimeout,
-        final NodeClient client,
-        final ActionListener<ClusterStateResponse> listener
-    ) {
-
-        final ClusterStateRequest request = new ClusterStateRequest();
-        request.indices(indices);
-        request.indicesOptions(indicesOptions);
-        request.masterNodeTimeout(masterNodeTimeout);
-
-        client.admin().cluster().state(request, listener);
-    }
-
-    private static void sendClusterHealthRequest(
-        final String[] indices,
-        final IndicesOptions indicesOptions,
-        final TimeValue masterNodeTimeout,
-        final NodeClient client,
-        final ActionListener<ClusterHealthResponse> listener
-    ) {
-
-        final ClusterHealthRequest request = new ClusterHealthRequest();
-        request.indices(indices);
-        request.indicesOptions(indicesOptions);
-        request.masterNodeTimeout(masterNodeTimeout);
-
-        client.admin().cluster().health(request, listener);
-    }
-
-    private static void sendIndicesStatsRequest(
-        final String[] indices,
-        final IndicesOptions indicesOptions,
-        final boolean includeUnloadedSegments,
-        final NodeClient client,
-        final ActionListener<IndicesStatsResponse> listener
-    ) {
-
-        final IndicesStatsRequest request = new IndicesStatsRequest();
-        request.indices(indices);
-        request.indicesOptions(indicesOptions);
-        request.all();
-        request.includeUnloadedSegments(includeUnloadedSegments);
-
-        client.admin().indices().stats(request, listener);
-    }
-
-    private GroupedActionListener<ActionResponse> createGroupedListener(
-        final RestRequest request,
-        final int size,
-        final ActionListener<Table> listener
-    ) {
-        return new GroupedActionListener<>(new ActionListener.Delegating<>(listener) {
-            @Override
-            public void onResponse(final Collection<ActionResponse> responses) {
-                try {
-                    GetSettingsResponse settingsResponse = extractResponse(responses, GetSettingsResponse.class);
-                    Map<String, Settings> indicesSettings = settingsResponse.getIndexToSettings();
-
-                    ClusterStateResponse stateResponse = extractResponse(responses, ClusterStateResponse.class);
-                    Map<String, IndexMetadata> indicesStates = stateResponse.getState()
-                        .getMetadata()
-                        .stream()
-                        .collect(Collectors.toMap(indexMetadata -> indexMetadata.getIndex().getName(), Function.identity()));
-
-                    ClusterHealthResponse healthResponse = extractResponse(responses, ClusterHealthResponse.class);
-                    Map<String, ClusterIndexHealth> indicesHealths = healthResponse.getIndices();
-
-                    IndicesStatsResponse statsResponse = extractResponse(responses, IndicesStatsResponse.class);
-                    Map<String, IndexStats> indicesStats = statsResponse.getIndices();
-
-                    Table responseTable = buildTable(request, indicesSettings, indicesHealths, indicesStats, indicesStates);
-                    delegate.onResponse(responseTable);
-                } catch (Exception e) {
-                    onFailure(e);
-                }
+            })) {
+                // [NOTE: WHY GET SETTINGS] Use the Get Settings API to determine the indices we should return because:
+                //
+                // - the Cluster State API does not filter output based on index privileges, so they can't be used to determine which
+                // indices are authorized or not.
+                //
+                // - the Indices Stats API does not provide information for all existing indices (for example recovering indices or
+                // non-replicated closed indices are not reported in its response).
+                //
+                // Note that the user must be authorized to request the cluster state to use this API, which means they can use the cluster
+                // state API to get a list of all the indices in the cluster anyway. Thus filtering out the unauthorized indices has limited
+                // security benefits - it's more of a convenience thing.
+                client.admin()
+                    .indices()
+                    .prepareGetSettings(indices)
+                    .setIndicesOptions(indicesOptions)
+                    .setMasterNodeTimeout(masterNodeTimeout)
+                    .setNames(IndexSettings.INDEX_SEARCH_THROTTLED.getKey())
+                    .execute(listeners.acquire(indexSettingsRef::set));
+
+                // The other requests just provide additional detail, and wildcards may be resolved differently depending on the type of
+                // request in the presence of security plugins, so set the IndicesOptions for all the sub-requests to be as inclusive as
+                // possible.
+                final IndicesOptions subRequestIndicesOptions = IndicesOptions.lenientExpandHidden();
+
+                client.admin()
+                    .cluster()
+                    .prepareState()
+                    .clear()
+                    .setMetadata(true)
+                    .setRoutingTable(true)
+                    .setIndices(indices)
+                    .setIndicesOptions(subRequestIndicesOptions)
+                    .setMasterNodeTimeout(masterNodeTimeout)
+                    .execute(listeners.acquire(clusterStateRef::set));
+
+                client.admin()
+                    .indices()
+                    .prepareStats(indices)
+                    .setIndicesOptions(subRequestIndicesOptions)
+                    .all()
+                    .setIncludeUnloadedSegments(includeUnloadedSegments)
+                    .execute(listeners.acquire(indicesStatsRef::set));
             }
-        }, size);
+        };
     }
 
     @Override
@@ -608,54 +506,50 @@ protected Table getTableWithHeader(final RestRequest request) {
     Table buildTable(
         final RestRequest request,
         final Map<String, Settings> indicesSettings,
-        final Map<String, ClusterIndexHealth> indicesHealths,
-        final Map<String, IndexStats> indicesStats,
-        final Map<String, IndexMetadata> indicesMetadatas
+        final ClusterState clusterState,
+        final Map<String, IndexStats> indicesStats
     ) {
-
         final String healthParam = request.param("health");
+        final ClusterHealthStatus healthStatusFilter = healthParam == null ? null : ClusterHealthStatus.fromString(healthParam);
+
         final Table table = getTableWithHeader(request);
 
+        // Use indicesSettings to determine the indices returned - see [NOTE: WHY GET SETTINGS] above for details.
         indicesSettings.forEach((indexName, settings) -> {
-            if (indicesMetadatas.containsKey(indexName) == false) {
-                // the index exists in the Get Indices response but is not present in the cluster state:
-                // it is likely that the index was deleted in the meanwhile, so we ignore it.
+            final IndexMetadata indexMetadata = clusterState.metadata().index(indexName);
+
+            if (indexMetadata == null) {
+                // The index exists in indicesSettings but its metadata is missing, which means it was created or deleted
+                // concurrently with this action. However none of the requests returned an IndexNotFoundException so we do not require it to
+                // exist, and therefore we can just skip it.
                 return;
             }
 
-            final IndexMetadata indexMetadata = indicesMetadatas.get(indexName);
             final IndexMetadata.State indexState = indexMetadata.getState();
             final IndexStats indexStats = indicesStats.get(indexName);
             final boolean searchThrottled = IndexSettings.INDEX_SEARCH_THROTTLED.get(settings);
 
+            final IndexRoutingTable indexRoutingTable = clusterState.routingTable().index(indexName);
+            final ClusterHealthStatus indexHealthStatus = indexRoutingTable == null
+                ? ClusterHealthStatus.RED // no routing table => cluster not recovered
+                : new ClusterIndexHealth(indexMetadata, indexRoutingTable).getStatus();
+
+            if (healthStatusFilter != null && indexHealthStatus != healthStatusFilter) {
+                // index health does not match the one requested
+                return;
+            }
+
             final String health;
-            final ClusterIndexHealth indexHealth = indicesHealths.get(indexName);
-            if (indexHealth != null) {
-                health = indexHealth.getStatus().toString().toLowerCase(Locale.ROOT);
+            if (indexRoutingTable != null) {
+                health = indexHealthStatus.toString().toLowerCase(Locale.ROOT);
             } else if (indexStats != null) {
                 health = "red*";
             } else {
                 health = "";
             }
 
-            if (healthParam != null) {
-                final ClusterHealthStatus healthStatusFilter = ClusterHealthStatus.fromString(healthParam);
-                boolean skip;
-                if (indexHealth != null) {
-                    // index health is known but does not match the one requested
-                    skip = indexHealth.getStatus() != healthStatusFilter;
-                } else {
-                    // index health is unknown, skip if we don't explicitly request RED health
-                    skip = ClusterHealthStatus.RED != healthStatusFilter;
-                }
-                if (skip) {
-                    return;
-                }
-            }
-
             final CommonStats primaryStats;
             final CommonStats totalStats;
-
             if (indexStats == null || indexState == IndexMetadata.State.CLOSE) {
                 // TODO: expose docs stats for replicated closed indices
                 primaryStats = new CommonStats();
@@ -664,13 +558,14 @@ Table buildTable(
                 primaryStats = indexStats.getPrimaries();
                 totalStats = indexStats.getTotal();
             }
+
             table.startRow();
             table.addCell(health);
             table.addCell(indexState.toString().toLowerCase(Locale.ROOT));
             table.addCell(indexName);
             table.addCell(indexMetadata.getIndexUUID());
-            table.addCell(indexHealth == null ? null : indexHealth.getNumberOfShards());
-            table.addCell(indexHealth == null ? null : indexHealth.getNumberOfReplicas());
+            table.addCell(indexMetadata.getNumberOfShards());
+            table.addCell(indexMetadata.getNumberOfReplicas());
 
             table.addCell(primaryStats.getDocs() == null ? null : primaryStats.getDocs().getCount());
             table.addCell(primaryStats.getDocs() == null ? null : primaryStats.getDocs().getDeleted());
@@ -881,9 +776,4 @@ Table buildTable(
 
         return table;
     }
-
-    @SuppressWarnings("unchecked")
-    private static <A extends ActionResponse> A extractResponse(final Collection<? extends ActionResponse> responses, Class<A> c) {
-        return (A) responses.stream().filter(c::isInstance).findFirst().get();
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java
index 90f0254b67c7..56c9d8f55536 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java
@@ -66,10 +66,10 @@ protected Table getTableWithHeader(final RestRequest request) {
         return t;
     }
 
-    private Table buildTable(RestRequest request, PendingClusterTasksResponse tasks) {
+    private Table buildTable(RestRequest request, PendingClusterTasksResponse response) {
         Table t = getTableWithHeader(request);
 
-        for (PendingClusterTask task : tasks) {
+        for (PendingClusterTask task : response.pendingTasks()) {
             t.startRow();
             t.addCell(task.getInsertOrder());
             t.addCell(task.getTimeInQueue());
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestTable.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestTable.java
index 8b42f915e19f..015b1422d021 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestTable.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestTable.java
@@ -8,28 +8,38 @@
 
 package org.elasticsearch.rest.action.cat;
 
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.io.Streams;
-import org.elasticsearch.common.io.UTF8StreamWriter;
-import org.elasticsearch.common.io.stream.BytesStream;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.ReleasableBytesReference;
+import org.elasticsearch.common.collect.Iterators;
+import org.elasticsearch.common.io.stream.RecyclerBytesStreamOutput;
+import org.elasticsearch.common.recycler.Recycler;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.SizeValue;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.Booleans;
+import org.elasticsearch.core.Releasables;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.rest.ChunkedRestResponseBody;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -57,60 +67,146 @@ private static XContentType getResponseContentType(RestRequest request) {
     }
 
     public static RestResponse buildXContentBuilder(Table table, RestChannel channel) throws Exception {
-        RestRequest request = channel.request();
-        XContentBuilder builder = channel.newBuilder();
-        List<DisplayHeader> displayHeaders = buildDisplayHeaders(table, request);
-
-        builder.startArray();
-        List<Integer> rowOrder = getRowOrder(table, request);
-        for (Integer row : rowOrder) {
-            builder.startObject();
-            for (DisplayHeader header : displayHeaders) {
-                builder.field(header.display, renderValue(request, table.getAsMap().get(header.name).get(row).value));
-            }
-            builder.endObject();
-        }
-        builder.endArray();
-        return new RestResponse(RestStatus.OK, builder);
+        final RestRequest request = channel.request();
+        final List<Integer> rowOrder = getRowOrder(table, channel.request());
+        final List<DisplayHeader> displayHeaders = buildDisplayHeaders(table, request);
+
+        return new RestResponse(
+            RestStatus.OK,
+            ChunkedRestResponseBody.fromXContent(
+                ignored -> Iterators.concat(
+                    Iterators.single((builder, params) -> builder.startArray()),
+                    rowOrder.stream().<ToXContent>map(row -> (builder, params) -> {
+                        builder.startObject();
+                        for (DisplayHeader header : displayHeaders) {
+                            builder.field(header.display, renderValue(request, table.getAsMap().get(header.name).get(row).value));
+                        }
+                        builder.endObject();
+                        return builder;
+                    }).iterator(),
+                    Iterators.single((builder, params) -> builder.endArray())
+                ),
+                ToXContent.EMPTY_PARAMS,
+                channel
+            )
+        );
     }
 
-    public static RestResponse buildTextPlainResponse(Table table, RestChannel channel) throws IOException {
+    public static RestResponse buildTextPlainResponse(Table table, RestChannel channel) {
         RestRequest request = channel.request();
         boolean verbose = request.paramAsBoolean("v", false);
 
         List<DisplayHeader> headers = buildDisplayHeaders(table, request);
         int[] width = buildWidths(table, request, verbose, headers);
-
-        BytesStream bytesOut = Streams.flushOnCloseStream(channel.bytesOutput());
-        UTF8StreamWriter out = new UTF8StreamWriter().setOutput(bytesOut);
         int lastHeader = headers.size() - 1;
-        if (verbose) {
-            for (int col = 0; col < headers.size(); col++) {
-                DisplayHeader header = headers.get(col);
-                boolean isLastColumn = col == lastHeader;
-                pad(new Table.Cell(header.display, table.findHeaderByName(header.name)), width[col], request, out, isLastColumn);
-                if (isLastColumn == false) {
-                    out.append(" ");
+        List<Integer> rowOrder = getRowOrder(table, request);
+
+        if (verbose == false && rowOrder.isEmpty()) {
+            return new RestResponse(RestStatus.OK, RestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
+        }
+
+        return new RestResponse(RestStatus.OK, new ChunkedRestResponseBody() {
+
+            private boolean needsHeader = verbose;
+            private final Iterator<Integer> rowIterator = rowOrder.iterator();
+
+            private RecyclerBytesStreamOutput currentOutput;
+            private final Writer writer = new OutputStreamWriter(new OutputStream() {
+                @Override
+                public void write(int b) throws IOException {
+                    assert currentOutput != null;
+                    currentOutput.write(b);
                 }
+
+                @Override
+                public void write(byte[] b, int off, int len) throws IOException {
+                    assert currentOutput != null;
+                    currentOutput.write(b, off, len);
+                }
+
+                @Override
+                public void flush() {
+                    assert currentOutput != null;
+                    currentOutput.flush();
+                }
+
+                @Override
+                public void close() {
+                    assert currentOutput != null;
+                    currentOutput.flush();
+                }
+            }, StandardCharsets.UTF_8);
+
+            @Override
+            public boolean isDone() {
+                return needsHeader == false && rowIterator.hasNext() == false;
             }
-            out.append("\n");
-        }
 
-        List<Integer> rowOrder = getRowOrder(table, request);
+            @Override
+            public ReleasableBytesReference encodeChunk(int sizeHint, Recycler<BytesRef> recycler) throws IOException {
+                try {
+                    assert currentOutput == null;
+                    currentOutput = new RecyclerBytesStreamOutput(recycler);
+
+                    if (needsHeader) {
+                        needsHeader = false;
+                        for (int col = 0; col < headers.size(); col++) {
+                            DisplayHeader header = headers.get(col);
+                            boolean isLastColumn = col == lastHeader;
+                            pad(
+                                new Table.Cell(header.display, table.findHeaderByName(header.name)),
+                                width[col],
+                                request,
+                                writer,
+                                isLastColumn
+                            );
+                            if (isLastColumn == false) {
+                                writer.append(" ");
+                            }
+                        }
+                        writer.append("\n");
+                    }
 
-        for (Integer row : rowOrder) {
-            for (int col = 0; col < headers.size(); col++) {
-                DisplayHeader header = headers.get(col);
-                boolean isLastColumn = col == lastHeader;
-                pad(table.getAsMap().get(header.name).get(row), width[col], request, out, isLastColumn);
-                if (isLastColumn == false) {
-                    out.append(" ");
+                    while (rowIterator.hasNext() && currentOutput.size() < sizeHint) {
+                        final var row = rowIterator.next();
+                        for (int col = 0; col < headers.size(); col++) {
+                            DisplayHeader header = headers.get(col);
+                            boolean isLastColumn = col == lastHeader;
+                            pad(table.getAsMap().get(header.name).get(row), width[col], request, writer, isLastColumn);
+                            if (isLastColumn == false) {
+                                writer.append(" ");
+                            }
+                        }
+                        writer.append("\n");
+                    }
+
+                    if (rowIterator.hasNext()) {
+                        writer.flush();
+                    } else {
+                        writer.close();
+                    }
+
+                    final var chunkOutput = currentOutput;
+                    final var result = new ReleasableBytesReference(
+                        chunkOutput.bytes(),
+                        () -> Releasables.closeExpectNoException(chunkOutput)
+                    );
+                    currentOutput = null;
+                    return result;
+                } finally {
+                    if (currentOutput != null) {
+                        assert false : "failure encoding table chunk";
+                        Releasables.closeExpectNoException(currentOutput);
+                        currentOutput = null;
+                    }
                 }
             }
-            out.append("\n");
-        }
-        out.close();
-        return new RestResponse(RestStatus.OK, RestResponse.TEXT_CONTENT_TYPE, bytesOut.bytes());
+
+            @Override
+            public String getResponseContentTypeString() {
+                return RestResponse.TEXT_CONTENT_TYPE;
+            }
+        });
     }
 
     static List<Integer> getRowOrder(Table table, RestRequest request) {
@@ -297,11 +393,11 @@ private static int[] buildWidths(Table table, RestRequest request, boolean verbo
         return width;
     }
 
-    public static void pad(Table.Cell cell, int width, RestRequest request, UTF8StreamWriter out) throws IOException {
+    public static void pad(Table.Cell cell, int width, RestRequest request, Writer out) throws IOException {
         pad(cell, width, request, out, false);
     }
 
-    public static void pad(Table.Cell cell, int width, RestRequest request, UTF8StreamWriter out, boolean isLast) throws IOException {
+    public static void pad(Table.Cell cell, int width, RestRequest request, Writer out, boolean isLast) throws IOException {
         String sValue = renderValue(request, cell.value);
         int length = sValue == null ? 0 : sValue.length();
         byte leftOver = (byte) (width - length);
diff --git a/server/src/main/java/org/elasticsearch/script/AbstractFieldScript.java b/server/src/main/java/org/elasticsearch/script/AbstractFieldScript.java
index c31e5d3db352..4b9d69204779 100644
--- a/server/src/main/java/org/elasticsearch/script/AbstractFieldScript.java
+++ b/server/src/main/java/org/elasticsearch/script/AbstractFieldScript.java
@@ -10,14 +10,16 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
 
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 import static org.elasticsearch.core.TimeValue.timeValueMillis;
 
@@ -61,25 +63,36 @@ static <F> ScriptContext<F> newContext(String name, Class<F> factoryClass) {
         );
     }
 
+    @SuppressWarnings("unchecked")
     private static final Map<String, Function<Object, Object>> PARAMS_FUNCTIONS = Map.of(
         "_source",
-        value -> ((SourceLookup) value).source()
+        value -> ((Supplier<Source>) value).get().source()
     );
 
     protected final String fieldName;
-    protected final SourceLookup sourceLookup;
+    protected final Supplier<Source> source;
     private final Map<String, Object> params;
+    private final OnScriptError onScriptError;
+
+    public AbstractFieldScript(
+        String fieldName,
+        Map<String, Object> params,
+        SearchLookup searchLookup,
+        LeafReaderContext ctx,
+        OnScriptError onScriptError
+    ) {
+        this(fieldName, params, new DocValuesDocReader(searchLookup, ctx), onScriptError);
+    }
 
-    public AbstractFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
-        super(new DocValuesDocReader(searchLookup, ctx));
-
+    private AbstractFieldScript(String fieldName, Map<String, Object> params, DocReader docReader, OnScriptError onScriptError) {
+        super(docReader);
         this.fieldName = fieldName;
-        Map<String, Object> docAsMap = docAsMap();
-        this.sourceLookup = (SourceLookup) docAsMap.get("_source");
+        this.source = docReader.source();
         params = new HashMap<>(params);
-        params.put("_source", sourceLookup);
-        params.put("_fields", docAsMap.get("_fields"));
+        params.put("_source", this.source);
+        params.put("_fields", docReader.docAsMap().get("_fields"));
         this.params = new DynamicMap(params, PARAMS_FUNCTIONS);
+        this.onScriptError = onScriptError;
     }
 
     /**
@@ -90,7 +103,7 @@ public final Map<String, Object> getParams() {
     }
 
     protected List<Object> extractFromSource(String path) {
-        return XContentMapValues.extractRawValues(path, sourceLookup.source());
+        return XContentMapValues.extractRawValues(path, source.get().source());
     }
 
     protected final void emitFromCompositeScript(CompositeFieldScript compositeFieldScript) {
@@ -133,5 +146,24 @@ protected final void checkMaxSize(int currentSize) {
         }
     }
 
+    protected abstract void prepareExecute();
+
+    /**
+     * Execute the script for the provided {@code docId}.
+     */
+    public final void runForDoc(int docId) {
+        prepareExecute();
+        setDocument(docId);
+        try {
+            execute();
+        } catch (Exception e) {
+            if (onScriptError == OnScriptError.CONTINUE) {
+                // ignore
+            } else {
+                throw e;
+            }
+        }
+    }
+
     public abstract void execute();
 }
diff --git a/server/src/main/java/org/elasticsearch/script/AbstractLongFieldScript.java b/server/src/main/java/org/elasticsearch/script/AbstractLongFieldScript.java
index 10cb90607b5a..0717994d0066 100644
--- a/server/src/main/java/org/elasticsearch/script/AbstractLongFieldScript.java
+++ b/server/src/main/java/org/elasticsearch/script/AbstractLongFieldScript.java
@@ -10,6 +10,7 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.util.ArrayUtil;
+import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.util.Map;
@@ -22,17 +23,19 @@ public abstract class AbstractLongFieldScript extends AbstractFieldScript {
     private long[] values = new long[1];
     private int count;
 
-    public AbstractLongFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
-        super(fieldName, params, searchLookup, ctx);
+    public AbstractLongFieldScript(
+        String fieldName,
+        Map<String, Object> params,
+        SearchLookup searchLookup,
+        OnScriptError onScriptError,
+        LeafReaderContext ctx
+    ) {
+        super(fieldName, params, searchLookup, ctx, onScriptError);
     }
 
-    /**
-     * Execute the script for the provided {@code docId}.
-     */
-    public final void runForDoc(int docId) {
+    @Override
+    protected void prepareExecute() {
         count = 0;
-        setDocument(docId);
-        execute();
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/script/AbstractSortScript.java b/server/src/main/java/org/elasticsearch/script/AbstractSortScript.java
index 56e693b7ea4c..d0d8155b9a25 100644
--- a/server/src/main/java/org/elasticsearch/script/AbstractSortScript.java
+++ b/server/src/main/java/org/elasticsearch/script/AbstractSortScript.java
@@ -12,16 +12,18 @@
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.lucene.ScorerAware;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
 
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 abstract class AbstractSortScript extends DocBasedScript implements ScorerAware {
 
     private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(DynamicMap.class);
+    @SuppressWarnings("unchecked")
     private static final Map<String, Function<Object, Object>> PARAMS_FUNCTIONS = Map.of("doc", value -> {
         deprecationLogger.warn(
             DeprecationCategory.SCRIPTING,
@@ -36,7 +38,7 @@ abstract class AbstractSortScript extends DocBasedScript implements ScorerAware
             "Accessing variable [doc] via [params._doc] from within an sort-script " + "is deprecated in favor of directly accessing [doc]."
         );
         return value;
-    }, "_source", value -> ((SourceLookup) value).source());
+    }, "_source", value -> ((Supplier<Source>) value).get().source());
 
     /**
      * The generic runtime parameters for the script.
diff --git a/server/src/main/java/org/elasticsearch/script/AggregationScript.java b/server/src/main/java/org/elasticsearch/script/AggregationScript.java
index ee545e2d4e57..0fee40934684 100644
--- a/server/src/main/java/org/elasticsearch/script/AggregationScript.java
+++ b/server/src/main/java/org/elasticsearch/script/AggregationScript.java
@@ -14,12 +14,13 @@
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.lucene.ScorerAware;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
 
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 public abstract class AggregationScript extends DocBasedScript implements ScorerAware {
 
@@ -28,6 +29,8 @@ public abstract class AggregationScript extends DocBasedScript implements Scorer
     public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("aggs", Factory.class);
 
     private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(DynamicMap.class);
+
+    @SuppressWarnings("unchecked")
     private static final Map<String, Function<Object, Object>> PARAMS_FUNCTIONS = Map.of("doc", value -> {
         deprecationLogger.warn(
             DeprecationCategory.SCRIPTING,
@@ -44,7 +47,7 @@ public abstract class AggregationScript extends DocBasedScript implements Scorer
                 + "is deprecated in favor of directly accessing [doc]."
         );
         return value;
-    }, "_source", value -> ((SourceLookup) value).source());
+    }, "_source", value -> ((Supplier<Source>) value).get().source());
 
     /**
      * The generic runtime parameters for the script.
@@ -59,9 +62,13 @@ public abstract class AggregationScript extends DocBasedScript implements Scorer
     private Object value;
 
     public AggregationScript(Map<String, Object> params, SearchLookup lookup, LeafReaderContext leafContext) {
-        super(new DocValuesDocReader(lookup, leafContext));
+        this(params, new DocValuesDocReader(lookup, leafContext));
+    }
+
+    private AggregationScript(Map<String, Object> params, DocReader docReader) {
+        super(docReader);
         this.params = new DynamicMap(new HashMap<>(params), PARAMS_FUNCTIONS);
-        this.params.putAll(docAsMap());
+        this.params.putAll(docReader.docAsMap());
     }
 
     protected AggregationScript() {
diff --git a/server/src/main/java/org/elasticsearch/script/BooleanFieldScript.java b/server/src/main/java/org/elasticsearch/script/BooleanFieldScript.java
index 5591ff6d796f..704172cfb9a5 100644
--- a/server/src/main/java/org/elasticsearch/script/BooleanFieldScript.java
+++ b/server/src/main/java/org/elasticsearch/script/BooleanFieldScript.java
@@ -10,6 +10,7 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.core.Booleans;
+import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.util.Map;
@@ -22,8 +23,8 @@ public abstract class BooleanFieldScript extends AbstractFieldScript {
 
     public static final Factory PARSE_FROM_SOURCE = new Factory() {
         @Override
-        public LeafFactory newFactory(String field, Map<String, Object> params, SearchLookup lookup) {
-            return ctx -> new BooleanFieldScript(field, params, lookup, ctx) {
+        public LeafFactory newFactory(String field, Map<String, Object> params, SearchLookup lookup, OnScriptError onScriptError) {
+            return ctx -> new BooleanFieldScript(field, params, lookup, OnScriptError.FAIL, ctx) {
                 @Override
                 public void execute() {
                     emitFromSource();
@@ -38,11 +39,11 @@ public boolean isResultDeterministic() {
     };
 
     public static Factory leafAdapter(Function<SearchLookup, CompositeFieldScript.LeafFactory> parentFactory) {
-        return (leafFieldName, params, searchLookup) -> {
+        return (leafFieldName, params, searchLookup, onScriptError) -> {
             CompositeFieldScript.LeafFactory parentLeafFactory = parentFactory.apply(searchLookup);
             return (LeafFactory) ctx -> {
                 CompositeFieldScript compositeFieldScript = parentLeafFactory.newInstance(ctx);
-                return new BooleanFieldScript(leafFieldName, params, searchLookup, ctx) {
+                return new BooleanFieldScript(leafFieldName, params, searchLookup, onScriptError, ctx) {
                     @Override
                     public void setDocument(int docId) {
                         compositeFieldScript.setDocument(docId);
@@ -61,7 +62,7 @@ public void execute() {
     public static final String[] PARAMETERS = {};
 
     public interface Factory extends ScriptFactory {
-        LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup);
+        LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup, OnScriptError onScriptError);
     }
 
     public interface LeafFactory {
@@ -71,18 +72,20 @@ public interface LeafFactory {
     private int trues;
     private int falses;
 
-    public BooleanFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
-        super(fieldName, params, searchLookup, ctx);
+    public BooleanFieldScript(
+        String fieldName,
+        Map<String, Object> params,
+        SearchLookup searchLookup,
+        OnScriptError onScriptError,
+        LeafReaderContext ctx
+    ) {
+        super(fieldName, params, searchLookup, ctx, onScriptError);
     }
 
-    /**
-     * Execute the script for the provided {@code docId}.
-     */
-    public final void runForDoc(int docId) {
+    @Override
+    protected void prepareExecute() {
         trues = 0;
         falses = 0;
-        setDocument(docId);
-        execute();
     }
 
     public final void runForDoc(int docId, Consumer<Boolean> consumer) {
diff --git a/server/src/main/java/org/elasticsearch/script/CompositeFieldScript.java b/server/src/main/java/org/elasticsearch/script/CompositeFieldScript.java
index 253d83fd2596..a1c44145864a 100644
--- a/server/src/main/java/org/elasticsearch/script/CompositeFieldScript.java
+++ b/server/src/main/java/org/elasticsearch/script/CompositeFieldScript.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.script;
 
 import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.util.ArrayList;
@@ -27,7 +28,12 @@ public abstract class CompositeFieldScript extends AbstractFieldScript {
     public static final String[] PARAMETERS = {};
 
     public interface Factory extends ScriptFactory {
-        CompositeFieldScript.LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup);
+        CompositeFieldScript.LeafFactory newFactory(
+            String fieldName,
+            Map<String, Object> params,
+            SearchLookup searchLookup,
+            OnScriptError onScriptError
+        );
     }
 
     public interface LeafFactory {
@@ -36,8 +42,14 @@ public interface LeafFactory {
 
     private final Map<String, List<Object>> fieldValues = new HashMap<>();
 
-    public CompositeFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
-        super(fieldName, params, searchLookup, ctx);
+    public CompositeFieldScript(
+        String fieldName,
+        Map<String, Object> params,
+        SearchLookup searchLookup,
+        OnScriptError onScriptError,
+        LeafReaderContext ctx
+    ) {
+        super(fieldName, params, searchLookup, ctx, onScriptError);
     }
 
     /**
@@ -47,17 +59,19 @@ public CompositeFieldScript(String fieldName, Map<String, Object> params, Search
      */
     public final List<Object> getValues(String field) {
         // TODO for now we re-run the script every time a leaf field is accessed, but we could cache the values?
-        fieldValues.clear();
+        prepareExecute();
         execute();
         List<Object> values = fieldValues.get(field);
         fieldValues.clear();    // don't hold on to values unnecessarily
         return values;
     }
 
-    public final Map<String, List<Object>> runForDoc(int doc) {
-        setDocument(doc);
+    @Override
+    protected void prepareExecute() {
         fieldValues.clear();
-        execute();
+    }
+
+    public final Map<String, List<Object>> getFieldValues() {
         return fieldValues;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/script/DateFieldScript.java b/server/src/main/java/org/elasticsearch/script/DateFieldScript.java
index bf4ce0b047c1..b6207d0c203f 100644
--- a/server/src/main/java/org/elasticsearch/script/DateFieldScript.java
+++ b/server/src/main/java/org/elasticsearch/script/DateFieldScript.java
@@ -10,6 +10,7 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.util.Map;
@@ -21,8 +22,14 @@ public abstract class DateFieldScript extends AbstractLongFieldScript {
 
     public static final Factory PARSE_FROM_SOURCE = new Factory() {
         @Override
-        public LeafFactory newFactory(String field, Map<String, Object> params, SearchLookup lookup, DateFormatter formatter) {
-            return ctx -> new DateFieldScript(field, params, lookup, formatter, ctx) {
+        public LeafFactory newFactory(
+            String field,
+            Map<String, Object> params,
+            SearchLookup lookup,
+            DateFormatter formatter,
+            OnScriptError onScriptError
+        ) {
+            return ctx -> new DateFieldScript(field, params, lookup, formatter, OnScriptError.FAIL, ctx) {
                 @Override
                 public void execute() {
                     emitFromSource();
@@ -37,11 +44,11 @@ public boolean isResultDeterministic() {
     };
 
     public static Factory leafAdapter(Function<SearchLookup, CompositeFieldScript.LeafFactory> parentFactory) {
-        return (leafFieldName, params, searchLookup, formatter) -> {
+        return (leafFieldName, params, searchLookup, formatter, onScriptError) -> {
             CompositeFieldScript.LeafFactory parentLeafFactory = parentFactory.apply(searchLookup);
             return (LeafFactory) ctx -> {
                 CompositeFieldScript compositeFieldScript = parentLeafFactory.newInstance(ctx);
-                return new DateFieldScript(leafFieldName, params, searchLookup, formatter, ctx) {
+                return new DateFieldScript(leafFieldName, params, searchLookup, formatter, onScriptError, ctx) {
                     @Override
                     public void setDocument(int docId) {
                         compositeFieldScript.setDocument(docId);
@@ -60,7 +67,13 @@ public void execute() {
     public static final String[] PARAMETERS = {};
 
     public interface Factory extends ScriptFactory {
-        LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup, DateFormatter formatter);
+        LeafFactory newFactory(
+            String fieldName,
+            Map<String, Object> params,
+            SearchLookup searchLookup,
+            DateFormatter formatter,
+            OnScriptError onScriptError
+        );
     }
 
     public interface LeafFactory {
@@ -74,9 +87,10 @@ public DateFieldScript(
         Map<String, Object> params,
         SearchLookup searchLookup,
         DateFormatter formatter,
+        OnScriptError onScriptError,
         LeafReaderContext ctx
     ) {
-        super(fieldName, params, searchLookup, ctx);
+        super(fieldName, params, searchLookup, onScriptError, ctx);
         this.formatter = formatter;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/script/DocBasedScript.java b/server/src/main/java/org/elasticsearch/script/DocBasedScript.java
index ae26532ddfc8..3e7476532fbd 100644
--- a/server/src/main/java/org/elasticsearch/script/DocBasedScript.java
+++ b/server/src/main/java/org/elasticsearch/script/DocBasedScript.java
@@ -11,9 +11,12 @@
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.script.field.EmptyField;
 import org.elasticsearch.script.field.Field;
+import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.xcontent.XContentType;
 
 import java.util.Collections;
 import java.util.Map;
+import java.util.function.Supplier;
 import java.util.stream.Stream;
 
 public abstract class DocBasedScript {
@@ -53,6 +56,13 @@ public Map<String, Object> docAsMap() {
         return docReader.docAsMap();
     }
 
+    public Supplier<Source> source() {
+        if (docReader == null) {
+            return () -> Source.empty(XContentType.JSON);
+        }
+        return docReader.source();
+    }
+
     /**
      * The doc lookup for the Lucene segment this script was created for.
      */
diff --git a/server/src/main/java/org/elasticsearch/script/DocReader.java b/server/src/main/java/org/elasticsearch/script/DocReader.java
index 19f6f157f542..061c7033afa8 100644
--- a/server/src/main/java/org/elasticsearch/script/DocReader.java
+++ b/server/src/main/java/org/elasticsearch/script/DocReader.java
@@ -10,8 +10,10 @@
 
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.script.field.Field;
+import org.elasticsearch.search.lookup.Source;
 
 import java.util.Map;
+import java.util.function.Supplier;
 import java.util.stream.Stream;
 
 /**
@@ -36,4 +38,7 @@ public interface DocReader {
 
     /** Old-style doc['field'] access */
     Map<String, ScriptDocValues<?>> doc();
+
+    /** Helper for source access */
+    Supplier<Source> source();
 }
diff --git a/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java b/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java
index 24cd38bfbe7c..3c83dce4b5af 100644
--- a/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java
+++ b/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java
@@ -15,8 +15,10 @@
 import org.elasticsearch.search.lookup.LeafDocLookup;
 import org.elasticsearch.search.lookup.LeafSearchLookup;
 import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.search.lookup.Source;
 
 import java.util.Map;
+import java.util.function.Supplier;
 import java.util.stream.Stream;
 
 /**
@@ -73,4 +75,9 @@ public Map<String, ScriptDocValues<?>> doc() {
     public LeafReaderContext getLeafReaderContext() {
         return leafReaderContext;
     }
+
+    @Override
+    public Supplier<Source> source() {
+        return leafSearchLookup.source();
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/script/DoubleFieldScript.java b/server/src/main/java/org/elasticsearch/script/DoubleFieldScript.java
index f59759e65bdd..089a96238e29 100644
--- a/server/src/main/java/org/elasticsearch/script/DoubleFieldScript.java
+++ b/server/src/main/java/org/elasticsearch/script/DoubleFieldScript.java
@@ -10,6 +10,7 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.util.ArrayUtil;
+import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.util.Map;
@@ -21,8 +22,8 @@ public abstract class DoubleFieldScript extends AbstractFieldScript {
 
     public static final Factory PARSE_FROM_SOURCE = new Factory() {
         @Override
-        public LeafFactory newFactory(String field, Map<String, Object> params, SearchLookup lookup) {
-            return ctx -> new DoubleFieldScript(field, params, lookup, ctx) {
+        public LeafFactory newFactory(String field, Map<String, Object> params, SearchLookup lookup, OnScriptError onScriptError) {
+            return ctx -> new DoubleFieldScript(field, params, lookup, OnScriptError.FAIL, ctx) {
                 @Override
                 public void execute() {
                     emitFromSource();
@@ -37,11 +38,11 @@ public boolean isResultDeterministic() {
     };
 
     public static Factory leafAdapter(Function<SearchLookup, CompositeFieldScript.LeafFactory> parentFactory) {
-        return (leafFieldName, params, searchLookup) -> {
+        return (leafFieldName, params, searchLookup, onScriptError) -> {
             CompositeFieldScript.LeafFactory parentLeafFactory = parentFactory.apply(searchLookup);
             return (LeafFactory) ctx -> {
                 CompositeFieldScript compositeFieldScript = parentLeafFactory.newInstance(ctx);
-                return new DoubleFieldScript(leafFieldName, params, searchLookup, ctx) {
+                return new DoubleFieldScript(leafFieldName, params, searchLookup, onScriptError, ctx) {
                     @Override
                     public void setDocument(int docId) {
                         compositeFieldScript.setDocument(docId);
@@ -60,7 +61,7 @@ public void execute() {
     public static final String[] PARAMETERS = {};
 
     public interface Factory extends ScriptFactory {
-        LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup);
+        LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup, OnScriptError onScriptError);
     }
 
     public interface LeafFactory {
@@ -70,17 +71,19 @@ public interface LeafFactory {
     private double[] values = new double[1];
     private int count;
 
-    public DoubleFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
-        super(fieldName, params, searchLookup, ctx);
+    public DoubleFieldScript(
+        String fieldName,
+        Map<String, Object> params,
+        SearchLookup searchLookup,
+        OnScriptError onScriptError,
+        LeafReaderContext ctx
+    ) {
+        super(fieldName, params, searchLookup, ctx, onScriptError);
     }
 
-    /**
-     * Execute the script for the provided {@code docId}.
-     */
-    public final void runForDoc(int docId) {
+    @Override
+    protected void prepareExecute() {
         count = 0;
-        setDocument(docId);
-        execute();
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/script/FieldScript.java b/server/src/main/java/org/elasticsearch/script/FieldScript.java
index 3f6012f2d19f..af4a0f81227b 100644
--- a/server/src/main/java/org/elasticsearch/script/FieldScript.java
+++ b/server/src/main/java/org/elasticsearch/script/FieldScript.java
@@ -12,12 +12,13 @@
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
 
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 /**
  * A script to produce dynamic values for return fields.
@@ -27,6 +28,8 @@ public abstract class FieldScript extends DocBasedScript {
     public static final String[] PARAMETERS = {};
 
     private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(DynamicMap.class);
+
+    @SuppressWarnings("unchecked")
     private static final Map<String, Function<Object, Object>> PARAMS_FUNCTIONS = Map.of("doc", value -> {
         deprecationLogger.warn(
             DeprecationCategory.SCRIPTING,
@@ -42,15 +45,19 @@ public abstract class FieldScript extends DocBasedScript {
                 + "is deprecated in favor of directly accessing [doc]."
         );
         return value;
-    }, "_source", value -> ((SourceLookup) value).source());
+    }, "_source", value -> ((Supplier<Source>) value).get().source());
 
     /** The generic runtime parameters for the script. */
     private final Map<String, Object> params;
 
     public FieldScript(Map<String, Object> params, SearchLookup lookup, LeafReaderContext leafContext) {
-        super(new DocValuesDocReader(lookup, leafContext));
+        this(params, new DocValuesDocReader(lookup, leafContext));
+    }
+
+    private FieldScript(Map<String, Object> params, DocReader docReader) {
+        super(docReader);
         params = new HashMap<>(params);
-        params.putAll(docAsMap());
+        params.putAll(docReader.docAsMap());
         this.params = new DynamicMap(params, PARAMS_FUNCTIONS);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/script/GeoPointFieldScript.java b/server/src/main/java/org/elasticsearch/script/GeoPointFieldScript.java
index d89cf9d4d069..ffdda12dbde7 100644
--- a/server/src/main/java/org/elasticsearch/script/GeoPointFieldScript.java
+++ b/server/src/main/java/org/elasticsearch/script/GeoPointFieldScript.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.util.Collections;
@@ -31,8 +32,8 @@ public abstract class GeoPointFieldScript extends AbstractFieldScript {
 
     public static final Factory PARSE_FROM_SOURCE = new Factory() {
         @Override
-        public LeafFactory newFactory(String field, Map<String, Object> params, SearchLookup lookup) {
-            return ctx -> new GeoPointFieldScript(field, params, lookup, ctx) {
+        public LeafFactory newFactory(String field, Map<String, Object> params, SearchLookup lookup, OnScriptError onScriptError) {
+            return ctx -> new GeoPointFieldScript(field, params, lookup, OnScriptError.FAIL, ctx) {
                 @Override
                 public void execute() {
                     emitFromSource();
@@ -47,11 +48,11 @@ public boolean isResultDeterministic() {
     };
 
     public static Factory leafAdapter(Function<SearchLookup, CompositeFieldScript.LeafFactory> parentFactory) {
-        return (leafFieldName, params, searchLookup) -> {
+        return (leafFieldName, params, searchLookup, onScriptError) -> {
             CompositeFieldScript.LeafFactory parentLeafFactory = parentFactory.apply(searchLookup);
             return (LeafFactory) ctx -> {
                 CompositeFieldScript compositeFieldScript = parentLeafFactory.newInstance(ctx);
-                return new GeoPointFieldScript(leafFieldName, params, searchLookup, ctx) {
+                return new GeoPointFieldScript(leafFieldName, params, searchLookup, onScriptError, ctx) {
                     @Override
                     public void setDocument(int docId) {
                         compositeFieldScript.setDocument(docId);
@@ -70,7 +71,7 @@ public void execute() {
     public static final String[] PARAMETERS = {};
 
     public interface Factory extends ScriptFactory {
-        LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup);
+        LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup, OnScriptError onScriptError);
     }
 
     public interface LeafFactory {
@@ -81,17 +82,19 @@ public interface LeafFactory {
     private double[] lons = new double[1];
     private int count;
 
-    public GeoPointFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
-        super(fieldName, params, searchLookup, ctx);
+    public GeoPointFieldScript(
+        String fieldName,
+        Map<String, Object> params,
+        SearchLookup searchLookup,
+        OnScriptError onScriptError,
+        LeafReaderContext ctx
+    ) {
+        super(fieldName, params, searchLookup, ctx, onScriptError);
     }
 
-    /**
-     * Execute the script for the provided {@code docId}.
-     */
-    public final void runForDoc(int docId) {
+    @Override
+    protected void prepareExecute() {
         count = 0;
-        setDocument(docId);
-        execute();
     }
 
     /**
@@ -135,7 +138,7 @@ public final int count() {
 
     @Override
     protected List<Object> extractFromSource(String path) {
-        Object value = XContentMapValues.extractValue(path, sourceLookup.source());
+        Object value = XContentMapValues.extractValue(path, source.get().source());
         if (value instanceof List<?>) {
             @SuppressWarnings("unchecked")
             List<Object> list = (List<Object>) value;
diff --git a/server/src/main/java/org/elasticsearch/script/IpFieldScript.java b/server/src/main/java/org/elasticsearch/script/IpFieldScript.java
index 665eaee24ced..0e9afdd7bfda 100644
--- a/server/src/main/java/org/elasticsearch/script/IpFieldScript.java
+++ b/server/src/main/java/org/elasticsearch/script/IpFieldScript.java
@@ -14,6 +14,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.index.mapper.IpFieldMapper;
+import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.net.Inet4Address;
@@ -42,8 +43,8 @@ public abstract class IpFieldScript extends AbstractFieldScript {
 
     public static final Factory PARSE_FROM_SOURCE = new Factory() {
         @Override
-        public LeafFactory newFactory(String field, Map<String, Object> params, SearchLookup lookup) {
-            return ctx -> new IpFieldScript(field, params, lookup, ctx) {
+        public LeafFactory newFactory(String field, Map<String, Object> params, SearchLookup lookup, OnScriptError onScriptError) {
+            return ctx -> new IpFieldScript(field, params, lookup, OnScriptError.FAIL, ctx) {
                 @Override
                 public void execute() {
                     emitFromSource();
@@ -58,11 +59,11 @@ public boolean isResultDeterministic() {
     };
 
     public static Factory leafAdapter(Function<SearchLookup, CompositeFieldScript.LeafFactory> parentFactory) {
-        return (leafFieldName, params, searchLookup) -> {
+        return (leafFieldName, params, searchLookup, onScriptError) -> {
             CompositeFieldScript.LeafFactory parentLeafFactory = parentFactory.apply(searchLookup);
             return (LeafFactory) ctx -> {
                 CompositeFieldScript compositeFieldScript = parentLeafFactory.newInstance(ctx);
-                return new IpFieldScript(leafFieldName, params, searchLookup, ctx) {
+                return new IpFieldScript(leafFieldName, params, searchLookup, onScriptError, ctx) {
                     @Override
                     public void setDocument(int docId) {
                         compositeFieldScript.setDocument(docId);
@@ -81,7 +82,7 @@ public void execute() {
     public static final String[] PARAMETERS = {};
 
     public interface Factory extends ScriptFactory {
-        LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup);
+        LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup, OnScriptError onScriptError);
     }
 
     public interface LeafFactory {
@@ -91,17 +92,19 @@ public interface LeafFactory {
     private BytesRef[] values = new BytesRef[1];
     private int count;
 
-    public IpFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
-        super(fieldName, params, searchLookup, ctx);
+    public IpFieldScript(
+        String fieldName,
+        Map<String, Object> params,
+        SearchLookup searchLookup,
+        OnScriptError onScriptError,
+        LeafReaderContext ctx
+    ) {
+        super(fieldName, params, searchLookup, ctx, onScriptError);
     }
 
-    /**
-     * Execute the script for the provided {@code docId}.
-     */
-    public final void runForDoc(int docId) {
+    @Override
+    protected void prepareExecute() {
         count = 0;
-        setDocument(docId);
-        execute();
     }
 
     public final void runForDoc(int docId, Consumer<InetAddress> consumer) {
diff --git a/server/src/main/java/org/elasticsearch/script/LongFieldScript.java b/server/src/main/java/org/elasticsearch/script/LongFieldScript.java
index bfecad9a6869..bca5eb0b9eb8 100644
--- a/server/src/main/java/org/elasticsearch/script/LongFieldScript.java
+++ b/server/src/main/java/org/elasticsearch/script/LongFieldScript.java
@@ -10,6 +10,7 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.util.Map;
@@ -20,8 +21,8 @@ public abstract class LongFieldScript extends AbstractLongFieldScript {
 
     public static final Factory PARSE_FROM_SOURCE = new Factory() {
         @Override
-        public LeafFactory newFactory(String field, Map<String, Object> params, SearchLookup lookup) {
-            return ctx -> new LongFieldScript(field, params, lookup, ctx) {
+        public LeafFactory newFactory(String field, Map<String, Object> params, SearchLookup lookup, OnScriptError onScriptError) {
+            return ctx -> new LongFieldScript(field, params, lookup, OnScriptError.FAIL, ctx) {
                 @Override
                 public void execute() {
                     emitFromSource();
@@ -36,11 +37,11 @@ public boolean isResultDeterministic() {
     };
 
     public static Factory leafAdapter(Function<SearchLookup, CompositeFieldScript.LeafFactory> parentFactory) {
-        return (leafFieldName, params, searchLookup) -> {
+        return (leafFieldName, params, searchLookup, onScriptError) -> {
             CompositeFieldScript.LeafFactory parentLeafFactory = parentFactory.apply(searchLookup);
             return (LeafFactory) ctx -> {
                 CompositeFieldScript compositeFieldScript = parentLeafFactory.newInstance(ctx);
-                return new LongFieldScript(leafFieldName, params, searchLookup, ctx) {
+                return new LongFieldScript(leafFieldName, params, searchLookup, onScriptError, ctx) {
                     @Override
                     public void setDocument(int docId) {
                         compositeFieldScript.setDocument(docId);
@@ -59,15 +60,21 @@ public void execute() {
     public static final String[] PARAMETERS = {};
 
     public interface Factory extends ScriptFactory {
-        LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup);
+        LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup, OnScriptError onScriptError);
     }
 
     public interface LeafFactory {
         LongFieldScript newInstance(LeafReaderContext ctx);
     }
 
-    public LongFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
-        super(fieldName, params, searchLookup, ctx);
+    public LongFieldScript(
+        String fieldName,
+        Map<String, Object> params,
+        SearchLookup searchLookup,
+        OnScriptError onScriptError,
+        LeafReaderContext ctx
+    ) {
+        super(fieldName, params, searchLookup, onScriptError, ctx);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/script/Metadata.java b/server/src/main/java/org/elasticsearch/script/Metadata.java
index 0c2e6fb6b62f..1afbdb63ca3a 100644
--- a/server/src/main/java/org/elasticsearch/script/Metadata.java
+++ b/server/src/main/java/org/elasticsearch/script/Metadata.java
@@ -55,15 +55,35 @@ public class Metadata {
     public static FieldProperty<Number> LongField = new FieldProperty<>(Number.class).withValidation(FieldProperty.LONGABLE_NUMBER);
 
     protected final Map<String, Object> map;
-    protected final Map<String, FieldProperty<?>> properties;
+    private final Map<String, FieldProperty<?>> properties;
     protected static final FieldProperty<?> BAD_KEY = new FieldProperty<>(null, false, false, null);
 
-    public Metadata(Map<String, Object> map, Map<String, FieldProperty<?>> properties) {
+    /**
+     * Constructs a new Metadata object represented by the given map and properties.
+     * <p>
+     * The passed-in map is used directly -- subsequent modifications to it outside the methods of this class may result in
+     * undefined behavior. Note also that mutation-like methods (e.g. setters, etc) on this class rely on the map being mutable,
+     * which is the expected use for this class.
+     * <p>
+     * The properties map is used directly as well, but we verify at runtime that it <b>must</b> be an immutable map (i.e. constructed
+     * via a call to {@link Map#of()} (or similar) in production, or via {@link Map#copyOf(Map)}} in tests). Since it must be an
+     * immutable map, subsequent modifications are not possible.
+     *
+     * @param map the backing map for this metadata instance
+     * @param properties the immutable map of defined properties for the type of metadata represented by this instance
+     */
+    protected Metadata(Map<String, Object> map, Map<String, FieldProperty<?>> properties) {
         this.map = map;
-        this.properties = Collections.unmodifiableMap(properties);
+        // we can't tell the compiler that properties must be a java.util.ImmutableCollections.AbstractImmutableMap, but
+        // we can use this copyOf + assert to verify that at runtime.
+        this.properties = Map.copyOf(properties);
+        assert this.properties == properties : "properties map must be constructed via Map.of(...) or Map.copyOf(...)";
         validateMetadata();
     }
 
+    // a 'not found' sentinel value for use in validateMetadata below
+    private static final Object NOT_FOUND = new Object();
+
     /**
      * Check that all metadata map contains only valid metadata and no extraneous keys
      */
@@ -71,10 +91,14 @@ protected void validateMetadata() {
         int numMetadata = 0;
         for (Map.Entry<String, FieldProperty<?>> entry : properties.entrySet()) {
             String key = entry.getKey();
-            if (map.containsKey(key)) {
+            Object value = map.getOrDefault(key, NOT_FOUND); // getOrDefault is faster than containsKey + get
+            if (value == NOT_FOUND) {
+                // check whether it's permissible to *not* have a value for the property
+                entry.getValue().check(MapOperation.INIT, key, null);
+            } else {
                 numMetadata++;
+                entry.getValue().check(MapOperation.INIT, key, value);
             }
-            entry.getValue().check(MapOperation.INIT, key, map.get(key));
         }
         if (numMetadata < map.size()) {
             Set<String> keys = new HashSet<>(map.keySet());
@@ -239,7 +263,7 @@ public int size() {
 
     @Override
     public Metadata clone() {
-        // properties is an UnmodifiableMap, no need to create a copy
+        // properties is an unmodifiable map, no need to create a copy here
         return new Metadata(new HashMap<>(map), properties);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/script/ScoreScript.java b/server/src/main/java/org/elasticsearch/script/ScoreScript.java
index 9f885b038e86..503bd11fb434 100644
--- a/server/src/main/java/org/elasticsearch/script/ScoreScript.java
+++ b/server/src/main/java/org/elasticsearch/script/ScoreScript.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -20,6 +20,7 @@
 import java.util.Map;
 import java.util.function.DoubleSupplier;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 /**
  * A script used for adjusting the score on a per document basis.
@@ -51,6 +52,7 @@ public Explanation get(double score, Explanation subQueryExplanation) {
     }
 
     private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(DynamicMap.class);
+    @SuppressWarnings("unchecked")
     private static final Map<String, Function<Object, Object>> PARAMS_FUNCTIONS = Map.of("doc", value -> {
         deprecationLogger.warn(
             DeprecationCategory.SCRIPTING,
@@ -66,7 +68,7 @@ public Explanation get(double score, Explanation subQueryExplanation) {
                 + "is deprecated in favor of directly accessing [doc]."
         );
         return value;
-    }, "_source", value -> ((SourceLookup) value).source());
+    }, "_source", value -> ((Supplier<Source>) value).get().source());
 
     public static final String[] PARAMETERS = new String[] { "explanation" };
 
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java b/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java
index acde189431c4..6e5eba4834ac 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.script;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -45,10 +45,10 @@ public ScriptContextStats(StreamInput in) throws IOException {
         compilations = in.readVLong();
         cacheEvictions = in.readVLong();
         compilationLimitTriggered = in.readVLong();
-        if (in.getVersion().onOrAfter(Version.V_8_1_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
             compilationsHistory = new TimeSeries(in);
             cacheEvictionsHistory = new TimeSeries(in);
-        } else if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+        } else if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             compilationsHistory = new TimeSeries(in).withTotal(compilations);
             cacheEvictionsHistory = new TimeSeries(in).withTotal(cacheEvictions);
         } else {
@@ -63,7 +63,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(compilations);
         out.writeVLong(cacheEvictions);
         out.writeVLong(compilationLimitTriggered);
-        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             compilationsHistory.writeTo(out);
             cacheEvictionsHistory.writeTo(out);
         }
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptException.java b/server/src/main/java/org/elasticsearch/script/ScriptException.java
index 05a291b6562b..b066ddb24d1e 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptException.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptException.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.script;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -78,7 +78,7 @@ public ScriptException(StreamInput in) throws IOException {
         scriptStack = Arrays.asList(in.readStringArray());
         script = in.readString();
         lang = in.readString();
-        if (in.getVersion().onOrAfter(Version.V_7_7_0) && in.readBoolean()) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_7_0) && in.readBoolean()) {
             pos = new Position(in);
         } else {
             pos = null;
@@ -91,7 +91,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeStringArray(scriptStack.toArray(new String[0]));
         out.writeString(script);
         out.writeString(lang);
-        if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_7_0)) {
             if (pos == null) {
                 out.writeBoolean(false);
             } else {
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptMetadata.java b/server/src/main/java/org/elasticsearch/script/ScriptMetadata.java
index f63593df5c44..a3c4bfa575b1 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptMetadata.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptMetadata.java
@@ -10,7 +10,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.DiffableUtils;
@@ -20,8 +20,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.xcontent.ToXContentFragment;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParser.Token;
 
@@ -29,13 +28,14 @@
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
 
 /**
  * {@link ScriptMetadata} is used to store user-defined scripts
  * as part of the {@link ClusterState} using only an id as the key.
  */
-public final class ScriptMetadata implements Metadata.Custom, Writeable, ToXContentFragment {
+public final class ScriptMetadata implements Metadata.Custom, Writeable {
 
     /**
      * Standard logger used to warn about dropped scripts.
@@ -127,8 +127,8 @@ public void writeTo(StreamOutput out) throws IOException {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT.minimumCompatibilityVersion();
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT.minimumCompatibilityVersion();
         }
     }
 
@@ -260,25 +260,12 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeMap(scripts, StreamOutput::writeString, (o, v) -> v.writeTo(o));
     }
 
-    /**
-     * This will write XContent from {@link ScriptMetadata}.  The following format will be written:
-     *
-     * {@code
-     * {
-     *     "<id>" : "<{@link StoredScriptSource#toXContent(XContentBuilder, Params)}>",
-     *     "<id>" : "<{@link StoredScriptSource#toXContent(XContentBuilder, Params)}>",
-     *     ...
-     * }
-     * }
-     */
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        for (Map.Entry<String, StoredScriptSource> entry : scripts.entrySet()) {
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return scripts.entrySet().stream().map(entry -> (ToXContent) (builder, params) -> {
             builder.field(entry.getKey());
-            entry.getValue().toXContent(builder, params);
-        }
-
-        return builder;
+            return entry.getValue().toXContent(builder, params);
+        }).iterator();
     }
 
     @Override
@@ -292,8 +279,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.CURRENT.minimumCompatibilityVersion();
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.CURRENT.minimumCompatibilityVersion();
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptStats.java b/server/src/main/java/org/elasticsearch/script/ScriptStats.java
index 0b161ff7bca8..76f512d62781 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptStats.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptStats.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.script;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -76,7 +76,7 @@ public ScriptStats(ScriptContextStats context) {
     }
 
     public ScriptStats(StreamInput in) throws IOException {
-        if (in.getVersion().onOrAfter(Version.V_8_1_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
             compilationsHistory = new TimeSeries(in);
             cacheEvictionsHistory = new TimeSeries(in);
             compilations = compilationsHistory.total;
@@ -93,7 +93,7 @@ public ScriptStats(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getVersion().onOrAfter(Version.V_8_1_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
             compilationsHistory.writeTo(out);
             cacheEvictionsHistory.writeTo(out);
         } else {
diff --git a/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java b/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java
index 774c98a201f0..d0a81c1db172 100644
--- a/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java
+++ b/server/src/main/java/org/elasticsearch/script/ScriptedMetricAggContexts.java
@@ -15,13 +15,14 @@
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
 
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 public class ScriptedMetricAggContexts {
 
@@ -55,6 +56,8 @@ public interface Factory extends ScriptFactory {
     public abstract static class MapScript extends DocBasedScript {
 
         private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(DynamicMap.class);
+
+        @SuppressWarnings("unchecked")
         private static final Map<String, Function<Object, Object>> PARAMS_FUNCTIONS = Map.of("doc", value -> {
             deprecationLogger.warn(
                 DeprecationCategory.SCRIPTING,
@@ -79,17 +82,23 @@ public abstract static class MapScript extends DocBasedScript {
                     + "is deprecated in favor of using [state]."
             );
             return value;
-        }, "_source", value -> ((SourceLookup) value).source());
+        }, "_source", value -> ((Supplier<Source>) value).get().source());
 
         private final Map<String, Object> params;
         private final Map<String, Object> state;
         private Scorable scorer;
 
         public MapScript(Map<String, Object> params, Map<String, Object> state, SearchLookup lookup, LeafReaderContext leafContext) {
-            super(leafContext == null ? null : new DocValuesDocReader(lookup, leafContext));
+            this(params, state, leafContext == null ? null : new DocValuesDocReader(lookup, leafContext));
+        }
+
+        private MapScript(Map<String, Object> params, Map<String, Object> state, DocReader docReader) {
+            super(docReader);
             this.state = state;
             params = new HashMap<>(params); // copy params so we aren't modifying input
-            params.putAll(docAsMap()); // add lookup vars
+            if (docReader != null) {
+                params.putAll(docReader.docAsMap()); // add lookup vars
+            }
             this.params = new DynamicMap(params, PARAMS_FUNCTIONS); // wrap with deprecations
         }
 
diff --git a/server/src/main/java/org/elasticsearch/script/SortedNumericDocValuesLongFieldScript.java b/server/src/main/java/org/elasticsearch/script/SortedNumericDocValuesLongFieldScript.java
index 9d4e3f5f79fc..eb7163be9ce6 100644
--- a/server/src/main/java/org/elasticsearch/script/SortedNumericDocValuesLongFieldScript.java
+++ b/server/src/main/java/org/elasticsearch/script/SortedNumericDocValuesLongFieldScript.java
@@ -10,6 +10,7 @@
 
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.script.field.LongDocValuesField;
 import org.elasticsearch.search.lookup.SearchLookup;
 
@@ -21,7 +22,7 @@ public class SortedNumericDocValuesLongFieldScript extends AbstractLongFieldScri
     final LongDocValuesField longDocValuesField;
 
     public SortedNumericDocValuesLongFieldScript(String fieldName, SearchLookup lookup, LeafReaderContext ctx) {
-        super(fieldName, Map.of(), lookup, ctx);
+        super(fieldName, Map.of(), lookup, OnScriptError.FAIL, ctx);
         try {
             longDocValuesField = new LongDocValuesField(DocValues.getSortedNumeric(ctx.reader(), fieldName), fieldName);
         } catch (IOException e) {
diff --git a/server/src/main/java/org/elasticsearch/script/SortedSetDocValuesStringFieldScript.java b/server/src/main/java/org/elasticsearch/script/SortedSetDocValuesStringFieldScript.java
index c39d887922e9..b2b523f12fd3 100644
--- a/server/src/main/java/org/elasticsearch/script/SortedSetDocValuesStringFieldScript.java
+++ b/server/src/main/java/org/elasticsearch/script/SortedSetDocValuesStringFieldScript.java
@@ -12,6 +12,7 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
@@ -23,7 +24,7 @@ public class SortedSetDocValuesStringFieldScript extends StringFieldScript {
     boolean hasValue = false;
 
     public SortedSetDocValuesStringFieldScript(String fieldName, SearchLookup searchLookup, LeafReaderContext ctx) {
-        super(fieldName, Map.of(), searchLookup, ctx);
+        super(fieldName, Map.of(), searchLookup, OnScriptError.FAIL, ctx);
         try {
             sortedSetDocValues = DocValues.getSortedSet(ctx.reader(), fieldName);
         } catch (IOException e) {
diff --git a/server/src/main/java/org/elasticsearch/script/StringFieldScript.java b/server/src/main/java/org/elasticsearch/script/StringFieldScript.java
index 7e366f4b72b1..2f0ba635b754 100644
--- a/server/src/main/java/org/elasticsearch/script/StringFieldScript.java
+++ b/server/src/main/java/org/elasticsearch/script/StringFieldScript.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.script;
 
 import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.util.ArrayList;
@@ -28,8 +29,8 @@ public abstract class StringFieldScript extends AbstractFieldScript {
 
     public static final StringFieldScript.Factory PARSE_FROM_SOURCE = new Factory() {
         @Override
-        public LeafFactory newFactory(String field, Map<String, Object> params, SearchLookup lookup) {
-            return ctx -> new StringFieldScript(field, params, lookup, ctx) {
+        public LeafFactory newFactory(String field, Map<String, Object> params, SearchLookup lookup, OnScriptError onScriptError) {
+            return ctx -> new StringFieldScript(field, params, lookup, OnScriptError.FAIL, ctx) {
                 @Override
                 public void execute() {
                     emitFromSource();
@@ -44,11 +45,11 @@ public boolean isResultDeterministic() {
     };
 
     public static Factory leafAdapter(Function<SearchLookup, CompositeFieldScript.LeafFactory> parentFactory) {
-        return (leafFieldName, params, searchLookup) -> {
+        return (leafFieldName, params, searchLookup, onScriptError) -> {
             CompositeFieldScript.LeafFactory parentLeafFactory = parentFactory.apply(searchLookup);
             return (LeafFactory) ctx -> {
                 CompositeFieldScript compositeFieldScript = parentLeafFactory.newInstance(ctx);
-                return new StringFieldScript(leafFieldName, params, searchLookup, ctx) {
+                return new StringFieldScript(leafFieldName, params, searchLookup, onScriptError, ctx) {
                     @Override
                     public void setDocument(int docId) {
                         compositeFieldScript.setDocument(docId);
@@ -67,7 +68,7 @@ public void execute() {
     public static final String[] PARAMETERS = {};
 
     public interface Factory extends ScriptFactory {
-        LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup);
+        LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup, OnScriptError onScriptError);
     }
 
     public interface LeafFactory {
@@ -77,26 +78,32 @@ public interface LeafFactory {
     private final List<String> results = new ArrayList<>();
     private long chars;
 
-    public StringFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
-        super(fieldName, params, searchLookup, ctx);
+    public StringFieldScript(
+        String fieldName,
+        Map<String, Object> params,
+        SearchLookup searchLookup,
+        OnScriptError onScriptError,
+        LeafReaderContext ctx
+    ) {
+        super(fieldName, params, searchLookup, ctx, onScriptError);
     }
 
-    /**
-     * Execute the script for the provided {@code docId}.
-     *
-     * @return a mutable {@link List} that contains the results of the script
-     * and will be modified the next time you call {@linkplain #resultsForDoc}.
-     */
-    public final List<String> resultsForDoc(int docId) {
+    @Override
+    protected void prepareExecute() {
         results.clear();
         chars = 0;
-        setDocument(docId);
-        execute();
-        return results;
     }
 
     public final void runForDoc(int docId, Consumer<String> consumer) {
-        resultsForDoc(docId).forEach(consumer);
+        runForDoc(docId);
+        results.forEach(consumer);
+    }
+
+    /**
+     * Values from the last time runForDoc(int) was called. This list is mutable and will change with the next call of runForDoc(int).
+     */
+    public List<String> getValues() {
+        return results;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/script/TermsSetQueryScript.java b/server/src/main/java/org/elasticsearch/script/TermsSetQueryScript.java
index 90c8c9c9d18c..17fbf894ec66 100644
--- a/server/src/main/java/org/elasticsearch/script/TermsSetQueryScript.java
+++ b/server/src/main/java/org/elasticsearch/script/TermsSetQueryScript.java
@@ -13,12 +13,13 @@
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.search.lookup.LeafSearchLookup;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
 
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 public abstract class TermsSetQueryScript {
 
@@ -27,6 +28,7 @@ public abstract class TermsSetQueryScript {
     public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("terms_set", Factory.class);
 
     private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(DynamicMap.class);
+    @SuppressWarnings("unchecked")
     private static final Map<String, Function<Object, Object>> PARAMS_FUNCTIONS = Map.of("doc", value -> {
         deprecationLogger.warn(
             DeprecationCategory.SCRIPTING,
@@ -43,7 +45,7 @@ public abstract class TermsSetQueryScript {
                 + "is deprecated in favor of directly accessing [doc]."
         );
         return value;
-    }, "_source", value -> ((SourceLookup) value).source());
+    }, "_source", value -> ((Supplier<Source>) value).get().source());
 
     /**
      * The generic runtime parameters for the script.
diff --git a/server/src/main/java/org/elasticsearch/script/TimeSeries.java b/server/src/main/java/org/elasticsearch/script/TimeSeries.java
index 8399a65a57e0..0311d3322e45 100644
--- a/server/src/main/java/org/elasticsearch/script/TimeSeries.java
+++ b/server/src/main/java/org/elasticsearch/script/TimeSeries.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.script;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -49,7 +49,7 @@ public TimeSeries(StreamInput in) throws IOException {
         fiveMinutes = in.readVLong();
         fifteenMinutes = in.readVLong();
         twentyFourHours = in.readVLong();
-        if (in.getVersion().onOrAfter(Version.V_8_1_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
             total = in.readVLong();
         } else {
             total = 0;
@@ -70,7 +70,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(fiveMinutes);
         out.writeVLong(fifteenMinutes);
         out.writeVLong(twentyFourHours);
-        if (out.getVersion().onOrAfter(Version.V_8_1_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
             out.writeVLong(total);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java b/server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java
index 78efbaa373b3..ab3b12434f86 100644
--- a/server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java
+++ b/server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java
@@ -52,18 +52,18 @@ public static class ByteDenseVectorFunction extends DenseVectorFunction {
         public ByteDenseVectorFunction(ScoreScript scoreScript, DenseVectorDocValuesField field, List<Number> queryVector) {
             super(scoreScript, field);
             DenseVector.checkDimensions(field.get().getDims(), queryVector.size());
-
-            float[] vector = new float[queryVector.size()];
             this.queryVector = new byte[queryVector.size()];
+            float[] validateValues = new float[queryVector.size()];
             int queryMagnitude = 0;
             for (int i = 0; i < queryVector.size(); i++) {
-                float value = queryVector.get(i).floatValue();
-                vector[i] = value;
-                this.queryVector[i] = (byte) value;
+                final Number number = queryVector.get(i);
+                byte value = number.byteValue();
+                this.queryVector[i] = value;
                 queryMagnitude += value * value;
+                validateValues[i] = number.floatValue();
             }
             this.qvMagnitude = (float) Math.sqrt(queryMagnitude);
-            field.getElementType().checkVectorBounds(vector);
+            field.getElementType().checkVectorBounds(validateValues);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVector.java b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVector.java
index 3b0817bcf230..a6bbd541bae1 100644
--- a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVector.java
+++ b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVector.java
@@ -8,36 +8,34 @@
 
 package org.elasticsearch.script.field.vectors;
 
-import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.VectorUtil;
 import org.elasticsearch.core.SuppressForbidden;
 
 import java.util.List;
 
 public class ByteKnnDenseVector implements DenseVector {
 
-    protected final BytesRef docVector;
+    protected final byte[] docVector;
 
     protected float[] floatDocVector;
     protected boolean magnitudeCalculated = false;
     protected float magnitude;
 
-    public ByteKnnDenseVector(BytesRef vector) {
+    public ByteKnnDenseVector(byte[] vector) {
         this.docVector = vector;
     }
 
     @Override
     public float[] getVector() {
+        // TODO it would be really nice if we didn't transform the `byte[]` arrays to `float[]`
         if (floatDocVector == null) {
             floatDocVector = new float[docVector.length];
-
             int i = 0;
-            int j = docVector.offset;
-
             while (i < docVector.length) {
-                floatDocVector[i++] = docVector.bytes[j++];
+                floatDocVector[i] = docVector[i];
+                i++;
             }
         }
-
         return floatDocVector;
     }
 
@@ -52,13 +50,7 @@ public float getMagnitude() {
 
     @Override
     public int dotProduct(byte[] queryVector) {
-        int result = 0;
-        int i = 0;
-        int j = docVector.offset;
-        while (i < docVector.length) {
-            result += docVector.bytes[j++] * queryVector[i++];
-        }
-        return result;
+        return (int) VectorUtil.dotProduct(docVector, queryVector);
     }
 
     @Override
@@ -70,9 +62,9 @@ public double dotProduct(float[] queryVector) {
     public double dotProduct(List<Number> queryVector) {
         int result = 0;
         int i = 0;
-        int j = docVector.offset;
         while (i < docVector.length) {
-            result += docVector.bytes[j++] * queryVector.get(i++).intValue();
+            result += docVector[i] * queryVector.get(i).intValue();
+            i++;
         }
         return result;
     }
@@ -86,9 +78,9 @@ private int abs(int value) {
     public int l1Norm(byte[] queryVector) {
         int result = 0;
         int i = 0;
-        int j = docVector.offset;
         while (i < docVector.length) {
-            result += abs(docVector.bytes[j++] - queryVector[i++]);
+            result += abs(docVector[i] - queryVector[i]);
+            i++;
         }
         return result;
     }
@@ -102,9 +94,9 @@ public double l1Norm(float[] queryVector) {
     public double l1Norm(List<Number> queryVector) {
         int result = 0;
         int i = 0;
-        int j = docVector.offset;
         while (i < docVector.length) {
-            result += abs(docVector.bytes[j++] - queryVector.get(i++).intValue());
+            result += abs(docVector[i] - queryVector.get(i).intValue());
+            i++;
         }
         return result;
     }
@@ -113,10 +105,10 @@ public double l1Norm(List<Number> queryVector) {
     public double l2Norm(byte[] queryVector) {
         int result = 0;
         int i = 0;
-        int j = docVector.offset;
         while (i < docVector.length) {
-            int diff = docVector.bytes[j++] - queryVector[i++];
+            int diff = docVector[i] - queryVector[i];
             result += diff * diff;
+            i++;
         }
         return Math.sqrt(result);
     }
@@ -130,10 +122,10 @@ public double l2Norm(float[] queryVector) {
     public double l2Norm(List<Number> queryVector) {
         int result = 0;
         int i = 0;
-        int j = docVector.offset;
         while (i < docVector.length) {
-            int diff = docVector.bytes[j++] - queryVector.get(i++).intValue();
+            int diff = docVector[i] - queryVector.get(i).intValue();
             result += diff * diff;
+            i++;
         }
         return Math.sqrt(result);
     }
diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVectorDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVectorDocValuesField.java
index 4c071301ce3c..a2a9ba1c1d75 100644
--- a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVectorDocValuesField.java
+++ b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVectorDocValuesField.java
@@ -8,8 +8,7 @@
 
 package org.elasticsearch.script.field.vectors;
 
-import org.apache.lucene.index.VectorValues;
-import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.index.ByteVectorValues;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType;
 import org.elasticsearch.index.mapper.vectors.DenseVectorScriptDocValues;
@@ -19,12 +18,12 @@
 import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
 
 public class ByteKnnDenseVectorDocValuesField extends DenseVectorDocValuesField {
-    protected VectorValues input; // null if no vectors
-    protected BytesRef vector;
+    protected ByteVectorValues input; // null if no vectors
+    protected byte[] vector;
     protected final int dims;
 
-    public ByteKnnDenseVectorDocValuesField(@Nullable VectorValues input, String name, ElementType elementType, int dims) {
-        super(name, elementType);
+    public ByteKnnDenseVectorDocValuesField(@Nullable ByteVectorValues input, String name, int dims) {
+        super(name, ElementType.BYTE);
         this.dims = dims;
         this.input = input;
     }
@@ -38,11 +37,11 @@ public void setNextDocId(int docId) throws IOException {
         if (currentDoc == NO_MORE_DOCS || docId < currentDoc) {
             vector = null;
         } else if (docId == currentDoc) {
-            vector = input.binaryValue();
+            vector = input.vectorValue();
         } else {
             currentDoc = input.advance(docId);
             if (currentDoc == docId) {
-                vector = input.binaryValue();
+                vector = input.vectorValue();
             } else {
                 vector = null;
             }
diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/DenseVector.java b/server/src/main/java/org/elasticsearch/script/field/vectors/DenseVector.java
index a4d63786c4bc..84649d9954b6 100644
--- a/server/src/main/java/org/elasticsearch/script/field/vectors/DenseVector.java
+++ b/server/src/main/java/org/elasticsearch/script/field/vectors/DenseVector.java
@@ -8,8 +8,6 @@
 
 package org.elasticsearch.script.field.vectors;
 
-import org.apache.lucene.util.BytesRef;
-
 import java.util.List;
 
 /**
@@ -160,13 +158,13 @@ static float getMagnitude(byte[] vector) {
         return (float) Math.sqrt(mag);
     }
 
-    static float getMagnitude(BytesRef vector, int dims) {
+    static float getMagnitude(byte[] vector, int dims) {
         int mag = 0;
         int i = 0;
-        int j = vector.offset;
-        while (i++ < dims) {
-            int elem = vector.bytes[j++];
+        while (i < dims) {
+            int elem = vector[i];
             mag += elem * elem;
+            i++;
         }
         return (float) Math.sqrt(mag);
     }
diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/KnnDenseVectorDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/KnnDenseVectorDocValuesField.java
index ad9d0c7df78b..b10d83cbdb52 100644
--- a/server/src/main/java/org/elasticsearch/script/field/vectors/KnnDenseVectorDocValuesField.java
+++ b/server/src/main/java/org/elasticsearch/script/field/vectors/KnnDenseVectorDocValuesField.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.script.field.vectors;
 
-import org.apache.lucene.index.VectorValues;
+import org.apache.lucene.index.FloatVectorValues;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType;
 import org.elasticsearch.index.mapper.vectors.DenseVectorScriptDocValues;
@@ -18,12 +18,12 @@
 import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
 
 public class KnnDenseVectorDocValuesField extends DenseVectorDocValuesField {
-    protected VectorValues input; // null if no vectors
+    protected FloatVectorValues input; // null if no vectors
     protected float[] vector;
     protected final int dims;
 
-    public KnnDenseVectorDocValuesField(@Nullable VectorValues input, String name, ElementType elementType, int dims) {
-        super(name, elementType);
+    public KnnDenseVectorDocValuesField(@Nullable FloatVectorValues input, String name, int dims) {
+        super(name, ElementType.FLOAT);
         this.dims = dims;
         this.input = input;
     }
diff --git a/server/src/main/java/org/elasticsearch/search/CanMatchShardResponse.java b/server/src/main/java/org/elasticsearch/search/CanMatchShardResponse.java
index 0a9c98e0c901..d432a5b4f89d 100644
--- a/server/src/main/java/org/elasticsearch/search/CanMatchShardResponse.java
+++ b/server/src/main/java/org/elasticsearch/search/CanMatchShardResponse.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.sort.MinAndMax;
@@ -25,7 +25,7 @@ public final class CanMatchShardResponse extends SearchPhaseResult {
     public CanMatchShardResponse(StreamInput in) throws IOException {
         super(in);
         this.canMatch = in.readBoolean();
-        if (in.getVersion().onOrAfter(Version.V_7_6_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_6_0)) {
             estimatedMinAndMax = in.readOptionalWriteable(MinAndMax::new);
         } else {
             estimatedMinAndMax = null;
@@ -40,7 +40,7 @@ public CanMatchShardResponse(boolean canMatch, MinAndMax<?> estimatedMinAndMax)
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(canMatch);
-        if (out.getVersion().onOrAfter(Version.V_7_6_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_6_0)) {
             out.writeOptionalWriteable(estimatedMinAndMax);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java
index 8a4541ee0aa7..9f508bf68420 100644
--- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java
+++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -241,14 +241,14 @@ public DateTime(StreamInput in) throws IOException {
             this.formatter = DateFormatter.forPattern(formatterPattern).withZone(this.timeZone);
             this.parser = formatter.toDateMathParser();
             this.resolution = DateFieldMapper.Resolution.ofOrdinal(in.readVInt());
-            if (in.getVersion().onOrAfter(Version.V_7_7_0) && in.getVersion().before(Version.V_8_0_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_7_0) && in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 /* when deserialising from 7.7+ nodes expect a flag indicating if a pattern is of joda style
                    This is only used to support joda style indices in 7.x, in 8 we no longer support this.
                    All indices in 8 should use java style pattern. Hence we can ignore this flag.
                 */
                 in.readBoolean();
             }
-            if (in.getVersion().onOrAfter(Version.V_7_13_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0)) {
                 this.formatSortValues = in.readBoolean();
             } else {
                 this.formatSortValues = false;
@@ -265,14 +265,15 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(formatter.pattern());
             out.writeString(timeZone.getId());
             out.writeVInt(resolution.ordinal());
-            if (out.getVersion().onOrAfter(Version.V_7_7_0) && out.getVersion().before(Version.V_8_0_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_7_0)
+                && out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 /* when serializing to 7.7+  send out a flag indicating if a pattern is of joda style
                    This is only used to support joda style indices in 7.x, in 8 we no longer support this.
                    All indices in 8 should use java style pattern. Hence this flag is always false.
                 */
                 out.writeBoolean(false);
             }
-            if (out.getVersion().onOrAfter(Version.V_7_13_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0)) {
                 out.writeBoolean(formatSortValues);
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/search/SearchExtBuilder.java b/server/src/main/java/org/elasticsearch/search/SearchExtBuilder.java
index 26a5f630461c..76dc2cb69122 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchExtBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchExtBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.VersionedNamedWriteable;
@@ -42,7 +42,7 @@ public abstract class SearchExtBuilder implements VersionedNamedWriteable, ToXCo
     public abstract boolean equals(Object obj);
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java
index db059476206f..7a8fefe937de 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchHit.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java
@@ -10,6 +10,7 @@
 
 import org.apache.lucene.search.Explanation;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
@@ -33,7 +34,6 @@
 import org.elasticsearch.search.fetch.subphase.LookupField;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
 import org.elasticsearch.search.lookup.Source;
-import org.elasticsearch.search.lookup.SourceLookup;
 import org.elasticsearch.transport.RemoteClusterAware;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ObjectParser;
@@ -135,7 +135,7 @@ public SearchHit(StreamInput in) throws IOException {
         docId = -1;
         score = in.readFloat();
         id = in.readOptionalText();
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             in.readOptionalText();
         }
         nestedIdentity = in.readOptionalWriteable(NestedIdentity::new);
@@ -149,7 +149,7 @@ public SearchHit(StreamInput in) throws IOException {
         if (in.readBoolean()) {
             explanation = readExplanation(in);
         }
-        if (in.getVersion().onOrAfter(Version.V_7_8_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_8_0)) {
             documentFields.putAll(in.readMap(StreamInput::readString, DocumentField::new));
             metaFields.putAll(in.readMap(StreamInput::readString, DocumentField::new));
         } else {
@@ -402,7 +402,7 @@ public Map<String, Object> getSourceAsMap() {
             return sourceAsMap;
         }
 
-        sourceAsMap = SourceLookup.sourceAsMap(source);
+        sourceAsMap = Source.fromBytes(source).source();
         return sourceAsMap;
     }
 
@@ -1107,6 +1107,9 @@ public Source extractSource(Source root) {
                 if (nestedParsedSource == null) {
                     return Source.empty(root.sourceContentType());
                 }
+                if (nested.getOffset() > nestedParsedSource.size() - 1) {
+                    throw new IllegalStateException("Error retrieving path " + this.field);
+                }
                 rootSourceAsMap = (Map<String, Object>) nestedParsedSource.get(nested.getOffset());
                 if (nested.getChild() == null) {
                     current.put(nestedPath, rootSourceAsMap);
diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java
index fa2699a30b89..f7f9e5ce6651 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -82,6 +82,7 @@
 import org.elasticsearch.plugins.SearchPlugin.FetchPhaseConstructionContext;
 import org.elasticsearch.plugins.SearchPlugin.PipelineAggregationSpec;
 import org.elasticsearch.plugins.SearchPlugin.QuerySpec;
+import org.elasticsearch.plugins.SearchPlugin.QueryVectorBuilderSpec;
 import org.elasticsearch.plugins.SearchPlugin.RescorerSpec;
 import org.elasticsearch.plugins.SearchPlugin.ScoreFunctionSpec;
 import org.elasticsearch.plugins.SearchPlugin.SearchExtSpec;
@@ -244,6 +245,7 @@
 import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
 import org.elasticsearch.search.vectors.KnnScoreDocQueryBuilder;
 import org.elasticsearch.search.vectors.KnnVectorQueryBuilder;
+import org.elasticsearch.search.vectors.QueryVectorBuilder;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentParser;
@@ -305,6 +307,7 @@ public SearchModule(Settings settings, List<SearchPlugin> plugins) {
         registerSorts();
         registerValueFormats();
         registerSignificanceHeuristics(plugins);
+        registerQueryVectorBuilders(plugins);
         this.valuesSourceRegistry = registerAggregations(plugins);
         registerPipelineAggregations(plugins);
         registerFetchSubPhases(plugins);
@@ -980,6 +983,17 @@ private <T extends SignificanceHeuristic> void registerSignificanceHeuristic(Sig
         );
     }
 
+    private void registerQueryVectorBuilders(List<SearchPlugin> plugins) {
+        registerFromPlugin(plugins, SearchPlugin::getQueryVectorBuilders, this::registerQueryVectorBuilder);
+    }
+
+    private <T extends QueryVectorBuilder> void registerQueryVectorBuilder(QueryVectorBuilderSpec<?> spec) {
+        namedXContents.add(new NamedXContentRegistry.Entry(QueryVectorBuilder.class, spec.getName(), p -> spec.getParser().apply(p, null)));
+        namedWriteables.add(
+            new NamedWriteableRegistry.Entry(QueryVectorBuilder.class, spec.getName().getPreferredName(), spec.getReader())
+        );
+    }
+
     private void registerFetchSubPhases(List<SearchPlugin> plugins) {
         registerFetchSubPhase(new ExplainPhase());
         registerFetchSubPhase(new StoredFieldsPhase());
diff --git a/server/src/main/java/org/elasticsearch/search/SearchParseException.java b/server/src/main/java/org/elasticsearch/search/SearchParseException.java
deleted file mode 100644
index 04c463eb4104..000000000000
--- a/server/src/main/java/org/elasticsearch/search/SearchParseException.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.search;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentLocation;
-
-import java.io.IOException;
-
-public class SearchParseException extends SearchException {
-
-    public static final int UNKNOWN_POSITION = -1;
-    private final int lineNumber;
-    private final int columnNumber;
-
-    public SearchParseException(SearchShardTarget shardTarget, String msg, @Nullable XContentLocation location) {
-        this(shardTarget, msg, location, null);
-    }
-
-    public SearchParseException(SearchShardTarget shardTarget, String msg, @Nullable XContentLocation location, Throwable cause) {
-        super(shardTarget, msg, cause);
-        int lineNumber = UNKNOWN_POSITION;
-        int columnNumber = UNKNOWN_POSITION;
-        if (location != null) {
-            lineNumber = location.lineNumber();
-            columnNumber = location.columnNumber();
-        }
-        this.columnNumber = columnNumber;
-        this.lineNumber = lineNumber;
-    }
-
-    public SearchParseException(StreamInput in) throws IOException {
-        super(in);
-        lineNumber = in.readInt();
-        columnNumber = in.readInt();
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeInt(lineNumber);
-        out.writeInt(columnNumber);
-    }
-
-    @Override
-    public RestStatus status() {
-        return RestStatus.BAD_REQUEST;
-    }
-
-    @Override
-    protected void metadataToXContent(XContentBuilder builder, Params params) throws IOException {
-        if (lineNumber != UNKNOWN_POSITION) {
-            builder.field("line", lineNumber);
-            builder.field("col", columnNumber);
-        }
-    }
-
-    /**
-     * Line number of the location of the error
-     *
-     * @return the line number or -1 if unknown
-     */
-    public int getLineNumber() {
-        return lineNumber;
-    }
-
-    /**
-     * Column number of the location of the error
-     *
-     * @return the column number or -1 if unknown
-     */
-    public int getColumnNumber() {
-        return columnNumber;
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java
index 9257e28edd1f..d722ee8f14e5 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.composite;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper.TimeSeriesIdFieldType;
@@ -211,6 +211,9 @@ protected AggregatorFactory doBuild(
         AggregatorFactory parent,
         AggregatorFactories.Builder subfactoriesBuilder
     ) throws IOException {
+        if (context.isInSortOrderExecutionRequired()) {
+            throw new IllegalArgumentException("[composite] aggregation is incompatible with time series execution mode");
+        }
         AggregatorFactory invalid = validateParentAggregations(parent);
         if (invalid != null) {
             throw new IllegalArgumentException(
@@ -295,7 +298,7 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java
index 7d01e7598537..83fc95a43e17 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Rounding;
 import org.elasticsearch.core.Releasables;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.IndexSortConfig;
 import org.elasticsearch.lucene.queries.SearchAfterSortedDocQuery;
 import org.elasticsearch.search.DocValueFormat;
@@ -60,11 +61,13 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.function.BiConsumer;
 import java.util.function.LongUnaryOperator;
 
 import static org.elasticsearch.search.aggregations.MultiBucketConsumerService.MAX_BUCKET_SETTING;
 
 public final class CompositeAggregator extends BucketsAggregator implements SizedBucketAggregator {
+
     private final int size;
     private final List<String> sourceNames;
     private final int[] reverseMuls;
@@ -130,7 +133,7 @@ public final class CompositeAggregator extends BucketsAggregator implements Size
             }
         }
         this.innerSizedBucketAggregators = dateHistogramValuesSources.toArray(new DateHistogramValuesSource[0]);
-        this.queue = new CompositeValuesCollectorQueue(aggCtx.bigArrays(), sources, size);
+        this.queue = new CompositeValuesCollectorQueue(aggCtx.bigArrays(), sources, size, aggCtx.searcher().getIndexReader());
         if (rawAfterKey != null) {
             try {
                 this.queue.setAfterKey(rawAfterKey);
@@ -155,6 +158,14 @@ protected void doClose() {
         }
     }
 
+    @Override
+    public ScoreMode scoreMode() {
+        if (queue.mayDynamicallyPrune()) {
+            return super.scoreMode().needsScores() ? ScoreMode.TOP_DOCS_WITH_SCORES : ScoreMode.TOP_DOCS;
+        }
+        return super.scoreMode();
+    }
+
     @Override
     protected void doPreCollection() throws IOException {
         deferredCollectors = MultiBucketCollector.wrap(false, Arrays.asList(subAggregators));
@@ -492,6 +503,15 @@ public void collect(int doc, long zeroBucket) throws IOException {
                         assert zeroBucket == 0L;
                         inner.collect(doc);
                     }
+
+                    @Override
+                    public DocIdSetIterator competitiveIterator() throws IOException {
+                        if (queue.mayDynamicallyPrune()) {
+                            return inner.competitiveIterator();
+                        } else {
+                            return null;
+                        }
+                    }
                 };
             }
         }
@@ -607,6 +627,14 @@ public double bucketSize(Rounding.DateTimeUnit unit) {
         return innerSizedBucketAggregators[0].bucketSize(unit);
     }
 
+    @Override
+    public void collectDebugInfo(BiConsumer<String, Object> add) {
+        super.collectDebugInfo(add);
+        if (sources[0]instanceof GlobalOrdinalValuesSource globalOrdinalValuesSource) {
+            globalOrdinalValuesSource.collectDebugInfo(Strings.format("sources.%s", sourceConfigs[0].name()), add);
+        }
+    }
+
     private static class Entry {
         final AggregationExecutionContext aggCtx;
         final DocIdSet docIdSet;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueue.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueue.java
index ca09ef61dcad..7057b1dd55c8 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueue.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueue.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.composite;
 
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.CollectionTerminatedException;
 import org.apache.lucene.util.PriorityQueue;
@@ -48,6 +49,11 @@ public int hashCode() {
         }
     }
 
+    @FunctionalInterface
+    private interface CompetitiveBoundsChangedListener {
+        void boundsChanged(int topSlot) throws IOException;
+    }
+
     // the slot for the current candidate
     private static final int CANDIDATE_SLOT = Integer.MAX_VALUE;
 
@@ -55,6 +61,7 @@ public int hashCode() {
     private final int maxSize;
     private final Map<Slot, Integer> map;
     private final SingleDimensionValuesSource<?>[] arrays;
+    private final CompetitiveBoundsChangedListener competitiveBoundsChangedListener;
 
     private LongArray docCounts;
     private boolean afterKeyIsSet = false;
@@ -62,18 +69,83 @@ public int hashCode() {
     /**
      * Constructs a composite queue with the specified size and sources.
      *
-     * @param sources The list of {@link CompositeValuesSourceConfig} to build the composite buckets.
-     * @param size The number of composite buckets to keep.
+     * @param sources     The list of {@link CompositeValuesSourceConfig} to build the composite buckets.
+     * @param size        The number of composite buckets to keep.
+     * @param indexReader
      */
-    CompositeValuesCollectorQueue(BigArrays bigArrays, SingleDimensionValuesSource<?>[] sources, int size) {
+    CompositeValuesCollectorQueue(BigArrays bigArrays, SingleDimensionValuesSource<?>[] sources, int size, IndexReader indexReader) {
         super(size);
         this.bigArrays = bigArrays;
         this.maxSize = size;
         this.arrays = sources;
+
+        // If the leading source is a GlobalOrdinalValuesSource we can apply an optimization which requires
+        // tracking the highest competitive value.
+        if (arrays[0]instanceof GlobalOrdinalValuesSource globalOrdinalValuesSource) {
+            if (shouldApplyGlobalOrdinalDynamicPruningForLeadingSource(sources, size, indexReader)) {
+                competitiveBoundsChangedListener = topSlot -> globalOrdinalValuesSource.updateHighestCompetitiveValue(topSlot);
+            } else {
+                competitiveBoundsChangedListener = null;
+            }
+        } else {
+            competitiveBoundsChangedListener = null;
+        }
+
         this.map = Maps.newMapWithExpectedSize(size);
         this.docCounts = bigArrays.newLongArray(1, false);
     }
 
+    private static boolean shouldApplyGlobalOrdinalDynamicPruningForLeadingSource(
+        SingleDimensionValuesSource<?>[] sources,
+        int size,
+        IndexReader indexReader
+    ) {
+        if (sources.length == 0) {
+            return false;
+        }
+        if (sources[0]instanceof GlobalOrdinalValuesSource firstSource) {
+            if (firstSource.mayDynamicallyPrune(indexReader) == false) {
+                return false;
+            }
+
+            long approximateTotalNumberOfBuckets = firstSource.getUniqueValueCount();
+            if (sources.length > 1) {
+                // When there are multiple sources, it's hard to guess how many
+                // unique buckets there might be. Let's be conservative and
+                // assume that other sources increase the number of buckets by
+                // 3x.
+                approximateTotalNumberOfBuckets *= 3L;
+            }
+            // If the size is not significantly less than the total number of
+            // buckets then dynamic pruning can't help much.
+            if (size >= approximateTotalNumberOfBuckets / 8) {
+                return false;
+            }
+
+            // Try to estimate the width of the ordinal range that might be
+            // returned on each page. Since not all ordinals might match the
+            // query, we're increasing `size` by 25%.
+            long rangeWidthPerPage = size + (size / 4);
+            if (sources.length > 1) {
+                // Again assume other sources bump the number of buckets by 3x
+                rangeWidthPerPage /= 3;
+            }
+            if (rangeWidthPerPage > GlobalOrdinalValuesSource.MAX_TERMS_FOR_DYNAMIC_PRUNING) {
+                return false;
+            }
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * Return true if this queue produces a {@link LeafBucketCollector} that may
+     * dynamically prune hits that are not competitive.
+     */
+    public boolean mayDynamicallyPrune() {
+        return competitiveBoundsChangedListener != null;
+    }
+
     /**
      * Sets after key
      * @param afterKey composite key
@@ -221,7 +293,14 @@ CompositeKey toCompositeKey(int slot) throws IOException {
      * The provided collector <code>in</code> is called on each composite bucket.
      */
     LeafBucketCollector getLeafCollector(LeafReaderContext context, LeafBucketCollector in) throws IOException {
-        return getLeafCollector(null, context, in);
+        LeafBucketCollector leafBucketCollector = getLeafCollector(null, context, in);
+
+        // As we are starting to collect from a new segment we need to update the topChangedListener if present
+        // and if the queue is full.
+        if (competitiveBoundsChangedListener != null && size() >= maxSize) {
+            competitiveBoundsChangedListener.boundsChanged(top());
+        }
+        return leafBucketCollector;
     }
 
     /**
@@ -249,7 +328,7 @@ LeafBucketCollector getLeafCollector(Comparable<?> forceLeadSourceValue, LeafRea
      * Check if the current candidate should be added in the queue.
      * @return <code>true</code> if the candidate is competitive (added or already in the queue).
      */
-    boolean addIfCompetitive(long inc) {
+    boolean addIfCompetitive(long inc) throws IOException {
         return addIfCompetitive(0, inc);
     }
 
@@ -263,7 +342,7 @@ boolean addIfCompetitive(long inc) {
      *
      * @throws CollectionTerminatedException if the current collection can be terminated early due to index sorting.
      */
-    boolean addIfCompetitive(int indexSortSourcePrefix, long inc) {
+    boolean addIfCompetitive(int indexSortSourcePrefix, long inc) throws IOException {
         // checks if the candidate key is competitive
         Integer topSlot = compareCurrent();
         if (topSlot != null) {
@@ -312,6 +391,10 @@ boolean addIfCompetitive(int indexSortSourcePrefix, long inc) {
         copyCurrent(newSlot, inc);
         map.put(new Slot(newSlot), newSlot);
         add(newSlot);
+
+        if (competitiveBoundsChangedListener != null && size() >= maxSize) {
+            competitiveBoundsChangedListener.boundsChanged(top());
+        }
         return true;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java
index 980fa1ca4551..49c29bf722a9 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.composite;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -55,7 +55,7 @@ public abstract class CompositeValuesSourceBuilder<AB extends CompositeValuesSou
             this.userValueTypeHint = ValueType.readFromStream(in);
         }
         this.missingBucket = in.readBoolean();
-        if (in.getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             this.missingOrder = MissingOrder.readFromStream(in);
         }
         this.order = SortOrder.readFromStream(in);
@@ -77,7 +77,7 @@ public final void writeTo(StreamOutput out) throws IOException {
             userValueTypeHint.writeTo(out);
         }
         out.writeBoolean(missingBucket);
-        if (out.getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             missingOrder.writeTo(out);
         }
         order.writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java
index 7468af95b053..036a96dc8757 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.search.aggregations.bucket.composite;
 
 import org.apache.lucene.index.IndexReader;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Rounding;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -105,7 +105,7 @@ protected DateHistogramValuesSourceBuilder(StreamInput in) throws IOException {
         super(in);
         dateHistogramInterval = new DateIntervalWrapper(in);
         timeZone = in.readOptionalZoneId();
-        if (in.getVersion().onOrAfter(Version.V_7_6_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_6_0)) {
             offset = in.readLong();
         }
     }
@@ -114,7 +114,7 @@ protected DateHistogramValuesSourceBuilder(StreamInput in) throws IOException {
     protected void innerWriteTo(StreamOutput out) throws IOException {
         dateHistogramInterval.writeTo(out);
         out.writeOptionalZoneId(timeZone);
-        if (out.getVersion().onOrAfter(Version.V_7_6_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_6_0)) {
             out.writeLong(offset);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java
index 55a9a1f9ecbe..53e8356350b2 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java
@@ -82,7 +82,8 @@ static void register(ValuesSourceRegistry.Builder builder) {
                 ValuesSource.GeoPoint geoPoint = (ValuesSource.GeoPoint) valuesSourceConfig.getValuesSource();
                 // is specified in the builder.
                 final MappedFieldType fieldType = valuesSourceConfig.fieldType();
-                GeoTileCellIdSource cellIdSource = new GeoTileCellIdSource(geoPoint, precision, boundingBox);
+                // TODO: support adding circuit breaker in composite
+                GeoTileCellIdSource cellIdSource = new GeoTileCellIdSource(geoPoint, precision, boundingBox, cb -> {});
                 return new CompositeValuesSourceConfig(
                     name,
                     fieldType,
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GlobalOrdinalValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GlobalOrdinalValuesSource.java
index 30009bad1acc..855b45654631 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GlobalOrdinalValuesSource.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GlobalOrdinalValuesSource.java
@@ -10,20 +10,33 @@
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.PostingsEnum;
 import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.PriorityQueue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.LongArray;
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.core.Releasables;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.StringFieldType;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.ArrayDeque;
+import java.util.List;
+import java.util.function.BiConsumer;
 
 import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS;
 
@@ -31,13 +44,23 @@
  * A {@link SingleDimensionValuesSource} for global ordinals.
  */
 class GlobalOrdinalValuesSource extends SingleDimensionValuesSource<BytesRef> {
+
+    private static final Logger logger = LogManager.getLogger(GlobalOrdinalValuesSource.class);
+
+    public static final int MAX_TERMS_FOR_DYNAMIC_PRUNING = 128;
+
     public static final long MISSING_VALUE_FLAG = -1L;
+    private final long uniqueValueCount;
     private final CheckedFunction<LeafReaderContext, SortedSetDocValues, IOException> docValuesFunc;
     private LongArray values;
     private SortedSetDocValues lookup;
     private long currentValue;
     private Long afterValueGlobalOrd;
+    private Long highestCompetitiveValueGlobalOrd;
     private boolean isTopValueInsertionPoint;
+    private CompetitiveIterator currentCompetitiveIterator;
+    private int segmentsDynamicPruningUsed;
+    private int totalSegments;
 
     private long lastLookupOrd = -1;
     private BytesRef lastLookupValue;
@@ -45,6 +68,7 @@ class GlobalOrdinalValuesSource extends SingleDimensionValuesSource<BytesRef> {
     GlobalOrdinalValuesSource(
         BigArrays bigArrays,
         MappedFieldType type,
+        long uniqueValueCount,
         CheckedFunction<LeafReaderContext, SortedSetDocValues, IOException> docValuesFunc,
         DocValueFormat format,
         boolean missingBucket,
@@ -53,10 +77,58 @@ class GlobalOrdinalValuesSource extends SingleDimensionValuesSource<BytesRef> {
         int reverseMul
     ) {
         super(bigArrays, format, type, missingBucket, missingOrder, size, reverseMul);
+        this.uniqueValueCount = uniqueValueCount;
         this.docValuesFunc = docValuesFunc;
         this.values = bigArrays.newLongArray(Math.min(size, 100), false);
     }
 
+    /**
+     * Return the number of unique values of this source. Note that some unique
+     * values might not produce buckets if the query doesn't match documents
+     * that contain these values.
+     */
+    long getUniqueValueCount() {
+        return uniqueValueCount;
+    }
+
+    /**
+     * Return whether the source can be used for dynamic pruning.
+     */
+    boolean mayDynamicallyPrune(IndexReader indexReader) {
+        // If missing bucket is requested we have no way to tell lucene to efficiently match
+        // buckets in a range AND missing values.
+        if (missingBucket) {
+            return false;
+        }
+
+        // If we do not know the field type, the field is not indexed (e.g. runtime field) or the field
+        // has no doc values we should not apply the optimization.
+        if (fieldType == null || fieldType.isIndexed() == false || fieldType.hasDocValues() == false) {
+            return false;
+        }
+
+        // We also need to check if there are terms for the field in question.
+        // Some field types do not despite being global ordinals (e.g. IP addresses).
+        return hasTerms(indexReader);
+    }
+
+    private boolean hasTerms(IndexReader indexReader) {
+        assert fieldType != null;
+        List<LeafReaderContext> leaves = indexReader.leaves();
+        for (LeafReaderContext leaf : leaves) {
+            Terms terms;
+            try {
+                terms = leaf.reader().terms(fieldType.name());
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+            if (terms != null) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     @Override
     void copyCurrent(int slot) {
         values = bigArrays.grow(values, slot + 1);
@@ -133,11 +205,21 @@ BytesRef toComparable(int slot) throws IOException {
 
     @Override
     LeafBucketCollector getLeafCollector(LeafReaderContext context, LeafBucketCollector next) throws IOException {
+        totalSegments++;
         final SortedSetDocValues dvs = docValuesFunc.apply(context);
         if (lookup == null) {
             initLookup(dvs);
         }
+
+        // We create a competitive iterator that allows us to optimize by narrowing down the search
+        // to terms that are within the range based on what the composite queue is tracking.
+        // For example, if the composite agg size is 5, and we have seen terms with ordinals [1, 4, 5, 10, 11],
+        // we know that we never need to look at terms with and ordinal higher than 11.
+        final CompetitiveIterator competitiveIterator = fieldType == null ? null : new CompetitiveIterator(context, fieldType.name());
+        currentCompetitiveIterator = competitiveIterator;
+
         return new LeafBucketCollector() {
+
             @Override
             public void collect(int doc, long bucket) throws IOException {
                 if (dvs.advanceExact(doc)) {
@@ -151,6 +233,11 @@ public void collect(int doc, long bucket) throws IOException {
                     next.collect(doc, bucket);
                 }
             }
+
+            @Override
+            public DocIdSetIterator competitiveIterator() {
+                return competitiveIterator;
+            }
         };
     }
 
@@ -160,6 +247,7 @@ LeafBucketCollector getLeafCollector(Comparable<BytesRef> value, LeafReaderConte
         if (value.getClass() != BytesRef.class) {
             throw new IllegalArgumentException("Expected BytesRef, got " + value.getClass());
         }
+        totalSegments++;
         BytesRef term = (BytesRef) value;
         final SortedSetDocValues dvs = docValuesFunc.apply(context);
         if (lookup == null) {
@@ -214,4 +302,170 @@ private void initLookup(SortedSetDocValues dvs) throws IOException {
             }
         }
     }
+
+    public void updateHighestCompetitiveValue(int slot) throws IOException {
+        highestCompetitiveValueGlobalOrd = values.get(slot);
+        logger.trace("Highest observed set to [{}]", highestCompetitiveValueGlobalOrd);
+        final CompetitiveIterator competitiveIterator = currentCompetitiveIterator;
+        if (competitiveIterator != null) {
+            competitiveIterator.updateBounds();
+        }
+    }
+
+    void collectDebugInfo(String namespace, BiConsumer<String, Object> add) {
+        add.accept(Strings.format("%s.segments_collected", namespace), totalSegments);
+        add.accept(Strings.format("%s.segments_dynamic_pruning_used", namespace), segmentsDynamicPruningUsed);
+    }
+
+    private record PostingsEnumAndOrd(PostingsEnum postings, long ord) {}
+
+    private class CompetitiveIterator extends DocIdSetIterator {
+
+        private final LeafReaderContext context;
+        private final int maxDoc;
+        private final String field;
+        private int doc = -1;
+        private ArrayDeque<PostingsEnumAndOrd> postings;
+        private DocIdSetIterator docsWithField;
+        private PriorityQueue<PostingsEnumAndOrd> disjunction;
+
+        CompetitiveIterator(LeafReaderContext context, String field) {
+            this.context = context;
+            this.maxDoc = context.reader().maxDoc();
+            this.field = field;
+        }
+
+        @Override
+        public int docID() {
+            return doc;
+        }
+
+        @Override
+        public int nextDoc() throws IOException {
+            return advance(docID() + 1);
+        }
+
+        @Override
+        public int advance(int target) throws IOException {
+            if (target >= maxDoc) {
+                return doc = NO_MORE_DOCS;
+            } else if (disjunction == null) {
+                if (docsWithField != null) {
+                    return doc = docsWithField.advance(target);
+                } else {
+                    // We haven't started skipping yet
+                    return doc = target;
+                }
+            } else {
+                PostingsEnumAndOrd top = disjunction.top();
+                if (top == null) {
+                    // priority queue is empty, none of the remaining documents are competitive
+                    return doc = NO_MORE_DOCS;
+                }
+                while (top.postings.docID() < target) {
+                    top.postings.advance(target);
+                    top = disjunction.updateTop();
+                }
+                return doc = top.postings.docID();
+            }
+        }
+
+        @Override
+        public long cost() {
+            return context.reader().maxDoc();
+        }
+
+        /**
+         * Update this iterator to only match postings whose term has an ordinal between {@code minOrd}
+         * included and {@code maxOrd} included.
+         */
+        private void update(long minOrd, long maxOrd) throws IOException {
+            final int maxTerms = Math.min(MAX_TERMS_FOR_DYNAMIC_PRUNING, IndexSearcher.getMaxClauseCount());
+            final long size = Math.max(0, maxOrd - minOrd + 1);
+            if (size > maxTerms) {
+                if (docsWithField == null) {
+                    docsWithField = docValuesFunc.apply(context);
+                }
+            } else if (postings == null) {
+                init(minOrd, maxOrd);
+            } else {
+                boolean removed = false;
+                // Zero or more ords got removed
+                while (postings.isEmpty() == false && postings.getFirst().ord < minOrd) {
+                    removed = true;
+                    postings.removeFirst();
+                }
+                while (postings.isEmpty() == false && postings.getLast().ord > maxOrd) {
+                    removed = true;
+                    postings.removeLast();
+                }
+                if (removed) {
+                    disjunction.clear();
+                    disjunction.addAll(postings);
+                }
+            }
+        }
+
+        /**
+         * For the first time, this iterator is allowed to skip documents. It needs to pull {@link
+         * PostingsEnum}s from the terms dictionary of the inverted index and create a priority queue
+         * out of them.
+         */
+        private void init(long minOrd, long maxOrd) throws IOException {
+            segmentsDynamicPruningUsed++;
+
+            final int size = (int) Math.max(0, maxOrd - minOrd + 1);
+            postings = new ArrayDeque<>(size);
+            if (size > 0) {
+                Terms terms = context.reader().terms(field);
+                if (terms != null) {
+                    BytesRef minTerm = BytesRef.deepCopyOf(lookup.lookupOrd(minOrd));
+                    TermsEnum termsEnum = terms.iterator();
+                    TermsEnum.SeekStatus seekStatus = termsEnum.seekCeil(minTerm);
+                    if (seekStatus != TermsEnum.SeekStatus.END) {
+                        BytesRef maxTerm = BytesRef.deepCopyOf(lookup.lookupOrd(maxOrd));
+
+                        TermsEnum globalTermsEnum = lookup.termsEnum();
+                        globalTermsEnum.seekExact(minOrd);
+
+                        for (BytesRef term = termsEnum.term(); term != null && term.compareTo(maxTerm) <= 0; term = termsEnum.next()) {
+                            // Compute the global ordinal of this term by advancing the global terms enum to the same term and retrieving
+                            // the term ordinal. This is cheaper than calling lookupTerm for every term.
+                            while (globalTermsEnum.term().compareTo(term) < 0) {
+                                BytesRef nextGlobalTerm = globalTermsEnum.next();
+                                assert nextGlobalTerm != null;
+                            }
+                            assert globalTermsEnum.term().equals(term);
+                            final long globalOrd = globalTermsEnum.ord();
+                            postings.add(new PostingsEnumAndOrd(termsEnum.postings(null, PostingsEnum.NONE), globalOrd));
+                        }
+                    }
+                }
+            }
+            disjunction = new PriorityQueue<>(size) {
+                @Override
+                protected boolean lessThan(PostingsEnumAndOrd a, PostingsEnumAndOrd b) {
+                    return a.postings.docID() < b.postings.docID();
+                }
+            };
+            disjunction.addAll(postings);
+        }
+
+        public void updateBounds() throws IOException {
+            if (highestCompetitiveValueGlobalOrd == null) {
+                return;
+            }
+
+            // TODO If this is the only source, we know we are done with the buckets of the after_key.
+            // We could optimize even further by skipping to the next global ordinal after the after_key.
+
+            long lowOrd;
+            if (afterValueGlobalOrd != null && afterValueGlobalOrd != MISSING_VALUE_FLAG) {
+                lowOrd = afterValueGlobalOrd;
+            } else {
+                lowOrd = reverseMul == 1 ? 0 : lookup.getValueCount() - 1;
+            }
+            update(Math.min(highestCompetitiveValueGlobalOrd, lowOrd), Math.max(highestCompetitiveValueGlobalOrd, lowOrd));
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java
index d8aa4aa73dc1..12d1be0e02ef 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.PriorityQueue;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.DocValueFormat;
@@ -80,7 +80,7 @@ public InternalComposite(StreamInput in) throws IOException {
             formats.add(in.readNamedWriteable(DocValueFormat.class));
         }
         this.reverseMuls = in.readIntArray();
-        if (in.getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             this.missingOrders = in.readArray(MissingOrder::readFromStream, MissingOrder[]::new);
         } else {
             this.missingOrders = new MissingOrder[reverseMuls.length];
@@ -88,7 +88,7 @@ public InternalComposite(StreamInput in) throws IOException {
         }
         this.buckets = in.readList((input) -> new InternalBucket(input, sourceNames, formats, reverseMuls, missingOrders));
         this.afterKey = in.readOptionalWriteable(CompositeKey::new);
-        this.earlyTerminated = in.getVersion().onOrAfter(Version.V_7_6_0) ? in.readBoolean() : false;
+        this.earlyTerminated = in.getTransportVersion().onOrAfter(TransportVersion.V_7_6_0) ? in.readBoolean() : false;
     }
 
     @Override
@@ -99,12 +99,12 @@ protected void doWriteTo(StreamOutput out) throws IOException {
             out.writeNamedWriteable(format);
         }
         out.writeIntArray(reverseMuls);
-        if (out.getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             out.writeArray((o, order) -> order.writeTo(o), missingOrders);
         }
         out.writeList(buckets);
         out.writeOptionalWriteable(afterKey);
-        if (out.getVersion().onOrAfter(Version.V_7_6_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_6_0)) {
             out.writeBoolean(earlyTerminated);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/TermsValuesSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/TermsValuesSourceBuilder.java
index dcbaed78ba06..e5e89d94c803 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/TermsValuesSourceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/TermsValuesSourceBuilder.java
@@ -168,9 +168,16 @@ static void register(ValuesSourceRegistry.Builder builder) {
 
                     if (valuesSourceConfig.hasOrdinals() && reader instanceof DirectoryReader) {
                         ValuesSource.Bytes.WithOrdinals vs = (ValuesSource.Bytes.WithOrdinals) compositeValuesSourceConfig.valuesSource();
+                        long maxOrd;
+                        try {
+                            maxOrd = vs.globalMaxOrd(reader);
+                        } catch (IOException e) {
+                            throw new UnsupportedOperationException(e);
+                        }
                         return new GlobalOrdinalValuesSource(
                             bigArrays,
                             compositeValuesSourceConfig.fieldType(),
+                            maxOrd,
                             vs::globalOrdinalsValues,
                             compositeValuesSourceConfig.format(),
                             compositeValuesSourceConfig.missingBucket(),
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
index 1837afbbaf84..9a85f3fff9f5 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.filter;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -150,8 +150,8 @@ public QueryBuilder getFilter() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     public static class FilterAggregatorFactory extends AggregatorFactory {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterByFilterAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterByFilterAggregator.java
index eaa35d5fe9c1..312df1cf8b3e 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterByFilterAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterByFilterAggregator.java
@@ -13,7 +13,6 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorable;
 import org.apache.lucene.util.Bits;
-import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.search.aggregations.AdaptingAggregator;
 import org.elasticsearch.search.aggregations.AggregationExecutionContext;
@@ -70,7 +69,7 @@ public AdapterBuilder(
             this.cardinality = cardinality;
             this.metadata = metadata;
             this.rewrittenTopLevelQuery = aggCtx.searcher().rewrite(aggCtx.query());
-            this.valid = parent == null && otherBucketKey == null;
+            this.valid = parent == null && otherBucketKey == null && aggCtx.isInSortOrderExecutionRequired() == false;
         }
 
         /**
@@ -224,6 +223,9 @@ protected LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCt
             return LeafBucketCollector.NO_OP_COLLECTOR;
         }
         Bits live = aggCtx.getLeafReaderContext().reader().getLiveDocs();
+        if (live != null) {
+            segmentsWithDeletedDocs++;
+        }
         if (false == docCountProvider.alwaysOne()) {
             segmentsWithDocCountField++;
         }
@@ -304,25 +306,4 @@ public void collectDebugInfo(BiConsumer<String, Object> add) {
         add.accept("segments_with_doc_count_field", segmentsWithDocCountField);
     }
 
-    CheckedSupplier<Boolean, IOException> canUseMetadata(LeafReaderContext ctx) {
-        return new CheckedSupplier<Boolean, IOException>() {
-            Boolean canUse;
-
-            @Override
-            public Boolean get() throws IOException {
-                if (canUse == null) {
-                    canUse = canUse();
-                }
-                return canUse;
-            }
-
-            private boolean canUse() throws IOException {
-                if (ctx.reader().getLiveDocs() != null) {
-                    return false;
-                }
-                docCountProvider.setLeafReaderContext(ctx);
-                return docCountProvider.alwaysOne();
-            }
-        };
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregationBuilder.java
index ad91632402ac..da3f783bd022 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.filter;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -360,7 +360,7 @@ public String getType() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java
index 9189fd6c636d..4e5d92b191bc 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java
@@ -9,13 +9,13 @@
 package org.elasticsearch.search.aggregations.bucket.filter;
 
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BulkScorer;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.LeafCollector;
 import org.apache.lucene.search.PointRangeQuery;
 import org.apache.lucene.search.Query;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java
index 54a6a7989f02..d3de50aa1635 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 
 import java.io.IOException;
+import java.util.function.LongConsumer;
 
 /**
  * Base class to help convert {@link MultiGeoPointValues} to {@link CellMultiValues}
@@ -34,11 +35,14 @@ public abstract class CellIdSource extends ValuesSource.Numeric {
     private final GeoBoundingBox geoBoundingBox;
     private final boolean crossesDateline;
 
-    protected CellIdSource(GeoPoint valuesSource, int precision, GeoBoundingBox geoBoundingBox) {
+    protected final LongConsumer circuitBreakerConsumer;
+
+    protected CellIdSource(GeoPoint valuesSource, int precision, GeoBoundingBox geoBoundingBox, LongConsumer circuitBreakerConsumer) {
         this.valuesSource = valuesSource;
         this.precision = precision;
         this.geoBoundingBox = geoBoundingBox;
         this.crossesDateline = geoBoundingBox.left() > geoBoundingBox.right();
+        this.circuitBreakerConsumer = circuitBreakerConsumer;
     }
 
     protected final int precision() {
@@ -99,7 +103,7 @@ public final SortedBinaryDocValues bytesValues(LeafReaderContext ctx) {
      *
      * This method maybe faster than having to compute the bounding box for each point grid.
      * */
-    protected boolean validPoint(double lon, double lat) {
+    protected boolean pointInBounds(double lon, double lat) {
         if (geoBoundingBox.top() > lat && geoBoundingBox.bottom() < lat) {
             if (crossesDateline) {
                 return geoBoundingBox.left() < lon || geoBoundingBox.right() > lon;
@@ -119,7 +123,8 @@ protected abstract static class CellMultiValues extends AbstractSortingNumericDo
         private final MultiGeoPointValues geoValues;
         protected final int precision;
 
-        protected CellMultiValues(MultiGeoPointValues geoValues, int precision) {
+        protected CellMultiValues(MultiGeoPointValues geoValues, int precision, LongConsumer circuitBreakerConsumer) {
+            super(circuitBreakerConsumer);
             this.geoValues = geoValues;
             this.precision = precision;
         }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
index 0d5f708d3875..5adb8bbebd47 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.search.aggregations.bucket.geogrid;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.geo.GeoBoundingBox;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -91,7 +91,7 @@ public GeoGridAggregationBuilder(StreamInput in) throws IOException {
         precision = in.readVInt();
         requiredSize = in.readVInt();
         shardSize = in.readVInt();
-        if (in.getVersion().onOrAfter(Version.V_7_6_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_6_0)) {
             geoBoundingBox = new GeoBoundingBox(in);
         }
     }
@@ -111,7 +111,7 @@ protected void innerWriteTo(StreamOutput out) throws IOException {
         out.writeVInt(precision);
         out.writeVInt(requiredSize);
         out.writeVInt(shardSize);
-        if (out.getVersion().onOrAfter(Version.V_7_6_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_6_0)) {
             geoBoundingBox.writeTo(out);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregator.java
index 1a898304af42..cc5e666096e5 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregator.java
@@ -29,6 +29,8 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
+import java.util.function.LongConsumer;
 
 /**
  * Aggregates data expressed as longs (for efficiency's sake) but formats results as aggregation-specific strings.
@@ -43,7 +45,7 @@ public abstract class GeoGridAggregator<T extends InternalGeoGrid<?>> extends Bu
     protected GeoGridAggregator(
         String name,
         AggregatorFactories factories,
-        ValuesSource.Numeric valuesSource,
+        Function<LongConsumer, ValuesSource.Numeric> valuesSource,
         int requiredSize,
         int shardSize,
         AggregationContext aggregationContext,
@@ -52,7 +54,7 @@ protected GeoGridAggregator(
         Map<String, Object> metadata
     ) throws IOException {
         super(name, factories, aggregationContext, parent, CardinalityUpperBound.MANY, metadata);
-        this.valuesSource = valuesSource;
+        this.valuesSource = valuesSource.apply(this::addRequestCircuitBreakerBytes);
         this.requiredSize = requiredSize;
         this.shardSize = shardSize;
         bucketOrds = LongKeyedBucketOrds.build(bigArrays(), cardinality);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashCellIdSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashCellIdSource.java
index dfd83085a675..71d85a09c2b6 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashCellIdSource.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashCellIdSource.java
@@ -15,13 +15,20 @@
 import org.elasticsearch.index.fielddata.MultiGeoPointValues;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 
+import java.util.function.LongConsumer;
+
 /**
  * {@link CellIdSource} implementation for Geohash aggregation
  */
 public class GeoHashCellIdSource extends CellIdSource {
 
-    public GeoHashCellIdSource(ValuesSource.GeoPoint valuesSource, int precision, GeoBoundingBox geoBoundingBox) {
-        super(valuesSource, precision, geoBoundingBox);
+    public GeoHashCellIdSource(
+        ValuesSource.GeoPoint valuesSource,
+        int precision,
+        GeoBoundingBox geoBoundingBox,
+        LongConsumer circuitBreakerConsumer
+    ) {
+        super(valuesSource, precision, geoBoundingBox, circuitBreakerConsumer);
     }
 
     @Override
@@ -42,7 +49,7 @@ protected NumericDocValues boundedCellSingleValue(GeoPointValues values, GeoBoun
             @Override
             protected boolean advance(org.elasticsearch.common.geo.GeoPoint target) {
                 final String hash = Geohash.stringEncode(target.getLon(), target.getLat(), precision);
-                if (validPoint(target.getLon(), target.getLat()) || predicate.validHash(hash)) {
+                if (pointInBounds(target.getLon(), target.getLat()) || predicate.validHash(hash)) {
                     value = Geohash.longEncode(hash);
                     return true;
                 }
@@ -53,7 +60,7 @@ protected boolean advance(org.elasticsearch.common.geo.GeoPoint target) {
 
     @Override
     protected SortedNumericDocValues unboundedCellMultiValues(MultiGeoPointValues values) {
-        return new CellMultiValues(values, precision()) {
+        return new CellMultiValues(values, precision(), circuitBreakerConsumer) {
             @Override
             protected int advanceValue(org.elasticsearch.common.geo.GeoPoint target, int valuesIdx) {
                 values[valuesIdx] = Geohash.longEncode(target.getLon(), target.getLat(), precision);
@@ -65,11 +72,11 @@ protected int advanceValue(org.elasticsearch.common.geo.GeoPoint target, int val
     @Override
     protected SortedNumericDocValues boundedCellMultiValues(MultiGeoPointValues values, GeoBoundingBox boundingBox) {
         final GeoHashBoundedPredicate predicate = new GeoHashBoundedPredicate(precision(), boundingBox);
-        return new CellMultiValues(values, precision()) {
+        return new CellMultiValues(values, precision(), circuitBreakerConsumer) {
             @Override
             protected int advanceValue(org.elasticsearch.common.geo.GeoPoint target, int valuesIdx) {
                 final String hash = Geohash.stringEncode(target.getLon(), target.getLat(), precision);
-                if (validPoint(target.getLon(), target.getLat()) || predicate.validHash(hash)) {
+                if (pointInBounds(target.getLon(), target.getLat()) || predicate.validHash(hash)) {
                     values[valuesIdx] = Geohash.longEncode(hash);
                     return valuesIdx + 1;
                 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java
index ed7d84a16d38..ac51799acf70 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.geogrid;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.geo.GeoBoundingBox;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -115,7 +115,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java
index 2de98c3d569b..9d1c6f04ef28 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java
@@ -17,6 +17,8 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
+import java.util.function.LongConsumer;
 
 /**
  * Aggregates data expressed as GeoHash longs (for efficiency's sake) but formats results as Geohash strings.
@@ -26,7 +28,7 @@ public class GeoHashGridAggregator extends GeoGridAggregator<InternalGeoHashGrid
     public GeoHashGridAggregator(
         String name,
         AggregatorFactories factories,
-        ValuesSource.Numeric valuesSource,
+        Function<LongConsumer, ValuesSource.Numeric> valuesSource,
         int requiredSize,
         int shardSize,
         AggregationContext context,
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java
index 91638be9901b..d47e6aefd0a6 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java
@@ -91,20 +91,28 @@ static void registerAggregators(ValuesSourceRegistry.Builder builder) {
         builder.register(
             GeoHashGridAggregationBuilder.REGISTRY_KEY,
             CoreValuesSourceType.GEOPOINT,
-            (name, factories, valuesSource, precision, geoBoundingBox, requiredSize, shardSize, context, parent, cardinality, metadata) -> {
-                GeoHashCellIdSource cellIdSource = new GeoHashCellIdSource((ValuesSource.GeoPoint) valuesSource, precision, geoBoundingBox);
-                return new GeoHashGridAggregator(
+            (
+                name,
+                factories,
+                valuesSource,
+                precision,
+                geoBoundingBox,
+                requiredSize,
+                shardSize,
+                context,
+                parent,
+                cardinality,
+                metadata) -> new GeoHashGridAggregator(
                     name,
                     factories,
-                    cellIdSource,
+                    cb -> new GeoHashCellIdSource((ValuesSource.GeoPoint) valuesSource, precision, geoBoundingBox, cb),
                     requiredSize,
                     shardSize,
                     context,
                     parent,
                     cardinality,
                     metadata
-                );
-            },
+                ),
             true
         );
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileBoundedPredicate.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileBoundedPredicate.java
index 292d7f5b458b..ac64e8f24627 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileBoundedPredicate.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileBoundedPredicate.java
@@ -29,7 +29,7 @@ public GeoTileBoundedPredicate(int precision, GeoBoundingBox bbox) {
             leftX = rightX = minY = maxY = -1;
             maxTiles = 0;
         } else {
-            final long tiles = 1L << precision;
+            final int tiles = 1 << precision;
             // compute minX, minY
             final int minX = GeoTileUtils.getXTile(bbox.left(), tiles);
             final int minY = GeoTileUtils.getYTile(bbox.top(), tiles);
@@ -45,7 +45,7 @@ public GeoTileBoundedPredicate(int precision, GeoBoundingBox bbox) {
             this.rightX = maxTile.getMinX() == bbox.right() ? maxX : maxX + 1;
             this.maxY = maxTile.getMaxY() == bbox.bottom() ? maxY : maxY + 1;
             if (crossesDateline) {
-                this.maxTiles = (tiles + this.rightX - this.leftX) * (this.maxY - this.minY);
+                this.maxTiles = ((long) tiles + this.rightX - this.leftX) * (this.maxY - this.minY);
             } else {
                 this.maxTiles = (long) (this.rightX - this.leftX) * (this.maxY - this.minY);
             }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileCellIdSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileCellIdSource.java
index fdb226a4d628..71231fd6cab1 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileCellIdSource.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileCellIdSource.java
@@ -13,13 +13,15 @@
 import org.elasticsearch.index.fielddata.GeoPointValues;
 import org.elasticsearch.index.fielddata.MultiGeoPointValues;
 
+import java.util.function.LongConsumer;
+
 /**
  * {@link CellIdSource} implementation for GeoTile aggregation
  */
 public class GeoTileCellIdSource extends CellIdSource {
 
-    public GeoTileCellIdSource(GeoPoint valuesSource, int precision, GeoBoundingBox geoBoundingBox) {
-        super(valuesSource, precision, geoBoundingBox);
+    public GeoTileCellIdSource(GeoPoint valuesSource, int precision, GeoBoundingBox geoBoundingBox, LongConsumer circuitBreakerConsumer) {
+        super(valuesSource, precision, geoBoundingBox, circuitBreakerConsumer);
     }
 
     @Override
@@ -36,7 +38,7 @@ protected boolean advance(org.elasticsearch.common.geo.GeoPoint target) {
     @Override
     protected NumericDocValues boundedCellSingleValue(GeoPointValues values, GeoBoundingBox boundingBox) {
         final GeoTileBoundedPredicate predicate = new GeoTileBoundedPredicate(precision(), boundingBox);
-        final long tiles = 1L << precision();
+        final int tiles = 1 << precision();
         return new CellSingleValue(values, precision()) {
             @Override
             protected boolean advance(org.elasticsearch.common.geo.GeoPoint target) {
@@ -53,7 +55,7 @@ protected boolean advance(org.elasticsearch.common.geo.GeoPoint target) {
 
     @Override
     protected SortedNumericDocValues unboundedCellMultiValues(MultiGeoPointValues values) {
-        return new CellMultiValues(values, precision()) {
+        return new CellMultiValues(values, precision(), circuitBreakerConsumer) {
             @Override
             protected int advanceValue(org.elasticsearch.common.geo.GeoPoint target, int valuesIdx) {
                 values[valuesIdx] = GeoTileUtils.longEncode(target.getLon(), target.getLat(), precision);
@@ -65,8 +67,8 @@ protected int advanceValue(org.elasticsearch.common.geo.GeoPoint target, int val
     @Override
     protected SortedNumericDocValues boundedCellMultiValues(MultiGeoPointValues values, GeoBoundingBox boundingBox) {
         final GeoTileBoundedPredicate predicate = new GeoTileBoundedPredicate(precision(), boundingBox);
-        final long tiles = 1L << precision();
-        return new CellMultiValues(values, precision()) {
+        final int tiles = 1 << precision();
+        return new CellMultiValues(values, precision(), circuitBreakerConsumer) {
             @Override
             protected int advanceValue(org.elasticsearch.common.geo.GeoPoint target, int valuesIdx) {
                 final int x = GeoTileUtils.getXTile(target.getLon(), tiles);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java
index 5b227392436a..7d6c6bbd15e3 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.geogrid;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.geo.GeoBoundingBox;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -111,7 +111,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_0_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_0_0;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java
index 73975302fa4c..ebe25477bec8 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java
@@ -18,6 +18,8 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
+import java.util.function.LongConsumer;
 
 /**
  * Aggregates data expressed as geotile longs (for efficiency's sake) but formats results as geotile strings.
@@ -27,7 +29,7 @@ public class GeoTileGridAggregator extends GeoGridAggregator<InternalGeoTileGrid
     public GeoTileGridAggregator(
         String name,
         AggregatorFactories factories,
-        ValuesSource.Numeric valuesSource,
+        Function<LongConsumer, ValuesSource.Numeric> valuesSource,
         int requiredSize,
         int shardSize,
         AggregationContext context,
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java
index 3851d0d9bb65..8825c3b36aac 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java
@@ -98,20 +98,17 @@ static void registerAggregators(ValuesSourceRegistry.Builder builder) {
                 aggregationContext,
                 parent,
                 cardinality,
-                metadata) -> {
-                GeoTileCellIdSource cellIdSource = new GeoTileCellIdSource((ValuesSource.GeoPoint) valuesSource, precision, geoBoundingBox);
-                return new GeoTileGridAggregator(
+                metadata) -> new GeoTileGridAggregator(
                     name,
                     factories,
-                    cellIdSource,
+                    cb -> new GeoTileCellIdSource((ValuesSource.GeoPoint) valuesSource, precision, geoBoundingBox, cb),
                     requiredSize,
                     shardSize,
                     aggregationContext,
                     parent,
                     cardinality,
                     metadata
-                );
-            },
+                ),
             true
         );
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java
index f541f4e20c16..a8abfd258f38 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java
@@ -35,7 +35,11 @@ public final class GeoTileUtils {
 
     private GeoTileUtils() {}
 
-    private static final double PI_DIV_2 = Math.PI / 2;
+    private static final double PI_DIV_2 = Math.PI / 2.0;
+
+    private static final double PI_TIMES_2 = Math.PI * 2.0;
+
+    private static final double PI_TIMES_4 = Math.PI * 4.0;
 
     /**
      * Largest number of tiles (precision) to use.
@@ -104,24 +108,14 @@ public static int checkPrecisionRange(int precision) {
      * @param longitude the longitude to use when determining the tile x-coordinate
      * @param tiles     the number of tiles per row for a pre-determined zoom-level
      */
-    public static int getXTile(double longitude, long tiles) {
+    public static int getXTile(double longitude, int tiles) {
         // normalizeLon treats this as 180, which is not friendly for tile mapping
         if (longitude == -180) {
             return 0;
         }
-
-        int xTile = (int) Math.floor((normalizeLon(longitude) + 180) / 360 * tiles);
-
+        final double xTile = (normalizeLon(longitude) + 180.0) / 360.0 * tiles;
         // Edge values may generate invalid values, and need to be clipped.
-        // For example, polar regions (above/below lat 85.05112878) get normalized.
-        if (xTile < 0) {
-            return 0;
-        }
-        if (xTile >= tiles) {
-            return (int) tiles - 1;
-        }
-
-        return xTile;
+        return Math.max(0, Math.min(tiles - 1, (int) Math.floor(xTile)));
     }
 
     /**
@@ -131,18 +125,12 @@ public static int getXTile(double longitude, long tiles) {
      * @param latitude  the latitude to use when determining the tile y-coordinate
      * @param tiles     the number of tiles per column for a pre-determined zoom-level
      */
-    public static int getYTile(double latitude, long tiles) {
-        double latSin = SloppyMath.cos(PI_DIV_2 - Math.toRadians(normalizeLat(latitude)));
-        int yTile = (int) Math.floor((0.5 - (ESSloppyMath.log((1 + latSin) / (1 - latSin)) / (4 * Math.PI))) * tiles);
-
-        if (yTile < 0) {
-            yTile = 0;
-        }
-        if (yTile >= tiles) {
-            return (int) tiles - 1;
-        }
-
-        return yTile;
+    public static int getYTile(double latitude, int tiles) {
+        final double latSin = SloppyMath.cos(PI_DIV_2 - Math.toRadians(normalizeLat(latitude)));
+        final double yTile = (0.5 - (ESSloppyMath.log((1.0 + latSin) / (1.0 - latSin)) / PI_TIMES_4)) * tiles;
+        // Edge values may generate invalid values, and need to be clipped.
+        // For example, polar regions (above/below lat 85.05112878) get normalized.
+        return Math.max(0, Math.min(tiles - 1, (int) Math.floor(yTile)));
     }
 
     /**
@@ -153,10 +141,8 @@ public static int getYTile(double latitude, long tiles) {
     public static long longEncode(double longitude, double latitude, int precision) {
         // Mathematics for this code was adapted from https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Java
         // Number of tiles for the current zoom level along the X and Y axis
-        final long tiles = 1 << checkPrecisionRange(precision);
-        long xTile = getXTile(longitude, tiles);
-        long yTile = getYTile(latitude, tiles);
-        return longEncodeTiles(precision, xTile, yTile);
+        final int tiles = 1 << checkPrecisionRange(precision);
+        return longEncodeTiles(precision, getXTile(longitude, tiles), getYTile(latitude, tiles));
     }
 
     /**
@@ -166,38 +152,28 @@ public static long longEncode(double longitude, double latitude, int precision)
      * @return long encoded value of the given string hash
      */
     public static long longEncode(String hashAsString) {
-        int[] parsed = parseHash(hashAsString);
-        return longEncode((long) parsed[0], (long) parsed[1], (long) parsed[2]);
+        final int[] parsed = parseHash(hashAsString);
+        return longEncodeTiles(parsed[0], parsed[1], parsed[2]);
     }
 
-    public static long longEncodeTiles(int precision, long xTile, long yTile) {
+    public static long longEncodeTiles(int precision, int xTile, int yTile) {
         // Zoom value is placed in front of all the bits used for the geotile
         // e.g. when max zoom is 29, the largest index would use 58 bits (57th..0th),
         // leaving 5 bits unused for zoom. See MAX_ZOOM comment above.
-        return ((long) precision << ZOOM_SHIFT) | (xTile << MAX_ZOOM) | yTile;
+        return ((long) precision << ZOOM_SHIFT) | ((long) xTile << MAX_ZOOM) | yTile;
     }
 
     /**
      * Parse geotile hash as zoom, x, y integers.
      */
     private static int[] parseHash(long hash) {
-        final int zoom = (int) (hash >>> ZOOM_SHIFT);
-        final int xTile = (int) ((hash >>> MAX_ZOOM) & X_Y_VALUE_MASK);
-        final int yTile = (int) (hash & X_Y_VALUE_MASK);
-        return new int[] { zoom, xTile, yTile };
-    }
-
-    private static long longEncode(long precision, long xTile, long yTile) {
-        // Zoom value is placed in front of all the bits used for the geotile
-        // e.g. when max zoom is 29, the largest index would use 58 bits (57th..0th),
-        // leaving 5 bits unused for zoom. See MAX_ZOOM comment above.
-        return (precision << ZOOM_SHIFT) | (xTile << MAX_ZOOM) | yTile;
+        return new int[] { (int) (hash >>> ZOOM_SHIFT), (int) ((hash >>> MAX_ZOOM) & X_Y_VALUE_MASK), (int) (hash & X_Y_VALUE_MASK) };
     }
 
     /**
      * Parse geotile String hash format in "zoom/x/y" into an array of integers
      */
-    private static int[] parseHash(String hashAsString) {
+    public static int[] parseHash(String hashAsString) {
         final String[] parts = hashAsString.split("/", 4);
         if (parts.length != 3) {
             throw new IllegalArgumentException(
@@ -218,7 +194,7 @@ private static int[] parseHash(String hashAsString) {
      * Encode to a geotile string from the geotile based long format
      */
     public static String stringEncode(long hash) {
-        int[] res = parseHash(hash);
+        final int[] res = parseHash(hash);
         validateZXY(res[0], res[1], res[2]);
         return "" + res[0] + "/" + res[1] + "/" + res[2];
     }
@@ -227,7 +203,7 @@ public static String stringEncode(long hash) {
      * Decode long hash as a GeoPoint (center of the tile)
      */
     static GeoPoint hashToGeoPoint(long hash) {
-        int[] res = parseHash(hash);
+        final int[] res = parseHash(hash);
         return zxyToGeoPoint(res[0], res[1], res[2]);
     }
 
@@ -235,12 +211,12 @@ static GeoPoint hashToGeoPoint(long hash) {
      * Decode a string bucket key in "zoom/x/y" format to a GeoPoint (center of the tile)
      */
     static GeoPoint keyToGeoPoint(String hashAsString) {
-        int[] hashAsInts = parseHash(hashAsString);
+        final int[] hashAsInts = parseHash(hashAsString);
         return zxyToGeoPoint(hashAsInts[0], hashAsInts[1], hashAsInts[2]);
     }
 
     public static Rectangle toBoundingBox(long hash) {
-        int[] hashAsInts = parseHash(hash);
+        final int[] hashAsInts = parseHash(hash);
         return toBoundingBox(hashAsInts[1], hashAsInts[2], hashAsInts[0]);
     }
 
@@ -248,7 +224,7 @@ public static Rectangle toBoundingBox(long hash) {
      * Decode a string bucket key in "zoom/x/y" format to a bounding box of the tile corners
      */
     public static Rectangle toBoundingBox(String hash) {
-        int[] hashAsInts = parseHash(hash);
+        final int[] hashAsInts = parseHash(hash);
         return toBoundingBox(hashAsInts[1], hashAsInts[2], hashAsInts[0]);
     }
 
@@ -257,25 +233,34 @@ public static Rectangle toBoundingBox(String hash) {
      */
     public static Rectangle toBoundingBox(int xTile, int yTile, int precision) {
         final double tiles = validateZXY(precision, xTile, yTile);
-        final double minY = tileToLat(yTile + 1, tiles);
-        final double minX = tileToLon(xTile, tiles);
-        final double maxY = tileToLat(yTile, tiles);
-        final double maxX = tileToLon(xTile + 1, tiles);
-        return new Rectangle(minX, maxX, maxY, minY);
+        return new Rectangle(
+            tileToLon(xTile, tiles),            // minLon
+            tileToLon(xTile + 1, tiles),  // maxLon
+            tileToLat(yTile, tiles),            // maxLat
+            tileToLat(yTile + 1, tiles)   // minLat
+        );
     }
 
     /**
      * Decode a xTile into its longitude value
      */
-    public static double tileToLon(int xTile, double tiles) {
-        return (xTile / tiles * 360.0) - 180;
+    public static double tileToLon(int xTile, int tiles) {
+        return tileToLon(xTile, (double) tiles);
+    }
+
+    private static double tileToLon(double xTile, double tiles) {
+        return (xTile / tiles * 360.0) - 180.0;
     }
 
     /**
      * Decode a yTile into its latitude value
      */
-    public static double tileToLat(int yTile, double tiles) {
-        final double n = Math.PI - (2.0 * Math.PI * yTile) / tiles;
+    public static double tileToLat(int yTile, int tiles) {
+        return tileToLat(yTile, (double) tiles);
+    }
+
+    private static double tileToLat(double yTile, double tiles) {
+        final double n = Math.PI - (PI_TIMES_2 * yTile) / tiles;
         return Math.toDegrees(ESSloppyMath.atan(ESSloppyMath.sinh(n)));
     }
 
@@ -296,10 +281,7 @@ private static int validateZXY(int zoom, int xTile, int yTile) {
      * Converts zoom/x/y integers into a GeoPoint.
      */
     private static GeoPoint zxyToGeoPoint(int zoom, int xTile, int yTile) {
-        final int tiles = validateZXY(zoom, xTile, yTile);
-        final double n = Math.PI - (2.0 * Math.PI * (yTile + 0.5)) / tiles;
-        final double lat = Math.toDegrees(ESSloppyMath.atan(ESSloppyMath.sinh(n)));
-        final double lon = ((xTile + 0.5) / tiles * 360.0) - 180;
-        return new GeoPoint(lat, lon);
+        final double tiles = validateZXY(zoom, xTile, yTile);
+        return new GeoPoint(tileToLat(yTile + 0.5, tiles), tileToLon(xTile + 0.5, tiles));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java
index 42775dfbce6a..3d34824644df 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.global;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
@@ -79,7 +79,7 @@ public String getType() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
index bd1dca9f5e89..2c1537e17f67 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.histogram;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Rounding;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -156,7 +156,7 @@ public DateHistogramAggregationBuilder(StreamInput in) throws IOException {
         dateHistogramInterval = new DateIntervalWrapper(in);
         offset = in.readLong();
         extendedBounds = in.readOptionalWriteable(LongBounds::new);
-        if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             hardBounds = in.readOptionalWriteable(LongBounds::new);
         }
     }
@@ -179,7 +179,7 @@ protected void innerWriteTo(StreamOutput out) throws IOException {
         dateHistogramInterval.writeTo(out);
         out.writeLong(offset);
         out.writeOptionalWriteable(extendedBounds);
-        if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             out.writeOptionalWriteable(hardBounds);
         }
     }
@@ -518,8 +518,8 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java
index 8dcdab4059dc..6753e8151a85 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.histogram;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -145,7 +146,7 @@ public HistogramAggregationBuilder(StreamInput in) throws IOException {
         minDocCount = in.readVLong();
         interval = in.readDouble();
         offset = in.readDouble();
-        if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             extendedBounds = in.readOptionalWriteable(DoubleBounds::new);
             hardBounds = in.readOptionalWriteable(DoubleBounds::new);
         } else {
@@ -441,8 +442,8 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregationBuilder.java
index 82d41b7a02f8..4b7962bda6e7 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.histogram;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
@@ -241,7 +241,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_9_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_9_0;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java
index ce4e15caf1e3..c198ed4d038f 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.missing;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -81,7 +81,7 @@ protected void innerWriteTo(StreamOutput out) {
     }
 
     @Override
-    protected boolean serializeTargetValueType(Version version) {
+    protected boolean serializeTargetValueType(TransportVersion version) {
         return true;
     }
 
@@ -119,7 +119,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java
index ea430c142cef..11cc6a43cb0d 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.nested;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -155,7 +155,7 @@ public String getType() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java
index df65aa27bdb3..ce8616b8fc53 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.nested;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -176,7 +176,7 @@ public String getType() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java
index 93127021d08e..304d3426c4f3 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/prefix/IpPrefixAggregationBuilder.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.search.aggregations.bucket.prefix;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -314,7 +314,7 @@ public int hashCode() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_1_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_1_0;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java
index 4407e71a351f..fa53153d50d6 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.range;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
@@ -129,8 +129,8 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java
index 4472c3f0628c..61ee79c633aa 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.range;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.geo.GeoDistance;
 import org.elasticsearch.common.geo.GeoPoint;
@@ -502,8 +502,8 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
index 48de2424198f..88777f8a2928 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.search.aggregations.bucket.range;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.DocValueFormat;
@@ -72,8 +72,8 @@ private static String generateKey(BytesRef from, BytesRef to, DocValueFormat for
         private static Bucket createFromStream(StreamInput in, DocValueFormat format, boolean keyed) throws IOException {
             // NOTE: the key is required in version == 8.0.0 and version <= 7.17.0,
             // while it is optional for all subsequent versions.
-            String key = in.getVersion().equals(Version.V_8_0_0) ? in.readString()
-                : in.getVersion().onOrAfter(Version.V_7_17_1) ? in.readOptionalString()
+            String key = in.getTransportVersion().equals(TransportVersion.V_8_0_0) ? in.readString()
+                : in.getTransportVersion().onOrAfter(TransportVersion.V_7_17_1) ? in.readOptionalString()
                 : in.readString();
             BytesRef from = in.readBoolean() ? in.readBytesRef() : null;
             BytesRef to = in.readBoolean() ? in.readBytesRef() : null;
@@ -85,9 +85,9 @@ private static Bucket createFromStream(StreamInput in, DocValueFormat format, bo
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            if (out.getVersion().equals(Version.V_8_0_0)) {
+            if (out.getTransportVersion().equals(TransportVersion.V_8_0_0)) {
                 out.writeString(key == null ? generateKey(from, to, format) : key);
-            } else if (out.getVersion().onOrAfter(Version.V_7_17_1)) {
+            } else if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_17_1)) {
                 out.writeOptionalString(key);
             } else {
                 out.writeString(key == null ? generateKey(from, to, format) : key);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
index 8f539348d5a6..f821aa344030 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.search.aggregations.bucket.range;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.DocValueFormat;
@@ -160,19 +160,19 @@ private static String generateKey(double from, double to, DocValueFormat format)
         public void writeTo(StreamOutput out) throws IOException {
             // NOTE: the key is required in version == 8.0.0 and version <= 7.17.0,
             // while it is optional for all subsequent versions.
-            if (out.getVersion().equals(Version.V_8_0_0)) {
+            if (out.getTransportVersion().equals(TransportVersion.V_8_0_0)) {
                 out.writeString(key == null ? generateKey(from, to, format) : key);
-            } else if (out.getVersion().onOrAfter(Version.V_7_17_1)) {
+            } else if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_17_1)) {
                 out.writeOptionalString(key);
             } else {
                 out.writeString(key == null ? generateKey(from, to, format) : key);
             }
             out.writeDouble(from);
-            if (out.getVersion().onOrAfter(Version.V_7_17_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_17_0)) {
                 out.writeOptionalDouble(from);
             }
             out.writeDouble(to);
-            if (out.getVersion().onOrAfter(Version.V_7_17_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_17_0)) {
                 out.writeOptionalDouble(to);
             }
             out.writeVLong(docCount);
@@ -270,11 +270,11 @@ public InternalRange(StreamInput in) throws IOException {
         for (int i = 0; i < size; i++) {
             // NOTE: the key is required in version == 8.0.0 and version <= 7.17.0,
             // while it is optional for all subsequent versions.
-            final String key = in.getVersion().equals(Version.V_8_0_0) ? in.readString()
-                : in.getVersion().onOrAfter(Version.V_7_17_1) ? in.readOptionalString()
+            final String key = in.getTransportVersion().equals(TransportVersion.V_8_0_0) ? in.readString()
+                : in.getTransportVersion().onOrAfter(TransportVersion.V_7_17_1) ? in.readOptionalString()
                 : in.readString();
             double from = in.readDouble();
-            if (in.getVersion().onOrAfter(Version.V_7_17_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_17_0)) {
                 final Double originalFrom = in.readOptionalDouble();
                 if (originalFrom != null) {
                     from = originalFrom;
@@ -283,7 +283,7 @@ public InternalRange(StreamInput in) throws IOException {
                 }
             }
             double to = in.readDouble();
-            if (in.getVersion().onOrAfter(Version.V_7_17_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_17_0)) {
                 final Double originalTo = in.readOptionalDouble();
                 if (originalTo != null) {
                     to = originalTo;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java
index d680f16b1270..cfe55fc6c4ac 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java
@@ -9,7 +9,7 @@
 
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -426,7 +426,7 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java
index 7d716e4595b6..c94c0176983b 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.range;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -210,7 +210,7 @@ private static String generateKey(double from, double to, DocValueFormat format)
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java
index 377fc15302fc..55616a4de92e 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java
@@ -9,7 +9,7 @@
 
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.ScorerSupplier;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -174,8 +174,8 @@ public Range(StreamInput in) throws IOException {
             toAsStr = in.readOptionalString();
             from = in.readDouble();
             to = in.readDouble();
-            originalFrom = in.getVersion().onOrAfter(Version.V_7_17_0) ? in.readOptionalDouble() : Double.valueOf(from);
-            originalTo = in.getVersion().onOrAfter(Version.V_7_17_0) ? in.readOptionalDouble() : Double.valueOf(to);
+            originalFrom = in.getTransportVersion().onOrAfter(TransportVersion.V_7_17_0) ? in.readOptionalDouble() : Double.valueOf(from);
+            originalTo = in.getTransportVersion().onOrAfter(TransportVersion.V_7_17_0) ? in.readOptionalDouble() : Double.valueOf(to);
         }
 
         @Override
@@ -185,7 +185,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalString(toAsStr);
             out.writeDouble(from);
             out.writeDouble(to);
-            if (out.getVersion().onOrAfter(Version.V_7_17_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_17_0)) {
                 out.writeOptionalDouble(originalFrom);
                 out.writeOptionalDouble(originalTo);
             }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java
index fbac625ee2a0..b05a68f6f4f9 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.sampler;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -211,7 +211,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java
index c2048b3ce631..116a0d6dbb32 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.sampler;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -144,7 +144,7 @@ public String getType() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregationBuilder.java
index a9d7a4873af7..becbc8d809e0 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.sampler.random;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -168,8 +168,8 @@ public String getType() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_2_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_2_0;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/IncludeExclude.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/IncludeExclude.java
index d1f67533bf4a..26468dc8fc54 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/IncludeExclude.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/IncludeExclude.java
@@ -21,7 +21,7 @@
 import org.apache.lucene.util.automaton.Operations;
 import org.apache.lucene.util.hppc.BitMixer;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -387,7 +387,7 @@ public IncludeExclude(StreamInput in) throws IOException {
             include = includeString == null ? null : new RegExp(includeString);
             String excludeString = in.readOptionalString();
             exclude = excludeString == null ? null : new RegExp(excludeString);
-            if (in.getVersion().before(Version.V_7_11_0)) {
+            if (in.getTransportVersion().before(TransportVersion.V_7_11_0)) {
                 incZeroBasedPartition = 0;
                 incNumPartitions = 0;
                 includeValues = null;
@@ -427,7 +427,7 @@ public void writeTo(StreamOutput out) throws IOException {
         if (regexBased) {
             out.writeOptionalString(include == null ? null : include.getOriginalString());
             out.writeOptionalString(exclude == null ? null : exclude.getOriginalString());
-            if (out.getVersion().before(Version.V_7_11_0)) {
+            if (out.getTransportVersion().before(TransportVersion.V_7_11_0)) {
                 return;
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalMappedTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalMappedTerms.java
index 445ee82d982d..fb3b262e5f0b 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalMappedTerms.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalMappedTerms.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.terms;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.DocValueFormat;
@@ -63,7 +63,7 @@ protected InternalMappedTerms(
      */
     protected InternalMappedTerms(StreamInput in, Bucket.Reader<B> bucketReader) throws IOException {
         super(in);
-        if (in.getVersion().onOrAfter(Version.V_7_15_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_15_0)) {
             if (in.readBoolean()) {
                 docCountError = in.readZLong();
             } else {
@@ -81,7 +81,7 @@ protected InternalMappedTerms(StreamInput in, Bucket.Reader<B> bucketReader) thr
 
     @Override
     protected final void writeTermTypeInfoTo(StreamOutput out) throws IOException {
-        if (out.getVersion().onOrAfter(Version.V_7_15_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_15_0)) {
             if (docCountError != null) {
                 out.writeBoolean(true);
                 out.writeZLong(docCountError);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
index c29a59d1a40d..318f8c16893b 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.search.aggregations.bucket.terms;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.DocValueFormat;
@@ -197,7 +197,7 @@ protected InternalTerms(
     protected InternalTerms(StreamInput in) throws IOException {
         super(in);
         reduceOrder = InternalOrder.Streams.readOrder(in);
-        if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             order = InternalOrder.Streams.readOrder(in);
         } else {
             order = reduceOrder;
@@ -208,7 +208,7 @@ protected InternalTerms(StreamInput in) throws IOException {
 
     @Override
     protected final void doWriteTo(StreamOutput out) throws IOException {
-        if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             reduceOrder.writeTo(out);
         }
         order.writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregationBuilder.java
index ae76236d71d7..66e91ef01e0a 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregationBuilder.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.search.aggregations.bucket.terms;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -110,7 +110,7 @@ public RareTermsAggregationBuilder(StreamInput in) throws IOException {
     }
 
     @Override
-    protected boolean serializeTargetValueType(Version version) {
+    protected boolean serializeTargetValueType(TransportVersion version) {
         return true;
     }
 
@@ -249,7 +249,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_3_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_3_0;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregationBuilder.java
index b8bdd65aec3c..493730df35f4 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregationBuilder.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.search.aggregations.bucket.terms;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -172,7 +172,7 @@ protected void innerWriteTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    protected boolean serializeTargetValueType(Version version) {
+    protected boolean serializeTargetValueType(TransportVersion version) {
         return true;
     }
 
@@ -383,7 +383,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_3_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_3_0;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregationBuilder.java
index 99725bb2cda0..f0a697090f29 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.terms;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
@@ -399,7 +399,7 @@ public String getType() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_3_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_3_0;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java
index 1d22bf6f1e92..e087c54b7b3d 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java
@@ -15,9 +15,11 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.BytesRefHash;
 import org.elasticsearch.common.util.ObjectArray;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.TextSearchInfo;
@@ -43,7 +45,9 @@
 import org.elasticsearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.SamplingContext;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.search.lookup.SourceFilter;
+import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.search.profile.Timer;
 
 import java.io.IOException;
@@ -224,7 +228,7 @@ private CollectorSource createCollectorSource() {
             .toArray(String[]::new);
         if (context.profiling()) {
             return new ProfilingSignificantTextCollectorSource(
-                context.lookup().source(),
+                context.lookup(),
                 context.bigArrays(),
                 fieldType,
                 analyzer,
@@ -233,7 +237,7 @@ private CollectorSource createCollectorSource() {
             );
         }
         return new SignificantTextCollectorSource(
-            context.lookup().source(),
+            context.lookup(),
             context.bigArrays(),
             fieldType,
             analyzer,
@@ -243,7 +247,8 @@ private CollectorSource createCollectorSource() {
     }
 
     private static class SignificantTextCollectorSource implements MapStringTermsAggregator.CollectorSource {
-        private final SourceLookup sourceLookup;
+        private final SourceProvider sourceProvider;
+        private final SourceFilter sourceFilter;
         private final BigArrays bigArrays;
         private final MappedFieldType fieldType;
         private final Analyzer analyzer;
@@ -252,18 +257,19 @@ private static class SignificantTextCollectorSource implements MapStringTermsAgg
         private ObjectArray<DuplicateByteSequenceSpotter> dupSequenceSpotters;
 
         SignificantTextCollectorSource(
-            SourceLookup sourceLookup,
+            SourceProvider sourceProvider,
             BigArrays bigArrays,
             MappedFieldType fieldType,
             Analyzer analyzer,
             String[] sourceFieldNames,
             boolean filterDuplicateText
         ) {
-            this.sourceLookup = sourceLookup;
+            this.sourceProvider = sourceProvider;
             this.bigArrays = bigArrays;
             this.fieldType = fieldType;
             this.analyzer = analyzer;
             this.sourceFieldNames = sourceFieldNames;
+            this.sourceFilter = new SourceFilter(sourceFieldNames, Strings.EMPTY_ARRAY);
             dupSequenceSpotters = filterDuplicateText ? bigArrays.newObjectArray(1) : null;
         }
 
@@ -306,12 +312,11 @@ public void collect(int doc, long owningBucketOrd) throws IOException {
                 }
 
                 private void collectFromSource(int doc, long owningBucketOrd, DuplicateByteSequenceSpotter spotter) throws IOException {
-                    sourceLookup.setSegmentAndDocument(ctx, doc);
                     BytesRefHash inDocTerms = new BytesRefHash(256, bigArrays);
-
+                    Source source = sourceProvider.getSource(ctx, doc).filter(sourceFilter);
                     try {
                         for (String sourceField : sourceFieldNames) {
-                            Iterator<String> itr = extractRawValues(sourceField).stream().map(obj -> {
+                            Iterator<String> itr = extractRawValues(source, sourceField).stream().map(obj -> {
                                 if (obj == null) {
                                     return null;
                                 }
@@ -400,8 +405,8 @@ protected void processTokenStream(
         /**
          * Extract values from {@code _source}.
          */
-        protected List<Object> extractRawValues(String field) {
-            return sourceLookup.extractRawValuesWithoutCaching(field);
+        protected List<Object> extractRawValues(Source source, String field) {
+            return XContentMapValues.extractRawValues(field, source.source());
         }
 
         @Override
@@ -417,14 +422,14 @@ private static class ProfilingSignificantTextCollectorSource extends Significant
         private long charsFetched;
 
         private ProfilingSignificantTextCollectorSource(
-            SourceLookup sourceLookup,
+            SourceProvider sourceProvider,
             BigArrays bigArrays,
             MappedFieldType fieldType,
             Analyzer analyzer,
             String[] sourceFieldNames,
             boolean filterDuplicateText
         ) {
-            super(sourceLookup, bigArrays, fieldType, analyzer, sourceFieldNames, filterDuplicateText);
+            super(sourceProvider, bigArrays, fieldType, analyzer, sourceFieldNames, filterDuplicateText);
         }
 
         @Override
@@ -464,10 +469,10 @@ protected void processTokenStream(
         }
 
         @Override
-        protected List<Object> extractRawValues(String field) {
+        protected List<Object> extractRawValues(Source source, String field) {
             extract.start();
             try {
-                return super.extractRawValues(field);
+                return super.extractRawValues(source, field);
             } finally {
                 extract.stop();
             }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
index c0574cabae4c..52ea574cdb53 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.search.aggregations.bucket.terms;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
@@ -161,7 +161,7 @@ public TermsAggregationBuilder(StreamInput in) throws IOException {
     }
 
     @Override
-    protected boolean serializeTargetValueType(Version version) {
+    protected boolean serializeTargetValueType(TransportVersion version) {
         return true;
     }
 
@@ -440,7 +440,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
index 904d8b72d70d..5bea0f14db0c 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
@@ -354,7 +354,7 @@ static SubAggCollectionMode pickSubAggColectMode(AggregatorFactories factories,
      */
     private static long getMaxOrd(ValuesSource source, IndexSearcher searcher) throws IOException {
         if (source instanceof ValuesSource.Bytes.WithOrdinals valueSourceWithOrdinals) {
-            return valueSourceWithOrdinals.globalMaxOrd(searcher);
+            return valueSourceWithOrdinals.globalMaxOrd(searcher.getIndexReader());
         } else {
             return -1;
         }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalHDRPercentiles.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalHDRPercentiles.java
index 04e5ebdc65ce..367b259c2d71 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalHDRPercentiles.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalHDRPercentiles.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.search.aggregations.metrics;
 
 import org.HdrHistogram.DoubleHistogram;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.DocValueFormat;
@@ -27,6 +28,8 @@
 
 abstract class AbstractInternalHDRPercentiles extends InternalNumericMetricsAggregation.MultiValue {
 
+    private static final DoubleHistogram EMPTY_HISTOGRAM = new DoubleHistogram(3);
+
     protected final double[] keys;
     protected final DoubleHistogram state;
     protected final boolean keyed;
@@ -51,29 +54,55 @@ abstract class AbstractInternalHDRPercentiles extends InternalNumericMetricsAggr
     protected AbstractInternalHDRPercentiles(StreamInput in) throws IOException {
         super(in);
         keys = in.readDoubleArray();
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            if (in.readBoolean()) {
+                state = decode(in);
+            } else {
+                state = null;
+            }
+        } else {
+            state = decode(in);
+        }
+        keyed = in.readBoolean();
+    }
+
+    private DoubleHistogram decode(StreamInput in) throws IOException {
         long minBarForHighestToLowestValueRatio = in.readLong();
         final int serializedLen = in.readVInt();
         byte[] bytes = new byte[serializedLen];
         in.readBytes(bytes, 0, serializedLen);
         ByteBuffer stateBuffer = ByteBuffer.wrap(bytes);
         try {
-            state = DoubleHistogram.decodeFromCompressedByteBuffer(stateBuffer, minBarForHighestToLowestValueRatio);
+            return DoubleHistogram.decodeFromCompressedByteBuffer(stateBuffer, minBarForHighestToLowestValueRatio);
         } catch (DataFormatException e) {
             throw new IOException("Failed to decode DoubleHistogram for aggregation [" + name + "]", e);
         }
-        keyed = in.readBoolean();
     }
 
     @Override
     protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeNamedWriteable(format);
         out.writeDoubleArray(keys);
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            if (this.state != null) {
+                out.writeBoolean(true);
+                encode(this.state, out);
+            } else {
+                out.writeBoolean(false);
+            }
+        } else {
+            DoubleHistogram state = this.state != null ? this.state : EMPTY_HISTOGRAM;
+            encode(state, out);
+        }
+        out.writeBoolean(keyed);
+    }
+
+    private static void encode(DoubleHistogram state, StreamOutput out) throws IOException {
         out.writeLong(state.getHighestToLowestValueRatio());
         ByteBuffer stateBuffer = ByteBuffer.allocate(state.getNeededByteBufferCapacity());
         final int serializedLen = state.encodeIntoCompressedByteBuffer(stateBuffer);
         out.writeVInt(serializedLen);
         out.writeBytes(stateBuffer.array(), 0, serializedLen);
-        out.writeBoolean(keyed);
     }
 
     @Override
@@ -95,10 +124,6 @@ public DocValueFormat formatter() {
 
     public abstract double value(double key);
 
-    public long getEstimatedMemoryFootprint() {
-        return state.getEstimatedFootprintInBytes();
-    }
-
     /**
      * Return the internal {@link DoubleHistogram} sketch for this metric.
      */
@@ -125,12 +150,18 @@ public AbstractInternalHDRPercentiles reduce(List<InternalAggregation> aggregati
         DoubleHistogram merged = null;
         for (InternalAggregation aggregation : aggregations) {
             final AbstractInternalHDRPercentiles percentiles = (AbstractInternalHDRPercentiles) aggregation;
+            if (percentiles.state == null) {
+                continue;
+            }
             if (merged == null) {
                 merged = new DoubleHistogram(percentiles.state);
                 merged.setAutoResize(true);
             }
             merged.add(percentiles.state);
         }
+        if (merged == null) {
+            merged = EMPTY_HISTOGRAM;
+        }
         return createReduced(getName(), keys, merged, keyed, getMetadata());
     }
 
@@ -149,6 +180,7 @@ protected abstract AbstractInternalHDRPercentiles createReduced(
 
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        DoubleHistogram state = this.state != null ? this.state : EMPTY_HISTOGRAM;
         if (keyed) {
             builder.startObject(CommonFields.VALUES.getPreferredName());
             for (int i = 0; i < keys.length; ++i) {
@@ -196,8 +228,8 @@ public int hashCode() {
             super.hashCode(),
             keyed,
             Arrays.hashCode(keys),
-            state.getIntegerToDoubleValueConversionRatio(),
-            state.getTotalCount()
+            state != null ? state.getIntegerToDoubleValueConversionRatio() : 0,
+            state != null ? state.getTotalCount() : 0
         );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractPercentilesAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractPercentilesAggregationBuilder.java
index 407b7b61d348..55976bb41825 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractPercentilesAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractPercentilesAggregationBuilder.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.TriFunction;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -146,7 +146,7 @@ public static <T extends AbstractPercentilesAggregationBuilder<T>> ConstructingO
         this.valuesField = valuesField;
         values = in.readDoubleArray();
         keyed = in.readBoolean();
-        if (in.getVersion().onOrAfter(Version.V_7_8_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_8_0)) {
             percentilesConfig = (PercentilesConfig) in.readOptionalWriteable((Reader<Writeable>) PercentilesConfig::fromStream);
         } else {
             int numberOfSignificantValueDigits = in.readVInt();
@@ -165,7 +165,7 @@ public boolean supportsSampling() {
     protected void innerWriteTo(StreamOutput out) throws IOException {
         out.writeDoubleArray(values);
         out.writeBoolean(keyed);
-        if (out.getVersion().onOrAfter(Version.V_7_8_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_8_0)) {
             out.writeOptionalWriteable(percentilesConfig);
         } else {
             // Legacy method serialized both SigFigs and compression, even though we only need one. So we need
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregationBuilder.java
index a1bfa0ec197e..13f2f79c1282 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -104,7 +104,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregationBuilder.java
index bf4d8cc591e9..7da9aafa59c8 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -85,7 +85,7 @@ public CardinalityAggregationBuilder(StreamInput in) throws IOException {
         if (in.readBoolean()) {
             precisionThreshold = in.readLong();
         }
-        if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             executionHint = in.readOptionalString();
         }
     }
@@ -102,7 +102,7 @@ protected void innerWriteTo(StreamOutput out) throws IOException {
         if (hasPrecisionThreshold) {
             out.writeLong(precisionThreshold);
         }
-        if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             out.writeOptionalString(executionHint);
         }
     }
@@ -113,7 +113,7 @@ public boolean supportsSampling() {
     }
 
     @Override
-    protected boolean serializeTargetValueType(Version version) {
+    protected boolean serializeTargetValueType(TransportVersion version) {
         return true;
     }
 
@@ -218,7 +218,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorFactory.java
index 4481254aba36..1b1ab107c73c 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorFactory.java
@@ -157,7 +157,7 @@ public static void registerAggregators(ValuesSourceRegistry.Builder builder) {
                 if (valuesSourceConfig.hasValues()) {
                     if (valuesSourceConfig.getValuesSource()instanceof final ValuesSource.Bytes.WithOrdinals source) {
                         if (executionMode.useGlobalOrdinals(context, source, precision)) {
-                            final long maxOrd = source.globalMaxOrd(context.searcher());
+                            final long maxOrd = source.globalMaxOrd(context.searcher().getIndexReader());
                             return new GlobalOrdCardinalityAggregator(
                                 name,
                                 source,
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java
index ffb483286c6e..ee9895cade71 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -149,7 +149,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java
index 099514515d84..4c7b25da338f 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -156,7 +156,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java
index 830c1255ebe1..1a3a6966521c 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -111,7 +111,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregator.java
index 5f6c68dc024b..03c405963c4b 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregator.java
@@ -45,10 +45,7 @@ public InternalAggregation buildAggregation(long owningBucketOrdinal) {
 
     @Override
     public InternalAggregation buildEmptyAggregation() {
-        DoubleHistogram state;
-        state = new DoubleHistogram(numberOfSignificantValueDigits);
-        state.setAutoResize(true);
-        return new InternalHDRPercentileRanks(name, keys, state, keyed, format, metadata());
+        return new InternalHDRPercentileRanks(name, keys, null, keyed, format, metadata());
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesAggregator.java
index fe0eee731e66..78c93281d6b6 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesAggregator.java
@@ -55,9 +55,6 @@ public double metric(String name, long bucketOrd) {
 
     @Override
     public InternalAggregation buildEmptyAggregation() {
-        DoubleHistogram state;
-        state = new DoubleHistogram(numberOfSignificantValueDigits);
-        state.setAutoResize(true);
-        return new InternalHDRPercentiles(name, keys, state, keyed, format, metadata());
+        return new InternalHDRPercentiles(name, keys, null, keyed, format, metadata());
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoCentroid.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoCentroid.java
index be441c215b4f..4987bfed72e6 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoCentroid.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoCentroid.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.search.aggregations.metrics;
 
 import org.apache.lucene.geo.GeoEncodingUtils;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.SpatialPoint;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -60,7 +60,7 @@ public InternalGeoCentroid(StreamInput in) throws IOException {
 
     @Override
     protected GeoPoint centroidFromStream(StreamInput in) throws IOException {
-        if (in.getVersion().onOrAfter(Version.V_7_2_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_2_0)) {
             return new GeoPoint(in.readDouble(), in.readDouble());
         } else {
             final long hash = in.readLong();
@@ -70,7 +70,7 @@ protected GeoPoint centroidFromStream(StreamInput in) throws IOException {
 
     @Override
     protected void centroidToStream(StreamOutput out) throws IOException {
-        if (out.getVersion().onOrAfter(Version.V_7_2_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_2_0)) {
             out.writeDouble(centroid.getY());
             out.writeDouble(centroid.getX());
         } else {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentileRanks.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentileRanks.java
index a307b67c77d9..7a2bf3f5b820 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentileRanks.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentileRanks.java
@@ -73,7 +73,7 @@ protected AbstractInternalHDRPercentiles createReduced(
     }
 
     public static double percentileRank(DoubleHistogram state, double value) {
-        if (state.getTotalCount() == 0) {
+        if (state == null || state.getTotalCount() == 0) {
             return Double.NaN;
         }
         double percentileRank = state.getPercentileAtOrBelowValue(value);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentiles.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentiles.java
index 83ba92991764..d8a71d19506e 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentiles.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalHDRPercentiles.java
@@ -48,7 +48,7 @@ public Iterator<Percentile> iterator() {
 
     @Override
     public double percentile(double percent) {
-        if (state.getTotalCount() == 0) {
+        if (state == null || state.getTotalCount() == 0) {
             return Double.NaN;
         }
         return state.getValueAtPercentile(percent);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetric.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetric.java
index 7c1c5e5dcafa..1cdbc057834f 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetric.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetric.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.CollectionUtils;
@@ -45,7 +45,7 @@ public class InternalScriptedMetric extends InternalAggregation implements Scrip
     public InternalScriptedMetric(StreamInput in) throws IOException {
         super(in);
         reduceScript = in.readOptionalWriteable(Script::new);
-        if (in.getVersion().before(Version.V_7_8_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_7_8_0)) {
             aggregations = singletonList(in.readGenericValue());
         } else {
             aggregations = in.readList(StreamInput::readGenericValue);
@@ -55,7 +55,7 @@ public InternalScriptedMetric(StreamInput in) throws IOException {
     @Override
     protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(reduceScript);
-        if (out.getVersion().before(Version.V_7_8_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_7_8_0)) {
             if (aggregations.size() > 1) {
                 /*
                  * If aggregations has more than one entry we're trying to
@@ -162,4 +162,8 @@ public int hashCode() {
         return Objects.hash(super.hashCode(), reduceScript, aggregations);
     }
 
+    @Override
+    public String toString() {
+        return "InternalScriptedMetric{" + reduceScript + "}";
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregationBuilder.java
index f85e948fc4db..57b97bbb6fe9 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -108,7 +108,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorFactory.java
index 4effad4e687d..3e4cfc5261c4 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorFactory.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.search.aggregations.CardinalityUpperBound;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
@@ -29,7 +30,12 @@ class MaxAggregatorFactory extends ValuesSourceAggregatorFactory {
     static void registerAggregators(ValuesSourceRegistry.Builder builder) {
         builder.register(
             MaxAggregationBuilder.REGISTRY_KEY,
-            List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN),
+            List.of(
+                CoreValuesSourceType.NUMERIC,
+                CoreValuesSourceType.DATE,
+                CoreValuesSourceType.BOOLEAN,
+                TimeSeriesValuesSourceType.COUNTER
+            ),
             MaxAggregator::new,
             true
         );
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregationBuilder.java
index f3365d9258bc..bee4413bbbba 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -165,7 +165,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregationBuilder.java
index b214070289b3..fff112e04f7b 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -110,7 +110,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorFactory.java
index c6e7f5eee089..553319304172 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorFactory.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.search.aggregations.CardinalityUpperBound;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
@@ -29,7 +30,12 @@ class MinAggregatorFactory extends ValuesSourceAggregatorFactory {
     static void registerAggregators(ValuesSourceRegistry.Builder builder) {
         builder.register(
             MinAggregationBuilder.REGISTRY_KEY,
-            List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN),
+            List.of(
+                CoreValuesSourceType.NUMERIC,
+                CoreValuesSourceType.DATE,
+                CoreValuesSourceType.BOOLEAN,
+                TimeSeriesValuesSourceType.COUNTER
+            ),
             MinAggregator::new,
             true
         );
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregationBuilder.java
index 34f35833f6ac..e7b869091976 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
@@ -115,7 +115,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregationBuilder.java
index 61b8d0f56c20..96fd8052c0cc 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
@@ -151,7 +151,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregationBuilder.java
index 51b3f830a9c6..8d92ccc16e09 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.script.Script;
@@ -298,8 +298,8 @@ public String getType() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregationBuilder.java
index f0ce97424353..f13fe41d9b04 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -115,8 +115,8 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregationBuilder.java
index 5f394b9dc06b..201c9e87d63a 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -108,7 +108,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java
index 14c790928bae..3772eec79871 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -129,7 +129,7 @@ public TopHitsAggregationBuilder(StreamInput in) throws IOException {
         trackScores = in.readBoolean();
         version = in.readBoolean();
         seqNoAndPrimaryTerm = in.readBoolean();
-        if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             if (in.readBoolean()) {
                 fetchFields = in.readList(FieldAndFormat::new);
             }
@@ -162,7 +162,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeBoolean(trackScores);
         out.writeBoolean(version);
         out.writeBoolean(seqNoAndPrimaryTerm);
-        if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             out.writeBoolean(fetchFields != null);
             if (fetchFields != null) {
                 out.writeList(fetchFields);
@@ -917,7 +917,7 @@ public String getType() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregationBuilder.java
index 12f48d326097..465fe3880076 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -86,7 +86,7 @@ protected void innerWriteTo(StreamOutput out) {
     }
 
     @Override
-    protected boolean serializeTargetValueType(Version version) {
+    protected boolean serializeTargetValueType(TransportVersion version) {
         return true;
     }
 
@@ -117,7 +117,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregationBuilder.java
index fb426be5b05b..e0ccf799d18c 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -46,8 +46,8 @@ public class WeightedAvgAggregationBuilder extends MultiValuesSourceAggregationB
     );
     static {
         MultiValuesSourceParseHelper.declareCommon(PARSER, true, ValueType.NUMERIC);
-        MultiValuesSourceParseHelper.declareField(VALUE_FIELD.getPreferredName(), PARSER, true, false, false, false);
-        MultiValuesSourceParseHelper.declareField(WEIGHT_FIELD.getPreferredName(), PARSER, true, false, false, false);
+        MultiValuesSourceParseHelper.declareField(VALUE_FIELD.getPreferredName(), PARSER, true, false, false, false, false);
+        MultiValuesSourceParseHelper.declareField(WEIGHT_FIELD.getPreferredName(), PARSER, true, false, false, false, false);
     }
 
     public static void registerUsage(ValuesSourceRegistry.Builder builder) {
@@ -129,7 +129,7 @@ public String getType() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketPipelineAggregationBuilder.java
index 3306e6bb6dbb..ad34437e8ed3 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketPipelineAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -62,7 +62,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java
index 0f73941ef46f..b3aa03cb9b40 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.Maps;
@@ -223,7 +223,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java
index d989e1b60568..639358b023f1 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.DocValueFormat;
@@ -128,7 +128,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_4_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_4_0;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java
index 251643def323..452d28ca8cc6 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -99,7 +99,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketPipelineAggregationBuilder.java
index b3466a97b130..7d1a35d051fd 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketPipelineAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -62,7 +62,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MinBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MinBucketPipelineAggregationBuilder.java
index fe30fecb5a84..566be88cb017 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MinBucketPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MinBucketPipelineAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -62,7 +62,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgPipelineAggregationBuilder.java
index ca715858af8c..4de5c6099523 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgPipelineAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -75,7 +75,7 @@ public final String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java
index 23ecc3ab2cec..be0d0b6a6e2b 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.ParseField;
@@ -118,8 +118,8 @@ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params)
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     public static final PipelineAggregator.Parser PARSER = new BucketMetricsParser() {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java
index bca172e1dd5d..243271fba361 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -248,7 +248,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketPipelineAggregationBuilder.java
index 5cd070b8fa6b..c46aceaf3733 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketPipelineAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -62,7 +62,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SumBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SumBucketPipelineAggregationBuilder.java
index 5f45e1355b8a..3c499b9acf51 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SumBucketPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SumBucketPipelineAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -62,7 +62,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceFieldConfig.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceFieldConfig.java
index ad9e4c406c4a..22d1cbe6864d 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceFieldConfig.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceFieldConfig.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.aggregations.support;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -17,6 +17,7 @@
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -38,6 +39,7 @@ public class MultiValuesSourceFieldConfig implements Writeable, ToXContentObject
     private final QueryBuilder filter;
     // supported only if heterogeneous == true
     private final ValueType userValueTypeHint;
+    private final IncludeExclude includeExclude;
     private final String format;
 
     private static final String NAME = "field_config";
@@ -50,6 +52,7 @@ public class MultiValuesSourceFieldConfig implements Writeable, ToXContentObject
      * @param timezoneAware - allows specifying timezone
      * @param filtered - allows specifying filters on the values
      * @param heterogeneous - allows specifying value-source specific format and user value type hint
+     * @param supportsIncludesExcludes - allows specifying includes and excludes
      * @param <C> - parser context
      * @return configured parser
      */
@@ -57,7 +60,8 @@ public static <C> ObjectParser<MultiValuesSourceFieldConfig.Builder, C> parserBu
         boolean scriptable,
         boolean timezoneAware,
         boolean filtered,
-        boolean heterogeneous
+        boolean heterogeneous,
+        boolean supportsIncludesExcludes
     ) {
 
         ObjectParser<MultiValuesSourceFieldConfig.Builder, C> parser = new ObjectParser<>(
@@ -116,6 +120,23 @@ public static <C> ObjectParser<MultiValuesSourceFieldConfig.Builder, C> parserBu
                 ObjectParser.ValueType.STRING
             );
         }
+
+        if (supportsIncludesExcludes) {
+            parser.declareField(
+                (b, v) -> b.setIncludeExclude(IncludeExclude.merge(v, b.getIncludeExclude())),
+                IncludeExclude::parseInclude,
+                IncludeExclude.INCLUDE_FIELD,
+                ObjectParser.ValueType.OBJECT_ARRAY_OR_STRING
+            );
+
+            parser.declareField(
+                (b, v) -> b.setIncludeExclude(IncludeExclude.merge(b.getIncludeExclude(), v)),
+                IncludeExclude::parseExclude,
+                IncludeExclude.EXCLUDE_FIELD,
+                ObjectParser.ValueType.STRING_ARRAY
+            );
+        }
+
         return parser;
     };
 
@@ -126,7 +147,8 @@ protected MultiValuesSourceFieldConfig(
         ZoneId timeZone,
         QueryBuilder filter,
         ValueType userValueTypeHint,
-        String format
+        String format,
+        IncludeExclude includeExclude
     ) {
         this.fieldName = fieldName;
         this.missing = missing;
@@ -135,10 +157,11 @@ protected MultiValuesSourceFieldConfig(
         this.filter = filter;
         this.userValueTypeHint = userValueTypeHint;
         this.format = format;
+        this.includeExclude = includeExclude;
     }
 
     public MultiValuesSourceFieldConfig(StreamInput in) throws IOException {
-        if (in.getVersion().onOrAfter(Version.V_7_6_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_6_0)) {
             this.fieldName = in.readOptionalString();
         } else {
             this.fieldName = in.readString();
@@ -146,18 +169,23 @@ public MultiValuesSourceFieldConfig(StreamInput in) throws IOException {
         this.missing = in.readGenericValue();
         this.script = in.readOptionalWriteable(Script::new);
         this.timeZone = in.readOptionalZoneId();
-        if (in.getVersion().onOrAfter(Version.V_7_8_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_8_0)) {
             this.filter = in.readOptionalNamedWriteable(QueryBuilder.class);
         } else {
             this.filter = null;
         }
-        if (in.getVersion().onOrAfter(Version.V_7_12_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_12_0)) {
             this.userValueTypeHint = in.readOptionalWriteable(ValueType::readFromStream);
             this.format = in.readOptionalString();
         } else {
             this.userValueTypeHint = null;
             this.format = null;
         }
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            this.includeExclude = in.readOptionalWriteable(IncludeExclude::new);
+        } else {
+            this.includeExclude = null;
+        }
     }
 
     public Object getMissing() {
@@ -188,9 +216,13 @@ public String getFormat() {
         return format;
     }
 
+    public IncludeExclude getIncludeExclude() {
+        return includeExclude;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getVersion().onOrAfter(Version.V_7_6_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_6_0)) {
             out.writeOptionalString(fieldName);
         } else {
             out.writeString(fieldName);
@@ -198,13 +230,16 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeGenericValue(missing);
         out.writeOptionalWriteable(script);
         out.writeOptionalZoneId(timeZone);
-        if (out.getVersion().onOrAfter(Version.V_7_8_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_8_0)) {
             out.writeOptionalNamedWriteable(filter);
         }
-        if (out.getVersion().onOrAfter(Version.V_7_12_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_12_0)) {
             out.writeOptionalWriteable(userValueTypeHint);
             out.writeOptionalString(format);
         }
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            out.writeOptionalWriteable(includeExclude);
+        }
     }
 
     @Override
@@ -232,6 +267,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (format != null) {
             builder.field(AggregationBuilder.CommonFields.FORMAT.getPreferredName(), format);
         }
+        if (includeExclude != null) {
+            includeExclude.toXContent(builder, params);
+        }
+
         builder.endObject();
         return builder;
     }
@@ -247,12 +286,13 @@ public boolean equals(Object o) {
             && Objects.equals(timeZone, that.timeZone)
             && Objects.equals(filter, that.filter)
             && Objects.equals(userValueTypeHint, that.userValueTypeHint)
-            && Objects.equals(format, that.format);
+            && Objects.equals(format, that.format)
+            && Objects.equals(includeExclude, that.includeExclude);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(fieldName, missing, script, timeZone, filter, userValueTypeHint, format);
+        return Objects.hash(fieldName, missing, script, timeZone, filter, userValueTypeHint, format, includeExclude);
     }
 
     @Override
@@ -268,6 +308,7 @@ public static class Builder {
         private QueryBuilder filter = null;
         private ValueType userValueTypeHint = null;
         private String format = null;
+        private IncludeExclude includeExclude = null;
 
         public String getFieldName() {
             return fieldName;
@@ -328,6 +369,15 @@ public String getFormat() {
             return format;
         }
 
+        public Builder setIncludeExclude(IncludeExclude includeExclude) {
+            this.includeExclude = includeExclude;
+            return this;
+        }
+
+        public IncludeExclude getIncludeExclude() {
+            return includeExclude;
+        }
+
         public MultiValuesSourceFieldConfig build() {
             if (Strings.isNullOrEmpty(fieldName) && script == null) {
                 throw new IllegalArgumentException(
@@ -351,7 +401,16 @@ public MultiValuesSourceFieldConfig build() {
                 );
             }
 
-            return new MultiValuesSourceFieldConfig(fieldName, missing, script, timeZone, filter, userValueTypeHint, format);
+            return new MultiValuesSourceFieldConfig(
+                fieldName,
+                missing,
+                script,
+                timeZone,
+                filter,
+                userValueTypeHint,
+                format,
+                includeExclude
+            );
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParseHelper.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParseHelper.java
index 4fe17985fdc0..867380ba9cce 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParseHelper.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParseHelper.java
@@ -65,12 +65,19 @@ public static <VS extends ValuesSource, T> void declareField(
         boolean scriptable,
         boolean timezoneAware,
         boolean filterable,
-        boolean heterogeneous
+        boolean heterogeneous,
+        boolean supportsIncludesExcludes
     ) {
 
         objectParser.declareField(
             (o, fieldConfig) -> o.field(fieldName, fieldConfig.build()),
-            (p, c) -> MultiValuesSourceFieldConfig.parserBuilder(scriptable, timezoneAware, filterable, heterogeneous).parse(p, null),
+            (p, c) -> MultiValuesSourceFieldConfig.parserBuilder(
+                scriptable,
+                timezoneAware,
+                filterable,
+                heterogeneous,
+                supportsIncludesExcludes
+            ).parse(p, null),
             new ParseField(fieldName),
             ObjectParser.ValueType.OBJECT
         );
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
index 21f7d91120cb..ca9fa237f7fc 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
@@ -14,7 +14,6 @@
 import org.apache.lucene.index.OrdinalMap;
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Scorable;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Rounding;
@@ -249,8 +248,7 @@ public boolean hasOrdinals() {
              * Get the maximum global ordinal. Requires {@link #globalOrdinalsValues}
              * so see the note about its performance.
              */
-            public long globalMaxOrd(IndexSearcher indexSearcher) throws IOException {
-                IndexReader indexReader = indexSearcher.getIndexReader();
+            public long globalMaxOrd(IndexReader indexReader) throws IOException {
                 if (indexReader.leaves().isEmpty()) {
                     return 0;
                 } else {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
index 2a63e8cda13e..e953a4c85e77 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.search.aggregations.support;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.script.Script;
@@ -217,7 +217,7 @@ protected ValuesSourceAggregationBuilder(
      */
     protected ValuesSourceAggregationBuilder(StreamInput in) throws IOException {
         super(in);
-        if (serializeTargetValueType(in.getVersion())) {
+        if (serializeTargetValueType(in.getTransportVersion())) {
             ValueType valueType = in.readOptionalWriteable(ValueType::readFromStream);
             assert valueType == null;
         }
@@ -242,7 +242,7 @@ private void read(StreamInput in) throws IOException {
 
     @Override
     protected final void doWriteTo(StreamOutput out) throws IOException {
-        if (serializeTargetValueType(out.getVersion())) {
+        if (serializeTargetValueType(out.getTransportVersion())) {
             // TODO: deprecate this so we don't need to carry around a useless null in the wire format
             out.writeOptionalWriteable(null);
         }
@@ -275,7 +275,7 @@ protected final void doWriteTo(StreamOutput out) throws IOException {
      *
      * @param version For backwards compatibility, subclasses can change behavior based on the version
      */
-    protected boolean serializeTargetValueType(Version version) {
+    protected boolean serializeTargetValueType(TransportVersion version) {
         return false;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
index 64821a33bed7..ce2aa412bce3 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
@@ -408,7 +408,11 @@ public Function<Rounding, Rounding.Prepared> roundingPreparer(AggregationContext
     }
 
     /**
-     * Check if this values source supports using global and segment ordinals.
+     * Check if this values source supports segment ordinals. Global ordinals might or might not be supported.
+     * <p>
+     * If this returns {@code true} then it is safe to cast it to {@link ValuesSource.Bytes.WithOrdinals}.
+     * Call {@link ValuesSource.Bytes.WithOrdinals#supportsGlobalOrdinalsMapping} to find out if global ordinals are supported.
+     *
      */
     public boolean hasOrdinals() {
         return valuesSource.hasOrdinals();
diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index fb71a7e6b5b9..32c702cd1a77 100644
--- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.search.builder;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -50,6 +50,7 @@
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParseException;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
 
@@ -130,7 +131,7 @@ public static HighlightBuilder highlight() {
 
     private QueryBuilder postQueryBuilder;
 
-    private KnnSearchBuilder knnSearch;
+    private List<KnnSearchBuilder> knnSearch = new ArrayList<>();
 
     private int from = -1;
 
@@ -239,17 +240,22 @@ public SearchSourceBuilder(StreamInput in) throws IOException {
         sliceBuilder = in.readOptionalWriteable(SliceBuilder::new);
         collapse = in.readOptionalWriteable(CollapseBuilder::new);
         trackTotalHitsUpTo = in.readOptionalInt();
-        if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             if (in.readBoolean()) {
                 fetchFields = in.readList(FieldAndFormat::new);
             }
             pointInTimeBuilder = in.readOptionalWriteable(PointInTimeBuilder::new);
         }
-        if (in.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             runtimeMappings = in.readMap();
         }
-        if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
-            knnSearch = in.readOptionalWriteable(KnnSearchBuilder::new);
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
+            if (in.getTransportVersion().before(TransportVersion.V_8_7_0)) {
+                KnnSearchBuilder searchBuilder = in.readOptionalWriteable(KnnSearchBuilder::new);
+                knnSearch = searchBuilder != null ? List.of(searchBuilder) : List.of();
+            } else {
+                knnSearch = in.readList(KnnSearchBuilder::new);
+            }
         }
     }
 
@@ -302,24 +308,35 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(sliceBuilder);
         out.writeOptionalWriteable(collapse);
         out.writeOptionalInt(trackTotalHitsUpTo);
-        if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             out.writeBoolean(fetchFields != null);
             if (fetchFields != null) {
                 out.writeList(fetchFields);
             }
             out.writeOptionalWriteable(pointInTimeBuilder);
         }
-        if (out.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             out.writeGenericMap(runtimeMappings);
         } else {
             if (false == runtimeMappings.isEmpty()) {
                 throw new IllegalArgumentException(
-                    "Versions before 7.11.0 don't support [runtime_mappings] and search was sent to [" + out.getVersion() + "]"
+                    "Versions before 7110099 don't support [runtime_mappings] and search was sent to [" + out.getTransportVersion() + "]"
                 );
             }
         }
-        if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
-            out.writeOptionalWriteable(knnSearch);
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
+            if (out.getTransportVersion().before(TransportVersion.V_8_7_0)) {
+                if (knnSearch.size() > 1) {
+                    throw new IllegalArgumentException(
+                        "Versions before 8070099 don't support multiple [knn] search clauses and search was sent to ["
+                            + out.getTransportVersion()
+                            + "]"
+                    );
+                }
+                out.writeOptionalWriteable(knnSearch.isEmpty() ? null : knnSearch.get(0));
+            } else {
+                out.writeCollection(knnSearch);
+            }
         }
     }
 
@@ -361,16 +378,16 @@ public QueryBuilder postFilter() {
      * Defines a kNN search. If a query is also provided, the kNN hits
      * are combined with the query hits.
      */
-    public SearchSourceBuilder knnSearch(KnnSearchBuilder knnSearch) {
-        this.knnSearch = knnSearch;
+    public SearchSourceBuilder knnSearch(List<KnnSearchBuilder> knnSearch) {
+        this.knnSearch = Objects.requireNonNull(knnSearch);
         return this;
     }
 
     /**
      * An optional kNN search definition.
      */
-    public KnnSearchBuilder knnSearch() {
-        return knnSearch;
+    public List<KnnSearchBuilder> knnSearch() {
+        return Collections.unmodifiableList(knnSearch);
     }
 
     /**
@@ -986,7 +1003,7 @@ public List<SearchExtBuilder> ext() {
      * @return true if the source only has suggest
      */
     public boolean isSuggestOnly() {
-        return suggestBuilder != null && queryBuilder == null && knnSearch == null && aggregations == null;
+        return suggestBuilder != null && queryBuilder == null && knnSearch.isEmpty() && aggregations == null;
     }
 
     /**
@@ -1039,10 +1056,7 @@ public SearchSourceBuilder rewrite(QueryRewriteContext context) throws IOExcepti
         if (this.postQueryBuilder != null) {
             postQueryBuilder = this.postQueryBuilder.rewrite(context);
         }
-        KnnSearchBuilder knnSearch = null;
-        if (this.knnSearch != null) {
-            knnSearch = this.knnSearch.rewrite(context);
-        }
+        List<KnnSearchBuilder> knnSearch = Rewriteable.rewrite(this.knnSearch, context);
         AggregatorFactories.Builder aggregations = null;
         if (this.aggregations != null) {
             aggregations = this.aggregations.rewrite(context);
@@ -1092,7 +1106,7 @@ public SearchSourceBuilder shallowCopy() {
     private SearchSourceBuilder shallowCopy(
         QueryBuilder queryBuilder,
         QueryBuilder postQueryBuilder,
-        KnnSearchBuilder knnSearch,
+        List<KnnSearchBuilder> knnSearch,
         AggregatorFactories.Builder aggregations,
         SliceBuilder slice,
         List<SortBuilder<?>> sorts,
@@ -1243,7 +1257,7 @@ private SearchSourceBuilder parseXContent(XContentParser parser, boolean checkTr
                     postQueryBuilder = parseTopLevelQuery(parser, searchUsage::trackQueryUsage);
                     searchUsage.trackSectionUsage(POST_FILTER_FIELD.getPreferredName());
                 } else if (KNN_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                    knnSearch = KnnSearchBuilder.fromXContent(parser);
+                    knnSearch = List.of(KnnSearchBuilder.fromXContent(parser));
                     searchUsage.trackSectionUsage(KNN_FIELD.getPreferredName());
                 } else if (_SOURCE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     fetchSourceContext = FetchSourceContext.fromXContent(parser);
@@ -1420,6 +1434,19 @@ private SearchSourceBuilder parseXContent(XContentParser parser, boolean checkTr
                 } else if (SEARCH_AFTER.match(currentFieldName, parser.getDeprecationHandler())) {
                     searchAfterBuilder = SearchAfterBuilder.fromXContent(parser);
                     searchUsage.trackSectionUsage(SEARCH_AFTER.getPreferredName());
+                } else if (KNN_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                    knnSearch = new ArrayList<>();
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        if (token == XContentParser.Token.START_OBJECT) {
+                            knnSearch.add(KnnSearchBuilder.fromXContent(parser));
+                        } else {
+                            throw new XContentParseException(
+                                parser.getTokenLocation(),
+                                "malformed knn format, within the knn search array only objects are allowed; found " + token
+                            );
+                        }
+                    }
+                    searchUsage.trackSectionUsage(KNN_FIELD.getPreferredName());
                 } else {
                     throw new ParsingException(
                         parser.getTokenLocation(),
@@ -1469,10 +1496,14 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
             builder.field(POST_FILTER_FIELD.getPreferredName(), postQueryBuilder);
         }
 
-        if (knnSearch != null) {
-            builder.startObject(KNN_FIELD.getPreferredName());
-            knnSearch.toXContent(builder, params);
-            builder.endObject();
+        if (knnSearch.isEmpty() == false) {
+            builder.startArray(KNN_FIELD.getPreferredName());
+            for (KnnSearchBuilder knnSearchBuilder : knnSearch) {
+                builder.startObject();
+                knnSearchBuilder.toXContent(builder, params);
+                builder.endObject();
+            }
+            builder.endArray();
         }
 
         if (minScore != null) {
diff --git a/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java b/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java
index f40e00e8f7aa..582d231f7a42 100644
--- a/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java
@@ -16,7 +16,6 @@
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.TermStatistics;
 import org.apache.lucene.search.TopScoreDocCollector;
-import org.apache.lucene.search.Weight;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.internal.SearchContext;
@@ -24,15 +23,18 @@
 import org.elasticsearch.search.profile.dfs.DfsTimingType;
 import org.elasticsearch.search.profile.query.CollectorResult;
 import org.elasticsearch.search.profile.query.InternalProfileCollector;
+import org.elasticsearch.search.profile.query.QueryProfiler;
 import org.elasticsearch.search.rescore.RescoreContext;
 import org.elasticsearch.search.vectors.KnnSearchBuilder;
 import org.elasticsearch.search.vectors.KnnVectorQueryBuilder;
 import org.elasticsearch.tasks.TaskCancelledException;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Consumer;
 
 /**
  * DFS phase of a search request, used to make scoring 100% accurate by collecting additional info from each shard before the query phase.
@@ -61,6 +63,16 @@ private void collectStatistics(SearchContext context) throws IOException {
 
         Map<String, CollectionStatistics> fieldStatistics = new HashMap<>();
         Map<Term, TermStatistics> stats = new HashMap<>();
+        final Consumer<DfsTimingType> maybeStart = dtt -> {
+            if (profiler != null) {
+                profiler.startTimer(dtt);
+            }
+        };
+        final Consumer<DfsTimingType> maybeStop = dtt -> {
+            if (profiler != null) {
+                profiler.stopTimer(dtt);
+            }
+        };
 
         IndexSearcher searcher = new IndexSearcher(context.searcher().getIndexReader()) {
             @Override
@@ -68,11 +80,7 @@ public TermStatistics termStatistics(Term term, int docFreq, long totalTermFreq)
                 if (context.isCancelled()) {
                     throw new TaskCancelledException("cancelled");
                 }
-
-                if (profiler != null) {
-                    profiler.startTimer(DfsTimingType.TERM_STATISTICS);
-                }
-
+                maybeStart.accept(DfsTimingType.TERM_STATISTICS);
                 try {
                     TermStatistics ts = super.termStatistics(term, docFreq, totalTermFreq);
                     if (ts != null) {
@@ -80,9 +88,7 @@ public TermStatistics termStatistics(Term term, int docFreq, long totalTermFreq)
                     }
                     return ts;
                 } finally {
-                    if (profiler != null) {
-                        profiler.stopTimer(DfsTimingType.TERM_STATISTICS);
-                    }
+                    maybeStop.accept(DfsTimingType.TERM_STATISTICS);
                 }
             }
 
@@ -91,11 +97,7 @@ public CollectionStatistics collectionStatistics(String field) throws IOExceptio
                 if (context.isCancelled()) {
                     throw new TaskCancelledException("cancelled");
                 }
-
-                if (profiler != null) {
-                    profiler.startTimer(DfsTimingType.COLLECTION_STATISTICS);
-                }
-
+                maybeStart.accept(DfsTimingType.COLLECTION_STATISTICS);
                 try {
                     CollectionStatistics cs = super.collectionStatistics(field);
                     if (cs != null) {
@@ -103,39 +105,7 @@ public CollectionStatistics collectionStatistics(String field) throws IOExceptio
                     }
                     return cs;
                 } finally {
-                    if (profiler != null) {
-                        profiler.stopTimer(DfsTimingType.COLLECTION_STATISTICS);
-                    }
-                }
-            }
-
-            @Override
-            public Weight createWeight(Query query, ScoreMode scoreMode, float boost) throws IOException {
-                if (profiler != null) {
-                    profiler.startTimer(DfsTimingType.CREATE_WEIGHT);
-                }
-
-                try {
-                    return super.createWeight(query, scoreMode, boost);
-                } finally {
-                    if (profiler != null) {
-                        profiler.stopTimer(DfsTimingType.CREATE_WEIGHT);
-                    }
-                }
-            }
-
-            @Override
-            public Query rewrite(Query original) throws IOException {
-                if (profiler != null) {
-                    profiler.startTimer(DfsTimingType.REWRITE);
-                }
-
-                try {
-                    return super.rewrite(original);
-                } finally {
-                    if (profiler != null) {
-                        profiler.stopTimer(DfsTimingType.REWRITE);
-                    }
+                    maybeStop.accept(DfsTimingType.COLLECTION_STATISTICS);
                 }
             }
         };
@@ -145,10 +115,27 @@ public Query rewrite(Query original) throws IOException {
         }
 
         try {
-            searcher.createWeight(context.rewrittenQuery(), ScoreMode.COMPLETE, 1);
+            try {
+                maybeStart.accept(DfsTimingType.CREATE_WEIGHT);
+                searcher.createWeight(context.rewrittenQuery(), ScoreMode.COMPLETE, 1);
+            } finally {
+                maybeStop.accept(DfsTimingType.CREATE_WEIGHT);
+            }
             for (RescoreContext rescoreContext : context.rescore()) {
                 for (Query query : rescoreContext.getQueries()) {
-                    searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE, 1);
+                    final Query rewritten;
+                    try {
+                        maybeStart.accept(DfsTimingType.REWRITE);
+                        rewritten = searcher.rewrite(query);
+                    } finally {
+                        maybeStop.accept(DfsTimingType.REWRITE);
+                    }
+                    try {
+                        maybeStart.accept(DfsTimingType.CREATE_WEIGHT);
+                        searcher.createWeight(rewritten, ScoreMode.COMPLETE, 1);
+                    } finally {
+                        maybeStop.accept(DfsTimingType.CREATE_WEIGHT);
+                    }
                 }
             }
         } finally {
@@ -171,34 +158,42 @@ public Query rewrite(Query original) throws IOException {
 
     private void executeKnnVectorQuery(SearchContext context) throws IOException {
         SearchSourceBuilder source = context.request().source();
-        if (source == null || source.knnSearch() == null) {
+        if (source == null || source.knnSearch().isEmpty()) {
             return;
         }
 
         SearchExecutionContext searchExecutionContext = context.getSearchExecutionContext();
-        KnnSearchBuilder knnSearch = context.request().source().knnSearch();
-        KnnVectorQueryBuilder knnVectorQueryBuilder = knnSearch.toQueryBuilder();
+        List<KnnSearchBuilder> knnSearch = context.request().source().knnSearch();
+        List<KnnVectorQueryBuilder> knnVectorQueryBuilders = knnSearch.stream().map(KnnSearchBuilder::toQueryBuilder).toList();
 
         if (context.request().getAliasFilter().getQueryBuilder() != null) {
-            knnVectorQueryBuilder.addFilterQuery(context.request().getAliasFilter().getQueryBuilder());
+            for (KnnVectorQueryBuilder knnVectorQueryBuilder : knnVectorQueryBuilders) {
+                knnVectorQueryBuilder.addFilterQuery(context.request().getAliasFilter().getQueryBuilder());
+            }
         }
-
-        Query query = searchExecutionContext.toQuery(knnVectorQueryBuilder).query();
-        TopScoreDocCollector topScoreDocCollector = TopScoreDocCollector.create(knnSearch.k(), Integer.MAX_VALUE);
-        Collector collector = topScoreDocCollector;
-
+        List<DfsKnnResults> knnResults = new ArrayList<>(knnVectorQueryBuilders.size());
+        for (int i = 0; i < knnSearch.size(); i++) {
+            Query knnQuery = searchExecutionContext.toQuery(knnVectorQueryBuilders.get(i)).query();
+            TopScoreDocCollector topScoreDocCollector = TopScoreDocCollector.create(knnSearch.get(i).k(), Integer.MAX_VALUE);
+            Collector collector = topScoreDocCollector;
+            if (context.getProfilers() != null) {
+                InternalProfileCollector ipc = new InternalProfileCollector(
+                    topScoreDocCollector,
+                    CollectorResult.REASON_SEARCH_TOP_HITS,
+                    List.of()
+                );
+                QueryProfiler knnProfiler = context.getProfilers().getDfsProfiler().addQueryProfiler(ipc);
+                collector = ipc;
+                // Set the current searcher profiler to gather query profiling information for gathering top K docs
+                context.searcher().setProfiler(knnProfiler);
+            }
+            context.searcher().search(knnQuery, collector);
+            knnResults.add(new DfsKnnResults(topScoreDocCollector.topDocs().scoreDocs));
+        }
+        // Set profiler back after running KNN searches
         if (context.getProfilers() != null) {
-            InternalProfileCollector ipc = new InternalProfileCollector(
-                topScoreDocCollector,
-                CollectorResult.REASON_SEARCH_TOP_HITS,
-                List.of()
-            );
-            context.getProfilers().getDfsProfiler().setCollector(ipc);
-            collector = ipc;
+            context.searcher().setProfiler(context.getProfilers().getCurrentQueryProfiler());
         }
-
-        context.searcher().search(query, collector);
-        DfsKnnResults knnResults = new DfsKnnResults(topScoreDocCollector.topDocs().scoreDocs);
         context.dfsResult().knnResults(knnResults);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java b/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java
index 9d4b87e1a205..9c12bf3a4e49 100644
--- a/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/dfs/DfsSearchResult.java
@@ -12,7 +12,7 @@
 import org.apache.lucene.search.CollectionStatistics;
 import org.apache.lucene.search.TermStatistics;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.SearchPhaseResult;
@@ -23,6 +23,7 @@
 
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 public class DfsSearchResult extends SearchPhaseResult {
@@ -32,7 +33,7 @@ public class DfsSearchResult extends SearchPhaseResult {
     private Term[] terms;
     private TermStatistics[] termStatistics;
     private Map<String, CollectionStatistics> fieldStatistics = new HashMap<>();
-    private DfsKnnResults knnResults;
+    private List<DfsKnnResults> knnResults;
     private int maxDoc;
     private SearchProfileDfsPhaseResult searchProfileDfsPhaseResult;
 
@@ -52,13 +53,18 @@ public DfsSearchResult(StreamInput in) throws IOException {
         fieldStatistics = readFieldStats(in);
 
         maxDoc = in.readVInt();
-        if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             setShardSearchRequest(in.readOptionalWriteable(ShardSearchRequest::new));
         }
-        if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
-            knnResults = in.readOptionalWriteable(DfsKnnResults::new);
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+                knnResults = in.readOptionalList(DfsKnnResults::new);
+            } else {
+                DfsKnnResults results = in.readOptionalWriteable(DfsKnnResults::new);
+                knnResults = results != null ? List.of(results) : List.of();
+            }
         }
-        if (in.getVersion().onOrAfter(Version.V_8_6_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_6_0)) {
             searchProfileDfsPhaseResult = in.readOptionalWriteable(SearchProfileDfsPhaseResult::new);
         }
     }
@@ -89,7 +95,7 @@ public DfsSearchResult fieldStatistics(Map<String, CollectionStatistics> fieldSt
         return this;
     }
 
-    public DfsSearchResult knnResults(DfsKnnResults knnResults) {
+    public DfsSearchResult knnResults(List<DfsKnnResults> knnResults) {
         this.knnResults = knnResults;
         return this;
     }
@@ -111,7 +117,7 @@ public Map<String, CollectionStatistics> fieldStatistics() {
         return fieldStatistics;
     }
 
-    public DfsKnnResults knnResults() {
+    public List<DfsKnnResults> knnResults() {
         return knnResults;
     }
 
@@ -129,13 +135,24 @@ public void writeTo(StreamOutput out) throws IOException {
         writeTermStats(out, termStatistics);
         writeFieldStats(out, fieldStatistics);
         out.writeVInt(maxDoc);
-        if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             out.writeOptionalWriteable(getShardSearchRequest());
         }
-        if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
-            out.writeOptionalWriteable(knnResults);
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+                out.writeOptionalCollection(knnResults);
+            } else {
+                if (knnResults != null && knnResults.size() > 1) {
+                    throw new IllegalArgumentException(
+                        "Cannot serialize multiple KNN results to nodes using previous transport version ["
+                            + out.getTransportVersion()
+                            + "], minimum required transport version is [8070099]"
+                    );
+                }
+                out.writeOptionalWriteable(knnResults == null || knnResults.isEmpty() ? null : knnResults.get(0));
+            }
         }
-        if (out.getVersion().onOrAfter(Version.V_8_6_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_6_0)) {
             out.writeOptionalWriteable(searchProfileDfsPhaseResult);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
index 7a9d93f02b17..4304539dac2b 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
@@ -26,7 +26,7 @@
 import org.elasticsearch.search.fetch.subphase.InnerHitsPhase;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.lookup.Source;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.search.profile.ProfileResult;
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.tasks.TaskCancelledException;
@@ -84,6 +84,16 @@ public void execute(SearchContext context) {
         }
     }
 
+    private static class PreloadedSourceProvider implements SourceProvider {
+
+        Source source;
+
+        @Override
+        public Source getSource(LeafReaderContext ctx, int doc) throws IOException {
+            return source;
+        }
+    }
+
     private SearchHits buildSearchHits(SearchContext context, Profiler profiler) {
 
         FetchContext fetchContext = new FetchContext(context);
@@ -102,6 +112,9 @@ private SearchHits buildSearchHits(SearchContext context, Profiler profiler) {
 
         NestedDocuments nestedDocuments = context.getSearchExecutionContext().getNestedDocuments();
 
+        PreloadedSourceProvider sourceProvider = new PreloadedSourceProvider();
+        context.getSearchExecutionContext().setSourceProvider(sourceProvider);
+
         FetchPhaseDocsIterator docsIterator = new FetchPhaseDocsIterator() {
 
             LeafReaderContext ctx;
@@ -137,6 +150,7 @@ protected SearchHit nextDoc(int doc) throws IOException {
                     ctx,
                     leafSourceLoader
                 );
+                sourceProvider.source = hit.source();
                 for (FetchSubPhaseProcessor processor : processors) {
                     processor.process(hit);
                 }
@@ -187,15 +201,7 @@ private static HitContext prepareHitContext(
         SourceLoader.Leaf sourceLoader
     ) throws IOException {
         if (nestedDocuments.advance(docId - subReaderContext.docBase) == null) {
-            return prepareNonNestedHitContext(
-                context,
-                requiresSource,
-                profiler,
-                leafStoredFieldLoader,
-                docId,
-                subReaderContext,
-                sourceLoader
-            );
+            return prepareNonNestedHitContext(requiresSource, profiler, leafStoredFieldLoader, docId, subReaderContext, sourceLoader);
         } else {
             return prepareNestedHitContext(
                 context,
@@ -217,7 +223,6 @@ private static HitContext prepareHitContext(
      *     allows fetch subphases that use the hit context to access the preloaded source.
      */
     private static HitContext prepareNonNestedHitContext(
-        SearchContext context,
         boolean requiresSource,
         Profiler profiler,
         LeafStoredFieldLoader leafStoredFieldLoader,
@@ -240,9 +245,6 @@ private static HitContext prepareNonNestedHitContext(
                 try {
                     profiler.startLoadingSource();
                     source = sourceLoader.source(leafStoredFieldLoader, subDocId);
-                    SourceLookup scriptSourceLookup = context.getSearchExecutionContext().lookup().source();
-                    scriptSourceLookup.setSegmentAndDocument(subReaderContext, subDocId);
-                    scriptSourceLookup.setSourceProvider(new SourceLookup.BytesSourceProvider(source.internalSourceRef()));
                 } finally {
                     profiler.stopLoadingSource();
                 }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java
index 5798dd46354b..f7b51343c47d 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchSearchResult.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.fetch;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.SearchHit;
@@ -40,7 +40,7 @@ public FetchSearchResult(StreamInput in) throws IOException {
         super(in);
         contextId = new ShardSearchContextId(in);
         hits = new SearchHits(in);
-        if (in.getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             profileResult = in.readOptionalWriteable(ProfileResult::new);
         } else {
             profileResult = null;
@@ -51,7 +51,7 @@ public FetchSearchResult(StreamInput in) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         contextId.writeTo(out);
         hits.writeTo(out);
-        if (out.getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             out.writeOptionalWriteable(profileResult);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
index fff55b15ea97..1cd6a5d71246 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/ShardFetchSearchRequest.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.search.fetch;
 
 import org.apache.lucene.search.ScoreDoc;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -53,7 +53,7 @@ public ShardFetchSearchRequest(
     public ShardFetchSearchRequest(StreamInput in) throws IOException {
         super(in);
         originalIndices = OriginalIndices.readOriginalIndices(in);
-        if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             shardSearchRequest = in.readOptionalWriteable(ShardSearchRequest::new);
             rescoreDocIds = new RescoreDocIds(in);
             aggregatedDfs = in.readOptionalWriteable(AggregatedDfs::new);
@@ -68,7 +68,7 @@ public ShardFetchSearchRequest(StreamInput in) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         OriginalIndices.writeOriginalIndices(originalIndices, out);
-        if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             out.writeOptionalWriteable(shardSearchRequest);
             rescoreDocIds.writeTo(out);
             out.writeOptionalWriteable(aggregatedDfs);
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java
index 8ae6262357fc..3fb9d8a1930c 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.fetch.subphase;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -133,7 +133,7 @@ public FieldAndFormat(String field, @Nullable String format, @Nullable Boolean i
     public FieldAndFormat(StreamInput in) throws IOException {
         this.field = in.readString();
         format = in.readOptionalString();
-        if (in.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             this.includeUnmapped = in.readOptionalBoolean();
         } else {
             this.includeUnmapped = null;
@@ -144,7 +144,7 @@ public FieldAndFormat(StreamInput in) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(field);
         out.writeOptionalString(format);
-        if (out.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             out.writeOptionalBoolean(this.includeUnmapped);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java
index 3bddde70d697..bbb93bc1d7df 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.search.highlight.SimpleFragmenter;
 import org.apache.lucene.search.highlight.SimpleSpanFragmenter;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -161,7 +161,7 @@ protected AbstractHighlighterBuilder(StreamInput in) throws IOException {
             options(in.readMap());
         }
         requireFieldMatch(in.readOptionalBoolean());
-        if (in.getVersion().onOrAfter(Version.V_7_12_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_12_0)) {
             maxAnalyzedOffset(in.readOptionalInt());
         }
     }
@@ -205,7 +205,7 @@ public final void writeTo(StreamOutput out) throws IOException {
             out.writeGenericMap(options);
         }
         out.writeOptionalBoolean(requireFieldMatch);
-        if (out.getVersion().onOrAfter(Version.V_7_12_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_12_0)) {
             out.writeOptionalInt(maxAnalyzedOffset);
         }
         doWriteTo(out);
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java b/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java
index 965e0b97f456..9efd666a05dc 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java
@@ -9,17 +9,18 @@
 package org.elasticsearch.search.internal;
 
 import org.apache.lucene.codecs.StoredFieldsReader;
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FilterDirectoryReader;
 import org.apache.lucene.index.FilterLeafReader;
 import org.apache.lucene.index.FilterVectorValues;
+import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.PointValues;
 import org.apache.lucene.index.QueryTimeout;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.VectorValues;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.suggest.document.CompletionTerms;
@@ -127,12 +128,51 @@ protected StoredFieldsReader doGetSequentialStoredFieldsReader(StoredFieldsReade
         }
 
         @Override
-        public VectorValues getVectorValues(String field) throws IOException {
-            VectorValues vectorValues = in.getVectorValues(field);
+        public ByteVectorValues getByteVectorValues(String field) throws IOException {
+            ByteVectorValues vectorValues = in.getByteVectorValues(field);
             if (vectorValues == null) {
                 return null;
             }
-            return queryCancellation.isEnabled() ? new ExitableVectorValues(vectorValues, queryCancellation) : vectorValues;
+            return queryCancellation.isEnabled() ? new ExitableByteVectorValues(queryCancellation, vectorValues) : vectorValues;
+        }
+
+        @Override
+        public TopDocs searchNearestVectors(String field, byte[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
+            if (queryCancellation.isEnabled() == false) {
+                return in.searchNearestVectors(field, target, k, acceptDocs, visitedLimit);
+            }
+            // when acceptDocs is null due to no doc deleted, we will instantiate a new one that would
+            // match all docs to allow timeout checking.
+            final Bits updatedAcceptDocs = acceptDocs == null ? new Bits.MatchAllBits(maxDoc()) : acceptDocs;
+            Bits timeoutCheckingAcceptDocs = new Bits() {
+                private static final int MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK = 10;
+                private int calls;
+
+                @Override
+                public boolean get(int index) {
+                    if (calls++ % MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK == 0) {
+                        queryCancellation.checkCancelled();
+                    }
+
+                    return updatedAcceptDocs.get(index);
+                }
+
+                @Override
+                public int length() {
+                    return updatedAcceptDocs.length();
+                }
+            };
+
+            return in.searchNearestVectors(field, target, k, timeoutCheckingAcceptDocs, visitedLimit);
+        }
+
+        @Override
+        public FloatVectorValues getFloatVectorValues(String field) throws IOException {
+            FloatVectorValues vectorValues = in.getFloatVectorValues(field);
+            if (vectorValues == null) {
+                return null;
+            }
+            return queryCancellation.isEnabled() ? new ExitableFloatVectorValues(vectorValues, queryCancellation) : vectorValues;
         }
 
         @Override
@@ -173,7 +213,7 @@ static class ExitableTerms extends FilterLeafReader.FilterTerms {
 
         private final QueryCancellation queryCancellation;
 
-        private ExitableTerms(Terms terms, QueryCancellation queryCancellation) {
+        ExitableTerms(Terms terms, QueryCancellation queryCancellation) {
             super(terms);
             this.queryCancellation = queryCancellation;
         }
@@ -187,6 +227,16 @@ public TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm) throw
         public TermsEnum iterator() throws IOException {
             return new ExitableTermsEnum(in.iterator(), queryCancellation);
         }
+
+        @Override
+        public BytesRef getMin() throws IOException {
+            return in.getMin();
+        }
+
+        @Override
+        public BytesRef getMax() throws IOException {
+            return in.getMax();
+        }
     }
 
     /**
@@ -423,11 +473,62 @@ public void grow(int count) {
         }
     }
 
-    private static class ExitableVectorValues extends FilterVectorValues {
+    private static class ExitableByteVectorValues extends ByteVectorValues {
+        private int calls;
+        private final QueryCancellation queryCancellation;
+        private final ByteVectorValues in;
+
+        private ExitableByteVectorValues(QueryCancellation queryCancellation, ByteVectorValues in) {
+            this.queryCancellation = queryCancellation;
+            this.in = in;
+        }
+
+        @Override
+        public int dimension() {
+            return in.dimension();
+        }
+
+        @Override
+        public int size() {
+            return in.size();
+        }
+
+        @Override
+        public byte[] vectorValue() throws IOException {
+            return in.vectorValue();
+        }
+
+        @Override
+        public int docID() {
+            return in.docID();
+        }
+
+        @Override
+        public int nextDoc() throws IOException {
+            final int nextDoc = in.nextDoc();
+            checkAndThrowWithSampling();
+            return nextDoc;
+        }
+
+        @Override
+        public int advance(int target) throws IOException {
+            final int advance = in.advance(target);
+            checkAndThrowWithSampling();
+            return advance;
+        }
+
+        private void checkAndThrowWithSampling() {
+            if ((calls++ & ExitableIntersectVisitor.MAX_CALLS_BEFORE_QUERY_TIMEOUT_CHECK) == 0) {
+                this.queryCancellation.checkCancelled();
+            }
+        }
+    }
+
+    private static class ExitableFloatVectorValues extends FilterVectorValues {
         private int calls;
         private final QueryCancellation queryCancellation;
 
-        ExitableVectorValues(VectorValues vectorValues, QueryCancellation queryCancellation) {
+        ExitableFloatVectorValues(FloatVectorValues vectorValues, QueryCancellation queryCancellation) {
             super(vectorValues);
             this.queryCancellation = queryCancellation;
             this.queryCancellation.checkCancelled();
@@ -457,10 +558,5 @@ private void checkAndThrowWithSampling() {
         public float[] vectorValue() throws IOException {
             return in.vectorValue();
         }
-
-        @Override
-        public BytesRef binaryValue() throws IOException {
-            return in.binaryValue();
-        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java b/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java
index 5372f0b3abcd..0c5f0aec3798 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java
@@ -10,10 +10,12 @@
 
 import org.apache.lucene.codecs.StoredFieldsReader;
 import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.FilterDirectoryReader;
+import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.ImpactsEnum;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.NumericDocValues;
@@ -23,9 +25,10 @@
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.StoredFields;
+import org.apache.lucene.index.TermVectors;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.VectorValues;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.suggest.document.CompletionTerms;
 import org.apache.lucene.util.Bits;
@@ -90,11 +93,11 @@ public interface FieldUsageNotifier {
         void onKnnVectorsUsed(String field);
     }
 
-    public static final class FieldUsageTrackingLeafReader extends SequentialStoredFieldsLeafReader {
+    static final class FieldUsageTrackingLeafReader extends SequentialStoredFieldsLeafReader {
 
         private final FieldUsageNotifier notifier;
 
-        public FieldUsageTrackingLeafReader(LeafReader in, FieldUsageNotifier notifier) {
+        FieldUsageTrackingLeafReader(LeafReader in, FieldUsageNotifier notifier) {
             super(in);
             this.notifier = notifier;
         }
@@ -108,6 +111,21 @@ public Fields getTermVectors(int docID) throws IOException {
             return f;
         }
 
+        @Override
+        public TermVectors termVectors() throws IOException {
+            TermVectors termVectors = super.termVectors();
+            return new TermVectors() {
+                @Override
+                public Fields get(int doc) throws IOException {
+                    Fields f = termVectors.get(doc);
+                    if (f != null) {
+                        f = new FieldUsageTrackingTermVectorFields(f);
+                    }
+                    return f;
+                }
+            };
+        }
+
         @Override
         public PointValues getPointValues(String field) throws IOException {
             PointValues pointValues = super.getPointValues(field);
@@ -126,6 +144,21 @@ public void document(final int docID, final StoredFieldVisitor visitor) throws I
             }
         }
 
+        @Override
+        public StoredFields storedFields() throws IOException {
+            StoredFields storedFields = super.storedFields();
+            return new StoredFields() {
+                @Override
+                public void document(int docID, StoredFieldVisitor visitor) throws IOException {
+                    if (visitor instanceof FieldNamesProvidingStoredFieldsVisitor) {
+                        storedFields.document(docID, new FieldUsageFieldsVisitor((FieldNamesProvidingStoredFieldsVisitor) visitor));
+                    } else {
+                        storedFields.document(docID, new FieldUsageStoredFieldVisitor(visitor));
+                    }
+                }
+            };
+        }
+
         @Override
         public Terms terms(String field) throws IOException {
             Terms terms = super.terms(field);
@@ -187,14 +220,32 @@ public NumericDocValues getNormValues(String field) throws IOException {
         }
 
         @Override
-        public VectorValues getVectorValues(String field) throws IOException {
-            VectorValues vectorValues = super.getVectorValues(field);
+        public FloatVectorValues getFloatVectorValues(String field) throws IOException {
+            FloatVectorValues vectorValues = super.getFloatVectorValues(field);
+            if (vectorValues != null) {
+                notifier.onKnnVectorsUsed(field);
+            }
+            return vectorValues;
+        }
+
+        @Override
+        public ByteVectorValues getByteVectorValues(String field) throws IOException {
+            ByteVectorValues vectorValues = super.getByteVectorValues(field);
             if (vectorValues != null) {
                 notifier.onKnnVectorsUsed(field);
             }
             return vectorValues;
         }
 
+        @Override
+        public TopDocs searchNearestVectors(String field, byte[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
+            TopDocs topDocs = super.searchNearestVectors(field, target, k, acceptDocs, visitedLimit);
+            if (topDocs != null) {
+                notifier.onKnnVectorsUsed(field);
+            }
+            return topDocs;
+        }
+
         @Override
         public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
             TopDocs topDocs = super.searchNearestVectors(field, target, k, acceptDocs, visitedLimit);
@@ -223,8 +274,8 @@ class FieldUsageTrackingStoredFieldsReader extends StoredFieldsReader {
             }
 
             @Override
-            public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
-                reader.visitDocument(docID, new FieldUsageStoredFieldVisitor(visitor));
+            public void document(int docID, StoredFieldVisitor visitor) throws IOException {
+                reader.document(docID, new FieldUsageStoredFieldVisitor(visitor));
             }
 
             @Override
@@ -248,7 +299,7 @@ public void close() throws IOException {
             }
         }
 
-        private class FieldUsageTrackingTerms extends FilterTerms {
+        class FieldUsageTrackingTerms extends FilterTerms {
 
             private final String field;
 
@@ -286,6 +337,16 @@ public long getSumTotalTermFreq() throws IOException {
             public long getSumDocFreq() throws IOException {
                 return in.getSumDocFreq();
             }
+
+            @Override
+            public BytesRef getMin() throws IOException {
+                return in.getMin();
+            }
+
+            @Override
+            public BytesRef getMax() throws IOException {
+                return in.getMax();
+            }
         }
 
         private class FieldUsageTrackingTermsEnum extends FilterTermsEnum {
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchContextId.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchContextId.java
index 5513968f94a9..29dccd06605f 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchContextId.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchContextId.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.internal;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -34,12 +34,12 @@ public ShardSearchContextId(String sessionId, long id, String searcherId) {
 
     public ShardSearchContextId(StreamInput in) throws IOException {
         this.id = in.readLong();
-        if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_7_0)) {
             this.sessionId = in.readString();
         } else {
             this.sessionId = "";
         }
-        if (in.getVersion().onOrAfter(Version.V_7_12_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_12_0)) {
             this.searcherId = in.readOptionalString();
         } else {
             this.searcherId = null;
@@ -49,10 +49,10 @@ public ShardSearchContextId(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeLong(id);
-        if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_7_0)) {
             out.writeString(sessionId);
         }
-        if (out.getVersion().onOrAfter(Version.V_7_12_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_12_0)) {
             out.writeOptionalString(searcherId);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
index 09ea70bc608a..edfdd0141ad7 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.internal;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.search.SearchRequest;
@@ -88,7 +88,7 @@ public class ShardSearchRequest extends TransportRequest implements IndicesReque
     private final ShardSearchContextId readerId;
     private final TimeValue keepAlive;
 
-    private final Version channelVersion;
+    private final TransportVersion channelVersion;
 
     /**
      * Should this request force {@link SourceLoader.Synthetic synthetic source}?
@@ -236,7 +236,7 @@ public ShardSearchRequest(
         this.readerId = readerId;
         this.keepAlive = keepAlive;
         assert keepAlive == null || readerId != null : "readerId: " + readerId + " keepAlive: " + keepAlive;
-        this.channelVersion = Version.CURRENT;
+        this.channelVersion = TransportVersion.CURRENT;
         this.waitForCheckpoint = waitForCheckpoint;
         this.waitForCheckpointsTimeout = waitForCheckpointsTimeout;
         this.forceSyntheticSource = forceSyntheticSource;
@@ -270,11 +270,11 @@ public ShardSearchRequest(StreamInput in) throws IOException {
         super(in);
         shardId = new ShardId(in);
         searchType = SearchType.fromId(in.readByte());
-        shardRequestIndex = in.getVersion().onOrAfter(Version.V_7_11_0) ? in.readVInt() : -1;
+        shardRequestIndex = in.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0) ? in.readVInt() : -1;
         numberOfShards = in.readVInt();
         scroll = in.readOptionalWriteable(Scroll::new);
         source = in.readOptionalWriteable(SearchSourceBuilder::new);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             // types no longer relevant so ignore
             String[] types = in.readStringArray();
             if (types.length > 0) {
@@ -289,11 +289,11 @@ public ShardSearchRequest(StreamInput in) throws IOException {
         requestCache = in.readOptionalBoolean();
         clusterAlias = in.readOptionalString();
         allowPartialSearchResults = in.readBoolean();
-        if (in.getVersion().before(Version.V_7_11_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_7_11_0)) {
             in.readStringArray();
             in.readOptionalString();
         }
-        if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_7_0)) {
             canReturnNullResponseIfMatchNoDocs = in.readBoolean();
             bottomSortValues = in.readOptionalWriteable(SearchSortValuesAndFormats::new);
             readerId = in.readOptionalWriteable(ShardSearchContextId::new);
@@ -305,15 +305,15 @@ public ShardSearchRequest(StreamInput in) throws IOException {
             keepAlive = null;
         }
         assert keepAlive == null || readerId != null : "readerId: " + readerId + " keepAlive: " + keepAlive;
-        channelVersion = Version.min(Version.readVersion(in), in.getVersion());
-        if (in.getVersion().onOrAfter(Version.V_7_16_0)) {
+        channelVersion = TransportVersion.min(TransportVersion.readVersion(in), in.getTransportVersion());
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             waitForCheckpoint = in.readLong();
             waitForCheckpointsTimeout = in.readTimeValue();
         } else {
             waitForCheckpoint = SequenceNumbers.UNASSIGNED_SEQ_NO;
             waitForCheckpointsTimeout = SearchService.NO_TIMEOUT;
         }
-        if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             forceSyntheticSource = in.readBoolean();
         } else {
             /*
@@ -337,14 +337,14 @@ protected final void innerWriteTo(StreamOutput out, boolean asKey) throws IOExce
         shardId.writeTo(out);
         out.writeByte(searchType.id());
         if (asKey == false) {
-            if (out.getVersion().onOrAfter(Version.V_7_11_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
                 out.writeVInt(shardRequestIndex);
             }
             out.writeVInt(numberOfShards);
         }
         out.writeOptionalWriteable(scroll);
         out.writeOptionalWriteable(source);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             // types not supported so send an empty array to previous versions
             out.writeStringArray(Strings.EMPTY_ARRAY);
         }
@@ -356,32 +356,32 @@ protected final void innerWriteTo(StreamOutput out, boolean asKey) throws IOExce
         out.writeOptionalBoolean(requestCache);
         out.writeOptionalString(clusterAlias);
         out.writeBoolean(allowPartialSearchResults);
-        if (asKey == false && out.getVersion().before(Version.V_7_11_0)) {
+        if (asKey == false && out.getTransportVersion().before(TransportVersion.V_7_11_0)) {
             out.writeStringArray(Strings.EMPTY_ARRAY);
             out.writeOptionalString(null);
         }
-        if (asKey == false && out.getVersion().onOrAfter(Version.V_7_7_0)) {
+        if (asKey == false && out.getTransportVersion().onOrAfter(TransportVersion.V_7_7_0)) {
             out.writeBoolean(canReturnNullResponseIfMatchNoDocs);
             out.writeOptionalWriteable(bottomSortValues);
             out.writeOptionalWriteable(readerId);
             out.writeOptionalTimeValue(keepAlive);
         }
-        Version.writeVersion(channelVersion, out);
-        Version waitForCheckpointsVersion = Version.V_7_16_0;
-        if (out.getVersion().onOrAfter(waitForCheckpointsVersion)) {
+        TransportVersion.writeVersion(channelVersion, out);
+        TransportVersion waitForCheckpointsVersion = TransportVersion.V_7_16_0;
+        if (out.getTransportVersion().onOrAfter(waitForCheckpointsVersion)) {
             out.writeLong(waitForCheckpoint);
             out.writeTimeValue(waitForCheckpointsTimeout);
         } else if (waitForCheckpoint != SequenceNumbers.UNASSIGNED_SEQ_NO) {
             throw new IllegalArgumentException(
                 "Remote node version ["
-                    + out.getVersion()
+                    + out.getTransportVersion()
                     + " incompatible with "
                     + "wait_for_checkpoints. All nodes must be version ["
                     + waitForCheckpointsVersion
                     + "] or greater."
             );
         }
-        if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             out.writeBoolean(forceSyntheticSource);
         } else {
             if (forceSyntheticSource) {
@@ -663,10 +663,10 @@ public final Map<String, Object> getRuntimeMappings() {
 
     /**
      * Returns the minimum version of the channel that the request has been passed. If the request never passes around, then the channel
-     * version is {@link Version#CURRENT}; otherwise, it's the minimum version of the coordinating node and data node (and the proxy node
-     * in case the request is sent to the proxy node of the remote cluster before reaching the data node).
+     * version is {@link TransportVersion#CURRENT}; otherwise, it's the minimum transport version of the coordinating node and data node
+     * (and the proxy node in case the request is sent to the proxy node of the remote cluster before reaching the data node).
      */
-    public Version getChannelVersion() {
+    public TransportVersion getChannelVersion() {
         return channelVersion;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/lookup/LeafSearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/LeafSearchLookup.java
index 320e61e00857..9117f109ed22 100644
--- a/server/src/main/java/org/elasticsearch/search/lookup/LeafSearchLookup.java
+++ b/server/src/main/java/org/elasticsearch/search/lookup/LeafSearchLookup.java
@@ -10,35 +10,44 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 
+import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.Map;
+import java.util.function.Supplier;
 
 /**
  * Per-segment version of {@link SearchLookup}.
  */
 public class LeafSearchLookup {
 
-    private final LeafReaderContext ctx;
+    private int doc;
     private final LeafDocLookup docMap;
-    private final SourceLookup sourceLookup;
     private final LeafStoredFieldsLookup fieldsLookup;
     private final Map<String, Object> asMap;
+    private final Supplier<Source> source;
 
-    public LeafSearchLookup(LeafReaderContext ctx, LeafDocLookup docMap, SourceLookup sourceLookup, LeafStoredFieldsLookup fieldsLookup) {
-        this.ctx = ctx;
+    public LeafSearchLookup(
+        LeafReaderContext ctx,
+        LeafDocLookup docMap,
+        SourceProvider sourceProvider,
+        LeafStoredFieldsLookup fieldsLookup
+    ) {
         this.docMap = docMap;
-        this.sourceLookup = sourceLookup;
         this.fieldsLookup = fieldsLookup;
-        this.asMap = Map.of("doc", docMap, "_doc", docMap, "_source", sourceLookup, "_fields", fieldsLookup);
+        this.source = () -> {
+            try {
+                return sourceProvider.getSource(ctx, doc);
+            } catch (IOException e) {
+                throw new UncheckedIOException("Couldn't load source", e);
+            }
+        };
+        this.asMap = Map.of("doc", docMap, "_doc", docMap, "_source", source, "_fields", fieldsLookup);
     }
 
     public Map<String, Object> asMap() {
         return this.asMap;
     }
 
-    public SourceLookup source() {
-        return this.sourceLookup;
-    }
-
     public LeafStoredFieldsLookup fields() {
         return this.fieldsLookup;
     }
@@ -47,9 +56,13 @@ public LeafDocLookup doc() {
         return this.docMap;
     }
 
+    public Supplier<Source> source() {
+        return this.source;
+    }
+
     public void setDocument(int docId) {
+        this.doc = docId;
         docMap.setDocument(docId);
-        sourceLookup.setSegmentAndDocument(ctx, docId);
         fieldsLookup.setDocument(docId);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java
index 264e38e21e96..e76e54ea0467 100644
--- a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java
+++ b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.Objects;
@@ -20,7 +21,7 @@
 import java.util.function.Function;
 import java.util.function.Supplier;
 
-public class SearchLookup {
+public class SearchLookup implements SourceProvider {
     /**
      * The maximum depth of field dependencies.
      * When a runtime field's doc values depends on another runtime field's doc values,
@@ -39,7 +40,7 @@ public class SearchLookup {
      * {@code b}'s chain will contain {@code ["a", "b"]}.
      */
     private final Set<String> fieldChain;
-    private final SourceLookup sourceLookup;
+    private final SourceProvider sourceProvider;
     private final Function<String, MappedFieldType> fieldTypeLookup;
     private final TriFunction<
         MappedFieldType,
@@ -54,11 +55,11 @@ public class SearchLookup {
     public SearchLookup(
         Function<String, MappedFieldType> fieldTypeLookup,
         TriFunction<MappedFieldType, Supplier<SearchLookup>, MappedFieldType.FielddataOperation, IndexFieldData<?>> fieldDataLookup,
-        SourceLookup.SourceProvider sourceProvider
+        SourceProvider sourceProvider
     ) {
         this.fieldTypeLookup = fieldTypeLookup;
         this.fieldChain = Collections.emptySet();
-        this.sourceLookup = new SourceLookup(sourceProvider);
+        this.sourceProvider = sourceProvider;
         this.fieldDataLookup = fieldDataLookup;
     }
 
@@ -71,7 +72,7 @@ public SearchLookup(
      */
     private SearchLookup(SearchLookup searchLookup, Set<String> fieldChain) {
         this.fieldChain = Collections.unmodifiableSet(fieldChain);
-        this.sourceLookup = searchLookup.sourceLookup;
+        this.sourceProvider = searchLookup.sourceProvider;
         this.fieldTypeLookup = searchLookup.fieldTypeLookup;
         this.fieldDataLookup = searchLookup.fieldDataLookup;
     }
@@ -101,7 +102,7 @@ public LeafSearchLookup getLeafSearchLookup(LeafReaderContext context) {
         return new LeafSearchLookup(
             context,
             new LeafDocLookup(fieldTypeLookup, this::getForField, context),
-            sourceLookup,
+            sourceProvider,
             new LeafStoredFieldsLookup(fieldTypeLookup, (doc, visitor) -> context.reader().document(doc, visitor))
         );
     }
@@ -114,7 +115,8 @@ public IndexFieldData<?> getForField(MappedFieldType fieldType, MappedFieldType.
         return fieldDataLookup.apply(fieldType, () -> forkAndTrackFieldReferences(fieldType.name()), options);
     }
 
-    public SourceLookup source() {
-        return sourceLookup;
+    @Override
+    public Source getSource(LeafReaderContext ctx, int doc) throws IOException {
+        return sourceProvider.getSource(ctx, doc);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java
deleted file mode 100644
index c262e0989fb0..000000000000
--- a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java
+++ /dev/null
@@ -1,426 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.search.lookup;
-
-import org.apache.lucene.codecs.StoredFieldsReader;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.CheckedBiConsumer;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.support.XContentMapValues;
-import org.elasticsearch.core.MemoizedSupplier;
-import org.elasticsearch.core.Tuple;
-import org.elasticsearch.index.fieldvisitor.FieldsVisitor;
-import org.elasticsearch.xcontent.XContentType;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.Supplier;
-
-public class SourceLookup implements Source, Map<String, Object> {
-    private SourceProvider sourceProvider;
-
-    private int docId = -1;
-
-    public SourceLookup(SourceProvider sourceProvider) {
-        this.sourceProvider = sourceProvider;
-    }
-
-    @Override
-    public XContentType sourceContentType() {
-        return sourceProvider.sourceContentType();
-    }
-
-    public int docId() {
-        return docId;
-    }
-
-    @Override
-    public Map<String, Object> source() {
-        return sourceProvider.source();
-    }
-
-    public void setSourceProvider(SourceProvider sourceProvider) {
-        this.sourceProvider = sourceProvider;
-    }
-
-    public void setSegmentAndDocument(LeafReaderContext context, int docId) {
-        sourceProvider.setSegmentAndDocument(context, docId);
-        this.docId = docId;
-    }
-
-    /**
-     * Internal source representation, might be compressed....
-     */
-    @Override
-    public BytesReference internalSourceRef() {
-        return sourceProvider.sourceAsBytes();
-    }
-
-    @Override
-    public Source filter(SourceFilter sourceFilter) {
-        return sourceFilter.filterMap(this);
-    }
-
-    /**
-     * Checks if the source has been deserialized as a {@link Map} of java objects.
-     */
-    public boolean hasSourceAsMap() {
-        return sourceProvider.hasSourceAsMap();
-    }
-
-    /**
-     * Returns the values associated with the path. Those are "low" level values, and it can
-     * handle path expression where an array/list is navigated within.
-     *
-     * This method will:
-     *
-     *  - not cache source if it's not already parsed
-     *  - will only extract the desired values from the compressed source instead of deserializing the whole object
-     *
-     * This is useful when the caller only wants a single value from source and does not care of source is fully parsed and cached
-     * for later use.
-     * @param path The path from which to extract the values from source
-     * @return The list of found values or an empty list if none are found
-     */
-    public List<Object> extractRawValuesWithoutCaching(String path) {
-        return sourceProvider.extractRawValuesWithoutCaching(path);
-    }
-
-    private static Tuple<XContentType, Map<String, Object>> sourceAsMapAndType(BytesReference source) throws ElasticsearchParseException {
-        return XContentHelper.convertToMap(source, false);
-    }
-
-    /**
-     * Get the source as a {@link Map} of java objects.
-     * <p>
-     * Important: This can lose precision on numbers with a decimal point. It
-     * converts numbers like {@code "n": 1234.567} to a {@code double} which
-     * only has 52 bits of precision in the mantissa. This will come up most
-     * frequently when folks write nanosecond precision dates as a decimal
-     * number.
-     */
-    public static Map<String, Object> sourceAsMap(BytesReference source) throws ElasticsearchParseException {
-        return sourceAsMapAndType(source).v2();
-    }
-
-    @Override
-    public Object get(Object key) {
-        return source().get(key);
-    }
-
-    @Override
-    public int size() {
-        return source().size();
-    }
-
-    @Override
-    public boolean isEmpty() {
-        return source().isEmpty();
-    }
-
-    @Override
-    public boolean containsKey(Object key) {
-        return source().containsKey(key);
-    }
-
-    @Override
-    public boolean containsValue(Object value) {
-        return source().containsValue(value);
-    }
-
-    @Override
-    public Set<String> keySet() {
-        return source().keySet();
-    }
-
-    @Override
-    public Collection<Object> values() {
-        return source().values();
-    }
-
-    @Override
-    public Set<Map.Entry<String, Object>> entrySet() {
-        return source().entrySet();
-    }
-
-    @Override
-    public Object put(String key, Object value) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public Object remove(Object key) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    @SuppressWarnings("rawtypes")
-    public void putAll(Map m) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void clear() {
-        throw new UnsupportedOperationException();
-    }
-
-    /**
-     * SourceProvider describes how SourceLookup will access the source.
-     */
-    public interface SourceProvider {
-        Map<String, Object> source();
-
-        XContentType sourceContentType();
-
-        BytesReference sourceAsBytes();
-
-        List<Object> extractRawValuesWithoutCaching(String path);
-
-        boolean hasSourceAsMap();
-
-        void setSegmentAndDocument(LeafReaderContext context, int docId);
-    }
-
-    /**
-     * Null source provider when we expect to be given a real source provider in the future.
-     * Exceptions expected to be primarily seen by scripts attempting to access synthetic source.
-     */
-    public static class NullSourceProvider implements SourceProvider {
-        @Override
-        public Map<String, Object> source() {
-            throw new IllegalArgumentException("no source available");
-        }
-
-        @Override
-        public XContentType sourceContentType() {
-            throw new IllegalArgumentException("no source available");
-        }
-
-        @Override
-        public BytesReference sourceAsBytes() {
-            throw new IllegalArgumentException("no source available");
-        }
-
-        @Override
-        public List<Object> extractRawValuesWithoutCaching(String path) {
-            throw new IllegalArgumentException("no source available");
-        }
-
-        @Override
-        public boolean hasSourceAsMap() {
-            return false;
-        }
-
-        @Override
-        public void setSegmentAndDocument(LeafReaderContext context, int docId) {
-            // nothing to do
-        }
-    }
-
-    /**
-     * Provider for source using a given map and optional content type.
-     */
-    private static class MapSourceProvider implements SourceProvider {
-        private final Map<String, Object> source;
-        private final XContentType sourceContentType;
-
-        private MapSourceProvider(Map<String, Object> source) {
-            this.source = source;
-            this.sourceContentType = XContentType.JSON;
-        }
-
-        @Override
-        public Map<String, Object> source() {
-            return source;
-        }
-
-        @Override
-        public XContentType sourceContentType() {
-            return sourceContentType;
-        }
-
-        @Override
-        public BytesReference sourceAsBytes() {
-            throw new UnsupportedOperationException("source as bytes unavailable - empty or already parsed to map");
-        }
-
-        @Override
-        public List<Object> extractRawValuesWithoutCaching(String path) {
-            return XContentMapValues.extractRawValues(path, source);
-        }
-
-        @Override
-        public boolean hasSourceAsMap() {
-            return true;
-        }
-
-        @Override
-        public void setSegmentAndDocument(LeafReaderContext context, int docId) {
-            //
-        }
-    }
-
-    /**
-     * Provider for source using given source bytes.
-     */
-    public static class BytesSourceProvider implements SourceProvider {
-        private BytesReference sourceAsBytes;
-        private Map<String, Object> source;
-        private XContentType sourceContentType;
-
-        public BytesSourceProvider(BytesReference sourceAsBytes) {
-            this.sourceAsBytes = sourceAsBytes;
-        }
-
-        void parseSource() {
-            Tuple<XContentType, Map<String, Object>> tuple = sourceAsMapAndType(sourceAsBytes);
-            this.source = tuple.v2();
-            this.sourceContentType = tuple.v1();
-        }
-
-        @Override
-        public Map<String, Object> source() {
-            if (source == null) {
-                parseSource();
-            }
-            return source;
-        }
-
-        @Override
-        public XContentType sourceContentType() {
-            if (source == null) {
-                parseSource();
-            }
-            return sourceContentType;
-        }
-
-        @Override
-        public BytesReference sourceAsBytes() {
-            return sourceAsBytes;
-        }
-
-        @Override
-        public List<Object> extractRawValuesWithoutCaching(String path) {
-            if (source != null) {
-                return XContentMapValues.extractRawValues(path, source);
-            }
-            return XContentMapValues.extractRawValues(
-                path,
-                XContentHelper.convertToMap(sourceAsBytes, false, null, Set.of(path), null).v2()
-            );
-        }
-
-        @Override
-        public boolean hasSourceAsMap() {
-            return source != null;
-        }
-
-        @Override
-        public void setSegmentAndDocument(LeafReaderContext context, int docId) {
-            //
-        }
-    }
-
-    /**
-     * Provider for source using a fields visitor.
-     */
-    public static class ReaderSourceProvider implements SourceProvider {
-        private LeafReader reader;
-        private CheckedBiConsumer<Integer, FieldsVisitor, IOException> fieldReader;
-        private int docId = -1;
-
-        private SourceProvider sourceProvider;
-
-        public void setSegmentAndDocument(LeafReaderContext context, int docId) {
-            // if we are called with the same document, don't invalidate source
-            if (this.reader == context.reader() && this.docId == docId) {
-                return;
-            }
-
-            // only reset reader and fieldReader when reader changes
-            if (this.reader != context.reader()) {
-                this.reader = context.reader();
-
-                // All the docs to fetch are adjacent but Lucene stored fields are optimized
-                // for random access and don't optimize for sequential access - except for merging.
-                // So we do a little hack here and pretend we're going to do merges in order to
-                // get better sequential access.
-                if (context.reader()instanceof SequentialStoredFieldsLeafReader lf) {
-                    // Avoid eagerly loading the stored fields reader, since this can be expensive
-                    Supplier<StoredFieldsReader> supplier = new MemoizedSupplier<>(lf::getSequentialStoredFieldsReader);
-                    fieldReader = (d, v) -> supplier.get().visitDocument(d, v);
-                } else {
-                    fieldReader = context.reader()::document;
-                }
-            }
-            this.sourceProvider = null;
-            this.docId = docId;
-        }
-
-        @Override
-        public Map<String, Object> source() {
-            if (sourceProvider == null) {
-                readSourceBytes();
-            }
-            return sourceProvider.source();
-        }
-
-        private void readSourceBytes() {
-            try {
-                FieldsVisitor sourceFieldVisitor = new FieldsVisitor(true);
-                fieldReader.accept(docId, sourceFieldVisitor);
-                BytesReference sourceAsBytes = sourceFieldVisitor.source();
-                if (sourceAsBytes == null) {
-                    sourceProvider = new MapSourceProvider(Collections.emptyMap());
-                } else {
-                    sourceProvider = new BytesSourceProvider(sourceAsBytes);
-                    ((BytesSourceProvider) sourceProvider).parseSource();
-                }
-            } catch (Exception e) {
-                throw new ElasticsearchParseException("failed to parse / load source", e);
-            }
-        }
-
-        @Override
-        public XContentType sourceContentType() {
-            if (sourceProvider == null) {
-                readSourceBytes();
-            }
-            return sourceProvider.sourceContentType();
-        }
-
-        @Override
-        public BytesReference sourceAsBytes() {
-            if (sourceProvider == null) {
-                readSourceBytes();
-            }
-            return sourceProvider.sourceAsBytes();
-        }
-
-        @Override
-        public List<Object> extractRawValuesWithoutCaching(String path) {
-            if (sourceProvider == null) {
-                readSourceBytes();
-            }
-            return sourceProvider.extractRawValuesWithoutCaching(path);
-        }
-
-        @Override
-        public boolean hasSourceAsMap() {
-            return sourceProvider != null && sourceProvider.hasSourceAsMap();
-        }
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java
new file mode 100644
index 000000000000..96de69abec85
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.lookup;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader;
+import org.elasticsearch.index.fieldvisitor.StoredFieldLoader;
+
+import java.io.IOException;
+
+/**
+ * Provides access to the Source of a document
+ */
+public interface SourceProvider {
+
+    /**
+     * Get the Source for the given doc within the given context
+     */
+    Source getSource(LeafReaderContext ctx, int doc) throws IOException;
+
+    /**
+     * A SourceProvider that loads source from stored fields
+     */
+    static SourceProvider fromStoredFields() {
+        StoredFieldLoader storedFieldLoader = StoredFieldLoader.sequentialSource();
+        return new SourceProvider() {
+
+            // TODO we can make this segment thread safe by keeping an array of LeafStoredFieldLoader/doc/Source
+            // records, indexed by the ordinal of the LeafReaderContext
+
+            LeafReaderContext ctx;
+            int doc;
+            LeafStoredFieldLoader leafStoredFieldLoader;
+            Source source;
+
+            @Override
+            public Source getSource(LeafReaderContext ctx, int doc) throws IOException {
+                if (this.ctx == ctx) {
+                    if (this.doc == doc) {
+                        return source;
+                    }
+                } else {
+                    leafStoredFieldLoader = storedFieldLoader.getLoader(ctx, null);
+                }
+                this.ctx = ctx;
+                this.doc = doc;
+                leafStoredFieldLoader.advanceTo(doc);
+                return source = Source.fromBytes(leafStoredFieldLoader.source());
+            }
+        };
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/profile/ProfileResult.java b/server/src/main/java/org/elasticsearch/search/profile/ProfileResult.java
index 0ccc4b27f413..0fb8c13b14c4 100644
--- a/server/src/main/java/org/elasticsearch/search/profile/ProfileResult.java
+++ b/server/src/main/java/org/elasticsearch/search/profile/ProfileResult.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.profile;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -75,7 +75,7 @@ public ProfileResult(StreamInput in) throws IOException {
         this.description = in.readString();
         this.nodeTime = in.readLong();
         breakdown = in.readMap(StreamInput::readString, StreamInput::readLong);
-        if (in.getVersion().onOrAfter(Version.V_7_9_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_9_0)) {
             debug = in.readMap(StreamInput::readString, StreamInput::readGenericValue);
         } else {
             debug = Map.of();
@@ -89,7 +89,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(description);
         out.writeLong(nodeTime);            // not Vlong because can be negative
         out.writeMap(breakdown, StreamOutput::writeString, StreamOutput::writeLong);
-        if (out.getVersion().onOrAfter(Version.V_7_9_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_9_0)) {
             out.writeMap(debug, StreamOutput::writeString, StreamOutput::writeGenericValue);
         }
         out.writeList(children);
diff --git a/server/src/main/java/org/elasticsearch/search/profile/Profilers.java b/server/src/main/java/org/elasticsearch/search/profile/Profilers.java
index 4f8f3af24dd2..f3c9bf22d516 100644
--- a/server/src/main/java/org/elasticsearch/search/profile/Profilers.java
+++ b/server/src/main/java/org/elasticsearch/search/profile/Profilers.java
@@ -66,7 +66,7 @@ public AggregationProfiler getAggregationProfiler() {
      */
     public DfsProfiler getDfsProfiler() {
         if (dfsProfiler == null) {
-            dfsProfiler = new DfsProfiler(getCurrentQueryProfiler());
+            dfsProfiler = new DfsProfiler();
         }
 
         return dfsProfiler;
diff --git a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileDfsPhaseResult.java b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileDfsPhaseResult.java
index adef4bfe47dd..4d3205f185a6 100644
--- a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileDfsPhaseResult.java
+++ b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileDfsPhaseResult.java
@@ -8,10 +8,12 @@
 
 package org.elasticsearch.search.profile;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.search.profile.query.CollectorResult;
 import org.elasticsearch.search.profile.query.QueryProfileShardResult;
 import org.elasticsearch.xcontent.InstantiatingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -21,6 +23,8 @@
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Objects;
 
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
@@ -28,23 +32,35 @@
 public class SearchProfileDfsPhaseResult implements Writeable, ToXContentObject {
 
     private final ProfileResult dfsShardResult;
-    private final QueryProfileShardResult queryProfileShardResult;
+    private final List<QueryProfileShardResult> queryProfileShardResult;
 
     @ParserConstructor
-    public SearchProfileDfsPhaseResult(@Nullable ProfileResult dfsShardResult, @Nullable QueryProfileShardResult queryProfileShardResult) {
+    public SearchProfileDfsPhaseResult(
+        @Nullable ProfileResult dfsShardResult,
+        @Nullable List<QueryProfileShardResult> queryProfileShardResult
+    ) {
         this.dfsShardResult = dfsShardResult;
         this.queryProfileShardResult = queryProfileShardResult;
     }
 
     public SearchProfileDfsPhaseResult(StreamInput in) throws IOException {
         dfsShardResult = in.readOptionalWriteable(ProfileResult::new);
-        queryProfileShardResult = in.readOptionalWriteable(QueryProfileShardResult::new);
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            queryProfileShardResult = in.readOptionalList(QueryProfileShardResult::new);
+        } else {
+            QueryProfileShardResult singleResult = in.readOptionalWriteable(QueryProfileShardResult::new);
+            queryProfileShardResult = singleResult != null ? List.of(singleResult) : null;
+        }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(dfsShardResult);
-        out.writeOptionalWriteable(queryProfileShardResult);
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            out.writeOptionalCollection(queryProfileShardResult);
+        } else {
+            out.writeOptionalWriteable(combineQueryProfileShardResults());
+        }
     }
 
     private static final ParseField STATISTICS = new ParseField("statistics");
@@ -58,7 +74,7 @@ public void writeTo(StreamOutput out) throws IOException {
             SearchProfileDfsPhaseResult.class
         );
         parser.declareObject(optionalConstructorArg(), (p, c) -> ProfileResult.fromXContent(p), STATISTICS);
-        parser.declareObject(optionalConstructorArg(), (p, c) -> QueryProfileShardResult.fromXContent(p), KNN);
+        parser.declareObjectArray(optionalConstructorArg(), (p, c) -> QueryProfileShardResult.fromXContent(p), KNN);
         PARSER = parser.build();
     }
 
@@ -74,8 +90,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             dfsShardResult.toXContent(builder, params);
         }
         if (queryProfileShardResult != null) {
-            builder.field(KNN.getPreferredName());
-            queryProfileShardResult.toXContent(builder, params);
+            builder.startArray(KNN.getPreferredName());
+            for (QueryProfileShardResult qpsr : queryProfileShardResult) {
+                qpsr.toXContent(builder, params);
+            }
+            builder.endArray();
         }
         builder.endObject();
         return builder;
@@ -103,4 +122,33 @@ public String toString() {
             + queryProfileShardResult
             + '}';
     }
+
+    public ProfileResult getDfsShardResult() {
+        return dfsShardResult;
+    }
+
+    public List<QueryProfileShardResult> getQueryProfileShardResult() {
+        return queryProfileShardResult;
+    }
+
+    QueryProfileShardResult combineQueryProfileShardResults() {
+        if (queryProfileShardResult == null) {
+            return null;
+        }
+        List<CollectorResult> subCollectorResults = new ArrayList<>(queryProfileShardResult.size());
+        long totalRewriteTime = 0;
+        long totalCollectionTime = 0;
+        List<ProfileResult> profileResults = new ArrayList<>();
+        for (QueryProfileShardResult queryProfiler : queryProfileShardResult) {
+            totalRewriteTime += queryProfiler.getRewriteTime();
+            profileResults.addAll(queryProfiler.getQueryResults());
+            subCollectorResults.add(queryProfiler.getCollectorResult());
+            totalCollectionTime += queryProfiler.getCollectorResult().getTime();
+        }
+        return new QueryProfileShardResult(
+            profileResults,
+            totalRewriteTime,
+            new CollectorResult("KnnQueryCollector", CollectorResult.REASON_SEARCH_MULTI, totalCollectionTime, subCollectorResults)
+        );
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileQueryPhaseResult.java b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileQueryPhaseResult.java
index caeac1a74e37..04855716fe30 100644
--- a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileQueryPhaseResult.java
+++ b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileQueryPhaseResult.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.profile;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -42,7 +42,7 @@ public SearchProfileQueryPhaseResult(
     }
 
     public SearchProfileQueryPhaseResult(StreamInput in) throws IOException {
-        if (in.getVersion().onOrAfter(Version.V_8_6_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_6_0)) {
             searchProfileDfsPhaseResult = in.readOptionalWriteable(SearchProfileDfsPhaseResult::new);
         }
         int profileSize = in.readVInt();
@@ -57,7 +57,7 @@ public SearchProfileQueryPhaseResult(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getVersion().onOrAfter(Version.V_8_6_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_6_0)) {
             out.writeOptionalWriteable(searchProfileDfsPhaseResult);
         }
         out.writeVInt(queryProfileResults.size());
diff --git a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java
index 84d46af3036a..40a251ab24b8 100644
--- a/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java
+++ b/server/src/main/java/org/elasticsearch/search/profile/SearchProfileResults.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.profile;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -45,7 +45,7 @@ public SearchProfileResults(Map<String, SearchProfileShardResult> shardResults)
     }
 
     public SearchProfileResults(StreamInput in) throws IOException {
-        if (in.getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             shardResults = in.readMap(StreamInput::readString, SearchProfileShardResult::new);
         } else {
             // Before 8.0.0 we only send the query phase result
@@ -58,7 +58,7 @@ public SearchProfileResults(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             out.writeMap(shardResults, StreamOutput::writeString, (o, r) -> r.writeTo(o));
         } else {
             // Before 8.0.0 we only send the query phase
diff --git a/server/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResult.java b/server/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResult.java
index 685ebdcb6d66..a49231780f54 100644
--- a/server/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResult.java
+++ b/server/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResult.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.search.profile.aggregation;
 
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -20,7 +21,6 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.Objects;
 
 import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 
@@ -83,7 +83,7 @@ public int hashCode() {
 
     @Override
     public String toString() {
-        return Objects.toString(this);
+        return Strings.toString(this);
     }
 
     public static AggregationProfileShardResult fromXContent(XContentParser parser) throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingLeafBucketCollector.java b/server/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingLeafBucketCollector.java
index 1e8d8ee1cd5d..8e84a19d1daf 100644
--- a/server/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingLeafBucketCollector.java
+++ b/server/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingLeafBucketCollector.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.search.profile.aggregation;
 
+import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Scorable;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.profile.Timer;
@@ -34,6 +35,11 @@ public void collect(int doc, long bucket) throws IOException {
         }
     }
 
+    @Override
+    public DocIdSetIterator competitiveIterator() throws IOException {
+        return delegate.competitiveIterator();
+    }
+
     @Override
     public void setScorer(Scorable scorer) throws IOException {
         delegate.setScorer(scorer);
diff --git a/server/src/main/java/org/elasticsearch/search/profile/dfs/DfsProfiler.java b/server/src/main/java/org/elasticsearch/search/profile/dfs/DfsProfiler.java
index 6eeba4cce1e7..a74840455816 100644
--- a/server/src/main/java/org/elasticsearch/search/profile/dfs/DfsProfiler.java
+++ b/server/src/main/java/org/elasticsearch/search/profile/dfs/DfsProfiler.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.search.profile.query.QueryProfileShardResult;
 import org.elasticsearch.search.profile.query.QueryProfiler;
 
+import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -28,12 +29,11 @@ public class DfsProfiler extends AbstractProfileBreakdown<DfsTimingType> {
     private long startTime;
     private long totalTime;
 
-    private final QueryProfiler queryProfiler;
+    private final List<QueryProfiler> knnQueryProfilers = new ArrayList<>();
     private boolean collectorSet = false;
 
-    public DfsProfiler(QueryProfiler queryProfiler) {
+    public DfsProfiler() {
         super(DfsTimingType.class);
-        this.queryProfiler = queryProfiler;
     }
 
     public void start() {
@@ -52,9 +52,12 @@ public void stopTimer(DfsTimingType dfsTimingType) {
         getTimer(dfsTimingType).stop();
     }
 
-    public void setCollector(InternalProfileCollector collector) {
+    public QueryProfiler addQueryProfiler(InternalProfileCollector collector) {
+        QueryProfiler queryProfiler = new QueryProfiler();
         queryProfiler.setCollector(collector);
+        knnQueryProfilers.add(queryProfiler);
         collectorSet = true;
+        return queryProfiler;
     }
 
     public SearchProfileDfsPhaseResult buildDfsPhaseResults() {
@@ -66,9 +69,17 @@ public SearchProfileDfsPhaseResult buildDfsPhaseResults() {
             totalTime,
             List.of()
         );
-        QueryProfileShardResult queryProfileShardResult = collectorSet
-            ? new QueryProfileShardResult(queryProfiler.getTree(), queryProfiler.getRewriteTime(), queryProfiler.getCollector())
-            : null;
+        final List<QueryProfileShardResult> queryProfileShardResult;
+        if (collectorSet) {
+            queryProfileShardResult = new ArrayList<>(knnQueryProfilers.size());
+            for (QueryProfiler queryProfiler : knnQueryProfilers) {
+                queryProfileShardResult.add(
+                    new QueryProfileShardResult(queryProfiler.getTree(), queryProfiler.getRewriteTime(), queryProfiler.getCollector())
+                );
+            }
+        } else {
+            queryProfileShardResult = null;
+        }
         return new SearchProfileDfsPhaseResult(dfsProfileResult, queryProfileShardResult);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/profile/query/ProfileCollector.java b/server/src/main/java/org/elasticsearch/search/profile/query/ProfileCollector.java
index 9d025d99e117..84728499db0f 100644
--- a/server/src/main/java/org/elasticsearch/search/profile/query/ProfileCollector.java
+++ b/server/src/main/java/org/elasticsearch/search/profile/query/ProfileCollector.java
@@ -10,6 +10,7 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.FilterCollector;
 import org.apache.lucene.search.FilterLeafCollector;
 import org.apache.lucene.search.LeafCollector;
@@ -64,6 +65,11 @@ public void collect(int doc) throws IOException {
                 }
             }
 
+            @Override
+            public DocIdSetIterator competitiveIterator() throws IOException {
+                return in.competitiveIterator();
+            }
+
             @Override
             public void setScorer(Scorable scorer) throws IOException {
                 final long start = System.nanoTime();
diff --git a/server/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java b/server/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java
index 2114f7321ad7..877854f5cff9 100644
--- a/server/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java
+++ b/server/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java
@@ -11,6 +11,7 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.BulkScorer;
 import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.Matches;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.ScorerSupplier;
@@ -102,7 +103,13 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio
 
     @Override
     public int count(LeafReaderContext context) throws IOException {
-        return subQueryWeight.count(context);
+        Timer timer = profile.getTimer(QueryTimingType.COUNT_WEIGHT);
+        timer.start();
+        try {
+            return subQueryWeight.count(context);
+        } finally {
+            timer.stop();
+        }
     }
 
     @Override
@@ -110,4 +117,7 @@ public boolean isCacheable(LeafReaderContext ctx) {
         return false;
     }
 
+    public Matches matches(LeafReaderContext context, int doc) throws IOException {
+        return subQueryWeight.matches(context, doc);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/profile/query/QueryTimingType.java b/server/src/main/java/org/elasticsearch/search/profile/query/QueryTimingType.java
index fc8ae88e27ff..03444a8b52c6 100644
--- a/server/src/main/java/org/elasticsearch/search/profile/query/QueryTimingType.java
+++ b/server/src/main/java/org/elasticsearch/search/profile/query/QueryTimingType.java
@@ -12,6 +12,7 @@
 
 public enum QueryTimingType {
     CREATE_WEIGHT,
+    COUNT_WEIGHT,
     BUILD_SCORER,
     NEXT_DOC,
     ADVANCE,
diff --git a/server/src/main/java/org/elasticsearch/search/query/EarlyTerminatingCollector.java b/server/src/main/java/org/elasticsearch/search/query/EarlyTerminatingCollector.java
index 85ab5b14bb0f..bc4bc079aa4b 100644
--- a/server/src/main/java/org/elasticsearch/search/query/EarlyTerminatingCollector.java
+++ b/server/src/main/java/org/elasticsearch/search/query/EarlyTerminatingCollector.java
@@ -14,6 +14,7 @@
 import org.apache.lucene.search.FilterCollector;
 import org.apache.lucene.search.FilterLeafCollector;
 import org.apache.lucene.search.LeafCollector;
+import org.apache.lucene.search.ScoreMode;
 
 import java.io.IOException;
 
@@ -45,6 +46,16 @@ static final class EarlyTerminationException extends RuntimeException {
         this.forceTermination = forceTermination;
     }
 
+    @Override
+    public ScoreMode scoreMode() {
+        // Let the query know that this collector doesn't intend to collect all hits.
+        ScoreMode scoreMode = super.scoreMode();
+        if (scoreMode.isExhaustive()) {
+            scoreMode = scoreMode.needsScores() ? ScoreMode.TOP_DOCS_WITH_SCORES : ScoreMode.TOP_DOCS;
+        }
+        return scoreMode;
+    }
+
     @Override
     public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
         if (numCollected >= maxCountHits) {
diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index 5fbaed04e31b..2e300be8d40b 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -129,21 +129,15 @@ static boolean executeInternal(SearchContext searchContext) throws QueryPhaseExe
             }
 
             final LinkedList<QueryCollectorContext> collectors = new LinkedList<>();
-            // whether the chain contains a collector that filters documents
-            boolean hasFilterCollector = false;
             if (searchContext.terminateAfter() != SearchContext.DEFAULT_TERMINATE_AFTER) {
                 // add terminate_after before the filter collectors
                 // it will only be applied on documents accepted by these filter collectors
                 collectors.add(createEarlyTerminationCollectorContext(searchContext.terminateAfter()));
-                // this collector can filter documents during the collection
-                hasFilterCollector = true;
             }
             if (searchContext.parsedPostFilter() != null) {
                 // add post filters before aggregations
                 // it will only be applied to top hits
                 collectors.add(createFilteredCollectorContext(searcher, searchContext.parsedPostFilter().query()));
-                // this collector can filter documents during the collection
-                hasFilterCollector = true;
             }
             if (searchContext.queryCollectors().isEmpty() == false) {
                 // plug in additional collectors, like aggregations
@@ -152,8 +146,6 @@ static boolean executeInternal(SearchContext searchContext) throws QueryPhaseExe
             if (searchContext.minimumScore() != null) {
                 // apply the minimum score after multi collector so we filter aggs as well
                 collectors.add(createMinScoreCollectorContext(searchContext.minimumScore()));
-                // this collector can filter documents during the collection
-                hasFilterCollector = true;
             }
 
             boolean timeoutSet = scrollContext == null
@@ -176,7 +168,7 @@ static boolean executeInternal(SearchContext searchContext) throws QueryPhaseExe
             }
 
             try {
-                boolean shouldRescore = searchWithCollector(searchContext, searcher, query, collectors, hasFilterCollector, timeoutSet);
+                boolean shouldRescore = searchWithCollector(searchContext, searcher, query, collectors, timeoutSet);
                 ExecutorService executor = searchContext.indexShard().getThreadPool().executor(ThreadPool.Names.SEARCH);
                 assert executor instanceof EWMATrackingEsThreadPoolExecutor
                     || (executor instanceof EsThreadPoolExecutor == false /* in case thread pool is mocked out in tests */)
@@ -203,11 +195,10 @@ private static boolean searchWithCollector(
         ContextIndexSearcher searcher,
         Query query,
         LinkedList<QueryCollectorContext> collectors,
-        boolean hasFilterCollector,
         boolean timeoutSet
     ) throws IOException {
         // create the top docs collector last when the other collectors are known
-        final TopDocsCollectorContext topDocsFactory = createTopDocsCollectorContext(searchContext, hasFilterCollector);
+        final TopDocsCollectorContext topDocsFactory = createTopDocsCollectorContext(searchContext);
         // add the top docs collector, the first collector context in the chain
         collectors.addFirst(topDocsFactory);
 
diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java
index 0bfb49e1764f..36392a92f3b4 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.query;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.search.SearchShardTask;
@@ -51,7 +51,7 @@ public QuerySearchRequest(StreamInput in) throws IOException {
         contextId = new ShardSearchContextId(in);
         dfs = new AggregatedDfs(in);
         originalIndices = OriginalIndices.readOriginalIndices(in);
-        if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             this.shardSearchRequest = in.readOptionalWriteable(ShardSearchRequest::new);
         } else {
             this.shardSearchRequest = null;
@@ -64,7 +64,7 @@ public void writeTo(StreamOutput out) throws IOException {
         contextId.writeTo(out);
         dfs.writeTo(out);
         OriginalIndices.writeOriginalIndices(originalIndices, out);
-        if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             out.writeOptionalWriteable(shardSearchRequest);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
index 0741f13ca1f6..1dbf9bbf0dd4 100644
--- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
+++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.TotalHits;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.DelayableWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -75,7 +75,7 @@ public QuerySearchResult(StreamInput in) throws IOException {
      */
     public QuerySearchResult(StreamInput in, boolean delayedAggregations) throws IOException {
         super(in);
-        if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_7_0)) {
             isNull = in.readBoolean();
         } else {
             isNull = false;
@@ -369,7 +369,7 @@ private void readFromWithId(ShardSearchContextId id, StreamInput in, boolean del
             hasProfileResults = profileShardResults != null;
             serviceTimeEWMA = in.readZLong();
             nodeQueueSize = in.readInt();
-            if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
                 setShardSearchRequest(in.readOptionalWriteable(ShardSearchRequest::new));
                 setRescoreDocIds(new RescoreDocIds(in));
             }
@@ -388,7 +388,7 @@ public void writeTo(StreamOutput out) throws IOException {
         if (aggregations != null && aggregations.isSerialized()) {
             throw new IllegalStateException("cannot send serialized version since it will leak");
         }
-        if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_7_0)) {
             out.writeBoolean(isNull);
         }
         if (isNull == false) {
@@ -421,7 +421,7 @@ public void writeToNoId(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(profileShardResults);
         out.writeZLong(serviceTimeEWMA);
         out.writeInt(nodeQueueSize);
-        if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             out.writeOptionalWriteable(getShardSearchRequest());
             getRescoreDocIds().writeTo(out);
         }
diff --git a/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java b/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java
index 5f5b8569da2f..1f49654c9ddf 100644
--- a/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java
+++ b/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java
@@ -8,30 +8,17 @@
 
 package org.elasticsearch.search.query;
 
-import org.apache.lucene.index.DocValuesType;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.PointValues;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.Terms;
 import org.apache.lucene.queries.spans.SpanQuery;
 import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.Collector;
-import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.FieldDoc;
-import org.apache.lucene.search.FieldExistsQuery;
-import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MultiCollector;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TopDocsCollector;
 import org.apache.lucene.search.TopFieldCollector;
@@ -74,13 +61,6 @@ abstract class TopDocsCollectorContext extends QueryCollectorContext {
         this.numHits = numHits;
     }
 
-    /**
-     * Returns the number of top docs to retrieve
-     */
-    final int numHits() {
-        return numHits;
-    }
-
     /**
      * Returns true if the top docs should be re-scored after initial search
      */
@@ -93,20 +73,7 @@ static class EmptyTopDocsCollectorContext extends TopDocsCollectorContext {
         private final Collector collector;
         private final Supplier<TotalHits> hitCountSupplier;
 
-        /**
-         * Ctr
-         * @param reader The index reader
-         * @param query The query to execute
-         * @param trackTotalHitsUpTo True if the total number of hits should be tracked
-         * @param hasFilterCollector True if the collector chain contains a filter
-         */
-        private EmptyTopDocsCollectorContext(
-            IndexReader reader,
-            Query query,
-            @Nullable SortAndFormats sortAndFormats,
-            int trackTotalHitsUpTo,
-            boolean hasFilterCollector
-        ) throws IOException {
+        private EmptyTopDocsCollectorContext(@Nullable SortAndFormats sortAndFormats, int trackTotalHitsUpTo) {
             super(REASON_SEARCH_COUNT, 0);
             this.sort = sortAndFormats == null ? null : sortAndFormats.sort;
             if (trackTotalHitsUpTo == SearchContext.TRACK_TOTAL_HITS_DISABLED) {
@@ -116,22 +83,16 @@ private EmptyTopDocsCollectorContext(
             } else {
                 TotalHitCountCollector hitCountCollector = new TotalHitCountCollector();
                 // implicit total hit counts are valid only when there is no filter collector in the chain
-                int hitCount = hasFilterCollector ? -1 : shortcutTotalHitCount(reader, query);
-                if (hitCount == -1) {
-                    if (trackTotalHitsUpTo == SearchContext.TRACK_TOTAL_HITS_ACCURATE) {
-                        this.collector = hitCountCollector;
-                        this.hitCountSupplier = () -> new TotalHits(hitCountCollector.getTotalHits(), TotalHits.Relation.EQUAL_TO);
-                    } else {
-                        EarlyTerminatingCollector col = new EarlyTerminatingCollector(hitCountCollector, trackTotalHitsUpTo, false);
-                        this.collector = col;
-                        this.hitCountSupplier = () -> new TotalHits(
-                            hitCountCollector.getTotalHits(),
-                            col.hasEarlyTerminated() ? TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO : TotalHits.Relation.EQUAL_TO
-                        );
-                    }
+                if (trackTotalHitsUpTo == SearchContext.TRACK_TOTAL_HITS_ACCURATE) {
+                    this.collector = hitCountCollector;
+                    this.hitCountSupplier = () -> new TotalHits(hitCountCollector.getTotalHits(), TotalHits.Relation.EQUAL_TO);
                 } else {
-                    this.collector = new EarlyTerminatingCollector(hitCountCollector, 0, false);
-                    this.hitCountSupplier = () -> new TotalHits(hitCount, TotalHits.Relation.EQUAL_TO);
+                    EarlyTerminatingCollector col = new EarlyTerminatingCollector(hitCountCollector, trackTotalHitsUpTo, false);
+                    this.collector = col;
+                    this.hitCountSupplier = () -> new TotalHits(
+                        hitCountCollector.getTotalHits(),
+                        col.hasEarlyTerminated() ? TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO : TotalHits.Relation.EQUAL_TO
+                    );
                 }
             }
         }
@@ -191,7 +152,7 @@ private CollapsingTopDocsCollectorContext(
         }
 
         @Override
-        Collector create(Collector in) throws IOException {
+        Collector create(Collector in) {
             assert in == null;
             return topDocsCollector;
         }
@@ -226,25 +187,21 @@ private static TopDocsCollector<?> createCollector(
 
         /**
          * Ctr
-         * @param reader The index reader
          * @param query The Lucene query
          * @param sortAndFormats The query sort
          * @param numHits The number of top hits to retrieve
          * @param searchAfter The doc this request should "search after"
          * @param trackMaxScore True if max score should be tracked
          * @param trackTotalHitsUpTo True if the total number of hits should be tracked
-         * @param hasFilterCollector True if the collector chain contains at least one collector that can filters document
          */
         private SimpleTopDocsCollectorContext(
-            IndexReader reader,
             Query query,
             @Nullable SortAndFormats sortAndFormats,
             @Nullable ScoreDoc searchAfter,
             int numHits,
             boolean trackMaxScore,
-            int trackTotalHitsUpTo,
-            boolean hasFilterCollector
-        ) throws IOException {
+            int trackTotalHitsUpTo
+        ) {
             super(REASON_SEARCH_TOP_HITS, numHits);
             this.sortAndFormats = sortAndFormats;
 
@@ -262,18 +219,9 @@ private SimpleTopDocsCollectorContext(
                 topDocsSupplier = new CachedSupplier<>(topDocsCollector::topDocs);
                 totalHitsSupplier = () -> new TotalHits(0, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO);
             } else {
-                // implicit total hit counts are valid only when there is no filter collector in the chain
-                final int hitCount = hasFilterCollector ? -1 : shortcutTotalHitCount(reader, query);
-                if (hitCount == -1) {
-                    topDocsCollector = createCollector(sortAndFormats, numHits, searchAfter, trackTotalHitsUpTo);
-                    topDocsSupplier = new CachedSupplier<>(topDocsCollector::topDocs);
-                    totalHitsSupplier = () -> topDocsSupplier.get().totalHits;
-                } else {
-                    // don't compute hit counts via the collector
-                    topDocsCollector = createCollector(sortAndFormats, numHits, searchAfter, 1);
-                    topDocsSupplier = new CachedSupplier<>(topDocsCollector::topDocs);
-                    totalHitsSupplier = () -> new TotalHits(hitCount, TotalHits.Relation.EQUAL_TO);
-                }
+                topDocsCollector = createCollector(sortAndFormats, numHits, searchAfter, trackTotalHitsUpTo);
+                topDocsSupplier = new CachedSupplier<>(topDocsCollector::topDocs);
+                totalHitsSupplier = () -> topDocsSupplier.get().totalHits;
             }
             MaxScoreCollector maxScoreCollector = null;
             if (sortAndFormats == null) {
@@ -315,7 +263,7 @@ TopDocsAndMaxScore newTopDocs() {
         }
 
         @Override
-        void postProcess(QuerySearchResult result) throws IOException {
+        void postProcess(QuerySearchResult result) {
             final TopDocsAndMaxScore topDocs = newTopDocs();
             result.topDocs(topDocs, sortAndFormats == null ? null : sortAndFormats.formats);
         }
@@ -326,32 +274,21 @@ static class ScrollingTopDocsCollectorContext extends SimpleTopDocsCollectorCont
         private final int numberOfShards;
 
         private ScrollingTopDocsCollectorContext(
-            IndexReader reader,
             Query query,
             ScrollContext scrollContext,
             @Nullable SortAndFormats sortAndFormats,
             int numHits,
             boolean trackMaxScore,
             int numberOfShards,
-            int trackTotalHitsUpTo,
-            boolean hasFilterCollector
-        ) throws IOException {
-            super(
-                reader,
-                query,
-                sortAndFormats,
-                scrollContext.lastEmittedDoc,
-                numHits,
-                trackMaxScore,
-                trackTotalHitsUpTo,
-                hasFilterCollector
-            );
+            int trackTotalHitsUpTo
+        ) {
+            super(query, sortAndFormats, scrollContext.lastEmittedDoc, numHits, trackMaxScore, trackTotalHitsUpTo);
             this.scrollContext = Objects.requireNonNull(scrollContext);
             this.numberOfShards = numberOfShards;
         }
 
         @Override
-        void postProcess(QuerySearchResult result) throws IOException {
+        void postProcess(QuerySearchResult result) {
             final TopDocsAndMaxScore topDocs = newTopDocs();
             if (scrollContext.totalHits == null) {
                 // first round
@@ -374,138 +311,68 @@ void postProcess(QuerySearchResult result) throws IOException {
         }
     }
 
-    /**
-     * Returns query total hit count if the <code>query</code> is a {@link MatchAllDocsQuery}
-     * or a {@link TermQuery} and the <code>reader</code> has no deletions,
-     * -1 otherwise.
-     */
-    static int shortcutTotalHitCount(IndexReader reader, Query query) throws IOException {
-        while (true) {
-            // remove wrappers that don't matter for counts
-            // this is necessary so that we don't only optimize match_all
-            // queries but also match_all queries that are nested in
-            // a constant_score query
-            if (query instanceof ConstantScoreQuery) {
-                query = ((ConstantScoreQuery) query).getQuery();
-            } else if (query instanceof BoostQuery) {
-                query = ((BoostQuery) query).getQuery();
-            } else {
-                break;
-            }
-        }
-        if (query.getClass() == MatchAllDocsQuery.class) {
-            return reader.numDocs();
-        } else if (query.getClass() == TermQuery.class && reader.hasDeletions() == false) {
-            final Term term = ((TermQuery) query).getTerm();
-            int count = 0;
-            for (LeafReaderContext context : reader.leaves()) {
-                count += context.reader().docFreq(term);
-            }
-            return count;
-        } else if (query.getClass() == FieldExistsQuery.class && reader.hasDeletions() == false) {
-            final String field = ((FieldExistsQuery) query).getField();
-            int count = 0;
-            for (LeafReaderContext context : reader.leaves()) {
-                FieldInfos fieldInfos = context.reader().getFieldInfos();
-                FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
-                if (fieldInfo != null) {
-                    if (fieldInfo.getDocValuesType() == DocValuesType.NONE) {
-                        // no shortcut possible: it's a text field, empty values are counted as no value.
-                        return -1;
-                    }
-                    if (fieldInfo.getPointIndexDimensionCount() > 0) {
-                        PointValues points = context.reader().getPointValues(field);
-                        if (points != null) {
-                            count += points.getDocCount();
-                        }
-                    } else if (fieldInfo.getIndexOptions() != IndexOptions.NONE) {
-                        Terms terms = context.reader().terms(field);
-                        if (terms != null) {
-                            count += terms.getDocCount();
-                        }
-                    } else {
-                        return -1; // no shortcut possible for fields that are not indexed
-                    }
-                }
-            }
-            return count;
-        } else {
-            return -1;
-        }
-    }
-
     /**
      * Creates a {@link TopDocsCollectorContext} from the provided <code>searchContext</code>.
-     * @param hasFilterCollector True if the collector chain contains at least one collector that can filters document.
      */
-    static TopDocsCollectorContext createTopDocsCollectorContext(SearchContext searchContext, boolean hasFilterCollector)
-        throws IOException {
-        final IndexReader reader = searchContext.searcher().getIndexReader();
-        final Query query = searchContext.rewrittenQuery();
-        // top collectors don't like a size of 0
-        final int totalNumDocs = Math.max(1, reader.numDocs());
+    static TopDocsCollectorContext createTopDocsCollectorContext(SearchContext searchContext) {
         if (searchContext.size() == 0) {
             // no matter what the value of from is
-            return new EmptyTopDocsCollectorContext(
-                reader,
-                query,
-                searchContext.sort(),
-                searchContext.trackTotalHitsUpTo(),
-                hasFilterCollector
-            );
-        } else if (searchContext.scrollContext() != null) {
-            // we can disable the tracking of total hits after the initial scroll query
-            // since the total hits is preserved in the scroll context.
-            int trackTotalHitsUpTo = searchContext.scrollContext().totalHits != null
-                ? SearchContext.TRACK_TOTAL_HITS_DISABLED
-                : SearchContext.TRACK_TOTAL_HITS_ACCURATE;
-            // no matter what the value of from is
-            int numDocs = Math.min(searchContext.size(), totalNumDocs);
-            return new ScrollingTopDocsCollectorContext(
-                reader,
-                query,
-                searchContext.scrollContext(),
-                searchContext.sort(),
-                numDocs,
-                searchContext.trackScores(),
-                searchContext.numberOfShards(),
-                trackTotalHitsUpTo,
-                hasFilterCollector
-            );
-        } else if (searchContext.collapse() != null) {
-            boolean trackScores = searchContext.sort() == null ? true : searchContext.trackScores();
-            int numDocs = Math.min(searchContext.from() + searchContext.size(), totalNumDocs);
-            return new CollapsingTopDocsCollectorContext(
-                searchContext.collapse(),
-                searchContext.sort(),
-                numDocs,
-                trackScores,
-                searchContext.searchAfter()
-            );
+            return new EmptyTopDocsCollectorContext(searchContext.sort(), searchContext.trackTotalHitsUpTo());
         } else {
-            int numDocs = Math.min(searchContext.from() + searchContext.size(), totalNumDocs);
-            final boolean rescore = searchContext.rescore().isEmpty() == false;
-            if (rescore) {
-                assert searchContext.sort() == null;
-                for (RescoreContext rescoreContext : searchContext.rescore()) {
-                    numDocs = Math.max(numDocs, rescoreContext.getWindowSize());
+            final IndexReader reader = searchContext.searcher().getIndexReader();
+            final Query query = searchContext.rewrittenQuery();
+            // top collectors don't like a size of 0
+            final int totalNumDocs = Math.max(1, reader.numDocs());
+            if (searchContext.scrollContext() != null) {
+                // we can disable the tracking of total hits after the initial scroll query
+                // since the total hits is preserved in the scroll context.
+                int trackTotalHitsUpTo = searchContext.scrollContext().totalHits != null
+                    ? SearchContext.TRACK_TOTAL_HITS_DISABLED
+                    : SearchContext.TRACK_TOTAL_HITS_ACCURATE;
+                // no matter what the value of from is
+                int numDocs = Math.min(searchContext.size(), totalNumDocs);
+                return new ScrollingTopDocsCollectorContext(
+                    query,
+                    searchContext.scrollContext(),
+                    searchContext.sort(),
+                    numDocs,
+                    searchContext.trackScores(),
+                    searchContext.numberOfShards(),
+                    trackTotalHitsUpTo
+                );
+            } else if (searchContext.collapse() != null) {
+                boolean trackScores = searchContext.sort() == null ? true : searchContext.trackScores();
+                int numDocs = Math.min(searchContext.from() + searchContext.size(), totalNumDocs);
+                return new CollapsingTopDocsCollectorContext(
+                    searchContext.collapse(),
+                    searchContext.sort(),
+                    numDocs,
+                    trackScores,
+                    searchContext.searchAfter()
+                );
+            } else {
+                int numDocs = Math.min(searchContext.from() + searchContext.size(), totalNumDocs);
+                final boolean rescore = searchContext.rescore().isEmpty() == false;
+                if (rescore) {
+                    assert searchContext.sort() == null;
+                    for (RescoreContext rescoreContext : searchContext.rescore()) {
+                        numDocs = Math.max(numDocs, rescoreContext.getWindowSize());
+                    }
                 }
+                return new SimpleTopDocsCollectorContext(
+                    query,
+                    searchContext.sort(),
+                    searchContext.searchAfter(),
+                    numDocs,
+                    searchContext.trackScores(),
+                    searchContext.trackTotalHitsUpTo()
+                ) {
+                    @Override
+                    boolean shouldRescore() {
+                        return rescore;
+                    }
+                };
             }
-            return new SimpleTopDocsCollectorContext(
-                reader,
-                query,
-                searchContext.sort(),
-                searchContext.searchAfter(),
-                numDocs,
-                searchContext.trackScores(),
-                searchContext.trackTotalHitsUpTo(),
-                hasFilterCollector
-            ) {
-                @Override
-                boolean shouldRescore() {
-                    return rescore;
-                }
-            };
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java b/server/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java
index 59e6fca75a4d..228eda3edb12 100644
--- a/server/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.rescore;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -93,8 +93,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/search/runtime/AbstractStringScriptFieldQuery.java b/server/src/main/java/org/elasticsearch/search/runtime/AbstractStringScriptFieldQuery.java
index bf2be13aab77..dde67d2143d7 100644
--- a/server/src/main/java/org/elasticsearch/search/runtime/AbstractStringScriptFieldQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/runtime/AbstractStringScriptFieldQuery.java
@@ -24,7 +24,8 @@ abstract class AbstractStringScriptFieldQuery extends AbstractScriptFieldQuery<S
 
     @Override
     protected final boolean matches(StringFieldScript scriptContext, int docId) {
-        return matches(scriptContext.resultsForDoc(docId));
+        scriptContext.runForDoc(docId);
+        return matches(scriptContext.getValues());
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java b/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java
index 895aa5bed5bf..fdd7aeb8f3b3 100644
--- a/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -90,7 +90,7 @@ public SliceBuilder(String field, int id, int max) {
     }
 
     public SliceBuilder(StreamInput in) throws IOException {
-        if (in.getVersion().before(Version.V_7_15_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_7_15_0)) {
             field = in.readString();
         } else {
             field = in.readOptionalString();
@@ -103,7 +103,7 @@ public SliceBuilder(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         // Before 7.15.0 we always defaulted to _id when the field wasn't provided
-        if (out.getVersion().before(Version.V_7_15_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_7_15_0)) {
             String sliceField = field != null ? field : IdFieldMapper.NAME;
             out.writeString(sliceField);
         } else {
diff --git a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
index 19326a214a94..86e4a96ad328 100644
--- a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
@@ -15,7 +15,7 @@
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.search.SortField;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -132,7 +132,7 @@ public FieldSortBuilder(String fieldName) {
      */
     public FieldSortBuilder(StreamInput in) throws IOException {
         fieldName = in.readString();
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             if (in.readOptionalNamedWriteable(QueryBuilder.class) != null || in.readOptionalString() != null) {
                 throw new IOException(
                     "the [sort] options [nested_path] and [nested_filter] are removed in 8.x, " + "please use [nested] instead"
@@ -144,10 +144,10 @@ public FieldSortBuilder(StreamInput in) throws IOException {
         sortMode = in.readOptionalWriteable(SortMode::readFromStream);
         unmappedType = in.readOptionalString();
         nestedSort = in.readOptionalWriteable(NestedSortBuilder::new);
-        if (in.getVersion().onOrAfter(Version.V_7_2_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_2_0)) {
             numericType = in.readOptionalString();
         }
-        if (in.getVersion().onOrAfter(Version.V_7_13_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0)) {
             format = in.readOptionalString();
         }
     }
@@ -155,7 +155,7 @@ public FieldSortBuilder(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(fieldName);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeOptionalNamedWriteable(null);
             out.writeOptionalString(null);
         }
@@ -164,10 +164,10 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(sortMode);
         out.writeOptionalString(unmappedType);
         out.writeOptionalWriteable(nestedSort);
-        if (out.getVersion().onOrAfter(Version.V_7_2_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_2_0)) {
             out.writeOptionalString(numericType);
         }
-        if (out.getVersion().onOrAfter(Version.V_7_13_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0)) {
             out.writeOptionalString(format);
         } else {
             if (format != null) {
@@ -710,8 +710,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
index f58e24c68662..7f82db82a3cc 100644
--- a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
@@ -18,7 +18,7 @@
 import org.apache.lucene.search.comparators.DoubleComparator;
 import org.apache.lucene.util.BitSet;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.geo.GeoDistance;
 import org.elasticsearch.common.geo.GeoPoint;
@@ -158,7 +158,7 @@ public GeoDistanceSortBuilder(StreamInput in) throws IOException {
         unit = DistanceUnit.readFromStream(in);
         order = SortOrder.readFromStream(in);
         sortMode = in.readOptionalWriteable(SortMode::readFromStream);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             if (in.readOptionalNamedWriteable(QueryBuilder.class) != null || in.readOptionalString() != null) {
                 throw new IOException(
                     "the [sort] options [nested_path] and [nested_filter] are removed in 8.x, " + "please use [nested] instead"
@@ -179,7 +179,7 @@ public void writeTo(StreamOutput out) throws IOException {
         unit.writeTo(out);
         order.writeTo(out);
         out.writeOptionalWriteable(sortMode);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeOptionalNamedWriteable(null);
             out.writeOptionalString(null);
         }
@@ -352,8 +352,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java
index 915788af6373..77c5ac3516e1 100644
--- a/server/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Scorable;
 import org.apache.lucene.search.SortField;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.BigArrays;
@@ -159,8 +159,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
index b8f3c30905b7..79935c91a5bb 100644
--- a/server/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
@@ -14,7 +14,7 @@
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -109,7 +109,7 @@ public ScriptSortBuilder(StreamInput in) throws IOException {
         type = ScriptSortType.readFromStream(in);
         order = SortOrder.readFromStream(in);
         sortMode = in.readOptionalWriteable(SortMode::readFromStream);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             if (in.readOptionalNamedWriteable(QueryBuilder.class) != null || in.readOptionalString() != null) {
                 throw new IOException(
                     "the [sort] options [nested_path] and [nested_filter] are removed in 8.x, " + "please use [nested] instead"
@@ -125,7 +125,7 @@ public void writeTo(StreamOutput out) throws IOException {
         type.writeTo(out);
         order.writeTo(out);
         out.writeOptionalWriteable(sortMode);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeOptionalString(null);
             out.writeOptionalNamedWriteable(null);
         }
@@ -458,8 +458,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     public enum ScriptSortType implements Writeable {
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java b/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java
index d10b10c09d66..6d22de25279c 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.search.suggest.completion;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -327,8 +327,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java b/server/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java
index 198207535280..61f81f77add0 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
@@ -120,7 +120,7 @@ public WordScorerFactory buildWordScorerFactory() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java b/server/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java
index a3b0dfcbed8f..27f111fb6d82 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -177,7 +177,7 @@ public WordScorerFactory buildWordScorerFactory() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java b/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
index 23a30934fae9..c2252023fde9 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
@@ -9,7 +9,7 @@
 
 import org.apache.lucene.analysis.Analyzer;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -698,8 +698,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java b/server/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java
index 965913136048..0c18cc67f759 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
@@ -123,7 +123,7 @@ public WordScorerFactory buildWordScorerFactory() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java b/server/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
index bf9f26864bfb..e5c091553164 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
@@ -15,7 +15,7 @@
 import org.apache.lucene.search.spell.NGramDistance;
 import org.apache.lucene.search.spell.StringDistance;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -458,8 +458,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQuery.java
index 405c3c061a41..9f2d68ab3291 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQuery.java
@@ -30,7 +30,7 @@
  * A query that matches the provided docs with their scores.
  *
  * Note: this query was adapted from Lucene's DocAndScoreQuery from the class
- * {@link org.apache.lucene.search.KnnVectorQuery}, which is package-private.
+ * {@link org.apache.lucene.search.KnnFloatVectorQuery}, which is package-private.
  * There are no changes to the behavior, just some renames.
  */
 class KnnScoreDocQuery extends Query {
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java
index 20dc8e7ce82d..7e53ff3b4c85 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnScoreDocQueryBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.Lucene;
@@ -147,7 +147,7 @@ protected int doHashCode() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_4_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_4_0;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java
index a4679b6f891c..e650aa199211 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java
@@ -8,6 +8,9 @@
 
 package org.elasticsearch.search.vectors;
 
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -27,8 +30,11 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
+import java.util.function.Supplier;
 
+import static org.elasticsearch.common.Strings.format;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 /**
  * Defines a kNN search to run in the search request.
@@ -39,24 +45,36 @@ public class KnnSearchBuilder implements Writeable, ToXContentFragment, Rewritea
     public static final ParseField K_FIELD = new ParseField("k");
     public static final ParseField NUM_CANDS_FIELD = new ParseField("num_candidates");
     public static final ParseField QUERY_VECTOR_FIELD = new ParseField("query_vector");
+    public static final ParseField QUERY_VECTOR_BUILDER_FIELD = new ParseField("query_vector_builder");
     public static final ParseField FILTER_FIELD = new ParseField("filter");
     public static final ParseField BOOST_FIELD = AbstractQueryBuilder.BOOST_FIELD;
 
     private static final ConstructingObjectParser<KnnSearchBuilder, Void> PARSER = new ConstructingObjectParser<>("knn", args -> {
         @SuppressWarnings("unchecked")
+        // TODO optimize parsing for when BYTE values are provided
         List<Float> vector = (List<Float>) args[1];
-        float[] vectorArray = new float[vector.size()];
-        for (int i = 0; i < vector.size(); i++) {
-            vectorArray[i] = vector.get(i);
+        final float[] vectorArray;
+        if (vector != null) {
+            vectorArray = new float[vector.size()];
+            for (int i = 0; i < vector.size(); i++) {
+                vectorArray[i] = vector.get(i);
+            }
+        } else {
+            vectorArray = null;
         }
-        return new KnnSearchBuilder((String) args[0], vectorArray, (int) args[2], (int) args[3]);
+        return new KnnSearchBuilder((String) args[0], vectorArray, (QueryVectorBuilder) args[4], (int) args[2], (int) args[3]);
     });
 
     static {
         PARSER.declareString(constructorArg(), FIELD_FIELD);
-        PARSER.declareFloatArray(constructorArg(), QUERY_VECTOR_FIELD);
+        PARSER.declareFloatArray(optionalConstructorArg(), QUERY_VECTOR_FIELD);
         PARSER.declareInt(constructorArg(), K_FIELD);
         PARSER.declareInt(constructorArg(), NUM_CANDS_FIELD);
+        PARSER.declareNamedObject(
+            optionalConstructorArg(),
+            (p, c, n) -> p.namedObject(QueryVectorBuilder.class, n, c),
+            QUERY_VECTOR_BUILDER_FIELD
+        );
         PARSER.declareFieldArray(
             KnnSearchBuilder::addFilterQueries,
             (p, c) -> AbstractQueryBuilder.parseTopLevelQuery(p),
@@ -72,6 +90,8 @@ public static KnnSearchBuilder fromXContent(XContentParser parser) throws IOExce
 
     final String field;
     final float[] queryVector;
+    final QueryVectorBuilder queryVectorBuilder;
+    private final Supplier<float[]> querySupplier;
     final int k;
     final int numCands;
     final List<QueryBuilder> filterQueries;
@@ -86,6 +106,27 @@ public static KnnSearchBuilder fromXContent(XContentParser parser) throws IOExce
      * @param numCands    the number of nearest neighbor candidates to consider per shard
      */
     public KnnSearchBuilder(String field, float[] queryVector, int k, int numCands) {
+        this(field, Objects.requireNonNull(queryVector, format("[%s] cannot be null", QUERY_VECTOR_FIELD)), null, k, numCands);
+    }
+
+    /**
+     * Defines a kNN search where the query vector will be provided by the queryVectorBuilder
+     * @param field              the name of the vector field to search against
+     * @param queryVectorBuilder the query vector builder
+     * @param k                  the final number of nearest neighbors to return as top hits
+     * @param numCands           the number of nearest neighbor candidates to consider per shard
+     */
+    public KnnSearchBuilder(String field, QueryVectorBuilder queryVectorBuilder, int k, int numCands) {
+        this(
+            field,
+            null,
+            Objects.requireNonNull(queryVectorBuilder, format("[%s] cannot be null", QUERY_VECTOR_BUILDER_FIELD.getPreferredName())),
+            k,
+            numCands
+        );
+    }
+
+    private KnnSearchBuilder(String field, float[] queryVector, QueryVectorBuilder queryVectorBuilder, int k, int numCands) {
         if (k < 1) {
             throw new IllegalArgumentException("[" + K_FIELD.getPreferredName() + "] must be greater than 0");
         }
@@ -97,11 +138,41 @@ public KnnSearchBuilder(String field, float[] queryVector, int k, int numCands)
         if (numCands > NUM_CANDS_LIMIT) {
             throw new IllegalArgumentException("[" + NUM_CANDS_FIELD.getPreferredName() + "] cannot exceed [" + NUM_CANDS_LIMIT + "]");
         }
+        if (queryVector == null && queryVectorBuilder == null) {
+            throw new IllegalArgumentException(
+                format(
+                    "either [%s] or [%s] must be provided",
+                    QUERY_VECTOR_BUILDER_FIELD.getPreferredName(),
+                    QUERY_VECTOR_FIELD.getPreferredName()
+                )
+            );
+        }
+        if (queryVector != null && queryVectorBuilder != null) {
+            throw new IllegalArgumentException(
+                format(
+                    "cannot provide both [%s] and [%s]",
+                    QUERY_VECTOR_BUILDER_FIELD.getPreferredName(),
+                    QUERY_VECTOR_FIELD.getPreferredName()
+                )
+            );
+        }
         this.field = field;
-        this.queryVector = queryVector;
+        this.queryVector = queryVector == null ? new float[0] : queryVector;
+        this.queryVectorBuilder = queryVectorBuilder;
         this.k = k;
         this.numCands = numCands;
         this.filterQueries = new ArrayList<>();
+        this.querySupplier = null;
+    }
+
+    private KnnSearchBuilder(String field, Supplier<float[]> querySupplier, int k, int numCands, List<QueryBuilder> filterQueries) {
+        this.field = field;
+        this.queryVector = new float[0];
+        this.queryVectorBuilder = null;
+        this.k = k;
+        this.numCands = numCands;
+        this.filterQueries = filterQueries;
+        this.querySupplier = querySupplier;
     }
 
     public KnnSearchBuilder(StreamInput in) throws IOException {
@@ -111,6 +182,12 @@ public KnnSearchBuilder(StreamInput in) throws IOException {
         this.queryVector = in.readFloatArray();
         this.filterQueries = in.readNamedWriteableList(QueryBuilder.class);
         this.boost = in.readFloat();
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            this.queryVectorBuilder = in.readOptionalNamedWriteable(QueryVectorBuilder.class);
+        } else {
+            this.queryVectorBuilder = null;
+        }
+        this.querySupplier = null;
     }
 
     public int k() {
@@ -139,6 +216,32 @@ public KnnSearchBuilder boost(float boost) {
 
     @Override
     public KnnSearchBuilder rewrite(QueryRewriteContext ctx) throws IOException {
+        if (querySupplier != null) {
+            if (querySupplier.get() == null) {
+                return this;
+            }
+            return new KnnSearchBuilder(field, querySupplier.get(), k, numCands).boost(boost).addFilterQueries(filterQueries);
+        }
+        if (queryVectorBuilder != null) {
+            SetOnce<float[]> toSet = new SetOnce<>();
+            ctx.registerAsyncAction((c, l) -> queryVectorBuilder.buildVector(c, ActionListener.wrap(v -> {
+                toSet.set(v);
+                if (v == null) {
+                    l.onFailure(
+                        new IllegalArgumentException(
+                            format(
+                                "[%s] with name [%s] returned null query_vector",
+                                QUERY_VECTOR_BUILDER_FIELD.getPreferredName(),
+                                queryVectorBuilder.getWriteableName()
+                            )
+                        )
+                    );
+                    return;
+                }
+                l.onResponse(null);
+            }, l::onFailure)));
+            return new KnnSearchBuilder(field, toSet::get, k, numCands, filterQueries).boost(boost);
+        }
         boolean changed = false;
         List<QueryBuilder> rewrittenQueries = new ArrayList<>(filterQueries.size());
         for (QueryBuilder query : filterQueries) {
@@ -155,6 +258,9 @@ public KnnSearchBuilder rewrite(QueryRewriteContext ctx) throws IOException {
     }
 
     public KnnVectorQueryBuilder toQueryBuilder() {
+        if (queryVectorBuilder != null) {
+            throw new IllegalArgumentException("missing rewrite");
+        }
         return new KnnVectorQueryBuilder(field, queryVector, numCands).boost(boost).addFilterQueries(filterQueries);
     }
 
@@ -167,21 +273,38 @@ public boolean equals(Object o) {
             && numCands == that.numCands
             && Objects.equals(field, that.field)
             && Arrays.equals(queryVector, that.queryVector)
+            && Objects.equals(queryVectorBuilder, that.queryVectorBuilder)
+            && Objects.equals(querySupplier, that.querySupplier)
             && Objects.equals(filterQueries, that.filterQueries)
             && boost == that.boost;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(field, k, numCands, Arrays.hashCode(queryVector), Objects.hashCode(filterQueries), boost);
+        return Objects.hash(
+            field,
+            k,
+            numCands,
+            querySupplier,
+            queryVectorBuilder,
+            Arrays.hashCode(queryVector),
+            Objects.hashCode(filterQueries),
+            boost
+        );
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field(FIELD_FIELD.getPreferredName(), field)
             .field(K_FIELD.getPreferredName(), k)
-            .field(NUM_CANDS_FIELD.getPreferredName(), numCands)
-            .array(QUERY_VECTOR_FIELD.getPreferredName(), queryVector);
+            .field(NUM_CANDS_FIELD.getPreferredName(), numCands);
+        if (queryVectorBuilder != null) {
+            builder.startObject(QUERY_VECTOR_BUILDER_FIELD.getPreferredName());
+            builder.field(queryVectorBuilder.getWriteableName(), queryVectorBuilder);
+            builder.endObject();
+        } else {
+            builder.array(QUERY_VECTOR_FIELD.getPreferredName(), queryVector);
+        }
 
         if (filterQueries.isEmpty() == false) {
             builder.startArray(FILTER_FIELD.getPreferredName());
@@ -200,11 +323,26 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
+        if (querySupplier != null) {
+            throw new IllegalStateException("missing a rewriteAndFetch?");
+        }
         out.writeString(field);
         out.writeVInt(k);
         out.writeVInt(numCands);
         out.writeFloatArray(queryVector);
         out.writeNamedWriteableList(filterQueries);
         out.writeFloat(boost);
+        if (out.getTransportVersion().before(TransportVersion.V_8_7_0) && queryVectorBuilder != null) {
+            throw new IllegalArgumentException(
+                format(
+                    "cannot serialize [%s] to older node of version [%s]",
+                    QUERY_VECTOR_BUILDER_FIELD.getPreferredName(),
+                    out.getTransportVersion()
+                )
+            );
+        }
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            out.writeOptionalNamedWriteable(queryVectorBuilder);
+        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java
index 116ef025c15e..511b0da7f135 100644
--- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java
@@ -10,11 +10,11 @@
 
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.KnnVectorQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.DenseVectorFieldType;
@@ -32,7 +32,8 @@
 import java.util.Objects;
 
 /**
- * A query that performs kNN search using Lucene's {@link KnnVectorQuery}.
+ * A query that performs kNN search using Lucene's {@link org.apache.lucene.search.KnnFloatVectorQuery} or
+ * {@link org.apache.lucene.search.KnnByteVectorQuery}.
  *
  * NOTE: this is an internal class and should not be used outside of core Elasticsearch code.
  */
@@ -41,12 +42,22 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder<KnnVectorQueryBu
 
     private final String fieldName;
     private final float[] queryVector;
+    private final byte[] byteQueryVector;
     private final int numCands;
     private final List<QueryBuilder> filterQueries;
 
     public KnnVectorQueryBuilder(String fieldName, float[] queryVector, int numCands) {
         this.fieldName = fieldName;
-        this.queryVector = queryVector;
+        this.queryVector = Objects.requireNonNull(queryVector);
+        this.byteQueryVector = null;
+        this.numCands = numCands;
+        this.filterQueries = new ArrayList<>();
+    }
+
+    public KnnVectorQueryBuilder(String fieldName, byte[] queryVector, int numCands) {
+        this.fieldName = fieldName;
+        this.queryVector = null;
+        this.byteQueryVector = Objects.requireNonNull(queryVector);
         this.numCands = numCands;
         this.filterQueries = new ArrayList<>();
     }
@@ -55,8 +66,14 @@ public KnnVectorQueryBuilder(StreamInput in) throws IOException {
         super(in);
         this.fieldName = in.readString();
         this.numCands = in.readVInt();
-        this.queryVector = in.readFloatArray();
-        if (in.getVersion().before(Version.V_8_2_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_7_0)) {
+            this.queryVector = in.readFloatArray();
+            this.byteQueryVector = null;
+        } else {
+            this.queryVector = in.readBoolean() ? in.readFloatArray() : null;
+            this.byteQueryVector = in.readBoolean() ? in.readByteArray() : null;
+        }
+        if (in.getTransportVersion().before(TransportVersion.V_8_2_0)) {
             this.filterQueries = new ArrayList<>();
         } else {
             this.filterQueries = readQueries(in);
@@ -67,10 +84,16 @@ public String getFieldName() {
         return fieldName;
     }
 
+    @Nullable
     public float[] queryVector() {
         return queryVector;
     }
 
+    @Nullable
+    public byte[] getByteQueryVector() {
+        return byteQueryVector;
+    }
+
     public int numCands() {
         return numCands;
     }
@@ -95,15 +118,40 @@ public KnnVectorQueryBuilder addFilterQueries(List<QueryBuilder> filterQueries)
     protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeString(fieldName);
         out.writeVInt(numCands);
-        out.writeFloatArray(queryVector);
-        if (out.getVersion().onOrAfter(Version.V_8_2_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            boolean queryVectorNotNull = queryVector != null;
+            out.writeBoolean(queryVectorNotNull);
+            if (queryVectorNotNull) {
+                out.writeFloatArray(queryVector);
+            }
+            boolean byteVectorNotNull = byteQueryVector != null;
+            out.writeBoolean(byteVectorNotNull);
+            if (byteVectorNotNull) {
+                out.writeByteArray(byteQueryVector);
+            }
+        } else {
+            final float[] f;
+            if (queryVector != null) {
+                f = queryVector;
+            } else {
+                f = new float[byteQueryVector.length];
+                for (int i = 0; i < byteQueryVector.length; i++) {
+                    f[i] = byteQueryVector[i];
+                }
+            }
+            out.writeFloatArray(f);
+        }
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
             writeQueries(out, filterQueries);
         }
     }
 
     @Override
     protected void doXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(NAME).field("field", fieldName).field("vector", queryVector).field("num_candidates", numCands);
+        builder.startObject(NAME)
+            .field("field", fieldName)
+            .field("vector", queryVector != null ? queryVector : byteQueryVector)
+            .field("num_candidates", numCands);
         if (filterQueries.isEmpty() == false) {
             builder.startArray("filters");
             for (QueryBuilder filterQuery : filterQueries) {
@@ -135,7 +183,9 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws
             rewrittenQueries.add(rewrittenQuery);
         }
         if (changed) {
-            return new KnnVectorQueryBuilder(fieldName, queryVector, numCands).addFilterQueries(rewrittenQueries);
+            return byteQueryVector != null
+                ? new KnnVectorQueryBuilder(fieldName, byteQueryVector, numCands).addFilterQueries(rewrittenQueries)
+                : new KnnVectorQueryBuilder(fieldName, queryVector, numCands).addFilterQueries(rewrittenQueries);
         }
         return this;
     }
@@ -161,24 +211,27 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException {
         Query filterQuery = booleanQuery.clauses().isEmpty() ? null : booleanQuery;
 
         DenseVectorFieldType vectorFieldType = (DenseVectorFieldType) fieldType;
-        return vectorFieldType.createKnnQuery(queryVector, numCands, filterQuery);
+        return queryVector != null
+            ? vectorFieldType.createKnnQuery(queryVector, numCands, filterQuery)
+            : vectorFieldType.createKnnQuery(byteQueryVector, numCands, filterQuery);
     }
 
     @Override
     protected int doHashCode() {
-        return Objects.hash(fieldName, Arrays.hashCode(queryVector), numCands, filterQueries);
+        return Objects.hash(fieldName, Arrays.hashCode(queryVector), Arrays.hashCode(byteQueryVector), numCands, filterQueries);
     }
 
     @Override
     protected boolean doEquals(KnnVectorQueryBuilder other) {
         return Objects.equals(fieldName, other.fieldName)
             && Arrays.equals(queryVector, other.queryVector)
+            && Arrays.equals(byteQueryVector, other.byteQueryVector)
             && numCands == other.numCands
             && Objects.equals(filterQueries, other.filterQueries);
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_0_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_0_0;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/vectors/QueryVectorBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/QueryVectorBuilder.java
new file mode 100644
index 000000000000..aa35657c6054
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/vectors/QueryVectorBuilder.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.vectors;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.common.io.stream.VersionedNamedWriteable;
+import org.elasticsearch.xcontent.ToXContentObject;
+
+/**
+ * Provides a mechanism for building a KNN query vector in an asynchronous manner during the rewrite phase
+ */
+public interface QueryVectorBuilder extends VersionedNamedWriteable, ToXContentObject {
+
+    /**
+     * Method for building a vector via the client. This method is called during RerwiteAndFetch.
+     * Typical implementation for this method will:
+     *  1. call some asynchronous client action
+     *  2. Handle failure/success for that action (usually passing failure to the provided listener)
+     *  3. Parse the success case and extract the query vector
+     *  4. Pass the extracted query vector to the provided listener
+     *
+     * @param client for performing asynchronous actions against the cluster
+     * @param listener listener to accept the created vector
+     */
+    void buildVector(Client client, ActionListener<float[]> listener);
+
+}
diff --git a/server/src/main/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorService.java b/server/src/main/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorService.java
index 8ce0500c8225..59d84f396e02 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorService.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorService.java
@@ -72,7 +72,7 @@ public String name() {
     }
 
     @Override
-    public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
+    public HealthIndicatorResult calculate(boolean verbose, int maxAffectedResourcesCount, HealthInfo healthInfo) {
         var snapshotMetadata = clusterService.state().metadata().custom(RepositoriesMetadata.TYPE, RepositoriesMetadata.EMPTY);
 
         if (snapshotMetadata.repositories().isEmpty()) {
@@ -133,7 +133,12 @@ public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
                 )
                 : HealthIndicatorDetails.EMPTY,
             impacts,
-            List.of(new Diagnosis(CORRUPTED_REPOSITORY, List.of(new Diagnosis.Resource(Type.SNAPSHOT_REPOSITORY, corrupted))))
+            List.of(
+                new Diagnosis(
+                    CORRUPTED_REPOSITORY,
+                    List.of(new Diagnosis.Resource(Type.SNAPSHOT_REPOSITORY, limitSize(corrupted, maxAffectedResourcesCount)))
+                )
+            )
         );
     }
 
diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
index dae877937fc9..f9c8999ca0e6 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.StepListener;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
-import org.elasticsearch.action.support.GroupedActionListener;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateApplier;
@@ -124,8 +124,7 @@
  * First {@link #restoreSnapshot(RestoreSnapshotRequest, org.elasticsearch.action.ActionListener)}
  * method reads information about snapshot and metadata from repository. In update cluster state task it checks restore
  * preconditions, restores global state if needed, creates {@link RestoreInProgress} record with list of shards that needs
- * to be restored and adds this shard to the routing table using
- * {@link RoutingTable.Builder#addAsRestore(IndexMetadata, SnapshotRecoverySource)} method.
+ * to be restored and adds this shard to the routing table using {@link RoutingTable.Builder#addAsRestore} method.
  * <p>
  * Individual shards are getting restored as part of normal recovery process in
  * {@link IndexShard#restoreFromRepository} )}
@@ -250,8 +249,8 @@ public void restoreSnapshot(
     ) {
         try {
             // Try and fill in any missing repository UUIDs in case they're needed during the restore
-            final StepListener<Void> repositoryUuidRefreshListener = new StepListener<>();
-            refreshRepositoryUuids(refreshRepositoryUuidOnRestore, repositoriesService, repositoryUuidRefreshListener);
+            final var repositoryUuidRefreshStep = new StepListener<Void>();
+            refreshRepositoryUuids(refreshRepositoryUuidOnRestore, repositoriesService, () -> repositoryUuidRefreshStep.onResponse(null));
 
             // Read snapshot info and metadata from the repository
             final String repositoryName = request.repository();
@@ -259,7 +258,7 @@ public void restoreSnapshot(
             final StepListener<RepositoryData> repositoryDataListener = new StepListener<>();
             repository.getRepositoryData(repositoryDataListener);
 
-            repositoryDataListener.whenComplete(repositoryData -> repositoryUuidRefreshListener.whenComplete(ignored -> {
+            repositoryDataListener.whenComplete(repositoryData -> repositoryUuidRefreshStep.whenComplete(ignored -> {
                 final String snapshotName = request.snapshot();
                 final Optional<SnapshotId> matchingSnapshotId = repositoryData.getSnapshotIds()
                     .stream()
@@ -493,56 +492,37 @@ private void setRefreshRepositoryUuidOnRestore(boolean refreshRepositoryUuidOnRe
      *
      * @param enabled If {@code false} this method completes the listener immediately
      * @param repositoriesService Supplies the repositories to check
-     * @param refreshListener Listener that is completed when all repositories have been refreshed.
+     * @param onCompletion Action that is executed when all repositories have been refreshed.
      */
     // Exposed for tests
-    static void refreshRepositoryUuids(boolean enabled, RepositoriesService repositoriesService, ActionListener<Void> refreshListener) {
-
-        if (enabled == false) {
-            logger.debug("repository UUID refresh is disabled");
-            refreshListener.onResponse(null);
-            return;
-        }
-
-        // We only care about BlobStoreRepositories because they're the only ones that can contain a searchable snapshot, and we only care
-        // about ones with missing UUIDs. It's possible to have the UUID change from under us if, e.g., the repository was wiped by an
-        // external force, but in this case any searchable snapshots are lost anyway so it doesn't really matter.
-        final List<Repository> repositories = repositoriesService.getRepositories()
-            .values()
-            .stream()
-            .filter(
-                repository -> repository instanceof BlobStoreRepository
-                    && repository.getMetadata().uuid().equals(RepositoryData.MISSING_UUID)
-            )
-            .toList();
-        if (repositories.isEmpty()) {
-            logger.debug("repository UUID refresh is not required");
-            refreshListener.onResponse(null);
-            return;
-        }
-
-        logger.info(
-            "refreshing repository UUIDs for repositories [{}]",
-            repositories.stream().map(repository -> repository.getMetadata().name()).collect(Collectors.joining(","))
-        );
-        final ActionListener<RepositoryData> groupListener = new GroupedActionListener<>(new ActionListener<Collection<Void>>() {
-            @Override
-            public void onResponse(Collection<Void> ignored) {
-                logger.debug("repository UUID refresh completed");
-                refreshListener.onResponse(null);
+    static void refreshRepositoryUuids(boolean enabled, RepositoriesService repositoriesService, Runnable onCompletion) {
+        try (var refs = new RefCountingRunnable(onCompletion)) {
+            if (enabled == false) {
+                logger.debug("repository UUID refresh is disabled");
+                return;
             }
 
-            @Override
-            public void onFailure(Exception e) {
-                logger.debug("repository UUID refresh failed", e);
-                refreshListener.onResponse(null); // this refresh is best-effort, the restore should proceed either way
-            }
-        }, repositories.size()).map(repositoryData -> null /* don't collect the RepositoryData */);
+            for (Repository repository : repositoriesService.getRepositories().values()) {
+                // We only care about BlobStoreRepositories because they're the only ones that can contain a searchable snapshot, and we
+                // only care about ones with missing UUIDs. It's possible to have the UUID change from under us if, e.g., the repository was
+                // wiped by an external force, but in this case any searchable snapshots are lost anyway so it doesn't really matter.
+                if (repository instanceof BlobStoreRepository && repository.getMetadata().uuid().equals(RepositoryData.MISSING_UUID)) {
+                    final var repositoryName = repository.getMetadata().name();
+                    logger.info("refreshing repository UUID for repository [{}]", repositoryName);
+                    repository.getRepositoryData(ActionListener.releaseAfter(new ActionListener<>() {
+                        @Override
+                        public void onResponse(RepositoryData repositoryData) {
+                            logger.debug(() -> format("repository UUID [{}] refresh completed", repositoryName));
+                        }
 
-        for (Repository repository : repositories) {
-            repository.getRepositoryData(groupListener);
+                        @Override
+                        public void onFailure(Exception e) {
+                            logger.debug(() -> format("repository UUID [{}] refresh failed", repositoryName), e);
+                        }
+                    }, refs.acquire()));
+                }
+            }
         }
-
     }
 
     private boolean isSystemIndex(IndexMetadata indexMetadata) {
@@ -1279,7 +1259,10 @@ public ClusterState execute(ClusterState currentState) {
             // Updating cluster state
             final Metadata.Builder mdBuilder = Metadata.builder(currentState.metadata());
             final ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
-            final RoutingTable.Builder rtBuilder = RoutingTable.builder(currentState.routingTable());
+            final RoutingTable.Builder rtBuilder = RoutingTable.builder(
+                allocationService.getShardRoutingRoleStrategy(),
+                currentState.routingTable()
+            );
 
             final Map<ShardId, ShardRestoreStatus> shards = new HashMap<>();
 
diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
index f5372b359e89..0742036d81df 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
@@ -497,7 +497,7 @@ public SnapshotInfo withoutIndices() {
      */
     public static SnapshotInfo readFrom(final StreamInput in) throws IOException {
         final Snapshot snapshot;
-        if (in.getVersion().onOrAfter(GetSnapshotsRequest.PAGINATED_GET_SNAPSHOTS_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(GetSnapshotsRequest.PAGINATED_GET_SNAPSHOTS_VERSION)) {
             snapshot = new Snapshot(in);
         } else {
             snapshot = new Snapshot(UNKNOWN_REPO_NAME, new SnapshotId(in));
@@ -1018,7 +1018,7 @@ public static SnapshotInfo fromXContentInternal(final String repoName, final XCo
 
     @Override
     public void writeTo(final StreamOutput out) throws IOException {
-        if (out.getVersion().onOrAfter(GetSnapshotsRequest.PAGINATED_GET_SNAPSHOTS_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(GetSnapshotsRequest.PAGINATED_GET_SNAPSHOTS_VERSION)) {
             snapshot.writeTo(out);
         } else {
             snapshot.getSnapshotId().writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
index 8357e3467336..d6b5ffd5f42d 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
@@ -237,15 +237,7 @@ private void startNewSnapshots(List<SnapshotsInProgress.Entry> snapshotsInProgre
                                 + snapshotStatus.generation()
                                 + "] for snapshot with old-format compatibility";
                         shardSnapshotTasks.add(
-                            newShardSnapshotTask(
-                                shardId,
-                                snapshot,
-                                indexId,
-                                entry.userMetadata(),
-                                snapshotStatus,
-                                entry.version(),
-                                entry.startTime()
-                            )
+                            newShardSnapshotTask(shardId, snapshot, indexId, snapshotStatus, entry.version(), entry.startTime())
                         );
                     }
 
@@ -276,55 +268,45 @@ private Runnable newShardSnapshotTask(
         final ShardId shardId,
         final Snapshot snapshot,
         final IndexId indexId,
-        final Map<String, Object> userMetadata,
         final IndexShardSnapshotStatus snapshotStatus,
         final Version entryVersion,
         final long entryStartTime
     ) {
         // separate method to make sure this lambda doesn't capture any heavy local objects like a SnapshotsInProgress.Entry
-        return () -> snapshot(
-            shardId,
-            snapshot,
-            indexId,
-            userMetadata,
-            snapshotStatus,
-            entryVersion,
-            entryStartTime,
-            new ActionListener<>() {
-                @Override
-                public void onResponse(ShardSnapshotResult shardSnapshotResult) {
-                    final ShardGeneration newGeneration = shardSnapshotResult.getGeneration();
-                    assert newGeneration != null;
-                    assert newGeneration.equals(snapshotStatus.generation());
-                    if (logger.isDebugEnabled()) {
-                        final IndexShardSnapshotStatus.Copy lastSnapshotStatus = snapshotStatus.asCopy();
-                        logger.debug(
-                            "[{}][{}] completed snapshot to [{}] with status [{}] at generation [{}]",
-                            shardId,
-                            snapshot,
-                            snapshot.getRepository(),
-                            lastSnapshotStatus,
-                            snapshotStatus.generation()
-                        );
-                    }
-                    notifySuccessfulSnapshotShard(snapshot, shardId, shardSnapshotResult);
+        return () -> snapshot(shardId, snapshot, indexId, snapshotStatus, entryVersion, entryStartTime, new ActionListener<>() {
+            @Override
+            public void onResponse(ShardSnapshotResult shardSnapshotResult) {
+                final ShardGeneration newGeneration = shardSnapshotResult.getGeneration();
+                assert newGeneration != null;
+                assert newGeneration.equals(snapshotStatus.generation());
+                if (logger.isDebugEnabled()) {
+                    final IndexShardSnapshotStatus.Copy lastSnapshotStatus = snapshotStatus.asCopy();
+                    logger.debug(
+                        "[{}][{}] completed snapshot to [{}] with status [{}] at generation [{}]",
+                        shardId,
+                        snapshot,
+                        snapshot.getRepository(),
+                        lastSnapshotStatus,
+                        snapshotStatus.generation()
+                    );
                 }
+                notifySuccessfulSnapshotShard(snapshot, shardId, shardSnapshotResult);
+            }
 
-                @Override
-                public void onFailure(Exception e) {
-                    final String failure;
-                    if (e instanceof AbortedSnapshotException) {
-                        failure = "aborted";
-                        logger.debug(() -> format("[%s][%s] aborted shard snapshot", shardId, snapshot), e);
-                    } else {
-                        failure = summarizeFailure(e);
-                        logger.warn(() -> format("[%s][%s] failed to snapshot shard", shardId, snapshot), e);
-                    }
-                    snapshotStatus.moveToFailed(threadPool.absoluteTimeInMillis(), failure);
-                    notifyFailedSnapshotShard(snapshot, shardId, failure, snapshotStatus.generation());
+            @Override
+            public void onFailure(Exception e) {
+                final String failure;
+                if (e instanceof AbortedSnapshotException) {
+                    failure = "aborted";
+                    logger.debug(() -> format("[%s][%s] aborted shard snapshot", shardId, snapshot), e);
+                } else {
+                    failure = summarizeFailure(e);
+                    logger.warn(() -> format("[%s][%s] failed to snapshot shard", shardId, snapshot), e);
                 }
+                snapshotStatus.moveToFailed(threadPool.absoluteTimeInMillis(), failure);
+                notifyFailedSnapshotShard(snapshot, shardId, failure, snapshotStatus.generation());
             }
-        );
+        });
     }
 
     // package private for testing
@@ -359,13 +341,12 @@ private void snapshot(
         final ShardId shardId,
         final Snapshot snapshot,
         final IndexId indexId,
-        final Map<String, Object> userMetadata,
         final IndexShardSnapshotStatus snapshotStatus,
         Version version,
         final long entryStartTime,
-        ActionListener<ShardSnapshotResult> listener
+        ActionListener<ShardSnapshotResult> resultListener
     ) {
-        try {
+        ActionListener.run(resultListener, listener -> {
             if (snapshotStatus.isAborted()) {
                 throw new AbortedSnapshotException();
             }
@@ -398,7 +379,6 @@ private void snapshot(
                         getShardStateId(indexShard, snapshotRef.getIndexCommit()),
                         snapshotStatus,
                         version,
-                        userMetadata,
                         entryStartTime,
                         listener
                     )
@@ -407,9 +387,7 @@ private void snapshot(
                 IOUtils.close(snapshotRef);
                 throw e;
             }
-        } catch (Exception e) {
-            listener.onFailure(e);
-        }
+        });
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
index ddc7d8004595..e58949b8409a 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
@@ -292,7 +292,6 @@ public void createSnapshot(final CreateSnapshotRequest request, final ActionList
             featureStatesSet = Collections.emptySet();
         }
 
-        final Map<String, Object> userMeta = repository.adaptUserMetadata(request.userMetadata());
         repository.executeConsistentStateUpdate(repositoryData -> new ClusterStateUpdateTask(request.masterNodeTimeout()) {
 
             private SnapshotsInProgress.Entry newEntry;
@@ -412,7 +411,7 @@ public ClusterState execute(ClusterState currentState) {
                     threadPool.absoluteTimeInMillis(),
                     repositoryData.getGenId(),
                     shards,
-                    userMeta,
+                    request.userMetadata(),
                     version,
                     List.copyOf(featureStates)
                 );
@@ -602,8 +601,8 @@ private void startCloning(Repository repository, SnapshotsInProgress.Entry clone
 
         final StepListener<Collection<Tuple<IndexId, Integer>>> allShardCountsListener = new StepListener<>();
         final GroupedActionListener<Tuple<IndexId, Integer>> shardCountListener = new GroupedActionListener<>(
-            allShardCountsListener,
-            indices.size()
+            indices.size(),
+            allShardCountsListener
         );
         snapshotInfoListener.whenComplete(snapshotInfo -> {
             for (IndexId indexId : indices) {
diff --git a/server/src/main/java/org/elasticsearch/tasks/RemovedTaskListener.java b/server/src/main/java/org/elasticsearch/tasks/RemovedTaskListener.java
new file mode 100644
index 000000000000..494718533eeb
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/tasks/RemovedTaskListener.java
@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.tasks;
+
+@FunctionalInterface
+public interface RemovedTaskListener {
+
+    void onRemoved(Task task);
+}
diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java b/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java
index b6742815f44a..122a25a61391 100644
--- a/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java
+++ b/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java
@@ -12,12 +12,14 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ResultDeduplicator;
 import org.elasticsearch.action.StepListener;
 import org.elasticsearch.action.support.ChannelActionListener;
+import org.elasticsearch.action.support.CountDownActionListener;
 import org.elasticsearch.action.support.GroupedActionListener;
+import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -99,30 +101,42 @@ void doCancelTaskAndDescendants(CancellableTask task, String reason, boolean wai
         if (task.shouldCancelChildrenOnCancellation()) {
             logger.trace("cancelling task [{}] and its descendants", taskId);
             StepListener<Void> completedListener = new StepListener<>();
-            GroupedActionListener<Void> groupedListener = new GroupedActionListener<>(completedListener.map(r -> null), 3);
-            Collection<Transport.Connection> childConnections = taskManager.startBanOnChildTasks(task.getId(), reason, () -> {
-                logger.trace("child tasks of parent [{}] are completed", taskId);
-                groupedListener.onResponse(null);
-            });
-            taskManager.cancel(task, reason, () -> {
-                logger.trace("task [{}] is cancelled", taskId);
-                groupedListener.onResponse(null);
-            });
             StepListener<Void> setBanListener = new StepListener<>();
+
+            Collection<Transport.Connection> childConnections;
+            try (var refs = new RefCountingRunnable(() -> setBanListener.addListener(completedListener))) {
+                var banChildrenRef = refs.acquire();
+                var cancelTaskRef = refs.acquire();
+
+                childConnections = taskManager.startBanOnChildTasks(task.getId(), reason, () -> {
+                    logger.trace("child tasks of parent [{}] are completed", taskId);
+                    banChildrenRef.close();
+                });
+
+                taskManager.cancel(task, reason, () -> {
+                    logger.trace("task [{}] is cancelled", taskId);
+                    cancelTaskRef.close();
+                });
+            }
             setBanOnChildConnections(reason, waitForCompletion, task, childConnections, setBanListener);
-            setBanListener.addListener(groupedListener);
-            // If we start unbanning when the last child task completed and that child task executed with a specific user, then unban
-            // requests are denied because internal requests can't run with a user. We need to remove bans with the current thread context.
-            final Runnable removeBansRunnable = transportService.getThreadPool()
-                .getThreadContext()
-                .preserveContext(() -> removeBanOnChildConnections(task, childConnections));
+
             // We remove bans after all child tasks are completed although in theory we can do it on a per-connection basis.
-            completedListener.whenComplete(r -> removeBansRunnable.run(), e -> removeBansRunnable.run());
-            // if wait_for_completion is true, then only return when (1) bans are placed on child connections, (2) child tasks are
-            // completed or failed, (3) the main task is cancelled. Otherwise, return after bans are placed on child connections.
+            completedListener.addListener(
+                ActionListener.wrap(
+                    transportService.getThreadPool()
+                        .getThreadContext()
+                        // If we start unbanning when the last child task completed and that child task executed with a specific user, then
+                        // unban requests are denied because internal requests can't run with a user. We need to remove bans with the
+                        // current thread context.
+                        .preserveContext(() -> removeBanOnChildConnections(task, childConnections))
+                )
+            );
+
             if (waitForCompletion) {
+                // Wait until (1) bans are placed on child connections, (2) child tasks are completed or failed, (3) main task is cancelled.
                 completedListener.addListener(listener);
             } else {
+                // Only wait until bans are placed on child connections
                 setBanListener.addListener(listener);
             }
         } else {
@@ -149,7 +163,7 @@ private void setBanOnChildConnections(
         }
         final TaskId taskId = new TaskId(localNodeId(), task.getId());
         logger.trace("cancelling child tasks of [{}] on child connections {}", taskId, childConnections);
-        GroupedActionListener<Void> groupedListener = new GroupedActionListener<>(listener.map(r -> null), childConnections.size());
+        CountDownActionListener countDownListener = new CountDownActionListener(childConnections.size(), listener);
         final BanParentTaskRequest banRequest = BanParentTaskRequest.createSetBanParentTaskRequest(taskId, reason, waitForCompletion);
         for (Transport.Connection connection : childConnections) {
             assert TransportService.unwrapConnection(connection) == connection : "Child connection must be unwrapped";
@@ -162,7 +176,7 @@ private void setBanOnChildConnections(
                     @Override
                     public void handleResponse(TransportResponse.Empty response) {
                         logger.trace("sent ban for tasks with the parent [{}] for connection [{}]", taskId, connection);
-                        groupedListener.onResponse(null);
+                        countDownListener.onResponse(null);
                     }
 
                     @Override
@@ -188,7 +202,7 @@ public void handleException(TransportException exp) {
                             );
                         }
 
-                        groupedListener.onFailure(exp);
+                        countDownListener.onFailure(exp);
                     }
                 }
             );
@@ -280,7 +294,7 @@ private BanParentTaskRequest(StreamInput in) throws IOException {
             parentTaskId = TaskId.readFromStream(in);
             ban = in.readBoolean();
             reason = ban ? in.readString() : null;
-            if (in.getVersion().onOrAfter(Version.V_7_8_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_8_0)) {
                 waitForCompletion = in.readBoolean();
             } else {
                 waitForCompletion = false;
@@ -295,7 +309,7 @@ public void writeTo(StreamOutput out) throws IOException {
             if (ban) {
                 out.writeString(reason);
             }
-            if (out.getVersion().onOrAfter(Version.V_7_8_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_8_0)) {
                 out.writeBoolean(waitForCompletion);
             }
         }
@@ -313,8 +327,8 @@ public void messageReceived(final BanParentTaskRequest request, final TransportC
                 );
                 final List<CancellableTask> childTasks = taskManager.setBan(request.parentTaskId, request.reason, channel);
                 final GroupedActionListener<Void> listener = new GroupedActionListener<>(
-                    new ChannelActionListener<>(channel, BAN_PARENT_ACTION_NAME, request).map(r -> TransportResponse.Empty.INSTANCE),
-                    childTasks.size() + 1
+                    childTasks.size() + 1,
+                    new ChannelActionListener<>(channel, BAN_PARENT_ACTION_NAME, request).map(r -> TransportResponse.Empty.INSTANCE)
                 );
                 for (CancellableTask childTask : childTasks) {
                     cancelTaskAndDescendants(childTask, request.reason, request.waitForCompletion, listener);
diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskManager.java b/server/src/main/java/org/elasticsearch/tasks/TaskManager.java
index e4cec96f73ab..8fc38e520d0d 100644
--- a/server/src/main/java/org/elasticsearch/tasks/TaskManager.java
+++ b/server/src/main/java/org/elasticsearch/tasks/TaskManager.java
@@ -12,8 +12,6 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.Assertions;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
@@ -53,6 +51,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
@@ -95,6 +94,8 @@ public class TaskManager implements ClusterStateApplier {
     private final Map<TcpChannel, ChannelPendingTaskTracker> channelPendingTaskTrackers = ConcurrentCollections.newConcurrentMap();
     private final SetOnce<TaskCancellationService> cancellationService = new SetOnce<>();
 
+    private final List<RemovedTaskListener> removedTaskListeners = new CopyOnWriteArrayList<>();
+
     // For testing
     public TaskManager(Settings settings, ThreadPool threadPool, Set<String> taskHeaders) {
         this(settings, threadPool, taskHeaders, Tracer.NOOP);
@@ -292,9 +293,20 @@ public Task unregister(Task task) {
             }
         } finally {
             tracer.stopTrace("task-" + task.getId());
+            for (RemovedTaskListener listener : removedTaskListeners) {
+                listener.onRemoved(task);
+            }
         }
     }
 
+    public void registerRemovedTaskListener(RemovedTaskListener removedTaskListener) {
+        removedTaskListeners.add(removedTaskListener);
+    }
+
+    public void unregisterRemovedTaskListener(RemovedTaskListener removedTaskListener) {
+        removedTaskListeners.remove(removedTaskListener);
+    }
+
     /**
      * Register a connection on which a child task will execute on the target connection. The returned {@link Releasable} must be called
      * to unregister the child connection once the child task is completed or failed.
@@ -533,23 +545,6 @@ public void applyClusterState(ClusterChangedEvent event) {
         lastDiscoveryNodes = event.state().getNodes();
     }
 
-    /**
-     * Blocks the calling thread, waiting for the task to vanish from the TaskManager.
-     */
-    public void waitForTaskCompletion(Task task, long untilInNanos) {
-        while (System.nanoTime() - untilInNanos < 0) {
-            if (getTask(task.getId()) == null) {
-                return;
-            }
-            try {
-                Thread.sleep(WAIT_FOR_COMPLETION_POLL.millis());
-            } catch (InterruptedException e) {
-                throw new ElasticsearchException("Interrupted waiting for completion of [{}]", e, task);
-            }
-        }
-        throw new ElasticsearchTimeoutException("Timed out waiting for completion of [{}]", task);
-    }
-
     private static class CancellableTaskHolder {
         private final CancellableTask task;
         private boolean finished = false;
diff --git a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
index ce501ce3b00d..3256773196dc 100644
--- a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
+++ b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
@@ -15,6 +15,8 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.SizeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
@@ -187,9 +189,12 @@ public ThreadPool(final Settings settings, final ExecutorBuilder<?>... customBui
             new ScalingExecutorBuilder(Names.GENERIC, 4, genericThreadPoolMax, TimeValue.timeValueSeconds(30), false)
         );
         builders.put(Names.WRITE, new FixedExecutorBuilder(settings, Names.WRITE, allocatedProcessors, 10000, false));
-        builders.put(Names.GET, new FixedExecutorBuilder(settings, Names.GET, allocatedProcessors, 1000, false));
+        builders.put(Names.GET, new FixedExecutorBuilder(settings, Names.GET, searchOrGetThreadPoolSize(allocatedProcessors), 1000, false));
         builders.put(Names.ANALYZE, new FixedExecutorBuilder(settings, Names.ANALYZE, 1, 16, false));
-        builders.put(Names.SEARCH, new FixedExecutorBuilder(settings, Names.SEARCH, searchThreadPoolSize(allocatedProcessors), 1000, true));
+        builders.put(
+            Names.SEARCH,
+            new FixedExecutorBuilder(settings, Names.SEARCH, searchOrGetThreadPoolSize(allocatedProcessors), 1000, true)
+        );
         builders.put(Names.SEARCH_COORDINATION, new FixedExecutorBuilder(settings, Names.SEARCH_COORDINATION, halfProcMaxAt5, 1000, true));
         builders.put(
             Names.AUTO_COMPLETE,
@@ -198,12 +203,13 @@ public ThreadPool(final Settings settings, final ExecutorBuilder<?>... customBui
         builders.put(Names.SEARCH_THROTTLED, new FixedExecutorBuilder(settings, Names.SEARCH_THROTTLED, 1, 100, true));
         builders.put(
             Names.MANAGEMENT,
-            new ScalingExecutorBuilder(Names.MANAGEMENT, 2, boundedBy(allocatedProcessors, 2, 5), TimeValue.timeValueMinutes(5), false)
+            new ScalingExecutorBuilder(Names.MANAGEMENT, 1, boundedBy(allocatedProcessors, 1, 5), TimeValue.timeValueMinutes(5), false)
         );
         builders.put(Names.FLUSH, new ScalingExecutorBuilder(Names.FLUSH, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5), false));
         builders.put(Names.REFRESH, new ScalingExecutorBuilder(Names.REFRESH, 1, halfProcMaxAt10, TimeValue.timeValueMinutes(5), false));
         builders.put(Names.WARMER, new ScalingExecutorBuilder(Names.WARMER, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5), false));
-        builders.put(Names.SNAPSHOT, new ScalingExecutorBuilder(Names.SNAPSHOT, 1, 10, TimeValue.timeValueMinutes(5), false));
+        final int maxSnapshotCores = getMaxSnapshotThreadPoolSize(allocatedProcessors);
+        builders.put(Names.SNAPSHOT, new ScalingExecutorBuilder(Names.SNAPSHOT, 1, maxSnapshotCores, TimeValue.timeValueMinutes(5), false));
         builders.put(
             Names.SNAPSHOT_META,
             new ScalingExecutorBuilder(
@@ -557,10 +563,24 @@ static int oneEighthAllocatedProcessors(final int allocatedProcessors) {
         return boundedBy(allocatedProcessors / 8, 1, Integer.MAX_VALUE);
     }
 
-    public static int searchThreadPoolSize(final int allocatedProcessors) {
+    public static int searchOrGetThreadPoolSize(final int allocatedProcessors) {
         return ((allocatedProcessors * 3) / 2) + 1;
     }
 
+    static int getMaxSnapshotThreadPoolSize(int allocatedProcessors) {
+        final ByteSizeValue maxHeapSize = ByteSizeValue.ofBytes(Runtime.getRuntime().maxMemory());
+        return getMaxSnapshotThreadPoolSize(allocatedProcessors, maxHeapSize);
+    }
+
+    static int getMaxSnapshotThreadPoolSize(int allocatedProcessors, final ByteSizeValue maxHeapSize) {
+        // While on larger data nodes, larger snapshot threadpool size improves snapshotting on high latency blob stores,
+        // smaller instances can run into OOM issues and need a smaller snapshot threadpool size.
+        if (maxHeapSize.compareTo(new ByteSizeValue(750, ByteSizeUnit.MB)) < 0) {
+            return halfAllocatedProcessorsMaxFive(allocatedProcessors);
+        }
+        return 10;
+    }
+
     static class ThreadedRunnable implements Runnable {
 
         private final Runnable runnable;
diff --git a/server/src/main/java/org/elasticsearch/transport/ActionTransportException.java b/server/src/main/java/org/elasticsearch/transport/ActionTransportException.java
index 8f6f0fd477b1..444a34e4bcc3 100644
--- a/server/src/main/java/org/elasticsearch/transport/ActionTransportException.java
+++ b/server/src/main/java/org/elasticsearch/transport/ActionTransportException.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.transport;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.network.NetworkAddress;
@@ -24,7 +24,7 @@ public class ActionTransportException extends TransportException {
 
     public ActionTransportException(StreamInput in) throws IOException {
         super(in);
-        if (in.getVersion().before(Version.V_8_1_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_1_0)) {
             in.readOptionalWriteable(TransportAddress::new);
             in.readOptionalString();
         }
@@ -45,7 +45,7 @@ public ActionTransportException(String name, InetSocketAddress address, String a
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        if (out.getVersion().before(Version.V_8_1_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_1_0)) {
             out.writeMissingWriteable(TransportAddress.class);
             out.writeMissingString(); // action
         }
diff --git a/server/src/main/java/org/elasticsearch/transport/BytesTransportRequest.java b/server/src/main/java/org/elasticsearch/transport/BytesTransportRequest.java
index 1f05adbac033..51aca540f7aa 100644
--- a/server/src/main/java/org/elasticsearch/transport/BytesTransportRequest.java
+++ b/server/src/main/java/org/elasticsearch/transport/BytesTransportRequest.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.transport;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.bytes.ReleasableBytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -24,20 +24,20 @@
 public class BytesTransportRequest extends TransportRequest implements RefCounted {
 
     final ReleasableBytesReference bytes;
-    private final Version version;
+    private final TransportVersion version;
 
     public BytesTransportRequest(StreamInput in) throws IOException {
         super(in);
         bytes = in.readReleasableBytesReference();
-        version = in.getVersion();
+        version = in.getTransportVersion();
     }
 
-    public BytesTransportRequest(ReleasableBytesReference bytes, Version version) {
+    public BytesTransportRequest(ReleasableBytesReference bytes, TransportVersion version) {
         this.bytes = bytes;
         this.version = version;
     }
 
-    public Version version() {
+    public TransportVersion version() {
         return this.version;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/transport/ClusterConnectionManager.java b/server/src/main/java/org/elasticsearch/transport/ClusterConnectionManager.java
index 89a26ff8524f..6c500829d520 100644
--- a/server/src/main/java/org/elasticsearch/transport/ClusterConnectionManager.java
+++ b/server/src/main/java/org/elasticsearch/transport/ClusterConnectionManager.java
@@ -75,7 +75,7 @@ public void removeListener(TransportConnectionListener listener) {
     @Override
     public void openConnection(DiscoveryNode node, ConnectionProfile connectionProfile, ActionListener<Transport.Connection> listener) {
         ConnectionProfile resolvedProfile = ConnectionProfile.resolveConnectionProfile(connectionProfile, defaultProfile);
-        if (connectingRefCounter.tryIncRef()) {
+        if (acquireConnectingRef()) {
             var success = false;
             final var release = new RunOnce(connectingRefCounter::decRef);
             try {
@@ -136,7 +136,7 @@ private void connectToNodeOrRetry(
             return;
         }
 
-        if (connectingRefCounter.tryIncRef() == false) {
+        if (acquireConnectingRef() == false) {
             listener.onFailure(new ConnectTransportException(node, "connection manager is closed"));
             return;
         }
@@ -215,6 +215,7 @@ private void connectToNodeOrRetry(
             ActionListener.wrap(
                 conn -> connectionValidator.validate(conn, resolvedProfile, ActionListener.runAfter(ActionListener.wrap(ignored -> {
                     assert Transports.assertNotTransportThread("connection validator success");
+                    final var managerRefs = AbstractRefCounted.of(conn::onRemoved);
                     try {
                         if (connectedNodes.putIfAbsent(node, conn) != null) {
                             assert false : "redundant connection to " + node;
@@ -222,13 +223,14 @@ private void connectToNodeOrRetry(
                             IOUtils.closeWhileHandlingException(conn);
                         } else {
                             logger.debug("connected to node [{}]", node);
+                            managerRefs.incRef();
                             try {
                                 connectionListener.onNodeConnected(node, conn);
                             } finally {
                                 conn.addCloseListener(ActionListener.wrap(() -> {
                                     connectedNodes.remove(node, conn);
                                     connectionListener.onNodeDisconnected(node, conn);
-                                    conn.onRemoved();
+                                    managerRefs.decRef();
                                 }));
 
                                 conn.addCloseListener(ActionListener.wrap(() -> {
@@ -249,6 +251,7 @@ private void connectToNodeOrRetry(
                     } finally {
                         ListenableFuture<Transport.Connection> future = pendingConnections.remove(node);
                         assert future == currentListener : "Listener in pending map is different than the expected listener";
+                        managerRefs.decRef();
                         releaseOnce.run();
                         future.onResponse(conn);
                     }
@@ -391,4 +394,8 @@ public ConnectionProfile getConnectionProfile() {
         return defaultProfile;
     }
 
+    private boolean acquireConnectingRef() {
+        return closing.get() == false && connectingRefCounter.tryIncRef();
+    }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/transport/Compression.java b/server/src/main/java/org/elasticsearch/transport/Compression.java
index 5d653a4d9f24..a0290a8c92af 100644
--- a/server/src/main/java/org/elasticsearch/transport/Compression.java
+++ b/server/src/main/java/org/elasticsearch/transport/Compression.java
@@ -12,7 +12,7 @@
 import net.jpountz.lz4.LZ4Factory;
 import net.jpountz.lz4.LZ4FastDecompressor;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.lz4.ESLZ4Compressor;
@@ -27,7 +27,7 @@ public enum Scheme {
         LZ4,
         DEFLATE;
 
-        static final Version LZ4_VERSION = Version.V_7_14_0;
+        static final TransportVersion LZ4_VERSION = TransportVersion.V_7_14_0;
         static final int HEADER_LENGTH = 4;
         private static final byte[] DEFLATE_HEADER = new byte[] { 'D', 'F', 'L', '\0' };
         private static final byte[] LZ4_HEADER = new byte[] { 'L', 'Z', '4', '\0' };
diff --git a/server/src/main/java/org/elasticsearch/transport/ConnectTransportException.java b/server/src/main/java/org/elasticsearch/transport/ConnectTransportException.java
index eddd6d6f108b..e6e566e26b03 100644
--- a/server/src/main/java/org/elasticsearch/transport/ConnectTransportException.java
+++ b/server/src/main/java/org/elasticsearch/transport/ConnectTransportException.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.transport;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -35,7 +35,7 @@ public ConnectTransportException(DiscoveryNode node, String msg, String action,
 
     public ConnectTransportException(StreamInput in) throws IOException {
         super(in);
-        if (in.getVersion().before(Version.V_8_1_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_1_0)) {
             in.readOptionalWriteable(DiscoveryNode::new);
         }
     }
@@ -43,7 +43,7 @@ public ConnectTransportException(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        if (out.getVersion().before(Version.V_8_1_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_1_0)) {
             out.writeMissingWriteable(DiscoveryNode.class);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/transport/ConnectionProfile.java b/server/src/main/java/org/elasticsearch/transport/ConnectionProfile.java
index c4796e85f8ed..9aff4347ccdd 100644
--- a/server/src/main/java/org/elasticsearch/transport/ConnectionProfile.java
+++ b/server/src/main/java/org/elasticsearch/transport/ConnectionProfile.java
@@ -134,6 +134,7 @@ public static ConnectionProfile buildSingleChannelProfile(
     private final TimeValue pingInterval;
     private final Compression.Enabled compressionEnabled;
     private final Compression.Scheme compressionScheme;
+    private final String transportProfile;
 
     private ConnectionProfile(
         List<ConnectionTypeHandle> handles,
@@ -142,7 +143,8 @@ private ConnectionProfile(
         TimeValue handshakeTimeout,
         TimeValue pingInterval,
         Compression.Enabled compressionEnabled,
-        Compression.Scheme compressionScheme
+        Compression.Scheme compressionScheme,
+        String transportProfile
     ) {
         this.handles = handles;
         this.numConnections = numConnections;
@@ -151,6 +153,7 @@ private ConnectionProfile(
         this.pingInterval = pingInterval;
         this.compressionEnabled = compressionEnabled;
         this.compressionScheme = compressionScheme;
+        this.transportProfile = Objects.requireNonNull(transportProfile, "transport profile name must not be null");
     }
 
     /**
@@ -165,6 +168,7 @@ public static class Builder {
         private Compression.Enabled compressionEnabled;
         private Compression.Scheme compressionScheme;
         private TimeValue pingInterval;
+        private String transportProfile = TransportSettings.DEFAULT_PROFILE;
 
         /** create an empty builder */
         public Builder() {}
@@ -179,6 +183,7 @@ public Builder(ConnectionProfile source) {
             compressionEnabled = source.getCompressionEnabled();
             compressionScheme = source.getCompressionScheme();
             pingInterval = source.getPingInterval();
+            transportProfile = source.getTransportProfile();
         }
 
         /**
@@ -247,6 +252,11 @@ public Builder addConnections(int numConnections, TransportRequestOptions.Type..
             return this;
         }
 
+        public Builder setTransportProfile(String transportProfile) {
+            this.transportProfile = transportProfile;
+            return this;
+        }
+
         /**
          * Creates a new {@link ConnectionProfile} based on the added connections.
          * @throws IllegalStateException if any of the {@link org.elasticsearch.transport.TransportRequestOptions.Type} enum is missing
@@ -264,7 +274,8 @@ public ConnectionProfile build() {
                 handshakeTimeout,
                 pingInterval,
                 compressionEnabled,
-                compressionScheme
+                compressionScheme,
+                transportProfile
             );
         }
 
@@ -314,6 +325,10 @@ public int getNumConnections() {
         return numConnections;
     }
 
+    public String getTransportProfile() {
+        return transportProfile;
+    }
+
     /**
      * Returns the number of connections per type for this profile. This might return a count that is shared with other types such
      * that the sum of all connections per type might be higher than {@link #getNumConnections()}. For instance if
diff --git a/server/src/main/java/org/elasticsearch/transport/Header.java b/server/src/main/java/org/elasticsearch/transport/Header.java
index a1fa445f13dc..f84eddec7685 100644
--- a/server/src/main/java/org/elasticsearch/transport/Header.java
+++ b/server/src/main/java/org/elasticsearch/transport/Header.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.transport;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Tuple;
@@ -22,7 +22,7 @@ public class Header {
     private static final String RESPONSE_NAME = "NO_ACTION_NAME_FOR_RESPONSES";
 
     private final int networkMessageSize;
-    private final Version version;
+    private final TransportVersion version;
     private final long requestId;
     private final byte status;
     // These are directly set by tests
@@ -30,7 +30,7 @@ public class Header {
     Tuple<Map<String, String>, Map<String, Set<String>>> headers;
     private Compression.Scheme compressionScheme = null;
 
-    Header(int networkMessageSize, long requestId, byte status, Version version) {
+    Header(int networkMessageSize, long requestId, byte status, TransportVersion version) {
         this.networkMessageSize = networkMessageSize;
         this.version = version;
         this.requestId = requestId;
@@ -41,7 +41,7 @@ public int getNetworkMessageSize() {
         return networkMessageSize;
     }
 
-    Version getVersion() {
+    TransportVersion getVersion() {
         return version;
     }
 
@@ -93,7 +93,7 @@ void finishParsingHeader(StreamInput input) throws IOException {
         this.headers = ThreadContext.readHeadersFromStream(input);
 
         if (isRequest()) {
-            if (version.before(Version.V_8_0_0)) {
+            if (version.before(TransportVersion.V_8_0_0)) {
                 // discard features
                 input.readStringArray();
             }
diff --git a/server/src/main/java/org/elasticsearch/transport/InboundAggregator.java b/server/src/main/java/org/elasticsearch/transport/InboundAggregator.java
index 3c82c5459274..0dbd3f8fe00a 100644
--- a/server/src/main/java/org/elasticsearch/transport/InboundAggregator.java
+++ b/server/src/main/java/org/elasticsearch/transport/InboundAggregator.java
@@ -119,7 +119,7 @@ public InboundMessage finishAggregation() throws IOException {
                 checkBreaker(aggregated.getHeader(), aggregated.getContentLength(), breakerControl);
             }
             if (isShortCircuited()) {
-                aggregated.close();
+                aggregated.decRef();
                 success = true;
                 return new InboundMessage(aggregated.getHeader(), aggregationException);
             } else {
@@ -130,7 +130,7 @@ public InboundMessage finishAggregation() throws IOException {
         } finally {
             resetCurrentAggregation();
             if (success == false) {
-                aggregated.close();
+                aggregated.decRef();
             }
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/transport/InboundDecoder.java b/server/src/main/java/org/elasticsearch/transport/InboundDecoder.java
index 399614175410..ba399f6936df 100644
--- a/server/src/main/java/org/elasticsearch/transport/InboundDecoder.java
+++ b/server/src/main/java/org/elasticsearch/transport/InboundDecoder.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.transport;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.bytes.ReleasableBytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -25,7 +25,7 @@ public class InboundDecoder implements Releasable {
     static final Object PING = new Object();
     static final Object END_CONTENT = new Object();
 
-    private final Version version;
+    private final TransportVersion version;
     private final Recycler<BytesRef> recycler;
     private TransportDecompressor decompressor;
     private int totalNetworkSize = -1;
@@ -33,7 +33,7 @@ public class InboundDecoder implements Releasable {
     private boolean isCompressed = false;
     private boolean isClosed = false;
 
-    public InboundDecoder(Version version, Recycler<BytesRef> recycler) {
+    public InboundDecoder(TransportVersion version, Recycler<BytesRef> recycler) {
         this.version = version;
         this.recycler = recycler;
     }
@@ -154,7 +154,7 @@ private static int headerBytesToRead(BytesReference reference) {
             return 0;
         }
 
-        Version remoteVersion = Version.fromId(reference.getInt(TcpHeader.VERSION_POSITION));
+        TransportVersion remoteVersion = TransportVersion.fromId(reference.getInt(TcpHeader.VERSION_POSITION));
         int fixedHeaderSize = TcpHeader.headerSize(remoteVersion);
         if (fixedHeaderSize > reference.length()) {
             return 0;
@@ -172,12 +172,12 @@ private static int headerBytesToRead(BytesReference reference) {
     }
 
     // exposed for use in tests
-    static Header readHeader(Version version, int networkMessageSize, BytesReference bytesReference) throws IOException {
+    static Header readHeader(TransportVersion version, int networkMessageSize, BytesReference bytesReference) throws IOException {
         try (StreamInput streamInput = bytesReference.streamInput()) {
             streamInput.skip(TcpHeader.BYTES_REQUIRED_FOR_MESSAGE_SIZE);
             long requestId = streamInput.readLong();
             byte status = streamInput.readByte();
-            Version remoteVersion = Version.fromId(streamInput.readInt());
+            TransportVersion remoteVersion = TransportVersion.fromId(streamInput.readInt());
             Header header = new Header(networkMessageSize, requestId, status, remoteVersion);
             final IllegalStateException invalidVersion = ensureVersionCompatibility(remoteVersion, version, header.isHandshake());
             if (invalidVersion != null) {
@@ -203,14 +203,20 @@ private void ensureOpen() {
         }
     }
 
-    static IllegalStateException ensureVersionCompatibility(Version remoteVersion, Version currentVersion, boolean isHandshake) {
+    static IllegalStateException ensureVersionCompatibility(
+        TransportVersion remoteVersion,
+        TransportVersion currentVersion,
+        boolean isHandshake
+    ) {
         // for handshakes we are compatible with N-2 since otherwise we can't figure out our initial version
         // since we are compatible with N-1 and N+1 so we always send our minCompatVersion as the initial version in the
         // handshake. This looks odd but it's required to establish the connection correctly we check for real compatibility
         // once the connection is established
-        final Version compatibilityVersion = isHandshake ? currentVersion.minimumCompatibilityVersion() : currentVersion;
+        final TransportVersion compatibilityVersion = isHandshake ? currentVersion.calculateMinimumCompatVersion() : currentVersion;
         if (remoteVersion.isCompatible(compatibilityVersion) == false) {
-            final Version minCompatibilityVersion = isHandshake ? compatibilityVersion : compatibilityVersion.minimumCompatibilityVersion();
+            final TransportVersion minCompatibilityVersion = isHandshake
+                ? compatibilityVersion
+                : compatibilityVersion.calculateMinimumCompatVersion();
             String msg = "Received " + (isHandshake ? "handshake " : "") + "message from unsupported version: [";
             return new IllegalStateException(msg + remoteVersion + "] minimal compatible version is: [" + minCompatibilityVersion + "]");
         }
diff --git a/server/src/main/java/org/elasticsearch/transport/InboundHandler.java b/server/src/main/java/org/elasticsearch/transport/InboundHandler.java
index 3ef4b5f6769b..7c69f1e84d0a 100644
--- a/server/src/main/java/org/elasticsearch/transport/InboundHandler.java
+++ b/server/src/main/java/org/elasticsearch/transport/InboundHandler.java
@@ -11,7 +11,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -19,6 +19,8 @@
 import org.elasticsearch.common.network.HandlingTimeTracker;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Releasables;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -133,34 +135,17 @@ private void messageReceived(TcpChannel channel, InboundMessage message, long st
                 }
                 // ignore if its null, the service logs it
                 if (responseHandler != null) {
-                    final StreamInput streamInput;
-                    if (message.getContentLength() > 0 || header.getVersion().equals(Version.CURRENT) == false) {
-                        streamInput = namedWriteableStream(message.openOrGetStreamInput());
+                    if (message.getContentLength() > 0 || header.getVersion().equals(TransportVersion.CURRENT) == false) {
+                        final StreamInput streamInput = namedWriteableStream(message.openOrGetStreamInput());
                         assertRemoteVersion(streamInput, header.getVersion());
                         if (header.isError()) {
-                            handlerResponseError(streamInput, responseHandler);
+                            handlerResponseError(streamInput, message, responseHandler);
                         } else {
-                            handleResponse(remoteAddress, streamInput, responseHandler);
-                        }
-                        // Check the entire message has been read
-                        final int nextByte = streamInput.read();
-                        // calling read() is useful to make sure the message is fully read, even if there is an EOS marker
-                        if (nextByte != -1) {
-                            final IllegalStateException exception = new IllegalStateException(
-                                "Message not fully read (response) for requestId ["
-                                    + requestId
-                                    + "], handler ["
-                                    + responseHandler
-                                    + "], error ["
-                                    + header.isError()
-                                    + "]; resetting"
-                            );
-                            assert ignoreDeserializationErrors : exception;
-                            throw exception;
+                            handleResponse(remoteAddress, streamInput, responseHandler, message);
                         }
                     } else {
                         assert header.isError() == false;
-                        handleResponse(remoteAddress, EMPTY_STREAM_INPUT, responseHandler);
+                        handleResponse(remoteAddress, EMPTY_STREAM_INPUT, responseHandler, message);
                     }
                 }
             }
@@ -189,10 +174,30 @@ private void messageReceived(TcpChannel channel, InboundMessage message, long st
         }
     }
 
+    private void verifyResponseReadFully(Header header, TransportResponseHandler<?> responseHandler, StreamInput streamInput)
+        throws IOException {
+        // Check the entire message has been read
+        final int nextByte = streamInput.read();
+        // calling read() is useful to make sure the message is fully read, even if there is an EOS marker
+        if (nextByte != -1) {
+            final IllegalStateException exception = new IllegalStateException(
+                "Message not fully read (response) for requestId ["
+                    + header.getRequestId()
+                    + "], handler ["
+                    + responseHandler
+                    + "], error ["
+                    + header.isError()
+                    + "]; resetting"
+            );
+            assert ignoreDeserializationErrors : exception;
+            throw exception;
+        }
+    }
+
     private <T extends TransportRequest> void handleRequest(TcpChannel channel, Header header, InboundMessage message) throws IOException {
         final String action = header.getActionName();
         final long requestId = header.getRequestId();
-        final Version version = header.getVersion();
+        final TransportVersion version = header.getVersion();
         if (header.isHandshake()) {
             messageListener.onRequestReceived(requestId, action);
             // Cannot short circuit handshakes
@@ -212,7 +217,7 @@ private <T extends TransportRequest> void handleRequest(TcpChannel channel, Head
             try {
                 handshaker.handleHandshake(transportChannel, requestId, stream);
             } catch (Exception e) {
-                if (Version.CURRENT.isCompatible(header.getVersion())) {
+                if (TransportVersion.CURRENT.isCompatible(header.getVersion())) {
                     sendErrorResponse(action, transportChannel, e);
                 } else {
                     logger.warn(
@@ -335,10 +340,49 @@ private static void sendErrorResponse(String actionName, TransportChannel transp
     private <T extends TransportResponse> void handleResponse(
         InetSocketAddress remoteAddress,
         final StreamInput stream,
-        final TransportResponseHandler<T> handler
+        final TransportResponseHandler<T> handler,
+        final InboundMessage inboundMessage
+    ) {
+        final String executor = handler.executor();
+        if (ThreadPool.Names.SAME.equals(executor)) {
+            // no need to provide a buffer release here, we never escape the buffer when handling directly
+            doHandleResponse(handler, remoteAddress, stream, inboundMessage.getHeader(), () -> {});
+        } else {
+            inboundMessage.incRef();
+            // release buffer once we deserialize the message, but have a fail-safe in #onAfter below in case that didn't work out
+            final Releasable releaseBuffer = Releasables.releaseOnce(inboundMessage::decRef);
+            threadPool.executor(executor).execute(new ForkingResponseHandlerRunnable(handler, null) {
+                @Override
+                protected void doRun() {
+                    doHandleResponse(handler, remoteAddress, stream, inboundMessage.getHeader(), releaseBuffer);
+                }
+
+                @Override
+                public void onAfter() {
+                    Releasables.closeExpectNoException(releaseBuffer);
+                }
+            });
+        }
+    }
+
+    /**
+     *
+     * @param handler response handler
+     * @param remoteAddress remote address that the message was sent from
+     * @param stream bytes stream for reading the message
+     * @param header message header
+     * @param releaseResponseBuffer releasable that will be released once the message has been read from the {@code stream}
+     * @param <T> response message type
+     */
+    private <T extends TransportResponse> void doHandleResponse(
+        TransportResponseHandler<T> handler,
+        InetSocketAddress remoteAddress,
+        final StreamInput stream,
+        final Header header,
+        Releasable releaseResponseBuffer
     ) {
         final T response;
-        try {
+        try (releaseResponseBuffer) {
             response = handler.read(stream);
             response.remoteAddress(remoteAddress);
         } catch (Exception e) {
@@ -348,24 +392,11 @@ private <T extends TransportResponse> void handleResponse(
             );
             logger.warn(() -> "Failed to deserialize response from [" + remoteAddress + "]", serializationException);
             assert ignoreDeserializationErrors : e;
-            handleException(handler, serializationException);
+            doHandleException(handler, serializationException);
             return;
         }
-        final String executor = handler.executor();
-        if (ThreadPool.Names.SAME.equals(executor)) {
-            doHandleResponse(handler, response);
-        } else {
-            threadPool.executor(executor).execute(new ForkingResponseHandlerRunnable(handler, null) {
-                @Override
-                protected void doRun() {
-                    doHandleResponse(handler, response);
-                }
-            });
-        }
-    }
-
-    private static <T extends TransportResponse> void doHandleResponse(TransportResponseHandler<T> handler, T response) {
         try {
+            verifyResponseReadFully(header, handler, stream);
             handler.handleResponse(response);
         } catch (Exception e) {
             doHandleException(handler, new ResponseHandlerFailureTransportException(e));
@@ -374,10 +405,11 @@ private static <T extends TransportResponse> void doHandleResponse(TransportResp
         }
     }
 
-    private void handlerResponseError(StreamInput stream, final TransportResponseHandler<?> handler) {
+    private void handlerResponseError(StreamInput stream, InboundMessage message, final TransportResponseHandler<?> handler) {
         Exception error;
         try {
             error = stream.readException();
+            verifyResponseReadFully(message.getHeader(), handler, stream);
         } catch (Exception e) {
             error = new TransportSerializationException(
                 "Failed to deserialize exception response from stream for handler [" + handler + "]",
@@ -418,7 +450,8 @@ private StreamInput namedWriteableStream(StreamInput delegate) {
         return new NamedWriteableAwareStreamInput(delegate, namedWriteableRegistry);
     }
 
-    static void assertRemoteVersion(StreamInput in, Version version) {
-        assert version.equals(in.getVersion()) : "Stream version [" + in.getVersion() + "] does not match version [" + version + "]";
+    static void assertRemoteVersion(StreamInput in, TransportVersion version) {
+        assert version.equals(in.getTransportVersion())
+            : "Stream version [" + in.getTransportVersion() + "] does not match version [" + version + "]";
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/transport/InboundMessage.java b/server/src/main/java/org/elasticsearch/transport/InboundMessage.java
index 8d050ba19cef..d420ccc8e2e4 100644
--- a/server/src/main/java/org/elasticsearch/transport/InboundMessage.java
+++ b/server/src/main/java/org/elasticsearch/transport/InboundMessage.java
@@ -11,13 +11,14 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.bytes.ReleasableBytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.core.AbstractRefCounted;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Releasable;
 
 import java.io.IOException;
 import java.util.Objects;
 
-public class InboundMessage implements Releasable {
+public class InboundMessage extends AbstractRefCounted {
 
     private final Header header;
     private final ReleasableBytesReference content;
@@ -82,15 +83,21 @@ public Releasable takeBreakerReleaseControl() {
 
     public StreamInput openOrGetStreamInput() throws IOException {
         assert isPing == false && content != null;
+        assert hasReferences();
         if (streamInput == null) {
             streamInput = content.streamInput();
-            streamInput.setVersion(header.getVersion());
+            streamInput.setTransportVersion(header.getVersion());
         }
         return streamInput;
     }
 
     @Override
-    public void close() {
+    public String toString() {
+        return "InboundMessage{" + header + "}";
+    }
+
+    @Override
+    protected void closeInternal() {
         try {
             IOUtils.close(streamInput, content, breakerRelease);
         } catch (Exception e) {
@@ -98,9 +105,4 @@ public void close() {
             throw new ElasticsearchException(e);
         }
     }
-
-    @Override
-    public String toString() {
-        return "InboundMessage{" + header + "}";
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/transport/InboundPipeline.java b/server/src/main/java/org/elasticsearch/transport/InboundPipeline.java
index 7a8cc9173abb..eb24dadd776b 100644
--- a/server/src/main/java/org/elasticsearch/transport/InboundPipeline.java
+++ b/server/src/main/java/org/elasticsearch/transport/InboundPipeline.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.transport;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.bytes.CompositeBytesReference;
 import org.elasticsearch.common.bytes.ReleasableBytesReference;
@@ -40,7 +40,7 @@ public class InboundPipeline implements Releasable {
     private boolean isClosed = false;
 
     public InboundPipeline(
-        Version version,
+        TransportVersion version,
         StatsTracker statsTracker,
         Recycler<BytesRef> recycler,
         LongSupplier relativeTimeInMillis,
@@ -144,9 +144,12 @@ private void forwardFragments(TcpChannel channel, ArrayList<Object> fragments) t
                 messageHandler.accept(channel, PING_MESSAGE);
             } else if (fragment == InboundDecoder.END_CONTENT) {
                 assert aggregator.isAggregating();
-                try (InboundMessage aggregated = aggregator.finishAggregation()) {
+                InboundMessage aggregated = aggregator.finishAggregation();
+                try {
                     statsTracker.markMessageReceived();
                     messageHandler.accept(channel, aggregated);
+                } finally {
+                    aggregated.decRef();
                 }
             } else {
                 assert aggregator.isAggregating();
diff --git a/server/src/main/java/org/elasticsearch/transport/NetworkMessage.java b/server/src/main/java/org/elasticsearch/transport/NetworkMessage.java
index 976da4268c3d..f71159df29eb 100644
--- a/server/src/main/java/org/elasticsearch/transport/NetworkMessage.java
+++ b/server/src/main/java/org/elasticsearch/transport/NetworkMessage.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.transport;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 
@@ -17,13 +17,19 @@
  */
 public abstract class NetworkMessage {
 
-    protected final Version version;
+    protected final TransportVersion version;
     protected final Writeable threadContext;
     protected final long requestId;
     protected final byte status;
     protected final Compression.Scheme compressionScheme;
 
-    NetworkMessage(ThreadContext threadContext, Version version, byte status, long requestId, Compression.Scheme compressionScheme) {
+    NetworkMessage(
+        ThreadContext threadContext,
+        TransportVersion version,
+        byte status,
+        long requestId,
+        Compression.Scheme compressionScheme
+    ) {
         this.threadContext = threadContext.captureAsWriteable();
         this.version = version;
         this.requestId = requestId;
@@ -35,7 +41,7 @@ public abstract class NetworkMessage {
         }
     }
 
-    public Version getVersion() {
+    public TransportVersion getVersion() {
         return version;
     }
 
@@ -63,7 +69,7 @@ boolean isError() {
         return TransportStatus.isError(status);
     }
 
-    private static Compression.Scheme adjustedScheme(Version version, Compression.Scheme compressionScheme) {
+    private static Compression.Scheme adjustedScheme(TransportVersion version, Compression.Scheme compressionScheme) {
         return compressionScheme == Compression.Scheme.LZ4 && version.before(Compression.Scheme.LZ4_VERSION) ? null : compressionScheme;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/transport/OutboundHandler.java b/server/src/main/java/org/elasticsearch/transport/OutboundHandler.java
index 253cc2ec6d24..bedfa8d7225a 100644
--- a/server/src/main/java/org/elasticsearch/transport/OutboundHandler.java
+++ b/server/src/main/java/org/elasticsearch/transport/OutboundHandler.java
@@ -13,7 +13,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -36,7 +36,7 @@ final class OutboundHandler {
     private static final Logger logger = LogManager.getLogger(OutboundHandler.class);
 
     private final String nodeName;
-    private final Version version;
+    private final TransportVersion version;
     private final StatsTracker statsTracker;
     private final ThreadPool threadPool;
     private final Recycler<BytesRef> recycler;
@@ -49,7 +49,7 @@ final class OutboundHandler {
 
     OutboundHandler(
         String nodeName,
-        Version version,
+        TransportVersion version,
         StatsTracker statsTracker,
         ThreadPool threadPool,
         Recycler<BytesRef> recycler,
@@ -84,11 +84,11 @@ void sendRequest(
         final String action,
         final TransportRequest request,
         final TransportRequestOptions options,
-        final Version channelVersion,
+        final TransportVersion transportVersion,
         final Compression.Scheme compressionScheme,
         final boolean isHandshake
     ) throws IOException, TransportException {
-        Version version = Version.min(this.version, channelVersion);
+        TransportVersion version = TransportVersion.min(this.version, transportVersion);
         OutboundMessage.Request message = new OutboundMessage.Request(
             threadPool.getThreadContext(),
             request,
@@ -115,10 +115,10 @@ void sendRequest(
      * Sends the response to the given channel. This method should be used to send {@link TransportResponse}
      * objects back to the caller.
      *
-     * @see #sendErrorResponse(Version, TcpChannel, long, String, Exception) for sending error responses
+     * @see #sendErrorResponse(TransportVersion, TcpChannel, long, String, Exception) for sending error responses
      */
     void sendResponse(
-        final Version nodeVersion,
+        final TransportVersion transportVersion,
         final TcpChannel channel,
         final long requestId,
         final String action,
@@ -126,7 +126,7 @@ void sendResponse(
         final Compression.Scheme compressionScheme,
         final boolean isHandshake
     ) throws IOException {
-        Version version = Version.min(this.version, nodeVersion);
+        TransportVersion version = TransportVersion.min(this.version, transportVersion);
         OutboundMessage.Response message = new OutboundMessage.Response(
             threadPool.getThreadContext(),
             response,
@@ -148,13 +148,13 @@ void sendResponse(
      * Sends back an error response to the caller via the given channel
      */
     void sendErrorResponse(
-        final Version nodeVersion,
+        final TransportVersion transportVersion,
         final TcpChannel channel,
         final long requestId,
         final String action,
         final Exception error
     ) throws IOException {
-        Version version = Version.min(this.version, nodeVersion);
+        TransportVersion version = TransportVersion.min(this.version, transportVersion);
         RemoteTransportException tx = new RemoteTransportException(nodeName, channel.getLocalAddress(), action, error);
         OutboundMessage.Response message = new OutboundMessage.Response(threadPool.getThreadContext(), tx, version, requestId, false, null);
         sendMessage(channel, message, () -> messageListener.onResponseSent(requestId, action, error));
diff --git a/server/src/main/java/org/elasticsearch/transport/OutboundMessage.java b/server/src/main/java/org/elasticsearch/transport/OutboundMessage.java
index eae263cecf84..492aa82a8eab 100644
--- a/server/src/main/java/org/elasticsearch/transport/OutboundMessage.java
+++ b/server/src/main/java/org/elasticsearch/transport/OutboundMessage.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.transport;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -28,7 +28,7 @@ abstract class OutboundMessage extends NetworkMessage {
 
     OutboundMessage(
         ThreadContext threadContext,
-        Version version,
+        TransportVersion version,
         byte status,
         long requestId,
         Compression.Scheme compressionScheme,
@@ -39,7 +39,7 @@ abstract class OutboundMessage extends NetworkMessage {
     }
 
     BytesReference serialize(RecyclerBytesStreamOutput bytesStream) throws IOException {
-        bytesStream.setVersion(version);
+        bytesStream.setTransportVersion(version);
         bytesStream.skip(TcpHeader.headerSize(version));
 
         // The compressible bytes stream will not close the underlying bytes stream
@@ -56,7 +56,7 @@ BytesReference serialize(RecyclerBytesStreamOutput bytesStream) throws IOExcepti
         final StreamOutput stream = compress ? wrapCompressed(bytesStream) : bytesStream;
         final BytesReference zeroCopyBuffer;
         try {
-            stream.setVersion(version);
+            stream.setTransportVersion(version);
             if (variableHeaderLength == -1) {
                 writeVariableHeader(stream);
             }
@@ -115,7 +115,7 @@ static class Request extends OutboundMessage {
         Request(
             ThreadContext threadContext,
             Writeable message,
-            Version version,
+            TransportVersion version,
             String action,
             long requestId,
             boolean isHandshake,
@@ -128,7 +128,7 @@ static class Request extends OutboundMessage {
         @Override
         protected void writeVariableHeader(StreamOutput stream) throws IOException {
             super.writeVariableHeader(stream);
-            if (version.before(Version.V_8_0_0)) {
+            if (version.before(TransportVersion.V_8_0_0)) {
                 // empty features array
                 stream.writeStringArray(Strings.EMPTY_ARRAY);
             }
@@ -165,7 +165,7 @@ static class Response extends OutboundMessage {
         Response(
             ThreadContext threadContext,
             Writeable message,
-            Version version,
+            TransportVersion version,
             long requestId,
             boolean isHandshake,
             Compression.Scheme compressionScheme
diff --git a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
index d49e760ab85a..a410f0e912f8 100644
--- a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
+++ b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.transport;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterName;
@@ -316,7 +317,7 @@ public ProxyModeInfo(String address, String serverName, int maxSocketConnections
 
         private ProxyModeInfo(StreamInput input) throws IOException {
             address = input.readString();
-            if (input.getVersion().onOrAfter(Version.V_7_7_0)) {
+            if (input.getTransportVersion().onOrAfter(TransportVersion.V_7_7_0)) {
                 serverName = input.readString();
             } else {
                 serverName = null;
@@ -337,7 +338,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(address);
-            if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_7_0)) {
                 out.writeString(serverName);
             }
             out.writeVInt(maxSocketConnections);
diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterPortSettings.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterPortSettings.java
new file mode 100644
index 000000000000..97324a461561
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterPortSettings.java
@@ -0,0 +1,170 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.transport;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
+
+import java.util.List;
+import java.util.function.Function;
+
+import static org.elasticsearch.common.settings.Setting.boolSetting;
+import static org.elasticsearch.common.settings.Setting.intSetting;
+import static org.elasticsearch.common.settings.Setting.listSetting;
+import static org.elasticsearch.transport.TransportSettings.BIND_HOST_PROFILE;
+import static org.elasticsearch.transport.TransportSettings.PORT_PROFILE;
+import static org.elasticsearch.transport.TransportSettings.PUBLISH_HOST_PROFILE;
+import static org.elasticsearch.transport.TransportSettings.PUBLISH_PORT_PROFILE;
+import static org.elasticsearch.transport.TransportSettings.TCP_KEEP_ALIVE_PROFILE;
+import static org.elasticsearch.transport.TransportSettings.TCP_KEEP_COUNT_PROFILE;
+import static org.elasticsearch.transport.TransportSettings.TCP_KEEP_IDLE_PROFILE;
+import static org.elasticsearch.transport.TransportSettings.TCP_KEEP_INTERVAL_PROFILE;
+import static org.elasticsearch.transport.TransportSettings.TCP_NO_DELAY_PROFILE;
+import static org.elasticsearch.transport.TransportSettings.TCP_RECEIVE_BUFFER_SIZE_PROFILE;
+import static org.elasticsearch.transport.TransportSettings.TCP_REUSE_ADDRESS_PROFILE;
+import static org.elasticsearch.transport.TransportSettings.TCP_SEND_BUFFER_SIZE_PROFILE;
+
+/**
+ * Contains the settings and some associated logic for the settings related to the Remote Access port, used by Remote Cluster Security 2.0.
+ */
+public class RemoteClusterPortSettings {
+    public static final String REMOTE_CLUSTER_PROFILE = "_remote_cluster";
+    public static final String REMOTE_CLUSTER_PREFIX = "remote_cluster.";
+
+    public static final Setting<Boolean> REMOTE_CLUSTER_PORT_ENABLED = boolSetting(
+        REMOTE_CLUSTER_PREFIX + "enabled",
+        false,
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting<List<String>> HOST = listSetting(
+        REMOTE_CLUSTER_PREFIX + "host",
+        TransportSettings.BIND_HOST,
+        Function.identity(),
+        Setting.Property.NodeScope
+    );
+    public static final Setting<List<String>> PUBLISH_HOST = listSetting(
+        REMOTE_CLUSTER_PREFIX + "publish_host",
+        HOST,
+        Function.identity(),
+        Setting.Property.NodeScope
+    );
+    public static final Setting<List<String>> BIND_HOST = listSetting(
+        REMOTE_CLUSTER_PREFIX + "bind_host",
+        HOST,
+        Function.identity(),
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting<Integer> PORT = intSetting(REMOTE_CLUSTER_PREFIX + "port", 9443, 0, 65535, Setting.Property.NodeScope);
+
+    // The default value of -1 means it will use the default bind port as shown above
+    public static final Setting<Integer> PUBLISH_PORT = intSetting(
+        REMOTE_CLUSTER_PREFIX + "publish_port",
+        -1,
+        -1,
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting<Boolean> TCP_KEEP_ALIVE = boolSetting(
+        REMOTE_CLUSTER_PREFIX + "tcp.keep_alive",
+        TransportSettings.TCP_KEEP_ALIVE,
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting<Integer> TCP_KEEP_IDLE = intSetting(
+        REMOTE_CLUSTER_PREFIX + "tcp.keep_idle",
+        TransportSettings.TCP_KEEP_IDLE,
+        -1,
+        300,
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting<Integer> TCP_KEEP_INTERVAL = intSetting(
+        REMOTE_CLUSTER_PREFIX + "tcp.keep_interval",
+        TransportSettings.TCP_KEEP_INTERVAL,
+        -1,
+        300,
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting<Integer> TCP_KEEP_COUNT = intSetting(
+        REMOTE_CLUSTER_PREFIX + "tcp.keep_count",
+        TransportSettings.TCP_KEEP_COUNT,
+        -1,
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting<Boolean> TCP_NO_DELAY = boolSetting(
+        REMOTE_CLUSTER_PREFIX + "tcp.no_delay",
+        TransportSettings.TCP_NO_DELAY,
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting<Boolean> TCP_REUSE_ADDRESS = boolSetting(
+        REMOTE_CLUSTER_PREFIX + "tcp.reuse_address",
+        TransportSettings.TCP_REUSE_ADDRESS,
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting<ByteSizeValue> TCP_SEND_BUFFER_SIZE = Setting.byteSizeSetting(
+        REMOTE_CLUSTER_PREFIX + "tcp.send_buffer_size",
+        TransportSettings.TCP_SEND_BUFFER_SIZE,
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting<ByteSizeValue> TCP_RECEIVE_BUFFER_SIZE = Setting.byteSizeSetting(
+        REMOTE_CLUSTER_PREFIX + "tcp.receive_buffer_size",
+        TransportSettings.TCP_RECEIVE_BUFFER_SIZE,
+        Setting.Property.NodeScope
+    );
+
+    static void validateRemoteAccessSettings(Settings settings) {
+        if (REMOTE_CLUSTER_PORT_ENABLED.get(settings)
+            && settings.getGroups("transport.profiles.", true).keySet().contains(REMOTE_CLUSTER_PROFILE)) {
+            throw new IllegalArgumentException(
+                "Remote Access settings should not be configured using the ["
+                    + REMOTE_CLUSTER_PROFILE
+                    + "] profile. "
+                    + "Use the ["
+                    + REMOTE_CLUSTER_PREFIX
+                    + "] settings instead."
+            );
+        }
+    }
+
+    public static TcpTransport.ProfileSettings buildRemoteAccessProfileSettings(Settings settings) {
+        validateRemoteAccessSettings(settings);
+
+        // Build a synthetic settings object with the `_remote_access` profile properly configured per the friendlier settings,
+        Settings syntheticRemoteAccessProfile = Settings.builder()
+            .put(settings)
+            .put(TCP_KEEP_ALIVE_PROFILE.getConcreteSettingForNamespace(REMOTE_CLUSTER_PROFILE).getKey(), TCP_KEEP_ALIVE.get(settings))
+            .put(TCP_KEEP_IDLE_PROFILE.getConcreteSettingForNamespace(REMOTE_CLUSTER_PROFILE).getKey(), TCP_KEEP_IDLE.get(settings))
+            .put(TCP_KEEP_INTERVAL_PROFILE.getConcreteSettingForNamespace(REMOTE_CLUSTER_PROFILE).getKey(), TCP_KEEP_INTERVAL.get(settings))
+            .put(TCP_KEEP_COUNT_PROFILE.getConcreteSettingForNamespace(REMOTE_CLUSTER_PROFILE).getKey(), TCP_KEEP_COUNT.get(settings))
+            .put(TCP_NO_DELAY_PROFILE.getConcreteSettingForNamespace(REMOTE_CLUSTER_PROFILE).getKey(), TCP_NO_DELAY.get(settings))
+            .put(TCP_REUSE_ADDRESS_PROFILE.getConcreteSettingForNamespace(REMOTE_CLUSTER_PROFILE).getKey(), TCP_REUSE_ADDRESS.get(settings))
+            .put(
+                TCP_SEND_BUFFER_SIZE_PROFILE.getConcreteSettingForNamespace(REMOTE_CLUSTER_PROFILE).getKey(),
+                TCP_SEND_BUFFER_SIZE.get(settings)
+            )
+            .put(
+                TCP_RECEIVE_BUFFER_SIZE_PROFILE.getConcreteSettingForNamespace(REMOTE_CLUSTER_PROFILE).getKey(),
+                TCP_RECEIVE_BUFFER_SIZE.get(settings)
+            )
+            .putList(BIND_HOST_PROFILE.getConcreteSettingForNamespace(REMOTE_CLUSTER_PROFILE).getKey(), BIND_HOST.get(settings))
+            .putList(PUBLISH_HOST_PROFILE.getConcreteSettingForNamespace(REMOTE_CLUSTER_PROFILE).getKey(), PUBLISH_HOST.get(settings))
+            .put(PORT_PROFILE.getConcreteSettingForNamespace(REMOTE_CLUSTER_PROFILE).getKey(), PORT.get(settings))
+            .put(PUBLISH_PORT_PROFILE.getConcreteSettingForNamespace(REMOTE_CLUSTER_PROFILE).getKey(), PUBLISH_PORT.get(settings))
+            .build();
+        return new TcpTransport.ProfileSettings(syntheticRemoteAccessProfile, REMOTE_CLUSTER_PROFILE);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
index aff4a6952db4..bf380df5bbf7 100644
--- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
+++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
@@ -12,7 +12,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.OriginalIndices;
-import org.elasticsearch.action.support.GroupedActionListener;
+import org.elasticsearch.action.support.CountDownActionListener;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.internal.Client;
@@ -332,14 +332,14 @@ synchronized void updateRemoteCluster(String clusterAlias, Settings newSettings,
      */
     void initializeRemoteClusters() {
         final TimeValue timeValue = REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING.get(settings);
-        final PlainActionFuture<Collection<Void>> future = new PlainActionFuture<>();
+        final PlainActionFuture<Void> future = new PlainActionFuture<>();
         Set<String> enabledClusters = RemoteClusterAware.getEnabledRemoteClusters(settings);
 
         if (enabledClusters.isEmpty()) {
             return;
         }
 
-        GroupedActionListener<Void> listener = new GroupedActionListener<>(future, enabledClusters.size());
+        CountDownActionListener listener = new CountDownActionListener(enabledClusters.size(), future);
         for (String clusterAlias : enabledClusters) {
             updateRemoteCluster(clusterAlias, settings, listener);
         }
diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionInfo.java b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionInfo.java
index fbbd2483e693..b21455829fb0 100644
--- a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionInfo.java
+++ b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionInfo.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.transport;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -40,7 +40,7 @@ public RemoteConnectionInfo(String clusterAlias, ModeInfo modeInfo, TimeValue in
     }
 
     public RemoteConnectionInfo(StreamInput input) throws IOException {
-        if (input.getVersion().onOrAfter(Version.V_7_6_0)) {
+        if (input.getTransportVersion().onOrAfter(TransportVersion.V_7_6_0)) {
             RemoteConnectionStrategy.ConnectionStrategy mode = input.readEnum(RemoteConnectionStrategy.ConnectionStrategy.class);
             modeInfo = mode.getReader().read(input);
             initialConnectionTimeout = input.readTimeValue();
@@ -79,7 +79,7 @@ public boolean isSkipUnavailable() {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getVersion().onOrAfter(Version.V_7_6_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_6_0)) {
             out.writeEnum(modeInfo.modeType());
             modeInfo.writeTo(out);
             out.writeTimeValue(initialConnectionTimeout);
diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionManager.java b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionManager.java
index 16296ac85f1e..e6612ca58b3c 100644
--- a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionManager.java
+++ b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionManager.java
@@ -7,6 +7,7 @@
  */
 package org.elasticsearch.transport;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -17,6 +18,8 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -81,13 +84,20 @@ public void removeListener(TransportConnectionListener listener) {
 
     @Override
     public void openConnection(DiscoveryNode node, ConnectionProfile profile, ActionListener<Transport.Connection> listener) {
-        delegate.openConnection(node, profile, listener);
+        delegate.openConnection(
+            node,
+            profile,
+            ActionListener.wrap(
+                connection -> listener.onResponse(new InternalRemoteConnection(connection, clusterAlias)),
+                listener::onFailure
+            )
+        );
     }
 
     @Override
     public Transport.Connection getConnection(DiscoveryNode node) {
         try {
-            return delegate.getConnection(node);
+            return getConnectionInternal(node);
         } catch (NodeNotConnectedException e) {
             return new ProxyConnection(getAnyRemoteConnection(), node);
         }
@@ -116,7 +126,7 @@ public Transport.Connection getAnyRemoteConnection() {
         if (localConnectedNodes.isEmpty() == false) {
             DiscoveryNode nextNode = localConnectedNodes.get(Math.floorMod(curr, localConnectedNodes.size()));
             try {
-                return delegate.getConnection(nextNode);
+                return getConnectionInternal(nextNode);
             } catch (NodeNotConnectedException e) {
                 // Ignore. We will manually create an iterator of open nodes
             }
@@ -124,7 +134,7 @@ public Transport.Connection getAnyRemoteConnection() {
         Set<DiscoveryNode> allConnectionNodes = getAllConnectedNodes();
         for (DiscoveryNode connectedNode : allConnectionNodes) {
             try {
-                return delegate.getConnection(connectedNode);
+                return getConnectionInternal(connectedNode);
             } catch (NodeNotConnectedException e) {
                 // Ignore. We will try the next one until all are exhausted.
             }
@@ -152,6 +162,26 @@ public void closeNoBlock() {
         delegate.closeNoBlock();
     }
 
+    /**
+     * This method returns a remote cluster alias for the given transport connection if it targets a node in the remote cluster.
+     * This method will return an optional empty in case the connection targets the local node or the node in the local cluster.
+     *
+     * @param connection the transport connection for which to resolve a remote cluster alias
+     * @return a cluster alias if the connection target a node in the remote cluster, otherwise an empty result
+     */
+    public static Optional<String> resolveRemoteClusterAlias(Transport.Connection connection) {
+        Transport.Connection unwrapped = TransportService.unwrapConnection(connection);
+        if (unwrapped instanceof InternalRemoteConnection remoteConnection) {
+            return Optional.of(remoteConnection.getClusterAlias());
+        }
+        return Optional.empty();
+    }
+
+    private Transport.Connection getConnectionInternal(DiscoveryNode node) throws NodeNotConnectedException {
+        Transport.Connection connection = delegate.getConnection(node);
+        return new InternalRemoteConnection(connection, clusterAlias);
+    }
+
     private synchronized void addConnectedNode(DiscoveryNode addedNode) {
         this.connectedNodes = CollectionUtils.appendToCopy(this.connectedNodes, addedNode);
     }
@@ -218,6 +248,11 @@ public Version getVersion() {
             return connection.getVersion();
         }
 
+        @Override
+        public TransportVersion getTransportVersion() {
+            return connection.getTransportVersion();
+        }
+
         @Override
         public Object getCacheKey() {
             return connection.getCacheKey();
@@ -249,4 +284,90 @@ public boolean hasReferences() {
         @Override
         public void onRemoved() {}
     }
+
+    private static final class InternalRemoteConnection implements Transport.Connection {
+
+        private final Transport.Connection connection;
+        private final String clusterAlias;
+
+        InternalRemoteConnection(Transport.Connection connection, String clusterAlias) {
+            this.clusterAlias = Objects.requireNonNull(clusterAlias);
+            this.connection = Objects.requireNonNull(connection);
+        }
+
+        public String getClusterAlias() {
+            return clusterAlias;
+        }
+
+        @Override
+        public DiscoveryNode getNode() {
+            return connection.getNode();
+        }
+
+        @Override
+        public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
+            throws IOException, TransportException {
+            connection.sendRequest(requestId, action, request, options);
+        }
+
+        @Override
+        public void addCloseListener(ActionListener<Void> listener) {
+            connection.addCloseListener(listener);
+        }
+
+        @Override
+        public boolean isClosed() {
+            return connection.isClosed();
+        }
+
+        @Override
+        public Version getVersion() {
+            return connection.getVersion();
+        }
+
+        @Override
+        public TransportVersion getTransportVersion() {
+            return connection.getTransportVersion();
+        }
+
+        @Override
+        public Object getCacheKey() {
+            return connection.getCacheKey();
+        }
+
+        @Override
+        public void close() {
+            connection.close();
+        }
+
+        @Override
+        public void onRemoved() {
+            connection.onRemoved();
+        }
+
+        @Override
+        public void addRemovedListener(ActionListener<Void> listener) {
+            connection.addRemovedListener(listener);
+        }
+
+        @Override
+        public void incRef() {
+            connection.incRef();
+        }
+
+        @Override
+        public boolean tryIncRef() {
+            return connection.tryIncRef();
+        }
+
+        @Override
+        public boolean decRef() {
+            return connection.decRef();
+        }
+
+        @Override
+        public boolean hasReferences() {
+            return connection.hasReferences();
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java
index d493479ee528..c40d67af84d1 100644
--- a/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java
+++ b/server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java
@@ -44,6 +44,7 @@
 import java.util.stream.Stream;
 
 import static org.elasticsearch.core.Strings.format;
+import static org.elasticsearch.transport.RemoteClusterService.REMOTE_CLUSTER_AUTHORIZATION;
 
 public abstract class RemoteConnectionStrategy implements TransportConnectionListener, Closeable {
 
@@ -141,6 +142,10 @@ public Writeable.Reader<RemoteConnectionInfo.ModeInfo> getReader() {
     }
 
     static ConnectionProfile buildConnectionProfile(String clusterAlias, Settings settings) {
+        final String transportProfile = REMOTE_CLUSTER_AUTHORIZATION.getConcreteSettingForNamespace(clusterAlias).exists(settings)
+            ? RemoteClusterPortSettings.REMOTE_CLUSTER_PROFILE
+            : TransportSettings.DEFAULT_PROFILE;
+
         ConnectionStrategy mode = REMOTE_CONNECTION_MODE.getConcreteSettingForNamespace(clusterAlias).get(settings);
         ConnectionProfile.Builder builder = new ConnectionProfile.Builder().setConnectTimeout(
             TransportSettings.CONNECT_TIMEOUT.get(settings)
@@ -158,7 +163,8 @@ static ConnectionProfile buildConnectionProfile(String clusterAlias, Settings se
                 TransportRequestOptions.Type.RECOVERY,
                 TransportRequestOptions.Type.PING
             )
-            .addConnections(mode.numberOfChannels, TransportRequestOptions.Type.REG);
+            .addConnections(mode.numberOfChannels, TransportRequestOptions.Type.REG)
+            .setTransportProfile(transportProfile);
         return builder.build();
     }
 
diff --git a/server/src/main/java/org/elasticsearch/transport/TaskTransportChannel.java b/server/src/main/java/org/elasticsearch/transport/TaskTransportChannel.java
index 1c3384847ab1..2288de598b33 100644
--- a/server/src/main/java/org/elasticsearch/transport/TaskTransportChannel.java
+++ b/server/src/main/java/org/elasticsearch/transport/TaskTransportChannel.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.transport;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.core.Releasable;
 
 import java.io.IOException;
@@ -52,7 +52,7 @@ public void sendResponse(Exception exception) throws IOException {
     }
 
     @Override
-    public Version getVersion() {
+    public TransportVersion getVersion() {
         return channel.getVersion();
     }
 
diff --git a/server/src/main/java/org/elasticsearch/transport/TcpHeader.java b/server/src/main/java/org/elasticsearch/transport/TcpHeader.java
index 60e08225f7be..fce72973aa9f 100644
--- a/server/src/main/java/org/elasticsearch/transport/TcpHeader.java
+++ b/server/src/main/java/org/elasticsearch/transport/TcpHeader.java
@@ -8,14 +8,14 @@
 
 package org.elasticsearch.transport;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
 import java.io.IOException;
 
 public class TcpHeader {
 
-    public static final Version VERSION_WITH_HEADER_SIZE = Version.V_7_6_0;
+    public static final TransportVersion VERSION_WITH_HEADER_SIZE = TransportVersion.V_7_6_0;
 
     public static final int MARKER_BYTES_SIZE = 2;
 
@@ -41,7 +41,7 @@ public class TcpHeader {
 
     private static final int HEADER_SIZE = PRE_76_HEADER_SIZE + VARIABLE_HEADER_SIZE;
 
-    public static int headerSize(Version version) {
+    public static int headerSize(TransportVersion version) {
         if (version.onOrAfter(VERSION_WITH_HEADER_SIZE)) {
             return HEADER_SIZE;
         } else {
@@ -55,7 +55,7 @@ public static void writeHeader(
         StreamOutput output,
         long requestId,
         byte status,
-        Version version,
+        TransportVersion version,
         int contentSize,
         int variableHeaderSize
     ) throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java
index aac192e3762a..442ae7f7eaab 100644
--- a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java
+++ b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java
@@ -13,6 +13,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Build;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ThreadedActionListener;
@@ -82,6 +83,8 @@
 import static org.elasticsearch.common.transport.NetworkExceptionHelper.isConnectException;
 import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap;
 import static org.elasticsearch.core.Strings.format;
+import static org.elasticsearch.transport.RemoteClusterPortSettings.REMOTE_CLUSTER_PORT_ENABLED;
+import static org.elasticsearch.transport.RemoteClusterPortSettings.REMOTE_CLUSTER_PROFILE;
 
 public abstract class TcpTransport extends AbstractLifecycleComponent implements Transport {
     private static final Logger logger = LogManager.getLogger(TcpTransport.class);
@@ -130,7 +133,7 @@ public static boolean isUntrustedRemoteClusterEnabled() {
     protected final NetworkService networkService;
     protected final Set<ProfileSettings> profileSettingsSet;
     protected final boolean rstOnClose;
-    private final Version version;
+    private final TransportVersion version;
     private final CircuitBreakerService circuitBreakerService;
 
     private final ConcurrentMap<String, BoundTransportAddress> profileBoundAddresses = newConcurrentMap();
@@ -154,7 +157,7 @@ public static boolean isUntrustedRemoteClusterEnabled() {
 
     public TcpTransport(
         Settings settings,
-        Version version,
+        TransportVersion version,
         ThreadPool threadPool,
         PageCacheRecycler pageCacheRecycler,
         CircuitBreakerService circuitBreakerService,
@@ -214,7 +217,7 @@ public TcpTransport(
         );
     }
 
-    public Version getVersion() {
+    public TransportVersion getVersion() {
         return version;
     }
 
@@ -253,12 +256,17 @@ public final class NodeChannels extends CloseableConnection {
         private final Map<TransportRequestOptions.Type, ConnectionProfile.ConnectionTypeHandle> typeMapping;
         private final List<TcpChannel> channels;
         private final DiscoveryNode node;
-        private final Version version;
+        private final TransportVersion version;
         private final Compression.Enabled compress;
         private final Compression.Scheme compressionScheme;
         private final AtomicBoolean isClosing = new AtomicBoolean(false);
 
-        NodeChannels(DiscoveryNode node, List<TcpChannel> channels, ConnectionProfile connectionProfile, Version handshakeVersion) {
+        NodeChannels(
+            DiscoveryNode node,
+            List<TcpChannel> channels,
+            ConnectionProfile connectionProfile,
+            TransportVersion handshakeVersion
+        ) {
             this.node = node;
             this.channels = Collections.unmodifiableList(channels);
             assert channels.size() == connectionProfile.getNumConnections()
@@ -275,6 +283,13 @@ public final class NodeChannels extends CloseableConnection {
 
         @Override
         public Version getVersion() {
+            // TODO: this should be the below, but in some cases the node version does not match the passed-in version.
+            // return node.getVersion();
+            return Version.fromId(version.id);
+        }
+
+        @Override
+        public TransportVersion getTransportVersion() {
             return version;
         }
 
@@ -325,7 +340,7 @@ public void sendRequest(long requestId, String action, TransportRequest request,
             }
 
             final Compression.Scheme schemeToUse = getCompressionScheme(wrapped);
-            outboundHandler.sendRequest(node, channel, requestId, action, request, options, getVersion(), schemeToUse, false);
+            outboundHandler.sendRequest(node, channel, requestId, action, request, options, getTransportVersion(), schemeToUse, false);
         }
 
         private Compression.Scheme getCompressionScheme(TransportRequest request) {
@@ -388,7 +403,7 @@ private void initiateConnection(DiscoveryNode node, ConnectionProfile connection
 
         for (int i = 0; i < numConnections; ++i) {
             try {
-                final TcpChannel channel = initiateChannel(node);
+                final TcpChannel channel = initiateChannel(node, connectionProfile);
                 if (logger.isTraceEnabled()) {
                     channel.addConnectListener(new ChannelOpenTraceLogger(channel));
                 }
@@ -408,7 +423,7 @@ private void initiateConnection(DiscoveryNode node, ConnectionProfile connection
             node,
             connectionProfile,
             channels,
-            new ThreadedActionListener<>(logger, threadPool, ThreadPool.Names.GENERIC, listener, false)
+            new ThreadedActionListener<>(threadPool.generic(), listener)
         );
 
         for (TcpChannel channel : channels) {
@@ -424,6 +439,11 @@ public BoundTransportAddress boundAddress() {
         return this.boundAddress;
     }
 
+    @Override
+    public BoundTransportAddress boundRemoteIngressAddress() {
+        return profileBoundAddresses().get(REMOTE_CLUSTER_PROFILE);
+    }
+
     @Override
     public Map<String, BoundTransportAddress> profileBoundAddresses() {
         return unmodifiableMap(new HashMap<>(profileBoundAddresses));
@@ -787,10 +807,11 @@ protected void serverAcceptedChannel(TcpChannel channel) {
      * Initiate a single tcp socket channel.
      *
      * @param node for the initiated connection
+     * @param connectionProfile the connection profile to use when connecting to the node
      * @return the pending connection
      * @throws IOException if an I/O exception occurs while opening the channel
      */
-    protected abstract TcpChannel initiateChannel(DiscoveryNode node) throws IOException;
+    protected abstract TcpChannel initiateChannel(DiscoveryNode node, ConnectionProfile connectionProfile) throws IOException;
 
     /**
      * Called to tear down internal resources
@@ -933,7 +954,12 @@ public HttpRequestOnTransportException(StreamInput in) throws IOException {
         }
     }
 
-    public void executeHandshake(DiscoveryNode node, TcpChannel channel, ConnectionProfile profile, ActionListener<Version> listener) {
+    public void executeHandshake(
+        DiscoveryNode node,
+        TcpChannel channel,
+        ConnectionProfile profile,
+        ActionListener<TransportVersion> listener
+    ) {
         long requestId = responseHandlers.newRequestId();
         handshaker.sendHandshake(requestId, node, channel, profile.getHandshakeTimeout(), listener);
     }
@@ -993,6 +1019,11 @@ public final TransportStats getStats() {
      */
     public static Set<ProfileSettings> getProfileSettings(Settings settings) {
         HashSet<ProfileSettings> profiles = new HashSet<>();
+        // Process remote cluster port first so that errors are consistently reported if there
+        // is direct usage of the _remote_cluster profile
+        if (REMOTE_CLUSTER_PORT_ENABLED.get(settings)) {
+            profiles.add(RemoteClusterPortSettings.buildRemoteAccessProfileSettings(settings));
+        }
         boolean isDefaultSet = false;
         for (String profile : settings.getGroups("transport.profiles.", true).keySet()) {
             profiles.add(new ProfileSettings(settings, profile));
@@ -1003,6 +1034,7 @@ public static Set<ProfileSettings> getProfileSettings(Settings settings) {
         if (isDefaultSet == false) {
             profiles.add(new ProfileSettings(settings, TransportSettings.DEFAULT_PROFILE));
         }
+        // Add the remote access profile
         return Collections.unmodifiableSet(profiles);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java b/server/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java
index 96d655e44e8b..c3ecd0e93d49 100644
--- a/server/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java
+++ b/server/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.transport;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.core.Releasable;
 
 import java.io.IOException;
@@ -21,7 +21,7 @@ public final class TcpTransportChannel implements TransportChannel {
     private final TcpChannel channel;
     private final String action;
     private final long requestId;
-    private final Version version;
+    private final TransportVersion version;
     private final Compression.Scheme compressionScheme;
     private final boolean isHandshake;
     private final Releasable breakerRelease;
@@ -31,7 +31,7 @@ public final class TcpTransportChannel implements TransportChannel {
         TcpChannel channel,
         String action,
         long requestId,
-        Version version,
+        TransportVersion version,
         Compression.Scheme compressionScheme,
         boolean isHandshake,
         Releasable breakerRelease
@@ -88,7 +88,7 @@ public String getChannelType() {
     }
 
     @Override
-    public Version getVersion() {
+    public TransportVersion getVersion() {
         return version;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/transport/Transport.java b/server/src/main/java/org/elasticsearch/transport/Transport.java
index 23331e050af8..acc11177ac41 100644
--- a/server/src/main/java/org/elasticsearch/transport/Transport.java
+++ b/server/src/main/java/org/elasticsearch/transport/Transport.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.transport;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -54,6 +55,11 @@ default boolean isSecure() {
      */
     BoundTransportAddress boundAddress();
 
+    /**
+     * The address the Remote Access port is bound on, or <code>null</code> if it is not bound.
+     */
+    BoundTransportAddress boundRemoteIngressAddress();
+
     /**
      * Further profile bound addresses
      * @return <code>null</code> iff profiles are unsupported, otherwise a map with name of profile and its bound transport address
@@ -118,12 +124,19 @@ void sendRequest(long requestId, String action, TransportRequest request, Transp
         boolean isClosed();
 
         /**
-         * Returns the version of the node this connection was established with.
+         * Returns the version of the node on the other side of this channel.
          */
         default Version getVersion() {
             return getNode().getVersion();
         }
 
+        /**
+         * Returns the version of the data to communicate in this channel.
+         */
+        default TransportVersion getTransportVersion() {
+            return getVersion().transportVersion;
+        }
+
         /**
          * Returns a key that this connection can be cached on. Delegating subclasses must delegate method call to
          * the original connection.
diff --git a/server/src/main/java/org/elasticsearch/transport/TransportChannel.java b/server/src/main/java/org/elasticsearch/transport/TransportChannel.java
index 185b9ce6b88a..c56eca6a9e4a 100644
--- a/server/src/main/java/org/elasticsearch/transport/TransportChannel.java
+++ b/server/src/main/java/org/elasticsearch/transport/TransportChannel.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.transport;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 
 import java.io.IOException;
 
@@ -26,9 +26,9 @@ public interface TransportChannel {
     void sendResponse(Exception exception) throws IOException;
 
     /**
-     * Returns the version of the other party that this channel will send a response to.
+     * Returns the version of the data to communicate in this channel.
      */
-    default Version getVersion() {
-        return Version.CURRENT;
+    default TransportVersion getVersion() {
+        return TransportVersion.CURRENT;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/transport/TransportHandshaker.java b/server/src/main/java/org/elasticsearch/transport/TransportHandshaker.java
index ca64d15c986b..ce09fa9593a8 100644
--- a/server/src/main/java/org/elasticsearch/transport/TransportHandshaker.java
+++ b/server/src/main/java/org/elasticsearch/transport/TransportHandshaker.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.transport;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -26,22 +26,103 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
- * Sends and receives transport-level connection handshakes. This class will send the initial handshake,
- * manage state/timeouts while the handshake is in transit, and handle the eventual response.
+ * Sends and receives transport-level connection handshakes. This class will send the initial handshake, manage state/timeouts while the
+ * handshake is in transit, and handle the eventual response.
  */
 final class TransportHandshaker {
 
+    /*
+     * The transport-level handshake allows the node that opened the connection to determine the newest protocol version with which it can
+     * communicate with the remote node. Each node sends its maximum acceptable protocol version to the other, but the responding node
+     * ignores the body of the request. After the handshake, the OutboundHandler uses the min(local,remote) protocol version for all later
+     * messages.
+     *
+     * This version supports two handshake protocols, v6080099 and v7170099, which respectively have the same message structure as the
+     * transport protocols of v6.8.0 and v7.17.0. This node only sends v7170099 requests, but it can send a valid response to any v6080099
+     * requests that it receives.
+     *
+     * Here are some example messages, broken down to show their structure:
+     *
+     * ## v6080099 Request:
+     *
+     * 45 53                            -- 'ES' marker
+     * 00 00 00 34                      -- total message length
+     *    00 00 00 00 00 00 00 01       -- request ID
+     *    08                            -- status flags (0b1000 == handshake request)
+     *    00 5c c6 63                   -- handshake protocol version (0x5cc663 == 6080099)
+     *    00                            -- no request headers [1]
+     *    00                            -- no response headers [1]
+     *    01                            -- one feature [2]
+     *       06                         -- feature name length
+     *          78 2d 70 61 63 6b       -- feature name 'x-pack'
+     *    16                            -- action string size
+     *       69 6e 74 65 72 6e 61 6c    }
+     *       3a 74 63 70 2f 68 61 6e    }- ASCII representation of HANDSHAKE_ACTION_NAME
+     *       64 73 68 61 6b 65          }
+     *    00                            -- no parent task ID [3]
+     *    04                            -- payload length
+     *       8b d5 b5 03                -- max acceptable protocol version (vInt: 00000011 10110101 11010101 10001011 == 7170699)
+     *
+     * ## v6080099 Response:
+     *
+     * 45 53                            -- 'ES' marker
+     * 00 00 00 13                      -- total message length
+     *    00 00 00 00 00 00 00 01       -- request ID (copied from request)
+     *    09                            -- status flags (0b1001 == handshake response)
+     *    00 5c c6 63                   -- handshake protocol version (0x5cc663 == 6080099, copied from request)
+     *    00                            -- no request headers [1]
+     *    00                            -- no response headers [1]
+     *    c3 f9 eb 03                   -- max acceptable protocol version (vInt: 00000011 11101011 11111001 11000011 == 8060099)
+     *
+     *
+     * ## v7170099 Request:
+     *
+     * 45 53                            -- 'ES' marker
+     * 00 00 00 31                      -- total message length
+     *    00 00 00 00 00 00 00 01       -- request ID
+     *    08                            -- status flags (0b1000 == handshake request)
+     *    00 6d 68 33                   -- handshake protocol version (0x6d6833 == 7170099)
+     *    00 00 00 1a                   -- length of variable portion of header
+     *       00                         -- no request headers [1]
+     *       00                         -- no response headers [1]
+     *       00                         -- no features [2]
+     *       16                         -- action string size
+     *       69 6e 74 65 72 6e 61 6c    }
+     *       3a 74 63 70 2f 68 61 6e    }- ASCII representation of HANDSHAKE_ACTION_NAME
+     *       64 73 68 61 6b 65          }
+     *    00                            -- no parent task ID [3]
+     *    04                            -- payload length
+     *       c3 f9 eb 03                -- max acceptable protocol version (vInt: 00000011 11101011 11111001 11000011 == 8060099)
+     *
+     * ## v7170099 Response:
+     *
+     * 45 53                            -- 'ES' marker
+     * 00 00 00 17                      -- total message length
+     *    00 00 00 00 00 00 00 01       -- request ID (copied from request)
+     *    09                            -- status flags (0b1001 == handshake response)
+     *    00 6d 68 33                   -- handshake protocol version (0x6d6833 == 7170099, copied from request)
+     *    00 00 00 02                   -- length of following variable portion of header
+     *       00                         -- no request headers [1]
+     *       00                         -- no response headers [1]
+     *    c3 f9 eb 03                   -- max acceptable protocol version (vInt: 00000011 11101011 11111001 11000011 == 8060099)
+     *
+     * [1] Thread context headers should be empty; see org.elasticsearch.common.util.concurrent.ThreadContext.ThreadContextStruct.writeTo
+     *     for their structure.
+     * [2] A list of strings, which can safely be ignored
+     * [3] Parent task ID should be empty; see org.elasticsearch.tasks.TaskId.writeTo for its structure.
+     */
+
     static final String HANDSHAKE_ACTION_NAME = "internal:tcp/handshake";
     private final ConcurrentMap<Long, HandshakeResponseHandler> pendingHandshakes = new ConcurrentHashMap<>();
     private final CounterMetric numHandshakes = new CounterMetric();
 
-    private final Version version;
+    private final TransportVersion version;
     private final ThreadPool threadPool;
     private final HandshakeRequestSender handshakeRequestSender;
     private final boolean ignoreDeserializationErrors;
 
     TransportHandshaker(
-        Version version,
+        TransportVersion version,
         ThreadPool threadPool,
         HandshakeRequestSender handshakeRequestSender,
         boolean ignoreDeserializationErrors
@@ -52,7 +133,13 @@ final class TransportHandshaker {
         this.ignoreDeserializationErrors = ignoreDeserializationErrors;
     }
 
-    void sendHandshake(long requestId, DiscoveryNode node, TcpChannel channel, TimeValue timeout, ActionListener<Version> listener) {
+    void sendHandshake(
+        long requestId,
+        DiscoveryNode node,
+        TcpChannel channel,
+        TimeValue timeout,
+        ActionListener<TransportVersion> listener
+    ) {
         numHandshakes.inc();
         final HandshakeResponseHandler handler = new HandshakeResponseHandler(requestId, version, listener);
         pendingHandshakes.put(requestId, handler);
@@ -64,7 +151,7 @@ void sendHandshake(long requestId, DiscoveryNode node, TcpChannel channel, TimeV
             // for the request we use the minCompatVersion since we don't know what's the version of the node we talk to
             // we also have no payload on the request but the response will contain the actual version of the node we talk
             // to as the payload.
-            final Version minCompatVersion = version.minimumCompatibilityVersion();
+            TransportVersion minCompatVersion = version.calculateMinimumCompatVersion();
             handshakeRequestSender.sendRequest(node, channel, requestId, minCompatVersion);
 
             threadPool.schedule(
@@ -123,11 +210,11 @@ long getNumHandshakes() {
     private class HandshakeResponseHandler implements TransportResponseHandler<HandshakeResponse> {
 
         private final long requestId;
-        private final Version currentVersion;
-        private final ActionListener<Version> listener;
+        private final TransportVersion currentVersion;
+        private final ActionListener<TransportVersion> listener;
         private final AtomicBoolean isDone = new AtomicBoolean(false);
 
-        private HandshakeResponseHandler(long requestId, Version currentVersion, ActionListener<Version> listener) {
+        private HandshakeResponseHandler(long requestId, TransportVersion currentVersion, ActionListener<TransportVersion> listener) {
             this.requestId = requestId;
             this.currentVersion = currentVersion;
             this.listener = listener;
@@ -141,14 +228,14 @@ public HandshakeResponse read(StreamInput in) throws IOException {
         @Override
         public void handleResponse(HandshakeResponse response) {
             if (isDone.compareAndSet(false, true)) {
-                Version responseVersion = response.responseVersion;
+                TransportVersion responseVersion = response.responseVersion;
                 if (currentVersion.isCompatible(responseVersion) == false) {
                     listener.onFailure(
                         new IllegalStateException(
                             "Received message from unsupported version: ["
                                 + responseVersion
                                 + "] minimal compatible version is: ["
-                                + currentVersion.minimumCompatibilityVersion()
+                                + currentVersion.calculateMinimumCompatVersion()
                                 + "]"
                         )
                     );
@@ -174,9 +261,9 @@ void handleLocalException(TransportException e) {
 
     static final class HandshakeRequest extends TransportRequest {
 
-        private final Version version;
+        private final TransportVersion version;
 
-        HandshakeRequest(Version version) {
+        HandshakeRequest(TransportVersion version) {
             this.version = version;
         }
 
@@ -192,7 +279,7 @@ static final class HandshakeRequest extends TransportRequest {
                 version = null;
             } else {
                 try (StreamInput messageStreamInput = remainingMessage.streamInput()) {
-                    this.version = Version.readVersion(messageStreamInput);
+                    this.version = TransportVersion.readVersion(messageStreamInput);
                 }
             }
         }
@@ -202,7 +289,7 @@ public void writeTo(StreamOutput streamOutput) throws IOException {
             super.writeTo(streamOutput);
             assert version != null;
             try (BytesStreamOutput messageStreamOutput = new BytesStreamOutput(4)) {
-                Version.writeVersion(version, messageStreamOutput);
+                TransportVersion.writeVersion(version, messageStreamOutput);
                 BytesReference reference = messageStreamOutput.bytes();
                 streamOutput.writeBytesReference(reference);
             }
@@ -211,24 +298,24 @@ public void writeTo(StreamOutput streamOutput) throws IOException {
 
     static final class HandshakeResponse extends TransportResponse {
 
-        private final Version responseVersion;
+        private final TransportVersion responseVersion;
 
-        HandshakeResponse(Version responseVersion) {
+        HandshakeResponse(TransportVersion responseVersion) {
             this.responseVersion = responseVersion;
         }
 
         private HandshakeResponse(StreamInput in) throws IOException {
             super(in);
-            responseVersion = Version.readVersion(in);
+            responseVersion = TransportVersion.readVersion(in);
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             assert responseVersion != null;
-            Version.writeVersion(responseVersion, out);
+            TransportVersion.writeVersion(responseVersion, out);
         }
 
-        Version getResponseVersion() {
+        TransportVersion getResponseVersion() {
             return responseVersion;
         }
     }
@@ -236,6 +323,6 @@ Version getResponseVersion() {
     @FunctionalInterface
     interface HandshakeRequestSender {
 
-        void sendRequest(DiscoveryNode node, TcpChannel channel, long requestId, Version version) throws IOException;
+        void sendRequest(DiscoveryNode node, TcpChannel channel, long requestId, TransportVersion version) throws IOException;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/transport/TransportLogger.java b/server/src/main/java/org/elasticsearch/transport/TransportLogger.java
index 34b6720d10e3..c4b0381ef452 100644
--- a/server/src/main/java/org/elasticsearch/transport/TransportLogger.java
+++ b/server/src/main/java/org/elasticsearch/transport/TransportLogger.java
@@ -9,7 +9,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressorFactory;
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
@@ -77,8 +77,8 @@ private static String format(TcpChannel channel, BytesReference message, String
                 final byte status = streamInput.readByte();
                 final boolean isRequest = TransportStatus.isRequest(status);
                 final String type = isRequest ? "request" : "response";
-                final Version version = Version.fromId(streamInput.readInt());
-                streamInput.setVersion(version);
+                final TransportVersion version = TransportVersion.fromId(streamInput.readInt());
+                streamInput.setTransportVersion(version);
                 sb.append(" [length: ").append(messageLengthWithHeader);
                 sb.append(", request id: ").append(requestId);
                 sb.append(", type: ").append(type);
@@ -95,7 +95,7 @@ private static String format(TcpChannel channel, BytesReference message, String
                 ThreadContext.readHeadersFromStream(streamInput);
 
                 if (isRequest) {
-                    if (version.before(Version.V_8_0_0)) {
+                    if (version.before(TransportVersion.V_8_0_0)) {
                         // discard features
                         streamInput.readStringArray();
                     }
diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java
index df7b15f817c7..f46b4aaddc11 100644
--- a/server/src/main/java/org/elasticsearch/transport/TransportService.java
+++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java
@@ -11,6 +11,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Build;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionListenerResponseHandler;
@@ -415,6 +416,10 @@ public BoundTransportAddress boundAddress() {
         return transport.boundAddress();
     }
 
+    public BoundTransportAddress boundRemoteAccessAddress() {
+        return transport.boundRemoteIngressAddress();
+    }
+
     public List<String> getDefaultSeedAddresses() {
         return transport.getDefaultSeedAddresses();
     }
@@ -1387,6 +1392,10 @@ void setTimeoutHandler(TimeoutHandler timeoutHandler) {
             this.handler = timeoutHandler;
         }
 
+        // for tests
+        TransportResponseHandler<T> unwrap() {
+            return delegate;
+        }
     }
 
     static class DirectResponseChannel implements TransportChannel {
@@ -1513,8 +1522,8 @@ public String getChannelType() {
         }
 
         @Override
-        public Version getVersion() {
-            return localNode.getVersion();
+        public TransportVersion getVersion() {
+            return localNode.getVersion().transportVersion;
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/transport/TransportStats.java b/server/src/main/java/org/elasticsearch/transport/TransportStats.java
index fe0f1742f9eb..694fb0628d14 100644
--- a/server/src/main/java/org/elasticsearch/transport/TransportStats.java
+++ b/server/src/main/java/org/elasticsearch/transport/TransportStats.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.transport;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -59,7 +60,7 @@ public TransportStats(StreamInput in) throws IOException {
         rxSize = in.readVLong();
         txCount = in.readVLong();
         txSize = in.readVLong();
-        if (in.getVersion().onOrAfter(Version.V_8_1_0) && in.readBoolean()) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0) && in.readBoolean()) {
             inboundHandlingTimeBucketFrequencies = new long[HandlingTimeTracker.BUCKET_COUNT];
             for (int i = 0; i < inboundHandlingTimeBucketFrequencies.length; i++) {
                 inboundHandlingTimeBucketFrequencies[i] = in.readVLong();
@@ -83,7 +84,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(rxSize);
         out.writeVLong(txCount);
         out.writeVLong(txSize);
-        if (out.getVersion().onOrAfter(Version.V_8_1_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
             assert (inboundHandlingTimeBucketFrequencies.length > 0) == (outboundHandlingTimeBucketFrequencies.length > 0);
             out.writeBoolean(inboundHandlingTimeBucketFrequencies.length > 0);
             for (long handlingTimeBucketFrequency : inboundHandlingTimeBucketFrequencies) {
diff --git a/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java b/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java
index 5fda8daf0038..a83d673bbc45 100644
--- a/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java
+++ b/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.upgrades;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.DiffableUtils;
 import org.elasticsearch.cluster.NamedDiff;
@@ -16,16 +16,18 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -38,7 +40,7 @@
  */
 public class FeatureMigrationResults implements Metadata.Custom {
     public static final String TYPE = "system_index_migration";
-    public static final Version MIGRATION_ADDED_VERSION = Version.V_8_0_0;
+    public static final TransportVersion MIGRATION_ADDED_VERSION = TransportVersion.V_8_0_0;
 
     private static final ParseField RESULTS_FIELD = new ParseField("results");
 
@@ -78,9 +80,8 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(RESULTS_FIELD.getPreferredName(), featureStatuses);
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return ChunkedToXContentHelper.xContentValuesMap(RESULTS_FIELD.getPreferredName(), featureStatuses);
     }
 
     public static FeatureMigrationResults fromXContent(XContentParser parser) {
@@ -121,7 +122,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
+    public TransportVersion getMinimalSupportedVersion() {
         return MIGRATION_ADDED_VERSION;
     }
 
@@ -191,7 +192,7 @@ static Diff<SingleFeatureMigrationResult> readDiffFrom(StreamInput in) throws IO
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
+        public TransportVersion getMinimalSupportedVersion() {
             return MIGRATION_ADDED_VERSION;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParams.java b/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParams.java
index 001ce91f5afd..b574e5bb895e 100644
--- a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParams.java
+++ b/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParams.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.upgrades;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.persistent.PersistentTaskParams;
@@ -68,7 +68,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
+    public TransportVersion getMinimalSupportedVersion() {
         return MIGRATION_ADDED_VERSION;
     }
 
diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat
new file mode 100644
index 000000000000..e3165f3c425f
--- /dev/null
+++ b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat
@@ -0,0 +1 @@
+org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat
diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
index 91890d721b27..119a70883294 100644
--- a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
+++ b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
@@ -1 +1,2 @@
 org.elasticsearch.index.codec.bloomfilter.ES85BloomFilterPostingsFormat
+org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat
diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy
index 3b21c273d3a1..71bcaba7580e 100644
--- a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy
+++ b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy
@@ -28,6 +28,9 @@ grant codeBase "${codebase.elasticsearch}" {
   // for module layer
   permission java.lang.RuntimePermission "createClassLoader";
   permission java.lang.RuntimePermission "getClassLoader";
+
+  // for plugin api dynamic settings instances
+  permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.reflect";
 };
 
 //// Very special jar permissions:
diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
index 8907c4d463d3..ecd49fa4faea 100644
--- a/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
+++ b/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
@@ -83,6 +83,7 @@ grant codeBase "${codebase.mocksocket}" {
 grant codeBase "${codebase.elasticsearch}" {
   permission java.lang.RuntimePermission "setSecurityManager";
 };
+
 // this is the test-framework, but the jar is horribly named
 grant codeBase "${codebase.framework}" {
   permission java.lang.RuntimePermission "setSecurityManager";
@@ -125,4 +126,6 @@ grant codeBase "${codebase.netty-transport}" {
 grant {
   permission java.net.SocketPermission "127.0.0.1", "connect, resolve";
   permission java.nio.file.LinkPermission "symbolic";
+  // needed for keystore tests
+  permission java.lang.RuntimePermission "accessUserInformation";
 };
diff --git a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json
new file mode 100644
index 000000000000..cac82799c50a
--- /dev/null
+++ b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json
@@ -0,0 +1,9 @@
+{
+  "INITIAL_MASTER_NODES": "important-settings.html#initial_master_nodes",
+  "DISCOVERY_TROUBLESHOOTING": "discovery-troubleshooting.html",
+  "UNSTABLE_CLUSTER_TROUBLESHOOTING": "cluster-fault-detection.html#cluster-fault-detection-troubleshooting",
+  "LAGGING_NODE_TROUBLESHOOTING": "cluster-fault-detection.html#_diagnosing_lagging_nodes",
+  "SHARD_LOCK_TROUBLESHOOTING": "cluster-fault-detection.html#_diagnosing_shardlockobtainfailedexception_failures",
+  "CONCURRENT_REPOSITORY_WRITERS": "add-repository.html",
+  "ARCHIVE_INDICES": "archive-indices.html"
+}
diff --git a/server/src/main/resources/org/elasticsearch/plugins/scanners/extensibles.json b/server/src/main/resources/org/elasticsearch/plugins/scanners/extensibles.json
index 64d123055df9..b196de2bddfa 100644
--- a/server/src/main/resources/org/elasticsearch/plugins/scanners/extensibles.json
+++ b/server/src/main/resources/org/elasticsearch/plugins/scanners/extensibles.json
@@ -1,6 +1,6 @@
 {
-  "org.elasticsearch.plugin.analysis.api.AnalyzerFactory":"org.elasticsearch.plugin.analysis.api.AnalyzerFactory",
-  "org.elasticsearch.plugin.analysis.api.CharFilterFactory":"org.elasticsearch.plugin.analysis.api.CharFilterFactory",
-  "org.elasticsearch.plugin.analysis.api.TokenFilterFactory":"org.elasticsearch.plugin.analysis.api.TokenFilterFactory",
-  "org.elasticsearch.plugin.analysis.api.TokenizerFactory":"org.elasticsearch.plugin.analysis.api.TokenizerFactory"
+  "org.elasticsearch.plugin.analysis.AnalyzerFactory":"org.elasticsearch.plugin.analysis.api.AnalyzerFactory",
+  "org.elasticsearch.plugin.analysis.CharFilterFactory":"org.elasticsearch.plugin.analysis.api.CharFilterFactory",
+  "org.elasticsearch.plugin.analysis.TokenFilterFactory":"org.elasticsearch.plugin.analysis.api.TokenFilterFactory",
+  "org.elasticsearch.plugin.analysis.TokenizerFactory":"org.elasticsearch.plugin.analysis.api.TokenizerFactory"
 }
diff --git a/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java b/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java
index 1c89e93467ad..1c54ad4e6b94 100644
--- a/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java
+++ b/server/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java
@@ -37,7 +37,6 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.script.ScriptException;
 import org.elasticsearch.search.SearchContextMissingException;
-import org.elasticsearch.search.SearchParseException;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.ShardSearchContextId;
 import org.elasticsearch.test.ESTestCase;
@@ -46,7 +45,6 @@
 import org.elasticsearch.xcontent.XContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
-import org.elasticsearch.xcontent.XContentLocation;
 import org.elasticsearch.xcontent.XContentParseException;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
@@ -63,7 +61,6 @@
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singleton;
 import static java.util.Collections.singletonList;
-import static org.elasticsearch.test.TestSearchContext.SHARD_TARGET;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
 import static org.hamcrest.CoreMatchers.hasItem;
 import static org.hamcrest.CoreMatchers.hasItems;
@@ -543,12 +540,6 @@ public void testToXContent() throws IOException {
                   }
                 }""");
         }
-        {
-            ElasticsearchException e = new SearchParseException(SHARD_TARGET, "foo", new XContentLocation(1, 0));
-
-            assertExceptionAsJson(e, """
-                {"type":"search_parse_exception","reason":"foo","line":1,"col":0}""");
-        }
         {
             ElasticsearchException ex = new ElasticsearchException(
                 "foo",
@@ -597,9 +588,10 @@ public void testToXContent() throws IOException {
                 builder.endObject();
                 actual = Strings.toString(builder);
             }
-            assertThat(actual, startsWith(formatted("""
+            Object[] args = new Object[] { Constants.WINDOWS ? "\\r\\n" : "\\n" };
+            assertThat(actual, startsWith(Strings.format("""
                 {"type":"exception","reason":"foo","caused_by":{"type":"illegal_state_exception","reason":"bar",\
-                "stack_trace":"java.lang.IllegalStateException: bar%s\\tat org.elasticsearch.""", Constants.WINDOWS ? "\\r\\n" : "\\n")));
+                "stack_trace":"java.lang.IllegalStateException: bar%s\\tat org.elasticsearch.""", args)));
         }
     }
 
@@ -1202,7 +1194,7 @@ public static Tuple<Throwable, ElasticsearchException> randomExceptions() {
         Throwable actual;
         ElasticsearchException expected;
 
-        int type = randomIntBetween(0, 5);
+        int type = randomIntBetween(0, 4);
         switch (type) {
             case 0 -> {
                 actual = new ClusterBlockException(singleton(NoMasterBlockService.NO_MASTER_BLOCK_WRITES));
@@ -1215,17 +1207,13 @@ public static Tuple<Throwable, ElasticsearchException> randomExceptions() {
                 expected = new ElasticsearchException("Elasticsearch exception [type=parsing_exception, reason=Unknown identifier]");
             }
             case 2 -> {
-                actual = new SearchParseException(SHARD_TARGET, "Parse failure", new XContentLocation(12, 98));
-                expected = new ElasticsearchException("Elasticsearch exception [type=search_parse_exception, reason=Parse failure]");
-            }
-            case 3 -> {
                 actual = new IllegalArgumentException("Closed resource", new RuntimeException("Resource"));
                 expected = new ElasticsearchException(
                     "Elasticsearch exception [type=illegal_argument_exception, reason=Closed resource]",
                     new ElasticsearchException("Elasticsearch exception [type=runtime_exception, reason=Resource]")
                 );
             }
-            case 4 -> {
+            case 3 -> {
                 actual = new SearchPhaseExecutionException(
                     "search",
                     "all shards failed",
@@ -1240,7 +1228,7 @@ public static Tuple<Throwable, ElasticsearchException> randomExceptions() {
                 );
                 expected.addMetadata("es.phase", "search");
             }
-            case 5 -> {
+            case 4 -> {
                 actual = new ElasticsearchException(
                     "Parsing failed",
                     new ParsingException(9, 42, "Wrong state", new NullPointerException("Unexpected null value"))
diff --git a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
index 270d79afb20a..e57d91c859e2 100644
--- a/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
@@ -69,7 +69,6 @@
 import org.elasticsearch.rest.action.admin.indices.AliasesNotFoundException;
 import org.elasticsearch.search.SearchContextMissingException;
 import org.elasticsearch.search.SearchException;
-import org.elasticsearch.search.SearchParseException;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
 import org.elasticsearch.search.aggregations.UnsupportedAggregationOnDownsampledIndex;
@@ -80,6 +79,7 @@
 import org.elasticsearch.snapshots.SnapshotInProgressException;
 import org.elasticsearch.snapshots.SnapshotNameAlreadyInUseException;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.transport.ActionNotFoundTransportException;
 import org.elasticsearch.transport.ActionTransportException;
@@ -87,7 +87,6 @@
 import org.elasticsearch.transport.NoSeedNodeLeftException;
 import org.elasticsearch.transport.NoSuchRemoteClusterException;
 import org.elasticsearch.transport.TcpTransport;
-import org.elasticsearch.xcontent.XContentLocation;
 
 import java.io.EOFException;
 import java.io.FileNotFoundException;
@@ -118,7 +117,6 @@
 import static java.util.Collections.emptySet;
 import static java.util.Collections.singleton;
 import static org.elasticsearch.cluster.routing.TestShardRouting.newShardRouting;
-import static org.elasticsearch.test.TestSearchContext.SHARD_TARGET;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
@@ -170,10 +168,10 @@ private void checkClass(Class<?> clazz) {
                 if (isEsException(clazz) == false) {
                     return;
                 }
-                if (ElasticsearchException.isRegistered(clazz.asSubclass(Throwable.class), Version.CURRENT) == false
+                if (ElasticsearchException.isRegistered(clazz.asSubclass(Throwable.class), TransportVersion.CURRENT) == false
                     && ElasticsearchException.class.equals(clazz.getEnclosingClass()) == false) {
                     notRegistered.add(clazz);
-                } else if (ElasticsearchException.isRegistered(clazz.asSubclass(Throwable.class), Version.CURRENT)) {
+                } else if (ElasticsearchException.isRegistered(clazz.asSubclass(Throwable.class), TransportVersion.CURRENT)) {
                     registered.add(clazz);
                     try {
                         if (clazz.getMethod("writeTo", StreamOutput.class) != null) {
@@ -227,16 +225,16 @@ public TestException(StreamInput in) throws IOException {
     }
 
     private <T extends Exception> T serialize(T exception) throws IOException {
-        return serialize(exception, VersionUtils.randomVersion(random()));
+        return serialize(exception, TransportVersionUtils.randomVersion(random()));
     }
 
-    private <T extends Exception> T serialize(T exception, Version version) throws IOException {
+    private <T extends Exception> T serialize(T exception, TransportVersion version) throws IOException {
         BytesStreamOutput out = new BytesStreamOutput();
-        out.setVersion(version);
+        out.setTransportVersion(version);
         out.writeException(exception);
 
         StreamInput in = out.bytes().streamInput();
-        in.setVersion(version);
+        in.setTransportVersion(version);
         return in.readException();
     }
 
@@ -350,10 +348,10 @@ public void testActionTransportException() throws IOException {
 
     public void testSearchContextMissingException() throws IOException {
         ShardSearchContextId contextId = new ShardSearchContextId(UUIDs.randomBase64UUID(), randomLong());
-        Version version = VersionUtils.randomVersion(random());
+        TransportVersion version = TransportVersionUtils.randomVersion(random());
         SearchContextMissingException ex = serialize(new SearchContextMissingException(contextId), version);
         assertThat(ex.contextId().getId(), equalTo(contextId.getId()));
-        if (version.onOrAfter(Version.V_7_7_0)) {
+        if (version.onOrAfter(TransportVersion.V_7_7_0)) {
             assertThat(ex.contextId().getSessionId(), equalTo(contextId.getSessionId()));
         } else {
             assertThat(ex.contextId().getSessionId(), equalTo(""));
@@ -363,7 +361,7 @@ public void testSearchContextMissingException() throws IOException {
     public void testCircuitBreakingException() throws IOException {
         CircuitBreakingException ex = serialize(
             new CircuitBreakingException("Too large", 0, 100, CircuitBreaker.Durability.TRANSIENT),
-            Version.V_7_0_0
+            TransportVersion.V_7_0_0
         );
         assertEquals("Too large", ex.getMessage());
         assertEquals(100, ex.getByteLimit());
@@ -372,7 +370,7 @@ public void testCircuitBreakingException() throws IOException {
     }
 
     public void testTooManyBucketsException() throws IOException {
-        Version version = VersionUtils.randomCompatibleVersion(random(), Version.CURRENT);
+        TransportVersion version = TransportVersionUtils.randomCompatibleVersion(random(), TransportVersion.CURRENT);
         MultiBucketConsumerService.TooManyBucketsException ex = serialize(
             new MultiBucketConsumerService.TooManyBucketsException("Too many buckets", 100),
             version
@@ -394,13 +392,6 @@ public void testAliasesMissingException() throws IOException {
         assertArrayEquals(new String[] { "one", "two", "three" }, ex.getResourceId().toArray(new String[0]));
     }
 
-    public void testSearchParseException() throws IOException {
-        SearchParseException ex = serialize(new SearchParseException(SHARD_TARGET, "foo", new XContentLocation(66, 666)));
-        assertEquals("foo", ex.getMessage());
-        assertEquals(66, ex.getLineNumber());
-        assertEquals(666, ex.getColumnNumber());
-    }
-
     public void testIllegalIndexShardStateException() throws IOException {
         ShardId id = new ShardId("foo", "_na_", 1);
         IndexShardState state = randomFrom(IndexShardState.values());
@@ -740,7 +731,7 @@ public void testIds() {
         ids.put(69, org.elasticsearch.snapshots.SnapshotMissingException.class);
         ids.put(70, org.elasticsearch.action.PrimaryMissingActionException.class);
         ids.put(71, org.elasticsearch.action.FailedNodeException.class);
-        ids.put(72, org.elasticsearch.search.SearchParseException.class);
+        ids.put(72, null); // was SearchParseException, only used in tests since 7.11
         ids.put(73, org.elasticsearch.snapshots.ConcurrentSnapshotExecutionException.class);
         ids.put(74, org.elasticsearch.common.blobstore.BlobStoreException.class);
         ids.put(75, org.elasticsearch.cluster.IncompatibleClusterStateVersionException.class);
@@ -897,7 +888,7 @@ public void testShardLockObtainFailedException() throws IOException {
         ShardId shardId = new ShardId("foo", "_na_", 1);
         ShardLockObtainFailedException orig = new ShardLockObtainFailedException(shardId, "boom");
         Version version = VersionUtils.randomIndexCompatibleVersion(random());
-        ShardLockObtainFailedException ex = serialize(orig, version);
+        ShardLockObtainFailedException ex = serialize(orig, version.transportVersion);
         assertEquals(orig.getMessage(), ex.getMessage());
         assertEquals(orig.getShardId(), ex.getShardId());
     }
@@ -905,7 +896,7 @@ public void testShardLockObtainFailedException() throws IOException {
     public void testSnapshotInProgressException() throws IOException {
         SnapshotInProgressException orig = new SnapshotInProgressException("boom");
         Version version = VersionUtils.randomIndexCompatibleVersion(random());
-        SnapshotInProgressException ex = serialize(orig, version);
+        SnapshotInProgressException ex = serialize(orig, version.transportVersion);
         assertEquals(orig.getMessage(), ex.getMessage());
     }
 
diff --git a/server/src/test/java/org/elasticsearch/TransportVersionTests.java b/server/src/test/java/org/elasticsearch/TransportVersionTests.java
new file mode 100644
index 000000000000..1d995319b0f5
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/TransportVersionTests.java
@@ -0,0 +1,186 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.TransportVersionUtils;
+
+import java.lang.reflect.Modifier;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.sameInstance;
+
+public class TransportVersionTests extends ESTestCase {
+
+    public void testVersionComparison() {
+        TransportVersion V_7_2_0 = TransportVersion.V_7_2_0;
+        TransportVersion V_8_0_0 = TransportVersion.V_8_0_0;
+        assertThat(V_7_2_0.before(V_8_0_0), is(true));
+        assertThat(V_7_2_0.before(V_7_2_0), is(false));
+        assertThat(V_8_0_0.before(V_7_2_0), is(false));
+
+        assertThat(V_7_2_0.onOrBefore(V_8_0_0), is(true));
+        assertThat(V_7_2_0.onOrBefore(V_7_2_0), is(true));
+        assertThat(V_8_0_0.onOrBefore(V_7_2_0), is(false));
+
+        assertThat(V_7_2_0.after(V_8_0_0), is(false));
+        assertThat(V_7_2_0.after(V_7_2_0), is(false));
+        assertThat(V_8_0_0.after(V_7_2_0), is(true));
+
+        assertThat(V_7_2_0.onOrAfter(V_8_0_0), is(false));
+        assertThat(V_7_2_0.onOrAfter(V_7_2_0), is(true));
+        assertThat(V_8_0_0.onOrAfter(V_7_2_0), is(true));
+
+        assertThat(V_7_2_0, is(lessThan(V_8_0_0)));
+        assertThat(V_7_2_0.compareTo(V_7_2_0), is(0));
+        assertThat(V_8_0_0, is(greaterThan(V_7_2_0)));
+    }
+
+    private static String padNumber(String number) {
+        return number.length() == 1 ? "0" + number : number;
+    }
+
+    public static class CorrectFakeVersion {
+        public static final TransportVersion V1 = new TransportVersion(1, "1");
+        public static final TransportVersion V2 = new TransportVersion(2, "2");
+        public static final TransportVersion V3 = new TransportVersion(3, "3");
+    }
+
+    public static class DuplicatedIdFakeVersion {
+        public static final TransportVersion V1 = new TransportVersion(1, "1");
+        public static final TransportVersion V2 = new TransportVersion(2, "2");
+        public static final TransportVersion V3 = new TransportVersion(2, "3");
+    }
+
+    public static class DuplicatedStringIdFakeVersion {
+        public static final TransportVersion V1 = new TransportVersion(1, "1");
+        public static final TransportVersion V2 = new TransportVersion(2, "2");
+        public static final TransportVersion V3 = new TransportVersion(3, "2");
+    }
+
+    public void testStaticTransportVersionChecks() {
+        assertThat(
+            TransportVersion.getAllVersionIds(CorrectFakeVersion.class),
+            equalTo(Map.of(1, CorrectFakeVersion.V1, 2, CorrectFakeVersion.V2, 3, CorrectFakeVersion.V3))
+        );
+        expectThrows(AssertionError.class, () -> TransportVersion.getAllVersionIds(DuplicatedIdFakeVersion.class));
+        expectThrows(AssertionError.class, () -> TransportVersion.getAllVersionIds(DuplicatedStringIdFakeVersion.class));
+    }
+
+    public void testDefinedConstants() throws IllegalAccessException {
+        Pattern historicalVersion = Pattern.compile("^V_(\\d{1,2})_(\\d{1,2})_(\\d{1,2})$");
+        Pattern transportVersion = Pattern.compile("^V_(\\d{2,})_(\\d{3})_(\\d{3})$");
+        Set<String> ignore = Set.of("ZERO", "CURRENT", "MINIMUM_COMPATIBLE");
+
+        for (java.lang.reflect.Field field : TransportVersion.class.getFields()) {
+            if (Modifier.isStatic(field.getModifiers())
+                && field.getType() == TransportVersion.class
+                && ignore.contains(field.getName()) == false) {
+                Matcher historical = historicalVersion.matcher(field.getName());
+                Matcher transport;
+                if (historical.matches()) {
+                    // old-style version constant
+                    String idString = historical.group(1) + padNumber(historical.group(2)) + padNumber(historical.group(3)) + "99";
+                    assertEquals(
+                        "Field " + field.getName() + " does not have expected id " + idString,
+                        idString,
+                        field.get(null).toString()
+                    );
+                } else if ((transport = transportVersion.matcher(field.getName())).matches()) {
+                    String idString = transport.group(1) + transport.group(2) + transport.group(3);
+                    assertEquals(
+                        "Field " + field.getName() + " does not have expected id " + idString,
+                        idString,
+                        field.get(null).toString()
+                    );
+                } else {
+                    fail("Field " + field.getName() + " does not have expected format");
+                }
+            }
+        }
+    }
+
+    public void testMin() {
+        assertEquals(
+            TransportVersionUtils.getPreviousVersion(),
+            TransportVersion.min(TransportVersion.CURRENT, TransportVersionUtils.getPreviousVersion())
+        );
+        assertEquals(
+            TransportVersion.fromId(1_01_01_99),
+            TransportVersion.min(TransportVersion.fromId(1_01_01_99), TransportVersion.CURRENT)
+        );
+        TransportVersion version = TransportVersionUtils.randomVersion();
+        TransportVersion version1 = TransportVersionUtils.randomVersion();
+        if (version.id <= version1.id) {
+            assertEquals(version, TransportVersion.min(version1, version));
+        } else {
+            assertEquals(version1, TransportVersion.min(version1, version));
+        }
+    }
+
+    public void testMax() {
+        assertEquals(TransportVersion.CURRENT, TransportVersion.max(TransportVersion.CURRENT, TransportVersionUtils.getPreviousVersion()));
+        assertEquals(TransportVersion.CURRENT, TransportVersion.max(TransportVersion.fromId(1_01_01_99), TransportVersion.CURRENT));
+        TransportVersion version = TransportVersionUtils.randomVersion();
+        TransportVersion version1 = TransportVersionUtils.randomVersion();
+        if (version.id >= version1.id) {
+            assertEquals(version, TransportVersion.max(version1, version));
+        } else {
+            assertEquals(version1, TransportVersion.max(version1, version));
+        }
+    }
+
+    public void testVersionConstantPresent() {
+        // TODO those versions are not cached at the moment, perhaps we should add them to idToVersion set too?
+        Set<TransportVersion> ignore = Set.of(TransportVersion.ZERO, TransportVersion.CURRENT, TransportVersion.MINIMUM_COMPATIBLE);
+        assertThat(TransportVersion.CURRENT, sameInstance(TransportVersion.fromId(TransportVersion.CURRENT.id)));
+        final int iters = scaledRandomIntBetween(20, 100);
+        for (int i = 0; i < iters; i++) {
+            TransportVersion version = TransportVersionUtils.randomVersion(ignore);
+
+            assertThat(version, sameInstance(TransportVersion.fromId(version.id)));
+        }
+    }
+
+    public void testCURRENTIsLatest() {
+        final int iters = scaledRandomIntBetween(100, 1000);
+        for (int i = 0; i < iters; i++) {
+            TransportVersion version = TransportVersionUtils.randomVersion();
+            if (version != TransportVersion.CURRENT) {
+                assertThat(
+                    "Version: " + version + " should be before: " + Version.CURRENT + " but wasn't",
+                    version.before(TransportVersion.CURRENT),
+                    is(true)
+                );
+            }
+        }
+    }
+
+    public void testToString() {
+        assertEquals("5000099", TransportVersion.fromId(5_00_00_99).toString());
+        assertEquals("2030099", TransportVersion.fromId(2_03_00_99).toString());
+        assertEquals("1000099", TransportVersion.fromId(1_00_00_99).toString());
+        assertEquals("2000099", TransportVersion.fromId(2_00_00_99).toString());
+        assertEquals("5000099", TransportVersion.fromId(5_00_00_99).toString());
+    }
+
+    public void testMinCompatVersion() {
+        Version minCompat = Version.fromId(TransportVersion.V_8_0_0.calculateMinimumCompatVersion().id);
+        assertEquals(7, minCompat.major);
+        assertEquals("This needs to be updated when 7.18 is released", 17, minCompat.minor);
+        assertEquals(0, minCompat.revision);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/VersionTests.java b/server/src/test/java/org/elasticsearch/VersionTests.java
index 72e958a5a3ed..26a5415473d0 100644
--- a/server/src/test/java/org/elasticsearch/VersionTests.java
+++ b/server/src/test/java/org/elasticsearch/VersionTests.java
@@ -34,7 +34,7 @@
 
 public class VersionTests extends ESTestCase {
 
-    public void testVersionComparison() throws Exception {
+    public void testVersionComparison() {
         Version V_7_2_0 = Version.fromString("7.2.0");
         Version V_8_0_0 = Version.fromString("8.0.0");
         assertThat(V_7_2_0.before(V_8_0_0), is(true));
@@ -178,7 +178,7 @@ public void testParseVersion() {
         for (int i = 0; i < iters; i++) {
             Version version = randomVersion(random());
             if (random().nextBoolean()) {
-                version = new Version(version.id, version.luceneVersion);
+                version = new Version(version.id, version.transportVersion, version.luceneVersion);
             }
             Version parsedVersion = Version.fromString(version.toString());
             assertEquals(version, parsedVersion);
diff --git a/server/src/test/java/org/elasticsearch/action/ActionListenerTests.java b/server/src/test/java/org/elasticsearch/action/ActionListenerTests.java
index 84fb08f03adc..70f11037fe86 100644
--- a/server/src/test/java/org/elasticsearch/action/ActionListenerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ActionListenerTests.java
@@ -7,43 +7,146 @@
  */
 package org.elasticsearch.action;
 
+import org.apache.lucene.store.AlreadyClosedException;
+import org.elasticsearch.Assertions;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.core.CheckedRunnable;
+import org.elasticsearch.core.Releasable;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ReachabilityChecker;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
 
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
 public class ActionListenerTests extends ESTestCase {
 
-    public void testWrap() {
+    public void testWrapConsumers() {
         AtomicReference<Boolean> reference = new AtomicReference<>();
         AtomicReference<Exception> exReference = new AtomicReference<>();
 
-        CheckedConsumer<Boolean, ? extends Exception> handler = (o) -> {
-            if (Boolean.FALSE.equals(o)) {
-                throw new IllegalArgumentException("must not be false");
+        ActionListener<Boolean> wrap = ActionListener.wrap(new CheckedConsumer<>() {
+            @Override
+            public void accept(Boolean o) {
+                if (Boolean.FALSE.equals(o)) {
+                    throw new IllegalArgumentException("must not be false");
+                }
+                reference.set(o);
             }
-            reference.set(o);
-        };
-        ActionListener<Boolean> wrap = ActionListener.wrap(handler, exReference::set);
+
+            @Override
+            public String toString() {
+                return "test handler";
+            }
+        }, new Consumer<>() {
+            @Override
+            public void accept(Exception newValue) {
+                exReference.set(newValue);
+            }
+
+            @Override
+            public String toString() {
+                return "test exception handler";
+            }
+        });
+
+        assertEquals("WrappedActionListener{test handler}{test exception handler}", wrap.toString());
+
         wrap.onResponse(Boolean.FALSE);
-        assertNull(reference.get());
-        assertNotNull(exReference.get());
-        assertEquals("must not be false", exReference.get().getMessage());
-        exReference.set(null);
+        assertNull(reference.getAndSet(null));
+        assertEquals("must not be false", exReference.getAndSet(null).getMessage());
 
         wrap.onResponse(Boolean.TRUE);
-        assertTrue(reference.get());
-        assertNull(exReference.get());
+        assertTrue(reference.getAndSet(null));
+        assertNull(exReference.getAndSet(null));
+
+        wrap.onFailure(new RuntimeException("test exception"));
+        assertNull(reference.getAndSet(null));
+        assertEquals("test exception", exReference.getAndSet(null).getMessage());
+    }
+
+    public void testWrapRunnable() {
+        var executed = new AtomicBoolean();
+        var listener = ActionListener.wrap(new Runnable() {
+            @Override
+            public void run() {
+                assertTrue(executed.compareAndSet(false, true));
+            }
+
+            @Override
+            public String toString() {
+                return "test runnable";
+            }
+        });
+
+        assertEquals("RunnableWrappingActionListener{test runnable}", listener.toString());
+
+        listener.onResponse(new Object());
+        assertTrue(executed.getAndSet(false));
+
+        listener.onFailure(new Exception("simulated"));
+        assertTrue(executed.getAndSet(false));
+
+        expectThrows(
+            AssertionError.class,
+            () -> ActionListener.wrap(() -> { throw new UnsupportedOperationException(); }).onResponse(null)
+        );
+    }
+
+    public void testWrapListener() {
+        var succeeded = new AtomicBoolean();
+        var failed = new AtomicBoolean();
+
+        var listener = ActionListener.wrap(new ActionListener<>() {
+            @Override
+            public void onResponse(Object o) {
+                assertTrue(succeeded.compareAndSet(false, true));
+                if (o instanceof RuntimeException e) {
+                    throw e;
+                }
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                assertTrue(failed.compareAndSet(false, true));
+                assertEquals("test exception", e.getMessage());
+                if (e instanceof UnsupportedOperationException uoe) {
+                    throw uoe;
+                }
+            }
+
+            @Override
+            public String toString() {
+                return "test listener";
+            }
+        });
+
+        assertEquals("wrapped{test listener}", listener.toString());
+
+        listener.onResponse(new Object());
+        assertTrue(succeeded.getAndSet(false));
+        assertFalse(failed.getAndSet(false));
+
+        listener.onFailure(new RuntimeException("test exception"));
+        assertFalse(succeeded.getAndSet(false));
+        assertTrue(failed.getAndSet(false));
+
+        listener.onResponse(new RuntimeException("test exception"));
+        assertTrue(succeeded.getAndSet(false));
+        assertTrue(failed.getAndSet(false));
     }
 
     public void testOnResponse() {
@@ -209,6 +312,62 @@ public void onFailure(Exception e) {
         }
     }
 
+    public void testNotifyOnceReleasesDelegate() {
+        final var reachabilityChecker = new ReachabilityChecker();
+        final var listener = ActionListener.notifyOnce(reachabilityChecker.register(ActionListener.wrap(() -> {})));
+        reachabilityChecker.checkReachable();
+        listener.onResponse(null);
+        reachabilityChecker.ensureUnreachable();
+        assertEquals("notifyOnce[null]", listener.toString());
+    }
+
+    public void testConcurrentNotifyOnce() throws InterruptedException {
+        final var completed = new AtomicBoolean();
+        final var listener = ActionListener.notifyOnce(new ActionListener<Void>() {
+            @Override
+            public void onResponse(Void o) {
+                assertTrue(completed.compareAndSet(false, true));
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                assertTrue(completed.compareAndSet(false, true));
+            }
+
+            @Override
+            public String toString() {
+                return "inner-listener";
+            }
+        });
+        assertThat(listener.toString(), equalTo("notifyOnce[inner-listener]"));
+
+        final var threads = new Thread[between(1, 10)];
+        final var startBarrier = new CyclicBarrier(threads.length);
+        for (int i = 0; i < threads.length; i++) {
+            threads[i] = new Thread(() -> {
+                try {
+                    startBarrier.await(10, TimeUnit.SECONDS);
+                } catch (Exception e) {
+                    throw new AssertionError(e);
+                }
+                if (randomBoolean()) {
+                    listener.onResponse(null);
+                } else {
+                    listener.onFailure(new RuntimeException("test"));
+                }
+            });
+        }
+
+        for (Thread thread : threads) {
+            thread.start();
+        }
+        for (Thread thread : threads) {
+            thread.join();
+        }
+
+        assertTrue(completed.get());
+    }
+
     public void testCompleteWith() {
         PlainActionFuture<Integer> onResponseListener = new PlainActionFuture<>();
         ActionListener.completeWith(onResponseListener, () -> 100);
@@ -298,4 +457,169 @@ public void onFailure(Exception e) {
         mapped.onFailure(new IllegalStateException());
         assertThat(exReference.get(), instanceOf(IllegalStateException.class));
     }
+
+    public void testRunBeforeThrowsAssertionErrorIfExecutedMoreThanOnce() {
+        assumeTrue("test only works with assertions enabled", Assertions.ENABLED);
+        final String description = randomAlphaOfLength(10);
+        final ActionListener<Void> runBefore = ActionListener.runBefore(ActionListener.noop(), makeCheckedRunnable(description));
+
+        completeListener(randomBoolean(), runBefore);
+
+        var error = expectThrows(AssertionError.class, () -> completeListener(true, runBefore));
+        assertThat(error.getMessage(), containsString(description));
+    }
+
+    public void testRunAfterThrowsAssertionErrorIfExecutedMoreThanOnce() {
+        assumeTrue("test only works with assertions enabled", Assertions.ENABLED);
+        final String description = randomAlphaOfLength(10);
+        final ActionListener<Void> runAfter = randomBoolean()
+            ? ActionListener.runAfter(ActionListener.noop(), makeRunnable(description))
+            : ActionListener.releaseAfter(ActionListener.noop(), makeReleasable(description, new AtomicBoolean()));
+
+        completeListener(randomBoolean(), runAfter);
+
+        var error = expectThrows(AssertionError.class, () -> completeListener(true, runAfter));
+        assertThat(error.getMessage(), containsString(description));
+    }
+
+    public void testWrappedRunBeforeOrAfterThrowsAssertionErrorIfExecutedMoreThanOnce() {
+        assumeTrue("test only works with assertions enabled", Assertions.ENABLED);
+        final ActionListener<Void> throwingListener = new ActionListener<>() {
+            @Override
+            public void onResponse(Void o) {
+                throw new AlreadyClosedException("throwing on purpose");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError("should not be called");
+            }
+        };
+
+        final String description = randomAlphaOfLength(10);
+        final ActionListener<Void> runBeforeOrAfterListener = randomBoolean()
+            ? ActionListener.runBefore(throwingListener, makeCheckedRunnable(description))
+            : ActionListener.runAfter(throwingListener, makeRunnable(description));
+
+        final ActionListener<Void> wrappedListener = ActionListener.wrap(new AbstractRunnable() {
+            @Override
+            public void onFailure(Exception e) {
+                runBeforeOrAfterListener.onFailure(e);
+            }
+
+            @Override
+            protected void doRun() {
+                runBeforeOrAfterListener.onResponse(null);
+            }
+        });
+
+        var error = expectThrows(AssertionError.class, () -> completeListener(true, wrappedListener));
+        assertThat(error.getMessage(), containsString(description));
+    }
+
+    public void testReleasing() {
+        runReleasingTest(true);
+        runReleasingTest(false);
+    }
+
+    private static void runReleasingTest(boolean successResponse) {
+        final AtomicBoolean releasedFlag = new AtomicBoolean();
+        final String description = randomAlphaOfLength(10);
+        final ActionListener<Void> l = ActionListener.releasing(makeReleasable(description, releasedFlag));
+        assertThat(l.toString(), containsString("release[" + description + "]}"));
+        completeListener(successResponse, l);
+        assertTrue(releasedFlag.get());
+    }
+
+    private static void completeListener(boolean successResponse, ActionListener<Void> listener) {
+        if (successResponse) {
+            try {
+                listener.onResponse(null);
+            } catch (Exception e) {
+                // ok
+            }
+        } else {
+            listener.onFailure(new RuntimeException("simulated"));
+        }
+    }
+
+    public void testReleaseAfter() {
+        runReleaseAfterTest(true, false);
+        runReleaseAfterTest(true, true);
+        runReleaseAfterTest(false, false);
+    }
+
+    private static void runReleaseAfterTest(boolean successResponse, final boolean throwFromOnResponse) {
+        final AtomicBoolean released = new AtomicBoolean();
+        final String description = randomAlphaOfLength(10);
+        final ActionListener<Void> l = ActionListener.releaseAfter(new ActionListener<>() {
+            @Override
+            public void onResponse(Void unused) {
+                if (throwFromOnResponse) {
+                    throw new RuntimeException("onResponse");
+                }
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                // ok
+            }
+
+            @Override
+            public String toString() {
+                return "test listener";
+            }
+        }, makeReleasable(description, released));
+        assertThat(l.toString(), containsString("test listener/release[" + description + "]"));
+
+        if (successResponse) {
+            try {
+                l.onResponse(null);
+            } catch (Exception e) {
+                // ok
+            }
+        } else {
+            l.onFailure(new RuntimeException("supplied"));
+        }
+
+        assertTrue(released.get());
+    }
+
+    private static Releasable makeReleasable(String description, AtomicBoolean releasedFlag) {
+        return new Releasable() {
+            @Override
+            public void close() {
+                assertTrue(releasedFlag.compareAndSet(false, true));
+            }
+
+            @Override
+            public String toString() {
+                return description;
+            }
+        };
+    }
+
+    private static Runnable makeRunnable(String description) {
+        return new Runnable() {
+            @Override
+            public void run() {}
+
+            @Override
+            public String toString() {
+                return description;
+            }
+        };
+    }
+
+    private static CheckedRunnable<?> makeCheckedRunnable(String description) {
+        return new CheckedRunnable<>() {
+            @Override
+            public void run() {}
+
+            @Override
+            public String toString() {
+                return description;
+            }
+        };
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/NotifyOnceListenerTests.java b/server/src/test/java/org/elasticsearch/action/NotifyOnceListenerTests.java
deleted file mode 100644
index fa6761b2bf3c..000000000000
--- a/server/src/test/java/org/elasticsearch/action/NotifyOnceListenerTests.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.action;
-
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.concurrent.atomic.AtomicReference;
-
-public class NotifyOnceListenerTests extends ESTestCase {
-
-    public void testWhenSuccessCannotNotifyMultipleTimes() {
-        AtomicReference<String> response = new AtomicReference<>();
-        AtomicReference<Exception> exception = new AtomicReference<>();
-
-        NotifyOnceListener<String> listener = new NotifyOnceListener<String>() {
-            @Override
-            public void innerOnResponse(String s) {
-                response.set(s);
-            }
-
-            @Override
-            public void innerOnFailure(Exception e) {
-                exception.set(e);
-            }
-        };
-
-        listener.onResponse("response");
-        listener.onResponse("wrong-response");
-        listener.onFailure(new RuntimeException());
-
-        assertNull(exception.get());
-        assertEquals("response", response.get());
-    }
-
-    public void testWhenErrorCannotNotifyMultipleTimes() {
-        AtomicReference<String> response = new AtomicReference<>();
-        AtomicReference<Exception> exception = new AtomicReference<>();
-
-        NotifyOnceListener<String> listener = new NotifyOnceListener<String>() {
-            @Override
-            public void innerOnResponse(String s) {
-                response.set(s);
-            }
-
-            @Override
-            public void innerOnFailure(Exception e) {
-                exception.set(e);
-            }
-        };
-
-        RuntimeException expected = new RuntimeException();
-        listener.onFailure(expected);
-        listener.onFailure(new IllegalArgumentException());
-        listener.onResponse("response");
-
-        assertNull(response.get());
-        assertSame(expected, exception.get());
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java b/server/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java
index 5d87e3b7a496..5b3fe6e904f1 100644
--- a/server/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java
+++ b/server/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -16,7 +16,7 @@
 
 import java.io.IOException;
 
-import static org.elasticsearch.test.VersionUtils.randomCompatibleVersion;
+import static org.elasticsearch.test.TransportVersionUtils.randomCompatibleVersion;
 import static org.hamcrest.CoreMatchers.equalTo;
 
 public class OriginalIndicesTests extends ESTestCase {
@@ -34,18 +34,18 @@ public void testOriginalIndicesSerialization() throws IOException {
             OriginalIndices originalIndices = randomOriginalIndices();
 
             BytesStreamOutput out = new BytesStreamOutput();
-            out.setVersion(randomCompatibleVersion(random(), Version.CURRENT));
+            out.setTransportVersion(randomCompatibleVersion(random(), TransportVersion.CURRENT));
             OriginalIndices.writeOriginalIndices(originalIndices, out);
 
             StreamInput in = out.bytes().streamInput();
-            in.setVersion(out.getVersion());
+            in.setTransportVersion(out.getTransportVersion());
             OriginalIndices originalIndices2 = OriginalIndices.readOriginalIndices(in);
 
             assertThat(originalIndices2.indices(), equalTo(originalIndices.indices()));
             // indices options are not equivalent when sent to an older version and re-read due
             // to the addition of hidden indices as expand to hidden indices is always true when
             // read from a prior version
-            if (out.getVersion().onOrAfter(Version.V_7_7_0) || originalIndices.indicesOptions().expandWildcardsHidden()) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_7_0) || originalIndices.indicesOptions().expandWildcardsHidden()) {
                 assertThat(originalIndices2.indicesOptions(), equalTo(originalIndices.indicesOptions()));
             } else if (originalIndices.indicesOptions().expandWildcardsHidden()) {
                 assertThat(originalIndices2.indicesOptions(), equalTo(originalIndices.indicesOptions()));
diff --git a/server/src/test/java/org/elasticsearch/action/StepListenerTests.java b/server/src/test/java/org/elasticsearch/action/StepListenerTests.java
index 6d1258b56e51..329f1c743f14 100644
--- a/server/src/test/java/org/elasticsearch/action/StepListenerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/StepListenerTests.java
@@ -10,6 +10,7 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ReachabilityChecker;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.RemoteTransportException;
@@ -164,4 +165,23 @@ public void testThenCombine() throws Exception {
             assertThat(exception.getMessage(), containsString("simulated"));
         }
     }
+
+    public void testAddedListenersReleasedOnCompletion() {
+        final StepListener<Void> step = new StepListener<>();
+        final ReachabilityChecker reachabilityChecker = new ReachabilityChecker();
+
+        for (int i = between(1, 3); i > 0; i--) {
+            step.addListener(reachabilityChecker.register(ActionListener.wrap(() -> {})));
+        }
+        reachabilityChecker.checkReachable();
+        if (randomBoolean()) {
+            step.onResponse(null);
+        } else {
+            step.onFailure(new ElasticsearchException("simulated"));
+        }
+        reachabilityChecker.ensureUnreachable();
+
+        step.addListener(reachabilityChecker.register(ActionListener.wrap(() -> {})));
+        reachabilityChecker.ensureUnreachable();
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java
index bf5709656083..61599b00e293 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java
@@ -10,6 +10,7 @@
 
 import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingNode;
@@ -77,7 +78,7 @@ public ShardAllocationDecision decideShardAllocation(ShardRouting shard, Routing
                         throw new UnsupportedOperationException("cannot explain");
                     }
                 }
-            }, null, null)
+            }, null, null, TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
         );
 
         assertEquals(shard.currentNodeId(), cae.getCurrentNode().getId());
@@ -93,44 +94,36 @@ public ShardAllocationDecision decideShardAllocation(ShardRouting shard, Routing
         } else {
             explanation = "the shard is in the process of initializing on node [], " + "wait until initialization has completed";
         }
-        assertEquals(
-            XContentHelper.stripWhitespace(
-                formatted(
+        Object[] args = new Object[] {
+            shardRoutingState.toString().toLowerCase(Locale.ROOT),
+            shard.unassignedInfo() != null
+                ? Strings.format(
                     """
-                        {
-                          "index": "idx",
-                          "shard": 0,
-                          "primary": true,
-                          "current_state": "%s"
-                          %s,
-                          "current_node": {
-                            "id": "%s",
-                            "name": "%s",
-                            "transport_address": "%s"
-                          },
-                          "explanation": "%s"
-                        }""",
-                    shardRoutingState.toString().toLowerCase(Locale.ROOT),
-                    shard.unassignedInfo() != null
-                        ? formatted(
-                            """
-                                ,"unassigned_info": {"reason": "%s", "at": "%s", "last_allocation_status": "%s"}
-                                """,
-                            shard.unassignedInfo().getReason(),
-                            UnassignedInfo.DATE_TIME_FORMATTER.format(
-                                Instant.ofEpochMilli(shard.unassignedInfo().getUnassignedTimeInMillis())
-                            ),
-                            AllocationDecision.fromAllocationStatus(shard.unassignedInfo().getLastAllocationStatus())
-                        )
-                        : "",
-                    cae.getCurrentNode().getId(),
-                    cae.getCurrentNode().getName(),
-                    cae.getCurrentNode().getAddress(),
-                    explanation
+                        ,"unassigned_info": {"reason": "%s", "at": "%s", "last_allocation_status": "%s"}
+                        """,
+                    shard.unassignedInfo().getReason(),
+                    UnassignedInfo.DATE_TIME_FORMATTER.format(Instant.ofEpochMilli(shard.unassignedInfo().getUnassignedTimeInMillis())),
+                    AllocationDecision.fromAllocationStatus(shard.unassignedInfo().getLastAllocationStatus())
                 )
-            ),
-            Strings.toString(builder)
-        );
+                : "",
+            cae.getCurrentNode().getId(),
+            cae.getCurrentNode().getName(),
+            cae.getCurrentNode().getAddress(),
+            explanation };
+        assertEquals(XContentHelper.stripWhitespace(Strings.format("""
+            {
+              "index": "idx",
+              "shard": 0,
+              "primary": true,
+              "current_state": "%s"
+              %s,
+              "current_node": {
+                "id": "%s",
+                "name": "%s",
+                "transport_address": "%s"
+              },
+              "explanation": "%s"
+            }""", args)), Strings.toString(builder));
     }
 
     public void testFindAnyUnassignedShardToExplain() {
@@ -151,7 +144,7 @@ public void testFindAnyUnassignedShardToExplain() {
         final String redIndex = randomBoolean() ? "idx1" : "idx2";
         final RoutingTable.Builder routingTableBuilder = RoutingTable.builder(clusterState.routingTable());
         for (final IndexRoutingTable indexRoutingTable : clusterState.routingTable()) {
-            final IndexRoutingTable.Builder indexBuilder = new IndexRoutingTable.Builder(indexRoutingTable.getIndex());
+            final IndexRoutingTable.Builder indexBuilder = IndexRoutingTable.builder(indexRoutingTable.getIndex());
             for (int shardId = 0; shardId < indexRoutingTable.size(); shardId++) {
                 IndexShardRoutingTable indexShardRoutingTable = indexRoutingTable.shard(shardId);
                 final IndexShardRoutingTable.Builder shardBuilder = new IndexShardRoutingTable.Builder(indexShardRoutingTable.shardId());
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
index 071fe0c9df02..e9d016329462 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
@@ -84,7 +84,7 @@ public void testExplanationToXContent() throws Exception {
         ClusterAllocationExplanation cae = randomClusterAllocationExplanation(true, true);
         XContentBuilder builder = XContentFactory.jsonBuilder();
         cae.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        assertEquals(XContentHelper.stripWhitespace(formatted("""
+        assertEquals(XContentHelper.stripWhitespace(Strings.format("""
             {
               "index": "idx",
               "shard": 0,
@@ -112,7 +112,7 @@ public void testRandomShardExplanationToXContent() throws Exception {
             actual,
             equalTo(
                 XContentHelper.stripWhitespace(
-                    formatted(
+                    Strings.format(
                         """
                             {
                               "note": "%s",
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponseTests.java
index 3403a3e244ba..9d5dcca152db 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponseTests.java
@@ -7,10 +7,13 @@
  */
 package org.elasticsearch.action.admin.cluster.allocation;
 
-import org.elasticsearch.cluster.routing.AllocationId;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.allocation.allocator.ClusterBalanceStats;
 import org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceStats;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentFactory;
@@ -20,10 +23,13 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import static java.util.function.Function.identity;
+import static java.util.stream.Collectors.toMap;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+
 public class DesiredBalanceResponseTests extends AbstractWireSerializingTestCase<DesiredBalanceResponse> {
 
     @Override
@@ -33,10 +39,10 @@ protected Writeable.Reader<DesiredBalanceResponse> instanceReader() {
 
     @Override
     protected DesiredBalanceResponse createTestInstance() {
-        return new DesiredBalanceResponse(randomStats(), randomRoutingTable());
+        return new DesiredBalanceResponse(randomDesiredBalanceStats(), randomClusterBalanceStats(), randomRoutingTable());
     }
 
-    private DesiredBalanceStats randomStats() {
+    private DesiredBalanceStats randomDesiredBalanceStats() {
         return new DesiredBalanceStats(
             randomNonNegativeLong(),
             randomBoolean(),
@@ -49,6 +55,40 @@ private DesiredBalanceStats randomStats() {
         );
     }
 
+    private ClusterBalanceStats randomClusterBalanceStats() {
+        return new ClusterBalanceStats(
+            randomBoolean()
+                ? Map.of(DiscoveryNodeRole.DATA_CONTENT_NODE_ROLE.roleName(), randomTierBalanceStats())
+                : randomSubsetOf(
+                    List.of(
+                        DiscoveryNodeRole.DATA_HOT_NODE_ROLE,
+                        DiscoveryNodeRole.DATA_WARM_NODE_ROLE,
+                        DiscoveryNodeRole.DATA_COLD_NODE_ROLE,
+                        DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE
+                    )
+                ).stream().map(DiscoveryNodeRole::roleName).collect(toMap(identity(), ignore -> randomTierBalanceStats())),
+            randomList(10, () -> randomAlphaOfLength(10)).stream().collect(toMap(identity(), ignore -> randomNodeBalanceStats()))
+        );
+    }
+
+    private ClusterBalanceStats.TierBalanceStats randomTierBalanceStats() {
+        return new ClusterBalanceStats.TierBalanceStats(
+            new ClusterBalanceStats.MetricStats(randomDouble(), randomDouble(), randomDouble(), randomDouble(), randomDouble()),
+            new ClusterBalanceStats.MetricStats(randomDouble(), randomDouble(), randomDouble(), randomDouble(), randomDouble()),
+            new ClusterBalanceStats.MetricStats(randomDouble(), randomDouble(), randomDouble(), randomDouble(), randomDouble()),
+            new ClusterBalanceStats.MetricStats(randomDouble(), randomDouble(), randomDouble(), randomDouble(), randomDouble())
+        );
+    }
+
+    private ClusterBalanceStats.NodeBalanceStats randomNodeBalanceStats() {
+        return new ClusterBalanceStats.NodeBalanceStats(
+            randomIntBetween(0, Integer.MAX_VALUE),
+            randomDouble(),
+            randomLongBetween(0, Long.MAX_VALUE),
+            randomLongBetween(0, Long.MAX_VALUE)
+        );
+    }
+
     private Map<String, Map<Integer, DesiredBalanceResponse.DesiredShards>> randomRoutingTable() {
         Map<String, Map<Integer, DesiredBalanceResponse.DesiredShards>> routingTable = new HashMap<>();
         for (int i = 0; i < randomInt(8); i++) {
@@ -70,7 +110,8 @@ private Map<String, Map<Integer, DesiredBalanceResponse.DesiredShards>> randomRo
                                     randomBoolean(),
                                     shardId,
                                     indexName,
-                                    AllocationId.newInitializing()
+                                    randomBoolean() ? randomDouble() : null,
+                                    randomBoolean() ? randomLong() : null
                                 )
                             )
                             .toList(),
@@ -89,24 +130,41 @@ private Map<String, Map<Integer, DesiredBalanceResponse.DesiredShards>> randomRo
     }
 
     @Override
-    protected DesiredBalanceResponse mutateInstance(DesiredBalanceResponse instance) throws IOException {
-        return switch (randomInt(2)) {
+    protected DesiredBalanceResponse mutateInstance(DesiredBalanceResponse instance) {
+        return switch (randomInt(3)) {
             case 0 -> new DesiredBalanceResponse(
+                randomValueOtherThan(instance.getStats(), this::randomDesiredBalanceStats),
+                instance.getClusterBalanceStats(),
+                instance.getRoutingTable()
+            );
+            case 1 -> new DesiredBalanceResponse(
                 instance.getStats(),
+                randomValueOtherThan(instance.getClusterBalanceStats(), this::randomClusterBalanceStats),
+                instance.getRoutingTable()
+            );
+            case 2 -> new DesiredBalanceResponse(
+                instance.getStats(),
+                instance.getClusterBalanceStats(),
                 randomValueOtherThan(instance.getRoutingTable(), this::randomRoutingTable)
             );
-            case 1 -> new DesiredBalanceResponse(randomStats(), instance.getRoutingTable());
             default -> randomValueOtherThan(instance, this::createTestInstance);
         };
     }
 
     @SuppressWarnings("unchecked")
     public void testToXContent() throws IOException {
-        DesiredBalanceResponse response = new DesiredBalanceResponse(randomStats(), randomRoutingTable());
+        DesiredBalanceResponse response = new DesiredBalanceResponse(
+            randomDesiredBalanceStats(),
+            randomClusterBalanceStats(),
+            randomRoutingTable()
+        );
 
-        Map<String, Object> json = createParser(response.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)).map();
-        assertEquals(Set.of("stats", "routing_table"), json.keySet());
+        Map<String, Object> json = createParser(
+            ChunkedToXContent.wrapAsToXContent(response).toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)
+        ).map();
+        assertThat(json.keySet(), containsInAnyOrder("stats", "cluster_balance_stats", "routing_table"));
 
+        // stats
         Map<String, Object> stats = (Map<String, Object>) json.get("stats");
         assertEquals(stats.get("computation_active"), response.getStats().computationActive());
         assertEquals(stats.get("computation_submitted"), response.getStats().computationSubmitted());
@@ -117,6 +175,69 @@ public void testToXContent() throws IOException {
         assertEquals(stats.get("computation_time_in_millis"), response.getStats().cumulativeComputationTime());
         assertEquals(stats.get("reconciliation_time_in_millis"), response.getStats().cumulativeReconciliationTime());
 
+        // cluster balance stats
+        Map<String, Object> clusterBalanceStats = (Map<String, Object>) json.get("cluster_balance_stats");
+        assertThat(clusterBalanceStats.keySet(), containsInAnyOrder("tiers", "nodes"));
+
+        // tier balance stats
+        Map<String, Object> tiers = (Map<String, Object>) clusterBalanceStats.get("tiers");
+        assertEquals(tiers.keySet(), response.getClusterBalanceStats().tiers().keySet());
+        for (var entry : response.getClusterBalanceStats().tiers().entrySet()) {
+            Map<String, Object> tierStats = (Map<String, Object>) tiers.get(entry.getKey());
+            assertThat(
+                tierStats.keySet(),
+                containsInAnyOrder("shard_count", "forecast_write_load", "forecast_disk_usage", "actual_disk_usage")
+            );
+
+            Map<String, Object> shardCountStats = (Map<String, Object>) tierStats.get("shard_count");
+            assertThat(shardCountStats.keySet(), containsInAnyOrder("total", "average", "min", "max", "std_dev"));
+            assertEquals(shardCountStats.get("total"), entry.getValue().shardCount().total());
+            assertEquals(shardCountStats.get("average"), entry.getValue().shardCount().average());
+            assertEquals(shardCountStats.get("min"), entry.getValue().shardCount().min());
+            assertEquals(shardCountStats.get("max"), entry.getValue().shardCount().max());
+            assertEquals(shardCountStats.get("std_dev"), entry.getValue().shardCount().stdDev());
+
+            Map<String, Object> forecastWriteLoadStats = (Map<String, Object>) tierStats.get("forecast_write_load");
+            assertThat(forecastWriteLoadStats.keySet(), containsInAnyOrder("total", "average", "min", "max", "std_dev"));
+            assertEquals(forecastWriteLoadStats.get("total"), entry.getValue().forecastWriteLoad().total());
+            assertEquals(forecastWriteLoadStats.get("average"), entry.getValue().forecastWriteLoad().average());
+            assertEquals(forecastWriteLoadStats.get("min"), entry.getValue().forecastWriteLoad().min());
+            assertEquals(forecastWriteLoadStats.get("max"), entry.getValue().forecastWriteLoad().max());
+            assertEquals(forecastWriteLoadStats.get("std_dev"), entry.getValue().forecastWriteLoad().stdDev());
+
+            Map<String, Object> forecastDiskUsageStats = (Map<String, Object>) tierStats.get("forecast_disk_usage");
+            assertThat(forecastDiskUsageStats.keySet(), containsInAnyOrder("total", "average", "min", "max", "std_dev"));
+            assertEquals(forecastDiskUsageStats.get("total"), entry.getValue().forecastShardSize().total());
+            assertEquals(forecastDiskUsageStats.get("average"), entry.getValue().forecastShardSize().average());
+            assertEquals(forecastDiskUsageStats.get("min"), entry.getValue().forecastShardSize().min());
+            assertEquals(forecastDiskUsageStats.get("max"), entry.getValue().forecastShardSize().max());
+            assertEquals(forecastDiskUsageStats.get("std_dev"), entry.getValue().forecastShardSize().stdDev());
+
+            Map<String, Object> actualDiskUsageStats = (Map<String, Object>) tierStats.get("actual_disk_usage");
+            assertThat(actualDiskUsageStats.keySet(), containsInAnyOrder("total", "average", "min", "max", "std_dev"));
+            assertEquals(actualDiskUsageStats.get("total"), entry.getValue().actualShardSize().total());
+            assertEquals(actualDiskUsageStats.get("average"), entry.getValue().actualShardSize().average());
+            assertEquals(actualDiskUsageStats.get("min"), entry.getValue().actualShardSize().min());
+            assertEquals(actualDiskUsageStats.get("max"), entry.getValue().actualShardSize().max());
+            assertEquals(actualDiskUsageStats.get("std_dev"), entry.getValue().actualShardSize().stdDev());
+        }
+        // node balance stats
+        Map<String, Object> nodes = (Map<String, Object>) clusterBalanceStats.get("nodes");
+        assertEquals(nodes.keySet(), response.getClusterBalanceStats().nodes().keySet());
+        for (var entry : response.getClusterBalanceStats().nodes().entrySet()) {
+            Map<String, Object> nodesStats = (Map<String, Object>) nodes.get(entry.getKey());
+            assertThat(
+                nodesStats.keySet(),
+                containsInAnyOrder("shard_count", "forecast_write_load", "forecast_disk_usage_bytes", "actual_disk_usage_bytes")
+            );
+
+            assertEquals(nodesStats.get("shard_count"), entry.getValue().shards());
+            assertEquals(nodesStats.get("forecast_write_load"), entry.getValue().forecastWriteLoad());
+            assertEquals(nodesStats.get("forecast_disk_usage_bytes"), entry.getValue().forecastShardSize());
+            assertEquals(nodesStats.get("actual_disk_usage_bytes"), entry.getValue().actualShardSize());
+        }
+
+        // routing table
         Map<String, Object> jsonRoutingTable = (Map<String, Object>) json.get("routing_table");
         assertEquals(jsonRoutingTable.keySet(), response.getRoutingTable().keySet());
         for (var indexEntry : response.getRoutingTable().entrySet()) {
@@ -128,7 +249,7 @@ public void testToXContent() throws IOException {
             for (var shardEntry : indexEntry.getValue().entrySet()) {
                 DesiredBalanceResponse.DesiredShards desiredShards = shardEntry.getValue();
                 Map<String, Object> jsonDesiredShard = (Map<String, Object>) jsonIndexShards.get(String.valueOf(shardEntry.getKey()));
-                assertEquals(Set.of("current", "desired"), jsonDesiredShard.keySet());
+                assertThat(jsonDesiredShard.keySet(), containsInAnyOrder("current", "desired"));
                 List<Map<String, Object>> jsonCurrent = (List<Map<String, Object>>) jsonDesiredShard.get("current");
                 for (int i = 0; i < jsonCurrent.size(); i++) {
                     Map<String, Object> jsonShard = jsonCurrent.get(i);
@@ -141,6 +262,8 @@ public void testToXContent() throws IOException {
                     assertEquals(jsonShard.get("relocating_node_is_desired"), shardView.relocatingNodeIsDesired());
                     assertEquals(jsonShard.get("shard_id"), shardView.shardId());
                     assertEquals(jsonShard.get("index"), shardView.index());
+                    assertEquals(jsonShard.get("forecast_write_load"), shardView.forecastWriteLoad());
+                    assertEquals(jsonShard.get("forecast_shard_size_in_bytes"), shardView.forecastShardSizeInBytes());
                 }
 
                 Map<String, Object> jsonDesired = (Map<String, Object>) jsonDesiredShard.get("desired");
@@ -152,4 +275,11 @@ public void testToXContent() throws IOException {
             }
         }
     }
+
+    public void testChunking() {
+        AbstractChunkedSerializingTestCase.assertChunkCount(
+            new DesiredBalanceResponse(randomDesiredBalanceStats(), randomClusterBalanceStats(), randomRoutingTable()),
+            response -> response.getRoutingTable().size() + 2
+        );
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java
index d29535aaf1f8..7505d4c4e8f0 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionTests.java
@@ -8,31 +8,39 @@
 package org.elasticsearch.action.admin.cluster.allocation;
 
 import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.cluster.ClusterInfo;
+import org.elasticsearch.cluster.ClusterInfoService;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster;
 import org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalance;
 import org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceStats;
 import org.elasticsearch.cluster.routing.allocation.allocator.ShardAssignment;
+import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
-import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
-import org.junit.Before;
 import org.mockito.ArgumentCaptor;
 
 import java.util.HashMap;
@@ -42,47 +50,57 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.cluster.ClusterModule.BALANCED_ALLOCATOR;
+import static org.elasticsearch.cluster.ClusterModule.DESIRED_BALANCE_ALLOCATOR;
+import static org.elasticsearch.cluster.ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING;
+import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-public class TransportGetDesiredBalanceActionTests extends ESTestCase {
+public class TransportGetDesiredBalanceActionTests extends ESAllocationTestCase {
 
     private final DesiredBalanceShardsAllocator desiredBalanceShardsAllocator = mock(DesiredBalanceShardsAllocator.class);
-    private final ClusterState clusterState = mock(ClusterState.class);
-    private final Metadata metadata = mock(Metadata.class);
+    private final ClusterInfoService clusterInfoService = mock(ClusterInfoService.class);
     private final TransportGetDesiredBalanceAction transportGetDesiredBalanceAction = new TransportGetDesiredBalanceAction(
         mock(TransportService.class),
         mock(ClusterService.class),
         mock(ThreadPool.class),
         mock(ActionFilters.class),
         mock(IndexNameExpressionResolver.class),
-        desiredBalanceShardsAllocator
+        desiredBalanceShardsAllocator,
+        clusterInfoService,
+        TEST_WRITE_LOAD_FORECASTER
     );
     @SuppressWarnings("unchecked")
     private final ActionListener<DesiredBalanceResponse> listener = mock(ActionListener.class);
 
-    @Before
-    public void setUpMocks() throws Exception {
-        when(clusterState.metadata()).thenReturn(metadata);
-    }
-
     public void testReturnsErrorIfAllocatorIsNotDesiredBalanced() throws Exception {
-        when(metadata.settings()).thenReturn(Settings.builder().put("cluster.routing.allocation.type", "balanced").build());
+        var clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadataWithConfiguredAllocator(BALANCED_ALLOCATOR)).build();
 
-        transportGetDesiredBalanceAction.masterOperation(mock(Task.class), mock(DesiredBalanceRequest.class), clusterState, listener);
+        new TransportGetDesiredBalanceAction(
+            mock(TransportService.class),
+            mock(ClusterService.class),
+            mock(ThreadPool.class),
+            mock(ActionFilters.class),
+            mock(IndexNameExpressionResolver.class),
+            mock(ShardsAllocator.class),
+            mock(ClusterInfoService.class),
+            mock(WriteLoadForecaster.class)
+        ).masterOperation(mock(Task.class), mock(DesiredBalanceRequest.class), clusterState, listener);
 
         ArgumentCaptor<ResourceNotFoundException> exceptionArgumentCaptor = ArgumentCaptor.forClass(ResourceNotFoundException.class);
         verify(listener).onFailure(exceptionArgumentCaptor.capture());
 
-        assertEquals(
-            "Expected the shard balance allocator to be `desired_balance`, but got `balanced`",
-            exceptionArgumentCaptor.getValue().getMessage()
-        );
+        final var exception = exceptionArgumentCaptor.getValue();
+        assertEquals("Desired balance allocator is not in use, no desired balance found", exception.getMessage());
+        assertThat(exception.status(), equalTo(RestStatus.NOT_FOUND));
     }
 
     public void testReturnsErrorIfDesiredBalanceIsNotAvailable() throws Exception {
-        when(metadata.settings()).thenReturn(Settings.builder().put("cluster.routing.allocation.type", "desired_balance").build());
+        var clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metadata(metadataWithConfiguredAllocator(DESIRED_BALANCE_ALLOCATOR))
+            .build();
 
         transportGetDesiredBalanceAction.masterOperation(mock(Task.class), mock(DesiredBalanceRequest.class), clusterState, listener);
 
@@ -94,9 +112,27 @@ public void testReturnsErrorIfDesiredBalanceIsNotAvailable() throws Exception {
 
     public void testGetDesiredBalance() throws Exception {
         Set<String> nodeIds = randomUnique(() -> randomAlphaOfLength(8), randomIntBetween(1, 32));
+        DiscoveryNodes.Builder discoveryNodes = DiscoveryNodes.builder();
+        for (String nodeId : nodeIds) {
+            discoveryNodes.add(newNode(nodeId, Set.of(DiscoveryNodeRole.DATA_ROLE)));
+        }
+        Metadata.Builder metadataBuilder = metadataWithConfiguredAllocator(DESIRED_BALANCE_ALLOCATOR);
         RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
         for (int i = 0; i < randomInt(8); i++) {
-            Index index = new Index(randomAlphaOfLength(8), UUIDs.randomBase64UUID());
+            String indexName = randomAlphaOfLength(8);
+            IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexName)
+                .settings(settings(Version.CURRENT))
+                .numberOfShards(1)
+                .numberOfReplicas(0);
+            if (randomBoolean()) {
+                indexMetadataBuilder.indexWriteLoadForecast(randomDoubleBetween(0.0, 8.0, true));
+            }
+            if (randomBoolean()) {
+                indexMetadataBuilder.shardSizeInBytesForecast(randomLongBetween(0, 1024));
+            }
+            IndexMetadata indexMetadata = indexMetadataBuilder.build();
+            Index index = indexMetadata.getIndex();
+            metadataBuilder.put(indexMetadata, false);
             IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(index);
             for (int j = 0; j < randomIntBetween(1, 16); j++) {
                 String nodeId = randomFrom(nodeIds);
@@ -118,10 +154,9 @@ public void testGetDesiredBalance() throws Exception {
                                 )
                             );
                         }
-                        break;
                     }
                     case 2 -> indexRoutingTableBuilder.addShard(
-                        TestShardRouting.newShardRouting(new ShardId(index, j), null, false, ShardRoutingState.UNASSIGNED)
+                        TestShardRouting.newShardRouting(new ShardId(index, j), null, true, ShardRoutingState.UNASSIGNED)
                     );
                     case 3 -> {
                         ShardRouting shard = TestShardRouting.newShardRouting(
@@ -144,7 +179,6 @@ public void testGetDesiredBalance() throws Exception {
             routingTableBuilder.add(indexRoutingTableBuilder.build());
         }
         RoutingTable routingTable = routingTableBuilder.build();
-        when(clusterState.routingTable()).thenReturn(routingTable);
 
         List<ShardId> shardIds = routingTable.allShards().stream().map(ShardRouting::shardId).toList();
         Map<String, Set<ShardId>> indexShards = shardIds.stream()
@@ -173,7 +207,13 @@ public void testGetDesiredBalance() throws Exception {
             randomInt(Integer.MAX_VALUE)
         );
         when(desiredBalanceShardsAllocator.getStats()).thenReturn(desiredBalanceStats);
-        when(metadata.settings()).thenReturn(Settings.builder().put("cluster.routing.allocation.type", "desired_balance").build());
+        when(clusterInfoService.getClusterInfo()).thenReturn(ClusterInfo.EMPTY);
+
+        var clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metadata(metadataBuilder.build())
+            .nodes(discoveryNodes.build())
+            .routingTable(routingTable)
+            .build();
 
         transportGetDesiredBalanceAction.masterOperation(mock(Task.class), mock(DesiredBalanceRequest.class), clusterState, listener);
 
@@ -181,6 +221,7 @@ public void testGetDesiredBalance() throws Exception {
         verify(listener).onResponse(desiredBalanceResponseCaptor.capture());
         DesiredBalanceResponse desiredBalanceResponse = desiredBalanceResponseCaptor.getValue();
         assertEquals(desiredBalanceStats, desiredBalanceResponse.getStats());
+        assertNotNull(desiredBalanceResponse.getClusterBalanceStats());
         assertEquals(indexShards.keySet(), desiredBalanceResponse.getRoutingTable().keySet());
         for (var e : desiredBalanceResponse.getRoutingTable().entrySet()) {
             String index = e.getKey();
@@ -189,7 +230,8 @@ public void testGetDesiredBalance() throws Exception {
             for (var shardDesiredBalance : shardsMap.entrySet()) {
                 DesiredBalanceResponse.DesiredShards desiredShard = shardDesiredBalance.getValue();
                 int shardId = shardDesiredBalance.getKey();
-                IndexShardRoutingTable indexShardRoutingTable = routingTable.shardRoutingTable(index, shardId);
+                IndexMetadata indexMetadata = clusterState.metadata().index(index);
+                IndexShardRoutingTable indexShardRoutingTable = clusterState.getRoutingTable().shardRoutingTable(index, shardId);
                 for (int idx = 0; idx < indexShardRoutingTable.size(); idx++) {
                     ShardRouting shard = indexShardRoutingTable.shard(idx);
                     DesiredBalanceResponse.ShardView shardView = desiredShard.current().get(idx);
@@ -199,7 +241,13 @@ public void testGetDesiredBalance() throws Exception {
                     assertEquals(shard.relocatingNodeId(), shardView.relocatingNode());
                     assertEquals(shard.index().getName(), shardView.index());
                     assertEquals(shard.shardId().id(), shardView.shardId());
-                    assertEquals(shard.allocationId(), shardView.allocationId());
+                    var forecastedWriteLoad = TEST_WRITE_LOAD_FORECASTER.getForecastedWriteLoad(indexMetadata);
+                    assertEquals(forecastedWriteLoad.isPresent() ? forecastedWriteLoad.getAsDouble() : null, shardView.forecastWriteLoad());
+                    var forecastedShardSizeInBytes = indexMetadata.getForecastedShardSizeInBytes();
+                    assertEquals(
+                        forecastedShardSizeInBytes.isPresent() ? forecastedShardSizeInBytes.getAsLong() : null,
+                        shardView.forecastShardSizeInBytes()
+                    );
                     Set<String> desiredNodeIds = Optional.ofNullable(shardAssignments.get(shard.shardId()))
                         .map(ShardAssignment::nodeIds)
                         .orElse(Set.of());
@@ -224,4 +272,8 @@ public void testGetDesiredBalance() throws Exception {
             }
         }
     }
+
+    private static Metadata.Builder metadataWithConfiguredAllocator(String allocator) {
+        return Metadata.builder().persistentSettings(Settings.builder().put(SHARDS_ALLOCATOR_TYPE_SETTING.getKey(), allocator).build());
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/GetDesiredNodesResponseSerializationTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/GetDesiredNodesResponseSerializationTests.java
index 65b921fefecc..8c7d1b490b2f 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/GetDesiredNodesResponseSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/GetDesiredNodesResponseSerializationTests.java
@@ -11,8 +11,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
-
 import static org.elasticsearch.cluster.metadata.DesiredNodesTestCase.randomDesiredNodes;
 
 public class GetDesiredNodesResponseSerializationTests extends AbstractWireSerializingTestCase<GetDesiredNodesAction.Response> {
@@ -27,7 +25,7 @@ protected GetDesiredNodesAction.Response createTestInstance() {
     }
 
     @Override
-    protected GetDesiredNodesAction.Response mutateInstance(GetDesiredNodesAction.Response instance) throws IOException {
+    protected GetDesiredNodesAction.Response mutateInstance(GetDesiredNodesAction.Response instance) {
         return new GetDesiredNodesAction.Response(randomDesiredNodes());
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesRequestSerializationTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesRequestSerializationTests.java
index dd3dc184b6bf..cf491c52b49d 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesRequestSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesRequestSerializationTests.java
@@ -12,8 +12,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
-
 public class UpdateDesiredNodesRequestSerializationTests extends AbstractWireSerializingTestCase<UpdateDesiredNodesRequest> {
     @Override
     protected Writeable.Reader<UpdateDesiredNodesRequest> instanceReader() {
@@ -21,7 +19,7 @@ protected Writeable.Reader<UpdateDesiredNodesRequest> instanceReader() {
     }
 
     @Override
-    protected UpdateDesiredNodesRequest mutateInstance(UpdateDesiredNodesRequest request) throws IOException {
+    protected UpdateDesiredNodesRequest mutateInstance(UpdateDesiredNodesRequest request) {
         return new UpdateDesiredNodesRequest(request.getHistoryID(), request.getVersion() + 1, request.getNodes(), request.isDryRun());
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesResponseSerializationTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesResponseSerializationTests.java
index 081a4898a138..e8cd9a250602 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesResponseSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesResponseSerializationTests.java
@@ -11,8 +11,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
-
 public class UpdateDesiredNodesResponseSerializationTests extends AbstractWireSerializingTestCase<UpdateDesiredNodesResponse> {
     @Override
     protected Writeable.Reader<UpdateDesiredNodesResponse> instanceReader() {
@@ -25,7 +23,7 @@ protected UpdateDesiredNodesResponse createTestInstance() {
     }
 
     @Override
-    protected UpdateDesiredNodesResponse mutateInstance(UpdateDesiredNodesResponse instance) throws IOException {
+    protected UpdateDesiredNodesResponse mutateInstance(UpdateDesiredNodesResponse instance) {
         return new UpdateDesiredNodesResponse(instance.hasReplacedExistingHistoryId() == false);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthActionTests.java
index 14d881e2ddee..4cc9e909a23a 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthActionTests.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRoutingRoleStrategy;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.Randomness;
@@ -72,24 +73,28 @@ ClusterState randomClusterStateWithInitializingShards(String index, final int in
             .build();
 
         final List<ShardRoutingState> shardRoutingStates = new ArrayList<>();
-        IntStream.range(0, between(1, 30))
-            .forEach(
-                i -> shardRoutingStates.add(
-                    randomFrom(ShardRoutingState.STARTED, ShardRoutingState.UNASSIGNED, ShardRoutingState.RELOCATING)
-                )
-            );
-        IntStream.range(0, initializingShards).forEach(i -> shardRoutingStates.add(ShardRoutingState.INITIALIZING));
-        Randomness.shuffle(shardRoutingStates);
-
-        // primary can not be unassigned, otherwise replicas can't in initializing or relocating state.
-        // (assertion in RoutingNodes disallows this)
-        if (shardRoutingStates.get(0) == ShardRoutingState.UNASSIGNED) {
-            // Don't randomly pick ShardRoutingState.UNASSIGNED, since that already has randomly been inserted based on initializingShards
-            shardRoutingStates.set(0, randomFrom(ShardRoutingState.STARTED, ShardRoutingState.RELOCATING));
+        if (initializingShards == 1 && randomBoolean()) {
+            shardRoutingStates.add(ShardRoutingState.INITIALIZING);
+            IntStream.range(0, between(0, 30)).forEach(i -> shardRoutingStates.add(ShardRoutingState.UNASSIGNED));
+        } else {
+            IntStream.range(0, between(0, 30))
+                .forEach(
+                    i -> shardRoutingStates.add(
+                        randomFrom(ShardRoutingState.STARTED, ShardRoutingState.UNASSIGNED, ShardRoutingState.RELOCATING)
+                    )
+                );
+            IntStream.range(0, initializingShards).forEach(i -> shardRoutingStates.add(ShardRoutingState.INITIALIZING));
+            Randomness.shuffle(shardRoutingStates);
+
+            // primary must be active, otherwise replicas can't in initializing or relocating state.
+            shardRoutingStates.add(0, randomFrom(ShardRoutingState.STARTED, ShardRoutingState.RELOCATING));
         }
 
         final ShardId shardId = new ShardId(indexMetadata.getIndex(), 0);
-        final IndexRoutingTable.Builder routingTable = new IndexRoutingTable.Builder(indexMetadata.getIndex());
+        final IndexRoutingTable.Builder routingTable = new IndexRoutingTable.Builder(
+            ShardRoutingRoleStrategy.NO_SHARD_CREATION,
+            indexMetadata.getIndex()
+        );
 
         // Primary
         {
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponseTests.java
index 67feac2aa438..b7242c4ffcee 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponseTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.Collections;
 
 import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.ERROR;
@@ -41,7 +40,7 @@ protected GetFeatureUpgradeStatusResponse createTestInstance() {
     }
 
     @Override
-    protected GetFeatureUpgradeStatusResponse mutateInstance(GetFeatureUpgradeStatusResponse instance) throws IOException {
+    protected GetFeatureUpgradeStatusResponse mutateInstance(GetFeatureUpgradeStatusResponse instance) {
         return new GetFeatureUpgradeStatusResponse(
             randomList(
                 8,
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeResponseTests.java
index 0704eaa32774..1c6486de4612 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeResponseTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 
@@ -43,7 +42,7 @@ protected PostFeatureUpgradeResponse createTestInstance() {
     }
 
     @Override
-    protected PostFeatureUpgradeResponse mutateInstance(PostFeatureUpgradeResponse instance) throws IOException {
+    protected PostFeatureUpgradeResponse mutateInstance(PostFeatureUpgradeResponse instance) {
         if (instance.isAccepted()) {
             return new PostFeatureUpgradeResponse(
                 true,
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodePrevalidateShardPathRequestSerializationTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodePrevalidateShardPathRequestSerializationTests.java
new file mode 100644
index 000000000000..16a81f212458
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodePrevalidateShardPathRequestSerializationTests.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.cluster.node.shutdown;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import static org.elasticsearch.action.admin.cluster.node.shutdown.PrevalidateShardPathRequestSerializationTests.createSetMutation;
+
+public class NodePrevalidateShardPathRequestSerializationTests extends AbstractWireSerializingTestCase<NodePrevalidateShardPathRequest> {
+
+    @Override
+    protected Writeable.Reader<NodePrevalidateShardPathRequest> instanceReader() {
+        return NodePrevalidateShardPathRequest::new;
+    }
+
+    @Override
+    protected NodePrevalidateShardPathRequest createTestInstance() {
+        return new NodePrevalidateShardPathRequest(randomSet(0, 50, PrevalidateShardPathRequestSerializationTests::randomShardId));
+    }
+
+    @Override
+    protected NodePrevalidateShardPathRequest mutateInstance(NodePrevalidateShardPathRequest request) {
+        return new NodePrevalidateShardPathRequest(
+            createSetMutation(request.getShardIds(), PrevalidateShardPathRequestSerializationTests::randomShardId)
+        );
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodePrevalidateShardPathResponseSerializationTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodePrevalidateShardPathResponseSerializationTests.java
new file mode 100644
index 000000000000..0e41952b69a5
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodePrevalidateShardPathResponseSerializationTests.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.cluster.node.shutdown;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import static org.elasticsearch.action.admin.cluster.node.shutdown.PrevalidateShardPathRequestSerializationTests.createSetMutation;
+
+public class NodePrevalidateShardPathResponseSerializationTests extends AbstractWireSerializingTestCase<NodePrevalidateShardPathResponse> {
+
+    @Override
+    protected Writeable.Reader<NodePrevalidateShardPathResponse> instanceReader() {
+        return NodePrevalidateShardPathResponse::new;
+    }
+
+    @Override
+    protected NodePrevalidateShardPathResponse createTestInstance() {
+        return getRandomResponse();
+    }
+
+    public static NodePrevalidateShardPathResponse getRandomResponse() {
+        return new NodePrevalidateShardPathResponse(
+            getRandomNode(),
+            randomSet(0, 100, PrevalidateShardPathRequestSerializationTests::randomShardId)
+        );
+    }
+
+    public static DiscoveryNode getRandomNode() {
+        return new DiscoveryNode(randomAlphaOfLength(10), buildNewFakeTransportAddress(), Version.CURRENT);
+    }
+
+    @Override
+    protected NodePrevalidateShardPathResponse mutateInstance(NodePrevalidateShardPathResponse response) {
+        if (randomBoolean()) {
+            return new NodePrevalidateShardPathResponse(getRandomNode(), response.getShardIds());
+        }
+        return new NodePrevalidateShardPathResponse(
+            response.getNode(),
+            createSetMutation(response.getShardIds(), PrevalidateShardPathRequestSerializationTests::randomShardId)
+        );
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodesRemovalPrevalidationSerializationTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodesRemovalPrevalidationSerializationTests.java
index 54b428bdd2cd..f1ac17e8379f 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodesRemovalPrevalidationSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/shutdown/NodesRemovalPrevalidationSerializationTests.java
@@ -36,7 +36,7 @@ protected NodesRemovalPrevalidation createTestInstance() {
     }
 
     @Override
-    protected NodesRemovalPrevalidation mutateInstance(NodesRemovalPrevalidation instance) throws IOException {
+    protected NodesRemovalPrevalidation mutateInstance(NodesRemovalPrevalidation instance) {
         return randomBoolean() ? mutateOverallResult(instance) : mutateNodes(instance);
     }
 
@@ -87,7 +87,6 @@ public static NodeResult mutateNodeResult(final NodeResult instance) {
     public static NodesRemovalPrevalidation randomNodesRemovalPrevalidation() {
         int noOfNodes = randomIntBetween(1, 10);
         List<NodesRemovalPrevalidation.NodeResult> nodes = new ArrayList<>(noOfNodes);
-        NodesRemovalPrevalidation.Result result = randomResult();
         for (int i = 0; i < noOfNodes; i++) {
             nodes.add(
                 new NodesRemovalPrevalidation.NodeResult(
@@ -102,6 +101,10 @@ public static NodesRemovalPrevalidation randomNodesRemovalPrevalidation() {
     }
 
     private static NodesRemovalPrevalidation.Result randomResult() {
-        return new NodesRemovalPrevalidation.Result(randomBoolean(), randomAlphaOfLengthBetween(0, 1000));
+        return new NodesRemovalPrevalidation.Result(
+            randomBoolean(),
+            randomFrom(NodesRemovalPrevalidation.Reason.values()),
+            randomAlphaOfLengthBetween(0, 1000)
+        );
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalRequestSerializationTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalRequestSerializationTests.java
index 616124a57083..584ad10a5a16 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalRequestSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalRequestSerializationTests.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.Arrays;
 
 public class PrevalidateNodeRemovalRequestSerializationTests extends AbstractWireSerializingTestCase<PrevalidateNodeRemovalRequest> {
@@ -27,7 +26,7 @@ protected PrevalidateNodeRemovalRequest createTestInstance() {
     }
 
     @Override
-    protected PrevalidateNodeRemovalRequest mutateInstance(PrevalidateNodeRemovalRequest request) throws IOException {
+    protected PrevalidateNodeRemovalRequest mutateInstance(PrevalidateNodeRemovalRequest request) {
         int i = randomIntBetween(0, 2);
         return switch (i) {
             case 0 -> PrevalidateNodeRemovalRequest.builder()
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalResponseSerializationTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalResponseSerializationTests.java
index 9968e9e7afc6..88381146d910 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalResponseSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateNodeRemovalResponseSerializationTests.java
@@ -27,7 +27,7 @@ protected PrevalidateNodeRemovalResponse createTestInstance() {
     }
 
     @Override
-    protected PrevalidateNodeRemovalResponse mutateInstance(PrevalidateNodeRemovalResponse instance) throws IOException {
+    protected PrevalidateNodeRemovalResponse mutateInstance(PrevalidateNodeRemovalResponse instance) {
         return new PrevalidateNodeRemovalResponse(NodesRemovalPrevalidationSerializationTests.mutateNodes(instance.getPrevalidation()));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateShardPathRequestSerializationTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateShardPathRequestSerializationTests.java
new file mode 100644
index 000000000000..81123a0180c7
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateShardPathRequestSerializationTests.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.cluster.node.shutdown;
+
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.function.IntFunction;
+import java.util.function.Supplier;
+
+public class PrevalidateShardPathRequestSerializationTests extends AbstractWireSerializingTestCase<PrevalidateShardPathRequest> {
+
+    @Override
+    protected Writeable.Reader<PrevalidateShardPathRequest> instanceReader() {
+        return PrevalidateShardPathRequest::new;
+    }
+
+    @Override
+    protected PrevalidateShardPathRequest createTestInstance() {
+        Set<ShardId> shardIds = randomSet(0, 100, PrevalidateShardPathRequestSerializationTests::randomShardId);
+        String[] nodeIds = randomArray(1, 5, String[]::new, () -> randomAlphaOfLength(20));
+        PrevalidateShardPathRequest request = new PrevalidateShardPathRequest(shardIds, nodeIds);
+        return randomBoolean() ? request : request.timeout(randomTimeValue());
+    }
+
+    @Override
+    protected PrevalidateShardPathRequest mutateInstance(PrevalidateShardPathRequest request) {
+        int i = randomInt(2);
+        return switch (i) {
+            case 0 -> new PrevalidateShardPathRequest(
+                createSetMutation(request.getShardIds(), PrevalidateShardPathRequestSerializationTests::randomShardId),
+                request.nodesIds()
+            ).timeout(request.timeout());
+            case 1 -> new PrevalidateShardPathRequest(
+                request.getShardIds(),
+                createArrayMutation(request.nodesIds(), () -> randomAlphaOfLength(20), String[]::new)
+            ).timeout(request.timeout());
+            case 2 -> new PrevalidateShardPathRequest(request.getShardIds(), request.nodesIds()).timeout(
+                randomValueOtherThan(request.timeout(), () -> new TimeValue(randomLongBetween(1000, 10000)))
+            );
+            default -> throw new IllegalStateException("unexpected value: " + i);
+        };
+    }
+
+    public static ShardId randomShardId() {
+        return new ShardId(randomAlphaOfLength(20), UUIDs.randomBase64UUID(), randomIntBetween(0, 25));
+    }
+
+    public static <T> void mutateList(List<T> list, Supplier<T> supplier) {
+        if (list.size() > 0 && randomBoolean()) {
+            // just remove one
+            list.remove(randomInt(list.size() - 1));
+        } else {
+            list.add(supplier.get());
+        }
+    }
+
+    public static <T> Set<T> createSetMutation(Set<T> set, Supplier<T> supplier) {
+        List<T> list = new ArrayList<>(set);
+        mutateList(list, supplier);
+        return new HashSet<>(list);
+    }
+
+    public static <T> T[] createArrayMutation(T[] array, Supplier<T> supplier, IntFunction<T[]> arrayConstructor) {
+        List<T> list = new ArrayList<>(Arrays.asList(array));
+        mutateList(list, supplier);
+        return list.toArray(arrayConstructor.apply(list.size()));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java
index 6b8a38e372e9..adb575315132 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java
@@ -635,8 +635,9 @@ private static ShardStats createShardStats(ShardId shardId) {
         ShardRouting shardRouting = ShardRouting.newUnassigned(
             shardId,
             true,
-            RecoverySource.PeerRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "message")
+            RecoverySource.EmptyStoreRecoverySource.INSTANCE,
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "message"),
+            ShardRouting.Role.DEFAULT
         );
         Path path = createTempDir().resolve("indices")
             .resolve(shardRouting.shardId().getIndex().getUUID())
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
index cf3c30b4f512..a9c6b4fe093b 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
@@ -10,7 +10,7 @@
 import com.carrotsearch.randomizedtesting.RandomizedContext;
 import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksAction;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
@@ -408,7 +408,7 @@ public void testRegisterAndExecuteChildTaskWhileParentTaskIsBeingCanceled() thro
                 threadPool,
                 "test",
                 randomNonNegativeLong(),
-                Version.CURRENT
+                TransportVersion.CURRENT
             )
         );
         CancellableNodesRequest childRequest = new CancellableNodesRequest("child");
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
index 8752f493f456..0f41f1c30638 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.action.admin.cluster.node.tasks;
 
 import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.TransportCancelTasksAction;
@@ -188,7 +189,7 @@ public TestNode(String name, ThreadPool threadPool, Settings settings) {
                 settings,
                 new Netty4Transport(
                     settings,
-                    Version.CURRENT,
+                    TransportVersion.CURRENT,
                     threadPool,
                     new NetworkService(Collections.emptyList()),
                     PageCacheRecycler.NON_RECYCLING_INSTANCE,
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
index d7c0cb347e7e..c00b6cbf2dca 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
@@ -7,6 +7,8 @@
  */
 package org.elasticsearch.action.admin.cluster.node.tasks;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
@@ -69,6 +71,8 @@
  */
 public class TestTaskPlugin extends Plugin implements ActionPlugin, NetworkPlugin {
 
+    private static final Logger logger = LogManager.getLogger(TestTaskPlugin.class);
+
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
         return Arrays.asList(
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
index 87ffd5ea5be9..e27372ad785f 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
@@ -889,14 +890,12 @@ public void testTasksToXContentGrouping() throws Exception {
 
     private Map<String, Object> serialize(ListTasksResponse response, boolean byParents) throws IOException {
         XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
-        builder.startObject();
         if (byParents) {
             DiscoveryNodes nodes = testNodes[0].clusterService.state().nodes();
-            response.toXContentGroupedByNode(builder, ToXContent.EMPTY_PARAMS, nodes);
+            ChunkedToXContent.wrapAsToXContent(response.groupedByNode(() -> nodes)).toXContent(builder, ToXContent.EMPTY_PARAMS);
         } else {
-            response.toXContentGroupedByParents(builder, ToXContent.EMPTY_PARAMS);
+            ChunkedToXContent.wrapAsToXContent(response.groupedByParent()).toXContent(builder, ToXContent.EMPTY_PARAMS);
         }
-        builder.endObject();
         builder.flush();
         logger.info(Strings.toString(builder));
         return XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2();
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java
index 98f8e7385a2f..6d84e6392b5d 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java
@@ -22,18 +22,20 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.json.JsonXContent;
 
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.common.util.CollectionUtils.appendToCopy;
-import static org.hamcrest.Matchers.equalTo;
+import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 
 public class ClusterRerouteResponseTests extends ESTestCase {
 
@@ -43,54 +45,45 @@ protected List<String> filteredWarnings() {
     }
 
     public void testToXContent() throws IOException {
-        var clusterState = createClusterState();
-        var clusterRerouteResponse = createClusterRerouteResponse(clusterState);
-
-        var result = toXContent(clusterRerouteResponse, new ToXContent.MapParams(Map.of("metric", "none")));
-
-        assertThat(result, equalTo(XContentHelper.stripWhitespace("""
+        assertXContent(createClusterRerouteResponse(createClusterState()), new ToXContent.MapParams(Map.of("metric", "none")), 2, """
             {
               "acknowledged": true
-            }""")));
+            }""");
     }
 
-    public void testToXContentWithExplain() throws IOException {
+    public void testToXContentWithExplain() {
         var clusterState = createClusterState();
-        var clusterRerouteResponse = createClusterRerouteResponse(clusterState);
-
-        var result = toXContent(clusterRerouteResponse, new ToXContent.MapParams(Map.of("explain", "true", "metric", "none")));
-
-        assertThat(result, equalTo(XContentHelper.stripWhitespace(formatted("""
-            {
-              "acknowledged": true,
-              "explanations": [
+        assertXContent(
+            createClusterRerouteResponse(clusterState),
+            new ToXContent.MapParams(Map.of("explain", "true", "metric", "none")),
+            2,
+            Strings.format("""
                 {
-                  "command": "allocate_replica",
-                  "parameters": {
-                    "index": "index",
-                    "shard": 0,
-                    "node": "node0"
-                  },
-                  "decisions": [
+                  "acknowledged": true,
+                  "explanations": [
                     {
-                      "decider": null,
-                      "decision": "YES",
-                      "explanation": "none"
+                      "command": "allocate_replica",
+                      "parameters": {
+                        "index": "index",
+                        "shard": 0,
+                        "node": "node0"
+                      },
+                      "decisions": [
+                        {
+                          "decider": null,
+                          "decision": "YES",
+                          "explanation": "none"
+                        }
+                      ]
                     }
                   ]
-                }
-              ]
-            }""", clusterState.stateUUID()))));
-
+                }""", clusterState.stateUUID())
+        );
     }
 
-    public void testToXContentWithDeprecatedClusterState() throws IOException {
+    public void testToXContentWithDeprecatedClusterState() {
         var clusterState = createClusterState();
-        var clusterRerouteResponse = createClusterRerouteResponse(clusterState);
-
-        var result = toXContent(clusterRerouteResponse, ToXContent.EMPTY_PARAMS);
-
-        assertThat(result, equalTo(XContentHelper.stripWhitespace(formatted("""
+        assertXContent(createClusterRerouteResponse(clusterState), ToXContent.EMPTY_PARAMS, 32, Strings.format("""
             {
               "acknowledged": true,
               "state": {
@@ -121,7 +114,8 @@ public void testToXContentWithDeprecatedClusterState() throws IOException {
                       "search",
                       "transform",
                       "voting_only"
-                    ]
+                    ],
+                    "version": "%s"
                   }
                 },
                 "metadata": {
@@ -185,77 +179,110 @@ public void testToXContentWithDeprecatedClusterState() throws IOException {
                   }
                 }
               }
-            }""", clusterState.stateUUID(), clusterState.getNodes().get("node0").getEphemeralId(), Version.CURRENT.id))));
+            }""", clusterState.stateUUID(), clusterState.getNodes().get("node0").getEphemeralId(), Version.CURRENT, Version.CURRENT.id), """
+            The [state] field in the response to the reroute API is deprecated and will be removed in a future version. \
+            Specify ?metric=none to adopt the future behaviour.""");
     }
 
-    public void testToXContentWithDeprecatedClusterStateAndMetadata() throws IOException {
-        var clusterState = createClusterState();
-        var clusterRerouteResponse = createClusterRerouteResponse(clusterState);
-
-        var result = toXContent(
-            clusterRerouteResponse,
-            new ToXContent.MapParams(Map.of("metric", "metadata", "settings_filter", "index.number*,index.version.created"))
-        );
-
-        assertThat(result, equalTo(XContentHelper.stripWhitespace("""
-            {
-              "acknowledged" : true,
-              "state" : {
-                "cluster_uuid" : "_na_",
-                "metadata" : {
-                  "cluster_uuid" : "_na_",
-                  "cluster_uuid_committed" : false,
-                  "cluster_coordination" : {
-                    "term" : 0,
-                    "last_committed_config" : [ ],
-                    "last_accepted_config" : [ ],
-                    "voting_config_exclusions" : [ ]
-                  },
-                  "templates" : { },
-                  "indices" : {
-                    "index" : {
-                      "version" : 1,
-                      "mapping_version" : 1,
-                      "settings_version" : 1,
-                      "aliases_version" : 1,
-                      "routing_num_shards" : 1,
-                      "state" : "open",
-                      "settings" : {
+    public void testToXContentWithDeprecatedClusterStateAndMetadata() {
+        assertXContent(
+            createClusterRerouteResponse(createClusterState()),
+            new ToXContent.MapParams(Map.of("metric", "metadata", "settings_filter", "index.number*,index.version.created")),
+            19,
+            """
+                {
+                  "acknowledged" : true,
+                  "state" : {
+                    "cluster_uuid" : "_na_",
+                    "metadata" : {
+                      "cluster_uuid" : "_na_",
+                      "cluster_uuid_committed" : false,
+                      "cluster_coordination" : {
+                        "term" : 0,
+                        "last_committed_config" : [ ],
+                        "last_accepted_config" : [ ],
+                        "voting_config_exclusions" : [ ]
+                      },
+                      "templates" : { },
+                      "indices" : {
                         "index" : {
-                          "max_script_fields" : "10",
-                          "shard" : {
-                            "check_on_startup" : "true"
+                          "version" : 1,
+                          "mapping_version" : 1,
+                          "settings_version" : 1,
+                          "aliases_version" : 1,
+                          "routing_num_shards" : 1,
+                          "state" : "open",
+                          "settings" : {
+                            "index" : {
+                              "max_script_fields" : "10",
+                              "shard" : {
+                                "check_on_startup" : "true"
+                              }
+                            }
+                          },
+                          "mappings" : { },
+                          "aliases" : [ ],
+                          "primary_terms" : {
+                            "0" : 0
+                          },
+                          "in_sync_allocations" : {
+                            "0" : [ ]
+                          },
+                          "rollover_info" : { },
+                          "system" : false,
+                          "timestamp_range" : {
+                            "shards" : [ ]
                           }
                         }
                       },
-                      "mappings" : { },
-                      "aliases" : [ ],
-                      "primary_terms" : {
-                        "0" : 0
-                      },
-                      "in_sync_allocations" : {
-                        "0" : [ ]
+                      "index-graveyard" : {
+                        "tombstones" : [ ]
                       },
-                      "rollover_info" : { },
-                      "system" : false,
-                      "timestamp_range" : {
-                        "shards" : [ ]
-                      }
+                      "reserved_state":{}
                     }
-                  },
-                  "index-graveyard" : {
-                    "tombstones" : [ ]
-                  },
-                  "reserved_state":{}
-                }
-              }
-            }""")));
+                  }
+                }""",
+            """
+                The [state] field in the response to the reroute API is deprecated and will be removed in a future version. \
+                Specify ?metric=none to adopt the future behaviour."""
+        );
     }
 
-    private static String toXContent(ClusterRerouteResponse clusterRerouteResponse, ToXContent.Params params) throws IOException {
-        var builder = JsonXContent.contentBuilder().prettyPrint();
-        clusterRerouteResponse.toXContent(builder, params);
-        return XContentHelper.stripWhitespace(Strings.toString(builder));
+    private void assertXContent(
+        ClusterRerouteResponse response,
+        ToXContent.Params params,
+        int expectedChunks,
+        String expectedBody,
+        String... criticalDeprecationWarnings
+    ) {
+        try {
+            var builder = jsonBuilder();
+            if (randomBoolean()) {
+                builder.prettyPrint();
+            }
+            ChunkedToXContent.wrapAsToXContent(response).toXContent(builder, params);
+            assertEquals(XContentHelper.stripWhitespace(expectedBody), XContentHelper.stripWhitespace(Strings.toString(builder)));
+        } catch (IOException e) {
+            throw new AssertionError("unexpected", e);
+        }
+
+        AbstractChunkedSerializingTestCase.assertChunkCount(response, params, ignored -> expectedChunks);
+        assertCriticalWarnings(criticalDeprecationWarnings);
+
+        // check the v7 API too
+        AbstractChunkedSerializingTestCase.assertChunkCount(new ChunkedToXContent() {
+            @Override
+            public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
+                return response.toXContentChunkedV7(outerParams);
+            }
+
+            @Override
+            public boolean isFragment() {
+                return response.isFragment();
+            }
+        }, params, ignored -> expectedChunks);
+        // the v7 API should not emit any deprecation warnings
+        assertCriticalWarnings();
     }
 
     private static ClusterRerouteResponse createClusterRerouteResponse(ClusterState clusterState) {
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
index 19a699a5eaee..af5317149226 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -75,7 +76,8 @@ public void testClusterStateUpdateTaskInDryRun() {
             new TestGatewayAllocator(),
             new BalancedShardsAllocator(Settings.EMPTY),
             EmptyClusterInfoService.INSTANCE,
-            EmptySnapshotsInfoService.INSTANCE
+            EmptySnapshotsInfoService.INSTANCE,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
         ClusterState clusterState = createInitialClusterState(allocationService);
 
@@ -106,7 +108,8 @@ public void testClusterStateUpdateTask() {
             new TestGatewayAllocator(),
             new BalancedShardsAllocator(Settings.EMPTY),
             EmptyClusterInfoService.INSTANCE,
-            EmptySnapshotsInfoService.INSTANCE
+            EmptySnapshotsInfoService.INSTANCE,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
         ClusterState clusterState = createInitialClusterState(allocationService);
 
@@ -167,7 +170,7 @@ private ClusterState createInitialClusterState(AllocationService service) {
         Metadata.Builder metaBuilder = Metadata.builder();
         metaBuilder.put(IndexMetadata.builder("idx").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0));
         Metadata metadata = metaBuilder.build();
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         routingTableBuilder.addAsNew(metadata.index("idx"));
 
         RoutingTable routingTable = routingTableBuilder.build();
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponseTests.java
index 5a1fb26866be..802b8c37342e 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/GetSnapshottableFeaturesResponseTests.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -36,7 +35,7 @@ protected GetSnapshottableFeaturesResponse createTestInstance() {
     }
 
     @Override
-    protected GetSnapshottableFeaturesResponse mutateInstance(GetSnapshottableFeaturesResponse instance) throws IOException {
+    protected GetSnapshottableFeaturesResponse mutateInstance(GetSnapshottableFeaturesResponse instance) {
         int minSize = 0;
         if (instance.getSnapshottableFeatures().size() == 0) {
             minSize = 1;
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateResponseTests.java
index afa7afab1d5d..84377782c824 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/features/ResetFeatureStateResponseTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
@@ -46,7 +45,7 @@ protected ResetFeatureStateResponse createTestInstance() {
     }
 
     @Override
-    protected ResetFeatureStateResponse mutateInstance(ResetFeatureStateResponse instance) throws IOException {
+    protected ResetFeatureStateResponse mutateInstance(ResetFeatureStateResponse instance) {
         int minSize = 0;
         if (instance.getFeatureStateResetStatuses().size() == 0) {
             minSize = 1;
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java
index c2a3a265a33c..594177302660 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotInfoTestUtils;
 import org.elasticsearch.snapshots.SnapshotShardFailure;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentParser;
@@ -34,7 +35,6 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -177,15 +177,8 @@ public void testFromXContent() throws IOException {
             .test();
     }
 
-    public void testToChunkedXContent() {
-        final GetSnapshotsResponse response = createTestInstance();
-        final Iterator<ToXContent> serialization = response.toXContentChunked();
-        int chunks = 0;
-        while (serialization.hasNext()) {
-            serialization.next();
-            chunks++;
-        }
-        assertEquals(chunks, response.getSnapshots().size() + 2);
+    public void testChunking() {
+        AbstractChunkedSerializingTestCase.assertChunkCount(createTestInstance(), response -> response.getSnapshots().size() + 2);
     }
 
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/GetShardSnapshotRequestSerializationTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/GetShardSnapshotRequestSerializationTests.java
index eb38e878571e..fc75fb6650b1 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/GetShardSnapshotRequestSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/shard/GetShardSnapshotRequestSerializationTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 
@@ -39,7 +38,7 @@ protected GetShardSnapshotRequest createTestInstance() {
     }
 
     @Override
-    protected GetShardSnapshotRequest mutateInstance(GetShardSnapshotRequest instance) throws IOException {
+    protected GetShardSnapshotRequest mutateInstance(GetShardSnapshotRequest instance) {
         ShardId shardId = randomShardId();
         if (instance.getFromAllRepositories()) {
             return GetShardSnapshotRequest.latestSnapshotInAllRepositories(shardId);
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java
index fe5d4e92bd22..9c28930f1238 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.cluster.SnapshotsInProgress;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
@@ -55,7 +56,7 @@ public void testToString() throws Exception {
             case FAILURE -> failedShards++;
         }
 
-        String expected = formatted(
+        String expected = Strings.format(
             """
                 {
                   "snapshot" : "test-snap",
@@ -166,6 +167,11 @@ protected SnapshotStatus createTestInstance() {
         return new SnapshotStatus(snapshot, state, snapshotIndexShardStatuses, includeGlobalState, 0L, 0L);
     }
 
+    @Override
+    protected SnapshotStatus mutateInstance(SnapshotStatus instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Predicate<String> getRandomFieldsExcludeFilter() {
         // Do not place random fields in the indices field or shards field since their fields correspond to names.
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponseTests.java
index df4fe643ab25..f90b37f75fa4 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponseTests.java
@@ -45,25 +45,20 @@ protected SnapshotsStatusResponse createTestInstance() {
         return new SnapshotsStatusResponse(snapshotStatuses);
     }
 
+    @Override
+    protected SnapshotsStatusResponse mutateInstance(SnapshotsStatusResponse instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<SnapshotsStatusResponse> instanceReader() {
         return SnapshotsStatusResponse::new;
     }
 
-    public void testChunkCount() {
-        final var instance = createTestInstance();
-        // open and close chunk
-        int chunksExpected = 2;
-        for (SnapshotStatus snapshot : instance.getSnapshots()) {
-            // open and close chunk + one chunk per index
-            chunksExpected += 2 + snapshot.getIndices().size();
-        }
-        final var iterator = instance.toXContentChunked();
-        int chunksSeen = 0;
-        while (iterator.hasNext()) {
-            iterator.next();
-            chunksSeen++;
-        }
-        assertEquals(chunksExpected, chunksSeen);
+    public void testChunking() {
+        AbstractChunkedSerializingTestCase.assertChunkCount(
+            createTestInstance(),
+            instance -> 2 + instance.getSnapshots().stream().mapToInt(i -> 2 + i.getIndices().size()).sum()
+        );
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponseTests.java
index 7de06ad4906a..37829d40ff16 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponseTests.java
@@ -46,6 +46,11 @@ protected ClusterStateResponse createTestInstance() {
         return new ClusterStateResponse(clusterName, clusterState, randomBoolean());
     }
 
+    @Override
+    protected ClusterStateResponse mutateInstance(ClusterStateResponse instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<ClusterStateResponse> instanceReader() {
         return ClusterStateResponse::new;
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/AnalysisStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/AnalysisStatsTests.java
index 66cf0ccdc6a1..ef74123779ac 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/AnalysisStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/AnalysisStatsTests.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
@@ -89,7 +88,7 @@ protected AnalysisStats createTestInstance() {
     }
 
     @Override
-    protected AnalysisStats mutateInstance(AnalysisStats instance) throws IOException {
+    protected AnalysisStats mutateInstance(AnalysisStats instance) {
         switch (randomInt(7)) {
             case 0 -> {
                 Set<IndexFeatureStats> charFilters = new HashSet<>(instance.getUsedCharFilterTypes());
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodesTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodesTests.java
index 66305cdf5fef..8f642d48de12 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodesTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodesTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.stats.IndexingPressureStats;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentType;
@@ -92,7 +93,7 @@ public void testIngestStats() throws Exception {
             long failedCount = statValues[1];
             long current = statValues[2];
             long timeInMillis = statValues[3];
-            processorStatsString.append(formatted("""
+            processorStatsString.append(Strings.format("""
                 "%s":{"count":%s,"failed":%s,"current":%s,"time_in_millis":%s}\
                 """, entry.getKey(), count, failedCount, current, timeInMillis));
             if (iter.hasNext()) {
@@ -100,7 +101,7 @@ public void testIngestStats() throws Exception {
             }
         }
         processorStatsString.append("}");
-        assertThat(toXContent(stats, XContentType.JSON, false).utf8ToString(), equalTo(formatted("""
+        assertThat(toXContent(stats, XContentType.JSON, false).utf8ToString(), equalTo(Strings.format("""
             {"ingest":{"number_of_pipelines":%s,"processor_stats":%s}}\
             """, stats.pipelineCount, processorStatsString)));
     }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/IndexFeatureStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/IndexFeatureStatsTests.java
index 8e6000b601ea..9cce74ff21ad 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/IndexFeatureStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/IndexFeatureStatsTests.java
@@ -26,9 +26,14 @@ protected IndexFeatureStats createTestInstance() {
         return indexFeatureStats;
     }
 
+    @Override
+    protected IndexFeatureStats mutateInstance(IndexFeatureStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testToXContent() {
         IndexFeatureStats testInstance = createTestInstance();
-        assertEquals(formatted("""
+        assertEquals(Strings.format("""
             {"name":"%s","count":%s,"index_count":%s}\
             """, testInstance.name, testInstance.count, testInstance.indexCount), Strings.toString(testInstance));
     }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java
index 9c115924798f..cdaaa56e6b53 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java
@@ -96,7 +96,7 @@ public class MappingStatsTests extends AbstractWireSerializingTestCase<MappingSt
     private static final String SCRIPT_4 = scriptAsJSON("params._source.field");
 
     public void testToXContent() {
-        String mapping = formatted(MAPPING_TEMPLATE, SCRIPT_1, SCRIPT_2, SCRIPT_3, SCRIPT_4, SCRIPT_3, SCRIPT_4, SCRIPT_1);
+        String mapping = Strings.format(MAPPING_TEMPLATE, SCRIPT_1, SCRIPT_2, SCRIPT_3, SCRIPT_4, SCRIPT_3, SCRIPT_4, SCRIPT_1);
         IndexMetadata meta = IndexMetadata.builder("index").settings(SINGLE_SHARD_NO_REPLICAS).putMapping(mapping).build();
         IndexMetadata meta2 = IndexMetadata.builder("index2").settings(SINGLE_SHARD_NO_REPLICAS).putMapping(mapping).build();
         Metadata metadata = Metadata.builder().put(meta, false).put(meta2, false).build();
@@ -204,10 +204,19 @@ public void testToXContent() {
     public void testToXContentWithSomeSharedMappings() {
         IndexMetadata meta = IndexMetadata.builder("index")
             .settings(SINGLE_SHARD_NO_REPLICAS)
-            .putMapping(formatted(MAPPING_TEMPLATE, SCRIPT_1, SCRIPT_2, SCRIPT_3, SCRIPT_4, SCRIPT_3, SCRIPT_4, SCRIPT_1))
+            .putMapping(Strings.format(MAPPING_TEMPLATE, SCRIPT_1, SCRIPT_2, SCRIPT_3, SCRIPT_4, SCRIPT_3, SCRIPT_4, SCRIPT_1))
             .build();
         // make mappings that are slightly different because we shuffled 2 scripts between fields
-        final String mappingString2 = formatted(MAPPING_TEMPLATE, SCRIPT_1, SCRIPT_2, SCRIPT_3, SCRIPT_4, SCRIPT_4, SCRIPT_3, SCRIPT_1);
+        final String mappingString2 = Strings.format(
+            MAPPING_TEMPLATE,
+            SCRIPT_1,
+            SCRIPT_2,
+            SCRIPT_3,
+            SCRIPT_4,
+            SCRIPT_4,
+            SCRIPT_3,
+            SCRIPT_1
+        );
         IndexMetadata meta2 = IndexMetadata.builder("index2").settings(SINGLE_SHARD_NO_REPLICAS).putMapping(mappingString2).build();
         IndexMetadata meta3 = IndexMetadata.builder("index3").settings(SINGLE_SHARD_NO_REPLICAS).putMapping(mappingString2).build();
         Metadata metadata = Metadata.builder().put(meta, false).put(meta2, false).put(meta3, false).build();
@@ -379,7 +388,7 @@ private static RuntimeFieldStats randomRuntimeFieldStats(String type) {
 
     @SuppressWarnings("OptionalGetWithoutIsPresent")
     @Override
-    protected MappingStats mutateInstance(MappingStats instance) throws IOException {
+    protected MappingStats mutateInstance(MappingStats instance) {
         List<FieldStats> fieldTypes = new ArrayList<>(instance.getFieldTypeStats());
         List<RuntimeFieldStats> runtimeFieldTypes = new ArrayList<>(instance.getRuntimeFieldStats());
         long totalFieldCount = instance.getTotalFieldCount().getAsLong();
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStatsTests.java
index 4b578006bf94..1e8bc1e17d52 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/SearchUsageStatsTests.java
@@ -75,7 +75,7 @@ protected SearchUsageStats createTestInstance() {
     }
 
     @Override
-    protected SearchUsageStats mutateInstance(SearchUsageStats instance) throws IOException {
+    protected SearchUsageStats mutateInstance(SearchUsageStats instance) {
         if (randomBoolean()) {
             return new SearchUsageStats(
                 randomQueryUsage(instance.getQueryUsage().size() + 1),
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java
index b31b26fff961..04e07b212fc5 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.index.store.StoreStats;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -52,7 +51,7 @@ protected VersionStats createTestInstance() {
     }
 
     @Override
-    protected VersionStats mutateInstance(VersionStats instance) throws IOException {
+    protected VersionStats mutateInstance(VersionStats instance) {
         return new VersionStats(instance.versionStats().stream().map(svs -> {
             return switch (randomIntBetween(1, 4)) {
                 case 1 -> new VersionStats.SingleVersionStats(
@@ -103,8 +102,9 @@ public void testCreation() {
         ShardRouting shardRouting = ShardRouting.newUnassigned(
             shardId,
             true,
-            RecoverySource.PeerRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "message")
+            RecoverySource.EmptyStoreRecoverySource.INSTANCE,
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "message"),
+            ShardRouting.Role.DEFAULT
         );
         Path path = createTempDir().resolve("indices")
             .resolve(shardRouting.shardId().getIndex().getUUID())
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextResponseTests.java
index df1bcdfcd9fe..efb1e61e19fa 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptContextResponseTests.java
@@ -35,7 +35,7 @@ protected GetScriptContextResponse doParseInstance(XContentParser parser) throws
     }
 
     @Override
-    protected GetScriptContextResponse mutateInstance(GetScriptContextResponse instance) throws IOException {
+    protected GetScriptContextResponse mutateInstance(GetScriptContextResponse instance) {
         return new GetScriptContextResponse(ScriptContextInfoSerializingTests.mutateOne(instance.contexts.values()));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageResponseTests.java
index f7a255b65881..ff70e7e6756e 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetScriptLanguageResponseTests.java
@@ -47,7 +47,7 @@ protected Writeable.Reader<GetScriptLanguageResponse> instanceReader() {
     }
 
     @Override
-    protected GetScriptLanguageResponse mutateInstance(GetScriptLanguageResponse instance) throws IOException {
+    protected GetScriptLanguageResponse mutateInstance(GetScriptLanguageResponse instance) {
         switch (randomInt(2)) {
             case 0:
                 // mutate typesAllowed
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java
index e5ecc8722146..cf7bda6fc4c0 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java
@@ -14,7 +14,7 @@
 
 import java.io.IOException;
 
-import static org.elasticsearch.test.VersionUtils.randomVersion;
+import static org.elasticsearch.test.TransportVersionUtils.randomVersion;
 import static org.hamcrest.CoreMatchers.equalTo;
 
 public class GetStoredScriptRequestTests extends ESTestCase {
@@ -22,11 +22,11 @@ public void testGetIndexedScriptRequestSerialization() throws IOException {
         GetStoredScriptRequest request = new GetStoredScriptRequest("id");
 
         BytesStreamOutput out = new BytesStreamOutput();
-        out.setVersion(randomVersion(random()));
+        out.setTransportVersion(randomVersion(random()));
         request.writeTo(out);
 
         StreamInput in = out.bytes().streamInput();
-        in.setVersion(out.getVersion());
+        in.setTransportVersion(out.getTransportVersion());
         GetStoredScriptRequest request2 = new GetStoredScriptRequest(in);
 
         assertThat(request2.id(), equalTo(request.id()));
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptResponseTests.java
index 7c5c2b0e998a..500080fa9f11 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptResponseTests.java
@@ -32,6 +32,11 @@ protected GetStoredScriptResponse createTestInstance() {
         return new GetStoredScriptResponse(randomAlphaOfLengthBetween(1, 10), randomScriptSource());
     }
 
+    @Override
+    protected GetStoredScriptResponse mutateInstance(GetStoredScriptResponse instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<GetStoredScriptResponse> instanceReader() {
         return GetStoredScriptResponse::new;
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/ScriptContextInfoSerializingTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/ScriptContextInfoSerializingTests.java
index 56e36af437db..0fb7f966fb01 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/ScriptContextInfoSerializingTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/ScriptContextInfoSerializingTests.java
@@ -42,7 +42,7 @@ protected Writeable.Reader<ScriptContextInfo> instanceReader() {
     }
 
     @Override
-    protected ScriptContextInfo mutateInstance(ScriptContextInfo instance) throws IOException {
+    protected ScriptContextInfo mutateInstance(ScriptContextInfo instance) {
         return mutate(instance, null);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/ScriptMethodInfoSerializingTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/ScriptMethodInfoSerializingTests.java
index 0ea64c81fc31..b2b690babf8a 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/ScriptMethodInfoSerializingTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/ScriptMethodInfoSerializingTests.java
@@ -57,7 +57,7 @@ protected Writeable.Reader<ScriptMethodInfo> instanceReader() {
     }
 
     @Override
-    protected ScriptMethodInfo mutateInstance(ScriptMethodInfo instance) throws IOException {
+    protected ScriptMethodInfo mutateInstance(ScriptMethodInfo instance) {
         return mutate(instance);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/ScriptParameterInfoSerializingTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/ScriptParameterInfoSerializingTests.java
index 2fc200f49390..cefd1488edde 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/ScriptParameterInfoSerializingTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/ScriptParameterInfoSerializingTests.java
@@ -42,7 +42,7 @@ protected Writeable.Reader<ParameterInfo> instanceReader() {
     }
 
     @Override
-    protected ParameterInfo mutateInstance(ParameterInfo instance) throws IOException {
+    protected ParameterInfo mutateInstance(ParameterInfo instance) {
         return mutate(instance);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksResponseTests.java
new file mode 100644
index 000000000000..8bdd43ffa1ea
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksResponseTests.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.cluster.tasks;
+
+import org.elasticsearch.cluster.service.PendingClusterTask;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.ArrayList;
+
+public class PendingClusterTasksResponseTests extends ESTestCase {
+    public void testPendingClusterTasksResponseChunking() {
+        final var tasks = new ArrayList<PendingClusterTask>();
+        for (int i = between(0, 10); i > 0; i--) {
+            tasks.add(
+                new PendingClusterTask(
+                    randomNonNegativeLong(),
+                    randomFrom(Priority.values()),
+                    new Text(randomAlphaOfLengthBetween(1, 10)),
+                    randomNonNegativeLong(),
+                    randomBoolean()
+                )
+            );
+        }
+        AbstractChunkedSerializingTestCase.assertChunkCount(
+            new PendingClusterTasksResponse(tasks),
+            response -> response.pendingTasks().size() + 2
+        );
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java
index f5a1340c17cb..b940d33cf824 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java
@@ -32,7 +32,7 @@
 import org.elasticsearch.index.analysis.TokenizerFactory;
 import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
-import org.elasticsearch.indices.analysis.AnalysisModuleTests.AppendCharFilter;
+import org.elasticsearch.indices.analysis.lucene.AppendCharFilter;
 import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.scanners.StablePluginsRegistry;
 import org.elasticsearch.test.ESTestCase;
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java
index cd56af0b6ea6..625ed23850af 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java
@@ -93,7 +93,7 @@ protected AnalyzeAction.Response createTestInstance() {
      * Either add a token to the token list or change the details token list name
      */
     @Override
-    protected AnalyzeAction.Response mutateInstance(AnalyzeAction.Response instance) throws IOException {
+    protected AnalyzeAction.Response mutateInstance(AnalyzeAction.Response instance) {
         if (instance.getTokens() != null) {
             return new AnalyzeAction.Response(
                 CollectionUtils.appendToCopy(instance.getTokens(), RandomObjects.randomToken(random())),
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java
index d72d998eb30b..e958b32fe256 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java
@@ -8,14 +8,14 @@
 
 package org.elasticsearch.action.admin.indices.close;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.TransportVersionUtils;
 
 public class CloseIndexRequestTests extends ESTestCase {
 
@@ -41,11 +41,11 @@ public void testBwcSerialization() throws Exception {
         {
             final CloseIndexRequest request = randomRequest();
             try (BytesStreamOutput out = new BytesStreamOutput()) {
-                out.setVersion(VersionUtils.randomCompatibleVersion(random(), Version.CURRENT));
+                out.setTransportVersion(TransportVersionUtils.randomCompatibleVersion(random(), TransportVersion.CURRENT));
                 request.writeTo(out);
 
                 try (StreamInput in = out.bytes().streamInput()) {
-                    in.setVersion(out.getVersion());
+                    in.setTransportVersion(out.getTransportVersion());
                     assertEquals(request.getParentTask(), TaskId.readFromStream(in));
                     assertEquals(request.masterNodeTimeout(), in.readTimeValue());
                     assertEquals(request.timeout(), in.readTimeValue());
@@ -55,10 +55,10 @@ public void testBwcSerialization() throws Exception {
                     // to the addition of hidden indices as expand to hidden indices is always true when
                     // read from a prior version
                     // TODO update version on backport!
-                    if (out.getVersion().onOrAfter(Version.V_7_7_0) || request.indicesOptions().expandWildcardsHidden()) {
+                    if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_7_0) || request.indicesOptions().expandWildcardsHidden()) {
                         assertEquals(request.indicesOptions(), indicesOptions);
                     }
-                    if (in.getVersion().onOrAfter(Version.V_7_2_0)) {
+                    if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_2_0)) {
                         assertEquals(request.waitForActiveShards(), ActiveShardCount.readFrom(in));
                     } else {
                         assertEquals(0, in.available());
@@ -68,21 +68,21 @@ public void testBwcSerialization() throws Exception {
         }
         {
             final CloseIndexRequest sample = randomRequest();
-            final Version version = VersionUtils.randomCompatibleVersion(random(), Version.CURRENT);
+            final TransportVersion version = TransportVersionUtils.randomCompatibleVersion(random(), TransportVersion.CURRENT);
             try (BytesStreamOutput out = new BytesStreamOutput()) {
-                out.setVersion(version);
+                out.setTransportVersion(version);
                 sample.getParentTask().writeTo(out);
                 out.writeTimeValue(sample.masterNodeTimeout());
                 out.writeTimeValue(sample.timeout());
                 out.writeStringArray(sample.indices());
                 sample.indicesOptions().writeIndicesOptions(out);
-                if (out.getVersion().onOrAfter(Version.V_7_2_0)) {
+                if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_2_0)) {
                     sample.waitForActiveShards().writeTo(out);
                 }
 
                 final CloseIndexRequest deserializedRequest;
                 try (StreamInput in = out.bytes().streamInput()) {
-                    in.setVersion(version);
+                    in.setTransportVersion(version);
                     deserializedRequest = new CloseIndexRequest(in);
                 }
                 assertEquals(sample.getParentTask(), deserializedRequest.getParentTask());
@@ -93,10 +93,10 @@ public void testBwcSerialization() throws Exception {
                 // to the addition of hidden indices as expand to hidden indices is always true when
                 // read from a prior version
                 // TODO change version on backport
-                if (out.getVersion().onOrAfter(Version.V_7_7_0) || sample.indicesOptions().expandWildcardsHidden()) {
+                if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_7_0) || sample.indicesOptions().expandWildcardsHidden()) {
                     assertEquals(sample.indicesOptions(), deserializedRequest.indicesOptions());
                 }
-                if (out.getVersion().onOrAfter(Version.V_7_2_0)) {
+                if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_2_0)) {
                     assertEquals(sample.waitForActiveShards(), deserializedRequest.waitForActiveShards());
                 } else {
                     assertEquals(ActiveShardCount.NONE, deserializedRequest.waitForActiveShards());
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponseTests.java
index b96e54207d53..47ad152b0cc5 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponseTests.java
@@ -42,6 +42,11 @@ protected CloseIndexResponse createTestInstance() {
         return randomResponse();
     }
 
+    @Override
+    protected CloseIndexResponse mutateInstance(CloseIndexResponse instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<CloseIndexResponse> instanceReader() {
         return CloseIndexResponse::new;
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseActionTests.java
index 8dd0e89e1cbb..cf096e35bdbc 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseActionTests.java
@@ -234,7 +234,7 @@ public void testUnavailableShardsMarkedAsStale() throws Exception {
         final long primaryTerm = indexMetadata.primaryTerm(0);
 
         final Set<String> inSyncAllocationIds = indexMetadata.inSyncAllocationIds(0);
-        final Set<String> trackedShards = shardRoutingTable.getAllAllocationIds();
+        final Set<String> trackedShards = shardRoutingTable.getPromotableAllocationIds();
 
         List<ShardRouting> unavailableShards = randomSubsetOf(randomIntBetween(1, nbReplicas), shardRoutingTable.replicaShards());
         IndexShardRoutingTable.Builder shardRoutingTableBuilder = new IndexShardRoutingTable.Builder(shardRoutingTable);
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java
index 7803b47527c2..45ccfd08015c 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java
@@ -10,6 +10,7 @@
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.client.NoOpClient;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -51,11 +52,11 @@ public void testSetSource() throws IOException {
 
         ElasticsearchParseException e = expectThrows(
             ElasticsearchParseException.class,
-            () -> builder.setSource(formatted("{ \"%s\": \"%s\" }", KEY, VALUE), XContentType.JSON)
+            () -> { builder.setSource(Strings.format("{ \"%s\": \"%s\" }", KEY, VALUE), XContentType.JSON); }
         );
-        assertEquals(formatted("unknown key [%s] for create index", KEY), e.getMessage());
+        assertEquals(Strings.format("unknown key [%s] for create index", KEY), e.getMessage());
 
-        builder.setSource(formatted("{ \"settings\": { \"%s\": \"%s\" }}", KEY, VALUE), XContentType.JSON);
+        builder.setSource(Strings.format("{ \"settings\": { \"%s\": \"%s\" }}", KEY, VALUE), XContentType.JSON);
         assertEquals(VALUE, builder.request().settings().get(KEY));
 
         XContentBuilder xContent = XContentFactory.jsonBuilder()
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java
index 6c7654378bb2..3886a90bece8 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java
@@ -181,4 +181,9 @@ protected Writeable.Reader<CreateIndexRequest> instanceReader() {
     protected CreateIndexRequest createTestInstance() {
         return RandomCreateIndexGenerator.randomCreateIndexRequest();
     }
+
+    @Override
+    protected CreateIndexRequest mutateInstance(CreateIndexRequest instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java
index 54a8585af7f5..47822a3a332a 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerTests.java
@@ -13,8 +13,8 @@
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
 import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
-import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat;
 import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
 import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
 import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
@@ -24,7 +24,7 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.IntPoint;
-import org.apache.lucene.document.KnnVectorField;
+import org.apache.lucene.document.KnnFloatVectorField;
 import org.apache.lucene.document.LatLonShape;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.NumericDocValuesField;
@@ -80,6 +80,7 @@
 
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
@@ -256,14 +257,16 @@ public void testKnnVectors() throws Exception {
 
             indexRandomly(dir, codec, numDocs, doc -> {
                 float[] vector = randomVector(dimension);
-                doc.add(new KnnVectorField("vector", vector, similarity));
+                doc.add(new KnnFloatVectorField("vector", vector, similarity));
             });
             final IndexDiskUsageStats stats = IndexDiskUsageAnalyzer.analyze(testShardId(), lastCommit(dir), () -> {});
             logger.info("--> stats {}", stats);
 
-            int dataBytes = numDocs * dimension * Float.BYTES; // size of flat vector data
-            int indexBytesEstimate = numDocs * Integer.BYTES * Lucene94HnswVectorsFormat.DEFAULT_MAX_CONN * 2; // rough size of HNSW graph
-            assertTrue(stats.total().getKnnVectorsBytes() > dataBytes + indexBytesEstimate);
+            long dataBytes = (long) numDocs * dimension * Float.BYTES; // size of flat vector data
+            long indexBytesEstimate = (long) numDocs * (Lucene95HnswVectorsFormat.DEFAULT_MAX_CONN / 4); // rough size of HNSW graph
+            assertThat("numDocs=" + numDocs + ";dimension=" + dimension, stats.total().getKnnVectorsBytes(), greaterThan(dataBytes));
+            long connectionOverhead = stats.total().getKnnVectorsBytes() - dataBytes;
+            assertThat("numDocs=" + numDocs, connectionOverhead, greaterThan(indexBytesEstimate));
         }
     }
 
@@ -323,7 +326,7 @@ public void testTriangle() throws Exception {
     public void testCompletionField() throws Exception {
         IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
             .setUseCompoundFile(false)
-            .setCodec(new Lucene94Codec(Lucene94Codec.Mode.BEST_SPEED) {
+            .setCodec(new Lucene95Codec(Lucene95Codec.Mode.BEST_SPEED) {
                 @Override
                 public PostingsFormat getPostingsFormatForField(String field) {
                     if (field.startsWith("suggest_")) {
@@ -410,25 +413,25 @@ private static void addFieldsToDoc(Document doc, IndexableField[] fields) {
     enum CodecMode {
         BEST_SPEED {
             @Override
-            Lucene94Codec.Mode mode() {
-                return Lucene94Codec.Mode.BEST_SPEED;
+            Lucene95Codec.Mode mode() {
+                return Lucene95Codec.Mode.BEST_SPEED;
             }
         },
 
         BEST_COMPRESSION {
             @Override
-            Lucene94Codec.Mode mode() {
-                return Lucene94Codec.Mode.BEST_COMPRESSION;
+            Lucene95Codec.Mode mode() {
+                return Lucene95Codec.Mode.BEST_COMPRESSION;
             }
         };
 
-        abstract Lucene94Codec.Mode mode();
+        abstract Lucene95Codec.Mode mode();
     }
 
     static void indexRandomly(Directory directory, CodecMode codecMode, int numDocs, Consumer<Document> addFields) throws IOException {
         IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
             .setUseCompoundFile(randomBoolean())
-            .setCodec(new Lucene94Codec(codecMode.mode()));
+            .setCodec(new Lucene95Codec(codecMode.mode()));
         try (IndexWriter writer = new IndexWriter(directory, config)) {
             for (int i = 0; i < numDocs; i++) {
                 final Document doc = new Document();
@@ -517,7 +520,7 @@ static void addRandomTermVectors(Document doc) {
     static void addRandomKnnVectors(Document doc) {
         int numFields = randomFrom(1, 3);
         for (int f = 0; f < numFields; f++) {
-            doc.add(new KnnVectorField("knnvector-" + f, randomVector(DEFAULT_VECTOR_DIMENSION)));
+            doc.add(new KnnFloatVectorField("knnvector-" + f, randomVector(DEFAULT_VECTOR_DIMENSION)));
         }
     }
 
@@ -636,7 +639,7 @@ static void rewriteIndexWithPerFieldCodec(Directory source, CodecMode mode, Dire
         try (DirectoryReader reader = DirectoryReader.open(source)) {
             IndexWriterConfig config = new IndexWriterConfig().setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
                 .setUseCompoundFile(randomBoolean())
-                .setCodec(new Lucene94Codec(mode.mode()) {
+                .setCodec(new Lucene95Codec(mode.mode()) {
                     @Override
                     public PostingsFormat getPostingsFormatForField(String field) {
                         return new Lucene90PostingsFormat();
@@ -649,7 +652,7 @@ public DocValuesFormat getDocValuesFormatForField(String field) {
 
                     @Override
                     public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
-                        return new Lucene94HnswVectorsFormat();
+                        return new Lucene95HnswVectorsFormat();
                     }
 
                     @Override
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexRequestTests.java
index f4974d626714..21c5d0bee47e 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexRequestTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.action.admin.indices.get;
 
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestRequestTests;
 import org.elasticsearch.test.ESTestCase;
@@ -68,6 +69,6 @@ public void testInvalidFeatures() {
 
         RestRequest request = RestRequestTests.contentRestRequest("", Map.of("features", String.join(",", invalidFeatures)));
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> GetIndexRequest.Feature.fromRequest(request));
-        assertThat(e.getMessage(), containsString(formatted("Invalid features specified [%s]", String.join(",", invalidFeatures))));
+        assertThat(e.getMessage(), containsString(Strings.format("Invalid features specified [%s]", String.join(",", invalidFeatures))));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexResponseTests.java
index 3f6f0baaabf2..884f1f8eb747 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexResponseTests.java
@@ -17,8 +17,10 @@
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.RandomCreateIndexGenerator;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
@@ -73,4 +75,13 @@ protected GetIndexResponse createTestInstance() {
         }
         return new GetIndexResponse(indices, mappings, aliases, settings, defaultSettings, dataStreams);
     }
+
+    @Override
+    protected GetIndexResponse mutateInstance(GetIndexResponse instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
+    public void testChunking() throws IOException {
+        AbstractChunkedSerializingTestCase.assertChunkCount(createTestInstance(), response -> response.getIndices().length + 2);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java
index 7bfe9810ddd2..ead622b8d113 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java
@@ -115,6 +115,11 @@ protected GetFieldMappingsResponse createTestInstance() {
         return new GetFieldMappingsResponse(randomMapping());
     }
 
+    @Override
+    protected GetFieldMappingsResponse mutateInstance(GetFieldMappingsResponse instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<GetFieldMappingsResponse> instanceReader() {
         return GetFieldMappingsResponse::new;
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponseTests.java
index 520737cc07c3..22202fd4c353 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponseTests.java
@@ -11,10 +11,10 @@
 import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.test.EqualsHashCodeTestUtils;
 
-import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
@@ -42,7 +42,7 @@ private static GetMappingsResponse mutate(GetMappingsResponse original) {
     }
 
     @Override
-    protected GetMappingsResponse mutateInstance(GetMappingsResponse instance) throws IOException {
+    protected GetMappingsResponse mutateInstance(GetMappingsResponse instance) {
         return mutate(instance);
     }
 
@@ -66,20 +66,15 @@ protected GetMappingsResponse createTestInstance() {
         return resp;
     }
 
-    public void testChunkedXContentUsesChunkPerIndex() {
-        final int indexCount = randomIntBetween(1, 10);
-        final var response = new GetMappingsResponse(
-            IntStream.range(0, indexCount)
-                .mapToObj(i -> "index-" + i)
-                .collect(Collectors.toUnmodifiableMap(Function.identity(), k -> createMappingsForIndex()))
+    public void testChunking() {
+        AbstractChunkedSerializingTestCase.assertChunkCount(
+            new GetMappingsResponse(
+                IntStream.range(0, randomIntBetween(1, 10))
+                    .mapToObj(i -> "index-" + i)
+                    .collect(Collectors.toUnmodifiableMap(Function.identity(), k -> createMappingsForIndex()))
+            ),
+            response -> response.mappings().size() + 2
         );
-        final var chunks = response.toXContentChunked();
-        int chunkCount = 0;
-        while (chunks.hasNext()) {
-            chunks.next();
-            chunkCount++;
-        }
-        assertEquals(2 + indexCount, chunkCount);
     }
 
     // Not meant to be exhaustive
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/recovery/RecoveryResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/recovery/RecoveryResponseTests.java
index 6cf36a8c12d2..a34119fa17a3 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/recovery/RecoveryResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/recovery/RecoveryResponseTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.recovery.RecoveryState;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.List;
@@ -32,37 +33,34 @@ public void testChunkedToXContent() {
         DiscoveryNode sourceNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
         DiscoveryNode targetNode = new DiscoveryNode("bar", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
         final int shards = randomInt(50);
-        final RecoveryResponse recoveryResponse = new RecoveryResponse(
-            successfulShards + failedShards,
-            successfulShards,
-            failedShards,
-            IntStream.range(0, shards)
-                .boxed()
-                .collect(
-                    Collectors.toUnmodifiableMap(
-                        i -> "index-" + i,
-                        i -> List.of(
-                            new RecoveryState(
-                                ShardRouting.newUnassigned(
-                                    new ShardId("index-" + i, "index-uuid-" + i, 0),
-                                    randomBoolean(),
-                                    RecoverySource.PeerRecoverySource.INSTANCE,
-                                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)
-                                ).initialize(sourceNode.getId(), null, randomNonNegativeLong()),
-                                sourceNode,
-                                targetNode
+        AbstractChunkedSerializingTestCase.assertChunkCount(
+            new RecoveryResponse(
+                successfulShards + failedShards,
+                successfulShards,
+                failedShards,
+                IntStream.range(0, shards)
+                    .boxed()
+                    .collect(
+                        Collectors.toUnmodifiableMap(
+                            i -> "index-" + i,
+                            i -> List.of(
+                                new RecoveryState(
+                                    ShardRouting.newUnassigned(
+                                        new ShardId("index-" + i, "index-uuid-" + i, 0),
+                                        false,
+                                        RecoverySource.PeerRecoverySource.INSTANCE,
+                                        new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null),
+                                        ShardRouting.Role.DEFAULT
+                                    ).initialize(sourceNode.getId(), null, randomNonNegativeLong()),
+                                    sourceNode,
+                                    targetNode
+                                )
                             )
                         )
-                    )
-                ),
-            List.of()
+                    ),
+                List.of()
+            ),
+            ignored -> shards + 2
         );
-        final var iterator = recoveryResponse.toXContentChunked();
-        int chunks = 0;
-        while (iterator.hasNext()) {
-            iterator.next();
-            chunks++;
-        }
-        assertEquals(shards + 2, chunks);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexRequestTests.java
index b9e8b0d77353..9362242eb2dc 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexRequestTests.java
@@ -36,4 +36,9 @@ protected Request createTestInstance() {
         );
         return new Request(names, indicesOptions);
     }
+
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexResponseTests.java
index c542b216fd59..4b53e0bdfed0 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexResponseTests.java
@@ -65,6 +65,11 @@ protected Response createTestInstance() {
         return new Response(indices, aliases, dataStreams);
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     private static ResolvedIndex createTestResolvedIndexInstance() {
         String name = randomAlphaOfLength(6);
         String[] aliases = randomStringArray(0, 5);
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java
index b70affac4c09..15a8b33457ab 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java
@@ -15,15 +15,22 @@
 import org.elasticsearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedIndex;
 import org.elasticsearch.action.admin.indices.resolve.ResolveIndexAction.TransportAction;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.AliasMetadata;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
-import org.elasticsearch.cluster.metadata.IndexAbstractionResolver;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.indices.EmptySystemIndices;
+import org.elasticsearch.indices.SystemIndexDescriptor;
+import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.indices.TestIndexNameExpressionResolver;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
@@ -38,10 +45,17 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
+import static org.elasticsearch.indices.SystemIndices.EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY;
+import static org.elasticsearch.indices.SystemIndices.SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY;
 import static org.hamcrest.Matchers.arrayContaining;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.oneOf;
 import static org.hamcrest.core.IsNull.notNullValue;
 
@@ -62,8 +76,10 @@ public class ResolveIndexTests extends ESTestCase {
         { "logs-mysql-prod", 4 },
         { "logs-mysql-test", 2 } };
 
-    private Metadata metadata;
-    private final IndexAbstractionResolver resolver = new IndexAbstractionResolver(TestIndexNameExpressionResolver.newInstance());
+    private ClusterState clusterState;
+    private ThreadContext threadContext;
+
+    private IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance();
 
     private long epochMillis;
     private String dateString;
@@ -71,8 +87,10 @@ public class ResolveIndexTests extends ESTestCase {
     @Before
     public void setup() {
         epochMillis = randomLongBetween(1580536800000L, 1583042400000L);
+        threadContext = createThreadContext();
+        resolver = new IndexNameExpressionResolver(threadContext, EmptySystemIndices.INSTANCE);
         dateString = DataStream.DATE_FORMATTER.formatMillis(epochMillis);
-        metadata = buildMetadata(dataStreams, indices);
+        clusterState = ClusterState.builder(new ClusterName("_name")).metadata(buildMetadata(dataStreams, indices)).build();
     }
 
     public void testResolveStarWithDefaultOptions() {
@@ -82,7 +100,7 @@ public void testResolveStarWithDefaultOptions() {
         List<ResolvedAlias> aliases = new ArrayList<>();
         List<ResolvedDataStream> dataStreams = new ArrayList<>();
 
-        TransportAction.resolveIndices(names, indicesOptions, metadata, resolver, indices, aliases, dataStreams, true);
+        TransportAction.resolveIndices(names, indicesOptions, clusterState, resolver, indices, aliases, dataStreams);
 
         validateIndices(
             indices,
@@ -100,13 +118,16 @@ public void testResolveStarWithDefaultOptions() {
     }
 
     public void testResolveStarWithAllOptions() {
-        String[] names = new String[] { "*" };
-        IndicesOptions indicesOptions = IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN;
+        String[] names = randomFrom(new String[] { "*" }, new String[] { "_all" });
+        IndicesOptions indicesOptions = randomFrom(
+            IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN,
+            IndicesOptions.STRICT_EXPAND_OPEN_CLOSED_HIDDEN
+        );
         List<ResolvedIndex> indices = new ArrayList<>();
         List<ResolvedAlias> aliases = new ArrayList<>();
         List<ResolvedDataStream> dataStreams = new ArrayList<>();
 
-        TransportAction.resolveIndices(names, indicesOptions, metadata, resolver, indices, aliases, dataStreams, true);
+        TransportAction.resolveIndices(names, indicesOptions, clusterState, resolver, indices, aliases, dataStreams);
         validateIndices(
             indices,
             ".ds-logs-mysql-prod-" + dateString + "-000001",
@@ -136,7 +157,7 @@ public void testResolveWithPattern() {
         List<ResolvedAlias> aliases = new ArrayList<>();
         List<ResolvedDataStream> dataStreams = new ArrayList<>();
 
-        TransportAction.resolveIndices(names, indicesOptions, metadata, resolver, indices, aliases, dataStreams, true);
+        TransportAction.resolveIndices(names, indicesOptions, clusterState, resolver, indices, aliases, dataStreams);
 
         validateIndices(
             indices,
@@ -163,7 +184,7 @@ public void testResolveWithMultipleNames() {
         List<ResolvedAlias> aliases = new ArrayList<>();
         List<ResolvedDataStream> dataStreams = new ArrayList<>();
 
-        TransportAction.resolveIndices(names, indicesOptions, metadata, resolver, indices, aliases, dataStreams, true);
+        TransportAction.resolveIndices(names, indicesOptions, clusterState, resolver, indices, aliases, dataStreams);
         validateIndices(indices, ".ds-logs-mysql-prod-" + dateString + "-000003", "logs-pgsql-test-20200102");
         validateAliases(aliases, "one-off-alias");
         validateDataStreams(dataStreams, "logs-mysql-test");
@@ -180,21 +201,13 @@ public void testResolvePreservesBackingIndexOrdering() {
 
         DataStream ds = DataStreamTestHelper.newInstance(dataStreamName, backingIndices.stream().map(IndexMetadata::getIndex).toList());
         builder.put(ds);
+        ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metadata(builder.build()).build();
 
         IndicesOptions indicesOptions = IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN;
         List<ResolvedIndex> indices = new ArrayList<>();
         List<ResolvedAlias> aliases = new ArrayList<>();
         List<ResolvedDataStream> dataStreams = new ArrayList<>();
-        TransportAction.resolveIndices(
-            new String[] { "*" },
-            indicesOptions,
-            builder.build(),
-            resolver,
-            indices,
-            aliases,
-            dataStreams,
-            true
-        );
+        TransportAction.resolveIndices(new String[] { "*" }, indicesOptions, clusterState, resolver, indices, aliases, dataStreams);
 
         assertThat(dataStreams.size(), equalTo(1));
         assertThat(dataStreams.get(0).getBackingIndices(), arrayContaining(names));
@@ -210,18 +223,98 @@ public void testResolveHiddenProperlyWithDateMath() {
             // name, isClosed, isHidden, isFrozen, dataStream, aliases
             { "logs-pgsql-prod-" + todaySuffix, false, true, false, false, null, Strings.EMPTY_ARRAY },
             { "logs-pgsql-prod-" + tomorrowSuffix, false, true, false, false, null, Strings.EMPTY_ARRAY } };
-        Metadata metadata = buildMetadata(new Object[][] {}, indices);
-
-        String requestedIndex = "<logs-pgsql-prod-{now/d}>";
-        List<String> resolvedIndices = resolver.resolveIndexAbstractions(
-            List.of(requestedIndex),
-            IndicesOptions.LENIENT_EXPAND_OPEN,
-            metadata,
-            List.of("logs-pgsql-prod-" + todaySuffix, "logs-pgsql-prod-" + tomorrowSuffix),
-            randomBoolean()
-        );
+        ClusterState clusterState = ClusterState.builder(new ClusterName("_name"))
+            .metadata(buildMetadata(new Object[][] {}, indices))
+            .build();
+        String[] requestedIndex = new String[] { "<logs-pgsql-prod-{now/d}>" };
+        Set<String> resolvedIndices = resolver.resolveExpressions(clusterState, IndicesOptions.LENIENT_EXPAND_OPEN, true, requestedIndex);
         assertThat(resolvedIndices.size(), is(1));
-        assertThat(resolvedIndices.get(0), oneOf("logs-pgsql-prod-" + todaySuffix, "logs-pgsql-prod-" + tomorrowSuffix));
+        assertThat(resolvedIndices, contains(oneOf("logs-pgsql-prod-" + todaySuffix, "logs-pgsql-prod-" + tomorrowSuffix)));
+    }
+
+    public void testSystemIndexAccess() {
+        Metadata.Builder mdBuilder = buildMetadata(dataStreams, indices);
+        SystemIndices systemIndices = addSystemIndex(mdBuilder);
+        clusterState = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
+        {
+            threadContext = createThreadContext();
+            if (randomBoolean()) {
+                threadContext.putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, "false");
+            }
+            threadContext.putHeader(EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, "whatever");
+            resolver = new IndexNameExpressionResolver(threadContext, systemIndices);
+            List<ResolvedIndex> indices = new ArrayList<>();
+            List<ResolvedAlias> aliases = new ArrayList<>();
+            List<ResolvedDataStream> dataStreams = new ArrayList<>();
+            TransportAction.resolveIndices(
+                new String[] { ".*" },
+                IndicesOptions.LENIENT_EXPAND_OPEN,
+                clusterState,
+                resolver,
+                indices,
+                aliases,
+                dataStreams
+            );
+            // non net-new system indices are allowed even when no system indices are allowed
+            assertThat(indices.stream().map(ResolvedIndex::getName).collect(Collectors.toList()), hasItem(is(".test-system-1")));
+        }
+        {
+            threadContext = createThreadContext();
+            threadContext.putHeader(EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, "test-net-new-system");
+            resolver = new IndexNameExpressionResolver(threadContext, systemIndices);
+            List<ResolvedIndex> indices = new ArrayList<>();
+            List<ResolvedAlias> aliases = new ArrayList<>();
+            List<ResolvedDataStream> dataStreams = new ArrayList<>();
+            TransportAction.resolveIndices(
+                new String[] { ".*" },
+                IndicesOptions.STRICT_EXPAND_OPEN,
+                clusterState,
+                resolver,
+                indices,
+                aliases,
+                dataStreams
+            );
+            assertThat(indices.stream().map(ResolvedIndex::getName).collect(Collectors.toList()), hasItem(is(".test-system-1")));
+            assertThat(indices.stream().map(ResolvedIndex::getName).collect(Collectors.toList()), hasItem(is(".test-net-new-system-1")));
+            indices = new ArrayList<>();
+            TransportAction.resolveIndices(
+                new String[] { ".test-net*" },
+                IndicesOptions.STRICT_EXPAND_OPEN,
+                clusterState,
+                resolver,
+                indices,
+                aliases,
+                dataStreams
+            );
+            assertThat(indices.stream().map(ResolvedIndex::getName).collect(Collectors.toList()), not(hasItem(is(".test-system-1"))));
+            assertThat(indices.stream().map(ResolvedIndex::getName).collect(Collectors.toList()), hasItem(is(".test-net-new-system-1")));
+        }
+    }
+
+    public void testIgnoreUnavailableFalse() {
+        String[] names = new String[] { "missing", "logs*" };
+        IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
+        List<ResolvedIndex> indices = new ArrayList<>();
+        List<ResolvedAlias> aliases = new ArrayList<>();
+        List<ResolvedDataStream> dataStreams = new ArrayList<>();
+        IndexNotFoundException infe = expectThrows(
+            IndexNotFoundException.class,
+            () -> TransportAction.resolveIndices(names, indicesOptions, clusterState, resolver, indices, aliases, dataStreams)
+        );
+        assertThat(infe.getMessage(), containsString("no such index [missing]"));
+    }
+
+    public void testAllowNoIndicesFalse() {
+        String[] names = new String[] { "missing", "missing*" };
+        IndicesOptions indicesOptions = IndicesOptions.fromOptions(true, false, true, true);
+        List<ResolvedIndex> indices = new ArrayList<>();
+        List<ResolvedAlias> aliases = new ArrayList<>();
+        List<ResolvedDataStream> dataStreams = new ArrayList<>();
+        IndexNotFoundException infe = expectThrows(
+            IndexNotFoundException.class,
+            () -> TransportAction.resolveIndices(names, indicesOptions, clusterState, resolver, indices, aliases, dataStreams)
+        );
+        assertThat(infe.getMessage(), containsString("no such index [missing*]"));
     }
 
     private void validateIndices(List<ResolvedIndex> resolvedIndices, String... expectedIndices) {
@@ -283,7 +376,7 @@ private void validateDataStreams(List<ResolvedDataStream> resolvedDataStreams, S
         }
     }
 
-    Metadata buildMetadata(Object[][] dataStreams, Object[][] indices) {
+    Metadata.Builder buildMetadata(Object[][] dataStreams, Object[][] indices) {
         Metadata.Builder builder = Metadata.builder();
 
         List<IndexMetadata> allIndices = new ArrayList<>();
@@ -316,7 +409,7 @@ Metadata buildMetadata(Object[][] dataStreams, Object[][] indices) {
             builder.put(index, false);
         }
 
-        return builder.build();
+        return builder;
     }
 
     private static IndexMetadata createIndexMetadata(
@@ -394,4 +487,54 @@ private static String[] flagsToAttributes(Object[] indexInfo) {
         attributes.sort(String::compareTo);
         return attributes.toArray(Strings.EMPTY_ARRAY);
     }
+
+    private SystemIndices addSystemIndex(Metadata.Builder mdBuilder) {
+        mdBuilder.put(indexBuilder(".test-system-1", SystemIndexDescriptor.DEFAULT_SETTINGS).state(IndexMetadata.State.OPEN).system(true))
+            .put(
+                indexBuilder(".test-net-new-system-1", SystemIndexDescriptor.DEFAULT_SETTINGS).state(IndexMetadata.State.OPEN).system(true)
+            );
+        SystemIndices systemIndices = new SystemIndices(
+            List.of(
+                new SystemIndices.Feature(
+                    "test-system-feature",
+                    "test system index",
+                    List.of(
+                        new SystemIndexDescriptor(
+                            ".test-system*",
+                            "test-system-description",
+                            SystemIndexDescriptor.Type.EXTERNAL_UNMANAGED,
+                            List.of("test-system")
+                        ),
+                        SystemIndexDescriptor.builder()
+                            .setIndexPattern(".test-net-new-system*")
+                            .setDescription("test-net-new-system-description")
+                            .setType(SystemIndexDescriptor.Type.EXTERNAL_MANAGED)
+                            .setAllowedElasticProductOrigins(List.of("test-net-new-system"))
+                            .setNetNew()
+                            .setSettings(Settings.EMPTY)
+                            .setMappings("{ \"_doc\": { \"_meta\": { \"version\": \"8.0.0\" } } }")
+                            .setPrimaryIndex(".test-net-new-system-1")
+                            .setVersionMetaKey("version")
+                            .setOrigin("system")
+                            .build()
+                    )
+                )
+            )
+        );
+        return systemIndices;
+    }
+
+    private static IndexMetadata.Builder indexBuilder(String index, Settings additionalSettings) {
+        return IndexMetadata.builder(index).settings(settings(additionalSettings));
+    }
+
+    private static Settings.Builder settings(Settings additionalSettings) {
+        return settings(Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(additionalSettings);
+    }
+
+    private ThreadContext createThreadContext() {
+        return new ThreadContext(Settings.EMPTY);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveIndexActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveIndexActionTests.java
index 9b751702dfd6..cf3fd1efb37f 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveIndexActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveIndexActionTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.action.admin.indices.resolve;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilter;
@@ -54,8 +55,10 @@ public void testCCSCompatibilityCheck() throws Exception {
                 @Override
                 public void writeTo(StreamOutput out) throws IOException {
                     super.writeTo(out);
-                    if (out.getVersion().before(Version.CURRENT)) {
-                        throw new IllegalArgumentException("This request isn't serializable to nodes before " + Version.CURRENT);
+                    if (out.getTransportVersion().before(TransportVersion.CURRENT)) {
+                        throw new IllegalArgumentException(
+                            "This request isn't serializable before transport version " + TransportVersion.CURRENT
+                        );
                     }
                 }
             };
@@ -81,7 +84,10 @@ public void writeTo(StreamOutput out) throws IOException {
 
             assertThat(ex.getMessage(), containsString("not compatible with version"));
             assertThat(ex.getMessage(), containsString("and the 'search.check_ccs_compatibility' setting is enabled."));
-            assertEquals("This request isn't serializable to nodes before " + Version.CURRENT, ex.getCause().getMessage());
+            assertEquals(
+                "This request isn't serializable before transport version " + TransportVersion.CURRENT,
+                ex.getCause().getMessage()
+            );
         } finally {
             assertTrue(ESTestCase.terminate(threadPool));
         }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java
index 9500af1d086f..5628848f38bc 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java
@@ -276,7 +276,7 @@ public void testGenerateRolloverIndexName() {
         String indexEndingInNumbers = indexPrefix + "-" + num;
         assertThat(
             MetadataRolloverService.generateRolloverIndexName(indexEndingInNumbers),
-            equalTo(indexPrefix + "-" + formatted("%06d", num + 1))
+            equalTo(indexPrefix + "-" + Strings.format("%06d", num + 1))
         );
         assertThat(MetadataRolloverService.generateRolloverIndexName("index-name-1"), equalTo("index-name-000002"));
         assertThat(MetadataRolloverService.generateRolloverIndexName("index-name-2"), equalTo("index-name-000003"));
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
index 1a5b05812793..0c78701b6946 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
@@ -478,7 +478,8 @@ public static IndicesStatsResponse randomIndicesStatsResponse(final IndexMetadat
                     shardId,
                     primary,
                     primary ? RecoverySource.EmptyStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE,
-                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)
+                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null),
+                    ShardRouting.Role.DEFAULT
                 );
                 shardRouting = shardRouting.initialize("node-0", null, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
                 shardRouting = shardRouting.moveToStarted(ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponseTests.java
index efc68d2bbf97..5fe7a528d68c 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponseTests.java
@@ -13,15 +13,17 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.index.engine.Segment;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
+import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS;
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 
 public class IndicesSegmentResponseTests extends ESTestCase {
@@ -42,15 +44,12 @@ public void testToXContentSerialiationWithSortedFields() throws Exception {
             0,
             Collections.emptyList()
         );
-        var serialization = response.toXContentChunked();
         try (XContentBuilder builder = jsonBuilder()) {
-            while (serialization.hasNext()) {
-                serialization.next().toXContent(builder, ToXContent.EMPTY_PARAMS);
-            }
+            ChunkedToXContent.wrapAsToXContent(response).toXContent(builder, EMPTY_PARAMS);
         }
     }
 
-    public void testSerializesOneChunkPerIndex() {
+    public void testChunking() {
         final int indices = randomIntBetween(1, 10);
         final List<ShardRouting> routings = new ArrayList<>(indices);
         for (int i = 0; i < indices; i++) {
@@ -60,19 +59,15 @@ public void testSerializesOneChunkPerIndex() {
         SortField sortField = new SortField("foo", SortField.Type.STRING);
         sortField.setMissingValue(SortField.STRING_LAST);
         segment.segmentSort = new Sort(sortField);
-        IndicesSegmentResponse response = new IndicesSegmentResponse(
-            routings.stream().map(routing -> new ShardSegments(routing, List.of(segment))).toArray(ShardSegments[]::new),
-            indices,
-            indices,
-            0,
-            Collections.emptyList()
+        AbstractChunkedSerializingTestCase.assertChunkCount(
+            new IndicesSegmentResponse(
+                routings.stream().map(routing -> new ShardSegments(routing, List.of(segment))).toArray(ShardSegments[]::new),
+                indices,
+                indices,
+                0,
+                Collections.emptyList()
+            ),
+            response -> response.getIndices().size() + 4
         );
-        int chunks = 0;
-        final var iterator = response.toXContentChunked();
-        while (iterator.hasNext()) {
-            iterator.next();
-            chunks++;
-        }
-        assertEquals(indices + 2, chunks);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsResponseTests.java
index 8e007bfcac39..6c9297bb41ae 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsResponseTests.java
@@ -58,6 +58,11 @@ protected GetSettingsResponse createTestInstance() {
         return new GetSettingsResponse(indexToSettings, indexToDefaultSettings);
     }
 
+    @Override
+    protected GetSettingsResponse mutateInstance(GetSettingsResponse instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<GetSettingsResponse> instanceReader() {
         return GetSettingsResponse::new;
@@ -74,15 +79,8 @@ protected Predicate<String> getRandomFieldsExcludeFilter() {
         return f -> f.equals("") || f.contains(".settings") || f.contains(".defaults");
     }
 
-    public void testOneChunkPerIndex() {
-        final var instance = createTestInstance();
-        final var iterator = instance.toXContentChunked();
-        int chunks = 0;
-        while (iterator.hasNext()) {
-            chunks++;
-            iterator.next();
-        }
-        assertEquals(2 + instance.getIndexToSettings().size(), chunks);
+    public void testChunking() {
+        AbstractChunkedSerializingTestCase.assertChunkCount(createTestInstance(), response -> 2 + response.getIndexToSettings().size());
     }
 
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestTests.java
index 8ec0ed74af5c..afc3fc71e57b 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestTests.java
@@ -171,4 +171,9 @@ protected ResizeRequest createTestInstance() {
         }
         return resizeRequest;
     }
+
+    @Override
+    protected ResizeRequest mutateInstance(ResizeRequest instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java
index 132a9c08f208..1188ec2a828e 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
@@ -58,7 +59,7 @@ private ClusterState createClusterState(String name, int numShards, int numRepli
             .build();
         metaBuilder.put(indexMetadata, false);
         Metadata metadata = metaBuilder.build();
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         routingTableBuilder.addAsNew(metadata.index(name));
 
         RoutingTable routingTable = routingTableBuilder.build();
@@ -135,7 +136,8 @@ public void testErrorCondition() {
             new TestGatewayAllocator(),
             new BalancedShardsAllocator(Settings.EMPTY),
             EmptyClusterInfoService.INSTANCE,
-            EmptySnapshotsInfoService.INSTANCE
+            EmptySnapshotsInfoService.INSTANCE,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
 
         RoutingTable routingTable = service.reroute(clusterState, "reroute", ActionListener.noop()).routingTable();
@@ -164,7 +166,8 @@ public void testPassNumRoutingShards() {
             new TestGatewayAllocator(),
             new BalancedShardsAllocator(Settings.EMPTY),
             EmptyClusterInfoService.INSTANCE,
-            EmptySnapshotsInfoService.INSTANCE
+            EmptySnapshotsInfoService.INSTANCE,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
 
         RoutingTable routingTable = service.reroute(clusterState, "reroute", ActionListener.noop()).routingTable();
@@ -206,7 +209,8 @@ public void testPassNumRoutingShardsAndFail() {
             new TestGatewayAllocator(),
             new BalancedShardsAllocator(Settings.EMPTY),
             EmptyClusterInfoService.INSTANCE,
-            EmptySnapshotsInfoService.INSTANCE
+            EmptySnapshotsInfoService.INSTANCE,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
 
         RoutingTable routingTable = service.reroute(clusterState, "reroute", ActionListener.noop()).routingTable();
@@ -253,7 +257,8 @@ public void testShrinkIndexSettings() {
             new TestGatewayAllocator(),
             new BalancedShardsAllocator(Settings.EMPTY),
             EmptyClusterInfoService.INSTANCE,
-            EmptySnapshotsInfoService.INSTANCE
+            EmptySnapshotsInfoService.INSTANCE,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
 
         RoutingTable routingTable = service.reroute(clusterState, "reroute", ActionListener.noop()).routingTable();
@@ -317,7 +322,8 @@ public void testShrinkWithMaxPrimaryShardSize() {
             new TestGatewayAllocator(),
             new BalancedShardsAllocator(Settings.EMPTY),
             EmptyClusterInfoService.INSTANCE,
-            EmptySnapshotsInfoService.INSTANCE
+            EmptySnapshotsInfoService.INSTANCE,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
 
         RoutingTable routingTable = service.reroute(clusterState, "reroute", ActionListener.noop()).routingTable();
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponseTests.java
new file mode 100644
index 000000000000..6fb53760de5c
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponseTests.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.admin.indices.stats;
+
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.index.search.stats.FieldUsageStats;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.List;
+import java.util.Map;
+
+public class FieldUsageStatsResponseTests extends ESTestCase {
+
+    public void testToXContentChunkPerIndex() {
+        final int indices = randomIntBetween(0, 100);
+        final Map<String, List<FieldUsageShardResponse>> perIndex = Maps.newMapWithExpectedSize(indices);
+        for (int i = 0; i < indices; i++) {
+            perIndex.put(
+                "index-" + i,
+                List.of(
+                    new FieldUsageShardResponse(
+                        "tracking_id",
+                        TestShardRouting.newShardRouting(
+                            new ShardId("index" + i, UUIDs.randomBase64UUID(random()), 0),
+                            "node_id",
+                            true,
+                            ShardRoutingState.STARTED
+                        ),
+                        0,
+                        new FieldUsageStats()
+                    )
+                )
+            );
+        }
+
+        AbstractChunkedSerializingTestCase.assertChunkCount(
+            new FieldUsageStatsResponse(indices, indices, 0, List.of(), perIndex),
+            ignored -> indices + 2
+        );
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java
index e1873b7714bc..453343ae2805 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java
@@ -16,10 +16,12 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardPath;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.json.JsonXContent;
 
+import java.io.IOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -36,12 +38,20 @@
 public class IndicesStatsResponseTests extends ESTestCase {
 
     public void testInvalidLevel() {
-        final IndicesStatsResponse response = new IndicesStatsResponse(new ShardStats[0], 0, 0, 0, null, ClusterState.EMPTY_STATE);
+        final IndicesStatsResponse response = new IndicesStatsResponse(
+            new ShardStats[0],
+            0,
+            0,
+            0,
+            null,
+            ClusterState.EMPTY_STATE.getMetadata(),
+            ClusterState.EMPTY_STATE.routingTable()
+        );
         final String level = randomAlphaOfLength(16);
         final ToXContent.Params params = new ToXContent.MapParams(Collections.singletonMap("level", level));
         final IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
-            () -> response.toXContent(JsonXContent.contentBuilder(), params)
+            () -> response.toXContentChunked(params).next().toXContent(JsonXContent.contentBuilder(), params)
         );
         assertThat(
             e,
@@ -64,7 +74,7 @@ public void testGetIndices() {
                 ShardId shId = new ShardId(index, shardId);
                 Path path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve(String.valueOf(shardId));
                 ShardPath shardPath = new ShardPath(false, path, path, shId);
-                ShardRouting routing = createShardRouting(index, shId, (shardId == 0));
+                ShardRouting routing = createShardRouting(shId, (shardId == 0));
                 shards.add(new ShardStats(routing, shardPath, null, null, null, null));
                 AtomicLong primaryShardsCounter = expectedIndexToPrimaryShardsCount.computeIfAbsent(
                     index.getName(),
@@ -83,7 +93,8 @@ public void testGetIndices() {
             0,
             0,
             null,
-            ClusterState.EMPTY_STATE
+            ClusterState.EMPTY_STATE.getMetadata(),
+            ClusterState.EMPTY_STATE.routingTable()
         );
         Map<String, IndexStats> indexStats = indicesStatsResponse.getIndices();
 
@@ -105,7 +116,38 @@ public void testGetIndices() {
         }
     }
 
-    private ShardRouting createShardRouting(Index index, ShardId shardId, boolean isPrimary) {
+    public void testChunkedEncodingPerIndex() throws IOException {
+        final int shards = randomIntBetween(1, 10);
+        final List<ShardStats> stats = new ArrayList<>(shards);
+        for (int i = 0; i < shards; i++) {
+            ShardId shId = new ShardId(createIndex("index-" + i), randomIntBetween(0, 1));
+            Path path = createTempDir().resolve("indices").resolve(shId.getIndex().getUUID()).resolve(String.valueOf(shId.id()));
+            ShardPath shardPath = new ShardPath(false, path, path, shId);
+            ShardRouting routing = createShardRouting(shId, (shId.id() == 0));
+            stats.add(new ShardStats(routing, shardPath, new CommonStats(), null, null, null));
+        }
+        final IndicesStatsResponse indicesStatsResponse = new IndicesStatsResponse(
+            stats.toArray(new ShardStats[0]),
+            shards,
+            shards,
+            0,
+            null,
+            ClusterState.EMPTY_STATE.getMetadata(),
+            ClusterState.EMPTY_STATE.routingTable()
+        );
+        AbstractChunkedSerializingTestCase.assertChunkCount(
+            indicesStatsResponse,
+            new ToXContent.MapParams(Map.of("level", "cluster")),
+            ignored1 -> 3
+        );
+        AbstractChunkedSerializingTestCase.assertChunkCount(
+            indicesStatsResponse,
+            new ToXContent.MapParams(Map.of("level", "indices")),
+            ignored -> 4 + shards
+        );
+    }
+
+    private ShardRouting createShardRouting(ShardId shardId, boolean isPrimary) {
         return TestShardRouting.newShardRouting(shardId, randomAlphaOfLength(4), isPrimary, ShardRoutingState.STARTED);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
index 1ba21924a36c..25d04275ae28 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
@@ -195,6 +195,14 @@ public static IndicesStatsResponse newIndicesStatsResponse(
         List<DefaultShardOperationFailedException> shardFailures,
         ClusterState clusterState
     ) {
-        return new IndicesStatsResponse(shards, totalShards, successfulShards, failedShards, shardFailures, clusterState);
+        return new IndicesStatsResponse(
+            shards,
+            totalShards,
+            successfulShards,
+            failedShards,
+            shardFailures,
+            clusterState.getMetadata(),
+            clusterState.routingTable()
+        );
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/delete/DeleteComposableIndexTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/delete/DeleteComposableIndexTemplateRequestTests.java
index 42f9ce2952d9..b06e7824ce5d 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/delete/DeleteComposableIndexTemplateRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/delete/DeleteComposableIndexTemplateRequestTests.java
@@ -11,8 +11,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
-
 public class DeleteComposableIndexTemplateRequestTests extends AbstractWireSerializingTestCase<
     DeleteComposableIndexTemplateAction.Request> {
     @Override
@@ -26,8 +24,7 @@ protected DeleteComposableIndexTemplateAction.Request createTestInstance() {
     }
 
     @Override
-    protected DeleteComposableIndexTemplateAction.Request mutateInstance(DeleteComposableIndexTemplateAction.Request instance)
-        throws IOException {
+    protected DeleteComposableIndexTemplateAction.Request mutateInstance(DeleteComposableIndexTemplateAction.Request instance) {
         return randomValueOtherThan(instance, this::createTestInstance);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java
index 392f5243e959..3b24f90b9d85 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -37,7 +36,7 @@ protected GetComponentTemplateAction.Response createTestInstance() {
     }
 
     @Override
-    protected GetComponentTemplateAction.Response mutateInstance(GetComponentTemplateAction.Response instance) throws IOException {
+    protected GetComponentTemplateAction.Response mutateInstance(GetComponentTemplateAction.Response instance) {
         return randomValueOtherThan(instance, this::createTestInstance);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateRequestTests.java
index 8e039519741c..31ae4c358fda 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateRequestTests.java
@@ -11,8 +11,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
-
 public class GetComposableIndexTemplateRequestTests extends AbstractWireSerializingTestCase<GetComposableIndexTemplateAction.Request> {
     @Override
     protected Writeable.Reader<GetComposableIndexTemplateAction.Request> instanceReader() {
@@ -25,8 +23,7 @@ protected GetComposableIndexTemplateAction.Request createTestInstance() {
     }
 
     @Override
-    protected GetComposableIndexTemplateAction.Request mutateInstance(GetComposableIndexTemplateAction.Request instance)
-        throws IOException {
+    protected GetComposableIndexTemplateAction.Request mutateInstance(GetComposableIndexTemplateAction.Request instance) {
         return randomValueOtherThan(instance, this::createTestInstance);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateResponseTests.java
index 57da2a88e0d1..fd13d1cdc98b 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateResponseTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -37,8 +36,7 @@ protected GetComposableIndexTemplateAction.Response createTestInstance() {
     }
 
     @Override
-    protected GetComposableIndexTemplateAction.Response mutateInstance(GetComposableIndexTemplateAction.Response instance)
-        throws IOException {
+    protected GetComposableIndexTemplateAction.Response mutateInstance(GetComposableIndexTemplateAction.Response instance) {
         return randomValueOtherThan(instance, this::createTestInstance);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponseTests.java
index 549430b0dd39..03ea2062e3e3 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponseTests.java
@@ -57,6 +57,11 @@ protected GetIndexTemplatesResponse createTestInstance() {
         return new GetIndexTemplatesResponse(templates);
     }
 
+    @Override
+    protected GetIndexTemplatesResponse mutateInstance(GetIndexTemplatesResponse instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<GetIndexTemplatesResponse> instanceReader() {
         return GetIndexTemplatesResponse::new;
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java
index 2a6d7ce70ee5..c5d03e456638 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.List;
 
 import static org.hamcrest.Matchers.is;
@@ -41,7 +40,7 @@ protected SimulateIndexTemplateRequest createTestInstance() {
     }
 
     @Override
-    protected SimulateIndexTemplateRequest mutateInstance(SimulateIndexTemplateRequest instance) throws IOException {
+    protected SimulateIndexTemplateRequest mutateInstance(SimulateIndexTemplateRequest instance) {
         return randomValueOtherThan(instance, this::createTestInstance);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java
index 5566df80753b..b7c8e691f095 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.List;
 
 import static org.hamcrest.Matchers.is;
@@ -41,7 +40,7 @@ protected SimulateTemplateAction.Request createTestInstance() {
     }
 
     @Override
-    protected SimulateTemplateAction.Request mutateInstance(SimulateTemplateAction.Request instance) throws IOException {
+    protected SimulateTemplateAction.Request mutateInstance(SimulateTemplateAction.Request instance) {
         return randomValueOtherThan(instance, this::createTestInstance);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutComposableIndexTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutComposableIndexTemplateRequestTests.java
index 94bb72145c29..31855a5c7bf6 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutComposableIndexTemplateRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutComposableIndexTemplateRequestTests.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -41,8 +40,7 @@ protected PutComposableIndexTemplateAction.Request createTestInstance() {
     }
 
     @Override
-    protected PutComposableIndexTemplateAction.Request mutateInstance(PutComposableIndexTemplateAction.Request instance)
-        throws IOException {
+    protected PutComposableIndexTemplateAction.Request mutateInstance(PutComposableIndexTemplateAction.Request instance) {
         return randomValueOtherThan(instance, this::createTestInstance);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java
index 8bd4a80e0bf6..89e4bd59f0c3 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettingProviders;
 import org.elasticsearch.index.mapper.MapperService;
@@ -49,7 +50,6 @@
 
 import java.io.IOException;
 import java.util.Collections;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 
@@ -804,7 +804,7 @@ public void testBlockUsingReservedComponentTemplates() throws Exception {
 
         try (
             XContentParser parser = XContentType.JSON.xContent()
-                .createParser(XContentParserConfiguration.EMPTY, String.format(Locale.ROOT, composableTemplate, "template_1"))
+                .createParser(XContentParserConfiguration.EMPTY, Strings.format(composableTemplate, "template_1"))
         ) {
             var request = action.fromXContent(parser).composableTemplates().get(0);
             assertTrue(
@@ -817,7 +817,7 @@ public void testBlockUsingReservedComponentTemplates() throws Exception {
 
         try (
             XContentParser parser = XContentType.JSON.xContent()
-                .createParser(XContentParserConfiguration.EMPTY, String.format(Locale.ROOT, composableTemplate, "template_2"))
+                .createParser(XContentParserConfiguration.EMPTY, Strings.format(composableTemplate, "template_2"))
         ) {
             var request = action.fromXContent(parser).composableTemplates().get(0);
             // this should just work, no failure
@@ -829,7 +829,7 @@ public void testTemplatesWithReservedPrefix() throws Exception {
         final String conflictingTemplateName = "validate_template";
 
         // Reserve the validate_template name in the reserved metadata
-        String composableTemplate = String.format(Locale.ROOT, """
+        String composableTemplate = Strings.format("""
             {
               "composable_index_templates": {
                 "%s": {
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/validate/query/QueryExplanationTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/validate/query/QueryExplanationTests.java
index 3ecf9e82d320..7806a02ce100 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/validate/query/QueryExplanationTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/validate/query/QueryExplanationTests.java
@@ -42,6 +42,11 @@ protected QueryExplanation createTestInstance() {
         return createRandomQueryExplanation();
     }
 
+    @Override
+    protected QueryExplanation mutateInstance(QueryExplanation instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<QueryExplanation> instanceReader() {
         return QueryExplanation::new;
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessor2Tests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessor2Tests.java
new file mode 100644
index 000000000000..0aa9ad5a721a
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessor2Tests.java
@@ -0,0 +1,458 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.bulk;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.RemoteTransportException;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiConsumer;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+public class BulkProcessor2Tests extends ESTestCase {
+
+    private ThreadPool threadPool;
+    private final Logger logger = LogManager.getLogger(BulkProcessor2Tests.class);
+
+    @Before
+    public void startThreadPool() {
+        threadPool = new TestThreadPool("BulkProcessor2Tests");
+    }
+
+    @After
+    public void stopThreadPool() throws InterruptedException {
+        terminate(threadPool);
+    }
+
+    public void testRetry() throws Exception {
+        final int maxAttempts = between(1, 3);
+        final AtomicInteger attemptRef = new AtomicInteger();
+
+        final BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer = (request, listener) -> {
+            final int attempt = attemptRef.incrementAndGet();
+            assertThat(attempt, lessThanOrEqualTo(maxAttempts));
+            if (attempt != 1) {
+                assertThat(Thread.currentThread().getName(), containsString("[BulkProcessor2Tests][generic]"));
+            }
+
+            if (attempt == maxAttempts) {
+                listener.onFailure(new ElasticsearchException("final failure"));
+            } else {
+                listener.onFailure(new RemoteTransportException("remote", new EsRejectedExecutionException("retryable failure")));
+            }
+        };
+
+        final CountDownLatch countDownLatch = new CountDownLatch(1);
+        final BulkProcessor2.Listener listener = new BulkProcessor2.Listener() {
+
+            @Override
+            public void beforeBulk(long executionId, BulkRequest request) {}
+
+            @Override
+            public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
+                fail("afterBulk should not return success");
+            }
+
+            @Override
+            public void afterBulk(long executionId, BulkRequest request, Exception failure) {
+                assertThat(failure, instanceOf(ElasticsearchException.class));
+                assertThat(failure.getMessage(), equalTo("final failure"));
+                countDownLatch.countDown();
+            }
+        };
+
+        BulkProcessor2 bulkProcessor = BulkProcessor2.builder(consumer, listener, threadPool)
+            .setMaxNumberOfRetries(maxAttempts)
+            .setFlushInterval(TimeValue.timeValueMillis(1))
+            .build();
+        try {
+            bulkProcessor.add(new IndexRequest());
+            assertTrue(countDownLatch.await(5, TimeUnit.MINUTES));
+            assertThat(bulkProcessor.getTotalBytesInFlight(), equalTo(0L));
+        } finally {
+            bulkProcessor.awaitClose(1, TimeUnit.SECONDS);
+        }
+
+        assertThat(attemptRef.get(), equalTo(maxAttempts));
+    }
+
+    // @TestLogging(
+    // value = "org.elasticsearch.action.bulk:trace",
+    // reason = "Logging information about locks useful for tracking down deadlock"
+    // )
+    public void testConcurrentExecutions() throws Exception {
+        final AtomicReference<Throwable> exceptionRef = new AtomicReference<>();
+        int estimatedTimeForTest = Integer.MAX_VALUE;
+        final int simulateWorkTimeInMillis = 5;
+        int concurrentClients = 0;
+        int concurrentBulkRequests = 0;
+        AtomicInteger expectedExecutions = new AtomicInteger(0);
+        int maxBatchSize = 0;
+        int maxDocuments = 0;
+        int iterations = 0;
+        boolean runTest = true;
+        // find some randoms that allow this test to take under ~ 10 seconds
+        while (estimatedTimeForTest > 10_000) {
+            if (iterations++ > 1_000) { // extremely unlikely
+                runTest = false;
+                break;
+            }
+            maxBatchSize = randomIntBetween(1, 100);
+            maxDocuments = randomIntBetween(maxBatchSize, 100_000);
+            concurrentClients = randomIntBetween(1, 20);
+            concurrentBulkRequests = randomIntBetween(0, 20);
+            expectedExecutions.set(maxDocuments / maxBatchSize);
+            estimatedTimeForTest = (expectedExecutions.get() * simulateWorkTimeInMillis) / Math.min(
+                concurrentBulkRequests + 1,
+                concurrentClients
+            );
+        }
+        assumeTrue("failed to find random values that allows test to run quickly", runTest);
+        BulkResponse bulkResponse = new BulkResponse(
+            new BulkItemResponse[] { BulkItemResponse.success(0, randomFrom(DocWriteRequest.OpType.values()), mockResponse()) },
+            0
+        );
+        AtomicInteger failureCount = new AtomicInteger(0);
+        AtomicInteger successCount = new AtomicInteger(0);
+        AtomicInteger requestCount = new AtomicInteger(0);
+        AtomicInteger docCount = new AtomicInteger(0);
+        ScheduledExecutorService consumerExecutor = Executors.newScheduledThreadPool(2 * concurrentBulkRequests);
+        // All consumers are expected to be async:
+        BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer = (request, listener) -> {
+            consumerExecutor.schedule(() -> {
+                try {
+                    Thread.sleep(simulateWorkTimeInMillis); // simulate work
+                    listener.onResponse(bulkResponse);
+                } catch (InterruptedException e) {
+                    // should never happen
+                    Thread.currentThread().interrupt();
+                    failureCount.getAndIncrement();
+                    exceptionRef.set(ExceptionsHelper.useOrSuppress(exceptionRef.get(), e));
+                }
+            }, 0, TimeUnit.SECONDS);
+        };
+        BulkProcessor2 bulkProcessor = new BulkProcessor2(
+            consumer,
+            0,
+            countingListener(requestCount, successCount, failureCount, docCount, exceptionRef),
+            maxBatchSize,
+            ByteSizeValue.ofBytes(Integer.MAX_VALUE),
+            new ByteSizeValue(50, ByteSizeUnit.MB),
+            null,
+            threadPool
+        );
+        try {
+            IndexRequest indexRequest = new IndexRequest();
+            for (final AtomicInteger i = new AtomicInteger(0); i.getAndIncrement() < maxDocuments;) {
+                bulkProcessor.add(indexRequest);
+            }
+
+            assertBusy(() -> {
+                String message = """
+
+                    Expected Bulks: %s
+                    Requested Bulks: %s
+                    Successful Bulks: %s
+                    Failed Bulks: %ds
+                    """;
+                String formattedMessage = String.format(
+                    Locale.ROOT,
+                    message,
+                    expectedExecutions.get(),
+                    requestCount.get(),
+                    successCount.get(),
+                    failureCount.get()
+                );
+                assertTrue(
+                    formattedMessage,
+                    failureCount.get() == 0 && successCount.get() == expectedExecutions.get() && requestCount.get() == successCount.get()
+                );
+            }, 30, TimeUnit.SECONDS);
+            if (failureCount.get() > 0 || successCount.get() != expectedExecutions.get() || requestCount.get() != successCount.get()) {
+                if (exceptionRef.get() != null) {
+                    logger.error("exception(s) caught during test", exceptionRef.get());
+                }
+                String message = """
+
+                    Expected Bulks: %s
+                    Requested Bulks: %s
+                    Successful Bulks: %s
+                    Expected Bulks: %s
+                    Failed Bulks: %ds
+                    Max Documents: %s
+                    Max Batch Size: %s
+                    Concurrent Clients: %s
+                    Concurrent Bulk Requests: %s
+                    """;
+                fail(
+                    String.format(
+                        Locale.ROOT,
+                        message,
+                        expectedExecutions,
+                        requestCount.get(),
+                        successCount.get(),
+                        expectedExecutions.get(),
+                        failureCount.get(),
+                        maxDocuments,
+                        maxBatchSize,
+                        concurrentClients,
+                        concurrentBulkRequests
+                    )
+                );
+            }
+        } finally {
+            bulkProcessor.awaitClose(1, TimeUnit.SECONDS);
+        }
+
+        assertThat(bulkProcessor.getTotalBytesInFlight(), equalTo(0L));
+        // count total docs after processor is closed since there may have been partial batches that are flushed on close.
+        assertEquals(docCount.get(), maxDocuments);
+        consumerExecutor.shutdown();
+    }
+
+    public void testConcurrentExecutionsWithFlush() throws Exception {
+        final AtomicReference<Throwable> exceptionRef = new AtomicReference<>();
+        final int maxDocuments = 100_000;
+        final int concurrentClients = 2;
+        final int maxBatchSize = Integer.MAX_VALUE; // don't flush based on size
+        final int concurrentBulkRequests = randomIntBetween(0, 20);
+        final int simulateWorkTimeInMillis = 5;
+        BulkResponse bulkResponse = new BulkResponse(
+            new BulkItemResponse[] { BulkItemResponse.success(0, randomFrom(DocWriteRequest.OpType.values()), mockResponse()) },
+            0
+        );
+        AtomicInteger failureCount = new AtomicInteger(0);
+        AtomicInteger successCount = new AtomicInteger(0);
+        AtomicInteger requestCount = new AtomicInteger(0);
+        AtomicInteger docCount = new AtomicInteger(0);
+        BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer = (request, listener) -> {
+            try {
+                Thread.sleep(simulateWorkTimeInMillis); // simulate work
+                listener.onResponse(bulkResponse);
+            } catch (InterruptedException e) {
+                // should never happen
+                Thread.currentThread().interrupt();
+                failureCount.getAndIncrement();
+                exceptionRef.set(ExceptionsHelper.useOrSuppress(exceptionRef.get(), e));
+            }
+        };
+        BulkProcessor2 bulkProcessor = new BulkProcessor2(
+            consumer,
+            0,
+            countingListener(requestCount, successCount, failureCount, docCount, exceptionRef),
+            maxBatchSize,
+            ByteSizeValue.ofBytes(Integer.MAX_VALUE),
+            new ByteSizeValue(50, ByteSizeUnit.MB),
+            TimeValue.timeValueMillis(simulateWorkTimeInMillis * 2),
+            threadPool
+        );
+        try {
+
+            ExecutorService executorService = Executors.newFixedThreadPool(concurrentClients);
+            IndexRequest indexRequest = new IndexRequest();
+            List<Future<?>> futures = new ArrayList<>();
+            CountDownLatch startGate = new CountDownLatch(1 + concurrentClients);
+            for (final AtomicInteger i = new AtomicInteger(0); i.getAndIncrement() < maxDocuments;) {
+                futures.add(executorService.submit(() -> {
+                    try {
+                        // don't start any work until all tasks are submitted
+                        startGate.countDown();
+                        startGate.await();
+                        // alternate between ways to add to the bulk processor
+                        bulkProcessor.add(indexRequest);
+                    } catch (Exception e) {
+                        throw ExceptionsHelper.convertToRuntime(e);
+                    }
+                }));
+            }
+            startGate.countDown();
+            startGate.await();
+
+            for (Future<?> f : futures) {
+                try {
+                    f.get();
+                } catch (Exception e) {
+                    failureCount.incrementAndGet();
+                    exceptionRef.set(ExceptionsHelper.useOrSuppress(exceptionRef.get(), e));
+                }
+            }
+            executorService.shutdown();
+            executorService.awaitTermination(10, TimeUnit.SECONDS);
+        } finally {
+            bulkProcessor.awaitClose(1, TimeUnit.SECONDS);
+        }
+        assertThat(bulkProcessor.getTotalBytesInFlight(), equalTo(0L));
+
+        if (failureCount.get() > 0 || requestCount.get() != successCount.get() || maxDocuments != docCount.get()) {
+            if (exceptionRef.get() != null) {
+                logger.error("exception(s) caught during test", exceptionRef.get());
+            }
+            String message = """
+
+                Requested Bulks: %d
+                Successful Bulks: %d
+                Failed Bulks: %d
+                Total Documents: %d
+                Max Documents: %d
+                Max Batch Size: %d
+                Concurrent Clients: %d
+                Concurrent Bulk Requests: %d
+                """;
+            fail(
+                String.format(
+                    Locale.ROOT,
+                    message,
+                    requestCount.get(),
+                    successCount.get(),
+                    failureCount.get(),
+                    docCount.get(),
+                    maxDocuments,
+                    maxBatchSize,
+                    concurrentClients,
+                    concurrentBulkRequests
+                )
+            );
+        }
+    }
+
+    public void testRejections() throws Exception {
+        /*
+        * This test loads data into a BulkProcessor2 with a "max bytes in flight" value, and expects to see an
+        * EsERejectedExecutionException.
+        */
+        final int simulateWorkTimeInMillis = 5;
+        int maxBatchSize = randomIntBetween(2, 100);
+        int maxDocuments = randomIntBetween(maxBatchSize, 1_000_000);
+        int concurrentBulkRequests = randomIntBetween(0, 20);
+        BulkResponse bulkResponse = new BulkResponse(
+            new BulkItemResponse[] { BulkItemResponse.success(0, randomFrom(DocWriteRequest.OpType.values()), mockResponse()) },
+            0
+        );
+        ScheduledExecutorService consumerExecutor = Executors.newScheduledThreadPool(2 * concurrentBulkRequests);
+        // All consumers are expected to be async:
+        BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer = (request, listener) -> {
+            consumerExecutor.schedule(() -> {
+                try {
+                    Thread.sleep(simulateWorkTimeInMillis); // simulate work
+                    listener.onResponse(bulkResponse);
+                } catch (InterruptedException e) {
+                    // should never happen
+                    fail("Test interrupted");
+                }
+            }, 0, TimeUnit.SECONDS);
+        };
+        BulkProcessor2 bulkProcessor = new BulkProcessor2(
+            consumer,
+            0,
+            emptyListener(),
+            maxBatchSize,
+            ByteSizeValue.ofBytes(Integer.MAX_VALUE),
+            ByteSizeValue.ofBytes(500),
+            null,
+            threadPool
+        );
+        try {
+            IndexRequest indexRequest = new IndexRequest();
+            boolean rejectedRequests = false;
+            for (int i = 0; i < maxDocuments; i++) {
+                try {
+                    bulkProcessor.add(indexRequest);
+                } catch (EsRejectedExecutionException e) {
+                    rejectedRequests = true;
+                    break;
+                }
+            }
+            assertThat(rejectedRequests, equalTo(true));
+        } finally {
+            bulkProcessor.awaitClose(1, TimeUnit.SECONDS);
+        }
+        assertThat(bulkProcessor.getTotalBytesInFlight(), equalTo(0L));
+        consumerExecutor.shutdown();
+    }
+
+    private BulkProcessor2.Listener emptyListener() {
+        return new BulkProcessor2.Listener() {
+            @Override
+            public void beforeBulk(long executionId, BulkRequest request) {}
+
+            @Override
+            public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {}
+
+            @Override
+            public void afterBulk(long executionId, BulkRequest request, Exception failure) {}
+        };
+    }
+
+    private BulkProcessor2.Listener countingListener(
+        AtomicInteger requestCount,
+        AtomicInteger successCount,
+        AtomicInteger failureCount,
+        AtomicInteger docCount,
+        AtomicReference<Throwable> exceptionRef
+    ) {
+
+        return new BulkProcessor2.Listener() {
+            @Override
+            public void beforeBulk(long executionId, BulkRequest request) {
+                requestCount.incrementAndGet();
+            }
+
+            @Override
+            public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
+                successCount.incrementAndGet();
+                docCount.addAndGet(request.requests().size());
+            }
+
+            @Override
+            public void afterBulk(long executionId, BulkRequest request, Exception failure) {
+                if (failure != null) {
+                    failureCount.incrementAndGet();
+                    exceptionRef.set(ExceptionsHelper.useOrSuppress(exceptionRef.get(), failure));
+
+                }
+            }
+        };
+    }
+
+    private DocWriteResponse mockResponse() {
+        return new IndexResponse(new ShardId("index", "uid", 0), "id", 1, 1, 1, true);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessorTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessorTests.java
index 7605a305f028..aa4ad9824fd0 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessorTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessorTests.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
@@ -289,7 +290,7 @@ public void testConcurrentExecutions() throws Exception {
                     Concurrent Bulk Requests: %s
                     """;
                 fail(
-                    formatted(
+                    Strings.format(
                         message,
                         expectedExecutions,
                         requestCount.get(),
@@ -420,7 +421,7 @@ public void testConcurrentExecutionsWithFlush() throws Exception {
                 Concurrent Bulk Requests: %d
                 """;
             fail(
-                formatted(
+                Strings.format(
                     message,
                     requestCount.get(),
                     successCount.get(),
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/Retry2Tests.java b/server/src/test/java/org/elasticsearch/action/bulk/Retry2Tests.java
new file mode 100644
index 000000000000..e77339b296b2
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/bulk/Retry2Tests.java
@@ -0,0 +1,321 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.action.bulk;
+
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest.OpType;
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.client.NoOpClient;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class Retry2Tests extends ESTestCase {
+    private static final int CALLS_TO_FAIL = 5;
+
+    private MockBulkClient bulkClient;
+    /**
+     * Headers that are expected to be sent with all bulk requests.
+     */
+    private Map<String, String> expectedHeaders = new HashMap<>();
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        this.bulkClient = new MockBulkClient(getTestName(), CALLS_TO_FAIL);
+        // Stash some random headers so we can assert that we preserve them
+        bulkClient.threadPool().getThreadContext().stashContext();
+        expectedHeaders.clear();
+        expectedHeaders.put(randomAlphaOfLength(5), randomAlphaOfLength(5));
+        bulkClient.threadPool().getThreadContext().putHeader(expectedHeaders);
+    }
+
+    @Override
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+        this.bulkClient.close();
+    }
+
+    private BulkRequest createBulkRequest() {
+        BulkRequest request = new BulkRequest();
+        request.add(new UpdateRequest("shop", "1").doc(Map.of(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10))));
+        request.add(new UpdateRequest("shop", "2").doc(Map.of(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10))));
+        request.add(new UpdateRequest("shop", "3").doc(Map.of(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10))));
+        request.add(new UpdateRequest("shop", "4").doc(Map.of(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10))));
+        request.add(new UpdateRequest("shop", "5").doc(Map.of(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10))));
+        return request;
+    }
+
+    public void testRetryBacksOff() throws Exception {
+        BulkRequest bulkRequest = createBulkRequest();
+        Retry2 retry2 = new Retry2(CALLS_TO_FAIL);
+        PlainActionFuture<BulkResponse> future = PlainActionFuture.newFuture();
+        retry2.consumeRequestWithRetries(bulkClient::bulk, bulkRequest, future);
+        BulkResponse response = future.actionGet();
+
+        assertFalse(response.hasFailures());
+        assertThat(response.getItems().length, equalTo(bulkRequest.numberOfActions()));
+    }
+
+    public void testRetryFailsAfterRetry() throws Exception {
+        BulkRequest bulkRequest = createBulkRequest();
+        try {
+            Retry2 retry2 = new Retry2(CALLS_TO_FAIL - 1);
+            PlainActionFuture<BulkResponse> future = PlainActionFuture.newFuture();
+            retry2.consumeRequestWithRetries(bulkClient::bulk, bulkRequest, future);
+            BulkResponse response = future.actionGet();
+            /*
+            * If the last failure was an item failure we'll end up here
+            */
+            assertTrue(response.hasFailures());
+            assertThat(response.getItems().length, equalTo(bulkRequest.numberOfActions()));
+        } catch (EsRejectedExecutionException e) {
+            /*
+             * If the last failure was a rejection we'll end up here.
+             */
+            assertThat(e.getMessage(), equalTo("pretend the coordinating thread pool is stuffed"));
+        }
+
+    }
+
+    public void testRetryWithListener() throws Exception {
+        AssertingListener listener = new AssertingListener();
+        BulkRequest bulkRequest = createBulkRequest();
+        Retry2 retry = new Retry2(CALLS_TO_FAIL);
+        retry.consumeRequestWithRetries(bulkClient::bulk, bulkRequest, listener);
+
+        listener.awaitCallbacksCalled();
+        listener.assertOnResponseCalled();
+        listener.assertResponseWithoutFailures();
+        listener.assertResponseWithNumberOfItems(bulkRequest.numberOfActions());
+        listener.assertOnFailureNeverCalled();
+    }
+
+    public void testRetryWithListenerFailsAfterBacksOff() throws Exception {
+        AssertingListener listener = new AssertingListener();
+        BulkRequest bulkRequest = createBulkRequest();
+        Retry2 retry = new Retry2(CALLS_TO_FAIL - 1);
+        retry.consumeRequestWithRetries(bulkClient::bulk, bulkRequest, listener);
+
+        listener.awaitCallbacksCalled();
+
+        if (listener.lastFailure == null) {
+            /*
+             * If the last failure was an item failure we'll end up here.
+             */
+            listener.assertOnResponseCalled();
+            listener.assertResponseWithFailures();
+            listener.assertResponseWithNumberOfItems(bulkRequest.numberOfActions());
+        } else {
+            /*
+             * If the last failure was a rejection we'll end up here.
+             */
+            assertThat(listener.lastFailure, instanceOf(EsRejectedExecutionException.class));
+            assertThat(listener.lastFailure.getMessage(), equalTo("pretend the coordinating thread pool is stuffed"));
+        }
+    }
+
+    public void testAwaitClose() throws Exception {
+        /*
+         * awaitClose() is called immediately, and we make sure that subsequent requests are rejected.
+         */
+        {
+            Retry2 retry = new Retry2(CALLS_TO_FAIL);
+            retry.awaitClose(200, TimeUnit.MILLISECONDS);
+            AssertingListener listener = new AssertingListener();
+            BulkRequest bulkRequest = createBulkRequest();
+            retry.consumeRequestWithRetries(bulkClient::bulk, bulkRequest, listener);
+            listener.awaitCallbacksCalled();
+            assertNotNull(listener.lastFailure);
+            assertThat(listener.lastFailure, instanceOf(EsRejectedExecutionException.class));
+        }
+        /*
+         * awaitClose() returns without exception if all requests complete quickly, whether they were successes or failures
+         */
+        {
+            Retry2 retry = new Retry2(CALLS_TO_FAIL);
+            List<AssertingListener> listeners = new ArrayList<>();
+            BulkRequest bulkRequest = createBulkRequest();
+            for (int i = 0; i < randomIntBetween(1, 100); i++) {
+                AssertingListener listener = new AssertingListener();
+                listeners.add(listener);
+                retry.consumeRequestWithRetries((bulkRequest1, listener1) -> {
+                    if (randomBoolean()) {
+                        listener1.onResponse(new BulkResponse(new BulkItemResponse[0], 5));
+                    } else {
+                        listener1.onFailure(new RuntimeException("Some failure"));
+                    }
+                }, bulkRequest, listener);
+            }
+            retry.awaitClose(1, TimeUnit.SECONDS);
+        }
+        /*
+         * Most requests are complete but one request is hung so awaitClose ought to wait the full timeout period
+         */
+        {
+            Retry2 retry = new Retry2(CALLS_TO_FAIL);
+            List<AssertingListener> listeners = new ArrayList<>();
+            BulkRequest bulkRequest = createBulkRequest();
+            for (int i = 0; i < randomIntBetween(0, 100); i++) {
+                AssertingListener listener = new AssertingListener();
+                listeners.add(listener);
+                retry.consumeRequestWithRetries((bulkRequest1, listener1) -> {
+                    if (randomBoolean()) {
+                        listener1.onResponse(new BulkResponse(new BulkItemResponse[0], 5));
+                    } else {
+                        listener1.onFailure(new RuntimeException("Some failure"));
+                    }
+                }, bulkRequest, listener);
+            }
+            for (AssertingListener listener : listeners) {
+                listener.awaitCallbacksCalled();
+            }
+            AssertingListener listener = new AssertingListener();
+            retry.consumeRequestWithRetries((bulkRequest1, listener1) -> {
+                // never calls onResponse or onFailure
+            }, bulkRequest, listener);
+            long waitTimeMillis = randomLongBetween(20, 200);
+            // Make sure that awaitClose completes without exception, and takes at least waitTimeMillis
+            long startTimeNanos = System.nanoTime();
+            retry.awaitClose(waitTimeMillis, TimeUnit.MILLISECONDS);
+            long stopTimeNanos = System.nanoTime();
+            long runtimeMillis = TimeValue.timeValueNanos((stopTimeNanos - startTimeNanos)).millis();
+            assertThat(runtimeMillis, greaterThanOrEqualTo(waitTimeMillis));
+            assertThat(runtimeMillis, lessThanOrEqualTo(2 * waitTimeMillis));
+        }
+    }
+
+    private static class AssertingListener implements ActionListener<BulkResponse> {
+        private final CountDownLatch latch;
+        private final AtomicInteger countOnResponseCalled = new AtomicInteger();
+        private volatile Throwable lastFailure;
+        private volatile BulkResponse response;
+
+        private AssertingListener() {
+            latch = new CountDownLatch(1);
+        }
+
+        public void awaitCallbacksCalled() throws InterruptedException {
+            latch.await();
+        }
+
+        @Override
+        public void onResponse(BulkResponse bulkItemResponses) {
+            this.response = bulkItemResponses;
+            countOnResponseCalled.incrementAndGet();
+            latch.countDown();
+        }
+
+        @Override
+        public void onFailure(Exception e) {
+            this.lastFailure = e;
+            latch.countDown();
+        }
+
+        public void assertOnResponseCalled() {
+            assertThat(countOnResponseCalled.get(), equalTo(1));
+        }
+
+        public void assertResponseWithNumberOfItems(int numItems) {
+            assertThat(response.getItems().length, equalTo(numItems));
+        }
+
+        public void assertResponseWithoutFailures() {
+            assertThat(response, notNullValue());
+            assertFalse("Response should not have failures", response.hasFailures());
+        }
+
+        public void assertResponseWithFailures() {
+            assertThat(response, notNullValue());
+            assertTrue("Response should have failures", response.hasFailures());
+        }
+
+        public void assertOnFailureNeverCalled() {
+            assertThat(lastFailure, nullValue());
+        }
+    }
+
+    private class MockBulkClient extends NoOpClient {
+        private int numberOfCallsToFail;
+
+        private MockBulkClient(String testName, int numberOfCallsToFail) {
+            super(testName);
+            this.numberOfCallsToFail = numberOfCallsToFail;
+        }
+
+        @Override
+        public ActionFuture<BulkResponse> bulk(BulkRequest request) {
+            PlainActionFuture<BulkResponse> responseFuture = new PlainActionFuture<>();
+            bulk(request, responseFuture);
+            return responseFuture;
+        }
+
+        @Override
+        public void bulk(BulkRequest request, ActionListener<BulkResponse> listener) {
+            if (false == expectedHeaders.equals(threadPool().getThreadContext().getHeaders())) {
+                listener.onFailure(
+                    new RuntimeException("Expected " + expectedHeaders + " but got " + threadPool().getThreadContext().getHeaders())
+                );
+                return;
+            }
+
+            // do everything synchronously, that's fine for a test
+            boolean shouldFail = numberOfCallsToFail > 0;
+            numberOfCallsToFail--;
+
+            if (shouldFail && randomBoolean()) {
+                listener.onFailure(new EsRejectedExecutionException("pretend the coordinating thread pool is stuffed"));
+                return;
+            }
+
+            BulkItemResponse[] itemResponses = new BulkItemResponse[request.requests().size()];
+            // if we have to fail, we need to fail at least once "reliably", the rest can be random
+            int itemToFail = randomInt(request.requests().size() - 1);
+            for (int idx = 0; idx < request.requests().size(); idx++) {
+                if (shouldFail && (randomBoolean() || idx == itemToFail)) {
+                    itemResponses[idx] = failedResponse();
+                } else {
+                    itemResponses[idx] = successfulResponse();
+                }
+            }
+            listener.onResponse(new BulkResponse(itemResponses, 1000L));
+        }
+
+        private BulkItemResponse successfulResponse() {
+            return BulkItemResponse.success(1, OpType.DELETE, new DeleteResponse(new ShardId("test", "test", 0), "test", 0, 0, 0, false));
+        }
+
+        private BulkItemResponse failedResponse() {
+            BulkItemResponse.Failure failure = new BulkItemResponse.Failure("test", "1", new EsRejectedExecutionException("pool full"));
+            return BulkItemResponse.failure(1, OpType.INDEX, failure);
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java
index 06ff4a8fed51..3deae4081816 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java
@@ -100,6 +100,7 @@ private void indicesThatCannotBeCreatedTestCase(
         when(state.getMetadata()).thenReturn(Metadata.EMPTY_METADATA);
         when(state.metadata()).thenReturn(Metadata.EMPTY_METADATA);
         when(clusterService.state()).thenReturn(state);
+        when(clusterService.getSettings()).thenReturn(Settings.EMPTY);
 
         DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class);
         when(state.getNodes()).thenReturn(discoveryNodes);
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
index 9da30ae04339..3343f1f9f6eb 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
@@ -188,6 +188,7 @@ public void setupAction() {
         DiscoveryNode localNode = mock(DiscoveryNode.class);
         when(localNode.isIngestNode()).thenAnswer(stub -> localIngest);
         when(clusterService.localNode()).thenReturn(localNode);
+        when(clusterService.getSettings()).thenReturn(Settings.EMPTY);
         remoteNode1 = mock(DiscoveryNode.class);
         remoteNode2 = mock(DiscoveryNode.class);
         nodes = mock(DiscoveryNodes.class);
@@ -290,9 +291,9 @@ public void testIngestLocal() throws Exception {
         verify(ingestService).executeBulkRequest(
             eq(bulkRequest.numberOfActions()),
             bulkDocsItr.capture(),
+            any(),
             failureHandler.capture(),
             completionHandler.capture(),
-            any(),
             eq(Names.WRITE)
         );
         completionHandler.getValue().accept(null, exception);
@@ -332,9 +333,9 @@ public void testSingleItemBulkActionIngestLocal() throws Exception {
         verify(ingestService).executeBulkRequest(
             eq(1),
             bulkDocsItr.capture(),
+            any(),
             failureHandler.capture(),
             completionHandler.capture(),
-            any(),
             eq(Names.WRITE)
         );
         completionHandler.getValue().accept(null, exception);
@@ -378,9 +379,9 @@ public void testIngestSystemLocal() throws Exception {
         verify(ingestService).executeBulkRequest(
             eq(bulkRequest.numberOfActions()),
             bulkDocsItr.capture(),
+            any(),
             failureHandler.capture(),
             completionHandler.capture(),
-            any(),
             eq(Names.SYSTEM_WRITE)
         );
         completionHandler.getValue().accept(null, exception);
@@ -535,9 +536,9 @@ private void validatePipelineWithBulkUpsert(@Nullable String indexRequestIndexNa
         verify(ingestService).executeBulkRequest(
             eq(bulkRequest.numberOfActions()),
             bulkDocsItr.capture(),
+            any(),
             failureHandler.capture(),
             completionHandler.capture(),
-            any(),
             eq(Names.WRITE)
         );
         assertEquals(indexRequest1.getPipeline(), "default_pipeline");
@@ -583,9 +584,9 @@ public void testDoExecuteCalledTwiceCorrectly() throws Exception {
         verify(ingestService).executeBulkRequest(
             eq(1),
             bulkDocsItr.capture(),
+            any(),
             failureHandler.capture(),
             completionHandler.capture(),
-            any(),
             eq(Names.WRITE)
         );
         completionHandler.getValue().accept(null, exception);
@@ -677,9 +678,9 @@ public void testFindDefaultPipelineFromTemplateMatch() {
         verify(ingestService).executeBulkRequest(
             eq(1),
             bulkDocsItr.capture(),
+            any(),
             failureHandler.capture(),
             completionHandler.capture(),
-            any(),
             eq(Names.WRITE)
         );
     }
@@ -721,9 +722,9 @@ public void testFindDefaultPipelineFromV2TemplateMatch() {
         verify(ingestService).executeBulkRequest(
             eq(1),
             bulkDocsItr.capture(),
+            any(),
             failureHandler.capture(),
             completionHandler.capture(),
-            any(),
             eq(Names.WRITE)
         );
     }
@@ -751,9 +752,9 @@ public void testIngestCallbackExceptionHandled() throws Exception {
         verify(ingestService).executeBulkRequest(
             eq(bulkRequest.numberOfActions()),
             bulkDocsItr.capture(),
+            any(),
             failureHandler.capture(),
             completionHandler.capture(),
-            any(),
             eq(Names.WRITE)
         );
         indexRequest1.autoGenerateId();
@@ -788,9 +789,9 @@ private void validateDefaultPipeline(IndexRequest indexRequest) {
         verify(ingestService).executeBulkRequest(
             eq(1),
             bulkDocsItr.capture(),
+            any(),
             failureHandler.capture(),
             completionHandler.capture(),
-            any(),
             eq(Names.WRITE)
         );
         assertEquals(indexRequest.getPipeline(), "default_pipeline");
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java
index 7a5b6079349a..7dd943cf1ddf 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java
@@ -213,12 +213,12 @@ public void testSkipBulkIndexRequestIfAborted() throws Exception {
                 // since at least 1 item passed, the tran log location should exist,
                 assertThat(((WritePrimaryResult<BulkShardRequest, BulkShardResponse>) result).location, notNullValue());
                 // and the response should exist and match the item count
-                assertThat(result.finalResponseIfSuccessful, notNullValue());
-                assertThat(result.finalResponseIfSuccessful.getResponses(), arrayWithSize(items.length));
+                assertThat(result.replicationResponse, notNullValue());
+                assertThat(result.replicationResponse.getResponses(), arrayWithSize(items.length));
 
                 // check each response matches the input item, including the rejection
                 for (int i = 0; i < items.length; i++) {
-                    BulkItemResponse response = result.finalResponseIfSuccessful.getResponses()[i];
+                    BulkItemResponse response = result.replicationResponse.getResponses()[i];
                     assertThat(response.getItemId(), equalTo(i));
                     assertThat(response.getIndex(), equalTo("index"));
                     assertThat(response.getId(), equalTo("id_" + i));
diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/ModifyDataStreamsRequestTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/ModifyDataStreamsRequestTests.java
index c0fce3246dc0..c9bf4b38cd43 100644
--- a/server/src/test/java/org/elasticsearch/action/datastreams/ModifyDataStreamsRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/datastreams/ModifyDataStreamsRequestTests.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -36,7 +35,7 @@ protected Request createTestInstance() {
     }
 
     @Override
-    protected Request mutateInstance(Request request) throws IOException {
+    protected Request mutateInstance(Request request) {
         final int moreActions = randomIntBetween(1, 5);
         List<DataStreamAction> actions = new ArrayList<>(request.getActions());
         for (int k = 1; k <= moreActions; k++) {
diff --git a/server/src/test/java/org/elasticsearch/action/explain/ExplainResponseTests.java b/server/src/test/java/org/elasticsearch/action/explain/ExplainResponseTests.java
index 06ba312cb778..31fcfe342eb3 100644
--- a/server/src/test/java/org/elasticsearch/action/explain/ExplainResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/explain/ExplainResponseTests.java
@@ -66,6 +66,11 @@ protected ExplainResponse createTestInstance() {
         return new ExplainResponse(index, id, exist, explanation, getResult);
     }
 
+    @Override
+    protected ExplainResponse mutateInstance(ExplainResponse instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Predicate<String> getRandomFieldsExcludeFilter() {
         return field -> field.equals("get") || field.startsWith("get.fields") || field.startsWith("get._source");
diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeRequestTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeRequestTests.java
index 549bd4e29973..e19c651bb7f9 100644
--- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeRequestTests.java
@@ -20,12 +20,13 @@
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.equalTo;
+
 public class FieldCapabilitiesNodeRequestTests extends AbstractWireSerializingTestCase<FieldCapabilitiesNodeRequest> {
 
     @Override
@@ -92,7 +93,7 @@ protected Writeable.Reader<FieldCapabilitiesNodeRequest> instanceReader() {
     }
 
     @Override
-    protected FieldCapabilitiesNodeRequest mutateInstance(FieldCapabilitiesNodeRequest instance) throws IOException {
+    protected FieldCapabilitiesNodeRequest mutateInstance(FieldCapabilitiesNodeRequest instance) {
         switch (random().nextInt(7)) {
             case 0 -> {
                 List<ShardId> shardIds = randomShardIds(instance.shardIds().size() + 1);
@@ -203,4 +204,30 @@ protected FieldCapabilitiesNodeRequest mutateInstance(FieldCapabilitiesNodeReque
             default -> throw new IllegalStateException("The test should only allow 7 parameters mutated");
         }
     }
+
+    public void testDescription() {
+        FieldCapabilitiesNodeRequest r1 = new FieldCapabilitiesNodeRequest(
+            List.of(new ShardId("index-1", "n/a", 0), new ShardId("index-2", "n/a", 3)),
+            new String[] { "field-1", "field-2" },
+            Strings.EMPTY_ARRAY,
+            Strings.EMPTY_ARRAY,
+            randomOriginalIndices(1),
+            null,
+            randomNonNegativeLong(),
+            Map.of()
+        );
+        assertThat(r1.getDescription(), equalTo("shards[[index-1][0],[index-2][3]], fields[field-1,field-2], filters[], types[]"));
+
+        FieldCapabilitiesNodeRequest r2 = new FieldCapabilitiesNodeRequest(
+            List.of(new ShardId("index-1", "n/a", 0)),
+            new String[] { "*" },
+            new String[] { "-nested", "-metadata" },
+            Strings.EMPTY_ARRAY,
+            randomOriginalIndices(1),
+            null,
+            randomNonNegativeLong(),
+            Map.of()
+        );
+        assertThat(r2.getDescription(), equalTo("shards[[index-1][0]], fields[*], filters[-nested,-metadata], types[]"));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java
index d4d6c33a0e9c..4ad220fb5f21 100644
--- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java
@@ -112,7 +112,7 @@ public void testFailureParsing() throws IOException {
         boolean humanReadable = randomBoolean();
         XContentType xContentType = randomFrom(XContentType.values());
         BytesReference originalBytes = toShuffledXContent(
-            ChunkedToXContent.wrapAsXContentObject(randomResponse),
+            ChunkedToXContent.wrapAsToXContent(randomResponse),
             xContentType,
             ToXContent.EMPTY_PARAMS,
             humanReadable
diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java
index 9c354451c409..cdad234c73e9 100644
--- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java
+++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java
@@ -11,6 +11,7 @@
 import org.apache.lucene.util.ArrayUtil;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.iterable.Iterables;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.mapper.TimeSeriesParams;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
 import org.elasticsearch.xcontent.XContentParser;
@@ -21,7 +22,6 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.IntStream;
@@ -80,7 +80,7 @@ public void testBuilder() {
 
         builder = new FieldCapabilities.Builder("field", "type");
         builder.add(new String[] { "index1" }, false, false, true, true, null, Collections.emptyMap());
-        builder.add(new String[] { "index2" }, false, true, false, false, TimeSeriesParams.MetricType.counter, Collections.emptyMap());
+        builder.add(new String[] { "index2" }, false, true, false, false, TimeSeriesParams.MetricType.COUNTER, Collections.emptyMap());
         builder.add(new String[] { "index3" }, false, false, false, false, null, Collections.emptyMap());
         {
             FieldCapabilities cap1 = builder.build(false);
@@ -108,15 +108,15 @@ public void testBuilder() {
         }
 
         builder = new FieldCapabilities.Builder("field", "type");
-        builder.add(new String[] { "index1" }, false, true, true, true, TimeSeriesParams.MetricType.counter, Collections.emptyMap());
-        builder.add(new String[] { "index2" }, false, true, true, true, TimeSeriesParams.MetricType.counter, Map.of("foo", "bar"));
-        builder.add(new String[] { "index3" }, false, true, true, true, TimeSeriesParams.MetricType.counter, Map.of("foo", "quux"));
+        builder.add(new String[] { "index1" }, false, true, true, true, TimeSeriesParams.MetricType.COUNTER, Collections.emptyMap());
+        builder.add(new String[] { "index2" }, false, true, true, true, TimeSeriesParams.MetricType.COUNTER, Map.of("foo", "bar"));
+        builder.add(new String[] { "index3" }, false, true, true, true, TimeSeriesParams.MetricType.COUNTER, Map.of("foo", "quux"));
         {
             FieldCapabilities cap1 = builder.build(false);
             assertThat(cap1.isSearchable(), equalTo(true));
             assertThat(cap1.isAggregatable(), equalTo(true));
             assertThat(cap1.isDimension(), equalTo(true));
-            assertThat(cap1.getMetricType(), equalTo(TimeSeriesParams.MetricType.counter));
+            assertThat(cap1.getMetricType(), equalTo(TimeSeriesParams.MetricType.COUNTER));
             assertNull(cap1.indices());
             assertNull(cap1.nonSearchableIndices());
             assertNull(cap1.nonAggregatableIndices());
@@ -127,7 +127,7 @@ public void testBuilder() {
             assertThat(cap2.isSearchable(), equalTo(true));
             assertThat(cap2.isAggregatable(), equalTo(true));
             assertThat(cap2.isDimension(), equalTo(true));
-            assertThat(cap2.getMetricType(), equalTo(TimeSeriesParams.MetricType.counter));
+            assertThat(cap2.getMetricType(), equalTo(TimeSeriesParams.MetricType.COUNTER));
             assertThat(cap2.indices().length, equalTo(3));
             assertThat(cap2.indices(), equalTo(new String[] { "index1", "index2", "index3" }));
             assertNull(cap2.nonSearchableIndices());
@@ -137,9 +137,9 @@ public void testBuilder() {
         }
 
         builder = new FieldCapabilities.Builder("field", "type");
-        builder.add(new String[] { "index1" }, false, true, true, true, TimeSeriesParams.MetricType.counter, Collections.emptyMap());
-        builder.add(new String[] { "index2" }, false, true, true, true, TimeSeriesParams.MetricType.gauge, Map.of("foo", "bar"));
-        builder.add(new String[] { "index3" }, false, true, true, true, TimeSeriesParams.MetricType.counter, Map.of("foo", "quux"));
+        builder.add(new String[] { "index1" }, false, true, true, true, TimeSeriesParams.MetricType.COUNTER, Collections.emptyMap());
+        builder.add(new String[] { "index2" }, false, true, true, true, TimeSeriesParams.MetricType.GAUGE, Map.of("foo", "bar"));
+        builder.add(new String[] { "index3" }, false, true, true, true, TimeSeriesParams.MetricType.COUNTER, Map.of("foo", "quux"));
         {
             FieldCapabilities cap1 = builder.build(false);
             assertThat(cap1.isSearchable(), equalTo(true));
@@ -167,9 +167,7 @@ public void testBuilder() {
     }
 
     public void testRandomBuilder() {
-        String[] indices = IntStream.range(0, randomIntBetween(1, 50))
-            .mapToObj(n -> String.format(Locale.ROOT, "index_%2d", n))
-            .toArray(String[]::new);
+        String[] indices = IntStream.range(0, randomIntBetween(1, 50)).mapToObj(n -> Strings.format("index_%2d", n)).toArray(String[]::new);
 
         List<String> nonSearchableIndices = new ArrayList<>();
         List<String> nonAggregatableIndices = new ArrayList<>();
@@ -238,7 +236,7 @@ public void testRandomBuilder() {
     }
 
     public void testBuilderSingleMetricType() {
-        List<String> indices = IntStream.range(0, randomIntBetween(1, 50)).mapToObj(n -> formatted("index_%2d", n)).toList();
+        List<String> indices = IntStream.range(0, randomIntBetween(1, 50)).mapToObj(n -> Strings.format("index_%2d", n)).toList();
         TimeSeriesParams.MetricType metric = randomBoolean() ? null : randomFrom(TimeSeriesParams.MetricType.values());
         FieldCapabilities.Builder builder = new FieldCapabilities.Builder("field", "type");
         for (String index : indices) {
@@ -250,7 +248,7 @@ public void testBuilderSingleMetricType() {
     }
 
     public void testBuilderMixedMetricType() {
-        List<String> indices = IntStream.range(0, randomIntBetween(1, 50)).mapToObj(n -> formatted("index_%2d", n)).toList();
+        List<String> indices = IntStream.range(0, randomIntBetween(1, 50)).mapToObj(n -> Strings.format("index_%2d", n)).toList();
         Map<String, TimeSeriesParams.MetricType> metricTypes = new HashMap<>();
         for (String index : indices) {
             if (randomBoolean()) {
diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java
index 94c339e417c0..36c8a05b0226 100644
--- a/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java
@@ -166,7 +166,7 @@ private static FieldCapabilitiesResponse createSimpleResponse() {
                 true,
                 false,
                 false,
-                TimeSeriesParams.MetricType.counter,
+                TimeSeriesParams.MetricType.COUNTER,
                 new String[] { "index1", "index2" },
                 null,
                 new String[] { "index1" },
@@ -204,30 +204,12 @@ private static FieldCapabilitiesResponse createSimpleResponse() {
         return new FieldCapabilitiesResponse(new String[] { "index1", "index2", "index3", "index4" }, responses, failureMap);
     }
 
-    public void testExpectedChunkSizes() {
-        {
-            final FieldCapabilitiesResponse instance = FieldCapabilitiesResponseTests.createResponseWithFailures();
-            final var iterator = instance.toXContentChunked();
-            int chunks = 0;
-            while (iterator.hasNext()) {
-                iterator.next();
-                chunks++;
-            }
-            if (instance.getFailures().isEmpty()) {
-                assertEquals(2, chunks);
-            } else {
-                assertEquals(3 + instance.get().size() + instance.getFailures().size(), chunks);
-            }
-        }
-        {
-            final FieldCapabilitiesResponse instance = createTestInstance();
-            final var iterator = instance.toXContentChunked();
-            int chunks = 0;
-            while (iterator.hasNext()) {
-                iterator.next();
-                chunks++;
-            }
-            assertEquals(2 + instance.get().size(), chunks);
-        }
+    public void testChunking() {
+        AbstractChunkedSerializingTestCase.assertChunkCount(
+            FieldCapabilitiesResponseTests.createResponseWithFailures(),
+            instance -> instance.getFailures().isEmpty() ? 2 : (3 + instance.get().size() + instance.getFailures().size())
+        );
+
+        AbstractChunkedSerializingTestCase.assertChunkCount(createTestInstance(), instance -> 2 + instance.get().size());
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/RequestDispatcherTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/RequestDispatcherTests.java
index f1aea8852d66..97f8a5a5dd73 100644
--- a/server/src/test/java/org/elasticsearch/action/fieldcaps/RequestDispatcherTests.java
+++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/RequestDispatcherTests.java
@@ -11,12 +11,14 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -77,6 +79,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import static org.elasticsearch.common.settings.ClusterSettings.createBuiltInClusterSettings;
 import static org.elasticsearch.test.VersionUtils.randomVersion;
 import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.equalTo;
@@ -741,7 +744,7 @@ static TestTransportService newTestTransportService() {
             final TestThreadPool threadPool = new TestThreadPool("test");
             TcpTransport transport = new Netty4Transport(
                 Settings.EMPTY,
-                Version.CURRENT,
+                TransportVersion.CURRENT,
                 threadPool,
                 new NetworkService(Collections.emptyList()),
                 PageCacheRecycler.NON_RECYCLING_INSTANCE,
@@ -893,7 +896,7 @@ static Task newRandomParentTask() {
     }
 
     private ClusterState newClusterState(Metadata metadata, DiscoveryNodes discoveryNodes) {
-        final RoutingTable.Builder routingTable = RoutingTable.builder();
+        final RoutingTable.Builder routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         for (IndexMetadata imd : metadata) {
             routingTable.addAsNew(metadata.index(imd.getIndex()));
         }
@@ -903,10 +906,10 @@ private ClusterState newClusterState(Metadata metadata, DiscoveryNodes discovery
             .routingTable(routingTable.build())
             .build();
         final Settings settings = Settings.EMPTY;
-        final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        final ClusterSettings clusterSettings = createBuiltInClusterSettings(settings);
         final ArrayList<AllocationDecider> deciders = new ArrayList<>();
-        deciders.add(new EnableAllocationDecider(settings, clusterSettings));
-        deciders.add(new SameShardAllocationDecider(settings, clusterSettings));
+        deciders.add(new EnableAllocationDecider(clusterSettings));
+        deciders.add(new SameShardAllocationDecider(clusterSettings));
         deciders.add(new ReplicaAfterPrimaryActiveAllocationDecider());
         Collections.shuffle(deciders, random());
         final MockAllocationService allocationService = new MockAllocationService(
diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/ResponseRewriterTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/ResponseRewriterTests.java
index e757bdf0423b..9264db0c5aef 100644
--- a/server/src/test/java/org/elasticsearch/action/fieldcaps/ResponseRewriterTests.java
+++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/ResponseRewriterTests.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.fieldcaps;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.test.ESTestCase;
 
@@ -26,7 +26,7 @@ public void testExcludeMetadata() {
         );
 
         Map<String, IndexFieldCapabilities> rewritten = ResponseRewriter.rewriteOldResponses(
-            Version.V_8_0_0,
+            TransportVersion.V_8_0_0,
             oldResponse,
             new String[] { "-metadata" },
             Strings.EMPTY_ARRAY
@@ -45,7 +45,7 @@ public void testIncludeOnlyMetadata() {
         );
 
         Map<String, IndexFieldCapabilities> rewritten = ResponseRewriter.rewriteOldResponses(
-            Version.V_8_0_0,
+            TransportVersion.V_8_0_0,
             oldResponse,
             new String[] { "+metadata" },
             Strings.EMPTY_ARRAY
@@ -66,7 +66,7 @@ public void testExcludeNested() {
         );
 
         Map<String, IndexFieldCapabilities> rewritten = ResponseRewriter.rewriteOldResponses(
-            Version.V_8_0_0,
+            TransportVersion.V_8_0_0,
             oldResponse,
             new String[] { "-nested" },
             Strings.EMPTY_ARRAY
@@ -90,7 +90,7 @@ public void testExcludeMultifield() {
         );
 
         Map<String, IndexFieldCapabilities> rewritten = ResponseRewriter.rewriteOldResponses(
-            Version.V_8_0_0,
+            TransportVersion.V_8_0_0,
             oldResponse,
             new String[] { "-multifield" },
             Strings.EMPTY_ARRAY
@@ -112,7 +112,7 @@ public void testExcludeParents() {
         );
 
         Map<String, IndexFieldCapabilities> rewritten = ResponseRewriter.rewriteOldResponses(
-            Version.V_8_0_0,
+            TransportVersion.V_8_0_0,
             oldResponse,
             new String[] { "-parent" },
             Strings.EMPTY_ARRAY
@@ -134,7 +134,7 @@ public void testAllowedTypes() {
         );
 
         Map<String, IndexFieldCapabilities> rewritten = ResponseRewriter.rewriteOldResponses(
-            Version.V_8_0_0,
+            TransportVersion.V_8_0_0,
             oldResponse,
             Strings.EMPTY_ARRAY,
             new String[] { "text", "keyword" }
diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesActionTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesActionTests.java
index 1e5f9762b1b5..a390924b2e20 100644
--- a/server/src/test/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesActionTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.action.fieldcaps;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilter;
@@ -56,8 +57,10 @@ public void testCCSCompatibilityCheck() throws Exception {
             fieldCapsRequest.indexFilter(new DummyQueryBuilder() {
                 @Override
                 protected void doWriteTo(StreamOutput out) throws IOException {
-                    if (out.getVersion().before(Version.CURRENT)) {
-                        throw new IllegalArgumentException("This query isn't serializable to nodes before " + Version.CURRENT);
+                    if (out.getTransportVersion().before(TransportVersion.CURRENT)) {
+                        throw new IllegalArgumentException(
+                            "This query isn't serializable before transport version " + TransportVersion.CURRENT
+                        );
                     }
                 }
             });
@@ -88,7 +91,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
                 containsString("[class org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest] is not compatible with version")
             );
             assertThat(ex.getMessage(), containsString("and the 'search.check_ccs_compatibility' setting is enabled."));
-            assertEquals("This query isn't serializable to nodes before " + Version.CURRENT, ex.getCause().getMessage());
+            assertEquals("This query isn't serializable before transport version " + TransportVersion.CURRENT, ex.getCause().getMessage());
         } finally {
             assertTrue(ESTestCase.terminate(threadPool));
         }
diff --git a/server/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java b/server/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java
index 2546e4ef2a0e..879e4c73dd3c 100644
--- a/server/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.action.get;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -18,7 +18,7 @@
 
 import java.io.IOException;
 
-import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
+import static org.elasticsearch.test.TransportVersionUtils.randomVersionBetween;
 import static org.hamcrest.CoreMatchers.equalTo;
 
 public class MultiGetShardRequestTests extends ESTestCase {
@@ -26,15 +26,15 @@ public void testSerialization() throws IOException {
         MultiGetShardRequest multiGetShardRequest = createTestInstance(randomBoolean());
 
         BytesStreamOutput out = new BytesStreamOutput();
-        Version minVersion = Version.CURRENT.minimumCompatibilityVersion();
+        TransportVersion minVersion = TransportVersion.CURRENT.minimumCompatibilityVersion();
         if (multiGetShardRequest.isForceSyntheticSource()) {
-            minVersion = Version.V_8_4_0;
+            minVersion = TransportVersion.V_8_4_0;
         }
-        out.setVersion(randomVersionBetween(random(), minVersion, Version.CURRENT));
+        out.setTransportVersion(randomVersionBetween(random(), minVersion, TransportVersion.CURRENT));
         multiGetShardRequest.writeTo(out);
 
         StreamInput in = out.bytes().streamInput();
-        in.setVersion(out.getVersion());
+        in.setTransportVersion(out.getTransportVersion());
         MultiGetShardRequest multiGetShardRequest2 = new MultiGetShardRequest(in);
         assertThat(multiGetShardRequest2.index(), equalTo(multiGetShardRequest.index()));
         assertThat(multiGetShardRequest2.preference(), equalTo(multiGetShardRequest.preference()));
@@ -58,7 +58,7 @@ public void testSerialization() throws IOException {
     public void testForceSyntheticUnsupported() {
         MultiGetShardRequest request = createTestInstance(true);
         StreamOutput out = new BytesStreamOutput();
-        out.setVersion(Version.V_8_3_0);
+        out.setTransportVersion(TransportVersion.V_8_3_0);
         Exception e = expectThrows(IllegalArgumentException.class, () -> request.writeTo(out));
         assertEquals(e.getMessage(), "force_synthetic_source is not supported before 8.4.0");
     }
diff --git a/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java
index 58fc7c9040f5..8840335c6c06 100644
--- a/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.FormatNames;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.seqno.SequenceNumbers;
@@ -145,23 +146,19 @@ public void testIndexResponse() {
         assertEquals(total, indexResponse.getShardInfo().getTotal());
         assertEquals(successful, indexResponse.getShardInfo().getSuccessful());
         assertEquals(forcedRefresh, indexResponse.forcedRefresh());
-        assertEquals(
-            formatted(
-                """
-                    IndexResponse[index=%s,id=%s,version=%s,result=%s,seqNo=%s,primaryTerm=%s,shards=\
-                    {"total":%s,"successful":%s,"failed":0}]\
-                    """,
-                shardId.getIndexName(),
-                id,
-                version,
-                created ? "created" : "updated",
-                SequenceNumbers.UNASSIGNED_SEQ_NO,
-                0,
-                total,
-                successful
-            ),
-            indexResponse.toString()
-        );
+        Object[] args = new Object[] {
+            shardId.getIndexName(),
+            id,
+            version,
+            created ? "created" : "updated",
+            SequenceNumbers.UNASSIGNED_SEQ_NO,
+            0,
+            total,
+            successful };
+        assertEquals(Strings.format("""
+            IndexResponse[index=%s,id=%s,version=%s,result=%s,seqNo=%s,primaryTerm=%s,shards=\
+            {"total":%s,"successful":%s,"failed":0}]\
+            """, args), indexResponse.toString());
     }
 
     public void testIndexRequestXContentSerialization() throws IOException {
@@ -257,7 +254,7 @@ public void testToStringSizeLimit() {
         request.source(source, XContentType.JSON);
         assertEquals("index {[index][null], source[" + source + "]}", request.toString());
 
-        source = formatted("""
+        source = Strings.format("""
             {"name":"%s"}
             """, randomUnicodeOfLength(IndexRequest.MAX_SOURCE_LENGTH_IN_TOSTRING));
         request.source(source, XContentType.JSON);
diff --git a/server/src/test/java/org/elasticsearch/action/ingest/ReservedPipelineActionTests.java b/server/src/test/java/org/elasticsearch/action/ingest/ReservedPipelineActionTests.java
index 9562e469435f..cbf9033c9d67 100644
--- a/server/src/test/java/org/elasticsearch/action/ingest/ReservedPipelineActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ingest/ReservedPipelineActionTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -47,7 +46,6 @@
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.empty;
 import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
@@ -130,15 +128,8 @@ public void setup() {
         );
 
         fileSettingsService = spy(
-            new FileSettingsService(
-                clusterService,
-                mock(ReservedClusterStateService.class),
-                newEnvironment(Settings.EMPTY),
-                mock(NodeClient.class)
-            )
+            new FileSettingsService(clusterService, mock(ReservedClusterStateService.class), newEnvironment(Settings.EMPTY))
         );
-
-        doReturn(response).when(fileSettingsService).nodeInfos();
     }
 
     private TransformState processJSON(ReservedPipelineAction action, TransformState prevState, String json) throws Exception {
@@ -147,36 +138,10 @@ private TransformState processJSON(ReservedPipelineAction action, TransformState
         }
     }
 
-    public void testValidation() throws Exception {
-        ClusterState state = ClusterState.builder(new ClusterName("elasticsearch")).build();
-        TransformState prevState = new TransformState(state, Collections.emptySet());
-        ReservedPipelineAction action = makeSpiedAction();
-
-        String badPolicyJSON = """
-            {
-               "my_ingest_pipeline": {
-                   "description": "_description",
-                   "processors": [
-                      {
-                        "foo" : {
-                          "field": "pipeline",
-                          "value": "pipeline"
-                        }
-                      }
-                   ]
-               }
-            }""";
-
-        assertEquals(
-            "Error processing ingest pipelines",
-            expectThrows(IllegalArgumentException.class, () -> processJSON(action, prevState, badPolicyJSON)).getMessage()
-        );
-    }
-
     public void testAddRemoveIngestPipeline() throws Exception {
         ClusterState state = ClusterState.builder(new ClusterName("elasticsearch")).build();
         TransformState prevState = new TransformState(state, Collections.emptySet());
-        ReservedPipelineAction action = makeSpiedAction();
+        ReservedPipelineAction action = new ReservedPipelineAction();
 
         String emptyJSON = "";
 
@@ -235,9 +200,4 @@ public void testAddRemoveIngestPipeline() throws Exception {
         updatedState = processJSON(action, prevState, emptyJSON);
         assertThat(updatedState.keys(), empty());
     }
-
-    @SuppressWarnings("unchecked")
-    private ReservedPipelineAction makeSpiedAction() {
-        return spy(new ReservedPipelineAction(ingestService, fileSettingsService));
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java
index 21b25d974571..d76df88e9818 100644
--- a/server/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java
@@ -61,8 +61,8 @@ public void testSerialization() throws IOException {
 
     static SimulateProcessorResult createTestInstance(boolean isSuccessful, boolean isIgnoredException, boolean hasCondition) {
         String type = randomAlphaOfLengthBetween(1, 10);
-        String processorTag = randomAlphaOfLengthBetween(1, 10);
-        String description = randomAlphaOfLengthBetween(1, 10);
+        String processorTag = randomBoolean() ? null : randomAlphaOfLengthBetween(1, 10);
+        String description = randomBoolean() ? null : randomAlphaOfLengthBetween(1, 10);
         Tuple<String, Boolean> conditionWithResult = hasCondition ? new Tuple<>(randomAlphaOfLengthBetween(1, 10), randomBoolean()) : null;
         SimulateProcessorResult simulateProcessorResult;
         if (isSuccessful) {
diff --git a/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java b/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java
index 6d70802bab02..f4cbfac402a5 100644
--- a/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java
@@ -71,7 +71,10 @@ public void testEqualsAndHashcode() throws Exception {
         if (randomBoolean()) {
             numFields = randomIntBetween(1, IngestDocument.Metadata.values().length);
             for (int i = 0; i < numFields; i++) {
-                Tuple<String, Object> metadata = TestIngestDocument.randomMetadata();
+                Tuple<String, Object> metadata = randomValueOtherThanMany(
+                    t -> t.v2().equals(sourceAndMetadata.get(t.v1())),
+                    TestIngestDocument::randomMetadata
+                );
                 otherSourceAndMetadata.put(metadata.v1(), metadata.v2());
             }
             changed = true;
diff --git a/server/src/test/java/org/elasticsearch/action/main/MainResponseTests.java b/server/src/test/java/org/elasticsearch/action/main/MainResponseTests.java
index b7f569a51908..f591a40315ae 100644
--- a/server/src/test/java/org/elasticsearch/action/main/MainResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/main/MainResponseTests.java
@@ -57,7 +57,7 @@ public void testToXContent() throws IOException {
         response.toXContent(builder, ToXContent.EMPTY_PARAMS);
         assertEquals(
             XContentHelper.stripWhitespace(
-                formatted(
+                Strings.format(
                     """
                         {
                             "name": "nodeName",
diff --git a/server/src/test/java/org/elasticsearch/action/resync/TransportResyncReplicationActionTests.java b/server/src/test/java/org/elasticsearch/action/resync/TransportResyncReplicationActionTests.java
index 32adf7b33487..919737caf2c7 100644
--- a/server/src/test/java/org/elasticsearch/action/resync/TransportResyncReplicationActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/resync/TransportResyncReplicationActionTests.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.action.resync;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
@@ -102,7 +102,7 @@ public void testResyncDoesNotBlockOnPrimaryAction() throws Exception {
             try (
                 TcpTransport transport = new Netty4Transport(
                     Settings.EMPTY,
-                    Version.CURRENT,
+                    TransportVersion.CURRENT,
                     threadPool,
                     new NetworkService(emptyList()),
                     PageCacheRecycler.NON_RECYCLING_INSTANCE,
@@ -152,7 +152,7 @@ public void testResyncDoesNotBlockOnPrimaryAction() throws Exception {
                     new ReplicationGroup(
                         shardRoutingTable,
                         clusterService.state().metadata().index(index).inSyncAllocationIds(shardId.id()),
-                        shardRoutingTable.getAllAllocationIds(),
+                        shardRoutingTable.getPromotableAllocationIds(),
                         0
                     )
                 );
diff --git a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
index d6d072f29fe0..3ab1c5aefac6 100644
--- a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java
@@ -243,8 +243,6 @@ public void testShardNotAvailableWithDisallowPartialFailures() {
         SearchShardIterator skipIterator = new SearchShardIterator(null, null, Collections.emptyList(), null);
         skipIterator.resetAndSkip();
         action.skipShard(skipIterator);
-        // expect at least 2 shards, so onPhaseDone should report failure.
-        action.onPhaseDone();
         assertThat(exception.get(), instanceOf(SearchPhaseExecutionException.class));
         SearchPhaseExecutionException searchPhaseExecutionException = (SearchPhaseExecutionException) exception.get();
         assertEquals("Partial shards failure (" + (numShards - 1) + " shards unavailable)", searchPhaseExecutionException.getMessage());
diff --git a/server/src/test/java/org/elasticsearch/action/search/KnnSearchSingleNodeTests.java b/server/src/test/java/org/elasticsearch/action/search/KnnSearchSingleNodeTests.java
index 65c0df0f25e7..a9801ff8ff7e 100644
--- a/server/src/test/java/org/elasticsearch/action/search/KnnSearchSingleNodeTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/KnnSearchSingleNodeTests.java
@@ -8,11 +8,14 @@
 
 package org.elasticsearch.action.search;
 
+import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.indices.TermsLookup;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.metrics.InternalStats;
 import org.elasticsearch.search.vectors.KnnSearchBuilder;
 import org.elasticsearch.search.vectors.KnnVectorQueryBuilder;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -20,12 +23,58 @@
 import org.elasticsearch.xcontent.XContentFactory;
 
 import java.io.IOException;
+import java.util.List;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 
 public class KnnSearchSingleNodeTests extends ESSingleNodeTestCase {
     private static final int VECTOR_DIMENSION = 10;
 
+    public void testKnnSearchRemovedVector() throws IOException {
+        int numShards = 1 + randomInt(3);
+        Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards).build();
+
+        XContentBuilder builder = XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("properties")
+            .startObject("vector")
+            .field("type", "dense_vector")
+            .field("dims", VECTOR_DIMENSION)
+            .field("index", true)
+            .field("similarity", "l2_norm")
+            .endObject()
+            .startObject("text")
+            .field("type", "text")
+            .endObject()
+            .endObject()
+            .endObject();
+        createIndex("index", indexSettings, builder);
+
+        for (int doc = 0; doc < 10; doc++) {
+            client().prepareIndex("index").setId(Integer.toString(doc)).setSource("vector", randomVector(), "text", "hello world").get();
+            client().prepareIndex("index").setSource("text", "goodnight world").get();
+        }
+
+        client().admin().indices().prepareRefresh("index").get();
+        client().prepareUpdate("index", "0").setDoc("vector", (Object) null).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get();
+
+        float[] queryVector = randomVector();
+        KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 20, 50).boost(5.0f);
+        SearchResponse response = client().prepareSearch("index")
+            .setKnnSearch(List.of(knnSearch))
+            .setQuery(QueryBuilders.matchQuery("text", "goodnight"))
+            .setSize(10)
+            .get();
+
+        // Originally indexed 20 documents, but deleted vector field with an update, so only 19 should be hit
+        assertHitCount(response, 19);
+        assertEquals(10, response.getHits().getHits().length);
+        // Make sure we still have 20 docs
+        assertHitCount(client().prepareSearch("index").setSize(0).setTrackTotalHits(true).get(), 20);
+    }
+
     public void testKnnWithQuery() throws IOException {
         int numShards = 1 + randomInt(3);
         Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards).build();
@@ -56,7 +105,7 @@ public void testKnnWithQuery() throws IOException {
         float[] queryVector = randomVector();
         KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 5, 50).boost(5.0f);
         SearchResponse response = client().prepareSearch("index")
-            .setKnnSearch(knnSearch)
+            .setKnnSearch(List.of(knnSearch))
             .setQuery(QueryBuilders.matchQuery("text", "goodnight"))
             .addFetchField("*")
             .setSize(10)
@@ -101,7 +150,7 @@ public void testKnnFilter() throws IOException {
         KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 5, 50).addFilterQuery(
             QueryBuilders.termsQuery("field", "second")
         );
-        SearchResponse response = client().prepareSearch("index").setKnnSearch(knnSearch).addFetchField("*").setSize(10).get();
+        SearchResponse response = client().prepareSearch("index").setKnnSearch(List.of(knnSearch)).addFetchField("*").setSize(10).get();
 
         assertHitCount(response, 5);
         assertEquals(5, response.getHits().getHits().length);
@@ -144,12 +193,145 @@ public void testKnnFilterWithRewrite() throws IOException {
         KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 5, 50).addFilterQuery(
             QueryBuilders.termsLookupQuery("field", new TermsLookup("index", "lookup-doc", "other-field"))
         );
-        SearchResponse response = client().prepareSearch("index").setKnnSearch(knnSearch).setSize(10).get();
+        SearchResponse response = client().prepareSearch("index").setKnnSearch(List.of(knnSearch)).setSize(10).get();
 
         assertHitCount(response, 5);
         assertEquals(5, response.getHits().getHits().length);
     }
 
+    public void testMultiKnnClauses() throws IOException {
+        // This tests the recall from vectors being searched in different docs
+        int numShards = 1 + randomInt(3);
+        Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards).build();
+
+        XContentBuilder builder = XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("properties")
+            .startObject("vector")
+            .field("type", "dense_vector")
+            .field("dims", VECTOR_DIMENSION)
+            .field("index", true)
+            .field("similarity", "l2_norm")
+            .endObject()
+            .startObject("vector_2")
+            .field("type", "dense_vector")
+            .field("dims", VECTOR_DIMENSION)
+            .field("index", true)
+            .field("similarity", "l2_norm")
+            .endObject()
+            .startObject("text")
+            .field("type", "text")
+            .endObject()
+            .startObject("number")
+            .field("type", "long")
+            .endObject()
+            .endObject()
+            .endObject();
+        createIndex("index", indexSettings, builder);
+
+        for (int doc = 0; doc < 10; doc++) {
+            client().prepareIndex("index").setSource("vector", randomVector(1.0f, 2.0f), "text", "hello world", "number", 1).get();
+            client().prepareIndex("index").setSource("vector_2", randomVector(20f, 21f), "text", "hello world", "number", 2).get();
+            client().prepareIndex("index").setSource("text", "goodnight world", "number", 3).get();
+        }
+        client().admin().indices().prepareRefresh("index").get();
+
+        float[] queryVector = randomVector(20f, 21f);
+        KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 5, 50).boost(5.0f);
+        KnnSearchBuilder knnSearch2 = new KnnSearchBuilder("vector_2", queryVector, 5, 50).boost(10.0f);
+        SearchResponse response = client().prepareSearch("index")
+            .setKnnSearch(List.of(knnSearch, knnSearch2))
+            .setQuery(QueryBuilders.matchQuery("text", "goodnight"))
+            .addFetchField("*")
+            .setSize(10)
+            .addAggregation(AggregationBuilders.stats("stats").field("number"))
+            .get();
+
+        // The total hits is k plus the number of text matches
+        assertHitCount(response, 20);
+        assertEquals(10, response.getHits().getHits().length);
+        InternalStats agg = response.getAggregations().get("stats");
+        assertThat(agg.getCount(), equalTo(20L));
+        assertThat(agg.getMax(), equalTo(3.0));
+        assertThat(agg.getMin(), equalTo(1.0));
+        assertThat(agg.getAvg(), equalTo(2.25));
+        assertThat(agg.getSum(), equalTo(45.0));
+
+        // Because of the boost & vector distributions, vector_2 results should appear first
+        assertNotNull(response.getHits().getAt(0).field("vector_2"));
+    }
+
+    public void testMultiKnnClausesSameDoc() throws IOException {
+        int numShards = 1 + randomInt(3);
+        Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards).build();
+
+        XContentBuilder builder = XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("properties")
+            .startObject("vector")
+            .field("type", "dense_vector")
+            .field("dims", VECTOR_DIMENSION)
+            .field("index", true)
+            .field("similarity", "l2_norm")
+            .endObject()
+            .startObject("vector_2")
+            .field("type", "dense_vector")
+            .field("dims", VECTOR_DIMENSION)
+            .field("index", true)
+            .field("similarity", "l2_norm")
+            .endObject()
+            .startObject("number")
+            .field("type", "long")
+            .endObject()
+            .endObject()
+            .endObject();
+        createIndex("index", indexSettings, builder);
+
+        for (int doc = 0; doc < 10; doc++) {
+            // Make them have hte same vector. This will allow us to test the recall is the same but scores take into account both fields
+            float[] vector = randomVector();
+            client().prepareIndex("index").setSource("vector", vector, "vector_2", vector, "number", doc).get();
+        }
+        client().admin().indices().prepareRefresh("index").get();
+
+        float[] queryVector = randomVector();
+        // Having the same query vector and same docs should mean our KNN scores are linearly combined if the same doc is matched
+        KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 5, 50);
+        KnnSearchBuilder knnSearch2 = new KnnSearchBuilder("vector_2", queryVector, 5, 50);
+        SearchResponse responseOneKnn = client().prepareSearch("index")
+            .setKnnSearch(List.of(knnSearch))
+            .addFetchField("*")
+            .setSize(10)
+            .addAggregation(AggregationBuilders.stats("stats").field("number"))
+            .get();
+        SearchResponse responseBothKnn = client().prepareSearch("index")
+            .setKnnSearch(List.of(knnSearch, knnSearch2))
+            .addFetchField("*")
+            .setSize(10)
+            .addAggregation(AggregationBuilders.stats("stats").field("number"))
+            .get();
+
+        // The total hits is k matched docs
+        assertHitCount(responseOneKnn, 5);
+        assertHitCount(responseBothKnn, 5);
+        assertEquals(5, responseOneKnn.getHits().getHits().length);
+        assertEquals(5, responseBothKnn.getHits().getHits().length);
+
+        for (int i = 0; i < responseOneKnn.getHits().getHits().length; i++) {
+            SearchHit oneHit = responseOneKnn.getHits().getHits()[i];
+            SearchHit bothHit = responseBothKnn.getHits().getHits()[i];
+            assertThat(bothHit.getId(), equalTo(oneHit.getId()));
+            assertThat(bothHit.getScore(), greaterThan(oneHit.getScore()));
+        }
+        InternalStats oneAgg = responseOneKnn.getAggregations().get("stats");
+        InternalStats bothAgg = responseBothKnn.getAggregations().get("stats");
+        assertThat(bothAgg.getCount(), equalTo(oneAgg.getCount()));
+        assertThat(bothAgg.getAvg(), equalTo(oneAgg.getAvg()));
+        assertThat(bothAgg.getMax(), equalTo(oneAgg.getMax()));
+        assertThat(bothAgg.getSum(), equalTo(oneAgg.getSum()));
+        assertThat(bothAgg.getMin(), equalTo(oneAgg.getMin()));
+    }
+
     public void testKnnFilteredAlias() throws IOException {
         int numShards = 1 + randomInt(3);
         Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards).build();
@@ -184,7 +366,7 @@ public void testKnnFilteredAlias() throws IOException {
 
         float[] queryVector = randomVector();
         KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", queryVector, 10, 50);
-        SearchResponse response = client().prepareSearch("test-alias").setKnnSearch(knnSearch).setSize(10).get();
+        SearchResponse response = client().prepareSearch("test-alias").setKnnSearch(List.of(knnSearch)).setSize(10).get();
 
         assertHitCount(response, expectedHits);
         assertEquals(expectedHits, response.getHits().getHits().length);
@@ -234,4 +416,12 @@ private float[] randomVector() {
         }
         return vector;
     }
+
+    private float[] randomVector(float dimLower, float dimUpper) {
+        float[] vector = new float[VECTOR_DIMENSION];
+        for (int i = 0; i < vector.length; i++) {
+            vector[i] = (float) randomDoubleBetween(dimLower, dimUpper, true);
+        }
+        return vector;
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
index 8657f595986b..d13bf82b5cd9 100644
--- a/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
@@ -195,7 +195,7 @@ public void testResponseErrorToXContent() throws IOException {
             tookInMillis
         );
 
-        assertEquals(XContentHelper.stripWhitespace(formatted("""
+        assertEquals(XContentHelper.stripWhitespace(Strings.format("""
             {
               "took": %s,
               "responses": [
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
index f5f6228a1c12..74ee3cddde85 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
@@ -724,7 +724,8 @@ static List<SearchShardIterator> getShardsIter(
                 new ShardId(index, i),
                 true,
                 RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar")
+                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"),
+                ShardRouting.Role.DEFAULT
             );
             if (primaryNode != null) {
                 routing = routing.initialize(primaryNode.getId(), i + "p", 0);
@@ -736,7 +737,8 @@ static List<SearchShardIterator> getShardsIter(
                     new ShardId(index, i),
                     false,
                     RecoverySource.PeerRecoverySource.INSTANCE,
-                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar")
+                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"),
+                    ShardRouting.Role.DEFAULT
                 );
                 if (replicaNode != null) {
                     routing = routing.initialize(replicaNode.getId(), i + "r", 0);
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
index 7c3857ec8aec..9f466bdaf69e 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java
@@ -270,7 +270,8 @@ private void testMixedVersionsShardsSearch(Version oldVersion, Version newVersio
             new ShardId(new Index("idx", "_na_"), 0),
             true,
             RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"),
+            ShardRouting.Role.DEFAULT
         );
         routingNewVersionShard = routingNewVersionShard.initialize(newVersionNode.getId(), "p0", 0);
         routingNewVersionShard.started();
@@ -280,7 +281,8 @@ private void testMixedVersionsShardsSearch(Version oldVersion, Version newVersio
             new ShardId(new Index("idx", "_na_"), 1),
             true,
             RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"),
+            ShardRouting.Role.DEFAULT
         );
         routingOldVersionShard = routingOldVersionShard.initialize(oldVersionNode.getId(), "p1", 0);
         routingOldVersionShard.started();
@@ -368,7 +370,8 @@ public void testMinimumVersionSameAsOldVersion() throws Exception {
             new ShardId(new Index("idx", "_na_"), 0),
             true,
             RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"),
+            ShardRouting.Role.DEFAULT
         );
         routingNewVersionShard = routingNewVersionShard.initialize(newVersionNode.getId(), "p0", 0);
         routingNewVersionShard.started();
@@ -378,7 +381,8 @@ public void testMinimumVersionSameAsOldVersion() throws Exception {
             new ShardId(new Index("idx", "_na_"), 1),
             true,
             RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"),
+            ShardRouting.Role.DEFAULT
         );
         routingOldVersionShard = routingOldVersionShard.initialize(oldVersionNode.getId(), "p1", 0);
         routingOldVersionShard.started();
@@ -510,7 +514,8 @@ public void testMinimumVersionShardDuringPhaseExecution() throws Exception {
             new ShardId(new Index("idx", "_na_"), 0),
             true,
             RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"),
+            ShardRouting.Role.DEFAULT
         );
         routingNewVersionShard1 = routingNewVersionShard1.initialize(newVersionNode1.getId(), "p0", 0);
         routingNewVersionShard1.started();
@@ -520,7 +525,8 @@ public void testMinimumVersionShardDuringPhaseExecution() throws Exception {
             new ShardId(new Index("idx", "_na_"), 1),
             true,
             RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"),
+            ShardRouting.Role.DEFAULT
         );
         routingNewVersionShard2 = routingNewVersionShard2.initialize(newVersionNode2.getId(), "p1", 0);
         routingNewVersionShard2.started();
@@ -620,7 +626,8 @@ public void run() {
             new ShardId(new Index("idx", "_na_"), 2),
             true,
             RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"),
+            ShardRouting.Role.DEFAULT
         );
         SearchShardIterator shardIt = new SearchShardIterator(
             null,
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
index 44838951fc42..84fc17c86fa8 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.search.builder.PointInTimeBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.rescore.QueryRescorerBuilder;
+import org.elasticsearch.search.vectors.KnnSearchBuilder;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
@@ -85,6 +86,38 @@ public void testSerialization() throws Exception {
         assertNotSame(deserializedRequest, searchRequest);
     }
 
+    public void testSerializationMultiKNN() throws Exception {
+        SearchRequest searchRequest = createSearchRequest();
+        if (searchRequest.source() == null) {
+            searchRequest.source(new SearchSourceBuilder());
+        }
+        searchRequest.source()
+            .knnSearch(
+                List.of(
+                    new KnnSearchBuilder(randomAlphaOfLength(10), new float[] { 1, 2 }, 5, 10),
+                    new KnnSearchBuilder(randomAlphaOfLength(10), new float[] { 4, 12, 41 }, 3, 5)
+                )
+            );
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> copyWriteable(
+                searchRequest,
+                namedWriteableRegistry,
+                SearchRequest::new,
+                VersionUtils.randomVersionBetween(random(), Version.V_8_4_0, Version.V_8_6_0)
+            )
+        );
+
+        searchRequest.source().knnSearch(List.of(new KnnSearchBuilder(randomAlphaOfLength(10), new float[] { 1, 2 }, 5, 10)));
+        // Shouldn't throw because its just one KNN request
+        copyWriteable(
+            searchRequest,
+            namedWriteableRegistry,
+            SearchRequest::new,
+            VersionUtils.randomVersionBetween(random(), Version.V_8_4_0, Version.V_8_6_0)
+        );
+    }
+
     public void testRandomVersionSerialization() throws IOException {
         SearchRequest searchRequest = createSearchRequest();
         Version version = VersionUtils.randomVersion(random());
@@ -93,9 +126,13 @@ public void testRandomVersionSerialization() throws IOException {
             searchRequest.source().runtimeMappings(emptyMap());
         }
         if (version.before(Version.V_8_4_0)) {
-            // Versionse before 8.4.0 don't support force_synthetic_source
+            // Versions before 8.4.0 don't support force_synthetic_source
             searchRequest.setForceSyntheticSource(false);
         }
+        if (version.before(Version.V_8_7_0) && searchRequest.hasKnnSearch() && searchRequest.source().knnSearch().size() > 1) {
+            // Versions before 8.7.0 don't support more than one KNN clause
+            searchRequest.source().knnSearch(List.of(searchRequest.source().knnSearch().get(0)));
+        }
         SearchRequest deserializedRequest = copyWriteable(searchRequest, namedWriteableRegistry, SearchRequest::new, version);
         assertEquals(searchRequest.isCcsMinimizeRoundtrips(), deserializedRequest.isCcsMinimizeRoundtrips());
         assertEquals(searchRequest.getLocalClusterAlias(), deserializedRequest.getLocalClusterAlias());
diff --git a/server/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java b/server/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java
index 2fab9b5dae8b..a74be0297f4d 100644
--- a/server/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.action.search;
 
+import org.elasticsearch.action.NoShardAvailableActionException;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -28,17 +29,17 @@
 
 public class ShardSearchFailureTests extends ESTestCase {
 
+    private static SearchShardTarget randomShardTarget(String indexUuid) {
+        String nodeId = randomAlphaOfLengthBetween(5, 10);
+        String indexName = randomAlphaOfLengthBetween(5, 10);
+        String clusterAlias = randomBoolean() ? randomAlphaOfLengthBetween(5, 10) : null;
+        return new SearchShardTarget(nodeId, new ShardId(new Index(indexName, indexUuid), randomInt()), clusterAlias);
+    }
+
     public static ShardSearchFailure createTestItem(String indexUuid) {
         String randomMessage = randomAlphaOfLengthBetween(3, 20);
         Exception ex = new ParsingException(0, 0, randomMessage, new IllegalArgumentException("some bad argument"));
-        SearchShardTarget searchShardTarget = null;
-        if (randomBoolean()) {
-            String nodeId = randomAlphaOfLengthBetween(5, 10);
-            String indexName = randomAlphaOfLengthBetween(5, 10);
-            String clusterAlias = randomBoolean() ? randomAlphaOfLengthBetween(5, 10) : null;
-            searchShardTarget = new SearchShardTarget(nodeId, new ShardId(new Index(indexName, indexUuid), randomInt()), clusterAlias);
-        }
-        return new ShardSearchFailure(ex, searchShardTarget);
+        return new ShardSearchFailure(ex, randomBoolean() ? randomShardTarget(indexUuid) : null);
     }
 
     public void testFromXContent() throws IOException {
@@ -110,6 +111,22 @@ public void testToXContent() throws IOException {
             }"""), xContent.utf8ToString());
     }
 
+    public void testToXContentForNoShardAvailable() throws IOException {
+        ShardId shardId = new ShardId(new Index("indexName", "indexUuid"), 123);
+        ShardSearchFailure failure = new ShardSearchFailure(
+            NoShardAvailableActionException.forOnShardFailureWrapper("shard unassigned"),
+            new SearchShardTarget("nodeId", shardId, null)
+        );
+        BytesReference xContent = toXContent(failure, XContentType.JSON, randomBoolean());
+        assertEquals(XContentHelper.stripWhitespace("""
+            {
+              "shard": 123,
+              "index": "indexName",
+              "node": "nodeId",
+              "reason":{"type":"no_shard_available_action_exception","reason":"shard unassigned"}
+            }"""), xContent.utf8ToString());
+    }
+
     public void testToXContentWithClusterAlias() throws IOException {
         ShardSearchFailure failure = new ShardSearchFailure(
             new ParsingException(0, 0, "some message", null),
@@ -131,17 +148,25 @@ public void testToXContentWithClusterAlias() throws IOException {
     }
 
     public void testSerialization() throws IOException {
-        ShardSearchFailure testItem = createTestItem(randomAlphaOfLength(12));
-        ShardSearchFailure deserializedInstance = copyWriteable(
-            testItem,
-            writableRegistry(),
-            ShardSearchFailure::new,
-            VersionUtils.randomVersion(random())
-        );
-        assertEquals(testItem.index(), deserializedInstance.index());
-        assertEquals(testItem.shard(), deserializedInstance.shard());
-        assertEquals(testItem.shardId(), deserializedInstance.shardId());
-        assertEquals(testItem.reason(), deserializedInstance.reason());
-        assertEquals(testItem.status(), deserializedInstance.status());
+        for (int runs = 0; runs < 25; runs++) {
+            final ShardSearchFailure testItem;
+            if (randomBoolean()) {
+                testItem = createTestItem(randomAlphaOfLength(12));
+            } else {
+                SearchShardTarget target = randomShardTarget(randomAlphaOfLength(12));
+                testItem = new ShardSearchFailure(NoShardAvailableActionException.forOnShardFailureWrapper("unavailable"), target);
+            }
+            ShardSearchFailure deserializedInstance = copyWriteable(
+                testItem,
+                writableRegistry(),
+                ShardSearchFailure::new,
+                VersionUtils.randomVersion(random())
+            );
+            assertEquals(testItem.index(), deserializedInstance.index());
+            assertEquals(testItem.shard(), deserializedInstance.shard());
+            assertEquals(testItem.shardId(), deserializedInstance.shardId());
+            assertEquals(testItem.reason(), deserializedInstance.reason());
+            assertEquals(testItem.status(), deserializedInstance.status());
+        }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
index 0a94f946d5b8..03ce6a322e88 100644
--- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
@@ -1051,7 +1051,7 @@ public void testShouldMinimizeRoundtrips() throws Exception {
             SearchSourceBuilder source = searchRequest.source();
             if (source != null) {
                 source.pointInTimeBuilder(null);
-                source.knnSearch(null);
+                source.knnSearch(List.of());
                 CollapseBuilder collapse = source.collapse();
                 if (collapse != null) {
                     collapse.setInnerHits(Collections.emptyList());
@@ -1065,7 +1065,7 @@ public void testShouldMinimizeRoundtrips() throws Exception {
         {
             SearchRequest searchRequest = new SearchRequest();
             SearchSourceBuilder source = new SearchSourceBuilder();
-            source.knnSearch(new KnnSearchBuilder("field", new float[] { 1, 2, 3 }, 10, 50));
+            source.knnSearch(List.of(new KnnSearchBuilder("field", new float[] { 1, 2, 3 }, 10, 50)));
             searchRequest.source(source);
 
             searchRequest.setCcsMinimizeRoundtrips(true);
@@ -1080,7 +1080,7 @@ public void testAdjustSearchType() {
             // If the search includes kNN, we should always use DFS_QUERY_THEN_FETCH
             SearchRequest searchRequest = new SearchRequest();
             SearchSourceBuilder source = new SearchSourceBuilder();
-            source.knnSearch(new KnnSearchBuilder("field", new float[] { 1, 2, 3 }, 10, 50));
+            source.knnSearch(List.of(new KnnSearchBuilder("field", new float[] { 1, 2, 3 }, 10, 50)));
             searchRequest.source(source);
 
             TransportSearchAction.adjustSearchType(searchRequest, randomBoolean());
diff --git a/server/src/test/java/org/elasticsearch/action/support/ActiveShardCountTests.java b/server/src/test/java/org/elasticsearch/action/support/ActiveShardCountTests.java
index 1e330a4062f0..aab2e1399951 100644
--- a/server/src/test/java/org/elasticsearch/action/support/ActiveShardCountTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/ActiveShardCountTests.java
@@ -11,12 +11,14 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingRoleStrategy;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -101,6 +103,73 @@ public void testEnoughShardsActiveLevelDefault() {
         runTestForOneActiveShard(ActiveShardCount.DEFAULT);
     }
 
+    public void testEnoughShardsActiveLevelDefaultWithSearchOnlyRole() {
+        final String indexName = "test-idx";
+        final int numberOfShards = randomIntBetween(1, 5);
+        final int numberOfReplicas = randomIntBetween(4, 7);
+        final ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT;
+        ClusterState clusterState = initializeWithNewIndex(indexName, numberOfShards, numberOfReplicas, createCustomRoleStrategy(1));
+        assertFalse(waitForActiveShards.enoughShardsActive(clusterState, indexName));
+        clusterState = startPrimaries(clusterState, indexName);
+        assertFalse(waitForActiveShards.enoughShardsActive(clusterState, indexName));
+        clusterState = startLessThanWaitOnShards(clusterState, indexName, 1);
+        assertTrue(waitForActiveShards.enoughShardsActive(clusterState, indexName));
+        clusterState = startAllShards(clusterState, indexName);
+        assertTrue(waitForActiveShards.enoughShardsActive(clusterState, indexName));
+    }
+
+    public void testEnoughShardsActiveCustomLevelWithSearchOnlyRole() {
+        final String indexName = "test-idx";
+        final int numberOfShards = randomIntBetween(1, 5);
+        final int numberOfReplicas = randomIntBetween(4, 7);
+        final int activeShardCount = randomIntBetween(2, numberOfReplicas);
+        final ActiveShardCount waitForActiveShards = ActiveShardCount.from(activeShardCount);
+        ClusterState clusterState = initializeWithNewIndex(indexName, numberOfShards, numberOfReplicas, createCustomRoleStrategy(1));
+        assertFalse(waitForActiveShards.enoughShardsActive(clusterState, indexName));
+        clusterState = startPrimaries(clusterState, indexName);
+        assertFalse(waitForActiveShards.enoughShardsActive(clusterState, indexName));
+        clusterState = startLessThanWaitOnShards(clusterState, indexName, activeShardCount - 2);
+        assertFalse(waitForActiveShards.enoughShardsActive(clusterState, indexName));
+        clusterState = startWaitOnShards(clusterState, indexName, activeShardCount);
+        assertTrue(waitForActiveShards.enoughShardsActive(clusterState, indexName));
+        clusterState = startAllShards(clusterState, indexName);
+        assertTrue(waitForActiveShards.enoughShardsActive(clusterState, indexName));
+    }
+
+    public void testEnoughShardsActiveWithNoSearchOnlyRoles() {
+        final String indexName = "test-idx";
+        final int numberOfShards = randomIntBetween(1, 5);
+        final int numberOfReplicas = randomIntBetween(4, 7);
+        final ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT;
+        ClusterState clusterState = initializeWithNewIndex(
+            indexName,
+            numberOfShards,
+            numberOfReplicas,
+            createCustomRoleStrategy(numberOfReplicas + 1)
+        );
+        assertFalse(waitForActiveShards.enoughShardsActive(clusterState, indexName));
+        clusterState = startPrimaries(clusterState, indexName);
+        assertTrue(waitForActiveShards.enoughShardsActive(clusterState, indexName));
+        clusterState = startLessThanWaitOnShards(clusterState, indexName, 1);
+        assertTrue(waitForActiveShards.enoughShardsActive(clusterState, indexName));
+        clusterState = startAllShards(clusterState, indexName);
+        assertTrue(waitForActiveShards.enoughShardsActive(clusterState, indexName));
+    }
+
+    private static ShardRoutingRoleStrategy createCustomRoleStrategy(int indexShardCount) {
+        return new ShardRoutingRoleStrategy() {
+            @Override
+            public ShardRouting.Role newEmptyRole(int copyIndex) {
+                return copyIndex < indexShardCount ? ShardRouting.Role.INDEX_ONLY : ShardRouting.Role.SEARCH_ONLY;
+            }
+
+            @Override
+            public ShardRouting.Role newReplicaRole() {
+                return ShardRouting.Role.SEARCH_ONLY;
+            }
+        };
+    }
+
     public void testEnoughShardsActiveRandom() {
         final String indexName = "test-idx";
         final int numberOfShards = randomIntBetween(1, 5);
@@ -165,12 +234,11 @@ public void testEnoughShardsActiveWithClosedIndex() {
         }
     }
 
-    private void runTestForOneActiveShard(final ActiveShardCount activeShardCount) {
+    private void runTestForOneActiveShard(final ActiveShardCount waitForActiveShards) {
         final String indexName = "test-idx";
         final int numberOfShards = randomIntBetween(1, 5);
         final int numberOfReplicas = randomIntBetween(4, 7);
-        assert activeShardCount == ActiveShardCount.ONE || activeShardCount == ActiveShardCount.DEFAULT;
-        final ActiveShardCount waitForActiveShards = activeShardCount;
+        assert waitForActiveShards == ActiveShardCount.ONE || waitForActiveShards == ActiveShardCount.DEFAULT;
         ClusterState clusterState = initializeWithNewIndex(indexName, numberOfShards, numberOfReplicas);
         assertFalse(waitForActiveShards.enoughShardsActive(clusterState, indexName));
         clusterState = startPrimaries(clusterState, indexName);
@@ -179,7 +247,11 @@ private void runTestForOneActiveShard(final ActiveShardCount activeShardCount) {
         assertTrue(waitForActiveShards.enoughShardsActive(clusterState, indexName));
     }
 
-    private ClusterState initializeWithNewIndex(final String indexName, final int numShards, final int numReplicas) {
+    private ClusterState initializeWithNewIndex(String indexName, int numShards, int numReplicas) {
+        return initializeWithNewIndex(indexName, numShards, numReplicas, TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
+    }
+
+    private ClusterState initializeWithNewIndex(String indexName, int numShards, int numReplicas, ShardRoutingRoleStrategy strategy) {
         // initial index creation and new routing table info
         final IndexMetadata indexMetadata = IndexMetadata.builder(indexName)
             .settings(settings(Version.CURRENT).put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()))
@@ -187,7 +259,7 @@ private ClusterState initializeWithNewIndex(final String indexName, final int nu
             .numberOfReplicas(numReplicas)
             .build();
         final Metadata metadata = Metadata.builder().put(indexMetadata, true).build();
-        final RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build();
+        final RoutingTable routingTable = RoutingTable.builder(strategy).addAsNew(indexMetadata).build();
         return ClusterState.builder(new ClusterName("test_cluster")).metadata(metadata).routingTable(routingTable).build();
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/support/AdapterActionFutureTests.java b/server/src/test/java/org/elasticsearch/action/support/AdapterActionFutureTests.java
deleted file mode 100644
index 26344cca238d..000000000000
--- a/server/src/test/java/org/elasticsearch/action/support/AdapterActionFutureTests.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.action.support;
-
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException;
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.transport.RemoteTransportException;
-
-import java.util.Objects;
-import java.util.concurrent.BrokenBarrierException;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-public class AdapterActionFutureTests extends ESTestCase {
-
-    public void testInterruption() throws Exception {
-        final AdapterActionFuture<String, Integer> adapter = new AdapterActionFuture<String, Integer>() {
-            @Override
-            protected String convert(final Integer listenerResponse) {
-                return Objects.toString(listenerResponse);
-            }
-        };
-
-        // test all possible methods that can be interrupted
-        final Runnable runnable = () -> {
-            final int method = randomIntBetween(0, 4);
-            switch (method) {
-                case 0 -> adapter.actionGet();
-                case 1 -> adapter.actionGet("30s");
-                case 2 -> adapter.actionGet(30000);
-                case 3 -> adapter.actionGet(TimeValue.timeValueSeconds(30));
-                case 4 -> adapter.actionGet(30, TimeUnit.SECONDS);
-                default -> throw new AssertionError(method);
-            }
-        };
-
-        final CyclicBarrier barrier = new CyclicBarrier(2);
-        final Thread main = Thread.currentThread();
-        final Thread thread = new Thread(() -> {
-            try {
-                barrier.await();
-            } catch (final BrokenBarrierException | InterruptedException e) {
-                throw new RuntimeException(e);
-            }
-            main.interrupt();
-        });
-        thread.start();
-
-        final AtomicBoolean interrupted = new AtomicBoolean();
-
-        barrier.await();
-
-        try {
-            runnable.run();
-        } catch (final IllegalStateException e) {
-            interrupted.set(Thread.interrupted());
-        }
-        // we check this here instead of in the catch block to ensure that the catch block executed
-        assertTrue(interrupted.get());
-
-        thread.join();
-    }
-
-    public void testUnwrapException() {
-        checkUnwrap(new RemoteTransportException("test", new RuntimeException()), RuntimeException.class, RemoteTransportException.class);
-        checkUnwrap(
-            new RemoteTransportException("test", new Exception()),
-            UncategorizedExecutionException.class,
-            RemoteTransportException.class
-        );
-        checkUnwrap(new Exception(), UncategorizedExecutionException.class, Exception.class);
-        checkUnwrap(new ElasticsearchException("test", new Exception()), ElasticsearchException.class, ElasticsearchException.class);
-    }
-
-    private void checkUnwrap(Exception exception, Class<? extends Exception> actionGetException, Class<? extends Exception> getException) {
-        final AdapterActionFuture<Void, Void> adapter = new AdapterActionFuture<Void, Void>() {
-            @Override
-            protected Void convert(Void listenerResponse) {
-                fail();
-                return null;
-            }
-        };
-
-        adapter.onFailure(exception);
-        assertEquals(actionGetException, expectThrows(RuntimeException.class, adapter::actionGet).getClass());
-        assertEquals(actionGetException, expectThrows(RuntimeException.class, () -> adapter.actionGet(10, TimeUnit.SECONDS)).getClass());
-        assertEquals(getException, expectThrows(ExecutionException.class, () -> adapter.get()).getCause().getClass());
-        assertEquals(getException, expectThrows(ExecutionException.class, () -> adapter.get(10, TimeUnit.SECONDS)).getCause().getClass());
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/action/support/CountDownActionListenerTests.java b/server/src/test/java/org/elasticsearch/action/support/CountDownActionListenerTests.java
new file mode 100644
index 000000000000..54b98e26b8e6
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/support/CountDownActionListenerTests.java
@@ -0,0 +1,189 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.action.support;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+
+public class CountDownActionListenerTests extends ESTestCase {
+
+    public void testNotifications() throws InterruptedException {
+        AtomicBoolean called = new AtomicBoolean(false);
+        ActionListener<Void> result = new ActionListener<>() {
+            @Override
+            public void onResponse(Void ignored) {
+                called.set(true);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError(e);
+            }
+        };
+        final int groupSize = randomIntBetween(10, 1000);
+        AtomicInteger count = new AtomicInteger();
+        CountDownActionListener listener = new CountDownActionListener(groupSize, result);
+        int numThreads = randomIntBetween(2, 5);
+        Thread[] threads = new Thread[numThreads];
+        CyclicBarrier barrier = new CyclicBarrier(numThreads);
+        for (int i = 0; i < numThreads; i++) {
+            threads[i] = new Thread(() -> {
+                try {
+                    barrier.await(10, TimeUnit.SECONDS);
+                } catch (Exception e) {
+                    throw new AssertionError(e);
+                }
+                while (count.incrementAndGet() <= groupSize) {
+                    listener.onResponse(null);
+                }
+            });
+            threads[i].start();
+        }
+        for (Thread t : threads) {
+            t.join();
+        }
+        assertTrue(called.get());
+    }
+
+    public void testFailed() {
+        AtomicBoolean called = new AtomicBoolean(false);
+        AtomicReference<Exception> excRef = new AtomicReference<>();
+
+        ActionListener<Void> result = new ActionListener<>() {
+            @Override
+            public void onResponse(Void ignored) {
+                called.set(true);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                excRef.set(e);
+            }
+        };
+        int size = randomIntBetween(3, 4);
+        CountDownActionListener listener = new CountDownActionListener(size, result);
+        listener.onResponse(null);
+        IOException ioException = new IOException();
+        RuntimeException rtException = new RuntimeException();
+        listener.onFailure(rtException);
+        listener.onFailure(ioException);
+        if (size == 4) {
+            listener.onResponse(null);
+        }
+        assertNotNull(excRef.get());
+        assertEquals(rtException, excRef.get());
+        assertEquals(1, excRef.get().getSuppressed().length);
+        assertEquals(ioException, excRef.get().getSuppressed()[0]);
+        assertFalse(called.get());
+    }
+
+    public void testValidation() throws InterruptedException {
+        AtomicBoolean called = new AtomicBoolean(false);
+        ActionListener<Void> result = new ActionListener<>() {
+            @Override
+            public void onResponse(Void ignored) {
+                called.compareAndSet(false, true);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                called.compareAndSet(false, true);
+            }
+        };
+
+        // can't use a groupSize of 0
+        expectThrows(AssertionError.class, () -> new CountDownActionListener(0, result));
+
+        // can't use a null listener or runnable
+        expectThrows(NullPointerException.class, () -> new CountDownActionListener(1, (ActionListener<Void>) null));
+
+        final int overage = randomIntBetween(1, 10);
+        AtomicInteger assertionsTriggered = new AtomicInteger();
+        final int groupSize = randomIntBetween(10, 1000);
+        AtomicInteger count = new AtomicInteger();
+        CountDownActionListener listener = new CountDownActionListener(groupSize, result);
+        int numThreads = randomIntBetween(2, 5);
+        Thread[] threads = new Thread[numThreads];
+        CyclicBarrier barrier = new CyclicBarrier(numThreads);
+        for (int i = 0; i < numThreads; i++) {
+            threads[i] = new Thread(() -> {
+                try {
+                    barrier.await(10, TimeUnit.SECONDS);
+                } catch (Exception e) {
+                    throw new AssertionError(e);
+                }
+                int c;
+                while ((c = count.incrementAndGet()) <= groupSize + overage) {
+                    try {
+                        if (c % 10 == 1) { // a mix of failures and non-failures
+                            listener.onFailure(new RuntimeException());
+                        } else {
+                            listener.onResponse(null);
+                        }
+                    } catch (AssertionError e) {
+                        assertionsTriggered.incrementAndGet();
+                    }
+                }
+            });
+            threads[i].start();
+        }
+        for (Thread t : threads) {
+            t.join();
+        }
+        assertTrue(called.get());
+        assertEquals(overage, assertionsTriggered.get());
+    }
+
+    public void testConcurrentFailures() throws InterruptedException {
+        AtomicReference<Exception> finalException = new AtomicReference<>();
+        int numGroups = randomIntBetween(10, 100);
+        CountDownActionListener listener = new CountDownActionListener(numGroups, ActionListener.wrap(r -> {}, finalException::set));
+        ExecutorService executorService = Executors.newFixedThreadPool(numGroups);
+        for (int i = 0; i < numGroups; i++) {
+            executorService.submit(() -> listener.onFailure(new IOException()));
+        }
+
+        executorService.shutdown();
+        executorService.awaitTermination(10, TimeUnit.SECONDS);
+
+        Exception exception = finalException.get();
+        assertNotNull(exception);
+        assertThat(exception, instanceOf(IOException.class));
+        assertEquals(numGroups - 1, exception.getSuppressed().length);
+    }
+
+    /*
+     * It can happen that the same exception causes a grouped listener to be notified of the failure multiple times. Since we suppress
+     * additional exceptions into the first exception, we have to guard against suppressing into the same exception, which could occur if we
+     * are notified of with the same failure multiple times. This test verifies that the guard against self-suppression remains.
+     */
+    public void testRepeatNotificationForTheSameException() {
+        final AtomicReference<Exception> finalException = new AtomicReference<>();
+        final CountDownActionListener listener = new CountDownActionListener(2, ActionListener.wrap(r -> {}, finalException::set));
+        final Exception e = new Exception();
+        // repeat notification for the same exception
+        listener.onFailure(e);
+        listener.onFailure(e);
+        assertThat(finalException.get(), not(nullValue()));
+        assertThat(finalException.get(), equalTo(e));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/support/GroupedActionListenerTests.java b/server/src/test/java/org/elasticsearch/action/support/GroupedActionListenerTests.java
index 418c78122f17..10d53ed208ec 100644
--- a/server/src/test/java/org/elasticsearch/action/support/GroupedActionListenerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/GroupedActionListenerTests.java
@@ -43,7 +43,7 @@ public void onFailure(Exception e) {
         };
         final int groupSize = randomIntBetween(10, 1000);
         AtomicInteger count = new AtomicInteger();
-        GroupedActionListener<Integer> listener = new GroupedActionListener<>(result, groupSize);
+        GroupedActionListener<Integer> listener = new GroupedActionListener<>(groupSize, result);
         int numThreads = randomIntBetween(2, 5);
         Thread[] threads = new Thread[numThreads];
         CyclicBarrier barrier = new CyclicBarrier(numThreads);
@@ -90,7 +90,7 @@ public void onFailure(Exception e) {
             }
         };
         int size = randomIntBetween(3, 4);
-        GroupedActionListener<Integer> listener = new GroupedActionListener<>(result, size);
+        GroupedActionListener<Integer> listener = new GroupedActionListener<>(size, result);
         listener.onResponse(0);
         IOException ioException = new IOException();
         RuntimeException rtException = new RuntimeException();
@@ -111,7 +111,7 @@ public void onFailure(Exception e) {
     public void testConcurrentFailures() throws InterruptedException {
         AtomicReference<Exception> finalException = new AtomicReference<>();
         int numGroups = randomIntBetween(10, 100);
-        GroupedActionListener<Void> listener = new GroupedActionListener<>(ActionListener.wrap(r -> {}, finalException::set), numGroups);
+        GroupedActionListener<Void> listener = new GroupedActionListener<>(numGroups, ActionListener.wrap(r -> {}, finalException::set));
         ExecutorService executorService = Executors.newFixedThreadPool(numGroups);
         for (int i = 0; i < numGroups; i++) {
             executorService.submit(() -> listener.onFailure(new IOException()));
@@ -133,7 +133,7 @@ public void testConcurrentFailures() throws InterruptedException {
      */
     public void testRepeatNotificationForTheSameException() {
         final AtomicReference<Exception> finalException = new AtomicReference<>();
-        final GroupedActionListener<Void> listener = new GroupedActionListener<>(ActionListener.wrap(r -> {}, finalException::set), 2);
+        final GroupedActionListener<Void> listener = new GroupedActionListener<>(2, ActionListener.wrap(r -> {}, finalException::set));
         final Exception e = new Exception();
         // repeat notification for the same exception
         listener.onFailure(e);
diff --git a/server/src/test/java/org/elasticsearch/action/support/ListenableActionFutureTests.java b/server/src/test/java/org/elasticsearch/action/support/ListenableActionFutureTests.java
index 8954fd6ede9d..ae19ad7560e0 100644
--- a/server/src/test/java/org/elasticsearch/action/support/ListenableActionFutureTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/ListenableActionFutureTests.java
@@ -7,9 +7,11 @@
  */
 package org.elasticsearch.action.support;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ReachabilityChecker;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.Transports;
@@ -159,4 +161,23 @@ private static void awaitSafe(CyclicBarrier barrier) {
             throw new AssertionError("unexpected", e);
         }
     }
+
+    public void testAddedListenersReleasedOnCompletion() {
+        final ListenableActionFuture<Void> future = new ListenableActionFuture<>();
+        final ReachabilityChecker reachabilityChecker = new ReachabilityChecker();
+
+        for (int i = between(1, 3); i > 0; i--) {
+            future.addListener(reachabilityChecker.register(ActionListener.wrap(() -> {})));
+        }
+        reachabilityChecker.checkReachable();
+        if (randomBoolean()) {
+            future.onResponse(null);
+        } else {
+            future.onFailure(new ElasticsearchException("simulated"));
+        }
+        reachabilityChecker.ensureUnreachable();
+
+        future.addListener(reachabilityChecker.register(ActionListener.wrap(() -> {})));
+        reachabilityChecker.ensureUnreachable();
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/support/NodeResponseTrackerTests.java b/server/src/test/java/org/elasticsearch/action/support/NodeResponseTrackerTests.java
deleted file mode 100644
index 11d2ee1f12a0..000000000000
--- a/server/src/test/java/org/elasticsearch/action/support/NodeResponseTrackerTests.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.action.support;
-
-import org.elasticsearch.test.ESTestCase;
-
-public class NodeResponseTrackerTests extends ESTestCase {
-
-    public void testAllResponsesReceived() throws Exception {
-        int nodes = randomIntBetween(1, 10);
-        NodeResponseTracker intermediateNodeResponses = new NodeResponseTracker(nodes);
-        for (int i = 0; i < nodes; i++) {
-            boolean isLast = i == nodes - 1;
-            assertEquals(
-                isLast,
-                intermediateNodeResponses.trackResponseAndCheckIfLast(i, randomBoolean() ? i : new Exception("from node " + i))
-            );
-        }
-
-        assertFalse(intermediateNodeResponses.responsesDiscarded());
-        assertEquals(nodes, intermediateNodeResponses.getExpectedResponseCount());
-        for (int i = 0; i < nodes; i++) {
-            assertNotNull(intermediateNodeResponses.getResponse(i));
-            if (intermediateNodeResponses.getResponse(i)instanceof Integer nodeResponse) {
-                assertEquals(i, nodeResponse.intValue());
-            }
-        }
-    }
-
-    public void testDiscardingResults() {
-        int nodes = randomIntBetween(1, 10);
-        int cancelAt = randomIntBetween(0, Math.max(0, nodes - 2));
-        NodeResponseTracker intermediateNodeResponses = new NodeResponseTracker(nodes);
-        for (int i = 0; i < nodes; i++) {
-            if (i == cancelAt) {
-                intermediateNodeResponses.discardIntermediateResponses(new Exception("simulated"));
-            }
-            boolean isLast = i == nodes - 1;
-            assertEquals(
-                isLast,
-                intermediateNodeResponses.trackResponseAndCheckIfLast(i, randomBoolean() ? i : new Exception("from node " + i))
-            );
-        }
-
-        assertTrue(intermediateNodeResponses.responsesDiscarded());
-        assertEquals(nodes, intermediateNodeResponses.getExpectedResponseCount());
-        expectThrows(NodeResponseTracker.DiscardedResponsesException.class, () -> intermediateNodeResponses.getResponse(0));
-    }
-
-    public void testResponseIsRegisteredOnlyOnce() {
-        NodeResponseTracker intermediateNodeResponses = new NodeResponseTracker(1);
-        assertTrue(intermediateNodeResponses.trackResponseAndCheckIfLast(0, "response1"));
-        expectThrows(AssertionError.class, () -> intermediateNodeResponses.trackResponseAndCheckIfLast(0, "response2"));
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/action/support/PlainActionFutureTests.java b/server/src/test/java/org/elasticsearch/action/support/PlainActionFutureTests.java
new file mode 100644
index 000000000000..d9f735d454de
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/support/PlainActionFutureTests.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.support;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.RemoteTransportException;
+
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class PlainActionFutureTests extends ESTestCase {
+
+    public void testInterruption() throws Exception {
+        final PlainActionFuture<Object> adapter = new PlainActionFuture<>() {
+            @Override
+            public void onResponse(Object value) {
+                throw new AssertionError("should not be called");
+            }
+        };
+
+        // test all possible methods that can be interrupted
+        final Runnable runnable = () -> {
+            final int method = randomIntBetween(0, 4);
+            switch (method) {
+                case 0 -> adapter.actionGet();
+                case 1 -> adapter.actionGet("30s");
+                case 2 -> adapter.actionGet(30000);
+                case 3 -> adapter.actionGet(TimeValue.timeValueSeconds(30));
+                case 4 -> adapter.actionGet(30, TimeUnit.SECONDS);
+                default -> throw new AssertionError(method);
+            }
+        };
+
+        final CyclicBarrier barrier = new CyclicBarrier(2);
+        final Thread main = Thread.currentThread();
+        final Thread thread = new Thread(() -> {
+            try {
+                barrier.await();
+            } catch (final BrokenBarrierException | InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+            main.interrupt();
+        });
+        thread.start();
+
+        final AtomicBoolean interrupted = new AtomicBoolean();
+
+        barrier.await();
+
+        try {
+            runnable.run();
+        } catch (final IllegalStateException e) {
+            interrupted.set(Thread.interrupted());
+        }
+        // we check this here instead of in the catch block to ensure that the catch block executed
+        assertTrue(interrupted.get());
+
+        thread.join();
+    }
+
+    public void testUnwrapException() {
+        checkUnwrap(new RemoteTransportException("test", new RuntimeException()), RuntimeException.class, RemoteTransportException.class);
+        checkUnwrap(
+            new RemoteTransportException("test", new Exception()),
+            UncategorizedExecutionException.class,
+            RemoteTransportException.class
+        );
+        checkUnwrap(new Exception(), UncategorizedExecutionException.class, Exception.class);
+        checkUnwrap(new ElasticsearchException("test", new Exception()), ElasticsearchException.class, ElasticsearchException.class);
+    }
+
+    private void checkUnwrap(Exception exception, Class<? extends Exception> actionGetException, Class<? extends Exception> getException) {
+        final PlainActionFuture<Void> adapter = new PlainActionFuture<>() {
+            @Override
+            public void onResponse(Void value) {
+                throw new AssertionError("should not be called");
+            }
+        };
+
+        adapter.onFailure(exception);
+        assertEquals(actionGetException, expectThrows(RuntimeException.class, adapter::actionGet).getClass());
+        assertEquals(actionGetException, expectThrows(RuntimeException.class, () -> adapter.actionGet(10, TimeUnit.SECONDS)).getClass());
+        assertEquals(getException, expectThrows(ExecutionException.class, () -> adapter.get()).getCause().getClass());
+        assertEquals(getException, expectThrows(ExecutionException.class, () -> adapter.get(10, TimeUnit.SECONDS)).getCause().getClass());
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/support/RefCountingListenerTests.java b/server/src/test/java/org/elasticsearch/action/support/RefCountingListenerTests.java
new file mode 100644
index 000000000000..71424008da31
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/support/RefCountingListenerTests.java
@@ -0,0 +1,242 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.support;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ReachabilityChecker;
+
+import java.util.ArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
+
+import static org.elasticsearch.common.util.concurrent.EsExecutors.DIRECT_EXECUTOR_SERVICE;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+
+public class RefCountingListenerTests extends ESTestCase {
+
+    public void testBasicOperation() throws InterruptedException {
+        final var executed = new AtomicBoolean();
+        final var exceptionCount = new AtomicInteger();
+        final var threads = new Thread[between(0, 3)];
+        final var exceptionLimit = Math.max(1, between(0, threads.length + 2));
+
+        boolean async = false;
+        final var startLatch = new CountDownLatch(1);
+
+        try (var refs = new RefCountingListener(exceptionLimit, new ActionListener<>() {
+            @Override
+            public void onResponse(Void unused) {
+                assertTrue(executed.compareAndSet(false, true));
+                assertEquals(0, exceptionCount.get());
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                assertTrue(executed.compareAndSet(false, true));
+                assertThat(exceptionCount.get(), greaterThan(0));
+                Throwable[] suppressed = e.getSuppressed();
+                if (exceptionCount.get() > exceptionLimit) {
+                    assertEquals(exceptionLimit, suppressed.length);
+                    for (int i = 0; i < suppressed.length; i++) {
+                        Throwable throwable = suppressed[i];
+                        if (i == suppressed.length - 1) {
+                            assertThat(
+                                throwable.getMessage(),
+                                equalTo((exceptionCount.get() - exceptionLimit) + " further exceptions were dropped")
+                            );
+                        } else {
+                            assertThat(throwable.getMessage(), equalTo("simulated"));
+                        }
+                    }
+                } else {
+                    assertEquals(exceptionCount.get() - 1, suppressed.length);
+                    for (Throwable throwable : suppressed) {
+                        assertThat(throwable.getMessage(), equalTo("simulated"));
+                    }
+                }
+            }
+
+            @Override
+            public String toString() {
+                return "test listener";
+            }
+        })) {
+            assertEquals("refCounting[test listener]", refs.toString());
+
+            var listener = refs.acquire();
+            assertThat(listener.toString(), containsString("refCounting[test listener]"));
+            if (randomBoolean()) {
+                listener.onResponse(null);
+            } else {
+                listener.onFailure(new ElasticsearchException("simulated"));
+                exceptionCount.incrementAndGet();
+            }
+
+            var reachChecker = new ReachabilityChecker();
+            var consumed = new AtomicBoolean();
+            var consumingListener = refs.acquire(reachChecker.register(new Consumer<String>() {
+                @Override
+                public void accept(String s) {
+                    assertEquals("test response", s);
+                    consumed.set(true);
+                }
+
+                @Override
+                public String toString() {
+                    return "test consumer";
+                }
+            }));
+            assertFalse(consumed.get());
+            assertThat(consumingListener.toString(), containsString("refCounting[test listener][test consumer]"));
+            if (randomBoolean()) {
+                consumingListener.onResponse("test response");
+                assertTrue(consumed.get());
+            } else {
+                consumingListener.onFailure(new ElasticsearchException("simulated"));
+                assertFalse(consumed.get());
+                exceptionCount.incrementAndGet();
+            }
+            reachChecker.ensureUnreachable();
+            assertThat(consumingListener.toString(), containsString("refCounting[test listener][null]"));
+
+            for (int i = 0; i < threads.length; i++) {
+                if (randomBoolean()) {
+                    async = true;
+                    var ref = refs.acquire();
+                    threads[i] = new Thread(() -> {
+                        try {
+                            assertTrue(startLatch.await(10, TimeUnit.SECONDS));
+                        } catch (InterruptedException e) {
+                            throw new AssertionError(e);
+                        }
+                        assertFalse(executed.get());
+                        if (randomBoolean()) {
+                            ref.onResponse(null);
+                        } else {
+                            exceptionCount.incrementAndGet();
+                            ref.onFailure(new ElasticsearchException("simulated"));
+                        }
+                    });
+                }
+            }
+
+            assertFalse(executed.get());
+        }
+
+        assertNotEquals(async, executed.get());
+
+        for (Thread thread : threads) {
+            if (thread != null) {
+                thread.start();
+            }
+        }
+
+        startLatch.countDown();
+
+        for (Thread thread : threads) {
+            if (thread != null) {
+                thread.join();
+            }
+        }
+
+        assertTrue(executed.get());
+    }
+
+    @SuppressWarnings("resource")
+    public void testNullCheck() {
+        expectThrows(NullPointerException.class, () -> new RefCountingListener(between(1, 10), null));
+    }
+
+    public void testValidation() {
+        final var callCount = new AtomicInteger();
+        final var refs = new RefCountingListener(Integer.MAX_VALUE, ActionListener.wrap(callCount::incrementAndGet));
+        refs.close();
+        assertEquals(1, callCount.get());
+
+        for (int i = between(1, 5); i > 0; i--) {
+            final ThrowingRunnable throwingRunnable;
+            final String expectedMessage;
+            if (randomBoolean()) {
+                throwingRunnable = refs::acquire;
+                expectedMessage = RefCountingRunnable.ALREADY_CLOSED_MESSAGE;
+            } else {
+                throwingRunnable = refs::close;
+                expectedMessage = "already closed";
+            }
+
+            assertEquals(expectedMessage, expectThrows(AssertionError.class, throwingRunnable).getMessage());
+            assertEquals(1, callCount.get());
+        }
+    }
+
+    public void testJavaDocExample() {
+        final var flag = new AtomicBoolean();
+        runExample(ActionListener.wrap(() -> assertTrue(flag.compareAndSet(false, true))));
+        assertTrue(flag.get());
+    }
+
+    private void runExample(ActionListener<Void> finalListener) {
+        final var collection = randomList(10, Object::new);
+        final var otherCollection = randomList(10, Object::new);
+        final var flag = randomBoolean();
+        @SuppressWarnings("UnnecessaryLocalVariable")
+        final var executorService = DIRECT_EXECUTOR_SERVICE;
+        final var results = new ArrayList<>();
+
+        try (var refs = new RefCountingListener(finalListener)) {
+            for (var item : collection) {
+                if (condition(item)) {
+                    runAsyncAction(item, refs.acquire(results::add));
+                }
+            }
+            if (flag) {
+                runOneOffAsyncAction(refs.acquire(results::add));
+                return;
+            }
+            for (var item : otherCollection) {
+                var itemRef = refs.acquire(); // delays completion while the background action is pending
+                executorService.execute(() -> {
+                    try {
+                        if (condition(item)) {
+                            runOtherAsyncAction(item, refs.acquire(results::add));
+                        }
+                    } finally {
+                        itemRef.onResponse(null);
+                    }
+                });
+            }
+        }
+    }
+
+    @SuppressWarnings("unused")
+    private boolean condition(Object item) {
+        return randomBoolean();
+    }
+
+    @SuppressWarnings("unused")
+    private void runAsyncAction(Object item, ActionListener<Void> listener) {
+        listener.onResponse(null);
+    }
+
+    @SuppressWarnings("unused")
+    private void runOtherAsyncAction(Object item, ActionListener<Void> listener) {
+        listener.onResponse(null);
+    }
+
+    private void runOneOffAsyncAction(ActionListener<Void> listener) {
+        listener.onResponse(null);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/support/RefCountingRunnableTests.java b/server/src/test/java/org/elasticsearch/action/support/RefCountingRunnableTests.java
new file mode 100644
index 000000000000..41e9e4044024
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/support/RefCountingRunnableTests.java
@@ -0,0 +1,234 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.support;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.elasticsearch.common.util.concurrent.EsExecutors.DIRECT_EXECUTOR_SERVICE;
+import static org.hamcrest.Matchers.containsString;
+
+public class RefCountingRunnableTests extends ESTestCase {
+
+    public void testBasicOperation() throws InterruptedException {
+        final var executed = new AtomicBoolean();
+        final var threads = new Thread[between(0, 3)];
+        boolean async = false;
+        final var startLatch = new CountDownLatch(1);
+
+        try (var refs = new RefCountingRunnable(new Runnable() {
+            @Override
+            public void run() {
+                assertTrue(executed.compareAndSet(false, true));
+            }
+
+            @Override
+            public String toString() {
+                return "test runnable";
+            }
+        })) {
+            assertEquals("refCounted[test runnable]", refs.toString());
+            try (var ref = refs.acquire()) {
+                assertEquals("refCounted[test runnable]", ref.toString());
+            }
+            var listener = refs.acquireListener();
+            assertThat(listener.toString(), containsString("release[refCounted[test runnable]]"));
+            listener.onResponse(null);
+
+            for (int i = 0; i < threads.length; i++) {
+                if (randomBoolean()) {
+                    async = true;
+                    var ref = refs.acquire();
+                    threads[i] = new Thread(() -> {
+                        try (var ignored = ref) {
+                            assertTrue(startLatch.await(10, TimeUnit.SECONDS));
+                            assertFalse(executed.get());
+                        } catch (InterruptedException e) {
+                            throw new AssertionError(e);
+                        }
+                    });
+                }
+            }
+
+            assertFalse(executed.get());
+        }
+
+        assertNotEquals(async, executed.get());
+
+        for (Thread thread : threads) {
+            if (thread != null) {
+                thread.start();
+            }
+        }
+
+        startLatch.countDown();
+
+        for (Thread thread : threads) {
+            if (thread != null) {
+                thread.join();
+            }
+        }
+
+        assertTrue(executed.get());
+    }
+
+    @SuppressWarnings("resource")
+    public void testNullCheck() {
+        expectThrows(NullPointerException.class, () -> new RefCountingRunnable(null));
+    }
+
+    public void testAsyncAcquire() throws InterruptedException {
+        final var completionLatch = new CountDownLatch(1);
+        final var executorService = EsExecutors.newScaling(
+            "test",
+            0,
+            between(1, 10),
+            10,
+            TimeUnit.SECONDS,
+            true,
+            EsExecutors.daemonThreadFactory("test"),
+            new ThreadContext(Settings.EMPTY)
+        );
+        final var asyncPermits = new Semaphore(between(0, 1000));
+
+        try (var refs = new RefCountingRunnable(() -> {
+            assertEquals(1, completionLatch.getCount());
+            completionLatch.countDown();
+        })) {
+            class AsyncAction extends AbstractRunnable {
+                private final Releasable ref;
+
+                AsyncAction(Releasable ref) {
+                    this.ref = ref;
+                }
+
+                @Override
+                protected void doRun() throws Exception {
+                    if (asyncPermits.tryAcquire()) {
+                        executorService.execute(new AsyncAction(refs.acquire()));
+                    }
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    assert e instanceof EsRejectedExecutionException esre && esre.isExecutorShutdown() : e;
+                }
+
+                @Override
+                public void onAfter() {
+                    ref.close();
+                }
+            }
+
+            for (int i = between(0, 5); i >= 0; i--) {
+                executorService.execute(new AsyncAction(refs.acquire()));
+            }
+
+            assertEquals(1, completionLatch.getCount());
+        }
+
+        if (randomBoolean()) {
+            assertTrue(completionLatch.await(10, TimeUnit.SECONDS));
+            assertFalse(asyncPermits.tryAcquire());
+        }
+
+        executorService.shutdown();
+        assertTrue(executorService.awaitTermination(10, TimeUnit.SECONDS));
+
+        assertTrue(completionLatch.await(10, TimeUnit.SECONDS));
+    }
+
+    public void testValidation() {
+        final var callCount = new AtomicInteger();
+        final var refs = new RefCountingRunnable(callCount::incrementAndGet);
+        refs.close();
+        assertEquals(1, callCount.get());
+
+        for (int i = between(1, 5); i > 0; i--) {
+            final ThrowingRunnable throwingRunnable;
+            final String expectedMessage;
+            if (randomBoolean()) {
+                throwingRunnable = randomBoolean() ? refs::acquire : refs::acquireListener;
+                expectedMessage = RefCountingRunnable.ALREADY_CLOSED_MESSAGE;
+            } else {
+                throwingRunnable = refs::close;
+                expectedMessage = "already closed";
+            }
+
+            assertEquals(expectedMessage, expectThrows(AssertionError.class, throwingRunnable).getMessage());
+            assertEquals(1, callCount.get());
+        }
+    }
+
+    public void testJavaDocExample() {
+        final var flag = new AtomicBoolean();
+        runExample(() -> assertTrue(flag.compareAndSet(false, true)));
+        assertTrue(flag.get());
+    }
+
+    private void runExample(Runnable finalRunnable) {
+        final var collection = randomList(10, Object::new);
+        final var otherCollection = randomList(10, Object::new);
+        final var flag = randomBoolean();
+        @SuppressWarnings("UnnecessaryLocalVariable")
+        final var executorService = DIRECT_EXECUTOR_SERVICE;
+
+        try (var refs = new RefCountingRunnable(finalRunnable)) {
+            for (var item : collection) {
+                if (condition(item)) {
+                    runAsyncAction(item, refs.acquire());
+                }
+            }
+            if (flag) {
+                runOneOffAsyncAction(refs.acquire());
+                return;
+            }
+            for (var item : otherCollection) {
+                var itemRef = refs.acquire(); // delays completion while the background action is pending
+                executorService.execute(() -> {
+                    try (var ignored = itemRef) {
+                        if (condition(item)) {
+                            runOtherAsyncAction(item, refs.acquire());
+                        }
+                    }
+                });
+            }
+        }
+    }
+
+    @SuppressWarnings("unused")
+    private boolean condition(Object item) {
+        return randomBoolean();
+    }
+
+    @SuppressWarnings("unused")
+    private void runAsyncAction(Object item, Releasable releasable) {
+        releasable.close();
+    }
+
+    @SuppressWarnings("unused")
+    private void runOtherAsyncAction(Object item, Releasable releasable) {
+        releasable.close();
+    }
+
+    private void runOneOffAsyncAction(Releasable releasable) {
+        releasable.close();
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/support/ThreadedActionListenerTests.java b/server/src/test/java/org/elasticsearch/action/support/ThreadedActionListenerTests.java
index 88af6e68e50e..d5eb188d0138 100644
--- a/server/src/test/java/org/elasticsearch/action/support/ThreadedActionListenerTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/ThreadedActionListenerTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.FixedExecutorBuilder;
@@ -47,11 +48,9 @@ public void testRejectionHandling() throws InterruptedException {
                 for (int i = 0; i < listenerCount; i++) {
                     final var pool = randomFrom(pools);
                     final var listener = new ThreadedActionListener<Void>(
-                        logger,
-                        threadPool,
-                        pool,
-                        ActionListener.wrap(countdownLatch::countDown),
-                        (pool.equals("fixed-bounded-queue") || pool.startsWith("scaling")) && rarely()
+                        threadPool.executor(pool),
+                        (pool.equals("fixed-bounded-queue") || pool.startsWith("scaling")) && rarely(),
+                        ActionListener.wrap(countdownLatch::countDown)
                     );
                     synchronized (closeFlag) {
                         if (closeFlag.get() && shutdownUnsafePools.contains(pool)) {
@@ -76,4 +75,30 @@ public void testRejectionHandling() throws InterruptedException {
         assertTrue(countdownLatch.await(10, TimeUnit.SECONDS));
     }
 
+    public void testToString() {
+        var deterministicTaskQueue = new DeterministicTaskQueue();
+
+        assertEquals(
+            "ThreadedActionListener[DeterministicTaskQueue/forkingExecutor/NoopActionListener]",
+            new ThreadedActionListener<Void>(deterministicTaskQueue.getThreadPool().generic(), randomBoolean(), ActionListener.noop())
+                .toString()
+        );
+
+        assertEquals(
+            "ThreadedActionListener[DeterministicTaskQueue/forkingExecutor/NoopActionListener]/onResponse",
+            PlainActionFuture.get(future -> new ThreadedActionListener<Void>(deterministicTaskQueue.getThreadPool(s -> {
+                future.onResponse(s.toString());
+                return s;
+            }).generic(), randomBoolean(), ActionListener.noop()).onResponse(null))
+        );
+
+        assertEquals(
+            "ThreadedActionListener[DeterministicTaskQueue/forkingExecutor/NoopActionListener]/onFailure",
+            PlainActionFuture.get(future -> new ThreadedActionListener<Void>(deterministicTaskQueue.getThreadPool(s -> {
+                future.onResponse(s.toString());
+                return s;
+            }).generic(), randomBoolean(), ActionListener.noop()).onFailure(new ElasticsearchException("test")))
+        );
+    }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
index 44e4e80652e3..7e76190e60a0 100644
--- a/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
@@ -37,7 +37,9 @@
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -50,6 +52,7 @@
 import org.elasticsearch.tasks.TaskCancelHelper;
 import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ReachabilityChecker;
 import org.elasticsearch.test.transport.CapturingTransport;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -63,6 +66,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -71,6 +75,7 @@
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.IntStream;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
@@ -79,6 +84,7 @@
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.Matchers.anEmptyMap;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.object.HasToString.hasToString;
 
 public class TransportBroadcastByNodeActionTests extends ESTestCase {
@@ -91,9 +97,9 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase {
     private CapturingTransport transport;
 
     private TestTransportBroadcastByNodeAction action;
+    private TransportService transportService;
 
     public static class Request extends BroadcastRequest<Request> {
-
         public Request(StreamInput in) throws IOException {
             super(in);
         }
@@ -113,46 +119,42 @@ public Response(int totalShards, int successfulShards, int failedShards, List<De
         }
     }
 
-    class TestTransportBroadcastByNodeAction extends TransportBroadcastByNodeAction<
-        Request,
-        Response,
-        TransportBroadcastByNodeAction.EmptyResult> {
+    // empty per-shard result, but not a singleton so we can check each instance is released on cancellation
+    public static class ShardResult implements Writeable {
+        public ShardResult() {}
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {}
+    }
+
+    class TestTransportBroadcastByNodeAction extends TransportBroadcastByNodeAction<Request, Response, ShardResult> {
         private final Map<ShardRouting, Object> shards = new HashMap<>();
 
-        TestTransportBroadcastByNodeAction(
-            TransportService transportService,
-            ActionFilters actionFilters,
-            IndexNameExpressionResolver indexNameExpressionResolver,
-            Writeable.Reader<Request> request,
-            String executor
-        ) {
+        TestTransportBroadcastByNodeAction(String actionName) {
             super(
-                "indices:admin/test",
-                TransportBroadcastByNodeActionTests.this.clusterService,
+                actionName,
+                clusterService,
                 transportService,
-                actionFilters,
-                indexNameExpressionResolver,
-                request,
-                executor
+                new ActionFilters(Set.of()),
+                new MyResolver(),
+                Request::new,
+                ThreadPool.Names.SAME
             );
         }
 
         @Override
-        protected EmptyResult readShardResult(StreamInput in) throws IOException {
-            return EmptyResult.readEmptyResultFrom(in);
+        protected ShardResult readShardResult(StreamInput in) {
+            return new ShardResult();
         }
 
         @Override
-        protected Response newResponse(
-            Request request,
-            int totalShards,
-            int successfulShards,
-            int failedShards,
-            List<EmptyResult> emptyResults,
-            List<DefaultShardOperationFailedException> shardFailures,
-            ClusterState clusterState
-        ) {
-            return new Response(totalShards, successfulShards, failedShards, shardFailures);
+        protected ResponseFactory<Response, ShardResult> getResponseFactory(Request request, ClusterState clusterState) {
+            return (totalShards, successfulShards, failedShards, emptyResults, shardFailures) -> new Response(
+                totalShards,
+                successfulShards,
+                failedShards,
+                shardFailures
+            );
         }
 
         @Override
@@ -161,11 +163,11 @@ protected Request readRequestFrom(StreamInput in) throws IOException {
         }
 
         @Override
-        protected void shardOperation(Request request, ShardRouting shardRouting, Task task, ActionListener<EmptyResult> listener) {
+        protected void shardOperation(Request request, ShardRouting shardRouting, Task task, ActionListener<ShardResult> listener) {
             ActionListener.completeWith(listener, () -> {
                 if (rarely()) {
                     shards.put(shardRouting, Boolean.TRUE);
-                    return EmptyResult.INSTANCE;
+                    return new ShardResult();
                 } else {
                     ElasticsearchException e = new ElasticsearchException("operation failed");
                     shards.put(shardRouting, e);
@@ -192,6 +194,7 @@ protected ClusterBlockException checkRequestBlock(ClusterState state, Request re
         public Map<ShardRouting, Object> getResults() {
             return shards;
         }
+
     }
 
     static class MyResolver extends IndexNameExpressionResolver {
@@ -215,7 +218,7 @@ public void setUp() throws Exception {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
-        TransportService transportService = transport.createTransportService(
+        transportService = transport.createTransportService(
             clusterService.getSettings(),
             THREAD_POOL,
             TransportService.NOOP_TRANSPORT_INTERCEPTOR,
@@ -225,14 +228,8 @@ public void setUp() throws Exception {
         );
         transportService.start();
         transportService.acceptIncomingRequests();
-        setClusterState(clusterService, TEST_INDEX);
-        action = new TestTransportBroadcastByNodeAction(
-            transportService,
-            new ActionFilters(new HashSet<>()),
-            new MyResolver(),
-            Request::new,
-            ThreadPool.Names.SAME
-        );
+        setClusterState(clusterService);
+        action = new TestTransportBroadcastByNodeAction("indices:admin/test");
     }
 
     @After
@@ -241,10 +238,10 @@ public void tearDown() throws Exception {
         clusterService.close();
     }
 
-    void setClusterState(ClusterService clusterService, String index) {
+    void setClusterState(ClusterService clusterService) {
         int numberOfNodes = randomIntBetween(3, 5);
         DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
-        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(new Index(index, "_na_"));
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(new Index(TEST_INDEX, "_na_"));
 
         int shardIndex = -1;
         int totalIndexShards = 0;
@@ -254,9 +251,9 @@ void setClusterState(ClusterService clusterService, String index) {
             int numberOfShards = randomIntBetween(1, 10);
             totalIndexShards += numberOfShards;
             for (int j = 0; j < numberOfShards; j++) {
-                final ShardId shardId = new ShardId(index, "_na_", ++shardIndex);
+                final ShardId shardId = new ShardId(TEST_INDEX, "_na_", ++shardIndex);
                 ShardRouting shard = TestShardRouting.newShardRouting(
-                    index,
+                    TEST_INDEX,
                     shardId.getId(),
                     node.getId(),
                     true,
@@ -271,7 +268,7 @@ void setClusterState(ClusterService clusterService, String index) {
         discoBuilder.masterNodeId(newNode(numberOfNodes - 1).getId());
         ClusterState.Builder stateBuilder = ClusterState.builder(new ClusterName(TEST_CLUSTER));
         stateBuilder.nodes(discoBuilder);
-        final IndexMetadata.Builder indexMetadata = IndexMetadata.builder(index)
+        final IndexMetadata.Builder indexMetadata = IndexMetadata.builder(TEST_INDEX)
             .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT))
             .numberOfReplicas(0)
             .numberOfShards(totalIndexShards);
@@ -294,14 +291,14 @@ public static void destroyThreadPool() {
     }
 
     public void testGlobalBlock() {
-        Request request = new Request(new String[] { TEST_INDEX });
+        Request request = new Request(TEST_INDEX);
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
 
         ClusterBlocks.Builder block = ClusterBlocks.builder()
             .addGlobalBlock(new ClusterBlock(1, "test-block", false, true, false, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
         setState(clusterService, ClusterState.builder(clusterService.state()).blocks(block));
         try {
-            action.new AsyncAction(null, request, listener).start();
+            action.doExecute(null, request, listener);
             fail("expected ClusterBlockException");
         } catch (ClusterBlockException expected) {
             assertEquals("blocked by: [SERVICE_UNAVAILABLE/1/test-block];", expected.getMessage());
@@ -309,7 +306,7 @@ public void testGlobalBlock() {
     }
 
     public void testRequestBlock() {
-        Request request = new Request(new String[] { TEST_INDEX });
+        Request request = new Request(TEST_INDEX);
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
 
         ClusterBlocks.Builder block = ClusterBlocks.builder()
@@ -319,7 +316,7 @@ public void testRequestBlock() {
             );
         setState(clusterService, ClusterState.builder(clusterService.state()).blocks(block));
         try {
-            action.new AsyncAction(null, request, listener).start();
+            action.doExecute(null, request, listener);
             fail("expected ClusterBlockException");
         } catch (ClusterBlockException expected) {
             assertEquals("index [" + TEST_INDEX + "] blocked by: [SERVICE_UNAVAILABLE/1/test-block];", expected.getMessage());
@@ -327,10 +324,10 @@ public void testRequestBlock() {
     }
 
     public void testOneRequestIsSentToEachNodeHoldingAShard() {
-        Request request = new Request(new String[] { TEST_INDEX });
+        Request request = new Request(TEST_INDEX);
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
 
-        action.new AsyncAction(null, request, listener).start();
+        action.doExecute(null, request, listener);
         Map<String, List<CapturingTransport.CapturedRequest>> capturedRequests = transport.getCapturedRequestsByTargetNodeAndClear();
 
         ShardsIterator shardIt = clusterService.state().routingTable().allShards(new String[] { TEST_INDEX });
@@ -359,16 +356,15 @@ public void testNoShardOperationsExecutedIfTaskCancelled() throws Exception {
                 shards.add(shard);
             }
         }
-        final TransportBroadcastByNodeAction<
-            Request,
-            Response,
-            TransportBroadcastByNodeAction.EmptyResult>.BroadcastByNodeTransportRequestHandler handler =
-                action.new BroadcastByNodeTransportRequestHandler();
+        final TransportBroadcastByNodeAction<Request, Response, ShardResult>.BroadcastByNodeTransportRequestHandler handler =
+            action.new BroadcastByNodeTransportRequestHandler();
 
         final PlainActionFuture<TransportResponse> future = PlainActionFuture.newFuture();
         TestTransportChannel channel = new TestTransportChannel(future);
 
-        handler.messageReceived(action.new NodeRequest(nodeId, new Request(), new ArrayList<>(shards)), channel, cancelledTask());
+        final CancellableTask cancellableTask = new CancellableTask(randomLong(), "transport", "action", "", null, emptyMap());
+        TaskCancelHelper.cancel(cancellableTask, "simulated");
+        handler.messageReceived(action.new NodeRequest(new Request(), new ArrayList<>(shards), nodeId), channel, cancellableTask);
         expectThrows(TaskCancelledException.class, future::actionGet);
 
         assertThat(action.getResults(), anEmptyMap());
@@ -379,7 +375,7 @@ public void testNoShardOperationsExecutedIfTaskCancelled() throws Exception {
     // that node will not be in the local cluster state on any node that has detected the master as failing
     // in this case, such a shard should be treated as unassigned
     public void testRequestsAreNotSentToFailedMaster() {
-        Request request = new Request(new String[] { TEST_INDEX });
+        Request request = new Request(TEST_INDEX);
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
 
         DiscoveryNode masterNode = clusterService.state().nodes().getMasterNode();
@@ -388,7 +384,7 @@ public void testRequestsAreNotSentToFailedMaster() {
 
         setState(clusterService, ClusterState.builder(clusterService.state()).nodes(builder));
 
-        action.new AsyncAction(null, request, listener).start();
+        action.doExecute(null, request, listener);
 
         Map<String, List<CapturingTransport.CapturedRequest>> capturedRequests = transport.getCapturedRequestsByTargetNodeAndClear();
 
@@ -421,16 +417,13 @@ public void testOperationExecution() throws Exception {
                 shards.add(shard);
             }
         }
-        final TransportBroadcastByNodeAction<
-            Request,
-            Response,
-            TransportBroadcastByNodeAction.EmptyResult>.BroadcastByNodeTransportRequestHandler handler =
-                action.new BroadcastByNodeTransportRequestHandler();
+        final TransportBroadcastByNodeAction<Request, Response, ShardResult>.BroadcastByNodeTransportRequestHandler handler =
+            action.new BroadcastByNodeTransportRequestHandler();
 
         final PlainActionFuture<TransportResponse> future = PlainActionFuture.newFuture();
         TestTransportChannel channel = new TestTransportChannel(future);
 
-        handler.messageReceived(action.new NodeRequest(nodeId, new Request(), new ArrayList<>(shards)), channel, null);
+        handler.messageReceived(action.new NodeRequest(new Request(), new ArrayList<>(shards), nodeId), channel, null);
 
         // check the operation was executed only on the expected shards
         assertEquals(shards, action.getResults().keySet());
@@ -452,13 +445,12 @@ public void testOperationExecution() throws Exception {
                 failedShards++;
             }
         }
-
         // check the operation results
         assertEquals("successful shards", successfulShards, nodeResponse.getSuccessfulShards());
         assertEquals("total shards", action.getResults().size(), nodeResponse.getTotalShards());
-        assertEquals("failed shards", failedShards, nodeResponse.getExceptions().size());
         @SuppressWarnings("unchecked")
         List<BroadcastShardOperationFailedException> exceptions = nodeResponse.getExceptions();
+        assertEquals("exceptions count", failedShards, exceptions.size());
         for (BroadcastShardOperationFailedException exception : exceptions) {
             assertThat(exception.getMessage(), is("operation indices:admin/test failed"));
             assertThat(exception.getCause(), hasToString(containsString("operation failed")));
@@ -466,7 +458,7 @@ public void testOperationExecution() throws Exception {
     }
 
     public void testResultAggregation() throws ExecutionException, InterruptedException {
-        Request request = new Request(new String[] { TEST_INDEX });
+        Request request = new Request(TEST_INDEX);
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
 
         // simulate removing the master
@@ -481,7 +473,7 @@ public void testResultAggregation() throws ExecutionException, InterruptedExcept
             setState(clusterService, ClusterState.builder(clusterService.state()).nodes(builder));
         }
 
-        action.new AsyncAction(null, request, listener).start();
+        action.doExecute(null, request, listener);
         Map<String, List<CapturingTransport.CapturedRequest>> capturedRequests = transport.getCapturedRequestsByTargetNodeAndClear();
 
         ShardsIterator shardIt = clusterService.state().getRoutingTable().allShards(new String[] { TEST_INDEX });
@@ -506,7 +498,7 @@ public void testResultAggregation() throws ExecutionException, InterruptedExcept
                 transport.handleRemoteError(requestId, new Exception());
             } else {
                 List<ShardRouting> shards = map.get(entry.getKey());
-                List<TransportBroadcastByNodeAction.EmptyResult> shardResults = new ArrayList<>();
+                List<ShardResult> shardResults = new ArrayList<>();
                 for (ShardRouting shard : shards) {
                     totalShards++;
                     if (rarely()) {
@@ -514,12 +506,13 @@ public void testResultAggregation() throws ExecutionException, InterruptedExcept
                         totalFailedShards++;
                         exceptions.add(new BroadcastShardOperationFailedException(shard.shardId(), "operation indices:admin/test failed"));
                     } else {
-                        shardResults.add(TransportBroadcastByNodeAction.EmptyResult.INSTANCE);
+                        shardResults.add(new ShardResult());
                     }
                 }
                 totalSuccessfulShards += shardResults.size();
-                TransportBroadcastByNodeAction<Request, Response, TransportBroadcastByNodeAction.EmptyResult>.NodeResponse nodeResponse =
-                    action.new NodeResponse(entry.getKey(), shards.size(), shardResults, exceptions);
+                TransportBroadcastByNodeAction<Request, Response, ShardResult>.NodeResponse nodeResponse = action.new NodeResponse(
+                    entry.getKey(), shards.size(), shardResults, exceptions
+                );
                 transport.handleResponse(requestId, nodeResponse);
             }
         }
@@ -534,33 +527,161 @@ public void testResultAggregation() throws ExecutionException, InterruptedExcept
         assertEquals("accumulated exceptions", totalFailedShards, response.getShardFailures().length);
     }
 
-    public void testNoResultAggregationIfTaskCancelled() {
-        Request request = new Request(new String[] { TEST_INDEX });
-        PlainActionFuture<Response> listener = new PlainActionFuture<>();
-        final CancellableTask task = new CancellableTask(randomLong(), "transport", "action", "", null, emptyMap());
-        TransportBroadcastByNodeAction<Request, Response, TransportBroadcastByNodeAction.EmptyResult>.AsyncAction asyncAction =
-            action.new AsyncAction(task, request, listener);
-        asyncAction.start();
-        Map<String, List<CapturingTransport.CapturedRequest>> capturedRequests = transport.getCapturedRequestsByTargetNodeAndClear();
-        int cancelAt = randomIntBetween(0, Math.max(0, capturedRequests.size() - 2));
-        int i = 0;
-        for (Map.Entry<String, List<CapturingTransport.CapturedRequest>> entry : capturedRequests.entrySet()) {
-            if (cancelAt == i) {
-                TaskCancelHelper.cancel(task, "simulated");
+    public void testResponsesReleasedOnCancellation() {
+        final CancellableTask cancellableTask = new CancellableTask(randomLong(), "transport", "action", "", null, emptyMap());
+        final PlainActionFuture<Response> listener = new PlainActionFuture<>();
+        action.execute(cancellableTask, new Request(TEST_INDEX), listener);
+
+        final List<CapturingTransport.CapturedRequest> capturedRequests = new ArrayList<>(
+            Arrays.asList(transport.getCapturedRequestsAndClear())
+        );
+        Randomness.shuffle(capturedRequests);
+
+        final ReachabilityChecker reachabilityChecker = new ReachabilityChecker();
+        final Runnable nextRequestProcessor = () -> {
+            final var capturedRequest = capturedRequests.remove(0);
+            if (randomBoolean()) {
+                // transport.handleResponse may de/serialize the response, releasing it early, so send the response straight to the handler
+                transport.getTransportResponseHandler(capturedRequest.requestId())
+                    .handleResponse(
+                        action.new NodeResponse(
+                            capturedRequest.node().getId(), 1, List.of(reachabilityChecker.register(new ShardResult())), List.of()
+                        )
+                    );
+            } else {
+                // handleRemoteError may de/serialize the exception, releasing it early, so just use handleLocalError
+                transport.handleLocalError(
+                    capturedRequest.requestId(),
+                    reachabilityChecker.register(new ElasticsearchException("simulated"))
+                );
             }
-            transport.handleRemoteError(entry.getValue().get(0).requestId(), new ElasticsearchException("simulated"));
-            i++;
+        };
+
+        assertThat(capturedRequests.size(), greaterThan(2));
+        final var responsesBeforeCancellation = between(1, capturedRequests.size() - 2);
+        for (int i = 0; i < responsesBeforeCancellation; i++) {
+            nextRequestProcessor.run();
+        }
+
+        reachabilityChecker.checkReachable();
+        TaskCancelHelper.cancel(cancellableTask, "simulated");
+
+        // responses captured before cancellation are now unreachable
+        reachabilityChecker.ensureUnreachable();
+
+        while (capturedRequests.size() > 0) {
+            // a response sent after cancellation is dropped immediately
+            assertFalse(listener.isDone());
+            nextRequestProcessor.run();
+            reachabilityChecker.ensureUnreachable();
         }
 
-        assertTrue(listener.isDone());
-        assertTrue(asyncAction.getNodeResponseTracker().responsesDiscarded());
-        expectThrows(ExecutionException.class, TaskCancelledException.class, listener::get);
+        expectThrows(TaskCancelledException.class, () -> listener.actionGet(10, TimeUnit.SECONDS));
     }
 
-    private static Task cancelledTask() {
+    public void testShardLevelOperationsStopOnCancellation() throws Exception {
+        action = new TestTransportBroadcastByNodeAction("indices:admin/shard_level_test") {
+            int expectedShardId;
+
+            @Override
+            protected void shardOperation(Request request, ShardRouting shardRouting, Task task, ActionListener<ShardResult> listener) {
+                // this test runs a node-level operation on three shards, cancelling the task some time during the execution on the second
+                if (task instanceof CancellableTask cancellableTask) {
+                    assertEquals(expectedShardId++, shardRouting.shardId().id());
+                    switch (shardRouting.shardId().id()) {
+                        case 0 -> {
+                            assertFalse(cancellableTask.isCancelled());
+                            listener.onResponse(new ShardResult());
+                        }
+                        case 1 -> {
+                            assertFalse(cancellableTask.isCancelled());
+                            TaskCancelHelper.cancel(cancellableTask, "simulated");
+                            if (randomBoolean()) {
+                                listener.onResponse(new ShardResult());
+                            } else {
+                                assertTrue(cancellableTask.notifyIfCancelled(listener));
+                            }
+                        }
+                        default -> fail("unexpected shard execution: " + shardRouting);
+                    }
+                } else {
+                    fail("task was not cancellable");
+                }
+            }
+        };
+
+        final PlainActionFuture<TransportResponse> nodeResponseFuture = new PlainActionFuture<>();
+
+        action.new BroadcastByNodeTransportRequestHandler().messageReceived(
+            action.new NodeRequest(
+                new Request(), IntStream.range(0, 3)
+                    .mapToObj(shardId -> TestShardRouting.newShardRouting(TEST_INDEX, shardId, "node-id", true, ShardRoutingState.STARTED))
+                    .toList(), "node-id"
+            ),
+            new TestTransportChannel(nodeResponseFuture),
+            new CancellableTask(randomLong(), "transport", "action", "", null, emptyMap())
+        );
+
+        assertTrue(nodeResponseFuture.isDone());
+        assertEquals(
+            "task cancelled [simulated]",
+            expectThrows(
+                java.util.concurrent.ExecutionException.class,
+                org.elasticsearch.tasks.TaskCancelledException.class,
+                nodeResponseFuture::get
+            ).getMessage()
+        );
+    }
+
+    public void testShardResultsReleasedOnCancellation() throws Exception {
+        final var listeners = new ArrayList<ActionListener<ShardResult>>();
+
+        action = new TestTransportBroadcastByNodeAction("indices:admin/shard_level_gc_test") {
+            @Override
+            protected void shardOperation(Request request, ShardRouting shardRouting, Task task, ActionListener<ShardResult> listener) {
+                listeners.add(listener);
+            }
+        };
+
+        final PlainActionFuture<TransportResponse> nodeResponseFuture = new PlainActionFuture<>();
         final CancellableTask task = new CancellableTask(randomLong(), "transport", "action", "", null, emptyMap());
+
+        action.new BroadcastByNodeTransportRequestHandler().messageReceived(
+            action.new NodeRequest(
+                new Request(), IntStream.range(0, 3)
+                    .mapToObj(shardId -> TestShardRouting.newShardRouting(TEST_INDEX, shardId, "node-id", true, ShardRoutingState.STARTED))
+                    .toList(), "node-id"
+            ),
+            new TestTransportChannel(nodeResponseFuture),
+            task
+        );
+
+        assertEquals(3, listeners.size());
+
+        final var reachabilityChecker = new ReachabilityChecker();
+        listeners.get(0).onResponse(reachabilityChecker.register(new ShardResult()));
+        reachabilityChecker.checkReachable();
+
         TaskCancelHelper.cancel(task, "simulated");
-        return task;
+        reachabilityChecker.ensureUnreachable();
+
+        listeners.get(1).onResponse(reachabilityChecker.register(new ShardResult()));
+        reachabilityChecker.ensureUnreachable();
+
+        assertFalse(nodeResponseFuture.isDone());
+
+        listeners.get(2).onResponse(reachabilityChecker.register(new ShardResult()));
+        reachabilityChecker.ensureUnreachable();
+
+        assertTrue(nodeResponseFuture.isDone());
+        assertEquals(
+            "task cancelled [simulated]",
+            expectThrows(
+                java.util.concurrent.ExecutionException.class,
+                org.elasticsearch.tasks.TaskCancelledException.class,
+                nodeResponseFuture::get
+            ).getMessage()
+        );
     }
 
 }
diff --git a/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
index d904310d13ff..04e3220e7a20 100644
--- a/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.action.support.ActionTestUtils;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.action.support.ThreadedActionListener;
 import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
@@ -243,12 +242,6 @@ class Action extends TransportMasterNodeAction<Request, Response> {
             );
         }
 
-        @Override
-        protected void doExecute(Task task, final Request request, ActionListener<Response> listener) {
-            // remove unneeded threading by wrapping listener with SAME to prevent super.doExecute from wrapping it with LISTENER
-            super.doExecute(task, request, new ThreadedActionListener<>(logger, threadPool, ThreadPool.Names.SAME, listener, false));
-        }
-
         @Override
         protected void masterOperation(Task task, Request request, ClusterState state, ActionListener<Response> listener) throws Exception {
             listener.onResponse(new Response()); // default implementation, overridden in specific tests
diff --git a/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java b/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
index 8c9af6be2459..2d8a7bce9e72 100644
--- a/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
@@ -8,10 +8,10 @@
 
 package org.elasticsearch.action.support.nodes;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.NodeResponseTracker;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeActionTests;
 import org.elasticsearch.cluster.ClusterName;
@@ -20,6 +20,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -28,11 +29,13 @@
 import org.elasticsearch.tasks.TaskCancelHelper;
 import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ReachabilityChecker;
 import org.elasticsearch.test.transport.CapturingTransport;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportService;
+import org.hamcrest.Matchers;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -40,19 +43,19 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Supplier;
 
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
 import static org.elasticsearch.test.ClusterServiceUtils.setState;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.mockito.Mockito.mock;
 
 public class TransportNodesActionTests extends ESTestCase {
@@ -63,11 +66,10 @@ public class TransportNodesActionTests extends ESTestCase {
     private CapturingTransport transport;
     private TransportService transportService;
 
-    public void testRequestIsSentToEachNode() throws Exception {
+    public void testRequestIsSentToEachNode() {
         TransportNodesAction<TestNodesRequest, TestNodesResponse, TestNodeRequest, TestNodeResponse> action = getTestTransportNodesAction();
         TestNodesRequest request = new TestNodesRequest();
-        PlainActionFuture<TestNodesResponse> listener = new PlainActionFuture<>();
-        action.new AsyncAction(null, request, listener).start();
+        action.execute(null, request, new PlainActionFuture<>());
         Map<String, List<CapturingTransport.CapturedRequest>> capturedRequests = transport.getCapturedRequestsByTargetNodeAndClear();
         int numNodes = clusterService.state().getNodes().getSize();
         // check a request was sent to the right number of nodes
@@ -79,7 +81,7 @@ public void testNodesSelectors() {
         int numSelectors = randomIntBetween(1, 5);
         Set<String> nodeSelectors = new HashSet<>();
         for (int i = 0; i < numSelectors; i++) {
-            nodeSelectors.add(randomFrom(NodeSelector.values()).selector);
+            nodeSelectors.add(randomFrom("_local", "_master", "master:true", "data:true", "attr:value"));
         }
         int numNodeIds = randomIntBetween(0, 3);
         String[] nodeIds = clusterService.state().nodes().getNodes().keySet().toArray(new String[0]);
@@ -87,57 +89,18 @@ public void testNodesSelectors() {
             String nodeId = randomFrom(nodeIds);
             nodeSelectors.add(nodeId);
         }
-        String[] finalNodesIds = nodeSelectors.toArray(new String[nodeSelectors.size()]);
+        String[] finalNodesIds = nodeSelectors.toArray(String[]::new);
         TestNodesRequest request = new TestNodesRequest(finalNodesIds);
-        action.new AsyncAction(null, request, new PlainActionFuture<>()).start();
+        action.execute(null, request, new PlainActionFuture<>());
         Map<String, List<CapturingTransport.CapturedRequest>> capturedRequests = transport.getCapturedRequestsByTargetNodeAndClear();
         assertEquals(clusterService.state().nodes().resolveNodes(finalNodesIds).length, capturedRequests.size());
     }
 
-    public void testNewResponseNullArray() throws Exception {
-        TransportNodesAction<TestNodesRequest, TestNodesResponse, TestNodeRequest, TestNodeResponse> action = getTestTransportNodesAction();
-        final PlainActionFuture<TestNodesResponse> future = new PlainActionFuture<>();
-        action.newResponse(new Task(1, "test", "test", "", null, emptyMap()), new TestNodesRequest(), null, future);
-        expectThrows(NullPointerException.class, future::actionGet);
-    }
-
-    public void testNewResponse() throws Exception {
-        TestTransportNodesAction action = getTestTransportNodesAction();
-        TestNodesRequest request = new TestNodesRequest();
-        List<TestNodeResponse> expectedNodeResponses = mockList(TestNodeResponse::new, randomIntBetween(0, 2));
-        expectedNodeResponses.add(new TestNodeResponse());
-        List<FailedNodeException> failures = mockList(
-            () -> new FailedNodeException(
-                randomAlphaOfLength(8),
-                randomAlphaOfLength(8),
-                new IllegalStateException(randomAlphaOfLength(8))
-            ),
-            randomIntBetween(0, 2)
-        );
-
-        List<Object> allResponses = new ArrayList<>(expectedNodeResponses);
-        allResponses.addAll(failures);
-
-        Collections.shuffle(allResponses, random());
-
-        NodeResponseTracker nodeResponseCollector = new NodeResponseTracker(allResponses);
-
-        final PlainActionFuture<TestNodesResponse> future = new PlainActionFuture<>();
-        action.newResponse(new Task(1, "test", "test", "", null, emptyMap()), request, nodeResponseCollector, future);
-        TestNodesResponse response = future.actionGet();
-
-        assertSame(request, response.request);
-        // note: I shuffled the overall list, so it's not possible to guarantee that it's in the right order
-        assertTrue(expectedNodeResponses.containsAll(response.getNodes()));
-        assertTrue(failures.containsAll(response.failures()));
-    }
-
-    public void testCustomResolving() throws Exception {
+    public void testCustomResolving() {
         TransportNodesAction<TestNodesRequest, TestNodesResponse, TestNodeRequest, TestNodeResponse> action =
             getDataNodesOnlyTransportNodesAction(transportService);
         TestNodesRequest request = new TestNodesRequest(randomBoolean() ? null : generateRandomStringArray(10, 5, false, true));
-        PlainActionFuture<TestNodesResponse> listener = new PlainActionFuture<>();
-        action.new AsyncAction(null, request, listener).start();
+        action.execute(null, request, new PlainActionFuture<>());
         Map<String, List<CapturingTransport.CapturedRequest>> capturedRequests = transport.getCapturedRequestsByTargetNodeAndClear();
         // check requests were only sent to data nodes
         for (String nodeTarget : capturedRequests.keySet()) {
@@ -146,61 +109,98 @@ public void testCustomResolving() throws Exception {
         assertEquals(clusterService.state().nodes().getDataNodes().size(), capturedRequests.size());
     }
 
-    public void testTaskCancellation() {
-        TransportNodesAction<TestNodesRequest, TestNodesResponse, TestNodeRequest, TestNodeResponse> action = getTestTransportNodesAction();
-        List<String> nodeIds = new ArrayList<>();
-        for (DiscoveryNode node : clusterService.state().nodes()) {
-            nodeIds.add(node.getId());
-        }
+    public void testResponseAggregation() {
+        final TestTransportNodesAction action = getTestTransportNodesAction();
 
-        TestNodesRequest request = new TestNodesRequest(nodeIds.toArray(new String[0]));
-        PlainActionFuture<TestNodesResponse> listener = new PlainActionFuture<>();
-        CancellableTask cancellableTask = new CancellableTask(randomLong(), "transport", "action", "", null, emptyMap());
-        TransportNodesAction<TestNodesRequest, TestNodesResponse, TestNodeRequest, TestNodeResponse>.AsyncAction asyncAction =
-            action.new AsyncAction(cancellableTask, request, listener);
-        asyncAction.start();
-        Map<String, List<CapturingTransport.CapturedRequest>> capturedRequests = transport.getCapturedRequestsByTargetNodeAndClear();
-        int cancelAt = randomIntBetween(0, Math.max(0, capturedRequests.values().size() - 2));
-        int requestCount = 0;
-        for (List<CapturingTransport.CapturedRequest> requests : capturedRequests.values()) {
-            if (requestCount == cancelAt) {
-                TaskCancelHelper.cancel(cancellableTask, "simulated");
-            }
-            for (CapturingTransport.CapturedRequest capturedRequest : requests) {
+        final PlainActionFuture<TestNodesResponse> listener = new PlainActionFuture<>();
+        action.execute(null, new TestNodesRequest(), listener);
+        assertFalse(listener.isDone());
+
+        final Set<String> failedNodeIds = new HashSet<>();
+        final Set<DiscoveryNode> successfulNodes = new HashSet<>();
+
+        for (CapturingTransport.CapturedRequest capturedRequest : transport.getCapturedRequestsAndClear()) {
+            if (randomBoolean()) {
+                successfulNodes.add(capturedRequest.node());
+                transport.handleResponse(capturedRequest.requestId(), new TestNodeResponse(capturedRequest.node()));
+            } else {
+                failedNodeIds.add(capturedRequest.node().getId());
                 if (randomBoolean()) {
-                    transport.handleResponse(capturedRequest.requestId(), new TestNodeResponse(capturedRequest.node()));
+                    transport.handleRemoteError(capturedRequest.requestId(), new ElasticsearchException("simulated"));
                 } else {
-                    transport.handleRemoteError(capturedRequest.requestId(), new TaskCancelledException("simulated"));
+                    transport.handleLocalError(capturedRequest.requestId(), new ElasticsearchException("simulated"));
                 }
             }
-            requestCount++;
         }
 
-        assertTrue(listener.isDone());
-        assertTrue(asyncAction.getNodeResponseTracker().responsesDiscarded());
-        expectThrows(ExecutionException.class, TaskCancelledException.class, listener::get);
-    }
+        TestNodesResponse response = listener.actionGet(10, TimeUnit.SECONDS);
 
-    private <T> List<T> mockList(Supplier<T> supplier, int size) {
-        List<T> failures = new ArrayList<>(size);
-        for (int i = 0; i < size; ++i) {
-            failures.add(supplier.get());
+        for (TestNodeResponse nodeResponse : response.getNodes()) {
+            assertThat(successfulNodes, Matchers.hasItem(nodeResponse.getNode()));
         }
-        return failures;
+        assertEquals(successfulNodes.size(), response.getNodes().size());
+
+        assertNotEquals(failedNodeIds.isEmpty(), response.hasFailures());
+        for (FailedNodeException failure : response.failures()) {
+            assertThat(failedNodeIds, Matchers.hasItem(failure.nodeId()));
+            if (failure.getCause()instanceof ElasticsearchException elasticsearchException) {
+                final var cause = elasticsearchException.unwrapCause();
+                assertEquals("simulated", cause.getMessage());
+            } else {
+                throw new AssertionError("unexpected exception", failure);
+            }
+        }
+        assertEquals(failedNodeIds.size(), response.failures().size());
     }
 
-    private enum NodeSelector {
-        LOCAL("_local"),
-        ELECTED_MASTER("_master"),
-        MASTER_ELIGIBLE("master:true"),
-        DATA("data:true"),
-        CUSTOM_ATTRIBUTE("attr:value");
+    public void testResponsesReleasedOnCancellation() {
+        final TestTransportNodesAction action = getTestTransportNodesAction();
+
+        final CancellableTask cancellableTask = new CancellableTask(randomLong(), "transport", "action", "", null, emptyMap());
+        final PlainActionFuture<TestNodesResponse> listener = new PlainActionFuture<>();
+        action.execute(cancellableTask, new TestNodesRequest(), listener);
 
-        private final String selector;
+        final List<CapturingTransport.CapturedRequest> capturedRequests = new ArrayList<>(
+            Arrays.asList(transport.getCapturedRequestsAndClear())
+        );
+        Randomness.shuffle(capturedRequests);
+
+        final ReachabilityChecker reachabilityChecker = new ReachabilityChecker();
+        final Runnable nextRequestProcessor = () -> {
+            var capturedRequest = capturedRequests.remove(0);
+            if (randomBoolean()) {
+                // transport.handleResponse may de/serialize the response, releasing it early, so send the response straight to the handler
+                transport.getTransportResponseHandler(capturedRequest.requestId())
+                    .handleResponse(reachabilityChecker.register(new TestNodeResponse(capturedRequest.node())));
+            } else {
+                // handleRemoteError may de/serialize the exception, releasing it early, so just use handleLocalError
+                transport.handleLocalError(
+                    capturedRequest.requestId(),
+                    reachabilityChecker.register(new ElasticsearchException("simulated"))
+                );
+            }
+        };
 
-        NodeSelector(String selector) {
-            this.selector = selector;
+        assertThat(capturedRequests.size(), greaterThan(2));
+        final var responsesBeforeCancellation = between(1, capturedRequests.size() - 2);
+        for (int i = 0; i < responsesBeforeCancellation; i++) {
+            nextRequestProcessor.run();
         }
+
+        reachabilityChecker.checkReachable();
+        TaskCancelHelper.cancel(cancellableTask, "simulated");
+
+        // responses captured before cancellation are now unreachable
+        reachabilityChecker.ensureUnreachable();
+
+        while (capturedRequests.size() > 0) {
+            // a response sent after cancellation is dropped immediately
+            assertFalse(listener.isDone());
+            nextRequestProcessor.run();
+            reachabilityChecker.ensureUnreachable();
+        }
+
+        expectThrows(TaskCancelledException.class, () -> listener.actionGet(10, TimeUnit.SECONDS));
     }
 
     @BeforeClass
diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
index 51363e76d8ad..4759724bfc3d 100644
--- a/server/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
@@ -7,7 +7,7 @@
  */
 package org.elasticsearch.action.support.replication;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.NoShardAvailableActionException;
 import org.elasticsearch.action.UnavailableShardsException;
@@ -88,7 +88,7 @@ public void setUp() throws Exception {
         super.setUp();
         TcpTransport transport = new Netty4Transport(
             Settings.EMPTY,
-            Version.CURRENT,
+            TransportVersion.CURRENT,
             threadPool,
             new NetworkService(Collections.emptyList()),
             PageCacheRecycler.NON_RECYCLING_INSTANCE,
@@ -254,15 +254,11 @@ private class TestBroadcastReplicationAction extends TransportBroadcastReplicati
                 null,
                 actionFilters,
                 indexNameExpressionResolver,
-                null
+                null,
+                ThreadPool.Names.SAME
             );
         }
 
-        @Override
-        protected ReplicationResponse newShardResponse() {
-            return new ReplicationResponse();
-        }
-
         @Override
         protected BasicReplicationRequest newShardRequest(DummyBroadcastRequest request, ShardId shardId) {
             return new BasicReplicationRequest(shardId);
diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java
index 8dab09fb6015..543b673635ee 100644
--- a/server/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java
@@ -507,7 +507,7 @@ public void testPrimaryFailureHandlingReplicaResponse() throws Exception {
 
         final Set<String> inSyncAllocationIds = indexMetadata.inSyncAllocationIds(0);
         final IndexShardRoutingTable shardRoutingTable = state.routingTable().index(index).shard(shardId.id());
-        final Set<String> trackedShards = shardRoutingTable.getAllAllocationIds();
+        final Set<String> trackedShards = shardRoutingTable.getPromotableAllocationIds();
         final ReplicationGroup initialReplicationGroup = new ReplicationGroup(shardRoutingTable, inSyncAllocationIds, trackedShards, 0);
 
         final Thread testThread = Thread.currentThread();
diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/ReplicationResponseTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/ReplicationResponseTests.java
index ca39be3055e8..1f3243931628 100644
--- a/server/src/test/java/org/elasticsearch/action/support/replication/ReplicationResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/replication/ReplicationResponseTests.java
@@ -36,7 +36,7 @@ public void testShardInfoToString() {
         final int total = 5;
         final int successful = randomIntBetween(1, total);
         final ShardInfo shardInfo = new ShardInfo(total, successful);
-        assertEquals(formatted("ShardInfo{total=5, successful=%d, failures=[]}", successful), shardInfo.toString());
+        assertEquals(Strings.format("ShardInfo{total=5, successful=%d, failures=[]}", successful), shardInfo.toString());
     }
 
     public void testShardInfoToXContent() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
index 31c2d6e29deb..e64dddff3cdd 100644
--- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.UnavailableShardsException;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
@@ -47,6 +47,7 @@
 import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexService;
@@ -88,7 +89,6 @@
 import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Locale;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
@@ -928,7 +928,12 @@ public void testSeqNoIsSetOnPrimary() {
         Set<String> inSyncIds = randomBoolean()
             ? singleton(routingEntry.allocationId().getId())
             : clusterService.state().metadata().index(index).inSyncAllocationIds(0);
-        ReplicationGroup replicationGroup = new ReplicationGroup(shardRoutingTable, inSyncIds, shardRoutingTable.getAllAllocationIds(), 0);
+        ReplicationGroup replicationGroup = new ReplicationGroup(
+            shardRoutingTable,
+            inSyncIds,
+            shardRoutingTable.getPromotableAllocationIds(),
+            0
+        );
         when(shard.getReplicationGroup()).thenReturn(replicationGroup);
         PendingReplicationActions replicationActions = new PendingReplicationActions(shardId, threadPool);
         replicationActions.accept(replicationGroup);
@@ -1282,7 +1287,7 @@ public void testRetryOnReplicaWithRealTransport() throws Exception {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
         final Transport transport = new Netty4Transport(
             Settings.EMPTY,
-            Version.CURRENT,
+            TransportVersion.CURRENT,
             threadPool,
             new NetworkService(Collections.emptyList()),
             PageCacheRecycler.NON_RECYCLING_INSTANCE,
@@ -1582,7 +1587,7 @@ private IndexShard mockIndexShard(ShardId shardId, ClusterService clusterService
             final long primaryTerm = indexShard.getPendingPrimaryTerm();
             if (term < primaryTerm) {
                 throw new IllegalArgumentException(
-                    String.format(Locale.ROOT, "%s operation term [%d] is too old (current [%d])", shardId, term, primaryTerm)
+                    Strings.format("%s operation term [%d] is too old (current [%d])", shardId, term, primaryTerm)
                 );
             }
             count.incrementAndGet();
diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
index cdb34b7a75d5..3cd45e374fd9 100644
--- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexingPressure;
@@ -57,7 +58,6 @@
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashSet;
-import java.util.Locale;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -129,7 +129,7 @@ public void testPrimaryNoRefreshCall() throws Exception {
         TestAction testAction = new TestAction();
         testAction.dispatchedShardOperationOnPrimary(request, indexShard, ActionTestUtils.assertNoFailureListener(result -> {
             CapturingActionListener<TestResponse> listener = new CapturingActionListener<>();
-            result.runPostReplicationActions(listener.map(ignore -> result.finalResponseIfSuccessful));
+            result.runPostReplicationActions(listener.map(ignore -> result.replicationResponse));
             assertNotNull(listener.response);
             assertNull(listener.failure);
             verify(indexShard, never()).refresh(any());
@@ -158,7 +158,7 @@ public void testPrimaryImmediateRefresh() throws Exception {
         TestAction testAction = new TestAction();
         testAction.dispatchedShardOperationOnPrimary(request, indexShard, ActionTestUtils.assertNoFailureListener(result -> {
             CapturingActionListener<TestResponse> listener = new CapturingActionListener<>();
-            result.runPostReplicationActions(listener.map(ignore -> result.finalResponseIfSuccessful));
+            result.runPostReplicationActions(listener.map(ignore -> result.replicationResponse));
             assertNotNull(listener.response);
             assertNull(listener.failure);
             assertTrue(listener.response.forcedRefresh);
@@ -189,7 +189,7 @@ public void testPrimaryWaitForRefresh() throws Exception {
         TestAction testAction = new TestAction();
         testAction.dispatchedShardOperationOnPrimary(request, indexShard, ActionTestUtils.assertNoFailureListener(result -> {
             CapturingActionListener<TestResponse> listener = new CapturingActionListener<>();
-            result.runPostReplicationActions(listener.map(ignore -> result.finalResponseIfSuccessful));
+            result.runPostReplicationActions(listener.map(ignore -> result.replicationResponse));
             assertNull(listener.response); // Haven't really responded yet
 
             @SuppressWarnings({ "unchecked", "rawtypes" })
@@ -228,15 +228,21 @@ public void testReplicaWaitForRefresh() throws Exception {
         assertNull(listener.failure);
     }
 
-    public void testDocumentFailureInShardOperationOnPrimary() throws Exception {
-        TestRequest request = new TestRequest();
-        TestAction testAction = new TestAction(true, true);
-        testAction.dispatchedShardOperationOnPrimary(request, indexShard, ActionTestUtils.assertNoFailureListener(result -> {
-            CapturingActionListener<TestResponse> listener = new CapturingActionListener<>();
-            result.runPostReplicationActions(listener.map(ignore -> result.finalResponseIfSuccessful));
-            assertNull(listener.response);
-            assertNotNull(listener.failure);
-        }));
+    public void testDocumentFailureInShardOperationOnPrimary() {
+        assertEquals(
+            "simulated",
+            expectThrows(
+                RuntimeException.class,
+                () -> PlainActionFuture.get(
+                    (PlainActionFuture<TransportReplicationAction.PrimaryResult<TestRequest, TestResponse>> future) -> new TestAction(
+                        true,
+                        randomBoolean()
+                    ).dispatchedShardOperationOnPrimary(new TestRequest(), indexShard, future),
+                    0,
+                    TimeUnit.SECONDS
+                )
+            ).getMessage()
+        );
     }
 
     public void testDocumentFailureInShardOperationOnReplica() throws Exception {
@@ -447,9 +453,9 @@ protected void dispatchedShardOperationOnPrimary(
         ) {
             ActionListener.completeWith(listener, () -> {
                 if (withDocumentFailureOnPrimary) {
-                    return new WritePrimaryResult<>(request, null, null, new RuntimeException("simulated"), primary, logger);
+                    throw new RuntimeException("simulated");
                 } else {
-                    return new WritePrimaryResult<>(request, new TestResponse(), location, null, primary, logger);
+                    return new WritePrimaryResult<>(request, new TestResponse(), location, primary, logger);
                 }
             });
         }
@@ -521,7 +527,7 @@ private IndexShard mockIndexShard(ShardId shardId, ClusterService clusterService
             final long primaryTerm = indexShard.getPendingPrimaryTerm();
             if (term < primaryTerm) {
                 throw new IllegalArgumentException(
-                    String.format(Locale.ROOT, "%s operation term [%d] is too old (current [%d])", shardId, term, primaryTerm)
+                    Strings.format("%s operation term [%d] is too old (current [%d])", shardId, term, primaryTerm)
                 );
             }
             count.incrementAndGet();
diff --git a/server/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java b/server/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java
index 12f7af90536c..2c0b03194bb7 100644
--- a/server/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java
@@ -24,6 +24,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.analysis.PreConfiguredTokenizer;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.indices.analysis.AnalysisModule;
@@ -191,7 +192,7 @@ public void testRandomPayloadWithDelimitedPayloadTokenFilter() throws IOExceptio
             for (int k = 0; k < docsAndPositions.freq(); k++) {
                 docsAndPositions.nextPosition();
                 if (docsAndPositions.getPayload() != null) {
-                    String infoString = formatted("""
+                    String infoString = Strings.format("""
 
                         term: %s has payload\s
                         %s
diff --git a/server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java b/server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java
index a56c8531ee74..843a65aa877a 100644
--- a/server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java
+++ b/server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java
@@ -24,6 +24,7 @@
 import org.hamcrest.Matcher;
 
 import java.net.InetAddress;
+import java.nio.ByteOrder;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -725,4 +726,23 @@ public void testDiscoveryConfiguredCheck() throws NodeValidationException {
         ensureChecksPass.accept(Settings.builder().putList(DiscoveryModule.DISCOVERY_SEED_PROVIDERS_SETTING.getKey()));
         ensureChecksPass.accept(Settings.builder().putList(SettingsBasedSeedHostsProvider.DISCOVERY_SEED_HOSTS_SETTING.getKey()));
     }
+
+    public void testByteOrderCheck() throws NodeValidationException {
+        ByteOrder[] reference = new ByteOrder[] { ByteOrder.BIG_ENDIAN };
+        BootstrapChecks.ByteOrderCheck byteOrderCheck = new BootstrapChecks.ByteOrderCheck() {
+            @Override
+            ByteOrder nativeByteOrder() {
+                return reference[0];
+            }
+        };
+
+        final NodeValidationException e = expectThrows(
+            NodeValidationException.class,
+            () -> BootstrapChecks.check(emptyContext, true, List.of(byteOrderCheck))
+        );
+        assertThat(e.getMessage(), containsString("Little-endian native byte order is required to run Elasticsearch"));
+
+        reference[0] = ByteOrder.LITTLE_ENDIAN;
+        BootstrapChecks.check(emptyContext, true, List.of(byteOrderCheck));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
index 1b8d5822217f..82b189799940 100644
--- a/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.cluster;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexGraveyard;
@@ -299,8 +300,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT;
         }
 
         @Override
@@ -320,8 +321,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT;
         }
 
         @Override
@@ -501,7 +502,7 @@ private static IndexMetadata createIndexMetadata(final Index index, final long v
 
     // Create the routing table for a cluster state.
     private static RoutingTable createRoutingTable(final long version, final Metadata metadata) {
-        final RoutingTable.Builder builder = RoutingTable.builder().version(version);
+        final RoutingTable.Builder builder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).version(version);
         for (IndexMetadata indexMetadata : metadata.indices().values()) {
             builder.addAsNew(indexMetadata);
         }
diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java
index 2c16c8e2fb94..66c2fd4b3df3 100644
--- a/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/ClusterInfoTests.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -35,7 +34,7 @@ protected ClusterInfo createTestInstance() {
     }
 
     @Override
-    protected ClusterInfo mutateInstance(ClusterInfo instance) throws IOException {
+    protected ClusterInfo mutateInstance(ClusterInfo instance) {
         return createTestInstance();
     }
 
@@ -44,12 +43,13 @@ private static Map<String, DiskUsage> randomDiskUsage() {
         Map<String, DiskUsage> builder = new HashMap<>(numEntries);
         for (int i = 0; i < numEntries; i++) {
             String key = randomAlphaOfLength(32);
+            final int totalBytes = randomIntBetween(0, Integer.MAX_VALUE);
             DiskUsage diskUsage = new DiskUsage(
                 randomAlphaOfLength(4),
                 randomAlphaOfLength(4),
                 randomAlphaOfLength(4),
-                randomIntBetween(0, Integer.MAX_VALUE),
-                randomIntBetween(0, Integer.MAX_VALUE)
+                totalBytes,
+                randomIntBetween(0, totalBytes)
             );
             builder.put(key, diskUsage);
         }
diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
index fe4147d5f8cb..3abc91dbac04 100644
--- a/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
@@ -8,7 +8,6 @@
 
 package org.elasticsearch.cluster;
 
-import org.elasticsearch.cluster.routing.RerouteService;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
@@ -154,16 +153,9 @@ public void testRegisterAllocationDeciderDuplicate() {
             () -> new ClusterModule(Settings.EMPTY, clusterService, Collections.<ClusterPlugin>singletonList(new ClusterPlugin() {
                 @Override
                 public Collection<AllocationDecider> createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) {
-                    return Collections.singletonList(new EnableAllocationDecider(settings, clusterSettings));
+                    return Collections.singletonList(new EnableAllocationDecider(clusterSettings));
                 }
-            }),
-                clusterInfoService,
-                null,
-                threadPool,
-                EmptySystemIndices.INSTANCE,
-                ClusterModuleTests::getFakeRerouteService,
-                WriteLoadForecaster.DEFAULT
-            )
+            }), clusterInfoService, null, threadPool, EmptySystemIndices.INSTANCE, WriteLoadForecaster.DEFAULT)
         );
         assertEquals(e.getMessage(), "Cannot specify allocation decider [" + EnableAllocationDecider.class.getName() + "] twice");
     }
@@ -174,14 +166,7 @@ public void testRegisterAllocationDecider() {
             public Collection<AllocationDecider> createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) {
                 return Collections.singletonList(new FakeAllocationDecider());
             }
-        }),
-            clusterInfoService,
-            null,
-            threadPool,
-            EmptySystemIndices.INSTANCE,
-            ClusterModuleTests::getFakeRerouteService,
-            WriteLoadForecaster.DEFAULT
-        );
+        }), clusterInfoService, null, threadPool, EmptySystemIndices.INSTANCE, WriteLoadForecaster.DEFAULT);
         assertTrue(module.deciderList.stream().anyMatch(d -> d.getClass().equals(FakeAllocationDecider.class)));
     }
 
@@ -191,14 +176,7 @@ private ClusterModule newClusterModuleWithShardsAllocator(Settings settings, Str
             public Map<String, Supplier<ShardsAllocator>> getShardsAllocators(Settings settings, ClusterSettings clusterSettings) {
                 return Collections.singletonMap(name, supplier);
             }
-        }),
-            clusterInfoService,
-            null,
-            threadPool,
-            EmptySystemIndices.INSTANCE,
-            ClusterModuleTests::getFakeRerouteService,
-            WriteLoadForecaster.DEFAULT
-        );
+        }), clusterInfoService, null, threadPool, EmptySystemIndices.INSTANCE, WriteLoadForecaster.DEFAULT);
     }
 
     public void testRegisterShardsAllocator() {
@@ -227,7 +205,6 @@ public void testUnknownShardsAllocator() {
                 null,
                 threadPool,
                 EmptySystemIndices.INSTANCE,
-                ClusterModuleTests::getFakeRerouteService,
                 WriteLoadForecaster.DEFAULT
             )
         );
@@ -286,7 +263,6 @@ public void testRejectsReservedExistingShardsAllocatorName() {
             null,
             threadPool,
             EmptySystemIndices.INSTANCE,
-            ClusterModuleTests::getFakeRerouteService,
             WriteLoadForecaster.DEFAULT
         );
         expectThrows(IllegalArgumentException.class, () -> clusterModule.setExistingShardsAllocators(new TestGatewayAllocator()));
@@ -301,7 +277,6 @@ public void testRejectsDuplicateExistingShardsAllocatorName() {
             null,
             threadPool,
             EmptySystemIndices.INSTANCE,
-            ClusterModuleTests::getFakeRerouteService,
             WriteLoadForecaster.DEFAULT
         );
         expectThrows(IllegalArgumentException.class, () -> clusterModule.setExistingShardsAllocators(new TestGatewayAllocator()));
@@ -315,8 +290,4 @@ public Map<String, ExistingShardsAllocator> getExistingShardsAllocators() {
             }
         };
     }
-
-    private static RerouteService getFakeRerouteService() {
-        return (s, p, r) -> { throw new AssertionError("should not be called"); };
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java
index 79dfd0be6b23..f26caf2f98e5 100644
--- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java
@@ -134,179 +134,209 @@ public void testToXContent() throws IOException {
         IndexRoutingTable index = clusterState.getRoutingTable().getIndicesRouting().get("index");
 
         String ephemeralId = clusterState.getNodes().get("nodeId1").getEphemeralId();
-        String allocationId = index.shard(0).getAllAllocationIds().iterator().next();
+        String allocationId = index.shard(0).getPromotableAllocationIds().iterator().next();
 
         XContentBuilder builder = JsonXContent.contentBuilder();
         builder.startObject();
-        clusterState.toXContent(builder, new ToXContent.MapParams(singletonMap(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API)));
+        writeChunks(
+            clusterState,
+            builder,
+            new ToXContent.MapParams(singletonMap(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API)),
+            34
+        );
         builder.endObject();
 
-        assertEquals(XContentHelper.stripWhitespace(formatted("""
-            {
-              "cluster_uuid": "clusterUUID",
-              "version": 0,
-              "state_uuid": "stateUUID",
-              "master_node": "nodeId1",
-              "blocks": {
-                "global": {
-                  "1": {
-                    "description": "description",
-                    "retryable": true,
-                    "disable_state_persistence": true,
-                    "levels": [
-                      "read",
-                      "write",
-                      "metadata_read",
-                      "metadata_write"
-                    ]
-                  }
-                },
-                "indices": {
-                  "index": {
-                    "2": {
-                      "description": "description2",
-                      "retryable": false,
-                      "levels": [
-                        "read",
-                        "write",
-                        "metadata_read",
-                        "metadata_write"
-                      ]
-                    }
-                  }
-                }
-              },
-              "nodes": {
-                "nodeId1": {
-                  "name": "",
-                  "ephemeral_id": "%s",
-                  "transport_address": "127.0.0.1:111",
-                  "external_id": "",
-                  "attributes": {},
-                  "roles": [
-                    "data",
-                    "data_cold",
-                    "data_content",
-                    "data_frozen",
-                    "data_hot",
-                    "data_warm",
-                    "index",
-                    "ingest",
-                    "master",
-                    "ml",
-                    "remote_cluster_client",
-                    "search",
-                    "transform",
-                    "voting_only"
-                  ]
-                }
-              },
-              "metadata": {
-                "cluster_uuid": "clusterUUID",
-                "cluster_uuid_committed": false,
-                "cluster_coordination": {
-                  "term": 1,
-                  "last_committed_config": [
-                    "commitedConfigurationNodeId"
-                  ],
-                  "last_accepted_config": [
-                    "acceptedConfigurationNodeId"
-                  ],
-                  "voting_config_exclusions": [
-                    {
-                      "node_id": "exlucdedNodeId",
-                      "node_name": "excludedNodeName"
-                    }
-                  ]
-                },
-                "templates": {
-                  "template": {
-                    "order": 0,
-                    "index_patterns": [
-                      "pattern1",
-                      "pattern2"
-                    ],
-                    "settings": {
-                      "index": {
-                        "version": {
-                          "created": "%s"
-                        }
+        assertEquals(
+            XContentHelper.stripWhitespace(Strings.format("""
+                {
+                  "cluster_uuid": "clusterUUID",
+                  "version": 0,
+                  "state_uuid": "stateUUID",
+                  "master_node": "nodeId1",
+                  "blocks": {
+                    "global": {
+                      "1": {
+                        "description": "description",
+                        "retryable": true,
+                        "disable_state_persistence": true,
+                        "levels": [
+                          "read",
+                          "write",
+                          "metadata_read",
+                          "metadata_write"
+                        ]
                       }
                     },
-                    "mappings": {
-                      "key1": {}
-                    },
-                    "aliases": {}
-                  }
-                },
-                "indices": {
-                  "index": {
-                    "version": 1,
-                    "mapping_version": 1,
-                    "settings_version": 1,
-                    "aliases_version": 1,
-                    "routing_num_shards": 1,
-                    "state": "open",
-                    "settings": {
+                    "indices": {
                       "index": {
-                        "number_of_shards": "1",
-                        "number_of_replicas": "2",
-                        "version": {
-                          "created": "%s"
+                        "2": {
+                          "description": "description2",
+                          "retryable": false,
+                          "levels": [
+                            "read",
+                            "write",
+                            "metadata_read",
+                            "metadata_write"
+                          ]
                         }
                       }
-                    },
-                    "mappings": {
-                      "type": {
-                        "type1": {
-                          "key": "value"
+                    }
+                  },
+                  "nodes": {
+                    "nodeId1": {
+                      "name": "",
+                      "ephemeral_id": "%s",
+                      "transport_address": "127.0.0.1:111",
+                      "external_id": "",
+                      "attributes": {},
+                      "roles": [
+                        "data",
+                        "data_cold",
+                        "data_content",
+                        "data_frozen",
+                        "data_hot",
+                        "data_warm",
+                        "index",
+                        "ingest",
+                        "master",
+                        "ml",
+                        "remote_cluster_client",
+                        "search",
+                        "transform",
+                        "voting_only"
+                      ],
+                      "version": "%s"
+                    }
+                  },
+                  "metadata": {
+                    "cluster_uuid": "clusterUUID",
+                    "cluster_uuid_committed": false,
+                    "cluster_coordination": {
+                      "term": 1,
+                      "last_committed_config": [
+                        "commitedConfigurationNodeId"
+                      ],
+                      "last_accepted_config": [
+                        "acceptedConfigurationNodeId"
+                      ],
+                      "voting_config_exclusions": [
+                        {
+                          "node_id": "exlucdedNodeId",
+                          "node_name": "excludedNodeName"
                         }
-                      }
-                    },
-                    "aliases": [
-                      "alias"
-                    ],
-                    "primary_terms": {
-                      "0": 1
-                    },
-                    "in_sync_allocations": {
-                      "0": [
-                        "allocationId"
                       ]
                     },
-                    "rollover_info": {
-                      "rolloveAlias": {
-                        "met_conditions": {},
-                        "time": 1
+                    "templates": {
+                      "template": {
+                        "order": 0,
+                        "index_patterns": [
+                          "pattern1",
+                          "pattern2"
+                        ],
+                        "settings": {
+                          "index": {
+                            "version": {
+                              "created": "%s"
+                            }
+                          }
+                        },
+                        "mappings": {
+                          "key1": {}
+                        },
+                        "aliases": {}
                       }
                     },
-                    "system": false,
-                    "timestamp_range": {
-                      "shards": []
-                    },
-                    "stats": {
-                        "write_load": {
-                          "loads": [-1.0],
-                          "uptimes": [-1]
+                    "indices": {
+                      "index": {
+                        "version": 1,
+                        "mapping_version": 1,
+                        "settings_version": 1,
+                        "aliases_version": 1,
+                        "routing_num_shards": 1,
+                        "state": "open",
+                        "settings": {
+                          "index": {
+                            "number_of_shards": "1",
+                            "number_of_replicas": "2",
+                            "version": {
+                              "created": "%s"
+                            }
+                          }
                         },
-                        "avg_size": {
-                            "total_size_in_bytes": 120,
-                            "shard_count": 1
-                        }
+                        "mappings": {
+                          "type": {
+                            "type1": {
+                              "key": "value"
+                            }
+                          }
+                        },
+                        "aliases": [
+                          "alias"
+                        ],
+                        "primary_terms": {
+                          "0": 1
+                        },
+                        "in_sync_allocations": {
+                          "0": [
+                            "allocationId"
+                          ]
+                        },
+                        "rollover_info": {
+                          "rolloveAlias": {
+                            "met_conditions": {},
+                            "time": 1
+                          }
+                        },
+                        "system": false,
+                        "timestamp_range": {
+                          "shards": []
+                        },
+                        "stats": {
+                            "write_load": {
+                              "loads": [-1.0],
+                              "uptimes": [-1]
+                            },
+                            "avg_size": {
+                                "total_size_in_bytes": 120,
+                                "shard_count": 1
+                            }
+                        },
+                        "write_load_forecast" : 8.0
+                      }
                     },
-                    "write_load_forecast" : 8.0
-                  }
-                },
-                "index-graveyard": {
-                  "tombstones": []
-                },
-                "reserved_state" : { }
-              },
-              "routing_table": {
-                "indices": {
-                  "index": {
-                    "shards": {
-                      "0": [
+                    "index-graveyard": {
+                      "tombstones": []
+                    },
+                    "reserved_state" : { }
+                  },
+                  "routing_table": {
+                    "indices": {
+                      "index": {
+                        "shards": {
+                          "0": [
+                            {
+                              "state": "STARTED",
+                              "primary": true,
+                              "node": "nodeId2",
+                              "relocating_node": null,
+                              "shard": 0,
+                              "index": "index",
+                              "allocation_id": {
+                                "id": "%s"
+                              },
+                              "relocation_failure_info" : {
+                                "failed_attempts" : 0
+                              }
+                            }
+                          ]
+                        }
+                      }
+                    }
+                  },
+                  "routing_nodes": {
+                    "unassigned": [],
+                    "nodes": {
+                      "nodeId2": [
                         {
                           "state": "STARTED",
                           "primary": true,
@@ -321,34 +351,13 @@ public void testToXContent() throws IOException {
                             "failed_attempts" : 0
                           }
                         }
-                      ]
+                      ],
+                      "nodeId1": []
                     }
                   }
-                }
-              },
-              "routing_nodes": {
-                "unassigned": [],
-                "nodes": {
-                  "nodeId2": [
-                    {
-                      "state": "STARTED",
-                      "primary": true,
-                      "node": "nodeId2",
-                      "relocating_node": null,
-                      "shard": 0,
-                      "index": "index",
-                      "allocation_id": {
-                        "id": "%s"
-                      },
-                      "relocation_failure_info" : {
-                        "failed_attempts" : 0
-                      }
-                    }
-                  ],
-                  "nodeId1": []
-                }
-              }
-            }""", ephemeralId, Version.CURRENT.id, Version.CURRENT.id, allocationId, allocationId)), Strings.toString(builder));
+                }""", ephemeralId, Version.CURRENT, Version.CURRENT.id, Version.CURRENT.id, allocationId, allocationId)),
+            Strings.toString(builder)
+        );
 
     }
 
@@ -365,175 +374,200 @@ public void testToXContent_FlatSettingTrue_ReduceMappingFalse() throws IOExcepti
         IndexRoutingTable index = clusterState.getRoutingTable().getIndicesRouting().get("index");
 
         String ephemeralId = clusterState.getNodes().get("nodeId1").getEphemeralId();
-        String allocationId = index.shard(0).getAllAllocationIds().iterator().next();
+        String allocationId = index.shard(0).getPromotableAllocationIds().iterator().next();
 
         XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();
         builder.startObject();
-        clusterState.toXContent(builder, new ToXContent.MapParams(mapParams));
+        writeChunks(clusterState, builder, new ToXContent.MapParams(mapParams), 34);
         builder.endObject();
 
-        assertEquals(formatted("""
-            {
-              "cluster_uuid" : "clusterUUID",
-              "version" : 0,
-              "state_uuid" : "stateUUID",
-              "master_node" : "nodeId1",
-              "blocks" : {
-                "global" : {
-                  "1" : {
-                    "description" : "description",
-                    "retryable" : true,
-                    "disable_state_persistence" : true,
-                    "levels" : [
-                      "read",
-                      "write",
-                      "metadata_read",
-                      "metadata_write"
-                    ]
-                  }
-                },
-                "indices" : {
-                  "index" : {
-                    "2" : {
-                      "description" : "description2",
-                      "retryable" : false,
-                      "levels" : [
-                        "read",
-                        "write",
-                        "metadata_read",
-                        "metadata_write"
-                      ]
-                    }
-                  }
-                }
-              },
-              "nodes" : {
-                "nodeId1" : {
-                  "name" : "",
-                  "ephemeral_id" : "%s",
-                  "transport_address" : "127.0.0.1:111",
-                  "external_id" : "",
-                  "attributes" : { },
-                  "roles" : [
-                    "data",
-                    "data_cold",
-                    "data_content",
-                    "data_frozen",
-                    "data_hot",
-                    "data_warm",
-                    "index",
-                    "ingest",
-                    "master",
-                    "ml",
-                    "remote_cluster_client",
-                    "search",
-                    "transform",
-                    "voting_only"
-                  ]
-                }
-              },
-              "metadata" : {
-                "cluster_uuid" : "clusterUUID",
-                "cluster_uuid_committed" : false,
-                "cluster_coordination" : {
-                  "term" : 1,
-                  "last_committed_config" : [
-                    "commitedConfigurationNodeId"
-                  ],
-                  "last_accepted_config" : [
-                    "acceptedConfigurationNodeId"
-                  ],
-                  "voting_config_exclusions" : [
-                    {
-                      "node_id" : "exlucdedNodeId",
-                      "node_name" : "excludedNodeName"
-                    }
-                  ]
-                },
-                "templates" : {
-                  "template" : {
-                    "order" : 0,
-                    "index_patterns" : [
-                      "pattern1",
-                      "pattern2"
-                    ],
-                    "settings" : {
-                      "index.version.created" : "%s"
-                    },
-                    "mappings" : {
-                      "key1" : { }
-                    },
-                    "aliases" : { }
-                  }
-                },
-                "indices" : {
-                  "index" : {
-                    "version" : 1,
-                    "mapping_version" : 1,
-                    "settings_version" : 1,
-                    "aliases_version" : 1,
-                    "routing_num_shards" : 1,
-                    "state" : "open",
-                    "settings" : {
-                      "index.number_of_replicas" : "2",
-                      "index.number_of_shards" : "1",
-                      "index.version.created" : "%s"
+        assertEquals(
+            Strings.format("""
+                {
+                  "cluster_uuid" : "clusterUUID",
+                  "version" : 0,
+                  "state_uuid" : "stateUUID",
+                  "master_node" : "nodeId1",
+                  "blocks" : {
+                    "global" : {
+                      "1" : {
+                        "description" : "description",
+                        "retryable" : true,
+                        "disable_state_persistence" : true,
+                        "levels" : [
+                          "read",
+                          "write",
+                          "metadata_read",
+                          "metadata_write"
+                        ]
+                      }
                     },
-                    "mappings" : {
-                      "type" : {
-                        "type1" : {
-                          "key" : "value"
+                    "indices" : {
+                      "index" : {
+                        "2" : {
+                          "description" : "description2",
+                          "retryable" : false,
+                          "levels" : [
+                            "read",
+                            "write",
+                            "metadata_read",
+                            "metadata_write"
+                          ]
                         }
                       }
-                    },
-                    "aliases" : [
-                      "alias"
-                    ],
-                    "primary_terms" : {
-                      "0" : 1
-                    },
-                    "in_sync_allocations" : {
-                      "0" : [
-                        "allocationId"
+                    }
+                  },
+                  "nodes" : {
+                    "nodeId1" : {
+                      "name" : "",
+                      "ephemeral_id" : "%s",
+                      "transport_address" : "127.0.0.1:111",
+                      "external_id" : "",
+                      "attributes" : { },
+                      "roles" : [
+                        "data",
+                        "data_cold",
+                        "data_content",
+                        "data_frozen",
+                        "data_hot",
+                        "data_warm",
+                        "index",
+                        "ingest",
+                        "master",
+                        "ml",
+                        "remote_cluster_client",
+                        "search",
+                        "transform",
+                        "voting_only"
+                      ],
+                      "version" : "%s"
+                    }
+                  },
+                  "metadata" : {
+                    "cluster_uuid" : "clusterUUID",
+                    "cluster_uuid_committed" : false,
+                    "cluster_coordination" : {
+                      "term" : 1,
+                      "last_committed_config" : [
+                        "commitedConfigurationNodeId"
+                      ],
+                      "last_accepted_config" : [
+                        "acceptedConfigurationNodeId"
+                      ],
+                      "voting_config_exclusions" : [
+                        {
+                          "node_id" : "exlucdedNodeId",
+                          "node_name" : "excludedNodeName"
+                        }
                       ]
                     },
-                    "rollover_info" : {
-                      "rolloveAlias" : {
-                        "met_conditions" : { },
-                        "time" : 1
+                    "templates" : {
+                      "template" : {
+                        "order" : 0,
+                        "index_patterns" : [
+                          "pattern1",
+                          "pattern2"
+                        ],
+                        "settings" : {
+                          "index.version.created" : "%s"
+                        },
+                        "mappings" : {
+                          "key1" : { }
+                        },
+                        "aliases" : { }
                       }
                     },
-                    "system" : false,
-                    "timestamp_range" : {
-                      "shards" : [ ]
-                    },
-                    "stats" : {
-                      "write_load" : {
-                        "loads" : [
-                          -1.0
+                    "indices" : {
+                      "index" : {
+                        "version" : 1,
+                        "mapping_version" : 1,
+                        "settings_version" : 1,
+                        "aliases_version" : 1,
+                        "routing_num_shards" : 1,
+                        "state" : "open",
+                        "settings" : {
+                          "index.number_of_replicas" : "2",
+                          "index.number_of_shards" : "1",
+                          "index.version.created" : "%s"
+                        },
+                        "mappings" : {
+                          "type" : {
+                            "type1" : {
+                              "key" : "value"
+                            }
+                          }
+                        },
+                        "aliases" : [
+                          "alias"
                         ],
-                        "uptimes" : [
-                          -1
-                        ]
-                      },
-                      "avg_size" : {
-                        "total_size_in_bytes" : 120,
-                        "shard_count" : 1
+                        "primary_terms" : {
+                          "0" : 1
+                        },
+                        "in_sync_allocations" : {
+                          "0" : [
+                            "allocationId"
+                          ]
+                        },
+                        "rollover_info" : {
+                          "rolloveAlias" : {
+                            "met_conditions" : { },
+                            "time" : 1
+                          }
+                        },
+                        "system" : false,
+                        "timestamp_range" : {
+                          "shards" : [ ]
+                        },
+                        "stats" : {
+                          "write_load" : {
+                            "loads" : [
+                              -1.0
+                            ],
+                            "uptimes" : [
+                              -1
+                            ]
+                          },
+                          "avg_size" : {
+                            "total_size_in_bytes" : 120,
+                            "shard_count" : 1
+                          }
+                        },
+                        "write_load_forecast" : 8.0
                       }
                     },
-                    "write_load_forecast" : 8.0
-                  }
-                },
-                "index-graveyard" : {
-                  "tombstones" : [ ]
-                },
-                "reserved_state" : { }
-              },
-              "routing_table" : {
-                "indices" : {
-                  "index" : {
-                    "shards" : {
-                      "0" : [
+                    "index-graveyard" : {
+                      "tombstones" : [ ]
+                    },
+                    "reserved_state" : { }
+                  },
+                  "routing_table" : {
+                    "indices" : {
+                      "index" : {
+                        "shards" : {
+                          "0" : [
+                            {
+                              "state" : "STARTED",
+                              "primary" : true,
+                              "node" : "nodeId2",
+                              "relocating_node" : null,
+                              "shard" : 0,
+                              "index" : "index",
+                              "allocation_id" : {
+                                "id" : "%s"
+                              },
+                              "relocation_failure_info" : {
+                                "failed_attempts" : 0
+                              }
+                            }
+                          ]
+                        }
+                      }
+                    }
+                  },
+                  "routing_nodes" : {
+                    "unassigned" : [ ],
+                    "nodes" : {
+                      "nodeId2" : [
                         {
                           "state" : "STARTED",
                           "primary" : true,
@@ -548,34 +582,13 @@ public void testToXContent_FlatSettingTrue_ReduceMappingFalse() throws IOExcepti
                             "failed_attempts" : 0
                           }
                         }
-                      ]
+                      ],
+                      "nodeId1" : [ ]
                     }
                   }
-                }
-              },
-              "routing_nodes" : {
-                "unassigned" : [ ],
-                "nodes" : {
-                  "nodeId2" : [
-                    {
-                      "state" : "STARTED",
-                      "primary" : true,
-                      "node" : "nodeId2",
-                      "relocating_node" : null,
-                      "shard" : 0,
-                      "index" : "index",
-                      "allocation_id" : {
-                        "id" : "%s"
-                      },
-                      "relocation_failure_info" : {
-                        "failed_attempts" : 0
-                      }
-                    }
-                  ],
-                  "nodeId1" : [ ]
-                }
-              }
-            }""", ephemeralId, Version.CURRENT.id, Version.CURRENT.id, allocationId, allocationId), Strings.toString(builder));
+                }""", ephemeralId, Version.CURRENT, Version.CURRENT.id, Version.CURRENT.id, allocationId, allocationId),
+            Strings.toString(builder)
+        );
 
     }
 
@@ -593,181 +606,206 @@ public void testToXContent_FlatSettingFalse_ReduceMappingTrue() throws IOExcepti
         IndexRoutingTable index = clusterState.getRoutingTable().getIndicesRouting().get("index");
 
         String ephemeralId = clusterState.getNodes().get("nodeId1").getEphemeralId();
-        String allocationId = index.shard(0).getAllAllocationIds().iterator().next();
+        String allocationId = index.shard(0).getPromotableAllocationIds().iterator().next();
 
         XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();
         builder.startObject();
-        clusterState.toXContent(builder, new ToXContent.MapParams(mapParams));
+        writeChunks(clusterState, builder, new ToXContent.MapParams(mapParams), 34);
         builder.endObject();
 
-        assertEquals(formatted("""
-            {
-              "cluster_uuid" : "clusterUUID",
-              "version" : 0,
-              "state_uuid" : "stateUUID",
-              "master_node" : "nodeId1",
-              "blocks" : {
-                "global" : {
-                  "1" : {
-                    "description" : "description",
-                    "retryable" : true,
-                    "disable_state_persistence" : true,
-                    "levels" : [
-                      "read",
-                      "write",
-                      "metadata_read",
-                      "metadata_write"
-                    ]
-                  }
-                },
-                "indices" : {
-                  "index" : {
-                    "2" : {
-                      "description" : "description2",
-                      "retryable" : false,
-                      "levels" : [
-                        "read",
-                        "write",
-                        "metadata_read",
-                        "metadata_write"
-                      ]
-                    }
-                  }
-                }
-              },
-              "nodes" : {
-                "nodeId1" : {
-                  "name" : "",
-                  "ephemeral_id" : "%s",
-                  "transport_address" : "127.0.0.1:111",
-                  "external_id" : "",
-                  "attributes" : { },
-                  "roles" : [
-                    "data",
-                    "data_cold",
-                    "data_content",
-                    "data_frozen",
-                    "data_hot",
-                    "data_warm",
-                    "index",
-                    "ingest",
-                    "master",
-                    "ml",
-                    "remote_cluster_client",
-                    "search",
-                    "transform",
-                    "voting_only"
-                  ]
-                }
-              },
-              "metadata" : {
-                "cluster_uuid" : "clusterUUID",
-                "cluster_uuid_committed" : false,
-                "cluster_coordination" : {
-                  "term" : 1,
-                  "last_committed_config" : [
-                    "commitedConfigurationNodeId"
-                  ],
-                  "last_accepted_config" : [
-                    "acceptedConfigurationNodeId"
-                  ],
-                  "voting_config_exclusions" : [
-                    {
-                      "node_id" : "exlucdedNodeId",
-                      "node_name" : "excludedNodeName"
-                    }
-                  ]
-                },
-                "templates" : {
-                  "template" : {
-                    "order" : 0,
-                    "index_patterns" : [
-                      "pattern1",
-                      "pattern2"
-                    ],
-                    "settings" : {
-                      "index" : {
-                        "version" : {
-                          "created" : "%s"
-                        }
+        assertEquals(
+            Strings.format("""
+                {
+                  "cluster_uuid" : "clusterUUID",
+                  "version" : 0,
+                  "state_uuid" : "stateUUID",
+                  "master_node" : "nodeId1",
+                  "blocks" : {
+                    "global" : {
+                      "1" : {
+                        "description" : "description",
+                        "retryable" : true,
+                        "disable_state_persistence" : true,
+                        "levels" : [
+                          "read",
+                          "write",
+                          "metadata_read",
+                          "metadata_write"
+                        ]
                       }
                     },
-                    "mappings" : { },
-                    "aliases" : { }
-                  }
-                },
-                "indices" : {
-                  "index" : {
-                    "version" : 1,
-                    "mapping_version" : 1,
-                    "settings_version" : 1,
-                    "aliases_version" : 1,
-                    "routing_num_shards" : 1,
-                    "state" : "open",
-                    "settings" : {
+                    "indices" : {
                       "index" : {
-                        "number_of_shards" : "1",
-                        "number_of_replicas" : "2",
-                        "version" : {
-                          "created" : "%s"
+                        "2" : {
+                          "description" : "description2",
+                          "retryable" : false,
+                          "levels" : [
+                            "read",
+                            "write",
+                            "metadata_read",
+                            "metadata_write"
+                          ]
                         }
                       }
-                    },
-                    "mappings" : {
-                      "type" : {
-                        "type1" : {
-                          "key" : "value"
+                    }
+                  },
+                  "nodes" : {
+                    "nodeId1" : {
+                      "name" : "",
+                      "ephemeral_id" : "%s",
+                      "transport_address" : "127.0.0.1:111",
+                      "external_id" : "",
+                      "attributes" : { },
+                      "roles" : [
+                        "data",
+                        "data_cold",
+                        "data_content",
+                        "data_frozen",
+                        "data_hot",
+                        "data_warm",
+                        "index",
+                        "ingest",
+                        "master",
+                        "ml",
+                        "remote_cluster_client",
+                        "search",
+                        "transform",
+                        "voting_only"
+                      ],
+                      "version" : "%s"
+                    }
+                  },
+                  "metadata" : {
+                    "cluster_uuid" : "clusterUUID",
+                    "cluster_uuid_committed" : false,
+                    "cluster_coordination" : {
+                      "term" : 1,
+                      "last_committed_config" : [
+                        "commitedConfigurationNodeId"
+                      ],
+                      "last_accepted_config" : [
+                        "acceptedConfigurationNodeId"
+                      ],
+                      "voting_config_exclusions" : [
+                        {
+                          "node_id" : "exlucdedNodeId",
+                          "node_name" : "excludedNodeName"
                         }
-                      }
-                    },
-                    "aliases" : [
-                      "alias"
-                    ],
-                    "primary_terms" : {
-                      "0" : 1
-                    },
-                    "in_sync_allocations" : {
-                      "0" : [
-                        "allocationId"
                       ]
                     },
-                    "rollover_info" : {
-                      "rolloveAlias" : {
-                        "met_conditions" : { },
-                        "time" : 1
+                    "templates" : {
+                      "template" : {
+                        "order" : 0,
+                        "index_patterns" : [
+                          "pattern1",
+                          "pattern2"
+                        ],
+                        "settings" : {
+                          "index" : {
+                            "version" : {
+                              "created" : "%s"
+                            }
+                          }
+                        },
+                        "mappings" : { },
+                        "aliases" : { }
                       }
                     },
-                    "system" : false,
-                    "timestamp_range" : {
-                      "shards" : [ ]
-                    },
-                    "stats" : {
-                      "write_load" : {
-                        "loads" : [
-                          -1.0
+                    "indices" : {
+                      "index" : {
+                        "version" : 1,
+                        "mapping_version" : 1,
+                        "settings_version" : 1,
+                        "aliases_version" : 1,
+                        "routing_num_shards" : 1,
+                        "state" : "open",
+                        "settings" : {
+                          "index" : {
+                            "number_of_shards" : "1",
+                            "number_of_replicas" : "2",
+                            "version" : {
+                              "created" : "%s"
+                            }
+                          }
+                        },
+                        "mappings" : {
+                          "type" : {
+                            "type1" : {
+                              "key" : "value"
+                            }
+                          }
+                        },
+                        "aliases" : [
+                          "alias"
                         ],
-                        "uptimes" : [
-                          -1
-                        ]
-                      },
-                      "avg_size" : {
-                        "total_size_in_bytes" : 120,
-                        "shard_count" : 1
+                        "primary_terms" : {
+                          "0" : 1
+                        },
+                        "in_sync_allocations" : {
+                          "0" : [
+                            "allocationId"
+                          ]
+                        },
+                        "rollover_info" : {
+                          "rolloveAlias" : {
+                            "met_conditions" : { },
+                            "time" : 1
+                          }
+                        },
+                        "system" : false,
+                        "timestamp_range" : {
+                          "shards" : [ ]
+                        },
+                        "stats" : {
+                          "write_load" : {
+                            "loads" : [
+                              -1.0
+                            ],
+                            "uptimes" : [
+                              -1
+                            ]
+                          },
+                          "avg_size" : {
+                            "total_size_in_bytes" : 120,
+                            "shard_count" : 1
+                          }
+                        },
+                        "write_load_forecast" : 8.0
                       }
                     },
-                    "write_load_forecast" : 8.0
-                  }
-                },
-                "index-graveyard" : {
-                  "tombstones" : [ ]
-                },
-                "reserved_state" : { }
-              },
-              "routing_table" : {
-                "indices" : {
-                  "index" : {
-                    "shards" : {
-                      "0" : [
+                    "index-graveyard" : {
+                      "tombstones" : [ ]
+                    },
+                    "reserved_state" : { }
+                  },
+                  "routing_table" : {
+                    "indices" : {
+                      "index" : {
+                        "shards" : {
+                          "0" : [
+                            {
+                              "state" : "STARTED",
+                              "primary" : true,
+                              "node" : "nodeId2",
+                              "relocating_node" : null,
+                              "shard" : 0,
+                              "index" : "index",
+                              "allocation_id" : {
+                                "id" : "%s"
+                              },
+                              "relocation_failure_info" : {
+                                "failed_attempts" : 0
+                              }
+                            }
+                          ]
+                        }
+                      }
+                    }
+                  },
+                  "routing_nodes" : {
+                    "unassigned" : [ ],
+                    "nodes" : {
+                      "nodeId2" : [
                         {
                           "state" : "STARTED",
                           "primary" : true,
@@ -782,34 +820,13 @@ public void testToXContent_FlatSettingFalse_ReduceMappingTrue() throws IOExcepti
                             "failed_attempts" : 0
                           }
                         }
-                      ]
+                      ],
+                      "nodeId1" : [ ]
                     }
                   }
-                }
-              },
-              "routing_nodes" : {
-                "unassigned" : [ ],
-                "nodes" : {
-                  "nodeId2" : [
-                    {
-                      "state" : "STARTED",
-                      "primary" : true,
-                      "node" : "nodeId2",
-                      "relocating_node" : null,
-                      "shard" : 0,
-                      "index" : "index",
-                      "allocation_id" : {
-                        "id" : "%s"
-                      },
-                      "relocation_failure_info" : {
-                        "failed_attempts" : 0
-                      }
-                    }
-                  ],
-                  "nodeId1" : [ ]
-                }
-              }
-            }""", ephemeralId, Version.CURRENT.id, Version.CURRENT.id, allocationId, allocationId), Strings.toString(builder));
+                }""", ephemeralId, Version.CURRENT, Version.CURRENT.id, Version.CURRENT.id, allocationId, allocationId),
+            Strings.toString(builder)
+        );
 
     }
 
@@ -849,10 +866,10 @@ public void testToXContentSameTypeName() throws IOException {
 
         XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();
         builder.startObject();
-        clusterState.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        writeChunks(clusterState, builder, ToXContent.EMPTY_PARAMS, 25);
         builder.endObject();
 
-        assertEquals(formatted("""
+        assertEquals(Strings.format("""
             {
               "cluster_uuid" : "clusterUUID",
               "version" : 0,
@@ -1040,4 +1057,15 @@ public void testNodesIfRecovered() throws IOException {
             .build();
         assertEquals(DiscoveryNodes.EMPTY_NODES, notRecoveredState.nodesIfRecovered());
     }
+
+    private static void writeChunks(ClusterState clusterState, XContentBuilder builder, ToXContent.Params params, int expectedChunks)
+        throws IOException {
+        final var iterator = clusterState.toXContentChunked(params);
+        int chunks = 0;
+        while (iterator.hasNext()) {
+            iterator.next().toXContent(builder, params);
+            chunks += 1;
+        }
+        assertEquals(expectedChunks, chunks);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java b/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
index dee994f9cfd5..0340b5b2ae17 100644
--- a/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
@@ -101,7 +101,8 @@ public void testFillShardLevelInfo() {
             new ShardId(index, 0),
             false,
             PeerRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         test_0 = ShardRoutingHelper.initialize(test_0, "node1");
         test_0 = ShardRoutingHelper.moveToStarted(test_0);
@@ -112,7 +113,8 @@ public void testFillShardLevelInfo() {
             new ShardId(index, 1),
             false,
             PeerRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         test_1 = ShardRoutingHelper.initialize(test_1, "node2");
         test_1 = ShardRoutingHelper.moveToStarted(test_1);
diff --git a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
index 461fec42d9fc..d2a90aa7bfdc 100644
--- a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
@@ -9,7 +9,6 @@
 package org.elasticsearch.cluster;
 
 import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.Build;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
@@ -21,7 +20,6 @@
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleListener;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
@@ -55,7 +53,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
@@ -97,9 +97,7 @@ public void testEventuallyConnectsOnlyToAppliedNodes() throws Exception {
         final AtomicBoolean keepGoing = new AtomicBoolean(true);
         final Thread reconnectionThread = new Thread(() -> {
             while (keepGoing.get()) {
-                final PlainActionFuture<Void> future = new PlainActionFuture<>();
-                service.ensureConnections(() -> future.onResponse(null));
-                future.actionGet();
+                ensureConnections(service);
             }
         }, "reconnection thread");
         reconnectionThread.start();
@@ -109,34 +107,18 @@ public void testEventuallyConnectsOnlyToAppliedNodes() throws Exception {
         final boolean isDisrupting = randomBoolean();
         final Thread disruptionThread = new Thread(() -> {
             while (isDisrupting && keepGoing.get()) {
-                final Transport.Connection connection;
-                try {
-                    connection = transportService.getConnection(randomFrom(allNodes));
-                } catch (NodeNotConnectedException e) {
-                    continue;
-                }
-
-                final PlainActionFuture<Void> future = new PlainActionFuture<>();
-                connection.addRemovedListener(future);
-                connection.close();
-                future.actionGet(10, TimeUnit.SECONDS);
+                closeConnection(transportService, randomFrom(allNodes));
             }
         }, "disruption thread");
         disruptionThread.start();
 
         for (int i = 0; i < 10; i++) {
-            final DiscoveryNodes connectNodes = discoveryNodesFromList(randomSubsetOf(allNodes));
-            final PlainActionFuture<Void> future = new PlainActionFuture<>();
-            service.connectToNodes(connectNodes, () -> future.onResponse(null));
-            future.actionGet(10, TimeUnit.SECONDS);
-            final DiscoveryNodes disconnectExceptNodes = discoveryNodesFromList(randomSubsetOf(allNodes));
-            service.disconnectFromNodesExcept(disconnectExceptNodes);
+            connectToNodes(service, discoveryNodesFromList(randomSubsetOf(allNodes)));
+            service.disconnectFromNodesExcept(discoveryNodesFromList(randomSubsetOf(allNodes)));
         }
 
         final DiscoveryNodes nodes = discoveryNodesFromList(randomSubsetOf(allNodes));
-        final PlainActionFuture<Void> connectFuture = new PlainActionFuture<>();
-        service.connectToNodes(nodes, () -> connectFuture.onResponse(null));
-        connectFuture.actionGet(10, TimeUnit.SECONDS);
+        connectToNodes(service, nodes);
         service.disconnectFromNodesExcept(nodes);
 
         assertTrue(keepGoing.compareAndSet(true, false));
@@ -144,14 +126,59 @@ public void testEventuallyConnectsOnlyToAppliedNodes() throws Exception {
         disruptionThread.join();
 
         if (isDisrupting) {
-            final PlainActionFuture<Void> ensureFuture = new PlainActionFuture<>();
-            service.ensureConnections(() -> ensureFuture.onResponse(null));
-            ensureFuture.actionGet(10, TimeUnit.SECONDS);
+            ensureConnections(service);
         }
 
+        assertConnected(transportService, nodes);
         assertBusy(() -> assertConnectedExactlyToNodes(nodes));
     }
 
+    public void testConcurrentConnectAndDisconnect() throws Exception {
+        final NodeConnectionsService service = new NodeConnectionsService(Settings.EMPTY, threadPool, transportService);
+
+        final AtomicBoolean keepGoing = new AtomicBoolean(true);
+        final Thread reconnectionThread = new Thread(() -> {
+            while (keepGoing.get()) {
+                ensureConnections(service);
+            }
+        }, "reconnection thread");
+        reconnectionThread.start();
+
+        final var node = new DiscoveryNode("node", buildNewFakeTransportAddress(), Map.of(), Set.of(), Version.CURRENT);
+        final var nodes = discoveryNodesFromList(List.of(node));
+
+        final Thread disruptionThread = new Thread(() -> {
+            while (keepGoing.get()) {
+                closeConnection(transportService, node);
+            }
+        }, "disruption thread");
+        disruptionThread.start();
+
+        final var reconnectPermits = new Semaphore(1000);
+        final var reconnectThreads = 10;
+        final var reconnectCountDown = new CountDownLatch(reconnectThreads);
+        for (int i = 0; i < reconnectThreads; i++) {
+            threadPool.generic().execute(new Runnable() {
+                @Override
+                public void run() {
+                    if (reconnectPermits.tryAcquire()) {
+                        service.connectToNodes(nodes, () -> threadPool.generic().execute(this));
+                    } else {
+                        reconnectCountDown.countDown();
+                    }
+                }
+            });
+        }
+
+        assertTrue(reconnectCountDown.await(10, TimeUnit.SECONDS));
+        assertTrue(keepGoing.compareAndSet(true, false));
+        reconnectionThread.join();
+        disruptionThread.join();
+
+        ensureConnections(service);
+        assertConnectedExactlyToNodes(nodes);
+    }
+
     public void testPeriodicReconnection() {
         final Settings.Builder settings = Settings.builder();
         final long reconnectIntervalMillis;
@@ -234,30 +261,24 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti
         // connect to one node
         final DiscoveryNode node0 = new DiscoveryNode("node0", buildNewFakeTransportAddress(), Version.CURRENT);
         final DiscoveryNodes nodes0 = DiscoveryNodes.builder().add(node0).build();
-        final PlainActionFuture<Void> future0 = new PlainActionFuture<>();
-        service.connectToNodes(nodes0, () -> future0.onResponse(null));
-        future0.actionGet(10, TimeUnit.SECONDS);
+        connectToNodes(service, nodes0);
         assertConnectedExactlyToNodes(nodes0);
 
         // connection attempts to node0 block indefinitely
         final CyclicBarrier connectionBarrier = new CyclicBarrier(2);
         try {
-            nodeConnectionBlocks.put(node0, connectionBarrier::await);
+            nodeConnectionBlocks.put(node0, () -> connectionBarrier.await(10, TimeUnit.SECONDS));
             transportService.disconnectFromNode(node0);
 
             // can still connect to another node without blocking
             final DiscoveryNode node1 = new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT);
             final DiscoveryNodes nodes1 = DiscoveryNodes.builder().add(node1).build();
             final DiscoveryNodes nodes01 = DiscoveryNodes.builder(nodes0).add(node1).build();
-            final PlainActionFuture<Void> future1 = new PlainActionFuture<>();
-            service.connectToNodes(nodes01, () -> future1.onResponse(null));
-            future1.actionGet(10, TimeUnit.SECONDS);
+            connectToNodes(service, nodes01);
             assertConnectedExactlyToNodes(nodes1);
 
             // can also disconnect from node0 without blocking
-            final PlainActionFuture<Void> future2 = new PlainActionFuture<>();
-            service.connectToNodes(nodes1, () -> future2.onResponse(null));
-            future2.actionGet(10, TimeUnit.SECONDS);
+            connectToNodes(service, nodes1);
             service.disconnectFromNodesExcept(nodes1);
             assertConnectedExactlyToNodes(nodes1);
 
@@ -273,17 +294,15 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti
 
             // the reconnection is also blocked but the connection future doesn't wait, it completes straight away
             transportService.disconnectFromNode(node0);
-            final PlainActionFuture<Void> future4 = new PlainActionFuture<>();
-            service.connectToNodes(nodes01, () -> future4.onResponse(null));
-            future4.actionGet(10, TimeUnit.SECONDS);
+            connectToNodes(service, nodes01);
             assertConnectedExactlyToNodes(nodes1);
 
             // a blocked reconnection attempt doesn't also block the node from being deregistered
             service.disconnectFromNodesExcept(nodes1);
-            final PlainActionFuture<DiscoveryNode> disconnectFuture1 = new PlainActionFuture<>();
-            assertTrue(disconnectListenerRef.compareAndSet(null, disconnectFuture1));
-            connectionBarrier.await();
-            assertThat(disconnectFuture1.actionGet(10, TimeUnit.SECONDS), equalTo(node0)); // node0 connects briefly, must wait here
+            assertThat(PlainActionFuture.get(disconnectFuture1 -> {
+                assertTrue(disconnectListenerRef.compareAndSet(null, disconnectFuture1));
+                connectionBarrier.await(10, TimeUnit.SECONDS);
+            }, 10, TimeUnit.SECONDS), equalTo(node0)); // node0 connects briefly, must wait here
             assertConnectedExactlyToNodes(nodes1);
 
             // a blocked connection attempt to a new node also doesn't prevent an immediate deregistration
@@ -294,10 +313,10 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti
             service.disconnectFromNodesExcept(nodes1);
             assertConnectedExactlyToNodes(nodes1);
 
-            final PlainActionFuture<DiscoveryNode> disconnectFuture2 = new PlainActionFuture<>();
-            assertTrue(disconnectListenerRef.compareAndSet(null, disconnectFuture2));
-            connectionBarrier.await(10, TimeUnit.SECONDS);
-            assertThat(disconnectFuture2.actionGet(10, TimeUnit.SECONDS), equalTo(node0)); // node0 connects briefly, must wait here
+            assertThat(PlainActionFuture.get(disconnectFuture2 -> {
+                assertTrue(disconnectListenerRef.compareAndSet(null, disconnectFuture2));
+                connectionBarrier.await(10, TimeUnit.SECONDS);
+            }, 10, TimeUnit.SECONDS), equalTo(node0)); // node0 connects briefly, must wait here
             assertConnectedExactlyToNodes(nodes1);
             assertTrue(future5.isDone());
         } finally {
@@ -310,7 +329,7 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti
         reason = "testing that DEBUG-level logging is reasonable",
         value = "org.elasticsearch.cluster.NodeConnectionsService:DEBUG"
     )
-    public void testDebugLogging() throws IllegalAccessException {
+    public void testDebugLogging() {
         final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue();
 
         MockTransport transport = new MockTransport(deterministicTaskQueue.getThreadPool());
@@ -336,9 +355,7 @@ public void testDebugLogging() throws IllegalAccessException {
             transportService.disconnectFromNode(disconnectedNode);
         }
         MockLogAppender appender = new MockLogAppender();
-        try {
-            appender.start();
-            Loggers.addAppender(LogManager.getLogger("org.elasticsearch.cluster.NodeConnectionsService"), appender);
+        try (var ignored = appender.capturing(NodeConnectionsService.class)) {
             for (DiscoveryNode targetNode : targetNodes) {
                 if (disconnectedNodes.contains(targetNode)) {
                     appender.addExpectation(
@@ -379,10 +396,8 @@ public void testDebugLogging() throws IllegalAccessException {
 
             runTasksUntil(deterministicTaskQueue, CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.get(Settings.EMPTY).millis());
             appender.assertAllExpectationsMatched();
-        } finally {
-            Loggers.removeAppender(LogManager.getLogger("org.elasticsearch.cluster.NodeConnectionsService"), appender);
-            appender.stop();
         }
+
         for (DiscoveryNode disconnectedNode : disconnectedNodes) {
             transportService.disconnectFromNode(disconnectedNode);
         }
@@ -393,9 +408,7 @@ public void testDebugLogging() throws IllegalAccessException {
             transportService.disconnectFromNode(disconnectedNode);
         }
         appender = new MockLogAppender();
-        try {
-            appender.start();
-            Loggers.addAppender(LogManager.getLogger("org.elasticsearch.cluster.NodeConnectionsService"), appender);
+        try (var ignored = appender.capturing(NodeConnectionsService.class)) {
             for (DiscoveryNode targetNode : targetNodes) {
                 if (disconnectedNodes.contains(targetNode) && newTargetNodes.get(targetNode.getId()) != null) {
                     appender.addExpectation(
@@ -476,9 +489,6 @@ public void testDebugLogging() throws IllegalAccessException {
             service.connectToNodes(newTargetNodes, () -> {});
             deterministicTaskQueue.runAllRunnableTasks();
             appender.assertAllExpectationsMatched();
-        } finally {
-            Loggers.removeAppender(LogManager.getLogger("org.elasticsearch.cluster.NodeConnectionsService"), appender);
-            appender.stop();
         }
     }
 
@@ -573,6 +583,11 @@ public BoundTransportAddress boundAddress() {
             return null;
         }
 
+        @Override
+        public BoundTransportAddress boundRemoteIngressAddress() {
+            return null;
+        }
+
         @Override
         public Map<String, BoundTransportAddress> profileBoundAddresses() {
             return null;
@@ -706,4 +721,23 @@ public RequestHandlers getRequestHandlers() {
             return requestHandlers;
         }
     }
+
+    private static void connectToNodes(NodeConnectionsService service, DiscoveryNodes discoveryNodes) {
+        PlainActionFuture.get(future -> service.connectToNodes(discoveryNodes, () -> future.onResponse(null)), 10, TimeUnit.SECONDS);
+    }
+
+    private static void ensureConnections(NodeConnectionsService service) {
+        PlainActionFuture.get(future -> service.ensureConnections(() -> future.onResponse(null)), 10, TimeUnit.SECONDS);
+    }
+
+    private static void closeConnection(TransportService transportService, DiscoveryNode discoveryNode) {
+        try {
+            final var connection = transportService.getConnection(discoveryNode);
+            connection.close();
+            PlainActionFuture.get(connection::addRemovedListener, 10, TimeUnit.SECONDS);
+        } catch (NodeNotConnectedException e) {
+            // ok
+        }
+    }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/RepositoryCleanupInProgressTests.java b/server/src/test/java/org/elasticsearch/cluster/RepositoryCleanupInProgressTests.java
new file mode 100644
index 000000000000..84d695b858f2
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/RepositoryCleanupInProgressTests.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.cluster;
+
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
+import org.elasticsearch.test.ESTestCase;
+
+public class RepositoryCleanupInProgressTests extends ESTestCase {
+    public void testChunking() {
+        AbstractChunkedSerializingTestCase.assertChunkCount(
+            new RepositoryCleanupInProgress(
+                randomList(10, () -> new RepositoryCleanupInProgress.Entry(randomAlphaOfLength(10), randomNonNegativeLong()))
+            ),
+            i -> i.entries().size() + 2
+        );
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/RestoreInProgressTests.java b/server/src/test/java/org/elasticsearch/cluster/RestoreInProgressTests.java
new file mode 100644
index 000000000000..e56d747e3496
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/RestoreInProgressTests.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.cluster;
+
+import org.elasticsearch.snapshots.Snapshot;
+import org.elasticsearch.snapshots.SnapshotId;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class RestoreInProgressTests extends ESTestCase {
+    public void testChunking() throws IOException {
+        final var ripBuilder = new RestoreInProgress.Builder();
+        final var entryCount = between(0, 5);
+        for (int i = 0; i < entryCount; i++) {
+            ripBuilder.add(
+                new RestoreInProgress.Entry(
+                    "uuid-" + i,
+                    new Snapshot(randomAlphaOfLength(10), new SnapshotId(randomAlphaOfLength(10), randomAlphaOfLength(10))),
+                    randomFrom(RestoreInProgress.State.values()),
+                    randomBoolean(),
+                    List.of(),
+                    Map.of()
+                )
+            );
+        }
+
+        AbstractChunkedSerializingTestCase.assertChunkCount(ripBuilder.build(), ignored -> entryCount + 2);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/SnapshotDeletionsInProgressTests.java b/server/src/test/java/org/elasticsearch/cluster/SnapshotDeletionsInProgressTests.java
index b9946a2bd603..f7d18cb0aca5 100644
--- a/server/src/test/java/org/elasticsearch/cluster/SnapshotDeletionsInProgressTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/SnapshotDeletionsInProgressTests.java
@@ -9,7 +9,9 @@
 package org.elasticsearch.cluster;
 
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -38,7 +40,7 @@ public void testXContent() throws IOException {
         try (XContentBuilder builder = jsonBuilder()) {
             builder.humanReadable(true);
             builder.startObject();
-            sdip.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            ChunkedToXContent.wrapAsToXContent(sdip).toXContent(builder, ToXContent.EMPTY_PARAMS);
             builder.endObject();
             String json = Strings.toString(builder);
             assertThat(json, equalTo(XContentHelper.stripWhitespace("""
@@ -56,4 +58,22 @@ public void testXContent() throws IOException {
                 }""")));
         }
     }
+
+    public void testChunking() {
+        AbstractChunkedSerializingTestCase.assertChunkCount(
+            SnapshotDeletionsInProgress.of(
+                randomList(
+                    10,
+                    () -> new SnapshotDeletionsInProgress.Entry(
+                        Collections.emptyList(),
+                        randomAlphaOfLength(10),
+                        randomNonNegativeLong(),
+                        randomNonNegativeLong(),
+                        randomFrom(SnapshotDeletionsInProgress.State.values())
+                    )
+                )
+            ),
+            instance -> instance.getEntries().size() + 2
+        );
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java
index aa7f21825aaa..091e840f3325 100644
--- a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.action.shard.ShardStateAction.FailedShardEntry;
 import org.elasticsearch.cluster.action.shard.ShardStateAction.FailedShardUpdateTask;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -77,7 +78,7 @@ public void setUp() throws Exception {
                     .primaryTerm(0, randomIntBetween(2, 10))
             )
             .build();
-        routingTable = RoutingTable.builder().addAsNew(metadata.index(INDEX)).build();
+        routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(metadata.index(INDEX)).build();
         clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
             .routingTable(routingTable)
diff --git a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStartedClusterStateTaskExecutorTests.java b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStartedClusterStateTaskExecutorTests.java
index baf1132dbb4a..142b5228f7d9 100644
--- a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStartedClusterStateTaskExecutorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStartedClusterStateTaskExecutorTests.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardLongFieldRange;
 
-import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
@@ -142,9 +141,9 @@ public void testNonInitializingShardAreMarkedAsSuccessful() throws Exception {
         assertSame(clusterState, executeTasks(clusterState, tasks));
     }
 
-    public void testStartedShards() throws Exception {
+    public void testStartPrimary() throws Exception {
         final String indexName = "test";
-        final ClusterState clusterState = state(indexName, randomBoolean(), ShardRoutingState.INITIALIZING, ShardRoutingState.INITIALIZING);
+        final ClusterState clusterState = state(indexName, randomBoolean(), ShardRoutingState.INITIALIZING);
 
         final IndexMetadata indexMetadata = clusterState.metadata().index(indexName);
         final ShardId shardId = new ShardId(indexMetadata.getIndex(), 0);
@@ -152,35 +151,47 @@ public void testStartedShards() throws Exception {
         final ShardRouting primaryShard = clusterState.routingTable().shardRoutingTable(shardId).primaryShard();
         final String primaryAllocationId = primaryShard.allocationId().getId();
 
-        final List<StartedShardUpdateTask> tasks = new ArrayList<>();
-        tasks.add(
-            new StartedShardUpdateTask(
-                new StartedShardEntry(shardId, primaryAllocationId, primaryTerm, "test", ShardLongFieldRange.UNKNOWN),
-                createTestListener()
-            )
+        final var task = new StartedShardUpdateTask(
+            new StartedShardEntry(shardId, primaryAllocationId, primaryTerm, "test", ShardLongFieldRange.UNKNOWN),
+            createTestListener()
         );
-        if (randomBoolean()) {
-            final ShardRouting replicaShard = clusterState.routingTable().shardRoutingTable(shardId).replicaShards().iterator().next();
-            final String replicaAllocationId = replicaShard.allocationId().getId();
-            tasks.add(
-                new StartedShardUpdateTask(
-                    new StartedShardEntry(shardId, replicaAllocationId, primaryTerm, "test", ShardLongFieldRange.UNKNOWN),
-                    createTestListener()
-                )
-            );
-        }
 
-        final var resultingState = executeTasks(clusterState, tasks);
+        final var resultingState = executeTasks(clusterState, List.of(task));
         assertNotSame(clusterState, resultingState);
-        for (final var task : tasks) {
-            assertThat(
-                resultingState.routingTable()
-                    .shardRoutingTable(task.getEntry().shardId)
-                    .getByAllocationId(task.getEntry().allocationId)
-                    .state(),
-                is(ShardRoutingState.STARTED)
-            );
-        }
+        assertThat(
+            resultingState.routingTable()
+                .shardRoutingTable(task.getEntry().shardId)
+                .getByAllocationId(task.getEntry().allocationId)
+                .state(),
+            is(ShardRoutingState.STARTED)
+        );
+    }
+
+    public void testStartReplica() throws Exception {
+        final String indexName = "test";
+        final ClusterState clusterState = state(indexName, randomBoolean(), ShardRoutingState.STARTED, ShardRoutingState.INITIALIZING);
+
+        final IndexMetadata indexMetadata = clusterState.metadata().index(indexName);
+        final ShardId shardId = new ShardId(indexMetadata.getIndex(), 0);
+        final long primaryTerm = indexMetadata.primaryTerm(shardId.id());
+        final ShardRouting primaryShard = clusterState.routingTable().shardRoutingTable(shardId).primaryShard();
+
+        final ShardRouting replicaShard = clusterState.routingTable().shardRoutingTable(shardId).replicaShards().iterator().next();
+        final String replicaAllocationId = replicaShard.allocationId().getId();
+        final var task = new StartedShardUpdateTask(
+            new StartedShardEntry(shardId, replicaAllocationId, primaryTerm, "test", ShardLongFieldRange.UNKNOWN),
+            createTestListener()
+        );
+
+        final var resultingState = executeTasks(clusterState, List.of(task));
+        assertNotSame(clusterState, resultingState);
+        assertThat(
+            resultingState.routingTable()
+                .shardRoutingTable(task.getEntry().shardId)
+                .getByAllocationId(task.getEntry().allocationId)
+                .state(),
+            is(ShardRoutingState.STARTED)
+        );
     }
 
     public void testDuplicateStartsAreOkay() throws Exception {
@@ -215,12 +226,12 @@ public void testDuplicateStartsAreOkay() throws Exception {
         }
     }
 
-    public void testPrimaryTermsMismatch() throws Exception {
+    public void testPrimaryTermsMismatchOnPrimary() throws Exception {
         final String indexName = "test";
         final int shard = 0;
         final int primaryTerm = 2 + randomInt(200);
 
-        ClusterState clusterState = state(indexName, randomBoolean(), ShardRoutingState.INITIALIZING, ShardRoutingState.INITIALIZING);
+        ClusterState clusterState = state(indexName, randomBoolean(), ShardRoutingState.INITIALIZING);
         clusterState = ClusterState.builder(clusterState)
             .metadata(
                 Metadata.builder(clusterState.metadata())
@@ -272,8 +283,23 @@ public void testPrimaryTermsMismatch() throws Exception {
                     .state(),
                 is(ShardRoutingState.STARTED)
             );
-            clusterState = resultingState;
         }
+    }
+
+    public void testPrimaryTermsMismatchOnReplica() throws Exception {
+        final String indexName = "test";
+        final int shard = 0;
+        final int primaryTerm = 2 + randomInt(200);
+
+        ClusterState clusterState = state(indexName, randomBoolean(), ShardRoutingState.STARTED, ShardRoutingState.INITIALIZING);
+        clusterState = ClusterState.builder(clusterState)
+            .metadata(
+                Metadata.builder(clusterState.metadata())
+                    .put(IndexMetadata.builder(clusterState.metadata().index(indexName)).primaryTerm(shard, primaryTerm).build(), true)
+                    .build()
+            )
+            .build();
+        final ShardId shardId = new ShardId(clusterState.metadata().index(indexName).getIndex(), shard);
         {
             final long replicaPrimaryTerm = randomBoolean() ? primaryTerm : primaryTerm - 1;
             final String replicaAllocationId = clusterState.routingTable()
@@ -301,9 +327,9 @@ public void testPrimaryTermsMismatch() throws Exception {
         }
     }
 
-    public void testExpandsTimestampRange() throws Exception {
+    public void testExpandsTimestampRangeForPrimary() throws Exception {
         final String indexName = "test";
-        final ClusterState clusterState = state(indexName, randomBoolean(), ShardRoutingState.INITIALIZING, ShardRoutingState.INITIALIZING);
+        final ClusterState clusterState = state(indexName, randomBoolean(), ShardRoutingState.INITIALIZING);
 
         final IndexMetadata indexMetadata = clusterState.metadata().index(indexName);
         final ShardId shardId = new ShardId(indexMetadata.getIndex(), 0);
@@ -317,47 +343,66 @@ public void testExpandsTimestampRange() throws Exception {
             : randomBoolean() ? ShardLongFieldRange.EMPTY
             : ShardLongFieldRange.of(1606407943000L, 1606407944000L);
 
-        final List<StartedShardUpdateTask> tasks = new ArrayList<>();
-        tasks.add(
-            new StartedShardUpdateTask(
-                new StartedShardEntry(shardId, primaryAllocationId, primaryTerm, "test", shardTimestampRange),
-                createTestListener()
-            )
+        final var task = new StartedShardUpdateTask(
+            new StartedShardEntry(shardId, primaryAllocationId, primaryTerm, "test", shardTimestampRange),
+            createTestListener()
         );
-        if (randomBoolean()) {
-            final ShardRouting replicaShard = clusterState.routingTable().shardRoutingTable(shardId).replicaShards().iterator().next();
-            final String replicaAllocationId = replicaShard.allocationId().getId();
-            tasks.add(
-                new StartedShardUpdateTask(
-                    new StartedShardEntry(shardId, replicaAllocationId, primaryTerm, "test", shardTimestampRange),
-                    createTestListener()
-                )
-            );
-        }
-        final var resultingState = executeTasks(clusterState, tasks);
+
+        final var resultingState = executeTasks(clusterState, List.of(task));
         assertNotSame(clusterState, resultingState);
-        for (final var task : tasks) {
-            assertThat(
-                resultingState.routingTable()
-                    .shardRoutingTable(task.getEntry().shardId)
-                    .getByAllocationId(task.getEntry().allocationId)
-                    .state(),
-                is(ShardRoutingState.STARTED)
-            );
+        assertThat(
+            resultingState.routingTable()
+                .shardRoutingTable(task.getEntry().shardId)
+                .getByAllocationId(task.getEntry().allocationId)
+                .state(),
+            is(ShardRoutingState.STARTED)
+        );
 
-            final var timestampRange = resultingState.metadata().index(indexName).getTimestampRange();
-            if (shardTimestampRange == ShardLongFieldRange.UNKNOWN) {
-                assertThat(timestampRange, sameInstance(IndexLongFieldRange.UNKNOWN));
-            } else if (shardTimestampRange == ShardLongFieldRange.EMPTY) {
-                assertThat(timestampRange, sameInstance(IndexLongFieldRange.EMPTY));
-            } else {
-                assertTrue(timestampRange.isComplete());
-                assertThat(timestampRange.getMin(), equalTo(shardTimestampRange.getMin()));
-                assertThat(timestampRange.getMax(), equalTo(shardTimestampRange.getMax()));
-            }
+        final var timestampRange = resultingState.metadata().index(indexName).getTimestampRange();
+        if (shardTimestampRange == ShardLongFieldRange.UNKNOWN) {
+            assertThat(timestampRange, sameInstance(IndexLongFieldRange.UNKNOWN));
+        } else if (shardTimestampRange == ShardLongFieldRange.EMPTY) {
+            assertThat(timestampRange, sameInstance(IndexLongFieldRange.EMPTY));
+        } else {
+            assertTrue(timestampRange.isComplete());
+            assertThat(timestampRange.getMin(), equalTo(shardTimestampRange.getMin()));
+            assertThat(timestampRange.getMax(), equalTo(shardTimestampRange.getMax()));
         }
     }
 
+    public void testExpandsTimestampRangeForReplica() throws Exception {
+        final String indexName = "test";
+        final ClusterState clusterState = state(indexName, randomBoolean(), ShardRoutingState.STARTED, ShardRoutingState.INITIALIZING);
+
+        final IndexMetadata indexMetadata = clusterState.metadata().index(indexName);
+        final ShardId shardId = new ShardId(indexMetadata.getIndex(), 0);
+        final long primaryTerm = indexMetadata.primaryTerm(shardId.id());
+
+        assertThat(indexMetadata.getTimestampRange(), sameInstance(IndexLongFieldRange.UNKNOWN));
+
+        final ShardLongFieldRange shardTimestampRange = randomBoolean() ? ShardLongFieldRange.UNKNOWN
+            : randomBoolean() ? ShardLongFieldRange.EMPTY
+            : ShardLongFieldRange.of(1606407943000L, 1606407944000L);
+
+        final ShardRouting replicaShard = clusterState.routingTable().shardRoutingTable(shardId).replicaShards().iterator().next();
+        final String replicaAllocationId = replicaShard.allocationId().getId();
+        final var task = new StartedShardUpdateTask(
+            new StartedShardEntry(shardId, replicaAllocationId, primaryTerm, "test", shardTimestampRange),
+            createTestListener()
+        );
+        final var resultingState = executeTasks(clusterState, List.of(task));
+        assertNotSame(clusterState, resultingState);
+        assertThat(
+            resultingState.routingTable()
+                .shardRoutingTable(task.getEntry().shardId)
+                .getByAllocationId(task.getEntry().allocationId)
+                .state(),
+            is(ShardRoutingState.STARTED)
+        );
+
+        assertThat(resultingState.metadata().index(indexName).getTimestampRange(), sameInstance(IndexLongFieldRange.UNKNOWN));
+    }
+
     private ClusterState executeTasks(final ClusterState state, final List<StartedShardUpdateTask> tasks) throws Exception {
         return ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(state, executor, tasks);
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterBootstrapServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterBootstrapServiceTests.java
index f4878220bd86..8915550be539 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterBootstrapServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterBootstrapServiceTests.java
@@ -8,16 +8,15 @@
 package org.elasticsearch.cluster.coordination;
 
 import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockLogAppender;
@@ -661,13 +660,9 @@ public void testFailBootstrapNonMasterEligibleNodeWithSingleNodeDiscovery() {
         );
     }
 
-    public void testBootstrapStateLogging() throws IllegalAccessException {
+    public void testBootstrapStateLogging() {
         final var mockAppender = new MockLogAppender();
-        mockAppender.start();
-        final var serviceLogger = LogManager.getLogger(ClusterBootstrapService.class);
-        Loggers.addAppender(serviceLogger, mockAppender);
-
-        try {
+        try (var ignored = mockAppender.capturing(ClusterBootstrapService.class)) {
             mockAppender.addExpectation(
                 new MockLogAppender.SeenEventExpectation(
                     "fresh node message",
@@ -710,14 +705,18 @@ public void testBootstrapStateLogging() throws IllegalAccessException {
 
             mockAppender.assertAllExpectationsMatched();
 
+            final var warningMessagePattern = """
+                this node is locked into cluster UUID [test-uuid] but [cluster.initial_master_nodes] is set to [node1, node2]; \
+                remove this setting to avoid possible data loss caused by subsequent cluster bootstrap attempts; \
+                for further information see \
+                https://www.elastic.co/guide/en/elasticsearch/reference/*/important-settings.html#initial_master_nodes""";
+
             mockAppender.addExpectation(
                 new MockLogAppender.SeenEventExpectation(
                     "bootstrapped node message if bootstrapping still configured",
                     ClusterBootstrapService.class.getCanonicalName(),
                     Level.WARN,
-                    """
-                        this node is locked into cluster UUID [test-uuid] but [cluster.initial_master_nodes] is set to [node1, node2]; \
-                        remove this setting to avoid possible data loss caused by subsequent cluster bootstrap attempts"""
+                    warningMessagePattern
                 )
             );
 
@@ -731,9 +730,22 @@ public void testBootstrapStateLogging() throws IllegalAccessException {
 
             mockAppender.assertAllExpectationsMatched();
 
-        } finally {
-            Loggers.removeAppender(serviceLogger, mockAppender);
-            mockAppender.stop();
+            mockAppender.addExpectation(
+                new MockLogAppender.SeenEventExpectation(
+                    "bootstrapped node message if bootstrapping still configured",
+                    ClusterBootstrapService.class.getCanonicalName(),
+                    Level.WARN,
+                    warningMessagePattern
+                )
+            );
+
+            var startTime = deterministicTaskQueue.getCurrentTimeMillis();
+            while (deterministicTaskQueue.getCurrentTimeMillis() <= startTime + TimeValue.timeValueHours(12).millis()) {
+                deterministicTaskQueue.runAllRunnableTasks();
+                deterministicTaskQueue.advanceTime();
+            }
+
+            mockAppender.assertAllExpectationsMatched();
         }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelperTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelperTests.java
index d63141dd58d4..c471e28a3da2 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelperTests.java
@@ -7,6 +7,7 @@
  */
 package org.elasticsearch.cluster.coordination;
 
+import org.apache.logging.log4j.Level;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
@@ -25,6 +26,7 @@
 import org.elasticsearch.monitor.StatusInfo;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.elasticsearch.test.MockLogAppender;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -54,6 +56,9 @@ public class ClusterFormationFailureHelperTests extends ESTestCase {
 
     private static final ElectionStrategy electionStrategy = ElectionStrategy.DEFAULT_INSTANCE;
 
+    // Hard-coding the class name here because it is also mentioned in the troubleshooting docs, so should not be renamed without care.
+    private static final String LOGGER_NAME = "org.elasticsearch.cluster.coordination.ClusterFormationFailureHelper";
+
     public void testScheduling() {
         final long expectedDelayMillis;
         final Settings.Builder settingsBuilder = Settings.builder();
@@ -103,16 +108,32 @@ public void testScheduling() {
         final long startTimeMillis = deterministicTaskQueue.getCurrentTimeMillis();
         clusterFormationFailureHelper.start();
 
-        while (warningCount.get() == 0) {
-            assertTrue(clusterFormationFailureHelper.isRunning());
-            if (deterministicTaskQueue.hasRunnableTasks()) {
-                deterministicTaskQueue.runRandomTask();
-            } else {
-                deterministicTaskQueue.advanceTime();
+        var mockLogAppender = new MockLogAppender();
+        mockLogAppender.addExpectation(
+            new MockLogAppender.SeenEventExpectation("master not discovered", LOGGER_NAME, Level.WARN, "master not discovered")
+        );
+        mockLogAppender.addExpectation(
+            new MockLogAppender.SeenEventExpectation(
+                "troubleshooting link",
+                LOGGER_NAME,
+                Level.WARN,
+                "* for troubleshooting guidance, see "
+                    + "https://www.elastic.co/guide/en/elasticsearch/reference/*/discovery-troubleshooting.html*"
+            )
+        );
+        try (var ignored = mockLogAppender.capturing(ClusterFormationFailureHelper.class)) {
+            while (warningCount.get() == 0) {
+                assertTrue(clusterFormationFailureHelper.isRunning());
+                if (deterministicTaskQueue.hasRunnableTasks()) {
+                    deterministicTaskQueue.runRandomTask();
+                } else {
+                    deterministicTaskQueue.advanceTime();
+                }
             }
+            assertThat(warningCount.get(), is(1L));
+            assertThat(deterministicTaskQueue.getCurrentTimeMillis() - startTimeMillis, is(expectedDelayMillis));
+            mockLogAppender.assertAllExpectationsMatched();
         }
-        assertThat(warningCount.get(), is(1L));
-        assertThat(deterministicTaskQueue.getCurrentTimeMillis() - startTimeMillis, is(expectedDelayMillis));
 
         while (warningCount.get() < 5) {
             assertTrue(clusterFormationFailureHelper.isRunning());
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java
index 7d208c508395..1e9fb53e7049 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java
@@ -8,11 +8,9 @@
 package org.elasticsearch.cluster.coordination;
 
 import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.core.LogEvent;
-import org.apache.lucene.util.Constants;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AbstractNamedDiffable;
 import org.elasticsearch.cluster.ClusterState;
@@ -29,7 +27,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.Settings.Builder;
@@ -44,11 +41,13 @@
 import org.elasticsearch.test.MockLogAppender;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -117,7 +116,6 @@ public void testCanUpdateClusterStateAfterStabilisation() {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/89860")
     public void testDoesNotElectNonMasterNode() {
         try (Cluster cluster = new Cluster(randomIntBetween(1, 5), false, Settings.EMPTY)) {
             cluster.runRandomly();
@@ -567,7 +565,6 @@ public void testUnresponsiveLeaderDetectedEventually() {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/90158")
     public void testUnhealthyLeaderIsReplaced() {
         final AtomicReference<StatusInfo> nodeHealthServiceStatus = new AtomicReference<>(new StatusInfo(HEALTHY, "healthy-info"));
         final int initialClusterSize = between(1, 3);
@@ -1180,10 +1177,8 @@ class DelayedCustom extends AbstractNamedDiffable<ClusterState.Custom> implement
             }
 
             @Override
-            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-                builder.startObject();
-                builder.endObject();
-                return builder;
+            public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+                return Collections.emptyIterator();
             }
 
             @Override
@@ -1192,8 +1187,8 @@ public String getWriteableName() {
             }
 
             @Override
-            public Version getMinimalSupportedVersion() {
-                return Version.CURRENT;
+            public TransportVersion getMinimalSupportedVersion() {
+                return TransportVersion.CURRENT;
             }
 
             @Override
@@ -1487,12 +1482,8 @@ public void testNodeCannotJoinIfJoinValidationFailsOnMaster() {
         reason = "test includes assertions about ClusterBootstrapService logging",
         value = "org.elasticsearch.cluster.coordination.ClusterBootstrapService:INFO"
     )
-    public void testClusterUUIDLogging() throws IllegalAccessException {
+    public void testClusterUUIDLogging() {
         final var mockAppender = new MockLogAppender();
-        mockAppender.start();
-        final var serviceLogger = LogManager.getLogger(ClusterBootstrapService.class);
-        Loggers.addAppender(serviceLogger, mockAppender);
-
         mockAppender.addExpectation(
             new MockLogAppender.SeenEventExpectation(
                 "fresh node message",
@@ -1501,8 +1492,7 @@ public void testClusterUUIDLogging() throws IllegalAccessException {
                 "this node has not joined a bootstrapped cluster yet; [cluster.initial_master_nodes] is set to []"
             )
         );
-
-        try (var cluster = new Cluster(randomIntBetween(1, 3))) {
+        try (var ignored = mockAppender.capturing(ClusterBootstrapService.class); var cluster = new Cluster(randomIntBetween(1, 3))) {
             cluster.runRandomly();
             cluster.stabilise();
             mockAppender.assertAllExpectationsMatched();
@@ -1522,9 +1512,6 @@ public void testClusterUUIDLogging() throws IllegalAccessException {
             cluster.clusterNodes.replaceAll(cn -> cn == restartingNode ? cn.restartedNode() : cn);
             cluster.stabilise();
             mockAppender.assertAllExpectationsMatched();
-        } finally {
-            Loggers.removeAppender(serviceLogger, mockAppender);
-            mockAppender.stop();
         }
     }
 
@@ -1532,7 +1519,7 @@ public void testClusterUUIDLogging() throws IllegalAccessException {
         reason = "test includes assertions about Coordinator and JoinHelper logging",
         value = "org.elasticsearch.cluster.coordination.Coordinator:WARN,org.elasticsearch.cluster.coordination.JoinHelper:INFO"
     )
-    public void testNodeCannotJoinIfJoinPingValidationFailsOnMaster() throws IllegalAccessException {
+    public void testNodeCannotJoinIfJoinPingValidationFailsOnMaster() {
         try (Cluster cluster = new Cluster(randomIntBetween(1, 3))) {
             cluster.runRandomly();
             cluster.stabilise();
@@ -1544,12 +1531,7 @@ public void testNodeCannotJoinIfJoinPingValidationFailsOnMaster() throws Illegal
             final long previousClusterStateVersion = cluster.getAnyLeader().getLastAppliedClusterState().version();
 
             MockLogAppender mockAppender = new MockLogAppender();
-            mockAppender.start();
-            Logger joinLogger = LogManager.getLogger(JoinHelper.class);
-            Logger coordinatorLogger = LogManager.getLogger(Coordinator.class);
-            Loggers.addAppender(joinLogger, mockAppender);
-            Loggers.addAppender(coordinatorLogger, mockAppender);
-            try {
+            try (var ignored = mockAppender.capturing(JoinHelper.class, Coordinator.class)) {
                 mockAppender.addExpectation(
                     new MockLogAppender.SeenEventExpectation(
                         "failed to join",
@@ -1568,10 +1550,6 @@ public void testNodeCannotJoinIfJoinPingValidationFailsOnMaster() throws Illegal
                 );
                 cluster.runFor(10000, "failing joins");
                 mockAppender.assertAllExpectationsMatched();
-            } finally {
-                Loggers.removeAppender(coordinatorLogger, mockAppender);
-                Loggers.removeAppender(joinLogger, mockAppender);
-                mockAppender.stop();
             }
 
             assertTrue(addedNodes.stream().allMatch(ClusterNode::isCandidate));
@@ -1636,7 +1614,7 @@ public void testClusterCannotFormWithFailingJoinValidation() {
         reason = "test includes assertions about JoinHelper logging",
         value = "org.elasticsearch.cluster.coordination.JoinHelper:INFO"
     )
-    public void testCannotJoinClusterWithDifferentUUID() throws IllegalAccessException {
+    public void testCannotJoinClusterWithDifferentUUID() {
         try (Cluster cluster1 = new Cluster(randomIntBetween(1, 3))) {
             cluster1.runRandomly();
             cluster1.stabilise();
@@ -1658,18 +1636,12 @@ public void testCannotJoinClusterWithDifferentUUID() throws IllegalAccessExcepti
             cluster1.clusterNodes.add(newNode);
 
             MockLogAppender mockAppender = new MockLogAppender();
-            mockAppender.start();
             mockAppender.addExpectation(
                 new MockLogAppender.SeenEventExpectation("test1", JoinHelper.class.getCanonicalName(), Level.INFO, "*failed to join*")
             );
-            Logger joinLogger = LogManager.getLogger(JoinHelper.class);
-            Loggers.addAppender(joinLogger, mockAppender);
-            cluster1.runFor(DEFAULT_STABILISATION_TIME, "failing join validation");
-            try {
+            try (var ignored = mockAppender.capturing(JoinHelper.class)) {
+                cluster1.runFor(DEFAULT_STABILISATION_TIME, "failing join validation");
                 mockAppender.assertAllExpectationsMatched();
-            } finally {
-                Loggers.removeAppender(joinLogger, mockAppender);
-                mockAppender.stop();
             }
             assertEquals(0, newNode.getLastAppliedClusterState().version());
 
@@ -1688,7 +1660,7 @@ public void testCannotJoinClusterWithDifferentUUID() throws IllegalAccessExcepti
         reason = "test includes assertions about logging",
         value = "org.elasticsearch.cluster.coordination.Coordinator:WARN,org.elasticsearch.cluster.coordination.JoinHelper:INFO"
     )
-    public void testReportsConnectBackProblemsDuringJoining() throws IllegalAccessException {
+    public void testReportsConnectBackProblemsDuringJoining() {
         try (var cluster = new Cluster(3)) {
             cluster.runRandomly();
             cluster.stabilise();
@@ -1703,7 +1675,6 @@ public void testReportsConnectBackProblemsDuringJoining() throws IllegalAccessEx
             leader.addActionBlock(TransportService.HANDSHAKE_ACTION_NAME);
 
             final var mockAppender = new MockLogAppender();
-            mockAppender.start();
             mockAppender.addExpectation(
                 new MockLogAppender.SeenEventExpectation(
                     "connect-back failure",
@@ -1755,11 +1726,7 @@ public void assertMatched() {
                     assertTrue(matched);
                 }
             });
-            final var coordinatorLogger = LogManager.getLogger(Coordinator.class);
-            Loggers.addAppender(coordinatorLogger, mockAppender);
-            final var joinHelperLogger = LogManager.getLogger(JoinHelper.class);
-            Loggers.addAppender(joinHelperLogger, mockAppender);
-            try {
+            try (var ignored = mockAppender.capturing(Coordinator.class, JoinHelper.class)) {
                 cluster.runFor(
                     // This expects 8 tasks to be executed after PeerFinder handling wakeup:
                     //
@@ -1776,10 +1743,6 @@ public void assertMatched() {
                     "allowing time for join attempt"
                 );
                 mockAppender.assertAllExpectationsMatched();
-            } finally {
-                Loggers.removeAppender(joinHelperLogger, mockAppender);
-                Loggers.removeAppender(coordinatorLogger, mockAppender);
-                mockAppender.stop();
             }
 
             leader.clearActionBlocks();
@@ -1969,8 +1932,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.V_EMPTY;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.ZERO;
         }
 
         @Override
@@ -1979,8 +1942,8 @@ public void writeTo(StreamOutput out) throws IOException {
         }
 
         @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            return builder;
+        public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+            return Collections.emptyIterator();
         }
 
     }
@@ -2024,7 +1987,7 @@ public void testClusterRecoversAfterExceptionDuringSerialization() {
         reason = "testing ClusterFormationFailureHelper logging",
         value = "org.elasticsearch.cluster.coordination.ClusterFormationFailureHelper:WARN"
     )
-    public void testLogsWarningPeriodicallyIfClusterNotFormed() throws IllegalAccessException {
+    public void testLogsWarningPeriodicallyIfClusterNotFormed() {
         final long warningDelayMillis;
         final Settings settings;
         if (randomBoolean()) {
@@ -2061,9 +2024,7 @@ public void testLogsWarningPeriodicallyIfClusterNotFormed() throws IllegalAccess
 
             for (int i = scaledRandomIntBetween(1, 10); i >= 0; i--) {
                 final MockLogAppender mockLogAppender = new MockLogAppender();
-                try {
-                    mockLogAppender.start();
-                    Loggers.addAppender(LogManager.getLogger(ClusterFormationFailureHelper.class), mockLogAppender);
+                try (var ignored = mockLogAppender.capturing(ClusterFormationFailureHelper.class)) {
                     mockLogAppender.addExpectation(new MockLogAppender.LoggingExpectation() {
                         final Set<DiscoveryNode> nodesLogged = new HashSet<>();
 
@@ -2106,9 +2067,6 @@ public void assertMatched() {
                     });
                     cluster.runFor(warningDelayMillis + DEFAULT_DELAY_VARIABILITY, "waiting for warning to be emitted");
                     mockLogAppender.assertAllExpectationsMatched();
-                } finally {
-                    Loggers.removeAppender(LogManager.getLogger(ClusterFormationFailureHelper.class), mockLogAppender);
-                    mockLogAppender.stop();
                 }
             }
 
@@ -2122,7 +2080,7 @@ public void assertMatched() {
         reason = "testing warning of a single-node cluster having discovery seed hosts",
         value = "org.elasticsearch.cluster.coordination.Coordinator:WARN"
     )
-    public void testLogsWarningPeriodicallyIfSingleNodeClusterHasSeedHosts() throws IllegalAccessException {
+    public void testLogsWarningPeriodicallyIfSingleNodeClusterHasSeedHosts() {
         final long warningDelayMillis;
         final Settings settings;
         final String fakeSeedHost = buildNewFakeTransportAddress().toString();
@@ -2144,9 +2102,7 @@ public void testLogsWarningPeriodicallyIfSingleNodeClusterHasSeedHosts() throws
 
             for (int i = scaledRandomIntBetween(1, 10); i >= 0; i--) {
                 final MockLogAppender mockLogAppender = new MockLogAppender();
-                try {
-                    mockLogAppender.start();
-                    Loggers.addAppender(LogManager.getLogger(Coordinator.class), mockLogAppender);
+                try (var ignored = mockLogAppender.capturing(Coordinator.class)) {
                     mockLogAppender.addExpectation(new MockLogAppender.LoggingExpectation() {
                         String loggedClusterUuid;
 
@@ -2165,9 +2121,6 @@ public void assertMatched() {
                     });
                     cluster.runFor(warningDelayMillis + DEFAULT_DELAY_VARIABILITY, "waiting for warning to be emitted");
                     mockLogAppender.assertAllExpectationsMatched();
-                } finally {
-                    Loggers.removeAppender(LogManager.getLogger(Coordinator.class), mockLogAppender);
-                    mockLogAppender.stop();
                 }
             }
         }
@@ -2186,17 +2139,14 @@ public void testInvariantWhenTwoNodeClusterBecomesSingleNodeCluster() {
         value = "org.elasticsearch.cluster.coordination.LagDetector:DEBUG,"
             + "org.elasticsearch.cluster.coordination.Coordinator.CoordinatorPublication:INFO"
     )
-    public void testLogsMessagesIfPublicationDelayed() throws IllegalAccessException {
+    public void testLogsMessagesIfPublicationDelayed() {
         try (Cluster cluster = new Cluster(between(3, 5))) {
             cluster.runRandomly();
             cluster.stabilise();
             final ClusterNode brokenNode = cluster.getAnyNodeExcept(cluster.getAnyLeader());
 
             final MockLogAppender mockLogAppender = new MockLogAppender();
-            try {
-                mockLogAppender.start();
-                Loggers.addAppender(LogManager.getLogger(Coordinator.CoordinatorPublication.class), mockLogAppender);
-                Loggers.addAppender(LogManager.getLogger(LagDetector.class), mockLogAppender);
+            try (var ignored = mockLogAppender.capturing(Coordinator.CoordinatorPublication.class, LagDetector.class)) {
 
                 mockLogAppender.addExpectation(
                     new MockLogAppender.SeenEventExpectation(
@@ -2236,20 +2186,16 @@ public void testLogsMessagesIfPublicationDelayed() throws IllegalAccessException
                     )
                 );
 
-                if (Constants.WINDOWS == false) {
-                    // log messages containing control characters are hidden from the log assertions framework, and this includes the
-                    // `\r` that Windows uses in its line endings, so we only see this message on systems with `\n` line endings:
-                    mockLogAppender.addExpectation(
-                        new MockLogAppender.SeenEventExpectation(
-                            "hot threads from lagging node",
-                            LagDetector.class.getCanonicalName(),
-                            Level.DEBUG,
-                            "hot threads from node ["
-                                + brokenNode.getLocalNode().descriptionWithoutAttributes()
-                                + "] lagging at version [*] despite commit of cluster state version [*]:\nHot threads at*"
-                        )
-                    );
-                }
+                mockLogAppender.addExpectation(
+                    new MockLogAppender.SeenEventExpectation(
+                        "hot threads from lagging node",
+                        LagDetector.class.getCanonicalName(),
+                        Level.DEBUG,
+                        "hot threads from node ["
+                            + brokenNode.getLocalNode().descriptionWithoutAttributes()
+                            + "] lagging at version [*] despite commit of cluster state version [*]*"
+                    )
+                );
 
                 // drop the publication messages to one node, but then restore connectivity so it remains in the cluster and does not fail
                 // health checks
@@ -2277,10 +2223,6 @@ public String toString() {
                 );
 
                 mockLogAppender.assertAllExpectationsMatched();
-            } finally {
-                Loggers.removeAppender(LogManager.getLogger(Coordinator.CoordinatorPublication.class), mockLogAppender);
-                Loggers.removeAppender(LogManager.getLogger(LagDetector.class), mockLogAppender);
-                mockLogAppender.stop();
             }
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinReasonServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinReasonServiceTests.java
index 0a301f04374e..424a9503e3bc 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinReasonServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinReasonServiceTests.java
@@ -12,9 +12,11 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matcher;
 
 import java.util.Collections;
 import java.util.concurrent.atomic.AtomicLong;
@@ -40,25 +42,25 @@ public void testJoinReasonService() {
 
         final DiscoveryNodes withNode = DiscoveryNodes.builder(withoutNode).add(discoveryNode).build();
 
-        assertThat(joinReasonService.getJoinReason(discoveryNode, CANDIDATE), equalTo("completing election"));
-        assertThat(joinReasonService.getJoinReason(discoveryNode, LEADER), equalTo("joining"));
+        assertThat(joinReasonService.getJoinReason(discoveryNode, CANDIDATE), matches("completing election"));
+        assertThat(joinReasonService.getJoinReason(discoveryNode, LEADER), matches("joining"));
 
         joinReasonService.onClusterStateApplied(withoutNode);
 
-        assertThat(joinReasonService.getJoinReason(discoveryNode, CANDIDATE), equalTo("completing election"));
-        assertThat(joinReasonService.getJoinReason(discoveryNode, LEADER), equalTo("joining"));
+        assertThat(joinReasonService.getJoinReason(discoveryNode, CANDIDATE), matches("completing election"));
+        assertThat(joinReasonService.getJoinReason(discoveryNode, LEADER), matches("joining"));
 
         joinReasonService.onClusterStateApplied(withNode);
 
-        assertThat(joinReasonService.getJoinReason(discoveryNode, CANDIDATE), equalTo("completing election"));
-        assertThat(joinReasonService.getJoinReason(discoveryNode, LEADER), equalTo("rejoining"));
+        assertThat(joinReasonService.getJoinReason(discoveryNode, CANDIDATE), matches("completing election"));
+        assertThat(joinReasonService.getJoinReason(discoveryNode, LEADER), matches("rejoining"));
 
         joinReasonService.onClusterStateApplied(withoutNode);
         currentTimeMillis.addAndGet(1234L);
 
         assertThat(
             joinReasonService.getJoinReason(discoveryNode, LEADER),
-            equalTo("joining, removed [1.2s/1234ms] ago by [" + master.getName() + "]")
+            matchesNeedingGuidance("joining, removed [1.2s/1234ms] ago by [" + master.getName() + "]")
         );
 
         joinReasonService.onNodeRemoved(discoveryNode, "test removal");
@@ -66,7 +68,7 @@ public void testJoinReasonService() {
 
         assertThat(
             joinReasonService.getJoinReason(discoveryNode, LEADER),
-            equalTo("joining, removed [5.5s/5555ms] ago with reason [test removal]")
+            matchesNeedingGuidance("joining, removed [5.5s/5555ms] ago with reason [test removal]")
         );
 
         joinReasonService.onClusterStateApplied(withNode);
@@ -74,14 +76,14 @@ public void testJoinReasonService() {
 
         assertThat(
             joinReasonService.getJoinReason(discoveryNode, LEADER),
-            equalTo("joining, removed [0ms] ago by [" + master.getName() + "], [2] total removals")
+            matchesNeedingGuidance("joining, removed [0ms] ago by [" + master.getName() + "], [2] total removals")
         );
 
         joinReasonService.onNodeRemoved(discoveryNode, "second test removal");
 
         assertThat(
             joinReasonService.getJoinReason(discoveryNode, LEADER),
-            equalTo("joining, removed [0ms] ago with reason [second test removal], [2] total removals")
+            matchesNeedingGuidance("joining, removed [0ms] ago with reason [second test removal], [2] total removals")
         );
 
         final DiscoveryNode rebootedNode = new DiscoveryNode(
@@ -98,7 +100,7 @@ public void testJoinReasonService() {
 
         assertThat(
             joinReasonService.getJoinReason(rebootedNode, LEADER),
-            equalTo("joining after restart, removed [0ms] ago with reason [second test removal]")
+            matches("joining after restart, removed [0ms] ago with reason [second test removal]")
         );
 
         final DiscoveryNodes withRebootedNode = DiscoveryNodes.builder(withoutNode).add(rebootedNode).build();
@@ -108,7 +110,7 @@ public void testJoinReasonService() {
 
         assertThat(
             joinReasonService.getJoinReason(rebootedNode, LEADER),
-            equalTo("joining, removed [0ms] ago with reason [third test removal]")
+            matchesNeedingGuidance("joining, removed [0ms] ago with reason [third test removal]")
         );
 
         joinReasonService.onClusterStateApplied(withRebootedNode);
@@ -117,7 +119,7 @@ public void testJoinReasonService() {
 
         assertThat(
             joinReasonService.getJoinReason(rebootedNode, LEADER),
-            equalTo("joining, removed [0ms] ago with reason [fourth test removal], [2] total removals")
+            matchesNeedingGuidance("joining, removed [0ms] ago with reason [fourth test removal], [2] total removals")
         );
 
         joinReasonService.onClusterStateApplied(withRebootedNode);
@@ -126,12 +128,12 @@ public void testJoinReasonService() {
 
         assertThat(
             joinReasonService.getJoinReason(discoveryNode, LEADER),
-            equalTo("joining, removed [0ms] ago by [" + master.getName() + "]")
+            matchesNeedingGuidance("joining, removed [0ms] ago by [" + master.getName() + "]")
         );
 
         assertThat(
             joinReasonService.getJoinReason(rebootedNode, LEADER),
-            equalTo("joining after restart, removed [0ms] ago by [" + master.getName() + "]")
+            matches("joining after restart, removed [0ms] ago by [" + master.getName() + "]")
         );
     }
 
@@ -178,11 +180,14 @@ public void testCleanup() {
 
         // remove almost enough other nodes and verify that we're still tracking the target node
         joinReasonService.onClusterStateApplied(almostCleanupNodes);
-        assertThat(joinReasonService.getJoinReason(targetNode, LEADER), equalTo("joining, removed [1ms] ago with reason [test]"));
+        assertThat(
+            joinReasonService.getJoinReason(targetNode, LEADER),
+            matchesNeedingGuidance("joining, removed [1ms] ago with reason [test]")
+        );
 
         // remove one more node to trigger the cleanup and forget about the target node
         joinReasonService.onClusterStateApplied(cleanupNodes);
-        assertThat(joinReasonService.getJoinReason(targetNode, LEADER), equalTo("joining"));
+        assertThat(joinReasonService.getJoinReason(targetNode, LEADER), matches("joining"));
     }
 
     private DiscoveryNode randomDiscoveryNode() {
@@ -199,4 +204,12 @@ private DiscoveryNode randomDiscoveryNode() {
             Version.CURRENT
         );
     }
+
+    private static Matcher<JoinReason> matches(String message) {
+        return equalTo(new JoinReason(message, null));
+    }
+
+    private static Matcher<JoinReason> matchesNeedingGuidance(String message) {
+        return equalTo(new JoinReason(message, ReferenceDocs.UNSTABLE_CLUSTER_TROUBLESHOOTING));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinStatusTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinStatusTests.java
index cf67ecc02b16..36161da21b89 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinStatusTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinStatusTests.java
@@ -47,7 +47,7 @@ private JoinStatus mutateJoinStatus(JoinStatus originalJoinStatus) {
                 return new JoinStatus(
                     originalJoinStatus.remoteNode(),
                     originalJoinStatus.term(),
-                    randomAlphaOfLengthBetween(0, 30),
+                    randomValueOtherThan(originalJoinStatus.message(), () -> randomAlphaOfLengthBetween(0, 30)),
                     originalJoinStatus.age()
                 );
             }
@@ -55,8 +55,8 @@ private JoinStatus mutateJoinStatus(JoinStatus originalJoinStatus) {
                 return new JoinStatus(
                     originalJoinStatus.remoteNode(),
                     originalJoinStatus.term(),
-                    randomAlphaOfLength(30),
-                    randomNonNegativeTimeValue()
+                    originalJoinStatus.message(),
+                    randomValueOtherThan(originalJoinStatus.age(), this::randomNonNegativeTimeValue)
                 );
             }
             case 4 -> {
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinTaskExecutorTests.java
deleted file mode 100644
index 1554cd555b42..000000000000
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinTaskExecutorTests.java
+++ /dev/null
@@ -1,598 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.cluster.coordination;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.NotMasterException;
-import org.elasticsearch.cluster.block.ClusterBlocks;
-import org.elasticsearch.cluster.metadata.DesiredNodeWithStatus;
-import org.elasticsearch.cluster.metadata.DesiredNodes;
-import org.elasticsearch.cluster.metadata.DesiredNodesTestCase;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.cluster.metadata.Metadata;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodeRole;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.RerouteService;
-import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.service.ClusterStateTaskExecutorUtils;
-import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
-
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Stream;
-
-import static org.elasticsearch.cluster.metadata.DesiredNodesTestCase.assertDesiredNodesStatusIsCorrect;
-import static org.elasticsearch.cluster.metadata.DesiredNodesTestCase.randomDesiredNode;
-import static org.elasticsearch.test.VersionUtils.maxCompatibleVersion;
-import static org.elasticsearch.test.VersionUtils.randomCompatibleVersion;
-import static org.elasticsearch.test.VersionUtils.randomVersion;
-import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.lessThan;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyBoolean;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class JoinTaskExecutorTests extends ESTestCase {
-
-    private static final ActionListener<Void> NOT_COMPLETED_LISTENER = ActionListener.wrap(
-        () -> { throw new AssertionError("should not complete publication"); }
-    );
-
-    public void testPreventJoinClusterWithNewerIndices() {
-        Settings.builder().build();
-        Metadata.Builder metaBuilder = Metadata.builder();
-        IndexMetadata indexMetadata = IndexMetadata.builder("test")
-            .settings(settings(Version.CURRENT))
-            .numberOfShards(1)
-            .numberOfReplicas(1)
-            .build();
-        metaBuilder.put(indexMetadata, false);
-        Metadata metadata = metaBuilder.build();
-        JoinTaskExecutor.ensureIndexCompatibility(Version.CURRENT, metadata);
-
-        expectThrows(
-            IllegalStateException.class,
-            () -> JoinTaskExecutor.ensureIndexCompatibility(VersionUtils.getPreviousVersion(Version.CURRENT), metadata)
-        );
-    }
-
-    public void testPreventJoinClusterWithUnsupportedIndices() {
-        Settings.builder().build();
-        Metadata.Builder metaBuilder = Metadata.builder();
-        IndexMetadata indexMetadata = IndexMetadata.builder("test")
-            .settings(settings(Version.fromString("6.8.0"))) // latest V6 released version
-            .numberOfShards(1)
-            .numberOfReplicas(1)
-            .build();
-        metaBuilder.put(indexMetadata, false);
-        Metadata metadata = metaBuilder.build();
-        expectThrows(IllegalStateException.class, () -> JoinTaskExecutor.ensureIndexCompatibility(Version.CURRENT, metadata));
-    }
-
-    public void testPreventJoinClusterWithUnsupportedNodeVersions() {
-        DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
-        final Version version = randomCompatibleVersion(random(), Version.CURRENT);
-        builder.add(new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), version));
-        builder.add(new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), randomCompatibleVersion(random(), version)));
-        DiscoveryNodes nodes = builder.build();
-
-        final Version maxNodeVersion = nodes.getMaxNodeVersion();
-        final Version minNodeVersion = nodes.getMinNodeVersion();
-
-        final Version tooLow = Version.fromId(maxNodeVersion.minimumCompatibilityVersion().id - 100);
-        expectThrows(IllegalStateException.class, () -> {
-            if (randomBoolean()) {
-                JoinTaskExecutor.ensureNodesCompatibility(tooLow, nodes);
-            } else {
-                JoinTaskExecutor.ensureNodesCompatibility(tooLow, minNodeVersion, maxNodeVersion);
-            }
-        });
-
-        final Version oldVersion = randomValueOtherThanMany(
-            v -> v.onOrAfter(minNodeVersion),
-            () -> rarely() ? Version.fromId(minNodeVersion.id - 1) : randomVersion(random())
-        );
-        expectThrows(IllegalStateException.class, () -> JoinTaskExecutor.ensureVersionBarrier(oldVersion, minNodeVersion));
-
-        final Version minGoodVersion = maxNodeVersion.major == minNodeVersion.major ?
-        // we have to stick with the same major
-            minNodeVersion : maxNodeVersion.minimumCompatibilityVersion();
-        final Version justGood = randomVersionBetween(random(), minGoodVersion, maxCompatibleVersion(minNodeVersion));
-
-        if (randomBoolean()) {
-            JoinTaskExecutor.ensureNodesCompatibility(justGood, nodes);
-        } else {
-            JoinTaskExecutor.ensureNodesCompatibility(justGood, minNodeVersion, maxNodeVersion);
-        }
-    }
-
-    public void testSuccess() {
-        Settings.builder().build();
-        Metadata.Builder metaBuilder = Metadata.builder();
-        IndexMetadata indexMetadata = IndexMetadata.builder("test")
-            .settings(randomCompatibleVersionSettings())
-            .numberOfShards(1)
-            .numberOfReplicas(1)
-            .build();
-        metaBuilder.put(indexMetadata, false);
-        indexMetadata = IndexMetadata.builder("test1")
-            .settings(randomCompatibleVersionSettings())
-            .numberOfShards(1)
-            .numberOfReplicas(1)
-            .build();
-        metaBuilder.put(indexMetadata, false);
-        Metadata metadata = metaBuilder.build();
-        JoinTaskExecutor.ensureIndexCompatibility(Version.CURRENT, metadata);
-    }
-
-    public static Settings.Builder randomCompatibleVersionSettings() {
-        Settings.Builder builder = Settings.builder();
-        if (randomBoolean()) {
-            builder.put(IndexMetadata.SETTING_VERSION_CREATED, getRandomCompatibleVersion());
-            if (randomBoolean()) {
-                builder.put(IndexMetadata.SETTING_VERSION_COMPATIBILITY, getRandomCompatibleVersion());
-            }
-        } else {
-            builder.put(IndexMetadata.SETTING_VERSION_CREATED, randomFrom(Version.fromString("5.0.0"), Version.fromString("6.0.0")));
-            builder.put(IndexMetadata.SETTING_VERSION_COMPATIBILITY, getRandomCompatibleVersion());
-        }
-        return builder;
-    }
-
-    private static Version getRandomCompatibleVersion() {
-        return VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumIndexCompatibilityVersion(), Version.CURRENT);
-    }
-
-    public void testUpdatesNodeWithNewRoles() throws Exception {
-        // Node roles vary by version, and new roles are suppressed for BWC. This means we can receive a join from a node that's already
-        // in the cluster but with a different set of roles: the node didn't change roles, but the cluster state came via an older master.
-        // In this case we must properly process its join to ensure that the roles are correct.
-
-        final AllocationService allocationService = createAllocationService();
-        when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]);
-        final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
-
-        final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor(allocationService, rerouteService);
-
-        final DiscoveryNode masterNode = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT);
-
-        final DiscoveryNode actualNode = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT);
-        final DiscoveryNode bwcNode = new DiscoveryNode(
-            actualNode.getName(),
-            actualNode.getId(),
-            actualNode.getEphemeralId(),
-            actualNode.getHostName(),
-            actualNode.getHostAddress(),
-            actualNode.getAddress(),
-            actualNode.getAttributes(),
-            new HashSet<>(randomSubsetOf(actualNode.getRoles())),
-            actualNode.getVersion()
-        );
-        final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
-            .nodes(DiscoveryNodes.builder().add(masterNode).localNodeId(masterNode.getId()).masterNodeId(masterNode.getId()).add(bwcNode))
-            .build();
-
-        final var resultingState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(
-            clusterState,
-            joinTaskExecutor,
-            List.of(JoinTask.singleNode(actualNode, "test", NOT_COMPLETED_LISTENER, 0L))
-        );
-
-        assertThat(resultingState.getNodes().get(actualNode.getId()).getRoles(), equalTo(actualNode.getRoles()));
-    }
-
-    public void testRejectsStatesWithStaleTerm() {
-        final var allocationService = createAllocationService();
-        final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
-
-        final long executorTerm = randomLongBetween(0L, Long.MAX_VALUE - 1);
-        final var joinTaskExecutor = new JoinTaskExecutor(allocationService, rerouteService);
-
-        final var masterNode = new DiscoveryNode(UUIDs.randomBase64UUID(random()), buildNewFakeTransportAddress(), Version.CURRENT);
-        final var clusterState = ClusterState.builder(ClusterName.DEFAULT)
-            .nodes(DiscoveryNodes.builder().add(masterNode).localNodeId(masterNode.getId()).masterNodeId(masterNode.getId()).build())
-            .metadata(
-                Metadata.builder()
-                    .coordinationMetadata(CoordinationMetadata.builder().term(randomLongBetween(executorTerm + 1, Long.MAX_VALUE)).build())
-                    .build()
-            )
-            .build();
-
-        assertThat(
-            expectThrows(
-                NotMasterException.class,
-                () -> ClusterStateTaskExecutorUtils.executeHandlingResults(
-                    clusterState,
-                    joinTaskExecutor,
-                    randomBoolean()
-                        ? List.of(JoinTask.singleNode(masterNode, "test", NOT_COMPLETED_LISTENER, executorTerm))
-                        : List.of(
-                            JoinTask.completingElection(
-                                Stream.of(new JoinTask.NodeJoinTask(masterNode, "test", NOT_COMPLETED_LISTENER)),
-                                executorTerm
-                            )
-                        ),
-                    t -> fail("should not succeed"),
-                    (t, e) -> assertThat(e, instanceOf(NotMasterException.class))
-                )
-            ).getMessage(),
-            containsString("there is a newer master")
-        );
-    }
-
-    public void testRejectsStatesWithOtherMaster() {
-        final var allocationService = createAllocationService();
-        final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
-
-        final long executorTerm = randomNonNegativeLong();
-        final var joinTaskExecutor = new JoinTaskExecutor(allocationService, rerouteService);
-
-        final var masterNode = new DiscoveryNode(UUIDs.randomBase64UUID(random()), buildNewFakeTransportAddress(), Version.CURRENT);
-        final var localNode = new DiscoveryNode(UUIDs.randomBase64UUID(random()), buildNewFakeTransportAddress(), Version.CURRENT);
-        final var clusterState = ClusterState.builder(ClusterName.DEFAULT)
-            .nodes(
-                DiscoveryNodes.builder()
-                    .add(localNode)
-                    .add(masterNode)
-                    .localNodeId(localNode.getId())
-                    .masterNodeId(masterNode.getId())
-                    .build()
-            )
-            .metadata(
-                Metadata.builder()
-                    .coordinationMetadata(CoordinationMetadata.builder().term(randomLongBetween(0L, executorTerm)).build())
-                    .build()
-            )
-            .build();
-
-        assertThat(
-            expectThrows(
-                NotMasterException.class,
-                () -> ClusterStateTaskExecutorUtils.executeHandlingResults(
-                    clusterState,
-                    joinTaskExecutor,
-                    randomBoolean()
-                        ? List.of(JoinTask.singleNode(masterNode, "test", NOT_COMPLETED_LISTENER, executorTerm))
-                        : List.of(
-                            JoinTask.completingElection(
-                                Stream.of(new JoinTask.NodeJoinTask(masterNode, "test", NOT_COMPLETED_LISTENER)),
-                                executorTerm
-                            )
-                        ),
-                    t -> fail("should not succeed"),
-                    (t, e) -> assertThat(e, instanceOf(NotMasterException.class))
-                )
-            ).getMessage(),
-            containsString("not master for join request")
-        );
-    }
-
-    public void testRejectsStatesWithNoMasterIfNotBecomingMaster() {
-        final var allocationService = createAllocationService();
-        final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
-
-        final long executorTerm = randomNonNegativeLong();
-        final var joinTaskExecutor = new JoinTaskExecutor(allocationService, rerouteService);
-
-        final var masterNode = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT);
-        final var clusterState = ClusterState.builder(ClusterName.DEFAULT)
-            .nodes(DiscoveryNodes.builder().add(masterNode).localNodeId(masterNode.getId()).build())
-            .metadata(
-                Metadata.builder()
-                    .coordinationMetadata(CoordinationMetadata.builder().term(randomLongBetween(0L, executorTerm)).build())
-                    .build()
-            )
-            .build();
-
-        assertThat(
-            expectThrows(
-                NotMasterException.class,
-                () -> ClusterStateTaskExecutorUtils.executeHandlingResults(
-                    clusterState,
-                    joinTaskExecutor,
-                    List.of(JoinTask.singleNode(masterNode, "test", NOT_COMPLETED_LISTENER, executorTerm)),
-                    t -> fail("should not succeed"),
-                    (t, e) -> assertThat(e, instanceOf(NotMasterException.class))
-                )
-            ).getMessage(),
-            containsString("not master for join request")
-        );
-    }
-
-    public void testRemovesOlderNodeInstancesWhenBecomingMaster() throws Exception {
-        final var allocationService = createAllocationService();
-        final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
-
-        final long executorTerm = randomLongBetween(1, Long.MAX_VALUE);
-        final var joinTaskExecutor = new JoinTaskExecutor(allocationService, rerouteService);
-
-        final var masterNode = new DiscoveryNode(UUIDs.randomBase64UUID(random()), buildNewFakeTransportAddress(), Version.CURRENT);
-        final var otherNodeOld = new DiscoveryNode(UUIDs.randomBase64UUID(random()), buildNewFakeTransportAddress(), Version.CURRENT);
-        final var otherNodeNew = new DiscoveryNode(
-            otherNodeOld.getName(),
-            otherNodeOld.getId(),
-            UUIDs.randomBase64UUID(random()),
-            otherNodeOld.getHostName(),
-            otherNodeOld.getHostAddress(),
-            otherNodeOld.getAddress(),
-            otherNodeOld.getAttributes(),
-            otherNodeOld.getRoles(),
-            otherNodeOld.getVersion()
-        );
-
-        final var afterElectionClusterState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(
-            ClusterState.builder(ClusterName.DEFAULT)
-                .nodes(DiscoveryNodes.builder().add(masterNode).add(otherNodeOld).localNodeId(masterNode.getId()).build())
-                .blocks(ClusterBlocks.builder().addGlobalBlock(NoMasterBlockService.NO_MASTER_BLOCK_ALL).build())
-                .metadata(
-                    Metadata.builder()
-                        .coordinationMetadata(CoordinationMetadata.builder().term(randomLongBetween(0, executorTerm - 1)).build())
-                        .build()
-                )
-                .build(),
-            joinTaskExecutor,
-            List.of(
-                JoinTask.completingElection(
-                    Stream.of(
-                        new JoinTask.NodeJoinTask(masterNode, "test", NOT_COMPLETED_LISTENER),
-                        new JoinTask.NodeJoinTask(otherNodeNew, "test", NOT_COMPLETED_LISTENER)
-                    ),
-                    executorTerm
-                )
-            )
-        );
-        assertThat(afterElectionClusterState.term(), equalTo(executorTerm));
-        assertThat(afterElectionClusterState.nodes().getMasterNode(), equalTo(masterNode));
-        assertFalse(afterElectionClusterState.blocks().hasGlobalBlock(NoMasterBlockService.NO_MASTER_BLOCK_ALL));
-        assertThat(
-            "existing node should be replaced by new one in an election",
-            afterElectionClusterState.nodes().get(otherNodeOld.getId()).getEphemeralId(),
-            equalTo(otherNodeNew.getEphemeralId())
-        );
-
-        assertThat(
-            "existing node should not be replaced if not completing an election",
-            ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(
-                afterElectionClusterState,
-                joinTaskExecutor,
-                List.of(
-                    JoinTask.singleNode(masterNode, "test", NOT_COMPLETED_LISTENER, executorTerm),
-                    JoinTask.singleNode(otherNodeOld, "test", NOT_COMPLETED_LISTENER, executorTerm)
-                )
-            ).nodes().get(otherNodeNew.getId()).getEphemeralId(),
-            equalTo(otherNodeNew.getEphemeralId())
-        );
-    }
-
-    public void testUpdatesVotingConfigExclusionsIfNeeded() throws Exception {
-        final var allocationService = createAllocationService();
-        final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
-
-        final long executorTerm = randomLongBetween(1, Long.MAX_VALUE);
-        final var joinTaskExecutor = new JoinTaskExecutor(allocationService, rerouteService);
-
-        final var masterNode = new DiscoveryNode(UUIDs.randomBase64UUID(random()), buildNewFakeTransportAddress(), Version.CURRENT);
-        final var otherNode = new DiscoveryNode(
-            UUIDs.randomBase64UUID(random()),
-            UUIDs.randomBase64UUID(random()),
-            buildNewFakeTransportAddress(),
-            Map.of(),
-            Set.of(DiscoveryNodeRole.MASTER_ROLE),
-            Version.CURRENT
-        );
-
-        var clusterState = ClusterState.builder(ClusterName.DEFAULT)
-            .nodes(DiscoveryNodes.builder().add(masterNode).localNodeId(masterNode.getId()).build())
-            .blocks(ClusterBlocks.builder().addGlobalBlock(NoMasterBlockService.NO_MASTER_BLOCK_ALL).build())
-            .metadata(
-                Metadata.builder()
-                    .coordinationMetadata(
-                        CoordinationMetadata.builder()
-                            .term(randomLongBetween(0, executorTerm - 1))
-                            .addVotingConfigExclusion(
-                                new CoordinationMetadata.VotingConfigExclusion(
-                                    CoordinationMetadata.VotingConfigExclusion.MISSING_VALUE_MARKER,
-                                    otherNode.getName()
-                                )
-                            )
-                            .build()
-                    )
-                    .build()
-            )
-            .build();
-
-        logger.info("--> {}", clusterState);
-        logger.info("--> {}", otherNode);
-
-        if (randomBoolean()) {
-            clusterState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(
-                clusterState,
-                joinTaskExecutor,
-                List.of(
-                    JoinTask.completingElection(
-                        Stream.of(
-                            new JoinTask.NodeJoinTask(masterNode, "test", NOT_COMPLETED_LISTENER),
-                            new JoinTask.NodeJoinTask(otherNode, "test", NOT_COMPLETED_LISTENER)
-                        ),
-                        executorTerm
-                    )
-                )
-            );
-        } else {
-            clusterState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(
-                clusterState,
-                joinTaskExecutor,
-                List.of(
-                    JoinTask.completingElection(
-                        Stream.of(new JoinTask.NodeJoinTask(masterNode, "test", NOT_COMPLETED_LISTENER)),
-                        executorTerm
-                    )
-                )
-            );
-            clusterState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(
-                clusterState,
-                joinTaskExecutor,
-                List.of(JoinTask.singleNode(otherNode, "test", NOT_COMPLETED_LISTENER, executorTerm))
-            );
-        }
-
-        assertThat(clusterState.term(), equalTo(executorTerm));
-        assertThat(clusterState.nodes().getMasterNode(), equalTo(masterNode));
-        assertThat(
-            clusterState.coordinationMetadata().getVotingConfigExclusions().iterator().next().getNodeId(),
-            equalTo(otherNode.getId())
-        );
-    }
-
-    public void testIgnoresOlderTerms() throws Exception {
-        final var allocationService = createAllocationService();
-        final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
-
-        final long currentTerm = randomLongBetween(100, 1000);
-        final var joinTaskExecutor = new JoinTaskExecutor(allocationService, rerouteService);
-
-        final var masterNode = new DiscoveryNode(UUIDs.randomBase64UUID(random()), buildNewFakeTransportAddress(), Version.CURRENT);
-        final var clusterState = ClusterState.builder(ClusterName.DEFAULT)
-            .nodes(DiscoveryNodes.builder().add(masterNode).localNodeId(masterNode.getId()).masterNodeId(masterNode.getId()).build())
-            .metadata(Metadata.builder().coordinationMetadata(CoordinationMetadata.builder().term(currentTerm).build()).build())
-            .build();
-
-        var tasks = Stream.concat(
-            Stream.generate(() -> createRandomTask(masterNode, "outdated", randomLongBetween(0, currentTerm - 1)))
-                .limit(randomLongBetween(1, 10)),
-            Stream.of(createRandomTask(masterNode, "current", currentTerm))
-        ).toList();
-
-        ClusterStateTaskExecutorUtils.executeHandlingResults(
-            clusterState,
-            joinTaskExecutor,
-            tasks,
-            t -> assertThat(t.term(), equalTo(currentTerm)),
-            (t, e) -> {
-                assertThat(t.term(), lessThan(currentTerm));
-                assertThat(e, instanceOf(NotMasterException.class));
-            }
-        );
-    }
-
-    public void testDesiredNodesMembershipIsUpgradedWhenNewNodesJoin() throws Exception {
-        final var allocationService = createAllocationService();
-        final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
-        final var joinTaskExecutor = new JoinTaskExecutor(allocationService, rerouteService);
-
-        final var actualizedDesiredNodes = randomList(0, 5, this::createActualizedDesiredNode);
-        final var pendingDesiredNodes = randomList(0, 5, this::createPendingDesiredNode);
-        final var joiningDesiredNodes = randomList(1, 5, this::createPendingDesiredNode);
-
-        final List<DiscoveryNode> joiningNodes = joiningDesiredNodes.stream()
-            .map(desiredNode -> DesiredNodesTestCase.newDiscoveryNode(desiredNode.externalId()))
-            .toList();
-
-        final var clusterState = DesiredNodesTestCase.createClusterStateWithDiscoveryNodesAndDesiredNodes(
-            actualizedDesiredNodes,
-            pendingDesiredNodes,
-            joiningDesiredNodes,
-            true,
-            false
-        );
-        final var desiredNodes = DesiredNodes.latestFromClusterState(clusterState);
-
-        var tasks = joiningNodes.stream().map(node -> JoinTask.singleNode(node, "join", NOT_COMPLETED_LISTENER, 0L)).toList();
-
-        final var updatedClusterState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(clusterState, joinTaskExecutor, tasks);
-
-        final var updatedDesiredNodes = DesiredNodes.latestFromClusterState(clusterState);
-        assertThat(updatedDesiredNodes, is(notNullValue()));
-
-        assertThat(updatedDesiredNodes.nodes(), hasSize(desiredNodes.nodes().size()));
-        assertDesiredNodesStatusIsCorrect(
-            updatedClusterState,
-            Stream.concat(actualizedDesiredNodes.stream(), joiningDesiredNodes.stream()).map(DesiredNodeWithStatus::desiredNode).toList(),
-            pendingDesiredNodes.stream().map(DesiredNodeWithStatus::desiredNode).toList()
-        );
-    }
-
-    public void testDesiredNodesMembershipIsUpgradedWhenANewMasterIsElected() throws Exception {
-        final var allocationService = createAllocationService();
-        final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
-        final var joinTaskExecutor = new JoinTaskExecutor(allocationService, rerouteService);
-
-        final var actualizedDesiredNodes = randomList(1, 5, this::createPendingDesiredNode);
-        final var pendingDesiredNodes = randomList(0, 5, this::createPendingDesiredNode);
-
-        final var clusterState = DesiredNodesTestCase.createClusterStateWithDiscoveryNodesAndDesiredNodes(
-            actualizedDesiredNodes,
-            pendingDesiredNodes,
-            Collections.emptyList(),
-            false,
-            false
-        );
-        final var desiredNodes = DesiredNodes.latestFromClusterState(clusterState);
-
-        final var completingElectionTask = JoinTask.completingElection(
-            clusterState.nodes().stream().map(node -> new JoinTask.NodeJoinTask(node, "test", NOT_COMPLETED_LISTENER)),
-            1L
-        );
-
-        final var updatedClusterState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(
-            clusterState,
-            joinTaskExecutor,
-            List.of(completingElectionTask)
-        );
-
-        final var updatedDesiredNodes = DesiredNodes.latestFromClusterState(updatedClusterState);
-        assertThat(updatedDesiredNodes, is(notNullValue()));
-
-        assertThat(updatedDesiredNodes.nodes(), hasSize(desiredNodes.nodes().size()));
-        assertDesiredNodesStatusIsCorrect(
-            updatedClusterState,
-            actualizedDesiredNodes.stream().map(DesiredNodeWithStatus::desiredNode).toList(),
-            pendingDesiredNodes.stream().map(DesiredNodeWithStatus::desiredNode).toList()
-        );
-    }
-
-    private DesiredNodeWithStatus createActualizedDesiredNode() {
-        return new DesiredNodeWithStatus(randomDesiredNode(), DesiredNodeWithStatus.Status.ACTUALIZED);
-    }
-
-    private DesiredNodeWithStatus createPendingDesiredNode() {
-        return new DesiredNodeWithStatus(randomDesiredNode(), DesiredNodeWithStatus.Status.PENDING);
-    }
-
-    private static JoinTask createRandomTask(DiscoveryNode node, String reason, long term) {
-        return randomBoolean()
-            ? JoinTask.singleNode(node, reason, NOT_COMPLETED_LISTENER, term)
-            : JoinTask.completingElection(Stream.of(new JoinTask.NodeJoinTask(node, reason, NOT_COMPLETED_LISTENER)), term);
-    }
-
-    private static AllocationService createAllocationService() {
-        final var allocationService = mock(AllocationService.class);
-        when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]);
-        when(allocationService.disassociateDeadNodes(any(), anyBoolean(), any())).then(
-            invocationOnMock -> invocationOnMock.getArguments()[0]
-        );
-        return allocationService;
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinValidationServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinValidationServiceTests.java
index 0b0ee295cea9..812b020a86a8 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinValidationServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinValidationServiceTests.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.cluster.coordination;
 
 import org.elasticsearch.Build;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionListenerResponseHandler;
@@ -41,10 +42,11 @@
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Random;
 import java.util.Set;
@@ -232,8 +234,8 @@ public void testJoinValidationRejectsUnreadableClusterState() {
         class BadCustom implements SimpleDiffable<ClusterState.Custom>, ClusterState.Custom {
 
             @Override
-            public XContentBuilder toXContent(XContentBuilder builder, Params params) {
-                return builder;
+            public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+                return Collections.emptyIterator();
             }
 
             @Override
@@ -242,8 +244,8 @@ public String getWriteableName() {
             }
 
             @Override
-            public Version getMinimalSupportedVersion() {
-                return Version.CURRENT;
+            public TransportVersion getMinimalSupportedVersion() {
+                return TransportVersion.CURRENT;
             }
 
             @Override
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/LagDetectorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/LagDetectorTests.java
index 052da3dd2cfe..22ea808e00a6 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/LagDetectorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/LagDetectorTests.java
@@ -7,11 +7,17 @@
  */
 package org.elasticsearch.cluster.coordination;
 
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.ReferenceDocs;
+import org.elasticsearch.common.logging.ChunkedLoggingStreamTests;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.junit.Before;
 
 import java.util.Arrays;
@@ -243,4 +249,23 @@ public void testLagDetection() {
         deterministicTaskQueue.runAllTasksInTimeOrder();
         assertThat(failedNodes, empty()); // nodes added after a lag detector was started are also ignored
     }
+
+    @TestLogging(reason = "testing LagDetector logging", value = "org.elasticsearch.cluster.coordination.LagDetector:DEBUG")
+    public void testHotThreadsChunkedLoggingEncoding() {
+        final var node = new DiscoveryNode("test", buildNewFakeTransportAddress(), Version.CURRENT);
+        final var expectedBody = randomUnicodeOfLengthBetween(1, 20000);
+        assertEquals(
+            expectedBody,
+            ChunkedLoggingStreamTests.getDecodedLoggedBody(
+                LogManager.getLogger(LagDetector.class),
+                Level.DEBUG,
+                "hot threads from node ["
+                    + node.descriptionWithoutAttributes()
+                    + "] lagging at version [1] despite commit of cluster state version [2]",
+                ReferenceDocs.LAGGING_NODE_TROUBLESHOOTING,
+                new LagDetector.HotThreadsLoggingTask(node, 1, 2, expectedBody)::run
+            ).utf8ToString()
+        );
+    }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinExecutorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinExecutorTests.java
new file mode 100644
index 000000000000..bb657ff210c5
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinExecutorTests.java
@@ -0,0 +1,688 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.cluster.coordination;
+
+import org.apache.logging.log4j.Level;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateTaskConfig;
+import org.elasticsearch.cluster.NotMasterException;
+import org.elasticsearch.cluster.block.ClusterBlocks;
+import org.elasticsearch.cluster.metadata.DesiredNodeWithStatus;
+import org.elasticsearch.cluster.metadata.DesiredNodes;
+import org.elasticsearch.cluster.metadata.DesiredNodesTestCase;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RerouteService;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.service.ClusterStateTaskExecutorUtils;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.ReferenceDocs;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ClusterServiceUtils;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.MockLogAppender;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.cluster.metadata.DesiredNodesTestCase.assertDesiredNodesStatusIsCorrect;
+import static org.elasticsearch.cluster.metadata.DesiredNodesTestCase.randomDesiredNode;
+import static org.elasticsearch.test.VersionUtils.maxCompatibleVersion;
+import static org.elasticsearch.test.VersionUtils.randomCompatibleVersion;
+import static org.elasticsearch.test.VersionUtils.randomVersion;
+import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class NodeJoinExecutorTests extends ESTestCase {
+
+    private static final ActionListener<Void> NOT_COMPLETED_LISTENER = ActionListener.wrap(
+        () -> { throw new AssertionError("should not complete publication"); }
+    );
+
+    public void testPreventJoinClusterWithNewerIndices() {
+        Settings.builder().build();
+        Metadata.Builder metaBuilder = Metadata.builder();
+        IndexMetadata indexMetadata = IndexMetadata.builder("test")
+            .settings(settings(Version.CURRENT))
+            .numberOfShards(1)
+            .numberOfReplicas(1)
+            .build();
+        metaBuilder.put(indexMetadata, false);
+        Metadata metadata = metaBuilder.build();
+        NodeJoinExecutor.ensureIndexCompatibility(Version.CURRENT, metadata);
+
+        expectThrows(
+            IllegalStateException.class,
+            () -> NodeJoinExecutor.ensureIndexCompatibility(VersionUtils.getPreviousVersion(Version.CURRENT), metadata)
+        );
+    }
+
+    public void testPreventJoinClusterWithUnsupportedIndices() {
+        Settings.builder().build();
+        Metadata.Builder metaBuilder = Metadata.builder();
+        IndexMetadata indexMetadata = IndexMetadata.builder("test")
+            .settings(settings(Version.fromString("6.8.0"))) // latest V6 released version
+            .numberOfShards(1)
+            .numberOfReplicas(1)
+            .build();
+        metaBuilder.put(indexMetadata, false);
+        Metadata metadata = metaBuilder.build();
+        expectThrows(IllegalStateException.class, () -> NodeJoinExecutor.ensureIndexCompatibility(Version.CURRENT, metadata));
+    }
+
+    public void testPreventJoinClusterWithUnsupportedNodeVersions() {
+        DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
+        final Version version = randomCompatibleVersion(random(), Version.CURRENT);
+        builder.add(new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), version));
+        builder.add(new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), randomCompatibleVersion(random(), version)));
+        DiscoveryNodes nodes = builder.build();
+
+        final Version maxNodeVersion = nodes.getMaxNodeVersion();
+        final Version minNodeVersion = nodes.getMinNodeVersion();
+
+        final Version tooLow = Version.fromId(maxNodeVersion.minimumCompatibilityVersion().id - 100);
+        expectThrows(IllegalStateException.class, () -> {
+            if (randomBoolean()) {
+                NodeJoinExecutor.ensureNodesCompatibility(tooLow, nodes);
+            } else {
+                NodeJoinExecutor.ensureNodesCompatibility(tooLow, minNodeVersion, maxNodeVersion);
+            }
+        });
+
+        final Version oldVersion = randomValueOtherThanMany(
+            v -> v.onOrAfter(minNodeVersion),
+            () -> rarely() ? Version.fromId(minNodeVersion.id - 1) : randomVersion(random())
+        );
+        expectThrows(IllegalStateException.class, () -> NodeJoinExecutor.ensureVersionBarrier(oldVersion, minNodeVersion));
+
+        final Version minGoodVersion = maxNodeVersion.major == minNodeVersion.major ?
+        // we have to stick with the same major
+            minNodeVersion : maxNodeVersion.minimumCompatibilityVersion();
+        final Version justGood = randomVersionBetween(random(), minGoodVersion, maxCompatibleVersion(minNodeVersion));
+
+        if (randomBoolean()) {
+            NodeJoinExecutor.ensureNodesCompatibility(justGood, nodes);
+        } else {
+            NodeJoinExecutor.ensureNodesCompatibility(justGood, minNodeVersion, maxNodeVersion);
+        }
+    }
+
+    public void testSuccess() {
+        Settings.builder().build();
+        Metadata.Builder metaBuilder = Metadata.builder();
+        IndexMetadata indexMetadata = IndexMetadata.builder("test")
+            .settings(randomCompatibleVersionSettings())
+            .numberOfShards(1)
+            .numberOfReplicas(1)
+            .build();
+        metaBuilder.put(indexMetadata, false);
+        indexMetadata = IndexMetadata.builder("test1")
+            .settings(randomCompatibleVersionSettings())
+            .numberOfShards(1)
+            .numberOfReplicas(1)
+            .build();
+        metaBuilder.put(indexMetadata, false);
+        Metadata metadata = metaBuilder.build();
+        NodeJoinExecutor.ensureIndexCompatibility(Version.CURRENT, metadata);
+    }
+
+    public static Settings.Builder randomCompatibleVersionSettings() {
+        Settings.Builder builder = Settings.builder();
+        if (randomBoolean()) {
+            builder.put(IndexMetadata.SETTING_VERSION_CREATED, getRandomCompatibleVersion());
+            if (randomBoolean()) {
+                builder.put(IndexMetadata.SETTING_VERSION_COMPATIBILITY, getRandomCompatibleVersion());
+            }
+        } else {
+            builder.put(IndexMetadata.SETTING_VERSION_CREATED, randomFrom(Version.fromString("5.0.0"), Version.fromString("6.0.0")));
+            builder.put(IndexMetadata.SETTING_VERSION_COMPATIBILITY, getRandomCompatibleVersion());
+        }
+        return builder;
+    }
+
+    private static Version getRandomCompatibleVersion() {
+        return VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumIndexCompatibilityVersion(), Version.CURRENT);
+    }
+
+    private static final JoinReason TEST_REASON = new JoinReason("test", null);
+
+    public void testUpdatesNodeWithNewRoles() throws Exception {
+        // Node roles vary by version, and new roles are suppressed for BWC. This means we can receive a join from a node that's already
+        // in the cluster but with a different set of roles: the node didn't change roles, but the cluster state came via an older master.
+        // In this case we must properly process its join to ensure that the roles are correct.
+
+        final AllocationService allocationService = createAllocationService();
+        when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]);
+        final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
+
+        final NodeJoinExecutor executor = new NodeJoinExecutor(allocationService, rerouteService);
+
+        final DiscoveryNode masterNode = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT);
+
+        final DiscoveryNode actualNode = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT);
+        final DiscoveryNode bwcNode = new DiscoveryNode(
+            actualNode.getName(),
+            actualNode.getId(),
+            actualNode.getEphemeralId(),
+            actualNode.getHostName(),
+            actualNode.getHostAddress(),
+            actualNode.getAddress(),
+            actualNode.getAttributes(),
+            new HashSet<>(randomSubsetOf(actualNode.getRoles())),
+            actualNode.getVersion()
+        );
+        final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .nodes(DiscoveryNodes.builder().add(masterNode).localNodeId(masterNode.getId()).masterNodeId(masterNode.getId()).add(bwcNode))
+            .build();
+
+        final var resultingState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(
+            clusterState,
+            executor,
+            List.of(JoinTask.singleNode(actualNode, TEST_REASON, NOT_COMPLETED_LISTENER, 0L))
+        );
+
+        assertThat(resultingState.getNodes().get(actualNode.getId()).getRoles(), equalTo(actualNode.getRoles()));
+    }
+
+    public void testRejectsStatesWithStaleTerm() {
+        final var allocationService = createAllocationService();
+        final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
+
+        final long executorTerm = randomLongBetween(0L, Long.MAX_VALUE - 1);
+        final var executor = new NodeJoinExecutor(allocationService, rerouteService);
+
+        final var masterNode = new DiscoveryNode(UUIDs.randomBase64UUID(random()), buildNewFakeTransportAddress(), Version.CURRENT);
+        final var clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .nodes(DiscoveryNodes.builder().add(masterNode).localNodeId(masterNode.getId()).masterNodeId(masterNode.getId()).build())
+            .metadata(
+                Metadata.builder()
+                    .coordinationMetadata(CoordinationMetadata.builder().term(randomLongBetween(executorTerm + 1, Long.MAX_VALUE)).build())
+                    .build()
+            )
+            .build();
+
+        assertThat(
+            expectThrows(
+                NotMasterException.class,
+                () -> ClusterStateTaskExecutorUtils.executeHandlingResults(
+                    clusterState,
+                    executor,
+                    randomBoolean()
+                        ? List.of(JoinTask.singleNode(masterNode, TEST_REASON, NOT_COMPLETED_LISTENER, executorTerm))
+                        : List.of(
+                            JoinTask.completingElection(
+                                Stream.of(new JoinTask.NodeJoinTask(masterNode, TEST_REASON, NOT_COMPLETED_LISTENER)),
+                                executorTerm
+                            )
+                        ),
+                    t -> fail("should not succeed"),
+                    (t, e) -> assertThat(e, instanceOf(NotMasterException.class))
+                )
+            ).getMessage(),
+            containsString("there is a newer master")
+        );
+    }
+
+    public void testRejectsStatesWithOtherMaster() {
+        final var allocationService = createAllocationService();
+        final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
+
+        final long executorTerm = randomNonNegativeLong();
+        final var executor = new NodeJoinExecutor(allocationService, rerouteService);
+
+        final var masterNode = new DiscoveryNode(UUIDs.randomBase64UUID(random()), buildNewFakeTransportAddress(), Version.CURRENT);
+        final var localNode = new DiscoveryNode(UUIDs.randomBase64UUID(random()), buildNewFakeTransportAddress(), Version.CURRENT);
+        final var clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .nodes(
+                DiscoveryNodes.builder()
+                    .add(localNode)
+                    .add(masterNode)
+                    .localNodeId(localNode.getId())
+                    .masterNodeId(masterNode.getId())
+                    .build()
+            )
+            .metadata(
+                Metadata.builder()
+                    .coordinationMetadata(CoordinationMetadata.builder().term(randomLongBetween(0L, executorTerm)).build())
+                    .build()
+            )
+            .build();
+
+        assertThat(
+            expectThrows(
+                NotMasterException.class,
+                () -> ClusterStateTaskExecutorUtils.executeHandlingResults(
+                    clusterState,
+                    executor,
+                    randomBoolean()
+                        ? List.of(JoinTask.singleNode(masterNode, TEST_REASON, NOT_COMPLETED_LISTENER, executorTerm))
+                        : List.of(
+                            JoinTask.completingElection(
+                                Stream.of(new JoinTask.NodeJoinTask(masterNode, TEST_REASON, NOT_COMPLETED_LISTENER)),
+                                executorTerm
+                            )
+                        ),
+                    t -> fail("should not succeed"),
+                    (t, e) -> assertThat(e, instanceOf(NotMasterException.class))
+                )
+            ).getMessage(),
+            containsString("not master for join request")
+        );
+    }
+
+    public void testRejectsStatesWithNoMasterIfNotBecomingMaster() {
+        final var allocationService = createAllocationService();
+        final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
+
+        final long executorTerm = randomNonNegativeLong();
+        final var executor = new NodeJoinExecutor(allocationService, rerouteService);
+
+        final var masterNode = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT);
+        final var clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .nodes(DiscoveryNodes.builder().add(masterNode).localNodeId(masterNode.getId()).build())
+            .metadata(
+                Metadata.builder()
+                    .coordinationMetadata(CoordinationMetadata.builder().term(randomLongBetween(0L, executorTerm)).build())
+                    .build()
+            )
+            .build();
+
+        assertThat(
+            expectThrows(
+                NotMasterException.class,
+                () -> ClusterStateTaskExecutorUtils.executeHandlingResults(
+                    clusterState,
+                    executor,
+                    List.of(JoinTask.singleNode(masterNode, TEST_REASON, NOT_COMPLETED_LISTENER, executorTerm)),
+                    t -> fail("should not succeed"),
+                    (t, e) -> assertThat(e, instanceOf(NotMasterException.class))
+                )
+            ).getMessage(),
+            containsString("not master for join request")
+        );
+    }
+
+    public void testRemovesOlderNodeInstancesWhenBecomingMaster() throws Exception {
+        final var allocationService = createAllocationService();
+        final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
+
+        final long executorTerm = randomLongBetween(1, Long.MAX_VALUE);
+        final var executor = new NodeJoinExecutor(allocationService, rerouteService);
+
+        final var masterNode = new DiscoveryNode(UUIDs.randomBase64UUID(random()), buildNewFakeTransportAddress(), Version.CURRENT);
+        final var otherNodeOld = new DiscoveryNode(UUIDs.randomBase64UUID(random()), buildNewFakeTransportAddress(), Version.CURRENT);
+        final var otherNodeNew = new DiscoveryNode(
+            otherNodeOld.getName(),
+            otherNodeOld.getId(),
+            UUIDs.randomBase64UUID(random()),
+            otherNodeOld.getHostName(),
+            otherNodeOld.getHostAddress(),
+            otherNodeOld.getAddress(),
+            otherNodeOld.getAttributes(),
+            otherNodeOld.getRoles(),
+            otherNodeOld.getVersion()
+        );
+
+        final var afterElectionClusterState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(
+            ClusterState.builder(ClusterName.DEFAULT)
+                .nodes(DiscoveryNodes.builder().add(masterNode).add(otherNodeOld).localNodeId(masterNode.getId()).build())
+                .blocks(ClusterBlocks.builder().addGlobalBlock(NoMasterBlockService.NO_MASTER_BLOCK_ALL).build())
+                .metadata(
+                    Metadata.builder()
+                        .coordinationMetadata(CoordinationMetadata.builder().term(randomLongBetween(0, executorTerm - 1)).build())
+                        .build()
+                )
+                .build(),
+            executor,
+            List.of(
+                JoinTask.completingElection(
+                    Stream.of(
+                        new JoinTask.NodeJoinTask(masterNode, TEST_REASON, NOT_COMPLETED_LISTENER),
+                        new JoinTask.NodeJoinTask(otherNodeNew, TEST_REASON, NOT_COMPLETED_LISTENER)
+                    ),
+                    executorTerm
+                )
+            )
+        );
+        assertThat(afterElectionClusterState.term(), equalTo(executorTerm));
+        assertThat(afterElectionClusterState.nodes().getMasterNode(), equalTo(masterNode));
+        assertFalse(afterElectionClusterState.blocks().hasGlobalBlock(NoMasterBlockService.NO_MASTER_BLOCK_ALL));
+        assertThat(
+            "existing node should be replaced by new one in an election",
+            afterElectionClusterState.nodes().get(otherNodeOld.getId()).getEphemeralId(),
+            equalTo(otherNodeNew.getEphemeralId())
+        );
+
+        assertThat(
+            "existing node should not be replaced if not completing an election",
+            ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(
+                afterElectionClusterState,
+                executor,
+                List.of(
+                    JoinTask.singleNode(masterNode, TEST_REASON, NOT_COMPLETED_LISTENER, executorTerm),
+                    JoinTask.singleNode(otherNodeOld, TEST_REASON, NOT_COMPLETED_LISTENER, executorTerm)
+                )
+            ).nodes().get(otherNodeNew.getId()).getEphemeralId(),
+            equalTo(otherNodeNew.getEphemeralId())
+        );
+    }
+
+    public void testUpdatesVotingConfigExclusionsIfNeeded() throws Exception {
+        final var allocationService = createAllocationService();
+        final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
+
+        final long executorTerm = randomLongBetween(1, Long.MAX_VALUE);
+        final var executor = new NodeJoinExecutor(allocationService, rerouteService);
+
+        final var masterNode = new DiscoveryNode(UUIDs.randomBase64UUID(random()), buildNewFakeTransportAddress(), Version.CURRENT);
+        final var otherNode = new DiscoveryNode(
+            UUIDs.randomBase64UUID(random()),
+            UUIDs.randomBase64UUID(random()),
+            buildNewFakeTransportAddress(),
+            Map.of(),
+            Set.of(DiscoveryNodeRole.MASTER_ROLE),
+            Version.CURRENT
+        );
+
+        var clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .nodes(DiscoveryNodes.builder().add(masterNode).localNodeId(masterNode.getId()).build())
+            .blocks(ClusterBlocks.builder().addGlobalBlock(NoMasterBlockService.NO_MASTER_BLOCK_ALL).build())
+            .metadata(
+                Metadata.builder()
+                    .coordinationMetadata(
+                        CoordinationMetadata.builder()
+                            .term(randomLongBetween(0, executorTerm - 1))
+                            .addVotingConfigExclusion(
+                                new CoordinationMetadata.VotingConfigExclusion(
+                                    CoordinationMetadata.VotingConfigExclusion.MISSING_VALUE_MARKER,
+                                    otherNode.getName()
+                                )
+                            )
+                            .build()
+                    )
+                    .build()
+            )
+            .build();
+
+        logger.info("--> {}", clusterState);
+        logger.info("--> {}", otherNode);
+
+        if (randomBoolean()) {
+            clusterState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(
+                clusterState,
+                executor,
+                List.of(
+                    JoinTask.completingElection(
+                        Stream.of(
+                            new JoinTask.NodeJoinTask(masterNode, TEST_REASON, NOT_COMPLETED_LISTENER),
+                            new JoinTask.NodeJoinTask(otherNode, TEST_REASON, NOT_COMPLETED_LISTENER)
+                        ),
+                        executorTerm
+                    )
+                )
+            );
+        } else {
+            clusterState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(
+                clusterState,
+                executor,
+                List.of(
+                    JoinTask.completingElection(
+                        Stream.of(new JoinTask.NodeJoinTask(masterNode, TEST_REASON, NOT_COMPLETED_LISTENER)),
+                        executorTerm
+                    )
+                )
+            );
+            clusterState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(
+                clusterState,
+                executor,
+                List.of(JoinTask.singleNode(otherNode, TEST_REASON, NOT_COMPLETED_LISTENER, executorTerm))
+            );
+        }
+
+        assertThat(clusterState.term(), equalTo(executorTerm));
+        assertThat(clusterState.nodes().getMasterNode(), equalTo(masterNode));
+        assertThat(
+            clusterState.coordinationMetadata().getVotingConfigExclusions().iterator().next().getNodeId(),
+            equalTo(otherNode.getId())
+        );
+    }
+
+    public void testIgnoresOlderTerms() throws Exception {
+        final var allocationService = createAllocationService();
+        final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
+
+        final long currentTerm = randomLongBetween(100, 1000);
+        final var executor = new NodeJoinExecutor(allocationService, rerouteService);
+
+        final var masterNode = new DiscoveryNode(UUIDs.randomBase64UUID(random()), buildNewFakeTransportAddress(), Version.CURRENT);
+        final var clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .nodes(DiscoveryNodes.builder().add(masterNode).localNodeId(masterNode.getId()).masterNodeId(masterNode.getId()).build())
+            .metadata(Metadata.builder().coordinationMetadata(CoordinationMetadata.builder().term(currentTerm).build()).build())
+            .build();
+
+        var tasks = Stream.concat(
+            Stream.generate(() -> createRandomTask(masterNode, randomLongBetween(0, currentTerm - 1))).limit(randomLongBetween(1, 10)),
+            Stream.of(createRandomTask(masterNode, currentTerm))
+        ).toList();
+
+        ClusterStateTaskExecutorUtils.executeHandlingResults(
+            clusterState,
+            executor,
+            tasks,
+            t -> assertThat(t.term(), equalTo(currentTerm)),
+            (t, e) -> {
+                assertThat(t.term(), lessThan(currentTerm));
+                assertThat(e, instanceOf(NotMasterException.class));
+            }
+        );
+    }
+
+    public void testDesiredNodesMembershipIsUpgradedWhenNewNodesJoin() throws Exception {
+        final var allocationService = createAllocationService();
+        final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
+        final var executor = new NodeJoinExecutor(allocationService, rerouteService);
+
+        final var actualizedDesiredNodes = randomList(0, 5, this::createActualizedDesiredNode);
+        final var pendingDesiredNodes = randomList(0, 5, this::createPendingDesiredNode);
+        final var joiningDesiredNodes = randomList(1, 5, this::createPendingDesiredNode);
+
+        final List<DiscoveryNode> joiningNodes = joiningDesiredNodes.stream()
+            .map(desiredNode -> DesiredNodesTestCase.newDiscoveryNode(desiredNode.externalId()))
+            .toList();
+
+        final var clusterState = DesiredNodesTestCase.createClusterStateWithDiscoveryNodesAndDesiredNodes(
+            actualizedDesiredNodes,
+            pendingDesiredNodes,
+            joiningDesiredNodes,
+            true,
+            false
+        );
+        final var desiredNodes = DesiredNodes.latestFromClusterState(clusterState);
+
+        var tasks = joiningNodes.stream().map(node -> JoinTask.singleNode(node, TEST_REASON, NOT_COMPLETED_LISTENER, 0L)).toList();
+
+        final var updatedClusterState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(clusterState, executor, tasks);
+
+        final var updatedDesiredNodes = DesiredNodes.latestFromClusterState(clusterState);
+        assertThat(updatedDesiredNodes, is(notNullValue()));
+
+        assertThat(updatedDesiredNodes.nodes(), hasSize(desiredNodes.nodes().size()));
+        assertDesiredNodesStatusIsCorrect(
+            updatedClusterState,
+            Stream.concat(actualizedDesiredNodes.stream(), joiningDesiredNodes.stream()).map(DesiredNodeWithStatus::desiredNode).toList(),
+            pendingDesiredNodes.stream().map(DesiredNodeWithStatus::desiredNode).toList()
+        );
+    }
+
+    public void testDesiredNodesMembershipIsUpgradedWhenANewMasterIsElected() throws Exception {
+        final var allocationService = createAllocationService();
+        final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
+        final var executor = new NodeJoinExecutor(allocationService, rerouteService);
+
+        final var actualizedDesiredNodes = randomList(1, 5, this::createPendingDesiredNode);
+        final var pendingDesiredNodes = randomList(0, 5, this::createPendingDesiredNode);
+
+        final var clusterState = DesiredNodesTestCase.createClusterStateWithDiscoveryNodesAndDesiredNodes(
+            actualizedDesiredNodes,
+            pendingDesiredNodes,
+            Collections.emptyList(),
+            false,
+            false
+        );
+        final var desiredNodes = DesiredNodes.latestFromClusterState(clusterState);
+
+        final var completingElectionTask = JoinTask.completingElection(
+            clusterState.nodes().stream().map(node -> new JoinTask.NodeJoinTask(node, TEST_REASON, NOT_COMPLETED_LISTENER)),
+            1L
+        );
+
+        final var updatedClusterState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(
+            clusterState,
+            executor,
+            List.of(completingElectionTask)
+        );
+
+        final var updatedDesiredNodes = DesiredNodes.latestFromClusterState(updatedClusterState);
+        assertThat(updatedDesiredNodes, is(notNullValue()));
+
+        assertThat(updatedDesiredNodes.nodes(), hasSize(desiredNodes.nodes().size()));
+        assertDesiredNodesStatusIsCorrect(
+            updatedClusterState,
+            actualizedDesiredNodes.stream().map(DesiredNodeWithStatus::desiredNode).toList(),
+            pendingDesiredNodes.stream().map(DesiredNodeWithStatus::desiredNode).toList()
+        );
+    }
+
+    public void testPerNodeLogging() {
+        final AllocationService allocationService = createAllocationService();
+        when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]);
+        final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null);
+
+        final NodeJoinExecutor executor = new NodeJoinExecutor(allocationService, rerouteService);
+
+        final DiscoveryNode masterNode = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT);
+        final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .nodes(DiscoveryNodes.builder().add(masterNode).localNodeId(masterNode.getId()).masterNodeId(masterNode.getId()))
+            .build();
+
+        final MockLogAppender appender = new MockLogAppender();
+        final ThreadPool threadPool = new TestThreadPool("test");
+        try (
+            var ignored = appender.capturing(NodeJoinExecutor.class);
+            var clusterService = ClusterServiceUtils.createClusterService(clusterState, threadPool)
+        ) {
+            final var node1 = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT);
+            appender.addExpectation(
+                new MockLogAppender.SeenEventExpectation(
+                    "info message",
+                    LOGGER_NAME,
+                    Level.INFO,
+                    "node-join: [" + node1.descriptionWithoutAttributes() + "] with reason [" + TEST_REASON.message() + "]"
+                )
+            );
+            assertNull(
+                PlainActionFuture.<Void, RuntimeException>get(
+                    future -> clusterService.getMasterService()
+                        .submitStateUpdateTask(
+                            "test",
+                            JoinTask.singleNode(node1, TEST_REASON, future, 0L),
+                            ClusterStateTaskConfig.build(Priority.NORMAL),
+                            executor
+                        ),
+                    10,
+                    TimeUnit.SECONDS
+                )
+            );
+            appender.assertAllExpectationsMatched();
+
+            final var node2 = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT);
+            final var testReasonWithLink = new JoinReason("test", ReferenceDocs.UNSTABLE_CLUSTER_TROUBLESHOOTING);
+            appender.addExpectation(
+                new MockLogAppender.SeenEventExpectation(
+                    "warn message with troubleshooting link",
+                    LOGGER_NAME,
+                    Level.WARN,
+                    "node-join: ["
+                        + node2.descriptionWithoutAttributes()
+                        + "] with reason ["
+                        + testReasonWithLink.message()
+                        + "]; for troubleshooting guidance, see https://www.elastic.co/guide/en/elasticsearch/reference/*"
+                )
+            );
+            assertNull(
+                PlainActionFuture.<Void, RuntimeException>get(
+                    future -> clusterService.getMasterService()
+                        .submitStateUpdateTask(
+                            "test",
+                            JoinTask.singleNode(node2, testReasonWithLink, future, 0L),
+                            ClusterStateTaskConfig.build(Priority.NORMAL),
+                            executor
+                        ),
+                    10,
+                    TimeUnit.SECONDS
+                )
+            );
+            appender.assertAllExpectationsMatched();
+        } finally {
+            TestThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
+        }
+    }
+
+    private DesiredNodeWithStatus createActualizedDesiredNode() {
+        return new DesiredNodeWithStatus(randomDesiredNode(), DesiredNodeWithStatus.Status.ACTUALIZED);
+    }
+
+    private DesiredNodeWithStatus createPendingDesiredNode() {
+        return new DesiredNodeWithStatus(randomDesiredNode(), DesiredNodeWithStatus.Status.PENDING);
+    }
+
+    private static JoinTask createRandomTask(DiscoveryNode node, long term) {
+        return randomBoolean()
+            ? JoinTask.singleNode(node, TEST_REASON, NOT_COMPLETED_LISTENER, term)
+            : JoinTask.completingElection(Stream.of(new JoinTask.NodeJoinTask(node, TEST_REASON, NOT_COMPLETED_LISTENER)), term);
+    }
+
+    private static AllocationService createAllocationService() {
+        final var allocationService = mock(AllocationService.class);
+        when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]);
+        when(allocationService.disassociateDeadNodes(any(), anyBoolean(), any())).then(
+            invocationOnMock -> invocationOnMock.getArguments()[0]
+        );
+        return allocationService;
+    }
+
+    // Hard-coding the class name here because it is also mentioned in the troubleshooting docs, so should not be renamed without care.
+    private static final String LOGGER_NAME = "org.elasticsearch.cluster.coordination.NodeJoinExecutor";
+
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
index 01602deb53d3..0cb3384daed4 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.Build;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
@@ -25,7 +26,6 @@
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.BaseFuture;
 import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
@@ -56,6 +56,7 @@
 import java.util.Set;
 import java.util.concurrent.BrokenBarrierException;
 import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
@@ -250,29 +251,16 @@ protected DiscoveryNode newNode(int i, boolean master) {
         return new DiscoveryNode(prefix + i, i + "", buildNewFakeTransportAddress(), emptyMap(), roles, Version.CURRENT);
     }
 
-    static class SimpleFuture extends BaseFuture<Void> {
-        final String description;
+    private Future<Void> joinNodeAsync(final JoinRequest joinRequest) {
+        final PlainActionFuture<Void> future = new PlainActionFuture<>() {
+            private final String description = "join of " + joinRequest + "]";
 
-        SimpleFuture(String description) {
-            this.description = description;
-        }
-
-        public void markAsDone() {
-            set(null);
-        }
-
-        public void markAsFailed(Throwable t) {
-            setException(t);
-        }
-
-        @Override
-        public String toString() {
-            return "future [" + description + "]";
-        }
-    }
+            @Override
+            public String toString() {
+                return "future [" + description + "]";
+            }
+        };
 
-    private SimpleFuture joinNodeAsync(final JoinRequest joinRequest) {
-        final SimpleFuture future = new SimpleFuture("join of " + joinRequest + "]");
         logger.debug("starting {}", future);
         // clone the node before submitting to simulate an incoming join, which is guaranteed to have a new
         // disco node object serialized off the network
@@ -283,20 +271,20 @@ private SimpleFuture joinNodeAsync(final JoinRequest joinRequest) {
                 @Override
                 public void onResponse(TransportResponse transportResponse) {
                     logger.debug("{} completed", future);
-                    future.markAsDone();
+                    future.onResponse(null);
                 }
 
                 @Override
                 public void onFailure(Exception e) {
                     logger.error(() -> format("unexpected error for %s", future), e);
-                    future.markAsFailed(e);
+                    future.onFailure(e);
                 }
             };
 
             joinHandler.processMessageReceived(joinRequest, new TestTransportChannel(listener));
         } catch (Exception e) {
             logger.error(() -> format("unexpected error for %s", future), e);
-            future.markAsFailed(e);
+            future.onFailure(e);
         }
         return future;
     }
@@ -306,7 +294,7 @@ private void joinNode(final JoinRequest joinRequest) {
     }
 
     private void joinNodeAndRun(final JoinRequest joinRequest) {
-        SimpleFuture fut = joinNodeAsync(joinRequest);
+        Future<Void> fut = joinNodeAsync(joinRequest);
         deterministicTaskQueue.runAllRunnableTasks();
         assertTrue(fut.isDone());
         FutureUtils.get(fut);
@@ -325,7 +313,7 @@ public void testJoinWithHigherTermElectsLeader() {
         assertFalse(isLocalNodeElectedMaster());
         assertNull(coordinator.getStateForMasterService().nodes().getMasterNodeId());
         long newTerm = initialTerm + randomLongBetween(1, 10);
-        SimpleFuture fut = joinNodeAsync(
+        Future<Void> fut = joinNodeAsync(
             new JoinRequest(node1, newTerm, Optional.of(new Join(node1, node0, newTerm, initialTerm, initialVersion)))
         );
         assertEquals(Coordinator.Mode.LEADER, coordinator.getMode());
@@ -427,13 +415,13 @@ public void testJoinAccumulation() {
         );
         assertFalse(isLocalNodeElectedMaster());
         long newTerm = initialTerm + randomLongBetween(1, 10);
-        SimpleFuture futNode0 = joinNodeAsync(
+        Future<Void> futNode0 = joinNodeAsync(
             new JoinRequest(node0, newTerm, Optional.of(new Join(node0, node0, newTerm, initialTerm, initialVersion)))
         );
         deterministicTaskQueue.runAllRunnableTasks();
         assertFalse(futNode0.isDone());
         assertFalse(isLocalNodeElectedMaster());
-        SimpleFuture futNode1 = joinNodeAsync(
+        Future<Void> futNode1 = joinNodeAsync(
             new JoinRequest(node1, newTerm, Optional.of(new Join(node1, node0, newTerm, initialTerm, initialVersion)))
         );
         deterministicTaskQueue.runAllRunnableTasks();
@@ -603,7 +591,7 @@ public void testBecomeFollowerFailsPendingJoin() throws Exception {
             () -> new StatusInfo(HEALTHY, "healthy-info")
         );
         long newTerm = initialTerm + randomLongBetween(1, 10);
-        SimpleFuture fut = joinNodeAsync(
+        Future<Void> fut = joinNodeAsync(
             new JoinRequest(node0, newTerm, Optional.of(new Join(node0, node0, newTerm, initialTerm, initialVersion)))
         );
         deterministicTaskQueue.runAllRunnableTasks();
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeLeftExecutorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeLeftExecutorTests.java
new file mode 100644
index 000000000000..92dbc694a73a
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeLeftExecutorTests.java
@@ -0,0 +1,158 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.cluster.coordination;
+
+import org.apache.logging.log4j.Level;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateTaskConfig;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.service.ClusterStateTaskExecutorUtils;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.test.ClusterServiceUtils;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.MockLogAppender;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class NodeLeftExecutorTests extends ESTestCase {
+
+    public void testRemovingNonExistentNodes() throws Exception {
+        final NodeLeftExecutor executor = new NodeLeftExecutor(null);
+        final DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
+        final int nodes = randomIntBetween(2, 16);
+        for (int i = 0; i < nodes; i++) {
+            builder.add(node(i));
+        }
+        final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).nodes(builder).build();
+
+        final DiscoveryNodes.Builder removeBuilder = DiscoveryNodes.builder();
+        for (int i = nodes; i < nodes + randomIntBetween(1, 16); i++) {
+            removeBuilder.add(node(i));
+        }
+        final List<NodeLeftExecutor.Task> tasks = removeBuilder.build()
+            .stream()
+            .map(node -> new NodeLeftExecutor.Task(node, randomBoolean() ? "left" : "failed", () -> {}))
+            .toList();
+
+        assertSame(clusterState, ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(clusterState, executor, tasks));
+    }
+
+    public void testRerouteAfterRemovingNodes() throws Exception {
+        final AllocationService allocationService = mock(AllocationService.class);
+        when(allocationService.disassociateDeadNodes(any(ClusterState.class), eq(true), any(String.class))).thenAnswer(
+            im -> im.getArguments()[0]
+        );
+
+        final AtomicReference<ClusterState> remainingNodesClusterState = new AtomicReference<>();
+        final NodeLeftExecutor executor = new NodeLeftExecutor(allocationService) {
+            @Override
+            protected ClusterState remainingNodesClusterState(ClusterState currentState, DiscoveryNodes.Builder remainingNodesBuilder) {
+                remainingNodesClusterState.set(super.remainingNodesClusterState(currentState, remainingNodesBuilder));
+                return remainingNodesClusterState.get();
+            }
+        };
+
+        final DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
+        final int nodes = randomIntBetween(2, 16);
+        final List<NodeLeftExecutor.Task> tasks = new ArrayList<>();
+        // to ensure that there is at least one removal
+        boolean first = true;
+        for (int i = 0; i < nodes; i++) {
+            final DiscoveryNode node = node(i);
+            builder.add(node);
+            if (first || randomBoolean()) {
+                tasks.add(new NodeLeftExecutor.Task(node, randomBoolean() ? "left" : "failed", () -> {}));
+            }
+            first = false;
+        }
+        final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).nodes(builder).build();
+
+        final var resultingState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(clusterState, executor, tasks);
+
+        verify(allocationService).disassociateDeadNodes(eq(remainingNodesClusterState.get()), eq(true), any(String.class));
+
+        for (final NodeLeftExecutor.Task task : tasks) {
+            assertNull(resultingState.nodes().get(task.node().getId()));
+        }
+    }
+
+    public void testPerNodeLogging() {
+        final AllocationService allocationService = mock(AllocationService.class);
+        when(allocationService.disassociateDeadNodes(any(ClusterState.class), eq(true), any(String.class))).thenAnswer(
+            im -> im.getArguments()[0]
+        );
+        final var executor = new NodeLeftExecutor(allocationService);
+
+        final DiscoveryNode masterNode = new DiscoveryNode("master", buildNewFakeTransportAddress(), Version.CURRENT);
+        final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .nodes(
+                DiscoveryNodes.builder()
+                    .add(masterNode)
+                    .localNodeId("master")
+                    .masterNodeId("master")
+                    .add(new DiscoveryNode("other", buildNewFakeTransportAddress(), Version.CURRENT))
+            )
+            .build();
+
+        final MockLogAppender appender = new MockLogAppender();
+        final ThreadPool threadPool = new TestThreadPool("test");
+        try (
+            var ignored = appender.capturing(NodeLeftExecutor.class);
+            var clusterService = ClusterServiceUtils.createClusterService(clusterState, threadPool)
+        ) {
+            final var nodeToRemove = clusterState.nodes().get("other");
+            appender.addExpectation(
+                new MockLogAppender.SeenEventExpectation(
+                    "info message",
+                    LOGGER_NAME,
+                    Level.INFO,
+                    "node-left: [" + nodeToRemove.descriptionWithoutAttributes() + "] with reason [test reason]"
+                )
+            );
+            assertNull(
+                PlainActionFuture.<Void, RuntimeException>get(
+                    future -> clusterService.getMasterService()
+                        .submitStateUpdateTask(
+                            "test",
+                            new NodeLeftExecutor.Task(nodeToRemove, "test reason", () -> future.onResponse(null)),
+                            ClusterStateTaskConfig.build(Priority.NORMAL),
+                            executor
+                        )
+                )
+            );
+            appender.assertAllExpectationsMatched();
+        } finally {
+            TestThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
+        }
+    }
+
+    private static DiscoveryNode node(final int id) {
+        return new DiscoveryNode(Integer.toString(id), buildNewFakeTransportAddress(), Version.CURRENT);
+    }
+
+    // Hard-coding the class name here because it is also mentioned in the troubleshooting docs, so should not be renamed without care.
+    private static final String LOGGER_NAME = "org.elasticsearch.cluster.coordination.NodeLeftExecutor";
+
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeRemovalClusterStateTaskExecutorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeRemovalClusterStateTaskExecutorTests.java
deleted file mode 100644
index dcd5cd8826f8..000000000000
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeRemovalClusterStateTaskExecutorTests.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.cluster.coordination;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.service.ClusterStateTaskExecutorUtils;
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicReference;
-
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-public class NodeRemovalClusterStateTaskExecutorTests extends ESTestCase {
-
-    public void testRemovingNonExistentNodes() throws Exception {
-        final NodeRemovalClusterStateTaskExecutor executor = new NodeRemovalClusterStateTaskExecutor(null);
-        final DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
-        final int nodes = randomIntBetween(2, 16);
-        for (int i = 0; i < nodes; i++) {
-            builder.add(node(i));
-        }
-        final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).nodes(builder).build();
-
-        final DiscoveryNodes.Builder removeBuilder = DiscoveryNodes.builder();
-        for (int i = nodes; i < nodes + randomIntBetween(1, 16); i++) {
-            removeBuilder.add(node(i));
-        }
-        final List<NodeRemovalClusterStateTaskExecutor.Task> tasks = removeBuilder.build()
-            .stream()
-            .map(node -> new NodeRemovalClusterStateTaskExecutor.Task(node, randomBoolean() ? "left" : "failed", () -> {}))
-            .toList();
-
-        assertSame(clusterState, ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(clusterState, executor, tasks));
-    }
-
-    public void testRerouteAfterRemovingNodes() throws Exception {
-        final AllocationService allocationService = mock(AllocationService.class);
-        when(allocationService.disassociateDeadNodes(any(ClusterState.class), eq(true), any(String.class))).thenAnswer(
-            im -> im.getArguments()[0]
-        );
-
-        final AtomicReference<ClusterState> remainingNodesClusterState = new AtomicReference<>();
-        final NodeRemovalClusterStateTaskExecutor executor = new NodeRemovalClusterStateTaskExecutor(allocationService) {
-            @Override
-            protected ClusterState remainingNodesClusterState(ClusterState currentState, DiscoveryNodes.Builder remainingNodesBuilder) {
-                remainingNodesClusterState.set(super.remainingNodesClusterState(currentState, remainingNodesBuilder));
-                return remainingNodesClusterState.get();
-            }
-        };
-
-        final DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
-        final int nodes = randomIntBetween(2, 16);
-        final List<NodeRemovalClusterStateTaskExecutor.Task> tasks = new ArrayList<>();
-        // to ensure that there is at least one removal
-        boolean first = true;
-        for (int i = 0; i < nodes; i++) {
-            final DiscoveryNode node = node(i);
-            builder.add(node);
-            if (first || randomBoolean()) {
-                tasks.add(new NodeRemovalClusterStateTaskExecutor.Task(node, randomBoolean() ? "left" : "failed", () -> {}));
-            }
-            first = false;
-        }
-        final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).nodes(builder).build();
-
-        final var resultingState = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(clusterState, executor, tasks);
-
-        verify(allocationService).disassociateDeadNodes(eq(remainingNodesClusterState.get()), eq(true), any(String.class));
-
-        for (final NodeRemovalClusterStateTaskExecutor.Task task : tasks) {
-            assertNull(resultingState.nodes().get(task.node().getId()));
-        }
-    }
-
-    private DiscoveryNode node(final int id) {
-        return new DiscoveryNode(Integer.toString(id), buildNewFakeTransportAddress(), Version.CURRENT);
-    }
-
-}
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/PublicationTransportHandlerTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/PublicationTransportHandlerTests.java
index 52cc92f73c74..a4b6b1f7fe87 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/PublicationTransportHandlerTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/PublicationTransportHandlerTests.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.IncompatibleClusterStateVersionException;
 import org.elasticsearch.cluster.coordination.CoordinationMetadata.VotingConfiguration;
+import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -30,6 +31,7 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.MockPageCacheRecycler;
+import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.tasks.Task;
@@ -42,17 +44,23 @@
 import org.elasticsearch.transport.BytesRefRecycler;
 import org.elasticsearch.transport.BytesTransportRequest;
 import org.elasticsearch.transport.RemoteTransportException;
+import org.elasticsearch.transport.TestTransportChannel;
 import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.cluster.service.MasterService.STATE_UPDATE_ACTION_NAME;
@@ -129,7 +137,7 @@ private static boolean isDiff(BytesTransportRequest request, DiscoveryNode node)
                 if (compressor != null) {
                     in = new InputStreamStreamInput(compressor.threadLocalInputStream(in));
                 }
-                in.setVersion(node.getVersion());
+                in.setTransportVersion(node.getVersion().transportVersion);
                 return in.readBoolean() == false;
             } finally {
                 IOUtils.close(in);
@@ -330,4 +338,170 @@ public void writeTo(StreamOutput out) throws IOException {
         }
     }
 
+    public void testIncludesLastCommittedFieldsInDiffSerialization() {
+        final var deterministicTaskQueue = new DeterministicTaskQueue();
+        final var threadPool = deterministicTaskQueue.getThreadPool();
+
+        final var transportsByNode = new HashMap<DiscoveryNode, MockTransport>();
+        final var transportHandlersByNode = new HashMap<DiscoveryNode, PublicationTransportHandler>();
+        final var transportServicesByNode = new HashMap<DiscoveryNode, TransportService>();
+        final var receivedStateRef = new AtomicReference<ClusterState>();
+        final var completed = new AtomicBoolean();
+
+        final var localNode = new DiscoveryNode("localNode", buildNewFakeTransportAddress(), Version.CURRENT);
+        final var otherNode = new DiscoveryNode(
+            "otherNode",
+            buildNewFakeTransportAddress(),
+            VersionUtils.randomCompatibleVersion(random(), Version.CURRENT)
+        );
+        for (final var discoveryNode : List.of(localNode, otherNode)) {
+            final var transport = new MockTransport() {
+                @Override
+                protected void onSendRequest(long requestId, String action, TransportRequest request, DiscoveryNode node) {
+                    @SuppressWarnings("unchecked")
+                    final var context = (ResponseContext<TransportResponse>) getResponseHandlers().remove(requestId);
+                    try {
+                        transportsByNode.get(node)
+                            .getRequestHandlers()
+                            .getHandler(action)
+                            .getHandler()
+                            .messageReceived(request, new TestTransportChannel(new ActionListener<>() {
+                                @Override
+                                public void onResponse(TransportResponse transportResponse) {
+                                    context.handler().handleResponse(transportResponse);
+                                }
+
+                                @Override
+                                public void onFailure(Exception e) {
+                                    throw new AssertionError("unexpected", e);
+                                }
+                            }), new Task(randomNonNegativeLong(), "test", "test", "", TaskId.EMPTY_TASK_ID, Map.of()));
+                    } catch (IncompatibleClusterStateVersionException e) {
+                        context.handler().handleException(new RemoteTransportException("wrapped", e));
+                    } catch (Exception e) {
+                        throw new AssertionError("unexpected", e);
+                    }
+                }
+            };
+            transportsByNode.put(discoveryNode, transport);
+
+            final var transportService = transport.createTransportService(
+                Settings.EMPTY,
+                threadPool,
+                TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+                ignored -> discoveryNode,
+                null,
+                Set.of()
+            );
+            transportServicesByNode.put(discoveryNode, transportService);
+
+            final var publicationTransportHandler = new PublicationTransportHandler(
+                transportService,
+                writableRegistry(),
+                publishRequest -> {
+                    assertTrue(receivedStateRef.compareAndSet(null, publishRequest.getAcceptedState()));
+                    return new PublishWithJoinResponse(
+                        new PublishResponse(publishRequest.getAcceptedState().term(), publishRequest.getAcceptedState().version()),
+                        Optional.empty()
+                    );
+                }
+            );
+            transportHandlersByNode.put(discoveryNode, publicationTransportHandler);
+        }
+
+        for (final var transportService : transportServicesByNode.values()) {
+            transportService.start();
+            transportService.acceptIncomingRequests();
+        }
+
+        threadPool.getThreadContext().markAsSystemContext();
+
+        final var clusterState0 = ClusterState.builder(ClusterState.EMPTY_STATE)
+            .nodes(DiscoveryNodes.builder().add(localNode).add(otherNode).localNodeId(localNode.getId()).masterNodeId(localNode.getId()))
+            .metadata(
+                Metadata.builder()
+                    .coordinationMetadata(
+                        CoordinationMetadata.builder().lastAcceptedConfiguration(VotingConfiguration.of(localNode)).build()
+                    )
+                    .generateClusterUuidIfNeeded()
+            )
+            .build();
+
+        final ClusterState receivedState0;
+        var context0 = transportHandlersByNode.get(localNode)
+            .newPublicationContext(
+                new ClusterStatePublicationEvent(
+                    new BatchSummary("test"),
+                    clusterState0,
+                    clusterState0,
+                    new Task(randomNonNegativeLong(), "test", "test", "", TaskId.EMPTY_TASK_ID, Map.of()),
+                    0L,
+                    0L
+                )
+            );
+        try {
+            context0.sendPublishRequest(
+                otherNode,
+                new PublishRequest(clusterState0),
+                ActionListener.wrap(() -> assertTrue(completed.compareAndSet(false, true)))
+            );
+            assertTrue(completed.getAndSet(false));
+            receivedState0 = receivedStateRef.getAndSet(null);
+            assertEquals(clusterState0.stateUUID(), receivedState0.stateUUID());
+            assertEquals(otherNode, receivedState0.nodes().getLocalNode());
+            assertFalse(receivedState0.metadata().clusterUUIDCommitted());
+            assertEquals(VotingConfiguration.of(), receivedState0.getLastCommittedConfiguration());
+            final var receivedStateStats = transportHandlersByNode.get(otherNode).stats();
+            assertEquals(0, receivedStateStats.getCompatibleClusterStateDiffReceivedCount());
+            assertEquals(1, receivedStateStats.getIncompatibleClusterStateDiffReceivedCount());
+            assertEquals(1, receivedStateStats.getFullClusterStateReceivedCount());
+        } finally {
+            context0.decRef();
+        }
+
+        final var committedClusterState0 = ClusterState.builder(clusterState0)
+            .metadata(clusterState0.metadata().withLastCommittedValues(true, clusterState0.getLastAcceptedConfiguration()))
+            .build();
+        assertEquals(clusterState0.stateUUID(), committedClusterState0.stateUUID());
+        assertEquals(clusterState0.term(), committedClusterState0.term());
+        assertEquals(clusterState0.version(), committedClusterState0.version());
+
+        final var clusterState1 = ClusterState.builder(committedClusterState0).incrementVersion().build();
+        assertSame(committedClusterState0.metadata(), clusterState1.metadata());
+
+        var context1 = transportHandlersByNode.get(localNode)
+            .newPublicationContext(
+                new ClusterStatePublicationEvent(
+                    new BatchSummary("test"),
+                    committedClusterState0,
+                    clusterState1,
+                    new Task(randomNonNegativeLong(), "test", "test", "", TaskId.EMPTY_TASK_ID, Map.of()),
+                    0L,
+                    0L
+                )
+            );
+        try {
+            context1.sendPublishRequest(
+                otherNode,
+                new PublishRequest(clusterState1),
+                ActionListener.wrap(() -> assertTrue(completed.compareAndSet(false, true)))
+            );
+            assertTrue(completed.getAndSet(false));
+            var receivedState1 = receivedStateRef.getAndSet(null);
+            assertEquals(clusterState1.stateUUID(), receivedState1.stateUUID());
+            assertEquals(otherNode, receivedState1.nodes().getLocalNode());
+            assertSame(receivedState0.nodes(), receivedState1.nodes()); // it was a diff
+            assertTrue(receivedState1.metadata().clusterUUIDCommitted());
+            assertEquals(VotingConfiguration.of(localNode), receivedState1.getLastCommittedConfiguration());
+            final var receivedStateStats = transportHandlersByNode.get(otherNode).stats();
+            assertEquals(1, receivedStateStats.getCompatibleClusterStateDiffReceivedCount());
+            assertEquals(1, receivedStateStats.getIncompatibleClusterStateDiffReceivedCount());
+            assertEquals(1, receivedStateStats.getFullClusterStateReceivedCount());
+        } finally {
+            context1.decRef();
+        }
+
+        assertFalse(deterministicTaskQueue.hasRunnableTasks());
+        assertFalse(deterministicTaskQueue.hasDeferredTasks());
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorServiceTests.java
index 83ba8fb64459..17b19b5d8989 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/StableMasterHealthIndicatorServiceTests.java
@@ -150,10 +150,9 @@ public void testGetHealthIndicatorResultNotGreenVerboseTrue() throws Exception {
         assertThat(nodeIdToNodeNameMap.get(node2.getId()), equalTo(node2.getName()));
         List<Diagnosis> diagnosis = result.diagnosisList();
         assertThat(diagnosis.size(), equalTo(1));
-        assertThat(diagnosis.get(0), is(StableMasterHealthIndicatorService.CONTACT_SUPPORT_USER_ACTION));
+        assertThat(diagnosis.get(0), is(StableMasterHealthIndicatorService.CONTACT_SUPPORT));
     }
 
-    @SuppressWarnings("unchecked")
     public void testGetHealthIndicatorResultNotGreenVerboseFalse() throws Exception {
         MasterHistoryService masterHistoryService = createMasterHistoryService();
         StableMasterHealthIndicatorService service = createStableMasterHealthIndicatorService(nullMasterClusterState, masterHistoryService);
@@ -180,7 +179,6 @@ public void testGetHealthIndicatorResultNotGreenVerboseFalse() throws Exception
         assertThat(diagnosis.size(), equalTo(0));
     }
 
-    @SuppressWarnings("unchecked")
     public void testGetHealthIndicatorResultGreenOrUnknown() throws Exception {
         MasterHistoryService masterHistoryService = createMasterHistoryService();
         StableMasterHealthIndicatorService service = createStableMasterHealthIndicatorService(nullMasterClusterState, masterHistoryService);
@@ -264,6 +262,16 @@ public void testCalculate() throws Exception {
 
     }
 
+    // We expose the indicator name and the diagnosis in the x-pack usage API, consequently these end up in the mapping of the telemetry
+    // index, any changes or additions that we want to track need to be added to the mapping.
+    public void testMappedFieldsForTelemetry() {
+        assertThat(StableMasterHealthIndicatorService.NAME, equalTo("master_is_stable"));
+        assertThat(
+            StableMasterHealthIndicatorService.CONTACT_SUPPORT.definition().getUniqueId(),
+            equalTo("elasticsearch:health:master_is_stable:diagnosis:contact_support")
+        );
+    }
+
     private ClusterState createClusterState(DiscoveryNode masterNode) {
         var routingTableBuilder = RoutingTable.builder();
         Metadata.Builder metadataBuilder = Metadata.builder();
diff --git a/server/src/test/java/org/elasticsearch/cluster/health/ClusterHealthAllocationTests.java b/server/src/test/java/org/elasticsearch/cluster/health/ClusterHealthAllocationTests.java
index d40dc34575c1..9b79bd423f6e 100644
--- a/server/src/test/java/org/elasticsearch/cluster/health/ClusterHealthAllocationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/health/ClusterHealthAllocationTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
@@ -32,7 +33,9 @@ public void testClusterHealth() {
         Metadata metadata = Metadata.builder()
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(1))
             .build();
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
         clusterState = ClusterState.builder(clusterState).metadata(metadata).routingTable(routingTable).build();
         MockAllocationService allocation = createAllocationService();
         clusterState = applyStartedShardsUntilNoChange(clusterState, allocation);
diff --git a/server/src/test/java/org/elasticsearch/cluster/health/ClusterIndexHealthTests.java b/server/src/test/java/org/elasticsearch/cluster/health/ClusterIndexHealthTests.java
index 92ee7af359a8..e85d0084885b 100644
--- a/server/src/test/java/org/elasticsearch/cluster/health/ClusterIndexHealthTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/health/ClusterIndexHealthTests.java
@@ -123,7 +123,7 @@ protected Predicate<String> getRandomFieldsExcludeFilter() {
     }
 
     @Override
-    protected ClusterIndexHealth mutateInstance(ClusterIndexHealth instance) throws IOException {
+    protected ClusterIndexHealth mutateInstance(ClusterIndexHealth instance) {
         String mutate = randomFrom(
             "index",
             "numberOfShards",
diff --git a/server/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java b/server/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
index 0d7f3b0f874e..41270275def5 100644
--- a/server/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
@@ -154,7 +155,7 @@ public void testClusterHealthWaitsForClusterStateApplication() throws Interrupte
             threadPool,
             new ActionFilters(new HashSet<>()),
             indexNameExpressionResolver,
-            new AllocationService(null, new TestGatewayAllocator(), null, null, null)
+            new AllocationService(null, new TestGatewayAllocator(), null, null, null, TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
         );
         PlainActionFuture<ClusterHealthResponse> listener = new PlainActionFuture<>();
         ActionTestUtils.execute(action, null, new ClusterHealthRequest().waitForGreenStatus(), listener);
@@ -322,7 +323,9 @@ private List<ClusterState> simulateIndexCreationStates(final String indexName, f
             .numberOfReplicas(numberOfReplicas)
             .build();
         final Metadata metadata = Metadata.builder().put(indexMetadata, true).build();
-        final RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build();
+        final RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(indexMetadata)
+            .build();
 
         ClusterState clusterState = ClusterState.builder(new ClusterName("test_cluster"))
             .metadata(metadata)
@@ -357,7 +360,9 @@ private List<ClusterState> simulateClusterRecoveryStates(
             indexMetadata = idxMetaWithAllocationIds.build();
         }
         final Metadata metadata = Metadata.builder().put(indexMetadata, true).build();
-        final RoutingTable routingTable = RoutingTable.builder().addAsRecovery(indexMetadata).build();
+        final RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsRecovery(indexMetadata)
+            .build();
 
         ClusterState clusterState = ClusterState.builder(new ClusterName("test_cluster"))
             .metadata(metadata)
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/AutoExpandReplicasTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/AutoExpandReplicasTests.java
index 919960e2dee1..c82b13918835 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/AutoExpandReplicasTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/AutoExpandReplicasTests.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.cluster.routing.RoutingNodesHelper;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.indices.cluster.ClusterStateChanges;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
@@ -100,7 +101,7 @@ public void testInvalidValues() {
     protected DiscoveryNode createNode(Version version, DiscoveryNodeRole... mustHaveRoles) {
         Set<DiscoveryNodeRole> roles = new HashSet<>(randomSubsetOf(DiscoveryNodeRole.roles()));
         Collections.addAll(roles, mustHaveRoles);
-        final String id = formatted("node_%03d", nodeIdGenerator.incrementAndGet());
+        final String id = Strings.format("node_%03d", nodeIdGenerator.incrementAndGet());
         return new DiscoveryNode(id, id, buildNewFakeTransportAddress(), Collections.emptyMap(), roles, version);
     }
 
@@ -161,7 +162,11 @@ public void testAutoExpandWhenNodeLeavesAndPossiblyRejoins() throws InterruptedE
                 postTable = state.routingTable().index("index").shard(0);
 
                 assertTrue("not all shards started in " + state.toString(), postTable.allShardsStarted());
-                assertThat(postTable.toString(), postTable.getAllAllocationIds(), everyItem(is(in(preTable.getAllAllocationIds()))));
+                assertThat(
+                    postTable.toString(),
+                    postTable.getPromotableAllocationIds(),
+                    everyItem(is(in(preTable.getPromotableAllocationIds())))
+                );
             } else {
                 // fake an election where conflicting nodes are removed and readded
                 state = ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).masterNodeId(null).build()).build();
@@ -198,7 +203,7 @@ public void testAutoExpandWhenNodeLeavesAndPossiblyRejoins() throws InterruptedE
                 .map(shr -> shr.allocationId().getId())
                 .collect(Collectors.toSet());
 
-            assertThat(postTable.toString(), unchangedAllocationIds, everyItem(is(in(postTable.getAllAllocationIds()))));
+            assertThat(postTable.toString(), unchangedAllocationIds, everyItem(is(in(postTable.getPromotableAllocationIds()))));
 
             RoutingNodesHelper.asStream(postTable).forEach(shardRouting -> {
                 if (shardRouting.assignedToNode() && unchangedAllocationIds.contains(shardRouting.allocationId().getId())) {
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadataTests.java
index ac2f80057d6c..48cac161839a 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadataTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.test.AbstractNamedWriteableTestCase;
 
-import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -30,7 +29,7 @@ protected ComponentTemplateMetadata createTestInstance() {
     }
 
     @Override
-    protected ComponentTemplateMetadata mutateInstance(ComponentTemplateMetadata instance) throws IOException {
+    protected ComponentTemplateMetadata mutateInstance(ComponentTemplateMetadata instance) {
         if (instance.componentTemplates().size() == 0) {
             // Not really much to mutate, so just generate a new one
             return randomValueOtherThan(instance, this::createTestInstance);
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
index f93051ec914b..c14385c56dc2 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
@@ -31,13 +31,7 @@
 public class ComponentTemplateTests extends SimpleDiffableSerializationTestCase<ComponentTemplate> {
     @Override
     protected ComponentTemplate makeTestChanges(ComponentTemplate testInstance) {
-        try {
-            return mutateInstance(testInstance);
-        } catch (IOException e) {
-            logger.error(e);
-            fail("mutating should not throw an exception, but got: " + e);
-            return null;
-        }
+        return mutateInstance(testInstance);
     }
 
     @Override
@@ -125,7 +119,7 @@ private static Map<String, Object> randomMeta() {
     }
 
     @Override
-    protected ComponentTemplate mutateInstance(ComponentTemplate orig) throws IOException {
+    protected ComponentTemplate mutateInstance(ComponentTemplate orig) {
         return mutateTemplate(orig);
     }
 
@@ -206,10 +200,10 @@ public void testMappingsEquals() throws IOException {
 
         {
             String randomString = randomAlphaOfLength(10);
-            CompressedXContent m1 = new CompressedXContent(formatted("""
+            CompressedXContent m1 = new CompressedXContent(Strings.format("""
                 {"properties":{"%s":{"type":"keyword"}}}
                 """, randomString));
-            CompressedXContent m2 = new CompressedXContent(formatted("""
+            CompressedXContent m2 = new CompressedXContent(Strings.format("""
                 {"properties":{"%s":{"type":"keyword"}}}
                 """, randomString));
             assertThat(Template.mappingsEquals(m1, m2), equalTo(true));
@@ -217,14 +211,14 @@ public void testMappingsEquals() throws IOException {
 
         {
             CompressedXContent m1 = randomMappings();
-            CompressedXContent m2 = new CompressedXContent(formatted("""
+            CompressedXContent m2 = new CompressedXContent(Strings.format("""
                 {"properties":{"%s":{"type":"keyword"}}}
                 """, randomAlphaOfLength(10)));
             assertThat(Template.mappingsEquals(m1, m2), equalTo(false));
         }
 
         {
-            Map<String, Object> map = XContentHelper.convertToMap(new BytesArray(formatted("""
+            Map<String, Object> map = XContentHelper.convertToMap(new BytesArray(Strings.format("""
                 {"%s":{"properties":{"%s":{"type":"keyword"}}}}
                 """, MapperService.SINGLE_MAPPING_NAME, randomAlphaOfLength(10))), true, XContentType.JSON).v2();
             Map<String, Object> reduceMap = Template.reduceMapping(map);
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadataTests.java
index 7ed15b669bf3..e4f0c0b6f24a 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadataTests.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.test.AbstractNamedWriteableTestCase;
 
-import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -30,7 +29,7 @@ protected ComposableIndexTemplateMetadata createTestInstance() {
     }
 
     @Override
-    protected ComposableIndexTemplateMetadata mutateInstance(ComposableIndexTemplateMetadata instance) throws IOException {
+    protected ComposableIndexTemplateMetadata mutateInstance(ComposableIndexTemplateMetadata instance) {
         return randomValueOtherThan(instance, this::createTestInstance);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java
index 45954541dd7b..649355fb8b7f 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java
@@ -26,13 +26,7 @@
 public class ComposableIndexTemplateTests extends SimpleDiffableSerializationTestCase<ComposableIndexTemplate> {
     @Override
     protected ComposableIndexTemplate makeTestChanges(ComposableIndexTemplate testInstance) {
-        try {
-            return mutateInstance(testInstance);
-        } catch (IOException e) {
-            logger.error(e);
-            fail("mutating should not throw an exception, but got: " + e);
-            return null;
-        }
+        return mutateInstance(testInstance);
     }
 
     @Override
@@ -81,6 +75,7 @@ public static ComposableIndexTemplate randomInstance() {
         }
 
         List<String> indexPatterns = randomList(1, 4, () -> randomAlphaOfLength(4));
+        List<String> ignoreMissingComponentTemplates = randomList(0, 4, () -> randomAlphaOfLength(4));
         return new ComposableIndexTemplate(
             indexPatterns,
             template,
@@ -89,7 +84,8 @@ public static ComposableIndexTemplate randomInstance() {
             randomBoolean() ? null : randomNonNegativeLong(),
             meta,
             dataStreamTemplate,
-            randomBoolean() ? null : randomBoolean()
+            randomBoolean() ? null : randomBoolean(),
+            ignoreMissingComponentTemplates
         );
     }
 
@@ -150,12 +146,12 @@ private static ComposableIndexTemplate.DataStreamTemplate randomDataStreamTempla
     }
 
     @Override
-    protected ComposableIndexTemplate mutateInstance(ComposableIndexTemplate orig) throws IOException {
+    protected ComposableIndexTemplate mutateInstance(ComposableIndexTemplate orig) {
         return mutateTemplate(orig);
     }
 
     public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate orig) {
-        switch (randomIntBetween(0, 6)) {
+        switch (randomIntBetween(0, 7)) {
             case 0:
                 List<String> newIndexPatterns = randomValueOtherThan(
                     orig.indexPatterns(),
@@ -183,7 +179,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori
                     orig.version(),
                     orig.metadata(),
                     orig.getDataStreamTemplate(),
-                    orig.getAllowAutoCreate()
+                    orig.getAllowAutoCreate(),
+                    orig.getIgnoreMissingComponentTemplates()
                 );
             case 2:
                 List<String> newComposedOf = randomValueOtherThan(orig.composedOf(), () -> randomList(0, 10, () -> randomAlphaOfLength(5)));
@@ -195,7 +192,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori
                     orig.version(),
                     orig.metadata(),
                     orig.getDataStreamTemplate(),
-                    orig.getAllowAutoCreate()
+                    orig.getAllowAutoCreate(),
+                    orig.getIgnoreMissingComponentTemplates()
                 );
             case 3:
                 return new ComposableIndexTemplate(
@@ -206,7 +204,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori
                     orig.version(),
                     orig.metadata(),
                     orig.getDataStreamTemplate(),
-                    orig.getAllowAutoCreate()
+                    orig.getAllowAutoCreate(),
+                    orig.getIgnoreMissingComponentTemplates()
                 );
             case 4:
                 return new ComposableIndexTemplate(
@@ -217,7 +216,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori
                     randomValueOtherThan(orig.version(), ESTestCase::randomNonNegativeLong),
                     orig.metadata(),
                     orig.getDataStreamTemplate(),
-                    orig.getAllowAutoCreate()
+                    orig.getAllowAutoCreate(),
+                    orig.getIgnoreMissingComponentTemplates()
                 );
             case 5:
                 return new ComposableIndexTemplate(
@@ -228,7 +228,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori
                     orig.version(),
                     randomValueOtherThan(orig.metadata(), ComposableIndexTemplateTests::randomMeta),
                     orig.getDataStreamTemplate(),
-                    orig.getAllowAutoCreate()
+                    orig.getAllowAutoCreate(),
+                    orig.getIgnoreMissingComponentTemplates()
                 );
             case 6:
                 return new ComposableIndexTemplate(
@@ -239,7 +240,24 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori
                     orig.version(),
                     orig.metadata(),
                     randomValueOtherThan(orig.getDataStreamTemplate(), ComposableIndexTemplateTests::randomDataStreamTemplate),
-                    orig.getAllowAutoCreate()
+                    orig.getAllowAutoCreate(),
+                    orig.getIgnoreMissingComponentTemplates()
+                );
+            case 7:
+                List<String> ignoreMissingComponentTemplates = randomValueOtherThan(
+                    orig.getIgnoreMissingComponentTemplates(),
+                    () -> randomList(1, 4, () -> randomAlphaOfLength(4))
+                );
+                return new ComposableIndexTemplate(
+                    orig.indexPatterns(),
+                    orig.template(),
+                    orig.composedOf(),
+                    orig.priority(),
+                    orig.version(),
+                    orig.metadata(),
+                    orig.getDataStreamTemplate(),
+                    orig.getAllowAutoCreate(),
+                    ignoreMissingComponentTemplates
                 );
             default:
                 throw new IllegalStateException("illegal randomization branch");
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamAliasTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamAliasTests.java
index 0fbf645b3159..91cd1c92d73f 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamAliasTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamAliasTests.java
@@ -8,15 +8,24 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
 import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.json.JsonXContent;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.cluster.metadata.DataStreamAlias.DATA_STREAMS_FIELD;
+import static org.elasticsearch.cluster.metadata.DataStreamAlias.OLD_FILTER_FIELD;
+import static org.elasticsearch.cluster.metadata.DataStreamAlias.WRITE_DATA_STREAM_FIELD;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
@@ -50,6 +59,11 @@ protected DataStreamAlias createTestInstance() {
         return DataStreamTestHelper.randomAliasInstance();
     }
 
+    @Override
+    protected DataStreamAlias mutateInstance(DataStreamAlias instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testUpdate() {
         // Add ds-3 to alias, a different instance is returned with ds-3 as one of the data streams being referred to
         DataStreamAlias alias = new DataStreamAlias("my-alias", List.of("ds-1", "ds-2"), null, null);
@@ -87,8 +101,9 @@ public void testUpdateFilter() {
             assertThat(result, not(sameInstance(alias)));
             assertThat(result.getDataStreams(), containsInAnyOrder("ds-1", "ds-2"));
             assertThat(result.getWriteDataStream(), nullValue());
-            assertThat(result.getFilter(), notNullValue());
-            assertThat(result.getFilter().string(), equalTo("""
+            assertThat(result.getFilter("ds-1"), nullValue());
+            assertThat(result.getFilter("ds-2"), notNullValue());
+            assertThat(result.getFilter("ds-2").string(), equalTo("""
                 {"term":{"field":"value"}}"""));
         }
         // noop update to filter:
@@ -97,7 +112,7 @@ public void testUpdateFilter() {
                 "my-alias",
                 List.of("ds-1", "ds-2"),
                 null,
-                Map.of("term", Map.of("field", "value"))
+                Map.of("ds-2", Map.of("term", Map.of("field", "value")))
             );
             DataStreamAlias result = alias.update("ds-2", null, Map.of("term", Map.of("field", "value")));
             assertThat(result, sameInstance(alias));
@@ -108,14 +123,15 @@ public void testUpdateFilter() {
                 "my-alias",
                 List.of("ds-1", "ds-2"),
                 null,
-                Map.of("term", Map.of("field", "value"))
+                Map.of("ds-2", Map.of("term", Map.of("field", "value")))
             );
             DataStreamAlias result = alias.update("ds-2", null, Map.of("term", Map.of("field", "value1")));
             assertThat(result, not(sameInstance(alias)));
             assertThat(result.getDataStreams(), containsInAnyOrder("ds-1", "ds-2"));
             assertThat(result.getWriteDataStream(), nullValue());
-            assertThat(result.getFilter(), notNullValue());
-            assertThat(result.getFilter().string(), equalTo("""
+            assertThat(result.getFilter("ds-1"), nullValue());
+            assertThat(result.getFilter("ds-2"), notNullValue());
+            assertThat(result.getFilter("ds-2").string(), equalTo("""
                 {"term":{"field":"value1"}}"""));
         }
         // Filter not specified, keep existing filter:
@@ -124,13 +140,14 @@ public void testUpdateFilter() {
                 "my-alias",
                 List.of("ds-1", "ds-2"),
                 null,
-                Map.of("term", Map.of("field", "value"))
+                Map.of("ds-2", Map.of("term", Map.of("field", "value")))
             );
             DataStreamAlias result = alias.update("ds-2", null, null);
             assertThat(result, sameInstance(alias));
             assertThat(result.getDataStreams(), containsInAnyOrder("ds-1", "ds-2"));
             assertThat(result.getWriteDataStream(), nullValue());
-            assertThat(result.getFilter().string(), equalTo("""
+            assertThat(result.getFilter("ds-1"), nullValue());
+            assertThat(result.getFilter("ds-2").string(), equalTo("""
                 {"term":{"field":"value"}}"""));
         }
     }
@@ -268,4 +285,53 @@ public void testRestoreDataStreamWithWriteDataStreamThatDoesNotExistInOriginalAl
         assertThat(result.getDataStreams(), containsInAnyOrder("ds-1", "ds-2", "ds-3"));
         assertThat(result.getWriteDataStream(), equalTo("ds-3"));
     }
+
+    public void testSupportsXContentWithSingleFilter() throws IOException {
+        /*
+         * Before 8.7.0, DataStreamAlias only supported a single filter shared by all DataStreams. As of 8.7.0 the "filter" XContent field
+         * is no longer written, but this tests that we can still read it (needed when the cluster state is read on a cluster upgrade).
+         */
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        XContentBuilder builder = new XContentBuilder(JsonXContent.jsonXContent, baos);
+        builder.startObject();
+        String aliasName = randomAlphaOfLength(20);
+        builder.startObject(aliasName);
+        List<String> dataStreams = randomList(10, () -> randomAlphaOfLength(20));
+        builder.stringListField(DATA_STREAMS_FIELD.getPreferredName(), dataStreams);
+        String writeDataStream = dataStreams.isEmpty() ? null : randomFrom(dataStreams);
+        if (writeDataStream != null) {
+            builder.field(WRITE_DATA_STREAM_FIELD.getPreferredName(), writeDataStream);
+        }
+        boolean binary = randomBoolean();
+        Map<String, Object> filterMap = new HashMap<>();
+        filterMap.put(randomAlphaOfLength(10), randomAlphaOfLength(20));
+        CompressedXContent filter = randomBoolean() ? null : new CompressedXContent(filterMap);
+        if (filter != null) {
+            if (binary) {
+                builder.field(OLD_FILTER_FIELD.getPreferredName(), filter.compressed());
+            } else {
+                builder.field(OLD_FILTER_FIELD.getPreferredName(), XContentHelper.convertToMap(filter.uncompressed(), true).v2());
+            }
+        }
+        builder.endObject();
+        builder.endObject();
+        builder.close();
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, baos.toByteArray())) {
+            parser.nextToken();
+            parser.nextToken();
+            DataStreamAlias outputAlias = DataStreamAlias.fromXContent(parser);
+            assertNotNull(outputAlias);
+            assertThat(outputAlias.getName(), equalTo(aliasName));
+            assertThat(outputAlias.getDataStreams(), equalTo(dataStreams));
+            assertThat(outputAlias.getWriteDataStream(), equalTo(writeDataStream));
+            if (filter == null) {
+                assertTrue(outputAlias.dataStreamToFilterMap.isEmpty());
+            } else {
+                assertThat(outputAlias.dataStreamToFilterMap.size(), equalTo(dataStreams.size()));
+                for (String dataStreamName : dataStreams) {
+                    assertThat(outputAlias.getFilter(dataStreamName), equalTo(filter));
+                }
+            }
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java
index 9cb4d5b70902..d186dafd6c7c 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java
@@ -10,25 +10,23 @@
 
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.test.AbstractNamedWriteableTestCase;
-import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
+import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
-import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;
+import static org.hamcrest.Matchers.equalTo;
 
-public class DataStreamMetadataTests extends AbstractNamedWriteableTestCase<DataStreamMetadata> {
-
-    public void testFromXContent() throws IOException {
-        xContentTester(this::createParser, this::createTestInstance, ToXContent.EMPTY_PARAMS, DataStreamMetadata::fromXContent)
-            .assertEqualsConsumer(this::assertEqualInstances)
-            .test();
-    }
+public class DataStreamMetadataTests extends AbstractChunkedSerializingTestCase<DataStreamMetadata> {
 
     @Override
     protected DataStreamMetadata createTestInstance() {
@@ -47,7 +45,7 @@ protected DataStreamMetadata createTestInstance() {
     }
 
     @Override
-    protected DataStreamMetadata mutateInstance(DataStreamMetadata instance) throws IOException {
+    protected DataStreamMetadata mutateInstance(DataStreamMetadata instance) {
         return randomValueOtherThan(instance, this::createTestInstance);
     }
 
@@ -61,7 +59,68 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() {
     }
 
     @Override
-    protected Class<DataStreamMetadata> categoryClass() {
-        return DataStreamMetadata.class;
+    protected DataStreamMetadata doParseInstance(XContentParser parser) throws IOException {
+        return DataStreamMetadata.fromXContent(parser);
+    }
+
+    @Override
+    protected Writeable.Reader<DataStreamMetadata> instanceReader() {
+        return DataStreamMetadata::new;
+    }
+
+    public void testWithAlias() {
+        Index index1 = new Index("data-stream-1-index", "1");
+        Index index2 = new Index("data-stream-2-index", "2");
+        DataStream dataStream1 = new DataStream(
+            "data-stream-1",
+            List.of(index1),
+            1,
+            Map.of(),
+            false,
+            false,
+            false,
+            false,
+            IndexMode.STANDARD
+        );
+        DataStream dataStream2 = new DataStream(
+            "data-stream-2",
+            List.of(index2),
+            1,
+            Map.of(),
+            false,
+            false,
+            false,
+            false,
+            IndexMode.STANDARD
+        );
+        ImmutableOpenMap<String, DataStream> dataStreams = new ImmutableOpenMap.Builder<String, DataStream>().fPut(
+            "data-stream-1",
+            dataStream1
+        ).fPut("data-stream-2", dataStream2).build();
+        ImmutableOpenMap<String, DataStreamAlias> dataStreamAliases = new ImmutableOpenMap.Builder<String, DataStreamAlias>().build();
+        DataStreamMetadata dataStreamMetadata = new DataStreamMetadata(dataStreams, dataStreamAliases);
+        dataStreamMetadata = dataStreamMetadata.withAlias("alias1", "data-stream-2", false, null);
+        dataStreamMetadata = dataStreamMetadata.withAlias("alias1", "data-stream-1", false, """
+            {
+                "term": {
+                    "reaction": "report"
+                }
+                }""");
+
+        Map<String, DataStreamAlias> aliasMap = dataStreamMetadata.getDataStreamAliases();
+        assertNotNull(aliasMap);
+        assertThat(aliasMap.size(), equalTo(1));
+        DataStreamAlias alias1 = aliasMap.get("alias1");
+        assertNotNull(alias1);
+        Map<String, DataStream> dataStreamMap = dataStreamMetadata.dataStreams();
+        assertNotNull(dataStreamMap);
+        assertThat(dataStreamMap.size(), equalTo(2));
+        DataStream resultDataStream1 = dataStreamMap.get("data-stream-1");
+        assertNotNull(resultDataStream1);
+        DataStream resultDataStream2 = dataStreamMap.get("data-stream-2");
+        assertNotNull(resultDataStream2);
+
+        assertNotNull(alias1.getFilter(resultDataStream1.getName()));
+        assertNull(alias1.getFilter(resultDataStream2.getName()));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java
index ee774f9f2209..6033da5d1a68 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTemplateTests.java
@@ -32,6 +32,11 @@ protected DataStreamTemplate createTestInstance() {
         return randomInstance();
     }
 
+    @Override
+    protected DataStreamTemplate mutateInstance(DataStreamTemplate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static DataStreamTemplate randomInstance() {
         IndexMode indexMode = randomBoolean() ? randomFrom(IndexMode.values()) : null;
         return new ComposableIndexTemplate.DataStreamTemplate(randomBoolean(), randomBoolean());
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
index 6a49a437d024..86a3d165ac62 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
@@ -39,6 +39,7 @@
 import static org.hamcrest.Matchers.in;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 
 public class DataStreamTests extends AbstractXContentSerializingTestCase<DataStream> {
 
@@ -57,6 +58,11 @@ protected DataStream createTestInstance() {
         return DataStreamTestHelper.randomInstance();
     }
 
+    @Override
+    protected DataStream mutateInstance(DataStream instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testRollover() {
         DataStream ds = DataStreamTestHelper.randomInstance().promoteDataStream();
         Tuple<String, Long> newCoordinates = ds.nextWriteIndexAndGeneration(Metadata.EMPTY_METADATA);
@@ -96,7 +102,7 @@ public void testRolloverWithConflictingBackingIndexName() {
         assertThat(rolledDs.getIndexMode(), equalTo(ds.getIndexMode()));
     }
 
-    public void testRolloverIndexMode() {
+    public void testRolloverUpgradeToTsdbDataStream() {
         IndexMode indexMode = randomBoolean() ? IndexMode.STANDARD : null;
         DataStream ds = DataStreamTestHelper.randomInstance().promoteDataStream();
         // Unsure index_mode=null
@@ -123,7 +129,7 @@ public void testRolloverIndexMode() {
         assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TIME_SERIES));
     }
 
-    public void testRolloverIndexMode_keepIndexMode() {
+    public void testRolloverDowngradeToRegularDataStream() {
         DataStream ds = DataStreamTestHelper.randomInstance().promoteDataStream();
         ds = new DataStream(
             ds.getName(),
@@ -145,7 +151,7 @@ public void testRolloverIndexMode_keepIndexMode() {
         assertThat(rolledDs.getIndices().size(), equalTo(ds.getIndices().size() + 1));
         assertTrue(rolledDs.getIndices().containsAll(ds.getIndices()));
         assertTrue(rolledDs.getIndices().contains(rolledDs.getWriteIndex()));
-        assertThat(rolledDs.getIndexMode(), equalTo(IndexMode.TIME_SERIES));
+        assertThat(rolledDs.getIndexMode(), nullValue());
     }
 
     public void testRemoveBackingIndex() {
@@ -183,7 +189,7 @@ public void testRemoveBackingIndexThatDoesNotExist() {
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> original.removeBackingIndex(indexToRemove));
         assertThat(
             e.getMessage(),
-            equalTo(String.format(Locale.ROOT, "index [%s] is not part of data stream [%s]", indexToRemove.getName(), dataStreamName))
+            equalTo(Strings.format("index [%s] is not part of data stream [%s]", indexToRemove.getName(), dataStreamName))
         );
     }
 
@@ -398,7 +404,7 @@ public void testDefaultBackingIndexName() {
         long epochMillis = randomLongBetween(1580536800000L, 1583042400000L);
         String dateString = DataStream.DATE_FORMATTER.formatMillis(epochMillis);
         String defaultBackingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, backingIndexNum, epochMillis);
-        String expectedBackingIndexName = String.format(Locale.ROOT, ".ds-%s-%s-%06d", dataStreamName, dateString, backingIndexNum);
+        String expectedBackingIndexName = Strings.format(".ds-%s-%s-%06d", dataStreamName, dateString, backingIndexNum);
         assertThat(defaultBackingIndexName, equalTo(expectedBackingIndexName));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodeSerializationTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodeSerializationTests.java
index 3223981ff2f2..20f7298fffe0 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodeSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodeSerializationTests.java
@@ -34,7 +34,7 @@ protected DesiredNode createTestInstance() {
     }
 
     @Override
-    protected DesiredNode mutateInstance(DesiredNode instance) throws IOException {
+    protected DesiredNode mutateInstance(DesiredNode instance) {
         return mutateDesiredNode(instance);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodeTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodeTests.java
index 5451da0c2235..826edafbae35 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodeTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodeTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.settings.Settings;
@@ -201,8 +202,8 @@ public void testDesiredNodeIsCompatible() {
                 ByteSizeValue.ofGb(1),
                 Version.CURRENT
             );
-            assertThat(desiredNode.isCompatibleWithVersion(Version.V_8_2_0), is(equalTo(false)));
-            assertThat(desiredNode.isCompatibleWithVersion(Version.V_8_3_0), is(equalTo(true)));
+            assertThat(desiredNode.isCompatibleWithVersion(TransportVersion.V_8_2_0), is(equalTo(false)));
+            assertThat(desiredNode.isCompatibleWithVersion(TransportVersion.V_8_3_0), is(equalTo(true)));
         }
 
         {
@@ -213,14 +214,14 @@ public void testDesiredNodeIsCompatible() {
                 ByteSizeValue.ofGb(1),
                 Version.CURRENT
             );
-            assertThat(desiredNode.isCompatibleWithVersion(Version.V_8_2_0), is(equalTo(false)));
-            assertThat(desiredNode.isCompatibleWithVersion(Version.V_8_3_0), is(equalTo(true)));
+            assertThat(desiredNode.isCompatibleWithVersion(TransportVersion.V_8_2_0), is(equalTo(false)));
+            assertThat(desiredNode.isCompatibleWithVersion(TransportVersion.V_8_3_0), is(equalTo(true)));
         }
 
         {
             final var desiredNode = new DesiredNode(settings, 2.0f, ByteSizeValue.ofGb(1), ByteSizeValue.ofGb(1), Version.CURRENT);
-            assertThat(desiredNode.isCompatibleWithVersion(Version.V_8_2_0), is(equalTo(true)));
-            assertThat(desiredNode.isCompatibleWithVersion(Version.V_8_3_0), is(equalTo(true)));
+            assertThat(desiredNode.isCompatibleWithVersion(TransportVersion.V_8_2_0), is(equalTo(true)));
+            assertThat(desiredNode.isCompatibleWithVersion(TransportVersion.V_8_3_0), is(equalTo(true)));
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodeWithStatusSerializationTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodeWithStatusSerializationTests.java
index 32b5775cc1b1..aff5a6a785c2 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodeWithStatusSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodeWithStatusSerializationTests.java
@@ -35,7 +35,7 @@ protected DesiredNodeWithStatus createTestInstance() {
     }
 
     @Override
-    protected DesiredNodeWithStatus mutateInstance(DesiredNodeWithStatus instance) throws IOException {
+    protected DesiredNodeWithStatus mutateInstance(DesiredNodeWithStatus instance) {
         return mutateDesiredNodeWithStatus(instance);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodesMetadataSerializationTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodesMetadataSerializationTests.java
index d3fc52f05b05..717e74f2878c 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodesMetadataSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodesMetadataSerializationTests.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.test.SimpleDiffableSerializationTestCase;
+import org.elasticsearch.test.ChunkedToXContentDiffableSerializationTestCase;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -20,7 +20,7 @@
 import static org.elasticsearch.cluster.metadata.DesiredNodesSerializationTests.mutateDesiredNodes;
 import static org.elasticsearch.cluster.metadata.DesiredNodesTestCase.randomDesiredNodes;
 
-public class DesiredNodesMetadataSerializationTests extends SimpleDiffableSerializationTestCase<Metadata.Custom> {
+public class DesiredNodesMetadataSerializationTests extends ChunkedToXContentDiffableSerializationTestCase<Metadata.Custom> {
     @Override
     protected Metadata.Custom makeTestChanges(Metadata.Custom testInstance) {
         if (randomBoolean()) {
@@ -58,6 +58,11 @@ protected Metadata.Custom createTestInstance() {
         return randomDesiredNodesMetadata();
     }
 
+    @Override
+    protected Metadata.Custom mutateInstance(Metadata.Custom instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     private static DesiredNodesMetadata randomDesiredNodesMetadata() {
         return new DesiredNodesMetadata(randomDesiredNodes());
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodesSerializationTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodesSerializationTests.java
index 9e5adc9c2bdd..e82aaf5a1b79 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodesSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodesSerializationTests.java
@@ -34,7 +34,7 @@ protected DesiredNodes doParseInstance(XContentParser parser) throws IOException
     }
 
     @Override
-    protected DesiredNodes mutateInstance(DesiredNodes instance) throws IOException {
+    protected DesiredNodes mutateInstance(DesiredNodes instance) {
         return mutateDesiredNodes(instance);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ExpressionListTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ExpressionListTests.java
new file mode 100644
index 000000000000..5235ca967131
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ExpressionListTests.java
@@ -0,0 +1,308 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.Context;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ExpressionList;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ExpressionList.Expression;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ExpressionListTests extends ESTestCase {
+
+    public void testEmpty() {
+        ExpressionList expressionList = new ExpressionList(getContextWithOptions(getExpandWildcardsIndicesOptions()), List.of());
+        assertThat(expressionList.iterator().hasNext(), is(false));
+        assertThat(expressionList.hasWildcard(), is(false));
+        expressionList = new ExpressionList(getContextWithOptions(getNoExpandWildcardsIndicesOptions()), List.of());
+        assertThat(expressionList.iterator().hasNext(), is(false));
+        assertThat(expressionList.hasWildcard(), is(false));
+    }
+
+    public void testExplicitSingleNameExpression() {
+        for (IndicesOptions indicesOptions : List.of(getExpandWildcardsIndicesOptions(), getNoExpandWildcardsIndicesOptions())) {
+            for (String expressionString : List.of("non_wildcard", "-non_exclusion")) {
+                ExpressionList expressionList = new ExpressionList(getContextWithOptions(indicesOptions), List.of(expressionString));
+                assertThat(expressionList.hasWildcard(), is(false));
+                if (randomBoolean()) {
+                    expressionList = new ExpressionList(getContextWithOptions(indicesOptions), List.of(expressionString));
+                }
+                Iterator<Expression> expressionIterator = expressionList.iterator();
+                assertThat(expressionIterator.hasNext(), is(true));
+                if (randomBoolean()) {
+                    expressionIterator = expressionList.iterator();
+                }
+                Expression expression = expressionIterator.next();
+                assertThat(expression.isExclusion(), is(false));
+                assertThat(expression.isWildcard(), is(false));
+                assertThat(expression.get(), is(expressionString));
+                assertThat(expressionIterator.hasNext(), is(false));
+            }
+        }
+    }
+
+    public void testWildcardSingleExpression() {
+        for (String wildcardTest : List.of("*", "a*", "*b", "a*b", "a-*b", "a*-b", "-*", "-a*", "-*b", "**", "*-*")) {
+            ExpressionList expressionList = new ExpressionList(
+                getContextWithOptions(getExpandWildcardsIndicesOptions()),
+                List.of(wildcardTest)
+            );
+            assertThat(expressionList.hasWildcard(), is(true));
+            if (randomBoolean()) {
+                expressionList = new ExpressionList(getContextWithOptions(getExpandWildcardsIndicesOptions()), List.of(wildcardTest));
+            }
+            Iterator<Expression> expressionIterator = expressionList.iterator();
+            assertThat(expressionIterator.hasNext(), is(true));
+            if (randomBoolean()) {
+                expressionIterator = expressionList.iterator();
+            }
+            Expression expression = expressionIterator.next();
+            assertThat(expression.isExclusion(), is(false));
+            assertThat(expression.isWildcard(), is(true));
+            assertThat(expression.get(), is(wildcardTest));
+            assertThat(expressionIterator.hasNext(), is(false));
+        }
+    }
+
+    public void testWildcardLongerExpression() {
+        List<String> onlyExplicits = randomList(7, () -> randomAlphaOfLengthBetween(0, 5));
+        String wildcard = randomFrom("*", "*b", "-*", "*-", "c*", "a*b", "**");
+        List<String> expressionList = new ArrayList<>(onlyExplicits.size() + 1);
+        expressionList.addAll(randomSubsetOf(onlyExplicits));
+        int wildcardPos = expressionList.size();
+        expressionList.add(wildcard);
+        for (String item : onlyExplicits) {
+            if (expressionList.contains(item) == false) {
+                expressionList.add(item);
+            }
+        }
+        ExpressionList expressionIterable = new ExpressionList(getContextWithOptions(getExpandWildcardsIndicesOptions()), expressionList);
+        assertThat(expressionIterable.hasWildcard(), is(true));
+        if (randomBoolean()) {
+            expressionIterable = new ExpressionList(getContextWithOptions(getExpandWildcardsIndicesOptions()), expressionList);
+        }
+        int i = 0;
+        for (Expression expression : expressionIterable) {
+            assertThat(expression.isExclusion(), is(false));
+            if (i != wildcardPos) {
+                assertThat(expression.isWildcard(), is(false));
+            } else {
+                assertThat(expression.isWildcard(), is(true));
+            }
+            assertThat(expression.get(), is(expressionList.get(i++)));
+        }
+    }
+
+    public void testWildcardsNoExclusionExpressions() {
+        for (List<String> wildcardExpression : List.of(
+            List.of("*"),
+            List.of("a", "*"),
+            List.of("-b", "*c"),
+            List.of("-", "a", "c*"),
+            List.of("*", "a*", "*b"),
+            List.of("-*", "a", "b*")
+        )) {
+            ExpressionList expressionList = new ExpressionList(
+                getContextWithOptions(getExpandWildcardsIndicesOptions()),
+                wildcardExpression
+            );
+            assertThat(expressionList.hasWildcard(), is(true));
+            if (randomBoolean()) {
+                expressionList = new ExpressionList(getContextWithOptions(getExpandWildcardsIndicesOptions()), wildcardExpression);
+            }
+            int i = 0;
+            for (Expression expression : expressionList) {
+                assertThat(expression.isExclusion(), is(false));
+                if (wildcardExpression.get(i).contains("*")) {
+                    assertThat(expression.isWildcard(), is(true));
+                } else {
+                    assertThat(expression.isWildcard(), is(false));
+                }
+                assertThat(expression.get(), is(wildcardExpression.get(i++)));
+            }
+        }
+    }
+
+    public void testWildcardExpressionNoExpandOptions() {
+        for (List<String> wildcardExpression : List.of(
+            List.of("*"),
+            List.of("a", "*"),
+            List.of("-b", "*c"),
+            List.of("*d", "-"),
+            List.of("*", "-*"),
+            List.of("-", "a", "c*"),
+            List.of("*", "a*", "*b")
+        )) {
+            ExpressionList expressionList = new ExpressionList(
+                getContextWithOptions(getNoExpandWildcardsIndicesOptions()),
+                wildcardExpression
+            );
+            assertThat(expressionList.hasWildcard(), is(false));
+            if (randomBoolean()) {
+                expressionList = new ExpressionList(getContextWithOptions(getNoExpandWildcardsIndicesOptions()), wildcardExpression);
+            }
+            int i = 0;
+            for (Expression expression : expressionList) {
+                assertThat(expression.isWildcard(), is(false));
+                assertThat(expression.isExclusion(), is(false));
+                assertThat(expression.get(), is(wildcardExpression.get(i++)));
+            }
+        }
+    }
+
+    public void testSingleExclusionExpression() {
+        String wildcard = randomFrom("*", "*b", "-*", "*-", "c*", "a*b", "**", "*-*");
+        int wildcardPos = randomIntBetween(0, 3);
+        String exclusion = randomFrom("-*", "-", "-c*", "-ab", "--");
+        int exclusionPos = randomIntBetween(wildcardPos + 1, 7);
+        List<String> exclusionExpression = new ArrayList<>();
+        for (int i = 0; i < wildcardPos; i++) {
+            exclusionExpression.add(randomAlphaOfLengthBetween(0, 5));
+        }
+        exclusionExpression.add(wildcard);
+        for (int i = wildcardPos + 1; i < exclusionPos; i++) {
+            exclusionExpression.add(randomAlphaOfLengthBetween(0, 5));
+        }
+        exclusionExpression.add(exclusion);
+        for (int i = 0; i < randomIntBetween(0, 3); i++) {
+            exclusionExpression.add(randomAlphaOfLengthBetween(0, 5));
+        }
+        ExpressionList expressionList = new ExpressionList(getContextWithOptions(getExpandWildcardsIndicesOptions()), exclusionExpression);
+        if (randomBoolean()) {
+            assertThat(expressionList.hasWildcard(), is(true));
+        }
+        int i = 0;
+        for (Expression expression : expressionList) {
+            if (i == wildcardPos) {
+                assertThat(expression.isWildcard(), is(true));
+                assertThat(expression.isExclusion(), is(false));
+                assertThat(expression.get(), is(exclusionExpression.get(i++)));
+            } else if (i == exclusionPos) {
+                assertThat(expression.isExclusion(), is(true));
+                assertThat(expression.isWildcard(), is(exclusionExpression.get(i).contains("*")));
+                assertThat(expression.get(), is(exclusionExpression.get(i++).substring(1)));
+            } else {
+                assertThat(expression.isWildcard(), is(false));
+                assertThat(expression.isExclusion(), is(false));
+                assertThat(expression.get(), is(exclusionExpression.get(i++)));
+            }
+        }
+    }
+
+    public void testExclusionsExpression() {
+        for (Tuple<List<String>, List<Boolean>> exclusionExpression : List.of(
+            new Tuple<>(List.of("-a", "*", "-a"), List.of(false, false, true)),
+            new Tuple<>(List.of("-b*", "c", "-a"), List.of(false, false, true)),
+            new Tuple<>(List.of("*d", "-", "*b"), List.of(false, true, false)),
+            new Tuple<>(List.of("-", "--", "-*", "", "-*"), List.of(false, false, false, false, true)),
+            new Tuple<>(List.of("*-", "-*", "a", "-b"), List.of(false, true, false, true)),
+            new Tuple<>(List.of("a", "-b", "-*", "-b", "*", "-b"), List.of(false, false, false, true, false, true)),
+            new Tuple<>(List.of("-a", "*d", "-a", "-*b", "-b", "--"), List.of(false, false, true, true, true, true))
+        )) {
+            ExpressionList expressionList = new ExpressionList(
+                getContextWithOptions(getExpandWildcardsIndicesOptions()),
+                exclusionExpression.v1()
+            );
+            if (randomBoolean()) {
+                assertThat(expressionList.hasWildcard(), is(true));
+            }
+            int i = 0;
+            for (Expression expression : expressionList) {
+                boolean isExclusion = exclusionExpression.v2().get(i);
+                assertThat(expression.isExclusion(), is(isExclusion));
+                assertThat(expression.isWildcard(), is(exclusionExpression.v1().get(i).contains("*")));
+                if (isExclusion) {
+                    assertThat(expression.get(), is(exclusionExpression.v1().get(i++).substring(1)));
+                } else {
+                    assertThat(expression.get(), is(exclusionExpression.v1().get(i++)));
+                }
+            }
+        }
+    }
+
+    private IndicesOptions getExpandWildcardsToOpenOnlyIndicesOptions() {
+        return IndicesOptions.fromOptions(
+            randomBoolean(),
+            randomBoolean(),
+            true,
+            false,
+            randomBoolean(),
+            randomBoolean(),
+            randomBoolean(),
+            randomBoolean(),
+            randomBoolean()
+        );
+    }
+
+    private IndicesOptions getExpandWildcardsToCloseOnlyIndicesOptions() {
+        return IndicesOptions.fromOptions(
+            randomBoolean(),
+            randomBoolean(),
+            false,
+            true,
+            randomBoolean(),
+            randomBoolean(),
+            randomBoolean(),
+            randomBoolean(),
+            randomBoolean()
+        );
+    }
+
+    private IndicesOptions getExpandWildcardsToOpenCloseIndicesOptions() {
+        return IndicesOptions.fromOptions(
+            randomBoolean(),
+            randomBoolean(),
+            true,
+            true,
+            randomBoolean(),
+            randomBoolean(),
+            randomBoolean(),
+            randomBoolean(),
+            randomBoolean()
+        );
+    }
+
+    private IndicesOptions getExpandWildcardsIndicesOptions() {
+        return ESTestCase.<Supplier<IndicesOptions>>randomFrom(
+            this::getExpandWildcardsToOpenOnlyIndicesOptions,
+            this::getExpandWildcardsToCloseOnlyIndicesOptions,
+            this::getExpandWildcardsToOpenCloseIndicesOptions
+        ).get();
+    }
+
+    private IndicesOptions getNoExpandWildcardsIndicesOptions() {
+        return IndicesOptions.fromOptions(
+            randomBoolean(),
+            randomBoolean(),
+            false,
+            false,
+            randomBoolean(),
+            randomBoolean(),
+            randomBoolean(),
+            randomBoolean(),
+            randomBoolean()
+        );
+    }
+
+    private Context getContextWithOptions(IndicesOptions indicesOptions) {
+        Context context = mock(Context.class);
+        when(context.getOptions()).thenReturn(indicesOptions);
+        return context;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java
index cbf1f6dc4f7a..e539087de7b8 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java
@@ -13,8 +13,10 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.common.xcontent.XContentElasticsearchExtension;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -57,7 +59,7 @@ public void testXContent() throws IOException {
         final IndexGraveyard graveyard = createRandom();
         final XContentBuilder builder = JsonXContent.contentBuilder();
         builder.startObject();
-        graveyard.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        ChunkedToXContent.wrapAsToXContent(graveyard).toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
         if (graveyard.getTombstones().size() > 0) {
             // check that date properly printed
@@ -75,6 +77,10 @@ public void testXContent() throws IOException {
         assertThat(IndexGraveyard.fromXContent(parser), equalTo(graveyard));
     }
 
+    public void testChunking() {
+        AbstractChunkedSerializingTestCase.assertChunkCount(createRandom(), graveyard -> graveyard.getTombstones().size() + 2);
+    }
+
     public void testAddTombstones() {
         final IndexGraveyard graveyard1 = createRandom();
         final IndexGraveyard.Builder graveyardBuidler = IndexGraveyard.builder(graveyard1);
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataStatsSerializationTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataStatsSerializationTests.java
index df7c95b450de..6844e4c1f5e3 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataStatsSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataStatsSerializationTests.java
@@ -37,7 +37,7 @@ protected IndexMetadataStats createTestInstance() {
     }
 
     @Override
-    protected IndexMetadataStats mutateInstance(IndexMetadataStats originalStats) throws IOException {
+    protected IndexMetadataStats mutateInstance(IndexMetadataStats originalStats) {
         final IndexWriteLoad originalWriteLoad = originalStats.writeLoad();
 
         final int newNumberOfShards;
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataStatsTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataStatsTests.java
index 9684c9c6d3e5..926f96b4a6a6 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataStatsTests.java
@@ -110,7 +110,8 @@ private ShardStats createShardStats(
             new ShardId(indexName, "__na__", shard),
             primary,
             recoverySource,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         shardRouting = ShardRoutingHelper.initialize(shardRouting, UUIDs.randomBase64UUID());
         shardRouting = ShardRoutingHelper.moveToStarted(shardRouting);
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java
index 6e7682079bcb..783790b9aed8 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java
@@ -47,6 +47,7 @@
 import static org.elasticsearch.cluster.metadata.IndexMetadata.parseIndexNameCounter;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.is;
 
 public class IndexMetadataTests extends ESTestCase {
@@ -497,16 +498,40 @@ public void testLifeCyclePolicyName() {
     }
 
     public void testIndexAndAliasWithSameName() {
-        final IllegalArgumentException iae = expectThrows(
-            IllegalArgumentException.class,
-            () -> IndexMetadata.builder("index")
-                .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT))
-                .numberOfShards(1)
-                .numberOfReplicas(0)
-                .putAlias(AliasMetadata.builder("index").build())
-                .build()
-        );
-        assertEquals("alias name [index] self-conflicts with index name", iae.getMessage());
+        {
+            final IllegalArgumentException iae = expectThrows(
+                IllegalArgumentException.class,
+                () -> IndexMetadata.builder("index")
+                    .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT))
+                    .numberOfShards(1)
+                    .numberOfReplicas(0)
+                    .putAlias(AliasMetadata.builder("index").build())
+                    .build(randomBoolean())
+            );
+            assertEquals("alias name [index] self-conflicts with index name", iae.getMessage());
+        }
+        {
+            final IllegalArgumentException iae = expectThrows(
+                IllegalArgumentException.class,
+                () -> IndexMetadata.builder("index")
+                    .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.V_8_5_0))
+                    .numberOfShards(1)
+                    .numberOfReplicas(0)
+                    .putAlias(AliasMetadata.builder("index").build())
+                    .build(false)
+            );
+            assertEquals("alias name [index] self-conflicts with index name", iae.getMessage());
+        }
+    }
+
+    public void testRepairIndexAndAliasWithSameName() {
+        final IndexMetadata indexMetadata = IndexMetadata.builder("index")
+            .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.V_8_5_0))
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .putAlias(AliasMetadata.builder("index").build())
+            .build(true);
+        assertThat(indexMetadata.getAliases(), hasKey("index-alias-corrupted-by-8-5"));
     }
 
     private static Settings indexSettingsWithDataTier(String dataTier) {
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
index d08166eaf411..04a4b8471588 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
@@ -753,11 +753,43 @@ public void testIndexOptionsEmptyCluster() {
             IndicesOptions.fromOptions(true, false, true, false),
             SystemIndexAccessLevel.NONE
         );
-        IndexNotFoundException infe = expectThrows(
-            IndexNotFoundException.class,
-            () -> indexNameExpressionResolver.concreteIndexNames(context3, Strings.EMPTY_ARRAY)
+        {
+            IndexNotFoundException infe = expectThrows(
+                IndexNotFoundException.class,
+                () -> indexNameExpressionResolver.concreteIndexNames(context3, Strings.EMPTY_ARRAY)
+            );
+            assertThat(infe.getResourceId().toString(), equalTo("[_all]"));
+        }
+
+        // no wildcard expand
+        final IndexNameExpressionResolver.Context context4 = new IndexNameExpressionResolver.Context(
+            state,
+            IndicesOptions.fromOptions(false, true, false, false),
+            randomFrom(SystemIndexAccessLevel.values())
         );
-        assertThat(infe.getResourceId().toString(), equalTo("[_all]"));
+        results = indexNameExpressionResolver.concreteIndexNames(context4, Strings.EMPTY_ARRAY);
+        assertThat(results, emptyArray());
+        {
+            IndexNotFoundException infe = expectThrows(
+                IndexNotFoundException.class,
+                () -> indexNameExpressionResolver.concreteIndexNames(context4, "foo")
+            );
+            assertThat(infe.getIndex().getName(), equalTo("foo"));
+        }
+        {
+            IndexNotFoundException infe = expectThrows(
+                IndexNotFoundException.class,
+                () -> indexNameExpressionResolver.concreteIndexNames(context4, "foo*")
+            );
+            assertThat(infe.getIndex().getName(), equalTo("foo*"));
+        }
+        {
+            IndexNotFoundException infe = expectThrows(
+                IndexNotFoundException.class,
+                () -> indexNameExpressionResolver.concreteIndexNames(context4, "bar", "foo*")
+            );
+            assertThat(infe.getIndex().getName(), equalTo("bar"));
+        }
     }
 
     public void testConcreteIndicesIgnoreIndicesOneMissingIndex() {
@@ -768,12 +800,25 @@ public void testConcreteIndicesIgnoreIndicesOneMissingIndex() {
             IndicesOptions.strictExpandOpen(),
             SystemIndexAccessLevel.NONE
         );
-
         IndexNotFoundException infe = expectThrows(
             IndexNotFoundException.class,
             () -> indexNameExpressionResolver.concreteIndexNames(context, "testZZZ")
         );
         assertThat(infe.getMessage(), is("no such index [testZZZ]"));
+        // same as above, but DO NOT expand wildcards
+        IndexNameExpressionResolver.Context context_no_expand = new IndexNameExpressionResolver.Context(
+            state,
+            new IndicesOptions(
+                EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES),
+                randomFrom(EnumSet.noneOf(IndicesOptions.WildcardStates.class), EnumSet.of(IndicesOptions.WildcardStates.HIDDEN))
+            ),
+            randomFrom(SystemIndexAccessLevel.values())
+        );
+        IndexNotFoundException infe_no_expand = expectThrows(
+            IndexNotFoundException.class,
+            () -> indexNameExpressionResolver.concreteIndexNames(context_no_expand, "testZZZ")
+        );
+        assertThat(infe_no_expand.getMessage(), is("no such index [testZZZ]"));
     }
 
     public void testConcreteIndicesIgnoreIndicesOneMissingIndexOtherFound() {
@@ -805,6 +850,20 @@ public void testConcreteIndicesIgnoreIndicesAllMissing() {
             () -> indexNameExpressionResolver.concreteIndexNames(context, "testMo", "testMahdy")
         );
         assertThat(infe.getMessage(), is("no such index [testMo]"));
+        // same as above, but DO NOT expand wildcards
+        IndexNameExpressionResolver.Context context_no_expand = new IndexNameExpressionResolver.Context(
+            state,
+            new IndicesOptions(
+                EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES),
+                randomFrom(EnumSet.noneOf(IndicesOptions.WildcardStates.class), EnumSet.of(IndicesOptions.WildcardStates.HIDDEN))
+            ),
+            randomFrom(SystemIndexAccessLevel.values())
+        );
+        IndexNotFoundException infe_no_expand = expectThrows(
+            IndexNotFoundException.class,
+            () -> indexNameExpressionResolver.concreteIndexNames(context_no_expand, "testMo", "testMahdy")
+        );
+        assertThat(infe_no_expand.getMessage(), is("no such index [testMo]"));
     }
 
     public void testConcreteIndicesIgnoreIndicesEmptyRequest() {
@@ -993,7 +1052,7 @@ public void testConcreteIndicesWildcardAndAliases() {
 
         // when ignoreAliases option is set, concreteIndexNames resolves the provided expressions
         // only against the defined indices
-        IndicesOptions ignoreAliasesOptions = IndicesOptions.fromOptions(false, false, true, false, true, false, true, false);
+        IndicesOptions ignoreAliasesOptions = IndicesOptions.fromOptions(false, randomBoolean(), true, false, true, false, true, false);
 
         String[] indexNamesIndexWildcard = indexNameExpressionResolver.concreteIndexNames(state, ignoreAliasesOptions, "foo*");
 
@@ -1019,9 +1078,32 @@ public void testConcreteIndicesWildcardAndAliases() {
             iae.getMessage()
         );
 
+        // same as above, but DO NOT expand wildcards
+        iae = expectThrows(
+            IllegalArgumentException.class,
+            () -> indexNameExpressionResolver.concreteIndexNames(
+                state,
+                IndicesOptions.fromOptions(false, randomBoolean(), false, false, true, false, true, false),
+                "foo"
+            )
+        );
+        assertEquals(
+            "The provided expression [foo] matches an alias, specify the corresponding concrete indices instead.",
+            iae.getMessage()
+        );
+
         // when ignoreAliases option is not set, concreteIndexNames resolves the provided
         // expressions against the defined indices and aliases
-        IndicesOptions indicesAndAliasesOptions = IndicesOptions.fromOptions(false, false, true, false, true, false, false, false);
+        IndicesOptions indicesAndAliasesOptions = IndicesOptions.fromOptions(
+            false,
+            randomBoolean(),
+            true,
+            false,
+            true,
+            false,
+            false,
+            false
+        );
 
         List<String> indexNames = Arrays.asList(indexNameExpressionResolver.concreteIndexNames(state, indicesAndAliasesOptions, "foo*"));
         assertEquals(2, indexNames.size());
@@ -1042,6 +1124,13 @@ public void testConcreteIndicesWildcardAndAliases() {
         assertEquals(2, indexNames.size());
         assertTrue(indexNames.contains("foo_foo"));
         assertTrue(indexNames.contains("bar_bar"));
+
+        // same as above, but DO NOT expand wildcards
+        indicesAndAliasesOptions = IndicesOptions.fromOptions(false, randomBoolean(), false, false, true, false, false, false);
+        indexNames = Arrays.asList(indexNameExpressionResolver.concreteIndexNames(state, indicesAndAliasesOptions, "foo"));
+        assertEquals(2, indexNames.size());
+        assertTrue(indexNames.contains("foo_foo"));
+        assertTrue(indexNames.contains("bar_bar"));
     }
 
     public void testHiddenAliasAndHiddenIndexResolution() {
@@ -1583,6 +1672,21 @@ public void testIndexAliasesDataStreamAliases() {
             String[] result = indexNameExpressionResolver.indexAliases(state, index, x -> true, x -> true, false, resolvedExpressions);
             assertThat(result, nullValue());
         }
+
+        {
+            // Only resolve aliases that refer to dataStreamName1 where the filter is not null
+            Set<String> resolvedExpressions = indexNameExpressionResolver.resolveExpressions(state, "l*");
+            String index = backingIndex1.getIndex().getName();
+            String[] result = indexNameExpressionResolver.indexAliases(
+                state,
+                index,
+                x -> true,
+                DataStreamAlias::filteringRequired,
+                true,
+                resolvedExpressions
+            );
+            assertThat(result, arrayContainingInAnyOrder("logs", "logs_foo"));
+        }
     }
 
     public void testIndexAliasesSkipIdentity() {
@@ -1858,13 +1962,42 @@ public void testDeleteIndexIgnoresAliases() {
             assertEquals("does_not_exist", infe.getIndex().getName());
             assertEquals("no such index [does_not_exist]", infe.getMessage());
         }
+        {
+            // same delete request but with request options that DO NOT expand wildcards
+            DeleteIndexRequest request = new DeleteIndexRequest("does_not_exist").indicesOptions(
+                new IndicesOptions(
+                    EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES),
+                    randomFrom(EnumSet.noneOf(IndicesOptions.WildcardStates.class), EnumSet.of(IndicesOptions.WildcardStates.HIDDEN))
+                )
+            );
+            IndexNotFoundException infe = expectThrows(
+                IndexNotFoundException.class,
+                () -> indexNameExpressionResolver.concreteIndexNames(state, request)
+            );
+            assertEquals("does_not_exist", infe.getIndex().getName());
+            assertEquals("no such index [does_not_exist]", infe.getMessage());
+        }
         {
             IllegalArgumentException iae = expectThrows(
                 IllegalArgumentException.class,
                 () -> indexNameExpressionResolver.concreteIndexNames(state, new DeleteIndexRequest("test-alias"))
             );
             assertEquals(
-                "The provided expression [test-alias] matches an alias, " + "specify the corresponding concrete indices instead.",
+                "The provided expression [test-alias] matches an alias, specify the corresponding concrete indices instead.",
+                iae.getMessage()
+            );
+        }
+        {
+            // same delete request but with request options that DO NOT expand wildcards
+            DeleteIndexRequest request = new DeleteIndexRequest("test-alias").indicesOptions(
+                IndicesOptions.fromOptions(false, true, false, false, false, false, true, false)
+            );
+            IllegalArgumentException iae = expectThrows(
+                IllegalArgumentException.class,
+                () -> indexNameExpressionResolver.concreteIndexNames(state, request)
+            );
+            assertEquals(
+                "The provided expression [test-alias] matches an alias, specify the corresponding concrete indices instead.",
                 iae.getMessage()
             );
         }
@@ -1874,6 +2007,13 @@ public void testDeleteIndexIgnoresAliases() {
             String[] indices = indexNameExpressionResolver.concreteIndexNames(state, deleteIndexRequest);
             assertEquals(0, indices.length);
         }
+        {
+            // same request as above but with request options that DO NOT expand wildcards
+            DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest("test-alias");
+            deleteIndexRequest.indicesOptions(IndicesOptions.fromOptions(true, true, false, false, false, false, true, false));
+            String[] indices = indexNameExpressionResolver.concreteIndexNames(state, deleteIndexRequest);
+            assertEquals(0, indices.length);
+        }
         {
             DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest("test-a*");
             deleteIndexRequest.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), false, true, true, false, false, true, false));
@@ -1892,6 +2032,16 @@ public void testDeleteIndexIgnoresAliases() {
             assertEquals(1, indices.length);
             assertEquals("test-index", indices[0]);
         }
+        {
+            String[] indices = indexNameExpressionResolver.concreteIndexNames(
+                state,
+                new DeleteIndexRequest("test-index").indicesOptions(
+                    IndicesOptions.fromOptions(false, true, false, false, false, false, false, false)
+                )
+            );
+            assertEquals(1, indices.length);
+            assertEquals("test-index", indices[0]);
+        }
         {
             String[] indices = indexNameExpressionResolver.concreteIndexNames(state, new DeleteIndexRequest("test-*"));
             assertEquals(1, indices.length);
@@ -2411,8 +2561,11 @@ public void testDataStreams() {
             assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 2, epochMillis)));
         }
         {
-            // Ignore data streams
-            IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
+            // Ignore data streams,allow no indices and expand wildcards
+            IndicesOptions indicesOptions = randomFrom(
+                IndicesOptions.STRICT_EXPAND_OPEN,
+                new IndicesOptions(EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES), EnumSet.of(IndicesOptions.WildcardStates.OPEN))
+            );
             Exception e = expectThrows(
                 IndexNotFoundException.class,
                 () -> indexNameExpressionResolver.concreteIndices(state, indicesOptions, false, "my-data-stream")
@@ -2420,10 +2573,10 @@ public void testDataStreams() {
             assertThat(e.getMessage(), equalTo("no such index [my-data-stream]"));
         }
         {
-            // Ignore data streams and allow no indices
+            // Ignore data streams and DO NOT expand wildcards
             IndicesOptions indicesOptions = new IndicesOptions(
                 EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES),
-                EnumSet.of(IndicesOptions.WildcardStates.OPEN)
+                randomFrom(EnumSet.noneOf(IndicesOptions.WildcardStates.class), EnumSet.of(IndicesOptions.WildcardStates.HIDDEN))
             );
             Exception e = expectThrows(
                 IndexNotFoundException.class,
@@ -2440,11 +2593,29 @@ public void testDataStreams() {
             Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, false, "my-data-stream");
             assertThat(result.length, equalTo(0));
         }
+        {
+            // Ignore data streams, allow no indices, ignore unavailable and DO NOT expand wildcards
+            IndicesOptions indicesOptions = new IndicesOptions(
+                EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES, IndicesOptions.Option.IGNORE_UNAVAILABLE),
+                randomFrom(EnumSet.noneOf(IndicesOptions.WildcardStates.class), EnumSet.of(IndicesOptions.WildcardStates.HIDDEN))
+            );
+            Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, false, "my-data-stream");
+            assertThat(result.length, equalTo(0));
+        }
         {
             IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
             Index result = indexNameExpressionResolver.concreteWriteIndex(state, indicesOptions, "my-data-stream", false, true);
             assertThat(result.getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 2, epochMillis)));
         }
+        {
+            // same as above but don't expand wildcards
+            IndicesOptions indicesOptions = new IndicesOptions(
+                EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES),
+                randomFrom(EnumSet.noneOf(IndicesOptions.WildcardStates.class), EnumSet.of(IndicesOptions.WildcardStates.HIDDEN))
+            );
+            Index result = indexNameExpressionResolver.concreteWriteIndex(state, indicesOptions, "my-data-stream", false, true);
+            assertThat(result.getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 2, epochMillis)));
+        }
         {
             // Ignore data streams
             IndicesOptions indicesOptions = new IndicesOptions(
@@ -2457,6 +2628,18 @@ public void testDataStreams() {
             );
             assertThat(e.getMessage(), equalTo("no such index [my-data-stream]"));
         }
+        {
+            // same as above but don't expand wildcards
+            IndicesOptions indicesOptions = new IndicesOptions(
+                EnumSet.noneOf(IndicesOptions.Option.class),
+                EnumSet.noneOf(IndicesOptions.WildcardStates.class)
+            );
+            Exception e = expectThrows(
+                IndexNotFoundException.class,
+                () -> indexNameExpressionResolver.concreteWriteIndex(state, indicesOptions, "my-data-stream", true, false)
+            );
+            assertThat(e.getMessage(), equalTo("no such index [my-data-stream]"));
+        }
         {
             // Ignore data streams and allow no indices
             IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
@@ -2466,6 +2649,18 @@ public void testDataStreams() {
             );
             assertThat(e.getMessage(), equalTo("no such index [my-data-stream]"));
         }
+        {
+            // same as above but don't expand wildcards
+            IndicesOptions indicesOptions = new IndicesOptions(
+                EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES),
+                randomFrom(EnumSet.noneOf(IndicesOptions.WildcardStates.class), EnumSet.of(IndicesOptions.WildcardStates.HIDDEN))
+            );
+            Exception e = expectThrows(
+                IndexNotFoundException.class,
+                () -> indexNameExpressionResolver.concreteWriteIndex(state, indicesOptions, "my-data-stream", false, false)
+            );
+            assertThat(e.getMessage(), equalTo("no such index [my-data-stream]"));
+        }
         {
             // Ignore data streams, allow no indices and ignore unavailable
             IndicesOptions indicesOptions = new IndicesOptions(
@@ -2478,6 +2673,18 @@ public void testDataStreams() {
             );
             assertThat(e.getMessage(), equalTo("no such index [my-data-stream]"));
         }
+        {
+            // same as above but don't expand wildcards
+            IndicesOptions indicesOptions = new IndicesOptions(
+                EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES, IndicesOptions.Option.IGNORE_UNAVAILABLE),
+                randomFrom(EnumSet.noneOf(IndicesOptions.WildcardStates.class), EnumSet.of(IndicesOptions.WildcardStates.HIDDEN))
+            );
+            Exception e = expectThrows(
+                IndexNotFoundException.class,
+                () -> indexNameExpressionResolver.concreteWriteIndex(state, indicesOptions, "my-data-stream", false, false)
+            );
+            assertThat(e.getMessage(), equalTo("no such index [my-data-stream]"));
+        }
     }
 
     public void testDataStreamAliases() {
@@ -2511,17 +2718,35 @@ public void testDataStreamAliases() {
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
 
         {
-            IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
+            IndicesOptions indicesOptions = randomFrom(
+                IndicesOptions.STRICT_EXPAND_OPEN,
+                new IndicesOptions(
+                    EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES),
+                    randomFrom(EnumSet.noneOf(IndicesOptions.WildcardStates.class), EnumSet.of(IndicesOptions.WildcardStates.HIDDEN))
+                )
+            );
             Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, dataStreamAlias1);
             assertThat(result, arrayContainingInAnyOrder(index1.getIndex(), index2.getIndex(), index3.getIndex(), index4.getIndex()));
         }
         {
-            IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
+            IndicesOptions indicesOptions = randomFrom(
+                IndicesOptions.STRICT_EXPAND_OPEN,
+                new IndicesOptions(
+                    EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES),
+                    randomFrom(EnumSet.noneOf(IndicesOptions.WildcardStates.class), EnumSet.of(IndicesOptions.WildcardStates.HIDDEN))
+                )
+            );
             Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, dataStreamAlias2);
             assertThat(result, arrayContainingInAnyOrder(index3.getIndex(), index4.getIndex()));
         }
         {
-            IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
+            IndicesOptions indicesOptions = randomFrom(
+                IndicesOptions.STRICT_EXPAND_OPEN,
+                new IndicesOptions(
+                    EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES),
+                    randomFrom(EnumSet.noneOf(IndicesOptions.WildcardStates.class), EnumSet.of(IndicesOptions.WildcardStates.HIDDEN))
+                )
+            );
             Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, dataStreamAlias3);
             assertThat(result, arrayContainingInAnyOrder(index5.getIndex(), index6.getIndex()));
         }
@@ -2533,6 +2758,18 @@ public void testDataStreamAliases() {
             );
             assertThat(e.getMessage(), equalTo("no such index [" + dataStreamAlias1 + "]"));
         }
+        {
+            // same as above but DO NOT expand wildcards
+            IndicesOptions indicesOptions = new IndicesOptions(
+                EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES),
+                randomFrom(EnumSet.noneOf(IndicesOptions.WildcardStates.class), EnumSet.of(IndicesOptions.WildcardStates.HIDDEN))
+            );
+            Exception e = expectThrows(
+                IndexNotFoundException.class,
+                () -> indexNameExpressionResolver.concreteIndices(state, indicesOptions, false, dataStreamAlias1)
+            );
+            assertThat(e.getMessage(), equalTo("no such index [" + dataStreamAlias1 + "]"));
+        }
         {
             IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
             Exception e = expectThrows(
@@ -2541,6 +2778,18 @@ public void testDataStreamAliases() {
             );
             assertThat(e.getMessage(), equalTo("no such index [" + dataStreamAlias2 + "]"));
         }
+        {
+            // same as above but DO NOT expand wildcards
+            IndicesOptions indicesOptions = new IndicesOptions(
+                EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES),
+                randomFrom(EnumSet.noneOf(IndicesOptions.WildcardStates.class), EnumSet.of(IndicesOptions.WildcardStates.HIDDEN))
+            );
+            Exception e = expectThrows(
+                IndexNotFoundException.class,
+                () -> indexNameExpressionResolver.concreteIndices(state, indicesOptions, false, dataStreamAlias2)
+            );
+            assertThat(e.getMessage(), equalTo("no such index [" + dataStreamAlias2 + "]"));
+        }
         {
             IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
             Exception e = expectThrows(
@@ -2549,6 +2798,18 @@ public void testDataStreamAliases() {
             );
             assertThat(e.getMessage(), equalTo("no such index [" + dataStreamAlias3 + "]"));
         }
+        {
+            // same as above but DO NOT expand wildcards
+            IndicesOptions indicesOptions = new IndicesOptions(
+                EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES),
+                randomFrom(EnumSet.noneOf(IndicesOptions.WildcardStates.class), EnumSet.of(IndicesOptions.WildcardStates.HIDDEN))
+            );
+            Exception e = expectThrows(
+                IndexNotFoundException.class,
+                () -> indexNameExpressionResolver.concreteIndices(state, indicesOptions, false, dataStreamAlias3)
+            );
+            assertThat(e.getMessage(), equalTo("no such index [" + dataStreamAlias3 + "]"));
+        }
         {
             IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
             Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-alias*");
@@ -2575,6 +2836,16 @@ public void testDataStreamAliases() {
             assertThat(result, notNullValue());
             assertThat(result.getName(), backingIndexEqualTo(dataStream2, 2));
         }
+        {
+            // same as above but DO NOT expand wildcards
+            IndicesOptions indicesOptions = new IndicesOptions(
+                EnumSet.of(IndicesOptions.Option.ALLOW_NO_INDICES),
+                randomFrom(EnumSet.noneOf(IndicesOptions.WildcardStates.class), EnumSet.of(IndicesOptions.WildcardStates.HIDDEN))
+            );
+            Index result = indexNameExpressionResolver.concreteWriteIndex(state, indicesOptions, dataStreamAlias1, false, true);
+            assertThat(result, notNullValue());
+            assertThat(result.getName(), backingIndexEqualTo(dataStream2, 2));
+        }
     }
 
     public void testDataStreamsWithWildcardExpression() {
@@ -2824,6 +3095,12 @@ public void testRemoteIndex() {
                 "Cross-cluster calls are not supported in this context but remote indices were requested: [cluster:index]",
                 iae.getMessage()
             );
+            // but datemath with colon doesn't trip cross-cluster check
+            IndexNotFoundException e = expectThrows(
+                IndexNotFoundException.class,
+                () -> indexNameExpressionResolver.concreteIndexNames(context, "<datemath-{2001-01-01-13||+1h/h{yyyy-MM-dd-HH|-07:00}}>")
+            );
+            assertThat(e.getMessage(), containsString("no such index [datemath-2001-01-01-14"));
         }
         {
             IndicesOptions options = IndicesOptions.fromOptions(true, true, randomBoolean(), randomBoolean(), randomBoolean());
@@ -2842,27 +3119,43 @@ public void testResolveWriteIndexAbstraction() {
             List.of(new Tuple<>("logs-foobar", 1)),
             List.of("my-index")
         );
-        state = ClusterState.builder(state)
+        final ClusterState finalState = ClusterState.builder(state)
             .metadata(
                 Metadata.builder(state.getMetadata())
                     .put(IndexMetadata.builder(state.getMetadata().index("my-index")).putAlias(new AliasMetadata.Builder("my-alias")))
                     .build()
             )
             .build();
-        DocWriteRequest<?> request = new IndexRequest("logs-foobar");
-        IndexAbstraction result = indexNameExpressionResolver.resolveWriteIndexAbstraction(state, request);
-        assertThat(result.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM));
-        assertThat(result.getName(), equalTo("logs-foobar"));
-
-        request = new IndexRequest("my-index");
-        result = indexNameExpressionResolver.resolveWriteIndexAbstraction(state, request);
-        assertThat(result.getName(), equalTo("my-index"));
-        assertThat(result.getType(), equalTo(IndexAbstraction.Type.CONCRETE_INDEX));
-
-        request = new IndexRequest("my-alias");
-        result = indexNameExpressionResolver.resolveWriteIndexAbstraction(state, request);
-        assertThat(result.getName(), equalTo("my-alias"));
-        assertThat(result.getType(), equalTo(IndexAbstraction.Type.ALIAS));
+        Function<String, List<DocWriteRequest<?>>> docWriteRequestsForName = (name) -> List.of(
+            new IndexRequest(name).opType(DocWriteRequest.OpType.INDEX),
+            new IndexRequest(name).opType(DocWriteRequest.OpType.CREATE),
+            new DeleteRequest(name),
+            new UpdateRequest(name, randomAlphaOfLength(8))
+        );
+        for (DocWriteRequest<?> request : docWriteRequestsForName.apply("logs-foobar")) {
+            if (request.opType() == DocWriteRequest.OpType.CREATE) {
+                IndexAbstraction result = indexNameExpressionResolver.resolveWriteIndexAbstraction(finalState, request);
+                assertThat(result.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM));
+                assertThat(result.getName(), equalTo("logs-foobar"));
+            } else {
+                IndexNotFoundException infe = expectThrows(
+                    IndexNotFoundException.class,
+                    () -> indexNameExpressionResolver.resolveWriteIndexAbstraction(finalState, request)
+                );
+                assertThat(infe.toString(), containsString("logs-foobar"));
+                assertThat(infe.getMetadataKeys().contains(IndexNameExpressionResolver.EXCLUDED_DATA_STREAMS_KEY), is(true));
+            }
+        }
+        for (DocWriteRequest<?> request : docWriteRequestsForName.apply("my-index")) {
+            IndexAbstraction result = indexNameExpressionResolver.resolveWriteIndexAbstraction(finalState, request);
+            assertThat(result.getName(), equalTo("my-index"));
+            assertThat(result.getType(), equalTo(IndexAbstraction.Type.CONCRETE_INDEX));
+        }
+        for (DocWriteRequest<?> request : docWriteRequestsForName.apply("my-alias")) {
+            IndexAbstraction result = indexNameExpressionResolver.resolveWriteIndexAbstraction(finalState, request);
+            assertThat(result.getName(), equalTo("my-alias"));
+            assertThat(result.getType(), equalTo(IndexAbstraction.Type.ALIAS));
+        }
     }
 
     public void testResolveWriteIndexAbstractionNoWriteIndexForAlias() {
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadataTests.java
index 768d86ee42ff..bea57d5175a2 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadataTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.DeprecationHandler;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -31,7 +32,7 @@ public class IndexTemplateMetadataTests extends ESTestCase {
 
     public void testIndexTemplateMetadataXContentRoundTrip() throws Exception {
 
-        String template = formatted("""
+        String template = Strings.format("""
             {
               "index_patterns": [ ".test-*" ],
               "order": 1000,
@@ -112,7 +113,7 @@ public void testValidateInvalidIndexPatterns() throws Exception {
         });
         assertThat(nullPatternError.getMessage(), equalTo("Index patterns must not be null or empty; got null"));
 
-        final String templateWithEmptyPattern = formatted("""
+        final String templateWithEmptyPattern = Strings.format("""
             {
               "index_patterns": [],
               "order": 1000,
@@ -145,7 +146,7 @@ public void testValidateInvalidIndexPatterns() throws Exception {
             assertThat(ex.getMessage(), equalTo("Index patterns must not be null or empty; got []"));
         }
 
-        final String templateWithoutPattern = formatted("""
+        final String templateWithoutPattern = Strings.format("""
             {
               "order": 1000,
               "settings": {
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ItemUsageTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ItemUsageTests.java
index 63e7257d8007..8b6e5cf9bf34 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ItemUsageTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ItemUsageTests.java
@@ -41,7 +41,7 @@ protected ItemUsage copyInstance(ItemUsage instance, Version version) throws IOE
     }
 
     @Override
-    protected ItemUsage mutateInstance(ItemUsage instance) throws IOException {
-        return super.mutateInstance(instance);
+    protected ItemUsage mutateInstance(ItemUsage instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java
index c8400ac9197d..a8527a181881 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java
@@ -109,7 +109,7 @@ public void testCreateDataStreamWithAliasFromTemplate() throws Exception {
             var actualAlias = newState.metadata().dataStreamAliases().get(aliasName);
             assertThat(actualAlias, is(notNullValue()));
             assertThat(actualAlias.getName(), equalTo(expectedAlias.alias()));
-            assertThat(actualAlias.getFilter(), equalTo(expectedAlias.filter()));
+            assertThat(actualAlias.getFilter(dataStreamName), equalTo(expectedAlias.filter()));
             assertThat(actualAlias.getWriteDataStream(), equalTo(expectedAlias.writeIndex() ? dataStreamName : null));
         }
 
@@ -183,7 +183,7 @@ public void testCreateDataStreamWithAliasFromComponentTemplate() throws Exceptio
                 var actualAlias = newState.metadata().dataStreamAliases().get(alias.alias());
                 assertThat(actualAlias, is(notNullValue()));
                 assertThat(actualAlias.getName(), equalTo(alias.alias()));
-                assertThat(actualAlias.getFilter(), equalTo(alias.filter()));
+                assertThat(actualAlias.getFilter(dataStreamName), equalTo(alias.filter()));
                 assertThat(actualAlias.getWriteDataStream(), equalTo(alias.writeIndex() ? dataStreamName : null));
             }
         }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java
index e9dc4a6f6f49..94e4c2664f42 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
@@ -150,7 +151,7 @@ private ClusterState createClusterState(String name, int numShards, int numRepli
             .build();
         metaBuilder.put(indexMetadata, false);
         Metadata metadata = metaBuilder.build();
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         routingTableBuilder.addAsNew(metadata.index(name));
 
         RoutingTable routingTable = routingTableBuilder.build();
@@ -267,7 +268,8 @@ public void testValidateShrinkIndex() {
             new TestGatewayAllocator(),
             new BalancedShardsAllocator(Settings.EMPTY),
             EmptyClusterInfoService.INSTANCE,
-            EmptySnapshotsInfoService.INSTANCE
+            EmptySnapshotsInfoService.INSTANCE,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
 
         RoutingTable routingTable = service.reroute(clusterState, "reroute", ActionListener.noop()).routingTable();
@@ -369,7 +371,8 @@ public void testValidateSplitIndex() {
             new TestGatewayAllocator(),
             new BalancedShardsAllocator(Settings.EMPTY),
             EmptyClusterInfoService.INSTANCE,
-            EmptySnapshotsInfoService.INSTANCE
+            EmptySnapshotsInfoService.INSTANCE,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
 
         RoutingTable routingTable = service.reroute(clusterState, "reroute", ActionListener.noop()).routingTable();
@@ -517,7 +520,8 @@ private void runPrepareResizeIndexSettingsTest(
             new TestGatewayAllocator(),
             new BalancedShardsAllocator(Settings.EMPTY),
             EmptyClusterInfoService.INSTANCE,
-            EmptySnapshotsInfoService.INSTANCE
+            EmptySnapshotsInfoService.INSTANCE,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
 
         final RoutingTable initialRoutingTable = service.reroute(initialClusterState, "reroute", ActionListener.noop()).routingTable();
@@ -985,8 +989,16 @@ public void testClusterStateCreateIndexThrowsWriteIndexValidationException() thr
             .build();
 
         assertThat(
-            expectThrows(IllegalStateException.class, () -> clusterStateCreateIndex(currentClusterState, Set.of(), newIndex, null))
-                .getMessage(),
+            expectThrows(
+                IllegalStateException.class,
+                () -> clusterStateCreateIndex(
+                    currentClusterState,
+                    Set.of(),
+                    newIndex,
+                    null,
+                    TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
+                )
+            ).getMessage(),
             startsWith("alias [alias1] has more than one write index [")
         );
     }
@@ -1005,7 +1017,8 @@ public void testClusterStateCreateIndex() {
             currentClusterState,
             Set.of(INDEX_READ_ONLY_BLOCK),
             newIndexMetadata,
-            null
+            null,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
         assertThat(updatedClusterState.blocks().getIndexBlockWithId("test", INDEX_READ_ONLY_BLOCK.id()), is(INDEX_READ_ONLY_BLOCK));
         assertThat(updatedClusterState.routingTable().index("test"), is(notNullValue()));
@@ -1050,7 +1063,8 @@ public void testClusterStateCreateIndexWithMetadataTransaction() {
             currentClusterState,
             Set.of(INDEX_READ_ONLY_BLOCK),
             newIndexMetadata,
-            metadataTransformer
+            metadataTransformer,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
         assertTrue(updatedClusterState.metadata().findAllAliases(new String[] { "my-index" }).containsKey("my-index"));
         assertNotNull(updatedClusterState.metadata().findAllAliases(new String[] { "my-index" }).get("my-index"));
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java
index e26a970d3e1a..bed2b9de9710 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.SnapshotsInProgress;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
@@ -144,7 +145,7 @@ public void testDeleteIndexWithAnAlias() {
             .build();
         ClusterState before = ClusterState.builder(ClusterName.DEFAULT)
             .metadata(Metadata.builder().put(idxMetadata, false))
-            .routingTable(RoutingTable.builder().addAsNew(idxMetadata).build())
+            .routingTable(RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(idxMetadata).build())
             .blocks(ClusterBlocks.builder().addBlocks(idxMetadata))
             .build();
 
@@ -232,7 +233,7 @@ private ClusterState clusterState(String index) {
             .build();
         return ClusterState.builder(ClusterName.DEFAULT)
             .metadata(Metadata.builder().put(indexMetadata, false))
-            .routingTable(RoutingTable.builder().addAsNew(indexMetadata).build())
+            .routingTable(RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(indexMetadata).build())
             .blocks(ClusterBlocks.builder().addBlocks(indexMetadata))
             .build();
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java
index 4534727b7592..4f99f5ea943c 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesServiceTests.java
@@ -10,8 +10,10 @@
 
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesClusterStateUpdateRequest;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.service.ClusterStateTaskExecutorUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.Tuple;
@@ -292,13 +294,13 @@ public void testAddWriteOnlyWithNoExistingAliases() {
 
         ClusterState after = service.applyAliasActions(
             before,
-            Arrays.asList(new AliasAction.Add("test", "alias", null, null, null, false, null))
+            List.of(new AliasAction.Add("test", "alias", null, null, null, false, null))
         );
         assertFalse(after.metadata().index("test").getAliases().get("alias").writeIndex());
         assertNull(after.metadata().getIndicesLookup().get("alias").getWriteIndex());
         assertAliasesVersionIncreased("test", before, after);
 
-        after = service.applyAliasActions(before, Arrays.asList(new AliasAction.Add("test", "alias", null, null, null, null, null)));
+        after = service.applyAliasActions(before, List.of(new AliasAction.Add("test", "alias", null, null, null, null, null)));
         assertNull(after.metadata().index("test").getAliases().get("alias").writeIndex());
         assertThat(
             after.metadata().index(after.metadata().getIndicesLookup().get("alias").getWriteIndex()),
@@ -306,7 +308,7 @@ public void testAddWriteOnlyWithNoExistingAliases() {
         );
         assertAliasesVersionIncreased("test", before, after);
 
-        after = service.applyAliasActions(before, Arrays.asList(new AliasAction.Add("test", "alias", null, null, null, true, null)));
+        after = service.applyAliasActions(before, List.of(new AliasAction.Add("test", "alias", null, null, null, true, null)));
         assertTrue(after.metadata().index("test").getAliases().get("alias").writeIndex());
         assertThat(
             after.metadata().index(after.metadata().getIndicesLookup().get("alias").getWriteIndex()),
@@ -329,10 +331,7 @@ public void testAddWriteOnlyWithExistingWriteIndex() {
             .metadata(Metadata.builder().put(indexMetadata).put(indexMetadata2))
             .build();
 
-        ClusterState after = service.applyAliasActions(
-            before,
-            Arrays.asList(new AliasAction.Add("test", "alias", null, null, null, null, null))
-        );
+        ClusterState after = service.applyAliasActions(before, List.of(new AliasAction.Add("test", "alias", null, null, null, null, null)));
         assertNull(after.metadata().index("test").getAliases().get("alias").writeIndex());
         assertThat(
             after.metadata().index(after.metadata().getIndicesLookup().get("alias").getWriteIndex()),
@@ -343,7 +342,7 @@ public void testAddWriteOnlyWithExistingWriteIndex() {
 
         Exception exception = expectThrows(
             IllegalStateException.class,
-            () -> service.applyAliasActions(before, Arrays.asList(new AliasAction.Add("test", "alias", null, null, null, true, null)))
+            () -> service.applyAliasActions(before, List.of(new AliasAction.Add("test", "alias", null, null, null, true, null)))
         );
         assertThat(exception.getMessage(), startsWith("alias [alias] has more than one write index ["));
     }
@@ -402,7 +401,7 @@ public void testAddWriteOnlyWithExistingNonWriteIndices() {
 
         ClusterState after = service.applyAliasActions(
             before,
-            Arrays.asList(new AliasAction.Add("test3", "alias", null, null, null, true, null))
+            List.of(new AliasAction.Add("test3", "alias", null, null, null, true, null))
         );
         assertTrue(after.metadata().index("test3").getAliases().get("alias").writeIndex());
         assertThat(
@@ -675,6 +674,44 @@ public void testDataStreamAliasesWithWriteFlag() {
         assertThat(result.metadata().dataStreamAliases().get("logs-http"), nullValue());
     }
 
+    public void testAddAndRemoveAliasClusterStateUpdate() throws Exception {
+        // Create a state with a single index
+        String index = randomAlphaOfLength(5);
+        ClusterState before = createIndex(ClusterState.builder(ClusterName.DEFAULT).build(), index);
+        IndicesAliasesClusterStateUpdateRequest addAliasRequest = new IndicesAliasesClusterStateUpdateRequest(
+            List.of(new AliasAction.Add(index, "test", null, null, null, null, null))
+        );
+        IndicesAliasesClusterStateUpdateRequest removeAliasRequest = new IndicesAliasesClusterStateUpdateRequest(
+            List.of(new AliasAction.Remove(index, "test", true))
+        );
+
+        ClusterState after = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(
+            before,
+            service.getExecutor(),
+            List.of(
+                new MetadataIndexAliasesService.ApplyAliasesTask(addAliasRequest, null),
+                // Repeat the same change to ensure that the clte version won't increase
+                new MetadataIndexAliasesService.ApplyAliasesTask(addAliasRequest, null),
+                new MetadataIndexAliasesService.ApplyAliasesTask(removeAliasRequest, null),
+                new MetadataIndexAliasesService.ApplyAliasesTask(addAliasRequest, null)
+            )
+        );
+
+        IndexAbstraction alias = after.metadata().getIndicesLookup().get("test");
+        assertNotNull(alias);
+        assertThat(alias.getType(), equalTo(IndexAbstraction.Type.ALIAS));
+        assertThat(alias.getIndices(), contains(after.metadata().index(index).getIndex()));
+        assertAliasesVersionIncreased(new String[] { index }, before, after, 3);
+        assertThat(after.metadata().aliasedIndices("test"), contains(after.metadata().index(index).getIndex()));
+    }
+
+    public void testEmptyTaskListProducesSameClusterState() throws Exception {
+        String index = randomAlphaOfLength(5);
+        ClusterState before = createIndex(ClusterState.builder(ClusterName.DEFAULT).build(), index);
+        ClusterState after = ClusterStateTaskExecutorUtils.executeAndAssertSuccessful(before, service.getExecutor(), List.of());
+        assertSame(before, after);
+    }
+
     private ClusterState applyHiddenAliasMix(ClusterState before, Boolean isHidden1, Boolean isHidden2) {
         return service.applyAliasActions(
             before,
@@ -711,11 +748,19 @@ private void assertAliasesVersionIncreased(final String index, final ClusterStat
     }
 
     private void assertAliasesVersionIncreased(final String[] indices, final ClusterState before, final ClusterState after) {
+        assertAliasesVersionIncreased(indices, before, after, 1);
+    }
+
+    private void assertAliasesVersionIncreased(
+        final String[] indices,
+        final ClusterState before,
+        final ClusterState after,
+        final int diff
+    ) {
         for (final var index : indices) {
-            final long expected = 1 + before.metadata().index(index).getAliasesVersion();
+            final long expected = diff + before.metadata().index(index).getAliasesVersion();
             final long actual = after.metadata().index(index).getAliasesVersion();
             assertThat("index metadata aliases version mismatch", actual, equalTo(expected));
         }
     }
-
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java
index c6df4f7ad53b..0bf6dfd3441b 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.RestoreInProgress;
 import org.elasticsearch.cluster.SnapshotsInProgress;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
@@ -85,7 +86,12 @@ public void testCloseRoutingTable() {
             }
         }
 
-        final ClusterState updatedState = MetadataIndexStateService.closeRoutingTable(state, blockedIndices, results).v1();
+        final ClusterState updatedState = MetadataIndexStateService.closeRoutingTable(
+            state,
+            blockedIndices,
+            results,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
+        ).v1();
         assertThat(updatedState.metadata().indices().size(), equalTo(nonBlockedIndices.size() + blockedIndices.size()));
 
         for (Index nonBlockedIndex : nonBlockedIndices) {
@@ -114,7 +120,8 @@ public void testCloseRoutingTableWithRestoredIndex() {
         final ClusterState updatedState = MetadataIndexStateService.closeRoutingTable(
             state,
             Map.of(index, block),
-            Map.of(index, new IndexResult(index))
+            Map.of(index, new IndexResult(index)),
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         ).v1();
         assertIsOpened(index.getName(), updatedState);
         assertThat(updatedState.blocks().hasIndexBlockWithId(index.getName(), INDEX_CLOSED_BLOCK_ID), is(true));
@@ -132,7 +139,8 @@ public void testCloseRoutingTableWithSnapshottedIndex() {
         final ClusterState updatedState = MetadataIndexStateService.closeRoutingTable(
             state,
             Map.of(index, block),
-            Map.of(index, new IndexResult(index))
+            Map.of(index, new IndexResult(index)),
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         ).v1();
         assertIsOpened(index.getName(), updatedState);
         assertThat(updatedState.blocks().hasIndexBlockWithId(index.getName(), INDEX_CLOSED_BLOCK_ID), is(true));
@@ -302,7 +310,8 @@ public void testCloseFailedIfBlockDisappeared() {
         Collection<IndexResult> closingResults = MetadataIndexStateService.closeRoutingTable(
             state,
             blockedIndices,
-            Map.copyOf(verifyResults)
+            Map.copyOf(verifyResults),
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         ).v2();
         assertThat(closingResults, hasSize(numIndices));
         Set<Index> failedIndices = closingResults.stream()
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceUtils.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceUtils.java
index bcb310763e49..2159e2f37e1a 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceUtils.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceUtils.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.action.admin.indices.close.CloseIndexResponse;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.index.Index;
 
@@ -30,13 +31,14 @@ public static ClusterState addIndexClosedBlocks(
     }
 
     /**
-     * Allows to call {@link MetadataIndexStateService#closeRoutingTable(ClusterState, Map, Map)} which is a protected method.
+     * Allows to call {@link MetadataIndexStateService#closeRoutingTable} which is a protected method.
      */
     public static ClusterState closeRoutingTable(
         final ClusterState state,
         final Map<Index, ClusterBlock> blockedIndices,
         final Map<Index, CloseIndexResponse.IndexResult> results
     ) {
-        return MetadataIndexStateService.closeRoutingTable(state, blockedIndices, results).v1();
+        return MetadataIndexStateService.closeRoutingTable(state, blockedIndices, results, TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .v1();
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java
index 91bf2f1b8090..e8ad6d75736b 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java
@@ -597,6 +597,7 @@ public void testRemoveIndexTemplateV2Wildcards() throws Exception {
 
         ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "foo", template);
         assertThat(state.metadata().templatesV2().get("foo"), notNullValue());
+
         assertTemplatesEqual(state.metadata().templatesV2().get("foo"), template);
 
         Exception e = expectThrows(
@@ -1529,7 +1530,12 @@ public void testAddInvalidTemplate() throws Exception {
         });
 
         assertThat(e.name(), equalTo("template"));
-        assertThat(e.getMessage(), containsString("index template [template] specifies " + "component templates [bad] that do not exist"));
+        assertThat(
+            e.getMessage(),
+            containsString(
+                "index_template [template] invalid, cause [index template [template] specifies component templates [bad] that do not exist]"
+            )
+        );
     }
 
     public void testRemoveComponentTemplate() throws Exception {
@@ -1723,11 +1729,12 @@ public void testUpdateComponentTemplateFailsIfResolvedIndexTemplatesWouldBeInval
         );
 
         assertNotNull(e.getCause());
-        assertThat(e.getCause().getMessage(), containsString("invalid composite mappings for [my-template]"));
-
         assertNotNull(e.getCause().getCause());
+        assertThat(e.getCause().getCause().getMessage(), containsString("invalid composite mappings for [my-template]"));
+
+        assertNotNull(e.getCause().getCause().getCause());
         assertThat(
-            e.getCause().getCause().getMessage(),
+            e.getCause().getCause().getCause().getMessage(),
             containsString("can't merge a non object mapping [field2] with an object mapping")
         );
     }
@@ -2107,6 +2114,146 @@ public void testV2TemplateOverlaps() throws Exception {
         }
     }
 
+    /**
+     * Tests to add two component templates but ignores both with is valid
+     *
+     * @throws Exception
+     */
+    public void testIgnoreMissingComponentTemplateValid() throws Exception {
+
+        String indexTemplateName = "metric-test";
+
+        List<String> componentTemplates = new ArrayList<>();
+        componentTemplates.add("foo");
+        componentTemplates.add("bar");
+
+        // Order of params is mixed up on purpose
+        List<String> ignoreMissingComponentTemplates = new ArrayList<>();
+        ignoreMissingComponentTemplates.add("bar");
+        ignoreMissingComponentTemplates.add("foo");
+
+        ComposableIndexTemplate template = new ComposableIndexTemplate(
+            Arrays.asList("metrics-test-*"),
+            null,
+            componentTemplates,
+            1L,
+            null,
+            null,
+            null,
+            null,
+            ignoreMissingComponentTemplates
+        );
+        MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService();
+
+        ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, indexTemplateName, template);
+        MetadataIndexTemplateService.validateV2TemplateRequest(state.metadata(), indexTemplateName, template);
+    }
+
+    public void testIgnoreMissingComponentTemplateInvalid() throws Exception {
+
+        String indexTemplateName = "metric-test";
+
+        List<String> componentTemplates = new ArrayList<>();
+        componentTemplates.add("foo");
+        componentTemplates.add("fail");
+
+        List<String> ignoreMissingComponentTemplates = new ArrayList<>();
+        ignoreMissingComponentTemplates.add("bar");
+        ignoreMissingComponentTemplates.add("foo");
+
+        ComposableIndexTemplate template = new ComposableIndexTemplate(
+            Arrays.asList("metrics-foo-*"),
+            null,
+            componentTemplates,
+            1L,
+            null,
+            null,
+            null,
+            null,
+            ignoreMissingComponentTemplates
+        );
+
+        MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService();
+        ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, indexTemplateName, template);
+
+        // try now the same thing with validation on
+        InvalidIndexTemplateException e = expectThrows(
+            InvalidIndexTemplateException.class,
+            () -> MetadataIndexTemplateService.validateV2TemplateRequest(state.metadata(), indexTemplateName, template)
+
+        );
+        assertThat(e.getMessage(), containsString("specifies a missing component templates [fail] that does not exist"));
+    }
+
+    /**
+     * This is a similar test as above but with running the service
+     * @throws Exception
+     */
+    public void testAddInvalidTemplateIgnoreService() throws Exception {
+
+        String indexTemplateName = "metric-test";
+
+        List<String> componentTemplates = new ArrayList<>();
+        componentTemplates.add("foo");
+        componentTemplates.add("fail");
+
+        List<String> ignoreMissingComponentTemplates = new ArrayList<>();
+        ignoreMissingComponentTemplates.add("bar");
+        ignoreMissingComponentTemplates.add("foo");
+
+        ComposableIndexTemplate template = new ComposableIndexTemplate(
+            Arrays.asList("metrics-foo-*"),
+            null,
+            componentTemplates,
+            1L,
+            null,
+            null,
+            null,
+            null,
+            ignoreMissingComponentTemplates
+        );
+
+        ComponentTemplate ct = new ComponentTemplate(new Template(Settings.EMPTY, null, null), null, null);
+
+        final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
+        CountDownLatch ctLatch = new CountDownLatch(1);
+        // Makes ure the foo template exists
+        service.putComponentTemplate(
+            "api",
+            randomBoolean(),
+            "foo",
+            TimeValue.timeValueSeconds(5),
+            ct,
+            ActionListener.wrap(r -> ctLatch.countDown(), e -> {
+                logger.error("unexpected error", e);
+                fail("unexpected error");
+            })
+        );
+        ctLatch.await(5, TimeUnit.SECONDS);
+        InvalidIndexTemplateException e = expectThrows(InvalidIndexTemplateException.class, () -> {
+            CountDownLatch latch = new CountDownLatch(1);
+            AtomicReference<Exception> err = new AtomicReference<>();
+            service.putIndexTemplateV2(
+                "api",
+                randomBoolean(),
+                "template",
+                TimeValue.timeValueSeconds(30),
+                template,
+                ActionListener.wrap(r -> fail("should have failed!"), exception -> {
+                    err.set(exception);
+                    latch.countDown();
+                })
+            );
+            latch.await(5, TimeUnit.SECONDS);
+            if (err.get() != null) {
+                throw err.get();
+            }
+        });
+
+        assertThat(e.name(), equalTo("template"));
+        assertThat(e.getMessage(), containsString("missing component templates [fail] that does not exist"));
+    }
+
     private static List<Throwable> putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) {
         ThreadPool testThreadPool = mock(ThreadPool.class);
         ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool);
@@ -2199,6 +2346,6 @@ private MetadataIndexTemplateService getMetadataIndexTemplateService() {
     }
 
     public static void assertTemplatesEqual(ComposableIndexTemplate actual, ComposableIndexTemplate expected) {
-        assertTrue(Objects.equals(actual, expected));
+        assertEquals(actual, expected);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java
index 92b228b17a8f..95dbc190182f 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.cluster.ClusterModule;
@@ -28,12 +29,14 @@
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.alias.RandomAliasActionsGenerator;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.plugins.MapperPlugin;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.xcontent.ToXContent;
@@ -42,11 +45,14 @@
 import org.elasticsearch.xcontent.json.JsonXContent;
 
 import java.io.IOException;
+import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -2269,6 +2275,97 @@ public void testEmptyDiffReturnsSameInstance() throws IOException {
         assertSame(instance, deserializedDiff.apply(instance));
     }
 
+    public void testChunkedToXContent() throws IOException {
+        final int datastreams = randomInt(10);
+        // 2 chunks at the beginning
+        // 1 chunk for each index + 2 to wrap the indices field
+        // 2 chunks for wrapping reserved state + 1 chunk for each item
+        // 2 chunks wrapping templates and one chunk per template
+        // 2 chunks to wrap each custom
+        // 1 chunk per datastream, 4 chunks to wrap ds and ds-aliases, or 0 if there are no datastreams
+        // 2 chunks to wrap index graveyard and one per tombstone
+        // 2 chunks to wrap component templates and one per component template
+        // 2 chunks to wrap v2 templates and one per v2 template
+        // 1 chunk to close metadata
+        AbstractChunkedSerializingTestCase.assertChunkCount(randomMetadata(datastreams), instance -> {
+            // 2 chunks at the beginning
+            // 1 chunk for each index + 2 to wrap the indices field
+            final int indicesChunks = instance.indices().size() + 2;
+            // 2 chunks for wrapping reserved state + 1 chunk for each item
+            final int reservedStateChunks = instance.reservedStateMetadata().size() + 2;
+            // 2 chunks wrapping templates and one chunk per template
+            final int templatesChunks = instance.templates().size() + 2;
+            // 2 chunks to wrap each custom
+            final int customChunks = 2 * instance.customs().size();
+            // 1 chunk per datastream, 4 chunks to wrap ds and ds-aliases, or 0 if there are no datastreams
+            final int dsChunks = datastreams == 0 ? 0 : (datastreams + 4);
+            // 2 chunks to wrap index graveyard and one per tombstone
+            final int graveYardChunks = instance.indexGraveyard().getTombstones().size() + 2;
+            // 2 chunks to wrap component templates and one per component template
+            final int componentTemplateChunks = instance.componentTemplates().size() + 2;
+            // 2 chunks to wrap v2 templates and one per v2 template
+            final int v2TemplateChunks = instance.templatesV2().size() + 2;
+            // 1 chunk to close metadata
+
+            return 2 + indicesChunks + reservedStateChunks + templatesChunks + customChunks + dsChunks + graveYardChunks
+                + componentTemplateChunks + v2TemplateChunks + 1;
+        });
+    }
+
+    /**
+     * With this test we ensure that we consider whether a new field added to Metadata should be checked
+     * in Metadata.isGlobalStateEquals. We force the instance fields to be either in the checked list
+     * or in the excluded list.
+     * <p>
+     * This prevents from accidentally forgetting that a new field should be checked in isGlobalStateEquals.
+     */
+    @SuppressForbidden(reason = "need access to all fields, they are mostly private")
+    public void testEnsureMetadataFieldCheckedForGlobalStateChanges() {
+        Set<String> checkedForGlobalStateChanges = Set.of(
+            "coordinationMetadata",
+            "persistentSettings",
+            "hashesOfConsistentSettings",
+            "templates",
+            "clusterUUID",
+            "clusterUUIDCommitted",
+            "customs",
+            "reservedStateMetadata"
+        );
+        Set<String> excludedFromGlobalStateCheck = Set.of(
+            "version",
+            "transientSettings",
+            "settings",
+            "indices",
+            "aliasedIndices",
+            "totalNumberOfShards",
+            "totalOpenIndexShards",
+            "allIndices",
+            "visibleIndices",
+            "allOpenIndices",
+            "visibleOpenIndices",
+            "allClosedIndices",
+            "visibleClosedIndices",
+            "indicesLookup",
+            "mappingsByHash",
+            "oldestIndexVersion"
+        );
+
+        var diff = new HashSet<>(checkedForGlobalStateChanges);
+        diff.removeAll(excludedFromGlobalStateCheck);
+
+        // sanity check that the two field sets are mutually exclusive
+        assertEquals(checkedForGlobalStateChanges, diff);
+
+        // any declared non-static field in metadata must be either in the list of fields
+        // we check for global state changes, or in the fields excluded from the global state check.
+        var unclassifiedFields = Arrays.stream(Metadata.class.getDeclaredFields())
+            .filter(f -> Modifier.isStatic(f.getModifiers()) == false)
+            .map(f -> f.getName())
+            .filter(n -> (checkedForGlobalStateChanges.contains(n) || excludedFromGlobalStateCheck.contains(n)) == false)
+            .collect(Collectors.toSet());
+        assertThat(unclassifiedFields, empty());
+    }
+
     public static Metadata randomMetadata() {
         return randomMetadata(1);
     }
@@ -2348,9 +2445,10 @@ private static class CreateIndexResult {
     }
 
     private static class TestCustomMetadata implements Metadata.Custom {
+
         @Override
-        public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
-            return null;
+        public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+            return Collections.emptyIterator();
         }
 
         @Override
@@ -2369,7 +2467,7 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
+        public TransportVersion getMinimalSupportedVersion() {
             return null;
         }
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadataTests.java
index 381d64a249ab..70cf38755ab1 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadataTests.java
@@ -17,7 +17,7 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.test.SimpleDiffableSerializationTestCase;
+import org.elasticsearch.test.ChunkedToXContentDiffableSerializationTestCase;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -35,7 +35,7 @@
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
-public class NodesShutdownMetadataTests extends SimpleDiffableSerializationTestCase<Metadata.Custom> {
+public class NodesShutdownMetadataTests extends ChunkedToXContentDiffableSerializationTestCase<Metadata.Custom> {
 
     public void testInsertNewNodeShutdownMetadata() {
         NodesShutdownMetadata nodesShutdownMetadata = new NodesShutdownMetadata(new HashMap<>());
@@ -139,7 +139,7 @@ protected Metadata.Custom makeTestChanges(Metadata.Custom testInstance) {
     }
 
     @Override
-    protected Metadata.Custom mutateInstance(Metadata.Custom instance) throws IOException {
+    protected Metadata.Custom mutateInstance(Metadata.Custom instance) {
         return makeTestChanges(instance);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java
index ffb21eaa0b2c..200fd596b4cb 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -189,7 +190,7 @@ public void testUpdateTemplates() {
         }
         Map<String, BytesReference> additions = Maps.newMapWithExpectedSize(additionsCount);
         for (int i = 0; i < additionsCount; i++) {
-            additions.put("add_template_" + i, new BytesArray(formatted("""
+            additions.put("add_template_" + i, new BytesArray(Strings.format("""
                 {"index_patterns" : "*", "order" : %s}
                 """, i)));
         }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java
index 9bd25e82cdc5..061189e2304a 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java
@@ -8,12 +8,14 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.rollover.RolloverInfo;
 import org.elasticsearch.cluster.coordination.CoordinationMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TestCustomMetadata;
 import org.elasticsearch.xcontent.ToXContent;
@@ -226,10 +228,10 @@ public void testToXContentGateway_FlatSettingTrue_ReduceMappingFalse() throws IO
         Metadata metadata = buildMetadata();
         XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();
         builder.startObject();
-        metadata.toXContent(builder, new ToXContent.MapParams(mapParams));
+        ChunkedToXContent.wrapAsToXContent(metadata).toXContent(builder, new ToXContent.MapParams(mapParams));
         builder.endObject();
 
-        assertEquals(formatted("""
+        assertEquals(Strings.format("""
             {
               "meta-data" : {
                 "version" : 0,
@@ -314,10 +316,10 @@ public void testToXContentAPI_SameTypeName() throws IOException {
             .build();
         XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();
         builder.startObject();
-        metadata.toXContent(builder, new ToXContent.MapParams(mapParams));
+        ChunkedToXContent.wrapAsToXContent(metadata).toXContent(builder, new ToXContent.MapParams(mapParams));
         builder.endObject();
 
-        assertEquals(formatted("""
+        assertEquals(Strings.format("""
             {
               "metadata" : {
                 "cluster_uuid" : "clusterUUID",
@@ -385,10 +387,10 @@ public void testToXContentGateway_FlatSettingFalse_ReduceMappingTrue() throws IO
         Metadata metadata = buildMetadata();
         XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();
         builder.startObject();
-        metadata.toXContent(builder, new ToXContent.MapParams(mapParams));
+        ChunkedToXContent.wrapAsToXContent(metadata).toXContent(builder, new ToXContent.MapParams(mapParams));
         builder.endObject();
 
-        assertEquals(formatted("""
+        assertEquals(Strings.format("""
             {
               "meta-data" : {
                 "version" : 0,
@@ -451,10 +453,10 @@ public void testToXContentAPI_FlatSettingTrue_ReduceMappingFalse() throws IOExce
 
         XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();
         builder.startObject();
-        metadata.toXContent(builder, new ToXContent.MapParams(mapParams));
+        ChunkedToXContent.wrapAsToXContent(metadata).toXContent(builder, new ToXContent.MapParams(mapParams));
         builder.endObject();
 
-        assertEquals(formatted("""
+        assertEquals(Strings.format("""
             {
               "metadata" : {
                 "cluster_uuid" : "clusterUUID",
@@ -554,10 +556,10 @@ public void testToXContentAPI_FlatSettingFalse_ReduceMappingTrue() throws IOExce
 
         XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();
         builder.startObject();
-        metadata.toXContent(builder, new ToXContent.MapParams(mapParams));
+        ChunkedToXContent.wrapAsToXContent(metadata).toXContent(builder, new ToXContent.MapParams(mapParams));
         builder.endObject();
 
-        assertEquals(formatted("""
+        assertEquals(Strings.format("""
             {
               "metadata" : {
                 "cluster_uuid" : "clusterUUID",
@@ -689,10 +691,10 @@ public void testToXContentAPIReservedMetadata() throws IOException {
 
         XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();
         builder.startObject();
-        metadata.toXContent(builder, new ToXContent.MapParams(mapParams));
+        ChunkedToXContent.wrapAsToXContent(metadata).toXContent(builder, new ToXContent.MapParams(mapParams));
         builder.endObject();
 
-        assertEquals(formatted("""
+        assertEquals(Strings.format("""
             {
               "metadata" : {
                 "cluster_uuid" : "clusterUUID",
@@ -899,8 +901,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT;
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java
index 3356033d5b73..65cdd4c92666 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/WildcardExpressionResolverTests.java
@@ -28,6 +28,7 @@
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
 
 public class WildcardExpressionResolverTests extends ESTestCase {
 
@@ -96,7 +97,7 @@ public void testConvertWildcardsJustIndicesTests() {
         } else if (indicesOptions == IndicesOptions.strictExpandOpen()) {
             IndexNotFoundException infe = expectThrows(
                 IndexNotFoundException.class,
-                () -> IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Arrays.asList("testXXX", "-testXXX"))
+                () -> IndexNameExpressionResolver.resolveExpressions(context, "testXXX", "-testXXX")
             );
             assertEquals("-testXXX", infe.getIndex().getName());
         }
@@ -178,6 +179,23 @@ public void testConvertWildcardsOpenClosedIndicesTests() {
             newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Collections.singletonList("testX*"))),
             equalTo(newHashSet("testXXX", "testXXY"))
         );
+        context = new IndexNameExpressionResolver.Context(
+            state,
+            IndicesOptions.fromOptions(true, true, false, false),
+            SystemIndexAccessLevel.NONE
+        );
+        assertThat(IndexNameExpressionResolver.resolveExpressions(context, "testX*").size(), equalTo(0));
+        context = new IndexNameExpressionResolver.Context(
+            state,
+            IndicesOptions.fromOptions(false, true, false, false),
+            SystemIndexAccessLevel.NONE
+        );
+        IndexNameExpressionResolver.Context finalContext = context;
+        IndexNotFoundException infe = expectThrows(
+            IndexNotFoundException.class,
+            () -> IndexNameExpressionResolver.resolveExpressions(finalContext, "testX*")
+        );
+        assertThat(infe.getIndex().getName(), is("testX*"));
     }
 
     // issue #13334
@@ -237,9 +255,29 @@ public void testAll() {
             SystemIndexAccessLevel.NONE
         );
         assertThat(
-            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolve(context, Collections.singletonList("_all"))),
+            newHashSet(IndexNameExpressionResolver.WildcardExpressionResolver.resolveAll(context)),
             equalTo(newHashSet("testXXX", "testXYY", "testYYY"))
         );
+        assertThat(
+            newHashSet(IndexNameExpressionResolver.resolveExpressions(context, "_all")),
+            equalTo(newHashSet("testXXX", "testXYY", "testYYY"))
+        );
+        IndicesOptions noExpandOptions = IndicesOptions.fromOptions(
+            randomBoolean(),
+            true,
+            false,
+            false,
+            randomBoolean(),
+            randomBoolean(),
+            randomBoolean(),
+            randomBoolean()
+        );
+        IndexNameExpressionResolver.Context noExpandContext = new IndexNameExpressionResolver.Context(
+            state,
+            noExpandOptions,
+            SystemIndexAccessLevel.NONE
+        );
+        assertThat(IndexNameExpressionResolver.resolveExpressions(noExpandContext, "_all").size(), equalTo(0));
     }
 
     public void testResolveEmpty() {
@@ -420,13 +458,48 @@ public void testResolveAliases() {
         {
             IllegalArgumentException iae = expectThrows(
                 IllegalArgumentException.class,
-                () -> IndexNameExpressionResolver.WildcardExpressionResolver.resolve(
-                    skipAliasesStrictContext,
-                    Collections.singletonList("foo_alias")
-                )
+                () -> IndexNameExpressionResolver.resolveExpressions(skipAliasesStrictContext, "foo_alias")
+            );
+            assertEquals(
+                "The provided expression [foo_alias] matches an alias, specify the corresponding concrete indices instead.",
+                iae.getMessage()
+            );
+        }
+        IndicesOptions noExpandNoAliasesIndicesOptions = IndicesOptions.fromOptions(true, false, false, false, true, false, true, false);
+        IndexNameExpressionResolver.Context noExpandNoAliasesContext = new IndexNameExpressionResolver.Context(
+            state,
+            noExpandNoAliasesIndicesOptions,
+            SystemIndexAccessLevel.NONE
+        );
+        {
+            Collection<String> indices = IndexNameExpressionResolver.WildcardExpressionResolver.resolve(
+                noExpandNoAliasesContext,
+                List.of("foo_alias")
+            );
+            assertThat(indices, containsInAnyOrder("foo_alias"));
+        }
+        IndicesOptions strictNoExpandNoAliasesIndicesOptions = IndicesOptions.fromOptions(
+            false,
+            true,
+            false,
+            false,
+            true,
+            false,
+            true,
+            false
+        );
+        IndexNameExpressionResolver.Context strictNoExpandNoAliasesContext = new IndexNameExpressionResolver.Context(
+            state,
+            strictNoExpandNoAliasesIndicesOptions,
+            SystemIndexAccessLevel.NONE
+        );
+        {
+            IllegalArgumentException iae = expectThrows(
+                IllegalArgumentException.class,
+                () -> IndexNameExpressionResolver.resolveExpressions(strictNoExpandNoAliasesContext, "foo_alias")
             );
             assertEquals(
-                "The provided expression [foo_alias] matches an alias, " + "specify the corresponding concrete indices instead.",
+                "The provided expression [foo_alias] matches an alias, specify the corresponding concrete indices instead.",
                 iae.getMessage()
             );
         }
@@ -638,11 +711,11 @@ public void testMatchesConcreteIndicesWildcardAndAliases() {
         assertThat(matches, containsInAnyOrder("foo_alias"));
         IllegalArgumentException iae = expectThrows(
             IllegalArgumentException.class,
-            () -> IndexNameExpressionResolver.WildcardExpressionResolver.resolve(onlyIndicesContext, List.of("foo_alias"))
+            () -> IndexNameExpressionResolver.resolveExpressions(onlyIndicesContext, "foo_alias")
         );
         assertThat(
             iae.getMessage(),
-            containsString("The provided expression [foo_alias] matches an alias, specify the corresponding " + "concrete indices instead")
+            containsString("The provided expression [foo_alias] matches an alias, specify the corresponding concrete indices instead")
         );
     }
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeTests.java b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeTests.java
index 71cf71ecb4c7..27c7ddfc0790 100644
--- a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeTests.java
@@ -14,14 +14,11 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xcontent.XContentType;
 
 import java.net.InetAddress;
 import java.util.Collections;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -30,12 +27,10 @@
 import static org.elasticsearch.test.NodeRoles.nonRemoteClusterClientNode;
 import static org.elasticsearch.test.NodeRoles.remoteClusterClientNode;
 import static org.hamcrest.Matchers.allOf;
-import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasItem;
-import static org.hamcrest.Matchers.hasItems;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.startsWith;
@@ -167,7 +162,7 @@ public void testDiscoveryNodeDescriptionWithoutAttributes() {
         final DiscoveryNode node = new DiscoveryNode(
             "test-id",
             buildNewFakeTransportAddress(),
-            Collections.singletonMap("test-attr", "val"),
+            Map.of("test-attr", "val"),
             DiscoveryNodeRole.roles(),
             Version.CURRENT
         );
@@ -189,39 +184,58 @@ public void testDiscoveryNodeToXContent() {
             "test-hostname",
             "test-hostaddr",
             transportAddress,
-            Collections.singletonMap("test-attr", "val"),
+            Map.of("test-attr", "val"),
             DiscoveryNodeRole.roles(),
             Version.CURRENT,
             withExternalId ? "test-external-id" : null
         );
 
-        final String jsonString = Strings.toString(node, randomBoolean(), randomBoolean());
-        final Map<String, Object> topLevelMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), jsonString, randomBoolean());
-
-        assertThat(topLevelMap.toString(), topLevelMap.size(), equalTo(1));
-        assertTrue(topLevelMap.toString(), topLevelMap.containsKey("test-id"));
-
-        @SuppressWarnings("unchecked")
-        final Map<String, Object> detailsMap = (Map<String, Object>) topLevelMap.get("test-id");
+        assertThat(Strings.toString(node, true, false), equalTo(Strings.format("""
+            {
+              "test-id" : {
+                "name" : "test-name",
+                "ephemeral_id" : "test-ephemeral-id",
+                "transport_address" : "%s",
+                "external_id" : "%s",
+                "attributes" : {
+                  "test-attr" : "val"
+                },
+                "roles" : [
+                  "data",
+                  "data_cold",
+                  "data_content",
+                  "data_frozen",
+                  "data_hot",
+                  "data_warm",
+                  "index",
+                  "ingest",
+                  "master",
+                  "ml",
+                  "remote_cluster_client",
+                  "search",
+                  "transform",
+                  "voting_only"
+                ],
+                "version" : "%s"
+              }
+            }""", transportAddress, withExternalId ? "test-external-id" : "test-name", Version.CURRENT)));
+    }
 
-        assertThat(topLevelMap.toString(), detailsMap.remove("name"), equalTo("test-name"));
-        assertThat(topLevelMap.toString(), detailsMap.remove("ephemeral_id"), equalTo("test-ephemeral-id"));
-        assertThat(topLevelMap.toString(), detailsMap.remove("transport_address"), equalTo(transportAddress.toString()));
-        assertThat(
-            topLevelMap.toString(),
-            detailsMap.remove("external_id"),
-            withExternalId ? equalTo("test-external-id") : equalTo("test-name")
+    public void testDiscoveryNodeToString() {
+        var node = new DiscoveryNode(
+            "test-id",
+            buildNewFakeTransportAddress(),
+            Map.of("test-attr", "val"),
+            DiscoveryNodeRole.roles(),
+            Version.CURRENT
         );
-
-        @SuppressWarnings("unchecked")
-        final Map<String, Object> attributes = (Map<String, Object>) detailsMap.remove("attributes");
-        assertThat(topLevelMap.toString(), attributes.get("test-attr"), equalTo("val"));
-
-        @SuppressWarnings("unchecked")
-        final List<String> roles = (List<String>) detailsMap.remove("roles");
-        assertThat(roles, hasItems("master", "data_content", "remote_cluster_client"));
-
-        assertThat(detailsMap.toString(), detailsMap, anEmptyMap());
+        var toString = node.toString();
+
+        assertThat(toString, containsString("{" + node.getId() + "}"));
+        assertThat(toString, containsString("{" + node.getEphemeralId() + "}"));
+        assertThat(toString, containsString("{" + node.getAddress() + "}"));
+        assertThat(toString, containsString("{IScdfhilmrstvw}"));// roles
+        assertThat(toString, containsString("{" + node.getVersion() + "}"));
+        assertThat(toString, containsString("{test-attr=val}"));// attributes
     }
-
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java
index 7f068201ab2e..e425b0e30505 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java
@@ -31,7 +31,8 @@ public void testShardToStarted() {
             new ShardId("test", "_na_", 0),
             true,
             ExistingStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null),
+            ShardRouting.Role.DEFAULT
         );
         assertThat(shard.allocationId(), nullValue());
 
@@ -56,7 +57,8 @@ public void testSuccessfulRelocation() {
             new ShardId("test", "_na_", 0),
             true,
             ExistingStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null),
+            ShardRouting.Role.DEFAULT
         );
         shard = shard.initialize("node1", null, -1);
         shard = shard.moveToStarted(ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
@@ -84,7 +86,8 @@ public void testCancelRelocation() {
             new ShardId("test", "_na_", 0),
             true,
             ExistingStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null),
+            ShardRouting.Role.DEFAULT
         );
         shard = shard.initialize("node1", null, -1);
         shard = shard.moveToStarted(ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
@@ -109,7 +112,8 @@ public void testMoveToUnassigned() {
             new ShardId("test", "_na_", 0),
             true,
             ExistingStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null),
+            ShardRouting.Role.DEFAULT
         );
         shard = shard.initialize("node1", null, -1);
         shard = shard.moveToStarted(ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
index 2e7a60fcdb84..37e5b205ebdb 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
@@ -82,7 +83,7 @@ public void testNoDelayedUnassigned() {
             .build();
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
-            .routingTable(RoutingTable.builder().addAsNew(metadata.index("test")).build())
+            .routingTable(RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(metadata.index("test")).build())
             .build();
         clusterState = ClusterState.builder(clusterState)
             .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")).localNodeId("node1").masterNodeId("node1"))
@@ -128,7 +129,7 @@ public void testDelayedUnassignedScheduleReroute() throws Exception {
             .build();
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
-            .routingTable(RoutingTable.builder().addAsNew(metadata.index("test")).build())
+            .routingTable(RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(metadata.index("test")).build())
             .build();
         clusterState = ClusterState.builder(clusterState)
             .nodes(
@@ -243,7 +244,12 @@ public void testDelayedUnassignedScheduleRerouteAfterDelayedReroute() throws Exc
             .build();
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
-            .routingTable(RoutingTable.builder().addAsNew(metadata.index("short_delay")).addAsNew(metadata.index("long_delay")).build())
+            .routingTable(
+                RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+                    .addAsNew(metadata.index("short_delay"))
+                    .addAsNew(metadata.index("long_delay"))
+                    .build()
+            )
             .nodes(
                 DiscoveryNodes.builder()
                     .add(newNode("node0", singleton(DiscoveryNodeRole.MASTER_ROLE)))
@@ -427,7 +433,12 @@ public void testDelayedUnassignedScheduleRerouteRescheduledOnShorterDelay() {
             .build();
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
-            .routingTable(RoutingTable.builder().addAsNew(metadata.index("foo")).addAsNew(metadata.index("bar")).build())
+            .routingTable(
+                RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+                    .addAsNew(metadata.index("foo"))
+                    .addAsNew(metadata.index("bar"))
+                    .build()
+            )
             .build();
         clusterState = ClusterState.builder(clusterState)
             .nodes(
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java
index 1e5e40b16354..a1e45a72407b 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.health.ClusterStateHealth;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
@@ -61,10 +62,14 @@ public void setUp() throws Exception {
         Metadata metadata = Metadata.builder().put(createIndexMetadata(TEST_INDEX_1)).put(createIndexMetadata(TEST_INDEX_2)).build();
 
         RoutingTable routingTable = new RoutingTable.Builder().add(
-            new IndexRoutingTable.Builder(metadata.index(TEST_INDEX_1).getIndex()).initializeAsNew(metadata.index(TEST_INDEX_1)).build()
+            new IndexRoutingTable.Builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY, metadata.index(TEST_INDEX_1).getIndex())
+                .initializeAsNew(metadata.index(TEST_INDEX_1))
+                .build()
         )
             .add(
-                new IndexRoutingTable.Builder(metadata.index(TEST_INDEX_2).getIndex()).initializeAsNew(metadata.index(TEST_INDEX_2)).build()
+                new IndexRoutingTable.Builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY, metadata.index(TEST_INDEX_2).getIndex())
+                    .initializeAsNew(metadata.index(TEST_INDEX_2))
+                    .build()
             )
             .build();
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/RelocationFailureInfoTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/RelocationFailureInfoTests.java
index 5e76189bc6b2..b2aa04b306ce 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/RelocationFailureInfoTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/RelocationFailureInfoTests.java
@@ -11,8 +11,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
-
 public class RelocationFailureInfoTests extends AbstractWireSerializingTestCase<RelocationFailureInfo> {
 
     @Override
@@ -26,7 +24,7 @@ protected RelocationFailureInfo createTestInstance() {
     }
 
     @Override
-    protected RelocationFailureInfo mutateInstance(RelocationFailureInfo instance) throws IOException {
+    protected RelocationFailureInfo mutateInstance(RelocationFailureInfo instance) {
         return new RelocationFailureInfo(instance.failedRelocations() + 1);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/RoutingNodeTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/RoutingNodeTests.java
index f78c0ff3fe6b..15bd91d9a561 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/RoutingNodeTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/RoutingNodeTests.java
@@ -96,23 +96,22 @@ public void testRemove() {
     }
 
     public void testNumberOfShardsWithState() {
-        assertThat(routingNode.numberOfShardsWithState(ShardRoutingState.INITIALIZING, ShardRoutingState.STARTED), equalTo(2));
         assertThat(routingNode.numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(1));
         assertThat(routingNode.numberOfShardsWithState(ShardRoutingState.RELOCATING), equalTo(1));
         assertThat(routingNode.numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1));
     }
 
     public void testShardsWithState() {
-        assertThat(routingNode.shardsWithState(ShardRoutingState.STARTED).size(), equalTo(1));
-        assertThat(routingNode.shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(1));
-        assertThat(routingNode.shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1));
+        assertThat(routingNode.shardsWithState(ShardRoutingState.STARTED).count(), equalTo(1L));
+        assertThat(routingNode.shardsWithState(ShardRoutingState.RELOCATING).count(), equalTo(1L));
+        assertThat(routingNode.shardsWithState(ShardRoutingState.INITIALIZING).count(), equalTo(1L));
     }
 
     public void testShardsWithStateInIndex() {
-        assertThat(routingNode.shardsWithState("test", ShardRoutingState.INITIALIZING, ShardRoutingState.STARTED).size(), equalTo(2));
-        assertThat(routingNode.shardsWithState("test", ShardRoutingState.STARTED).size(), equalTo(1));
-        assertThat(routingNode.shardsWithState("test", ShardRoutingState.RELOCATING).size(), equalTo(1));
-        assertThat(routingNode.shardsWithState("test", ShardRoutingState.INITIALIZING).size(), equalTo(1));
+        assertThat(routingNode.shardsWithState("test", ShardRoutingState.INITIALIZING, ShardRoutingState.STARTED).count(), equalTo(2L));
+        assertThat(routingNode.shardsWithState("test", ShardRoutingState.STARTED).count(), equalTo(1L));
+        assertThat(routingNode.shardsWithState("test", ShardRoutingState.RELOCATING).count(), equalTo(1L));
+        assertThat(routingNode.shardsWithState("test", ShardRoutingState.INITIALIZING).count(), equalTo(1L));
     }
 
     public void testNumberOfOwningShards() {
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/RoutingTableGenerator.java b/server/src/test/java/org/elasticsearch/cluster/routing/RoutingTableGenerator.java
index 43a9599d273e..2b09da7c6991 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/RoutingTableGenerator.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/RoutingTableGenerator.java
@@ -26,7 +26,7 @@ private ShardRouting genShardRouting(String index, int shardId, boolean primary)
         int stateRandomizer = RandomizedContext.current().getRandom().nextInt(40);
         if (stateRandomizer > 5) {
             state = ShardRoutingState.STARTED;
-        } else if (stateRandomizer > 3) {
+        } else if (stateRandomizer > 3 || primary) {
             state = ShardRoutingState.RELOCATING;
         } else {
             state = ShardRoutingState.INITIALIZING;
@@ -36,7 +36,7 @@ private ShardRouting genShardRouting(String index, int shardId, boolean primary)
             case STARTED -> TestShardRouting.newShardRouting(
                 index,
                 shardId,
-                "node_" + Integer.toString(node_id++),
+                "node_" + (node_id++),
                 null,
                 primary,
                 ShardRoutingState.STARTED
@@ -44,7 +44,7 @@ private ShardRouting genShardRouting(String index, int shardId, boolean primary)
             case INITIALIZING -> TestShardRouting.newShardRouting(
                 index,
                 shardId,
-                "node_" + Integer.toString(node_id++),
+                "node_" + (node_id++),
                 null,
                 primary,
                 ShardRoutingState.INITIALIZING
@@ -52,8 +52,8 @@ private ShardRouting genShardRouting(String index, int shardId, boolean primary)
             case RELOCATING -> TestShardRouting.newShardRouting(
                 index,
                 shardId,
-                "node_" + Integer.toString(node_id++),
-                "node_" + Integer.toString(node_id++),
+                "node_" + (node_id++),
+                "node_" + (node_id++),
                 primary,
                 ShardRoutingState.RELOCATING
             );
@@ -65,13 +65,13 @@ private ShardRouting genShardRouting(String index, int shardId, boolean primary)
     public IndexShardRoutingTable.Builder genShardRoutingTable(IndexMetadata indexMetadata, int shardId, ShardCounter counter) {
         final String index = indexMetadata.getIndex().getName();
         IndexShardRoutingTable.Builder builder = new IndexShardRoutingTable.Builder(new ShardId(index, "_na_", shardId));
-        ShardRouting shardRouting = genShardRouting(index, shardId, true);
-        counter.update(shardRouting);
-        builder.addShard(shardRouting);
+        final ShardRouting primary = genShardRouting(index, shardId, true);
+        counter.update(primary);
+        builder.addShard(primary);
         for (int replicas = indexMetadata.getNumberOfReplicas(); replicas > 0; replicas--) {
-            shardRouting = genShardRouting(index, shardId, false);
-            counter.update(shardRouting);
-            builder.addShard(shardRouting);
+            final ShardRouting replica = genShardRouting(index, shardId, false);
+            counter.update(replica);
+            builder.addShard(replica);
         }
 
         return builder;
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java
index d3855023e616..0fef9db2f5d2 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.MetadataIndexStateService;
@@ -74,10 +75,14 @@ public void setUp() throws Exception {
         Metadata metadata = Metadata.builder().put(createIndexMetadata(TEST_INDEX_1)).put(createIndexMetadata(TEST_INDEX_2)).build();
 
         RoutingTable testRoutingTable = new RoutingTable.Builder().add(
-            new IndexRoutingTable.Builder(metadata.index(TEST_INDEX_1).getIndex()).initializeAsNew(metadata.index(TEST_INDEX_1)).build()
+            new IndexRoutingTable.Builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY, metadata.index(TEST_INDEX_1).getIndex())
+                .initializeAsNew(metadata.index(TEST_INDEX_1))
+                .build()
         )
             .add(
-                new IndexRoutingTable.Builder(metadata.index(TEST_INDEX_2).getIndex()).initializeAsNew(metadata.index(TEST_INDEX_2)).build()
+                new IndexRoutingTable.Builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY, metadata.index(TEST_INDEX_2).getIndex())
+                    .initializeAsNew(metadata.index(TEST_INDEX_2))
+                    .build()
             )
             .build();
         this.clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
@@ -327,7 +332,7 @@ public void testAllShardsForMultipleIndices() {
     }
 
     public void testRoutingTableBuiltMoreThanOnce() {
-        RoutingTable.Builder b = RoutingTable.builder();
+        RoutingTable.Builder b = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         b.build(); // Ok the first time
         try {
             b.build();
@@ -414,14 +419,18 @@ public void testDistinctNodes() {
     public void testAddAsRecovery() {
         {
             final IndexMetadata indexMetadata = createIndexMetadata(TEST_INDEX_1).state(IndexMetadata.State.OPEN).build();
-            final RoutingTable routingTable = new RoutingTable.Builder().addAsRecovery(indexMetadata).build();
+            final RoutingTable routingTable = new RoutingTable.Builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsRecovery(
+                indexMetadata
+            ).build();
             assertThat(routingTable.hasIndex(TEST_INDEX_1), is(true));
             assertThat(routingTable.allShards(TEST_INDEX_1).size(), is(this.shardsPerIndex));
             assertThat(routingTable.index(TEST_INDEX_1).shardsWithState(UNASSIGNED).size(), is(this.shardsPerIndex));
         }
         {
             final IndexMetadata indexMetadata = createIndexMetadata(TEST_INDEX_1).state(IndexMetadata.State.CLOSE).build();
-            final RoutingTable routingTable = new RoutingTable.Builder().addAsRecovery(indexMetadata).build();
+            final RoutingTable routingTable = new RoutingTable.Builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsRecovery(
+                indexMetadata
+            ).build();
             assertThat(routingTable.hasIndex(TEST_INDEX_1), is(false));
             expectThrows(IndexNotFoundException.class, () -> routingTable.allShards(TEST_INDEX_1));
         }
@@ -436,7 +445,9 @@ public void testAddAsRecovery() {
                         .build()
                 )
                 .settingsVersion(indexMetadata.getSettingsVersion() + 1);
-            final RoutingTable routingTable = new RoutingTable.Builder().addAsRecovery(indexMetadataBuilder.build()).build();
+            final RoutingTable routingTable = new RoutingTable.Builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsRecovery(
+                indexMetadataBuilder.build()
+            ).build();
             assertThat(routingTable.hasIndex(TEST_INDEX_1), is(true));
             assertThat(routingTable.allShards(TEST_INDEX_1).size(), is(this.shardsPerIndex));
             assertThat(routingTable.index(TEST_INDEX_1).shardsWithState(UNASSIGNED).size(), is(this.shardsPerIndex));
@@ -454,7 +465,7 @@ public void testRemoveReplicasInRightOrder() {
         shards.add(TestShardRouting.newShardRouting(shardId, "node3", false, STARTED));
 
         for (int removeReplicaNumber = 0; removeReplicaNumber <= rightRemoveOrder.size(); removeReplicaNumber++) {
-            IndexRoutingTable.Builder builder = new IndexRoutingTable.Builder(index);
+            IndexRoutingTable.Builder builder = new IndexRoutingTable.Builder(ShardRoutingRoleStrategy.NO_SHARD_CREATION, index);
             Randomness.shuffle(shards);
             for (ShardRouting shard : shards) {
                 builder.addShard(shard);
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java
index 74b923c21cf7..1d61395b58d9 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java
@@ -11,16 +11,159 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.shard.ShardIdTests;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.Objects;
 
-public class ShardRoutingTests extends ESTestCase {
+import static java.util.Objects.requireNonNullElseGet;
+import static org.hamcrest.Matchers.containsString;
+
+public class ShardRoutingTests extends AbstractWireSerializingTestCase<ShardRouting> {
+
+    @Override
+    protected Writeable.Reader<ShardRouting> instanceReader() {
+        return ShardRouting::new;
+    }
+
+    @Override
+    protected ShardRouting createTestInstance() {
+        var state = randomFrom(ShardRoutingState.values());
+        var primary = randomBoolean();
+        return new ShardRouting(
+            new ShardId(randomIdentifier(), UUIDs.randomBase64UUID(), randomIntBetween(0, 99)),
+            state == ShardRoutingState.UNASSIGNED ? null : randomIdentifier(),
+            state == ShardRoutingState.UNASSIGNED || state == ShardRoutingState.STARTED ? null : randomIdentifier(),
+            primary,
+            state,
+            TestShardRouting.buildRecoveryTarget(primary, state),
+            TestShardRouting.buildUnassignedInfo(state),
+            TestShardRouting.buildRelocationFailureInfo(state),
+            TestShardRouting.buildAllocationId(state),
+            randomLongBetween(-1, 1024),
+            randomFrom(ShardRouting.Role.DEFAULT, (primary ? ShardRouting.Role.INDEX_ONLY : ShardRouting.Role.SEARCH_ONLY))
+        );
+    }
+
+    @Override
+    protected ShardRouting mutateInstance(ShardRouting instance) {
+        var mutation = randomInt(4);
+        if (mutation == 0) {
+            return mutateShardId(instance);
+        } else if (mutation == 1) {
+            return mutateState(instance);
+        } else if (mutation == 2 && instance.state() == ShardRoutingState.STARTED) {
+            return mutateCurrentNodeId(instance);
+        } else if (mutation == 3) {
+            return mutateRole(instance);
+        } else {
+            return randomValueOtherThan(instance, this::createTestInstance);
+        }
+    }
+
+    private static ShardRouting mutateShardId(ShardRouting instance) {
+        return new ShardRouting(
+            ShardIdTests.mutate(instance.shardId()),
+            instance.currentNodeId(),
+            instance.relocatingNodeId(),
+            instance.primary(),
+            instance.state(),
+            instance.recoverySource(),
+            instance.unassignedInfo(),
+            instance.relocationFailureInfo(),
+            instance.allocationId(),
+            instance.getExpectedShardSize(),
+            instance.role()
+        );
+    }
+
+    private static ShardRouting mutateState(ShardRouting instance) {
+        var newState = randomValueOtherThan(instance.state(), () -> randomFrom(ShardRoutingState.values()));
+        return new ShardRouting(
+            instance.shardId(),
+            newState == ShardRoutingState.UNASSIGNED ? null : requireNonNullElseGet(instance.currentNodeId(), ESTestCase::randomIdentifier),
+            newState == ShardRoutingState.UNASSIGNED || newState == ShardRoutingState.STARTED
+                ? null
+                : requireNonNullElseGet(instance.relocatingNodeId(), ESTestCase::randomIdentifier),
+            instance.primary(),
+            newState,
+            newState == ShardRoutingState.STARTED || newState == ShardRoutingState.RELOCATING
+                ? null
+                : requireNonNullElseGet(
+                    instance.recoverySource(),
+                    () -> TestShardRouting.buildRecoveryTarget(instance.primary(), newState)
+                ),
+            newState == ShardRoutingState.STARTED || newState == ShardRoutingState.RELOCATING
+                ? null
+                : requireNonNullElseGet(instance.unassignedInfo(), () -> TestShardRouting.buildUnassignedInfo(newState)),
+            instance.relocationFailureInfo(),
+            switch (newState) {
+            case UNASSIGNED -> null;
+            case INITIALIZING, STARTED -> requireNonNullElseGet(instance.allocationId(), AllocationId::newInitializing);
+            case RELOCATING -> AllocationId.newRelocation(requireNonNullElseGet(instance.allocationId(), AllocationId::newInitializing));
+            },
+            instance.getExpectedShardSize(),
+            instance.role()
+        );
+    }
+
+    private static ShardRouting mutateCurrentNodeId(ShardRouting instance) {
+        return new ShardRouting(
+            instance.shardId(),
+            randomValueOtherThan(instance.currentNodeId(), ESTestCase::randomIdentifier),
+            instance.relocatingNodeId(),
+            instance.primary(),
+            instance.state(),
+            instance.recoverySource(),
+            instance.unassignedInfo(),
+            instance.relocationFailureInfo(),
+            instance.allocationId(),
+            instance.getExpectedShardSize(),
+            instance.role()
+        );
+    }
+
+    private static ShardRouting mutateRole(ShardRouting instance) {
+        return new ShardRouting(
+            instance.shardId(),
+            instance.currentNodeId(),
+            instance.relocatingNodeId(),
+            instance.primary(),
+            instance.state(),
+            instance.recoverySource(),
+            instance.unassignedInfo(),
+            instance.relocationFailureInfo(),
+            instance.allocationId(),
+            instance.getExpectedShardSize(),
+            randomValueOtherThan(
+                instance.role(),
+                () -> randomFrom(
+                    ShardRouting.Role.DEFAULT,
+                    (instance.primary() ? ShardRouting.Role.INDEX_ONLY : ShardRouting.Role.SEARCH_ONLY)
+                )
+            )
+        );
+    }
+
+    private ShardRouting randomShardRouting(String index, int shard) {
+        ShardRoutingState state = randomFrom(ShardRoutingState.values());
+        return TestShardRouting.newShardRouting(
+            index,
+            shard,
+            state == ShardRoutingState.UNASSIGNED ? null : "1",
+            state == ShardRoutingState.RELOCATING ? "2" : null,
+            state != ShardRoutingState.UNASSIGNED && randomBoolean(),
+            state
+        );
+    }
 
     public void testIsSameAllocation() {
         ShardRouting unassignedShard0 = TestShardRouting.newShardRouting("test", 0, null, false, ShardRoutingState.UNASSIGNED);
@@ -46,18 +189,6 @@ public void testIsSameAllocation() {
         assertFalse(unassignedShard0.isSameAllocation(startedShard1));
     }
 
-    private ShardRouting randomShardRouting(String index, int shard) {
-        ShardRoutingState state = randomFrom(ShardRoutingState.values());
-        return TestShardRouting.newShardRouting(
-            index,
-            shard,
-            state == ShardRoutingState.UNASSIGNED ? null : "1",
-            state == ShardRoutingState.RELOCATING ? "2" : null,
-            state != ShardRoutingState.UNASSIGNED && randomBoolean(),
-            state
-        );
-    }
-
     public void testIsSourceTargetRelocation() {
         ShardRouting unassignedShard0 = TestShardRouting.newShardRouting("test", 0, null, false, ShardRoutingState.UNASSIGNED);
         ShardRouting initializingShard0 = TestShardRouting.newShardRouting(
@@ -138,7 +269,8 @@ public void testEqualsIgnoringVersion() {
                         otherRouting.unassignedInfo(),
                         otherRouting.relocationFailureInfo(),
                         otherRouting.allocationId(),
-                        otherRouting.getExpectedShardSize()
+                        otherRouting.getExpectedShardSize(),
+                        otherRouting.role()
                     );
                     break;
                 case 1:
@@ -153,7 +285,8 @@ public void testEqualsIgnoringVersion() {
                         otherRouting.unassignedInfo(),
                         otherRouting.relocationFailureInfo(),
                         otherRouting.allocationId(),
-                        otherRouting.getExpectedShardSize()
+                        otherRouting.getExpectedShardSize(),
+                        otherRouting.role()
                     );
                     break;
                 case 2:
@@ -171,7 +304,8 @@ public void testEqualsIgnoringVersion() {
                             otherRouting.unassignedInfo(),
                             otherRouting.relocationFailureInfo(),
                             otherRouting.allocationId(),
-                            otherRouting.getExpectedShardSize()
+                            otherRouting.getExpectedShardSize(),
+                            otherRouting.role()
                         );
                     }
                     break;
@@ -190,7 +324,8 @@ public void testEqualsIgnoringVersion() {
                             otherRouting.unassignedInfo(),
                             otherRouting.relocationFailureInfo(),
                             otherRouting.allocationId(),
-                            otherRouting.getExpectedShardSize()
+                            otherRouting.getExpectedShardSize(),
+                            otherRouting.role()
                         );
                     }
                     break;
@@ -214,7 +349,8 @@ public void testEqualsIgnoringVersion() {
                             otherRouting.unassignedInfo(),
                             otherRouting.relocationFailureInfo(),
                             otherRouting.allocationId(),
-                            otherRouting.getExpectedShardSize()
+                            otherRouting.getExpectedShardSize(),
+                            otherRouting.role()
                         );
                     }
                     break;
@@ -232,32 +368,25 @@ public void testEqualsIgnoringVersion() {
                     break;
                 case 6:
                     // change state
-                    ShardRoutingState newState;
-                    do {
-                        newState = randomFrom(ShardRoutingState.values());
-                    } while (newState == otherRouting.state());
-
-                    UnassignedInfo unassignedInfo = otherRouting.unassignedInfo();
-                    if (unassignedInfo == null
-                        && (newState == ShardRoutingState.UNASSIGNED || newState == ShardRoutingState.INITIALIZING)) {
-                        unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "test");
-                    }
-
+                    ShardRoutingState newState = randomValueOtherThan(otherRouting.state(), () -> randomFrom(ShardRoutingState.values()));
                     otherRouting = TestShardRouting.newShardRouting(
                         otherRouting.getIndexName(),
                         otherRouting.id(),
-                        newState == ShardRoutingState.UNASSIGNED
-                            ? null
-                            : (otherRouting.currentNodeId() == null ? "1" : otherRouting.currentNodeId()),
+                        newState == ShardRoutingState.UNASSIGNED ? null : Objects.requireNonNullElse(otherRouting.currentNodeId(), "1"),
                         newState == ShardRoutingState.RELOCATING ? "2" : null,
                         otherRouting.primary(),
                         newState,
-                        unassignedInfo
+                        newState == ShardRoutingState.UNASSIGNED || newState == ShardRoutingState.INITIALIZING
+                            ? Objects.requireNonNullElse(
+                                otherRouting.unassignedInfo(),
+                                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "test")
+                            )
+                            : null
                     );
                     break;
             }
 
-            if (randomBoolean()) {
+            if (randomBoolean() && otherRouting.state() == ShardRoutingState.UNASSIGNED) {
                 // change unassigned info
                 otherRouting = TestShardRouting.newShardRouting(
                     otherRouting.getIndexName(),
@@ -315,4 +444,20 @@ public void testExpectedSize() throws IOException {
             }
         }
     }
+
+    public void testSummaryContainsImportantFields() {
+        var shard = createTestInstance();
+
+        assertThat("index name", shard.shortSummary(), containsString('[' + shard.getIndexName() + ']'));
+        assertThat("shard id", shard.shortSummary(), containsString(shard.shardId().toString()));
+        assertThat("primary/replica", shard.shortSummary(), containsString(shard.primary() ? "[P]" : "[R]"));
+        assertThat("current node id", shard.shortSummary(), containsString("node[" + shard.currentNodeId() + ']'));
+        if (shard.relocating()) {
+            assertThat("relocating node id", shard.shortSummary(), containsString("relocating [" + shard.relocatingNodeId() + ']'));
+        }
+        assertThat("state", shard.shortSummary(), containsString("s[" + shard.state() + "]"));
+        if (shard.role() != ShardRouting.Role.DEFAULT) {
+            assertThat("role", shard.shortSummary(), containsString("[" + shard.role() + "]"));
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
index 186226a106fe..90cd855fef53 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
@@ -77,7 +78,8 @@ public void testReasonOrdinalOrder() {
             UnassignedInfo.Reason.FORCED_EMPTY_PRIMARY,
             UnassignedInfo.Reason.MANUAL_ALLOCATION,
             UnassignedInfo.Reason.INDEX_CLOSED,
-            UnassignedInfo.Reason.NODE_RESTARTING };
+            UnassignedInfo.Reason.NODE_RESTARTING,
+            UnassignedInfo.Reason.UNPROMOTABLE_REPLICA };
         for (int i = 0; i < order.length; i++) {
             assertThat(order[i].ordinal(), equalTo(i));
         }
@@ -151,7 +153,7 @@ public void testIndexCreated() {
             .build();
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
-            .routingTable(RoutingTable.builder().addAsNew(metadata.index("test")).build())
+            .routingTable(RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(metadata.index("test")).build())
             .build();
         for (ShardRouting shard : shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED)) {
             assertThat(shard.unassignedInfo().getReason(), equalTo(UnassignedInfo.Reason.INDEX_CREATED));
@@ -169,7 +171,9 @@ public void testClusterRecovered() {
             .build();
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
-            .routingTable(RoutingTable.builder().addAsRecovery(metadata.index("test")).build())
+            .routingTable(
+                RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsRecovery(metadata.index("test")).build()
+            )
             .build();
         for (ShardRouting shard : shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED)) {
             assertThat(shard.unassignedInfo().getReason(), equalTo(UnassignedInfo.Reason.CLUSTER_RECOVERED));
@@ -199,7 +203,9 @@ public void testIndexClosedAndReopened() {
                         .add(newNode("node-5"))
                 )
                 .metadata(metadata0)
-                .routingTable(RoutingTable.builder().addAsNew(metadata0.index("test")).build())
+                .routingTable(
+                    RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(metadata0.index("test")).build()
+                )
                 .build(),
             allocationService
         );
@@ -226,7 +232,10 @@ public void testIndexClosedAndReopened() {
             .build();
         final var clusterState2 = ClusterState.builder(clusterState1)
             .metadata(metadata1)
-            .routingTable(RoutingTable.builder(clusterState1.routingTable()).addAsFromOpenToClose(metadata1.index("test")))
+            .routingTable(
+                RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY, clusterState1.routingTable())
+                    .addAsFromOpenToClose(metadata1.index("test"))
+            )
             .build();
 
         assertLastAllocatedNodeIdsAssigned(
@@ -245,7 +254,10 @@ public void testIndexClosedAndReopened() {
             .build();
         final var clusterState4 = ClusterState.builder(clusterState3)
             .metadata(metadata4)
-            .routingTable(RoutingTable.builder(clusterState3.routingTable()).addAsFromCloseToOpen(metadata4.index("test")))
+            .routingTable(
+                RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY, clusterState3.routingTable())
+                    .addAsFromCloseToOpen(metadata4.index("test"))
+            )
             .build();
 
         assertLastAllocatedNodeIdsAssigned(
@@ -313,7 +325,9 @@ public void testIndexReopened() {
             .build();
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
-            .routingTable(RoutingTable.builder().addAsFromCloseToOpen(metadata.index("test")).build())
+            .routingTable(
+                RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsFromCloseToOpen(metadata.index("test")).build()
+            )
             .build();
         for (ShardRouting shard : shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED)) {
             assertThat(shard.unassignedInfo().getReason(), equalTo(UnassignedInfo.Reason.INDEX_REOPENED));
@@ -332,7 +346,7 @@ public void testNewIndexRestored() {
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
             .routingTable(
-                RoutingTable.builder()
+                RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
                     .addAsNewRestore(
                         metadata.index("test"),
                         new SnapshotRecoverySource(
@@ -374,7 +388,9 @@ public void testExistingIndexRestored() {
                         .add(newNode("node-5"))
                 )
                 .metadata(metadata0)
-                .routingTable(RoutingTable.builder().addAsNew(metadata0.index("test")).build())
+                .routingTable(
+                    RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(metadata0.index("test")).build()
+                )
                 .build(),
             allocationService
         );
@@ -386,7 +402,10 @@ public void testExistingIndexRestored() {
             .build();
         final var clusterState1 = ClusterState.builder(clusterState0)
             .metadata(metadata1)
-            .routingTable(RoutingTable.builder(clusterState0.routingTable()).addAsFromOpenToClose(metadata1.index("test")))
+            .routingTable(
+                RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY, clusterState0.routingTable())
+                    .addAsFromOpenToClose(metadata1.index("test"))
+            )
             .build();
 
         assertLastAllocatedNodeIdsAssigned(
@@ -411,7 +430,7 @@ public void testExistingIndexRestored() {
         final var clusterState3 = ClusterState.builder(clusterState2)
             .metadata(metadata3)
             .routingTable(
-                RoutingTable.builder(clusterState2.routingTable())
+                RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY, clusterState2.routingTable())
                     .addAsRestore(
                         metadata3.index("test"),
                         new SnapshotRecoverySource(
@@ -442,7 +461,9 @@ public void testDanglingIndexImported() {
             .build();
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
-            .routingTable(RoutingTable.builder().addAsFromDangling(metadata.index("test")).build())
+            .routingTable(
+                RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsFromDangling(metadata.index("test")).build()
+            )
             .build();
         for (ShardRouting shard : shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED)) {
             assertThat(shard.unassignedInfo().getReason(), equalTo(UnassignedInfo.Reason.DANGLING_INDEX_IMPORTED));
@@ -457,7 +478,7 @@ public void testReplicaAdded() {
         final Index index = metadata.index("test").getIndex();
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
-            .routingTable(RoutingTable.builder().addAsNew(metadata.index(index)).build())
+            .routingTable(RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(metadata.index(index)).build())
             .build();
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
         clusterState = allocation.reroute(clusterState, "reroute", ActionListener.noop());
@@ -468,7 +489,7 @@ public void testReplicaAdded() {
         for (int i = 0; i < indexRoutingTable.size(); i++) {
             builder.addIndexShard(new IndexShardRoutingTable.Builder(indexRoutingTable.shard(i)));
         }
-        builder.addReplica();
+        builder.addReplica(ShardRouting.Role.DEFAULT);
         clusterState = ClusterState.builder(clusterState)
             .routingTable(RoutingTable.builder(clusterState.routingTable()).add(builder).build())
             .build();
@@ -512,7 +533,7 @@ public void testNodeLeave() {
             .build();
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
-            .routingTable(RoutingTable.builder().addAsNew(metadata.index("test")).build())
+            .routingTable(RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(metadata.index("test")).build())
             .build();
         clusterState = ClusterState.builder(clusterState)
             .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")))
@@ -550,7 +571,7 @@ public void testFailedShard() {
             .build();
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
-            .routingTable(RoutingTable.builder().addAsNew(metadata.index("test")).build())
+            .routingTable(RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(metadata.index("test")).build())
             .build();
         clusterState = ClusterState.builder(clusterState)
             .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")))
@@ -761,7 +782,12 @@ public void testNumberOfDelayedUnassigned() throws Exception {
             .build();
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
-            .routingTable(RoutingTable.builder().addAsNew(metadata.index("test1")).addAsNew(metadata.index("test2")).build())
+            .routingTable(
+                RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+                    .addAsNew(metadata.index("test1"))
+                    .addAsNew(metadata.index("test2"))
+                    .build()
+            )
             .build();
         clusterState = ClusterState.builder(clusterState)
             .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")))
@@ -802,7 +828,12 @@ public void testFindNextDelayedAllocation() {
             .build();
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
-            .routingTable(RoutingTable.builder().addAsNew(metadata.index("test1")).addAsNew(metadata.index("test2")).build())
+            .routingTable(
+                RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+                    .addAsNew(metadata.index("test1"))
+                    .addAsNew(metadata.index("test2"))
+                    .build()
+            )
             .build();
         clusterState = ClusterState.builder(clusterState)
             .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")))
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java
index f2186b83564e..4e11d3b21be5 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -25,7 +26,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.hamcrest.Matcher;
-import org.hamcrest.Matchers;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -33,6 +33,8 @@
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 
 public class AddIncrementallyTests extends ESAllocationTestCase {
@@ -47,18 +49,18 @@ public void testAddNodesAndIndices() {
         AllocationService service = createAllocationService(settings.build());
 
         ClusterState clusterState = initCluster(service, 1, 3, 3, 1);
-        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(STARTED).size(), equalTo(9));
+        assertThat(clusterState.getRoutingNodes().node("node0").numberOfShardsWithState(STARTED), equalTo(9));
         assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(9));
         int nodeOffset = 1;
         clusterState = addNodes(clusterState, service, 1, nodeOffset++);
-        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(STARTED).size(), equalTo(9));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(9));
+        assertThat(clusterState.getRoutingNodes().node("node0").numberOfShardsWithState(STARTED), equalTo(9));
+        assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(9));
         assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(0));
         assertNumIndexShardsPerNode(clusterState, equalTo(3));
         clusterState = addNodes(clusterState, service, 1, nodeOffset++);
         assertNumIndexShardsPerNode(clusterState, equalTo(2));
         clusterState = addNodes(clusterState, service, 1, nodeOffset++);
-        assertNumIndexShardsPerNode(clusterState, Matchers.lessThanOrEqualTo(2));
+        assertNumIndexShardsPerNode(clusterState, lessThanOrEqualTo(2));
         assertAtLeastOneIndexShardPerNode(clusterState);
         clusterState = removeNodes(clusterState, service, 1);
         assertNumIndexShardsPerNode(clusterState, equalTo(2));
@@ -66,20 +68,20 @@ public void testAddNodesAndIndices() {
         clusterState = addIndex(clusterState, service, 3, 2, 3);
         assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(2));
         assertNumIndexShardsPerNode(clusterState, "test3", equalTo(2));
-        assertNumIndexShardsPerNode(clusterState, Matchers.lessThanOrEqualTo(2));
+        assertNumIndexShardsPerNode(clusterState, lessThanOrEqualTo(2));
 
         clusterState = addIndex(clusterState, service, 4, 2, 3);
         assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(4));
         assertNumIndexShardsPerNode(clusterState, "test4", equalTo(2));
-        assertNumIndexShardsPerNode(clusterState, Matchers.lessThanOrEqualTo(2));
+        assertNumIndexShardsPerNode(clusterState, lessThanOrEqualTo(2));
         clusterState = addNodes(clusterState, service, 1, nodeOffset++);
-        assertNumIndexShardsPerNode(clusterState, Matchers.lessThanOrEqualTo(2));
+        assertNumIndexShardsPerNode(clusterState, lessThanOrEqualTo(2));
         assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(0));
         clusterState = removeNodes(clusterState, service, 1);
         assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(4));
-        assertNumIndexShardsPerNode(clusterState, Matchers.lessThanOrEqualTo(2));
+        assertNumIndexShardsPerNode(clusterState, lessThanOrEqualTo(2));
         clusterState = addNodes(clusterState, service, 1, nodeOffset++);
-        assertNumIndexShardsPerNode(clusterState, Matchers.lessThanOrEqualTo(2));
+        assertNumIndexShardsPerNode(clusterState, lessThanOrEqualTo(2));
         assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(0));
         logger.debug("ClusterState: {}", clusterState.getRoutingNodes());
     }
@@ -93,12 +95,12 @@ public void testMinimalRelocations() {
         AllocationService service = createAllocationService(settings.build());
 
         ClusterState clusterState = initCluster(service, 1, 3, 3, 1);
-        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(STARTED).size(), equalTo(9));
+        assertThat(clusterState.getRoutingNodes().node("node0").numberOfShardsWithState(STARTED), equalTo(9));
         assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(9));
         int nodeOffset = 1;
         clusterState = addNodes(clusterState, service, 1, nodeOffset++);
-        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(STARTED).size(), equalTo(9));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(9));
+        assertThat(clusterState.getRoutingNodes().node("node0").numberOfShardsWithState(STARTED), equalTo(9));
+        assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(9));
         assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(0));
         assertNumIndexShardsPerNode(clusterState, equalTo(3));
 
@@ -110,36 +112,36 @@ public void testMinimalRelocations() {
         clusterState = service.reroute(clusterState, "reroute", ActionListener.noop());
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
 
-        assertThat(routingNodes.node("node2").shardsWithState(INITIALIZING).size(), equalTo(2));
-        assertThat(routingNodes.node("node0").shardsWithState(INITIALIZING).size(), equalTo(0));
-        assertThat(routingNodes.node("node1").shardsWithState(INITIALIZING).size(), equalTo(0));
+        assertThat(routingNodes.node("node2").numberOfShardsWithState(INITIALIZING), equalTo(2));
+        assertThat(routingNodes.node("node0").numberOfShardsWithState(INITIALIZING), equalTo(0));
+        assertThat(routingNodes.node("node1").numberOfShardsWithState(INITIALIZING), equalTo(0));
 
         ClusterState newState = startInitializingShardsAndReroute(service, clusterState);
         assertThat(newState, not(equalTo(clusterState)));
         clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
-        assertThat(routingNodes.node("node2").shardsWithState(STARTED).size(), equalTo(2));
-        assertThat(routingNodes.node("node2").shardsWithState(INITIALIZING).size(), equalTo(2));
-        assertThat(routingNodes.node("node0").shardsWithState(INITIALIZING).size(), equalTo(0));
-        assertThat(routingNodes.node("node1").shardsWithState(INITIALIZING).size(), equalTo(0));
+        assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), equalTo(2));
+        assertThat(routingNodes.node("node2").numberOfShardsWithState(INITIALIZING), equalTo(2));
+        assertThat(routingNodes.node("node0").numberOfShardsWithState(INITIALIZING), equalTo(0));
+        assertThat(routingNodes.node("node1").numberOfShardsWithState(INITIALIZING), equalTo(0));
 
         newState = startInitializingShardsAndReroute(service, clusterState);
         assertThat(newState, not(equalTo(clusterState)));
         clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
-        assertThat(routingNodes.node("node2").shardsWithState(STARTED).size(), equalTo(4));
-        assertThat(routingNodes.node("node2").shardsWithState(INITIALIZING).size(), equalTo(2));
-        assertThat(routingNodes.node("node0").shardsWithState(INITIALIZING).size(), equalTo(0));
-        assertThat(routingNodes.node("node1").shardsWithState(INITIALIZING).size(), equalTo(0));
+        assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), equalTo(4));
+        assertThat(routingNodes.node("node2").numberOfShardsWithState(INITIALIZING), equalTo(2));
+        assertThat(routingNodes.node("node0").numberOfShardsWithState(INITIALIZING), equalTo(0));
+        assertThat(routingNodes.node("node1").numberOfShardsWithState(INITIALIZING), equalTo(0));
 
         newState = startInitializingShardsAndReroute(service, clusterState);
         assertThat(newState, not(equalTo(clusterState)));
         clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
-        assertThat(routingNodes.node("node2").shardsWithState(STARTED).size(), equalTo(6));
-        assertThat(routingNodes.node("node2").shardsWithState(INITIALIZING).size(), equalTo(0));
-        assertThat(routingNodes.node("node0").shardsWithState(INITIALIZING).size(), equalTo(0));
-        assertThat(routingNodes.node("node1").shardsWithState(INITIALIZING).size(), equalTo(0));
+        assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), equalTo(6));
+        assertThat(routingNodes.node("node2").numberOfShardsWithState(INITIALIZING), equalTo(0));
+        assertThat(routingNodes.node("node0").numberOfShardsWithState(INITIALIZING), equalTo(0));
+        assertThat(routingNodes.node("node1").numberOfShardsWithState(INITIALIZING), equalTo(0));
 
         newState = startInitializingShardsAndReroute(service, clusterState);
         assertThat(newState, equalTo(clusterState));
@@ -158,12 +160,12 @@ public void testMinimalRelocationsNoLimit() {
         AllocationService service = createAllocationService(settings.build());
 
         ClusterState clusterState = initCluster(service, 1, 3, 3, 1);
-        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(STARTED).size(), equalTo(9));
+        assertThat(clusterState.getRoutingNodes().node("node0").numberOfShardsWithState(STARTED), equalTo(9));
         assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(9));
         int nodeOffset = 1;
         clusterState = addNodes(clusterState, service, 1, nodeOffset++);
-        assertThat(clusterState.getRoutingNodes().node("node0").shardsWithState(STARTED).size(), equalTo(9));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(9));
+        assertThat(clusterState.getRoutingNodes().node("node0").numberOfShardsWithState(STARTED), equalTo(9));
+        assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(9));
         assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(0));
         assertNumIndexShardsPerNode(clusterState, equalTo(3));
 
@@ -175,36 +177,36 @@ public void testMinimalRelocationsNoLimit() {
         clusterState = service.reroute(clusterState, "reroute", ActionListener.noop());
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
 
-        assertThat(routingNodes.node("node2").shardsWithState(INITIALIZING).size(), equalTo(2));
-        assertThat(routingNodes.node("node0").shardsWithState(INITIALIZING).size(), equalTo(0));
-        assertThat(routingNodes.node("node1").shardsWithState(INITIALIZING).size(), equalTo(0));
+        assertThat(routingNodes.node("node2").numberOfShardsWithState(INITIALIZING), equalTo(2));
+        assertThat(routingNodes.node("node0").numberOfShardsWithState(INITIALIZING), equalTo(0));
+        assertThat(routingNodes.node("node1").numberOfShardsWithState(INITIALIZING), equalTo(0));
 
         ClusterState newState = startInitializingShardsAndReroute(service, clusterState);
         assertThat(newState, not(equalTo(clusterState)));
         clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
-        assertThat(routingNodes.node("node2").shardsWithState(STARTED).size(), equalTo(2));
-        assertThat(routingNodes.node("node2").shardsWithState(INITIALIZING).size(), equalTo(2));
-        assertThat(routingNodes.node("node0").shardsWithState(INITIALIZING).size(), equalTo(0));
-        assertThat(routingNodes.node("node1").shardsWithState(INITIALIZING).size(), equalTo(0));
+        assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), equalTo(2));
+        assertThat(routingNodes.node("node2").numberOfShardsWithState(INITIALIZING), equalTo(2));
+        assertThat(routingNodes.node("node0").numberOfShardsWithState(INITIALIZING), equalTo(0));
+        assertThat(routingNodes.node("node1").numberOfShardsWithState(INITIALIZING), equalTo(0));
 
         newState = startInitializingShardsAndReroute(service, clusterState);
         assertThat(newState, not(equalTo(clusterState)));
         clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
-        assertThat(routingNodes.node("node2").shardsWithState(STARTED).size(), equalTo(4));
-        assertThat(routingNodes.node("node2").shardsWithState(INITIALIZING).size(), equalTo(2));
-        assertThat(routingNodes.node("node0").shardsWithState(INITIALIZING).size(), equalTo(0));
-        assertThat(routingNodes.node("node1").shardsWithState(INITIALIZING).size(), equalTo(0));
+        assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), equalTo(4));
+        assertThat(routingNodes.node("node2").numberOfShardsWithState(INITIALIZING), equalTo(2));
+        assertThat(routingNodes.node("node0").numberOfShardsWithState(INITIALIZING), equalTo(0));
+        assertThat(routingNodes.node("node1").numberOfShardsWithState(INITIALIZING), equalTo(0));
 
         newState = startInitializingShardsAndReroute(service, clusterState);
         assertThat(newState, not(equalTo(clusterState)));
         clusterState = newState;
         routingNodes = clusterState.getRoutingNodes();
-        assertThat(routingNodes.node("node2").shardsWithState(STARTED).size(), equalTo(6));
-        assertThat(routingNodes.node("node2").shardsWithState(INITIALIZING).size(), equalTo(0));
-        assertThat(routingNodes.node("node0").shardsWithState(INITIALIZING).size(), equalTo(0));
-        assertThat(routingNodes.node("node1").shardsWithState(INITIALIZING).size(), equalTo(0));
+        assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), equalTo(6));
+        assertThat(routingNodes.node("node2").numberOfShardsWithState(INITIALIZING), equalTo(0));
+        assertThat(routingNodes.node("node0").numberOfShardsWithState(INITIALIZING), equalTo(0));
+        assertThat(routingNodes.node("node1").numberOfShardsWithState(INITIALIZING), equalTo(0));
 
         newState = startInitializingShardsAndReroute(service, clusterState);
         assertThat(newState, equalTo(clusterState));
@@ -220,7 +222,7 @@ private void assertNumIndexShardsPerNode(ClusterState state, Matcher<Integer> ma
 
     private void assertNumIndexShardsPerNode(ClusterState state, String index, Matcher<Integer> matcher) {
         for (RoutingNode node : state.getRoutingNodes()) {
-            assertThat(node.shardsWithState(index, STARTED).size(), matcher);
+            assertThat(Math.toIntExact(node.shardsWithState(index, STARTED).count()), matcher);
         }
     }
 
@@ -228,7 +230,7 @@ private void assertAtLeastOneIndexShardPerNode(ClusterState state) {
         for (String index : state.routingTable().indicesRouting().keySet()) {
 
             for (RoutingNode node : state.getRoutingNodes()) {
-                assertThat(node.shardsWithState(index, STARTED).size(), Matchers.greaterThanOrEqualTo(1));
+                assertThat(node.shardsWithState(index, STARTED).count(), greaterThanOrEqualTo(1L));
             }
         }
 
@@ -257,7 +259,7 @@ private ClusterState initCluster(
         int numberOfReplicas
     ) {
         Metadata.Builder metadataBuilder = Metadata.builder();
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
 
         for (int i = 0; i < numberOfIndices; i++) {
             IndexMetadata.Builder index = IndexMetadata.builder("test" + i)
@@ -303,7 +305,10 @@ private ClusterState addIndex(
         int numberOfReplicas
     ) {
         Metadata.Builder metadataBuilder = Metadata.builder(clusterState.getMetadata());
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(clusterState.routingTable());
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
+            clusterState.routingTable()
+        );
 
         IndexMetadata.Builder index = IndexMetadata.builder("test" + indexOrdinal)
             .settings(settings(Version.CURRENT))
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecisionTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecisionTests.java
index 31814d9c1f42..064411b8be59 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecisionTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecisionTests.java
@@ -13,13 +13,13 @@
 import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.Locale;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
@@ -58,10 +58,7 @@ public void testNoDecision() {
         if (allocationStatus == AllocationStatus.FETCHING_SHARD_DATA) {
             assertEquals(Explanations.Allocation.AWAITING_INFO, noDecision.getExplanation());
         } else if (allocationStatus == AllocationStatus.DELAYED_ALLOCATION) {
-            assertThat(
-                noDecision.getExplanation(),
-                equalTo(String.format(Locale.ROOT, Explanations.Allocation.DELAYED_WITHOUT_ALTERNATIVE, "0s"))
-            );
+            assertThat(noDecision.getExplanation(), equalTo(Strings.format(Explanations.Allocation.DELAYED_WITHOUT_ALTERNATIVE, "0s")));
         } else {
             assertThat(noDecision.getExplanation(), equalTo(Explanations.Allocation.NO_COPIES));
         }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
index 80fe61f1abf3..5785e040c616 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
@@ -13,10 +13,10 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterInfo;
-import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -58,8 +58,6 @@
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singleton;
-import static org.elasticsearch.cluster.ClusterModule.BALANCED_ALLOCATOR;
-import static org.elasticsearch.cluster.ClusterModule.DESIRED_BALANCE_ALLOCATOR;
 import static org.elasticsearch.cluster.routing.RoutingNodesHelper.shardsWithState;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
@@ -76,17 +74,16 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
 
     public void testMoveShardCommand() {
         AllocationService allocation = createAllocationService(
-            Settings.builder()
-                .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
-                .put(ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING.getKey(), randomShardsAllocator())
-                .build()
+            Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).build()
         );
 
         logger.info("creating an index with 1 shard, no replica");
         Metadata metadata = Metadata.builder()
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
             .build();
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
         ).metadata(metadata).routingTable(routingTable).build();
@@ -141,7 +138,6 @@ public void testAllocateCommand() {
             Settings.builder()
                 .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "none")
                 .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none")
-                .put(ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING.getKey(), randomShardsAllocator())
                 .build()
         );
         final String index = "test";
@@ -159,7 +155,9 @@ public void testAllocateCommand() {
             .build();
         // shard routing is added as "from recovery" instead of "new index creation" so that we can test below that allocating an empty
         // primary with accept_data_loss flag set to false fails
-        RoutingTable routingTable = RoutingTable.builder().addAsRecovery(metadata.index(index)).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsRecovery(metadata.index(index))
+            .build();
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
         ).metadata(metadata).routingTable(routingTable).build();
@@ -294,13 +292,13 @@ public void testAllocateCommand() {
         assertThat(newState, not(equalTo(clusterState)));
         clusterState = newState;
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(INITIALIZING).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(INITIALIZING), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
 
         logger.info("--> start the primary shard");
         clusterState = startInitializingShardsAndReroute(allocation, clusterState);
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
 
         logger.info("--> allocate the replica shard on the primary shard node, should fail");
@@ -328,16 +326,16 @@ public void testAllocateCommand() {
         assertThat(newState, not(equalTo(clusterState)));
         clusterState = newState;
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(INITIALIZING), equalTo(1));
 
         logger.info("--> start the replica shard");
         clusterState = startInitializingShardsAndReroute(allocation, clusterState);
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(STARTED), equalTo(1));
 
         logger.info("--> verify that we fail when there are no unassigned shards");
         try {
@@ -358,7 +356,6 @@ public void testAllocateStalePrimaryCommand() {
             Settings.builder()
                 .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "none")
                 .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none")
-                .put(ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING.getKey(), randomShardsAllocator())
                 .build()
         );
         final String index = "test";
@@ -376,7 +373,9 @@ public void testAllocateStalePrimaryCommand() {
             .build();
         // shard routing is added as "from recovery" instead of "new index creation" so that we can test below that allocating an empty
         // primary with accept_data_loss flag set to false fails
-        RoutingTable routingTable = RoutingTable.builder().addAsRecovery(metadata.index(index)).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsRecovery(metadata.index(index))
+            .build();
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
             .routingTable(routingTable)
@@ -402,14 +401,14 @@ public void testAllocateStalePrimaryCommand() {
         ).clusterState();
         RoutingNode routingNode1 = clusterState.getRoutingNodes().node(node1);
         assertThat(routingNode1.size(), equalTo(1));
-        assertThat(routingNode1.shardsWithState(INITIALIZING).size(), equalTo(1));
+        assertThat(routingNode1.numberOfShardsWithState(INITIALIZING), equalTo(1));
         Set<String> inSyncAllocationIds = clusterState.metadata().index(index).inSyncAllocationIds(0);
         assertThat(inSyncAllocationIds, equalTo(Collections.singleton(RecoverySource.ExistingStoreRecoverySource.FORCED_ALLOCATION_ID)));
 
         clusterState = startInitializingShardsAndReroute(allocation, clusterState);
         routingNode1 = clusterState.getRoutingNodes().node(node1);
         assertThat(routingNode1.size(), equalTo(1));
-        assertThat(routingNode1.shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(routingNode1.numberOfShardsWithState(STARTED), equalTo(1));
         inSyncAllocationIds = clusterState.metadata().index(index).inSyncAllocationIds(0);
         assertThat(inSyncAllocationIds, hasSize(1));
         assertThat(inSyncAllocationIds, not(Collections.singleton(RecoverySource.ExistingStoreRecoverySource.FORCED_ALLOCATION_ID)));
@@ -420,7 +419,6 @@ public void testCancelCommand() {
             Settings.builder()
                 .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "none")
                 .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none")
-                .put(ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING.getKey(), randomShardsAllocator())
                 .build()
         );
 
@@ -428,7 +426,9 @@ public void testCancelCommand() {
         Metadata metadata = Metadata.builder()
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
         ).metadata(metadata).routingTable(routingTable).build();
@@ -452,7 +452,7 @@ public void testCancelCommand() {
         assertThat(newState, not(equalTo(clusterState)));
         clusterState = newState;
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(INITIALIZING).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(INITIALIZING), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
 
         logger.info("--> cancel primary allocation, make sure it fails...");
@@ -471,7 +471,7 @@ public void testCancelCommand() {
         logger.info("--> start the primary shard");
         clusterState = startInitializingShardsAndReroute(allocation, clusterState);
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
 
         logger.info("--> cancel primary allocation, make sure it fails...");
@@ -499,9 +499,9 @@ public void testCancelCommand() {
         assertThat(newState, not(equalTo(clusterState)));
         clusterState = newState;
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(INITIALIZING), equalTo(1));
 
         logger.info("--> cancel the relocation allocation");
         newState = allocation.reroute(
@@ -515,7 +515,7 @@ public void testCancelCommand() {
         assertThat(newState, not(equalTo(clusterState)));
         clusterState = newState;
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
         assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(0));
 
@@ -531,9 +531,9 @@ public void testCancelCommand() {
         assertThat(newState, not(equalTo(clusterState)));
         clusterState = newState;
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(INITIALIZING), equalTo(1));
 
         logger.info("--> cancel the primary being replicated, make sure it fails");
         try {
@@ -551,9 +551,9 @@ public void testCancelCommand() {
         logger.info("--> start the replica shard");
         clusterState = startInitializingShardsAndReroute(allocation, clusterState);
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(STARTED), equalTo(1));
 
         logger.info("--> cancel allocation of the replica shard");
         newState = allocation.reroute(
@@ -567,7 +567,7 @@ public void testCancelCommand() {
         assertThat(newState, not(equalTo(clusterState)));
         clusterState = newState;
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
         assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(0));
 
@@ -583,15 +583,15 @@ public void testCancelCommand() {
         assertThat(newState, not(equalTo(clusterState)));
         clusterState = newState;
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(INITIALIZING), equalTo(1));
         logger.info("--> start the replica shard");
         clusterState = startInitializingShardsAndReroute(allocation, clusterState);
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(STARTED), equalTo(1));
 
         logger.info("--> move the replica shard");
         clusterState = allocation.reroute(
@@ -603,11 +603,11 @@ public void testCancelCommand() {
             ActionListener.noop()
         ).clusterState();
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(RELOCATING).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(RELOCATING), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState(INITIALIZING).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node3").numberOfShardsWithState(INITIALIZING), equalTo(1));
 
         if (randomBoolean()) {
             logger.info("--> cancel the primary allocation (with allow_primary set to true)");
@@ -622,7 +622,7 @@ public void testCancelCommand() {
             assertThat(newState, not(equalTo(clusterState)));
             clusterState = newState;
             assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(0));
-            assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).iterator().next().primary(), equalTo(true));
+            assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).findFirst().get().primary(), equalTo(true));
             assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(0));
         } else {
             logger.info("--> cancel the move of the replica shard");
@@ -635,9 +635,9 @@ public void testCancelCommand() {
                 ActionListener.noop()
             ).clusterState();
             assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-            assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
+            assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(1));
             assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
-            assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), equalTo(1));
+            assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(STARTED), equalTo(1));
 
             logger.info("--> move the replica shard again");
             clusterState = allocation.reroute(
@@ -649,11 +649,11 @@ public void testCancelCommand() {
                 ActionListener.noop()
             ).clusterState();
             assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-            assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
+            assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(1));
             assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
-            assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(RELOCATING).size(), equalTo(1));
+            assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(RELOCATING), equalTo(1));
             assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(1));
-            assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState(INITIALIZING).size(), equalTo(1));
+            assertThat(clusterState.getRoutingNodes().node("node3").numberOfShardsWithState(INITIALIZING), equalTo(1));
 
             logger.info("--> cancel the source replica shard");
             clusterState = allocation.reroute(
@@ -665,18 +665,21 @@ public void testCancelCommand() {
                 ActionListener.noop()
             ).clusterState();
             assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-            assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
+            assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(1));
             assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
             assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(1));
-            assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState(INITIALIZING).size(), equalTo(1));
-            assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState(INITIALIZING).get(0).relocatingNodeId(), nullValue());
+            assertThat(clusterState.getRoutingNodes().node("node3").numberOfShardsWithState(INITIALIZING), equalTo(1));
+            assertThat(
+                clusterState.getRoutingNodes().node("node3").shardsWithState(INITIALIZING).findFirst().get().relocatingNodeId(),
+                nullValue()
+            );
 
             logger.info("--> start the former target replica shard");
             clusterState = startInitializingShardsAndReroute(allocation, clusterState);
             assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-            assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
+            assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(STARTED), equalTo(1));
             assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
-            assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState(STARTED).size(), equalTo(1));
+            assertThat(clusterState.getRoutingNodes().node("node3").numberOfShardsWithState(STARTED), equalTo(1));
 
             logger.info("--> cancel the primary allocation (with allow_primary set to true)");
             newState = allocation.reroute(
@@ -689,7 +692,7 @@ public void testCancelCommand() {
             ).clusterState();
             assertThat(newState, not(equalTo(clusterState)));
             clusterState = newState;
-            assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState(STARTED).iterator().next().primary(), equalTo(true));
+            assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState(STARTED).findFirst().get().primary(), equalTo(true));
             assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(0));
             assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
         }
@@ -712,32 +715,7 @@ public void testSerialization() throws Exception {
         in = new NamedWriteableAwareStreamInput(in, namedWriteableRegistry);
 
         // Now we can read them!
-        AllocationCommands sCommands = AllocationCommands.readFrom(in);
-
-        assertThat(sCommands.commands().size(), equalTo(5));
-        assertThat(((AllocateEmptyPrimaryAllocationCommand) (sCommands.commands().get(0))).shardId(), equalTo(1));
-        assertThat(((AllocateEmptyPrimaryAllocationCommand) (sCommands.commands().get(0))).index(), equalTo("test"));
-        assertThat(((AllocateEmptyPrimaryAllocationCommand) (sCommands.commands().get(0))).node(), equalTo("node1"));
-        assertThat(((AllocateEmptyPrimaryAllocationCommand) (sCommands.commands().get(0))).acceptDataLoss(), equalTo(true));
-
-        assertThat(((AllocateStalePrimaryAllocationCommand) (sCommands.commands().get(1))).shardId(), equalTo(2));
-        assertThat(((AllocateStalePrimaryAllocationCommand) (sCommands.commands().get(1))).index(), equalTo("test"));
-        assertThat(((AllocateStalePrimaryAllocationCommand) (sCommands.commands().get(1))).node(), equalTo("node1"));
-        assertThat(((AllocateStalePrimaryAllocationCommand) (sCommands.commands().get(1))).acceptDataLoss(), equalTo(true));
-
-        assertThat(((AllocateReplicaAllocationCommand) (sCommands.commands().get(2))).shardId(), equalTo(2));
-        assertThat(((AllocateReplicaAllocationCommand) (sCommands.commands().get(2))).index(), equalTo("test"));
-        assertThat(((AllocateReplicaAllocationCommand) (sCommands.commands().get(2))).node(), equalTo("node1"));
-
-        assertThat(((MoveAllocationCommand) (sCommands.commands().get(3))).shardId(), equalTo(3));
-        assertThat(((MoveAllocationCommand) (sCommands.commands().get(3))).index(), equalTo("test"));
-        assertThat(((MoveAllocationCommand) (sCommands.commands().get(3))).fromNode(), equalTo("node2"));
-        assertThat(((MoveAllocationCommand) (sCommands.commands().get(3))).toNode(), equalTo("node3"));
-
-        assertThat(((CancelAllocationCommand) (sCommands.commands().get(4))).shardId(), equalTo(4));
-        assertThat(((CancelAllocationCommand) (sCommands.commands().get(4))).index(), equalTo("test"));
-        assertThat(((CancelAllocationCommand) (sCommands.commands().get(4))).node(), equalTo("node5"));
-        assertThat(((CancelAllocationCommand) (sCommands.commands().get(4))).allowPrimary(), equalTo(true));
+        assertThat(AllocationCommands.readFrom(in), equalTo(commands));
     }
 
     public void testXContent() throws Exception {
@@ -790,32 +768,19 @@ public void testXContent() throws Exception {
         // move two tokens, parser expected to be "on" `commands` field
         parser.nextToken();
         parser.nextToken();
-        AllocationCommands sCommands = AllocationCommands.fromXContent(parser);
-
-        assertThat(sCommands.commands().size(), equalTo(5));
-        assertThat(((AllocateEmptyPrimaryAllocationCommand) (sCommands.commands().get(0))).shardId(), equalTo(1));
-        assertThat(((AllocateEmptyPrimaryAllocationCommand) (sCommands.commands().get(0))).index(), equalTo("test"));
-        assertThat(((AllocateEmptyPrimaryAllocationCommand) (sCommands.commands().get(0))).node(), equalTo("node1"));
-        assertThat(((AllocateEmptyPrimaryAllocationCommand) (sCommands.commands().get(0))).acceptDataLoss(), equalTo(true));
-
-        assertThat(((AllocateStalePrimaryAllocationCommand) (sCommands.commands().get(1))).shardId(), equalTo(2));
-        assertThat(((AllocateStalePrimaryAllocationCommand) (sCommands.commands().get(1))).index(), equalTo("test"));
-        assertThat(((AllocateStalePrimaryAllocationCommand) (sCommands.commands().get(1))).node(), equalTo("node1"));
-        assertThat(((AllocateStalePrimaryAllocationCommand) (sCommands.commands().get(1))).acceptDataLoss(), equalTo(true));
-
-        assertThat(((AllocateReplicaAllocationCommand) (sCommands.commands().get(2))).shardId(), equalTo(2));
-        assertThat(((AllocateReplicaAllocationCommand) (sCommands.commands().get(2))).index(), equalTo("test"));
-        assertThat(((AllocateReplicaAllocationCommand) (sCommands.commands().get(2))).node(), equalTo("node1"));
-
-        assertThat(((MoveAllocationCommand) (sCommands.commands().get(3))).shardId(), equalTo(3));
-        assertThat(((MoveAllocationCommand) (sCommands.commands().get(3))).index(), equalTo("test"));
-        assertThat(((MoveAllocationCommand) (sCommands.commands().get(3))).fromNode(), equalTo("node2"));
-        assertThat(((MoveAllocationCommand) (sCommands.commands().get(3))).toNode(), equalTo("node3"));
-
-        assertThat(((CancelAllocationCommand) (sCommands.commands().get(4))).shardId(), equalTo(4));
-        assertThat(((CancelAllocationCommand) (sCommands.commands().get(4))).index(), equalTo("test"));
-        assertThat(((CancelAllocationCommand) (sCommands.commands().get(4))).node(), equalTo("node5"));
-        assertThat(((CancelAllocationCommand) (sCommands.commands().get(4))).allowPrimary(), equalTo(true));
+
+        assertThat(
+            AllocationCommands.fromXContent(parser),
+            equalTo(
+                new AllocationCommands(
+                    new AllocateEmptyPrimaryAllocationCommand("test", 1, "node1", true),
+                    new AllocateStalePrimaryAllocationCommand("test", 2, "node1", true),
+                    new AllocateReplicaAllocationCommand("test", 2, "node1"),
+                    new MoveAllocationCommand("test", 3, "node2", "node3"),
+                    new CancelAllocationCommand("test", 4, "node5", true)
+                )
+            )
+        );
     }
 
     @Override
@@ -825,17 +790,16 @@ protected NamedXContentRegistry xContentRegistry() {
 
     public void testMoveShardToNonDataNode() {
         AllocationService allocation = createAllocationService(
-            Settings.builder()
-                .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
-                .put(ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING.getKey(), randomShardsAllocator())
-                .build()
+            Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).build()
         );
 
         logger.info("creating an index with 1 shard, no replica");
         Metadata metadata = Metadata.builder()
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
             .build();
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
         ).metadata(metadata).routingTable(routingTable).build();
@@ -896,17 +860,16 @@ public void testMoveShardToNonDataNode() {
 
     public void testMoveShardFromNonDataNode() {
         AllocationService allocation = createAllocationService(
-            Settings.builder()
-                .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
-                .put(ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING.getKey(), randomShardsAllocator())
-                .build()
+            Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).build()
         );
 
         logger.info("creating an index with 1 shard, no replica");
         Metadata metadata = Metadata.builder()
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
             .build();
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
         ).metadata(metadata).routingTable(routingTable).build();
@@ -969,7 +932,6 @@ public void testConflictingCommandsInSingleRequest() {
             Settings.builder()
                 .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "none")
                 .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none")
-                .put(ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING.getKey(), randomShardsAllocator())
                 .build()
         );
 
@@ -1003,7 +965,7 @@ public void testConflictingCommandsInSingleRequest() {
                     .putInSyncAllocationIds(1, Collections.singleton("randomAllocID2"))
             )
             .build();
-        RoutingTable routingTable = RoutingTable.builder()
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsRecovery(metadata.index(index1))
             .addAsRecovery(metadata.index(index2))
             .addAsRecovery(metadata.index(index3))
@@ -1058,7 +1020,7 @@ public void testConflictingCommandsInSingleRequest() {
         clusterState = startInitializingShardsAndReroute(allocation, clusterState);
 
         final ClusterState updatedClusterState = clusterState;
-        assertThat(updatedClusterState.getRoutingNodes().node(node1).shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(updatedClusterState.getRoutingNodes().node(node1).numberOfShardsWithState(STARTED), equalTo(1));
 
         logger.info("--> subsequent replica allocation fails as all configured replicas have been allocated");
         assertThat(expectThrows(IllegalArgumentException.class, () -> {
@@ -1075,8 +1037,4 @@ public void testConflictingCommandsInSingleRequest() {
             );
         }).getMessage(), containsString("all copies of [" + index3 + "][0] are already assigned. Use the move allocation command instead"));
     }
-
-    private static String randomShardsAllocator() {
-        return randomFrom(BALANCED_ALLOCATOR, DESIRED_BALANCE_ALLOCATOR);
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationPriorityTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationPriorityTests.java
index 320ceaf603ca..002e6f121f29 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationPriorityTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationPriorityTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -65,7 +66,7 @@ public void testPrioritizedIndicesAllocatedFirst() {
                     .numberOfReplicas(1)
             )
             .build();
-        RoutingTable initialRoutingTable = RoutingTable.builder()
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("first"))
             .addAsNew(metadata.index("second"))
             .build();
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationServiceTests.java
index 4afb957ea98b..68eaf2911f8f 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationServiceTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -49,6 +50,7 @@
 import static org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING;
 import static org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING;
 import static org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING;
+import static org.elasticsearch.common.settings.ClusterSettings.createBuiltInClusterSettings;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
@@ -108,13 +110,10 @@ public void testAssignsPrimariesInPriorityOrderThenReplicas() {
             .put(CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING.getKey(), 1)
             .put(CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.getKey(), Integer.MAX_VALUE)
             .build();
-        final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        final ClusterSettings clusterSettings = createBuiltInClusterSettings(settings);
         final AllocationService allocationService = new AllocationService(
             new AllocationDeciders(
-                Arrays.asList(
-                    new SameShardAllocationDecider(settings, clusterSettings),
-                    new ThrottlingAllocationDecider(settings, clusterSettings)
-                )
+                Arrays.asList(new SameShardAllocationDecider(clusterSettings), new ThrottlingAllocationDecider(clusterSettings))
             ),
             new ShardsAllocator() {
                 @Override
@@ -130,7 +129,8 @@ public ShardAllocationDecision decideShardAllocation(ShardRouting shard, Routing
                 }
             },
             new EmptyClusterInfoService(),
-            EmptySnapshotsInfoService.INSTANCE
+            EmptySnapshotsInfoService.INSTANCE,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
 
         final String unrealisticAllocatorName = "unrealistic";
@@ -169,7 +169,7 @@ public ShardAllocationDecision decideShardAllocation(ShardRouting shard, Routing
                 )
             );
 
-        final RoutingTable.Builder routingTableBuilder = RoutingTable.builder()
+        final RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsRecovery(metadata.get("highPriority"))
             .addAsRecovery(metadata.get("mediumPriority"))
             .addAsRecovery(metadata.get("lowPriority"))
@@ -229,7 +229,13 @@ public ShardAllocationDecision decideShardAllocation(ShardRouting shard, Routing
     }
 
     public void testExplainsNonAllocationOfShardWithUnknownAllocator() {
-        final AllocationService allocationService = new AllocationService(null, null, null, null);
+        final AllocationService allocationService = new AllocationService(
+            null,
+            null,
+            null,
+            null,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
+        );
         allocationService.setExistingShardsAllocators(
             Collections.singletonMap(GatewayAllocator.ALLOCATOR_NAME, new TestGatewayAllocator())
         );
@@ -246,7 +252,8 @@ public void testExplainsNonAllocationOfShardWithUnknownAllocator() {
                 )
             );
 
-        final RoutingTable.Builder routingTableBuilder = RoutingTable.builder().addAsRecovery(metadata.get("index"));
+        final RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsRecovery(metadata.get("index"));
 
         final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
             .nodes(nodesBuilder)
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java
index 2f2129ca9e6a..5c3617bed33d 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -68,7 +69,9 @@ public void testMoveShardOnceNewNodeWithAttributeAdded1() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -141,7 +144,9 @@ public void testMoveShardOnceNewNodeWithAttributeAdded2() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -220,7 +225,9 @@ public void testMoveShardOnceNewNodeWithAttributeAdded3() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(5).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -316,7 +323,7 @@ public void testMoveShardOnceNewNodeWithAttributeAdded4() {
             .put(IndexMetadata.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(5).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder()
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("test1"))
             .addAsNew(metadata.index("test2"))
             .build();
@@ -416,7 +423,9 @@ public void testMoveShardOnceNewNodeWithAttributeAdded5() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(2))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -504,7 +513,9 @@ public void testMoveShardOnceNewNodeWithAttributeAdded6() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(3))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -595,7 +606,9 @@ public void testFullAwareness1() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -668,7 +681,9 @@ public void testFullAwareness2() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -748,7 +763,7 @@ public void testFullAwareness3() {
             .put(IndexMetadata.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(5).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder()
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("test1"))
             .addAsNew(metadata.index("test2"))
             .build();
@@ -837,7 +852,9 @@ public void testUnbalancedZones() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(5).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -899,7 +916,9 @@ public void testUnassignedShardsWithUnbalancedZones() {
             .put(IndexMetadata.builder("test").settings(indexSettings).numberOfShards(1).numberOfReplicas(4))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -930,17 +949,21 @@ public void testUnassignedShardsWithUnbalancedZones() {
         assertThat(shardsWithState(clusterState.getRoutingNodes(), INITIALIZING).size(), equalTo(3));
         assertThat(shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).size(), equalTo(1)); // Unassigned shard is expected.
 
-        // Cancel all initializing shards and move started primary to another node.
         AllocationCommands commands = new AllocationCommands();
-        String primaryNode = null;
+        // Cancel all initializing shards
         for (ShardRouting routing : clusterState.routingTable().allShards()) {
-            if (routing.primary()) {
-                primaryNode = routing.currentNodeId();
-            } else if (routing.initializing()) {
+            if (routing.initializing()) {
                 commands.add(new CancelAllocationCommand(routing.shardId().getIndexName(), routing.id(), routing.currentNodeId(), false));
             }
         }
-        commands.add(new MoveAllocationCommand("test", 0, primaryNode, "A-4"));
+        // Move started primary to another node.
+        for (ShardRouting routing : clusterState.routingTable().allShards()) {
+            if (routing.primary()) {
+                var currentNodeId = routing.currentNodeId();
+                var otherNodeId = randomValueOtherThan(currentNodeId, clusterState.nodes().getNodes().keySet().iterator()::next);
+                commands.add(new MoveAllocationCommand("test", 0, currentNodeId, otherNodeId));
+            }
+        }
 
         clusterState = strategy.reroute(clusterState, commands, false, false, false, ActionListener.noop()).clusterState();
 
@@ -966,7 +989,9 @@ public void testMultipleAwarenessAttributes() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -1018,7 +1043,9 @@ public void testDisabledByAutoExpandReplicas() {
         final ClusterState clusterState = applyStartedShardsUntilNoChange(
             ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(Settings.EMPTY))
                 .metadata(metadata)
-                .routingTable(RoutingTable.builder().addAsNew(metadata.index("test")).build())
+                .routingTable(
+                    RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(metadata.index("test")).build()
+                )
                 .nodes(
                     DiscoveryNodes.builder()
                         .add(newNode("A-0", singletonMap("zone", "a")))
@@ -1055,7 +1082,9 @@ public void testNodesWithoutAttributeAreIgnored() {
         final ClusterState clusterState = applyStartedShardsUntilNoChange(
             ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(Settings.EMPTY))
                 .metadata(metadata)
-                .routingTable(RoutingTable.builder().addAsNew(metadata.index("test")).build())
+                .routingTable(
+                    RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(metadata.index("test")).build()
+                )
                 .nodes(
                     DiscoveryNodes.builder()
                         .add(newNode("A-0", singletonMap("zone", "a")))
@@ -1120,7 +1149,9 @@ private void testExplanation(
         final ClusterState clusterState = applyStartedShardsUntilNoChange(
             ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(Settings.EMPTY))
                 .metadata(metadata)
-                .routingTable(RoutingTable.builder().addAsNew(metadata.index("test")).build())
+                .routingTable(
+                    RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(metadata.index("test")).build()
+                )
                 .nodes(
                     nodesOperator.apply(
                         DiscoveryNodes.builder()
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
index 3eb5c353585a..1649a9556740 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
@@ -10,28 +10,22 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.lucene.util.ArrayUtil;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
-import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
-import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.snapshots.EmptySnapshotsInfoService;
-import org.elasticsearch.test.gateway.TestGatewayAllocator;
 import org.hamcrest.Matchers;
 
 import java.util.stream.Collectors;
@@ -39,6 +33,8 @@
 import static org.elasticsearch.cluster.routing.RoutingNodesHelper.shardsWithState;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 public class BalanceConfigurationTests extends ESAllocationTestCase {
 
@@ -64,7 +60,7 @@ public void testIndexBalance() {
         settings.put(BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING.getKey(), replicaBalance);
         settings.put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), balanceThreshold);
 
-        AllocationService strategy = createAllocationService(settings.build(), new TestGatewayAllocator());
+        AllocationService strategy = createAllocationService(settings.build());
 
         ClusterState clusterState = initCluster(strategy);
         assertIndexBalance(
@@ -115,7 +111,7 @@ public void testReplicaBalance() {
         settings.put(BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING.getKey(), replicaBalance);
         settings.put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), balanceThreshold);
 
-        AllocationService strategy = createAllocationService(settings.build(), new TestGatewayAllocator());
+        AllocationService strategy = createAllocationService(settings.build());
 
         ClusterState clusterState = initCluster(strategy);
         assertReplicaBalance(
@@ -150,7 +146,7 @@ public void testReplicaBalance() {
 
     private ClusterState initCluster(AllocationService strategy) {
         Metadata.Builder metadataBuilder = Metadata.builder();
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
 
         for (int i = 0; i < numberOfIndices; i++) {
             IndexMetadata.Builder index = IndexMetadata.builder("test" + i)
@@ -255,8 +251,8 @@ private void assertReplicaBalance(
         final int maxAvgNumberOfShards = Math.round(Math.round(Math.ceil(avgNumShards + threshold)));
 
         for (RoutingNode node : nodes) {
-            assertThat(node.shardsWithState(STARTED).size(), Matchers.greaterThanOrEqualTo(minAvgNumberOfShards));
-            assertThat(node.shardsWithState(STARTED).size(), Matchers.lessThanOrEqualTo(maxAvgNumberOfShards));
+            assertThat(node.numberOfShardsWithState(STARTED), greaterThanOrEqualTo(minAvgNumberOfShards));
+            assertThat(node.numberOfShardsWithState(STARTED), lessThanOrEqualTo(maxAvgNumberOfShards));
         }
     }
 
@@ -277,8 +273,8 @@ private void assertIndexBalance(
 
         for (String index : routingTable.indicesRouting().keySet()) {
             for (RoutingNode node : nodes) {
-                assertThat(node.shardsWithState(index, STARTED).size(), Matchers.greaterThanOrEqualTo(minAvgNumberOfShards));
-                assertThat(node.shardsWithState(index, STARTED).size(), Matchers.lessThanOrEqualTo(maxAvgNumberOfShards));
+                assertThat(Math.toIntExact(node.shardsWithState(index, STARTED).count()), greaterThanOrEqualTo(minAvgNumberOfShards));
+                assertThat(Math.toIntExact(node.shardsWithState(index, STARTED).count()), lessThanOrEqualTo(maxAvgNumberOfShards));
             }
         }
     }
@@ -288,8 +284,8 @@ public void testPersistedSettings() {
         settings.put(BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING.getKey(), 0.2);
         settings.put(BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING.getKey(), 0.3);
         settings.put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), 2.0);
-        ClusterSettings service = new ClusterSettings(Settings.builder().build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        BalancedShardsAllocator allocator = new BalancedShardsAllocator(settings.build(), service, WriteLoadForecaster.DEFAULT);
+        ClusterSettings service = new ClusterSettings(settings.build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        BalancedShardsAllocator allocator = new BalancedShardsAllocator(service, WriteLoadForecaster.DEFAULT);
         assertThat(allocator.getIndexBalance(), Matchers.equalTo(0.2f));
         assertThat(allocator.getShardBalance(), Matchers.equalTo(0.3f));
         assertThat(allocator.getThreshold(), Matchers.equalTo(2.0f));
@@ -316,155 +312,4 @@ public void testPersistedSettings() {
         assertThat(allocator.getShardBalance(), Matchers.equalTo(0.1f));
         assertThat(allocator.getThreshold(), Matchers.equalTo(3.0f));
     }
-
-    public void testNoRebalanceOnPrimaryOverload() {
-        Settings.Builder settings = Settings.builder();
-        AllocationService strategy = new AllocationService(
-            randomAllocationDeciders(
-                settings.build(),
-                new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-                random()
-            ),
-            new TestGatewayAllocator(),
-            new ShardsAllocator() {
-                /*
-                 *  // this allocator tries to rebuild this scenario where a rebalance is
-                 *  // triggered solely by the primary overload on node [1] where a shard
-                 *  // is rebalanced to node 0
-                    routing_nodes:
-                    -----node_id[0][V]
-                    --------[test][0], node[0], [R], s[STARTED]
-                    --------[test][4], node[0], [R], s[STARTED]
-                    -----node_id[1][V]
-                    --------[test][0], node[1], [P], s[STARTED]
-                    --------[test][1], node[1], [P], s[STARTED]
-                    --------[test][3], node[1], [R], s[STARTED]
-                    -----node_id[2][V]
-                    --------[test][1], node[2], [R], s[STARTED]
-                    --------[test][2], node[2], [R], s[STARTED]
-                    --------[test][4], node[2], [P], s[STARTED]
-                    -----node_id[3][V]
-                    --------[test][2], node[3], [P], s[STARTED]
-                    --------[test][3], node[3], [P], s[STARTED]
-                    ---- unassigned
-                */
-                public void allocate(RoutingAllocation allocation) {
-                    RoutingNodes.UnassignedShards unassigned = allocation.routingNodes().unassigned();
-                    ShardRouting[] drain = unassigned.drain();
-                    ArrayUtil.timSort(drain, (a, b) -> { return a.primary() ? -1 : 1; }); // we have to allocate primaries first
-                    for (ShardRouting sr : drain) {
-                        switch (sr.id()) {
-                            case 0:
-                                if (sr.primary()) {
-                                    allocation.routingNodes().initializeShard(sr, "node1", null, -1, allocation.changes());
-                                } else {
-                                    allocation.routingNodes().initializeShard(sr, "node0", null, -1, allocation.changes());
-                                }
-                                break;
-                            case 1:
-                                if (sr.primary()) {
-                                    allocation.routingNodes().initializeShard(sr, "node1", null, -1, allocation.changes());
-                                } else {
-                                    allocation.routingNodes().initializeShard(sr, "node2", null, -1, allocation.changes());
-                                }
-                                break;
-                            case 2:
-                                if (sr.primary()) {
-                                    allocation.routingNodes().initializeShard(sr, "node3", null, -1, allocation.changes());
-                                } else {
-                                    allocation.routingNodes().initializeShard(sr, "node2", null, -1, allocation.changes());
-                                }
-                                break;
-                            case 3:
-                                if (sr.primary()) {
-                                    allocation.routingNodes().initializeShard(sr, "node3", null, -1, allocation.changes());
-                                } else {
-                                    allocation.routingNodes().initializeShard(sr, "node1", null, -1, allocation.changes());
-                                }
-                                break;
-                            case 4:
-                                if (sr.primary()) {
-                                    allocation.routingNodes().initializeShard(sr, "node2", null, -1, allocation.changes());
-                                } else {
-                                    allocation.routingNodes().initializeShard(sr, "node0", null, -1, allocation.changes());
-                                }
-                                break;
-                        }
-
-                    }
-                }
-
-                @Override
-                public ShardAllocationDecision decideShardAllocation(ShardRouting shard, RoutingAllocation allocation) {
-                    throw new UnsupportedOperationException("explain not supported");
-                }
-            },
-            EmptyClusterInfoService.INSTANCE,
-            EmptySnapshotsInfoService.INSTANCE
-        );
-        Metadata.Builder metadataBuilder = Metadata.builder();
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
-        IndexMetadata.Builder indexMeta = IndexMetadata.builder("test")
-            .settings(settings(Version.CURRENT))
-            .numberOfShards(5)
-            .numberOfReplicas(1);
-        metadataBuilder = metadataBuilder.put(indexMeta);
-        Metadata metadata = metadataBuilder.build();
-        for (IndexMetadata indexMetadata : metadata.indices().values()) {
-            routingTableBuilder.addAsNew(indexMetadata);
-        }
-        RoutingTable routingTable = routingTableBuilder.build();
-        DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
-        for (int i = 0; i < 4; i++) {
-            DiscoveryNode node = newNode("node" + i);
-            nodes.add(node);
-        }
-
-        ClusterState clusterState = ClusterState.builder(
-            org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
-        ).nodes(nodes).metadata(metadata).routingTable(routingTable).build();
-        routingTable = strategy.reroute(clusterState, "reroute", ActionListener.noop()).routingTable();
-        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
-        RoutingNodes routingNodes = clusterState.getRoutingNodes();
-
-        for (RoutingNode routingNode : routingNodes) {
-            for (ShardRouting shardRouting : routingNode) {
-                assertThat(shardRouting.state(), Matchers.equalTo(ShardRoutingState.INITIALIZING));
-            }
-        }
-        strategy = createAllocationService(settings.build(), new TestGatewayAllocator());
-
-        logger.info("use the new allocator and check if it moves shards");
-        routingTable = startInitializingShardsAndReroute(strategy, clusterState).routingTable();
-        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
-        routingNodes = clusterState.getRoutingNodes();
-        for (RoutingNode routingNode : routingNodes) {
-            for (ShardRouting shardRouting : routingNode) {
-                assertThat(shardRouting.state(), Matchers.equalTo(ShardRoutingState.STARTED));
-            }
-        }
-
-        logger.info("start the replica shards");
-        routingTable = startInitializingShardsAndReroute(strategy, clusterState).routingTable();
-        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
-        routingNodes = clusterState.getRoutingNodes();
-
-        for (RoutingNode routingNode : routingNodes) {
-            for (ShardRouting shardRouting : routingNode) {
-                assertThat(shardRouting.state(), Matchers.equalTo(ShardRoutingState.STARTED));
-            }
-        }
-
-        logger.info("rebalancing");
-        routingTable = strategy.reroute(clusterState, "reroute", ActionListener.noop()).routingTable();
-        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
-        routingNodes = clusterState.getRoutingNodes();
-
-        for (RoutingNode routingNode : routingNodes) {
-            for (ShardRouting shardRouting : routingNode) {
-                assertThat(shardRouting.state(), Matchers.equalTo(ShardRoutingState.STARTED));
-            }
-        }
-    }
-
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceUnbalancedClusterTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceUnbalancedClusterTests.java
index bbb1b8998084..df5b47a24107 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceUnbalancedClusterTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceUnbalancedClusterTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
@@ -51,7 +52,9 @@ protected ClusterState allocateNew(ClusterState state) {
             .put(IndexMetadata.builder(index).settings(settings(Version.CURRENT)).numberOfShards(5).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder(state.routingTable()).addAsNew(metadata.index(index)).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY, state.routingTable())
+            .addAsNew(metadata.index(index))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(state).metadata(metadata).routingTable(initialRoutingTable).build();
         clusterState = strategy.reroute(clusterState, "reroute", ActionListener.noop());
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java
index e1531a38efcb..7378ca56d2a4 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java
@@ -221,7 +221,7 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca
         allocator.allocate(routingAllocation);
         ShardRouting shardToRebalance = null;
         for (RoutingNode routingNode : routingAllocation.routingNodes()) {
-            List<ShardRouting> relocatingShards = routingNode.shardsWithState(ShardRoutingState.RELOCATING);
+            List<ShardRouting> relocatingShards = routingNode.shardsWithState(ShardRoutingState.RELOCATING).toList();
             if (relocatingShards.size() > 0) {
                 shardToRebalance = randomFrom(relocatingShards);
                 break;
@@ -276,7 +276,7 @@ public void testNodeDecisionsRanking() {
             if (node.numberOfShardsWithState(ShardRoutingState.STARTED) == 2) {
                 nodesWithTwoShards.add(node.nodeId());
                 if (shardToRebalance == null) {
-                    shardToRebalance = node.shardsWithState(ShardRoutingState.STARTED).get(0);
+                    shardToRebalance = node.shardsWithState(ShardRoutingState.STARTED).findFirst().get();
                 }
             } else {
                 assertEquals(3, node.numberOfShardsWithState(ShardRoutingState.STARTED));
@@ -298,6 +298,10 @@ public void testNodeDecisionsRanking() {
                 assertTrue(nodesWithTwoShards.contains(result.getNode().getId()));
             }
         }
+
+        assertCriticalWarnings("""
+            ignoring value [0.01] for [cluster.routing.allocation.balance.threshold] since it is smaller than 1.0; setting \
+            [cluster.routing.allocation.balance.threshold] to a value smaller than 1.0 will be forbidden in a future release""");
     }
 
     private MoveDecision executeRebalanceFor(
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java
index cc8673a0a356..8e7ff7063b82 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -107,17 +108,16 @@ public void testRun() throws IOException {
             }
             IndexMetadata idxMeta = idxMetaBuilder.build();
             builder.put(idxMeta, false);
-            IndexRoutingTable.Builder tableBuilder = new IndexRoutingTable.Builder(idxMeta.getIndex()).initializeAsRecovery(idxMeta);
-            Map<Integer, IndexShardRoutingTable> shardIdToRouting = new HashMap<>();
+            IndexRoutingTable.Builder tableBuilder = new IndexRoutingTable.Builder(
+                TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
+                idxMeta.getIndex()
+            ).initializeAsRecovery(idxMeta);
+            Map<Integer, IndexShardRoutingTable.Builder> shardIdToRouting = new HashMap<>();
             for (ShardRouting r : idx.routing) {
-                IndexShardRoutingTable refData = new IndexShardRoutingTable.Builder(r.shardId()).addShard(r).build();
-                if (shardIdToRouting.containsKey(r.getId())) {
-                    refData = new IndexShardRoutingTable.Builder(shardIdToRouting.get(r.getId())).addShard(r).build();
-                }
-                shardIdToRouting.put(r.getId(), refData);
+                shardIdToRouting.computeIfAbsent(r.shardId().getId(), i -> new IndexShardRoutingTable.Builder(r.shardId())).addShard(r);
             }
-            for (IndexShardRoutingTable t : shardIdToRouting.values()) {
-                tableBuilder.addIndexShard(new IndexShardRoutingTable.Builder(t));
+            for (IndexShardRoutingTable.Builder t : shardIdToRouting.values()) {
+                tableBuilder.addIndexShard(t);
             }
             IndexRoutingTable table = tableBuilder.build();
             routingTableBuilder.add(table);
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java
index c904100a5a80..1711af2773bd 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -54,7 +55,7 @@ public void testAlways() {
             .put(IndexMetadata.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder()
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("test1"))
             .addAsNew(metadata.index("test2"))
             .build();
@@ -148,7 +149,7 @@ public void testClusterPrimariesActive1() {
             )
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder()
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("test1"))
             .addAsNew(metadata.index("test2"))
             .build();
@@ -244,7 +245,7 @@ public void testClusterPrimariesActive2() {
 
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder()
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("test1"))
             .addAsNew(metadata.index("test2"))
             .build();
@@ -323,7 +324,7 @@ public void testClusterAllActive1() {
             .put(IndexMetadata.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder()
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("test1"))
             .addAsNew(metadata.index("test2"))
             .build();
@@ -434,7 +435,7 @@ public void testClusterAllActive2() {
             .put(IndexMetadata.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder()
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("test1"))
             .addAsNew(metadata.index("test2"))
             .build();
@@ -514,7 +515,7 @@ public void testClusterAllActive3() {
             .put(IndexMetadata.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder()
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("test1"))
             .addAsNew(metadata.index("test2"))
             .build();
@@ -617,7 +618,7 @@ public void allocateUnassigned(
             .put(IndexMetadata.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(0))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder()
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("test"))
             .addAsNew(metadata.index("test1"))
             .build();
@@ -724,7 +725,7 @@ public void beforeAllocation(RoutingAllocation allocation) {
 
         // we use a second index here (test1) that never gets assigned otherwise allocateUnassigned
         // is never called if we don't have unassigned shards.
-        RoutingTable initialRoutingTable = RoutingTable.builder()
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("test"))
             .addAsNew(metadata.index("test1"))
             .build();
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ConcurrentRebalanceRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ConcurrentRebalanceRoutingTests.java
index 9bd237e68d29..300a148bbb22 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ConcurrentRebalanceRoutingTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ConcurrentRebalanceRoutingTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -45,7 +46,9 @@ public void testClusterConcurrentRebalance() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(5).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java
index 737b09fcac87..01a4a4d6f78c 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -47,7 +48,9 @@ public void testSimpleDeadNodeOnStartedPrimaryShard() {
         Metadata metadata = Metadata.builder()
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
         ).metadata(metadata).routingTable(routingTable).build();
@@ -90,7 +93,9 @@ public void testLoggingOnNodeLeft() throws IllegalAccessException {
             ClusterState.builder(ClusterName.DEFAULT)
                 .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")).build())
                 .metadata(metadata)
-                .routingTable(RoutingTable.builder().addAsNew(metadata.index("test")).build())
+                .routingTable(
+                    RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(metadata.index("test")).build()
+                )
                 .build(),
             allocationService
         );
@@ -140,7 +145,9 @@ public void testDeadNodeWhileRelocatingOnToNode() {
         Metadata metadata = Metadata.builder()
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
         ).metadata(metadata).routingTable(routingTable).build();
@@ -219,7 +226,9 @@ public void testDeadNodeWhileRelocatingOnFromNode() {
         Metadata metadata = Metadata.builder()
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
         ).metadata(metadata).routingTable(routingTable).build();
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DecisionsImpactOnClusterHealthTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DecisionsImpactOnClusterHealthTests.java
index 44eba74bcae3..e2dd433e54cc 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DecisionsImpactOnClusterHealthTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DecisionsImpactOnClusterHealthTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.health.ClusterStateHealth;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -27,14 +28,14 @@
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.snapshots.EmptySnapshotsInfoService;
 import org.elasticsearch.test.gateway.TestGatewayAllocator;
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.Set;
+import java.util.List;
 
 import static org.hamcrest.Matchers.equalTo;
 
@@ -52,7 +53,7 @@ public void testPrimaryShardNoDecisionOnIndexCreation() throws IOException {
             .build();
         AllocationDecider decider = new TestAllocateDecision(Decision.NO);
         // if deciders say NO to allocating a primary shard, then the cluster health should be RED
-        runAllocationTest(settings, indexName, Collections.singleton(decider), ClusterHealthStatus.RED);
+        runAllocationTest(settings, indexName, decider, ClusterHealthStatus.RED);
     }
 
     public void testPrimaryShardThrottleDecisionOnIndexCreation() throws IOException {
@@ -62,7 +63,7 @@ public void testPrimaryShardThrottleDecisionOnIndexCreation() throws IOException
             .build();
         AllocationDecider decider = new TestAllocateDecision(Decision.THROTTLE);
         // if deciders THROTTLE allocating a primary shard, stay in YELLOW state
-        runAllocationTest(settings, indexName, Collections.singleton(decider), ClusterHealthStatus.YELLOW);
+        runAllocationTest(settings, indexName, decider, ClusterHealthStatus.YELLOW);
     }
 
     public void testPrimaryShardYesDecisionOnIndexCreation() throws IOException {
@@ -81,7 +82,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
             }
         };
         // if deciders say YES to allocating primary shards, stay in YELLOW state
-        ClusterState clusterState = runAllocationTest(settings, indexName, Collections.singleton(decider), ClusterHealthStatus.YELLOW);
+        ClusterState clusterState = runAllocationTest(settings, indexName, decider, ClusterHealthStatus.YELLOW);
         // make sure primaries are initialized
         final IndexRoutingTable indexRoutingTable = clusterState.routingTable().index(indexName);
         for (int i = 0; i < indexRoutingTable.size(); i++) {
@@ -92,12 +93,12 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
     private ClusterState runAllocationTest(
         final Settings settings,
         final String indexName,
-        final Set<AllocationDecider> allocationDeciders,
+        final AllocationDecider allocationDecider,
         final ClusterHealthStatus expectedStatus
     ) throws IOException {
 
         final String clusterName = "test-cluster";
-        final AllocationService allocationService = newAllocationService(settings, allocationDeciders);
+        final AllocationService allocationService = newAllocationService(settings, allocationDecider);
 
         logger.info("Building initial routing table");
         final int numShards = randomIntBetween(1, 5);
@@ -105,7 +106,9 @@ private ClusterState runAllocationTest(
             .put(IndexMetadata.builder(indexName).settings(settings(Version.CURRENT)).numberOfShards(numShards).numberOfReplicas(1))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index(indexName)).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index(indexName))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(new ClusterName(clusterName))
             .metadata(metadata)
@@ -133,13 +136,14 @@ private ClusterState runAllocationTest(
         return clusterState;
     }
 
-    private static AllocationService newAllocationService(Settings settings, Set<AllocationDecider> deciders) {
+    private static AllocationService newAllocationService(Settings settings, AllocationDecider decider) {
         return new AllocationService(
-            new AllocationDeciders(deciders),
+            new AllocationDeciders(List.of(decider, new ReplicaAfterPrimaryActiveAllocationDecider())),
             new TestGatewayAllocator(),
             new BalancedShardsAllocator(settings),
             EmptyClusterInfoService.INSTANCE,
-            EmptySnapshotsInfoService.INSTANCE
+            EmptySnapshotsInfoService.INSTANCE,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
     }
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java
index 1ac031357999..812cc46e9327 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.DiskUsage;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -36,6 +37,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.Releasable;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.MockLogAppender;
 import org.elasticsearch.test.junit.annotations.TestLogging;
@@ -90,7 +92,7 @@ private void doTestMarkFloodStageIndicesReadOnly(boolean testMaxHeadroom) {
                     .numberOfReplicas(0)
             )
             .build();
-        RoutingTable routingTable = RoutingTable.builder()
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("test"))
             .addAsNew(metadata.index("test_1"))
             .addAsNew(metadata.index("test_2"))
@@ -121,10 +123,10 @@ private void doTestMarkFloodStageIndicesReadOnly(boolean testMaxHeadroom) {
         ) {
 
             @Override
-            protected void updateIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionListener<Void> listener, boolean readOnly) {
+            protected void updateIndicesReadOnly(Set<String> indicesToMarkReadOnly, Releasable onCompletion, boolean readOnly) {
                 assertTrue(indices.compareAndSet(null, indicesToMarkReadOnly));
                 assertTrue(readOnly);
-                listener.onResponse(null);
+                onCompletion.close();
             }
         };
 
@@ -216,10 +218,10 @@ protected void updateIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionLi
             }
         ) {
             @Override
-            protected void updateIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionListener<Void> listener, boolean readOnly) {
+            protected void updateIndicesReadOnly(Set<String> indicesToMarkReadOnly, Releasable onCompletion, boolean readOnly) {
                 assertTrue(indices.compareAndSet(null, indicesToMarkReadOnly));
                 assertTrue(readOnly);
-                listener.onResponse(null);
+                onCompletion.close();
             }
         };
 
@@ -276,7 +278,7 @@ private void doTestDoesNotSubmitRerouteTaskTooFrequently(boolean testMaxHeadroom
             }
         ) {
             @Override
-            protected void updateIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionListener<Void> listener, boolean readOnly) {
+            protected void updateIndicesReadOnly(Set<String> indicesToMarkReadOnly, Releasable onCompletion, boolean readOnly) {
                 throw new AssertionError("unexpected");
             }
         };
@@ -434,7 +436,10 @@ private void doTestAutoReleaseIndices(boolean testMaxHeadroom) {
             .put(IndexMetadata.builder("test_1").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(1))
             .put(IndexMetadata.builder("test_2").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(1))
             .build();
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test_1")).addAsNew(metadata.index("test_2")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test_1"))
+            .addAsNew(metadata.index("test_2"))
+            .build();
         final ClusterState clusterState = applyStartedShardsUntilNoChange(
             ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metadata(metadata)
@@ -472,13 +477,13 @@ private void doTestAutoReleaseIndices(boolean testMaxHeadroom) {
             }
         ) {
             @Override
-            protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener<Void> listener, boolean readOnly) {
+            protected void updateIndicesReadOnly(Set<String> indicesToUpdate, Releasable onCompletion, boolean readOnly) {
                 if (readOnly) {
                     assertTrue(indicesToMarkReadOnly.compareAndSet(null, indicesToUpdate));
                 } else {
                     assertTrue(indicesToRelease.compareAndSet(null, indicesToUpdate));
                 }
-                listener.onResponse(null);
+                onCompletion.close();
             }
         };
         indicesToMarkReadOnly.set(null);
@@ -564,13 +569,13 @@ protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener
             }
         ) {
             @Override
-            protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener<Void> listener, boolean readOnly) {
+            protected void updateIndicesReadOnly(Set<String> indicesToUpdate, Releasable onCompletion, boolean readOnly) {
                 if (readOnly) {
                     assertTrue(indicesToMarkReadOnly.compareAndSet(null, indicesToUpdate));
                 } else {
                     assertTrue(indicesToRelease.compareAndSet(null, indicesToUpdate));
                 }
-                listener.onResponse(null);
+                onCompletion.close();
             }
         };
         // When free disk on any of node1 or node2 goes below the flood watermark, then apply index block on indices not having the block
@@ -767,7 +772,10 @@ private void doTestNoAutoReleaseOfIndicesOnReplacementNodes(boolean testMaxHeadr
             .put(IndexMetadata.builder("test_1").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(1))
             .put(IndexMetadata.builder("test_2").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(1))
             .build();
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test_1")).addAsNew(metadata.index("test_2")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test_1"))
+            .addAsNew(metadata.index("test_2"))
+            .build();
         final ClusterState clusterState = applyStartedShardsUntilNoChange(
             ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metadata(metadata)
@@ -807,13 +815,13 @@ private void doTestNoAutoReleaseOfIndicesOnReplacementNodes(boolean testMaxHeadr
             }
         ) {
             @Override
-            protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener<Void> listener, boolean readOnly) {
+            protected void updateIndicesReadOnly(Set<String> indicesToUpdate, Releasable onCompletion, boolean readOnly) {
                 if (readOnly) {
                     assertTrue(indicesToMarkReadOnly.compareAndSet(null, indicesToUpdate));
                 } else {
                     assertTrue(indicesToRelease.compareAndSet(null, indicesToUpdate));
                 }
-                listener.onResponse(null);
+                onCompletion.close();
             }
         };
         indicesToMarkReadOnly.set(null);
@@ -1051,8 +1059,8 @@ public long getAsLong() {
             (reason, priority, listener) -> listener.onResponse(clusterStateRef.get())
         ) {
             @Override
-            protected void updateIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionListener<Void> listener, boolean readOnly) {
-                listener.onResponse(null);
+            protected void updateIndicesReadOnly(Set<String> indicesToMarkReadOnly, Releasable onCompletion, boolean readOnly) {
+                onCompletion.close();
             }
 
             @Override
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ElectReplicaAsPrimaryDuringRelocationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ElectReplicaAsPrimaryDuringRelocationTests.java
index c805426c91ef..88b4b3d73514 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ElectReplicaAsPrimaryDuringRelocationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ElectReplicaAsPrimaryDuringRelocationTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -40,7 +41,9 @@ public void testElectReplicaAsPrimaryDuringRelocation() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java
index edd5ced7e986..341313d3b56f 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java
@@ -13,17 +13,21 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterInfo;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
-import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.shard.ShardId;
+
+import java.util.Map;
 
 import static org.elasticsearch.cluster.routing.RoutingNodesHelper.shardsWithState;
 import static org.hamcrest.Matchers.equalTo;
@@ -34,35 +38,18 @@ public class ExpectedShardSizeAllocationTests extends ESAllocationTestCase {
 
     public void testInitializingHasExpectedSize() {
         final long byteSize = randomIntBetween(0, Integer.MAX_VALUE);
-        AllocationService strategy = createAllocationService(Settings.EMPTY, () -> new ClusterInfo() {
-            @Override
-            public Long getShardSize(ShardRouting shardRouting) {
-                if (shardRouting.getIndexName().equals("test") && shardRouting.shardId().getId() == 0) {
-                    return byteSize;
-                }
-                return null;
-            }
-        });
+        final ClusterInfo clusterInfo = createClusterInfoWith(new ShardId("test", "_na_", 0), byteSize);
+        AllocationService strategy = createAllocationService(Settings.EMPTY, () -> clusterInfo);
 
         logger.info("Building initial routing table");
+        var indexMetadata = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1).build();
 
-        Metadata metadata = Metadata.builder()
-            .put(
-                IndexMetadata.builder("test")
-                    .settings(
-                        settings(Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
-                    )
-            )
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metadata(Metadata.builder().put(indexMetadata, false))
+            .routingTable(RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(indexMetadata))
+            .nodes(DiscoveryNodes.builder().add(newNode("node1")))
             .build();
-
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
-
-        ClusterState clusterState = ClusterState.builder(
-            org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
-        ).metadata(metadata).routingTable(routingTable).build();
         logger.info("Adding one node and performing rerouting");
-        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
         clusterState = strategy.reroute(clusterState, "reroute", ActionListener.noop());
 
         assertEquals(1, clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.INITIALIZING));
@@ -89,28 +76,17 @@ public Long getShardSize(ShardRouting shardRouting) {
 
     public void testExpectedSizeOnMove() {
         final long byteSize = randomIntBetween(0, Integer.MAX_VALUE);
-        final AllocationService allocation = createAllocationService(Settings.EMPTY, () -> new ClusterInfo() {
-            @Override
-            public Long getShardSize(ShardRouting shardRouting) {
-                if (shardRouting.getIndexName().equals("test") && shardRouting.shardId().getId() == 0) {
-                    return byteSize;
-                }
-                return null;
-            }
-        });
+        final ClusterInfo clusterInfo = createClusterInfoWith(new ShardId("test", "_na_", 0), byteSize);
+        final AllocationService allocation = createAllocationService(Settings.EMPTY, () -> clusterInfo);
         logger.info("creating an index with 1 shard, no replica");
-        Metadata metadata = Metadata.builder()
-            .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
+        var indexMetadata = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0).build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metadata(Metadata.builder().put(indexMetadata, false))
+            .routingTable(RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(indexMetadata))
+            .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")))
             .build();
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
-        ClusterState clusterState = ClusterState.builder(
-            org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
-        ).metadata(metadata).routingTable(routingTable).build();
 
         logger.info("adding two nodes and performing rerouting");
-        clusterState = ClusterState.builder(clusterState)
-            .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")))
-            .build();
         clusterState = allocation.reroute(clusterState, "reroute", ActionListener.noop());
 
         logger.info("start primary shard");
@@ -118,12 +94,8 @@ public Long getShardSize(ShardRouting shardRouting) {
 
         logger.info("move the shard");
         String existingNodeId = clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId();
-        String toNodeId;
-        if ("node1".equals(existingNodeId)) {
-            toNodeId = "node2";
-        } else {
-            toNodeId = "node1";
-        }
+        String toNodeId = "node1".equals(existingNodeId) ? "node2" : "node1";
+
         AllocationService.CommandsResult commandsResult = allocation.reroute(
             clusterState,
             new AllocationCommands(new MoveAllocationCommand("test", 0, existingNodeId, toNodeId)),
@@ -147,4 +119,18 @@ public Long getShardSize(ShardRouting shardRouting) {
         assertThat(clusterState.getRoutingNodes().node(toNodeId).iterator().next().state(), equalTo(ShardRoutingState.STARTED));
         assertEquals(clusterState.getRoutingNodes().node(toNodeId).iterator().next().getExpectedShardSize(), -1);
     }
+
+    private static ClusterInfo createClusterInfoWith(ShardId shardId, long size) {
+        return new ClusterInfo(
+            Map.of(),
+            Map.of(),
+            Map.ofEntries(
+                Map.entry(ClusterInfo.shardIdentifierFromRouting(shardId, true), size),
+                Map.entry(ClusterInfo.shardIdentifierFromRouting(shardId, false), size)
+            ),
+            Map.of(),
+            Map.of(),
+            Map.of()
+        );
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java
index 9c47c227972d..da7bb240d45e 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -29,6 +30,7 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.indices.cluster.ClusterStateChanges;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -67,7 +69,7 @@ public void testSimpleFailedNodeTest() {
             .put(IndexMetadata.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder()
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("test1"))
             .addAsNew(metadata.index("test2"))
             .build();
@@ -220,7 +222,7 @@ public ClusterState randomInitialClusterState() {
     protected DiscoveryNode createNode(DiscoveryNodeRole... mustHaveRoles) {
         Set<DiscoveryNodeRole> roles = new HashSet<>(randomSubsetOf(DiscoveryNodeRole.roles()));
         Collections.addAll(roles, mustHaveRoles);
-        final String id = formatted("node_%03d", nodeIdGenerator.incrementAndGet());
+        final String id = Strings.format("node_%03d", nodeIdGenerator.incrementAndGet());
         return new DiscoveryNode(
             id,
             id,
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java
index 7f814c4dd866..9bc7c13eb51c 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -41,7 +42,6 @@
 import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -60,7 +60,9 @@ public void testFailedShardPrimaryRelocatingToAndFrom() {
         Metadata metadata = Metadata.builder()
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
         ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
             .routingTable(routingTable)
@@ -151,6 +153,12 @@ public void testFailedShardPrimaryRelocatingToAndFrom() {
         // check promotion of replica to primary
         assertThat(clusterState.getRoutingNodes().node(origReplicaNodeId).iterator().next().state(), equalTo(STARTED));
         assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), equalTo(origReplicaNodeId));
+
+        if (clusterState.routingTable().index("test").shard(0).replicaShards().get(0).assignedToNode() == false) {
+            // desired node may be ignored due to previous failure - try again if so
+            clusterState = startShardsAndReroute(allocation, clusterState);
+        }
+
         assertThat(
             clusterState.routingTable().index("test").shard(0).replicaShards().get(0).currentNodeId(),
             anyOf(equalTo(origPrimaryNodeId), equalTo("node3"))
@@ -171,7 +179,9 @@ public void testFailPrimaryStartedCheckReplicaElected() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
@@ -261,7 +271,9 @@ public void testFirstAllocationFailureSingleNode() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
@@ -313,7 +325,9 @@ public void testSingleShardMultipleAllocationFailures() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(numberOfReplicas))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
@@ -380,7 +394,9 @@ public void testFirstAllocationFailureTwoNodes() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
@@ -438,7 +454,9 @@ public void testRebalanceFailure() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
@@ -501,10 +519,14 @@ public void testRebalanceFailure() {
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
 
         assertThat(clusterState.routingTable().index("test").size(), equalTo(2));
-        assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED, RELOCATING), equalTo(2));
-        assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED), lessThan(3));
-        assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED, RELOCATING), equalTo(2));
-        assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), lessThan(3));
+        assertThat(
+            routingNodes.node("node1").numberOfShardsWithState(STARTED) + routingNodes.node("node1").numberOfShardsWithState(RELOCATING),
+            equalTo(2)
+        );
+        assertThat(
+            routingNodes.node("node2").numberOfShardsWithState(STARTED) + routingNodes.node("node2").numberOfShardsWithState(RELOCATING),
+            equalTo(2)
+        );
         assertThat(routingNodes.node("node3").numberOfShardsWithState(INITIALIZING), equalTo(1));
 
         logger.info("Fail the shards on node 3");
@@ -515,10 +537,14 @@ public void testRebalanceFailure() {
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(clusterState.routingTable().index("test").size(), equalTo(2));
-        assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED, RELOCATING), equalTo(2));
-        assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED), lessThan(3));
-        assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED, RELOCATING), equalTo(2));
-        assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), lessThan(3));
+        assertThat(
+            routingNodes.node("node1").numberOfShardsWithState(STARTED) + routingNodes.node("node1").numberOfShardsWithState(RELOCATING),
+            equalTo(2)
+        );
+        assertThat(
+            routingNodes.node("node2").numberOfShardsWithState(STARTED) + routingNodes.node("node2").numberOfShardsWithState(RELOCATING),
+            equalTo(2)
+        );
 
         if (strategy.isBalancedShardsAllocator()) {
             assertThat(routingNodes.node("node3").numberOfShardsWithState(INITIALIZING), equalTo(1));
@@ -543,7 +569,9 @@ public void testFailAllReplicasInitializingOnPrimaryFail() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(2))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
@@ -582,7 +610,7 @@ public void testFailAllReplicasInitializingOnPrimaryFail() {
         );
         assertThat(shardsWithState(clusterState.getRoutingNodes(), STARTED).size(), equalTo(2));
         assertThat(shardsWithState(clusterState.getRoutingNodes(), INITIALIZING).size(), equalTo(1));
-        ShardRouting startedReplica = clusterState.getRoutingNodes().activeReplicaWithHighestVersion(shardId);
+        ShardRouting startedReplica = clusterState.getRoutingNodes().activePromotableReplicaWithHighestVersion(shardId);
 
         // fail the primary shard, check replicas get removed as well...
         ShardRouting primaryShardToFail = clusterState.routingTable().index("test").shard(0).primaryShard();
@@ -613,7 +641,9 @@ public void testFailAllReplicasInitializingOnPrimaryFailWhileHavingAReplicaToEle
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(2))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
@@ -669,7 +699,9 @@ public void testReplicaOnNewestVersionIsPromoted() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(3))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
@@ -732,7 +764,7 @@ public void testReplicaOnNewestVersionIsPromoted() {
         assertThat(shardsWithState(clusterState.getRoutingNodes(), STARTED).size(), equalTo(4));
         assertThat(shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).size(), equalTo(0));
 
-        ShardRouting startedReplica = clusterState.getRoutingNodes().activeReplicaWithHighestVersion(shardId);
+        ShardRouting startedReplica = clusterState.getRoutingNodes().activePromotableReplicaWithHighestVersion(shardId);
         logger.info("--> all shards allocated, replica that should be promoted: {}", startedReplica);
 
         // fail the primary shard again and make sure the correct replica is promoted
@@ -763,7 +795,7 @@ public void testReplicaOnNewestVersionIsPromoted() {
             );
         }
 
-        startedReplica = clusterState.getRoutingNodes().activeReplicaWithHighestVersion(shardId);
+        startedReplica = clusterState.getRoutingNodes().activePromotableReplicaWithHighestVersion(shardId);
         logger.info("--> failing primary shard a second time, should select: {}", startedReplica);
 
         // fail the primary shard again, and ensure the same thing happens
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java
index 028b9b476652..ed30e9c5d904 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -193,7 +194,9 @@ private void testClusterFilters(Settings.Builder allocationServiceSettings, Disc
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(1))
             .build();
 
-        final RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        final RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
@@ -310,7 +313,9 @@ private void testIndexFilters(Settings.Builder initialIndexSettings, Settings.Bu
             )
             .build();
 
-        final RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(initialMetadata.index("test")).build();
+        final RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(initialMetadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(initialMetadata)
@@ -370,7 +375,7 @@ public void testConcurrentRecoveriesAfterShardsCannotRemainOnNode() {
             .put(IndexMetadata.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(0))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder()
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("test1"))
             .addAsNew(metadata.index("test2"))
             .build();
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/InSyncAllocationIdTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/InSyncAllocationIdTests.java
index 7892061ecba4..f99e7142cf95 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/InSyncAllocationIdTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/InSyncAllocationIdTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction.FailedShardEntry;
 import org.elasticsearch.cluster.action.shard.ShardStateAction.FailedShardUpdateTask;
@@ -63,7 +64,7 @@ public void testInSyncAllocationIdsUpdated() {
                     .putInSyncAllocationIds(0, Set.of("x", "y"))
             )
             .build();
-        RoutingTable routingTable = RoutingTable.builder()
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("test"))
             .addAsRecovery(metadata.index("test-old"))
             .build();
@@ -324,7 +325,11 @@ public void testInSyncIdsNotTrimmedWhenNotGrowing() throws Exception {
 
         logger.info("decrease number of replicas to 0");
         clusterState = ClusterState.builder(clusterState)
-            .routingTable(RoutingTable.builder(clusterState.routingTable()).updateNumberOfReplicas(0, new String[] { "test" }).build())
+            .routingTable(
+                RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY, clusterState.routingTable())
+                    .updateNumberOfReplicas(0, new String[] { "test" })
+                    .build()
+            )
             .metadata(Metadata.builder(clusterState.metadata()).updateNumberOfReplicas(0, new String[] { "test" }))
             .build();
 
@@ -386,7 +391,9 @@ private ClusterState createOnePrimaryOneReplicaClusterState(AllocationService al
         Metadata metadata = Metadata.builder()
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
         ).metadata(metadata).routingTable(routingTable).build();
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/IndexBalanceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/IndexBalanceTests.java
index b814aba17c69..29cec2205557 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/IndexBalanceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/IndexBalanceTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -49,7 +50,7 @@ public void testBalanceAllNodesStarted() {
             .put(IndexMetadata.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(3).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder()
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("test"))
             .addAsNew(metadata.index("test1"))
             .build();
@@ -142,13 +143,13 @@ public void testBalanceAllNodesStarted() {
         assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), equalTo(4));
         assertThat(routingNodes.node("node3").numberOfShardsWithState(STARTED), equalTo(4));
 
-        assertThat(routingNodes.node("node1").shardsWithState("test", STARTED).size(), equalTo(2));
-        assertThat(routingNodes.node("node2").shardsWithState("test", STARTED).size(), equalTo(2));
-        assertThat(routingNodes.node("node3").shardsWithState("test", STARTED).size(), equalTo(2));
+        assertThat(routingNodes.node("node1").shardsWithState("test", STARTED).count(), equalTo(2L));
+        assertThat(routingNodes.node("node2").shardsWithState("test", STARTED).count(), equalTo(2L));
+        assertThat(routingNodes.node("node3").shardsWithState("test", STARTED).count(), equalTo(2L));
 
-        assertThat(routingNodes.node("node1").shardsWithState("test1", STARTED).size(), equalTo(2));
-        assertThat(routingNodes.node("node2").shardsWithState("test1", STARTED).size(), equalTo(2));
-        assertThat(routingNodes.node("node3").shardsWithState("test1", STARTED).size(), equalTo(2));
+        assertThat(routingNodes.node("node1").shardsWithState("test1", STARTED).count(), equalTo(2L));
+        assertThat(routingNodes.node("node2").shardsWithState("test1", STARTED).count(), equalTo(2L));
+        assertThat(routingNodes.node("node3").shardsWithState("test1", STARTED).count(), equalTo(2L));
     }
 
     public void testBalanceIncrementallyStartNodes() {
@@ -168,7 +169,7 @@ public void testBalanceIncrementallyStartNodes() {
             .put(IndexMetadata.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(3).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder()
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("test"))
             .addAsNew(metadata.index("test1"))
             .build();
@@ -265,13 +266,13 @@ public void testBalanceIncrementallyStartNodes() {
         assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(STARTED), equalTo(4));
         assertThat(clusterState.getRoutingNodes().node("node3").numberOfShardsWithState(STARTED), equalTo(4));
 
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState("test", STARTED).size(), equalTo(2));
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState("test", STARTED).size(), equalTo(2));
-        assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState("test", STARTED).size(), equalTo(2));
+        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState("test", STARTED).count(), equalTo(2L));
+        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState("test", STARTED).count(), equalTo(2L));
+        assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState("test", STARTED).count(), equalTo(2L));
 
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState("test1", STARTED).size(), equalTo(2));
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState("test1", STARTED).size(), equalTo(2));
-        assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState("test1", STARTED).size(), equalTo(2));
+        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState("test1", STARTED).count(), equalTo(2L));
+        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState("test1", STARTED).count(), equalTo(2L));
+        assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState("test1", STARTED).count(), equalTo(2L));
     }
 
     public void testBalanceAllNodesStartedAddIndex() {
@@ -290,7 +291,9 @@ public void testBalanceAllNodesStartedAddIndex() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(3).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -360,9 +363,9 @@ public void testBalanceAllNodesStartedAddIndex() {
         assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), equalTo(2));
         assertThat(routingNodes.node("node3").numberOfShardsWithState(STARTED), equalTo(2));
 
-        assertThat(routingNodes.node("node1").shardsWithState("test", STARTED).size(), equalTo(2));
-        assertThat(routingNodes.node("node2").shardsWithState("test", STARTED).size(), equalTo(2));
-        assertThat(routingNodes.node("node3").shardsWithState("test", STARTED).size(), equalTo(2));
+        assertThat(routingNodes.node("node1").shardsWithState("test", STARTED).count(), equalTo(2L));
+        assertThat(routingNodes.node("node2").shardsWithState("test", STARTED).count(), equalTo(2L));
+        assertThat(routingNodes.node("node3").shardsWithState("test", STARTED).count(), equalTo(2L));
 
         logger.info("Add new index 3 shards 1 replica");
 
@@ -375,9 +378,10 @@ public void testBalanceAllNodesStartedAddIndex() {
                     )
             )
             .build();
-        RoutingTable updatedRoutingTable = RoutingTable.builder(clusterState.routingTable())
-            .addAsNew(updatedMetadata.index("test1"))
-            .build();
+        RoutingTable updatedRoutingTable = RoutingTable.builder(
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
+            clusterState.routingTable()
+        ).addAsNew(updatedMetadata.index("test1")).build();
         clusterState = ClusterState.builder(clusterState).metadata(updatedMetadata).routingTable(updatedRoutingTable).build();
 
         assertThat(clusterState.routingTable().index("test1").size(), equalTo(3));
@@ -432,8 +436,8 @@ public void testBalanceAllNodesStartedAddIndex() {
         assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), equalTo(4));
         assertThat(routingNodes.node("node3").numberOfShardsWithState(STARTED), equalTo(4));
 
-        assertThat(routingNodes.node("node1").shardsWithState("test1", STARTED).size(), equalTo(2));
-        assertThat(routingNodes.node("node2").shardsWithState("test1", STARTED).size(), equalTo(2));
-        assertThat(routingNodes.node("node3").shardsWithState("test1", STARTED).size(), equalTo(2));
+        assertThat(routingNodes.node("node1").shardsWithState("test1", STARTED).count(), equalTo(2L));
+        assertThat(routingNodes.node("node2").shardsWithState("test1", STARTED).count(), equalTo(2L));
+        assertThat(routingNodes.node("node3").shardsWithState("test1", STARTED).count(), equalTo(2L));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java
index f2d69ceb3096..5b774b9794ce 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -37,6 +38,7 @@
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
+import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
@@ -49,14 +51,17 @@ public class MaxRetryAllocationDeciderTests extends ESAllocationTestCase {
         new TestGatewayAllocator(),
         new BalancedShardsAllocator(Settings.EMPTY),
         EmptyClusterInfoService.INSTANCE,
-        EmptySnapshotsInfoService.INSTANCE
+        EmptySnapshotsInfoService.INSTANCE,
+        TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
     );
 
     private ClusterState createInitialClusterState() {
         Metadata metadata = Metadata.builder()
             .put(IndexMetadata.builder("idx").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
             .build();
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("idx")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("idx"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
@@ -167,9 +172,16 @@ public void testFailedAllocation() {
             assertEquals(unassignedPrimary.state(), UNASSIGNED);
             assertThat(unassignedPrimary.unassignedInfo().getMessage(), containsString("boom"));
             // MaxRetryAllocationDecider#canForceAllocatePrimary should return a NO decision because canAllocate returns NO here
-            assertEquals(
-                Decision.Type.NO,
-                decider.canForceAllocatePrimary(unassignedPrimary, null, newRoutingAllocation(clusterState)).type()
+            final var allocation = newRoutingAllocation(clusterState);
+            allocation.debugDecision(true);
+            final var decision = decider.canForceAllocatePrimary(unassignedPrimary, null, allocation);
+            assertEquals(Decision.Type.NO, decision.type());
+            assertThat(
+                decision.getExplanation(),
+                allOf(
+                    containsString("shard has exceeded the maximum number of retries"),
+                    containsString("POST /_cluster/reroute?retry_failed&metric=none")
+                )
             );
         }
 
@@ -260,8 +272,16 @@ public void testFailedRelocation() {
 
         // shard could not be relocated when retries are exhausted
         withRoutingAllocation(clusterState, allocation -> {
-            var source = allocation.routingTable().index("idx").shard(0).shard(0);
-            assertThat(decider.canAllocate(source, allocation).type(), equalTo(Decision.Type.NO));
+            allocation.debugDecision(true);
+            final var decision = decider.canAllocate(allocation.routingTable().index("idx").shard(0).shard(0), allocation);
+            assertThat(decision.type(), equalTo(Decision.Type.NO));
+            assertThat(
+                decision.getExplanation(),
+                allOf(
+                    containsString("shard has exceeded the maximum number of retries"),
+                    containsString("POST /_cluster/reroute?retry_failed&metric=none")
+                )
+            );
         });
 
         // manually reset retry count
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
index cf1d56dfc872..43f34c200d9e 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -92,7 +93,9 @@ public void testDoNotAllocateFromPrimary() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(5).numberOfReplicas(2))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -183,7 +186,7 @@ public void testRandom() {
 
         logger.info("Building initial routing table");
         Metadata.Builder builder = Metadata.builder();
-        RoutingTable.Builder rtBuilder = RoutingTable.builder();
+        RoutingTable.Builder rtBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         int numIndices = between(1, 20);
         for (int i = 0; i < numIndices; i++) {
             builder.put(
@@ -245,7 +248,9 @@ public void testRollingRestart() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(5).numberOfReplicas(2))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -471,7 +476,7 @@ public void testRestoreDoesNotAllocateSnapshotOnOlderNodes() {
         ClusterState state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
             .routingTable(
-                RoutingTable.builder()
+                RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
                     .addAsRestore(
                         metadata.index("test"),
                         new SnapshotRecoverySource(UUIDs.randomBase64UUID(), snapshot, Version.CURRENT, indexId)
@@ -577,7 +582,9 @@ public void testMessages() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         RoutingNode newNode = RoutingNodesHelper.routingNode("newNode", newNode("newNode", Version.CURRENT));
         RoutingNode oldNode = RoutingNodesHelper.routingNode("oldNode", newNode("oldNode", VersionUtils.getPreviousVersion()));
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferLocalPrimariesToRelocatingPrimariesTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferLocalPrimariesToRelocatingPrimariesTests.java
index 54b4dab4642c..d6244474433e 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferLocalPrimariesToRelocatingPrimariesTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferLocalPrimariesToRelocatingPrimariesTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -53,7 +54,7 @@ public void testPreferLocalPrimaryAllocationOverFiltered() {
             .put(IndexMetadata.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(numberOfShards).numberOfReplicas(0))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder()
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("test1"))
             .addAsNew(metadata.index("test2"))
             .build();
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferPrimaryAllocationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferPrimaryAllocationTests.java
index b640cba42058..5eea4211375e 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferPrimaryAllocationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferPrimaryAllocationTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -45,7 +46,7 @@ public void testPreferPrimaryAllocationOverReplicas() {
             .put(IndexMetadata.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(10).numberOfReplicas(0))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder()
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("test1"))
             .addAsNew(metadata.index("test2"))
             .build();
@@ -66,7 +67,10 @@ public void testPreferPrimaryAllocationOverReplicas() {
 
         logger.info("increasing the number of replicas to 1, and perform a reroute (to get the replicas allocation going)");
         final String[] indices = { "test1", "test2" };
-        RoutingTable updatedRoutingTable = RoutingTable.builder(clusterState.routingTable()).updateNumberOfReplicas(1, indices).build();
+        RoutingTable updatedRoutingTable = RoutingTable.builder(
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
+            clusterState.routingTable()
+        ).updateNumberOfReplicas(1, indices).build();
         metadata = Metadata.builder(clusterState.metadata()).updateNumberOfReplicas(1, indices).build();
         clusterState = ClusterState.builder(clusterState).routingTable(updatedRoutingTable).metadata(metadata).build();
 
@@ -80,7 +84,9 @@ public void testPreferPrimaryAllocationOverReplicas() {
             .put(IndexMetadata.builder("new_index").settings(settings(Version.CURRENT)).numberOfShards(4).numberOfReplicas(0))
             .build();
 
-        updatedRoutingTable = RoutingTable.builder(clusterState.routingTable()).addAsNew(metadata.index("new_index")).build();
+        updatedRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY, clusterState.routingTable())
+            .addAsNew(metadata.index("new_index"))
+            .build();
 
         clusterState = ClusterState.builder(clusterState).metadata(metadata).routingTable(updatedRoutingTable).build();
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java
index 2dbcf8f62cab..fa2a26dd2257 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -42,7 +43,9 @@ public void testBackupElectionToPrimaryWhenPrimaryCanBeAllocatedToAnotherNode()
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -94,7 +97,9 @@ public void testRemovingInitializingReplicasIfPrimariesFails() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(1))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -126,8 +131,7 @@ public void testRemovingInitializingReplicasIfPrimariesFails() {
         assertThat(shardsWithState(routingNodes, STARTED).size(), equalTo(1));
         assertThat(shardsWithState(routingNodes, INITIALIZING).size(), equalTo(0));
         assertThat(shardsWithState(routingNodes, UNASSIGNED).size(), equalTo(3)); // 2 replicas and one primary
-        assertThat(routingNodes.node(nodeIdRemaining).shardsWithState(STARTED).get(0).primary(), equalTo(true));
+        assertThat(routingNodes.node(nodeIdRemaining).shardsWithState(STARTED).findFirst().get().primary(), equalTo(true));
         assertThat(clusterState.metadata().index("test").primaryTerm(0), equalTo(2L));
-
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryNotRelocatedWhileBeingRecoveredTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryNotRelocatedWhileBeingRecoveredTests.java
index 6a3fdc331afb..3aebcec496c6 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryNotRelocatedWhileBeingRecoveredTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryNotRelocatedWhileBeingRecoveredTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -44,7 +45,9 @@ public void testPrimaryNotRelocatedWhileBeingRecoveredFrom() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(5).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
index d8c2beb40688..2bc63eb190e6 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.Metadata.Builder;
@@ -27,7 +28,6 @@
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider;
-import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.snapshots.EmptySnapshotsInfoService;
 import org.elasticsearch.test.gateway.TestGatewayAllocator;
@@ -40,6 +40,7 @@
 
 import static org.elasticsearch.cluster.routing.RoutingNodesHelper.shardsWithState;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
+import static org.elasticsearch.common.settings.ClusterSettings.createBuiltInClusterSettings;
 import static org.hamcrest.Matchers.equalTo;
 
 public class RandomAllocationDeciderTests extends ESAllocationTestCase {
@@ -54,10 +55,7 @@ public void testRandomDecisions() {
             new AllocationDeciders(
                 new HashSet<>(
                     Arrays.asList(
-                        new SameShardAllocationDecider(
-                            Settings.EMPTY,
-                            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
-                        ),
+                        new SameShardAllocationDecider(createBuiltInClusterSettings()),
                         new ReplicaAfterPrimaryActiveAllocationDecider(),
                         randomAllocationDecider
                     )
@@ -66,7 +64,8 @@ public void testRandomDecisions() {
             new TestGatewayAllocator(),
             new BalancedShardsAllocator(Settings.EMPTY),
             EmptyClusterInfoService.INSTANCE,
-            EmptySnapshotsInfoService.INSTANCE
+            EmptySnapshotsInfoService.INSTANCE,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
         int indices = scaledRandomIntBetween(1, 20);
         Builder metaBuilder = Metadata.builder();
@@ -83,7 +82,7 @@ public void testRandomDecisions() {
 
         }
         Metadata metadata = metaBuilder.build();
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         for (int i = 0; i < indices; i++) {
             routingTableBuilder.addAsNew(metadata.index("INDEX_" + i));
         }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RebalanceAfterActiveTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RebalanceAfterActiveTests.java
index 9d47e84bafc5..a438890b7820 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RebalanceAfterActiveTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RebalanceAfterActiveTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -65,7 +66,9 @@ public Long getShardSize(ShardRouting shardRouting) {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(5).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ReplicaAllocatedAfterPrimaryTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ReplicaAllocatedAfterPrimaryTests.java
index 2934c2a1c790..e520ec219a10 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ReplicaAllocatedAfterPrimaryTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ReplicaAllocatedAfterPrimaryTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -42,7 +43,9 @@ public void testBackupIsAllocatedAfterPrimary() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java
index 52057b0383d5..6cca1271c65f 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -55,7 +56,8 @@ public void setUp() throws Exception {
             new TestGatewayAllocator(),
             new BalancedShardsAllocator(Settings.EMPTY),
             EmptyClusterInfoService.INSTANCE,
-            EmptySnapshotsInfoService.INSTANCE
+            EmptySnapshotsInfoService.INSTANCE,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
     }
 
@@ -69,7 +71,7 @@ private ClusterState createInitialClusterState(boolean startShards) {
                 .setRoutingNumShards(16)
         );
         Metadata metadata = metaBuilder.build();
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         routingTableBuilder.addAsNew(metadata.index("source"));
 
         RoutingTable routingTable = routingTableBuilder.build();
@@ -134,7 +136,10 @@ public void testShrink() { // we don't handle shrink yet
                 .numberOfReplicas(0)
         );
         Metadata metadata = metaBuilder.build();
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(clusterState.routingTable());
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
+            clusterState.routingTable()
+        );
         routingTableBuilder.addAsNew(metadata.index("target"));
 
         clusterState = ClusterState.builder(clusterState).routingTable(routingTableBuilder.build()).metadata(metadata).build();
@@ -173,7 +178,10 @@ public void testSourceNotActive() {
                 .numberOfReplicas(0)
         );
         Metadata metadata = metaBuilder.build();
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(clusterState.routingTable());
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
+            clusterState.routingTable()
+        );
         routingTableBuilder.addAsNew(metadata.index("target"));
 
         clusterState = ClusterState.builder(clusterState).routingTable(routingTableBuilder.build()).metadata(metadata).build();
@@ -230,7 +238,10 @@ public void testSourcePrimaryActive() {
                 .numberOfReplicas(0)
         );
         Metadata metadata = metaBuilder.build();
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(clusterState.routingTable());
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
+            clusterState.routingTable()
+        );
         routingTableBuilder.addAsNew(metadata.index("target"));
 
         clusterState = ClusterState.builder(clusterState).routingTable(routingTableBuilder.build()).metadata(metadata).build();
@@ -339,7 +350,8 @@ public void testGetForcedInitialShardAllocationToNodes() {
             new ShardId(target.getIndex(), 0),
             true,
             RecoverySource.LocalShardsRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "index created")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "index created"),
+            ShardRouting.Role.DEFAULT
         );
         assertThat(decider.getForcedInitialShardAllocationToNodes(localRecoveryShard, allocation), equalTo(Optional.of(Set.of("node-1"))));
 
@@ -347,7 +359,8 @@ public void testGetForcedInitialShardAllocationToNodes() {
             new ShardId(target.getIndex(), 0),
             true,
             RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "index created")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "index created"),
+            ShardRouting.Role.DEFAULT
         );
         assertThat(decider.getForcedInitialShardAllocationToNodes(newShard, allocation), equalTo(Optional.empty()));
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeSourceIndexSettingsUpdaterTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeSourceIndexSettingsUpdaterTests.java
index 84c581dad3cf..832227782f53 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeSourceIndexSettingsUpdaterTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeSourceIndexSettingsUpdaterTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -60,7 +61,9 @@ public void testResizeIndexSettingsRemovedAfterStart() {
             .build();
 
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
-            .routingTable(RoutingTable.builder().addAsNew(sourceMetadata.index(sourceIndex)))
+            .routingTable(
+                RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(sourceMetadata.index(sourceIndex))
+            )
             .metadata(sourceMetadata)
             .nodes(discoveryNodes)
             .build();
@@ -123,7 +126,10 @@ public void testResizeIndexSettingsRemovedAfterStart() {
             )
             .build();
         clusterState = ClusterState.builder(clusterState)
-            .routingTable(RoutingTable.builder(clusterState.routingTable()).addAsNew(clusterState.metadata().index(targetIndex)))
+            .routingTable(
+                RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY, clusterState.routingTable())
+                    .addAsNew(clusterState.metadata().index(targetIndex))
+            )
             .build();
 
         {
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RetryFailedAllocationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RetryFailedAllocationTests.java
index 41b41dda25f0..89bb4337b2f1 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RetryFailedAllocationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RetryFailedAllocationTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -43,7 +44,9 @@ public void setUp() throws Exception {
         Metadata metadata = Metadata.builder()
             .put(IndexMetadata.builder(INDEX_NAME).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index(INDEX_NAME)).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index(INDEX_NAME))
+            .build();
         clusterState = ClusterState.builder(ClusterName.DEFAULT)
             .metadata(metadata)
             .routingTable(routingTable)
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RoutingNodesIntegrityTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RoutingNodesIntegrityTests.java
index b6100e1844a3..2534e5342146 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RoutingNodesIntegrityTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RoutingNodesIntegrityTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -65,7 +66,7 @@ public void testBalanceAllNodesStarted() {
             .put(IndexMetadata.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(3).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder()
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("test"))
             .addAsNew(metadata.index("test1"))
             .build();
@@ -133,7 +134,7 @@ public void testBalanceIncrementallyStartNodes() {
             .put(IndexMetadata.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(3).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder()
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("test"))
             .addAsNew(metadata.index("test1"))
             .build();
@@ -168,13 +169,13 @@ public void testBalanceIncrementallyStartNodes() {
         assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(STARTED), equalTo(4));
         assertThat(clusterState.getRoutingNodes().node("node3").numberOfShardsWithState(STARTED), equalTo(4));
 
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState("test", STARTED).size(), equalTo(2));
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState("test", STARTED).size(), equalTo(2));
-        assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState("test", STARTED).size(), equalTo(2));
+        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState("test", STARTED).count(), equalTo(2L));
+        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState("test", STARTED).count(), equalTo(2L));
+        assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState("test", STARTED).count(), equalTo(2L));
 
-        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState("test1", STARTED).size(), equalTo(2));
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState("test1", STARTED).size(), equalTo(2));
-        assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState("test1", STARTED).size(), equalTo(2));
+        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState("test1", STARTED).count(), equalTo(2L));
+        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState("test1", STARTED).count(), equalTo(2L));
+        assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState("test1", STARTED).count(), equalTo(2L));
     }
 
     public void testBalanceAllNodesStartedAddIndex() {
@@ -194,7 +195,9 @@ public void testBalanceAllNodesStartedAddIndex() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(3).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -257,9 +260,9 @@ public void testBalanceAllNodesStartedAddIndex() {
         assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), equalTo(2));
         assertThat(routingNodes.node("node3").numberOfShardsWithState(STARTED), equalTo(2));
 
-        assertThat(routingNodes.node("node1").shardsWithState("test", STARTED).size(), equalTo(2));
-        assertThat(routingNodes.node("node2").shardsWithState("test", STARTED).size(), equalTo(2));
-        assertThat(routingNodes.node("node3").shardsWithState("test", STARTED).size(), equalTo(2));
+        assertThat(routingNodes.node("node1").shardsWithState("test", STARTED).count(), equalTo(2L));
+        assertThat(routingNodes.node("node2").shardsWithState("test", STARTED).count(), equalTo(2L));
+        assertThat(routingNodes.node("node3").shardsWithState("test", STARTED).count(), equalTo(2L));
 
         logger.info("Add new index 3 shards 1 replica");
 
@@ -272,7 +275,10 @@ public void testBalanceAllNodesStartedAddIndex() {
                     )
             )
             .build();
-        RoutingTable updatedRoutingTable = RoutingTable.builder(clusterState.routingTable()).addAsNew(metadata.index("test1")).build();
+        RoutingTable updatedRoutingTable = RoutingTable.builder(
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
+            clusterState.routingTable()
+        ).addAsNew(metadata.index("test1")).build();
         clusterState = ClusterState.builder(clusterState).metadata(metadata).routingTable(updatedRoutingTable).build();
         routingNodes = clusterState.getRoutingNodes();
 
@@ -318,9 +324,9 @@ public void testBalanceAllNodesStartedAddIndex() {
         assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), equalTo(4));
         assertThat(routingNodes.node("node3").numberOfShardsWithState(STARTED), equalTo(4));
 
-        assertThat(routingNodes.node("node1").shardsWithState("test1", STARTED).size(), equalTo(2));
-        assertThat(routingNodes.node("node2").shardsWithState("test1", STARTED).size(), equalTo(2));
-        assertThat(routingNodes.node("node3").shardsWithState("test1", STARTED).size(), equalTo(2));
+        assertThat(routingNodes.node("node1").shardsWithState("test1", STARTED).count(), equalTo(2L));
+        assertThat(routingNodes.node("node2").shardsWithState("test1", STARTED).count(), equalTo(2L));
+        assertThat(routingNodes.node("node3").shardsWithState("test1", STARTED).count(), equalTo(2L));
 
         logger.info("kill one node");
         IndexShardRoutingTable indexShardRoutingTable = clusterState.routingTable().index("test").shard(0);
@@ -368,7 +374,7 @@ public void testNodeInterleavedShardIterator() {
             )
             .build();
         final var metadata = Metadata.builder().put(indexMetadata, true).build();
-        final var routingTable = RoutingTable.builder().addAsNew(indexMetadata).build();
+        final var routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(indexMetadata).build();
 
         final var nodeCount = between(numberOfReplicas + 1, 6);
         final var discoveryNodes = DiscoveryNodes.builder();
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java
index e078934adce6..1125df5e18e5 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -26,8 +27,8 @@
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider;
-import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.snapshots.SnapshotShardSizeInfo;
 
@@ -41,6 +42,7 @@
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
 import static org.elasticsearch.cluster.routing.allocation.RoutingNodesUtils.numberOfShardsOfType;
+import static org.elasticsearch.common.settings.ClusterSettings.createBuiltInClusterSettings;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -60,7 +62,9 @@ public void testSameHost() {
             .put(IndexMetadata.builder("test").settings(indexSettings).numberOfShards(2).numberOfReplicas(1))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
         ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
             .routingTable(routingTable)
@@ -151,7 +155,9 @@ public void testSameHostCheckWithExplain() {
             .put(IndexMetadata.builder("test").settings(indexSettings).numberOfShards(2).numberOfReplicas(1))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
         ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
             .routingTable(routingTable)
@@ -200,10 +206,7 @@ public void testSameHostCheckWithExplain() {
         } else {
             final ShardRouting unassignedShard = unassignedShards.get(0);
 
-            final SameShardAllocationDecider decider = new SameShardAllocationDecider(
-                sameHostSetting,
-                new ClusterSettings(sameHostSetting, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
-            );
+            final SameShardAllocationDecider decider = new SameShardAllocationDecider(createBuiltInClusterSettings(sameHostSetting));
 
             final RoutingNode emptyNode = clusterState.getRoutingNodes()
                 .stream()
@@ -232,7 +235,7 @@ public void testSameHostCheckWithExplain() {
             assertThat(
                 decision.getExplanation(),
                 equalTo(
-                    formatted(
+                    Strings.format(
                         """
                             cannot allocate to node [%s] because a copy of this shard is already allocated to node [%s] with the same host \
                             address [%s] and [%s] is [true] which forbids more than one node on each host from holding a copy of this shard\
@@ -290,7 +293,9 @@ public void testSameHostCheckDisabledByAutoExpandReplicas() {
         final ClusterState clusterState = applyStartedShardsUntilNoChange(
             ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metadata(metadata)
-                .routingTable(RoutingTable.builder().addAsNew(metadata.index("test")).build())
+                .routingTable(
+                    RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(metadata.index("test")).build()
+                )
                 .nodes(DiscoveryNodes.builder().add(node1).add(node2))
                 .build(),
             strategy
@@ -300,10 +305,7 @@ public void testSameHostCheckDisabledByAutoExpandReplicas() {
     }
 
     public void testForceAllocatePrimaryOnSameNodeNotAllowed() {
-        SameShardAllocationDecider decider = new SameShardAllocationDecider(
-            Settings.EMPTY,
-            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
-        );
+        SameShardAllocationDecider decider = new SameShardAllocationDecider(createBuiltInClusterSettings());
         ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomIntBetween(2, 4), 1);
         Index index = clusterState.getMetadata().index("idx").getIndex();
         ShardRouting primaryShard = clusterState.routingTable().index(index).shard(0).primaryShard();
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardVersioningTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardVersioningTests.java
index 77578a7a16db..b6d85e477cbd 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardVersioningTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardVersioningTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -44,7 +45,10 @@ public void testSimple() {
             .put(IndexMetadata.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test1")).addAsNew(metadata.index("test2")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test1"))
+            .addAsNew(metadata.index("test2"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsAvailabilityHealthIndicatorServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsAvailabilityHealthIndicatorServiceTests.java
index 4a412010482f..775be5aaa64e 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsAvailabilityHealthIndicatorServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsAvailabilityHealthIndicatorServiceTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.NodesShutdownMetadata;
@@ -23,6 +24,7 @@
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.allocation.ShardsAvailabilityHealthIndicatorService.ShardAllocationStatus;
 import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
@@ -43,7 +45,12 @@
 import org.elasticsearch.health.node.HealthInfo;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.ExecutorNames;
+import org.elasticsearch.indices.SystemDataStreamDescriptor;
+import org.elasticsearch.indices.SystemIndexDescriptor;
+import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.mockito.stubbing.Answer;
 
 import java.util.ArrayList;
@@ -56,7 +63,10 @@
 import java.util.UUID;
 import java.util.stream.Collectors;
 
+import static java.util.Collections.emptyList;
 import static java.util.stream.Collectors.toMap;
+import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createBackingIndex;
+import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_PREFIX;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX;
 import static org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata.Type.RESTART;
@@ -71,7 +81,9 @@
 import static org.elasticsearch.cluster.routing.allocation.ShardsAvailabilityHealthIndicatorService.ACTION_INCREASE_SHARD_LIMIT_INDEX_SETTING;
 import static org.elasticsearch.cluster.routing.allocation.ShardsAvailabilityHealthIndicatorService.ACTION_INCREASE_SHARD_LIMIT_INDEX_SETTING_LOOKUP;
 import static org.elasticsearch.cluster.routing.allocation.ShardsAvailabilityHealthIndicatorService.ACTION_INCREASE_TIER_CAPACITY_LOOKUP;
+import static org.elasticsearch.cluster.routing.allocation.ShardsAvailabilityHealthIndicatorService.ACTION_MIGRATE_TIERS_AWAY_FROM_INCLUDE_DATA;
 import static org.elasticsearch.cluster.routing.allocation.ShardsAvailabilityHealthIndicatorService.ACTION_MIGRATE_TIERS_AWAY_FROM_INCLUDE_DATA_LOOKUP;
+import static org.elasticsearch.cluster.routing.allocation.ShardsAvailabilityHealthIndicatorService.ACTION_MIGRATE_TIERS_AWAY_FROM_REQUIRE_DATA;
 import static org.elasticsearch.cluster.routing.allocation.ShardsAvailabilityHealthIndicatorService.ACTION_MIGRATE_TIERS_AWAY_FROM_REQUIRE_DATA_LOOKUP;
 import static org.elasticsearch.cluster.routing.allocation.ShardsAvailabilityHealthIndicatorService.ACTION_RESTORE_FROM_SNAPSHOT;
 import static org.elasticsearch.cluster.routing.allocation.ShardsAvailabilityHealthIndicatorService.DIAGNOSIS_WAIT_FOR_OR_FIX_DELAYED_SHARDS;
@@ -83,14 +95,18 @@
 import static org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider.CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING;
 import static org.elasticsearch.common.util.CollectionUtils.concatLists;
 import static org.elasticsearch.core.TimeValue.timeValueSeconds;
+import static org.elasticsearch.health.Diagnosis.Resource.Type.FEATURE_STATE;
 import static org.elasticsearch.health.Diagnosis.Resource.Type.INDEX;
 import static org.elasticsearch.health.HealthStatus.GREEN;
 import static org.elasticsearch.health.HealthStatus.RED;
 import static org.elasticsearch.health.HealthStatus.YELLOW;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.emptyCollectionOf;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItems;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -105,7 +121,7 @@ public void testShouldBeGreenWhenAllPrimariesAndReplicasAreStarted() {
             ),
             List.of()
         );
-        var service = createAllocationHealthIndicatorService(clusterState);
+        var service = createShardsAvailabilityIndicatorService(clusterState);
 
         assertThat(
             service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO),
@@ -114,8 +130,8 @@ public void testShouldBeGreenWhenAllPrimariesAndReplicasAreStarted() {
                     GREEN,
                     "This cluster has all shards available.",
                     Map.of("started_primaries", 2, "started_replicas", 1),
-                    Collections.emptyList(),
-                    Collections.emptyList()
+                    emptyList(),
+                    emptyList()
                 )
             )
         );
@@ -135,7 +151,7 @@ public void testShouldBeYellowWhenThereAreUnassignedReplicas() {
             ),
             List.of()
         );
-        var service = createAllocationHealthIndicatorService(clusterState);
+        var service = createShardsAvailabilityIndicatorService(clusterState);
 
         assertThat(
             service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO),
@@ -170,20 +186,20 @@ public void testShouldBeYellowWhenThereAreUnassignedReplicas() {
         );
     }
 
-    public void testShouldBeRedWhenThereAreUnassignedPrimariesAndAssignedReplicas() {
+    public void testShouldBeRedWhenThereAreUnassignedPrimariesAndUnassignedReplicas() {
         var clusterState = createClusterStateWith(
-            List.of(index("red-index", new ShardAllocation(randomNodeId(), UNAVAILABLE), new ShardAllocation(randomNodeId(), AVAILABLE))),
+            List.of(index("red-index", new ShardAllocation(randomNodeId(), UNAVAILABLE), new ShardAllocation(randomNodeId(), UNAVAILABLE))),
             List.of()
         );
-        var service = createAllocationHealthIndicatorService(clusterState);
+        var service = createShardsAvailabilityIndicatorService(clusterState);
 
         assertThat(
             service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO),
             equalTo(
                 createExpectedResult(
                     RED,
-                    "This cluster has 1 unavailable primary shard.",
-                    Map.of("unassigned_primaries", 1, "started_replicas", 1),
+                    "This cluster has 1 unavailable primary shard, 1 unavailable replica shard.",
+                    Map.of("unassigned_primaries", 1, "unassigned_replicas", 1),
                     List.of(
                         new HealthIndicatorImpact(
                             NAME,
@@ -203,7 +219,7 @@ public void testShouldBeRedWhenThereAreUnassignedPrimariesAndAssignedReplicas()
 
     public void testShouldBeRedWhenThereAreUnassignedPrimariesAndNoReplicas() {
         var clusterState = createClusterStateWith(List.of(index("red-index", new ShardAllocation(randomNodeId(), UNAVAILABLE))), List.of());
-        var service = createAllocationHealthIndicatorService(clusterState);
+        var service = createShardsAvailabilityIndicatorService(clusterState);
 
         assertThat(
             service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO),
@@ -234,7 +250,7 @@ public void testShouldBeRedWhenThereAreUnassignedPrimariesAndUnassignedReplicasO
             List.of(index("red-index", new ShardAllocation(randomNodeId(), UNAVAILABLE), new ShardAllocation(randomNodeId(), UNAVAILABLE))),
             List.of()
         );
-        var service = createAllocationHealthIndicatorService(clusterState);
+        var service = createShardsAvailabilityIndicatorService(clusterState);
 
         HealthIndicatorResult result = service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO);
         assertEquals(RED, result.status());
@@ -259,14 +275,14 @@ public void testShouldBeRedWhenThereAreUnassignedPrimariesAndUnassignedReplicasO
         var clusterState = createClusterStateWith(
             indexMetadataList,
             List.of(
-                index("red-index", new ShardAllocation(randomNodeId(), UNAVAILABLE), new ShardAllocation(randomNodeId(), AVAILABLE)),
+                index("red-index", new ShardAllocation(randomNodeId(), UNAVAILABLE)),
                 index("yellow-index-1", new ShardAllocation(randomNodeId(), AVAILABLE), new ShardAllocation(randomNodeId(), UNAVAILABLE)),
                 index("yellow-index-2", new ShardAllocation(randomNodeId(), AVAILABLE), new ShardAllocation(randomNodeId(), UNAVAILABLE))
             ),
             List.of(),
             List.of()
         );
-        var service = createAllocationHealthIndicatorService(clusterState);
+        var service = createShardsAvailabilityIndicatorService(clusterState);
 
         HealthIndicatorResult result = service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO);
         assertEquals(RED, result.status());
@@ -314,7 +330,7 @@ public void testSortByIndexPriority() {
             List.of(),
             List.of()
         );
-        var service = createAllocationHealthIndicatorService(clusterState);
+        var service = createShardsAvailabilityIndicatorService(clusterState);
 
         HealthIndicatorResult result = service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO);
         // index-2 has the higher priority so it ought to be listed first, followed by index-1 then index-3 which have the same priority:
@@ -344,7 +360,7 @@ public void testShouldBeGreenWhenThereAreRestartingReplicas() {
             ),
             List.of(new NodeShutdown("node-0", RESTART, 60))
         );
-        var service = createAllocationHealthIndicatorService(clusterState);
+        var service = createShardsAvailabilityIndicatorService(clusterState);
 
         assertThat(
             service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO),
@@ -353,8 +369,8 @@ public void testShouldBeGreenWhenThereAreRestartingReplicas() {
                     GREEN,
                     "This cluster has 1 restarting replica shard.",
                     Map.of("started_primaries", 1, "restarting_replicas", 1),
-                    Collections.emptyList(),
-                    Collections.emptyList()
+                    emptyList(),
+                    emptyList()
                 )
             )
         );
@@ -365,7 +381,7 @@ public void testShouldBeGreenWhenThereAreNoReplicasExpected() {
             List.of(index("primaries-only-index", new ShardAllocation(randomNodeId(), AVAILABLE))),
             List.of(new NodeShutdown("node-0", RESTART, 60))
         );
-        var service = createAllocationHealthIndicatorService(clusterState);
+        var service = createShardsAvailabilityIndicatorService(clusterState);
 
         assertThat(
             service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO),
@@ -374,8 +390,8 @@ public void testShouldBeGreenWhenThereAreNoReplicasExpected() {
                     GREEN,
                     "This cluster has all shards available.",
                     Map.of("started_primaries", 1),
-                    Collections.emptyList(),
-                    Collections.emptyList()
+                    emptyList(),
+                    emptyList()
                 )
             )
         );
@@ -392,7 +408,7 @@ public void testShouldBeYellowWhenRestartingReplicasReachedAllocationDelay() {
             ),
             List.of(new NodeShutdown("node-0", RESTART, 60))
         );
-        var service = createAllocationHealthIndicatorService(clusterState);
+        var service = createShardsAvailabilityIndicatorService(clusterState);
 
         assertThat(
             service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO),
@@ -427,7 +443,7 @@ public void testShouldBeGreenWhenThereAreInitializingPrimaries() {
             List.of(index("restarting-index", new ShardAllocation("node-0", INITIALIZING))),
             List.of()
         );
-        var service = createAllocationHealthIndicatorService(clusterState);
+        var service = createShardsAvailabilityIndicatorService(clusterState);
 
         assertThat(
             service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO),
@@ -436,8 +452,8 @@ public void testShouldBeGreenWhenThereAreInitializingPrimaries() {
                     GREEN,
                     "This cluster has 1 creating primary shard.",
                     Map.of("creating_primaries", 1),
-                    Collections.emptyList(),
-                    Collections.emptyList()
+                    emptyList(),
+                    emptyList()
                 )
             )
         );
@@ -448,7 +464,7 @@ public void testShouldBeGreenWhenThereAreRestartingPrimaries() {
             List.of(index("restarting-index", new ShardAllocation("node-0", RESTARTING, System.nanoTime()))),
             List.of(new NodeShutdown("node-0", RESTART, 60))
         );
-        var service = createAllocationHealthIndicatorService(clusterState);
+        var service = createShardsAvailabilityIndicatorService(clusterState);
 
         assertThat(
             service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO),
@@ -457,8 +473,8 @@ public void testShouldBeGreenWhenThereAreRestartingPrimaries() {
                     GREEN,
                     "This cluster has 1 restarting primary shard.",
                     Map.of("restarting_primaries", 1),
-                    Collections.emptyList(),
-                    Collections.emptyList()
+                    emptyList(),
+                    emptyList()
                 )
             )
         );
@@ -474,7 +490,7 @@ public void testShouldBeRedWhenRestartingPrimariesReachedAllocationDelayAndNoRep
             ),
             List.of(new NodeShutdown("node-0", RESTART, 60))
         );
-        var service = createAllocationHealthIndicatorService(clusterState);
+        var service = createShardsAvailabilityIndicatorService(clusterState);
 
         assertThat(
             service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO),
@@ -503,7 +519,7 @@ public void testShouldBeRedWhenRestartingPrimariesReachedAllocationDelayAndNoRep
         );
     }
 
-    public void testUserActionsNotGeneratedWhenNotDrillingDown() {
+    public void testDiagnosisNotGeneratedWhenNotDrillingDown() {
         // Index definition, 1 primary no replicas
         IndexMetadata indexMetadata = IndexMetadata.builder("red-index")
             .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build())
@@ -519,7 +535,7 @@ public void testUserActionsNotGeneratedWhenNotDrillingDown() {
             List.of()
         );
 
-        var service = createAllocationHealthIndicatorService(clusterState);
+        var service = createShardsAvailabilityIndicatorService(clusterState);
 
         assertThat(
             service.calculate(false, HealthInfo.EMPTY_HEALTH_INFO),
@@ -555,13 +571,156 @@ public void testDiagnoseRestoreIndexAfterDataLoss() {
             new ShardAllocation(randomNodeId(), UNAVAILABLE, noShardCopy())
         );
 
-        var service = createAllocationHealthIndicatorService();
+        var service = createShardsAvailabilityIndicatorService();
         List<Diagnosis.Definition> definitions = service.diagnoseUnassignedShardRouting(shardRouting, ClusterState.EMPTY_STATE);
 
         assertThat(definitions, hasSize(1));
         assertThat(definitions, contains(ACTION_RESTORE_FROM_SNAPSHOT));
     }
 
+    public void testRestoreFromSnapshotReportsFeatureStates() {
+        // this test adds a mix of regular and system indices and data streams
+        // we'll test the `shards_availability` indicator correctly reports the
+        // affected feature states and indices
+
+        IndexMetadata featureIndex = IndexMetadata.builder(".feature-index")
+            .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build())
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .build();
+
+        IndexMetadata regularIndex = IndexMetadata.builder("regular-index")
+            .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build())
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .build();
+
+        String featureDataStreamName = ".test-ds-feature";
+        IndexMetadata backingIndex = createBackingIndex(featureDataStreamName, 1).build();
+
+        ShardRouting featureIndexRouting = createShardRouting(
+            new ShardId(featureIndex.getIndex(), 0),
+            true,
+            new ShardAllocation(randomNodeId(), UNAVAILABLE, noShardCopy())
+        );
+
+        ShardRouting regularIndexRouting = createShardRouting(
+            new ShardId(regularIndex.getIndex(), 0),
+            true,
+            new ShardAllocation(randomNodeId(), UNAVAILABLE, noShardCopy())
+        );
+
+        ShardRouting backingIndexRouting = createShardRouting(
+            new ShardId(backingIndex.getIndex(), 0),
+            true,
+            new ShardAllocation(randomNodeId(), UNAVAILABLE, noShardCopy())
+        );
+
+        var clusterState = createClusterStateWith(
+            List.of(featureIndex, regularIndex, backingIndex),
+            List.of(
+                IndexRoutingTable.builder(featureIndex.getIndex()).addShard(featureIndexRouting).build(),
+                IndexRoutingTable.builder(regularIndex.getIndex()).addShard(regularIndexRouting).build(),
+                IndexRoutingTable.builder(backingIndex.getIndex()).addShard(backingIndexRouting).build()
+            ),
+            List.of(),
+            List.of()
+        );
+
+        // add the data stream to the cluster state
+        Metadata.Builder mdBuilder = Metadata.builder(clusterState.metadata())
+            .put(newInstance(featureDataStreamName, List.of(backingIndex.getIndex())));
+        ClusterState state = ClusterState.builder(clusterState).metadata(mdBuilder).build();
+
+        var service = createAllocationHealthIndicatorService(
+            Settings.EMPTY,
+            state,
+            Map.of(),
+            getSystemIndices(featureDataStreamName, ".test-ds-*", ".feature-*")
+        );
+        HealthIndicatorResult result = service.calculate(true, HealthInfo.EMPTY_HEALTH_INFO);
+
+        assertThat(result.status(), is(HealthStatus.RED));
+        assertThat(result.diagnosisList().size(), is(1));
+        Diagnosis diagnosis = result.diagnosisList().get(0);
+        List<Diagnosis.Resource> affectedResources = diagnosis.affectedResources();
+        assertThat("expecting we report a resource of type INDEX and one of type FEATURE_STATE", affectedResources.size(), is(2));
+        for (Diagnosis.Resource resource : affectedResources) {
+            if (resource.getType() == INDEX) {
+                assertThat(resource.getValues(), hasItems("regular-index"));
+            } else {
+                assertThat(resource.getType(), is(FEATURE_STATE));
+                assertThat(resource.getValues(), hasItems("feature-with-system-data-stream", "feature-with-system-index"));
+            }
+        }
+    }
+
+    public void testGetRestoreFromSnapshotAffectedResources() {
+        String featureDataStreamName = ".test-ds-feature";
+        IndexMetadata backingIndex = createBackingIndex(featureDataStreamName, 1).build();
+
+        List<IndexMetadata> indexMetadataList = List.of(
+            IndexMetadata.builder(".feature-index")
+                .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build())
+                .numberOfShards(1)
+                .numberOfReplicas(0)
+                .build(),
+            IndexMetadata.builder("regular-index")
+                .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build())
+                .numberOfShards(1)
+                .numberOfReplicas(0)
+                .build(),
+            backingIndex
+        );
+
+        Metadata.Builder metadataBuilder = Metadata.builder();
+        Map<String, IndexMetadata> indexMetadataMap = new HashMap<>();
+        for (IndexMetadata indexMetadata : indexMetadataList) {
+            indexMetadataMap.put(indexMetadata.getIndex().getName(), indexMetadata);
+        }
+        metadataBuilder.indices(indexMetadataMap);
+        metadataBuilder.put(newInstance(featureDataStreamName, List.of(backingIndex.getIndex())));
+        Metadata metadata = metadataBuilder.build();
+        {
+            List<Diagnosis.Resource> affectedResources = ShardAllocationStatus.getRestoreFromSnapshotAffectedResources(
+                metadata,
+                getSystemIndices(featureDataStreamName, ".test-ds-*", ".feature-*"),
+                Set.of(backingIndex.getIndex().getName(), ".feature-index", "regular-index"),
+                10
+            );
+
+            assertThat(affectedResources.size(), is(2));
+            for (Diagnosis.Resource resource : affectedResources) {
+                if (resource.getType() == INDEX) {
+                    assertThat(resource.getValues(), hasItems("regular-index"));
+                } else {
+                    assertThat(resource.getType(), is(FEATURE_STATE));
+                    assertThat(resource.getValues(), hasItems("feature-with-system-data-stream", "feature-with-system-index"));
+                }
+            }
+        }
+
+        {
+            List<Diagnosis.Resource> affectedResources = ShardAllocationStatus.getRestoreFromSnapshotAffectedResources(
+                metadata,
+                getSystemIndices(featureDataStreamName, ".test-ds-*", ".feature-*"),
+                Set.of(backingIndex.getIndex().getName(), ".feature-index", "regular-index"),
+                0
+            );
+
+            assertThat(affectedResources.size(), is(2));
+            for (Diagnosis.Resource resource : affectedResources) {
+                if (resource.getType() == INDEX) {
+                    assertThat(resource.getValues(), emptyCollectionOf(String.class));
+                } else {
+                    assertThat(resource.getType(), is(FEATURE_STATE));
+                    assertThat(resource.getValues(), emptyCollectionOf(String.class));
+                }
+            }
+        }
+
+    }
+
     public void testDiagnoseUnknownAllocationDeciderIssue() {
         // Index definition, 1 primary no replicas
         IndexMetadata indexMetadata = IndexMetadata.builder("red-index")
@@ -601,7 +760,7 @@ public void testDiagnoseUnknownAllocationDeciderIssue() {
                 MoveDecision.NOT_TAKEN
             )
         );
-        var service = createAllocationHealthIndicatorService(clusterState, decisionMap);
+        var service = createShardsAvailabilityIndicatorService(clusterState, decisionMap);
 
         // Get the list of user actions that are generated for this unassigned index shard
         ShardRouting shardRouting = clusterState.routingTable().index(indexMetadata.getIndex()).shard(0).primaryShard();
@@ -624,7 +783,7 @@ public void testDiagnoseEnableIndexAllocation() {
             .numberOfReplicas(0)
             .build();
 
-        var service = createAllocationHealthIndicatorService();
+        var service = createShardsAvailabilityIndicatorService();
 
         // Get the list of user actions that are generated for this unassigned index shard
         List<Diagnosis.Definition> actions = service.checkIsAllocationDisabled(
@@ -652,7 +811,7 @@ public void testDiagnoseEnableClusterAllocation() {
             .build();
 
         // Disallow allocations in cluster settings
-        var service = createAllocationHealthIndicatorService(
+        var service = createShardsAvailabilityIndicatorService(
             Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "none").build(),
             ClusterState.EMPTY_STATE,
             Map.of()
@@ -689,7 +848,7 @@ public void testDiagnoseEnableRoutingAllocation() {
             .build();
 
         // Disallow allocations in cluster settings
-        var service = createAllocationHealthIndicatorService(
+        var service = createShardsAvailabilityIndicatorService(
             Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "none").build(),
             ClusterState.EMPTY_STATE,
             Map.of()
@@ -726,7 +885,7 @@ public void testDiagnoseEnableDataTiers() {
             .numberOfReplicas(0)
             .build();
 
-        var service = createAllocationHealthIndicatorService();
+        var service = createShardsAvailabilityIndicatorService();
 
         // Get the list of user actions that are generated for this unassigned index shard
         List<Diagnosis.Definition> actions = service.checkDataTierRelatedIssues(
@@ -785,7 +944,7 @@ public void testDiagnoseIncreaseShardLimitIndexSettingInTier() {
             List.of(),
             List.of(hotNode)
         );
-        var service = createAllocationHealthIndicatorService();
+        var service = createShardsAvailabilityIndicatorService();
 
         // Get the list of user actions that are generated for this unassigned index shard
         List<Diagnosis.Definition> actions = service.checkDataTierRelatedIssues(
@@ -846,7 +1005,7 @@ public void testDiagnoseIncreaseShardLimitClusterSettingInTier() {
         );
 
         // Configure at most 1 shard per node
-        var service = createAllocationHealthIndicatorService(
+        var service = createShardsAvailabilityIndicatorService(
             Settings.builder().put(CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), 1).build(),
             clusterState,
             Map.of()
@@ -910,7 +1069,7 @@ public void testDiagnoseIncreaseShardLimitIndexSettingInGeneral() {
             List.of(),
             List.of(dataNode)
         );
-        var service = createAllocationHealthIndicatorService();
+        var service = createShardsAvailabilityIndicatorService();
 
         // Get the list of user actions that are generated for this unassigned index shard
         List<Diagnosis.Definition> actions = service.checkDataTierRelatedIssues(
@@ -971,7 +1130,7 @@ public void testDiagnoseIncreaseShardLimitClusterSettingInGeneral() {
         );
 
         // Configure at most 1 shard per node
-        var service = createAllocationHealthIndicatorService(
+        var service = createShardsAvailabilityIndicatorService(
             Settings.builder().put(CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), 1).build(),
             clusterState,
             Map.of()
@@ -1010,7 +1169,7 @@ public void testDiagnoseMigrateDataRequiredToDataTiers() {
             .numberOfReplicas(0)
             .build();
 
-        var service = createAllocationHealthIndicatorService();
+        var service = createShardsAvailabilityIndicatorService();
 
         // Get the list of user actions that are generated for this unassigned index shard
         List<Diagnosis.Definition> actions = service.checkDataTierRelatedIssues(
@@ -1046,7 +1205,7 @@ public void testDiagnoseMigrateDataIncludedToDataTiers() {
             .numberOfReplicas(0)
             .build();
 
-        var service = createAllocationHealthIndicatorService();
+        var service = createShardsAvailabilityIndicatorService();
 
         // Get the list of user actions that are generated for this unassigned index shard
         List<Diagnosis.Definition> actions = service.checkDataTierRelatedIssues(
@@ -1081,7 +1240,7 @@ public void testDiagnoseOtherFilteringIssue() {
             .numberOfReplicas(0)
             .build();
 
-        var service = createAllocationHealthIndicatorService();
+        var service = createShardsAvailabilityIndicatorService();
 
         // Get the list of user actions that are generated for this unassigned index shard
         List<Diagnosis.Definition> actions = service.checkDataTierRelatedIssues(
@@ -1116,7 +1275,7 @@ public void testDiagnoseIncreaseTierCapacity() {
             .numberOfReplicas(0)
             .build();
 
-        var service = createAllocationHealthIndicatorService();
+        var service = createShardsAvailabilityIndicatorService();
 
         // Get the list of user actions that are generated for this unassigned index shard
         List<Diagnosis.Definition> actions = service.checkDataTierRelatedIssues(
@@ -1156,7 +1315,7 @@ public void testDiagnoseIncreaseNodeCapacity() {
             .numberOfReplicas(0)
             .build();
 
-        var service = createAllocationHealthIndicatorService();
+        var service = createShardsAvailabilityIndicatorService();
 
         // Get the list of user actions that are generated for this unassigned index shard
         List<Diagnosis.Definition> actions = service.checkDataTierRelatedIssues(
@@ -1183,6 +1342,195 @@ public void testDiagnoseIncreaseNodeCapacity() {
         assertThat(actions, contains(ACTION_INCREASE_NODE_CAPACITY));
     }
 
+    public void testLimitNumberOfAffectedResources() {
+        var clusterState = createClusterStateWith(
+            List.of(
+                index("red-index1", new ShardAllocation(randomNodeId(), UNAVAILABLE)),
+                index("red-index2", new ShardAllocation(randomNodeId(), UNAVAILABLE)),
+                index("red-index3", new ShardAllocation(randomNodeId(), UNAVAILABLE)),
+                index("red-index4", new ShardAllocation(randomNodeId(), UNAVAILABLE)),
+                index("red-index5", new ShardAllocation(randomNodeId(), UNAVAILABLE))
+            ),
+            List.of()
+        );
+        var service = createShardsAvailabilityIndicatorService(clusterState);
+
+        {
+            // assert the full result to check that details, impacts, and symptoms use the correct count of affected indices (5)
+            assertThat(
+                service.calculate(true, 2, HealthInfo.EMPTY_HEALTH_INFO),
+                equalTo(
+                    createExpectedResult(
+                        RED,
+                        "This cluster has 5 unavailable primary shards.",
+                        Map.of("unassigned_primaries", 5),
+                        List.of(
+                            new HealthIndicatorImpact(
+                                NAME,
+                                ShardsAvailabilityHealthIndicatorService.PRIMARY_UNASSIGNED_IMPACT_ID,
+                                1,
+                                "Cannot add data to 5 indices [red-index1, red-index2, red-index3, red-index4, red-index5]. Searches might "
+                                    + "return incomplete results.",
+                                List.of(ImpactArea.INGEST, ImpactArea.SEARCH)
+                            )
+                        ),
+                        List.of(
+                            new Diagnosis(
+                                ACTION_CHECK_ALLOCATION_EXPLAIN_API,
+                                List.of(new Diagnosis.Resource(INDEX, List.of("red-index1", "red-index2")))
+                            )
+                        )
+                    )
+                )
+            );
+        }
+
+        {
+            // larger number of affected resources
+            assertThat(
+                service.calculate(true, 2_000, HealthInfo.EMPTY_HEALTH_INFO).diagnosisList(),
+                equalTo(
+                    List.of(
+                        new Diagnosis(
+                            ACTION_CHECK_ALLOCATION_EXPLAIN_API,
+                            List.of(
+                                new Diagnosis.Resource(INDEX, List.of("red-index1", "red-index2", "red-index3", "red-index4", "red-index5"))
+                            )
+                        )
+                    )
+                )
+            );
+
+        }
+
+        {
+            // 0 affected resources
+            assertThat(
+                service.calculate(true, 0, HealthInfo.EMPTY_HEALTH_INFO).diagnosisList(),
+                equalTo(List.of(new Diagnosis(ACTION_CHECK_ALLOCATION_EXPLAIN_API, List.of(new Diagnosis.Resource(INDEX, List.of())))))
+            );
+        }
+    }
+
+    /**
+     * Creates the {@link SystemIndices} with one standalone system index and a system data stream
+     */
+    private SystemIndices getSystemIndices(
+        String featureDataStreamName,
+        String systemDataStreamPattern,
+        String standaloneSystemIndexPattern
+    ) {
+        return new SystemIndices(
+            List.of(
+                new SystemIndices.Feature(
+                    "feature-with-system-index",
+                    "testing",
+                    List.of(new SystemIndexDescriptor(standaloneSystemIndexPattern, "feature with index"))
+                ),
+                new SystemIndices.Feature(
+                    "feature-with-system-data-stream",
+                    "feature with data stream",
+                    List.of(),
+                    List.of(
+                        new SystemDataStreamDescriptor(
+                            featureDataStreamName,
+                            "description",
+                            SystemDataStreamDescriptor.Type.EXTERNAL,
+                            new ComposableIndexTemplate(
+                                List.of(systemDataStreamPattern),
+                                null,
+                                null,
+                                null,
+                                null,
+                                null,
+                                new ComposableIndexTemplate.DataStreamTemplate()
+                            ),
+                            Map.of(),
+                            List.of("test"),
+                            new ExecutorNames(
+                                ThreadPool.Names.SYSTEM_CRITICAL_READ,
+                                ThreadPool.Names.SYSTEM_READ,
+                                ThreadPool.Names.SYSTEM_WRITE
+                            )
+                        )
+                    )
+                )
+            )
+        );
+    }
+
+    // We expose the indicator name and the diagnoses in the x-pack usage API. In order to index them properly in a telemetry index
+    // they need to be declared in the health-api-indexer.edn in the telemetry repository.
+    public void testMappedFieldsForTelemetry() {
+        assertThat(ShardsAvailabilityHealthIndicatorService.NAME, equalTo("shards_availability"));
+        assertThat(
+            ACTION_RESTORE_FROM_SNAPSHOT.getUniqueId(),
+            equalTo("elasticsearch:health:shards_availability:diagnosis:restore_from_snapshot")
+        );
+        assertThat(
+            ACTION_CHECK_ALLOCATION_EXPLAIN_API.getUniqueId(),
+            equalTo("elasticsearch:health:shards_availability:diagnosis:explain_allocations")
+        );
+        assertThat(
+            DIAGNOSIS_WAIT_FOR_OR_FIX_DELAYED_SHARDS.getUniqueId(),
+            equalTo("elasticsearch:health:shards_availability:diagnosis:delayed_shard_allocations")
+        );
+        assertThat(
+            ACTION_ENABLE_INDEX_ROUTING_ALLOCATION.getUniqueId(),
+            equalTo("elasticsearch:health:shards_availability:diagnosis:enable_index_allocations")
+        );
+        assertThat(
+            ACTION_ENABLE_CLUSTER_ROUTING_ALLOCATION.getUniqueId(),
+            equalTo("elasticsearch:health:shards_availability:diagnosis:enable_cluster_allocations")
+        );
+        assertThat(
+            ACTION_INCREASE_SHARD_LIMIT_CLUSTER_SETTING.getUniqueId(),
+            equalTo("elasticsearch:health:shards_availability:diagnosis:increase_shard_limit_cluster_setting")
+        );
+        assertThat(
+            ACTION_INCREASE_SHARD_LIMIT_INDEX_SETTING.getUniqueId(),
+            equalTo("elasticsearch:health:shards_availability:diagnosis:increase_shard_limit_index_setting")
+        );
+        assertThat(
+            ACTION_MIGRATE_TIERS_AWAY_FROM_REQUIRE_DATA.getUniqueId(),
+            equalTo("elasticsearch:health:shards_availability:diagnosis:migrate_data_tiers_require_data")
+        );
+        assertThat(
+            ACTION_MIGRATE_TIERS_AWAY_FROM_INCLUDE_DATA.getUniqueId(),
+            equalTo("elasticsearch:health:shards_availability:diagnosis:migrate_data_tiers_include_data")
+        );
+        assertThat(
+            ACTION_INCREASE_NODE_CAPACITY.getUniqueId(),
+            equalTo("elasticsearch:health:shards_availability:diagnosis:increase_node_capacity_for_allocations")
+        );
+        for (String tier : List.of("data_content", "data_hot", "data_warm", "data_cold", "data_frozen")) {
+            assertThat(
+                ACTION_ENABLE_TIERS_LOOKUP.get(tier).getUniqueId(),
+                equalTo("elasticsearch:health:shards_availability:diagnosis:enable_data_tiers:tier:" + tier)
+            );
+            assertThat(
+                ACTION_INCREASE_SHARD_LIMIT_INDEX_SETTING_LOOKUP.get(tier).getUniqueId(),
+                equalTo("elasticsearch:health:shards_availability:diagnosis:increase_shard_limit_index_setting:tier:" + tier)
+            );
+            assertThat(
+                ACTION_INCREASE_SHARD_LIMIT_CLUSTER_SETTING_LOOKUP.get(tier).getUniqueId(),
+                equalTo("elasticsearch:health:shards_availability:diagnosis:increase_shard_limit_cluster_setting:tier:" + tier)
+            );
+            assertThat(
+                ACTION_MIGRATE_TIERS_AWAY_FROM_REQUIRE_DATA_LOOKUP.get(tier).getUniqueId(),
+                equalTo("elasticsearch:health:shards_availability:diagnosis:migrate_data_tiers_require_data:tier:" + tier)
+            );
+            assertThat(
+                ACTION_MIGRATE_TIERS_AWAY_FROM_INCLUDE_DATA_LOOKUP.get(tier).getUniqueId(),
+                equalTo("elasticsearch:health:shards_availability:diagnosis:migrate_data_tiers_include_data:tier:" + tier)
+            );
+            assertThat(
+                ACTION_INCREASE_TIER_CAPACITY_LOOKUP.get(tier).getUniqueId(),
+                equalTo("elasticsearch:health:shards_availability:diagnosis:increase_tier_capacity_for_allocations:tier:" + tier)
+            );
+        }
+    }
+
     private HealthIndicatorResult createExpectedResult(
         HealthStatus status,
         String symptom,
@@ -1201,7 +1549,7 @@ private HealthIndicatorResult createExpectedResult(
     }
 
     private HealthIndicatorResult createExpectedTruncatedResult(HealthStatus status, String symptom, List<HealthIndicatorImpact> impacts) {
-        return new HealthIndicatorResult(NAME, status, symptom, HealthIndicatorDetails.EMPTY, impacts, Collections.emptyList());
+        return new HealthIndicatorResult(NAME, status, symptom, HealthIndicatorDetails.EMPTY, impacts, emptyList());
     }
 
     private static ClusterState createClusterStateWith(List<IndexRoutingTable> indexRoutes, List<NodeShutdown> nodeShutdowns) {
@@ -1334,7 +1682,8 @@ private static ShardRouting createShardRouting(ShardId shardId, boolean primary,
             shardId,
             primary,
             getSource(primary, allocation.state),
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null),
+            ShardRouting.Role.DEFAULT
         );
         if (allocation.state == INITIALIZING) {
             return routing;
@@ -1407,7 +1756,7 @@ private static String randomNodeId() {
 
     private static UnassignedInfo noShardCopy() {
         return new UnassignedInfo(
-            UnassignedInfo.Reason.NODE_LEFT,
+            randomBoolean() ? UnassignedInfo.Reason.NODE_LEFT : UnassignedInfo.Reason.CLUSTER_RECOVERED,
             null,
             null,
             0,
@@ -1452,25 +1801,34 @@ private static UnassignedInfo decidersNo() {
 
     private record ShardRoutingKey(String index, int shard, boolean primary) {}
 
-    private static ShardsAvailabilityHealthIndicatorService createAllocationHealthIndicatorService() {
-        return createAllocationHealthIndicatorService(ClusterState.EMPTY_STATE, Collections.emptyMap());
+    private static ShardsAvailabilityHealthIndicatorService createShardsAvailabilityIndicatorService() {
+        return createShardsAvailabilityIndicatorService(ClusterState.EMPTY_STATE, Collections.emptyMap());
     }
 
-    private static ShardsAvailabilityHealthIndicatorService createAllocationHealthIndicatorService(ClusterState clusterState) {
-        return createAllocationHealthIndicatorService(clusterState, Collections.emptyMap());
+    private static ShardsAvailabilityHealthIndicatorService createShardsAvailabilityIndicatorService(ClusterState clusterState) {
+        return createShardsAvailabilityIndicatorService(clusterState, Collections.emptyMap());
     }
 
-    private static ShardsAvailabilityHealthIndicatorService createAllocationHealthIndicatorService(
+    private static ShardsAvailabilityHealthIndicatorService createShardsAvailabilityIndicatorService(
         ClusterState clusterState,
         final Map<ShardRoutingKey, ShardAllocationDecision> decisions
     ) {
-        return createAllocationHealthIndicatorService(Settings.EMPTY, clusterState, decisions);
+        return createAllocationHealthIndicatorService(Settings.EMPTY, clusterState, decisions, new SystemIndices(List.of()));
     }
 
-    private static ShardsAvailabilityHealthIndicatorService createAllocationHealthIndicatorService(
+    private static ShardsAvailabilityHealthIndicatorService createShardsAvailabilityIndicatorService(
         Settings nodeSettings,
         ClusterState clusterState,
         final Map<ShardRoutingKey, ShardAllocationDecision> decisions
+    ) {
+        return createAllocationHealthIndicatorService(nodeSettings, clusterState, decisions, new SystemIndices(List.of()));
+    }
+
+    private static ShardsAvailabilityHealthIndicatorService createAllocationHealthIndicatorService(
+        Settings nodeSettings,
+        ClusterState clusterState,
+        final Map<ShardRoutingKey, ShardAllocationDecision> decisions,
+        SystemIndices systemIndices
     ) {
         var clusterService = mock(ClusterService.class);
         when(clusterService.state()).thenReturn(clusterState);
@@ -1482,6 +1840,6 @@ private static ShardsAvailabilityHealthIndicatorService createAllocationHealthIn
             var key = new ShardRoutingKey(shardRouting.getIndexName(), shardRouting.getId(), shardRouting.primary());
             return decisions.getOrDefault(key, ShardAllocationDecision.NOT_TAKEN);
         });
-        return new ShardsAvailabilityHealthIndicatorService(clusterService, allocationService);
+        return new ShardsAvailabilityHealthIndicatorService(clusterService, allocationService, systemIndices);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java
index ae5b6c38a136..2186702d2628 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -49,7 +50,9 @@ public void testIndexLevelShardsLimitAllocate() {
             )
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -96,7 +99,9 @@ public void testClusterLevelShardsLimitAllocate() {
             )
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -162,7 +167,9 @@ public void testIndexLevelShardsLimitRemain() {
             )
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -186,7 +193,10 @@ public void testIndexLevelShardsLimitRemain() {
                     )
             )
             .build();
-        RoutingTable updatedRoutingTable = RoutingTable.builder(clusterState.routingTable()).addAsNew(metadata.index("test1")).build();
+        RoutingTable updatedRoutingTable = RoutingTable.builder(
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
+            clusterState.routingTable()
+        ).addAsNew(metadata.index("test1")).build();
 
         clusterState = ClusterState.builder(clusterState).metadata(metadata).routingTable(updatedRoutingTable).build();
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardNoReplicasRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardNoReplicasRoutingTests.java
index d646ef7750c6..8635436574f6 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardNoReplicasRoutingTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardNoReplicasRoutingTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -55,7 +56,9 @@ public void testSingleIndexStartedShard() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -155,7 +158,8 @@ public void testSingleIndexShardFailed() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
             .build();
 
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder().addAsNew(metadata.index("test"));
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"));
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -182,7 +186,9 @@ public void testSingleIndexShardFailed() {
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
         newState = strategy.applyFailedShards(
             clusterState,
-            List.of(new FailedShard(routingNodes.node("node1").shardsWithState(INITIALIZING).get(0), null, null, randomBoolean())),
+            List.of(
+                new FailedShard(routingNodes.node("node1").shardsWithState(INITIALIZING).findFirst().get(), null, null, randomBoolean())
+            ),
             List.of()
         );
         assertThat(newState, not(equalTo(clusterState)));
@@ -214,7 +220,7 @@ public void testMultiIndexEvenDistribution() {
         }
         Metadata metadata = metadataBuilder.build();
 
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         for (int i = 0; i < numberOfIndices; i++) {
             routingTableBuilder.addAsNew(metadata.index("test" + i));
         }
@@ -325,7 +331,7 @@ public void testMultiIndexUnevenNodes() {
         }
         Metadata metadata = metadataBuilder.build();
 
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         for (int i = 0; i < numberOfIndices; i++) {
             routingTableBuilder.addAsNew(metadata.index("test" + i));
         }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardOneReplicaRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardOneReplicaRoutingTests.java
index 75a99105c79b..8a4cb4ee0862 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardOneReplicaRoutingTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardOneReplicaRoutingTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -42,7 +43,9 @@ public void testSingleIndexFirstStartPrimaryThenBackups() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/TenShardsOneReplicaRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/TenShardsOneReplicaRoutingTests.java
index f03683dd3675..a1ef61c048f2 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/TenShardsOneReplicaRoutingTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/TenShardsOneReplicaRoutingTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -53,7 +54,9 @@ public void testSingleIndexFirstStartPrimaryThenBackups() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(10).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -143,10 +146,16 @@ public void testSingleIndexFirstStartPrimaryThenBackups() {
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(clusterState.routingTable().index("test").size(), equalTo(10));
-        assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED, RELOCATING), equalTo(10));
         assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED), lessThan(10));
-        assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED, RELOCATING), equalTo(10));
+        assertThat(
+            routingNodes.node("node1").numberOfShardsWithState(STARTED) + routingNodes.node("node1").numberOfShardsWithState(RELOCATING),
+            equalTo(10)
+        );
         assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), lessThan(10));
+        assertThat(
+            routingNodes.node("node2").numberOfShardsWithState(STARTED) + routingNodes.node("node2").numberOfShardsWithState(RELOCATING),
+            equalTo(10)
+        );
         assertThat(routingNodes.node("node3").numberOfShardsWithState(INITIALIZING), equalTo(6));
 
         logger.info("Start the shards on node 3");
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java
index f7bb70b037f2..05ab419c2b99 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java
@@ -14,7 +14,9 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.EmptyClusterInfoService;
 import org.elasticsearch.cluster.RestoreInProgress;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -69,6 +71,7 @@ public void testPrimaryRecoveryThrottling() {
                 .put("cluster.routing.allocation.node_initial_primaries_recoveries", 3)
                 .build(),
             gatewayAllocator,
+            EmptyClusterInfoService.INSTANCE,
             snapshotsInfoService
         );
 
@@ -127,6 +130,7 @@ public void testReplicaAndPrimaryRecoveryThrottling() {
                 .put("cluster.routing.allocation.node_initial_primaries_recoveries", 3)
                 .build(),
             gatewayAllocator,
+            EmptyClusterInfoService.INSTANCE,
             snapshotsInfoService
         );
 
@@ -190,7 +194,12 @@ public void testThrottleIncomingAndOutgoing() {
             .put("cluster.routing.allocation.node_initial_primaries_recoveries", 5)
             .put("cluster.routing.allocation.cluster_concurrent_rebalance", 5)
             .build();
-        AllocationService strategy = createAllocationService(settings, gatewayAllocator, snapshotsInfoService);
+        AllocationService strategy = createAllocationService(
+            settings,
+            gatewayAllocator,
+            EmptyClusterInfoService.INSTANCE,
+            snapshotsInfoService
+        );
         logger.info("Building initial routing table");
 
         Metadata metadata = Metadata.builder()
@@ -249,6 +258,7 @@ public void testOutgoingThrottlesAllocation() {
         AllocationService strategy = createAllocationService(
             Settings.builder().put("cluster.routing.allocation.node_concurrent_outgoing_recoveries", 1).build(),
             gatewayAllocator,
+            EmptyClusterInfoService.INSTANCE,
             snapshotsInfoService
         );
 
@@ -348,7 +358,7 @@ private ClusterState createRecoveryStateAndInitializeAllocations(
     ) {
         DiscoveryNode node1 = newNode("node1");
         Metadata.Builder metadataBuilder = Metadata.builder(metadata);
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         Snapshot snapshot = new Snapshot("repo", new SnapshotId("snap", "randomId"));
         Set<String> snapshotIndices = new HashSet<>();
         String restoreUUID = UUIDs.randomBase64UUID();
@@ -444,7 +454,8 @@ private void addInSyncAllocationIds(
                 new ShardId(index, shard),
                 primary,
                 primary ? RecoverySource.EmptyStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE,
-                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "test")
+                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "test"),
+                ShardRouting.Role.DEFAULT
             );
             ShardRouting started = ShardRoutingHelper.moveToStarted(ShardRoutingHelper.initialize(unassigned, node1.getId()));
             indexMetadata.putInSyncAllocationIds(shard, Collections.singleton(started.allocationId().getId()));
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/TrackFailedAllocationNodesTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/TrackFailedAllocationNodesTests.java
index 90dd67e3965d..ba7ca84b0639 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/TrackFailedAllocationNodesTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/TrackFailedAllocationNodesTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -45,7 +46,7 @@ public void testTrackFailedNodes() {
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .nodes(discoNodes)
             .metadata(metadata)
-            .routingTable(RoutingTable.builder().addAsNew(metadata.index("idx")).build())
+            .routingTable(RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(metadata.index("idx")).build())
             .build();
         clusterState = allocationService.reroute(clusterState, "reroute", ActionListener.noop());
         Set<String> failedNodeIds = new HashSet<>();
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/UpdateNumberOfReplicasTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/UpdateNumberOfReplicasTests.java
index fceeb228afc6..16aaef88a235 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/UpdateNumberOfReplicasTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/UpdateNumberOfReplicasTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -42,7 +43,9 @@ public void testUpdateNumberOfReplicas() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -85,7 +88,10 @@ public void testUpdateNumberOfReplicas() {
 
         logger.info("add another replica");
         final String[] indices = { "test" };
-        RoutingTable updatedRoutingTable = RoutingTable.builder(clusterState.routingTable()).updateNumberOfReplicas(2, indices).build();
+        RoutingTable updatedRoutingTable = RoutingTable.builder(
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
+            clusterState.routingTable()
+        ).updateNumberOfReplicas(2, indices).build();
         metadata = Metadata.builder(clusterState.metadata()).updateNumberOfReplicas(2, indices).build();
         clusterState = ClusterState.builder(clusterState).routingTable(updatedRoutingTable).metadata(metadata).build();
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocatorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocatorTests.java
index f7fa7cfc88f1..e3cf4a34a3e1 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocatorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocatorTests.java
@@ -12,13 +12,14 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
 import org.elasticsearch.cluster.ClusterInfo;
+import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.AllocationId;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
@@ -31,33 +32,37 @@
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocateUnassignedDecision;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.snapshots.SnapshotShardSizeInfo;
+import org.elasticsearch.test.gateway.TestGatewayAllocator;
 
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.OptionalDouble;
 import java.util.Set;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-
+import java.util.stream.Collector;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+import static java.util.stream.Collectors.mapping;
+import static java.util.stream.Collectors.summingDouble;
+import static java.util.stream.Collectors.summingLong;
+import static java.util.stream.Collectors.toMap;
+import static java.util.stream.Collectors.toSet;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
 import static org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator.Balancer.getIndexDiskUsageInBytes;
-import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.everyItem;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
@@ -73,7 +78,10 @@ public void testDecideShardAllocation() {
         Metadata metadata = Metadata.builder(clusterState.metadata())
             .put(IndexMetadata.builder(index).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
             .build();
-        RoutingTable initialRoutingTable = RoutingTable.builder(clusterState.routingTable()).addAsNew(metadata.index(index)).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
+            clusterState.routingTable()
+        ).addAsNew(metadata.index(index)).build();
         clusterState = ClusterState.builder(clusterState).metadata(metadata).routingTable(initialRoutingTable).build();
 
         ShardRouting shard = clusterState.routingTable().index("idx_new").shard(0).primaryShard();
@@ -93,96 +101,183 @@ public void testDecideShardAllocation() {
         assertNotNull(allocateDecision.getTargetNode().getId(), assignedShards.get(0).currentNodeId());
     }
 
-    public void testBalanceByShardLoad() {
+    public void testBalanceByForecastWriteLoad() {
+
+        var allocationService = new MockAllocationService(
+            yesAllocationDeciders(),
+            new TestGatewayAllocator(),
+            new BalancedShardsAllocator(
+                new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+                TEST_WRITE_LOAD_FORECASTER
+            ),
+            EmptyClusterInfoService.INSTANCE,
+            SNAPSHOT_INFO_SERVICE_WITH_NO_SHARD_SIZES
+        );
 
-        var smallIndices = IntStream.range(1, 5)
-            .mapToObj(i -> IndexMetadata.builder("small-index-" + i))
-            .map(builder -> builder.settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
-            .map(builder -> builder.indexWriteLoadForecast(randomIngestLoad(1.5)));
+        var clusterState = applyStartedShardsUntilNoChange(
+            stateWithStartedIndices(
+                IndexMetadata.builder("heavy-index").indexWriteLoadForecast(8.0),
+                IndexMetadata.builder("light-index-1").indexWriteLoadForecast(1.0),
+                IndexMetadata.builder("light-index-2").indexWriteLoadForecast(2.0),
+                IndexMetadata.builder("light-index-3").indexWriteLoadForecast(3.0),
+                IndexMetadata.builder("zero-write-load-index").indexWriteLoadForecast(0.0),
+                IndexMetadata.builder("no-write-load-index")
+            ),
+            allocationService
+        );
 
-        var heavyIndex = IndexMetadata.builder("heavy-index")
-            .settings(settings(Version.CURRENT))
-            .numberOfShards(1)
-            .numberOfReplicas(0)
-            .indexWriteLoadForecast(8.0);
+        assertThat(
+            getShardsPerNode(clusterState).values(),
+            containsInAnyOrder(
+                Set.of("heavy-index"),
+                Set.of("light-index-1", "light-index-2", "light-index-3", "zero-write-load-index", "no-write-load-index")
+            )
+        );
 
-        var clusterState = stateWithStartedIndices(Stream.concat(smallIndices, Stream.of(heavyIndex)).toList());
+        assertThat(
+            getPerNode(
+                clusterState,
+                summingDouble(
+                    it -> TEST_WRITE_LOAD_FORECASTER.getForecastedWriteLoad(clusterState.metadata().index(it.index())).orElse(0.0)
+                )
+            ).values(),
+            everyItem(lessThanOrEqualTo(8.0))
+        );
+    }
 
-        var testWriteLoadForecaster = new WriteLoadForecaster() {
-            @Override
-            public Metadata.Builder withWriteLoadForecastForWriteIndex(String dataStreamName, Metadata.Builder metadata) {
-                throw new UnsupportedOperationException("Not required for test");
-            }
+    public void testBalanceByForecastDiskUsage() {
 
-            @Override
-            @SuppressForbidden(reason = "This is required to test balancing by ingest load")
-            public OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata) {
-                return indexMetadata.getForecastedWriteLoad();
-            }
-        };
+        var allocationService = createAllocationService(
+            Settings.builder()
+                // enable disk based balancing
+                .put(BalancedShardsAllocator.DISK_USAGE_BALANCE_FACTOR_SETTING.getKey(), "1e-9")
+                .build()
+        );
 
-        var settings = Settings.EMPTY;
-        var allocator = new BalancedShardsAllocator(
-            settings,
-            new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-            testWriteLoadForecaster
+        var clusterState = applyStartedShardsUntilNoChange(
+            stateWithStartedIndices(
+                IndexMetadata.builder("heavy-index").shardSizeInBytesForecast(ByteSizeValue.ofGb(8).getBytes()),
+                IndexMetadata.builder("light-index-1").shardSizeInBytesForecast(ByteSizeValue.ofGb(1).getBytes()),
+                IndexMetadata.builder("light-index-2").shardSizeInBytesForecast(ByteSizeValue.ofGb(2).getBytes()),
+                IndexMetadata.builder("light-index-3").shardSizeInBytesForecast(ByteSizeValue.ofGb(3).getBytes()),
+                IndexMetadata.builder("zero-disk-usage-index").shardSizeInBytesForecast(0L),
+                IndexMetadata.builder("no-disk-usage-index")
+            ),
+            allocationService
         );
-        var allocation = createRoutingAllocation(clusterState);
-        allocator.allocate(allocation);
 
-        assertThat(allocation.metadata().getTotalNumberOfShards(), allOf(greaterThanOrEqualTo(3), lessThanOrEqualTo(5)));
-        for (RoutingNode routingNode : allocation.routingNodes()) {
-            var nodeIngestLoad = 0.0;
-            for (ShardRouting shardRouting : routingNode) {
-                if (shardRouting.started() || shardRouting.initializing()) { // count load from the target node when relocating
-                    var indexMetadata = clusterState.metadata().index(shardRouting.index());
-                    nodeIngestLoad += testWriteLoadForecaster.getForecastedWriteLoad(indexMetadata).orElse(0.0);
-                }
-            }
-            assertThat(nodeIngestLoad, lessThanOrEqualTo(8.0 + 1.5));
-        }
+        assertThat(
+            getShardsPerNode(clusterState).values(),
+            containsInAnyOrder(
+                Set.of("heavy-index"),
+                Set.of("light-index-1", "light-index-2", "light-index-3", "zero-disk-usage-index", "no-disk-usage-index")
+            )
+        );
+
+        assertThat(
+            getPerNode(
+                clusterState,
+                summingLong(it -> clusterState.metadata().index(it.index()).getForecastedShardSizeInBytes().orElse(0L))
+            ).values(),
+            everyItem(lessThanOrEqualTo(ByteSizeValue.ofGb(8).getBytes()))
+        );
     }
 
-    private Double randomIngestLoad(double max) {
-        return switch (randomInt(3)) {
-            case 0 -> null;
-            case 1 -> 0.0;
-            default -> randomDoubleBetween(1.0, max, true);
-        };
+    public void testBalanceByActualDiskUsage() {
+
+        var allocationService = createAllocationService(
+            Settings.builder()
+                // enable disk based balancing
+                .put(BalancedShardsAllocator.DISK_USAGE_BALANCE_FACTOR_SETTING.getKey(), "1e-9")
+                .build(),
+            createClusterInfoService(
+                Map.ofEntries(
+                    Map.entry("heavy-index", ByteSizeValue.ofGb(8).getBytes()),
+                    Map.entry("light-index-1", ByteSizeValue.ofGb(1).getBytes()),
+                    Map.entry("light-index-2", ByteSizeValue.ofGb(2).getBytes()),
+                    Map.entry("light-index-3", ByteSizeValue.ofGb(3).getBytes()),
+                    Map.entry("zero-disk-usage-index", 0L)
+                    // no-disk-usage-index is intentionally not present in cluster info
+                )
+            )
+        );
+
+        var clusterState = applyStartedShardsUntilNoChange(
+            stateWithStartedIndices(
+                IndexMetadata.builder("heavy-index"),
+                IndexMetadata.builder("light-index-1"),
+                IndexMetadata.builder("light-index-2"),
+                IndexMetadata.builder("light-index-3"),
+                IndexMetadata.builder("zero-disk-usage-index"),
+                IndexMetadata.builder("no-disk-usage-index")
+            ),
+            allocationService
+        );
+
+        assertThat(
+            getShardsPerNode(clusterState).values(),
+            containsInAnyOrder(
+                Set.of("heavy-index"),
+                Set.of("light-index-1", "light-index-2", "light-index-3", "zero-disk-usage-index", "no-disk-usage-index")
+            )
+        );
+
+        assertThat(
+            getPerNode(
+                clusterState,
+                summingLong(it -> clusterState.metadata().index(it.index()).getForecastedShardSizeInBytes().orElse(0L))
+            ).values(),
+            everyItem(lessThanOrEqualTo(ByteSizeValue.ofGb(8).getBytes()))
+        );
     }
 
-    public void testBalanceByDiskUsage() {
+    public void testBalanceByActualAndForecastDiskUsage() {
 
-        var smallIndices = IntStream.range(1, 5)
-            .mapToObj(i -> IndexMetadata.builder("small-index-" + i))
-            .map(builder -> builder.settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
-            .map(builder -> builder.shardSizeInBytesForecast(ByteSizeValue.ofGb(1).getBytes()));
+        var allocationService = createAllocationService(
+            Settings.builder()
+                // enable disk based balancing
+                .put(BalancedShardsAllocator.DISK_USAGE_BALANCE_FACTOR_SETTING.getKey(), "1e-9")
+                .build(),
+            createClusterInfoService(Map.of("heavy-index", ByteSizeValue.ofGb(8).getBytes()))
+        );
 
-        var heavyIndex = IndexMetadata.builder("heavy-index")
-            .settings(settings(Version.CURRENT))
-            .numberOfShards(1)
-            .numberOfReplicas(0)
-            .shardSizeInBytesForecast(ByteSizeValue.ofGb(8).getBytes());
+        var clusterState = applyStartedShardsUntilNoChange(
+            stateWithStartedIndices(
+                IndexMetadata.builder("heavy-index"),// size is set in cluster info
+                IndexMetadata.builder("light-index-1").shardSizeInBytesForecast(ByteSizeValue.ofGb(1).getBytes()),
+                IndexMetadata.builder("light-index-2").shardSizeInBytesForecast(ByteSizeValue.ofGb(2).getBytes()),
+                IndexMetadata.builder("light-index-3").shardSizeInBytesForecast(ByteSizeValue.ofGb(3).getBytes()),
+                IndexMetadata.builder("zero-disk-usage-index").shardSizeInBytesForecast(0L),
+                IndexMetadata.builder("no-disk-usage-index")
+            ),
+            allocationService
+        );
 
-        var clusterState = stateWithStartedIndices(Stream.concat(smallIndices, Stream.of(heavyIndex)).toList());
+        assertThat(
+            getShardsPerNode(clusterState).values(),
+            containsInAnyOrder(
+                Set.of("heavy-index"),
+                Set.of("light-index-1", "light-index-2", "light-index-3", "zero-disk-usage-index", "no-disk-usage-index")
+            )
+        );
 
-        var allocator = new BalancedShardsAllocator(
-            Settings.builder().put("cluster.routing.allocation.balance.disk_usage", "1e-9").build()
+        assertThat(
+            getPerNode(
+                clusterState,
+                summingLong(it -> clusterState.metadata().index(it.index()).getForecastedShardSizeInBytes().orElse(0L))
+            ).values(),
+            everyItem(lessThanOrEqualTo(ByteSizeValue.ofGb(8).getBytes()))
         );
-        var allocation = createRoutingAllocation(clusterState);
-        allocator.allocate(allocation);
+    }
 
-        assertThat(allocation.metadata().getTotalNumberOfShards(), allOf(greaterThanOrEqualTo(3), lessThanOrEqualTo(5)));
-        for (RoutingNode routingNode : allocation.routingNodes()) {
-            var nodeDiskUsage = 0L;
-            for (ShardRouting shardRouting : routingNode) {
-                if (shardRouting.started() || shardRouting.initializing()) { // count load from the target node when relocating
-                    var indexMetadata = clusterState.metadata().index(shardRouting.index());
-                    nodeDiskUsage += indexMetadata.getForecastedShardSizeInBytes().orElse(0L);
-                }
-            }
-            assertThat(nodeDiskUsage, lessThanOrEqualTo(ByteSizeValue.ofGb(8 + 1).getBytes()));
-        }
+    private static Map<String, Set<String>> getShardsPerNode(ClusterState clusterState) {
+        return getPerNode(clusterState, mapping(ShardRouting::getIndexName, toSet()));
+    }
+
+    private static <T> Map<String, T> getPerNode(ClusterState clusterState, Collector<ShardRouting, ?, T> collector) {
+        return clusterState.getRoutingNodes()
+            .stream()
+            .collect(Collectors.toMap(RoutingNode::nodeId, it -> StreamSupport.stream(it.spliterator(), false).collect(collector)));
     }
 
     /**
@@ -191,7 +286,7 @@ public void testBalanceByDiskUsage() {
     public void testRebalanceImprovesTheBalanceOfTheShards() {
         var discoveryNodesBuilder = DiscoveryNodes.builder();
         for (int node = 0; node < 3; node++) {
-            discoveryNodesBuilder.add(createNode("node-" + node));
+            discoveryNodesBuilder.add(newNode("node-" + node));
         }
 
         var metadataBuilder = Metadata.builder();
@@ -310,6 +405,27 @@ public void testGetIndexDiskUsageInBytes() {
         }
     }
 
+    public void testThresholdLimit() {
+        final var badValue = (float) randomDoubleBetween(0.0, Math.nextDown(1.0f), true);
+        assertEquals(
+            1.0f,
+            new BalancedShardsAllocator(Settings.builder().put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), badValue).build())
+                .getThreshold(),
+            0.0f
+        );
+        assertCriticalWarnings("ignoring value [" + badValue + """
+            ] for [cluster.routing.allocation.balance.threshold] since it is smaller than 1.0; setting \
+            [cluster.routing.allocation.balance.threshold] to a value smaller than 1.0 will be forbidden in a future release""");
+
+        final var goodValue = (float) randomDoubleBetween(1.0, 10.0, true);
+        assertEquals(
+            goodValue,
+            new BalancedShardsAllocator(Settings.builder().put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), goodValue).build())
+                .getThreshold(),
+            0.0f
+        );
+    }
+
     private Map<String, Integer> getTargetShardPerNodeCount(IndexRoutingTable indexRoutingTable) {
         var counts = new HashMap<String, Integer>();
         for (int shardId = 0; shardId < indexRoutingTable.size(); shardId++) {
@@ -333,47 +449,40 @@ private RoutingAllocation createRoutingAllocation(ClusterState clusterState) {
         );
     }
 
-    private static ClusterState stateWithStartedIndices(List<IndexMetadata.Builder> indices) {
+    private static ClusterInfoService createClusterInfoService(Map<String, Long> indexSizes) {
+        return () -> new ClusterInfo(
+            Map.of(),
+            Map.of(),
+            indexSizes.entrySet()
+                .stream()
+                .collect(
+                    toMap(
+                        entry -> ClusterInfo.shardIdentifierFromRouting(new ShardId(entry.getKey(), "_na_", 0), true),
+                        Map.Entry::getValue
+                    )
+                ),
+            Map.of(),
+            Map.of(),
+            Map.of()
+        );
+    }
+
+    private static ClusterState stateWithStartedIndices(IndexMetadata.Builder... indices) {
         var metadataBuilder = Metadata.builder();
-        var routingTableBuilder = RoutingTable.builder();
+        var routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         for (var index : indices) {
-            var inSyncId = UUIDs.randomBase64UUID(random());
-            var build = index.putInSyncAllocationIds(0, Set.of(inSyncId)).build();
+            var build = index.settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0).build();
             metadataBuilder.put(build, false);
-            routingTableBuilder.add(
-                IndexRoutingTable.builder(build.getIndex())
-                    .addShard(
-                        TestShardRouting.newShardRouting(
-                            new ShardId(build.getIndex(), 0),
-                            randomFrom("node-1", "node-2"),
-                            null,
-                            true,
-                            ShardRoutingState.STARTED,
-                            AllocationId.newInitializing(inSyncId)
-                        )
-                    )
-            );
+            routingTableBuilder.addAsNew(build);
         }
 
         return ClusterState.builder(ClusterName.DEFAULT)
-            .nodes(DiscoveryNodes.builder().add(createNode("node-1")).add(createNode("node-2")))
+            .nodes(DiscoveryNodes.builder().add(newNode("node-1")).add(newNode("node-2")))
             .metadata(metadataBuilder)
             .routingTable(routingTableBuilder)
             .build();
     }
 
-    private static DiscoveryNode createNode(String nodeId) {
-        return new DiscoveryNode(
-            nodeId,
-            nodeId,
-            UUIDs.randomBase64UUID(random()),
-            buildNewFakeTransportAddress(),
-            Map.of(),
-            DiscoveryNodeRole.roles(),
-            Version.CURRENT
-        );
-    }
-
     private void addIndex(
         Metadata.Builder metadataBuilder,
         RoutingTable.Builder routingTableBuilder,
@@ -383,13 +492,9 @@ private void addIndex(
         var numberOfShards = assignments.entrySet().stream().mapToInt(Map.Entry::getValue).sum();
         var inSyncIds = randomList(numberOfShards, numberOfShards, () -> UUIDs.randomBase64UUID(random()));
         var indexMetadataBuilder = IndexMetadata.builder(name)
-            .settings(
-                Settings.builder()
-                    .put("index.number_of_shards", numberOfShards)
-                    .put("index.number_of_replicas", 0)
-                    .put("index.version.created", Version.CURRENT)
-                    .build()
-            );
+            .settings(settings(Version.CURRENT))
+            .numberOfShards(numberOfShards)
+            .numberOfReplicas(0);
 
         for (int shardId = 0; shardId < numberOfShards; shardId++) {
             indexMetadataBuilder.putInSyncAllocationIds(shardId, Set.of(inSyncIds.get(shardId)));
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterAllocationSimulationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterAllocationSimulationTests.java
index 6495de7b7441..34f0abd0a14b 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterAllocationSimulationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterAllocationSimulationTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.DiskUsage;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -62,6 +63,7 @@
 import static org.elasticsearch.cluster.node.DiscoveryNodeRole.DATA_HOT_NODE_ROLE;
 import static org.elasticsearch.cluster.node.DiscoveryNodeRole.DATA_WARM_NODE_ROLE;
 import static org.elasticsearch.cluster.node.DiscoveryNodeRole.MASTER_ROLE;
+import static org.elasticsearch.common.settings.ClusterSettings.createBuiltInClusterSettings;
 
 public class ClusterAllocationSimulationTests extends ESAllocationTestCase {
 
@@ -129,7 +131,7 @@ public void testBalanceQuality() throws IOException {
         }
         final var metadata = metadataBuilder.build();
 
-        final var routingTableBuilder = RoutingTable.builder();
+        final var routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         for (final var indexMetadata : metadata) {
             routingTableBuilder.addAsNew(indexMetadata);
         }
@@ -380,9 +382,8 @@ void addNode(long sizeBytes, int shardCount, double writeLoad) {
                 for (ShardRouting shardRouting : routingNode) {
                     shards += 1;
                     totalBytes += shardSizesByIndex.get(shardRouting.index().getName());
-                    totalWriteLoad += SIMULATION_WRITE_LOAD_FORECASTER.getForecastedWriteLoad(
-                        clusterState.metadata().index(shardRouting.index())
-                    ).orElseThrow(() -> new AssertionError("missing write load"));
+                    totalWriteLoad += TEST_WRITE_LOAD_FORECASTER.getForecastedWriteLoad(clusterState.metadata().index(shardRouting.index()))
+                        .orElseThrow(() -> new AssertionError("missing write load"));
                 }
 
                 results.startObject();
@@ -473,10 +474,10 @@ private Map.Entry<MockAllocationService, ShardsAllocator> createNewAllocationSer
     ) {
         var strategyRef = new SetOnce<AllocationService>();
         var desiredBalanceShardsAllocator = new DesiredBalanceShardsAllocator(
+            createBuiltInClusterSettings(),
             new BalancedShardsAllocator(
-                Settings.EMPTY,
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-                SIMULATION_WRITE_LOAD_FORECASTER
+                TEST_WRITE_LOAD_FORECASTER
             ),
             threadPool,
             clusterService,
@@ -484,11 +485,7 @@ private Map.Entry<MockAllocationService, ShardsAllocator> createNewAllocationSer
                 .executeWithRoutingAllocation(clusterState, "reconcile-desired-balance", routingAllocationAction)
         );
         var strategy = new MockAllocationService(
-            randomAllocationDeciders(
-                Settings.EMPTY,
-                new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-                random()
-            ),
+            randomAllocationDeciders(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)),
             new TestGatewayAllocator(),
             desiredBalanceShardsAllocator,
             clusterInfoService,
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterBalanceStatsTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterBalanceStatsTests.java
new file mode 100644
index 000000000000..db0e187d217f
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/ClusterBalanceStatsTests.java
@@ -0,0 +1,317 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.cluster.routing.allocation.allocator;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterInfo;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.shard.ShardId;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.elasticsearch.cluster.node.DiscoveryNodeRole.DATA_CONTENT_NODE_ROLE;
+import static org.elasticsearch.cluster.node.DiscoveryNodeRole.DATA_HOT_NODE_ROLE;
+import static org.elasticsearch.cluster.node.DiscoveryNodeRole.DATA_WARM_NODE_ROLE;
+import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
+import static org.elasticsearch.cluster.routing.TestShardRouting.newShardRouting;
+import static org.hamcrest.Matchers.equalTo;
+
+public class ClusterBalanceStatsTests extends ESAllocationTestCase {
+
+    public void testStatsForSingleTierClusterWithNoForecasts() {
+
+        var clusterState = createClusterState(
+            List.of(
+                newNode("node-1", "node-1", Set.of(DATA_CONTENT_NODE_ROLE)),
+                newNode("node-2", "node-2", Set.of(DATA_CONTENT_NODE_ROLE)),
+                newNode("node-3", "node-3", Set.of(DATA_CONTENT_NODE_ROLE))
+            ),
+            List.of(
+                startedIndex("index-1", null, null, "node-1", "node-2"),
+                startedIndex("index-2", null, null, "node-2", "node-3"),
+                startedIndex("index-3", null, null, "node-3", "node-1")
+            )
+        );
+        var clusterInfo = createClusterInfo(
+            List.of(indexSizes("index-1", 1L, 1L), indexSizes("index-2", 2L, 2L), indexSizes("index-3", 3L, 3L))
+        );
+
+        var stats = ClusterBalanceStats.createFrom(clusterState, clusterInfo, TEST_WRITE_LOAD_FORECASTER);
+
+        assertThat(
+            stats,
+            equalTo(
+                new ClusterBalanceStats(
+                    Map.of(
+                        DATA_CONTENT_NODE_ROLE.roleName(),
+                        new ClusterBalanceStats.TierBalanceStats(
+                            new ClusterBalanceStats.MetricStats(6.0, 2.0, 2.0, 2.0, 0.0),
+                            new ClusterBalanceStats.MetricStats(0.0, 0.0, 0.0, 0.0, 0.0),
+                            new ClusterBalanceStats.MetricStats(12.0, 3.0, 5.0, 4.0, stdDev(3.0, 5.0, 4.0)),
+                            new ClusterBalanceStats.MetricStats(12.0, 3.0, 5.0, 4.0, stdDev(3.0, 5.0, 4.0))
+                        )
+                    ),
+                    Map.ofEntries(
+                        Map.entry("node-1", new ClusterBalanceStats.NodeBalanceStats(2, 0.0, 4L, 4L)),
+                        Map.entry("node-2", new ClusterBalanceStats.NodeBalanceStats(2, 0.0, 3L, 3L)),
+                        Map.entry("node-3", new ClusterBalanceStats.NodeBalanceStats(2, 0.0, 5L, 5L))
+                    )
+                )
+            )
+        );
+    }
+
+    public void testStatsForSingleTierClusterWithForecasts() {
+
+        var clusterState = createClusterState(
+            List.of(
+                newNode("node-1", "node-1", Set.of(DATA_CONTENT_NODE_ROLE)),
+                newNode("node-2", "node-2", Set.of(DATA_CONTENT_NODE_ROLE)),
+                newNode("node-3", "node-3", Set.of(DATA_CONTENT_NODE_ROLE))
+            ),
+            List.of(
+                startedIndex("index-1", 1.5, 8L, "node-1", "node-2"),
+                startedIndex("index-2", 2.5, 4L, "node-2", "node-3"),
+                startedIndex("index-3", 2.0, 6L, "node-3", "node-1")
+            )
+        );
+        // intentionally different from forecast
+        var clusterInfo = createClusterInfo(
+            List.of(indexSizes("index-1", 1L, 1L), indexSizes("index-2", 2L, 2L), indexSizes("index-3", 3L, 3L))
+        );
+
+        var stats = ClusterBalanceStats.createFrom(clusterState, clusterInfo, TEST_WRITE_LOAD_FORECASTER);
+
+        assertThat(
+            stats,
+            equalTo(
+                new ClusterBalanceStats(
+                    Map.of(
+                        DATA_CONTENT_NODE_ROLE.roleName(),
+                        new ClusterBalanceStats.TierBalanceStats(
+                            new ClusterBalanceStats.MetricStats(6.0, 2.0, 2.0, 2.0, 0.0),
+                            new ClusterBalanceStats.MetricStats(12.0, 3.5, 4.5, 4.0, stdDev(3.5, 4.0, 4.5)),
+                            new ClusterBalanceStats.MetricStats(36.0, 10.0, 14.0, 12.0, stdDev(10.0, 12.0, 14.0)),
+                            new ClusterBalanceStats.MetricStats(12.0, 3.0, 5.0, 4.0, stdDev(3.0, 5.0, 4.0))
+                        )
+                    ),
+                    Map.ofEntries(
+                        Map.entry("node-1", new ClusterBalanceStats.NodeBalanceStats(2, 3.5, 14L, 4L)),
+                        Map.entry("node-2", new ClusterBalanceStats.NodeBalanceStats(2, 4.0, 12L, 3L)),
+                        Map.entry("node-3", new ClusterBalanceStats.NodeBalanceStats(2, 4.5, 10L, 5L))
+                    )
+                )
+            )
+        );
+    }
+
+    public void testStatsForHotWarmClusterWithForecasts() {
+
+        var clusterState = createClusterState(
+            List.of(
+                newNode("node-hot-1", "node-hot-1", Set.of(DATA_CONTENT_NODE_ROLE, DATA_HOT_NODE_ROLE)),
+                newNode("node-hot-2", "node-hot-2", Set.of(DATA_CONTENT_NODE_ROLE, DATA_HOT_NODE_ROLE)),
+                newNode("node-hot-3", "node-hot-3", Set.of(DATA_CONTENT_NODE_ROLE, DATA_HOT_NODE_ROLE)),
+                newNode("node-warm-1", "node-warm-1", Set.of(DATA_WARM_NODE_ROLE)),
+                newNode("node-warm-2", "node-warm-2", Set.of(DATA_WARM_NODE_ROLE)),
+                newNode("node-warm-3", "node-warm-3", Set.of(DATA_WARM_NODE_ROLE))
+            ),
+            List.of(
+                startedIndex("index-hot-1", 4.0, 4L, "node-hot-1", "node-hot-2", "node-hot-3"),
+                startedIndex("index-hot-2", 2.0, 6L, "node-hot-1", "node-hot-2"),
+                startedIndex("index-hot-3", 2.5, 6L, "node-hot-1", "node-hot-3"),
+                startedIndex("index-warm-1", 0.0, 12L, "node-warm-1", "node-warm-2"),
+                startedIndex("index-warm-2", 0.0, 18L, "node-warm-3")
+            )
+        );
+        // intentionally different from forecast
+        var clusterInfo = createClusterInfo(
+            List.of(
+                indexSizes("index-hot-1", 4L, 4L, 4L),
+                indexSizes("index-hot-2", 5L, 5L),
+                indexSizes("index-hot-3", 6L, 6L),
+                indexSizes("index-warm-1", 12L, 12L),
+                indexSizes("index-warm-2", 18L)
+            )
+        );
+
+        var stats = ClusterBalanceStats.createFrom(clusterState, clusterInfo, TEST_WRITE_LOAD_FORECASTER);
+
+        assertThat(
+            stats,
+            equalTo(
+                new ClusterBalanceStats(
+                    Map.of(
+                        DATA_CONTENT_NODE_ROLE.roleName(),
+                        new ClusterBalanceStats.TierBalanceStats(
+                            new ClusterBalanceStats.MetricStats(7.0, 2.0, 3.0, 7.0 / 3, stdDev(3.0, 2.0, 2.0)),
+                            new ClusterBalanceStats.MetricStats(21.0, 6.0, 8.5, 7.0, stdDev(6.0, 8.5, 6.5)),
+                            new ClusterBalanceStats.MetricStats(36.0, 10.0, 16.0, 12.0, stdDev(10.0, 10.0, 16.0)),
+                            new ClusterBalanceStats.MetricStats(34.0, 9.0, 15.0, 34.0 / 3, stdDev(9.0, 10.0, 15.0))
+                        ),
+                        DATA_HOT_NODE_ROLE.roleName(),
+                        new ClusterBalanceStats.TierBalanceStats(
+                            new ClusterBalanceStats.MetricStats(7.0, 2.0, 3.0, 7.0 / 3, stdDev(3.0, 2.0, 2.0)),
+                            new ClusterBalanceStats.MetricStats(21.0, 6.0, 8.5, 7.0, stdDev(6.0, 8.5, 6.5)),
+                            new ClusterBalanceStats.MetricStats(36.0, 10.0, 16.0, 12.0, stdDev(10.0, 10.0, 16.0)),
+                            new ClusterBalanceStats.MetricStats(34.0, 9.0, 15.0, 34.0 / 3, stdDev(9.0, 10.0, 15.0))
+                        ),
+                        DATA_WARM_NODE_ROLE.roleName(),
+                        new ClusterBalanceStats.TierBalanceStats(
+                            new ClusterBalanceStats.MetricStats(3.0, 1.0, 1.0, 1.0, 0.0),
+                            new ClusterBalanceStats.MetricStats(0.0, 0.0, 0.0, 0.0, 0.0),
+                            new ClusterBalanceStats.MetricStats(42.0, 12.0, 18.0, 14.0, stdDev(12.0, 12.0, 18.0)),
+                            new ClusterBalanceStats.MetricStats(42.0, 12.0, 18.0, 14.0, stdDev(12.0, 12.0, 18.0))
+                        )
+                    ),
+                    Map.ofEntries(
+                        Map.entry("node-hot-1", new ClusterBalanceStats.NodeBalanceStats(3, 8.5, 16L, 15L)),
+                        Map.entry("node-hot-2", new ClusterBalanceStats.NodeBalanceStats(2, 6.0, 10L, 9L)),
+                        Map.entry("node-hot-3", new ClusterBalanceStats.NodeBalanceStats(2, 6.5, 10L, 10L)),
+                        Map.entry("node-warm-1", new ClusterBalanceStats.NodeBalanceStats(1, 0.0, 12L, 12L)),
+                        Map.entry("node-warm-2", new ClusterBalanceStats.NodeBalanceStats(1, 0.0, 12L, 12L)),
+                        Map.entry("node-warm-3", new ClusterBalanceStats.NodeBalanceStats(1, 0.0, 18L, 18L))
+                    )
+                )
+            )
+        );
+    }
+
+    public void testStatsForNoIndicesInTier() {
+
+        var clusterState = createClusterState(
+            List.of(
+                newNode("node-1", "node-1", Set.of(DATA_CONTENT_NODE_ROLE)),
+                newNode("node-2", "node-2", Set.of(DATA_CONTENT_NODE_ROLE)),
+                newNode("node-3", "node-3", Set.of(DATA_CONTENT_NODE_ROLE))
+            ),
+            List.of()
+        );
+        var clusterInfo = createClusterInfo(List.of());
+
+        var stats = ClusterBalanceStats.createFrom(clusterState, clusterInfo, TEST_WRITE_LOAD_FORECASTER);
+
+        assertThat(
+            stats,
+            equalTo(
+                new ClusterBalanceStats(
+                    Map.of(
+                        DATA_CONTENT_NODE_ROLE.roleName(),
+                        new ClusterBalanceStats.TierBalanceStats(
+                            new ClusterBalanceStats.MetricStats(0.0, 0.0, 0.0, 0.0, 0.0),
+                            new ClusterBalanceStats.MetricStats(0.0, 0.0, 0.0, 0.0, 0.0),
+                            new ClusterBalanceStats.MetricStats(0.0, 0.0, 0.0, 0.0, 0.0),
+                            new ClusterBalanceStats.MetricStats(0.0, 0.0, 0.0, 0.0, 0.0)
+                        )
+                    ),
+                    Map.ofEntries(
+                        Map.entry("node-1", new ClusterBalanceStats.NodeBalanceStats(0, 0.0, 0L, 0L)),
+                        Map.entry("node-2", new ClusterBalanceStats.NodeBalanceStats(0, 0.0, 0L, 0L)),
+                        Map.entry("node-3", new ClusterBalanceStats.NodeBalanceStats(0, 0.0, 0L, 0L))
+                    )
+                )
+            )
+        );
+    }
+
+    private static ClusterState createClusterState(List<DiscoveryNode> nodes, List<Tuple<IndexMetadata.Builder, String[]>> indices) {
+        var discoveryNodesBuilder = DiscoveryNodes.builder();
+        for (DiscoveryNode node : nodes) {
+            discoveryNodesBuilder.add(node);
+        }
+
+        var metadataBuilder = Metadata.builder();
+        var routingTableBuilder = RoutingTable.builder();
+        for (var index : indices) {
+            var indexMetadata = index.v1()
+                .settings(settings(Version.CURRENT))
+                .numberOfShards(index.v2().length)
+                .numberOfReplicas(0)
+                .build();
+            metadataBuilder.put(indexMetadata, false);
+            var indexRoutingTableBuilder = IndexRoutingTable.builder(indexMetadata.getIndex());
+            for (int shardId = 0; shardId < index.v2().length; shardId++) {
+                indexRoutingTableBuilder.addShard(
+                    newShardRouting(new ShardId(indexMetadata.getIndex(), shardId), index.v2()[shardId], true, STARTED)
+                );
+            }
+            routingTableBuilder.add(indexRoutingTableBuilder.build());
+        }
+
+        return ClusterState.builder(ClusterName.DEFAULT)
+            .nodes(discoveryNodesBuilder)
+            .metadata(metadataBuilder)
+            .routingTable(routingTableBuilder)
+            .build();
+    }
+
+    private static Tuple<IndexMetadata.Builder, String[]> startedIndex(
+        String indexName,
+        @Nullable Double indexWriteLoadForecast,
+        @Nullable Long shardSizeInBytesForecast,
+        String... nodeId
+    ) {
+        return Tuple.tuple(
+            IndexMetadata.builder(indexName)
+                .indexWriteLoadForecast(indexWriteLoadForecast)
+                .shardSizeInBytesForecast(shardSizeInBytesForecast),
+            nodeId
+        );
+    }
+
+    private ClusterInfo createClusterInfo(List<Tuple<String, long[]>> shardSizes) {
+        return new ClusterInfo(
+            Map.of(),
+            Map.of(),
+            shardSizes.stream()
+                .flatMap(
+                    entry -> IntStream.range(0, entry.v2().length)
+                        .mapToObj(
+                            index -> Map.entry(
+                                ClusterInfo.shardIdentifierFromRouting(new ShardId(entry.v1(), "_na_", index), true),
+                                entry.v2()[index]
+                            )
+                        )
+                )
+                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)),
+            Map.of(),
+            Map.of(),
+            Map.of()
+        );
+    }
+
+    private static Tuple<String, long[]> indexSizes(String name, long... sizes) {
+        return Tuple.tuple(name, sizes);
+    }
+
+    private static double stdDev(double... data) {
+        double total = 0.0;
+        double total2 = 0.0;
+        int count = data.length;
+        for (double d : data) {
+            total += d;
+            total2 += Math.pow(d, 2);
+        }
+        return Math.sqrt(total2 / count - Math.pow(total / count, 2));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputerTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputerTests.java
index 123ca20b1567..a9e02c464840 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputerTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputerTests.java
@@ -8,12 +8,16 @@
 
 package org.elasticsearch.cluster.routing.allocation.allocator;
 
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.DiskUsage;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -36,11 +40,14 @@
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.snapshots.SnapshotShardSizeInfo;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.MockLogAppender;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -52,6 +59,7 @@
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_VERSION_CREATED;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
@@ -61,10 +69,13 @@
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
 import static org.elasticsearch.cluster.routing.TestShardRouting.newShardRouting;
+import static org.elasticsearch.common.settings.ClusterSettings.createBuiltInClusterSettings;
 import static org.hamcrest.Matchers.aMapWithSize;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasEntry;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class DesiredBalanceComputerTests extends ESTestCase {
 
@@ -327,32 +338,82 @@ public void testRespectsAssignmentOfUnknownReplicas() {
         );
     }
 
+    public void testRespectsAssignmentByGatewayAllocators() {
+        var desiredBalanceComputer = createDesiredBalanceComputer();
+        var clusterState = createInitialClusterState(3);
+        var index = clusterState.metadata().index(TEST_INDEX).getIndex();
+
+        final var routingAllocation = new RoutingAllocation(
+            new AllocationDeciders(List.of()),
+            clusterState.mutableRoutingNodes(),
+            clusterState,
+            ClusterInfo.EMPTY,
+            SnapshotShardSizeInfo.EMPTY,
+            0L
+        );
+        for (var iterator = routingAllocation.routingNodes().unassigned().iterator(); iterator.hasNext();) {
+            var shardRouting = iterator.next();
+            if (shardRouting.shardId().id() == 0 && shardRouting.primary()) {
+                routingAllocation.routingNodes()
+                    .startShard(
+                        logger,
+                        iterator.initialize("node-2", null, 0L, routingAllocation.changes()),
+                        routingAllocation.changes(),
+                        0L
+                    );
+                break;
+            }
+        }
+
+        var desiredBalance = desiredBalanceComputer.compute(
+            DesiredBalance.INITIAL,
+            DesiredBalanceInput.create(randomNonNegativeLong(), routingAllocation),
+            queue(),
+            input -> true
+        );
+
+        assertDesiredAssignments(
+            desiredBalance,
+            Map.of(
+                new ShardId(index, 0),
+                new ShardAssignment(Set.of("node-2", "node-1"), 2, 0, 0),
+                new ShardId(index, 1),
+                new ShardAssignment(Set.of("node-0", "node-1"), 2, 0, 0)
+            )
+        );
+
+    }
+
     public void testSimulatesAchievingDesiredBalanceBeforeDelegating() {
 
         var allocateCalled = new AtomicBoolean();
-        var desiredBalanceComputer = new DesiredBalanceComputer(new ShardsAllocator() {
-            @Override
-            public void allocate(RoutingAllocation allocation) {
-                assertTrue(allocateCalled.compareAndSet(false, true));
-                // whatever the allocation in the current cluster state, the desired balance service should start by moving all the
-                // known shards to their desired locations before delegating to the inner allocator
-                for (var routingNode : allocation.routingNodes()) {
-                    assertThat(
-                        allocation.routingNodes().toString(),
-                        routingNode.numberOfOwningShards(),
-                        equalTo(routingNode.nodeId().equals("node-2") ? 0 : 2)
-                    );
-                    for (var shardRouting : routingNode) {
-                        assertTrue(shardRouting.toString(), shardRouting.started());
+        var desiredBalanceComputer = new DesiredBalanceComputer(
+            createBuiltInClusterSettings(),
+            mock(ThreadPool.class),
+            new ShardsAllocator() {
+                @Override
+                public void allocate(RoutingAllocation allocation) {
+                    assertTrue(allocateCalled.compareAndSet(false, true));
+                    // whatever the allocation in the current cluster state, the desired balance service should start by moving all the
+                    // known shards to their desired locations before delegating to the inner allocator
+                    for (var routingNode : allocation.routingNodes()) {
+                        assertThat(
+                            allocation.routingNodes().toString(),
+                            routingNode.numberOfOwningShards(),
+                            equalTo(routingNode.nodeId().equals("node-2") ? 0 : 2)
+                        );
+                        for (var shardRouting : routingNode) {
+                            assertTrue(shardRouting.toString(), shardRouting.started());
+                        }
                     }
                 }
-            }
 
-            @Override
-            public ShardAllocationDecision decideShardAllocation(ShardRouting shard, RoutingAllocation allocation) {
-                throw new AssertionError("only used for allocation explain");
+                @Override
+                public ShardAllocationDecision decideShardAllocation(ShardRouting shard, RoutingAllocation allocation) {
+                    throw new AssertionError("only used for allocation explain");
+                }
             }
-        });
+        );
         var clusterState = createInitialClusterState(3);
         var index = clusterState.metadata().index(TEST_INDEX).getIndex();
 
@@ -570,7 +631,7 @@ public void testDesiredBalanceShouldConvergeInABigCluster() {
                     )
                 );
                 for (int replica = 0; replica < replicas; replica++) {
-                    var replicaNodeId = pickAndRemoveRandomValueFrom(remainingNodeIds);
+                    var replicaNodeId = primaryNodeId == null ? null : pickAndRemoveRandomValueFrom(remainingNodeIds);
                     indexRoutingTableBuilder.addShard(
                         newShardRouting(
                             shardId,
@@ -599,12 +660,11 @@ public void testDesiredBalanceShouldConvergeInABigCluster() {
             routingAllocationWithDecidersOf(clusterState, ClusterInfo.EMPTY, Settings.EMPTY),
             List.of()
         );
-        var desiredBalance = new DesiredBalanceComputer(new BalancedShardsAllocator(Settings.EMPTY)).compute(
-            DesiredBalance.INITIAL,
-            input,
-            queue(),
-            ignored -> iteration.incrementAndGet() < 1000
-        );
+        var desiredBalance = new DesiredBalanceComputer(
+            createBuiltInClusterSettings(),
+            mock(ThreadPool.class),
+            new BalancedShardsAllocator(Settings.EMPTY)
+        ).compute(DesiredBalance.INITIAL, input, queue(), ignored -> iteration.incrementAndGet() < 1000);
 
         try {
             assertThat(
@@ -765,7 +825,11 @@ public void testComputeConsideringShardSizes() {
             )
         );
 
-        var desiredBalance = new DesiredBalanceComputer(new BalancedShardsAllocator(settings)).compute(
+        var desiredBalance = new DesiredBalanceComputer(
+            createBuiltInClusterSettings(),
+            mock(ThreadPool.class),
+            new BalancedShardsAllocator(settings)
+        ).compute(
             initial,
             new DesiredBalanceInput(randomInt(), routingAllocationWithDecidersOf(clusterState, clusterInfo, settings), List.of()),
             queue(),
@@ -783,6 +847,98 @@ public void testComputeConsideringShardSizes() {
         assertThat(resultDiskUsage, allOf(aMapWithSize(2), hasEntry("node-0", 950L), hasEntry("node-1", 850L)));
     }
 
+    public void testShouldLogComputationIteration() {
+        checkIterationLogging(
+            999,
+            10L,
+            new MockLogAppender.UnseenEventExpectation(
+                "Should not report long computation too early",
+                DesiredBalanceComputer.class.getCanonicalName(),
+                Level.INFO,
+                "Desired balance computation for [*] is still not converged after [*] and [*] iterations"
+            )
+        );
+
+        checkIterationLogging(
+            1001,
+            10L,
+            new MockLogAppender.SeenEventExpectation(
+                "Should report long computation based on iteration count",
+                DesiredBalanceComputer.class.getCanonicalName(),
+                Level.INFO,
+                "Desired balance computation for [*] is still not converged after [10s] and [1000] iterations"
+            )
+        );
+
+        checkIterationLogging(
+            61,
+            1000L,
+            new MockLogAppender.SeenEventExpectation(
+                "Should report long computation based on time",
+                DesiredBalanceComputer.class.getCanonicalName(),
+                Level.INFO,
+                "Desired balance computation for [*] is still not converged after [1m] and [60] iterations"
+            )
+        );
+    }
+
+    private void checkIterationLogging(int iterations, long eachIterationDuration, MockLogAppender.AbstractEventExpectation expectation) {
+
+        var mockThreadPool = mock(ThreadPool.class);
+        var currentTime = new AtomicLong(0L);
+        when(mockThreadPool.relativeTimeInMillis()).thenAnswer(invocation -> currentTime.addAndGet(eachIterationDuration));
+
+        var desiredBalanceComputer = new DesiredBalanceComputer(createBuiltInClusterSettings(), mockThreadPool, new ShardsAllocator() {
+            @Override
+            public void allocate(RoutingAllocation allocation) {
+                final var unassignedIterator = allocation.routingNodes().unassigned().iterator();
+                while (unassignedIterator.hasNext()) {
+                    final var shardRouting = unassignedIterator.next();
+                    if (shardRouting.primary()) {
+                        unassignedIterator.initialize("node-0", null, 0L, allocation.changes());
+                    } else {
+                        unassignedIterator.removeAndIgnore(UnassignedInfo.AllocationStatus.NO_ATTEMPT, allocation.changes());
+                    }
+                }
+
+                // move shard on each iteration
+                for (var shard : allocation.routingNodes().node("node-0").shardsWithState(STARTED).toList()) {
+                    allocation.routingNodes().relocateShard(shard, "node-1", 0L, allocation.changes());
+                }
+                for (var shard : allocation.routingNodes().node("node-1").shardsWithState(STARTED).toList()) {
+                    allocation.routingNodes().relocateShard(shard, "node-0", 0L, allocation.changes());
+                }
+            }
+
+            @Override
+            public ShardAllocationDecision decideShardAllocation(ShardRouting shard, RoutingAllocation allocation) {
+                throw new AssertionError("only used for allocation explain");
+            }
+        });
+
+        MockLogAppender mockAppender = new MockLogAppender();
+        mockAppender.start();
+        mockAppender.addExpectation(expectation);
+
+        Logger logger = LogManager.getLogger(DesiredBalanceComputer.class);
+        Loggers.addAppender(logger, mockAppender);
+
+        try {
+            var iteration = new AtomicInteger(0);
+            desiredBalanceComputer.compute(
+                DesiredBalance.INITIAL,
+                createInput(createInitialClusterState(3)),
+                queue(),
+                input -> iteration.incrementAndGet() < iterations
+            );
+
+            mockAppender.assertAllExpectationsMatched();
+        } finally {
+            Loggers.removeAppender(logger, mockAppender);
+            mockAppender.stop();
+        }
+    }
+
     private static Map.Entry<String, Long> indexSize(ClusterState clusterState, String name, long size, boolean primary) {
         return Map.entry(ClusterInfo.shardIdentifierFromRouting(findShardId(clusterState, name), primary), size);
     }
@@ -809,14 +965,14 @@ static ClusterState createInitialClusterState(int dataNodesCount) {
         return ClusterState.builder(ClusterName.DEFAULT)
             .nodes(discoveryNodes.masterNodeId("master").localNodeId("master"))
             .metadata(Metadata.builder().put(indexMetadata, true))
-            .routingTable(RoutingTable.builder().addAsNew(indexMetadata))
+            .routingTable(RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(indexMetadata))
             .build();
     }
 
     static ClusterState mutateAllocationStatuses(ClusterState clusterState) {
         final var routingTableBuilder = RoutingTable.builder();
         for (final var indexRoutingTable : clusterState.routingTable()) {
-            final var indexRoutingTableBuilder = new IndexRoutingTable.Builder(indexRoutingTable.getIndex());
+            final var indexRoutingTableBuilder = IndexRoutingTable.builder(indexRoutingTable.getIndex());
             for (int shardId = 0; shardId < indexRoutingTable.size(); shardId++) {
                 final var shardRoutingTable = indexRoutingTable.shard(shardId);
                 final var shardRoutingTableBuilder = new IndexShardRoutingTable.Builder(shardRoutingTable.shardId());
@@ -873,7 +1029,7 @@ private static DiscoveryNode createDiscoveryNode(String id, Set<DiscoveryNodeRol
      * @return a {@link DesiredBalanceComputer} which allocates unassigned primaries to node-0 and unassigned replicas to node-1
      */
     private static DesiredBalanceComputer createDesiredBalanceComputer() {
-        return new DesiredBalanceComputer(new ShardsAllocator() {
+        return new DesiredBalanceComputer(createBuiltInClusterSettings(), mock(ThreadPool.class), new ShardsAllocator() {
             @Override
             public void allocate(RoutingAllocation allocation) {
                 final var unassignedIterator = allocation.routingNodes().unassigned().iterator();
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java
index 13df8b5e6af6..6d3b7a707dad 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.NodesShutdownMetadata;
@@ -85,7 +86,6 @@
 import static org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING;
 import static org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.oneOf;
@@ -201,7 +201,7 @@ public void testUnassignedPrimariesBeforeUnassignedReplicas() {
 
         final var discoveryNodes = discoveryNodes(2);
         final var metadata = Metadata.builder();
-        final var routingTable = RoutingTable.builder();
+        final var routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
 
         final var indexMetadata0 = randomPriorityIndex("index-0", 1, 1);
         metadata.put(indexMetadata0, true);
@@ -227,9 +227,9 @@ public void testUnassignedPrimariesBeforeUnassignedReplicas() {
 
         final var allocationService = createTestAllocationService(
             routingAllocation -> reconcile(routingAllocation, desiredBalance),
-            new SameShardAllocationDecider(settings, clusterSettings),
+            new SameShardAllocationDecider(clusterSettings),
             new ReplicaAfterPrimaryActiveAllocationDecider(),
-            new ThrottlingAllocationDecider(settings, clusterSettings),
+            new ThrottlingAllocationDecider(clusterSettings),
             new AllocationDecider() {
                 @Override
                 public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
@@ -285,7 +285,7 @@ public void testUnassignedShardsInterleaving() {
 
         final var discoveryNodes = discoveryNodes(4);
         final var metadata = Metadata.builder();
-        final var routingTable = RoutingTable.builder();
+        final var routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
 
         var shardsRemaining = 4;
         var indexNum = 0;
@@ -310,9 +310,9 @@ public void testUnassignedShardsInterleaving() {
 
         final var allocationService = createTestAllocationService(
             routingAllocation -> reconcile(routingAllocation, desiredBalance),
-            new SameShardAllocationDecider(settings, clusterSettings),
+            new SameShardAllocationDecider(clusterSettings),
             new ReplicaAfterPrimaryActiveAllocationDecider(),
-            new ThrottlingAllocationDecider(settings, clusterSettings)
+            new ThrottlingAllocationDecider(clusterSettings)
         );
 
         final var stateWithInitializingPrimaries = startInitializingShardsAndReroute(allocationService, clusterState);
@@ -367,7 +367,7 @@ public void testUnassignedShardsInterleaving() {
     public void testUnassignedShardsPriority() {
         final var discoveryNodes = discoveryNodes(2);
         final var metadata = Metadata.builder();
-        final var routingTable = RoutingTable.builder();
+        final var routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
 
         final var indexMetadata0 = randomPriorityIndex("index-0", 2, 1);
         final var indexMetadata1 = randomPriorityIndex("index-1", 2, 1);
@@ -400,9 +400,9 @@ public void testUnassignedShardsPriority() {
 
         final var allocationService = createTestAllocationService(
             routingAllocation -> reconcile(routingAllocation, desiredBalance),
-            new SameShardAllocationDecider(settings, clusterSettings),
+            new SameShardAllocationDecider(clusterSettings),
             new ReplicaAfterPrimaryActiveAllocationDecider(),
-            new ThrottlingAllocationDecider(settings, clusterSettings),
+            new ThrottlingAllocationDecider(clusterSettings),
             new AllocationDecider() {
                 @Override
                 public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
@@ -488,7 +488,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
     public void testUnassignedRespectsDesiredBalance() {
         final var discoveryNodes = discoveryNodes(5);
         final var metadata = Metadata.builder();
-        final var routingTable = RoutingTable.builder();
+        final var routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
 
         for (var i = 0; i < 5; i++) {
             final var indexMetadata = randomPriorityIndex("index-" + i, between(1, 5), between(0, 4));
@@ -508,7 +508,7 @@ public void testUnassignedRespectsDesiredBalance() {
         final var desiredBalance = desiredBalance(clusterState, (shardId, nodeId) -> true);
         final var allocationService = createTestAllocationService(
             routingAllocation -> reconcile(routingAllocation, desiredBalance),
-            new SameShardAllocationDecider(settings, clusterSettings),
+            new SameShardAllocationDecider(clusterSettings),
             new ReplicaAfterPrimaryActiveAllocationDecider()
         );
 
@@ -544,7 +544,7 @@ public void testUnassignedRespectsDesiredBalance() {
     public void testUnassignedAllocationPredictsDiskUsage() {
         final var discoveryNodes = discoveryNodes(1);
         final var metadata = Metadata.builder();
-        final var routingTable = RoutingTable.builder();
+        final var routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
 
         final var existingIndexMetadata = randomPriorityIndex("index-existing", 1, 0);
         metadata.put(existingIndexMetadata, true);
@@ -601,7 +601,7 @@ public void testUnassignedAllocationPredictsDiskUsage() {
             routingAllocation -> reconcile(routingAllocation, desiredBalance),
             () -> clusterInfo,
             () -> snapshotShardSizeInfo,
-            new SameShardAllocationDecider(settings, clusterSettings),
+            new SameShardAllocationDecider(clusterSettings),
             new ReplicaAfterPrimaryActiveAllocationDecider()
         );
 
@@ -619,7 +619,7 @@ public void testUnassignedAllocationPredictsDiskUsage() {
     public void testUnassignedSkipsEquivalentReplicas() {
         final var discoveryNodes = discoveryNodes(2);
         final var metadata = Metadata.builder();
-        final var routingTable = RoutingTable.builder();
+        final var routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
 
         final var indexMetadata = randomPriorityIndex("index-0", 1, between(0, 5));
         metadata.put(indexMetadata, true);
@@ -639,7 +639,7 @@ public void testUnassignedSkipsEquivalentReplicas() {
         final var desiredBalance = desiredBalance(clusterState, (shardId, nodeId) -> true);
         final var allocationService = createTestAllocationService(
             routingAllocation -> reconcile(routingAllocation, desiredBalance),
-            new SameShardAllocationDecider(settings, clusterSettings),
+            new SameShardAllocationDecider(clusterSettings),
             new ReplicaAfterPrimaryActiveAllocationDecider(),
             new AllocationDecider() {
                 @Override
@@ -678,7 +678,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
     public void testUnassignedSetsAllocationStatusOnUnassignedShards() {
         final var discoveryNodes = discoveryNodes(2);
         final var metadata = Metadata.builder();
-        final var routingTable = RoutingTable.builder();
+        final var routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
 
         final var indexMetadata = randomPriorityIndex("index-0", 1, between(0, 5));
         metadata.put(indexMetadata, true);
@@ -698,7 +698,7 @@ public void testUnassignedSetsAllocationStatusOnUnassignedShards() {
         final var desiredBalance = desiredBalance(clusterState, (shardId, nodeId) -> true);
         final var allocationService = createTestAllocationService(
             routingAllocation -> reconcile(routingAllocation, desiredBalance),
-            new SameShardAllocationDecider(settings, clusterSettings),
+            new SameShardAllocationDecider(clusterSettings),
             new ReplicaAfterPrimaryActiveAllocationDecider(),
             new AllocationDecider() {
                 @Override
@@ -736,7 +736,7 @@ public void testUnassignedPrimariesThrottlingAndFallback() {
 
         final var discoveryNodes = discoveryNodes(2);
         final var metadata = Metadata.builder();
-        final var routingTable = RoutingTable.builder();
+        final var routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
 
         final var indexMetadata0 = randomPriorityIndex("index-0", 2, 0);
         metadata.put(indexMetadata0, true);
@@ -756,9 +756,9 @@ public void testUnassignedPrimariesThrottlingAndFallback() {
 
         final var allocationService = createTestAllocationService(
             routingAllocation -> reconcile(routingAllocation, desiredBalance),
-            new SameShardAllocationDecider(settings, clusterSettings),
+            new SameShardAllocationDecider(clusterSettings),
             new ReplicaAfterPrimaryActiveAllocationDecider(),
-            new ThrottlingAllocationDecider(settings, clusterSettings),
+            new ThrottlingAllocationDecider(clusterSettings),
             new AllocationDecider() {
                 @Override
                 public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
@@ -807,7 +807,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
     public void testMoveShards() {
         final var discoveryNodes = discoveryNodes(4);
         final var metadata = Metadata.builder();
-        final var routingTable = RoutingTable.builder();
+        final var routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
 
         final var indexMetadata = randomPriorityIndex("index-0", 3, 1);
         metadata.put(indexMetadata, true);
@@ -834,9 +834,9 @@ public void testMoveShards() {
         final var desiredBalance = new AtomicReference<>(desiredBalance(clusterState, (shardId, nodeId) -> true));
         final var allocationService = createTestAllocationService(
             routingAllocation -> reconcile(routingAllocation, desiredBalance.get()),
-            new SameShardAllocationDecider(settings, clusterSettings),
+            new SameShardAllocationDecider(clusterSettings),
             new ReplicaAfterPrimaryActiveAllocationDecider(),
-            new ThrottlingAllocationDecider(settings, clusterSettings),
+            new ThrottlingAllocationDecider(clusterSettings),
             new FilterAllocationDecider(settings, clusterSettings),
             new NodeShutdownAllocationDecider(),
             new NodeReplacementAllocationDecider(),
@@ -883,8 +883,8 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocat
         // The next reroute starts moving shards to node-2 and node-3, but interleaves the decisions between node-0 and node-1 for fairness.
         // There's an inbound throttle of 1 but no outbound throttle, so without the interleaving one node would relocate 2 shards.
         final var reroutedState = allocationService.reroute(clusterState, "test", ActionListener.noop());
-        assertThat(reroutedState.getRoutingNodes().node("node-0").shardsWithState(ShardRoutingState.RELOCATING), hasSize(1));
-        assertThat(reroutedState.getRoutingNodes().node("node-1").shardsWithState(ShardRoutingState.RELOCATING), hasSize(1));
+        assertThat(reroutedState.getRoutingNodes().node("node-0").numberOfShardsWithState(ShardRoutingState.RELOCATING), equalTo(1));
+        assertThat(reroutedState.getRoutingNodes().node("node-1").numberOfShardsWithState(ShardRoutingState.RELOCATING), equalTo(1));
 
         // Ensuring that we check the shortcut two-param canAllocate() method up front
         canAllocateRef.set(Decision.NO);
@@ -925,13 +925,13 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocat
             "test",
             ActionListener.noop()
         );
-        assertThat(shuttingDownState.getRoutingNodes().node("node-2").shardsWithState(ShardRoutingState.INITIALIZING), hasSize(1));
+        assertThat(shuttingDownState.getRoutingNodes().node("node-2").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1));
     }
 
     public void testRebalance() {
         final var discoveryNodes = discoveryNodes(4);
         final var metadata = Metadata.builder();
-        final var routingTable = RoutingTable.builder();
+        final var routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
 
         final var indexMetadata = randomPriorityIndex("index-0", 3, 1);
         metadata.put(indexMetadata, true);
@@ -955,9 +955,9 @@ public void testRebalance() {
         );
         final var allocationService = createTestAllocationService(
             routingAllocation -> reconcile(routingAllocation, desiredBalance.get()),
-            new SameShardAllocationDecider(settings, clusterSettings),
+            new SameShardAllocationDecider(clusterSettings),
             new ReplicaAfterPrimaryActiveAllocationDecider(),
-            new ThrottlingAllocationDecider(settings, clusterSettings),
+            new ThrottlingAllocationDecider(clusterSettings),
             new AllocationDecider() {
                 @Override
                 public Decision canRebalance(RoutingAllocation allocation) {
@@ -1011,8 +1011,8 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocat
         // The next reroute starts moving shards to node-2 and node-3, but interleaves the decisions between node-0 and node-1 for fairness.
         // There's an inbound throttle of 1 but no outbound throttle, so without the interleaving one node would relocate 2 shards.
         final var reroutedState = allocationService.reroute(clusterState, "test", ActionListener.noop());
-        assertThat(reroutedState.getRoutingNodes().node("node-0").shardsWithState(ShardRoutingState.RELOCATING), hasSize(1));
-        assertThat(reroutedState.getRoutingNodes().node("node-1").shardsWithState(ShardRoutingState.RELOCATING), hasSize(1));
+        assertThat(reroutedState.getRoutingNodes().node("node-0").numberOfShardsWithState(ShardRoutingState.RELOCATING), equalTo(1));
+        assertThat(reroutedState.getRoutingNodes().node("node-1").numberOfShardsWithState(ShardRoutingState.RELOCATING), equalTo(1));
     }
 
     public void testDoNotRebalanceToTheNodeThatNoLongerExists() {
@@ -1024,7 +1024,6 @@ public void testDoNotRebalanceToTheNodeThatNoLongerExists() {
                     .put(SETTING_NUMBER_OF_REPLICAS, 0)
                     .put(SETTING_INDEX_VERSION_CREATED.getKey(), Version.CURRENT)
             )
-            .system(randomBoolean())
             .build();
         final var index = indexMetadata.getIndex();
         final var shardId = new ShardId(index, 0);
@@ -1101,7 +1100,7 @@ public void allocate(RoutingAllocation allocation) {
             public ShardAllocationDecision decideShardAllocation(ShardRouting shard, RoutingAllocation allocation) {
                 throw new AssertionError("should not be called");
             }
-        }, clusterInfoService, snapshotsInfoService);
+        }, clusterInfoService, snapshotsInfoService, TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         allocationService.setExistingShardsAllocators(Map.of(GatewayAllocator.ALLOCATOR_NAME, new NoOpExistingShardsAllocator()));
         return allocationService;
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocatorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocatorTests.java
index e41878f7107a..b1408049104d 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocatorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocatorTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
@@ -33,7 +34,6 @@
 import org.elasticsearch.cluster.service.ClusterApplierService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.FakeThreadPoolMasterService;
-import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
 import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor;
@@ -58,19 +58,20 @@
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_VERSION_CREATED;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
+import static org.elasticsearch.common.settings.ClusterSettings.createBuiltInClusterSettings;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
 
 public class DesiredBalanceShardsAllocatorTests extends ESTestCase {
 
+    private static final String LOCAL_NODE_ID = "node-1";
+    private static final String OTHER_NODE_ID = "node-2";
+
     public void testGatewayAllocatorPreemptsAllocation() {
+        final var nodeId = randomFrom(LOCAL_NODE_ID, OTHER_NODE_ID);
         testAllocate(
-            (allocation, unassignedAllocationHandler) -> unassignedAllocationHandler.initialize(
-                allocation.nodes().getLocalNodeId(),
-                null,
-                0L,
-                allocation.changes()
-            ),
-            routingTable -> assertTrue(routingTable.index("test-index").shard(0).primaryShard().assignedToNode())
+            (allocation, unassignedAllocationHandler) -> unassignedAllocationHandler.initialize(nodeId, null, 0L, allocation.changes()),
+            routingTable -> assertEquals(nodeId, routingTable.index("test-index").shard(0).primaryShard().currentNodeId())
         );
     }
 
@@ -106,18 +107,20 @@ public void testAllocate(
         var deterministicTaskQueue = new DeterministicTaskQueue();
         var threadPool = deterministicTaskQueue.getThreadPool();
 
-        var localNode = createDiscoveryNode("node-1");
+        var localNode = createDiscoveryNode(LOCAL_NODE_ID);
+        var otherNode = createDiscoveryNode(OTHER_NODE_ID);
         var initialState = ClusterState.builder(new ClusterName(ClusterServiceUtils.class.getSimpleName()))
-            .nodes(DiscoveryNodes.builder().add(localNode).localNodeId(localNode.getId()).masterNodeId(localNode.getId()))
+            .nodes(DiscoveryNodes.builder().add(localNode).add(otherNode).localNodeId(localNode.getId()).masterNodeId(localNode.getId()))
             .blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK)
             .build();
 
-        var clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        var settings = Settings.EMPTY;
+        var clusterSettings = createBuiltInClusterSettings(settings);
         var clusterService = new ClusterService(
-            Settings.EMPTY,
-            clusterSettings,
-            new FakeThreadPoolMasterService("node-1", "test", threadPool, deterministicTaskQueue::scheduleNow),
-            new ClusterApplierService("node-1", Settings.EMPTY, clusterSettings, threadPool) {
+            settings,
+            createBuiltInClusterSettings(settings),
+            new FakeThreadPoolMasterService(LOCAL_NODE_ID, "test", threadPool, deterministicTaskQueue::scheduleNow),
+            new ClusterApplierService(LOCAL_NODE_ID, Settings.EMPTY, clusterSettings, threadPool) {
                 @Override
                 protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() {
                     return deterministicTaskQueue.getPrioritizedEsThreadPoolExecutor();
@@ -139,10 +142,14 @@ public ClusterState apply(ClusterState clusterState, Consumer<RoutingAllocation>
             }
         };
 
-        var allocationService = createAllocationService(
-            new DesiredBalanceShardsAllocator(createShardsAllocator(), threadPool, clusterService, reconcileAction),
-            createGatewayAllocator(allocateUnassigned)
+        final var desiredBalanceShardsAllocator = new DesiredBalanceShardsAllocator(
+            clusterSettings,
+            createShardsAllocator(),
+            threadPool,
+            clusterService,
+            reconcileAction
         );
+        var allocationService = createAllocationService(desiredBalanceShardsAllocator, createGatewayAllocator(allocateUnassigned));
         allocationServiceRef.set(allocationService);
 
         var listenerCalled = new AtomicBoolean(false);
@@ -152,7 +159,10 @@ public ClusterState execute(ClusterState currentState) {
                 var indexMetadata = createIndex("test-index");
                 var newState = ClusterState.builder(currentState)
                     .metadata(Metadata.builder(currentState.metadata()).put(indexMetadata, true))
-                    .routingTable(RoutingTable.builder(currentState.routingTable()).addAsNew(indexMetadata))
+                    .routingTable(
+                        RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY, currentState.routingTable())
+                            .addAsNew(indexMetadata)
+                    )
                     .build();
                 return allocationService.reroute(
                     newState,
@@ -173,7 +183,17 @@ public void onFailure(Exception e) {
 
         try {
             assertTrue(listenerCalled.get());
-            verifier.accept(clusterService.state().routingTable());
+            final var routingTable = clusterService.state().routingTable();
+            verifier.accept(routingTable);
+            final var desiredBalance = desiredBalanceShardsAllocator.getDesiredBalance();
+            for (final var indexRoutingTable : routingTable) {
+                for (int shardId = 0; shardId < indexRoutingTable.size(); shardId++) {
+                    final var shardRoutingTable = indexRoutingTable.shard(shardId);
+                    for (final var assignedShard : shardRoutingTable.assignedShards()) {
+                        assertThat(desiredBalance.getAssignment(assignedShard.shardId()).nodeIds(), hasItem(assignedShard.currentNodeId()));
+                    }
+                }
+            }
         } finally {
             clusterService.close();
         }
@@ -207,7 +227,7 @@ public ClusterState apply(ClusterState clusterState, Consumer<RoutingAllocation>
             shardsAllocator,
             threadPool,
             clusterService,
-            new DesiredBalanceComputer(shardsAllocator) {
+            new DesiredBalanceComputer(createBuiltInClusterSettings(), threadPool, shardsAllocator) {
                 @Override
                 public DesiredBalance compute(
                     DesiredBalance previousDesiredBalance,
@@ -241,7 +261,10 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                 var indexMetadata = createIndex(indexName);
                 var newState = ClusterState.builder(currentState)
                     .metadata(Metadata.builder(currentState.metadata()).put(indexMetadata, true))
-                    .routingTable(RoutingTable.builder(currentState.routingTable()).addAsNew(indexMetadata))
+                    .routingTable(
+                        RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY, currentState.routingTable())
+                            .addAsNew(indexMetadata)
+                    )
                     .build();
                 return allocationService.reroute(newState, "test", ActionListener.wrap(response -> {
                     assertThat(
@@ -283,8 +306,8 @@ public void testFailListenersOnNoLongerMasterException() throws InterruptedExcep
         var newMasterElected = new CountDownLatch(1);
         var clusterStateUpdatesExecuted = new CountDownLatch(1);
 
-        var node1 = createDiscoveryNode("node-1");
-        var node2 = createDiscoveryNode("node-2");
+        var node1 = createDiscoveryNode(LOCAL_NODE_ID);
+        var node2 = createDiscoveryNode(OTHER_NODE_ID);
         var initial = ClusterState.builder(ClusterName.DEFAULT)
             .nodes(DiscoveryNodes.builder().add(node1).add(node2).localNodeId(node1.getId()).masterNodeId(node1.getId()))
             .build();
@@ -305,7 +328,7 @@ public ClusterState apply(ClusterState clusterState, Consumer<RoutingAllocation>
             shardsAllocator,
             threadPool,
             clusterService,
-            new DesiredBalanceComputer(shardsAllocator) {
+            new DesiredBalanceComputer(createBuiltInClusterSettings(), threadPool, shardsAllocator) {
                 @Override
                 public DesiredBalance compute(
                     DesiredBalance previousDesiredBalance,
@@ -334,7 +357,10 @@ public ClusterState execute(ClusterState currentState) {
                 var indexMetadata = createIndex("index-1");
                 var newState = ClusterState.builder(currentState)
                     .metadata(Metadata.builder(currentState.metadata()).put(indexMetadata, true))
-                    .routingTable(RoutingTable.builder(currentState.routingTable()).addAsNew(indexMetadata))
+                    .routingTable(
+                        RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY, currentState.routingTable())
+                            .addAsNew(indexMetadata)
+                    )
                     .build();
                 return allocationService.reroute(
                     newState,
@@ -406,7 +432,8 @@ private static AllocationService createAllocationService(
             gatewayAllocator,
             desiredBalanceShardsAllocator,
             () -> ClusterInfo.EMPTY,
-            () -> SnapshotShardSizeInfo.EMPTY
+            () -> SnapshotShardSizeInfo.EMPTY,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
     }
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceStatsTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceStatsTests.java
index 721dec54dedf..c8031a0d7cc3 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceStatsTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.Locale;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -39,7 +38,7 @@ protected DesiredBalanceStats createTestInstance() {
     }
 
     @Override
-    protected DesiredBalanceStats mutateInstance(DesiredBalanceStats instance) throws IOException {
+    protected DesiredBalanceStats mutateInstance(DesiredBalanceStats instance) {
         return createTestInstance();
     }
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersTests.java
index 533a499ce33e..98aa2a1a2a35 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersTests.java
@@ -328,7 +328,8 @@ private static ShardRouting createShardRouting(Index index) {
             new ShardId(index, 0),
             true,
             RecoverySource.ExistingStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "_message")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "_message"),
+            ShardRouting.Role.DEFAULT
         );
     }
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
index d063e4975f60..863169d34432 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.DiskUsage;
 import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.RestoreInProgress;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -55,24 +56,24 @@
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotShardSizeInfo;
+import org.elasticsearch.snapshots.SnapshotsInfoService;
 import org.elasticsearch.test.gateway.TestGatewayAllocator;
 
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Stream;
 
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.singleton;
 import static org.elasticsearch.cluster.routing.RoutingNodesHelper.shardsWithState;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
 import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING;
+import static org.elasticsearch.common.settings.ClusterSettings.createBuiltInClusterSettings;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
@@ -81,10 +82,6 @@
 
 public class DiskThresholdDeciderTests extends ESAllocationTestCase {
 
-    DiskThresholdDecider makeDecider(Settings settings) {
-        return new DiskThresholdDecider(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
-    }
-
     private void doTestDiskThreshold(boolean testMaxHeadroom) {
         Settings.Builder diskSettings = Settings.builder()
             .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true)
@@ -123,32 +120,12 @@ private void doTestDiskThreshold(boolean testMaxHeadroom) {
         shardSizes.put("[test][0][r]", exactFreeSpaceForHighWatermark);
         final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
 
-        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        AllocationDeciders deciders = new AllocationDeciders(
-            new HashSet<>(Arrays.asList(new SameShardAllocationDecider(Settings.EMPTY, clusterSettings), makeDecider(diskSettings.build())))
-        );
-
-        ClusterInfoService cis = () -> {
-            logger.info("--> calling fake getClusterInfo");
-            return clusterInfo;
-        };
-        AllocationService strategy = new AllocationService(
-            deciders,
-            new TestGatewayAllocator(),
-            new BalancedShardsAllocator(Settings.EMPTY),
-            cis,
-            EmptySnapshotsInfoService.INSTANCE
-        );
-
-        Metadata metadata = Metadata.builder()
-            .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
-            .build();
-
-        final RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        AllocationService strategy = createAllocationService(clusterInfo, createDiskThresholdDecider(diskSettings.build()));
 
+        var indexMetadata = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1).build();
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
-            .metadata(metadata)
-            .routingTable(initialRoutingTable)
+            .metadata(Metadata.builder().put(indexMetadata, false).build())
+            .routingTable(RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(indexMetadata).build())
             .build();
 
         logger.info("--> adding two nodes");
@@ -216,18 +193,7 @@ private void doTestDiskThreshold(boolean testMaxHeadroom) {
                 .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "60%")
                 .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.7);
         }
-
-        deciders = new AllocationDeciders(
-            new HashSet<>(Arrays.asList(new SameShardAllocationDecider(Settings.EMPTY, clusterSettings), makeDecider(diskSettings.build())))
-        );
-
-        strategy = new AllocationService(
-            deciders,
-            new TestGatewayAllocator(),
-            new BalancedShardsAllocator(Settings.EMPTY),
-            cis,
-            EmptySnapshotsInfoService.INSTANCE
-        );
+        strategy = createAllocationService(clusterInfo, createDiskThresholdDecider(diskSettings.build()));
 
         clusterState = strategy.reroute(clusterState, "reroute", ActionListener.noop());
         logShardStates(clusterState);
@@ -259,18 +225,7 @@ private void doTestDiskThreshold(boolean testMaxHeadroom) {
                 .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), 0.5)
                 .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), 0.6);
         }
-
-        deciders = new AllocationDeciders(
-            new HashSet<>(Arrays.asList(new SameShardAllocationDecider(Settings.EMPTY, clusterSettings), makeDecider(diskSettings.build())))
-        );
-
-        strategy = new AllocationService(
-            deciders,
-            new TestGatewayAllocator(),
-            new BalancedShardsAllocator(Settings.EMPTY),
-            cis,
-            EmptySnapshotsInfoService.INSTANCE
-        );
+        strategy = createAllocationService(clusterInfo, createDiskThresholdDecider(diskSettings.build()));
 
         clusterState = strategy.reroute(clusterState, "reroute", ActionListener.noop());
 
@@ -331,33 +286,12 @@ public void testDiskThresholdWithAbsoluteSizes() {
         shardSizes.put("[test][0][r]", 10L);
         final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
 
-        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        AllocationDeciders deciders = new AllocationDeciders(
-            new HashSet<>(Arrays.asList(new SameShardAllocationDecider(Settings.EMPTY, clusterSettings), makeDecider(diskSettings)))
-        );
-
-        ClusterInfoService cis = () -> {
-            logger.info("--> calling fake getClusterInfo");
-            return clusterInfo;
-        };
-
-        AllocationService strategy = new AllocationService(
-            deciders,
-            new TestGatewayAllocator(),
-            new BalancedShardsAllocator(Settings.EMPTY),
-            cis,
-            EmptySnapshotsInfoService.INSTANCE
-        );
-
-        Metadata metadata = Metadata.builder()
-            .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(2))
-            .build();
-
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        AllocationService strategy = createAllocationService(clusterInfo, createDiskThresholdDecider(diskSettings));
 
+        var indexMetadata = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(2).build();
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
-            .metadata(metadata)
-            .routingTable(initialRoutingTable)
+            .metadata(Metadata.builder().put(indexMetadata, false).build())
+            .routingTable(RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(indexMetadata).build())
             .build();
 
         logger.info("--> adding node1 and node2 node");
@@ -386,29 +320,21 @@ public void testDiskThresholdWithAbsoluteSizes() {
         usages = new HashMap<>(usages);
         usages.put(nodeWithoutPrimary, new DiskUsage(nodeWithoutPrimary, "", "/dev/null", 100, 35)); // 65% used
         final ClusterInfo clusterInfo2 = new DevNullClusterInfo(usages, usages, shardSizes);
-        cis = () -> {
-            logger.info("--> calling fake getClusterInfo");
-            return clusterInfo2;
-        };
-        strategy = new AllocationService(
-            deciders,
-            new TestGatewayAllocator(),
-            new BalancedShardsAllocator(Settings.EMPTY),
-            cis,
-            EmptySnapshotsInfoService.INSTANCE
-        );
 
-        clusterState = strategy.reroute(clusterState, "reroute", ActionListener.noop());
+        strategy = createAllocationService(clusterInfo2, createDiskThresholdDecider(diskSettings));
+
         logShardStates(clusterState);
 
+        logger.info("--> start the shards (primaries)");
+        clusterState = startInitializingShardsAndReroute(strategy, clusterState);
+
         // Now the replica should be able to initialize
-        assertThat(shardsWithState(clusterState.getRoutingNodes(), INITIALIZING).size(), equalTo(2));
+        assertThat(shardsWithState(clusterState.getRoutingNodes(), INITIALIZING).size(), equalTo(1));
 
-        logger.info("--> start the shards (primaries)");
         clusterState = startInitializingShardsAndReroute(strategy, clusterState);
 
         logShardStates(clusterState);
-        // Assert that we're able to start the primary and replica, since they were both initializing
+        // Assert that we're able to start the replica
         assertThat(shardsWithState(clusterState.getRoutingNodes(), ShardRoutingState.STARTED).size(), equalTo(2));
         // Assert that node1 got a single shard (the primary), even though its disk usage is too high
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
@@ -449,18 +375,7 @@ public void testDiskThresholdWithAbsoluteSizes() {
             .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "30b")
             .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "20b")
             .build();
-
-        deciders = new AllocationDeciders(
-            new HashSet<>(Arrays.asList(new SameShardAllocationDecider(Settings.EMPTY, clusterSettings), makeDecider(diskSettings)))
-        );
-
-        strategy = new AllocationService(
-            deciders,
-            new TestGatewayAllocator(),
-            new BalancedShardsAllocator(Settings.EMPTY),
-            cis,
-            EmptySnapshotsInfoService.INSTANCE
-        );
+        strategy = createAllocationService(clusterInfo2, createDiskThresholdDecider(diskSettings));
 
         clusterState = strategy.reroute(clusterState, "reroute", ActionListener.noop());
         logShardStates(clusterState);
@@ -482,18 +397,7 @@ public void testDiskThresholdWithAbsoluteSizes() {
             .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "40b")
             .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "30b")
             .build();
-
-        deciders = new AllocationDeciders(
-            new HashSet<>(Arrays.asList(new SameShardAllocationDecider(Settings.EMPTY, clusterSettings), makeDecider(diskSettings)))
-        );
-
-        strategy = new AllocationService(
-            deciders,
-            new TestGatewayAllocator(),
-            new BalancedShardsAllocator(Settings.EMPTY),
-            cis,
-            EmptySnapshotsInfoService.INSTANCE
-        );
+        strategy = createAllocationService(clusterInfo2, createDiskThresholdDecider(diskSettings));
 
         clusterState = strategy.reroute(clusterState, "reroute", ActionListener.noop());
 
@@ -521,10 +425,10 @@ public void testDiskThresholdWithAbsoluteSizes() {
         clusterState = startInitializingShardsAndReroute(strategy, clusterState);
 
         logShardStates(clusterState);
-        // primary shard already has been relocated away
-        assertThat(clusterState.getRoutingNodes().node(nodeWithPrimary).size(), equalTo(0));
-        // node with increased space still has its shard
-        assertThat(clusterState.getRoutingNodes().node(nodeWithoutPrimary).size(), equalTo(1));
+        assertThat(
+            clusterState.getRoutingNodes().node(nodeWithPrimary).size() + clusterState.getRoutingNodes().node(nodeWithoutPrimary).size(),
+            equalTo(1)
+        );
         assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node4").size(), equalTo(1));
 
@@ -587,51 +491,21 @@ private void doTestDiskThresholdWithShardSizes(boolean testMaxHeadroom) {
             Map.of("[test][0][p]", testMaxHeadroom ? ByteSizeValue.ofGb(10).getBytes() : 10L)
         );
 
-        AllocationDeciders deciders = new AllocationDeciders(
-            new HashSet<>(
-                Arrays.asList(
-                    new SameShardAllocationDecider(
-                        Settings.EMPTY,
-                        new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
-                    ),
-                    makeDecider(diskSettings.build())
-                )
-            )
-        );
-
-        ClusterInfoService cis = () -> {
-            logger.info("--> calling fake getClusterInfo");
-            return clusterInfo;
-        };
-
-        AllocationService strategy = new AllocationService(
-            deciders,
-            new TestGatewayAllocator(),
-            new BalancedShardsAllocator(Settings.EMPTY),
-            cis,
-            EmptySnapshotsInfoService.INSTANCE
-        );
-
-        Metadata metadata = Metadata.builder()
-            .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
-            .build();
-
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        AllocationService strategy = createAllocationService(clusterInfo, createDiskThresholdDecider(diskSettings.build()));
 
+        var indexMetadata = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0).build();
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
-            .metadata(metadata)
-            .routingTable(routingTable)
+            .metadata(Metadata.builder().put(indexMetadata, false).build())
+            .routingTable(RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(indexMetadata).build())
             .build();
         logger.info("--> adding node1");
         // node2 is added because DiskThresholdDecider automatically ignore single-node clusters
         clusterState = ClusterState.builder(clusterState)
             .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")))
             .build();
-        routingTable = strategy.reroute(clusterState, "reroute", ActionListener.noop()).routingTable();
-        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        clusterState = strategy.reroute(clusterState, "reroute", ActionListener.noop());
         logger.info("--> start the shards (primaries)");
-        routingTable = startInitializingShardsAndReroute(strategy, clusterState).routingTable();
-        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        clusterState = startInitializingShardsAndReroute(strategy, clusterState);
         logShardStates(clusterState);
 
         // Shard can't be allocated to node1 (or node2) because it would cause too much usage
@@ -664,40 +538,12 @@ public void testUnknownDiskUsage() {
         shardSizes.put("[test][0][r]", 10L); // 10 bytes
         final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
 
-        AllocationDeciders deciders = new AllocationDeciders(
-            new HashSet<>(
-                Arrays.asList(
-                    new SameShardAllocationDecider(
-                        Settings.EMPTY,
-                        new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
-                    ),
-                    makeDecider(diskSettings)
-                )
-            )
-        );
-
-        ClusterInfoService cis = () -> {
-            logger.info("--> calling fake getClusterInfo");
-            return clusterInfo;
-        };
-
-        AllocationService strategy = new AllocationService(
-            deciders,
-            new TestGatewayAllocator(),
-            new BalancedShardsAllocator(Settings.EMPTY),
-            cis,
-            EmptySnapshotsInfoService.INSTANCE
-        );
-
-        Metadata metadata = Metadata.builder()
-            .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
-            .build();
-
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        AllocationService strategy = createAllocationService(clusterInfo, createDiskThresholdDecider(diskSettings));
 
+        var indexMetadata = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0).build();
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
-            .metadata(metadata)
-            .routingTable(routingTable)
+            .metadata(Metadata.builder().put(indexMetadata, false).build())
+            .routingTable(RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(indexMetadata).build())
             .build();
         logger.info("--> adding node1");
         clusterState = ClusterState.builder(clusterState)
@@ -706,16 +552,14 @@ public void testUnknownDiskUsage() {
                                                                                      // automatically ignore single-node clusters
             )
             .build();
-        routingTable = strategy.reroute(clusterState, "reroute", ActionListener.noop()).routingTable();
-        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        clusterState = strategy.reroute(clusterState, "reroute", ActionListener.noop());
 
         // Shard can be allocated to node1, even though it only has 25% free,
         // because it's a primary that's never been allocated before
         assertThat(shardsWithState(clusterState.getRoutingNodes(), INITIALIZING).size(), equalTo(1));
 
         logger.info("--> start the shards (primaries)");
-        routingTable = startInitializingShardsAndReroute(strategy, clusterState).routingTable();
-        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        clusterState = startInitializingShardsAndReroute(strategy, clusterState);
         logShardStates(clusterState);
 
         // A single shard is started on node1, even though it normally would not
@@ -727,7 +571,6 @@ public void testUnknownDiskUsage() {
 
     public void testAverageUsage() {
         RoutingNode rn = RoutingNodesHelper.routingNode("node1", newNode("node1"));
-        DiskThresholdDecider decider = makeDecider(Settings.EMPTY);
 
         Map<String, DiskUsage> usages = new HashMap<>();
         usages.put("node2", new DiskUsage("node2", "n2", "/dev/null", 100, 50)); // 50% used
@@ -776,45 +619,33 @@ private void doTestShardRelocationsTakenIntoAccount(boolean testMaxHeadroom) {
         shardSizes.put("[test2][0][r]", testMaxHeadroom ? ByteSizeValue.ofGb(1).getBytes() : 1L);
         final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
 
-        DiskThresholdDecider decider = makeDecider(diskSettings.build());
-        final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        AllocationDeciders deciders = new AllocationDeciders(
-            new HashSet<>(
-                Arrays.asList(
-                    new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
-                    new EnableAllocationDecider(
-                        Settings.builder().put(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none").build(),
-                        clusterSettings
-                    ),
-                    decider
-                )
-            )
-        );
-
         final AtomicReference<ClusterInfo> clusterInfoReference = new AtomicReference<>(clusterInfo);
-        final ClusterInfoService cis = clusterInfoReference::get;
 
-        AllocationService strategy = new AllocationService(
-            deciders,
-            new TestGatewayAllocator(),
-            new BalancedShardsAllocator(Settings.EMPTY),
-            cis,
-            EmptySnapshotsInfoService.INSTANCE
+        AllocationService strategy = createAllocationService(
+            clusterInfoReference::get,
+            EmptySnapshotsInfoService.INSTANCE,
+            createEnableAllocationDecider(Settings.builder().put(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none").build()),
+            createDiskThresholdDecider(diskSettings.build())
         );
 
-        Metadata metadata = Metadata.builder()
-            .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
-            .put(IndexMetadata.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
+        var indexMetadata1 = IndexMetadata.builder("test")
+            .settings(settings(Version.CURRENT))
+            .numberOfShards(1)
+            .numberOfReplicas(1)
             .build();
-
-        RoutingTable initialRoutingTable = RoutingTable.builder()
-            .addAsNew(metadata.index("test"))
-            .addAsNew(metadata.index("test2"))
+        var indexMetadata2 = IndexMetadata.builder("test2")
+            .settings(settings(Version.CURRENT))
+            .numberOfShards(1)
+            .numberOfReplicas(1)
             .build();
-
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
-            .metadata(metadata)
-            .routingTable(initialRoutingTable)
+            .metadata(Metadata.builder().put(indexMetadata1, false).put(indexMetadata2, false).build())
+            .routingTable(
+                RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+                    .addAsNew(indexMetadata1)
+                    .addAsNew(indexMetadata2)
+                    .build()
+            )
             .build();
 
         logger.info("--> adding two nodes");
@@ -824,12 +655,17 @@ private void doTestShardRelocationsTakenIntoAccount(boolean testMaxHeadroom) {
         clusterState = strategy.reroute(clusterState, "reroute", ActionListener.noop());
         logShardStates(clusterState);
 
-        // shards should be initializing
-        assertThat(shardsWithState(clusterState.getRoutingNodes(), INITIALIZING).size(), equalTo(4));
+        // primaries should be initializing
+        assertThat(shardsWithState(clusterState.getRoutingNodes(), INITIALIZING).size(), equalTo(2));
 
         logger.info("--> start the shards");
         clusterState = startInitializingShardsAndReroute(strategy, clusterState);
 
+        // replicas should now be initializing
+        assertThat(shardsWithState(clusterState.getRoutingNodes(), INITIALIZING).size(), equalTo(2));
+
+        clusterState = startInitializingShardsAndReroute(strategy, clusterState);
+
         logShardStates(clusterState);
         // Assert that we're able to start the primary and replicas
         assertThat(shardsWithState(clusterState.getRoutingNodes(), ShardRoutingState.STARTED).size(), equalTo(4));
@@ -989,34 +825,19 @@ private void doTestCanRemainWithShardRelocatingAway(boolean testMaxHeadroom) {
 
         final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
 
-        DiskThresholdDecider diskThresholdDecider = makeDecider(diskSettings.build());
-        Metadata metadata = Metadata.builder()
-            .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(0))
-            .put(IndexMetadata.builder("foo").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
-            .build();
+        DiskThresholdDecider diskThresholdDecider = createDiskThresholdDecider(diskSettings.build());
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).addAsNew(metadata.index("foo")).build();
-
-        DiscoveryNode discoveryNode1 = new DiscoveryNode(
-            "node1",
-            buildNewFakeTransportAddress(),
-            emptyMap(),
-            MASTER_DATA_ROLES,
-            Version.CURRENT
-        );
-        DiscoveryNode discoveryNode2 = new DiscoveryNode(
-            "node2",
-            buildNewFakeTransportAddress(),
-            emptyMap(),
-            MASTER_DATA_ROLES,
-            Version.CURRENT
-        );
-        DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(discoveryNode1).add(discoveryNode2).build();
+        DiscoveryNode discoveryNode1 = newNode("node1");
+        DiscoveryNode discoveryNode2 = newNode("node2");
 
+        var testMetadata = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(0).build();
+        var fooMetadata = IndexMetadata.builder("foo").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0).build();
         ClusterState baseClusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
-            .metadata(metadata)
-            .routingTable(initialRoutingTable)
-            .nodes(discoveryNodes)
+            .metadata(Metadata.builder().put(testMetadata, false).put(fooMetadata, false).build())
+            .routingTable(
+                RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(testMetadata).addAsNew(fooMetadata).build()
+            )
+            .nodes(DiscoveryNodes.builder().add(discoveryNode1).add(discoveryNode2).build())
             .build();
 
         // Two shards consuming each 80% of disk space while 70% is allowed, so shard 0 isn't allowed here
@@ -1091,13 +912,13 @@ private void doTestCanRemainWithShardRelocatingAway(boolean testMaxHeadroom) {
             testMaxHeadroom
                 ? "there is enough disk on this node for the shard to remain, free: [4.9tb]"
                 : "there is enough disk on this node for the shard to remain, free: [60b]",
-            ((Decision.Single) decision).getExplanation()
+            decision.getExplanation()
         );
         decision = diskThresholdDecider.canAllocate(fooRouting, firstRoutingNode, routingAllocation);
         assertThat(decision.type(), equalTo(Decision.Type.NO));
         if (fooRouting.recoverySource().getType() == RecoverySource.Type.EMPTY_STORE) {
             assertThat(
-                ((Decision.Single) decision).getExplanation(),
+                decision.getExplanation(),
                 containsString(
                     testMaxHeadroom
                         ? "the node is above the high watermark cluster setting [cluster.routing.allocation.disk.watermark"
@@ -1109,7 +930,7 @@ private void doTestCanRemainWithShardRelocatingAway(boolean testMaxHeadroom) {
             );
         } else {
             assertThat(
-                ((Decision.Single) decision).getExplanation(),
+                decision.getExplanation(),
                 containsString(
                     testMaxHeadroom
                         ? "the node is above the low watermark cluster setting [cluster.routing.allocation.disk.watermark.low"
@@ -1122,28 +943,7 @@ private void doTestCanRemainWithShardRelocatingAway(boolean testMaxHeadroom) {
         }
 
         // Creating AllocationService instance and the services it depends on...
-        ClusterInfoService cis = () -> {
-            logger.info("--> calling fake getClusterInfo");
-            return clusterInfo;
-        };
-        AllocationDeciders deciders = new AllocationDeciders(
-            new HashSet<>(
-                Arrays.asList(
-                    new SameShardAllocationDecider(
-                        Settings.EMPTY,
-                        new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
-                    ),
-                    diskThresholdDecider
-                )
-            )
-        );
-        AllocationService strategy = new AllocationService(
-            deciders,
-            new TestGatewayAllocator(),
-            new BalancedShardsAllocator(Settings.EMPTY),
-            cis,
-            EmptySnapshotsInfoService.INSTANCE
-        );
+        AllocationService strategy = createAllocationService(clusterInfo, diskThresholdDecider);
         // Ensure that the reroute call doesn't alter the routing table, since the first primary is relocating away
         // and therefore we will have sufficient disk space on node1.
         ClusterState result = strategy.reroute(clusterState, "reroute", ActionListener.noop());
@@ -1194,60 +994,22 @@ private void doTestWatermarksEnabledForSingleDataNode(boolean testMaxHeadroom) {
         Map<String, Long> shardSizes = Map.of("[test][0][p]", testMaxHeadroom ? ByteSizeValue.ofGb(60).getBytes() : 40L);
         final ClusterInfo clusterInfo = new DevNullClusterInfo(usages, usages, shardSizes);
 
-        DiskThresholdDecider diskThresholdDecider = makeDecider(diskSettings);
-        Metadata metadata = Metadata.builder()
-            .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
-            .build();
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
-
-        DiscoveryNode masterNode = new DiscoveryNode(
-            "master",
-            "master",
-            buildNewFakeTransportAddress(),
-            emptyMap(),
-            singleton(DiscoveryNodeRole.MASTER_ROLE),
-            Version.CURRENT
-        );
-        DiscoveryNode dataNode = new DiscoveryNode(
-            "data",
-            "data",
-            buildNewFakeTransportAddress(),
-            emptyMap(),
-            singleton(DiscoveryNodeRole.DATA_ROLE),
-            Version.CURRENT
-        );
-        DiscoveryNodes.Builder discoveryNodesBuilder = DiscoveryNodes.builder().add(dataNode);
+        DiskThresholdDecider diskThresholdDecider = createDiskThresholdDecider(diskSettings);
+
+        var discoveryNodesBuilder = DiscoveryNodes.builder().add(newNode("data", "data", Set.of(DiscoveryNodeRole.DATA_ROLE)));
         if (randomBoolean()) {
-            discoveryNodesBuilder.add(masterNode);
+            discoveryNodesBuilder.add(newNode("master", "master", Set.of(DiscoveryNodeRole.MASTER_ROLE)));
         }
-        DiscoveryNodes discoveryNodes = discoveryNodesBuilder.build();
 
+        var testMetadata = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0).build();
         ClusterState clusterState = ClusterState.builder(new ClusterName("test"))
-            .nodes(discoveryNodes)
-            .metadata(metadata)
-            .routingTable(initialRoutingTable)
+            .nodes(discoveryNodesBuilder.build())
+            .metadata(Metadata.builder().put(testMetadata, false).build())
+            .routingTable(RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(testMetadata).build())
             .build();
 
         // validate that the shard cannot be allocated
-        ClusterInfoService cis = () -> clusterInfo;
-        AllocationDeciders deciders = new AllocationDeciders(
-            new HashSet<>(
-                Arrays.asList(
-                    new SameShardAllocationDecider(
-                        Settings.EMPTY,
-                        new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
-                    ),
-                    diskThresholdDecider
-                )
-            )
-        );
-        AllocationService strategy = new AllocationService(
-            deciders,
-            new TestGatewayAllocator(),
-            new BalancedShardsAllocator(Settings.EMPTY),
-            cis,
-            EmptySnapshotsInfoService.INSTANCE
-        );
+        AllocationService strategy = createAllocationService(clusterInfo, diskThresholdDecider);
         ClusterState result = strategy.reroute(clusterState, "reroute", ActionListener.noop());
 
         ShardRouting shardRouting = result.routingTable().index("test").shard(0).primaryShard();
@@ -1355,53 +1117,41 @@ private void doTestDiskThresholdWithSnapshotShardSizes(boolean testMaxHeadroom)
             new DiskUsage("node1", "n1", "/dev/null", totalBytes, testMaxHeadroom ? ByteSizeValue.ofGb(210).getBytes() : 21)
         );
         usages.put("node2", new DiskUsage("node2", "n2", "/dev/null", totalBytes, testMaxHeadroom ? ByteSizeValue.ofGb(1).getBytes() : 1));
-        final ClusterInfoService clusterInfoService = () -> new DevNullClusterInfo(usages, usages, Map.of());
-
-        final AllocationDeciders deciders = new AllocationDeciders(
-            new HashSet<>(
-                Arrays.asList(
-                    new RestoreInProgressAllocationDecider(),
-                    new SameShardAllocationDecider(
-                        Settings.EMPTY,
-                        new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
-                    ),
-                    makeDecider(diskSettings.build())
-                )
-            )
-        );
 
         final Snapshot snapshot = new Snapshot("_repository", new SnapshotId("_snapshot_name", UUIDs.randomBase64UUID(random())));
         final IndexId indexId = new IndexId("_indexid_name", UUIDs.randomBase64UUID(random()));
         final ShardId shardId = new ShardId(new Index("test", IndexMetadata.INDEX_UUID_NA_VALUE), 0);
 
-        final Metadata metadata = Metadata.builder()
-            .put(
-                IndexMetadata.builder("test")
-                    .settings(settings(Version.CURRENT))
-                    .numberOfShards(1)
-                    .numberOfReplicas(0)
-                    .putInSyncAllocationIds(0, Set.of(AllocationId.newInitializing().getId()))
-            )
-            .build();
-
-        final RoutingTable routingTable = RoutingTable.builder()
-            .addAsNewRestore(
-                metadata.index("test"),
-                new RecoverySource.SnapshotRecoverySource("_restore_uuid", snapshot, Version.CURRENT, indexId),
-                new HashSet<>()
-            )
+        var indexMetadata = IndexMetadata.builder("test")
+            .settings(settings(Version.CURRENT))
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .putInSyncAllocationIds(0, Set.of(AllocationId.newInitializing().getId()))
             .build();
-
-        Map<ShardId, RestoreInProgress.ShardRestoreStatus> shards = Map.of(shardId, new RestoreInProgress.ShardRestoreStatus("node1"));
-
-        final RestoreInProgress.Builder restores = new RestoreInProgress.Builder().add(
-            new RestoreInProgress.Entry("_restore_uuid", snapshot, RestoreInProgress.State.INIT, false, List.of("test"), shards)
-        );
-
         ClusterState clusterState = ClusterState.builder(new ClusterName(getTestName()))
-            .metadata(metadata)
-            .routingTable(routingTable)
-            .putCustom(RestoreInProgress.TYPE, restores.build())
+            .metadata(Metadata.builder().put(indexMetadata, false).build())
+            .routingTable(
+                RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+                    .addAsNewRestore(
+                        indexMetadata,
+                        new RecoverySource.SnapshotRecoverySource("_restore_uuid", snapshot, Version.CURRENT, indexId),
+                        new HashSet<>()
+                    )
+                    .build()
+            )
+            .putCustom(
+                RestoreInProgress.TYPE,
+                new RestoreInProgress.Builder().add(
+                    new RestoreInProgress.Entry(
+                        "_restore_uuid",
+                        snapshot,
+                        RestoreInProgress.State.INIT,
+                        false,
+                        List.of("test"),
+                        Map.of(shardId, new RestoreInProgress.ShardRestoreStatus("node1"))
+                    )
+                ).build()
+            )
             .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")) // node2 is added because DiskThresholdDecider
             // automatically ignore single-node clusters
             )
@@ -1422,12 +1172,12 @@ private void doTestDiskThresholdWithSnapshotShardSizes(boolean testMaxHeadroom)
         assertThat(shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).size(), equalTo(1));
 
         final AtomicReference<SnapshotShardSizeInfo> snapshotShardSizeInfoRef = new AtomicReference<>(SnapshotShardSizeInfo.EMPTY);
-        final AllocationService strategy = new AllocationService(
-            deciders,
-            new TestGatewayAllocator(),
-            new BalancedShardsAllocator(Settings.EMPTY),
-            clusterInfoService,
-            snapshotShardSizeInfoRef::get
+
+        final AllocationService strategy = createAllocationService(
+            () -> new DevNullClusterInfo(usages, usages, Map.of()),
+            snapshotShardSizeInfoRef::get,
+            new RestoreInProgressAllocationDecider(),
+            createDiskThresholdDecider(diskSettings.build())
         );
 
         // reroute triggers snapshot shard size fetching
@@ -1465,7 +1215,11 @@ private void doTestDiskThresholdWithSnapshotShardSizes(boolean testMaxHeadroom)
         logShardStates(clusterState);
 
         assertThat(shardsWithState(clusterState.getRoutingNodes(), UNASSIGNED).size(), equalTo(shouldAllocate ? 0 : 1));
-        assertThat(shardsWithState(clusterState.getRoutingNodes(), "test", INITIALIZING, STARTED).size(), equalTo(shouldAllocate ? 1 : 0));
+        assertThat(
+            shardsWithState(clusterState.getRoutingNodes(), "test", INITIALIZING).size() //
+                + shardsWithState(clusterState.getRoutingNodes(), "test", STARTED).size(),
+            equalTo(shouldAllocate ? 1 : 0)
+        );
     }
 
     public void testDiskThresholdWithSnapshotShardSizesWithPercentages() {
@@ -1495,6 +1249,42 @@ public void logShardStates(ClusterState state) {
         );
     }
 
+    private AllocationService createAllocationService(ClusterInfo clusterInfo, AllocationDecider... allocationDeciders) {
+        return createAllocationService(() -> clusterInfo, EmptySnapshotsInfoService.INSTANCE, allocationDeciders);
+    }
+
+    private AllocationService createAllocationService(
+        ClusterInfoService clusterInfoService,
+        SnapshotsInfoService snapshotShardSizeInfoService,
+        AllocationDecider... allocationDeciders
+    ) {
+        return new AllocationService(
+            new AllocationDeciders(
+                Stream.concat(
+                    Stream.of(createSameShardAllocationDecider(Settings.EMPTY), new ReplicaAfterPrimaryActiveAllocationDecider()),
+                    Stream.of(allocationDeciders)
+                ).toList()
+            ),
+            new TestGatewayAllocator(),
+            new BalancedShardsAllocator(Settings.EMPTY),
+            clusterInfoService,
+            snapshotShardSizeInfoService,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
+        );
+    }
+
+    private DiskThresholdDecider createDiskThresholdDecider(Settings settings) {
+        return new DiskThresholdDecider(settings, createBuiltInClusterSettings(settings));
+    }
+
+    private SameShardAllocationDecider createSameShardAllocationDecider(Settings settings) {
+        return new SameShardAllocationDecider(createBuiltInClusterSettings(settings));
+    }
+
+    private EnableAllocationDecider createEnableAllocationDecider(Settings settings) {
+        return new EnableAllocationDecider(createBuiltInClusterSettings(settings));
+    }
+
     /**
      * ClusterInfo that always reports /dev/null for the shards' data paths.
      */
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
index df1c40aaf160..2c7b8b3ffc2b 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.DiskUsage;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -74,7 +75,8 @@ public void testCanAllocateUsesMaxAvailableSpace() {
             new ShardId(index, 0),
             true,
             EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         DiscoveryNode node_0 = new DiscoveryNode(
             "node_0",
@@ -91,7 +93,9 @@ public void testCanAllocateUsesMaxAvailableSpace() {
             Version.CURRENT
         );
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
@@ -155,7 +159,8 @@ private void doTestCannotAllocateDueToLackOfDiskResources(boolean testMaxHeadroo
             new ShardId(index, 0),
             true,
             EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         DiscoveryNode node_0 = new DiscoveryNode(
             "node_0",
@@ -172,7 +177,9 @@ private void doTestCannotAllocateDueToLackOfDiskResources(boolean testMaxHeadroo
             Version.CURRENT
         );
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
@@ -280,7 +287,8 @@ private void doTestCanRemainUsesLeastAvailableSpace(boolean testMaxHeadroom) {
             new ShardId(indexMetadata.getIndex(), 0),
             true,
             EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         test_0 = ShardRoutingHelper.initialize(test_0, node_0.getId());
         test_0 = ShardRoutingHelper.moveToStarted(test_0);
@@ -290,7 +298,8 @@ private void doTestCanRemainUsesLeastAvailableSpace(boolean testMaxHeadroom) {
             new ShardId(indexMetadata.getIndex(), 1),
             true,
             EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         test_1 = ShardRoutingHelper.initialize(test_1, node_1.getId());
         test_1 = ShardRoutingHelper.moveToStarted(test_1);
@@ -300,7 +309,8 @@ private void doTestCanRemainUsesLeastAvailableSpace(boolean testMaxHeadroom) {
             new ShardId(indexMetadata.getIndex(), 2),
             true,
             EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         test_2 = ShardRoutingHelper.initialize(test_2, node_1.getId());
         test_2 = ShardRoutingHelper.moveToStarted(test_2);
@@ -310,13 +320,14 @@ private void doTestCanRemainUsesLeastAvailableSpace(boolean testMaxHeadroom) {
             new ShardId(indexMetadata.getIndex(), 3),
             true,
             EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         test_3 = ShardRoutingHelper.initialize(test_3, node_1.getId());
         test_3 = ShardRoutingHelper.moveToStarted(test_3);
         // Intentionally not in the shardRoutingMap. We want to test what happens when we don't know where it is.
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(indexMetadata).build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -450,7 +461,7 @@ public void testShardSizeAndRelocatingSize() {
                 .numberOfReplicas(1)
         );
         Metadata metadata = metaBuilder.build();
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         routingTableBuilder.addAsNew(metadata.index("test"));
         routingTableBuilder.addAsNew(metadata.index("other"));
         ClusterState clusterState = ClusterState.builder(
@@ -463,7 +474,8 @@ public void testShardSizeAndRelocatingSize() {
             new ShardId(index, 0),
             false,
             PeerRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         test_0 = ShardRoutingHelper.initialize(test_0, "node1");
         test_0 = ShardRoutingHelper.moveToStarted(test_0);
@@ -473,7 +485,8 @@ public void testShardSizeAndRelocatingSize() {
             new ShardId(index, 1),
             false,
             PeerRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         test_1 = ShardRoutingHelper.initialize(test_1, "node2");
         test_1 = ShardRoutingHelper.moveToStarted(test_1);
@@ -483,7 +496,8 @@ public void testShardSizeAndRelocatingSize() {
             new ShardId(index, 2),
             false,
             PeerRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         test_2 = ShardRoutingHelper.initialize(test_2, "node1");
         test_2 = ShardRoutingHelper.moveToStarted(test_2);
@@ -508,17 +522,20 @@ public void testShardSizeAndRelocatingSize() {
             new ShardId(index, 3),
             false,
             PeerRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         test_3 = ShardRoutingHelper.initialize(test_3, "node1");
         test_3 = ShardRoutingHelper.moveToStarted(test_3);
         assertEquals(0L, DiskThresholdDecider.getExpectedShardSize(test_3, 0L, allocation));
 
+        boolean primary = randomBoolean();
         ShardRouting other_0 = ShardRouting.newUnassigned(
             new ShardId("other", "5678", 0),
-            randomBoolean(),
-            PeerRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            primary,
+            primary ? EmptyStoreRecoverySource.INSTANCE : PeerRecoverySource.INSTANCE,
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         other_0 = ShardRoutingHelper.initialize(other_0, "node2");
         other_0 = ShardRoutingHelper.moveToStarted(other_0);
@@ -578,8 +595,9 @@ public void testTakesIntoAccountExpectedSizeForInitializingSearchableSnapshots()
                     ShardRouting.newUnassigned(
                         new ShardId(index, nodeShardCounter++),
                         true,
-                        PeerRecoverySource.INSTANCE,
-                        new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+                        EmptyStoreRecoverySource.INSTANCE,
+                        new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+                        ShardRouting.Role.DEFAULT
                     ),
                     nodeId
                 );
@@ -640,8 +658,9 @@ private ShardRouting createShard(Index index, String nodeId, int i, int expected
         var unassigned = ShardRouting.newUnassigned(
             new ShardId(index, i),
             true,
-            PeerRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            EmptyStoreRecoverySource.INSTANCE,
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         var initialized = ShardRoutingHelper.initialize(unassigned, nodeId, expectedSize);
         var started = ShardRoutingHelper.moveToStarted(initialized, expectedSize);
@@ -696,7 +715,7 @@ public void testSizeShrinkIndex() {
                 .numberOfReplicas(0)
         );
         Metadata metadata = metaBuilder.build();
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         routingTableBuilder.addAsNew(metadata.index("test"));
         routingTableBuilder.addAsNew(metadata.index("target"));
         routingTableBuilder.addAsNew(metadata.index("target2"));
@@ -721,7 +740,8 @@ public void testSizeShrinkIndex() {
             new ShardId(index, 0),
             true,
             LocalShardsRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         test_0 = ShardRoutingHelper.initialize(test_0, "node1");
         test_0 = ShardRoutingHelper.moveToStarted(test_0);
@@ -730,7 +750,8 @@ public void testSizeShrinkIndex() {
             new ShardId(index, 1),
             true,
             LocalShardsRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         test_1 = ShardRoutingHelper.initialize(test_1, "node2");
         test_1 = ShardRoutingHelper.moveToStarted(test_1);
@@ -739,7 +760,8 @@ public void testSizeShrinkIndex() {
             new ShardId(index, 2),
             true,
             LocalShardsRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         test_2 = ShardRoutingHelper.initialize(test_2, "node1");
 
@@ -747,7 +769,8 @@ public void testSizeShrinkIndex() {
             new ShardId(index, 3),
             true,
             LocalShardsRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         test_3 = ShardRoutingHelper.initialize(test_3, "node1");
         assertEquals(500L, DiskThresholdDecider.getExpectedShardSize(test_3, 0L, allocation));
@@ -759,7 +782,8 @@ public void testSizeShrinkIndex() {
             new ShardId(new Index("target", "5678"), 0),
             true,
             LocalShardsRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         assertEquals(1110L, DiskThresholdDecider.getExpectedShardSize(target, 0L, allocation));
 
@@ -767,7 +791,8 @@ public void testSizeShrinkIndex() {
             new ShardId(new Index("target2", "9101112"), 0),
             true,
             LocalShardsRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         assertEquals(110L, DiskThresholdDecider.getExpectedShardSize(target2, 0L, allocation));
 
@@ -775,7 +800,8 @@ public void testSizeShrinkIndex() {
             new ShardId(new Index("target2", "9101112"), 1),
             true,
             LocalShardsRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         assertEquals(1000L, DiskThresholdDecider.getExpectedShardSize(target2, 0L, allocation));
 
@@ -873,7 +899,8 @@ public void testDecidesYesIfWatermarksIgnored() {
             new ShardId(index, 0),
             true,
             EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         DiscoveryNode node_0 = new DiscoveryNode(
             "node_0",
@@ -890,7 +917,9 @@ public void testDecidesYesIfWatermarksIgnored() {
             Version.CURRENT
         );
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
@@ -949,7 +978,8 @@ public void testCannotForceAllocateOver100PercentUsage() {
             new ShardId(index, 0),
             true,
             EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"),
+            ShardRouting.Role.DEFAULT
         );
         DiscoveryNode node_0 = new DiscoveryNode(
             "node_0",
@@ -966,7 +996,9 @@ public void testCannotForceAllocateOver100PercentUsage() {
             Version.CURRENT
         );
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationShortCircuitTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationShortCircuitTests.java
index 1bddb4744511..38d5ef8c57b7 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationShortCircuitTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationShortCircuitTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -54,7 +55,7 @@ private static ClusterState createClusterStateWithAllShardsAssigned() {
         }
 
         final Metadata.Builder metadataBuilder = Metadata.builder();
-        final RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        final RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         for (int i = randomIntBetween(1, 10); i >= 0; i--) {
             final IndexMetadata indexMetadata = IndexMetadata.builder("test" + i)
                 .settings(settings(Version.CURRENT))
@@ -183,7 +184,9 @@ public void testAllocationSkippedIfDisabled() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
@@ -195,15 +198,16 @@ public void testAllocationSkippedIfDisabled() {
         assertThat(plugin.canAllocateAttempts, equalTo(0));
     }
 
-    private static AllocationService createAllocationService(Settings.Builder settings, ClusterPlugin plugin) {
-        final ClusterSettings emptyClusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+    private static AllocationService createAllocationService(Settings.Builder settingsBuilder, ClusterPlugin plugin) {
+        var settings = settingsBuilder.build();
+        var clusterSettings = ClusterSettings.createBuiltInClusterSettings(settings);
         List<AllocationDecider> deciders = new ArrayList<>(
-            ClusterModule.createAllocationDeciders(settings.build(), emptyClusterSettings, Collections.singletonList(plugin))
+            ClusterModule.createAllocationDeciders(settings, clusterSettings, Collections.singletonList(plugin))
         );
         return new MockAllocationService(
             new AllocationDeciders(deciders),
             new TestGatewayAllocator(),
-            new BalancedShardsAllocator(Settings.EMPTY),
+            new BalancedShardsAllocator(clusterSettings),
             EmptyClusterInfoService.INSTANCE,
             EmptySnapshotsInfoService.INSTANCE
         );
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationTests.java
index 38f8777af7e9..b405405bf1b4 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationTests.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -54,7 +55,9 @@ public void testClusterEnableNone() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -81,7 +84,9 @@ public void testClusterEnableOnlyPrimaries() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
@@ -115,7 +120,7 @@ public void testIndexEnableNone() {
             .put(IndexMetadata.builder("enabled").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder()
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("disabled"))
             .addAsNew(metadata.index("enabled"))
             .build();
@@ -153,7 +158,7 @@ public void testEnableClusterBalance() {
             .put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.getKey(), 10)
             .build();
         ClusterSettings clusterSettings = new ClusterSettings(build, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        AllocationService strategy = createAllocationService(build, clusterSettings, random());
+        AllocationService strategy = createAllocationService(build, clusterSettings);
         Settings indexSettings = useClusterSetting
             ? Settings.EMPTY
             : Settings.builder().put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), Rebalance.NONE).build();
@@ -174,7 +179,7 @@ public void testEnableClusterBalance() {
             )
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder()
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNew(metadata.index("test"))
             .addAsNew(metadata.index("always_disabled"))
             .build();
@@ -285,7 +290,7 @@ public void testEnableClusterBalanceNoReplicas() {
             .put(ConcurrentRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING.getKey(), 3)
             .build();
         ClusterSettings clusterSettings = new ClusterSettings(build, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        AllocationService strategy = createAllocationService(build, clusterSettings, random());
+        AllocationService strategy = createAllocationService(build, clusterSettings);
         Settings indexSettings = useClusterSetting
             ? Settings.EMPTY
             : Settings.builder().put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), Rebalance.NONE).build();
@@ -295,7 +300,9 @@ public void testEnableClusterBalanceNoReplicas() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT).put(indexSettings)).numberOfShards(6).numberOfReplicas(0))
             .build();
 
-        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable initialRoutingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(
             org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDeciderTests.java
index 832a488a30a1..72d7650ad72d 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDeciderTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -44,17 +45,18 @@
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
+import static org.elasticsearch.common.settings.ClusterSettings.createBuiltInClusterSettings;
 import static org.hamcrest.Matchers.equalTo;
 
 public class FilterAllocationDeciderTests extends ESAllocationTestCase {
 
     public void testFilterInitialRecovery() {
-        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        ClusterSettings clusterSettings = createBuiltInClusterSettings();
         FilterAllocationDecider filterAllocationDecider = new FilterAllocationDecider(Settings.EMPTY, clusterSettings);
         AllocationDeciders allocationDeciders = new AllocationDeciders(
             Arrays.asList(
                 filterAllocationDecider,
-                new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
+                new SameShardAllocationDecider(clusterSettings),
                 new ReplicaAfterPrimaryActiveAllocationDecider()
             )
         );
@@ -63,7 +65,8 @@ public void testFilterInitialRecovery() {
             new TestGatewayAllocator(),
             new BalancedShardsAllocator(Settings.EMPTY),
             EmptyClusterInfoService.INSTANCE,
-            EmptySnapshotsInfoService.INSTANCE
+            EmptySnapshotsInfoService.INSTANCE,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
         ClusterState state = createInitialClusterState(
             service,
@@ -194,7 +197,7 @@ private ClusterState createInitialClusterState(AllocationService service, Settin
             .numberOfReplicas(1);
         final IndexMetadata indexMetadata = indexMetadataBuilder.build();
         metadata.put(indexMetadata, false);
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         routingTableBuilder.addAsFromCloseToOpen(sourceIndex);
         routingTableBuilder.addAsNew(indexMetadata);
 
@@ -315,7 +318,8 @@ public void testGetForcedInitialShardAllocationToNodes() {
             new ShardId(index.getIndex(), 0),
             true,
             RecoverySource.LocalShardsRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "index created")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "index created"),
+            ShardRouting.Role.DEFAULT
         );
         assertThat(decider.getForcedInitialShardAllocationToNodes(localRecoveryShard, allocation), equalTo(Optional.of(Set.of("node-1"))));
 
@@ -323,7 +327,8 @@ public void testGetForcedInitialShardAllocationToNodes() {
             new ShardId(index.getIndex(), 0),
             true,
             RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "index created")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "index created"),
+            ShardRouting.Role.DEFAULT
         );
         assertThat(decider.getForcedInitialShardAllocationToNodes(newShard, allocation), equalTo(Optional.empty()));
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDeciderTests.java
index 4f5e6839d08b..ec3463ddf8d6 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDeciderTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.NodesShutdownMetadata;
@@ -38,6 +39,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 
+import static org.elasticsearch.common.settings.ClusterSettings.createBuiltInClusterSettings;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -49,14 +51,15 @@ public class NodeReplacementAllocationDeciderTests extends ESAllocationTestCase
         new ShardId("myindex", "myindex", 0),
         true,
         RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-        new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "index created")
+        new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "index created"),
+        ShardRouting.Role.DEFAULT
     );
-    private final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+    private final ClusterSettings clusterSettings = createBuiltInClusterSettings();
     private NodeReplacementAllocationDecider decider = new NodeReplacementAllocationDecider();
     private final AllocationDeciders allocationDeciders = new AllocationDeciders(
         Arrays.asList(
             decider,
-            new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
+            new SameShardAllocationDecider(clusterSettings),
             new ReplicaAfterPrimaryActiveAllocationDecider(),
             new NodeShutdownAllocationDecider()
         )
@@ -66,7 +69,8 @@ public class NodeReplacementAllocationDeciderTests extends ESAllocationTestCase
         new TestGatewayAllocator(),
         new BalancedShardsAllocator(Settings.EMPTY),
         EmptyClusterInfoService.INSTANCE,
-        EmptySnapshotsInfoService.INSTANCE
+        EmptySnapshotsInfoService.INSTANCE,
+        TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
     );
 
     private final String idxName = "test-idx";
@@ -115,7 +119,8 @@ public void testCanForceAllocate() {
             new ShardId("myindex", "myindex", 0),
             true,
             RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "index created")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "index created"),
+            ShardRouting.Role.DEFAULT
         );
         assignedShard = assignedShard.initialize(NODE_A.getId(), null, 1);
         assignedShard = assignedShard.moveToStarted(ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDeciderTests.java
index 1cf9803caecf..0cdfc6609e29 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDeciderTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.NodesShutdownMetadata;
@@ -38,6 +39,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 
+import static org.elasticsearch.common.settings.ClusterSettings.createBuiltInClusterSettings;
 import static org.hamcrest.Matchers.equalTo;
 
 public class NodeShutdownAllocationDeciderTests extends ESAllocationTestCase {
@@ -46,23 +48,21 @@ public class NodeShutdownAllocationDeciderTests extends ESAllocationTestCase {
         new ShardId("myindex", "myindex", 0),
         true,
         RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-        new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "index created")
+        new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "index created"),
+        ShardRouting.Role.DEFAULT
     );
-    private final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+    private final ClusterSettings clusterSettings = createBuiltInClusterSettings();
     private final NodeShutdownAllocationDecider decider = new NodeShutdownAllocationDecider();
     private final AllocationDeciders allocationDeciders = new AllocationDeciders(
-        Arrays.asList(
-            decider,
-            new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
-            new ReplicaAfterPrimaryActiveAllocationDecider()
-        )
+        Arrays.asList(decider, new SameShardAllocationDecider(clusterSettings), new ReplicaAfterPrimaryActiveAllocationDecider())
     );
     private final AllocationService service = new AllocationService(
         allocationDeciders,
         new TestGatewayAllocator(),
         new BalancedShardsAllocator(Settings.EMPTY),
         EmptyClusterInfoService.INSTANCE,
-        EmptySnapshotsInfoService.INSTANCE
+        EmptySnapshotsInfoService.INSTANCE,
+        TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
     );
 
     private final String idxName = "test-idx";
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java
index 2562d367cb6f..c1d5e2a64b07 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.RestoreInProgress;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -64,7 +65,7 @@ public void testCanAllocatePrimary() {
 
     public void testCannotAllocatePrimaryMissingInRestoreInProgress() {
         ClusterState clusterState = createInitialClusterState();
-        RoutingTable routingTable = RoutingTable.builder(clusterState.getRoutingTable())
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY, clusterState.getRoutingTable())
             .addAsRestore(clusterState.getMetadata().index("test"), createSnapshotRecoverySource("_missing"))
             .build();
 
@@ -90,7 +91,7 @@ public void testCanAllocatePrimaryExistingInRestoreInProgress() {
         RecoverySource.SnapshotRecoverySource recoverySource = createSnapshotRecoverySource("_existing");
 
         ClusterState clusterState = createInitialClusterState();
-        RoutingTable routingTable = RoutingTable.builder(clusterState.getRoutingTable())
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY, clusterState.getRoutingTable())
             .addAsRestore(clusterState.getMetadata().index("test"), recoverySource)
             .build();
 
@@ -183,7 +184,9 @@ private ClusterState createInitialClusterState() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
             .add(newNode("master", Collections.singleton(DiscoveryNodeRole.MASTER_ROLE)))
diff --git a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
index 2b676bc4d3e3..cb235475e323 100644
--- a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.cluster.serialization;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.AbstractNamedDiffable;
@@ -20,6 +21,7 @@
 import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.RestoreInProgress;
 import org.elasticsearch.cluster.SnapshotDeletionsInProgress;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
@@ -27,6 +29,7 @@
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -36,11 +39,12 @@
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
@@ -55,7 +59,9 @@ public void testClusterStateSerialization() throws Exception {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(10).numberOfReplicas(1))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         DiscoveryNodes nodes = DiscoveryNodes.builder()
             .add(newNode("node1"))
@@ -92,7 +98,9 @@ public void testRoutingTableSerialization() throws Exception {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(10).numberOfReplicas(1))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         DiscoveryNodes nodes = DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")).add(newNode("node3")).build();
 
@@ -202,7 +210,9 @@ public void testObjectReuseWhenApplyingClusterStateDiff() throws Exception {
             .build();
         Metadata metadata = Metadata.builder().put(indexMetadata, true).put(indexTemplateMetadata).build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState1 = ClusterState.builder(new ClusterName("clusterName1"))
             .metadata(metadata)
@@ -285,13 +295,12 @@ public void writeTo(StreamOutput out) throws IOException {
         }
 
         @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            {
-                builder.field("custom_string_object", strObject);
-            }
-            builder.endObject();
-            return builder;
+        public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+            return Iterators.concat(
+                Iterators.single((builder, params) -> builder.startObject()),
+                Iterators.single((builder, params) -> builder.field("custom_string_object", strObject)),
+                Iterators.single((builder, params) -> builder.endObject())
+            );
         }
 
         @Override
@@ -304,8 +313,8 @@ public static NamedDiff<Custom> readDiffFrom(StreamInput in) throws IOException
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT;
         }
 
     }
@@ -329,13 +338,12 @@ public void writeTo(StreamOutput out) throws IOException {
         }
 
         @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            {
-                builder.field("custom_integer_object", intObject);
-            }
-            builder.endObject();
-            return builder;
+        public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+            return Iterators.concat(
+                Iterators.single((builder, params) -> builder.startObject()),
+                Iterators.single((builder, params) -> builder.field("custom_integer_object", intObject)),
+                Iterators.single((builder, params) -> builder.endObject())
+            );
         }
 
         @Override
@@ -348,8 +356,8 @@ public static NamedDiff<Custom> readDiffFrom(StreamInput in) throws IOException
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT.minimumCompatibilityVersion();
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT.minimumCompatibilityVersion();
         }
 
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java
index b50c227e3c61..55cb7b1e4fd9 100644
--- a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
@@ -39,7 +40,9 @@ public void testClusterStateSerialization() throws Exception {
             )
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test_idx")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test_idx"))
+            .build();
 
         DiscoveryNodes nodes = DiscoveryNodes.builder()
             .add(new DiscoveryNode("node_foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT))
diff --git a/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierRecordingServiceStatsTests.java b/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierRecordingServiceStatsTests.java
index ef17d1e3fee8..1db95a51c7a4 100644
--- a/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierRecordingServiceStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierRecordingServiceStatsTests.java
@@ -32,4 +32,9 @@ protected Stats createTestInstance() {
         }
         return new Stats(recordings);
     }
+
+    @Override
+    protected Stats mutateInstance(Stats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/service/ClusterStateUpdateStatsWireSerializationTests.java b/server/src/test/java/org/elasticsearch/cluster/service/ClusterStateUpdateStatsWireSerializationTests.java
index 9e6fd663d94c..9c2db57c2745 100644
--- a/server/src/test/java/org/elasticsearch/cluster/service/ClusterStateUpdateStatsWireSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/service/ClusterStateUpdateStatsWireSerializationTests.java
@@ -12,8 +12,6 @@
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
 
-import java.io.IOException;
-
 public class ClusterStateUpdateStatsWireSerializationTests extends AbstractWireSerializingTestCase<ClusterStateUpdateStats> {
 
     @Override
@@ -51,7 +49,7 @@ private static long not(long l) {
     }
 
     @Override
-    protected ClusterStateUpdateStats mutateInstance(ClusterStateUpdateStats instance) throws IOException {
+    protected ClusterStateUpdateStats mutateInstance(ClusterStateUpdateStats instance) {
         switch (between(1, 19)) {
             case 1:
                 return new ClusterStateUpdateStats(
diff --git a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java
index 6ed1436327dd..685c805f7d2c 100644
--- a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java
@@ -9,8 +9,6 @@
 package org.elasticsearch.cluster.service;
 
 import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.core.LogEvent;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ElasticsearchException;
@@ -34,7 +32,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.BaseFuture;
@@ -77,6 +74,7 @@
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
@@ -354,7 +352,6 @@ public void clusterStatePublished(ClusterState newClusterState) {
     @TestLogging(value = "org.elasticsearch.cluster.service:TRACE", reason = "to ensure that we log cluster state events on TRACE level")
     public void testClusterStateUpdateLogging() throws Exception {
         MockLogAppender mockAppender = new MockLogAppender();
-        mockAppender.start();
         mockAppender.addExpectation(
             new MockLogAppender.SeenEventExpectation(
                 "test1 start",
@@ -447,9 +444,7 @@ public void testClusterStateUpdateLogging() throws Exception {
             )
         );
 
-        Logger clusterLogger = LogManager.getLogger(MasterService.class);
-        Loggers.addAppender(clusterLogger, mockAppender);
-        try (MasterService masterService = createMasterService(true)) {
+        try (var ignored = mockAppender.capturing(MasterService.class); var masterService = createMasterService(true)) {
             masterService.submitUnbatchedStateUpdateTask("test1", new ClusterStateUpdateTask() {
                 @Override
                 public ClusterState execute(ClusterState currentState) {
@@ -512,9 +507,6 @@ public void onFailure(Exception e) {
                 }
             });
             assertBusy(mockAppender::assertAllExpectationsMatched);
-        } finally {
-            Loggers.removeAppender(clusterLogger, mockAppender);
-            mockAppender.stop();
         }
     }
 
@@ -903,8 +895,10 @@ class Task implements ClusterStateTaskListener {
 
             final ActionListener<ClusterState> publishListener;
             final String responseHeaderValue;
+            final boolean expectFailure;
 
-            Task(String responseHeaderValue, ActionListener<ClusterState> publishListener) {
+            Task(boolean expectFailure, String responseHeaderValue, ActionListener<ClusterState> publishListener) {
+                this.expectFailure = expectFailure;
                 this.responseHeaderValue = responseHeaderValue;
                 this.publishListener = publishListener;
             }
@@ -920,6 +914,8 @@ public void onFailure(Exception e) {
 
         final var testResponseHeaderName = "test-response-header";
 
+        final var taskFailedExceptionMessage = "simulated task failure";
+
         final var executor = new ClusterStateTaskExecutor<Task>() {
             @Override
             @SuppressForbidden(reason = "consuming published cluster state for legacy reasons")
@@ -928,16 +924,22 @@ public ClusterState execute(BatchExecutionContext<Task> batchExecutionContext) {
                     try (var ignored = taskContext.captureResponseHeaders()) {
                         threadPool.getThreadContext().addResponseHeader(testResponseHeaderName, taskContext.getTask().responseHeaderValue);
                     }
-                    taskContext.success(taskContext.getTask().publishListener::onResponse);
+                    if (taskContext.getTask().expectFailure) {
+                        taskContext.onFailure(new ElasticsearchException(taskFailedExceptionMessage));
+                    } else {
+                        taskContext.success(taskContext.getTask().publishListener::onResponse);
+                    }
                 }
                 return ClusterState.builder(batchExecutionContext.initialState()).build();
             }
         };
 
+        final var blockedState = new AtomicReference<ClusterState>();
         final var executionBarrier = new CyclicBarrier(2);
         final ClusterStateUpdateTask blockMasterTask = new ClusterStateUpdateTask() {
             @Override
             public ClusterState execute(ClusterState currentState) throws Exception {
+                assertTrue(blockedState.compareAndSet(null, currentState));
                 executionBarrier.await(10, TimeUnit.SECONDS); // notify test thread that the master service is blocked
                 executionBarrier.await(10, TimeUnit.SECONDS); // wait for test thread to release us
                 return currentState;
@@ -955,9 +957,13 @@ public void onFailure(Exception e) {
 
             masterService.submitUnbatchedStateUpdateTask("block", blockMasterTask);
             executionBarrier.await(10, TimeUnit.SECONDS); // wait for the master service to be blocked
+            final var stateBeforeSuccess = blockedState.get();
+            assertNotNull(stateBeforeSuccess);
 
             final AtomicReference<ClusterState> publishedState = new AtomicReference<>();
             masterService.setClusterStatePublisher((clusterStatePublicationEvent, publishListener, ackListener) -> {
+                assertSame(stateBeforeSuccess, clusterStatePublicationEvent.getOldState());
+                assertNotSame(stateBeforeSuccess, clusterStatePublicationEvent.getNewState());
                 assertTrue(publishedState.compareAndSet(null, clusterStatePublicationEvent.getNewState()));
                 ClusterServiceUtils.setAllElapsedMillis(clusterStatePublicationEvent);
                 publishListener.onResponse(null);
@@ -971,18 +977,30 @@ public void onFailure(Exception e) {
                     final var testContextHeaderValue = randomAlphaOfLength(10);
                     final var testResponseHeaderValue = randomAlphaOfLength(10);
                     threadContext.putHeader(testContextHeaderName, testContextHeaderValue);
-                    final var task = new Task(testResponseHeaderValue, new ActionListener<>() {
+                    final var expectFailure = randomBoolean();
+                    final var taskComplete = new AtomicBoolean();
+                    final var task = new Task(expectFailure, testResponseHeaderValue, new ActionListener<>() {
                         @Override
                         public void onResponse(ClusterState clusterState) {
+                            assertFalse(expectFailure);
                             assertEquals(testContextHeaderValue, threadContext.getHeader(testContextHeaderName));
                             assertEquals(List.of(testResponseHeaderValue), threadContext.getResponseHeaders().get(testResponseHeaderName));
                             assertSame(publishedState.get(), clusterState);
+                            assertNotSame(stateBeforeSuccess, publishedState.get());
+                            assertTrue(taskComplete.compareAndSet(false, true));
                             publishSuccessCountdown.countDown();
                         }
 
                         @Override
                         public void onFailure(Exception e) {
-                            throw new AssertionError(e);
+                            assertTrue(expectFailure);
+                            assertThat(e, instanceOf(ElasticsearchException.class));
+                            assertThat(e.getMessage(), equalTo(taskFailedExceptionMessage));
+                            assertEquals(List.of(testResponseHeaderValue), threadContext.getResponseHeaders().get(testResponseHeaderName));
+                            assertNotNull(publishedState.get());
+                            assertNotSame(stateBeforeSuccess, publishedState.get());
+                            assertTrue(taskComplete.compareAndSet(false, true));
+                            publishSuccessCountdown.countDown();
                         }
                     });
 
@@ -996,13 +1014,21 @@ public void onFailure(Exception e) {
 
             // failure case: submit some tasks, possibly in different contexts, and verify that the expected listener is completed
 
+            assertNotNull(blockedState.getAndSet(null));
+            assertNotNull(publishedState.getAndSet(null));
             masterService.submitUnbatchedStateUpdateTask("block", blockMasterTask);
             executionBarrier.await(10, TimeUnit.SECONDS); // wait for the master service to be blocked
+            final var stateBeforeFailure = blockedState.get();
+            assertNotNull(stateBeforeFailure);
+
+            final var publicationFailedExceptionMessage = "simulated publication failure";
 
-            final String exceptionMessage = "simulated";
             masterService.setClusterStatePublisher((clusterStatePublicationEvent, publishListener, ackListener) -> {
+                assertSame(stateBeforeFailure, clusterStatePublicationEvent.getOldState());
+                assertNotSame(stateBeforeFailure, clusterStatePublicationEvent.getNewState());
+                assertTrue(publishedState.compareAndSet(null, clusterStatePublicationEvent.getNewState()));
                 ClusterServiceUtils.setAllElapsedMillis(clusterStatePublicationEvent);
-                publishListener.onFailure(new FailedToCommitClusterStateException(exceptionMessage));
+                publishListener.onFailure(new FailedToCommitClusterStateException(publicationFailedExceptionMessage));
             });
 
             toSubmit = between(1, 10);
@@ -1013,7 +1039,9 @@ public void onFailure(Exception e) {
                     final String testContextHeaderValue = randomAlphaOfLength(10);
                     final String testResponseHeaderValue = randomAlphaOfLength(10);
                     threadContext.putHeader(testContextHeaderName, testContextHeaderValue);
-                    final var task = new Task(testResponseHeaderValue, new ActionListener<>() {
+                    final var expectFailure = randomBoolean();
+                    final var taskComplete = new AtomicBoolean();
+                    final var task = new Task(expectFailure, testResponseHeaderValue, new ActionListener<>() {
                         @Override
                         public void onResponse(ClusterState clusterState) {
                             throw new AssertionError("should not succeed");
@@ -1024,7 +1052,16 @@ public void onFailure(Exception e) {
                             assertEquals(testContextHeaderValue, threadContext.getHeader(testContextHeaderName));
                             assertEquals(List.of(testResponseHeaderValue), threadContext.getResponseHeaders().get(testResponseHeaderName));
                             assertThat(e, instanceOf(FailedToCommitClusterStateException.class));
-                            assertThat(e.getMessage(), equalTo(exceptionMessage));
+                            assertThat(e.getMessage(), equalTo(publicationFailedExceptionMessage));
+                            if (expectFailure) {
+                                assertThat(e.getSuppressed().length, greaterThan(0));
+                                var suppressed = e.getSuppressed()[0];
+                                assertThat(suppressed, instanceOf(ElasticsearchException.class));
+                                assertThat(suppressed.getMessage(), equalTo(taskFailedExceptionMessage));
+                            }
+                            assertNotNull(publishedState.get());
+                            assertNotSame(stateBeforeFailure, publishedState.get());
+                            assertTrue(taskComplete.compareAndSet(false, true));
                             publishFailureCountdown.countDown();
                         }
                     });
@@ -1081,7 +1118,6 @@ public void testBlockingCallInClusterStateTaskListenerFails() throws Interrupted
     @TestLogging(value = "org.elasticsearch.cluster.service:WARN", reason = "to ensure that we log cluster state events on WARN level")
     public void testLongClusterStateUpdateLogging() throws Exception {
         MockLogAppender mockAppender = new MockLogAppender();
-        mockAppender.start();
         mockAppender.addExpectation(
             new MockLogAppender.UnseenEventExpectation(
                 "test1 shouldn't log because it was fast enough",
@@ -1131,13 +1167,12 @@ public void testLongClusterStateUpdateLogging() throws Exception {
             )
         );
 
-        Logger clusterLogger = LogManager.getLogger(MasterService.class);
-        Loggers.addAppender(clusterLogger, mockAppender);
         final Settings settings = Settings.builder()
             .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), MasterServiceTests.class.getSimpleName())
             .put(Node.NODE_NAME_SETTING.getKey(), "test_node")
             .build();
         try (
+            var ignored = mockAppender.capturing(MasterService.class);
             MasterService masterService = new MasterService(
                 settings,
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
@@ -1304,11 +1339,8 @@ public void onFailure(Exception e) {
                 }
             });
             latch.await();
-        } finally {
-            Loggers.removeAppender(clusterLogger, mockAppender);
-            mockAppender.stop();
+            mockAppender.assertAllExpectationsMatched();
         }
-        mockAppender.assertAllExpectationsMatched();
     }
 
     public void testAcking() throws InterruptedException {
@@ -1660,11 +1692,7 @@ public void testStarvationLogging() throws Exception {
         final long taskDurationMillis = TimeValue.timeValueSeconds(1).millis();
 
         MockLogAppender mockAppender = new MockLogAppender();
-        mockAppender.start();
-
-        Logger clusterLogger = LogManager.getLogger(MasterService.class);
-        Loggers.addAppender(clusterLogger, mockAppender);
-        try (MasterService masterService = createMasterService(true)) {
+        try (MasterService masterService = createMasterService(true); var ignored = mockAppender.capturing(MasterService.class)) {
             final AtomicBoolean keepRunning = new AtomicBoolean(true);
 
             final Runnable await = new Runnable() {
@@ -1766,10 +1794,6 @@ public void onFailure(Exception e) {
             keepRunning.set(false);
             awaitNextTask.run();
             assertTrue(starvedTaskExecuted.await(10, TimeUnit.SECONDS));
-
-        } finally {
-            Loggers.removeAppender(clusterLogger, mockAppender);
-            mockAppender.stop();
         }
     }
 
@@ -1779,11 +1803,7 @@ public void onFailure(Exception e) {
     )
     public void testBatchedUpdateSummaryLogging() throws Exception {
         MockLogAppender mockAppender = new MockLogAppender();
-        mockAppender.start();
-
-        Logger masterServiceLogger = LogManager.getLogger(MasterService.class);
-        Loggers.addAppender(masterServiceLogger, mockAppender);
-        try (MasterService masterService = createMasterService(true)) {
+        try (var ignored = mockAppender.capturing(MasterService.class); var masterService = createMasterService(true)) {
 
             final var barrier = new CyclicBarrier(2);
             final var blockingTask = new ClusterStateUpdateTask() {
@@ -1907,9 +1927,6 @@ public boolean innerMatch(LogEvent event) {
             assertTrue(manySourceExecutor.semaphore.tryAcquire(2048, 10, TimeUnit.SECONDS));
             assertTrue(manyTasksPerSourceExecutor.semaphore.tryAcquire(2048, 10, TimeUnit.SECONDS));
             mockAppender.assertAllExpectationsMatched();
-        } finally {
-            Loggers.removeAppender(masterServiceLogger, mockAppender);
-            mockAppender.stop();
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java b/server/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java
index 144b71ef5202..e641816fdfa7 100644
--- a/server/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -85,7 +86,9 @@ public void testIterator1() {
         Metadata metadata = Metadata.builder()
             .put(IndexMetadata.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(2))
             .build();
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test1")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test1"))
+            .build();
 
         ShardIterator shardIterator = routingTable.index("test1").shard(0).shardsIt(0);
         assertThat(shardIterator.size(), equalTo(3));
@@ -112,7 +115,10 @@ public void testIterator2() {
             .put(IndexMetadata.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test1")).addAsNew(metadata.index("test2")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test1"))
+            .addAsNew(metadata.index("test2"))
+            .build();
 
         ShardIterator shardIterator = routingTable.index("test1").shard(0).shardsIt(0);
         assertThat(shardIterator.size(), equalTo(2));
@@ -189,7 +195,10 @@ public void testRandomRouting() {
             .put(IndexMetadata.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test1")).addAsNew(metadata.index("test2")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test1"))
+            .addAsNew(metadata.index("test2"))
+            .build();
 
         ShardIterator shardIterator = routingTable.index("test1").shard(0).shardsRandomIt();
         ShardRouting shardRouting1 = shardIterator.nextOrNull();
@@ -219,7 +228,9 @@ public void testNodeSelectorRouting() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
@@ -326,7 +337,9 @@ public void testShardsAndPreferNodeRouting() {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(5).numberOfReplicas(1))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
@@ -399,7 +412,9 @@ public void testReplicaShardPreferenceIters() throws Exception {
             .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(2))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.index("test"))
+            .build();
 
         final ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metadata(metadata)
diff --git a/server/src/test/java/org/elasticsearch/common/ReferenceDocsTests.java b/server/src/test/java/org/elasticsearch/common/ReferenceDocsTests.java
new file mode 100644
index 000000000000..92e37ff005e5
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/common/ReferenceDocsTests.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParseException;
+
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+
+import static org.elasticsearch.common.ReferenceDocs.getVersionComponent;
+
+public class ReferenceDocsTests extends ESTestCase {
+
+    public void testVersionComponent() {
+        // Snapshot x.y.0 versions are unreleased so link to master
+        assertEquals("master", getVersionComponent(Version.V_8_7_0, true));
+
+        // Snapshot x.y.z versions with z>0 mean that x.y.0 is released so have a properly versioned docs link
+        assertEquals("8.5", getVersionComponent(Version.V_8_5_1, true));
+
+        // Non-snapshot versions are to be released so have a properly versioned docs link
+        assertEquals("8.7", getVersionComponent(Version.V_8_7_0, false));
+    }
+
+    public void testResourceValidation() throws Exception {
+
+        try (var builder = XContentFactory.jsonBuilder()) {
+            builder.startObject();
+            for (ReferenceDocs link : ReferenceDocs.values()) {
+                builder.field(link.name(), "TEST");
+            }
+            builder.endObject();
+
+            var map = ReferenceDocs.readLinksBySymbol(BytesReference.bytes(builder).streamInput());
+            assertEquals(ReferenceDocs.values().length, map.size());
+            for (ReferenceDocs link : ReferenceDocs.values()) {
+                assertEquals("TEST", map.get(link.name()));
+            }
+        }
+
+        try (var stream = new ByteArrayInputStream("{\"invalid\":".getBytes(StandardCharsets.UTF_8))) {
+            expectThrows(XContentParseException.class, () -> ReferenceDocs.readLinksBySymbol(stream));
+        }
+
+        try (var builder = XContentFactory.jsonBuilder()) {
+            builder.startObject();
+            for (ReferenceDocs link : ReferenceDocs.values()) {
+                builder.field(link.name(), "TEST");
+            }
+            builder.startObject("UNEXPECTED").endObject().endObject();
+
+            try (var stream = BytesReference.bytes(builder).streamInput()) {
+                expectThrows(IllegalStateException.class, () -> ReferenceDocs.readLinksBySymbol(stream));
+            }
+        }
+
+        try (var builder = XContentFactory.jsonBuilder()) {
+            builder.startObject();
+            for (ReferenceDocs link : ReferenceDocs.values()) {
+                builder.field(link.name(), "TEST");
+            }
+            builder.field("EXTRA", "TEST").endObject();
+
+            try (var stream = BytesReference.bytes(builder).streamInput()) {
+                expectThrows(IllegalStateException.class, () -> ReferenceDocs.readLinksBySymbol(stream));
+            }
+        }
+
+        try (var builder = XContentFactory.jsonBuilder()) {
+            builder.startObject();
+            var skipped = randomFrom(ReferenceDocs.values());
+            for (ReferenceDocs link : ReferenceDocs.values()) {
+                if (link != skipped) {
+                    builder.field(link.name(), "TEST");
+                }
+            }
+            builder.endObject();
+
+            try (var stream = BytesReference.bytes(builder).streamInput()) {
+                expectThrows(IllegalStateException.class, () -> ReferenceDocs.readLinksBySymbol(stream));
+            }
+        }
+
+        try (var builder = XContentFactory.jsonBuilder()) {
+            var shuffled = Arrays.copyOf(ReferenceDocs.values(), ReferenceDocs.values().length);
+            var i = between(0, ReferenceDocs.values().length - 1);
+            var j = randomValueOtherThan(i, () -> between(0, ReferenceDocs.values().length - 1));
+            var tmp = shuffled[i];
+            shuffled[i] = shuffled[j];
+            shuffled[j] = tmp;
+
+            builder.startObject();
+            for (ReferenceDocs link : shuffled) {
+                builder.field(link.name(), "TEST");
+            }
+            builder.endObject();
+
+            try (var stream = BytesReference.bytes(builder).streamInput()) {
+                expectThrows(IllegalStateException.class, () -> ReferenceDocs.readLinksBySymbol(stream));
+            }
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/common/ReleasablesTests.java b/server/src/test/java/org/elasticsearch/common/ReleasablesTests.java
index 4e62179c5936..120a28c9802a 100644
--- a/server/src/test/java/org/elasticsearch/common/ReleasablesTests.java
+++ b/server/src/test/java/org/elasticsearch/common/ReleasablesTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ReachabilityChecker;
 
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -24,4 +25,15 @@ public void testReleaseOnce() {
         releasable.close();
         assertEquals(1, count.get());
     }
+
+    public void testReleaseOnceReleasesDelegate() {
+        final var reachabilityChecker = new ReachabilityChecker();
+        final var releaseOnce = Releasables.releaseOnce(reachabilityChecker.register(this::noop));
+        reachabilityChecker.checkReachable();
+        releaseOnce.close();
+        reachabilityChecker.ensureUnreachable();
+        assertEquals("releaseOnce[null]", releaseOnce.toString());
+    }
+
+    private void noop() {}
 }
diff --git a/server/src/test/java/org/elasticsearch/common/RoundingWireTests.java b/server/src/test/java/org/elasticsearch/common/RoundingWireTests.java
index 5c0808809e78..9ff46116a693 100644
--- a/server/src/test/java/org/elasticsearch/common/RoundingWireTests.java
+++ b/server/src/test/java/org/elasticsearch/common/RoundingWireTests.java
@@ -33,6 +33,11 @@ protected Rounding createTestInstance() {
         return builder.build();
     }
 
+    @Override
+    protected Rounding mutateInstance(Rounding instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<Rounding> instanceReader() {
         return Rounding::read;
diff --git a/server/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java b/server/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java
index 028d07cc5777..d75cd363925f 100644
--- a/server/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java
+++ b/server/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java
@@ -83,7 +83,7 @@ public void testGetLongLE() {
         assertThat(ref.getLongLE(0), equalTo(888L));
         assertThat(ref.getLongLE(8), equalTo(Long.MAX_VALUE));
         Exception e = expectThrows(ArrayIndexOutOfBoundsException.class, () -> ref.getLongLE(9));
-        assertThat(e.getMessage(), equalTo("Index 16 out of bounds for length 16"));
+        assertThat(e.getMessage(), equalTo("Index 9 out of bounds for length 9"));
     }
 
     public void testGetDoubleLE() {
diff --git a/server/src/test/java/org/elasticsearch/common/bytes/CompositeBytesReferenceTests.java b/server/src/test/java/org/elasticsearch/common/bytes/CompositeBytesReferenceTests.java
index e6043837ccd7..99de6ed47114 100644
--- a/server/src/test/java/org/elasticsearch/common/bytes/CompositeBytesReferenceTests.java
+++ b/server/src/test/java/org/elasticsearch/common/bytes/CompositeBytesReferenceTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
+import org.elasticsearch.common.util.ByteUtils;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
@@ -190,4 +191,29 @@ public void testGetDoubleLE() {
         // We can assert the exception message if -XX:-OmitStackTraceInFastThrow is set in gradle test task.
         expectThrows(IndexOutOfBoundsException.class, () -> comp.getDoubleLE(17));
     }
+
+    public void testGetLongLE() {
+        // first long = 2, second long = 44, third long = 512
+        // tag::noformat
+        byte[] data = new byte[] {
+            0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+            0x2C, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+            0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
+        // end::noformat
+
+        byte[] d = new byte[8];
+        ByteUtils.writeLongLE(2, d, 0);
+
+        List<BytesReference> refs = new ArrayList<>();
+        int bytesPerChunk = randomFrom(4, 16);
+        for (int offset = 0; offset < data.length; offset += bytesPerChunk) {
+            int length = Math.min(bytesPerChunk, data.length - offset);
+            refs.add(new BytesArray(data, offset, length));
+        }
+        BytesReference comp = CompositeBytesReference.of(refs.toArray(BytesReference[]::new));
+        assertThat(comp.getLongLE(0), equalTo(2L));
+        assertThat(comp.getLongLE(8), equalTo(44L));
+        assertThat(comp.getLongLE(16), equalTo(512L));
+        expectThrows(IndexOutOfBoundsException.class, () -> comp.getLongLE(17));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/bytes/ReleasableBytesReferenceStreamInputTests.java b/server/src/test/java/org/elasticsearch/common/bytes/ReleasableBytesReferenceStreamInputTests.java
index 5103a685beb5..dc5e9fa0ec33 100644
--- a/server/src/test/java/org/elasticsearch/common/bytes/ReleasableBytesReferenceStreamInputTests.java
+++ b/server/src/test/java/org/elasticsearch/common/bytes/ReleasableBytesReferenceStreamInputTests.java
@@ -12,8 +12,10 @@
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.ByteArray;
 import org.elasticsearch.common.util.DoubleArray;
 import org.elasticsearch.common.util.IntArray;
+import org.elasticsearch.common.util.LongArray;
 import org.junit.After;
 
 import java.io.IOException;
@@ -100,4 +102,42 @@ public void testBigDoubleArrayLivesAfterReleasableIsDecremented() throws IOExcep
         assertThat(ref.hasReferences(), equalTo(false));
     }
 
+    public void testBigLongArrayLivesAfterReleasableIsDecremented() throws IOException {
+        LongArray testData = BigArrays.NON_RECYCLING_INSTANCE.newLongArray(1, false);
+        testData.set(0, 1);
+
+        BytesStreamOutput out = new BytesStreamOutput();
+        testData.writeTo(out);
+
+        ReleasableBytesReference ref = ReleasableBytesReference.wrap(out.bytes());
+
+        try (LongArray in = LongArray.readFrom(ref.streamInput())) {
+            ref.decRef();
+            assertThat(ref.hasReferences(), equalTo(true));
+
+            assertThat(in.size(), equalTo(testData.size()));
+            assertThat(in.get(0), equalTo(1L));
+        }
+        assertThat(ref.hasReferences(), equalTo(false));
+    }
+
+    public void testBigByteArrayLivesAfterReleasableIsDecremented() throws IOException {
+        ByteArray testData = BigArrays.NON_RECYCLING_INSTANCE.newByteArray(1, false);
+        testData.set(0L, (byte) 1);
+
+        BytesStreamOutput out = new BytesStreamOutput();
+        testData.writeTo(out);
+
+        ReleasableBytesReference ref = ReleasableBytesReference.wrap(out.bytes());
+
+        try (ByteArray in = ByteArray.readFrom(ref.streamInput())) {
+            ref.decRef();
+            assertThat(ref.hasReferences(), equalTo(true));
+
+            assertThat(in.size(), equalTo(testData.size()));
+            assertThat(in.get(0), equalTo((byte) 1));
+        }
+        assertThat(ref.hasReferences(), equalTo(false));
+    }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/common/collect/IteratorsTests.java b/server/src/test/java/org/elasticsearch/common/collect/IteratorsTests.java
index 3a750419c209..8d3bcdd61439 100644
--- a/server/src/test/java/org/elasticsearch/common/collect/IteratorsTests.java
+++ b/server/src/test/java/org/elasticsearch/common/collect/IteratorsTests.java
@@ -18,6 +18,7 @@
 import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.IntStream;
 
 public class IteratorsTests extends ESTestCase {
     public void testConcatentation() {
@@ -153,6 +154,35 @@ public void testArrayIteratorOnNull() {
         expectThrows(NullPointerException.class, "Unable to iterate over a null array", () -> Iterators.forArray(null));
     }
 
+    public void testFlatMap() {
+        final var array = randomIntegerArray();
+        assertEmptyIterator(Iterators.flatMap(Iterators.forArray(array), i -> Iterators.concat()));
+        assertEmptyIterator(Iterators.flatMap(Iterators.concat(), i -> Iterators.single("foo")));
+
+        final var index = new AtomicInteger();
+        Iterators.flatMap(Iterators.forArray(array), Iterators::single)
+            .forEachRemaining(i -> assertEquals(array[index.getAndIncrement()], i));
+        assertEquals(array.length, index.get());
+
+        index.set(0);
+        Iterators.flatMap(Iterators.forArray(array), i -> Iterators.forArray(new Integer[] { i, i, i }))
+            .forEachRemaining(i -> assertEquals(array[(index.getAndIncrement() / 3)], i));
+        assertEquals(array.length * 3, index.get());
+
+        final var input = new ArrayList<>();
+        for (int i = 1; i <= 4; i++) {
+            IntStream.range(0, between(0, 2)).forEachOrdered(ignored -> input.add(0));
+            input.add(i);
+            IntStream.range(0, between(0, 2)).forEachOrdered(ignored -> input.add(0));
+        }
+
+        index.set(0);
+        final var expectedArray = new Integer[] { 0, 0, 1, 0, 1, 2, 0, 1, 2, 3 };
+        Iterators.flatMap(input.listIterator(), i -> IntStream.range(0, (int) i).iterator())
+            .forEachRemaining(i -> assertEquals(expectedArray[(index.getAndIncrement())], i));
+        assertEquals(expectedArray.length, index.get());
+    }
+
     private static Integer[] randomIntegerArray() {
         return Randomness.get().ints(randomIntBetween(0, 1000)).boxed().toArray(Integer[]::new);
     }
diff --git a/server/src/test/java/org/elasticsearch/common/io/stream/AbstractStreamTests.java b/server/src/test/java/org/elasticsearch/common/io/stream/AbstractStreamTests.java
index 04918f5ba845..0d07c8cdda5d 100644
--- a/server/src/test/java/org/elasticsearch/common/io/stream/AbstractStreamTests.java
+++ b/server/src/test/java/org/elasticsearch/common/io/stream/AbstractStreamTests.java
@@ -14,12 +14,15 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.ByteArray;
 import org.elasticsearch.common.util.DoubleArray;
 import org.elasticsearch.common.util.IntArray;
+import org.elasticsearch.common.util.LongArray;
 import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.ESTestCase;
 
@@ -75,7 +78,7 @@ public void testBooleanSerialization() throws IOException {
         final byte[] corruptBytes = new byte[] { randomFrom(set) };
         final BytesReference corrupt = new BytesArray(corruptBytes);
         final IllegalStateException e = expectThrows(IllegalStateException.class, () -> getStreamInput(corrupt).readBoolean());
-        final String message = formatted("unexpected byte [0x%02x]", corruptBytes[0]);
+        final String message = Strings.format("unexpected byte [0x%02x]", corruptBytes[0]);
         assertThat(e, hasToString(containsString(message)));
     }
 
@@ -110,7 +113,7 @@ public void testOptionalBooleanSerialization() throws IOException {
         final byte[] corruptBytes = new byte[] { randomFrom(set) };
         final BytesReference corrupt = new BytesArray(corruptBytes);
         final IllegalStateException e = expectThrows(IllegalStateException.class, () -> getStreamInput(corrupt).readOptionalBoolean());
-        final String message = formatted("unexpected byte [0x%02x]", corruptBytes[0]);
+        final String message = Strings.format("unexpected byte [0x%02x]", corruptBytes[0]);
         assertThat(e, hasToString(containsString(message)));
     }
 
@@ -271,6 +274,56 @@ private void assertBigDoubleArray(int size) throws IOException {
         }
     }
 
+    public void testSmallBigLongArray() throws IOException {
+        assertBigLongArray(between(0, PageCacheRecycler.LONG_PAGE_SIZE));
+    }
+
+    public void testLargeBigLongArray() throws IOException {
+        assertBigLongArray(between(PageCacheRecycler.LONG_PAGE_SIZE, 10000));
+    }
+
+    private void assertBigLongArray(int size) throws IOException {
+        LongArray testData = BigArrays.NON_RECYCLING_INSTANCE.newLongArray(size, false);
+        for (int i = 0; i < size; i++) {
+            testData.set(i, randomLong());
+        }
+
+        BytesStreamOutput out = new BytesStreamOutput();
+        testData.writeTo(out);
+
+        try (LongArray in = LongArray.readFrom(getStreamInput(out.bytes()))) {
+            assertThat(in.size(), equalTo(testData.size()));
+            for (int i = 0; i < size; i++) {
+                assertThat(in.get(i), equalTo(testData.get(i)));
+            }
+        }
+    }
+
+    public void testSmallBigByteArray() throws IOException {
+        assertBigByteArray(between(0, PageCacheRecycler.BYTE_PAGE_SIZE / 10));
+    }
+
+    public void testLargeBigByteArray() throws IOException {
+        assertBigByteArray(between(PageCacheRecycler.BYTE_PAGE_SIZE / 10, PageCacheRecycler.BYTE_PAGE_SIZE * 10));
+    }
+
+    private void assertBigByteArray(int size) throws IOException {
+        ByteArray testData = BigArrays.NON_RECYCLING_INSTANCE.newByteArray(size, false);
+        for (int i = 0; i < size; i++) {
+            testData.set(i, randomByte());
+        }
+
+        BytesStreamOutput out = new BytesStreamOutput();
+        testData.writeTo(out);
+
+        try (ByteArray in = ByteArray.readFrom(getStreamInput(out.bytes()))) {
+            assertThat(in.size(), equalTo(testData.size()));
+            for (int i = 0; i < size; i++) {
+                assertThat(in.get(i), equalTo(testData.get(i)));
+            }
+        }
+    }
+
     public void testCollection() throws IOException {
         class FooBar implements Writeable {
 
diff --git a/server/src/test/java/org/elasticsearch/common/io/stream/DelayableWriteableTests.java b/server/src/test/java/org/elasticsearch/common/io/stream/DelayableWriteableTests.java
index 5fd585cdb35b..078a09bf44fe 100644
--- a/server/src/test/java/org/elasticsearch/common/io/stream/DelayableWriteableTests.java
+++ b/server/src/test/java/org/elasticsearch/common/io/stream/DelayableWriteableTests.java
@@ -8,9 +8,9 @@
 
 package org.elasticsearch.common.io.stream;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.TransportVersionUtils;
 
 import java.io.IOException;
 
@@ -87,19 +87,19 @@ public int hashCode() {
     }
 
     private static class SneakOtherSideVersionOnWire implements Writeable {
-        private final Version version;
+        private final TransportVersion version;
 
         SneakOtherSideVersionOnWire() {
-            version = Version.CURRENT;
+            version = TransportVersion.CURRENT;
         }
 
         SneakOtherSideVersionOnWire(StreamInput in) throws IOException {
-            version = Version.readVersion(in);
+            version = TransportVersion.readVersion(in);
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            Version.writeVersion(out.getVersion(), out);
+            TransportVersion.writeVersion(out.getTransportVersion(), out);
         }
     }
 
@@ -144,7 +144,7 @@ public void testRoundTripFromDelayedFromOldVersionWithNamedWriteable() throws IO
     }
 
     public void testSerializesWithRemoteVersion() throws IOException {
-        Version remoteVersion = VersionUtils.randomCompatibleVersion(random(), Version.CURRENT);
+        TransportVersion remoteVersion = TransportVersionUtils.randomCompatibleVersion(random(), TransportVersion.CURRENT);
         DelayableWriteable<SneakOtherSideVersionOnWire> original = DelayableWriteable.referencing(new SneakOtherSideVersionOnWire());
         assertThat(roundTrip(original, SneakOtherSideVersionOnWire::new, remoteVersion).expand().version, equalTo(remoteVersion));
     }
@@ -157,14 +157,14 @@ public void testAsSerializedIsNoopOnSerialized() throws IOException {
     }
 
     private <T extends Writeable> void roundTripTestCase(DelayableWriteable<T> original, Writeable.Reader<T> reader) throws IOException {
-        DelayableWriteable<T> roundTripped = roundTrip(original, reader, Version.CURRENT);
+        DelayableWriteable<T> roundTripped = roundTrip(original, reader, TransportVersion.CURRENT);
         assertThat(roundTripped.expand(), equalTo(original.expand()));
     }
 
     private <T extends Writeable> DelayableWriteable<T> roundTrip(
         DelayableWriteable<T> original,
         Writeable.Reader<T> reader,
-        Version version
+        TransportVersion version
     ) throws IOException {
         DelayableWriteable<T> delayed = copyInstance(
             original,
@@ -192,7 +192,10 @@ protected NamedWriteableRegistry writableRegistry() {
         return new NamedWriteableRegistry(singletonList(new NamedWriteableRegistry.Entry(Example.class, "example", Example::new)));
     }
 
-    private static Version randomOldVersion() {
-        return randomValueOtherThanMany(Version.CURRENT::before, () -> VersionUtils.randomCompatibleVersion(random(), Version.CURRENT));
+    private static TransportVersion randomOldVersion() {
+        return randomValueOtherThanMany(
+            TransportVersion.CURRENT::before,
+            () -> TransportVersionUtils.randomCompatibleVersion(random(), TransportVersion.CURRENT)
+        );
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/io/stream/VersionCheckingStreamOutputTests.java b/server/src/test/java/org/elasticsearch/common/io/stream/VersionCheckingStreamOutputTests.java
index 43766005be5f..a6fd21042685 100644
--- a/server/src/test/java/org/elasticsearch/common/io/stream/VersionCheckingStreamOutputTests.java
+++ b/server/src/test/java/org/elasticsearch/common/io/stream/VersionCheckingStreamOutputTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.common.io.stream;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ESTestCase;
@@ -28,8 +29,8 @@ public String getWriteableName() {
         public void writeTo(StreamOutput out) throws IOException {}
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT;
         }
     }
 
@@ -44,9 +45,9 @@ public void testCheckVersionCompatibility() throws IOException {
             );
             assertEquals(
                 "[test_writable] was released first in version "
-                    + Version.CURRENT
+                    + TransportVersion.CURRENT
                     + ", failed compatibility check trying to send it to node with version "
-                    + streamVersion,
+                    + streamVersion.transportVersion,
                 e.getMessage()
             );
         }
diff --git a/server/src/test/java/org/elasticsearch/common/logging/ChunkedLoggingStreamTests.java b/server/src/test/java/org/elasticsearch/common/logging/ChunkedLoggingStreamTests.java
new file mode 100644
index 000000000000..a0534ea52dfc
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/common/logging/ChunkedLoggingStreamTests.java
@@ -0,0 +1,196 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common.logging;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.appender.AbstractAppender;
+import org.apache.logging.log4j.core.config.Property;
+import org.elasticsearch.common.ReferenceDocs;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.core.CheckedRunnable;
+import org.elasticsearch.core.Streams;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.junit.annotations.TestLogging;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.stream.IntStream;
+import java.util.zip.GZIPInputStream;
+
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+public class ChunkedLoggingStreamTests extends ESTestCase {
+
+    public static final Logger logger = LogManager.getLogger(ChunkedLoggingStreamTests.class);
+
+    @TestLogging(reason = "testing logging", value = "org.elasticsearch.common.logging.ChunkedLoggingStreamTests:DEBUG")
+    public void testLogMessageChunking() {
+        // bugs are most likely near chunk boundaries, so test sizes that are within +/- 3 bytes of 0, 1, and 2 chunks:
+        IntStream.rangeClosed(-3, 3)
+            .flatMap(i -> IntStream.iterate(i, j -> j + ChunkedLoggingStream.CHUNK_SIZE).limit(3))
+            .filter(i -> i >= 0)
+            .sorted()
+            .forEach(ChunkedLoggingStreamTests::runChunkingTest);
+    }
+
+    private static void runChunkingTest(int size) {
+        final var bytes = new byte[size];
+        Arrays.fill(bytes, (byte) '.');
+        final var expectedBody = new String(bytes, StandardCharsets.ISO_8859_1);
+        final var prefix = randomAlphaOfLength(10);
+        final var level = randomFrom(Level.DEBUG, Level.INFO, Level.WARN, Level.ERROR);
+        final var referenceDocs = randomFrom(ReferenceDocs.values());
+        assertEquals(expectedBody, getLoggedBody(logger, level, prefix, referenceDocs, () -> {
+            try (var stream = new ChunkedLoggingStream(logger, level, prefix, referenceDocs)) {
+                writeRandomly(stream, bytes);
+            }
+        }));
+    }
+
+    @TestLogging(reason = "testing logging", value = "org.elasticsearch.common.logging.ChunkedLoggingStreamTests:DEBUG")
+    public void testEncodingRoundTrip() {
+        final var bytes = randomByteArrayOfLength(between(0, 10000));
+        final var level = randomFrom(Level.DEBUG, Level.INFO, Level.WARN, Level.ERROR);
+        final var referenceDocs = randomFrom(ReferenceDocs.values());
+        assertEquals(new BytesArray(bytes), getDecodedLoggedBody(logger, level, "prefix", referenceDocs, () -> {
+            try (var stream = ChunkedLoggingStream.create(logger, level, "prefix", referenceDocs)) {
+                writeRandomly(stream, bytes);
+            }
+        }));
+    }
+
+    private static String getLoggedBody(
+        Logger captureLogger,
+        final Level level,
+        String prefix,
+        final ReferenceDocs referenceDocs,
+        CheckedRunnable<Exception> runnable
+    ) {
+        class ChunkReadingAppender extends AbstractAppender {
+            final StringBuilder encodedResponseBuilder = new StringBuilder();
+            int chunks;
+            boolean seenTotal;
+
+            ChunkReadingAppender() {
+                super("mock", null, null, false, Property.EMPTY_ARRAY);
+            }
+
+            @Override
+            public void append(LogEvent event) {
+                if (event.getLevel() != level) {
+                    return;
+                }
+                if (event.getLoggerName().equals(captureLogger.getName()) == false) {
+                    return;
+                }
+                assertFalse(seenTotal);
+                final var message = event.getMessage().getFormattedMessage();
+                final var onePartPrefix = prefix + " (gzip compressed and base64-encoded; for details see " + referenceDocs + "): ";
+                final var partPrefix = prefix + " [part " + (chunks + 1) + "]: ";
+                if (message.startsWith(partPrefix)) {
+                    chunks += 1;
+                    final var chunk = message.substring(partPrefix.length());
+                    assertThat(chunk.length(), lessThanOrEqualTo(ChunkedLoggingStream.CHUNK_SIZE));
+                    encodedResponseBuilder.append(chunk);
+                } else if (message.startsWith(onePartPrefix)) {
+                    assertEquals(0, chunks);
+                    chunks += 1;
+                    final var chunk = message.substring(onePartPrefix.length());
+                    assertThat(chunk.length(), lessThanOrEqualTo(ChunkedLoggingStream.CHUNK_SIZE));
+                    encodedResponseBuilder.append(chunk);
+                    seenTotal = true;
+                } else {
+                    assertEquals(
+                        prefix
+                            + " (gzip compressed, base64-encoded, and split into "
+                            + chunks
+                            + " parts on preceding log lines; for details see "
+                            + referenceDocs
+                            + ")",
+                        message
+                    );
+                    assertThat(chunks, greaterThan(1));
+                    seenTotal = true;
+                }
+            }
+        }
+
+        final var appender = new ChunkReadingAppender();
+        try {
+            appender.start();
+            Loggers.addAppender(captureLogger, appender);
+            runnable.run();
+        } catch (Exception e) {
+            throw new AssertionError("unexpected", e);
+        } finally {
+            Loggers.removeAppender(captureLogger, appender);
+            appender.stop();
+        }
+
+        assertThat(appender.chunks, greaterThan(0));
+        assertTrue(appender.seenTotal);
+
+        return appender.encodedResponseBuilder.toString();
+    }
+
+    /**
+     * Test utility function which captures the logged output from a {@link ChunkedLoggingStream}, combines the chunks, Base64-decodes it
+     * and Gzip-decompresses it to retrieve the original data.
+     *
+     * @param captureLogger The logger whose output should be captured.
+     * @param level         The log level for the data.
+     * @param prefix        The prefix used by the logging stream.
+     * @param referenceDocs A link to the reference docs about the output.
+     * @param runnable      The action which emits the logs.
+     * @return              A {@link BytesReference} containing the captured data.
+     */
+    public static BytesReference getDecodedLoggedBody(
+        Logger captureLogger,
+        Level level,
+        String prefix,
+        ReferenceDocs referenceDocs,
+        CheckedRunnable<Exception> runnable
+    ) {
+        final var loggedBody = getLoggedBody(captureLogger, level, prefix, referenceDocs, runnable);
+
+        try (
+            var bytesStreamOutput = new BytesStreamOutput();
+            var byteArrayInputStream = new ByteArrayInputStream(Base64.getDecoder().decode(loggedBody));
+            var gzipInputStream = new GZIPInputStream(byteArrayInputStream)
+        ) {
+            Streams.copy(gzipInputStream, bytesStreamOutput);
+            return bytesStreamOutput.bytes();
+        } catch (Exception e) {
+            throw new AssertionError("unexpected", e);
+        }
+    }
+
+    private static void writeRandomly(OutputStream stream, byte[] bytes) throws IOException {
+        for (var pos = 0; pos < bytes.length;) {
+            if (randomBoolean()) {
+                stream.write(bytes[pos++]);
+            } else {
+                var len = between(1, bytes.length - pos);
+                stream.write(bytes, pos, len);
+                pos += len;
+            }
+        }
+    }
+
+}
diff --git a/server/src/test/java/org/elasticsearch/common/logging/LoggingOutputStreamTests.java b/server/src/test/java/org/elasticsearch/common/logging/LoggingOutputStreamTests.java
index 0640efdf8643..62e3046dc63f 100644
--- a/server/src/test/java/org/elasticsearch/common/logging/LoggingOutputStreamTests.java
+++ b/server/src/test/java/org/elasticsearch/common/logging/LoggingOutputStreamTests.java
@@ -29,20 +29,22 @@ class TestLoggingOutputStream extends LoggingOutputStream {
         List<String> lines = new ArrayList<>();
 
         TestLoggingOutputStream() {
-            super(null, null);
+            super(null, null, messageFilters);
         }
 
         @Override
-        void log(String msg) {
+        protected void log0(String msg) {
             lines.add(msg);
         }
     }
 
+    List<String> messageFilters = new ArrayList<>();
     TestLoggingOutputStream loggingStream;
     PrintStream printStream;
 
     @Before
     public void createStream() {
+        messageFilters.clear();
         loggingStream = new TestLoggingOutputStream();
         printStream = new PrintStream(loggingStream, false, StandardCharsets.UTF_8);
     }
@@ -115,4 +117,12 @@ public void testThreadIsolation() throws Exception {
         printStream.flush();
         assertThat(loggingStream.lines, contains("from thread 2", "from thread 1"));
     }
+
+    public void testMessageFilters() throws Exception {
+        messageFilters.add("foo bar");
+        printStream.println("prefix foo bar suffix");
+        printStream.println("non-filtered message");
+        printStream.flush();
+        assertThat(loggingStream.lines, contains("non-filtered message"));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/lucene/store/BytesReferenceIndexInputTests.java b/server/src/test/java/org/elasticsearch/common/lucene/store/BytesReferenceIndexInputTests.java
new file mode 100644
index 000000000000..39691ed64a10
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/common/lucene/store/BytesReferenceIndexInputTests.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common.lucene.store;
+
+import org.elasticsearch.common.bytes.BytesArray;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class BytesReferenceIndexInputTests extends ESIndexInputTestCase {
+    public void testRandomReads() throws IOException {
+        for (int i = 0; i < 100; i++) {
+            byte[] input = randomUnicodeOfLength(randomIntBetween(1, 1000)).getBytes(StandardCharsets.UTF_8);
+            BytesReferenceIndexInput indexInput = new BytesReferenceIndexInput("test", new BytesArray(input));
+            assertEquals(input.length, indexInput.length());
+            assertEquals(0, indexInput.getFilePointer());
+            byte[] output = randomReadAndSlice(indexInput, input.length);
+            assertArrayEquals(input, output);
+        }
+    }
+
+    public void testRandomOverflow() throws IOException {
+        for (int i = 0; i < 100; i++) {
+            byte[] input = randomUnicodeOfLength(randomIntBetween(1, 1000)).getBytes(StandardCharsets.UTF_8);
+            BytesReferenceIndexInput indexInput = new BytesReferenceIndexInput("test", new BytesArray(input));
+            int firstReadLen = randomIntBetween(0, input.length - 1);
+            randomReadAndSlice(indexInput, firstReadLen);
+            int bytesLeft = input.length - firstReadLen;
+
+            // read using int size
+            int secondReadLen = bytesLeft + randomIntBetween(1, 100);
+            expectThrows(EOFException.class, () -> indexInput.readBytes(new byte[secondReadLen], 0, secondReadLen));
+        }
+    }
+
+    public void testSeekOverflow() throws IOException {
+        for (int i = 0; i < 100; i++) {
+            byte[] input = randomUnicodeOfLength(randomIntBetween(1, 1000)).getBytes(StandardCharsets.UTF_8);
+            BytesReferenceIndexInput indexInput = new BytesReferenceIndexInput("test", new BytesArray(input));
+            int firstReadLen = randomIntBetween(0, input.length - 1);
+            randomReadAndSlice(indexInput, firstReadLen);
+            try {
+                switch (randomIntBetween(0, 2)) {
+                    case 0 -> indexInput.seek(Integer.MAX_VALUE + 4L);
+                    case 1 -> indexInput.seek(-randomIntBetween(1, 10));
+                    case 2 -> {
+                        int seek = input.length + randomIntBetween(1, 100);
+                        indexInput.seek(seek);
+                    }
+                    default -> fail();
+                }
+                fail();
+            } catch (EOFException ex) {
+                // ok
+            } catch (IllegalArgumentException ex) {
+                assertThat(ex.getMessage(), containsString("negative position"));
+            }
+        }
+    }
+
+}
diff --git a/server/src/test/java/org/elasticsearch/common/lucene/store/InputStreamIndexInputTests.java b/server/src/test/java/org/elasticsearch/common/lucene/store/InputStreamIndexInputTests.java
index 1e73989b12af..75380c2b38bb 100644
--- a/server/src/test/java/org/elasticsearch/common/lucene/store/InputStreamIndexInputTests.java
+++ b/server/src/test/java/org/elasticsearch/common/lucene/store/InputStreamIndexInputTests.java
@@ -241,4 +241,36 @@ public void testMarkRest() throws Exception {
         assertThat(is.read(), equalTo(1));
         assertThat(is.read(), equalTo(2));
     }
+
+    public void testReadZeroShouldReturnZero() throws IOException {
+        try (Directory dir = new ByteBuffersDirectory()) {
+            try (IndexOutput output = dir.createOutput("test", IOContext.DEFAULT)) {
+                output.writeByte((byte) 1);
+            }
+            try (IndexInput input = dir.openInput("test", IOContext.DEFAULT)) {
+                assertEquals(0, new InputStreamIndexInput(input, input.length()).read(new byte[randomIntBetween(0, 16)], 0, 0));
+            }
+        }
+    }
+
+    public void testReadAllBytes() throws IOException {
+        try (Directory dir = new ByteBuffersDirectory()) {
+            // Need to be bigger than InputStream#DEFAULT_BUFFER_SIZE in order to test that `readAllBytes`
+            // will call `read` again after calling it with length 0
+            int size = randomIntBetween(8193, 10_000);
+            try (IndexOutput output = dir.createOutput("test", IOContext.DEFAULT)) {
+                output.writeBytes(randomByteArrayOfLength(size), size);
+            }
+            try (IndexInput input = dir.openInput("test", IOContext.DEFAULT)) {
+                byte[] bytes = new InputStreamIndexInput(input, input.length()).readAllBytes();
+                assertEquals(size, bytes.length);
+            }
+            try (IndexInput input = dir.openInput("test", IOContext.DEFAULT)) {
+                // Verify that the respect the limit condition
+                long limit = randomLongBetween(0, input.length());
+                byte[] bytes = new InputStreamIndexInput(input, limit).readAllBytes();
+                assertEquals(limit, bytes.length);
+            }
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java
index 315712191e26..03c7c37026e3 100644
--- a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java
+++ b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java
@@ -10,6 +10,7 @@
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
@@ -19,6 +20,7 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndVersion;
 import org.elasticsearch.index.mapper.IdFieldMapper;
@@ -52,7 +54,7 @@ public void testSimple() throws Exception {
         writer.addDocument(new Document());
         DirectoryReader reader = DirectoryReader.open(writer);
         LeafReaderContext segment = reader.leaves().get(0);
-        PerThreadIDVersionAndSeqNoLookup lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME);
+        PerThreadIDVersionAndSeqNoLookup lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME, false);
         // found doc
         DocIdAndVersion result = lookup.lookupVersion(new BytesRef("6"), randomBoolean(), segment);
         assertNotNull(result);
@@ -65,7 +67,7 @@ public void testSimple() throws Exception {
         reader.close();
         reader = DirectoryReader.open(writer);
         segment = reader.leaves().get(0);
-        lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME);
+        lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME, false);
         assertNull(lookup.lookupVersion(new BytesRef("6"), randomBoolean(), segment));
         reader.close();
         writer.close();
@@ -88,7 +90,7 @@ public void testTwoDocuments() throws Exception {
         writer.addDocument(new Document());
         DirectoryReader reader = DirectoryReader.open(writer);
         LeafReaderContext segment = reader.leaves().get(0);
-        PerThreadIDVersionAndSeqNoLookup lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME);
+        PerThreadIDVersionAndSeqNoLookup lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME, false);
         // return the last doc when there are duplicates
         DocIdAndVersion result = lookup.lookupVersion(new BytesRef("6"), randomBoolean(), segment);
         assertNotNull(result);
@@ -99,7 +101,7 @@ public void testTwoDocuments() throws Exception {
         reader.close();
         reader = DirectoryReader.open(writer);
         segment = reader.leaves().get(0);
-        lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME);
+        lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME, false);
         result = lookup.lookupVersion(new BytesRef("6"), randomBoolean(), segment);
         assertNotNull(result);
         assertEquals(87, result.version);
@@ -109,10 +111,45 @@ public void testTwoDocuments() throws Exception {
         reader.close();
         reader = DirectoryReader.open(writer);
         segment = reader.leaves().get(0);
-        lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME);
+        lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME, false);
         assertNull(lookup.lookupVersion(new BytesRef("6"), randomBoolean(), segment));
         reader.close();
         writer.close();
         dir.close();
     }
+
+    public void testLoadTimestampRange() throws Exception {
+        Directory dir = newDirectory();
+        IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER).setMergePolicy(NoMergePolicy.INSTANCE));
+        Document doc = new Document();
+        doc.add(new Field(IdFieldMapper.NAME, "6", ProvidedIdFieldMapper.Defaults.FIELD_TYPE));
+        doc.add(new LongPoint(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, 1_000));
+        doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 87));
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, randomNonNegativeLong()));
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE)));
+        writer.addDocument(doc);
+        doc = new Document();
+        doc.add(new Field(IdFieldMapper.NAME, "8", ProvidedIdFieldMapper.Defaults.FIELD_TYPE));
+        doc.add(new LongPoint(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, 1_000_000));
+        doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 1));
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, randomNonNegativeLong()));
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE)));
+        writer.addDocument(doc);
+        DirectoryReader reader = DirectoryReader.open(writer);
+
+        LeafReaderContext segment = reader.leaves().get(0);
+        PerThreadIDVersionAndSeqNoLookup lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME, true);
+        assertTrue(lookup.loadedTimestampRange);
+        assertEquals(lookup.minTimestamp, 1_000L);
+        assertEquals(lookup.maxTimestamp, 1_000_000L);
+
+        lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), IdFieldMapper.NAME, false);
+        assertFalse(lookup.loadedTimestampRange);
+        assertEquals(lookup.minTimestamp, 0L);
+        assertEquals(lookup.maxTimestamp, Long.MAX_VALUE);
+
+        reader.close();
+        writer.close();
+        dir.close();
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java
index d5ebac5e7bb4..4554fc544efe 100644
--- a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java
+++ b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java
@@ -9,15 +9,21 @@
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.routing.IndexRouting;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
@@ -30,8 +36,10 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import static org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.loadDocIdAndVersion;
+import static org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.timeSeriesLoadDocIdAndVersion;
+import static org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper.createId;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 public class VersionsTests extends ESTestCase {
@@ -54,7 +62,7 @@ public void testVersions() throws Exception {
         Directory dir = newDirectory();
         IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
         DirectoryReader directoryReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "_na_", 1));
-        assertThat(loadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()), nullValue());
+        assertThat(timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()), nullValue());
 
         Document doc = new Document();
         doc.add(new Field(IdFieldMapper.NAME, "1", ProvidedIdFieldMapper.Defaults.FIELD_TYPE));
@@ -63,7 +71,7 @@ public void testVersions() throws Exception {
         doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE)));
         writer.updateDocument(new Term(IdFieldMapper.NAME, "1"), doc);
         directoryReader = reopen(directoryReader);
-        assertThat(loadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()).version, equalTo(1L));
+        assertThat(timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()).version, equalTo(1L));
 
         doc = new Document();
         Field uid = new Field(IdFieldMapper.NAME, "1", ProvidedIdFieldMapper.Defaults.FIELD_TYPE);
@@ -74,7 +82,7 @@ public void testVersions() throws Exception {
         doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE)));
         writer.updateDocument(new Term(IdFieldMapper.NAME, "1"), doc);
         directoryReader = reopen(directoryReader);
-        assertThat(loadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()).version, equalTo(2L));
+        assertThat(timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()).version, equalTo(2L));
 
         // test reuse of uid field
         doc = new Document();
@@ -86,11 +94,11 @@ public void testVersions() throws Exception {
         writer.updateDocument(new Term(IdFieldMapper.NAME, "1"), doc);
 
         directoryReader = reopen(directoryReader);
-        assertThat(loadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()).version, equalTo(3L));
+        assertThat(timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()).version, equalTo(3L));
 
         writer.deleteDocuments(new Term(IdFieldMapper.NAME, "1"));
         directoryReader = reopen(directoryReader);
-        assertThat(loadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()), nullValue());
+        assertThat(timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()), nullValue());
         directoryReader.close();
         writer.close();
         dir.close();
@@ -118,18 +126,18 @@ public void testNestedDocuments() throws IOException {
 
         writer.updateDocuments(new Term(IdFieldMapper.NAME, "1"), docs);
         DirectoryReader directoryReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "_na_", 1));
-        assertThat(loadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()).version, equalTo(5L));
+        assertThat(timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()).version, equalTo(5L));
 
         version.setLongValue(6L);
         writer.updateDocuments(new Term(IdFieldMapper.NAME, "1"), docs);
         version.setLongValue(7L);
         writer.updateDocuments(new Term(IdFieldMapper.NAME, "1"), docs);
         directoryReader = reopen(directoryReader);
-        assertThat(loadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()).version, equalTo(7L));
+        assertThat(timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()).version, equalTo(7L));
 
         writer.deleteDocuments(new Term(IdFieldMapper.NAME, "1"));
         directoryReader = reopen(directoryReader);
-        assertThat(loadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()), nullValue());
+        assertThat(timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()), nullValue());
         directoryReader.close();
         writer.close();
         dir.close();
@@ -149,10 +157,10 @@ public void testCache() throws Exception {
         writer.addDocument(doc);
         DirectoryReader reader = DirectoryReader.open(writer);
         // should increase cache size by 1
-        assertEquals(87, loadDocIdAndVersion(reader, new Term(IdFieldMapper.NAME, "6"), randomBoolean()).version);
+        assertEquals(87, timeSeriesLoadDocIdAndVersion(reader, new Term(IdFieldMapper.NAME, "6"), randomBoolean()).version);
         assertEquals(size + 1, VersionsAndSeqNoResolver.lookupStates.size());
         // should be cache hit
-        assertEquals(87, loadDocIdAndVersion(reader, new Term(IdFieldMapper.NAME, "6"), randomBoolean()).version);
+        assertEquals(87, timeSeriesLoadDocIdAndVersion(reader, new Term(IdFieldMapper.NAME, "6"), randomBoolean()).version);
         assertEquals(size + 1, VersionsAndSeqNoResolver.lookupStates.size());
 
         reader.close();
@@ -175,11 +183,11 @@ public void testCacheFilterReader() throws Exception {
         doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE)));
         writer.addDocument(doc);
         DirectoryReader reader = DirectoryReader.open(writer);
-        assertEquals(87, loadDocIdAndVersion(reader, new Term(IdFieldMapper.NAME, "6"), randomBoolean()).version);
+        assertEquals(87, timeSeriesLoadDocIdAndVersion(reader, new Term(IdFieldMapper.NAME, "6"), randomBoolean()).version);
         assertEquals(size + 1, VersionsAndSeqNoResolver.lookupStates.size());
         // now wrap the reader
         DirectoryReader wrapped = ElasticsearchDirectoryReader.wrap(reader, new ShardId("bogus", "_na_", 5));
-        assertEquals(87, loadDocIdAndVersion(wrapped, new Term(IdFieldMapper.NAME, "6"), randomBoolean()).version);
+        assertEquals(87, timeSeriesLoadDocIdAndVersion(wrapped, new Term(IdFieldMapper.NAME, "6"), randomBoolean()).version);
         // same size map: core cache key is shared
         assertEquals(size + 1, VersionsAndSeqNoResolver.lookupStates.size());
 
@@ -210,4 +218,67 @@ public void testLuceneVersionOnUnknownVersions() {
         version = Version.fromId(Version.CURRENT.id + 100);
         assertEquals(Version.CURRENT.luceneVersion, version.luceneVersion);
     }
+
+    public void testTimeSeriesLoadDocIdAndVersion() throws Exception {
+        Directory dir = newDirectory();
+        IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
+        DirectoryReader directoryReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "_na_", 1));
+        String id = createTSDBId(1000L);
+        assertThat(
+            VersionsAndSeqNoResolver.timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), id, randomBoolean()),
+            nullValue()
+        );
+
+        Document doc = new Document();
+        doc.add(new Field(IdFieldMapper.NAME, "1", ProvidedIdFieldMapper.Defaults.FIELD_TYPE));
+        doc.add(new LongPoint(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, 1_000));
+        doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 1));
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, 0L));
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, 1L));
+        writer.updateDocument(new Term(IdFieldMapper.NAME, "1"), doc);
+
+        doc = new Document();
+        doc.add(new Field(IdFieldMapper.NAME, "2", ProvidedIdFieldMapper.Defaults.FIELD_TYPE));
+        doc.add(new LongPoint(DataStream.TimestampField.FIXED_TIMESTAMP_FIELD, 10_000));
+        doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 1));
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, 0L));
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, 1L));
+        writer.updateDocument(new Term(IdFieldMapper.NAME, "2"), doc);
+        directoryReader = reopen(directoryReader);
+
+        id = createTSDBId(randomLongBetween(1000, 10000));
+        assertThat(
+            VersionsAndSeqNoResolver.timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), id, true),
+            notNullValue()
+        );
+        assertThat(
+            VersionsAndSeqNoResolver.timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "2"), id, true),
+            notNullValue()
+        );
+        id = createTSDBId(randomBoolean() ? randomLongBetween(0, 999) : randomLongBetween(10001, Long.MAX_VALUE));
+        assertThat(
+            VersionsAndSeqNoResolver.timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), id, true),
+            nullValue()
+        );
+        assertThat(
+            VersionsAndSeqNoResolver.timeSeriesLoadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "2"), id, true),
+            nullValue()
+        );
+
+        directoryReader.close();
+        writer.close();
+        dir.close();
+    }
+
+    private static String createTSDBId(long timestamp) {
+        Settings.Builder b = Settings.builder()
+            .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field");
+        IndexMetadata indexMetadata = IndexMetadata.builder("idx").settings(b).numberOfShards(1).numberOfReplicas(0).build();
+        IndexRouting.ExtractFromSource.Builder routingBuilder = ((IndexRouting.ExtractFromSource) IndexRouting.fromIndexMetadata(
+            indexMetadata
+        )).builder();
+        routingBuilder.addMatching("field", new BytesRef("value"));
+        return createId(false, routingBuilder, new BytesRef("tsid"), timestamp, new byte[16]);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/metrics/CountersTests.java b/server/src/test/java/org/elasticsearch/common/metrics/CountersTests.java
index 22a352851d21..426b5b949bc0 100644
--- a/server/src/test/java/org/elasticsearch/common/metrics/CountersTests.java
+++ b/server/src/test/java/org/elasticsearch/common/metrics/CountersTests.java
@@ -32,7 +32,7 @@ public void testCounters() {
         assertThat(counters.get("foo.bar"), equalTo(1L));
         assertThat(counters.get("foo.baz"), equalTo(2L));
         expectThrows(IllegalArgumentException.class, () -> counters.get("foo"));
-        Map<String, Object> map = counters.toNestedMap();
+        Map<String, Object> map = counters.toMutableNestedMap();
         assertThat(map, hasEntry("f", 200L));
         assertThat(map, hasKey("foo"));
         assertThat(map.get("foo"), instanceOf(Map.class));
@@ -58,6 +58,19 @@ public void testMerging() {
         assertThat(mergedCounters.get("bar.foo"), equalTo(1L));
     }
 
+    @SuppressWarnings("unchecked")
+    public void testNestedMapMutability() {
+        Counters counters = new Counters();
+        counters.inc("foo.baz");
+        Map<String, Object> map = counters.toMutableNestedMap();
+        try {
+            map.put("test", 1);
+            ((Map<String, Object>) map.get("foo")).put("nested-test", 2);
+        } catch (Exception e) {
+            fail("The map of the counters should be mutable.");
+        }
+    }
+
     public void testPathConflictNestedMapConversion() {
         Counters counters = new Counters();
         counters.inc("foo.bar");
@@ -67,7 +80,7 @@ public void testPathConflictNestedMapConversion() {
         assertThat(counters.get("foo"), equalTo(1L));
 
         try {
-            counters.toNestedMap();
+            counters.toMutableNestedMap();
             fail("Expected an IllegalStateException but got no exception");
         } catch (IllegalStateException e) {
             assertThat(e.getMessage(), containsString("Failed to convert counter 'foo"));
diff --git a/server/src/test/java/org/elasticsearch/common/regex/RegexTests.java b/server/src/test/java/org/elasticsearch/common/regex/RegexTests.java
index 09fb8a410653..ccff89491fbf 100644
--- a/server/src/test/java/org/elasticsearch/common/regex/RegexTests.java
+++ b/server/src/test/java/org/elasticsearch/common/regex/RegexTests.java
@@ -189,4 +189,27 @@ private void assertMatchesNone(Automaton automaton, String... strings) {
             assertFalse(run.run(s));
         }
     }
+
+    public void testSimpleMatcher() {
+        assertFalse(Regex.simpleMatcher((String[]) null).test("abc"));
+        assertFalse(Regex.simpleMatcher().test("abc"));
+        assertTrue(Regex.simpleMatcher("abc").test("abc"));
+        assertFalse(Regex.simpleMatcher("abc").test("abd"));
+
+        assertTrue(Regex.simpleMatcher("abc", "xyz").test("abc"));
+        assertTrue(Regex.simpleMatcher("abc", "xyz").test("xyz"));
+        assertFalse(Regex.simpleMatcher("abc", "xyz").test("abd"));
+        assertFalse(Regex.simpleMatcher("abc", "xyz").test("xyy"));
+
+        assertTrue(Regex.simpleMatcher("abc", "*").test("abc"));
+        assertTrue(Regex.simpleMatcher("abc", "*").test("abd"));
+
+        assertTrue(Regex.simpleMatcher("a*c").test("abc"));
+        assertFalse(Regex.simpleMatcher("a*c").test("abd"));
+
+        assertTrue(Regex.simpleMatcher("a*c", "x*z").test("abc"));
+        assertTrue(Regex.simpleMatcher("a*c", "x*z").test("xyz"));
+        assertFalse(Regex.simpleMatcher("a*c", "x*z").test("abd"));
+        assertFalse(Regex.simpleMatcher("a*c", "x*z").test("xyy"));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/settings/LocallyMountedSecretsTests.java b/server/src/test/java/org/elasticsearch/common/settings/LocallyMountedSecretsTests.java
new file mode 100644
index 000000000000..e443fffc7d73
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/common/settings/LocallyMountedSecretsTests.java
@@ -0,0 +1,149 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common.settings;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.security.GeneralSecurityException;
+import java.security.MessageDigest;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+
+public class LocallyMountedSecretsTests extends ESTestCase {
+    Environment env;
+
+    private static String testJSON = """
+        {
+             "metadata": {
+                 "version": "1",
+                 "compatibility": "8.4.0"
+             },
+             "secrets": {
+                 "aaa": "bbb",
+                 "ccc": "ddd"
+             }
+        }""";
+
+    private static String noMetadataJSON = """
+        {
+             "secrets": {
+                 "aaa": "bbb",
+                 "ccc": "ddd"
+             }
+        }""";
+
+    @Before
+    public void setupEnv() {
+        env = newEnvironment();
+    }
+
+    public void testCreate() {
+        SecureSettings secrets = new LocallyMountedSecrets(env);
+        assertTrue(secrets.isLoaded());
+    }
+
+    public void testProcessSettingsFile() throws IOException {
+        writeTestFile(env.configFile().resolve("secrets").resolve("secrets.json"), testJSON);
+        LocallyMountedSecrets secrets = new LocallyMountedSecrets(env);
+        assertTrue(secrets.isLoaded());
+        assertThat(secrets.getSettingNames(), containsInAnyOrder("aaa", "ccc"));
+        assertEquals("bbb", secrets.getString("aaa").toString());
+        assertEquals("ddd", secrets.getString("ccc").toString());
+    }
+
+    public void testSettingsGetFile() throws IOException, GeneralSecurityException {
+        writeTestFile(env.configFile().resolve("secrets").resolve("secrets.json"), testJSON);
+        LocallyMountedSecrets secrets = new LocallyMountedSecrets(env);
+        assertTrue(secrets.isLoaded());
+        assertThat(secrets.getSettingNames(), containsInAnyOrder("aaa", "ccc"));
+        try (InputStream stream = secrets.getFile("aaa")) {
+            for (int i = 0; i < 3; ++i) {
+                int got = stream.read();
+                if (got < 0) {
+                    fail("Expected 3 bytes but read " + i);
+                }
+                assertEquals('b', got);
+            }
+            assertEquals(-1, stream.read()); // nothing left
+        }
+    }
+
+    public void testSettingsSHADigest() throws IOException, GeneralSecurityException {
+        writeTestFile(env.configFile().resolve("secrets").resolve("secrets.json"), testJSON);
+        LocallyMountedSecrets secrets = new LocallyMountedSecrets(env);
+        assertTrue(secrets.isLoaded());
+        assertThat(secrets.getSettingNames(), containsInAnyOrder("aaa", "ccc"));
+
+        final byte[] stringSettingHash = MessageDigest.getInstance("SHA-256").digest("bbb".getBytes(StandardCharsets.UTF_8));
+        assertThat(secrets.getSHA256Digest("aaa"), equalTo(stringSettingHash));
+    }
+
+    public void testProcessBadSettingsFile() throws IOException {
+        writeTestFile(env.configFile().resolve("secrets").resolve("secrets.json"), noMetadataJSON);
+        assertThat(
+            expectThrows(IllegalArgumentException.class, () -> new LocallyMountedSecrets(env)).getMessage(),
+            containsString("Required [metadata]")
+        );
+    }
+
+    public void testSerializationWithSecrets() throws Exception {
+        writeTestFile(env.configFile().resolve("secrets").resolve("secrets.json"), testJSON);
+        LocallyMountedSecrets secrets = new LocallyMountedSecrets(env);
+
+        final BytesStreamOutput out = new BytesStreamOutput();
+        secrets.writeTo(out);
+        final LocallyMountedSecrets fromStream = new LocallyMountedSecrets(out.bytes().streamInput());
+
+        assertThat(fromStream.getSettingNames(), hasSize(2));
+        assertThat(fromStream.getSettingNames(), containsInAnyOrder("aaa", "ccc"));
+
+        assertEquals(secrets.getString("aaa"), fromStream.getString("aaa"));
+        assertTrue(fromStream.isLoaded());
+    }
+
+    public void testSerializationNewlyCreated() throws Exception {
+        LocallyMountedSecrets secrets = new LocallyMountedSecrets(env);
+
+        final BytesStreamOutput out = new BytesStreamOutput();
+        secrets.writeTo(out);
+        final LocallyMountedSecrets fromStream = new LocallyMountedSecrets(out.bytes().streamInput());
+
+        assertTrue(fromStream.isLoaded());
+    }
+
+    public void testClose() throws IOException {
+        writeTestFile(env.configFile().resolve("secrets").resolve("secrets.json"), testJSON);
+        LocallyMountedSecrets secrets = new LocallyMountedSecrets(env);
+        assertEquals("bbb", secrets.getString("aaa").toString());
+        assertEquals("ddd", secrets.getString("ccc").toString());
+        secrets.close();
+        assertNull(secrets.getString("aaa"));
+        assertNull(secrets.getString("ccc"));
+    }
+
+    private void writeTestFile(Path path, String contents) throws IOException {
+        Path tempFilePath = createTempFile();
+
+        Files.write(tempFilePath, contents.getBytes(StandardCharsets.UTF_8));
+        Files.createDirectories(path.getParent());
+        Files.move(tempFilePath, path, StandardCopyOption.ATOMIC_MOVE);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/common/settings/SettingsTests.java b/server/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
index 2cc11e68f9c5..a415d3b8c2ef 100644
--- a/server/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
+++ b/server/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -26,6 +27,7 @@
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -485,6 +487,43 @@ public void testSecureSettingIllegalName() {
         assertTrue(e.getMessage().contains("does not match the allowed setting name pattern"));
     }
 
+    public void testStatelessSecureSettingsWithoutStateless() {
+        Setting<?> setting = SecureSetting.secureString(StatelessSecureSettings.PREFIX + "key", null);
+
+        final Settings settings = Settings.builder()
+            .put(DiscoveryNode.STATELESS_ENABLED_SETTING_NAME, false)
+            .put(setting.getKey(), "yaml.value")
+            .build();
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> StatelessSecureSettings.install(settings));
+        assertTrue(e.getMessage().contains("supported only in stateless"));
+    }
+
+    public void testStatelessSecureSettings() throws Exception {
+        boolean testFileSettingInsteadOfStringSetting = randomBoolean();
+
+        Setting<?> yamlSetting = Setting.simpleString(StatelessSecureSettings.PREFIX + "stateless.key");
+        Setting<?> secureSetting = testFileSettingInsteadOfStringSetting
+            ? SecureSetting.secureFile("stateless.key", null)
+            : SecureSetting.secureString("stateless.key", null);
+
+        final Settings settings = Settings.builder()
+            .put(DiscoveryNode.STATELESS_ENABLED_SETTING_NAME, true)
+            .put(yamlSetting.getKey(), "stateless.yaml.value")
+            .build();
+
+        Settings newSettings = StatelessSecureSettings.install(settings);
+        assertTrue(secureSetting.exists(newSettings));
+        assertEquals(
+            "stateless.yaml.value",
+            testFileSettingInsteadOfStringSetting
+                ? new String(((InputStream) secureSetting.get(newSettings)).readAllBytes(), StandardCharsets.UTF_8)
+                : secureSetting.get(newSettings).toString()
+        );
+        assertTrue(yamlSetting.exists(newSettings));
+        assertEquals("stateless.yaml.value", yamlSetting.get(newSettings).toString());
+    }
+
     public void testGetAsArrayFailsOnDuplicates() {
         final IllegalStateException e = expectThrows(
             IllegalStateException.class,
diff --git a/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java b/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java
index 627861f0a9f7..1b17b266d9cb 100644
--- a/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java
+++ b/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.common.time;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 
 import java.time.Instant;
@@ -403,7 +404,7 @@ private void assertDateEquals(long gotMillis, String original, String expected)
         long expectedMillis = parser.parse(expected, () -> 0).toEpochMilli();
         if (gotMillis != expectedMillis) {
             ZonedDateTime zonedDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(gotMillis), ZoneOffset.UTC);
-            fail(formatted("""
+            fail(Strings.format("""
                 Date math not equal
                 Original              : %s
                 Parsed                : %s
diff --git a/server/src/test/java/org/elasticsearch/common/unit/ProcessorsSerializationTests.java b/server/src/test/java/org/elasticsearch/common/unit/ProcessorsSerializationTests.java
index 25d024f23c35..a8bd71c9fdef 100644
--- a/server/src/test/java/org/elasticsearch/common/unit/ProcessorsSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/common/unit/ProcessorsSerializationTests.java
@@ -11,8 +11,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
-
 public class ProcessorsSerializationTests extends AbstractWireSerializingTestCase<Processors> {
     @Override
     protected Writeable.Reader<Processors> instanceReader() {
@@ -25,7 +23,7 @@ protected Processors createTestInstance() {
     }
 
     @Override
-    protected Processors mutateInstance(Processors instance) throws IOException {
+    protected Processors mutateInstance(Processors instance) {
         return Processors.of(instance.count() + randomDoubleBetween(0.01, 1, true));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/util/ArrayUtilsTests.java b/server/src/test/java/org/elasticsearch/common/util/ArrayUtilsTests.java
index 38019410adae..b22d72988751 100644
--- a/server/src/test/java/org/elasticsearch/common/util/ArrayUtilsTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/ArrayUtilsTests.java
@@ -14,6 +14,7 @@
 import java.util.Arrays;
 import java.util.BitSet;
 
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
 public class ArrayUtilsTests extends ESTestCase {
@@ -79,4 +80,29 @@ public void testConcat() {
         }
         assertArrayEquals(sourceOfTruth.toArray(new String[0]), ArrayUtils.concat(first, second));
     }
+
+    public void testReverseSubArray() {
+        final int length = randomIntBetween(10, 50);
+        final double[] array = new double[length];
+        for (int i = 0; i < array.length; i++) {
+            array[i] = randomDoubleBetween(Double.NEGATIVE_INFINITY, Double.MAX_VALUE, true);
+        }
+        assertReverseSubArray(array, 0, array.length);
+        for (int i = 0; i < 10; i++) {
+            final int start = randomInt(length);
+            final int end = randomIntBetween(start, length);
+            assertReverseSubArray(array, start, end - start);
+        }
+    }
+
+    private void assertReverseSubArray(double[] array, int from, int length) {
+        final double[] copy = array.clone();
+        ArrayUtils.reverseSubArray(copy, from, length);
+        final int endOffset = from + length - 1;
+        for (int i = 0; i < length; i++) {
+            assertThat(copy[endOffset - i], equalTo(array[from + i]));
+        }
+        ArrayUtils.reverseSubArray(copy, from, length);
+        assertThat(copy, equalTo(array));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/util/CollectionUtilsTests.java b/server/src/test/java/org/elasticsearch/common/util/CollectionUtilsTests.java
index 89f55a6981ff..3b3d205a8ebb 100644
--- a/server/src/test/java/org/elasticsearch/common/util/CollectionUtilsTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/CollectionUtilsTests.java
@@ -8,10 +8,10 @@
 
 package org.elasticsearch.common.util;
 
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -19,9 +19,10 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
-import static java.util.Collections.emptyMap;
 import static org.elasticsearch.common.util.CollectionUtils.eagerPartition;
+import static org.elasticsearch.common.util.CollectionUtils.ensureNoSelfReferences;
 import static org.elasticsearch.common.util.CollectionUtils.limitSize;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -29,7 +30,7 @@
 
 public class CollectionUtilsTests extends ESTestCase {
     public void testRotateEmpty() {
-        assertTrue(CollectionUtils.rotate(Collections.emptyList(), randomInt()).isEmpty());
+        assertTrue(CollectionUtils.rotate(List.of(), randomInt()).isEmpty());
     }
 
     public void testRotate() {
@@ -77,59 +78,109 @@ public void testUniquify() {
     }
 
     public void testEmptyPartition() {
-        assertEquals(Collections.emptyList(), eagerPartition(Collections.emptyList(), 1));
+        assertEquals(List.of(), eagerPartition(List.of(), 1));
     }
 
     public void testSimplePartition() {
-        assertEquals(
-            Arrays.asList(Arrays.asList(1, 2), Arrays.asList(3, 4), Arrays.asList(5)),
-            eagerPartition(Arrays.asList(1, 2, 3, 4, 5), 2)
-        );
+        assertEquals(List.of(List.of(1, 2), List.of(3, 4), List.of(5)), eagerPartition(List.of(1, 2, 3, 4, 5), 2));
     }
 
     public void testSingletonPartition() {
-        assertEquals(
-            Arrays.asList(Arrays.asList(1), Arrays.asList(2), Arrays.asList(3), Arrays.asList(4), Arrays.asList(5)),
-            eagerPartition(Arrays.asList(1, 2, 3, 4, 5), 1)
-        );
+        assertEquals(List.of(List.of(1), List.of(2), List.of(3), List.of(4), List.of(5)), eagerPartition(List.of(1, 2, 3, 4, 5), 1));
     }
 
     public void testOversizedPartition() {
-        assertEquals(Arrays.asList(Arrays.asList(1, 2, 3, 4, 5)), eagerPartition(Arrays.asList(1, 2, 3, 4, 5), 15));
+        assertEquals(List.of(List.of(1, 2, 3, 4, 5)), eagerPartition(List.of(1, 2, 3, 4, 5), 15));
     }
 
     public void testPerfectPartition() {
         assertEquals(
-            Arrays.asList(Arrays.asList(1, 2, 3, 4, 5, 6), Arrays.asList(7, 8, 9, 10, 11, 12)),
-            eagerPartition(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12), 6)
+            List.of(List.of(1, 2, 3, 4, 5, 6), List.of(7, 8, 9, 10, 11, 12)),
+            eagerPartition(List.of(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12), 6)
         );
     }
 
     public void testEnsureNoSelfReferences() {
-        CollectionUtils.ensureNoSelfReferences(emptyMap(), "test with empty map");
-        CollectionUtils.ensureNoSelfReferences(null, "test with null");
+        ensureNoSelfReferences("string", "test with a string");
+        ensureNoSelfReferences(2, "test with a number");
+        ensureNoSelfReferences(true, "test with a boolean");
+        ensureNoSelfReferences(Map.of(), "test with an empty map");
+        ensureNoSelfReferences(Set.of(), "test with an empty set");
+        ensureNoSelfReferences(List.of(), "test with an empty list");
+        ensureNoSelfReferences(new Object[0], "test with an empty array");
+        ensureNoSelfReferences((Iterable<?>) Collections::emptyIterator, "test with an empty iterable");
+    }
 
+    public void testEnsureNoSelfReferencesMap() {
+        // map value
         {
             Map<String, Object> map = new HashMap<>();
             map.put("field", map);
 
             IllegalArgumentException e = expectThrows(
                 IllegalArgumentException.class,
-                () -> CollectionUtils.ensureNoSelfReferences(map, "test with self ref value")
+                () -> ensureNoSelfReferences(map, "test with self ref value")
             );
             assertThat(e.getMessage(), containsString("Iterable object is self-referencing itself (test with self ref value)"));
         }
+        // map key
         {
             Map<Object, Object> map = new HashMap<>();
             map.put(map, 1);
 
             IllegalArgumentException e = expectThrows(
                 IllegalArgumentException.class,
-                () -> CollectionUtils.ensureNoSelfReferences(map, "test with self ref key")
+                () -> ensureNoSelfReferences(map, "test with self ref key")
             );
             assertThat(e.getMessage(), containsString("Iterable object is self-referencing itself (test with self ref key)"));
         }
+        // nested map value
+        {
+            Map<String, Object> map = new HashMap<>();
+            map.put("field", Set.of(List.of((Iterable<?>) () -> Iterators.single(new Object[] { map }))));
 
+            IllegalArgumentException e = expectThrows(
+                IllegalArgumentException.class,
+                () -> ensureNoSelfReferences(map, "test with self ref nested value")
+            );
+            assertThat(e.getMessage(), containsString("Iterable object is self-referencing itself (test with self ref nested value)"));
+        }
+    }
+
+    public void testEnsureNoSelfReferencesSet() {
+        Set<Object> set = new HashSet<>();
+        set.add("foo");
+        set.add(set);
+
+        IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> ensureNoSelfReferences(set, "test with self ref set")
+        );
+        assertThat(e.getMessage(), containsString("Iterable object is self-referencing itself (test with self ref set)"));
+    }
+
+    public void testEnsureNoSelfReferencesList() {
+        List<Object> list = new ArrayList<>();
+        list.add("foo");
+        list.add(list);
+
+        IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> ensureNoSelfReferences(list, "test with self ref list")
+        );
+        assertThat(e.getMessage(), containsString("Iterable object is self-referencing itself (test with self ref list)"));
+    }
+
+    public void testEnsureNoSelfReferencesArray() {
+        Object[] array = new Object[2];
+        array[0] = "foo";
+        array[1] = array;
+
+        IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> ensureNoSelfReferences(array, "test with self ref array")
+        );
+        assertThat(e.getMessage(), containsString("Iterable object is self-referencing itself (test with self ref array)"));
     }
 
     public void testLimitSizeOfShortList() {
diff --git a/server/src/test/java/org/elasticsearch/common/util/CopyOnFirstWriteMapTests.java b/server/src/test/java/org/elasticsearch/common/util/CopyOnFirstWriteMapTests.java
new file mode 100644
index 000000000000..41870862c437
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/common/util/CopyOnFirstWriteMapTests.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common.util;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.sameInstance;
+
+public class CopyOnFirstWriteMapTests extends ESTestCase {
+
+    public void testShouldNotCopyIfThereWereNoUpdates() {
+        var source = Map.of("key", "value");
+        var copyOnFirstWrite = new CopyOnFirstWriteMap<>(source);
+        source.get("key");
+        var copy = copyOnFirstWrite.toImmutableMap();
+
+        assertThat(copy, sameInstance(source));
+        assertThat(copy, equalTo(source));
+    }
+
+    public void testShouldBeUpdatable() {
+        var source = Map.of("key", "value");
+        var copyOnFirstWrite = new CopyOnFirstWriteMap<>(source);
+        copyOnFirstWrite.put("key", "new_value");
+        var copy = copyOnFirstWrite.toImmutableMap();
+
+        assertThat(copy, not(sameInstance(source)));
+        assertThat(copy, equalTo(Map.of("key", "new_value")));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/common/util/SetBackedScalingCuckooFilterTests.java b/server/src/test/java/org/elasticsearch/common/util/SetBackedScalingCuckooFilterTests.java
index 9fbbaacbc928..a5315a9b4f13 100644
--- a/server/src/test/java/org/elasticsearch/common/util/SetBackedScalingCuckooFilterTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/SetBackedScalingCuckooFilterTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.HashSet;
 import java.util.Set;
 
@@ -41,7 +40,7 @@ protected Writeable.Reader<SetBackedScalingCuckooFilter> instanceReader() {
     }
 
     @Override
-    protected SetBackedScalingCuckooFilter mutateInstance(SetBackedScalingCuckooFilter instance) throws IOException {
+    protected SetBackedScalingCuckooFilter mutateInstance(SetBackedScalingCuckooFilter instance) {
         SetBackedScalingCuckooFilter newInstance = new SetBackedScalingCuckooFilter(
             instance.getThreshold(),
             instance.getRng(),
diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/AbstractThrottledTaskRunnerTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/AbstractThrottledTaskRunnerTests.java
new file mode 100644
index 000000000000..53df99d2641e
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/AbstractThrottledTaskRunnerTests.java
@@ -0,0 +1,210 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common.util.concurrent;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+public class AbstractThrottledTaskRunnerTests extends ESTestCase {
+
+    private static final ThreadFactory threadFactory = EsExecutors.daemonThreadFactory("test");
+    private static final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+
+    private ExecutorService executor;
+    private int maxThreads;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        maxThreads = between(1, 10);
+        executor = EsExecutors.newScaling("test", maxThreads, maxThreads, 0, TimeUnit.MILLISECONDS, false, threadFactory, threadContext);
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        terminate(executor);
+    }
+
+    public void testMultiThreadedEnqueue() throws Exception {
+        final int maxTasks = randomIntBetween(1, 2 * maxThreads);
+        final var permits = new Semaphore(maxTasks);
+        final int totalTasks = randomIntBetween(2 * maxTasks, 10 * maxTasks);
+        final var latch = new CountDownLatch(totalTasks);
+
+        class TestTask implements ActionListener<Releasable> {
+
+            private final ExecutorService taskExecutor = randomFrom(executor, EsExecutors.DIRECT_EXECUTOR_SERVICE);
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError(e);
+            }
+
+            @Override
+            public void onResponse(Releasable releasable) {
+                assertTrue(permits.tryAcquire());
+                try {
+                    Thread.sleep(between(0, 10));
+                } catch (InterruptedException e) {
+                    throw new AssertionError(e);
+                }
+                taskExecutor.execute(() -> {
+                    permits.release();
+                    releasable.close();
+                    latch.countDown();
+                });
+            }
+        }
+
+        final BlockingQueue<TestTask> queue = ConcurrentCollections.newBlockingQueue();
+        final AbstractThrottledTaskRunner<TestTask> taskRunner = new AbstractThrottledTaskRunner<>("test", maxTasks, executor, queue);
+
+        final var threadBlocker = new CyclicBarrier(totalTasks);
+        for (int i = 0; i < totalTasks; i++) {
+            new Thread(() -> {
+                CyclicBarrierUtils.await(threadBlocker);
+                taskRunner.enqueueTask(new TestTask());
+                assertThat(taskRunner.runningTasks(), lessThanOrEqualTo(maxTasks));
+            }).start();
+        }
+        // Eventually all tasks are executed
+        assertTrue(latch.await(10, TimeUnit.SECONDS));
+        assertTrue(queue.isEmpty());
+        assertTrue(permits.tryAcquire(maxTasks));
+        assertNoRunningTasks(taskRunner);
+    }
+
+    public void testEnqueueSpawnsNewTasksUpToMax() throws Exception {
+        int maxTasks = randomIntBetween(1, maxThreads);
+        final int enqueued = maxTasks - 1; // So that it is possible to run at least one more task
+        final int newTasks = randomIntBetween(1, 10);
+
+        CountDownLatch taskBlocker = new CountDownLatch(1);
+        CountDownLatch executedCountDown = new CountDownLatch(enqueued + newTasks);
+
+        class TestTask implements ActionListener<Releasable> {
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError(e);
+            }
+
+            @Override
+            public void onResponse(Releasable releasable) {
+                try {
+                    taskBlocker.await();
+                } catch (InterruptedException e) {
+                    throw new AssertionError(e);
+                } finally {
+                    executedCountDown.countDown();
+                    releasable.close();
+                }
+            }
+        }
+
+        final BlockingQueue<TestTask> queue = ConcurrentCollections.newBlockingQueue();
+        final AbstractThrottledTaskRunner<TestTask> taskRunner = new AbstractThrottledTaskRunner<>("test", maxTasks, executor, queue);
+        for (int i = 0; i < enqueued; i++) {
+            taskRunner.enqueueTask(new TestTask());
+            assertThat(taskRunner.runningTasks(), equalTo(i + 1));
+            assertTrue(queue.isEmpty());
+        }
+        // Enqueueing one or more new tasks would create only one new running task
+        for (int i = 0; i < newTasks; i++) {
+            taskRunner.enqueueTask(new TestTask());
+            assertThat(taskRunner.runningTasks(), equalTo(maxTasks));
+            assertThat(queue.size(), equalTo(i));
+        }
+        taskBlocker.countDown();
+        /// Eventually all tasks are executed
+        assertTrue(executedCountDown.await(10, TimeUnit.SECONDS));
+        assertTrue(queue.isEmpty());
+        assertNoRunningTasks(taskRunner);
+    }
+
+    public void testFailsTasksOnRejectionOrShutdown() throws Exception {
+        final var executor = randomBoolean()
+            ? EsExecutors.newScaling("test", maxThreads, maxThreads, 0, TimeUnit.MILLISECONDS, true, threadFactory, threadContext)
+            : EsExecutors.newFixed("test", maxThreads, between(1, 5), threadFactory, threadContext, false);
+
+        final var totalPermits = between(1, maxThreads * 2);
+        final var permits = new Semaphore(totalPermits);
+        final var taskCompleted = new CountDownLatch(between(1, maxThreads * 2));
+        final var rejectionCountDown = new CountDownLatch(between(1, maxThreads * 2));
+
+        class TestTask implements ActionListener<Releasable> {
+
+            @Override
+            public void onFailure(Exception e) {
+                rejectionCountDown.countDown();
+                permits.release();
+            }
+
+            @Override
+            public void onResponse(Releasable releasable) {
+                permits.release();
+                taskCompleted.countDown();
+                releasable.close();
+            }
+        }
+
+        final BlockingQueue<TestTask> queue = ConcurrentCollections.newBlockingQueue();
+        final AbstractThrottledTaskRunner<TestTask> taskRunner = new AbstractThrottledTaskRunner<>(
+            "test",
+            between(1, maxThreads * 2),
+            executor,
+            queue
+        );
+
+        final var spawnThread = new Thread(() -> {
+            try {
+                while (true) {
+                    assertTrue(permits.tryAcquire(10, TimeUnit.SECONDS));
+                    taskRunner.enqueueTask(new TestTask());
+                }
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+        });
+        spawnThread.start();
+        assertTrue(taskCompleted.await(10, TimeUnit.SECONDS));
+        executor.shutdown();
+        assertTrue(executor.awaitTermination(30, TimeUnit.SECONDS));
+        assertTrue(rejectionCountDown.await(10, TimeUnit.SECONDS));
+        spawnThread.interrupt();
+        spawnThread.join();
+        assertThat(taskRunner.runningTasks(), equalTo(0));
+        assertTrue(queue.isEmpty());
+        assertTrue(permits.tryAcquire(totalPermits));
+    }
+
+    private void assertNoRunningTasks(AbstractThrottledTaskRunner<?> taskRunner) {
+        final var barrier = new CyclicBarrier(maxThreads + 1);
+        for (int i = 0; i < maxThreads; i++) {
+            executor.execute(() -> CyclicBarrierUtils.await(barrier));
+        }
+        CyclicBarrierUtils.await(barrier);
+        assertThat(taskRunner.runningTasks(), equalTo(0));
+    }
+
+}
diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/ListenableFutureTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/ListenableFutureTests.java
index c79864bfba14..ef0abce6d8ba 100644
--- a/server/src/test/java/org/elasticsearch/common/util/concurrent/ListenableFutureTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/ListenableFutureTests.java
@@ -8,9 +8,11 @@
 
 package org.elasticsearch.common.util.concurrent;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ReachabilityChecker;
 import org.junit.After;
 
 import java.util.concurrent.BrokenBarrierException;
@@ -124,4 +126,23 @@ public void testConcurrentListenerRegistrationAndCompletion() throws BrokenBarri
         assertEquals(numberOfThreads - 1, numResponses.get());
         assertEquals(0, numExceptions.get());
     }
+
+    public void testAddedListenersReleasedOnCompletion() {
+        final ListenableFuture<Void> future = new ListenableFuture<>();
+        final ReachabilityChecker reachabilityChecker = new ReachabilityChecker();
+
+        for (int i = between(1, 3); i > 0; i--) {
+            future.addListener(reachabilityChecker.register(ActionListener.wrap(() -> {})));
+        }
+        reachabilityChecker.checkReachable();
+        if (randomBoolean()) {
+            future.onResponse(null);
+        } else {
+            future.onFailure(new ElasticsearchException("simulated"));
+        }
+        reachabilityChecker.ensureUnreachable();
+
+        future.addListener(reachabilityChecker.register(ActionListener.wrap(() -> {})));
+        reachabilityChecker.ensureUnreachable();
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/PrioritizedThrottledTaskRunnerTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/PrioritizedThrottledTaskRunnerTests.java
index 3768f5556b4e..6ded72fd1f62 100644
--- a/server/src/test/java/org/elasticsearch/common/util/concurrent/PrioritizedThrottledTaskRunnerTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/PrioritizedThrottledTaskRunnerTests.java
@@ -8,40 +8,45 @@
 
 package org.elasticsearch.common.util.concurrent;
 
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
-import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Executor;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 public class PrioritizedThrottledTaskRunnerTests extends ESTestCase {
-    private ThreadPool threadPool;
-    private Executor executor;
+
+    private static final ThreadFactory threadFactory = EsExecutors.daemonThreadFactory("test");
+    private static final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+
+    private ExecutorService executor;
+    private int maxThreads;
 
     @Override
     public void setUp() throws Exception {
         super.setUp();
-        threadPool = new TestThreadPool("test");
-        executor = threadPool.executor(ThreadPool.Names.GENERIC);
+        maxThreads = between(1, 10);
+        executor = EsExecutors.newScaling("test", maxThreads, maxThreads, 0, TimeUnit.NANOSECONDS, false, threadFactory, threadContext);
     }
 
     @Override
     public void tearDown() throws Exception {
         super.tearDown();
-        TestThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS);
+        TestThreadPool.terminate(executor, 30, TimeUnit.SECONDS);
     }
 
-    static class TestTask implements Comparable<TestTask>, Runnable {
-
+    static class TestTask extends AbstractRunnable implements Comparable<TestTask> {
         private final Runnable runnable;
         private final int priority;
 
@@ -56,122 +61,188 @@ public int compareTo(TestTask o) {
         }
 
         @Override
-        public void run() {
+        public void doRun() {
             runnable.run();
         }
+
+        @Override
+        public void onFailure(Exception e) {
+            throw new AssertionError("unexpected", e);
+        }
     }
 
     public void testMultiThreadedEnqueue() throws Exception {
-        final int maxTasks = randomIntBetween(1, threadPool.info(ThreadPool.Names.GENERIC).getMax());
+        final int maxTasks = randomIntBetween(1, maxThreads);
         PrioritizedThrottledTaskRunner<TestTask> taskRunner = new PrioritizedThrottledTaskRunner<>("test", maxTasks, executor);
         final int enqueued = randomIntBetween(2 * maxTasks, 10 * maxTasks);
-        AtomicInteger executed = new AtomicInteger();
-        CountDownLatch threadBlocker = new CountDownLatch(enqueued);
+        final var threadBlocker = new CyclicBarrier(enqueued);
+        final var executedCountDown = new CountDownLatch(enqueued);
         for (int i = 0; i < enqueued; i++) {
+            final int taskId = i;
             new Thread(() -> {
-                try {
-                    threadBlocker.countDown();
-                    threadBlocker.await();
-                } catch (InterruptedException e) {
-                    throw new RuntimeException(e);
-                }
+                CyclicBarrierUtils.await(threadBlocker);
                 taskRunner.enqueueTask(new TestTask(() -> {
                     try {
                         Thread.sleep(randomLongBetween(0, 10));
                     } catch (InterruptedException e) {
-                        throw new RuntimeException(e);
+                        throw new AssertionError(e);
                     }
-                    executed.incrementAndGet();
+                    executedCountDown.countDown();
                 }, getRandomPriority()));
                 assertThat(taskRunner.runningTasks(), lessThanOrEqualTo(maxTasks));
             }).start();
         }
         // Eventually all tasks are executed
-        assertBusy(() -> {
-            assertThat(executed.get(), equalTo(enqueued));
-            assertThat(taskRunner.runningTasks(), equalTo(0));
-        });
+        assertTrue(executedCountDown.await(10, TimeUnit.SECONDS));
         assertThat(taskRunner.queueSize(), equalTo(0));
+        assertNoRunningTasks(taskRunner);
     }
 
     public void testTasksRunInOrder() throws Exception {
-        final int n = randomIntBetween(1, threadPool.info(ThreadPool.Names.GENERIC).getMax());
-        final int enqueued = randomIntBetween(2 * n, 10 * n);
+        final int n = randomIntBetween(1, maxThreads);
         // To check that tasks are run in the order (based on their priority), limit max running tasks to 1
         // and wait until all tasks are enqueued.
-        CountDownLatch workerBlocker = new CountDownLatch(1);
-        PrioritizedThrottledTaskRunner<TestTask> taskRunner = new PrioritizedThrottledTaskRunner<>("test", 1, executor) {
-            @Override
-            protected void pollAndSpawn() {
-                try {
-                    workerBlocker.await();
-                } catch (InterruptedException e) {
-                    throw new RuntimeException(e);
-                }
-                super.pollAndSpawn();
-            }
-        };
+
+        final var taskRunner = new PrioritizedThrottledTaskRunner<TestTask>("test", 1, executor);
+
+        final var blockBarrier = new CyclicBarrier(2);
+        taskRunner.enqueueTask(new TestTask(() -> {
+            CyclicBarrierUtils.await(blockBarrier); // notify main thread that the runner is blocked
+            CyclicBarrierUtils.await(blockBarrier); // wait for main thread to finish enqueuing tasks
+        }, getRandomPriority()));
+
+        blockBarrier.await(10, TimeUnit.SECONDS); // wait for blocking task to start executing
+
+        final int enqueued = randomIntBetween(2 * n, 10 * n);
         List<Integer> taskPriorities = new ArrayList<>(enqueued);
         List<Integer> executedPriorities = new ArrayList<>(enqueued);
+        final var enqueuedBarrier = new CyclicBarrier(enqueued + 1);
+        final var executedCountDown = new CountDownLatch(enqueued);
         for (int i = 0; i < enqueued; i++) {
+            final int taskId = i;
             final int priority = getRandomPriority();
             taskPriorities.add(priority);
-            new Thread(() -> taskRunner.enqueueTask(new TestTask(() -> executedPriorities.add(priority), priority))).start();
+            new Thread(() -> {
+                // wait until all threads are ready so the enqueueTask() calls are as concurrent as possible
+                CyclicBarrierUtils.await(enqueuedBarrier);
+                taskRunner.enqueueTask(new TestTask(() -> {
+                    executedPriorities.add(priority);
+                    executedCountDown.countDown();
+                }, priority));
+                CyclicBarrierUtils.await(enqueuedBarrier); // notify main thread that the task is enqueued
+            }).start();
         }
-        assertBusy(() -> assertThat(taskRunner.queueSize(), equalTo(enqueued)));
-        assertThat(taskRunner.runningTasks(), equalTo(0));
-        workerBlocker.countDown();
+        // release all the threads at once
+        enqueuedBarrier.await(10, TimeUnit.SECONDS);
+        // wait for all threads to confirm the task is enqueued
+        enqueuedBarrier.await(10, TimeUnit.SECONDS);
+        assertThat(taskRunner.queueSize(), equalTo(enqueued));
+
+        blockBarrier.await(10, TimeUnit.SECONDS); // notify blocking task that it can continue
+
         // Eventually all tasks are executed
-        assertBusy(() -> {
-            assertThat(executedPriorities.size(), equalTo(enqueued));
-            assertThat(taskRunner.runningTasks(), equalTo(0));
-        });
+        assertTrue(executedCountDown.await(10, TimeUnit.SECONDS));
+        assertThat(executedPriorities.size(), equalTo(enqueued));
         assertThat(taskRunner.queueSize(), equalTo(0));
         Collections.sort(taskPriorities);
         assertThat(executedPriorities, equalTo(taskPriorities));
+        assertNoRunningTasks(taskRunner);
     }
 
     public void testEnqueueSpawnsNewTasksUpToMax() throws Exception {
-        int maxTasks = randomIntBetween(1, threadPool.info(ThreadPool.Names.GENERIC).getMax());
+        int maxTasks = randomIntBetween(1, maxThreads);
+        final int enqueued = maxTasks - 1; // So that it is possible to run at least one more task
+        final int newTasks = randomIntBetween(1, 10);
+
         CountDownLatch taskBlocker = new CountDownLatch(1);
-        AtomicInteger executed = new AtomicInteger();
+        CountDownLatch executedCountDown = new CountDownLatch(enqueued + newTasks);
         PrioritizedThrottledTaskRunner<TestTask> taskRunner = new PrioritizedThrottledTaskRunner<>("test", maxTasks, executor);
-        final int enqueued = maxTasks - 1; // So that it is possible to run at least one more task
         for (int i = 0; i < enqueued; i++) {
+            final int taskId = i;
             taskRunner.enqueueTask(new TestTask(() -> {
                 try {
                     taskBlocker.await();
                 } catch (InterruptedException e) {
                     throw new RuntimeException(e);
                 }
-                executed.incrementAndGet();
+                executedCountDown.countDown();
             }, getRandomPriority()));
             assertThat(taskRunner.runningTasks(), equalTo(i + 1));
         }
         // Enqueueing one or more new tasks would create only one new running task
-        final int newTasks = randomIntBetween(1, 10);
         for (int i = 0; i < newTasks; i++) {
+            final int taskId = i;
             taskRunner.enqueueTask(new TestTask(() -> {
                 try {
                     taskBlocker.await();
                 } catch (InterruptedException e) {
                     throw new RuntimeException(e);
                 }
-                executed.incrementAndGet();
+                executedCountDown.countDown();
             }, getRandomPriority()));
             assertThat(taskRunner.runningTasks(), equalTo(maxTasks));
         }
         assertThat(taskRunner.queueSize(), equalTo(newTasks - 1));
         taskBlocker.countDown();
         /// Eventually all tasks are executed
-        assertBusy(() -> {
-            assertThat(executed.get(), equalTo(enqueued + newTasks));
-            assertThat(taskRunner.runningTasks(), equalTo(0));
+        assertTrue(executedCountDown.await(10, TimeUnit.SECONDS));
+        assertThat(taskRunner.queueSize(), equalTo(0));
+        assertNoRunningTasks(taskRunner);
+    }
+
+    public void testFailsTasksOnRejectionOrShutdown() throws Exception {
+        final var executor = randomBoolean()
+            ? EsExecutors.newScaling("test", maxThreads, maxThreads, 0, TimeUnit.MILLISECONDS, true, threadFactory, threadContext)
+            : EsExecutors.newFixed("test", maxThreads, between(1, 5), threadFactory, threadContext, false);
+        final var taskRunner = new PrioritizedThrottledTaskRunner<TestTask>("test", between(1, maxThreads * 2), executor);
+        final var totalPermits = between(1, maxThreads * 2);
+        final var permits = new Semaphore(totalPermits);
+        final var taskCompleted = new CountDownLatch(between(1, maxThreads * 2));
+        final var rejectionCountDown = new CountDownLatch(between(1, maxThreads * 2));
+
+        final var spawnThread = new Thread(() -> {
+            try {
+                while (true) {
+                    assertTrue(permits.tryAcquire(10, TimeUnit.SECONDS));
+                    taskRunner.enqueueTask(new TestTask(taskCompleted::countDown, getRandomPriority()) {
+                        @Override
+                        public void onRejection(Exception e) {
+                            rejectionCountDown.countDown();
+                        }
+
+                        @Override
+                        public void onAfter() {
+                            permits.release();
+                        }
+                    });
+                }
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
         });
+        spawnThread.start();
+        assertTrue(taskCompleted.await(10, TimeUnit.SECONDS));
+        executor.shutdown();
+        assertTrue(executor.awaitTermination(30, TimeUnit.SECONDS));
+        assertTrue(rejectionCountDown.await(10, TimeUnit.SECONDS));
+        spawnThread.interrupt();
+        spawnThread.join();
+        assertThat(taskRunner.runningTasks(), equalTo(0));
         assertThat(taskRunner.queueSize(), equalTo(0));
+        assertTrue(permits.tryAcquire(totalPermits));
     }
 
     private int getRandomPriority() {
         return randomIntBetween(-1000, 1000);
     }
+
+    private void assertNoRunningTasks(PrioritizedThrottledTaskRunner<TestTask> taskRunner) {
+        final var barrier = new CyclicBarrier(maxThreads + 1);
+        for (int i = 0; i < maxThreads; i++) {
+            executor.execute(() -> CyclicBarrierUtils.await(barrier));
+        }
+        CyclicBarrierUtils.await(barrier);
+        assertThat(taskRunner.runningTasks(), equalTo(0));
+    }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/RunOnceTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/RunOnceTests.java
index d097fd601837..1fe436347a50 100644
--- a/server/src/test/java/org/elasticsearch/common/util/concurrent/RunOnceTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/RunOnceTests.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.common.util.concurrent;
 
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ReachabilityChecker;
 
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -88,4 +89,15 @@ public void onAfter() {
             assertTrue(runOnce.hasRun());
         }
     }
+
+    public void testReleasesDelegate() {
+        final var reachabilityChecker = new ReachabilityChecker();
+        final var runOnce = new RunOnce(reachabilityChecker.register(this::noop));
+        reachabilityChecker.checkReachable();
+        runOnce.run();
+        reachabilityChecker.ensureUnreachable();
+        assertEquals("RunOnce[null]", runOnce.toString());
+    }
+
+    private void noop() {}
 }
diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
index f17962ea7574..717b87799ea5 100644
--- a/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
@@ -33,6 +33,7 @@
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
 public class ThreadContextTests extends ESTestCase {
@@ -73,7 +74,8 @@ public void testNewContextWithClearedTransients() {
         // foo is the only existing transient header that is cleared
         try (
             ThreadContext.StoredContext stashed = threadContext.newStoredContext(
-                randomFrom(List.of("foo", "foo"), List.of("foo"), List.of("foo", "acme"))
+                randomFrom(List.of("foo", "foo"), List.of("foo"), List.of("foo", "acme")),
+                List.of()
             )
         ) {
             // only the requested transient header is cleared
@@ -113,7 +115,8 @@ public void testNewContextWithClearedTransients() {
         // test stashed missing header stays missing
         try (
             ThreadContext.StoredContext stashed = threadContext.newStoredContext(
-                randomFrom(Arrays.asList("acme", "acme"), Arrays.asList("acme"))
+                randomFrom(Arrays.asList("acme", "acme"), Arrays.asList("acme")),
+                List.of()
             )
         ) {
             assertNull(threadContext.getTransient("acme"));
@@ -122,6 +125,284 @@ public void testNewContextWithClearedTransients() {
         assertNull(threadContext.getTransient("acme"));
     }
 
+    public void testNewContextWithClearedRequestHeaders() {
+        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+
+        final Map<String, String> requestHeaders = Map.ofEntries(
+            Map.entry(randomAlphaOfLengthBetween(3, 8), randomAlphaOfLengthBetween(3, 8)),
+            Map.entry(Task.X_OPAQUE_ID_HTTP_HEADER, randomAlphaOfLength(10)),
+            Map.entry(Task.TRACE_ID, randomAlphaOfLength(20)),
+            Map.entry("_username", "elastic-admin")
+        );
+        threadContext.putHeader(requestHeaders);
+
+        final Map<String, Object> transientHeaders = Map.ofEntries(
+            Map.entry(randomAlphaOfLengthBetween(3, 8), randomAlphaOfLengthBetween(3, 8)),
+            Map.entry("_random", randomAlphaOfLengthBetween(3, 8)),
+            Map.entry("_map", Map.of("key", new Object())),
+            Map.entry("_address", "125.124.123.122"),
+            Map.entry("_object", new Object()),
+            Map.entry("_number", 42)
+        );
+        transientHeaders.forEach((k, v) -> threadContext.putTransient(k, v));
+
+        final Map<String, String> responseHeaders = Map.ofEntries(
+            Map.entry(randomAlphaOfLengthBetween(3, 6), randomAlphaOfLengthBetween(3, 8)),
+            Map.entry("_response_message", "All good."),
+            Map.entry("Warning", "Some warning!")
+        );
+        responseHeaders.forEach((k, v) -> threadContext.addResponseHeader(k, v));
+
+        // this is missing or null
+        if (randomBoolean()) {
+            threadContext.putHeader("_missing_or_null", null);
+        }
+
+        // mark as system context
+        boolean setSystemContext = randomBoolean();
+        if (setSystemContext) {
+            threadContext.markAsSystemContext();
+        }
+
+        // adding password header here to simplify assertions
+        threadContext.putHeader("_password", "elastic-password");
+
+        // password is the only request header that should be cleared
+        try (
+            ThreadContext.StoredContext stashed = threadContext.newStoredContext(
+                List.of(),
+                randomFrom(List.of("_password", "_password"), List.of("_password"), List.of("_password", "_missing_or_null"))
+            )
+        ) {
+            // only the requested header is cleared
+            assertThat(threadContext.getHeader("_password"), nullValue());
+            // system context boolean is preserved
+            assertThat(threadContext.isSystemContext(), equalTo(setSystemContext));
+            // missing header is still missing
+            assertThat(threadContext.getHeader("_missing_or_null"), nullValue());
+            // other headers are preserved
+            requestHeaders.forEach((k, v) -> assertThat(threadContext.getHeader(k), equalTo(v)));
+            transientHeaders.forEach((k, v) -> assertThat(threadContext.getTransient(k), equalTo(v)));
+            responseHeaders.forEach((k, v) -> assertThat(threadContext.getResponseHeaders().get(k).get(0), equalTo(v)));
+            // warning header count is still equal to 1
+            assertThat(threadContext.getResponseHeaders().get("Warning").size(), equalTo(1));
+
+            // try override stashed header
+            threadContext.putHeader("_password", "new-password");
+            assertThat(threadContext.getHeader("_password"), equalTo("new-password"));
+            // add new headers
+            threadContext.addResponseHeader("_new_response_header", randomAlphaOfLengthBetween(3, 8));
+            threadContext.putTransient("_new_transient_header", randomAlphaOfLengthBetween(3, 8));
+            threadContext.putHeader("_new_request_header", randomAlphaOfLengthBetween(3, 8));
+            threadContext.addResponseHeader("Warning", randomAlphaOfLengthBetween(3, 8));
+            // warning header is now equal to 2
+            assertThat(threadContext.getResponseHeaders().get("Warning").size(), equalTo(2));
+        }
+
+        // original "password" header is restored (it is not overridden)
+        assertThat(threadContext.getHeader("_password"), equalTo("elastic-password"));
+        // headers added inside the stash are NOT preserved
+        assertThat(threadContext.getResponseHeaders().get("_new_response_header"), nullValue());
+        assertThat(threadContext.getTransient("_new_transient_header"), nullValue());
+        assertThat(threadContext.getHeader("_new_request_header"), nullValue());
+        // warning header is restored to 1
+        assertThat(threadContext.getResponseHeaders().get("Warning").size(), equalTo(1));
+        assertThat(threadContext.getResponseHeaders().get("Warning").get(0), equalTo("Some warning!"));
+        // original headers are restored
+        requestHeaders.forEach((k, v) -> assertThat(threadContext.getHeader(k), equalTo(v)));
+        transientHeaders.forEach((k, v) -> assertThat(threadContext.getTransient(k), equalTo(v)));
+        responseHeaders.forEach((k, v) -> assertThat(threadContext.getResponseHeaders().get(k).get(0), equalTo(v)));
+        // system context boolean is unchanged
+        assertThat(threadContext.isSystemContext(), equalTo(setSystemContext));
+
+        // test stashed missing header stays missing
+        try (
+            ThreadContext.StoredContext stashed = threadContext.newStoredContext(
+                randomFrom(Arrays.asList("_missing_or_null", "_missing_or_null"), Arrays.asList("_missing_or_null")),
+                List.of()
+            )
+        ) {
+            assertThat(threadContext.getHeader("_missing_or_null"), nullValue());
+            threadContext.putHeader("_missing_or_null", "not_null");
+        }
+        assertThat(threadContext.getHeader("_missing_or_null"), nullValue());
+    }
+
+    public void testNewContextWithoutClearingTransientAndRequestHeaders() {
+        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+
+        final Map<String, String> requestHeaders = Map.ofEntries(
+            Map.entry(randomAlphaOfLengthBetween(3, 8), randomAlphaOfLengthBetween(3, 8)),
+            Map.entry(Task.X_OPAQUE_ID_HTTP_HEADER, randomAlphaOfLength(10)),
+            Map.entry(Task.TRACE_ID, randomAlphaOfLength(20)),
+            Map.entry("_username", "elastic-admin")
+        );
+        threadContext.putHeader(requestHeaders);
+
+        final Map<String, Object> transientHeaders = Map.ofEntries(
+            Map.entry(randomAlphaOfLengthBetween(3, 8), randomAlphaOfLengthBetween(3, 8)),
+            Map.entry("_random", randomAlphaOfLengthBetween(3, 8)),
+            Map.entry("_map", Map.of("key", new Object())),
+            Map.entry("_address", "125.124.123.122"),
+            Map.entry("_object", new Object()),
+            Map.entry("_number", 42)
+        );
+        transientHeaders.forEach((k, v) -> threadContext.putTransient(k, v));
+
+        final Map<String, String> responseHeaders = Map.ofEntries(
+            Map.entry(randomAlphaOfLengthBetween(3, 6), randomAlphaOfLengthBetween(3, 8)),
+            Map.entry("_response_message", "All good."),
+            Map.entry("Warning", "Some warning!")
+        );
+        responseHeaders.forEach((k, v) -> threadContext.addResponseHeader(k, v));
+
+        // mark as system context
+        boolean setSystemContext = randomBoolean();
+        if (setSystemContext) {
+            threadContext.markAsSystemContext();
+        }
+
+        // test nothing is cleared when empty collections are passed
+        try (ThreadContext.StoredContext stashed = threadContext.newStoredContext(List.of(), List.of())) {
+            // system context boolean is preserved
+            assertThat(threadContext.isSystemContext(), equalTo(setSystemContext));
+            // other headers are preserved
+            assertThat(threadContext.getHeaders().size(), equalTo(requestHeaders.size()));
+            assertThat(threadContext.getResponseHeaders().size(), equalTo(responseHeaders.size()));
+            assertThat(threadContext.getTransientHeaders().size(), equalTo(transientHeaders.size()));
+            requestHeaders.forEach((k, v) -> assertThat(threadContext.getHeader(k), equalTo(v)));
+            transientHeaders.forEach((k, v) -> assertThat(threadContext.getTransient(k), equalTo(v)));
+            responseHeaders.forEach((k, v) -> assertThat(threadContext.getResponseHeaders().get(k).get(0), equalTo(v)));
+            // warning header count is still equal to 1
+            assertThat(threadContext.getResponseHeaders().get("Warning").size(), equalTo(1));
+            // add new headers
+            threadContext.addResponseHeader("_new_response_header", randomAlphaOfLengthBetween(3, 8));
+            threadContext.putTransient("_new_transient_header", randomAlphaOfLengthBetween(3, 8));
+            threadContext.putHeader("_new_request_header", randomAlphaOfLengthBetween(3, 8));
+            threadContext.addResponseHeader("Warning", randomAlphaOfLengthBetween(3, 8));
+            // warning header is now equal to 2
+            assertThat(threadContext.getResponseHeaders().get("Warning").size(), equalTo(2));
+        }
+
+        // headers added inside the stash are NOT preserved
+        assertThat(threadContext.getResponseHeaders().get("_new_response_header"), nullValue());
+        assertThat(threadContext.getTransient("_new_transient_header"), nullValue());
+        assertThat(threadContext.getHeader("_new_request_header"), nullValue());
+        // original headers are unchanged
+        assertThat(threadContext.getHeaders().size(), equalTo(requestHeaders.size()));
+        assertThat(threadContext.getResponseHeaders().size(), equalTo(responseHeaders.size()));
+        assertThat(threadContext.getTransientHeaders().size(), equalTo(transientHeaders.size()));
+        requestHeaders.forEach((k, v) -> assertThat(threadContext.getHeader(k), equalTo(v)));
+        transientHeaders.forEach((k, v) -> assertThat(threadContext.getTransient(k), equalTo(v)));
+        responseHeaders.forEach((k, v) -> assertThat(threadContext.getResponseHeaders().get(k).get(0), equalTo(v)));
+        // system context boolean is unchanged
+        assertThat(threadContext.isSystemContext(), equalTo(setSystemContext));
+        // warning header is unchanged
+        assertThat(threadContext.getResponseHeaders().get("Warning").size(), equalTo(1));
+        assertThat(threadContext.getResponseHeaders().get("Warning").get(0), equalTo("Some warning!"));
+    }
+
+    public void testNewContextPreservingResponseHeadersWithClearedTransientAndRequestHeaders() {
+        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+
+        final Map<String, String> requestHeaders = Map.ofEntries(
+            Map.entry(randomAlphaOfLengthBetween(3, 8), randomAlphaOfLengthBetween(3, 8)),
+            Map.entry(Task.X_OPAQUE_ID_HTTP_HEADER, randomAlphaOfLength(10)),
+            Map.entry(Task.TRACE_ID, randomAlphaOfLength(20)),
+            Map.entry("_username", "elastic-admin")
+        );
+        threadContext.putHeader(requestHeaders);
+
+        final Map<String, Object> transientHeaders = Map.ofEntries(
+            Map.entry(randomAlphaOfLengthBetween(3, 8), randomAlphaOfLengthBetween(3, 8)),
+            Map.entry("_random", randomAlphaOfLengthBetween(3, 8)),
+            Map.entry("_map", Map.of("key", new Object())),
+            Map.entry("_address", "125.124.123.122"),
+            Map.entry("_object", new Object()),
+            Map.entry("_number", 42)
+        );
+        transientHeaders.forEach((k, v) -> threadContext.putTransient(k, v));
+
+        final Map<String, String> responseHeaders = Map.ofEntries(
+            Map.entry(randomAlphaOfLengthBetween(3, 6), randomAlphaOfLengthBetween(3, 8)),
+            Map.entry("_response_message", "All good."),
+            Map.entry("Warning", "Some warning!")
+        );
+        responseHeaders.forEach((k, v) -> threadContext.addResponseHeader(k, v));
+
+        // this is missing or null
+        if (randomBoolean()) {
+            threadContext.putHeader("_missing_or_null", null);
+            threadContext.putTransient("_missing_or_null", null);
+        }
+
+        // mark as system context
+        boolean setSystemContext = randomBoolean();
+        if (setSystemContext) {
+            threadContext.markAsSystemContext();
+        }
+
+        // adding request and transient headers to be cleared later
+        threadContext.putHeader("_password", "elastic-password");
+        threadContext.putTransient("_transient_to_be_cleared", "original-transient-value");
+
+        // password is the only request header that should be cleared
+        try (
+            ThreadContext.StoredContext stashed = threadContext.newStoredContextPreservingResponseHeaders(
+                randomFrom(
+                    List.of("_transient_to_be_cleared"),
+                    List.of("_transient_to_be_cleared", "_transient_to_be_cleared"),
+                    List.of("_transient_to_be_cleared", "_missing_or_null")
+                ),
+                randomFrom(List.of("_password", "_password"), List.of("_password"), List.of("_password", "_missing_or_null"))
+            )
+        ) {
+            // only the requested headers are cleared
+            assertThat(threadContext.getHeader("_password"), nullValue());
+            assertThat(threadContext.getTransient("_transient_to_be_cleared"), nullValue());
+            // system context boolean is preserved
+            assertThat(threadContext.isSystemContext(), equalTo(setSystemContext));
+            // missing header is still missing
+            assertThat(threadContext.getHeader("_missing_or_null"), nullValue());
+            assertThat(threadContext.getTransient("_missing_or_null"), nullValue());
+            // other headers are preserved
+            requestHeaders.forEach((k, v) -> assertThat(threadContext.getHeader(k), equalTo(v)));
+            transientHeaders.forEach((k, v) -> assertThat(threadContext.getTransient(k), equalTo(v)));
+            responseHeaders.forEach((k, v) -> assertThat(threadContext.getResponseHeaders().get(k).get(0), equalTo(v)));
+            // warning header count is still equal to 1
+            assertThat(threadContext.getResponseHeaders().get("Warning").size(), equalTo(1));
+
+            // try override stashed headers
+            threadContext.putHeader("_password", "new-password");
+            threadContext.putTransient("_transient_to_be_cleared", "new-transient-value");
+            assertThat(threadContext.getHeader("_password"), equalTo("new-password"));
+            assertThat(threadContext.getTransient("_transient_to_be_cleared"), equalTo("new-transient-value"));
+            // add new headers
+            threadContext.addResponseHeader("_new_response_header", "value-which-should-be-preserved");
+            threadContext.putTransient("_new_transient_header", randomAlphaOfLengthBetween(3, 8));
+            threadContext.putHeader("_new_request_header", randomAlphaOfLengthBetween(3, 8));
+            threadContext.addResponseHeader("Warning", "Another warning!");
+            // warning header is now equal to 2
+            assertThat(threadContext.getResponseHeaders().get("Warning").size(), equalTo(2));
+        }
+
+        // originally cleared headers should be restored (and not overridden)
+        assertThat(threadContext.getHeader("_password"), equalTo("elastic-password"));
+        assertThat(threadContext.getTransient("_transient_to_be_cleared"), equalTo("original-transient-value"));
+        requestHeaders.forEach((k, v) -> assertThat(threadContext.getHeader(k), equalTo(v)));
+        transientHeaders.forEach((k, v) -> assertThat(threadContext.getTransient(k), equalTo(v)));
+        // headers added inside the stash are NOT preserved
+        assertThat(threadContext.getTransient("_new_transient_header"), nullValue());
+        assertThat(threadContext.getHeader("_new_request_header"), nullValue());
+        // except for response headers which should be preserved
+        assertThat(threadContext.getResponseHeaders().get("_new_response_header").get(0), equalTo("value-which-should-be-preserved"));
+        assertThat(threadContext.getResponseHeaders().get("Warning").size(), equalTo(2));
+        assertThat(threadContext.getResponseHeaders().get("Warning").get(0), equalTo("Some warning!"));
+        assertThat(threadContext.getResponseHeaders().get("Warning").get(1), equalTo("Another warning!"));
+        // system context boolean is unchanged
+        assertThat(threadContext.isSystemContext(), equalTo(setSystemContext));
+    }
+
     public void testStashWithOrigin() {
         final String origin = randomAlphaOfLengthBetween(4, 16);
         final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
diff --git a/server/src/test/java/org/elasticsearch/common/util/set/SetsTests.java b/server/src/test/java/org/elasticsearch/common/util/set/SetsTests.java
index aad4eaa604b8..8112864a9926 100644
--- a/server/src/test/java/org/elasticsearch/common/util/set/SetsTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/set/SetsTests.java
@@ -25,7 +25,9 @@
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.hamcrest.Matchers.sameInstance;
 
 public class SetsTests extends ESTestCase {
 
@@ -77,6 +79,14 @@ public void testIntersection() {
             .filter(i -> (sets.v1().contains(i) && sets.v2().contains(i)))
             .collect(Collectors.toSet());
         assertThat(intersection, containsInAnyOrder(expectedIntersection.toArray(new Integer[0])));
+
+        final Set<Integer> emptyIntersection = Sets.intersection(
+            Sets.difference(sets.v1(), intersection),
+            Sets.difference(sets.v2(), intersection)
+        );
+        assertThat(emptyIntersection.isEmpty(), is(true));
+        // as an implementation detail, it's not just *some* empty set, but precisely *this* empty set
+        assertThat(emptyIntersection, sameInstance(Set.of()));
     }
 
     public void testNewHashSetWithExpectedSize() {
diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java b/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java
index 6623276c84eb..97ef5a356830 100644
--- a/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java
+++ b/server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.NamedObjectNotFoundException;
@@ -954,7 +953,7 @@ public void testEnsureNoSelfReferences() throws IOException {
 
     /**
      * Test that the same map written multiple times do not trigger the self-reference check in
-     * {@link CollectionUtils#ensureNoSelfReferences(Object, String)} (Object)}
+     * {@link XContentBuilder#ensureNoSelfReferences(Object)}
      */
     public void testRepeatedMapsAndNoSelfReferences() throws Exception {
         Map<String, Object> mapB = singletonMap("b", "B");
diff --git a/server/src/test/java/org/elasticsearch/discovery/FileBasedSeedHostsProviderTests.java b/server/src/test/java/org/elasticsearch/discovery/FileBasedSeedHostsProviderTests.java
index 8614ca64822d..ebd43ee5c85b 100644
--- a/server/src/test/java/org/elasticsearch/discovery/FileBasedSeedHostsProviderTests.java
+++ b/server/src/test/java/org/elasticsearch/discovery/FileBasedSeedHostsProviderTests.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.discovery;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
@@ -70,7 +70,7 @@ public void tearDown() throws Exception {
     private void createTransportSvc() {
         final TcpTransport transport = new Netty4Transport(
             Settings.EMPTY,
-            Version.CURRENT,
+            TransportVersion.CURRENT,
             threadPool,
             new NetworkService(Collections.emptyList()),
             PageCacheRecycler.NON_RECYCLING_INSTANCE,
diff --git a/server/src/test/java/org/elasticsearch/discovery/HandshakingTransportAddressConnectorTests.java b/server/src/test/java/org/elasticsearch/discovery/HandshakingTransportAddressConnectorTests.java
index 10e1a6308eab..5b0051a39b8b 100644
--- a/server/src/test/java/org/elasticsearch/discovery/HandshakingTransportAddressConnectorTests.java
+++ b/server/src/test/java/org/elasticsearch/discovery/HandshakingTransportAddressConnectorTests.java
@@ -9,8 +9,6 @@
 package org.elasticsearch.discovery;
 
 import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.Build;
 import org.elasticsearch.ElasticsearchException;
@@ -18,7 +16,6 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.core.Nullable;
@@ -161,7 +158,6 @@ public void testLogsFullConnectionFailureAfterSuccessfulHandshake() throws Excep
         FailureListener failureListener = new FailureListener();
 
         MockLogAppender mockAppender = new MockLogAppender();
-        mockAppender.start();
         mockAppender.addExpectation(
             new MockLogAppender.SeenEventExpectation(
                 "message",
@@ -176,16 +172,10 @@ public void testLogsFullConnectionFailureAfterSuccessfulHandshake() throws Excep
                     + "] failed"
             )
         );
-        Logger targetLogger = LogManager.getLogger(HandshakingTransportAddressConnector.class);
-        Loggers.addAppender(targetLogger, mockAppender);
-
-        try {
+        try (var ignored = mockAppender.capturing(HandshakingTransportAddressConnector.class)) {
             handshakingTransportAddressConnector.connectToRemoteMasterNode(discoveryAddress, failureListener);
             assertThat(failureListener.getFailureMessage(), containsString("simulated"));
             mockAppender.assertAllExpectationsMatched();
-        } finally {
-            Loggers.removeAppender(targetLogger, mockAppender);
-            mockAppender.stop();
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/discovery/PeerFinderTests.java b/server/src/test/java/org/elasticsearch/discovery/PeerFinderTests.java
index 7e32209a6562..15652ae6c2d5 100644
--- a/server/src/test/java/org/elasticsearch/discovery/PeerFinderTests.java
+++ b/server/src/test/java/org/elasticsearch/discovery/PeerFinderTests.java
@@ -9,7 +9,6 @@
 package org.elasticsearch.discovery;
 
 import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.core.LogEvent;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
@@ -19,7 +18,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.node.DiscoveryNodes.Builder;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
@@ -789,10 +787,7 @@ public void testLogsWarningsIfActiveForLongEnough() throws IllegalAccessExceptio
             .millis();
 
         MockLogAppender appender = new MockLogAppender();
-        try {
-            appender.start();
-            Loggers.addAppender(LogManager.getLogger("org.elasticsearch.discovery.PeerFinder"), appender);
-
+        try (var ignored = appender.capturing(PeerFinder.class)) {
             appender.addExpectation(
                 new MockLogAppender.SeenEventExpectation(
                     "discovery result",
@@ -811,10 +806,6 @@ public boolean innerMatch(LogEvent event) {
                 runAllRunnableTasks();
             }
             appender.assertAllExpectationsMatched();
-
-        } finally {
-            Loggers.removeAppender(LogManager.getLogger("org.elasticsearch.discovery.PeerFinder"), appender);
-            appender.stop();
         }
     }
 
@@ -830,9 +821,7 @@ public void testLogsStackTraceInDiscoveryResultMessages() throws IllegalAccessEx
             .millis();
 
         MockLogAppender appender = new MockLogAppender();
-        try {
-            appender.start();
-            Loggers.addAppender(LogManager.getLogger("org.elasticsearch.discovery.PeerFinder"), appender);
+        try (var ignored = appender.capturing(PeerFinder.class)) {
             appender.addExpectation(
                 new MockLogAppender.SeenEventExpectation(
                     "discovery result",
@@ -869,10 +858,6 @@ public boolean innerMatch(LogEvent event) {
                 runAllRunnableTasks();
             }
             appender.assertAllExpectationsMatched();
-
-        } finally {
-            Loggers.removeAppender(LogManager.getLogger("org.elasticsearch.discovery.PeerFinder"), appender);
-            appender.stop();
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/discovery/SeedHostsResolverTests.java b/server/src/test/java/org/elasticsearch/discovery/SeedHostsResolverTests.java
index e7480c0ae7cc..8e652bba1fc1 100644
--- a/server/src/test/java/org/elasticsearch/discovery/SeedHostsResolverTests.java
+++ b/server/src/test/java/org/elasticsearch/discovery/SeedHostsResolverTests.java
@@ -9,12 +9,9 @@
 package org.elasticsearch.discovery;
 
 import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
@@ -139,7 +136,7 @@ public void testRemovingLocalAddresses() {
         final InetAddress loopbackAddress = InetAddress.getLoopbackAddress();
         final Transport transport = new Netty4Transport(
             Settings.EMPTY,
-            Version.CURRENT,
+            TransportVersion.CURRENT,
             threadPool,
             networkService,
             PageCacheRecycler.NON_RECYCLING_INSTANCE,
@@ -179,13 +176,13 @@ public BoundTransportAddress boundAddress() {
         assertThat(ports, equalTo(IntStream.range(9303, 9310).boxed().collect(Collectors.toSet())));
     }
 
-    public void testUnknownHost() throws IllegalAccessException {
+    public void testUnknownHost() {
         final NetworkService networkService = new NetworkService(Collections.emptyList());
         final String hostname = randomAlphaOfLength(8);
         final UnknownHostException unknownHostException = new UnknownHostException(hostname);
         final Transport transport = new Netty4Transport(
             Settings.EMPTY,
-            Version.CURRENT,
+            TransportVersion.CURRENT,
             threadPool,
             networkService,
             PageCacheRecycler.NON_RECYCLING_INSTANCE,
@@ -222,13 +219,11 @@ public TransportAddress[] addressesFromString(String address) throws UnknownHost
         closeables.push(transportService);
         recreateSeedHostsResolver(transportService);
 
-        final Logger logger = LogManager.getLogger(SeedHostsResolver.class);
         final MockLogAppender appender = new MockLogAppender();
-        appender.start();
         appender.addExpectation(
             new MockLogAppender.ExceptionSeenEventExpectation(
                 getTestName(),
-                logger.getName(),
+                SeedHostsResolver.class.getCanonicalName(),
                 Level.WARN,
                 "failed to resolve host [" + hostname + "]",
                 UnknownHostException.class,
@@ -236,24 +231,18 @@ public TransportAddress[] addressesFromString(String address) throws UnknownHost
             )
         );
 
-        try {
-            Loggers.addAppender(logger, appender);
-            final List<TransportAddress> transportAddresses = seedHostsResolver.resolveHosts(Collections.singletonList(hostname));
-
-            assertThat(transportAddresses, empty());
+        try (var ignored = appender.capturing(SeedHostsResolver.class)) {
+            assertThat(seedHostsResolver.resolveHosts(Collections.singletonList(hostname)), empty());
             appender.assertAllExpectationsMatched();
-        } finally {
-            Loggers.removeAppender(logger, appender);
-            appender.stop();
         }
     }
 
-    public void testResolveTimeout() throws IllegalAccessException {
+    public void testResolveTimeout() {
         final NetworkService networkService = new NetworkService(Collections.emptyList());
         final CountDownLatch latch = new CountDownLatch(1);
         final Transport transport = new Netty4Transport(
             Settings.EMPTY,
-            Version.CURRENT,
+            TransportVersion.CURRENT,
             threadPool,
             networkService,
             PageCacheRecycler.NON_RECYCLING_INSTANCE,
@@ -301,13 +290,11 @@ public TransportAddress[] addressesFromString(String address) throws UnknownHost
         closeables.push(transportService);
         recreateSeedHostsResolver(transportService);
 
-        final Logger logger = LogManager.getLogger(SeedHostsResolver.class);
         final MockLogAppender appender = new MockLogAppender();
-        appender.start();
         appender.addExpectation(
             new MockLogAppender.SeenEventExpectation(
                 getTestName(),
-                logger.getName(),
+                SeedHostsResolver.class.getCanonicalName(),
                 Level.WARN,
                 "timed out after [*] ([discovery.seed_resolver.timeout]=["
                     + SeedHostsResolver.getResolveTimeout(Settings.EMPTY)
@@ -315,15 +302,10 @@ public TransportAddress[] addressesFromString(String address) throws UnknownHost
             )
         );
 
-        try {
-            Loggers.addAppender(logger, appender);
-            final List<TransportAddress> transportAddresses = seedHostsResolver.resolveHosts(Arrays.asList("hostname1", "hostname2"));
-
-            assertThat(transportAddresses, hasSize(1));
+        try (var ignored = appender.capturing(SeedHostsResolver.class)) {
+            assertThat(seedHostsResolver.resolveHosts(Arrays.asList("hostname1", "hostname2")), hasSize(1));
             appender.assertAllExpectationsMatched();
         } finally {
-            Loggers.removeAppender(logger, appender);
-            appender.stop();
             latch.countDown();
         }
     }
@@ -334,7 +316,7 @@ public void testCancellationOnClose() throws InterruptedException {
         final CountDownLatch conditionLatch = new CountDownLatch(1);
         final Transport transport = new Netty4Transport(
             Settings.EMPTY,
-            Version.CURRENT,
+            TransportVersion.CURRENT,
             threadPool,
             networkService,
             PageCacheRecycler.NON_RECYCLING_INSTANCE,
@@ -394,10 +376,10 @@ public TransportAddress[] addressesFromString(String address) throws UnknownHost
         assertThat(FutureUtils.get(fut, 10, TimeUnit.SECONDS), hasSize(0));
     }
 
-    public void testInvalidHosts() throws IllegalAccessException {
+    public void testInvalidHosts() {
         final Transport transport = new Netty4Transport(
             Settings.EMPTY,
-            Version.CURRENT,
+            TransportVersion.CURRENT,
             threadPool,
             new NetworkService(Collections.emptyList()),
             PageCacheRecycler.NON_RECYCLING_INSTANCE,
@@ -427,20 +409,17 @@ public BoundTransportAddress boundAddress() {
         closeables.push(transportService);
         recreateSeedHostsResolver(transportService);
 
-        final Logger logger = LogManager.getLogger(SeedHostsResolver.class);
         final MockLogAppender appender = new MockLogAppender();
-        appender.start();
         appender.addExpectation(
             new MockLogAppender.SeenEventExpectation(
                 getTestName(),
-                logger.getName(),
+                SeedHostsResolver.class.getCanonicalName(),
                 Level.WARN,
                 "failed to resolve host [127.0.0.1:9300:9300]"
             )
         );
 
-        try {
-            Loggers.addAppender(logger, appender);
+        try (var ignored = appender.capturing(SeedHostsResolver.class)) {
             final List<TransportAddress> transportAddresses = seedHostsResolver.resolveHosts(
                 Arrays.asList("127.0.0.1:9300:9300", "127.0.0.1:9301")
             );
@@ -448,9 +427,6 @@ public BoundTransportAddress boundAddress() {
             assertThat(transportAddresses.get(0).getAddress(), equalTo("127.0.0.1"));
             assertThat(transportAddresses.get(0).getPort(), equalTo(9301));
             appender.assertAllExpectationsMatched();
-        } finally {
-            Loggers.removeAppender(logger, appender);
-            appender.stop();
         }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
index 33f7f8a3d24d..6451c76e99e9 100644
--- a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
+++ b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
@@ -7,6 +7,7 @@
  */
 package org.elasticsearch.env;
 
+import org.apache.logging.log4j.Level;
 import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
@@ -34,7 +35,9 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.test.MockLogAppender;
 import org.elasticsearch.test.NodeRoles;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -114,44 +117,63 @@ public void testSegmentInfosTracing() {
         }
     }
 
+    // using a literal string here because the logger is mentioned in the docs, and therefore must only be changed with care
+    private static final String NODE_ENVIRONMENT_LOGGER_NAME = "org.elasticsearch.env.NodeEnvironment";
+
+    @TestLogging(reason = "test includes assertions about DEBUG logging", value = NODE_ENVIRONMENT_LOGGER_NAME + ":DEBUG")
     public void testShardLock() throws Exception {
-        final NodeEnvironment env = newNodeEnvironment();
+        try (var env = newNodeEnvironment()) {
 
-        Index index = new Index("foo", "fooUUID");
-        ShardLock fooLock = env.shardLock(new ShardId(index, 0), "1");
-        assertEquals(new ShardId(index, 0), fooLock.getShardId());
+            Index index = new Index("foo", "fooUUID");
 
-        try {
-            env.shardLock(new ShardId(index, 0), "2");
-            fail("shard is locked");
-        } catch (ShardLockObtainFailedException ex) {
-            // expected
-        }
-        for (Path path : env.indexPaths(index)) {
-            Files.createDirectories(path.resolve("0"));
-            Files.createDirectories(path.resolve("1"));
-        }
-        try {
-            env.lockAllForIndex(index, idxSettings, "3", randomIntBetween(0, 10));
-            fail("shard 0 is locked");
-        } catch (ShardLockObtainFailedException ex) {
-            // expected
-        }
+            var appender = new MockLogAppender();
+            appender.addExpectation(
+                new MockLogAppender.SeenEventExpectation(
+                    "hot threads logging",
+                    NODE_ENVIRONMENT_LOGGER_NAME,
+                    Level.DEBUG,
+                    "hot threads while failing to obtain shard lock for [foo][0]: obtaining shard lock for [2] timed out after *"
+                )
+            );
+            appender.addExpectation(
+                new MockLogAppender.UnseenEventExpectation(
+                    "second attempt should be suppressed due to throttling",
+                    NODE_ENVIRONMENT_LOGGER_NAME,
+                    Level.DEBUG,
+                    "hot threads while failing to obtain shard lock for [foo][0]: obtaining shard lock for [3] timed out after *"
+                )
+            );
 
-        fooLock.close();
-        // can lock again?
-        env.shardLock(new ShardId(index, 0), "4").close();
+            try (var ignored = appender.capturing(NodeEnvironment.class); var lock = env.shardLock(new ShardId(index, 0), "1")) {
+                assertEquals(new ShardId(index, 0), lock.getShardId());
 
-        List<ShardLock> locks = env.lockAllForIndex(index, idxSettings, "5", randomIntBetween(0, 10));
-        try {
-            env.shardLock(new ShardId(index, 0), "6");
-            fail("shard is locked");
-        } catch (ShardLockObtainFailedException ex) {
-            // expected
+                expectThrows(ShardLockObtainFailedException.class, () -> env.shardLock(new ShardId(index, 0), "2"));
+
+                for (Path path : env.indexPaths(index)) {
+                    Files.createDirectories(path.resolve("0"));
+                    Files.createDirectories(path.resolve("1"));
+                }
+                expectThrows(
+                    ShardLockObtainFailedException.class,
+                    () -> env.lockAllForIndex(index, idxSettings, "3", randomIntBetween(0, 10))
+                );
+
+                appender.assertAllExpectationsMatched();
+            }
+
+            // can lock again?
+            env.shardLock(new ShardId(index, 0), "4").close();
+
+            List<ShardLock> locks = new ArrayList<>();
+            try {
+                locks.addAll(env.lockAllForIndex(index, idxSettings, "5", randomIntBetween(0, 10)));
+                expectThrows(ShardLockObtainFailedException.class, () -> env.shardLock(new ShardId(index, 0), "6"));
+            } finally {
+                IOUtils.close(locks);
+            }
+
+            assertTrue("LockedShards: " + env.lockedShards(), env.lockedShards().isEmpty());
         }
-        IOUtils.close(locks);
-        assertTrue("LockedShards: " + env.lockedShards(), env.lockedShards().isEmpty());
-        env.close();
     }
 
     public void testAvailableIndexFolders() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java b/server/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
index 1eaf80d9d48c..9d3a1ac68e5b 100644
--- a/server/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
@@ -9,8 +9,10 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.nodes.BaseNodeResponse;
+import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -372,13 +374,23 @@ static class Entry {
 
         private final ThreadPool threadPool;
         private final Map<String, Entry> simulations = new ConcurrentHashMap<>();
-        private AtomicInteger reroute = new AtomicInteger();
+        private final AtomicInteger reroute = new AtomicInteger();
 
         TestFetch(ThreadPool threadPool) {
-            super(LogManager.getLogger(TestFetch.class), "test", new ShardId("test", "_na_", 1), "", null);
+            super(LogManager.getLogger(TestFetch.class), "test", new ShardId("test", "_na_", 1), "");
             this.threadPool = threadPool;
         }
 
+        @Override
+        protected void list(
+            ShardId shardId,
+            String customDataPath,
+            DiscoveryNode[] nodes,
+            ActionListener<BaseNodesResponse<Response>> listener
+        ) {
+            fail("should not call list()");
+        }
+
         public void addSimulation(String nodeId, Response response) {
             simulations.put(nodeId, new Entry(response, null));
         }
diff --git a/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java b/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java
index ad915f45f721..a57833d2c66a 100644
--- a/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.coordination.CoordinationMetadata;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -204,7 +205,7 @@ public void testUpdateRoutingTable() {
         assertFalse(initialState.routingTable().hasIndex(index));
 
         {
-            final ClusterState newState = updateRoutingTable(initialState);
+            final ClusterState newState = updateRoutingTable(initialState, TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
             assertTrue(newState.routingTable().hasIndex(index));
             assertThat(newState.routingTable().version(), is(0L));
             assertThat(newState.routingTable().allShards(index.getName()).size(), is(numOfShards));
@@ -217,7 +218,8 @@ public void testUpdateRoutingTable() {
                             .put(IndexMetadata.builder(initialState.metadata().index("test")).state(IndexMetadata.State.CLOSE))
                             .build()
                     )
-                    .build()
+                    .build(),
+                TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
             );
             assertFalse(newState.routingTable().hasIndex(index));
         }
@@ -238,7 +240,8 @@ public void testUpdateRoutingTable() {
                             )
                             .build()
                     )
-                    .build()
+                    .build(),
+                TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
             );
             assertTrue(newState.routingTable().hasIndex(index));
             assertThat(newState.routingTable().version(), is(0L));
diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
index 97835d3bf1d3..4a138be17da4 100644
--- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.gateway;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexMetadataVerifier;
@@ -167,8 +168,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT;
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
index 8d6be3708671..fe37b198531d 100644
--- a/server/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -39,7 +40,13 @@ private GatewayService createService(final Settings.Builder settings) {
             null,
             (TaskManager) null
         );
-        return new GatewayService(settings.build(), (reason, priority, listener) -> fail("should not reroute"), clusterService, null);
+        return new GatewayService(
+            settings.build(),
+            (reason, priority, listener) -> fail("should not reroute"),
+            clusterService,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
+            null
+        );
     }
 
     public void testDefaultRecoverAfterTime() {
diff --git a/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java
index f09257922aee..855e51e689d7 100644
--- a/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java
@@ -8,8 +8,6 @@
 package org.elasticsearch.gateway;
 
 import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
@@ -48,7 +46,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -1524,7 +1521,6 @@ public void testDebugLogging() throws IOException, IllegalAccessException {
             }
 
             MockLogAppender mockAppender = new MockLogAppender();
-            mockAppender.start();
             mockAppender.addExpectation(
                 new MockLogAppender.SeenEventExpectation(
                     "should see checkindex message",
@@ -1558,16 +1554,10 @@ public void testDebugLogging() throws IOException, IllegalAccessException {
                 )
             );
 
-            Logger classLogger = LogManager.getLogger(PersistedClusterStateService.class);
-            Loggers.addAppender(classLogger, mockAppender);
-
-            try {
+            try (var ignored = mockAppender.capturing(PersistedClusterStateService.class)) {
                 persistedClusterStateService.loadBestOnDiskState();
-            } finally {
-                Loggers.removeAppender(classLogger, mockAppender);
-                mockAppender.stop();
+                mockAppender.assertAllExpectationsMatched();
             }
-            mockAppender.assertAllExpectationsMatched();
         }
     }
 
@@ -1852,24 +1842,17 @@ private void assertExpectedLogs(
         ClusterState clusterState,
         PersistedClusterStateService.Writer writer,
         MockLogAppender.LoggingExpectation expectation
-    ) throws IllegalAccessException, IOException {
+    ) throws IOException {
         MockLogAppender mockAppender = new MockLogAppender();
-        mockAppender.start();
         mockAppender.addExpectation(expectation);
-        Logger classLogger = LogManager.getLogger(PersistedClusterStateService.class);
-        Loggers.addAppender(classLogger, mockAppender);
-
-        try {
+        try (var ignored = mockAppender.capturing(PersistedClusterStateService.class)) {
             if (previousState == null) {
                 writer.writeFullStateAndCommit(currentTerm, clusterState);
             } else {
                 writer.writeIncrementalStateAndCommit(currentTerm, previousState, clusterState);
             }
-        } finally {
-            Loggers.removeAppender(classLogger, mockAppender);
-            mockAppender.stop();
+            mockAppender.assertAllExpectationsMatched();
         }
-        mockAppender.assertAllExpectationsMatched();
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java b/server/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
index ffee1c72e979..8f474adec6a4 100644
--- a/server/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.health.ClusterStateHealth;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -469,7 +470,7 @@ private RoutingAllocation getRestoreRoutingAllocation(AllocationDeciders allocat
             .build();
 
         final Snapshot snapshot = new Snapshot("test", new SnapshotId("test", UUIDs.randomBase64UUID()));
-        RoutingTable routingTable = RoutingTable.builder()
+        RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsRestore(
                 metadata.index(shardId.getIndex()),
                 new SnapshotRecoverySource(
@@ -507,7 +508,7 @@ private RoutingAllocation routingAllocationWithOnePrimaryNoReplicas(
                     .putInSyncAllocationIds(shardId.id(), Sets.newHashSet(activeAllocationIds))
             )
             .build();
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         switch (reason) {
             case INDEX_CREATED -> routingTableBuilder.addAsNew(metadata.index(shardId.getIndex()));
             case CLUSTER_RECOVERED -> routingTableBuilder.addAsRecovery(metadata.index(shardId.getIndex()));
diff --git a/server/src/test/java/org/elasticsearch/gateway/PriorityComparatorTests.java b/server/src/test/java/org/elasticsearch/gateway/PriorityComparatorTests.java
index f4c65d9cf576..6495a730bc37 100644
--- a/server/src/test/java/org/elasticsearch/gateway/PriorityComparatorTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/PriorityComparatorTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
 
@@ -198,7 +199,7 @@ public void testPriorityComparatorSort() {
             }
             // else sometimes just use the defaults
 
-            indices[i] = IndexMetadata.builder(formatted("idx_%04d", i))
+            indices[i] = IndexMetadata.builder(Strings.format("idx_%04d", i))
                 .system(isSystem)
                 .settings(buildSettings(creationDate, priority))
                 .build();
diff --git a/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java b/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
index be67e5574ffd..12ea5490028e 100644
--- a/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
@@ -33,7 +33,6 @@
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider;
-import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.TimeValue;
@@ -60,6 +59,7 @@
 
 import static java.util.Collections.unmodifiableMap;
 import static org.elasticsearch.cluster.routing.RoutingNodesHelper.shardsWithState;
+import static org.elasticsearch.common.settings.ClusterSettings.createBuiltInClusterSettings;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
@@ -369,10 +369,7 @@ public void testThrottleWhenAllocatingToMatchingNode() {
             new AllocationDeciders(
                 Arrays.asList(
                     new TestAllocateDecision(Decision.YES),
-                    new SameShardAllocationDecider(
-                        Settings.EMPTY,
-                        new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
-                    ),
+                    new SameShardAllocationDecider(createBuiltInClusterSettings()),
                     new AllocationDecider() {
                         @Override
                         public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
@@ -563,7 +560,8 @@ private RoutingAllocation onePrimaryOnNode1And1Replica(AllocationDeciders decide
                                         UnassignedInfo.AllocationStatus.NO_ATTEMPT,
                                         Collections.emptySet(),
                                         lastAllocatedNodeId
-                                    )
+                                    ),
+                                    ShardRouting.Role.DEFAULT
                                 )
                             )
                     )
diff --git a/server/src/test/java/org/elasticsearch/health/GetHealthRequestTests.java b/server/src/test/java/org/elasticsearch/health/GetHealthRequestTests.java
new file mode 100644
index 000000000000..da0717967352
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/health/GetHealthRequestTests.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.health;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class GetHealthRequestTests extends ESTestCase {
+
+    public void testValidation() {
+        var req = new GetHealthAction.Request(true, -1);
+        ActionRequestValidationException validationException = req.validate();
+        assertThat(validationException, notNullValue());
+        assertThat(validationException.validationErrors().size(), is(1));
+        assertThat(validationException.validationErrors().get(0), containsString("The size parameter must be a positive integer"));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/health/GetHealthResponseTests.java b/server/src/test/java/org/elasticsearch/health/GetHealthResponseTests.java
index 454383d3ecf3..0c16f22bc765 100644
--- a/server/src/test/java/org/elasticsearch/health/GetHealthResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/health/GetHealthResponseTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.health.GetHealthAction.Response;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 
@@ -24,6 +23,7 @@
 import java.util.Locale;
 import java.util.Map;
 
+import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS;
 import static org.hamcrest.Matchers.is;
 
 public class GetHealthResponseTests extends ESTestCase {
@@ -35,9 +35,9 @@ public void testToXContent() throws IOException {
         Response response = new Response(ClusterName.DEFAULT, indicatorResults, true);
 
         XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
-        response.toXContentChunked().forEachRemaining(xcontent -> {
+        response.toXContentChunked(EMPTY_PARAMS).forEachRemaining(xcontent -> {
             try {
-                xcontent.toXContent(builder, ToXContent.EMPTY_PARAMS);
+                xcontent.toXContent(builder, EMPTY_PARAMS);
             } catch (IOException e) {
                 logger.error(e.getMessage(), e);
                 fail(e.getMessage());
diff --git a/server/src/test/java/org/elasticsearch/health/HealthIndicatorResultTests.java b/server/src/test/java/org/elasticsearch/health/HealthIndicatorResultTests.java
index df0cf83cd8bb..058995c52607 100644
--- a/server/src/test/java/org/elasticsearch/health/HealthIndicatorResultTests.java
+++ b/server/src/test/java/org/elasticsearch/health/HealthIndicatorResultTests.java
@@ -10,6 +10,7 @@
 
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -70,7 +71,7 @@ public void testToXContent() throws Exception {
         HealthIndicatorResult result = new HealthIndicatorResult(name, status, symptom, details, impacts, diagnosisList);
         XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
 
-        result.toXContentChunked().forEachRemaining(xcontent -> {
+        result.toXContentChunked(ToXContent.EMPTY_PARAMS).forEachRemaining(xcontent -> {
             try {
                 xcontent.toXContent(builder, ToXContent.EMPTY_PARAMS);
             } catch (IOException e) {
@@ -106,7 +107,7 @@ public void testToXContent() throws Exception {
 
             if (diagnosis1.affectedResources() != null) {
                 XContentBuilder diagnosisXContent = XContentFactory.jsonBuilder().prettyPrint();
-                diagnosis1.toXContentChunked().forEachRemaining(xcontent -> {
+                diagnosis1.toXContentChunked(ToXContent.EMPTY_PARAMS).forEachRemaining(xcontent -> {
                     try {
                         xcontent.toXContent(diagnosisXContent, ToXContent.EMPTY_PARAMS);
                     } catch (IOException e) {
@@ -131,7 +132,7 @@ public void testToXContent() throws Exception {
             expectedDiagnosis2.put("help_url", diagnosis2.definition().helpURL());
             if (diagnosis2.affectedResources() != null) {
                 XContentBuilder diagnosisXContent = XContentFactory.jsonBuilder().prettyPrint();
-                diagnosis2.toXContentChunked().forEachRemaining(xcontent -> {
+                diagnosis2.toXContentChunked(ToXContent.EMPTY_PARAMS).forEachRemaining(xcontent -> {
                     try {
                         xcontent.toXContent(diagnosisXContent, ToXContent.EMPTY_PARAMS);
                     } catch (IOException e) {
@@ -198,13 +199,6 @@ public void testChunkCount() {
 
         // -> each Diagnosis yields 5 chunks => 10 chunks from both diagnosis
         // -> HealthIndicatorResult surrounds the diagnosis list by 2 chunks
-        int chunksExpected = 12;
-        var iterator = result.toXContentChunked();
-        int chunksSeen = 0;
-        while (iterator.hasNext()) {
-            iterator.next();
-            chunksSeen++;
-        }
-        assertEquals(chunksExpected, chunksSeen);
+        AbstractChunkedSerializingTestCase.assertChunkCount(result, ignored -> 12);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/health/HealthIndicatorServiceTests.java b/server/src/test/java/org/elasticsearch/health/HealthIndicatorServiceTests.java
index 1d6b4505ac07..557e6fee722d 100644
--- a/server/src/test/java/org/elasticsearch/health/HealthIndicatorServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/health/HealthIndicatorServiceTests.java
@@ -90,7 +90,7 @@ public String name() {
             }
 
             @Override
-            public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
+            public HealthIndicatorResult calculate(boolean verbose, int maxAffectedResourcesCount, HealthInfo healthInfo) {
                 return null;
             }
         };
diff --git a/server/src/test/java/org/elasticsearch/health/HealthServiceTests.java b/server/src/test/java/org/elasticsearch/health/HealthServiceTests.java
index 14c69ed90d37..bc1d30976bf3 100644
--- a/server/src/test/java/org/elasticsearch/health/HealthServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/health/HealthServiceTests.java
@@ -88,6 +88,7 @@ private void assertExpectedHealthIndicatorResults(
             client,
             indicatorName,
             false,
+            1000,
             getExpectedHealthIndicatorResultsActionListener(onResponseCalled, expectedHealthIndicatorResults)
         );
         assertBusy(() -> assertThat(onResponseCalled.get(), equalTo(true)));
@@ -138,11 +139,24 @@ public void testMissingIndicator() throws Exception {
         );
     }
 
+    @SuppressWarnings("unchecked")
+    public void testValidateSize() {
+        var shardsAvailable = new HealthIndicatorResult("shards_availability", GREEN, null, null, null, null);
+
+        var service = new HealthService(Collections.emptyList(), List.of(createMockHealthIndicatorService(shardsAvailable)), threadPool);
+        NodeClient client = getTestClient(HealthInfo.EMPTY_HEALTH_INFO);
+        IllegalArgumentException illegalArgumentException = expectThrows(
+            IllegalArgumentException.class,
+            () -> service.getHealth(client, null, true, -1, ActionListener.NOOP)
+        );
+        assertThat(illegalArgumentException.getMessage(), is("The max number of resources must be a positive integer"));
+    }
+
     private <T extends Throwable> void assertGetHealthThrowsException(
         HealthService service,
         NodeClient client,
         String indicatorName,
-        boolean explain,
+        boolean verbose,
         Class<T> expectedType,
         String expectedMessage,
         boolean expectOnFailCalled
@@ -154,7 +168,7 @@ private <T extends Throwable> void assertGetHealthThrowsException(
             expectedMessage
         );
         try {
-            service.getHealth(client, indicatorName, explain, listener);
+            service.getHealth(client, indicatorName, verbose, 1000, listener);
         } catch (Throwable t) {
             if (expectOnFailCalled || (expectedType.isInstance(t) == false)) {
                 throw new RuntimeException("Unexpected throwable", t);
@@ -325,7 +339,7 @@ public void onFailure(Exception e) {
                 throw new RuntimeException(e);
             }
         };
-        service.getHealth(client, indicatorName, false, listener);
+        service.getHealth(client, indicatorName, false, 1000, listener);
         assertBusy(() -> assertNotNull(resultReference.get()));
         return resultReference.get();
     }
@@ -366,7 +380,7 @@ public String name() {
             }
 
             @Override
-            public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
+            public HealthIndicatorResult calculate(boolean verbose, int maxAffectedResourcesCount, HealthInfo healthInfo) {
                 if (expectedHealthInfo != null) {
                     assertThat(healthInfo, equalTo(expectedHealthInfo));
                 }
diff --git a/server/src/test/java/org/elasticsearch/health/HealthStatusTests.java b/server/src/test/java/org/elasticsearch/health/HealthStatusTests.java
index 5666936ab183..3bb977263df2 100644
--- a/server/src/test/java/org/elasticsearch/health/HealthStatusTests.java
+++ b/server/src/test/java/org/elasticsearch/health/HealthStatusTests.java
@@ -67,6 +67,11 @@ protected HealthStatus createTestInstance() {
         return randomFrom(HealthStatus.values());
     }
 
+    @Override
+    protected HealthStatus mutateInstance(HealthStatus instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected void assertEqualInstances(HealthStatus expectedInstance, HealthStatus newInstance) {
         assertThat(newInstance, equalTo(expectedInstance));
diff --git a/server/src/test/java/org/elasticsearch/health/metadata/HealthMetadataSerializationTests.java b/server/src/test/java/org/elasticsearch/health/metadata/HealthMetadataSerializationTests.java
index 3661726f3d72..fb89ef991a3f 100644
--- a/server/src/test/java/org/elasticsearch/health/metadata/HealthMetadataSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/health/metadata/HealthMetadataSerializationTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.RatioValue;
 import org.elasticsearch.common.unit.RelativeByteSizeValue;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.test.SimpleDiffableWireSerializationTestCase;
 
 import java.util.List;
@@ -51,6 +52,11 @@ protected ClusterState.Custom createTestInstance() {
         return randomHealthMetadata();
     }
 
+    @Override
+    protected ClusterState.Custom mutateInstance(ClusterState.Custom instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     private static HealthMetadata randomHealthMetadata() {
         return new HealthMetadata(randomDiskMetadata());
     }
@@ -102,4 +108,8 @@ static HealthMetadata.Disk mutateDiskMetadata(HealthMetadata.Disk base) {
     private HealthMetadata mutate(HealthMetadata base) {
         return new HealthMetadata(mutateDiskMetadata(base.getDiskMetadata()));
     }
+
+    public void testChunking() {
+        AbstractChunkedSerializingTestCase.assertChunkCount(createTestInstance(), ignored -> 1);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/health/metadata/HealthMetadataTests.java b/server/src/test/java/org/elasticsearch/health/metadata/HealthMetadataTests.java
index 2453c4efae22..83a0f998261c 100644
--- a/server/src/test/java/org/elasticsearch/health/metadata/HealthMetadataTests.java
+++ b/server/src/test/java/org/elasticsearch/health/metadata/HealthMetadataTests.java
@@ -30,11 +30,14 @@ public void testFreeBytesCalculationOfAbsoluteValue() {
     public void testFreeBytesCalculationMaxHeadroom() {
         HealthMetadata.Disk metadata = HealthMetadata.Disk.newBuilder()
             .highWatermark("90%", "ratio-high")
+            .highMaxHeadroom(ByteSizeValue.ofBytes(10))
             .floodStageWatermark("95%", "ratio-flood")
+            .floodStageMaxHeadroom(ByteSizeValue.ofBytes(5))
             .frozenFloodStageWatermark("95%", "ratio-frozen-flood")
             .frozenFloodStageMaxHeadroom("20B", "headroom")
             .build();
-        // For now only the frozen tier is using the max headroom setting
+        assertThat(metadata.getFreeBytesHighWatermark(ByteSizeValue.ofBytes(1000)), equalTo(ByteSizeValue.ofBytes(10)));
+        assertThat(metadata.getFreeBytesFloodStageWatermark(ByteSizeValue.ofBytes(1000)), equalTo(ByteSizeValue.ofBytes(5)));
         assertThat(metadata.getFreeBytesFrozenFloodStageWatermark(ByteSizeValue.ofBytes(1000)), equalTo(ByteSizeValue.ofBytes(20)));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/health/node/DiskHealthIndicatorServiceTests.java b/server/src/test/java/org/elasticsearch/health/node/DiskHealthIndicatorServiceTests.java
index d0f04588cd53..75610ada727b 100644
--- a/server/src/test/java/org/elasticsearch/health/node/DiskHealthIndicatorServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/health/node/DiskHealthIndicatorServiceTests.java
@@ -57,6 +57,7 @@
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED;
+import static org.elasticsearch.health.HealthIndicatorService.MAX_AFFECTED_RESOURCES_COUNT;
 import static org.elasticsearch.health.ImpactArea.DEPLOYMENT_MANAGEMENT;
 import static org.elasticsearch.health.node.DiskHealthIndicatorService.DiskHealthAnalyzer.INDICES_WITH_READONLY_BLOCK;
 import static org.elasticsearch.health.node.DiskHealthIndicatorService.DiskHealthAnalyzer.NODES_OVER_FLOOD_STAGE_WATERMARK;
@@ -882,6 +883,115 @@ public void testNodeHealthStatusCounts() {
         }
     }
 
+    public void testLimitNumberOfAffectedResources() {
+        Set<DiscoveryNodeRole> otherRoles = new HashSet<>(randomNonEmptySubsetOf(OTHER_ROLES));
+        Set<DiscoveryNodeRole> dataRoles = new HashSet<>(randomNonEmptySubsetOf(DATA_ROLES));
+        Set<DiscoveryNodeRole> masterRole = Set.of(DiscoveryNodeRole.MASTER_ROLE);
+        Set<DiscoveryNode> dataNodes = createNodes(30, dataRoles);
+        Set<DiscoveryNode> masterNodes = createNodes(20, masterRole);
+        Set<DiscoveryNode> otherNodes = createNodes(10, otherRoles);
+        ClusterService clusterService = createClusterService(Sets.union(Sets.union(dataNodes, masterNodes), otherNodes), true);
+        DiskHealthIndicatorService diskHealthIndicatorService = new DiskHealthIndicatorService(clusterService);
+        int numberOfRedMasterNodes = masterNodes.size();
+        int numberOfRedOtherNodes = otherNodes.size();
+        int numberOfYellowDataNodes = dataNodes.size();
+        HealthInfo healthInfo = createHealthInfo(
+            List.of(
+                new HealthInfoConfig(HealthStatus.YELLOW, numberOfYellowDataNodes, dataNodes),
+                new HealthInfoConfig(HealthStatus.RED, numberOfRedMasterNodes, masterNodes),
+                new HealthInfoConfig(HealthStatus.RED, numberOfRedOtherNodes, otherNodes)
+            )
+        );
+        {
+            HealthIndicatorResult result = diskHealthIndicatorService.calculate(true, 0, healthInfo);
+            List<Diagnosis> diagnosisList = result.diagnosisList();
+            assertThat(diagnosisList.size(), equalTo(3));
+            {
+                Diagnosis diagnosis = diagnosisList.get(0);
+                List<Diagnosis.Resource> dataAffectedResources = diagnosis.affectedResources();
+                assertThat(dataAffectedResources.size(), equalTo(2));
+                assertThat(dataAffectedResources.get(0).getType(), is(Diagnosis.Resource.Type.NODE));
+                assertThat(dataAffectedResources.get(0).getNodes().size(), is(0));
+                assertThat(dataAffectedResources.get(1).getType(), is(Diagnosis.Resource.Type.INDEX));
+                assertThat(dataAffectedResources.get(1).getValues().size(), is(0));
+            }
+            {
+                Diagnosis diagnosis = diagnosisList.get(1);
+                List<Diagnosis.Resource> masterAffectedResources = diagnosis.affectedResources();
+                assertThat(masterAffectedResources.size(), equalTo(1));
+                assertThat(masterAffectedResources.get(0).getType(), is(Diagnosis.Resource.Type.NODE));
+                assertThat(masterAffectedResources.get(0).getNodes().size(), is(0));
+            }
+
+            {
+                Diagnosis diagnosis = diagnosisList.get(2);
+                List<Diagnosis.Resource> nonDataNonMasterAffectedResources = diagnosis.affectedResources();
+                assertThat(nonDataNonMasterAffectedResources.size(), equalTo(1));
+                assertThat(nonDataNonMasterAffectedResources.get(0).getType(), is(Diagnosis.Resource.Type.NODE));
+                assertThat(nonDataNonMasterAffectedResources.get(0).getNodes().size(), is(0));
+            }
+        }
+
+        {
+            HealthIndicatorResult result = diskHealthIndicatorService.calculate(true, 10, healthInfo);
+            List<Diagnosis> diagnosisList = result.diagnosisList();
+            assertThat(diagnosisList.size(), equalTo(3));
+            {
+                Diagnosis diagnosis = diagnosisList.get(0);
+                List<Diagnosis.Resource> dataAffectedResources = diagnosis.affectedResources();
+                assertThat(dataAffectedResources.size(), equalTo(2));
+                assertThat(dataAffectedResources.get(0).getType(), is(Diagnosis.Resource.Type.NODE));
+                assertThat(dataAffectedResources.get(0).getNodes().size(), is(10));
+                assertThat(dataAffectedResources.get(1).getType(), is(Diagnosis.Resource.Type.INDEX));
+                assertThat(dataAffectedResources.get(1).getValues().size(), is(1));
+            }
+            {
+                Diagnosis diagnosis = diagnosisList.get(1);
+                List<Diagnosis.Resource> masterAffectedResources = diagnosis.affectedResources();
+                assertThat(masterAffectedResources.size(), equalTo(1));
+                assertThat(masterAffectedResources.get(0).getType(), is(Diagnosis.Resource.Type.NODE));
+                assertThat(masterAffectedResources.get(0).getNodes().size(), is(10));
+            }
+
+            {
+                Diagnosis diagnosis = diagnosisList.get(2);
+                List<Diagnosis.Resource> nonDataNonMasterAffectedResources = diagnosis.affectedResources();
+                assertThat(nonDataNonMasterAffectedResources.size(), equalTo(1));
+                assertThat(nonDataNonMasterAffectedResources.get(0).getType(), is(Diagnosis.Resource.Type.NODE));
+                assertThat(nonDataNonMasterAffectedResources.get(0).getNodes().size(), is(10));
+            }
+        }
+
+    }
+
+    // We expose the indicator name and the diagnoses in the x-pack usage API. In order to index them properly in a telemetry index
+    // they need to be declared in the health-api-indexer.edn in the telemetry repository.
+    public void testMappedFieldsForTelemetry() {
+        assertThat(DiskHealthIndicatorService.NAME, equalTo("disk"));
+        assertThat(
+            DiskHealthIndicatorService.DiskHealthAnalyzer.createDataNodeDiagnosis(0, List.of()).definition().getUniqueId(),
+            equalTo("elasticsearch:health:disk:diagnosis:add_disk_capacity_data_nodes")
+        );
+        assertThat(
+            DiskHealthIndicatorService.DiskHealthAnalyzer.createNonDataNodeDiagnosis(
+                HealthStatus.RED,
+                List.of(),
+                MAX_AFFECTED_RESOURCES_COUNT,
+                true
+            ).definition().getUniqueId(),
+            equalTo("elasticsearch:health:disk:diagnosis:add_disk_capacity_master_nodes")
+        );
+        assertThat(
+            DiskHealthIndicatorService.DiskHealthAnalyzer.createNonDataNodeDiagnosis(
+                HealthStatus.RED,
+                List.of(),
+                MAX_AFFECTED_RESOURCES_COUNT,
+                false
+            ).definition().getUniqueId(),
+            equalTo("elasticsearch:health:disk:diagnosis:add_disk_capacity")
+        );
+    }
+
     private Set<DiscoveryNode> createNodesWithAllRoles() {
         return createNodes(DiscoveryNodeRole.roles());
     }
diff --git a/server/src/test/java/org/elasticsearch/health/node/LocalHealthMonitorTests.java b/server/src/test/java/org/elasticsearch/health/node/LocalHealthMonitorTests.java
index cc6fdacdb422..a8083966ae31 100644
--- a/server/src/test/java/org/elasticsearch/health/node/LocalHealthMonitorTests.java
+++ b/server/src/test/java/org/elasticsearch/health/node/LocalHealthMonitorTests.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -43,7 +44,9 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
+import static org.elasticsearch.action.support.replication.ClusterStateCreationUtils.state;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
@@ -306,6 +309,49 @@ public void testFrozenRedDiskStatus() {
         assertThat(diskHealth, equalTo(new DiskHealthInfo(HealthStatus.RED, DiskHealthInfo.Cause.FROZEN_NODE_OVER_FLOOD_STAGE_THRESHOLD)));
     }
 
+    public void testYellowStatusForNonDataNode() {
+        DiscoveryNode dedicatedMasterNode = new DiscoveryNode(
+            "master-node",
+            "master-node-1",
+            ESTestCase.buildNewFakeTransportAddress(),
+            Collections.emptyMap(),
+            Set.of(DiscoveryNodeRole.MASTER_ROLE),
+            Version.CURRENT
+        );
+        clusterState = ClusterStateCreationUtils.state(
+            dedicatedMasterNode,
+            dedicatedMasterNode,
+            node,
+            new DiscoveryNode[] { node, dedicatedMasterNode }
+        ).copyAndUpdate(b -> b.putCustom(HealthMetadata.TYPE, healthMetadata));
+
+        initializeIncreasedDiskSpaceUsage();
+        LocalHealthMonitor.DiskCheck diskMonitor = new LocalHealthMonitor.DiskCheck(nodeService);
+        DiskHealthInfo diskHealth = diskMonitor.getHealth(healthMetadata, clusterState);
+        assertThat(diskHealth, equalTo(new DiskHealthInfo(HealthStatus.YELLOW, DiskHealthInfo.Cause.NODE_OVER_HIGH_THRESHOLD)));
+    }
+
+    public void testHasRelocatingShards() {
+        String indexName = "my-index";
+        final ClusterState state = state(indexName, true, ShardRoutingState.RELOCATING);
+        // local node coincides with the node hosting the (relocating) primary shard
+        DiscoveryNode localNode = state.nodes().getLocalNode();
+        assertThat(LocalHealthMonitor.DiskCheck.hasRelocatingShards(state, localNode), is(true));
+
+        DiscoveryNode dedicatedMasterNode = new DiscoveryNode(
+            "master-node",
+            "master-node-1",
+            ESTestCase.buildNewFakeTransportAddress(),
+            Collections.emptyMap(),
+            Set.of(DiscoveryNodeRole.MASTER_ROLE),
+            Version.CURRENT
+        );
+        ClusterState newState = ClusterState.builder(state)
+            .nodes(new DiscoveryNodes.Builder(state.nodes()).add(dedicatedMasterNode))
+            .build();
+        assertThat(LocalHealthMonitor.DiskCheck.hasRelocatingShards(newState, dedicatedMasterNode), is(false));
+    }
+
     private void simulateDiskOutOfSpace() {
         when(
             nodeService.stats(
diff --git a/server/src/test/java/org/elasticsearch/health/node/action/TransportHealthNodeActionTests.java b/server/src/test/java/org/elasticsearch/health/node/action/TransportHealthNodeActionTests.java
index 91d924d8b97c..8b9801724ed9 100644
--- a/server/src/test/java/org/elasticsearch/health/node/action/TransportHealthNodeActionTests.java
+++ b/server/src/test/java/org/elasticsearch/health/node/action/TransportHealthNodeActionTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.ActionTestUtils;
 import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.action.support.ThreadedActionListener;
 import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -187,12 +186,6 @@ class Action extends TransportHealthNodeAction<Request, Response> {
             );
         }
 
-        @Override
-        protected void doExecute(Task task, final Request request, ActionListener<Response> listener) {
-            // remove unneeded threading by wrapping listener with SAME to prevent super.doExecute from wrapping it with LISTENER
-            super.doExecute(task, request, new ThreadedActionListener<>(logger, threadPool, ThreadPool.Names.SAME, listener, false));
-        }
-
         @Override
         protected void healthOperation(Task task, Request request, ClusterState state, ActionListener<Response> listener) {
             listener.onResponse(new Response());
diff --git a/server/src/test/java/org/elasticsearch/health/node/selection/HealthNodeTaskParamsTests.java b/server/src/test/java/org/elasticsearch/health/node/selection/HealthNodeTaskParamsTests.java
index 64fecf8707f7..a2cea20a9200 100644
--- a/server/src/test/java/org/elasticsearch/health/node/selection/HealthNodeTaskParamsTests.java
+++ b/server/src/test/java/org/elasticsearch/health/node/selection/HealthNodeTaskParamsTests.java
@@ -24,6 +24,11 @@ protected HealthNodeTaskParams createTestInstance() {
         return new HealthNodeTaskParams();
     }
 
+    @Override
+    protected HealthNodeTaskParams mutateInstance(HealthNodeTaskParams instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected HealthNodeTaskParams copyInstance(HealthNodeTaskParams instance, Version version) throws IOException {
         return new HealthNodeTaskParams();
diff --git a/server/src/test/java/org/elasticsearch/health/stats/HealthApiStatsActionResponseTests.java b/server/src/test/java/org/elasticsearch/health/stats/HealthApiStatsActionResponseTests.java
index 7be9dd1789eb..b03fa68f0cff 100644
--- a/server/src/test/java/org/elasticsearch/health/stats/HealthApiStatsActionResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/health/stats/HealthApiStatsActionResponseTests.java
@@ -12,12 +12,9 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.metrics.Counters;
-import org.elasticsearch.health.HealthStatus;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.List;
-import java.util.Map;
-import java.util.Set;
 
 import static org.hamcrest.Matchers.equalTo;
 
@@ -33,11 +30,6 @@ public void testMerging() {
             counters.inc("merged.metric", randomIntBetween(1, 10));
             counters.inc("only.one.metric", randomIntBetween(1, 10));
             nodeResponse1.setStats(counters);
-            nodeResponse1.setStatuses(Set.of(HealthStatus.RED, HealthStatus.GREEN));
-            nodeResponse1.setIndicators(Map.of(HealthStatus.RED, Set.of("one_red_indicator", "shared_red_indicator")));
-            nodeResponse1.setDiagnoses(
-                Map.of(HealthStatus.RED, Set.of("one:red:diagnosis"), HealthStatus.YELLOW, Set.of("one:yellow:diagnosis"))
-            );
         }
         HealthApiStatsAction.Response.Node nodeResponse2 = new HealthApiStatsAction.Response.Node(
             new DiscoveryNode("remote_node", buildNewFakeTransportAddress(), Version.CURRENT)
@@ -51,13 +43,6 @@ public void testMerging() {
             counters.inc("merged.metric", randomIntBetween(1, 10));
             counters.inc("only.third.metric", randomIntBetween(1, 10));
             nodeResponse3.setStats(counters);
-            nodeResponse3.setStatuses(Set.of(HealthStatus.YELLOW, HealthStatus.GREEN));
-            nodeResponse3.setIndicators(
-                Map.of(HealthStatus.RED, Set.of("shared_red_indicator"), HealthStatus.YELLOW, Set.of("one_yellow_indicator"))
-            );
-            nodeResponse3.setDiagnoses(
-                Map.of(HealthStatus.RED, Set.of("another:red:diagnosis"), HealthStatus.YELLOW, Set.of("another:yellow:diagnosis"))
-            );
         }
         HealthApiStatsAction.Response response = new HealthApiStatsAction.Response(
             ClusterName.DEFAULT,
@@ -71,28 +56,5 @@ public void testMerging() {
         );
         assertThat(stats.get("only.one.metric"), equalTo(nodeResponse1.getStats().get("only.one.metric")));
         assertThat(stats.get("only.third.metric"), equalTo(nodeResponse3.getStats().get("only.third.metric")));
-        assertThat(response.getStatuses(), equalTo(Set.of(HealthStatus.RED, HealthStatus.YELLOW, HealthStatus.GREEN)));
-        assertThat(
-            response.getIndicators(),
-            equalTo(
-                Map.of(
-                    HealthStatus.RED,
-                    Set.of("shared_red_indicator", "one_red_indicator"),
-                    HealthStatus.YELLOW,
-                    Set.of("one_yellow_indicator")
-                )
-            )
-        );
-        assertThat(
-            response.getDiagnoses(),
-            equalTo(
-                Map.of(
-                    HealthStatus.RED,
-                    Set.of("one:red:diagnosis", "another:red:diagnosis"),
-                    HealthStatus.YELLOW,
-                    Set.of("one:yellow:diagnosis", "another:yellow:diagnosis")
-                )
-            )
-        );
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/health/stats/HealthApiStatsTests.java b/server/src/test/java/org/elasticsearch/health/stats/HealthApiStatsTests.java
index 08cf0382cd92..bc579d2518f2 100644
--- a/server/src/test/java/org/elasticsearch/health/stats/HealthApiStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/health/stats/HealthApiStatsTests.java
@@ -18,7 +18,6 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import static org.elasticsearch.health.HealthStatus.GREEN;
 import static org.elasticsearch.health.HealthStatus.RED;
@@ -30,7 +29,7 @@ public class HealthApiStatsTests extends ESTestCase {
     public void testEmptyCounters() {
         HealthApiStats healthApiStats = new HealthApiStats();
         assertThat(healthApiStats.getStats().get("invocations.total"), equalTo(0L));
-        Map<String, Object> metricMap = healthApiStats.getStats().toNestedMap();
+        Map<String, Object> metricMap = healthApiStats.getStats().toMutableNestedMap();
         assertThat(metricMap.containsKey("statuses"), equalTo(false));
         assertThat(metricMap.containsKey("indicators"), equalTo(false));
         assertThat(metricMap.containsKey("diagnoses"), equalTo(false));
@@ -53,13 +52,10 @@ public void testCounters() {
             assertThat(healthApiStats.getStats().get("invocations.verbose_false"), equalTo(1L));
             expectThrows(IllegalArgumentException.class, () -> healthApiStats.getStats().get("invocations.verbose_true"));
             assertThat(healthApiStats.getStats().get("statuses.green"), equalTo(1L));
-            Map<String, Object> metricMap = healthApiStats.getStats().toNestedMap();
+            Map<String, Object> metricMap = healthApiStats.getStats().toMutableNestedMap();
             assertThat(((Map<String, Object>) metricMap.get("statuses")).size(), equalTo(1));
             assertThat(metricMap.containsKey("indicators"), equalTo(false));
             assertThat(metricMap.containsKey("diagnoses"), equalTo(false));
-            assertThat(healthApiStats.getStatuses(), equalTo(Set.of(GREEN)));
-            assertThat(healthApiStats.getIndicators().isEmpty(), equalTo(true));
-            assertThat(healthApiStats.getDiagnoses().isEmpty(), equalTo(true));
         }
         {
             // Yellow no diagnosis
@@ -76,11 +72,8 @@ public void testCounters() {
             assertThat(healthApiStats.getStats().get("statuses.green"), equalTo(1L));
             assertThat(healthApiStats.getStats().get("statuses.yellow"), equalTo(1L));
             assertThat(healthApiStats.getStats().get("indicators.yellow.shards_availability"), equalTo(1L));
-            Map<String, Object> metricMap = healthApiStats.getStats().toNestedMap();
+            Map<String, Object> metricMap = healthApiStats.getStats().toMutableNestedMap();
             assertThat(metricMap.containsKey("diagnoses"), equalTo(false));
-            assertThat(healthApiStats.getStatuses(), equalTo(Set.of(GREEN, YELLOW)));
-            assertThat(healthApiStats.getIndicators().get(YELLOW), equalTo(Set.of("shards_availability")));
-            assertThat(healthApiStats.getDiagnoses().isEmpty(), equalTo(true));
         }
         {
             // Yellow and Red with diagnoses
@@ -109,17 +102,6 @@ public void testCounters() {
                 healthApiStats.getStats().get("diagnoses.red.elasticsearch:health:shards_availability:diagnosis:primary_unassigned"),
                 equalTo(1L)
             );
-            assertThat(healthApiStats.getStatuses(), equalTo(Set.of(GREEN, YELLOW, RED)));
-            assertThat(healthApiStats.getIndicators().get(YELLOW), equalTo(Set.of("shards_availability", "disk")));
-            assertThat(healthApiStats.getIndicators().get(RED), equalTo(Set.of("shards_availability")));
-            assertThat(
-                healthApiStats.getDiagnoses().get(YELLOW),
-                equalTo(Set.of("elasticsearch:health:disk:diagnosis:add_disk_capacity_data_nodes"))
-            );
-            assertThat(
-                healthApiStats.getDiagnoses().get(RED),
-                equalTo(Set.of("elasticsearch:health:shards_availability:diagnosis:primary_unassigned"))
-            );
         }
         {
             // Increment counters
@@ -153,22 +135,6 @@ public void testCounters() {
                 healthApiStats.getStats().get("diagnoses.red.elasticsearch:health:disk:diagnosis:add_disk_capacity_data_nodes"),
                 equalTo(1L)
             );
-            assertThat(healthApiStats.getStatuses(), equalTo(Set.of(GREEN, YELLOW, RED)));
-            assertThat(healthApiStats.getIndicators().get(YELLOW), equalTo(Set.of("shards_availability", "disk")));
-            assertThat(healthApiStats.getIndicators().get(RED), equalTo(Set.of("shards_availability", "disk")));
-            assertThat(
-                healthApiStats.getDiagnoses().get(YELLOW),
-                equalTo(Set.of("elasticsearch:health:disk:diagnosis:add_disk_capacity_data_nodes"))
-            );
-            assertThat(
-                healthApiStats.getDiagnoses().get(RED),
-                equalTo(
-                    Set.of(
-                        "elasticsearch:health:shards_availability:diagnosis:primary_unassigned",
-                        "elasticsearch:health:disk:diagnosis:add_disk_capacity_data_nodes"
-                    )
-                )
-            );
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/http/HttpTracerTests.java b/server/src/test/java/org/elasticsearch/http/HttpTracerTests.java
new file mode 100644
index 000000000000..d754c3ab0a07
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/http/HttpTracerTests.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.http;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.MockLogAppender;
+import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+
+import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.Map;
+
+public class HttpTracerTests extends ESTestCase {
+
+    @TestLogging(reason = "Get HttpTracer to output trace logs", value = "org.elasticsearch.http.HttpTracer:TRACE")
+    public void testLogging() {
+        String httpTracerLog = HttpTracer.class.getName();
+
+        MockLogAppender appender = new MockLogAppender();
+        try {
+            appender.start();
+            Loggers.addAppender(LogManager.getLogger(httpTracerLog), appender);
+
+            appender.addExpectation(
+                new MockLogAppender.PatternSeenEventExpectation(
+                    "request log",
+                    httpTracerLog,
+                    Level.TRACE,
+                    "\\[\\d+]\\[idHeader]\\[GET]\\[uri] received request from \\[.*] trace.id: 4bf92f3577b34da6a3ce929d0e0e4736"
+                )
+            );
+            appender.addExpectation(
+                new MockLogAppender.PatternSeenEventExpectation(
+                    "response log",
+                    httpTracerLog,
+                    Level.TRACE,
+                    "\\[\\d+]\\[idHeader]\\[ACCEPTED]\\[text/plain; charset=UTF-8]\\[length] sent response to \\[.*] success \\[true]"
+                )
+            );
+
+            RestRequest request = new FakeRestRequest.Builder(new NamedXContentRegistry(List.of())).withMethod(RestRequest.Method.GET)
+                .withPath("uri")
+                .withHeaders(
+                    Map.of(
+                        Task.X_OPAQUE_ID_HTTP_HEADER,
+                        List.of("idHeader"),
+                        "traceparent",
+                        List.of("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01")
+                    )
+                )
+                .build();
+
+            HttpTracer tracer = new HttpTracer().maybeLogRequest(request, null);
+            assertNotNull(tracer);
+
+            tracer.logResponse(
+                new RestResponse(RestStatus.ACCEPTED, ""),
+                new FakeRestRequest.FakeHttpChannel(InetSocketAddress.createUnresolved("127.0.0.1", 9200)),
+                "length",
+                "idHeader",
+                1L,
+                true
+            );
+
+            appender.assertAllExpectationsMatched();
+        } finally {
+            Loggers.removeAppender(LogManager.getLogger(httpTracerLog), appender);
+            appender.stop();
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/CompositeIndexEventListenerTests.java b/server/src/test/java/org/elasticsearch/index/CompositeIndexEventListenerTests.java
new file mode 100644
index 000000000000..f910081286df
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/CompositeIndexEventListenerTests.java
@@ -0,0 +1,112 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index;
+
+import org.apache.logging.log4j.Level;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRunnable;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.core.CheckedRunnable;
+import org.elasticsearch.index.shard.IndexEventListener;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.IndexShardTestCase;
+import org.elasticsearch.test.MockLogAppender;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.hamcrest.Matchers;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+public class CompositeIndexEventListenerTests extends IndexShardTestCase {
+
+    private Exception getRootCause(Exception e) {
+        while (e.getCause()instanceof Exception ee) {
+            e = ee;
+        }
+        if (e.getCause() != null) {
+            throw new AssertionError(e.getCause());
+        }
+        return e;
+    }
+
+    public void testBeforeIndexShardRecoveryInOrder() throws Exception {
+        var shard = newShard(randomBoolean());
+        var appender = new MockLogAppender();
+        try (var ignored = appender.capturing(CompositeIndexEventListener.class)) {
+            final var stepNumber = new AtomicInteger();
+            final var stepCount = between(0, 20);
+            final var failAtStep = new AtomicInteger(-1);
+            final var indexEventListener = new CompositeIndexEventListener(
+                shard.indexSettings(),
+                IntStream.range(0, stepCount).mapToObj(step -> new IndexEventListener() {
+                    @Override
+                    public void beforeIndexShardRecovery(
+                        IndexShard indexShard,
+                        IndexSettings indexSettings,
+                        ActionListener<Void> listener
+                    ) {
+                        if (randomBoolean()) {
+                            // throws an exception sometimes
+                            runStep();
+                            listener.onResponse(null);
+                        } else {
+                            // fails the listener sometimes
+                            shard.getThreadPool()
+                                .executor(randomFrom(ThreadPool.Names.GENERIC, ThreadPool.Names.SAME))
+                                .execute(ActionRunnable.run(listener, this::runStep));
+                        }
+                    }
+
+                    private void runStep() {
+                        assertThat(step, Matchers.lessThanOrEqualTo(failAtStep.get()));
+                        assertTrue(stepNumber.compareAndSet(step, step + 1));
+                        if (step == failAtStep.get()) {
+                            throw new ElasticsearchException("simulated failure at step " + step);
+                        }
+                    }
+                }).collect(Collectors.toList())
+            );
+
+            final CheckedRunnable<Exception> beforeIndexShardRecoveryRunner = () -> assertNull(
+                PlainActionFuture.<Void, Exception>get(
+                    fut -> indexEventListener.beforeIndexShardRecovery(shard, shard.indexSettings(), fut),
+                    10,
+                    TimeUnit.SECONDS
+                )
+            );
+
+            failAtStep.set(stepCount);
+            beforeIndexShardRecoveryRunner.run();
+            assertEquals(stepCount, stepNumber.getAndSet(0));
+
+            if (stepCount > 0) {
+                appender.addExpectation(
+                    new MockLogAppender.SeenEventExpectation(
+                        "warning",
+                        CompositeIndexEventListener.class.getCanonicalName(),
+                        Level.WARN,
+                        "*failed to invoke the listener before the shard recovery starts for [index][0]"
+                    )
+                );
+
+                failAtStep.set(between(0, stepCount - 1));
+                final var rootCause = getRootCause(expectThrows(ElasticsearchException.class, beforeIndexShardRecoveryRunner::run));
+                assertEquals("simulated failure at step " + failAtStep.get(), rootCause.getMessage());
+                assertEquals(failAtStep.get() + 1, stepNumber.getAndSet(0));
+                appender.assertAllExpectationsMatched();
+            }
+
+        } finally {
+            closeShards(shard);
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
index 61fa1809a665..88ae556ec554 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
@@ -11,6 +11,7 @@
 import org.apache.lucene.analysis.standard.StandardTokenizer;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.CollectionStatistics;
 import org.apache.lucene.search.QueryCachingPolicy;
@@ -23,13 +24,21 @@
 import org.apache.lucene.tests.index.AssertingDirectoryReader;
 import org.apache.lucene.util.SetOnce.AlreadySetException;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -51,19 +60,27 @@
 import org.elasticsearch.index.cache.query.QueryCache;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.EngineTestCase;
+import org.elasticsearch.index.engine.InternalEngine;
 import org.elasticsearch.index.engine.InternalEngineFactory;
+import org.elasticsearch.index.engine.SafeCommitInfo;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.Uid;
+import org.elasticsearch.index.seqno.ReplicationTracker;
+import org.elasticsearch.index.seqno.RetentionLeaseSyncer;
+import org.elasticsearch.index.seqno.RetentionLeases;
 import org.elasticsearch.index.shard.IndexEventListener;
+import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexingOperationListener;
+import org.elasticsearch.index.shard.ReplicationGroup;
 import org.elasticsearch.index.shard.SearchOperationListener;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardPath;
 import org.elasticsearch.index.similarity.NonNegativeScoresSimilarity;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.index.store.FsDirectoryFactory;
+import org.elasticsearch.index.store.Store;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.IndicesQueryCache;
 import org.elasticsearch.indices.TestIndexNameExpressionResolver;
@@ -84,24 +101,39 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.hamcrest.Matchers;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.function.LongConsumer;
+import java.util.function.LongSupplier;
+import java.util.function.Supplier;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.index.IndexService.IndexCreationContext.CREATE_INDEX;
+import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.sameInstance;
 import static org.mockito.Mockito.mock;
 
 public class IndexModuleTests extends ESTestCase {
@@ -272,8 +304,20 @@ public void testDirectoryWrapper() throws IOException {
 
         final ShardId shardId = new ShardId(indexSettings.getIndex(), randomIntBetween(0, 5));
         final Path dataPath = new NodeEnvironment.DataPath(homeDir).resolve(shardId);
-        final Directory directory = directoryFactory.newDirectory(indexSettings, new ShardPath(false, dataPath, dataPath, shardId));
+        Directory directory = directoryFactory.newDirectory(indexSettings, new ShardPath(false, dataPath, dataPath, shardId));
         assertThat(directory, instanceOf(WrappedDirectory.class));
+        assertThat(((WrappedDirectory) directory).shardRouting, nullValue());
+        assertThat(directory, instanceOf(FilterDirectory.class));
+
+        final ShardRouting shardRouting = TestShardRouting.newShardRouting(
+            shardId,
+            randomIdentifier(),
+            randomBoolean(),
+            ShardRoutingState.INITIALIZING
+        );
+        directory = directoryFactory.newDirectory(indexSettings, new ShardPath(false, dataPath, dataPath, shardId), shardRouting);
+        assertThat(directory, instanceOf(WrappedDirectory.class));
+        assertThat(((WrappedDirectory) directory).shardRouting, sameInstance(shardRouting));
         assertThat(directory, instanceOf(FilterDirectory.class));
 
         indexService.close("test done", false);
@@ -415,6 +459,8 @@ public void testFrozen() {
         assertEquals(msg, expectThrows(IllegalStateException.class, () -> module.setReaderWrapper(null)).getMessage());
         assertEquals(msg, expectThrows(IllegalStateException.class, () -> module.forceQueryCacheProvider(null)).getMessage());
         assertEquals(msg, expectThrows(IllegalStateException.class, () -> module.setDirectoryWrapper(null)).getMessage());
+        assertEquals(msg, expectThrows(IllegalStateException.class, () -> module.setIndexCommitListener(null)).getMessage());
+        assertEquals(msg, expectThrows(IllegalStateException.class, () -> module.setReplicationTrackerFactory(null)).getMessage());
     }
 
     public void testSetupUnknownSimilarity() {
@@ -610,12 +656,185 @@ public void testRegisterCustomRecoveryStateFactory() throws IOException {
         indexService.close("closing", false);
     }
 
+    public void testIndexCommitListenerIsBound() throws IOException, ExecutionException, InterruptedException {
+        IndexModule module = new IndexModule(
+            indexSettings,
+            emptyAnalysisRegistry,
+            InternalEngine::new,
+            Collections.emptyMap(),
+            () -> true,
+            indexNameExpressionResolver,
+            Collections.emptyMap()
+        );
+
+        final AtomicLong lastAcquiredPrimaryTerm = new AtomicLong();
+        final AtomicReference<Engine.IndexCommitRef> lastAcquiredCommit = new AtomicReference<>();
+        final AtomicReference<IndexCommit> lastDeletedCommit = new AtomicReference<>();
+
+        module.setIndexCommitListener(new Engine.IndexCommitListener() {
+            @Override
+            public void onNewCommit(
+                ShardId shardId,
+                Store store,
+                long primaryTerm,
+                Engine.IndexCommitRef indexCommitRef,
+                Set<String> additionalFiles
+            ) {
+                lastAcquiredPrimaryTerm.set(primaryTerm);
+                lastAcquiredCommit.set(indexCommitRef);
+            }
+
+            @Override
+            public void onIndexCommitDelete(ShardId shardId, IndexCommit deletedCommit) {
+                lastDeletedCommit.set(deletedCommit);
+            }
+        });
+
+        final List<Closeable> closeables = new ArrayList<>();
+        try {
+            ShardId shardId = new ShardId("index", UUIDs.randomBase64UUID(random()), 0);
+            ShardRouting shardRouting = ShardRouting.newUnassigned(
+                shardId,
+                true,
+                RecoverySource.EmptyStoreRecoverySource.INSTANCE,
+                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null),
+                ShardRouting.Role.DEFAULT
+            ).initialize("_node_id", null, -1);
+
+            IndexService indexService = newIndexService(module);
+            closeables.add(() -> indexService.close("close index service at end of test", false));
+
+            IndexShard indexShard = indexService.createShard(shardRouting, s -> {}, RetentionLeaseSyncer.EMPTY);
+            closeables.add(() -> indexShard.close("close shard at end of test", true));
+            indexShard.markAsRecovering(
+                "test",
+                new RecoveryState(
+                    shardRouting,
+                    new DiscoveryNode(
+                        "_node_id",
+                        "_node_id",
+                        buildNewFakeTransportAddress(),
+                        Collections.emptyMap(),
+                        DiscoveryNodeRole.roles(),
+                        Version.CURRENT
+                    ),
+                    null
+                )
+            );
+
+            final PlainActionFuture<Boolean> recoveryFuture = PlainActionFuture.newFuture();
+            indexShard.recoverFromStore(recoveryFuture);
+            recoveryFuture.get();
+
+            assertThat(lastAcquiredPrimaryTerm.get(), equalTo(indexShard.getOperationPrimaryTerm()));
+            Engine.IndexCommitRef lastCommitRef = lastAcquiredCommit.get();
+            assertThat(lastCommitRef, notNullValue());
+            IndexCommit lastCommit = lastCommitRef.getIndexCommit();
+            assertThat(lastCommit.getGeneration(), equalTo(2L));
+            IndexCommit lastDeleted = lastDeletedCommit.get();
+            assertThat(lastDeleted, nullValue());
+
+            lastCommitRef.close();
+
+            indexShard.flush(new FlushRequest("index").force(true));
+
+            lastDeleted = lastDeletedCommit.get();
+            assertThat(lastDeleted.getGeneration(), equalTo(lastCommit.getGeneration()));
+            assertThat(lastDeleted.getSegmentsFileName(), equalTo(lastCommit.getSegmentsFileName()));
+            assertThat(lastDeleted.isDeleted(), equalTo(true));
+
+            lastCommitRef = lastAcquiredCommit.get();
+            assertThat(lastCommitRef, notNullValue());
+            lastCommit = lastCommitRef.getIndexCommit();
+            assertThat(lastCommit.getGeneration(), equalTo(3L));
+
+            lastCommitRef.close();
+        } finally {
+            IOUtils.close(closeables);
+        }
+    }
+
+    public void testCustomReplicationTrackerFactory() throws IOException {
+        IndexModule module = new IndexModule(
+            indexSettings,
+            emptyAnalysisRegistry,
+            InternalEngine::new,
+            Collections.emptyMap(),
+            () -> true,
+            indexNameExpressionResolver,
+            Collections.emptyMap()
+        );
+
+        class CustomReplicationTracker extends ReplicationTracker {
+            CustomReplicationTracker(
+                ShardId shardId,
+                String allocationId,
+                IndexSettings indexSettings,
+                long operationPrimaryTerm,
+                long globalCheckpoint,
+                LongConsumer onGlobalCheckpointUpdated,
+                LongSupplier currentTimeMillisSupplier,
+                BiConsumer<RetentionLeases, ActionListener<ReplicationResponse>> onSyncRetentionLeases,
+                Supplier<SafeCommitInfo> safeCommitInfoSupplier,
+                Consumer<ReplicationGroup> onReplicationGroupUpdated
+            ) {
+                super(
+                    shardId,
+                    allocationId,
+                    indexSettings,
+                    operationPrimaryTerm,
+                    globalCheckpoint,
+                    onGlobalCheckpointUpdated,
+                    currentTimeMillisSupplier,
+                    onSyncRetentionLeases,
+                    safeCommitInfoSupplier,
+                    onReplicationGroupUpdated
+                );
+            }
+        }
+        module.setReplicationTrackerFactory(
+            (
+                shardId,
+                allocationId,
+                indexSettings,
+                operationPrimaryTerm,
+                onGlobalCheckpointUpdated,
+                currentTimeMillisSupplier,
+                onSyncRetentionLeases,
+                safeCommitInfoSupplier,
+                onReplicationGroupUpdated) -> new CustomReplicationTracker(
+                    shardId,
+                    allocationId,
+                    indexSettings,
+                    operationPrimaryTerm,
+                    UNASSIGNED_SEQ_NO,
+                    onGlobalCheckpointUpdated,
+                    currentTimeMillisSupplier,
+                    onSyncRetentionLeases,
+                    safeCommitInfoSupplier,
+                    onReplicationGroupUpdated
+                )
+        );
+        final IndexService indexService = newIndexService(module);
+        ShardId shardId = new ShardId("index", UUIDs.randomBase64UUID(random()), 0);
+        ShardRouting shardRouting = ShardRouting.newUnassigned(
+            shardId,
+            true,
+            RecoverySource.EmptyStoreRecoverySource.INSTANCE,
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null),
+            ShardRouting.Role.DEFAULT
+        ).initialize("_node_id", null, -1);
+        IndexShard indexShard = indexService.createShard(shardRouting, s -> {}, RetentionLeaseSyncer.EMPTY);
+        assertThat(indexShard.getReplicationTracker(), instanceOf(CustomReplicationTracker.class));
+    }
+
     private ShardRouting createInitializedShardRouting() {
         ShardRouting shard = ShardRouting.newUnassigned(
             new ShardId("test", "_na_", 0),
             true,
             RecoverySource.ExistingStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null),
+            ShardRouting.Role.DEFAULT
         );
         shard = shard.initialize("node1", null, -1);
         return shard;
@@ -697,16 +916,20 @@ public DirectoryReader apply(DirectoryReader reader) {
     }
 
     private static final class TestDirectoryWrapper implements IndexModule.DirectoryWrapper {
+
         @Override
-        public Directory apply(Directory directory) {
-            return new WrappedDirectory(directory);
+        public Directory wrap(Directory delegate, ShardRouting shardRouting) {
+            return new WrappedDirectory(delegate, shardRouting);
         }
     }
 
     private static final class WrappedDirectory extends FilterDirectory {
 
-        protected WrappedDirectory(Directory in) {
+        final ShardRouting shardRouting;
+
+        protected WrappedDirectory(Directory in, ShardRouting shardRouting) {
             super(in);
+            this.shardRouting = shardRouting;
         }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java b/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java
index b6604c0c9302..30891c76b369 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java
@@ -57,6 +57,19 @@ public static CompressedXContent filter(QueryBuilder filterBuilder) throws IOExc
         return new CompressedXContent(Strings.toString(builder));
     }
 
+    public void testClosedIndexHasNullNodeMappingStats() throws Exception {
+        final IndexService indexService = createIndex("test", Settings.EMPTY);
+        ensureGreen("test");
+
+        final Index index = indexService.index();
+        assertAcked(client().admin().indices().prepareClose(index.getName()));
+        assertBusy(() -> assertTrue("Index not found: " + index.getName(), getInstanceFromNode(IndicesService.class).hasIndex(index)));
+
+        final IndexService closedIndexService = getInstanceFromNode(IndicesService.class).indexServiceSafe(index);
+        assertNotSame(indexService, closedIndexService);
+        assertNull(closedIndexService.getNodeMappingStats());
+    }
+
     public void testBaseAsyncTask() throws Exception {
         IndexService indexService = createIndex("test", Settings.EMPTY);
         AtomicReference<CountDownLatch> latch = new AtomicReference<>(new CountDownLatch(1));
diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
index a6b146925439..4bac7fc71729 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
@@ -30,6 +30,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
 
+import static org.elasticsearch.index.IndexSettings.NODE_DEFAULT_REFRESH_INTERVAL_SETTING;
 import static org.elasticsearch.index.IndexSettings.TIME_SERIES_END_TIME;
 import static org.elasticsearch.index.IndexSettings.TIME_SERIES_START_TIME;
 import static org.hamcrest.CoreMatchers.equalTo;
@@ -345,6 +346,38 @@ public void testRefreshInterval() {
         );
     }
 
+    public void testNodeDefaultRefreshInterval() {
+        String defaultRefreshInterval = getRandomTimeString();
+        IndexMetadata metadata = newIndexMeta(
+            "index",
+            Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build()
+        );
+        IndexSettings settings = new IndexSettings(
+            metadata,
+            Settings.builder().put(NODE_DEFAULT_REFRESH_INTERVAL_SETTING.getKey(), defaultRefreshInterval).build()
+        );
+        assertEquals(
+            TimeValue.parseTimeValue(
+                defaultRefreshInterval,
+                new TimeValue(1, TimeUnit.DAYS),
+                IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey()
+            ),
+            settings.getRefreshInterval()
+        );
+        String newRefreshInterval = getRandomTimeString();
+        settings.updateIndexMetadata(
+            newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), newRefreshInterval).build())
+        );
+        assertEquals(
+            TimeValue.parseTimeValue(
+                newRefreshInterval,
+                new TimeValue(1, TimeUnit.DAYS),
+                IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey()
+            ),
+            settings.getRefreshInterval()
+        );
+    }
+
     private String getRandomTimeString() {
         int refreshIntervalInt = randomFrom(-1, Math.abs(randomInt()));
         String refreshInterval = Integer.toString(refreshIntervalInt);
diff --git a/server/src/test/java/org/elasticsearch/index/IndexTests.java b/server/src/test/java/org/elasticsearch/index/IndexTests.java
index 34b938386512..e9b376c9ecea 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexTests.java
@@ -10,21 +10,47 @@
 
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractXContentSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xcontent.json.JsonXContent;
 
 import java.io.IOException;
 
 import static org.apache.lucene.tests.util.TestUtil.randomSimpleString;
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 
-public class IndexTests extends ESTestCase {
+public class IndexTests extends AbstractXContentSerializingTestCase<Index> {
+
+    @Override
+    protected Writeable.Reader<Index> instanceReader() {
+        return Index::new;
+    }
+
+    @Override
+    protected Index doParseInstance(XContentParser parser) throws IOException {
+        return Index.fromXContent(parser);
+    }
+
+    @Override
+    protected Index createTestInstance() {
+        return new Index(randomIdentifier(), UUIDs.randomBase64UUID());
+    }
+
+    @Override
+    protected Index mutateInstance(Index instance) {
+        return mutate(instance);
+    }
+
+    public static Index mutate(Index instance) {
+        return switch (randomInt(1)) {
+            case 0 -> new Index(randomValueOtherThan(instance.getName(), ESTestCase::randomIdentifier), instance.getUUID());
+            case 1 -> new Index(instance.getName(), randomValueOtherThan(instance.getUUID(), UUIDs::randomBase64UUID));
+            default -> throw new RuntimeException("unreachable");
+        };
+    }
+
     public void testToString() {
         assertEquals("[name/uuid]", new Index("name", "uuid").toString());
         assertEquals("[name]", new Index("name", ClusterState.UNKNOWN_UUID).toString());
@@ -40,30 +66,4 @@ public void testToString() {
             assertThat(random.toString(), containsString(random.getUUID()));
         }
     }
-
-    public void testXContent() throws IOException {
-        final String name = randomAlphaOfLengthBetween(4, 15);
-        final String uuid = UUIDs.randomBase64UUID();
-        final Index original = new Index(name, uuid);
-        final XContentBuilder builder = JsonXContent.contentBuilder();
-        original.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
-            parser.nextToken(); // the beginning of the parser
-            assertThat(Index.fromXContent(parser), equalTo(original));
-        }
-    }
-
-    public void testEquals() {
-        Index index1 = new Index("a", "a");
-        Index index2 = new Index("a", "a");
-        Index index3 = new Index("a", "b");
-        Index index4 = new Index("b", "a");
-        String s = "Some random other object";
-        assertEquals(index1, index1);
-        assertEquals(index1, index2);
-        assertNotEquals(index1, null);
-        assertNotEquals(index1, s);
-        assertNotEquals(index1, index3);
-        assertNotEquals(index1, index4);
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/MergePolicyConfigTests.java b/server/src/test/java/org/elasticsearch/index/MergePolicyConfigTests.java
new file mode 100644
index 000000000000..318f9cdcb0e2
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/MergePolicyConfigTests.java
@@ -0,0 +1,312 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.index;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.SegmentCommitInfo;
+import org.apache.lucene.index.SegmentReader;
+import org.apache.lucene.index.TieredMergePolicy;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.index.IndexSettingsTests.newIndexMeta;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+public class MergePolicyConfigTests extends ESTestCase {
+    protected final ShardId shardId = new ShardId("index", "_na_", 1);
+
+    public void testCompoundFileSettings() throws IOException {
+        assertCompoundThreshold(Settings.EMPTY, 1.0, ByteSizeValue.ofGb(1));
+        assertCompoundThreshold(build(true), 1.0, ByteSizeValue.ofBytes(Long.MAX_VALUE));
+        assertCompoundThreshold(build(0.5), 0.5, ByteSizeValue.ofBytes(Long.MAX_VALUE));
+        assertCompoundThreshold(build(1.0), 1.0, ByteSizeValue.ofBytes(Long.MAX_VALUE));
+        assertCompoundThreshold(build("true"), 1.0, ByteSizeValue.ofBytes(Long.MAX_VALUE));
+        assertCompoundThreshold(build("True"), 1.0, ByteSizeValue.ofBytes(Long.MAX_VALUE));
+        assertCompoundThreshold(build("False"), 0.0, ByteSizeValue.ofBytes(Long.MAX_VALUE));
+        assertCompoundThreshold(build("false"), 0.0, ByteSizeValue.ofBytes(Long.MAX_VALUE));
+        assertCompoundThreshold(build(false), 0.0, ByteSizeValue.ofBytes(Long.MAX_VALUE));
+        assertCompoundThreshold(build(0), 0.0, ByteSizeValue.ofBytes(Long.MAX_VALUE));
+        assertCompoundThreshold(build(0.0), 0.0, ByteSizeValue.ofBytes(Long.MAX_VALUE));
+        assertCompoundThreshold(build(0.0), 0.0, ByteSizeValue.ofBytes(Long.MAX_VALUE));
+        assertCompoundThreshold(build("100MB"), 1.0, ByteSizeValue.ofMb(100));
+        assertCompoundThreshold(build("0MB"), 1.0, ByteSizeValue.ofBytes(0));
+        assertCompoundThreshold(build("0B"), 1.0, ByteSizeValue.ofBytes(0));
+    }
+
+    private void assertCompoundThreshold(Settings settings, double noCFSRatio, ByteSizeValue maxCFSSize) {
+        MergePolicy mp = new MergePolicyConfig(logger, indexSettings(settings)).getMergePolicy();
+        assertThat(mp.getNoCFSRatio(), equalTo(noCFSRatio));
+        assertThat(mp.getMaxCFSSegmentSizeMB(), equalTo(maxCFSSize.getMbFrac()));
+    }
+
+    private static IndexSettings indexSettings(Settings settings) {
+        return new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY);
+    }
+
+    public void testNoMerges() {
+        MergePolicyConfig mp = new MergePolicyConfig(
+            logger,
+            indexSettings(Settings.builder().put(MergePolicyConfig.INDEX_MERGE_ENABLED, false).build())
+        );
+        assertTrue(mp.getMergePolicy() instanceof NoMergePolicy);
+    }
+
+    public void testUpdateSettings() throws IOException {
+        IndexSettings indexSettings = indexSettings(Settings.EMPTY);
+        assertThat(indexSettings.getMergePolicy().getNoCFSRatio(), equalTo(1.0));
+        assertThat(indexSettings.getMergePolicy().getMaxCFSSegmentSizeMB(), equalTo(1024d));
+        indexSettings = indexSettings(build(0.9));
+        assertThat((indexSettings.getMergePolicy()).getNoCFSRatio(), equalTo(0.9));
+        assertThat(indexSettings.getMergePolicy().getMaxCFSSegmentSizeMB(), equalTo(ByteSizeValue.ofBytes(Long.MAX_VALUE).getMbFrac()));
+        indexSettings.updateIndexMetadata(newIndexMeta("index", build(0.1)));
+        assertThat((indexSettings.getMergePolicy()).getNoCFSRatio(), equalTo(0.1));
+        assertThat(indexSettings.getMergePolicy().getMaxCFSSegmentSizeMB(), equalTo(ByteSizeValue.ofBytes(Long.MAX_VALUE).getMbFrac()));
+        indexSettings.updateIndexMetadata(newIndexMeta("index", build(0.0)));
+        assertThat((indexSettings.getMergePolicy()).getNoCFSRatio(), equalTo(0.0));
+        assertThat(indexSettings.getMergePolicy().getMaxCFSSegmentSizeMB(), equalTo(ByteSizeValue.ofBytes(Long.MAX_VALUE).getMbFrac()));
+        indexSettings.updateIndexMetadata(newIndexMeta("index", build("true")));
+        assertThat((indexSettings.getMergePolicy()).getNoCFSRatio(), equalTo(1.0));
+        assertThat(indexSettings.getMergePolicy().getMaxCFSSegmentSizeMB(), equalTo(ByteSizeValue.ofBytes(Long.MAX_VALUE).getMbFrac()));
+        indexSettings.updateIndexMetadata(newIndexMeta("index", build("false")));
+        assertThat((indexSettings.getMergePolicy()).getNoCFSRatio(), equalTo(0.0));
+        assertThat(indexSettings.getMergePolicy().getMaxCFSSegmentSizeMB(), equalTo(ByteSizeValue.ofBytes(Long.MAX_VALUE).getMbFrac()));
+        indexSettings.updateIndexMetadata(newIndexMeta("index", build("100mb")));
+        assertThat((indexSettings.getMergePolicy()).getNoCFSRatio(), equalTo(1.0));
+        assertThat(indexSettings.getMergePolicy().getMaxCFSSegmentSizeMB(), equalTo(100d));
+    }
+
+    public void testTieredMergePolicySettingsUpdate() throws IOException {
+        IndexSettings indexSettings = indexSettings(Settings.EMPTY);
+        assertEquals(
+            ((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(),
+            MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED,
+            0.0d
+        );
+
+        indexSettings.updateIndexMetadata(
+            newIndexMeta(
+                "index",
+                Settings.builder()
+                    .put(
+                        MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING.getKey(),
+                        MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d
+                    )
+                    .build()
+            )
+        );
+        assertEquals(
+            ((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(),
+            MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d,
+            0.0d
+        );
+
+        assertEquals(
+            ((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(),
+            MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMbFrac(),
+            0
+        );
+        indexSettings.updateIndexMetadata(
+            newIndexMeta(
+                "index",
+                Settings.builder()
+                    .put(
+                        MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING.getKey(),
+                        new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB)
+                    )
+                    .build()
+            )
+        );
+        assertEquals(
+            ((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(),
+            new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB).getMbFrac(),
+            0.001
+        );
+
+        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
+        indexSettings.updateIndexMetadata(
+            newIndexMeta(
+                "index",
+                Settings.builder()
+                    .put(
+                        MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(),
+                        MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1
+                    )
+                    .build()
+            )
+        );
+        assertEquals(
+            ((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(),
+            MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1
+        );
+
+        assertEquals(
+            ((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(),
+            MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getMbFrac(),
+            0.0001
+        );
+        indexSettings.updateIndexMetadata(
+            newIndexMeta(
+                "index",
+                Settings.builder()
+                    .put(
+                        MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING.getKey(),
+                        ByteSizeValue.ofBytes(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1)
+                    )
+                    .build()
+            )
+        );
+        assertEquals(
+            ((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(),
+            ByteSizeValue.ofBytes(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(),
+            0.0001
+        );
+
+        assertEquals(
+            ((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(),
+            MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER,
+            0
+        );
+        indexSettings.updateIndexMetadata(
+            newIndexMeta(
+                "index",
+                Settings.builder()
+                    .put(
+                        MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(),
+                        MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1
+                    )
+                    .build()
+            )
+        );
+        assertEquals(
+            ((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(),
+            MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1,
+            0
+        );
+
+        assertEquals(
+            ((TieredMergePolicy) indexSettings.getMergePolicy()).getDeletesPctAllowed(),
+            MergePolicyConfig.DEFAULT_DELETES_PCT_ALLOWED,
+            0
+        );
+        indexSettings.updateIndexMetadata(
+            newIndexMeta(
+                "index",
+                Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING.getKey(), 22).build()
+            )
+        );
+        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getDeletesPctAllowed(), 22, 0);
+
+        IllegalArgumentException exc = expectThrows(
+            IllegalArgumentException.class,
+            () -> indexSettings.updateIndexMetadata(
+                newIndexMeta(
+                    "index",
+                    Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING.getKey(), 53).build()
+                )
+            )
+        );
+        final Throwable cause = exc.getCause();
+        assertThat(cause.getMessage(), containsString("must be <= 50.0"));
+        indexSettings.updateIndexMetadata(newIndexMeta("index", Settings.EMPTY)); // see if defaults are restored
+        assertEquals(
+            ((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(),
+            MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED,
+            0.0d
+        );
+        assertEquals(
+            ((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(),
+            new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb(), ByteSizeUnit.MB).getMbFrac(),
+            0.00
+        );
+        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
+        assertEquals(
+            ((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(),
+            ByteSizeValue.ofBytes(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(),
+            0.0001
+        );
+        assertEquals(
+            ((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(),
+            MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER,
+            0
+        );
+        assertEquals(
+            ((TieredMergePolicy) indexSettings.getMergePolicy()).getDeletesPctAllowed(),
+            MergePolicyConfig.DEFAULT_DELETES_PCT_ALLOWED,
+            0
+        );
+    }
+
+    public Settings build(String value) {
+        return Settings.builder().put(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING.getKey(), value).build();
+    }
+
+    public Settings build(double value) {
+        return Settings.builder().put(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING.getKey(), value).build();
+    }
+
+    public Settings build(int value) {
+        return Settings.builder().put(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING.getKey(), value).build();
+    }
+
+    public Settings build(boolean value) {
+        return Settings.builder().put(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING.getKey(), value).build();
+    }
+
+    private Settings build(ByteSizeValue value) {
+        return Settings.builder().put(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING.getKey(), value).build();
+    }
+
+    public void testCompoundFileConfiguredByByteSize() throws IOException {
+        try (Directory dir = newDirectory()) {
+            // index.compound_format: 1gb, the merge will use a compound file
+            MergePolicy mp = new MergePolicyConfig(logger, indexSettings(Settings.EMPTY)).getMergePolicy();
+            try (IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null).setMergePolicy(mp))) {
+                w.addDocument(new Document());
+                w.flush();
+                w.addDocument(new Document());
+                w.forceMerge(1);
+            }
+            try (DirectoryReader reader = DirectoryReader.open(dir)) {
+                LeafReader leaf = getOnlyLeafReader(reader);
+                SegmentCommitInfo sci = ((SegmentReader) leaf).getSegmentInfo();
+                assertEquals(IndexWriter.SOURCE_MERGE, sci.info.getDiagnostics().get(IndexWriter.SOURCE));
+                assertTrue(sci.info.getUseCompoundFile());
+            }
+        }
+
+        // index.compound_format: 1b, the merge will not use a compound file
+        try (Directory dir = newDirectory()) {
+            MergePolicy mp = new MergePolicyConfig(logger, indexSettings(build("1b"))).getMergePolicy();
+            try (IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null).setMergePolicy(mp))) {
+                w.addDocument(new Document());
+                w.flush();
+                w.addDocument(new Document());
+                w.forceMerge(1);
+            }
+            try (DirectoryReader reader = DirectoryReader.open(dir)) {
+                LeafReader leaf = getOnlyLeafReader(reader);
+                SegmentCommitInfo sci = ((SegmentReader) leaf).getSegmentInfo();
+                assertEquals(IndexWriter.SOURCE_MERGE, sci.info.getDiagnostics().get(IndexWriter.SOURCE));
+                assertFalse(sci.info.getUseCompoundFile());
+            }
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java b/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java
deleted file mode 100644
index d693048e8e9d..000000000000
--- a/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.index;
-
-import org.apache.lucene.index.NoMergePolicy;
-import org.apache.lucene.index.TieredMergePolicy;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.test.ESTestCase;
-
-import java.io.IOException;
-
-import static org.elasticsearch.index.IndexSettingsTests.newIndexMeta;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-
-public class MergePolicySettingsTests extends ESTestCase {
-    protected final ShardId shardId = new ShardId("index", "_na_", 1);
-
-    public void testCompoundFileSettings() throws IOException {
-        assertThat(new MergePolicyConfig(logger, indexSettings(Settings.EMPTY)).getMergePolicy().getNoCFSRatio(), equalTo(0.1));
-        assertThat(new MergePolicyConfig(logger, indexSettings(build(true))).getMergePolicy().getNoCFSRatio(), equalTo(1.0));
-        assertThat(new MergePolicyConfig(logger, indexSettings(build(0.5))).getMergePolicy().getNoCFSRatio(), equalTo(0.5));
-        assertThat(new MergePolicyConfig(logger, indexSettings(build(1.0))).getMergePolicy().getNoCFSRatio(), equalTo(1.0));
-        assertThat(new MergePolicyConfig(logger, indexSettings(build("true"))).getMergePolicy().getNoCFSRatio(), equalTo(1.0));
-        assertThat(new MergePolicyConfig(logger, indexSettings(build("True"))).getMergePolicy().getNoCFSRatio(), equalTo(1.0));
-        assertThat(new MergePolicyConfig(logger, indexSettings(build("False"))).getMergePolicy().getNoCFSRatio(), equalTo(0.0));
-        assertThat(new MergePolicyConfig(logger, indexSettings(build("false"))).getMergePolicy().getNoCFSRatio(), equalTo(0.0));
-        assertThat(new MergePolicyConfig(logger, indexSettings(build(false))).getMergePolicy().getNoCFSRatio(), equalTo(0.0));
-        assertThat(new MergePolicyConfig(logger, indexSettings(build(0))).getMergePolicy().getNoCFSRatio(), equalTo(0.0));
-        assertThat(new MergePolicyConfig(logger, indexSettings(build(0.0))).getMergePolicy().getNoCFSRatio(), equalTo(0.0));
-    }
-
-    private static IndexSettings indexSettings(Settings settings) {
-        return new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY);
-    }
-
-    public void testNoMerges() {
-        MergePolicyConfig mp = new MergePolicyConfig(
-            logger,
-            indexSettings(Settings.builder().put(MergePolicyConfig.INDEX_MERGE_ENABLED, false).build())
-        );
-        assertTrue(mp.getMergePolicy() instanceof NoMergePolicy);
-    }
-
-    public void testUpdateSettings() throws IOException {
-        IndexSettings indexSettings = indexSettings(Settings.EMPTY);
-        assertThat(indexSettings.getMergePolicy().getNoCFSRatio(), equalTo(0.1));
-        indexSettings = indexSettings(build(0.9));
-        assertThat((indexSettings.getMergePolicy()).getNoCFSRatio(), equalTo(0.9));
-        indexSettings.updateIndexMetadata(newIndexMeta("index", build(0.1)));
-        assertThat((indexSettings.getMergePolicy()).getNoCFSRatio(), equalTo(0.1));
-        indexSettings.updateIndexMetadata(newIndexMeta("index", build(0.0)));
-        assertThat((indexSettings.getMergePolicy()).getNoCFSRatio(), equalTo(0.0));
-        indexSettings.updateIndexMetadata(newIndexMeta("index", build("true")));
-        assertThat((indexSettings.getMergePolicy()).getNoCFSRatio(), equalTo(1.0));
-        indexSettings.updateIndexMetadata(newIndexMeta("index", build("false")));
-        assertThat((indexSettings.getMergePolicy()).getNoCFSRatio(), equalTo(0.0));
-    }
-
-    public void testTieredMergePolicySettingsUpdate() throws IOException {
-        IndexSettings indexSettings = indexSettings(Settings.EMPTY);
-        assertEquals(
-            ((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(),
-            MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED,
-            0.0d
-        );
-
-        indexSettings.updateIndexMetadata(
-            newIndexMeta(
-                "index",
-                Settings.builder()
-                    .put(
-                        MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING.getKey(),
-                        MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d
-                    )
-                    .build()
-            )
-        );
-        assertEquals(
-            ((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(),
-            MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d,
-            0.0d
-        );
-
-        assertEquals(
-            ((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(),
-            MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMbFrac(),
-            0
-        );
-        indexSettings.updateIndexMetadata(
-            newIndexMeta(
-                "index",
-                Settings.builder()
-                    .put(
-                        MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING.getKey(),
-                        new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB)
-                    )
-                    .build()
-            )
-        );
-        assertEquals(
-            ((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(),
-            new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB).getMbFrac(),
-            0.001
-        );
-
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
-        indexSettings.updateIndexMetadata(
-            newIndexMeta(
-                "index",
-                Settings.builder()
-                    .put(
-                        MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(),
-                        MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1
-                    )
-                    .build()
-            )
-        );
-        assertEquals(
-            ((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(),
-            MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1
-        );
-
-        assertEquals(
-            ((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(),
-            MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getMbFrac(),
-            0.0001
-        );
-        indexSettings.updateIndexMetadata(
-            newIndexMeta(
-                "index",
-                Settings.builder()
-                    .put(
-                        MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING.getKey(),
-                        ByteSizeValue.ofBytes(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1)
-                    )
-                    .build()
-            )
-        );
-        assertEquals(
-            ((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(),
-            ByteSizeValue.ofBytes(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(),
-            0.0001
-        );
-
-        assertEquals(
-            ((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(),
-            MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER,
-            0
-        );
-        indexSettings.updateIndexMetadata(
-            newIndexMeta(
-                "index",
-                Settings.builder()
-                    .put(
-                        MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(),
-                        MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1
-                    )
-                    .build()
-            )
-        );
-        assertEquals(
-            ((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(),
-            MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1,
-            0
-        );
-
-        assertEquals(
-            ((TieredMergePolicy) indexSettings.getMergePolicy()).getDeletesPctAllowed(),
-            MergePolicyConfig.DEFAULT_DELETES_PCT_ALLOWED,
-            0
-        );
-        indexSettings.updateIndexMetadata(
-            newIndexMeta(
-                "index",
-                Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING.getKey(), 22).build()
-            )
-        );
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getDeletesPctAllowed(), 22, 0);
-
-        IllegalArgumentException exc = expectThrows(
-            IllegalArgumentException.class,
-            () -> indexSettings.updateIndexMetadata(
-                newIndexMeta(
-                    "index",
-                    Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING.getKey(), 53).build()
-                )
-            )
-        );
-        final Throwable cause = exc.getCause();
-        assertThat(cause.getMessage(), containsString("must be <= 50.0"));
-        indexSettings.updateIndexMetadata(newIndexMeta("index", Settings.EMPTY)); // see if defaults are restored
-        assertEquals(
-            ((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(),
-            MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED,
-            0.0d
-        );
-        assertEquals(
-            ((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(),
-            new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb(), ByteSizeUnit.MB).getMbFrac(),
-            0.00
-        );
-        assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
-        assertEquals(
-            ((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(),
-            ByteSizeValue.ofBytes(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(),
-            0.0001
-        );
-        assertEquals(
-            ((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(),
-            MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER,
-            0
-        );
-        assertEquals(
-            ((TieredMergePolicy) indexSettings.getMergePolicy()).getDeletesPctAllowed(),
-            MergePolicyConfig.DEFAULT_DELETES_PCT_ALLOWED,
-            0
-        );
-    }
-
-    public Settings build(String value) {
-        return Settings.builder().put(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING.getKey(), value).build();
-    }
-
-    public Settings build(double value) {
-        return Settings.builder().put(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING.getKey(), value).build();
-    }
-
-    public Settings build(int value) {
-        return Settings.builder().put(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING.getKey(), value).build();
-    }
-
-    public Settings build(boolean value) {
-        return Settings.builder().put(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING.getKey(), value).build();
-    }
-
-}
diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java
index 2b6f29d2e79f..3377aca2c048 100644
--- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.index.mapper.MapperServiceTestCase;
+import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.StringFieldScript;
@@ -264,7 +265,12 @@ protected <T> T compileScript(Script script, ScriptContext<T> context) {
         if (context.equals(StringFieldScript.CONTEXT) && script.getLang().equals("mock")) {
             return (T) new StringFieldScript.Factory() {
                 @Override
-                public LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup) {
+                public LeafFactory newFactory(
+                    String fieldName,
+                    Map<String, Object> params,
+                    SearchLookup searchLookup,
+                    OnScriptError onScriptError
+                ) {
                     throw new UnsupportedOperationException("error should be thrown before getting here");
                 }
             };
diff --git a/server/src/test/java/org/elasticsearch/index/cache/bitset/BitSetFilterCacheTests.java b/server/src/test/java/org/elasticsearch/index/cache/bitset/BitSetFilterCacheTests.java
index 3428c8d3653b..1c164e898426 100644
--- a/server/src/test/java/org/elasticsearch/index/cache/bitset/BitSetFilterCacheTests.java
+++ b/server/src/test/java/org/elasticsearch/index/cache/bitset/BitSetFilterCacheTests.java
@@ -11,6 +11,7 @@
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
@@ -20,6 +21,7 @@
 import org.apache.lucene.index.LogByteSizeMergePolicy;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.join.BitSetProducer;
 import org.apache.lucene.store.ByteBuffersDirectory;
@@ -31,6 +33,7 @@
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.lucene.util.MatchAllBitSet;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 
@@ -39,6 +42,10 @@
 import java.util.concurrent.atomic.AtomicLong;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.lessThan;
 
 public class BitSetFilterCacheTests extends ESTestCase {
 
@@ -168,6 +175,48 @@ public void onRemoval(ShardId shardId, Accountable accountable) {
         assertEquals(0, stats.get());
     }
 
+    public void testStats() throws IOException {
+        Directory directory = newDirectory();
+        IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig());
+        int numDocs = randomIntBetween(2000, 5000);
+        for (int i = 0; i < numDocs; i++) {
+            Document d = new Document();
+            d.add(new LongPoint("f", i));
+            writer.addDocument(d);
+        }
+        writer.commit();
+        writer.forceMerge(1);
+        IndexReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("test", "_na_", 0));
+        assertThat(reader.leaves(), hasSize(1));
+        assertThat(reader.numDocs(), equalTo(numDocs));
+
+        final AtomicLong stats = new AtomicLong();
+        final BitsetFilterCache cache = new BitsetFilterCache(INDEX_SETTINGS, new BitsetFilterCache.Listener() {
+            @Override
+            public void onCache(ShardId shardId, Accountable accountable) {
+                stats.addAndGet(accountable.ramBytesUsed());
+            }
+
+            @Override
+            public void onRemoval(ShardId shardId, Accountable accountable) {
+                stats.addAndGet(-accountable.ramBytesUsed());
+            }
+        });
+        // match all
+        Query matchAll = randomBoolean() ? LongPoint.newRangeQuery("f", 0, numDocs + between(0, 1000)) : new MatchAllDocsQuery();
+        BitSetProducer bitSetProducer = cache.getBitSetProducer(matchAll);
+        BitSet bitset = bitSetProducer.getBitSet(reader.leaves().get(0));
+        assertThat(bitset, instanceOf(MatchAllBitSet.class));
+        long usedBytes = stats.get();
+        assertThat(usedBytes, lessThan(32L));
+        // range
+        bitSetProducer = cache.getBitSetProducer(LongPoint.newRangeQuery("f", 0, between(1000, 2000)));
+        bitSetProducer.getBitSet(reader.leaves().get(0));
+        usedBytes = stats.get() - usedBytes;
+        assertThat(usedBytes, greaterThan(256L));
+        IOUtils.close(cache, reader, writer, directory);
+    }
+
     public void testSetNullListener() {
         try {
             new BitsetFilterCache(INDEX_SETTINGS, null);
diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java
index 9d8ee36d3dd7..df39f06c88da 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
@@ -42,21 +42,21 @@ public class CodecTests extends ESTestCase {
     public void testResolveDefaultCodecs() throws Exception {
         CodecService codecService = createCodecService();
         assertThat(codecService.codec("default"), instanceOf(PerFieldMapperCodec.class));
-        assertThat(codecService.codec("default"), instanceOf(Lucene94Codec.class));
+        assertThat(codecService.codec("default"), instanceOf(Lucene95Codec.class));
     }
 
     public void testDefault() throws Exception {
         Codec codec = createCodecService().codec("default");
-        assertStoredFieldsCompressionEquals(Lucene94Codec.Mode.BEST_SPEED, codec);
+        assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_SPEED, codec);
     }
 
     public void testBestCompression() throws Exception {
         Codec codec = createCodecService().codec("best_compression");
-        assertStoredFieldsCompressionEquals(Lucene94Codec.Mode.BEST_COMPRESSION, codec);
+        assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_COMPRESSION, codec);
     }
 
     // write some docs with it, inspect .si to see this was the used compression
-    private void assertStoredFieldsCompressionEquals(Lucene94Codec.Mode expected, Codec actual) throws Exception {
+    private void assertStoredFieldsCompressionEquals(Lucene95Codec.Mode expected, Codec actual) throws Exception {
         Directory dir = newDirectory();
         IndexWriterConfig iwc = newIndexWriterConfig(null);
         iwc.setCodec(actual);
@@ -68,7 +68,7 @@ private void assertStoredFieldsCompressionEquals(Lucene94Codec.Mode expected, Co
         SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader();
         String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY);
         assertNotNull(v);
-        assertEquals(expected, Lucene94Codec.Mode.valueOf(v));
+        assertEquals(expected, Lucene95Codec.Mode.valueOf(v));
         ir.close();
         dir.close();
     }
diff --git a/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java
new file mode 100644
index 000000000000..42299c6c6baf
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.codec;
+
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.MapperTestUtils;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.is;
+
+public class PerFieldMapperCodecTests extends ESTestCase {
+
+    public void testUseBloomFilter() throws IOException {
+        PerFieldMapperCodec perFieldMapperCodec = createCodec(false, randomBoolean(), false);
+        assertThat(perFieldMapperCodec.useBloomFilter("_id"), is(true));
+        assertThat(perFieldMapperCodec.useBloomFilter("another_field"), is(false));
+    }
+
+    public void testUseBloomFilterWithTimestampFieldEnabled() throws IOException {
+        PerFieldMapperCodec perFieldMapperCodec = createCodec(true, true, false);
+        assertThat(perFieldMapperCodec.useBloomFilter("_id"), is(true));
+        assertThat(perFieldMapperCodec.useBloomFilter("another_field"), is(false));
+    }
+
+    public void testUseBloomFilterWithTimestampFieldEnabled_noTimeSeriesMode() throws IOException {
+        PerFieldMapperCodec perFieldMapperCodec = createCodec(true, false, false);
+        assertThat(perFieldMapperCodec.useBloomFilter("_id"), is(false));
+    }
+
+    public void testUseBloomFilterWithTimestampFieldEnabled_disableBloomFilter() throws IOException {
+        PerFieldMapperCodec perFieldMapperCodec = createCodec(true, true, true);
+        assertThat(perFieldMapperCodec.useBloomFilter("_id"), is(false));
+        assertWarnings(
+            "[index.bloom_filter_for_id_field.enabled] setting was deprecated in Elasticsearch and will be removed in a future release."
+        );
+    }
+
+    private PerFieldMapperCodec createCodec(boolean timestampField, boolean timeSeries, boolean disableBloomFilter) throws IOException {
+        Settings.Builder settings = Settings.builder();
+        if (timeSeries) {
+            settings.put(IndexSettings.MODE.getKey(), "time_series");
+            settings.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field");
+        }
+        if (disableBloomFilter) {
+            settings.put(IndexSettings.BLOOM_FILTER_ID_FIELD_ENABLED_SETTING.getKey(), false);
+        }
+        MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), settings.build(), "test");
+        if (timestampField) {
+            String mapping = """
+                {
+                    "_data_stream_timestamp": {
+                        "enabled": true
+                    },
+                    "properties": {
+                        "@timestamp": {
+                            "type": "date"
+                        }
+                    }
+                }
+                """;
+            mapperService.merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
+        }
+        return new PerFieldMapperCodec(Lucene95Codec.Mode.BEST_SPEED, mapperService, BigArrays.NON_RECYCLING_INSTANCE);
+    }
+
+}
diff --git a/server/src/test/java/org/elasticsearch/index/codec/bloomfilter/ES85BloomFilterPostingsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/bloomfilter/ES85BloomFilterPostingsFormatTests.java
index 6c3af1880074..a209a9e336e6 100644
--- a/server/src/test/java/org/elasticsearch/index/codec/bloomfilter/ES85BloomFilterPostingsFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/index/codec/bloomfilter/ES85BloomFilterPostingsFormatTests.java
@@ -34,8 +34,8 @@ public class ES85BloomFilterPostingsFormatTests extends BasePostingsFormatTestCa
 
     @Override
     protected Codec getCodec() {
-        return TestUtil.alwaysPostingsFormat(new ES85BloomFilterPostingsFormat(BigArrays.NON_RECYCLING_INSTANCE, field -> {
-            PostingsFormat postingsFormat = Codec.getDefault().postingsFormat();
+        return TestUtil.alwaysPostingsFormat(new ES85BloomFilterRWPostingsFormat(BigArrays.NON_RECYCLING_INSTANCE, field -> {
+            PostingsFormat postingsFormat = TestUtil.getDefaultPostingsFormat();
             if (postingsFormat instanceof PerFieldPostingsFormat) {
                 postingsFormat = TestUtil.getDefaultPostingsFormat();
             }
diff --git a/server/src/test/java/org/elasticsearch/index/codec/bloomfilter/ES85BloomFilterRWPostingsFormat.java b/server/src/test/java/org/elasticsearch/index/codec/bloomfilter/ES85BloomFilterRWPostingsFormat.java
new file mode 100644
index 000000000000..d6c803183da1
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/codec/bloomfilter/ES85BloomFilterRWPostingsFormat.java
@@ -0,0 +1,170 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.codec.bloomfilter;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.FieldsConsumer;
+import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.FilterLeafReader;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.lucene.store.IndexOutputOutputStream;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.ByteArray;
+import org.elasticsearch.core.IOUtils;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+
+final class ES85BloomFilterRWPostingsFormat extends ES85BloomFilterPostingsFormat {
+
+    private final Function<String, PostingsFormat> postingsFormats;
+    private final BigArrays bigArrays;
+
+    ES85BloomFilterRWPostingsFormat(BigArrays bigArrays, Function<String, PostingsFormat> postingsFormats) {
+        super();
+        this.bigArrays = Objects.requireNonNull(bigArrays);
+        this.postingsFormats = Objects.requireNonNull(postingsFormats);
+    }
+
+    @Override
+    public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+        if (postingsFormats == null || bigArrays == null) {
+            assert false : BLOOM_CODEC_NAME + " was initialized with a wrong constructor";
+            throw new UnsupportedOperationException(BLOOM_CODEC_NAME + " was initialized with a wrong constructor");
+        }
+        return new FieldsWriter(state);
+    }
+
+    final class FieldsWriter extends FieldsConsumer {
+        private final SegmentWriteState state;
+        private final IndexOutput indexOut;
+        private final List<BloomFilter> bloomFilters = new ArrayList<>();
+        private final List<FieldsGroup> fieldsGroups = new ArrayList<>();
+        private final List<Closeable> toCloses = new ArrayList<>();
+        private boolean closed;
+
+        FieldsWriter(SegmentWriteState state) throws IOException {
+            this.state = state;
+            boolean success = false;
+            try {
+                indexOut = state.directory.createOutput(indexFile(state.segmentInfo, state.segmentSuffix), state.context);
+                toCloses.add(indexOut);
+                CodecUtil.writeIndexHeader(indexOut, BLOOM_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
+                success = true;
+            } finally {
+                if (success == false) {
+                    IOUtils.closeWhileHandlingException(toCloses);
+                }
+            }
+        }
+
+        @Override
+        public void write(Fields fields, NormsProducer norms) throws IOException {
+            writePostings(fields, norms);
+            writeBloomFilters(fields);
+        }
+
+        private void writePostings(Fields fields, NormsProducer norms) throws IOException {
+            final Map<PostingsFormat, FieldsGroup> currentGroups = new HashMap<>();
+            for (String field : fields) {
+                final PostingsFormat postingsFormat = postingsFormats.apply(field);
+                if (postingsFormat == null) {
+                    throw new IllegalStateException("PostingsFormat for field [" + field + "] wasn't specified");
+                }
+                FieldsGroup group = currentGroups.get(postingsFormat);
+                if (group == null) {
+                    group = new FieldsGroup(postingsFormat, Integer.toString(fieldsGroups.size()), new ArrayList<>());
+                    currentGroups.put(postingsFormat, group);
+                    fieldsGroups.add(group);
+                }
+                group.fields().add(field);
+            }
+            for (FieldsGroup group : currentGroups.values()) {
+                final FieldsConsumer writer = group.postingsFormat().fieldsConsumer(new SegmentWriteState(state, group.suffix()));
+                toCloses.add(writer);
+                final Fields maskedFields = new FilterLeafReader.FilterFields(fields) {
+                    @Override
+                    public Iterator<String> iterator() {
+                        return group.fields().iterator();
+                    }
+                };
+                writer.write(maskedFields, norms);
+            }
+        }
+
+        private void writeBloomFilters(Fields fields) throws IOException {
+            for (String field : fields) {
+                final Terms terms = fields.terms(field);
+                if (terms == null) {
+                    continue;
+                }
+                final int bloomFilterSize = bloomFilterSize(state.segmentInfo.maxDoc());
+                final int numBytes = numBytesForBloomFilter(bloomFilterSize);
+                try (ByteArray buffer = bigArrays.newByteArray(numBytes)) {
+                    final TermsEnum termsEnum = terms.iterator();
+                    while (true) {
+                        final BytesRef term = termsEnum.next();
+                        if (term == null) {
+                            break;
+                        }
+                        final int hash = hashTerm(term) % bloomFilterSize;
+                        final int pos = hash >> 3;
+                        final int mask = 1 << (hash & 0x7);
+                        final byte val = (byte) (buffer.get(pos) | mask);
+                        buffer.set(pos, val);
+                    }
+                    bloomFilters.add(new BloomFilter(field, indexOut.getFilePointer(), bloomFilterSize));
+                    final BytesReference bytes = BytesReference.fromByteArray(buffer, numBytes);
+                    bytes.writeTo(new IndexOutputOutputStream(indexOut));
+                }
+            }
+        }
+
+        @Override
+        public void close() throws IOException {
+            if (closed) {
+                return;
+            }
+            closed = true;
+            try {
+                CodecUtil.writeFooter(indexOut);
+            } finally {
+                IOUtils.close(toCloses);
+            }
+            try (IndexOutput metaOut = state.directory.createOutput(metaFile(state.segmentInfo, state.segmentSuffix), state.context)) {
+                CodecUtil.writeIndexHeader(metaOut, BLOOM_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
+                // write postings formats
+                metaOut.writeVInt(fieldsGroups.size());
+                for (FieldsGroup group : fieldsGroups) {
+                    group.writeTo(metaOut, state.fieldInfos);
+                }
+                // Write bloom filters
+                metaOut.writeVInt(bloomFilters.size());
+                for (BloomFilter bloomFilter : bloomFilters) {
+                    bloomFilter.writeTo(metaOut, state.fieldInfos);
+                }
+                CodecUtil.writeFooter(metaOut);
+            }
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/codec/bloomfilter/ES87BloomFilterPostingsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/bloomfilter/ES87BloomFilterPostingsFormatTests.java
new file mode 100644
index 000000000000..f88f53d4be90
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/codec/bloomfilter/ES87BloomFilterPostingsFormatTests.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.codec.bloomfilter;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.tests.index.BasePostingsFormatTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.GraalVMThreadsFilter;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat.hashTerm;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+
+@ThreadLeakFilters(filters = { GraalVMThreadsFilter.class })
+public class ES87BloomFilterPostingsFormatTests extends BasePostingsFormatTestCase {
+
+    @Override
+    protected Codec getCodec() {
+        return TestUtil.alwaysPostingsFormat(new ES87BloomFilterPostingsFormat(BigArrays.NON_RECYCLING_INSTANCE, field -> {
+            PostingsFormat postingsFormat = TestUtil.getDefaultPostingsFormat();
+            if (postingsFormat instanceof PerFieldPostingsFormat) {
+                postingsFormat = TestUtil.getDefaultPostingsFormat();
+            }
+            return postingsFormat;
+        }));
+    }
+
+    public void testBloomFilterSize() {
+        assertThat(ES87BloomFilterPostingsFormat.bloomFilterSize(1000), equalTo(10_000));
+        assertThat(ES87BloomFilterPostingsFormat.bloomFilterSize(999), equalTo(9992)); // rounded to next multiple of 8
+        assertThat(ES87BloomFilterPostingsFormat.bloomFilterSize(1001), equalTo(10_016)); // rounded to next multiple of 8
+        assertThat(ES87BloomFilterPostingsFormat.bloomFilterSize(IndexWriter.MAX_DOCS - random().nextInt(10)), equalTo(Integer.MAX_VALUE));
+        assertThat(ES87BloomFilterPostingsFormat.numBytesForBloomFilter(16384), equalTo(2048));
+        assertThat(ES87BloomFilterPostingsFormat.numBytesForBloomFilter(16383), equalTo(2048));
+        assertThat(ES87BloomFilterPostingsFormat.numBytesForBloomFilter(Integer.MAX_VALUE), equalTo(1 << 28));
+    }
+
+    public void testHashTermsV2() {
+        // The following tests are "intentionally brittle" - the implementation of the hash function is relevant for backward-compatibility,
+        // therefore these tests test the *internals* of the hash function and not only the external interface.
+        Map<String, List<Integer>> testStrings = Map.of(
+            "elastic",
+            List.of(924743812, 1179558664, 1689188368, 51334424, 1070593832, 1580223536, 451999296),
+            "bloom_filter",
+            List.of(1660214456, 1257667067, 452572289, 1794961159, 184771603, 1527160473, 2064454565)
+        );
+
+        for (Map.Entry<String, List<Integer>> e : testStrings.entrySet()) {
+            String term = e.getKey();
+            int[] hashes = new int[7];
+            final BytesRef bytesRef = randomBytesRef(term.getBytes(StandardCharsets.UTF_8));
+            hashTerm(bytesRef, hashes);
+            for (int i = 0; i < 7; ++i) {
+                assertThat("term=" + term, hashes[i], equalTo(e.getValue().get(i)));
+            }
+        }
+
+        byte[] bytes = ESTestCase.randomByteArrayOfLength(ESTestCase.between(0, 1000));
+        int[] hashes = { 0, 0, 0, 0, 0, 0, 0 };
+        hashTerm(randomBytesRef(bytes), hashes);
+        for (int hash : hashes) {
+            assertThat(hash, greaterThanOrEqualTo(0));
+        }
+    }
+
+    private static BytesRef randomBytesRef(byte[] bytes) {
+        if (random().nextBoolean()) {
+            final BytesRefBuilder builder = new BytesRefBuilder();
+            // prefix
+            int offset = ESTestCase.randomIntBetween(0, 10);
+            builder.append(new BytesRef(ESTestCase.randomByteArrayOfLength(offset)));
+            // term
+            builder.append(bytes, 0, bytes.length);
+            // suffix
+            int suffixLength = ESTestCase.between(0, 10);
+            builder.append(new BytesRef(ESTestCase.randomByteArrayOfLength(suffixLength)));
+            return new BytesRef(builder.bytes(), offset, bytes.length);
+        } else {
+            return new BytesRef(bytes);
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtilTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtilTests.java
new file mode 100644
index 000000000000..78ee03882f6e
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtilTests.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.codec.tsdb;
+
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
+
+import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.store.ByteArrayDataOutput;
+import org.apache.lucene.store.ByteBuffersDirectory;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.packed.PackedInts;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Random;
+
+public class DocValuesForUtilTests extends LuceneTestCase {
+    private final ES87TSDBDocValuesEncoder encoder = new ES87TSDBDocValuesEncoder();
+    private static final ForUtil forUtil = new ForUtil();
+
+    public void testEncodeDecode() throws IOException {
+        final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);
+        final long[] values = new long[iterations * encoder.getBlockSize()];
+        final int[] bpvs = new int[iterations];
+
+        for (int i = 0; i < iterations; ++i) {
+            final int bpv = TestUtil.nextInt(random(), 1, 64);
+            bpvs[i] = bpv;
+            for (int j = 0; j < encoder.getBlockSize(); ++j) {
+                values[i * encoder.getBlockSize() + j] = bpv == 64
+                    ? random().nextLong()
+                    : TestUtil.nextLong(random(), 0, PackedInts.maxValue(bpv));
+            }
+        }
+
+        final Directory d = new ByteBuffersDirectory();
+        final long endPointer;
+
+        {
+            // encode
+            IndexOutput out = d.createOutput("test.bin", IOContext.DEFAULT);
+            final DocValuesForUtil forUtil = new DocValuesForUtil(encoder.getBlockSize());
+
+            for (int i = 0; i < iterations; ++i) {
+                long[] source = new long[encoder.getBlockSize()];
+                for (int j = 0; j < encoder.getBlockSize(); ++j) {
+                    source[j] = values[i * encoder.getBlockSize() + j];
+                }
+                out.writeByte((byte) bpvs[i]);
+                forUtil.encode(source, bpvs[i], out);
+            }
+            endPointer = out.getFilePointer();
+            out.close();
+        }
+
+        {
+            // decode
+            IndexInput in = d.openInput("test.bin", IOContext.READONCE);
+            final DocValuesForUtil forUtil = new DocValuesForUtil(encoder.getBlockSize());
+            final long[] restored = new long[encoder.getBlockSize()];
+            for (int i = 0; i < iterations; ++i) {
+                final int bitsPerValue = in.readByte();
+                forUtil.decode(bitsPerValue, in, restored);
+                assertArrayEquals(
+                    Arrays.toString(restored),
+                    ArrayUtil.copyOfSubArray(values, i * encoder.getBlockSize(), (i + 1) * encoder.getBlockSize()),
+                    restored
+                );
+            }
+            assertEquals(endPointer, in.getFilePointer());
+            in.close();
+        }
+
+        d.close();
+    }
+
+    public void testEncodeDecodeBitsPerValue() throws IOException {
+        int size = 128;
+        byte[] dataOutputBuffer = new byte[16 * 1024];
+        byte[] dataInputBuffer = new byte[16 * 1024];
+        long[] decodeBuffer = new long[size];
+
+        for (int bitsPerValue = 1; bitsPerValue < 32; bitsPerValue++) {
+            long[] values = randomValues(bitsPerValue, size);
+
+            // Encode
+            DataOutput dataOutput = new ByteArrayDataOutput(dataOutputBuffer);
+            long[] encodeBuffer = Arrays.copyOf(values, values.length);
+            forUtil.encode(encodeBuffer, bitsPerValue, dataOutput);
+
+            // Prepare for decoding
+            DataInput dataInput = new ByteArrayDataInput(dataInputBuffer);
+            System.arraycopy(dataOutputBuffer, 0, dataInputBuffer, 0, dataOutputBuffer.length);
+
+            // Decode
+            forUtil.decode(bitsPerValue, dataInput, decodeBuffer);
+
+            assertArrayEquals(decodeBuffer, values);
+        }
+    }
+
+    private long[] randomValues(int bitsPerValue, int size) {
+        final Random random = new Random(0);
+        long[] values = new long[size];
+        long upperBound = 1L << bitsPerValue;
+        for (int i = 0; i < size; i++) {
+            values[i] = random.nextLong(upperBound);
+        }
+        return values;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesEncoderTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesEncoderTests.java
new file mode 100644
index 000000000000..c52f0334cafd
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesEncoderTests.java
@@ -0,0 +1,156 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.codec.tsdb;
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.util.NumericUtils;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+public class ES87TSDBDocValuesEncoderTests extends LuceneTestCase {
+
+    private final ES87TSDBDocValuesEncoder encoder;
+    private final int blockSize = ES87TSDBDocValuesFormat.DEFAULT_NUMERIC_BLOCK_SIZE;
+
+    public ES87TSDBDocValuesEncoderTests() {
+        this.encoder = new ES87TSDBDocValuesEncoder(blockSize);
+    }
+
+    public void testRandomValues() throws IOException {
+        long[] arr = new long[blockSize];
+        for (int i = 0; i < blockSize; ++i) {
+            arr[i] = random().nextLong();
+        }
+        arr[4] = Long.MAX_VALUE / 2;
+        arr[100] = Long.MIN_VALUE / 2 - 1;
+        final long expectedNumBytes = 2 // token
+            + (blockSize * 64) / Byte.SIZE; // data
+        doTest(arr, expectedNumBytes);
+    }
+
+    public void testAllEqual() throws IOException {
+        long[] arr = new long[blockSize];
+        Arrays.fill(arr, 3);
+        final long expectedNumBytes = 2; // token + min value
+        doTest(arr, expectedNumBytes);
+    }
+
+    public void testSmallPositiveValues() throws IOException {
+        long[] arr = new long[blockSize];
+        for (int i = 0; i < blockSize; ++i) {
+            arr[i] = (i + 2) & 0x03; // 2 bits per value
+        }
+        final long expectedNumBytes = 1 // token
+            + (blockSize * 2) / Byte.SIZE; // data
+        doTest(arr, expectedNumBytes);
+    }
+
+    public void testSmallPositiveValuesWithOffset() throws IOException {
+        long[] arr = new long[blockSize];
+        for (int i = 0; i < blockSize; ++i) {
+            arr[i] = 1000 + ((i + 2) & 0x03); // 2 bits per value
+        }
+        final long expectedNumBytes = 3 // token + min value (1000 -> 2 bytes)
+            + (blockSize * 2) / Byte.SIZE; // data
+        doTest(arr, expectedNumBytes);
+    }
+
+    public void testSmallPositiveValuesWithNegativeOffset() throws IOException {
+        long[] arr = new long[blockSize];
+        for (int i = 0; i < blockSize; ++i) {
+            arr[i] = -1000 + ((i + 2) & 0x03); // 2 bits per value
+        }
+        final long expectedNumBytes = 3 // token + min value (-1000 -> 2 bytes)
+            + (blockSize * 2) / Byte.SIZE; // data
+        doTest(arr, expectedNumBytes);
+    }
+
+    /**
+     * Integers as doubles, GCD compression should help here given high numbers of trailing zeroes.
+     */
+    public void testIntegersAsDoubles() throws IOException {
+        long[] arr = new long[blockSize];
+        for (int i = 0; i < blockSize; ++i) {
+            arr[i] = NumericUtils.doubleToSortableLong((i + 2) & 0x03); // 0, 1 or 2
+        }
+        final long expectedNumBytes = 9 // token + GCD (8 bytes)
+            + blockSize * 12 / Byte.SIZE; // 12 bits per value -> 26 longs
+        doTest(arr, expectedNumBytes);
+    }
+
+    public void testDecreasingValues() throws IOException {
+        long[] arr = new long[blockSize];
+        for (int i = 0; i < blockSize; ++i) {
+            arr[i] = 1000 - 30 * i;
+        }
+        final long expectedNumBytes = 4; // token + min value + delta
+        doTest(arr, expectedNumBytes);
+    }
+
+    public void testTwoValues() throws IOException {
+        long[] arr = new long[blockSize];
+        for (int i = 0; i < blockSize; ++i) {
+            arr[i] = i % 3 == 1 ? 42 : 100;
+        }
+        final long expectedNumBytes = 3 // token + min value (1 byte) + GCD (1 byte)
+            + (blockSize * 1) / Byte.SIZE; // data
+        doTest(arr, expectedNumBytes);
+    }
+
+    /** Monotonic timestamps: delta coding + GCD compression. */
+    public void testMonotonicTimestamps() throws IOException {
+        long offset = 2 * 60 * 60 * 1000; // 2h offset = timezone
+        // a timestamp near Jan 1st 2021
+        long first = (2021 - 1970) * 365L * 24 * 60 * 60 * 1000 + offset;
+        long granularity = 24L * 60 * 60 * 1000; // 1 day
+
+        long[] arr = new long[blockSize];
+        arr[0] = first;
+        for (int i = 1; i < blockSize; ++i) {
+            arr[i] = arr[i - 1] + (5 + i % 4) * granularity; // increment by a multiple of granularity
+        }
+        final long expectedNumBytes = 16 // token + min delta + GCD + first
+            + (blockSize * 2) / Byte.SIZE; // data
+        doTest(arr, expectedNumBytes);
+    }
+
+    public void testZeroOrMinValue() throws IOException {
+        long[] arr = new long[blockSize];
+        for (int i = 0; i < blockSize; ++i) {
+            arr[i] = i % 3 == 1 ? Long.MIN_VALUE : 0;
+        }
+        final long expectedNumBytes = 10 // token + GCD (9 byte)
+            + (blockSize * 1) / Byte.SIZE; // data
+        doTest(arr, expectedNumBytes);
+    }
+
+    private void doTest(long[] arr, long expectedNumBytes) throws IOException {
+        final long[] expected = arr.clone();
+        try (Directory dir = newDirectory()) {
+            try (IndexOutput out = dir.createOutput("tests.bin", IOContext.DEFAULT)) {
+                encoder.encode(arr, out);
+                assertEquals(expectedNumBytes, out.getFilePointer());
+            }
+            try (IndexInput in = dir.openInput("tests.bin", IOContext.DEFAULT)) {
+                long[] decoded = new long[blockSize];
+                for (int i = 0; i < decoded.length; ++i) {
+                    decoded[i] = random().nextLong();
+                }
+                encoder.decode(in, decoded);
+                assertEquals(in.length(), in.getFilePointer());
+                assertArrayEquals(expected, decoded);
+            }
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormatTests.java
new file mode 100644
index 000000000000..1fe24f2e740e
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormatTests.java
@@ -0,0 +1,418 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.codec.tsdb;
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.tests.index.BaseDocValuesFormatTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+
+import java.io.IOException;
+import java.util.function.Supplier;
+
+public class ES87TSDBDocValuesFormatTests extends BaseDocValuesFormatTestCase {
+
+    private final Codec codec = TestUtil.alwaysDocValuesFormat(new ES87TSDBDocValuesFormat());
+
+    @Override
+    protected Codec getCodec() {
+        return codec;
+    }
+
+    // NOTE: here and below we disable tests dealing with non-numeric fields
+    // because ES87TSDBDocValuesFormat only deals with numeric fields.
+    @Override
+    public void testTwoBinaryValues() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testVariouslyCompressibleBinaryValues() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testTwoFieldsMixed() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testThreeFieldsMixed() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testThreeFieldsMixed2() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testBytes() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testBytesTwoDocumentsMerged() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testBytesMergeAwayAllValues() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedBytes() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedBytesTwoDocuments() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedBytesThreeDocuments() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedBytesTwoDocumentsMerged() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedMergeAwayAllValues() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testBytesWithNewline() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testMissingSortedBytes() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedTermsEnum() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testEmptySortedBytes() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testEmptyBytes() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testVeryLargeButLegalBytes() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testVeryLargeButLegalSortedBytes() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testCodecUsesOwnBytes() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testCodecUsesOwnSortedBytes() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testDocValuesSimple() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testRandomSortedBytes() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testBinaryFixedLengthVsStoredFields() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSparseBinaryFixedLengthVsStoredFields() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testBinaryVariableLengthVsStoredFields() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSparseBinaryVariableLengthVsStoredFields() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void doTestBinaryVariableLengthVsStoredFields(double density) throws Exception {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    protected void doTestSortedVsStoredFields(int numDocs, double density, Supplier<byte[]> bytes) throws Exception {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedFixedLengthVsStoredFields() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSparseSortedFixedLengthVsStoredFields() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedVariableLengthVsStoredFields() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSparseSortedVariableLengthVsStoredFields() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    protected void doTestSortedVsStoredFields(int numDocs, double density, int minLength, int maxLength) throws Exception {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetOneValue() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetTwoFields() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetTwoDocumentsMerged() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetTwoValues() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetTwoValuesUnordered() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetThreeValuesTwoDocs() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetTwoDocumentsLastMissing() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetTwoDocumentsLastMissingMerge() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetTwoDocumentsFirstMissing() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetTwoDocumentsFirstMissingMerge() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetMergeAwayAllValues() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetTermsEnum() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetFixedLengthVsStoredFields() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetVariableLengthVsStoredFields() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetFixedLengthSingleValuedVsStoredFields() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetVariableLengthSingleValuedVsStoredFields() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetFixedLengthFewUniqueSetsVsStoredFields() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetVariableLengthFewUniqueSetsVsStoredFields() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetVariableLengthManyValuesPerDocVsStoredFields() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetFixedLengthManyValuesPerDocVsStoredFields() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSparseGCDCompression() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testTwoBytesOneMissing() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testTwoBytesOneMissingWithMerging() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testThreeBytesOneMissingWithMerging() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testThreads() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testEmptyBinaryValueOnPageSizes() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testOneSortedNumber() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testOneSortedNumberOneMissing() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testTwoSortedNumber() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testTwoSortedNumberSameValue() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testTwoSortedNumberOneMissing() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedNumberMerge() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedNumberMergeAwayAllValues() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedEnumAdvanceIndependently() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetEnumAdvanceIndependently() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedMergeAwayAllValuesLargeSegment() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testSortedSetMergeAwayAllValuesLargeSegment() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testBinaryMergeAwayAllValuesLargeSegment() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testRandomAdvanceBinary() throws IOException {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testHighOrdsSortedSetDV() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testCheckIntegrityReadsAllBytes() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testMergeStability() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+
+    @Override
+    public void testRandomExceptions() {
+        assumeTrue("doc values format only supports numerics", false);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/engine/CombinedDeletionPolicyTests.java b/server/src/test/java/org/elasticsearch/index/engine/CombinedDeletionPolicyTests.java
index 5d96464c8c97..b50251aef011 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/CombinedDeletionPolicyTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/CombinedDeletionPolicyTests.java
@@ -10,6 +10,7 @@
 
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.store.Directory;
+import org.elasticsearch.common.lucene.FilterIndexCommit;
 import org.elasticsearch.index.seqno.RetentionLeases;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.translog.Translog;
@@ -19,14 +20,22 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 
 import static org.elasticsearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.not;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
@@ -266,12 +275,269 @@ public void testCheckUnreferencedCommits() throws Exception {
         }
     }
 
+    public void testCommitsListener() throws Exception {
+        final List<IndexCommit> acquiredCommits = new ArrayList<>();
+        final List<IndexCommit> deletedCommits = new ArrayList<>();
+        final Set<String> newCommitFiles = new HashSet<>();
+        final CombinedDeletionPolicy.CommitsListener commitsListener = new CombinedDeletionPolicy.CommitsListener() {
+            @Override
+            public void onNewAcquiredCommit(IndexCommit commit, Set<String> additionalFiles) {
+                assertThat(commit, instanceOf(FilterIndexCommit.class));
+                assertThat(acquiredCommits.add(((FilterIndexCommit) commit).getIndexCommit()), equalTo(true));
+                assertThat(additionalFiles, not(empty()));
+                newCommitFiles.clear();
+                newCommitFiles.addAll(additionalFiles);
+            }
+
+            @Override
+            public void onDeletedCommit(IndexCommit commit) {
+                assertThat(acquiredCommits.remove(commit), equalTo(true));
+                assertThat(deletedCommits.add(commit), equalTo(true));
+                assertThat(commit.isDeleted(), equalTo(true));
+            }
+        };
+
+        final AtomicLong globalCheckpoint = new AtomicLong(0L);
+        final TranslogDeletionPolicy translogDeletionPolicy = new TranslogDeletionPolicy();
+        final SoftDeletesPolicy softDeletesPolicy = new SoftDeletesPolicy(
+            globalCheckpoint::get,
+            NO_OPS_PERFORMED,
+            0L,
+            () -> RetentionLeases.EMPTY
+        );
+        final CombinedDeletionPolicy combinedDeletionPolicy = new CombinedDeletionPolicy(
+            logger,
+            translogDeletionPolicy,
+            softDeletesPolicy,
+            globalCheckpoint::get,
+            commitsListener
+        ) {
+            @Override
+            protected int getDocCountOfCommit(IndexCommit indexCommit) {
+                return 10;
+            }
+
+            @Override
+            synchronized boolean releaseCommit(IndexCommit indexCommit) {
+                return super.releaseCommit(wrapCommit(indexCommit));
+            }
+        };
+
+        final UUID translogUUID = UUID.randomUUID();
+        final IndexCommit commit0 = mockIndexCommit(NO_OPS_PERFORMED, NO_OPS_PERFORMED, translogUUID, Set.of("segments_0"));
+        combinedDeletionPolicy.onInit(List.of(commit0));
+
+        assertThat(acquiredCommits, contains(commit0));
+        assertThat(deletedCommits, hasSize(0));
+        assertThat(newCommitFiles, contains("segments_0"));
+
+        final IndexCommit commit1 = mockIndexCommit(10L, 10L, translogUUID, Set.of("_0.cfe", "_0.si", "_0.cfs", "segments_1"));
+        combinedDeletionPolicy.onCommit(List.of(commit0, commit1));
+
+        assertThat(acquiredCommits, contains(commit0, commit1));
+        assertThat(deletedCommits, hasSize(0));
+        assertThat(newCommitFiles, containsInAnyOrder(equalTo("_0.cfe"), equalTo("_0.si"), equalTo("_0.cfs"), equalTo("segments_1")));
+
+        globalCheckpoint.set(10L);
+        final IndexCommit commit2 = mockIndexCommit(
+            20L,
+            20L,
+            translogUUID,
+            Set.of("_1.cfs", "_0.cfe", "_0.si", "_1.cfe", "_1.si", "_0.cfs", "segments_2")
+        );
+        combinedDeletionPolicy.onCommit(List.of(commit0, commit1, commit2));
+
+        assertThat(acquiredCommits, contains(commit0, commit1, commit2));
+        assertThat(deletedCommits, hasSize(0));
+        assertThat(newCommitFiles, containsInAnyOrder(equalTo("_1.cfe"), equalTo("_1.si"), equalTo("_1.cfs"), equalTo("segments_2")));
+
+        boolean maybeCleanUpCommits = combinedDeletionPolicy.releaseCommit(commit0);
+        assertThat(maybeCleanUpCommits, equalTo(true));
+
+        globalCheckpoint.set(20L);
+        final IndexCommit commit3 = mockIndexCommit(
+            30L,
+            30L,
+            translogUUID,
+            Set.of(
+                "_3.fdx",
+                "_3_Lucene90_0.tip",
+                "_3_Lucene90_0.dvm",
+                "_3.si",
+                "_3_0.tmd",
+                "_3_0.tim",
+                "_3_ES85BloomFilter_0.bfi",
+                "_3_0.pos",
+                "_3_ES85BloomFilter_0.bfm",
+                "_3_0.tip",
+                "_3_Lucene90_0.doc",
+                "_3.nvd",
+                "_3.nvm",
+                "_3.fnm",
+                "_3_0.doc",
+                "segments_3",
+                "_3.kdd",
+                "_3_Lucene90_0.tmd",
+                "_3.fdm",
+                "_3.kdi",
+                "_3_Lucene90_0.dvd",
+                "_3_Lucene90_0.pos",
+                "_3.kdm",
+                "_3.fdt",
+                "_3_Lucene90_0.tim"
+            )
+        );
+        combinedDeletionPolicy.onCommit(List.of(commit0, commit1, commit2, commit3));
+
+        assertThat(acquiredCommits, contains(commit1, commit2, commit3));
+        assertThat(deletedCommits, contains(commit0));
+        assertThat(
+            newCommitFiles,
+            containsInAnyOrder(
+                equalTo("_3.fdx"),
+                equalTo("_3_Lucene90_0.tip"),
+                equalTo("_3_Lucene90_0.dvm"),
+                equalTo("_3.si"),
+                equalTo("_3_0.tmd"),
+                equalTo("_3_0.tim"),
+                equalTo("_3_ES85BloomFilter_0.bfi"),
+                equalTo("_3_0.pos"),
+                equalTo("_3_ES85BloomFilter_0.bfm"),
+                equalTo("_3_0.tip"),
+                equalTo("_3_Lucene90_0.doc"),
+                equalTo("_3.nvd"),
+                equalTo("_3.nvm"),
+                equalTo("_3.fnm"),
+                equalTo("_3_0.doc"),
+                equalTo("segments_3"),
+                equalTo("_3.kdd"),
+                equalTo("_3_Lucene90_0.tmd"),
+                equalTo("_3.fdm"),
+                equalTo("_3.kdi"),
+                equalTo("_3_Lucene90_0.dvd"),
+                equalTo("_3_Lucene90_0.pos"),
+                equalTo("_3.kdm"),
+                equalTo("_3.fdt"),
+                equalTo("_3_Lucene90_0.tim")
+            )
+        );
+
+        maybeCleanUpCommits = combinedDeletionPolicy.releaseCommit(commit2);
+        assertThat("No commits to clean up (commit #2 is the safe commit)", maybeCleanUpCommits, equalTo(false));
+
+        globalCheckpoint.set(30L);
+        final IndexCommit commit4 = mockIndexCommit(
+            40L,
+            40L,
+            translogUUID,
+            Set.of(
+                "_3.fdx",
+                "_3_Lucene90_0.tip",
+                "_3_Lucene90_0.dvm",
+                "_3.si",
+                "_3_0.tmd",
+                "_3_0.tim",
+                "_3_ES85BloomFilter_0.bfi",
+                "_3_0.pos",
+                "_3_ES85BloomFilter_0.bfm",
+                "_3_0.tip",
+                "_3_Lucene90_0.doc",
+                "_3.nvd",
+                "_4.cfe",
+                "_3.nvm",
+                "_3.fnm",
+                "_3_0.doc",
+                "segments_4",
+                "_3.kdd",
+                "_3_Lucene90_0.tmd",
+                "_3.fdm",
+                "_3.kdi",
+                "_4.cfs",
+                "_3_Lucene90_0.dvd",
+                "_3_Lucene90_0.pos",
+                "_3.kdm",
+                "_4.si",
+                "_3.fdt",
+                "_3_Lucene90_0.tim"
+            )
+        );
+        combinedDeletionPolicy.onCommit(List.of(commit1, commit2, commit3, commit4));
+
+        assertThat(acquiredCommits, contains(commit1, commit3, commit4));
+        assertThat(deletedCommits, contains(commit0, commit2));
+        assertThat(newCommitFiles, containsInAnyOrder(equalTo("_4.cfe"), equalTo("_4.si"), equalTo("_4.cfs"), equalTo("segments_4")));
+
+        maybeCleanUpCommits = combinedDeletionPolicy.releaseCommit(commit3);
+        assertThat("No commits to clean up (commit #3 is the safe commit)", maybeCleanUpCommits, equalTo(false));
+
+        maybeCleanUpCommits = combinedDeletionPolicy.releaseCommit(commit4);
+        assertThat("No commits to clean up (commit #4 is the last commit)", maybeCleanUpCommits, equalTo(false));
+
+        maybeCleanUpCommits = combinedDeletionPolicy.releaseCommit(commit1);
+        assertThat(maybeCleanUpCommits, equalTo(true));
+
+        final boolean globalCheckpointCatchUp = randomBoolean();
+        globalCheckpoint.set(globalCheckpointCatchUp ? 50L : 40L);
+
+        final IndexCommit commit5 = mockIndexCommit(
+            50L,
+            50L,
+            translogUUID,
+            Set.of(
+                "_3.fdx",
+                "_3_Lucene90_0.tip",
+                "_3_Lucene90_0.dvm",
+                "_3.si",
+                "_3_0.tmd",
+                "_3_0.tim",
+                "_3_ES85BloomFilter_0.bfi",
+                "_3_0.pos",
+                "_3_ES85BloomFilter_0.bfm",
+                "_3_0.tip",
+                "_5.si",
+                "_3_Lucene90_0.doc",
+                "segments_5",
+                "_3.nvd",
+                "_5.cfs",
+                "_4.cfe",
+                "_3.nvm",
+                "_3.fnm",
+                "_3_0.doc",
+                "_3.kdd",
+                "_3_Lucene90_0.tmd",
+                "_3.fdm",
+                "_3.kdi",
+                "_5.cfe",
+                "_4.cfs",
+                "_3_Lucene90_0.dvd",
+                "_3_Lucene90_0.pos",
+                "_3.kdm",
+                "_4.si",
+                "_3.fdt",
+                "_3_Lucene90_0.tim"
+            )
+        );
+        combinedDeletionPolicy.onCommit(List.of(commit1, commit3, commit4, commit5));
+
+        if (globalCheckpointCatchUp) {
+            assertThat(acquiredCommits, contains(commit5));
+            assertThat(deletedCommits, contains(commit0, commit2, commit1, commit3, commit4));
+        } else {
+            assertThat(acquiredCommits, contains(commit4, commit5));
+            assertThat(deletedCommits, contains(commit0, commit2, commit1, commit3));
+        }
+        assertThat(newCommitFiles, containsInAnyOrder(equalTo("_5.cfe"), equalTo("_5.si"), equalTo("_5.cfs"), equalTo("segments_5")));
+
+        maybeCleanUpCommits = combinedDeletionPolicy.releaseCommit(commit5);
+        assertThat("No commits to clean up (commit #5 is the last commit)", maybeCleanUpCommits, equalTo(false));
+    }
+
     private CombinedDeletionPolicy newCombinedDeletionPolicy(
         TranslogDeletionPolicy translogPolicy,
         SoftDeletesPolicy softDeletesPolicy,
         AtomicLong globalCheckpoint
     ) {
-        return new CombinedDeletionPolicy(logger, translogPolicy, softDeletesPolicy, globalCheckpoint::get) {
+        return new CombinedDeletionPolicy(logger, translogPolicy, softDeletesPolicy, globalCheckpoint::get, null) {
             @Override
             protected int getDocCountOfCommit(IndexCommit indexCommit) throws IOException {
                 if (randomBoolean()) {
@@ -284,6 +550,10 @@ protected int getDocCountOfCommit(IndexCommit indexCommit) throws IOException {
     }
 
     IndexCommit mockIndexCommit(long localCheckpoint, long maxSeqNo, UUID translogUUID) throws IOException {
+        return mockIndexCommit(localCheckpoint, maxSeqNo, translogUUID, Set.of());
+    }
+
+    IndexCommit mockIndexCommit(long localCheckpoint, long maxSeqNo, UUID translogUUID, Set<String> fileNames) throws IOException {
         final Map<String, String> userData = new HashMap<>();
         userData.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, Long.toString(localCheckpoint));
         userData.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(maxSeqNo));
@@ -292,6 +562,7 @@ IndexCommit mockIndexCommit(long localCheckpoint, long maxSeqNo, UUID translogUU
         final Directory directory = mock(Directory.class);
         when(commit.getUserData()).thenReturn(userData);
         when(commit.getDirectory()).thenReturn(directory);
+        when(commit.getFileNames()).thenReturn(fileNames);
         resetDeletion(commit);
         return commit;
     }
diff --git a/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java b/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java
index 6ff58b260ac6..7f13629cf31e 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/CompletionStatsCacheTests.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.index.engine;
 
 import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
@@ -45,7 +45,7 @@ public void testExceptionsAreNotCached() {
     public void testCompletionStatsCache() throws IOException, InterruptedException {
         final IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
         final PostingsFormat postingsFormat = new Completion90PostingsFormat();
-        indexWriterConfig.setCodec(new Lucene94Codec() {
+        indexWriterConfig.setCodec(new Lucene95Codec() {
             @Override
             public PostingsFormat getPostingsFormatForField(String field) {
                 return postingsFormat; // all fields are suggest fields
diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 9fabf2333daf..5200388af888 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -193,6 +193,7 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.emptyArray;
+import static org.hamcrest.Matchers.emptyIterable;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -1549,7 +1550,7 @@ public void testLookupVersionWithPrunedAwayIds() throws IOException {
                 writer.forceMerge(1);
                 try (DirectoryReader reader = DirectoryReader.open(writer)) {
                     assertEquals(1, reader.leaves().size());
-                    assertNull(VersionsAndSeqNoResolver.loadDocIdAndVersion(reader, new Term(IdFieldMapper.NAME, "1"), false));
+                    assertNull(VersionsAndSeqNoResolver.timeSeriesLoadDocIdAndVersion(reader, new Term(IdFieldMapper.NAME, "1"), false));
                 }
             }
         }
@@ -3588,7 +3589,9 @@ public void testRecoverFromForeignTranslog() throws IOException {
             primaryTerm::get,
             IndexModule.DEFAULT_SNAPSHOT_COMMIT_SUPPLIER,
             null,
-            config.getRelativeTimeInNanosSupplier()
+            config.getRelativeTimeInNanosSupplier(),
+            null,
+            true
         );
         expectThrows(EngineCreationFailureException.class, () -> new InternalEngine(brokenConfig));
 
@@ -5426,7 +5429,9 @@ public void testRefreshScopedSearcher() throws IOException {
                 engine.index(primaryResponse);
             }
             assertTrue(engine.refreshNeeded());
-            engine.refresh("test", Engine.SearcherScope.INTERNAL, true);
+            var refreshResult = engine.refresh("test", Engine.SearcherScope.INTERNAL, true);
+            assertTrue(refreshResult.refreshed());
+            assertNotEquals(refreshResult.generation(), Engine.RefreshResult.UNKNOWN_GENERATION);
             try (
                 Engine.Searcher getSearcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL);
                 Engine.Searcher searchSearcher = engine.acquireSearcher("test", Engine.SearcherScope.EXTERNAL)
@@ -7257,7 +7262,9 @@ public void testNotWarmUpSearcherInEngineCtor() throws Exception {
                 config.getPrimaryTermSupplier(),
                 config.getSnapshotCommitSupplier(),
                 config.getLeafSorter(),
-                config.getRelativeTimeInNanosSupplier()
+                config.getRelativeTimeInNanosSupplier(),
+                config.getIndexCommitListener(),
+                config.isPromotableToPrimary()
             );
             try (InternalEngine engine = createEngine(configWithWarmer)) {
                 assertThat(warmedUpReaders, empty());
@@ -7471,4 +7478,159 @@ public void testTrimUnsafeCommitHasESVersionInUserData() throws IOException {
             assertThat(userDataAfterTrimUnsafeCommits.get(ES_VERSION), is(equalTo(Version.CURRENT.toString())));
         }
     }
+
+    public void testIndexCommitsListener() throws Exception {
+        final Map<IndexCommit, Engine.IndexCommitRef> acquiredCommits = new HashMap<>();
+        final List<IndexCommit> deletedCommits = new ArrayList<>();
+        final List<Long> acquiredPrimaryTerms = new ArrayList<>();
+
+        final Engine.IndexCommitListener indexCommitListener = new Engine.IndexCommitListener() {
+            @Override
+            public void onNewCommit(
+                ShardId shardId,
+                Store store,
+                long primaryTerm,
+                Engine.IndexCommitRef indexCommitRef,
+                Set<String> additionalFiles
+            ) {
+                assertNotNull(store);
+                assertTrue(store.hasReferences());
+                assertThat(acquiredCommits.put(indexCommitRef.getIndexCommit(), indexCommitRef), nullValue());
+                assertThat(shardId, equalTo(InternalEngineTests.this.shardId));
+                assertThat(primaryTerm, greaterThanOrEqualTo(0L));
+                acquiredPrimaryTerms.add(primaryTerm);
+            }
+
+            @Override
+            public void onIndexCommitDelete(ShardId shardId, IndexCommit deletedCommit) {
+                assertThat(acquiredCommits.remove(deletedCommit), notNullValue());
+                assertThat(deletedCommits.add(deletedCommit), equalTo(true));
+                assertThat(deletedCommit.isDeleted(), equalTo(true));
+                assertThat(shardId, equalTo(InternalEngineTests.this.shardId));
+            }
+        };
+
+        final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
+        try (
+            Store store = createStore();
+            InternalEngine engine = createEngine(
+                config(
+                    defaultSettings,
+                    store,
+                    createTempDir(),
+                    NoMergePolicy.INSTANCE,
+                    null,
+                    null,
+                    null,
+                    globalCheckpoint::get,
+                    () -> RetentionLeases.EMPTY,
+                    new NoneCircuitBreakerService(),
+                    indexCommitListener
+                )
+            )
+        ) {
+            assertCommitGenerations(acquiredCommits, List.of(2L));
+            assertCommitGenerations(deletedCommits, List.of());
+
+            engine.index(indexForDoc(createParsedDoc("a", EngineTestCase.randomIdFieldType(), null)));
+            engine.flush();
+
+            assertCommitGenerations(acquiredCommits, List.of(2L, 3L));
+            assertCommitGenerations(deletedCommits, List.of());
+
+            globalCheckpoint.set(0L);
+            engine.index(indexForDoc(createParsedDoc("b", EngineTestCase.randomIdFieldType(), null)));
+            engine.flush();
+
+            assertCommitGenerations(acquiredCommits, List.of(2L, 3L, 4L));
+            assertCommitGenerations(deletedCommits, List.of());
+
+            releaseCommitRef(acquiredCommits, 2L);
+
+            globalCheckpoint.set(1L);
+            engine.index(indexForDoc(createParsedDoc("c", EngineTestCase.randomIdFieldType(), null)));
+            engine.flush();
+
+            assertCommitGenerations(acquiredCommits, List.of(3L, 4L, 5L));
+            assertCommitGenerations(deletedCommits, List.of(2L));
+
+            releaseCommitRef(acquiredCommits, 4L);
+
+            globalCheckpoint.set(2L);
+            engine.index(indexForDoc(createParsedDoc("d", EngineTestCase.randomIdFieldType(), null)));
+            engine.flush();
+
+            assertCommitGenerations(acquiredCommits, List.of(3L, 5L, 6L));
+            assertCommitGenerations(deletedCommits, List.of(2L, 4L));
+
+            releaseCommitRef(acquiredCommits, 5L);
+            releaseCommitRef(acquiredCommits, 6L);
+            releaseCommitRef(acquiredCommits, 3L);
+
+            final boolean globalCheckpointCatchUp = randomBoolean();
+            globalCheckpoint.set(globalCheckpointCatchUp ? 4L : 3L);
+
+            engine.index(indexForDoc(createParsedDoc("e", EngineTestCase.randomIdFieldType(), null)));
+            engine.flush();
+
+            if (globalCheckpointCatchUp) {
+                assertCommitGenerations(acquiredCommits, List.of(7L));
+                assertCommitGenerations(deletedCommits, List.of(2L, 3L, 4L, 5L, 6L));
+            } else {
+                assertCommitGenerations(acquiredCommits, List.of(6L, 7L));
+                assertCommitGenerations(deletedCommits, List.of(2L, 3L, 4L, 5L));
+            }
+
+            releaseCommitRef(acquiredCommits, 7L);
+
+            final long primaryTerm = engine.config().getPrimaryTermSupplier().getAsLong();
+            assertThat(acquiredPrimaryTerms.stream().allMatch(value -> value == primaryTerm), is(true));
+        }
+    }
+
+    public void testRefreshResult() throws IOException {
+        try (
+            Store store = createStore();
+            InternalEngine engine =
+                // disable merges to make sure that the reader doesn't change unexpectedly during the test
+                createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE)
+        ) {
+            var refresh1Result = engine.refresh("warm_up");
+            assertTrue(refresh1Result.refreshed());
+            assertNotEquals("when refreshed, generation must be set", refresh1Result.generation(), Engine.RefreshResult.UNKNOWN_GENERATION);
+            for (int i = 0; i < 10; i++) {
+                engine.index(indexForDoc(createParsedDoc(String.valueOf(i), EngineTestCase.randomIdFieldType(), null)));
+            }
+            assertTrue(engine.refreshNeeded());
+            var refresh2Result = engine.refresh("test", Engine.SearcherScope.INTERNAL, true);
+            assertTrue(refresh2Result.refreshed());
+            assertThat(refresh2Result.generation(), greaterThanOrEqualTo(refresh1Result.generation()));
+            engine.flush(true, true);
+            var refresh3Result = engine.refresh("test");
+            assertTrue(refresh3Result.refreshed());
+            assertThat(refresh3Result.generation(), greaterThan(refresh2Result.generation()));
+        }
+    }
+
+    private static void assertCommitGenerations(Map<IndexCommit, Engine.IndexCommitRef> commits, List<Long> expectedGenerations) {
+        assertCommitGenerations(commits.values().stream().map(Engine.IndexCommitRef::getIndexCommit).toList(), expectedGenerations);
+    }
+
+    private static void assertCommitGenerations(List<IndexCommit> commits, List<Long> expectedGenerations) {
+        assertThat(
+            commits.stream().map(IndexCommit::getGeneration).sorted().toList(),
+            expectedGenerations.isEmpty() ? emptyIterable() : equalTo(expectedGenerations)
+        );
+    }
+
+    private static void releaseCommitRef(Map<IndexCommit, Engine.IndexCommitRef> commits, long generation) {
+        var releasable = commits.keySet().stream().filter(c -> c.getGeneration() == generation).findFirst();
+        assertThat(releasable.isPresent(), is(true));
+        Engine.IndexCommitRef indexCommitRef = commits.get(releasable.get());
+        try {
+            indexCommitRef.close();
+        } catch (IOException e) {
+            throw new AssertionError("Failed to release IndexCommitRef for commit generation " + generation, e);
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java
index ac47d48e4707..3fef14fe6df8 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java
@@ -101,7 +101,8 @@ public <IFD extends IndexFieldData<?>> IFD getForField(String type, String field
                 ScriptCompiler.NONE,
                 false,
                 true,
-                Version.CURRENT
+                Version.CURRENT,
+                null
             ).docValues(docValues).build(context).fieldType();
         } else if (type.equals("double")) {
             fieldType = new NumberFieldMapper.Builder(
@@ -110,7 +111,8 @@ public <IFD extends IndexFieldData<?>> IFD getForField(String type, String field
                 ScriptCompiler.NONE,
                 false,
                 true,
-                Version.CURRENT
+                Version.CURRENT,
+                null
             ).docValues(docValues).build(context).fieldType();
         } else if (type.equals("long")) {
             fieldType = new NumberFieldMapper.Builder(
@@ -119,7 +121,8 @@ public <IFD extends IndexFieldData<?>> IFD getForField(String type, String field
                 ScriptCompiler.NONE,
                 false,
                 true,
-                Version.CURRENT
+                Version.CURRENT,
+                null
             ).docValues(docValues).build(context).fieldType();
         } else if (type.equals("int")) {
             fieldType = new NumberFieldMapper.Builder(
@@ -128,7 +131,8 @@ public <IFD extends IndexFieldData<?>> IFD getForField(String type, String field
                 ScriptCompiler.NONE,
                 false,
                 true,
-                Version.CURRENT
+                Version.CURRENT,
+                null
             ).docValues(docValues).build(context).fieldType();
         } else if (type.equals("short")) {
             fieldType = new NumberFieldMapper.Builder(
@@ -137,7 +141,8 @@ public <IFD extends IndexFieldData<?>> IFD getForField(String type, String field
                 ScriptCompiler.NONE,
                 false,
                 true,
-                Version.CURRENT
+                Version.CURRENT,
+                null
             ).docValues(docValues).build(context).fieldType();
         } else if (type.equals("byte")) {
             fieldType = new NumberFieldMapper.Builder(
@@ -146,7 +151,8 @@ public <IFD extends IndexFieldData<?>> IFD getForField(String type, String field
                 ScriptCompiler.NONE,
                 false,
                 true,
-                Version.CURRENT
+                Version.CURRENT,
+                null
             ).docValues(docValues).build(context).fieldType();
         } else if (type.equals("geo_point")) {
             fieldType = new GeoPointFieldMapper.Builder(fieldName, ScriptCompiler.NONE, false, Version.CURRENT).docValues(docValues)
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
index 02a673d1c783..32412c0094af 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
@@ -40,7 +40,6 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -85,10 +84,11 @@ public void testGetForFieldDefaults() {
         assertTrue(fd instanceof SortedSetOrdinalsIndexFieldData);
 
         for (MappedFieldType mapper : Arrays.asList(
-            new NumberFieldMapper.Builder("int", BYTE, ScriptCompiler.NONE, false, true, Version.CURRENT).build(context).fieldType(),
-            new NumberFieldMapper.Builder("int", SHORT, ScriptCompiler.NONE, false, true, Version.CURRENT).build(context).fieldType(),
-            new NumberFieldMapper.Builder("int", INTEGER, ScriptCompiler.NONE, false, true, Version.CURRENT).build(context).fieldType(),
-            new NumberFieldMapper.Builder("long", LONG, ScriptCompiler.NONE, false, true, Version.CURRENT).build(context).fieldType()
+            new NumberFieldMapper.Builder("int", BYTE, ScriptCompiler.NONE, false, true, Version.CURRENT, null).build(context).fieldType(),
+            new NumberFieldMapper.Builder("int", SHORT, ScriptCompiler.NONE, false, true, Version.CURRENT, null).build(context).fieldType(),
+            new NumberFieldMapper.Builder("int", INTEGER, ScriptCompiler.NONE, false, true, Version.CURRENT, null).build(context)
+                .fieldType(),
+            new NumberFieldMapper.Builder("long", LONG, ScriptCompiler.NONE, false, true, Version.CURRENT, null).build(context).fieldType()
         )) {
             ifdService.clear();
             fd = ifdService.getForField(mapper, FieldDataContext.noRuntimeFields("test"));
@@ -101,7 +101,8 @@ public void testGetForFieldDefaults() {
             ScriptCompiler.NONE,
             false,
             true,
-            Version.CURRENT
+            Version.CURRENT,
+            null
         ).build(context).fieldType();
         ifdService.clear();
         fd = ifdService.getForField(floatMapper, FieldDataContext.noRuntimeFields("test"));
@@ -113,7 +114,8 @@ public void testGetForFieldDefaults() {
             ScriptCompiler.NONE,
             false,
             true,
-            Version.CURRENT
+            Version.CURRENT,
+            null
         ).build(context).fieldType();
         ifdService.clear();
         fd = ifdService.getForField(doubleMapper, FieldDataContext.noRuntimeFields("test"));
@@ -136,7 +138,7 @@ public void testGetForFieldRuntimeField() {
             searchLookupSetOnce.set(fdc.lookupSupplier());
             return (IndexFieldData.Builder) (cache, breakerService) -> null;
         });
-        SearchLookup searchLookup = new SearchLookup(null, null, new SourceLookup.ReaderSourceProvider());
+        SearchLookup searchLookup = new SearchLookup(null, null, (ctx, doc) -> null);
         ifdService.getForField(ft, new FieldDataContext("qualified", () -> searchLookup, null, MappedFieldType.FielddataOperation.SEARCH));
         assertSame(searchLookup, searchLookupSetOnce.get().get());
     }
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldDataTests.java
index dbd0e695ae5c..d61cb5fba1a6 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldDataTests.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldDataTests.java
@@ -8,9 +8,7 @@
 
 package org.elasticsearch.index.fielddata;
 
-import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBooleanIndexFieldData.SourceValueFetcherSortedBooleanDocValues;
-import org.elasticsearch.search.lookup.SourceLookup;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -29,12 +27,7 @@ public void testSourceValueFetcherSortedBooleanDocValues() throws IOException {
         SourceValueFetcherSortedBooleanDocValues values = new SourceValueFetcherSortedBooleanDocValues(
             null,
             (source, doc, ignoredValues) -> docs.get(doc),
-            new SourceLookup(null) {
-                @Override
-                public void setSegmentAndDocument(LeafReaderContext context, int docId) {
-                    // do nothing
-                }
-            }
+            (ctx, doc) -> null
         );
 
         for (int doc = 0; doc < docs.size(); ++doc) {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java
index 04d9846ad292..a4cdd842dbe0 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java
@@ -8,9 +8,15 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -18,6 +24,7 @@
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.query.ExistsQueryBuilder;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.script.BooleanFieldScript;
@@ -30,7 +37,7 @@
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.StringFieldScript;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
@@ -38,7 +45,9 @@
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.BiConsumer;
 
 import static org.hamcrest.Matchers.containsString;
@@ -157,6 +166,45 @@ public void testFieldCaps() throws Exception {
         assertEquals(concreteIndexType.isAggregatable(), scriptFieldType.isAggregatable());
     }
 
+    /**
+     * Check that running query on a runtime field script that fails has the expected behaviour according to its configuration
+     */
+    public final void testOnScriptError() throws IOException {
+        try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {
+            iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [1]}"))));
+            try (DirectoryReader reader = iw.getReader()) {
+                IndexSearcher searcher = newUnthreadedSearcher(reader);
+                {
+                    AbstractScriptFieldType<?> fieldType = build("error", Collections.emptyMap(), OnScriptError.CONTINUE);
+                    SearchExecutionContext searchExecutionContext = mockContext(true, fieldType);
+                    Query query = new ExistsQueryBuilder("test").rewrite(searchExecutionContext).toQuery(searchExecutionContext);
+                    assertEquals(0, searcher.count(query));
+                }
+                {
+                    AbstractScriptFieldType<?> fieldType = build("error", Collections.emptyMap(), OnScriptError.FAIL);
+                    SearchExecutionContext searchExecutionContext = mockContext(true, fieldType);
+                    Query query = new ExistsQueryBuilder("test").rewrite(searchExecutionContext).toQuery(searchExecutionContext);
+                    expectThrows(RuntimeException.class, () -> searcher.count(query));
+                }
+            }
+        }
+    }
+
+    public final void testOnScriptErrorFail() throws IOException {
+        try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {
+            iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [1]}"))));
+            try (DirectoryReader reader = iw.getReader()) {
+                IndexSearcher searcher = newUnthreadedSearcher(reader);
+                AbstractScriptFieldType<?> fieldType = build("error", Collections.emptyMap(), OnScriptError.FAIL);
+                SearchExecutionContext searchExecutionContext = mockContext(true, fieldType);
+                Query query = new ExistsQueryBuilder("test").rewrite(searchExecutionContext).toQuery(searchExecutionContext);
+                expectThrows(RuntimeException.class, () -> searcher.count(query));
+            }
+        }
+    }
+
+    protected abstract AbstractScriptFieldType<?> build(String error, Map<String, Object> emptyMap, OnScriptError onScriptError);
+
     @SuppressWarnings("unused")
     public abstract void testDocValues() throws IOException;
 
@@ -211,13 +259,13 @@ protected boolean supportsRangeQueries() {
     }
 
     protected static SearchExecutionContext mockContext(boolean allowExpensiveQueries, MappedFieldType mappedFieldType) {
-        return mockContext(allowExpensiveQueries, mappedFieldType, new SourceLookup.ReaderSourceProvider());
+        return mockContext(allowExpensiveQueries, mappedFieldType, SourceProvider.fromStoredFields());
     }
 
     protected static SearchExecutionContext mockContext(
         boolean allowExpensiveQueries,
         MappedFieldType mappedFieldType,
-        SourceLookup.SourceProvider sourceProvider
+        SourceProvider sourceProvider
     ) {
         SearchExecutionContext context = mock(SearchExecutionContext.class);
         if (mappedFieldType != null) {
@@ -237,6 +285,7 @@ protected static SearchExecutionContext mockContext(
             return ft.fielddataBuilder(new FieldDataContext("test", context::lookup, context::sourcePath, fdo))
                 .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService());
         });
+        when(context.getMatchingFieldNames(any())).thenReturn(Set.of("dummy_field"));
         return context;
     }
 
@@ -405,7 +454,7 @@ protected <T> T compileScript(Script script, ScriptContext<T> context) {
 
     /**
      * We need to make sure we don't randomize the useThreads parameter for subtests of this abstract test case
-     * because scripted fields use {@link SourceLookup} which isn't thread-safe.
+     * because scripted fields use {@link SearchLookup#getSource(LeafReaderContext, int)} which isn't thread-safe.
      * Also Elasticsearch doesn't support concurrent searches so far, so we don't need to test it.
      */
     protected IndexSearcher newUnthreadedSearcher(IndexReader reader) {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
index b392755b2f44..21a7e27998b3 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
@@ -258,7 +258,13 @@ protected IngestScriptSupport ingestScriptSupport() {
         return new IngestScriptSupport() {
             @Override
             protected BooleanFieldScript.Factory emptyFieldScript() {
-                return (fieldName, params, searchLookup) -> ctx -> new BooleanFieldScript(fieldName, params, searchLookup, ctx) {
+                return (fieldName, params, searchLookup, onScriptError) -> ctx -> new BooleanFieldScript(
+                    fieldName,
+                    params,
+                    searchLookup,
+                    OnScriptError.FAIL,
+                    ctx
+                ) {
                     @Override
                     public void execute() {}
                 };
@@ -266,7 +272,13 @@ public void execute() {}
 
             @Override
             protected BooleanFieldScript.Factory nonEmptyFieldScript() {
-                return (fieldName, params, searchLookup) -> ctx -> new BooleanFieldScript(fieldName, params, searchLookup, ctx) {
+                return (fieldName, params, searchLookup, onScriptError) -> ctx -> new BooleanFieldScript(
+                    fieldName,
+                    params,
+                    searchLookup,
+                    OnScriptError.FAIL,
+                    ctx
+                ) {
                     @Override
                     public void execute() {
                         emit(true);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java
index 082c4f79354d..d628990aecb7 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java
@@ -17,17 +17,17 @@
 import org.elasticsearch.script.BooleanFieldScript;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
 public class BooleanFieldScriptTests extends FieldScriptTestCase<BooleanFieldScript.Factory> {
-    public static final BooleanFieldScript.Factory DUMMY = (fieldName, params, lookup) -> ctx -> new BooleanFieldScript(
+    public static final BooleanFieldScript.Factory DUMMY = (fieldName, params, lookup, onScriptError) -> ctx -> new BooleanFieldScript(
         fieldName,
         params,
         lookup,
+        OnScriptError.FAIL,
         ctx
     ) {
         @Override
@@ -58,7 +58,8 @@ public void testTooManyValues() throws IOException {
                 BooleanFieldScript script = new BooleanFieldScript(
                     "test",
                     Map.of(),
-                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()),
+                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, (ctx, doc) -> null),
+                    OnScriptError.FAIL,
                     reader.leaves().get(0)
                 ) {
                     @Override
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java
index 6fa75dbc1d46..9be06048daec 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.fielddata.BooleanScriptFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.query.SearchExecutionContext;
@@ -253,7 +252,10 @@ public void testTermQuery() throws IOException {
                 assertThat(searcher.count(simpleMappedFieldType().termQuery(true, mockContext())), equalTo(1));
                 assertThat(searcher.count(simpleMappedFieldType().termQuery("true", mockContext())), equalTo(1));
                 assertThat(searcher.count(simpleMappedFieldType().termQuery(false, mockContext())), equalTo(0));
-                assertThat(searcher.count(build("xor_param", Map.of("param", false)).termQuery(true, mockContext())), equalTo(1));
+                assertThat(
+                    searcher.count(build("xor_param", Map.of("param", false), OnScriptError.FAIL).termQuery(true, mockContext())),
+                    equalTo(1)
+                );
             }
         }
         try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {
@@ -264,7 +266,10 @@ public void testTermQuery() throws IOException {
                 assertThat(searcher.count(simpleMappedFieldType().termQuery("false", mockContext())), equalTo(1));
                 assertThat(searcher.count(simpleMappedFieldType().termQuery(null, mockContext())), equalTo(1));
                 assertThat(searcher.count(simpleMappedFieldType().termQuery(true, mockContext())), equalTo(0));
-                assertThat(searcher.count(build("xor_param", Map.of("param", false)).termQuery(false, mockContext())), equalTo(1));
+                assertThat(
+                    searcher.count(build("xor_param", Map.of("param", false), OnScriptError.FAIL).termQuery(false, mockContext())),
+                    equalTo(1)
+                );
             }
         }
     }
@@ -322,13 +327,7 @@ public void testDualingQueries() throws IOException {
             String source = "{\"foo\": " + values + "}";
             XContentParser parser = createParser(JsonXContent.jsonXContent, source);
             SourceToParse sourceToParse = new SourceToParse("test", new BytesArray(source), XContentType.JSON);
-            DocumentParserContext ctx = new TestDocumentParserContext(
-                MappingLookup.EMPTY,
-                MapperTestCase.createIndexSettings(Version.CURRENT, Settings.EMPTY),
-                null,
-                null,
-                sourceToParse
-            ) {
+            DocumentParserContext ctx = new TestDocumentParserContext(MappingLookup.EMPTY, sourceToParse) {
                 @Override
                 public XContentParser parser() {
                     return parser;
@@ -400,12 +399,12 @@ private void assertSameCount(IndexSearcher searcher, String source, Object query
 
     @Override
     protected BooleanScriptFieldType simpleMappedFieldType() {
-        return build("read_foo", Map.of());
+        return build("read_foo", Map.of(), OnScriptError.FAIL);
     }
 
     @Override
     protected MappedFieldType loopFieldType() {
-        return build("loop", Map.of());
+        return build("loop", Map.of(), OnScriptError.FAIL);
     }
 
     @Override
@@ -413,38 +412,63 @@ protected String typeName() {
         return "boolean";
     }
 
-    private static BooleanScriptFieldType build(String code, Map<String, Object> params) {
-        return build(new Script(ScriptType.INLINE, "test", code, params));
+    protected BooleanScriptFieldType build(String code, Map<String, Object> params, OnScriptError onScriptError) {
+        Script script = new Script(ScriptType.INLINE, "test", code, params);
+        return new BooleanScriptFieldType("test", factory(script), script, emptyMap(), onScriptError);
     }
 
     private static BooleanFieldScript.Factory factory(Script script) {
         return switch (script.getIdOrCode()) {
-            case "read_foo" -> (fieldName, params, lookup) -> (ctx) -> new BooleanFieldScript(fieldName, params, lookup, ctx) {
+            case "read_foo" -> (fieldName, params, lookup, onScriptError) -> (ctx) -> new BooleanFieldScript(
+                fieldName,
+                params,
+                lookup,
+                onScriptError,
+                ctx
+            ) {
                 @Override
+                @SuppressWarnings("unchecked")
                 public void execute() {
-                    for (Object foo : (List<?>) lookup.source().source().get("foo")) {
+                    Map<String, Object> source = (Map<String, Object>) this.getParams().get("_source");
+                    for (Object foo : (List<?>) source.get("foo")) {
                         emit((Boolean) foo);
                     }
                 }
             };
-            case "xor_param" -> (fieldName, params, lookup) -> (ctx) -> new BooleanFieldScript(fieldName, params, lookup, ctx) {
+            case "xor_param" -> (fieldName, params, lookup, onScriptError) -> (ctx) -> new BooleanFieldScript(
+                fieldName,
+                params,
+                lookup,
+                onScriptError,
+                ctx
+            ) {
                 @Override
+                @SuppressWarnings("unchecked")
                 public void execute() {
-                    for (Object foo : (List<?>) lookup.source().source().get("foo")) {
+                    Map<String, Object> source = (Map<String, Object>) this.getParams().get("_source");
+                    for (Object foo : (List<?>) source.get("foo")) {
                         emit((Boolean) foo ^ ((Boolean) getParams().get("param")));
                     }
                 }
             };
-            case "loop" -> (fieldName, params, lookup) -> {
+            case "loop" -> (fieldName, params, lookup, onScriptError) -> {
                 // Indicate that this script wants the field call "test", which *is* the name of this field
                 lookup.forkAndTrackFieldReferences("test");
                 throw new IllegalStateException("should have thrown on the line above");
             };
+            case "error" -> (fieldName, params, lookup, onScriptError) -> ctx -> new BooleanFieldScript(
+                fieldName,
+                params,
+                lookup,
+                onScriptError,
+                ctx
+            ) {
+                @Override
+                public void execute() {
+                    throw new RuntimeException("test error");
+                }
+            };
             default -> throw new IllegalArgumentException("unsupported script [" + script.getIdOrCode() + "]");
         };
     }
-
-    private static BooleanScriptFieldType build(Script script) {
-        return new BooleanScriptFieldType("test", factory(script), script, emptyMap());
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptMapperTests.java
index 5bf46fd8a258..ef0fcc5cb7ff 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptMapperTests.java
@@ -21,11 +21,16 @@ public class BooleanScriptMapperTests extends MapperScriptTestCase<BooleanFieldS
     private static BooleanFieldScript.Factory factory(Consumer<BooleanFieldScript> executor) {
         return new BooleanFieldScript.Factory() {
             @Override
-            public BooleanFieldScript.LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup) {
+            public BooleanFieldScript.LeafFactory newFactory(
+                String fieldName,
+                Map<String, Object> params,
+                SearchLookup searchLookup,
+                OnScriptError onScriptError
+            ) {
                 return new BooleanFieldScript.LeafFactory() {
                     @Override
                     public BooleanFieldScript newInstance(LeafReaderContext ctx) {
-                        return new BooleanFieldScript(fieldName, params, searchLookup, ctx) {
+                        return new BooleanFieldScript(fieldName, params, searchLookup, OnScriptError.FAIL, ctx) {
                             @Override
                             public void execute() {
                                 executor.accept(this);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java
index b2ce1482ce02..162b45cef971 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java
@@ -16,7 +16,7 @@
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.search.lookup.LeafSearchLookup;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
 
@@ -33,10 +33,11 @@ public class CompositeRuntimeFieldTests extends MapperServiceTestCase {
     @SuppressWarnings("unchecked")
     protected <T> T compileScript(Script script, ScriptContext<T> context) {
         if (context == CompositeFieldScript.CONTEXT) {
-            return (T) (CompositeFieldScript.Factory) (fieldName, params, searchLookup) -> ctx -> new CompositeFieldScript(
+            return (T) (CompositeFieldScript.Factory) (fieldName, params, searchLookup, onScriptError) -> ctx -> new CompositeFieldScript(
                 fieldName,
                 params,
                 searchLookup,
+                OnScriptError.FAIL,
                 ctx
             ) {
                 @Override
@@ -52,7 +53,13 @@ public void execute() {
             };
         }
         if (context == LongFieldScript.CONTEXT) {
-            return (T) (LongFieldScript.Factory) (field, params, lookup) -> ctx -> new LongFieldScript(field, params, lookup, ctx) {
+            return (T) (LongFieldScript.Factory) (field, params, lookup, onScriptError) -> ctx -> new LongFieldScript(
+                field,
+                params,
+                lookup,
+                OnScriptError.FAIL,
+                ctx
+            ) {
                 @Override
                 public void execute() {
 
@@ -338,7 +345,7 @@ public void testParseDocumentSubFieldAccess() throws IOException {
                 (mft, lookupSupplier, fdo) -> mft.fielddataBuilder(
                     new FieldDataContext("test", lookupSupplier, mapperService.mappingLookup()::sourcePaths, fdo)
                 ).build(null, null),
-                new SourceLookup.ReaderSourceProvider()
+                SourceProvider.fromStoredFields()
             );
 
             LeafSearchLookup leafSearchLookup = searchLookup.getLeafSearchLookup(reader.leaves().get(0));
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java
index 70156768d63f..f78fbf1f5d2d 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/CopyToMapperTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.xcontent.ToXContent;
@@ -91,9 +92,12 @@ public void testCopyToFieldsParsing() throws Exception {
         assertThat(doc.getFields("cyclic_test")[1].stringValue(), equalTo("bar"));
 
         assertThat(doc.getFields("int_to_str_test").length, equalTo(1));
+        assertThat(doc.getFields("int_to_str_test")[0].fieldType().docValuesType(), equalTo(DocValuesType.NONE));
         assertThat(doc.getFields("int_to_str_test")[0].numericValue().intValue(), equalTo(42));
 
-        assertThat(doc.getFields("new_field").length, equalTo(2)); // new field has doc values
+        assertThat(doc.getFields("new_field").length, equalTo(1));
+        // new_field has doc values
+        assertThat(doc.getFields("new_field")[0].fieldType().docValuesType(), equalTo(DocValuesType.SORTED_NUMERIC));
         assertThat(doc.getFields("new_field")[0].numericValue().intValue(), equalTo(42));
 
         assertNotNull(parsedDoc.dynamicMappingsUpdate());
@@ -379,31 +383,37 @@ public void testCopyToNestedField() throws Exception {
         assertFieldValue(nested, "n1.n2.target", 3L);
         assertFieldValue(nested, "n1.target");
         assertFieldValue(nested, "target");
+        assertEquals(1, nested.getFields(NestedPathFieldMapper.NAME).length);
 
         nested = doc.docs().get(1);
         assertFieldValue(nested, "n1.n2.target", 5L);
         assertFieldValue(nested, "n1.target");
         assertFieldValue(nested, "target");
+        assertEquals(1, nested.getFields(NestedPathFieldMapper.NAME).length);
 
         nested = doc.docs().get(3);
         assertFieldValue(nested, "n1.n2.target", 7L);
         assertFieldValue(nested, "n1.target");
         assertFieldValue(nested, "target");
+        assertEquals(1, nested.getFields(NestedPathFieldMapper.NAME).length);
 
         LuceneDocument parent = doc.docs().get(2);
         assertFieldValue(parent, "target");
         assertFieldValue(parent, "n1.target", 3L, 5L);
         assertFieldValue(parent, "n1.n2.target");
+        assertEquals(1, parent.getFields(NestedPathFieldMapper.NAME).length);
 
         parent = doc.docs().get(4);
         assertFieldValue(parent, "target");
         assertFieldValue(parent, "n1.target", 7L);
         assertFieldValue(parent, "n1.n2.target");
+        assertEquals(1, parent.getFields(NestedPathFieldMapper.NAME).length);
 
         LuceneDocument root = doc.docs().get(5);
         assertFieldValue(root, "target", 3L, 5L, 7L);
         assertFieldValue(root, "n1.target");
         assertFieldValue(root, "n1.n2.target");
+        assertEquals(0, root.getFields(NestedPathFieldMapper.NAME).length);
     }
 
     public void testCopyToChildNested() {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
index 68b161421f7a..74ad20ea3988 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType;
 import org.elasticsearch.script.DateFieldScript;
@@ -88,16 +89,14 @@ public void testDefaults() throws Exception {
         ParsedDocument doc = mapper.parse(source(b -> b.field("field", "2016-03-11")));
 
         IndexableField[] fields = doc.rootDoc().getFields("field");
-        assertEquals(2, fields.length);
-        IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
-        assertEquals(8, pointField.fieldType().pointNumBytes());
-        assertFalse(pointField.fieldType().stored());
-        assertEquals(1457654400000L, pointField.numericValue().longValue());
-        IndexableField dvField = fields[1];
-        assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
-        assertEquals(1457654400000L, dvField.numericValue().longValue());
-        assertFalse(dvField.fieldType().stored());
+        assertEquals(1, fields.length);
+        IndexableField field = fields[0];
+        assertEquals(1, field.fieldType().pointIndexDimensionCount());
+        assertEquals(8, field.fieldType().pointNumBytes());
+        assertFalse(field.fieldType().stored());
+        assertEquals("LongField <field:1457654400000>", field.toString());
+        assertEquals(DocValuesType.SORTED_NUMERIC, field.fieldType().docValuesType());
+        assertFalse(field.fieldType().stored());
     }
 
     public void testNotIndexed() throws Exception {
@@ -131,12 +130,11 @@ public void testStore() throws Exception {
         ParsedDocument doc = mapper.parse(source(b -> b.field("field", "2016-03-11")));
 
         IndexableField[] fields = doc.rootDoc().getFields("field");
-        assertEquals(3, fields.length);
-        IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
-        IndexableField dvField = fields[1];
-        assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
-        IndexableField storedField = fields[2];
+        assertEquals(2, fields.length);
+        IndexableField field = fields[0];
+        assertEquals(1, field.fieldType().pointIndexDimensionCount());
+        assertEquals(DocValuesType.SORTED_NUMERIC, field.fieldType().docValuesType());
+        IndexableField storedField = fields[1];
         assertTrue(storedField.fieldType().stored());
         assertEquals(1457654400000L, storedField.numericValue().longValue());
     }
@@ -169,9 +167,9 @@ public void testChangeFormat() throws IOException {
         ParsedDocument doc = mapper.parse(source(b -> b.field("field", 1457654400)));
 
         IndexableField[] fields = doc.rootDoc().getFields("field");
-        assertEquals(2, fields.length);
-        IndexableField pointField = fields[0];
-        assertEquals(1457654400000L, pointField.numericValue().longValue());
+        assertEquals(1, fields.length);
+        IndexableField field = fields[0];
+        assertEquals(1457654400000L, field.numericValue().longValue());
     }
 
     public void testChangeLocale() throws IOException {
@@ -193,16 +191,14 @@ public void testNullValue() throws IOException {
 
         doc = mapper.parse(source(b -> b.nullField("field")));
         IndexableField[] fields = doc.rootDoc().getFields("field");
-        assertEquals(2, fields.length);
-        IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
-        assertEquals(8, pointField.fieldType().pointNumBytes());
-        assertFalse(pointField.fieldType().stored());
-        assertEquals(1457654400000L, pointField.numericValue().longValue());
-        IndexableField dvField = fields[1];
-        assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
-        assertEquals(1457654400000L, dvField.numericValue().longValue());
-        assertFalse(dvField.fieldType().stored());
+        assertEquals(1, fields.length);
+        IndexableField field = fields[0];
+        assertEquals(1, field.fieldType().pointIndexDimensionCount());
+        assertEquals(8, field.fieldType().pointNumBytes());
+        assertEquals("LongField <field:1457654400000>", field.toString());
+        assertEquals(DocValuesType.SORTED_NUMERIC, field.fieldType().docValuesType());
+        assertEquals(1457654400000L, field.numericValue().longValue());
+        assertFalse(field.fieldType().stored());
     }
 
     public void testNanosNullValue() throws IOException {
@@ -220,16 +216,13 @@ public void testNanosNullValue() throws IOException {
 
         doc = mapperService.documentMapper().parse(source(b -> b.nullField("field")));
         IndexableField[] fields = doc.rootDoc().getFields("field");
-        assertEquals(2, fields.length);
-        IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
-        assertEquals(8, pointField.fieldType().pointNumBytes());
-        assertFalse(pointField.fieldType().stored());
-        assertEquals(expectedNullValue, pointField.numericValue().longValue());
-        IndexableField dvField = fields[1];
-        assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
-        assertEquals(expectedNullValue, dvField.numericValue().longValue());
-        assertFalse(dvField.fieldType().stored());
+        assertEquals(1, fields.length);
+        IndexableField field = fields[0];
+        assertEquals(1, field.fieldType().pointIndexDimensionCount());
+        assertEquals(8, field.fieldType().pointNumBytes());
+        assertEquals(DocValuesType.SORTED_NUMERIC, field.fieldType().docValuesType());
+        assertEquals(expectedNullValue, field.numericValue().longValue());
+        assertFalse(field.fieldType().stored());
     }
 
     public void testBadNullValue() throws IOException {
@@ -272,7 +265,7 @@ public void testTimeZoneParsing() throws Exception {
         ParsedDocument doc = mapper.parse(source(b -> b.field("field", formatter.format(randomDate))));
 
         IndexableField[] fields = doc.rootDoc().getFields("field");
-        assertEquals(2, fields.length);
+        assertEquals(1, fields.length);
 
         long millis = randomDate.withZoneSameInstant(ZoneOffset.UTC).toInstant().toEpochMilli();
         assertEquals(millis, fields[0].numericValue().longValue());
@@ -531,7 +524,7 @@ private MapperService dateNanosMapperService() throws IOException {
     private String randomIs8601Nanos(long maxMillis) {
         String date = DateFieldMapper.DEFAULT_DATE_TIME_NANOS_FORMATTER.formatMillis(randomLongBetween(0, maxMillis));
         date = date.substring(0, date.length() - 1);  // Strip off trailing "Z"
-        return date + formatted("%06d", between(0, 999999)) + "Z";  // Add nanos and the "Z"
+        return date + Strings.format("%06d", between(0, 999999)) + "Z";  // Add nanos and the "Z"
     }
 
     private String randomDecimalNanos(long maxMillis) {
@@ -669,11 +662,12 @@ protected IngestScriptSupport ingestScriptSupport() {
         return new IngestScriptSupport() {
             @Override
             protected DateFieldScript.Factory emptyFieldScript() {
-                return (fieldName, params, searchLookup, formatter) -> ctx -> new DateFieldScript(
+                return (fieldName, params, searchLookup, formatter, onScriptError) -> ctx -> new DateFieldScript(
                     fieldName,
                     params,
                     searchLookup,
                     formatter,
+                    OnScriptError.FAIL,
                     ctx
                 ) {
                     @Override
@@ -683,11 +677,12 @@ public void execute() {}
 
             @Override
             protected DateFieldScript.Factory nonEmptyFieldScript() {
-                return (fieldName, params, searchLookup, formatter) -> ctx -> new DateFieldScript(
+                return (fieldName, params, searchLookup, formatter, onScriptError) -> ctx -> new DateFieldScript(
                     fieldName,
                     params,
                     searchLookup,
                     formatter,
+                    OnScriptError.FAIL,
                     ctx
                 ) {
                     @Override
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java
index 17e3578a3ea1..685dc52eb063 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java
@@ -19,7 +19,7 @@
 import org.elasticsearch.script.DateFieldScript;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.json.JsonXContent;
 
@@ -32,11 +32,12 @@
 import static org.hamcrest.Matchers.equalTo;
 
 public class DateFieldScriptTests extends FieldScriptTestCase<DateFieldScript.Factory> {
-    public static final DateFieldScript.Factory DUMMY = (fieldName, params, lookup, formatter) -> ctx -> new DateFieldScript(
+    public static final DateFieldScript.Factory DUMMY = (fieldName, params, lookup, formatter, onScriptError) -> ctx -> new DateFieldScript(
         fieldName,
         params,
         lookup,
         formatter,
+        OnScriptError.FAIL,
         ctx
     ) {
         @Override
@@ -67,8 +68,9 @@ public void testTooManyValues() throws IOException {
                 DateFieldScript script = new DateFieldScript(
                     "test",
                     Map.of(),
-                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()),
+                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, (ctx, doc) -> null),
                     DateFormatter.forPattern(randomDateFormatterPattern()).withLocale(randomLocale(random())),
+                    OnScriptError.FAIL,
                     reader.leaves().get(0)
                 ) {
                     @Override
@@ -103,8 +105,9 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException {
                 DateFieldScript.LeafFactory leafFactory = fromSource().newFactory(
                     "field",
                     Collections.emptyMap(),
-                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()),
-                    DateFormatter.forPattern("epoch_millis")
+                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, SourceProvider.fromStoredFields()),
+                    DateFormatter.forPattern("epoch_millis"),
+                    OnScriptError.FAIL
                 );
                 DateFieldScript dateFieldScript = leafFactory.newInstance(reader.leaves().get(0));
                 List<Long> results = new ArrayList<>();
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java
index 05f80a812d93..14c48348fcd4 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java
@@ -16,9 +16,9 @@
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.MultiReader;
 import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
+import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.Version;
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java
index 382d113a50bd..f34bcaca292b 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java
@@ -128,7 +128,11 @@ public void testFormat() throws IOException {
 
     public void testFormatDuel() throws IOException {
         DateFormatter formatter = DateFormatter.forPattern(randomDateFormatterPattern()).withLocale(randomLocale(random()));
-        DateScriptFieldType scripted = build(new Script(ScriptType.INLINE, "test", "read_timestamp", Map.of()), formatter);
+        DateScriptFieldType scripted = build(
+            new Script(ScriptType.INLINE, "test", "read_timestamp", Map.of()),
+            formatter,
+            OnScriptError.FAIL
+        );
         DateFieldMapper.DateFieldType indexed = new DateFieldMapper.DateFieldType("test", formatter);
         for (int i = 0; i < 100; i++) {
             long date = randomDate();
@@ -149,7 +153,7 @@ public void testDocValues() throws IOException {
             List<Long> results = new ArrayList<>();
             try (DirectoryReader reader = iw.getReader()) {
                 IndexSearcher searcher = newUnthreadedSearcher(reader);
-                DateScriptFieldType ft = build("add_days", Map.of("days", 1));
+                DateScriptFieldType ft = build("add_days", Map.of("days", 1), OnScriptError.FAIL);
                 DateScriptFieldData ifd = ft.fielddataBuilder(mockFielddataContext()).build(null, null);
                 searcher.search(new MatchAllDocsQuery(), new Collector() {
                     @Override
@@ -381,7 +385,9 @@ public void testTermQuery() throws IOException {
                 assertThat(searcher.count(simpleMappedFieldType().termQuery(1595432181354L, mockContext())), equalTo(1));
                 assertThat(searcher.count(simpleMappedFieldType().termQuery(2595432181354L, mockContext())), equalTo(0));
                 assertThat(
-                    searcher.count(build("add_days", Map.of("days", 1)).termQuery("2020-07-23T15:36:21.354Z", mockContext())),
+                    searcher.count(
+                        build("add_days", Map.of("days", 1), OnScriptError.FAIL).termQuery("2020-07-23T15:36:21.354Z", mockContext())
+                    ),
                     equalTo(1)
                 );
                 checkBadDate(() -> searcher.count(simpleMappedFieldType().termQuery("2020-07-22(-■_■)15:36:21.354Z", mockContext())));
@@ -462,7 +468,11 @@ protected MappedFieldType loopFieldType() {
     }
 
     private DateScriptFieldType coolFormattedFieldType() {
-        return build(simpleMappedFieldType().script, DateFormatter.forPattern("yyyy-MM-dd(-■_■)HH:mm:ss.SSSz||epoch_millis"));
+        return build(
+            simpleMappedFieldType().script,
+            DateFormatter.forPattern("yyyy-MM-dd(-■_■)HH:mm:ss.SSSz||epoch_millis"),
+            OnScriptError.FAIL
+        );
     }
 
     @Override
@@ -470,41 +480,47 @@ protected String typeName() {
         return "date";
     }
 
-    private static DateScriptFieldType build(String code) {
-        return build(code, Map.of());
+    private DateScriptFieldType build(String code) {
+        return build(code, Map.of(), OnScriptError.FAIL);
     }
 
-    private static DateScriptFieldType build(String code, Map<String, Object> params) {
-        return build(new Script(ScriptType.INLINE, "test", code, params), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER);
+    protected DateScriptFieldType build(String code, Map<String, Object> params, OnScriptError onScriptError) {
+        return build(new Script(ScriptType.INLINE, "test", code, params), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, onScriptError);
     }
 
     private static DateFieldScript.Factory factory(Script script) {
         return switch (script.getIdOrCode()) {
-            case "read_timestamp" -> (fieldName, params, lookup, formatter) -> ctx -> new DateFieldScript(
+            case "read_timestamp" -> (fieldName, params, lookup, formatter, onScriptError) -> ctx -> new DateFieldScript(
                 fieldName,
                 params,
                 lookup,
                 formatter,
+                onScriptError,
                 ctx
             ) {
                 @Override
+                @SuppressWarnings("unchecked")
                 public void execute() {
-                    for (Object timestamp : (List<?>) lookup.source().source().get("timestamp")) {
+                    Map<String, Object> source = (Map<String, Object>) this.getParams().get("_source");
+                    for (Object timestamp : (List<?>) source.get("timestamp")) {
                         Parse parse = new Parse(this);
                         emit(parse.parse(timestamp));
                     }
                 }
             };
-            case "add_days" -> (fieldName, params, lookup, formatter) -> ctx -> new DateFieldScript(
+            case "add_days" -> (fieldName, params, lookup, formatter, onScriptError) -> ctx -> new DateFieldScript(
                 fieldName,
                 params,
                 lookup,
                 formatter,
+                onScriptError,
                 ctx
             ) {
                 @Override
+                @SuppressWarnings("unchecked")
                 public void execute() {
-                    for (Object timestamp : (List<?>) lookup.source().source().get("timestamp")) {
+                    Map<String, Object> source = (Map<String, Object>) this.getParams().get("_source");
+                    for (Object timestamp : (List<?>) source.get("timestamp")) {
                         long epoch = (Long) timestamp;
                         ZonedDateTime dt = ZonedDateTime.ofInstant(Instant.ofEpochMilli(epoch), ZoneId.of("UTC"));
                         dt = dt.plus(((Number) params.get("days")).longValue(), ChronoUnit.DAYS);
@@ -512,17 +528,30 @@ public void execute() {
                     }
                 }
             };
-            case "loop" -> (fieldName, params, lookup, formatter) -> {
+            case "loop" -> (fieldName, params, lookup, formatter, onScriptError) -> {
                 // Indicate that this script wants the field call "test", which *is* the name of this field
                 lookup.forkAndTrackFieldReferences("test");
-                throw new IllegalStateException("shoud have thrown on the line above");
+                throw new IllegalStateException("should have thrown on the line above");
+            };
+            case "error" -> (fieldName, params, lookup, formatter, onScriptError) -> ctx -> new DateFieldScript(
+                fieldName,
+                params,
+                lookup,
+                formatter,
+                onScriptError,
+                ctx
+            ) {
+                @Override
+                public void execute() {
+                    throw new RuntimeException("test error");
+                }
             };
             default -> throw new IllegalArgumentException("unsupported script [" + script.getIdOrCode() + "]");
         };
     }
 
-    private static DateScriptFieldType build(Script script, DateFormatter dateTimeFormatter) {
-        return new DateScriptFieldType("test", factory(script), dateTimeFormatter, script, emptyMap());
+    private static DateScriptFieldType build(Script script, DateFormatter dateTimeFormatter, OnScriptError onScriptError) {
+        return new DateScriptFieldType("test", factory(script), dateTimeFormatter, script, emptyMap(), onScriptError);
     }
 
     private static long randomDate() {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateScriptMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateScriptMapperTests.java
index 77672693ec63..6e7338bfe1b2 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DateScriptMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DateScriptMapperTests.java
@@ -26,12 +26,13 @@ public DateFieldScript.LeafFactory newFactory(
                 String fieldName,
                 Map<String, Object> params,
                 SearchLookup searchLookup,
-                DateFormatter formatter
+                DateFormatter formatter,
+                OnScriptError onScriptError
             ) {
                 return new DateFieldScript.LeafFactory() {
                     @Override
                     public DateFieldScript newInstance(LeafReaderContext ctx) {
-                        return new DateFieldScript(fieldName, params, searchLookup, formatter, ctx) {
+                        return new DateFieldScript(fieldName, params, searchLookup, formatter, OnScriptError.FAIL, ctx) {
                             @Override
                             public void execute() {
                                 executor.accept(this);
@@ -73,11 +74,9 @@ protected DateFieldScript.Factory multipleValuesScript() {
 
     @Override
     protected void assertMultipleValues(IndexableField[] fields) {
-        assertEquals(4, fields.length);
-        assertEquals("LongPoint <field:1516729294000>", fields[0].toString());
-        assertEquals("docValuesType=SORTED_NUMERIC<field:1516729294000>", fields[1].toString());
-        assertEquals("LongPoint <field:1516729295000>", fields[2].toString());
-        assertEquals("docValuesType=SORTED_NUMERIC<field:1516729295000>", fields[3].toString());
+        assertEquals(2, fields.length);
+        assertEquals("LongField <field:1516729294000>", fields[0].toString());
+        assertEquals("LongField <field:1516729295000>", fields[1].toString());
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java
index ba3d4d5b8f13..3a172e970222 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java
@@ -110,4 +110,35 @@ public void testSyntheticSourceMany() throws IOException {
             }
         });
     }
+
+    public void testSyntheticSourceManyDoNotHave() throws IOException {
+        MapperService mapper = createMapperService(syntheticSourceMapping(b -> b.startObject("doc").field("type", "integer").endObject()));
+        List<Integer> counts = randomList(2, 10000, () -> randomBoolean() ? null : between(1, Integer.MAX_VALUE));
+        withLuceneIndex(mapper, iw -> {
+            int d = 0;
+            for (Integer c : counts) {
+                int doc = d++;
+                iw.addDocument(mapper.documentMapper().parse(source(b -> {
+                    b.field("doc", doc);
+                    if (c != null) {
+                        b.field(CONTENT_TYPE, c);
+                    }
+                })).rootDoc());
+            }
+        }, reader -> {
+            SourceLoader loader = mapper.mappingLookup().newSourceLoader();
+            assertTrue(loader.requiredStoredFields().isEmpty());
+            for (LeafReaderContext leaf : reader.leaves()) {
+                int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray();
+                SourceLoader.Leaf sourceLoaderLeaf = loader.leaf(leaf.reader(), docIds);
+                LeafStoredFieldLoader storedFieldLoader = StoredFieldLoader.empty().getLoader(leaf, docIds);
+                for (int docId : docIds) {
+                    String source = sourceLoaderLeaf.source(storedFieldLoader, docId).internalSourceRef().utf8ToString();
+                    int doc = (int) JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, source).map().get("doc");
+                    String docCountPart = counts.get(doc) == null ? "" : "\"_doc_count\":" + counts.get(doc) + ",";
+                    assertThat("doc " + docId, source, equalTo("{" + docCountPart + "\"doc\":" + doc + "}"));
+                }
+            }
+        });
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java
index d0b6591738e1..1273f8c8bf6b 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java
@@ -63,7 +63,7 @@ public void testAddFields() throws Exception {
         assertThat(stage1.mappers().getMapper("age"), nullValue());
         assertThat(stage1.mappers().getMapper("obj1.prop1"), nullValue());
         // but merged should
-        DocumentParser documentParser = new DocumentParser(null, null, null, null);
+        DocumentParser documentParser = new DocumentParser(null, null);
         DocumentMapper mergedMapper = new DocumentMapper(documentParser, merged, merged.toCompressedXContent());
         assertThat(mergedMapper.mappers().getMapper("age"), notNullValue());
         assertThat(mergedMapper.mappers().getMapper("obj1.prop1"), notNullValue());
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
index 9f25bd647713..546029039ca5 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
@@ -269,8 +269,8 @@ public void testRuntimeFieldDoesNotShadowObjectChildren() throws IOException {
 
             assertNotNull(doc.rootDoc().getField("timestamp"));
             assertNotNull(doc.rootDoc().getField("_source"));
-            assertThat(doc.rootDoc().getFields("location.lat").length, equalTo(4));
-            assertThat(doc.rootDoc().getFields("location.lon").length, equalTo(4));
+            assertThat(doc.rootDoc().getFields("location.lat").length, equalTo(2));
+            assertThat(doc.rootDoc().getFields("location.lon").length, equalTo(2));
             assertNotNull(doc.rootDoc().getField("concrete"));
             assertNull(doc.rootDoc().getField("country"));
         }
@@ -290,8 +290,8 @@ public void testRuntimeFieldDoesNotShadowObjectChildren() throws IOException {
 
             assertNotNull(doc.rootDoc().getField("timestamp"));
             assertNotNull(doc.rootDoc().getField("_source"));
-            assertThat(doc.rootDoc().getFields("location.lat").length, equalTo(4));
-            assertThat(doc.rootDoc().getFields("location.lon").length, equalTo(4));
+            assertThat(doc.rootDoc().getFields("location.lat").length, equalTo(2));
+            assertThat(doc.rootDoc().getFields("location.lon").length, equalTo(2));
             assertNotNull(doc.rootDoc().getField("concrete"));
             assertNull(doc.rootDoc().getField("country"));
         }
@@ -382,13 +382,10 @@ public void testDotsWithExistingMapper() throws Exception {
         assertNull(doc.dynamicMappingsUpdate()); // no update!
 
         IndexableField[] fields = doc.rootDoc().getFields("foo.bar.baz");
-        assertEquals(6, fields.length);
-        assertEquals(123, fields[0].numericValue());
-        assertEquals("123", fields[1].stringValue());
-        assertEquals(456, fields[2].numericValue());
-        assertEquals("456", fields[3].stringValue());
-        assertEquals(789, fields[4].numericValue());
-        assertEquals("789", fields[5].stringValue());
+        assertEquals(3, fields.length);
+        assertEquals("IntField <foo.bar.baz:123>", fields[0].toString());
+        assertEquals("IntField <foo.bar.baz:456>", fields[1].toString());
+        assertEquals("IntField <foo.bar.baz:789>", fields[2].toString());
     }
 
     public void testDotsWithExistingNestedMapper() throws Exception {
@@ -803,13 +800,13 @@ public void testDynamicLongArrayWithTemplate() throws Exception {
         }));
 
         ParsedDocument doc = mapper.parse(source(b -> b.startArray("foo").value(0).value(1).endArray()));
-        assertEquals(4, doc.rootDoc().getFields("foo").length);
+        assertEquals(2, doc.rootDoc().getFields("foo").length);
     }
 
     public void testDynamicLongArray() throws Exception {
         DocumentMapper mapper = createDocumentMapper(mapping(b -> {}));
         ParsedDocument doc = mapper.parse(source(b -> b.startArray("foo").value(0).value(1).endArray()));
-        assertEquals(4, doc.rootDoc().getFields("foo").length);
+        assertEquals(2, doc.rootDoc().getFields("foo").length);
     }
 
     public void testDynamicFalseLongArray() throws Exception {
@@ -891,7 +888,7 @@ public void testMappedGeoPointArray() throws Exception {
     public void testMappedLongArray() throws Exception {
         DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "long")));
         ParsedDocument doc = mapper.parse(source(b -> b.startArray("field").value(0).value(1).endArray()));
-        assertEquals(4, doc.rootDoc().getFields("field").length);
+        assertEquals(2, doc.rootDoc().getFields("field").length);
     }
 
     public void testDynamicObjectWithTemplate() throws Exception {
@@ -1033,7 +1030,7 @@ public void testDynamicDottedFieldNameLongArray() throws Exception {
         DocumentMapper mapper = createDocumentMapper(mapping(b -> {}));
 
         ParsedDocument doc = mapper.parse(source(b -> b.startArray("foo.bar.baz").value(0).value(1).endArray()));
-        assertEquals(4, doc.rootDoc().getFields("foo.bar.baz").length);
+        assertEquals(2, doc.rootDoc().getFields("foo.bar.baz").length);
         Mapper fooMapper = doc.dynamicMappingsUpdate().getRoot().getMapper("foo");
         assertNotNull(fooMapper);
         assertThat(fooMapper, instanceOf(ObjectMapper.class));
@@ -1064,7 +1061,7 @@ public void testDynamicDottedFieldNameLongArrayWithParentTemplate() throws Excep
         }));
 
         ParsedDocument doc = mapper.parse(source(b -> b.startArray("foo.bar.baz").value(0).value(1).endArray()));
-        assertEquals(4, doc.rootDoc().getFields("foo.bar.baz").length);
+        assertEquals(2, doc.rootDoc().getFields("foo.bar.baz").length);
         Mapper fooMapper = doc.dynamicMappingsUpdate().getRoot().getMapper("foo");
         assertNotNull(fooMapper);
         assertThat(fooMapper, instanceOf(ObjectMapper.class));
@@ -1240,7 +1237,7 @@ public void testDynamicDottedFieldNameLongArrayWithExistingParent() throws Excep
         DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "object")));
         ParsedDocument doc = mapper.parse(source(b -> b.startArray("field.bar.baz").value(0).value(1).endArray()));
 
-        assertEquals(4, doc.rootDoc().getFields("field.bar.baz").length);
+        assertEquals(2, doc.rootDoc().getFields("field.bar.baz").length);
         Mapper fooMapper = doc.dynamicMappingsUpdate().getRoot().getMapper("field");
         assertNotNull(fooMapper);
         assertThat(fooMapper, instanceOf(ObjectMapper.class));
@@ -1279,7 +1276,7 @@ public void testDynamicStrictDottedFieldNameLongArray() throws Exception {
     public void testDynamicDottedFieldNameLong() throws Exception {
         DocumentMapper mapper = createDocumentMapper(mapping(b -> {}));
         ParsedDocument doc = mapper.parse(source(b -> b.field("foo.bar.baz", 0)));
-        assertEquals(2, doc.rootDoc().getFields("foo.bar.baz").length);
+        assertEquals(1, doc.rootDoc().getFields("foo.bar.baz").length);
         Mapper fooMapper = doc.dynamicMappingsUpdate().getRoot().getMapper("foo");
         assertNotNull(fooMapper);
         assertThat(fooMapper, instanceOf(ObjectMapper.class));
@@ -1310,7 +1307,7 @@ public void testDynamicDottedFieldNameLongWithParentTemplate() throws Exception
         }));
 
         ParsedDocument doc = mapper.parse(source(b -> b.field("foo.bar.baz", 0)));
-        assertEquals(2, doc.rootDoc().getFields("foo.bar.baz").length);
+        assertEquals(1, doc.rootDoc().getFields("foo.bar.baz").length);
         Mapper fooMapper = doc.dynamicMappingsUpdate().getRoot().getMapper("foo");
         assertNotNull(fooMapper);
         assertThat(fooMapper, instanceOf(ObjectMapper.class));
@@ -1325,7 +1322,7 @@ public void testDynamicDottedFieldNameLongWithParentTemplate() throws Exception
     public void testDynamicDottedFieldNameLongWithExistingParent() throws Exception {
         DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "object")));
         ParsedDocument doc = mapper.parse(source(b -> b.field("field.bar.baz", 0)));
-        assertEquals(2, doc.rootDoc().getFields("field.bar.baz").length);
+        assertEquals(1, doc.rootDoc().getFields("field.bar.baz").length);
         Mapper fooMapper = doc.dynamicMappingsUpdate().getRoot().getMapper("field");
         assertNotNull(fooMapper);
         assertThat(fooMapper, instanceOf(ObjectMapper.class));
@@ -1364,7 +1361,7 @@ public void testDynamicStrictDottedFieldNameLong() throws Exception {
     public void testDynamicDottedFieldNameObject() throws Exception {
         DocumentMapper mapper = createDocumentMapper(mapping(b -> {}));
         ParsedDocument doc = mapper.parse(source(b -> b.startObject("foo.bar.baz").field("a", 0).endObject()));
-        assertEquals(2, doc.rootDoc().getFields("foo.bar.baz.a").length);
+        assertEquals(1, doc.rootDoc().getFields("foo.bar.baz.a").length);
         Mapper fooMapper = doc.dynamicMappingsUpdate().getRoot().getMapper("foo");
         assertNotNull(fooMapper);
         assertThat(fooMapper, instanceOf(ObjectMapper.class));
@@ -1399,7 +1396,7 @@ public void testDynamicDottedFieldNameObjectWithParentTemplate() throws Exceptio
 
         ParsedDocument doc = mapper.parse(source(b -> b.startObject("foo.bar.baz").field("a", 0).endObject()));
 
-        assertEquals(2, doc.rootDoc().getFields("foo.bar.baz.a").length);
+        assertEquals(1, doc.rootDoc().getFields("foo.bar.baz.a").length);
         Mapper fooMapper = doc.dynamicMappingsUpdate().getRoot().getMapper("foo");
         assertNotNull(fooMapper);
         assertThat(fooMapper, instanceOf(ObjectMapper.class));
@@ -1417,7 +1414,7 @@ public void testDynamicDottedFieldNameObjectWithParentTemplate() throws Exceptio
     public void testDynamicDottedFieldNameObjectWithExistingParent() throws Exception {
         DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "object")));
         ParsedDocument doc = mapper.parse(source(b -> b.startObject("field.bar.baz").field("a", 0).endObject()));
-        assertEquals(2, doc.rootDoc().getFields("field.bar.baz.a").length);
+        assertEquals(1, doc.rootDoc().getFields("field.bar.baz.a").length);
         Mapper fooMapper = doc.dynamicMappingsUpdate().getRoot().getMapper("field");
         assertNotNull(fooMapper);
         assertThat(fooMapper, instanceOf(ObjectMapper.class));
@@ -2493,9 +2490,9 @@ same name need to be part  of the same mappings (hence the same document). If th
         assertEquals("foo.bar.baz", baz.name());
         assertEquals("baz", baz.simpleName());
         IndexableField[] fields = doc.rootDoc().getFields("foo.bar.baz");
-        assertEquals(4, fields.length);
-        long[] longs = Arrays.stream(fields).mapToLong(value -> value.numericValue().longValue()).toArray();
-        assertArrayEquals(new long[] { 1, 1, 2, 2 }, longs);
+        assertEquals(2, fields.length);
+        String[] fieldStrings = Arrays.stream(fields).map(Object::toString).toArray(String[]::new);
+        assertArrayEquals(new String[] { "LongField <foo.bar.baz:1>", "LongField <foo.bar.baz:2>" }, fieldStrings);
 
         // merge without going through toXContent and reparsing, otherwise the potential leaf path issue gets fixed on its own
         Mapping newMapping = MapperService.mergeMappings(mapperService.documentMapper(), mapping, MapperService.MergeReason.MAPPING_UPDATE);
@@ -2511,9 +2508,8 @@ same name need to be part  of the same mappings (hence the same document). If th
             """));
         assertNull(doc2.dynamicMappingsUpdate());
         IndexableField[] fields2 = doc2.rootDoc().getFields("foo.bar.baz");
-        assertEquals(2, fields2.length);
-        long[] longs2 = Arrays.stream(fields2).mapToLong(value -> value.numericValue().longValue()).toArray();
-        assertArrayEquals(new long[] { 10, 10 }, longs2);
+        assertEquals(1, fields2.length);
+        assertEquals("LongField <foo.bar.baz:10>", fields2[0].toString());
     }
 
     public void testDeeplyNestedDocument() throws Exception {
@@ -2575,7 +2571,13 @@ protected String contentType() {
 
             @Override
             public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
-                throw new UnsupportedOperationException();
+                return new StringStoredFieldFieldLoader(name(), simpleName(), null) {
+                    @Override
+                    protected void write(XContentBuilder b, Object value) throws IOException {
+                        BytesRef ref = (BytesRef) value;
+                        b.utf8Value(ref.bytes, ref.offset, ref.length);
+                    }
+                };
             }
 
             private static final TypeParser PARSER = new FixedTypeParser(c -> new MockMetadataMapper());
@@ -2604,7 +2606,8 @@ protected RuntimeField createRuntimeField(MappingParserContext parserContext) {
                 protected RuntimeField createChildRuntimeField(
                     MappingParserContext parserContext,
                     String parentName,
-                    Function<SearchLookup, CompositeFieldScript.LeafFactory> parentScriptFactory
+                    Function<SearchLookup, CompositeFieldScript.LeafFactory> parentScriptFactory,
+                    OnScriptError onScriptError
                 ) {
                     throw new UnsupportedOperationException();
                 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java
index 3c16b5acfe51..b6b9dfcfea9f 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java
@@ -105,7 +105,13 @@ protected <T> T compileOtherScript(Script script, ScriptContext<T> context) {
 
             @Override
             protected DoubleFieldScript.Factory emptyFieldScript() {
-                return (fieldName, params, searchLookup) -> ctx -> new DoubleFieldScript(fieldName, params, searchLookup, ctx) {
+                return (fieldName, params, searchLookup, onScriptError) -> ctx -> new DoubleFieldScript(
+                    fieldName,
+                    params,
+                    searchLookup,
+                    OnScriptError.FAIL,
+                    ctx
+                ) {
                     @Override
                     public void execute() {}
                 };
@@ -113,7 +119,13 @@ public void execute() {}
 
             @Override
             protected DoubleFieldScript.Factory nonEmptyFieldScript() {
-                return (fieldName, params, searchLookup) -> ctx -> new DoubleFieldScript(fieldName, params, searchLookup, ctx) {
+                return (fieldName, params, searchLookup, onScriptError) -> ctx -> new DoubleFieldScript(
+                    fieldName,
+                    params,
+                    searchLookup,
+                    OnScriptError.FAIL,
+                    ctx
+                ) {
                     @Override
                     public void execute() {
                         emit(1.0);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java
index 29b68d8f3085..b9d86e3a1629 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java
@@ -18,7 +18,7 @@
 import org.elasticsearch.script.DoubleFieldScript;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.json.JsonXContent;
 
@@ -31,10 +31,11 @@
 import static org.hamcrest.Matchers.equalTo;
 
 public class DoubleFieldScriptTests extends FieldScriptTestCase<DoubleFieldScript.Factory> {
-    public static final DoubleFieldScript.Factory DUMMY = (fieldName, params, lookup) -> ctx -> new DoubleFieldScript(
+    public static final DoubleFieldScript.Factory DUMMY = (fieldName, params, lookup, onScriptError) -> ctx -> new DoubleFieldScript(
         fieldName,
         params,
         lookup,
+        OnScriptError.FAIL,
         ctx
     ) {
         @Override
@@ -65,7 +66,8 @@ public void testTooManyValues() throws IOException {
                 DoubleFieldScript script = new DoubleFieldScript(
                     "test",
                     Map.of(),
-                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()),
+                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, (ctx, doc) -> null),
+                    OnScriptError.FAIL,
                     reader.leaves().get(0)
                 ) {
                     @Override
@@ -100,7 +102,8 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException {
                 DoubleFieldScript.LeafFactory leafFactory = fromSource().newFactory(
                     "field",
                     Collections.emptyMap(),
-                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider())
+                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, SourceProvider.fromStoredFields()),
+                    OnScriptError.FAIL
                 );
                 DoubleFieldScript doubleFieldScript = leafFactory.newInstance(reader.leaves().get(0));
                 List<Double> results = new ArrayList<>();
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java
index bac1115e059c..aaf1fd3a0a83 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java
@@ -64,7 +64,7 @@ public void testDocValues() throws IOException {
             List<Double> results = new ArrayList<>();
             try (DirectoryReader reader = iw.getReader()) {
                 IndexSearcher searcher = newUnthreadedSearcher(reader);
-                DoubleScriptFieldType ft = build("add_param", Map.of("param", 1));
+                DoubleScriptFieldType ft = build("add_param", Map.of("param", 1), OnScriptError.FAIL);
                 DoubleScriptFieldData ifd = ft.fielddataBuilder(mockFielddataContext()).build(null, null);
                 searcher.search(new MatchAllDocsQuery(), new Collector() {
                     @Override
@@ -190,7 +190,10 @@ public void testTermQuery() throws IOException {
                 assertThat(searcher.count(simpleMappedFieldType().termQuery("1", mockContext())), equalTo(1));
                 assertThat(searcher.count(simpleMappedFieldType().termQuery(1, mockContext())), equalTo(1));
                 assertThat(searcher.count(simpleMappedFieldType().termQuery(1.1, mockContext())), equalTo(0));
-                assertThat(searcher.count(build("add_param", Map.of("param", 1)).termQuery(2, mockContext())), equalTo(1));
+                assertThat(
+                    searcher.count(build("add_param", Map.of("param", 1), OnScriptError.FAIL).termQuery(2, mockContext())),
+                    equalTo(1)
+                );
             }
         }
     }
@@ -223,12 +226,12 @@ protected Query randomTermsQuery(MappedFieldType ft, SearchExecutionContext ctx)
 
     @Override
     protected DoubleScriptFieldType simpleMappedFieldType() {
-        return build("read_foo", Map.of());
+        return build("read_foo", Map.of(), OnScriptError.FAIL);
     }
 
     @Override
     protected MappedFieldType loopFieldType() {
-        return build("loop", Map.of());
+        return build("loop", Map.of(), OnScriptError.FAIL);
     }
 
     @Override
@@ -236,38 +239,63 @@ protected String typeName() {
         return "double";
     }
 
-    private static DoubleScriptFieldType build(String code, Map<String, Object> params) {
-        return build(new Script(ScriptType.INLINE, "test", code, params));
+    protected DoubleScriptFieldType build(String code, Map<String, Object> params, OnScriptError onScriptError) {
+        Script script = new Script(ScriptType.INLINE, "test", code, params);
+        return new DoubleScriptFieldType("test", factory(script), script, emptyMap(), onScriptError);
     }
 
     private static DoubleFieldScript.Factory factory(Script script) {
         return switch (script.getIdOrCode()) {
-            case "read_foo" -> (fieldName, params, lookup) -> (ctx) -> new DoubleFieldScript(fieldName, params, lookup, ctx) {
+            case "read_foo" -> (fieldName, params, lookup, onScriptError) -> (ctx) -> new DoubleFieldScript(
+                fieldName,
+                params,
+                lookup,
+                onScriptError,
+                ctx
+            ) {
                 @Override
+                @SuppressWarnings("unchecked")
                 public void execute() {
-                    for (Object foo : (List<?>) lookup.source().source().get("foo")) {
+                    Map<String, Object> source = (Map<String, Object>) this.getParams().get("_source");
+                    for (Object foo : (List<?>) source.get("foo")) {
                         emit(((Number) foo).doubleValue());
                     }
                 }
             };
-            case "add_param" -> (fieldName, params, lookup) -> (ctx) -> new DoubleFieldScript(fieldName, params, lookup, ctx) {
+            case "add_param" -> (fieldName, params, lookup, onScriptError) -> (ctx) -> new DoubleFieldScript(
+                fieldName,
+                params,
+                lookup,
+                onScriptError,
+                ctx
+            ) {
                 @Override
+                @SuppressWarnings("unchecked")
                 public void execute() {
-                    for (Object foo : (List<?>) lookup.source().source().get("foo")) {
+                    Map<String, Object> source = (Map<String, Object>) this.getParams().get("_source");
+                    for (Object foo : (List<?>) source.get("foo")) {
                         emit(((Number) foo).doubleValue() + ((Number) getParams().get("param")).doubleValue());
                     }
                 }
             };
-            case "loop" -> (fieldName, params, lookup) -> {
+            case "loop" -> (fieldName, params, lookup, onScriptError) -> {
                 // Indicate that this script wants the field call "test", which *is* the name of this field
                 lookup.forkAndTrackFieldReferences("test");
-                throw new IllegalStateException("shoud have thrown on the line above");
+                throw new IllegalStateException("should have thrown on the line above");
+            };
+            case "error" -> (fieldName, params, lookup, onScriptError) -> ctx -> new DoubleFieldScript(
+                fieldName,
+                params,
+                lookup,
+                onScriptError,
+                ctx
+            ) {
+                @Override
+                public void execute() {
+                    throw new RuntimeException("test error");
+                }
             };
             default -> throw new IllegalArgumentException("unsupported script [" + script.getIdOrCode() + "]");
         };
     }
-
-    private static DoubleScriptFieldType build(Script script) {
-        return new DoubleScriptFieldType("test", factory(script), script, emptyMap());
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptMapperTests.java
index 6131f7473f9a..2985f4ae8931 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptMapperTests.java
@@ -21,11 +21,16 @@ public class DoubleScriptMapperTests extends MapperScriptTestCase<DoubleFieldScr
     private static DoubleFieldScript.Factory factory(Consumer<DoubleFieldScript> executor) {
         return new DoubleFieldScript.Factory() {
             @Override
-            public DoubleFieldScript.LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup) {
+            public DoubleFieldScript.LeafFactory newFactory(
+                String fieldName,
+                Map<String, Object> params,
+                SearchLookup searchLookup,
+                OnScriptError onScriptError
+            ) {
                 return new DoubleFieldScript.LeafFactory() {
                     @Override
                     public DoubleFieldScript newInstance(LeafReaderContext ctx) {
-                        return new DoubleFieldScript(fieldName, params, searchLookup, ctx) {
+                        return new DoubleFieldScript(fieldName, params, searchLookup, OnScriptError.FAIL, ctx) {
                             @Override
                             public void execute() {
                                 executor.accept(this);
@@ -67,11 +72,9 @@ protected DoubleFieldScript.Factory multipleValuesScript() {
 
     @Override
     protected void assertMultipleValues(IndexableField[] fields) {
-        assertEquals(4, fields.length);
-        assertEquals("DoublePoint <field:3.14>", fields[0].toString());
-        assertEquals("docValuesType=SORTED_NUMERIC<field:4614253070214989087>", fields[1].toString());
-        assertEquals("DoublePoint <field:2.78>", fields[2].toString());
-        assertEquals("docValuesType=SORTED_NUMERIC<field:4613442422282062397>", fields[3].toString());
+        assertEquals(2, fields.length);
+        assertEquals("DoubleField <field:3.14>", fields[0].toString());
+        assertEquals("DoubleField <field:2.78>", fields[1].toString());
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
index 56bddfa16407..ab7e5588d874 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
@@ -887,7 +887,7 @@ public void testArraysOfObjectsDynamicFalse() throws IOException {
             b.field("myfield", 2);
         }));
 
-        assertThat(doc.rootDoc().getFields("myfield"), arrayWithSize(2));
+        assertThat(doc.rootDoc().getFields("myfield"), arrayWithSize(1));
         assertThat(doc.rootDoc().getFields("objects.subfield"), arrayWithSize(2));
         assertThat(doc.rootDoc().getFields("objects.unmapped"), arrayWithSize(0));
         assertEquals(XContentHelper.stripWhitespace("""
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java
index bb8f244740e0..8d15d754a39c 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.script.GeoPointFieldScript;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
 
 import java.io.IOException;
 import java.util.List;
@@ -26,10 +25,11 @@
 import static org.hamcrest.Matchers.equalTo;
 
 public class GeoPointFieldScriptTests extends FieldScriptTestCase<GeoPointFieldScript.Factory> {
-    public static final GeoPointFieldScript.Factory DUMMY = (fieldName, params, lookup) -> ctx -> new GeoPointFieldScript(
+    public static final GeoPointFieldScript.Factory DUMMY = (fieldName, params, lookup, onScriptError) -> ctx -> new GeoPointFieldScript(
         fieldName,
         params,
         lookup,
+        OnScriptError.FAIL,
         ctx
     ) {
         @Override
@@ -60,7 +60,8 @@ public void testTooManyValues() throws IOException {
                 GeoPointFieldScript script = new GeoPointFieldScript(
                     "test",
                     Map.of(),
-                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()),
+                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, (ctx, doc) -> null),
+                    OnScriptError.FAIL,
                     reader.leaves().get(0)
                 ) {
                     @Override
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java
index e740e438bbfc..bd5aed2a6511 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.MultiValueMode;
+import org.elasticsearch.search.lookup.Source;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -64,7 +65,7 @@ public void testDocValues() throws IOException {
             List<Object> results = new ArrayList<>();
             try (DirectoryReader reader = iw.getReader()) {
                 IndexSearcher searcher = newUnthreadedSearcher(reader);
-                GeoPointScriptFieldType ft = build("fromLatLon", Map.of());
+                GeoPointScriptFieldType ft = build("fromLatLon", Map.of(), OnScriptError.FAIL);
                 GeoPointScriptFieldData ifd = ft.fielddataBuilder(mockFielddataContext()).build(null, null);
                 searcher.search(new MatchAllDocsQuery(), new Collector() {
                     @Override
@@ -109,16 +110,16 @@ public void testFetch() throws IOException {
                 {"foo": {"lat": 45.0, "lon" : 45.0}}"""))));
             try (DirectoryReader reader = iw.getReader()) {
                 SearchExecutionContext searchContext = mockContext(true, simpleMappedFieldType());
-                searchContext.lookup().source().setSegmentAndDocument(reader.leaves().get(0), 0);
+                Source source = searchContext.lookup().getSource(reader.leaves().get(0), 0);
                 ValueFetcher fetcher = simpleMappedFieldType().valueFetcher(searchContext, randomBoolean() ? null : "geojson");
                 fetcher.setNextReader(reader.leaves().get(0));
                 assertThat(
-                    fetcher.fetchValues(searchContext.lookup().source(), 0, null),
+                    fetcher.fetchValues(source, 0, null),
                     equalTo(List.of(Map.of("type", "Point", "coordinates", List.of(45.0, 45.0))))
                 );
                 fetcher = simpleMappedFieldType().valueFetcher(searchContext, "wkt");
                 fetcher.setNextReader(reader.leaves().get(0));
-                assertThat(fetcher.fetchValues(searchContext.lookup().source(), 0, null), equalTo(List.of("POINT (45.0 45.0)")));
+                assertThat(fetcher.fetchValues(source, 0, null), equalTo(List.of("POINT (45.0 45.0)")));
             }
         }
     }
@@ -213,12 +214,12 @@ protected Query randomTermsQuery(MappedFieldType ft, SearchExecutionContext ctx)
 
     @Override
     protected GeoPointScriptFieldType simpleMappedFieldType() {
-        return build("fromLatLon", Map.of());
+        return build("fromLatLon", Map.of(), OnScriptError.FAIL);
     }
 
     @Override
     protected MappedFieldType loopFieldType() {
-        return build("loop", Map.of());
+        return build("loop", Map.of(), OnScriptError.FAIL);
     }
 
     @Override
@@ -226,29 +227,46 @@ protected String typeName() {
         return "geo_point";
     }
 
-    private static GeoPointScriptFieldType build(String code, Map<String, Object> params) {
-        return build(new Script(ScriptType.INLINE, "test", code, params));
+    protected GeoPointScriptFieldType build(String code, Map<String, Object> params, OnScriptError onScriptError) {
+        Script script = new Script(ScriptType.INLINE, "test", code, params);
+        return new GeoPointScriptFieldType("test", factory(script), script, emptyMap(), onScriptError);
     }
 
     private static GeoPointFieldScript.Factory factory(Script script) {
         return switch (script.getIdOrCode()) {
-            case "fromLatLon" -> (fieldName, params, lookup) -> (ctx) -> new GeoPointFieldScript(fieldName, params, lookup, ctx) {
+            case "fromLatLon" -> (fieldName, params, lookup, onScriptError) -> (ctx) -> new GeoPointFieldScript(
+                fieldName,
+                params,
+                lookup,
+                onScriptError,
+                ctx
+            ) {
                 @Override
+                @SuppressWarnings("unchecked")
                 public void execute() {
-                    Map<?, ?> foo = (Map<?, ?>) lookup.source().source().get("foo");
+                    Map<String, Object> source = (Map<String, Object>) this.getParams().get("_source");
+                    Map<?, ?> foo = (Map<?, ?>) source.get("foo");
                     emit(((Number) foo.get("lat")).doubleValue(), ((Number) foo.get("lon")).doubleValue());
                 }
             };
-            case "loop" -> (fieldName, params, lookup) -> {
+            case "loop" -> (fieldName, params, lookup, onScriptError) -> {
                 // Indicate that this script wants the field call "test", which *is* the name of this field
                 lookup.forkAndTrackFieldReferences("test");
-                throw new IllegalStateException("shoud have thrown on the line above");
+                throw new IllegalStateException("should have thrown on the line above");
+            };
+            case "error" -> (fieldName, params, lookup, onScriptError) -> ctx -> new GeoPointFieldScript(
+                fieldName,
+                params,
+                lookup,
+                onScriptError,
+                ctx
+            ) {
+                @Override
+                public void execute() {
+                    throw new RuntimeException("test error");
+                }
             };
             default -> throw new IllegalArgumentException("unsupported script [" + script.getIdOrCode() + "]");
         };
     }
-
-    private static GeoPointScriptFieldType build(Script script) {
-        return new GeoPointScriptFieldType("test", factory(script), script, emptyMap());
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptMapperTests.java
index 70bbf0672daa..5ef7cebbf4a1 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptMapperTests.java
@@ -21,11 +21,16 @@ public class GeoPointScriptMapperTests extends MapperScriptTestCase<GeoPointFiel
     private static GeoPointFieldScript.Factory factory(Consumer<GeoPointFieldScript.Emit> executor) {
         return new GeoPointFieldScript.Factory() {
             @Override
-            public GeoPointFieldScript.LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup) {
+            public GeoPointFieldScript.LeafFactory newFactory(
+                String fieldName,
+                Map<String, Object> params,
+                SearchLookup searchLookup,
+                OnScriptError onScriptError
+            ) {
                 return new GeoPointFieldScript.LeafFactory() {
                     @Override
                     public GeoPointFieldScript newInstance(LeafReaderContext ctx) {
-                        return new GeoPointFieldScript(fieldName, params, searchLookup, ctx) {
+                        return new GeoPointFieldScript(fieldName, params, searchLookup, OnScriptError.FAIL, ctx) {
                             @Override
                             public void execute() {
                                 executor.accept(new Emit(this));
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java
index 9fdadd443617..e54a31d4941d 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java
@@ -14,7 +14,8 @@
 import org.apache.lucene.search.IndexSearcher;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -64,20 +65,15 @@ public void testFetchIgnoredFieldValue() throws IOException {
             mapperService,
             iw -> { iw.addDocument(mapperService.documentMapper().parse(source(b -> b.field("field", "value"))).rootDoc()); },
             iw -> {
-                SearchLookup lookup = new SearchLookup(
-                    mapperService::fieldType,
-                    fieldDataLookup(mapperService),
-                    new SourceLookup.ReaderSourceProvider()
-                );
+                SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup(mapperService), (ctx, doc) -> null);
                 SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class);
                 when(searchExecutionContext.lookup()).thenReturn(lookup);
                 IgnoredFieldMapper.IgnoredFieldType ft = (IgnoredFieldMapper.IgnoredFieldType) mapperService.fieldType("_ignored");
                 ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null);
                 IndexSearcher searcher = newSearcher(iw);
                 LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
-                lookup.source().setSegmentAndDocument(context, 0);
                 valueFetcher.setNextReader(context);
-                assertEquals(List.of("field"), valueFetcher.fetchValues(lookup.source(), 0, new ArrayList<>()));
+                assertEquals(List.of("field"), valueFetcher.fetchValues(Source.empty(XContentType.JSON), 0, new ArrayList<>()));
             }
         );
     }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java
index e069c96fbbc6..d461fac38049 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java
@@ -12,7 +12,6 @@
 import org.apache.lucene.search.IndexSearcher;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -60,18 +59,13 @@ public void testFetchFieldValue() throws IOException {
             iw.addDocument(mapperService.documentMapper().parse(source).rootDoc());
         }, iw -> {
             IndexFieldMapper.IndexFieldType ft = (IndexFieldMapper.IndexFieldType) mapperService.fieldType("_index");
-            SearchLookup lookup = new SearchLookup(
-                mapperService::fieldType,
-                fieldDataLookup(mapperService),
-                new SourceLookup.ReaderSourceProvider()
-            );
+            SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup(mapperService), (ctx, doc) -> null);
             SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService);
             ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null);
             IndexSearcher searcher = newSearcher(iw);
             LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
-            lookup.source().setSegmentAndDocument(context, 0);
             valueFetcher.setNextReader(context);
-            assertEquals(List.of(index), valueFetcher.fetchValues(lookup.source(), 0, Collections.emptyList()));
+            assertEquals(List.of(index), valueFetcher.fetchValues(lookup.getSource(context, 0), 0, Collections.emptyList()));
         });
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexTimeScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexTimeScriptTests.java
index c2f94713ba7b..59f86556ab32 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IndexTimeScriptTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexTimeScriptTests.java
@@ -37,9 +37,8 @@ public void testSimpleFieldReference() throws IOException {
 
         ParsedDocument doc = mapper.parse(source(b -> b.field("message", "this is some text")));
         IndexableField[] lengthFields = doc.rootDoc().getFields("message_length");
-        assertEquals(2, lengthFields.length);
-        assertEquals("LongPoint <message_length:17>", lengthFields[0].toString());
-        assertEquals("docValuesType=SORTED_NUMERIC<message_length:17>", lengthFields[1].toString());
+        assertEquals(1, lengthFields.length);
+        assertEquals("LongField <message_length:17>", lengthFields[0].toString());
     }
 
     public void testDocAccess() throws IOException {
@@ -65,7 +64,7 @@ public void testDoublesAccess() throws IOException {
         }));
 
         ParsedDocument doc = mapper.parse(source(b -> b.field("double_field", 4.5)));
-        assertEquals(doc.rootDoc().getField("double_field_plus_two").numericValue(), 6.5);
+        assertEquals(doc.rootDoc().getField("double_field_plus_two").toString(), "DoubleField <double_field_plus_two:6.5>");
     }
 
     public void testCrossReferences() throws IOException {
@@ -85,9 +84,9 @@ public void testCrossReferences() throws IOException {
             b.endObject();
         }));
         ParsedDocument doc = mapper.parse(source(b -> b.field("message", "this is a message")));
-        assertEquals(doc.rootDoc().getField("message_length_plus_two").numericValue(), 19L);
-        assertEquals(doc.rootDoc().getField("message_length").numericValue(), 17L);
-        assertEquals(doc.rootDoc().getField("message_length_plus_four").numericValue(), 21d);
+        assertEquals(doc.rootDoc().getField("message_length_plus_two").toString(), "LongField <message_length_plus_two:19>");
+        assertEquals(doc.rootDoc().getField("message_length").toString(), "LongField <message_length:17>");
+        assertEquals(doc.rootDoc().getField("message_length_plus_four").toString(), "DoubleField <message_length_plus_four:21.0>");
     }
 
     public void testLoopDetection() throws IOException {
@@ -128,7 +127,7 @@ public void testCannotReferToRuntimeFields() throws IOException {
     @SuppressWarnings("unchecked")
     protected <T> T compileScript(Script script, ScriptContext<T> context) {
         if (context.factoryClazz == LongFieldScript.Factory.class) {
-            return (T) (LongFieldScript.Factory) (n, p, l) -> ctx -> new TestLongFieldScript(
+            return (T) (LongFieldScript.Factory) (n, p, l, onScriptError) -> ctx -> new TestLongFieldScript(
                 n,
                 p,
                 l,
@@ -137,7 +136,7 @@ protected <T> T compileScript(Script script, ScriptContext<T> context) {
             );
         }
         if (context.factoryClazz == DoubleFieldScript.Factory.class) {
-            return (T) (DoubleFieldScript.Factory) (n, p, l) -> ctx -> new TestDoubleFieldScript(
+            return (T) (DoubleFieldScript.Factory) (n, p, l, onScriptError) -> ctx -> new TestDoubleFieldScript(
                 n,
                 p,
                 l,
@@ -195,7 +194,7 @@ private static class TestLongFieldScript extends LongFieldScript {
             LeafReaderContext ctx,
             Consumer<TestLongFieldScript> executor
         ) {
-            super(fieldName, params, searchLookup, ctx);
+            super(fieldName, params, searchLookup, OnScriptError.FAIL, ctx);
             this.executor = executor;
         }
 
@@ -224,7 +223,7 @@ private static class TestDoubleFieldScript extends DoubleFieldScript {
             LeafReaderContext ctx,
             Consumer<TestDoubleFieldScript> executor
         ) {
-            super(fieldName, params, searchLookup, ctx);
+            super(fieldName, params, searchLookup, OnScriptError.FAIL, ctx);
             this.executor = executor;
         }
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
index ddf8b0a3cb03..aa69051fe93d 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
@@ -388,7 +388,13 @@ protected IngestScriptSupport ingestScriptSupport() {
         return new IngestScriptSupport() {
             @Override
             protected IpFieldScript.Factory emptyFieldScript() {
-                return (fieldName, params, searchLookup) -> ctx -> new IpFieldScript(fieldName, params, searchLookup, ctx) {
+                return (fieldName, params, searchLookup, onScriptError) -> ctx -> new IpFieldScript(
+                    fieldName,
+                    params,
+                    searchLookup,
+                    OnScriptError.FAIL,
+                    ctx
+                ) {
                     @Override
                     public void execute() {}
                 };
@@ -396,7 +402,13 @@ public void execute() {}
 
             @Override
             protected IpFieldScript.Factory nonEmptyFieldScript() {
-                return (fieldName, params, searchLookup) -> ctx -> new IpFieldScript(fieldName, params, searchLookup, ctx) {
+                return (fieldName, params, searchLookup, onScriptError) -> ctx -> new IpFieldScript(
+                    fieldName,
+                    params,
+                    searchLookup,
+                    OnScriptError.FAIL,
+                    ctx
+                ) {
                     @Override
                     public void execute() {
                         emit("192.168.0.1");
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java
index 3b7a35d8971f..5db04a76d417 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java
@@ -18,7 +18,7 @@
 import org.elasticsearch.script.IpFieldScript;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.json.JsonXContent;
 
@@ -32,10 +32,11 @@
 import static org.hamcrest.Matchers.equalTo;
 
 public class IpFieldScriptTests extends FieldScriptTestCase<IpFieldScript.Factory> {
-    public static final IpFieldScript.Factory DUMMY = (fieldName, params, lookup) -> ctx -> new IpFieldScript(
+    public static final IpFieldScript.Factory DUMMY = (fieldName, params, lookup, onScriptError) -> ctx -> new IpFieldScript(
         fieldName,
         params,
         lookup,
+        OnScriptError.FAIL,
         ctx
     ) {
         @Override
@@ -66,7 +67,8 @@ public void testTooManyValues() throws IOException {
                 IpFieldScript script = new IpFieldScript(
                     "test",
                     Map.of(),
-                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()),
+                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, (ctx, doc) -> null),
+                    OnScriptError.FAIL,
                     reader.leaves().get(0)
                 ) {
                     @Override
@@ -101,7 +103,8 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException {
                 IpFieldScript.LeafFactory leafFactory = fromSource().newFactory(
                     "field",
                     Collections.emptyMap(),
-                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider())
+                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, SourceProvider.fromStoredFields()),
+                    OnScriptError.FAIL
                 );
                 IpFieldScript ipFieldScript = leafFactory.newInstance(reader.leaves().get(0));
                 List<InetAddress> results = new ArrayList<>();
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java
index e223753cf6bb..a00a344a0073 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java
@@ -66,7 +66,7 @@ public void testDocValues() throws IOException {
             List<Object> results = new ArrayList<>();
             try (DirectoryReader reader = iw.getReader()) {
                 IndexSearcher searcher = newUnthreadedSearcher(reader);
-                IpScriptFieldType ft = build("append_param", Map.of("param", ".1"));
+                IpScriptFieldType ft = build("append_param", Map.of("param", ".1"), OnScriptError.FAIL);
                 BinaryScriptFieldData ifd = ft.fielddataBuilder(mockFielddataContext()).build(null, null);
                 DocValueFormat format = ft.docValueFormat(null, null);
                 searcher.search(new MatchAllDocsQuery(), new Collector() {
@@ -198,7 +198,7 @@ public void testTermQuery() throws IOException {
             iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"200.0.0\"]}"))));
             try (DirectoryReader reader = iw.getReader()) {
                 IndexSearcher searcher = newUnthreadedSearcher(reader);
-                IpScriptFieldType fieldType = build("append_param", Map.of("param", ".1"));
+                IpScriptFieldType fieldType = build("append_param", Map.of("param", ".1"), OnScriptError.FAIL);
                 assertThat(searcher.count(fieldType.termQuery("192.168.0.1", mockContext())), equalTo(1));
                 assertThat(searcher.count(fieldType.termQuery("192.168.0.7", mockContext())), equalTo(0));
                 assertThat(searcher.count(fieldType.termQuery("192.168.0.0/16", mockContext())), equalTo(2));
@@ -240,12 +240,12 @@ protected Query randomTermsQuery(MappedFieldType ft, SearchExecutionContext ctx)
 
     @Override
     protected IpScriptFieldType simpleMappedFieldType() {
-        return build("read_foo", Map.of());
+        return build("read_foo", Map.of(), OnScriptError.FAIL);
     }
 
     @Override
     protected MappedFieldType loopFieldType() {
-        return build("loop", Map.of());
+        return build("loop", Map.of(), OnScriptError.FAIL);
     }
 
     @Override
@@ -253,38 +253,63 @@ protected String typeName() {
         return "ip";
     }
 
-    private static IpScriptFieldType build(String code, Map<String, Object> params) {
-        return build(new Script(ScriptType.INLINE, "test", code, params));
+    protected IpScriptFieldType build(String code, Map<String, Object> params, OnScriptError onScriptError) {
+        Script script = new Script(ScriptType.INLINE, "test", code, params);
+        return new IpScriptFieldType("test", factory(script), script, emptyMap(), onScriptError);
     }
 
     private static IpFieldScript.Factory factory(Script script) {
         return switch (script.getIdOrCode()) {
-            case "read_foo" -> (fieldName, params, lookup) -> (ctx) -> new IpFieldScript(fieldName, params, lookup, ctx) {
+            case "read_foo" -> (fieldName, params, lookup, onScriptError) -> (ctx) -> new IpFieldScript(
+                fieldName,
+                params,
+                lookup,
+                OnScriptError.FAIL,
+                ctx
+            ) {
                 @Override
+                @SuppressWarnings("unchecked")
                 public void execute() {
-                    for (Object foo : (List<?>) lookup.source().source().get("foo")) {
+                    Map<String, Object> source = (Map<String, Object>) this.getParams().get("_source");
+                    for (Object foo : (List<?>) source.get("foo")) {
                         emit(foo.toString());
                     }
                 }
             };
-            case "append_param" -> (fieldName, params, lookup) -> (ctx) -> new IpFieldScript(fieldName, params, lookup, ctx) {
+            case "append_param" -> (fieldName, params, lookup, onScriptError) -> (ctx) -> new IpFieldScript(
+                fieldName,
+                params,
+                lookup,
+                OnScriptError.FAIL,
+                ctx
+            ) {
                 @Override
+                @SuppressWarnings("unchecked")
                 public void execute() {
-                    for (Object foo : (List<?>) lookup.source().source().get("foo")) {
+                    Map<String, Object> source = (Map<String, Object>) this.getParams().get("_source");
+                    for (Object foo : (List<?>) source.get("foo")) {
                         emit(foo.toString() + getParams().get("param"));
                     }
                 }
             };
-            case "loop" -> (fieldName, params, lookup) -> {
+            case "loop" -> (fieldName, params, lookup, onScriptError) -> {
                 // Indicate that this script wants the field call "test", which *is* the name of this field
                 lookup.forkAndTrackFieldReferences("test");
-                throw new IllegalStateException("shoud have thrown on the line above");
+                throw new IllegalStateException("should have thrown on the line above");
+            };
+            case "error" -> (fieldName, params, lookup, onScriptError) -> ctx -> new IpFieldScript(
+                fieldName,
+                params,
+                lookup,
+                onScriptError,
+                ctx
+            ) {
+                @Override
+                public void execute() {
+                    throw new RuntimeException("test error");
+                }
             };
             default -> throw new IllegalArgumentException("unsupported script [" + script.getIdOrCode() + "]");
         };
     }
-
-    private static IpScriptFieldType build(Script script) {
-        return new IpScriptFieldType("test", factory(script), script, emptyMap());
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptMapperTests.java
index 941e8e100c95..e61e16fedbb7 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptMapperTests.java
@@ -21,11 +21,16 @@ public class IpScriptMapperTests extends MapperScriptTestCase<IpFieldScript.Fact
     private static IpFieldScript.Factory factory(Consumer<IpFieldScript> executor) {
         return new IpFieldScript.Factory() {
             @Override
-            public IpFieldScript.LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup) {
+            public IpFieldScript.LeafFactory newFactory(
+                String fieldName,
+                Map<String, Object> params,
+                SearchLookup searchLookup,
+                OnScriptError onScriptError
+            ) {
                 return new IpFieldScript.LeafFactory() {
                     @Override
                     public IpFieldScript newInstance(LeafReaderContext ctx) {
-                        return new IpFieldScript(fieldName, params, searchLookup, ctx) {
+                        return new IpFieldScript(fieldName, params, searchLookup, OnScriptError.FAIL, ctx) {
                             @Override
                             public void execute() {
                                 executor.accept(this);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
index 877d38b4b1f7..b58cde556f34 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
@@ -631,6 +631,23 @@ public void testKeywordFieldUtf8LongerThan32766() throws Exception {
         assertThat(e.getCause().getMessage(), containsString("UTF8 encoding is longer than the max length"));
     }
 
+    /**
+     * Test that we don't error on exceeding field size if field is neither indexed nor has doc values
+     */
+    public void testKeywordFieldUtf8LongerThan32766SourceOnly() throws Exception {
+        DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
+            b.field("type", "keyword");
+            b.field("index", false);
+            b.field("doc_values", false);
+            b.field("store", false);
+        }));
+        StringBuilder stringBuilder = new StringBuilder(32768);
+        for (int i = 0; i < 32768; i++) {
+            stringBuilder.append("a");
+        }
+        mapper.parse(source(b -> b.field("field", stringBuilder.toString())));
+    }
+
     @Override
     protected boolean supportsIgnoreMalformed() {
         return false;
@@ -729,7 +746,13 @@ protected IngestScriptSupport ingestScriptSupport() {
         return new IngestScriptSupport() {
             @Override
             protected StringFieldScript.Factory emptyFieldScript() {
-                return (fieldName, params, searchLookup) -> ctx -> new StringFieldScript(fieldName, params, searchLookup, ctx) {
+                return (fieldName, params, searchLookup, onScriptError) -> ctx -> new StringFieldScript(
+                    fieldName,
+                    params,
+                    searchLookup,
+                    OnScriptError.FAIL,
+                    ctx
+                ) {
                     @Override
                     public void execute() {}
                 };
@@ -737,7 +760,13 @@ public void execute() {}
 
             @Override
             protected StringFieldScript.Factory nonEmptyFieldScript() {
-                return (fieldName, params, searchLookup) -> ctx -> new StringFieldScript(fieldName, params, searchLookup, ctx) {
+                return (fieldName, params, searchLookup, onScriptError) -> ctx -> new StringFieldScript(
+                    fieldName,
+                    params,
+                    searchLookup,
+                    OnScriptError.FAIL,
+                    ctx
+                ) {
                     @Override
                     public void execute() {
                         emit("foo");
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java
index b43c4b71e0ca..f12e422f329a 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java
@@ -40,7 +40,6 @@
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.script.StringFieldScript;
 import org.elasticsearch.search.MultiValueMode;
-import org.elasticsearch.search.lookup.SourceLookup;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -60,7 +59,7 @@ public void testDocValues() throws IOException {
             List<String> results = new ArrayList<>();
             try (DirectoryReader reader = iw.getReader()) {
                 IndexSearcher searcher = newUnthreadedSearcher(reader);
-                KeywordScriptFieldType ft = build("append_param", Map.of("param", "-suffix"));
+                KeywordScriptFieldType ft = build("append_param", Map.of("param", "-suffix"), OnScriptError.FAIL);
                 StringScriptFieldData ifd = ft.fielddataBuilder(mockFielddataContext()).build(null, null);
                 searcher.search(new MatchAllDocsQuery(), new Collector() {
                     @Override
@@ -287,29 +286,12 @@ public void testTermQuery() throws IOException {
             iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [2]}"))));
             try (DirectoryReader reader = iw.getReader()) {
                 IndexSearcher searcher = newUnthreadedSearcher(reader);
-                KeywordScriptFieldType fieldType = build("append_param", Map.of("param", "-suffix"));
+                KeywordScriptFieldType fieldType = build("append_param", Map.of("param", "-suffix"), OnScriptError.FAIL);
                 assertThat(searcher.count(fieldType.termQuery("1-suffix", mockContext())), equalTo(1));
             }
         }
     }
 
-    public void testSyntheticSourceAccess() throws IOException {
-        try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {
-            iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [1]}"))));
-            iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [2]}"))));
-            try (DirectoryReader reader = iw.getReader()) {
-                IndexSearcher searcher = newUnthreadedSearcher(reader);
-                KeywordScriptFieldType fieldType = build("append_param", Map.of("param", "-suffix"));
-                expectThrows(
-                    IllegalArgumentException.class,
-                    () -> {
-                        searcher.count(fieldType.termQuery("1-suffix", mockContext(true, null, new SourceLookup.NullSourceProvider())));
-                    }
-                );
-            }
-        }
-    }
-
     @Override
     protected Query randomTermQuery(MappedFieldType ft, SearchExecutionContext ctx) {
         return ft.termQuery(randomAlphaOfLengthBetween(1, 1000), ctx);
@@ -375,7 +357,7 @@ public void testMatchQuery() throws IOException {
             iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [2]}"))));
             try (DirectoryReader reader = iw.getReader()) {
                 IndexSearcher searcher = newUnthreadedSearcher(reader);
-                KeywordScriptFieldType fieldType = build("append_param", Map.of("param", "-Suffix"));
+                KeywordScriptFieldType fieldType = build("append_param", Map.of("param", "-Suffix"), OnScriptError.FAIL);
                 SearchExecutionContext searchExecutionContext = mockContext(true, fieldType);
                 Query query = new MatchQueryBuilder("test", "1-Suffix").toQuery(searchExecutionContext);
                 assertThat(searcher.count(query), equalTo(1));
@@ -385,12 +367,12 @@ public void testMatchQuery() throws IOException {
 
     @Override
     protected KeywordScriptFieldType simpleMappedFieldType() {
-        return build("read_foo", Map.of());
+        return build("read_foo", Map.of(), OnScriptError.FAIL);
     }
 
     @Override
     protected KeywordScriptFieldType loopFieldType() {
-        return build("loop", Map.of());
+        return build("loop", Map.of(), OnScriptError.FAIL);
     }
 
     @Override
@@ -398,38 +380,63 @@ protected String typeName() {
         return "keyword";
     }
 
-    private static KeywordScriptFieldType build(String code, Map<String, Object> params) {
-        return build(new Script(ScriptType.INLINE, "test", code, params));
+    protected KeywordScriptFieldType build(String code, Map<String, Object> params, OnScriptError onScriptError) {
+        Script script = new Script(ScriptType.INLINE, "test", code, params);
+        return new KeywordScriptFieldType("test", factory(script), script, emptyMap(), onScriptError);
     }
 
     private static StringFieldScript.Factory factory(Script script) {
         return switch (script.getIdOrCode()) {
-            case "read_foo" -> (fieldName, params, lookup) -> ctx -> new StringFieldScript(fieldName, params, lookup, ctx) {
+            case "read_foo" -> (fieldName, params, lookup, onScriptError) -> ctx -> new StringFieldScript(
+                fieldName,
+                params,
+                lookup,
+                OnScriptError.FAIL,
+                ctx
+            ) {
                 @Override
+                @SuppressWarnings("unchecked")
                 public void execute() {
-                    for (Object foo : (List<?>) lookup.source().source().get("foo")) {
+                    Map<String, Object> source = (Map<String, Object>) this.getParams().get("_source");
+                    for (Object foo : (List<?>) source.get("foo")) {
                         emit(foo.toString());
                     }
                 }
             };
-            case "append_param" -> (fieldName, params, lookup) -> ctx -> new StringFieldScript(fieldName, params, lookup, ctx) {
+            case "append_param" -> (fieldName, params, lookup, onScriptError) -> ctx -> new StringFieldScript(
+                fieldName,
+                params,
+                lookup,
+                OnScriptError.FAIL,
+                ctx
+            ) {
                 @Override
+                @SuppressWarnings("unchecked")
                 public void execute() {
-                    for (Object foo : (List<?>) lookup.source().source().get("foo")) {
+                    Map<String, Object> source = (Map<String, Object>) this.getParams().get("_source");
+                    for (Object foo : (List<?>) source.get("foo")) {
                         emit(foo.toString() + getParams().get("param").toString());
                     }
                 }
             };
-            case "loop" -> (fieldName, params, lookup) -> {
+            case "loop" -> (fieldName, params, lookup, onScriptError) -> {
                 // Indicate that this script wants the field call "test", which *is* the name of this field
                 lookup.forkAndTrackFieldReferences("test");
-                throw new IllegalStateException("shoud have thrown on the line above");
+                throw new IllegalStateException("should have thrown on the line above");
+            };
+            case "error" -> (fieldName, params, lookup, onScriptError) -> ctx -> new StringFieldScript(
+                fieldName,
+                params,
+                lookup,
+                onScriptError,
+                ctx
+            ) {
+                @Override
+                public void execute() {
+                    throw new RuntimeException("test error");
+                }
             };
             default -> throw new IllegalArgumentException("unsupported script [" + script.getIdOrCode() + "]");
         };
     }
-
-    private static KeywordScriptFieldType build(Script script) {
-        return new KeywordScriptFieldType("test", factory(script), script, emptyMap());
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptMapperTests.java
index 8cb8e463451a..15c1909fc835 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptMapperTests.java
@@ -22,11 +22,16 @@ public class KeywordScriptMapperTests extends MapperScriptTestCase<StringFieldSc
     private static StringFieldScript.Factory factory(Consumer<StringFieldScript> executor) {
         return new StringFieldScript.Factory() {
             @Override
-            public StringFieldScript.LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup) {
+            public StringFieldScript.LeafFactory newFactory(
+                String fieldName,
+                Map<String, Object> params,
+                SearchLookup searchLookup,
+                OnScriptError onScriptError
+            ) {
                 return new StringFieldScript.LeafFactory() {
                     @Override
                     public StringFieldScript newInstance(LeafReaderContext ctx) {
-                        return new StringFieldScript(fieldName, params, searchLookup, ctx) {
+                        return new StringFieldScript(fieldName, params, searchLookup, OnScriptError.FAIL, ctx) {
                             @Override
                             public void execute() {
                                 executor.accept(this);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java
index 76264302114b..ac5f96a8f83d 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java
@@ -97,9 +97,9 @@ public void testLongIndexingCoercesIntoRange() throws Exception {
         // the following two strings are in-range for a long after coercion
         DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping));
         ParsedDocument doc = mapper.parse(source(b -> b.field("field", "9223372036854775807.9")));
-        assertThat(doc.rootDoc().getFields("field"), arrayWithSize(2));
+        assertThat(doc.rootDoc().getFields("field"), arrayWithSize(1));
         doc = mapper.parse(source(b -> b.field("field", "-9223372036854775808.9")));
-        assertThat(doc.rootDoc().getFields("field"), arrayWithSize(2));
+        assertThat(doc.rootDoc().getFields("field"), arrayWithSize(1));
     }
 
     @Override
@@ -132,7 +132,13 @@ protected <T> T compileOtherScript(Script script, ScriptContext<T> context) {
 
             @Override
             protected LongFieldScript.Factory emptyFieldScript() {
-                return (fieldName, params, searchLookup) -> ctx -> new LongFieldScript(fieldName, params, searchLookup, ctx) {
+                return (fieldName, params, searchLookup, onScriptError) -> ctx -> new LongFieldScript(
+                    fieldName,
+                    params,
+                    searchLookup,
+                    OnScriptError.FAIL,
+                    ctx
+                ) {
                     @Override
                     public void execute() {}
                 };
@@ -140,7 +146,13 @@ public void execute() {}
 
             @Override
             protected LongFieldScript.Factory nonEmptyFieldScript() {
-                return (fieldName, params, searchLookup) -> ctx -> new LongFieldScript(fieldName, params, searchLookup, ctx) {
+                return (fieldName, params, searchLookup, onScriptError) -> ctx -> new LongFieldScript(
+                    fieldName,
+                    params,
+                    searchLookup,
+                    OnScriptError.FAIL,
+                    ctx
+                ) {
                     @Override
                     public void execute() {
                         emit(1);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java
index 7aab950f0eaa..756d0699826a 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java
@@ -18,7 +18,7 @@
 import org.elasticsearch.script.LongFieldScript;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.json.JsonXContent;
 
@@ -31,10 +31,11 @@
 import static org.hamcrest.Matchers.equalTo;
 
 public class LongFieldScriptTests extends FieldScriptTestCase<LongFieldScript.Factory> {
-    public static final LongFieldScript.Factory DUMMY = (fieldName, params, lookup) -> ctx -> new LongFieldScript(
+    public static final LongFieldScript.Factory DUMMY = (fieldName, params, lookup, onScriptError) -> ctx -> new LongFieldScript(
         fieldName,
         params,
         lookup,
+        OnScriptError.FAIL,
         ctx
     ) {
         @Override
@@ -65,7 +66,8 @@ public void testTooManyValues() throws IOException {
                 LongFieldScript script = new LongFieldScript(
                     "test",
                     Map.of(),
-                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()),
+                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, (ctx, doc) -> null),
+                    OnScriptError.FAIL,
                     reader.leaves().get(0)
                 ) {
                     @Override
@@ -100,7 +102,8 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException {
                 LongFieldScript.LeafFactory leafFactory = fromSource().newFactory(
                     "field",
                     Collections.emptyMap(),
-                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider())
+                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, SourceProvider.fromStoredFields()),
+                    OnScriptError.FAIL
                 );
                 LongFieldScript longFieldScript = leafFactory.newInstance(reader.leaves().get(0));
                 List<Long> results = new ArrayList<>();
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java
index a84f0d658d6e..7e044f02779a 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java
@@ -76,7 +76,7 @@ public void testDocValues() throws IOException {
             List<Long> results = new ArrayList<>();
             try (DirectoryReader reader = iw.getReader()) {
                 IndexSearcher searcher = newUnthreadedSearcher(reader);
-                LongScriptFieldType ft = build("add_param", Map.of("param", 1));
+                LongScriptFieldType ft = build("add_param", Map.of("param", 1), OnScriptError.FAIL);
                 LongScriptFieldData ifd = ft.fielddataBuilder(mockFielddataContext()).build(null, null);
                 searcher.search(new MatchAllDocsQuery(), new Collector() {
                     @Override
@@ -132,7 +132,8 @@ public void testNow() throws IOException {
             iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181356]}"))));
             try (DirectoryReader reader = iw.getReader()) {
                 IndexSearcher searcher = newUnthreadedSearcher(reader);
-                LongScriptFieldData ifd = build("millis_ago", Map.of()).fielddataBuilder(mockFielddataContext()).build(null, null);
+                LongScriptFieldData ifd = build("millis_ago", Map.of(), OnScriptError.FAIL).fielddataBuilder(mockFielddataContext())
+                    .build(null, null);
                 SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false);
                 TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf));
                 assertThat(readSource(reader, docs.scoreDocs[0].doc), equalTo("{\"timestamp\": [1595432181356]}"));
@@ -222,7 +223,10 @@ public void testTermQuery() throws IOException {
                 assertThat(searcher.count(simpleMappedFieldType().termQuery("1", mockContext())), equalTo(1));
                 assertThat(searcher.count(simpleMappedFieldType().termQuery(1, mockContext())), equalTo(1));
                 assertThat(searcher.count(simpleMappedFieldType().termQuery(1.1, mockContext())), equalTo(0));
-                assertThat(searcher.count(build("add_param", Map.of("param", 1)).termQuery(2, mockContext())), equalTo(1));
+                assertThat(
+                    searcher.count(build("add_param", Map.of("param", 1), OnScriptError.FAIL).termQuery(2, mockContext())),
+                    equalTo(1)
+                );
             }
         }
     }
@@ -255,12 +259,12 @@ protected Query randomTermsQuery(MappedFieldType ft, SearchExecutionContext ctx)
 
     @Override
     protected LongScriptFieldType simpleMappedFieldType() {
-        return build("read_foo", Map.of());
+        return build("read_foo", Map.of(), OnScriptError.FAIL);
     }
 
     @Override
     protected LongScriptFieldType loopFieldType() {
-        return build("loop", Map.of());
+        return build("loop", Map.of(), OnScriptError.FAIL);
     }
 
     @Override
@@ -268,26 +272,43 @@ protected String typeName() {
         return "long";
     }
 
-    private static LongScriptFieldType build(String code, Map<String, Object> params) {
-        return build(new Script(ScriptType.INLINE, "test", code, params));
+    protected LongScriptFieldType build(String code, Map<String, Object> params, OnScriptError onScriptError) {
+        Script script = new Script(ScriptType.INLINE, "test", code, params);
+        return new LongScriptFieldType("test", factory(script), script, emptyMap(), onScriptError);
     }
 
     private static LongFieldScript.Factory factory(Script script) {
         switch (script.getIdOrCode()) {
             case "read_foo":
-                return (fieldName, params, lookup) -> (ctx) -> new LongFieldScript(fieldName, params, lookup, ctx) {
+                return (fieldName, params, lookup, onScriptError) -> (ctx) -> new LongFieldScript(
+                    fieldName,
+                    params,
+                    lookup,
+                    onScriptError,
+                    ctx
+                ) {
                     @Override
+                    @SuppressWarnings("unchecked")
                     public void execute() {
-                        for (Object foo : (List<?>) lookup.source().source().get("foo")) {
+                        Map<String, Object> source = (Map<String, Object>) this.getParams().get("_source");
+                        for (Object foo : (List<?>) source.get("foo")) {
                             emit(((Number) foo).longValue());
                         }
                     }
                 };
             case "add_param":
-                return (fieldName, params, lookup) -> (ctx) -> new LongFieldScript(fieldName, params, lookup, ctx) {
+                return (fieldName, params, lookup, onScriptError) -> (ctx) -> new LongFieldScript(
+                    fieldName,
+                    params,
+                    lookup,
+                    onScriptError,
+                    ctx
+                ) {
                     @Override
+                    @SuppressWarnings("unchecked")
                     public void execute() {
-                        for (Object foo : (List<?>) lookup.source().source().get("foo")) {
+                        Map<String, Object> source = (Map<String, Object>) this.getParams().get("_source");
+                        for (Object foo : (List<?>) source.get("foo")) {
                             emit(((Number) foo).longValue() + ((Number) getParams().get("param")).longValue());
                         }
                     }
@@ -295,26 +316,43 @@ public void execute() {
             case "millis_ago":
                 // Painless actually call System.currentTimeMillis. We could mock the time but this works fine too.
                 long now = System.currentTimeMillis();
-                return (fieldName, params, lookup) -> (ctx) -> new LongFieldScript(fieldName, params, lookup, ctx) {
+                return (fieldName, params, lookup, onScriptError) -> (ctx) -> new LongFieldScript(
+                    fieldName,
+                    params,
+                    lookup,
+                    onScriptError,
+                    ctx
+                ) {
                     @Override
+                    @SuppressWarnings("unchecked")
                     public void execute() {
-                        for (Object timestamp : (List<?>) lookup.source().source().get("timestamp")) {
+                        Map<String, Object> source = (Map<String, Object>) this.getParams().get("_source");
+                        for (Object timestamp : (List<?>) source.get("timestamp")) {
                             emit(now - ((Number) timestamp).longValue());
                         }
                     }
                 };
             case "loop":
-                return (fieldName, params, lookup) -> {
+                return (fieldName, params, lookup, onScriptError) -> {
                     // Indicate that this script wants the field call "test", which *is* the name of this field
                     lookup.forkAndTrackFieldReferences("test");
-                    throw new IllegalStateException("shoud have thrown on the line above");
+                    throw new IllegalStateException("should have thrown on the line above");
+                };
+            case "error":
+                return (fieldName, params, lookup, onScriptError) -> ctx -> new LongFieldScript(
+                    fieldName,
+                    params,
+                    lookup,
+                    onScriptError,
+                    ctx
+                ) {
+                    @Override
+                    public void execute() {
+                        throw new RuntimeException("test error");
+                    }
                 };
             default:
                 throw new IllegalArgumentException("unsupported script [" + script.getIdOrCode() + "]");
         }
     }
-
-    private static LongScriptFieldType build(Script script) {
-        return new LongScriptFieldType("test", factory(script), script, emptyMap());
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongScriptMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongScriptMapperTests.java
index 6eb670d70c5c..e35d907273b4 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/LongScriptMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/LongScriptMapperTests.java
@@ -21,11 +21,16 @@ public class LongScriptMapperTests extends MapperScriptTestCase<LongFieldScript.
     private static LongFieldScript.Factory factory(Consumer<LongFieldScript> executor) {
         return new LongFieldScript.Factory() {
             @Override
-            public LongFieldScript.LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup) {
+            public LongFieldScript.LeafFactory newFactory(
+                String fieldName,
+                Map<String, Object> params,
+                SearchLookup searchLookup,
+                OnScriptError onScriptError
+            ) {
                 return new LongFieldScript.LeafFactory() {
                     @Override
                     public LongFieldScript newInstance(LeafReaderContext ctx) {
-                        return new LongFieldScript(fieldName, params, searchLookup, ctx) {
+                        return new LongFieldScript(fieldName, params, searchLookup, OnScriptError.FAIL, ctx) {
                             @Override
                             public void execute() {
                                 executor.accept(this);
@@ -67,11 +72,9 @@ protected LongFieldScript.Factory multipleValuesScript() {
 
     @Override
     protected void assertMultipleValues(IndexableField[] fields) {
-        assertEquals(4, fields.length);
-        assertEquals("LongPoint <field:1>", fields[0].toString());
-        assertEquals("docValuesType=SORTED_NUMERIC<field:1>", fields[1].toString());
-        assertEquals("LongPoint <field:2>", fields[2].toString());
-        assertEquals("docValuesType=SORTED_NUMERIC<field:2>", fields[3].toString());
+        assertEquals(2, fields.length);
+        assertEquals("LongField <field:1>", fields[0].toString());
+        assertEquals("LongField <field:2>", fields[1].toString());
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java
index 9adb99da5231..947b64901c99 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingParserTests.java
@@ -25,7 +25,6 @@
 import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.Map;
-import java.util.function.Supplier;
 
 public class MappingParserTests extends MapperServiceTestCase {
 
@@ -39,13 +38,12 @@ private static MappingParser createMappingParser(Settings settings, Version vers
         IndexAnalyzers indexAnalyzers = createIndexAnalyzers();
         SimilarityService similarityService = new SimilarityService(indexSettings, scriptService, Collections.emptyMap());
         MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();
-        Supplier<MappingParserContext> parserContextSupplier = () -> new MappingParserContext(
+        MappingParserContext mappingParserContext = new MappingParserContext(
             similarityService::getSimilarity,
             type -> mapperRegistry.getMapperParser(type, indexSettings.getIndexVersionCreated()),
             mapperRegistry.getRuntimeFieldParsers()::get,
             indexSettings.getIndexVersionCreated(),
             () -> { throw new UnsupportedOperationException(); },
-            null,
             scriptService,
             indexAnalyzers,
             indexSettings,
@@ -55,13 +53,13 @@ private static MappingParser createMappingParser(Settings settings, Version vers
             indexSettings.getIndexVersionCreated()
         );
         Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> metadataMappers = new LinkedHashMap<>();
-        metadataMapperParsers.values().stream().map(parser -> parser.getDefault(parserContextSupplier.get())).forEach(m -> {
+        metadataMapperParsers.values().stream().map(parser -> parser.getDefault(mappingParserContext)).forEach(m -> {
             if (m != null) {
                 metadataMappers.put(m.getClass(), m);
             }
         });
         return new MappingParser(
-            parserContextSupplier,
+            mappingParserContext,
             metadataMapperParsers,
             () -> metadataMappers,
             type -> MapperService.SINGLE_MAPPING_NAME
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java
index d106908fad5d..a8b804017d1c 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java
@@ -1439,4 +1439,79 @@ public void testNestedDoesNotSupportSubobjectsParameter() {
         );
         assertEquals("Failed to parse mapping: Nested type [nested1] does not support [subobjects] parameter", exception.getMessage());
     }
+
+    public void testIndexTemplatesMergeIncludes() throws IOException {
+        {
+            MapperService mapperService = createMapperService("""
+                { "_doc" : { "properties" : {
+                    "field" : {
+                        "type" : "nested",
+                        "include_in_root" : true,
+                        "properties" : {
+                            "text" : { "type" : "text" }
+                        }
+                    }
+                }}}
+                """);
+            merge(mapperService, MergeReason.INDEX_TEMPLATE, """
+                { "_doc" : { "properties" : {
+                    "field" : {
+                        "type" : "nested",
+                        "include_in_parent" : true,
+                        "properties" : {
+                            "text" : { "type" : "text" }
+                        }
+                    }
+                }}}
+                """);
+            assertThat(Strings.toString(mapperService.documentMapper().mapping()), containsString("""
+                {"type":"nested","include_in_parent":true,"properties":{"""));
+        }
+        {
+            MapperService mapperService = createMapperService("""
+                { "_doc" : { "properties" : {
+                    "field" : {
+                        "type" : "nested",
+                        "include_in_parent" : true,
+                        "properties" : {
+                            "text" : { "type" : "text" }
+                        }
+                    }
+                }}}
+                """);
+            merge(mapperService, MergeReason.INDEX_TEMPLATE, """
+                { "_doc" : { "properties" : {
+                    "field" : {
+                        "type" : "nested",
+                        "include_in_root" : true,
+                        "properties" : {
+                            "text" : { "type" : "text" }
+                        }
+                    }
+                }}}
+                """);
+            assertThat(Strings.toString(mapperService.documentMapper().mapping()), containsString("""
+                {"type":"nested","include_in_parent":true,"properties":{"""));
+        }
+    }
+
+    public void testMergeNested() {
+        NestedObjectMapper firstMapper = new NestedObjectMapper.Builder("nested1", Version.CURRENT).includeInParent(true)
+            .includeInRoot(true)
+            .build(MapperBuilderContext.root(false));
+        NestedObjectMapper secondMapper = new NestedObjectMapper.Builder("nested1", Version.CURRENT).includeInParent(false)
+            .includeInRoot(true)
+            .build(MapperBuilderContext.root(false));
+
+        MapperException e = expectThrows(MapperException.class, () -> firstMapper.merge(secondMapper, MapperBuilderContext.root(false)));
+        assertThat(e.getMessage(), containsString("[include_in_parent] parameter can't be updated on a nested object mapping"));
+
+        NestedObjectMapper result = (NestedObjectMapper) firstMapper.merge(
+            secondMapper,
+            MapperService.MergeReason.INDEX_TEMPLATE,
+            MapperBuilderContext.root(false)
+        );
+        assertFalse(result.isIncludeInParent());
+        assertTrue(result.isIncludeInRoot());
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
index 9bafe9673be6..dd5d1630ccd6 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
@@ -10,8 +10,11 @@
 
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.NumberFieldTypeTests.OutOfRangeSpec;
 import org.elasticsearch.script.DoubleFieldScript;
 import org.elasticsearch.script.LongFieldScript;
@@ -23,6 +26,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.function.Function;
 import java.util.function.Supplier;
@@ -31,6 +35,7 @@
 import static org.hamcrest.Matchers.both;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.matchesPattern;
 
 public abstract class NumberFieldMapperTests extends MapperTestCase {
@@ -70,7 +75,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException {
                 minimalMapping(b);
                 b.field("script", "test");
                 b.field("on_script_error", "continue");
-            }, m -> assertThat((m).onScriptError, equalTo("continue")));
+            }, m -> assertThat((m).onScriptError, is(OnScriptError.CONTINUE)));
         }
     }
 
@@ -104,14 +109,10 @@ public void testDefaults() throws Exception {
         ParsedDocument doc = mapper.parse(source(b -> b.field("field", 123)));
 
         IndexableField[] fields = doc.rootDoc().getFields("field");
-        assertEquals(2, fields.length);
-        IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
-        assertFalse(pointField.fieldType().stored());
-        assertEquals(123, pointField.numericValue().doubleValue(), 0d);
-        IndexableField dvField = fields[1];
-        assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
-        assertFalse(dvField.fieldType().stored());
+        // One field indexes points
+        assertEquals(1, Arrays.stream(fields).filter(f -> f.fieldType().pointIndexDimensionCount() != 0).count());
+        // One field indexes doc values
+        assertEquals(1, Arrays.stream(fields).filter(f -> f.fieldType().docValuesType() != DocValuesType.NONE).count());
     }
 
     public void testNotIndexed() throws Exception {
@@ -149,15 +150,13 @@ public void testStore() throws Exception {
         ParsedDocument doc = mapper.parse(source(b -> b.field("field", 123)));
 
         IndexableField[] fields = doc.rootDoc().getFields("field");
-        assertEquals(3, fields.length);
-        IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
-        assertEquals(123, pointField.numericValue().doubleValue(), 0d);
-        IndexableField dvField = fields[1];
-        assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
-        IndexableField storedField = fields[2];
-        assertTrue(storedField.fieldType().stored());
-        assertEquals(123, storedField.numericValue().doubleValue(), 0d);
+
+        // One field indexes points
+        assertEquals(1, Arrays.stream(fields).filter(f -> f.fieldType().pointIndexDimensionCount() != 0).count());
+        // One field indexes doc values
+        assertEquals(1, Arrays.stream(fields).filter(f -> f.fieldType().docValuesType() != DocValuesType.NONE).count());
+        // One field is stored
+        assertEquals(1, Arrays.stream(fields).filter(f -> f.fieldType().stored()).count());
     }
 
     public void testCoerce() throws IOException {
@@ -165,12 +164,10 @@ public void testCoerce() throws IOException {
         ParsedDocument doc = mapper.parse(source(b -> b.field("field", "123")));
 
         IndexableField[] fields = doc.rootDoc().getFields("field");
-        assertEquals(2, fields.length);
-        IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
-        assertEquals(123, pointField.numericValue().doubleValue(), 0d);
-        IndexableField dvField = fields[1];
-        assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
+        // One field indexes points
+        assertEquals(1, Arrays.stream(fields).filter(f -> f.fieldType().pointIndexDimensionCount() != 0).count());
+        // One field indexes doc values
+        assertEquals(1, Arrays.stream(fields).filter(f -> f.fieldType().docValuesType() != DocValuesType.NONE).count());
 
         DocumentMapper mapper2 = createDocumentMapper(fieldMapping(b -> {
             minimalMapping(b);
@@ -264,7 +261,7 @@ public void testDimension() throws IOException {
 
     public void testMetricType() throws IOException {
         // Test default setting
-        MapperService mapperService = createMapperService(fieldMapping(b -> minimalMapping(b)));
+        MapperService mapperService = createMapperService(fieldMapping(this::minimalMapping));
         NumberFieldMapper.NumberFieldType ft = (NumberFieldMapper.NumberFieldType) mapperService.fieldType("field");
         assertNull(ft.getMetricType());
 
@@ -295,6 +292,19 @@ public void testMetricType() throws IOException {
         }
     }
 
+    public void testTimeSeriesIndexDefault() throws Exception {
+        var randomMetricType = randomFrom(TimeSeriesParams.MetricType.values());
+        var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName())
+            .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field");
+        var mapperService = createMapperService(indexSettings.build(), fieldMapping(b -> {
+            minimalMapping(b);
+            b.field("time_series_metric", randomMetricType.toString());
+        }));
+        var ft = (NumberFieldMapper.NumberFieldType) mapperService.fieldType("field");
+        assertThat(ft.getMetricType(), equalTo(randomMetricType));
+        assertThat(ft.isIndexed(), is(false));
+    }
+
     public void testMetricAndDocvalues() {
         Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {
             minimalMapping(b);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java
index e2f2e4a9ad8e..167f6e358320 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java
@@ -20,9 +20,9 @@
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.sandbox.document.HalfFloatPoint;
-import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Sort;
@@ -824,7 +824,8 @@ public void testFetchSourceValue() throws IOException {
             ScriptCompiler.NONE,
             false,
             true,
-            Version.CURRENT
+            Version.CURRENT,
+            null
         ).build(MapperBuilderContext.root(false)).fieldType();
         assertEquals(List.of(3), fetchSourceValue(mapper, 3.14));
         assertEquals(List.of(42), fetchSourceValue(mapper, "42.9"));
@@ -836,7 +837,8 @@ public void testFetchSourceValue() throws IOException {
             ScriptCompiler.NONE,
             false,
             true,
-            Version.CURRENT
+            Version.CURRENT,
+            null
         ).nullValue(2.71f).build(MapperBuilderContext.root(false)).fieldType();
         assertEquals(List.of(2.71f), fetchSourceValue(nullValueMapper, ""));
         assertEquals(List.of(2.71f), fetchSourceValue(nullValueMapper, null));
@@ -849,7 +851,8 @@ public void testFetchHalfFloatFromSource() throws IOException {
             ScriptCompiler.NONE,
             false,
             true,
-            Version.CURRENT
+            Version.CURRENT,
+            null
         ).build(MapperBuilderContext.root(false)).fieldType();
         /*
          * Half float loses a fair bit of precision compared to float but
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java
index 089e1692a1a0..5749ce4650b1 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java
@@ -13,8 +13,6 @@
 
 import java.util.Collections;
 
-import static org.hamcrest.Matchers.containsString;
-
 public class ObjectMapperMergeTests extends ESTestCase {
 
     private final RootObjectMapper rootObjectMapper = createMapping(false, true, true, false);
@@ -117,26 +115,6 @@ public void testMergeDisabledRootMapper() {
         assertEquals("test", merged.runtimeFields().iterator().next().name());
     }
 
-    public void testMergeNested() {
-        NestedObjectMapper firstMapper = new NestedObjectMapper.Builder("nested1", Version.CURRENT).includeInParent(true)
-            .includeInRoot(true)
-            .build(MapperBuilderContext.root(false));
-        NestedObjectMapper secondMapper = new NestedObjectMapper.Builder("nested1", Version.CURRENT).includeInParent(false)
-            .includeInRoot(true)
-            .build(MapperBuilderContext.root(false));
-
-        MapperException e = expectThrows(MapperException.class, () -> firstMapper.merge(secondMapper, MapperBuilderContext.root(false)));
-        assertThat(e.getMessage(), containsString("[include_in_parent] parameter can't be updated on a nested object mapping"));
-
-        NestedObjectMapper result = (NestedObjectMapper) firstMapper.merge(
-            secondMapper,
-            MapperService.MergeReason.INDEX_TEMPLATE,
-            MapperBuilderContext.root(false)
-        );
-        assertFalse(result.isIncludeInParent());
-        assertTrue(result.isIncludeInRoot());
-    }
-
     public void testMergedFieldNamesFieldWithDotsSubobjectsFalseAtRoot() {
         RootObjectMapper mergeInto = createRootSubobjectFalseLeafWithDots();
         RootObjectMapper mergeWith = createRootSubobjectFalseLeafWithDots();
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java
index e25ff342e43b..77bed713ecf8 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java
@@ -33,6 +33,7 @@
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 
@@ -44,9 +45,14 @@
 public class ParametrizedMapperTests extends MapperServiceTestCase {
 
     public enum DummyEnumType {
-        name1,
-        name2,
-        name3
+        NAME1,
+        NAME2,
+        NAME3;
+
+        @Override
+        public final String toString() {
+            return name().toLowerCase(Locale.ROOT);
+        }
     }
 
     public static class TestPlugin extends Plugin implements MapperPlugin {
@@ -127,7 +133,7 @@ public static class Builder extends FieldMapper.Builder {
             "enum_field",
             true,
             m -> toType(m).enumField,
-            DummyEnumType.name1,
+            DummyEnumType.NAME1,
             DummyEnumType.class
         );
 
@@ -135,9 +141,9 @@ public static class Builder extends FieldMapper.Builder {
             "restricted_enum_field",
             true,
             m -> toType(m).restrictedEnumField,
-            DummyEnumType.name1,
+            DummyEnumType.NAME1,
             DummyEnumType.class,
-            EnumSet.of(DummyEnumType.name1, DummyEnumType.name2)
+            EnumSet.of(DummyEnumType.NAME1, DummyEnumType.NAME2)
         );
 
         protected Builder(String name) {
@@ -261,14 +267,13 @@ private static TestMapper fromMapping(String mapping, Version version, boolean f
             name -> null,
             version,
             () -> null,
-            null,
             ScriptCompiler.NONE,
             mapperService.getIndexAnalyzers(),
             mapperService.getIndexSettings(),
             mapperService.getIndexSettings().getMode().idFieldMapperWithoutFieldData()
         );
         if (fromDynamicTemplate) {
-            pc = new MappingParserContext.DynamicTemplateParserContext(pc);
+            pc = pc.createDynamicTemplateContext(null);
         }
         return (TestMapper) new TypeParser().parse("field", XContentHelper.convertToMap(JsonXContent.jsonXContent, mapping, true), pc)
             .build(MapperBuilderContext.root(false));
@@ -648,13 +653,13 @@ public void testEnumField() {
             String mapping = """
                 {"type":"test_mapper","required":"a","enum_field":"name3"}""";
             TestMapper mapper = fromMapping(mapping);
-            assertEquals(DummyEnumType.name3, mapper.enumField);
+            assertEquals(DummyEnumType.NAME3, mapper.enumField);
         }
         {
             String mapping = """
                 {"type":"test_mapper","required":"a"}""";
             TestMapper mapper = fromMapping(mapping);
-            assertEquals(DummyEnumType.name1, mapper.enumField);
+            assertEquals(DummyEnumType.NAME1, mapper.enumField);
         }
     }
 
@@ -675,13 +680,13 @@ public void testRestrictedEnumField() {
             String mapping = """
                 {"type":"test_mapper","required":"a","restricted_enum_field":"name2"}""";
             TestMapper mapper = fromMapping(mapping);
-            assertEquals(DummyEnumType.name2, mapper.restrictedEnumField);
+            assertEquals(DummyEnumType.NAME2, mapper.restrictedEnumField);
         }
         {
             String mapping = """
                 {"type":"test_mapper","required":"a"}""";
             TestMapper mapper = fromMapping(mapping);
-            assertEquals(DummyEnumType.name1, mapper.restrictedEnumField);
+            assertEquals(DummyEnumType.NAME1, mapper.restrictedEnumField);
         }
     }
 
@@ -699,5 +704,4 @@ public void testCustomMergeValidation() throws IOException {
         })));
         assertThat(e.getMessage(), containsString("int_value"));
     }
-
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java
index b20358348712..26b8ed8e0cb3 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java
@@ -18,7 +18,8 @@
 import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
 import org.elasticsearch.search.fetch.subphase.FieldFetcher;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.util.Collections;
@@ -65,7 +66,7 @@ public void testFetchValue() throws Exception {
             SearchLookup lookup = new SearchLookup(
                 mapperService::fieldType,
                 fieldDataLookup(mapperService),
-                new SourceLookup.ReaderSourceProvider()
+                SourceProvider.fromStoredFields()
             );
             SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService);
             FieldFetcher fieldFetcher = FieldFetcher.create(
@@ -74,8 +75,8 @@ public void testFetchValue() throws Exception {
             );
             IndexSearcher searcher = newSearcher(iw);
             LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
-            lookup.source().setSegmentAndDocument(context, 0);
-            Map<String, DocumentField> fields = fieldFetcher.fetch(lookup.source(), 0);
+            Source source = lookup.getSource(context, 0);
+            Map<String, DocumentField> fields = fieldFetcher.fetch(source, 0);
             assertEquals(1, fields.size());
             assertEquals(List.of("value"), fields.get("field").getValues());
         });
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java
index 99daa3770c55..71173ccdf6a0 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java
@@ -12,13 +12,12 @@
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.IndexSearcher;
-import org.elasticsearch.Version;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.search.lookup.SourceProvider;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -78,7 +77,7 @@ public void testFetchIdFieldValue() throws IOException {
                 SearchLookup lookup = new SearchLookup(
                     mapperService::fieldType,
                     fieldDataLookup(mapperService),
-                    new SourceLookup.ReaderSourceProvider()
+                    SourceProvider.fromStoredFields()
                 );
                 SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class);
                 when(searchExecutionContext.lookup()).thenReturn(lookup);
@@ -86,9 +85,9 @@ public void testFetchIdFieldValue() throws IOException {
                 ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null);
                 IndexSearcher searcher = newSearcher(iw);
                 LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
-                lookup.source().setSegmentAndDocument(context, 0);
+                Source source = lookup.getSource(context, 0);
                 valueFetcher.setNextReader(context);
-                assertEquals(List.of(id), valueFetcher.fetchValues(lookup.source(), 0, new ArrayList<>()));
+                assertEquals(List.of(id), valueFetcher.fetchValues(source, 0, new ArrayList<>()));
             }
         );
     }
@@ -97,13 +96,7 @@ public void testSourceDescription() throws IOException {
         String id = randomAlphaOfLength(4);
         assertThat(
             ProvidedIdFieldMapper.NO_FIELD_DATA.documentDescription(
-                new TestDocumentParserContext(
-                    MappingLookup.EMPTY,
-                    MapperTestCase.createIndexSettings(Version.CURRENT, Settings.EMPTY),
-                    null,
-                    null,
-                    source(id, b -> {}, randomAlphaOfLength(2))
-                )
+                new TestDocumentParserContext(MappingLookup.EMPTY, source(id, b -> {}, randomAlphaOfLength(2)))
             ),
             equalTo("document with id '" + id + "'")
         );
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java
index c46c7a062019..b899c5da58b5 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java
@@ -141,6 +141,7 @@ public void testRangeQueryIntersectsAdjacentValues() throws Exception {
     /**
      * check that we catch cases where the user specifies larger "from" than "to" value, not counting the include upper/lower settings
      */
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/86284")
     public void testFromLargerToErrors() throws Exception {
         SearchExecutionContext context = createContext();
         RangeFieldType ft = createDefaultFieldType();
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java
index 0d6c963958e3..896be2640b46 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
 
@@ -74,20 +74,15 @@ public void testFetchRoutingFieldValue() throws IOException {
             mapperService,
             iw -> { iw.addDocument(mapperService.documentMapper().parse(source("1", b -> {}, "abcd")).rootDoc()); },
             iw -> {
-                SearchLookup lookup = new SearchLookup(
-                    mapperService::fieldType,
-                    fieldDataLookup(mapperService),
-                    new SourceLookup.ReaderSourceProvider()
-                );
+                SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup(mapperService), (ctx, doc) -> null);
                 SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class);
                 when(searchExecutionContext.lookup()).thenReturn(lookup);
                 RoutingFieldMapper.RoutingFieldType ft = (RoutingFieldMapper.RoutingFieldType) mapperService.fieldType("_routing");
                 ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null);
                 IndexSearcher searcher = newSearcher(iw);
                 LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
-                lookup.source().setSegmentAndDocument(context, 0);
                 valueFetcher.setNextReader(context);
-                assertEquals(List.of("abcd"), valueFetcher.fetchValues(lookup.source(), 0, new ArrayList<>()));
+                assertEquals(List.of("abcd"), valueFetcher.fetchValues(Source.empty(XContentType.JSON), 0, new ArrayList<>()));
             }
         );
     }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java
index 6d55a3e33917..40c719367159 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java
@@ -18,7 +18,7 @@
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.StringFieldScript;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.json.JsonXContent;
 
@@ -30,10 +30,11 @@
 import static org.hamcrest.Matchers.equalTo;
 
 public class StringFieldScriptTests extends FieldScriptTestCase<StringFieldScript.Factory> {
-    public static final StringFieldScript.Factory DUMMY = (fieldName, params, lookup) -> ctx -> new StringFieldScript(
+    public static final StringFieldScript.Factory DUMMY = (fieldName, params, lookup, onScriptError) -> ctx -> new StringFieldScript(
         fieldName,
         params,
         lookup,
+        OnScriptError.FAIL,
         ctx
     ) {
         @Override
@@ -64,7 +65,8 @@ public void testTooManyValues() throws IOException {
                 StringFieldScript script = new StringFieldScript(
                     "test",
                     Map.of(),
-                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()),
+                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, (ctx, doc) -> null),
+                    OnScriptError.FAIL,
                     reader.leaves().get(0)
                 ) {
                     @Override
@@ -90,7 +92,8 @@ public void testTooManyChars() throws IOException {
                 StringFieldScript script = new StringFieldScript(
                     "test",
                     Map.of(),
-                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider()),
+                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, (ctx, doc) -> null),
+                    OnScriptError.FAIL,
                     reader.leaves().get(0)
                 ) {
                     @Override
@@ -130,11 +133,12 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException {
                 StringFieldScript.LeafFactory leafFactory = fromSource().newFactory(
                     "field",
                     Collections.emptyMap(),
-                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider())
+                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, SourceProvider.fromStoredFields()),
+                    OnScriptError.FAIL
                 );
                 StringFieldScript stringFieldScript = leafFactory.newInstance(reader.leaves().get(0));
-                List<String> results = stringFieldScript.resultsForDoc(0);
-                assertEquals(numValues, results.size());
+                stringFieldScript.runForDoc(0);
+                assertEquals(numValues, stringFieldScript.getValues().size());
             }
         }
     }
@@ -159,11 +163,12 @@ public final void testFromSourceDoesNotEnforceCharsLimit() throws IOException {
                 StringFieldScript.LeafFactory leafFactory = fromSource().newFactory(
                     "field",
                     Collections.emptyMap(),
-                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, new SourceLookup.ReaderSourceProvider())
+                    new SearchLookup(field -> null, (ft, lookup, fdt) -> null, SourceProvider.fromStoredFields()),
+                    OnScriptError.FAIL
                 );
                 StringFieldScript stringFieldScript = leafFactory.newInstance(reader.leaves().get(0));
-                List<String> results = stringFieldScript.resultsForDoc(0);
-                assertEquals(5, results.size());
+                stringFieldScript.runForDoc(0);
+                assertEquals(5, stringFieldScript.getValues().size());
             }
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java
index d41a62ea27e2..982ae98c08d2 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java
@@ -614,9 +614,6 @@ public void testSourceDescription() throws IOException {
     private TestDocumentParserContext documentParserContext(IndexableField... fields) throws IOException {
         TestDocumentParserContext ctx = new TestDocumentParserContext(
             mapperService().mappingLookup(),
-            MapperTestCase.createIndexSettings(Version.CURRENT, Settings.EMPTY),
-            null,
-            null,
             source(null, testCase.randomSource(), null)
         );
         for (IndexableField f : fields) {
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java
index 98d0e96696d8..85183941c88f 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java
@@ -77,7 +77,6 @@ public void testMultiFieldWithinMultiField() throws IOException {
             type -> null,
             olderVersion,
             null,
-            null,
             ScriptCompiler.NONE,
             mapperService.getIndexAnalyzers(),
             mapperService.getIndexSettings(),
@@ -103,7 +102,6 @@ public void testMultiFieldWithinMultiField() throws IOException {
             type -> null,
             version,
             null,
-            null,
             ScriptCompiler.NONE,
             mapperService.getIndexAnalyzers(),
             mapperService.getIndexSettings(),
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java
index a4145c02ed1a..911d4355e063 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java
@@ -15,7 +15,8 @@
 import org.apache.lucene.search.IndexSearcher;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -64,18 +65,13 @@ public void testFetchFieldValue() throws IOException {
             iw.addDocument(parsedDoc.rootDoc());
         }, iw -> {
             VersionFieldMapper.VersionFieldType ft = (VersionFieldMapper.VersionFieldType) mapperService.fieldType("_version");
-            SearchLookup lookup = new SearchLookup(
-                mapperService::fieldType,
-                fieldDataLookup(mapperService),
-                new SourceLookup.ReaderSourceProvider()
-            );
+            SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup(mapperService), (ctx, doc) -> null);
             SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService);
             ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null);
             IndexSearcher searcher = newSearcher(iw);
             LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
-            lookup.source().setSegmentAndDocument(context, 0);
             valueFetcher.setNextReader(context);
-            assertEquals(List.of(version), valueFetcher.fetchValues(lookup.source(), 0, Collections.emptyList()));
+            assertEquals(List.of(version), valueFetcher.fetchValues(Source.empty(XContentType.JSON), 0, Collections.emptyList()));
         });
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java
index d8ef71ee71cf..6ca882a766fb 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java
@@ -12,9 +12,10 @@
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat;
 import org.apache.lucene.document.BinaryDocValuesField;
-import org.apache.lucene.document.KnnVectorField;
+import org.apache.lucene.document.KnnByteVectorField;
+import org.apache.lucene.document.KnnFloatVectorField;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.FieldExistsQuery;
 import org.apache.lucene.search.Query;
@@ -42,8 +43,8 @@
 import java.util.Arrays;
 import java.util.List;
 
-import static org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
-import static org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat.DEFAULT_MAX_CONN;
+import static org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
+import static org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat.DEFAULT_MAX_CONN;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -223,7 +224,7 @@ public void testDefaults() throws Exception {
     public void testIndexedVector() throws Exception {
         VectorSimilarity similarity = RandomPicks.randomFrom(random(), VectorSimilarity.values());
         DocumentMapper mapper = createDocumentMapper(
-            fieldMapping(b -> b.field("type", "dense_vector").field("dims", 3).field("index", true).field("similarity", similarity.name()))
+            fieldMapping(b -> b.field("type", "dense_vector").field("dims", 3).field("index", true).field("similarity", similarity))
         );
 
         float[] vector = { -0.5f, 0.5f, 0.7071f };
@@ -231,9 +232,9 @@ public void testIndexedVector() throws Exception {
 
         IndexableField[] fields = doc1.rootDoc().getFields("field");
         assertEquals(1, fields.length);
-        assertThat(fields[0], instanceOf(KnnVectorField.class));
+        assertThat(fields[0], instanceOf(KnnFloatVectorField.class));
 
-        KnnVectorField vectorField = (KnnVectorField) fields[0];
+        KnnFloatVectorField vectorField = (KnnFloatVectorField) fields[0];
         assertArrayEquals("Parsed vector is not equal to original.", vector, vectorField.vectorValue(), 0.001f);
         assertEquals(similarity.function, vectorField.fieldType().vectorSimilarityFunction());
     }
@@ -245,24 +246,24 @@ public void testIndexedByteVector() throws Exception {
                 b -> b.field("type", "dense_vector")
                     .field("dims", 3)
                     .field("index", true)
-                    .field("similarity", similarity.name())
+                    .field("similarity", similarity)
                     .field("element_type", "byte")
             )
         );
 
-        float[] vector = { (byte) -1, (byte) 1, (byte) 127 };
+        byte[] vector = { (byte) -1, (byte) 1, (byte) 127 };
         ParsedDocument doc1 = mapper.parse(source(b -> b.array("field", vector)));
 
         IndexableField[] fields = doc1.rootDoc().getFields("field");
         assertEquals(1, fields.length);
-        assertThat(fields[0], instanceOf(KnnVectorField.class));
+        assertThat(fields[0], instanceOf(KnnByteVectorField.class));
 
-        KnnVectorField vectorField = (KnnVectorField) fields[0];
-        vectorField.binaryValue();
-        assertEquals(
+        KnnByteVectorField vectorField = (KnnByteVectorField) fields[0];
+        vectorField.vectorValue();
+        assertArrayEquals(
             "Parsed vector is not equal to original.",
-            new BytesRef(new byte[] { (byte) -1, (byte) 1, (byte) 127 }),
-            vectorField.binaryValue()
+            new byte[] { (byte) -1, (byte) 1, (byte) 127 },
+            vectorField.vectorValue()
         );
         assertEquals(similarity.function, vectorField.fieldType().vectorSimilarityFunction());
     }
@@ -270,7 +271,7 @@ public void testIndexedByteVector() throws Exception {
     public void testDotProductWithInvalidNorm() throws Exception {
         DocumentMapper mapper = createDocumentMapper(
             fieldMapping(
-                b -> b.field("type", "dense_vector").field("dims", 3).field("index", true).field("similarity", VectorSimilarity.dot_product)
+                b -> b.field("type", "dense_vector").field("dims", 3).field("index", true).field("similarity", VectorSimilarity.DOT_PRODUCT)
             )
         );
         float[] vector = { -12.1f, 2.7f, -4 };
@@ -285,7 +286,7 @@ public void testDotProductWithInvalidNorm() throws Exception {
 
         DocumentMapper mapperWithLargerDim = createDocumentMapper(
             fieldMapping(
-                b -> b.field("type", "dense_vector").field("dims", 6).field("index", true).field("similarity", VectorSimilarity.dot_product)
+                b -> b.field("type", "dense_vector").field("dims", 6).field("index", true).field("similarity", VectorSimilarity.DOT_PRODUCT)
             )
         );
         float[] largerVector = { -12.1f, 2.7f, -4, 1.05f, 10.0f, 29.9f };
@@ -303,7 +304,7 @@ public void testDotProductWithInvalidNorm() throws Exception {
     public void testCosineWithZeroVector() throws Exception {
         DocumentMapper mapper = createDocumentMapper(
             fieldMapping(
-                b -> b.field("type", "dense_vector").field("dims", 3).field("index", true).field("similarity", VectorSimilarity.cosine)
+                b -> b.field("type", "dense_vector").field("dims", 3).field("index", true).field("similarity", VectorSimilarity.COSINE)
             )
         );
         float[] vector = { -0.0f, 0.0f, 0.0f };
@@ -323,7 +324,7 @@ public void testCosineWithZeroByteVector() throws Exception {
                 b -> b.field("type", "dense_vector")
                     .field("dims", 3)
                     .field("index", true)
-                    .field("similarity", VectorSimilarity.cosine)
+                    .field("similarity", VectorSimilarity.COSINE)
                     .field("element_type", "byte")
             )
         );
@@ -332,9 +333,7 @@ public void testCosineWithZeroByteVector() throws Exception {
         assertNotNull(e.getCause());
         assertThat(
             e.getCause().getMessage(),
-            containsString(
-                "The [cosine] similarity does not support vectors with zero magnitude. Preview of invalid vector: [-0.0, 0.0, 0.0]"
-            )
+            containsString("The [cosine] similarity does not support vectors with zero magnitude. Preview of invalid vector: [0, 0, 0]")
         );
     }
 
@@ -492,7 +491,7 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
 
     @Override
     protected boolean allowsNullValues() {
-        return false;       // TODO should this allow null values?
+        return true;
     }
 
     public void testCannotBeUsedInMultifields() {
@@ -534,7 +533,7 @@ public void testByteVectorIndexBoundaries() throws IOException {
                     .field("element_type", "byte")
                     .field("dims", 3)
                     .field("index", true)
-                    .field("similarity", VectorSimilarity.cosine)
+                    .field("similarity", VectorSimilarity.COSINE)
             )
         );
 
@@ -544,7 +543,13 @@ public void testByteVectorIndexBoundaries() throws IOException {
         );
         assertThat(
             e.getCause().getMessage(),
-            containsString("element_type [byte] vectors only support integers between [-128, 127] but found [128.0] at dim [0];")
+            containsString("element_type [byte] vectors only support integers between [-128, 127] but found [128] at dim [0];")
+        );
+
+        e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.array("field", new float[] { 18.2f, 0, 0 }))));
+        assertThat(
+            e.getCause().getMessage(),
+            containsString("element_type [byte] vectors only support non-decimal values but found decimal value [18.2] at dim [0];")
         );
 
         e = expectThrows(
@@ -553,7 +558,7 @@ public void testByteVectorIndexBoundaries() throws IOException {
         );
         assertThat(
             e.getCause().getMessage(),
-            containsString("element_type [byte] vectors only support integers between [-128, 127] but found [-129.0] at dim [2];")
+            containsString("element_type [byte] vectors only support integers between [-128, 127] but found [-129] at dim [2];")
         );
     }
 
@@ -693,8 +698,8 @@ public void testKnnVectorsFormat() throws IOException {
         Codec codec = codecService.codec("default");
         assertThat(codec, instanceOf(PerFieldMapperCodec.class));
         KnnVectorsFormat knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
-        assertThat(knnVectorsFormat, instanceOf(Lucene94HnswVectorsFormat.class));
-        String expectedString = "Lucene94HnswVectorsFormat(name=Lucene94HnswVectorsFormat, maxConn="
+        assertThat(knnVectorsFormat, instanceOf(Lucene95HnswVectorsFormat.class));
+        String expectedString = "Lucene95HnswVectorsFormat(name=Lucene95HnswVectorsFormat, maxConn="
             + m
             + ", beamWidth="
             + efConstruction
@@ -725,14 +730,12 @@ private static class DenseVectorSyntheticSourceSupport implements SyntheticSourc
 
         @Override
         public SyntheticSourceExample example(int maxValues) throws IOException {
-            List<Float> value = randomList(dims, dims, this::randomValue);
+            Object value = elementType == ElementType.BYTE
+                ? randomList(dims, dims, ESTestCase::randomByte)
+                : randomList(dims, dims, ESTestCase::randomFloat);
             return new SyntheticSourceExample(value, value, this::mapping);
         }
 
-        private float randomValue() {
-            return elementType == ElementType.BYTE ? ESTestCase.randomByte() : ESTestCase.randomFloat();
-        }
-
         private void mapping(XContentBuilder b) throws IOException {
             b.field("type", "dense_vector");
             b.field("dims", dims);
@@ -753,7 +756,7 @@ private void mapping(XContentBuilder b) throws IOException {
         }
 
         @Override
-        public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
+        public List<SyntheticSourceInvalidExample> invalidExample() {
             return List.of();
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java
index 9993bed3d94e..b07ef7ba6c6b 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java
@@ -36,7 +36,7 @@ private DenseVectorFieldType createFloatFieldType() {
             DenseVectorFieldMapper.ElementType.FLOAT,
             5,
             indexed,
-            VectorSimilarity.cosine,
+            VectorSimilarity.COSINE,
             Collections.emptyMap()
         );
     }
@@ -48,7 +48,7 @@ private DenseVectorFieldType createByteFieldType() {
             DenseVectorFieldMapper.ElementType.BYTE,
             5,
             true,
-            VectorSimilarity.cosine,
+            VectorSimilarity.COSINE,
             Collections.emptyMap()
         );
     }
@@ -113,7 +113,7 @@ public void testFloatCreateKnnQuery() {
             DenseVectorFieldMapper.ElementType.FLOAT,
             3,
             false,
-            VectorSimilarity.cosine,
+            VectorSimilarity.COSINE,
             Collections.emptyMap()
         );
         IllegalArgumentException e = expectThrows(
@@ -128,7 +128,7 @@ public void testFloatCreateKnnQuery() {
             DenseVectorFieldMapper.ElementType.FLOAT,
             3,
             true,
-            VectorSimilarity.dot_product,
+            VectorSimilarity.DOT_PRODUCT,
             Collections.emptyMap()
         );
         e = expectThrows(IllegalArgumentException.class, () -> dotProductField.createKnnQuery(new float[] { 0.3f, 0.1f, 1.0f }, 10, null));
@@ -140,7 +140,7 @@ public void testFloatCreateKnnQuery() {
             DenseVectorFieldMapper.ElementType.FLOAT,
             3,
             true,
-            VectorSimilarity.cosine,
+            VectorSimilarity.COSINE,
             Collections.emptyMap()
         );
         e = expectThrows(IllegalArgumentException.class, () -> cosineField.createKnnQuery(new float[] { 0.0f, 0.0f, 0.0f }, 10, null));
@@ -154,7 +154,7 @@ public void testByteCreateKnnQuery() {
             DenseVectorFieldMapper.ElementType.BYTE,
             3,
             false,
-            VectorSimilarity.cosine,
+            VectorSimilarity.COSINE,
             Collections.emptyMap()
         );
         IllegalArgumentException e = expectThrows(
@@ -169,10 +169,13 @@ public void testByteCreateKnnQuery() {
             DenseVectorFieldMapper.ElementType.BYTE,
             3,
             true,
-            VectorSimilarity.cosine,
+            VectorSimilarity.COSINE,
             Collections.emptyMap()
         );
         e = expectThrows(IllegalArgumentException.class, () -> cosineField.createKnnQuery(new float[] { 0.0f, 0.0f, 0.0f }, 10, null));
         assertThat(e.getMessage(), containsString("The [cosine] similarity does not support vectors with zero magnitude."));
+
+        e = expectThrows(IllegalArgumentException.class, () -> cosineField.createKnnQuery(new byte[] { 0, 0, 0 }, 10, null));
+        assertThat(e.getMessage(), containsString("The [cosine] similarity does not support vectors with zero magnitude."));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/KnnDenseVectorScriptDocValuesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/KnnDenseVectorScriptDocValuesTests.java
index 51ef11618537..ce71236b3524 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/KnnDenseVectorScriptDocValuesTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/KnnDenseVectorScriptDocValuesTests.java
@@ -8,9 +8,8 @@
 
 package org.elasticsearch.index.mapper.vectors;
 
-import org.apache.lucene.index.VectorValues;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.VectorUtil;
+import org.apache.lucene.index.ByteVectorValues;
+import org.apache.lucene.index.FloatVectorValues;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType;
 import org.elasticsearch.script.field.vectors.ByteKnnDenseVectorDocValuesField;
 import org.elasticsearch.script.field.vectors.DenseVector;
@@ -19,7 +18,6 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
-import java.nio.ByteBuffer;
 
 import static org.hamcrest.Matchers.containsString;
 
@@ -30,12 +28,7 @@ public void testFloatGetVectorValueAndGetMagnitude() throws IOException {
         float[][] vectors = { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } };
         float[] expectedMagnitudes = { 1.7320f, 2.4495f, 3.3166f };
 
-        DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(
-            wrap(vectors, ElementType.FLOAT),
-            "test",
-            ElementType.FLOAT,
-            dims
-        );
+        DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(wrap(vectors), "test", dims);
         DenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues();
         for (int i = 0; i < vectors.length; i++) {
             field.setNextDocId(i);
@@ -51,12 +44,7 @@ public void testByteGetVectorValueAndGetMagnitude() throws IOException {
         float[][] vectors = { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } };
         float[] expectedMagnitudes = { 1.7320f, 2.4495f, 3.3166f };
 
-        DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(
-            wrap(vectors, ElementType.BYTE),
-            "test",
-            ElementType.BYTE,
-            dims
-        );
+        DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(wrapBytes(vectors), "test", dims);
         DenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues();
         for (int i = 0; i < vectors.length; i++) {
             field.setNextDocId(i);
@@ -70,12 +58,7 @@ public void testByteGetVectorValueAndGetMagnitude() throws IOException {
     public void testFloatMetadataAndIterator() throws IOException {
         int dims = 3;
         float[][] vectors = fill(new float[randomIntBetween(1, 5)][dims], ElementType.FLOAT);
-        DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(
-            wrap(vectors, ElementType.FLOAT),
-            "test",
-            ElementType.FLOAT,
-            dims
-        );
+        DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(wrap(vectors), "test", dims);
         for (int i = 0; i < vectors.length; i++) {
             field.setNextDocId(i);
             DenseVector dv = field.get();
@@ -94,12 +77,7 @@ public void testFloatMetadataAndIterator() throws IOException {
     public void testByteMetadataAndIterator() throws IOException {
         int dims = 3;
         float[][] vectors = fill(new float[randomIntBetween(1, 5)][dims], ElementType.BYTE);
-        DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(
-            wrap(vectors, ElementType.BYTE),
-            "test",
-            ElementType.BYTE,
-            dims
-        );
+        DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(wrapBytes(vectors), "test", dims);
         for (int i = 0; i < vectors.length; i++) {
             field.setNextDocId(i);
             DenseVector dv = field.get();
@@ -127,12 +105,7 @@ protected float[][] fill(float[][] vectors, ElementType elementType) {
     public void testFloatMissingValues() throws IOException {
         int dims = 3;
         float[][] vectors = { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } };
-        DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(
-            wrap(vectors, ElementType.FLOAT),
-            "test",
-            ElementType.FLOAT,
-            dims
-        );
+        DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(wrap(vectors), "test", dims);
         DenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues();
 
         field.setNextDocId(3);
@@ -146,12 +119,7 @@ public void testFloatMissingValues() throws IOException {
     public void testByteMissingValues() throws IOException {
         int dims = 3;
         float[][] vectors = { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } };
-        DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(
-            wrap(vectors, ElementType.BYTE),
-            "test",
-            ElementType.BYTE,
-            dims
-        );
+        DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(wrapBytes(vectors), "test", dims);
         DenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues();
 
         field.setNextDocId(3);
@@ -165,12 +133,7 @@ public void testByteMissingValues() throws IOException {
     public void testFloatGetFunctionIsNotAccessible() throws IOException {
         int dims = 3;
         float[][] vectors = { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } };
-        DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(
-            wrap(vectors, ElementType.FLOAT),
-            "test",
-            ElementType.FLOAT,
-            dims
-        );
+        DenseVectorDocValuesField field = new KnnDenseVectorDocValuesField(wrap(vectors), "test", dims);
         DenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues();
 
         field.setNextDocId(0);
@@ -186,12 +149,7 @@ public void testFloatGetFunctionIsNotAccessible() throws IOException {
     public void testByteGetFunctionIsNotAccessible() throws IOException {
         int dims = 3;
         float[][] vectors = { { 1, 1, 1 }, { 1, 1, 2 }, { 1, 1, 3 } };
-        DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(
-            wrap(vectors, ElementType.BYTE),
-            "test",
-            ElementType.BYTE,
-            dims
-        );
+        DenseVectorDocValuesField field = new ByteKnnDenseVectorDocValuesField(wrapBytes(vectors), "test", dims);
         DenseVectorScriptDocValues scriptDocValues = field.toScriptDocValues();
 
         field.setNextDocId(0);
@@ -206,7 +164,7 @@ public void testByteGetFunctionIsNotAccessible() throws IOException {
 
     public void testFloatMissingVectorValues() throws IOException {
         int dims = 7;
-        DenseVectorDocValuesField emptyKnn = new KnnDenseVectorDocValuesField(null, "test", ElementType.FLOAT, dims);
+        DenseVectorDocValuesField emptyKnn = new KnnDenseVectorDocValuesField(null, "test", dims);
 
         emptyKnn.setNextDocId(0);
         assertEquals(0, emptyKnn.toScriptDocValues().size());
@@ -220,7 +178,7 @@ public void testFloatMissingVectorValues() throws IOException {
 
     public void testByteMissingVectorValues() throws IOException {
         int dims = 7;
-        DenseVectorDocValuesField emptyKnn = new ByteKnnDenseVectorDocValuesField(null, "test", ElementType.BYTE, dims);
+        DenseVectorDocValuesField emptyKnn = new ByteKnnDenseVectorDocValuesField(null, "test", dims);
 
         emptyKnn.setNextDocId(0);
         assertEquals(0, emptyKnn.toScriptDocValues().size());
@@ -232,9 +190,10 @@ public void testByteMissingVectorValues() throws IOException {
         assertEquals("Cannot iterate over single valued dense_vector field, use get() instead", e.getMessage());
     }
 
-    public static VectorValues wrap(float[][] vectors, ElementType elementType) {
-        return new VectorValues() {
+    public static ByteVectorValues wrapBytes(float[][] vectors) {
+        return new ByteVectorValues() {
             int index = 0;
+            byte[] byteVector = new byte[vectors[0].length];
 
             @Override
             public int dimension() {
@@ -247,23 +206,11 @@ public int size() {
             }
 
             @Override
-            public float[] vectorValue() {
-                return vectors[index];
-            }
-
-            @Override
-            public BytesRef binaryValue() {
-                if (elementType == ElementType.FLOAT) {
-                    ByteBuffer byteBuffer = ByteBuffer.allocate(elementType.elementBytes * vectors[index].length);
-                    for (float value : vectors[index]) {
-                        elementType.writeValue(byteBuffer, value);
-                    }
-                    return new BytesRef(byteBuffer.array());
-                } else if (elementType == ElementType.BYTE) {
-                    return VectorUtil.toBytesRef(vectors[index]);
-                } else {
-                    throw new IllegalStateException("unknown element_type [" + elementType + "]");
+            public byte[] vectorValue() {
+                for (int i = 0; i < byteVector.length; i++) {
+                    byteVector[i] = (byte) vectors[index][i];
                 }
+                return byteVector;
             }
 
             @Override
@@ -283,10 +230,44 @@ public int advance(int target) {
                 }
                 return index = target;
             }
+        };
+    }
+
+    public static FloatVectorValues wrap(float[][] vectors) {
+        return new FloatVectorValues() {
+            int index = 0;
+
+            @Override
+            public int dimension() {
+                return 0;
+            }
 
             @Override
-            public long cost() {
-                return size();
+            public int size() {
+                return vectors.length;
+            }
+
+            @Override
+            public float[] vectorValue() {
+                return vectors[index];
+            }
+
+            @Override
+            public int docID() {
+                return index;
+            }
+
+            @Override
+            public int nextDoc() {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public int advance(int target) {
+                if (target >= size()) {
+                    return NO_MORE_DOCS;
+                }
+                return index = target;
             }
         };
     }
diff --git a/server/src/test/java/org/elasticsearch/index/query/AbstractTermQueryTestCase.java b/server/src/test/java/org/elasticsearch/index/query/AbstractTermQueryTestCase.java
index 71c71e846dbe..052582e178ac 100644
--- a/server/src/test/java/org/elasticsearch/index/query/AbstractTermQueryTestCase.java
+++ b/server/src/test/java/org/elasticsearch/index/query/AbstractTermQueryTestCase.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.index.query;
 
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.xcontent.json.JsonStringEncoder;
 
@@ -39,7 +40,7 @@ protected Map<String, QB> getAlternateVersions() {
         } else {
             value = testQuery.value();
         }
-        String contentString = formatted("""
+        String contentString = Strings.format("""
             {
                 "%s" : {
                     "%s" : %s
diff --git a/server/src/test/java/org/elasticsearch/index/query/CombineIntervalsSourceProviderTests.java b/server/src/test/java/org/elasticsearch/index/query/CombineIntervalsSourceProviderTests.java
index eceb866feaec..91ed2e3e642d 100644
--- a/server/src/test/java/org/elasticsearch/index/query/CombineIntervalsSourceProviderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/CombineIntervalsSourceProviderTests.java
@@ -27,7 +27,7 @@ protected Combine createTestInstance() {
     }
 
     @Override
-    protected Combine mutateInstance(Combine instance) throws IOException {
+    protected Combine mutateInstance(Combine instance) {
         List<IntervalsSourceProvider> subSources = instance.getSubSources();
         boolean ordered = instance.isOrdered();
         int maxGaps = instance.getMaxGaps();
diff --git a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java
index 522e4a512b0d..67b28479d2ad 100644
--- a/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/CombinedFieldsQueryBuilderTests.java
@@ -14,6 +14,7 @@
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -103,7 +104,7 @@ public void testMinumumShouldMatchFromXContent() throws IOException {
         Object[] expectedValues = new Object[] { "2", "2", "2%", null };
         int i = 0;
         for (Object value : testValues) {
-            String json = formatted("""
+            String json = Strings.format("""
                 {
                   "combined_fields" : {
                     "query" : "quick brown fox",
diff --git a/server/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java
index b2e71ede251a..d0878a905ff7 100644
--- a/server/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java
@@ -12,6 +12,7 @@
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -62,7 +63,7 @@ public void testFilterElement() throws IOException {
      * test that "filter" does not accept an array of queries, throws {@link ParsingException}
      */
     public void testNoArrayAsFilterElements() throws IOException {
-        String queryString = formatted("""
+        String queryString = Strings.format("""
             {
               "%s": {
                 "filter": [ { "term": { "foo": "a" } }, { "term": { "foo": "x" } } ]
diff --git a/server/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
index c5041ebb4dbc..f7bf98efb11d 100644
--- a/server/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
@@ -13,6 +13,7 @@
 import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -62,7 +63,7 @@ protected Map<String, DisMaxQueryBuilder> getAlternateVersions() {
         QueryBuilder innerQuery = createTestQueryBuilder().innerQueries().get(0);
         DisMaxQueryBuilder expectedQuery = new DisMaxQueryBuilder();
         expectedQuery.add(innerQuery);
-        String contentString = formatted("""
+        String contentString = Strings.format("""
             {
               "dis_max": {
                 "queries": %s
@@ -78,7 +79,7 @@ public void testIllegalArguments() {
     }
 
     public void testToQueryInnerPrefixQuery() throws Exception {
-        String queryAsString = formatted("""
+        String queryAsString = Strings.format("""
             {
               "dis_max": {
                 "queries": [
diff --git a/server/src/test/java/org/elasticsearch/index/query/DisjunctionIntervalsSourceProviderTests.java b/server/src/test/java/org/elasticsearch/index/query/DisjunctionIntervalsSourceProviderTests.java
index 7e100eefa643..160badb6161c 100644
--- a/server/src/test/java/org/elasticsearch/index/query/DisjunctionIntervalsSourceProviderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/DisjunctionIntervalsSourceProviderTests.java
@@ -27,7 +27,7 @@ protected Disjunction createTestInstance() {
     }
 
     @Override
-    protected Disjunction mutateInstance(Disjunction instance) throws IOException {
+    protected Disjunction mutateInstance(Disjunction instance) {
         List<IntervalsSourceProvider> subSources = instance.getSubSources();
         IntervalsSourceProvider.IntervalFilter filter = instance.getFilter();
         if (randomBoolean()) {
diff --git a/server/src/test/java/org/elasticsearch/index/query/DistanceFeatureQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/DistanceFeatureQueryBuilderTests.java
index f926539c788a..e4d1f716cfee 100644
--- a/server/src/test/java/org/elasticsearch/index/query/DistanceFeatureQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/DistanceFeatureQueryBuilderTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.unit.DistanceUnit;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType;
@@ -88,7 +89,7 @@ public void testFromJsonDateFieldType() throws IOException {
         // origin as string
         String origin = "2018-01-01T13:10:30Z";
         String pivot = "7d";
-        String json = formatted("""
+        String json = Strings.format("""
             {
               "distance_feature": {
                 "field": "%s",
@@ -105,7 +106,7 @@ public void testFromJsonDateFieldType() throws IOException {
 
         // origin as long
         long originLong = 1514812230999L;
-        json = formatted("""
+        json = Strings.format("""
             {
               "distance_feature": {
                 "field": "%s",
@@ -122,7 +123,7 @@ public void testFromJsonDateNanosFieldType() throws IOException {
         // origin as string
         String origin = "2018-01-01T13:10:30.323456789Z";
         String pivot = "100000000nanos";
-        String json = formatted("""
+        String json = Strings.format("""
             {
               "distance_feature": {
                 "field": "%s",
@@ -139,7 +140,7 @@ public void testFromJsonDateNanosFieldType() throws IOException {
 
         // origin as long
         long originLong = 1514812230999L;
-        json = formatted("""
+        json = Strings.format("""
             {
               "distance_feature": {
                 "field": "%s",
@@ -157,7 +158,7 @@ public void testFromJsonGeoFieldType() throws IOException {
         final String pivot = "1km";
 
         // origin as string
-        String json = formatted("""
+        String json = Strings.format("""
             {
               "distance_feature": {
                 "field": "%s",
@@ -173,7 +174,7 @@ public void testFromJsonGeoFieldType() throws IOException {
         assertEquals(json, 2.0, parsed.boost(), 0.0001);
 
         // origin as array
-        json = formatted("""
+        json = Strings.format("""
             {
               "distance_feature": {
                 "field": "%s",
@@ -186,7 +187,7 @@ public void testFromJsonGeoFieldType() throws IOException {
         assertEquals(json, origin, parsed.origin().origin());
 
         // origin as object
-        json = formatted("""
+        json = Strings.format("""
             {
               "distance_feature": {
                 "field": "%s",
@@ -217,7 +218,7 @@ public void testQueryMatchNoDocsQueryWithUnmappedField() throws IOException {
     }
 
     public void testQueryFailsWithWrongFieldType() {
-        String query = formatted("""
+        String query = Strings.format("""
             {
               "distance_feature": {
                 "field": "%s",
diff --git a/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java
index 9c25617f47c7..da4ebbd333a6 100644
--- a/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java
@@ -14,6 +14,7 @@
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -63,7 +64,7 @@ public void testIllegalArguments() {
     }
 
     public void testFromJson() throws IOException {
-        String json = formatted("""
+        String json = Strings.format("""
             {
               "%s" : {
                 "query" : {
@@ -87,7 +88,7 @@ public void testFromJson() throws IOException {
     }
 
     public void testJsonWithTopLevelBoost() throws IOException {
-        String json = formatted("""
+        String json = Strings.format("""
             {
               "%s" : {
                 "query" : {
diff --git a/server/src/test/java/org/elasticsearch/index/query/FilterIntervalsSourceProviderTests.java b/server/src/test/java/org/elasticsearch/index/query/FilterIntervalsSourceProviderTests.java
index d709e7814c64..68f9db438600 100644
--- a/server/src/test/java/org/elasticsearch/index/query/FilterIntervalsSourceProviderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/FilterIntervalsSourceProviderTests.java
@@ -29,7 +29,7 @@ protected IntervalFilter createTestInstance() {
     }
 
     @Override
-    protected IntervalFilter mutateInstance(IntervalFilter instance) throws IOException {
+    protected IntervalFilter mutateInstance(IntervalFilter instance) {
         return mutateFilter(instance);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/query/FuzzyIntervalsSourceProviderTests.java b/server/src/test/java/org/elasticsearch/index/query/FuzzyIntervalsSourceProviderTests.java
index 1e12806a6d04..343386d4ced6 100644
--- a/server/src/test/java/org/elasticsearch/index/query/FuzzyIntervalsSourceProviderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/FuzzyIntervalsSourceProviderTests.java
@@ -30,7 +30,7 @@ protected Fuzzy createTestInstance() {
     }
 
     @Override
-    protected Fuzzy mutateInstance(Fuzzy instance) throws IOException {
+    protected Fuzzy mutateInstance(Fuzzy instance) {
         String term = instance.getTerm();
         int prefixLength = instance.getPrefixLength();
         boolean isTranspositions = instance.isTranspositions();
diff --git a/server/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java
index 4f0a9cfc5bde..df454b89b2a8 100644
--- a/server/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -52,7 +53,7 @@ protected FuzzyQueryBuilder doCreateTestQueryBuilder() {
     protected Map<String, FuzzyQueryBuilder> getAlternateVersions() {
         Map<String, FuzzyQueryBuilder> alternateVersions = new HashMap<>();
         FuzzyQueryBuilder fuzzyQuery = new FuzzyQueryBuilder(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10));
-        String contentString = formatted("""
+        String contentString = Strings.format("""
             {
                 "fuzzy" : {
                     "%s" : "%s"
@@ -84,7 +85,7 @@ public void testIllegalArguments() {
     }
 
     public void testToQueryWithStringField() throws IOException {
-        String query = formatted("""
+        String query = Strings.format("""
             {
                 "fuzzy":{
                     "%s":{
@@ -107,7 +108,7 @@ public void testToQueryWithStringField() throws IOException {
     }
 
     public void testToQueryWithStringFieldDefinedFuzziness() throws IOException {
-        String query = formatted("""
+        String query = Strings.format("""
             {
                 "fuzzy":{
                     "%s":{
@@ -130,7 +131,7 @@ public void testToQueryWithStringFieldDefinedFuzziness() throws IOException {
     }
 
     public void testToQueryWithStringFieldDefinedWrongFuzziness() throws IOException {
-        String queryMissingFuzzinessUpLimit = formatted("""
+        String queryMissingFuzzinessUpLimit = Strings.format("""
             {
                 "fuzzy":{
                     "%s":{
@@ -148,7 +149,7 @@ public void testToQueryWithStringFieldDefinedWrongFuzziness() throws IOException
         String msg = "failed to find low and high distance values";
         assertTrue(e.getMessage() + " didn't contain: " + msg + " but: " + e.getMessage(), e.getMessage().contains(msg));
 
-        String queryHavingNegativeFuzzinessLowLimit = formatted("""
+        String queryHavingNegativeFuzzinessLowLimit = Strings.format("""
             {
                 "fuzzy":{
                     "%s":{
@@ -166,7 +167,7 @@ public void testToQueryWithStringFieldDefinedWrongFuzziness() throws IOException
         );
         assertTrue(e2.getMessage() + " didn't contain: " + msg2 + " but: " + e.getMessage(), e.getMessage().contains(msg));
 
-        String queryMissingFuzzinessUpLimit2 = formatted("""
+        String queryMissingFuzzinessUpLimit2 = Strings.format("""
             {
                 "fuzzy":{
                     "%s":{
@@ -183,7 +184,7 @@ public void testToQueryWithStringFieldDefinedWrongFuzziness() throws IOException
         );
         assertTrue(e.getMessage() + " didn't contain: " + msg + " but: " + e.getMessage(), e.getMessage().contains(msg));
 
-        String queryMissingFuzzinessLowLimit = formatted("""
+        String queryMissingFuzzinessLowLimit = Strings.format("""
             {
                 "fuzzy":{
                     "%s":{
@@ -203,7 +204,7 @@ public void testToQueryWithStringFieldDefinedWrongFuzziness() throws IOException
     }
 
     public void testToQueryWithNumericField() throws IOException {
-        String query = formatted("""
+        String query = Strings.format("""
             {
                 "fuzzy":{
                     "%s":{
diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
index 22d70167bbd9..990bb0259a9e 100644
--- a/server/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
@@ -15,6 +15,7 @@
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.geo.GeometryTestUtils;
 import org.elasticsearch.geometry.LinearRing;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -161,7 +162,7 @@ public void testParsingAndToQueryParsingExceptions() throws IOException {
     }
 
     public void testParsingAndToQuery1() throws IOException {
-        String query = formatted("""
+        String query = Strings.format("""
             {
               "geo_polygon": {
                 "%s": {
@@ -178,7 +179,7 @@ public void testParsingAndToQuery1() throws IOException {
     }
 
     public void testParsingAndToQuery2() throws IOException {
-        String query = formatted("""
+        String query = Strings.format("""
             {
               "geo_polygon": {
                 "%s": {
@@ -191,7 +192,7 @@ public void testParsingAndToQuery2() throws IOException {
     }
 
     public void testParsingAndToQuery3() throws IOException {
-        String query = formatted("""
+        String query = Strings.format("""
             {
               "geo_polygon": {
                 "%s": {
@@ -204,7 +205,7 @@ public void testParsingAndToQuery3() throws IOException {
     }
 
     public void testParsingAndToQuery4() throws IOException {
-        String query = formatted("""
+        String query = Strings.format("""
             {
               "geo_polygon": {
                 "%s": {
@@ -256,7 +257,7 @@ public void testIgnoreUnmapped() throws IOException {
 
     public void testPointValidation() throws IOException {
         SearchExecutionContext context = createSearchExecutionContext();
-        String queryInvalidLat = formatted("""
+        String queryInvalidLat = Strings.format("""
             {
               "geo_polygon": {
                 "%s": {
@@ -272,7 +273,7 @@ public void testPointValidation() throws IOException {
         QueryShardException e1 = expectThrows(QueryShardException.class, () -> parseQuery(queryInvalidLat).toQuery(context));
         assertThat(e1.getMessage(), containsString("illegal latitude value [140.0] for [geo_polygon]"));
 
-        String queryInvalidLon = formatted("""
+        String queryInvalidLon = Strings.format("""
             {
               "geo_polygon": {
                 "%s": {
diff --git a/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java
index 855f0ca07250..79cc850c4b8c 100644
--- a/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java
@@ -180,7 +180,7 @@ public IntervalQueryBuilder mutateInstance(IntervalQueryBuilder instance) throws
 
     public void testMatchInterval() throws IOException {
 
-        String json = formatted("""
+        String json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -196,7 +196,7 @@ public void testMatchInterval() throws IOException {
 
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
-        json = formatted("""
+        json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -215,7 +215,7 @@ public void testMatchInterval() throws IOException {
         );
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
-        json = formatted("""
+        json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -235,7 +235,7 @@ public void testMatchInterval() throws IOException {
         );
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
-        json = formatted("""
+        json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -256,7 +256,7 @@ public void testMatchInterval() throws IOException {
         );
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
-        json = formatted("""
+        json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -278,7 +278,7 @@ public void testMatchInterval() throws IOException {
         );
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
-        json = formatted("""
+        json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -312,7 +312,7 @@ public void testMatchInterval() throws IOException {
 
     public void testOrInterval() throws IOException {
 
-        String json = formatted("""
+        String json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -337,7 +337,7 @@ public void testOrInterval() throws IOException {
         Query expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.or(Intervals.term("one"), Intervals.term("two")));
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
-        json = formatted("""
+        json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -375,7 +375,7 @@ public void testOrInterval() throws IOException {
 
     public void testCombineInterval() throws IOException {
 
-        String json = formatted("""
+        String json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -427,7 +427,7 @@ public void testCombineInterval() throws IOException {
     }
 
     public void testCombineDisjunctionInterval() throws IOException {
-        String json = formatted("""
+        String json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -503,7 +503,7 @@ public void testNonIndexedFields() throws IOException {
         });
         assertThat(e.getMessage(), equalTo("Cannot create intervals over field [" + NO_POSITIONS_FIELD + "] with no positions indexed"));
 
-        String json = formatted("""
+        String json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -526,7 +526,7 @@ public void testNonIndexedFields() throws IOException {
     }
 
     public void testMultipleProviders() {
-        String json = formatted("""
+        String json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -566,7 +566,7 @@ public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryTyp
             }
         };
 
-        String json = formatted("""
+        String json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -594,7 +594,7 @@ public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryTyp
 
     public void testPrefixes() throws IOException {
 
-        String json = formatted("""
+        String json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -608,7 +608,7 @@ public void testPrefixes() throws IOException {
         Query expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.prefix(new BytesRef("term")));
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
-        String no_positions_json = formatted("""
+        String no_positions_json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -623,7 +623,7 @@ public void testPrefixes() throws IOException {
             builder1.toQuery(createSearchExecutionContext());
         });
 
-        String no_positions_fixed_field_json = formatted("""
+        String no_positions_fixed_field_json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -639,7 +639,7 @@ public void testPrefixes() throws IOException {
             builder1.toQuery(createSearchExecutionContext());
         });
 
-        String prefix_json = formatted("""
+        String prefix_json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -653,7 +653,7 @@ public void testPrefixes() throws IOException {
         expected = new IntervalQuery(PREFIXED_FIELD, Intervals.fixField(PREFIXED_FIELD + "._index_prefix", Intervals.term("term")));
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
-        String short_prefix_json = formatted("""
+        String short_prefix_json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -670,7 +670,7 @@ public void testPrefixes() throws IOException {
         );
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
-        String fix_field_prefix_json = formatted("""
+        String fix_field_prefix_json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -689,7 +689,7 @@ public void testPrefixes() throws IOException {
         );
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
-        String keyword_json = formatted("""
+        String keyword_json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -704,7 +704,7 @@ public void testPrefixes() throws IOException {
         expected = new IntervalQuery(PREFIXED_FIELD, Intervals.fixField(PREFIXED_FIELD + "._index_prefix", Intervals.term("Term")));
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
-        String keyword_fix_field_json = formatted("""
+        String keyword_fix_field_json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -727,7 +727,7 @@ public void testPrefixes() throws IOException {
 
     public void testWildcard() throws IOException {
 
-        String json = formatted("""
+        String json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -742,7 +742,7 @@ public void testWildcard() throws IOException {
         Query expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.wildcard(new BytesRef("te?m")));
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
-        String no_positions_json = formatted("""
+        String no_positions_json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -758,7 +758,7 @@ public void testWildcard() throws IOException {
             builder1.toQuery(createSearchExecutionContext());
         });
 
-        String keyword_json = formatted("""
+        String keyword_json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -774,7 +774,7 @@ public void testWildcard() throws IOException {
         expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.wildcard(new BytesRef("Te?m")));
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
-        String fixed_field_json = formatted("""
+        String fixed_field_json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -790,7 +790,7 @@ public void testWildcard() throws IOException {
         expected = new IntervalQuery(TEXT_FIELD_NAME, Intervals.fixField(MASKED_FIELD, Intervals.wildcard(new BytesRef("te?m"))));
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
-        String fixed_field_json_no_positions = formatted("""
+        String fixed_field_json_no_positions = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -806,7 +806,7 @@ public void testWildcard() throws IOException {
             builder1.toQuery(createSearchExecutionContext());
         });
 
-        String fixed_field_analyzer_json = formatted("""
+        String fixed_field_analyzer_json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -831,7 +831,7 @@ private static IntervalsSource buildFuzzySource(String term, String label, int p
 
     public void testFuzzy() throws IOException {
 
-        String json = formatted("""
+        String json = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -849,7 +849,7 @@ public void testFuzzy() throws IOException {
         );
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
-        String json_with_prefix = formatted("""
+        String json_with_prefix = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -864,7 +864,7 @@ public void testFuzzy() throws IOException {
         expected = new IntervalQuery(TEXT_FIELD_NAME, buildFuzzySource("term", "term", 2, true, Fuzziness.AUTO.asDistance("term")));
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
-        String json_with_fuzziness = formatted("""
+        String json_with_fuzziness = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -880,7 +880,7 @@ public void testFuzzy() throws IOException {
         expected = new IntervalQuery(TEXT_FIELD_NAME, buildFuzzySource("term", "term", 2, true, Fuzziness.ONE.asDistance("term")));
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
-        String json_no_transpositions = formatted("""
+        String json_no_transpositions = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -896,7 +896,7 @@ public void testFuzzy() throws IOException {
         expected = new IntervalQuery(TEXT_FIELD_NAME, buildFuzzySource("term", "term", 2, false, Fuzziness.AUTO.asDistance("term")));
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
-        String json_with_analyzer = formatted("""
+        String json_with_analyzer = Strings.format("""
             {
               "intervals": {
                 "%s": {
@@ -912,7 +912,7 @@ public void testFuzzy() throws IOException {
         expected = new IntervalQuery(TEXT_FIELD_NAME, buildFuzzySource("Term", "Term", 2, true, Fuzziness.AUTO.asDistance("term")));
         assertEquals(expected, builder.toQuery(createSearchExecutionContext()));
 
-        String json_with_fixfield = formatted("""
+        String json_with_fixfield = Strings.format("""
             {
               "intervals": {
                 "%s": {
diff --git a/server/src/test/java/org/elasticsearch/index/query/MatchIntervalsSourceProviderTests.java b/server/src/test/java/org/elasticsearch/index/query/MatchIntervalsSourceProviderTests.java
index 9e66873d1c7d..10cfb2dba717 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MatchIntervalsSourceProviderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MatchIntervalsSourceProviderTests.java
@@ -26,7 +26,7 @@ protected Match createTestInstance() {
     }
 
     @Override
-    protected Match mutateInstance(Match instance) throws IOException {
+    protected Match mutateInstance(Match instance) {
         String query = instance.getQuery();
         int maxGaps = instance.getMaxGaps();
         boolean isOrdered = instance.isOrdered();
diff --git a/server/src/test/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilderTests.java
index d33b445c897b..b5ae7cdc7964 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilderTests.java
@@ -14,6 +14,7 @@
 import org.apache.lucene.search.SynonymQuery;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -68,7 +69,7 @@ protected Map<String, MatchPhrasePrefixQueryBuilder> getAlternateVersions() {
             randomAlphaOfLengthBetween(1, 10),
             randomAlphaOfLengthBetween(1, 10)
         );
-        String contentString = formatted("""
+        String contentString = Strings.format("""
             {
                 "match_phrase_prefix" : {
                     "%s" : "%s"
diff --git a/server/src/test/java/org/elasticsearch/index/query/MatchPhraseQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MatchPhraseQueryBuilderTests.java
index e73468eb1430..011e074f3bba 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MatchPhraseQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MatchPhraseQueryBuilderTests.java
@@ -18,6 +18,7 @@
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -77,7 +78,7 @@ protected Map<String, MatchPhraseQueryBuilder> getAlternateVersions() {
             randomAlphaOfLengthBetween(1, 10),
             randomAlphaOfLengthBetween(1, 10)
         );
-        String contentString = formatted("""
+        String contentString = Strings.format("""
             {
                 "match_phrase" : {
                     "%s" : "%s"
diff --git a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
index bb76bf8453bc..954f0881fca0 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
@@ -128,7 +128,7 @@ protected MatchQueryBuilder doCreateTestQueryBuilder() {
     protected Map<String, MatchQueryBuilder> getAlternateVersions() {
         Map<String, MatchQueryBuilder> alternateVersions = new HashMap<>();
         MatchQueryBuilder matchQuery = new MatchQueryBuilder(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10));
-        String contentString = formatted("""
+        String contentString = Strings.format("""
             {
                 "match" : {
                     "%s" : "%s"
diff --git a/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
index 7242a4ef9b77..aba90ec5fda4 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.query.MultiMatchQueryBuilder.Type;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.hamcrest.Matchers;
@@ -329,7 +330,7 @@ public void testFuzzinessNotAllowedTypes() throws IOException {
             Type.PHRASE.parseField().getPreferredName(),
             Type.PHRASE_PREFIX.parseField().getPreferredName() };
         for (String type : notAllowedTypes) {
-            String json = formatted("""
+            String json = Strings.format("""
                 {
                   "multi_match": {
                     "query": "quick brown fox",
diff --git a/server/src/test/java/org/elasticsearch/index/query/PrefixIntervalsSourceProviderTests.java b/server/src/test/java/org/elasticsearch/index/query/PrefixIntervalsSourceProviderTests.java
index a94d4db44e5b..2385906e1c20 100644
--- a/server/src/test/java/org/elasticsearch/index/query/PrefixIntervalsSourceProviderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/PrefixIntervalsSourceProviderTests.java
@@ -28,7 +28,7 @@ protected Prefix createTestInstance() {
     }
 
     @Override
-    protected Prefix mutateInstance(Prefix instance) throws IOException {
+    protected Prefix mutateInstance(Prefix instance) {
         String prefix = instance.getPrefix();
         String analyzer = instance.getAnalyzer();
         String useField = instance.getUseField();
diff --git a/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
index 59d9411dcba0..676f5dda96ff 100644
--- a/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
@@ -15,6 +15,7 @@
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.hamcrest.Matchers;
 
@@ -41,7 +42,7 @@ protected PrefixQueryBuilder doCreateTestQueryBuilder() {
     protected Map<String, PrefixQueryBuilder> getAlternateVersions() {
         Map<String, PrefixQueryBuilder> alternateVersions = new HashMap<>();
         PrefixQueryBuilder prefixQuery = randomPrefixQuery();
-        String contentString = formatted("""
+        String contentString = Strings.format("""
             {
                 "prefix" : {
                     "%s" : "%s"
diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
index a5ee2ac66515..2f3433cec643 100644
--- a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
@@ -873,7 +873,7 @@ public void testLenientFlag() throws Exception {
     }
 
     public void testTimezone() throws Exception {
-        String queryAsString = formatted("""
+        String queryAsString = Strings.format("""
             {
                 "query_string":{
                     "time_zone":"Europe/Paris",
@@ -885,7 +885,7 @@ public void testTimezone() throws Exception {
         QueryStringQueryBuilder queryStringQueryBuilder = (QueryStringQueryBuilder) queryBuilder;
         assertThat(queryStringQueryBuilder.timeZone(), equalTo(ZoneId.of("Europe/Paris")));
 
-        String invalidQueryAsString = formatted("""
+        String invalidQueryAsString = Strings.format("""
             {
                 "query_string":{
                     "time_zone":"This timezone does not exist",
diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
index b8d27941530f..8bd37b7fa831 100644
--- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.lucene.BytesRefs;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -123,7 +124,7 @@ protected Map<String, RangeQueryBuilder> getAlternateVersions() {
             rangeQueryBuilder.to(randomIntBetween(101, 200));
         }
 
-        String query = formatted(
+        String query = Strings.format(
             """
                 {
                     "range":{
@@ -272,7 +273,7 @@ public void testToQueryNumericField() throws IOException {
 
     public void testDateRangeQueryFormat() throws IOException {
         // We test 01/01/2012 from gte and 2030 for lt
-        String query = formatted("""
+        String query = Strings.format("""
             {
                 "range" : {
                     "%s" : {
@@ -297,7 +298,7 @@ public void testDateRangeQueryFormat() throws IOException {
         );
 
         // Test Invalid format
-        final String invalidQuery = formatted("""
+        final String invalidQuery = Strings.format("""
             {
                 "range" : {
                     "%s" : {
@@ -311,7 +312,7 @@ public void testDateRangeQueryFormat() throws IOException {
     }
 
     public void testDateRangeBoundaries() throws IOException {
-        String query = formatted("""
+        String query = Strings.format("""
             {
                 "range" : {
                     "%s" : {
@@ -334,7 +335,7 @@ public void testDateRangeBoundaries() throws IOException {
             parsedQuery
         );
 
-        query = formatted("""
+        query = Strings.format("""
             {
                 "range" : {
                     "%s" : {
@@ -358,7 +359,7 @@ public void testDateRangeBoundaries() throws IOException {
     }
 
     public void testDateRangeQueryTimezone() throws IOException {
-        String query = formatted("""
+        String query = Strings.format("""
             {
                 "range" : {
                     "%s" : {
@@ -377,7 +378,7 @@ public void testDateRangeQueryTimezone() throws IOException {
         assertThat(parsedQuery, instanceOf(PointRangeQuery.class));
         // TODO what else can we assert
 
-        query = formatted("""
+        query = Strings.format("""
             {
                 "range" : {
                     "%s" : {
@@ -552,7 +553,7 @@ public void testParseFailsWithMultipleFields() {
     }
 
     public void testParseFailsWithMultipleFieldsWhenOneIsDate() {
-        String json = formatted("""
+        String json = Strings.format("""
             {
               "range": {
                 "age": {
diff --git a/server/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java
index 17bd93166e17..b45522cd81cd 100644
--- a/server/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java
@@ -11,6 +11,7 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.RegexpQuery;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -51,7 +52,7 @@ protected RegexpQueryBuilder doCreateTestQueryBuilder() {
     protected Map<String, RegexpQueryBuilder> getAlternateVersions() {
         Map<String, RegexpQueryBuilder> alternateVersions = new HashMap<>();
         RegexpQueryBuilder regexpQuery = randomRegexpQuery();
-        String contentString = formatted("""
+        String contentString = Strings.format("""
             {
                 "regexp" : {
                     "%s" : "%s"
diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java
index 7bd6cc2abf08..2c42c0efff82 100644
--- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java
@@ -25,6 +25,7 @@
 import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
@@ -57,6 +58,7 @@
 import org.elasticsearch.index.mapper.ObjectMapper;
 import org.elasticsearch.index.mapper.RootObjectMapper;
 import org.elasticsearch.index.mapper.RuntimeField;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.TestRuntimeField;
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.indices.IndicesModule;
@@ -69,10 +71,13 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import org.elasticsearch.search.lookup.LeafDocLookup;
 import org.elasticsearch.search.lookup.LeafSearchLookup;
+import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.search.sort.BucketedSort;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
 import org.mockito.stubbing.Answer;
 
 import java.io.IOException;
@@ -390,6 +395,30 @@ public void testSearchRequestRuntimeFieldsAndMultifieldDetection() {
         assertTrue(mappingLookup.isMultiField("cat.subfield"));
     }
 
+    public void testSyntheticSourceScriptLoading() throws IOException {
+
+        // Build a mapping using synthetic source
+        SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null).setSynthetic().build();
+        RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).build(MapperBuilderContext.root(true));
+        Mapping mapping = new Mapping(root, new MetadataFieldMapper[] { sourceMapper }, Map.of());
+        MappingLookup lookup = MappingLookup.fromMapping(mapping);
+
+        SearchExecutionContext sec = createSearchExecutionContext("index", "", lookup, Map.of());
+
+        // Attempting to access synthetic source via this context should throw an error
+        SearchLookup searchLookup = sec.lookup();
+        Exception e = expectThrows(IllegalArgumentException.class, () -> searchLookup.getSource(null, 0));
+        assertThat(e.getMessage(), equalTo("Cannot access source from scripts in synthetic mode"));
+
+        // Setting the source provider explicitly then gives us a new SearchLookup that can use source
+        Source source = Source.fromMap(Map.of("field", "value"), XContentType.JSON);
+        sec.setSourceProvider((ctx, doc) -> source);
+        SearchLookup searchLookup1 = sec.lookup();
+        assertNotSame(searchLookup, searchLookup1);
+        assertSame(source, searchLookup1.getSource(null, 0));
+
+    }
+
     public static SearchExecutionContext createSearchExecutionContext(String indexUuid, String clusterAlias) {
         return createSearchExecutionContext(indexUuid, clusterAlias, MappingLookup.EMPTY, Map.of());
     }
@@ -462,7 +491,6 @@ private static MapperService createMapperService(IndexSettings indexSettings, Ma
                 mapperRegistry.getRuntimeFieldParsers()::get,
                 indexSettings.getIndexVersionCreated(),
                 searchExecutionContextSupplier,
-                null,
                 ScriptCompiler.NONE,
                 indexAnalyzers,
                 indexSettings,
diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java
index 3cb1813199da..f040fd42f772 100644
--- a/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java
@@ -27,7 +27,7 @@
 import org.apache.lucene.search.TopTermsRewrite;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.index.RandomIndexWriter;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -166,8 +166,8 @@ public String fieldName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.V_EMPTY;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.ZERO;
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/query/WildcardIntervalsSourceProviderTests.java b/server/src/test/java/org/elasticsearch/index/query/WildcardIntervalsSourceProviderTests.java
index 702dd96b15b8..2e675c010132 100644
--- a/server/src/test/java/org/elasticsearch/index/query/WildcardIntervalsSourceProviderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/WildcardIntervalsSourceProviderTests.java
@@ -32,7 +32,7 @@ static Wildcard createRandomWildcard() {
     }
 
     @Override
-    protected Wildcard mutateInstance(Wildcard instance) throws IOException {
+    protected Wildcard mutateInstance(Wildcard instance) {
         String wildcard = instance.getPattern();
         String analyzer = instance.getAnalyzer();
         String useField = instance.getUseField();
diff --git a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
index 53dc2ba8edca..a6e90c6c2261 100644
--- a/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
@@ -13,6 +13,7 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.WildcardQuery;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -38,7 +39,7 @@ protected WildcardQueryBuilder doCreateTestQueryBuilder() {
     protected Map<String, WildcardQueryBuilder> getAlternateVersions() {
         Map<String, WildcardQueryBuilder> alternateVersions = new HashMap<>();
         WildcardQueryBuilder wildcardQuery = randomWildcardQuery();
-        String contentString = formatted("""
+        String contentString = Strings.format("""
             {
                 "wildcard" : {
                     "%s" : "%s"
diff --git a/server/src/test/java/org/elasticsearch/index/reindex/BulkByScrollResponseTests.java b/server/src/test/java/org/elasticsearch/index/reindex/BulkByScrollResponseTests.java
index 04ebb70ec328..244a66338ce9 100644
--- a/server/src/test/java/org/elasticsearch/index/reindex/BulkByScrollResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/index/reindex/BulkByScrollResponseTests.java
@@ -10,7 +10,7 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
 import org.elasticsearch.client.internal.transport.NoNodeAvailableException;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -82,7 +82,7 @@ private List<ScrollableHitSource.SearchFailure> randomSearchFailures() {
 
     private void assertResponseEquals(BulkByScrollResponse expected, BulkByScrollResponse actual) {
         assertEquals(expected.getTook(), actual.getTook());
-        BulkByScrollTaskStatusTests.assertTaskStatusEquals(Version.CURRENT, expected.getStatus(), actual.getStatus());
+        BulkByScrollTaskStatusTests.assertTaskStatusEquals(TransportVersion.CURRENT, expected.getStatus(), actual.getStatus());
         assertEquals(expected.getBulkFailures().size(), actual.getBulkFailures().size());
         for (int i = 0; i < expected.getBulkFailures().size(); i++) {
             Failure expectedFailure = expected.getBulkFailures().get(i);
diff --git a/server/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskStatusTests.java b/server/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskStatusTests.java
index b341e8c3503c..1d2ec8ae082e 100644
--- a/server/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskStatusTests.java
+++ b/server/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskStatusTests.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -44,14 +44,14 @@ public void testBulkByTaskStatus() throws IOException {
         BytesStreamOutput out = new BytesStreamOutput();
         status.writeTo(out);
         BulkByScrollTask.Status tripped = new BulkByScrollTask.Status(out.bytes().streamInput());
-        assertTaskStatusEquals(out.getVersion(), status, tripped);
+        assertTaskStatusEquals(out.getTransportVersion(), status, tripped);
     }
 
     /**
      * Assert that two task statuses are equal after serialization.
      * @param version the version at which expected was serialized
      */
-    public static void assertTaskStatusEquals(Version version, BulkByScrollTask.Status expected, BulkByScrollTask.Status actual) {
+    public static void assertTaskStatusEquals(TransportVersion version, BulkByScrollTask.Status expected, BulkByScrollTask.Status actual) {
         assertEquals(expected.getTotal(), actual.getTotal());
         assertEquals(expected.getUpdated(), actual.getUpdated());
         assertEquals(expected.getCreated(), actual.getCreated());
diff --git a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseStatsWireSerializingTests.java b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseStatsWireSerializingTests.java
index 475bd93454d1..cb302be67719 100644
--- a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseStatsWireSerializingTests.java
+++ b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseStatsWireSerializingTests.java
@@ -38,6 +38,11 @@ protected RetentionLeaseStats createTestInstance() {
         return new RetentionLeaseStats(new RetentionLeases(primaryTerm, version, leases));
     }
 
+    @Override
+    protected RetentionLeaseStats mutateInstance(RetentionLeaseStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<RetentionLeaseStats> instanceReader() {
         return RetentionLeaseStats::new;
diff --git a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionTests.java b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionTests.java
index 628fb6190441..7b8bded11de4 100644
--- a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionTests.java
@@ -108,7 +108,7 @@ public void testRetentionLeaseSyncActionOnPrimary() {
             // we should forward the request containing the current retention leases to the replica
             assertThat(result.replicaRequest(), sameInstance(request));
             // we should start with an empty replication response
-            assertNull(result.finalResponseIfSuccessful.getShardInfo());
+            assertNull(result.replicationResponse.getShardInfo());
         }));
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexLongFieldRangeWireTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexLongFieldRangeWireTests.java
index f66c2e93941d..685f251a804d 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/IndexLongFieldRangeWireTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/IndexLongFieldRangeWireTests.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.Arrays;
 
 import static org.elasticsearch.index.shard.IndexLongFieldRangeTestUtils.checkForSameInstances;
@@ -29,7 +28,7 @@ protected IndexLongFieldRange createTestInstance() {
     }
 
     @Override
-    protected IndexLongFieldRange mutateInstance(IndexLongFieldRange instance) throws IOException {
+    protected IndexLongFieldRange mutateInstance(IndexLongFieldRange instance) {
         if (instance == IndexLongFieldRange.UNKNOWN) {
             return IndexLongFieldRangeTestUtils.randomSpecificRange();
         }
diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java
index e0e9e7a46e3f..60a83219340f 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardOperationPermitsTests.java
@@ -7,6 +7,7 @@
  */
 package org.elasticsearch.index.shard;
 
+import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.settings.Settings;
@@ -16,6 +17,7 @@
 import org.elasticsearch.core.CheckedRunnable;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.FixedExecutorBuilder;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPoolStats;
@@ -56,6 +58,8 @@ public class IndexShardOperationPermitsTests extends ESTestCase {
 
     private IndexShardOperationPermits permits;
 
+    private static final String REJECTING_EXECUTOR = "rejecting";
+
     @BeforeClass
     public static void setupThreadPool() {
         int writeThreadPoolSize = randomIntBetween(1, 2);
@@ -65,7 +69,8 @@ public static void setupThreadPool() {
             Settings.builder()
                 .put("thread_pool." + ThreadPool.Names.WRITE + ".size", writeThreadPoolSize)
                 .put("thread_pool." + ThreadPool.Names.WRITE + ".queue_size", writeThreadPoolQueueSize)
-                .build()
+                .build(),
+            new FixedExecutorBuilder(Settings.EMPTY, REJECTING_EXECUTOR, 1, 0, REJECTING_EXECUTOR, false)
         );
         assertThat(threadPool.executor(ThreadPool.Names.WRITE), instanceOf(EsThreadPoolExecutor.class));
         assertThat(((EsThreadPoolExecutor) threadPool.executor(ThreadPool.Names.WRITE)).getCorePoolSize(), equalTo(writeThreadPoolSize));
@@ -96,8 +101,6 @@ public void checkNoInflightOperations() {
     public void testAllOperationsInvoked() throws InterruptedException, TimeoutException {
         int numThreads = 10;
 
-        class DummyException extends RuntimeException {}
-
         List<PlainActionFuture<Releasable>> futures = new ArrayList<>();
         List<Thread> operationThreads = new ArrayList<>();
         CountDownLatch latch = new CountDownLatch(numThreads / 4);
@@ -105,30 +108,17 @@ class DummyException extends RuntimeException {}
         for (int i = 0; i < numThreads; i++) {
             // the write thread pool uses a bounded size and can get rejections, see setupThreadPool
             String threadPoolName = randomFrom(ThreadPool.Names.WRITE, ThreadPool.Names.GENERIC);
-            boolean failingListener = randomBoolean();
             PlainActionFuture<Releasable> future = new PlainActionFuture<Releasable>() {
                 @Override
                 public void onResponse(Releasable releasable) {
                     releasable.close();
-                    if (failingListener) {
-                        throw new DummyException();
-                    } else {
-                        super.onResponse(releasable);
-                    }
-                }
-            };
-            Thread thread = new Thread() {
-                public void run() {
-                    latch.countDown();
-                    try {
-                        permits.acquire(future, threadPoolName, forceExecution, "");
-                    } catch (DummyException dummyException) {
-                        // ok, notify future
-                        assertTrue(failingListener);
-                        future.onFailure(dummyException);
-                    }
+                    super.onResponse(releasable);
                 }
             };
+            Thread thread = new Thread(() -> {
+                latch.countDown();
+                permits.acquire(future, threadPoolName, forceExecution, "");
+            });
             futures.add(future);
             operationThreads.add(thread);
         }
@@ -159,11 +149,10 @@ public void run() {
                 if (closeAfterBlocking) {
                     assertThat(
                         e.getCause(),
-                        either(instanceOf(DummyException.class)).or(instanceOf(EsRejectedExecutionException.class))
-                            .or(instanceOf(IndexShardClosedException.class))
+                        either(instanceOf(EsRejectedExecutionException.class)).or(instanceOf(IndexShardClosedException.class))
                     );
                 } else {
-                    assertThat(e.getCause(), either(instanceOf(DummyException.class)).or(instanceOf(EsRejectedExecutionException.class)));
+                    assertThat(e.getCause(), instanceOf(EsRejectedExecutionException.class));
                 }
             }
         }
@@ -531,26 +520,85 @@ public void testActiveOperationsCount() throws ExecutionException, InterruptedEx
         assertThat(permits.getActiveOperationsCount(), equalTo(0));
     }
 
-    public void testAsyncBlockOperationsOnFailure() throws InterruptedException {
-        final AtomicReference<Exception> reference = new AtomicReference<>();
-        final CountDownLatch onFailureLatch = new CountDownLatch(1);
-        permits.blockOperations(new ActionListener<Releasable>() {
-            @Override
-            public void onResponse(Releasable releasable) {
-                try (Releasable ignored = releasable) {
-                    throw new RuntimeException("simulated");
-                }
+    public void testAsyncBlockOperationsOnRejection() {
+        final PlainActionFuture<Void> threadBlock = new PlainActionFuture<>();
+        try (
+            Releasable firstPermit = PlainActionFuture.get(
+                f -> permits.acquire(f, ThreadPool.Names.GENERIC, false, ""),
+                0,
+                TimeUnit.SECONDS
+            )
+        ) {
+            assertNotNull(firstPermit);
+
+            threadPool.executor(REJECTING_EXECUTOR).execute(threadBlock::actionGet);
+            expectThrows(
+                EsRejectedExecutionException.class,
+                () -> PlainActionFuture.<Releasable, RuntimeException>get(
+                    f -> permits.blockOperations(f, 1, TimeUnit.HOURS, REJECTING_EXECUTOR)
+                )
+            );
+
+            // ensure that the exception means no block was put in place
+            try (
+                Releasable secondPermit = PlainActionFuture.get(
+                    f -> permits.acquire(f, ThreadPool.Names.GENERIC, false, ""),
+                    0,
+                    TimeUnit.SECONDS
+                )
+            ) {
+                assertNotNull(secondPermit);
             }
+        } finally {
+            threadBlock.onResponse(null);
+        }
 
-            @Override
-            public void onFailure(final Exception e) {
-                reference.set(e);
-                onFailureLatch.countDown();
+        // ensure that another block can still be acquired
+        try (Releasable block = PlainActionFuture.get(f -> permits.blockOperations(f, 1, TimeUnit.HOURS, ThreadPool.Names.GENERIC))) {
+            assertNotNull(block);
+        }
+    }
+
+    public void testAsyncBlockOperationsOnTimeout() {
+        final PlainActionFuture<Void> threadBlock = new PlainActionFuture<>();
+        try (
+            Releasable firstPermit = PlainActionFuture.get(
+                f -> permits.acquire(f, ThreadPool.Names.GENERIC, false, ""),
+                0,
+                TimeUnit.SECONDS
+            )
+        ) {
+            assertNotNull(firstPermit);
+
+            assertEquals(
+                "timeout while blocking operations after [0s]",
+                expectThrows(
+                    ElasticsearchTimeoutException.class,
+                    () -> PlainActionFuture.<Releasable, RuntimeException>get(
+                        f -> permits.blockOperations(f, 0, TimeUnit.SECONDS, ThreadPool.Names.GENERIC)
+                    )
+                ).getMessage()
+            );
+
+            // ensure that the exception means no block was put in place
+            try (
+                Releasable secondPermit = PlainActionFuture.get(
+                    f -> permits.acquire(f, ThreadPool.Names.GENERIC, false, ""),
+                    0,
+                    TimeUnit.SECONDS
+                )
+            ) {
+                assertNotNull(secondPermit);
             }
-        }, 10, TimeUnit.MINUTES, ThreadPool.Names.GENERIC);
-        onFailureLatch.await();
-        assertThat(reference.get(), instanceOf(RuntimeException.class));
-        assertThat(reference.get(), hasToString(containsString("simulated")));
+
+        } finally {
+            threadBlock.onResponse(null);
+        }
+
+        // ensure that another block can still be acquired
+        try (Releasable block = PlainActionFuture.get(f -> permits.blockOperations(f, 1, TimeUnit.HOURS, ThreadPool.Names.GENERIC))) {
+            assertNotNull(block);
+        }
     }
 
     public void testTimeout() throws BrokenBarrierException, InterruptedException {
diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 9d49cca516da..5a253f2e6bd1 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -482,7 +482,9 @@ public void onFailure(Exception e) {
         promoteReplica(
             indexShard,
             Collections.singleton(replicaRouting.allocationId().getId()),
-            new IndexShardRoutingTable.Builder(replicaRouting.shardId()).addShard(replicaRouting).build()
+            new IndexShardRoutingTable.Builder(replicaRouting.shardId()).addShard(
+                TestShardRouting.newShardRouting(replicaRouting.shardId(), "ignored", true, ShardRoutingState.STARTED)
+            ).addShard(replicaRouting).build()
         );
 
         final int delayedOperations = scaledRandomIntBetween(1, 64);
@@ -570,7 +572,9 @@ public void testPublishingOrderOnPromotion() throws IOException, InterruptedExce
         promoteReplica(
             indexShard,
             Collections.singleton(replicaRouting.allocationId().getId()),
-            new IndexShardRoutingTable.Builder(replicaRouting.shardId()).addShard(replicaRouting).build()
+            new IndexShardRoutingTable.Builder(replicaRouting.shardId()).addShard(
+                TestShardRouting.newShardRouting(replicaRouting.shardId(), "ignored", true, ShardRoutingState.STARTED)
+            ).addShard(replicaRouting).build()
         );
 
         stop.set(true);
@@ -593,7 +597,9 @@ public void testPrimaryFillsSeqNoGapsOnPromotion() throws Exception {
         promoteReplica(
             indexShard,
             Collections.singleton(replicaRouting.allocationId().getId()),
-            new IndexShardRoutingTable.Builder(replicaRouting.shardId()).addShard(replicaRouting).build()
+            new IndexShardRoutingTable.Builder(replicaRouting.shardId()).addShard(
+                TestShardRouting.newShardRouting(replicaRouting.shardId(), "ignored", true, ShardRoutingState.STARTED)
+            ).addShard(replicaRouting).build()
         );
 
         /*
@@ -2900,11 +2906,15 @@ public void testRecoverFromTranslog() throws IOException {
     }
 
     public void testShardActiveDuringInternalRecovery() throws IOException {
-        IndexShard shard = newStartedShard(true);
+        boolean isPrimary = randomBoolean();
+        IndexShard shard = newStartedShard(isPrimary);
         indexDoc(shard, "_doc", "0");
         shard = reinitShard(shard);
         DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
-        shard.markAsRecovering("for testing", new RecoveryState(shard.routingEntry(), localNode, null));
+        DiscoveryNode sourceNode = isPrimary
+            ? null
+            : new DiscoveryNode("bar", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
+        shard.markAsRecovering("for testing", new RecoveryState(shard.routingEntry(), localNode, sourceNode));
         // Shard is still inactive since we haven't started recovering yet
         assertFalse(shard.isActive());
         shard.prepareForIndexRecovery();
@@ -4519,7 +4529,9 @@ public void testCloseShardWhileEngineIsWarming() throws Exception {
                 config.getPrimaryTermSupplier(),
                 IndexModule.DEFAULT_SNAPSHOT_COMMIT_SUPPLIER,
                 config.getLeafSorter(),
-                config.getRelativeTimeInNanosSupplier()
+                config.getRelativeTimeInNanosSupplier(),
+                config.getIndexCommitListener(),
+                config.isPromotableToPrimary()
             );
             return new InternalEngine(configWithWarmer);
         });
diff --git a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
index 7f7a1e0cf2b4..9c62e725b3fe 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexModule;
@@ -65,7 +66,6 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import java.util.Locale;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -152,7 +152,9 @@ public void onFailedEngine(String reason, @Nullable Exception e) {
             () -> primaryTerm,
             IndexModule.DEFAULT_SNAPSHOT_COMMIT_SUPPLIER,
             null,
-            System::nanoTime
+            System::nanoTime,
+            null,
+            true
         );
         engine = new InternalEngine(config);
         engine.recoverFromTranslog((e, s) -> 0, Long.MAX_VALUE);
@@ -401,11 +403,11 @@ public void testLotsOfThreads() throws Exception {
         // These threads add and block until the refresh makes the change visible and then do a non-realtime get.
         Thread[] indexers = new Thread[threadCount];
         for (int thread = 0; thread < threadCount; thread++) {
-            final String threadId = formatted("%04d", thread);
+            final String threadId = Strings.format("%04d", thread);
             indexers[thread] = new Thread(() -> {
                 for (int iteration = 1; iteration <= 50; iteration++) {
                     try {
-                        String testFieldValue = String.format(Locale.ROOT, "%s%04d", threadId, iteration);
+                        String testFieldValue = Strings.format("%s%04d", threadId, iteration);
                         Engine.IndexResult index = index(threadId, testFieldValue);
                         assertEquals(iteration, index.getVersion());
 
diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardCountStatsTest.java b/server/src/test/java/org/elasticsearch/index/shard/ShardCountStatsTest.java
index 9e7d13018203..f052775371a1 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/ShardCountStatsTest.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/ShardCountStatsTest.java
@@ -36,4 +36,9 @@ protected Writeable.Reader<ShardCountStats> instanceReader() {
     protected ShardCountStats createTestInstance() {
         return new ShardCountStats(randomIntBetween(0, Integer.MAX_VALUE));
     }
+
+    @Override
+    protected ShardCountStats mutateInstance(ShardCountStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java b/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java
index 657a745d6b1e..db25e368704d 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.EngineTestCase;
@@ -131,7 +132,7 @@ private void runGetFromTranslogWithOptions(
             .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
             .build();
 
-        IndexMetadata metadata = IndexMetadata.builder("test").putMapping(formatted("""
+        IndexMetadata metadata = IndexMetadata.builder("test").putMapping(Strings.format("""
             {
               "properties": {
                 "foo": {
diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardIdTests.java b/server/src/test/java/org/elasticsearch/index/shard/ShardIdTests.java
index ae431a5bea3a..662521a58ae3 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/ShardIdTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/ShardIdTests.java
@@ -9,10 +9,35 @@
 package org.elasticsearch.index.shard;
 
 import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.index.IndexTests;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-public class ShardIdTests extends ESTestCase {
+public class ShardIdTests extends AbstractWireSerializingTestCase<ShardId> {
+
+    @Override
+    protected Writeable.Reader<ShardId> instanceReader() {
+        return ShardId::new;
+    }
+
+    @Override
+    protected ShardId createTestInstance() {
+        return new ShardId(randomIdentifier(), UUIDs.randomBase64UUID(), randomIntBetween(0, 99));
+    }
+
+    @Override
+    protected ShardId mutateInstance(ShardId instance) {
+        return mutate(instance);
+    }
+
+    public static ShardId mutate(ShardId instance) {
+        return switch (randomInt(1)) {
+            case 0 -> new ShardId(IndexTests.mutate(instance.getIndex()), instance.id());
+            case 1 -> new ShardId(instance.getIndex(), randomValueOtherThan(instance.id(), () -> randomIntBetween(0, 99)));
+            default -> throw new RuntimeException("unreachable");
+        };
+    }
 
     public void testShardIdFromString() {
         String indexName = randomAlphaOfLengthBetween(3, 50);
@@ -39,20 +64,4 @@ public void testShardIdFromString() {
         String badId3 = "[" + indexName + "][" + shardId; // missing closing bracket
         ex = expectThrows(IllegalArgumentException.class, () -> ShardId.fromString(badId3));
     }
-
-    public void testEquals() {
-        Index index1 = new Index("a", "a");
-        Index index2 = new Index("a", "b");
-        ShardId shardId1 = new ShardId(index1, 0);
-        ShardId shardId2 = new ShardId(index1, 0);
-        ShardId shardId3 = new ShardId(index2, 0);
-        ShardId shardId4 = new ShardId(index1, 1);
-        String s = "Some random other object";
-        assertEquals(shardId1, shardId1);
-        assertEquals(shardId1, shardId2);
-        assertNotEquals(shardId1, null);
-        assertNotEquals(shardId1, s);
-        assertNotEquals(shardId1, shardId3);
-        assertNotEquals(shardId1, shardId4);
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardLongFieldRangeWireTests.java b/server/src/test/java/org/elasticsearch/index/shard/ShardLongFieldRangeWireTests.java
index 7266b82e6ea4..4b4a5019498a 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/ShardLongFieldRangeWireTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/ShardLongFieldRangeWireTests.java
@@ -11,8 +11,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
-
 public class ShardLongFieldRangeWireTests extends AbstractWireSerializingTestCase<ShardLongFieldRange> {
     @Override
     protected Writeable.Reader<ShardLongFieldRange> instanceReader() {
@@ -39,7 +37,7 @@ static ShardLongFieldRange randomSpecificRange() {
     }
 
     @Override
-    protected ShardLongFieldRange mutateInstance(ShardLongFieldRange instance) throws IOException {
+    protected ShardLongFieldRange mutateInstance(ShardLongFieldRange instance) {
         if (instance == ShardLongFieldRange.UNKNOWN) {
             return randomBoolean() ? ShardLongFieldRange.EMPTY : randomSpecificRange();
         }
diff --git a/server/src/test/java/org/elasticsearch/index/store/ByteSizeCachingDirectoryTests.java b/server/src/test/java/org/elasticsearch/index/store/ByteSizeCachingDirectoryTests.java
index 332563e3e351..34960e58558b 100644
--- a/server/src/test/java/org/elasticsearch/index/store/ByteSizeCachingDirectoryTests.java
+++ b/server/src/test/java/org/elasticsearch/index/store/ByteSizeCachingDirectoryTests.java
@@ -87,6 +87,26 @@ public void testBasics() throws IOException {
             assertEquals(18, countingDir.numFileLengthCalls);
             assertEquals(15, cachingDir.estimateSizeInBytes());
             assertEquals(18, countingDir.numFileLengthCalls);
+
+            // Close more than once
+            IndexOutput out = cachingDir.createOutput("foo", IOContext.DEFAULT);
+            try {
+                out.writeBytes(new byte[5], 5);
+
+                cachingDir.estimateSizeInBytes();
+                // +3 because there are 3 files
+                assertEquals(21, countingDir.numFileLengthCalls);
+                // An index output is open so no caching
+                cachingDir.estimateSizeInBytes();
+                assertEquals(24, countingDir.numFileLengthCalls);
+            } finally {
+                out.close();
+                assertEquals(20, cachingDir.estimateSizeInBytes());
+                assertEquals(27, countingDir.numFileLengthCalls);
+            }
+            out.close();
+            assertEquals(20, cachingDir.estimateSizeInBytes());
+            assertEquals(27, countingDir.numFileLengthCalls);
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java
index 3d3cac048042..f4d94948bd78 100644
--- a/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java
+++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java
@@ -92,7 +92,8 @@ private Tuple<List<TranslogReader>, TranslogWriter> createReadersAndWriter() thr
                 new TragicExceptionHolder(),
                 seqNo -> {},
                 BigArrays.NON_RECYCLING_INSTANCE,
-                TranslogTests.RANDOMIZING_IO_BUFFERS
+                TranslogTests.RANDOMIZING_IO_BUFFERS,
+                (d, s, l) -> {}
             );
             writer = Mockito.spy(writer);
             byte[] bytes = new byte[4];
diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
index e4b5a6a20524..da7b257b087e 100644
--- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
+++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
@@ -76,6 +76,7 @@
 import java.io.Closeable;
 import java.io.EOFException;
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.file.CopyOption;
@@ -98,6 +99,7 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Queue;
 import java.util.Set;
 import java.util.concurrent.ArrayBlockingQueue;
@@ -275,6 +277,10 @@ private TranslogConfig getTranslogConfig(final Path path) {
     }
 
     private TranslogConfig getTranslogConfig(final Path path, final Settings settings) {
+        return getTranslogConfig(path, settings, null);
+    }
+
+    private TranslogConfig getTranslogConfig(final Path path, final Settings settings, OperationListener listener) {
         final ByteSizeValue bufferSize = randomFrom(
             TranslogConfig.DEFAULT_BUFFER_SIZE,
             new ByteSizeValue(8, ByteSizeUnit.KB),
@@ -288,7 +294,8 @@ private TranslogConfig getTranslogConfig(final Path path, final Settings setting
             indexSettings,
             NON_RECYCLING_INSTANCE,
             bufferSize,
-            randomBoolean() ? DiskIoBufferPool.INSTANCE : RANDOMIZING_IO_BUFFERS
+            randomBoolean() ? DiskIoBufferPool.INSTANCE : RANDOMIZING_IO_BUFFERS,
+            Objects.requireNonNullElse(listener, (d, s, l) -> {})
         );
     }
 
@@ -530,7 +537,7 @@ public void testStats() throws Exception {
                 builder.startObject();
                 copy.toXContent(builder, ToXContent.EMPTY_PARAMS);
                 builder.endObject();
-                assertThat(Strings.toString(builder), equalTo(XContentHelper.stripWhitespace(formatted("""
+                assertThat(Strings.toString(builder), equalTo(XContentHelper.stripWhitespace(Strings.format("""
                     {
                       "translog": {
                         "operations": 4,
@@ -1579,6 +1586,55 @@ ChannelFactory getChannelFactory() {
         }
     }
 
+    public void testTranslogOperationListener() throws IOException {
+        Path tempDir = createTempDir();
+        final Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, org.elasticsearch.Version.CURRENT).build();
+
+        final ArrayList<Long> seqNos = new ArrayList<>();
+        final ArrayList<Location> locations = new ArrayList<>();
+        final ArrayList<BytesReference> datas = new ArrayList<>();
+        OperationListener listener = (data, seqNo, location) -> {
+            seqNos.add(seqNo);
+            locations.add(location);
+            try (BytesStreamOutput output = new BytesStreamOutput()) {
+                try {
+                    data.writeTo(output);
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+                datas.add(output.bytes());
+            }
+
+        };
+        TranslogConfig config = getTranslogConfig(tempDir, settings, listener);
+
+        try (Translog translog = createTranslog(config)) {
+            Location location1 = translog.add(indexOp(randomAlphaOfLength(10), 0, primaryTerm.get()));
+            Location location2 = translog.add(TranslogOperationsUtils.indexOp(randomAlphaOfLength(10), 1, primaryTerm.get()));
+            long firstGeneration = translog.getGeneration().translogFileGeneration;
+            assertThat(location1.generation, equalTo(firstGeneration));
+            assertThat(location2.generation, equalTo(firstGeneration));
+
+            translog.rollGeneration();
+
+            Location location3 = translog.add(TranslogOperationsUtils.indexOp(randomAlphaOfLength(10), 3, primaryTerm.get()));
+            Location location4 = translog.add(TranslogOperationsUtils.indexOp(randomAlphaOfLength(10), 2, primaryTerm.get()));
+            long secondGeneration = translog.getGeneration().translogFileGeneration;
+            assertThat(location3.generation, equalTo(secondGeneration));
+            assertThat(location4.generation, equalTo(secondGeneration));
+
+            assertThat(seqNos, equalTo(List.of(0L, 1L, 3L, 2L)));
+            assertThat(locations, equalTo(List.of(location1, location2, location3, location4)));
+
+            for (int i = 0; i < 4; ++i) {
+                try (BufferedChecksumStreamInput stream = new BufferedChecksumStreamInput(datas.get(i).streamInput(), "test")) {
+                    Translog.Operation operation = Translog.readOperation(stream);
+                    assertThat(operation, equalTo(translog.readOperation(locations.get(i))));
+                }
+            }
+        }
+    }
+
     public void testCloseIntoReader() throws IOException {
         try (TranslogWriter writer = translog.createWriter(translog.currentFileGeneration() + 1)) {
             final int numOps = randomIntBetween(8, 128);
diff --git a/server/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/server/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
index b8cc5b2fd1c1..e49c027d25d1 100644
--- a/server/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
@@ -401,7 +401,9 @@ EngineConfig configWithRefreshListener(EngineConfig config, ReferenceManager.Ref
             config.getPrimaryTermSupplier(),
             config.getSnapshotCommitSupplier(),
             config.getLeafSorter(),
-            config.getRelativeTimeInNanosSupplier()
+            config.getRelativeTimeInNanosSupplier(),
+            config.getIndexCommitListener(),
+            config.isPromotableToPrimary()
         );
     }
 
diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
index c5b1c90df708..e1dcc3d09fdd 100644
--- a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
@@ -610,13 +610,18 @@ public void testConflictingEngineFactories() {
 
     public void testBuildAliasFilter() {
         var indicesService = getIndicesService();
-
         Metadata.Builder mdBuilder = Metadata.builder()
             .put(
                 indexBuilder("test-0").state(IndexMetadata.State.OPEN)
                     .putAlias(AliasMetadata.builder("test-alias-0").filter(Strings.toString(QueryBuilders.termQuery("foo", "bar"))))
                     .putAlias(AliasMetadata.builder("test-alias-1").filter(Strings.toString(QueryBuilders.termQuery("foo", "baz"))))
                     .putAlias(AliasMetadata.builder("test-alias-non-filtering"))
+            )
+            .put(
+                indexBuilder("test-1").state(IndexMetadata.State.OPEN)
+                    .putAlias(AliasMetadata.builder("test-alias-0").filter(Strings.toString(QueryBuilders.termQuery("foo", "bar"))))
+                    .putAlias(AliasMetadata.builder("test-alias-1").filter(Strings.toString(QueryBuilders.termQuery("foo", "bax"))))
+                    .putAlias(AliasMetadata.builder("test-alias-non-filtering"))
             );
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
         {
@@ -624,11 +629,21 @@ public void testBuildAliasFilter() {
             assertThat(result.getAliases(), arrayContainingInAnyOrder("test-alias-0"));
             assertThat(result.getQueryBuilder(), equalTo(QueryBuilders.termQuery("foo", "bar")));
         }
+        {
+            AliasFilter result = indicesService.buildAliasFilter(state, "test-1", Set.of("test-alias-0"));
+            assertThat(result.getAliases(), arrayContainingInAnyOrder("test-alias-0"));
+            assertThat(result.getQueryBuilder(), equalTo(QueryBuilders.termQuery("foo", "bar")));
+        }
         {
             AliasFilter result = indicesService.buildAliasFilter(state, "test-0", Set.of("test-alias-1"));
             assertThat(result.getAliases(), arrayContainingInAnyOrder("test-alias-1"));
             assertThat(result.getQueryBuilder(), equalTo(QueryBuilders.termQuery("foo", "baz")));
         }
+        {
+            AliasFilter result = indicesService.buildAliasFilter(state, "test-1", Set.of("test-alias-1"));
+            assertThat(result.getAliases(), arrayContainingInAnyOrder("test-alias-1"));
+            assertThat(result.getQueryBuilder(), equalTo(QueryBuilders.termQuery("foo", "bax")));
+        }
         {
             AliasFilter result = indicesService.buildAliasFilter(state, "test-0", Set.of("test-alias-0", "test-alias-1"));
             assertThat(result.getAliases(), arrayContainingInAnyOrder("test-alias-0", "test-alias-1"));
@@ -638,6 +653,15 @@ public void testBuildAliasFilter() {
             assertThat(filter.mustNot(), empty());
             assertThat(filter.should(), containsInAnyOrder(QueryBuilders.termQuery("foo", "baz"), QueryBuilders.termQuery("foo", "bar")));
         }
+        {
+            AliasFilter result = indicesService.buildAliasFilter(state, "test-1", Set.of("test-alias-0", "test-alias-1"));
+            assertThat(result.getAliases(), arrayContainingInAnyOrder("test-alias-0", "test-alias-1"));
+            BoolQueryBuilder filter = (BoolQueryBuilder) result.getQueryBuilder();
+            assertThat(filter.filter(), empty());
+            assertThat(filter.must(), empty());
+            assertThat(filter.mustNot(), empty());
+            assertThat(filter.should(), containsInAnyOrder(QueryBuilders.termQuery("foo", "bax"), QueryBuilders.termQuery("foo", "bar")));
+        }
         {
             AliasFilter result = indicesService.buildAliasFilter(
                 state,
@@ -647,18 +671,33 @@ public void testBuildAliasFilter() {
             assertThat(result.getAliases(), emptyArray());
             assertThat(result.getQueryBuilder(), nullValue());
         }
+        {
+            AliasFilter result = indicesService.buildAliasFilter(
+                state,
+                "test-1",
+                Set.of("test-alias-0", "test-alias-1", "test-alias-non-filtering")
+            );
+            assertThat(result.getAliases(), emptyArray());
+            assertThat(result.getQueryBuilder(), nullValue());
+        }
     }
 
     public void testBuildAliasFilterDataStreamAliases() {
         var indicesService = getIndicesService();
 
         final String dataStreamName1 = "logs-foobar";
+        final String dataStreamName2 = "logs-foobaz";
         IndexMetadata backingIndex1 = createBackingIndex(dataStreamName1, 1).build();
+        IndexMetadata backingIndex2 = createBackingIndex(dataStreamName2, 1).build();
         Metadata.Builder mdBuilder = Metadata.builder()
             .put(backingIndex1, false)
-            .put(DataStreamTestHelper.newInstance(dataStreamName1, List.of(backingIndex1.getIndex())));
+            .put(backingIndex2, false)
+            .put(DataStreamTestHelper.newInstance(dataStreamName1, List.of(backingIndex1.getIndex())))
+            .put(DataStreamTestHelper.newInstance(dataStreamName2, List.of(backingIndex2.getIndex())));
         mdBuilder.put("logs_foo", dataStreamName1, null, Strings.toString(QueryBuilders.termQuery("foo", "bar")));
+        mdBuilder.put("logs_foo", dataStreamName2, null, Strings.toString(QueryBuilders.termQuery("foo", "baz")));
         mdBuilder.put("logs", dataStreamName1, null, Strings.toString(QueryBuilders.termQuery("foo", "baz")));
+        mdBuilder.put("logs", dataStreamName2, null, Strings.toString(QueryBuilders.termQuery("foo", "bax")));
         mdBuilder.put("logs_bar", dataStreamName1, null, null);
         ClusterState state = ClusterState.builder(new ClusterName("_name")).metadata(mdBuilder).build();
         {
@@ -667,6 +706,12 @@ public void testBuildAliasFilterDataStreamAliases() {
             assertThat(result.getAliases(), arrayContainingInAnyOrder("logs_foo"));
             assertThat(result.getQueryBuilder(), equalTo(QueryBuilders.termQuery("foo", "bar")));
         }
+        {
+            String index = backingIndex2.getIndex().getName();
+            AliasFilter result = indicesService.buildAliasFilter(state, index, Set.of("logs_foo"));
+            assertThat(result.getAliases(), arrayContainingInAnyOrder("logs_foo"));
+            assertThat(result.getQueryBuilder(), equalTo(QueryBuilders.termQuery("foo", "baz")));
+        }
         {
             String index = backingIndex1.getIndex().getName();
             AliasFilter result = indicesService.buildAliasFilter(state, index, Set.of("logs_foo", "logs"));
@@ -677,6 +722,16 @@ public void testBuildAliasFilterDataStreamAliases() {
             assertThat(filter.mustNot(), empty());
             assertThat(filter.should(), containsInAnyOrder(QueryBuilders.termQuery("foo", "baz"), QueryBuilders.termQuery("foo", "bar")));
         }
+        {
+            String index = backingIndex2.getIndex().getName();
+            AliasFilter result = indicesService.buildAliasFilter(state, index, Set.of("logs_foo", "logs"));
+            assertThat(result.getAliases(), arrayContainingInAnyOrder("logs_foo", "logs"));
+            BoolQueryBuilder filter = (BoolQueryBuilder) result.getQueryBuilder();
+            assertThat(filter.filter(), empty());
+            assertThat(filter.must(), empty());
+            assertThat(filter.mustNot(), empty());
+            assertThat(filter.should(), containsInAnyOrder(QueryBuilders.termQuery("foo", "baz"), QueryBuilders.termQuery("foo", "bax")));
+        }
         {
             String index = backingIndex1.getIndex().getName();
             AliasFilter result = indicesService.buildAliasFilter(state, index, Set.of("logs_foo", "logs", "logs_bar"));
diff --git a/server/src/test/java/org/elasticsearch/indices/SystemIndexManagerTests.java b/server/src/test/java/org/elasticsearch/indices/SystemIndexManagerTests.java
index 986c51074dfd..8694a5680588 100644
--- a/server/src/test/java/org/elasticsearch/indices/SystemIndexManagerTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/SystemIndexManagerTests.java
@@ -316,7 +316,8 @@ private ClusterState markShardsUnavailable(ClusterState.Builder clusterStateBuil
                                     new ShardId(prevIndex, 0),
                                     true,
                                     RecoverySource.ExistingStoreRecoverySource.INSTANCE,
-                                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")
+                                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""),
+                                    ShardRouting.Role.DEFAULT
                                 )
                                     .initialize(UUIDs.randomBase64UUID(random()), null, 0L)
                                     .moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, ""))
@@ -375,7 +376,8 @@ private static RoutingTable buildIndexRoutingTable(Index index) {
             new ShardId(index, 0),
             true,
             RecoverySource.ExistingStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""),
+            ShardRouting.Role.DEFAULT
         );
         String nodeId = ESTestCase.randomAlphaOfLength(8);
         return RoutingTable.builder()
diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java
index 68913d53c4f0..ef07c5efe43c 100644
--- a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java
@@ -9,23 +9,14 @@
 package org.elasticsearch.indices.analysis;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.CharFilter;
-import org.apache.lucene.analysis.FilteringTokenFilter;
-import org.apache.lucene.analysis.TokenFilter;
-import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.charfilter.MappingCharFilter;
-import org.apache.lucene.analysis.charfilter.NormalizeCharMap;
 import org.apache.lucene.analysis.hunspell.Dictionary;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.util.CharTokenizer;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.analysis.MockTokenizer;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.index.IndexSettings;
@@ -43,12 +34,9 @@
 import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.index.analysis.TokenizerFactory;
 import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
-import org.elasticsearch.plugin.analysis.api.AnalysisMode;
-import org.elasticsearch.plugin.api.NamedComponent;
+import org.elasticsearch.indices.analysis.lucene.AppendCharFilter;
+import org.elasticsearch.indices.analysis.lucene.AppendTokenFilter;
 import org.elasticsearch.plugins.AnalysisPlugin;
-import org.elasticsearch.plugins.scanners.NameToPluginInfo;
-import org.elasticsearch.plugins.scanners.NamedComponentReader;
-import org.elasticsearch.plugins.scanners.PluginInfo;
 import org.elasticsearch.plugins.scanners.StablePluginsRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
@@ -59,9 +47,6 @@
 import java.io.BufferedWriter;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.Reader;
-import java.io.StringReader;
-import java.io.UncheckedIOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -70,7 +55,6 @@
 import java.util.Map;
 import java.util.Set;
 
-import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
 import static java.util.Collections.singletonMap;
 import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents;
@@ -476,247 +460,4 @@ public Map<String, Dictionary> getHunspellDictionaries() {
         assertSame(dictionary, module.getHunspellService().getDictionary("foo"));
     }
 
-    @NamedComponent("stableCharFilterFactory")
-    public static class TestCharFilterFactory implements org.elasticsearch.plugin.analysis.api.CharFilterFactory {
-        @SuppressForbidden(reason = "need a public constructor")
-        public TestCharFilterFactory() {}
-
-        @Override
-        public Reader create(Reader reader) {
-            return new ReplaceHash(reader);
-        }
-
-        @Override
-        public Reader normalize(Reader reader) {
-            return new ReplaceHash(reader);
-        }
-
-    }
-
-    static class ReplaceHash extends MappingCharFilter {
-
-        ReplaceHash(Reader in) {
-            super(charMap(), in);
-        }
-
-        private static NormalizeCharMap charMap() {
-            NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
-            builder.add("#", "3");
-            return builder.build();
-        }
-    }
-
-    @NamedComponent("stableTokenFilterFactory")
-    public static class TestTokenFilterFactory implements org.elasticsearch.plugin.analysis.api.TokenFilterFactory {
-
-        @SuppressForbidden(reason = "need a public constructor")
-        public TestTokenFilterFactory() {}
-
-        @Override
-        public TokenStream create(TokenStream tokenStream) {
-
-            return new Skip1TokenFilter(tokenStream);
-        }
-
-        @Override
-        public TokenStream normalize(TokenStream tokenStream) {
-            return new AppendTokenFilter(tokenStream, "1");
-        }
-
-        @Override
-        public AnalysisMode getAnalysisMode() {
-            return org.elasticsearch.plugin.analysis.api.TokenFilterFactory.super.getAnalysisMode();
-        }
-
-    }
-
-    static class Skip1TokenFilter extends FilteringTokenFilter {
-
-        private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
-
-        Skip1TokenFilter(TokenStream in) {
-            super(in);
-        }
-
-        @Override
-        protected boolean accept() throws IOException {
-            return termAtt.buffer()[0] != '1';
-        }
-    }
-
-    @NamedComponent("stableTokenizerFactory")
-    public static class TestTokenizerFactory implements org.elasticsearch.plugin.analysis.api.TokenizerFactory {
-        @SuppressForbidden(reason = "need a public constructor")
-        public TestTokenizerFactory() {}
-
-        @Override
-        public Tokenizer create() {
-            return new UnderscoreTokenizer();
-        }
-
-    }
-
-    static class UnderscoreTokenizer extends CharTokenizer {
-
-        @Override
-        protected boolean isTokenChar(int c) {
-            return c != '_';
-        }
-    }
-
-    @NamedComponent("stableAnalyzerFactory")
-    public static class TestAnalyzerFactory implements org.elasticsearch.plugin.analysis.api.AnalyzerFactory {
-
-        @Override
-        public Analyzer create() {
-            return new CustomAnalyzer();
-        }
-
-        static class CustomAnalyzer extends Analyzer {
-
-            @Override
-            protected TokenStreamComponents createComponents(String fieldName) {
-                var tokenizer = new UnderscoreTokenizer();
-                var tokenFilter = new Skip1TokenFilter(tokenizer);
-                return new TokenStreamComponents(r -> tokenizer.setReader(new ReplaceHash(r)), tokenFilter);
-            }
-        }
-    }
-
-    public void testStablePlugins() throws IOException {
-        ClassLoader classLoader = getClass().getClassLoader();
-        AnalysisRegistry registry = new AnalysisModule(
-            TestEnvironment.newEnvironment(emptyNodeSettings),
-            emptyList(),
-            new StablePluginsRegistry(
-                new NamedComponentReader(),
-                Map.of(
-                    org.elasticsearch.plugin.analysis.api.CharFilterFactory.class.getCanonicalName(),
-                    new NameToPluginInfo(
-                        Map.of(
-                            "stableCharFilterFactory",
-                            new PluginInfo("stableCharFilterFactory", TestCharFilterFactory.class.getName(), classLoader)
-                        )
-                    ),
-                    org.elasticsearch.plugin.analysis.api.TokenFilterFactory.class.getCanonicalName(),
-                    new NameToPluginInfo(
-                        Map.of(
-                            "stableTokenFilterFactory",
-                            new PluginInfo("stableTokenFilterFactory", TestTokenFilterFactory.class.getName(), classLoader)
-                        )
-                    ),
-                    org.elasticsearch.plugin.analysis.api.TokenizerFactory.class.getCanonicalName(),
-                    new NameToPluginInfo(
-                        Map.of(
-                            "stableTokenizerFactory",
-                            new PluginInfo("stableTokenizerFactory", TestTokenizerFactory.class.getName(), classLoader)
-                        )
-                    ),
-                    org.elasticsearch.plugin.analysis.api.AnalyzerFactory.class.getCanonicalName(),
-                    new NameToPluginInfo(
-                        Map.of(
-                            "stableAnalyzerFactory",
-                            new PluginInfo("stableAnalyzerFactory", TestAnalyzerFactory.class.getName(), classLoader)
-                        )
-                    )
-                )
-            )
-        ).getAnalysisRegistry();
-
-        Version version = VersionUtils.randomVersion(random());
-        IndexAnalyzers analyzers = getIndexAnalyzers(
-            registry,
-            Settings.builder()
-                .put("index.analysis.analyzer.char_filter_test.tokenizer", "standard")
-                .put("index.analysis.analyzer.char_filter_test.char_filter", "stableCharFilterFactory")
-
-                .put("index.analysis.analyzer.token_filter_test.tokenizer", "standard")
-                .put("index.analysis.analyzer.token_filter_test.filter", "stableTokenFilterFactory")
-
-                .put("index.analysis.analyzer.tokenizer_test.tokenizer", "stableTokenizerFactory")
-
-                .put("index.analysis.analyzer.analyzer_provider_test.type", "stableAnalyzerFactory")
-
-                .put(IndexMetadata.SETTING_VERSION_CREATED, version)
-                .build()
-        );
-        assertTokenStreamContents(analyzers.get("char_filter_test").tokenStream("", "t#st"), new String[] { "t3st" });
-        assertTokenStreamContents(
-            analyzers.get("token_filter_test").tokenStream("", "1test 2test 1test 3test "),
-            new String[] { "2test", "3test" }
-        );
-        assertTokenStreamContents(analyzers.get("tokenizer_test").tokenStream("", "x_y_z"), new String[] { "x", "y", "z" });
-        assertTokenStreamContents(analyzers.get("analyzer_provider_test").tokenStream("", "1x_y_#z"), new String[] { "y", "3z" });
-
-        assertThat(analyzers.get("char_filter_test").normalize("", "t#st").utf8ToString(), equalTo("t3st"));
-        assertThat(
-            analyzers.get("token_filter_test").normalize("", "1test 2test 1test 3test ").utf8ToString(),
-            equalTo("1test 2test 1test 3test 1")
-        );
-
-        // TODO does it makes sense to test normalize on tokenizer and analyzer?
-    }
-
-    // Simple char filter that appends text to the term
-    public static class AppendCharFilter extends CharFilter {
-
-        static Reader append(Reader input, String appendMe) {
-            try {
-                return new StringReader(Streams.copyToString(input) + appendMe);
-            } catch (IOException e) {
-                throw new UncheckedIOException(e);
-            }
-        }
-
-        public AppendCharFilter(Reader input, String appendMe) {
-            super(append(input, appendMe));
-        }
-
-        @Override
-        protected int correct(int currentOff) {
-            return currentOff;
-        }
-
-        @Override
-        public int read(char[] cbuf, int off, int len) throws IOException {
-            return input.read(cbuf, off, len);
-        }
-    }
-
-    // Simple token filter that appends text to the term
-    private static class AppendTokenFilter extends TokenFilter {
-        public static TokenFilterFactory factoryForSuffix(String suffix) {
-            return new TokenFilterFactory() {
-                @Override
-                public String name() {
-                    return suffix;
-                }
-
-                @Override
-                public TokenStream create(TokenStream tokenStream) {
-                    return new AppendTokenFilter(tokenStream, suffix);
-                }
-            };
-        }
-
-        private final CharTermAttribute term = addAttribute(CharTermAttribute.class);
-        private final char[] appendMe;
-
-        protected AppendTokenFilter(TokenStream input, String appendMe) {
-            super(input);
-            this.appendMe = appendMe.toCharArray();
-        }
-
-        @Override
-        public boolean incrementToken() throws IOException {
-            if (false == input.incrementToken()) {
-                return false;
-            }
-            term.resizeBuffer(term.length() + appendMe.length);
-            System.arraycopy(appendMe, 0, term.buffer(), term.length(), appendMe.length);
-            term.setLength(term.length() + appendMe.length);
-            return true;
-        }
-    }
-
 }
diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/IncorrectSetupStablePluginsTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/IncorrectSetupStablePluginsTests.java
new file mode 100644
index 000000000000..68d879686269
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/indices/analysis/IncorrectSetupStablePluginsTests.java
@@ -0,0 +1,153 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.indices.analysis;
+
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AnalysisRegistry;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.indices.analysis.lucene.ReplaceCharToNumber;
+import org.elasticsearch.plugin.Inject;
+import org.elasticsearch.plugin.NamedComponent;
+import org.elasticsearch.plugin.analysis.CharFilterFactory;
+import org.elasticsearch.plugin.settings.AnalysisSettings;
+import org.elasticsearch.plugins.scanners.NameToPluginInfo;
+import org.elasticsearch.plugins.scanners.NamedComponentReader;
+import org.elasticsearch.plugins.scanners.PluginInfo;
+import org.elasticsearch.plugins.scanners.StablePluginsRegistry;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.test.VersionUtils;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.Map;
+
+import static java.util.Collections.emptyList;
+import static org.hamcrest.Matchers.equalTo;
+
+public class IncorrectSetupStablePluginsTests extends ESTestCase {
+    ClassLoader classLoader = getClass().getClassLoader();
+
+    private final Settings emptyNodeSettings = Settings.builder()
+        .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+        .build();
+
+    public @interface IncorrectAnnotation {
+    }
+
+    @IncorrectAnnotation
+    public interface IncorrectlyAnnotatedSettings {}
+
+    @NamedComponent("incorrectlyAnnotatedSettings")
+    public static class IncorrectlyAnnotatedSettingsCharFilter extends AbstractCharFilterFactory {
+        @Inject
+        public IncorrectlyAnnotatedSettingsCharFilter(IncorrectlyAnnotatedSettings settings) {}
+    }
+
+    public void testIncorrectlyAnnotatedSettingsClass() throws IOException {
+        var e = expectThrows(
+            IllegalArgumentException.class,
+            () -> getIndexAnalyzers(
+                Settings.builder()
+                    .put("index.analysis.analyzer.char_filter_test.tokenizer", "standard")
+                    .put("index.analysis.analyzer.char_filter_test.char_filter", "incorrectlyAnnotatedSettings")
+                    .put(IndexMetadata.SETTING_VERSION_CREATED, VersionUtils.randomVersion(random()))
+                    .build(),
+                Map.of(
+                    "incorrectlyAnnotatedSettings",
+                    new PluginInfo("incorrectlyAnnotatedSettings", IncorrectlyAnnotatedSettingsCharFilter.class.getName(), classLoader)
+                )
+            )
+        );
+        assertThat(e.getMessage(), equalTo("Parameter is not instance of a class annotated with settings annotation."));
+    }
+
+    @AnalysisSettings
+    public interface OkAnalysisSettings {}
+
+    @NamedComponent("noInjectCharFilter")
+    public static class NoInjectCharFilter extends AbstractCharFilterFactory {
+
+        public NoInjectCharFilter(OkAnalysisSettings settings) {}
+    }
+
+    public void testIncorrectlyAnnotatedConstructor() throws IOException {
+        var e = expectThrows(
+            IllegalStateException.class,
+            () -> getIndexAnalyzers(
+                Settings.builder()
+                    .put("index.analysis.analyzer.char_filter_test.tokenizer", "standard")
+                    .put("index.analysis.analyzer.char_filter_test.char_filter", "noInjectCharFilter")
+                    .put(IndexMetadata.SETTING_VERSION_CREATED, VersionUtils.randomVersion(random()))
+                    .build(),
+                Map.of("noInjectCharFilter", new PluginInfo("noInjectCharFilter", NoInjectCharFilter.class.getName(), classLoader))
+            )
+        );
+        assertThat(e.getMessage(), equalTo("Missing @Inject annotation for constructor with settings."));
+    }
+
+    @NamedComponent("multipleConstructors")
+    public static class MultipleConstructors extends AbstractCharFilterFactory {
+        public MultipleConstructors() {}
+
+        public MultipleConstructors(OkAnalysisSettings settings) {}
+    }
+
+    public void testMultiplePublicConstructors() throws IOException {
+        var e = expectThrows(
+            IllegalStateException.class,
+            () -> getIndexAnalyzers(
+                Settings.builder()
+                    .put("index.analysis.analyzer.char_filter_test.tokenizer", "standard")
+                    .put("index.analysis.analyzer.char_filter_test.char_filter", "multipleConstructors")
+                    .put(IndexMetadata.SETTING_VERSION_CREATED, VersionUtils.randomVersion(random()))
+                    .build(),
+                Map.of("multipleConstructors", new PluginInfo("multipleConstructors", MultipleConstructors.class.getName(), classLoader))
+            )
+        );
+        assertThat(e.getMessage(), equalTo("Plugin can only have one public constructor."));
+    }
+
+    public IndexAnalyzers getIndexAnalyzers(Settings settings, Map<String, PluginInfo> mapOfCharFilters) throws IOException {
+        AnalysisRegistry registry = setupRegistry(mapOfCharFilters);
+
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
+        return registry.build(idxSettings);
+    }
+
+    private AnalysisRegistry setupRegistry(Map<String, PluginInfo> mapOfCharFilters) throws IOException {
+
+        AnalysisRegistry registry = new AnalysisModule(
+            TestEnvironment.newEnvironment(emptyNodeSettings),
+            emptyList(),
+            new StablePluginsRegistry(
+                new NamedComponentReader(),
+                Map.of(CharFilterFactory.class.getCanonicalName(), new NameToPluginInfo(mapOfCharFilters))
+            )
+        ).getAnalysisRegistry();
+        return registry;
+    }
+
+    public abstract static class AbstractCharFilterFactory implements CharFilterFactory {
+
+        @Override
+        public Reader create(Reader reader) {
+            return new ReplaceCharToNumber(reader, "#", 3);
+        }
+
+        @Override
+        public Reader normalize(Reader reader) {
+            return new ReplaceCharToNumber(reader, "#", 3);
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsNoSettingsTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsNoSettingsTests.java
new file mode 100644
index 000000000000..76d3ff14c9f4
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsNoSettingsTests.java
@@ -0,0 +1,198 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.indices.analysis;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AnalysisRegistry;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.indices.analysis.lucene.AppendTokenFilter;
+import org.elasticsearch.indices.analysis.lucene.CharSkippingTokenizer;
+import org.elasticsearch.indices.analysis.lucene.ReplaceCharToNumber;
+import org.elasticsearch.indices.analysis.lucene.SkipStartingWithDigitTokenFilter;
+import org.elasticsearch.plugin.NamedComponent;
+import org.elasticsearch.plugin.analysis.AnalysisMode;
+import org.elasticsearch.plugin.analysis.AnalyzerFactory;
+import org.elasticsearch.plugin.analysis.CharFilterFactory;
+import org.elasticsearch.plugin.analysis.TokenFilterFactory;
+import org.elasticsearch.plugin.analysis.TokenizerFactory;
+import org.elasticsearch.plugins.scanners.NameToPluginInfo;
+import org.elasticsearch.plugins.scanners.NamedComponentReader;
+import org.elasticsearch.plugins.scanners.PluginInfo;
+import org.elasticsearch.plugins.scanners.StablePluginsRegistry;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.test.VersionUtils;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.emptyList;
+import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents;
+import static org.hamcrest.Matchers.equalTo;
+
+public class StableAnalysisPluginsNoSettingsTests extends ESTestCase {
+    private final Settings emptyNodeSettings = Settings.builder()
+        .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+        .build();
+
+    public IndexAnalyzers getIndexAnalyzers(Settings settings) throws IOException {
+        AnalysisRegistry registry = setupRegistry();
+
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
+        return registry.build(idxSettings);
+    }
+
+    public void testStablePlugins() throws IOException {
+        Version version = VersionUtils.randomVersion(random());
+        IndexAnalyzers analyzers = getIndexAnalyzers(
+            Settings.builder()
+                .put("index.analysis.analyzer.char_filter_test.tokenizer", "standard")
+                .put("index.analysis.analyzer.char_filter_test.char_filter", "stableCharFilterFactory")
+
+                .put("index.analysis.analyzer.token_filter_test.tokenizer", "standard")
+                .put("index.analysis.analyzer.token_filter_test.filter", "stableTokenFilterFactory")
+
+                .put("index.analysis.analyzer.tokenizer_test.tokenizer", "stableTokenizerFactory")
+
+                .put("index.analysis.analyzer.analyzer_provider_test.type", "stableAnalyzerFactory")
+
+                .put(IndexMetadata.SETTING_VERSION_CREATED, version)
+                .build()
+        );
+        assertTokenStreamContents(analyzers.get("char_filter_test").tokenStream("", "t#st"), new String[] { "t3st" });
+        assertTokenStreamContents(
+            analyzers.get("token_filter_test").tokenStream("", "1test 2test 1test 3test "),
+            new String[] { "2test", "3test" }
+        );
+        assertTokenStreamContents(analyzers.get("tokenizer_test").tokenStream("", "x_y_z"), new String[] { "x", "y", "z" });
+        assertTokenStreamContents(analyzers.get("analyzer_provider_test").tokenStream("", "1x_y_#z"), new String[] { "y", "3z" });
+
+        assertThat(analyzers.get("char_filter_test").normalize("", "t#st").utf8ToString(), equalTo("t3st"));
+        assertThat(
+            analyzers.get("token_filter_test").normalize("", "1test 2test 1test 3test ").utf8ToString(),
+            equalTo("1test 2test 1test 3test 1")
+        );
+    }
+
+    @NamedComponent("stableCharFilterFactory")
+    public static class TestCharFilterFactory implements CharFilterFactory {
+
+        @Override
+        public Reader create(Reader reader) {
+            return new ReplaceCharToNumber(reader, "#", 3);
+        }
+
+        @Override
+        public Reader normalize(Reader reader) {
+            return new ReplaceCharToNumber(reader, "#", 3);
+        }
+    }
+
+    @NamedComponent("stableTokenFilterFactory")
+    public static class TestTokenFilterFactory implements TokenFilterFactory {
+
+        @Override
+        public TokenStream create(TokenStream tokenStream) {
+            return new SkipStartingWithDigitTokenFilter(tokenStream, 1);
+        }
+
+        @Override
+        public TokenStream normalize(TokenStream tokenStream) {
+            return new AppendTokenFilter(tokenStream, "1");
+        }
+
+        @Override
+        public AnalysisMode getAnalysisMode() {
+            return TokenFilterFactory.super.getAnalysisMode();
+        }
+
+    }
+
+    @NamedComponent("stableTokenizerFactory")
+    public static class TestTokenizerFactory implements TokenizerFactory {
+
+        @Override
+        public Tokenizer create() {
+            return new CharSkippingTokenizer(List.of("_"));
+        }
+
+    }
+
+    @NamedComponent("stableAnalyzerFactory")
+    public static class TestAnalyzerFactory implements AnalyzerFactory {
+
+        @Override
+        public Analyzer create() {
+            return new CustomAnalyzer();
+        }
+
+        static class CustomAnalyzer extends Analyzer {
+
+            @Override
+            protected TokenStreamComponents createComponents(String fieldName) {
+                var tokenizer = new CharSkippingTokenizer(List.of("_"));
+                var tokenFilter = new SkipStartingWithDigitTokenFilter(tokenizer, 1);
+                return new TokenStreamComponents(r -> tokenizer.setReader(new ReplaceCharToNumber(r, "#", 3)), tokenFilter);
+            }
+        }
+    }
+
+    private AnalysisRegistry setupRegistry() throws IOException {
+        ClassLoader classLoader = getClass().getClassLoader();
+
+        AnalysisRegistry registry = new AnalysisModule(
+            TestEnvironment.newEnvironment(emptyNodeSettings),
+            emptyList(),
+            new StablePluginsRegistry(
+                new NamedComponentReader(),
+                Map.of(
+                    CharFilterFactory.class.getCanonicalName(),
+                    new NameToPluginInfo(
+                        Map.of(
+                            "stableCharFilterFactory",
+                            new PluginInfo("stableCharFilterFactory", TestCharFilterFactory.class.getName(), classLoader)
+                        )
+                    ),
+                    TokenFilterFactory.class.getCanonicalName(),
+                    new NameToPluginInfo(
+                        Map.of(
+                            "stableTokenFilterFactory",
+                            new PluginInfo("stableTokenFilterFactory", TestTokenFilterFactory.class.getName(), classLoader)
+                        )
+                    ),
+                    TokenizerFactory.class.getCanonicalName(),
+                    new NameToPluginInfo(
+                        Map.of(
+                            "stableTokenizerFactory",
+                            new PluginInfo("stableTokenizerFactory", TestTokenizerFactory.class.getName(), classLoader)
+                        )
+                    ),
+                    AnalyzerFactory.class.getCanonicalName(),
+                    new NameToPluginInfo(
+                        Map.of(
+                            "stableAnalyzerFactory",
+                            new PluginInfo("stableAnalyzerFactory", TestAnalyzerFactory.class.getName(), classLoader)
+                        )
+                    )
+                )
+            )
+        ).getAnalysisRegistry();
+        return registry;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsWithSettingsTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsWithSettingsTests.java
new file mode 100644
index 000000000000..0a8d38b32c09
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/indices/analysis/StableAnalysisPluginsWithSettingsTests.java
@@ -0,0 +1,300 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.indices.analysis;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AnalysisRegistry;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.indices.analysis.lucene.AppendTokenFilter;
+import org.elasticsearch.indices.analysis.lucene.CharSkippingTokenizer;
+import org.elasticsearch.indices.analysis.lucene.ReplaceCharToNumber;
+import org.elasticsearch.indices.analysis.lucene.SkipStartingWithDigitTokenFilter;
+import org.elasticsearch.plugin.Inject;
+import org.elasticsearch.plugin.NamedComponent;
+import org.elasticsearch.plugin.analysis.AnalysisMode;
+import org.elasticsearch.plugin.analysis.AnalyzerFactory;
+import org.elasticsearch.plugin.analysis.CharFilterFactory;
+import org.elasticsearch.plugin.analysis.TokenFilterFactory;
+import org.elasticsearch.plugin.analysis.TokenizerFactory;
+import org.elasticsearch.plugin.settings.AnalysisSettings;
+import org.elasticsearch.plugin.settings.BooleanSetting;
+import org.elasticsearch.plugin.settings.IntSetting;
+import org.elasticsearch.plugin.settings.ListSetting;
+import org.elasticsearch.plugin.settings.LongSetting;
+import org.elasticsearch.plugin.settings.StringSetting;
+import org.elasticsearch.plugins.scanners.NameToPluginInfo;
+import org.elasticsearch.plugins.scanners.NamedComponentReader;
+import org.elasticsearch.plugins.scanners.PluginInfo;
+import org.elasticsearch.plugins.scanners.StablePluginsRegistry;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.test.VersionUtils;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.Map;
+
+import static java.util.Collections.emptyList;
+import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents;
+import static org.hamcrest.Matchers.equalTo;
+
+public class StableAnalysisPluginsWithSettingsTests extends ESTestCase {
+
+    protected final Settings emptyNodeSettings = Settings.builder()
+        .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+        .build();
+
+    public void testCharFilters() throws IOException {
+        IndexAnalyzers analyzers = getIndexAnalyzers(
+            Settings.builder()
+                .put("index.analysis.char_filter.my_char_filter.type", "stableCharFilterFactory")
+                .put("index.analysis.char_filter.my_char_filter.old_char", "#")
+                .put("index.analysis.char_filter.my_char_filter.new_number", 3)
+
+                .put("index.analysis.analyzer.char_filter_test.tokenizer", "standard")
+                .put("index.analysis.analyzer.char_filter_test.char_filter", "my_char_filter")
+
+                .put("index.analysis.analyzer.char_filter_with_defaults_test.tokenizer", "standard")
+                .put("index.analysis.analyzer.char_filter_with_defaults_test.char_filter", "stableCharFilterFactory")
+
+                .put(IndexMetadata.SETTING_VERSION_CREATED, VersionUtils.randomVersion(random()))
+                .build()
+        );
+        assertTokenStreamContents(analyzers.get("char_filter_test").tokenStream("", "t#st"), new String[] { "t3st" });
+        assertTokenStreamContents(analyzers.get("char_filter_with_defaults_test").tokenStream("", "t t"), new String[] { "t0t" });
+        assertThat(analyzers.get("char_filter_test").normalize("", "t#st").utf8ToString(), equalTo("t3st"));
+    }
+
+    public void testTokenFilters() throws IOException {
+        IndexAnalyzers analyzers = getIndexAnalyzers(
+            Settings.builder()
+                .put("index.analysis.filter.my_token_filter.type", "stableTokenFilterFactory")
+                .put("index.analysis.filter.my_token_filter.token_filter_number", 1L)
+
+                .put("index.analysis.analyzer.token_filter_test.tokenizer", "standard")
+                .put("index.analysis.analyzer.token_filter_test.filter", "my_token_filter")
+                .put(IndexMetadata.SETTING_VERSION_CREATED, VersionUtils.randomVersion(random()))
+                .build()
+        );
+        assertTokenStreamContents(
+            analyzers.get("token_filter_test").tokenStream("", "1test 2test 1test 3test "),
+            new String[] { "2test", "3test" }
+        );
+
+        assertThat(
+            analyzers.get("token_filter_test").normalize("", "1test 2test 1test 3test ").utf8ToString(),
+            equalTo("1test 2test 1test 3test 1")
+        );
+    }
+
+    public void testTokenizer() throws IOException {
+        IndexAnalyzers analyzers = getIndexAnalyzers(
+            Settings.builder()
+                .put("index.analysis.tokenizer.my_tokenizer.type", "stableTokenizerFactory")
+                .putList("index.analysis.tokenizer.my_tokenizer.tokenizer_list_of_chars", "_", " ")
+
+                .put("index.analysis.analyzer.tokenizer_test.tokenizer", "my_tokenizer")
+                .put(IndexMetadata.SETTING_VERSION_CREATED, VersionUtils.randomVersion(random()))
+                .build()
+        );
+        assertTokenStreamContents(analyzers.get("tokenizer_test").tokenStream("", "x_y z"), new String[] { "x", "y", "z" });
+    }
+
+    public void testAnalyzer() throws IOException {
+        IndexAnalyzers analyzers = getIndexAnalyzers(
+            Settings.builder()
+                .put("index.analysis.analyzer.analyzer_provider_test.type", "stableAnalyzerFactory")
+                .putList("index.analysis.analyzer.analyzer_provider_test.tokenizer_list_of_chars", "_", " ")
+                .put("index.analysis.analyzer.analyzer_provider_test.token_filter_number", 1L)
+                .put("index.analysis.analyzer.analyzer_provider_test.old_char", "#")
+                .put("index.analysis.analyzer.analyzer_provider_test.new_number", 3)
+                .put("index.analysis.analyzer.analyzer_provider_test.analyzerUseTokenListOfChars", true)
+                .put(IndexMetadata.SETTING_VERSION_CREATED, VersionUtils.randomVersion(random()))
+                .build()
+        );
+        assertTokenStreamContents(analyzers.get("analyzer_provider_test").tokenStream("", "1x_y_#z"), new String[] { "y", "3z" });
+    }
+
+    protected IndexAnalyzers getIndexAnalyzers(Settings settings) throws IOException {
+        AnalysisRegistry registry = setupRegistry();
+
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
+        return registry.build(idxSettings);
+    }
+
+    private AnalysisRegistry setupRegistry() throws IOException {
+        ClassLoader classLoader = getClass().getClassLoader();
+
+        AnalysisRegistry registry = new AnalysisModule(
+            TestEnvironment.newEnvironment(emptyNodeSettings),
+            emptyList(),
+            new StablePluginsRegistry(
+                new NamedComponentReader(),
+                Map.of(
+                    CharFilterFactory.class.getCanonicalName(),
+                    new NameToPluginInfo(
+                        Map.of(
+                            "stableCharFilterFactory",
+                            new PluginInfo("stableCharFilterFactory", TestCharFilterFactory.class.getName(), classLoader)
+                        )
+                    ),
+                    TokenFilterFactory.class.getCanonicalName(),
+                    new NameToPluginInfo(
+                        Map.of(
+                            "stableTokenFilterFactory",
+                            new PluginInfo("stableTokenFilterFactory", TestTokenFilterFactory.class.getName(), classLoader)
+                        )
+                    ),
+                    TokenizerFactory.class.getCanonicalName(),
+                    new NameToPluginInfo(
+                        Map.of(
+                            "stableTokenizerFactory",
+                            new PluginInfo("stableTokenizerFactory", TestTokenizerFactory.class.getName(), classLoader)
+                        )
+                    ),
+                    AnalyzerFactory.class.getCanonicalName(),
+                    new NameToPluginInfo(
+                        Map.of(
+                            "stableAnalyzerFactory",
+                            new PluginInfo("stableAnalyzerFactory", TestAnalyzerFactory.class.getName(), classLoader)
+                        )
+                    )
+                )
+            )
+        ).getAnalysisRegistry();
+        return registry;
+    }
+
+    @AnalysisSettings
+    public interface TestAnalysisSettings {
+        @StringSetting(path = "old_char", defaultValue = " ")
+        String oldChar();
+
+        @IntSetting(path = "new_number", defaultValue = 0)
+        int newNumber();
+
+        @LongSetting(path = "token_filter_number", defaultValue = 0L)
+        long tokenFilterNumber();
+
+        @ListSetting(path = "tokenizer_list_of_chars")
+        java.util.List<String> tokenizerListOfChars();
+
+        @BooleanSetting(path = "analyzerUseTokenListOfChars", defaultValue = false)
+        boolean analyzerUseTokenListOfChars();
+    }
+
+    @NamedComponent("stableAnalyzerFactory")
+    public static class TestAnalyzerFactory implements AnalyzerFactory {
+
+        private final TestAnalysisSettings settings;
+
+        @Inject
+        public TestAnalyzerFactory(TestAnalysisSettings settings) {
+            this.settings = settings;
+        }
+
+        @Override
+        public Analyzer create() {
+            return new CustomAnalyzer(settings);
+        }
+
+        static class CustomAnalyzer extends Analyzer {
+
+            private final TestAnalysisSettings settings;
+
+            CustomAnalyzer(TestAnalysisSettings settings) {
+                this.settings = settings;
+            }
+
+            @Override
+            protected TokenStreamComponents createComponents(String fieldName) {
+                var tokenizer = new CharSkippingTokenizer(settings.tokenizerListOfChars());
+                long tokenFilterNumber = settings.analyzerUseTokenListOfChars() ? settings.tokenFilterNumber() : -1;
+                var tokenFilter = new SkipStartingWithDigitTokenFilter(tokenizer, tokenFilterNumber);
+                return new TokenStreamComponents(
+                    r -> tokenizer.setReader(new ReplaceCharToNumber(r, settings.oldChar(), settings.newNumber())),
+                    tokenFilter
+                );
+            }
+        }
+    }
+
+    @NamedComponent("stableCharFilterFactory")
+    public static class TestCharFilterFactory implements CharFilterFactory {
+        private final String oldChar;
+        private final int newNumber;
+
+        @Inject
+        public TestCharFilterFactory(TestAnalysisSettings settings) {
+            oldChar = settings.oldChar();
+            newNumber = settings.newNumber();
+        }
+
+        @Override
+        public Reader create(Reader reader) {
+            return new ReplaceCharToNumber(reader, oldChar, newNumber);
+        }
+
+        @Override
+        public Reader normalize(Reader reader) {
+            return new ReplaceCharToNumber(reader, oldChar, newNumber);
+        }
+
+    }
+
+    @NamedComponent("stableTokenFilterFactory")
+    public static class TestTokenFilterFactory implements TokenFilterFactory {
+
+        private final long tokenFilterNumber;
+
+        @Inject
+        public TestTokenFilterFactory(TestAnalysisSettings settings) {
+            this.tokenFilterNumber = settings.tokenFilterNumber();
+        }
+
+        @Override
+        public TokenStream create(TokenStream tokenStream) {
+            return new SkipStartingWithDigitTokenFilter(tokenStream, tokenFilterNumber);
+        }
+
+        @Override
+        public TokenStream normalize(TokenStream tokenStream) {
+            return new AppendTokenFilter(tokenStream, String.valueOf(tokenFilterNumber));
+        }
+
+        @Override
+        public AnalysisMode getAnalysisMode() {
+            return TokenFilterFactory.super.getAnalysisMode();
+        }
+
+    }
+
+    @NamedComponent("stableTokenizerFactory")
+    public static class TestTokenizerFactory implements TokenizerFactory {
+        private final java.util.List<String> tokenizerListOfChars;
+
+        @Inject
+        public TestTokenizerFactory(TestAnalysisSettings settings) {
+            this.tokenizerListOfChars = settings.tokenizerListOfChars();
+        }
+
+        @Override
+        public Tokenizer create() {
+            return new CharSkippingTokenizer(tokenizerListOfChars);
+        }
+
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/lucene/AppendCharFilter.java b/server/src/test/java/org/elasticsearch/indices/analysis/lucene/AppendCharFilter.java
new file mode 100644
index 000000000000..ce04642a88bf
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/indices/analysis/lucene/AppendCharFilter.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.indices.analysis.lucene;
+
+import org.apache.lucene.analysis.CharFilter;
+import org.elasticsearch.common.io.Streams;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.io.UncheckedIOException;
+
+// Simple char filter that appends text to the term
+public class AppendCharFilter extends CharFilter {
+
+    static Reader append(Reader input, String appendMe) {
+        try {
+            return new StringReader(Streams.copyToString(input) + appendMe);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public AppendCharFilter(Reader input, String appendMe) {
+        super(append(input, appendMe));
+    }
+
+    @Override
+    protected int correct(int currentOff) {
+        return currentOff;
+    }
+
+    @Override
+    public int read(char[] cbuf, int off, int len) throws IOException {
+        return input.read(cbuf, off, len);
+    }
+}
diff --git a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/AppendTokenFilter.java b/server/src/test/java/org/elasticsearch/indices/analysis/lucene/AppendTokenFilter.java
similarity index 87%
rename from plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/AppendTokenFilter.java
rename to server/src/test/java/org/elasticsearch/indices/analysis/lucene/AppendTokenFilter.java
index b9235e49e2eb..0a45604deebf 100644
--- a/plugins/examples/stable-analysis/src/main/java/org/elasticsearch/example/analysis/lucene/AppendTokenFilter.java
+++ b/server/src/test/java/org/elasticsearch/indices/analysis/lucene/AppendTokenFilter.java
@@ -6,16 +6,17 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.example.analysis.lucene;
+package org.elasticsearch.indices.analysis.lucene;
 
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.elasticsearch.plugin.analysis.api.TokenFilterFactory;
+import org.elasticsearch.index.analysis.TokenFilterFactory;
 
 import java.io.IOException;
 
-public class AppendTokenFilter extends TokenFilter {
+// Simple token filter that appends text to the term
+public final class AppendTokenFilter extends TokenFilter {
     public static TokenFilterFactory factoryForSuffix(String suffix) {
         return new TokenFilterFactory() {
             @Override
diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/lucene/CharSkippingTokenizer.java b/server/src/test/java/org/elasticsearch/indices/analysis/lucene/CharSkippingTokenizer.java
new file mode 100644
index 000000000000..c32cfe696bf0
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/indices/analysis/lucene/CharSkippingTokenizer.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.indices.analysis.lucene;
+
+import org.apache.lucene.analysis.util.CharTokenizer;
+
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class CharSkippingTokenizer extends CharTokenizer {
+
+    private final Set<Integer> setOfChars;
+
+    public CharSkippingTokenizer(List<String> tokenizerListOfChars) {
+        this.setOfChars = tokenizerListOfChars.stream().map(s -> (int) s.charAt(0)).collect(Collectors.toSet());
+    }
+
+    @Override
+    protected boolean isTokenChar(int c) {
+        return setOfChars.contains(c) == false;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/lucene/ReplaceCharToNumber.java b/server/src/test/java/org/elasticsearch/indices/analysis/lucene/ReplaceCharToNumber.java
new file mode 100644
index 000000000000..19d4e6f28ed0
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/indices/analysis/lucene/ReplaceCharToNumber.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.indices.analysis.lucene;
+
+import org.apache.lucene.analysis.charfilter.MappingCharFilter;
+import org.apache.lucene.analysis.charfilter.NormalizeCharMap;
+
+import java.io.Reader;
+
+public class ReplaceCharToNumber extends MappingCharFilter {
+
+    public ReplaceCharToNumber(Reader in, String oldChar, int newNumber) {
+        super(charMap(oldChar, newNumber), in);
+    }
+
+    private static NormalizeCharMap charMap(String oldChar, int newNumber) {
+        NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
+        builder.add(oldChar, String.valueOf(newNumber));
+        return builder.build();
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/lucene/SkipStartingWithDigitTokenFilter.java b/server/src/test/java/org/elasticsearch/indices/analysis/lucene/SkipStartingWithDigitTokenFilter.java
new file mode 100644
index 000000000000..3b53366f8f04
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/indices/analysis/lucene/SkipStartingWithDigitTokenFilter.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.indices.analysis.lucene;
+
+import org.apache.lucene.analysis.FilteringTokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+
+import java.io.IOException;
+
+public class SkipStartingWithDigitTokenFilter extends FilteringTokenFilter {
+
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private final long asciiDigitsToSkip;
+
+    public SkipStartingWithDigitTokenFilter(TokenStream in, long asciiDigitsToSkip) {
+        super(in);
+        this.asciiDigitsToSkip = asciiDigitsToSkip;
+    }
+
+    @Override
+    protected boolean accept() throws IOException {
+        return termAtt.buffer()[0] != (char) (asciiDigitsToSkip + '0');
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/wrappers/StableApiWrappersTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/wrappers/StableApiWrappersTests.java
index 508d0c77f6f8..3ddf8feb543f 100644
--- a/server/src/test/java/org/elasticsearch/indices/analysis/wrappers/StableApiWrappersTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/analysis/wrappers/StableApiWrappersTests.java
@@ -14,8 +14,12 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.indices.analysis.AnalysisModule;
-import org.elasticsearch.plugin.analysis.api.AnalysisMode;
-import org.elasticsearch.plugin.api.NamedComponent;
+import org.elasticsearch.plugin.NamedComponent;
+import org.elasticsearch.plugin.analysis.AnalysisMode;
+import org.elasticsearch.plugin.analysis.AnalyzerFactory;
+import org.elasticsearch.plugin.analysis.CharFilterFactory;
+import org.elasticsearch.plugin.analysis.TokenFilterFactory;
+import org.elasticsearch.plugin.analysis.TokenizerFactory;
 import org.elasticsearch.plugins.scanners.PluginInfo;
 import org.elasticsearch.plugins.scanners.StablePluginsRegistry;
 import org.elasticsearch.test.ESTestCase;
@@ -36,9 +40,8 @@ public class StableApiWrappersTests extends ESTestCase {
 
     public void testUnknownClass() throws IOException {
         StablePluginsRegistry registry = Mockito.mock(StablePluginsRegistry.class);
-        Mockito.when(
-            registry.getPluginInfosForExtensible(eq(org.elasticsearch.plugin.analysis.api.AnalyzerFactory.class.getCanonicalName()))
-        ).thenReturn(List.of(new PluginInfo("namedComponentName1", "someRandomName", getClass().getClassLoader())));
+        Mockito.when(registry.getPluginInfosForExtensible(eq(AnalyzerFactory.class.getCanonicalName())))
+            .thenReturn(List.of(new PluginInfo("namedComponentName1", "someRandomName", getClass().getClassLoader())));
 
         Map<String, AnalysisModule.AnalysisProvider<org.elasticsearch.index.analysis.AnalyzerProvider<?>>> analysisProviderMap =
             StableApiWrappers.oldApiForAnalyzerFactory(registry);
@@ -56,9 +59,7 @@ public void testUnknownClass() throws IOException {
 
     public void testStablePluginHasNoArgConstructor() throws IOException {
         StablePluginsRegistry registry = Mockito.mock(StablePluginsRegistry.class);
-        Mockito.when(
-            registry.getPluginInfosForExtensible(eq(org.elasticsearch.plugin.analysis.api.AnalyzerFactory.class.getCanonicalName()))
-        )
+        Mockito.when(registry.getPluginInfosForExtensible(eq(AnalyzerFactory.class.getCanonicalName())))
             .thenReturn(
                 List.of(new PluginInfo("namedComponentName1", DefaultConstrAnalyzerFactory.class.getName(), getClass().getClassLoader()))
             );
@@ -74,14 +75,13 @@ public void testStablePluginHasNoArgConstructor() throws IOException {
             IllegalStateException.class,
             () -> oldTokenFilter.get(null, mock(Environment.class), null, null)
         );
-        assertThat(illegalStateException.getCause(), instanceOf(NoSuchMethodException.class));
+        assertThat(illegalStateException.getMessage(), equalTo("Missing @Inject annotation for constructor with settings."));
     }
 
     public void testAnalyzerFactoryDelegation() throws IOException {
         StablePluginsRegistry registry = Mockito.mock(StablePluginsRegistry.class);
-        Mockito.when(
-            registry.getPluginInfosForExtensible(eq(org.elasticsearch.plugin.analysis.api.AnalyzerFactory.class.getCanonicalName()))
-        ).thenReturn(List.of(new PluginInfo("namedComponentName1", TestAnalyzerFactory.class.getName(), getClass().getClassLoader())));
+        Mockito.when(registry.getPluginInfosForExtensible(eq(AnalyzerFactory.class.getCanonicalName())))
+            .thenReturn(List.of(new PluginInfo("namedComponentName1", TestAnalyzerFactory.class.getName(), getClass().getClassLoader())));
 
         Map<String, AnalysisModule.AnalysisProvider<org.elasticsearch.index.analysis.AnalyzerProvider<?>>> analysisProviderMap =
             StableApiWrappers.oldApiForAnalyzerFactory(registry);
@@ -107,9 +107,8 @@ public void testAnalyzerFactoryDelegation() throws IOException {
 
     public void testTokenizerFactoryDelegation() throws IOException {
         StablePluginsRegistry registry = Mockito.mock(StablePluginsRegistry.class);
-        Mockito.when(
-            registry.getPluginInfosForExtensible(eq(org.elasticsearch.plugin.analysis.api.TokenizerFactory.class.getCanonicalName()))
-        ).thenReturn(List.of(new PluginInfo("namedComponentName1", TestTokenizerFactory.class.getName(), getClass().getClassLoader())));
+        Mockito.when(registry.getPluginInfosForExtensible(eq(TokenizerFactory.class.getCanonicalName())))
+            .thenReturn(List.of(new PluginInfo("namedComponentName1", TestTokenizerFactory.class.getName(), getClass().getClassLoader())));
 
         Map<String, AnalysisModule.AnalysisProvider<org.elasticsearch.index.analysis.TokenizerFactory>> analysisProviderMap =
             StableApiWrappers.oldApiForTokenizerFactory(registry);
@@ -130,9 +129,10 @@ public void testTokenizerFactoryDelegation() throws IOException {
 
     public void testTokenFilterFactoryDelegation() throws IOException {
         StablePluginsRegistry registry = Mockito.mock(StablePluginsRegistry.class);
-        Mockito.when(
-            registry.getPluginInfosForExtensible(eq(org.elasticsearch.plugin.analysis.api.TokenFilterFactory.class.getCanonicalName()))
-        ).thenReturn(List.of(new PluginInfo("namedComponentName1", TestTokenFilterFactory.class.getName(), getClass().getClassLoader())));
+        Mockito.when(registry.getPluginInfosForExtensible(eq(TokenFilterFactory.class.getCanonicalName())))
+            .thenReturn(
+                List.of(new PluginInfo("namedComponentName1", TestTokenFilterFactory.class.getName(), getClass().getClassLoader()))
+            );
 
         Map<String, AnalysisModule.AnalysisProvider<org.elasticsearch.index.analysis.TokenFilterFactory>> analysisProviderMap =
             StableApiWrappers.oldApiForTokenFilterFactory(registry);
@@ -168,9 +168,8 @@ public void testTokenFilterFactoryDelegation() throws IOException {
 
     public void testCharFilterFactoryDelegation() throws IOException {
         StablePluginsRegistry registry = Mockito.mock(StablePluginsRegistry.class);
-        Mockito.when(
-            registry.getPluginInfosForExtensible(eq(org.elasticsearch.plugin.analysis.api.CharFilterFactory.class.getCanonicalName()))
-        ).thenReturn(List.of(new PluginInfo("namedComponentName1", TestCharFilterFactory.class.getName(), getClass().getClassLoader())));
+        Mockito.when(registry.getPluginInfosForExtensible(eq(CharFilterFactory.class.getCanonicalName())))
+            .thenReturn(List.of(new PluginInfo("namedComponentName1", TestCharFilterFactory.class.getName(), getClass().getClassLoader())));
 
         Map<String, AnalysisModule.AnalysisProvider<org.elasticsearch.index.analysis.CharFilterFactory>> analysisProviderMap =
             StableApiWrappers.oldApiForStableCharFilterFactory(registry);
@@ -198,7 +197,7 @@ public void testCharFilterFactoryDelegation() throws IOException {
     }
 
     @NamedComponent("DefaultConstrAnalyzerFactory")
-    public static class DefaultConstrAnalyzerFactory implements org.elasticsearch.plugin.analysis.api.AnalyzerFactory {
+    public static class DefaultConstrAnalyzerFactory implements AnalyzerFactory {
 
         public DefaultConstrAnalyzerFactory(int x) {}
 
@@ -210,7 +209,7 @@ public Analyzer create() {
     }
 
     @NamedComponent("TestAnalyzerFactory")
-    public static class TestAnalyzerFactory implements org.elasticsearch.plugin.analysis.api.AnalyzerFactory {
+    public static class TestAnalyzerFactory implements AnalyzerFactory {
 
         @Override
         public Analyzer create() {
@@ -220,7 +219,7 @@ public Analyzer create() {
     }
 
     @NamedComponent("TestTokenizerFactory")
-    public static class TestTokenizerFactory implements org.elasticsearch.plugin.analysis.api.TokenizerFactory {
+    public static class TestTokenizerFactory implements TokenizerFactory {
 
         @Override
         public Tokenizer create() {
@@ -229,7 +228,7 @@ public Tokenizer create() {
     }
 
     @NamedComponent("TestTokenFilterFactory")
-    public static class TestTokenFilterFactory implements org.elasticsearch.plugin.analysis.api.TokenFilterFactory {
+    public static class TestTokenFilterFactory implements TokenFilterFactory {
 
         @Override
         public TokenStream create(TokenStream tokenStream) {
@@ -254,7 +253,7 @@ public AnalysisMode getAnalysisMode() {
     }
 
     @NamedComponent("TestCharFilterFactory")
-    public static class TestCharFilterFactory implements org.elasticsearch.plugin.analysis.api.CharFilterFactory {
+    public static class TestCharFilterFactory implements CharFilterFactory {
 
         @Override
         public Reader create(Reader reader) {
diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
index 2954cdcbbdcd..8ccd7d75e944 100644
--- a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
+++ b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
@@ -40,14 +40,16 @@
 import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.ClusterStateTaskListener;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction.FailedShardUpdateTask;
 import org.elasticsearch.cluster.action.shard.ShardStateAction.StartedShardEntry;
 import org.elasticsearch.cluster.action.shard.ShardStateAction.StartedShardUpdateTask;
 import org.elasticsearch.cluster.block.ClusterBlock;
+import org.elasticsearch.cluster.coordination.JoinReason;
 import org.elasticsearch.cluster.coordination.JoinTask;
-import org.elasticsearch.cluster.coordination.JoinTaskExecutor;
-import org.elasticsearch.cluster.coordination.NodeRemovalClusterStateTaskExecutor;
+import org.elasticsearch.cluster.coordination.NodeJoinExecutor;
+import org.elasticsearch.cluster.coordination.NodeLeftExecutor;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexMetadataVerifier;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -111,6 +113,7 @@
 
 import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom;
 import static java.util.stream.Collectors.toMap;
+import static org.elasticsearch.common.settings.ClusterSettings.createBuiltInClusterSettings;
 import static org.elasticsearch.env.Environment.PATH_HOME_SETTING;
 import static org.elasticsearch.test.CheckedFunctionUtils.anyCheckedFunction;
 import static org.elasticsearch.test.ESTestCase.between;
@@ -136,16 +139,16 @@ public class ClusterStateChanges {
     private final TransportClusterRerouteAction transportClusterRerouteAction;
     private final TransportCreateIndexAction transportCreateIndexAction;
 
-    private final NodeRemovalClusterStateTaskExecutor nodeRemovalExecutor;
+    private final NodeLeftExecutor nodeLeftExecutor;
 
     @SuppressWarnings("unchecked")
     public ClusterStateChanges(NamedXContentRegistry xContentRegistry, ThreadPool threadPool) {
-        ClusterSettings clusterSettings = new ClusterSettings(SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        ClusterSettings clusterSettings = createBuiltInClusterSettings(SETTINGS);
         allocationService = new AllocationService(
             new AllocationDeciders(
                 new HashSet<>(
                     Arrays.asList(
-                        new SameShardAllocationDecider(SETTINGS, clusterSettings),
+                        new SameShardAllocationDecider(clusterSettings),
                         new ReplicaAfterPrimaryActiveAllocationDecider(),
                         new RandomAllocationDeciderTests.RandomAllocationDecider(getRandom())
                     )
@@ -154,7 +157,8 @@ public ClusterStateChanges(NamedXContentRegistry xContentRegistry, ThreadPool th
             new TestGatewayAllocator(),
             new BalancedShardsAllocator(SETTINGS),
             EmptyClusterInfoService.INSTANCE,
-            EmptySnapshotsInfoService.INSTANCE
+            EmptySnapshotsInfoService.INSTANCE,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
         shardFailedClusterStateTaskExecutor = new ShardStateAction.ShardFailedClusterStateTaskExecutor(allocationService, null);
         shardStartedClusterStateTaskExecutor = new ShardStateAction.ShardStartedClusterStateTaskExecutor(allocationService, null);
@@ -347,7 +351,7 @@ public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, Version mi
             EmptySystemIndices.INSTANCE
         );
 
-        nodeRemovalExecutor = new NodeRemovalClusterStateTaskExecutor(allocationService);
+        nodeLeftExecutor = new NodeLeftExecutor(allocationService);
     }
 
     private void resetMasterService() {
@@ -396,14 +400,16 @@ public ClusterState reroute(ClusterState state, ClusterRerouteRequest request) {
         return execute(transportClusterRerouteAction, request, state);
     }
 
+    private static final JoinReason DUMMY_REASON = new JoinReason("dummy reason", null);
+
     public ClusterState addNode(ClusterState clusterState, DiscoveryNode discoveryNode) {
         return runTasks(
-            new JoinTaskExecutor(allocationService, (s, p, r) -> {}),
+            new NodeJoinExecutor(allocationService, (s, p, r) -> {}),
             clusterState,
             List.of(
                 JoinTask.singleNode(
                     discoveryNode,
-                    "dummy reason",
+                    DUMMY_REASON,
                     ActionListener.wrap(() -> { throw new AssertionError("should not complete publication"); }),
                     clusterState.term()
                 )
@@ -413,7 +419,7 @@ public ClusterState addNode(ClusterState clusterState, DiscoveryNode discoveryNo
 
     public ClusterState joinNodesAndBecomeMaster(ClusterState clusterState, List<DiscoveryNode> nodes) {
         return runTasks(
-            new JoinTaskExecutor(allocationService, (s, p, r) -> {}),
+            new NodeJoinExecutor(allocationService, (s, p, r) -> {}),
             clusterState,
             List.of(
                 JoinTask.completingElection(
@@ -421,7 +427,7 @@ public ClusterState joinNodesAndBecomeMaster(ClusterState clusterState, List<Dis
                         .map(
                             node -> new JoinTask.NodeJoinTask(
                                 node,
-                                "dummy reason",
+                                DUMMY_REASON,
                                 ActionListener.wrap(() -> { throw new AssertionError("should not complete publication"); })
                             )
                         ),
@@ -433,9 +439,9 @@ public ClusterState joinNodesAndBecomeMaster(ClusterState clusterState, List<Dis
 
     public ClusterState removeNodes(ClusterState clusterState, List<DiscoveryNode> nodes) {
         return runTasks(
-            nodeRemovalExecutor,
+            nodeLeftExecutor,
             clusterState,
-            nodes.stream().map(n -> new NodeRemovalClusterStateTaskExecutor.Task(n, "dummy reason", () -> {})).toList()
+            nodes.stream().map(n -> new NodeLeftExecutor.Task(n, "dummy reason", () -> {})).toList()
         );
     }
 
diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
index 66b92a5aa3ef..e255fc86c65d 100644
--- a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
@@ -157,7 +157,7 @@ public void testJoiningNewClusterOnlyRemovesInMemoryIndexStructures() {
         // a cluster state derived from the initial state that includes a created index
         String name = "index_" + randomAlphaOfLength(8).toLowerCase(Locale.ROOT);
         ShardRoutingState[] replicaStates = new ShardRoutingState[randomIntBetween(0, 3)];
-        Arrays.fill(replicaStates, ShardRoutingState.INITIALIZING);
+        Arrays.fill(replicaStates, ShardRoutingState.UNASSIGNED);
         ClusterState stateWithIndex = ClusterStateCreationUtils.state(name, randomBoolean(), ShardRoutingState.INITIALIZING, replicaStates);
 
         // the initial state which is derived from the newly created cluster state but doesn't contain the index
@@ -498,7 +498,7 @@ public ClusterState randomlyUpdateClusterState(
     protected DiscoveryNode createNode(DiscoveryNodeRole... mustHaveRoles) {
         Set<DiscoveryNodeRole> roles = new HashSet<>(randomSubsetOf(DiscoveryNodeRole.roles()));
         Collections.addAll(roles, mustHaveRoles);
-        final String id = formatted("node_%03d", nodeIdGenerator.incrementAndGet());
+        final String id = format("node_%03d", nodeIdGenerator.incrementAndGet());
         return new DiscoveryNode(id, id, buildNewFakeTransportAddress(), Collections.emptyMap(), roles, Version.CURRENT);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/MultiFileWriterTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/MultiFileWriterTests.java
new file mode 100644
index 000000000000..3d59bda185ff
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/indices/recovery/MultiFileWriterTests.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.indices.recovery;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.store.ByteBuffersDataOutput;
+import org.apache.lucene.store.ByteBuffersIndexOutput;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.tests.util.TestUtil;
+import org.elasticsearch.Version;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.IndexShardTestCase;
+import org.elasticsearch.index.store.Store;
+import org.elasticsearch.index.store.StoreFileMetadata;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.Collections;
+
+import static org.mockito.ArgumentMatchers.anyCollection;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+
+public class MultiFileWriterTests extends IndexShardTestCase {
+
+    private IndexShard indexShard;
+    private Directory directory;
+    private Directory directorySpy;
+    private Store store;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        indexShard = newShard(true);
+        directory = newMockFSDirectory(indexShard.shardPath().resolveIndex());
+        directorySpy = spy(directory);
+        store = createStore(indexShard.shardId(), indexShard.indexSettings(), directorySpy);
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        directory.close();
+        closeShards(indexShard);
+    }
+
+    public void testWritesFile() throws IOException {
+        var fileWriter = createMultiFileWriter(true);
+        var file = createFile("file");
+
+        fileWriter.writeFile(file.metadata, 10, new ByteArrayInputStream(file.bytes));
+
+        verify(directorySpy).createOutput("temp_file", IOContext.DEFAULT);
+        verify(directorySpy).sync(Collections.singleton("temp_file"));
+    }
+
+    public void testFailsToWriteFileWithIncorrectChecksum() throws IOException {
+        var fileWriter = createMultiFileWriter(true);
+        var file = createFile("file");
+
+        expectThrows(
+            CorruptIndexException.class,
+            () -> fileWriter.writeFile(withWrongChecksum(file.metadata), 10, new ByteArrayInputStream(file.bytes))
+        );
+
+        verify(directorySpy).createOutput("temp_file", IOContext.DEFAULT);
+        verify(directorySpy).deleteFile("temp_file");
+        verify(directorySpy, never()).sync(anyCollection());
+    }
+
+    public void testWritesFileWithIncorrectChecksumWithoutVerification() throws IOException {
+        var fileWriter = createMultiFileWriter(false);
+        var file = createFile("file");
+
+        fileWriter.writeFile(withWrongChecksum(file.metadata), 10, new ByteArrayInputStream(file.bytes));
+
+        verify(directorySpy).createOutput("temp_file", IOContext.DEFAULT);
+        verify(directorySpy).sync(Collections.singleton("temp_file"));
+    }
+
+    private MultiFileWriter createMultiFileWriter(boolean verifyOutput) {
+        return new MultiFileWriter(store, mock(RecoveryState.Index.class), "temp_", logger, mock(Runnable.class), verifyOutput);
+    }
+
+    private record FileAndMetadata(byte[] bytes, StoreFileMetadata metadata) {}
+
+    private static FileAndMetadata createFile(String name) throws IOException {
+        var buffer = new ByteBuffersDataOutput();
+        var output = new ByteBuffersIndexOutput(buffer, "test", name);
+        output.writeString(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
+        CodecUtil.writeBEInt(output, CodecUtil.FOOTER_MAGIC);
+        CodecUtil.writeBEInt(output, 0);
+        String checksum = Store.digestToString(output.getChecksum());
+        CodecUtil.writeBELong(output, output.getChecksum());
+        output.close();
+
+        return new FileAndMetadata(buffer.toArrayCopy(), new StoreFileMetadata(name, buffer.size(), checksum, Version.CURRENT.toString()));
+    }
+
+    private static StoreFileMetadata withWrongChecksum(StoreFileMetadata metadata) {
+        var newChecksum = randomValueOtherThan(metadata.checksum(), () -> randomAlphaOfLength(6));
+        return new StoreFileMetadata(metadata.name(), metadata.length(), newChecksum, metadata.writtenBy());
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
index ad9ddd189917..99672372aeb4 100644
--- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
@@ -45,6 +45,7 @@
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.VersionType;
@@ -506,7 +507,7 @@ private TimeSeriesModeIndexOpFactory() throws IOException {
 
         @Override
         public Engine.Index createIndexOp(int docIdent) {
-            SourceToParse source = new SourceToParse(null, new BytesArray(formatted("""
+            SourceToParse source = new SourceToParse(null, new BytesArray(Strings.format("""
                 {
                     "@timestamp": %s,
                     "dim": "dim"
diff --git a/server/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java b/server/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
index 26c139040c09..8aa0141598ad 100644
--- a/server/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
@@ -42,16 +42,12 @@ public void createLocalNode() {
         localNode = new DiscoveryNode("abc", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
     }
 
-    public void testShardCanBeDeletedNoShardRouting() {
-        IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", "_na_", 1));
-        assertFalse(IndicesStore.shardCanBeDeleted(localNode.getId(), routingTable.build()));
-    }
-
     public void testShardCanBeDeletedNoShardStarted() {
         final var numShardCopies = randomInt(3);
         final var shardId = new ShardId("test", "_na_", 0);
         final var routingTable = new IndexShardRoutingTable.Builder(shardId);
         final var unStartedShard = randomInt(numShardCopies);
+        boolean activePrimary = false;
         for (int j = 0; j <= numShardCopies; j++) {
             ShardRoutingState state;
             if (j == unStartedShard) {
@@ -59,6 +55,11 @@ public void testShardCanBeDeletedNoShardStarted() {
             } else {
                 state = randomFrom(ShardRoutingState.values());
             }
+            if (j == 0) {
+                activePrimary = state == ShardRoutingState.STARTED || state == ShardRoutingState.RELOCATING;
+            } else if (activePrimary == false) {
+                state = ShardRoutingState.UNASSIGNED;
+            }
             UnassignedInfo unassignedInfo = null;
             if (state == ShardRoutingState.UNASSIGNED) {
                 unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null);
diff --git a/server/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java
index 8c6c44bd17f5..c44ad7fe9f48 100644
--- a/server/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java
+++ b/server/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java
@@ -19,7 +19,7 @@
 import java.util.function.Consumer;
 import java.util.function.LongSupplier;
 
-import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
@@ -60,7 +60,7 @@ public boolean isAsync() {
                 return isAsync;
             }
         };
-        CompoundProcessor compoundProcessor = new CompoundProcessor(relativeTimeProvider, false, processor);
+        CompoundProcessor compoundProcessor = new CompoundProcessor(false, List.of(processor), List.of(), relativeTimeProvider);
         ingestDocument.setFieldValue("compoundProcessor", compoundProcessor); // ugly hack to assert current count = 1
         assertThat(compoundProcessor.getProcessors().size(), equalTo(1));
         assertThat(compoundProcessor.getProcessors().get(0), sameInstance(processor));
@@ -76,7 +76,7 @@ public void testSingleProcessorWithException() throws Exception {
         TestProcessor processor = new TestProcessor(new RuntimeException("error"));
         LongSupplier relativeTimeProvider = mock(LongSupplier.class);
         when(relativeTimeProvider.getAsLong()).thenReturn(0L);
-        CompoundProcessor compoundProcessor = new CompoundProcessor(relativeTimeProvider, false, processor);
+        CompoundProcessor compoundProcessor = new CompoundProcessor(false, List.of(processor), List.of(), relativeTimeProvider);
         assertThat(compoundProcessor.getProcessors().size(), equalTo(1));
         assertThat(compoundProcessor.getProcessors().get(0), sameInstance(processor));
         assertThat(compoundProcessor.getOnFailureProcessors().isEmpty(), is(true));
@@ -249,7 +249,7 @@ public void testCompoundProcessorExceptionFailWithoutOnFailure() throws Exceptio
         LongSupplier relativeTimeProvider = mock(LongSupplier.class);
         when(relativeTimeProvider.getAsLong()).thenReturn(0L);
 
-        CompoundProcessor failCompoundProcessor = new CompoundProcessor(relativeTimeProvider, false, firstProcessor);
+        CompoundProcessor failCompoundProcessor = new CompoundProcessor(false, List.of(firstProcessor), List.of(), relativeTimeProvider);
 
         CompoundProcessor compoundProcessor = new CompoundProcessor(
             false,
@@ -313,7 +313,7 @@ public void testCompoundProcessorExceptionFailInOnFailure() throws Exception {
         CompoundProcessor failCompoundProcessor = new CompoundProcessor(
             false,
             List.of(firstProcessor),
-            List.of(new CompoundProcessor(relativeTimeProvider, false, failProcessor))
+            List.of(new CompoundProcessor(false, List.of(failProcessor), List.of(), relativeTimeProvider))
         );
 
         CompoundProcessor compoundProcessor = new CompoundProcessor(
@@ -480,7 +480,7 @@ public void testMultipleProcessors() {
         for (int i = 0; i < processorsCount; i++) {
             processors[i] = getTestProcessor(Integer.toString(i), randomBoolean(), randomBoolean());
         }
-        CompoundProcessor compoundProcessor = new CompoundProcessor(relativeTimeProvider, true, processors);
+        CompoundProcessor compoundProcessor = new CompoundProcessor(true, List.of(processors), List.of(), relativeTimeProvider);
         executeCompound(
             compoundProcessor,
             ingestDocument,
@@ -508,7 +508,7 @@ public void testMultipleProcessorsDoNotIgnoreFailures() {
         for (int i = goodProcessorsCount + 1; i < totalProcessorsCount; i++) {
             processors[i] = getTestProcessor(Integer.toString(i), randomBoolean(), false);
         }
-        CompoundProcessor compoundProcessor = new CompoundProcessor(relativeTimeProvider, false, processors);
+        CompoundProcessor compoundProcessor = new CompoundProcessor(false, List.of(processors), List.of(), relativeTimeProvider);
         executeCompound(compoundProcessor, ingestDocument, (result, e) -> {});
         for (int i = 0; i < goodProcessorsCount + 1; i++) {
             assertThat(
diff --git a/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java
index e505dfc2ce64..cab001478a54 100644
--- a/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java
+++ b/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java
@@ -22,7 +22,6 @@
 
 import java.text.ParseException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -37,8 +36,8 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
-import static org.hamcrest.core.Is.is;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -52,12 +51,12 @@ public void testChecksCondition() throws Exception {
         String trueValue = "truthy";
         ScriptService scriptService = new ScriptService(
             Settings.builder().build(),
-            Collections.singletonMap(
+            Map.of(
                 Script.DEFAULT_SCRIPT_LANG,
                 new MockScriptEngine(
                     Script.DEFAULT_SCRIPT_LANG,
-                    Collections.singletonMap(scriptName, ctx -> trueValue.equals(ctx.get(conditionalField))),
-                    Collections.emptyMap()
+                    Map.of(scriptName, ctx -> trueValue.equals(ctx.get(conditionalField))),
+                    Map.of()
                 )
             ),
             new HashMap<>(ScriptModule.CORE_CONTEXTS),
@@ -69,7 +68,7 @@ public void testChecksCondition() throws Exception {
         ConditionalProcessor processor = new ConditionalProcessor(
             randomAlphaOfLength(10),
             "description",
-            new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()),
+            new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Map.of()),
             scriptService,
             new Processor() {
                 @Override
@@ -150,13 +149,10 @@ public void testTypeDeprecation() throws Exception {
 
         ScriptService scriptService = new ScriptService(
             Settings.builder().build(),
-            Collections.singletonMap(
-                Script.DEFAULT_SCRIPT_LANG,
-                new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> {
-                    ctx.get("_type");
-                    return true;
-                }), Collections.emptyMap())
-            ),
+            Map.of(Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Map.of(scriptName, ctx -> {
+                ctx.get("_type");
+                return true;
+            }), Map.of())),
             new HashMap<>(ScriptModule.CORE_CONTEXTS),
             () -> 1L
         );
@@ -166,7 +162,7 @@ public void testTypeDeprecation() throws Exception {
         ConditionalProcessor processor = new ConditionalProcessor(
             randomAlphaOfLength(10),
             "description",
-            new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()),
+            new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Map.of()),
             scriptService,
             new Processor() {
                 @Override
@@ -192,7 +188,7 @@ public String getDescription() {
             relativeTimeProvider
         );
 
-        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Map.of());
         execProcessor(processor, ingestDocument, (result, e) -> {});
         assertWarnings("[types removal] Looking up doc types [_type] in scripts is deprecated.");
     }
@@ -255,25 +251,22 @@ private static void assertMutatingCtxThrows(Consumer<Map<String, Object>> mutati
         CompletableFuture<Exception> expectedException = new CompletableFuture<>();
         ScriptService scriptService = new ScriptService(
             Settings.builder().build(),
-            Collections.singletonMap(
-                Script.DEFAULT_SCRIPT_LANG,
-                new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> {
-                    try {
-                        mutation.accept(ctx);
-                    } catch (Exception e) {
-                        expectedException.complete(e);
-                    }
-                    return false;
-                }), Collections.emptyMap())
-            ),
+            Map.of(Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Map.of(scriptName, ctx -> {
+                try {
+                    mutation.accept(ctx);
+                } catch (Exception e) {
+                    expectedException.complete(e);
+                }
+                return false;
+            }), Map.of())),
             new HashMap<>(ScriptModule.CORE_CONTEXTS),
             () -> 1L
         );
         Map<String, Object> document = new HashMap<>();
         ConditionalProcessor processor = new ConditionalProcessor(
             randomAlphaOfLength(10),
-            "desription",
-            new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()),
+            "description",
+            new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Map.of()),
             scriptService,
             new FakeProcessor(null, null, null, null)
         );
diff --git a/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java b/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
index ace57f6a2ebc..5e9e03bbb626 100644
--- a/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
+++ b/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
@@ -28,8 +28,8 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java
index dd9e81318336..7d6dd2d4e23a 100644
--- a/server/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java
+++ b/server/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java
@@ -13,6 +13,8 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -46,7 +48,7 @@ public void testFromXContent() throws IOException {
         XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
         builder.prettyPrint();
         builder.startObject();
-        ingestMetadata.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        ChunkedToXContent.wrapAsToXContent(ingestMetadata).toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
         XContentBuilder shuffled = shuffleXContent(builder);
         try (XContentParser parser = createParser(shuffled)) {
@@ -121,4 +123,18 @@ public void testDiff() throws Exception {
             equalTo(new PipelineConfiguration("2", new BytesArray("{\"key\" : \"value\"}"), XContentType.JSON))
         );
     }
+
+    public void testChunkedToXContent() {
+        final BytesReference pipelineConfig = new BytesArray("{}");
+        final int pipelines = randomInt(10);
+        final Map<String, PipelineConfiguration> pipelineConfigurations = new HashMap<>();
+        for (int i = 0; i < pipelines; i++) {
+            final String id = Integer.toString(i);
+            pipelineConfigurations.put(id, new PipelineConfiguration(id, pipelineConfig, XContentType.JSON));
+        }
+        AbstractChunkedSerializingTestCase.assertChunkCount(
+            new IngestMetadata(pipelineConfigurations),
+            response -> 2 + response.getPipelines().size()
+        );
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java
index 196cb62bf018..ffafe12ce0dd 100644
--- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java
@@ -46,6 +46,7 @@
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.VersionType;
@@ -55,6 +56,7 @@
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.script.TemplateScript;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockLogAppender;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -70,13 +72,13 @@
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
@@ -87,8 +89,6 @@
 import java.util.function.LongSupplier;
 import java.util.stream.Collectors;
 
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.emptySet;
 import static org.elasticsearch.cluster.service.ClusterStateTaskExecutorUtils.executeAndAssertSuccessful;
 import static org.elasticsearch.core.Tuple.tuple;
 import static org.hamcrest.Matchers.containsInAnyOrder;
@@ -101,13 +101,13 @@
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyInt;
-import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.ArgumentMatchers.argThat;
-import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyInt;
+import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.argThat;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
@@ -119,7 +119,7 @@ public class IngestServiceTests extends ESTestCase {
     private static final IngestPlugin DUMMY_PLUGIN = new IngestPlugin() {
         @Override
         public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
-            return Collections.singletonMap("foo", (factories, tag, description, config) -> null);
+            return Map.of("foo", (factories, tag, description, config) -> null);
         }
     };
 
@@ -140,7 +140,7 @@ public void testIngestPlugin() {
             null,
             null,
             null,
-            Collections.singletonList(DUMMY_PLUGIN),
+            List.of(DUMMY_PLUGIN),
             client
         );
         Map<String, Processor.Factory> factories = ingestService.getProcessorFactories();
@@ -152,15 +152,7 @@ public void testIngestPluginDuplicate() {
         Client client = mock(Client.class);
         IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
-            () -> new IngestService(
-                mock(ClusterService.class),
-                threadPool,
-                null,
-                null,
-                null,
-                Arrays.asList(DUMMY_PLUGIN, DUMMY_PLUGIN),
-                client
-            )
+            () -> new IngestService(mock(ClusterService.class), threadPool, null, null, null, List.of(DUMMY_PLUGIN, DUMMY_PLUGIN), client)
         );
         assertTrue(e.getMessage(), e.getMessage().contains("already registered"));
     }
@@ -173,11 +165,11 @@ public void testExecuteIndexPipelineDoesNotExist() {
             null,
             null,
             null,
-            Collections.singletonList(DUMMY_PLUGIN),
+            List.of(DUMMY_PLUGIN),
             client
         );
         final IndexRequest indexRequest = new IndexRequest("_index").id("_id")
-            .source(emptyMap())
+            .source(Map.of())
             .setPipeline("_id")
             .setFinalPipeline("_none");
 
@@ -192,14 +184,7 @@ public void testExecuteIndexPipelineDoesNotExist() {
         @SuppressWarnings("unchecked")
         final BiConsumer<Thread, Exception> completionHandler = mock(BiConsumer.class);
 
-        ingestService.executeBulkRequest(
-            1,
-            Collections.singletonList(indexRequest),
-            failureHandler,
-            completionHandler,
-            indexReq -> {},
-            Names.WRITE
-        );
+        ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE);
 
         assertTrue(failure.get());
         verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
@@ -214,7 +199,7 @@ public void testUpdatePipelines() {
 
         PipelineConfiguration pipeline = new PipelineConfiguration("_id", new BytesArray("""
             {"processors": [{"set" : {"field": "_field", "value": "_value"}}]}"""), XContentType.JSON);
-        IngestMetadata ingestMetadata = new IngestMetadata(Collections.singletonMap("_id", pipeline));
+        IngestMetadata ingestMetadata = new IngestMetadata(Map.of("_id", pipeline));
         clusterState = ClusterState.builder(clusterState)
             .metadata(Metadata.builder().putCustom(IngestMetadata.TYPE, ingestMetadata))
             .build();
@@ -291,7 +276,7 @@ public void testDelete() {
         IngestService ingestService = createWithProcessors();
         PipelineConfiguration config = new PipelineConfiguration("_id", new BytesArray("""
             {"processors": [{"set" : {"field": "_field", "value": "_value"}}]}"""), XContentType.JSON);
-        IngestMetadata ingestMetadata = new IngestMetadata(Collections.singletonMap("_id", config));
+        IngestMetadata ingestMetadata = new IngestMetadata(Map.of("_id", config));
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
         ClusterState previousClusterState = clusterState;
         clusterState = ClusterState.builder(clusterState)
@@ -323,19 +308,13 @@ public void testValidateNoIngestInfo() throws Exception {
         var pipelineConfig = XContentHelper.convertToMap(putRequest.getSource(), false, putRequest.getXContentType()).v2();
         Exception e = expectThrows(
             IllegalStateException.class,
-            () -> ingestService.validatePipeline(emptyMap(), putRequest.getId(), pipelineConfig)
+            () -> ingestService.validatePipeline(Map.of(), putRequest.getId(), pipelineConfig)
         );
         assertEquals("Ingest info is empty", e.getMessage());
 
-        DiscoveryNode discoveryNode = new DiscoveryNode(
-            "_node_id",
-            buildNewFakeTransportAddress(),
-            emptyMap(),
-            emptySet(),
-            Version.CURRENT
-        );
-        IngestInfo ingestInfo = new IngestInfo(Collections.singletonList(new ProcessorInfo("set")));
-        ingestService.validatePipeline(Collections.singletonMap(discoveryNode, ingestInfo), putRequest.getId(), pipelineConfig);
+        DiscoveryNode discoveryNode = new DiscoveryNode("_node_id", buildNewFakeTransportAddress(), Map.of(), Set.of(), Version.CURRENT);
+        IngestInfo ingestInfo = new IngestInfo(List.of(new ProcessorInfo("set")));
+        ingestService.validatePipeline(Map.of(discoveryNode, ingestInfo), putRequest.getId(), pipelineConfig);
     }
 
     public void testValidateNotInUse() {
@@ -543,13 +522,10 @@ public void testGetProcessorsInPipelineComplexConditional() throws Exception {
         String scriptName = "conditionalScript";
         ScriptService scriptService = new ScriptService(
             Settings.builder().build(),
-            Collections.singletonMap(
-                Script.DEFAULT_SCRIPT_LANG,
-                new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> {
-                    ctx.get("_type");
-                    return true;
-                }), Collections.emptyMap())
-            ),
+            Map.of(Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Map.of(scriptName, ctx -> {
+                ctx.get("_type");
+                return true;
+            }), Map.of())),
             new HashMap<>(ScriptModule.CORE_CONTEXTS),
             () -> 1L
         );
@@ -562,12 +538,12 @@ public void testGetProcessorsInPipelineComplexConditional() throws Exception {
             return new ConditionalProcessor(
                 randomAlphaOfLength(10),
                 null,
-                new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()),
+                new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Map.of()),
                 scriptService,
                 new ConditionalProcessor(
                     randomAlphaOfLength(10) + "-nested",
                     null,
-                    new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()),
+                    new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Map.of()),
                     scriptService,
                     new FakeProcessor("complexSet", tag, description, (ingestDocument) -> ingestDocument.setFieldValue(field, value))
                 )
@@ -777,7 +753,7 @@ public void testDeleteWithIndexUsePipeline() {
         IngestService ingestService = createWithProcessors();
         PipelineConfiguration config = new PipelineConfiguration("_id", new BytesArray("""
             {"processors": [{"set" : {"field": "_field", "value": "_value"}}]}"""), XContentType.JSON);
-        IngestMetadata ingestMetadata = new IngestMetadata(Collections.singletonMap("_id", config));
+        IngestMetadata ingestMetadata = new IngestMetadata(Map.of("_id", config));
         Metadata.Builder builder = Metadata.builder();
         for (int i = 0; i < randomIntBetween(2, 10); i++) {
             builder.put(
@@ -896,11 +872,11 @@ public void testValidate() throws Exception {
             }"""), XContentType.JSON);
         var pipelineConfig = XContentHelper.convertToMap(putRequest.getSource(), false, putRequest.getXContentType()).v2();
 
-        DiscoveryNode node1 = new DiscoveryNode("_node_id1", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
-        DiscoveryNode node2 = new DiscoveryNode("_node_id2", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
+        DiscoveryNode node1 = new DiscoveryNode("_node_id1", buildNewFakeTransportAddress(), Map.of(), Set.of(), Version.CURRENT);
+        DiscoveryNode node2 = new DiscoveryNode("_node_id2", buildNewFakeTransportAddress(), Map.of(), Set.of(), Version.CURRENT);
         Map<DiscoveryNode, IngestInfo> ingestInfos = new HashMap<>();
-        ingestInfos.put(node1, new IngestInfo(Arrays.asList(new ProcessorInfo("set"), new ProcessorInfo("remove"))));
-        ingestInfos.put(node2, new IngestInfo(Arrays.asList(new ProcessorInfo("set"))));
+        ingestInfos.put(node1, new IngestInfo(List.of(new ProcessorInfo("set"), new ProcessorInfo("remove"))));
+        ingestInfos.put(node2, new IngestInfo(List.of(new ProcessorInfo("set"))));
 
         ElasticsearchParseException e = expectThrows(
             ElasticsearchParseException.class,
@@ -911,13 +887,13 @@ public void testValidate() throws Exception {
         assertEquals("tag2", e.getMetadata("es.processor_tag").get(0));
 
         var pipelineConfig2 = XContentHelper.convertToMap(putRequest.getSource(), false, putRequest.getXContentType()).v2();
-        ingestInfos.put(node2, new IngestInfo(Arrays.asList(new ProcessorInfo("set"), new ProcessorInfo("remove"))));
+        ingestInfos.put(node2, new IngestInfo(List.of(new ProcessorInfo("set"), new ProcessorInfo("remove"))));
         ingestService.validatePipeline(ingestInfos, putRequest.getId(), pipelineConfig2);
     }
 
     public void testExecuteIndexPipelineExistsButFailedParsing() {
         IngestService ingestService = createWithProcessors(
-            Collections.singletonMap("mock", (factories, tag, description, config) -> new AbstractProcessor("mock", "description") {
+            Map.of("mock", (factories, tag, description, config) -> new AbstractProcessor("mock", "description") {
                 @Override
                 public IngestDocument execute(IngestDocument ingestDocument) {
                     throw new IllegalStateException("error");
@@ -943,11 +919,11 @@ public String getType() {
 
         BulkRequest bulkRequest = new BulkRequest();
         final IndexRequest indexRequest1 = new IndexRequest("_index").id("_id1")
-            .source(emptyMap())
+            .source(Map.of())
             .setPipeline("_none")
             .setFinalPipeline("_none");
         bulkRequest.add(indexRequest1);
-        IndexRequest indexRequest2 = new IndexRequest("_index").id("_id2").source(emptyMap()).setPipeline(id).setFinalPipeline("_none");
+        IndexRequest indexRequest2 = new IndexRequest("_index").id("_id2").source(Map.of()).setPipeline(id).setFinalPipeline("_none");
         bulkRequest.add(indexRequest2);
 
         final BiConsumer<Integer, Exception> failureHandler = (slot, e) -> {
@@ -963,9 +939,9 @@ public String getType() {
         ingestService.executeBulkRequest(
             bulkRequest.numberOfActions(),
             bulkRequest.requests(),
+            indexReq -> {},
             failureHandler,
             completionHandler,
-            indexReq -> {},
             Names.WRITE
         );
 
@@ -975,7 +951,7 @@ public String getType() {
 
     public void testExecuteBulkPipelineDoesNotExist() {
         IngestService ingestService = createWithProcessors(
-            Collections.singletonMap("mock", (factories, tag, description, config) -> mockCompoundProcessor())
+            Map.of("mock", (factories, tag, description, config) -> mockCompoundProcessor())
         );
 
         PutPipelineRequest putRequest = new PutPipelineRequest(
@@ -990,15 +966,12 @@ public void testExecuteBulkPipelineDoesNotExist() {
 
         BulkRequest bulkRequest = new BulkRequest();
 
-        IndexRequest indexRequest1 = new IndexRequest("_index").id("_id1")
-            .source(emptyMap())
-            .setPipeline("_none")
-            .setFinalPipeline("_none");
+        IndexRequest indexRequest1 = new IndexRequest("_index").id("_id1").source(Map.of()).setPipeline("_none").setFinalPipeline("_none");
         bulkRequest.add(indexRequest1);
-        IndexRequest indexRequest2 = new IndexRequest("_index").id("_id2").source(emptyMap()).setPipeline("_id").setFinalPipeline("_none");
+        IndexRequest indexRequest2 = new IndexRequest("_index").id("_id2").source(Map.of()).setPipeline("_id").setFinalPipeline("_none");
         bulkRequest.add(indexRequest2);
         IndexRequest indexRequest3 = new IndexRequest("_index").id("_id3")
-            .source(Collections.emptyMap())
+            .source(Map.of())
             .setPipeline("does_not_exist")
             .setFinalPipeline("_none");
         bulkRequest.add(indexRequest3);
@@ -1009,9 +982,9 @@ public void testExecuteBulkPipelineDoesNotExist() {
         ingestService.executeBulkRequest(
             bulkRequest.numberOfActions(),
             bulkRequest.requests(),
+            indexReq -> {},
             failureHandler,
             completionHandler,
-            indexReq -> {},
             Names.WRITE
         );
         verify(failureHandler, times(1)).accept(
@@ -1023,7 +996,7 @@ public void testExecuteBulkPipelineDoesNotExist() {
 
     public void testExecuteSuccess() {
         IngestService ingestService = createWithProcessors(
-            Collections.singletonMap("mock", (factories, tag, description, config) -> mockCompoundProcessor())
+            Map.of("mock", (factories, tag, description, config) -> mockCompoundProcessor())
         );
         PutPipelineRequest putRequest = new PutPipelineRequest(
             "_id",
@@ -1035,28 +1008,21 @@ public void testExecuteSuccess() {
         clusterState = executePut(putRequest, clusterState);
         ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
         final IndexRequest indexRequest = new IndexRequest("_index").id("_id")
-            .source(emptyMap())
+            .source(Map.of())
             .setPipeline("_id")
             .setFinalPipeline("_none");
         @SuppressWarnings("unchecked")
         final BiConsumer<Integer, Exception> failureHandler = mock(BiConsumer.class);
         @SuppressWarnings("unchecked")
         final BiConsumer<Thread, Exception> completionHandler = mock(BiConsumer.class);
-        ingestService.executeBulkRequest(
-            1,
-            Collections.singletonList(indexRequest),
-            failureHandler,
-            completionHandler,
-            indexReq -> {},
-            Names.WRITE
-        );
+        ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE);
         verify(failureHandler, never()).accept(any(), any());
         verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
     }
 
     public void testDynamicTemplates() throws Exception {
         IngestService ingestService = createWithProcessors(
-            Collections.singletonMap(
+            Map.of(
                 "set",
                 (factories, tag, description, config) -> new FakeProcessor(
                     "set",
@@ -1076,26 +1042,19 @@ public void testDynamicTemplates() throws Exception {
         clusterState = executePut(putRequest, clusterState);
         ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
         final IndexRequest indexRequest = new IndexRequest("_index").id("_id")
-            .source(emptyMap())
+            .source(Map.of())
             .setPipeline("_id")
             .setFinalPipeline("_none");
         CountDownLatch latch = new CountDownLatch(1);
         final BiConsumer<Integer, Exception> failureHandler = (v, e) -> { throw new AssertionError("must never fail", e); };
         final BiConsumer<Thread, Exception> completionHandler = (t, e) -> latch.countDown();
-        ingestService.executeBulkRequest(
-            1,
-            Collections.singletonList(indexRequest),
-            failureHandler,
-            completionHandler,
-            indexReq -> {},
-            Names.WRITE
-        );
+        ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE);
         latch.await();
         assertThat(indexRequest.getDynamicTemplates(), equalTo(Map.of("foo", "bar", "foo.bar", "baz")));
     }
 
     public void testExecuteEmptyPipeline() throws Exception {
-        IngestService ingestService = createWithProcessors(emptyMap());
+        IngestService ingestService = createWithProcessors(Map.of());
         PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray("""
             {"processors": [], "description": "_description"}"""), XContentType.JSON);
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); // Start empty
@@ -1103,30 +1062,21 @@ public void testExecuteEmptyPipeline() throws Exception {
         clusterState = executePut(putRequest, clusterState);
         ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
         final IndexRequest indexRequest = new IndexRequest("_index").id("_id")
-            .source(emptyMap())
+            .source(Map.of())
             .setPipeline("_id")
             .setFinalPipeline("_none");
         @SuppressWarnings("unchecked")
         final BiConsumer<Integer, Exception> failureHandler = mock(BiConsumer.class);
         @SuppressWarnings("unchecked")
         final BiConsumer<Thread, Exception> completionHandler = mock(BiConsumer.class);
-        ingestService.executeBulkRequest(
-            1,
-            Collections.singletonList(indexRequest),
-            failureHandler,
-            completionHandler,
-            indexReq -> {},
-            Names.WRITE
-        );
+        ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE);
         verify(failureHandler, never()).accept(any(), any());
         verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
     }
 
     public void testExecutePropagateAllMetadataUpdates() throws Exception {
         final CompoundProcessor processor = mockCompoundProcessor();
-        IngestService ingestService = createWithProcessors(
-            Collections.singletonMap("mock", (factories, tag, description, config) -> processor)
-        );
+        IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> processor));
         PutPipelineRequest putRequest = new PutPipelineRequest(
             "_id",
             new BytesArray("{\"processors\": [{\"mock\" : {}}]}"),
@@ -1166,21 +1116,14 @@ public void testExecutePropagateAllMetadataUpdates() throws Exception {
             return null;
         }).when(processor).execute(any(), any());
         final IndexRequest indexRequest = new IndexRequest("_index").id("_id")
-            .source(emptyMap())
+            .source(Map.of())
             .setPipeline("_id")
             .setFinalPipeline("_none");
         @SuppressWarnings("unchecked")
         final BiConsumer<Integer, Exception> failureHandler = mock(BiConsumer.class);
         @SuppressWarnings("unchecked")
         final BiConsumer<Thread, Exception> completionHandler = mock(BiConsumer.class);
-        ingestService.executeBulkRequest(
-            1,
-            Collections.singletonList(indexRequest),
-            failureHandler,
-            completionHandler,
-            indexReq -> {},
-            Names.WRITE
-        );
+        ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE);
         verify(processor).execute(any(), any());
         verify(failureHandler, never()).accept(any(), any());
         verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
@@ -1196,36 +1139,41 @@ public void testExecutePropagateAllMetadataUpdates() throws Exception {
     public void testExecuteFailure() throws Exception {
         final CompoundProcessor processor = mockCompoundProcessor();
         IngestService ingestService = createWithProcessors(
-            Collections.singletonMap("mock", (factories, tag, description, config) -> processor)
+            Map.of(
+                "mock",
+                (factories, tag, description, config) -> processor,
+                "set",
+                (factories, tag, description, config) -> new FakeProcessor("set", "", "", (ingestDocument) -> fail())
+            )
         );
-        PutPipelineRequest putRequest = new PutPipelineRequest(
-            "_id",
+        PutPipelineRequest putRequest1 = new PutPipelineRequest(
+            "_id1",
             new BytesArray("{\"processors\": [{\"mock\" : {}}]}"),
             XContentType.JSON
         );
+        // given that set -> fail() above, it's a failure if a document executes against this pipeline
+        PutPipelineRequest putRequest2 = new PutPipelineRequest(
+            "_id2",
+            new BytesArray("{\"processors\": [{\"set\" : {}}]}"),
+            XContentType.JSON
+        );
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); // Start empty
         ClusterState previousClusterState = clusterState;
-        clusterState = executePut(putRequest, clusterState);
+        clusterState = executePut(putRequest1, clusterState);
+        clusterState = executePut(putRequest2, clusterState);
         ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
         final IndexRequest indexRequest = new IndexRequest("_index").id("_id")
-            .source(emptyMap())
-            .setPipeline("_id")
-            .setFinalPipeline("_none");
+            .source(Map.of())
+            .setPipeline("_id1")
+            .setFinalPipeline("_id2");
         doThrow(new RuntimeException()).when(processor)
-            .execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), emptyMap()), any());
+            .execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), Map.of()), any());
         @SuppressWarnings("unchecked")
         final BiConsumer<Integer, Exception> failureHandler = mock(BiConsumer.class);
         @SuppressWarnings("unchecked")
         final BiConsumer<Thread, Exception> completionHandler = mock(BiConsumer.class);
-        ingestService.executeBulkRequest(
-            1,
-            Collections.singletonList(indexRequest),
-            failureHandler,
-            completionHandler,
-            indexReq -> {},
-            Names.WRITE
-        );
-        verify(processor).execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), emptyMap()), any());
+        ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE);
+        verify(processor).execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), Map.of()), any());
         verify(failureHandler, times(1)).accept(eq(0), any(RuntimeException.class));
         verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
     }
@@ -1239,7 +1187,7 @@ public void testExecuteSuccessWithOnFailure() throws Exception {
             BiConsumer<IngestDocument, Exception> handler = (BiConsumer) args.getArguments()[1];
             handler.accept(null, new RuntimeException());
             return null;
-        }).when(processor).execute(eqIndexTypeId(emptyMap()), any());
+        }).when(processor).execute(eqIndexTypeId(Map.of()), any());
 
         final Processor onFailureProcessor = mock(Processor.class);
         doAnswer(args -> {
@@ -1248,16 +1196,14 @@ public void testExecuteSuccessWithOnFailure() throws Exception {
             BiConsumer<IngestDocument, Exception> handler = (BiConsumer) args.getArguments()[1];
             handler.accept(ingestDocument, null);
             return null;
-        }).when(onFailureProcessor).execute(eqIndexTypeId(emptyMap()), any());
+        }).when(onFailureProcessor).execute(eqIndexTypeId(Map.of()), any());
 
         final CompoundProcessor compoundProcessor = new CompoundProcessor(
             false,
-            Collections.singletonList(processor),
-            Collections.singletonList(new CompoundProcessor(onFailureProcessor))
-        );
-        IngestService ingestService = createWithProcessors(
-            Collections.singletonMap("mock", (factories, tag, description, config) -> compoundProcessor)
+            List.of(processor),
+            List.of(new CompoundProcessor(onFailureProcessor))
         );
+        IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> compoundProcessor));
         PutPipelineRequest putRequest = new PutPipelineRequest(
             "_id",
             new BytesArray("{\"processors\": [{\"mock\" : {}}]}"),
@@ -1268,21 +1214,14 @@ public void testExecuteSuccessWithOnFailure() throws Exception {
         clusterState = executePut(putRequest, clusterState);
         ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
         final IndexRequest indexRequest = new IndexRequest("_index").id("_id")
-            .source(emptyMap())
+            .source(Map.of())
             .setPipeline("_id")
             .setFinalPipeline("_none");
         @SuppressWarnings("unchecked")
         final BiConsumer<Integer, Exception> failureHandler = mock(BiConsumer.class);
         @SuppressWarnings("unchecked")
         final BiConsumer<Thread, Exception> completionHandler = mock(BiConsumer.class);
-        ingestService.executeBulkRequest(
-            1,
-            Collections.singletonList(indexRequest),
-            failureHandler,
-            completionHandler,
-            indexReq -> {},
-            Names.WRITE
-        );
+        ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE);
         verify(failureHandler, never()).accept(eq(0), any(IngestProcessorException.class));
         verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
     }
@@ -1294,16 +1233,14 @@ public void testExecuteFailureWithNestedOnFailure() throws Exception {
         when(onFailureProcessor.isAsync()).thenReturn(true);
         final Processor onFailureOnFailureProcessor = mock(Processor.class);
         when(onFailureOnFailureProcessor.isAsync()).thenReturn(true);
-        final List<Processor> processors = Collections.singletonList(onFailureProcessor);
-        final List<Processor> onFailureProcessors = Collections.singletonList(onFailureOnFailureProcessor);
+        final List<Processor> processors = List.of(onFailureProcessor);
+        final List<Processor> onFailureProcessors = List.of(onFailureOnFailureProcessor);
         final CompoundProcessor compoundProcessor = new CompoundProcessor(
             false,
-            Collections.singletonList(processor),
-            Collections.singletonList(new CompoundProcessor(false, processors, onFailureProcessors))
-        );
-        IngestService ingestService = createWithProcessors(
-            Collections.singletonMap("mock", (factories, tag, description, config) -> compoundProcessor)
+            List.of(processor),
+            List.of(new CompoundProcessor(false, processors, onFailureProcessors))
         );
+        IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> compoundProcessor));
         PutPipelineRequest putRequest = new PutPipelineRequest(
             "_id",
             new BytesArray("{\"processors\": [{\"mock\" : {}}]}"),
@@ -1314,28 +1251,21 @@ public void testExecuteFailureWithNestedOnFailure() throws Exception {
         clusterState = executePut(putRequest, clusterState);
         ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
         final IndexRequest indexRequest = new IndexRequest("_index").id("_id")
-            .source(emptyMap())
+            .source(Map.of())
             .setPipeline("_id")
             .setFinalPipeline("_none");
         doThrow(new RuntimeException()).when(onFailureOnFailureProcessor)
-            .execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), emptyMap()), any());
+            .execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), Map.of()), any());
         doThrow(new RuntimeException()).when(onFailureProcessor)
-            .execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), emptyMap()), any());
+            .execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), Map.of()), any());
         doThrow(new RuntimeException()).when(processor)
-            .execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), emptyMap()), any());
+            .execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), Map.of()), any());
         @SuppressWarnings("unchecked")
         final BiConsumer<Integer, Exception> failureHandler = mock(BiConsumer.class);
         @SuppressWarnings("unchecked")
         final BiConsumer<Thread, Exception> completionHandler = mock(BiConsumer.class);
-        ingestService.executeBulkRequest(
-            1,
-            Collections.singletonList(indexRequest),
-            failureHandler,
-            completionHandler,
-            indexReq -> {},
-            Names.WRITE
-        );
-        verify(processor).execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), emptyMap()), any());
+        ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE);
+        verify(processor).execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), Map.of()), any());
         verify(failureHandler, times(1)).accept(eq(0), any(RuntimeException.class));
         verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
     }
@@ -1365,7 +1295,7 @@ public void testBulkRequestExecutionWithFailures() throws Exception {
 
         CompoundProcessor processor = mock(CompoundProcessor.class);
         when(processor.isAsync()).thenReturn(true);
-        when(processor.getProcessors()).thenReturn(Collections.singletonList(mock(Processor.class)));
+        when(processor.getProcessors()).thenReturn(List.of(mock(Processor.class)));
         Exception error = new RuntimeException();
         doAnswer(args -> {
             @SuppressWarnings("unchecked")
@@ -1373,9 +1303,7 @@ public void testBulkRequestExecutionWithFailures() throws Exception {
             handler.accept(null, error);
             return null;
         }).when(processor).execute(any(), any());
-        IngestService ingestService = createWithProcessors(
-            Collections.singletonMap("mock", (factories, tag, description, config) -> processor)
-        );
+        IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> processor));
         PutPipelineRequest putRequest = new PutPipelineRequest(
             "_id",
             new BytesArray("{\"processors\": [{\"mock\" : {}}]}"),
@@ -1393,9 +1321,9 @@ public void testBulkRequestExecutionWithFailures() throws Exception {
         ingestService.executeBulkRequest(
             numRequest,
             bulkRequest.requests(),
+            indexReq -> {},
             requestItemErrorHandler,
             completionHandler,
-            indexReq -> {},
             Names.WRITE
         );
 
@@ -1447,9 +1375,9 @@ public void testBulkRequestExecution() throws Exception {
         ingestService.executeBulkRequest(
             numRequest,
             bulkRequest.requests(),
+            indexReq -> {},
             requestItemErrorHandler,
             completionHandler,
-            indexReq -> {},
             Names.WRITE
         );
 
@@ -1462,6 +1390,112 @@ public void testBulkRequestExecution() throws Exception {
         }
     }
 
+    public void testIngestAndPipelineStats() throws Exception {
+        final Processor processor = mock(Processor.class);
+        when(processor.getType()).thenReturn("mock");
+        when(processor.getTag()).thenReturn("mockTag");
+        when(processor.isAsync()).thenReturn(true);
+
+        // avoid returning null and dropping the document
+        doAnswer(args -> {
+            @SuppressWarnings("unchecked")
+            BiConsumer<IngestDocument, Exception> handler = (BiConsumer) args.getArguments()[1];
+            handler.accept(RandomDocumentPicks.randomIngestDocument(random()), null);
+            return null;
+        }).when(processor).execute(any(IngestDocument.class), any());
+
+        // mock up an ingest service for returning a pipeline, this is used by the pipeline processor
+        final Pipeline[] pipelineToReturn = new Pipeline[1];
+        final IngestService pipelineIngestService = mock(IngestService.class);
+        when(pipelineIngestService.getPipeline(anyString())).thenAnswer(inv -> pipelineToReturn[0]);
+
+        IngestService ingestService = createWithProcessors(
+            Map.of(
+                "pipeline",
+                (factories, tag, description, config) -> new PipelineProcessor(tag, description, (params) -> new TemplateScript(params) {
+                    @Override
+                    public String execute() {
+                        return "_id3";
+                    } // this pipeline processor will always execute the '_id3' processor
+                }, false, pipelineIngestService),
+                "mock",
+                (factories, tag, description, config) -> processor
+            )
+        );
+
+        {
+            // all zeroes since nothing has executed
+            final IngestStats ingestStats = ingestService.stats();
+            assertThat(ingestStats.getPipelineStats().size(), equalTo(0));
+            assertStats(ingestStats.getTotalStats(), 0, 0, 0);
+        }
+
+        // put some pipelines, and now there are pipeline and processor stats, too
+        PutPipelineRequest putRequest1 = new PutPipelineRequest(
+            "_id1",
+            new BytesArray("{\"processors\": [{\"mock\" : {}}]}"),
+            XContentType.JSON
+        );
+        // n.b. this 'pipeline' processor will always run the '_id3' pipeline, see the mocking/plumbing above and below
+        PutPipelineRequest putRequest2 = new PutPipelineRequest(
+            "_id2",
+            new BytesArray("{\"processors\": [{\"pipeline\" : {}}]}"),
+            XContentType.JSON
+        );
+        PutPipelineRequest putRequest3 = new PutPipelineRequest(
+            "_id3",
+            new BytesArray("{\"processors\": [{\"mock\" : {}}]}"),
+            XContentType.JSON
+        );
+        ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); // Start empty
+        ClusterState previousClusterState = clusterState;
+        clusterState = executePut(putRequest1, clusterState);
+        clusterState = executePut(putRequest2, clusterState);
+        clusterState = executePut(putRequest3, clusterState);
+        ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
+
+        // hook up the mock ingest service to return pipeline3 when asked by the pipeline processor
+        pipelineToReturn[0] = ingestService.getPipeline("_id3");
+
+        {
+            final IngestStats ingestStats = ingestService.stats();
+            assertThat(ingestStats.getPipelineStats().size(), equalTo(3));
+
+            // total
+            assertStats(ingestStats.getTotalStats(), 0, 0, 0);
+            // pipeline
+            assertPipelineStats(ingestStats.getPipelineStats(), "_id1", 0, 0, 0);
+            assertPipelineStats(ingestStats.getPipelineStats(), "_id2", 0, 0, 0);
+            assertPipelineStats(ingestStats.getPipelineStats(), "_id3", 0, 0, 0);
+            // processor
+            assertProcessorStats(0, ingestStats, "_id1", 0, 0, 0);
+            assertProcessorStats(0, ingestStats, "_id2", 0, 0, 0);
+            assertProcessorStats(0, ingestStats, "_id3", 0, 0, 0);
+        }
+
+        // put a single document through ingest processing
+        final IndexRequest indexRequest = new IndexRequest("_index");
+        indexRequest.setPipeline("_id1").setFinalPipeline("_id2");
+        indexRequest.source(randomAlphaOfLength(10), randomAlphaOfLength(10));
+        ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, (integer, e) -> {}, (thread, e) -> {}, Names.WRITE);
+
+        {
+            final IngestStats ingestStats = ingestService.stats();
+            assertThat(ingestStats.getPipelineStats().size(), equalTo(3));
+
+            // total
+            assertStats(ingestStats.getTotalStats(), 1, 0, 0);
+            // pipeline
+            assertPipelineStats(ingestStats.getPipelineStats(), "_id1", 1, 0, 0);
+            assertPipelineStats(ingestStats.getPipelineStats(), "_id2", 1, 0, 0);
+            assertPipelineStats(ingestStats.getPipelineStats(), "_id3", 1, 0, 0);
+            // processor
+            assertProcessorStats(0, ingestStats, "_id1", 1, 0, 0);
+            assertProcessorStats(0, ingestStats, "_id2", 1, 0, 0);
+            assertProcessorStats(0, ingestStats, "_id3", 1, 0, 0);
+        }
+    }
+
     public void testStats() throws Exception {
         final Processor processor = mock(Processor.class);
         final Processor processorFailure = mock(Processor.class);
@@ -1514,14 +1548,7 @@ public void testStats() throws Exception {
         final IndexRequest indexRequest = new IndexRequest("_index");
         indexRequest.setPipeline("_id1").setFinalPipeline("_none");
         indexRequest.source(randomAlphaOfLength(10), randomAlphaOfLength(10));
-        ingestService.executeBulkRequest(
-            1,
-            Collections.singletonList(indexRequest),
-            failureHandler,
-            completionHandler,
-            indexReq -> {},
-            Names.WRITE
-        );
+        ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE);
         final IngestStats afterFirstRequestStats = ingestService.stats();
         assertThat(afterFirstRequestStats.getPipelineStats().size(), equalTo(2));
 
@@ -1538,14 +1565,7 @@ public void testStats() throws Exception {
         assertProcessorStats(0, afterFirstRequestStats, "_id2", 0, 0, 0);
 
         indexRequest.setPipeline("_id2");
-        ingestService.executeBulkRequest(
-            1,
-            Collections.singletonList(indexRequest),
-            failureHandler,
-            completionHandler,
-            indexReq -> {},
-            Names.WRITE
-        );
+        ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE);
         final IngestStats afterSecondRequestStats = ingestService.stats();
         assertThat(afterSecondRequestStats.getPipelineStats().size(), equalTo(2));
         // total
@@ -1567,14 +1587,7 @@ public void testStats() throws Exception {
         clusterState = executePut(putRequest, clusterState);
         ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
         indexRequest.setPipeline("_id1");
-        ingestService.executeBulkRequest(
-            1,
-            Collections.singletonList(indexRequest),
-            failureHandler,
-            completionHandler,
-            indexReq -> {},
-            Names.WRITE
-        );
+        ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE);
         final IngestStats afterThirdRequestStats = ingestService.stats();
         assertThat(afterThirdRequestStats.getPipelineStats().size(), equalTo(2));
         // total
@@ -1597,14 +1610,7 @@ public void testStats() throws Exception {
         clusterState = executePut(putRequest, clusterState);
         ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
         indexRequest.setPipeline("_id1");
-        ingestService.executeBulkRequest(
-            1,
-            Collections.singletonList(indexRequest),
-            failureHandler,
-            completionHandler,
-            indexReq -> {},
-            Names.WRITE
-        );
+        ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, failureHandler, completionHandler, Names.WRITE);
         final IngestStats afterForthRequestStats = ingestService.stats();
         assertThat(afterForthRequestStats.getPipelineStats().size(), equalTo(2));
         // total
@@ -1678,15 +1684,12 @@ public String getDescription() {
 
         BulkRequest bulkRequest = new BulkRequest();
         final IndexRequest indexRequest1 = new IndexRequest("_index").id("_id1")
-            .source(Collections.emptyMap())
+            .source(Map.of())
             .setPipeline("_none")
             .setFinalPipeline("_none");
         bulkRequest.add(indexRequest1);
 
-        IndexRequest indexRequest2 = new IndexRequest("_index").id("_id2")
-            .source(Collections.emptyMap())
-            .setPipeline("_id")
-            .setFinalPipeline("_none");
+        IndexRequest indexRequest2 = new IndexRequest("_index").id("_id2").source(Map.of()).setPipeline("_id").setFinalPipeline("_none");
         bulkRequest.add(indexRequest2);
 
         @SuppressWarnings("unchecked")
@@ -1698,9 +1701,9 @@ public String getDescription() {
         ingestService.executeBulkRequest(
             bulkRequest.numberOfActions(),
             bulkRequest.requests(),
+            dropHandler,
             failureHandler,
             completionHandler,
-            dropHandler,
             Names.WRITE
         );
         verify(failureHandler, never()).accept(any(), any());
@@ -1718,7 +1721,7 @@ public void testIngestClusterStateListeners_orderOfExecution() {
         IngestPlugin testPlugin = new IngestPlugin() {
             @Override
             public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
-                return Collections.singletonMap("test", (factories, tag, description, config) -> {
+                return Map.of("test", (factories, tag, description, config) -> {
                     assertThat(counter.compareAndSet(1, 2), is(true));
                     return new FakeProcessor("test", tag, description, ingestDocument -> {});
                 });
@@ -1757,7 +1760,7 @@ public void testCBORParsing() throws Exception {
         AtomicReference<Object> reference = new AtomicReference<>();
         Consumer<IngestDocument> executor = doc -> reference.set(doc.getFieldValueAsBytes("data"));
         final IngestService ingestService = createWithProcessors(
-            Collections.singletonMap("foo", (factories, tag, description, config) -> new FakeProcessor("foo", tag, description, executor))
+            Map.of("foo", (factories, tag, description, config) -> new FakeProcessor("foo", tag, description, executor))
         );
 
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
@@ -1781,46 +1784,87 @@ public void testCBORParsing() throws Exception {
                 .setPipeline("_id")
                 .setFinalPipeline("_none");
 
-            ingestService.executeBulkRequest(
-                1,
-                Collections.singletonList(indexRequest),
-                (integer, e) -> {},
-                (thread, e) -> {},
-                indexReq -> {},
-                Names.WRITE
-            );
+            ingestService.executeBulkRequest(1, List.of(indexRequest), indexReq -> {}, (integer, e) -> {}, (thread, e) -> {}, Names.WRITE);
         }
 
         assertThat(reference.get(), is(instanceOf(byte[].class)));
     }
 
-    public void testPostIngest() {
+    public void testSetsRawTimestamp() {
         IngestService ingestService = createWithProcessors(
-            Collections.singletonMap("mock", (factories, tag, description, config) -> mockCompoundProcessor())
+            Map.of(
+                "mock",
+                (factories, tag, description, config) -> mockCompoundProcessor(),
+                "set",
+                (factories, tag, description, config) -> new FakeProcessor(
+                    "set",
+                    tag,
+                    description,
+                    // just always set the timestamp field to 100
+                    (ingestDocument) -> ingestDocument.setFieldValue(TimestampField.FIXED_TIMESTAMP_FIELD, 100)
+                )
+            )
         );
 
-        PutPipelineRequest putRequest = new PutPipelineRequest(
-            "_id",
+        PutPipelineRequest putRequest1 = new PutPipelineRequest(
+            "_id1",
             new BytesArray("{\"processors\": [{\"mock\" : {}}]}"),
             XContentType.JSON
         );
+        PutPipelineRequest putRequest2 = new PutPipelineRequest(
+            "_id2",
+            new BytesArray("{\"processors\": [{\"set\" : {}}]}"),
+            XContentType.JSON
+        );
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); // Start empty
         ClusterState previousClusterState = clusterState;
-        clusterState = executePut(putRequest, clusterState);
+        clusterState = executePut(putRequest1, clusterState);
+        clusterState = executePut(putRequest2, clusterState);
         ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
 
+        // feed a document with no timestamp through four scenarios
+        Map<String, Object> doc1 = Map.of("foo", "bar");
+
+        // neither a request nor a final pipeline
+        IndexRequest indexRequest1 = new IndexRequest("idx").setPipeline("_none").setFinalPipeline("_none").source(doc1);
+        // just a request pipeline
+        IndexRequest indexRequest2 = new IndexRequest("idx").setPipeline("_id1").setFinalPipeline("_none").source(doc1);
+        // just a final pipeline
+        IndexRequest indexRequest3 = new IndexRequest("idx").setPipeline("_none").setFinalPipeline("_id2").source(doc1);
+        // both a request and a final pipeline
+        IndexRequest indexRequest4 = new IndexRequest("idx").setPipeline("_id1").setFinalPipeline("_id2").source(doc1);
+
+        // feed a document with a timestamp through four scenarios
+        Map<String, Object> doc2 = Map.of(TimestampField.FIXED_TIMESTAMP_FIELD, 10);
+
+        // neither a request nor a final pipeline
+        IndexRequest indexRequest5 = new IndexRequest("idx").setPipeline("_none").setFinalPipeline("_none").source(doc2);
+        // just a request pipeline
+        IndexRequest indexRequest6 = new IndexRequest("idx").setPipeline("_id1").setFinalPipeline("_none").source(doc2);
+        // just a final pipeline
+        IndexRequest indexRequest7 = new IndexRequest("idx").setPipeline("_none").setFinalPipeline("_id2").source(doc2);
+        // both a request and a final pipeline
+        IndexRequest indexRequest8 = new IndexRequest("idx").setPipeline("_id1").setFinalPipeline("_id2").source(doc2);
+
         BulkRequest bulkRequest = new BulkRequest();
-        IndexRequest indexRequest1 = new IndexRequest("idx").setPipeline("_id")
-            .setFinalPipeline("_id")
-            .source(Map.of(TimestampField.FIXED_TIMESTAMP_FIELD, 10));
-        IndexRequest indexRequest2 = new IndexRequest("idx").setPipeline("_id").setFinalPipeline("_id").source(Map.of("foo", "bar"));
         bulkRequest.add(indexRequest1);
         bulkRequest.add(indexRequest2);
-
-        ingestService.executeBulkRequest(2, bulkRequest.requests(), (integer, e) -> {}, (thread, e) -> {}, indexReq -> {}, Names.WRITE);
-
-        assertThat(indexRequest1.getRawTimestamp(), equalTo(10));
+        bulkRequest.add(indexRequest3);
+        bulkRequest.add(indexRequest4);
+        bulkRequest.add(indexRequest5);
+        bulkRequest.add(indexRequest6);
+        bulkRequest.add(indexRequest7);
+        bulkRequest.add(indexRequest8);
+        ingestService.executeBulkRequest(8, bulkRequest.requests(), indexReq -> {}, (integer, e) -> {}, (thread, e) -> {}, Names.WRITE);
+
+        assertThat(indexRequest1.getRawTimestamp(), nullValue());
         assertThat(indexRequest2.getRawTimestamp(), nullValue());
+        assertThat(indexRequest3.getRawTimestamp(), equalTo(100));
+        assertThat(indexRequest4.getRawTimestamp(), equalTo(100));
+        assertThat(indexRequest5.getRawTimestamp(), nullValue());
+        assertThat(indexRequest6.getRawTimestamp(), equalTo(10));
+        assertThat(indexRequest7.getRawTimestamp(), equalTo(100));
+        assertThat(indexRequest8.getRawTimestamp(), equalTo(100));
     }
 
     public void testResolveRequiredOrDefaultPipelineDefaultPipeline() {
@@ -1975,7 +2019,7 @@ public void testUpdatingRandomPipelineWithoutChangesIsNoOp() throws Exception {
 
     public void testUpdatingPipelineWithoutChangesIsNoOp() throws Exception {
         var value = randomAlphaOfLength(5);
-        var pipelineString = formatted("""
+        var pipelineString = Strings.format("""
             {"processors": [{"set" : {"field": "_field", "value": "%s"}}]}
             """, value);
         testUpdatingPipeline(pipelineString);
@@ -1991,15 +2035,7 @@ private void testUpdatingPipeline(String pipelineString) throws Exception {
         Client client = mock(Client.class);
         ClusterService clusterService = mock(ClusterService.class);
         when(clusterService.state()).thenReturn(clusterState);
-        IngestService ingestService = new IngestService(
-            clusterService,
-            threadPool,
-            null,
-            null,
-            null,
-            Collections.singletonList(DUMMY_PLUGIN),
-            client
-        );
+        IngestService ingestService = new IngestService(clusterService, threadPool, null, null, null, List.of(DUMMY_PLUGIN), client);
         ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, clusterState));
 
         CountDownLatch latch = new CountDownLatch(1);
@@ -2108,10 +2144,7 @@ public void testPutPipelineWithVersionedUpdateSpecifiesSameVersion() throws Exce
 
         var request = new PutPipelineRequest(pipelineId, new BytesArray(pipelineString), XContentType.JSON, version);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> executeFailingPut(request, clusterState));
-        assertThat(
-            e.getMessage(),
-            equalTo(String.format(Locale.ROOT, "cannot update pipeline [%s] with the same version [%s]", pipelineId, version))
-        );
+        assertThat(e.getMessage(), equalTo(Strings.format("cannot update pipeline [%s] with the same version [%s]", pipelineId, version)));
     }
 
     public void testPutPipelineWithVersionedUpdateSpecifiesValidVersion() throws Exception {
@@ -2151,6 +2184,90 @@ public void testPutPipelineWithVersionedUpdateIncrementsVersion() throws Excepti
         assertThat(updatedConfig.getVersion(), equalTo(existingVersion + 1));
     }
 
+    public void testResolvePipelinesWithNonePipeline() {
+        // _none request pipeline:
+        {
+            Metadata metadata = Metadata.builder().build();
+            IndexRequest indexRequest = new IndexRequest("idx").setPipeline(IngestService.NOOP_PIPELINE_NAME);
+            boolean result = IngestService.resolvePipelines(indexRequest, indexRequest, metadata);
+            assertThat(result, is(false));
+            assertThat(indexRequest.isPipelineResolved(), is(true));
+            assertThat(indexRequest.getPipeline(), equalTo(IngestService.NOOP_PIPELINE_NAME));
+        }
+
+        // _none default pipeline:
+        {
+            IndexMetadata.Builder builder = IndexMetadata.builder("idx")
+                .settings(settings(Version.CURRENT).put(IndexSettings.DEFAULT_PIPELINE.getKey(), IngestService.NOOP_PIPELINE_NAME))
+                .numberOfShards(1)
+                .numberOfReplicas(0);
+            Metadata metadata = Metadata.builder().put(builder).build();
+            IndexRequest indexRequest = new IndexRequest("idx");
+            boolean result = IngestService.resolvePipelines(indexRequest, indexRequest, metadata);
+            assertThat(result, is(false));
+            assertThat(indexRequest.isPipelineResolved(), is(true));
+            assertThat(indexRequest.getPipeline(), equalTo(IngestService.NOOP_PIPELINE_NAME));
+        }
+
+        // _none default pipeline with request pipeline:
+        {
+            IndexMetadata.Builder builder = IndexMetadata.builder("idx")
+                .settings(settings(Version.CURRENT).put(IndexSettings.DEFAULT_PIPELINE.getKey(), IngestService.NOOP_PIPELINE_NAME))
+                .numberOfShards(1)
+                .numberOfReplicas(0);
+            Metadata metadata = Metadata.builder().put(builder).build();
+            IndexRequest indexRequest = new IndexRequest("idx").setPipeline("pipeline1");
+            boolean result = IngestService.resolvePipelines(indexRequest, indexRequest, metadata);
+            assertThat(result, is(true));
+            assertThat(indexRequest.isPipelineResolved(), is(true));
+            assertThat(indexRequest.getPipeline(), equalTo("pipeline1"));
+        }
+
+        // _none request pipeline with default pipeline:
+        {
+            IndexMetadata.Builder builder = IndexMetadata.builder("idx")
+                .settings(settings(Version.CURRENT).put(IndexSettings.DEFAULT_PIPELINE.getKey(), "default-pipeline"))
+                .numberOfShards(1)
+                .numberOfReplicas(0);
+            Metadata metadata = Metadata.builder().put(builder).build();
+            IndexRequest indexRequest = new IndexRequest("idx").setPipeline(IngestService.NOOP_PIPELINE_NAME);
+            boolean result = IngestService.resolvePipelines(indexRequest, indexRequest, metadata);
+            assertThat(result, is(false));
+            assertThat(indexRequest.isPipelineResolved(), is(true));
+            assertThat(indexRequest.getPipeline(), equalTo(IngestService.NOOP_PIPELINE_NAME));
+        }
+
+        // _none request pipeline with final pipeline:
+        {
+            IndexMetadata.Builder builder = IndexMetadata.builder("idx")
+                .settings(settings(Version.CURRENT).put(IndexSettings.FINAL_PIPELINE.getKey(), "final-pipeline"))
+                .numberOfShards(1)
+                .numberOfReplicas(0);
+            Metadata metadata = Metadata.builder().put(builder).build();
+            IndexRequest indexRequest = new IndexRequest("idx").setPipeline(IngestService.NOOP_PIPELINE_NAME);
+            boolean result = IngestService.resolvePipelines(indexRequest, indexRequest, metadata);
+            assertThat(result, is(true));
+            assertThat(indexRequest.isPipelineResolved(), is(true));
+            assertThat(indexRequest.getPipeline(), equalTo(IngestService.NOOP_PIPELINE_NAME));
+            assertThat(indexRequest.getFinalPipeline(), equalTo("final-pipeline"));
+        }
+
+        // _none final pipeline:
+        {
+            IndexMetadata.Builder builder = IndexMetadata.builder("idx")
+                .settings(settings(Version.CURRENT).put(IndexSettings.FINAL_PIPELINE.getKey(), IngestService.NOOP_PIPELINE_NAME))
+                .numberOfShards(1)
+                .numberOfReplicas(0);
+            Metadata metadata = Metadata.builder().put(builder).build();
+            IndexRequest indexRequest = new IndexRequest("idx").setPipeline("pipeline1");
+            boolean result = IngestService.resolvePipelines(indexRequest, indexRequest, metadata);
+            assertThat(result, is(true));
+            assertThat(indexRequest.isPipelineResolved(), is(true));
+            assertThat(indexRequest.getPipeline(), equalTo("pipeline1"));
+            assertThat(indexRequest.getFinalPipeline(), equalTo(IngestService.NOOP_PIPELINE_NAME));
+        }
+    }
+
     private static Tuple<String, Object> randomMapEntry() {
         return tuple(randomAlphaOfLength(5), randomObject());
     }
@@ -2191,12 +2308,11 @@ private static IngestService createWithProcessors() {
     }
 
     private static IngestService createWithProcessors(Map<String, Processor.Factory> processors) {
-
         Client client = mock(Client.class);
         ThreadPool threadPool = mock(ThreadPool.class);
         when(threadPool.generic()).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE);
         when(threadPool.executor(anyString())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE);
-        return new IngestService(mock(ClusterService.class), threadPool, null, null, null, Collections.singletonList(new IngestPlugin() {
+        return new IngestService(mock(ClusterService.class), threadPool, null, null, null, List.of(new IngestPlugin() {
             @Override
             public Map<String, Processor.Factory> getProcessors(final Processor.Parameters parameters) {
                 return processors;
diff --git a/server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java b/server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java
index 9ff13e861a32..dde949a7371a 100644
--- a/server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java
+++ b/server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.AbstractXContentTestCase;
 import org.elasticsearch.xcontent.ContextParser;
 import org.elasticsearch.xcontent.DeprecationHandler;
@@ -98,7 +99,7 @@ public void testGetVersion() {
         {
             // null version
             int version = randomInt();
-            String configJson = formatted("""
+            String configJson = Strings.format("""
                 {"version": %d, "description": "blah", "_meta" : {"foo": "bar"}}
                 """, version);
             PipelineConfiguration configuration = new PipelineConfiguration(
diff --git a/server/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java b/server/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java
index 574c631d0917..12a44e652f5f 100644
--- a/server/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java
+++ b/server/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java
@@ -13,14 +13,12 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
 
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Mockito.mock;
@@ -42,11 +40,8 @@ public void testCreate() throws Exception {
         if (metadata != null) {
             pipelineConfig.put(Pipeline.META_KEY, metadata);
         }
-        pipelineConfig.put(
-            Pipeline.PROCESSORS_KEY,
-            Arrays.asList(Collections.singletonMap("test", processorConfig0), Collections.singletonMap("test", processorConfig1))
-        );
-        Map<String, Processor.Factory> processorRegistry = Collections.singletonMap("test", new TestProcessor.Factory());
+        pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of(Map.of("test", processorConfig0), Map.of("test", processorConfig1)));
+        Map<String, Processor.Factory> processorRegistry = Map.of("test", new TestProcessor.Factory());
         Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService);
         assertThat(pipeline.getId(), equalTo("_id"));
         assertThat(pipeline.getDescription(), equalTo("_description"));
@@ -66,7 +61,7 @@ public void testCreateWithNoProcessorsField() throws Exception {
             pipelineConfig.put(Pipeline.META_KEY, metadata);
         }
         try {
-            Pipeline.create("_id", pipelineConfig, Collections.emptyMap(), scriptService);
+            Pipeline.create("_id", pipelineConfig, Map.of(), scriptService);
             fail("should fail, missing required [processors] field");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[processors] required property is missing"));
@@ -80,7 +75,7 @@ public void testCreateWithEmptyProcessorsField() throws Exception {
         if (metadata != null) {
             pipelineConfig.put(Pipeline.META_KEY, metadata);
         }
-        pipelineConfig.put(Pipeline.PROCESSORS_KEY, Collections.emptyList());
+        pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of());
         Pipeline pipeline = Pipeline.create("_id", pipelineConfig, null, scriptService);
         assertThat(pipeline.getId(), equalTo("_id"));
         assertThat(pipeline.getDescription(), equalTo("_description"));
@@ -96,9 +91,9 @@ public void testCreateWithPipelineOnFailure() throws Exception {
         if (metadata != null) {
             pipelineConfig.put(Pipeline.META_KEY, metadata);
         }
-        pipelineConfig.put(Pipeline.PROCESSORS_KEY, Collections.singletonList(Collections.singletonMap("test", processorConfig)));
-        pipelineConfig.put(Pipeline.ON_FAILURE_KEY, Collections.singletonList(Collections.singletonMap("test", processorConfig)));
-        Map<String, Processor.Factory> processorRegistry = Collections.singletonMap("test", new TestProcessor.Factory());
+        pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of(Map.of("test", processorConfig)));
+        pipelineConfig.put(Pipeline.ON_FAILURE_KEY, List.of(Map.of("test", processorConfig)));
+        Map<String, Processor.Factory> processorRegistry = Map.of("test", new TestProcessor.Factory());
         Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService);
         assertThat(pipeline.getId(), equalTo("_id"));
         assertThat(pipeline.getDescription(), equalTo("_description"));
@@ -117,9 +112,9 @@ public void testCreateWithPipelineEmptyOnFailure() throws Exception {
         if (metadata != null) {
             pipelineConfig.put(Pipeline.META_KEY, metadata);
         }
-        pipelineConfig.put(Pipeline.PROCESSORS_KEY, Collections.singletonList(Collections.singletonMap("test", processorConfig)));
-        pipelineConfig.put(Pipeline.ON_FAILURE_KEY, Collections.emptyList());
-        Map<String, Processor.Factory> processorRegistry = Collections.singletonMap("test", new TestProcessor.Factory());
+        pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of(Map.of("test", processorConfig)));
+        pipelineConfig.put(Pipeline.ON_FAILURE_KEY, List.of());
+        Map<String, Processor.Factory> processorRegistry = Map.of("test", new TestProcessor.Factory());
         Exception e = expectThrows(
             ElasticsearchParseException.class,
             () -> Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService)
@@ -129,15 +124,15 @@ public void testCreateWithPipelineEmptyOnFailure() throws Exception {
 
     public void testCreateWithPipelineEmptyOnFailureInProcessor() throws Exception {
         Map<String, Object> processorConfig = new HashMap<>();
-        processorConfig.put(Pipeline.ON_FAILURE_KEY, Collections.emptyList());
+        processorConfig.put(Pipeline.ON_FAILURE_KEY, List.of());
         Map<String, Object> pipelineConfig = new HashMap<>();
         pipelineConfig.put(Pipeline.DESCRIPTION_KEY, "_description");
         pipelineConfig.put(Pipeline.VERSION_KEY, versionString);
         if (metadata != null) {
             pipelineConfig.put(Pipeline.META_KEY, metadata);
         }
-        pipelineConfig.put(Pipeline.PROCESSORS_KEY, Collections.singletonList(Collections.singletonMap("test", processorConfig)));
-        Map<String, Processor.Factory> processorRegistry = Collections.singletonMap("test", new TestProcessor.Factory());
+        pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of(Map.of("test", processorConfig)));
+        Map<String, Processor.Factory> processorRegistry = Map.of("test", new TestProcessor.Factory());
         Exception e = expectThrows(
             ElasticsearchParseException.class,
             () -> Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService)
@@ -149,14 +144,14 @@ public void testCreateWithPipelineIgnoreFailure() throws Exception {
         Map<String, Object> processorConfig = new HashMap<>();
         processorConfig.put("ignore_failure", true);
 
-        Map<String, Processor.Factory> processorRegistry = Collections.singletonMap("test", new TestProcessor.Factory());
+        Map<String, Processor.Factory> processorRegistry = Map.of("test", new TestProcessor.Factory());
         Map<String, Object> pipelineConfig = new HashMap<>();
         pipelineConfig.put(Pipeline.DESCRIPTION_KEY, "_description");
         pipelineConfig.put(Pipeline.VERSION_KEY, versionString);
         if (metadata != null) {
             pipelineConfig.put(Pipeline.META_KEY, metadata);
         }
-        pipelineConfig.put(Pipeline.PROCESSORS_KEY, Collections.singletonList(Collections.singletonMap("test", processorConfig)));
+        pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of(Map.of("test", processorConfig)));
 
         Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService);
         assertThat(pipeline.getId(), equalTo("_id"));
@@ -179,8 +174,8 @@ public void testCreateUnusedProcessorOptions() throws Exception {
         if (metadata != null) {
             pipelineConfig.put(Pipeline.META_KEY, metadata);
         }
-        pipelineConfig.put(Pipeline.PROCESSORS_KEY, Collections.singletonList(Collections.singletonMap("test", processorConfig)));
-        Map<String, Processor.Factory> processorRegistry = Collections.singletonMap("test", new TestProcessor.Factory());
+        pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of(Map.of("test", processorConfig)));
+        Map<String, Processor.Factory> processorRegistry = Map.of("test", new TestProcessor.Factory());
         Exception e = expectThrows(
             ElasticsearchParseException.class,
             () -> Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService)
@@ -190,7 +185,7 @@ public void testCreateUnusedProcessorOptions() throws Exception {
 
     public void testCreateProcessorsWithOnFailureProperties() throws Exception {
         Map<String, Object> processorConfig = new HashMap<>();
-        processorConfig.put(Pipeline.ON_FAILURE_KEY, Collections.singletonList(Collections.singletonMap("test", new HashMap<>())));
+        processorConfig.put(Pipeline.ON_FAILURE_KEY, List.of(Map.of("test", new HashMap<>())));
 
         Map<String, Object> pipelineConfig = new HashMap<>();
         pipelineConfig.put(Pipeline.DESCRIPTION_KEY, "_description");
@@ -198,8 +193,8 @@ public void testCreateProcessorsWithOnFailureProperties() throws Exception {
         if (metadata != null) {
             pipelineConfig.put(Pipeline.META_KEY, metadata);
         }
-        pipelineConfig.put(Pipeline.PROCESSORS_KEY, Collections.singletonList(Collections.singletonMap("test", processorConfig)));
-        Map<String, Processor.Factory> processorRegistry = Collections.singletonMap("test", new TestProcessor.Factory());
+        pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of(Map.of("test", processorConfig)));
+        Map<String, Processor.Factory> processorRegistry = Map.of("test", new TestProcessor.Factory());
         Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService);
         assertThat(pipeline.getId(), equalTo("_id"));
         assertThat(pipeline.getDescription(), equalTo("_description"));
@@ -211,11 +206,7 @@ public void testCreateProcessorsWithOnFailureProperties() throws Exception {
     public void testFlattenProcessors() throws Exception {
         TestProcessor testProcessor = new TestProcessor(ingestDocument -> {});
         CompoundProcessor processor1 = new CompoundProcessor(testProcessor, testProcessor);
-        CompoundProcessor processor2 = new CompoundProcessor(
-            false,
-            Collections.singletonList(testProcessor),
-            Collections.singletonList(testProcessor)
-        );
+        CompoundProcessor processor2 = new CompoundProcessor(false, List.of(testProcessor), List.of(testProcessor));
         Pipeline pipeline = new Pipeline("_id", "_description", version, null, new CompoundProcessor(processor1, processor2));
         List<Processor> flattened = pipeline.flattenAllProcessors();
         assertThat(flattened.size(), equalTo(4));
diff --git a/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java
index 68ec92118659..b15d37fdbc16 100644
--- a/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java
+++ b/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java
@@ -13,8 +13,6 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -22,8 +20,8 @@
 import java.util.concurrent.TimeUnit;
 import java.util.function.LongSupplier;
 
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -61,7 +59,7 @@ public String getDescription() {
         PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService);
         Map<String, Object> config = new HashMap<>();
         config.put("name", pipelineId);
-        factory.create(Collections.emptyMap(), null, null, config).execute(testIngestDocument, (result, e) -> {});
+        factory.create(Map.of(), null, null, config).execute(testIngestDocument, (result, e) -> {});
         assertEquals(testIngestDocument, invoked.get());
     }
 
@@ -72,8 +70,7 @@ public void testThrowsOnMissingPipeline() throws Exception {
         Map<String, Object> config = new HashMap<>();
         config.put("name", "missingPipelineId");
         IllegalStateException[] e = new IllegalStateException[1];
-        factory.create(Collections.emptyMap(), null, null, config)
-            .execute(testIngestDocument, (result, e1) -> e[0] = (IllegalStateException) e1);
+        factory.create(Map.of(), null, null, config).execute(testIngestDocument, (result, e1) -> e[0] = (IllegalStateException) e1);
         assertEquals("Pipeline processor configured for non-existent pipeline [missingPipelineId]", e[0].getMessage());
     }
 
@@ -87,7 +84,7 @@ public void testIgnoreMissingPipeline() throws Exception {
 
         var r = new IngestDocument[1];
         var e = new Exception[1];
-        var processor = factory.create(Collections.emptyMap(), null, null, config);
+        var processor = factory.create(Map.of(), null, null, config);
         processor.execute(testIngestDocument, (result, e1) -> {
             r[0] = result;
             e[0] = e1;
@@ -109,7 +106,7 @@ public void testThrowsOnRecursivePipelineInvocations() throws Exception {
             null,
             null,
             null,
-            new CompoundProcessor(factory.create(Collections.emptyMap(), null, null, outerConfig))
+            new CompoundProcessor(factory.create(Map.of(), null, null, outerConfig))
         );
         Map<String, Object> innerConfig = new HashMap<>();
         innerConfig.put("name", outerPipelineId);
@@ -118,14 +115,13 @@ public void testThrowsOnRecursivePipelineInvocations() throws Exception {
             null,
             null,
             null,
-            new CompoundProcessor(factory.create(Collections.emptyMap(), null, null, innerConfig))
+            new CompoundProcessor(factory.create(Map.of(), null, null, innerConfig))
         );
         when(ingestService.getPipeline(outerPipelineId)).thenReturn(outer);
         when(ingestService.getPipeline(innerPipelineId)).thenReturn(inner);
         outerConfig.put("name", innerPipelineId);
         ElasticsearchException[] e = new ElasticsearchException[1];
-        factory.create(Collections.emptyMap(), null, null, outerConfig)
-            .execute(testIngestDocument, (result, e1) -> e[0] = (ElasticsearchException) e1);
+        factory.create(Map.of(), null, null, outerConfig).execute(testIngestDocument, (result, e1) -> e[0] = (ElasticsearchException) e1);
         assertEquals("Cycle detected for pipeline: inner", e[0].getRootCause().getMessage());
     }
 
@@ -138,7 +134,7 @@ public void testAllowsRepeatedPipelineInvocations() throws Exception {
         PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService);
         Pipeline inner = new Pipeline(innerPipelineId, null, null, null, new CompoundProcessor());
         when(ingestService.getPipeline(innerPipelineId)).thenReturn(inner);
-        Processor outerProc = factory.create(Collections.emptyMap(), null, null, outerConfig);
+        Processor outerProc = factory.create(Map.of(), null, null, outerConfig);
         outerProc.execute(testIngestDocument, (result, e) -> {});
         outerProc.execute(testIngestDocument, (result, e) -> {});
     }
@@ -152,11 +148,11 @@ public void testPipelineProcessorWithPipelineChain() throws Exception {
 
         Map<String, Object> pipeline1ProcessorConfig = new HashMap<>();
         pipeline1ProcessorConfig.put("name", pipeline2Id);
-        PipelineProcessor pipeline1Processor = factory.create(Collections.emptyMap(), null, null, pipeline1ProcessorConfig);
+        PipelineProcessor pipeline1Processor = factory.create(Map.of(), null, null, pipeline1ProcessorConfig);
 
         Map<String, Object> pipeline2ProcessorConfig = new HashMap<>();
         pipeline2ProcessorConfig.put("name", pipeline3Id);
-        PipelineProcessor pipeline2Processor = factory.create(Collections.emptyMap(), null, null, pipeline2ProcessorConfig);
+        PipelineProcessor pipeline2Processor = factory.create(Map.of(), null, null, pipeline2ProcessorConfig);
 
         LongSupplier relativeTimeProvider = mock(LongSupplier.class);
         when(relativeTimeProvider.getAsLong()).thenReturn(0L);
@@ -172,11 +168,8 @@ public void testPipelineProcessorWithPipelineChain() throws Exception {
             null,
             new CompoundProcessor(
                 true,
-                Arrays.asList(
-                    new TestProcessor(ingestDocument -> { ingestDocument.setFieldValue(key1, randomInt()); }),
-                    pipeline2Processor
-                ),
-                Collections.emptyList()
+                List.of(new TestProcessor(ingestDocument -> { ingestDocument.setFieldValue(key1, randomInt()); }), pipeline2Processor),
+                List.of()
             ),
             relativeTimeProvider
         );
diff --git a/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java
index 7054778b8528..dfeb418918fd 100644
--- a/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java
+++ b/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java
@@ -20,8 +20,6 @@
 import org.mockito.Mockito;
 
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -31,11 +29,11 @@
 import static org.elasticsearch.ingest.CompoundProcessor.ON_FAILURE_PROCESSOR_TYPE_FIELD;
 import static org.elasticsearch.ingest.PipelineProcessorTests.createIngestService;
 import static org.elasticsearch.ingest.TrackingResultProcessor.decorate;
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.not;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
 import static org.mockito.Mockito.verify;
@@ -103,14 +101,8 @@ public void testActualCompoundProcessorWithOnFailure() throws Exception {
         TestProcessor onFailureProcessor = new TestProcessor("success", "test", null, ingestDocument -> {});
         CompoundProcessor actualProcessor = new CompoundProcessor(
             false,
-            Arrays.asList(
-                new CompoundProcessor(
-                    false,
-                    Arrays.asList(failProcessor, onFailureProcessor),
-                    Arrays.asList(onFailureProcessor, failProcessor)
-                )
-            ),
-            Arrays.asList(onFailureProcessor)
+            List.of(new CompoundProcessor(false, List.of(failProcessor, onFailureProcessor), List.of(onFailureProcessor, failProcessor))),
+            List.of(onFailureProcessor)
         );
         CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
         trackingProcessor.execute(ingestDocument, (result, e) -> {});
@@ -161,10 +153,7 @@ public void testActualCompoundProcessorWithOnFailureAndTrueCondition() throws Ex
         String scriptName = "conditionalScript";
         ScriptService scriptService = new ScriptService(
             Settings.builder().build(),
-            Collections.singletonMap(
-                Script.DEFAULT_SCRIPT_LANG,
-                new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> true), Collections.emptyMap())
-            ),
+            Map.of(Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Map.of(scriptName, ctx -> true), Map.of())),
             new HashMap<>(ScriptModule.CORE_CONTEXTS),
             () -> 1L
         );
@@ -173,16 +162,12 @@ public void testActualCompoundProcessorWithOnFailureAndTrueCondition() throws Ex
         ConditionalProcessor conditionalProcessor = new ConditionalProcessor(
             randomAlphaOfLength(10),
             null,
-            new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()),
+            new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Map.of()),
             scriptService,
             failProcessor
         );
         TestProcessor onFailureProcessor = new TestProcessor("success", "test", null, ingestDocument -> {});
-        CompoundProcessor actualProcessor = new CompoundProcessor(
-            false,
-            Arrays.asList(conditionalProcessor),
-            Arrays.asList(onFailureProcessor)
-        );
+        CompoundProcessor actualProcessor = new CompoundProcessor(false, List.of(conditionalProcessor), List.of(onFailureProcessor));
         CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
         trackingProcessor.execute(ingestDocument, (result, e) -> {});
 
@@ -223,7 +208,7 @@ public void testActualCompoundProcessorWithOnFailureAndTrueCondition() throws Ex
     public void testActualCompoundProcessorWithIgnoreFailure() throws Exception {
         RuntimeException exception = new RuntimeException("processor failed");
         TestProcessor testProcessor = new TestProcessor(ingestDocument -> { throw exception; });
-        CompoundProcessor actualProcessor = new CompoundProcessor(true, Collections.singletonList(testProcessor), Collections.emptyList());
+        CompoundProcessor actualProcessor = new CompoundProcessor(true, List.of(testProcessor), List.of());
         CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
 
         trackingProcessor.execute(ingestDocument, (result, e) -> {});
@@ -250,9 +235,9 @@ public void testActualCompoundProcessorWithFalseConditional() throws Exception {
         String scriptName = "conditionalScript";
         ScriptService scriptService = new ScriptService(
             Settings.builder().build(),
-            Collections.singletonMap(
+            Map.of(
                 Script.DEFAULT_SCRIPT_LANG,
-                new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> false), Collections.emptyMap())
+                new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Map.of(scriptName, ctx -> false), Map.of())
             ),
             new HashMap<>(ScriptModule.CORE_CONTEXTS),
             () -> 1L
@@ -263,7 +248,7 @@ public void testActualCompoundProcessorWithFalseConditional() throws Exception {
             new ConditionalProcessor(
                 randomAlphaOfLength(10),
                 null,
-                new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()),
+                new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Map.of()),
                 scriptService,
                 new TestProcessor(ingestDocument -> { ingestDocument.setFieldValue(key2, randomInt()); })
             ),
@@ -322,7 +307,7 @@ public void testActualPipelineProcessor() throws Exception {
         );
         when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline);
 
-        PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, null, pipelineConfig);
+        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig);
         CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
 
         CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@@ -378,10 +363,7 @@ public void testActualPipelineProcessorWithTrueConditional() throws Exception {
 
         ScriptService scriptService = new ScriptService(
             Settings.builder().build(),
-            Collections.singletonMap(
-                Script.DEFAULT_SCRIPT_LANG,
-                new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> true), Collections.emptyMap())
-            ),
+            Map.of(Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Map.of(scriptName, ctx -> true), Map.of())),
             new HashMap<>(ScriptModule.CORE_CONTEXTS),
             () -> 1L
         );
@@ -396,9 +378,9 @@ public void testActualPipelineProcessorWithTrueConditional() throws Exception {
                 new ConditionalProcessor(
                     randomAlphaOfLength(10),
                     null,
-                    new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()),
+                    new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Map.of()),
                     scriptService,
-                    factory.create(Collections.emptyMap(), "pipeline1", null, pipelineConfig2)
+                    factory.create(Map.of(), "pipeline1", null, pipelineConfig2)
                 ),
                 new TestProcessor(ingestDocument -> { ingestDocument.setFieldValue(key3, randomInt()); })
             )
@@ -415,7 +397,7 @@ public void testActualPipelineProcessorWithTrueConditional() throws Exception {
         when(ingestService.getPipeline(pipelineId1)).thenReturn(pipeline1);
         when(ingestService.getPipeline(pipelineId2)).thenReturn(pipeline2);
 
-        PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), "pipeline0", null, pipelineConfig0);
+        PipelineProcessor pipelineProcessor = factory.create(Map.of(), "pipeline0", null, pipelineConfig0);
         CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
 
         CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@@ -478,9 +460,9 @@ public void testActualPipelineProcessorWithFalseConditional() throws Exception {
 
         ScriptService scriptService = new ScriptService(
             Settings.builder().build(),
-            Collections.singletonMap(
+            Map.of(
                 Script.DEFAULT_SCRIPT_LANG,
-                new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> false), Collections.emptyMap())
+                new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Map.of(scriptName, ctx -> false), Map.of())
             ),
             new HashMap<>(ScriptModule.CORE_CONTEXTS),
             () -> 1L
@@ -496,9 +478,9 @@ public void testActualPipelineProcessorWithFalseConditional() throws Exception {
                 new ConditionalProcessor(
                     randomAlphaOfLength(10),
                     null,
-                    new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()),
+                    new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Map.of()),
                     scriptService,
-                    factory.create(Collections.emptyMap(), null, null, pipelineConfig2)
+                    factory.create(Map.of(), null, null, pipelineConfig2)
                 ),
                 new TestProcessor(ingestDocument -> { ingestDocument.setFieldValue(key3, randomInt()); })
             )
@@ -515,7 +497,7 @@ public void testActualPipelineProcessorWithFalseConditional() throws Exception {
         when(ingestService.getPipeline(pipelineId1)).thenReturn(pipeline1);
         when(ingestService.getPipeline(pipelineId2)).thenReturn(pipeline2);
 
-        PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, null, pipelineConfig0);
+        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig0);
         CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
 
         CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@@ -573,15 +555,15 @@ public void testActualPipelineProcessorWithHandledFailure() throws Exception {
                 new TestProcessor(ingestDocument -> { ingestDocument.setFieldValue(key1, randomInt()); }),
                 new CompoundProcessor(
                     false,
-                    Collections.singletonList(new TestProcessor(ingestDocument -> { throw exception; })),
-                    Collections.singletonList(new TestProcessor(ingestDocument -> { ingestDocument.setFieldValue(key2, randomInt()); }))
+                    List.of(new TestProcessor(ingestDocument -> { throw exception; })),
+                    List.of(new TestProcessor(ingestDocument -> { ingestDocument.setFieldValue(key2, randomInt()); }))
                 ),
                 new TestProcessor(ingestDocument -> { ingestDocument.setFieldValue(key3, randomInt()); })
             )
         );
         when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline);
 
-        PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, null, pipelineConfig);
+        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig);
         CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
 
         CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@@ -642,7 +624,7 @@ public void testActualPipelineProcessorWithUnhandledFailure() throws Exception {
         );
         when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline);
 
-        PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, null, pipelineConfig);
+        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig);
         CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
 
         CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@@ -684,7 +666,7 @@ public void testActualPipelineProcessorWithCycle() throws Exception {
             null,
             null,
             null,
-            new CompoundProcessor(factory.create(Collections.emptyMap(), null, null, pipelineConfig2))
+            new CompoundProcessor(factory.create(Map.of(), null, null, pipelineConfig2))
         );
 
         Pipeline pipeline2 = new Pipeline(
@@ -692,13 +674,13 @@ public void testActualPipelineProcessorWithCycle() throws Exception {
             null,
             null,
             null,
-            new CompoundProcessor(factory.create(Collections.emptyMap(), null, null, pipelineConfig1))
+            new CompoundProcessor(factory.create(Map.of(), null, null, pipelineConfig1))
         );
 
         when(ingestService.getPipeline(pipelineId1)).thenReturn(pipeline1);
         when(ingestService.getPipeline(pipelineId2)).thenReturn(pipeline2);
 
-        PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, null, pipelineConfig0);
+        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig0);
         CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
 
         CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@@ -718,7 +700,7 @@ public void testActualPipelineProcessorRepeatedInvocation() throws Exception {
         PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService);
 
         String key1 = randomAlphaOfLength(10);
-        PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, null, pipelineConfig);
+        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig);
         Pipeline pipeline = new Pipeline(
             pipelineId,
             null,
diff --git a/server/src/test/java/org/elasticsearch/ingest/ValueSourceTests.java b/server/src/test/java/org/elasticsearch/ingest/ValueSourceTests.java
index f774bc3ed77e..84b7db730159 100644
--- a/server/src/test/java/org/elasticsearch/ingest/ValueSourceTests.java
+++ b/server/src/test/java/org/elasticsearch/ingest/ValueSourceTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -20,8 +19,8 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.sameInstance;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -34,7 +33,7 @@ public void testDeepCopy() {
         for (int i = 0; i < iterations; i++) {
             Map<String, Object> map = RandomDocumentPicks.randomSource(random());
             ValueSource valueSource = ValueSource.wrap(map, TestTemplateService.instance());
-            Object copy = valueSource.copyAndResolve(Collections.emptyMap());
+            Object copy = valueSource.copyAndResolve(Map.of());
             assertThat("iteration: " + i, copy, equalTo(map));
             assertThat("iteration: " + i, copy, not(sameInstance(map)));
         }
@@ -86,7 +85,7 @@ public void testScriptShouldCompile() {
         String compiledValue = randomAlphaOfLength(10);
         when(scriptService.compile(any(), any())).thenReturn(new TestTemplateService.MockTemplateScript.Factory(compiledValue));
         ValueSource result = ValueSource.wrap(propertyValue, scriptService);
-        assertThat(result.copyAndResolve(Collections.emptyMap()), equalTo(compiledValue));
+        assertThat(result.copyAndResolve(Map.of()), equalTo(compiledValue));
         verify(scriptService, times(1)).compile(any(), any());
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/lucene/util/BitSetsTests.java b/server/src/test/java/org/elasticsearch/lucene/util/BitSetsTests.java
new file mode 100644
index 000000000000..178963b8d5d8
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/lucene/util/BitSetsTests.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.lucene.util;
+
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.BitSet;
+import org.apache.lucene.util.BitSetIterator;
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class BitSetsTests extends ESTestCase {
+
+    public void testRandomBitset() throws Exception {
+        int maxDocs = randomIntBetween(1, 1024);
+        int numDocs = 0;
+        FixedBitSet matches = new FixedBitSet(maxDocs);
+        for (int i = 0; i < maxDocs; i++) {
+            if (numDocs < maxDocs && randomBoolean()) {
+                numDocs++;
+                matches.set(i);
+            }
+        }
+        DocIdSetIterator it = new BitSetIterator(matches, randomIntBetween(0, numDocs));
+        BitSet bitSet = BitSets.of(it, maxDocs);
+        assertThat(bitSet.cardinality(), equalTo(numDocs));
+        assertThat(bitSet.length(), equalTo(maxDocs));
+        for (int i = 0; i < maxDocs; i++) {
+            assertThat(bitSet.get(i), equalTo(matches.get(i)));
+            assertThat(bitSet.nextSetBit(i), equalTo(matches.nextSetBit(i)));
+            assertThat(bitSet.prevSetBit(i), equalTo(matches.prevSetBit(i)));
+        }
+    }
+
+    public void testMatchAllBitSet() throws Exception {
+        int maxDocs = randomIntBetween(1, 128);
+        FixedBitSet matches = new FixedBitSet(maxDocs);
+        for (int i = 0; i < maxDocs; i++) {
+            matches.set(i);
+        }
+        DocIdSetIterator it = new BitSetIterator(matches, randomNonNegativeLong());
+        BitSet bitSet = BitSets.of(it, maxDocs);
+        assertThat(bitSet, instanceOf(MatchAllBitSet.class));
+        for (int i = 0; i < maxDocs; i++) {
+            assertTrue(bitSet.get(i));
+            assertThat(bitSet.nextSetBit(i), equalTo(matches.nextSetBit(i)));
+            assertThat(bitSet.prevSetBit(i), equalTo(matches.prevSetBit(i)));
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/monitor/fs/FsHealthServiceTests.java b/server/src/test/java/org/elasticsearch/monitor/fs/FsHealthServiceTests.java
index b482e5c1bbc8..7ad53f37855a 100644
--- a/server/src/test/java/org/elasticsearch/monitor/fs/FsHealthServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/monitor/fs/FsHealthServiceTests.java
@@ -9,11 +9,8 @@
 package org.elasticsearch.monitor.fs;
 
 import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.apache.lucene.tests.mockfile.FilterFileChannel;
 import org.apache.lucene.tests.mockfile.FilterFileSystemProvider;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
@@ -131,11 +128,7 @@ public void testLoggingOnHungIO() throws Exception {
         final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
 
         MockLogAppender mockAppender = new MockLogAppender();
-        mockAppender.start();
-
-        Logger logger = LogManager.getLogger(FsHealthService.class);
-        Loggers.addAppender(logger, mockAppender);
-        try (NodeEnvironment env = newNodeEnvironment()) {
+        try (NodeEnvironment env = newNodeEnvironment(); var ignored = mockAppender.capturing(FsHealthService.class)) {
             FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env);
             int counter = 0;
             for (Path path : env.nodeDataPaths()) {
@@ -155,8 +148,6 @@ public void testLoggingOnHungIO() throws Exception {
             assertEquals(env.nodeDataPaths().length, disruptFileSystemProvider.getInjectedPathCount());
             assertBusy(mockAppender::assertAllExpectationsMatched);
         } finally {
-            Loggers.removeAppender(logger, mockAppender);
-            mockAppender.stop();
             PathUtilsForTesting.teardown();
             ThreadPool.terminate(testThreadPool, 500, TimeUnit.MILLISECONDS);
         }
diff --git a/server/src/test/java/org/elasticsearch/monitor/jvm/HotThreadsTests.java b/server/src/test/java/org/elasticsearch/monitor/jvm/HotThreadsTests.java
index 44898f5af05a..8fc530d45d38 100644
--- a/server/src/test/java/org/elasticsearch/monitor/jvm/HotThreadsTests.java
+++ b/server/src/test/java/org/elasticsearch/monitor/jvm/HotThreadsTests.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.monitor.jvm;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 import org.mockito.ArgumentMatchers;
@@ -46,7 +47,7 @@ public void testSupportedThreadsReportType() {
             try {
                 new HotThreads().type(HotThreads.ReportType.of(type));
             } catch (IllegalArgumentException e) {
-                fail(formatted("IllegalArgumentException called when creating HotThreads for supported type [%s]", type));
+                fail(Strings.format("IllegalArgumentException called when creating HotThreads for supported type [%s]", type));
             }
         }
     }
@@ -219,7 +220,7 @@ private ThreadInfo makeThreadInfoMocksHelper(ThreadMXBean mockedMXBean, String t
         when(mockedMXBean.getThreadCpuTime(threadId)).thenReturn(0L).thenReturn(threadId * cpuMultiplier);
         ThreadInfo mockedThreadInfo = mock(ThreadInfo.class);
         when(mockedMXBean.getThreadInfo(eq(threadId), anyInt())).thenReturn(mockedThreadInfo);
-        when(mockedThreadInfo.getThreadName()).thenReturn(String.format(Locale.ROOT, "%s %d", threadPrefix, threadId));
+        when(mockedThreadInfo.getThreadName()).thenReturn(Strings.format("%s %d", threadPrefix, threadId));
 
         // We create some variability for the blocked and waited times. Odd and even.
         when(mockedThreadInfo.getBlockedCount()).thenReturn(0L).thenReturn(threadId % 2);
@@ -234,7 +235,7 @@ private ThreadInfo makeThreadInfoMocksHelper(ThreadMXBean mockedMXBean, String t
 
         StackTraceElement[] stack = makeThreadStackHelper(
             List.of(
-                new String[] { "org.elasticsearch.monitor.test", formatted("method_%d", (threadId) % 2) },
+                new String[] { "org.elasticsearch.monitor.test", Strings.format("method_%d", (threadId) % 2) },
                 new String[] { "org.elasticsearch.monitor.testOther", "methodFinal" }
             )
         ).toArray(new StackTraceElement[0]);
diff --git a/server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java b/server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java
index a485e128e8fe..78ea94feebab 100644
--- a/server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java
+++ b/server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java
@@ -10,6 +10,7 @@
 
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.common.unit.Processors;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -17,7 +18,6 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import java.util.Locale;
 
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.anyOf;
@@ -48,7 +48,7 @@ List<String> readOsRelease() {
                 if (prettyName != null) {
                     final String quote = randomFrom("\"", "'", "");
                     final String space = randomFrom(" ", "");
-                    final String prettyNameLine = String.format(Locale.ROOT, "PRETTY_NAME=%s%s%s%s", quote, prettyName, quote, space);
+                    final String prettyNameLine = Strings.format("PRETTY_NAME=%s%s%s%s", quote, prettyName, quote, space);
                     return Arrays.asList("NAME=" + randomAlphaOfLength(16), prettyNameLine);
                 } else {
                     return Collections.singletonList("NAME=" + randomAlphaOfLength(16));
diff --git a/server/src/test/java/org/elasticsearch/node/NodeTests.java b/server/src/test/java/org/elasticsearch/node/NodeTests.java
index 0982442dc0d8..645cdf077596 100644
--- a/server/src/test/java/org/elasticsearch/node/NodeTests.java
+++ b/server/src/test/java/org/elasticsearch/node/NodeTests.java
@@ -55,6 +55,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.TimeUnit;
@@ -370,8 +371,8 @@ public static class MockRecoveryPlannerPlugin extends Plugin implements Recovery
         public MockRecoveryPlannerPlugin() {}
 
         @Override
-        public RecoveryPlannerService createRecoveryPlannerService(ShardSnapshotsService shardSnapshotsService) {
-            return mock(RecoveryPlannerService.class);
+        public Optional<RecoveryPlannerService> createRecoveryPlannerService(ShardSnapshotsService shardSnapshotsService) {
+            return Optional.of(mock(RecoveryPlannerService.class));
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskRequestTests.java
index 4b8120e7e6a8..70aa0504e8b0 100644
--- a/server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/persistent/CancelPersistentTaskRequestTests.java
@@ -20,6 +20,11 @@ protected Request createTestInstance() {
         return new Request(randomAsciiOfLength(10));
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/server/src/test/java/org/elasticsearch/persistent/CompletionPersistentTaskRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/CompletionPersistentTaskRequestTests.java
index dece61938576..9cae2f320f24 100644
--- a/server/src/test/java/org/elasticsearch/persistent/CompletionPersistentTaskRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/persistent/CompletionPersistentTaskRequestTests.java
@@ -22,6 +22,11 @@ protected Request createTestInstance() {
         }
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java
index 5e5d97a7bee2..77cac3e3d001 100644
--- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.NodesShutdownMetadata;
@@ -50,6 +51,7 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
@@ -527,7 +529,7 @@ public void testPeriodicRecheckOffMaster() {
         assertFalse(service.getPeriodicRechecker().isScheduled());
     }
 
-    public void testUnassignTask() {
+    public void testUnassignTask() throws InterruptedException {
         ClusterState clusterState = initialState();
         ClusterState.Builder builder = ClusterState.builder(clusterState);
         PersistentTasksCustomMetadata.Builder tasks = PersistentTasksCustomMetadata.builder(
@@ -545,14 +547,18 @@ public void testUnassignTask() {
         clusterState = builder.metadata(metadata).nodes(nodes).build();
         setState(clusterService, clusterState);
         PersistentTasksClusterService service = createService((params, candidateNodes, currentState) -> new Assignment("_node_2", "test"));
+        final var countDownLatch = new CountDownLatch(1);
         service.unassignPersistentTask(unassignedId, tasks.getLastAllocationId(), "unassignment test", ActionListener.wrap(task -> {
             assertThat(task.getAssignment().getExecutorNode(), is(nullValue()));
             assertThat(task.getId(), equalTo(unassignedId));
             assertThat(task.getAssignment().getExplanation(), equalTo("unassignment test"));
+            countDownLatch.countDown();
         }, e -> fail()));
+
+        assertTrue(countDownLatch.await(10, TimeUnit.SECONDS));
     }
 
-    public void testUnassignNonExistentTask() {
+    public void testUnassignNonExistentTask() throws InterruptedException {
         ClusterState clusterState = initialState();
         ClusterState.Builder builder = ClusterState.builder(clusterState);
         PersistentTasksCustomMetadata.Builder tasks = PersistentTasksCustomMetadata.builder(
@@ -568,12 +574,18 @@ public void testUnassignNonExistentTask() {
         clusterState = builder.metadata(metadata).nodes(nodes).build();
         setState(clusterService, clusterState);
         PersistentTasksClusterService service = createService((params, candidateNodes, currentState) -> new Assignment("_node_2", "test"));
+        final var countDownLatch = new CountDownLatch(1);
         service.unassignPersistentTask(
             "missing-task",
             tasks.getLastAllocationId(),
             "unassignment test",
-            ActionListener.wrap(task -> fail(), e -> assertThat(e, instanceOf(ResourceNotFoundException.class)))
+            ActionListener.wrap(task -> fail(), e -> {
+                assertThat(e, instanceOf(ResourceNotFoundException.class));
+                countDownLatch.countDown();
+            })
         );
+
+        assertTrue(countDownLatch.await(10, TimeUnit.SECONDS));
     }
 
     public void testTasksNotAssignedToShuttingDownNodes() {
@@ -851,7 +863,10 @@ private ClusterState significantChange(ClusterState clusterState) {
             // change routing table to simulate a change
             logger.info("changed routing table");
             Metadata.Builder metadata = Metadata.builder(clusterState.metadata());
-            RoutingTable.Builder routingTable = RoutingTable.builder(clusterState.routingTable());
+            RoutingTable.Builder routingTable = RoutingTable.builder(
+                TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
+                clusterState.routingTable()
+            );
             changeRoutingTable(metadata, routingTable);
             builder.metadata(metadata).routingTable(routingTable.build());
         }
@@ -899,7 +914,10 @@ private ClusterState insignificantChange(ClusterState clusterState) {
                 logger.info("changed routing table");
                 Metadata.Builder metadata = Metadata.builder(clusterState.metadata());
                 metadata.putCustom(PersistentTasksCustomMetadata.TYPE, tasksBuilder.build());
-                RoutingTable.Builder routingTable = RoutingTable.builder(clusterState.routingTable());
+                RoutingTable.Builder routingTable = RoutingTable.builder(
+                    TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
+                    clusterState.routingTable()
+                );
                 changeRoutingTable(metadata, routingTable);
                 builder.metadata(metadata).routingTable(routingTable.build());
                 return builder.build();
@@ -1006,7 +1024,7 @@ private DiscoveryNode newNode(String nodeId) {
 
     private ClusterState initialState() {
         Metadata.Builder metadata = Metadata.builder();
-        RoutingTable.Builder routingTable = RoutingTable.builder();
+        RoutingTable.Builder routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         int randomIndices = randomIntBetween(0, 5);
         for (int i = 0; i < randomIndices; i++) {
             changeRoutingTable(metadata, routingTable);
diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetadataTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetadataTests.java
index a143150f8ba3..a8b7a8937a4b 100644
--- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetadataTests.java
+++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetadataTests.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.persistent;
 
 import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
@@ -32,7 +33,7 @@
 import org.elasticsearch.persistent.TestPersistentTasksPlugin.State;
 import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestParams;
 import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor;
-import org.elasticsearch.test.SimpleDiffableSerializationTestCase;
+import org.elasticsearch.test.ChunkedToXContentDiffableSerializationTestCase;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
@@ -58,7 +59,7 @@
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.sameInstance;
 
-public class PersistentTasksCustomMetadataTests extends SimpleDiffableSerializationTestCase<Custom> {
+public class PersistentTasksCustomMetadataTests extends ChunkedToXContentDiffableSerializationTestCase<Custom> {
 
     @Override
     protected PersistentTasksCustomMetadata createTestInstance() {
@@ -75,6 +76,11 @@ protected PersistentTasksCustomMetadata createTestInstance() {
         return tasks.build();
     }
 
+    @Override
+    protected Custom mutateInstance(Custom instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Custom> instanceReader() {
         return PersistentTasksCustomMetadata::new;
@@ -174,7 +180,7 @@ public void testSerializationContext() throws Exception {
         );
 
         XContentType xContentType = randomFrom(XContentType.values());
-        BytesReference shuffled = toShuffledXContent(testInstance, xContentType, params, false);
+        BytesReference shuffled = toShuffledXContent(asXContent(testInstance), xContentType, params, false);
 
         PersistentTasksCustomMetadata newInstance;
         try (XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled)) {
@@ -260,7 +266,7 @@ public void testMinVersionSerialization() throws IOException {
             TestPersistentTasksExecutor.NAME,
             new TestParams(
                 null,
-                randomVersionBetween(random(), minVersion, streamVersion),
+                randomVersionBetween(random(), minVersion, streamVersion).transportVersion,
                 randomBoolean() ? Optional.empty() : Optional.of("test")
             ),
             randomAssignment()
@@ -270,7 +276,7 @@ public void testMinVersionSerialization() throws IOException {
             TestPersistentTasksExecutor.NAME,
             new TestParams(
                 null,
-                randomVersionBetween(random(), compatibleFutureVersion(streamVersion), Version.CURRENT),
+                randomVersionBetween(random(), compatibleFutureVersion(streamVersion), Version.CURRENT).transportVersion,
                 randomBoolean() ? Optional.empty() : Optional.of("test")
             ),
             randomAssignment()
@@ -380,8 +386,8 @@ public String getWriteableName() {
             }
 
             @Override
-            public Version getMinimalSupportedVersion() {
-                return Version.CURRENT;
+            public TransportVersion getMinimalSupportedVersion() {
+                return TransportVersion.CURRENT;
             }
         };
     }
diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java
index 4f691cf36ac2..0fec0a9bc8e5 100644
--- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorResponseTests.java
@@ -35,6 +35,11 @@ protected PersistentTaskResponse createTestInstance() {
         }
     }
 
+    @Override
+    protected PersistentTaskResponse mutateInstance(PersistentTaskResponse instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<PersistentTaskResponse> instanceReader() {
         return PersistentTaskResponse::new;
diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceStatusTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceStatusTests.java
index 859abe5309f5..9d136b324498 100644
--- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceStatusTests.java
+++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksNodeServiceStatusTests.java
@@ -20,6 +20,11 @@ protected Status createTestInstance() {
         return new Status(randomFrom(AllocatedPersistentTask.State.values()));
     }
 
+    @Override
+    protected Status mutateInstance(Status instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Status> instanceReader() {
         return Status::new;
diff --git a/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java
index bc9b25fa6170..92f6aa707e0c 100644
--- a/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/persistent/StartPersistentActionRequestTests.java
@@ -32,6 +32,11 @@ protected Request createTestInstance() {
         return new Request(UUIDs.base64UUID(), randomAlphaOfLengthBetween(1, 20), testParams);
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java
index 0d1d62e189d4..81c39dab03f7 100644
--- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java
+++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java
@@ -10,7 +10,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -127,7 +127,7 @@ public static class TestParams implements PersistentTaskParams {
             REQUEST_PARSER.declareString(constructorArg(), new ParseField("param"));
         }
 
-        private final Version minVersion;
+        private final TransportVersion minVersion;
         private final Optional<String> feature;
 
         private String executorNodeAttr = null;
@@ -141,10 +141,10 @@ public TestParams() {
         }
 
         public TestParams(String testParam) {
-            this(testParam, Version.CURRENT, Optional.empty());
+            this(testParam, TransportVersion.CURRENT, Optional.empty());
         }
 
-        public TestParams(String testParam, Version minVersion, Optional<String> feature) {
+        public TestParams(String testParam, TransportVersion minVersion, Optional<String> feature) {
             this.testParam = testParam;
             this.minVersion = minVersion;
             this.feature = feature;
@@ -154,7 +154,7 @@ public TestParams(StreamInput in) throws IOException {
             executorNodeAttr = in.readOptionalString();
             responseNode = in.readOptionalString();
             testParam = in.readOptionalString();
-            minVersion = Version.readVersion(in);
+            minVersion = TransportVersion.readVersion(in);
             feature = Optional.ofNullable(in.readOptionalString());
         }
 
@@ -184,7 +184,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalString(executorNodeAttr);
             out.writeOptionalString(responseNode);
             out.writeOptionalString(testParam);
-            Version.writeVersion(minVersion, out);
+            TransportVersion.writeVersion(minVersion, out);
             out.writeOptionalString(feature.orElse(null));
         }
 
@@ -216,7 +216,7 @@ public int hashCode() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
+        public TransportVersion getMinimalSupportedVersion() {
             return minVersion;
         }
 
diff --git a/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java b/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java
index ec173145ca0f..62da3023ef7d 100644
--- a/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/persistent/UpdatePersistentTaskRequestTests.java
@@ -24,6 +24,11 @@ protected Request createTestInstance() {
         return new Request(UUIDs.base64UUID(), randomLong(), new State(randomAlphaOfLength(10)));
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/server/src/test/java/org/elasticsearch/plugins/PluginIntrospectorTests.java b/server/src/test/java/org/elasticsearch/plugins/PluginIntrospectorTests.java
index c20fdeb98b18..55d47d303b72 100644
--- a/server/src/test/java/org/elasticsearch/plugins/PluginIntrospectorTests.java
+++ b/server/src/test/java/org/elasticsearch/plugins/PluginIntrospectorTests.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -151,8 +151,8 @@ public Map<String, DirectoryFactory> getDirectoryFactories() {
             }
 
             @Override
-            public RecoveryPlannerService createRecoveryPlannerService(ShardSnapshotsService shardSnapshotsService) {
-                return null;
+            public Optional<RecoveryPlannerService> createRecoveryPlannerService(ShardSnapshotsService shardSnapshotsService) {
+                return Optional.empty();
             }
 
             @Override
@@ -272,7 +272,7 @@ public Collection<Object> createComponents(
                 IndexNameExpressionResolver indexNameExpressionResolver,
                 Supplier<RepositoriesService> repositoriesServiceSupplier,
                 Tracer tracer,
-                AllocationDeciders allocationDeciders
+                AllocationService allocationService
             ) {
                 return null;
             }
diff --git a/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java b/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java
index d06410ff0ac2..b655a240759d 100644
--- a/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java
@@ -17,6 +17,8 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.plugin.analysis.CharFilterFactory;
+import org.elasticsearch.plugins.scanners.PluginInfo;
 import org.elasticsearch.plugins.spi.BarPlugin;
 import org.elasticsearch.plugins.spi.BarTestService;
 import org.elasticsearch.plugins.spi.TestService;
@@ -148,7 +150,7 @@ public void testStartupWithRemovingMarker() throws IOException {
             "false"
         );
         final IllegalStateException e = expectThrows(IllegalStateException.class, () -> newPluginsService(settings));
-        final String expected = formatted(
+        final String expected = Strings.format(
             "found file [%s] from a failed attempt to remove the plugin [fake]; execute [elasticsearch-plugin remove fake]",
             removing
         );
@@ -792,8 +794,8 @@ public void testStablePluginLoading() throws Exception {
         JarUtils.createJarWithEntries(jar, Map.of("p/A.class", InMemoryJavaCompiler.compile("p.A", """
             package p;
             import java.util.Map;
-            import org.elasticsearch.plugin.analysis.api.CharFilterFactory;
-            import org.elasticsearch.plugin.api.NamedComponent;
+            import org.elasticsearch.plugin.analysis.CharFilterFactory;
+            import org.elasticsearch.plugin.NamedComponent;
             import java.io.Reader;
             @NamedComponent( "a_name")
             public class A  implements CharFilterFactory {
@@ -806,7 +808,7 @@ public Reader create(Reader reader) {
         Path namedComponentFile = plugin.resolve("named_components.json");
         Files.writeString(namedComponentFile, """
             {
-              "org.elasticsearch.plugin.analysis.api.CharFilterFactory": {
+              "org.elasticsearch.plugin.analysis.CharFilterFactory": {
                 "a_name": "p.A"
               }
             }
@@ -823,6 +825,22 @@ public Reader create(Reader reader) {
             assertThat(pluginInfos.get(0).descriptor().getName(), equalTo("stable-plugin"));
             assertThat(pluginInfos.get(0).descriptor().isStable(), is(true));
 
+            // check ubermodule classloader usage
+            Collection<PluginInfo> stablePluginInfos = pluginService.getStablePluginRegistry()
+                .getPluginInfosForExtensible("org.elasticsearch.plugin.analysis.CharFilterFactory");
+            assertThat(stablePluginInfos, hasSize(1));
+            ClassLoader stablePluginClassLoader = stablePluginInfos.stream().findFirst().orElseThrow().loader();
+            assertThat(stablePluginClassLoader, instanceOf(UberModuleClassLoader.class));
+
+            if (CharFilterFactory.class.getModule().isNamed() == false) {
+                // test frameworks run with stable api classes on classpath, so we
+                // have no choice but to let our class read the unnamed module that
+                // owns the stable api classes
+                ((UberModuleClassLoader) stablePluginClassLoader).addReadsSystemClassLoaderUnnamedModule();
+            }
+
+            Class<?> stableClass = stablePluginClassLoader.loadClass("p.A");
+            assertThat(stableClass.getModule().getName(), equalTo("synthetic.stable.plugin"));
             // TODO should we add something to pluginInfos.get(0).pluginApiInfo() ?
         } finally {
             closePluginLoaders(pluginService);
@@ -838,13 +856,27 @@ public void testCanCreateAClassLoader() {
         assertEquals(this.getClass().getClassLoader(), loader.getParent());
     }
 
+    public void testToModuleName() {
+        assertThat(PluginsService.toModuleName("module.name"), equalTo("module.name"));
+        assertThat(PluginsService.toModuleName("module-name"), equalTo("module.name"));
+        assertThat(PluginsService.toModuleName("module-name1"), equalTo("module.name1"));
+        assertThat(PluginsService.toModuleName("1module-name"), equalTo("module.name"));
+        assertThat(PluginsService.toModuleName("module-name!"), equalTo("module.name"));
+        assertThat(PluginsService.toModuleName("module!@#name!"), equalTo("module.name"));
+        assertThat(PluginsService.toModuleName("!module-name!"), equalTo("module.name"));
+        assertThat(PluginsService.toModuleName("module_name"), equalTo("module_name"));
+        assertThat(PluginsService.toModuleName("-module-name-"), equalTo("module.name"));
+        assertThat(PluginsService.toModuleName("_module_name"), equalTo("_module_name"));
+        assertThat(PluginsService.toModuleName("_"), equalTo("_"));
+    }
+
     static final class Loader extends ClassLoader {
         Loader(ClassLoader parent) {
             super(parent);
         }
     }
 
-    // Closes the URLClassLoaders of plugins loaded by the given plugin service.
+    // Closes the URLClassLoaders and UberModuleClassloaders of plugins loaded by the given plugin service.
     static void closePluginLoaders(PluginsService pluginService) {
         for (var lp : pluginService.plugins()) {
             if (lp.loader()instanceof URLClassLoader urlClassLoader) {
@@ -854,6 +886,13 @@ static void closePluginLoaders(PluginsService pluginService) {
                     throw new UncheckedIOException(unexpected);
                 }
             }
+            if (lp.loader()instanceof UberModuleClassLoader loader) {
+                try {
+                    PrivilegedOperations.closeURLClassLoader(loader.getInternalLoader());
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+            }
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/plugins/PluginsUtilsTests.java b/server/src/test/java/org/elasticsearch/plugins/PluginsUtilsTests.java
index 72dc0806e7ba..755f8dcf482b 100644
--- a/server/src/test/java/org/elasticsearch/plugins/PluginsUtilsTests.java
+++ b/server/src/test/java/org/elasticsearch/plugins/PluginsUtilsTests.java
@@ -368,40 +368,35 @@ public void testJarHellSpiConflict() throws Exception {
         assertThat(e.getCause().getMessage(), containsString("DummyClass1"));
     }
 
-    public void testIncompatibleElasticsearchVersion() throws Exception {
-        PluginDescriptor info = new PluginDescriptor(
-            "my_plugin",
-            "desc",
-            "1.0",
-            Version.fromId(6000099),
-            "1.8",
-            "FakePlugin",
-            null,
-            Collections.emptyList(),
-            false,
-            false,
-            false,
-            false
+    public void testStableEarlierElasticsearchVersion() throws Exception {
+        PluginDescriptor info = getPluginDescriptorForVersion(Version.fromId(Version.CURRENT.id + 1), "1.8", true);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> PluginsUtils.verifyCompatibility(info));
+        assertThat(
+            e.getMessage(),
+            containsString(
+                "was built for Elasticsearch version "
+                    + Version.fromId(Version.CURRENT.id + 1)
+                    + " but earlier version "
+                    + Version.CURRENT
+                    + " is running"
+            )
         );
+    }
+
+    public void testStableIncompatibleElasticsearchVersion() throws Exception {
+        PluginDescriptor info = getPluginDescriptorForVersion(Version.fromId(6000099), "1.8", true);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> PluginsUtils.verifyCompatibility(info));
+        assertThat(e.getMessage(), containsString("was built for Elasticsearch major version 6"));
+    }
+
+    public void testIncompatibleElasticsearchVersion() throws Exception {
+        PluginDescriptor info = getPluginDescriptorForVersion(Version.fromId(6000099), "1.8", false);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> PluginsUtils.verifyCompatibility(info));
         assertThat(e.getMessage(), containsString("was built for Elasticsearch version 6.0.0"));
     }
 
     public void testIncompatibleJavaVersion() throws Exception {
-        PluginDescriptor info = new PluginDescriptor(
-            "my_plugin",
-            "desc",
-            "1.0",
-            Version.CURRENT,
-            "1000",
-            "FakePlugin",
-            null,
-            Collections.emptyList(),
-            false,
-            false,
-            false,
-            false
-        );
+        PluginDescriptor info = getPluginDescriptorForVersion(Version.CURRENT, "1000", false);
         IllegalStateException e = expectThrows(IllegalStateException.class, () -> PluginsUtils.verifyCompatibility(info));
         assertThat(e.getMessage(), containsString("my_plugin requires Java"));
     }
@@ -434,4 +429,22 @@ public void testFindPluginDirs() throws IOException {
         assertThat(PluginsUtils.findPluginDirs(plugins), containsInAnyOrder(fake));
     }
 
+    private static PluginDescriptor getPluginDescriptorForVersion(Version id, String javaVersion, boolean isStable) {
+        PluginDescriptor info = new PluginDescriptor(
+            "my_plugin",
+            "desc",
+            "1.0",
+            id,
+            javaVersion,
+            "FakePlugin",
+            null,
+            Collections.emptyList(),
+            false,
+            false,
+            false,
+            isStable
+        );
+        return info;
+    }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/plugins/UberModuleClassLoaderTests.java b/server/src/test/java/org/elasticsearch/plugins/UberModuleClassLoaderTests.java
index 45544a471584..6359b9425c93 100644
--- a/server/src/test/java/org/elasticsearch/plugins/UberModuleClassLoaderTests.java
+++ b/server/src/test/java/org/elasticsearch/plugins/UberModuleClassLoaderTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.plugins;
 
+import org.apache.lucene.util.Constants;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.compiler.InMemoryJavaCompiler;
@@ -465,8 +466,9 @@ public static String demo() {
         JarUtils.createJarWithEntries(jar, jarEntries);
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/91609")
     public void testServiceLoadingWithOptionalDependencies() throws Exception {
-
+        assumeFalse("Tests frequently fail on Windows", Constants.WINDOWS);
         try (UberModuleClassLoader loader = getServiceTestLoader(true)) {
 
             Class<?> serviceCallerClass = loader.loadClass("q.caller.ServiceCaller");
@@ -483,8 +485,9 @@ public void testServiceLoadingWithOptionalDependencies() throws Exception {
         }
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/91609")
     public void testServiceLoadingWithoutOptionalDependencies() throws Exception {
-
+        assumeFalse("Tests frequently fail on Windows", Constants.WINDOWS);
         try (UberModuleClassLoader loader = getServiceTestLoader(false)) {
 
             Class<?> serviceCallerClass = loader.loadClass("q.caller.ServiceCaller");
diff --git a/server/src/test/java/org/elasticsearch/readiness/ReadinessServiceTests.java b/server/src/test/java/org/elasticsearch/readiness/ReadinessServiceTests.java
index 9c9167a8d546..b22eda974ea9 100644
--- a/server/src/test/java/org/elasticsearch/readiness/ReadinessServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/readiness/ReadinessServiceTests.java
@@ -239,6 +239,7 @@ public void testStatusChange() throws Exception {
             .build();
         ClusterChangedEvent event = new ClusterChangedEvent("test", newState, previousState);
         readinessService.clusterChanged(event);
+        readinessService.settingsChanged();
 
         // sending a cluster state with active master should bring up the service
         assertTrue(readinessService.ready());
diff --git a/server/src/test/java/org/elasticsearch/repositories/IndexIdTests.java b/server/src/test/java/org/elasticsearch/repositories/IndexIdTests.java
index 39e3d3cbee8e..580fdaf6f7f7 100644
--- a/server/src/test/java/org/elasticsearch/repositories/IndexIdTests.java
+++ b/server/src/test/java/org/elasticsearch/repositories/IndexIdTests.java
@@ -10,7 +10,8 @@
 
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -22,36 +23,25 @@
 /**
  * Tests for the {@link IndexId} class.
  */
-public class IndexIdTests extends ESTestCase {
+public class IndexIdTests extends AbstractWireSerializingTestCase<IndexId> {
 
-    public void testEqualsAndHashCode() {
-        // assert equals and hashcode
-        String name = randomAlphaOfLength(8);
-        String id = UUIDs.randomBase64UUID();
-        IndexId indexId1 = new IndexId(name, id);
-        IndexId indexId2 = new IndexId(name, id);
-        assertEquals(indexId1, indexId2);
-        assertEquals(indexId1.hashCode(), indexId2.hashCode());
-        // assert equals when using index name for id
-        id = name;
-        indexId1 = new IndexId(name, id);
-        indexId2 = new IndexId(name, id);
-        assertEquals(indexId1, indexId2);
-        assertEquals(indexId1.hashCode(), indexId2.hashCode());
-        // assert not equals when name or id differ
-        indexId2 = new IndexId(randomAlphaOfLength(8), id);
-        assertNotEquals(indexId1, indexId2);
-        assertNotEquals(indexId1.hashCode(), indexId2.hashCode());
-        indexId2 = new IndexId(name, UUIDs.randomBase64UUID());
-        assertNotEquals(indexId1, indexId2);
-        assertNotEquals(indexId1.hashCode(), indexId2.hashCode());
+    @Override
+    protected Writeable.Reader<IndexId> instanceReader() {
+        return IndexId::new;
     }
 
-    public void testSerialization() throws IOException {
-        IndexId indexId = new IndexId(randomAlphaOfLength(8), UUIDs.randomBase64UUID());
-        BytesStreamOutput out = new BytesStreamOutput();
-        indexId.writeTo(out);
-        assertEquals(indexId, new IndexId(out.bytes().streamInput()));
+    @Override
+    protected IndexId createTestInstance() {
+        return new IndexId(randomIdentifier(), UUIDs.randomBase64UUID());
+    }
+
+    @Override
+    protected IndexId mutateInstance(IndexId instance) {
+        return switch (randomInt(1)) {
+            case 0 -> new IndexId(randomValueOtherThan(instance.getName(), ESTestCase::randomIdentifier), instance.getId());
+            case 1 -> new IndexId(instance.getName(), randomValueOtherThan(instance.getId(), UUIDs::randomBase64UUID));
+            default -> throw new RuntimeException("unreachable");
+        };
     }
 
     public void testXContent() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/ShardSnapshotTaskRunnerTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/ShardSnapshotTaskRunnerTests.java
index 5cfd872a7b1b..f0aa256ff031 100644
--- a/server/src/test/java/org/elasticsearch/repositories/blobstore/ShardSnapshotTaskRunnerTests.java
+++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/ShardSnapshotTaskRunnerTests.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.store.ByteBuffersDirectory;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.GroupedActionListener;
+import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
@@ -30,7 +30,6 @@
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 
-import java.util.Collections;
 import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -69,16 +68,10 @@ public void setTaskRunner(ShardSnapshotTaskRunner taskRunner) {
 
         public void snapshotShard(SnapshotShardContext context) {
             int filesToUpload = randomIntBetween(0, 10);
-            if (filesToUpload == 0) {
-                finishedShardSnapshots.incrementAndGet();
-            } else {
-                expectedFileSnapshotTasks.addAndGet(filesToUpload);
-                ActionListener<Void> uploadListener = new GroupedActionListener<>(
-                    ActionListener.wrap(finishedShardSnapshots::incrementAndGet),
-                    filesToUpload
-                );
+            expectedFileSnapshotTasks.addAndGet(filesToUpload);
+            try (var refs = new RefCountingRunnable(finishedShardSnapshots::incrementAndGet)) {
                 for (int i = 0; i < filesToUpload; i++) {
-                    taskRunner.enqueueFileSnapshot(context, ShardSnapshotTaskRunnerTests::dummyFileInfo, uploadListener);
+                    taskRunner.enqueueFileSnapshot(context, ShardSnapshotTaskRunnerTests::dummyFileInfo, refs.acquireListener());
                 }
             }
             finishedShardSnapshotTasks.incrementAndGet();
@@ -137,7 +130,6 @@ public static SnapshotShardContext dummyContext(final SnapshotId snapshotId, fin
             null,
             IndexShardSnapshotStatus.newInitializing(null),
             Version.CURRENT,
-            Collections.emptyMap(),
             startTime,
             ActionListener.noop()
         );
diff --git a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java
index c76b131a4cb1..b990dd44330c 100644
--- a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java
+++ b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java
@@ -60,7 +60,6 @@
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 
@@ -112,7 +111,6 @@ public void testSnapshotAndRestore() throws IOException {
                     null,
                     snapshotStatus,
                     Version.CURRENT,
-                    Collections.emptyMap(),
                     randomMillisUpToYear9999(),
                     snapshot1Future
                 )
@@ -127,7 +125,8 @@ public void testSnapshotAndRestore() throws IOException {
                 shardId,
                 true,
                 new RecoverySource.SnapshotRecoverySource("test", new Snapshot("foo", snapshotId), Version.CURRENT, indexId),
-                new UnassignedInfo(UnassignedInfo.Reason.EXISTING_INDEX_RESTORED, "")
+                new UnassignedInfo(UnassignedInfo.Reason.EXISTING_INDEX_RESTORED, ""),
+                ShardRouting.Role.DEFAULT
             );
             routing = ShardRoutingHelper.initialize(routing, localNode.getId(), 0);
             RecoveryState state = new RecoveryState(routing, localNode, null);
@@ -155,7 +154,6 @@ public void testSnapshotAndRestore() throws IOException {
                     null,
                     snapshotStatus2,
                     Version.CURRENT,
-                    Collections.emptyMap(),
                     randomMillisUpToYear9999(),
                     snapshot2future
                 )
diff --git a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java
index 4c5b2ae78203..66fee30cc466 100644
--- a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java
@@ -8,14 +8,7 @@
 
 package org.elasticsearch.reservedstate.service;
 
-import org.elasticsearch.Build;
 import org.elasticsearch.Version;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
-import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
-import org.elasticsearch.client.internal.ClusterAdminClient;
-import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -25,12 +18,11 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.ingest.IngestInfo;
-import org.elasticsearch.ingest.ProcessorInfo;
 import org.elasticsearch.reservedstate.action.ReservedClusterSettingsAction;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xcontent.XContentParser;
 import org.junit.After;
 import org.junit.Before;
 import org.mockito.Mockito;
@@ -52,16 +44,11 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
 
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.emptySet;
-import static org.hamcrest.Matchers.allOf;
-import static org.hamcrest.Matchers.hasToString;
-import static org.hamcrest.Matchers.instanceOf;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyInt;
-import static org.mockito.Mockito.clearInvocations;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
@@ -75,9 +62,6 @@ public class FileSettingsServiceTests extends ESTestCase {
     private FileSettingsService fileSettingsService;
     private ReservedClusterStateService controller;
     private ThreadPool threadpool;
-    private NodeClient nodeClient;
-    private ClusterAdminClient clusterAdminClient;
-    private NodeInfo nodeInfo;
 
     @Before
     @SuppressWarnings("unchecked")
@@ -109,42 +93,7 @@ public void setUp() throws Exception {
         ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
 
         controller = new ReservedClusterStateService(clusterService, List.of(new ReservedClusterSettingsAction(clusterSettings)));
-
-        DiscoveryNode discoveryNode = new DiscoveryNode(
-            "_node_id",
-            buildNewFakeTransportAddress(),
-            emptyMap(),
-            emptySet(),
-            Version.CURRENT
-        );
-
-        nodeInfo = new NodeInfo(
-            Version.CURRENT,
-            Build.CURRENT,
-            discoveryNode,
-            Settings.EMPTY,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            new IngestInfo(Collections.singletonList(new ProcessorInfo("set"))),
-            null,
-            null
-        );
-        NodesInfoResponse response = new NodesInfoResponse(new ClusterName("elasticsearch"), List.of(nodeInfo), List.of());
-
-        clusterAdminClient = mock(ClusterAdminClient.class);
-        doAnswer(i -> {
-            ((ActionListener<NodesInfoResponse>) i.getArgument(1)).onResponse(response);
-            return null;
-        }).when(clusterAdminClient).nodesInfo(any(), any());
-
-        nodeClient = mock(NodeClient.class);
-        fileSettingsService = spy(new FileSettingsService(clusterService, controller, env, nodeClient));
-        doAnswer(i -> clusterAdminClient).when(fileSettingsService).clusterAdminClient();
+        fileSettingsService = spy(new FileSettingsService(clusterService, controller, env));
     }
 
     @After
@@ -188,6 +137,7 @@ public void testWatchedFile() throws Exception {
 
     public void testStartStop() {
         fileSettingsService.start();
+        fileSettingsService.startWatcher(clusterService.state());
         assertTrue(fileSettingsService.watching());
         fileSettingsService.stop();
         assertFalse(fileSettingsService.watching());
@@ -204,6 +154,7 @@ public void testCallsProcessing() throws Exception {
         }).when(service).processFileSettings(any());
 
         service.start();
+        service.startWatcher(clusterService.state());
         assertTrue(service.watching());
 
         Files.createDirectories(service.operatorSettingsDir());
@@ -214,8 +165,8 @@ public void testCallsProcessing() throws Exception {
         // on Linux is instantaneous. Windows is instantaneous too.
         processFileLatch.await(30, TimeUnit.SECONDS);
 
-        verify(service, Mockito.atLeast(1)).watchedFileChanged(any());
-        verify(service, times(1)).processFileSettings(any());
+        verify(service, Mockito.atLeast(1)).processSettingsAndNotifyListeners();
+        verify(service, Mockito.atLeast(1)).processFileSettings(any());
 
         service.stop();
         assertFalse(service.watching());
@@ -223,50 +174,81 @@ public void testCallsProcessing() throws Exception {
     }
 
     @SuppressWarnings("unchecked")
-    public void testInitialFile() throws Exception {
+    public void testInitialFileError() throws Exception {
         ReservedClusterStateService stateService = mock(ReservedClusterStateService.class);
 
         doAnswer((Answer<Void>) invocation -> {
             ((Consumer<Exception>) invocation.getArgument(2)).accept(new IllegalStateException("Some exception"));
             return null;
-        }).when(stateService).process(any(), (ReservedStateChunk) any(), any());
+        }).when(stateService).process(any(), (XContentParser) any(), any());
 
-        FileSettingsService service = spy(new FileSettingsService(clusterService, stateService, env, nodeClient));
-        doAnswer(i -> clusterAdminClient).when(service).clusterAdminClient();
+        AtomicBoolean settingsChanged = new AtomicBoolean(false);
+        CountDownLatch latch = new CountDownLatch(1);
 
-        Files.createDirectories(service.operatorSettingsDir());
+        final FileSettingsService service = spy(new FileSettingsService(clusterService, stateService, env));
+
+        service.addFileSettingsChangedListener(() -> settingsChanged.set(true));
+
+        doAnswer((Answer<Void>) invocation -> {
+            invocation.callRealMethod();
+            latch.countDown();
+            return null;
+        }).when(service).processSettingsAndNotifyListeners();
 
+        Files.createDirectories(service.operatorSettingsDir());
         // contents of the JSON don't matter, we just need a file to exist
         writeTestFile(service.operatorSettingsFile(), "{}");
 
-        Exception startupException = expectThrows(IllegalStateException.class, () -> service.start());
-        assertThat(
-            startupException.getCause(),
-            allOf(
-                instanceOf(FileSettingsService.FileSettingsStartupException.class),
-                hasToString(
-                    "org.elasticsearch.reservedstate.service.FileSettingsService$FileSettingsStartupException: "
-                        + "Error applying operator settings"
-                )
-            )
-        );
+        service.start();
+        service.startWatcher(clusterService.state());
+
+        // wait until the watcher thread has started, and it has discovered the file
+        assertTrue(latch.await(20, TimeUnit.SECONDS));
 
         verify(service, times(1)).processFileSettings(any());
+        // assert we never notified any listeners of successful application of file based settings
+        assertFalse(settingsChanged.get());
 
         service.stop();
+        service.close();
+    }
 
-        clearInvocations(service);
+    @SuppressWarnings("unchecked")
+    public void testInitialFileWorks() throws Exception {
+        ReservedClusterStateService stateService = mock(ReservedClusterStateService.class);
 
         // Let's check that if we didn't throw an error that everything works
         doAnswer((Answer<Void>) invocation -> {
             ((Consumer<Exception>) invocation.getArgument(2)).accept(null);
             return null;
-        }).when(stateService).process(any(), (ReservedStateChunk) any(), any());
+        }).when(stateService).process(any(), (XContentParser) any(), any());
+
+        AtomicBoolean settingsChanged = new AtomicBoolean(false);
+        CountDownLatch latch = new CountDownLatch(1);
+
+        final FileSettingsService service = spy(new FileSettingsService(clusterService, stateService, env));
+
+        service.addFileSettingsChangedListener(() -> settingsChanged.set(true));
+
+        doAnswer((Answer<Void>) invocation -> {
+            invocation.callRealMethod();
+            latch.countDown();
+            return null;
+        }).when(service).processSettingsAndNotifyListeners();
+
+        Files.createDirectories(service.operatorSettingsDir());
+        // contents of the JSON don't matter, we just need a file to exist
+        writeTestFile(service.operatorSettingsFile(), "{}");
 
         service.start();
-        service.startWatcher(clusterService.state(), true);
+        service.startWatcher(clusterService.state());
+
+        // wait until the watcher thread has started, and it has discovered the file
+        assertTrue(latch.await(20, TimeUnit.SECONDS));
 
         verify(service, times(1)).processFileSettings(any());
+        // assert we notified the listeners the file settings have changed, they were successfully applied
+        assertTrue(settingsChanged.get());
 
         service.stop();
         service.close();
@@ -275,9 +257,8 @@ public void testInitialFile() throws Exception {
     @SuppressWarnings("unchecked")
     public void testStopWorksInMiddleOfProcessing() throws Exception {
         var spiedController = spy(controller);
-        var fsService = new FileSettingsService(clusterService, spiedController, env, nodeClient);
+        var fsService = new FileSettingsService(clusterService, spiedController, env);
         FileSettingsService service = spy(fsService);
-        doAnswer(i -> clusterAdminClient).when(service).clusterAdminClient();
 
         CountDownLatch processFileLatch = new CountDownLatch(1);
         CountDownLatch deadThreadLatch = new CountDownLatch(1);
@@ -297,6 +278,7 @@ public void testStopWorksInMiddleOfProcessing() throws Exception {
         }).when(spiedController).parse(any(String.class), any());
 
         service.start();
+        service.startWatcher(clusterService.state());
         assertTrue(service.watching());
 
         Files.createDirectories(service.operatorSettingsDir());
@@ -319,10 +301,8 @@ public void testStopWorksInMiddleOfProcessing() throws Exception {
     @SuppressWarnings("unchecked")
     public void testStopWorksIfProcessingDidntReturnYet() throws Exception {
         var spiedController = spy(controller);
-        var fsService = new FileSettingsService(clusterService, spiedController, env, nodeClient);
+        var service = new FileSettingsService(clusterService, spiedController, env);
 
-        FileSettingsService service = spy(fsService);
-        doAnswer(i -> clusterAdminClient).when(service).clusterAdminClient();
         CountDownLatch processFileLatch = new CountDownLatch(1);
         CountDownLatch deadThreadLatch = new CountDownLatch(1);
 
@@ -343,6 +323,7 @@ public void testStopWorksIfProcessingDidntReturnYet() throws Exception {
         }).when(spiedController).parse(any(String.class), any());
 
         service.start();
+        service.startWatcher(clusterService.state());
         assertTrue(service.watching());
 
         Files.createDirectories(service.operatorSettingsDir());
@@ -362,120 +343,6 @@ public void testStopWorksIfProcessingDidntReturnYet() throws Exception {
         deadThreadLatch.countDown();
     }
 
-    @SuppressWarnings("unchecked")
-    public void testNodeInfosRefresh() throws Exception {
-        var spiedController = spy(controller);
-        var csAdminClient = spy(clusterAdminClient);
-        var response = new NodesInfoResponse(new ClusterName("elasticsearch"), List.of(nodeInfo), List.of());
-
-        doAnswer(i -> {
-            ((ActionListener<NodesInfoResponse>) i.getArgument(1)).onResponse(response);
-            return null;
-        }).when(csAdminClient).nodesInfo(any(), any());
-
-        var service = spy(new FileSettingsService(clusterService, spiedController, env, nodeClient));
-        doAnswer(i -> csAdminClient).when(service).clusterAdminClient();
-
-        doAnswer(
-            (Answer<ReservedStateChunk>) invocation -> new ReservedStateChunk(
-                Collections.emptyMap(),
-                new ReservedStateVersion(1L, Version.CURRENT)
-            )
-        ).when(spiedController).parse(any(String.class), any());
-
-        Files.createDirectories(service.operatorSettingsDir());
-        // Make some fake settings file to cause the file settings service to process it
-        writeTestFile(service.operatorSettingsFile(), "{}");
-
-        clearInvocations(csAdminClient);
-        clearInvocations(spiedController);
-
-        // we haven't fetched the node infos ever, since we haven't done any file processing
-        assertNull(service.nodeInfos());
-
-        // call the processing twice
-        service.processFileSettings(service.operatorSettingsFile()).whenComplete((o, e) -> {
-            if (e != null) {
-                fail("shouldn't get an exception");
-            }
-        });
-        // after the first processing we should have node infos
-        assertEquals(1, service.nodeInfos().getNodes().size());
-
-        service.processFileSettings(service.operatorSettingsFile()).whenComplete((o, e) -> {
-            if (e != null) {
-                fail("shouldn't get an exception");
-            }
-        });
-
-        // node infos should have been fetched only once
-        verify(csAdminClient, times(1)).nodesInfo(any(), any());
-        verify(spiedController, times(2)).process(any(), any(ReservedStateChunk.class), any());
-
-        // pretend we added a new node
-
-        final DiscoveryNode localNode = new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT);
-
-        NodeInfo localNodeInfo = new NodeInfo(
-            Version.CURRENT,
-            Build.CURRENT,
-            localNode,
-            Settings.EMPTY,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            new IngestInfo(Collections.singletonList(new ProcessorInfo("set"))),
-            null,
-            null
-        );
-        var newResponse = new NodesInfoResponse(new ClusterName("elasticsearch"), List.of(nodeInfo, localNodeInfo), List.of());
-
-        final ClusterState prevState = clusterService.state();
-        final ClusterState clusterState = ClusterState.builder(prevState)
-            .nodes(
-                DiscoveryNodes.builder(prevState.getNodes()).add(localNode).localNodeId(localNode.getId()).masterNodeId(localNode.getId())
-            )
-            .build();
-
-        ClusterChangedEvent event = new ClusterChangedEvent("transport", clusterState, prevState);
-        assertTrue(event.nodesChanged());
-        service.clusterChanged(event);
-
-        doAnswer(i -> {
-            ((ActionListener<NodesInfoResponse>) i.getArgument(1)).onResponse(newResponse);
-            return null;
-        }).when(csAdminClient).nodesInfo(any(), any());
-
-        // this wouldn't change yet, node fetch transport action is invoked on demand, when we need to process file changes,
-        // not every time we update the cluster state
-        assertEquals(1, service.nodeInfos().getNodes().size());
-
-        // call the processing twice
-        service.processFileSettings(service.operatorSettingsFile()).whenComplete((o, e) -> {
-            if (e != null) {
-                fail("shouldn't get an exception");
-            }
-        });
-
-        assertEquals(2, service.nodeInfos().getNodes().size());
-
-        service.processFileSettings(service.operatorSettingsFile()).whenComplete((o, e) -> {
-            if (e != null) {
-                fail("shouldn't get an exception");
-            }
-        });
-
-        assertEquals(2, service.nodeInfos().getNodes().size());
-
-        // node infos should have been fetched one more time
-        verify(csAdminClient, times(2)).nodesInfo(any(), any());
-        verify(spiedController, times(4)).process(any(), any(ReservedStateChunk.class), any());
-    }
-
     public void testRegisterWatchKeyRetry() throws IOException, InterruptedException {
         var service = spy(fileSettingsService);
         doAnswer(i -> 0L).when(service).retryDelayMillis(anyInt());
diff --git a/server/src/test/java/org/elasticsearch/rest/ChunkedRestResponseBodyTests.java b/server/src/test/java/org/elasticsearch/rest/ChunkedRestResponseBodyTests.java
index 72bd2c65fc6b..de26b8fc88ae 100644
--- a/server/src/test/java/org/elasticsearch/rest/ChunkedRestResponseBodyTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/ChunkedRestResponseBodyTests.java
@@ -30,16 +30,18 @@
 public class ChunkedRestResponseBodyTests extends ESTestCase {
 
     public void testEncodesChunkedXContentCorrectly() throws IOException {
-        final ChunkedToXContent chunkedToXContent = () -> Iterators.forArray(new ToXContent[] { (b, p) -> b.startObject(), (b, p) -> {
-            if (randomBoolean()) {
-                b.flush();
-            }
-            b.mapContents(Map.of("foo", "bar", "some_other_key", "some_other_value"));
-            return b;
-        }, (b, p) -> b.stringListField("list_field", List.of("string", "otherString")).endObject() });
+        final ChunkedToXContent chunkedToXContent = (ToXContent.Params outerParams) -> Iterators.forArray(
+            new ToXContent[] { (b, p) -> b.startObject(), (b, p) -> {
+                if (randomBoolean()) {
+                    b.flush();
+                }
+                b.mapContents(Map.of("foo", "bar", "some_other_key", "some_other_value"));
+                return b;
+            }, (b, p) -> b.stringListField("list_field", List.of("string", "otherString")).endObject() }
+        );
         final XContent randomXContent = randomFrom(XContentType.values()).xContent();
         final XContentBuilder builderDirect = XContentBuilder.builder(randomXContent);
-        var iter = chunkedToXContent.toXContentChunked();
+        var iter = chunkedToXContent.toXContentChunked(ToXContent.EMPTY_PARAMS);
         while (iter.hasNext()) {
             iter.next().toXContent(builderDirect, ToXContent.EMPTY_PARAMS);
         }
diff --git a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
index f585cf56b24b..8b70c920a217 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
@@ -938,6 +938,21 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c
         );
     }
 
+    public void testRegisterWithReservedPath() {
+        final RestController restController = new RestController(Set.of(), null, client, circuitBreakerService, usageService, tracer);
+        for (String path : RestController.RESERVED_PATHS) {
+            IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> {
+                restController.registerHandler(
+                    new Route(randomFrom(RestRequest.Method.values()), path),
+                    (request, channel, client) -> channel.sendResponse(
+                        new RestResponse(RestStatus.OK, RestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)
+                    )
+                );
+            });
+            assertThat(iae.getMessage(), containsString("path [" + path + "] is a reserved path and may not be registered"));
+        }
+    }
+
     private static final class TestHttpServerTransport extends AbstractLifecycleComponent implements HttpServerTransport {
 
         TestHttpServerTransport() {}
diff --git a/server/src/test/java/org/elasticsearch/rest/RestResponseTests.java b/server/src/test/java/org/elasticsearch/rest/RestResponseTests.java
index 57b456642650..1224a1071324 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestResponseTests.java
@@ -19,11 +19,13 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
 import org.elasticsearch.transport.RemoteTransportException;
+import org.elasticsearch.xcontent.MediaType;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
@@ -31,11 +33,13 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.ElasticsearchExceptionTests.assertDeepEquals;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 
@@ -357,6 +361,21 @@ public void testNoErrorFromXContent() throws IOException {
         assertEquals("Failed to parse elasticsearch status exception: no exception was found", e.getMessage());
     }
 
+    public void testResponseContentTypeUponException() throws Exception {
+        String mediaType = XContentType.VND_JSON.toParsedMediaType()
+            .responseContentTypeHeader(
+                Map.of(MediaType.COMPATIBLE_WITH_PARAMETER_NAME, String.valueOf(RestApiVersion.minimumSupported().major))
+            );
+        List<String> mediaTypeList = Collections.singletonList(mediaType);
+
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(Map.of("Accept", mediaTypeList)).build();
+        RestChannel channel = new SimpleExceptionRestChannel(request);
+
+        Exception t = new ElasticsearchException("an error occurred reading data", new FileNotFoundException("/foo/bar"));
+        RestResponse response = new RestResponse(channel, t);
+        assertThat(response.contentType(), equalTo(mediaType));
+    }
+
     public static class WithHeadersException extends ElasticsearchException {
 
         WithHeadersException() {
diff --git a/server/src/test/java/org/elasticsearch/rest/RestUtilsTests.java b/server/src/test/java/org/elasticsearch/rest/RestUtilsTests.java
index 70f8327448fa..85fd5ae92103 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestUtilsTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestUtilsTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.rest;
 
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
@@ -34,7 +35,7 @@ public void testDecodeQueryString() {
         assertThat(params.get("test"), equalTo("value"));
 
         params.clear();
-        uri = formatted("something?test=value%ctest1=value1", randomDelimiter());
+        uri = Strings.format("something?test=value%ctest1=value1", randomDelimiter());
         RestUtils.decodeQueryString(uri, uri.indexOf('?') + 1, params);
         assertThat(params.size(), equalTo(2));
         assertThat(params.get("test"), equalTo("value"));
@@ -59,12 +60,12 @@ public void testDecodeQueryStringEdgeCases() {
         assertThat(params.size(), equalTo(0));
 
         params.clear();
-        uri = formatted("something?%c", randomDelimiter());
+        uri = Strings.format("something?%c", randomDelimiter());
         RestUtils.decodeQueryString(uri, uri.indexOf('?') + 1, params);
         assertThat(params.size(), equalTo(0));
 
         params.clear();
-        uri = String.format(Locale.ROOT, "something?p=v%c%cp1=v1", randomDelimiter(), randomDelimiter());
+        uri = Strings.format("something?p=v%c%cp1=v1", randomDelimiter(), randomDelimiter());
         RestUtils.decodeQueryString(uri, uri.indexOf('?') + 1, params);
         assertThat(params.size(), equalTo(2));
         assertThat(params.get("p"), equalTo("v"));
@@ -76,7 +77,7 @@ public void testDecodeQueryStringEdgeCases() {
         assertThat(params.size(), equalTo(0));
 
         params.clear();
-        uri = formatted("something?%c=", randomDelimiter());
+        uri = Strings.format("something?%c=", randomDelimiter());
         RestUtils.decodeQueryString(uri, uri.indexOf('?') + 1, params);
         assertThat(params.size(), equalTo(0));
 
@@ -87,14 +88,14 @@ public void testDecodeQueryStringEdgeCases() {
         assertThat(params.get("a"), equalTo(""));
 
         params.clear();
-        uri = formatted("something?p=v%ca", randomDelimiter());
+        uri = Strings.format("something?p=v%ca", randomDelimiter());
         RestUtils.decodeQueryString(uri, uri.indexOf('?') + 1, params);
         assertThat(params.size(), equalTo(2));
         assertThat(params.get("a"), equalTo(""));
         assertThat(params.get("p"), equalTo("v"));
 
         params.clear();
-        uri = String.format(Locale.ROOT, "something?p=v%ca%cp1=v1", randomDelimiter(), randomDelimiter());
+        uri = Strings.format("something?p=v%ca%cp1=v1", randomDelimiter(), randomDelimiter());
         RestUtils.decodeQueryString(uri, uri.indexOf('?') + 1, params);
         assertThat(params.size(), equalTo(3));
         assertThat(params.get("a"), equalTo(""));
@@ -102,7 +103,7 @@ public void testDecodeQueryStringEdgeCases() {
         assertThat(params.get("p1"), equalTo("v1"));
 
         params.clear();
-        uri = String.format(Locale.ROOT, "something?p=v%ca%cb%cp1=v1", randomDelimiter(), randomDelimiter(), randomDelimiter());
+        uri = Strings.format("something?p=v%ca%cb%cp1=v1", randomDelimiter(), randomDelimiter(), randomDelimiter());
         RestUtils.decodeQueryString(uri, uri.indexOf('?') + 1, params);
         assertThat(params.size(), equalTo(4));
         assertThat(params.get("a"), equalTo(""));
@@ -167,7 +168,7 @@ private void assertCorsSettingRegexMatches(String settingsValue, boolean expectM
         Pattern pattern = RestUtils.checkCorsSettingForRegex(settingsValue);
         for (String candidate : candidates) {
             assertThat(
-                String.format(Locale.ROOT, "Expected pattern %s to match against %s: %s", settingsValue, candidate, expectMatch),
+                Strings.format("Expected pattern %s to match against %s: %s", settingsValue, candidate, expectMatch),
                 pattern.matcher(candidate).matches(),
                 is(expectMatch)
             );
diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesActionTests.java
index 164fc38d15c4..5885c6f8c988 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesActionTests.java
@@ -9,7 +9,6 @@
 package org.elasticsearch.rest.action.admin.indices;
 
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.test.ESTestCase;
@@ -37,7 +36,7 @@ public void testIncludeTypeNamesWarning() throws IOException {
             Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)
         ).withMethod(RestRequest.Method.GET).withPath("/some_index").withParams(params).build();
 
-        RestGetIndicesAction handler = new RestGetIndicesAction(new DeterministicTaskQueue().getThreadPool());
+        RestGetIndicesAction handler = new RestGetIndicesAction();
         handler.prepareRequest(request, mock(NodeClient.class));
         assertCriticalWarnings(RestGetIndicesAction.TYPES_DEPRECATION_MESSAGE);
 
@@ -58,7 +57,7 @@ public void testIncludeTypeNamesWarningExists() throws IOException {
             Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)
         ).withMethod(RestRequest.Method.HEAD).withPath("/some_index").withParams(params).build();
 
-        RestGetIndicesAction handler = new RestGetIndicesAction(new DeterministicTaskQueue().getThreadPool());
+        RestGetIndicesAction handler = new RestGetIndicesAction();
         handler.prepareRequest(request, mock(NodeClient.class));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestCatComponentTemplateActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestCatComponentTemplateActionTests.java
index cab92c4e2292..7b22091341bb 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestCatComponentTemplateActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestCatComponentTemplateActionTests.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponseUtils;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.client.NoOpNodeClient;
 import org.elasticsearch.test.rest.FakeRestChannel;
@@ -86,7 +87,7 @@ public void testRestCatComponentAction() throws Exception {
         // validate results
         assertThat(channel.responses().get(), equalTo(1));
         assertThat(channel.capturedResponse().status(), equalTo(RestStatus.OK));
-        assertThat(channel.capturedResponse().content().utf8ToString(), equalTo("test_ct 2 0 0 2 0 []\n"));
+        assertThat(RestResponseUtils.getBodyContent(channel.capturedResponse()).utf8ToString(), equalTo("test_ct 2 0 0 2 0 []\n"));
     }
 
     public void testRestCatComponentActionWithParam() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestCatRecoveryActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestCatRecoveryActionTests.java
index 10d98160938f..3db5faa505c9 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestCatRecoveryActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestCatRecoveryActionTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.XContentElasticsearchExtension;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
@@ -192,7 +193,7 @@ public void testRestRecoveryAction() {
     }
 
     private static String percent(float percent) {
-        return formatted("%1.1f%%", percent);
+        return Strings.format("%1.1f%%", percent);
     }
 
 }
diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
index 3cbd7aa6a9ab..b6a2f05164a1 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
@@ -11,10 +11,14 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.health.ClusterIndexHealth;
+import org.elasticsearch.cluster.health.ClusterStateHealth;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.Table;
@@ -26,6 +30,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
 
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
@@ -33,7 +38,6 @@
 import java.util.stream.IntStream;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -42,9 +46,10 @@ public class RestIndicesActionTests extends ESTestCase {
     public void testBuildTable() {
         final int numIndices = randomIntBetween(3, 20);
         final Map<String, Settings> indicesSettings = new LinkedHashMap<>();
-        final Map<String, IndexMetadata> indicesMetadatas = new LinkedHashMap<>();
-        final Map<String, ClusterIndexHealth> indicesHealths = new LinkedHashMap<>();
-        final Map<String, IndexStats> indicesStats = new LinkedHashMap<>();
+        final Map<String, IndexStats> indicesStats = new HashMap<>();
+
+        final Metadata.Builder metadata = Metadata.builder();
+        final RoutingTable.Builder routingTable = RoutingTable.builder();
 
         for (int i = 0; i < numIndices; i++) {
             String indexName = "index-" + i;
@@ -68,7 +73,7 @@ public void testBuildTable() {
                     .numberOfReplicas(numberOfReplicas)
                     .state(indexState)
                     .build();
-                indicesMetadatas.put(indexName, indexMetadata);
+                metadata.put(indexMetadata, false);
 
                 if (frequently()) {
                     Index index = indexMetadata.getIndex();
@@ -101,7 +106,7 @@ public void testBuildTable() {
                         case RED:
                             break;
                     }
-                    indicesHealths.put(indexName, new ClusterIndexHealth(indexMetadata, indexRoutingTable.build()));
+                    routingTable.add(indexRoutingTable);
 
                     if (frequently()) {
                         IndexStats indexStats = mock(IndexStats.class);
@@ -113,8 +118,13 @@ public void testBuildTable() {
             }
         }
 
+        final ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE)
+            .metadata(metadata)
+            .routingTable(randomBoolean() ? routingTable : RoutingTable.builder())
+            .build();
+
         final RestIndicesAction action = new RestIndicesAction();
-        final Table table = action.buildTable(new FakeRestRequest(), indicesSettings, indicesHealths, indicesStats, indicesMetadatas);
+        final Table table = action.buildTable(new FakeRestRequest(), indicesSettings, clusterState, indicesStats);
 
         // now, verify the table is correct
         List<Table.Cell> headers = table.getHeaders();
@@ -126,14 +136,16 @@ public void testBuildTable() {
         assertThat(headers.get(5).value, equalTo("rep"));
 
         final List<List<Table.Cell>> rows = table.getRows();
-        assertThat(rows.size(), equalTo(indicesMetadatas.size()));
+        assertThat(rows.size(), equalTo(clusterState.metadata().indices().size()));
+
+        final var clusterStateHealth = new ClusterStateHealth(clusterState);
 
         for (final List<Table.Cell> row : rows) {
             final String indexName = (String) row.get(2).value;
 
-            ClusterIndexHealth indexHealth = indicesHealths.get(indexName);
+            ClusterIndexHealth indexHealth = clusterStateHealth.getIndices().get(indexName);
             IndexStats indexStats = indicesStats.get(indexName);
-            IndexMetadata indexMetadata = indicesMetadatas.get(indexName);
+            IndexMetadata indexMetadata = clusterState.metadata().index(indexName);
 
             if (indexHealth != null) {
                 assertThat(row.get(0).value, equalTo(indexHealth.getStatus().toString().toLowerCase(Locale.ROOT)));
@@ -146,13 +158,8 @@ public void testBuildTable() {
             assertThat(row.get(1).value, equalTo(indexMetadata.getState().toString().toLowerCase(Locale.ROOT)));
             assertThat(row.get(2).value, equalTo(indexName));
             assertThat(row.get(3).value, equalTo(indexMetadata.getIndexUUID()));
-            if (indexHealth != null) {
-                assertThat(row.get(4).value, equalTo(indexMetadata.getNumberOfShards()));
-                assertThat(row.get(5).value, equalTo(indexMetadata.getNumberOfReplicas()));
-            } else {
-                assertThat(row.get(4).value, nullValue());
-                assertThat(row.get(5).value, nullValue());
-            }
+            assertThat(row.get(4).value, equalTo(indexMetadata.getNumberOfShards()));
+            assertThat(row.get(5).value, equalTo(indexMetadata.getNumberOfReplicas()));
         }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java
index 0856c5c787e6..ff3b72463d86 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java
@@ -8,7 +8,10 @@
 
 package org.elasticsearch.rest.action.cat;
 
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Table;
+import org.elasticsearch.common.recycler.Recycler;
+import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.rest.AbstractRestChannel;
 import org.elasticsearch.rest.RestResponse;
@@ -17,17 +20,20 @@
 import org.elasticsearch.xcontent.XContentType;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.elasticsearch.rest.action.cat.RestTable.buildDisplayHeaders;
 import static org.elasticsearch.rest.action.cat.RestTable.buildResponse;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.equalToIgnoringCase;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.not;
 
@@ -253,6 +259,96 @@ public void testMultiSort() {
         assertEquals(Arrays.asList(1, 0, 2), rowOrder);
     }
 
+    public void testPlainTextChunking() throws Exception {
+        final var cells = randomArray(8, 8, String[]::new, () -> randomAlphaOfLengthBetween(1, 5));
+        final var expectedRow = String.join(" ", cells) + "\n";
+
+        // OutputStreamWriter has an 8kiB buffer so all chunks are at least that big
+        final var bufferSize = ByteSizeUnit.KB.toIntBytes(8);
+        final var rowLength = expectedRow.length();
+        final var expectedRowsPerChunk = 1 + bufferSize / rowLength; // end chunk after first row which overflows the buffer
+        final var expectedChunkSize = expectedRowsPerChunk * rowLength;
+
+        final var rowCount = between(expectedRowsPerChunk + 1, expectedRowsPerChunk * 10);
+        final var expectedChunkCount = 1 + (rowCount * rowLength - 1) / expectedChunkSize; // ceil(rowCount * rowLength / expectedChunkSize)
+        assertThat(expectedChunkCount, greaterThan(1));
+
+        final var expectedBody = new StringBuilder();
+        for (int i = 0; i < rowCount; i++) {
+            table.startRow();
+            for (final var cell : cells) {
+                table.addCell(cell);
+            }
+            table.endRow();
+            expectedBody.append(expectedRow);
+        }
+
+        final var request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(
+            Collections.singletonMap(ACCEPT, Collections.singletonList(TEXT_PLAIN))
+        ).build();
+
+        final var response = buildResponse(table, new AbstractRestChannel(request, true) {
+            @Override
+            public void sendResponse(RestResponse response) {}
+        });
+
+        final var bodyChunks = getBodyChunks(response, bufferSize);
+        assertEquals("chunk count", expectedChunkCount, bodyChunks.size());
+        assertEquals("first chunk size", expectedChunkSize, bodyChunks.get(0).length());
+        assertEquals("body contents", expectedBody.toString(), String.join("", bodyChunks));
+    }
+
+    public void testEmptyTable() throws Exception {
+        final var request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(
+            Collections.singletonMap(ACCEPT, Collections.singletonList(TEXT_PLAIN))
+        ).build();
+
+        final var response = buildResponse(table, new AbstractRestChannel(request, true) {
+            @Override
+            public void sendResponse(RestResponse response) {}
+        });
+
+        assertFalse(response.isChunked());
+        assertThat(response.content().length(), equalTo(0));
+    }
+
+    public void testXContentChunking() throws Exception {
+        final var jsonTableRow = JSON_TABLE_BODY.substring(1, JSON_TABLE_BODY.length() - 2).replaceAll("\\s+", "");
+        final var expectedBody = new StringBuilder("[");
+        final var rowCount = between(10000, 20000);
+        for (int i = 0; i < rowCount; i++) {
+            if (i != 0) {
+                expectedBody.append(",");
+            }
+            table.startRow();
+            table.addCell("foo");
+            table.addCell("foo");
+            table.addCell("foo");
+            table.addCell("foo");
+            table.addCell("foo");
+            table.addCell("foo");
+            table.addCell("foo");
+            table.addCell("foo");
+            table.endRow();
+            expectedBody.append(jsonTableRow);
+        }
+        expectedBody.append("]");
+
+        final var request = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(
+            Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_JSON))
+        ).build();
+
+        final var response = buildResponse(table, new AbstractRestChannel(request, true) {
+            @Override
+            public void sendResponse(RestResponse response) {}
+        });
+
+        // layers of buffering make it hard to be precise here:
+        final var bodyChunks = getBodyChunks(response, 8192);
+        assertEquals(expectedBody.toString(), String.join("", bodyChunks));
+        assertThat(bodyChunks.size(), greaterThan(1));
+    }
+
     private RestResponse assertResponseContentType(Map<String, List<String>> headers, String mediaType) throws Exception {
         FakeRestRequest requestWithAcceptHeader = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(headers).build();
         table.startRow();
@@ -277,7 +373,50 @@ public void sendResponse(RestResponse response) {}
 
     private void assertResponse(Map<String, List<String>> headers, String mediaType, String body) throws Exception {
         RestResponse response = assertResponseContentType(headers, mediaType);
-        assertThat(response.content().utf8ToString(), equalTo(body));
+        assertTrue(response.isChunked());
+        assertThat(String.join("", getBodyChunks(response, between(1, 1024))), equalTo(body));
+    }
+
+    private static List<String> getBodyChunks(RestResponse response, final int pageSize) throws IOException {
+        assertTrue(response.isChunked());
+
+        final var openPages = new AtomicInteger();
+        final var recycler = new Recycler<BytesRef>() {
+            @Override
+            public V<BytesRef> obtain() {
+                openPages.incrementAndGet();
+                return new V<>() {
+                    final BytesRef page = new BytesRef(new byte[pageSize], 0, pageSize);
+
+                    @Override
+                    public BytesRef v() {
+                        return page;
+                    }
+
+                    @Override
+                    public boolean isRecycled() {
+                        return false;
+                    }
+
+                    @Override
+                    public void close() {
+                        openPages.decrementAndGet();
+                    }
+                };
+            }
+        };
+
+        final var bodyChunks = new ArrayList<String>();
+        final var chunkedRestResponseBody = response.chunkedContent();
+
+        while (chunkedRestResponseBody.isDone() == false) {
+            try (var chunk = chunkedRestResponseBody.encodeChunk(pageSize, recycler)) {
+                assertThat(chunk.length(), greaterThan(0));
+                bodyChunks.add(chunk.utf8ToString());
+            }
+        }
+        assertEquals(0, openPages.get());
+        return bodyChunks;
     }
 
     private List<String> getHeaderNames(List<RestTable.DisplayHeader> headers) {
diff --git a/server/src/test/java/org/elasticsearch/rest/action/search/RestSearchActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/search/RestSearchActionTests.java
index a7b05e450456..9f88540e6af6 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/search/RestSearchActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/search/RestSearchActionTests.java
@@ -104,7 +104,7 @@ public void testValidateSearchRequest() {
 
             SearchRequest searchRequest = new SearchRequest();
             KnnSearchBuilder knnSearch = new KnnSearchBuilder("vector", new float[] { 1, 1, 1 }, 10, 100);
-            searchRequest.source(new SearchSourceBuilder().knnSearch(knnSearch));
+            searchRequest.source(new SearchSourceBuilder().knnSearch(List.of(knnSearch)));
 
             Exception ex = expectThrows(
                 IllegalArgumentException.class,
diff --git a/server/src/test/java/org/elasticsearch/script/CompositeFieldScriptTests.java b/server/src/test/java/org/elasticsearch/script/CompositeFieldScriptTests.java
index 9fb7d6094c57..0a80f4310b89 100644
--- a/server/src/test/java/org/elasticsearch/script/CompositeFieldScriptTests.java
+++ b/server/src/test/java/org/elasticsearch/script/CompositeFieldScriptTests.java
@@ -13,8 +13,8 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -32,7 +32,8 @@ public void testTooManyValues() throws IOException {
                 CompositeFieldScript script = new CompositeFieldScript(
                     "composite",
                     Collections.emptyMap(),
-                    new SearchLookup(field -> null, (ft, lookup, ftd) -> null, new SourceLookup.ReaderSourceProvider()),
+                    new SearchLookup(field -> null, (ft, lookup, ftd) -> null, (ctx, doc) -> null),
+                    OnScriptError.FAIL,
                     reader.leaves().get(0)
                 ) {
                     @Override
@@ -63,7 +64,8 @@ public void testTooManyChars() throws IOException {
                 CompositeFieldScript script = new CompositeFieldScript(
                     "composite",
                     Collections.emptyMap(),
-                    new SearchLookup(field -> null, (ft, lookup, ftd) -> null, new SourceLookup.ReaderSourceProvider()),
+                    new SearchLookup(field -> null, (ft, lookup, ftd) -> null, (ctx, doc) -> null),
+                    OnScriptError.FAIL,
                     reader.leaves().get(0)
                 ) {
                     @Override
@@ -76,7 +78,8 @@ public void execute() {
                 StringFieldScript stringFieldScript = new StringFieldScript(
                     "composite.leaf",
                     Collections.emptyMap(),
-                    new SearchLookup(field -> null, (ft, lookup, ftd) -> null, new SourceLookup.ReaderSourceProvider()),
+                    new SearchLookup(field -> null, (ft, lookup, ftd) -> null, (ctx, doc) -> null),
+                    OnScriptError.FAIL,
                     reader.leaves().get(0)
                 ) {
                     @Override
diff --git a/server/src/test/java/org/elasticsearch/script/CtxMapTests.java b/server/src/test/java/org/elasticsearch/script/CtxMapTests.java
index 67b906e24eaa..fe692faaea93 100644
--- a/server/src/test/java/org/elasticsearch/script/CtxMapTests.java
+++ b/server/src/test/java/org/elasticsearch/script/CtxMapTests.java
@@ -25,7 +25,7 @@ public class CtxMapTests extends ESTestCase {
     @Before
     public void setUp() throws Exception {
         super.setUp();
-        map = new CtxMap<>(new HashMap<>(), new Metadata(new HashMap<>(), new HashMap<>()));
+        map = new CtxMap<>(new HashMap<>(), new Metadata(Map.of(), Map.of()));
     }
 
     public void testAddingJunkToCtx() {
diff --git a/server/src/test/java/org/elasticsearch/script/MetadataTests.java b/server/src/test/java/org/elasticsearch/script/MetadataTests.java
index 843c5a6c1c97..a99524888c24 100644
--- a/server/src/test/java/org/elasticsearch/script/MetadataTests.java
+++ b/server/src/test/java/org/elasticsearch/script/MetadataTests.java
@@ -58,7 +58,7 @@ private static Map<String, Metadata.FieldProperty<?>> allowAllValidators(String.
         for (String key : keys) {
             validators.put(key, Metadata.FieldProperty.ALLOW_ALL);
         }
-        return validators;
+        return Map.copyOf(validators);
     }
 
     public void testValidateMetadata() {
@@ -264,4 +264,18 @@ public void testFieldPropertyExtendedValidation() {
             odd.check(op, key, value);
         }
     }
+
+    public void testImmutablePropertiesMap() {
+        // a mutable metadata properties map is not permitted
+        AssertionError e1 = expectThrows(AssertionError.class, () -> new Metadata(Map.of(), new HashMap<>()));
+        assertEquals("properties map must be constructed via Map.of(...) or Map.copyOf(...)", e1.getMessage());
+
+        // furthermore, for optimization reasons, just calling Collections.unmodifiableMap() is not sufficient
+        AssertionError e2 = expectThrows(AssertionError.class, () -> new Metadata(Map.of(), Collections.unmodifiableMap(new HashMap<>())));
+        assertEquals("properties map must be constructed via Map.of(...) or Map.copyOf(...)", e1.getMessage());
+
+        // Map.of and Map.copyOf are permissible (the former for code that should be fast, and the latter for e.g. tests)
+        new Metadata(Map.of(), Map.of());
+        new Metadata(Map.of(), Map.copyOf(new HashMap<>()));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/script/ScriptMetadataTests.java b/server/src/test/java/org/elasticsearch/script/ScriptMetadataTests.java
index 51e30000474c..1d778501b4b5 100644
--- a/server/src/test/java/org/elasticsearch/script/ScriptMetadataTests.java
+++ b/server/src/test/java/org/elasticsearch/script/ScriptMetadataTests.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.xcontent.DeprecationHandler;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -28,7 +28,7 @@
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.hasKey;
 
-public class ScriptMetadataTests extends AbstractXContentSerializingTestCase<ScriptMetadata> {
+public class ScriptMetadataTests extends AbstractChunkedSerializingTestCase<ScriptMetadata> {
 
     public void testGetScript() throws Exception {
         ScriptMetadata.Builder builder = new ScriptMetadata.Builder(null);
diff --git a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java
index 96018f0cd16f..390fc4f0a6cf 100644
--- a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java
@@ -20,7 +20,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Locale;
 import java.util.function.Function;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -109,17 +108,17 @@ public void testSerializeTimeSeries() throws IOException {
         XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
         mkContextStats.apply(series).toXContent(builder, ToXContent.EMPTY_PARAMS);
 
-        assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 2222, 0, 0, 5)));
+        assertThat(Strings.toString(builder), equalTo(Strings.format(format, 2222, 0, 0, 5)));
 
         series = new TimeSeries(0, 7, 1234, 5678);
         builder = XContentFactory.jsonBuilder().prettyPrint();
         mkContextStats.apply(series).toXContent(builder, ToXContent.EMPTY_PARAMS);
-        assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 5678, 0, 7, 1234)));
+        assertThat(Strings.toString(builder), equalTo(Strings.format(format, 5678, 0, 7, 1234)));
 
         series = new TimeSeries(123, 456, 789, 91011);
         builder = XContentFactory.jsonBuilder().prettyPrint();
         mkContextStats.apply(series).toXContent(builder, ToXContent.EMPTY_PARAMS);
-        assertThat(Strings.toString(builder), equalTo(String.format(Locale.ROOT, format, 91011, 123, 456, 789)));
+        assertThat(Strings.toString(builder), equalTo(Strings.format(format, 91011, 123, 456, 789)));
     }
 
     public void testTimeSeriesIsEmpty() {
diff --git a/server/src/test/java/org/elasticsearch/script/VectorScoreScriptUtilsTests.java b/server/src/test/java/org/elasticsearch/script/VectorScoreScriptUtilsTests.java
index 1dc29532e9de..841a80927ba8 100644
--- a/server/src/test/java/org/elasticsearch/script/VectorScoreScriptUtilsTests.java
+++ b/server/src/test/java/org/elasticsearch/script/VectorScoreScriptUtilsTests.java
@@ -55,12 +55,7 @@ public void testFloatVectorClassBindings() throws IOException {
                 dims,
                 Version.CURRENT
             ),
-            new KnnDenseVectorDocValuesField(
-                KnnDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }, ElementType.FLOAT),
-                "test",
-                ElementType.FLOAT,
-                dims
-            )
+            new KnnDenseVectorDocValuesField(KnnDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }), "test", dims)
         );
         for (DenseVectorDocValuesField field : fields) {
             field.setNextDocId(0);
@@ -140,12 +135,7 @@ public void testByteVectorClassBindings() throws IOException {
                 ElementType.BYTE,
                 dims
             ),
-            new ByteKnnDenseVectorDocValuesField(
-                KnnDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }, ElementType.BYTE),
-                "test",
-                ElementType.BYTE,
-                dims
-            )
+            new ByteKnnDenseVectorDocValuesField(KnnDenseVectorScriptDocValuesTests.wrapBytes(new float[][] { docVector }), "test", dims)
         );
         for (DenseVectorDocValuesField field : fields) {
             field.setNextDocId(0);
@@ -233,24 +223,14 @@ public void testByteVsFloatSimilarity() throws IOException {
                 dims,
                 Version.CURRENT
             ),
-            new KnnDenseVectorDocValuesField(
-                KnnDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }, ElementType.FLOAT),
-                "field2",
-                ElementType.FLOAT,
-                dims
-            ),
+            new KnnDenseVectorDocValuesField(KnnDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }), "field2", dims),
             new ByteBinaryDenseVectorDocValuesField(
                 BinaryDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }, ElementType.BYTE, Version.CURRENT),
                 "field3",
                 ElementType.BYTE,
                 dims
             ),
-            new ByteKnnDenseVectorDocValuesField(
-                KnnDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }, ElementType.BYTE),
-                "field4",
-                ElementType.BYTE,
-                dims
-            )
+            new ByteKnnDenseVectorDocValuesField(KnnDenseVectorScriptDocValuesTests.wrapBytes(new float[][] { docVector }), "field4", dims)
         );
         for (DenseVectorDocValuesField field : fields) {
             field.setNextDocId(0);
@@ -388,12 +368,7 @@ public void testByteBoundaries() throws IOException {
                 ElementType.BYTE,
                 dims
             ),
-            new ByteKnnDenseVectorDocValuesField(
-                KnnDenseVectorScriptDocValuesTests.wrap(new float[][] { docVector }, ElementType.BYTE),
-                "test",
-                ElementType.BYTE,
-                dims
-            )
+            new ByteKnnDenseVectorDocValuesField(KnnDenseVectorScriptDocValuesTests.wrapBytes(new float[][] { docVector }), "test", dims)
         );
 
         for (DenseVectorDocValuesField field : fields) {
diff --git a/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java b/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java
index a449775cb253..c63e43122918 100644
--- a/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java
+++ b/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.script.AbstractFieldScript;
 import org.elasticsearch.script.SortedNumericDocValuesLongFieldScript;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -40,7 +39,7 @@ public void testValuesLimitIsNotEnforced() throws IOException {
             try (DirectoryReader reader = iw.getReader()) {
                 SortedNumericDocValuesLongFieldScript docValues = new SortedNumericDocValuesLongFieldScript(
                     "test",
-                    new SearchLookup(field -> null, (ft, lookup, ftd) -> null, new SourceLookup.ReaderSourceProvider()),
+                    new SearchLookup(field -> null, (ft, lookup, ftd) -> null, (ctx, doc) -> null),
                     reader.leaves().get(0)
                 );
                 List<Long> values = new ArrayList<>();
diff --git a/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java b/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java
index d4385b261abd..3bbea2b3c6b7 100644
--- a/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java
+++ b/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.script.AbstractFieldScript;
 import org.elasticsearch.script.SortedSetDocValuesStringFieldScript;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -41,7 +40,7 @@ public void testValuesLimitIsNotEnforced() throws IOException {
             try (DirectoryReader reader = iw.getReader()) {
                 SortedSetDocValuesStringFieldScript docValues = new SortedSetDocValuesStringFieldScript(
                     "test",
-                    new SearchLookup(field -> null, (ft, lookup, ftd) -> null, new SourceLookup.ReaderSourceProvider()),
+                    new SearchLookup(field -> null, (ft, lookup, ftd) -> null, (ctx, doc) -> null),
                     reader.leaves().get(0)
                 );
                 List<String> values = new ArrayList<>();
diff --git a/server/src/test/java/org/elasticsearch/script/field/vectors/DenseVectorTests.java b/server/src/test/java/org/elasticsearch/script/field/vectors/DenseVectorTests.java
index 6f9255f06550..41d97f792043 100644
--- a/server/src/test/java/org/elasticsearch/script/field/vectors/DenseVectorTests.java
+++ b/server/src/test/java/org/elasticsearch/script/field/vectors/DenseVectorTests.java
@@ -101,7 +101,7 @@ public void testByteVsListQueryVector() {
             listQV.add(q);
         }
 
-        ByteKnnDenseVector knn = new ByteKnnDenseVector(new BytesRef(docVector));
+        ByteKnnDenseVector knn = new ByteKnnDenseVector(docVector);
         assertEquals(knn.dotProduct(arrayQV), knn.dotProduct(listQV), 0.001f);
         assertEquals(knn.dotProduct((Object) listQV), knn.dotProduct((Object) arrayQV), 0.001f);
 
@@ -139,7 +139,7 @@ public void testByteUnsupported() {
             queryVector[i] = randomByte();
         }
 
-        ByteKnnDenseVector knn = new ByteKnnDenseVector(new BytesRef(docVector));
+        ByteKnnDenseVector knn = new ByteKnnDenseVector(docVector);
         UnsupportedOperationException e;
 
         e = expectThrows(UnsupportedOperationException.class, () -> knn.dotProduct(queryVector));
diff --git a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java
index a6f80c705341..19f6400badcf 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java
@@ -10,15 +10,15 @@
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.IntPoint;
-import org.apache.lucene.document.KnnVectorField;
+import org.apache.lucene.document.KnnFloatVectorField;
 import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.PointValues;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.VectorValues;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.TotalHitCountCollector;
@@ -67,7 +67,7 @@ private static void indexRandomDocuments(RandomIndexWriter w, int numDocs) throw
             Document doc = new Document();
             doc.add(new StringField(STRING_FIELD_NAME, "a".repeat(i), Field.Store.NO));
             doc.add(new IntPoint(POINT_FIELD_NAME, i, i + 1));
-            doc.add(new KnnVectorField(KNN_FIELD_NAME, new float[] { 1.0f, 0.5f, 42.0f }));
+            doc.add(new KnnFloatVectorField(KNN_FIELD_NAME, new float[] { 1.0f, 0.5f, 42.0f }));
             w.addDocument(doc);
         }
     }
@@ -197,20 +197,20 @@ public void testExitableDirectoryReaderVectors() throws IOException {
         );
         searcher.addQueryCancellation(cancellation);
         final LeafReader leaf = searcher.getIndexReader().leaves().get(0).reader();
-        expectThrows(TaskCancelledException.class, () -> leaf.getVectorValues(KNN_FIELD_NAME));
+        expectThrows(TaskCancelledException.class, () -> leaf.getFloatVectorValues(KNN_FIELD_NAME));
         expectThrows(
             TaskCancelledException.class,
             () -> leaf.searchNearestVectors(KNN_FIELD_NAME, new float[] { 1f, 1f, 1f }, 2, leaf.getLiveDocs(), Integer.MAX_VALUE)
         );
 
         cancelled.set(false); // Avoid exception during construction of the wrapper objects
-        VectorValues vectorValues = searcher.getIndexReader().leaves().get(0).reader().getVectorValues(KNN_FIELD_NAME);
+        FloatVectorValues vectorValues = searcher.getIndexReader().leaves().get(0).reader().getFloatVectorValues(KNN_FIELD_NAME);
         cancelled.set(true);
         // On the first doc when already canceled, it throws
         expectThrows(TaskCancelledException.class, vectorValues::nextDoc);
 
         cancelled.set(false); // Avoid exception during construction of the wrapper objects
-        VectorValues uncancelledVectorValues = searcher.getIndexReader().leaves().get(0).reader().getVectorValues(KNN_FIELD_NAME);
+        FloatVectorValues uncancelledVectorValues = searcher.getIndexReader().leaves().get(0).reader().getFloatVectorValues(KNN_FIELD_NAME);
         cancelled.set(true);
         searcher.removeQueryCancellation(cancellation);
         // On the first doc when already canceled, it throws, but with the cancellation removed, it should not
diff --git a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java
index c50e0777bd6d..96634dad16c7 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java
@@ -148,6 +148,11 @@ protected SearchHit createTestInstance() {
         return createTestItem(randomFrom(XContentType.values()).canonical(), randomBoolean(), randomBoolean());
     }
 
+    @Override
+    protected SearchHit mutateInstance(SearchHit instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testFromXContent() throws IOException {
         XContentType xContentType = randomFrom(XContentType.values()).canonical();
         SearchHit searchHit = createTestItem(xContentType, true, false);
diff --git a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java
index 4c81141be2ea..6dadb5b03f67 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java
@@ -9,7 +9,7 @@
 
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.util.CharsRefBuilder;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -544,8 +544,8 @@ private static TestAggregationBuilder fromXContent(String name, XContentParser p
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.V_EMPTY;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.ZERO;
         }
     }
 
@@ -599,8 +599,8 @@ private static TestPipelineAggregationBuilder fromXContent(String name, XContent
         protected void validate(ValidationContext context) {}
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.V_EMPTY;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.ZERO;
         }
     }
 
@@ -649,8 +649,8 @@ public RescoreContext innerBuildContext(int windowSize, SearchExecutionContext c
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.V_EMPTY;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.ZERO;
         }
     }
 
@@ -717,8 +717,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.V_EMPTY;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.ZERO;
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesBuilderTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesBuilderTests.java
index 2c0072cc7ae4..6c4ecf238815 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesBuilderTests.java
@@ -84,6 +84,11 @@ protected Builder createTestInstance() {
         return builder;
     }
 
+    @Override
+    protected Builder mutateInstance(Builder instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<Builder> instanceReader() {
         return AggregatorFactories.Builder::new;
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java
index 646604b0cc16..06ded5285fe8 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.search.aggregations;
 
 import org.apache.lucene.util.SetOnce;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
@@ -347,8 +347,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.V_EMPTY;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.ZERO;
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/InternalOrderTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/InternalOrderTests.java
index 7034eafd9bfd..b6cdf1a2825d 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/InternalOrderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/InternalOrderTests.java
@@ -100,7 +100,7 @@ public void testAggregationOrderEqualsAndHashCode() {
     }
 
     @Override
-    protected BucketOrder mutateInstance(BucketOrder instance) throws IOException {
+    protected BucketOrder mutateInstance(BucketOrder instance) {
         if (instance == InternalOrder.KEY_ASC) {
             return InternalOrder.COUNT_ASC;
         } else if (instance == InternalOrder.KEY_DESC) {
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/TopBucketBuilderTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/TopBucketBuilderTests.java
index 6c633916e8a3..a5be04ab92ed 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/TopBucketBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/TopBucketBuilderTests.java
@@ -10,6 +10,7 @@
 
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation.InternalBucket;
 import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
@@ -135,7 +136,7 @@ public void testHugeQueueError() {
     }
 
     private String bucketKey(int index) {
-        return formatted("%06d", index);
+        return Strings.format("%06d", index);
     }
 
     private InternalBucket bucket(int index) {
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java
index a4d1d69c5925..16b7d30e9ab9 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java
@@ -44,6 +44,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.DateFormatters;
+import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.DateFieldMapper;
@@ -113,10 +114,13 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.search.aggregations.bucket.nested.NestedAggregatorTests.nestedObject;
+import static org.elasticsearch.test.MapMatcher.assertMap;
+import static org.elasticsearch.test.MapMatcher.matchesMap;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
@@ -3255,6 +3259,310 @@ public void testCompositeWithSamplingAndOneSmallBucket() throws Exception {
         );
     }
 
+    public void testWithKeywordGivenNoIndexSortingAndDynamicPruningIsNotApplicableDueToMissingBucket() throws Exception {
+        final CompositeAggregationBuilder aggregationBuilder = new CompositeAggregationBuilder(
+            "name",
+            List.of(new TermsValuesSourceBuilder("leading").field("keyword").missingBucket(true))
+        ).size(2);
+        final MappedFieldType keywordMapping = new KeywordFieldMapper.KeywordFieldType("keyword");
+        final MappedFieldType fooMapping = new KeywordFieldMapper.KeywordFieldType("foo");
+
+        CheckedConsumer<RandomIndexWriter, IOException> buildIndex = iw -> {
+            for (int i = 1; i <= 100; i++) {
+                addDocWithKeywordFields(iw, "keyword", "a_" + i, "foo", "bar");
+            }
+        };
+
+        debugTestCase(
+            aggregationBuilder,
+            new TermQuery(new Term("foo", "bar")),
+            buildIndex,
+            (InternalComposite result, Class<? extends Aggregator> impl, Map<String, Map<String, Object>> debug) -> {
+                assertThat(result.getBuckets(), hasSize(2));
+                assertEquals(CompositeAggregator.class, impl);
+                assertEquals("{leading=a_10}", result.afterKey().toString());
+                assertEquals("{leading=a_1}", result.getBuckets().get(0).getKeyAsString());
+                assertEquals(1L, result.getBuckets().get(0).getDocCount());
+                assertEquals("{leading=a_10}", result.getBuckets().get(1).getKeyAsString());
+                assertEquals(1L, result.getBuckets().get(1).getDocCount());
+                assertMap(
+                    debug,
+                    matchesMap().entry(
+                        "name",
+                        matchesMap().entry("sources.leading.segments_dynamic_pruning_used", equalTo(0))
+                            .entry("sources.leading.segments_collected", greaterThanOrEqualTo(1))
+                    )
+                );
+            },
+            keywordMapping,
+            fooMapping
+        );
+    }
+
+    public void testWithKeywordGivenNoIndexSortingAndDynamicPruningIsNotApplicableDueToSize() throws Exception {
+        final CompositeAggregationBuilder aggregationBuilder = new CompositeAggregationBuilder(
+            "name",
+            List.of(new TermsValuesSourceBuilder("leading").field("keyword").missingBucket(true))
+        ).size(13); // We need a size that is more than 1/8 of the total count.
+        final MappedFieldType keywordMapping = new KeywordFieldMapper.KeywordFieldType("keyword");
+        final MappedFieldType fooMapping = new KeywordFieldMapper.KeywordFieldType("foo");
+
+        CheckedConsumer<RandomIndexWriter, IOException> buildIndex = iw -> {
+            for (int i = 1; i <= 100; i++) {
+                addDocWithKeywordFields(iw, "keyword", "a_" + i, "foo", "bar");
+            }
+        };
+
+        debugTestCase(
+            aggregationBuilder,
+            new TermQuery(new Term("foo", "bar")),
+            buildIndex,
+            (InternalComposite result, Class<? extends Aggregator> impl, Map<String, Map<String, Object>> debug) -> {
+                assertThat(result.getBuckets(), hasSize(13));
+                assertEquals(CompositeAggregator.class, impl);
+                assertMap(
+                    debug,
+                    matchesMap().entry(
+                        "name",
+                        matchesMap().entry("sources.leading.segments_dynamic_pruning_used", equalTo(0))
+                            .entry("sources.leading.segments_collected", greaterThanOrEqualTo(1))
+                    )
+                );
+            },
+            keywordMapping,
+            fooMapping
+        );
+    }
+
+    public void testWithKeywordGivenNoIndexSortingAndDynamicPruningIsApplicableAndAscendingOrder() throws Exception {
+        CompositeAggregationBuilder aggregationBuilder = new CompositeAggregationBuilder(
+            "name",
+            List.of(new TermsValuesSourceBuilder("leading").field("keyword"))
+        ).size(2);
+        final MappedFieldType keywordMapping = new KeywordFieldMapper.KeywordFieldType("keyword");
+        final MappedFieldType fooMapping = new KeywordFieldMapper.KeywordFieldType("foo");
+
+        CheckedConsumer<RandomIndexWriter, IOException> buildIndex = iw -> {
+            for (int i = 1; i <= 100; i++) {
+                addDocWithKeywordFields(iw, "keyword", "a_" + i, "foo", "bar");
+            }
+        };
+
+        debugTestCase(
+            aggregationBuilder,
+            new TermQuery(new Term("foo", "bar")),
+            buildIndex,
+            (InternalComposite result, Class<? extends Aggregator> impl, Map<String, Map<String, Object>> debug) -> {
+                assertThat(result.getBuckets(), hasSize(2));
+                assertEquals(CompositeAggregator.class, impl);
+                assertEquals("{leading=a_10}", result.afterKey().toString());
+                assertEquals("{leading=a_1}", result.getBuckets().get(0).getKeyAsString());
+                assertEquals(1L, result.getBuckets().get(0).getDocCount());
+                assertEquals("{leading=a_10}", result.getBuckets().get(1).getKeyAsString());
+                assertEquals(1L, result.getBuckets().get(1).getDocCount());
+                assertMap(
+                    debug,
+                    matchesMap().entry(
+                        "name",
+                        matchesMap().entry("sources.leading.segments_dynamic_pruning_used", greaterThanOrEqualTo(1))
+                            .entry("sources.leading.segments_collected", greaterThanOrEqualTo(1))
+                    )
+                );
+            },
+            keywordMapping,
+            fooMapping
+        );
+
+        aggregationBuilder = new CompositeAggregationBuilder("name", List.of(new TermsValuesSourceBuilder("leading").field("keyword")))
+            .size(2)
+            .aggregateAfter(Collections.singletonMap("leading", "a_10"));
+        debugTestCase(
+            aggregationBuilder,
+            new TermQuery(new Term("foo", "bar")),
+            buildIndex,
+            (InternalComposite result, Class<? extends Aggregator> impl, Map<String, Map<String, Object>> debug) -> {
+                assertThat(result.getBuckets(), hasSize(2));
+                assertEquals(CompositeAggregator.class, impl);
+                assertEquals("{leading=a_11}", result.afterKey().toString());
+                assertEquals("{leading=a_100}", result.getBuckets().get(0).getKeyAsString());
+                assertEquals(1L, result.getBuckets().get(0).getDocCount());
+                assertEquals("{leading=a_11}", result.getBuckets().get(1).getKeyAsString());
+                assertEquals(1L, result.getBuckets().get(1).getDocCount());
+                assertMap(
+                    debug,
+                    matchesMap().entry(
+                        "name",
+                        matchesMap().entry("sources.leading.segments_dynamic_pruning_used", greaterThanOrEqualTo(1))
+                            .entry("sources.leading.segments_collected", greaterThanOrEqualTo(1))
+                    )
+                );
+            },
+            keywordMapping,
+            fooMapping
+        );
+    }
+
+    public void testWithKeywordGivenNoIndexSortingAndDynamicPruningIsApplicableAndDescendingOrder() throws Exception {
+        CompositeAggregationBuilder aggregationBuilder = new CompositeAggregationBuilder(
+            "name",
+            List.of(new TermsValuesSourceBuilder("leading").field("keyword").order(SortOrder.DESC))
+        ).size(2);
+        final MappedFieldType keywordMapping = new KeywordFieldMapper.KeywordFieldType("keyword");
+        final MappedFieldType fooMapping = new KeywordFieldMapper.KeywordFieldType("foo");
+
+        CheckedConsumer<RandomIndexWriter, IOException> buildIndex = iw -> {
+            for (int i = 1; i <= 100; i++) {
+                addDocWithKeywordFields(iw, "keyword", "a_" + i, "foo", "bar");
+            }
+        };
+
+        debugTestCase(
+            aggregationBuilder,
+            new TermQuery(new Term("foo", "bar")),
+            buildIndex,
+            (InternalComposite result, Class<? extends Aggregator> impl, Map<String, Map<String, Object>> debug) -> {
+                assertThat(result.getBuckets(), hasSize(2));
+                assertEquals(CompositeAggregator.class, impl);
+                assertEquals("{leading=a_98}", result.afterKey().toString());
+                assertEquals("{leading=a_99}", result.getBuckets().get(0).getKeyAsString());
+                assertEquals(1L, result.getBuckets().get(0).getDocCount());
+                assertEquals("{leading=a_98}", result.getBuckets().get(1).getKeyAsString());
+                assertEquals(1L, result.getBuckets().get(1).getDocCount());
+                assertMap(
+                    debug,
+                    matchesMap().entry(
+                        "name",
+                        matchesMap().entry("sources.leading.segments_dynamic_pruning_used", greaterThanOrEqualTo(1))
+                            .entry("sources.leading.segments_collected", greaterThanOrEqualTo(1))
+                    )
+                );
+            },
+            keywordMapping,
+            fooMapping
+        );
+
+        aggregationBuilder = new CompositeAggregationBuilder(
+            "name",
+            List.of(new TermsValuesSourceBuilder("leading").field("keyword").order(SortOrder.DESC))
+        ).size(2).aggregateAfter(Collections.singletonMap("leading", "a_98"));
+        debugTestCase(
+            aggregationBuilder,
+            new TermQuery(new Term("foo", "bar")),
+            buildIndex,
+            (InternalComposite result, Class<? extends Aggregator> impl, Map<String, Map<String, Object>> debug) -> {
+                assertThat(result.getBuckets(), hasSize(2));
+                assertEquals(CompositeAggregator.class, impl);
+                assertEquals("{leading=a_96}", result.afterKey().toString());
+                assertEquals("{leading=a_97}", result.getBuckets().get(0).getKeyAsString());
+                assertEquals(1L, result.getBuckets().get(0).getDocCount());
+                assertEquals("{leading=a_96}", result.getBuckets().get(1).getKeyAsString());
+                assertEquals(1L, result.getBuckets().get(1).getDocCount());
+                assertMap(
+                    debug,
+                    matchesMap().entry(
+                        "name",
+                        matchesMap().entry("sources.leading.segments_dynamic_pruning_used", greaterThanOrEqualTo(1))
+                            .entry("sources.leading.segments_collected", greaterThanOrEqualTo(1))
+                    )
+                );
+            },
+            keywordMapping,
+            fooMapping
+        );
+    }
+
+    public void testWithKeywordGivenSecondarySourceAndDynamicPruningIsApplicable() throws Exception {
+        CompositeAggregationBuilder aggregationBuilder = new CompositeAggregationBuilder(
+            "name",
+            List.of(
+                new TermsValuesSourceBuilder("leading_keyword").field("leading_keyword"),
+                new TermsValuesSourceBuilder("secondary_keyword").field("secondary_keyword")
+            )
+        ).size(2);
+        final MappedFieldType leadingKeywordMapping = new KeywordFieldMapper.KeywordFieldType("leading_keyword");
+        final MappedFieldType secondaryKeywordMapping = new KeywordFieldMapper.KeywordFieldType("secondary_keyword");
+        final MappedFieldType fooMapping = new KeywordFieldMapper.KeywordFieldType("foo");
+
+        CheckedConsumer<RandomIndexWriter, IOException> buildIndex = iw -> {
+            for (int i = 1; i <= 100; i++) {
+                addDocWithKeywordFields(iw, "leading_keyword", "a_" + i, "secondary_keyword", "alpha", "foo", "bar");
+                addDocWithKeywordFields(iw, "leading_keyword", "a_" + i, "secondary_keyword", "beta", "foo", "bar");
+            }
+        };
+
+        debugTestCase(
+            aggregationBuilder,
+            new TermQuery(new Term("foo", "bar")),
+            buildIndex,
+            (InternalComposite result, Class<? extends Aggregator> impl, Map<String, Map<String, Object>> debug) -> {
+                assertThat(result.getBuckets(), hasSize(2));
+                assertEquals(CompositeAggregator.class, impl);
+                assertEquals("{leading_keyword=a_1, secondary_keyword=beta}", result.afterKey().toString());
+                assertEquals("{leading_keyword=a_1, secondary_keyword=alpha}", result.getBuckets().get(0).getKeyAsString());
+                assertEquals(1L, result.getBuckets().get(0).getDocCount());
+                assertEquals("{leading_keyword=a_1, secondary_keyword=beta}", result.getBuckets().get(1).getKeyAsString());
+                assertEquals(1L, result.getBuckets().get(1).getDocCount());
+                assertMap(
+                    debug,
+                    matchesMap().entry(
+                        "name",
+                        matchesMap().entry("sources.leading_keyword.segments_dynamic_pruning_used", greaterThanOrEqualTo(1))
+                            .entry("sources.leading_keyword.segments_collected", greaterThanOrEqualTo(1))
+                    )
+                );
+            },
+            leadingKeywordMapping,
+            secondaryKeywordMapping,
+            fooMapping
+        );
+
+        aggregationBuilder = new CompositeAggregationBuilder(
+            "name",
+            List.of(
+                new TermsValuesSourceBuilder("leading_keyword").field("leading_keyword"),
+                new TermsValuesSourceBuilder("secondary_keyword").field("secondary_keyword")
+            )
+        ).size(2).aggregateAfter(Map.of("leading_keyword", "a_1", "secondary_keyword", "beta"));
+        debugTestCase(
+            aggregationBuilder,
+            new TermQuery(new Term("foo", "bar")),
+            buildIndex,
+            (InternalComposite result, Class<? extends Aggregator> impl, Map<String, Map<String, Object>> debug) -> {
+                assertThat(result.getBuckets(), hasSize(2));
+                assertEquals(CompositeAggregator.class, impl);
+                assertEquals("{leading_keyword=a_10, secondary_keyword=beta}", result.afterKey().toString());
+                assertEquals("{leading_keyword=a_10, secondary_keyword=alpha}", result.getBuckets().get(0).getKeyAsString());
+                assertEquals(1L, result.getBuckets().get(0).getDocCount());
+                assertEquals("{leading_keyword=a_10, secondary_keyword=beta}", result.getBuckets().get(1).getKeyAsString());
+                assertEquals(1L, result.getBuckets().get(1).getDocCount());
+                assertMap(
+                    debug,
+                    matchesMap().entry(
+                        "name",
+                        matchesMap().entry("sources.leading_keyword.segments_dynamic_pruning_used", greaterThanOrEqualTo(1))
+                            .entry("sources.leading_keyword.segments_collected", greaterThanOrEqualTo(1))
+                    )
+                );
+            },
+            leadingKeywordMapping,
+            secondaryKeywordMapping,
+            fooMapping
+        );
+    }
+
+    private static void addDocWithKeywordFields(RandomIndexWriter iw, String... fieldValuePairs) throws IOException {
+        assertThat(fieldValuePairs.length, greaterThan(0));
+        assertThat(fieldValuePairs.length % 2, equalTo(0));
+        List<Field> fields = new ArrayList<>();
+        for (int i = 0; i < fieldValuePairs.length; i = i + 2) {
+            String field = fieldValuePairs[i];
+            String value = fieldValuePairs[i + 1];
+            fields.add(new StringField(field, value, Field.Store.NO));
+            fields.add(new SortedSetDocValuesField(field, new BytesRef(value)));
+        }
+        iw.addDocument(fields);
+    }
+
     private <T extends AggregationBuilder, V extends InternalAggregation> void testSearchCase(
         List<Query> queries,
         List<Map<String, List<Object>>> dataset,
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java
index c7a396a23965..c78caced15a4 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java
@@ -17,6 +17,7 @@
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReaderContext;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.CollectionTerminatedException;
@@ -39,6 +40,7 @@
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -51,6 +53,8 @@
 import static org.elasticsearch.index.mapper.NumberFieldMapper.NumberType.LONG;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class CompositeValuesCollectorQueueTests extends AggregatorTestCase {
     static class ClassAndName {
@@ -63,6 +67,16 @@ static class ClassAndName {
         }
     }
 
+    private IndexReader indexReader;
+
+    @Before
+    public void setUpMocks() {
+        indexReader = mock(IndexReader.class);
+        IndexReaderContext indexReaderContext = mock(IndexReaderContext.class);
+        when(indexReaderContext.leaves()).thenReturn(List.of());
+        when(indexReader.getContext()).thenReturn(indexReaderContext);
+    }
+
     public void testRandomLong() throws IOException {
         testRandomCase(new ClassAndName(createNumber("long", LONG), Long.class));
     }
@@ -275,6 +289,7 @@ private void testRandomCase(boolean forceMerge, boolean missingBucket, int index
                         sources[i] = new GlobalOrdinalValuesSource(
                             bigArrays,
                             fieldType,
+                            DocValues.getSortedSet(reader.leaves().get(0).reader(), fieldType.name()).getValueCount(),
                             context -> DocValues.getSortedSet(context.reader(), fieldType.name()),
                             DocValueFormat.RAW,
                             missingBucket,
@@ -308,7 +323,8 @@ private void testRandomCase(boolean forceMerge, boolean missingBucket, int index
                     final CompositeValuesCollectorQueue queue = new CompositeValuesCollectorQueue(
                         BigArrays.NON_RECYCLING_INSTANCE,
                         sources,
-                        size
+                        size,
+                        indexReader
                     );
                     if (last != null) {
                         queue.setAfterKey(last);
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/InternalCompositeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/InternalCompositeTests.java
index da86027f893b..e95bc3a46013 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/InternalCompositeTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/InternalCompositeTests.java
@@ -183,7 +183,7 @@ protected InternalComposite createTestInstance(String name, Map<String, Object>
     }
 
     @Override
-    protected InternalComposite mutateInstance(InternalComposite instance) throws IOException {
+    protected InternalComposite mutateInstance(InternalComposite instance) {
         List<InternalComposite.InternalBucket> buckets = instance.getBuckets();
         Map<String, Object> metadata = instance.getMetadata();
         int code = randomIntBetween(0, 2);
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/SingleDimensionValuesSourceTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/SingleDimensionValuesSourceTests.java
index 56b430c4ee66..4f5700cff91d 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/SingleDimensionValuesSourceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/SingleDimensionValuesSourceTests.java
@@ -96,6 +96,7 @@ public void testGlobalOrdinalsSorted() {
         GlobalOrdinalValuesSource source = new GlobalOrdinalValuesSource(
             BigArrays.NON_RECYCLING_INSTANCE,
             keyword,
+            0L,
             context -> null,
             DocValueFormat.RAW,
             false,
@@ -113,6 +114,7 @@ public void testGlobalOrdinalsSorted() {
         source = new GlobalOrdinalValuesSource(
             BigArrays.NON_RECYCLING_INSTANCE,
             keyword,
+            0L,
             context -> null,
             DocValueFormat.RAW,
             true,
@@ -127,6 +129,7 @@ public void testGlobalOrdinalsSorted() {
         source = new GlobalOrdinalValuesSource(
             BigArrays.NON_RECYCLING_INSTANCE,
             keyword,
+            0L,
             context -> null,
             DocValueFormat.RAW,
             false,
@@ -141,6 +144,7 @@ public void testGlobalOrdinalsSorted() {
         source = new GlobalOrdinalValuesSource(
             BigArrays.NON_RECYCLING_INSTANCE,
             ip,
+            0L,
             context -> null,
             DocValueFormat.RAW,
             false,
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java
index a3c1ed5d7793..548d8a140cad 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java
@@ -665,6 +665,12 @@ public void onCache(ShardId shardId, Accountable accountable) {}
                     LongPoint.newRangeQuery("t", 5, Long.MAX_VALUE)
                 );
                 IndexSearcher searcher = newIndexSearcher(limitedReader);
+                int segmentsWithLiveDocs = (int) searcher.getIndexReader()
+                    .leaves()
+                    .stream()
+                    .map(LeafReaderContext::reader)
+                    .filter(leafReader -> leafReader.getLiveDocs() != null)
+                    .count();
                 debugTestCase(
                     builder,
                     new MatchAllDocsQuery(),
@@ -679,7 +685,7 @@ public void onCache(ShardId shardId, Accountable accountable) {}
                                 matchesMap().entry("segments_counted", greaterThanOrEqualTo(1))
                                     .entry("segments_collected", 0)
                                     .entry("segments_with_doc_count_field", 0)
-                                    .entry("segments_with_deleted_docs", 0)
+                                    .entry("segments_with_deleted_docs", segmentsWithLiveDocs)
                                     .entry(
                                         "filters",
                                         matchesList().item(
@@ -730,6 +736,12 @@ public void onCache(ShardId shardId, Accountable accountable) {}
                     LongPoint.newRangeQuery("t", 5, Long.MAX_VALUE)
                 );
                 IndexSearcher searcher = newIndexSearcher(limitedReader);
+                int segmentsWithLiveDocs = (int) searcher.getIndexReader()
+                    .leaves()
+                    .stream()
+                    .map(LeafReaderContext::reader)
+                    .filter(leafReader -> leafReader.getLiveDocs() != null)
+                    .count();
                 debugTestCase(
                     builder,
                     new MatchAllDocsQuery(),
@@ -744,7 +756,7 @@ public void onCache(ShardId shardId, Accountable accountable) {}
                                 matchesMap().entry("segments_counted", greaterThanOrEqualTo(1))
                                     .entry("segments_collected", 0)
                                     .entry("segments_with_doc_count_field", 0)
-                                    .entry("segments_with_deleted_docs", 0)
+                                    .entry("segments_with_deleted_docs", segmentsWithLiveDocs)
                                     .entry(
                                         "filters",
                                         matchesList().item(
@@ -792,6 +804,12 @@ public void onCache(ShardId shardId, Accountable accountable) {}
                     LongPoint.newRangeQuery("t", Long.MIN_VALUE, Long.MAX_VALUE)
                 );
                 IndexSearcher searcher = newIndexSearcher(limitedReader);
+                int segmentsWithLiveDocs = (int) searcher.getIndexReader()
+                    .leaves()
+                    .stream()
+                    .map(LeafReaderContext::reader)
+                    .filter(leafReader -> leafReader.getLiveDocs() != null)
+                    .count();
 
                 debugTestCase(
                     builder,
@@ -807,7 +825,7 @@ public void onCache(ShardId shardId, Accountable accountable) {}
                                 matchesMap().entry("segments_counted", greaterThanOrEqualTo(1))
                                     .entry("segments_collected", 0)
                                     .entry("segments_with_doc_count_field", 0)
-                                    .entry("segments_with_deleted_docs", 0)
+                                    .entry("segments_with_deleted_docs", segmentsWithLiveDocs)
                                     .entry(
                                         "filters",
                                         matchesList().item(
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java
index ec30b125fe93..dc4b3c32b3ed 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.search.aggregations.bucket.geogrid;
 
 import org.elasticsearch.common.unit.DistanceUnit;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.geo.GeometryTestUtils;
 import org.elasticsearch.geometry.Rectangle;
 import org.elasticsearch.test.ESTestCase;
@@ -23,7 +24,7 @@
 public class GeoHashGridParserTests extends ESTestCase {
     public void testParseValidFromInts() throws Exception {
         int precision = randomIntBetween(1, 12);
-        XContentParser stParser = createParser(JsonXContent.jsonXContent, formatted("""
+        XContentParser stParser = createParser(JsonXContent.jsonXContent, Strings.format("""
             {"field":"my_loc", "precision":%s, "size": 500, "shard_size": 550}""", precision));
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
@@ -33,7 +34,7 @@ public void testParseValidFromInts() throws Exception {
 
     public void testParseValidFromStrings() throws Exception {
         int precision = randomIntBetween(1, 12);
-        XContentParser stParser = createParser(JsonXContent.jsonXContent, formatted("""
+        XContentParser stParser = createParser(JsonXContent.jsonXContent, Strings.format("""
             {"field":"my_loc", "precision":"%s", "size": "500", "shard_size": "550"}
             """, precision));
         XContentParser.Token token = stParser.nextToken();
@@ -49,7 +50,7 @@ public void testParseDistanceUnitPrecision() throws Exception {
             distance = 5600 + randomDouble(); // 5.6cm is approx. smallest distance represented by precision 12
         }
         String distanceString = distance + unit.toString();
-        XContentParser stParser = createParser(JsonXContent.jsonXContent, formatted("""
+        XContentParser stParser = createParser(JsonXContent.jsonXContent, Strings.format("""
             {"field":"my_loc", "precision": "%s", "size": "500", "shard_size": "550"}
             """, distanceString));
         XContentParser.Token token = stParser.nextToken();
@@ -116,7 +117,7 @@ public void testParseErrorOnPrecisionOutOfRange() throws Exception {
 
     public void testParseValidBounds() throws Exception {
         Rectangle bbox = GeometryTestUtils.randomRectangle();
-        XContentParser stParser = createParser(JsonXContent.jsonXContent, formatted("""
+        XContentParser stParser = createParser(JsonXContent.jsonXContent, Strings.format("""
             {
               "field": "my_loc",
               "precision": 5,
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java
index e56ca35675f4..6a34eb104fd1 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java
@@ -60,7 +60,7 @@ protected GeoBoundingBox randomBBox() {
 
     @Override
     protected Rectangle getTile(double lng, double lat, int precision) {
-        long tiles = 1 << precision;
+        int tiles = 1 << precision;
         int x = GeoTileUtils.getXTile(lng, tiles);
         int y = GeoTileUtils.getYTile(lat, tiles);
         Rectangle r1 = GeoTileUtils.toBoundingBox(x, y, precision);
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java
index f91600db7e67..fbec707f79c5 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.search.aggregations.bucket.geogrid;
 
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.geo.GeometryTestUtils;
 import org.elasticsearch.geometry.Rectangle;
 import org.elasticsearch.test.ESTestCase;
@@ -21,7 +22,7 @@
 public class GeoTileGridParserTests extends ESTestCase {
     public void testParseValidFromInts() throws Exception {
         int precision = randomIntBetween(0, GeoTileUtils.MAX_ZOOM);
-        XContentParser stParser = createParser(JsonXContent.jsonXContent, formatted("""
+        XContentParser stParser = createParser(JsonXContent.jsonXContent, Strings.format("""
             {"field":"my_loc", "precision":%s, "size": 500, "shard_size": 550}""", precision));
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
@@ -31,7 +32,7 @@ public void testParseValidFromInts() throws Exception {
 
     public void testParseValidFromStrings() throws Exception {
         int precision = randomIntBetween(0, GeoTileUtils.MAX_ZOOM);
-        XContentParser stParser = createParser(JsonXContent.jsonXContent, formatted("""
+        XContentParser stParser = createParser(JsonXContent.jsonXContent, Strings.format("""
             {"field":"my_loc", "precision":"%s", "size": "500", "shard_size": "550"}
             """, precision));
         XContentParser.Token token = stParser.nextToken();
@@ -71,7 +72,7 @@ public void testParseErrorOnPrecisionOutOfRange() throws Exception {
 
     public void testParseValidBounds() throws Exception {
         Rectangle bbox = GeometryTestUtils.randomRectangle();
-        XContentParser stParser = createParser(JsonXContent.jsonXContent, formatted("""
+        XContentParser stParser = createParser(JsonXContent.jsonXContent, Strings.format("""
             {
               "field": "my_loc",
               "precision": 5,
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
index c86fdb8a315a..4add53513278 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
@@ -520,13 +521,13 @@ public void testNestedOrdering_random() throws IOException {
             for (int j = 0; j < numChapters; j++) {
                 chapters[j] = randomIntBetween(2, 64);
             }
-            books.add(Tuple.tuple(formatted("%03d", i), chapters));
+            books.add(Tuple.tuple(Strings.format("%03d", i), chapters));
         }
         try (Directory directory = newDirectory()) {
             try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {
                 int id = 0;
                 for (Tuple<String, int[]> book : books) {
-                    iw.addDocuments(generateBook(formatted("%03d", id), new String[] { book.v1() }, book.v2()));
+                    iw.addDocuments(generateBook(Strings.format("%03d", id), new String[] { book.v1() }, book.v2()));
                     id++;
                 }
             }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/prefix/InternalIpPrefixTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/prefix/InternalIpPrefixTests.java
index 7266eb921827..079953f64f8b 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/prefix/InternalIpPrefixTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/prefix/InternalIpPrefixTests.java
@@ -178,4 +178,9 @@ public void testPartialReduceNoMinDocCount() {
         assertThat(reduced.getBuckets().get(0).getDocCount(), equalTo(1L));
         assertThat(reduced.getBuckets().get(1).getDocCount(), equalTo(2L));
     }
+
+    @Override
+    protected InternalIpPrefix mutateInstance(InternalIpPrefix instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java
index 83679268cb8c..6c3d5b03ffdc 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilderTests.java
@@ -64,7 +64,7 @@ protected RangeAggregationBuilder createTestInstance() {
     }
 
     @Override
-    protected RangeAggregationBuilder mutateInstance(RangeAggregationBuilder builder) throws IOException {
+    protected RangeAggregationBuilder mutateInstance(RangeAggregationBuilder builder) {
         String name = builder.getName();
         boolean keyed = builder.keyed();
         String field = builder.field();
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java
index 7c4d565499d3..85481d94f257 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
+import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.script.LongFieldScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.StringFieldScript;
@@ -35,7 +36,6 @@
 import org.elasticsearch.search.aggregations.CardinalityUpperBound;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
 import org.elasticsearch.search.runtime.StringScriptFieldTermQuery;
 
 import java.io.IOException;
@@ -598,8 +598,8 @@ public void testOverlappingRanges() throws IOException {
      */
     public void testRuntimeFieldTopLevelQueryNotOptimized() throws IOException {
         long totalDocs = (long) RangeAggregator.DOCS_PER_RANGE_TO_USE_FILTERS * 4;
-        SearchLookup lookup = new SearchLookup(s -> null, (ft, l, ftd) -> null, new SourceLookup.ReaderSourceProvider());
-        StringFieldScript.LeafFactory scriptFactory = ctx -> new StringFieldScript("dummy", Map.of(), lookup, ctx) {
+        SearchLookup lookup = new SearchLookup(s -> null, (ft, l, ftd) -> null, (ctx, doc) -> null);
+        StringFieldScript.LeafFactory scriptFactory = ctx -> new StringFieldScript("dummy", Map.of(), lookup, OnScriptError.FAIL, ctx) {
             @Override
             public void execute() {
                 emit("cat");
@@ -641,13 +641,19 @@ public void execute() {
      */
     public void testRuntimeFieldRangesNotOptimized() throws IOException {
         long totalDocs = (long) RangeAggregator.DOCS_PER_RANGE_TO_USE_FILTERS * 4;
-        LongFieldScript.Factory scriptFactory = (fieldName, params, l) -> ctx -> new LongFieldScript(fieldName, Map.of(), l, ctx) {
+        LongFieldScript.Factory scriptFactory = (fieldName, params, l, onScriptError) -> ctx -> new LongFieldScript(
+            fieldName,
+            Map.of(),
+            l,
+            OnScriptError.FAIL,
+            ctx
+        ) {
             @Override
             public void execute() {
                 emit((long) getDoc().get(NUMBER_FIELD_NAME).get(0));
             }
         };
-        MappedFieldType dummyFt = new LongScriptFieldType("dummy", scriptFactory, new Script("test"), Map.of());
+        MappedFieldType dummyFt = new LongScriptFieldType("dummy", scriptFactory, new Script("test"), Map.of(), OnScriptError.FAIL);
         MappedFieldType numberFt = new NumberFieldMapper.NumberFieldType(NUMBER_FIELD_NAME, NumberFieldMapper.NumberType.INTEGER);
         debugTestCase(
             new RangeAggregationBuilder("r").field("dummy").addRange(0, 1).addRange(1, 2).addRange(2, 3),
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java
index 70f8598da6e1..a072855ffbcc 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java
@@ -708,7 +708,7 @@ private void addMixedTextDocs(IndexWriter w) throws IOException {
             }
 
             doc.add(new Field("text", text.toString(), TextFieldMapper.Defaults.FIELD_TYPE));
-            String json = formatted("""
+            String json = Strings.format("""
                 { "text" : "%s" }""", text.toString());
             doc.add(new StoredField("_source", new BytesRef(json)));
 
@@ -727,7 +727,7 @@ private void addManyMixedTextDocs(IndexWriter w) throws IOException {
             }
 
             doc.add(new Field("text", text.toString(), TextFieldMapper.Defaults.FIELD_TYPE));
-            String json = formatted("""
+            String json = Strings.format("""
                 { "text" : "%s" }""", text.toString());
             doc.add(new StoredField("_source", new BytesRef(json)));
 
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
index c343dd9613d9..19a59dd21354 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
@@ -44,6 +44,7 @@
 import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType;
 import org.elasticsearch.index.mapper.DocCountFieldMapper;
 import org.elasticsearch.index.mapper.GeoPointFieldMapper;
@@ -60,6 +61,7 @@
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
 import org.elasticsearch.index.mapper.ObjectMapper;
+import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
 import org.elasticsearch.index.mapper.RangeFieldMapper;
 import org.elasticsearch.index.mapper.RangeType;
@@ -112,7 +114,6 @@
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
 import org.elasticsearch.search.runtime.StringScriptFieldTermQuery;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.ScoreSortBuilder;
@@ -331,7 +332,7 @@ public void testManyTerms() throws Exception {
              */
             List<List<? extends IndexableField>> docs = new ArrayList<>();
             for (int i = 0; i < TermsAggregatorFactory.MAX_ORDS_TO_TRY_FILTERS - 200; i++) {
-                String s = formatted("b%03d", i);
+                String s = Strings.format("b%03d", i);
                 docs.add(doc(fieldType, s));
                 if (i % 100 == 7) {
                     docs.add(doc(fieldType, s));
@@ -365,7 +366,7 @@ public void testManyTermsOrderBySubAgg() throws Exception {
              */
             List<List<? extends IndexableField>> docs = new ArrayList<>();
             for (int i = 0; i < TermsAggregatorFactory.MAX_ORDS_TO_TRY_FILTERS - 200; i++) {
-                String s = formatted("b%03d", i);
+                String s = Strings.format("b%03d", i);
                 List<IndexableField> doc = doc(kft, s);
                 doc.add(new SortedNumericDocValuesField("long", i));
                 docs.add(doc);
@@ -375,7 +376,7 @@ public void testManyTermsOrderBySubAgg() throws Exception {
             List<String> expected = LongStream.range(
                 TermsAggregatorFactory.MAX_ORDS_TO_TRY_FILTERS - 210,
                 TermsAggregatorFactory.MAX_ORDS_TO_TRY_FILTERS - 200
-            ).mapToObj(l -> formatted("b%03d", l)).collect(toList());
+            ).mapToObj(l -> Strings.format("b%03d", l)).collect(toList());
             Collections.reverse(expected);
             assertThat(result.getBuckets().stream().map(StringTerms.Bucket::getKey).collect(toList()), equalTo(expected));
         }, new AggTestConfig(aggregationBuilder, kft, lft));
@@ -395,9 +396,9 @@ public void testDelaysSubAggs() throws Exception {
             .subAggregation(new TermsAggregationBuilder("sub").field("string2").shardSize(500));
         withIndex(iw -> {
             for (int i1 = 0; i1 < 1000; i1++) {
-                String s1 = formatted("b%03d", i1);
+                String s1 = Strings.format("b%03d", i1);
                 for (int i2 = 0; i2 < 50; i2++) {
-                    String s2 = formatted("b%03d", i2);
+                    String s2 = Strings.format("b%03d", i2);
                     List<IndexableField> doc = new ArrayList<>();
                     doc.addAll(doc(s1ft, s1));
                     doc.addAll(doc(s2ft, s2));
@@ -1993,8 +1994,8 @@ public void testWithFilterAndPreciseSize() throws IOException {
      */
     public void testRuntimeFieldTopLevelNotOptimized() throws IOException {
         long totalDocs = 500;
-        SearchLookup lookup = new SearchLookup(s -> null, (ft, l, ftd) -> null, new SourceLookup.ReaderSourceProvider());
-        StringFieldScript.LeafFactory scriptFactory = ctx -> new StringFieldScript("dummy", Map.of(), lookup, ctx) {
+        SearchLookup lookup = new SearchLookup(s -> null, (ft, l, ftd) -> null, (ctx, doc) -> null);
+        StringFieldScript.LeafFactory scriptFactory = ctx -> new StringFieldScript("dummy", Map.of(), lookup, OnScriptError.FAIL, ctx) {
             @Override
             public void execute() {
                 emit("cat");
@@ -2043,10 +2044,11 @@ public void execute() {
      */
     public void testRuntimeFieldTermsNotOptimized() throws IOException {
         long totalDocs = 500;
-        StringFieldScript.Factory scriptFactory = (fieldName, params, lookup) -> ctx -> new StringFieldScript(
+        StringFieldScript.Factory scriptFactory = (fieldName, params, lookup, onScriptError) -> ctx -> new StringFieldScript(
             fieldName,
             Map.of(),
             lookup,
+            OnScriptError.FAIL,
             ctx
         ) {
             @Override
@@ -2056,7 +2058,7 @@ public void execute() {
         };
         BytesRef[] values = new BytesRef[] { new BytesRef("stuff"), new BytesRef("more_stuff"), new BytesRef("other_stuff"), };
         MappedFieldType keywordFt = new KeywordFieldType("k", true, true, Collections.emptyMap());
-        MappedFieldType dummyFt = new KeywordScriptFieldType("dummy", scriptFactory, new Script("test"), Map.of());
+        MappedFieldType dummyFt = new KeywordScriptFieldType("dummy", scriptFactory, new Script("test"), Map.of(), OnScriptError.FAIL);
         debugTestCase(new TermsAggregationBuilder("t").field("dummy"), new MatchAllDocsQuery(), iw -> {
             for (int d = 0; d < totalDocs; d++) {
                 BytesRef value = values[d % values.length];
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMedianAbsoluteDeviationTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMedianAbsoluteDeviationTests.java
index cd5afc42e863..5509695bb2fd 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMedianAbsoluteDeviationTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMedianAbsoluteDeviationTests.java
@@ -73,7 +73,7 @@ protected void assertFromXContent(InternalMedianAbsoluteDeviation internalMAD, P
     }
 
     @Override
-    protected InternalMedianAbsoluteDeviation mutateInstance(InternalMedianAbsoluteDeviation instance) throws IOException {
+    protected InternalMedianAbsoluteDeviation mutateInstance(InternalMedianAbsoluteDeviation instance) {
         String name = instance.getName();
         TDigestState valuesSketch = instance.getValuesSketch();
         Map<String, Object> metadata = instance.getMetadata();
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetricTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetricTests.java
index 000c66d44f52..45ad84990a45 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetricTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetricTests.java
@@ -247,7 +247,7 @@ protected Predicate<String> excludePathsFromXContentInsertion() {
     }
 
     @Override
-    protected InternalScriptedMetric mutateInstance(InternalScriptedMetric instance) throws IOException {
+    protected InternalScriptedMetric mutateInstance(InternalScriptedMetric instance) {
         String name = instance.getName();
         List<Object> aggregationsList = instance.aggregationsList();
         Script reduceScript = instance.reduceScript;
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java
index 03341670d6db..a0c50ffd6b2e 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalStatsTests.java
@@ -227,23 +227,14 @@ public void testDoXContentBody() throws IOException {
         internalStats.doXContentBody(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
 
-        String expected = formatted(
-            """
-                {
-                  "count" : %s,
-                  "min" : %s,
-                  "max" : %s,
-                  "avg" : %s,
-                  "sum" : %s
-                  %s
-                }""",
+        Object[] args = new Object[] {
             count,
             min,
             max,
             internalStats.getAvg(),
             sum,
             format != DocValueFormat.RAW
-                ? formatted(
+                ? Strings.format(
                     """
                         ,
                         "min_as_string" : "%s",
@@ -256,8 +247,16 @@ public void testDoXContentBody() throws IOException {
                     format.format(internalStats.getAvg()),
                     format.format(internalStats.getSum())
                 )
-                : ""
-        );
+                : "" };
+        String expected = Strings.format("""
+            {
+              "count" : %s,
+              "min" : %s,
+              "max" : %s,
+              "avg" : %s,
+              "sum" : %s
+              %s
+            }""", args);
         assertEquals(XContentHelper.stripWhitespace(expected), Strings.toString(builder));
 
         // count is zero
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregationBuilderTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregationBuilderTests.java
index bc8f488fa07c..2b4cd8e8f871 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregationBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregationBuilderTests.java
@@ -59,6 +59,11 @@ protected WeightedAvgAggregationBuilder createTestInstance() {
         return aggregationBuilder;
     }
 
+    @Override
+    protected WeightedAvgAggregationBuilder mutateInstance(WeightedAvgAggregationBuilder instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<WeightedAvgAggregationBuilder> instanceReader() {
         return WeightedAvgAggregationBuilder::new;
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java
index 5605e68a7d34..27314a8112f1 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java
@@ -35,6 +35,35 @@
 import static org.hamcrest.Matchers.equalTo;
 
 public class IncludeExcludeTests extends ESTestCase {
+
+    public static IncludeExclude randomIncludeExclude() {
+        switch (randomInt(7)) {
+            case 0:
+                return new IncludeExclude("incl*de", null, null, null);
+            case 1:
+                return new IncludeExclude("incl*de", "excl*de", null, null);
+            case 2:
+                return new IncludeExclude("incl*de", null, null, new TreeSet<>(Set.of(newBytesRef("exclude"))));
+            case 3:
+                return new IncludeExclude(null, "excl*de", null, null);
+            case 4:
+                return new IncludeExclude(null, "excl*de", new TreeSet<>(Set.of(newBytesRef("include"))), null);
+            case 5:
+                return new IncludeExclude(null, null, new TreeSet<>(Set.of(newBytesRef("include"))), null);
+            case 6:
+                return new IncludeExclude(
+                    null,
+                    null,
+                    new TreeSet<>(Set.of(newBytesRef("include"))),
+                    new TreeSet<>(Set.of(newBytesRef("exclude")))
+                );
+            case 7:
+                return new IncludeExclude(null, null, null, new TreeSet<>(Set.of(newBytesRef("exclude"))));
+            default:
+                throw new IllegalArgumentException("got unexpected parameter, expected 0 <= x <= 7");
+        }
+    }
+
     public void testEmptyTermsWithOrds() throws IOException {
         IncludeExclude inexcl = new IncludeExclude(null, null, new TreeSet<>(Set.of(new BytesRef("foo"))), null);
         OrdinalsFilter filter = inexcl.convertToOrdinalsFilter(DocValueFormat.RAW);
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceFieldConfigTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceFieldConfigTests.java
index 05569547bd24..d30850fe2358 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceFieldConfigTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceFieldConfigTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentParser;
@@ -30,7 +31,7 @@ public class MultiValuesSourceFieldConfigTests extends AbstractXContentSerializi
 
     @Override
     protected MultiValuesSourceFieldConfig doParseInstance(XContentParser parser) throws IOException {
-        return MultiValuesSourceFieldConfig.parserBuilder(true, true, true, true).apply(parser, null).build();
+        return MultiValuesSourceFieldConfig.parserBuilder(true, true, true, true, true).apply(parser, null).build();
     }
 
     @Override
@@ -43,6 +44,7 @@ protected MultiValuesSourceFieldConfig createTestInstance() {
         ValueType userValueTypeHint = randomBoolean()
             ? randomFrom(ValueType.STRING, ValueType.DOUBLE, ValueType.LONG, ValueType.DATE, ValueType.IP, ValueType.BOOLEAN)
             : null;
+        IncludeExclude includeExclude = randomBoolean() ? IncludeExcludeTests.randomIncludeExclude() : null;
         return new MultiValuesSourceFieldConfig.Builder().setFieldName(field)
             .setMissing(missing)
             .setScript(null)
@@ -50,9 +52,15 @@ protected MultiValuesSourceFieldConfig createTestInstance() {
             .setFilter(filter)
             .setFormat(format)
             .setUserValueTypeHint(userValueTypeHint)
+            .setIncludeExclude(includeExclude)
             .build();
     }
 
+    @Override
+    protected MultiValuesSourceFieldConfig mutateInstance(MultiValuesSourceFieldConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<MultiValuesSourceFieldConfig> instanceReader() {
         return MultiValuesSourceFieldConfig::new;
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java
index 85bc4e2d00cf..ff6b4ffcdc28 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.search.aggregations.support.values.ScriptLongValues;
 import org.elasticsearch.search.lookup.LeafSearchLookup;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -37,7 +36,7 @@ private static class FakeAggregationScript extends AggregationScript {
         int index;
 
         FakeAggregationScript(Object[][] values) {
-            super(Collections.emptyMap(), new SearchLookup(null, null, new SourceLookup.ReaderSourceProvider()) {
+            super(Collections.emptyMap(), new SearchLookup(null, null, (ctx, doc) -> null) {
 
                 @Override
                 public LeafSearchLookup getLeafSearchLookup(LeafReaderContext context) {
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java
index 928c8c77e0e4..0a296383b457 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java
@@ -405,4 +405,24 @@ public void testFieldAlias() throws Exception {
             assertTrue(config.alignesWithSearchIndex());
         });
     }
+
+    public void testFlattened() throws Exception {
+        MapperService mapperService = createMapperService(fieldMapping(b -> b.field("type", "flattened")));
+        withAggregationContext(mapperService, List.of(source(b -> b.startObject("field").field("key", "abc").endObject())), context -> {
+            ValuesSourceConfig config;
+            config = ValuesSourceConfig.resolve(context, null, "field.key", null, null, null, null, CoreValuesSourceType.KEYWORD);
+
+            ValuesSource.Bytes valuesSource = (ValuesSource.Bytes) config.getValuesSource();
+            LeafReaderContext ctx = context.searcher().getIndexReader().leaves().get(0);
+            SortedBinaryDocValues values = valuesSource.bytesValues(ctx);
+            assertTrue(values.advanceExact(0));
+            assertEquals(1, values.docValueCount());
+            assertEquals(new BytesRef("abc"), values.nextValue());
+
+            // flattened supports ordinals, but _not_ global ordinals
+            assertTrue(config.hasOrdinals());
+            ValuesSource.Bytes.WithOrdinals valuesSourceWithOrdinals = (ValuesSource.Bytes.WithOrdinals) valuesSource;
+            assertFalse(valuesSourceWithOrdinals.supportsGlobalOrdinalsMapping());
+        });
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/builder/PointInTimeBuilderTests.java b/server/src/test/java/org/elasticsearch/search/builder/PointInTimeBuilderTests.java
index 87033902dd98..7c26ddbbbdb7 100644
--- a/server/src/test/java/org/elasticsearch/search/builder/PointInTimeBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/builder/PointInTimeBuilderTests.java
@@ -33,4 +33,9 @@ protected PointInTimeBuilder createTestInstance() {
         }
         return pointInTime;
     }
+
+    @Override
+    protected PointInTimeBuilder mutateInstance(PointInTimeBuilder instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index 0c6e62019c4a..1ee8fe3e0829 100644
--- a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.search.builder;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.admin.cluster.stats.SearchUsageStats;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
@@ -45,7 +45,7 @@
 import org.elasticsearch.search.vectors.KnnSearchBuilder;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.EqualsHashCodeTestUtils;
-import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.usage.SearchUsageHolder;
 import org.elasticsearch.usage.UsageService;
 import org.elasticsearch.xcontent.ToXContent;
@@ -58,6 +58,7 @@
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
@@ -116,9 +117,13 @@ public void testSerialization() throws IOException {
 
     public void testSerializingWithRuntimeFieldsBeforeSupportedThrows() {
         SearchSourceBuilder original = new SearchSourceBuilder().runtimeMappings(randomRuntimeMappings());
-        Version v = VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, VersionUtils.getPreviousVersion(Version.V_7_11_0));
+        TransportVersion v = TransportVersionUtils.randomVersionBetween(
+            random(),
+            TransportVersion.V_7_0_0,
+            TransportVersionUtils.getPreviousVersion(TransportVersion.V_7_11_0)
+        );
         Exception e = expectThrows(IllegalArgumentException.class, () -> copyBuilder(original, v));
-        assertThat(e.getMessage(), equalTo("Versions before 7.11.0 don't support [runtime_mappings] and search was sent to [" + v + "]"));
+        assertThat(e.getMessage(), equalTo("Versions before 7110099 don't support [runtime_mappings] and search was sent to [" + v + "]"));
     }
 
     public void testShallowCopy() {
@@ -135,10 +140,10 @@ public void testEqualsAndHashcode() throws IOException {
     }
 
     private SearchSourceBuilder copyBuilder(SearchSourceBuilder original) throws IOException {
-        return copyBuilder(original, Version.CURRENT);
+        return copyBuilder(original, TransportVersion.CURRENT);
     }
 
-    private SearchSourceBuilder copyBuilder(SearchSourceBuilder original, Version version) throws IOException {
+    private SearchSourceBuilder copyBuilder(SearchSourceBuilder original, TransportVersion version) throws IOException {
         return ESTestCase.copyWriteable(original, namedWriteableRegistry, SearchSourceBuilder::new, version);
     }
 
@@ -377,7 +382,7 @@ public void testParseRescore() throws IOException {
 
     public void testTimeoutWithUnits() throws IOException {
         final String timeout = randomTimeValue();
-        final String query = formatted("""
+        final String query = Strings.format("""
             { "query": { "match_all": {}}, "timeout": "%s"}
             """, timeout);
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, query)) {
@@ -388,7 +393,7 @@ public void testTimeoutWithUnits() throws IOException {
 
     public void testTimeoutWithoutUnits() throws IOException {
         final int timeout = randomIntBetween(1, 1024);
-        final String query = formatted("""
+        final String query = Strings.format("""
             { "query": { "match_all": {}}, "timeout": "%s"}
             """, timeout);
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, query)) {
@@ -739,7 +744,7 @@ public void testSearchSectionsUsageCollection() throws IOException {
         searchSourceBuilder.fetchField("field");
         // these are not correct runtime mappings but they are counted compared to empty object
         searchSourceBuilder.runtimeMappings(Collections.singletonMap("field", "keyword"));
-        searchSourceBuilder.knnSearch(new KnnSearchBuilder("field", new float[] {}, 2, 5));
+        searchSourceBuilder.knnSearch(List.of(new KnnSearchBuilder("field", new float[] {}, 2, 5)));
         searchSourceBuilder.pointInTimeBuilder(new PointInTimeBuilder("pitid"));
         searchSourceBuilder.docValueField("field");
         searchSourceBuilder.storedField("field");
diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourceContextTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourceContextTests.java
index 14ce899dda9c..01dea381b5d2 100644
--- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourceContextTests.java
+++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourceContextTests.java
@@ -39,6 +39,11 @@ protected FetchSourceContext createTestInstance() {
         );
     }
 
+    @Override
+    protected FetchSourceContext mutateInstance(FetchSourceContext instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testFromXContentException() throws IOException {
         XContentBuilder builder = XContentFactory.jsonBuilder();
         int value = randomInt();
diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java
index 43de95cf9821..1a97d6dc37c7 100644
--- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java
+++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java
@@ -35,7 +35,7 @@
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.search.lookup.SourceFilter;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
@@ -247,10 +247,9 @@ public void testMetadataFields() throws IOException {
             LeafReaderContext readerContext = searcher.getIndexReader().leaves().get(0);
             fieldFetcher.setNextReader(readerContext);
 
-            SourceLookup sourceLookup = new SourceLookup(new SourceLookup.ReaderSourceProvider());
-            sourceLookup.setSegmentAndDocument(readerContext, 0);
+            Source s = SourceProvider.fromStoredFields().getSource(readerContext, 0);
 
-            Map<String, DocumentField> fetchedFields = fieldFetcher.fetch(sourceLookup, 0);
+            Map<String, DocumentField> fetchedFields = fieldFetcher.fetch(s, 0);
             assertThat(fetchedFields.size(), equalTo(5));
             assertEquals(docId, fetchedFields.get("_id").getValue());
             assertEquals(routing, fetchedFields.get("_routing").getValue());
@@ -1143,7 +1142,8 @@ public void testFetchRuntimeFieldWithSourceDisabled() throws IOException {
             IndexSearcher searcher = newSearcher(iw);
             LeafReaderContext readerContext = searcher.getIndexReader().leaves().get(0);
             fieldFetcher.setNextReader(readerContext);
-            Map<String, DocumentField> fields = fieldFetcher.fetch(new SourceLookup(new SourceLookup.ReaderSourceProvider()), 0);
+            Source source = SourceProvider.fromStoredFields().getSource(readerContext, 0);
+            Map<String, DocumentField> fields = fieldFetcher.fetch(source, 0);
             assertEquals(1, fields.size());
             DocumentField field = fields.get("runtime_field");
             assertEquals(1L, (long) field.getValue());
@@ -1175,9 +1175,7 @@ public void testFetchMetadataFieldWithSourceDisabled() throws IOException {
             IndexSearcher searcher = newSearcher(iw);
             LeafReaderContext readerContext = searcher.getIndexReader().leaves().get(0);
             fieldFetcher.setNextReader(readerContext);
-            SourceLookup sourceLookup = new SourceLookup(new SourceLookup.ReaderSourceProvider());
-            sourceLookup.setSegmentAndDocument(readerContext, 0);
-            Map<String, DocumentField> fields = fieldFetcher.fetch(sourceLookup, 0);
+            Map<String, DocumentField> fields = fieldFetcher.fetch(Source.empty(XContentType.JSON), 0);
             assertEquals(1, fields.size());
             DocumentField field = fields.get("_id");
             assertEquals("1", field.getValue());
diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/LookupFieldTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/LookupFieldTests.java
index 42d5d8b09af7..b0833a33cc80 100644
--- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/LookupFieldTests.java
+++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/LookupFieldTests.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.junit.Before;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -75,7 +74,7 @@ private static List<FieldAndFormat> randomFetchFields() {
     }
 
     @Override
-    protected LookupField mutateInstance(LookupField old) throws IOException {
+    protected LookupField mutateInstance(LookupField old) {
         String lookupIndex = old.targetIndex();
         QueryBuilder query = old.query();
         List<FieldAndFormat> fetchFields = old.fetchFields();
diff --git a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java
index 23a951e20ab7..34ea932da040 100644
--- a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java
+++ b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java
@@ -16,6 +16,7 @@
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FilterDirectoryReader;
+import org.apache.lucene.index.FilterLeafReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
@@ -24,6 +25,7 @@
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.PostingsEnum;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.BulkScorer;
@@ -271,6 +273,43 @@ public void onRemoval(ShardId shardId, Accountable accountable) {
         IOUtils.close(reader, w, dir);
     }
 
+    public void testExitableTermsMinAndMax() throws IOException {
+        Directory dir = newDirectory();
+        IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(null));
+        Document doc = new Document();
+        StringField fooField = new StringField("foo", "bar", Field.Store.NO);
+        doc.add(fooField);
+        w.addDocument(doc);
+        w.flush();
+
+        DirectoryReader directoryReader = DirectoryReader.open(w);
+        for (LeafReaderContext lfc : directoryReader.leaves()) {
+            Terms terms = lfc.reader().terms("foo");
+            FilterLeafReader.FilterTerms filterTerms = new ExitableTerms(terms, new ExitableDirectoryReader.QueryCancellation() {
+                @Override
+                public boolean isEnabled() {
+                    return false;
+                }
+
+                @Override
+                public void checkCancelled() {
+
+                }
+            }) {
+                @Override
+                public TermsEnum iterator() {
+                    fail("Retrieving min and max should retrieve values from block tree instead of iterating");
+                    return null;
+                }
+            };
+            assertEquals("bar", filterTerms.getMin().utf8ToString());
+            assertEquals("bar", filterTerms.getMax().utf8ToString());
+        }
+        w.close();
+        directoryReader.close();
+        dir.close();
+    }
+
     private SparseFixedBitSet query(LeafReaderContext leaf, String field, String value) throws IOException {
         SparseFixedBitSet sparseFixedBitSet = new SparseFixedBitSet(leaf.reader().maxDoc());
         TermsEnum tenum = leaf.reader().terms(field).iterator();
diff --git a/server/src/test/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReaderTests.java b/server/src/test/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReaderTests.java
new file mode 100644
index 000000000000..f17cbf83aa4e
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReaderTests.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.internal;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class FieldUsageTrackingDirectoryReaderTests extends ESTestCase {
+
+    public void testTermsMinAndMax() throws IOException {
+        Directory dir = newDirectory();
+        IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(null));
+        Document doc = new Document();
+        StringField fooField = new StringField("foo", "bar", Field.Store.NO);
+        doc.add(fooField);
+        w.addDocument(doc);
+        w.flush();
+
+        DirectoryReader directoryReader = DirectoryReader.open(w);
+        for (LeafReaderContext lrc : directoryReader.leaves()) {
+            FieldUsageTrackingDirectoryReader.FieldUsageTrackingLeafReader leafReader =
+                new FieldUsageTrackingDirectoryReader.FieldUsageTrackingLeafReader(lrc.reader(), new TestFieldUsageNotifier());
+            FieldUsageTrackingDirectoryReader.FieldUsageTrackingLeafReader.FieldUsageTrackingTerms terms =
+                leafReader.new FieldUsageTrackingTerms("foo", lrc.reader().terms("foo")) {
+                    @Override
+                    public TermsEnum iterator() {
+                        fail("Retrieving min and max should retrieve values from block tree instead of iterating");
+                        return null;
+                    }
+                };
+            assertEquals("bar", terms.getMin().utf8ToString());
+            assertEquals("bar", terms.getMax().utf8ToString());
+        }
+        w.close();
+        directoryReader.close();
+        dir.close();
+    }
+
+    private static class TestFieldUsageNotifier implements FieldUsageTrackingDirectoryReader.FieldUsageNotifier {
+        @Override
+        public void onTermsUsed(String field) {
+
+        }
+
+        @Override
+        public void onPostingsUsed(String field) {
+
+        }
+
+        @Override
+        public void onTermFrequenciesUsed(String field) {
+
+        }
+
+        @Override
+        public void onPositionsUsed(String field) {
+
+        }
+
+        @Override
+        public void onOffsetsUsed(String field) {
+
+        }
+
+        @Override
+        public void onDocValuesUsed(String field) {
+
+        }
+
+        @Override
+        public void onStoredFieldsUsed(String field) {
+
+        }
+
+        @Override
+        public void onNormsUsed(String field) {
+
+        }
+
+        @Override
+        public void onPayloadsUsed(String field) {
+
+        }
+
+        @Override
+        public void onPointsUsed(String field) {
+
+        }
+
+        @Override
+        public void onTermVectorsUsed(String field) {
+
+        }
+
+        @Override
+        public void onKnnVectorsUsed(String field) {
+
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/internal/ShardSearchRequestTests.java b/server/src/test/java/org/elasticsearch/search/internal/ShardSearchRequestTests.java
index 2f1b23d2c27d..11a4f4d180a0 100644
--- a/server/src/test/java/org/elasticsearch/search/internal/ShardSearchRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/search/internal/ShardSearchRequestTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.search.internal;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.search.SearchRequest;
@@ -29,7 +30,8 @@
 import org.elasticsearch.indices.InvalidAliasNameException;
 import org.elasticsearch.search.AbstractSearchTestCase;
 import org.elasticsearch.search.SearchSortValuesAndFormatsTests;
-import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.xcontent.DeprecationHandler;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -38,6 +40,8 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.List;
+import java.util.Optional;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.elasticsearch.index.query.AbstractQueryBuilder.parseTopLevelQuery;
@@ -215,16 +219,20 @@ public QueryBuilder aliasFilter(IndexMetadata indexMetadata, String... aliasName
 
     public void testChannelVersion() throws Exception {
         ShardSearchRequest request = createShardSearchRequest();
-        Version channelVersion = Version.CURRENT;
+        TransportVersion channelVersion = TransportVersion.CURRENT;
         assertThat(request.getChannelVersion(), equalTo(channelVersion));
         int iterations = between(0, 5);
         // New version
         for (int i = 0; i < iterations; i++) {
-            Version version = request.isForceSyntheticSource()
-                ? VersionUtils.randomVersionBetween(random(), Version.V_8_4_0, Version.CURRENT)
-                : VersionUtils.randomCompatibleVersion(random(), Version.CURRENT);
+            TransportVersion version = TransportVersionUtils.randomCompatibleVersion(random(), TransportVersion.CURRENT);
+            if (request.isForceSyntheticSource()) {
+                version = TransportVersionUtils.randomVersionBetween(random(), TransportVersion.V_8_4_0, TransportVersion.CURRENT);
+            }
+            if (Optional.ofNullable(request.source()).map(SearchSourceBuilder::knnSearch).map(List::size).orElse(0) > 1) {
+                version = TransportVersionUtils.randomVersionBetween(random(), TransportVersion.V_8_7_0, TransportVersion.CURRENT);
+            }
             request = copyWriteable(request, namedWriteableRegistry, ShardSearchRequest::new, version);
-            channelVersion = Version.min(channelVersion, version);
+            channelVersion = TransportVersion.min(channelVersion, version);
             assertThat(request.getChannelVersion(), equalTo(channelVersion));
             if (randomBoolean()) {
                 request = new ShardSearchRequest(request);
@@ -248,7 +256,7 @@ public void testForceSyntheticUnsupported() throws IOException {
         request.setForceSyntheticSource(true);
         ShardSearchRequest shardRequest = createShardSearchReqest(request);
         StreamOutput out = new BytesStreamOutput();
-        out.setVersion(Version.V_8_3_0);
+        out.setTransportVersion(TransportVersion.V_8_3_0);
         Exception e = expectThrows(IllegalArgumentException.class, () -> shardRequest.writeTo(out));
         assertEquals(e.getMessage(), "force_synthetic_source is not supported before 8.4.0");
     }
diff --git a/server/src/test/java/org/elasticsearch/search/lookup/SourceLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/SourceLookupTests.java
deleted file mode 100644
index e1f38f6aa69e..000000000000
--- a/server/src/test/java/org/elasticsearch/search/lookup/SourceLookupTests.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.search.lookup;
-
-import org.apache.lucene.codecs.StoredFieldsReader;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.StoredField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.tests.index.RandomIndexWriter;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader;
-import org.elasticsearch.test.ESTestCase;
-
-import java.io.IOException;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.instanceOf;
-
-public class SourceLookupTests extends ESTestCase {
-
-    public void testSetSegmentAndDocument() throws IOException {
-        try (Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) {
-            Document doc = new Document();
-            doc.add(new StringField("field", "value", Field.Store.YES));
-            doc.add(new StoredField("_source", new BytesRef("{\"field\": \"value\"}")));
-            iw.addDocument(doc);
-
-            try (IndexReader reader = iw.getReader()) {
-                LeafReaderContext readerContext = reader.leaves().get(0);
-
-                SourceLookup sourceLookup = new SourceLookup(new SourceLookup.ReaderSourceProvider());
-                sourceLookup.setSegmentAndDocument(readerContext, 0);
-                sourceLookup.source();
-                assertNotNull(sourceLookup.internalSourceRef());
-
-                // Source should be preserved if we pass in the same reader and document
-                sourceLookup.setSegmentAndDocument(readerContext, 0);
-                assertNotNull(sourceLookup.internalSourceRef());
-
-                // Check that the stored fields reader is not loaded eagerly
-                LeafReader throwingReader = new SequentialStoredFieldsLeafReader(readerContext.reader()) {
-                    @Override
-                    protected StoredFieldsReader doGetSequentialStoredFieldsReader(StoredFieldsReader reader) {
-                        throw new UnsupportedOperationException("attempted to load stored fields reader");
-                    }
-
-                    @Override
-                    public CacheHelper getReaderCacheHelper() {
-                        return in.getReaderCacheHelper();
-                    }
-
-                    @Override
-                    public CacheHelper getCoreCacheHelper() {
-                        return in.getCoreCacheHelper();
-                    }
-                };
-
-                sourceLookup.setSegmentAndDocument(throwingReader.getContext(), 0);
-                ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, sourceLookup::source);
-                assertThat(e.getCause(), instanceOf(UnsupportedOperationException.class));
-                assertThat(e.getCause().getMessage(), containsString("attempted to load stored fields reader"));
-            }
-        }
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/search/lookup/SourceProviderTests.java b/server/src/test/java/org/elasticsearch/search/lookup/SourceProviderTests.java
new file mode 100644
index 000000000000..96591f3e28ec
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/lookup/SourceProviderTests.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.lookup;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class SourceProviderTests extends ESTestCase {
+
+    public void testStoredFieldsSourceProvider() throws IOException {
+        try (Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) {
+            Document doc = new Document();
+            doc.add(new StringField("field", "value", Field.Store.YES));
+            doc.add(new StoredField("_source", new BytesRef("{\"field\": \"value\"}")));
+            iw.addDocument(doc);
+
+            try (IndexReader reader = iw.getReader()) {
+                LeafReaderContext readerContext = reader.leaves().get(0);
+
+                SourceProvider sourceProvider = SourceProvider.fromStoredFields();
+                Source source = sourceProvider.getSource(readerContext, 0);
+
+                assertNotNull(source.internalSourceRef());
+
+                // Source should be preserved if we pass in the same reader and document
+                Source s2 = sourceProvider.getSource(readerContext, 0);
+                assertSame(s2, source);
+            }
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/profile/ProfileResultTests.java b/server/src/test/java/org/elasticsearch/search/profile/ProfileResultTests.java
index bd829ab10cc5..49e89d3d4a87 100644
--- a/server/src/test/java/org/elasticsearch/search/profile/ProfileResultTests.java
+++ b/server/src/test/java/org/elasticsearch/search/profile/ProfileResultTests.java
@@ -59,6 +59,11 @@ protected ProfileResult createTestInstance() {
         return createTestItem(2);
     }
 
+    @Override
+    protected ProfileResult mutateInstance(ProfileResult instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<ProfileResult> instanceReader() {
         return ProfileResult::new;
diff --git a/server/src/test/java/org/elasticsearch/search/profile/SearchProfileDfsPhaseResultTests.java b/server/src/test/java/org/elasticsearch/search/profile/SearchProfileDfsPhaseResultTests.java
index 783fea30aad4..c89edb29b505 100644
--- a/server/src/test/java/org/elasticsearch/search/profile/SearchProfileDfsPhaseResultTests.java
+++ b/server/src/test/java/org/elasticsearch/search/profile/SearchProfileDfsPhaseResultTests.java
@@ -9,18 +9,25 @@
 package org.elasticsearch.search.profile;
 
 import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.search.profile.query.CollectorResult;
+import org.elasticsearch.search.profile.query.QueryProfileShardResult;
 import org.elasticsearch.search.profile.query.QueryProfileShardResultTests;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
 
 public class SearchProfileDfsPhaseResultTests extends AbstractXContentSerializingTestCase<SearchProfileDfsPhaseResult> {
 
     static SearchProfileDfsPhaseResult createTestItem() {
         return new SearchProfileDfsPhaseResult(
             randomBoolean() ? null : ProfileResultTests.createTestItem(1),
-            randomBoolean() ? null : QueryProfileShardResultTests.createTestItem()
+            randomBoolean() ? null : randomList(1, 10, QueryProfileShardResultTests::createTestItem)
         );
     }
 
@@ -29,6 +36,11 @@ protected SearchProfileDfsPhaseResult createTestInstance() {
         return createTestItem();
     }
 
+    @Override
+    protected SearchProfileDfsPhaseResult mutateInstance(SearchProfileDfsPhaseResult instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<SearchProfileDfsPhaseResult> instanceReader() {
         return SearchProfileDfsPhaseResult::new;
@@ -38,4 +50,26 @@ protected Reader<SearchProfileDfsPhaseResult> instanceReader() {
     protected SearchProfileDfsPhaseResult doParseInstance(XContentParser parser) throws IOException {
         return SearchProfileDfsPhaseResult.fromXContent(parser);
     }
+
+    public void testCombineQueryProfileShardResults() {
+        assertThat(new SearchProfileDfsPhaseResult(null, null).combineQueryProfileShardResults(), is(nullValue()));
+
+        List<QueryProfileShardResult> resultList = randomList(5, 5, QueryProfileShardResultTests::createTestItem);
+
+        SearchProfileDfsPhaseResult result = new SearchProfileDfsPhaseResult(null, resultList);
+        QueryProfileShardResult queryProfileShardResult = result.combineQueryProfileShardResults();
+        assertThat(
+            queryProfileShardResult.getRewriteTime(),
+            equalTo(resultList.stream().mapToLong(QueryProfileShardResult::getRewriteTime).sum())
+        );
+        assertThat(
+            queryProfileShardResult.getCollectorResult().getTime(),
+            equalTo(resultList.stream().map(QueryProfileShardResult::getCollectorResult).mapToLong(CollectorResult::getTime).sum())
+        );
+        assertThat(queryProfileShardResult.getCollectorResult().getProfiledChildren().size(), equalTo(resultList.size()));
+        assertThat(
+            queryProfileShardResult.getQueryResults().size(),
+            equalTo((int) resultList.stream().mapToLong(q -> q.getQueryResults().size()).sum())
+        );
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/profile/SearchProfileQueryPhaseResultTests.java b/server/src/test/java/org/elasticsearch/search/profile/SearchProfileQueryPhaseResultTests.java
index eb99b1ef26f9..b14412c70103 100644
--- a/server/src/test/java/org/elasticsearch/search/profile/SearchProfileQueryPhaseResultTests.java
+++ b/server/src/test/java/org/elasticsearch/search/profile/SearchProfileQueryPhaseResultTests.java
@@ -41,6 +41,11 @@ protected SearchProfileQueryPhaseResult createTestInstance() {
         return createTestItem();
     }
 
+    @Override
+    protected SearchProfileQueryPhaseResult mutateInstance(SearchProfileQueryPhaseResult instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<SearchProfileQueryPhaseResult> instanceReader() {
         return SearchProfileQueryPhaseResult::new;
diff --git a/server/src/test/java/org/elasticsearch/search/profile/SearchProfileResultsTests.java b/server/src/test/java/org/elasticsearch/search/profile/SearchProfileResultsTests.java
index 42ae9d6595e3..159ccccf7430 100644
--- a/server/src/test/java/org/elasticsearch/search/profile/SearchProfileResultsTests.java
+++ b/server/src/test/java/org/elasticsearch/search/profile/SearchProfileResultsTests.java
@@ -37,6 +37,11 @@ protected SearchProfileResults createTestInstance() {
         return createTestItem();
     }
 
+    @Override
+    protected SearchProfileResults mutateInstance(SearchProfileResults instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<SearchProfileResults> instanceReader() {
         return SearchProfileResults::new;
diff --git a/server/src/test/java/org/elasticsearch/search/profile/SearchProfileShardResultTests.java b/server/src/test/java/org/elasticsearch/search/profile/SearchProfileShardResultTests.java
index 99228843c817..565c9ac297dc 100644
--- a/server/src/test/java/org/elasticsearch/search/profile/SearchProfileShardResultTests.java
+++ b/server/src/test/java/org/elasticsearch/search/profile/SearchProfileShardResultTests.java
@@ -23,6 +23,11 @@ protected SearchProfileShardResult createTestInstance() {
         return createTestItem();
     }
 
+    @Override
+    protected SearchProfileShardResult mutateInstance(SearchProfileShardResult instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<SearchProfileShardResult> instanceReader() {
         return SearchProfileShardResult::new;
diff --git a/server/src/test/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResultTests.java b/server/src/test/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResultTests.java
index 531405344abb..1dec0978bdd0 100644
--- a/server/src/test/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResultTests.java
+++ b/server/src/test/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResultTests.java
@@ -44,6 +44,11 @@ protected AggregationProfileShardResult createTestInstance() {
         return createTestItem(2);
     }
 
+    @Override
+    protected AggregationProfileShardResult mutateInstance(AggregationProfileShardResult instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected AggregationProfileShardResult doParseInstance(XContentParser parser) throws IOException {
         XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/CollectorResultTests.java b/server/src/test/java/org/elasticsearch/search/profile/query/CollectorResultTests.java
index 9fb391499053..4786230df72c 100644
--- a/server/src/test/java/org/elasticsearch/search/profile/query/CollectorResultTests.java
+++ b/server/src/test/java/org/elasticsearch/search/profile/query/CollectorResultTests.java
@@ -47,6 +47,11 @@ protected CollectorResult createTestInstance() {
         return createTestItem(1);
     }
 
+    @Override
+    protected CollectorResult mutateInstance(CollectorResult instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected CollectorResult doParseInstance(XContentParser parser) throws IOException {
         ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/ProfileScorerTests.java b/server/src/test/java/org/elasticsearch/search/profile/query/ProfileScorerTests.java
index 9e0d4bdeae25..a478e80ca1b4 100644
--- a/server/src/test/java/org/elasticsearch/search/profile/query/ProfileScorerTests.java
+++ b/server/src/test/java/org/elasticsearch/search/profile/query/ProfileScorerTests.java
@@ -8,10 +8,14 @@
 
 package org.elasticsearch.search.profile.query;
 
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.MultiReader;
 import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Matches;
+import org.apache.lucene.search.MatchesIterator;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
@@ -19,6 +23,9 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
 
 public class ProfileScorerTests extends ESTestCase {
 
@@ -56,6 +63,85 @@ public void setMinCompetitiveScore(float minScore) {
         }
     }
 
+    private static class FakeWeight extends Weight {
+
+        protected FakeWeight(Query query) {
+            super(query);
+        }
+
+        @Override
+        public Explanation explain(LeafReaderContext context, int doc) throws IOException {
+            return Explanation.match(1, "fake_description");
+        }
+
+        @Override
+        public Scorer scorer(LeafReaderContext context) throws IOException {
+            FakeScorer fakeScorer = new FakeScorer(this);
+            fakeScorer.maxScore = 42f;
+            return fakeScorer;
+        }
+
+        @Override
+        public boolean isCacheable(LeafReaderContext ctx) {
+            return false;
+        }
+
+        @Override
+        public Matches matches(LeafReaderContext context, int doc) throws IOException {
+            return new Matches() {
+                @Override
+                public MatchesIterator getMatches(String field) throws IOException {
+                    return new MatchesIterator() {
+                        @Override
+                        public boolean next() throws IOException {
+                            return false;
+                        }
+
+                        @Override
+                        public int startPosition() {
+                            return 42;
+                        }
+
+                        @Override
+                        public int endPosition() {
+                            return 43;
+                        }
+
+                        @Override
+                        public int startOffset() throws IOException {
+                            return 44;
+                        }
+
+                        @Override
+                        public int endOffset() throws IOException {
+                            return 45;
+                        }
+
+                        @Override
+                        public MatchesIterator getSubMatches() throws IOException {
+                            return null;
+                        }
+
+                        @Override
+                        public Query getQuery() {
+                            return parentQuery;
+                        }
+                    };
+                }
+
+                @Override
+                public Collection<Matches> getSubMatches() {
+                    return Collections.emptyList();
+                }
+
+                @Override
+                public Iterator<String> iterator() {
+                    return null;
+                }
+            };
+        }
+    }
+
     public void testPropagateMinCompetitiveScore() throws IOException {
         Query query = new MatchAllDocsQuery();
         Weight weight = query.createWeight(new IndexSearcher(new MultiReader()), ScoreMode.TOP_SCORES, 1f);
@@ -78,4 +164,15 @@ public void testPropagateMaxScore() throws IOException {
         fakeScorer.maxScore = 42f;
         assertEquals(42f, profileScorer.getMaxScore(DocIdSetIterator.NO_MORE_DOCS), 0f);
     }
+
+    // tests that ProfileWeight correctly propagates the wrapped inner weight
+    public void testPropagateSubWeight() throws IOException {
+        Query query = new MatchAllDocsQuery();
+        Weight fakeWeight = new FakeWeight(query);
+        QueryProfileBreakdown profile = new QueryProfileBreakdown();
+        ProfileWeight profileWeight = new ProfileWeight(query, fakeWeight, profile);
+        assertEquals(42f, profileWeight.scorer(null).getMaxScore(DocIdSetIterator.NO_MORE_DOCS), 0f);
+        assertEquals(42, profileWeight.matches(null, 1).getMatches("some_field").startPosition());
+        assertEquals("fake_description", profileWeight.explain(null, 1).getDescription());
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfileShardResultTests.java b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfileShardResultTests.java
index 05ee1bf8bffa..f8c8d38e9280 100644
--- a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfileShardResultTests.java
+++ b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfileShardResultTests.java
@@ -41,6 +41,11 @@ protected QueryProfileShardResult createTestInstance() {
         return createTestItem();
     }
 
+    @Override
+    protected QueryProfileShardResult mutateInstance(QueryProfileShardResult instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected QueryProfileShardResult doParseInstance(XContentParser parser) throws IOException {
         ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
index c52a67ed29bf..65e3195ec44f 100644
--- a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
+++ b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
@@ -106,6 +106,7 @@ public void testBasic() throws IOException {
         assertEquals(1, results.size());
         Map<String, Long> breakdown = results.get(0).getTimeBreakdown();
         assertThat(breakdown.get(QueryTimingType.CREATE_WEIGHT.toString()), greaterThan(0L));
+        assertThat(breakdown.get(QueryTimingType.COUNT_WEIGHT.toString()), equalTo(0L));
         assertThat(breakdown.get(QueryTimingType.BUILD_SCORER.toString()), greaterThan(0L));
         assertThat(breakdown.get(QueryTimingType.NEXT_DOC.toString()), greaterThan(0L));
         assertThat(breakdown.get(QueryTimingType.ADVANCE.toString()), equalTo(0L));
@@ -113,6 +114,7 @@ public void testBasic() throws IOException {
         assertThat(breakdown.get(QueryTimingType.MATCH.toString()), equalTo(0L));
 
         assertThat(breakdown.get(QueryTimingType.CREATE_WEIGHT.toString() + "_count"), greaterThan(0L));
+        assertThat(breakdown.get(QueryTimingType.COUNT_WEIGHT.toString() + "_count"), equalTo(0L));
         assertThat(breakdown.get(QueryTimingType.BUILD_SCORER.toString() + "_count"), greaterThan(0L));
         assertThat(breakdown.get(QueryTimingType.NEXT_DOC.toString() + "_count"), greaterThan(0L));
         assertThat(breakdown.get(QueryTimingType.ADVANCE.toString() + "_count"), equalTo(0L));
@@ -149,6 +151,7 @@ public void testNoScoring() throws IOException {
         assertEquals(1, results.size());
         Map<String, Long> breakdown = results.get(0).getTimeBreakdown();
         assertThat(breakdown.get(QueryTimingType.CREATE_WEIGHT.toString()), greaterThan(0L));
+        assertThat(breakdown.get(QueryTimingType.COUNT_WEIGHT.toString()), equalTo(0L));
         assertThat(breakdown.get(QueryTimingType.BUILD_SCORER.toString()), greaterThan(0L));
         assertThat(breakdown.get(QueryTimingType.NEXT_DOC.toString()), greaterThan(0L));
         assertThat(breakdown.get(QueryTimingType.ADVANCE.toString()), equalTo(0L));
@@ -156,6 +159,7 @@ public void testNoScoring() throws IOException {
         assertThat(breakdown.get(QueryTimingType.MATCH.toString()), equalTo(0L));
 
         assertThat(breakdown.get(QueryTimingType.CREATE_WEIGHT.toString() + "_count"), greaterThan(0L));
+        assertThat(breakdown.get(QueryTimingType.COUNT_WEIGHT.toString() + "_count"), equalTo(0L));
         assertThat(breakdown.get(QueryTimingType.BUILD_SCORER.toString() + "_count"), greaterThan(0L));
         assertThat(breakdown.get(QueryTimingType.NEXT_DOC.toString() + "_count"), greaterThan(0L));
         assertThat(breakdown.get(QueryTimingType.ADVANCE.toString() + "_count"), equalTo(0L));
@@ -189,6 +193,7 @@ public void testApproximations() throws IOException {
         assertEquals(1, results.size());
         Map<String, Long> breakdown = results.get(0).getTimeBreakdown();
         assertThat(breakdown.get(QueryTimingType.CREATE_WEIGHT.toString()), greaterThan(0L));
+        assertThat(breakdown.get(QueryTimingType.COUNT_WEIGHT.toString()), greaterThan(0L));
         assertThat(breakdown.get(QueryTimingType.BUILD_SCORER.toString()), greaterThan(0L));
         assertThat(breakdown.get(QueryTimingType.NEXT_DOC.toString()), greaterThan(0L));
         assertThat(breakdown.get(QueryTimingType.ADVANCE.toString()), equalTo(0L));
@@ -196,6 +201,7 @@ public void testApproximations() throws IOException {
         assertThat(breakdown.get(QueryTimingType.MATCH.toString()), greaterThan(0L));
 
         assertThat(breakdown.get(QueryTimingType.CREATE_WEIGHT.toString() + "_count"), greaterThan(0L));
+        assertThat(breakdown.get(QueryTimingType.COUNT_WEIGHT.toString() + "_count"), greaterThan(0L));
         assertThat(breakdown.get(QueryTimingType.BUILD_SCORER.toString() + "_count"), greaterThan(0L));
         assertThat(breakdown.get(QueryTimingType.NEXT_DOC.toString() + "_count"), greaterThan(0L));
         assertThat(breakdown.get(QueryTimingType.ADVANCE.toString() + "_count"), equalTo(0L));
diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
index 2c3a45f49138..8c2ba79a8181 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
@@ -634,7 +634,7 @@ public void testDisableTopScoreCollection() throws Exception {
         context.parsedQuery(new ParsedQuery(q));
         context.setSize(3);
         context.trackTotalHitsUpTo(3);
-        TopDocsCollectorContext topDocsContext = TopDocsCollectorContext.createTopDocsCollectorContext(context, false);
+        TopDocsCollectorContext topDocsContext = TopDocsCollectorContext.createTopDocsCollectorContext(context);
         assertEquals(topDocsContext.create(null).scoreMode(), org.apache.lucene.search.ScoreMode.COMPLETE);
         QueryPhase.executeInternal(context);
         assertEquals(5, context.queryResult().topDocs().topDocs.totalHits.value);
@@ -642,7 +642,7 @@ public void testDisableTopScoreCollection() throws Exception {
         assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(3));
 
         context.sort(new SortAndFormats(new Sort(new SortField("other", SortField.Type.INT)), new DocValueFormat[] { DocValueFormat.RAW }));
-        topDocsContext = TopDocsCollectorContext.createTopDocsCollectorContext(context, false);
+        topDocsContext = TopDocsCollectorContext.createTopDocsCollectorContext(context);
         assertEquals(topDocsContext.create(null).scoreMode(), org.apache.lucene.search.ScoreMode.TOP_DOCS);
         QueryPhase.executeInternal(context);
         assertEquals(5, context.queryResult().topDocs().topDocs.totalHits.value);
diff --git a/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java b/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java
index b332c5f66def..bc118cb2e718 100644
--- a/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.lookup.LeafSearchLookup;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.After;
 import org.junit.Before;
@@ -50,7 +49,7 @@ public class ScriptScoreQueryTests extends ESTestCase {
     private DirectoryReader reader;
     private IndexSearcher searcher;
     private LeafReaderContext leafReaderContext;
-    private final SearchLookup lookup = new SearchLookup(null, null, new SourceLookup.ReaderSourceProvider());
+    private final SearchLookup lookup = new SearchLookup(null, null, (ctx, doc) -> null);
 
     @Before
     public void initSearcher() throws IOException {
@@ -179,7 +178,7 @@ public boolean needs_score() {
             }
 
             @Override
-            public ScoreScript newInstance(DocReader docReader) throws IOException {
+            public ScoreScript newInstance(DocReader docReader) {
                 return new ScoreScript(script.getParams(), lookup, docReader) {
                     @Override
                     public double execute(ExplanationHolder explanation) {
diff --git a/server/src/test/java/org/elasticsearch/search/query/TopDocsCollectorContextTests.java b/server/src/test/java/org/elasticsearch/search/query/TopDocsCollectorContextTests.java
deleted file mode 100644
index 783232d3b033..000000000000
--- a/server/src/test/java/org/elasticsearch/search/query/TopDocsCollectorContextTests.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.search.query;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.IntPoint;
-import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.document.SortedDocValuesField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.search.FieldExistsQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.tests.index.RandomIndexWriter;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.test.ESTestCase;
-
-import java.io.IOException;
-
-public class TopDocsCollectorContextTests extends ESTestCase {
-
-    public void testShortcutTotalHitCountTextField() throws IOException {
-        try (Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) {
-            Document doc = new Document();
-            doc.add(new TextField("text", "value", Field.Store.NO));
-            iw.addDocument(doc);
-            doc = new Document();
-            doc.add(new TextField("text", "", Field.Store.NO));
-            iw.addDocument(doc);
-            iw.addDocument(new Document());
-            iw.commit();
-            try (IndexReader reader = iw.getReader()) {
-                final Query testQuery = new FieldExistsQuery("text");
-                int hitCount = TopDocsCollectorContext.shortcutTotalHitCount(reader, testQuery);
-                assertEquals(-1, hitCount);
-            }
-        }
-    }
-
-    public void testShortcutTotalHitCountStringField() throws IOException {
-        try (Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) {
-            Document doc = new Document();
-            doc.add(new StringField("string", "value", Field.Store.NO));
-            doc.add(new SortedDocValuesField("string", new BytesRef("value")));
-            iw.addDocument(doc);
-            doc = new Document();
-            doc.add(new StringField("string", "", Field.Store.NO));
-            doc.add(new SortedDocValuesField("string", new BytesRef("")));
-            iw.addDocument(doc);
-            iw.addDocument(new Document());
-            iw.commit();
-            try (IndexReader reader = iw.getReader()) {
-                final Query testQuery = new FieldExistsQuery("string");
-                int hitCount = TopDocsCollectorContext.shortcutTotalHitCount(reader, testQuery);
-                assertEquals(2, hitCount);
-            }
-        }
-    }
-
-    public void testShortcutTotalHitCountNumericField() throws IOException {
-        try (Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) {
-            Document doc = new Document();
-            doc.add(new IntPoint("int", 10));
-            doc.add(new NumericDocValuesField("int", 10));
-            iw.addDocument(doc);
-            iw.addDocument(new Document());
-            iw.commit();
-            try (IndexReader reader = iw.getReader()) {
-                final Query testQuery = new FieldExistsQuery("int");
-                int hitCount = TopDocsCollectorContext.shortcutTotalHitCount(reader, testQuery);
-                assertEquals(1, hitCount);
-            }
-        }
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java b/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java
index eb1a57c98677..b6530e3c1c6b 100644
--- a/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java
+++ b/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java
@@ -21,11 +21,12 @@
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.index.mapper.GeoPointScriptFieldType;
+import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.script.AbstractLongFieldScript;
 import org.elasticsearch.script.GeoPointFieldScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.SourceProvider;
 
 import java.io.IOException;
 import java.util.List;
@@ -83,16 +84,19 @@ public void testMatches() throws IOException {
             iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"location\": [-3.56, -45.98]}"))));
             try (DirectoryReader reader = iw.getReader()) {
                 IndexSearcher searcher = newSearcher(reader);
-                SearchLookup searchLookup = new SearchLookup(null, null, new SourceLookup.ReaderSourceProvider());
+                SearchLookup searchLookup = new SearchLookup(null, null, SourceProvider.fromStoredFields());
                 Function<LeafReaderContext, GeoPointFieldScript> leafFactory = ctx -> new GeoPointFieldScript(
                     "test",
                     Map.of(),
                     searchLookup,
+                    OnScriptError.FAIL,
                     ctx
                 ) {
                     @Override
+                    @SuppressWarnings("unchecked")
                     public void execute() {
-                        GeoPoint point = GeoUtils.parseGeoPoint(searchLookup.source().source().get("location"), true);
+                        Map<String, Object> source = (Map<String, Object>) this.getParams().get("_source");
+                        GeoPoint point = GeoUtils.parseGeoPoint(source.get("location"), true);
                         emit(point.lat(), point.lon());
                     }
                 };
diff --git a/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java b/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java
index fcdf236188ea..e369abe36ded 100644
--- a/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java
+++ b/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java
@@ -17,11 +17,12 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.script.AbstractLongFieldScript;
 import org.elasticsearch.script.DateFieldScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -69,17 +70,20 @@ public void testMatches() throws IOException {
             iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181351]}"))));
             try (DirectoryReader reader = iw.getReader()) {
                 IndexSearcher searcher = newSearcher(reader);
-                SearchLookup searchLookup = new SearchLookup(null, null, new SourceLookup.ReaderSourceProvider());
+                SearchLookup searchLookup = new SearchLookup(null, null, SourceProvider.fromStoredFields());
                 Function<LeafReaderContext, AbstractLongFieldScript> leafFactory = ctx -> new DateFieldScript(
                     "test",
                     Map.of(),
                     searchLookup,
                     null,
+                    OnScriptError.FAIL,
                     ctx
                 ) {
                     @Override
+                    @SuppressWarnings("unchecked")
                     public void execute() {
-                        for (Object timestamp : (List<?>) searchLookup.source().source().get("timestamp")) {
+                        Map<String, Object> source = (Map<String, Object>) this.getParams().get("_source");
+                        for (Object timestamp : (List<?>) source.get("timestamp")) {
                             emit(((Number) timestamp).longValue());
                         }
                     }
diff --git a/server/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java
index 39e879de13ac..47636aafa72b 100644
--- a/server/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.search.sort;
 
 import org.apache.lucene.search.SortField;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.json.JsonXContent;
@@ -47,7 +48,7 @@ public void testIllegalOrder() {
      */
     public void testParseOrder() throws IOException {
         SortOrder order = randomBoolean() ? SortOrder.ASC : SortOrder.DESC;
-        String scoreSortString = formatted("""
+        String scoreSortString = Strings.format("""
             { "_score": { "order": "%s" }}
             """, order.toString());
         XContentParser parser = createParser(JsonXContent.jsonXContent, scoreSortString);
diff --git a/server/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java
index c9e198c9bf58..eee98297c7a1 100644
--- a/server/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java
@@ -54,7 +54,7 @@ public static void afterClass() throws Exception {
      */
     public void testSingleFieldSort() throws IOException {
         SortOrder order = randomBoolean() ? SortOrder.ASC : SortOrder.DESC;
-        String json = formatted("{ \"sort\" : { \"field1\" : \"%s\" }}", order);
+        String json = Strings.format("{ \"sort\" : { \"field1\" : \"%s\" }}", order);
         List<SortBuilder<?>> result = parseSort(json);
         assertEquals(1, result.size());
         SortBuilder<?> sortBuilder = result.get(0);
diff --git a/server/src/test/java/org/elasticsearch/search/sort/SortValueTests.java b/server/src/test/java/org/elasticsearch/search/sort/SortValueTests.java
index 0879d786ed61..5c59319b64a2 100644
--- a/server/src/test/java/org/elasticsearch/search/sort/SortValueTests.java
+++ b/server/src/test/java/org/elasticsearch/search/sort/SortValueTests.java
@@ -61,7 +61,7 @@ protected SortValue createTestInstance() {
     }
 
     @Override
-    protected SortValue mutateInstance(SortValue instance) throws IOException {
+    protected SortValue mutateInstance(SortValue instance) {
         return randomValueOtherThanMany(mut -> instance.getKey().equals(mut.getKey()), this::createTestInstance);
     }
 
@@ -242,4 +242,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             }
         });
     }
+
+    @Override
+    public void testToString() throws Exception {
+        // override parent since empty sort value serializes to empty string
+        assertNotNull(createTestInstance().toString());
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilderTests.java b/server/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilderTests.java
index 8d808f7fb869..6855b227ceae 100644
--- a/server/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilderTests.java
@@ -28,7 +28,7 @@ protected PhraseSuggestionBuilder randomSuggestionBuilder() {
     public static PhraseSuggestionBuilder randomPhraseSuggestionBuilder() {
         PhraseSuggestionBuilder testBuilder = new PhraseSuggestionBuilder(randomAlphaOfLengthBetween(2, 20));
         setCommonPropertiesOnRandomBuilder(testBuilder);
-        maybeSet(testBuilder::maxErrors, randomFloat());
+        maybeSet(testBuilder::maxErrors, Math.max(randomFloat(), Float.MIN_NORMAL));
         maybeSet(testBuilder::separator, randomAlphaOfLengthBetween(1, 10));
         maybeSet(testBuilder::realWordErrorLikelihood, randomFloat());
         maybeSet(testBuilder::confidence, randomFloat());
diff --git a/server/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java b/server/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java
index 264ccb3077aa..833bf34b8f9e 100644
--- a/server/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java
@@ -10,6 +10,7 @@
 
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.search.suggest.AbstractSuggestionBuilderTestCase;
 import org.elasticsearch.search.suggest.SortBy;
 import org.elasticsearch.search.suggest.SuggestBuilder;
@@ -177,7 +178,7 @@ public void testDefaultValuesSet() {
 
     public void testMalformedJson() {
         final String field = RandomStrings.randomAsciiOfLength(random(), 10).toLowerCase(Locale.ROOT);
-        String suggest = formatted("""
+        String suggest = Strings.format("""
             {
               "bad-payload" : {
                 "text" : "the amsterdma meetpu",
diff --git a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java
new file mode 100644
index 000000000000..19e3a9fc3e14
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java
@@ -0,0 +1,239 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.vectors;
+
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.KnnByteVectorQuery;
+import org.apache.lucene.search.KnnFloatVectorQuery;
+import org.apache.lucene.search.Query;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
+import org.elasticsearch.index.query.MatchNoneQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.test.AbstractBuilderTestCase;
+import org.elasticsearch.test.AbstractQueryTestCase;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.instanceOf;
+
+abstract class AbstractKnnVectorQueryBuilderTestCase extends AbstractQueryTestCase<KnnVectorQueryBuilder> {
+    private static final String VECTOR_FIELD = "vector";
+    private static final String VECTOR_ALIAS_FIELD = "vector_alias";
+    private static final int VECTOR_DIMENSION = 3;
+
+    abstract DenseVectorFieldMapper.ElementType elementType();
+
+    @Override
+    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("properties")
+            .startObject(VECTOR_FIELD)
+            .field("type", "dense_vector")
+            .field("dims", VECTOR_DIMENSION)
+            .field("index", true)
+            .field("similarity", "l2_norm")
+            .field("element_type", elementType())
+            .endObject()
+            .startObject(VECTOR_ALIAS_FIELD)
+            .field("type", "alias")
+            .field("path", VECTOR_FIELD)
+            .endObject()
+            .endObject()
+            .endObject();
+        mapperService.merge(
+            MapperService.SINGLE_MAPPING_NAME,
+            new CompressedXContent(Strings.toString(builder)),
+            MapperService.MergeReason.MAPPING_UPDATE
+        );
+    }
+
+    @Override
+    protected KnnVectorQueryBuilder doCreateTestQueryBuilder() {
+        String fieldName = randomBoolean() ? VECTOR_FIELD : VECTOR_ALIAS_FIELD;
+        byte[] byteVector = new byte[VECTOR_DIMENSION];
+        float[] vector = new float[VECTOR_DIMENSION];
+        for (int i = 0; i < vector.length; i++) {
+            vector[i] = randomFloat();
+            byteVector[i] = randomByte();
+        }
+        int numCands = randomIntBetween(1, 1000);
+
+        KnnVectorQueryBuilder queryBuilder = switch (elementType()) {
+            case BYTE -> new KnnVectorQueryBuilder(fieldName, byteVector, numCands);
+            case FLOAT -> new KnnVectorQueryBuilder(fieldName, vector, numCands);
+        };
+
+        if (randomBoolean()) {
+            List<QueryBuilder> filters = new ArrayList<>();
+            int numFilters = randomIntBetween(1, 5);
+            for (int i = 0; i < numFilters; i++) {
+                String filterFieldName = randomBoolean() ? KEYWORD_FIELD_NAME : TEXT_FIELD_NAME;
+                filters.add(QueryBuilders.termQuery(filterFieldName, randomAlphaOfLength(10)));
+            }
+            queryBuilder.addFilterQueries(filters);
+        }
+        return queryBuilder;
+    }
+
+    @Override
+    protected void doAssertLuceneQuery(KnnVectorQueryBuilder queryBuilder, Query query, SearchExecutionContext context) throws IOException {
+        switch (elementType()) {
+            case FLOAT -> assertTrue(query instanceof KnnFloatVectorQuery);
+            case BYTE -> assertTrue(query instanceof KnnByteVectorQuery);
+        }
+
+        BooleanQuery.Builder builder = new BooleanQuery.Builder();
+        for (QueryBuilder qb : queryBuilder.filterQueries()) {
+            builder.add(qb.toQuery(context), BooleanClause.Occur.FILTER);
+        }
+        BooleanQuery booleanQuery = builder.build();
+        Query filterQuery = booleanQuery.clauses().isEmpty() ? null : booleanQuery;
+        // The field should always be resolved to the concrete field
+        Query knnVectorQueryBuilt = switch (elementType()) {
+            case BYTE -> new KnnByteVectorQuery(VECTOR_FIELD, queryBuilder.getByteQueryVector(), queryBuilder.numCands(), filterQuery);
+            case FLOAT -> new KnnFloatVectorQuery(VECTOR_FIELD, queryBuilder.queryVector(), queryBuilder.numCands(), filterQuery);
+        };
+        assertEquals(query, knnVectorQueryBuilt);
+    }
+
+    public void testWrongDimension() {
+        SearchExecutionContext context = createSearchExecutionContext();
+        KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f }, 10);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context));
+        assertThat(e.getMessage(), containsString("the query vector has a different dimension [2] than the index vectors [3]"));
+    }
+
+    public void testNonexistentField() {
+        SearchExecutionContext context = createSearchExecutionContext();
+        KnnVectorQueryBuilder query = new KnnVectorQueryBuilder("nonexistent", new float[] { 1.0f, 1.0f, 1.0f }, 10);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context));
+        assertThat(e.getMessage(), containsString("field [nonexistent] does not exist in the mapping"));
+    }
+
+    public void testWrongFieldType() {
+        SearchExecutionContext context = createSearchExecutionContext();
+        KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(
+            AbstractBuilderTestCase.KEYWORD_FIELD_NAME,
+            new float[] { 1.0f, 1.0f, 1.0f },
+            10
+        );
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context));
+        assertThat(e.getMessage(), containsString("[knn] queries are only supported on [dense_vector] fields"));
+    }
+
+    @Override
+    public void testValidOutput() {
+        KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f, 3.0f }, 10);
+        String expected = """
+            {
+              "knn" : {
+                "field" : "vector",
+                "vector" : [
+                  1.0,
+                  2.0,
+                  3.0
+                ],
+                "num_candidates" : 10
+              }
+            }""";
+        assertEquals(expected, query.toString());
+    }
+
+    @Override
+    public void testMustRewrite() throws IOException {
+        SearchExecutionContext context = createSearchExecutionContext();
+        context.setAllowUnmappedFields(true);
+        TermQueryBuilder termQuery = new TermQueryBuilder("unmapped_field", 42);
+        KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f, 3.0f }, VECTOR_DIMENSION);
+        query.addFilterQuery(termQuery);
+
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> query.toQuery(context));
+        assertEquals("Rewrite first", e.getMessage());
+
+        QueryBuilder rewrittenQuery = query.rewrite(context);
+        assertThat(rewrittenQuery, instanceOf(MatchNoneQueryBuilder.class));
+    }
+
+    public void testBWCVersionSerialization() throws IOException {
+        float[] bwcFloat = new float[VECTOR_DIMENSION];
+        KnnVectorQueryBuilder query = createTestQueryBuilder();
+        if (query.queryVector() != null) {
+            bwcFloat = query.queryVector();
+        } else {
+            for (int i = 0; i < query.getByteQueryVector().length; i++) {
+                bwcFloat[i] = query.getByteQueryVector()[i];
+            }
+        }
+        KnnVectorQueryBuilder queryWithNoFilters = new KnnVectorQueryBuilder(query.getFieldName(), bwcFloat, query.numCands()).queryName(
+            query.queryName()
+        ).boost(query.boost());
+
+        KnnVectorQueryBuilder queryNoByteQuery = new KnnVectorQueryBuilder(query.getFieldName(), bwcFloat, query.numCands()).queryName(
+            query.queryName()
+        ).boost(query.boost()).addFilterQueries(query.filterQueries());
+
+        Version newVersion = VersionUtils.randomVersionBetween(random(), Version.V_8_7_0, Version.CURRENT);
+        Version beforeByteQueryVersion = VersionUtils.randomVersionBetween(random(), Version.V_8_2_0, Version.V_8_6_0);
+        Version beforeFilterVersion = VersionUtils.randomVersionBetween(random(), Version.V_8_0_0, Version.V_8_1_0);
+
+        assertSerialization(query, newVersion);
+        assertSerialization(queryNoByteQuery, beforeByteQueryVersion);
+        assertSerialization(queryWithNoFilters, beforeFilterVersion);
+
+        for (var tuple : List.of(
+            Tuple.tuple(beforeByteQueryVersion, queryNoByteQuery),
+            Tuple.tuple(beforeFilterVersion, queryWithNoFilters)
+        )) {
+            try (BytesStreamOutput output = new BytesStreamOutput()) {
+                output.setVersion(tuple.v1());
+                output.writeNamedWriteable(query);
+                try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry())) {
+                    in.setVersion(tuple.v1());
+                    KnnVectorQueryBuilder deserializedQuery = (KnnVectorQueryBuilder) in.readNamedWriteable(QueryBuilder.class);
+                    assertEquals(tuple.v2(), deserializedQuery);
+                    assertEquals(tuple.v2().hashCode(), deserializedQuery.hashCode());
+                }
+            }
+        }
+    }
+
+    @Override
+    public void testUnknownObjectException() {
+        assumeTrue("Test isn't relevant, since query is never parsed from xContent", false);
+    }
+
+    @Override
+    public void testFromXContent() {
+        assumeTrue("Test isn't relevant, since query is never parsed from xContent", false);
+    }
+
+    @Override
+    public void testUnknownField() {
+        assumeTrue("Test isn't relevant, since query is never parsed from xContent", false);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/vectors/AbstractQueryVectorBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/vectors/AbstractQueryVectorBuilderTestCase.java
new file mode 100644
index 000000000000..c889f71324cd
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/vectors/AbstractQueryVectorBuilderTestCase.java
@@ -0,0 +1,164 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.vectors;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.index.query.Rewriteable;
+import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.test.AbstractXContentTestCase;
+import org.elasticsearch.test.client.NoOpClient;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.search.vectors.KnnSearchBuilderTests.randomVector;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+/**
+ * Tests a query vector builder
+ * @param <T> the query vector builder type to test
+ */
+public abstract class AbstractQueryVectorBuilderTestCase<T extends QueryVectorBuilder> extends AbstractXContentSerializingTestCase<T> {
+
+    private NamedWriteableRegistry namedWriteableRegistry;
+    private NamedXContentRegistry namedXContentRegistry;
+
+    protected List<SearchPlugin> additionalPlugins() {
+        return List.of();
+    }
+
+    @Before
+    public void registerNamedXContents() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, additionalPlugins());
+        namedXContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
+        namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return namedXContentRegistry;
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return namedWriteableRegistry;
+    }
+
+    // Just in case the vector builder needs to know the expected value when testing
+    protected T createTestInstance(float[] expected) {
+        return createTestInstance();
+    }
+
+    public final void testKnnSearchBuilderXContent() throws Exception {
+        AbstractXContentTestCase.XContentTester<KnnSearchBuilder> tester = AbstractXContentTestCase.xContentTester(
+            this::createParser,
+            () -> new KnnSearchBuilder(randomAlphaOfLength(10), createTestInstance(), 5, 10),
+            getToXContentParams(),
+            KnnSearchBuilder::fromXContent
+        );
+        tester.test();
+    }
+
+    public final void testKnnSearchBuilderWireSerialization() throws IOException {
+        for (int i = 0; i < NUMBER_OF_TEST_RUNS; i++) {
+            KnnSearchBuilder searchBuilder = new KnnSearchBuilder(randomAlphaOfLength(10), createTestInstance(), 5, 10);
+            KnnSearchBuilder serialized = copyWriteable(
+                searchBuilder,
+                getNamedWriteableRegistry(),
+                KnnSearchBuilder::new,
+                TransportVersion.CURRENT
+            );
+            assertThat(serialized, equalTo(searchBuilder));
+            assertNotSame(serialized, searchBuilder);
+        }
+    }
+
+    public final void testKnnSearchRewrite() throws Exception {
+        for (int i = 0; i < NUMBER_OF_TEST_RUNS; i++) {
+            float[] expected = randomVector(randomIntBetween(10, 1024));
+            T queryVectorBuilder = createTestInstance(expected);
+            KnnSearchBuilder searchBuilder = new KnnSearchBuilder(randomAlphaOfLength(10), queryVectorBuilder, 5, 10);
+            KnnSearchBuilder serialized = copyWriteable(
+                searchBuilder,
+                getNamedWriteableRegistry(),
+                KnnSearchBuilder::new,
+                TransportVersion.CURRENT
+            );
+            try (NoOpClient client = new AssertingClient(expected, queryVectorBuilder)) {
+                QueryRewriteContext context = new QueryRewriteContext(null, null, client, null);
+                PlainActionFuture<KnnSearchBuilder> future = new PlainActionFuture<>();
+                Rewriteable.rewriteAndFetch(randomFrom(serialized, searchBuilder), context, future);
+                KnnSearchBuilder rewritten = future.get();
+                assertThat(rewritten.queryVector, equalTo(expected));
+                assertThat(rewritten.queryVectorBuilder, nullValue());
+            }
+        }
+    }
+
+    public final void testVectorFetch() throws Exception {
+        float[] expected = randomVector(randomIntBetween(10, 1024));
+        T queryVectorBuilder = createTestInstance(expected);
+        try (NoOpClient client = new AssertingClient(expected, queryVectorBuilder)) {
+            PlainActionFuture<float[]> future = new PlainActionFuture<>();
+            queryVectorBuilder.buildVector(client, future);
+            assertThat(future.get(), equalTo(expected));
+        }
+    }
+
+    /**
+     * Assert that the client action request is correct given this provided random builder
+     * @param request The built request to be executed by the client
+     * @param builder The builder used when generating this request
+     */
+    abstract void doAssertClientRequest(ActionRequest request, T builder);
+
+    /**
+     * Create a response given this expected array that is acceptable to the query builder
+     * @param array The expected final array
+     * @param builder The original randomly built query vector builder
+     * @return An action response to be handled by the query vector builder
+     */
+    abstract ActionResponse createResponse(float[] array, T builder);
+
+    private class AssertingClient extends NoOpClient {
+
+        private final float[] array;
+        private final T queryVectorBuilder;
+
+        AssertingClient(float[] array, T queryVectorBuilder) {
+            super("query_vector_builder_tests");
+            this.array = array;
+            this.queryVectorBuilder = queryVectorBuilder;
+        }
+
+        @Override
+        @SuppressWarnings("unchecked")
+        protected <Request extends ActionRequest, Response extends ActionResponse> void doExecute(
+            ActionType<Response> action,
+            Request request,
+            ActionListener<Response> listener
+        ) {
+            doAssertClientRequest(request, queryVectorBuilder);
+            listener.onResponse((Response) createResponse(array, queryVectorBuilder));
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnByteVectorQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnByteVectorQueryBuilderTests.java
new file mode 100644
index 000000000000..3bf92a60275d
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnByteVectorQueryBuilderTests.java
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.vectors;
+
+import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
+
+public class KnnByteVectorQueryBuilderTests extends AbstractKnnVectorQueryBuilderTestCase {
+    @Override
+    DenseVectorFieldMapper.ElementType elementType() {
+        return DenseVectorFieldMapper.ElementType.BYTE;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java
index 7d3304ae733f..9d66f06cc9bf 100644
--- a/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java
@@ -8,25 +8,37 @@
 
 package org.elasticsearch.search.vectors;
 
+import org.apache.lucene.search.Query;
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.index.query.Rewriteable;
+import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.junit.Before;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 
 import static java.util.Collections.emptyList;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.nullValue;
 
 public class KnnSearchBuilderTests extends AbstractXContentSerializingTestCase<KnnSearchBuilder> {
     private NamedWriteableRegistry namedWriteableRegistry;
@@ -85,7 +97,7 @@ protected KnnSearchBuilder createTestInstance() {
     }
 
     @Override
-    protected KnnSearchBuilder mutateInstance(KnnSearchBuilder instance) throws IOException {
+    protected KnnSearchBuilder mutateInstance(KnnSearchBuilder instance) {
         switch (random().nextInt(6)) {
 
             case 0:
@@ -161,11 +173,80 @@ public void testInvalidK() {
         assertThat(e.getMessage(), containsString("[k] must be greater than 0"));
     }
 
-    private static float[] randomVector(int dim) {
+    public void testRewrite() throws Exception {
+        float[] expectedArray = randomVector(randomIntBetween(10, 1024));
+        KnnSearchBuilder searchBuilder = new KnnSearchBuilder(
+            "field",
+            new TestQueryVectorBuilderPlugin.TestQueryVectorBuilder(expectedArray),
+            5,
+            10
+        );
+        searchBuilder.boost(randomFloat());
+        searchBuilder.addFilterQueries(List.of(new RewriteableQuery()));
+
+        QueryRewriteContext context = new QueryRewriteContext(null, null, null, null);
+        PlainActionFuture<KnnSearchBuilder> future = new PlainActionFuture<>();
+        Rewriteable.rewriteAndFetch(searchBuilder, context, future);
+        KnnSearchBuilder rewritten = future.get();
+
+        assertThat(rewritten.field, equalTo(searchBuilder.field));
+        assertThat(rewritten.boost, equalTo(searchBuilder.boost));
+        assertThat(rewritten.queryVector, equalTo(expectedArray));
+        assertThat(rewritten.queryVectorBuilder, nullValue());
+        assertThat(rewritten.filterQueries, hasSize(1));
+        assertThat(((RewriteableQuery) rewritten.filterQueries.get(0)).rewrites, equalTo(1));
+    }
+
+    static float[] randomVector(int dim) {
         float[] vector = new float[dim];
         for (int i = 0; i < vector.length; i++) {
             vector[i] = randomFloat();
         }
         return vector;
     }
+
+    private static class RewriteableQuery extends AbstractQueryBuilder<RewriteableQuery> {
+        private int rewrites;
+
+        @Override
+        public String getWriteableName() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public TransportVersion getMinimalSupportedVersion() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        protected void doWriteTo(StreamOutput out) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        protected void doXContent(XContentBuilder builder, Params params) throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        protected Query doToQuery(SearchExecutionContext context) throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        protected boolean doEquals(RewriteableQuery other) {
+            return true;
+        }
+
+        @Override
+        protected int doHashCode() {
+            return Objects.hashCode(RewriteableQuery.class);
+        }
+
+        @Override
+        protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) {
+            rewrites++;
+            return this;
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilderTests.java
index d52cafc8e685..6b1166cdd16d 100644
--- a/server/src/test/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilderTests.java
@@ -8,196 +8,11 @@
 
 package org.elasticsearch.search.vectors;
 
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.KnnVectorQuery;
-import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.query.MatchNoneQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.SearchExecutionContext;
-import org.elasticsearch.index.query.TermQueryBuilder;
-import org.elasticsearch.test.AbstractBuilderTestCase;
-import org.elasticsearch.test.AbstractQueryTestCase;
-import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.instanceOf;
-
-public class KnnVectorQueryBuilderTests extends AbstractQueryTestCase<KnnVectorQueryBuilder> {
-    private static final String VECTOR_FIELD = "vector";
-    private static final String VECTOR_ALIAS_FIELD = "vector_alias";
-    private static final int VECTOR_DIMENSION = 3;
-
-    @Override
-    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
-        XContentBuilder builder = XContentFactory.jsonBuilder()
-            .startObject()
-            .startObject("properties")
-            .startObject(VECTOR_FIELD)
-            .field("type", "dense_vector")
-            .field("dims", VECTOR_DIMENSION)
-            .field("index", true)
-            .field("similarity", "l2_norm")
-            .endObject()
-            .startObject(VECTOR_ALIAS_FIELD)
-            .field("type", "alias")
-            .field("path", VECTOR_FIELD)
-            .endObject()
-            .endObject()
-            .endObject();
-        mapperService.merge(
-            MapperService.SINGLE_MAPPING_NAME,
-            new CompressedXContent(Strings.toString(builder)),
-            MapperService.MergeReason.MAPPING_UPDATE
-        );
-    }
-
-    @Override
-    protected KnnVectorQueryBuilder doCreateTestQueryBuilder() {
-        String fieldName = randomBoolean() ? VECTOR_FIELD : VECTOR_ALIAS_FIELD;
-        float[] vector = new float[VECTOR_DIMENSION];
-        for (int i = 0; i < vector.length; i++) {
-            vector[i] = randomFloat();
-        }
-        int numCands = randomIntBetween(1, 1000);
-
-        KnnVectorQueryBuilder queryBuilder = new KnnVectorQueryBuilder(fieldName, vector, numCands);
-
-        if (randomBoolean()) {
-            List<QueryBuilder> filters = new ArrayList<>();
-            int numFilters = randomIntBetween(1, 5);
-            for (int i = 0; i < numFilters; i++) {
-                String filterFieldName = randomBoolean() ? KEYWORD_FIELD_NAME : TEXT_FIELD_NAME;
-                filters.add(QueryBuilders.termQuery(filterFieldName, randomAlphaOfLength(10)));
-            }
-            queryBuilder.addFilterQueries(filters);
-        }
-        return queryBuilder;
-    }
-
-    @Override
-    protected void doAssertLuceneQuery(KnnVectorQueryBuilder queryBuilder, Query query, SearchExecutionContext context) throws IOException {
-        assertTrue(query instanceof KnnVectorQuery);
-        KnnVectorQuery knnVectorQuery = (KnnVectorQuery) query;
-
-        BooleanQuery.Builder builder = new BooleanQuery.Builder();
-        for (QueryBuilder qb : queryBuilder.filterQueries()) {
-            builder.add(qb.toQuery(context), BooleanClause.Occur.FILTER);
-        }
-        BooleanQuery booleanQuery = builder.build();
-        Query filterQuery = booleanQuery.clauses().isEmpty() ? null : booleanQuery;
-        // The field should always be resolved to the concrete field
-        Query knnVectorQueryBuilt = new KnnVectorQuery(VECTOR_FIELD, queryBuilder.queryVector(), queryBuilder.numCands(), filterQuery);
-        assertEquals(knnVectorQuery, knnVectorQueryBuilt);
-    }
-
-    public void testWrongDimension() {
-        SearchExecutionContext context = createSearchExecutionContext();
-        KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f }, 10);
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context));
-        assertThat(e.getMessage(), containsString("the query vector has a different dimension [2] than the index vectors [3]"));
-    }
-
-    public void testNonexistentField() {
-        SearchExecutionContext context = createSearchExecutionContext();
-        KnnVectorQueryBuilder query = new KnnVectorQueryBuilder("nonexistent", new float[] { 1.0f, 1.0f, 1.0f }, 10);
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context));
-        assertThat(e.getMessage(), containsString("field [nonexistent] does not exist in the mapping"));
-    }
-
-    public void testWrongFieldType() {
-        SearchExecutionContext context = createSearchExecutionContext();
-        KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(
-            AbstractBuilderTestCase.KEYWORD_FIELD_NAME,
-            new float[] { 1.0f, 1.0f, 1.0f },
-            10
-        );
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> query.doToQuery(context));
-        assertThat(e.getMessage(), containsString("[knn] queries are only supported on [dense_vector] fields"));
-    }
-
-    @Override
-    public void testValidOutput() {
-        KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f, 3.0f }, 10);
-        String expected = """
-            {
-              "knn" : {
-                "field" : "vector",
-                "vector" : [
-                  1.0,
-                  2.0,
-                  3.0
-                ],
-                "num_candidates" : 10
-              }
-            }""";
-        assertEquals(expected, query.toString());
-    }
-
-    @Override
-    public void testMustRewrite() throws IOException {
-        SearchExecutionContext context = createSearchExecutionContext();
-        context.setAllowUnmappedFields(true);
-        TermQueryBuilder termQuery = new TermQueryBuilder("unmapped_field", 42);
-        KnnVectorQueryBuilder query = new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 1.0f, 2.0f, 3.0f }, VECTOR_DIMENSION);
-        query.addFilterQuery(termQuery);
-
-        IllegalStateException e = expectThrows(IllegalStateException.class, () -> query.toQuery(context));
-        assertEquals("Rewrite first", e.getMessage());
-
-        QueryBuilder rewrittenQuery = query.rewrite(context);
-        assertThat(rewrittenQuery, instanceOf(MatchNoneQueryBuilder.class));
-    }
-
-    public void testOldVersionSerialization() throws IOException {
-        KnnVectorQueryBuilder query = createTestQueryBuilder();
-        KnnVectorQueryBuilder queryWithNoFilters = new KnnVectorQueryBuilder(query.getFieldName(), query.queryVector(), query.numCands());
-        queryWithNoFilters.queryName(query.queryName()).boost(query.boost());
-
-        Version newVersion = VersionUtils.randomVersionBetween(random(), Version.V_8_2_0, Version.CURRENT);
-        Version oldVersion = VersionUtils.randomVersionBetween(random(), Version.V_8_0_0, Version.V_8_1_0);
-
-        assertSerialization(query, newVersion);
-        assertSerialization(queryWithNoFilters, oldVersion);
-
-        try (BytesStreamOutput output = new BytesStreamOutput()) {
-            output.setVersion(newVersion);
-            output.writeNamedWriteable(query);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry())) {
-                in.setVersion(oldVersion);
-                KnnVectorQueryBuilder deserializedQuery = (KnnVectorQueryBuilder) in.readNamedWriteable(QueryBuilder.class);
-                assertEquals(queryWithNoFilters, deserializedQuery);
-                assertEquals(queryWithNoFilters.hashCode(), deserializedQuery.hashCode());
-            }
-        }
-    }
-
-    @Override
-    public void testUnknownObjectException() throws IOException {
-        // Test isn't relevant, since query is never parsed from xContent
-    }
-
-    @Override
-    public void testFromXContent() throws IOException {
-        // Test isn't relevant, since query is never parsed from xContent
-    }
+import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 
+public class KnnVectorQueryBuilderTests extends AbstractKnnVectorQueryBuilderTestCase {
     @Override
-    public void testUnknownField() throws IOException {
-        // Test isn't relevant, since query is never parsed from xContent
+    DenseVectorFieldMapper.ElementType elementType() {
+        return DenseVectorFieldMapper.ElementType.FLOAT;
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/vectors/QueryVectorBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/QueryVectorBuilderTests.java
new file mode 100644
index 000000000000..e8b879948fb7
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/vectors/QueryVectorBuilderTests.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.vectors;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Test the query vector builder logic with a test plugin
+ */
+public class QueryVectorBuilderTests extends AbstractQueryVectorBuilderTestCase<TestQueryVectorBuilderPlugin.TestQueryVectorBuilder> {
+
+    @Override
+    protected List<SearchPlugin> additionalPlugins() {
+        return List.of(new TestQueryVectorBuilderPlugin());
+    }
+
+    @Override
+    protected Writeable.Reader<TestQueryVectorBuilderPlugin.TestQueryVectorBuilder> instanceReader() {
+        return TestQueryVectorBuilderPlugin.TestQueryVectorBuilder::new;
+    }
+
+    @Override
+    protected TestQueryVectorBuilderPlugin.TestQueryVectorBuilder createTestInstance() {
+        return new TestQueryVectorBuilderPlugin.TestQueryVectorBuilder(randomList(2, 1024, ESTestCase::randomFloat));
+    }
+
+    @Override
+    protected TestQueryVectorBuilderPlugin.TestQueryVectorBuilder createTestInstance(float[] expected) {
+        return new TestQueryVectorBuilderPlugin.TestQueryVectorBuilder(expected);
+    }
+
+    @Override
+    protected TestQueryVectorBuilderPlugin.TestQueryVectorBuilder mutateInstance(
+        TestQueryVectorBuilderPlugin.TestQueryVectorBuilder instance
+    ) throws IOException {
+        return createTestInstance();
+    }
+
+    @Override
+    protected TestQueryVectorBuilderPlugin.TestQueryVectorBuilder doParseInstance(XContentParser parser) throws IOException {
+        return TestQueryVectorBuilderPlugin.TestQueryVectorBuilder.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected void doAssertClientRequest(ActionRequest request, TestQueryVectorBuilderPlugin.TestQueryVectorBuilder builder) {
+        // Nothing to assert here as this object does not make client calls
+    }
+
+    @Override
+    protected ActionResponse createResponse(float[] array, TestQueryVectorBuilderPlugin.TestQueryVectorBuilder builder) {
+        return new ActionResponse.Empty();
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/vectors/TestQueryVectorBuilderPlugin.java b/server/src/test/java/org/elasticsearch/search/vectors/TestQueryVectorBuilderPlugin.java
new file mode 100644
index 000000000000..bdec097f355f
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/vectors/TestQueryVectorBuilderPlugin.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.search.vectors;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A SearchPlugin to exercise query vector builder
+ */
+class TestQueryVectorBuilderPlugin implements SearchPlugin {
+
+    static class TestQueryVectorBuilder implements QueryVectorBuilder {
+        private static final String NAME = "test_query_vector_builder";
+
+        private static final ParseField QUERY_VECTOR = new ParseField("query_vector");
+
+        @SuppressWarnings("unchecked")
+        static final ConstructingObjectParser<TestQueryVectorBuilder, Void> PARSER = new ConstructingObjectParser<>(
+            NAME + "_parser",
+            true,
+            a -> new TestQueryVectorBuilder((List<Float>) a[0])
+        );
+
+        static {
+            PARSER.declareFloatArray(ConstructingObjectParser.constructorArg(), QUERY_VECTOR);
+        }
+
+        private List<Float> vectorToBuild;
+
+        TestQueryVectorBuilder(List<Float> vectorToBuild) {
+            this.vectorToBuild = vectorToBuild;
+        }
+
+        TestQueryVectorBuilder(float[] expected) {
+            this.vectorToBuild = new ArrayList<>(expected.length);
+            for (float f : expected) {
+                vectorToBuild.add(f);
+            }
+        }
+
+        TestQueryVectorBuilder(StreamInput in) throws IOException {
+            this.vectorToBuild = in.readList(StreamInput::readFloat);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return builder.startObject().field(QUERY_VECTOR.getPreferredName(), vectorToBuild).endObject();
+        }
+
+        @Override
+        public String getWriteableName() {
+            return NAME;
+        }
+
+        @Override
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeCollection(vectorToBuild, StreamOutput::writeFloat);
+        }
+
+        @Override
+        public void buildVector(Client client, ActionListener<float[]> listener) {
+            float[] response = new float[vectorToBuild.size()];
+            int i = 0;
+            for (Float f : vectorToBuild) {
+                response[i++] = f;
+            }
+            listener.onResponse(response);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            TestQueryVectorBuilder that = (TestQueryVectorBuilder) o;
+            return Objects.equals(vectorToBuild, that.vectorToBuild);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(vectorToBuild);
+        }
+    }
+
+    @Override
+    public List<QueryVectorBuilderSpec<?>> getQueryVectorBuilders() {
+        return List.of(
+            new QueryVectorBuilderSpec<>(TestQueryVectorBuilder.NAME, TestQueryVectorBuilder::new, TestQueryVectorBuilder.PARSER::apply)
+        );
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatTests.java b/server/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatTests.java
index 46416922a5df..a65d50e897d1 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatTests.java
@@ -16,8 +16,8 @@
 import org.elasticsearch.common.blobstore.fs.FsBlobStore;
 import org.elasticsearch.common.blobstore.support.BlobMetadata;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.core.Streams;
 import org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.ToXContent;
@@ -29,6 +29,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Map;
+import java.util.function.Function;
 
 import static org.hamcrest.Matchers.greaterThan;
 
@@ -80,7 +81,8 @@ public void testBlobStoreOperations() throws IOException {
         ChecksumBlobStoreFormat<BlobObj> checksumSMILE = new ChecksumBlobStoreFormat<>(
             BLOB_CODEC,
             "%s",
-            (repoName, parser) -> BlobObj.fromXContent(parser)
+            (repoName, parser) -> BlobObj.fromXContent(parser),
+            Function.identity()
         );
 
         // Write blobs in different formats
@@ -105,7 +107,8 @@ public void testCompressionIsApplied() throws IOException {
         ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(
             BLOB_CODEC,
             "%s",
-            (repo, parser) -> BlobObj.fromXContent(parser)
+            (repo, parser) -> BlobObj.fromXContent(parser),
+            Function.identity()
         );
         BlobObj blobObj = new BlobObj(veryRedundantText.toString());
         checksumFormat.write(blobObj, blobContainer, "blob-comp", true);
@@ -124,7 +127,8 @@ public void testBlobCorruption() throws IOException {
         ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(
             BLOB_CODEC,
             "%s",
-            (repo, parser) -> BlobObj.fromXContent(parser)
+            (repo, parser) -> BlobObj.fromXContent(parser),
+            Function.identity()
         );
         checksumFormat.write(blobObj, blobContainer, "test-path", randomBoolean());
         assertEquals(checksumFormat.read("repo", blobContainer, "test-path", xContentRegistry()).getText(), testString);
diff --git a/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java
index 72fb1f17a4b1..db6f41e949f8 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.RestoreInProgress;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -436,7 +437,11 @@ private ClusterState addUnassignedShards(final ClusterState currentState, String
         final Index index = indexMetadata.getIndex();
 
         final RoutingTable.Builder routingTable = RoutingTable.builder(currentState.routingTable());
-        routingTable.add(IndexRoutingTable.builder(index).initializeAsNewRestore(indexMetadata, recoverySource, new HashSet<>()).build());
+        routingTable.add(
+            IndexRoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY, index)
+                .initializeAsNewRestore(indexMetadata, recoverySource, new HashSet<>())
+                .build()
+        );
 
         final RestoreInProgress.Builder restores = new RestoreInProgress.Builder(
             currentState.custom(RestoreInProgress.TYPE, RestoreInProgress.EMPTY)
diff --git a/server/src/test/java/org/elasticsearch/snapshots/RepositoriesMetadataSerializationTests.java b/server/src/test/java/org/elasticsearch/snapshots/RepositoriesMetadataSerializationTests.java
index 91dd9983ca70..2feb9a6c6582 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/RepositoriesMetadataSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/RepositoriesMetadataSerializationTests.java
@@ -16,7 +16,7 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.SimpleDiffableSerializationTestCase;
+import org.elasticsearch.test.ChunkedToXContentDiffableSerializationTestCase;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -24,7 +24,7 @@
 import java.util.Comparator;
 import java.util.List;
 
-public class RepositoriesMetadataSerializationTests extends SimpleDiffableSerializationTestCase<Custom> {
+public class RepositoriesMetadataSerializationTests extends ChunkedToXContentDiffableSerializationTestCase<Custom> {
 
     @Override
     protected Custom createTestInstance() {
@@ -116,5 +116,4 @@ protected Custom doParseInstance(XContentParser parser) throws IOException {
         repos.sort(Comparator.comparing(RepositoryMetadata::name));
         return new RepositoriesMetadata(repos);
     }
-
 }
diff --git a/server/src/test/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceTests.java
index 002b695897ec..35713eec56f5 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceTests.java
@@ -28,6 +28,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.common.util.CollectionUtils.appendToCopy;
 import static org.elasticsearch.health.HealthStatus.GREEN;
@@ -115,6 +116,52 @@ public void testIsGreenWhenNoMetadata() {
         );
     }
 
+    // We expose the indicator name and the diagnoses in the x-pack usage API. In order to index them properly in a telemetry index
+    // they need to be declared in the health-api-indexer.edn in the telemetry repository.
+    public void testMappedFieldsForTelemetry() {
+        assertThat(RepositoryIntegrityHealthIndicatorService.NAME, equalTo("repository_integrity"));
+        assertThat(
+            CORRUPTED_REPOSITORY.getUniqueId(),
+            equalTo("elasticsearch:health:repository_integrity:diagnosis:corrupt_repo_integrity")
+        );
+    }
+
+    public void testLimitNumberOfAffectedResources() {
+        List<RepositoryMetadata> repos = Stream.iterate(0, n -> n + 1)
+            .limit(20)
+            .map(i -> createRepositoryMetadata("corrupted-repo" + i, true))
+            .toList();
+        var clusterState = createClusterStateWith(new RepositoriesMetadata(repos));
+        var service = createRepositoryCorruptionHealthIndicatorService(clusterState);
+
+        {
+            assertThat(
+                service.calculate(true, 10, HealthInfo.EMPTY_HEALTH_INFO).diagnosisList(),
+                equalTo(
+                    List.of(
+                        new Diagnosis(
+                            CORRUPTED_REPOSITORY,
+                            List.of(
+                                new Diagnosis.Resource(
+                                    Type.SNAPSHOT_REPOSITORY,
+                                    repos.stream().limit(10).map(RepositoryMetadata::name).toList()
+                                )
+                            )
+                        )
+                    )
+                )
+            );
+        }
+
+        {
+            assertThat(
+                service.calculate(true, 0, HealthInfo.EMPTY_HEALTH_INFO).diagnosisList(),
+                equalTo(List.of(new Diagnosis(CORRUPTED_REPOSITORY, List.of(new Diagnosis.Resource(Type.SNAPSHOT_REPOSITORY, List.of())))))
+            );
+        }
+
+    }
+
     private static ClusterState createClusterStateWith(RepositoriesMetadata metadata) {
         var builder = ClusterState.builder(new ClusterName("test-cluster"));
         if (metadata != null) {
diff --git a/server/src/test/java/org/elasticsearch/snapshots/RestoreServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/RestoreServiceTests.java
index cfd80d2b00cf..a1f39c4f16be 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/RestoreServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/RestoreServiceTests.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
-import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -33,7 +32,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
@@ -122,16 +121,14 @@ public void testPrefixNotChanged() {
     }
 
     public void testRefreshRepositoryUuidsDoesNothingIfDisabled() {
-        final PlainActionFuture<Void> listener = new PlainActionFuture<>();
         final RepositoriesService repositoriesService = mock(RepositoriesService.class);
-        RestoreService.refreshRepositoryUuids(false, repositoriesService, listener);
-        assertTrue(listener.isDone());
+        final AtomicBoolean called = new AtomicBoolean();
+        RestoreService.refreshRepositoryUuids(false, repositoriesService, () -> assertTrue(called.compareAndSet(false, true)));
+        assertTrue(called.get());
         verifyNoMoreInteractions(repositoriesService);
     }
 
-    public void testRefreshRepositoryUuidsRefreshesAsNeeded() throws Exception {
-        final PlainActionFuture<Void> listener = new PlainActionFuture<>();
-
+    public void testRefreshRepositoryUuidsRefreshesAsNeeded() {
         final int repositoryCount = between(1, 5);
         final Map<String, Repository> repositories = Maps.newMapWithExpectedSize(repositoryCount);
         final Set<String> pendingRefreshes = new HashSet<>();
@@ -177,8 +174,9 @@ public void testRefreshRepositoryUuidsRefreshesAsNeeded() throws Exception {
 
         final RepositoriesService repositoriesService = mock(RepositoriesService.class);
         when(repositoriesService.getRepositories()).thenReturn(repositories);
-        RestoreService.refreshRepositoryUuids(true, repositoriesService, listener);
-        assertNull(listener.get(0L, TimeUnit.SECONDS));
+        final AtomicBoolean completed = new AtomicBoolean();
+        RestoreService.refreshRepositoryUuids(true, repositoriesService, () -> assertTrue(completed.compareAndSet(false, true)));
+        assertTrue(completed.get());
         assertThat(pendingRefreshes, empty());
         finalAssertions.forEach(Runnable::run);
     }
diff --git a/server/src/test/java/org/elasticsearch/snapshots/ShardSnapshotResultWireSerializationTests.java b/server/src/test/java/org/elasticsearch/snapshots/ShardSnapshotResultWireSerializationTests.java
index 5939293a0c29..453636bf5818 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/ShardSnapshotResultWireSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/ShardSnapshotResultWireSerializationTests.java
@@ -14,8 +14,6 @@
 import org.elasticsearch.repositories.ShardSnapshotResult;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
-
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
 
@@ -32,7 +30,7 @@ protected ShardSnapshotResult createTestInstance() {
     }
 
     @Override
-    protected ShardSnapshotResult mutateInstance(ShardSnapshotResult instance) throws IOException {
+    protected ShardSnapshotResult mutateInstance(ShardSnapshotResult instance) {
         return mutateShardSnapshotResult(instance);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/snapshots/ShardSnapshotStatusWireSerializationTests.java b/server/src/test/java/org/elasticsearch/snapshots/ShardSnapshotStatusWireSerializationTests.java
index 2caa1b54bc38..7257cc913e33 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/ShardSnapshotStatusWireSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/ShardSnapshotStatusWireSerializationTests.java
@@ -14,8 +14,6 @@
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.hamcrest.Matchers;
 
-import java.io.IOException;
-
 import static org.elasticsearch.snapshots.ShardSnapshotResultWireSerializationTests.randomShardSnapshotResult;
 import static org.hamcrest.Matchers.containsString;
 
@@ -40,7 +38,7 @@ protected SnapshotsInProgress.ShardSnapshotStatus createTestInstance() {
     }
 
     @Override
-    protected SnapshotsInProgress.ShardSnapshotStatus mutateInstance(SnapshotsInProgress.ShardSnapshotStatus instance) throws IOException {
+    protected SnapshotsInProgress.ShardSnapshotStatus mutateInstance(SnapshotsInProgress.ShardSnapshotStatus instance) {
         if (instance.state() == SnapshotsInProgress.ShardState.QUEUED) {
             assert instance == SnapshotsInProgress.ShardSnapshotStatus.UNASSIGNED_QUEUED;
             return randomValueOtherThanMany(i -> i.state() == SnapshotsInProgress.ShardState.QUEUED, this::createTestInstance);
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotFeatureInfoTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotFeatureInfoTests.java
index 8c5070ed41e5..28684f14e3a4 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotFeatureInfoTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotFeatureInfoTests.java
@@ -38,7 +38,7 @@ public static SnapshotFeatureInfo randomSnapshotFeatureInfo() {
     }
 
     @Override
-    protected SnapshotFeatureInfo mutateInstance(SnapshotFeatureInfo instance) throws IOException {
+    protected SnapshotFeatureInfo mutateInstance(SnapshotFeatureInfo instance) {
         if (randomBoolean()) {
             return new SnapshotFeatureInfo(
                 randomValueOtherThan(instance.getPluginName(), () -> randomAlphaOfLengthBetween(5, 20)),
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoBlobSerializationTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoBlobSerializationTests.java
index a171beb6c905..43d8d0d4fabd 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoBlobSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoBlobSerializationTests.java
@@ -24,7 +24,7 @@ protected SnapshotInfo createTestInstance() {
     }
 
     @Override
-    protected SnapshotInfo mutateInstance(SnapshotInfo instance) throws IOException {
+    protected SnapshotInfo mutateInstance(SnapshotInfo instance) {
         return SnapshotInfoTestUtils.mutateSnapshotInfo(instance);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
index 3515728b69a2..7853b47a0db3 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
@@ -174,10 +174,10 @@
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.disruption.DisruptableMockTransport;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.tracing.Tracer;
 import org.elasticsearch.transport.BytesRefRecycler;
+import org.elasticsearch.transport.DisruptableMockTransport;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.junit.After;
@@ -665,8 +665,8 @@ public void testBulkSnapshotDeleteWithAbort() {
         final int inProgressSnapshots = randomIntBetween(1, 5);
         final StepListener<Collection<CreateSnapshotResponse>> createOtherSnapshotResponseStepListener = new StepListener<>();
         final ActionListener<CreateSnapshotResponse> createSnapshotListener = new GroupedActionListener<>(
-            createOtherSnapshotResponseStepListener,
-            inProgressSnapshots
+            inProgressSnapshots,
+            createOtherSnapshotResponseStepListener
         );
 
         continueOrDie(createSnapshotResponseStepListener, createSnapshotResponse -> {
@@ -825,7 +825,7 @@ public void testConcurrentSnapshotDeleteAndDeleteIndex() throws IOException {
             firstIndex.set(masterNode.clusterService.state().metadata().index(index).getIndex());
             // create a few more indices to make it more likely that the subsequent index delete operation happens before snapshot
             // finalization
-            final GroupedActionListener<CreateIndexResponse> listener = new GroupedActionListener<>(createIndicesListener, indices);
+            final GroupedActionListener<CreateIndexResponse> listener = new GroupedActionListener<>(indices, createIndicesListener);
             for (int i = 0; i < indices; ++i) {
                 client().admin().indices().create(new CreateIndexRequest("index-" + i), listener);
             }
@@ -1172,8 +1172,8 @@ public void testRunConcurrentSnapshots() {
 
         final StepListener<Collection<CreateSnapshotResponse>> allSnapshotsListener = new StepListener<>();
         final ActionListener<CreateSnapshotResponse> snapshotListener = new GroupedActionListener<>(
-            allSnapshotsListener,
-            snapshotNames.size()
+            snapshotNames.size(),
+            allSnapshotsListener
         );
         final AtomicBoolean doneIndexing = new AtomicBoolean(false);
         continueOrDie(createRepoAndIndex(repoName, index, shards), createIndexResponse -> {
@@ -1652,7 +1652,7 @@ protected void connectToNodesAndWait(ClusterState newClusterState) {
                     }
                 );
                 recoverySettings = new RecoverySettings(settings, clusterSettings);
-                mockTransport = new DisruptableMockTransport(node, logger, deterministicTaskQueue) {
+                mockTransport = new DisruptableMockTransport(node, deterministicTaskQueue) {
                     @Override
                     protected ConnectionStatus getConnectionStatus(DiscoveryNode destination) {
                         if (node.equals(destination)) {
@@ -1831,6 +1831,25 @@ protected void assertSnapshotOrGenericThread() {
                     PeerOnlyRecoveryPlannerService.INSTANCE
                 );
 
+                final ResponseCollectorService responseCollectorService = new ResponseCollectorService(clusterService);
+                final SearchTransportService searchTransportService = new SearchTransportService(
+                    transportService,
+                    client,
+                    SearchExecutionStatsCollector.makeWrapper(responseCollectorService)
+                );
+                final SearchService searchService = new SearchService(
+                    clusterService,
+                    indicesService,
+                    threadPool,
+                    scriptService,
+                    bigArrays,
+                    new FetchPhase(Collections.emptyList()),
+                    responseCollectorService,
+                    new NoneCircuitBreakerService(),
+                    EmptySystemIndices.INSTANCE.getExecutorSelector(),
+                    Tracer.NOOP
+                );
+
                 final SnapshotFilesProvider snapshotFilesProvider = new SnapshotFilesProvider(repositoriesService);
                 indicesClusterStateService = new IndicesClusterStateService(
                     settings,
@@ -1840,7 +1859,7 @@ protected void assertSnapshotOrGenericThread() {
                     new PeerRecoveryTargetService(threadPool, transportService, recoverySettings, clusterService, snapshotFilesProvider),
                     shardStateAction,
                     repositoriesService,
-                    mock(SearchService.class),
+                    searchService,
                     peerRecoverySourceService,
                     snapshotShardsService,
                     new PrimaryReplicaSyncer(
@@ -1964,24 +1983,7 @@ protected void assertSnapshotOrGenericThread() {
                         EmptySystemIndices.INSTANCE
                     )
                 );
-                final ResponseCollectorService responseCollectorService = new ResponseCollectorService(clusterService);
-                final SearchTransportService searchTransportService = new SearchTransportService(
-                    transportService,
-                    client,
-                    SearchExecutionStatsCollector.makeWrapper(responseCollectorService)
-                );
-                final SearchService searchService = new SearchService(
-                    clusterService,
-                    indicesService,
-                    threadPool,
-                    scriptService,
-                    bigArrays,
-                    new FetchPhase(Collections.emptyList()),
-                    responseCollectorService,
-                    new NoneCircuitBreakerService(),
-                    EmptySystemIndices.INSTANCE.getExecutorSelector(),
-                    Tracer.NOOP
-                );
+
                 SearchPhaseController searchPhaseController = new SearchPhaseController(searchService::aggReduceContextBuilder);
                 actions.put(
                     SearchAction.INSTANCE,
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java
index afc5005f8fd1..c6a6ef643ae7 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java
@@ -26,11 +26,10 @@
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.ShardGeneration;
 import org.elasticsearch.repositories.ShardSnapshotResult;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.SimpleDiffableWireSerializationTestCase;
 import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -42,7 +41,6 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -418,115 +416,110 @@ public void testXContent() throws IOException {
             )
         );
 
-        try (XContentBuilder builder = jsonBuilder()) {
-            builder.humanReadable(true);
-            builder.startObject();
-            sip.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            builder.endObject();
-            String json = Strings.toString(builder);
-            assertThat(
-                json,
-                anyOf(
-                    equalTo(XContentHelper.stripWhitespace("""
+        AbstractChunkedSerializingTestCase.assertChunkCount(sip, instance -> Math.toIntExact(instance.asStream().count() + 2));
+        final var json = Strings.toString(sip, false, true);
+        assertThat(
+            json,
+            anyOf(
+                equalTo(XContentHelper.stripWhitespace("""
+                    {
+                      "snapshots": [
                         {
-                          "snapshots": [
+                          "repository": "repo",
+                          "snapshot": "name",
+                          "uuid": "uuid",
+                          "include_global_state": true,
+                          "partial": true,
+                          "state": "SUCCESS",
+                          "indices": [ { "name": "index", "id": "uuid" } ],
+                          "start_time": "1970-01-01T00:20:34.567Z",
+                          "start_time_millis": 1234567,
+                          "repository_state_id": 0,
+                          "shards": [
                             {
-                              "repository": "repo",
-                              "snapshot": "name",
-                              "uuid": "uuid",
-                              "include_global_state": true,
-                              "partial": true,
+                              "index": {
+                                "index_name": "index",
+                                "index_uuid": "uuid"
+                              },
+                              "shard": 0,
                               "state": "SUCCESS",
-                              "indices": [ { "name": "index", "id": "uuid" } ],
-                              "start_time": "1970-01-01T00:20:34.567Z",
-                              "start_time_millis": 1234567,
-                              "repository_state_id": 0,
-                              "shards": [
-                                {
-                                  "index": {
-                                    "index_name": "index",
-                                    "index_uuid": "uuid"
-                                  },
-                                  "shard": 0,
-                                  "state": "SUCCESS",
-                                  "generation": "shardgen",
-                                  "node": "nodeId",
-                                  "result": {
-                                    "generation": "shardgen",
-                                    "size": "1b",
-                                    "size_in_bytes": 1,
-                                    "segments": 1
-                                  }
-                                },
-                                {
-                                  "index": {
-                                    "index_name": "index",
-                                    "index_uuid": "uuid"
-                                  },
-                                  "shard": 1,
-                                  "state": "FAILED",
-                                  "generation": "fail-gen",
-                                  "node": "nodeId",
-                                  "reason": "failure-reason"
-                                }
-                              ],
-                              "feature_states": [],
-                              "data_streams": []
+                              "generation": "shardgen",
+                              "node": "nodeId",
+                              "result": {
+                                "generation": "shardgen",
+                                "size": "1b",
+                                "size_in_bytes": 1,
+                                "segments": 1
+                              }
+                            },
+                            {
+                              "index": {
+                                "index_name": "index",
+                                "index_uuid": "uuid"
+                              },
+                              "shard": 1,
+                              "state": "FAILED",
+                              "generation": "fail-gen",
+                              "node": "nodeId",
+                              "reason": "failure-reason"
                             }
-                          ]
-                        }""")),
-                    // or the shards might be in the other order:
-                    equalTo(XContentHelper.stripWhitespace("""
+                          ],
+                          "feature_states": [],
+                          "data_streams": []
+                        }
+                      ]
+                    }""")),
+                // or the shards might be in the other order:
+                equalTo(XContentHelper.stripWhitespace("""
+                    {
+                      "snapshots": [
                         {
-                          "snapshots": [
+                          "repository": "repo",
+                          "snapshot": "name",
+                          "uuid": "uuid",
+                          "include_global_state": true,
+                          "partial": true,
+                          "state": "SUCCESS",
+                          "indices": [ { "name": "index", "id": "uuid" } ],
+                          "start_time": "1970-01-01T00:20:34.567Z",
+                          "start_time_millis": 1234567,
+                          "repository_state_id": 0,
+                          "shards": [
+                            {
+                              "index": {
+                                "index_name": "index",
+                                "index_uuid": "uuid"
+                              },
+                              "shard": 1,
+                              "state": "FAILED",
+                              "generation": "fail-gen",
+                              "node": "nodeId",
+                              "reason": "failure-reason"
+                            },
                             {
-                              "repository": "repo",
-                              "snapshot": "name",
-                              "uuid": "uuid",
-                              "include_global_state": true,
-                              "partial": true,
+                              "index": {
+                                "index_name": "index",
+                                "index_uuid": "uuid"
+                              },
+                              "shard": 0,
                               "state": "SUCCESS",
-                              "indices": [ { "name": "index", "id": "uuid" } ],
-                              "start_time": "1970-01-01T00:20:34.567Z",
-                              "start_time_millis": 1234567,
-                              "repository_state_id": 0,
-                              "shards": [
-                                {
-                                  "index": {
-                                    "index_name": "index",
-                                    "index_uuid": "uuid"
-                                  },
-                                  "shard": 1,
-                                  "state": "FAILED",
-                                  "generation": "fail-gen",
-                                  "node": "nodeId",
-                                  "reason": "failure-reason"
-                                },
-                                {
-                                  "index": {
-                                    "index_name": "index",
-                                    "index_uuid": "uuid"
-                                  },
-                                  "shard": 0,
-                                  "state": "SUCCESS",
-                                  "generation": "shardgen",
-                                  "node": "nodeId",
-                                  "result": {
-                                    "generation": "shardgen",
-                                    "size": "1b",
-                                    "size_in_bytes": 1,
-                                    "segments": 1
-                                  }
-                                }
-                              ],
-                              "feature_states": [],
-                              "data_streams": []
+                              "generation": "shardgen",
+                              "node": "nodeId",
+                              "result": {
+                                "generation": "shardgen",
+                                "size": "1b",
+                                "size_in_bytes": 1,
+                                "segments": 1
+                              }
                             }
-                          ]
-                        }"""))
-                )
-            );
-        }
+                          ],
+                          "feature_states": [],
+                          "data_streams": []
+                        }
+                      ]
+                    }"""))
+            )
+        );
     }
 
     public static State randomState(Map<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shards) {
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java
index 76f8b0c454e1..1de240b3bbde 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java
@@ -19,9 +19,12 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.service.ClusterStateTaskExecutorUtils;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
@@ -467,7 +470,22 @@ private static ClusterState stateWithUnassignedIndices(String... indexNames) {
         final RoutingTable.Builder routingTable = RoutingTable.builder();
         for (String index : indexNames) {
             final Index idx = metaBuilder.get(index).getIndex();
-            routingTable.add(IndexRoutingTable.builder(idx).addIndexShard(IndexShardRoutingTable.builder(new ShardId(idx, 0))));
+            final ShardId shardId = new ShardId(idx, 0);
+            routingTable.add(
+                IndexRoutingTable.builder(idx)
+                    .addIndexShard(
+                        IndexShardRoutingTable.builder(shardId)
+                            .addShard(
+                                ShardRouting.newUnassigned(
+                                    shardId,
+                                    true,
+                                    RecoverySource.EmptyStoreRecoverySource.INSTANCE,
+                                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "test"),
+                                    ShardRouting.Role.DEFAULT
+                                )
+                            )
+                    )
+            );
         }
         return ClusterState.builder(ClusterState.EMPTY_STATE).metadata(metaBuilder).routingTable(routingTable.build()).build();
     }
diff --git a/server/src/test/java/org/elasticsearch/tasks/CancelTasksResponseTests.java b/server/src/test/java/org/elasticsearch/tasks/CancelTasksResponseTests.java
index c44611c0eedf..d00e0cc828f4 100644
--- a/server/src/test/java/org/elasticsearch/tasks/CancelTasksResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/tasks/CancelTasksResponseTests.java
@@ -13,8 +13,11 @@
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.test.AbstractXContentTestCase;
 import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -27,12 +30,20 @@
 
 import static org.hamcrest.Matchers.equalTo;
 
-public class CancelTasksResponseTests extends AbstractXContentTestCase<CancelTasksResponse> {
+public class CancelTasksResponseTests extends AbstractXContentTestCase<CancelTasksResponseTests.CancelTasksResponseWrapper> {
+
+    // CancelTasksResponse doesn't directly implement ToXContent because it has multiple XContent representations, so we must wrap here
+    public record CancelTasksResponseWrapper(CancelTasksResponse in) implements ToXContentObject {
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+            return ChunkedToXContent.wrapAsToXContent(in.groupedByNone()).toXContent(builder, params);
+        }
+    }
 
     @Override
-    protected CancelTasksResponse createTestInstance() {
+    protected CancelTasksResponseWrapper createTestInstance() {
         List<TaskInfo> randomTasks = randomTasks();
-        return new CancelTasksResponse(randomTasks, Collections.emptyList(), Collections.emptyList());
+        return new CancelTasksResponseWrapper(new CancelTasksResponse(randomTasks, Collections.emptyList(), Collections.emptyList()));
     }
 
     private static List<TaskInfo> randomTasks() {
@@ -50,7 +61,9 @@ protected Predicate<String> getRandomFieldsExcludeFilter() {
     }
 
     @Override
-    protected void assertEqualInstances(CancelTasksResponse expectedInstance, CancelTasksResponse newInstance) {
+    protected void assertEqualInstances(CancelTasksResponseWrapper expectedInstanceWrapper, CancelTasksResponseWrapper newInstanceWrapper) {
+        final var expectedInstance = expectedInstanceWrapper.in();
+        final var newInstance = newInstanceWrapper.in();
         assertNotSame(expectedInstance, newInstance);
         assertThat(newInstance.getTasks(), equalTo(expectedInstance.getTasks()));
         ListTasksResponseTests.assertOnNodeFailures(newInstance.getNodeFailures(), expectedInstance.getNodeFailures());
@@ -58,8 +71,8 @@ protected void assertEqualInstances(CancelTasksResponse expectedInstance, Cancel
     }
 
     @Override
-    protected CancelTasksResponse doParseInstance(XContentParser parser) {
-        return CancelTasksResponse.fromXContent(parser);
+    protected CancelTasksResponseWrapper doParseInstance(XContentParser parser) {
+        return new CancelTasksResponseWrapper(CancelTasksResponse.fromXContent(parser));
     }
 
     @Override
@@ -78,7 +91,7 @@ protected boolean assertToXContentEquivalence() {
      * without failures, and this other test with failures where we disable asserting on xcontent equivalence at the end.
      */
     public void testFromXContentWithFailures() throws IOException {
-        Supplier<CancelTasksResponse> instanceSupplier = CancelTasksResponseTests::createTestInstanceWithFailures;
+        Supplier<CancelTasksResponseWrapper> instanceSupplier = CancelTasksResponseTests::createTestInstanceWithFailures;
         // with random fields insertion in the inner exceptions, some random stuff may be parsed back as metadata,
         // but that does not bother our assertions, as we only want to test that we don't break.
         boolean supportsUnknownFields = true;
@@ -98,7 +111,7 @@ public void testFromXContentWithFailures() throws IOException {
         );
     }
 
-    private static CancelTasksResponse createTestInstanceWithFailures() {
+    private static CancelTasksResponseWrapper createTestInstanceWithFailures() {
         int numNodeFailures = randomIntBetween(0, 3);
         List<FailedNodeException> nodeFailures = new ArrayList<>(numNodeFailures);
         for (int i = 0; i < numNodeFailures; i++) {
@@ -109,7 +122,7 @@ private static CancelTasksResponse createTestInstanceWithFailures() {
         for (int i = 0; i < numTaskFailures; i++) {
             taskFailures.add(new TaskOperationFailure(randomAlphaOfLength(5), randomLong(), new IllegalStateException()));
         }
-        return new CancelTasksResponse(randomTasks(), taskFailures, nodeFailures);
+        return new CancelTasksResponseWrapper(new CancelTasksResponse(randomTasks(), taskFailures, nodeFailures));
     }
 
 }
diff --git a/server/src/test/java/org/elasticsearch/tasks/ListTasksResponseTests.java b/server/src/test/java/org/elasticsearch/tasks/ListTasksResponseTests.java
index 49d01f75a4d0..b2225e7d73e8 100644
--- a/server/src/test/java/org/elasticsearch/tasks/ListTasksResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/tasks/ListTasksResponseTests.java
@@ -12,9 +12,13 @@
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.test.AbstractXContentTestCase;
-import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -27,10 +31,19 @@
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
+import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
-public class ListTasksResponseTests extends AbstractXContentTestCase<ListTasksResponse> {
+public class ListTasksResponseTests extends AbstractXContentTestCase<ListTasksResponseTests.ListTasksResponseWrapper> {
+
+    // ListTasksResponse doesn't directly implement ToXContent because it has multiple XContent representations, so we must wrap here
+    public record ListTasksResponseWrapper(ListTasksResponse in) implements ToXContentObject {
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return ChunkedToXContent.wrapAsToXContent(in.groupedByNone()).toXContent(builder, params);
+        }
+    }
 
     public void testEmptyToString() {
         assertEquals("""
@@ -79,9 +92,9 @@ public void testNonEmptyToString() {
     }
 
     @Override
-    protected ListTasksResponse createTestInstance() {
+    protected ListTasksResponseWrapper createTestInstance() {
         // failures are tested separately, so we can test xcontent equivalence at least when we have no failures
-        return new ListTasksResponse(randomTasks(), Collections.emptyList(), Collections.emptyList());
+        return new ListTasksResponseWrapper(new ListTasksResponse(randomTasks(), Collections.emptyList(), Collections.emptyList()));
     }
 
     private static List<TaskInfo> randomTasks() {
@@ -93,8 +106,8 @@ private static List<TaskInfo> randomTasks() {
     }
 
     @Override
-    protected ListTasksResponse doParseInstance(XContentParser parser) {
-        return ListTasksResponse.fromXContent(parser);
+    protected ListTasksResponseWrapper doParseInstance(XContentParser parser) {
+        return new ListTasksResponseWrapper(ListTasksResponse.fromXContent(parser));
     }
 
     @Override
@@ -109,7 +122,9 @@ protected Predicate<String> getRandomFieldsExcludeFilter() {
     }
 
     @Override
-    protected void assertEqualInstances(ListTasksResponse expectedInstance, ListTasksResponse newInstance) {
+    protected void assertEqualInstances(ListTasksResponseWrapper expectedInstanceWrapper, ListTasksResponseWrapper newInstanceWrapper) {
+        final var expectedInstance = expectedInstanceWrapper.in();
+        final var newInstance = newInstanceWrapper.in();
         assertNotSame(expectedInstance, newInstance);
         assertThat(newInstance.getTasks(), equalTo(expectedInstance.getTasks()));
         assertOnNodeFailures(newInstance.getNodeFailures(), expectedInstance.getNodeFailures());
@@ -149,7 +164,7 @@ protected static void assertOnTaskFailures(List<TaskOperationFailure> taskFailur
      * without failures, and this other test with failures where we disable asserting on xcontent equivalence at the end.
      */
     public void testFromXContentWithFailures() throws IOException {
-        Supplier<ListTasksResponse> instanceSupplier = ListTasksResponseTests::createTestInstanceWithFailures;
+        Supplier<ListTasksResponseWrapper> instanceSupplier = ListTasksResponseTests::createTestInstanceWithFailures;
         // with random fields insertion in the inner exceptions, some random stuff may be parsed back as metadata,
         // but that does not bother our assertions, as we only want to test that we don't break.
         boolean supportsUnknownFields = true;
@@ -165,11 +180,21 @@ public void testFromXContentWithFailures() throws IOException {
             this::doParseInstance,
             this::assertEqualInstances,
             assertToXContentEquivalence,
-            ToXContent.EMPTY_PARAMS
+            EMPTY_PARAMS
+        );
+    }
+
+    public void testChunkedEncoding() {
+        final var response = createTestInstanceWithFailures().in();
+        AbstractChunkedSerializingTestCase.assertChunkCount(response.groupedByNone(), o -> response.getTasks().size() + 2);
+        AbstractChunkedSerializingTestCase.assertChunkCount(response.groupedByParent(), o -> response.getTaskGroups().size() + 2);
+        AbstractChunkedSerializingTestCase.assertChunkCount(
+            response.groupedByNode(() -> DiscoveryNodes.EMPTY_NODES),
+            o -> 2 + response.getPerNodeTasks().values().stream().mapToInt(entry -> 2 + entry.size()).sum()
         );
     }
 
-    private static ListTasksResponse createTestInstanceWithFailures() {
+    private static ListTasksResponseWrapper createTestInstanceWithFailures() {
         int numNodeFailures = randomIntBetween(0, 3);
         List<FailedNodeException> nodeFailures = new ArrayList<>(numNodeFailures);
         for (int i = 0; i < numNodeFailures; i++) {
@@ -180,6 +205,6 @@ private static ListTasksResponse createTestInstanceWithFailures() {
         for (int i = 0; i < numTaskFailures; i++) {
             taskFailures.add(new TaskOperationFailure(randomAlphaOfLength(5), randomLong(), new IllegalStateException()));
         }
-        return new ListTasksResponse(randomTasks(), taskFailures, nodeFailures);
+        return new ListTasksResponseWrapper(new ListTasksResponse(randomTasks(), taskFailures, nodeFailures));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/tasks/TaskManagerTests.java b/server/src/test/java/org/elasticsearch/tasks/TaskManagerTests.java
index a1e7aad45293..05343247fe66 100644
--- a/server/src/test/java/org/elasticsearch/tasks/TaskManagerTests.java
+++ b/server/src/test/java/org/elasticsearch/tasks/TaskManagerTests.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.tasks;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
@@ -184,7 +184,7 @@ void cancelTaskAndDescendants(CancellableTask task, String reason, boolean waitF
                             threadPool,
                             "action-" + i,
                             randomIntBetween(0, 1000),
-                            Version.CURRENT
+                            TransportVersion.CURRENT
                         );
                         taskManager.setBan(taskId, "test", tcpTransportChannel);
                     }
@@ -233,7 +233,7 @@ void cancelTaskAndDescendants(CancellableTask task, String reason, boolean waitF
                     threadPool,
                     "action",
                     randomIntBetween(1, 10000),
-                    Version.CURRENT
+                    TransportVersion.CURRENT
                 )
             );
         }
diff --git a/server/src/test/java/org/elasticsearch/threadpool/ESThreadPoolTestCase.java b/server/src/test/java/org/elasticsearch/threadpool/ESThreadPoolTestCase.java
index e43879320b09..ccae5f946d14 100644
--- a/server/src/test/java/org/elasticsearch/threadpool/ESThreadPoolTestCase.java
+++ b/server/src/test/java/org/elasticsearch/threadpool/ESThreadPoolTestCase.java
@@ -41,7 +41,7 @@ protected final void terminateThreadPoolIfNeeded(final ThreadPool threadPool) th
 
     static String randomThreadPool(final ThreadPool.ThreadPoolType type) {
         return randomFrom(
-            ThreadPool.THREAD_POOL_TYPES.entrySet().stream().filter(t -> t.getValue().equals(type)).map(Map.Entry::getKey).toList()
+            ThreadPool.THREAD_POOL_TYPES.entrySet().stream().filter(t -> t.getValue().equals(type)).map(Map.Entry::getKey).sorted().toList()
         );
     }
 
diff --git a/server/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java b/server/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
index 07b8a629c81e..1d0e8fa76800 100644
--- a/server/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
+++ b/server/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
@@ -30,6 +30,7 @@
 import java.util.function.BiConsumer;
 import java.util.function.Function;
 
+import static org.elasticsearch.threadpool.ThreadPool.getMaxSnapshotThreadPoolSize;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
@@ -48,12 +49,7 @@ public void testScalingThreadPoolConfiguration() throws InterruptedException {
             core = randomIntBetween(0, 8);
             builder.put("thread_pool." + threadPoolName + ".core", core);
         } else {
-            // the defaults
-            core = switch (threadPoolName) {
-                case "generic" -> 4;
-                case "management" -> 2;
-                default -> 1;
-            };
+            core = "generic".equals(threadPoolName) ? 4 : 1; // the defaults
         }
 
         final int availableProcessors = Runtime.getRuntime().availableProcessors();
@@ -73,7 +69,9 @@ public void testScalingThreadPoolConfiguration() throws InterruptedException {
             expectedMax = randomIntBetween(Math.max(1, core), 16);
             builder.put("thread_pool." + threadPoolName + ".max", expectedMax);
         } else {
-            expectedMax = threadPoolName.equals(ThreadPool.Names.SNAPSHOT) ? 10 : maxBasedOnNumberOfProcessors;
+            expectedMax = threadPoolName.equals(ThreadPool.Names.SNAPSHOT)
+                ? getMaxSnapshotThreadPoolSize(processors)
+                : maxBasedOnNumberOfProcessors;
         }
 
         final long keepAlive;
@@ -111,7 +109,7 @@ public void testScalingThreadPoolConfiguration() throws InterruptedException {
     private int expectedSize(final String threadPoolName, final int numberOfProcessors) {
         final Map<String, Function<Integer, Integer>> sizes = new HashMap<>();
         sizes.put(ThreadPool.Names.GENERIC, n -> ThreadPool.boundedBy(4 * n, 128, 512));
-        sizes.put(ThreadPool.Names.MANAGEMENT, n -> ThreadPool.boundedBy(n, 2, 5));
+        sizes.put(ThreadPool.Names.MANAGEMENT, n -> ThreadPool.boundedBy(n, 1, 5));
         sizes.put(ThreadPool.Names.FLUSH, ThreadPool::halfAllocatedProcessorsMaxFive);
         sizes.put(ThreadPool.Names.REFRESH, ThreadPool::halfAllocatedProcessorsMaxTen);
         sizes.put(ThreadPool.Names.WARMER, ThreadPool::halfAllocatedProcessorsMaxFive);
@@ -154,11 +152,7 @@ public void testScalingThreadPoolIsBounded() throws InterruptedException {
 
     public void testScalingThreadPoolThreadsAreTerminatedAfterKeepAlive() throws InterruptedException {
         final String threadPoolName = randomThreadPool(ThreadPool.ThreadPoolType.SCALING);
-        final int min = switch (threadPoolName) {
-            case "generic" -> 4;
-            case "management" -> 2;
-            default -> 1;
-        };
+        final int min = "generic".equals(threadPoolName) ? 4 : 1;
         final Settings settings = Settings.builder()
             .put("thread_pool." + threadPoolName + ".max", 128)
             .put("thread_pool." + threadPoolName + ".keep_alive", "1ms")
diff --git a/server/src/test/java/org/elasticsearch/threadpool/ScheduleWithFixedDelayTests.java b/server/src/test/java/org/elasticsearch/threadpool/ScheduleWithFixedDelayTests.java
index 9e21ccfa4904..efa83b672ecc 100644
--- a/server/src/test/java/org/elasticsearch/threadpool/ScheduleWithFixedDelayTests.java
+++ b/server/src/test/java/org/elasticsearch/threadpool/ScheduleWithFixedDelayTests.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.threadpool;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.BaseFuture;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
@@ -178,9 +179,9 @@ public void testCancellingRunnable() throws Exception {
     }
 
     public void testBlockingCallOnSchedulerThreadFails() throws Exception {
-        final BaseFuture<Object> future = new BaseFuture<Object>() {
+        final BaseFuture<Object> future = new BaseFuture<>() {
         };
-        final TestFuture resultsFuture = new TestFuture();
+        final PlainActionFuture<Object> resultsFuture = new PlainActionFuture<>();
         final boolean getWithTimeout = randomBoolean();
 
         final Runnable runnable = () -> {
@@ -191,9 +192,9 @@ public void testBlockingCallOnSchedulerThreadFails() throws Exception {
                 } else {
                     obj = future.get();
                 }
-                resultsFuture.futureDone(obj);
+                resultsFuture.onResponse(obj);
             } catch (Throwable t) {
-                resultsFuture.futureDone(t);
+                resultsFuture.onResponse(t);
             }
         };
 
@@ -208,8 +209,14 @@ public void testBlockingCallOnSchedulerThreadFails() throws Exception {
     }
 
     public void testBlockingCallOnNonSchedulerThreadAllowed() throws Exception {
-        final TestFuture future = new TestFuture();
-        final TestFuture resultsFuture = new TestFuture();
+        class TestFuture extends BaseFuture<Object> {
+            public boolean set(Object o) {
+                return super.set(o);
+            }
+        }
+        final TestFuture future = new TestFuture() {
+        };
+        final PlainActionFuture<Object> resultsFuture = new PlainActionFuture<>();
         final boolean rethrow = randomBoolean();
         final boolean getWithTimeout = randomBoolean();
 
@@ -221,9 +228,9 @@ public void testBlockingCallOnNonSchedulerThreadAllowed() throws Exception {
                 } else {
                     obj = future.get();
                 }
-                resultsFuture.futureDone(obj);
+                resultsFuture.onResponse(obj);
             } catch (Throwable t) {
-                resultsFuture.futureDone(t);
+                resultsFuture.onResponse(t);
                 if (rethrow) {
                     throw new RuntimeException(t);
                 }
@@ -234,7 +241,7 @@ public void testBlockingCallOnNonSchedulerThreadAllowed() throws Exception {
         assertFalse(resultsFuture.isDone());
 
         final Object o = new Object();
-        future.futureDone(o);
+        future.set(o);
 
         final Object resultingObject = resultsFuture.get();
         assertThat(resultingObject, sameInstance(o));
@@ -288,10 +295,4 @@ public void testRunnableDoesNotRunAfterCancellation() throws Exception {
             assertBusy(() -> assertThat(counter.get(), equalTo(iterations)), 5 * interval.millis(), TimeUnit.MILLISECONDS);
         }
     }
-
-    static final class TestFuture extends BaseFuture<Object> {
-        boolean futureDone(Object value) {
-            return set(value);
-        }
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java b/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java
index d22538e986cc..f4ad6075ef16 100644
--- a/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java
+++ b/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java
@@ -13,6 +13,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.core.TimeValue;
@@ -25,6 +26,8 @@
 import static org.elasticsearch.threadpool.ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING;
 import static org.elasticsearch.threadpool.ThreadPool.LATE_TIME_INTERVAL_WARN_THRESHOLD_SETTING;
 import static org.elasticsearch.threadpool.ThreadPool.assertCurrentMethodIsNotCalledRecursively;
+import static org.elasticsearch.threadpool.ThreadPool.getMaxSnapshotThreadPoolSize;
+import static org.elasticsearch.threadpool.ThreadPool.halfAllocatedProcessorsMaxFive;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 
@@ -328,4 +331,21 @@ public void testForceMergeThreadPoolSize() {
             assertTrue(terminate(threadPool));
         }
     }
+
+    public void testGetMaxSnapshotCores() {
+        int allocatedProcessors = randomIntBetween(1, 16);
+        assertThat(
+            getMaxSnapshotThreadPoolSize(allocatedProcessors, ByteSizeValue.ofMb(400)),
+            equalTo(halfAllocatedProcessorsMaxFive(allocatedProcessors))
+        );
+        allocatedProcessors = randomIntBetween(1, 16);
+        assertThat(
+            getMaxSnapshotThreadPoolSize(allocatedProcessors, ByteSizeValue.ofMb(749)),
+            equalTo(halfAllocatedProcessorsMaxFive(allocatedProcessors))
+        );
+        allocatedProcessors = randomIntBetween(1, 16);
+        assertThat(getMaxSnapshotThreadPoolSize(allocatedProcessors, ByteSizeValue.ofMb(750)), equalTo(10));
+        allocatedProcessors = randomIntBetween(1, 16);
+        assertThat(getMaxSnapshotThreadPoolSize(allocatedProcessors, ByteSizeValue.ofGb(4)), equalTo(10));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java b/server/src/test/java/org/elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java
index c82d5038c932..8e5cfce61938 100644
--- a/server/src/test/java/org/elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java
+++ b/server/src/test/java/org/elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java
@@ -121,11 +121,7 @@ public void testScalingExecutorType() throws InterruptedException {
                 .put("node.name", "testScalingExecutorType")
                 .build();
             threadPool = new ThreadPool(nodeSettings);
-            final int expectedMinimum = switch (threadPoolName) {
-                case "generic" -> 4;
-                case "management" -> 2;
-                default -> 1;
-            };
+            final int expectedMinimum = "generic".equals(threadPoolName) ? 4 : 1;
             assertThat(info(threadPool, threadPoolName).getMin(), equalTo(expectedMinimum));
             assertThat(info(threadPool, threadPoolName).getMax(), equalTo(10));
             final long expectedKeepAlive = "generic".equals(threadPoolName) || Names.SNAPSHOT_META.equals(threadPoolName) ? 30 : 300;
diff --git a/server/src/test/java/org/elasticsearch/transport/ClusterConnectionManagerTests.java b/server/src/test/java/org/elasticsearch/transport/ClusterConnectionManagerTests.java
index ff67abcb171d..17e58544ead4 100644
--- a/server/src/test/java/org/elasticsearch/transport/ClusterConnectionManagerTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/ClusterConnectionManagerTests.java
@@ -9,17 +9,18 @@
 package org.elasticsearch.transport;
 
 import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.util.concurrent.RunOnce;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.AbstractRefCounted;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.core.TimeValue;
@@ -42,6 +43,7 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
@@ -49,6 +51,7 @@
 import static org.elasticsearch.test.ActionListenerUtils.anyActionListener;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
@@ -111,15 +114,16 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti
 
         assertFalse(connectionManager.nodeConnected(node));
 
-        AtomicReference<Transport.Connection> connectionRef = new AtomicReference<>();
+        final var validatedConnectionRef = new AtomicReference<Transport.Connection>();
         ConnectionManager.ConnectionValidator validator = (c, p, l) -> {
-            connectionRef.set(c);
+            validatedConnectionRef.set(c);
             l.onResponse(null);
         };
         PlainActionFuture.get(fut -> connectionManager.connectToNode(node, connectionProfile, validator, fut.map(x -> null)));
 
         assertFalse(connection.isClosed());
         assertTrue(connectionManager.nodeConnected(node));
+        assertSame(connection, validatedConnectionRef.get());
         assertSame(connection, connectionManager.getConnection(node));
         assertEquals(1, connectionManager.size());
         assertEquals(1, nodeConnectedCount.get());
@@ -140,7 +144,7 @@ public void onNodeDisconnected(DiscoveryNode node, Transport.Connection connecti
         reason = "testing log messages emitted on disconnect",
         value = "org.elasticsearch.transport.ClusterConnectionManager:TRACE"
     )
-    public void testDisconnectLogging() throws IllegalAccessException {
+    public void testDisconnectLogging() {
         final Supplier<DiscoveryNode> nodeFactory = () -> new DiscoveryNode(
             randomAlphaOfLength(10),
             new TransportAddress(InetAddress.getLoopbackAddress(), 0),
@@ -170,16 +174,12 @@ public void testDisconnectLogging() throws IllegalAccessException {
         final Releasable localConnectionRef = toClose.getAndSet(null);
         assertThat(localConnectionRef, notNullValue());
 
-        final String loggerName = "org.elasticsearch.transport.ClusterConnectionManager";
-        final Logger logger = LogManager.getLogger(loggerName);
         final MockLogAppender appender = new MockLogAppender();
-        try {
-            appender.start();
-            Loggers.addAppender(logger, appender);
+        try (var ignored = appender.capturing(ClusterConnectionManager.class)) {
             appender.addExpectation(
                 new MockLogAppender.SeenEventExpectation(
                     "locally-triggered close message",
-                    loggerName,
+                    ClusterConnectionManager.class.getCanonicalName(),
                     Level.DEBUG,
                     "closing unused transport connection to [" + localClose + "]"
                 )
@@ -187,7 +187,7 @@ public void testDisconnectLogging() throws IllegalAccessException {
             appender.addExpectation(
                 new MockLogAppender.SeenEventExpectation(
                     "remotely-triggered close message",
-                    loggerName,
+                    ClusterConnectionManager.class.getCanonicalName(),
                     Level.INFO,
                     "transport connection to [" + remoteClose.descriptionWithoutAttributes() + "] closed by remote"
                 )
@@ -195,7 +195,7 @@ public void testDisconnectLogging() throws IllegalAccessException {
             appender.addExpectation(
                 new MockLogAppender.SeenEventExpectation(
                     "shutdown-triggered close message",
-                    loggerName,
+                    ClusterConnectionManager.class.getCanonicalName(),
                     Level.TRACE,
                     "connection manager shut down, closing transport connection to [" + shutdownClose + "]"
                 )
@@ -206,9 +206,6 @@ public void testDisconnectLogging() throws IllegalAccessException {
             connectionManager.close();
 
             appender.assertAllExpectationsMatched();
-        } finally {
-            Loggers.removeAppender(logger, appender);
-            appender.stop();
         }
     }
 
@@ -346,6 +343,128 @@ public void testConcurrentConnects() throws Exception {
         }
     }
 
+    public void testConcurrentConnectsDuringClose() throws Exception {
+
+        // This test ensures that closing the connection manager doesn't block forever, even if there's a constant stream of attempts to
+        // open connections. Note that closing the connection manager _does_ block while there are in-flight connection attempts, and in
+        // practice each attempt will (eventually) finish, so we're just trying to test that constant open attempts do not cause starvation.
+        //
+        // It works by spawning connection-open attempts in several concurrent loops, putting a Runnable to complete each attempt into a
+        // queue, and then consuming and completing the enqueued runnables in a separate thread. The consuming thread is throttled via a
+        // Semaphore, from which the main thread steals a permit which ensures that there's always at least one pending connection while the
+        // close is ongoing even though no connection attempt blocks forever.
+
+        final var pendingConnectionPermits = new Semaphore(0);
+        final var pendingConnections = ConcurrentCollections.<Runnable>newQueue();
+
+        // transport#openConnection enqueues a Runnable to complete the connection attempt
+        doAnswer(invocationOnMock -> {
+            @SuppressWarnings("unchecked")
+            final var listener = (ActionListener<Transport.Connection>) invocationOnMock.getArguments()[2];
+            final var targetNode = (DiscoveryNode) invocationOnMock.getArguments()[0];
+            pendingConnections.add(() -> listener.onResponse(new TestConnect(targetNode)));
+            pendingConnectionPermits.release();
+            return null;
+        }).when(transport).openConnection(any(), eq(connectionProfile), anyActionListener());
+
+        final ConnectionManager.ConnectionValidator validator = (c, p, l) -> l.onResponse(null);
+
+        // Once we start to see connections being rejected, we give back the stolen permit so that the last connection can complete
+        final var onConnectException = new RunOnce(pendingConnectionPermits::release);
+
+        // Create a few threads which open connections in a loop. Must be at least 2 so that there's always more connections incoming.
+        final var connectionLoops = between(2, 4);
+        final var connectionLoopCountDown = new CountDownLatch(connectionLoops);
+        final var expectConnectionFailures = new AtomicBoolean(); // unexpected failures would make this test pass vacuously
+
+        class ConnectionLoop extends AbstractRunnable {
+            private final boolean useConnectToNode = randomBoolean();
+
+            @Override
+            public void onFailure(Exception e) {
+                assert false : e;
+            }
+
+            @Override
+            protected void doRun() throws Exception {
+                final var discoveryNode = new DiscoveryNode("", new TransportAddress(InetAddress.getLoopbackAddress(), 0), Version.CURRENT);
+                final var listener = new ActionListener<Releasable>() {
+                    @Override
+                    public void onResponse(Releasable releasable) {
+                        releasable.close();
+                        threadPool.generic().execute(ConnectionLoop.this);
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        assertTrue(expectConnectionFailures.get());
+                        assertThat(e, instanceOf(ConnectTransportException.class));
+                        assertThat(e.getMessage(), containsString("connection manager is closed"));
+                        onConnectException.run();
+                        connectionLoopCountDown.countDown();
+                    }
+                };
+
+                if (useConnectToNode) {
+                    connectionManager.connectToNode(discoveryNode, connectionProfile, validator, listener);
+                } else {
+                    connectionManager.openConnection(discoveryNode, connectionProfile, listener.map(c -> c::close));
+                }
+            }
+        }
+
+        for (int i = 0; i < connectionLoops; i++) {
+            threadPool.generic().execute(new ConnectionLoop());
+        }
+
+        // Create a separate thread to complete pending connection attempts, throttled by the pendingConnectionPermits semaphore
+        final var completionThread = new Thread(() -> {
+            while (true) {
+                try {
+                    assertTrue(pendingConnectionPermits.tryAcquire(10, TimeUnit.SECONDS));
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                    // There could still be items in the queue when we are interrupted, so drain the queue before exiting:
+                    while (pendingConnectionPermits.tryAcquire()) {
+                        // noinspection ConstantConditions
+                        pendingConnections.poll().run();
+                    }
+                    return;
+                }
+                // noinspection ConstantConditions
+                pendingConnections.poll().run();
+            }
+        });
+        completionThread.start();
+
+        // Steal a permit so that the consumer lags behind the producers ...
+        assertTrue(pendingConnectionPermits.tryAcquire(10, TimeUnit.SECONDS));
+        // ... and then send a connection attempt through the system to ensure that the lagging has started
+        Releasables.closeExpectNoException(
+            PlainActionFuture.<Releasable, RuntimeException>get(
+                fut -> connectionManager.connectToNode(
+                    new DiscoveryNode("", new TransportAddress(InetAddress.getLoopbackAddress(), 0), Version.CURRENT),
+                    connectionProfile,
+                    validator,
+                    fut
+                ),
+                30,
+                TimeUnit.SECONDS
+            )
+        );
+
+        // Now close the connection manager
+        expectConnectionFailures.set(true);
+        connectionManager.close();
+        // Success! The close call returned
+
+        // Clean up and check everything completed properly
+        assertTrue(connectionLoopCountDown.await(10, TimeUnit.SECONDS));
+        completionThread.interrupt();
+        completionThread.join();
+        assertTrue(pendingConnections.isEmpty());
+    }
+
     public void testConcurrentConnectsAndDisconnects() throws Exception {
         final DiscoveryNode node = new DiscoveryNode("", new TransportAddress(InetAddress.getLoopbackAddress(), 0), Version.CURRENT);
         doAnswer(invocationOnMock -> {
@@ -368,14 +487,17 @@ public void testConcurrentConnectsAndDisconnects() throws Exception {
             });
         };
 
-        final Semaphore pendingConnections = new Semaphore(between(1, 1000));
+        final int connectionCount = between(1, 1000);
+        final int disconnectionCount = randomFrom(connectionCount, connectionCount - 1, between(0, connectionCount - 1));
+        final var connectionPermits = new Semaphore(connectionCount);
+        final var disconnectionPermits = new Semaphore(disconnectionCount);
         final int threadCount = between(1, 10);
-        final CountDownLatch countDownLatch = new CountDownLatch(threadCount);
+        final var countDownLatch = new CountDownLatch(threadCount);
 
         final Runnable action = new Runnable() {
             @Override
             public void run() {
-                if (pendingConnections.tryAcquire()) {
+                if (connectionPermits.tryAcquire()) {
                     connectionManager.connectToNode(node, null, validator, new ActionListener<>() {
                         @Override
                         public void onResponse(Releasable releasable) {
@@ -383,20 +505,26 @@ public void onResponse(Releasable releasable) {
                                 final String description = releasable.toString();
                                 fail(description);
                             }
-                            Releasables.close(releasable);
-                            threadPool.generic().execute(() -> run());
+                            if (disconnectionPermits.tryAcquire()) {
+                                Releasables.close(releasable);
+                            }
+                            runAgain();
                         }
 
                         @Override
                         public void onFailure(Exception e) {
                             if (e instanceof ConnectTransportException
                                 && e.getMessage().contains("concurrently connecting and disconnecting")) {
-                                pendingConnections.release();
-                                threadPool.generic().execute(() -> run());
+                                connectionPermits.release();
+                                runAgain();
                             } else {
                                 throw new AssertionError("unexpected", e);
                             }
                         }
+
+                        private void runAgain() {
+                            threadPool.generic().execute(() -> run());
+                        }
                     });
                 } else {
                     countDownLatch.countDown();
@@ -410,7 +538,116 @@ public void onFailure(Exception e) {
 
         assertTrue("threads did not all complete", countDownLatch.await(10, TimeUnit.SECONDS));
         assertTrue("validatorPermits not all released", validatorPermits.tryAcquire(Integer.MAX_VALUE, 10, TimeUnit.SECONDS));
-        assertFalse("node still connected", connectionManager.nodeConnected(node));
+        assertEquals("node still connected", disconnectionCount < connectionCount, connectionManager.nodeConnected(node));
+        connectionManager.close();
+    }
+
+    @TestLogging(reason = "ignore copious 'closed by remote' messages", value = "org.elasticsearch.transport.ClusterConnectionManager:WARN")
+    public void testConcurrentConnectsAndCloses() throws Exception {
+        final DiscoveryNode node = new DiscoveryNode("", new TransportAddress(InetAddress.getLoopbackAddress(), 0), Version.CURRENT);
+        doAnswer(invocationOnMock -> {
+            @SuppressWarnings("unchecked")
+            ActionListener<Transport.Connection> listener = (ActionListener<Transport.Connection>) invocationOnMock.getArguments()[2];
+            listener.onResponse(new TestConnect(node));
+            return null;
+        }).when(transport).openConnection(eq(node), any(), anyActionListener());
+
+        final Semaphore validatorPermits = new Semaphore(Integer.MAX_VALUE);
+
+        final ConnectionManager.ConnectionValidator validator = (c, p, l) -> {
+            assertTrue(validatorPermits.tryAcquire());
+            threadPool.executor(randomFrom(ThreadPool.Names.GENERIC, ThreadPool.Names.SAME)).execute(() -> {
+                try {
+                    l.onResponse(null);
+                } finally {
+                    validatorPermits.release();
+                }
+            });
+        };
+
+        final var closePermits = new Semaphore(between(1, 1000));
+        final int connectThreadCount = between(1, 3);
+        final int closeThreadCount = between(1, 3);
+        final var countDownLatch = new CountDownLatch(connectThreadCount + closeThreadCount);
+
+        final var cleanlyOpenedConnectionFuture = new PlainActionFuture<Boolean>();
+        final var closingRefs = AbstractRefCounted.of(
+            () -> connectionManager.connectToNode(
+                node,
+                null,
+                validator,
+                cleanlyOpenedConnectionFuture.map(r -> connectionManager.nodeConnected(node))
+            )
+        );
+
+        final Runnable connectAction = new Runnable() {
+            private void runAgain() {
+                threadPool.generic().execute(this);
+            }
+
+            @Override
+            public void run() {
+                if (cleanlyOpenedConnectionFuture.isDone() == false) {
+                    connectionManager.connectToNode(node, null, validator, new ActionListener<>() {
+                        @Override
+                        public void onResponse(Releasable releasable) {
+                            runAgain();
+                        }
+
+                        @Override
+                        public void onFailure(Exception e) {
+                            if (e instanceof ConnectTransportException
+                                && e.getMessage().contains("concurrently connecting and disconnecting")) {
+                                runAgain();
+                            } else {
+                                throw new AssertionError("unexpected", e);
+                            }
+                        }
+
+                    });
+                } else {
+                    countDownLatch.countDown();
+                }
+            }
+        };
+
+        final Runnable closeAction = new Runnable() {
+            private void runAgain() {
+                threadPool.generic().execute(this);
+            }
+
+            @Override
+            public void run() {
+                closingRefs.decRef();
+                if (closePermits.tryAcquire() && closingRefs.tryIncRef()) {
+                    try {
+                        var connection = connectionManager.getConnection(node);
+                        connection.addRemovedListener(ActionListener.wrap(this::runAgain));
+                        connection.close();
+                    } catch (NodeNotConnectedException e) {
+                        closePermits.release();
+                        runAgain();
+                    }
+                } else {
+                    countDownLatch.countDown();
+                }
+            }
+        };
+
+        for (int i = 0; i < connectThreadCount; i++) {
+            connectAction.run();
+        }
+        for (int i = 0; i < closeThreadCount; i++) {
+            closingRefs.incRef();
+            closeAction.run();
+        }
+        closingRefs.decRef();
+
+        assertTrue("threads did not all complete", countDownLatch.await(10, TimeUnit.SECONDS));
+        assertFalse(closingRefs.hasReferences());
+        assertTrue(cleanlyOpenedConnectionFuture.actionGet(0, TimeUnit.SECONDS));
+
+        assertTrue("validatorPermits not all released", validatorPermits.tryAcquire(Integer.MAX_VALUE, 10, TimeUnit.SECONDS));
         connectionManager.close();
     }
 
diff --git a/server/src/test/java/org/elasticsearch/transport/ConnectionProfileTests.java b/server/src/test/java/org/elasticsearch/transport/ConnectionProfileTests.java
index 1bc0481e8e5b..256ffe2e20d4 100644
--- a/server/src/test/java/org/elasticsearch/transport/ConnectionProfileTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/ConnectionProfileTests.java
@@ -71,6 +71,12 @@ public void testBuildConnectionProfile() {
         );
         assertEquals("type [PING] is already registered", illegalArgumentException.getMessage());
         builder.addConnections(4, TransportRequestOptions.Type.REG);
+
+        final String transportProfile = randomFrom(TransportSettings.DEFAULT_PROFILE, randomAlphaOfLengthBetween(5, 12), null);
+        if (transportProfile != null) {
+            builder.setTransportProfile(transportProfile);
+        }
+
         ConnectionProfile build = builder.build();
         if (randomBoolean()) {
             build = new ConnectionProfile.Builder(build).build();
@@ -106,6 +112,12 @@ public void testBuildConnectionProfile() {
             assertNull(build.getPingInterval());
         }
 
+        if (transportProfile != null) {
+            assertEquals(transportProfile, build.getTransportProfile());
+        } else {
+            assertEquals(TransportSettings.DEFAULT_PROFILE, build.getTransportProfile());
+        }
+
         List<Integer> list = new ArrayList<>(10);
         for (int i = 0; i < 10; i++) {
             list.add(i);
@@ -246,6 +258,7 @@ public void testDefaultConnectionProfile() {
         assertEquals(TransportSettings.TRANSPORT_COMPRESS.get(Settings.EMPTY), profile.getCompressionEnabled());
         assertEquals(TransportSettings.TRANSPORT_COMPRESSION_SCHEME.get(Settings.EMPTY), profile.getCompressionScheme());
         assertEquals(TransportSettings.PING_SCHEDULE.get(Settings.EMPTY), profile.getPingInterval());
+        assertEquals(TransportSettings.DEFAULT_PROFILE, profile.getTransportProfile());
 
         profile = ConnectionProfile.buildDefaultConnectionProfile(nonMasterNode());
         assertEquals(12, profile.getNumConnections());
@@ -254,6 +267,7 @@ public void testDefaultConnectionProfile() {
         assertEquals(0, profile.getNumConnectionsPerType(TransportRequestOptions.Type.STATE));
         assertEquals(2, profile.getNumConnectionsPerType(TransportRequestOptions.Type.RECOVERY));
         assertEquals(3, profile.getNumConnectionsPerType(TransportRequestOptions.Type.BULK));
+        assertEquals(TransportSettings.DEFAULT_PROFILE, profile.getTransportProfile());
 
         profile = ConnectionProfile.buildDefaultConnectionProfile(nonDataNode());
         assertEquals(11, profile.getNumConnections());
@@ -262,6 +276,7 @@ public void testDefaultConnectionProfile() {
         assertEquals(1, profile.getNumConnectionsPerType(TransportRequestOptions.Type.STATE));
         assertEquals(0, profile.getNumConnectionsPerType(TransportRequestOptions.Type.RECOVERY));
         assertEquals(3, profile.getNumConnectionsPerType(TransportRequestOptions.Type.BULK));
+        assertEquals(TransportSettings.DEFAULT_PROFILE, profile.getTransportProfile());
 
         profile = ConnectionProfile.buildDefaultConnectionProfile(removeRoles(nonDataNode(), Set.of(DiscoveryNodeRole.MASTER_ROLE)));
         assertEquals(10, profile.getNumConnections());
@@ -270,5 +285,6 @@ public void testDefaultConnectionProfile() {
         assertEquals(0, profile.getNumConnectionsPerType(TransportRequestOptions.Type.STATE));
         assertEquals(0, profile.getNumConnectionsPerType(TransportRequestOptions.Type.RECOVERY));
         assertEquals(3, profile.getNumConnectionsPerType(TransportRequestOptions.Type.BULK));
+        assertEquals(TransportSettings.DEFAULT_PROFILE, profile.getTransportProfile());
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/transport/InboundAggregatorTests.java b/server/src/test/java/org/elasticsearch/transport/InboundAggregatorTests.java
index 97e7dddc720c..30ce676a36e8 100644
--- a/server/src/test/java/org/elasticsearch/transport/InboundAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/InboundAggregatorTests.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.transport;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.breaker.CircuitBreakingException;
 import org.elasticsearch.common.breaker.TestCircuitBreaker;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -54,7 +54,7 @@ public void setUp() throws Exception {
 
     public void testInboundAggregation() throws IOException {
         long requestId = randomNonNegativeLong();
-        Header header = new Header(randomInt(), requestId, TransportStatus.setRequest((byte) 0), Version.CURRENT);
+        Header header = new Header(randomInt(), requestId, TransportStatus.setRequest((byte) 0), TransportVersion.CURRENT);
         header.headers = new Tuple<>(Collections.emptyMap(), Collections.emptyMap());
         header.actionName = "action_name";
         // Initiate Message
@@ -89,11 +89,11 @@ public void testInboundAggregation() throws IOException {
         assertFalse(aggregated.isPing());
         assertTrue(aggregated.getHeader().isRequest());
         assertThat(aggregated.getHeader().getRequestId(), equalTo(requestId));
-        assertThat(aggregated.getHeader().getVersion(), equalTo(Version.CURRENT));
+        assertThat(aggregated.getHeader().getVersion(), equalTo(TransportVersion.CURRENT));
         for (ReleasableBytesReference reference : references) {
             assertTrue(reference.hasReferences());
         }
-        aggregated.close();
+        aggregated.decRef();
         for (ReleasableBytesReference reference : references) {
             assertFalse(reference.hasReferences());
         }
@@ -101,7 +101,7 @@ public void testInboundAggregation() throws IOException {
 
     public void testInboundUnknownAction() throws IOException {
         long requestId = randomNonNegativeLong();
-        Header header = new Header(randomInt(), requestId, TransportStatus.setRequest((byte) 0), Version.CURRENT);
+        Header header = new Header(randomInt(), requestId, TransportStatus.setRequest((byte) 0), TransportVersion.CURRENT);
         header.headers = new Tuple<>(Collections.emptyMap(), Collections.emptyMap());
         header.actionName = unknownAction;
         // Initiate Message
@@ -125,7 +125,12 @@ public void testInboundUnknownAction() throws IOException {
     public void testCircuitBreak() throws IOException {
         circuitBreaker.startBreaking();
         // Actions are breakable
-        Header breakableHeader = new Header(randomInt(), randomNonNegativeLong(), TransportStatus.setRequest((byte) 0), Version.CURRENT);
+        Header breakableHeader = new Header(
+            randomInt(),
+            randomNonNegativeLong(),
+            TransportStatus.setRequest((byte) 0),
+            TransportVersion.CURRENT
+        );
         breakableHeader.headers = new Tuple<>(Collections.emptyMap(), Collections.emptyMap());
         breakableHeader.actionName = "action_name";
         // Initiate Message
@@ -145,7 +150,12 @@ public void testCircuitBreak() throws IOException {
         assertThat(aggregated1.getException(), instanceOf(CircuitBreakingException.class));
 
         // Actions marked as unbreakable are not broken
-        Header unbreakableHeader = new Header(randomInt(), randomNonNegativeLong(), TransportStatus.setRequest((byte) 0), Version.CURRENT);
+        Header unbreakableHeader = new Header(
+            randomInt(),
+            randomNonNegativeLong(),
+            TransportStatus.setRequest((byte) 0),
+            TransportVersion.CURRENT
+        );
         unbreakableHeader.headers = new Tuple<>(Collections.emptyMap(), Collections.emptyMap());
         unbreakableHeader.actionName = unBreakableAction;
         // Initiate Message
@@ -164,7 +174,7 @@ public void testCircuitBreak() throws IOException {
 
         // Handshakes are not broken
         final byte handshakeStatus = TransportStatus.setHandshake(TransportStatus.setRequest((byte) 0));
-        Header handshakeHeader = new Header(randomInt(), randomNonNegativeLong(), handshakeStatus, Version.CURRENT);
+        Header handshakeHeader = new Header(randomInt(), randomNonNegativeLong(), handshakeStatus, TransportVersion.CURRENT);
         handshakeHeader.headers = new Tuple<>(Collections.emptyMap(), Collections.emptyMap());
         handshakeHeader.actionName = "handshake";
         // Initiate Message
@@ -184,7 +194,7 @@ public void testCircuitBreak() throws IOException {
 
     public void testCloseWillCloseContent() {
         long requestId = randomNonNegativeLong();
-        Header header = new Header(randomInt(), requestId, TransportStatus.setRequest((byte) 0), Version.CURRENT);
+        Header header = new Header(randomInt(), requestId, TransportStatus.setRequest((byte) 0), TransportVersion.CURRENT);
         header.headers = new Tuple<>(Collections.emptyMap(), Collections.emptyMap());
         header.actionName = "action_name";
         // Initiate Message
@@ -224,7 +234,7 @@ public void testFinishAggregationWillFinishHeader() throws IOException {
         } else {
             actionName = "action_name";
         }
-        Header header = new Header(randomInt(), requestId, TransportStatus.setRequest((byte) 0), Version.CURRENT);
+        Header header = new Header(randomInt(), requestId, TransportStatus.setRequest((byte) 0), TransportVersion.CURRENT);
         // Initiate Message
         aggregator.headerReceived(header);
 
diff --git a/server/src/test/java/org/elasticsearch/transport/InboundDecoderTests.java b/server/src/test/java/org/elasticsearch/transport/InboundDecoderTests.java
index 65e3cb1ad432..b676947cf112 100644
--- a/server/src/test/java/org/elasticsearch/transport/InboundDecoderTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/InboundDecoderTests.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.transport;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.bytes.ReleasableBytesReference;
@@ -17,7 +18,7 @@
 import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.TransportVersionUtils;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -53,7 +54,7 @@ public void testDecode() throws IOException {
             message = new OutboundMessage.Request(
                 threadContext,
                 new TestRequest(randomAlphaOfLength(100)),
-                Version.CURRENT,
+                TransportVersion.CURRENT,
                 action,
                 requestId,
                 false,
@@ -63,7 +64,7 @@ public void testDecode() throws IOException {
             message = new OutboundMessage.Response(
                 threadContext,
                 new TestResponse(randomAlphaOfLength(100)),
-                Version.CURRENT,
+                TransportVersion.CURRENT,
                 requestId,
                 false,
                 null
@@ -72,10 +73,12 @@ public void testDecode() throws IOException {
 
         try (RecyclerBytesStreamOutput os = new RecyclerBytesStreamOutput(recycler)) {
             final BytesReference totalBytes = message.serialize(os);
-            int totalHeaderSize = TcpHeader.headerSize(Version.CURRENT) + totalBytes.getInt(TcpHeader.VARIABLE_HEADER_SIZE_POSITION);
+            int totalHeaderSize = TcpHeader.headerSize(TransportVersion.CURRENT) + totalBytes.getInt(
+                TcpHeader.VARIABLE_HEADER_SIZE_POSITION
+            );
             final BytesReference messageBytes = totalBytes.slice(totalHeaderSize, totalBytes.length() - totalHeaderSize);
 
-            InboundDecoder decoder = new InboundDecoder(Version.CURRENT, recycler);
+            InboundDecoder decoder = new InboundDecoder(TransportVersion.CURRENT, recycler);
             final ArrayList<Object> fragments = new ArrayList<>();
             final ReleasableBytesReference releasable1 = ReleasableBytesReference.wrap(totalBytes);
             int bytesConsumed = decoder.decode(releasable1, fragments::add);
@@ -84,7 +87,7 @@ public void testDecode() throws IOException {
 
             final Header header = (Header) fragments.get(0);
             assertEquals(requestId, header.getRequestId());
-            assertEquals(Version.CURRENT, header.getVersion());
+            assertEquals(TransportVersion.CURRENT, header.getVersion());
             assertFalse(header.isCompressed());
             assertFalse(header.isHandshake());
             if (isRequest) {
@@ -122,7 +125,7 @@ public void testDecodePreHeaderSizeVariableInt() throws IOException {
         Compression.Scheme compressionScheme = randomFrom(Compression.Scheme.DEFLATE, Compression.Scheme.DEFLATE, null);
         String action = "test-request";
         long requestId = randomNonNegativeLong();
-        final Version preHeaderVariableInt = Version.V_7_5_0;
+        final TransportVersion preHeaderVariableInt = TransportVersion.V_7_5_0;
         final String contentValue = randomAlphaOfLength(100);
         // 8.0 is only compatible with handshakes on a pre-variable int version
         final OutboundMessage message = new OutboundMessage.Request(
@@ -139,7 +142,7 @@ public void testDecodePreHeaderSizeVariableInt() throws IOException {
             final BytesReference totalBytes = message.serialize(os);
             int partialHeaderSize = TcpHeader.headerSize(preHeaderVariableInt);
 
-            InboundDecoder decoder = new InboundDecoder(Version.CURRENT, recycler);
+            InboundDecoder decoder = new InboundDecoder(TransportVersion.CURRENT, recycler);
             final ArrayList<Object> fragments = new ArrayList<>();
             final ReleasableBytesReference releasable1 = ReleasableBytesReference.wrap(totalBytes);
             int bytesConsumed = decoder.decode(releasable1, fragments::add);
@@ -181,7 +184,7 @@ public void testDecodeHandshakeCompatibility() throws IOException {
         final String headerKey = randomAlphaOfLength(10);
         final String headerValue = randomAlphaOfLength(20);
         threadContext.putHeader(headerKey, headerValue);
-        Version handshakeCompat = Version.CURRENT.minimumCompatibilityVersion().minimumCompatibilityVersion();
+        TransportVersion handshakeCompat = TransportVersion.CURRENT.calculateMinimumCompatVersion().calculateMinimumCompatVersion();
         OutboundMessage message = new OutboundMessage.Request(
             threadContext,
             new TestRequest(randomAlphaOfLength(100)),
@@ -196,7 +199,7 @@ public void testDecodeHandshakeCompatibility() throws IOException {
             final BytesReference bytes = message.serialize(os);
             int totalHeaderSize = TcpHeader.headerSize(handshakeCompat);
 
-            InboundDecoder decoder = new InboundDecoder(Version.CURRENT, recycler);
+            InboundDecoder decoder = new InboundDecoder(TransportVersion.CURRENT, recycler);
             final ArrayList<Object> fragments = new ArrayList<>();
             final ReleasableBytesReference releasable1 = ReleasableBytesReference.wrap(bytes);
             int bytesConsumed = decoder.decode(releasable1, fragments::add);
@@ -232,10 +235,18 @@ public void testCompressedDecode() throws IOException {
         Compression.Scheme scheme = randomFrom(Compression.Scheme.DEFLATE, Compression.Scheme.LZ4);
         if (isRequest) {
             transportMessage = new TestRequest(randomAlphaOfLength(100));
-            message = new OutboundMessage.Request(threadContext, transportMessage, Version.CURRENT, action, requestId, false, scheme);
+            message = new OutboundMessage.Request(
+                threadContext,
+                transportMessage,
+                TransportVersion.CURRENT,
+                action,
+                requestId,
+                false,
+                scheme
+            );
         } else {
             transportMessage = new TestResponse(randomAlphaOfLength(100));
-            message = new OutboundMessage.Response(threadContext, transportMessage, Version.CURRENT, requestId, false, scheme);
+            message = new OutboundMessage.Response(threadContext, transportMessage, TransportVersion.CURRENT, requestId, false, scheme);
         }
 
         try (RecyclerBytesStreamOutput os = new RecyclerBytesStreamOutput(recycler)) {
@@ -243,9 +254,11 @@ public void testCompressedDecode() throws IOException {
             final BytesStreamOutput out = new BytesStreamOutput();
             transportMessage.writeTo(out);
             final BytesReference uncompressedBytes = out.bytes();
-            int totalHeaderSize = TcpHeader.headerSize(Version.CURRENT) + totalBytes.getInt(TcpHeader.VARIABLE_HEADER_SIZE_POSITION);
+            int totalHeaderSize = TcpHeader.headerSize(TransportVersion.CURRENT) + totalBytes.getInt(
+                TcpHeader.VARIABLE_HEADER_SIZE_POSITION
+            );
 
-            InboundDecoder decoder = new InboundDecoder(Version.CURRENT, recycler);
+            InboundDecoder decoder = new InboundDecoder(TransportVersion.CURRENT, recycler);
             final ArrayList<Object> fragments = new ArrayList<>();
             final ReleasableBytesReference releasable1 = ReleasableBytesReference.wrap(totalBytes);
             int bytesConsumed = decoder.decode(releasable1, fragments::add);
@@ -254,7 +267,7 @@ public void testCompressedDecode() throws IOException {
 
             final Header header = (Header) fragments.get(0);
             assertEquals(requestId, header.getRequestId());
-            assertEquals(Version.CURRENT, header.getVersion());
+            assertEquals(TransportVersion.CURRENT, header.getVersion());
             assertTrue(header.isCompressed());
             assertFalse(header.isHandshake());
             if (isRequest) {
@@ -294,7 +307,7 @@ public void testCompressedDecodeHandshakeCompatibility() throws IOException {
         final String headerKey = randomAlphaOfLength(10);
         final String headerValue = randomAlphaOfLength(20);
         threadContext.putHeader(headerKey, headerValue);
-        Version handshakeCompat = Version.CURRENT.minimumCompatibilityVersion().minimumCompatibilityVersion();
+        TransportVersion handshakeCompat = TransportVersion.CURRENT.calculateMinimumCompatVersion().calculateMinimumCompatVersion();
         OutboundMessage message = new OutboundMessage.Request(
             threadContext,
             new TestRequest(randomAlphaOfLength(100)),
@@ -309,7 +322,7 @@ public void testCompressedDecodeHandshakeCompatibility() throws IOException {
             final BytesReference bytes = message.serialize(os);
             int totalHeaderSize = TcpHeader.headerSize(handshakeCompat);
 
-            InboundDecoder decoder = new InboundDecoder(Version.CURRENT, recycler);
+            InboundDecoder decoder = new InboundDecoder(TransportVersion.CURRENT, recycler);
             final ArrayList<Object> fragments = new ArrayList<>();
             final ReleasableBytesReference releasable1 = ReleasableBytesReference.wrap(bytes);
             int bytesConsumed = decoder.decode(releasable1, fragments::add);
@@ -331,7 +344,7 @@ public void testCompressedDecodeHandshakeCompatibility() throws IOException {
     public void testVersionIncompatibilityDecodeException() throws IOException {
         String action = "test-request";
         long requestId = randomNonNegativeLong();
-        Version incompatibleVersion = Version.CURRENT.minimumCompatibilityVersion().minimumCompatibilityVersion();
+        TransportVersion incompatibleVersion = TransportVersion.CURRENT.calculateMinimumCompatVersion().calculateMinimumCompatVersion();
         OutboundMessage message = new OutboundMessage.Request(
             threadContext,
             new TestRequest(randomAlphaOfLength(100)),
@@ -346,7 +359,7 @@ public void testVersionIncompatibilityDecodeException() throws IOException {
         try (RecyclerBytesStreamOutput os = new RecyclerBytesStreamOutput(recycler)) {
             final BytesReference bytes = message.serialize(os);
 
-            InboundDecoder decoder = new InboundDecoder(Version.CURRENT, recycler);
+            InboundDecoder decoder = new InboundDecoder(TransportVersion.CURRENT, recycler);
             final ArrayList<Object> fragments = new ArrayList<>();
             try (ReleasableBytesReference r = ReleasableBytesReference.wrap(bytes)) {
                 releasable1 = r;
@@ -359,32 +372,36 @@ public void testVersionIncompatibilityDecodeException() throws IOException {
 
     public void testEnsureVersionCompatibility() throws IOException {
         IllegalStateException ise = InboundDecoder.ensureVersionCompatibility(
-            VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT),
-            Version.CURRENT,
+            TransportVersionUtils.randomVersionBetween(
+                random(),
+                TransportVersion.CURRENT.minimumCompatibilityVersion(),
+                TransportVersion.CURRENT
+            ),
+            TransportVersion.CURRENT,
             randomBoolean()
         );
         assertNull(ise);
 
-        final Version version = Version.fromString("7.0.0");
-        ise = InboundDecoder.ensureVersionCompatibility(Version.fromString("6.0.0"), version, true);
+        TransportVersion version = Version.fromString("7.0.0").transportVersion;
+        ise = InboundDecoder.ensureVersionCompatibility(Version.fromString("6.0.0").transportVersion, version, true);
         assertNull(ise);
 
-        ise = InboundDecoder.ensureVersionCompatibility(Version.fromString("6.0.0"), version, false);
+        ise = InboundDecoder.ensureVersionCompatibility(Version.fromString("6.0.0").transportVersion, version, false);
         assertEquals(
-            "Received message from unsupported version: [6.0.0] minimal compatible version is: ["
-                + version.minimumCompatibilityVersion()
+            "Received message from unsupported version: [6000099] minimal compatible version is: ["
+                + version.calculateMinimumCompatVersion()
                 + "]",
             ise.getMessage()
         );
 
         // For handshake we are compatible with N-2
-        ise = InboundDecoder.ensureVersionCompatibility(Version.fromString("5.6.0"), version, true);
+        ise = InboundDecoder.ensureVersionCompatibility(Version.fromString("5.6.0").transportVersion, version, true);
         assertNull(ise);
 
-        ise = InboundDecoder.ensureVersionCompatibility(Version.fromString("5.6.0"), version, false);
+        ise = InboundDecoder.ensureVersionCompatibility(Version.fromString("5.6.0").transportVersion, version, false);
         assertEquals(
-            "Received message from unsupported version: [5.6.0] minimal compatible version is: ["
-                + version.minimumCompatibilityVersion()
+            "Received message from unsupported version: [5060099] minimal compatible version is: ["
+                + version.calculateMinimumCompatVersion()
                 + "]",
             ise.getMessage()
         );
diff --git a/server/src/test/java/org/elasticsearch/transport/InboundHandlerTests.java b/server/src/test/java/org/elasticsearch/transport/InboundHandlerTests.java
index c63e875d96ab..32ab1fa6d398 100644
--- a/server/src/test/java/org/elasticsearch/transport/InboundHandlerTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/InboundHandlerTests.java
@@ -12,11 +12,12 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.bytes.ReleasableBytesReference;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.RecyclerBytesStreamOutput;
@@ -27,10 +28,11 @@
 import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockLogAppender;
-import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.tracing.Tracer;
@@ -50,7 +52,7 @@
 public class InboundHandlerTests extends ESTestCase {
 
     private final TestThreadPool threadPool = new TestThreadPool(getClass().getName());
-    private final Version version = Version.CURRENT;
+    private final TransportVersion version = TransportVersion.CURRENT;
 
     private TaskManager taskManager;
     private Transport.ResponseHandlers responseHandlers;
@@ -178,7 +180,7 @@ public TestResponse read(StreamInput in) throws IOException {
         handler.inboundMessage(channel, requestMessage);
 
         TransportChannel transportChannel = channelCaptor.get();
-        assertEquals(Version.CURRENT, transportChannel.getVersion());
+        assertEquals(TransportVersion.CURRENT, transportChannel.getVersion());
         assertEquals("transport", transportChannel.getChannelType());
         assertEquals(requestValue, requestCaptor.get().value);
 
@@ -213,7 +215,7 @@ public void testSendsErrorResponseToHandshakeFromCompatibleVersion() throws Exce
         // successful we can respond correctly in a format this old, but we do not guarantee that we can respond correctly with an error
         // response. However if the two nodes are from the same major version then we do guarantee compatibility of error responses.
 
-        final Version remoteVersion = VersionUtils.randomCompatibleVersion(random(), version);
+        final TransportVersion remoteVersion = TransportVersionUtils.randomCompatibleVersion(random(), version);
         final long requestId = randomNonNegativeLong();
         final Header requestHeader = new Header(
             between(0, 100),
@@ -256,7 +258,9 @@ public void testClosesChannelOnErrorInHandshakeWithIncompatibleVersion() throws
             final AtomicBoolean isClosed = new AtomicBoolean();
             channel.addCloseListener(ActionListener.wrap(() -> assertTrue(isClosed.compareAndSet(false, true))));
 
-            final Version remoteVersion = Version.fromId(randomIntBetween(0, version.minimumCompatibilityVersion().id - 1));
+            final TransportVersion remoteVersion = TransportVersion.fromId(
+                randomIntBetween(0, version.minimumCompatibilityVersion().id - 1)
+            );
             final long requestId = randomNonNegativeLong();
             final Header requestHeader = new Header(
                 between(0, 100),
@@ -285,7 +289,7 @@ public void testLogsSlowInboundProcessing() throws Exception {
 
         handler.setSlowLogThreshold(TimeValue.timeValueMillis(5L));
         try {
-            final Version remoteVersion = Version.CURRENT;
+            final TransportVersion remoteVersion = TransportVersion.CURRENT;
 
             mockAppender.addExpectation(
                 new MockLogAppender.SeenEventExpectation(
@@ -303,7 +307,10 @@ public void testLogsSlowInboundProcessing() throws Exception {
                 TransportStatus.setRequest(TransportStatus.setHandshake((byte) 0)),
                 remoteVersion
             );
-            final InboundMessage requestMessage = new InboundMessage(requestHeader, ReleasableBytesReference.empty(), () -> {
+            BytesStreamOutput byteData = new BytesStreamOutput();
+            TaskId.EMPTY_TASK_ID.writeTo(byteData);
+            TransportVersion.writeVersion(remoteVersion, byteData);
+            final InboundMessage requestMessage = new InboundMessage(requestHeader, ReleasableBytesReference.wrap(byteData.bytes()), () -> {
                 try {
                     TimeUnit.SECONDS.sleep(1L);
                 } catch (InterruptedException e) {
@@ -349,7 +356,7 @@ public void onResponseReceived(long requestId, Transport.ResponseContext context
         }
     }
 
-    private static InboundMessage unreadableInboundHandshake(Version remoteVersion, Header requestHeader) {
+    private static InboundMessage unreadableInboundHandshake(TransportVersion remoteVersion, Header requestHeader) {
         return new InboundMessage(requestHeader, ReleasableBytesReference.wrap(BytesArray.EMPTY), () -> {}) {
             @Override
             public StreamInput openOrGetStreamInput() {
@@ -359,7 +366,7 @@ public int read() {
                         throw new ElasticsearchException("unreadable handshake");
                     }
                 });
-                streamInput.setVersion(remoteVersion);
+                streamInput.setTransportVersion(remoteVersion);
                 return streamInput;
             }
         };
diff --git a/server/src/test/java/org/elasticsearch/transport/InboundPipelineTests.java b/server/src/test/java/org/elasticsearch/transport/InboundPipelineTests.java
index 726c876ba559..333cbde83681 100644
--- a/server/src/test/java/org/elasticsearch/transport/InboundPipelineTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/InboundPipelineTests.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.transport;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.breaker.CircuitBreakingException;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
@@ -29,7 +29,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Objects;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.BiConsumer;
 import java.util.function.LongSupplier;
@@ -52,7 +51,7 @@ public void testPipelineHandling() throws IOException {
             try {
                 final Header header = m.getHeader();
                 final MessageData actualData;
-                final Version version = header.getVersion();
+                final TransportVersion version = header.getVersion();
                 final boolean isRequest = header.isRequest();
                 final long requestId = header.getRequestId();
                 final Compression.Scheme compressionScheme = header.getCompressionScheme();
@@ -78,7 +77,7 @@ public void testPipelineHandling() throws IOException {
 
         final StatsTracker statsTracker = new StatsTracker();
         final LongSupplier millisSupplier = () -> TimeValue.nsecToMSec(System.nanoTime());
-        final InboundDecoder decoder = new InboundDecoder(Version.CURRENT, recycler);
+        final InboundDecoder decoder = new InboundDecoder(TransportVersion.CURRENT, recycler);
         final String breakThisAction = "break_this_action";
         final String actionName = "actionName";
         final Predicate<String> canTripBreaker = breakThisAction::equals;
@@ -98,7 +97,10 @@ public void testPipelineHandling() throws IOException {
             toRelease.clear();
             try (RecyclerBytesStreamOutput streamOutput = new RecyclerBytesStreamOutput(recycler)) {
                 while (streamOutput.size() < BYTE_THRESHOLD) {
-                    final Version version = randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion());
+                    final TransportVersion version = randomFrom(
+                        TransportVersion.CURRENT,
+                        TransportVersion.CURRENT.minimumCompatibilityVersion()
+                    );
                     final String value = randomRealisticUnicodeOfCodepointLength(randomIntBetween(200, 400));
                     final boolean isRequest = randomBoolean();
                     Compression.Scheme compressionScheme = getCompressionScheme(version);
@@ -192,7 +194,7 @@ public void testPipelineHandling() throws IOException {
         }
     }
 
-    private static Compression.Scheme getCompressionScheme(Version version) {
+    private static Compression.Scheme getCompressionScheme(TransportVersion version) {
         if (randomBoolean()) {
             return null;
         } else {
@@ -208,14 +210,15 @@ public void testDecodeExceptionIsPropagated() throws IOException {
         BiConsumer<TcpChannel, InboundMessage> messageHandler = (c, m) -> {};
         final StatsTracker statsTracker = new StatsTracker();
         final LongSupplier millisSupplier = () -> TimeValue.nsecToMSec(System.nanoTime());
-        final InboundDecoder decoder = new InboundDecoder(Version.CURRENT, recycler);
+        final InboundDecoder decoder = new InboundDecoder(TransportVersion.CURRENT, recycler);
         final Supplier<CircuitBreaker> breaker = () -> new NoopCircuitBreaker("test");
         final InboundAggregator aggregator = new InboundAggregator(breaker, (Predicate<String>) action -> true);
         final InboundPipeline pipeline = new InboundPipeline(statsTracker, millisSupplier, decoder, aggregator, messageHandler);
 
         try (RecyclerBytesStreamOutput streamOutput = new RecyclerBytesStreamOutput(recycler)) {
             String actionName = "actionName";
-            final Version invalidVersion = Version.CURRENT.minimumCompatibilityVersion().minimumCompatibilityVersion();
+            final TransportVersion invalidVersion = TransportVersion.CURRENT.calculateMinimumCompatVersion()
+                .calculateMinimumCompatVersion();
             final String value = randomAlphaOfLength(1000);
             final boolean isRequest = randomBoolean();
             final long requestId = randomNonNegativeLong();
@@ -253,14 +256,14 @@ public void testEnsureBodyIsNotPrematurelyReleased() throws IOException {
         BiConsumer<TcpChannel, InboundMessage> messageHandler = (c, m) -> {};
         final StatsTracker statsTracker = new StatsTracker();
         final LongSupplier millisSupplier = () -> TimeValue.nsecToMSec(System.nanoTime());
-        final InboundDecoder decoder = new InboundDecoder(Version.CURRENT, recycler);
+        final InboundDecoder decoder = new InboundDecoder(TransportVersion.CURRENT, recycler);
         final Supplier<CircuitBreaker> breaker = () -> new NoopCircuitBreaker("test");
         final InboundAggregator aggregator = new InboundAggregator(breaker, (Predicate<String>) action -> true);
         final InboundPipeline pipeline = new InboundPipeline(statsTracker, millisSupplier, decoder, aggregator, messageHandler);
 
         try (RecyclerBytesStreamOutput streamOutput = new RecyclerBytesStreamOutput(recycler)) {
             String actionName = "actionName";
-            final Version version = Version.CURRENT;
+            final TransportVersion version = TransportVersion.CURRENT;
             final String value = randomAlphaOfLength(1000);
             final boolean isRequest = randomBoolean();
             final long requestId = randomNonNegativeLong();
@@ -273,7 +276,7 @@ public void testEnsureBodyIsNotPrematurelyReleased() throws IOException {
             }
 
             final BytesReference reference = message.serialize(streamOutput);
-            final int fixedHeaderSize = TcpHeader.headerSize(Version.CURRENT);
+            final int fixedHeaderSize = TcpHeader.headerSize(TransportVersion.CURRENT);
             final int variableHeaderSize = reference.getInt(fixedHeaderSize - 4);
             final int totalHeaderSize = fixedHeaderSize + variableHeaderSize;
             final AtomicBoolean bodyReleased = new AtomicBoolean(false);
@@ -297,47 +300,12 @@ public void testEnsureBodyIsNotPrematurelyReleased() throws IOException {
         }
     }
 
-    private static class MessageData {
-
-        private final Version version;
-        private final long requestId;
-        private final boolean isRequest;
-        private final Compression.Scheme compressionScheme;
-        private final String value;
-        private final String actionName;
-
-        private MessageData(
-            Version version,
-            long requestId,
-            boolean isRequest,
-            Compression.Scheme compressionScheme,
-            String actionName,
-            String value
-        ) {
-            this.version = version;
-            this.requestId = requestId;
-            this.isRequest = isRequest;
-            this.compressionScheme = compressionScheme;
-            this.actionName = actionName;
-            this.value = value;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            MessageData that = (MessageData) o;
-            return requestId == that.requestId
-                && isRequest == that.isRequest
-                && Objects.equals(compressionScheme, that.compressionScheme)
-                && Objects.equals(version, that.version)
-                && Objects.equals(value, that.value)
-                && Objects.equals(actionName, that.actionName);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(version, requestId, isRequest, compressionScheme, value, actionName);
-        }
-    }
+    private record MessageData(
+        TransportVersion version,
+        long requestId,
+        boolean isRequest,
+        Compression.Scheme compressionScheme,
+        String actionName,
+        String value
+    ) {}
 }
diff --git a/server/src/test/java/org/elasticsearch/transport/OutboundHandlerTests.java b/server/src/test/java/org/elasticsearch/transport/OutboundHandlerTests.java
index 460ba850d4b6..69e14b2fdc37 100644
--- a/server/src/test/java/org/elasticsearch/transport/OutboundHandlerTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/OutboundHandlerTests.java
@@ -12,6 +12,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
@@ -72,10 +73,18 @@ public void setUp() throws Exception {
         node = new DiscoveryNode("", transportAddress, Version.CURRENT);
         StatsTracker statsTracker = new StatsTracker();
         compressionScheme = randomFrom(Compression.Scheme.DEFLATE, Compression.Scheme.LZ4);
-        handler = new OutboundHandler("node", Version.CURRENT, statsTracker, threadPool, recycler, new HandlingTimeTracker(), false);
+        handler = new OutboundHandler(
+            "node",
+            TransportVersion.CURRENT,
+            statsTracker,
+            threadPool,
+            recycler,
+            new HandlingTimeTracker(),
+            false
+        );
 
         final LongSupplier millisSupplier = () -> TimeValue.nsecToMSec(System.nanoTime());
-        final InboundDecoder decoder = new InboundDecoder(Version.CURRENT, this.recycler);
+        final InboundDecoder decoder = new InboundDecoder(TransportVersion.CURRENT, this.recycler);
         final Supplier<CircuitBreaker> breaker = () -> new NoopCircuitBreaker("test");
         final InboundAggregator aggregator = new InboundAggregator(breaker, (Predicate<String>) action -> true);
         pipeline = new InboundPipeline(statsTracker, millisSupplier, decoder, aggregator, (c, m) -> {
@@ -120,7 +129,7 @@ public void testSendRawBytes() {
 
     public void testSendRequest() throws IOException {
         ThreadContext threadContext = threadPool.getThreadContext();
-        Version version = randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion());
+        TransportVersion version = randomFrom(TransportVersion.CURRENT, TransportVersion.CURRENT.minimumCompatibilityVersion());
         String action = "handshake";
         long requestId = randomLongBetween(0, 300);
         boolean isHandshake = randomBoolean();
@@ -193,7 +202,7 @@ public void onRequestSent(
 
     public void testSendResponse() throws IOException {
         ThreadContext threadContext = threadPool.getThreadContext();
-        Version version = randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion());
+        TransportVersion version = randomFrom(TransportVersion.CURRENT, TransportVersion.CURRENT.minimumCompatibilityVersion());
         String action = "handshake";
         long requestId = randomLongBetween(0, 300);
         boolean isHandshake = randomBoolean();
@@ -260,7 +269,7 @@ public void onResponseSent(long requestId, String action, TransportResponse resp
 
     public void testErrorResponse() throws IOException {
         ThreadContext threadContext = threadPool.getThreadContext();
-        Version version = randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion());
+        TransportVersion version = randomFrom(TransportVersion.CURRENT, TransportVersion.CURRENT.minimumCompatibilityVersion());
         String action = "handshake";
         long requestId = randomLongBetween(0, 300);
         threadContext.putHeader("header", "header_value");
diff --git a/server/src/test/java/org/elasticsearch/transport/ProxyConnectionStrategyTests.java b/server/src/test/java/org/elasticsearch/transport/ProxyConnectionStrategyTests.java
index 0ce7f197ae33..65409f3b5835 100644
--- a/server/src/test/java/org/elasticsearch/transport/ProxyConnectionStrategyTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/ProxyConnectionStrategyTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.junit.annotations.TestLogging;
@@ -439,7 +440,7 @@ public void testModeSettingsCannotBeUsedWhenInDifferentMode() {
             Setting<?> concreteSetting = restrictedSetting.v1().getConcreteSettingForNamespace(clusterName);
             Settings invalid = Settings.builder().put(settings).put(concreteSetting.getKey(), restrictedSetting.v2()).build();
             IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> service.validate(invalid, true));
-            String expected = formatted("""
+            String expected = Strings.format("""
                 Setting "%s" cannot be used with the configured "cluster.remote.cluster_name.mode" \
                 [required=PROXY, configured=SNIFF]\
                 """, concreteSetting.getKey());
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterPortSettingsTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterPortSettingsTests.java
new file mode 100644
index 000000000000..06893906187f
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterPortSettingsTests.java
@@ -0,0 +1,206 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.transport;
+
+import org.elasticsearch.common.network.NetworkAddress;
+import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.List;
+
+import static org.elasticsearch.transport.RemoteClusterPortSettings.REMOTE_CLUSTER_PORT_ENABLED;
+import static org.elasticsearch.transport.RemoteClusterPortSettings.REMOTE_CLUSTER_PROFILE;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+public class RemoteClusterPortSettingsTests extends ESTestCase {
+    /**
+     * Tests that, if Remote Cluster Security 2.0 is enabled, we reject any configuration of that profile
+     * via the profile settings.
+     */
+    public void testRemoteClusterProfileCannotBeUsedWhenRcs2IsEnabled() {
+        List<Setting.AffixSetting<?>> transportProfileSettings = List.of(
+            TransportSettings.TCP_KEEP_ALIVE_PROFILE,
+            TransportSettings.TCP_KEEP_IDLE_PROFILE,
+            TransportSettings.TCP_KEEP_INTERVAL_PROFILE,
+            TransportSettings.TCP_KEEP_COUNT_PROFILE,
+            TransportSettings.TCP_NO_DELAY_PROFILE,
+            TransportSettings.TCP_REUSE_ADDRESS_PROFILE,
+            TransportSettings.TCP_SEND_BUFFER_SIZE_PROFILE,
+            TransportSettings.TCP_RECEIVE_BUFFER_SIZE_PROFILE,
+            TransportSettings.BIND_HOST_PROFILE,
+            TransportSettings.PUBLISH_HOST_PROFILE,
+            TransportSettings.PORT_PROFILE,
+            TransportSettings.PUBLISH_PORT_PROFILE
+        );
+
+        for (Setting.AffixSetting<?> profileSetting : transportProfileSettings) {
+            Settings testSettings = Settings.builder()
+                .put(REMOTE_CLUSTER_PORT_ENABLED.getKey(), true)
+                // We can just stick a random value in, even if it doesn't match the type - that validation happens at a different layer
+                .put(profileSetting.getConcreteSettingForNamespace(REMOTE_CLUSTER_PROFILE).getKey(), randomAlphaOfLength(5))
+                .build();
+            final IllegalArgumentException e = expectThrows(
+                IllegalArgumentException.class,
+                () -> RemoteClusterPortSettings.buildRemoteAccessProfileSettings(testSettings)
+            );
+            assertThat(
+                e.getMessage(),
+                containsString(
+                    "Remote Access settings should not be configured using the [_remote_cluster] profile. "
+                        + "Use the [remote_cluster.] settings instead."
+                )
+            );
+        }
+    }
+
+    public void testPortSettingsConstruction() {
+        String hostValue = NetworkAddress.format(randomIp(true));
+        Settings.Builder testSettingsBuilder = Settings.builder()
+            .put(REMOTE_CLUSTER_PORT_ENABLED.getKey(), true)
+            .put(randomFrom(RemoteClusterPortSettings.HOST, TransportSettings.BIND_HOST, TransportSettings.HOST).getKey(), hostValue);
+
+        boolean publishHostSet = randomBoolean();
+        String publishHostValue = publishHostSet ? NetworkAddress.format(randomIp(true)) : hostValue;
+        if (publishHostSet) {
+            testSettingsBuilder.put(RemoteClusterPortSettings.PUBLISH_HOST.getKey(), publishHostValue);
+        }
+
+        boolean bindHostSet = randomBoolean();
+        String bindHostValue = bindHostSet ? NetworkAddress.format(randomIp(true)) : hostValue;
+        if (bindHostSet) {
+            testSettingsBuilder.put(RemoteClusterPortSettings.BIND_HOST.getKey(), bindHostValue);
+        }
+
+        boolean portSet = randomBoolean();
+        int portValue = portSet ? randomInt(65535) : RemoteClusterPortSettings.PORT.getDefault(Settings.EMPTY);
+        if (portSet) {
+            testSettingsBuilder.put(RemoteClusterPortSettings.PORT.getKey(), portValue);
+        }
+
+        boolean publishPortSet = randomBoolean();
+        int publishPortValue = publishPortSet ? randomInt(65535) : -1; // Publish port is handled a bit weird in ProfileSettings
+        if (publishPortSet) {
+            testSettingsBuilder.put(RemoteClusterPortSettings.PUBLISH_PORT.getKey(), publishPortValue);
+        }
+
+        boolean keepAliveSet = randomBoolean();
+        boolean keepAliveValue = keepAliveSet ? randomBoolean() : NetworkService.TCP_KEEP_ALIVE.getDefault(Settings.EMPTY);
+        if (keepAliveSet) {
+            testSettingsBuilder.put(
+                randomFrom(RemoteClusterPortSettings.TCP_KEEP_ALIVE, TransportSettings.TCP_KEEP_ALIVE, NetworkService.TCP_KEEP_ALIVE)
+                    .getKey(),
+                keepAliveValue
+            );
+        }
+        boolean keepIdleSet = randomBoolean();
+        int keepIdleValue = keepIdleSet ? randomInt(300) : NetworkService.TCP_KEEP_IDLE.getDefault(Settings.EMPTY);
+        if (keepIdleSet) {
+            testSettingsBuilder.put(
+                randomFrom(RemoteClusterPortSettings.TCP_KEEP_IDLE, TransportSettings.TCP_KEEP_IDLE, NetworkService.TCP_KEEP_IDLE).getKey(),
+                keepIdleValue
+            );
+        }
+
+        boolean keepIntervalSet = randomBoolean();
+        int keepIntervalValue = keepIntervalSet ? randomInt(300) : NetworkService.TCP_KEEP_INTERVAL.getDefault(Settings.EMPTY);
+        if (keepIntervalSet) {
+            testSettingsBuilder.put(
+                randomFrom(
+                    RemoteClusterPortSettings.TCP_KEEP_INTERVAL,
+                    TransportSettings.TCP_KEEP_INTERVAL,
+                    NetworkService.TCP_KEEP_INTERVAL
+                ).getKey(),
+                keepIntervalValue
+            );
+        }
+        boolean keepCountSet = randomBoolean();
+        int keepCountValue = keepCountSet ? randomInt(1000000) : NetworkService.TCP_KEEP_COUNT.getDefault(Settings.EMPTY);
+        if (keepCountSet) {
+            testSettingsBuilder.put(
+                randomFrom(RemoteClusterPortSettings.TCP_KEEP_COUNT, TransportSettings.TCP_KEEP_COUNT, NetworkService.TCP_KEEP_COUNT)
+                    .getKey(),
+                keepCountValue
+            );
+        }
+        boolean noDelaySet = randomBoolean();
+        boolean noDelayValue = noDelaySet ? randomBoolean() : NetworkService.TCP_NO_DELAY.getDefault(Settings.EMPTY);
+        if (noDelaySet) {
+            testSettingsBuilder.put(
+                randomFrom(RemoteClusterPortSettings.TCP_NO_DELAY, TransportSettings.TCP_NO_DELAY).getKey(),
+                noDelayValue
+            );
+        }
+        boolean reuseAddressSet = randomBoolean();
+        boolean reuseAddressValue = reuseAddressSet ? randomBoolean() : NetworkService.TCP_REUSE_ADDRESS.getDefault(Settings.EMPTY);
+        if (reuseAddressSet) {
+            testSettingsBuilder.put(
+                randomFrom(
+                    RemoteClusterPortSettings.TCP_REUSE_ADDRESS,
+                    TransportSettings.TCP_REUSE_ADDRESS,
+                    NetworkService.TCP_REUSE_ADDRESS
+                ).getKey(),
+                reuseAddressValue
+            );
+        }
+        boolean sendBufferSizeSet = randomBoolean();
+        int sendBufSizeBytes = randomInt(10_000_000);
+        ByteSizeValue sendBufferSizeValue = sendBufferSizeSet
+            ? ByteSizeValue.ofBytes(sendBufSizeBytes)
+            : NetworkService.TCP_SEND_BUFFER_SIZE.getDefault(Settings.EMPTY);
+        if (sendBufferSizeSet) {
+            testSettingsBuilder.put(
+                randomFrom(
+                    RemoteClusterPortSettings.TCP_SEND_BUFFER_SIZE,
+                    TransportSettings.TCP_SEND_BUFFER_SIZE,
+                    NetworkService.TCP_SEND_BUFFER_SIZE
+                ).getKey(),
+                sendBufferSizeValue
+            );
+        }
+        boolean receiveBufferSizeSet = randomBoolean();
+        int rcvBufSizeBytes = randomInt(10_000_000);
+        ByteSizeValue receiveBufferSizeValue = receiveBufferSizeSet
+            ? ByteSizeValue.ofBytes(rcvBufSizeBytes)
+            : NetworkService.TCP_RECEIVE_BUFFER_SIZE.getDefault(Settings.EMPTY);
+        if (receiveBufferSizeSet) {
+            testSettingsBuilder.put(
+                randomFrom(
+                    RemoteClusterPortSettings.TCP_RECEIVE_BUFFER_SIZE,
+                    TransportSettings.TCP_RECEIVE_BUFFER_SIZE,
+                    NetworkService.TCP_RECEIVE_BUFFER_SIZE
+                ).getKey(),
+                receiveBufferSizeValue
+            );
+        }
+
+        Settings testSettings = testSettingsBuilder.build();
+        TcpTransport.ProfileSettings profileSettings = RemoteClusterPortSettings.buildRemoteAccessProfileSettings(testSettings);
+        assertThat(profileSettings.profileName, equalTo(REMOTE_CLUSTER_PROFILE));
+        assertThat(profileSettings.bindHosts, contains(bindHostValue));
+        assertThat(profileSettings.publishHosts, contains(publishHostValue));
+        assertThat(profileSettings.portOrRange, equalTo(Integer.toString(portValue)));
+        assertThat(profileSettings.publishPort, equalTo(publishPortValue));
+
+        assertThat(profileSettings.tcpNoDelay, equalTo(noDelayValue));
+        assertThat(profileSettings.tcpKeepAlive, equalTo(keepAliveValue));
+        assertThat(profileSettings.tcpKeepIdle, equalTo(keepIdleValue));
+        assertThat(profileSettings.tcpKeepInterval, equalTo(keepIntervalValue));
+        assertThat(profileSettings.tcpKeepCount, equalTo(keepCountValue));
+        assertThat(profileSettings.reuseAddress, equalTo(reuseAddressValue));
+        assertThat(profileSettings.sendBufferSize, equalTo(sendBufferSizeValue));
+        assertThat(profileSettings.receiveBufferSize, equalTo(receiveBufferSizeValue));
+        assertThat(profileSettings.isDefaultProfile, equalTo(false));
+    }
+
+}
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteConnectionManagerTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteConnectionManagerTests.java
index 031bafbaf78f..f2ecc9a3f06b 100644
--- a/server/src/test/java/org/elasticsearch/transport/RemoteConnectionManagerTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteConnectionManagerTests.java
@@ -19,7 +19,9 @@
 import java.net.InetAddress;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.concurrent.ExecutionException;
 
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItems;
 import static org.hamcrest.core.IsInstanceOf.instanceOf;
 import static org.mockito.ArgumentMatchers.any;
@@ -31,7 +33,9 @@ public class RemoteConnectionManagerTests extends ESTestCase {
     private Transport transport;
     private RemoteConnectionManager remoteConnectionManager;
     private ConnectionManager.ConnectionValidator validator = (connection, profile, listener) -> listener.onResponse(null);
+    private TransportAddress address = new TransportAddress(InetAddress.getLoopbackAddress(), 1000);
 
+    @SuppressWarnings("unchecked")
     @Override
     public void setUp() throws Exception {
         super.setUp();
@@ -40,18 +44,15 @@ public void setUp() throws Exception {
             "remote-cluster",
             new ClusterConnectionManager(Settings.EMPTY, transport, new ThreadContext(Settings.EMPTY))
         );
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testGetConnection() {
-        TransportAddress address = new TransportAddress(InetAddress.getLoopbackAddress(), 1000);
 
         doAnswer(invocationOnMock -> {
             ActionListener<Transport.Connection> listener = (ActionListener<Transport.Connection>) invocationOnMock.getArguments()[2];
             listener.onResponse(new TestRemoteConnection((DiscoveryNode) invocationOnMock.getArguments()[0]));
             return null;
         }).when(transport).openConnection(any(DiscoveryNode.class), any(ConnectionProfile.class), any(ActionListener.class));
+    }
 
+    public void testGetConnection() {
         DiscoveryNode node1 = new DiscoveryNode("node-1", address, Version.CURRENT);
         PlainActionFuture<Void> future1 = PlainActionFuture.newFuture();
         remoteConnectionManager.connectToRemoteClusterNode(node1, validator, future1);
@@ -89,6 +90,28 @@ public void testGetConnection() {
         assertEquals(1, versions.size());
     }
 
+    public void testResolveRemoteClusterAlias() throws ExecutionException, InterruptedException {
+        DiscoveryNode remoteNode1 = new DiscoveryNode("remote-node-1", address, Version.CURRENT);
+        PlainActionFuture<Void> future = PlainActionFuture.newFuture();
+        remoteConnectionManager.connectToRemoteClusterNode(remoteNode1, validator, future);
+        assertTrue(future.isDone());
+
+        Transport.Connection remoteConnection = remoteConnectionManager.getConnection(remoteNode1);
+        assertThat(RemoteConnectionManager.resolveRemoteClusterAlias(remoteConnection).get(), equalTo("remote-cluster"));
+
+        Transport.Connection localConnection = mock(Transport.Connection.class);
+        assertThat(RemoteConnectionManager.resolveRemoteClusterAlias(localConnection).isPresent(), equalTo(false));
+
+        DiscoveryNode remoteNode2 = new DiscoveryNode("remote-node-2", address, Version.CURRENT);
+        Transport.Connection proxyConnection = remoteConnectionManager.getConnection(remoteNode2);
+        assertThat(proxyConnection, instanceOf(RemoteConnectionManager.ProxyConnection.class));
+        assertThat(RemoteConnectionManager.resolveRemoteClusterAlias(proxyConnection).get(), equalTo("remote-cluster"));
+
+        PlainActionFuture<Transport.Connection> future2 = PlainActionFuture.newFuture();
+        remoteConnectionManager.openConnection(remoteNode1, null, future2);
+        assertThat(RemoteConnectionManager.resolveRemoteClusterAlias(future2.get()).get(), equalTo("remote-cluster"));
+    }
+
     private static class TestRemoteConnection extends CloseableConnection {
 
         private final DiscoveryNode node;
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteConnectionStrategyTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteConnectionStrategyTests.java
index 76cfafc41266..d27d805a2949 100644
--- a/server/src/test/java/org/elasticsearch/transport/RemoteConnectionStrategyTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteConnectionStrategyTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 
+import static org.elasticsearch.transport.RemoteClusterService.REMOTE_CLUSTER_AUTHORIZATION;
 import static org.mockito.Mockito.mock;
 
 public class RemoteConnectionStrategyTests extends ESTestCase {
@@ -120,6 +121,32 @@ public void testCorrectChannelNumber() {
         }
     }
 
+    public void testTransportProfile() {
+        String clusterAlias = "cluster-alias";
+
+        // New rcs connection with credentials
+        for (RemoteConnectionStrategy.ConnectionStrategy strategy : RemoteConnectionStrategy.ConnectionStrategy.values()) {
+            String settingKey = REMOTE_CLUSTER_AUTHORIZATION.getConcreteSettingForNamespace(clusterAlias).getKey();
+            final Settings settings = Settings.builder().put(settingKey, randomAlphaOfLength(20)).build();
+            ConnectionProfile profile = RemoteConnectionStrategy.buildConnectionProfile(clusterAlias, settings);
+            assertEquals(
+                "Incorrect transport profile for " + strategy.name(),
+                RemoteClusterPortSettings.REMOTE_CLUSTER_PROFILE,
+                profile.getTransportProfile()
+            );
+        }
+
+        // Legacy ones without credentials
+        for (RemoteConnectionStrategy.ConnectionStrategy strategy : RemoteConnectionStrategy.ConnectionStrategy.values()) {
+            ConnectionProfile profile = RemoteConnectionStrategy.buildConnectionProfile(clusterAlias, Settings.EMPTY);
+            assertEquals(
+                "Incorrect transport profile for " + strategy.name(),
+                TransportSettings.DEFAULT_PROFILE,
+                profile.getTransportProfile()
+            );
+        }
+    }
+
     private static class FakeConnectionStrategy extends RemoteConnectionStrategy {
 
         private final ConnectionStrategy strategy;
diff --git a/server/src/test/java/org/elasticsearch/transport/SingleResultDeduplicatorTests.java b/server/src/test/java/org/elasticsearch/transport/SingleResultDeduplicatorTests.java
new file mode 100644
index 000000000000..56bfe72241f2
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/transport/SingleResultDeduplicatorTests.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.transport;
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.SingleResultDeduplicator;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.ESTestCase;
+
+public class SingleResultDeduplicatorTests extends ESTestCase {
+
+    public void testDeduplicatesWithoutShowingStaleData() {
+        final SetOnce<ActionListener<Object>> firstListenerRef = new SetOnce<>();
+        final SetOnce<ActionListener<Object>> secondListenerRef = new SetOnce<>();
+        final var deduplicator = new SingleResultDeduplicator<>(new ThreadContext(Settings.EMPTY), l -> {
+            if (firstListenerRef.trySet(l) == false) {
+                secondListenerRef.set(l);
+            }
+        });
+        final Object result1 = new Object();
+        final Object result2 = new Object();
+
+        final int totalListeners = randomIntBetween(2, 10);
+        final boolean[] called = new boolean[totalListeners];
+        deduplicator.execute(new ActionListener<>() {
+            @Override
+            public void onResponse(Object response) {
+                assertFalse(called[0]);
+                called[0] = true;
+                assertEquals(result1, response);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError(e);
+            }
+        });
+
+        for (int i = 1; i < totalListeners; i++) {
+            final int index = i;
+            deduplicator.execute(new ActionListener<>() {
+
+                @Override
+                public void onResponse(Object response) {
+                    assertFalse(called[index]);
+                    called[index] = true;
+                    assertEquals(result2, response);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    throw new AssertionError(e);
+                }
+            });
+        }
+        for (int i = 0; i < totalListeners; i++) {
+            assertFalse(called[i]);
+        }
+        firstListenerRef.get().onResponse(result1);
+        assertTrue(called[0]);
+        for (int i = 1; i < totalListeners; i++) {
+            assertFalse(called[i]);
+        }
+        secondListenerRef.get().onResponse(result2);
+        for (int i = 0; i < totalListeners; i++) {
+            assertTrue(called[i]);
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java
index 772158e695c5..9cc5b043c07a 100644
--- a/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/SniffConnectionStrategyTests.java
@@ -937,7 +937,7 @@ public void testModeSettingsCannotBeUsedWhenInDifferentMode() {
             Setting<?> concreteSetting = restrictedSetting.v1().getConcreteSettingForNamespace(clusterName);
             Settings invalid = Settings.builder().put(settings).put(concreteSetting.getKey(), restrictedSetting.v2()).build();
             IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> service.validate(invalid, true));
-            String expected = formatted("""
+            String expected = Strings.format("""
                 Setting "%s" cannot be used with the configured "cluster.remote.cluster_name.mode" \
                 [required=SNIFF, configured=PROXY]\
                 """, concreteSetting.getKey());
diff --git a/server/src/test/java/org/elasticsearch/transport/TcpTransportTests.java b/server/src/test/java/org/elasticsearch/transport/TcpTransportTests.java
index 72941da7ceb1..532f2778ae21 100644
--- a/server/src/test/java/org/elasticsearch/transport/TcpTransportTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/TcpTransportTests.java
@@ -11,7 +11,7 @@
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.component.Lifecycle;
@@ -227,7 +227,7 @@ private void testDefaultSeedAddresses(final Settings settings, Matcher<Iterable<
         try {
             final TcpTransport tcpTransport = new TcpTransport(
                 settings,
-                Version.CURRENT,
+                TransportVersion.CURRENT,
                 testThreadPool,
                 new MockPageCacheRecycler(settings),
                 new NoneCircuitBreakerService(),
@@ -245,7 +245,7 @@ protected TcpServerChannel bind(String name, InetSocketAddress address) {
                 }
 
                 @Override
-                protected TcpChannel initiateChannel(DiscoveryNode node) {
+                protected TcpChannel initiateChannel(DiscoveryNode node, ConnectionProfile connectionProfile) {
                     throw new UnsupportedOperationException();
                 }
 
@@ -602,7 +602,7 @@ private void testExceptionHandling(
                 lifecycle,
                 new OutboundHandler(
                     randomAlphaOfLength(10),
-                    Version.CURRENT,
+                    TransportVersion.CURRENT,
                     new StatsTracker(),
                     testThreadPool,
                     new BytesRefRecycler(new MockPageCacheRecycler(Settings.EMPTY)),
diff --git a/server/src/test/java/org/elasticsearch/transport/TransportHandshakerTests.java b/server/src/test/java/org/elasticsearch/transport/TransportHandshakerTests.java
index e20ac4f5f637..2446d51e3720 100644
--- a/server/src/test/java/org/elasticsearch/transport/TransportHandshakerTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/TransportHandshakerTests.java
@@ -7,6 +7,7 @@
  */
 package org.elasticsearch.transport;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -52,7 +53,7 @@ public void setUp() throws Exception {
             Version.CURRENT
         );
         threadPool = new TestThreadPool("thread-poll");
-        handshaker = new TransportHandshaker(Version.CURRENT, threadPool, requestSender, false);
+        handshaker = new TransportHandshaker(TransportVersion.CURRENT, threadPool, requestSender, false);
     }
 
     @Override
@@ -62,15 +63,15 @@ public void tearDown() throws Exception {
     }
 
     public void testHandshakeRequestAndResponse() throws IOException {
-        PlainActionFuture<Version> versionFuture = PlainActionFuture.newFuture();
+        PlainActionFuture<TransportVersion> versionFuture = PlainActionFuture.newFuture();
         long reqId = randomLongBetween(1, 10);
         handshaker.sendHandshake(reqId, node, channel, new TimeValue(30, TimeUnit.SECONDS), versionFuture);
 
-        verify(requestSender).sendRequest(node, channel, reqId, Version.CURRENT.minimumCompatibilityVersion());
+        verify(requestSender).sendRequest(node, channel, reqId, TransportVersion.CURRENT.minimumCompatibilityVersion());
 
         assertFalse(versionFuture.isDone());
 
-        TransportHandshaker.HandshakeRequest handshakeRequest = new TransportHandshaker.HandshakeRequest(Version.CURRENT);
+        TransportHandshaker.HandshakeRequest handshakeRequest = new TransportHandshaker.HandshakeRequest(TransportVersion.CURRENT);
         BytesStreamOutput bytesStreamOutput = new BytesStreamOutput();
         handshakeRequest.writeTo(bytesStreamOutput);
         StreamInput input = bytesStreamOutput.bytes().streamInput();
@@ -82,16 +83,16 @@ public void testHandshakeRequestAndResponse() throws IOException {
         handler.handleResponse((TransportHandshaker.HandshakeResponse) responseFuture.actionGet());
 
         assertTrue(versionFuture.isDone());
-        assertEquals(Version.CURRENT, versionFuture.actionGet());
+        assertEquals(TransportVersion.CURRENT, versionFuture.actionGet());
     }
 
     public void testHandshakeRequestFutureVersionsCompatibility() throws IOException {
         long reqId = randomLongBetween(1, 10);
         handshaker.sendHandshake(reqId, node, channel, new TimeValue(30, TimeUnit.SECONDS), PlainActionFuture.newFuture());
 
-        verify(requestSender).sendRequest(node, channel, reqId, Version.CURRENT.minimumCompatibilityVersion());
+        verify(requestSender).sendRequest(node, channel, reqId, TransportVersion.CURRENT.minimumCompatibilityVersion());
 
-        TransportHandshaker.HandshakeRequest handshakeRequest = new TransportHandshaker.HandshakeRequest(Version.CURRENT);
+        TransportHandshaker.HandshakeRequest handshakeRequest = new TransportHandshaker.HandshakeRequest(TransportVersion.CURRENT);
         BytesStreamOutput currentHandshakeBytes = new BytesStreamOutput();
         handshakeRequest.writeTo(currentHandshakeBytes);
 
@@ -117,15 +118,15 @@ public void testHandshakeRequestFutureVersionsCompatibility() throws IOException
 
         TransportHandshaker.HandshakeResponse response = (TransportHandshaker.HandshakeResponse) responseFuture.actionGet();
 
-        assertEquals(Version.CURRENT, response.getResponseVersion());
+        assertEquals(TransportVersion.CURRENT, response.getResponseVersion());
     }
 
     public void testHandshakeError() throws IOException {
-        PlainActionFuture<Version> versionFuture = PlainActionFuture.newFuture();
+        PlainActionFuture<TransportVersion> versionFuture = PlainActionFuture.newFuture();
         long reqId = randomLongBetween(1, 10);
         handshaker.sendHandshake(reqId, node, channel, new TimeValue(30, TimeUnit.SECONDS), versionFuture);
 
-        verify(requestSender).sendRequest(node, channel, reqId, Version.CURRENT.minimumCompatibilityVersion());
+        verify(requestSender).sendRequest(node, channel, reqId, TransportVersion.CURRENT.minimumCompatibilityVersion());
 
         assertFalse(versionFuture.isDone());
 
@@ -138,9 +139,9 @@ public void testHandshakeError() throws IOException {
     }
 
     public void testSendRequestThrowsException() throws IOException {
-        PlainActionFuture<Version> versionFuture = PlainActionFuture.newFuture();
+        PlainActionFuture<TransportVersion> versionFuture = PlainActionFuture.newFuture();
         long reqId = randomLongBetween(1, 10);
-        Version compatibilityVersion = Version.CURRENT.minimumCompatibilityVersion();
+        TransportVersion compatibilityVersion = TransportVersion.CURRENT.minimumCompatibilityVersion();
         doThrow(new IOException("boom")).when(requestSender).sendRequest(node, channel, reqId, compatibilityVersion);
 
         handshaker.sendHandshake(reqId, node, channel, new TimeValue(30, TimeUnit.SECONDS), versionFuture);
@@ -152,11 +153,11 @@ public void testSendRequestThrowsException() throws IOException {
     }
 
     public void testHandshakeTimeout() throws IOException {
-        PlainActionFuture<Version> versionFuture = PlainActionFuture.newFuture();
+        PlainActionFuture<TransportVersion> versionFuture = PlainActionFuture.newFuture();
         long reqId = randomLongBetween(1, 10);
         handshaker.sendHandshake(reqId, node, channel, new TimeValue(100, TimeUnit.MILLISECONDS), versionFuture);
 
-        verify(requestSender).sendRequest(node, channel, reqId, Version.CURRENT.minimumCompatibilityVersion());
+        verify(requestSender).sendRequest(node, channel, reqId, TransportVersion.CURRENT.minimumCompatibilityVersion());
 
         ConnectTransportException cte = expectThrows(ConnectTransportException.class, versionFuture::actionGet);
         assertThat(cte.getMessage(), containsString("handshake_timeout"));
diff --git a/server/src/test/java/org/elasticsearch/transport/TransportLoggerTests.java b/server/src/test/java/org/elasticsearch/transport/TransportLoggerTests.java
index 2f5527a86e92..22781d1f04b9 100644
--- a/server/src/test/java/org/elasticsearch/transport/TransportLoggerTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/TransportLoggerTests.java
@@ -9,7 +9,7 @@
 
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsAction;
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequest;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -89,7 +89,7 @@ private BytesReference buildRequest() throws IOException {
             OutboundMessage.Request request = new OutboundMessage.Request(
                 new ThreadContext(Settings.EMPTY),
                 new ClusterStatsRequest(),
-                Version.CURRENT,
+                TransportVersion.CURRENT,
                 ClusterStatsAction.NAME,
                 randomInt(30),
                 false,
diff --git a/server/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java b/server/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java
index d1000048d838..027d3bfe01a0 100644
--- a/server/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.transport;
 
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -61,7 +62,7 @@ private TransportService startServices(
     ) {
         TcpTransport transport = new Netty4Transport(
             settings,
-            Version.CURRENT,
+            TransportVersion.CURRENT,
             threadPool,
             new NetworkService(Collections.emptyList()),
             PageCacheRecycler.NON_RECYCLING_INSTANCE,
diff --git a/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java b/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java
index 97457a4f014a..d7e204401076 100644
--- a/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.transport;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -135,7 +136,7 @@ public ExecutorService executor(String name) {
                     return super.executor(name);
                 }
             };
-            final var tcpTransport = MockTransportService.newMockTransport(Settings.EMPTY, Version.CURRENT, threadPool);
+            final var tcpTransport = MockTransportService.newMockTransport(Settings.EMPTY, TransportVersion.CURRENT, threadPool);
             transportService = new TransportService(
                 Settings.EMPTY,
                 tcpTransport,
diff --git a/server/src/test/java/org/elasticsearch/upgrades/FeatureMigrationResultsTests.java b/server/src/test/java/org/elasticsearch/upgrades/FeatureMigrationResultsTests.java
index 9d997843f70e..6dbae94492aa 100644
--- a/server/src/test/java/org/elasticsearch/upgrades/FeatureMigrationResultsTests.java
+++ b/server/src/test/java/org/elasticsearch/upgrades/FeatureMigrationResultsTests.java
@@ -12,12 +12,12 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.test.SimpleDiffableSerializationTestCase;
+import org.elasticsearch.test.ChunkedToXContentDiffableSerializationTestCase;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 
-public class FeatureMigrationResultsTests extends SimpleDiffableSerializationTestCase<Metadata.Custom> {
+public class FeatureMigrationResultsTests extends ChunkedToXContentDiffableSerializationTestCase<Metadata.Custom> {
 
     @Override
     protected FeatureMigrationResults createTestInstance() {
diff --git a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParamsTests.java b/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParamsTests.java
index 8a7e6cf73afa..d115b28c18a0 100644
--- a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParamsTests.java
+++ b/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParamsTests.java
@@ -24,6 +24,11 @@ protected SystemIndexMigrationTaskParams createTestInstance() {
         return new SystemIndexMigrationTaskParams();
     }
 
+    @Override
+    protected SystemIndexMigrationTaskParams mutateInstance(SystemIndexMigrationTaskParams instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected SystemIndexMigrationTaskParams copyInstance(SystemIndexMigrationTaskParams instance, Version version) throws IOException {
         return new SystemIndexMigrationTaskParams();
diff --git a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskStateTests.java b/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskStateTests.java
index 9526055867ec..f6176d5f56b2 100644
--- a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskStateTests.java
+++ b/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskStateTests.java
@@ -63,7 +63,7 @@ protected SystemIndexMigrationTaskState copyInstance(SystemIndexMigrationTaskSta
     }
 
     @Override
-    protected SystemIndexMigrationTaskState mutateInstance(SystemIndexMigrationTaskState instance) throws IOException {
+    protected SystemIndexMigrationTaskState mutateInstance(SystemIndexMigrationTaskState instance) {
         String index = instance.getCurrentIndex();
         String feature = instance.getCurrentFeature();
         Map<String, Object> featureMetadata = instance.getFeatureCallbackMetadata();
diff --git a/settings.gradle b/settings.gradle
index 0fbb994ec248..d98f65042e44 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -83,8 +83,9 @@ List projects = [
   'test:fixtures:url-fixture',
   'test:fixtures:nginx-fixture',
   'test:logger-usage',
-  'test:yaml-rest-runner',
-  'test:x-content'
+  'test:test-clusters',
+  'test:x-content',
+  'test:yaml-rest-runner'
 ]
 
 /**
@@ -135,6 +136,10 @@ project(":libs").children.each { libsProject ->
   }
 }
 
+project(":qa:stable-api").children.each { libsProject ->
+  libsProject.name = "elasticsearch-${libsProject.name}"
+}
+
 project(":test:external-modules").children.each { testProject ->
   testProject.name = "test-${testProject.name}"
 }
diff --git a/test/external-modules/build.gradle b/test/external-modules/build.gradle
index 7b2c74918124..538a0439f88f 100644
--- a/test/external-modules/build.gradle
+++ b/test/external-modules/build.gradle
@@ -3,7 +3,7 @@ import org.elasticsearch.gradle.internal.info.BuildParams
 
 subprojects {
   apply plugin: 'elasticsearch.base-internal-es-plugin'
-  apply plugin: 'elasticsearch.internal-yaml-rest-test'
+  apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 
   esplugin {
     name it.name
diff --git a/test/external-modules/delayed-aggs/src/main/java/org/elasticsearch/test/delayedshard/DelayedShardAggregationBuilder.java b/test/external-modules/delayed-aggs/src/main/java/org/elasticsearch/test/delayedshard/DelayedShardAggregationBuilder.java
index 42a15a28bb3f..0549d1b3053b 100644
--- a/test/external-modules/delayed-aggs/src/main/java/org/elasticsearch/test/delayedshard/DelayedShardAggregationBuilder.java
+++ b/test/external-modules/delayed-aggs/src/main/java/org/elasticsearch/test/delayedshard/DelayedShardAggregationBuilder.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.test.delayedshard;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.TimeValue;
@@ -133,7 +133,7 @@ public int hashCode() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_10_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_10_0;
     }
 }
diff --git a/test/external-modules/die-with-dignity/build.gradle b/test/external-modules/die-with-dignity/build.gradle
index 3466b7bb24ad..3046b045220a 100644
--- a/test/external-modules/die-with-dignity/build.gradle
+++ b/test/external-modules/die-with-dignity/build.gradle
@@ -1,7 +1,7 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.util.GradleUtils
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.internal-es-plugin'
 
 esplugin {
diff --git a/test/external-modules/error-query/src/main/java/org/elasticsearch/test/errorquery/ErrorQueryBuilder.java b/test/external-modules/error-query/src/main/java/org/elasticsearch/test/errorquery/ErrorQueryBuilder.java
index 5343069eb6d6..bf48adefee5f 100644
--- a/test/external-modules/error-query/src/main/java/org/elasticsearch/test/errorquery/ErrorQueryBuilder.java
+++ b/test/external-modules/error-query/src/main/java/org/elasticsearch/test/errorquery/ErrorQueryBuilder.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.HeaderWarning;
@@ -130,7 +130,7 @@ protected int doHashCode() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/TestUtils.java b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/TestUtils.java
similarity index 93%
rename from modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/TestUtils.java
rename to test/fixtures/gcs-fixture/src/main/java/fixture/gcs/TestUtils.java
index 800846fbbecf..3d831e5212e4 100644
--- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/TestUtils.java
+++ b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/TestUtils.java
@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-package org.elasticsearch.repositories.gcs;
+package fixture.gcs;
 
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
@@ -16,14 +16,14 @@
 import java.util.Random;
 import java.util.UUID;
 
-final class TestUtils {
+public final class TestUtils {
 
     private TestUtils() {}
 
     /**
      * Creates a random Service Account file for testing purpose
      */
-    static byte[] createServiceAccount(final Random random) {
+    public static byte[] createServiceAccount(final Random random) {
         try {
             final KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA");
             keyPairGenerator.initialize(2048);
diff --git a/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
index bed3271ca49d..85453971f761 100644
--- a/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -67,6 +68,11 @@ public static ClusterState state(
         ShardRoutingState primaryState,
         ShardRoutingState... replicaStates
     ) {
+        assert primaryState == ShardRoutingState.STARTED
+            || primaryState == ShardRoutingState.RELOCATING
+            || Arrays.stream(replicaStates).allMatch(s -> s == ShardRoutingState.UNASSIGNED)
+            : "invalid shard states [" + primaryState + "] vs [" + Arrays.toString(replicaStates) + "]";
+
         final int numberOfReplicas = replicaStates.length;
 
         int numberOfNodes = numberOfReplicas + 1;
diff --git a/test/framework/src/main/java/org/elasticsearch/cli/CommandTestCase.java b/test/framework/src/main/java/org/elasticsearch/cli/CommandTestCase.java
index 6c3573e2594d..a3b9703a16a6 100644
--- a/test/framework/src/main/java/org/elasticsearch/cli/CommandTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/cli/CommandTestCase.java
@@ -24,6 +24,7 @@
 /**
  * A base test case for cli tools.
  */
+@ESTestCase.WithoutSecurityManager
 public abstract class CommandTestCase extends ESTestCase {
 
     /** The terminal that execute uses */
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java
index c984f3a31556..18a611e39b5a 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java
@@ -48,19 +48,15 @@
 import java.util.List;
 import java.util.Map;
 import java.util.OptionalDouble;
-import java.util.Random;
 import java.util.Set;
 
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.cluster.ClusterModule.BALANCED_ALLOCATOR;
 import static org.elasticsearch.cluster.ClusterModule.DESIRED_BALANCE_ALLOCATOR;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
+import static org.elasticsearch.common.settings.ClusterSettings.createBuiltInClusterSettings;
 
 public abstract class ESAllocationTestCase extends ESTestCase {
-    private static final ClusterSettings EMPTY_CLUSTER_SETTINGS = new ClusterSettings(
-        Settings.EMPTY,
-        ClusterSettings.BUILT_IN_CLUSTER_SETTINGS
-    );
 
     public static final SnapshotsInfoService SNAPSHOT_INFO_SERVICE_WITH_NO_SHARD_SIZES = () -> new SnapshotShardSizeInfo(Map.of()) {
         @Override
@@ -71,21 +67,30 @@ public Long getShardSize(ShardRouting shardRouting) {
         }
     };
 
+    public static final WriteLoadForecaster TEST_WRITE_LOAD_FORECASTER = new WriteLoadForecaster() {
+        @Override
+        public Metadata.Builder withWriteLoadForecastForWriteIndex(String dataStreamName, Metadata.Builder metadata) {
+            throw new AssertionError("Not required for testing");
+        }
+
+        @Override
+        @SuppressForbidden(reason = "tests do not need a license to access the write load")
+        public OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata) {
+            return indexMetadata.getForecastedWriteLoad();
+        }
+    };
+
     public static MockAllocationService createAllocationService() {
         return createAllocationService(Settings.EMPTY);
     }
 
     public static MockAllocationService createAllocationService(Settings settings) {
-        return createAllocationService(settings, random());
+        return createAllocationService(settings, createBuiltInClusterSettings(settings));
     }
 
-    public static MockAllocationService createAllocationService(Settings settings, Random random) {
-        return createAllocationService(settings, EMPTY_CLUSTER_SETTINGS, random);
-    }
-
-    public static MockAllocationService createAllocationService(Settings settings, ClusterSettings clusterSettings, Random random) {
+    public static MockAllocationService createAllocationService(Settings settings, ClusterSettings clusterSettings) {
         return new MockAllocationService(
-            randomAllocationDeciders(settings, clusterSettings, random),
+            randomAllocationDeciders(settings, clusterSettings),
             new TestGatewayAllocator(),
             createShardsAllocator(settings),
             EmptyClusterInfoService.INSTANCE,
@@ -103,7 +108,13 @@ private static ShardsAllocator createShardsAllocator(Settings settings) {
 
     private static DesiredBalanceShardsAllocator createDesiredBalanceShardsAllocator(Settings settings) {
         var queue = new DeterministicTaskQueue();
-        return new DesiredBalanceShardsAllocator(new BalancedShardsAllocator(settings), queue.getThreadPool(), null, null) {
+        return new DesiredBalanceShardsAllocator(
+            createBuiltInClusterSettings(settings),
+            new BalancedShardsAllocator(settings),
+            queue.getThreadPool(),
+            null,
+            null
+        ) {
             private RoutingAllocation lastAllocation;
 
             @Override
@@ -126,43 +137,44 @@ protected void submitReconcileTask(DesiredBalance desiredBalance) {
         };
     }
 
-    public static MockAllocationService createAllocationService(Settings settings, ClusterInfoService clusterInfoService) {
-        return new MockAllocationService(
-            randomAllocationDeciders(settings, EMPTY_CLUSTER_SETTINGS, random()),
-            new TestGatewayAllocator(),
-            new BalancedShardsAllocator(settings),
-            clusterInfoService,
+    public static MockAllocationService createAllocationService(Settings settings, GatewayAllocator gatewayAllocator) {
+        return createAllocationService(
+            settings,
+            gatewayAllocator,
+            EmptyClusterInfoService.INSTANCE,
             SNAPSHOT_INFO_SERVICE_WITH_NO_SHARD_SIZES
         );
     }
 
-    public static MockAllocationService createAllocationService(Settings settings, GatewayAllocator gatewayAllocator) {
-        return createAllocationService(settings, gatewayAllocator, SNAPSHOT_INFO_SERVICE_WITH_NO_SHARD_SIZES);
+    public static MockAllocationService createAllocationService(Settings settings, ClusterInfoService clusterInfoService) {
+        return createAllocationService(settings, new TestGatewayAllocator(), clusterInfoService, SNAPSHOT_INFO_SERVICE_WITH_NO_SHARD_SIZES);
     }
 
     public static MockAllocationService createAllocationService(Settings settings, SnapshotsInfoService snapshotsInfoService) {
-        return createAllocationService(settings, new TestGatewayAllocator(), snapshotsInfoService);
+        return createAllocationService(settings, new TestGatewayAllocator(), EmptyClusterInfoService.INSTANCE, snapshotsInfoService);
     }
 
     public static MockAllocationService createAllocationService(
         Settings settings,
         GatewayAllocator gatewayAllocator,
+        ClusterInfoService clusterInfoService,
         SnapshotsInfoService snapshotsInfoService
     ) {
+        var clusterSettings = createBuiltInClusterSettings(settings);
         return new MockAllocationService(
-            randomAllocationDeciders(settings, EMPTY_CLUSTER_SETTINGS, random()),
+            randomAllocationDeciders(settings, clusterSettings),
             gatewayAllocator,
-            new BalancedShardsAllocator(settings),
-            EmptyClusterInfoService.INSTANCE,
+            new BalancedShardsAllocator(clusterSettings, WriteLoadForecaster.DEFAULT),
+            clusterInfoService,
             snapshotsInfoService
         );
     }
 
-    public static AllocationDeciders randomAllocationDeciders(Settings settings, ClusterSettings clusterSettings, Random random) {
+    public static AllocationDeciders randomAllocationDeciders(Settings settings, ClusterSettings clusterSettings) {
         List<AllocationDecider> deciders = new ArrayList<>(
             ClusterModule.createAllocationDeciders(settings, clusterSettings, Collections.emptyList())
         );
-        Collections.shuffle(deciders, random);
+        Collections.shuffle(deciders, random());
         return new AllocationDeciders(deciders);
     }
 
@@ -202,13 +214,7 @@ protected static ClusterState startRandomInitializingShard(ClusterState clusterS
 
     protected static AllocationDeciders yesAllocationDeciders() {
         return new AllocationDeciders(
-            Arrays.asList(
-                new TestAllocateDecision(Decision.YES),
-                new SameShardAllocationDecider(
-                    Settings.EMPTY,
-                    new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
-                )
-            )
+            Arrays.asList(new TestAllocateDecision(Decision.YES), new SameShardAllocationDecider(createBuiltInClusterSettings()))
         );
     }
 
@@ -218,13 +224,7 @@ protected static AllocationDeciders noAllocationDeciders() {
 
     protected static AllocationDeciders throttleAllocationDeciders() {
         return new AllocationDeciders(
-            Arrays.asList(
-                new TestAllocateDecision(Decision.THROTTLE),
-                new SameShardAllocationDecider(
-                    Settings.EMPTY,
-                    new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
-                )
-            )
+            Arrays.asList(new TestAllocateDecision(Decision.THROTTLE), new SameShardAllocationDecider(createBuiltInClusterSettings()))
         );
     }
 
@@ -263,7 +263,7 @@ public static ClusterState startInitializingShardsAndReroute(
         ClusterState clusterState,
         RoutingNode routingNode
     ) {
-        return startShardsAndReroute(allocationService, clusterState, routingNode.shardsWithState(INITIALIZING));
+        return startShardsAndReroute(allocationService, clusterState, routingNode.shardsWithState(INITIALIZING).toList());
     }
 
     /**
@@ -337,6 +337,8 @@ protected static class MockAllocationService extends AllocationService {
 
         private volatile long nanoTimeOverride = -1L;
 
+        private final GatewayAllocator gatewayAllocator;
+
         public MockAllocationService(
             AllocationDeciders allocationDeciders,
             GatewayAllocator gatewayAllocator,
@@ -344,7 +346,15 @@ public MockAllocationService(
             ClusterInfoService clusterInfoService,
             SnapshotsInfoService snapshotsInfoService
         ) {
-            super(allocationDeciders, gatewayAllocator, shardsAllocator, clusterInfoService, snapshotsInfoService);
+            super(
+                allocationDeciders,
+                gatewayAllocator,
+                shardsAllocator,
+                clusterInfoService,
+                snapshotsInfoService,
+                TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
+            );
+            this.gatewayAllocator = gatewayAllocator;
         }
 
         public void setNanoTimeOverride(long nanoTime) {
@@ -355,6 +365,10 @@ public void setNanoTimeOverride(long nanoTime) {
         protected long currentNanoTime() {
             return nanoTimeOverride == -1L ? super.currentNanoTime() : nanoTimeOverride;
         }
+
+        public GatewayAllocator getGatewayAllocator() {
+            return gatewayAllocator;
+        }
     }
 
     /**
@@ -397,18 +411,4 @@ public void allocateUnassigned(
             }
         }
     }
-
-    protected static final WriteLoadForecaster SIMULATION_WRITE_LOAD_FORECASTER = new WriteLoadForecaster() {
-        @Override
-        public Metadata.Builder withWriteLoadForecastForWriteIndex(String dataStreamName, Metadata.Builder metadata) {
-            throw new AssertionError("only called during rollover");
-        }
-
-        @Override
-        @SuppressForbidden(reason = "tests do not need a license to access the write load")
-        public OptionalDouble getForecastedWriteLoad(IndexMetadata indexMetadata) {
-            return indexMetadata.getForecastedWriteLoad();
-        }
-    };
-
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/TestShardRoutingRoleStrategies.java b/test/framework/src/main/java/org/elasticsearch/cluster/TestShardRoutingRoleStrategies.java
new file mode 100644
index 000000000000..e2f384f0a6e1
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/TestShardRoutingRoleStrategies.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.cluster;
+
+import org.elasticsearch.cluster.routing.ShardRoutingRoleStrategy;
+
+import java.util.List;
+
+public class TestShardRoutingRoleStrategies {
+
+    /**
+     * A strategy which only returns the default role in all situations. This is deliberately not available to production code to avoid any
+     * possibility of using it instead of the strategy provided by the plugin (if so configured).
+     */
+    public static final ShardRoutingRoleStrategy DEFAULT_ROLE_ONLY = ClusterModule.getShardRoutingRoleStrategy(List.of());
+
+    private TestShardRoutingRoleStrategies() {
+        // no instances
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java
index dd3247147f67..238a7defdfc3 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.NodeConnectionsService;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.coordination.AbstractCoordinatorTestCase.Cluster.ClusterNode;
 import org.elasticsearch.cluster.coordination.CoordinationMetadata.VotingConfiguration;
 import org.elasticsearch.cluster.coordination.LinearizabilityChecker.History;
@@ -53,7 +54,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
 import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor;
 import org.elasticsearch.common.util.set.Sets;
@@ -72,11 +72,11 @@
 import org.elasticsearch.monitor.NodeHealthService;
 import org.elasticsearch.monitor.StatusInfo;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.disruption.DisruptableMockTransport;
-import org.elasticsearch.test.disruption.DisruptableMockTransport.ConnectionStatus;
 import org.elasticsearch.threadpool.Scheduler;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.BytesRefRecycler;
+import org.elasticsearch.transport.DisruptableMockTransport;
+import org.elasticsearch.transport.DisruptableMockTransport.ConnectionStatus;
 import org.elasticsearch.transport.TransportInterceptor;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
@@ -271,6 +271,7 @@ public class Cluster implements Releasable {
         private final Map<Long, ClusterState> committedStatesByVersion = new HashMap<>();
         private final LinearizabilityChecker linearizabilityChecker = new LinearizabilityChecker();
         private final History history = new History();
+        private final CountingPageCacheRecycler countingPageCacheRecycler;
         private final Recycler<BytesRef> recycler;
         private final NodeHealthService nodeHealthService;
 
@@ -289,9 +290,8 @@ public Cluster(int initialNodeCount, boolean allNodesMasterEligible, Settings no
 
         Cluster(int initialNodeCount, boolean allNodesMasterEligible, Settings nodeSettings, NodeHealthService nodeHealthService) {
             this.nodeHealthService = nodeHealthService;
-            this.recycler = usually()
-                ? BytesRefRecycler.NON_RECYCLING_INSTANCE
-                : new BytesRefRecycler(new MockPageCacheRecycler(Settings.EMPTY));
+            this.countingPageCacheRecycler = new CountingPageCacheRecycler();
+            this.recycler = new BytesRefRecycler(countingPageCacheRecycler);
             deterministicTaskQueue.setExecutionDelayVariabilityMillis(DEFAULT_DELAY_VARIABILITY);
 
             assertThat(initialNodeCount, greaterThan(0));
@@ -876,6 +876,12 @@ public void close() {
             }
 
             clusterNodes.forEach(ClusterNode::close);
+
+            // Closing nodes may spawn some other background cleanup tasks that must also be run
+            runFor(DEFAULT_DELAY_VARIABILITY, "accumulate close-time tasks");
+            deterministicTaskQueue.runAllRunnableTasks();
+
+            countingPageCacheRecycler.assertAllPagesReleased();
         }
 
         protected List<NamedWriteableRegistry.Entry> extraNamedWriteables() {
@@ -1135,7 +1141,7 @@ public class ClusterNode {
             private void setUp() {
                 final ThreadPool threadPool = deterministicTaskQueue.getThreadPool(this::onNode);
                 clearableRecycler = new ClearableRecycler(recycler);
-                mockTransport = new DisruptableMockTransport(localNode, logger, deterministicTaskQueue) {
+                mockTransport = new DisruptableMockTransport(localNode, deterministicTaskQueue) {
                     @Override
                     protected void execute(Runnable runnable) {
                         deterministicTaskQueue.scheduleNow(onNode(runnable));
@@ -1289,6 +1295,7 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() {
                     settings,
                     new BatchedRerouteService(clusterService, allocationService::reroute),
                     clusterService,
+                    TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
                     threadPool
                 );
 
@@ -1390,13 +1397,13 @@ Runnable onNode(Runnable runnable) {
                     public void run() {
                         if (clusterNodes.contains(ClusterNode.this)) {
                             wrapped.run();
-                        } else if (runnable instanceof DisruptableMockTransport.RebootSensitiveRunnable) {
+                        } else if (runnable instanceof DisruptableMockTransport.RebootSensitiveRunnable rebootSensitiveRunnable) {
                             logger.trace(
                                 "completing reboot-sensitive runnable {} from node {} as node has been removed from cluster",
                                 runnable,
                                 localNode
                             );
-                            ((DisruptableMockTransport.RebootSensitiveRunnable) runnable).ifRebooted();
+                            rebootSensitiveRunnable.ifRebooted();
                         } else {
                             logger.trace("ignoring runnable {} from node {} as node has been removed from cluster", runnable, localNode);
                         }
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/CountingPageCacheRecycler.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/CountingPageCacheRecycler.java
new file mode 100644
index 000000000000..788a943e10d2
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/CountingPageCacheRecycler.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.cluster.coordination;
+
+import org.elasticsearch.common.recycler.Recycler;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.PageCacheRecycler;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+public class CountingPageCacheRecycler extends PageCacheRecycler {
+
+    private int openPages = 0;
+
+    public CountingPageCacheRecycler() {
+        super(Settings.EMPTY);
+    }
+
+    @Override
+    public Recycler.V<byte[]> bytePage(boolean clear) {
+        final var page = super.bytePage(clear);
+        openPages += 1;
+        return new Recycler.V<>() {
+            boolean closed = false;
+
+            @Override
+            public byte[] v() {
+                return page.v();
+            }
+
+            @Override
+            public boolean isRecycled() {
+                return page.isRecycled();
+            }
+
+            @Override
+            public void close() {
+                assertFalse(closed);
+                closed = true;
+                openPages -= 1;
+                page.close();
+            }
+        };
+    }
+
+    @Override
+    public Recycler.V<Object[]> objectPage() {
+        throw new AssertionError("unexpected call to objectPage()");
+    }
+
+    public void assertAllPagesReleased() {
+        assertEquals(0, openPages);
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java
index 4605bb31662e..d44116e36afc 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.admin.indices.rollover.MetadataRolloverService;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -451,6 +452,7 @@ public static MetadataRolloverService getMetadataRolloverService(
         when(env.sharedDataFile()).thenReturn(null);
         AllocationService allocationService = mock(AllocationService.class);
         when(allocationService.reroute(any(ClusterState.class), any(String.class), any())).then(i -> i.getArguments()[0]);
+        when(allocationService.getShardRoutingRoleStrategy()).thenReturn(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         MappingLookup mappingLookup = null;
         if (dataStream != null) {
             RootObjectMapper.Builder root = new RootObjectMapper.Builder("_doc", ObjectMapper.Defaults.SUBOBJECTS);
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/routing/RoutingNodesHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/routing/RoutingNodesHelper.java
index 4387f36effa0..4ed514fa1670 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/routing/RoutingNodesHelper.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/routing/RoutingNodesHelper.java
@@ -10,42 +10,27 @@
 
 import org.elasticsearch.cluster.node.DiscoveryNode;
 
-import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
+import static org.elasticsearch.common.util.CollectionUtils.iterableAsArrayList;
+
 public final class RoutingNodesHelper {
 
     private RoutingNodesHelper() {}
 
     public static List<ShardRouting> shardsWithState(RoutingNodes routingNodes, ShardRoutingState state) {
-        List<ShardRouting> shards = new ArrayList<>();
-        for (RoutingNode routingNode : routingNodes) {
-            shards.addAll(routingNode.shardsWithState(state));
-        }
-        if (state == ShardRoutingState.UNASSIGNED) {
-            routingNodes.unassigned().forEach(shards::add);
-        }
-        return shards;
+        return state == ShardRoutingState.UNASSIGNED
+            ? iterableAsArrayList(routingNodes.unassigned())
+            : routingNodes.stream().flatMap(routingNode -> routingNode.shardsWithState(state)).toList();
     }
 
-    public static List<ShardRouting> shardsWithState(RoutingNodes routingNodes, String index, ShardRoutingState... state) {
-        List<ShardRouting> shards = new ArrayList<>();
-        for (RoutingNode routingNode : routingNodes) {
-            shards.addAll(routingNode.shardsWithState(index, state));
-        }
-        for (ShardRoutingState s : state) {
-            if (s == ShardRoutingState.UNASSIGNED) {
-                for (ShardRouting unassignedShard : routingNodes.unassigned()) {
-                    if (unassignedShard.index().getName().equals(index)) {
-                        shards.add(unassignedShard);
-                    }
-                }
-                break;
-            }
-        }
-        return shards;
+    public static List<ShardRouting> shardsWithState(RoutingNodes routingNodes, String index, ShardRoutingState states) {
+        return shardsWithState(routingNodes, states).stream()
+            .filter(shardRouting -> Objects.equals(shardRouting.getIndexName(), index))
+            .toList();
     }
 
     /**
@@ -64,7 +49,6 @@ public static RoutingNode routingNode(String nodeId, DiscoveryNode node, ShardRo
         for (ShardRouting shardRouting : shards) {
             routingNode.add(shardRouting);
         }
-
         return routingNode;
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java
index c82a43ed6b10..53562b1b2ff4 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java
@@ -50,7 +50,8 @@ public static ShardRouting initWithSameId(ShardRouting copy, RecoverySource reco
             new UnassignedInfo(UnassignedInfo.Reason.REINITIALIZED, null),
             RelocationFailureInfo.NO_FAILURES,
             copy.allocationId(),
-            copy.getExpectedShardSize()
+            copy.getExpectedShardSize(),
+            copy.role()
         );
     }
 
@@ -69,7 +70,8 @@ public static ShardRouting newWithRestoreSource(ShardRouting routing, SnapshotRe
             routing.unassignedInfo(),
             routing.relocationFailureInfo(),
             routing.allocationId(),
-            routing.getExpectedShardSize()
+            routing.getExpectedShardSize(),
+            routing.role()
         );
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java
index 76858a91ff52..ecda06813489 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java
@@ -45,7 +45,8 @@ public static ShardRouting newShardRouting(ShardId shardId, String currentNodeId
             buildUnassignedInfo(state),
             buildRelocationFailureInfo(state),
             buildAllocationId(state),
-            -1
+            -1,
+            ShardRouting.Role.DEFAULT
         );
     }
 
@@ -66,7 +67,8 @@ public static ShardRouting newShardRouting(
             buildUnassignedInfo(state),
             buildRelocationFailureInfo(state),
             buildAllocationId(state),
-            -1
+            -1,
+            ShardRouting.Role.DEFAULT
         );
     }
 
@@ -104,7 +106,8 @@ public static ShardRouting newShardRouting(
             buildUnassignedInfo(state),
             buildRelocationFailureInfo(state),
             buildAllocationId(state),
-            -1
+            -1,
+            ShardRouting.Role.DEFAULT
         );
     }
 
@@ -145,7 +148,8 @@ public static ShardRouting newShardRouting(
             buildUnassignedInfo(state),
             buildRelocationFailureInfo(state),
             allocationId,
-            -1
+            -1,
+            ShardRouting.Role.DEFAULT
         );
     }
 
@@ -186,7 +190,8 @@ public static ShardRouting newShardRouting(
             unassignedInfo,
             buildRelocationFailureInfo(state),
             buildAllocationId(state),
-            -1
+            -1,
+            ShardRouting.Role.DEFAULT
         );
     }
 
@@ -194,49 +199,31 @@ public static ShardRouting relocate(ShardRouting shardRouting, String relocating
         return shardRouting.relocate(relocatingNodeId, expectedShardSize);
     }
 
-    private static RecoverySource buildRecoveryTarget(boolean primary, ShardRoutingState state) {
-        switch (state) {
-            case UNASSIGNED:
-            case INITIALIZING:
-                if (primary) {
-                    return randomFrom(
-                        RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-                        RecoverySource.ExistingStoreRecoverySource.INSTANCE
-                    );
-                } else {
-                    return RecoverySource.PeerRecoverySource.INSTANCE;
-                }
-            case STARTED:
-            case RELOCATING:
-                return null;
-            default:
-                throw new IllegalStateException("illegal state");
-        }
+    public static RecoverySource buildRecoveryTarget(boolean primary, ShardRoutingState state) {
+        return switch (state) {
+            case UNASSIGNED, INITIALIZING -> primary
+                ? randomFrom(RecoverySource.EmptyStoreRecoverySource.INSTANCE, RecoverySource.ExistingStoreRecoverySource.INSTANCE)
+                : RecoverySource.PeerRecoverySource.INSTANCE;
+            case STARTED, RELOCATING -> null;
+        };
     }
 
-    private static AllocationId buildAllocationId(ShardRoutingState state) {
-        switch (state) {
-            case UNASSIGNED:
-                return null;
-            case INITIALIZING:
-            case STARTED:
-                return AllocationId.newInitializing();
-            case RELOCATING:
-                AllocationId allocationId = AllocationId.newInitializing();
-                return AllocationId.newRelocation(allocationId);
-            default:
-                throw new IllegalStateException("illegal state");
-        }
+    public static AllocationId buildAllocationId(ShardRoutingState state) {
+        return switch (state) {
+            case UNASSIGNED -> null;
+            case INITIALIZING, STARTED -> AllocationId.newInitializing();
+            case RELOCATING -> AllocationId.newRelocation(AllocationId.newInitializing());
+        };
     }
 
-    private static UnassignedInfo buildUnassignedInfo(ShardRoutingState state) {
+    public static UnassignedInfo buildUnassignedInfo(ShardRoutingState state) {
         return switch (state) {
             case UNASSIGNED, INITIALIZING -> randomUnassignedInfo("auto generated for test");
             case STARTED, RELOCATING -> null;
         };
     }
 
-    private static RelocationFailureInfo buildRelocationFailureInfo(ShardRoutingState state) {
+    public static RelocationFailureInfo buildRelocationFailureInfo(ShardRoutingState state) {
         return switch (state) {
             case UNASSIGNED, INITIALIZING, STARTED -> RelocationFailureInfo.NO_FAILURES;
             case RELOCATING -> randomBoolean() ? RelocationFailureInfo.NO_FAILURES : new RelocationFailureInfo(randomIntBetween(1, 10));
diff --git a/test/framework/src/main/java/org/elasticsearch/common/lucene/store/ESIndexInputTestCase.java b/test/framework/src/main/java/org/elasticsearch/common/lucene/store/ESIndexInputTestCase.java
index 1117a953e7c0..c4132e9dbdb1 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/lucene/store/ESIndexInputTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/lucene/store/ESIndexInputTestCase.java
@@ -21,6 +21,8 @@
 import org.junit.BeforeClass;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -60,18 +62,29 @@ protected byte[] randomReadAndSlice(IndexInput indexInput, int length) throws IO
         int readPos = (int) indexInput.getFilePointer();
         byte[] output = new byte[length];
         while (readPos < length) {
-            switch (randomIntBetween(0, 5)) {
-                case 0:
-                    // Read by one byte at a time
-                    output[readPos++] = indexInput.readByte();
+            final var readStrategy = between(0, 8);
+            switch (readStrategy) {
+                case 0, 1, 2, 3:
+                    if (length - readPos >= Long.BYTES && readStrategy <= 0) {
+                        ByteBuffer.wrap(output, readPos, Long.BYTES).order(ByteOrder.LITTLE_ENDIAN).putLong(indexInput.readLong());
+                        readPos += Long.BYTES;
+                    } else if (length - readPos >= Integer.BYTES && readStrategy <= 1) {
+                        ByteBuffer.wrap(output, readPos, Integer.BYTES).order(ByteOrder.LITTLE_ENDIAN).putInt(indexInput.readInt());
+                        readPos += Integer.BYTES;
+                    } else if (length - readPos >= Short.BYTES && readStrategy <= 2) {
+                        ByteBuffer.wrap(output, readPos, Short.BYTES).order(ByteOrder.LITTLE_ENDIAN).putShort(indexInput.readShort());
+                        readPos += Short.BYTES;
+                    } else {
+                        output[readPos++] = indexInput.readByte();
+                    }
                     break;
-                case 1:
+                case 4:
                     // Read several bytes into target
                     int len = randomIntBetween(1, length - readPos);
                     indexInput.readBytes(output, readPos, len);
                     readPos += len;
                     break;
-                case 2:
+                case 5:
                     // Read several bytes into 0-offset target
                     len = randomIntBetween(1, length - readPos);
                     byte[] temp = new byte[len];
@@ -79,7 +92,7 @@ protected byte[] randomReadAndSlice(IndexInput indexInput, int length) throws IO
                     System.arraycopy(temp, 0, output, readPos, len);
                     readPos += len;
                     break;
-                case 3:
+                case 6:
                     // Read using slice
                     len = randomIntBetween(1, length - readPos);
                     final String sliceExtension = randomValueOtherThan(".cfs", ESIndexInputTestCase::randomFileExtension);
@@ -92,7 +105,7 @@ protected byte[] randomReadAndSlice(IndexInput indexInput, int length) throws IO
                     indexInput.seek(readPos);
                     assertEquals(readPos, indexInput.getFilePointer());
                     break;
-                case 4:
+                case 7:
                     // Seek at a random position and read a single byte,
                     // then seek back to original position
                     final int lastReadPos = readPos;
@@ -106,7 +119,7 @@ protected byte[] randomReadAndSlice(IndexInput indexInput, int length) throws IO
                     indexInput.seek(readPos);
                     assertEquals(readPos, indexInput.getFilePointer());
                     break;
-                case 5:
+                case 8:
                     // Read clone or slice concurrently
                     final int cloneCount = between(1, 3);
                     final CountDownLatch startLatch = new CountDownLatch(1 + cloneCount);
diff --git a/test/framework/src/main/java/org/elasticsearch/common/settings/MockSecureSettings.java b/test/framework/src/main/java/org/elasticsearch/common/settings/MockSecureSettings.java
index 49b55ed20785..8bfbc6f4f53b 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/settings/MockSecureSettings.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/settings/MockSecureSettings.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.common.settings;
 
 import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.common.io.stream.StreamOutput;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
@@ -113,4 +114,9 @@ public SecureSettings clone() {
         ensureOpen();
         return new MockSecureSettings(this);
     }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        throw new IllegalStateException("Not supported, implement me!");
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
index e1d106f6fff6..5c1d395ed28e 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
@@ -49,6 +49,11 @@
 public class MockBigArrays extends BigArrays {
     private static final Logger logger = LogManager.getLogger(MockBigArrays.class);
 
+    /**
+     * Error message thrown by {@link BigArrays} produced with {@link MockBigArrays#MockBigArrays(PageCacheRecycler, ByteSizeValue)}.
+     */
+    public static final String ERROR_MESSAGE = "over test limit";
+
     /**
      * Assert that a function returning a {@link Releasable} runs to completion
      * when allocated a breaker with that breaks when it uses more than {@code max}
@@ -407,6 +412,11 @@ public byte[] array() {
         public Collection<Accountable> getChildResources() {
             return Collections.singleton(Accountables.namedAccountable("delegate", in));
         }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            in.writeTo(out);
+        }
     }
 
     private class IntArrayWrapper extends AbstractArrayWrapper implements IntArray {
@@ -513,6 +523,10 @@ public Collection<Accountable> getChildResources() {
             return Collections.singleton(Accountables.namedAccountable("delegate", in));
         }
 
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            in.writeTo(out);
+        }
     }
 
     private class FloatArrayWrapper extends AbstractArrayWrapper implements FloatArray {
@@ -663,7 +677,7 @@ public LimitedBreaker(String name, ByteSizeValue max) {
         public void addEstimateBytesAndMaybeBreak(long bytes, String label) throws CircuitBreakingException {
             long total = used.addAndGet(bytes);
             if (total > max.getBytes()) {
-                throw new CircuitBreakingException("test error", bytes, max.getBytes(), Durability.TRANSIENT);
+                throw new CircuitBreakingException(ERROR_MESSAGE, bytes, max.getBytes(), Durability.TRANSIENT);
             }
         }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/common/util/concurrent/CyclicBarrierUtils.java b/test/framework/src/main/java/org/elasticsearch/common/util/concurrent/CyclicBarrierUtils.java
new file mode 100644
index 000000000000..7fd9e1122725
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/common/util/concurrent/CyclicBarrierUtils.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common.util.concurrent;
+
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
+
+public class CyclicBarrierUtils {
+    private CyclicBarrierUtils() {
+        // no instances
+    }
+
+    /**
+     * Await the given {@link CyclicBarrier}, failing the test after 10s with an {@link AssertionError}. Tests should not wait forever, so
+     * a timed wait is always appropriate.
+     */
+    public static void await(CyclicBarrier cyclicBarrier) {
+        try {
+            cyclicBarrier.await(10, TimeUnit.SECONDS);
+        } catch (Exception e) {
+            throw new AssertionError("unexpected", e);
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/common/util/concurrent/DeterministicTaskQueue.java b/test/framework/src/main/java/org/elasticsearch/common/util/concurrent/DeterministicTaskQueue.java
index b27ab2899c67..771e25786040 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/util/concurrent/DeterministicTaskQueue.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/util/concurrent/DeterministicTaskQueue.java
@@ -309,6 +309,11 @@ public <T> T invokeAny(Collection<? extends Callable<T>> tasks, long timeout, Ti
                 public void execute(Runnable command) {
                     scheduleNow(runnableWrapper.apply(command));
                 }
+
+                @Override
+                public String toString() {
+                    return "DeterministicTaskQueue/forkingExecutor";
+                }
             };
 
             @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
index ac48caafb8e8..68b4f18fbcfd 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
@@ -249,7 +249,7 @@ public static FieldType randomIdFieldType() {
         return randomBoolean() ? ProvidedIdFieldMapper.Defaults.FIELD_TYPE : TsidExtractingIdFieldMapper.FIELD_TYPE;
     }
 
-    public EngineConfig copy(EngineConfig config, LongSupplier globalCheckpointSupplier) {
+    public static EngineConfig copy(EngineConfig config, LongSupplier globalCheckpointSupplier) {
         return new EngineConfig(
             config.getShardId(),
             config.getThreadPool(),
@@ -259,7 +259,7 @@ public EngineConfig copy(EngineConfig config, LongSupplier globalCheckpointSuppl
             config.getMergePolicy(),
             config.getAnalyzer(),
             config.getSimilarity(),
-            newCodecService(),
+            config.getCodecService(),
             config.getEventListener(),
             config.getQueryCache(),
             config.getQueryCachingPolicy(),
@@ -274,7 +274,9 @@ public EngineConfig copy(EngineConfig config, LongSupplier globalCheckpointSuppl
             config.getPrimaryTermSupplier(),
             config.getSnapshotCommitSupplier(),
             config.getLeafSorter(),
-            config.getRelativeTimeInNanosSupplier()
+            config.getRelativeTimeInNanosSupplier(),
+            config.getIndexCommitListener(),
+            config.isPromotableToPrimary()
         );
     }
 
@@ -288,7 +290,7 @@ public EngineConfig copy(EngineConfig config, Analyzer analyzer) {
             config.getMergePolicy(),
             analyzer,
             config.getSimilarity(),
-            newCodecService(),
+            config.getCodecService(),
             config.getEventListener(),
             config.getQueryCache(),
             config.getQueryCachingPolicy(),
@@ -303,7 +305,9 @@ public EngineConfig copy(EngineConfig config, Analyzer analyzer) {
             config.getPrimaryTermSupplier(),
             config.getSnapshotCommitSupplier(),
             config.getLeafSorter(),
-            config.getRelativeTimeInNanosSupplier()
+            config.getRelativeTimeInNanosSupplier(),
+            config.getIndexCommitListener(),
+            config.isPromotableToPrimary()
         );
     }
 
@@ -317,7 +321,7 @@ public EngineConfig copy(EngineConfig config, MergePolicy mergePolicy) {
             mergePolicy,
             config.getAnalyzer(),
             config.getSimilarity(),
-            newCodecService(),
+            config.getCodecService(),
             config.getEventListener(),
             config.getQueryCache(),
             config.getQueryCachingPolicy(),
@@ -332,7 +336,9 @@ public EngineConfig copy(EngineConfig config, MergePolicy mergePolicy) {
             config.getPrimaryTermSupplier(),
             config.getSnapshotCommitSupplier(),
             config.getLeafSorter(),
-            config.getRelativeTimeInNanosSupplier()
+            config.getRelativeTimeInNanosSupplier(),
+            config.getIndexCommitListener(),
+            config.isPromotableToPrimary()
         );
     }
 
@@ -754,7 +760,8 @@ public EngineConfig config(
             indexSort,
             globalCheckpointSupplier,
             retentionLeasesSupplier,
-            new NoneCircuitBreakerService()
+            new NoneCircuitBreakerService(),
+            null
         );
     }
 
@@ -779,7 +786,8 @@ public EngineConfig config(
             indexSort,
             maybeGlobalCheckpointSupplier,
             maybeGlobalCheckpointSupplier == null ? null : () -> RetentionLeases.EMPTY,
-            breakerService
+            breakerService,
+            null
         );
     }
 
@@ -793,7 +801,8 @@ public EngineConfig config(
         final Sort indexSort,
         final @Nullable LongSupplier maybeGlobalCheckpointSupplier,
         final @Nullable Supplier<RetentionLeases> maybeRetentionLeasesSupplier,
-        final CircuitBreakerService breakerService
+        final CircuitBreakerService breakerService,
+        final @Nullable Engine.IndexCommitListener indexCommitListener
     ) {
         final IndexWriterConfig iwc = newIndexWriterConfig();
         final TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, BigArrays.NON_RECYCLING_INSTANCE);
@@ -851,7 +860,9 @@ public EngineConfig config(
             primaryTerm,
             IndexModule.DEFAULT_SNAPSHOT_COMMIT_SUPPLIER,
             null,
-            System::nanoTime
+            System::nanoTime,
+            indexCommitListener,
+            true
         );
     }
 
@@ -888,7 +899,9 @@ protected EngineConfig config(EngineConfig config, Store store, Path translogPat
             config.getPrimaryTermSupplier(),
             config.getSnapshotCommitSupplier(),
             config.getLeafSorter(),
-            config.getRelativeTimeInNanosSupplier()
+            config.getRelativeTimeInNanosSupplier(),
+            config.getIndexCommitListener(),
+            config.isPromotableToPrimary()
         );
     }
 
@@ -1620,7 +1633,7 @@ private static LazySoftDeletesDirectoryReaderWrapper.LazyBits lazyBits(LeafReade
         throw new IllegalStateException("Can not extract lazy bits from given index reader [" + reader + "]");
     }
 
-    static CodecService newCodecService() {
+    protected static CodecService newCodecService() {
         return new CodecService(null, BigArrays.NON_RECYCLING_INSTANCE);
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
index 505fe2bfe316..c661302d74bd 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.search.lookup.Source;
-import org.elasticsearch.search.lookup.SourceLookup;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentType;
 
@@ -38,9 +37,7 @@ private static SearchExecutionContext createMockSearchExecutionContext(boolean a
         SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class);
         when(searchExecutionContext.allowExpensiveQueries()).thenReturn(allowExpensiveQueries);
         when(searchExecutionContext.isSourceEnabled()).thenReturn(true);
-        SourceLookup sourceLookup = mock(SourceLookup.class);
         SearchLookup searchLookup = mock(SearchLookup.class);
-        when(searchLookup.source()).thenReturn(sourceLookup);
         when(searchExecutionContext.lookup()).thenReturn(searchLookup);
         when(searchExecutionContext.indexVersionCreated()).thenReturn(Version.CURRENT);
         return searchExecutionContext;
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java
index 4ef132026574..8c06d2e63404 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java
@@ -49,7 +49,8 @@
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.lookup.LeafStoredFieldsLookup;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.json.JsonXContent;
@@ -354,7 +355,7 @@ protected <T> void assertMetricType(String metricType, Function<T, Enum<TimeSeri
 
         @SuppressWarnings("unchecked") // Syntactic sugar in tests
         T fieldType = (T) mapperService.fieldType("field");
-        assertThat(checker.apply(fieldType).name(), equalTo(metricType));
+        assertThat(checker.apply(fieldType).toString(), equalTo(metricType));
     }
 
     public final void testEmptyName() {
@@ -388,8 +389,7 @@ public final void testMinimalSerializesToItself() throws IOException {
         assertParseMinimalWarnings();
     }
 
-    // TODO make this final once we remove FieldMapperTestCase2
-    public void testMinimalToMaximal() throws IOException {
+    public final void testMinimalToMaximal() throws IOException {
         XContentBuilder orig = JsonXContent.contentBuilder().startObject();
         createMapperService(fieldMapping(this::minimalMapping)).documentMapper().mapping().toXContent(orig, INCLUDE_DEFAULTS);
         orig.endObject();
@@ -510,14 +510,14 @@ protected final List<?> fetchFromDocValues(MapperService mapperService, MappedFi
                 SearchLookup lookup = new SearchLookup(
                     mapperService::fieldType,
                     fieldDataLookup(mapperService),
-                    new SourceLookup.ReaderSourceProvider()
+                    SourceProvider.fromStoredFields()
                 );
                 ValueFetcher valueFetcher = new DocValueFetcher(format, lookup.getForField(ft, MappedFieldType.FielddataOperation.SEARCH));
                 IndexSearcher searcher = newSearcher(iw);
                 LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
-                lookup.source().setSegmentAndDocument(context, 0);
+                Source source = lookup.getSource(context, 0);
                 valueFetcher.setNextReader(context);
-                result.set(valueFetcher.fetchValues(lookup.source(), 0, new ArrayList<>()));
+                result.set(valueFetcher.fetchValues(source, 0, new ArrayList<>()));
             }
         );
         return result.get();
@@ -531,7 +531,10 @@ protected final void assertScriptDocValues(MapperService mapperService, Object s
             iw -> {
                 IndexSearcher searcher = newSearcher(iw);
                 MappedFieldType ft = mapperService.fieldType("field");
-                SearchLookup searchLookup = new SearchLookup(null, null, mapperService.mappingLookup().getSourceProvider());
+                SourceProvider sourceProvider = mapperService.mappingLookup().isSourceSynthetic()
+                    ? (ctx, doc) -> { throw new IllegalArgumentException("Can't load source in scripts in synthetic mode"); }
+                    : SourceProvider.fromStoredFields();
+                SearchLookup searchLookup = new SearchLookup(null, null, sourceProvider);
                 IndexFieldData<?> sfd = ft.fielddataBuilder(
                     new FieldDataContext("", () -> searchLookup, Set::of, MappedFieldType.FielddataOperation.SCRIPT)
                 ).build(null, null);
@@ -812,12 +815,11 @@ protected void assertFetch(MapperService mapperService, String field, Object val
         ValueFetcher nativeFetcher = ft.valueFetcher(searchExecutionContext, format);
         ParsedDocument doc = mapperService.documentMapper().parse(source);
         withLuceneIndex(mapperService, iw -> iw.addDocuments(doc.docs()), ir -> {
-            SourceLookup sourceLookup = new SourceLookup(new SourceLookup.ReaderSourceProvider());
-            sourceLookup.setSegmentAndDocument(ir.leaves().get(0), 0);
+            Source s = SourceProvider.fromStoredFields().getSource(ir.leaves().get(0), 0);
             docValueFetcher.setNextReader(ir.leaves().get(0));
             nativeFetcher.setNextReader(ir.leaves().get(0));
-            List<Object> fromDocValues = docValueFetcher.fetchValues(sourceLookup, 0, new ArrayList<>());
-            List<Object> fromNative = nativeFetcher.fetchValues(sourceLookup, 0, new ArrayList<>());
+            List<Object> fromDocValues = docValueFetcher.fetchValues(s, 0, new ArrayList<>());
+            List<Object> fromNative = nativeFetcher.fetchValues(s, 0, new ArrayList<>());
             /*
              * The native fetcher uses byte, short, etc but doc values always
              * uses long or double. This difference is fine because on the outside
@@ -924,7 +926,7 @@ public final void testIndexTimeStoredFieldsAccess() throws IOException {
         SearchLookup lookup = new SearchLookup(
             f -> fieldType,
             (f, s, t) -> { throw new UnsupportedOperationException(); },
-            new SourceLookup.ReaderSourceProvider()
+            (ctx, docid) -> Source.fromBytes(doc.source())
         );
 
         withLuceneIndex(mapperService, iw -> iw.addDocument(doc.rootDoc()), ir -> {
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java
index b42d561de4d7..23308c7677d3 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java
@@ -10,13 +10,8 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.time.DateFormatter;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.xcontent.XContentParser;
 
-import java.util.function.Function;
-
 /**
  * Simplified version of {@link DocumentParserContext} to be used in tests.
  * Every non final method throws {@link UnsupportedOperationException} and can be implemented as needed.
@@ -33,21 +28,29 @@ public class TestDocumentParserContext extends DocumentParserContext {
      * Use with caution as it can cause {@link NullPointerException}s down the line.
      */
     public TestDocumentParserContext() {
-        super(MappingLookup.EMPTY, MapperTestCase.createIndexSettings(Version.CURRENT, Settings.EMPTY), null, null, null);
+        this(MappingLookup.EMPTY, null);
     }
 
     /**
      * More verbose way to create a context, to be used when one or more constructor arguments are needed as final methods
      * that depend on them are called while executing tests.
      */
-    public TestDocumentParserContext(
-        MappingLookup mappingLookup,
-        IndexSettings indexSettings,
-        IndexAnalyzers indexAnalyzers,
-        Function<DateFormatter, MappingParserContext> parserContextFunction,
-        SourceToParse source
-    ) {
-        super(mappingLookup, indexSettings, indexAnalyzers, parserContextFunction, source);
+    public TestDocumentParserContext(MappingLookup mappingLookup, SourceToParse source) {
+        super(
+            mappingLookup,
+            new MappingParserContext(
+                s -> null,
+                s -> null,
+                s -> null,
+                Version.CURRENT,
+                () -> null,
+                null,
+                null,
+                MapperTestCase.createIndexSettings(Version.CURRENT, Settings.EMPTY),
+                null
+            ),
+            source
+        );
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
index 0d873b92af4b..efe811ef38b6 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
@@ -572,7 +572,9 @@ private void updateAllocationIDsOnPrimary() throws IOException {
                 null,
                 currentClusterStateVersion.incrementAndGet(),
                 activeIds(),
-                routingTable(Function.identity())
+                primary.routingEntry().initializing()
+                    ? IndexShardRoutingTable.builder(primary.shardId()).addShard(primary.routingEntry()).build()
+                    : routingTable(Function.identity())
             );
         }
 
@@ -852,7 +854,7 @@ protected void performOnPrimary(IndexShard primary, BulkShardRequest request, Ac
             executeShardBulkOnPrimary(
                 primary,
                 request,
-                listener.map(result -> new PrimaryResult(result.replicaRequest(), result.finalResponseIfSuccessful))
+                listener.map(result -> new PrimaryResult(result.replicaRequest(), result.replicationResponse))
             );
         }
 
@@ -1031,7 +1033,7 @@ protected void performOnPrimary(IndexShard primary, ResyncReplicationRequest req
             ActionListener.completeWith(listener, () -> {
                 final TransportWriteAction.WritePrimaryResult<ResyncReplicationRequest, ResyncReplicationResponse> result =
                     executeResyncOnPrimary(primary, request);
-                return new PrimaryResult(result.replicaRequest(), result.finalResponseIfSuccessful);
+                return new PrimaryResult(result.replicaRequest(), result.replicationResponse);
             });
         }
 
@@ -1056,7 +1058,6 @@ private TransportWriteAction.WritePrimaryResult<ResyncReplicationRequest, Resync
                 TransportResyncReplicationAction.performOnPrimary(request),
                 new ResyncReplicationResponse(),
                 null,
-                null,
                 primary,
                 logger
             );
diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
index c7ffa47896e8..559bec543c96 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
@@ -503,7 +503,9 @@ protected IndexShard newShard(
                 retentionLeaseSyncer,
                 breakerService,
                 IndexModule.DEFAULT_SNAPSHOT_COMMIT_SUPPLIER,
-                relativeTimeSupplier
+                relativeTimeSupplier,
+                null,
+                ReplicationTracker.DEFAULT_FACTORY
             );
             indexShard.addShardFailureCallback(DEFAULT_SHARD_FAILURE_HANDLER);
             success = true;
@@ -675,7 +677,11 @@ protected void recoverShardFromStore(IndexShard primary) throws IOException {
 
     public static void updateRoutingEntry(IndexShard shard, ShardRouting shardRouting) throws IOException {
         Set<String> inSyncIds = shardRouting.active() ? Collections.singleton(shardRouting.allocationId().getId()) : Collections.emptySet();
-        IndexShardRoutingTable newRoutingTable = new IndexShardRoutingTable.Builder(shardRouting.shardId()).addShard(shardRouting).build();
+        final var builder = new IndexShardRoutingTable.Builder(shardRouting.shardId());
+        if (shardRouting.primary() == false) {
+            builder.addShard(TestShardRouting.newShardRouting(shardRouting.shardId(), "ignored", true, ShardRoutingState.STARTED));
+        }
+        IndexShardRoutingTable newRoutingTable = builder.addShard(shardRouting).build();
         shard.updateShardState(
             shardRouting,
             shard.getPendingPrimaryTerm(),
@@ -856,9 +862,9 @@ protected void promoteReplica(IndexShard replica, Set<String> inSyncIds, IndexSh
             replica.routingEntry().allocationId()
         );
 
-        final IndexShardRoutingTable newRoutingTable = new IndexShardRoutingTable.Builder(routingTable).removeShard(replica.routingEntry())
-            .addShard(routingEntry)
-            .build();
+        final IndexShardRoutingTable newRoutingTable = new IndexShardRoutingTable.Builder(routingTable).removeShard(
+            routingTable.primaryShard()
+        ).removeShard(replica.routingEntry()).addShard(routingEntry).build();
         replica.updateShardState(
             routingEntry,
             replica.getPendingPrimaryTerm() + 1,
@@ -1051,7 +1057,6 @@ protected ShardGeneration snapshotShard(final IndexShard shard, final Snapshot s
                     null,
                     snapshotStatus,
                     Version.CURRENT,
-                    Collections.emptyMap(),
                     randomMillisUpToYear9999(),
                     future
                 )
diff --git a/test/framework/src/main/java/org/elasticsearch/indices/CrankyCircuitBreakerService.java b/test/framework/src/main/java/org/elasticsearch/indices/CrankyCircuitBreakerService.java
new file mode 100644
index 000000000000..15ffa52569d0
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/indices/CrankyCircuitBreakerService.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.indices;
+
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.breaker.CircuitBreakingException;
+import org.elasticsearch.indices.breaker.AllCircuitBreakerStats;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.indices.breaker.CircuitBreakerStats;
+import org.elasticsearch.test.ESTestCase;
+
+/**
+ * {@link CircuitBreakerService} that fails one twentieth of the time when you
+ * add bytes. This is useful to make sure code responds sensibly to circuit
+ * breaks at unpredictable times.
+ */
+public class CrankyCircuitBreakerService extends CircuitBreakerService {
+    /**
+     * Error message thrown when the breaker randomly trips.
+     */
+    public static final String ERROR_MESSAGE = "cranky breaker";
+
+    private final CircuitBreaker breaker = new CircuitBreaker() {
+        @Override
+        public void circuitBreak(String fieldName, long bytesNeeded) {
+
+        }
+
+        @Override
+        public void addEstimateBytesAndMaybeBreak(long bytes, String label) throws CircuitBreakingException {
+            if (ESTestCase.random().nextInt(20) == 0) {
+                throw new CircuitBreakingException(ERROR_MESSAGE, Durability.PERMANENT);
+            }
+        }
+
+        @Override
+        public void addWithoutBreaking(long bytes) {
+
+        }
+
+        @Override
+        public long getUsed() {
+            return 0;
+        }
+
+        @Override
+        public long getLimit() {
+            return 0;
+        }
+
+        @Override
+        public double getOverhead() {
+            return 0;
+        }
+
+        @Override
+        public long getTrippedCount() {
+            return 0;
+        }
+
+        @Override
+        public String getName() {
+            return CircuitBreaker.FIELDDATA;
+        }
+
+        @Override
+        public Durability getDurability() {
+            return null;
+        }
+
+        @Override
+        public void setLimitAndOverhead(long limit, double overhead) {
+
+        }
+    };
+
+    @Override
+    public CircuitBreaker getBreaker(String name) {
+        return breaker;
+    }
+
+    @Override
+    public AllCircuitBreakerStats stats() {
+        return new AllCircuitBreakerStats(new CircuitBreakerStats[] { stats(CircuitBreaker.FIELDDATA) });
+    }
+
+    @Override
+    public CircuitBreakerStats stats(String name) {
+        return new CircuitBreakerStats(CircuitBreaker.FIELDDATA, -1, -1, 0, 0);
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/indices/recovery/AbstractIndexRecoveryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/indices/recovery/AbstractIndexRecoveryIntegTestCase.java
index 02d46839b29e..4d099a96644c 100644
--- a/test/framework/src/main/java/org/elasticsearch/indices/recovery/AbstractIndexRecoveryIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/indices/recovery/AbstractIndexRecoveryIntegTestCase.java
@@ -178,12 +178,11 @@ protected void checkTransientErrorsDuringRecoveryAreRetried(String recoveryActio
                 redTransportService.disconnectFromNode(blueTransportService.getLocalDiscoNode());
             }
         };
-        TransientReceiveRejected handlingBehavior = new TransientReceiveRejected(
-            recoveryActionToBlock,
-            recoveryStarted,
-            finalizeReceived,
-            connectionBreaker
-        );
+        TransientReceiveRejected handlingBehavior = new TransientReceiveRejected(recoveryActionToBlock, recoveryStarted, connectionBreaker);
+        redTransportService.addRequestHandlingBehavior(PeerRecoveryTargetService.Actions.FINALIZE, (handler, request, channel, task) -> {
+            finalizeReceived.set(true);
+            handler.messageReceived(request, channel, task);
+        });
         redTransportService.addRequestHandlingBehavior(recoveryActionToBlock, handlingBehavior);
 
         try {
@@ -580,19 +579,12 @@ private class TransientReceiveRejected implements StubbableTransport.RequestHand
 
         private final String actionName;
         private final AtomicBoolean recoveryStarted;
-        private final AtomicBoolean finalizeReceived;
         private final Runnable connectionBreaker;
         private final AtomicInteger blocksRemaining;
 
-        private TransientReceiveRejected(
-            String actionName,
-            AtomicBoolean recoveryStarted,
-            AtomicBoolean finalizeReceived,
-            Runnable connectionBreaker
-        ) {
+        private TransientReceiveRejected(String actionName, AtomicBoolean recoveryStarted, Runnable connectionBreaker) {
             this.actionName = actionName;
             this.recoveryStarted = recoveryStarted;
-            this.finalizeReceived = finalizeReceived;
             this.connectionBreaker = connectionBreaker;
             this.blocksRemaining = new AtomicInteger(randomIntBetween(1, 3));
         }
@@ -605,9 +597,6 @@ public void messageReceived(
             Task task
         ) throws Exception {
             recoveryStarted.set(true);
-            if (actionName.equals(PeerRecoveryTargetService.Actions.FINALIZE)) {
-                finalizeReceived.set(true);
-            }
             if (blocksRemaining.getAndUpdate(i -> i == 0 ? 0 : i - 1) != 0) {
                 String rejected = "rejected";
                 String circuit = "circuit";
diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestCtxMetadata.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestCtxMetadata.java
index e69d292f2dc6..18c935e3931e 100644
--- a/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestCtxMetadata.java
+++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestCtxMetadata.java
@@ -15,7 +15,7 @@
 
 public class TestIngestCtxMetadata extends IngestDocMetadata {
     public TestIngestCtxMetadata(Map<String, Object> map, Map<String, FieldProperty<?>> properties) {
-        super(map, properties, null);
+        super(map, Map.copyOf(properties), null);
     }
 
     public static TestIngestCtxMetadata withNullableVersion(Map<String, Object> map) {
diff --git a/test/framework/src/main/java/org/elasticsearch/plugins/PluginTestUtil.java b/test/framework/src/main/java/org/elasticsearch/plugins/PluginTestUtil.java
index bd3e9dec5e84..7edb46d3e378 100644
--- a/test/framework/src/main/java/org/elasticsearch/plugins/PluginTestUtil.java
+++ b/test/framework/src/main/java/org/elasticsearch/plugins/PluginTestUtil.java
@@ -60,4 +60,9 @@ private static void writeProperties(Path propertiesFile, String... stringProps)
             properties.store(out, "");
         }
     }
+
+    public static void writeNamedComponentsFile(Path pluginDir, String namedComponentsJson) throws IOException {
+        Path namedComponentsFile = pluginDir.resolve(PluginDescriptor.NAMED_COMPONENTS_FILENAME);
+        Files.writeString(namedComponentsFile, namedComponentsJson);
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java
index 8304901864ba..52e4cf7b7ea8 100644
--- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java
@@ -423,6 +423,7 @@ private static ClusterService mockClusterService(ClusterState initialState) {
         final ThreadPool threadPool = mock(ThreadPool.class);
         when(threadPool.getThreadContext()).thenReturn(threadContext);
         when(threadPool.executor(ThreadPool.Names.SNAPSHOT)).thenReturn(new SameThreadExecutorService());
+        when(threadPool.executor(ThreadPool.Names.SNAPSHOT_META)).thenReturn(new SameThreadExecutorService());
         when(threadPool.generic()).thenReturn(new SameThreadExecutorService());
         when(threadPool.info(ThreadPool.Names.SNAPSHOT)).thenReturn(
             new ThreadPool.Info(ThreadPool.Names.SNAPSHOT, ThreadPool.ThreadPoolType.FIXED, randomIntBetween(1, 10))
diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java
index b4ee5b8fff49..fad23991729e 100644
--- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java
@@ -23,8 +23,8 @@
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.blobstore.support.BlobMetadata;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Streams;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.RepositoriesService;
diff --git a/test/framework/src/main/java/org/elasticsearch/rest/RestResponseUtils.java b/test/framework/src/main/java/org/elasticsearch/rest/RestResponseUtils.java
new file mode 100644
index 000000000000..cd0d54104419
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/rest/RestResponseUtils.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.rest;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+
+import static org.elasticsearch.transport.BytesRefRecycler.NON_RECYCLING_INSTANCE;
+
+public class RestResponseUtils {
+    private RestResponseUtils() {}
+
+    public static BytesReference getBodyContent(RestResponse restResponse) {
+        if (restResponse.isChunked() == false) {
+            return restResponse.content();
+        }
+
+        final var chunkedRestResponseBody = restResponse.chunkedContent();
+        assert chunkedRestResponseBody.isDone() == false;
+
+        final int pageSize;
+        try (var page = NON_RECYCLING_INSTANCE.obtain()) {
+            pageSize = page.v().length;
+        }
+
+        try (var out = new BytesStreamOutput()) {
+            while (chunkedRestResponseBody.isDone() == false) {
+                try (var chunk = chunkedRestResponseBody.encodeChunk(pageSize, NON_RECYCLING_INSTANCE)) {
+                    chunk.writeTo(out);
+                }
+            }
+
+            out.flush();
+            return out.bytes();
+        } catch (Exception e) {
+            throw new AssertionError("unexpected", e);
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
index ea0201145978..811d7ed1d6f7 100644
--- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
+++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
@@ -15,6 +15,7 @@
 import org.apache.lucene.search.Scorable;
 import org.apache.lucene.search.SortField;
 import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.index.mapper.OnScriptError;
 import org.elasticsearch.index.query.IntervalFilterScript;
 import org.elasticsearch.index.similarity.ScriptedSimilarity.Doc;
 import org.elasticsearch.index.similarity.ScriptedSimilarity.Field;
@@ -269,7 +270,13 @@ public String execute() {
             IntervalFilterScript.Factory factory = mockCompiled::createIntervalFilterScript;
             return context.factoryClazz.cast(factory);
         } else if (context.instanceClazz.equals(BooleanFieldScript.class)) {
-            BooleanFieldScript.Factory booleanFieldScript = (f, p, s) -> ctx -> new BooleanFieldScript(f, p, s, ctx) {
+            BooleanFieldScript.Factory booleanFieldScript = (f, p, s, onScriptError) -> ctx -> new BooleanFieldScript(
+                f,
+                p,
+                s,
+                OnScriptError.FAIL,
+                ctx
+            ) {
                 @Override
                 public void execute() {
                     emit(true);
@@ -277,7 +284,13 @@ public void execute() {
             };
             return context.factoryClazz.cast(booleanFieldScript);
         } else if (context.instanceClazz.equals(StringFieldScript.class)) {
-            StringFieldScript.Factory stringFieldScript = (f, p, s) -> ctx -> new StringFieldScript(f, p, s, ctx) {
+            StringFieldScript.Factory stringFieldScript = (f, p, s, onScriptError) -> ctx -> new StringFieldScript(
+                f,
+                p,
+                s,
+                OnScriptError.FAIL,
+                ctx
+            ) {
                 @Override
                 public void execute() {
                     emit("test");
@@ -285,7 +298,13 @@ public void execute() {
             };
             return context.factoryClazz.cast(stringFieldScript);
         } else if (context.instanceClazz.equals(LongFieldScript.class)) {
-            LongFieldScript.Factory longFieldScript = (f, p, s) -> ctx -> new LongFieldScript(f, p, s, ctx) {
+            LongFieldScript.Factory longFieldScript = (f, p, s, onScriptError) -> ctx -> new LongFieldScript(
+                f,
+                p,
+                s,
+                OnScriptError.FAIL,
+                ctx
+            ) {
                 @Override
                 public void execute() {
                     emit(1L);
@@ -293,7 +312,13 @@ public void execute() {
             };
             return context.factoryClazz.cast(longFieldScript);
         } else if (context.instanceClazz.equals(DoubleFieldScript.class)) {
-            DoubleFieldScript.Factory doubleFieldScript = (f, p, s) -> ctx -> new DoubleFieldScript(f, p, s, ctx) {
+            DoubleFieldScript.Factory doubleFieldScript = (f, p, s, onScriptError) -> ctx -> new DoubleFieldScript(
+                f,
+                p,
+                s,
+                OnScriptError.FAIL,
+                ctx
+            ) {
                 @Override
                 public void execute() {
                     emit(1.2D);
@@ -301,7 +326,14 @@ public void execute() {
             };
             return context.factoryClazz.cast(doubleFieldScript);
         } else if (context.instanceClazz.equals(DateFieldScript.class)) {
-            DateFieldScript.Factory dateFieldScript = (f, p, s, formatter) -> ctx -> new DateFieldScript(f, p, s, formatter, ctx) {
+            DateFieldScript.Factory dateFieldScript = (f, p, s, formatter, onScriptError) -> ctx -> new DateFieldScript(
+                f,
+                p,
+                s,
+                formatter,
+                OnScriptError.FAIL,
+                ctx
+            ) {
                 @Override
                 public void execute() {
                     emit(123L);
@@ -309,7 +341,7 @@ public void execute() {
             };
             return context.factoryClazz.cast(dateFieldScript);
         } else if (context.instanceClazz.equals(IpFieldScript.class)) {
-            IpFieldScript.Factory ipFieldScript = (f, p, s) -> ctx -> new IpFieldScript(f, p, s, ctx) {
+            IpFieldScript.Factory ipFieldScript = (f, p, s, onScriptError) -> ctx -> new IpFieldScript(f, p, s, OnScriptError.FAIL, ctx) {
                 @Override
                 public void execute() {
                     emit("127.0.0.1");
@@ -317,7 +349,13 @@ public void execute() {
             };
             return context.factoryClazz.cast(ipFieldScript);
         } else if (context.instanceClazz.equals(GeoPointFieldScript.class)) {
-            GeoPointFieldScript.Factory geoPointFieldScript = (f, p, s) -> ctx -> new GeoPointFieldScript(f, p, s, ctx) {
+            GeoPointFieldScript.Factory geoPointFieldScript = (f, p, s, onScriptError) -> ctx -> new GeoPointFieldScript(
+                f,
+                p,
+                s,
+                OnScriptError.FAIL,
+                ctx
+            ) {
                 @Override
                 public void execute() {
                     emit(1.2D, 1.2D);
@@ -325,7 +363,13 @@ public void execute() {
             };
             return context.factoryClazz.cast(geoPointFieldScript);
         } else if (context.instanceClazz.equals(CompositeFieldScript.class)) {
-            CompositeFieldScript.Factory objectFieldScript = (f, p, s) -> ctx -> new CompositeFieldScript(f, p, s, ctx) {
+            CompositeFieldScript.Factory objectFieldScript = (f, p, s, onScriptError) -> ctx -> new CompositeFieldScript(
+                f,
+                p,
+                s,
+                OnScriptError.FAIL,
+                ctx
+            ) {
                 @Override
                 public void execute() {
                     emit("field1", "value1");
diff --git a/test/framework/src/main/java/org/elasticsearch/search/DummyQueryBuilder.java b/test/framework/src/main/java/org/elasticsearch/search/DummyQueryBuilder.java
index ce609e1370bf..b936feebd490 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/DummyQueryBuilder.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/DummyQueryBuilder.java
@@ -9,7 +9,7 @@
 package org.elasticsearch.search;
 
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
@@ -66,7 +66,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java
index 2189603d8802..1931dd3dd573 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java
@@ -248,15 +248,20 @@ public static SearchSourceBuilder randomSearchSourceBuilder(
         }
 
         if (randomBoolean()) {
-            String field = randomAlphaOfLength(6);
-            int dim = randomIntBetween(2, 30);
-            float[] vector = new float[dim];
-            for (int i = 0; i < vector.length; i++) {
-                vector[i] = randomFloat();
+            int numKClauses = randomIntBetween(1, 5);
+            List<KnnSearchBuilder> knnSearchBuilders = new ArrayList<>(numKClauses);
+            for (int j = 0; j < numKClauses; j++) {
+                String field = randomAlphaOfLength(6);
+                int dim = randomIntBetween(2, 30);
+                float[] vector = new float[dim];
+                for (int i = 0; i < vector.length; i++) {
+                    vector[i] = randomFloat();
+                }
+                int k = randomIntBetween(1, 100);
+                int numCands = randomIntBetween(k, 1000);
+                knnSearchBuilders.add(new KnnSearchBuilder(field, vector, k, numCands));
             }
-            int k = randomIntBetween(1, 100);
-            int numCands = randomIntBetween(k, 1000);
-            builder.knnSearch(new KnnSearchBuilder(field, vector, k, numCands));
+            builder.knnSearch(knnSearchBuilders);
         }
 
         if (randomBoolean()) {
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
index b0befd0911cc..aaee330e3f38 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
@@ -42,6 +42,7 @@
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.NumericUtils;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.CheckedBiConsumer;
@@ -61,6 +62,7 @@
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
@@ -103,11 +105,10 @@
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.CrankyCircuitBreakerService;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.analysis.AnalysisModule;
-import org.elasticsearch.indices.breaker.AllCircuitBreakerStats;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.indices.breaker.CircuitBreakerStats;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SearchPlugin;
@@ -450,9 +451,6 @@ protected ScriptService getMockScriptService() {
      * Collects all documents that match the provided query {@link Query} and
      * returns the reduced {@link InternalAggregation}.
      * <p>
-     * Half the time it aggregates each leaf individually and reduces all
-     * results together. The other half the time it aggregates across the entire
-     * index at once and runs a final reduction on the single resulting agg.
      * It runs the aggregation as well using a circuit breaker that randomly throws {@link CircuitBreakingException}
      * in order to mak sure the implementation does not leak.
      */
@@ -463,41 +461,24 @@ protected <A extends InternalAggregation, C extends Aggregator> A searchAndReduc
         runWithCrankyCircuitBreaker(indexSettings, searcher, aggTestConfig);
         // Second run it to the end
         CircuitBreakerService breakerService = new NoneCircuitBreakerService();
-        return searchAndReduce(
-            indexSettings,
-            searcher,
-            aggTestConfig.query(),
-            aggTestConfig.builder(),
-            aggTestConfig.maxBuckets(),
-            aggTestConfig.splitLeavesIntoSeparateAggregators(),
-            aggTestConfig.shouldBeCached(),
-            breakerService,
-            aggTestConfig.fieldTypes()
-        );
+        return searchAndReduce(indexSettings, searcher, breakerService, aggTestConfig);
     }
 
     /**
-     * This is extracted into a seperate function so that stack traces will indicate if a bad allocation happened in the
-     * cranky CB run or the happy path run.
+     * Run an aggregation test against the {@link CrankyCircuitBreakerService}
+     * which fails randomly. This is extracted into a separate function so that
+     * stack traces will indicate if a bad allocation happened in the cranky CB
+     * run or the happy path run.
      */
     private void runWithCrankyCircuitBreaker(IndexSettings indexSettings, IndexSearcher searcher, AggTestConfig aggTestConfig)
         throws IOException {
         CircuitBreakerService crankyService = new CrankyCircuitBreakerService();
         for (int i = 0; i < 5; i++) {
             try {
-                searchAndReduce(
-                    indexSettings,
-                    searcher,
-                    aggTestConfig.query(),
-                    aggTestConfig.builder(),
-                    aggTestConfig.maxBuckets(),
-                    aggTestConfig.splitLeavesIntoSeparateAggregators(),
-                    aggTestConfig.shouldBeCached(),
-                    crankyService,
-                    aggTestConfig.fieldTypes()
-                );
+                searchAndReduce(indexSettings, searcher, crankyService, aggTestConfig);
             } catch (CircuitBreakingException e) {
-                // expected
+                // Circuit breaks from the cranky breaker are expected - it randomly fails, after all
+                assertThat(e.getMessage(), equalTo(CrankyCircuitBreakerService.ERROR_MESSAGE));
             } catch (IOException e) {
                 throw e;
             }
@@ -508,14 +489,16 @@ private void runWithCrankyCircuitBreaker(IndexSettings indexSettings, IndexSearc
     private <A extends InternalAggregation, C extends Aggregator> A searchAndReduce(
         IndexSettings indexSettings,
         IndexSearcher searcher,
-        Query query,
-        AggregationBuilder builder,
-        int maxBucket,
-        boolean splitLeavesIntoSeparateAggregators,
-        boolean shouldBeCached,
         CircuitBreakerService breakerService,
-        MappedFieldType... fieldTypes
+        AggTestConfig aggTestConfig
     ) throws IOException {
+        Query query = aggTestConfig.query();
+        AggregationBuilder builder = aggTestConfig.builder();
+        int maxBucket = aggTestConfig.maxBuckets();
+        boolean splitLeavesIntoSeparateAggregators = aggTestConfig.splitLeavesIntoSeparateAggregators();
+        boolean shouldBeCached = aggTestConfig.shouldBeCached();
+        MappedFieldType[] fieldTypes = aggTestConfig.fieldTypes();
+
         final IndexReaderContext ctx = searcher.getTopReaderContext();
         final PipelineTree pipelines = builder.buildPipelineTree();
         List<InternalAggregation> aggs = new ArrayList<>();
@@ -588,7 +571,7 @@ private <A extends InternalAggregation, C extends Aggregator> A searchAndReduce(
 
         BigArrays bigArraysForReduction = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), breakerService);
         try {
-            if (randomBoolean() && aggs.size() > 1) {
+            if (aggTestConfig.incrementalReduce() && aggs.size() > 1) {
                 // sometimes do an incremental reduce
                 int toReduceSize = aggs.size();
                 Collections.shuffle(aggs, random());
@@ -1199,14 +1182,14 @@ private void writeTestDoc(MappedFieldType fieldType, String fieldName, RandomInd
 
             final RangeFieldMapper.Range range = new RangeFieldMapper.Range(rangeType, start, end, true, true);
             doc.add(new BinaryDocValuesField(fieldName, rangeType.encodeRanges(Collections.singleton(range))));
-            json = formatted("""
+            json = Strings.format("""
                 { "%s" : { "gte" : "%s", "lte" : "%s" } }
                 """, fieldName, start, end);
         } else if (vst.equals(CoreValuesSourceType.GEOPOINT)) {
             double lat = randomDouble();
             double lon = randomDouble();
             doc.add(new LatLonDocValuesField(fieldName, lat, lon));
-            json = formatted("""
+            json = Strings.format("""
                 { "%s" : "[%s,%s]" }""", fieldName, lon, lat);
         } else {
             throw new IllegalStateException("Unknown field type [" + typeName + "]");
@@ -1218,7 +1201,7 @@ private void writeTestDoc(MappedFieldType fieldType, String fieldName, RandomInd
 
     private static class MockParserContext extends MappingParserContext {
         MockParserContext(IndexSettings indexSettings) {
-            super(null, null, null, Version.CURRENT, null, null, ScriptCompiler.NONE, null, indexSettings, null);
+            super(null, null, null, Version.CURRENT, null, ScriptCompiler.NONE, null, indexSettings, null);
         }
 
         @Override
@@ -1381,8 +1364,8 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.V_EMPTY;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.ZERO;
         }
     }
 
@@ -1450,105 +1433,78 @@ public List<AggregationSpec> getAggregations() {
         }
     }
 
-    private static class CrankyCircuitBreakerService extends CircuitBreakerService {
-
-        private final CircuitBreaker breaker = new CircuitBreaker() {
-            @Override
-            public void circuitBreak(String fieldName, long bytesNeeded) {
-
-            }
-
-            @Override
-            public void addEstimateBytesAndMaybeBreak(long bytes, String label) throws CircuitBreakingException {
-                if (random().nextInt(20) == 0) {
-                    throw new CircuitBreakingException("fake error", Durability.PERMANENT);
-                }
-            }
-
-            @Override
-            public void addWithoutBreaking(long bytes) {
-
-            }
-
-            @Override
-            public long getUsed() {
-                return 0;
-            }
-
-            @Override
-            public long getLimit() {
-                return 0;
-            }
-
-            @Override
-            public double getOverhead() {
-                return 0;
-            }
-
-            @Override
-            public long getTrippedCount() {
-                return 0;
-            }
-
-            @Override
-            public String getName() {
-                return CircuitBreaker.FIELDDATA;
-            }
-
-            @Override
-            public Durability getDurability() {
-                return null;
-            }
-
-            @Override
-            public void setLimitAndOverhead(long limit, double overhead) {
-
-            }
-        };
-
-        @Override
-        public CircuitBreaker getBreaker(String name) {
-            return breaker;
-        }
-
-        @Override
-        public AllCircuitBreakerStats stats() {
-            return new AllCircuitBreakerStats(new CircuitBreakerStats[] { stats(CircuitBreaker.FIELDDATA) });
-        }
-
-        @Override
-        public CircuitBreakerStats stats(String name) {
-            return new CircuitBreakerStats(CircuitBreaker.FIELDDATA, -1, -1, 0, 0);
-        }
-    }
-
     public record AggTestConfig(
         Query query,
         AggregationBuilder builder,
         int maxBuckets,
         boolean splitLeavesIntoSeparateAggregators,
         boolean shouldBeCached,
+        boolean incrementalReduce,
         MappedFieldType... fieldTypes
     ) {
 
         public AggTestConfig(AggregationBuilder builder, MappedFieldType... fieldTypes) {
-            this(new MatchAllDocsQuery(), builder, DEFAULT_MAX_BUCKETS, randomBoolean(), true, fieldTypes);
+            this(new MatchAllDocsQuery(), builder, DEFAULT_MAX_BUCKETS, randomBoolean(), true, randomBoolean(), fieldTypes);
         }
 
         public AggTestConfig withQuery(Query query) {
-            return new AggTestConfig(query, builder, maxBuckets, splitLeavesIntoSeparateAggregators, shouldBeCached, fieldTypes);
+            return new AggTestConfig(
+                query,
+                builder,
+                maxBuckets,
+                splitLeavesIntoSeparateAggregators,
+                shouldBeCached,
+                incrementalReduce,
+                fieldTypes
+            );
         }
 
         public AggTestConfig withSplitLeavesIntoSeperateAggregators(boolean splitLeavesIntoSeparateAggregators) {
-            return new AggTestConfig(query, builder, maxBuckets, splitLeavesIntoSeparateAggregators, shouldBeCached, fieldTypes);
+            return new AggTestConfig(
+                query,
+                builder,
+                maxBuckets,
+                splitLeavesIntoSeparateAggregators,
+                shouldBeCached,
+                incrementalReduce,
+                fieldTypes
+            );
         }
 
         public AggTestConfig withShouldBeCached(boolean shouldBeCached) {
-            return new AggTestConfig(query, builder, maxBuckets, splitLeavesIntoSeparateAggregators, shouldBeCached, fieldTypes);
+            return new AggTestConfig(
+                query,
+                builder,
+                maxBuckets,
+                splitLeavesIntoSeparateAggregators,
+                shouldBeCached,
+                incrementalReduce,
+                fieldTypes
+            );
         }
 
         public AggTestConfig withMaxBuckets(int maxBuckets) {
-            return new AggTestConfig(query, builder, maxBuckets, splitLeavesIntoSeparateAggregators, shouldBeCached, fieldTypes);
+            return new AggTestConfig(
+                query,
+                builder,
+                maxBuckets,
+                splitLeavesIntoSeparateAggregators,
+                shouldBeCached,
+                incrementalReduce,
+                fieldTypes
+            );
+        }
+
+        public AggTestConfig withIncrementalReduce(boolean incrementalReduce) {
+            return new AggTestConfig(
+                query,
+                builder,
+                maxBuckets,
+                splitLeavesIntoSeparateAggregators,
+                shouldBeCached,
+                incrementalReduce,
+                fieldTypes
+            );
         }
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/search/geo/BaseShapeQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/geo/BaseShapeQueryTestCase.java
index 2761e35da49e..b1319e59e851 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/geo/BaseShapeQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/geo/BaseShapeQueryTestCase.java
@@ -106,7 +106,7 @@ public void testShapeFetchingPath() throws Exception {
         String geo = """
             "geo" : {"type":"polygon", "coordinates":[[[-10,-10],[10,-10],[10,10],[-10,10],[-10,-10]]]}""";
 
-        client().prepareIndex("shapes").setId("1").setSource(formatted("""
+        client().prepareIndex("shapes").setId("1").setSource(Strings.format("""
             { %s, "1" : { %s, "2" : { %s, "3" : { %s } }} }
             """, geo, geo, geo, geo), XContentType.JSON).setRefreshPolicy(IMMEDIATE).get();
         client().prepareIndex(defaultIndexName)
diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
index 8b1ebce4ec46..2c3dee7dfd4f 100644
--- a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
@@ -700,7 +700,7 @@ protected List<String> createNSnapshots(String repoName, int count) throws Excep
 
     public static List<String> createNSnapshots(Logger logger, String repoName, int count) throws Exception {
         final PlainActionFuture<Collection<CreateSnapshotResponse>> allSnapshotsDone = PlainActionFuture.newFuture();
-        final ActionListener<CreateSnapshotResponse> snapshotsListener = new GroupedActionListener<>(allSnapshotsDone, count);
+        final ActionListener<CreateSnapshotResponse> snapshotsListener = new GroupedActionListener<>(count, allSnapshotsDone);
         final List<String> snapshotNames = new ArrayList<>(count);
         final String prefix = RANDOM_SNAPSHOT_NAME_PREFIX + UUIDs.randomBase64UUID(random()).toLowerCase(Locale.ROOT) + "-";
         for (int i = 0; i < count; i++) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
index 8fe0efbff8e4..3e7d3f046d4c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
@@ -465,7 +465,7 @@ public void onRemoval(ShardId shardId, Accountable accountable) {
                     MapperService.MergeReason.MAPPING_UPDATE
                 );
                 // also add mappings for two inner field in the object field
-                mapperService.merge("_doc", new CompressedXContent(formatted("""
+                mapperService.merge("_doc", new CompressedXContent(Strings.format("""
                     {
                       "properties": {
                         "%s": {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractChunkedSerializingTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractChunkedSerializingTestCase.java
index b8182e3c7ba5..4e26869b492b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractChunkedSerializingTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractChunkedSerializingTestCase.java
@@ -15,24 +15,22 @@
 import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
+import java.util.function.ToIntFunction;
 
 import static org.elasticsearch.test.AbstractXContentTestCase.chunkedXContentTester;
+import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS;
+import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 
 public abstract class AbstractChunkedSerializingTestCase<T extends ChunkedToXContent & Writeable> extends AbstractSerializationTestCase<T> {
 
     @Override
     protected AbstractXContentTestCase.XContentTester<T> createXContentTester() {
-        return chunkedXContentTester(
-            this::createParser,
-            xContentType -> createTestInstance(),
-            getToXContentParams(),
-            this::doParseInstance
-        );
+        return chunkedXContentTester(this::createParser, this::createXContextTestInstance, getToXContentParams(), this::doParseInstance);
     }
 
     @Override
     protected ToXContent asXContent(T instance) {
-        return ChunkedToXContent.wrapAsXContentObject(instance);
+        return ChunkedToXContent.wrapAsToXContent(instance);
     }
 
     @Override
@@ -44,4 +42,32 @@ protected T createXContextTestInstance(XContentType xContentType) {
      * Parses to a new instance using the provided {@link XContentParser}
      */
     protected abstract T doParseInstance(XContentParser parser) throws IOException;
+
+    public static <T extends ChunkedToXContent> void assertChunkCount(T instance, ToIntFunction<T> expectedChunkCount) {
+        assertChunkCount(instance, EMPTY_PARAMS, expectedChunkCount);
+    }
+
+    public static <T extends ChunkedToXContent> void assertChunkCount(
+        T instance,
+        ToXContent.Params params,
+        ToIntFunction<T> expectedChunkCount
+    ) {
+        int chunkCount = 0;
+        try (var builder = jsonBuilder()) {
+            if (instance.isFragment()) {
+                builder.startObject();
+            }
+            final var iterator = instance.toXContentChunked(params);
+            while (iterator.hasNext()) {
+                iterator.next().toXContent(builder, params);
+                chunkCount += 1;
+            }
+            if (instance.isFragment()) {
+                builder.endObject();
+            }
+        } catch (IOException e) {
+            throw new AssertionError("unexpected", e);
+        } // closing the builder verifies that the XContent is well-formed
+        assertEquals(expectedChunkCount.applyAsInt(instance), chunkCount);
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractWireTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractWireTestCase.java
index fe98c63de712..e410070cc2c4 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractWireTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractWireTestCase.java
@@ -23,7 +23,9 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
+import static org.hamcrest.Matchers.emptyString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
 
 /**
  * Standard test case for testing wire serialization. If the class being tested
@@ -44,10 +46,7 @@ public abstract class AbstractWireTestCase<T> extends ESTestCase {
      * Returns an instance which is mutated slightly so it should not be equal
      * to the given instance.
      */
-    // TODO: Make this abstract when all sub-classes implement this (https://github.com/elastic/elasticsearch/issues/25929)
-    protected T mutateInstance(T instance) throws IOException {
-        return null;
-    }
+    protected abstract T mutateInstance(T instance) throws IOException;
 
     /**
      * Tests that the equals and hashcode methods are consistent and copied
@@ -110,7 +109,7 @@ protected void concurrentTest(Runnable r) throws InterruptedException, Execution
      * some hashCode implementations violate this assumption and it's very
      * surprising. This tries to fail when that assumption is violated.
      */
-    public final void testConcurrentHashCode() throws IOException, InterruptedException, ExecutionException {
+    public final void testConcurrentHashCode() throws InterruptedException, ExecutionException {
         T testInstance = createTestInstance();
         int firstHashCode = testInstance.hashCode();
 
@@ -127,6 +126,12 @@ public final void testConcurrentHashCode() throws IOException, InterruptedExcept
         });
     }
 
+    public void testToString() throws Exception {
+        final String toString = createTestInstance().toString();
+        assertNotNull(toString);
+        assertThat(toString, not(emptyString()));
+    }
+
     /**
      * Test serialization and deserialization of the test instance.
      */
@@ -148,7 +153,7 @@ public final void testSerialization() throws IOException {
      * operations like {@link ToXContent#toXContent}. This doesn't
      * check that exactly, but it's close.
      */
-    public final void testConcurrentSerialization() throws IOException, InterruptedException, ExecutionException {
+    public final void testConcurrentSerialization() throws InterruptedException, ExecutionException {
         T testInstance = createTestInstance();
 
         /*
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java
index 157276b8ef10..5e9ed3e26d97 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java
@@ -91,10 +91,16 @@ public static <T extends ChunkedToXContent> XContentTester<T> chunkedXContentTes
     ) {
         return new XContentTester<>(createParser, instanceSupplier, (testInstance, xContentType) -> {
             try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
-                var serialization = testInstance.toXContentChunked();
+                var serialization = testInstance.toXContentChunked(toXContentParams);
+                if (testInstance.isFragment()) {
+                    builder.startObject();
+                }
                 while (serialization.hasNext()) {
                     serialization.next().toXContent(builder, toXContentParams);
                 }
+                if (testInstance.isFragment()) {
+                    builder.endObject();
+                }
                 return BytesReference.bytes(builder);
             }
         }, fromXContent);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AnnotationTestOrdering.java b/test/framework/src/main/java/org/elasticsearch/test/AnnotationTestOrdering.java
new file mode 100644
index 000000000000..49f5716b6c79
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/AnnotationTestOrdering.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test;
+
+import com.carrotsearch.randomizedtesting.TestMethodAndParams;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.util.Comparator;
+
+/**
+ * Test case ordering to be used in conjunction with {@link com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering}. Tests are
+ * ordered with respect to ordinals defined with {@link Order} annotations placed on individual test methods.
+ */
+public class AnnotationTestOrdering implements Comparator<TestMethodAndParams> {
+    @Override
+    public int compare(TestMethodAndParams o1, TestMethodAndParams o2) {
+        return Integer.compare(
+            o1.getTestMethod().getAnnotation(Order.class).value(),
+            o2.getTestMethod().getAnnotation(Order.class).value()
+        );
+    }
+
+    @Retention(RetentionPolicy.RUNTIME)
+    @Target(ElementType.METHOD)
+    @Inherited
+    public @interface Order {
+        int value();
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ChunkedToXContentDiffableSerializationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ChunkedToXContentDiffableSerializationTestCase.java
new file mode 100644
index 000000000000..d1724ad831bd
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/ChunkedToXContentDiffableSerializationTestCase.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test;
+
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.Diffable;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
+
+import java.io.IOException;
+
+/**
+ * An abstract test case to ensure correct behavior of Diffable.
+ *
+ * This class can be used as a based class for tests of {@link org.elasticsearch.cluster.ClusterState.Custom} classes and other classes
+ * that support, Writable serialization, chunked XContent-based serialization and are diffable.
+ */
+public abstract class ChunkedToXContentDiffableSerializationTestCase<T extends Diffable<T> & ChunkedToXContent> extends
+    AbstractChunkedSerializingTestCase<T> {
+
+    /**
+     *  Introduces random changes into the test object
+     */
+    protected abstract T makeTestChanges(T testInstance);
+
+    protected abstract Reader<Diff<T>> diffReader();
+
+    public final void testDiffableSerialization() throws IOException {
+        DiffableTestUtils.testDiffableSerialization(
+            this::createTestInstance,
+            this::makeTestChanges,
+            getNamedWriteableRegistry(),
+            instanceReader(),
+            diffReader()
+        );
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 1e1381929e4b..cffceafa4eb4 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -91,6 +91,7 @@
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Nullable;
@@ -842,7 +843,11 @@ public void waitNoPendingTasksOnAll() throws Exception {
                 ClusterHealthResponse clusterHealth = client.admin().cluster().prepareHealth().setLocal(true).get();
                 assertThat("client " + client + " still has in flight fetch", clusterHealth.getNumberOfInFlightFetch(), equalTo(0));
                 PendingClusterTasksResponse pendingTasks = client.admin().cluster().preparePendingClusterTasks().setLocal(true).get();
-                assertThat("client " + client + " still has pending tasks " + pendingTasks, pendingTasks, Matchers.emptyIterable());
+                assertThat(
+                    "client " + client + " still has pending tasks " + pendingTasks,
+                    pendingTasks.pendingTasks(),
+                    Matchers.emptyIterable()
+                );
                 clusterHealth = client.admin().cluster().prepareHealth().setLocal(true).get();
                 assertThat("client " + client + " still has in flight fetch", clusterHealth.getNumberOfInFlightFetch(), equalTo(0));
             }
@@ -1157,13 +1162,13 @@ protected void ensureClusterStateCanBeReadByNodeTool() throws IOException {
 
                 XContentBuilder builder = SmileXContent.contentBuilder();
                 builder.startObject();
-                metadataWithoutIndices.toXContent(builder, serializationFormatParams);
+                ChunkedToXContent.wrapAsToXContent(metadataWithoutIndices).toXContent(builder, serializationFormatParams);
                 builder.endObject();
                 final BytesReference originalBytes = BytesReference.bytes(builder);
 
                 XContentBuilder compareBuilder = SmileXContent.contentBuilder();
                 compareBuilder.startObject();
-                metadataWithoutIndices.toXContent(compareBuilder, compareFormatParams);
+                ChunkedToXContent.wrapAsToXContent(metadataWithoutIndices).toXContent(compareBuilder, compareFormatParams);
                 compareBuilder.endObject();
                 final BytesReference compareOriginalBytes = BytesReference.bytes(compareBuilder);
 
@@ -1179,7 +1184,7 @@ protected void ensureClusterStateCanBeReadByNodeTool() throws IOException {
                 }
                 builder = SmileXContent.contentBuilder();
                 builder.startObject();
-                loadedMetadata.toXContent(builder, compareFormatParams);
+                ChunkedToXContent.wrapAsToXContent(loadedMetadata).toXContent(builder, compareFormatParams);
                 builder.endObject();
                 final BytesReference parsedBytes = BytesReference.bytes(builder);
 
@@ -2051,6 +2056,13 @@ protected boolean addMockInternalEngine() {
         return true;
     }
 
+    /**
+     * Returns {@code true} if this test cluster can use the {@link MockFSIndexStore} test plugin. Defaults to true.
+     */
+    protected boolean addMockFSIndexStore() {
+        return true;
+    }
+
     /**
      * Returns a function that allows to wrap / filter all clients that are exposed by the test cluster. This is useful
      * for debugging or request / response pre and post processing. It also allows to intercept all calls done by the test
@@ -2067,7 +2079,7 @@ protected Collection<Class<? extends Plugin>> getMockPlugins() {
             if (randomBoolean() && addMockTransportService()) {
                 mocks.add(MockTransportService.TestPlugin.class);
             }
-            if (randomBoolean()) {
+            if (addMockFSIndexStore() && randomBoolean()) {
                 mocks.add(MockFSIndexStore.TestPlugin.class);
             }
             if (randomBoolean()) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 52e403c33dde..edd62f90757a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -39,6 +39,7 @@
 import org.apache.lucene.tests.util.TestUtil;
 import org.apache.lucene.tests.util.TimeUnits;
 import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.bootstrap.BootstrapForTesting;
 import org.elasticsearch.client.internal.Requests;
@@ -72,6 +73,7 @@
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.PathUtilsForTesting;
 import org.elasticsearch.core.RestApiVersion;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.env.Environment;
@@ -158,6 +160,9 @@
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
+import java.util.stream.DoubleStream;
+import java.util.stream.IntStream;
+import java.util.stream.LongStream;
 import java.util.stream.Stream;
 
 import static java.util.Collections.emptyMap;
@@ -802,6 +807,14 @@ public static int randomInt() {
         return random().nextInt();
     }
 
+    public static IntStream randomInts() {
+        return random().ints();
+    }
+
+    public static IntStream randomInts(long streamSize) {
+        return random().ints(streamSize);
+    }
+
     /**
      * @return a <code>long</code> between <code>0</code> and <code>Long.MAX_VALUE</code> (inclusive) chosen uniformly at random.
      */
@@ -818,6 +831,14 @@ public static double randomDouble() {
         return random().nextDouble();
     }
 
+    public static DoubleStream randomDoubles() {
+        return random().doubles();
+    }
+
+    public static DoubleStream randomDoubles(long streamSize) {
+        return random().doubles(streamSize);
+    }
+
     /**
      * Returns a double value in the interval [start, end) if lowerInclusive is
      * set to true, (start, end) otherwise.
@@ -851,6 +872,14 @@ public static long randomLong() {
         return random().nextLong();
     }
 
+    public static LongStream randomLongs() {
+        return random().longs();
+    }
+
+    public static LongStream randomLongs(long streamSize) {
+        return random().longs(streamSize);
+    }
+
     /**
      * Returns a random BigInteger uniformly distributed over the range 0 to (2^64 - 1) inclusive
      * Currently BigIntegers are only used for unsigned_long field type, where the max value is 2^64 - 1.
@@ -911,6 +940,13 @@ public static String randomAlphaOfLength(int codeUnits) {
         return RandomizedTest.randomAsciiOfLength(codeUnits);
     }
 
+    /**
+     * Creates a valid random identifier such as node id or index name
+     */
+    public static String randomIdentifier() {
+        return randomAlphaOfLengthBetween(8, 12).toLowerCase(Locale.ROOT);
+    }
+
     public static String randomUnicodeOfLengthBetween(int minCodeUnits, int maxCodeUnits) {
         return RandomizedTest.randomUnicodeOfLengthBetween(minCodeUnits, maxCodeUnits);
     }
@@ -1436,18 +1472,32 @@ public static <T extends Writeable> T copyWriteable(
         NamedWriteableRegistry namedWriteableRegistry,
         Writeable.Reader<T> reader
     ) throws IOException {
-        return copyWriteable(original, namedWriteableRegistry, reader, Version.CURRENT);
+        return copyWriteable(original, namedWriteableRegistry, reader, TransportVersion.CURRENT);
     }
 
     /**
      * Same as {@link #copyWriteable(Writeable, NamedWriteableRegistry, Writeable.Reader)} but also allows to provide
      * a {@link Version} argument which will be used to write and read back the object.
      */
+    @Deprecated
     public static <T extends Writeable> T copyWriteable(
         T original,
         NamedWriteableRegistry namedWriteableRegistry,
         Writeable.Reader<T> reader,
         Version version
+    ) throws IOException {
+        return copyWriteable(original, namedWriteableRegistry, reader, version.transportVersion);
+    }
+
+    /**
+     * Same as {@link #copyWriteable(Writeable, NamedWriteableRegistry, Writeable.Reader)} but also allows to provide
+     * a {@link TransportVersion} argument which will be used to write and read back the object.
+     */
+    public static <T extends Writeable> T copyWriteable(
+        T original,
+        NamedWriteableRegistry namedWriteableRegistry,
+        Writeable.Reader<T> reader,
+        TransportVersion version
     ) throws IOException {
         return copyInstance(original, namedWriteableRegistry, (out, value) -> value.writeTo(out), reader, version);
     }
@@ -1461,7 +1511,7 @@ public static <C extends NamedWriteable, T extends C> C copyNamedWriteable(
         NamedWriteableRegistry namedWriteableRegistry,
         Class<C> categoryClass
     ) throws IOException {
-        return copyNamedWriteable(original, namedWriteableRegistry, categoryClass, Version.CURRENT);
+        return copyNamedWriteable(original, namedWriteableRegistry, categoryClass, TransportVersion.CURRENT);
     }
 
     /**
@@ -1470,11 +1520,27 @@ public static <C extends NamedWriteable, T extends C> C copyNamedWriteable(
      * @return
      */
     @SuppressWarnings("unchecked")
+    @Deprecated
     public static <C extends NamedWriteable, T extends C> C copyNamedWriteable(
         T original,
         NamedWriteableRegistry namedWriteableRegistry,
         Class<C> categoryClass,
         Version version
+    ) throws IOException {
+        return copyNamedWriteable(original, namedWriteableRegistry, categoryClass, version.transportVersion);
+    }
+
+    /**
+     * Same as {@link #copyNamedWriteable(NamedWriteable, NamedWriteableRegistry, Class)} but also allows to provide
+     * a {@link TransportVersion} argument which will be used to write and read back the object.
+     * @return
+     */
+    @SuppressWarnings("unchecked")
+    public static <C extends NamedWriteable, T extends C> C copyNamedWriteable(
+        T original,
+        NamedWriteableRegistry namedWriteableRegistry,
+        Class<C> categoryClass,
+        TransportVersion version
     ) throws IOException {
         return copyInstance(
             original,
@@ -1485,18 +1551,29 @@ public static <C extends NamedWriteable, T extends C> C copyNamedWriteable(
         );
     }
 
+    @Deprecated
     protected static <T> T copyInstance(
         T original,
         NamedWriteableRegistry namedWriteableRegistry,
         Writeable.Writer<T> writer,
         Writeable.Reader<T> reader,
         Version version
+    ) throws IOException {
+        return copyInstance(original, namedWriteableRegistry, writer, reader, version.transportVersion);
+    }
+
+    protected static <T> T copyInstance(
+        T original,
+        NamedWriteableRegistry namedWriteableRegistry,
+        Writeable.Writer<T> writer,
+        Writeable.Reader<T> reader,
+        TransportVersion version
     ) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
-            output.setVersion(version);
+            output.setTransportVersion(version);
             writer.write(output, original);
             try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
-                in.setVersion(version);
+                in.setTransportVersion(version);
                 return reader.read(in);
             }
         }
@@ -1730,7 +1807,7 @@ public static boolean inFipsJvm() {
     /**
      * Defines the maximum port that test workers should use. See also [NOTE: Port ranges for tests].
      */
-    private static final int MAX_PRIVATE_PORT = 36600;
+    private static final int MAX_PRIVATE_PORT = 32767;
 
     /**
      * Wrap around after reaching this worker ID.
@@ -1812,14 +1889,10 @@ public boolean equals(Object o) {
 
         @Override
         public String toString() {
-            return String.format(Locale.ROOT, "%s: %s", level.name(), message);
+            return Strings.format("%s: %s", level.name(), message);
         }
     }
 
-    protected static String formatted(String string, Object... args) {
-        return String.format(Locale.ROOT, string, args);
-    }
-
     /**
      * Call method at the beginning of a test to disable its execution
      * until a given Lucene version is released and integrated into Elasticsearch
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
index 150eff8f84ee..1fc865001795 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
@@ -10,7 +10,7 @@
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.SetOnce;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -401,6 +401,13 @@ private void collectSubBuilderNames(Map<String, Object> names, InternalAggregati
 
     public record BuilderAndToReduce<T> (AggregationBuilder builder, List<T> toReduce) {}
 
+    /**
+     * Does this aggregation support reductions when the internal buckets are not in-order
+     */
+    protected boolean supportsOutOfOrderReduce() {
+        return true;
+    }
+
     public void testReduceRandom() throws IOException {
         String name = randomAlphaOfLength(5);
         int size = between(1, 200);
@@ -412,7 +419,9 @@ public void testReduceRandom() throws IOException {
         MockBigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
         if (randomBoolean() && toReduce.size() > 1) {
             // sometimes do a partial reduce
-            Collections.shuffle(toReduce, random());
+            if (supportsOutOfOrderReduce()) {
+                Collections.shuffle(toReduce, random());
+            }
             int r = randomIntBetween(1, toReduce.size());
             List<InternalAggregation> toPartialReduce = toReduce.subList(0, r);
             // Sort aggs so that unmapped come last. This mimicks the behavior of InternalAggregations.reduce()
@@ -721,8 +730,8 @@ public BucketCardinality bucketCardinality() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.V_EMPTY;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.ZERO;
         }
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index 18f21cb7368f..d89c3c8bf54c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -152,7 +152,6 @@
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.nullValue;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
@@ -603,6 +602,12 @@ private static Settings getRandomNodeSettings(long seed) {
             );
         }
 
+        if (random.nextBoolean()) {
+            // Nodes may fail to join a cluster within the default 30s timeout so we must sometimes skip that wait to ensure there's nothing
+            // in the startup process which relies on having joined a cluster.
+            builder.put(INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s");
+        }
+
         return builder.build();
     }
 
@@ -731,14 +736,6 @@ private Settings getNodeSettings(final int nodeId, final long seed, final Settin
         updatedSettings.put("node.name", name);
         updatedSettings.put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), seed);
 
-        if (autoManageMasterNodes) {
-            assertThat(
-                "if master nodes are automatically managed then nodes must complete a join cycle when starting",
-                updatedSettings.get(INITIAL_STATE_TIMEOUT_SETTING.getKey()),
-                nullValue()
-            );
-        }
-
         return updatedSettings.build();
     }
 
@@ -1778,6 +1775,10 @@ private void rebuildUnicastHostFiles(List<NodeAndClient> newNodes) {
         }
     }
 
+    public Collection<Path> configPaths() {
+        return nodes.values().stream().map(nac -> nac.node.getEnvironment().configFile()).toList();
+    }
+
     private void stopNodesAndClient(NodeAndClient nodeAndClient) throws IOException {
         stopNodesAndClients(Collections.singleton(nodeAndClient));
     }
@@ -2515,9 +2516,9 @@ public void assertRequestsFinished() {
                                 "All incoming requests on node ["
                                     + nodeAndClient.name
                                     + "] should have finished. "
-                                    + "Expected 0 but got "
+                                    + "Expected 0 bytes for requests in-flight but got "
                                     + bytesUsed
-                                    + "; pending tasks ["
+                                    + " bytes; pending tasks ["
                                     + pendingTasks
                                     + "]"
                             );
diff --git a/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java b/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java
index 5e9ef9ef212b..298dae9d3e32 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java
@@ -10,7 +10,7 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Setting;
@@ -88,7 +88,7 @@ public Collection<Object> createComponents(
             IndexNameExpressionResolver expressionResolver,
             Supplier<RepositoriesService> repositoriesServiceSupplier,
             Tracer tracer,
-            AllocationDeciders allocationDeciders
+            AllocationService allocationService
         ) {
             return Collections.singletonList(listener);
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/MockLogAppender.java b/test/framework/src/main/java/org/elasticsearch/test/MockLogAppender.java
index 5a40407eb8e7..6e13a9b184a1 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/MockLogAppender.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/MockLogAppender.java
@@ -8,17 +8,25 @@
 package org.elasticsearch.test;
 
 import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.core.LogEvent;
 import org.apache.logging.log4j.core.appender.AbstractAppender;
 import org.apache.logging.log4j.core.config.Property;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.core.Releasable;
 
+import java.util.Arrays;
 import java.util.List;
+import java.util.Objects;
 import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.regex.Pattern;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
 
 /**
  * Test appender that can be used to verify that certain events were logged correctly
@@ -27,7 +35,7 @@ public class MockLogAppender extends AbstractAppender {
 
     private static final String COMMON_PREFIX = System.getProperty("es.logger.prefix", "org.elasticsearch.");
 
-    private final List<LoggingExpectation> expectations;
+    private final List<WrappedLoggingExpectation> expectations;
 
     public MockLogAppender() {
         super("mock", null, null, false, Property.EMPTY_ARRAY);
@@ -40,7 +48,7 @@ public MockLogAppender() {
     }
 
     public void addExpectation(LoggingExpectation expectation) {
-        expectations.add(expectation);
+        expectations.add(new WrappedLoggingExpectation(expectation));
     }
 
     @Override
@@ -208,4 +216,58 @@ private static String getLoggerName(String name) {
         }
         return COMMON_PREFIX + name;
     }
+
+    /**
+     * A wrapper around {@link LoggingExpectation} to detect if the assertMatched method has been called
+     */
+    private static class WrappedLoggingExpectation implements LoggingExpectation {
+
+        private final AtomicBoolean assertMatchedCalled = new AtomicBoolean(false);
+        private final LoggingExpectation delegate;
+
+        private WrappedLoggingExpectation(LoggingExpectation delegate) {
+            this.delegate = Objects.requireNonNull(delegate);
+        }
+
+        @Override
+        public void match(LogEvent event) {
+            delegate.match(event);
+        }
+
+        @Override
+        public void assertMatched() {
+            try {
+                delegate.assertMatched();
+            } finally {
+                assertMatchedCalled.set(true);
+            }
+        }
+
+        @Override
+        public String toString() {
+            return delegate.toString();
+        }
+    }
+
+    public Releasable capturing(Class<?>... classes) {
+        start();
+        final var loggers = Arrays.stream(classes).map(LogManager::getLogger).toArray(Logger[]::new);
+        for (final var logger : loggers) {
+            Loggers.addAppender(logger, this);
+        }
+        return () -> {
+            for (final var logger : loggers) {
+                Loggers.removeAppender(logger, this);
+            }
+            stop();
+            // check that all expectations have been evaluated before this is released
+            for (WrappedLoggingExpectation expectation : expectations) {
+                assertThat(
+                    "Method assertMatched() not called on LoggingExpectation instance before release: " + expectation,
+                    expectation.assertMatchedCalled.get(),
+                    is(true)
+                );
+            }
+        };
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ReachabilityChecker.java b/test/framework/src/main/java/org/elasticsearch/test/ReachabilityChecker.java
new file mode 100644
index 000000000000..f3a5fdf83619
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/ReachabilityChecker.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test;
+
+import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.lang.ref.PhantomReference;
+import java.lang.ref.ReferenceQueue;
+import java.util.Objects;
+import java.util.Queue;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Utility class for checking that objects become unreachable when expected.
+ *
+ * Registered objects become unreachable only when the GC notices them. This class attempts to trigger the GC at various points but does not
+ * guarantee a full GC. If we're unlucky, some of these objects may move to old enough heap generations that they are not subject to our GC
+ * attempts, resulting in test flakiness. Time will tell whether this is a problem in practice or not; if it is then we'll have to introduce
+ * some extra measures here.
+ */
+public class ReachabilityChecker {
+
+    private final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
+    private final Queue<Registered> references = ConcurrentCollections.newQueue();
+
+    public ReachabilityChecker() {
+        memoryMXBean.gc();
+    }
+
+    /**
+     * Register the given target object for reachability checks.
+     *
+     * @return the given target object.
+     */
+    public <T> T register(T target) {
+        var referenceQueue = new ReferenceQueue<>();
+        references.add(
+            new Registered(target.toString(), new PhantomReference<>(Objects.requireNonNull(target), referenceQueue), referenceQueue)
+        );
+        return target;
+    }
+
+    /**
+     * Ensure that all registered objects have become unreachable.
+     */
+    public void ensureUnreachable() {
+        ensureUnreachable(TimeUnit.SECONDS.toMillis(10));
+    }
+
+    void ensureUnreachable(long timeoutMillis) {
+        Registered registered;
+        while ((registered = references.poll()) != null) {
+            registered.assertReferenceEnqueuedForCollection(memoryMXBean, timeoutMillis);
+        }
+    }
+
+    /**
+     * From the objects registered since the most recent call to {@link #ensureUnreachable()} (or since the construction of this {@link
+     * ReachabilityChecker} if {@link #ensureUnreachable()} has not been called) this method chooses one at random and verifies that it has
+     * not yet become unreachable.
+     */
+    public void checkReachable() {
+        if (references.peek() == null) {
+            throw new AssertionError("no references registered");
+        }
+
+        var target = Randomness.get().nextInt(references.size());
+        var iterator = references.iterator();
+        for (int i = 0; i < target; i++) {
+            assertTrue(iterator.hasNext());
+            assertNotNull(iterator.next());
+        }
+
+        assertTrue(iterator.hasNext());
+        iterator.next().assertReferenceNotEnqueuedForCollection(memoryMXBean);
+    }
+
+    private static final class Registered {
+        private final String description;
+        private final PhantomReference<?> phantomReference;
+        private final ReferenceQueue<?> referenceQueue;
+
+        Registered(String description, PhantomReference<?> phantomReference, ReferenceQueue<?> referenceQueue) {
+            this.description = description;
+            this.phantomReference = phantomReference;
+            this.referenceQueue = referenceQueue;
+        }
+
+        /**
+         * Attempts to trigger the GC repeatedly until the {@link ReferenceQueue} yields a reference.
+         */
+        public void assertReferenceEnqueuedForCollection(MemoryMXBean memoryMXBean, long timeoutMillis) {
+            try {
+                final var timeoutAt = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis);
+                while (true) {
+                    memoryMXBean.gc();
+                    final var ref = referenceQueue.remove(500);
+                    if (ref != null) {
+                        ref.clear();
+                        return;
+                    }
+                    assertTrue("still reachable: " + description, System.nanoTime() < timeoutAt);
+                    assertNull(phantomReference.get()); // always succeeds, we're just doing this to use the phantomReference for something
+                }
+            } catch (Exception e) {
+                throw new AssertionError("unexpected", e);
+            }
+        }
+
+        /**
+         * Attempts to trigger the GC and verifies that the {@link ReferenceQueue} does not yield a reference.
+         */
+        public void assertReferenceNotEnqueuedForCollection(MemoryMXBean memoryMXBean) {
+            try {
+                memoryMXBean.gc();
+                assertNull("became unreachable: " + description, referenceQueue.remove(100));
+            } catch (Exception e) {
+                throw new AssertionError("unexpected", e);
+            }
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TaskAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/TaskAssertions.java
index b811b33cff53..b4ecc36fc5b9 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TaskAssertions.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TaskAssertions.java
@@ -32,10 +32,18 @@ public class TaskAssertions {
     private TaskAssertions() {}
 
     public static void awaitTaskWithPrefix(String actionPrefix) throws Exception {
+        awaitTaskWithPrefix(actionPrefix, internalCluster().getInstances(TransportService.class));
+    }
+
+    public static void awaitTaskWithPrefixOnMaster(String actionPrefix) throws Exception {
+        awaitTaskWithPrefix(actionPrefix, List.of(internalCluster().getCurrentMasterNodeInstance(TransportService.class)));
+    }
+
+    private static void awaitTaskWithPrefix(String actionPrefix, Iterable<TransportService> transportServiceInstances) throws Exception {
         logger.info("--> waiting for task with prefix [{}] to start", actionPrefix);
 
         assertBusy(() -> {
-            for (TransportService transportService : internalCluster().getInstances(TransportService.class)) {
+            for (TransportService transportService : transportServiceInstances) {
                 List<Task> matchingTasks = transportService.getTaskManager()
                     .getTasks()
                     .values()
@@ -61,12 +69,13 @@ public static void assertAllCancellableTasksAreCancelled(String actionPrefix) th
                 assertTrue(taskManager.assertCancellableTaskConsistency());
                 for (CancellableTask cancellableTask : taskManager.getCancellableTasks().values()) {
                     if (cancellableTask.getAction().startsWith(actionPrefix)) {
-                        logger.trace("--> found task with prefix [{}] marked as cancelled: [{}]", actionPrefix, cancellableTask);
+                        logger.trace("--> found task with prefix [{}]: [{}]", actionPrefix, cancellableTask);
                         foundTask = true;
                         assertTrue(
                             "task " + cancellableTask.getId() + "/" + cancellableTask.getAction() + " not cancelled",
                             cancellableTask.isCancelled()
                         );
+                        logger.trace("--> Task with prefix [{}] is marked as cancelled: [{}]", actionPrefix, cancellableTask);
                     }
                 }
             }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetadata.java b/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetadata.java
index e617a3ffa40e..5a63bc135393 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetadata.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetadata.java
@@ -12,12 +12,14 @@
 import org.elasticsearch.cluster.AbstractNamedDiffable;
 import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.function.Function;
 
 public abstract class TestCustomMetadata extends AbstractNamedDiffable<Metadata.Custom> implements Metadata.Custom {
@@ -91,9 +93,8 @@ public static <T extends Metadata.Custom> T fromXContent(Function<String, T> sup
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field("data", getData());
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return Iterators.single((builder, params) -> builder.field("data", getData()));
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TransportVersionUtils.java b/test/framework/src/main/java/org/elasticsearch/test/TransportVersionUtils.java
new file mode 100644
index 000000000000..40f1d5f6e18b
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/TransportVersionUtils.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.Version;
+import org.elasticsearch.core.Nullable;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class TransportVersionUtils {
+    private static final List<TransportVersion> ALL_VERSIONS = List.copyOf(TransportVersion.getAllVersions());
+
+    /** Returns a random {@link TransportVersion} from all available versions. */
+    public static TransportVersion randomVersion() {
+        return ESTestCase.randomFrom(ALL_VERSIONS);
+    }
+
+    /** Returns a random {@link TransportVersion} from all available versions without the ignore set */
+    public static TransportVersion randomVersion(Set<TransportVersion> ignore) {
+        return ESTestCase.randomFrom(ALL_VERSIONS.stream().filter(v -> ignore.contains(v) == false).collect(Collectors.toList()));
+    }
+
+    /** Returns a random {@link Version} from all available versions. */
+    public static TransportVersion randomVersion(Random random) {
+        return ALL_VERSIONS.get(random.nextInt(ALL_VERSIONS.size()));
+    }
+
+    /** Returns a random {@link Version} between <code>minVersion</code> and <code>maxVersion</code> (inclusive). */
+    public static TransportVersion randomVersionBetween(
+        Random random,
+        @Nullable TransportVersion minVersion,
+        @Nullable TransportVersion maxVersion
+    ) {
+        int minVersionIndex = 0;
+        if (minVersion != null) {
+            minVersionIndex = ALL_VERSIONS.indexOf(minVersion);
+        }
+        int maxVersionIndex = ALL_VERSIONS.size() - 1;
+        if (maxVersion != null) {
+            maxVersionIndex = ALL_VERSIONS.indexOf(maxVersion);
+        }
+        if (minVersionIndex == -1) {
+            throw new IllegalArgumentException("minVersion [" + minVersion + "] does not exist.");
+        } else if (maxVersionIndex == -1) {
+            throw new IllegalArgumentException("maxVersion [" + maxVersion + "] does not exist.");
+        } else if (minVersionIndex > maxVersionIndex) {
+            throw new IllegalArgumentException("maxVersion [" + maxVersion + "] cannot be less than minVersion [" + minVersion + "]");
+        } else {
+            // minVersionIndex is inclusive so need to add 1 to this index
+            int range = maxVersionIndex + 1 - minVersionIndex;
+            return ALL_VERSIONS.get(minVersionIndex + random.nextInt(range));
+        }
+    }
+
+    public static TransportVersion getPreviousVersion() {
+        TransportVersion version = getPreviousVersion(TransportVersion.CURRENT);
+        assert version.before(TransportVersion.CURRENT);
+        return version;
+    }
+
+    public static TransportVersion getPreviousVersion(TransportVersion version) {
+        int place = Collections.binarySearch(ALL_VERSIONS, version);
+        if (place < 0) {
+            // version does not exist - need the item before the index this version should be inserted
+            place = -(place + 1);
+        }
+        if (place <= 1) {
+            throw new IllegalArgumentException("couldn't find any released versions before [" + version + "]");
+        }
+        return ALL_VERSIONS.get(place - 1);
+    }
+
+    /** Returns a random {@link Version} from all available versions, that is compatible with the given version. */
+    public static TransportVersion randomCompatibleVersion(Random random, TransportVersion version) {
+        final List<TransportVersion> compatible = ALL_VERSIONS.stream().filter(version::isCompatible).toList();
+        return compatible.get(random.nextInt(compatible.size()));
+    }
+
+    public static TransportVersion randomPreviousCompatibleVersion(Random random, TransportVersion version) {
+        return randomVersionBetween(random, version.calculateMinimumCompatVersion(), getPreviousVersion(version));
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java
index bcc4c64f015f..070f0cda8617 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java
@@ -9,6 +9,7 @@
 package org.elasticsearch.test;
 
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.rest.ObjectPath;
 import org.elasticsearch.xcontent.DeprecationHandler;
@@ -41,6 +42,10 @@ private XContentTestUtils() {
 
     }
 
+    public static Map<String, Object> convertToMap(ChunkedToXContent chunkedToXContent) throws IOException {
+        return convertToMap(ChunkedToXContent.wrapAsToXContent(chunkedToXContent));
+    }
+
     public static Map<String, Object> convertToMap(ToXContent part) throws IOException {
         XContentBuilder builder = XContentFactory.jsonBuilder();
         if (part.isFragment()) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
index d0b0d00fa47f..f4cd8533059e 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
@@ -27,7 +27,7 @@
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.BaseBroadcastResponse;
 import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.IsAcknowledgedSupplier;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -99,7 +99,7 @@ public static void assertNoTimeout(ClusterHealthResponse response) {
         assertThat("ClusterHealthResponse has timed out - returned: [" + response + "]", response.isTimedOut(), is(false));
     }
 
-    public static void assertAcked(AcknowledgedResponse response) {
+    public static void assertAcked(IsAcknowledgedSupplier response) {
         assertThat(response.getClass().getSimpleName() + " failed - not acked", response.isAcknowledged(), equalTo(true));
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index 297ea9beafff..1b2bf23dd63f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -20,7 +20,6 @@
 import org.apache.http.ssl.SSLContextBuilder;
 import org.apache.http.ssl.SSLContexts;
 import org.apache.http.util.EntityUtils;
-import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction;
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
@@ -57,7 +56,6 @@
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.seqno.ReplicationTracker;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.snapshots.SnapshotState;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.DeprecationHandler;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -98,7 +96,6 @@
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 import java.util.regex.Pattern;
@@ -109,7 +106,6 @@
 import static java.util.Collections.sort;
 import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.core.Strings.format;
-import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -606,6 +602,8 @@ protected Set<String> preserveILMPolicyIds() {
             "180-days-default",
             "365-days-default",
             ".fleet-actions-results-ilm-policy",
+            ".fleet-file-data-ilm-policy",
+            ".fleet-files-ilm-policy",
             ".deprecation-indexing-ilm-policy",
             ".monitoring-8-ilm-policy"
         );
@@ -638,14 +636,6 @@ protected boolean preserveSearchableSnapshotsIndicesUponCompletion() {
         return false;
     }
 
-    /**
-     * Returns whether to wait to make absolutely certain that all snapshots
-     * have been deleted.
-     */
-    protected boolean waitForAllSnapshotsWiped() {
-        return false;
-    }
-
     private void wipeCluster() throws Exception {
 
         // Cleanup rollup before deleting indices. A rollup job might have bulks in-flight,
@@ -666,24 +656,7 @@ private void wipeCluster() throws Exception {
             wipeSearchableSnapshotsIndices();
         }
 
-        SetOnce<Map<String, List<Map<?, ?>>>> inProgressSnapshots = new SetOnce<>();
-        if (waitForAllSnapshotsWiped()) {
-            AtomicReference<Map<String, List<Map<?, ?>>>> snapshots = new AtomicReference<>();
-            try {
-                // Repeatedly delete the snapshots until there aren't any
-                assertBusy(() -> {
-                    snapshots.set(wipeSnapshots());
-                    assertThat(snapshots.get(), anEmptyMap());
-                }, 2, TimeUnit.MINUTES);
-                // At this point there should be no snaphots
-                inProgressSnapshots.set(snapshots.get());
-            } catch (AssertionError e) {
-                // This will cause an error at the end of this method, but do the rest of the cleanup first
-                inProgressSnapshots.set(snapshots.get());
-            }
-        } else {
-            inProgressSnapshots.set(wipeSnapshots());
-        }
+        wipeSnapshots();
 
         // wipe data streams before indices so that the backing indices for data streams are handled properly
         if (preserveDataStreamsUponCompletion() == false) {
@@ -816,8 +789,6 @@ private void wipeCluster() throws Exception {
         }
 
         deleteAllNodeShutdownMetadata();
-
-        assertThat("Found in progress snapshots [" + inProgressSnapshots.get() + "].", inProgressSnapshots.get(), anEmptyMap());
     }
 
     /**
@@ -1007,37 +978,21 @@ protected void wipeSearchableSnapshotsIndices() throws IOException {
 
     /**
      * Wipe fs snapshots we created one by one and all repositories so that the next test can create the repositories fresh and they'll
-     * start empty. There isn't an API to delete all snapshots. There is an API to delete all snapshot repositories but that leaves all of
-     * the snapshots intact in the repository.
-     * @return Map of repository name to list of snapshots found in unfinished state
+     * start empty.
      */
-    protected Map<String, List<Map<?, ?>>> wipeSnapshots() throws IOException {
-        final Map<String, List<Map<?, ?>>> inProgressSnapshots = new HashMap<>();
+    protected void wipeSnapshots() throws IOException {
         for (Map.Entry<String, ?> repo : entityAsMap(adminClient.performRequest(new Request("GET", "/_snapshot/_all"))).entrySet()) {
             String repoName = repo.getKey();
             Map<?, ?> repoSpec = (Map<?, ?>) repo.getValue();
             String repoType = (String) repoSpec.get("type");
             if (false == preserveSnapshotsUponCompletion() && repoType.equals("fs")) {
                 // All other repo types we really don't have a chance of being able to iterate properly, sadly.
-                Request listRequest = new Request("GET", "/_snapshot/" + repoName + "/_all");
-                listRequest.addParameter("ignore_unavailable", "true");
-
-                List<?> snapshots = (List<?>) entityAsMap(adminClient.performRequest(listRequest)).get("snapshots");
-                for (Object snapshot : snapshots) {
-                    Map<?, ?> snapshotInfo = (Map<?, ?>) snapshot;
-                    String name = (String) snapshotInfo.get("snapshot");
-                    if (SnapshotState.valueOf((String) snapshotInfo.get("state")).completed() == false) {
-                        inProgressSnapshots.computeIfAbsent(repoName, key -> new ArrayList<>()).add(snapshotInfo);
-                    }
-                    logger.debug("wiping snapshot [{}/{}]", repoName, name);
-                    adminClient().performRequest(new Request("DELETE", "/_snapshot/" + repoName + "/" + name));
-                }
+                adminClient().performRequest(new Request("DELETE", "/_snapshot/" + repoName + "/*"));
             }
             if (preserveReposUponCompletion() == false) {
                 deleteRepository(repoName);
             }
         }
-        return inProgressSnapshots;
     }
 
     protected void deleteRepository(String repoName) throws IOException {
@@ -1821,6 +1776,9 @@ protected static boolean isXPackTemplate(String name) {
         if (name.startsWith(".deprecation-")) {
             return true;
         }
+        if (name.startsWith(".fleet-")) {
+            return true;
+        }
         switch (name) {
             case ".watches":
             case "security_audit_log":
diff --git a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java
index a8a5c668fc85..0e89df35eca7 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.tasks.RemovedTaskListener;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskAwareRequest;
 import org.elasticsearch.tasks.TaskManager;
@@ -78,18 +79,11 @@ public Task unregister(Task task) {
     }
 
     @Override
-    public void waitForTaskCompletion(Task task, long untilInNanos) {
+    public void registerRemovedTaskListener(RemovedTaskListener removedTaskListener) {
         for (MockTaskManagerListener listener : listeners) {
-            try {
-                listener.waitForTaskCompletion(task);
-            } catch (Exception e) {
-                logger.warn(
-                    () -> format("failed to notify task manager listener about waitForTaskCompletion the task with id %s", task.getId()),
-                    e
-                );
-            }
+            listener.subscribeForRemovedTasks(removedTaskListener);
         }
-        super.waitForTaskCompletion(task, untilInNanos);
+        super.registerRemovedTaskListener(removedTaskListener);
     }
 
     public void addListener(MockTaskManagerListener listener) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManagerListener.java b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManagerListener.java
index 7258cd8591bb..bda259a70073 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManagerListener.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManagerListener.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.test.tasks;
 
+import org.elasticsearch.tasks.RemovedTaskListener;
 import org.elasticsearch.tasks.Task;
 
 /**
@@ -19,4 +20,6 @@ public interface MockTaskManagerListener {
     void onTaskUnregistered(Task task);
 
     void waitForTaskCompletion(Task task);
+
+    default void subscribeForRemovedTasks(RemovedTaskListener removedTaskListener) {}
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/FakeTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/FakeTransport.java
index d17b7aa02107..78ec5e7575fd 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/FakeTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/FakeTransport.java
@@ -47,6 +47,11 @@ public BoundTransportAddress boundAddress() {
         return null;
     }
 
+    @Override
+    public BoundTransportAddress boundRemoteIngressAddress() {
+        return null;
+    }
+
     @Override
     public Map<String, BoundTransportAddress> profileBoundAddresses() {
         return null;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java
index 72aab46b452c..efab83f84a2c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java
@@ -89,8 +89,7 @@ public MockTransport() {
      */
     @SuppressWarnings("unchecked")
     public <Response extends TransportResponse> void handleResponse(final long requestId, final Response response) {
-        final TransportResponseHandler<Response> transportResponseHandler = (TransportResponseHandler<Response>) getResponseHandlers()
-            .onResponseReceived(requestId, listener);
+        final TransportResponseHandler<Response> transportResponseHandler = getTransportResponseHandler(requestId);
         if (transportResponseHandler != null) {
             final Response deliveredResponse;
             try (BytesStreamOutput output = new BytesStreamOutput()) {
@@ -100,8 +99,14 @@ public <Response extends TransportResponse> void handleResponse(final long reque
                 );
             } catch (IOException | UnsupportedOperationException e) {
                 throw new AssertionError("failed to serialize/deserialize response " + response, e);
+            } finally {
+                response.decRef();
+            }
+            try {
+                transportResponseHandler.handleResponse(deliveredResponse);
+            } finally {
+                deliveredResponse.decRef();
             }
-            transportResponseHandler.handleResponse(deliveredResponse);
         }
     }
 
@@ -154,12 +159,17 @@ public void handleRemoteError(final long requestId, final Throwable t) {
      * @param e         the failure
      */
     public void handleError(final long requestId, final TransportException e) {
-        final TransportResponseHandler<?> transportResponseHandler = getResponseHandlers().onResponseReceived(requestId, listener);
+        final TransportResponseHandler<?> transportResponseHandler = getTransportResponseHandler(requestId);
         if (transportResponseHandler != null) {
             transportResponseHandler.handleException(e);
         }
     }
 
+    @SuppressWarnings("unchecked")
+    public <T extends TransportResponse> TransportResponseHandler<T> getTransportResponseHandler(long requestId) {
+        return (TransportResponseHandler<T>) getResponseHandlers().onResponseReceived(requestId, listener);
+    }
+
     public Connection createConnection(DiscoveryNode node) {
         return new CloseableConnection() {
             @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
index 57acb1b9e79f..e7a68d30f2df 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
@@ -10,6 +10,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterModule;
@@ -106,7 +107,7 @@ public static MockTransportService createNewService(
     ) {
         return createNewService(
             settings,
-            newMockTransport(settings, version, threadPool),
+            newMockTransport(settings, version.transportVersion, threadPool),
             version,
             threadPool,
             clusterSettings,
@@ -114,7 +115,7 @@ public static MockTransportService createNewService(
         );
     }
 
-    public static TcpTransport newMockTransport(Settings settings, Version version, ThreadPool threadPool) {
+    public static TcpTransport newMockTransport(Settings settings, TransportVersion version, ThreadPool threadPool) {
         settings = Settings.builder().put(TransportSettings.PORT.getKey(), ESTestCase.getPortRange()).put(settings).build();
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables());
         return new Netty4Transport(
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java
index dd6b33139a0b..ceaf25164f01 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/StubbableTransport.java
@@ -8,6 +8,7 @@
 
 package org.elasticsearch.test.transport;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -147,6 +148,11 @@ public BoundTransportAddress boundAddress() {
         return delegate.boundAddress();
     }
 
+    @Override
+    public BoundTransportAddress boundRemoteIngressAddress() {
+        return delegate.boundRemoteIngressAddress();
+    }
+
     @Override
     public TransportAddress[] addressesFromString(String address) throws UnknownHostException {
         return delegate.addressesFromString(address);
@@ -263,6 +269,11 @@ public Version getVersion() {
             return connection.getVersion();
         }
 
+        @Override
+        public TransportVersion getTransportVersion() {
+            return connection.getTransportVersion();
+        }
+
         @Override
         public Object getCacheKey() {
             return connection.getCacheKey();
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
index a0c826dacbbe..a17b3168b8cc 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
@@ -15,6 +15,7 @@
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionListenerResponseHandler;
@@ -120,7 +121,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
     protected volatile DiscoveryNode nodeB;
     protected volatile MockTransportService serviceB;
 
-    protected abstract Transport build(Settings settings, Version version, ClusterSettings clusterSettings, boolean doHandshake);
+    protected abstract Transport build(Settings settings, TransportVersion version, ClusterSettings clusterSettings, boolean doHandshake);
 
     protected int channelsPerNodeConnection() {
         // This is a customized profile for this test case.
@@ -206,7 +207,7 @@ private MockTransportService buildService(
         if (clusterSettings == null) {
             clusterSettings = new ClusterSettings(updatedSettings, getSupportedSettings());
         }
-        Transport transport = build(updatedSettings, version, clusterSettings, doHandshake);
+        Transport transport = build(updatedSettings, version.transportVersion, clusterSettings, doHandshake);
         MockTransportService service = MockTransportService.createNewService(
             updatedSettings,
             transport,
@@ -1466,7 +1467,7 @@ public static class Version1Request extends Version0Request {
 
         Version1Request(StreamInput in) throws IOException {
             super(in);
-            if (in.getVersion().onOrAfter(version1)) {
+            if (in.getTransportVersion().onOrAfter(version1.transportVersion)) {
                 value2 = in.readInt();
             }
         }
@@ -1474,7 +1475,7 @@ public static class Version1Request extends Version0Request {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            if (out.getVersion().onOrAfter(version1)) {
+            if (out.getTransportVersion().onOrAfter(version1.transportVersion)) {
                 out.writeInt(value2);
             }
         }
@@ -1509,7 +1510,7 @@ static class Version1Response extends Version0Response {
 
         Version1Response(StreamInput in) throws IOException {
             super(in);
-            if (in.getVersion().onOrAfter(version1)) {
+            if (in.getTransportVersion().onOrAfter(version1.transportVersion)) {
                 value2 = in.readInt();
             } else {
                 value2 = 0;
@@ -1519,7 +1520,7 @@ static class Version1Response extends Version0Response {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            if (out.getVersion().onOrAfter(version1)) {
+            if (out.getTransportVersion().onOrAfter(version1.transportVersion)) {
                 out.writeInt(value2);
             }
         }
@@ -1531,7 +1532,7 @@ public void testVersionFrom0to1() throws Exception {
             assertThat(request.value2, equalTo(0)); // not set, coming from service A
             Version1Response response = new Version1Response(1, 2);
             channel.sendResponse(response);
-            assertEquals(version0, channel.getVersion());
+            assertEquals(version0.transportVersion, channel.getVersion());
         });
 
         Version0Request version0Request = new Version0Request();
@@ -1568,7 +1569,7 @@ public void testVersionFrom1to0() throws Exception {
             assertThat(request.value1, equalTo(1));
             Version0Response response = new Version0Response(1);
             channel.sendResponse(response);
-            assertEquals(version0, channel.getVersion());
+            assertEquals(version0.transportVersion, channel.getVersion());
         });
 
         Version1Request version1Request = new Version1Request();
@@ -1609,7 +1610,7 @@ public void testVersionFrom1to1() throws Exception {
             assertThat(request.value2, equalTo(2));
             Version1Response response = new Version1Response(1, 2);
             channel.sendResponse(response);
-            assertEquals(version1, channel.getVersion());
+            assertEquals(version1.transportVersion, channel.getVersion());
         });
 
         Version1Request version1Request = new Version1Request();
@@ -1649,7 +1650,7 @@ public void testVersionFrom0to0() throws Exception {
             assertThat(request.value1, equalTo(1));
             Version0Response response = new Version0Response(1);
             channel.sendResponse(response);
-            assertEquals(version0, channel.getVersion());
+            assertEquals(version0.transportVersion, channel.getVersion());
         });
 
         Version0Request version0Request = new Version0Request();
@@ -2258,7 +2259,7 @@ public void testHandshakeUpdatesVersion() throws IOException {
                 TransportRequestOptions.Type.STATE
             );
             try (Transport.Connection connection = openConnection(serviceA, node, builder.build())) {
-                assertEquals(connection.getVersion(), version);
+                assertEquals(version.transportVersion, connection.getTransportVersion());
             }
         }
     }
@@ -2303,7 +2304,7 @@ public void testTcpHandshake() {
             PlainActionFuture<Transport.Connection> future = PlainActionFuture.newFuture();
             serviceA.getOriginalTransport().openConnection(node, connectionProfile, future);
             try (Transport.Connection connection = future.actionGet()) {
-                assertEquals(connection.getVersion(), Version.CURRENT);
+                assertEquals(TransportVersion.CURRENT, connection.getTransportVersion());
             }
         }
     }
@@ -2375,7 +2376,7 @@ public void run() {
                 ConnectTransportException.class,
                 () -> connectToNode(serviceA, dummy, builder.build())
             );
-            assertEquals(ex.getMessage(), "[][" + dummy.getAddress() + "] general node connection failure");
+            assertEquals("[][" + dummy.getAddress() + "] general node connection failure", ex.getMessage());
             assertThat(ex.getCause().getMessage(), startsWith("handshake failed"));
             t.join();
         }
@@ -2761,7 +2762,7 @@ public void handleException(TransportException exp) {
             TransportException exception = receivedException.get();
             assertNotNull(exception);
             BytesStreamOutput streamOutput = new BytesStreamOutput();
-            streamOutput.setVersion(version0);
+            streamOutput.setTransportVersion(version0.transportVersion);
             exception.writeTo(streamOutput);
             String failedMessage = "Unexpected read bytes size. The transport exception that was received=" + exception;
             // 53 bytes are the non-exception message bytes that have been received. It should include the initial
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java b/test/framework/src/main/java/org/elasticsearch/transport/DisruptableMockTransport.java
similarity index 82%
rename from test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java
rename to test/framework/src/main/java/org/elasticsearch/transport/DisruptableMockTransport.java
index e939cbda3eca..6c5abb9a8de2 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/DisruptableMockTransport.java
@@ -5,11 +5,13 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-package org.elasticsearch.test.disruption;
+package org.elasticsearch.transport;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.coordination.CleanableResponseHandler;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -21,19 +23,6 @@
 import org.elasticsearch.test.transport.MockTransport;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.tracing.Tracer;
-import org.elasticsearch.transport.CloseableConnection;
-import org.elasticsearch.transport.ConnectTransportException;
-import org.elasticsearch.transport.ConnectionProfile;
-import org.elasticsearch.transport.NodeNotConnectedException;
-import org.elasticsearch.transport.RemoteTransportException;
-import org.elasticsearch.transport.RequestHandlerRegistry;
-import org.elasticsearch.transport.TransportChannel;
-import org.elasticsearch.transport.TransportException;
-import org.elasticsearch.transport.TransportInterceptor;
-import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.transport.TransportRequestOptions;
-import org.elasticsearch.transport.TransportResponse;
-import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -48,14 +37,13 @@
 
 public abstract class DisruptableMockTransport extends MockTransport {
     private final DiscoveryNode localNode;
-    private final Logger logger;
+    private final Logger logger = LogManager.getLogger(DisruptableMockTransport.class);
     private final DeterministicTaskQueue deterministicTaskQueue;
     private final List<Runnable> blackholedRequests = new ArrayList<>();
     private final Set<String> blockedActions = new HashSet<>();
 
-    public DisruptableMockTransport(DiscoveryNode localNode, Logger logger, DeterministicTaskQueue deterministicTaskQueue) {
+    public DisruptableMockTransport(DiscoveryNode localNode, DeterministicTaskQueue deterministicTaskQueue) {
         this.localNode = localNode;
-        this.logger = logger;
         this.deterministicTaskQueue = deterministicTaskQueue;
     }
 
@@ -111,7 +99,12 @@ public DiscoveryNode getNode() {
                     public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
                         throws TransportException {
                         if (blockedActions.contains(action)) {
-                            execute(new Runnable() {
+                            execute(new RebootSensitiveRunnable() {
+                                @Override
+                                public void ifRebooted() {
+                                    cleanupResponseHandler(requestId);
+                                }
+
                                 @Override
                                 public void run() {
                                     handleError(
@@ -127,7 +120,7 @@ public void run() {
 
                                 @Override
                                 public String toString() {
-                                    return "error response delivery for action [" + action + "] on node [" + node + "]";
+                                    return "error response delivery for blocked action [" + action + "] on node [" + node + "]";
                                 }
                             });
                         } else {
@@ -172,28 +165,20 @@ public void run() {
             @Override
             public void ifRebooted() {
                 request.decRef();
-                deterministicTaskQueue.scheduleNow(new Runnable() {
+                execute(new RebootSensitiveRunnable() {
                     @Override
-                    public void run() {
-                        execute(new Runnable() {
-                            @Override
-                            public void run() {
-                                handleRemoteError(
-                                    requestId,
-                                    new NodeNotConnectedException(destinationTransport.getLocalNode(), "node rebooted")
-                                );
-                            }
+                    public void ifRebooted() {
+                        cleanupResponseHandler(requestId);
+                    }
 
-                            @Override
-                            public String toString() {
-                                return "error response (reboot) to " + internalToString();
-                            }
-                        });
+                    @Override
+                    public void run() {
+                        handleRemoteError(requestId, new NodeNotConnectedException(destinationTransport.getLocalNode(), "node rebooted"));
                     }
 
                     @Override
                     public String toString() {
-                        return "scheduling of error response (reboot) to " + internalToString();
+                        return "error response (reboot) to " + internalToString();
                     }
                 });
             }
@@ -209,8 +194,13 @@ private String internalToString() {
         });
     }
 
-    protected Runnable getDisconnectException(long requestId, String action, DiscoveryNode destination) {
-        return new Runnable() {
+    private Runnable getDisconnectException(long requestId, String action, DiscoveryNode destination) {
+        return new RebootSensitiveRunnable() {
+            @Override
+            public void ifRebooted() {
+                cleanupResponseHandler(requestId);
+            }
+
             @Override
             public void run() {
                 handleError(requestId, new ConnectTransportException(destination, "disconnected"));
@@ -223,11 +213,11 @@ public String toString() {
         };
     }
 
-    protected String getRequestDescription(long requestId, String action, DiscoveryNode destination) {
+    private String getRequestDescription(long requestId, String action, DiscoveryNode destination) {
         return format("[%s][%s] from %s to %s", requestId, action, getLocalNode(), destination);
     }
 
-    protected void onBlackholedDuringSend(long requestId, String action, DisruptableMockTransport destinationTransport) {
+    private void onBlackholedDuringSend(long requestId, String action, DisruptableMockTransport destinationTransport) {
         logger.trace("dropping {}", getRequestDescription(requestId, action, destinationTransport.getLocalNode()));
         // Delaying the response until explicitly instructed, to simulate a very long delay
         blackholedRequests.add(new Runnable() {
@@ -243,11 +233,11 @@ public String toString() {
         });
     }
 
-    protected void onDisconnectedDuringSend(long requestId, String action, DisruptableMockTransport destinationTransport) {
-        destinationTransport.execute(getDisconnectException(requestId, action, destinationTransport.getLocalNode()));
+    private void onDisconnectedDuringSend(long requestId, String action, DisruptableMockTransport destinationTransport) {
+        execute(getDisconnectException(requestId, action, destinationTransport.getLocalNode()));
     }
 
-    protected void onConnectedDuringSend(
+    private void onConnectedDuringSend(
         long requestId,
         String action,
         TransportRequest request,
@@ -272,13 +262,20 @@ public String getChannelType() {
 
             @Override
             public void sendResponse(final TransportResponse response) {
-                execute(new Runnable() {
+                execute(new RebootSensitiveRunnable() {
+                    @Override
+                    public void ifRebooted() {
+                        response.decRef();
+                        cleanupResponseHandler(requestId);
+                    }
+
                     @Override
                     public void run() {
                         final ConnectionStatus connectionStatus = destinationTransport.getConnectionStatus(getLocalNode());
                         switch (connectionStatus) {
                             case CONNECTED, BLACK_HOLE_REQUESTS_ONLY -> handleResponse(requestId, response);
                             case BLACK_HOLE, DISCONNECTED -> {
+                                response.decRef();
                                 logger.trace("delaying response to {}: channel is {}", requestDescription, connectionStatus);
                                 onBlackholedDuringSend(requestId, action, destinationTransport);
                             }
@@ -295,8 +292,12 @@ public String toString() {
 
             @Override
             public void sendResponse(Exception exception) {
+                execute(new RebootSensitiveRunnable() {
+                    @Override
+                    public void ifRebooted() {
+                        cleanupResponseHandler(requestId);
+                    }
 
-                execute(new Runnable() {
                     @Override
                     public void run() {
                         final ConnectionStatus connectionStatus = destinationTransport.getConnectionStatus(getLocalNode());
@@ -333,6 +334,18 @@ public String toString() {
             } catch (Exception ee) {
                 logger.warn("failed to send failure", e);
             }
+        } finally {
+            copiedRequest.decRef();
+        }
+    }
+
+    private void cleanupResponseHandler(long requestId) {
+        TransportResponseHandler<?> handler = getTransportResponseHandler(requestId);
+        while (handler instanceof TransportService.ContextRestoreResponseHandler<?> contextRestoreHandler) {
+            handler = contextRestoreHandler.unwrap();
+        }
+        if (handler instanceof CleanableResponseHandler<?> cleanableResponseHandler) {
+            cleanableResponseHandler.runCleanup();
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/TestTransportChannels.java b/test/framework/src/main/java/org/elasticsearch/transport/TestTransportChannels.java
index b0b17b714630..607232fc9b88 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/TestTransportChannels.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/TestTransportChannels.java
@@ -8,7 +8,7 @@
 
 package org.elasticsearch.transport;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.network.HandlingTimeTracker;
 import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -21,7 +21,7 @@ public static TcpTransportChannel newFakeTcpTransportChannel(
         ThreadPool threadPool,
         String action,
         long requestId,
-        Version version
+        TransportVersion version
     ) {
         BytesRefRecycler recycler = new BytesRefRecycler(PageCacheRecycler.NON_RECYCLING_INSTANCE);
         return new TcpTransportChannel(
diff --git a/test/framework/src/test/java/org/elasticsearch/test/ReachabilityCheckerTests.java b/test/framework/src/test/java/org/elasticsearch/test/ReachabilityCheckerTests.java
new file mode 100644
index 000000000000..d741527b969c
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/test/ReachabilityCheckerTests.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test;
+
+import org.hamcrest.Matchers;
+
+public class ReachabilityCheckerTests extends ESTestCase {
+
+    public void testSuccess() {
+        final var reachabilityChecker = new ReachabilityChecker();
+        var target = reachabilityChecker.register(createTarget());
+        reachabilityChecker.checkReachable();
+        target = null;
+        reachabilityChecker.ensureUnreachable();
+        assertNull(target);
+    }
+
+    public void testBecomesUnreachable() {
+        final var reachabilityChecker = new ReachabilityChecker();
+        var target = reachabilityChecker.register(createTarget());
+        reachabilityChecker.checkReachable();
+        target = null;
+        assertThat(
+            expectThrows(AssertionError.class, reachabilityChecker::checkReachable).getMessage(),
+            Matchers.startsWith("became unreachable: test object")
+        );
+        assertNull(target);
+    }
+
+    public void testStaysReachable() {
+        final var reachabilityChecker = new ReachabilityChecker();
+        var target = reachabilityChecker.register(createTarget());
+        reachabilityChecker.checkReachable();
+        assertThat(
+            expectThrows(AssertionError.class, () -> reachabilityChecker.ensureUnreachable(500)).getMessage(),
+            Matchers.startsWith("still reachable: test object")
+        );
+        assertNotNull(target);
+    }
+
+    private static Object createTarget() {
+        return new Object() {
+            @Override
+            public String toString() {
+                return "test object";
+            }
+        };
+    }
+}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/disruption/DisruptableMockTransportTests.java b/test/framework/src/test/java/org/elasticsearch/test/disruption/DisruptableMockTransportTests.java
deleted file mode 100644
index b3fafa729856..000000000000
--- a/test/framework/src/test/java/org/elasticsearch/test/disruption/DisruptableMockTransportTests.java
+++ /dev/null
@@ -1,443 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.test.disruption;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
-import org.elasticsearch.core.Releasable;
-import org.elasticsearch.core.Tuple;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.disruption.DisruptableMockTransport.ConnectionStatus;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.AbstractSimpleTransportTestCase;
-import org.elasticsearch.transport.ConnectTransportException;
-import org.elasticsearch.transport.TransportChannel;
-import org.elasticsearch.transport.TransportException;
-import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.transport.TransportRequestHandler;
-import org.elasticsearch.transport.TransportResponse;
-import org.elasticsearch.transport.TransportResponseHandler;
-import org.elasticsearch.transport.TransportService;
-import org.junit.Before;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Consumer;
-
-import static org.elasticsearch.transport.TransportService.NOOP_TRANSPORT_INTERCEPTOR;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.endsWith;
-import static org.hamcrest.Matchers.instanceOf;
-
-public class DisruptableMockTransportTests extends ESTestCase {
-
-    private DiscoveryNode node1;
-    private DiscoveryNode node2;
-
-    private TransportService service1;
-    private TransportService service2;
-
-    private DeterministicTaskQueue deterministicTaskQueue;
-
-    private Runnable deliverBlackholedRequests;
-
-    private Set<Tuple<DiscoveryNode, DiscoveryNode>> disconnectedLinks;
-    private Set<Tuple<DiscoveryNode, DiscoveryNode>> blackholedLinks;
-    private Set<Tuple<DiscoveryNode, DiscoveryNode>> blackholedRequestLinks;
-
-    private ConnectionStatus getConnectionStatus(DiscoveryNode sender, DiscoveryNode destination) {
-        Tuple<DiscoveryNode, DiscoveryNode> link = Tuple.tuple(sender, destination);
-        if (disconnectedLinks.contains(link)) {
-            assert blackholedLinks.contains(link) == false;
-            assert blackholedRequestLinks.contains(link) == false;
-            return ConnectionStatus.DISCONNECTED;
-        }
-        if (blackholedLinks.contains(link)) {
-            assert blackholedRequestLinks.contains(link) == false;
-            return ConnectionStatus.BLACK_HOLE;
-        }
-        if (blackholedRequestLinks.contains(link)) {
-            return ConnectionStatus.BLACK_HOLE_REQUESTS_ONLY;
-        }
-        return ConnectionStatus.CONNECTED;
-    }
-
-    @Before
-    public void initTransports() {
-        node1 = new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT);
-        node2 = new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT);
-
-        disconnectedLinks = new HashSet<>();
-        blackholedLinks = new HashSet<>();
-        blackholedRequestLinks = new HashSet<>();
-
-        List<DisruptableMockTransport> transports = new ArrayList<>();
-
-        deterministicTaskQueue = new DeterministicTaskQueue();
-
-        final DisruptableMockTransport transport1 = new DisruptableMockTransport(node1, logger, deterministicTaskQueue) {
-            @Override
-            protected ConnectionStatus getConnectionStatus(DiscoveryNode destination) {
-                return DisruptableMockTransportTests.this.getConnectionStatus(getLocalNode(), destination);
-            }
-
-            @Override
-            protected Optional<DisruptableMockTransport> getDisruptableMockTransport(TransportAddress address) {
-                return transports.stream().filter(t -> t.getLocalNode().getAddress().equals(address)).findAny();
-            }
-
-            @Override
-            protected void execute(Runnable runnable) {
-                deterministicTaskQueue.scheduleNow(runnable);
-            }
-        };
-
-        final DisruptableMockTransport transport2 = new DisruptableMockTransport(node2, logger, deterministicTaskQueue) {
-            @Override
-            protected ConnectionStatus getConnectionStatus(DiscoveryNode destination) {
-                return DisruptableMockTransportTests.this.getConnectionStatus(getLocalNode(), destination);
-            }
-
-            @Override
-            protected Optional<DisruptableMockTransport> getDisruptableMockTransport(TransportAddress address) {
-                return transports.stream().filter(t -> t.getLocalNode().getAddress().equals(address)).findAny();
-            }
-
-            @Override
-            protected void execute(Runnable runnable) {
-                deterministicTaskQueue.scheduleNow(runnable);
-            }
-        };
-
-        transports.add(transport1);
-        transports.add(transport2);
-
-        service1 = transport1.createTransportService(
-            Settings.EMPTY,
-            deterministicTaskQueue.getThreadPool(),
-            NOOP_TRANSPORT_INTERCEPTOR,
-            a -> node1,
-            null,
-            Collections.emptySet()
-        );
-        service2 = transport2.createTransportService(
-            Settings.EMPTY,
-            deterministicTaskQueue.getThreadPool(),
-            NOOP_TRANSPORT_INTERCEPTOR,
-            a -> node2,
-            null,
-            Collections.emptySet()
-        );
-
-        service1.start();
-        service2.start();
-
-        final PlainActionFuture<Releasable> fut1 = new PlainActionFuture<>();
-        service1.connectToNode(node2, fut1);
-        final PlainActionFuture<Releasable> fut2 = new PlainActionFuture<>();
-        service2.connectToNode(node1, fut2);
-        deterministicTaskQueue.runAllTasksInTimeOrder();
-        assertTrue(fut1.isDone());
-        assertTrue(fut2.isDone());
-
-        deliverBlackholedRequests = () -> transports.forEach(DisruptableMockTransport::deliverBlackholedRequests);
-    }
-
-    private TransportRequestHandler<TransportRequest.Empty> requestHandlerShouldNotBeCalled() {
-        return (request, channel, task) -> { throw new AssertionError("should not be called"); };
-    }
-
-    private TransportRequestHandler<TransportRequest.Empty> requestHandlerRepliesNormally() {
-        return (request, channel, task) -> {
-            logger.debug("got a dummy request, replying normally...");
-            channel.sendResponse(TransportResponse.Empty.INSTANCE);
-        };
-    }
-
-    private TransportRequestHandler<TransportRequest.Empty> requestHandlerRepliesExceptionally(Exception e) {
-        return (request, channel, task) -> {
-            logger.debug("got a dummy request, replying exceptionally...");
-            channel.sendResponse(e);
-        };
-    }
-
-    private TransportRequestHandler<TransportRequest.Empty> requestHandlerCaptures(Consumer<TransportChannel> channelConsumer) {
-        return (request, channel, task) -> {
-            logger.debug("got a dummy request...");
-            channelConsumer.accept(channel);
-        };
-    }
-
-    private TransportResponseHandler<TransportResponse> responseHandlerShouldNotBeCalled() {
-        return new TransportResponseHandler<>() {
-            @Override
-            public TransportResponse read(StreamInput in) {
-                throw new AssertionError("should not be called");
-            }
-
-            @Override
-            public void handleResponse(TransportResponse response) {
-                throw new AssertionError("should not be called");
-            }
-
-            @Override
-            public void handleException(TransportException exp) {
-                throw new AssertionError("should not be called");
-            }
-        };
-    }
-
-    private TransportResponseHandler<TransportResponse.Empty> responseHandlerShouldBeCalledNormally(Runnable onCalled) {
-        return new TransportResponseHandler.Empty() {
-            @Override
-            public void handleResponse(TransportResponse.Empty response) {
-                onCalled.run();
-            }
-
-            @Override
-            public void handleException(TransportException exp) {
-                throw new AssertionError("should not be called");
-            }
-        };
-    }
-
-    private TransportResponseHandler<TransportResponse> responseHandlerShouldBeCalledExceptionally(Consumer<TransportException> onCalled) {
-        return new TransportResponseHandler<>() {
-            @Override
-            public TransportResponse read(StreamInput in) {
-                throw new AssertionError("should not be called");
-            }
-
-            @Override
-            public void handleResponse(TransportResponse response) {
-                throw new AssertionError("should not be called");
-            }
-
-            @Override
-            public void handleException(TransportException exp) {
-                onCalled.accept(exp);
-            }
-        };
-    }
-
-    private void registerRequestHandler(TransportService transportService, TransportRequestHandler<TransportRequest.Empty> handler) {
-        transportService.registerRequestHandler("internal:dummy", ThreadPool.Names.GENERIC, TransportRequest.Empty::new, handler);
-    }
-
-    private void send(
-        TransportService transportService,
-        DiscoveryNode destinationNode,
-        TransportResponseHandler<? extends TransportResponse> responseHandler
-    ) {
-        transportService.sendRequest(destinationNode, "internal:dummy", TransportRequest.Empty.INSTANCE, responseHandler);
-    }
-
-    public void testSuccessfulResponse() {
-        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
-        registerRequestHandler(service2, requestHandlerRepliesNormally());
-        AtomicBoolean responseHandlerCalled = new AtomicBoolean();
-        send(service1, node2, responseHandlerShouldBeCalledNormally(() -> responseHandlerCalled.set(true)));
-        deterministicTaskQueue.runAllRunnableTasks();
-        assertTrue(responseHandlerCalled.get());
-    }
-
-    public void testExceptionalResponse() {
-        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
-        Exception e = new Exception("dummy exception");
-        registerRequestHandler(service2, requestHandlerRepliesExceptionally(e));
-        AtomicReference<TransportException> responseHandlerException = new AtomicReference<>();
-        send(service1, node2, responseHandlerShouldBeCalledExceptionally(responseHandlerException::set));
-        deterministicTaskQueue.runAllRunnableTasks();
-        assertNotNull(responseHandlerException.get());
-        assertNotNull(responseHandlerException.get().getCause());
-        assertThat(responseHandlerException.get().getCause().getMessage(), containsString("dummy exception"));
-    }
-
-    public void testDisconnectedOnRequest() {
-        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
-        registerRequestHandler(service2, requestHandlerShouldNotBeCalled());
-        disconnectedLinks.add(Tuple.tuple(node1, node2));
-        AtomicReference<TransportException> responseHandlerException = new AtomicReference<>();
-        send(service1, node2, responseHandlerShouldBeCalledExceptionally(responseHandlerException::set));
-        deterministicTaskQueue.runAllRunnableTasks();
-        assertNotNull(responseHandlerException.get());
-        assertThat(responseHandlerException.get().getMessage(), containsString("disconnected"));
-    }
-
-    public void testUnavailableOnRequest() {
-        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
-        registerRequestHandler(service2, requestHandlerShouldNotBeCalled());
-        blackholedLinks.add(Tuple.tuple(node1, node2));
-        send(service1, node2, responseHandlerShouldNotBeCalled());
-        deterministicTaskQueue.runAllRunnableTasks();
-    }
-
-    public void testUnavailableOnRequestOnly() {
-        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
-        registerRequestHandler(service2, requestHandlerShouldNotBeCalled());
-        blackholedRequestLinks.add(Tuple.tuple(node1, node2));
-        send(service1, node2, responseHandlerShouldNotBeCalled());
-        deterministicTaskQueue.runAllRunnableTasks();
-    }
-
-    public void testDisconnectedOnSuccessfulResponse() throws IOException {
-        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
-        AtomicReference<TransportChannel> responseHandlerChannel = new AtomicReference<>();
-        registerRequestHandler(service2, requestHandlerCaptures(responseHandlerChannel::set));
-
-        AtomicReference<TransportException> responseHandlerException = new AtomicReference<>();
-        send(service1, node2, responseHandlerShouldBeCalledExceptionally(responseHandlerException::set));
-        deterministicTaskQueue.runAllRunnableTasks();
-        assertNotNull(responseHandlerChannel.get());
-        assertNull(responseHandlerException.get());
-
-        disconnectedLinks.add(Tuple.tuple(node2, node1));
-        responseHandlerChannel.get().sendResponse(TransportResponse.Empty.INSTANCE);
-        deterministicTaskQueue.runAllTasks();
-        deliverBlackholedRequests.run();
-        deterministicTaskQueue.runAllTasks();
-
-        assertThat(responseHandlerException.get(), instanceOf(ConnectTransportException.class));
-    }
-
-    public void testDisconnectedOnExceptionalResponse() throws IOException {
-        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
-        AtomicReference<TransportChannel> responseHandlerChannel = new AtomicReference<>();
-        registerRequestHandler(service2, requestHandlerCaptures(responseHandlerChannel::set));
-
-        AtomicReference<TransportException> responseHandlerException = new AtomicReference<>();
-        send(service1, node2, responseHandlerShouldBeCalledExceptionally(responseHandlerException::set));
-        deterministicTaskQueue.runAllRunnableTasks();
-        assertNotNull(responseHandlerChannel.get());
-        assertNull(responseHandlerException.get());
-
-        disconnectedLinks.add(Tuple.tuple(node2, node1));
-        responseHandlerChannel.get().sendResponse(new Exception());
-        deterministicTaskQueue.runAllTasks();
-        deliverBlackholedRequests.run();
-        deterministicTaskQueue.runAllTasks();
-
-        assertThat(responseHandlerException.get(), instanceOf(ConnectTransportException.class));
-    }
-
-    public void testUnavailableOnSuccessfulResponse() throws IOException {
-        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
-        AtomicReference<TransportChannel> responseHandlerChannel = new AtomicReference<>();
-        registerRequestHandler(service2, requestHandlerCaptures(responseHandlerChannel::set));
-
-        send(service1, node2, responseHandlerShouldNotBeCalled());
-        deterministicTaskQueue.runAllRunnableTasks();
-        assertNotNull(responseHandlerChannel.get());
-
-        blackholedLinks.add(Tuple.tuple(node2, node1));
-        responseHandlerChannel.get().sendResponse(TransportResponse.Empty.INSTANCE);
-        deterministicTaskQueue.runAllRunnableTasks();
-    }
-
-    public void testUnavailableOnExceptionalResponse() throws IOException {
-        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
-        AtomicReference<TransportChannel> responseHandlerChannel = new AtomicReference<>();
-        registerRequestHandler(service2, requestHandlerCaptures(responseHandlerChannel::set));
-
-        send(service1, node2, responseHandlerShouldNotBeCalled());
-        deterministicTaskQueue.runAllRunnableTasks();
-        assertNotNull(responseHandlerChannel.get());
-
-        blackholedLinks.add(Tuple.tuple(node2, node1));
-        responseHandlerChannel.get().sendResponse(new Exception());
-        deterministicTaskQueue.runAllRunnableTasks();
-    }
-
-    public void testUnavailableOnRequestOnlyReceivesSuccessfulResponse() throws IOException {
-        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
-        AtomicReference<TransportChannel> responseHandlerChannel = new AtomicReference<>();
-        registerRequestHandler(service2, requestHandlerCaptures(responseHandlerChannel::set));
-
-        AtomicBoolean responseHandlerCalled = new AtomicBoolean();
-        send(service1, node2, responseHandlerShouldBeCalledNormally(() -> responseHandlerCalled.set(true)));
-
-        deterministicTaskQueue.runAllTasks();
-        assertNotNull(responseHandlerChannel.get());
-        assertFalse(responseHandlerCalled.get());
-
-        blackholedRequestLinks.add(Tuple.tuple(node1, node2));
-        blackholedRequestLinks.add(Tuple.tuple(node2, node1));
-        responseHandlerChannel.get().sendResponse(TransportResponse.Empty.INSTANCE);
-
-        deterministicTaskQueue.runAllRunnableTasks();
-        assertTrue(responseHandlerCalled.get());
-    }
-
-    public void testUnavailableOnRequestOnlyReceivesExceptionalResponse() throws IOException {
-        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
-        AtomicReference<TransportChannel> responseHandlerChannel = new AtomicReference<>();
-        registerRequestHandler(service2, requestHandlerCaptures(responseHandlerChannel::set));
-
-        AtomicBoolean responseHandlerCalled = new AtomicBoolean();
-        send(service1, node2, responseHandlerShouldBeCalledExceptionally(e -> responseHandlerCalled.set(true)));
-
-        deterministicTaskQueue.runAllTasks();
-        assertNotNull(responseHandlerChannel.get());
-        assertFalse(responseHandlerCalled.get());
-
-        blackholedRequestLinks.add(Tuple.tuple(node1, node2));
-        blackholedRequestLinks.add(Tuple.tuple(node2, node1));
-        responseHandlerChannel.get().sendResponse(new Exception());
-
-        deterministicTaskQueue.runAllRunnableTasks();
-        assertTrue(responseHandlerCalled.get());
-    }
-
-    public void testBrokenLinkFailsToConnect() {
-        service1.disconnectFromNode(node2);
-
-        disconnectedLinks.add(Tuple.tuple(node1, node2));
-        assertThat(
-            expectThrows(ConnectTransportException.class, () -> AbstractSimpleTransportTestCase.connectToNode(service1, node2))
-                .getMessage(),
-            endsWith("is [DISCONNECTED] not [CONNECTED]")
-        );
-        disconnectedLinks.clear();
-
-        blackholedLinks.add(Tuple.tuple(node1, node2));
-        assertThat(
-            expectThrows(ConnectTransportException.class, () -> AbstractSimpleTransportTestCase.connectToNode(service1, node2))
-                .getMessage(),
-            endsWith("is [BLACK_HOLE] not [CONNECTED]")
-        );
-        blackholedLinks.clear();
-
-        blackholedRequestLinks.add(Tuple.tuple(node1, node2));
-        assertThat(
-            expectThrows(ConnectTransportException.class, () -> AbstractSimpleTransportTestCase.connectToNode(service1, node2))
-                .getMessage(),
-            endsWith("is [BLACK_HOLE_REQUESTS_ONLY] not [CONNECTED]")
-        );
-        blackholedRequestLinks.clear();
-
-        final DiscoveryNode node3 = new DiscoveryNode("node3", buildNewFakeTransportAddress(), Version.CURRENT);
-        assertThat(
-            expectThrows(ConnectTransportException.class, () -> AbstractSimpleTransportTestCase.connectToNode(service1, node3))
-                .getMessage(),
-            endsWith("does not exist")
-        );
-    }
-}
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java b/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java
new file mode 100644
index 000000000000..460e3de541c4
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/transport/DisruptableMockTransportTests.java
@@ -0,0 +1,667 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.transport;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.cluster.coordination.CleanableResponseHandler;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
+import org.elasticsearch.core.AbstractRefCounted;
+import org.elasticsearch.core.RefCounted;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.DisruptableMockTransport.ConnectionStatus;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.transport.TransportService.NOOP_TRANSPORT_INTERCEPTOR;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.endsWith;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class DisruptableMockTransportTests extends ESTestCase {
+
+    private static final String TEST_ACTION = "internal:dummy";
+
+    private DiscoveryNode node1;
+    private DiscoveryNode node2;
+
+    private TransportService service1;
+    private TransportService service2;
+
+    private DeterministicTaskQueue deterministicTaskQueue;
+
+    private Runnable deliverBlackholedRequests;
+    private Runnable blockTestAction;
+
+    private Set<Tuple<DiscoveryNode, DiscoveryNode>> disconnectedLinks;
+    private Set<Tuple<DiscoveryNode, DiscoveryNode>> blackholedLinks;
+    private Set<Tuple<DiscoveryNode, DiscoveryNode>> blackholedRequestLinks;
+
+    private long activeRequestCount;
+    private final Set<DiscoveryNode> rebootedNodes = new HashSet<>();
+
+    private ConnectionStatus getConnectionStatus(DiscoveryNode sender, DiscoveryNode destination) {
+        Tuple<DiscoveryNode, DiscoveryNode> link = Tuple.tuple(sender, destination);
+        if (disconnectedLinks.contains(link)) {
+            assert blackholedLinks.contains(link) == false;
+            assert blackholedRequestLinks.contains(link) == false;
+            return ConnectionStatus.DISCONNECTED;
+        }
+        if (blackholedLinks.contains(link)) {
+            assert blackholedRequestLinks.contains(link) == false;
+            return ConnectionStatus.BLACK_HOLE;
+        }
+        if (blackholedRequestLinks.contains(link)) {
+            return ConnectionStatus.BLACK_HOLE_REQUESTS_ONLY;
+        }
+        return ConnectionStatus.CONNECTED;
+    }
+
+    @Before
+    public void initTransports() {
+        node1 = new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT);
+        node2 = new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT);
+
+        disconnectedLinks = new HashSet<>();
+        blackholedLinks = new HashSet<>();
+        blackholedRequestLinks = new HashSet<>();
+
+        List<DisruptableMockTransport> transports = new ArrayList<>();
+
+        deterministicTaskQueue = new DeterministicTaskQueue();
+
+        final DisruptableMockTransport transport1 = new DisruptableMockTransport(node1, deterministicTaskQueue) {
+            @Override
+            protected ConnectionStatus getConnectionStatus(DiscoveryNode destination) {
+                return DisruptableMockTransportTests.this.getConnectionStatus(getLocalNode(), destination);
+            }
+
+            @Override
+            protected Optional<DisruptableMockTransport> getDisruptableMockTransport(TransportAddress address) {
+                return transports.stream().filter(t -> t.getLocalNode().getAddress().equals(address)).findAny();
+            }
+
+            @Override
+            protected void execute(Runnable runnable) {
+                deterministicTaskQueue.scheduleNow(unlessRebooted(node1, runnable));
+            }
+        };
+
+        final DisruptableMockTransport transport2 = new DisruptableMockTransport(node2, deterministicTaskQueue) {
+            @Override
+            protected ConnectionStatus getConnectionStatus(DiscoveryNode destination) {
+                return DisruptableMockTransportTests.this.getConnectionStatus(getLocalNode(), destination);
+            }
+
+            @Override
+            protected Optional<DisruptableMockTransport> getDisruptableMockTransport(TransportAddress address) {
+                return transports.stream().filter(t -> t.getLocalNode().getAddress().equals(address)).findAny();
+            }
+
+            @Override
+            protected void execute(Runnable runnable) {
+                deterministicTaskQueue.scheduleNow(unlessRebooted(node2, runnable));
+            }
+        };
+
+        transports.add(transport1);
+        transports.add(transport2);
+
+        service1 = transport1.createTransportService(
+            Settings.EMPTY,
+            deterministicTaskQueue.getThreadPool(),
+            NOOP_TRANSPORT_INTERCEPTOR,
+            a -> node1,
+            null,
+            Collections.emptySet()
+        );
+        service2 = transport2.createTransportService(
+            Settings.EMPTY,
+            deterministicTaskQueue.getThreadPool(),
+            NOOP_TRANSPORT_INTERCEPTOR,
+            a -> node2,
+            null,
+            Collections.emptySet()
+        );
+
+        service1.start();
+        service2.start();
+
+        final PlainActionFuture<Releasable> fut1 = new PlainActionFuture<>();
+        service1.connectToNode(node2, fut1);
+        final PlainActionFuture<Releasable> fut2 = new PlainActionFuture<>();
+        service2.connectToNode(node1, fut2);
+        deterministicTaskQueue.runAllTasksInTimeOrder();
+        assertTrue(fut1.isDone());
+        assertTrue(fut2.isDone());
+
+        deliverBlackholedRequests = () -> transports.forEach(DisruptableMockTransport::deliverBlackholedRequests);
+
+        blockTestAction = new Runnable() {
+            @Override
+            public void run() {
+                transports.forEach(t -> t.addActionBlock(TEST_ACTION));
+            }
+
+            @Override
+            public String toString() {
+                return "add block for " + TEST_ACTION;
+            }
+        };
+
+        activeRequestCount = 0;
+        rebootedNodes.clear();
+    }
+
+    @After
+    public void assertAllRequestsReleased() {
+        assertEquals(0, activeRequestCount);
+    }
+
+    private Runnable reboot(DiscoveryNode discoveryNode) {
+        return new Runnable() {
+            @Override
+            public void run() {
+                rebootedNodes.add(discoveryNode);
+            }
+
+            @Override
+            public String toString() {
+                return "reboot " + discoveryNode;
+            }
+        };
+    }
+
+    private Runnable unlessRebooted(DiscoveryNode discoveryNode, Runnable runnable) {
+        return new Runnable() {
+            @Override
+            public void run() {
+                if (rebootedNodes.contains(discoveryNode)) {
+                    if (runnable instanceof DisruptableMockTransport.RebootSensitiveRunnable rebootSensitiveRunnable) {
+                        rebootSensitiveRunnable.ifRebooted();
+                    }
+                } else {
+                    runnable.run();
+                }
+            }
+
+            @Override
+            public String toString() {
+                return "unlessRebooted[" + discoveryNode.getId() + "/" + runnable + "]";
+            }
+        };
+    }
+
+    private TransportRequestHandler<TestRequest> requestHandlerShouldNotBeCalled() {
+        return (request, channel, task) -> { throw new AssertionError("should not be called"); };
+    }
+
+    private TransportRequestHandler<TestRequest> requestHandlerRepliesNormally() {
+        return (request, channel, task) -> {
+            logger.debug("got a dummy request, replying normally...");
+            channel.sendResponse(new TestResponse());
+        };
+    }
+
+    private TransportRequestHandler<TestRequest> requestHandlerRepliesExceptionally(Exception e) {
+        return (request, channel, task) -> {
+            logger.debug("got a dummy request, replying exceptionally...");
+            channel.sendResponse(e);
+        };
+    }
+
+    private TransportRequestHandler<TestRequest> requestHandlerCaptures(Consumer<TransportChannel> channelConsumer) {
+        return (request, channel, task) -> {
+            logger.debug("got a dummy request...");
+            channelConsumer.accept(channel);
+        };
+    }
+
+    private <T extends TransportResponse> TransportResponseHandler<T> responseHandlerShouldNotBeCalled() {
+        return new TransportResponseHandler<>() {
+            @Override
+            public T read(StreamInput in) {
+                throw new AssertionError("should not be called");
+            }
+
+            @Override
+            public void handleResponse(T response) {
+                throw new AssertionError("should not be called");
+            }
+
+            @Override
+            public void handleException(TransportException exp) {
+                throw new AssertionError("should not be called");
+            }
+        };
+    }
+
+    private TransportResponseHandler<TestResponse> responseHandlerShouldBeCalledNormally(Runnable onCalled) {
+        return new TransportResponseHandler<>() {
+            @Override
+            public TestResponse read(StreamInput in) throws IOException {
+                return new TestResponse(in);
+            }
+
+            @Override
+            public void handleResponse(TestResponse response) {
+                onCalled.run();
+            }
+
+            @Override
+            public void handleException(TransportException exp) {
+                throw new AssertionError("should not be called");
+            }
+        };
+    }
+
+    private <T extends TransportResponse> TransportResponseHandler<T> responseHandlerShouldBeCalledExceptionally(
+        Consumer<TransportException> onCalled
+    ) {
+        return new TransportResponseHandler<>() {
+            @Override
+            public T read(StreamInput in) {
+                throw new AssertionError("should not be called");
+            }
+
+            @Override
+            public void handleResponse(T response) {
+                throw new AssertionError("should not be called");
+            }
+
+            @Override
+            public void handleException(TransportException exp) {
+                onCalled.accept(exp);
+            }
+        };
+    }
+
+    private void registerRequestHandler(TransportService transportService, TransportRequestHandler<TestRequest> handler) {
+        transportService.registerRequestHandler(TEST_ACTION, ThreadPool.Names.GENERIC, TestRequest::new, handler);
+    }
+
+    private void send(
+        TransportService transportService,
+        DiscoveryNode destinationNode,
+        TransportResponseHandler<? extends TransportResponse> responseHandler
+    ) {
+        final var request = new TestRequest();
+        try {
+            transportService.sendRequest(destinationNode, TEST_ACTION, request, responseHandler);
+        } finally {
+            request.decRef();
+        }
+    }
+
+    public void testSuccessfulResponse() {
+        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
+        registerRequestHandler(service2, requestHandlerRepliesNormally());
+        AtomicBoolean responseHandlerCalled = new AtomicBoolean();
+        send(service1, node2, responseHandlerShouldBeCalledNormally(() -> responseHandlerCalled.set(true)));
+        deterministicTaskQueue.runAllRunnableTasks();
+        assertTrue(responseHandlerCalled.get());
+    }
+
+    public void testBlockedAction() {
+        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
+        registerRequestHandler(service2, requestHandlerRepliesNormally());
+        blockTestAction.run();
+        AtomicReference<TransportException> responseHandlerException = new AtomicReference<>();
+        send(service1, node2, responseHandlerShouldBeCalledExceptionally(responseHandlerException::set));
+        deterministicTaskQueue.runAllRunnableTasks();
+        assertNotNull(responseHandlerException.get());
+        assertNotNull(responseHandlerException.get().getCause());
+        assertThat(responseHandlerException.get().getCause().getMessage(), containsString("action [" + TEST_ACTION + "] is blocked"));
+    }
+
+    public void testExceptionalResponse() {
+        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
+        Exception e = new Exception("dummy exception");
+        registerRequestHandler(service2, requestHandlerRepliesExceptionally(e));
+        AtomicReference<TransportException> responseHandlerException = new AtomicReference<>();
+        send(service1, node2, responseHandlerShouldBeCalledExceptionally(responseHandlerException::set));
+        deterministicTaskQueue.runAllRunnableTasks();
+        assertNotNull(responseHandlerException.get());
+        assertNotNull(responseHandlerException.get().getCause());
+        assertThat(responseHandlerException.get().getCause().getMessage(), containsString("dummy exception"));
+    }
+
+    public void testDisconnectedOnRequest() {
+        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
+        registerRequestHandler(service2, requestHandlerShouldNotBeCalled());
+        disconnectedLinks.add(Tuple.tuple(node1, node2));
+        AtomicReference<TransportException> responseHandlerException = new AtomicReference<>();
+        send(service1, node2, responseHandlerShouldBeCalledExceptionally(responseHandlerException::set));
+        deterministicTaskQueue.runAllRunnableTasks();
+        assertNotNull(responseHandlerException.get());
+        assertThat(responseHandlerException.get().getMessage(), containsString("disconnected"));
+    }
+
+    public void testUnavailableOnRequest() {
+        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
+        registerRequestHandler(service2, requestHandlerShouldNotBeCalled());
+        blackholedLinks.add(Tuple.tuple(node1, node2));
+        send(service1, node2, responseHandlerShouldNotBeCalled());
+        deterministicTaskQueue.runAllRunnableTasks();
+    }
+
+    public void testUnavailableOnRequestOnly() {
+        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
+        registerRequestHandler(service2, requestHandlerShouldNotBeCalled());
+        blackholedRequestLinks.add(Tuple.tuple(node1, node2));
+        send(service1, node2, responseHandlerShouldNotBeCalled());
+        deterministicTaskQueue.runAllRunnableTasks();
+    }
+
+    public void testDisconnectedOnSuccessfulResponse() throws IOException {
+        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
+        AtomicReference<TransportChannel> responseHandlerChannel = new AtomicReference<>();
+        registerRequestHandler(service2, requestHandlerCaptures(responseHandlerChannel::set));
+
+        AtomicReference<TransportException> responseHandlerException = new AtomicReference<>();
+        send(service1, node2, responseHandlerShouldBeCalledExceptionally(responseHandlerException::set));
+        deterministicTaskQueue.runAllRunnableTasks();
+        assertNotNull(responseHandlerChannel.get());
+        assertNull(responseHandlerException.get());
+
+        disconnectedLinks.add(Tuple.tuple(node2, node1));
+        responseHandlerChannel.get().sendResponse(new TestResponse());
+        deterministicTaskQueue.runAllTasks();
+        deliverBlackholedRequests.run();
+        deterministicTaskQueue.runAllTasks();
+
+        assertThat(responseHandlerException.get(), instanceOf(ConnectTransportException.class));
+    }
+
+    public void testDisconnectedOnExceptionalResponse() throws IOException {
+        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
+        AtomicReference<TransportChannel> responseHandlerChannel = new AtomicReference<>();
+        registerRequestHandler(service2, requestHandlerCaptures(responseHandlerChannel::set));
+
+        AtomicReference<TransportException> responseHandlerException = new AtomicReference<>();
+        send(service1, node2, responseHandlerShouldBeCalledExceptionally(responseHandlerException::set));
+        deterministicTaskQueue.runAllRunnableTasks();
+        assertNotNull(responseHandlerChannel.get());
+        assertNull(responseHandlerException.get());
+
+        disconnectedLinks.add(Tuple.tuple(node2, node1));
+        responseHandlerChannel.get().sendResponse(new Exception());
+        deterministicTaskQueue.runAllTasks();
+        deliverBlackholedRequests.run();
+        deterministicTaskQueue.runAllTasks();
+
+        assertThat(responseHandlerException.get(), instanceOf(ConnectTransportException.class));
+    }
+
+    public void testUnavailableOnSuccessfulResponse() throws IOException {
+        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
+        AtomicReference<TransportChannel> responseHandlerChannel = new AtomicReference<>();
+        registerRequestHandler(service2, requestHandlerCaptures(responseHandlerChannel::set));
+
+        send(service1, node2, responseHandlerShouldNotBeCalled());
+        deterministicTaskQueue.runAllRunnableTasks();
+        assertNotNull(responseHandlerChannel.get());
+
+        blackholedLinks.add(Tuple.tuple(node2, node1));
+        responseHandlerChannel.get().sendResponse(new TestResponse());
+        deterministicTaskQueue.runAllRunnableTasks();
+    }
+
+    public void testUnavailableOnExceptionalResponse() throws IOException {
+        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
+        AtomicReference<TransportChannel> responseHandlerChannel = new AtomicReference<>();
+        registerRequestHandler(service2, requestHandlerCaptures(responseHandlerChannel::set));
+
+        send(service1, node2, responseHandlerShouldNotBeCalled());
+        deterministicTaskQueue.runAllRunnableTasks();
+        assertNotNull(responseHandlerChannel.get());
+
+        blackholedLinks.add(Tuple.tuple(node2, node1));
+        responseHandlerChannel.get().sendResponse(new Exception());
+        deterministicTaskQueue.runAllRunnableTasks();
+    }
+
+    public void testUnavailableOnRequestOnlyReceivesSuccessfulResponse() throws IOException {
+        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
+        AtomicReference<TransportChannel> responseHandlerChannel = new AtomicReference<>();
+        registerRequestHandler(service2, requestHandlerCaptures(responseHandlerChannel::set));
+
+        AtomicBoolean responseHandlerCalled = new AtomicBoolean();
+        send(service1, node2, responseHandlerShouldBeCalledNormally(() -> responseHandlerCalled.set(true)));
+
+        deterministicTaskQueue.runAllTasks();
+        assertNotNull(responseHandlerChannel.get());
+        assertFalse(responseHandlerCalled.get());
+
+        blackholedRequestLinks.add(Tuple.tuple(node1, node2));
+        blackholedRequestLinks.add(Tuple.tuple(node2, node1));
+        responseHandlerChannel.get().sendResponse(new TestResponse());
+
+        deterministicTaskQueue.runAllRunnableTasks();
+        assertTrue(responseHandlerCalled.get());
+    }
+
+    public void testUnavailableOnRequestOnlyReceivesExceptionalResponse() throws IOException {
+        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
+        AtomicReference<TransportChannel> responseHandlerChannel = new AtomicReference<>();
+        registerRequestHandler(service2, requestHandlerCaptures(responseHandlerChannel::set));
+
+        AtomicBoolean responseHandlerCalled = new AtomicBoolean();
+        send(service1, node2, responseHandlerShouldBeCalledExceptionally(e -> responseHandlerCalled.set(true)));
+
+        deterministicTaskQueue.runAllTasks();
+        assertNotNull(responseHandlerChannel.get());
+        assertFalse(responseHandlerCalled.get());
+
+        blackholedRequestLinks.add(Tuple.tuple(node1, node2));
+        blackholedRequestLinks.add(Tuple.tuple(node2, node1));
+        responseHandlerChannel.get().sendResponse(new Exception());
+
+        deterministicTaskQueue.runAllRunnableTasks();
+        assertTrue(responseHandlerCalled.get());
+    }
+
+    public void testResponseWithReboots() {
+        registerRequestHandler(service1, requestHandlerShouldNotBeCalled());
+        registerRequestHandler(
+            service2,
+            randomFrom(requestHandlerRepliesNormally(), requestHandlerRepliesExceptionally(new ElasticsearchException("simulated")))
+        );
+
+        final var linkDisruptions = List.of(
+            Tuple.tuple("blackhole", blackholedLinks),
+            Tuple.tuple("blackhole-request", blackholedRequestLinks),
+            Tuple.tuple("disconnected", disconnectedLinks)
+        );
+
+        for (Runnable runnable : Stream.concat(
+            Stream.of(reboot(node1), reboot(node2), blockTestAction),
+            Stream.of(Tuple.tuple(node1, node2), Tuple.tuple(node2, node1)).map(link -> {
+                final var disruption = randomFrom(linkDisruptions);
+                return new Runnable() {
+                    @Override
+                    public void run() {
+                        if (linkDisruptions.stream().noneMatch(otherDisruption -> otherDisruption.v2().contains(link))) {
+                            disruption.v2().add(link);
+                        }
+                    }
+
+                    @Override
+                    public String toString() {
+                        return disruption.v1() + ": " + link.v1().getId() + " to " + link.v2().getId();
+                    }
+                };
+            })
+        ).toList()) {
+            if (randomBoolean()) {
+                deterministicTaskQueue.scheduleNow(runnable);
+            }
+        }
+
+        AtomicBoolean responseHandlerCalled = new AtomicBoolean();
+        AtomicBoolean responseHandlerReleased = new AtomicBoolean();
+        deterministicTaskQueue.scheduleNow(new Runnable() {
+            @Override
+            public void run() {
+                DisruptableMockTransportTests.this.send(
+                    service1,
+                    node2,
+                    new CleanableResponseHandler<>(
+                        ActionListener.wrap(() -> assertFalse(responseHandlerCalled.getAndSet(true))),
+                        TestResponse::new,
+                        ThreadPool.Names.SAME,
+                        () -> assertFalse(responseHandlerReleased.getAndSet(true))
+                    )
+                );
+            }
+
+            @Override
+            public String toString() {
+                return "send test message";
+            }
+        });
+
+        deterministicTaskQueue.runAllRunnableTasks();
+        deliverBlackholedRequests.run();
+        deterministicTaskQueue.runAllRunnableTasks();
+
+        assertTrue(responseHandlerReleased.get());
+        assertTrue(rebootedNodes.contains(node1) || responseHandlerCalled.get());
+    }
+
+    public void testBrokenLinkFailsToConnect() {
+        service1.disconnectFromNode(node2);
+
+        disconnectedLinks.add(Tuple.tuple(node1, node2));
+        assertThat(
+            expectThrows(ConnectTransportException.class, () -> AbstractSimpleTransportTestCase.connectToNode(service1, node2))
+                .getMessage(),
+            endsWith("is [DISCONNECTED] not [CONNECTED]")
+        );
+        disconnectedLinks.clear();
+
+        blackholedLinks.add(Tuple.tuple(node1, node2));
+        assertThat(
+            expectThrows(ConnectTransportException.class, () -> AbstractSimpleTransportTestCase.connectToNode(service1, node2))
+                .getMessage(),
+            endsWith("is [BLACK_HOLE] not [CONNECTED]")
+        );
+        blackholedLinks.clear();
+
+        blackholedRequestLinks.add(Tuple.tuple(node1, node2));
+        assertThat(
+            expectThrows(ConnectTransportException.class, () -> AbstractSimpleTransportTestCase.connectToNode(service1, node2))
+                .getMessage(),
+            endsWith("is [BLACK_HOLE_REQUESTS_ONLY] not [CONNECTED]")
+        );
+        blackholedRequestLinks.clear();
+
+        final DiscoveryNode node3 = new DiscoveryNode("node3", buildNewFakeTransportAddress(), Version.CURRENT);
+        assertThat(
+            expectThrows(ConnectTransportException.class, () -> AbstractSimpleTransportTestCase.connectToNode(service1, node3))
+                .getMessage(),
+            endsWith("does not exist")
+        );
+    }
+
+    private class TestRequest extends TransportRequest {
+        private final RefCounted refCounted;
+
+        TestRequest() {
+            activeRequestCount++;
+            refCounted = AbstractRefCounted.of(() -> activeRequestCount--);
+        }
+
+        TestRequest(StreamInput in) throws IOException {
+            super(in);
+            activeRequestCount++;
+            refCounted = AbstractRefCounted.of(() -> activeRequestCount--);
+        }
+
+        @Override
+        public void incRef() {
+            refCounted.incRef();
+        }
+
+        @Override
+        public boolean tryIncRef() {
+            return refCounted.tryIncRef();
+        }
+
+        @Override
+        public boolean decRef() {
+            return refCounted.decRef();
+        }
+
+        @Override
+        public boolean hasReferences() {
+            return refCounted.hasReferences();
+        }
+    }
+
+    private class TestResponse extends TransportResponse {
+        private final RefCounted refCounted;
+
+        TestResponse() {
+            activeRequestCount++;
+            refCounted = AbstractRefCounted.of(() -> activeRequestCount--);
+        }
+
+        TestResponse(StreamInput in) throws IOException {
+            super(in);
+            activeRequestCount++;
+            refCounted = AbstractRefCounted.of(() -> activeRequestCount--);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) {}
+
+        @Override
+        public void incRef() {
+            refCounted.incRef();
+        }
+
+        @Override
+        public boolean tryIncRef() {
+            return refCounted.tryIncRef();
+        }
+
+        @Override
+        public boolean decRef() {
+            return refCounted.decRef();
+        }
+
+        @Override
+        public boolean hasReferences() {
+            return refCounted.hasReferences();
+        }
+    }
+}
diff --git a/test/test-clusters/build.gradle b/test/test-clusters/build.gradle
new file mode 100644
index 000000000000..48f55931c510
--- /dev/null
+++ b/test/test-clusters/build.gradle
@@ -0,0 +1,15 @@
+import org.elasticsearch.gradle.internal.conventions.util.Util
+
+apply plugin: 'elasticsearch.java'
+apply plugin: 'com.github.johnrengelman.shadow'
+
+dependencies {
+  shadow "junit:junit:${versions.junit}"
+  shadow "org.apache.logging.log4j:log4j-api:${versions.log4j}"
+
+  implementation "org.elasticsearch.gradle:reaper"
+}
+
+tasks.named("processResources").configure {
+  from(new File(Util.locateElasticsearchWorkspace(gradle), "build-tools-internal/version.properties"))
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/ClusterFactory.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/ClusterFactory.java
new file mode 100644
index 000000000000..313bfcc767fa
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/ClusterFactory.java
@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster;
+
+public interface ClusterFactory<S extends ClusterSpec, H extends ClusterHandle> {
+
+    H create(S spec);
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/ClusterHandle.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/ClusterHandle.java
new file mode 100644
index 000000000000..2a4e3e3958c5
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/ClusterHandle.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster;
+
+import org.elasticsearch.test.cluster.util.Version;
+
+import java.io.Closeable;
+
+/**
+ * A handle to an {@link ElasticsearchCluster}.
+ */
+public interface ClusterHandle extends Closeable {
+    /**
+     * Starts the cluster. This method will block until all nodes are started and cluster is ready to serve requests.
+     */
+    void start();
+
+    /**
+     * Stops the cluster. This method will block until all cluster node processes have exited. This method is thread-safe and subsequent
+     * calls will wait for the exiting termination to complete.
+     *
+     * @param forcibly whether to forcibly terminate the cluster
+     */
+    void stop(boolean forcibly);
+
+    /**
+     * Restarts the cluster. Effectively the same as calling {@link #stop(boolean)} followed by {@link #start()}
+     *
+     * @param forcibly whether to ficibly terminate the cluster
+     */
+    void restart(boolean forcibly);
+
+    /**
+     * Whether the cluster is started or not. This method makes no guarantees on cluster availability, only that the node processes have
+     * been started.
+     *
+     * @return whether the cluster has been started
+     */
+    boolean isStarted();
+
+    /**
+     * Returns a comma-separated list of HTTP transport endpoints for cluster. If this method is called on an unstarted cluster, the cluster
+     * will be started. This method is thread-safe and subsequent calls will wait for cluster start and availability.
+     *
+     * @return cluster node HTTP transport addresses
+     */
+    String getHttpAddresses();
+
+    /**
+     * Returns the HTTP transport endpoint for the node at the given index. If this method is called on an unstarted cluster, the cluster
+     * will be started. This method is thread-safe and subsequent calls will wait for cluster start and availability.
+     *
+     * @return cluster node HTTP transport addresses
+     */
+    String getHttpAddress(int index);
+
+    /**
+     * Returns a comma-separated list of TCP transport endpoints for cluster. If this method is called on an unstarted cluster, the cluster
+     * will be started. This method is thread-safe and subsequent calls will wait for cluster start and availability.
+     *
+     * @return cluster node TCP transport endpoints
+     */
+    String getTransportEndpoints();
+
+    /**
+     * Returns the TCP transport endpoint for the node at the given index. If this method is called on an unstarted cluster, the cluster
+     * will be started. This method is thread-safe and subsequent calls will wait for cluster start and availability.
+     *
+     * @return cluster node TCP transport endpoints
+     */
+    String getTransportEndpoint(int index);
+
+    /**
+     * Upgrades a single node to the given version. Method blocks until the node is back up and ready to respond to requests.
+     *
+     * @param index index of node ot upgrade
+     * @param version version to upgrade to
+     */
+    void upgradeNodeToVersion(int index, Version version);
+
+    /**
+     * Performs a "full cluster restart" upgrade to the given version. Method blocks until the cluster is restarted and available.
+     *
+     * @param version version to upgrade to
+     */
+    void upgradeToVersion(Version version);
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/ClusterSpec.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/ClusterSpec.java
new file mode 100644
index 000000000000..528a24509c3b
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/ClusterSpec.java
@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster;
+
+public interface ClusterSpec {}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/ElasticsearchCluster.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/ElasticsearchCluster.java
new file mode 100644
index 000000000000..02eb3fb73df6
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/ElasticsearchCluster.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster;
+
+import org.elasticsearch.test.cluster.local.DefaultLocalClusterSpecBuilder;
+import org.elasticsearch.test.cluster.local.LocalClusterSpecBuilder;
+import org.junit.rules.TestRule;
+
+/**
+ * <p>A JUnit test rule for orchestrating an Elasticsearch cluster for local integration testing. New clusters can be created via one of the
+ * various static builder methods. For example:</p>
+ * <pre>
+ * &#064;ClassRule
+ * public static ElasticsearchCluster myCluster = ElasticsearchCluster.local().build();
+ * </pre>
+ */
+public interface ElasticsearchCluster extends TestRule, ClusterHandle {
+
+    /**
+     * Creates a new {@link DefaultLocalClusterSpecBuilder} for defining a locally orchestrated cluster. Local clusters use a locally built
+     * Elasticsearch distribution.
+     *
+     * @return a builder for a local cluster
+     */
+    static LocalClusterSpecBuilder local() {
+        return new DefaultLocalClusterSpecBuilder();
+    }
+
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/EnvironmentProvider.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/EnvironmentProvider.java
new file mode 100644
index 000000000000..a63151b42f8f
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/EnvironmentProvider.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster;
+
+import org.elasticsearch.test.cluster.local.LocalClusterSpec.LocalNodeSpec;
+
+import java.util.Map;
+
+/**
+ * Functional interface for supplying environment variables to an Elasticsearch node. This interface is designed to be implemented by tests
+ * and fixtures wanting to provide settings to an {@link ElasticsearchCluster} in a dynamic fashion. Instances are evaluated lazily at
+ * cluster start time.
+ */
+public interface EnvironmentProvider {
+
+    /**
+     * Returns a collection of environment variables to apply to an Elasticsearch cluster node. This method is called when the cluster is
+     * started so implementors can return dynamic environment values that may or may not be based on the given node spec.
+     *
+     * @param nodeSpec the specification for the given node to apply settings to
+     * @return environment variables to add to the node
+     */
+    Map<String, String> get(LocalNodeSpec nodeSpec);
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java
new file mode 100644
index 000000000000..62103dcf47e7
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster;
+
+import org.elasticsearch.test.cluster.util.Version;
+
+/**
+ * Elasticsearch feature flags. Used in conjunction with {@link org.elasticsearch.test.cluster.local.LocalSpecBuilder#feature(FeatureFlag)}
+ * to indicate that this feature is required and should be enabled when appropriate.
+ */
+public enum FeatureFlag {
+    TIME_SERIES_MODE("es.index_mode_feature_flag_registered=true", Version.fromString("8.0.0"), null);
+
+    public final String systemProperty;
+    public final Version from;
+    public final Version until;
+
+    FeatureFlag(String systemProperty, Version from, Version until) {
+        this.systemProperty = systemProperty;
+        this.from = from;
+        this.until = until;
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/MutableSettingsProvider.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/MutableSettingsProvider.java
new file mode 100644
index 000000000000..172a114bc04d
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/MutableSettingsProvider.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster;
+
+import org.elasticsearch.test.cluster.local.LocalClusterSpec;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class MutableSettingsProvider implements SettingsProvider {
+    private final Map<String, String> settings = new HashMap<>();
+
+    @Override
+    public Map<String, String> get(LocalClusterSpec.LocalNodeSpec nodeSpec) {
+        return settings;
+    }
+
+    public void put(String setting, String value) {
+        settings.put(setting, value);
+    }
+
+    public void clear() {
+        settings.clear();
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/SettingsProvider.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/SettingsProvider.java
new file mode 100644
index 000000000000..3d4c694fad6c
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/SettingsProvider.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster;
+
+import org.elasticsearch.test.cluster.local.LocalClusterSpec.LocalNodeSpec;
+
+import java.util.Map;
+
+/**
+ * Functional interface for supplying settings to an Elasticsearch node. This interface is designed to be implemented by tests and fixtures
+ * wanting to provide settings to an {@link ElasticsearchCluster} in a dynamic fashion. Instances are evaluated lazily at cluster
+ * start time.
+ */
+public interface SettingsProvider {
+
+    /**
+     * Returns a collection of settings to apply to an Elasticsearch cluster node. This method is called when the cluster is started so
+     * implementors can return dynamic setting values that may or may not be based on the given node spec.
+     *
+     * @param nodeSpec the specification for the given node to apply settings to
+     * @return settings to add to the node
+     */
+    Map<String, String> get(LocalNodeSpec nodeSpec);
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java
new file mode 100644
index 000000000000..aa71ffdf27a7
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java
@@ -0,0 +1,233 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local;
+
+import org.elasticsearch.test.cluster.EnvironmentProvider;
+import org.elasticsearch.test.cluster.FeatureFlag;
+import org.elasticsearch.test.cluster.SettingsProvider;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.cluster.util.Version;
+import org.elasticsearch.test.cluster.util.resource.Resource;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Supplier;
+
+public abstract class AbstractLocalSpecBuilder<T extends LocalSpecBuilder<?>> implements LocalSpecBuilder<T> {
+    private final AbstractLocalSpecBuilder<?> parent;
+    private final List<SettingsProvider> settingsProviders = new ArrayList<>();
+    private final Map<String, String> settings = new HashMap<>();
+    private final List<EnvironmentProvider> environmentProviders = new ArrayList<>();
+    private final Map<String, String> environment = new HashMap<>();
+    private final Set<String> modules = new HashSet<>();
+    private final Set<String> plugins = new HashSet<>();
+    private final Set<FeatureFlag> features = new HashSet<>();
+    private final Map<String, String> keystoreSettings = new HashMap<>();
+    private final Map<String, Resource> keystoreFiles = new HashMap<>();
+    private final Map<String, Resource> extraConfigFiles = new HashMap<>();
+    private final Map<String, String> systemProperties = new HashMap<>();
+    private DistributionType distributionType;
+    private Version version;
+    private String keystorePassword;
+
+    protected AbstractLocalSpecBuilder(AbstractLocalSpecBuilder<?> parent) {
+        this.parent = parent;
+    }
+
+    @Override
+    public T settings(SettingsProvider settingsProvider) {
+        this.settingsProviders.add(settingsProvider);
+        return cast(this);
+    }
+
+    List<SettingsProvider> getSettingsProviders() {
+        return inherit(() -> parent.getSettingsProviders(), settingsProviders);
+    }
+
+    @Override
+    public T setting(String setting, String value) {
+        this.settings.put(setting, value);
+        return cast(this);
+    }
+
+    @Override
+    public T setting(String setting, Supplier<String> value) {
+        this.settingsProviders.add(s -> Map.of(setting, value.get()));
+        return cast(this);
+    }
+
+    Map<String, String> getSettings() {
+        return inherit(() -> parent.getSettings(), settings);
+    }
+
+    @Override
+    public T environment(EnvironmentProvider environmentProvider) {
+        this.environmentProviders.add(environmentProvider);
+        return cast(this);
+    }
+
+    List<EnvironmentProvider> getEnvironmentProviders() {
+        return inherit(() -> parent.getEnvironmentProviders(), environmentProviders);
+
+    }
+
+    @Override
+    public T environment(String key, String value) {
+        this.environment.put(key, value);
+        return cast(this);
+    }
+
+    Map<String, String> getEnvironment() {
+        return inherit(() -> parent.getEnvironment(), environment);
+    }
+
+    @Override
+    public T distribution(DistributionType type) {
+        this.distributionType = type;
+        return cast(this);
+    }
+
+    DistributionType getDistributionType() {
+        return inherit(() -> parent.getDistributionType(), distributionType);
+    }
+
+    @Override
+    public T module(String moduleName) {
+        this.modules.add(moduleName);
+        return cast(this);
+    }
+
+    Set<String> getModules() {
+        return inherit(() -> parent.getModules(), modules);
+    }
+
+    @Override
+    public T plugin(String pluginName) {
+        this.plugins.add(pluginName);
+        return cast(this);
+    }
+
+    Set<String> getPlugins() {
+        return inherit(() -> parent.getPlugins(), plugins);
+    }
+
+    @Override
+    public T feature(FeatureFlag feature) {
+        this.features.add(feature);
+        return cast(this);
+    }
+
+    Set<FeatureFlag> getFeatures() {
+        return inherit(() -> parent.getFeatures(), features);
+    }
+
+    @Override
+    public T keystore(String key, String value) {
+        this.keystoreSettings.put(key, value);
+        return cast(this);
+    }
+
+    public Map<String, String> getKeystoreSettings() {
+        return inherit(() -> parent.getKeystoreSettings(), keystoreSettings);
+    }
+
+    @Override
+    public T keystore(String key, Resource file) {
+        this.keystoreFiles.put(key, file);
+        return cast(this);
+    }
+
+    public Map<String, Resource> getKeystoreFiles() {
+        return inherit(() -> parent.getKeystoreFiles(), keystoreFiles);
+    }
+
+    @Override
+    public T configFile(String fileName, Resource configFile) {
+        this.extraConfigFiles.put(fileName, configFile);
+        return cast(this);
+    }
+
+    public Map<String, Resource> getExtraConfigFiles() {
+        return inherit(() -> parent.getExtraConfigFiles(), extraConfigFiles);
+    }
+
+    @Override
+    public T systemProperty(String property, String value) {
+        this.systemProperties.put(property, value);
+        return cast(this);
+    }
+
+    public Map<String, String> getSystemProperties() {
+        return inherit(() -> parent.getSystemProperties(), systemProperties);
+    }
+
+    @Override
+    public T keystorePassword(String password) {
+        this.keystorePassword = password;
+        return cast(this);
+    }
+
+    public String getKeystorePassword() {
+        return inherit(() -> parent.getKeystorePassword(), keystorePassword);
+    }
+
+    @Override
+    public T version(Version version) {
+        this.version = version;
+        return cast(this);
+    }
+
+    public Version getVersion() {
+        return inherit(() -> parent.getVersion(), version);
+    }
+
+    private <T> List<T> inherit(Supplier<List<T>> parent, List<T> child) {
+        List<T> combinedList = new ArrayList<>();
+        if (this.parent != null) {
+            combinedList.addAll(parent.get());
+        }
+        combinedList.addAll(child);
+        return combinedList;
+    }
+
+    private <T> Set<T> inherit(Supplier<Set<T>> parent, Set<T> child) {
+        Set<T> combinedSet = new HashSet<>();
+        if (this.parent != null) {
+            combinedSet.addAll(parent.get());
+        }
+        combinedSet.addAll(child);
+        return combinedSet;
+    }
+
+    private <K, V> Map<K, V> inherit(Supplier<Map<K, V>> parent, Map<K, V> child) {
+        Map<K, V> combinedMap = new HashMap<>();
+        if (this.parent != null) {
+            combinedMap.putAll(parent.get());
+        }
+        combinedMap.putAll(child);
+        return combinedMap;
+    }
+
+    private <T> T inherit(Supplier<T> parent, T child) {
+        T value = null;
+        if (this.parent != null) {
+            value = parent.get();
+        }
+        return child == null ? value : child;
+    }
+
+    @SuppressWarnings("unchecked")
+    private static <T> T cast(Object o) {
+        return (T) o;
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultEnvironmentProvider.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultEnvironmentProvider.java
new file mode 100644
index 000000000000..aa3982e076ae
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultEnvironmentProvider.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local;
+
+import org.elasticsearch.test.cluster.EnvironmentProvider;
+import org.elasticsearch.test.cluster.local.LocalClusterSpec.LocalNodeSpec;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.cluster.util.Version;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TimeZone;
+
+public class DefaultEnvironmentProvider implements EnvironmentProvider {
+    private static final String HOSTNAME_OVERRIDE = "LinuxDarwinHostname";
+    private static final String COMPUTERNAME_OVERRIDE = "WindowsComputername";
+    private static final String TESTS_RUNTIME_JAVA_SYSPROP = "tests.runtime.java";
+
+    @Override
+    public Map<String, String> get(LocalNodeSpec nodeSpec) {
+        Map<String, String> environment = new HashMap<>();
+
+        // If we are testing the current version of Elasticsearch, use the configured runtime Java, otherwise use the bundled JDK
+        if (nodeSpec.getDistributionType() == DistributionType.INTEG_TEST || nodeSpec.getVersion().equals(Version.CURRENT)) {
+            environment.put("ES_JAVA_HOME", System.getProperty(TESTS_RUNTIME_JAVA_SYSPROP));
+        }
+
+        // Override the system hostname variables for testing
+        environment.put("HOSTNAME", HOSTNAME_OVERRIDE);
+        environment.put("COMPUTERNAME", COMPUTERNAME_OVERRIDE);
+
+        // Use the same timezone as the test executor
+        environment.put("TZ", TimeZone.getDefault().getID());
+
+        return environment;
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterSpecBuilder.java
new file mode 100644
index 000000000000..fad762fa34c2
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterSpecBuilder.java
@@ -0,0 +1,187 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.LocalClusterSpec.LocalNodeSpec;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.cluster.local.model.User;
+import org.elasticsearch.test.cluster.util.Version;
+import org.elasticsearch.test.cluster.util.resource.Resource;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+public class DefaultLocalClusterSpecBuilder extends AbstractLocalSpecBuilder<LocalClusterSpecBuilder> implements LocalClusterSpecBuilder {
+    private String name = "test-cluster";
+    private final List<DefaultLocalNodeSpecBuilder> nodeBuilders = new ArrayList<>();
+    private final List<User> users = new ArrayList<>();
+    private final List<Resource> roleFiles = new ArrayList<>();
+    private final List<Supplier<LocalClusterConfigProvider>> lazyConfigProviders = new ArrayList<>();
+
+    public DefaultLocalClusterSpecBuilder() {
+        super(null);
+        this.apply(new FipsEnabledClusterConfigProvider());
+        this.settings(new DefaultSettingsProvider());
+        this.environment(new DefaultEnvironmentProvider());
+        this.rolesFile(Resource.fromClasspath("default_test_roles.yml"));
+    }
+
+    @Override
+    public DefaultLocalClusterSpecBuilder name(String name) {
+        this.name = name;
+        return this;
+    }
+
+    @Override
+    public DefaultLocalClusterSpecBuilder apply(LocalClusterConfigProvider configProvider) {
+        configProvider.apply(this);
+        return this;
+    }
+
+    @Override
+    public LocalClusterSpecBuilder apply(Supplier<LocalClusterConfigProvider> configProvider) {
+        lazyConfigProviders.add(configProvider);
+        return this;
+    }
+
+    @Override
+    public DefaultLocalClusterSpecBuilder nodes(int nodes) {
+        if (nodes < nodeBuilders.size()) {
+            throw new IllegalArgumentException(
+                "Cannot shrink cluster to " + nodes + ". " + nodeBuilders.size() + " nodes already configured"
+            );
+        }
+
+        int newNodes = nodes - nodeBuilders.size();
+        for (int i = 0; i < newNodes; i++) {
+            nodeBuilders.add(new DefaultLocalNodeSpecBuilder(this));
+        }
+
+        return this;
+    }
+
+    @Override
+    public DefaultLocalClusterSpecBuilder withNode(Consumer<? super LocalNodeSpecBuilder> config) {
+        DefaultLocalNodeSpecBuilder builder = new DefaultLocalNodeSpecBuilder(this);
+        config.accept(builder);
+        nodeBuilders.add(builder);
+        return this;
+    }
+
+    @Override
+    public DefaultLocalClusterSpecBuilder node(int index, Consumer<? super LocalNodeSpecBuilder> config) {
+        try {
+            DefaultLocalNodeSpecBuilder builder = nodeBuilders.get(index);
+            config.accept(builder);
+        } catch (IndexOutOfBoundsException e) {
+            throw new IllegalArgumentException(
+                "No node at index + " + index + " exists. Only " + nodeBuilders.size() + " nodes have been configured"
+            );
+        }
+        return this;
+    }
+
+    @Override
+    public DefaultLocalClusterSpecBuilder user(String username, String password) {
+        this.users.add(new User(username, password));
+        return this;
+    }
+
+    @Override
+    public DefaultLocalClusterSpecBuilder user(String username, String password, String role) {
+        this.users.add(new User(username, password, role));
+        return this;
+    }
+
+    @Override
+    public DefaultLocalClusterSpecBuilder rolesFile(Resource rolesFile) {
+        this.roleFiles.add(rolesFile);
+        return this;
+    }
+
+    @Override
+    public ElasticsearchCluster build() {
+        List<User> clusterUsers = users.isEmpty() ? List.of(User.DEFAULT_USER) : users;
+        LocalClusterSpec clusterSpec = new LocalClusterSpec(name, clusterUsers, roleFiles);
+        List<LocalNodeSpec> nodeSpecs;
+
+        if (nodeBuilders.isEmpty()) {
+            // No node-specific configuration so assume a single-node cluster
+            nodeSpecs = List.of(new DefaultLocalNodeSpecBuilder(this).build(clusterSpec));
+        } else {
+            nodeSpecs = nodeBuilders.stream().map(node -> node.build(clusterSpec)).toList();
+        }
+
+        clusterSpec.setNodes(nodeSpecs);
+        clusterSpec.validate();
+
+        return new LocalElasticsearchCluster(this);
+    }
+
+    LocalClusterSpec buildClusterSpec() {
+        // Apply lazily provided configuration
+        lazyConfigProviders.forEach(s -> s.get().apply(this));
+
+        List<User> clusterUsers = users.isEmpty() ? List.of(User.DEFAULT_USER) : users;
+        LocalClusterSpec clusterSpec = new LocalClusterSpec(name, clusterUsers, roleFiles);
+        List<LocalNodeSpec> nodeSpecs;
+
+        if (nodeBuilders.isEmpty()) {
+            // No node-specific configuration so assume a single-node cluster
+            nodeSpecs = List.of(new DefaultLocalNodeSpecBuilder(this).build(clusterSpec));
+        } else {
+            nodeSpecs = nodeBuilders.stream().map(node -> node.build(clusterSpec)).toList();
+        }
+
+        clusterSpec.setNodes(nodeSpecs);
+        clusterSpec.validate();
+
+        return clusterSpec;
+    }
+
+    public static class DefaultLocalNodeSpecBuilder extends AbstractLocalSpecBuilder<LocalNodeSpecBuilder> implements LocalNodeSpecBuilder {
+        private String name;
+
+        protected DefaultLocalNodeSpecBuilder(AbstractLocalSpecBuilder<?> parent) {
+            super(parent);
+        }
+
+        @Override
+        public DefaultLocalNodeSpecBuilder name(String name) {
+            this.name = name;
+            return this;
+        }
+
+        private LocalNodeSpec build(LocalClusterSpec cluster) {
+
+            return new LocalNodeSpec(
+                cluster,
+                name,
+                Optional.ofNullable(getVersion()).orElse(Version.CURRENT),
+                getSettingsProviders(),
+                getSettings(),
+                getEnvironmentProviders(),
+                getEnvironment(),
+                getModules(),
+                getPlugins(),
+                Optional.ofNullable(getDistributionType()).orElse(DistributionType.INTEG_TEST),
+                getFeatures(),
+                getKeystoreSettings(),
+                getKeystoreFiles(),
+                getKeystorePassword(),
+                getExtraConfigFiles(),
+                getSystemProperties()
+            );
+        }
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultSettingsProvider.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultSettingsProvider.java
new file mode 100644
index 000000000000..a1c5e5892ef1
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultSettingsProvider.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local;
+
+import org.elasticsearch.test.cluster.SettingsProvider;
+import org.elasticsearch.test.cluster.local.LocalClusterSpec.LocalNodeSpec;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class DefaultSettingsProvider implements SettingsProvider {
+    @Override
+    public Map<String, String> get(LocalNodeSpec nodeSpec) {
+        Map<String, String> settings = new HashMap<>();
+
+        settings.put("node.name", nodeSpec.getName());
+        settings.put("node.attr.testattr", "test");
+        settings.put("node.portsfile", "true");
+        settings.put("http.port", "0");
+        settings.put("transport.port", "0");
+        settings.put("network.host", "_local_");
+
+        if (nodeSpec.getDistributionType() == DistributionType.INTEG_TEST) {
+            settings.put("xpack.security.enabled", "false");
+        } else {
+            // Disable deprecation indexing which is enabled by default in 7.16
+            if (nodeSpec.getVersion().onOrAfter("7.16.0")) {
+                settings.put("cluster.deprecation_indexing.enabled", "false");
+            }
+        }
+
+        // Default the watermarks to absurdly low to prevent the tests from failing on nodes without enough disk space
+        settings.put("cluster.routing.allocation.disk.watermark.low", "1b");
+        settings.put("cluster.routing.allocation.disk.watermark.high", "1b");
+        settings.put("cluster.routing.allocation.disk.watermark.flood_stage", "1b");
+
+        // increase script compilation limit since tests can rapid-fire script compilations
+        if (nodeSpec.getVersion().onOrAfter("7.9.0")) {
+            settings.put("script.disable_max_compilations_rate", "true");
+        } else {
+            settings.put("script.max_compilations_rate", "2048/1m");
+        }
+
+        // Temporarily disable the real memory usage circuit breaker. It depends on real memory usage which we have no full control
+        // over and the REST client will not retry on circuit breaking exceptions yet (see #31986 for details). Once the REST client
+        // can retry on circuit breaking exceptions, we can revert again to the default configuration.
+        settings.put("indices.breaker.total.use_real_memory", "false");
+
+        // Don't wait for state, just start up quickly. This will also allow new and old nodes in the BWC case to become the master
+        settings.put("discovery.initial_state_timeout", "0s");
+
+        if (nodeSpec.getVersion().getMajor() >= 8) {
+            settings.put("cluster.service.slow_task_logging_threshold", "5s");
+            settings.put("cluster.service.slow_master_task_logging_threshold", "5s");
+        }
+
+        settings.put("action.destructive_requires_name", "false");
+
+        // Setup cluster discovery
+        String masterEligibleNodes = nodeSpec.getCluster()
+            .getNodes()
+            .stream()
+            .filter(LocalNodeSpec::isMasterEligible)
+            .map(LocalNodeSpec::getName)
+            .collect(Collectors.joining(","));
+
+        if (masterEligibleNodes.isEmpty()) {
+            throw new IllegalStateException(
+                "Cannot start cluster '" + nodeSpec.getCluster().getName() + "' as it configured with no master-eligible nodes."
+            );
+        }
+
+        settings.put("cluster.initial_master_nodes", "[" + masterEligibleNodes + "]");
+        settings.put("discovery.seed_providers", "file");
+        settings.put("discovery.seed_hosts", "[]");
+
+        return settings;
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/FipsEnabledClusterConfigProvider.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/FipsEnabledClusterConfigProvider.java
new file mode 100644
index 000000000000..37a159bc8172
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/FipsEnabledClusterConfigProvider.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local;
+
+import org.elasticsearch.test.cluster.util.resource.Resource;
+
+public class FipsEnabledClusterConfigProvider implements LocalClusterConfigProvider {
+
+    @Override
+    public void apply(LocalClusterSpecBuilder builder) {
+        if (isFipsEnabled()) {
+            builder.configFile(
+                "fips_java.security",
+                Resource.fromClasspath(isOracleJvm() ? "fips/fips_java_oracle.security" : "fips/fips_java.security")
+            )
+                .configFile("fips_java.policy", Resource.fromClasspath("fips/fips_java.policy"))
+                .configFile("cacerts.bcfks", Resource.fromClasspath("fips/cacerts.bcfks"))
+                .systemProperty("java.security.properties", "=${ES_PATH_CONF}/fips_java.security")
+                .systemProperty("java.security.policy", "=${ES_PATH_CONF}/fips_java.policy")
+                .systemProperty("javax.net.ssl.trustStore", "${ES_PATH_CONF}/cacerts.bcfks")
+                .systemProperty("javax.net.ssl.trustStorePassword", "password")
+                .systemProperty("javax.net.ssl.keyStorePassword", "password")
+                .systemProperty("javax.net.ssl.keyStoreType", "BCFKS")
+                .systemProperty("org.bouncycastle.fips.approved_only", "true")
+                .setting("network.host", "_local:ipv4_")
+                .setting("xpack.security.enabled", "false")
+                .setting("xpack.security.fips_mode.enabled", "true")
+                .setting("xpack.license.self_generated.type", "trial")
+                .setting("xpack.security.authc.password_hashing.algorithm", "pbkdf2_stretch")
+                .keystorePassword("keystore-password");
+        }
+    }
+
+    private static boolean isFipsEnabled() {
+        return Boolean.getBoolean("tests.fips.enabled");
+    }
+
+    private static boolean isOracleJvm() {
+        return System.getProperty("java.vendor").toLowerCase().contains("oracle");
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterConfigProvider.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterConfigProvider.java
new file mode 100644
index 000000000000..7e802247709d
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterConfigProvider.java
@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local;
+
+public interface LocalClusterConfigProvider {
+
+    void apply(LocalClusterSpecBuilder builder);
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterFactory.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterFactory.java
new file mode 100644
index 000000000000..5c490d8322b0
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterFactory.java
@@ -0,0 +1,633 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.test.cluster.ClusterFactory;
+import org.elasticsearch.test.cluster.local.LocalClusterSpec.LocalNodeSpec;
+import org.elasticsearch.test.cluster.local.distribution.DistributionDescriptor;
+import org.elasticsearch.test.cluster.local.distribution.DistributionResolver;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.cluster.local.model.User;
+import org.elasticsearch.test.cluster.util.IOUtils;
+import org.elasticsearch.test.cluster.util.OS;
+import org.elasticsearch.test.cluster.util.Pair;
+import org.elasticsearch.test.cluster.util.ProcessReaper;
+import org.elasticsearch.test.cluster.util.ProcessUtils;
+import org.elasticsearch.test.cluster.util.Retry;
+import org.elasticsearch.test.cluster.util.Version;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.UncheckedIOException;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.test.cluster.local.distribution.DistributionType.DEFAULT;
+import static org.elasticsearch.test.cluster.util.OS.WINDOWS;
+
+public class LocalClusterFactory implements ClusterFactory<LocalClusterSpec, LocalClusterHandle> {
+    private static final Logger LOGGER = LogManager.getLogger(LocalClusterFactory.class);
+    private static final Duration NODE_UP_TIMEOUT = Duration.ofMinutes(2);
+    private static final Map<Pair<Version, DistributionType>, DistributionDescriptor> TEST_DISTRIBUTIONS = new ConcurrentHashMap<>();
+    private static final String TESTS_CLUSTER_MODULES_PATH_SYSPROP = "tests.cluster.modules.path";
+    private static final String TESTS_CLUSTER_PLUGINS_PATH_SYSPROP = "tests.cluster.plugins.path";
+    private static final String TESTS_CLUSTER_FIPS_JAR_PATH_SYSPROP = "tests.cluster.fips.jars.path";
+    private static final String TESTS_CLUSTER_DEBUG_ENABLED_SYSPROP = "tests.cluster.debug.enabled";
+    private static final String ENABLE_DEBUG_JVM_ARGS = "-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=";
+    private static final int DEFAULT_DEBUG_PORT = 5007;
+
+    private final Path baseWorkingDir;
+    private final DistributionResolver distributionResolver;
+
+    public LocalClusterFactory(Path baseWorkingDir, DistributionResolver distributionResolver) {
+        this.baseWorkingDir = baseWorkingDir;
+        this.distributionResolver = distributionResolver;
+    }
+
+    @Override
+    public LocalClusterHandle create(LocalClusterSpec spec) {
+        return new LocalClusterHandle(spec.getName(), spec.getNodes().stream().map(Node::new).toList());
+    }
+
+    public class Node {
+        private final LocalNodeSpec spec;
+        private final Path workingDir;
+        private final Path repoDir;
+        private final Path dataDir;
+        private final Path logsDir;
+        private final Path configDir;
+        private final Path tempDir;
+
+        private Path distributionDir;
+        private Version currentVersion;
+        private Process process = null;
+        private DistributionDescriptor distributionDescriptor;
+
+        public Node(LocalNodeSpec spec) {
+            this.spec = spec;
+            this.workingDir = baseWorkingDir.resolve(spec.getCluster().getName()).resolve(spec.getName());
+            this.repoDir = baseWorkingDir.resolve("repo");
+            this.dataDir = workingDir.resolve("data");
+            this.logsDir = workingDir.resolve("logs");
+            this.configDir = workingDir.resolve("config");
+            this.tempDir = workingDir.resolve("tmp"); // elasticsearch temporary directory
+        }
+
+        public synchronized void start(Version version) {
+            LOGGER.info("Starting Elasticsearch node '{}'", spec.getName());
+            if (version != null) {
+                spec.setVersion(version);
+            }
+
+            if (currentVersion == null || currentVersion.equals(spec.getVersion()) == false) {
+                LOGGER.info("Creating installation for node '{}' in {}", spec.getName(), workingDir);
+                distributionDescriptor = resolveDistribution();
+                LOGGER.info("Distribution for node '{}': {}", spec.getName(), distributionDescriptor);
+                initializeWorkingDirectory(currentVersion != null);
+                createConfigDirectory();
+                copyExtraConfigFiles(); // extra config files might be needed for running cli tools like plugin install
+                copyExtraJarFiles();
+                installPlugins();
+                if (distributionDescriptor.getType() == DistributionType.INTEG_TEST) {
+                    installModules();
+                }
+                currentVersion = spec.getVersion();
+            } else {
+                createConfigDirectory();
+                copyExtraConfigFiles();
+            }
+
+            writeConfiguration();
+            createKeystore();
+            addKeystoreSettings();
+            addKeystoreFiles();
+            configureSecurity();
+
+            startElasticsearch();
+        }
+
+        public synchronized void stop(boolean forcibly) {
+            if (process != null) {
+                ProcessUtils.stopHandle(process.toHandle(), forcibly);
+                ProcessReaper.instance().unregister(getServiceName());
+            }
+            deletePortsFiles();
+        }
+
+        public void waitForExit() {
+            if (process != null) {
+                ProcessUtils.waitForExit(process.toHandle());
+            }
+        }
+
+        public String getHttpAddress() {
+            Path portFile = workingDir.resolve("logs").resolve("http.ports");
+            if (Files.notExists(portFile)) {
+                waitUntilReady();
+            }
+            return readPortsFile(portFile).get(0);
+        }
+
+        public String getTransportEndpoint() {
+            Path portsFile = workingDir.resolve("logs").resolve("transport.ports");
+            if (Files.notExists(portsFile)) {
+                waitUntilReady();
+            }
+            return readPortsFile(portsFile).get(0);
+        }
+
+        public void deletePortsFiles() {
+            try {
+                Path hostsFile = workingDir.resolve("config").resolve("unicast_hosts.txt");
+                Path httpPortsFile = workingDir.resolve("logs").resolve("http.ports");
+                Path transportPortsFile = workingDir.resolve("logs").resolve("transport.ports");
+
+                Files.deleteIfExists(hostsFile);
+                Files.deleteIfExists(httpPortsFile);
+                Files.deleteIfExists(transportPortsFile);
+            } catch (IOException e) {
+                throw new UncheckedIOException("Failed to write unicast_hosts for: " + this, e);
+            }
+        }
+
+        public LocalNodeSpec getSpec() {
+            return spec;
+        }
+
+        Path getWorkingDir() {
+            return workingDir;
+        }
+
+        public void waitUntilReady() {
+            try {
+                Retry.retryUntilTrue(NODE_UP_TIMEOUT, Duration.ofMillis(500), () -> {
+                    if (process.isAlive() == false) {
+                        throw new RuntimeException(
+                            "Elasticsearch process died while waiting for ports file. See console output for details."
+                        );
+                    }
+
+                    Path httpPorts = workingDir.resolve("logs").resolve("http.ports");
+                    Path transportPorts = workingDir.resolve("logs").resolve("transport.ports");
+
+                    return Files.exists(httpPorts) && Files.exists(transportPorts);
+                });
+            } catch (TimeoutException e) {
+                throw new RuntimeException("Timed out after " + NODE_UP_TIMEOUT + " waiting for ports files for: " + this);
+            } catch (ExecutionException e) {
+                throw new RuntimeException("An error occurred while waiting for ports file for: " + this, e);
+            }
+        }
+
+        private void createConfigDirectory() {
+            try {
+                IOUtils.deleteWithRetry(configDir);
+                Files.createDirectories(configDir);
+            } catch (IOException e) {
+                throw new UncheckedIOException("An error occurred creating config directory", e);
+            }
+        }
+
+        private List<String> readPortsFile(Path file) {
+            try (Stream<String> lines = Files.lines(file, StandardCharsets.UTF_8)) {
+                return lines.map(String::trim).collect(Collectors.toList());
+            } catch (IOException e) {
+                throw new UncheckedIOException("Unable to read ports file: " + file, e);
+            }
+        }
+
+        private void initializeWorkingDirectory(boolean preserveWorkingDirectory) {
+            try {
+                if (preserveWorkingDirectory == false) {
+                    IOUtils.deleteWithRetry(workingDir);
+                }
+
+                if (canUseSharedDistribution()) {
+                    distributionDir = distributionDescriptor.getDistributionDir();
+                } else {
+                    distributionDir = OS.conditional(
+                        // Use per-version distribution directories on Windows to avoid cleanup failures
+                        c -> c.onWindows(() -> workingDir.resolve("distro").resolve(distributionDescriptor.getVersion().toString()))
+                            .onUnix(() -> workingDir.resolve("distro"))
+                    );
+
+                    if (Files.exists(distributionDir)) {
+                        IOUtils.deleteWithRetry(distributionDir);
+                    }
+
+                    try {
+                        IOUtils.syncWithLinks(distributionDescriptor.getDistributionDir(), distributionDir);
+                    } catch (IOUtils.LinkCreationException e) {
+                        // Note does not work for network drives, e.g. Vagrant
+                        LOGGER.info("Failed to create working dir using hard links. Falling back to copy", e);
+                        // ensure we get a clean copy
+                        IOUtils.deleteWithRetry(distributionDir);
+                        IOUtils.syncWithCopy(distributionDescriptor.getDistributionDir(), distributionDir);
+                    }
+                }
+                Files.createDirectories(repoDir);
+                Files.createDirectories(dataDir);
+                Files.createDirectories(logsDir);
+                Files.createDirectories(tempDir);
+            } catch (IOException e) {
+                throw new UncheckedIOException("Failed to create working directory for node '" + spec.getName() + "'", e);
+            }
+        }
+
+        /*
+         * We can "share" a distribution directory across clusters so long as we aren't modifying it. That means we aren't installing any
+         * additional plugins, modules, or jars. This avoids having to copy the test distribution unnecessarily.
+         */
+        private boolean canUseSharedDistribution() {
+            return OS.current() != WINDOWS // Issues with long file paths on Windows in CI
+                && System.getProperty(TESTS_CLUSTER_FIPS_JAR_PATH_SYSPROP) == null
+                && (distributionDescriptor.getType() == DEFAULT || (getSpec().getPlugins().isEmpty() && getSpec().getModules().isEmpty()));
+        }
+
+        private void copyExtraJarFiles() {
+            String fipsJarsPath = System.getProperty(TESTS_CLUSTER_FIPS_JAR_PATH_SYSPROP);
+            if (fipsJarsPath != null) {
+                LOGGER.info("FIPS is enabled. Copying FIPS jars for node: {}", this);
+                Path libsDir = distributionDir.resolve("lib");
+                Arrays.stream(fipsJarsPath.split(File.pathSeparator)).map(Path::of).forEach(jar -> {
+                    LOGGER.info("Copying jar {} to {}", jar, libsDir);
+                    try {
+                        Files.copy(jar, libsDir.resolve(jar.getFileName()), StandardCopyOption.REPLACE_EXISTING);
+                    } catch (IOException e) {
+                        throw new UncheckedIOException("An error occurred copy extra jar files for: " + this, e);
+                    }
+                });
+            }
+        }
+
+        private DistributionDescriptor resolveDistribution() {
+            return TEST_DISTRIBUTIONS.computeIfAbsent(
+                Pair.of(spec.getVersion(), spec.getDistributionType()),
+                key -> distributionResolver.resolve(key.left, key.right)
+            );
+        }
+
+        private void writeConfiguration() {
+            Path configFile = configDir.resolve("elasticsearch.yml");
+            Path jvmOptionsFile = configDir.resolve("jvm.options");
+
+            try {
+                // Write settings to elasticsearch.yml
+                Map<String, String> finalSettings = new HashMap<>();
+                finalSettings.put("path.repo", repoDir.toString());
+                finalSettings.put("path.data", dataDir.toString());
+                finalSettings.put("path.logs", logsDir.toString());
+                finalSettings.putAll(spec.resolveSettings());
+
+                Files.writeString(
+                    configFile,
+                    finalSettings.entrySet()
+                        .stream()
+                        .map(entry -> entry.getKey() + ": " + entry.getValue())
+                        .collect(Collectors.joining("\n")),
+                    StandardOpenOption.TRUNCATE_EXISTING,
+                    StandardOpenOption.CREATE
+                );
+
+                // Copy additional configuration from distribution
+                try (Stream<Path> configFiles = Files.list(distributionDir.resolve("config"))) {
+                    for (Path file : configFiles.toList()) {
+                        Path dest = configFile.getParent().resolve(file.getFileName());
+                        if (Files.exists(dest) == false) {
+                            Files.copy(file, dest);
+                        }
+                    }
+                }
+
+                // Patch jvm.options file to update paths
+                String content = Files.readString(jvmOptionsFile);
+                Map<String, String> expansions = getJvmOptionsReplacements();
+                for (String key : expansions.keySet()) {
+                    if (content.contains(key) == false) {
+                        throw new IOException("Template property '" + key + "' not found in template.");
+                    }
+                    content = content.replace(key, expansions.get(key));
+                }
+                Files.writeString(jvmOptionsFile, content);
+            } catch (IOException e) {
+                throw new UncheckedIOException("Could not write config file: " + configFile, e);
+            }
+        }
+
+        private void copyExtraConfigFiles() {
+            spec.getExtraConfigFiles().forEach((fileName, resource) -> resource.writeTo(configDir.resolve(fileName)));
+        }
+
+        private void createKeystore() {
+            if (spec.getKeystorePassword() == null || spec.getKeystorePassword().isEmpty()) {
+                runToolScript("elasticsearch-keystore", null, "-v", "create");
+            } else {
+                runToolScript("elasticsearch-keystore", spec.getKeystorePassword() + "\n" + spec.getKeystorePassword(), "create", "-p");
+            }
+        }
+
+        private void addKeystoreSettings() {
+            spec.getKeystoreSettings().forEach((key, value) -> {
+                String input = spec.getKeystorePassword() == null || spec.getKeystorePassword().isEmpty()
+                    ? value
+                    : spec.getKeystorePassword() + "\n" + value;
+
+                runToolScript("elasticsearch-keystore", input, "add", key);
+            });
+        }
+
+        private void addKeystoreFiles() {
+            spec.getKeystoreFiles().forEach((key, file) -> {
+                try {
+                    Path path = Files.createTempFile(tempDir, key, null);
+                    file.writeTo(path);
+
+                    ProcessUtils.exec(
+                        spec.getKeystorePassword(),
+                        workingDir,
+                        OS.conditional(
+                            c -> c.onWindows(() -> distributionDir.resolve("bin").resolve("elasticsearch-keystore.bat"))
+                                .onUnix(() -> distributionDir.resolve("bin").resolve("elasticsearch-keystore"))
+                        ),
+                        getEnvironmentVariables(),
+                        false,
+                        "add-file",
+                        key,
+                        path.toString()
+                    ).waitFor();
+                } catch (InterruptedException | IOException e) {
+                    throw new RuntimeException(e);
+                }
+            });
+        }
+
+        private void configureSecurity() {
+            if (spec.isSecurityEnabled()) {
+                if (spec.getUsers().isEmpty() == false) {
+                    LOGGER.info("Setting up roles.yml for node '{}'", spec.getName());
+
+                    Path destination = workingDir.resolve("config").resolve("roles.yml");
+                    spec.getRolesFiles().forEach(rolesFile -> {
+                        try (
+                            Writer writer = Files.newBufferedWriter(destination, StandardOpenOption.APPEND);
+                            Reader reader = new BufferedReader(new InputStreamReader(rolesFile.asStream()))
+                        ) {
+                            reader.transferTo(writer);
+                        } catch (IOException e) {
+                            throw new UncheckedIOException("Failed to append roles file " + rolesFile + " to " + destination, e);
+                        }
+                    });
+                }
+
+                LOGGER.info("Creating users for node '{}'", spec.getName());
+                for (User user : spec.getUsers()) {
+                    runToolScript(
+                        "elasticsearch-users",
+                        null,
+                        "useradd",
+                        user.getUsername(),
+                        "-p",
+                        user.getPassword(),
+                        "-r",
+                        user.getRole()
+                    );
+                }
+            }
+        }
+
+        private void installPlugins() {
+            if (spec.getPlugins().isEmpty() == false) {
+                Pattern pattern = Pattern.compile("(.+)(?:-\\d\\.\\d\\.\\d-SNAPSHOT\\.zip)?");
+
+                LOGGER.info("Installing plugins {} into node '{}", spec.getPlugins(), spec.getName());
+                List<Path> pluginPaths = Arrays.stream(System.getProperty(TESTS_CLUSTER_PLUGINS_PATH_SYSPROP).split(File.pathSeparator))
+                    .map(Path::of)
+                    .toList();
+
+                List<String> toInstall = spec.getPlugins()
+                    .stream()
+                    .map(
+                        pluginName -> pluginPaths.stream()
+                            .map(path -> Pair.of(pattern.matcher(path.getFileName().toString()), path))
+                            .filter(pair -> pair.left.matches())
+                            .map(p -> p.right.getParent().resolve(p.left.group(1)))
+                            .findFirst()
+                            .orElseThrow(() -> {
+                                String taskPath = System.getProperty("tests.task");
+                                String project = taskPath.substring(0, taskPath.lastIndexOf(':'));
+
+                                throw new RuntimeException(
+                                    "Unable to locate plugin '"
+                                        + pluginName
+                                        + "'. Ensure you've added the following to the build script for project '"
+                                        + project
+                                        + "':\n\n"
+                                        + "dependencies {\n"
+                                        + "  clusterPlugins "
+                                        + "project(':plugins:"
+                                        + pluginName
+                                        + "')"
+                                        + "\n}"
+                                );
+                            })
+                    )
+                    .map(p -> p.toUri().toString())
+                    .toList();
+
+                if (spec.getVersion().onOrAfter("7.6.0")) {
+                    runToolScript(
+                        "elasticsearch-plugin",
+                        null,
+                        Stream.concat(Stream.of("install", "--batch"), toInstall.stream()).toArray(String[]::new)
+                    );
+                } else {
+                    toInstall.forEach(plugin -> runToolScript("elasticsearch-plugin", null, "install", "--batch", plugin));
+                }
+            }
+        }
+
+        private void installModules() {
+            if (spec.getModules().isEmpty() == false) {
+                LOGGER.info("Installing modules {} into node '{}", spec.getModules(), spec.getName());
+                List<Path> modulePaths = Arrays.stream(System.getProperty(TESTS_CLUSTER_MODULES_PATH_SYSPROP).split(File.pathSeparator))
+                    .map(Path::of)
+                    .toList();
+
+                spec.getModules().forEach(module -> installModule(module, modulePaths));
+            }
+        }
+
+        private void installModule(String moduleName, List<Path> modulePaths) {
+            Path destination = distributionDir.resolve("modules").resolve(moduleName);
+            if (Files.notExists(destination)) {
+                Path modulePath = modulePaths.stream().filter(path -> path.endsWith(moduleName)).findFirst().orElseThrow(() -> {
+                    String taskPath = System.getProperty("tests.task");
+                    String project = taskPath.substring(0, taskPath.lastIndexOf(':'));
+                    String moduleDependency = moduleName.startsWith("x-pack")
+                        ? "project(xpackModule('" + moduleName.substring(7) + "'))"
+                        : "project(':modules:" + moduleName + "')";
+
+                    throw new RuntimeException(
+                        "Unable to locate module '"
+                            + moduleName
+                            + "'. Ensure you've added the following to the build script for project '"
+                            + project
+                            + "':\n\n"
+                            + "dependencies {\n"
+                            + "  clusterModules "
+                            + moduleDependency
+                            + "\n}"
+                    );
+
+                });
+
+                IOUtils.syncWithCopy(modulePath, destination);
+
+                // Install any extended plugins
+                Properties pluginProperties = new Properties();
+                try (
+                    InputStream in = new BufferedInputStream(
+                        new FileInputStream(modulePath.resolve("plugin-descriptor.properties").toFile())
+                    )
+                ) {
+                    pluginProperties.load(in);
+                    String extendedProperty = pluginProperties.getProperty("extended.plugins");
+                    if (extendedProperty != null) {
+                        String[] extendedPlugins = extendedProperty.split(",");
+                        for (String plugin : extendedPlugins) {
+                            installModule(plugin, modulePaths);
+                        }
+                    }
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+            }
+        }
+
+        private void startElasticsearch() {
+            process = ProcessUtils.exec(
+                spec.getKeystorePassword() == null ? null : spec.getKeystorePassword() + "\n",
+                workingDir,
+                OS.conditional(
+                    c -> c.onWindows(() -> distributionDir.resolve("bin").resolve("elasticsearch.bat"))
+                        .onUnix(() -> distributionDir.resolve("bin").resolve("elasticsearch"))
+                ),
+                getEnvironmentVariables(),
+                true
+            );
+
+            ProcessReaper.instance().registerPid(getServiceName(), process.pid());
+        }
+
+        private Map<String, String> getEnvironmentVariables() {
+            Map<String, String> environment = new HashMap<>(spec.resolveEnvironment());
+            environment.put("ES_PATH_CONF", workingDir.resolve("config").toString());
+            environment.put("ES_TMPDIR", workingDir.resolve("tmp").toString());
+            // Windows requires this as it defaults to `c:\windows` despite ES_TMPDIR
+            environment.put("TMP", workingDir.resolve("tmp").toString());
+
+            String featureFlagProperties = "";
+            if (spec.getFeatures().isEmpty() == false && distributionDescriptor.isSnapshot() == false) {
+                featureFlagProperties = spec.getFeatures()
+                    .stream()
+                    .filter(f -> spec.getVersion().onOrAfter(f.from) && (f.until == null || spec.getVersion().before(f.until)))
+                    .map(f -> "-D" + f.systemProperty)
+                    .collect(Collectors.joining(" "));
+            }
+
+            String systemProperties = "";
+            if (spec.getSystemProperties().isEmpty() == false) {
+                systemProperties = spec.getSystemProperties()
+                    .entrySet()
+                    .stream()
+                    .map(entry -> "-D" + entry.getKey() + "=" + entry.getValue())
+                    .map(p -> p.replace("${ES_PATH_CONF}", configDir.toString()))
+                    .collect(Collectors.joining(" "));
+            }
+
+            String debugArgs = "";
+            if (Boolean.getBoolean(TESTS_CLUSTER_DEBUG_ENABLED_SYSPROP)) {
+                int port = DEFAULT_DEBUG_PORT + spec.getCluster().getNodes().indexOf(spec);
+                debugArgs = ENABLE_DEBUG_JVM_ARGS + port;
+            }
+
+            String heapSize = System.getProperty("tests.heap.size", "512m");
+            environment.put("ES_JAVA_OPTS", "-Xms" + heapSize + " -Xmx" + heapSize + " -ea -esa "
+            // Support passing in additional JVM arguments
+                + System.getProperty("tests.jvm.argline", "")
+                + " "
+                + featureFlagProperties
+                + systemProperties
+                + debugArgs);
+
+            return environment;
+        }
+
+        private Map<String, String> getJvmOptionsReplacements() {
+            Path relativeLogsDir = workingDir.relativize(logsDir);
+            return Map.of(
+                "-XX:HeapDumpPath=data",
+                "-XX:HeapDumpPath=" + relativeLogsDir,
+                "logs/gc.log",
+                relativeLogsDir.resolve("gc.log").toString(),
+                "-XX:ErrorFile=logs/hs_err_pid%p.log",
+                "-XX:ErrorFile=" + relativeLogsDir.resolve("hs_err_pid%p.log")
+            );
+        }
+
+        private void runToolScript(String tool, String input, String... args) {
+            try {
+                ProcessUtils.exec(
+                    input,
+                    distributionDir,
+                    distributionDir.resolve("bin")
+                        .resolve(OS.<String>conditional(c -> c.onWindows(() -> tool + ".bat").onUnix(() -> tool))),
+                    getEnvironmentVariables(),
+                    false,
+                    args
+                ).waitFor();
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        private String getServiceName() {
+            return baseWorkingDir.getFileName() + "-" + spec.getCluster().getName() + "-" + spec.getName();
+        }
+
+        @Override
+        public String toString() {
+            return "{ cluster: '" + spec.getCluster().getName() + "', node: '" + spec.getName() + "' }";
+        }
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java
new file mode 100644
index 000000000000..6ad270995729
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java
@@ -0,0 +1,235 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.test.cluster.ClusterHandle;
+import org.elasticsearch.test.cluster.local.LocalClusterFactory.Node;
+import org.elasticsearch.test.cluster.local.model.User;
+import org.elasticsearch.test.cluster.util.ExceptionUtils;
+import org.elasticsearch.test.cluster.util.Version;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.MalformedURLException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.ForkJoinWorkerThread;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class LocalClusterHandle implements ClusterHandle {
+    private static final Logger LOGGER = LogManager.getLogger(LocalClusterHandle.class);
+    private static final Duration CLUSTER_UP_TIMEOUT = Duration.ofSeconds(30);
+
+    public final ForkJoinPool executor = new ForkJoinPool(
+        Math.max(Runtime.getRuntime().availableProcessors(), 4),
+        new ForkJoinPool.ForkJoinWorkerThreadFactory() {
+            private final AtomicLong counter = new AtomicLong(0);
+
+            @Override
+            public ForkJoinWorkerThread newThread(ForkJoinPool pool) {
+                ForkJoinWorkerThread thread = ForkJoinPool.defaultForkJoinWorkerThreadFactory.newThread(pool);
+                thread.setName(name + "-node-executor-" + counter.getAndIncrement());
+                return thread;
+            }
+        },
+        null,
+        false
+    );
+    private final AtomicBoolean started = new AtomicBoolean(false);
+    private final String name;
+    private final List<Node> nodes;
+
+    public LocalClusterHandle(String name, List<Node> nodes) {
+        this.name = name;
+        this.nodes = nodes;
+    }
+
+    @Override
+    public void start() {
+        if (started.getAndSet(true) == false) {
+            LOGGER.info("Starting Elasticsearch test cluster '{}'", name);
+            execute(() -> nodes.parallelStream().forEach(n -> n.start(null)));
+        }
+        waitUntilReady();
+    }
+
+    @Override
+    public void stop(boolean forcibly) {
+        if (started.getAndSet(false)) {
+            LOGGER.info("Stopping Elasticsearch test cluster '{}', forcibly: {}", name, forcibly);
+            execute(() -> nodes.parallelStream().forEach(n -> n.stop(forcibly)));
+        } else {
+            // Make sure the process is stopped, otherwise wait
+            execute(() -> nodes.parallelStream().forEach(Node::waitForExit));
+        }
+    }
+
+    @Override
+    public void restart(boolean forcibly) {
+        stop(forcibly);
+        start();
+    }
+
+    @Override
+    public boolean isStarted() {
+        return started.get();
+    }
+
+    @Override
+    public void close() {
+        stop(false);
+
+        executor.shutdownNow();
+        try {
+            executor.awaitTermination(5, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public String getHttpAddresses() {
+        start();
+        return execute(() -> nodes.parallelStream().map(Node::getHttpAddress).collect(Collectors.joining(",")));
+    }
+
+    @Override
+    public String getHttpAddress(int index) {
+        return getHttpAddresses().split(",")[index];
+    }
+
+    @Override
+    public String getTransportEndpoints() {
+        start();
+        return execute(() -> nodes.parallelStream().map(Node::getTransportEndpoint).collect(Collectors.joining(",")));
+    }
+
+    @Override
+    public String getTransportEndpoint(int index) {
+        return getTransportEndpoints().split(",")[index];
+    }
+
+    @Override
+    public void upgradeNodeToVersion(int index, Version version) {
+        Node node = nodes.get(index);
+        node.stop(false);
+        LOGGER.info("Upgrading node '{}' to version {}", node.getSpec().getName(), version);
+        node.start(version);
+        waitUntilReady();
+    }
+
+    @Override
+    public void upgradeToVersion(Version version) {
+        stop(false);
+        if (started.getAndSet(true) == false) {
+            LOGGER.info("Upgrading Elasticsearch test cluster '{}' to version {}", name, version);
+            execute(() -> nodes.parallelStream().forEach(n -> n.start(version)));
+        }
+        waitUntilReady();
+    }
+
+    private void waitUntilReady() {
+        writeUnicastHostsFile();
+        try {
+            WaitForHttpResource wait = configureWaitForReady();
+            wait.waitFor(CLUSTER_UP_TIMEOUT.toMillis());
+        } catch (Exception e) {
+            throw new RuntimeException("An error occurred while checking cluster '" + name + "' status.", e);
+        }
+    }
+
+    private WaitForHttpResource configureWaitForReady() throws MalformedURLException {
+        Node node = nodes.get(0);
+        boolean securityEnabled = Boolean.parseBoolean(node.getSpec().getSetting("xpack.security.enabled", "true"));
+        boolean sslEnabled = Boolean.parseBoolean(node.getSpec().getSetting("xpack.security.http.ssl.enabled", "false"));
+        boolean securityAutoConfigured = isSecurityAutoConfigured(node);
+        String scheme = securityEnabled && (sslEnabled || securityAutoConfigured) ? "https" : "http";
+        WaitForHttpResource wait = new WaitForHttpResource(scheme, node.getHttpAddress(), nodes.size());
+        User credentials = node.getSpec().getUsers().get(0);
+        wait.setUsername(credentials.getUsername());
+        wait.setPassword(credentials.getPassword());
+        if (sslEnabled) {
+            configureWaitSecurity(wait, node);
+        } else if (securityAutoConfigured) {
+            wait.setCertificateAuthorities(node.getWorkingDir().resolve("config/certs/http_ca.crt").toFile());
+        }
+
+        return wait;
+    }
+
+    private void configureWaitSecurity(WaitForHttpResource wait, Node node) {
+        String caFile = node.getSpec().getSetting("xpack.security.http.ssl.certificate_authorities", null);
+        if (caFile != null) {
+            wait.setCertificateAuthorities(node.getWorkingDir().resolve("config").resolve(caFile).toFile());
+        }
+        String sslCertFile = node.getSpec().getSetting("xpack.security.http.ssl.certificate", null);
+        if (sslCertFile != null) {
+            wait.setCertificateAuthorities(node.getWorkingDir().resolve("config").resolve(sslCertFile).toFile());
+        }
+        String sslKeystoreFile = node.getSpec().getSetting("xpack.security.http.ssl.keystore.path", null);
+        if (sslKeystoreFile != null && caFile == null) { // Can not set both trust stores and CA
+            wait.setTrustStoreFile(node.getWorkingDir().resolve("config").resolve(sslKeystoreFile).toFile());
+        }
+        String keystorePassword = node.getSpec().getSetting("xpack.security.http.ssl.keystore.secure_password", null);
+        if (keystorePassword != null) {
+            wait.setTrustStorePassword(keystorePassword);
+        }
+    }
+
+    private boolean isSecurityAutoConfigured(Node node) {
+        Path configFile = node.getWorkingDir().resolve("config").resolve("elasticsearch.yml");
+        try (Stream<String> lines = Files.lines(configFile)) {
+            return lines.anyMatch(l -> l.contains("BEGIN SECURITY AUTO CONFIGURATION"));
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private void writeUnicastHostsFile() {
+        String transportUris = execute(() -> nodes.parallelStream().map(Node::getTransportEndpoint).collect(Collectors.joining("\n")));
+        execute(() -> nodes.parallelStream().forEach(node -> {
+            try {
+                Path hostsFile = node.getWorkingDir().resolve("config").resolve("unicast_hosts.txt");
+                if (Files.notExists(hostsFile)) {
+                    Files.writeString(hostsFile, transportUris);
+                }
+            } catch (IOException e) {
+                throw new UncheckedIOException("Failed to write unicast_hosts for: " + node, e);
+            }
+        }));
+    }
+
+    private <T> T execute(Callable<T> task) {
+        try {
+            return executor.submit(task).get();
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        } catch (ExecutionException e) {
+            throw new RuntimeException("An error occurred orchestrating test cluster.", ExceptionUtils.findRootCause(e));
+        }
+    }
+
+    private void execute(Runnable task) {
+        execute(() -> {
+            task.run();
+            return true;
+        });
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java
new file mode 100644
index 000000000000..2234b037381a
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java
@@ -0,0 +1,281 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local;
+
+import org.elasticsearch.test.cluster.ClusterSpec;
+import org.elasticsearch.test.cluster.EnvironmentProvider;
+import org.elasticsearch.test.cluster.FeatureFlag;
+import org.elasticsearch.test.cluster.SettingsProvider;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.cluster.local.model.User;
+import org.elasticsearch.test.cluster.util.Version;
+import org.elasticsearch.test.cluster.util.resource.Resource;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class LocalClusterSpec implements ClusterSpec {
+    private final String name;
+    private final List<User> users;
+    private final List<Resource> roleFiles;
+    private List<LocalNodeSpec> nodes;
+
+    public LocalClusterSpec(String name, List<User> users, List<Resource> roleFiles) {
+        this.name = name;
+        this.users = users;
+        this.roleFiles = roleFiles;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public List<User> getUsers() {
+        return users;
+    }
+
+    public List<Resource> getRoleFiles() {
+        return roleFiles;
+    }
+
+    public List<LocalNodeSpec> getNodes() {
+        return nodes;
+    }
+
+    public void setNodes(List<LocalNodeSpec> nodes) {
+        this.nodes = nodes;
+    }
+
+    void validate() {
+        // Ensure we don't have nodes with duplicate names
+        List<String> nodeNames = nodes.stream().map(LocalNodeSpec::getName).collect(Collectors.toList());
+        Set<String> uniqueNames = nodes.stream().map(LocalNodeSpec::getName).collect(Collectors.toSet());
+        uniqueNames.forEach(name -> nodeNames.remove(nodeNames.indexOf(name)));
+
+        if (nodeNames.isEmpty() == false) {
+            throw new IllegalArgumentException("Cluster cannot contain nodes with duplicates names: " + nodeNames);
+        }
+    }
+
+    public static class LocalNodeSpec {
+        private final LocalClusterSpec cluster;
+        private final String name;
+        private final List<SettingsProvider> settingsProviders;
+        private final Map<String, String> settings;
+        private final List<EnvironmentProvider> environmentProviders;
+        private final Map<String, String> environment;
+        private final Set<String> modules;
+        private final Set<String> plugins;
+        private final DistributionType distributionType;
+        private final Set<FeatureFlag> features;
+        private final Map<String, String> keystoreSettings;
+        private final Map<String, Resource> keystoreFiles;
+        private final String keystorePassword;
+        private final Map<String, Resource> extraConfigFiles;
+        private final Map<String, String> systemProperties;
+        private Version version;
+
+        public LocalNodeSpec(
+            LocalClusterSpec cluster,
+            String name,
+            Version version,
+            List<SettingsProvider> settingsProviders,
+            Map<String, String> settings,
+            List<EnvironmentProvider> environmentProviders,
+            Map<String, String> environment,
+            Set<String> modules,
+            Set<String> plugins,
+            DistributionType distributionType,
+            Set<FeatureFlag> features,
+            Map<String, String> keystoreSettings,
+            Map<String, Resource> keystoreFiles,
+            String keystorePassword,
+            Map<String, Resource> extraConfigFiles,
+            Map<String, String> systemProperties
+        ) {
+            this.cluster = cluster;
+            this.name = name;
+            this.version = version;
+            this.settingsProviders = settingsProviders;
+            this.settings = settings;
+            this.environmentProviders = environmentProviders;
+            this.environment = environment;
+            this.modules = modules;
+            this.plugins = plugins;
+            this.distributionType = distributionType;
+            this.features = features;
+            this.keystoreSettings = keystoreSettings;
+            this.keystoreFiles = keystoreFiles;
+            this.keystorePassword = keystorePassword;
+            this.extraConfigFiles = extraConfigFiles;
+            this.systemProperties = systemProperties;
+        }
+
+        void setVersion(Version version) {
+            this.version = version;
+        }
+
+        public LocalClusterSpec getCluster() {
+            return cluster;
+        }
+
+        public String getName() {
+            return name == null ? cluster.getName() + "-" + cluster.getNodes().indexOf(this) : name;
+        }
+
+        public Version getVersion() {
+            return version;
+        }
+
+        public List<User> getUsers() {
+            return cluster.getUsers();
+        }
+
+        public List<Resource> getRolesFiles() {
+            return cluster.getRoleFiles();
+        }
+
+        public DistributionType getDistributionType() {
+            return distributionType;
+        }
+
+        public Set<String> getModules() {
+            return modules;
+        }
+
+        public Set<String> getPlugins() {
+            return plugins;
+        }
+
+        public Set<FeatureFlag> getFeatures() {
+            return features;
+        }
+
+        public Map<String, String> getKeystoreSettings() {
+            return keystoreSettings;
+        }
+
+        public Map<String, Resource> getKeystoreFiles() {
+            return keystoreFiles;
+        }
+
+        public String getKeystorePassword() {
+            return keystorePassword;
+        }
+
+        public Map<String, Resource> getExtraConfigFiles() {
+            return extraConfigFiles;
+        }
+
+        public Map<String, String> getSystemProperties() {
+            return systemProperties;
+        }
+
+        public boolean isSecurityEnabled() {
+            return Boolean.parseBoolean(getSetting("xpack.security.enabled", getVersion().onOrAfter("8.0.0") ? "true" : "false"));
+        }
+
+        public boolean isMasterEligible() {
+            return getSetting("node.roles", "master").contains("master");
+        }
+
+        /**
+         * Return node configured setting or the provided default if no explicit value has been configured. This method returns all
+         * settings, to include security settings provided to the keystore
+         *
+         * @param setting the setting name
+         * @param defaultValue a default value
+         * @return the configured setting value or provided default
+         */
+        public String getSetting(String setting, String defaultValue) {
+            Map<String, String> allSettings = new HashMap<>();
+            allSettings.putAll(resolveSettings());
+            allSettings.putAll(keystoreSettings);
+
+            return allSettings.getOrDefault(setting, defaultValue);
+        }
+
+        /**
+         * Resolve node settings. Order of precedence is as follows:
+         * <ol>
+         *     <li>Settings from cluster configured {@link SettingsProvider}</li>
+         *     <li>Settings from node configured {@link SettingsProvider}</li>
+         *     <li>Explicit cluster settings</li>
+         *     <li>Explicit node settings</li>
+         * </ol>
+         *
+         * @return resolved settings for node
+         */
+        public Map<String, String> resolveSettings() {
+            Map<String, String> resolvedSettings = new HashMap<>();
+            settingsProviders.forEach(p -> resolvedSettings.putAll(p.get(getFilteredSpec(p))));
+            resolvedSettings.putAll(settings);
+            return resolvedSettings;
+        }
+
+        /**
+         * Resolve node environment variables. Order of precedence is as follows:
+         * <ol>
+         *     <li>Environment variables from cluster configured {@link EnvironmentProvider}</li>
+         *     <li>Environment variables from node configured {@link EnvironmentProvider}</li>
+         *     <li>Environment variables cluster settings</li>
+         *     <li>Environment variables node settings</li>
+         * </ol>
+         *
+         * @return resolved environment variables for node
+         */
+        public Map<String, String> resolveEnvironment() {
+            Map<String, String> resolvedEnvironment = new HashMap<>();
+            environmentProviders.forEach(p -> resolvedEnvironment.putAll(p.get(this)));
+            resolvedEnvironment.putAll(environment);
+            return resolvedEnvironment;
+        }
+
+        /**
+         * Returns a new {@link LocalNodeSpec} without the given {@link SettingsProvider}. This is needed when resolving settings from a
+         * settings provider to avoid infinite recursion.
+         *
+         * @param filteredProvider the provider to omit from the new node spec
+         * @return a new local node spec
+         */
+        private LocalNodeSpec getFilteredSpec(SettingsProvider filteredProvider) {
+            LocalClusterSpec newCluster = new LocalClusterSpec(cluster.name, cluster.users, cluster.roleFiles);
+
+            List<LocalNodeSpec> nodeSpecs = cluster.nodes.stream()
+                .map(
+                    n -> new LocalNodeSpec(
+                        newCluster,
+                        n.name,
+                        n.version,
+                        n.settingsProviders.stream().filter(s -> s != filteredProvider).toList(),
+                        n.settings,
+                        n.environmentProviders,
+                        n.environment,
+                        n.modules,
+                        n.plugins,
+                        n.distributionType,
+                        n.features,
+                        n.keystoreSettings,
+                        n.keystoreFiles,
+                        n.keystorePassword,
+                        n.extraConfigFiles,
+                        n.systemProperties
+                    )
+                )
+                .toList();
+
+            newCluster.setNodes(nodeSpecs);
+
+            return nodeSpecs.stream().filter(n -> n.getName().equals(this.getName())).findFirst().get();
+        }
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpecBuilder.java
new file mode 100644
index 000000000000..1f4086fd47fe
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpecBuilder.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.util.resource.Resource;
+
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+public interface LocalClusterSpecBuilder extends LocalSpecBuilder<LocalClusterSpecBuilder> {
+    /**
+     * Sets the node name. By default, "test-cluster" is used.
+     */
+    LocalClusterSpecBuilder name(String name);
+
+    /**
+     * Apply configuration from a {@link LocalClusterConfigProvider}. This configuration is applied eagerly. Subsequent calls to this
+     * builder will override provider settings.
+     */
+    LocalClusterSpecBuilder apply(LocalClusterConfigProvider configProvider);
+
+    /**
+     * Apply configuration from a {@link LocalClusterConfigProvider} created by the given {@link Supplier}. This configuration is applied
+     * lazily and will override existing builder settings.
+     */
+    LocalClusterSpecBuilder apply(Supplier<LocalClusterConfigProvider> configProvider);
+
+    /**
+     * Sets the number of nodes for the cluster.
+     */
+    LocalClusterSpecBuilder nodes(int nodes);
+
+    /**
+     * Adds a new node to the cluster and configures the node.
+     */
+    LocalClusterSpecBuilder withNode(Consumer<? super LocalNodeSpecBuilder> config);
+
+    /**
+     * Configures an existing node.
+     *
+     * @param index the index of the node to configure
+     * @param config configuration to apply to the node
+     */
+    LocalClusterSpecBuilder node(int index, Consumer<? super LocalNodeSpecBuilder> config);
+
+    /**
+     * Register a user using the default test role.
+     */
+    LocalClusterSpecBuilder user(String username, String password);
+
+    /**
+     * Register a user using the given role.
+     */
+    LocalClusterSpecBuilder user(String username, String password, String role);
+
+    /**
+     * Register a roles file with cluster via the supplied {@link Resource}.
+     */
+    LocalClusterSpecBuilder rolesFile(Resource rolesFile);
+
+    ElasticsearchCluster build();
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalElasticsearchCluster.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalElasticsearchCluster.java
new file mode 100644
index 000000000000..9a5e5666f5e9
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalElasticsearchCluster.java
@@ -0,0 +1,121 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.LocalDistributionResolver;
+import org.elasticsearch.test.cluster.local.distribution.ReleasedDistributionResolver;
+import org.elasticsearch.test.cluster.local.distribution.SnapshotDistributionResolver;
+import org.elasticsearch.test.cluster.util.Version;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+import java.nio.file.Path;
+
+public class LocalElasticsearchCluster implements ElasticsearchCluster {
+    private final DefaultLocalClusterSpecBuilder builder;
+    private LocalClusterSpec spec;
+    private LocalClusterHandle handle;
+
+    public LocalElasticsearchCluster(DefaultLocalClusterSpecBuilder builder) {
+        this.builder = builder;
+    }
+
+    @Override
+    public Statement apply(Statement base, Description description) {
+        return new Statement() {
+            @Override
+            public void evaluate() throws Throwable {
+                try {
+                    spec = builder.buildClusterSpec();
+                    handle = new LocalClusterFactory(
+                        Path.of(System.getProperty("java.io.tmpdir")).resolve(description.getDisplayName()).toAbsolutePath(),
+                        new LocalDistributionResolver(new SnapshotDistributionResolver(new ReleasedDistributionResolver()))
+                    ).create(spec);
+                    handle.start();
+                    base.evaluate();
+                } finally {
+                    close();
+                }
+            }
+        };
+    }
+
+    @Override
+    public void start() {
+        checkHandle();
+        handle.start();
+    }
+
+    @Override
+    public void stop(boolean forcibly) {
+        checkHandle();
+        handle.stop(forcibly);
+    }
+
+    @Override
+    public void restart(boolean forcibly) {
+        checkHandle();
+        handle.restart(forcibly);
+    }
+
+    @Override
+    public boolean isStarted() {
+        checkHandle();
+        return handle.isStarted();
+    }
+
+    @Override
+    public void close() {
+        checkHandle();
+        handle.close();
+    }
+
+    @Override
+    public String getHttpAddresses() {
+        checkHandle();
+        return handle.getHttpAddresses();
+    }
+
+    @Override
+    public String getHttpAddress(int index) {
+        checkHandle();
+        return handle.getHttpAddress(index);
+    }
+
+    @Override
+    public String getTransportEndpoints() {
+        checkHandle();
+        return handle.getTransportEndpoints();
+    }
+
+    @Override
+    public String getTransportEndpoint(int index) {
+        checkHandle();
+        return handle.getTransportEndpoint(index);
+    }
+
+    @Override
+    public void upgradeNodeToVersion(int index, Version version) {
+        checkHandle();
+        handle.upgradeNodeToVersion(index, version);
+    }
+
+    @Override
+    public void upgradeToVersion(Version version) {
+        checkHandle();
+        handle.upgradeToVersion(version);
+    }
+
+    private void checkHandle() {
+        if (handle == null) {
+            throw new IllegalStateException("Cluster handle has not been initialized. Did you forget the @ClassRule annotation?");
+        }
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalNodeSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalNodeSpecBuilder.java
new file mode 100644
index 000000000000..f9b28407aaa4
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalNodeSpecBuilder.java
@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local;
+
+public interface LocalNodeSpecBuilder extends LocalSpecBuilder<LocalNodeSpecBuilder> {
+
+    /**
+     * Sets the node name. By default, nodes are named after the cluster with an incrementing suffix (ex: my-cluster-0).
+     */
+    LocalNodeSpecBuilder name(String name);
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java
new file mode 100644
index 000000000000..d01d7d303748
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local;
+
+import org.elasticsearch.test.cluster.EnvironmentProvider;
+import org.elasticsearch.test.cluster.FeatureFlag;
+import org.elasticsearch.test.cluster.SettingsProvider;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.cluster.util.Version;
+import org.elasticsearch.test.cluster.util.resource.Resource;
+
+import java.util.function.Supplier;
+
+interface LocalSpecBuilder<T extends LocalSpecBuilder<?>> {
+    /**
+     * Register a {@link SettingsProvider}.
+     */
+    T settings(SettingsProvider settingsProvider);
+
+    /**
+     * Add a new node setting.
+     */
+    T setting(String setting, String value);
+
+    /**
+     * Add a new node setting computed by the given supplier.
+     */
+    T setting(String setting, Supplier<String> value);
+
+    /**
+     * Register a {@link EnvironmentProvider}.
+     */
+    T environment(EnvironmentProvider environmentProvider);
+
+    /**
+     * Add a new node environment variable.
+     */
+    T environment(String key, String value);
+
+    /**
+     * Set the cluster {@link DistributionType}. By default, the {@link DistributionType#INTEG_TEST} distribution is used.
+     */
+    T distribution(DistributionType type);
+
+    /**
+     * Ensure module is installed into the distribution when using the {@link DistributionType#INTEG_TEST} distribution. This is ignored
+     * when the {@link DistributionType#DEFAULT} is being used.
+     */
+    T module(String moduleName);
+
+    /**
+     * Ensure plugin is installed into the distribution.
+     */
+    T plugin(String pluginName);
+
+    /**
+     * Require feature to be enabled in the cluster.
+     */
+    T feature(FeatureFlag feature);
+
+    /**
+     * Adds a secure setting to the node keystore.
+     */
+    T keystore(String key, String value);
+
+    /**
+     * Adds a secure file to the node keystore.
+     */
+    T keystore(String key, Resource file);
+
+    /**
+     * Sets the security setting keystore password.
+     */
+    T keystorePassword(String password);
+
+    /**
+     * Adds a file to the node config directory
+     */
+    T configFile(String fileName, Resource configFile);
+
+    /**
+     * Sets the version of Elasticsearch. Defaults to {@link Version#CURRENT}.
+     */
+    T version(Version version);
+
+    /**
+     * Adds a system property to node JVM arguments.
+     */
+    T systemProperty(String property, String value);
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/WaitForHttpResource.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/WaitForHttpResource.java
new file mode 100644
index 000000000000..f00e6f13cb31
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/WaitForHttpResource.java
@@ -0,0 +1,219 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.SecureRandom;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateFactory;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManagerFactory;
+
+/**
+ * A utility to wait for a specific HTTP resource to be available, optionally with customized TLS trusted CAs.
+ * This is logically similar to using the Ant Get task to retrieve a resource, but with the difference that it can
+ * access resources that do not use the JRE's default trusted CAs.
+ */
+public class WaitForHttpResource {
+
+    private static final Logger logger = LogManager.getLogger(WaitForHttpResource.class);
+
+    private Set<Integer> validResponseCodes = Collections.singleton(200);
+    private URL url;
+    private Set<File> certificateAuthorities;
+    private File trustStoreFile;
+    private String trustStorePassword;
+    private String username;
+    private String password;
+
+    public WaitForHttpResource(String protocol, String host, int numberOfNodes) throws MalformedURLException {
+        this(new URL(protocol + "://" + host + "/_cluster/health?wait_for_nodes=>=" + numberOfNodes + "&wait_for_status=yellow"));
+    }
+
+    public WaitForHttpResource(URL url) {
+        this.url = url;
+    }
+
+    public void setValidResponseCodes(int... validResponseCodes) {
+        this.validResponseCodes = new HashSet<>(validResponseCodes.length);
+        for (int rc : validResponseCodes) {
+            this.validResponseCodes.add(rc);
+        }
+    }
+
+    public void setCertificateAuthorities(File... certificateAuthorities) {
+        this.certificateAuthorities = new HashSet<>(Arrays.asList(certificateAuthorities));
+    }
+
+    public void setTrustStoreFile(File trustStoreFile) {
+        this.trustStoreFile = trustStoreFile;
+    }
+
+    public void setTrustStorePassword(String trustStorePassword) {
+        this.trustStorePassword = trustStorePassword;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public boolean waitFor(long durationInMs) throws GeneralSecurityException, InterruptedException, IOException {
+        final long waitUntil = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(durationInMs);
+        final long sleep = Long.max(durationInMs / 10, 100);
+
+        final SSLContext ssl;
+        final KeyStore trustStore = buildTrustStore();
+        if (trustStore != null) {
+            ssl = createSslContext(trustStore);
+        } else {
+            ssl = null;
+        }
+        IOException failure = null;
+        while (true) {
+            try {
+                checkResource(ssl);
+                return true;
+            } catch (IOException e) {
+                logger.debug("Failed to access resource [{}]", url, e);
+                failure = e;
+            }
+            if (System.nanoTime() < waitUntil) {
+                Thread.sleep(sleep);
+            } else {
+                throw failure;
+            }
+        }
+    }
+
+    protected void checkResource(SSLContext ssl) throws IOException {
+        final HttpURLConnection connection = buildConnection(ssl);
+        connection.connect();
+        final Integer response = connection.getResponseCode();
+        if (validResponseCodes.contains(response)) {
+            logger.info("Got successful response [{}] from URL [{}]", response, url);
+            return;
+        } else {
+            throw new IOException(response + " " + connection.getResponseMessage());
+        }
+    }
+
+    HttpURLConnection buildConnection(SSLContext ssl) throws IOException {
+        final HttpURLConnection connection = (HttpURLConnection) this.url.openConnection();
+        configureSslContext(connection, ssl);
+        configureBasicAuth(connection);
+        connection.setRequestMethod("GET");
+        return connection;
+    }
+
+    private void configureSslContext(HttpURLConnection connection, SSLContext ssl) {
+        if (ssl != null) {
+            if (connection instanceof HttpsURLConnection) {
+                ((HttpsURLConnection) connection).setSSLSocketFactory(ssl.getSocketFactory());
+            } else {
+                throw new IllegalStateException("SSL trust has been configured, but [" + url + "] is not a 'https' URL");
+            }
+        }
+    }
+
+    private void configureBasicAuth(HttpURLConnection connection) {
+        if (username != null) {
+            if (password == null) {
+                throw new IllegalStateException("Basic Auth user [" + username + "] has been set, but no password has been configured");
+            }
+            connection.setRequestProperty(
+                "Authorization",
+                "Basic " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes(StandardCharsets.UTF_8))
+            );
+        }
+    }
+
+    KeyStore buildTrustStore() throws GeneralSecurityException, IOException {
+        if (this.certificateAuthorities != null) {
+            if (trustStoreFile != null) {
+                throw new IllegalStateException("Cannot specify both truststore and CAs");
+            }
+            return buildTrustStoreFromCA();
+        } else if (trustStoreFile != null) {
+            return buildTrustStoreFromFile();
+        } else {
+            return null;
+        }
+    }
+
+    private KeyStore buildTrustStoreFromFile() throws GeneralSecurityException, IOException {
+        KeyStore keyStore = KeyStore.getInstance(trustStoreFile.getName().endsWith(".jks") ? "JKS" : "PKCS12");
+        try (InputStream input = new FileInputStream(trustStoreFile)) {
+            keyStore.load(input, trustStorePassword == null ? null : trustStorePassword.toCharArray());
+        }
+        return keyStore;
+    }
+
+    private KeyStore buildTrustStoreFromCA() throws GeneralSecurityException, IOException {
+        final KeyStore store = KeyStore.getInstance(KeyStore.getDefaultType());
+        store.load(null, null);
+        final CertificateFactory certFactory = CertificateFactory.getInstance("X.509");
+        int counter = 0;
+        for (File ca : certificateAuthorities) {
+            try (InputStream input = new FileInputStream(ca)) {
+                for (Certificate certificate : certFactory.generateCertificates(input)) {
+                    store.setCertificateEntry("cert-" + counter, certificate);
+                    counter++;
+                }
+            }
+        }
+        return store;
+    }
+
+    private SSLContext createSslContext(KeyStore trustStore) throws GeneralSecurityException {
+        checkForTrustEntry(trustStore);
+        TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+        tmf.init(trustStore);
+        SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
+        sslContext.init(new KeyManager[0], tmf.getTrustManagers(), new SecureRandom());
+        return sslContext;
+    }
+
+    private void checkForTrustEntry(KeyStore trustStore) throws KeyStoreException {
+        Enumeration<String> enumeration = trustStore.aliases();
+        while (enumeration.hasMoreElements()) {
+            if (trustStore.isCertificateEntry(enumeration.nextElement())) {
+                // found trusted cert entry
+                return;
+            }
+        }
+        throw new IllegalStateException("Trust-store does not contain any trusted certificate entries");
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/DefaultDistributionDescriptor.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/DefaultDistributionDescriptor.java
new file mode 100644
index 000000000000..b42834d2694b
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/DefaultDistributionDescriptor.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local.distribution;
+
+import org.elasticsearch.test.cluster.util.Version;
+
+import java.nio.file.Path;
+
+public class DefaultDistributionDescriptor implements DistributionDescriptor {
+    private final Version version;
+    private final boolean snapshot;
+    private final Path distributionDir;
+    private final DistributionType type;
+
+    public DefaultDistributionDescriptor(Version version, boolean snapshot, Path extractedDir, DistributionType type) {
+        this.version = version;
+        this.snapshot = snapshot;
+        this.distributionDir = extractedDir;
+        this.type = type;
+    }
+
+    public Version getVersion() {
+        return version;
+    }
+
+    public boolean isSnapshot() {
+        return snapshot;
+    }
+
+    public Path getDistributionDir() {
+        return distributionDir.resolve("elasticsearch-" + version + (snapshot ? "-SNAPSHOT" : ""));
+    }
+
+    public DistributionType getType() {
+        return type;
+    }
+
+    @Override
+    public String toString() {
+        return "DefaultDistributionDescriptor{"
+            + "version="
+            + version
+            + ", snapshot="
+            + snapshot
+            + ", distributionDir="
+            + distributionDir
+            + ", type="
+            + type
+            + '}';
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/DistributionDescriptor.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/DistributionDescriptor.java
new file mode 100644
index 000000000000..fb213a87d038
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/DistributionDescriptor.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local.distribution;
+
+import org.elasticsearch.test.cluster.util.Version;
+
+import java.nio.file.Path;
+
+public interface DistributionDescriptor {
+    Version getVersion();
+
+    boolean isSnapshot();
+
+    Path getDistributionDir();
+
+    DistributionType getType();
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/DistributionResolver.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/DistributionResolver.java
new file mode 100644
index 000000000000..93e6d20d7071
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/DistributionResolver.java
@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local.distribution;
+
+import org.elasticsearch.test.cluster.util.Version;
+
+public interface DistributionResolver {
+
+    DistributionDescriptor resolve(Version version, DistributionType type);
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/DistributionType.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/DistributionType.java
new file mode 100644
index 000000000000..6e19c0fa160d
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/DistributionType.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local.distribution;
+
+public enum DistributionType {
+    INTEG_TEST("tests.integ-test.distribution"),
+    DEFAULT("tests.default.distribution");
+
+    private final String systemProperty;
+
+    DistributionType(String systemProperty) {
+        this.systemProperty = systemProperty;
+    }
+
+    public String getSystemProperty() {
+        return systemProperty;
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/LocalDistributionResolver.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/LocalDistributionResolver.java
new file mode 100644
index 000000000000..b9442b28e159
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/LocalDistributionResolver.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local.distribution;
+
+import org.elasticsearch.test.cluster.util.Version;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * A {@link DistributionResolver} for resolving locally built distributions for the current version of Elasticsearch.
+ */
+public class LocalDistributionResolver implements DistributionResolver {
+    private final DistributionResolver delegate;
+
+    public LocalDistributionResolver(DistributionResolver delegate) {
+        this.delegate = delegate;
+    }
+
+    @Override
+    public DistributionDescriptor resolve(Version version, DistributionType type) {
+        if (version.equals(Version.CURRENT)) {
+            String testDistributionPath = System.getProperty(type.getSystemProperty());
+
+            if (testDistributionPath == null) {
+                if (type == DistributionType.DEFAULT) {
+                    String taskPath = System.getProperty("tests.task");
+                    String project = taskPath.substring(0, taskPath.lastIndexOf(':'));
+                    String taskName = taskPath.substring(taskPath.lastIndexOf(':') + 1);
+
+                    throw new IllegalStateException(
+                        "Cannot locate Elasticsearch distribution. Ensure you've added the following to the build script for project '"
+                            + project
+                            + "':\n\n"
+                            + "tasks.named('"
+                            + taskName
+                            + "') {\n"
+                            + "  usesDefaultDistribution()\n"
+                            + "}"
+                    );
+                } else {
+                    throw new IllegalStateException(
+                        "Cannot locate Elasticsearch distribution. The '" + type.getSystemProperty() + "' system property is null."
+                    );
+                }
+            }
+
+            Path testDistributionDir = Path.of(testDistributionPath);
+            if (Files.notExists(testDistributionDir)) {
+                throw new IllegalStateException(
+                    "Cannot locate Elasticsearch distribution. Directory at '" + testDistributionDir + "' does not exist."
+                );
+            }
+
+            return new DefaultDistributionDescriptor(
+                version,
+                Boolean.parseBoolean(System.getProperty("build.snapshot", "true")),
+                testDistributionDir,
+                type
+            );
+        }
+
+        return delegate.resolve(version, type);
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/ReleasedDistributionResolver.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/ReleasedDistributionResolver.java
new file mode 100644
index 000000000000..12654be310ef
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/ReleasedDistributionResolver.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local.distribution;
+
+import org.elasticsearch.test.cluster.util.Version;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * A {@link DistributionResolver} for resolving previously released distributions of Elasticsearch.
+ */
+public class ReleasedDistributionResolver implements DistributionResolver {
+    private static final String BWC_DISTRIBUTION_SYSPROP_PREFIX = "tests.release.distribution.";
+
+    @Override
+    public DistributionDescriptor resolve(Version version, DistributionType type) {
+        String distributionPath = System.getProperty(BWC_DISTRIBUTION_SYSPROP_PREFIX + version.toString());
+
+        if (distributionPath == null) {
+            String taskPath = System.getProperty("tests.task");
+            String project = taskPath.substring(0, taskPath.lastIndexOf(':'));
+            String taskName = taskPath.substring(taskPath.lastIndexOf(':') + 1);
+
+            throw new IllegalStateException(
+                "Cannot locate Elasticsearch distribution. Ensure you've added the following to the build script for project '"
+                    + project
+                    + "':\n\n"
+                    + "tasks.named('"
+                    + taskName
+                    + "') {\n"
+                    + "  usesBwcDistribution("
+                    + version
+                    + ")\n"
+                    + "}"
+            );
+        }
+
+        Path distributionDir = Path.of(distributionPath);
+        if (Files.notExists(distributionDir)) {
+            throw new IllegalStateException(
+                "Cannot locate Elasticsearch distribution. Directory at '" + distributionDir + "' does not exist."
+            );
+        }
+
+        return new DefaultDistributionDescriptor(version, false, distributionDir, DistributionType.DEFAULT);
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/SnapshotDistributionResolver.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/SnapshotDistributionResolver.java
new file mode 100644
index 000000000000..c6cecf09e9b9
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/distribution/SnapshotDistributionResolver.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local.distribution;
+
+import org.elasticsearch.test.cluster.util.Version;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * A {@link DistributionResolver} for resolving snapshot versions of Elasticsearch for previous, backwards-compatible versions.
+ */
+public class SnapshotDistributionResolver implements DistributionResolver {
+    private static final String BWC_DISTRIBUTION_SYSPROP_PREFIX = "tests.snapshot.distribution.";
+    private final DistributionResolver delegate;
+
+    public SnapshotDistributionResolver(DistributionResolver delegate) {
+        this.delegate = delegate;
+    }
+
+    @Override
+    public DistributionDescriptor resolve(Version version, DistributionType type) {
+        String distributionPath = System.getProperty(BWC_DISTRIBUTION_SYSPROP_PREFIX + version.toString());
+
+        if (distributionPath != null) {
+            Path distributionDir = Path.of(distributionPath);
+            if (Files.notExists(distributionDir)) {
+                throw new IllegalStateException(
+                    "Cannot locate Elasticsearch distribution. Directory at '" + distributionDir + "' does not exist."
+                );
+            }
+
+            // Snapshot distributions are never release builds and always use the default distribution
+            return new DefaultDistributionDescriptor(version, true, distributionDir, DistributionType.DEFAULT);
+        }
+
+        return delegate.resolve(version, type);
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/model/User.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/model/User.java
new file mode 100644
index 000000000000..f056bbb4d0bd
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/model/User.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local.model;
+
+public class User {
+    public static final User DEFAULT_USER = new User("test_user", "x-pack-test-password", "_es_test_root");
+
+    private final String username;
+    private final String password;
+    private final String role;
+
+    public User(String username, String password) {
+        this.username = username;
+        this.password = password;
+        this.role = "_es_test_root";
+    }
+
+    public User(String username, String password, String role) {
+        this.username = username;
+        this.password = password;
+        this.role = role;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public String getRole() {
+        return role;
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/ExceptionUtils.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/ExceptionUtils.java
new file mode 100644
index 000000000000..dd3fc999d8b0
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/ExceptionUtils.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.util;
+
+public final class ExceptionUtils {
+    private ExceptionUtils() {}
+
+    public static Throwable findRootCause(Throwable t) {
+        Throwable cause = t.getCause();
+        if (cause == null) {
+            return t;
+        }
+
+        return findRootCause(cause);
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/IOUtils.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/IOUtils.java
new file mode 100644
index 000000000000..8b7f2c167cb1
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/IOUtils.java
@@ -0,0 +1,176 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.util.Comparator;
+import java.util.function.BiConsumer;
+import java.util.stream.Stream;
+
+public final class IOUtils {
+    private static final int RETRY_DELETE_MILLIS = OS.current() == OS.WINDOWS ? 500 : 0;
+    private static final int MAX_RETRY_DELETE_TIMES = OS.current() == OS.WINDOWS ? 15 : 0;
+
+    private IOUtils() {}
+
+    /**
+     * Deletes a path, retrying if necessary.
+     *
+     * @param path  the path to delete
+     * @throws IOException
+     *         if an I/O error occurs
+     */
+    public static void deleteWithRetry(Path path) throws IOException {
+        try {
+            deleteWithRetry0(path);
+        } catch (InterruptedException x) {
+            throw new IOException("Interrupted while deleting.", x);
+        }
+    }
+
+    /** Unchecked variant of deleteWithRetry. */
+    public static void uncheckedDeleteWithRetry(Path path) {
+        try {
+            deleteWithRetry0(path);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        } catch (InterruptedException x) {
+            throw new UncheckedIOException("Interrupted while deleting.", new IOException());
+        }
+    }
+
+    /**
+     * Does the equivalent of `cp -lr` and `chmod -r a-w` to save space and improve speed.
+     * We remove write permissions to make sure files are note mistakenly edited ( e.x. the config file ) and changes
+     * reflected across all copies. Permissions are retained to be able to replace the links.
+     *
+     * @param sourceRoot      where to copy from
+     * @param destinationRoot destination to link to
+     */
+    public static void syncWithLinks(Path sourceRoot, Path destinationRoot) {
+        sync(sourceRoot, destinationRoot, (Path d, Path s) -> {
+            try {
+                Files.createLink(d, s);
+            } catch (IOException e) {
+                // Note does not work for network drives, e.g. Vagrant
+                throw new LinkCreationException("Failed to create hard link " + d + " pointing to " + s, e);
+            }
+        });
+    }
+
+    /**
+     * Sync source folder to destination folder. This method does an actual copy of file contents. When possible,
+     * {@link #syncWithLinks(Path, Path)} is preferred for better performance when the synced contents don't need to be subsequently
+     * modified.
+     *
+     * @param sourceRoot      where to copy from
+     * @param destinationRoot destination to link to
+     */
+    public static void syncWithCopy(Path sourceRoot, Path destinationRoot) {
+        sync(sourceRoot, destinationRoot, (Path d, Path s) -> {
+            try {
+                Files.copy(s, d);
+            } catch (IOException e) {
+                throw new UncheckedIOException("Failed to copy " + s + " to " + d, e);
+            }
+        });
+    }
+
+    private static void sync(Path sourceRoot, Path destinationRoot, BiConsumer<Path, Path> syncMethod) {
+        assert Files.exists(destinationRoot) == false;
+        try (Stream<Path> stream = Files.walk(sourceRoot)) {
+            stream.forEach(source -> {
+                Path relativeDestination = sourceRoot.relativize(source);
+
+                Path destination = destinationRoot.resolve(relativeDestination);
+                if (Files.isDirectory(source)) {
+                    try {
+                        Files.createDirectories(destination);
+                    } catch (IOException e) {
+                        throw new UncheckedIOException("Can't create directory " + destination.getParent(), e);
+                    }
+                } else {
+                    try {
+                        Files.createDirectories(destination.getParent());
+                    } catch (IOException e) {
+                        throw new UncheckedIOException("Can't create directory " + destination.getParent(), e);
+                    }
+                    syncMethod.accept(destination, source);
+                }
+            });
+        } catch (UncheckedIOException e) {
+            if (e.getCause()instanceof NoSuchFileException cause) {
+                // Ignore these files that are sometimes left behind by the JVM
+                if (cause.getFile() == null || cause.getFile().contains(".attach_pid") == false) {
+                    throw new UncheckedIOException(cause);
+                }
+            } else {
+                throw e;
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException("Can't walk source " + sourceRoot, e);
+        }
+    }
+
+    // The exception handling here is loathsome, but necessary!
+    // TODO: Some of the loathsomeness here was copied from our Gradle plugin that was required because of Gradle exception wrapping. That
+    // may no longer be strictly necessary in this context.
+    private static void deleteWithRetry0(Path path) throws IOException, InterruptedException {
+        int times = 0;
+        IOException ioe = null;
+        while (true) {
+            try {
+                recursiveDelete(path);
+                times++;
+                // Checks for absence of the file. Semantics of Files.exists() is not the same.
+                while (Files.notExists(path) == false) {
+                    if (times > MAX_RETRY_DELETE_TIMES) {
+                        throw new IOException("File still exists after " + times + " waits.");
+                    }
+                    Thread.sleep(RETRY_DELETE_MILLIS);
+                    // retry
+                    recursiveDelete(path);
+                    times++;
+                }
+                break;
+            } catch (NoSuchFileException ignore) {
+                // already deleted, ignore
+                break;
+            } catch (IOException x) {
+                if (x.getCause() instanceof NoSuchFileException) {
+                    // already deleted, ignore
+                    break;
+                }
+                // Backoff/retry in case another process is accessing the file
+                times++;
+                if (ioe == null) ioe = new IOException();
+                ioe.addSuppressed(x);
+                if (times > MAX_RETRY_DELETE_TIMES) throw ioe;
+                Thread.sleep(RETRY_DELETE_MILLIS);
+            }
+        }
+    }
+
+    private static void recursiveDelete(Path path) throws IOException {
+        try (Stream<Path> files = Files.walk(path)) {
+            files.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
+        }
+    }
+
+    public static class LinkCreationException extends UncheckedIOException {
+        LinkCreationException(String message, IOException cause) {
+            super(message, cause);
+        }
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/OS.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/OS.java
new file mode 100644
index 000000000000..7ea0daaa3b24
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/OS.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.util;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+public enum OS {
+    WINDOWS,
+    MAC,
+    LINUX;
+
+    public static OS current() {
+        String os = System.getProperty("os.name", "");
+        if (os.startsWith("Windows")) {
+            return OS.WINDOWS;
+        }
+        if (os.startsWith("Linux") || os.startsWith("LINUX")) {
+            return OS.LINUX;
+        }
+        if (os.startsWith("Mac")) {
+            return OS.MAC;
+        }
+        throw new IllegalStateException("Can't determine OS from: " + os);
+    }
+
+    public static class Conditional<T> {
+
+        private final Map<OS, Supplier<? extends T>> conditions = new HashMap<>();
+
+        public Conditional<T> onWindows(Supplier<? extends T> supplier) {
+            conditions.put(WINDOWS, supplier);
+            return this;
+        }
+
+        public Conditional<T> onLinux(Supplier<? extends T> supplier) {
+            conditions.put(LINUX, supplier);
+            return this;
+        }
+
+        public Conditional<T> onMac(Supplier<? extends T> supplier) {
+            conditions.put(MAC, supplier);
+            return this;
+        }
+
+        public Conditional<T> onUnix(Supplier<? extends T> supplier) {
+            conditions.put(MAC, supplier);
+            conditions.put(LINUX, supplier);
+            return this;
+        }
+
+        T supply() {
+            HashSet<OS> missingOS = new HashSet<>(Arrays.asList(OS.values()));
+            missingOS.removeAll(conditions.keySet());
+            if (missingOS.isEmpty() == false) {
+                throw new IllegalArgumentException("No condition specified for " + missingOS);
+            }
+            return conditions.get(OS.current()).get();
+        }
+
+    }
+
+    public static <T> T conditional(Consumer<Conditional<T>> config) {
+        Conditional<T> conditional = new Conditional<>();
+        config.accept(conditional);
+
+        return conditional.supply();
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/Pair.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/Pair.java
new file mode 100644
index 000000000000..863cbce1c9e9
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/Pair.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.util;
+
+import java.util.Objects;
+
+public final class Pair<L, R> {
+    public final L left;
+    public final R right;
+
+    private Pair(L left, R right) {
+        this.left = left;
+        this.right = right;
+    }
+
+    public static <L, R> Pair<L, R> of(L left, R right) {
+        return new Pair<>(left, right);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Pair<?, ?> pair = (Pair<?, ?>) o;
+        return Objects.equals(left, pair.left) && Objects.equals(right, pair.right);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(left, right);
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/ProcessReaper.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/ProcessReaper.java
new file mode 100644
index 000000000000..7d3d19a204cb
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/ProcessReaper.java
@@ -0,0 +1,153 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.util;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class ProcessReaper {
+    private static final String REAPER_CLASS = "org/elasticsearch/gradle/reaper/Reaper.class";
+    private static final Pattern REAPER_JAR_PATH_PATTERN = Pattern.compile("file:(.*)!/" + REAPER_CLASS);
+    private static final Logger LOGGER = LogManager.getLogger(ProcessReaper.class);
+    private static final ProcessReaper INSTANCE = new ProcessReaper();
+    private final Path reaperDir;
+
+    private volatile Process process;
+
+    private ProcessReaper() {
+        try {
+            this.reaperDir = Files.createTempDirectory("reaper-");
+        } catch (IOException e) {
+            throw new UncheckedIOException("Failed to create process reaper working directory.", e);
+        }
+
+        Runtime.getRuntime().addShutdownHook(new Thread(this::shutdown, "process-reaper-shutdown"));
+    }
+
+    public static ProcessReaper instance() {
+        return INSTANCE;
+    }
+
+    /**
+     * Register a pid that will be killed by the reaper.
+     */
+    public void registerPid(String serviceId, long pid) {
+        String[] killPidCommand = OS.conditional(
+            c -> c.onWindows(() -> new String[] { "Taskkill", "/F", "/PID", String.valueOf(pid) })
+                .onUnix(() -> new String[] { "kill", "-9", String.valueOf(pid) })
+        );
+        registerCommand(serviceId, killPidCommand);
+    }
+
+    public void unregister(String serviceId) {
+        try {
+            Files.deleteIfExists(getCmdFile(serviceId));
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private void registerCommand(String serviceId, String... command) {
+        ensureReaperStarted();
+
+        try {
+            Files.writeString(getCmdFile(serviceId), String.join(" ", command));
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private Path getCmdFile(String serviceId) {
+        return reaperDir.resolve(serviceId.replaceAll("[^a-zA-Z0-9]", "-") + ".cmd");
+    }
+
+    void shutdown() {
+        if (process != null) {
+            ensureReaperAlive();
+            try {
+                process.getOutputStream().close();
+                LOGGER.info("Waiting for reaper to exit normally");
+                if (process.waitFor() != 0) {
+                    throw new RuntimeException("Reaper process failed. Check log at " + reaperDir.resolve("reaper.log") + " for details");
+                }
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+
+        }
+    }
+
+    private synchronized void ensureReaperStarted() {
+        if (process == null) {
+            try {
+                Path jarPath = locateReaperJar();
+
+                // ensure the input directory exists
+                Files.createDirectories(reaperDir);
+                // start the reaper
+                ProcessBuilder builder = new ProcessBuilder(
+                    findJavaHome().resolve("bin")
+                        .resolve(OS.<String>conditional(c -> c.onWindows(() -> "java.exe").onUnix(() -> "java")))
+                        .toString(), // same jvm as gradle
+                    "-Xms4m",
+                    "-Xmx16m", // no need for a big heap, just need to read some files and execute
+                    "-jar",
+                    jarPath.toString(),
+                    reaperDir.toString()
+                );
+                LOGGER.info("Launching reaper: " + String.join(" ", builder.command()));
+                // be explicit for stdin, we use closing of the pipe to signal shutdown to the reaper
+                builder.redirectInput(ProcessBuilder.Redirect.PIPE);
+                File logFile = reaperDir.resolve("reaper.log").toFile();
+                builder.redirectOutput(logFile);
+                builder.redirectErrorStream();
+                process = builder.start();
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        } else {
+            ensureReaperAlive();
+        }
+    }
+
+    private Path locateReaperJar() {
+        URL main = this.getClass().getClassLoader().getResource(REAPER_CLASS);
+        if (main != null) {
+            String mainPath = main.getFile();
+            Matcher matcher = REAPER_JAR_PATH_PATTERN.matcher(mainPath);
+
+            if (matcher.matches()) {
+                String path = matcher.group(1);
+                return Path.of(OS.<String>conditional(c -> c.onWindows(() -> path.substring(1)).onUnix(() -> path)));
+            }
+        }
+
+        throw new RuntimeException("Unable to locate " + REAPER_CLASS + " on classpath.");
+    }
+
+    private void ensureReaperAlive() {
+        if (process.isAlive() == false) {
+            throw new IllegalStateException("Reaper process died unexpectedly! Check the log at " + reaperDir.resolve("reaper.log"));
+        }
+    }
+
+    private Path findJavaHome() {
+        Path javaBase = Path.of(System.getProperty("java.home"));
+        return javaBase.endsWith("jre") && Files.exists(javaBase.getParent().resolve("bin/java")) ? javaBase.getParent() : javaBase;
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/ProcessUtils.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/ProcessUtils.java
new file mode 100644
index 000000000000..b70ff1d1242f
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/ProcessUtils.java
@@ -0,0 +1,170 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.util;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Consumer;
+
+public final class ProcessUtils {
+    private static final Logger LOGGER = LogManager.getLogger(ProcessUtils.class);
+    private static final Logger PROCESS_LOGGER = LogManager.getLogger("process-output");
+    private static final Duration PROCESS_DESTROY_TIMEOUT = Duration.ofSeconds(20);
+
+    private ProcessUtils() {}
+
+    public static Process exec(Path workingDir, Path executable, Map<String, String> environment, boolean inheritIO, String... args) {
+        return exec(null, workingDir, executable, environment, inheritIO, args);
+    }
+
+    public static Process exec(
+        String input,
+        Path workingDir,
+        Path executable,
+        Map<String, String> environment,
+        boolean inheritIO,
+        String... args
+    ) {
+        Process process;
+
+        if (Files.exists(executable) == false) {
+            throw new IllegalArgumentException("Can't run executable: `" + executable + "` does not exist.");
+        }
+
+        ProcessBuilder processBuilder = new ProcessBuilder();
+        List<String> command = new ArrayList<>();
+        command.addAll(
+            OS.conditional(
+                c -> c.onWindows(() -> List.of("cmd", "/c", workingDir.relativize(executable).toString()))
+                    .onUnix(() -> List.of(workingDir.relativize(executable).toString()))
+            )
+        );
+        command.addAll(Arrays.asList(args));
+
+        processBuilder.command(command);
+        processBuilder.directory(workingDir.toFile());
+        processBuilder.environment().clear();
+        processBuilder.environment().putAll(environment);
+
+        LOGGER.info("Executing '{}' in '{}'", command, workingDir);
+
+        try {
+            process = processBuilder.start();
+
+            startLoggingThread(
+                process.getInputStream(),
+                inheritIO ? System.out::println : PROCESS_LOGGER::info,
+                executable.getFileName().toString()
+            );
+
+            startLoggingThread(
+                process.getErrorStream(),
+                inheritIO ? System.err::println : PROCESS_LOGGER::error,
+                executable.getFileName().toString()
+            );
+
+            if (input != null) {
+                try (BufferedWriter writer = process.outputWriter()) {
+                    writer.write(input);
+                }
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException("Error executing process: " + executable.getFileName(), e);
+        }
+
+        return process;
+    }
+
+    public static void stopHandle(ProcessHandle processHandle, boolean forcibly) {
+        // No-op if the process has already exited by itself.
+        if (processHandle.isAlive() == false) {
+            return;
+        }
+
+        // Stop all children last - if the ML processes are killed before the ES JVM then
+        // they'll be recorded as having failed and won't restart when the cluster restarts.
+        // ES could actually be a child when there's some wrapper process like on Windows,
+        // and in that case the ML processes will be grandchildren of the wrapper.
+        List<ProcessHandle> children = processHandle.children().toList();
+        try {
+            LOGGER.info("Terminating Elasticsearch process {}: {}", forcibly ? " forcibly " : "gracefully", processHandle.info());
+
+            if (forcibly) {
+                processHandle.destroyForcibly();
+            } else {
+                processHandle.destroy();
+                waitForProcessToExit(processHandle);
+                if (processHandle.isAlive() == false) {
+                    return;
+                }
+                LOGGER.info("Process did not terminate after {}, stopping it forcefully", PROCESS_DESTROY_TIMEOUT);
+                processHandle.destroyForcibly();
+            }
+
+            waitForProcessToExit(processHandle);
+
+            if (processHandle.isAlive()) {
+                throw new RuntimeException("Failed to terminate terminate elasticsearch process.");
+            }
+        } finally {
+            children.forEach(each -> stopHandle(each, forcibly));
+        }
+    }
+
+    public static void waitForExit(ProcessHandle processHandle) {
+        // No-op if the process has already exited by itself.
+        if (processHandle.isAlive() == false) {
+            return;
+        }
+
+        waitForProcessToExit(processHandle);
+    }
+
+    private static void waitForProcessToExit(ProcessHandle processHandle) {
+        try {
+            Retry.retryUntilTrue(PROCESS_DESTROY_TIMEOUT, Duration.ofSeconds(1), () -> {
+                processHandle.destroy();
+                return processHandle.isAlive() == false;
+            });
+        } catch (ExecutionException e) {
+            LOGGER.info("Failure while waiting for process to exist", e);
+        } catch (TimeoutException e) {
+            LOGGER.info("Timed out waiting for process to exit", e);
+        }
+    }
+
+    private static void startLoggingThread(InputStream is, Consumer<String> logAppender, String name) {
+        new Thread(() -> {
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(is))) {
+                String line;
+                while ((line = reader.readLine()) != null) {
+                    logAppender.accept(line);
+                }
+            } catch (IOException e) {
+                throw new UncheckedIOException("Error reading output from process.", e);
+            }
+        }, name).start();
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/Retry.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/Retry.java
new file mode 100644
index 000000000000..6ea36a38b717
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/Retry.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.util;
+
+import java.time.Duration;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+public final class Retry {
+    private static final Executor EXECUTOR = new Executor() {
+        @Override
+        public void execute(Runnable command) {
+            new Thread(command).start();
+        }
+    };
+
+    private Retry() {}
+
+    public static void retryUntilTrue(Duration timeout, Duration delay, Callable<Boolean> predicate) throws TimeoutException,
+        ExecutionException {
+        getValueWithTimeout(timeout.toMillis(), TimeUnit.MILLISECONDS, () -> {
+            while (true) {
+                Boolean call = predicate.call();
+                if (call) {
+                    return true;
+                }
+
+                Thread.sleep(delay.toMillis());
+            }
+        });
+    }
+
+    private static <T> T getValueWithTimeout(long timeout, TimeUnit timeUnit, Callable<T> predicate) throws TimeoutException,
+        ExecutionException {
+        CompletableFuture<T> future = CompletableFuture.supplyAsync(() -> {
+            try {
+                return predicate.call();
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }, EXECUTOR);
+
+        try {
+            return future.get(timeout, timeUnit);
+        } catch (InterruptedException e) {
+            throw new TimeoutException();
+        }
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/Version.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/Version.java
new file mode 100644
index 000000000000..b83a6e8edf6c
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/Version.java
@@ -0,0 +1,177 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.test.cluster.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Serializable;
+import java.io.UncheckedIOException;
+import java.util.Objects;
+import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Encapsulates comparison and printing logic for an x.y.z version.
+ */
+public final class Version implements Comparable<Version>, Serializable {
+    public static final Version CURRENT;
+    private static final Pattern pattern = Pattern.compile("(\\d+)\\.(\\d+)\\.(\\d+)(?:-(alpha\\d+|beta\\d+|rc\\d+|SNAPSHOT))?");
+    private static final Pattern relaxedPattern = Pattern.compile(
+        "v?(\\d+)\\.(\\d+)(?:\\.(\\d+))?(?:[\\-+]+([a-zA-Z0-9_]+(?:-[a-zA-Z0-9]+)*))?"
+    );
+
+    private final int major;
+    private final int minor;
+    private final int revision;
+    private final int id;
+    private final String qualifier;
+
+    static {
+        Properties versionProperties = new Properties();
+        try (InputStream in = Version.class.getClassLoader().getResourceAsStream("version.properties")) {
+            versionProperties.load(in);
+            CURRENT = Version.fromString(versionProperties.getProperty("elasticsearch"));
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    /**
+     * Specifies how a version string should be parsed.
+     */
+    public enum Mode {
+        /**
+         * Strict parsing only allows known suffixes after the patch number: "-alpha", "-beta" or "-rc". The
+         * suffix "-SNAPSHOT" is also allowed, either after the patch number, or after the other suffices.
+         */
+        STRICT,
+
+        /**
+         * Relaxed parsing allows any alphanumeric suffix after the patch number.
+         */
+        RELAXED;
+    }
+
+    public Version(int major, int minor, int revision) {
+        this(major, minor, revision, null);
+    }
+
+    public Version(int major, int minor, int revision, String qualifier) {
+        this.major = major;
+        this.minor = minor;
+        this.revision = revision;
+
+        // currently qualifier is not taken into account
+        this.id = major * 10000000 + minor * 100000 + revision * 1000;
+
+        this.qualifier = qualifier;
+    }
+
+    public static Version fromString(final String s) {
+        return fromString(s, Mode.STRICT);
+    }
+
+    public static Version fromString(final String s, final Mode mode) {
+        Objects.requireNonNull(s);
+        Matcher matcher = mode == Mode.STRICT ? pattern.matcher(s) : relaxedPattern.matcher(s);
+        if (matcher.matches() == false) {
+            String expected = mode == Mode.STRICT
+                ? "major.minor.revision[-(alpha|beta|rc)Number|-SNAPSHOT]"
+                : "major.minor.revision[-extra]";
+            throw new IllegalArgumentException("Invalid version format: '" + s + "'. Should be " + expected);
+        }
+
+        String major = matcher.group(1);
+        String minor = matcher.group(2);
+        String revision = matcher.group(3);
+        String qualifier = matcher.group(4);
+
+        return new Version(Integer.parseInt(major), Integer.parseInt(minor), revision == null ? 0 : Integer.parseInt(revision), qualifier);
+    }
+
+    @Override
+    public String toString() {
+        return getMajor() + "." + getMinor() + "." + getRevision();
+    }
+
+    public boolean before(Version compareTo) {
+        return id < compareTo.getId();
+    }
+
+    public boolean before(String compareTo) {
+        return before(fromString(compareTo));
+    }
+
+    public boolean onOrBefore(Version compareTo) {
+        return id <= compareTo.getId();
+    }
+
+    public boolean onOrBefore(String compareTo) {
+        return onOrBefore(fromString(compareTo));
+    }
+
+    public boolean onOrAfter(Version compareTo) {
+        return id >= compareTo.getId();
+    }
+
+    public boolean onOrAfter(String compareTo) {
+        return onOrAfter(fromString(compareTo));
+    }
+
+    public boolean after(Version compareTo) {
+        return id > compareTo.getId();
+    }
+
+    public boolean after(String compareTo) {
+        return after(fromString(compareTo));
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        Version version = (Version) o;
+        return major == version.major && minor == version.minor && revision == version.revision;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(major, minor, revision, id);
+    }
+
+    public int getMajor() {
+        return major;
+    }
+
+    public int getMinor() {
+        return minor;
+    }
+
+    public int getRevision() {
+        return revision;
+    }
+
+    protected int getId() {
+        return id;
+    }
+
+    public String getQualifier() {
+        return qualifier;
+    }
+
+    @Override
+    public int compareTo(Version other) {
+        return Integer.compare(getId(), other.getId());
+    }
+
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/resource/ClasspathResource.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/resource/ClasspathResource.java
new file mode 100644
index 000000000000..6d6998a6b176
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/resource/ClasspathResource.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.util.resource;
+
+import java.io.InputStream;
+import java.util.HashSet;
+import java.util.Set;
+
+class ClasspathResource implements Resource {
+    private final String resourcePath;
+
+    ClasspathResource(String resourcePath) {
+        this.resourcePath = resourcePath;
+    }
+
+    @Override
+    public InputStream asStream() {
+        final Set<ClassLoader> classLoadersToSearch = new HashSet<>();
+        // try context and system classloaders as well
+        classLoadersToSearch.add(Thread.currentThread().getContextClassLoader());
+        classLoadersToSearch.add(ClassLoader.getSystemClassLoader());
+        classLoadersToSearch.add(ClasspathResource.class.getClassLoader());
+
+        for (final ClassLoader classLoader : classLoadersToSearch) {
+            if (classLoader == null) {
+                continue;
+            }
+
+            InputStream resource = classLoader.getResourceAsStream(resourcePath);
+            if (resource != null) {
+                return resource;
+            }
+
+            // Be lenient if an absolute path was given
+            if (resourcePath.startsWith("/")) {
+                resource = classLoader.getResourceAsStream(resourcePath.replaceFirst("/", ""));
+                if (resource != null) {
+                    return resource;
+                }
+            }
+        }
+
+        throw new IllegalArgumentException(
+            "Resource with path " + resourcePath + " could not be found on any of these classloaders: " + classLoadersToSearch
+        );
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/resource/FileResource.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/resource/FileResource.java
new file mode 100644
index 000000000000..bc39de394e47
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/resource/FileResource.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.util.resource;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+
+public class FileResource implements Resource {
+    private final Path file;
+
+    FileResource(File file) {
+        this.file = file.toPath();
+    }
+
+    @Override
+    public InputStream asStream() {
+        try {
+            return Files.newInputStream(file, StandardOpenOption.READ);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/resource/Resource.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/resource/Resource.java
new file mode 100644
index 000000000000..30217ba2083f
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/resource/Resource.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.util.resource;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+
+public interface Resource {
+    InputStream asStream();
+
+    static Resource fromString(String text) {
+        return new StringResource(text);
+    }
+
+    static Resource fromClasspath(String path) {
+        return new ClasspathResource(path);
+    }
+
+    static Resource fromFile(File file) {
+        return new FileResource(file);
+    }
+
+    default void writeTo(Path path) {
+        try (InputStream is = asStream()) {
+            Files.copy(is, path, StandardCopyOption.REPLACE_EXISTING);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+}
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/resource/StringResource.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/resource/StringResource.java
new file mode 100644
index 000000000000..2e16a4c4b979
--- /dev/null
+++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/util/resource/StringResource.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.util.resource;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+
+class StringResource implements Resource {
+    private final String text;
+
+    StringResource(String text) {
+        this.text = text;
+    }
+
+    @Override
+    public InputStream asStream() {
+        return new ByteArrayInputStream(text.getBytes());
+    }
+
+}
diff --git a/test/test-clusters/src/main/resources/default_test_roles.yml b/test/test-clusters/src/main/resources/default_test_roles.yml
new file mode 100644
index 000000000000..433dfdf79719
--- /dev/null
+++ b/test/test-clusters/src/main/resources/default_test_roles.yml
@@ -0,0 +1,13 @@
+
+# The roles below are automatically defined by the "testcluster" setup
+_es_test_root:
+  cluster: [ "ALL" ]
+  indices:
+    - names: [ "*" ]
+      allow_restricted_indices: true
+      privileges: [ "ALL" ]
+  run_as: [ "*" ]
+  applications:
+    - application: "*"
+      privileges: [ "*" ]
+      resources: [ "*" ]
diff --git a/test/test-clusters/src/main/resources/fips/cacerts.bcfks b/test/test-clusters/src/main/resources/fips/cacerts.bcfks
new file mode 100644
index 000000000000..9c3863eda60c
Binary files /dev/null and b/test/test-clusters/src/main/resources/fips/cacerts.bcfks differ
diff --git a/test/test-clusters/src/main/resources/fips/fips_java.policy b/test/test-clusters/src/main/resources/fips/fips_java.policy
new file mode 100644
index 000000000000..4ef62e03c254
--- /dev/null
+++ b/test/test-clusters/src/main/resources/fips/fips_java.policy
@@ -0,0 +1,16 @@
+grant {
+     permission java.security.SecurityPermission "putProviderProperty.BCFIPS";
+     permission java.security.SecurityPermission "putProviderProperty.BCJSSE";
+     permission java.lang.RuntimePermission "getProtectionDomain";
+     permission java.util.PropertyPermission "java.runtime.name", "read";
+     permission org.bouncycastle.crypto.CryptoServicesPermission "tlsAlgorithmsEnabled";
+     //io.netty.handler.codec.DecoderException
+     permission java.lang.RuntimePermission "accessClassInPackage.sun.security.internal.spec";
+     //java.security.InvalidAlgorithmParameterException: Cannot process GCMParameterSpec
+     permission java.lang.RuntimePermission "accessDeclaredMembers";
+     permission java.util.PropertyPermission "intellij.debug.agent", "read";
+     permission java.util.PropertyPermission "intellij.debug.agent", "write";
+     permission org.bouncycastle.crypto.CryptoServicesPermission "exportSecretKey";
+     permission org.bouncycastle.crypto.CryptoServicesPermission "exportPrivateKey";
+     permission java.io.FilePermission "${javax.net.ssl.trustStore}", "read";
+};
diff --git a/test/test-clusters/src/main/resources/fips/fips_java.security b/test/test-clusters/src/main/resources/fips/fips_java.security
new file mode 100644
index 000000000000..1018b1b96acc
--- /dev/null
+++ b/test/test-clusters/src/main/resources/fips/fips_java.security
@@ -0,0 +1,51 @@
+security.provider.1=org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider C:HYBRID;ENABLE{All};
+security.provider.2=org.bouncycastle.jsse.provider.BouncyCastleJsseProvider fips:BCFIPS
+security.provider.3=SUN
+security.provider.4=SunJGSS
+securerandom.source=file:/dev/urandom
+securerandom.strongAlgorithms=NativePRNGBlocking:SUN,DRBG:SUN
+securerandom.drbg.config=
+login.configuration.provider=sun.security.provider.ConfigFile
+policy.provider=sun.security.provider.PolicyFile
+policy.expandProperties=true
+policy.allowSystemProperty=true
+policy.ignoreIdentityScope=false
+keystore.type=BCFKS
+keystore.type.compat=true
+package.access=sun.misc.,\
+               sun.reflect.
+package.definition=sun.misc.,\
+                   sun.reflect.
+security.overridePropertiesFile=true
+ssl.KeyManagerFactory.algorithm=PKIX
+ssl.TrustManagerFactory.algorithm=PKIX
+networkaddress.cache.negative.ttl=10
+krb5.kdc.bad.policy = tryLast
+jdk.certpath.disabledAlgorithms=MD2, MD5, SHA1 jdkCA & usage TLSServer, \
+    RSA keySize < 1024, DSA keySize < 1024, EC keySize < 224
+jdk.jar.disabledAlgorithms=MD2, MD5, RSA keySize < 1024, \
+      DSA keySize < 1024
+jdk.tls.disabledAlgorithms=SSLv3, RC4, MD5withRSA, DH keySize < 1024, \
+    EC keySize < 224, DES40_CBC, RC4_40, 3DES_EDE_CBC
+jdk.tls.legacyAlgorithms= \
+        K_NULL, C_NULL, M_NULL, \
+        DH_anon, ECDH_anon, \
+        RC4_128, RC4_40, DES_CBC, DES40_CBC, \
+        3DES_EDE_CBC
+jdk.tls.keyLimits=AES/GCM/NoPadding KeyUpdate 2^37
+crypto.policy=unlimited
+jdk.xml.dsig.secureValidationPolicy=\
+    disallowAlg http://www.w3.org/TR/1999/REC-xslt-19991116,\
+    disallowAlg http://www.w3.org/2001/04/xmldsig-more#rsa-md5,\
+    disallowAlg http://www.w3.org/2001/04/xmldsig-more#hmac-md5,\
+    disallowAlg http://www.w3.org/2001/04/xmldsig-more#md5,\
+    maxTransforms 5,\
+    maxReferences 30,\
+    disallowReferenceUriSchemes file http https,\
+    minKeySize RSA 1024,\
+    minKeySize DSA 1024,\
+    minKeySize EC 224,\
+    noDuplicateIds,\
+    noRetrievalMethodLoops
+jceks.key.serialFilter = java.base/java.lang.Enum;java.base/java.security.KeyRep;\
+  java.base/java.security.KeyRep$Type;java.base/javax.crypto.spec.SecretKeySpec;!*
diff --git a/test/test-clusters/src/main/resources/fips/fips_java_oracle.security b/test/test-clusters/src/main/resources/fips/fips_java_oracle.security
new file mode 100644
index 000000000000..875151a058ee
--- /dev/null
+++ b/test/test-clusters/src/main/resources/fips/fips_java_oracle.security
@@ -0,0 +1,52 @@
+security.provider.1=org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider C:HYBRID;ENABLE{All};
+security.provider.2=org.bouncycastle.jsse.provider.BouncyCastleJsseProvider fips:BCFIPS
+security.provider.3=SUN
+security.provider.4=SunJGSS
+security.provider.5=SunRsaSign
+securerandom.source=file:/dev/urandom
+securerandom.strongAlgorithms=NativePRNGBlocking:SUN,DRBG:SUN
+securerandom.drbg.config=
+login.configuration.provider=sun.security.provider.ConfigFile
+policy.provider=sun.security.provider.PolicyFile
+policy.expandProperties=true
+policy.allowSystemProperty=true
+policy.ignoreIdentityScope=false
+keystore.type=BCFKS
+keystore.type.compat=true
+package.access=sun.misc.,\
+               sun.reflect.
+package.definition=sun.misc.,\
+                   sun.reflect.
+security.overridePropertiesFile=true
+ssl.KeyManagerFactory.algorithm=PKIX
+ssl.TrustManagerFactory.algorithm=PKIX
+networkaddress.cache.negative.ttl=10
+krb5.kdc.bad.policy = tryLast
+jdk.certpath.disabledAlgorithms=MD2, MD5, SHA1 jdkCA & usage TLSServer, \
+    RSA keySize < 1024, DSA keySize < 1024, EC keySize < 224
+jdk.jar.disabledAlgorithms=MD2, MD5, RSA keySize < 1024, \
+      DSA keySize < 1024
+jdk.tls.disabledAlgorithms=SSLv3, RC4, MD5withRSA, DH keySize < 1024, \
+    EC keySize < 224, DES40_CBC, RC4_40, 3DES_EDE_CBC
+jdk.tls.legacyAlgorithms= \
+        K_NULL, C_NULL, M_NULL, \
+        DH_anon, ECDH_anon, \
+        RC4_128, RC4_40, DES_CBC, DES40_CBC, \
+        3DES_EDE_CBC
+jdk.tls.keyLimits=AES/GCM/NoPadding KeyUpdate 2^37
+crypto.policy=unlimited
+jdk.xml.dsig.secureValidationPolicy=\
+    disallowAlg http://www.w3.org/TR/1999/REC-xslt-19991116,\
+    disallowAlg http://www.w3.org/2001/04/xmldsig-more#rsa-md5,\
+    disallowAlg http://www.w3.org/2001/04/xmldsig-more#hmac-md5,\
+    disallowAlg http://www.w3.org/2001/04/xmldsig-more#md5,\
+    maxTransforms 5,\
+    maxReferences 30,\
+    disallowReferenceUriSchemes file http https,\
+    minKeySize RSA 1024,\
+    minKeySize DSA 1024,\
+    minKeySize EC 224,\
+    noDuplicateIds,\
+    noRetrievalMethodLoops
+jceks.key.serialFilter = java.base/java.lang.Enum;java.base/java.security.KeyRep;\
+  java.base/java.security.KeyRep$Type;java.base/javax.crypto.spec.SecretKeySpec;!*
diff --git a/test/yaml-rest-runner/build.gradle b/test/yaml-rest-runner/build.gradle
index 1ae1315ac9ef..37d2a00a68dd 100644
--- a/test/yaml-rest-runner/build.gradle
+++ b/test/yaml-rest-runner/build.gradle
@@ -1,6 +1,6 @@
 apply plugin: 'elasticsearch.build'
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 dependencies {
   api project(':test:framework')
diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
index 76de2ae19844..4b0ac00deaa3 100644
--- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
+++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
@@ -24,7 +24,9 @@
 import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.client.WarningsHandler;
 import org.elasticsearch.client.sniff.ElasticsearchNodesSniffer;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Tuple;
@@ -36,6 +38,8 @@
 import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite;
 import org.elasticsearch.test.rest.yaml.section.ExecutableSection;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentBuilder;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -59,6 +63,8 @@
 import java.util.TreeSet;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
+
 /**
  * Runs a suite of yaml tests shared with all the official Elasticsearch
  * clients against an elasticsearch cluster.
@@ -115,11 +121,13 @@ protected ESClientYamlSuiteTestCase(ClientYamlTestCandidate testCandidate) {
         this.testCandidate = testCandidate;
     }
 
-    private static boolean useDefaultNumberOfShards;
+    private static Settings globalTemplateIndexSettings;
 
     @BeforeClass
-    public static void initializeUseDefaultNumberOfShards() {
-        useDefaultNumberOfShards = usually();
+    public static void initializeGlobalTemplateIndexSettings() {
+        globalTemplateIndexSettings = usually()
+            ? Settings.EMPTY
+            : Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2).build();
     }
 
     @Before
@@ -446,10 +454,25 @@ public void test() throws IOException {
             throw new IllegalArgumentException("No executable sections loaded for [" + testCandidate.getTestPath() + "]");
         }
 
-        if (useDefaultNumberOfShards == false
-            && testCandidate.getTestSection().getSkipSection().getFeatures().contains("default_shards") == false) {
+        assumeFalse(
+            "[" + testCandidate.getTestPath() + "] skipped, reason: in fips 140 mode",
+            inFipsJvm() && testCandidate.getTestSection().getSkipSection().getFeatures().contains("fips_140")
+        );
+
+        final Settings globalTemplateSettings = getGlobalTemplateSettings(testCandidate.getTestSection().getSkipSection().getFeatures());
+        if (globalTemplateSettings.isEmpty() == false) {
+            final XContentBuilder template = jsonBuilder();
+            template.startObject();
+            {
+                template.startArray("index_patterns").value("*").endArray();
+                template.startObject("settings");
+                globalTemplateSettings.toXContent(template, ToXContent.EMPTY_PARAMS);
+                template.endObject();
+            }
+            template.endObject();
+
             final Request request = new Request("PUT", "/_template/global");
-            request.setJsonEntity("{\"index_patterns\":[\"*\"],\"settings\":{\"index.number_of_shards\":2}}");
+            request.setJsonEntity(Strings.toString(template));
             // Because this has not yet transitioned to a composable template, it's possible that
             // this can overlap an installed composable template since this is a global (*)
             // template. In order to avoid this failing the test, we override the warnings handler
@@ -460,10 +483,6 @@ public void test() throws IOException {
             request.setOptions(builder.build());
             adminClient().performRequest(request);
         }
-        assumeFalse(
-            "[" + testCandidate.getTestPath() + "] skipped, reason: in fips 140 mode",
-            inFipsJvm() && testCandidate.getTestSection().getSkipSection().getFeatures().contains("fips_140")
-        );
 
         if (skipSetupSections() == false && testCandidate.getSetupSection().isEmpty() == false) {
             logger.debug("start setup test [{}]", testCandidate.getTestPath());
@@ -488,6 +507,14 @@ public void test() throws IOException {
         }
     }
 
+    protected Settings getGlobalTemplateSettings(List<String> features) {
+        if (features.contains("default_shards")) {
+            return Settings.EMPTY;
+        } else {
+            return globalTemplateIndexSettings;
+        }
+    }
+
     protected boolean skipSetupSections() {
         return false;
     }
diff --git a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java
index 8a8624c03708..5a9de8de3754 100644
--- a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java
+++ b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.client.NodeSelector;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.xcontent.XContentLocation;
 import org.elasticsearch.xcontent.yaml.YamlXContent;
 
@@ -19,7 +20,6 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.regex.Pattern;
@@ -500,7 +500,7 @@ public void testAddingDoWithWarningWithoutSkipWarnings() {
         doSection.setApiCallSection(new ApiCallSection("test"));
         ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection);
         Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
-        assertThat(e.getMessage(), containsString(formatted("""
+        assertThat(e.getMessage(), containsString(Strings.format("""
             api/name:
             attempted to add a [do] with a [warnings] section without a corresponding ["skip": "features": "warnings"] \
             so runners that do not support the [warnings] section can skip the test at line [%d]\
@@ -514,7 +514,7 @@ public void testAddingDoWithWarningRegexWithoutSkipWarnings() {
         doSection.setApiCallSection(new ApiCallSection("test"));
         ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection);
         Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
-        assertThat(e.getMessage(), containsString(formatted("""
+        assertThat(e.getMessage(), containsString(Strings.format("""
             api/name:
             attempted to add a [do] with a [warnings_regex] section without a corresponding ["skip": "features": "warnings_regex"] \
             so runners that do not support the [warnings_regex] section can skip the test at line [%d]\
@@ -528,7 +528,7 @@ public void testAddingDoWithAllowedWarningWithoutSkipAllowedWarnings() {
         doSection.setApiCallSection(new ApiCallSection("test"));
         ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection);
         Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
-        assertThat(e.getMessage(), containsString(formatted("""
+        assertThat(e.getMessage(), containsString(Strings.format("""
             api/name:
             attempted to add a [do] with a [allowed_warnings] section without a corresponding ["skip": "features": \
             "allowed_warnings"] so runners that do not support the [allowed_warnings] section can skip the test at \
@@ -543,7 +543,7 @@ public void testAddingDoWithAllowedWarningRegexWithoutSkipAllowedWarnings() {
         doSection.setApiCallSection(new ApiCallSection("test"));
         ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection);
         Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
-        assertThat(e.getMessage(), containsString(formatted("""
+        assertThat(e.getMessage(), containsString(Strings.format("""
             api/name:
             attempted to add a [do] with a [allowed_warnings_regex] section without a corresponding ["skip": "features": \
             "allowed_warnings_regex"] so runners that do not support the [allowed_warnings_regex] section can skip the test \
@@ -559,7 +559,7 @@ public void testAddingDoWithHeaderWithoutSkipHeaders() {
         doSection.setApiCallSection(apiCallSection);
         ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection);
         Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
-        assertThat(e.getMessage(), containsString(formatted("""
+        assertThat(e.getMessage(), containsString(Strings.format("""
             api/name:
             attempted to add a [do] with a [headers] section without a corresponding ["skip": "features": "headers"] \
             so runners that do not support the [headers] section can skip the test at line [%d]\
@@ -574,7 +574,7 @@ public void testAddingDoWithNodeSelectorWithoutSkipNodeSelector() {
         doSection.setApiCallSection(apiCall);
         ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, doSection);
         Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
-        assertThat(e.getMessage(), containsString(formatted("""
+        assertThat(e.getMessage(), containsString(Strings.format("""
             api/name:
             attempted to add a [do] with a [node_selector] section without a corresponding ["skip": "features": "node_selector"] \
             so runners that do not support the [node_selector] section can skip the test at line [%d]\
@@ -590,7 +590,7 @@ public void testAddingContainsWithoutSkipContains() {
         );
         ClientYamlTestSuite testSuite = createTestSuite(SkipSection.EMPTY, containsAssertion);
         Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
-        assertThat(e.getMessage(), containsString(formatted("""
+        assertThat(e.getMessage(), containsString(Strings.format("""
             api/name:
             attempted to add a [contains] assertion without a corresponding ["skip": "features": "contains"] \
             so runners that do not support the [contains] assertion can skip the test at line [%d]\
@@ -642,7 +642,7 @@ public void testMultipleValidationErrors() {
             sections
         );
         Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate);
-        assertEquals(String.format(Locale.ROOT, """
+        assertEquals(Strings.format("""
             api/name:
             attempted to add a [contains] assertion without a corresponding ["skip": "features": "contains"] so runners that \
             do not support the [contains] assertion can skip the test at line [%d],
diff --git a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java
index 2d48d9389f38..88c5fdfdb1e7 100644
--- a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java
+++ b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.client.NodeSelector;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.logging.HeaderWarning;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
@@ -239,7 +240,7 @@ public void testParseDoSectionNoParamsNoBody() throws Exception {
     public void testParseDoSectionWithJsonBody() throws Exception {
         String body = """
             { "include": { "field1": "v1", "field2": "v2" }, "count": 1 }""";
-        parser = createParser(YamlXContent.yamlXContent, formatted("""
+        parser = createParser(YamlXContent.yamlXContent, Strings.format("""
             index:
                 index:  test_1
                 type:   test
diff --git a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java
index 5c4b7eb62be3..5e4db061dbaa 100644
--- a/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java
+++ b/test/yaml-rest-runner/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java
@@ -10,6 +10,7 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.xcontent.yaml.YamlXContent;
 
@@ -71,7 +72,7 @@ public void testMessage() {
 
     public void testParseSkipSectionVersionNoFeature() throws Exception {
         Version version = VersionUtils.randomVersion(random());
-        parser = createParser(YamlXContent.yamlXContent, formatted("""
+        parser = createParser(YamlXContent.yamlXContent, Strings.format("""
             version:     " - %s"
             reason:      Delete ignores the parent param""", version));
 
diff --git a/x-pack/docs/en/rest-api/logstash/delete-pipeline.asciidoc b/x-pack/docs/en/rest-api/logstash/delete-pipeline.asciidoc
index d1533f14dbd5..ff7494d34e61 100644
--- a/x-pack/docs/en/rest-api/logstash/delete-pipeline.asciidoc
+++ b/x-pack/docs/en/rest-api/logstash/delete-pipeline.asciidoc
@@ -55,8 +55,7 @@ PUT _logstash/pipeline/my_pipeline
     "pipeline.batch.size": 125,
     "pipeline.batch.delay": 50,
     "queue.type": "memory",
-    "queue.max_bytes.number": 1,
-    "queue.max_bytes.units": "gb",
+    "queue.max_bytes": "1gb",
     "queue.checkpoint.writes": 1024
   }
 }
diff --git a/x-pack/docs/en/rest-api/logstash/get-pipeline.asciidoc b/x-pack/docs/en/rest-api/logstash/get-pipeline.asciidoc
index 252adba0a319..8409a5128d52 100644
--- a/x-pack/docs/en/rest-api/logstash/get-pipeline.asciidoc
+++ b/x-pack/docs/en/rest-api/logstash/get-pipeline.asciidoc
@@ -57,8 +57,7 @@ PUT _logstash/pipeline/my_pipeline
     "pipeline.batch.size": 125,
     "pipeline.batch.delay": 50,
     "queue.type": "memory",
-    "queue.max_bytes.number": 1,
-    "queue.max_bytes.units": "gb",
+    "queue.max_bytes": "1gb",
     "queue.checkpoint.writes": 1024
   }
 }
@@ -92,8 +91,7 @@ If the request succeeds, the body of the response contains the pipeline definiti
       "pipeline.batch.size": 125,
       "pipeline.batch.delay": 50,
       "queue.type": "memory",
-      "queue.max_bytes.number": 1,
-      "queue.max_bytes.units": "gb",
+      "queue.max_bytes": "1gb",
       "queue.checkpoint.writes": 1024
     }
   }
diff --git a/x-pack/docs/en/rest-api/logstash/put-pipeline.asciidoc b/x-pack/docs/en/rest-api/logstash/put-pipeline.asciidoc
index fb2d035aadc5..a0d2f02f2bf6 100644
--- a/x-pack/docs/en/rest-api/logstash/put-pipeline.asciidoc
+++ b/x-pack/docs/en/rest-api/logstash/put-pipeline.asciidoc
@@ -87,8 +87,7 @@ PUT _logstash/pipeline/my_pipeline
     "pipeline.batch.size": 125,
     "pipeline.batch.delay": 50,
     "queue.type": "memory",
-    "queue.max_bytes.number": 1,
-    "queue.max_bytes.units": "gb",
+    "queue.max_bytes": "1gb",
     "queue.checkpoint.writes": 1024
   }
 }
diff --git a/x-pack/docs/en/rest-api/security/create-roles.asciidoc b/x-pack/docs/en/rest-api/security/create-roles.asciidoc
index fa7e142f64a0..8a7b164773c4 100644
--- a/x-pack/docs/en/rest-api/security/create-roles.asciidoc
+++ b/x-pack/docs/en/rest-api/security/create-roles.asciidoc
@@ -123,3 +123,21 @@ created or updated.
 --------------------------------------------------
 
 <1> When an existing role is updated, `created` is set to false.
+
+The following example configures a role that can run SQL in JDBC:
+
+// tag::sql-queries-permission[]
+[source,console]
+--------------------------------------------------
+POST /_security/role/cli_or_drivers_minimal
+{
+  "cluster": ["cluster:monitor/main"],
+  "indices": [
+    {
+      "names": ["test"],
+      "privileges": ["read", "indices:admin/get"]
+    }
+  ]
+}
+--------------------------------------------------
+// end::sql-queries-permission[]
\ No newline at end of file
diff --git a/x-pack/docs/en/security/authentication/configuring-native-realm.asciidoc b/x-pack/docs/en/security/authentication/configuring-native-realm.asciidoc
index 110e852f227b..00089e6acf8d 100644
--- a/x-pack/docs/en/security/authentication/configuring-native-realm.asciidoc
+++ b/x-pack/docs/en/security/authentication/configuring-native-realm.asciidoc
@@ -1,6 +1,10 @@
-The native realm is available by default when no other realms are 
-configured. If other realm settings have been configured in `elasticsearch.yml`, 
-you must add the native realm to the realm chain.
+The native realm is available and enabled by default. You can disable it explicitly with the following snippet. 
+
+[source,yaml]
+----------------------------------------
+xpack.security.authc.realms.native.native1:
+  enabled: false
+----------------------------------------
 
 You can configure a `native` realm in the `xpack.security.authc.realms.native`
 namespace in `elasticsearch.yml`.
@@ -21,13 +25,8 @@ sets the `order` to zero so the realm is checked first:
 
 [source, yaml]
 ------------------------------------------------------------
-xpack:
-  security:
-    authc:
-      realms:
-        native:
-          native1:
-            order: 0
+xpack.security.authc.realms.native.native1:
+  order: 0
 ------------------------------------------------------------
 
 NOTE: To limit exposure to credential theft and mitigate credential compromise,
diff --git a/x-pack/docs/en/security/authentication/jwt-realm.asciidoc b/x-pack/docs/en/security/authentication/jwt-realm.asciidoc
index c3d3f2ab4dfd..a9993f950b18 100644
--- a/x-pack/docs/en/security/authentication/jwt-realm.asciidoc
+++ b/x-pack/docs/en/security/authentication/jwt-realm.asciidoc
@@ -12,35 +12,77 @@ between the _client_ that is connecting to {es}, and the _user_ on whose behalf
 the request should run. The JWT authenticates the user, and a separate credential
 authenticates the client.
 
-A common scenario for JWTs is when an existing front-end application uses
-OpenID Connect (OIDC) to authenticate and identify a user, and then accesses {es}
-on behalf of the authenticated user.
-
-TIP: If the front-end application does not exist, you can use the
-<<token-authentication-services>> instead.
+The JWT realm supports two token types, `id_token` (the default) and `access_token`.
+They are designed to work for the following two scenarios, respectively:
+
+1. `id_token` - An application authenticates and identifies a user with an authentication flow,
+e.g. OpenID Connect (OIDC), and then accesses {es} on behalf of the authenticated user using
+a JSON Web Token (JWT) conforming to OIDC ID Token specification.
+2. `access_token` - An application accesses {es} using its own identity, encoded as a JWT,
+e.g. The application authenticates itself to a central identity platform using an
+OAuth2 Client Credentials Flow and then uses the resulting JWT-based access token to connect to {es}.
+
+NOTE: A single JWT realm can only work with a single token type. To handle both token types,
+you must configure at least two JWT realms. You should choose the token type carefully based
+on the use case because it impacts on how validations are performed.
+
+The JWT realm validates the incoming JWT based on its configured token type.
+JSON Web Tokens (JWT) of both types must contain the following 5 pieces of information.
+While ID Tokens, based on the OIDC specification, have strict rules for what claims should provide these information,
+access tokens allow some claims to be configurable.
+
+[cols="3",frame=all]
+|====
+h|             2+^h| Claims
+h| Information     | ID Token   | Access Token
+ | Issuer          | `iss`      | `iss`
+ | Subject         | `sub`      | Defaults to `sub`, but can fall back to another claim if `sub` does not exist
+ | Audiences       | `aud`      | Defaults to `aud`, but can fall back to another claim if `aud` does not exist
+ | Issue Time      | `iat`      | `iat`
+ | Expiration Time | `exp`      | `exp`
+|====
+
+In addition, {es} also validates `nbf` and `auth_time` claims for ID Tokens if these claims are present.
+But these claims are ignored for access tokens.
+
+Overall, the access token type has more relaxed validation rules and is suitable for more generic JWTs,
+including self-signed ones.
 
 [[jwt-realm-oidc]]
-==== JWT uses OIDC workflows
+==== ID Tokens from OIDC workflows
 JWT authentication in {es} is derived from OIDC user workflows, where different
-tokens can be issued by an OIDC Provider (OP). One possible token is an
-_ID token_, which uses the JWT format. If the ID token is presented to a JWT
-realm, {es} can use it as a bearer token to authenticate, identify, and authorize an individual
-user.
+tokens can be issued by an OIDC Provider (OP), including ID Tokens.
+ID Tokens from an OIDC provider are well-defined JSON Web Tokens (JWT) and should be always compatible with
+a JWT realm of the `id_token` token type. The subject claim of an ID token represents the end-user.
+This means that ID tokens will generally have many allowed subjects.
+Therefore, a JWT realm of `id_token` token type does _not_ mandate the `allowed_subjects` validation.
 
 NOTE: Because JWTs are obtained external to {es}, you can define a custom workflow
 instead of using the OIDC workflow. However, the JWT format must still be JSON
 Web Signature (JWS). The JWS header and JWS signature are validated using OIDC
 ID token validation rules.
 
-{es} supports a separate <<oidc-realm,OpenID Connect realm>>, which provides
-stronger security guarantees than the JWT realm, and is preferred for any
+{es} supports a separate <<oidc-realm,OpenID Connect realm>>. It is preferred for any
 use case where {es} can act as an OIDC RP. The OIDC realm is the only supported
 way to enable OIDC authentication in {kib}.
 
-TIP: If JWTs are issued for the front-end application, the application is the realm client and JWT user.
-That is not supported by OIDC flows, but it may be supported by bespoke JWT issuers.
-In that case, use the client secret and JWT for the client application, and the
-`es-security-runas-user` HTTP request header for the different user. See <<jwt-realm-runas>>.
+TIP: Users authenticating with a JWT realm can optionally impersonate another user
+with the <<run-as-privilege,`run_as`>> feature. See also <<jwt-realm-runas>>.
+
+[[jwt-realm-oauth2]]
+==== Access Tokens
+A common method to obtain access tokens is with the OAuth2 Client Credentials Flow.
+A typical usage of this flow is for an application to get a credential for itself.
+This is the use case that the `access_token` token type is designed for.
+It is likely that this application also obtains ID Tokens for its end-users.
+To prevent end-user ID Tokens being used to authenticate with the JWT realm configured
+for the application, we mandate `allowed_subjects` validation when a JWT realm
+has token type `access_token`.
+
+NOTE: Not every access token is formatted as a JSON Web Token (JWT).
+For it to be compatible with the JWT realm, it must at least use the JWT format and satisfies
+relevant requirements in the above table.
+
 
 [[jwt-realm-configuration]]
 ==== Configure {es} to use a JWT realm
@@ -62,6 +104,7 @@ includes the most common settings, which are not intended for every use case:
 ----
 xpack.security.authc.realms.jwt.jwt1:
   order: 3
+  token_type: id_token
   client_authentication.type: shared_secret
   allowed_issuer: "https://issuer.example.com/jwt/"
   allowed_audiences: [ "8fb85eba-979c-496c-8ae2-a57fde3f12d0" ]
@@ -75,12 +118,15 @@ Specifies a realm `order` of `3`, which indicates the order in which the
 configured realm is checked when authenticating a user. Realms are consulted in
 ascending order, where the realm with the lowest order value is consulted first.
 
+`token_type`::
+Instructs the realm to treat and validate incoming JWTs as ID Tokens (`id_token`).
+
 `client_authentication.type`::
 Specifies the client authentication type as `shared_secret`, which means that
 the client is authenticated using an HTTP request header that must match a
 pre-configured secret value. The client must provide this shared secret with
-every request in the `ES-Client-Authentication` header. The value must be a
-case-insensitive match to the realm's `client_authentication.shared_secret`.
+every request in the `ES-Client-Authentication` header. The header value must be a
+case-sensitive match to the realm's `client_authentication.shared_secret`.
 
 `allowed_issuer`::
 Sets a verifiable identifier for your JWT issuer. This value is typically a
@@ -103,6 +149,53 @@ use an absolute path starting with `/app/config/`.
 `claims.principal`::
 The name of the JWT claim that contains the user's principal (username).
 
+The following is an example snippet for configure a JWT realm for handling
+access tokens:
+
+[source,yaml]
+----
+xpack.security.authc.realms.jwt.jwt2:
+  order: 4
+  token_type: access_token
+  client_authentication.type: shared_secret
+  allowed_issuer: "https://issuer.example.com/jwt/"
+  allowed_subjects: [ "123456-compute@developer.example.com" ]
+  allowed_audiences: [ "elasticsearch" ]
+  required_claims:
+    token_use: access
+    version: ["1.0", "2.0"]
+  allowed_signature_algorithms: [RS256,HS256]
+  pkc_jwkset_path: "https://idp-42.example.com/.well-known/configuration"
+  fallback_claims.sub: client_id
+  fallback_claims.aud: scope
+  claims.principal: sub
+----
+
+`token_type`::
+Instructs the realm to treat and validate incoming JWTs as access tokens (`access_token`).
+
+`allowed_subjects`::
+Specifies a list of JWT subjects that the realm will allow.
+These values are typically URLs, UUIDs, or other case-sensitive string values.
+
+NOTE: This setting is mandatory for when `token_type` is `access_token`.
+
+`required_claims`::
+Specifies a list of key/value pairs for additional verifications to be performed
+against a JWT. The values are either a string or an array of strings.
+
+`fallback_claims.sub`::
+The name of the JWT claim to extract the subject information if the `sub` claim does not exist.
+This setting is only available when `token_type` is `access_token`.
+The fallback is applied everywhere the `sub` claim is used.
+In the above snippet, it means the `claims.principal` will also fallback to `client_id`
+if `sub` does not exist.
+
+`fallback_claims.aud`::
+The name of the JWT claim to extract the audiences information if the `aud` claim does not exist.
+This setting is only available when `token_type` is `access_token`.
+The fallback is applied everywhere the `aud` claim is used.
+
 --
 
 . After defining settings, use the
@@ -186,11 +279,12 @@ as `HS256`. The algorithm must be in the realm's allow list.
 
 [[jwt-validation-payload]]
 ===== Payload claims
-OIDC ID tokens contain several claims, which provide information about the user
+Tokens contain several claims, which provide information about the user
 who is issuing the token, and the token itself.
+Depending on the token type, these information can optionally be identified
+by different claims.
 
-[[jwt-validation-payload-oidc]]
-====== OIDC payload claims
+====== JWT payload claims
 The following claims are validated by a subset of OIDC ID token rules.
 
 {es} doesn't validate `nonce` claims, but a custom JWT issuer can add a
@@ -205,10 +299,23 @@ creation (`iat`), not before (`nbf`), and expiration times (`exp`).
 (Required, String) Denotes the issuer that created the ID token. The value must
 be an exact, case-sensitive match to the value in the `allowed_issuer` setting.
 
+`sub`::
+(Required*, String) Indicates the subject that the ID token is created for.
+If the JWT realm is of the `id_token` type, this claim is mandatory.
+A JWT realm of the `id_token` type by defaults accepts all subjects.
+A JWT realm of the access_token type must specify the `allowed_subjects` setting and the subject value
+must be an exact, case-sensitive match to any of the CSV values in the
+allowed_subjects setting.
+A JWT realm of the access_token type can specify a fallback claim that will
+be used in place where the `sub` claim does not exist.
+
 `aud`::
-(Required, String) Indicates the audiences that the ID token is for, expressed as a
+(Required*, String) Indicates the audiences that the ID token is for, expressed as a
 comma-separated value (CSV). One of the values must be an exact, case-sensitive
 match to any of the CSV values in the `allowed_audiences` setting.
+If the JWT realm is of the `id_token` type, this claim is mandatory.
+A JWT realm of the `access_token` type can specify a fallback claim that will
+be used in place where the `aud` claim does not exist.
 
 `exp`::
 (Required, integer) Expiration time for the ID token, expressed in UTC
@@ -221,14 +328,19 @@ seconds since epoch.
 `nbf`::
 (Optional, integer) Indicates the time before which the JWT must not be accepted,
 expressed as UTC seconds since epoch.
+This claim is optional. If it exists, a JWT realm of `id_token` type will verify
+it, while a JWT realm of `access_token` will just ignore it.
 
 `auth_time`::
 (Optional, integer) Time when the user authenticated to the JWT issuer,
 expressed as UTC seconds since epoch.
+This claim is optional. If it exists, a JWT realm of `id_token` type will verify
+it, while a JWT realm of `access_token` will just ignore it.
+
 
 [[jwt-validation-payload-es]]
-====== {es} settings for consuming OIDC claims
-{es} uses OIDC ID token claims for the following settings.
+====== {es} settings for consuming JWT claims
+{es} uses JWT claims for the following settings.
 
 `principal`::
 (Required, String) Contains the user's principal (username). The value is
@@ -584,5 +696,4 @@ JWT realm itself.
 "metadata":{"jwt_claim_email":"user2@something.example.com","jwt_claim_aud":["es01","es02","es03"],
 "jwt_claim_sub":"user2","jwt_claim_iss":"my-issuer"},"enabled":true,"authentication_realm":
 {"name":"jwt2","type":"jwt"},"lookup_realm":{"name":"jwt2","type":"jwt"},"authentication_type":"realm"}
-%
 ----
diff --git a/x-pack/docs/en/security/authentication/remote-clusters-privileges.asciidoc b/x-pack/docs/en/security/authentication/remote-clusters-privileges.asciidoc
index d827ccd89bda..d0899494140a 100644
--- a/x-pack/docs/en/security/authentication/remote-clusters-privileges.asciidoc
+++ b/x-pack/docs/en/security/authentication/remote-clusters-privileges.asciidoc
@@ -34,7 +34,7 @@ the `read_ccr` cluster privilege, and `monitor` and `read` privileges on the
 leader index.
 
 NOTE: If requests will be issued <<run-as-privilege,on behalf of other users>>,
-then the the authenticating user must have the `run_as` privilege on the remote 
+then the authenticating user must have the `run_as` privilege on the remote 
 cluster.
 
 The following request creates a `remote-replication` role on the remote cluster:
@@ -134,7 +134,7 @@ On the remote cluster, the {ccs} role requires the `read` and
 `read_cross_cluster` privileges for the target indices.
 
 NOTE: If requests will be issued <<run-as-privilege,on behalf of other users>>,
-then the the authenticating user must have the `run_as` privilege on the remote 
+then the authenticating user must have the `run_as` privilege on the remote 
 cluster.
 
 The following request creates a `remote-search` role on the remote cluster:
diff --git a/x-pack/docs/en/security/authorization/built-in-roles.asciidoc b/x-pack/docs/en/security/authorization/built-in-roles.asciidoc
index 87e40d15e93c..7b75fba8fbe7 100644
--- a/x-pack/docs/en/security/authorization/built-in-roles.asciidoc
+++ b/x-pack/docs/en/security/authorization/built-in-roles.asciidoc
@@ -59,6 +59,8 @@ Grants full access to all features in {kib} (including Solutions) and read-only
 ===============================
 * This role provides read access to any index that is not prefixed with a dot.
 * This role automatically grants full access to new {kib} features as soon as they are released.
+* Some {kib} features may also require creation or write access to data indices. {ml-cap} {dfanalytics-jobs}
+  is an example. For such features those privileges must be defined in a separate role.
 ===============================
 
 --
diff --git a/x-pack/docs/en/security/troubleshooting.asciidoc b/x-pack/docs/en/security/troubleshooting.asciidoc
index ed30fc6c1455..23748172c92d 100644
--- a/x-pack/docs/en/security/troubleshooting.asciidoc
+++ b/x-pack/docs/en/security/troubleshooting.asciidoc
@@ -107,7 +107,20 @@ The role definition might be missing or invalid.
 
 |======================
 
-To help track down these possibilities, add the following lines to the end of
+To help track down these possibilities, enable additional logging to troubleshoot further. 
+You can enable debug logging by configuring the following persistent setting:
+
+[source, console]
+----
+PUT /_cluster/settings
+{
+  "persistent": {
+    "logger.org.elasticsearch.xpack.security.authc": "debug"
+  }
+}
+----
+
+Alternatively, you can add the following lines to the end of 
 the `log4j2.properties` configuration file in the `ES_PATH_CONF`:
 
 [source,properties]
@@ -116,6 +129,9 @@ logger.authc.name = org.elasticsearch.xpack.security.authc
 logger.authc.level = DEBUG
 ----------------
 
+Refer to <<configuring-logging-levels,configuring logging levels>> for more
+information.
+
 A successful authentication should produce debug statements that list groups and
 role mappings.
 --
diff --git a/x-pack/docs/en/watcher/example-watches.asciidoc b/x-pack/docs/en/watcher/example-watches.asciidoc
index 377f0c393556..60a42d4c55dd 100644
--- a/x-pack/docs/en/watcher/example-watches.asciidoc
+++ b/x-pack/docs/en/watcher/example-watches.asciidoc
@@ -1,15 +1,13 @@
 [role="xpack"]
 [[example-watches]]
 == Example watches
-The following examples show how to set up watches to:
+The following example shows how to set up a watch to:
 
 * <<watch-cluster-status,Monitor the status of an Elasticsearch cluster>>
-* <<watching-meetup-data,Monitor event data>>
 
 For more example watches you can use as a starting point for building custom
-watches, see the https://github.com/elastic/examples/tree/master/Alerting[Example
-watches] in the Elastic Examples repo.
+watches, refer to the
+https://github.com/elastic/examples/tree/master/Alerting[Example watches] in the
+Elastic Examples repo.
 
 include::example-watches/example-watch-clusterstatus.asciidoc[]
-
-include::example-watches/example-watch-meetupdata.asciidoc[]
diff --git a/x-pack/docs/en/watcher/example-watches/example-watch-meetupdata.asciidoc b/x-pack/docs/en/watcher/example-watches/example-watch-meetupdata.asciidoc
deleted file mode 100644
index 24be5c6cdb97..000000000000
--- a/x-pack/docs/en/watcher/example-watches/example-watch-meetupdata.asciidoc
+++ /dev/null
@@ -1,310 +0,0 @@
-[role="xpack"]
-[[watching-meetup-data]]
-=== Watching event data
-
-If you are indexing event data, such as log messages, network traffic, or a web feed, you can create a watch to email notifications when certain events occur.
-For example, if you index a feed of RSVPs for meetup events happening around the world, you can create a watch that alerts you to interesting events.
-
-To index the meetup data, you can use https://www.elastic.co/products/logstash[Logstash] to ingest live data from the Meetup.com streaming API, `http://stream.meetup.com/2/rsvps`.
-
-To ingest this data with Logstash:
-
-. https://www.elastic.co/downloads/logstash[Download Logstash] and unpack the
-  archive file.
-
-. Create a Logstash configuration file that uses the {logstash-ref}/plugins-inputs-stdin.html[Logstash standard input] and the {logstash-ref}/plugins-outputs-stdout.html[Logstash standard output] and save it in `logstash-{version}` directory as `livestream.conf`:
-+
---
-[source,ruby]
-----------------------------------------------------------
-input {
-  stdin {
-    codec => json <1>
-  }
-}
-filter {
-  date {
-    match => [ "event.time", "UNIX_MS" ]
-    target => "event_time"
-  }
-}
-output { <2>
-  stdout {
-    codec => rubydebug
-  }
-  elasticsearch {
-    hosts => "http://localhost:9200"
-    user  => "elastic"
-    password  => "x-pack-test-password"
-  }
-}
-
-----------------------------------------------------------
-// NOTCONSOLE
-<1> The meetup data is formatted in JSON.
-<2> Index the meetup data into Elasticsearch.
---
-
-. To start indexing the meetup data, pipe the RSVP stream into Logstash and specify your `livestream.conf` configuration file.
-+
---
-[source,shell]
-----------------------------------------------------------
-curl http://stream.meetup.com/2/rsvps | bin/logstash -f livestream.conf
-----------------------------------------------------------
-// NOTCONSOLE
--- 
-
-Now that you're indexing the meetup RSVPs, you can set up a watch that lets you know about events you might be interested in. For example, let's create a watch that runs every hour, looks for events that talk about _Open Source_, and sends an email with information about the events.
-
-
-To set up the watch:
-
-. Specify how often you want to run the watch by adding a schedule trigger to the watch:
-+
---
-[source,js]
---------------------------------------------------
-{
-  "trigger": {
-    "schedule": {
-      "interval": "1h"
-    }
-  },
---------------------------------------------------
-// NOTCONSOLE
---
-. Load data into the watch payload by creating an input that searches the meetup data for events that have _Open Source_ as a topic. You can use aggregations to group the data by city, consolidate references to the same events, and sort the events by date.
-+
---
-[source,js]
--------------------------------------------------
-"input": {
-    "search": {
-      "request": {
-        "indices": [
-          "logstash" <1>
-        ],
-        "body": {
-          "size": 0,
-          "query": {
-            "bool": {
-              "filter": [
-                {
-                  "range": {
-                    "@timestamp": {
-                      "gte": "now-3h"
-                    }
-                  }
-                },
-                {
-                  "match": {
-                    "group.group_topics.topic_name": "Open Source" <2>
-                  }
-                }
-              ]
-            }
-          },
-          "aggs": {
-            "group_by_city": {
-              "terms": {
-                "field": "group.group_city.keyword", <3>
-                "size": 5
-              },
-              "aggs": {
-                "group_by_event": {
-                  "terms": {
-                    "field": "event.event_url.keyword", <4>
-                    "size": 5
-                  },
-                  "aggs": {
-                    "get_latest": {
-                      "terms": {
-                        "field": "@timestamp", <5>
-                        "size": 1,
-                        "order": {
-                          "_key": "desc"
-                        }
-                      },
-                      "aggs": {
-                        "group_by_event_name": {
-                          "terms": {
-                            "field": "event.event_name.keyword" <6>
-                          }
-                        }
-                      }
-                    }
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  },
--------------------------------------------------
-// NOTCONSOLE
-<1> `logstash` is the default <<aliases,index alias>> for the {ls} indices
-containing the meetup data. By default, the {ls}
-<<index-lifecycle-management,{ilm} ({ilm-init})>> policy rolls this alias to a
-new index when the index size reaches 50GB or becomes 30 days old. For more
-information, see
-{logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ilm[{ilm-init}
-defaults in Logstash].
-<2> Find all of the RSVPs with `Open Source` as a topic.
-<3> Group the RSVPs by city.
-<4> Consolidate multiple RSVPs for the same event.
-<5> Sort the events so the latest events are listed first.
-<6> Group the events by name.
---
-
-. To determine whether or not there are any Open Source events, add a compare condition that checks the watch payload to see if there were any search hits.
-+
---
-[source,js]
---------------------------------------------------
-"compare" : { "ctx.payload.hits.total" : { "gt" : 0 }}
---------------------------------------------------
-// NOTCONSOLE
---
-
-. To send an email when _Open Source_ events are found, add an email action:
-+
---
-[source,js]
----------------------------------------------------
-"actions": {
-    "email_me": {
-      "throttle_period": "10m",
-      "email": {
-        "from": "<from:email address>",
-        "to": "<to:email address>",
-        "subject": "Open Source Events",
-        "body": {
-          "html": "Found events matching Open Source: <ul>{{#ctx.payload.aggregations.group_by_city.buckets}}<          li>{{key}} ({{doc_count}})<ul>{{#group_by_event.buckets}}
-          <li><a href=\"{{key}}\">{{get_latest.buckets.0.group_by_event_name.buckets.0.key}}</a>
-          ({{doc_count}})</li>{{/group_by_event.buckets}}</ul></li>
-          {{/ctx.payload.aggregations.group_by_city.buckets}}</ul>"
-        }
-      }
-    }
-  }
----------------------------------------------------
-// NOTCONSOLE
---
-
-NOTE: To enable Watcher to send emails, you must configure an email account in `elasticsearch.yml`. For more information, see <<configuring-email>>.
-
-
-The complete watch looks like this:
-
-[source,console]
---------------------------------------------------
-PUT _watcher/watch/meetup
-{
-  "trigger": {
-    "schedule": {
-      "interval": "1h"
-    }
-  },
-  "input": {
-    "search": {
-      "request": {
-        "indices": [
-          "logstash"
-        ],
-        "body": {
-          "size": 0,
-          "query": {
-            "bool": {
-              "filter": [
-                {
-                  "range": {
-                    "@timestamp": {
-                      "gte": "now-3h"
-                    }
-                  }
-                },
-                {
-                  "match": {
-                    "group.group_topics.topic_name": "Open Source"
-                  }
-                }
-              ]
-            }
-          },
-          "aggs": {
-            "group_by_city": {
-              "terms": {
-                "field": "group.group_city.keyword",
-                "size": 5
-              },
-              "aggs": {
-                "group_by_event": {
-                  "terms": {
-                    "field": "event.event_url.keyword",
-                    "size": 5
-                  },
-                  "aggs": {
-                    "get_latest": {
-                      "terms": {
-                        "field": "@timestamp",
-                        "size": 1,
-                        "order": {
-                          "_key": "desc"
-                        }
-                      },
-                      "aggs": {
-                        "group_by_event_name": {
-                          "terms": {
-                            "field": "event.event_name.keyword"
-                          }
-                        }
-                      }
-                    }
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  },
-  "condition": {
-    "compare": {
-      "ctx.payload.hits.total": {
-        "gt": 0
-      }
-    }
-  },
-  "actions": {  <1>
-    "email_me": {
-      "throttle_period": "10m",
-      "email": {
-        "from": "username@example.org",  <2>
-        "to": "recipient@example.org",   <3>
-        "subject": "Open Source events",
-        "body": {
-          "html": "Found events matching Open Source: <ul>{{#ctx.payload.aggregations.group_by_city.buckets}}<li>{{key}} ({{doc_count}})<ul>{{#group_by_event.buckets}}<li><a href=\"{{key}}\">{{get_latest.buckets.0.group_by_event_name.buckets.0.key}}</a> ({{doc_count}})</li>{{/group_by_event.buckets}}</ul></li>{{/ctx.payload.aggregations.group_by_city.buckets}}</ul>"
-         }
-      }
-    }
-  }
-}
---------------------------------------------------
-
-<1> The email body can include Mustache templates to reference data in the watch payload. By default,it will be <<email-html-sanitization,sanitized>> to block dangerous content.
-<2> Replace the `from` address with the email address you configured in `elasticsearch.yml`.
-<3> Replace the `to` address with your email address to receive notifications.
-
-
-Now that you've created your watch, you can use the
-{ref}/watcher-api-execute-watch.html[`_execute` API] to run it without waiting for the schedule to trigger execution:
-
-[source,console]
---------------------------------------------------
-POST _watcher/watch/meetup/_execute
---------------------------------------------------
-// TEST[continued]
diff --git a/x-pack/docs/en/watcher/input/http.asciidoc b/x-pack/docs/en/watcher/input/http.asciidoc
index 023884dd8e53..ca1738b5952d 100644
--- a/x-pack/docs/en/watcher/input/http.asciidoc
+++ b/x-pack/docs/en/watcher/input/http.asciidoc
@@ -139,6 +139,28 @@ http://openweathermap.org/appid[OpenWeatherMap] service:
 --------------------------------------------------
 // NOTCONSOLE
 
+===== Using token-based authentication
+
+You can also call an API using a `Bearer token` instead of basic authentication. The `request.headers` object contains the HTTP headers:
+
+[source,js]
+--------------------------------------------------
+"input" : {
+  "http" : {
+    "request" : {
+      "url": "https://api.example.com/v1/something",
+      "headers": {
+        "authorization" : "Bearer ABCD1234...",
+        "content-type": "application/json"
+        # other headers params..
+        },
+      "connection_timeout": "30s"
+    }
+  }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
 ==== Using templates
 
 The `http` input supports templating. You can use <<templates,templates>> when
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java
index fefa87c57714..d9340894159b 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java
@@ -10,12 +10,13 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.MapperPlugin;
@@ -42,6 +43,7 @@
 import org.elasticsearch.xpack.analytics.multiterms.MultiTermsAggregationBuilder;
 import org.elasticsearch.xpack.analytics.normalize.NormalizePipelineAggregationBuilder;
 import org.elasticsearch.xpack.analytics.rate.InternalRate;
+import org.elasticsearch.xpack.analytics.rate.InternalResetTrackingRate;
 import org.elasticsearch.xpack.analytics.rate.RateAggregationBuilder;
 import org.elasticsearch.xpack.analytics.stringstats.InternalStringStats;
 import org.elasticsearch.xpack.analytics.stringstats.StringStatsAggregationBuilder;
@@ -119,19 +121,27 @@ public List<AggregationSpec> getAggregations() {
                 TTestAggregationBuilder::new,
                 usage.track(AnalyticsStatsAction.Item.T_TEST, TTestAggregationBuilder.PARSER)
             ).addResultReader(InternalTTest::new).setAggregatorRegistrar(TTestAggregationBuilder::registerUsage),
-            new AggregationSpec(
-                RateAggregationBuilder.NAME,
-                RateAggregationBuilder::new,
-                usage.track(AnalyticsStatsAction.Item.RATE, RateAggregationBuilder.PARSER)
-            ).addResultReader(InternalRate::new).setAggregatorRegistrar(RateAggregationBuilder::registerAggregators),
             new AggregationSpec(
                 MultiTermsAggregationBuilder.NAME,
                 MultiTermsAggregationBuilder::new,
                 usage.track(AnalyticsStatsAction.Item.MULTI_TERMS, MultiTermsAggregationBuilder.PARSER)
-            ).addResultReader(InternalMultiTerms::new).setAggregatorRegistrar(MultiTermsAggregationBuilder::registerAggregators)
+            ).addResultReader(InternalMultiTerms::new).setAggregatorRegistrar(MultiTermsAggregationBuilder::registerAggregators),
+            rateAggregation()
         );
     }
 
+    private AggregationSpec rateAggregation() {
+        AggregationSpec rate = new AggregationSpec(
+            RateAggregationBuilder.NAME,
+            RateAggregationBuilder::new,
+            usage.track(AnalyticsStatsAction.Item.RATE, RateAggregationBuilder.PARSER)
+        ).addResultReader(InternalRate::new).setAggregatorRegistrar(RateAggregationBuilder::registerAggregators);
+        if (IndexSettings.isTimeSeriesModeEnabled()) {
+            rate.addResultReader(InternalResetTrackingRate.NAME, InternalResetTrackingRate::new);
+        }
+        return rate;
+    }
+
     @Override
     public List<ActionPlugin.ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
         return List.of(
@@ -180,7 +190,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         return List.of(usage);
     }
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregationBuilder.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregationBuilder.java
index 21c8bd93b82f..2239814a3845 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregationBuilder.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregationBuilder.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.analytics.boxplot;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -164,7 +164,7 @@ public Optional<Set<String>> getOutputFieldNames() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_7_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_7_0;
     }
 }
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java
index 0a4a98f0c365..b2cc9c358f2b 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/cumulativecardinality/CumulativeCardinalityPipelineAggregationBuilder.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.analytics.cumulativecardinality;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.DocValueFormat;
@@ -131,7 +131,7 @@ protected boolean overrideBucketsPath() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_4_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_4_0;
     }
 }
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java
index 05cd4f580f80..1f8055b7e3e0 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.fielddata.FieldDataContext;
+import org.elasticsearch.index.fielddata.FormattedDocValues;
 import org.elasticsearch.index.fielddata.HistogramValue;
 import org.elasticsearch.index.fielddata.HistogramValues;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -49,6 +50,7 @@
 import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.ArrayList;
 import java.util.Map;
 import java.util.stream.Stream;
@@ -196,6 +198,33 @@ public DocValuesScriptFieldFactory getScriptFieldFactory(String name) {
                             throw new UnsupportedOperationException("The [" + CONTENT_TYPE + "] field does not " + "support scripts");
                         }
 
+                        @Override
+                        public FormattedDocValues getFormattedValues(DocValueFormat format) {
+                            try {
+                                final BinaryDocValues values = DocValues.getBinary(context.reader(), fieldName);
+                                final InternalHistogramValue value = new InternalHistogramValue();
+                                return new FormattedDocValues() {
+                                    @Override
+                                    public boolean advanceExact(int docId) throws IOException {
+                                        return values.advanceExact(docId);
+                                    }
+
+                                    @Override
+                                    public int docValueCount() {
+                                        return 1;
+                                    }
+
+                                    @Override
+                                    public Object nextValue() throws IOException {
+                                        value.reset(values.binaryValue());
+                                        return value;
+                                    }
+                                };
+                            } catch (IOException e) {
+                                throw new UncheckedIOException("Unable to loead histogram doc values", e);
+                            }
+                        }
+
                         @Override
                         public SortedBinaryDocValues getBytesValues() {
                             throw new UnsupportedOperationException(
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/movingPercentiles/MovingPercentilesPipelineAggregationBuilder.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/movingPercentiles/MovingPercentilesPipelineAggregationBuilder.java
index a0135c045bef..fd9602b9964e 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/movingPercentiles/MovingPercentilesPipelineAggregationBuilder.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/movingPercentiles/MovingPercentilesPipelineAggregationBuilder.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.analytics.movingPercentiles;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
@@ -132,7 +132,7 @@ protected boolean overrideBucketsPath() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_9_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_9_0;
     }
 }
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregationBuilder.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregationBuilder.java
index a9dd0ed0d55e..40ced2fabf67 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregationBuilder.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregationBuilder.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.analytics.multiterms;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
@@ -64,7 +64,8 @@ public class MultiTermsAggregationBuilder extends AbstractAggregationBuilder<Mul
             true,
             true,
             false,
-            true
+            true,
+            false
         );
 
         PARSER.declareBoolean(MultiTermsAggregationBuilder::showTermDocCountError, MultiTermsAggregationBuilder.SHOW_TERM_DOC_COUNT_ERROR);
@@ -435,7 +436,7 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_12_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_12_0;
     }
 }
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/normalize/NormalizePipelineAggregationBuilder.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/normalize/NormalizePipelineAggregationBuilder.java
index f2cf85646f7b..3975e04de0c7 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/normalize/NormalizePipelineAggregationBuilder.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/normalize/NormalizePipelineAggregationBuilder.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.analytics.normalize;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.DocValueFormat;
@@ -155,7 +155,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_9_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_9_0;
     }
 }
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/InternalResetTrackingRate.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/InternalResetTrackingRate.java
new file mode 100644
index 000000000000..b3cab7189d43
--- /dev/null
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/InternalResetTrackingRate.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.analytics.rate;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.AggregationReduceContext;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+
+public class InternalResetTrackingRate extends InternalNumericMetricsAggregation.SingleValue implements Rate {
+
+    public static final String NAME = "rate_with_resets";
+
+    private final double startValue;
+    private final double endValue;
+    private final long startTime;
+    private final long endTime;
+    private final double resetCompensation;
+
+    protected InternalResetTrackingRate(
+        String name,
+        DocValueFormat format,
+        Map<String, Object> metadata,
+        double startValue,
+        double endValue,
+        long startTime,
+        long endTime,
+        double resetCompensation
+    ) {
+        super(name, format, metadata);
+        this.startValue = startValue;
+        this.endValue = endValue;
+        this.startTime = startTime;
+        this.endTime = endTime;
+        this.resetCompensation = resetCompensation;
+    }
+
+    public InternalResetTrackingRate(StreamInput in) throws IOException {
+        super(in, false);
+        this.startValue = in.readDouble();
+        this.endValue = in.readDouble();
+        this.startTime = in.readLong();
+        this.endTime = in.readLong();
+        this.resetCompensation = in.readDouble();
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeDouble(startValue);
+        out.writeDouble(endValue);
+        out.writeLong(startTime);
+        out.writeLong(endTime);
+        out.writeDouble(resetCompensation);
+    }
+
+    @Override
+    public InternalAggregation reduce(List<InternalAggregation> aggregations, AggregationReduceContext reduceContext) {
+        List<InternalResetTrackingRate> toReduce = aggregations.stream()
+            .map(r -> (InternalResetTrackingRate) r)
+            .sorted(Comparator.comparingLong(o -> o.startTime))
+            .toList();
+        double resetComp = toReduce.get(0).resetCompensation;
+        double startValue = toReduce.get(0).startValue;
+        double endValue = toReduce.get(0).endValue;
+        final int endIndex = toReduce.size() - 1;
+        for (int i = 1; i < endIndex + 1; i++) {
+            InternalResetTrackingRate rate = toReduce.get(i);
+            assert rate.startTime >= toReduce.get(i - 1).endTime;
+            resetComp += rate.resetCompensation;
+            if (endValue > rate.startValue) {
+                resetComp += endValue;
+            }
+            endValue = rate.endValue;
+        }
+        return new InternalResetTrackingRate(
+            name,
+            format,
+            metadata,
+            startValue,
+            endValue,
+            toReduce.get(0).startTime,
+            toReduce.get(endIndex).endTime,
+            resetComp
+        );
+    }
+
+    @Override
+    public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        return builder.field(CommonFields.VALUE.getPreferredName(), value());
+    }
+
+    @Override
+    public double value() {
+        return (endValue - startValue + resetCompensation) / (endTime - startTime);
+    }
+
+    @Override
+    public double getValue() {
+        return value();
+    }
+
+    boolean includes(InternalResetTrackingRate other) {
+        return this.startTime < other.startTime && this.endTime > other.endTime;
+    }
+}
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/RateAggregationBuilder.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/RateAggregationBuilder.java
index 9789aa56c7ec..c0b9650144fc 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/RateAggregationBuilder.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/RateAggregationBuilder.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.analytics.rate;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Rounding;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -87,7 +87,7 @@ public RateAggregationBuilder(StreamInput in) throws IOException {
         } else {
             rateUnit = null;
         }
-        if (in.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             if (in.readBoolean()) {
                 rateMode = in.readEnum(RateMode.class);
             }
@@ -106,7 +106,7 @@ protected void innerWriteTo(StreamOutput out) throws IOException {
         } else {
             out.writeByte((byte) 0);
         }
-        if (out.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             if (rateMode != null) {
                 out.writeBoolean(true);
                 out.writeEnum(rateMode);
@@ -214,7 +214,7 @@ public int hashCode() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_10_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_10_0;
     }
 }
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorFactory.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorFactory.java
index 01c69bbdbd2c..ab0c428ac421 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorFactory.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/RateAggregatorFactory.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
@@ -64,6 +65,12 @@ static void registerAggregators(ValuesSourceRegistry.Builder builder) {
             HistogramRateAggregator::new,
             true
         );
+        builder.register(
+            RateAggregationBuilder.REGISTRY_KEY,
+            Collections.singletonList(TimeSeriesValuesSourceType.COUNTER),
+            TimeSeriesRateAggregator::new,
+            true
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregator.java
new file mode 100644
index 000000000000..bf2fdc982422
--- /dev/null
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregator.java
@@ -0,0 +1,154 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.analytics.rate;
+
+import org.elasticsearch.common.Rounding;
+import org.elasticsearch.common.util.DoubleArray;
+import org.elasticsearch.common.util.LongArray;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.MultiValueMode;
+import org.elasticsearch.search.aggregations.AggregationExecutionContext;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
+import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregator;
+import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class TimeSeriesRateAggregator extends NumericMetricsAggregator.SingleValue {
+
+    protected final ValuesSource.Numeric valuesSource;
+
+    protected DoubleArray startValues;
+    protected DoubleArray endValues;
+    protected LongArray startTimes;
+    protected LongArray endTimes;
+    protected DoubleArray resetCompensations;
+
+    private long currentBucket = -1;
+    private long currentEndTime = -1;
+    private long currentStartTime = -1;
+    private double resetCompensation = 0;
+    private double currentEndValue = -1;
+    private double currentStartValue = -1;
+    private int currentTsid = -1;
+
+    // Unused parameters are so that the constructor implements `RateAggregatorSupplier`
+    protected TimeSeriesRateAggregator(
+        String name,
+        ValuesSourceConfig valuesSourceConfig,
+        Rounding.DateTimeUnit rateUnit,
+        RateMode rateMode,
+        AggregationContext context,
+        Aggregator parent,
+        Map<String, Object> metadata
+    ) throws IOException {
+        super(name, context, parent, metadata);
+        this.valuesSource = (ValuesSource.Numeric) valuesSourceConfig.getValuesSource();
+        this.startValues = bigArrays().newDoubleArray(1, true);
+        this.endValues = bigArrays().newDoubleArray(1, true);
+        this.startTimes = bigArrays().newLongArray(1, true);
+        this.endTimes = bigArrays().newLongArray(1, true);
+        this.resetCompensations = bigArrays().newDoubleArray(1, true);
+    }
+
+    @Override
+    public InternalAggregation buildEmptyAggregation() {
+        return new InternalResetTrackingRate(name, DocValueFormat.RAW, metadata(), 0, 0, 0, 0, 0);
+    }
+
+    private void calculateLastBucket() {
+        if (currentBucket != -1) {
+            startValues.set(currentBucket, currentStartValue);
+            endValues.set(currentBucket, currentEndValue);
+            startTimes.set(currentBucket, currentStartTime);
+            endTimes.set(currentBucket, currentEndTime);
+            resetCompensations.set(currentBucket, resetCompensation);
+            currentBucket = -1;
+        }
+    }
+
+    private double checkForResets(double latestValue) {
+        if (latestValue > currentStartValue) {
+            // reset detected
+            resetCompensation += currentEndValue;
+            currentEndValue = latestValue;
+        }
+        return latestValue;
+    }
+
+    @Override
+    protected LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCtx, LeafBucketCollector sub) throws IOException {
+        NumericDoubleValues leafValues = MultiValueMode.MAX.select(valuesSource.doubleValues(aggCtx.getLeafReaderContext()));
+        return new LeafBucketCollectorBase(sub, null) {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                leafValues.advanceExact(doc);   // TODO handle missing values
+                double latestValue = leafValues.doubleValue();
+
+                if (bucket != currentBucket) {
+                    startValues = bigArrays().grow(startValues, bucket + 1);
+                    endValues = bigArrays().grow(endValues, bucket + 1);
+                    startTimes = bigArrays().grow(startTimes, bucket + 1);
+                    endTimes = bigArrays().grow(endTimes, bucket + 1);
+                    resetCompensations = bigArrays().grow(resetCompensations, bucket + 1);
+                    if (currentTsid != aggCtx.getTsidOrd()) {
+                        // if we're on a new tsid then we need to calculate the last bucket
+                        calculateLastBucket();
+                        currentTsid = aggCtx.getTsidOrd();
+                    } else {
+                        // if we're in a new bucket but in the same tsid then we update the
+                        // timestamp and last value before we calculate the last bucket
+                        currentStartTime = aggCtx.getTimestamp();
+                        currentStartValue = checkForResets(latestValue);
+                        calculateLastBucket();
+                    }
+                    currentBucket = bucket;
+                    currentStartTime = currentEndTime = aggCtx.getTimestamp();
+                    currentStartValue = currentEndValue = latestValue;
+                    resetCompensation = 0;
+                } else {
+                    currentStartTime = aggCtx.getTimestamp();
+                    currentStartValue = checkForResets(latestValue);
+                }
+            }
+        };
+    }
+
+    @Override
+    public InternalResetTrackingRate buildAggregation(long owningBucketOrd) {
+        calculateLastBucket();
+        return new InternalResetTrackingRate(
+            name,
+            DocValueFormat.RAW,
+            metadata(),
+            startValues.get(owningBucketOrd),
+            endValues.get(owningBucketOrd),
+            startTimes.get(owningBucketOrd),
+            endTimes.get(owningBucketOrd),
+            resetCompensations.get(owningBucketOrd)
+        );
+    }
+
+    @Override
+    protected void doClose() {
+        Releasables.close(startValues, endValues, startTimes, endTimes, resetCompensations);
+    }
+
+    @Override
+    public double metric(long owningBucketOrd) {
+        return buildAggregation(owningBucketOrd).getValue();
+    }
+}
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregationBuilder.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregationBuilder.java
index a112fb201f19..da02f9bcb7a5 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregationBuilder.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregationBuilder.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.analytics.stringstats;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -157,7 +157,7 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_6_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_6_0;
     }
 }
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregationBuilder.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregationBuilder.java
index 495a392846d4..71b0beed29c5 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregationBuilder.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregationBuilder.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.analytics.topmetrics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
@@ -17,6 +17,7 @@
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.aggregations.support.MultiValuesSourceFieldConfig;
+import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry.RegistryKey;
 import org.elasticsearch.search.sort.SortBuilder;
@@ -49,7 +50,12 @@ public class TopMetricsAggregationBuilder extends AbstractAggregationBuilder<Top
 
     public static void registerAggregators(ValuesSourceRegistry.Builder registry) {
         registry.registerUsage(NAME);
-        registry.register(REGISTRY_KEY, List.of(CoreValuesSourceType.NUMERIC), TopMetricsAggregator::buildNumericMetricValues, false);
+        registry.register(
+            REGISTRY_KEY,
+            List.of(CoreValuesSourceType.NUMERIC, TimeSeriesValuesSourceType.COUNTER),
+            TopMetricsAggregator::buildNumericMetricValues,
+            false
+        );
         registry.register(
             REGISTRY_KEY,
             List.of(CoreValuesSourceType.BOOLEAN, CoreValuesSourceType.DATE),
@@ -96,6 +102,7 @@ public static void registerAggregators(ValuesSourceRegistry.Builder registry) {
             true,
             false,
             false,
+            false,
             false
         );
         PARSER.declareObjectArray(constructorArg(), (p, n) -> metricParser.parse(p, null).build(), METRIC_FIELD);
@@ -221,7 +228,7 @@ public Optional<Set<String>> getOutputFieldNames() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_7_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_7_0;
     }
 }
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregationBuilder.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregationBuilder.java
index 36b1218411f0..af367759d5ac 100644
--- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregationBuilder.java
+++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregationBuilder.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.analytics.ttest;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -46,8 +46,8 @@ public class TTestAggregationBuilder extends MultiValuesSourceAggregationBuilder
 
     static {
         MultiValuesSourceParseHelper.declareCommon(PARSER, true, ValueType.NUMERIC);
-        MultiValuesSourceParseHelper.declareField(A_FIELD.getPreferredName(), PARSER, true, false, true, false);
-        MultiValuesSourceParseHelper.declareField(B_FIELD.getPreferredName(), PARSER, true, false, true, false);
+        MultiValuesSourceParseHelper.declareField(A_FIELD.getPreferredName(), PARSER, true, false, true, false, false);
+        MultiValuesSourceParseHelper.declareField(B_FIELD.getPreferredName(), PARSER, true, false, true, false, false);
         PARSER.declareString(TTestAggregationBuilder::testType, TYPE_FIELD);
         PARSER.declareInt(TTestAggregationBuilder::tails, TAILS_FIELD);
     }
@@ -180,7 +180,7 @@ public String getType() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_8_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_8_0;
     }
 }
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsStatsActionNodeResponseTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsStatsActionNodeResponseTests.java
index a093f58c003e..f50e2527b3ee 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsStatsActionNodeResponseTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsStatsActionNodeResponseTests.java
@@ -36,6 +36,11 @@ protected AnalyticsStatsAction.NodeResponse createTestInstance() {
         return new AnalyticsStatsAction.NodeResponse(node, counters);
     }
 
+    @Override
+    protected AnalyticsStatsAction.NodeResponse mutateInstance(AnalyticsStatsAction.NodeResponse instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testItemEnum() {
         int i = 0;
         // We rely on the ordinals for serialization, so they shouldn't change between version
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregationBuilderTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregationBuilderTests.java
index f72623028454..8359f42c7c6c 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregationBuilderTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregationBuilderTests.java
@@ -63,6 +63,11 @@ protected BoxplotAggregationBuilder createTestInstance() {
         return aggregationBuilder;
     }
 
+    @Override
+    protected BoxplotAggregationBuilder mutateInstance(BoxplotAggregationBuilder instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<BoxplotAggregationBuilder> instanceReader() {
         return BoxplotAggregationBuilder::new;
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregationBuilderTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregationBuilderTests.java
index dd35c9166be2..2b150c39c0c9 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregationBuilderTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/multiterms/MultiTermsAggregationBuilderTests.java
@@ -102,6 +102,11 @@ protected MultiTermsAggregationBuilder createTestInstance() {
         return aggregationBuilder;
     }
 
+    @Override
+    protected MultiTermsAggregationBuilder mutateInstance(MultiTermsAggregationBuilder instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<MultiTermsAggregationBuilder> instanceReader() {
         return MultiTermsAggregationBuilder::new;
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/InternalResetTrackingRateTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/InternalResetTrackingRateTests.java
new file mode 100644
index 000000000000..04f88ccfca05
--- /dev/null
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/InternalResetTrackingRateTests.java
@@ -0,0 +1,138 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.analytics.rate;
+
+import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.ParsedAggregation;
+import org.elasticsearch.test.InternalAggregationTestCase;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xpack.analytics.AnalyticsPlugin;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.mockito.Mockito.mock;
+
+public class InternalResetTrackingRateTests extends InternalAggregationTestCase<InternalResetTrackingRate> {
+
+    @Override
+    protected SearchPlugin registerPlugin() {
+        return new AnalyticsPlugin();
+    }
+
+    @Override
+    protected InternalResetTrackingRate createTestInstance(String name, Map<String, Object> metadata) {
+        return new InternalResetTrackingRate(name, null, metadata, 0, 0, 0, 0, 0);
+    }
+
+    private static InternalResetTrackingRate rate(double startValue, double endValue, long startTime, long endTime, double resetComp) {
+        return new InternalResetTrackingRate("n", null, null, startValue, endValue, startTime, endTime, resetComp);
+    }
+
+    public void testReduction() {
+        List<InternalAggregation> rates = List.of(
+            rate(0, 10, 1000, 2000, 0),
+            rate(10, 20, 2000, 3000, 0),
+            rate(20, 5, 3000, 4000, 25), // internal reset
+            rate(5, 15, 4000, 5000, 0),
+            rate(0, 10, 5000, 6000, 0)  // cross-boundary reset
+        );
+        InternalAggregation reduced = rates.get(0).reduce(rates, null);
+        assertThat(reduced, instanceOf(Rate.class));
+        assertThat(((Rate) reduced).getValue(), equalTo(0.01));
+    }
+
+    @Override
+    protected void assertReduced(InternalResetTrackingRate reduced, List<InternalResetTrackingRate> inputs) {
+        for (InternalResetTrackingRate input : inputs) {
+            assertEquals(0.01f, input.getValue(), 0.001);
+        }
+        assertEquals(0.01f, reduced.getValue(), 0.001);
+    }
+
+    // Buckets must always be in-order so that we can detect resets between consecutive buckets
+    @Override
+    protected boolean supportsOutOfOrderReduce() {
+        return false;
+    }
+
+    @Override
+    protected BuilderAndToReduce<InternalResetTrackingRate> randomResultsToReduce(String name, int size) {
+        // generate a monotonically increasing counter, starting at 0 finishing at 1000 and increasing
+        // by 10 each time
+        // randomly reset to 0
+        // randomly break to a new rate
+        List<InternalResetTrackingRate> internalRates = new ArrayList<>();
+        double startValue = 0, currentValue = 0;
+        double resetComp = 0;
+        long startTime = 0;
+        long endTime = 0;
+        while (internalRates.size() < size - 1) {
+            endTime += 1000;
+            currentValue += 10;
+            if (randomInt(30) == 0) {
+                resetComp += currentValue;
+                currentValue = 0;
+            }
+            if (randomInt(45) == 0) {
+                internalRates.add(rate(startValue, currentValue, startTime, endTime, resetComp));
+                startValue = currentValue;
+                resetComp = 0;
+                startTime = endTime;
+            }
+        }
+        if (startTime == endTime) {
+            endTime += 1000;
+            currentValue += 10;
+        }
+        internalRates.add(rate(startValue, currentValue, startTime, endTime, resetComp));
+        return new BuilderAndToReduce<>(mock(RateAggregationBuilder.class), internalRates);
+    }
+
+    @Override
+    protected void assertFromXContent(InternalResetTrackingRate aggregation, ParsedAggregation parsedAggregation) throws IOException {
+
+    }
+
+    @Override
+    protected List<NamedXContentRegistry.Entry> getNamedXContents() {
+        return CollectionUtils.appendToCopy(
+            super.getNamedXContents(),
+            new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(InternalResetTrackingRate.NAME), (p, c) -> {
+                assumeTrue("There is no ParsedRate yet", false);
+                return null;
+            })
+        );
+    }
+
+    public void testIncludes() {
+        InternalResetTrackingRate big = new InternalResetTrackingRate("n", null, null, 0, 0, 1000, 3000, 0);
+        InternalResetTrackingRate small = new InternalResetTrackingRate("n", null, null, 0, 0, 1500, 2500, 0);
+        assertTrue(big.includes(small));
+        assertFalse(small.includes(big));
+
+        InternalResetTrackingRate unrelated = new InternalResetTrackingRate("n", null, null, 0, 0, 100000, 1000010, 0);
+        assertFalse(big.includes(unrelated));
+        assertFalse(unrelated.includes(big));
+        assertFalse(small.includes(unrelated));
+        assertFalse(unrelated.includes(small));
+    }
+
+    @Override
+    protected InternalResetTrackingRate mutateInstance(InternalResetTrackingRate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+}
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregationBuilderTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregationBuilderTests.java
index 5ae46895e8d0..deea5ac40d0e 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregationBuilderTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/RateAggregationBuilderTests.java
@@ -67,6 +67,11 @@ protected RateAggregationBuilder createTestInstance() {
         return aggregationBuilder;
     }
 
+    @Override
+    protected RateAggregationBuilder mutateInstance(RateAggregationBuilder instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<RateAggregationBuilder> instanceReader() {
         return RateAggregationBuilder::new;
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java
new file mode 100644
index 000000000000..65cfd05e6c5a
--- /dev/null
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java
@@ -0,0 +1,157 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.analytics.rate;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.elasticsearch.aggregations.AggregationsPlugin;
+import org.elasticsearch.aggregations.bucket.timeseries.InternalTimeSeries;
+import org.elasticsearch.aggregations.bucket.timeseries.TimeSeriesAggregationBuilder;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper;
+import org.elasticsearch.index.mapper.TimeSeriesParams;
+import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
+import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogram;
+import org.elasticsearch.xpack.analytics.AnalyticsPlugin;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Consumer;
+
+import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class TimeSeriesRateAggregatorTests extends AggregatorTestCase {
+
+    @Override
+    protected List<SearchPlugin> getSearchPlugins() {
+        return List.of(new AggregationsPlugin(), new AnalyticsPlugin());
+    }
+
+    public void testSimple() throws IOException {
+        RateAggregationBuilder builder = new RateAggregationBuilder("counter_field").field("counter_field");
+        TimeSeriesAggregationBuilder tsBuilder = new TimeSeriesAggregationBuilder("tsid");
+        tsBuilder.subAggregation(builder);
+        Consumer<InternalTimeSeries> verifier = r -> {
+            assertThat(r.getBuckets(), hasSize(2));
+            assertThat(((Rate) r.getBucketByKey("{dim=1}").getAggregations().asList().get(0)).getValue(), closeTo(59.0 / 3000.0, 0.00001));
+            assertThat(((Rate) r.getBucketByKey("{dim=2}").getAggregations().asList().get(0)).getValue(), closeTo(206.0 / 4000.0, 0.00001));
+        };
+        AggTestConfig aggTestConfig = new AggTestConfig(tsBuilder, timeStampField(), counterField("counter_field"))
+            .withSplitLeavesIntoSeperateAggregators(false);
+        testCase(iw -> {
+            iw.addDocuments(docs(1000, "1", 15, 37, 60, /*reset*/ 14));
+            iw.addDocuments(docs(1000, "2", 74, 150, /*reset*/ 50, 90, /*reset*/ 40));
+        }, verifier, aggTestConfig);
+    }
+
+    public void testNestedWithinDateHistogram() throws IOException {
+        RateAggregationBuilder builder = new RateAggregationBuilder("counter_field").field("counter_field");
+        DateHistogramAggregationBuilder dateBuilder = new DateHistogramAggregationBuilder("date");
+        dateBuilder.field("@timestamp");
+        dateBuilder.fixedInterval(DateHistogramInterval.seconds(2));
+        dateBuilder.subAggregation(builder);
+        TimeSeriesAggregationBuilder tsBuilder = new TimeSeriesAggregationBuilder("tsid");
+        tsBuilder.subAggregation(dateBuilder);
+
+        Consumer<InternalTimeSeries> verifier = r -> {
+            assertThat(r.getBuckets(), hasSize(2));
+            assertThat(r.getBucketByKey("{dim=1}"), instanceOf(InternalTimeSeries.InternalBucket.class));
+            InternalDateHistogram hb = r.getBucketByKey("{dim=1}").getAggregations().get("date");
+            {
+                Rate rate = hb.getBuckets().get(1).getAggregations().get("counter_field");
+                assertThat(rate.getValue(), closeTo((60 - 37 + 14) / 2000.0, 0.00001));
+            }
+            {
+                Rate rate = hb.getBuckets().get(0).getAggregations().get("counter_field");
+                assertThat(rate.getValue(), closeTo((37 - 15) / 1000.0, 0.00001));
+            }
+            hb = r.getBucketByKey("{dim=2}").getAggregations().get("date");
+            {
+                Rate rate = hb.getBuckets().get(0).getAggregations().get("counter_field");
+                assertThat(rate.getValue(), closeTo((150 - 74) / 1000.0, 0.00001));
+            }
+            {
+                Rate rate = hb.getBuckets().get(1).getAggregations().get("counter_field");
+                assertThat(rate.getValue(), closeTo(90 / 2000.0, 0.00001));
+            }
+        };
+
+        AggTestConfig aggTestConfig = new AggTestConfig(tsBuilder, timeStampField(), counterField("counter_field"))
+            .withSplitLeavesIntoSeperateAggregators(false);
+        testCase(iw -> {
+            iw.addDocuments(docs(2000, "1", 15, 37, 60, /*reset*/ 14));
+            iw.addDocuments(docs(2000, "2", 74, 150, /*reset*/ 50, 90, /*reset*/ 40));
+        }, verifier, aggTestConfig);
+    }
+
+    private List<Document> docs(long startTimestamp, String dim, long... values) throws IOException {
+
+        List<Document> documents = new ArrayList<>();
+        for (int i = 0; i < values.length; i++) {
+            documents.add(doc(startTimestamp + (i * 1000L), tsid(dim), values[i]));
+        }
+        return documents;
+    }
+
+    private static BytesReference tsid(String dim) throws IOException {
+        TimeSeriesIdFieldMapper.TimeSeriesIdBuilder idBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null);
+        idBuilder.addString("dim", dim);
+        return idBuilder.build();
+    }
+
+    private Document doc(long timestamp, BytesReference tsid, long counterValue) {
+        Document doc = new Document();
+        doc.add(new SortedNumericDocValuesField("@timestamp", timestamp));
+        doc.add(new SortedDocValuesField("_tsid", tsid.toBytesRef()));
+        doc.add(new NumericDocValuesField("counter_field", counterValue));
+        return doc;
+    }
+
+    private MappedFieldType counterField(String name) {
+        return new NumberFieldMapper.NumberFieldType(
+            name,
+            NumberFieldMapper.NumberType.LONG,
+            true,
+            false,
+            true,
+            false,
+            null,
+            Collections.emptyMap(),
+            null,
+            false,
+            TimeSeriesParams.MetricType.COUNTER
+        );
+    }
+
+    private DateFieldMapper.DateFieldType timeStampField() {
+        return new DateFieldMapper.DateFieldType(
+            "@timestamp",
+            true,
+            false,
+            true,
+            DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
+            DateFieldMapper.Resolution.MILLISECONDS,
+            null,
+            null,
+            Collections.emptyMap()
+        );
+    }
+
+}
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/InternalStringStatsTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/InternalStringStatsTests.java
index 3a58e5d3cca5..132f3b30b2f3 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/InternalStringStatsTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/InternalStringStatsTests.java
@@ -93,7 +93,7 @@ private InternalStringStats createTestInstance(String name, Map<String, Object>
     }
 
     @Override
-    protected InternalStringStats mutateInstance(InternalStringStats instance) throws IOException {
+    protected InternalStringStats mutateInstance(InternalStringStats instance) {
         String name = instance.getName();
         long count = instance.getCount();
         long totalLength = instance.getTotalLength();
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregationBuilderTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregationBuilderTests.java
index 5998de1817f8..cc4a6f1971c0 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregationBuilderTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/stringstats/StringStatsAggregationBuilderTests.java
@@ -67,7 +67,7 @@ protected StringStatsAggregationBuilder createTestInstance() {
     }
 
     @Override
-    protected StringStatsAggregationBuilder mutateInstance(StringStatsAggregationBuilder instance) throws IOException {
+    protected StringStatsAggregationBuilder mutateInstance(StringStatsAggregationBuilder instance) {
         if (randomBoolean()) {
             StringStatsAggregationBuilder mutant = new StringStatsAggregationBuilder(instance.getName());
             mutant.showDistribution(instance.showDistribution() == false);
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetricsTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetricsTests.java
index 31b56e1ff176..a4aa3da51c29 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetricsTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/InternalTopMetricsTests.java
@@ -323,7 +323,7 @@ private InternalTopMetrics createTestInstance(
     }
 
     @Override
-    protected InternalTopMetrics mutateInstance(InternalTopMetrics instance) throws IOException {
+    protected InternalTopMetrics mutateInstance(InternalTopMetrics instance) {
         String name = instance.getName();
         SortOrder instanceSortOrder = instance.getSortOrder();
         List<String> metricNames = instance.getMetricNames();
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregationBuilderTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregationBuilderTests.java
index 9969f9da1719..b99f50fd9274 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregationBuilderTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregationBuilderTests.java
@@ -86,6 +86,11 @@ protected TopMetricsAggregationBuilder createTestInstance() {
         return new TopMetricsAggregationBuilder(randomAlphaOfLength(5), sortBuilders, between(1, 100), metricFields);
     }
 
+    @Override
+    protected TopMetricsAggregationBuilder mutateInstance(TopMetricsAggregationBuilder instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testClientBuilder() throws IOException {
         AbstractXContentTestCase.xContentTester(this::createParser, this::createTestInstance, this::toXContentThroughClientBuilder, p -> {
             p.nextToken();
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java
index e65c451f630f..35db1a0d93ab 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java
@@ -435,7 +435,7 @@ public void testTonsOfBucketsTriggersBreaker() throws IOException {
                         }
                     }
                     CircuitBreakingException e = expectThrows(CircuitBreakingException.class, () -> leaf.collect(0, bucketThatBreaks));
-                    assertThat(e.getMessage(), equalTo("test error"));
+                    assertThat(e.getMessage(), equalTo(MockBigArrays.ERROR_MESSAGE));
                     assertThat(e.getByteLimit(), equalTo(max.getBytes()));
                     assertThat(e.getBytesWanted(), equalTo(5872L));
                 }
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregationBuilderTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregationBuilderTests.java
index d6e391ee1421..9d246b4866bc 100644
--- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregationBuilderTests.java
+++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregationBuilderTests.java
@@ -80,6 +80,11 @@ protected TTestAggregationBuilder createTestInstance() {
         return aggregationBuilder;
     }
 
+    @Override
+    protected TTestAggregationBuilder mutateInstance(TTestAggregationBuilder instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<TTestAggregationBuilder> instanceReader() {
         return TTestAggregationBuilder::new;
diff --git a/x-pack/plugin/async-search/qa/rest/build.gradle b/x-pack/plugin/async-search/qa/rest/build.gradle
index 07659ad9360c..759c756ccd88 100644
--- a/x-pack/plugin/async-search/qa/rest/build.gradle
+++ b/x-pack/plugin/async-search/qa/rest/build.gradle
@@ -1,8 +1,8 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 
 apply plugin: 'elasticsearch.base-internal-es-plugin'
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
   name 'x-pack-test-deprecated-query'
diff --git a/x-pack/plugin/async-search/qa/rest/src/main/java/org/elasticsearch/query/DeprecatedQueryBuilder.java b/x-pack/plugin/async-search/qa/rest/src/main/java/org/elasticsearch/query/DeprecatedQueryBuilder.java
index 37d04e269995..ef9ca6a7763c 100644
--- a/x-pack/plugin/async-search/qa/rest/src/main/java/org/elasticsearch/query/DeprecatedQueryBuilder.java
+++ b/x-pack/plugin/async-search/qa/rest/src/main/java/org/elasticsearch/query/DeprecatedQueryBuilder.java
@@ -9,7 +9,7 @@
 
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -76,7 +76,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/x-pack/plugin/async-search/qa/security/build.gradle b/x-pack/plugin/async-search/qa/security/build.gradle
index 0ae64a3bd81d..3e29a711b0a8 100644
--- a/x-pack/plugin/async-search/qa/security/build.gradle
+++ b/x-pack/plugin/async-search/qa/security/build.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 dependencies {
   javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
diff --git a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchActionIT.java b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchActionIT.java
index 6c39083bba69..3d375707e73a 100644
--- a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchActionIT.java
+++ b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/AsyncSearchActionIT.java
@@ -10,7 +10,7 @@
 import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.common.settings.Settings;
@@ -535,8 +535,8 @@ public void testMaxResponseSize() {
     public void testCCSCheckCompatibility() throws Exception {
         SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(new SearchSourceBuilder().query(new DummyQueryBuilder() {
             @Override
-            public Version getMinimalSupportedVersion() {
-                return Version.CURRENT;
+            public TransportVersion getMinimalSupportedVersion() {
+                return TransportVersion.CURRENT;
             }
         }), indexName);
 
diff --git a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/BlockingQueryBuilder.java b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/BlockingQueryBuilder.java
index 5e5b85fe0940..3b0d849b2ddf 100644
--- a/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/BlockingQueryBuilder.java
+++ b/x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/BlockingQueryBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Weight;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.search.Queries;
@@ -129,8 +129,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 
     /**
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java
index 88ed696419d4..9f11da41fb79 100644
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java
@@ -145,7 +145,7 @@ public void testToXContent() throws IOException {
         try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
             builder.prettyPrint();
             asyncSearchResponse.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            assertEquals(formatted("""
+            assertEquals(Strings.format("""
                 {
                   "id" : "id",
                   "is_partial" : true,
@@ -160,7 +160,7 @@ public void testToXContent() throws IOException {
             builder.humanReadable(true);
             asyncSearchResponse.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("human", "true")));
             assertEquals(
-                formatted(
+                Strings.format(
                     """
                         {
                           "id" : "id",
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncStatusResponseTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncStatusResponseTests.java
index 74136109566a..12231681f647 100644
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncStatusResponseTests.java
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncStatusResponseTests.java
@@ -78,23 +78,7 @@ protected AsyncStatusResponse mutateInstance(AsyncStatusResponse instance) {
     public void testToXContent() throws IOException {
         AsyncStatusResponse response = createTestInstance();
         try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
-            String expectedJson = formatted(
-                """
-                    {
-                      "id" : "%s",
-                      "is_running" : %s,
-                      "is_partial" : %s,
-                      "start_time_in_millis" : %s,
-                      "expiration_time_in_millis" : %s,
-                      "_shards" : {
-                        "total" : %s,
-                        "successful" : %s,
-                        "skipped" : %s,
-                        "failed" : %s
-                       }
-                      %s
-                    }
-                    """,
+            Object[] args = new Object[] {
                 response.getId(),
                 response.isRunning(),
                 response.isPartial(),
@@ -104,9 +88,24 @@ public void testToXContent() throws IOException {
                 response.getSuccessfulShards(),
                 response.getSkippedShards(),
                 response.getFailedShards(),
-                response.getCompletionStatus() == null ? "" : formatted("""
-                    ,"completion_status" : %s""", response.getCompletionStatus().getStatus())
-            );
+                response.getCompletionStatus() == null ? "" : Strings.format("""
+                    ,"completion_status" : %s""", response.getCompletionStatus().getStatus()) };
+            String expectedJson = Strings.format("""
+                {
+                  "id" : "%s",
+                  "is_running" : %s,
+                  "is_partial" : %s,
+                  "start_time_in_millis" : %s,
+                  "expiration_time_in_millis" : %s,
+                  "_shards" : {
+                    "total" : %s,
+                    "successful" : %s,
+                    "skipped" : %s,
+                    "failed" : %s
+                   }
+                  %s
+                }
+                """, args);
             response.toXContent(builder, ToXContent.EMPTY_PARAMS);
             assertEquals(XContentHelper.stripWhitespace(expectedJson), Strings.toString(builder));
         }
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/CancellingAggregationBuilder.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/CancellingAggregationBuilder.java
index d77e31e418e1..773c70c3b64e 100644
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/CancellingAggregationBuilder.java
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/CancellingAggregationBuilder.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.search;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -105,7 +105,7 @@ public BucketCardinality bucketCardinality() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_7_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_7_0;
     }
 }
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/DeleteAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/DeleteAsyncSearchRequestTests.java
index b4ceb12ee9c4..68ed2fd12731 100644
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/DeleteAsyncSearchRequestTests.java
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/DeleteAsyncSearchRequestTests.java
@@ -22,4 +22,9 @@ protected Writeable.Reader<DeleteAsyncResultRequest> instanceReader() {
     protected DeleteAsyncResultRequest createTestInstance() {
         return new DeleteAsyncResultRequest(randomSearchId());
     }
+
+    @Override
+    protected DeleteAsyncResultRequest mutateInstance(DeleteAsyncResultRequest instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java
index 655ebb1f10d7..17e7b804313d 100644
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/SubmitAsyncSearchRequestTests.java
@@ -65,6 +65,11 @@ protected SubmitAsyncSearchRequest createTestInstance() {
         return searchRequest;
     }
 
+    @Override
+    protected SubmitAsyncSearchRequest mutateInstance(SubmitAsyncSearchRequest instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     protected SearchSourceBuilder randomSearchSourceBuilder() {
         SearchSourceBuilder source = new SearchSourceBuilder();
         if (randomBoolean()) {
diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/ThrowingQueryBuilder.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/ThrowingQueryBuilder.java
index f28f6e5b15a1..57aa9245c17b 100644
--- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/ThrowingQueryBuilder.java
+++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/ThrowingQueryBuilder.java
@@ -11,7 +11,7 @@
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Weight;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.search.Queries;
@@ -108,7 +108,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java b/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java
index 14344eb816ef..cc94e4817079 100644
--- a/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java
+++ b/x-pack/plugin/async/src/main/java/org/elasticsearch/xpack/async/AsyncResultsIndexPlugin.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
@@ -73,7 +73,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         List<Object> components = new ArrayList<>();
         if (DiscoveryNode.canContainData(environment.settings())) {
diff --git a/x-pack/plugin/autoscaling/build.gradle b/x-pack/plugin/autoscaling/build.gradle
index 854f418b1863..4d82adfb0b4f 100644
--- a/x-pack/plugin/autoscaling/build.gradle
+++ b/x-pack/plugin/autoscaling/build.gradle
@@ -15,6 +15,7 @@ dependencies {
   compileOnly project(path: xpackModule('core'))
   testImplementation(testArtifact(project(xpackModule('core'))))
   testImplementation project(':modules:data-streams')
+  testImplementation project(path: xpackModule('blob-cache'))
   testImplementation project(path: xpackModule('searchable-snapshots'))
   testImplementation project(path: xpackModule('ilm'))
 
diff --git a/x-pack/plugin/autoscaling/qa/rest/build.gradle b/x-pack/plugin/autoscaling/qa/rest/build.gradle
index 143e271412d4..19254880a708 100644
--- a/x-pack/plugin/autoscaling/qa/rest/build.gradle
+++ b/x-pack/plugin/autoscaling/qa/rest/build.gradle
@@ -1,7 +1,7 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 dependencies {
   yamlRestTestImplementation(testArtifact(project(xpackModule('core'))))
diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/AbstractFrozenAutoscalingIntegTestCase.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/AbstractFrozenAutoscalingIntegTestCase.java
index c75de133789a..2fd1e6097051 100644
--- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/AbstractFrozenAutoscalingIntegTestCase.java
+++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/AbstractFrozenAutoscalingIntegTestCase.java
@@ -8,6 +8,8 @@
 package org.elasticsearch.xpack.autoscaling;
 
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
+import org.elasticsearch.blobcache.BlobCachePlugin;
+import org.elasticsearch.blobcache.shared.SharedBlobCacheService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.routing.allocation.DataTier;
@@ -25,7 +27,6 @@
 import org.elasticsearch.xpack.autoscaling.shards.LocalStateAutoscalingAndSearchableSnapshots;
 import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotAction;
 import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService;
 
 import java.util.Collection;
 import java.util.List;
@@ -63,7 +64,7 @@ protected boolean addMockInternalEngine() {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return List.of(LocalStateAutoscalingAndSearchableSnapshots.class);
+        return List.of(BlobCachePlugin.class, LocalStateAutoscalingAndSearchableSnapshots.class);
     }
 
     @Override
@@ -72,7 +73,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
             .put(super.nodeSettings(nodeOrdinal, otherSettings))
             .put(SELF_GENERATED_LICENSE_TYPE.getKey(), "trial");
         if (DiscoveryNode.hasRole(otherSettings, DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE)) {
-            builder.put(FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), new ByteSizeValue(10, ByteSizeUnit.MB));
+            builder.put(SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), new ByteSizeValue(10, ByteSizeUnit.MB));
         }
         return builder.build();
     }
diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java
index b657876323e2..14b849cf6d11 100644
--- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java
+++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.autoscaling.existence;
 
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.blobcache.BlobCachePlugin;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.settings.Settings;
@@ -71,7 +72,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return List.of(LocalStateAutoscalingAndSearchableSnapshotsAndIndexLifecycle.class);
+        return List.of(BlobCachePlugin.class, LocalStateAutoscalingAndSearchableSnapshotsAndIndexLifecycle.class);
     }
 
     public void testZeroToOne() throws Exception {
@@ -131,7 +132,7 @@ public void testZeroToOne() throws Exception {
             String[] indices = indices();
             assertThat(indices, arrayContaining(PARTIAL_INDEX_NAME));
             assertThat(indices, not(arrayContaining(INDEX_NAME)));
-        }, 30, TimeUnit.SECONDS);
+        }, 60, TimeUnit.SECONDS);
         ensureGreen();
     }
 
diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java
index 6dc2a8d8be88..e67406423dbd 100644
--- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java
+++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.ClusterInfoService;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.routing.allocation.DataTier;
@@ -23,7 +24,6 @@
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.NodeRoles;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.autoscaling.action.GetAutoscalingCapacityAction;
 import org.elasticsearch.xpack.autoscaling.action.PutAutoscalingPolicyAction;
@@ -231,7 +231,6 @@ public void testScaleFromEmptyLegacy() {
         );
     }
 
-    @TestLogging(value = "org.elasticsearch.monitor.fs.FsService:DEBUG", reason = "https://github.com/elastic/elasticsearch/issues/91083")
     public void testScaleWhileShrinking() throws Exception {
         internalCluster().startMasterOnlyNode();
         final String dataNode1Name = internalCluster().startDataOnlyNode();
@@ -515,7 +514,8 @@ public void testRoles() {
         ReactiveStorageDeciderService service = new ReactiveStorageDeciderService(
             Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-            null
+            null,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
         assertThat(
             service.roles().stream().sorted().collect(Collectors.toList()),
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java
index 94f716966c37..d1071a11e9ed 100644
--- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/Autoscaling.java
@@ -15,7 +15,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -92,7 +92,7 @@ public class Autoscaling extends Plugin implements ActionPlugin, ExtensiblePlugi
 
     private final List<AutoscalingExtension> autoscalingExtensions;
     private final SetOnce<ClusterService> clusterServiceHolder = new SetOnce<>();
-    private final SetOnce<AllocationDeciders> allocationDeciders = new SetOnce<>();
+    private final SetOnce<AllocationService> allocationServiceHolder = new SetOnce<>();
     private final AutoscalingLicenseChecker autoscalingLicenseChecker;
     private final SetOnce<ReservedAutoscalingPolicyAction> reservedAutoscalingPolicyAction = new SetOnce<>();
 
@@ -119,10 +119,10 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         this.clusterServiceHolder.set(clusterService);
-        this.allocationDeciders.set(allocationDeciders);
+        this.allocationServiceHolder.set(allocationService);
         var capacityServiceHolder = new AutoscalingCalculateCapacityService.Holder(this);
         this.reservedAutoscalingPolicyAction.set(new ReservedAutoscalingPolicyAction(capacityServiceHolder));
         return List.of(capacityServiceHolder, autoscalingLicenseChecker, new AutoscalingNodeInfoService(clusterService, client));
@@ -213,12 +213,23 @@ public void loadExtensions(ExtensionLoader loader) {
 
     @Override
     public Collection<AutoscalingDeciderService> deciders() {
-        assert allocationDeciders.get() != null;
+        final var allocationService = allocationServiceHolder.get();
+        assert allocationService != null;
         final ClusterService clusterService = clusterServiceHolder.get();
         return List.of(
             new FixedAutoscalingDeciderService(),
-            new ReactiveStorageDeciderService(clusterService.getSettings(), clusterService.getClusterSettings(), allocationDeciders.get()),
-            new ProactiveStorageDeciderService(clusterService.getSettings(), clusterService.getClusterSettings(), allocationDeciders.get()),
+            new ReactiveStorageDeciderService(
+                clusterService.getSettings(),
+                clusterService.getClusterSettings(),
+                allocationService.getAllocationDeciders(),
+                allocationService.getShardRoutingRoleStrategy()
+            ),
+            new ProactiveStorageDeciderService(
+                clusterService.getSettings(),
+                clusterService.getClusterSettings(),
+                allocationService.getAllocationDeciders(),
+                allocationService.getShardRoutingRoleStrategy()
+            ),
             new FrozenShardsDeciderService(),
             new FrozenStorageDeciderService(),
             new FrozenExistenceDeciderService()
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java
index 1f83a57985c7..33d9763acc2b 100644
--- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.autoscaling;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.DiffableUtils;
 import org.elasticsearch.cluster.NamedDiff;
@@ -15,15 +15,17 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.autoscaling.policy.AutoscalingPolicyMetadata;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.EnumSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -110,14 +112,13 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_8_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_8_0;
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(POLICIES_FIELD.getPreferredName(), policies);
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return ChunkedToXContentHelper.xContentValuesMap(POLICIES_FIELD.getPreferredName(), policies);
     }
 
     @Override
@@ -170,8 +171,8 @@ static Diff<AutoscalingPolicyMetadata> readFrom(final StreamInput in) throws IOE
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.V_7_8_0;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.V_7_8_0;
         }
     }
 }
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacity.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacity.java
index e005a6af0590..452cdbeb9512 100644
--- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacity.java
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCapacity.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.autoscaling.capacity;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -46,7 +46,7 @@ public AutoscalingResources(ByteSizeValue storage, ByteSizeValue memory, Process
         public AutoscalingResources(StreamInput in) throws IOException {
             this.storage = in.readOptionalWriteable(ByteSizeValue::readFrom);
             this.memory = in.readOptionalWriteable(ByteSizeValue::readFrom);
-            if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
                 this.processors = in.readOptionalWriteable(Processors::readFrom);
             } else {
                 this.processors = null;
@@ -93,7 +93,7 @@ public boolean isFragment() {
         public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalWriteable(storage);
             out.writeOptionalWriteable(memory);
-            if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
                 out.writeOptionalWriteable(processors);
             }
         }
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java
index c3450e0f4311..81d2122d7141 100644
--- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/FixedAutoscalingDeciderService.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.autoscaling.capacity;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -119,7 +119,7 @@ public FixedReason(StreamInput in) throws IOException {
             this.storage = in.readOptionalWriteable(ByteSizeValue::readFrom);
             this.memory = in.readOptionalWriteable(ByteSizeValue::readFrom);
             this.nodes = in.readInt();
-            if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
                 this.processors = in.readOptionalWriteable(Processors::readFrom);
             } else {
                 this.processors = null;
@@ -149,7 +149,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalWriteable(storage);
             out.writeOptionalWriteable(memory);
             out.writeInt(nodes);
-            if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
                 out.writeOptionalWriteable(processors);
             }
         }
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderService.java
index 5b076273f66c..24886e6d6e67 100644
--- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderService.java
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderService.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.autoscaling.storage;
 
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.cluster.routing.ShardRoutingRoleStrategy;
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -32,10 +33,17 @@ public class ProactiveStorageDeciderService implements AutoscalingDeciderService
 
     private final DiskThresholdSettings diskThresholdSettings;
     private final AllocationDeciders allocationDeciders;
-
-    public ProactiveStorageDeciderService(Settings settings, ClusterSettings clusterSettings, AllocationDeciders allocationDeciders) {
+    private final ShardRoutingRoleStrategy shardRoutingRoleStrategy;
+
+    public ProactiveStorageDeciderService(
+        Settings settings,
+        ClusterSettings clusterSettings,
+        AllocationDeciders allocationDeciders,
+        ShardRoutingRoleStrategy shardRoutingRoleStrategy
+    ) {
         this.diskThresholdSettings = new DiskThresholdSettings(settings, clusterSettings);
         this.allocationDeciders = allocationDeciders;
+        this.shardRoutingRoleStrategy = shardRoutingRoleStrategy;
     }
 
     @Override
@@ -61,7 +69,9 @@ public AutoscalingDeciderResult scale(Settings configuration, AutoscalingDecider
         ReactiveStorageDeciderService.AllocationState allocationState = new ReactiveStorageDeciderService.AllocationState(
             context,
             diskThresholdSettings,
-            allocationDeciders
+            allocationDeciders,
+            shardRoutingRoleStrategy
+
         );
         long unassignedBytesBeforeForecast = allocationState.storagePreventsAllocation().sizeInBytes();
         assert unassignedBytesBeforeForecast >= 0;
diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java
index d7b729e07850..e38ee682a477 100644
--- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java
+++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.autoscaling.storage;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.DiskUsage;
@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingRoleStrategy;
 import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
@@ -84,6 +85,7 @@ public class ReactiveStorageDeciderService implements AutoscalingDeciderService
 
     private final DiskThresholdSettings diskThresholdSettings;
     private final AllocationDeciders allocationDeciders;
+    private final ShardRoutingRoleStrategy shardRoutingRoleStrategy;
 
     private static final Predicate<String> REMOVE_NODE_LOCKED_FILTER_INITIAL = removeNodeLockedFilterPredicate(
         IndexMetadata.INDEX_ROUTING_INITIAL_RECOVERY_GROUP_SETTING.getKey()
@@ -103,9 +105,15 @@ private static Predicate<String> removeNodeLockedFilterPredicate(String settingP
         );
     }
 
-    public ReactiveStorageDeciderService(Settings settings, ClusterSettings clusterSettings, AllocationDeciders allocationDeciders) {
+    public ReactiveStorageDeciderService(
+        Settings settings,
+        ClusterSettings clusterSettings,
+        AllocationDeciders allocationDeciders,
+        ShardRoutingRoleStrategy shardRoutingRoleStrategy
+    ) {
         this.diskThresholdSettings = new DiskThresholdSettings(settings, clusterSettings);
         this.allocationDeciders = allocationDeciders;
+        this.shardRoutingRoleStrategy = shardRoutingRoleStrategy;
     }
 
     @Override
@@ -168,7 +176,7 @@ public AutoscalingDeciderResult scale(Settings configuration, AutoscalingDecider
     }
 
     AllocationState allocationState(AutoscalingDeciderContext context) {
-        return new AllocationState(context, diskThresholdSettings, allocationDeciders);
+        return new AllocationState(context, diskThresholdSettings, allocationDeciders, shardRoutingRoleStrategy);
     }
 
     static String message(long unassignedBytes, long assignedBytes) {
@@ -229,6 +237,7 @@ public static class AllocationState {
         private final ClusterState state;
         private final ClusterState originalState;
         private final AllocationDeciders allocationDeciders;
+        private final ShardRoutingRoleStrategy shardRoutingRoleStrategy;
         private final DiskThresholdSettings diskThresholdSettings;
         private final ClusterInfo info;
         private final SnapshotShardSizeInfo shardSizeInfo;
@@ -240,11 +249,13 @@ public static class AllocationState {
         AllocationState(
             AutoscalingDeciderContext context,
             DiskThresholdSettings diskThresholdSettings,
-            AllocationDeciders allocationDeciders
+            AllocationDeciders allocationDeciders,
+            ShardRoutingRoleStrategy shardRoutingRoleStrategy
         ) {
             this(
                 context.state(),
                 allocationDeciders,
+                shardRoutingRoleStrategy,
                 diskThresholdSettings,
                 context.info(),
                 context.snapshotShardSizeInfo(),
@@ -256,6 +267,7 @@ public static class AllocationState {
         AllocationState(
             ClusterState state,
             AllocationDeciders allocationDeciders,
+            ShardRoutingRoleStrategy shardRoutingRoleStrategy,
             DiskThresholdSettings diskThresholdSettings,
             ClusterInfo info,
             SnapshotShardSizeInfo shardSizeInfo,
@@ -265,6 +277,7 @@ public static class AllocationState {
             this.state = removeNodeLockFilters(state);
             this.originalState = state;
             this.allocationDeciders = allocationDeciders;
+            this.shardRoutingRoleStrategy = shardRoutingRoleStrategy;
             this.diskThresholdSettings = diskThresholdSettings;
             this.info = info;
             this.shardSizeInfo = shardSizeInfo;
@@ -554,7 +567,7 @@ private SingleForecast(Map<IndexMetadata, Long> additionalIndices, DataStream up
             }
 
             public void applyRouting(RoutingTable.Builder routing) {
-                additionalIndices.keySet().forEach(routing::addAsNew);
+                additionalIndices.keySet().forEach(indexMetadata -> routing.addAsNew(indexMetadata));
             }
 
             public void applyMetadata(Metadata.Builder metadataBuilder) {
@@ -592,7 +605,7 @@ public AllocationState forecast(long forecastWindow, long now) {
                 return this;
             }
             Metadata.Builder metadataBuilder = Metadata.builder(state.metadata());
-            RoutingTable.Builder routingTableBuilder = RoutingTable.builder(state.routingTable());
+            RoutingTable.Builder routingTableBuilder = RoutingTable.builder(shardRoutingRoleStrategy, state.routingTable());
             Map<String, Long> sizeBuilder = new HashMap<>();
             singleForecasts.forEach(p -> p.applyMetadata(metadataBuilder));
             singleForecasts.forEach(p -> p.applyRouting(routingTableBuilder));
@@ -604,6 +617,7 @@ public AllocationState forecast(long forecastWindow, long now) {
             return new AllocationState(
                 forecastClusterState,
                 allocationDeciders,
+                shardRoutingRoleStrategy,
                 diskThresholdSettings,
                 forecastInfo,
                 shardSizeInfo,
@@ -831,7 +845,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     public static class ReactiveReason implements AutoscalingDeciderResult.Reason {
 
         static final int MAX_AMOUNT_OF_SHARDS = 512;
-        private static final Version SHARD_IDS_OUTPUT_VERSION = Version.V_8_4_0;
+        private static final TransportVersion SHARD_IDS_OUTPUT_VERSION = TransportVersion.V_8_4_0;
 
         private final String reason;
         private final long unassigned;
@@ -861,7 +875,7 @@ public ReactiveReason(StreamInput in) throws IOException {
             this.reason = in.readString();
             this.unassigned = in.readLong();
             this.assigned = in.readLong();
-            if (in.getVersion().onOrAfter(SHARD_IDS_OUTPUT_VERSION)) {
+            if (in.getTransportVersion().onOrAfter(SHARD_IDS_OUTPUT_VERSION)) {
                 unassignedShardIds = Collections.unmodifiableSortedSet(new TreeSet<>(in.readSet(ShardId::new)));
                 assignedShardIds = Collections.unmodifiableSortedSet(new TreeSet<>(in.readSet(ShardId::new)));
             } else {
@@ -901,7 +915,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(reason);
             out.writeLong(unassigned);
             out.writeLong(assigned);
-            if (out.getVersion().onOrAfter(SHARD_IDS_OUTPUT_VERSION)) {
+            if (out.getTransportVersion().onOrAfter(SHARD_IDS_OUTPUT_VERSION)) {
                 out.writeCollection(unassignedShardIds);
                 out.writeCollection(assignedShardIds);
             }
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadataDiffableSerializationTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadataDiffableSerializationTests.java
index 75ed5bf32e33..1cecd7faf1df 100644
--- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadataDiffableSerializationTests.java
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadataDiffableSerializationTests.java
@@ -11,13 +11,12 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.test.SimpleDiffableSerializationTestCase;
+import org.elasticsearch.test.ChunkedToXContentDiffableSerializationTestCase;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.autoscaling.policy.AutoscalingPolicy;
 import org.elasticsearch.xpack.autoscaling.policy.AutoscalingPolicyMetadata;
 
-import java.io.IOException;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
@@ -25,7 +24,7 @@
 import static org.elasticsearch.xpack.autoscaling.AutoscalingTestCase.randomAutoscalingMetadata;
 import static org.elasticsearch.xpack.autoscaling.AutoscalingTestCase.randomAutoscalingPolicy;
 
-public class AutoscalingMetadataDiffableSerializationTests extends SimpleDiffableSerializationTestCase<Metadata.Custom> {
+public class AutoscalingMetadataDiffableSerializationTests extends ChunkedToXContentDiffableSerializationTestCase<Metadata.Custom> {
 
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
@@ -38,7 +37,7 @@ protected NamedXContentRegistry xContentRegistry() {
     }
 
     @Override
-    protected AutoscalingMetadata doParseInstance(final XContentParser parser) throws IOException {
+    protected AutoscalingMetadata doParseInstance(final XContentParser parser) {
         return AutoscalingMetadata.parse(parser);
     }
 
@@ -78,5 +77,4 @@ protected Metadata.Custom mutateInstance(final Metadata.Custom instance) {
     protected Writeable.Reader<Diff<Metadata.Custom>> diffReader() {
         return AutoscalingMetadata.AutoscalingMetadataDiff::new;
     }
-
 }
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/DeleteAutoscalingPolicyActionRequestWireSerializingTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/DeleteAutoscalingPolicyActionRequestWireSerializingTests.java
index 61fba15c6286..d524c58922fd 100644
--- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/DeleteAutoscalingPolicyActionRequestWireSerializingTests.java
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/DeleteAutoscalingPolicyActionRequestWireSerializingTests.java
@@ -23,4 +23,9 @@ protected DeleteAutoscalingPolicyAction.Request createTestInstance() {
         return new DeleteAutoscalingPolicyAction.Request(randomAlphaOfLength(8));
     }
 
+    @Override
+    protected DeleteAutoscalingPolicyAction.Request mutateInstance(DeleteAutoscalingPolicyAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
 }
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingCapacityActionRequestWireSerializingTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingCapacityActionRequestWireSerializingTests.java
index 2fb0685fe3dd..568604b13d98 100644
--- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingCapacityActionRequestWireSerializingTests.java
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingCapacityActionRequestWireSerializingTests.java
@@ -23,4 +23,9 @@ protected GetAutoscalingCapacityAction.Request createTestInstance() {
         return new GetAutoscalingCapacityAction.Request();
     }
 
+    @Override
+    protected GetAutoscalingCapacityAction.Request mutateInstance(GetAutoscalingCapacityAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
 }
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingCapacityActionResponseWireSerializingTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingCapacityActionResponseWireSerializingTests.java
index ef7ea15b4340..c5b5058e2d1c 100644
--- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingCapacityActionResponseWireSerializingTests.java
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingCapacityActionResponseWireSerializingTests.java
@@ -42,4 +42,9 @@ protected GetAutoscalingCapacityAction.Response createTestInstance() {
         return new GetAutoscalingCapacityAction.Response(Collections.unmodifiableSortedMap(results));
     }
 
+    @Override
+    protected GetAutoscalingCapacityAction.Response mutateInstance(GetAutoscalingCapacityAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
 }
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingPolicyActionRequestWireSerializingTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingPolicyActionRequestWireSerializingTests.java
index 78eac462121b..cd7d406af698 100644
--- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingPolicyActionRequestWireSerializingTests.java
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingPolicyActionRequestWireSerializingTests.java
@@ -23,4 +23,9 @@ protected GetAutoscalingPolicyAction.Request createTestInstance() {
         return new GetAutoscalingPolicyAction.Request(randomAlphaOfLength(8));
     }
 
+    @Override
+    protected GetAutoscalingPolicyAction.Request mutateInstance(GetAutoscalingPolicyAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
 }
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingPolicyActionResponseWireSerializingTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingPolicyActionResponseWireSerializingTests.java
index c1b65fe61a6e..86259c4fedd6 100644
--- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingPolicyActionResponseWireSerializingTests.java
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingPolicyActionResponseWireSerializingTests.java
@@ -27,6 +27,11 @@ protected GetAutoscalingPolicyAction.Response createTestInstance() {
         return new GetAutoscalingPolicyAction.Response(randomAutoscalingPolicy());
     }
 
+    @Override
+    protected GetAutoscalingPolicyAction.Response mutateInstance(GetAutoscalingPolicyAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
         return AutoscalingTestCase.getAutoscalingNamedWriteableRegistry();
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/PutAutoscalingPolicyActionRequestWireSerializingTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/PutAutoscalingPolicyActionRequestWireSerializingTests.java
index f71684d73001..f2e35732e6fb 100644
--- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/PutAutoscalingPolicyActionRequestWireSerializingTests.java
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/action/PutAutoscalingPolicyActionRequestWireSerializingTests.java
@@ -25,6 +25,11 @@ protected PutAutoscalingPolicyAction.Request createTestInstance() {
         return TransportPutAutoscalingPolicyActionTests.randomPutAutoscalingPolicyRequest();
     }
 
+    @Override
+    protected PutAutoscalingPolicyAction.Request mutateInstance(PutAutoscalingPolicyAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
         return AutoscalingTestCase.getAutoscalingNamedWriteableRegistry();
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultWireSerializingTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultWireSerializingTests.java
index 4ae4501949dc..93a911c28338 100644
--- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultWireSerializingTests.java
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultWireSerializingTests.java
@@ -29,4 +29,9 @@ protected AutoscalingDeciderResult createTestInstance() {
         return AutoscalingTestCase.randomAutoscalingDeciderResult();
     }
 
+    @Override
+    protected AutoscalingDeciderResult mutateInstance(AutoscalingDeciderResult instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
 }
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultsWireSerializingTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultsWireSerializingTests.java
index d98d02ac80dc..a5b7157619a0 100644
--- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultsWireSerializingTests.java
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderResultsWireSerializingTests.java
@@ -29,4 +29,9 @@ protected AutoscalingDeciderResults createTestInstance() {
         return AutoscalingTestCase.randomAutoscalingDeciderResults();
     }
 
+    @Override
+    protected AutoscalingDeciderResults mutateInstance(AutoscalingDeciderResults instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
 }
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/shards/FrozenShardsDeciderReasonWireSerializationTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/shards/FrozenShardsDeciderReasonWireSerializationTests.java
index aac03fe417ce..2a1e23ecc41e 100644
--- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/shards/FrozenShardsDeciderReasonWireSerializationTests.java
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/shards/FrozenShardsDeciderReasonWireSerializationTests.java
@@ -11,8 +11,6 @@
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
 
-import java.io.IOException;
-
 public class FrozenShardsDeciderReasonWireSerializationTests extends AbstractWireSerializingTestCase<
     FrozenShardsDeciderService.FrozenShardsReason> {
 
@@ -27,8 +25,7 @@ protected FrozenShardsDeciderService.FrozenShardsReason createTestInstance() {
     }
 
     @Override
-    protected FrozenShardsDeciderService.FrozenShardsReason mutateInstance(FrozenShardsDeciderService.FrozenShardsReason instance)
-        throws IOException {
+    protected FrozenShardsDeciderService.FrozenShardsReason mutateInstance(FrozenShardsDeciderService.FrozenShardsReason instance) {
         return new FrozenShardsDeciderService.FrozenShardsReason(
             randomValueOtherThan(instance.shards(), ESTestCase::randomNonNegativeLong)
         );
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderServiceTests.java
index 52f248408d40..39f82eca5921 100644
--- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderServiceTests.java
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderServiceTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.DiskUsage;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
@@ -70,7 +71,9 @@ public void testScale() {
         );
         ClusterState.Builder stateBuilder = ClusterState.builder(originalState);
         IntStream.range(0, between(1, 10)).forEach(i -> ReactiveStorageDeciderServiceTests.addNode(stateBuilder));
-        stateBuilder.routingTable(addRouting(originalState.metadata(), RoutingTable.builder()).build());
+        stateBuilder.routingTable(
+            addRouting(originalState.metadata(), RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)).build()
+        );
         long lastCreated = System.currentTimeMillis();
         applyCreatedDates(
             originalState,
@@ -92,7 +95,12 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
             }
         });
         AllocationDeciders allocationDeciders = new AllocationDeciders(allocationDecidersList);
-        ProactiveStorageDeciderService service = new ProactiveStorageDeciderService(Settings.EMPTY, clusterSettings, allocationDeciders);
+        ProactiveStorageDeciderService service = new ProactiveStorageDeciderService(
+            Settings.EMPTY,
+            clusterSettings,
+            allocationDeciders,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
+        );
         AutoscalingCapacity currentCapacity = ReactiveStorageDeciderDecisionTests.randomCurrentCapacity();
         ClusterInfo info = randomClusterInfo(state);
         AutoscalingDeciderContext context = new AutoscalingDeciderContext() {
@@ -168,11 +176,14 @@ public void testForecastNoDates() {
             randomBoolean()
         );
         ClusterState.Builder stateBuilder = ClusterState.builder(originalState);
-        stateBuilder.routingTable(addRouting(originalState.metadata(), RoutingTable.builder()).build());
+        stateBuilder.routingTable(
+            addRouting(originalState.metadata(), RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)).build()
+        );
         ClusterState state = stateBuilder.build();
         ReactiveStorageDeciderService.AllocationState allocationState = new ReactiveStorageDeciderService.AllocationState(
             state,
             null,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
             null,
             null,
             null,
@@ -191,7 +202,9 @@ public void testForecastZero() {
         );
         ClusterState.Builder stateBuilder = ClusterState.builder(originalState);
         IntStream.range(0, between(1, 10)).forEach(i -> ReactiveStorageDeciderServiceTests.addNode(stateBuilder));
-        stateBuilder.routingTable(addRouting(originalState.metadata(), RoutingTable.builder()).build());
+        stateBuilder.routingTable(
+            addRouting(originalState.metadata(), RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)).build()
+        );
         long lastCreated = randomNonNegativeLong();
         applyCreatedDates(
             originalState,
@@ -205,6 +218,7 @@ public void testForecastZero() {
         ReactiveStorageDeciderService.AllocationState allocationState = new ReactiveStorageDeciderService.AllocationState(
             state,
             null,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
             null,
             randomClusterInfo(state),
             null,
@@ -228,7 +242,9 @@ public void testForecast() {
             randomBoolean()
         );
         ClusterState.Builder stateBuilder = ClusterState.builder(originalState);
-        stateBuilder.routingTable(addRouting(originalState.metadata(), RoutingTable.builder()).build());
+        stateBuilder.routingTable(
+            addRouting(originalState.metadata(), RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)).build()
+        );
         IntStream.range(0, between(1, 10)).forEach(i -> ReactiveStorageDeciderServiceTests.addNode(stateBuilder));
         long lastCreated = randomNonNegativeLong();
         applyCreatedDates(
@@ -249,6 +265,7 @@ public void testForecast() {
         ReactiveStorageDeciderService.AllocationState allocationState = new ReactiveStorageDeciderService.AllocationState(
             state,
             null,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
             null,
             info,
             null,
@@ -357,7 +374,7 @@ private void startAll(RoutingAllocation allocation) {
     }
 
     private RoutingTable.Builder addRouting(Iterable<IndexMetadata> indices, RoutingTable.Builder builder) {
-        indices.forEach(builder::addAsNew);
+        indices.forEach(indexMetadata -> builder.addAsNew(indexMetadata));
         return builder;
     }
 
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java
index ed9278aca34f..7a15873b234d 100644
--- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.DiskUsage;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -52,7 +53,6 @@
 
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -409,7 +409,8 @@ private static void verify(
         ReactiveStorageDeciderService.AllocationState allocationState = new ReactiveStorageDeciderService.AllocationState(
             createContext(state, Set.of(role)),
             DISK_THRESHOLD_SETTINGS,
-            createAllocationDeciders(allocationDeciders)
+            createAllocationDeciders(allocationDeciders),
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
         assertThat(subject.invoke(allocationState), equalTo(expected));
     }
@@ -422,7 +423,8 @@ private static void verifyScale(ClusterState state, long expectedDifference, Str
         ReactiveStorageDeciderService decider = new ReactiveStorageDeciderService(
             Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-            createAllocationDeciders(allocationDeciders)
+            createAllocationDeciders(allocationDeciders),
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
         TestAutoscalingDeciderContext context = createContext(state, Set.of(DiscoveryNodeRole.DATA_HOT_NODE_ROLE));
         AutoscalingDeciderResult result = decider.scale(Settings.EMPTY, context);
@@ -470,14 +472,16 @@ private boolean hasStartedSubjectShard() {
     }
 
     private static AllocationDeciders createAllocationDeciders(AllocationDecider... extraDeciders) {
-        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        Collection<AllocationDecider> systemAllocationDeciders = ClusterModule.createAllocationDeciders(
+        ClusterSettings clusterSettings = ClusterSettings.createBuiltInClusterSettings(
             Settings.builder()
                 .put(
                     ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING.getKey(),
                     Integer.MAX_VALUE
                 )
-                .build(),
+                .build()
+        );
+        Collection<AllocationDecider> systemAllocationDeciders = ClusterModule.createAllocationDeciders(
+            Settings.EMPTY,
             clusterSettings,
             Collections.emptyList()
         );
@@ -515,7 +519,6 @@ private void startRandomShards() {
                 allocation.routingNodes(),
                 ShardRoutingState.INITIALIZING
             );
-            initializingShards.sort(Comparator.comparing(ShardRouting::shardId).thenComparing(ShardRouting::primary, Boolean::compare));
             List<ShardRouting> shards = randomSubsetOf(Math.min(randomIntBetween(1, 100), initializingShards.size()), initializingShards);
 
             // replicas before primaries, since replicas can be reinit'ed, resulting in a new ShardRouting instance.
@@ -652,7 +655,7 @@ private static ClusterState addRandomIndices(int minShards, int maxShardCopies,
         String[] tierSettingNames = new String[] { DataTier.TIER_PREFERENCE };
         int shards = randomIntBetween(minShards, 20);
         Metadata.Builder builder = Metadata.builder();
-        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         while (shards > 0) {
             IndexMetadata indexMetadata = IndexMetadata.builder("test" + "-" + shards)
                 .settings(settings(Version.CURRENT).put(randomFrom(tierSettingNames), "data_hot"))
diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java
index fe9e60899465..f9717b42497b 100644
--- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java
+++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.DiskUsage;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -224,7 +225,7 @@ public void testSizeOf() {
             .build();
         metaBuilder.put(indexMetadata, true);
         stateBuilder.metadata(metaBuilder);
-        stateBuilder.routingTable(RoutingTable.builder().addAsNew(indexMetadata).build());
+        stateBuilder.routingTable(RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(indexMetadata).build());
         addNode(stateBuilder);
         addNode(stateBuilder);
         ClusterState initialClusterState = stateBuilder.build();
@@ -285,7 +286,7 @@ public void testMaxNodeLockedSizeUsingAttributes() {
             .build();
         metaBuilder.put(indexMetadata, true);
         stateBuilder.metadata(metaBuilder);
-        stateBuilder.routingTable(RoutingTable.builder().addAsNew(indexMetadata).build());
+        stateBuilder.routingTable(RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(indexMetadata).build());
         ClusterState clusterState = stateBuilder.build();
 
         long baseSize = between(1, 10);
@@ -341,7 +342,12 @@ public void testNodeLockSplitClone() {
         metaBuilder.put(sourceIndexMetadata, true);
         metaBuilder.put(indexMetadata, true);
         stateBuilder.metadata(metaBuilder);
-        stateBuilder.routingTable(RoutingTable.builder().addAsNew(sourceIndexMetadata).addAsNew(indexMetadata).build());
+        stateBuilder.routingTable(
+            RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+                .addAsNew(sourceIndexMetadata)
+                .addAsNew(indexMetadata)
+                .build()
+        );
         ClusterState clusterState = stateBuilder.build();
 
         long sourceSize = between(1, 10);
@@ -374,6 +380,7 @@ private ReactiveStorageDeciderService.AllocationState createAllocationState(Map<
         ReactiveStorageDeciderService.AllocationState allocationState = new ReactiveStorageDeciderService.AllocationState(
             clusterState,
             null,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
             null,
             info,
             null,
@@ -419,7 +426,11 @@ public void testSizeOfSnapshot() {
             .build();
         metaBuilder.put(indexMetadata, true);
         stateBuilder.metadata(metaBuilder);
-        stateBuilder.routingTable(RoutingTable.builder().addAsNewRestore(indexMetadata, recoverySource, new HashSet<>()).build());
+        stateBuilder.routingTable(
+            RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+                .addAsNewRestore(indexMetadata, recoverySource, new HashSet<>())
+                .build()
+        );
         ClusterState clusterState = stateBuilder.build();
 
         int shardId = randomInt(indexMetadata.getNumberOfShards() - 1);
@@ -450,6 +461,7 @@ private void validateSizeOfSnapshotShard(
         ReactiveStorageDeciderService.AllocationState allocationState = new ReactiveStorageDeciderService.AllocationState(
             clusterState,
             null,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
             null,
             null,
             shardSizeInfo,
@@ -540,6 +552,7 @@ public void testUnmovableSize() {
         ReactiveStorageDeciderService.AllocationState allocationState = new ReactiveStorageDeciderService.AllocationState(
             clusterState,
             null,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
             thresholdSettings,
             info,
             null,
@@ -620,6 +633,7 @@ public boolean canRemainWithNoNodes(ClusterState clusterState, ShardRouting shar
         ReactiveStorageDeciderService.AllocationState allocationState = new ReactiveStorageDeciderService.AllocationState(
             clusterState,
             allocationDeciders,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
             new DiskThresholdSettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)),
             ClusterInfo.EMPTY,
             null,
@@ -723,6 +737,7 @@ private void verifyNeedsWarmTier(
         ReactiveStorageDeciderService.AllocationState allocationState = new ReactiveStorageDeciderService.AllocationState(
             clusterState,
             allocationDeciders,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY,
             new DiskThresholdSettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)),
             ClusterInfo.EMPTY,
             null,
diff --git a/x-pack/plugin/blob-cache/build.gradle b/x-pack/plugin/blob-cache/build.gradle
new file mode 100644
index 000000000000..9763d4af017d
--- /dev/null
+++ b/x-pack/plugin/blob-cache/build.gradle
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+apply plugin: 'elasticsearch.internal-es-plugin'
+apply plugin: 'elasticsearch.internal-test-artifact'
+
+esplugin {
+    name 'blob-cache'
+    description 'Provides a blob store cache'
+    classname 'org.elasticsearch.blobcache.BlobCachePlugin'
+}
+
+dependencies {
+    implementation project(path: 'preallocate')
+}
\ No newline at end of file
diff --git a/x-pack/plugin/searchable-snapshots/preallocate/build.gradle b/x-pack/plugin/blob-cache/preallocate/build.gradle
similarity index 100%
rename from x-pack/plugin/searchable-snapshots/preallocate/build.gradle
rename to x-pack/plugin/blob-cache/preallocate/build.gradle
diff --git a/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/LinuxPreallocator.java b/x-pack/plugin/blob-cache/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/LinuxPreallocator.java
similarity index 100%
rename from x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/LinuxPreallocator.java
rename to x-pack/plugin/blob-cache/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/LinuxPreallocator.java
diff --git a/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/MacOsPreallocator.java b/x-pack/plugin/blob-cache/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/MacOsPreallocator.java
similarity index 100%
rename from x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/MacOsPreallocator.java
rename to x-pack/plugin/blob-cache/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/MacOsPreallocator.java
diff --git a/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/NoNativePreallocator.java b/x-pack/plugin/blob-cache/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/NoNativePreallocator.java
similarity index 100%
rename from x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/NoNativePreallocator.java
rename to x-pack/plugin/blob-cache/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/NoNativePreallocator.java
diff --git a/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/Preallocate.java b/x-pack/plugin/blob-cache/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/Preallocate.java
similarity index 100%
rename from x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/Preallocate.java
rename to x-pack/plugin/blob-cache/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/Preallocate.java
diff --git a/x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/Preallocator.java b/x-pack/plugin/blob-cache/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/Preallocator.java
similarity index 100%
rename from x-pack/plugin/searchable-snapshots/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/Preallocator.java
rename to x-pack/plugin/blob-cache/preallocate/src/main/java/org/elasticsearch/xpack/searchablesnapshots/preallocate/Preallocator.java
diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCachePlugin.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCachePlugin.java
new file mode 100644
index 000000000000..e5466253333f
--- /dev/null
+++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCachePlugin.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.blobcache;
+
+import org.elasticsearch.blobcache.shared.SharedBlobCacheService;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.plugins.ExtensiblePlugin;
+import org.elasticsearch.plugins.Plugin;
+
+import java.util.List;
+
+public class BlobCachePlugin extends Plugin implements ExtensiblePlugin {
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        return List.of(
+            SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING,
+            SharedBlobCacheService.SHARED_CACHE_SIZE_MAX_HEADROOM_SETTING,
+            SharedBlobCacheService.SHARED_CACHE_REGION_SIZE_SETTING,
+            SharedBlobCacheService.SHARED_CACHE_RANGE_SIZE_SETTING,
+            SharedBlobCacheService.SHARED_CACHE_RECOVERY_RANGE_SIZE_SETTING,
+            SharedBlobCacheService.SHARED_CACHE_MAX_FREQ_SETTING,
+            SharedBlobCacheService.SHARED_CACHE_DECAY_INTERVAL_SETTING,
+            SharedBlobCacheService.SHARED_CACHE_MIN_TIME_DELTA_SETTING
+        );
+    }
+}
diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheUtils.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheUtils.java
new file mode 100644
index 000000000000..25e671a60634
--- /dev/null
+++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/BlobCacheUtils.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.blobcache;
+
+import org.apache.lucene.store.IndexInput;
+import org.elasticsearch.blobcache.common.ByteRange;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.core.Streams;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+
+import static org.elasticsearch.core.Strings.format;
+
+public class BlobCacheUtils {
+
+    /**
+     * We use {@code long} to represent offsets and lengths of files since they may be larger than 2GB, but {@code int} to represent
+     * offsets and lengths of arrays in memory which are limited to 2GB in size. We quite often need to convert from the file-based world
+     * of {@code long}s into the memory-based world of {@code int}s, knowing for certain that the result will not overflow. This method
+     * should be used to clarify that we're doing this.
+     */
+    public static int toIntBytes(long l) {
+        return ByteSizeUnit.BYTES.toIntBytes(l);
+    }
+
+    public static void throwEOF(long channelPos, long len, Object file) throws EOFException {
+        throw new EOFException(format("unexpected EOF reading [%d-%d] from %s", channelPos, channelPos + len, file));
+    }
+
+    public static void ensureSeek(long pos, IndexInput input) throws IOException {
+        final long length = input.length();
+        if (pos > length) {
+            throw new EOFException("Reading past end of file [position=" + pos + ", length=" + input.length() + "] for " + input);
+        } else if (pos < 0L) {
+            throw new IOException("Seeking to negative position [" + pos + "] for " + input);
+        }
+    }
+
+    public static ByteRange computeRange(long rangeSize, long position, long length) {
+        long start = (position / rangeSize) * rangeSize;
+        long end = Math.min(start + rangeSize, length);
+        return ByteRange.of(start, end);
+    }
+
+    public static void ensureSlice(String sliceName, long sliceOffset, long sliceLength, IndexInput input) {
+        if (sliceOffset < 0 || sliceLength < 0 || sliceOffset + sliceLength > input.length()) {
+            throw new IllegalArgumentException(
+                "slice() "
+                    + sliceName
+                    + " out of bounds: offset="
+                    + sliceOffset
+                    + ",length="
+                    + sliceLength
+                    + ",fileLength="
+                    + input.length()
+                    + ": "
+                    + input
+            );
+        }
+    }
+
+    /**
+     * Perform a single {@code read()} from {@code inputStream} into {@code copyBuffer}, handling an EOF by throwing an {@link EOFException}
+     * rather than returning {@code -1}. Returns the number of bytes read, which is always positive.
+     *
+     * Most of its arguments are there simply to make the message of the {@link EOFException} more informative.
+     */
+    public static int readSafe(InputStream inputStream, ByteBuffer copyBuffer, long rangeStart, long remaining, Object cacheFileReference)
+        throws IOException {
+        final int len = (remaining < copyBuffer.remaining()) ? toIntBytes(remaining) : copyBuffer.remaining();
+        final int bytesRead = Streams.read(inputStream, copyBuffer, len);
+        if (bytesRead <= 0) {
+            throwEOF(rangeStart, remaining, cacheFileReference);
+        }
+        return bytesRead;
+    }
+}
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/ByteRange.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/ByteRange.java
similarity index 97%
rename from x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/ByteRange.java
rename to x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/ByteRange.java
index 8ba31d1304cd..09c556e5ed3b 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/ByteRange.java
+++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/ByteRange.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.searchablesnapshots.cache.common;
+package org.elasticsearch.blobcache.common;
 
 import org.elasticsearch.core.Nullable;
 
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/ProgressListenableActionFuture.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/ProgressListenableActionFuture.java
similarity index 93%
rename from x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/ProgressListenableActionFuture.java
rename to x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/ProgressListenableActionFuture.java
index 9c2d489cd198..25cf338f3dd1 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/ProgressListenableActionFuture.java
+++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/ProgressListenableActionFuture.java
@@ -5,11 +5,11 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.searchablesnapshots.cache.common;
+package org.elasticsearch.blobcache.common;
 
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.AdapterActionFuture;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.core.Tuple;
 
 import java.util.ArrayList;
@@ -23,7 +23,7 @@
  * Listeners are executed within the thread that triggers the completion, the failure or the progress update and
  * the progress value passed to the listeners on execution is the last updated value.
  */
-class ProgressListenableActionFuture extends AdapterActionFuture<Long, Long> {
+class ProgressListenableActionFuture extends PlainActionFuture<Long> {
 
     protected final long start;
     protected final long end;
@@ -111,6 +111,10 @@ public void onProgress(final long progressValue) {
 
     @Override
     public void onResponse(Long result) {
+        if (result == null || result < start || end < result) {
+            assert false : start + " < " + result + " < " + end;
+            throw new IllegalArgumentException("Invalid completion value [start=" + start + ",end=" + end + ",response=" + result + ']');
+        }
         ensureNotCompleted();
         super.onResponse(result);
     }
@@ -180,15 +184,6 @@ private static void executeListener(final ActionListener<Long> listener, final S
         }
     }
 
-    @Override
-    protected Long convert(Long response) {
-        if (response == null || response < start || end < response) {
-            assert false : start + " < " + response + " < " + end;
-            throw new IllegalArgumentException("Invalid completion value [start=" + start + ",end=" + end + ",response=" + response + ']');
-        }
-        return response;
-    }
-
     @Override
     public String toString() {
         return "ProgressListenableActionFuture[start="
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/SparseFileTracker.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/SparseFileTracker.java
similarity index 93%
rename from x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/SparseFileTracker.java
rename to x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/SparseFileTracker.java
index 8e0bbbc56b2c..fb6803ac8472 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/SparseFileTracker.java
+++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/SparseFileTracker.java
@@ -4,11 +4,12 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.searchablesnapshots.cache.common;
+
+package org.elasticsearch.blobcache.common;
 
 import org.elasticsearch.Assertions;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.GroupedActionListener;
+import org.elasticsearch.action.support.RefCountingListener;
 import org.elasticsearch.core.Nullable;
 
 import java.util.ArrayList;
@@ -242,30 +243,7 @@ public List<Gap> waitForRange(final ByteRange range, final ByteRange subRange, f
                     .collect(Collectors.toList());
         }
 
-        // NB we work with ranges outside the mutex here, but only to interact with their completion listeners which are `final` so
-        // there is no risk of concurrent modification.
-
-        switch (requiredRanges.size()) {
-            case 0 ->
-                // no need to wait for the gaps to be filled, the listener can be executed immediately
-                wrappedListener.onResponse(null);
-            case 1 -> {
-                final Range requiredRange = requiredRanges.get(0);
-                requiredRange.completionListener.addListener(
-                    wrappedListener.map(progress -> null),
-                    Math.min(requiredRange.completionListener.end, subRange.end())
-                );
-            }
-            default -> {
-                final GroupedActionListener<Long> groupedActionListener = new GroupedActionListener<>(
-                    wrappedListener.map(progress -> null),
-                    requiredRanges.size()
-                );
-                requiredRanges.forEach(
-                    r -> r.completionListener.addListener(groupedActionListener, Math.min(r.completionListener.end, subRange.end()))
-                );
-            }
-        }
+        subscribeToCompletionListeners(requiredRanges, subRange.end(), wrappedListener);
 
         return Collections.unmodifiableList(gaps);
     }
@@ -331,31 +309,32 @@ public boolean waitForRangeIfPending(final ByteRange range, final ActionListener
             assert invariant();
         }
 
+        subscribeToCompletionListeners(pendingRanges, range.end(), wrappedListener);
+        return true;
+    }
+
+    private void subscribeToCompletionListeners(List<Range> requiredRanges, long rangeEnd, ActionListener<Void> listener) {
         // NB we work with ranges outside the mutex here, but only to interact with their completion listeners which are `final` so
         // there is no risk of concurrent modification.
-
-        switch (pendingRanges.size()) {
-            case 0 -> wrappedListener.onResponse(null);
+        switch (requiredRanges.size()) {
+            case 0 ->
+                // no need to wait for the gaps to be filled, the listener can be executed immediately
+                listener.onResponse(null);
             case 1 -> {
-                final Range pendingRange = pendingRanges.get(0);
-                pendingRange.completionListener.addListener(
-                    wrappedListener.map(progress -> null),
-                    Math.min(pendingRange.completionListener.end, range.end())
+                final Range requiredRange = requiredRanges.get(0);
+                requiredRange.completionListener.addListener(
+                    listener.map(progress -> null),
+                    Math.min(requiredRange.completionListener.end, rangeEnd)
                 );
-                return true;
             }
             default -> {
-                final GroupedActionListener<Long> groupedActionListener = new GroupedActionListener<>(
-                    wrappedListener.map(progress -> null),
-                    pendingRanges.size()
-                );
-                pendingRanges.forEach(
-                    r -> r.completionListener.addListener(groupedActionListener, Math.min(r.completionListener.end, range.end()))
-                );
-                return true;
+                try (var listeners = new RefCountingListener(listener)) {
+                    for (Range range : requiredRanges) {
+                        range.completionListener.addListener(listeners.acquire(l -> {}), Math.min(range.completionListener.end, rangeEnd));
+                    }
+                }
             }
         }
-        return true;
     }
 
     private ActionListener<Void> wrapWithAssertions(ActionListener<Void> listener) {
diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java
new file mode 100644
index 000000000000..2e84e6d85fdd
--- /dev/null
+++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java
@@ -0,0 +1,913 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.blobcache.shared;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.elasticsearch.Assertions;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.StepListener;
+import org.elasticsearch.blobcache.BlobCacheUtils;
+import org.elasticsearch.blobcache.common.ByteRange;
+import org.elasticsearch.blobcache.common.SparseFileTracker;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.cluster.routing.allocation.DataTier;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.RelativeByteSizeValue;
+import org.elasticsearch.common.util.concurrent.AbstractAsyncTask;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.KeyedLock;
+import org.elasticsearch.core.AbstractRefCounted;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Releasables;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.monitor.fs.FsProbe;
+import org.elasticsearch.node.NodeRoleSettings;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.LongAdder;
+import java.util.function.LongConsumer;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+public class SharedBlobCacheService<KeyType> implements Releasable {
+
+    private static final String SHARED_CACHE_SETTINGS_PREFIX = "xpack.searchable.snapshot.shared_cache.";
+
+    public static final Setting<ByteSizeValue> SHARED_CACHE_RANGE_SIZE_SETTING = new Setting<>(
+        SHARED_CACHE_SETTINGS_PREFIX + "range_size",
+        ByteSizeValue.ofMb(16).getStringRep(),
+        s -> ByteSizeValue.parseBytesSizeValue(s, SHARED_CACHE_SETTINGS_PREFIX + "range_size"),
+        getPositivePageSizeAlignedByteSizeValueValidator(SHARED_CACHE_SETTINGS_PREFIX + "range_size"),
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting<ByteSizeValue> SHARED_CACHE_RECOVERY_RANGE_SIZE_SETTING = new Setting<>(
+        SHARED_CACHE_SETTINGS_PREFIX + "recovery_range_size",
+        ByteSizeValue.ofKb(128L).getStringRep(),
+        s -> ByteSizeValue.parseBytesSizeValue(s, SHARED_CACHE_SETTINGS_PREFIX + "recovery_range_size"),
+        getPositivePageSizeAlignedByteSizeValueValidator(SHARED_CACHE_SETTINGS_PREFIX + "recovery_range_size"),
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting<ByteSizeValue> SHARED_CACHE_REGION_SIZE_SETTING = new Setting<>(
+        SHARED_CACHE_SETTINGS_PREFIX + "region_size",
+        SHARED_CACHE_RANGE_SIZE_SETTING,
+        s -> ByteSizeValue.parseBytesSizeValue(s, SHARED_CACHE_SETTINGS_PREFIX + "region_size"),
+        getPositivePageSizeAlignedByteSizeValueValidator(SHARED_CACHE_SETTINGS_PREFIX + "region_size"),
+        Setting.Property.NodeScope
+    );
+
+    private static Setting.Validator<ByteSizeValue> getPageSizeAlignedByteSizeValueValidator(String settingName) {
+        return value -> {
+            if (value.getBytes() == -1) {
+                throw new SettingsException("setting [{}] must be non-negative", settingName);
+            }
+            if (value.getBytes() % SharedBytes.PAGE_SIZE != 0L) {
+                throw new SettingsException("setting [{}] must be multiple of {}", settingName, SharedBytes.PAGE_SIZE);
+            }
+        };
+    }
+
+    private static Setting.Validator<ByteSizeValue> getPositivePageSizeAlignedByteSizeValueValidator(String settingName) {
+        return value -> {
+            if (value.getBytes() <= 0L) {
+                throw new SettingsException("setting [{}] must be greater than zero", settingName);
+            }
+            getPageSizeAlignedByteSizeValueValidator(settingName).validate(value);
+        };
+    }
+
+    public static final Setting<RelativeByteSizeValue> SHARED_CACHE_SIZE_SETTING = new Setting<>(
+        new Setting.SimpleKey(SHARED_CACHE_SETTINGS_PREFIX + "size"),
+        (settings) -> {
+            if (DiscoveryNode.isDedicatedFrozenNode(settings)) {
+                return "90%";
+            } else {
+                return ByteSizeValue.ZERO.getStringRep();
+            }
+        },
+        s -> RelativeByteSizeValue.parseRelativeByteSizeValue(s, SHARED_CACHE_SETTINGS_PREFIX + "size"),
+        new Setting.Validator<>() {
+
+            @Override
+            public void validate(final RelativeByteSizeValue value) {
+
+            }
+
+            @Override
+            public void validate(final RelativeByteSizeValue value, final Map<Setting<?>, Object> settings) {
+                if (value.isAbsolute() && value.getAbsolute().getBytes() == -1) {
+                    throw new SettingsException("setting [{}] must be non-negative", SHARED_CACHE_SETTINGS_PREFIX + "size");
+                }
+                if (value.isNonZeroSize()) {
+                    @SuppressWarnings("unchecked")
+                    final List<DiscoveryNodeRole> roles = (List<DiscoveryNodeRole>) settings.get(NodeRoleSettings.NODE_ROLES_SETTING);
+                    final var rolesSet = Set.copyOf(roles);
+                    if (DataTier.isFrozenNode(rolesSet) == false && rolesSet.contains(DiscoveryNodeRole.SEARCH_ROLE) == false) {
+                        throw new SettingsException(
+                            "setting [{}] to be positive [{}] is only permitted on nodes with the data_frozen role, roles are [{}]",
+                            SHARED_CACHE_SETTINGS_PREFIX + "size",
+                            value.getStringRep(),
+                            roles.stream().map(DiscoveryNodeRole::roleName).collect(Collectors.joining(","))
+                        );
+                    }
+
+                    @SuppressWarnings("unchecked")
+                    final List<String> dataPaths = (List<String>) settings.get(Environment.PATH_DATA_SETTING);
+                    if (dataPaths.size() > 1) {
+                        throw new SettingsException(
+                            "setting [{}={}] is not permitted on nodes with multiple data paths [{}]",
+                            SHARED_CACHE_SIZE_SETTING.getKey(),
+                            value.getStringRep(),
+                            String.join(",", dataPaths)
+                        );
+                    }
+                }
+            }
+
+            @Override
+            public Iterator<Setting<?>> settings() {
+                final List<Setting<?>> settings = List.of(NodeRoleSettings.NODE_ROLES_SETTING, Environment.PATH_DATA_SETTING);
+                return settings.iterator();
+            }
+
+        },
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting<ByteSizeValue> SHARED_CACHE_SIZE_MAX_HEADROOM_SETTING = new Setting<>(
+        new Setting.SimpleKey(SHARED_CACHE_SETTINGS_PREFIX + "size.max_headroom"),
+        (settings) -> {
+            if (SHARED_CACHE_SIZE_SETTING.exists(settings) == false && DiscoveryNode.isDedicatedFrozenNode(settings)) {
+                return "100GB";
+            }
+
+            return "-1";
+        },
+        (s) -> ByteSizeValue.parseBytesSizeValue(s, SHARED_CACHE_SETTINGS_PREFIX + "size.max_headroom"),
+        new Setting.Validator<>() {
+            private final Collection<Setting<?>> dependencies = List.of(SHARED_CACHE_SIZE_SETTING);
+
+            @Override
+            public Iterator<Setting<?>> settings() {
+                return dependencies.iterator();
+            }
+
+            @Override
+            public void validate(ByteSizeValue value) {
+                // ignore
+            }
+
+            @Override
+            public void validate(ByteSizeValue value, Map<Setting<?>, Object> settings, boolean isPresent) {
+                if (isPresent && value.getBytes() != -1) {
+                    RelativeByteSizeValue sizeValue = (RelativeByteSizeValue) settings.get(SHARED_CACHE_SIZE_SETTING);
+                    if (sizeValue.isAbsolute()) {
+                        throw new SettingsException(
+                            "setting [{}] cannot be specified for absolute [{}={}]",
+                            SHARED_CACHE_SIZE_MAX_HEADROOM_SETTING.getKey(),
+                            SHARED_CACHE_SIZE_SETTING.getKey(),
+                            sizeValue.getStringRep()
+                        );
+                    }
+                }
+            }
+        },
+        Setting.Property.NodeScope
+    );
+
+    public static final TimeValue MIN_SHARED_CACHE_DECAY_INTERVAL = TimeValue.timeValueSeconds(1L);
+    public static final Setting<TimeValue> SHARED_CACHE_DECAY_INTERVAL_SETTING = Setting.timeSetting(
+        SHARED_CACHE_SETTINGS_PREFIX + "decay.interval",
+        TimeValue.timeValueSeconds(60L),                        // default
+        MIN_SHARED_CACHE_DECAY_INTERVAL,                      // min
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
+
+    public static final Setting<Integer> SHARED_CACHE_MAX_FREQ_SETTING = Setting.intSetting(
+        SHARED_CACHE_SETTINGS_PREFIX + "max_freq",
+        100,                       // default
+        1,                            // min
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting<TimeValue> SHARED_CACHE_MIN_TIME_DELTA_SETTING = Setting.timeSetting(
+        SHARED_CACHE_SETTINGS_PREFIX + "min_time_delta",
+        TimeValue.timeValueSeconds(60L),                        // default
+        TimeValue.timeValueSeconds(0L),                         // min
+        Setting.Property.NodeScope
+    );
+
+    private static final Logger logger = LogManager.getLogger(SharedBlobCacheService.class);
+
+    private final ConcurrentHashMap<RegionKey<KeyType>, Entry<CacheFileRegion>> keyMapping;
+    private final ThreadPool threadPool;
+
+    private final KeyedLock<KeyType> keyedLock = new KeyedLock<>();
+
+    private final SharedBytes sharedBytes;
+    private final long cacheSize;
+    private final long regionSize;
+    private final ByteSizeValue rangeSize;
+    private final ByteSizeValue recoveryRangeSize;
+
+    private final int numRegions;
+    private final ConcurrentLinkedQueue<Integer> freeRegions = new ConcurrentLinkedQueue<>();
+    private final Entry<CacheFileRegion>[] freqs;
+    private final int maxFreq;
+    private final long minTimeDelta;
+
+    private final AtomicReference<CacheFileRegion>[] regionOwners; // to assert exclusive access of regions
+
+    private final CacheDecayTask decayTask;
+
+    private final LongAdder writeCount = new LongAdder();
+    private final LongAdder writeBytes = new LongAdder();
+
+    private final LongAdder readCount = new LongAdder();
+    private final LongAdder readBytes = new LongAdder();
+
+    private final LongAdder evictCount = new LongAdder();
+
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    public SharedBlobCacheService(NodeEnvironment environment, Settings settings, ThreadPool threadPool) {
+        this.threadPool = threadPool;
+        long totalFsSize;
+        try {
+            totalFsSize = FsProbe.getTotal(Environment.getFileStore(environment.nodeDataPaths()[0]));
+        } catch (IOException e) {
+            throw new IllegalStateException("unable to probe size of filesystem [" + environment.nodeDataPaths()[0] + "]");
+        }
+        this.cacheSize = calculateCacheSize(settings, totalFsSize);
+        final long regionSize = SHARED_CACHE_REGION_SIZE_SETTING.get(settings).getBytes();
+        this.numRegions = Math.toIntExact(cacheSize / regionSize);
+        keyMapping = new ConcurrentHashMap<>();
+        if (Assertions.ENABLED) {
+            regionOwners = new AtomicReference[numRegions];
+            for (int i = 0; i < numRegions; i++) {
+                regionOwners[i] = new AtomicReference<>();
+            }
+        } else {
+            regionOwners = null;
+        }
+        for (int i = 0; i < numRegions; i++) {
+            freeRegions.add(i);
+        }
+        this.regionSize = regionSize;
+        assert regionSize > 0L;
+        this.maxFreq = SHARED_CACHE_MAX_FREQ_SETTING.get(settings);
+        this.minTimeDelta = SHARED_CACHE_MIN_TIME_DELTA_SETTING.get(settings).millis();
+        freqs = new Entry[maxFreq];
+        try {
+            sharedBytes = new SharedBytes(numRegions, regionSize, environment, writeBytes::add, readBytes::add);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+        decayTask = new CacheDecayTask(threadPool, SHARED_CACHE_DECAY_INTERVAL_SETTING.get(settings));
+        decayTask.rescheduleIfNecessary();
+        this.rangeSize = SHARED_CACHE_RANGE_SIZE_SETTING.get(settings);
+        this.recoveryRangeSize = SHARED_CACHE_RECOVERY_RANGE_SIZE_SETTING.get(settings);
+    }
+
+    public static long calculateCacheSize(Settings settings, long totalFsSize) {
+        return SHARED_CACHE_SIZE_SETTING.get(settings)
+            .calculateValue(ByteSizeValue.ofBytes(totalFsSize), SHARED_CACHE_SIZE_MAX_HEADROOM_SETTING.get(settings))
+            .getBytes();
+    }
+
+    public int getRangeSize() {
+        return BlobCacheUtils.toIntBytes(rangeSize.getBytes());
+    }
+
+    public int getRecoveryRangeSize() {
+        return BlobCacheUtils.toIntBytes(recoveryRangeSize.getBytes());
+    }
+
+    private int getRegion(long position) {
+        return Math.toIntExact(position / regionSize);
+    }
+
+    private long getRegionRelativePosition(long position) {
+        return position % regionSize;
+    }
+
+    private long getRegionStart(int region) {
+        return region * regionSize;
+    }
+
+    private long getRegionEnd(int region) {
+        return (region + 1) * regionSize;
+    }
+
+    private int getEndingRegion(long position) {
+        assert position > 0L;
+        if (position % regionSize == 0L) {
+            return getRegion(position - 1);
+        }
+        return getRegion(position);
+    }
+
+    private ByteRange mapSubRangeToRegion(ByteRange range, int region) {
+        final long regionStart = getRegionStart(region);
+        final long regionEnd = getRegionEnd(region);
+        if (range.start() >= regionEnd || range.end() <= regionStart) {
+            return ByteRange.EMPTY;
+        }
+        final long rangeStart = Math.max(regionStart, range.start());
+        final long rangeEnd = Math.min(regionEnd, range.end());
+        if (rangeStart >= rangeEnd) {
+            return ByteRange.EMPTY;
+        }
+        return ByteRange.of(
+            getRegionRelativePosition(rangeStart),
+            rangeEnd == regionEnd ? regionSize : getRegionRelativePosition(rangeEnd)
+        );
+    }
+
+    private long getRegionSize(long fileLength, int region) {
+        assert fileLength > 0;
+        final int maxRegion = getEndingRegion(fileLength);
+        assert region >= 0 && region <= maxRegion : region + " - " + maxRegion;
+        final long effectiveRegionSize;
+        if (region == maxRegion && (region + 1) * regionSize != fileLength) {
+            assert getRegionRelativePosition(fileLength) != 0L;
+            effectiveRegionSize = getRegionRelativePosition(fileLength);
+        } else {
+            effectiveRegionSize = regionSize;
+        }
+        assert getRegionStart(region) + effectiveRegionSize <= fileLength;
+        return effectiveRegionSize;
+    }
+
+    public CacheFileRegion get(KeyType cacheKey, long fileLength, int region) {
+        final long effectiveRegionSize = getRegionSize(fileLength, region);
+        try (Releasable ignore = keyedLock.acquire(cacheKey)) {
+            final RegionKey<KeyType> regionKey = new RegionKey<>(cacheKey, region);
+            final long now = threadPool.relativeTimeInMillis();
+            final Entry<CacheFileRegion> entry = keyMapping.computeIfAbsent(
+                regionKey,
+                key -> new Entry<>(new CacheFileRegion(key, effectiveRegionSize), now)
+            );
+            if (entry.chunk.sharedBytesPos == -1) {
+                // new item
+                assert entry.freq == 0;
+                assert entry.prev == null;
+                assert entry.next == null;
+                final Integer freeSlot = freeRegions.poll();
+                if (freeSlot != null) {
+                    // no need to evict an item, just add
+                    entry.chunk.sharedBytesPos = freeSlot;
+                    assert regionOwners[freeSlot].compareAndSet(null, entry.chunk);
+                    synchronized (this) {
+                        pushEntryToBack(entry);
+                    }
+                } else {
+                    // need to evict something
+                    synchronized (this) {
+                        maybeEvict();
+                    }
+                    final Integer freeSlotRetry = freeRegions.poll();
+                    if (freeSlotRetry != null) {
+                        entry.chunk.sharedBytesPos = freeSlotRetry;
+                        assert regionOwners[freeSlotRetry].compareAndSet(null, entry.chunk);
+                        synchronized (this) {
+                            pushEntryToBack(entry);
+                        }
+                    } else {
+                        boolean removed = keyMapping.remove(regionKey, entry);
+                        assert removed;
+                        throw new AlreadyClosedException("no free region found");
+                    }
+                }
+            } else {
+                // check if we need to promote item
+                synchronized (this) {
+                    if (now - entry.lastAccessed >= minTimeDelta && entry.freq + 1 < maxFreq) {
+                        unlink(entry);
+                        entry.freq++;
+                        entry.lastAccessed = now;
+                        pushEntryToBack(entry);
+                    }
+                }
+            }
+            return entry.chunk;
+        }
+    }
+
+    public void onClose(CacheFileRegion chunk) {
+        assert regionOwners[chunk.sharedBytesPos].compareAndSet(chunk, null);
+        freeRegions.add(chunk.sharedBytesPos);
+    }
+
+    // used by tests
+    int freeRegionCount() {
+        return freeRegions.size();
+    }
+
+    public Stats getStats() {
+        return new Stats(
+            numRegions,
+            cacheSize,
+            regionSize,
+            evictCount.sum(),
+            writeCount.sum(),
+            writeBytes.sum(),
+            readCount.sum(),
+            readBytes.sum()
+        );
+    }
+
+    private synchronized boolean invariant(final Entry<CacheFileRegion> e, boolean present) {
+        boolean found = false;
+        for (int i = 0; i < maxFreq; i++) {
+            assert freqs[i] == null || freqs[i].prev != null;
+            assert freqs[i] == null || freqs[i].prev != freqs[i] || freqs[i].next == null;
+            assert freqs[i] == null || freqs[i].prev.next == null;
+            for (Entry<CacheFileRegion> entry = freqs[i]; entry != null; entry = entry.next) {
+                assert entry.next == null || entry.next.prev == entry;
+                assert entry.prev != null;
+                assert entry.prev.next == null || entry.prev.next == entry;
+                assert entry.freq == i;
+                if (entry == e) {
+                    found = true;
+                }
+            }
+            for (Entry<CacheFileRegion> entry = freqs[i]; entry != null && entry.prev != freqs[i]; entry = entry.prev) {
+                assert entry.next == null || entry.next.prev == entry;
+                assert entry.prev != null;
+                assert entry.prev.next == null || entry.prev.next == entry;
+                assert entry.freq == i;
+                if (entry == e) {
+                    found = true;
+                }
+            }
+        }
+        assert found == present;
+        return true;
+    }
+
+    private void maybeEvict() {
+        assert Thread.holdsLock(this);
+        for (int i = 0; i < maxFreq; i++) {
+            for (Entry<CacheFileRegion> entry = freqs[i]; entry != null; entry = entry.next) {
+                boolean evicted = entry.chunk.tryEvict();
+                if (evicted) {
+                    unlink(entry);
+                    keyMapping.remove(entry.chunk.regionKey, entry);
+                    return;
+                }
+            }
+        }
+    }
+
+    private void pushEntryToBack(final Entry<CacheFileRegion> entry) {
+        assert Thread.holdsLock(this);
+        assert invariant(entry, false);
+        assert entry.prev == null;
+        assert entry.next == null;
+        final Entry<CacheFileRegion> currFront = freqs[entry.freq];
+        if (currFront == null) {
+            freqs[entry.freq] = entry;
+            entry.prev = entry;
+            entry.next = null;
+        } else {
+            assert currFront.freq == entry.freq;
+            final Entry<CacheFileRegion> last = currFront.prev;
+            currFront.prev = entry;
+            last.next = entry;
+            entry.prev = last;
+            entry.next = null;
+        }
+        assert freqs[entry.freq].prev == entry;
+        assert freqs[entry.freq].prev.next == null;
+        assert entry.prev != null;
+        assert entry.prev.next == null || entry.prev.next == entry;
+        assert entry.next == null;
+        assert invariant(entry, true);
+    }
+
+    private void unlink(final Entry<CacheFileRegion> entry) {
+        assert Thread.holdsLock(this);
+        assert invariant(entry, true);
+        assert entry.prev != null;
+        final Entry<CacheFileRegion> currFront = freqs[entry.freq];
+        assert currFront != null;
+        if (currFront == entry) {
+            freqs[entry.freq] = entry.next;
+            if (entry.next != null) {
+                assert entry.prev != entry;
+                entry.next.prev = entry.prev;
+            }
+        } else {
+            if (entry.next != null) {
+                entry.next.prev = entry.prev;
+            }
+            entry.prev.next = entry.next;
+            if (currFront.prev == entry) {
+                currFront.prev = entry.prev;
+            }
+        }
+        entry.next = null;
+        entry.prev = null;
+        assert invariant(entry, false);
+    }
+
+    private void computeDecay() {
+        synchronized (this) {
+            long now = threadPool.relativeTimeInMillis();
+            for (int i = 0; i < maxFreq; i++) {
+                for (Entry<CacheFileRegion> entry = freqs[i]; entry != null; entry = entry.next) {
+                    if (now - entry.lastAccessed >= 2 * minTimeDelta) {
+                        if (entry.freq > 0) {
+                            unlink(entry);
+                            entry.freq--;
+                            pushEntryToBack(entry);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    public void removeFromCache(KeyType cacheKey) {
+        forceEvict(cacheKey::equals);
+    }
+
+    public void forceEvict(Predicate<KeyType> cacheKeyPredicate) {
+        final List<Entry<CacheFileRegion>> matchingEntries = new ArrayList<>();
+        keyMapping.forEach((key, value) -> {
+            if (cacheKeyPredicate.test(key.file)) {
+                matchingEntries.add(value);
+            }
+        });
+        if (matchingEntries.isEmpty() == false) {
+            synchronized (this) {
+                for (Entry<CacheFileRegion> entry : matchingEntries) {
+                    boolean evicted = entry.chunk.forceEvict();
+                    if (evicted) {
+                        unlink(entry);
+                        keyMapping.remove(entry.chunk.regionKey, entry);
+                    }
+                }
+            }
+        }
+    }
+
+    // used by tests
+    int getFreq(CacheFileRegion cacheFileRegion) {
+        return keyMapping.get(cacheFileRegion.regionKey).freq;
+    }
+
+    @Override
+    public void close() {
+        sharedBytes.decRef();
+        decayTask.close();
+    }
+
+    class CacheDecayTask extends AbstractAsyncTask {
+
+        CacheDecayTask(ThreadPool threadPool, TimeValue interval) {
+            super(logger, Objects.requireNonNull(threadPool), Objects.requireNonNull(interval), true);
+        }
+
+        @Override
+        protected boolean mustReschedule() {
+            return true;
+        }
+
+        @Override
+        public void runInternal() {
+            computeDecay();
+        }
+
+        @Override
+        protected String getThreadPool() {
+            return ThreadPool.Names.GENERIC;
+        }
+
+        @Override
+        public String toString() {
+            return "shared_cache_decay_task";
+        }
+    }
+
+    private record RegionKey<KeyType> (KeyType file, int region) {
+        @Override
+        public String toString() {
+            return "Chunk{" + "file=" + file + ", region=" + region + '}';
+        }
+    }
+
+    static class Entry<T> {
+        final T chunk;
+        Entry<T> prev;
+        Entry<T> next;
+        int freq;
+        long lastAccessed;
+
+        Entry(T chunk, long lastAccessed) {
+            this.chunk = chunk;
+            this.lastAccessed = lastAccessed;
+        }
+    }
+
+    class CacheFileRegion extends AbstractRefCounted {
+        final RegionKey<KeyType> regionKey;
+        final SparseFileTracker tracker;
+        volatile int sharedBytesPos = -1;
+
+        CacheFileRegion(RegionKey<KeyType> regionKey, long regionSize) {
+            this.regionKey = regionKey;
+            assert regionSize > 0L;
+            tracker = new SparseFileTracker("file", regionSize);
+        }
+
+        public long physicalStartOffset() {
+            return sharedBytes.getPhysicalOffset(sharedBytesPos);
+        }
+
+        public long physicalEndOffset() {
+            return sharedBytes.getPhysicalOffset(sharedBytesPos + 1);
+        }
+
+        // If true this file region has been evicted from the cache and should not be used any more
+        private final AtomicBoolean evicted = new AtomicBoolean(false);
+
+        // tries to evict this chunk if noone is holding onto its resources anymore
+        public boolean tryEvict() {
+            if (refCount() <= 1 && evicted.compareAndSet(false, true)) {
+                logger.trace("evicted {} with channel offset {}", regionKey, physicalStartOffset());
+                evictCount.increment();
+                decRef();
+                return true;
+            }
+            return false;
+        }
+
+        public boolean forceEvict() {
+            if (evicted.compareAndSet(false, true)) {
+                logger.trace("force evicted {} with channel offset {}", regionKey, physicalStartOffset());
+                evictCount.increment();
+                decRef();
+                return true;
+            }
+            return false;
+        }
+
+        public boolean isEvicted() {
+            return evicted.get();
+        }
+
+        @Override
+        protected void closeInternal() {
+            // now actually free the region associated with this chunk
+            onClose(this);
+            logger.trace("closed {} with channel offset {}", regionKey, physicalStartOffset());
+        }
+
+        private void ensureOpen() {
+            if (evicted.get()) {
+                throwAlreadyEvicted();
+            }
+        }
+
+        private static void throwAlreadyEvicted() {
+            throw new AlreadyClosedException("File chunk is evicted");
+        }
+
+        StepListener<Integer> populateAndRead(
+            final ByteRange rangeToWrite,
+            final ByteRange rangeToRead,
+            final RangeAvailableHandler reader,
+            final RangeMissingHandler writer,
+            final Executor executor
+        ) {
+            assert rangeToRead.length() > 0;
+            final StepListener<Integer> listener = new StepListener<>();
+            Releasable decrementRef = null;
+            try {
+                ensureOpen();
+                incRef();
+                decrementRef = Releasables.releaseOnce(this::decRef);
+                ensureOpen();
+                Releasable finalDecrementRef = decrementRef;
+                listener.whenComplete(integer -> finalDecrementRef.close(), throwable -> finalDecrementRef.close());
+                final SharedBytes.IO fileChannel = sharedBytes.getFileChannel(sharedBytesPos);
+                listener.whenComplete(integer -> fileChannel.decRef(), e -> fileChannel.decRef());
+                final ActionListener<Void> rangeListener = rangeListener(rangeToRead, reader, listener, fileChannel);
+                final List<SparseFileTracker.Gap> gaps = tracker.waitForRange(rangeToWrite, rangeToRead, rangeListener);
+
+                for (SparseFileTracker.Gap gap : gaps) {
+                    executor.execute(new AbstractRunnable() {
+
+                        @Override
+                        protected void doRun() throws Exception {
+                            if (CacheFileRegion.this.tryIncRef() == false) {
+                                throw new AlreadyClosedException("Cache file channel has been released and closed");
+                            }
+                            try {
+                                ensureOpen();
+                                final long start = gap.start();
+                                assert regionOwners[sharedBytesPos].get() == CacheFileRegion.this;
+                                writer.fillCacheRange(
+                                    fileChannel,
+                                    physicalStartOffset() + gap.start(),
+                                    gap.start(),
+                                    gap.end() - gap.start(),
+                                    progress -> gap.onProgress(start + progress)
+                                );
+                                writeCount.increment();
+                            } finally {
+                                decRef();
+                            }
+                            gap.onCompletion();
+                        }
+
+                        @Override
+                        public void onFailure(Exception e) {
+                            gap.onFailure(e);
+                        }
+                    });
+                }
+            } catch (Exception e) {
+                releaseAndFail(listener, decrementRef, e);
+            }
+            return listener;
+        }
+
+        private ActionListener<Void> rangeListener(
+            ByteRange rangeToRead,
+            RangeAvailableHandler reader,
+            ActionListener<Integer> listener,
+            SharedBytes.IO fileChannel
+        ) {
+            return ActionListener.wrap(success -> {
+                final long physicalStartOffset = physicalStartOffset();
+                assert regionOwners[sharedBytesPos].get() == CacheFileRegion.this;
+                final int read = reader.onRangeAvailable(
+                    fileChannel,
+                    physicalStartOffset + rangeToRead.start(),
+                    rangeToRead.start(),
+                    rangeToRead.length()
+                );
+                assert read == rangeToRead.length()
+                    : "partial read ["
+                        + read
+                        + "] does not match the range to read ["
+                        + rangeToRead.end()
+                        + '-'
+                        + rangeToRead.start()
+                        + ']';
+                readCount.increment();
+                listener.onResponse(read);
+            }, listener::onFailure);
+        }
+
+        private static void releaseAndFail(ActionListener<Integer> listener, Releasable decrementRef, Exception e) {
+            try {
+                Releasables.close(decrementRef);
+            } catch (Exception ex) {
+                e.addSuppressed(ex);
+            }
+            listener.onFailure(e);
+        }
+
+        @Override
+        protected void alreadyClosed() {
+            throwAlreadyEvicted();
+        }
+    }
+
+    public class CacheFile {
+
+        private final KeyType cacheKey;
+        private final long length;
+
+        private CacheFile(KeyType cacheKey, long length) {
+            this.cacheKey = cacheKey;
+            this.length = length;
+        }
+
+        public long getLength() {
+            return length;
+        }
+
+        public KeyType getCacheKey() {
+            return cacheKey;
+        }
+
+        public int populateAndRead(
+            final ByteRange rangeToWrite,
+            final ByteRange rangeToRead,
+            final RangeAvailableHandler reader,
+            final RangeMissingHandler writer,
+            final String executor
+        ) throws Exception {
+            StepListener<Integer> stepListener = null;
+            final long writeStart = rangeToWrite.start();
+            final long readStart = rangeToRead.start();
+            for (int region = getRegion(rangeToWrite.start()); region <= getEndingRegion(rangeToWrite.end()); region++) {
+                final ByteRange subRangeToWrite = mapSubRangeToRegion(rangeToWrite, region);
+                final ByteRange subRangeToRead = mapSubRangeToRegion(rangeToRead, region);
+                if (subRangeToRead.length() == 0L) {
+                    // nothing to read, skip
+                    if (stepListener == null) {
+                        stepListener = new StepListener<>();
+                        stepListener.onResponse(0);
+                    }
+                    continue;
+                }
+                final CacheFileRegion fileRegion = get(cacheKey, length, region);
+                final long regionStart = getRegionStart(region);
+                final long writeOffset = writeStart - regionStart;
+                final long readOffset = readStart - regionStart;
+                final StepListener<Integer> lis = fileRegion.populateAndRead(
+                    subRangeToWrite,
+                    subRangeToRead,
+                    (channel, channelPos, relativePos, len) -> {
+                        assert regionOwners[fileRegion.sharedBytesPos].get() == fileRegion;
+                        assert channelPos >= fileRegion.physicalStartOffset() && channelPos + len <= fileRegion.physicalEndOffset();
+                        return reader.onRangeAvailable(channel, channelPos, relativePos - readOffset, len);
+                    },
+                    (channel, channelPos, relativePos, len, progressUpdater) -> {
+                        assert regionOwners[fileRegion.sharedBytesPos].get() == fileRegion;
+                        assert channelPos >= fileRegion.physicalStartOffset() && channelPos + len <= fileRegion.physicalEndOffset();
+                        writer.fillCacheRange(channel, channelPos, relativePos - writeOffset, len, progressUpdater);
+                    },
+                    threadPool.executor(executor)
+                );
+                assert lis != null;
+                if (stepListener == null) {
+                    stepListener = lis;
+                } else {
+                    stepListener = stepListener.thenCombine(lis, Math::addExact);
+                }
+
+            }
+            return stepListener.asFuture().get();
+        }
+
+        @Override
+        public String toString() {
+            return "SharedCacheFile{" + "cacheKey=" + cacheKey + ", length=" + length + '}';
+        }
+    }
+
+    public CacheFile getCacheFile(KeyType cacheKey, long length) {
+        return new CacheFile(cacheKey, length);
+    }
+
+    @FunctionalInterface
+    public interface RangeAvailableHandler {
+        // caller that wants to read from x should instead do a positional read from x + relativePos
+        // caller should also only read up to length, further bytes will be offered by another call to this method
+        int onRangeAvailable(SharedBytes.IO channel, long channelPos, long relativePos, long length) throws IOException;
+    }
+
+    @FunctionalInterface
+    public interface RangeMissingHandler {
+        void fillCacheRange(SharedBytes.IO channel, long channelPos, long relativePos, long length, LongConsumer progressUpdater)
+            throws IOException;
+    }
+
+    public record Stats(
+        int numberOfRegions,
+        long size,
+        long regionSize,
+        long evictCount,
+        long writeCount,
+        long writeBytes,
+        long readCount,
+        long readBytes
+    ) {
+        public static final Stats EMPTY = new Stats(0, 0L, 0L, 0L, 0L, 0L, 0L, 0L);
+    }
+}
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/SharedBytes.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBytes.java
similarity index 99%
rename from x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/SharedBytes.java
rename to x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBytes.java
index 36df67ef0401..e6dcabc95543 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/SharedBytes.java
+++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBytes.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.searchablesnapshots.cache.shared;
+package org.elasticsearch.blobcache.shared;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/InMemoryNoOpCommitDirectory.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/store/InMemoryNoOpCommitDirectory.java
similarity index 97%
rename from x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/InMemoryNoOpCommitDirectory.java
rename to x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/store/InMemoryNoOpCommitDirectory.java
index c3c2ab9800c2..6b2c11d335ee 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/InMemoryNoOpCommitDirectory.java
+++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/store/InMemoryNoOpCommitDirectory.java
@@ -4,7 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.searchablesnapshots.store;
+package org.elasticsearch.blobcache.store;
 
 import org.apache.lucene.store.ByteBuffersDirectory;
 import org.apache.lucene.store.Directory;
@@ -33,7 +33,7 @@ public class InMemoryNoOpCommitDirectory extends FilterDirectory {
     private final Directory realDirectory;
     private final Set<String> deletedFiles = new CopyOnWriteArraySet<>();
 
-    InMemoryNoOpCommitDirectory(Directory realDirectory) {
+    public InMemoryNoOpCommitDirectory(Directory realDirectory) {
         super(new ByteBuffersDirectory(NoLockFactory.INSTANCE));
         this.realDirectory = realDirectory;
     }
diff --git a/x-pack/plugin/searchable-snapshots/src/main/plugin-metadata/plugin-security.codebases b/x-pack/plugin/blob-cache/src/main/plugin-metadata/plugin-security.codebases
similarity index 100%
rename from x-pack/plugin/searchable-snapshots/src/main/plugin-metadata/plugin-security.codebases
rename to x-pack/plugin/blob-cache/src/main/plugin-metadata/plugin-security.codebases
diff --git a/x-pack/plugin/searchable-snapshots/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/blob-cache/src/main/plugin-metadata/plugin-security.policy
similarity index 100%
rename from x-pack/plugin/searchable-snapshots/src/main/plugin-metadata/plugin-security.policy
rename to x-pack/plugin/blob-cache/src/main/plugin-metadata/plugin-security.policy
diff --git a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/BlobCacheTestUtils.java b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/BlobCacheTestUtils.java
new file mode 100644
index 000000000000..756cd9168fcb
--- /dev/null
+++ b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/BlobCacheTestUtils.java
@@ -0,0 +1,145 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.blobcache;
+
+import org.apache.lucene.tests.mockfile.FilterFileChannel;
+import org.apache.lucene.tests.mockfile.FilterFileSystemProvider;
+import org.apache.lucene.tests.mockfile.FilterPath;
+import org.elasticsearch.blobcache.common.ByteRange;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.PathUtilsForTesting;
+
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.file.FileSystem;
+import java.nio.file.OpenOption;
+import java.nio.file.Path;
+import java.nio.file.attribute.FileAttribute;
+import java.nio.file.spi.FileSystemProvider;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.elasticsearch.test.ESTestCase.randomLongBetween;
+
+public final class BlobCacheTestUtils {
+    private BlobCacheTestUtils() {}
+
+    public static long numberOfRanges(long fileSize, long rangeSize) {
+        return numberOfRanges(BlobCacheUtils.toIntBytes(fileSize), BlobCacheUtils.toIntBytes(rangeSize));
+    }
+
+    static long numberOfRanges(int fileSize, int rangeSize) {
+        long numberOfRanges = fileSize / rangeSize;
+        if (fileSize % rangeSize > 0) {
+            numberOfRanges++;
+        }
+        if (numberOfRanges == 0) {
+            numberOfRanges++;
+        }
+        return numberOfRanges;
+    }
+
+    /**
+     * Generates a sorted set of non-empty and non-contiguous random ranges that could fit into a file of a given maximum length.
+     */
+    public static SortedSet<ByteRange> randomRanges(long length) {
+        final SortedSet<ByteRange> randomRanges = new TreeSet<>();
+        for (long i = 0L; i < length;) {
+            long start = randomLongBetween(i, Math.max(0L, length - 1L));
+            long end = randomLongBetween(start + 1L, length); // +1 for non empty ranges
+            randomRanges.add(ByteRange.of(start, end));
+            i = end + 1L + randomLongBetween(0L, Math.max(0L, length - end)); // +1 for non contiguous ranges
+        }
+        return randomRanges;
+    }
+
+    public static SortedSet<ByteRange> mergeContiguousRanges(final SortedSet<ByteRange> ranges) {
+        return ranges.stream().collect(TreeSet::new, (gaps, gap) -> {
+            if (gaps.isEmpty()) {
+                gaps.add(gap);
+            } else {
+                final ByteRange previous = gaps.pollLast();
+                if (previous.end() == gap.start()) {
+                    gaps.add(ByteRange.of(previous.start(), gap.end()));
+                } else {
+                    gaps.add(previous);
+                    gaps.add(gap);
+                }
+            }
+        }, (gaps1, gaps2) -> {
+            if (gaps1.isEmpty() == false && gaps2.isEmpty() == false) {
+                final ByteRange last = gaps1.pollLast();
+                final ByteRange first = gaps2.pollFirst();
+                if (last.end() == first.start()) {
+                    gaps1.add(ByteRange.of(last.start(), first.end()));
+                } else {
+                    gaps1.add(last);
+                    gaps2.add(first);
+                }
+            }
+            gaps1.addAll(gaps2);
+        });
+    }
+
+    /**
+     * A {@link FileSystemProvider} that counts the number of times the method {@link FileChannel#force(boolean)} is executed on every
+     * files.
+     */
+    public static class FSyncTrackingFileSystemProvider extends FilterFileSystemProvider {
+
+        private final Map<Path, AtomicInteger> files = new ConcurrentHashMap<>();
+        private final AtomicBoolean failFSyncs = new AtomicBoolean();
+        private final FileSystem delegateInstance;
+        private final Path rootDir;
+
+        public FSyncTrackingFileSystemProvider(FileSystem delegate, Path rootDir) {
+            super("fsynccounting://", delegate);
+            this.rootDir = new FilterPath(rootDir, this.fileSystem);
+            this.delegateInstance = delegate;
+        }
+
+        public void failFSyncs(boolean shouldFail) {
+            failFSyncs.set(shouldFail);
+        }
+
+        public Path resolve(String other) {
+            return rootDir.resolve(other);
+        }
+
+        @Nullable
+        public Integer getNumberOfFSyncs(Path path) {
+            final AtomicInteger counter = files.get(path);
+            return counter != null ? counter.get() : null;
+        }
+
+        @Override
+        public FileChannel newFileChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
+            final AtomicInteger counter = files.computeIfAbsent(path, p -> new AtomicInteger(0));
+            return new FilterFileChannel(super.newFileChannel(path, options, attrs)) {
+
+                @Override
+                public void force(boolean metaData) throws IOException {
+                    if (failFSyncs.get()) {
+                        throw new IOException("simulated");
+                    }
+                    super.force(metaData);
+                    counter.incrementAndGet();
+                }
+            };
+        }
+
+        public void tearDown() {
+            PathUtilsForTesting.installMock(delegateInstance);
+        }
+    }
+}
diff --git a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/BlobCacheUtilsTests.java b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/BlobCacheUtilsTests.java
new file mode 100644
index 000000000000..7625a213b459
--- /dev/null
+++ b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/BlobCacheUtilsTests.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.blobcache;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.EOFException;
+import java.nio.ByteBuffer;
+
+public class BlobCacheUtilsTests extends ESTestCase {
+
+    public void testReadSafeThrows() {
+        final ByteBuffer buffer = ByteBuffer.allocate(randomIntBetween(1, 1025));
+        final int remaining = randomIntBetween(1, 1025);
+        expectThrows(EOFException.class, () -> BlobCacheUtils.readSafe(BytesArray.EMPTY.streamInput(), buffer, 0, remaining, null));
+    }
+}
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/ProgressListenableActionFutureTests.java b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/ProgressListenableActionFutureTests.java
similarity index 99%
rename from x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/ProgressListenableActionFutureTests.java
rename to x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/ProgressListenableActionFutureTests.java
index 82f287f236c1..a8963be9eaa9 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/ProgressListenableActionFutureTests.java
+++ b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/ProgressListenableActionFutureTests.java
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.searchablesnapshots.cache.common;
+package org.elasticsearch.blobcache.common;
 
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ElasticsearchException;
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/SparseFileTrackerTests.java b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/SparseFileTrackerTests.java
similarity index 98%
rename from x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/SparseFileTrackerTests.java
rename to x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/SparseFileTrackerTests.java
index e486f917a559..79842dc432a2 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/SparseFileTrackerTests.java
+++ b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/common/SparseFileTrackerTests.java
@@ -4,7 +4,8 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.searchablesnapshots.cache.common;
+
+package org.elasticsearch.blobcache.common;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
@@ -22,10 +23,10 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
 
+import static org.elasticsearch.blobcache.BlobCacheTestUtils.mergeContiguousRanges;
+import static org.elasticsearch.blobcache.BlobCacheTestUtils.randomRanges;
+import static org.elasticsearch.blobcache.BlobCacheUtils.toIntBytes;
 import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentSet;
-import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsUtils.toIntBytes;
-import static org.elasticsearch.xpack.searchablesnapshots.cache.common.TestUtils.mergeContiguousRanges;
-import static org.elasticsearch.xpack.searchablesnapshots.cache.common.TestUtils.randomRanges;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
diff --git a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java
new file mode 100644
index 000000000000..a368c53e256f
--- /dev/null
+++ b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java
@@ -0,0 +1,400 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.blobcache.shared;
+
+import org.elasticsearch.blobcache.common.ByteRange;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.RatioValue;
+import org.elasticsearch.common.unit.RelativeByteSizeValue;
+import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.node.NodeRoleSettings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.node.Node.NODE_NAME_SETTING;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class SharedBlobCacheServiceTests extends ESTestCase {
+
+    private static long size(long numPages) {
+        return numPages * SharedBytes.PAGE_SIZE;
+    }
+
+    public void testBasicEviction() throws IOException {
+        Settings settings = Settings.builder()
+            .put(NODE_NAME_SETTING.getKey(), "node")
+            .put(SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(500)).getStringRep())
+            .put(SharedBlobCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(100)).getStringRep())
+            .put("path.home", createTempDir())
+            .build();
+        final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue();
+        try (
+            NodeEnvironment environment = new NodeEnvironment(settings, TestEnvironment.newEnvironment(settings));
+            var cacheService = new SharedBlobCacheService<>(environment, settings, taskQueue.getThreadPool())
+        ) {
+            final var cacheKey = generateCacheKey();
+            assertEquals(5, cacheService.freeRegionCount());
+            final var region0 = cacheService.get(cacheKey, size(250), 0);
+            assertEquals(size(100), region0.tracker.getLength());
+            assertEquals(4, cacheService.freeRegionCount());
+            final var region1 = cacheService.get(cacheKey, size(250), 1);
+            assertEquals(size(100), region1.tracker.getLength());
+            assertEquals(3, cacheService.freeRegionCount());
+            final var region2 = cacheService.get(cacheKey, size(250), 2);
+            assertEquals(size(50), region2.tracker.getLength());
+            assertEquals(2, cacheService.freeRegionCount());
+
+            assertTrue(region1.tryEvict());
+            assertEquals(3, cacheService.freeRegionCount());
+            assertFalse(region1.tryEvict());
+            assertEquals(3, cacheService.freeRegionCount());
+            region0.populateAndRead(
+                ByteRange.of(0L, 1L),
+                ByteRange.of(0L, 1L),
+                (channel, channelPos, relativePos, length) -> 1,
+                (channel, channelPos, relativePos, length, progressUpdater) -> progressUpdater.accept(length),
+                taskQueue.getThreadPool().executor(ThreadPool.Names.GENERIC)
+            );
+            assertFalse(region0.tryEvict());
+            assertEquals(3, cacheService.freeRegionCount());
+            taskQueue.runAllRunnableTasks();
+            assertTrue(region0.tryEvict());
+            assertEquals(4, cacheService.freeRegionCount());
+            assertTrue(region2.tryEvict());
+            assertEquals(5, cacheService.freeRegionCount());
+        }
+    }
+
+    public void testAutoEviction() throws IOException {
+        Settings settings = Settings.builder()
+            .put(NODE_NAME_SETTING.getKey(), "node")
+            .put(SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(200)).getStringRep())
+            .put(SharedBlobCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(100)).getStringRep())
+            .put("path.home", createTempDir())
+            .build();
+        final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue();
+        try (
+            NodeEnvironment environment = new NodeEnvironment(settings, TestEnvironment.newEnvironment(settings));
+            var cacheService = new SharedBlobCacheService<>(environment, settings, taskQueue.getThreadPool())
+        ) {
+            final var cacheKey = generateCacheKey();
+            assertEquals(2, cacheService.freeRegionCount());
+            final var region0 = cacheService.get(cacheKey, size(250), 0);
+            assertEquals(size(100), region0.tracker.getLength());
+            assertEquals(1, cacheService.freeRegionCount());
+            final var region1 = cacheService.get(cacheKey, size(250), 1);
+            assertEquals(size(100), region1.tracker.getLength());
+            assertEquals(0, cacheService.freeRegionCount());
+            assertFalse(region0.isEvicted());
+            assertFalse(region1.isEvicted());
+
+            // acquire region 2, which should evict region 0 (oldest)
+            final var region2 = cacheService.get(cacheKey, size(250), 2);
+            assertEquals(size(50), region2.tracker.getLength());
+            assertEquals(0, cacheService.freeRegionCount());
+            assertTrue(region0.isEvicted());
+            assertFalse(region1.isEvicted());
+
+            // explicitly evict region 1
+            assertTrue(region1.tryEvict());
+            assertEquals(1, cacheService.freeRegionCount());
+        }
+    }
+
+    public void testForceEviction() throws IOException {
+        Settings settings = Settings.builder()
+            .put(NODE_NAME_SETTING.getKey(), "node")
+            .put(SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(500)).getStringRep())
+            .put(SharedBlobCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(100)).getStringRep())
+            .put("path.home", createTempDir())
+            .build();
+        final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue();
+        try (
+            NodeEnvironment environment = new NodeEnvironment(settings, TestEnvironment.newEnvironment(settings));
+            var cacheService = new SharedBlobCacheService<>(environment, settings, taskQueue.getThreadPool())
+        ) {
+            final var cacheKey1 = generateCacheKey();
+            final var cacheKey2 = generateCacheKey();
+            assertEquals(5, cacheService.freeRegionCount());
+            final var region0 = cacheService.get(cacheKey1, size(250), 0);
+            assertEquals(4, cacheService.freeRegionCount());
+            final var region1 = cacheService.get(cacheKey2, size(250), 1);
+            assertEquals(3, cacheService.freeRegionCount());
+            assertFalse(region0.isEvicted());
+            assertFalse(region1.isEvicted());
+            cacheService.removeFromCache(cacheKey1);
+            assertTrue(region0.isEvicted());
+            assertFalse(region1.isEvicted());
+            assertEquals(4, cacheService.freeRegionCount());
+        }
+    }
+
+    public void testDecay() throws IOException {
+        Settings settings = Settings.builder()
+            .put(NODE_NAME_SETTING.getKey(), "node")
+            .put(SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(500)).getStringRep())
+            .put(SharedBlobCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(100)).getStringRep())
+            .put("path.home", createTempDir())
+            .build();
+        final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue();
+        try (
+            NodeEnvironment environment = new NodeEnvironment(settings, TestEnvironment.newEnvironment(settings));
+            var cacheService = new SharedBlobCacheService<>(environment, settings, taskQueue.getThreadPool())
+        ) {
+            final var cacheKey1 = generateCacheKey();
+            final var cacheKey2 = generateCacheKey();
+            assertEquals(5, cacheService.freeRegionCount());
+            final var region0 = cacheService.get(cacheKey1, size(250), 0);
+            assertEquals(4, cacheService.freeRegionCount());
+            final var region1 = cacheService.get(cacheKey2, size(250), 1);
+            assertEquals(3, cacheService.freeRegionCount());
+
+            assertEquals(0, cacheService.getFreq(region0));
+            assertEquals(0, cacheService.getFreq(region1));
+
+            taskQueue.advanceTime();
+            taskQueue.runAllRunnableTasks();
+
+            final var region0Again = cacheService.get(cacheKey1, size(250), 0);
+            assertSame(region0Again, region0);
+            assertEquals(1, cacheService.getFreq(region0));
+            assertEquals(0, cacheService.getFreq(region1));
+
+            taskQueue.advanceTime();
+            taskQueue.runAllRunnableTasks();
+            cacheService.get(cacheKey1, size(250), 0);
+            assertEquals(2, cacheService.getFreq(region0));
+            cacheService.get(cacheKey1, size(250), 0);
+            assertEquals(2, cacheService.getFreq(region0));
+
+            // advance 2 ticks (decay only starts after 2 ticks)
+            taskQueue.advanceTime();
+            taskQueue.runAllRunnableTasks();
+            taskQueue.advanceTime();
+            taskQueue.runAllRunnableTasks();
+            assertEquals(1, cacheService.getFreq(region0));
+            assertEquals(0, cacheService.getFreq(region1));
+
+            // advance another tick
+            taskQueue.advanceTime();
+            taskQueue.runAllRunnableTasks();
+            assertEquals(0, cacheService.getFreq(region0));
+            assertEquals(0, cacheService.getFreq(region1));
+        }
+    }
+
+    public void testCacheSizeRejectedOnNonFrozenNodes() {
+        String cacheSize = randomBoolean()
+            ? ByteSizeValue.ofBytes(size(500)).getStringRep()
+            : (new RatioValue(between(1, 100))).formatNoTrailingZerosPercent();
+        final Settings settings = Settings.builder()
+            .put(SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), cacheSize)
+            .put(SharedBlobCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(100)).getStringRep())
+            .putList(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), DiscoveryNodeRole.DATA_HOT_NODE_ROLE.roleName())
+            .build();
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.get(settings)
+        );
+        assertThat(e.getCause(), notNullValue());
+        assertThat(e.getCause(), instanceOf(SettingsException.class));
+        assertThat(
+            e.getCause().getMessage(),
+            is(
+                "setting ["
+                    + SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey()
+                    + "] to be positive ["
+                    + cacheSize
+                    + "] is only permitted on nodes with the data_frozen role, roles are [data_hot]"
+            )
+        );
+    }
+
+    public void testMultipleDataPathsRejectedOnFrozenNodes() {
+        final Settings settings = Settings.builder()
+            .put(SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(500)).getStringRep())
+            .putList(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE.roleName())
+            .putList(Environment.PATH_DATA_SETTING.getKey(), List.of("a", "b"))
+            .build();
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.get(settings)
+        );
+        assertThat(e.getCause(), notNullValue());
+        assertThat(e.getCause(), instanceOf(SettingsException.class));
+        assertThat(
+            e.getCause().getMessage(),
+            is(
+                "setting ["
+                    + SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey()
+                    + "="
+                    + ByteSizeValue.ofBytes(size(500)).getStringRep()
+                    + "] is not permitted on nodes with multiple data paths [a,b]"
+            )
+        );
+    }
+
+    public void testDedicateFrozenCacheSizeDefaults() {
+        final Settings settings = Settings.builder()
+            .putList(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE.roleName())
+            .build();
+
+        RelativeByteSizeValue relativeCacheSize = SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.get(settings);
+        assertThat(relativeCacheSize.isAbsolute(), is(false));
+        assertThat(relativeCacheSize.isNonZeroSize(), is(true));
+        assertThat(relativeCacheSize.calculateValue(ByteSizeValue.ofBytes(10000), null), equalTo(ByteSizeValue.ofBytes(9000)));
+        assertThat(SharedBlobCacheService.SHARED_CACHE_SIZE_MAX_HEADROOM_SETTING.get(settings), equalTo(ByteSizeValue.ofGb(100)));
+    }
+
+    public void testNotDedicatedFrozenCacheSizeDefaults() {
+        final Settings settings = Settings.builder()
+            .putList(
+                NodeRoleSettings.NODE_ROLES_SETTING.getKey(),
+                Sets.union(
+                    Set.of(
+                        randomFrom(
+                            DiscoveryNodeRole.DATA_HOT_NODE_ROLE,
+                            DiscoveryNodeRole.DATA_COLD_NODE_ROLE,
+                            DiscoveryNodeRole.DATA_WARM_NODE_ROLE,
+                            DiscoveryNodeRole.DATA_CONTENT_NODE_ROLE
+                        )
+                    ),
+                    new HashSet<>(
+                        randomSubsetOf(
+                            between(0, 3),
+                            DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE,
+                            DiscoveryNodeRole.INGEST_ROLE,
+                            DiscoveryNodeRole.MASTER_ROLE
+                        )
+                    )
+                ).stream().map(DiscoveryNodeRole::roleName).collect(Collectors.toList())
+            )
+            .build();
+
+        RelativeByteSizeValue relativeCacheSize = SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.get(settings);
+        assertThat(relativeCacheSize.isNonZeroSize(), is(false));
+        assertThat(relativeCacheSize.isAbsolute(), is(true));
+        assertThat(relativeCacheSize.getAbsolute(), equalTo(ByteSizeValue.ZERO));
+        assertThat(SharedBlobCacheService.SHARED_CACHE_SIZE_MAX_HEADROOM_SETTING.get(settings), equalTo(ByteSizeValue.ofBytes(-1)));
+    }
+
+    public void testMaxHeadroomRejectedForAbsoluteCacheSize() {
+        String cacheSize = ByteSizeValue.ofBytes(size(500)).getStringRep();
+        final Settings settings = Settings.builder()
+            .put(SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), cacheSize)
+            .put(SharedBlobCacheService.SHARED_CACHE_SIZE_MAX_HEADROOM_SETTING.getKey(), ByteSizeValue.ofBytes(size(100)).getStringRep())
+            .putList(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE.roleName())
+            .build();
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> SharedBlobCacheService.SHARED_CACHE_SIZE_MAX_HEADROOM_SETTING.get(settings)
+        );
+        assertThat(e.getCause(), notNullValue());
+        assertThat(e.getCause(), instanceOf(SettingsException.class));
+        assertThat(
+            e.getCause().getMessage(),
+            is(
+                "setting ["
+                    + SharedBlobCacheService.SHARED_CACHE_SIZE_MAX_HEADROOM_SETTING.getKey()
+                    + "] cannot be specified for absolute ["
+                    + SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey()
+                    + "="
+                    + cacheSize
+                    + "]"
+            )
+        );
+    }
+
+    public void testCalculateCacheSize() {
+        long smallSize = 10000;
+        long largeSize = ByteSizeValue.ofTb(10).getBytes();
+        assertThat(SharedBlobCacheService.calculateCacheSize(Settings.EMPTY, smallSize), equalTo(0L));
+        final Settings settings = Settings.builder()
+            .putList(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE.roleName())
+            .build();
+        assertThat(SharedBlobCacheService.calculateCacheSize(settings, smallSize), equalTo(9000L));
+        assertThat(SharedBlobCacheService.calculateCacheSize(settings, largeSize), equalTo(largeSize - ByteSizeValue.ofGb(100).getBytes()));
+    }
+
+    private static Object generateCacheKey() {
+        return new Object();
+    }
+
+    public void testCacheSizeChanges() throws IOException {
+        ByteSizeValue val1 = new ByteSizeValue(randomIntBetween(1, 5), ByteSizeUnit.MB);
+        Settings settings = Settings.builder()
+            .put(NODE_NAME_SETTING.getKey(), "node")
+            .put(SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), val1.getStringRep())
+            .put(SharedBlobCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(100)).getStringRep())
+            .put("path.home", createTempDir())
+            .build();
+        final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue();
+        try (
+            NodeEnvironment environment = new NodeEnvironment(settings, TestEnvironment.newEnvironment(settings));
+            SharedBlobCacheService<?> cacheService = new SharedBlobCacheService<>(environment, settings, taskQueue.getThreadPool())
+        ) {
+            assertEquals(val1.getBytes(), cacheService.getStats().size());
+        }
+
+        ByteSizeValue val2 = new ByteSizeValue(randomIntBetween(1, 5), ByteSizeUnit.MB);
+        settings = Settings.builder()
+            .put(settings)
+            .put(SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), val2.getStringRep())
+            .build();
+        try (
+            NodeEnvironment environment = new NodeEnvironment(settings, TestEnvironment.newEnvironment(settings));
+            SharedBlobCacheService<?> cacheService = new SharedBlobCacheService<>(environment, settings, taskQueue.getThreadPool())
+        ) {
+            assertEquals(val2.getBytes(), cacheService.getStats().size());
+        }
+    }
+
+    private void assertThatNonPositiveRecoveryRangeSizeRejected(Setting<ByteSizeValue> setting) {
+        final String value = randomFrom(ByteSizeValue.MINUS_ONE, ByteSizeValue.ZERO).getStringRep();
+        final Settings settings = Settings.builder()
+            .put(SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(100)).getStringRep())
+            .putList(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE.roleName())
+            .put(setting.getKey(), value)
+            .build();
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> setting.get(settings));
+        assertThat(e.getCause(), notNullValue());
+        assertThat(e.getCause(), instanceOf(SettingsException.class));
+        assertThat(e.getCause().getMessage(), is("setting [" + setting.getKey() + "] must be greater than zero"));
+    }
+
+    public void testNonPositiveRegionSizeRejected() {
+        assertThatNonPositiveRecoveryRangeSizeRejected(SharedBlobCacheService.SHARED_CACHE_REGION_SIZE_SETTING);
+    }
+
+    public void testNonPositiveRangeSizeRejected() {
+        assertThatNonPositiveRecoveryRangeSizeRejected(SharedBlobCacheService.SHARED_CACHE_RANGE_SIZE_SETTING);
+    }
+
+    public void testNonPositiveRecoveryRangeSizeRejected() {
+        assertThatNonPositiveRecoveryRangeSizeRejected(SharedBlobCacheService.SHARED_CACHE_RECOVERY_RANGE_SIZE_SETTING);
+    }
+
+}
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/InMemoryNoOpCommitDirectoryTests.java b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/store/InMemoryNoOpCommitDirectoryTests.java
similarity index 99%
rename from x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/InMemoryNoOpCommitDirectoryTests.java
rename to x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/store/InMemoryNoOpCommitDirectoryTests.java
index 643820bb4827..cee344ddf08d 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/InMemoryNoOpCommitDirectoryTests.java
+++ b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/store/InMemoryNoOpCommitDirectoryTests.java
@@ -4,7 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-package org.elasticsearch.xpack.searchablesnapshots.store;
+package org.elasticsearch.blobcache.store;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle
index c293a46885e6..f86c68e253de 100644
--- a/x-pack/plugin/build.gradle
+++ b/x-pack/plugin/build.gradle
@@ -3,11 +3,10 @@ import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.util.GradleUtils
 
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 apply plugin: 'elasticsearch.validate-rest-spec'
 apply plugin: 'elasticsearch.internal-test-artifact'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
 
 archivesBaseName = 'x-pack'
 
diff --git a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
index 11695fe5139b..6f66e7e38606 100644
--- a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
+++ b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
@@ -912,7 +912,7 @@ public void testDataStreamsBiDirectionalReplication() throws Exception {
                 Request createDataStreamRequest = new Request("PUT", "/_data_stream/" + leaderDataStreamName);
                 assertOK(leaderClient.performRequest(createDataStreamRequest));
                 Request updateAliasesRequest = new Request("POST", "/_aliases");
-                updateAliasesRequest.setJsonEntity(formatted("""
+                updateAliasesRequest.setJsonEntity(format("""
                     {
                       "actions": [ { "add": { "index": "%s", "alias": "logs-http", "is_write_index": true } } ]
                     }""", leaderDataStreamName));
@@ -937,7 +937,7 @@ public void testDataStreamsBiDirectionalReplication() throws Exception {
             Request createDataStreamRequest = new Request("PUT", "/_data_stream/" + followerDataStreamName);
             assertOK(client().performRequest(createDataStreamRequest));
             Request updateAliasesRequest = new Request("POST", "/_aliases");
-            updateAliasesRequest.setJsonEntity(formatted("""
+            updateAliasesRequest.setJsonEntity(format("""
                 {
                   "actions": [ { "add": { "index": "%s", "alias": "logs-http", "is_write_index": true } } ]
                 }""", followerDataStreamName));
@@ -956,7 +956,7 @@ public void testDataStreamsBiDirectionalReplication() throws Exception {
             // (only set the write flag to logs-http-na)
             // Create alias in follower cluster that point to leader and follower data streams:
             updateAliasesRequest = new Request("POST", "/_aliases");
-            updateAliasesRequest.setJsonEntity(formatted("""
+            updateAliasesRequest.setJsonEntity(format("""
                 {
                   "actions": [ { "add": { "index": "%s", "alias": "logs-http" } } ]
                 }""", leaderDataStreamName));
@@ -973,7 +973,7 @@ public void testDataStreamsBiDirectionalReplication() throws Exception {
                     verifyDocuments(leaderClient, followerDataStreamName, numDocs);
                 });
                 updateAliasesRequest = new Request("POST", "/_aliases");
-                updateAliasesRequest.setJsonEntity(formatted("""
+                updateAliasesRequest.setJsonEntity(format("""
                     {
                       "actions": [ { "add": { "index": "%s", "alias": "logs-http" } } ]
                     }""", followerDataStreamName));
diff --git a/x-pack/plugin/ccr/qa/rest/build.gradle b/x-pack/plugin/ccr/qa/rest/build.gradle
index 98814f04d6c5..5f173b8831df 100644
--- a/x-pack/plugin/ccr/qa/rest/build.gradle
+++ b/x-pack/plugin/ccr/qa/rest/build.gradle
@@ -1,5 +1,5 @@
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 restResources {
   restApi {
diff --git a/x-pack/plugin/ccr/qa/security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java b/x-pack/plugin/ccr/qa/security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java
index 02e702c9adf2..8803a0159cc8 100644
--- a/x-pack/plugin/ccr/qa/security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java
+++ b/x-pack/plugin/ccr/qa/security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.CheckedRunnable;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.health.node.selection.HealthNode;
 import org.elasticsearch.index.seqno.ReplicationTracker;
 import org.elasticsearch.test.rest.ObjectPath;
@@ -218,7 +219,7 @@ public void testForgetFollower() throws IOException {
 
             try (RestClient leaderClient = buildLeaderClient(restAdminSettings())) {
                 final Request request = new Request("POST", "/" + forgetLeader + "/_ccr/forget_follower");
-                final String requestBody = formatted("""
+                final String requestBody = Strings.format("""
                     {
                       "follower_cluster": "follow-cluster",
                       "follower_index": "%s",
@@ -290,7 +291,7 @@ public void testUnPromoteAndFollowDataStream() throws Exception {
                 for (var i = 0; i < numDocs; i++) {
                     var indexRequest = new Request("POST", "/" + dataStreamName + "/_doc");
                     indexRequest.addParameter("refresh", "true");
-                    indexRequest.setJsonEntity(formatted("""
+                    indexRequest.setJsonEntity(Strings.format("""
                         {"@timestamp": "%s","message":"abc"}
                         """, dateFormat.format(new Date())));
                     assertOK(leaderClient.performRequest(indexRequest));
diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java
index fd752bc533cb..7b7dfbae58ab 100644
--- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java
+++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
@@ -217,7 +218,7 @@ public void testDocsAreRecovered() throws Exception {
         final int firstBatchNumDocs = randomIntBetween(1, 64);
         logger.info("Indexing [{}] docs as first batch", firstBatchNumDocs);
         for (int i = 0; i < firstBatchNumDocs; i++) {
-            final String source = formatted("{\"f\":%d}", i);
+            final String source = Strings.format("{\"f\":%d}", i);
             leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource(source, XContentType.JSON).get();
         }
 
@@ -286,7 +287,7 @@ public void testRateLimitingIsEmployed() throws Exception {
 
         logger.info("--> indexing some data");
         for (int i = 0; i < 100; i++) {
-            final String source = formatted("{\"f\":%d}", i);
+            final String source = Strings.format("{\"f\":%d}", i);
             leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource(source, XContentType.JSON).get();
         }
 
@@ -353,7 +354,7 @@ public void testIndividualActionsTimeout() throws Exception {
 
         logger.info("--> indexing some data");
         for (int i = 0; i < 100; i++) {
-            final String source = formatted("{\"f\":%d}", i);
+            final String source = Strings.format("{\"f\":%d}", i);
             leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource(source, XContentType.JSON).get();
         }
 
diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java
index fcdd173f7321..c9c03ee527c4 100644
--- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java
+++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java
@@ -139,7 +139,7 @@ private RestoreSnapshotRequest setUpRestoreSnapshotRequest(
 
         logger.info("indexing [{}] docs", numberOfDocuments);
         for (int i = 0; i < numberOfDocuments; i++) {
-            final String source = formatted("{\"f\":%d}", i);
+            final String source = Strings.format("{\"f\":%d}", i);
             leaderClient().prepareIndex(leaderIndex).setId(Integer.toString(i)).setSource(source, XContentType.JSON).get();
             if (rarely()) {
                 leaderClient().admin().indices().prepareFlush(leaderIndex).setForce(true).setWaitIfOngoing(true).get();
@@ -635,7 +635,7 @@ public void testRetentionLeaseAdvancesWhileFollowing() throws Exception {
         final int numberOfDocuments = randomIntBetween(128, 2048);
         logger.debug("indexing [{}] docs", numberOfDocuments);
         for (int i = 0; i < numberOfDocuments; i++) {
-            final String source = formatted("{\"f\":%d}", i);
+            final String source = Strings.format("{\"f\":%d}", i);
             leaderClient().prepareIndex(leaderIndex).setId(Integer.toString(i)).setSource(source, XContentType.JSON).get();
             if (rarely()) {
                 leaderClient().admin().indices().prepareFlush(leaderIndex).setForce(true).setWaitIfOngoing(true).get();
diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java
index 459b83ad699a..ad66e34cd371 100644
--- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java
+++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java
@@ -140,7 +140,8 @@ public void testFollowIndexAndCloseNode() throws Exception {
                 } catch (InterruptedException e) {
                     throw new AssertionError(e);
                 }
-                final String source = formatted("{\"f\":%d}", counter++);
+                Object[] args = new Object[] { counter++ };
+                final String source = Strings.format("{\"f\":%d}", args);
                 IndexResponse indexResp = leaderClient().prepareIndex("index1")
                     .setSource(source, XContentType.JSON)
                     .setTimeout(TimeValue.timeValueSeconds(1))
diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java
index 8dfea9367ebd..df4dd9dfd3eb 100644
--- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java
+++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java
@@ -65,6 +65,7 @@
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.CheckedRunnable;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
@@ -319,7 +320,7 @@ public void testFollowIndexWithoutWaitForComplete() throws Exception {
         final int firstBatchNumDocs = randomIntBetween(2, 64);
         logger.info("Indexing [{}] docs as first batch", firstBatchNumDocs);
         for (int i = 0; i < firstBatchNumDocs; i++) {
-            final String source = formatted("{\"f\":%d}", i);
+            final String source = Strings.format("{\"f\":%d}", i);
             leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource(source, XContentType.JSON).get();
         }
 
@@ -366,7 +367,7 @@ public void testSyncMappings() throws Exception {
 
         final long firstBatchNumDocs = randomIntBetween(2, 64);
         for (long i = 0; i < firstBatchNumDocs; i++) {
-            final String source = formatted("{\"f\":%d}", i);
+            final String source = Strings.format("{\"f\":%d}", i);
             leaderClient().prepareIndex("index1").setId(Long.toString(i)).setSource(source, XContentType.JSON).get();
         }
 
@@ -379,7 +380,7 @@ public void testSyncMappings() throws Exception {
 
         final int secondBatchNumDocs = randomIntBetween(2, 64);
         for (long i = firstBatchNumDocs; i < firstBatchNumDocs + secondBatchNumDocs; i++) {
-            final String source = formatted("{\"k\":%d}", i);
+            final String source = Strings.format("{\"k\":%d}", i);
             leaderClient().prepareIndex("index1").setId(Long.toString(i)).setSource(source, XContentType.JSON).get();
         }
 
@@ -522,7 +523,8 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure)
                 } catch (InterruptedException e) {
                     throw new AssertionError(e);
                 }
-                final String source = formatted("{\"f\":%d}", counter++);
+                Object[] args = new Object[] { counter++ };
+                final String source = Strings.format("{\"f\":%d}", args);
                 IndexRequest indexRequest = new IndexRequest("index1").source(source, XContentType.JSON)
                     .timeout(TimeValue.timeValueSeconds(1));
                 bulkProcessor.add(indexRequest);
@@ -638,7 +640,7 @@ public void testFollowIndexMaxOperationSizeInBytes() throws Exception {
         final int numDocs = between(10, 1024);
         logger.info("Indexing [{}] docs", numDocs);
         for (int i = 0; i < numDocs; i++) {
-            final String source = formatted("{\"f\":%d}", i);
+            final String source = Strings.format("{\"f\":%d}", i);
             leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource(source, XContentType.JSON).get();
         }
 
@@ -1155,7 +1157,7 @@ public void testUpdateAnalysisLeaderIndexSettings() throws Exception {
 
         final int secondBatchNumDocs = randomIntBetween(2, 64);
         for (long i = firstBatchNumDocs; i < firstBatchNumDocs + secondBatchNumDocs; i++) {
-            final String source = formatted("{\"new_field\":\"value %d\"}", i);
+            final String source = Strings.format("{\"new_field\":\"value %d\"}", i);
             leaderClient().prepareIndex("leader").setSource(source, XContentType.JSON).get();
         }
 
@@ -1403,7 +1405,7 @@ private void runFallBehindTest(
         final int numDocs = randomIntBetween(2, 64);
         logger.info("Indexing [{}] docs as first batch", numDocs);
         for (int i = 0; i < numDocs; i++) {
-            final String source = formatted("{\"f\":%d}", i);
+            final String source = Strings.format("{\"f\":%d}", i);
             leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource(source, XContentType.JSON).get();
         }
 
@@ -1423,7 +1425,7 @@ private void runFallBehindTest(
         afterPausingFollower.run();
 
         for (int i = 0; i < numDocs; i++) {
-            final String source = formatted("{\"f\":%d}", i * 2);
+            final String source = Strings.format("{\"f\":%d}", i * 2);
             leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource(source, XContentType.JSON).get();
         }
         leaderClient().prepareDelete("index1", "1").get();
diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/RestartIndexFollowingIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/RestartIndexFollowingIT.java
index 02f466e1e837..69cb86cccf08 100644
--- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/RestartIndexFollowingIT.java
+++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/RestartIndexFollowingIT.java
@@ -12,19 +12,26 @@
 import org.elasticsearch.action.admin.cluster.remote.RemoteInfoAction;
 import org.elasticsearch.action.admin.cluster.remote.RemoteInfoRequest;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.transport.RemoteConnectionInfo;
 import org.elasticsearch.transport.RemoteConnectionStrategy;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.CcrIntegTestCase;
+import org.elasticsearch.xpack.core.ccr.action.FollowStatsAction;
 import org.elasticsearch.xpack.core.ccr.action.PauseFollowAction;
 import org.elasticsearch.xpack.core.ccr.action.PutFollowAction;
+import org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction;
 import org.elasticsearch.xpack.core.ccr.action.UnfollowAction;
 
 import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
@@ -67,7 +74,7 @@ public void testFollowIndex() throws Exception {
         final long firstBatchNumDocs = randomIntBetween(10, 200);
         logger.info("Indexing [{}] docs as first batch", firstBatchNumDocs);
         for (int i = 0; i < firstBatchNumDocs; i++) {
-            final String source = formatted("{\"f\":%d}", i);
+            final String source = Strings.format("{\"f\":%d}", i);
             leaderClient().prepareIndex("index1").setId(Integer.toString(i)).setSource(source, XContentType.JSON).get();
         }
 
@@ -79,6 +86,7 @@ public void testFollowIndex() throws Exception {
         ensureFollowerGreen("index2");
 
         final long secondBatchNumDocs = randomIntBetween(10, 200);
+        logger.info("Indexing [{}] docs as second batch", secondBatchNumDocs);
         for (int i = 0; i < secondBatchNumDocs; i++) {
             leaderClient().prepareIndex("index1").setSource("{}", XContentType.JSON).get();
         }
@@ -87,16 +95,28 @@ public void testFollowIndex() throws Exception {
         ensureLeaderGreen("index1");
 
         final long thirdBatchNumDocs = randomIntBetween(10, 200);
+        logger.info("Indexing [{}] docs as third batch", thirdBatchNumDocs);
         for (int i = 0; i < thirdBatchNumDocs; i++) {
             leaderClient().prepareIndex("index1").setSource("{}", XContentType.JSON).get();
         }
 
-        assertBusy(
-            () -> assertThat(
-                followerClient().prepareSearch("index2").get().getHits().getTotalHits().value,
-                equalTo(firstBatchNumDocs + secondBatchNumDocs + thirdBatchNumDocs)
-            )
-        );
+        var totalDocs = firstBatchNumDocs + secondBatchNumDocs + thirdBatchNumDocs;
+        final AtomicBoolean resumeAfterDisconnectionOnce = new AtomicBoolean(false);
+        assertBusy(() -> {
+            if (resumeAfterDisconnectionOnce.get() == false && isFollowerStoppedBecauseOfRemoteClusterDisconnection("index2")) {
+                assertTrue(resumeAfterDisconnectionOnce.compareAndSet(false, true));
+                if (randomBoolean()) {
+                    logger.info("shard follow task has been stopped because of remote cluster disconnection, resuming");
+                    pauseFollow("index2");
+                    assertAcked(followerClient().execute(ResumeFollowAction.INSTANCE, resumeFollow("index2")).actionGet());
+                } else {
+                    logger.info("shard follow task has been stopped because of remote cluster disconnection, recreating");
+                    assertAcked(followerClient().admin().indices().prepareDelete("index2"));
+                    followerClient().execute(PutFollowAction.INSTANCE, putFollow("index1", "index2", ActiveShardCount.ALL)).actionGet();
+                }
+            }
+            assertThat(followerClient().prepareSearch("index2").get().getHits().getTotalHits().value, equalTo(totalDocs));
+        }, 30L, TimeUnit.SECONDS);
 
         cleanRemoteCluster();
         assertAcked(followerClient().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("index2")).actionGet());
@@ -151,4 +171,16 @@ private void cleanRemoteCluster() throws Exception {
             assertThat(infos.size(), equalTo(0));
         });
     }
+
+    private boolean isFollowerStoppedBecauseOfRemoteClusterDisconnection(String indexName) {
+        var request = new FollowStatsAction.StatsRequest();
+        request.setIndices(new String[] { indexName });
+        var response = followerClient().execute(FollowStatsAction.INSTANCE, request).actionGet();
+        return response.getStatsResponses().stream().map(r -> r.status().getFatalException()).filter(Objects::nonNull).anyMatch(e -> {
+            if (e.getCause()instanceof IllegalStateException ise) {
+                return ise.getMessage().contains("Unable to open any connections to remote cluster");
+            }
+            return false;
+        });
+    }
 }
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CCRInfoTransportAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CCRInfoTransportAction.java
index 82ef95374853..68974c5ed3ec 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CCRInfoTransportAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CCRInfoTransportAction.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.ccr;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -88,8 +88,8 @@ public Usage(StreamInput in) throws IOException {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.V_7_0_0;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.V_7_0_0;
         }
 
         public int getNumberOfFollowerIndices() {
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java
index c066bb056d3d..cbc40e0e21ea 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java
@@ -16,8 +16,8 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -185,7 +185,7 @@ public Collection<Object> createComponents(
         final IndexNameExpressionResolver expressionResolver,
         final Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         this.client = client;
         if (enabled == false) {
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java
index 39e32f948f72..50b7d83ee164 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java
@@ -594,11 +594,11 @@ private void checkAutoFollowPattern(
             Consumer<AutoFollowResult> resultHandler
         ) {
             final GroupedActionListener<Tuple<Index, Exception>> groupedListener = new GroupedActionListener<>(
+                leaderIndicesToFollow.size(),
                 ActionListener.wrap(
                     rs -> resultHandler.accept(new AutoFollowResult(autoFollowPattenName, new ArrayList<>(rs))),
                     e -> { throw new AssertionError("must never happen", e); }
-                ),
-                leaderIndicesToFollow.size()
+                )
             );
 
             // Loop through all the as-of-yet-unfollowed indices from the leader
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java
index 175ea917a426..179e4a7e2138 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.ccr.action;
 
 import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
@@ -266,7 +266,7 @@ public long getTookInMillis() {
             super(in);
             mappingVersion = in.readVLong();
             settingsVersion = in.readVLong();
-            if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_3_0)) {
                 aliasesVersion = in.readVLong();
             } else {
                 aliasesVersion = 0;
@@ -302,7 +302,7 @@ public long getTookInMillis() {
         public void writeTo(final StreamOutput out) throws IOException {
             out.writeVLong(mappingVersion);
             out.writeVLong(settingsVersion);
-            if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_3_0)) {
                 out.writeVLong(aliasesVersion);
             }
             out.writeZLong(globalCheckpoint);
@@ -447,11 +447,7 @@ private void globalCheckpointAdvanced(
             final ActionListener<Response> listener
         ) {
             logger.trace("{} global checkpoint advanced to [{}] after waiting for [{}]", shardId, globalCheckpoint, request.getFromSeqNo());
-            try {
-                super.asyncShardOperation(request, shardId, listener);
-            } catch (final IOException caught) {
-                listener.onFailure(caught);
-            }
+            ActionListener.run(listener, l -> super.asyncShardOperation(request, shardId, l));
         }
 
         private void globalCheckpointAdvancementFailure(
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportForgetFollowerAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportForgetFollowerAction.java
index 70eeff821d1f..8d56cc2183f8 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportForgetFollowerAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportForgetFollowerAction.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.Assertions;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
 import org.elasticsearch.action.support.replication.ReplicationResponse;
@@ -72,20 +71,20 @@ public TransportForgetFollowerAction(
 
     @Override
     protected EmptyResult readShardResult(final StreamInput in) {
-        return EmptyResult.readEmptyResultFrom(in);
+        return EmptyResult.INSTANCE;
     }
 
     @Override
-    protected BroadcastResponse newResponse(
-        final ForgetFollowerAction.Request request,
-        final int totalShards,
-        final int successfulShards,
-        final int failedShards,
-        List<EmptyResult> emptyResults,
-        final List<DefaultShardOperationFailedException> shardFailures,
-        final ClusterState clusterState
+    protected ResponseFactory<BroadcastResponse, EmptyResult> getResponseFactory(
+        ForgetFollowerAction.Request request,
+        ClusterState clusterState
     ) {
-        return new BroadcastResponse(totalShards, successfulShards, failedShards, shardFailures);
+        return (totalShards, successfulShards, failedShards, emptyResults, shardFailures) -> new BroadcastResponse(
+            totalShards,
+            successfulShards,
+            failedShards,
+            shardFailures
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java
index 154e3538eb36..caed65fd30a7 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java
@@ -124,23 +124,26 @@ public void clusterStateProcessed(final ClusterState oldState, final ClusterStat
                     return;
                 }
 
-                final GroupedActionListener<ActionResponse.Empty> groupListener = new GroupedActionListener<>(new ActionListener<>() {
+                final GroupedActionListener<ActionResponse.Empty> groupListener = new GroupedActionListener<>(
+                    numberOfShards,
+                    new ActionListener<>() {
 
-                    @Override
-                    public void onResponse(final Collection<ActionResponse.Empty> responses) {
-                        logger.trace(
-                            "[{}] removed retention lease [{}] on all leader primary shards",
-                            indexMetadata.getIndex(),
-                            retentionLeaseId
-                        );
-                        listener.onResponse(AcknowledgedResponse.TRUE);
-                    }
+                        @Override
+                        public void onResponse(final Collection<ActionResponse.Empty> responses) {
+                            logger.trace(
+                                "[{}] removed retention lease [{}] on all leader primary shards",
+                                indexMetadata.getIndex(),
+                                retentionLeaseId
+                            );
+                            listener.onResponse(AcknowledgedResponse.TRUE);
+                        }
 
-                    @Override
-                    public void onFailure(final Exception e) {
-                        onLeaseRemovalFailure(indexMetadata.getIndex(), retentionLeaseId, e);
+                        @Override
+                        public void onFailure(final Exception e) {
+                            onLeaseRemovalFailure(indexMetadata.getIndex(), retentionLeaseId, e);
+                        }
                     }
-                }, numberOfShards);
+                );
                 for (int i = 0; i < numberOfShards; i++) {
                     final ShardId followerShardId = new ShardId(indexMetadata.getIndex(), i);
                     final ShardId leaderShardId = new ShardId(leaderIndex, i);
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java
index 4b927811c673..15e61eb523c9 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java
@@ -97,17 +97,18 @@ protected void dispatchedShardOperationOnPrimary(
         if (logger.isTraceEnabled()) {
             logger.trace("index [{}] on the following primary shard {}", request.getOperations(), primary.routingEntry());
         }
-        ActionListener.completeWith(
-            listener,
-            () -> shardOperationOnPrimary(
+        ActionListener.completeWith(listener, () -> {
+            final var result = shardOperationOnPrimary(
                 request.shardId(),
                 request.getHistoryUUID(),
                 request.getOperations(),
                 request.getMaxSeqNoOfUpdatesOrDeletes(),
                 primary,
                 logger
-            )
-        );
+            );
+            result.replicaRequest().setParentTask(request.getParentTask());
+            return result;
+        });
     }
 
     @Override
@@ -219,7 +220,7 @@ public static WritePrimaryResult<BulkShardOperationsRequest, BulkShardOperations
             appliedOperations,
             maxSeqNoOfUpdatesOrDeletes
         );
-        return new WritePrimaryResult<>(replicaRequest, new BulkShardOperationsResponse(), location, null, primary, logger);
+        return new WritePrimaryResult<>(replicaRequest, new BulkShardOperationsResponse(), location, primary, logger);
     }
 
     @Override
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java
index 9e95befe0dfc..8f802644e72a 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionRunnable;
+import org.elasticsearch.action.SingleResultDeduplicator;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
@@ -137,6 +138,8 @@ public class CcrRepository extends AbstractLifecycleComponent implements Reposit
 
     private final CounterMetric throttledTime = new CounterMetric();
 
+    private final SingleResultDeduplicator<ClusterState> csDeduplicator;
+
     public CcrRepository(RepositoryMetadata metadata, Client client, Settings settings, CcrSettings ccrSettings, ThreadPool threadPool) {
         this.metadata = metadata;
         this.ccrSettings = ccrSettings;
@@ -145,6 +148,17 @@ public CcrRepository(RepositoryMetadata metadata, Client client, Settings settin
         this.remoteClusterAlias = Strings.split(metadata.name(), NAME_PREFIX)[1];
         this.client = client;
         this.threadPool = threadPool;
+        csDeduplicator = new SingleResultDeduplicator<>(
+            threadPool.getThreadContext(),
+            l -> getRemoteClusterClient().admin()
+                .cluster()
+                .prepareState()
+                .clear()
+                .setMetadata(true)
+                .setNodes(true)
+                .setMasterNodeTimeout(TimeValue.MAX_VALUE)
+                .execute(l.map(ClusterStateResponse::getState))
+        );
     }
 
     @Override
@@ -177,26 +191,22 @@ public void getSnapshotInfo(GetSnapshotInfoContext context) {
         assert snapshotIds.size() == 1 && SNAPSHOT_ID.equals(snapshotIds.iterator().next())
             : "RemoteClusterRepository only supports " + SNAPSHOT_ID + " as the SnapshotId but saw " + snapshotIds;
         try {
-            getRemoteClusterClient().admin()
-                .cluster()
-                .prepareState()
-                .clear()
-                .setMetadata(true)
-                .setNodes(true)
-                // fork to the snapshot meta pool because the context expects to run on it and asserts that it does
-                .execute(new ThreadedActionListener<>(logger, threadPool, ThreadPool.Names.SNAPSHOT_META, context.map(response -> {
-                    Metadata responseMetadata = response.getState().metadata();
+            csDeduplicator.execute(
+                new ThreadedActionListener<>(threadPool.executor(ThreadPool.Names.SNAPSHOT_META), context.map(response -> {
+                    Metadata responseMetadata = response.metadata();
                     Map<String, IndexMetadata> indicesMap = responseMetadata.indices();
                     return new SnapshotInfo(
                         new Snapshot(this.metadata.name(), SNAPSHOT_ID),
                         List.copyOf(indicesMap.keySet()),
                         List.copyOf(responseMetadata.dataStreams().keySet()),
                         List.of(),
-                        response.getState().getNodes().getMaxNodeVersion(),
+                        response.getNodes().getMaxNodeVersion(),
                         SnapshotState.SUCCESS
                     );
-                }), false));
+                }))
+            );
         } catch (Exception e) {
+            assert false : e;
             context.onFailure(e);
         }
     }
@@ -255,8 +265,8 @@ public IndexMetadata getSnapshotIndexMetaData(RepositoryData repositoryData, Sna
     @Override
     public void getRepositoryData(ActionListener<RepositoryData> listener) {
         try {
-            getRemoteClusterClient().admin().cluster().prepareState().clear().setMetadata(true).execute(listener.map(response -> {
-                final Metadata remoteMetadata = response.getState().getMetadata();
+            csDeduplicator.execute(listener.map(response -> {
+                final Metadata remoteMetadata = response.getMetadata();
                 final String[] concreteAllIndices = remoteMetadata.getConcreteAllIndices();
                 final Map<String, SnapshotId> copiedSnapshotIds = Maps.newMapWithExpectedSize(concreteAllIndices.length);
                 final Map<String, RepositoryData.SnapshotDetails> snapshotsDetails = Maps.newMapWithExpectedSize(concreteAllIndices.length);
@@ -285,6 +295,7 @@ public void getRepositoryData(ActionListener<RepositoryData> listener) {
                 );
             }));
         } catch (Exception e) {
+            assert false;
             listener.onFailure(e);
         }
     }
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestCcrStatsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestCcrStatsAction.java
index 76281e3f0b86..6e5c2e4f396c 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestCcrStatsAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestCcrStatsAction.java
@@ -7,10 +7,14 @@
 
 package org.elasticsearch.xpack.ccr.rest;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.support.ThreadedActionListener;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.rest.action.RestChunkedToXContentListener;
+import org.elasticsearch.xpack.ccr.Ccr;
 import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction;
 
 import java.util.List;
@@ -19,6 +23,8 @@
 
 public class RestCcrStatsAction extends BaseRestHandler {
 
+    private static final Logger logger = LogManager.getLogger(RestCcrStatsAction.class);
+
     @Override
     public List<Route> routes() {
         return List.of(new Route(GET, "/_ccr/stats"));
@@ -32,7 +38,14 @@ public String getName() {
     @Override
     protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) {
         final CcrStatsAction.Request request = new CcrStatsAction.Request();
-        return channel -> client.execute(CcrStatsAction.INSTANCE, request, new RestToXContentListener<>(channel));
+        return channel -> client.execute(
+            CcrStatsAction.INSTANCE,
+            request,
+            new ThreadedActionListener<>(
+                client.threadPool().executor(Ccr.CCR_THREAD_POOL_NAME),
+                new RestChunkedToXContentListener<>(channel)
+            )
+        );
     }
 
 }
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestFollowInfoAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestFollowInfoAction.java
index 555b05edd54e..6607b532c56e 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestFollowInfoAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestFollowInfoAction.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.rest.action.RestChunkedToXContentListener;
 import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction;
 
 import java.util.List;
@@ -34,7 +34,7 @@ public String getName() {
     protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) {
         final FollowInfoAction.Request request = new FollowInfoAction.Request();
         request.setFollowerIndices(Strings.splitStringByCommaToArray(restRequest.param("index")));
-        return channel -> client.execute(FollowInfoAction.INSTANCE, request, new RestToXContentListener<>(channel));
+        return channel -> client.execute(FollowInfoAction.INSTANCE, request, new RestChunkedToXContentListener<>(channel));
     }
 
 }
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestFollowStatsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestFollowStatsAction.java
index 38fbb0a1c060..07fdaef94637 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestFollowStatsAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestFollowStatsAction.java
@@ -7,11 +7,15 @@
 
 package org.elasticsearch.xpack.ccr.rest;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.support.ThreadedActionListener;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.rest.action.RestChunkedToXContentListener;
+import org.elasticsearch.xpack.ccr.Ccr;
 import org.elasticsearch.xpack.core.ccr.action.FollowStatsAction;
 
 import java.util.List;
@@ -20,6 +24,8 @@
 
 public class RestFollowStatsAction extends BaseRestHandler {
 
+    private static final Logger logger = LogManager.getLogger(RestFollowStatsAction.class);
+
     @Override
     public List<Route> routes() {
         return List.of(new Route(GET, "/{index}/_ccr/stats"));
@@ -34,7 +40,14 @@ public String getName() {
     protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) {
         final FollowStatsAction.StatsRequest request = new FollowStatsAction.StatsRequest();
         request.setIndices(Strings.splitStringByCommaToArray(restRequest.param("index")));
-        return channel -> client.execute(FollowStatsAction.INSTANCE, request, new RestToXContentListener<>(channel));
+        return channel -> client.execute(
+            FollowStatsAction.INSTANCE,
+            request,
+            new ThreadedActionListener<>(
+                client.threadPool().executor(Ccr.CCR_THREAD_POOL_NAME),
+                new RestChunkedToXContentListener<>(channel)
+            )
+        );
     }
 
 }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowMetadataTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowMetadataTests.java
index 58d0f7a4b6a3..066758a996d2 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowMetadataTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowMetadataTests.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
 
@@ -23,7 +23,7 @@
 import java.util.Map;
 import java.util.function.Predicate;
 
-public class AutoFollowMetadataTests extends AbstractXContentSerializingTestCase<AutoFollowMetadata> {
+public class AutoFollowMetadataTests extends AbstractChunkedSerializingTestCase<AutoFollowMetadata> {
 
     @Override
     protected Predicate<String> getRandomFieldsExcludeFilter() {
@@ -76,6 +76,11 @@ protected AutoFollowMetadata createTestInstance() {
         return new AutoFollowMetadata(configs, followedLeaderIndices, headers);
     }
 
+    @Override
+    protected AutoFollowMetadata mutateInstance(AutoFollowMetadata instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<AutoFollowMetadata> instanceReader() {
         return AutoFollowMetadata::new;
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java
index 0829d8ab3e13..df2cdf75fbdc 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java
@@ -23,6 +23,11 @@ protected CCRInfoTransportAction.Usage createTestInstance() {
         );
     }
 
+    @Override
+    protected CCRInfoTransportAction.Usage mutateInstance(CCRInfoTransportAction.Usage instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<CCRInfoTransportAction.Usage> instanceReader() {
         return CCRInfoTransportAction.Usage::new;
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ActivateAutoFollowPatternActionRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ActivateAutoFollowPatternActionRequestTests.java
index 28813a58b210..03b197b629d0 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ActivateAutoFollowPatternActionRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ActivateAutoFollowPatternActionRequestTests.java
@@ -22,6 +22,11 @@ protected ActivateAutoFollowPatternAction.Request createTestInstance() {
         return new ActivateAutoFollowPatternAction.Request(randomAlphaOfLength(5), randomBoolean());
     }
 
+    @Override
+    protected ActivateAutoFollowPatternAction.Request mutateInstance(ActivateAutoFollowPatternAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<ActivateAutoFollowPatternAction.Request> instanceReader() {
         return ActivateAutoFollowPatternAction.Request::new;
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowStatsResponseTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowStatsResponseTests.java
index 31cc73cb7022..a9786e066083 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowStatsResponseTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowStatsResponseTests.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.ccr.action;
 
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.ccr.AutoFollowStats;
 import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction;
@@ -35,4 +36,19 @@ protected CcrStatsAction.Response createTestInstance() {
         FollowStatsAction.StatsResponses statsResponse = createStatsResponse();
         return new CcrStatsAction.Response(autoFollowStats, statsResponse);
     }
+
+    @Override
+    protected CcrStatsAction.Response mutateInstance(CcrStatsAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
+    public void testChunking() {
+        AbstractChunkedSerializingTestCase.assertChunkCount(
+            createTestInstance(),
+            instance -> Math.toIntExact(
+                2 * instance.getFollowStats().getStatsResponses().stream().map(s -> s.status().followerIndex()).distinct().count()
+                    + instance.getFollowStats().getStatsResponses().size() + 4
+            )
+        );
+    }
 }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowStatsTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowStatsTests.java
index 826f0d01c5a5..44ac13394e4d 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowStatsTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowStatsTests.java
@@ -42,6 +42,11 @@ protected AutoFollowStats createTestInstance() {
         );
     }
 
+    @Override
+    protected AutoFollowStats mutateInstance(AutoFollowStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     static NavigableMap<String, Tuple<Long, ElasticsearchException>> randomReadExceptions() {
         final int count = randomIntBetween(0, 16);
         final NavigableMap<String, Tuple<Long, ElasticsearchException>> readExceptions = new TreeMap<>();
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/DeleteAutoFollowPatternRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/DeleteAutoFollowPatternRequestTests.java
index bf04ae47b488..b4a28fd14932 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/DeleteAutoFollowPatternRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/DeleteAutoFollowPatternRequestTests.java
@@ -21,4 +21,9 @@ protected Writeable.Reader<DeleteAutoFollowPatternAction.Request> instanceReader
     protected DeleteAutoFollowPatternAction.Request createTestInstance() {
         return new DeleteAutoFollowPatternAction.Request(randomAlphaOfLength(4));
     }
+
+    @Override
+    protected DeleteAutoFollowPatternAction.Request mutateInstance(DeleteAutoFollowPatternAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowInfoRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowInfoRequestTests.java
index 064aa90a0c3d..f08ab5aca1aa 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowInfoRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowInfoRequestTests.java
@@ -23,4 +23,9 @@ protected FollowInfoAction.Request createTestInstance() {
         request.setFollowerIndices(generateRandomStringArray(4, 4, true, false));
         return request;
     }
+
+    @Override
+    protected FollowInfoAction.Request mutateInstance(FollowInfoAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowInfoResponseTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowInfoResponseTests.java
index 0fd1b5b75628..a199a423f9a6 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowInfoResponseTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowInfoResponseTests.java
@@ -7,31 +7,31 @@
 package org.elasticsearch.xpack.ccr.action;
 
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction;
 import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction.Response.FollowerInfo;
 import org.elasticsearch.xpack.core.ccr.action.FollowParameters;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
 import static org.elasticsearch.xpack.core.ccr.action.FollowInfoAction.Response.FOLLOWER_INDICES_FIELD;
 import static org.elasticsearch.xpack.core.ccr.action.FollowInfoAction.Response.Status;
 
-public class FollowInfoResponseTests extends AbstractXContentSerializingTestCase<FollowInfoAction.Response> {
+public class FollowInfoResponseTests extends AbstractChunkedSerializingTestCase<FollowInfoAction.Response> {
 
-    static final ConstructingObjectParser<FollowerInfo, Void> INFO_PARSER = new ConstructingObjectParser<>("info_parser", args -> {
-        return new FollowerInfo(
+    static final ConstructingObjectParser<FollowerInfo, Void> INFO_PARSER = new ConstructingObjectParser<>(
+        "info_parser",
+        args -> new FollowerInfo(
             (String) args[0],
             (String) args[1],
             (String) args[2],
             Status.fromString((String) args[3]),
             (FollowParameters) args[4]
-        );
-    });
+        )
+    );
 
     static {
         INFO_PARSER.declareString(ConstructingObjectParser.constructorArg(), FollowerInfo.FOLLOWER_INDEX_FIELD);
@@ -48,7 +48,7 @@ public class FollowInfoResponseTests extends AbstractXContentSerializingTestCase
     @SuppressWarnings("unchecked")
     static final ConstructingObjectParser<FollowInfoAction.Response, Void> PARSER = new ConstructingObjectParser<>(
         "response",
-        args -> { return new FollowInfoAction.Response((List<FollowerInfo>) args[0]); }
+        args -> new FollowInfoAction.Response((List<FollowerInfo>) args[0])
     );
 
     static {
@@ -56,7 +56,7 @@ public class FollowInfoResponseTests extends AbstractXContentSerializingTestCase
     }
 
     @Override
-    protected FollowInfoAction.Response doParseInstance(XContentParser parser) throws IOException {
+    protected FollowInfoAction.Response doParseInstance(XContentParser parser) {
         return PARSER.apply(parser, null);
     }
 
@@ -87,4 +87,13 @@ protected FollowInfoAction.Response createTestInstance() {
         }
         return new FollowInfoAction.Response(infos);
     }
+
+    @Override
+    protected FollowInfoAction.Response mutateInstance(FollowInfoAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
+    public void testChunking() {
+        AbstractChunkedSerializingTestCase.assertChunkCount(createTestInstance(), instance -> instance.getFollowInfos().size() + 2);
+    }
 }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternRequestTests.java
index ab6af0d4ee8b..b451e07bd037 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternRequestTests.java
@@ -25,4 +25,9 @@ protected GetAutoFollowPatternAction.Request createTestInstance() {
         }
         return request;
     }
+
+    @Override
+    protected GetAutoFollowPatternAction.Request mutateInstance(GetAutoFollowPatternAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternResponseTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternResponseTests.java
index c2a0ddd858ea..3d9cd75d220a 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternResponseTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternResponseTests.java
@@ -53,4 +53,9 @@ protected GetAutoFollowPatternAction.Response createTestInstance() {
         }
         return new GetAutoFollowPatternAction.Response(patterns);
     }
+
+    @Override
+    protected GetAutoFollowPatternAction.Response mutateInstance(GetAutoFollowPatternAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java
index 107b9aa6de38..46bdab1a7463 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java
@@ -58,6 +58,11 @@ protected PutAutoFollowPatternAction.Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected PutAutoFollowPatternAction.Request mutateInstance(PutAutoFollowPatternAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected PutAutoFollowPatternAction.Request createXContextTestInstance(XContentType xContentType) {
         // follower index parameter is not part of the request body and is provided in the url path.
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java
index 4219a434b0cc..aed5b43d7ca4 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java
@@ -65,7 +65,7 @@ protected PutFollowAction.Request doParseInstance(XContentParser parser) throws
     }
 
     @Override
-    protected PutFollowAction.Request mutateInstance(PutFollowAction.Request instance) throws IOException {
+    protected PutFollowAction.Request mutateInstance(PutFollowAction.Request instance) {
         PutFollowAction.Request request = new PutFollowAction.Request();
         request.setFollowerIndex(instance.getFollowerIndex());
         request.waitForActiveShards(instance.waitForActiveShards());
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionResponseTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionResponseTests.java
index 14f21464182d..d43445edb768 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionResponseTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionResponseTests.java
@@ -21,4 +21,9 @@ protected Writeable.Reader<PutFollowAction.Response> instanceReader() {
     protected PutFollowAction.Response createTestInstance() {
         return new PutFollowAction.Response(randomBoolean(), randomBoolean(), randomBoolean());
     }
+
+    @Override
+    protected PutFollowAction.Response mutateInstance(PutFollowAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ResumeFollowActionRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ResumeFollowActionRequestTests.java
index ca57b0db3819..4df1da7d11eb 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ResumeFollowActionRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ResumeFollowActionRequestTests.java
@@ -39,6 +39,11 @@ protected ResumeFollowAction.Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected ResumeFollowAction.Request mutateInstance(ResumeFollowAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected ResumeFollowAction.Request createXContextTestInstance(XContentType type) {
         // follower index parameter is not part of the request body and is provided in the url path.
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesRequestTests.java
index cb52e3870690..b946fce15f94 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesRequestTests.java
@@ -23,6 +23,11 @@ protected ShardChangesAction.Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected ShardChangesAction.Request mutateInstance(ShardChangesAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<ShardChangesAction.Request> instanceReader() {
         return ShardChangesAction.Request::new;
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesResponseTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesResponseTests.java
index abf9f8c69bca..47cdf0372977 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesResponseTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesResponseTests.java
@@ -37,6 +37,11 @@ protected ShardChangesAction.Response createTestInstance() {
         );
     }
 
+    @Override
+    protected ShardChangesAction.Response mutateInstance(ShardChangesAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<ShardChangesAction.Response> instanceReader() {
         return ShardChangesAction.Response::new;
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTaskStatusTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTaskStatusTests.java
index 4960da8848b4..00c8cfe93436 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTaskStatusTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTaskStatusTests.java
@@ -67,6 +67,11 @@ protected ShardFollowNodeTaskStatus createTestInstance() {
         );
     }
 
+    @Override
+    protected ShardFollowNodeTaskStatus mutateInstance(ShardFollowNodeTaskStatus instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected void assertEqualInstances(final ShardFollowNodeTaskStatus expectedInstance, final ShardFollowNodeTaskStatus newInstance) {
         assertNotSame(expectedInstance, newInstance);
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java
index 479f2f08572b..c90df2dee69e 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java
@@ -808,7 +808,7 @@ protected void performOnPrimary(IndexShard primary, BulkShardOperationsRequest r
             } catch (InterruptedException | ExecutionException | IOException e) {
                 throw new RuntimeException(e);
             }
-            listener.onResponse(new PrimaryResult(ccrResult.replicaRequest(), ccrResult.finalResponseIfSuccessful));
+            listener.onResponse(new PrimaryResult(ccrResult.replicaRequest(), ccrResult.replicationResponse));
         }
 
         @Override
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/StatsRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/StatsRequestTests.java
index 5d06325449a7..42365f40ffd7 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/StatsRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/StatsRequestTests.java
@@ -25,4 +25,9 @@ protected FollowStatsAction.StatsRequest createTestInstance() {
         }
         return statsRequest;
     }
+
+    @Override
+    protected FollowStatsAction.StatsRequest mutateInstance(FollowStatsAction.StatsRequest instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/StatsResponsesTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/StatsResponsesTests.java
index 10759072b614..94c380d44921 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/StatsResponsesTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/StatsResponsesTests.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.ccr.ShardFollowNodeTaskStatus;
 import org.elasticsearch.xpack.core.ccr.action.FollowStatsAction;
@@ -28,6 +29,11 @@ protected FollowStatsAction.StatsResponses createTestInstance() {
         return createStatsResponse();
     }
 
+    @Override
+    protected FollowStatsAction.StatsResponses mutateInstance(FollowStatsAction.StatsResponses instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     static FollowStatsAction.StatsResponses createStatsResponse() {
         int numResponses = randomIntBetween(0, 8);
         List<FollowStatsAction.StatsResponse> responses = new ArrayList<>(numResponses);
@@ -67,4 +73,15 @@ static FollowStatsAction.StatsResponses createStatsResponse() {
         }
         return new FollowStatsAction.StatsResponses(Collections.emptyList(), Collections.emptyList(), responses);
     }
+
+    public void testChunking() {
+        AbstractChunkedSerializingTestCase.assertChunkCount(
+            createTestInstance(),
+            instance -> Math.toIntExact(
+                2 * instance.getStatsResponses().stream().map(s -> s.status().followerIndex()).distinct().count() + instance
+                    .getStatsResponses()
+                    .size() + 2
+            )
+        );
+    }
 }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/allocation/CcrPrimaryFollowerAllocationDeciderTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/allocation/CcrPrimaryFollowerAllocationDeciderTests.java
index b0cc279e6ded..6693855e5756 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/allocation/CcrPrimaryFollowerAllocationDeciderTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/allocation/CcrPrimaryFollowerAllocationDeciderTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -63,7 +64,7 @@ public void testRegularIndex() {
         DiscoveryNodes.Builder discoveryNodes = DiscoveryNodes.builder();
         nodes.forEach(discoveryNodes::add);
         Metadata metadata = Metadata.builder().put(indexMetadata).build();
-        RoutingTable.Builder routingTable = RoutingTable.builder();
+        RoutingTable.Builder routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         if (randomBoolean()) {
             routingTable.addAsNew(metadata.index(index));
         } else if (randomBoolean()) {
@@ -112,7 +113,8 @@ public void testAlreadyBootstrappedFollowerIndex() {
         DiscoveryNodes.Builder discoveryNodes = DiscoveryNodes.builder();
         nodes.forEach(discoveryNodes::add);
         Metadata metadata = Metadata.builder().put(indexMetadata).build();
-        RoutingTable.Builder routingTable = RoutingTable.builder().addAsRecovery(metadata.index(index));
+        RoutingTable.Builder routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsRecovery(metadata.index(index));
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .nodes(discoveryNodes)
             .metadata(metadata)
@@ -147,7 +149,7 @@ public void testBootstrappingFollowerIndex() {
         DiscoveryNode dataAndRemoteNode = newNode("dr1", Set.of(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE));
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(dataOnlyNode).add(dataAndRemoteNode).build();
         Metadata metadata = Metadata.builder().put(indexMetadata).build();
-        RoutingTable.Builder routingTable = RoutingTable.builder()
+        RoutingTable.Builder routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
             .addAsNewRestore(metadata.index(index), newSnapshotRecoverySource(), new HashSet<>());
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .nodes(discoveryNodes)
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java
index 8472b23865e1..8ff603d7f213 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java
@@ -287,7 +287,9 @@ public void onFailedEngine(String reason, Exception e) {
             () -> primaryTerm.get(),
             IndexModule.DEFAULT_SNAPSHOT_COMMIT_SUPPLIER,
             null,
-            System::nanoTime
+            System::nanoTime,
+            null,
+            true
         );
     }
 
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRepositoryRetentionLeaseTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRepositoryRetentionLeaseTests.java
index 841a8c45a8ff..a1c52728e9a7 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRepositoryRetentionLeaseTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRepositoryRetentionLeaseTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.seqno.RetentionLeaseActions;
 import org.elasticsearch.index.seqno.RetentionLeaseAlreadyExistsException;
@@ -54,13 +55,7 @@ public void testWhenRetentionLeaseAlreadyExistsWeTryToRenewIt() {
             CcrSettings.getSettings().stream().filter(Setting::hasNodeScope)
         ).collect(Collectors.toSet());
 
-        final CcrRepository repository = new CcrRepository(
-            repositoryMetadata,
-            mock(Client.class),
-            Settings.EMPTY,
-            new CcrSettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, settings)),
-            mock(ThreadPool.class)
-        );
+        final CcrRepository repository = createCcrRepository(repositoryMetadata, settings);
 
         final ShardId followerShardId = new ShardId(new Index("follower-index-name", "follower-index-uuid"), 0);
         final ShardId leaderShardId = new ShardId(new Index("leader-index-name", "leader-index-uuid"), 0);
@@ -110,6 +105,19 @@ public void testWhenRetentionLeaseAlreadyExistsWeTryToRenewIt() {
         verifyNoMoreInteractions(remoteClient);
     }
 
+    private static CcrRepository createCcrRepository(RepositoryMetadata repositoryMetadata, Set<Setting<?>> settings) {
+        final ThreadPool threadPool = mock(ThreadPool.class);
+        final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        when(threadPool.getThreadContext()).thenReturn(threadContext);
+        return new CcrRepository(
+            repositoryMetadata,
+            mock(Client.class),
+            Settings.EMPTY,
+            new CcrSettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, settings)),
+            threadPool
+        );
+    }
+
     public void testWhenRetentionLeaseExpiresBeforeWeCanRenewIt() {
         final RepositoryMetadata repositoryMetadata = mock(RepositoryMetadata.class);
         when(repositoryMetadata.name()).thenReturn(CcrRepository.NAME_PREFIX);
@@ -118,13 +126,7 @@ public void testWhenRetentionLeaseExpiresBeforeWeCanRenewIt() {
             CcrSettings.getSettings().stream().filter(Setting::hasNodeScope)
         ).collect(Collectors.toSet());
 
-        final CcrRepository repository = new CcrRepository(
-            repositoryMetadata,
-            mock(Client.class),
-            Settings.EMPTY,
-            new CcrSettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, settings)),
-            mock(ThreadPool.class)
-        );
+        final CcrRepository repository = createCcrRepository(repositoryMetadata, settings);
 
         final ShardId followerShardId = new ShardId(new Index("follower-index-name", "follower-index-uuid"), 0);
         final ShardId leaderShardId = new ShardId(new Index("leader-index-name", "leader-index-uuid"), 0);
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/AutoFollowStatsMonitoringDocTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/AutoFollowStatsMonitoringDocTests.java
index 07311bd7827b..3a728ae46732 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/AutoFollowStatsMonitoringDocTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/AutoFollowStatsMonitoringDocTests.java
@@ -108,7 +108,7 @@ public void testToXContent() throws IOException {
             xContent.utf8ToString(),
             equalTo(
                 XContentHelper.stripWhitespace(
-                    formatted(
+                    Strings.format(
                         """
                             {
                               "cluster_uuid": "_cluster",
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/FollowStatsMonitoringDocTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/FollowStatsMonitoringDocTests.java
index fe68cd927a60..71f5836fd9de 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/FollowStatsMonitoringDocTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/FollowStatsMonitoringDocTests.java
@@ -154,7 +154,7 @@ public void testToXContent() throws IOException {
             xContent.utf8ToString(),
             equalTo(
                 XContentHelper.stripWhitespace(
-                    formatted(
+                    Strings.format(
                         """
                             {
                               "cluster_uuid": "_cluster",
diff --git a/x-pack/plugin/core/build.gradle b/x-pack/plugin/core/build.gradle
index ba4218a0eab8..e09c82fa27e4 100644
--- a/x-pack/plugin/core/build.gradle
+++ b/x-pack/plugin/core/build.gradle
@@ -6,9 +6,9 @@ import java.nio.file.Paths
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.publish'
 apply plugin: 'elasticsearch.internal-cluster-test'
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.internal-java-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-test-artifact'
 
 archivesBaseName = 'x-pack-core'
@@ -26,6 +26,11 @@ tasks.named("dependencyLicenses").configure {
   mapping from: /commons-.*/, to: 'commons' // pulled in by rest client
 }
 
+configurations {
+  signedCerts
+  rootCert
+}
+
 dependencies {
   compileOnly project(":server")
   api project(':libs:elasticsearch-grok')
@@ -59,6 +64,8 @@ dependencies {
 
   yamlRestTestImplementation project(':x-pack:plugin:core')
   javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
+  signedCerts fileTree("src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed")
+  rootCert files("src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca.crt")
 }
 
 ext.expansions = [
diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java
index c0e295f90cb7..8e311b4932cb 100644
--- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java
+++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java
@@ -51,7 +51,7 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0, autoManageMasterNodes = false)
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
 public class DataTierAllocationDeciderIT extends ESIntegTestCase {
     private static final String index = "myindex";
 
@@ -63,7 +63,6 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     @Before
     public void setUpMasterNode() {
         // Ensure that master nodes cannot hold any data
-        internalCluster().setBootstrapMasterNodeIndex(0);
         internalCluster().startMasterOnlyNode();
     }
 
@@ -521,7 +520,7 @@ public void startHotOnlyNode(@Nullable String externalId) {
             nodeSettings.put(NODE_EXTERNAL_ID_SETTING.getKey(), externalId);
         }
 
-        internalCluster().startNode(nodeSettings);
+        internalCluster().startNode(nodeSettings.build());
     }
 
     public void startWarmOnlyNode() {
@@ -536,7 +535,7 @@ public String startWarmOnlyNode(@Nullable String externalId) {
         if (externalId != null) {
             nodeSettings.put(NODE_EXTERNAL_ID_SETTING.getKey(), externalId);
         }
-        return internalCluster().startNode(nodeSettings);
+        return internalCluster().startNode(nodeSettings.build());
     }
 
     public void startColdOnlyNode() {
@@ -552,7 +551,7 @@ public String startColdOnlyNode(@Nullable String externalId) {
             nodeSettings.put(NODE_EXTERNAL_ID_SETTING.getKey(), externalId);
         }
 
-        return internalCluster().startNode(nodeSettings);
+        return internalCluster().startNode(nodeSettings.build());
     }
 
     public void startFrozenOnlyNode() {
diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierShardAvailabilityHealthIndicatorIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierShardAvailabilityHealthIndicatorIT.java
index 93479a061426..12b327cfad37 100644
--- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierShardAvailabilityHealthIndicatorIT.java
+++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierShardAvailabilityHealthIndicatorIT.java
@@ -70,7 +70,7 @@ public void testIncreaseTierCapacityDiagnosisWhenCreated() throws Exception {
         ensureYellow("test");
         GetHealthAction.Response healthResponse = client().execute(
             GetHealthAction.INSTANCE,
-            new GetHealthAction.Request(ShardsAvailabilityHealthIndicatorService.NAME, true)
+            new GetHealthAction.Request(ShardsAvailabilityHealthIndicatorService.NAME, true, 1000)
         ).get();
         HealthIndicatorResult indicatorResult = healthResponse.findIndicator(ShardsAvailabilityHealthIndicatorService.NAME);
         assertThat(indicatorResult.status(), equalTo(HealthStatus.YELLOW));
@@ -107,7 +107,7 @@ public void testIncreaseTierCapacityDiagnosisWhenTierShrinksUnexpectedly() throw
         ensureYellow("test");
         GetHealthAction.Response healthResponse = client().execute(
             GetHealthAction.INSTANCE,
-            new GetHealthAction.Request(ShardsAvailabilityHealthIndicatorService.NAME, true)
+            new GetHealthAction.Request(ShardsAvailabilityHealthIndicatorService.NAME, true, 1000)
         ).get();
         ClusterAllocationExplanation explain = client().admin()
             .cluster()
@@ -152,7 +152,7 @@ public void testRemovingNodeReturnsYellowForDelayedIndex() throws Exception {
         ensureYellow("test");
         GetHealthAction.Response healthResponse = client().execute(
             GetHealthAction.INSTANCE,
-            new GetHealthAction.Request(ShardsAvailabilityHealthIndicatorService.NAME, true)
+            new GetHealthAction.Request(ShardsAvailabilityHealthIndicatorService.NAME, true, 1000)
         ).get();
         HealthIndicatorResult indicatorResult = healthResponse.findIndicator(ShardsAvailabilityHealthIndicatorService.NAME);
         assertThat(indicatorResult.status(), equalTo(HealthStatus.YELLOW));
diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/XPackUsageRestCancellationIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/XPackUsageRestCancellationIT.java
index 6f7cfd1cf908..65af5c0f956a 100644
--- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/XPackUsageRestCancellationIT.java
+++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/rest/action/XPackUsageRestCancellationIT.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.xpack.core.rest.action;
 
 import org.apache.http.client.methods.HttpGet;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -172,8 +172,8 @@ protected void masterOperation(
             blockActionLatch.await();
             listener.onResponse(new XPackUsageFeatureResponse(new XPackFeatureSet.Usage("test", false, false) {
                 @Override
-                public Version getMinimalSupportedVersion() {
-                    return Version.CURRENT;
+                public TransportVersion getMinimalSupportedVersion() {
+                    return TransportVersion.CURRENT;
                 }
             }));
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java
index a3f081882b09..768fea6218ea 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java
@@ -7,10 +7,12 @@
 package org.elasticsearch.index.engine.frozen;
 
 import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.LeafMetaData;
@@ -22,8 +24,9 @@
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.StoredFields;
+import org.apache.lucene.index.TermVectors;
 import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.VectorValues;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Bits;
@@ -212,7 +215,12 @@ public NumericDocValues getNormValues(String field) {
         }
 
         @Override
-        public VectorValues getVectorValues(String field) throws IOException {
+        public FloatVectorValues getFloatVectorValues(String field) throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public ByteVectorValues getByteVectorValues(String field) throws IOException {
             throw new UnsupportedOperationException();
         }
 
@@ -221,6 +229,11 @@ public TopDocs searchNearestVectors(String field, float[] target, int k, Bits ac
             throw new UnsupportedOperationException();
         }
 
+        @Override
+        public TopDocs searchNearestVectors(String field, byte[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
         @Override
         public FieldInfos getFieldInfos() {
             return fieldInfos;
@@ -249,6 +262,16 @@ public Fields getTermVectors(int docId) {
             throw new UnsupportedOperationException();
         }
 
+        @Override
+        public TermVectors termVectors() throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public StoredFields storedFields() throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
         @Override
         public int numDocs() {
             return numDocs;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetFeatureUsageResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetFeatureUsageResponse.java
index 8a91c9b0d245..91380980a58a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetFeatureUsageResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetFeatureUsageResponse.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.license;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -48,14 +48,14 @@ public FeatureUsageInfo(
         }
 
         public FeatureUsageInfo(StreamInput in) throws IOException {
-            if (in.getVersion().onOrAfter(Version.V_7_16_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
                 this.family = in.readOptionalString();
             } else {
                 this.family = null;
             }
             this.name = in.readString();
             this.lastUsedTime = ZonedDateTime.ofInstant(Instant.ofEpochSecond(in.readLong()), ZoneOffset.UTC);
-            if (in.getVersion().onOrAfter(Version.V_7_15_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_15_0)) {
                 this.context = in.readOptionalString();
             } else {
                 this.context = null;
@@ -65,12 +65,12 @@ public FeatureUsageInfo(StreamInput in) throws IOException {
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            if (out.getVersion().onOrAfter(Version.V_7_16_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
                 out.writeOptionalString(this.family);
             }
             out.writeString(name);
             out.writeLong(lastUsedTime.toEpochSecond());
-            if (out.getVersion().onOrAfter(Version.V_7_15_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_15_0)) {
                 out.writeOptionalString(this.context);
             }
             out.writeString(licenseLevel);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java
index 41cd4dbf5985..703b2cea267c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java
@@ -520,17 +520,21 @@ public void clusterChanged(ClusterChangedEvent event) {
 
             final LicensesMetadata prevLicensesMetadata = previousClusterState.getMetadata().custom(LicensesMetadata.TYPE);
             final LicensesMetadata currentLicensesMetadata = currentClusterState.getMetadata().custom(LicensesMetadata.TYPE);
-            if (logger.isDebugEnabled()) {
-                logger.debug("previous [{}]", prevLicensesMetadata);
-                logger.debug("current [{}]", currentLicensesMetadata);
-            }
             // notify all interested plugins
             if (previousClusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK) || prevLicensesMetadata == null) {
                 if (currentLicensesMetadata != null) {
+                    logger.debug("state recovered: previous license [{}]", prevLicensesMetadata);
+                    logger.debug("state recovered: current license [{}]", currentLicensesMetadata);
                     onUpdate(currentLicensesMetadata);
+                } else {
+                    logger.trace("state recovered: no current license");
                 }
             } else if (prevLicensesMetadata.equals(currentLicensesMetadata) == false) {
+                logger.debug("previous [{}]", prevLicensesMetadata);
+                logger.debug("current [{}]", currentLicensesMetadata);
                 onUpdate(currentLicensesMetadata);
+            } else {
+                logger.trace("license unchanged [{}]", currentLicensesMetadata);
             }
 
             License currentLicense = null;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensesMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensesMetadata.java
index 195d97e9b486..db5b363fdccc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensesMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensesMetadata.java
@@ -6,18 +6,21 @@
  */
 package org.elasticsearch.license;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AbstractNamedDiffable;
 import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.EnumSet;
+import java.util.Iterator;
 import java.util.Objects;
 
 /**
@@ -103,8 +106,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.CURRENT.minimumCompatibilityVersion();
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.CURRENT.minimumCompatibilityVersion();
     }
 
     @Override
@@ -138,18 +141,20 @@ public static LicensesMetadata fromXContent(XContentParser parser) throws IOExce
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        if (license == LICENSE_TOMBSTONE) {
-            builder.nullField(Fields.LICENSE);
-        } else {
-            builder.startObject(Fields.LICENSE);
-            license.toInnerXContent(builder, params);
-            builder.endObject();
-        }
-        if (trialVersion != null) {
-            builder.field(Fields.TRIAL_LICENSE, trialVersion.toString());
-        }
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return Iterators.single(((builder, params) -> {
+            if (license == LICENSE_TOMBSTONE) {
+                builder.nullField(Fields.LICENSE);
+            } else {
+                builder.startObject(Fields.LICENSE);
+                license.toInnerXContent(builder, params);
+                builder.endObject();
+            }
+            if (trialVersion != null) {
+                builder.field(Fields.TRIAL_LICENSE, trialVersion.toString());
+            }
+            return builder;
+        }));
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoRequest.java
index 9889a88c9138..dca4352690bd 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoRequest.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.protocol.xpack;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -57,7 +57,7 @@ public XPackInfoRequest(StreamInput in) throws IOException {
             categories.add(Category.valueOf(in.readString()));
         }
         this.categories = categories;
-        if (hasLicenseVersionField(in.getVersion())) {
+        if (hasLicenseVersionField(in.getTransportVersion())) {
             int ignoredLicenseVersion = in.readVInt();
         }
     }
@@ -91,12 +91,12 @@ public void writeTo(StreamOutput out) throws IOException {
         for (Category category : categories) {
             out.writeString(category.name());
         }
-        if (hasLicenseVersionField(out.getVersion())) {
+        if (hasLicenseVersionField(out.getTransportVersion())) {
             out.writeVInt(License.VERSION_CURRENT);
         }
     }
 
-    private static boolean hasLicenseVersionField(Version streamVersion) {
-        return streamVersion.onOrAfter(Version.V_7_8_1) && streamVersion.before(Version.V_8_0_0);
+    private static boolean hasLicenseVersionField(TransportVersion streamVersion) {
+        return streamVersion.onOrAfter(TransportVersion.V_7_8_1) && streamVersion.before(TransportVersion.V_8_0_0);
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java
index 05fffeca9cc2..820181c5c321 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.protocol.xpack;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -362,7 +362,7 @@ public FeatureSet(String name, boolean available, boolean enabled) {
 
             public FeatureSet(StreamInput in) throws IOException {
                 this(in.readString(), readAvailable(in), in.readBoolean());
-                if (in.getVersion().before(Version.V_8_0_0)) {
+                if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                     in.readMap(); // backcompat reading native code info, but no longer used here
                 }
             }
@@ -370,7 +370,7 @@ public FeatureSet(StreamInput in) throws IOException {
             // this is separated out so that the removed description can be read from the stream on construction
             // TODO: remove this for 8.0
             private static boolean readAvailable(StreamInput in) throws IOException {
-                if (in.getVersion().before(Version.V_7_3_0)) {
+                if (in.getTransportVersion().before(TransportVersion.V_7_3_0)) {
                     in.readOptionalString();
                 }
                 return in.readBoolean();
@@ -379,12 +379,12 @@ private static boolean readAvailable(StreamInput in) throws IOException {
             @Override
             public void writeTo(StreamOutput out) throws IOException {
                 out.writeString(name);
-                if (out.getVersion().before(Version.V_7_3_0)) {
+                if (out.getTransportVersion().before(TransportVersion.V_7_3_0)) {
                     out.writeOptionalString(null);
                 }
                 out.writeBoolean(available);
                 out.writeBoolean(enabled);
-                if (out.getVersion().before(Version.V_8_0_0)) {
+                if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                     out.writeGenericMap(Collections.emptyMap());
                 }
             }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/graph/GraphExploreRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/graph/GraphExploreRequest.java
index a2f73fcab266..94604dbd8227 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/graph/GraphExploreRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/graph/GraphExploreRequest.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.protocol.xpack.graph;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
@@ -111,7 +111,7 @@ public GraphExploreRequest(StreamInput in) throws IOException {
 
         indices = in.readStringArray();
         indicesOptions = IndicesOptions.readIndicesOptions(in);
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             String[] types = in.readStringArray();
             assert types.length == 0;
         }
@@ -180,7 +180,7 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeStringArray(indices);
         indicesOptions.writeIndicesOptions(out);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             out.writeStringArray(Strings.EMPTY_ARRAY);
         }
         out.writeOptionalString(routing);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotRepository.java
index 94e01e79d2d5..c0272fe91237 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotRepository.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotRepository.java
@@ -209,7 +209,6 @@ protected void closeInternal() {
                     context.stateIdentifier(),
                     context.status(),
                     context.getRepositoryMetaVersion(),
-                    context.userMetadata(),
                     context.snapshotStartTime(),
                     context
                 )
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersResponse.java
index 63acca783980..a89e0f561484 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersResponse.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.cluster.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -63,7 +63,7 @@ public MigrateToDataTiersResponse(StreamInput in) throws IOException {
         migratedPolicies = in.readStringList();
         migratedIndices = in.readStringList();
         dryRun = in.readBoolean();
-        if (in.getVersion().onOrAfter(Version.V_7_17_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_17_0)) {
             migratedLegacyTemplates = in.readStringList();
             migratedComposableTemplates = in.readStringList();
             migratedComponentTemplates = in.readStringList();
@@ -154,7 +154,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeStringCollection(migratedPolicies);
         out.writeStringCollection(migratedIndices);
         out.writeBoolean(dryRun);
-        if (out.getVersion().onOrAfter(Version.V_7_17_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_17_0)) {
             out.writeStringCollection(migratedLegacyTemplates);
             out.writeStringCollection(migratedComposableTemplates);
             out.writeStringCollection(migratedComponentTemplates);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
index 59ff0e2c53cd..b682b67c5b06 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
@@ -93,7 +93,7 @@ public static Map<String, String> getPersistableSafeSecurityHeaders(ThreadContex
         return maybeRewriteAuthenticationHeadersForVersion(
             filterSecurityHeaders(threadContext.getHeaders()),
             key -> new AuthenticationContextSerializer(key).readFromContext(threadContext),
-            clusterState.nodes().getMinNodeVersion()
+            clusterState.nodes().getMinNodeVersion().transportVersion // TODO revisit this once node's version is refactored
         );
     }
 
@@ -109,14 +109,14 @@ public static Map<String, String> getPersistableSafeSecurityHeaders(Map<String,
         return maybeRewriteAuthenticationHeadersForVersion(
             filterSecurityHeaders(headers),
             authenticationReader,
-            clusterState.nodes().getMinNodeVersion()
+            clusterState.nodes().getMinNodeVersion().transportVersion // TODO revisit this once node's version is refactored
         );
     }
 
     private static Map<String, String> maybeRewriteAuthenticationHeadersForVersion(
         Map<String, String> filteredHeaders,
         CheckedFunction<String, Authentication, IOException> authenticationReader,
-        Version minNodeVersion
+        TransportVersion minNodeVersion
     ) {
         Map<String, String> newHeaders = null;
 
@@ -154,11 +154,11 @@ private static Map<String, String> maybeRewriteAuthenticationHeadersForVersion(
     private static String maybeRewriteSingleAuthenticationHeaderForVersion(
         CheckedFunction<String, Authentication, IOException> authenticationReader,
         String authenticationHeaderKey,
-        Version minNodeVersion
+        TransportVersion minNodeVersion
     ) {
         try {
             final Authentication authentication = authenticationReader.apply(authenticationHeaderKey);
-            if (authentication != null && authentication.getEffectiveSubject().getVersion().after(minNodeVersion)) {
+            if (authentication != null && authentication.getEffectiveSubject().getTransportVersion().after(minNodeVersion)) {
                 return authentication.maybeRewriteForOlderVersion(minNodeVersion).encode();
             }
         } catch (IOException e) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersFeatureSetUsage.java
index 70be500a59a8..ec3d9f263111 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/DataTiersFeatureSetUsage.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -41,8 +41,8 @@ public DataTiersFeatureSetUsage(Map<String, TierSpecificStats> tierStats) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_10_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_10_0;
     }
 
     public Map<String, TierSpecificStats> getTierStats() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/HealthApiFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/HealthApiFeatureSetUsage.java
new file mode 100644
index 000000000000..740ae58beef3
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/HealthApiFeatureSetUsage.java
@@ -0,0 +1,157 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.core;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.metrics.Counters;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+/**
+ * Models the health api usage section in the XPack usage response. A sample response would look like this:
+ * {
+ *   "enabled": true,
+ *   "available": true,
+ *   "invocations": {
+ *     "total": 22,
+ *     "verbose_true": 12,
+ *     "verbose_false": 10
+ *   },
+ *   "statuses": {
+ *     "green": 10,
+ *     "yellow": 4,
+ *     "red": 8,
+ *     "values": ["green", "yellow", "red"]
+ *   },
+ *   "indicators": {
+ *     "red" : {
+ *       "master_stability": 2,
+ *       "ilm":2,
+ *       "slm": 4,
+ *       "values": ["master_stability", "ilm", "slm"]
+ *     },
+ *     "yellow": {
+ *       "disk": 1,
+ *       "shards_availability": 1,
+ *       "master_stability": 2,
+ *       "values": ["disk", "shards_availability", "master_stability"]
+ *     }
+ *   },
+ *   "diagnoses": {
+ *     "red": {
+ *       "elasticsearch:health:shards_availability:primary_unassigned": 1,
+ *       "elasticsearch:health:disk:add_disk_capacity_master_nodes": 3,
+ *       "values": [
+ *         "elasticsearch:health:shards_availability:primary_unassigned",
+ *         "elasticsearch:health:disk:add_disk_capacity_master_nodes"
+ *       ]
+ *     },
+ *     "yellow": {
+ *       "elasticsearch:health:disk:add_disk_capacity_data_nodes": 1,
+ *       "values": [""elasticsearch:health:disk:add_disk_capacity_data_nodes"]
+ *     }
+ *   }
+ * }
+
+ * Note: If the minimum version of the cluster is not after 8.7.0 then the response will look like this:
+ * {
+ *   "available": false,
+ *   "enabled": true
+ * }
+ */
+public class HealthApiFeatureSetUsage extends XPackFeatureSet.Usage {
+
+    private final Map<String, Object> usageStats;
+
+    public HealthApiFeatureSetUsage(StreamInput in) throws IOException {
+        super(in);
+        usageStats = in.readMap();
+    }
+
+    public HealthApiFeatureSetUsage(boolean available, boolean enabled, @Nullable Counters stats) {
+        super(XPackField.HEALTH_API, available, enabled);
+        if (stats != null) {
+            usageStats = stats.toMutableNestedMap();
+            enrichUsageStatsWithValues(usageStats);
+        } else {
+            usageStats = Map.of();
+        }
+    }
+
+    // This method enriches the stats map with a list of encountered values for the statuses, the indicators and the diagnoses stats.
+    // Visible for testing
+    @SuppressWarnings("unchecked")
+    static void enrichUsageStatsWithValues(Map<String, Object> usageStats) {
+        if (usageStats.containsKey("statuses")) {
+            Map<String, Object> statuses = (Map<String, Object>) usageStats.get("statuses");
+            if (statuses.isEmpty() == false) {
+                statuses.put("values", statuses.keySet().stream().sorted().collect(Collectors.toList()));
+            }
+        }
+        if (usageStats.containsKey("indicators")) {
+            Map<String, Map<String, Object>> indicatorsByStatus = (Map<String, Map<String, Object>>) usageStats.get("indicators");
+            for (String status : indicatorsByStatus.keySet()) {
+                Map<String, Object> indicators = indicatorsByStatus.get(status);
+                if (indicators.isEmpty() == false) {
+                    indicators.put("values", indicators.keySet().stream().sorted().collect(Collectors.toList()));
+                }
+            }
+        }
+        if (usageStats.containsKey("diagnoses")) {
+            Map<String, Map<String, Object>> diagnosesByStatus = (Map<String, Map<String, Object>>) usageStats.get("diagnoses");
+            for (String status : diagnosesByStatus.keySet()) {
+                Map<String, Object> diagnoses = diagnosesByStatus.get(status);
+                if (diagnoses.isEmpty() == false) {
+                    diagnoses.put("values", diagnoses.keySet().stream().sorted().collect(Collectors.toList()));
+                }
+            }
+        }
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_7_0;
+    }
+
+    public Map<String, Object> stats() {
+        return usageStats;
+    }
+
+    @Override
+    protected void innerXContent(XContentBuilder builder, Params params) throws IOException {
+        super.innerXContent(builder, params);
+        for (Map.Entry<String, Object> entry : usageStats.entrySet()) {
+            builder.field(entry.getKey(), entry.getValue());
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeGenericMap(usageStats);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        HealthApiFeatureSetUsage that = (HealthApiFeatureSetUsage) o;
+        return Objects.equals(usageStats, that.usageStats);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(usageStats);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/HealthApiUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/HealthApiUsageTransportAction.java
new file mode 100644
index 000000000000..a2df29141546
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/HealthApiUsageTransportAction.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.core;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.ContextPreservingActionListener;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.health.stats.HealthApiStatsAction;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
+
+/**
+ * This action provides telemetry of the cluster's health api usage.
+ */
+public class HealthApiUsageTransportAction extends XPackUsageFeatureTransportAction {
+    private final Client client;
+
+    @Inject
+    public HealthApiUsageTransportAction(
+        TransportService transportService,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        ActionFilters actionFilters,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        Client client
+    ) {
+        super(
+            XPackUsageFeatureAction.HEALTH.name(),
+            transportService,
+            clusterService,
+            threadPool,
+            actionFilters,
+            indexNameExpressionResolver
+        );
+        this.client = client;
+    }
+
+    @Override
+    protected void masterOperation(
+        Task task,
+        XPackUsageRequest request,
+        ClusterState state,
+        ActionListener<XPackUsageFeatureResponse> listener
+    ) {
+
+        ActionListener<XPackUsageFeatureResponse> preservingListener = ContextPreservingActionListener.wrapPreservingContext(
+            listener,
+            client.threadPool().getThreadContext()
+        );
+
+        if (state.nodesIfRecovered().getMinNodeVersion().onOrAfter(Version.V_8_7_0)) {
+            HealthApiStatsAction.Request statsRequest = new HealthApiStatsAction.Request();
+            statsRequest.setParentTask(clusterService.localNode().getId(), task.getId());
+            client.execute(HealthApiStatsAction.INSTANCE, statsRequest, ActionListener.wrap(r -> {
+                HealthApiFeatureSetUsage usage = new HealthApiFeatureSetUsage(true, true, r.getStats());
+                preservingListener.onResponse(new XPackUsageFeatureResponse(usage));
+            }, preservingListener::onFailure));
+        } else {
+            HealthApiFeatureSetUsage usage = new HealthApiFeatureSetUsage(false, true, null);
+            preservingListener.onResponse(new XPackUsageFeatureResponse(usage));
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
index de2475994139..64d92124d351 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
@@ -55,6 +55,7 @@
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleFeatureSetUsage;
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.ilm.LifecycleAction;
+import org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata;
 import org.elasticsearch.xpack.core.ilm.LifecycleType;
 import org.elasticsearch.xpack.core.ilm.MigrateAction;
 import org.elasticsearch.xpack.core.ilm.ReadOnlyAction;
@@ -151,6 +152,7 @@
 import org.elasticsearch.xpack.core.rollup.action.GetRollupJobsAction;
 import org.elasticsearch.xpack.core.rollup.action.PutRollupJobAction;
 import org.elasticsearch.xpack.core.rollup.action.RollupSearchAction;
+import org.elasticsearch.xpack.core.rollup.action.RollupShardStatus;
 import org.elasticsearch.xpack.core.rollup.action.StartRollupJobAction;
 import org.elasticsearch.xpack.core.rollup.action.StopRollupJobAction;
 import org.elasticsearch.xpack.core.rollup.job.RollupJob;
@@ -460,6 +462,7 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
             new NamedWriteableRegistry.Entry(PersistentTaskParams.class, RollupJob.NAME, RollupJob::new),
             new NamedWriteableRegistry.Entry(Task.Status.class, RollupJobStatus.NAME, RollupJobStatus::new),
             new NamedWriteableRegistry.Entry(PersistentTaskState.class, RollupJobStatus.NAME, RollupJobStatus::new),
+            new NamedWriteableRegistry.Entry(Task.Status.class, RollupShardStatus.NAME, RollupShardStatus::new),
             // ccr
             new NamedWriteableRegistry.Entry(AutoFollowMetadata.class, AutoFollowMetadata.TYPE, AutoFollowMetadata::new),
             new NamedWriteableRegistry.Entry(Metadata.Custom.class, AutoFollowMetadata.TYPE, AutoFollowMetadata::new),
@@ -479,6 +482,12 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 IndexLifecycleMetadata.TYPE,
                 IndexLifecycleMetadata.IndexLifecycleMetadataDiff::new
             ),
+            new NamedWriteableRegistry.Entry(Metadata.Custom.class, LifecycleOperationMetadata.TYPE, LifecycleOperationMetadata::new),
+            new NamedWriteableRegistry.Entry(
+                NamedDiff.class,
+                LifecycleOperationMetadata.TYPE,
+                LifecycleOperationMetadata.LifecycleOperationMetadataDiff::new
+            ),
             new NamedWriteableRegistry.Entry(Metadata.Custom.class, SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata::new),
             new NamedWriteableRegistry.Entry(
                 NamedDiff.class,
@@ -544,7 +553,9 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
             // Archive
             new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ARCHIVE, ArchiveFeatureSetUsage::new),
             // TSDB Downsampling
-            new NamedWriteableRegistry.Entry(LifecycleAction.class, DownsampleAction.NAME, DownsampleAction::new)
+            new NamedWriteableRegistry.Entry(LifecycleAction.class, DownsampleAction.NAME, DownsampleAction::new),
+            // Health API usage
+            new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.HEALTH_API, HealthApiFeatureSetUsage::new)
         );
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java
index dbc100e62ac1..bb1a8aa97be7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java
@@ -73,6 +73,8 @@ public final class XPackField {
     public static final String OPERATOR_PRIVILEGES = "operator_privileges";
     /** Name constant for the archive feature. */
     public static final String ARCHIVE = "archive";
+    /** Name constant for the health api feature. */
+    public static final String HEALTH_API = "health_api";
 
     private XPackField() {}
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java
index 1e09cb33219f..dd4e0788bd6a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java
@@ -19,9 +19,9 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.logging.DeprecationCategory;
@@ -307,7 +307,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver expressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         List<Object> components = new ArrayList<>();
 
@@ -339,6 +339,7 @@ public Collection<Object> createComponents(
         actions.add(new ActionHandler<>(XPackUsageFeatureAction.DATA_TIERS, DataTiersUsageTransportAction.class));
         actions.add(new ActionHandler<>(XPackUsageFeatureAction.DATA_STREAMS, DataStreamUsageTransportAction.class));
         actions.add(new ActionHandler<>(XPackInfoFeatureAction.DATA_STREAMS, DataStreamInfoTransportAction.class));
+        actions.add(new ActionHandler<>(XPackUsageFeatureAction.HEALTH, HealthApiUsageTransportAction.class));
         return actions;
     }
 
@@ -467,6 +468,7 @@ public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(Index
      */
     private SSLService createSSLService(Environment environment, ResourceWatcherService resourceWatcherService) {
         final Map<String, SslConfiguration> sslConfigurations = SSLService.getSSLConfigurations(environment);
+        // Must construct the reloader before the SSL service so that we don't miss any config changes, see #54867
         final SSLConfigurationReloader reloader = new SSLConfigurationReloader(resourceWatcherService, sslConfigurations.values());
         final SSLService sslService = new SSLService(environment, sslConfigurations);
         reloader.setSSLService(sslService);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
index 0e9bdf8b824a..8e2039efe241 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
@@ -13,6 +13,8 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.ssl.SslClientAuthenticationMode;
 import org.elasticsearch.common.ssl.SslVerificationMode;
+import org.elasticsearch.transport.RemoteClusterPortSettings;
+import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.xpack.core.security.SecurityField;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings;
@@ -98,6 +100,13 @@ private XPackSettings() {
         Setting.Property.NodeScope
     );
 
+    /** Setting for enabling or disabling remote cluster access TLS. Defaults to true. */
+    public static final Setting<Boolean> REMOTE_CLUSTER_SSL_ENABLED = Setting.boolSetting(
+        "xpack.security." + RemoteClusterPortSettings.REMOTE_CLUSTER_PREFIX + "ssl.enabled",
+        true,
+        Property.NodeScope
+    );
+
     /** Setting for enabling or disabling the reserved realm. Defaults to true */
     public static final Setting<Boolean> RESERVED_REALM_ENABLED_SETTING = Setting.boolSetting(
         "xpack.security.authc.reserved_realm.enabled",
@@ -235,6 +244,7 @@ public static Setting<String> defaultStoredHashAlgorithmSetting(String key, Func
 
     public static final SslClientAuthenticationMode CLIENT_AUTH_DEFAULT = SslClientAuthenticationMode.REQUIRED;
     public static final SslClientAuthenticationMode HTTP_CLIENT_AUTH_DEFAULT = SslClientAuthenticationMode.NONE;
+    public static final SslClientAuthenticationMode REMOTE_CLUSTER_CLIENT_AUTH_DEFAULT = SslClientAuthenticationMode.NONE;
     public static final SslVerificationMode VERIFICATION_MODE_DEFAULT = SslVerificationMode.FULL;
 
     // http specific settings
@@ -245,11 +255,21 @@ public static Setting<String> defaultStoredHashAlgorithmSetting(String key, Func
     public static final String TRANSPORT_SSL_PREFIX = SecurityField.setting("transport.ssl.");
     private static final SSLConfigurationSettings TRANSPORT_SSL = SSLConfigurationSettings.withPrefix(TRANSPORT_SSL_PREFIX, true);
 
+    public static final String REMOTE_CLUSTER_SSL_PREFIX = SecurityField.setting(RemoteClusterPortSettings.REMOTE_CLUSTER_PREFIX + "ssl.");
+
+    private static final SSLConfigurationSettings REMOTE_CLUSTER_SSL = SSLConfigurationSettings.withPrefix(
+        REMOTE_CLUSTER_SSL_PREFIX,
+        false
+    );
+
     /** Returns all settings created in {@link XPackSettings}. */
     public static List<Setting<?>> getAllSettings() {
         ArrayList<Setting<?>> settings = new ArrayList<>();
         settings.addAll(HTTP_SSL.getEnabledSettings());
         settings.addAll(TRANSPORT_SSL.getEnabledSettings());
+        if (TcpTransport.isUntrustedRemoteClusterEnabled()) {
+            settings.addAll(REMOTE_CLUSTER_SSL.getEnabledSettings());
+        }
         settings.add(SECURITY_ENABLED);
         settings.add(GRAPH_ENABLED);
         settings.add(MACHINE_LEARNING_ENABLED);
@@ -258,6 +278,9 @@ public static List<Setting<?>> getAllSettings() {
         settings.add(DLS_FLS_ENABLED);
         settings.add(TRANSPORT_SSL_ENABLED);
         settings.add(HTTP_SSL_ENABLED);
+        if (TcpTransport.isUntrustedRemoteClusterEnabled()) {
+            settings.add(REMOTE_CLUSTER_SSL_ENABLED);
+        }
         settings.add(RESERVED_REALM_ENABLED_SETTING);
         settings.add(TOKEN_SERVICE_ENABLED_SETTING);
         settings.add(API_KEY_SERVICE_ENABLED_SETTING);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportReloadAnalyzersAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportReloadAnalyzersAction.java
index 9e37b9a77135..aa4e8a324615 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportReloadAnalyzersAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportReloadAnalyzersAction.java
@@ -10,7 +10,6 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
@@ -81,28 +80,24 @@ protected ReloadResult readShardResult(StreamInput in) throws IOException {
     }
 
     @Override
-    protected ReloadAnalyzersResponse newResponse(
+    protected ResponseFactory<ReloadAnalyzersResponse, ReloadResult> getResponseFactory(
         ReloadAnalyzersRequest request,
-        int totalShards,
-        int successfulShards,
-        int failedShards,
-        List<ReloadResult> responses,
-        List<DefaultShardOperationFailedException> shardFailures,
         ClusterState clusterState
     ) {
-        Map<String, ReloadDetails> reloadedIndicesDetails = new HashMap<String, ReloadDetails>();
-        for (ReloadResult result : responses) {
-            if (reloadedIndicesDetails.containsKey(result.index)) {
-                reloadedIndicesDetails.get(result.index).merge(result);
-                ;
-            } else {
-                HashSet<String> nodeIds = new HashSet<String>();
-                nodeIds.add(result.nodeId);
-                ReloadDetails details = new ReloadDetails(result.index, nodeIds, new HashSet<String>(result.reloadedSearchAnalyzers));
-                reloadedIndicesDetails.put(result.index, details);
+        return (totalShards, successfulShards, failedShards, responses, shardFailures) -> {
+            Map<String, ReloadDetails> reloadedIndicesDetails = new HashMap<>();
+            for (ReloadResult result : responses) {
+                if (reloadedIndicesDetails.containsKey(result.index)) {
+                    reloadedIndicesDetails.get(result.index).merge(result);
+                } else {
+                    HashSet<String> nodeIds = new HashSet<>();
+                    nodeIds.add(result.nodeId);
+                    ReloadDetails details = new ReloadDetails(result.index, nodeIds, new HashSet<>(result.reloadedSearchAnalyzers));
+                    reloadedIndicesDetails.put(result.index, details);
+                }
             }
-        }
-        return new ReloadAnalyzersResponse(totalShards, successfulShards, failedShards, shardFailures, reloadedIndicesDetails);
+            return new ReloadAnalyzersResponse(totalShards, successfulShards, failedShards, shardFailures, reloadedIndicesDetails);
+        };
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java
index bfbac109012e..45220b884445 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java
@@ -12,10 +12,10 @@
 import java.util.List;
 
 /**
- * A base action for for usage of a feature plugin.
+ * A base action for usage of a feature plugin.
  *
  * This action is implemented by each feature plugin, bound to the public constants here. The
- * {@link XPackUsageAction} implementationn iterates over the {@link #ALL} list of actions to form
+ * {@link XPackUsageAction} implementation iterates over the {@link #ALL} list of actions to form
  * the complete usage result.
  */
 public class XPackUsageFeatureAction extends ActionType<XPackUsageFeatureResponse> {
@@ -45,6 +45,7 @@ public class XPackUsageFeatureAction extends ActionType<XPackUsageFeatureRespons
     public static final XPackUsageFeatureAction DATA_TIERS = new XPackUsageFeatureAction(XPackField.DATA_TIERS);
     public static final XPackUsageFeatureAction AGGREGATE_METRIC = new XPackUsageFeatureAction(XPackField.AGGREGATE_METRIC);
     public static final XPackUsageFeatureAction ARCHIVE = new XPackUsageFeatureAction(XPackField.ARCHIVE);
+    public static final XPackUsageFeatureAction HEALTH = new XPackUsageFeatureAction(XPackField.HEALTH_API);
 
     static final List<XPackUsageFeatureAction> ALL = List.of(
         AGGREGATE_METRIC,
@@ -68,7 +69,8 @@ public class XPackUsageFeatureAction extends ActionType<XPackUsageFeatureRespons
         TRANSFORM,
         VOTING_ONLY,
         WATCHER,
-        ARCHIVE
+        ARCHIVE,
+        HEALTH
     );
 
     // public for testing
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageResponse.java
index ec101b19908b..169cc4f4009e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageResponse.java
@@ -35,7 +35,7 @@ public List<XPackFeatureSet.Usage> getUsages() {
     public void writeTo(final StreamOutput out) throws IOException {
         // we can only write the usages with version the coordinating node is compatible with otherwise it will not know the named writeable
         final List<XPackFeatureSet.Usage> usagesToWrite = usages.stream()
-            .filter(usage -> out.getVersion().onOrAfter(usage.getMinimalSupportedVersion()))
+            .filter(usage -> out.getTransportVersion().onOrAfter(usage.getMinimalSupportedVersion()))
             .toList();
         writeTo(out, usagesToWrite);
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/aggregatemetric/AggregateMetricFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/aggregatemetric/AggregateMetricFeatureSetUsage.java
index b02601a8286d..441e62326640 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/aggregatemetric/AggregateMetricFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/aggregatemetric/AggregateMetricFeatureSetUsage.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.aggregatemetric;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
 import org.elasticsearch.xpack.core.XPackField;
@@ -26,8 +26,8 @@ public AggregateMetricFeatureSetUsage(boolean available, boolean enabled) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_11_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_11_0;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/AnalyticsFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/AnalyticsFeatureSetUsage.java
index 1593e5d54821..4c5b198a7565 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/AnalyticsFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/AnalyticsFeatureSetUsage.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.analytics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -45,8 +45,9 @@ protected void innerXContent(XContentBuilder builder, Params params) throws IOEx
         }
     }
 
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_4_0;
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_4_0;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/action/AnalyticsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/action/AnalyticsStatsAction.java
index 4d9d6b9824cd..dd0ed88d5f13 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/action/AnalyticsStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/action/AnalyticsStatsAction.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.analytics.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.nodes.BaseNodeResponse;
@@ -144,15 +144,15 @@ public NodeResponse(DiscoveryNode node, EnumCounters<Item> counters) {
 
         public NodeResponse(StreamInput in) throws IOException {
             super(in);
-            if (in.getVersion().onOrAfter(Version.V_7_8_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_8_0)) {
                 counters = new EnumCounters<>(in, Item.class);
             } else {
                 counters = new EnumCounters<>(Item.class);
-                if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
+                if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_7_0)) {
                     counters.inc(Item.BOXPLOT, in.readVLong());
                 }
                 counters.inc(Item.CUMULATIVE_CARDINALITY, in.readZLong());
-                if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
+                if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_7_0)) {
                     counters.inc(Item.STRING_STATS, in.readVLong());
                     counters.inc(Item.TOP_METRICS, in.readVLong());
                 }
@@ -162,14 +162,14 @@ public NodeResponse(StreamInput in) throws IOException {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            if (out.getVersion().onOrAfter(Version.V_7_8_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_8_0)) {
                 counters.writeTo(out);
             } else {
-                if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
+                if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_7_0)) {
                     out.writeVLong(counters.get(Item.BOXPLOT));
                 }
                 out.writeZLong(counters.get(Item.CUMULATIVE_CARDINALITY));
-                if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
+                if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_7_0)) {
                     out.writeVLong(counters.get(Item.STRING_STATS));
                     out.writeVLong(counters.get(Item.TOP_METRICS));
                 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/archive/ArchiveFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/archive/ArchiveFeatureSetUsage.java
index 543a16938483..79a386ce2147 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/archive/ArchiveFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/archive/ArchiveFeatureSetUsage.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.archive;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.ToXContent;
@@ -28,8 +28,8 @@ public ArchiveFeatureSetUsage(StreamInput input) throws IOException {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_3_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_3_0;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java
index d74be00ef18f..5a04b3d9be13 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java
@@ -7,18 +7,21 @@
 package org.elasticsearch.xpack.core.ccr;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.AbstractNamedDiffable;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
@@ -28,6 +31,7 @@
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -131,8 +135,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.CURRENT.minimumCompatibilityVersion();
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.CURRENT.minimumCompatibilityVersion();
     }
 
     @Override
@@ -147,31 +151,12 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(PATTERNS_FIELD.getPreferredName());
-        for (Map.Entry<String, AutoFollowPattern> entry : patterns.entrySet()) {
-            builder.startObject(entry.getKey());
-            builder.value(entry.getValue());
-            builder.endObject();
-        }
-        builder.endObject();
-
-        builder.startObject(FOLLOWED_LEADER_INDICES_FIELD.getPreferredName());
-        for (Map.Entry<String, List<String>> entry : followedLeaderIndexUUIDs.entrySet()) {
-            builder.field(entry.getKey(), entry.getValue());
-        }
-        builder.endObject();
-        builder.startObject(HEADERS.getPreferredName());
-        for (Map.Entry<String, Map<String, String>> entry : headers.entrySet()) {
-            builder.field(entry.getKey(), entry.getValue());
-        }
-        builder.endObject();
-        return builder;
-    }
-
-    @Override
-    public boolean isFragment() {
-        return true;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return Iterators.concat(
+            ChunkedToXContentHelper.xContentFragmentValuesMap(PATTERNS_FIELD.getPreferredName(), patterns),
+            ChunkedToXContentHelper.map(FOLLOWED_LEADER_INDICES_FIELD.getPreferredName(), followedLeaderIndexUUIDs),
+            ChunkedToXContentHelper.map(HEADERS.getPreferredName(), headers)
+        );
     }
 
     @Override
@@ -281,7 +266,7 @@ public static AutoFollowPattern readFrom(StreamInput in) throws IOException {
             final List<String> leaderIndexPatterns = in.readStringList();
             final String followIndexPattern = in.readOptionalString();
             final Settings settings;
-            if (in.getVersion().onOrAfter(Version.V_7_9_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_9_0)) {
                 settings = Settings.readSettingsFromStream(in);
             } else {
                 settings = Settings.EMPTY;
@@ -301,12 +286,12 @@ private AutoFollowPattern(
             this.leaderIndexPatterns = leaderIndexPatterns;
             this.followIndexPattern = followIndexPattern;
             this.settings = Objects.requireNonNull(settings);
-            if (in.getVersion().onOrAfter(Version.V_7_5_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_5_0)) {
                 this.active = in.readBoolean();
             } else {
                 this.active = true;
             }
-            if (in.getVersion().onOrAfter(Version.V_7_14_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_14_0)) {
                 this.leaderIndexExclusionPatterns = in.readStringList();
             } else {
                 this.leaderIndexExclusionPatterns = Collections.emptyList();
@@ -366,14 +351,14 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(remoteCluster);
             out.writeStringCollection(leaderIndexPatterns);
             out.writeOptionalString(followIndexPattern);
-            if (out.getVersion().onOrAfter(Version.V_7_9_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_9_0)) {
                 settings.writeTo(out);
             }
             super.writeTo(out);
-            if (out.getVersion().onOrAfter(Version.V_7_5_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_5_0)) {
                 out.writeBoolean(active);
             }
-            if (out.getVersion().onOrAfter(Version.V_7_14_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_14_0)) {
                 out.writeStringCollection(leaderIndexExclusionPatterns);
             }
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/ShardFollowNodeTaskStatus.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/ShardFollowNodeTaskStatus.java
index c0a926f1a0ae..4be42f37a9cd 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/ShardFollowNodeTaskStatus.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/ShardFollowNodeTaskStatus.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.xpack.core.ccr;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -413,7 +413,7 @@ public ShardFollowNodeTaskStatus(final StreamInput in) throws IOException {
         this.writeBufferSizeInBytes = in.readVLong();
         this.followerMappingVersion = in.readVLong();
         this.followerSettingsVersion = in.readVLong();
-        if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_3_0)) {
             this.followerAliasesVersion = in.readVLong();
         } else {
             this.followerAliasesVersion = 0L;
@@ -457,7 +457,7 @@ public void writeTo(final StreamOutput out) throws IOException {
         out.writeVLong(writeBufferSizeInBytes);
         out.writeVLong(followerMappingVersion);
         out.writeVLong(followerSettingsVersion);
-        if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_3_0)) {
             out.writeVLong(followerAliasesVersion);
         }
         out.writeVLong(totalReadTimeMillis);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java
index 1c9fb93b8f11..225c784ef143 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java
@@ -11,13 +11,16 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xpack.core.ccr.AutoFollowStats;
 
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.Objects;
 
 public class CcrStatsAction extends ActionType<CcrStatsAction.Response> {
@@ -48,7 +51,7 @@ public void writeTo(StreamOutput out) throws IOException {
         }
     }
 
-    public static class Response extends ActionResponse implements ToXContentObject {
+    public static class Response extends ActionResponse implements ChunkedToXContentObject {
 
         private final AutoFollowStats autoFollowStats;
         private final FollowStatsAction.StatsResponses followStats;
@@ -79,14 +82,14 @@ public void writeTo(StreamOutput out) throws IOException {
         }
 
         @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            {
-                builder.field("auto_follow_stats", autoFollowStats, params);
-                builder.field("follow_stats", followStats, params);
-            }
-            builder.endObject();
-            return builder;
+        public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerParams) {
+            return Iterators.concat(
+                Iterators.single(
+                    (builder, params) -> builder.startObject().field("auto_follow_stats", autoFollowStats, params).field("follow_stats")
+                ),
+                followStats.toXContentChunked(outerParams),
+                ChunkedToXContentHelper.endObject()
+            );
         }
 
         @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowInfoAction.java
index 5c37d2868468..037e413c89b4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowInfoAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowInfoAction.java
@@ -11,15 +11,19 @@
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
 
@@ -77,7 +81,7 @@ public int hashCode() {
         }
     }
 
-    public static class Response extends ActionResponse implements ToXContentObject {
+    public static class Response extends ActionResponse implements ChunkedToXContentObject {
 
         public static final ParseField FOLLOWER_INDICES_FIELD = new ParseField("follower_indices");
 
@@ -102,15 +106,12 @@ public void writeTo(StreamOutput out) throws IOException {
         }
 
         @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            builder.startArray(FOLLOWER_INDICES_FIELD.getPreferredName());
-            for (FollowerInfo followInfo : followInfos) {
-                followInfo.toXContent(builder, params);
-            }
-            builder.endArray();
-            builder.endObject();
-            return builder;
+        public Iterator<ToXContent> toXContentChunked(ToXContent.Params outerParams) {
+            return Iterators.concat(
+                Iterators.single((builder, params) -> builder.startObject().startArray(FOLLOWER_INDICES_FIELD.getPreferredName())),
+                followInfos.iterator(),
+                Iterators.single((builder, params) -> builder.endArray().endObject())
+            );
         }
 
         @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowStatsAction.java
index 1b0a67ee1a78..ac7d27d7f28b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowStatsAction.java
@@ -15,16 +15,21 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.tasks.BaseTasksRequest;
 import org.elasticsearch.action.support.tasks.BaseTasksResponse;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.Transports;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.ccr.ShardFollowNodeTaskStatus;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -40,7 +45,7 @@ private FollowStatsAction() {
         super(NAME, FollowStatsAction.StatsResponses::new);
     }
 
-    public static class StatsResponses extends BaseTasksResponse implements ToXContentObject {
+    public static class StatsResponses extends BaseTasksResponse implements ChunkedToXContentObject {
 
         private final List<StatsResponse> statsResponse;
 
@@ -69,36 +74,32 @@ public void writeTo(StreamOutput out) throws IOException {
         }
 
         @Override
-        public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
+        public Iterator<ToXContent> toXContentChunked(ToXContent.Params outerParams) {
             // sort by index name, then shard ID
+            Transports.assertNotTransportThread("collecting responses into a map may be expensive");
             final Map<String, Map<Integer, StatsResponse>> taskResponsesByIndex = new TreeMap<>();
             for (final StatsResponse response : statsResponse) {
                 taskResponsesByIndex.computeIfAbsent(response.status().followerIndex(), k -> new TreeMap<>())
                     .put(response.status().getShardId(), response);
             }
-            builder.startObject();
-            {
-                builder.startArray("indices");
-                {
-                    for (final Map.Entry<String, Map<Integer, StatsResponse>> index : taskResponsesByIndex.entrySet()) {
-                        builder.startObject();
-                        {
-                            builder.field("index", index.getKey());
-                            builder.startArray("shards");
-                            {
-                                for (final Map.Entry<Integer, StatsResponse> shard : index.getValue().entrySet()) {
-                                    shard.getValue().status().toXContent(builder, params);
-                                }
-                            }
-                            builder.endArray();
-                        }
-                        builder.endObject();
-                    }
-                }
-                builder.endArray();
-            }
-            builder.endObject();
-            return builder;
+            return innerToXContentChunked(taskResponsesByIndex);
+        }
+
+        private static Iterator<ToXContent> innerToXContentChunked(Map<String, Map<Integer, StatsResponse>> taskResponsesByIndex) {
+            return Iterators.concat(
+                Iterators.single((builder, params) -> builder.startObject().startArray("indices")),
+                Iterators.flatMap(
+                    taskResponsesByIndex.entrySet().iterator(),
+                    indexEntry -> Iterators.concat(
+                        Iterators.<ToXContent>single(
+                            (builder, params) -> builder.startObject().field("index", indexEntry.getKey()).startArray("shards")
+                        ),
+                        indexEntry.getValue().values().iterator(),
+                        Iterators.single((builder, params) -> builder.endArray().endObject())
+                    )
+                ),
+                Iterators.single((builder, params) -> builder.endArray().endObject())
+            );
         }
 
         @Override
@@ -178,7 +179,7 @@ public int hashCode() {
         }
     }
 
-    public static class StatsResponse implements Writeable {
+    public static class StatsResponse implements Writeable, ToXContentObject {
 
         private final ShardFollowNodeTaskStatus status;
 
@@ -211,6 +212,11 @@ public boolean equals(Object o) {
         public int hashCode() {
             return Objects.hash(status);
         }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+            return status.toXContent(builder, params);
+        }
     }
 
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java
index a7080131001e..46ed48512d2e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ccr.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
@@ -185,11 +185,11 @@ public Request(StreamInput in) throws IOException {
             remoteCluster = in.readString();
             leaderIndexPatterns = in.readStringList();
             followIndexNamePattern = in.readOptionalString();
-            if (in.getVersion().onOrAfter(Version.V_7_9_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_9_0)) {
                 settings = Settings.readSettingsFromStream(in);
             }
             parameters = new FollowParameters(in);
-            if (in.getVersion().onOrAfter(Version.V_7_14_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_14_0)) {
                 leaderIndexExclusionPatterns = in.readStringList();
             }
         }
@@ -201,11 +201,11 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(remoteCluster);
             out.writeStringCollection(leaderIndexPatterns);
             out.writeOptionalString(followIndexNamePattern);
-            if (out.getVersion().onOrAfter(Version.V_7_9_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_9_0)) {
                 settings.writeTo(out);
             }
             parameters.writeTo(out);
-            if (out.getVersion().onOrAfter(Version.V_7_14_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_14_0)) {
                 out.writeStringCollection(leaderIndexExclusionPatterns);
             }
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java
index edb4d9420de6..710d37b9c6c6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.ccr.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
@@ -193,12 +193,12 @@ public Request(StreamInput in) throws IOException {
             this.remoteCluster = in.readString();
             this.leaderIndex = in.readString();
             this.followerIndex = in.readString();
-            if (in.getVersion().onOrAfter(Version.V_7_9_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_9_0)) {
                 this.settings = Settings.readSettingsFromStream(in);
             }
             this.parameters = new FollowParameters(in);
             waitForActiveShards(ActiveShardCount.readFrom(in));
-            if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
                 this.dataStreamName = in.readOptionalString();
             }
         }
@@ -209,12 +209,12 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(remoteCluster);
             out.writeString(leaderIndex);
             out.writeString(followerIndex);
-            if (out.getVersion().onOrAfter(Version.V_7_9_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_9_0)) {
                 settings.writeTo(out);
             }
             parameters.writeTo(out);
             waitForActiveShards.writeTo(out);
-            if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
                 out.writeOptionalString(this.dataStreamName);
             }
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ShardFollowTask.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ShardFollowTask.java
index 38382d636624..911cc0f8dd52 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ShardFollowTask.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ShardFollowTask.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.ccr.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -196,7 +196,7 @@ public String toString() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.CURRENT.minimumCompatibilityVersion();
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.CURRENT.minimumCompatibilityVersion();
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/DataStreamFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/DataStreamFeatureSetUsage.java
index 42a99b5c25af..9fd73b660dd4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/DataStreamFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/DataStreamFeatureSetUsage.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.datastreams;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -39,8 +39,8 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_9_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_9_0;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/downsample/RollupIndexerAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/downsample/RollupIndexerAction.java
index 135578c52f1d..166d681c5230 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/downsample/RollupIndexerAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/downsample/RollupIndexerAction.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.rollup.action.RollupShardTask;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -258,6 +259,20 @@ public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             request.writeTo(out);
         }
+
+        @Override
+        public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+            return new RollupShardTask(
+                id,
+                type,
+                action,
+                parentTaskId,
+                request.rollupRequest.getSourceIndex(),
+                request.rollupRequest.getDownsampleConfig(),
+                headers,
+                shardId()
+            );
+        }
     }
 
     public static class ShardRollupResponse extends BroadcastShardResponse {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichFeatureSetUsage.java
index bd3729c0315f..8b5d85ec604b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichFeatureSetUsage.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.enrich;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
 import org.elasticsearch.xpack.core.XPackField;
@@ -25,7 +25,7 @@ public EnrichFeatureSetUsage(StreamInput input) throws IOException {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_5_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_5_0;
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/EnrichStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/EnrichStatsAction.java
index 912b6525baa0..9f6616427088 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/EnrichStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/EnrichStatsAction.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.enrich.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
@@ -62,7 +62,7 @@ public Response(StreamInput in) throws IOException {
             super(in);
             executingPolicies = in.readList(ExecutingPolicy::new);
             coordinatorStats = in.readList(CoordinatorStats::new);
-            cacheStats = in.getVersion().onOrAfter(Version.V_7_16_0) ? in.readList(CacheStats::new) : null;
+            cacheStats = in.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0) ? in.readList(CacheStats::new) : null;
         }
 
         public List<ExecutingPolicy> getExecutingPolicies() {
@@ -81,7 +81,7 @@ public List<CacheStats> getCacheStats() {
         public void writeTo(StreamOutput out) throws IOException {
             out.writeList(executingPolicies);
             out.writeList(coordinatorStats);
-            if (out.getVersion().onOrAfter(Version.V_7_16_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
                 out.writeList(cacheStats);
             }
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/ExecuteEnrichPolicyStatus.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/ExecuteEnrichPolicyStatus.java
index 9c207ff05bf6..89f9819409ca 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/ExecuteEnrichPolicyStatus.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/ExecuteEnrichPolicyStatus.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.enrich.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.tasks.Task;
@@ -46,7 +46,7 @@ public ExecuteEnrichPolicyStatus(ExecuteEnrichPolicyStatus status, String step)
 
     public ExecuteEnrichPolicyStatus(StreamInput in) throws IOException {
         this.phase = in.readString();
-        this.step = in.getVersion().onOrAfter(Version.V_7_16_0) ? in.readOptionalString() : null;
+        this.step = in.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0) ? in.readOptionalString() : null;
     }
 
     public String getPhase() {
@@ -69,7 +69,7 @@ public String getWriteableName() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(phase);
-        if (out.getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             out.writeOptionalString(step);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/eql/EqlFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/eql/EqlFeatureSetUsage.java
index 6f8f416e4ebc..77ebc85f9a1e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/eql/EqlFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/eql/EqlFeatureSetUsage.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.eql;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -52,8 +52,8 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_9_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_9_0;
     }
 
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/frozen/FrozenIndicesFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/frozen/FrozenIndicesFeatureSetUsage.java
index 4f70b2356c98..a5976c14d67c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/frozen/FrozenIndicesFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/frozen/FrozenIndicesFeatureSetUsage.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.frozen;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -26,8 +26,8 @@ public FrozenIndicesFeatureSetUsage(StreamInput input) throws IOException {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_4_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_4_0;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/graph/GraphFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/graph/GraphFeatureSetUsage.java
index c7730c88dbbc..9576be5c6ddb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/graph/GraphFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/graph/GraphFeatureSetUsage.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.graph;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
 import org.elasticsearch.xpack.core.XPackField;
@@ -24,8 +24,8 @@ public GraphFeatureSetUsage(boolean available, boolean enabled) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_0_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_0_0;
     }
 
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AllocateAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AllocateAction.java
index 743bb01a5240..6f0e41e89ca9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AllocateAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AllocateAction.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ilm;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
@@ -123,7 +123,7 @@ public AllocateAction(
     public AllocateAction(StreamInput in) throws IOException {
         this(
             in.readOptionalVInt(),
-            in.getVersion().onOrAfter(Version.V_7_16_0) ? in.readOptionalInt() : null,
+            in.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0) ? in.readOptionalInt() : null,
             (Map<String, String>) in.readGenericValue(),
             (Map<String, String>) in.readGenericValue(),
             (Map<String, String>) in.readGenericValue()
@@ -153,7 +153,7 @@ public Map<String, String> getRequire() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalVInt(numberOfReplicas);
-        if (out.getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             out.writeOptionalInt(totalShardsPerNode);
         }
         out.writeGenericValue(include);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java
index 62b71eaa19d6..879db231a99e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.ilm;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -305,7 +305,7 @@ public IndexLifecycleExplainResponse(StreamInput in) throws IOException {
             repositoryName = in.readOptionalString();
             snapshotName = in.readOptionalString();
             shrinkIndexName = in.readOptionalString();
-            if (in.getVersion().onOrAfter(Version.V_8_1_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
                 indexCreationDate = in.readOptionalLong();
             } else {
                 indexCreationDate = null;
@@ -352,7 +352,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalString(repositoryName);
             out.writeOptionalString(snapshotName);
             out.writeOptionalString(shrinkIndexName);
-            if (out.getVersion().onOrAfter(Version.V_8_1_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
                 out.writeOptionalLong(indexCreationDate);
             }
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleFeatureSetUsage.java
index 563a129c7f39..59b614fc805b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleFeatureSetUsage.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ilm;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -37,8 +37,8 @@ public IndexLifecycleFeatureSetUsage(StreamInput input) throws IOException {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_0_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_0_0;
     }
 
     @Override
@@ -425,12 +425,12 @@ public ActionConfigStats(StreamInput in) throws IOException {
             this.setPriorityPriority = in.readOptionalVInt();
             this.shrinkMaxPrimaryShardSize = in.readOptionalWriteable(ByteSizeValue::readFrom);
             this.shrinkNumberOfShards = in.readOptionalVInt();
-            if (in.getVersion().onOrAfter(Version.V_8_2_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
                 this.rolloverMaxPrimaryShardDocs = in.readOptionalVLong();
             } else {
                 this.rolloverMaxPrimaryShardDocs = null;
             }
-            if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
                 this.rolloverMinAge = in.readOptionalTimeValue();
                 this.rolloverMinDocs = in.readOptionalVLong();
                 this.rolloverMinPrimaryShardSize = in.readOptionalWriteable(ByteSizeValue::readFrom);
@@ -456,10 +456,10 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalVInt(setPriorityPriority);
             out.writeOptionalWriteable(shrinkMaxPrimaryShardSize);
             out.writeOptionalVInt(shrinkNumberOfShards);
-            if (out.getVersion().onOrAfter(Version.V_8_2_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
                 out.writeOptionalVLong(rolloverMaxPrimaryShardDocs);
             }
-            if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
                 out.writeOptionalTimeValue(rolloverMinAge);
                 out.writeOptionalVLong(rolloverMinDocs);
                 out.writeOptionalWriteable(rolloverMinPrimaryShardSize);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleMetadata.java
index dab9b1f420e0..d16ed099bb02 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleMetadata.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ilm;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.DiffableUtils;
 import org.elasticsearch.cluster.NamedDiff;
@@ -14,15 +14,18 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.Metadata.Custom;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.EnumSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -83,6 +86,10 @@ public Map<String, LifecyclePolicyMetadata> getPolicyMetadatas() {
         return policyMetadatas;
     }
 
+    /**
+     * @deprecated use {@link LifecycleOperationMetadata#getILMOperationMode()} instead. This may be incorrect.
+     */
+    @Deprecated(since = "8.7.0")
     public OperationMode getOperationMode() {
         return operationMode;
     }
@@ -100,15 +107,16 @@ public Diff<Custom> diff(Custom previousState) {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.xContentValuesMap(POLICIES_FIELD.getPreferredName(), policyMetadatas);
-        builder.field(OPERATION_MODE_FIELD.getPreferredName(), operationMode);
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return Iterators.concat(
+            ChunkedToXContentHelper.xContentValuesMap(POLICIES_FIELD.getPreferredName(), policyMetadatas),
+            Iterators.single((builder, params) -> builder.field(OPERATION_MODE_FIELD.getPreferredName(), operationMode))
+        );
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.CURRENT.minimumCompatibilityVersion();
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.CURRENT.minimumCompatibilityVersion();
     }
 
     @Override
@@ -183,8 +191,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT.minimumCompatibilityVersion();
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT.minimumCompatibilityVersion();
         }
 
         static Diff<LifecyclePolicyMetadata> readLifecyclePolicyDiffFrom(StreamInput in) throws IOException {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleOperationMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleOperationMetadata.java
new file mode 100644
index 000000000000..a468058e82f5
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleOperationMetadata.java
@@ -0,0 +1,202 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.ilm;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.NamedDiff;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Class that encapsulates the running operation mode of Index Lifecycle
+ * Management and Snapshot Lifecycle Management
+ */
+public class LifecycleOperationMetadata implements Metadata.Custom {
+    public static final String TYPE = "lifecycle_operation";
+    public static final ParseField ILM_OPERATION_MODE_FIELD = new ParseField("ilm_operation_mode");
+    public static final ParseField SLM_OPERATION_MODE_FIELD = new ParseField("slm_operation_mode");
+    public static final LifecycleOperationMetadata EMPTY = new LifecycleOperationMetadata(OperationMode.RUNNING, OperationMode.RUNNING);
+
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<LifecycleOperationMetadata, Void> PARSER = new ConstructingObjectParser<>(
+        TYPE,
+        a -> new LifecycleOperationMetadata(OperationMode.valueOf((String) a[0]), OperationMode.valueOf((String) a[1]))
+    );
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), ILM_OPERATION_MODE_FIELD);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), SLM_OPERATION_MODE_FIELD);
+    }
+
+    private final OperationMode ilmOperationMode;
+    private final OperationMode slmOperationMode;
+
+    public LifecycleOperationMetadata(OperationMode ilmOperationMode, OperationMode slmOperationMode) {
+        this.ilmOperationMode = ilmOperationMode;
+        this.slmOperationMode = slmOperationMode;
+    }
+
+    public LifecycleOperationMetadata(StreamInput in) throws IOException {
+        this.ilmOperationMode = in.readEnum(OperationMode.class);
+        this.slmOperationMode = in.readEnum(OperationMode.class);
+    }
+
+    /**
+     * Returns the current ILM mode based on the given cluster state. It first checks the newer
+     * storage mechanism ({@link LifecycleOperationMetadata#getILMOperationMode()}) before falling
+     * back to {@link IndexLifecycleMetadata#getOperationMode()}. If neither exist, the default
+     * value for an empty state is used.
+     */
+    @SuppressWarnings("deprecated")
+    public static OperationMode currentILMMode(final ClusterState state) {
+        IndexLifecycleMetadata oldMetadata = state.metadata().custom(IndexLifecycleMetadata.TYPE);
+        LifecycleOperationMetadata currentMetadata = state.metadata().custom(LifecycleOperationMetadata.TYPE);
+        return Optional.ofNullable(currentMetadata)
+            .map(LifecycleOperationMetadata::getILMOperationMode)
+            .orElse(
+                Optional.ofNullable(oldMetadata)
+                    .map(IndexLifecycleMetadata::getOperationMode)
+                    .orElseGet(LifecycleOperationMetadata.EMPTY::getILMOperationMode)
+            );
+    }
+
+    /**
+     * Returns the current ILM mode based on the given cluster state. It first checks the newer
+     * storage mechanism ({@link LifecycleOperationMetadata#getSLMOperationMode()}) before falling
+     * back to {@link SnapshotLifecycleMetadata#getOperationMode()}. If neither exist, the default
+     * value for an empty state is used.
+     */
+    @SuppressWarnings("deprecated")
+    public static OperationMode currentSLMMode(final ClusterState state) {
+        SnapshotLifecycleMetadata oldMetadata = state.metadata().custom(SnapshotLifecycleMetadata.TYPE);
+        LifecycleOperationMetadata currentMetadata = state.metadata().custom(LifecycleOperationMetadata.TYPE);
+        return Optional.ofNullable(currentMetadata)
+            .map(LifecycleOperationMetadata::getSLMOperationMode)
+            .orElse(
+                Optional.ofNullable(oldMetadata)
+                    .map(SnapshotLifecycleMetadata::getOperationMode)
+                    .orElseGet(LifecycleOperationMetadata.EMPTY::getSLMOperationMode)
+            );
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeEnum(ilmOperationMode);
+        out.writeEnum(slmOperationMode);
+    }
+
+    public OperationMode getILMOperationMode() {
+        return ilmOperationMode;
+    }
+
+    public OperationMode getSLMOperationMode() {
+        return slmOperationMode;
+    }
+
+    @Override
+    public Diff<Metadata.Custom> diff(Metadata.Custom previousState) {
+        return new LifecycleOperationMetadata.LifecycleOperationMetadataDiff((LifecycleOperationMetadata) previousState, this);
+    }
+
+    @Override
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        ToXContent ilmModeField = ((builder, params2) -> builder.field(ILM_OPERATION_MODE_FIELD.getPreferredName(), ilmOperationMode));
+        ToXContent slmModeField = ((builder, params2) -> builder.field(SLM_OPERATION_MODE_FIELD.getPreferredName(), slmOperationMode));
+        return Iterators.forArray(new ToXContent[] { ilmModeField, slmModeField });
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_7_0;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE;
+    }
+
+    @Override
+    public EnumSet<Metadata.XContentContext> context() {
+        // We do not store the lifecycle operation mode for ILM and SLM into a snapshot. This is so
+        // ILM and SLM can be operated independently from restoring snapshots.
+        return EnumSet.of(Metadata.XContentContext.API, Metadata.XContentContext.GATEWAY);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(ilmOperationMode, slmOperationMode);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        LifecycleOperationMetadata other = (LifecycleOperationMetadata) obj;
+        return Objects.equals(ilmOperationMode, other.ilmOperationMode) && Objects.equals(slmOperationMode, other.slmOperationMode);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true, true);
+    }
+
+    public static class LifecycleOperationMetadataDiff implements NamedDiff<Metadata.Custom> {
+
+        final OperationMode ilmOperationMode;
+        final OperationMode slmOperationMode;
+
+        LifecycleOperationMetadataDiff(LifecycleOperationMetadata before, LifecycleOperationMetadata after) {
+            this.ilmOperationMode = after.ilmOperationMode;
+            this.slmOperationMode = after.slmOperationMode;
+        }
+
+        public LifecycleOperationMetadataDiff(StreamInput in) throws IOException {
+            this.ilmOperationMode = in.readEnum(OperationMode.class);
+            this.slmOperationMode = in.readEnum(OperationMode.class);
+        }
+
+        @Override
+        public Metadata.Custom apply(Metadata.Custom part) {
+            return new LifecycleOperationMetadata(this.ilmOperationMode, this.slmOperationMode);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeEnum(ilmOperationMode);
+            out.writeEnum(slmOperationMode);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
+
+        @Override
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.V_8_7_0;
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverAction.java
index e7f04c028068..960f2ae58153 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverAction.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ilm;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -161,12 +161,12 @@ public RolloverAction(StreamInput in) throws IOException {
         maxPrimaryShardSize = in.readOptionalWriteable(ByteSizeValue::readFrom);
         maxAge = in.readOptionalTimeValue();
         maxDocs = in.readOptionalVLong();
-        if (in.getVersion().onOrAfter(Version.V_8_2_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
             maxPrimaryShardDocs = in.readOptionalVLong();
         } else {
             maxPrimaryShardDocs = null;
         }
-        if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             minSize = in.readOptionalWriteable(ByteSizeValue::readFrom);
             minPrimaryShardSize = in.readOptionalWriteable(ByteSizeValue::readFrom);
             minAge = in.readOptionalTimeValue();
@@ -187,10 +187,10 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(maxPrimaryShardSize);
         out.writeOptionalTimeValue(maxAge);
         out.writeOptionalVLong(maxDocs);
-        if (out.getVersion().onOrAfter(Version.V_8_2_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
             out.writeOptionalVLong(maxPrimaryShardDocs);
         }
-        if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             out.writeOptionalWriteable(minSize);
             out.writeOptionalWriteable(minPrimaryShardSize);
             out.writeOptionalTimeValue(minAge);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java
index b3020dddd5aa..1bdc35962f7f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.indexing;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -84,7 +84,7 @@ public IndexerJobStats(StreamInput in) throws IOException {
         this.indexFailures = in.readVLong();
         this.searchFailures = in.readVLong();
 
-        if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_7_0)) {
             this.processingTime = in.readVLong();
             this.processingTotal = in.readVLong();
         }
@@ -205,7 +205,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(searchTotal);
         out.writeVLong(indexFailures);
         out.writeVLong(searchFailures);
-        if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_7_0)) {
             out.writeVLong(processingTime);
             out.writeVLong(processingTotal);
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/logstash/LogstashFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/logstash/LogstashFeatureSetUsage.java
index d5f53f53fa6c..3152a780730f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/logstash/LogstashFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/logstash/LogstashFeatureSetUsage.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.logstash;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
 import org.elasticsearch.xpack.core.XPackField;
@@ -24,8 +24,8 @@ public LogstashFeatureSetUsage(boolean available) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_0_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_0_0;
     }
 
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningFeatureSetUsage.java
index 8861da43ad93..515def6fb56e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningFeatureSetUsage.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ml;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -64,8 +64,8 @@ public MachineLearningFeatureSetUsage(StreamInput in) throws IOException {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_0_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_0_0;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java
index 64645a64e4bc..c48d3f790396 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetadata.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ml;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.DiffableUtils;
@@ -14,19 +14,21 @@
 import org.elasticsearch.cluster.SimpleDiffable;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.EnumSet;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Objects;
 import java.util.SortedMap;
@@ -63,8 +65,8 @@ public boolean isResetMode() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.CURRENT.minimumCompatibilityVersion();
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.CURRENT.minimumCompatibilityVersion();
     }
 
     @Override
@@ -83,7 +85,7 @@ public Diff<Metadata.Custom> diff(Metadata.Custom previousState) {
     }
 
     public MlMetadata(StreamInput in) throws IOException {
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             int size = in.readVInt();
             for (int i = 0; i < size; i++) {
                 in.readString();
@@ -101,7 +103,7 @@ public MlMetadata(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             writeMap(Collections.emptySortedMap(), out);
             writeMap(Collections.emptySortedMap(), out);
         }
@@ -118,10 +120,11 @@ private static <T extends Writeable> void writeMap(Map<String, T> map, StreamOut
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(UPGRADE_MODE.getPreferredName(), upgradeMode);
-        builder.field(RESET_MODE.getPreferredName(), resetMode);
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return Iterators.single(
+            ((builder, params) -> builder.field(UPGRADE_MODE.getPreferredName(), upgradeMode)
+                .field(RESET_MODE.getPreferredName(), resetMode))
+        );
     }
 
     public static class MlMetadataDiff implements NamedDiff<Metadata.Custom> {
@@ -135,7 +138,7 @@ public static class MlMetadataDiff implements NamedDiff<Metadata.Custom> {
         }
 
         public MlMetadataDiff(StreamInput in) throws IOException {
-            if (in.getVersion().before(Version.V_8_0_0)) {
+            if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), Job::new, MlMetadataDiff::readJobDiffFrom);
                 DiffableUtils.readJdkMapDiff(
                     in,
@@ -160,7 +163,7 @@ public Metadata.Custom apply(Metadata.Custom part) {
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            if (out.getVersion().before(Version.V_8_0_0)) {
+            if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
                 SortedMap<String, Job> jobs = Collections.emptySortedMap();
                 DiffableUtils.diff(jobs, jobs, DiffableUtils.getStringKeySerializer()).writeTo(out);
                 SortedMap<String, DatafeedConfig> datafeeds = Collections.emptySortedMap();
@@ -176,8 +179,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT.minimumCompatibilityVersion();
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT.minimumCompatibilityVersion();
         }
 
         static Diff<Job> readJobDiffFrom(StreamInput in) throws IOException {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteJobAction.java
index e1759e804d40..37cf558c2b8c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteJobAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteJobAction.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
@@ -52,7 +52,7 @@ public Request(StreamInput in) throws IOException {
             super(in);
             jobId = in.readString();
             force = in.readBoolean();
-            if (in.getVersion().onOrAfter(Version.V_8_7_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
                 deleteUserAnnotations = in.readBoolean();
             } else {
                 deleteUserAnnotations = false;
@@ -110,7 +110,7 @@ public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             out.writeString(jobId);
             out.writeBoolean(force);
-            if (out.getVersion().onOrAfter(Version.V_8_7_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
                 out.writeBoolean(deleteUserAnnotations);
             }
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteTrainedModelAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteTrainedModelAction.java
index 15c1282f7697..ae1d787e8a11 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteTrainedModelAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteTrainedModelAction.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
@@ -41,7 +41,7 @@ public static class Request extends AcknowledgedRequest<Request> implements ToXC
         public Request(StreamInput in) throws IOException {
             super(in);
             id = in.readString();
-            if (in.getVersion().onOrAfter(Version.V_8_1_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
                 force = in.readBoolean();
             } else {
                 force = false;
@@ -88,7 +88,7 @@ public boolean equals(Object o) {
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             out.writeString(id);
-            if (out.getVersion().onOrAfter(Version.V_8_1_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
                 out.writeBoolean(force);
             }
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedRunningStateAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedRunningStateAction.java
index 09c116737aa0..b41581612db2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedRunningStateAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedRunningStateAction.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.tasks.BaseTasksRequest;
 import org.elasticsearch.action.support.tasks.BaseTasksResponse;
@@ -101,7 +101,7 @@ public RunningState(boolean realTimeConfigured, boolean realTimeRunning, @Nullab
             public RunningState(StreamInput in) throws IOException {
                 this.realTimeConfigured = in.readBoolean();
                 this.realTimeRunning = in.readBoolean();
-                if (in.getVersion().onOrAfter(Version.V_8_1_0)) {
+                if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
                     this.searchInterval = in.readOptionalWriteable(SearchInterval::new);
                 } else {
                     this.searchInterval = null;
@@ -127,7 +127,7 @@ public int hashCode() {
             public void writeTo(StreamOutput out) throws IOException {
                 out.writeBoolean(realTimeConfigured);
                 out.writeBoolean(realTimeRunning);
-                if (out.getVersion().onOrAfter(Version.V_8_1_0)) {
+                if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
                     out.writeOptionalWriteable(searchInterval);
                 }
             }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsAction.java
index a99b112490ab..bb26fdb841c3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsAction.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.core.ml.action;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -119,7 +119,7 @@ public TrainedModelStats(
 
             public TrainedModelStats(StreamInput in) throws IOException {
                 modelId = in.readString();
-                if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+                if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
                     modelSizeStats = in.readOptionalWriteable(TrainedModelSizeStats::new);
                 } else {
                     modelSizeStats = null;
@@ -127,7 +127,7 @@ public TrainedModelStats(StreamInput in) throws IOException {
                 ingestStats = new IngestStats(in);
                 pipelineCount = in.readVInt();
                 inferenceStats = in.readOptionalWriteable(InferenceStats::new);
-                if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+                if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
                     this.deploymentStats = in.readOptionalWriteable(AssignmentStats::new);
                 } else {
                     this.deploymentStats = null;
@@ -183,13 +183,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             @Override
             public void writeTo(StreamOutput out) throws IOException {
                 out.writeString(modelId);
-                if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+                if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
                     out.writeOptionalWriteable(modelSizeStats);
                 }
                 ingestStats.writeTo(out);
                 out.writeVInt(pipelineCount);
                 out.writeOptionalWriteable(inferenceStats);
-                if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+                if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
                     out.writeOptionalWriteable(deploymentStats);
                 }
             }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/InferModelAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/InferModelAction.java
index 72e1e2ed5220..cdf5f874b844 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/InferModelAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/InferModelAction.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
@@ -23,11 +23,12 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.ml.inference.results.InferenceResults;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.EmptyConfigUpdate;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfigUpdate;
-import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RegressionConfigUpdate;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -80,55 +81,78 @@ public static Builder parseRequest(String modelId, XContentParser parser) {
         private final List<Map<String, Object>> objectsToInfer;
         private final InferenceConfigUpdate update;
         private final boolean previouslyLicensed;
-        private final TimeValue timeout;
+        private TimeValue inferenceTimeout;
+        // textInput added for uses that accept a query string
+        // and do know which field the model expects to find its
+        // input and so cannot construct a document.
+        private final List<String> textInput;
 
-        public Request(String modelId, boolean previouslyLicensed) {
-            this(modelId, Collections.emptyList(), RegressionConfigUpdate.EMPTY_PARAMS, TimeValue.MAX_VALUE, previouslyLicensed);
-        }
-
-        public Request(
-            String modelId,
-            List<Map<String, Object>> objectsToInfer,
-            InferenceConfigUpdate inferenceConfig,
-            TimeValue timeout,
-            boolean previouslyLicensed
-        ) {
-            this.modelId = ExceptionsHelper.requireNonNull(modelId, MODEL_ID);
-            this.objectsToInfer = Collections.unmodifiableList(ExceptionsHelper.requireNonNull(objectsToInfer, "objects_to_infer"));
-            this.update = ExceptionsHelper.requireNonNull(inferenceConfig, "inference_config");
-            this.previouslyLicensed = previouslyLicensed;
-            this.timeout = timeout;
-        }
-
-        public Request(
+        public static Request forDocs(
             String modelId,
-            List<Map<String, Object>> objectsToInfer,
-            InferenceConfigUpdate inferenceConfig,
+            List<Map<String, Object>> docs,
+            InferenceConfigUpdate update,
             boolean previouslyLicensed
         ) {
-            this(modelId, objectsToInfer, inferenceConfig, TimeValue.MAX_VALUE, previouslyLicensed);
+            return new Request(
+                ExceptionsHelper.requireNonNull(modelId, InferModelAction.Request.MODEL_ID),
+                update,
+                ExceptionsHelper.requireNonNull(Collections.unmodifiableList(docs), DOCS),
+                null,
+                DEFAULT_TIMEOUT,
+                previouslyLicensed
+            );
         }
 
-        public Request(String modelId, Map<String, Object> objectToInfer, InferenceConfigUpdate update, boolean previouslyLicensed) {
-            this(
+        public static Request forTextInput(String modelId, InferenceConfigUpdate update, List<String> textInput) {
+            return new Request(
                 modelId,
-                Collections.singletonList(ExceptionsHelper.requireNonNull(objectToInfer, "objects_to_infer")),
                 update,
-                TimeValue.MAX_VALUE,
-                previouslyLicensed
+                List.of(),
+                ExceptionsHelper.requireNonNull(textInput, "inference text input"),
+                DEFAULT_TIMEOUT,
+                false
             );
         }
 
+        Request(
+            String modelId,
+            InferenceConfigUpdate inferenceConfigUpdate,
+            List<Map<String, Object>> docs,
+            List<String> textInput,
+            TimeValue inferenceTimeout,
+            boolean previouslyLicensed
+        ) {
+            this.modelId = ExceptionsHelper.requireNonNull(modelId, MODEL_ID);
+            this.objectsToInfer = Collections.unmodifiableList(ExceptionsHelper.requireNonNull(docs, DOCS.getPreferredName()));
+            this.update = ExceptionsHelper.requireNonNull(inferenceConfigUpdate, "inference_config");
+            this.textInput = textInput;
+            this.previouslyLicensed = previouslyLicensed;
+            this.inferenceTimeout = inferenceTimeout;
+        }
+
         public Request(StreamInput in) throws IOException {
             super(in);
             this.modelId = in.readString();
             this.objectsToInfer = in.readImmutableList(StreamInput::readMap);
             this.update = in.readNamedWriteable(InferenceConfigUpdate.class);
             this.previouslyLicensed = in.readBoolean();
-            if (in.getVersion().onOrAfter(Version.V_8_3_0)) {
-                this.timeout = in.readTimeValue();
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0)) {
+                this.inferenceTimeout = in.readTimeValue();
             } else {
-                this.timeout = TimeValue.MAX_VALUE;
+                this.inferenceTimeout = TimeValue.MAX_VALUE;
+            }
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+                textInput = in.readOptionalStringList();
+            } else {
+                textInput = null;
+            }
+        }
+
+        public int numberOfDocuments() {
+            if (textInput != null) {
+                return textInput.size();
+            } else {
+                return objectsToInfer.size();
             }
         }
 
@@ -140,6 +164,10 @@ public List<Map<String, Object>> getObjectsToInfer() {
             return objectsToInfer;
         }
 
+        public List<String> getTextInput() {
+            return textInput;
+        }
+
         public InferenceConfigUpdate getUpdate() {
             return update;
         }
@@ -148,8 +176,12 @@ public boolean isPreviouslyLicensed() {
             return previouslyLicensed;
         }
 
-        public TimeValue getTimeout() {
-            return timeout;
+        public TimeValue getInferenceTimeout() {
+            return inferenceTimeout;
+        }
+
+        public void setInferenceTimeout(TimeValue inferenceTimeout) {
+            this.inferenceTimeout = inferenceTimeout;
         }
 
         @Override
@@ -164,8 +196,11 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeCollection(objectsToInfer, StreamOutput::writeGenericMap);
             out.writeNamedWriteable(update);
             out.writeBoolean(previouslyLicensed);
-            if (out.getVersion().onOrAfter(Version.V_8_3_0)) {
-                out.writeTimeValue(timeout);
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0)) {
+                out.writeTimeValue(inferenceTimeout);
+            }
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+                out.writeOptionalStringCollection(textInput);
             }
         }
 
@@ -177,8 +212,9 @@ public boolean equals(Object o) {
             return Objects.equals(modelId, that.modelId)
                 && Objects.equals(update, that.update)
                 && Objects.equals(previouslyLicensed, that.previouslyLicensed)
-                && Objects.equals(timeout, that.timeout)
-                && Objects.equals(objectsToInfer, that.objectsToInfer);
+                && Objects.equals(inferenceTimeout, that.inferenceTimeout)
+                && Objects.equals(objectsToInfer, that.objectsToInfer)
+                && Objects.equals(textInput, that.textInput);
         }
 
         @Override
@@ -188,7 +224,7 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId,
 
         @Override
         public int hashCode() {
-            return Objects.hash(modelId, objectsToInfer, update, previouslyLicensed, timeout);
+            return Objects.hash(modelId, objectsToInfer, update, previouslyLicensed, inferenceTimeout, textInput);
         }
 
         public static class Builder {
@@ -196,7 +232,7 @@ public static class Builder {
             private String modelId;
             private List<Map<String, Object>> docs;
             private TimeValue timeout;
-            private InferenceConfigUpdate update;
+            private InferenceConfigUpdate update = new EmptyConfigUpdate();
 
             private Builder() {}
 
@@ -229,7 +265,7 @@ private Builder setInferenceTimeout(String inferenceTimeout) {
             }
 
             public Request build() {
-                return new Request(modelId, docs, update, timeout, false);
+                return new Request(modelId, update, docs, null, timeout, false);
             }
         }
 
@@ -302,12 +338,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         }
 
         public static class Builder {
-            private List<InferenceResults> inferenceResults;
+            private List<InferenceResults> inferenceResults = new ArrayList<>();
             private String modelId;
             private boolean isLicensed;
 
-            public Builder setInferenceResults(List<InferenceResults> inferenceResults) {
-                this.inferenceResults = inferenceResults;
+            public Builder addInferenceResults(List<InferenceResults> inferenceResults) {
+                this.inferenceResults.addAll(inferenceResults);
                 return this;
             }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/InferTrainedModelDeploymentAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/InferTrainedModelDeploymentAction.java
index 539bf59da174..bcb0ae53c76f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/InferTrainedModelDeploymentAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/InferTrainedModelDeploymentAction.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.tasks.BaseTasksRequest;
@@ -44,7 +44,14 @@ public class InferTrainedModelDeploymentAction extends ActionType<InferTrainedMo
 
     public static final InferTrainedModelDeploymentAction INSTANCE = new InferTrainedModelDeploymentAction();
 
-    // TODO Review security level
+    /**
+     * Do not call this action directly, use InferModelAction instead
+     * which will perform various checks and set the node the request
+     * should execute on.
+     *
+     * The action is poorly named as once it was publicly accessible
+     * and exposed through a REST API now it _must_ only called internally.
+     */
     public static final String NAME = "cluster:monitor/xpack/ml/trained_models/deployment/infer";
 
     public InferTrainedModelDeploymentAction() {
@@ -94,22 +101,38 @@ public static Request.Builder parseRequest(String modelId, XContentParser parser
         // textInput added for uses that accept a query string
         // and do know which field the model expects to find its
         // input and so cannot construct a document.
-        private final String textInput;
+        private final List<String> textInput;
 
-        public Request(String modelId, InferenceConfigUpdate update, List<Map<String, Object>> docs, TimeValue inferenceTimeout) {
-            this.modelId = ExceptionsHelper.requireNonNull(modelId, InferModelAction.Request.MODEL_ID);
-            this.docs = ExceptionsHelper.requireNonNull(Collections.unmodifiableList(docs), DOCS);
-            this.update = update;
-            this.inferenceTimeout = inferenceTimeout;
-            this.textInput = null;
+        public static Request forDocs(
+            String modelId,
+            InferenceConfigUpdate update,
+            List<Map<String, Object>> docs,
+            TimeValue inferenceTimeout
+        ) {
+            return new Request(
+                ExceptionsHelper.requireNonNull(modelId, InferModelAction.Request.MODEL_ID),
+                update,
+                ExceptionsHelper.requireNonNull(Collections.unmodifiableList(docs), DOCS),
+                null,
+                false,
+                inferenceTimeout
+            );
         }
 
-        public Request(String modelId, InferenceConfigUpdate update, String textInput, TimeValue inferenceTimeout) {
-            this.modelId = ExceptionsHelper.requireNonNull(modelId, InferModelAction.Request.MODEL_ID);
-            this.docs = List.of();
-            this.textInput = ExceptionsHelper.requireNonNull(textInput, "inference text input");
-            this.update = update;
-            this.inferenceTimeout = inferenceTimeout;
+        public static Request forTextInput(
+            String modelId,
+            InferenceConfigUpdate update,
+            List<String> textInput,
+            TimeValue inferenceTimeout
+        ) {
+            return new Request(
+                ExceptionsHelper.requireNonNull(modelId, InferModelAction.Request.MODEL_ID),
+                update,
+                List.of(),
+                ExceptionsHelper.requireNonNull(textInput, "inference text input"),
+                false,
+                inferenceTimeout
+            );
         }
 
         // for tests
@@ -117,7 +140,7 @@ public Request(String modelId, InferenceConfigUpdate update, String textInput, T
             String modelId,
             InferenceConfigUpdate update,
             List<Map<String, Object>> docs,
-            String textInput,
+            List<String> textInput,
             boolean skipQueue,
             TimeValue inferenceTimeout
         ) {
@@ -135,11 +158,11 @@ public Request(StreamInput in) throws IOException {
             docs = in.readImmutableList(StreamInput::readMap);
             update = in.readOptionalNamedWriteable(InferenceConfigUpdate.class);
             inferenceTimeout = in.readOptionalTimeValue();
-            if (in.getVersion().onOrAfter(Version.V_8_3_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0)) {
                 skipQueue = in.readBoolean();
             }
-            if (in.getVersion().onOrAfter(Version.V_8_6_0)) {
-                textInput = in.readOptionalString();
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+                textInput = in.readOptionalStringList();
             } else {
                 textInput = null;
             }
@@ -153,7 +176,7 @@ public List<Map<String, Object>> getDocs() {
             return docs;
         }
 
-        public String getTextInput() {
+        public List<String> getTextInput() {
             return textInput;
         }
 
@@ -196,10 +219,6 @@ public ActionRequestValidationException validate() {
                 if (docs.isEmpty() && textInput == null) {
                     validationException = addValidationError("at least one document is required ", validationException);
                 }
-                if (docs.size() > 1) {
-                    // TODO support multiple docs
-                    validationException = addValidationError("multiple documents are not supported", validationException);
-                }
             }
             return validationException;
         }
@@ -211,11 +230,11 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeCollection(docs, StreamOutput::writeGenericMap);
             out.writeOptionalNamedWriteable(update);
             out.writeOptionalTimeValue(inferenceTimeout);
-            if (out.getVersion().onOrAfter(Version.V_8_3_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0)) {
                 out.writeBoolean(skipQueue);
             }
-            if (out.getVersion().onOrAfter(Version.V_8_6_0)) {
-                out.writeOptionalString(textInput);
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+                out.writeOptionalStringCollection(textInput);
             }
         }
 
@@ -254,7 +273,7 @@ public static class Builder {
             private TimeValue timeout;
             private InferenceConfigUpdate update;
             private boolean skipQueue = false;
-            private String textInput;
+            private List<String> textInput;
 
             private Builder() {}
 
@@ -282,7 +301,7 @@ private Builder setInferenceTimeout(String inferenceTimeout) {
                 return setInferenceTimeout(TimeValue.parseTimeValue(inferenceTimeout, TIMEOUT.getPreferredName()));
             }
 
-            public Builder setTextInput(String textInput) {
+            public Builder setTextInput(List<String> textInput) {
                 this.textInput = textInput;
                 return this;
             }
@@ -300,50 +319,45 @@ public Request build() {
 
     public static class Response extends BaseTasksResponse implements Writeable, ToXContentObject {
 
-        private final InferenceResults results;
-        private long tookMillis;
+        private final List<InferenceResults> results;
 
-        public Response(InferenceResults result, long tookMillis) {
+        public Response(List<InferenceResults> results) {
             super(Collections.emptyList(), Collections.emptyList());
-            this.results = Objects.requireNonNull(result);
-            this.tookMillis = tookMillis;
+            this.results = Objects.requireNonNull(results);
         }
 
         public Response(StreamInput in) throws IOException {
             super(in);
-            results = in.readNamedWriteable(InferenceResults.class);
-            if (in.getVersion().onOrAfter(Version.V_8_6_0)) {
-                tookMillis = in.readVLong();
-            }
-        }
 
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            results.toXContent(builder, params);
-            builder.endObject();
-            return builder;
+            // Multiple results added in 8.6.1
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_6_1)) {
+                results = in.readNamedWriteableList(InferenceResults.class);
+            } else {
+                results = List.of(in.readNamedWriteable(InferenceResults.class));
+            }
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            out.writeNamedWriteable(results);
-            if (out.getVersion().onOrAfter(Version.V_8_6_0)) {
-                out.writeVLong(tookMillis);
+
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_6_1)) {
+                out.writeNamedWriteableList(results);
+            } else {
+                out.writeNamedWriteable(results.get(0));
             }
         }
 
-        public InferenceResults getResults() {
+        public List<InferenceResults> getResults() {
             return results;
         }
 
-        public long getTookMillis() {
-            return tookMillis;
-        }
-
-        public void setTookMillis(long tookMillis) {
-            this.tookMillis = tookMillis;
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            results.get(0).toXContent(builder, params);
+            builder.endObject();
+            return builder;
         }
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java
index e6a8a5601362..b7cb922b65fa 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
@@ -229,8 +229,8 @@ public String toString() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT.minimumCompatibilityVersion();
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT.minimumCompatibilityVersion();
         }
 
         @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PreviewDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PreviewDatafeedAction.java
index 88454b1500e8..5faf4b545ef7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PreviewDatafeedAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PreviewDatafeedAction.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
@@ -85,7 +85,7 @@ public Request(StreamInput in) throws IOException {
             datafeedId = in.readString();
             datafeedConfig = in.readOptionalWriteable(DatafeedConfig::new);
             jobConfig = in.readOptionalWriteable(Job.Builder::new);
-            if (in.getVersion().onOrAfter(Version.V_8_3_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0)) {
                 this.startTime = in.readOptionalLong();
                 this.endTime = in.readOptionalLong();
             } else {
@@ -163,7 +163,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(datafeedId);
             out.writeOptionalWriteable(datafeedConfig);
             out.writeOptionalWriteable(jobConfig);
-            if (out.getVersion().onOrAfter(Version.V_8_3_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0)) {
                 out.writeOptionalLong(startTime);
                 out.writeOptionalLong(endTime);
             }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelVocabularyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelVocabularyAction.java
index bb1c8fd111cb..270fc5312e2d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelVocabularyAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelVocabularyAction.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
@@ -66,7 +66,7 @@ public Request(StreamInput in) throws IOException {
             super(in);
             this.modelId = in.readString();
             this.vocabulary = in.readStringList();
-            if (in.getVersion().onOrAfter(Version.V_8_2_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
                 this.merges = in.readStringList();
             } else {
                 this.merges = List.of();
@@ -102,7 +102,7 @@ public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             out.writeString(modelId);
             out.writeStringCollection(vocabulary);
-            if (out.getVersion().onOrAfter(Version.V_8_2_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
                 out.writeStringCollection(merges);
             }
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ResetJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ResetJobAction.java
index f708051d0578..da9d79560f38 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ResetJobAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ResetJobAction.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.action;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
@@ -64,7 +65,7 @@ public Request(StreamInput in) throws IOException {
             super(in);
             jobId = in.readString();
             skipJobStateValidation = in.readBoolean();
-            if (in.getVersion().onOrAfter(Version.V_8_7_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
                 deleteUserAnnotations = in.readBoolean();
             } else {
                 deleteUserAnnotations = false;
@@ -76,7 +77,7 @@ public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             out.writeString(jobId);
             out.writeBoolean(skipJobStateValidation);
-            if (out.getVersion().onOrAfter(Version.V_8_7_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
                 out.writeBoolean(deleteUserAnnotations);
             }
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/SemanticSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/SemanticSearchAction.java
deleted file mode 100644
index a2fb072199c2..000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/SemanticSearchAction.java
+++ /dev/null
@@ -1,570 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.core.ml.action;
-
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.action.ActionType;
-import org.elasticsearch.action.IndicesRequest;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.index.query.AbstractQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.fetch.StoredFieldsContext;
-import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
-import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
-import org.elasticsearch.search.vectors.KnnSearchBuilder;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ObjectParser;
-import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfigUpdate;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Objects;
-
-import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
-
-public class SemanticSearchAction extends ActionType<SemanticSearchAction.Response> {
-
-    public static final String NAME = "indices:data/read/semantic_search";
-
-    public static final SemanticSearchAction INSTANCE = new SemanticSearchAction(NAME);
-
-    private SemanticSearchAction(String name) {
-        super(name, SemanticSearchAction.Response::new);
-    }
-
-    public static class Request extends ActionRequest implements IndicesRequest.Replaceable {
-
-        public static final ParseField QUERY_STRING = new ParseField("query_string"); // TODO a better name and update docs when changed
-        public static final ParseField TEXT_EMBEDDING_CONFIG = new ParseField("text_embedding_config");
-
-        static final ObjectParser<Request.Builder, Void> PARSER = new ObjectParser<>(NAME);
-
-        static {
-            PARSER.declareString(Request.Builder::setModelId, InferModelAction.Request.MODEL_ID);
-            PARSER.declareString(Request.Builder::setQueryString, QUERY_STRING);
-            PARSER.declareString(Request.Builder::setTimeout, SearchSourceBuilder.TIMEOUT_FIELD);
-            PARSER.declareObject(
-                Request.Builder::setUpdate,
-                (p, c) -> TextEmbeddingConfigUpdate.fromXContentStrict(p),
-                TEXT_EMBEDDING_CONFIG
-            );
-            PARSER.declareObject(
-                Request.Builder::setQueryBuilder,
-                (p, c) -> AbstractQueryBuilder.parseTopLevelQuery(p),
-                SearchSourceBuilder.QUERY_FIELD
-            );
-            PARSER.declareObject(Request.Builder::setKnnSearch, (p, c) -> KnnQueryOptions.fromXContent(p), SearchSourceBuilder.KNN_FIELD);
-            PARSER.declareField(
-                (p, request, c) -> request.setFetchSource(FetchSourceContext.fromXContent(p)),
-                SearchSourceBuilder._SOURCE_FIELD,
-                ObjectParser.ValueType.OBJECT_ARRAY_BOOLEAN_OR_STRING
-            );
-            PARSER.declareFieldArray(
-                Request.Builder::setFields,
-                (p, c) -> FieldAndFormat.fromXContent(p),
-                SearchSourceBuilder.FETCH_FIELDS_FIELD,
-                ObjectParser.ValueType.OBJECT_ARRAY
-            );
-            PARSER.declareFieldArray(
-                Request.Builder::setDocValueFields,
-                (p, c) -> FieldAndFormat.fromXContent(p),
-                SearchSourceBuilder.DOCVALUE_FIELDS_FIELD,
-                ObjectParser.ValueType.OBJECT_ARRAY
-            );
-            PARSER.declareField(
-                Request.Builder::setStoredFields,
-                (p, c) -> StoredFieldsContext.fromXContent(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), p),
-                SearchSourceBuilder.STORED_FIELDS_FIELD,
-                ObjectParser.ValueType.STRING_ARRAY
-            );
-            PARSER.declareInt(Request.Builder::setSize, SearchSourceBuilder.SIZE_FIELD);
-        }
-
-        public static Request parseRestRequest(RestRequest restRequest) throws IOException {
-            Builder builder = new Builder(Strings.splitStringByCommaToArray(restRequest.param("index")));
-            if (restRequest.hasContentOrSourceParam()) {
-                try (XContentParser contentParser = restRequest.contentOrSourceParamParser()) {
-                    PARSER.parse(contentParser, builder, null);
-                }
-            }
-            // Query parameters are preferred to body parameters.
-            if (restRequest.hasParam("size")) {
-                builder.setSize(restRequest.paramAsInt("size", -1));
-            }
-            builder.setRouting(restRequest.param("routing"));
-            return builder.build();
-        }
-
-        private String[] indices;
-        private final String routing;
-        private final String queryString;
-        private final String modelId;
-        private final TimeValue inferenceTimeout;
-        private final QueryBuilder query;
-        private final KnnQueryOptions knnQueryOptions;
-        private final TextEmbeddingConfigUpdate embeddingConfig;
-        private final FetchSourceContext fetchSource;
-        private final List<FieldAndFormat> fields;
-        private final List<FieldAndFormat> docValueFields;
-        private final StoredFieldsContext storedFields;
-        private final int size;
-
-        public Request(StreamInput in) throws IOException {
-            super(in);
-            indices = in.readStringArray();
-            routing = in.readOptionalString();
-            queryString = in.readString();
-            modelId = in.readString();
-            inferenceTimeout = in.readOptionalTimeValue();
-            query = in.readOptionalNamedWriteable(QueryBuilder.class);
-            knnQueryOptions = new KnnQueryOptions(in);
-            embeddingConfig = in.readOptionalWriteable(TextEmbeddingConfigUpdate::new);
-            fetchSource = in.readOptionalWriteable(FetchSourceContext::readFrom);
-            fields = in.readOptionalList(FieldAndFormat::new);
-            docValueFields = in.readOptionalList(FieldAndFormat::new);
-            storedFields = in.readOptionalWriteable(StoredFieldsContext::new);
-            size = in.readInt();
-        }
-
-        Request(
-            String[] indices,
-            String routing,
-            String queryString,
-            String modelId,
-            QueryBuilder query,
-            KnnQueryOptions knnQueryOptions,
-            TextEmbeddingConfigUpdate embeddingConfig,
-            TimeValue inferenceTimeout,
-            FetchSourceContext fetchSource,
-            List<FieldAndFormat> fields,
-            List<FieldAndFormat> docValueFields,
-            StoredFieldsContext storedFields,
-            int size
-        ) {
-            this.indices = Objects.requireNonNull(indices, "[indices] must not be null");
-            this.routing = routing;
-            this.queryString = queryString;
-            this.modelId = modelId;
-            this.query = query;
-            this.knnQueryOptions = knnQueryOptions;
-            this.embeddingConfig = embeddingConfig;
-            this.inferenceTimeout = inferenceTimeout;
-            this.fetchSource = fetchSource;
-            this.fields = fields;
-            this.docValueFields = docValueFields;
-            this.storedFields = storedFields;
-            this.size = size;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeStringArray(indices);
-            out.writeOptionalString(routing);
-            out.writeString(queryString);
-            out.writeString(modelId);
-            out.writeOptionalTimeValue(inferenceTimeout);
-            out.writeOptionalNamedWriteable(query);
-            knnQueryOptions.writeTo(out);
-            out.writeOptionalWriteable(embeddingConfig);
-            out.writeOptionalWriteable(fetchSource);
-            out.writeOptionalCollection(fields);
-            out.writeOptionalCollection(docValueFields);
-            out.writeOptionalWriteable(storedFields);
-            out.writeInt(size);
-        }
-
-        @Override
-        public String[] indices() {
-            return indices;
-        }
-
-        @Override
-        public IndicesOptions indicesOptions() {
-            return SearchRequest.DEFAULT_INDICES_OPTIONS;
-        }
-
-        @Override
-        public IndicesRequest indices(String... indices) {
-            Objects.requireNonNull(indices, "indices must not be null");
-            for (String index : indices) {
-                Objects.requireNonNull(index, "index must not be null");
-            }
-            this.indices = indices;
-            return this;
-        }
-
-        public String getRouting() {
-            return routing;
-        }
-
-        public String getQueryString() {
-            return queryString;
-        }
-
-        public String getModelId() {
-            return modelId;
-        }
-
-        public TimeValue getInferenceTimeout() {
-            return inferenceTimeout;
-        }
-
-        public QueryBuilder getQuery() {
-            return query;
-        }
-
-        public KnnQueryOptions getKnnQueryOptions() {
-            return knnQueryOptions;
-        }
-
-        public TextEmbeddingConfigUpdate getEmbeddingConfig() {
-            return embeddingConfig;
-        }
-
-        public FetchSourceContext getFetchSource() {
-            return fetchSource;
-        }
-
-        public List<FieldAndFormat> getFields() {
-            return fields;
-        }
-
-        public List<FieldAndFormat> getDocValueFields() {
-            return docValueFields;
-        }
-
-        public StoredFieldsContext getStoredFields() {
-            return storedFields;
-        }
-
-        public int getSize() {
-            return size;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            Request request = (Request) o;
-            return Arrays.equals(indices, request.indices)
-                && Objects.equals(routing, request.routing)
-                && Objects.equals(queryString, request.queryString)
-                && Objects.equals(modelId, request.modelId)
-                && Objects.equals(inferenceTimeout, request.inferenceTimeout)
-                && Objects.equals(query, request.query)
-                && Objects.equals(knnQueryOptions, request.knnQueryOptions)
-                && Objects.equals(embeddingConfig, request.embeddingConfig)
-                && Objects.equals(fetchSource, request.fetchSource)
-                && Objects.equals(fields, request.fields)
-                && Objects.equals(docValueFields, request.docValueFields)
-                && Objects.equals(storedFields, request.storedFields)
-                && size == request.size;
-        }
-
-        @Override
-        public int hashCode() {
-            int result = Objects.hash(
-                routing,
-                queryString,
-                modelId,
-                inferenceTimeout,
-                query,
-                knnQueryOptions,
-                embeddingConfig,
-                fetchSource,
-                fields,
-                docValueFields,
-                storedFields,
-                size
-            );
-            result = 31 * result + Arrays.hashCode(indices);
-            return result;
-        }
-
-        @Override
-        public ActionRequestValidationException validate() {
-            ActionRequestValidationException error = new ActionRequestValidationException();
-            if (queryString == null) {
-                error.addValidationError("query_string cannot be null");
-            }
-            if (modelId == null) {
-                error.addValidationError("model_id cannot be null");
-            }
-            if (knnQueryOptions == null) {
-                error.addValidationError("knn cannot be null");
-            }
-
-            return error.validationErrors().isEmpty() ? null : error;
-        }
-
-        public static class Builder {
-
-            private final String[] indices;
-            private String routing;
-            private String modelId;
-            private String queryString;
-            private TimeValue timeout;
-            private TextEmbeddingConfigUpdate update;
-            private QueryBuilder queryBuilder;
-            private KnnQueryOptions knnSearchBuilder;
-            private FetchSourceContext fetchSource;
-            private List<FieldAndFormat> fields;
-            private List<FieldAndFormat> docValueFields;
-            private StoredFieldsContext storedFields;
-            private int size = -1;
-
-            Builder(String[] indices) {
-                this.indices = Objects.requireNonNull(indices, "[indices] must not be null");
-            }
-
-            void setRouting(String routing) {
-                this.routing = routing;
-            }
-
-            void setModelId(String modelId) {
-                this.modelId = modelId;
-            }
-
-            void setQueryString(String queryString) {
-                this.queryString = queryString;
-            }
-
-            void setTimeout(TimeValue timeout) {
-                this.timeout = timeout;
-            }
-
-            void setTimeout(String timeout) {
-                setTimeout(TimeValue.parseTimeValue(timeout, SearchSourceBuilder.TIMEOUT_FIELD.getPreferredName()));
-            }
-
-            void setUpdate(TextEmbeddingConfigUpdate update) {
-                this.update = update;
-            }
-
-            void setKnnSearch(KnnQueryOptions knnSearchBuilder) {
-                this.knnSearchBuilder = knnSearchBuilder;
-            }
-
-            void setQueryBuilder(QueryBuilder queryBuilder) {
-                this.queryBuilder = queryBuilder;
-            }
-
-            private void setFetchSource(FetchSourceContext fetchSource) {
-                this.fetchSource = fetchSource;
-            }
-
-            private void setFields(List<FieldAndFormat> fields) {
-                this.fields = fields;
-            }
-
-            private void setDocValueFields(List<FieldAndFormat> docValueFields) {
-                this.docValueFields = docValueFields;
-            }
-
-            private void setStoredFields(StoredFieldsContext storedFields) {
-                this.storedFields = storedFields;
-            }
-
-            private void setSize(int size) {
-                this.size = size;
-            }
-
-            Request build() {
-                return new Request(
-                    indices,
-                    routing,
-                    queryString,
-                    modelId,
-                    queryBuilder,
-                    knnSearchBuilder,
-                    update,
-                    timeout,
-                    fetchSource,
-                    fields,
-                    docValueFields,
-                    storedFields,
-                    size
-                );
-            }
-        }
-    }
-
-    public static class Response extends ActionResponse implements ToXContentObject {
-
-        private final TimeValue searchTook;
-        private final TimeValue inferenceTook;
-        private final SearchResponse searchResponse;
-
-        public Response(TimeValue searchTook, TimeValue inferenceTook, SearchResponse searchResponse) {
-            this.searchTook = searchTook;
-            this.inferenceTook = inferenceTook;
-            this.searchResponse = searchResponse;
-        }
-
-        public Response(StreamInput in) throws IOException {
-            searchTook = in.readTimeValue();
-            inferenceTook = in.readTimeValue();
-            searchResponse = new SearchResponse(in);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeTimeValue(searchTook);
-            out.writeTimeValue(inferenceTook);
-            searchResponse.writeTo(out);
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            builder.field("inference_took", inferenceTook.millis());
-            searchResponse.innerToXContent(builder, params);
-            builder.endObject();
-            return builder;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            Response response1 = (Response) o;
-            return Objects.equals(searchTook, response1.searchTook)
-                && Objects.equals(inferenceTook, response1.inferenceTook)
-                && Objects.equals(searchResponse, response1.searchResponse);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(searchTook, inferenceTook, searchResponse);
-        }
-    }
-
-    public static class KnnQueryOptions implements Writeable {
-
-        private static final ConstructingObjectParser<KnnQueryOptions, Void> PARSER = new ConstructingObjectParser<>(
-            "knn",
-            true,
-            args -> new KnnQueryOptions((String) args[0], (int) args[1], (int) args[2])
-        );
-
-        static {
-            PARSER.declareString(constructorArg(), KnnSearchBuilder.FIELD_FIELD);
-            PARSER.declareInt(constructorArg(), KnnSearchBuilder.K_FIELD);
-            PARSER.declareInt(constructorArg(), KnnSearchBuilder.NUM_CANDS_FIELD);
-            PARSER.declareFieldArray(
-                KnnQueryOptions::addFilterQueries,
-                (p, c) -> AbstractQueryBuilder.parseTopLevelQuery(p),
-                KnnSearchBuilder.FILTER_FIELD,
-                ObjectParser.ValueType.OBJECT_ARRAY
-            );
-            PARSER.declareFloat(KnnQueryOptions::boost, KnnSearchBuilder.BOOST_FIELD);
-        }
-
-        public static KnnQueryOptions fromXContent(XContentParser parser) throws IOException {
-            return PARSER.parse(parser, null);
-        }
-
-        final String field;
-        final int k;
-        final int numCands;
-        final List<QueryBuilder> filterQueries;
-        float boost = AbstractQueryBuilder.DEFAULT_BOOST;
-
-        /**
-         * Holder for the knn search options.
-         * Validation occurs in when this is converted to a {@code KnnSearchBuilder}
-         *
-         * @param field       the name of the vector field to search against
-         * @param k           the final number of nearest neighbors to return as top hits
-         * @param numCands    the number of nearest neighbor candidates to consider per shard
-         */
-        public KnnQueryOptions(String field, int k, int numCands) {
-            this.field = field;
-            this.k = k;
-            this.numCands = numCands;
-            this.filterQueries = new ArrayList<>();
-        }
-
-        public KnnQueryOptions(StreamInput in) throws IOException {
-            this.field = in.readString();
-            this.k = in.readVInt();
-            this.numCands = in.readVInt();
-            this.filterQueries = in.readNamedWriteableList(QueryBuilder.class);
-            this.boost = in.readFloat();
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeString(field);
-            out.writeVInt(k);
-            out.writeVInt(numCands);
-            out.writeNamedWriteableList(filterQueries);
-            out.writeFloat(boost);
-        }
-
-        public void addFilterQueries(List<QueryBuilder> filterQueries) {
-            Objects.requireNonNull(filterQueries);
-            this.filterQueries.addAll(filterQueries);
-        }
-
-        /**
-         * Set a boost to apply to the kNN search scores.
-         */
-        public void boost(float boost) {
-            this.boost = boost;
-        }
-
-        public KnnSearchBuilder toKnnSearchBuilder(float[] queryVector) {
-            if (queryVector == null) {
-                throw new IllegalStateException("[query_vector] not set on the Knn query");
-            }
-            var builder = new KnnSearchBuilder(field, queryVector, k, numCands);
-            builder.boost(boost);
-            if (filterQueries.isEmpty() == false) {
-                builder.addFilterQueries(filterQueries);
-            }
-            return builder;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            KnnQueryOptions that = (KnnQueryOptions) o;
-            return k == that.k
-                && numCands == that.numCands
-                && Float.compare(that.boost, boost) == 0
-                && Objects.equals(field, that.field)
-                && Objects.equals(filterQueries, that.filterQueries);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(field, k, numCands, filterQueries, boost);
-        }
-    }
-
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsAction.java
index fec7800e6b69..363eca53fc21 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsAction.java
@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
@@ -200,8 +201,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return VERSION_INTRODUCED;
+        public TransportVersion getMinimalSupportedVersion() {
+            return VERSION_INTRODUCED.transportVersion;
         }
 
         @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java
index bef3f0c44485..d2da7ebb1f0e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.core.ml.action;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ValidateActions;
@@ -275,8 +275,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT.minimumCompatibilityVersion();
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT.minimumCompatibilityVersion();
         }
 
         @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java
index ddb01225e666..7249fd65c00a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.action;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
@@ -47,7 +48,7 @@ public class StartTrainedModelDeploymentAction extends ActionType<CreateTrainedM
     public static final StartTrainedModelDeploymentAction INSTANCE = new StartTrainedModelDeploymentAction();
     public static final String NAME = "cluster:admin/xpack/ml/trained_models/deployment/start";
 
-    public static final TimeValue DEFAULT_TIMEOUT = new TimeValue(20, TimeUnit.SECONDS);
+    public static final TimeValue DEFAULT_TIMEOUT = new TimeValue(30, TimeUnit.SECONDS);
 
     /**
      * This has been found to be approximately 300MB on linux by manual testing.
@@ -139,10 +140,10 @@ public Request(StreamInput in) throws IOException {
             numberOfAllocations = in.readVInt();
             threadsPerAllocation = in.readVInt();
             queueCapacity = in.readVInt();
-            if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
                 this.cacheSize = in.readOptionalWriteable(ByteSizeValue::readFrom);
             }
-            if (in.getVersion().onOrAfter(Version.V_8_6_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_6_0)) {
                 this.priority = in.readEnum(Priority.class);
             } else {
                 this.priority = Priority.NORMAL;
@@ -223,10 +224,10 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeVInt(numberOfAllocations);
             out.writeVInt(threadsPerAllocation);
             out.writeVInt(queueCapacity);
-            if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
                 out.writeOptionalWriteable(cacheSize);
             }
-            if (out.getVersion().onOrAfter(Version.V_8_6_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_6_0)) {
                 out.writeEnum(priority);
             }
         }
@@ -447,12 +448,12 @@ public TaskParams(StreamInput in) throws IOException {
             this.threadsPerAllocation = in.readVInt();
             this.numberOfAllocations = in.readVInt();
             this.queueCapacity = in.readVInt();
-            if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
                 this.cacheSize = in.readOptionalWriteable(ByteSizeValue::readFrom);
             } else {
                 this.cacheSize = null;
             }
-            if (in.getVersion().onOrAfter(Version.V_8_6_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_6_0)) {
                 this.priority = in.readEnum(Priority.class);
             } else {
                 this.priority = Priority.NORMAL;
@@ -483,10 +484,10 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeVInt(threadsPerAllocation);
             out.writeVInt(numberOfAllocations);
             out.writeVInt(queueCapacity);
-            if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
                 out.writeOptionalWriteable(cacheSize);
             }
-            if (out.getVersion().onOrAfter(Version.V_8_6_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_6_0)) {
                 out.writeEnum(priority);
             }
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/AggProvider.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/AggProvider.java
index 20906b2393ce..d5e39cf75ec2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/AggProvider.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/AggProvider.java
@@ -8,7 +8,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -117,7 +117,7 @@ static AggProvider fromStream(StreamInput in) throws IOException {
             in.readMap(),
             in.readOptionalWriteable(AggregatorFactories.Builder::new),
             in.readException(),
-            in.getVersion().onOrAfter(Version.V_8_0_0) ? in.readBoolean() : false
+            in.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0) ? in.readBoolean() : false
         );
     }
 
@@ -140,7 +140,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeGenericMap(aggs);
         out.writeOptionalWriteable(parsedAggs);
         out.writeException(parsingException);
-        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             out.writeBoolean(rewroteAggs);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlInferenceNamedXContentProvider.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlInferenceNamedXContentProvider.java
index bc0ae34aed0c..df6d564596c0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlInferenceNamedXContentProvider.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlInferenceNamedXContentProvider.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.xpack.core.ml.inference.results.PyTorchPassThroughResults;
 import org.elasticsearch.xpack.core.ml.inference.results.QuestionAnsweringInferenceResults;
 import org.elasticsearch.xpack.core.ml.inference.results.RegressionInferenceResults;
+import org.elasticsearch.xpack.core.ml.inference.results.SlimResults;
 import org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingResults;
 import org.elasticsearch.xpack.core.ml.inference.results.TextSimilarityInferenceResults;
 import org.elasticsearch.xpack.core.ml.inference.results.WarningInferenceResults;
@@ -56,6 +57,8 @@
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ResultsFieldUpdate;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RobertaTokenization;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RobertaTokenizationUpdate;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.SlimConfig;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.SlimConfigUpdate;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.StrictlyParsedInferenceConfig;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.StrictlyParsedTrainedModel;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.StrictlyParsedTrainedModelLocation;
@@ -315,6 +318,20 @@ public List<NamedXContentRegistry.Entry> getNamedXContentParsers() {
                 FillMaskConfig::fromXContentStrict
             )
         );
+        namedXContent.add(
+            new NamedXContentRegistry.Entry(
+                LenientlyParsedInferenceConfig.class,
+                new ParseField(SlimConfig.NAME),
+                SlimConfig::fromXContentLenient
+            )
+        );
+        namedXContent.add(
+            new NamedXContentRegistry.Entry(
+                StrictlyParsedInferenceConfig.class,
+                new ParseField(SlimConfig.NAME),
+                SlimConfig::fromXContentStrict
+            )
+        );
         namedXContent.add(
             new NamedXContentRegistry.Entry(
                 LenientlyParsedInferenceConfig.class,
@@ -436,6 +453,13 @@ public List<NamedXContentRegistry.Entry> getNamedXContentParsers() {
                 RegressionConfigUpdate::fromXContentStrict
             )
         );
+        namedXContent.add(
+            new NamedXContentRegistry.Entry(
+                InferenceConfigUpdate.class,
+                new ParseField(SlimConfigUpdate.NAME),
+                SlimConfigUpdate::fromXContentStrict
+            )
+        );
         namedXContent.add(
             new NamedXContentRegistry.Entry(
                 InferenceConfigUpdate.class,
@@ -588,6 +612,7 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         namedWriteables.add(
             new NamedWriteableRegistry.Entry(InferenceResults.class, PyTorchPassThroughResults.NAME, PyTorchPassThroughResults::new)
         );
+        namedWriteables.add(new NamedWriteableRegistry.Entry(InferenceResults.class, SlimResults.NAME, SlimResults::new));
         namedWriteables.add(new NamedWriteableRegistry.Entry(InferenceResults.class, TextEmbeddingResults.NAME, TextEmbeddingResults::new));
         namedWriteables.add(
             new NamedWriteableRegistry.Entry(
@@ -619,6 +644,7 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         );
         namedWriteables.add(new NamedWriteableRegistry.Entry(InferenceConfig.class, NerConfig.NAME, NerConfig::new));
         namedWriteables.add(new NamedWriteableRegistry.Entry(InferenceConfig.class, FillMaskConfig.NAME, FillMaskConfig::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(InferenceConfig.class, SlimConfig.NAME, SlimConfig::new));
         namedWriteables.add(
             new NamedWriteableRegistry.Entry(InferenceConfig.class, TextClassificationConfig.NAME, TextClassificationConfig::new)
         );
@@ -658,6 +684,7 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         namedWriteables.add(
             new NamedWriteableRegistry.Entry(InferenceConfigUpdate.class, ResultsFieldUpdate.NAME, ResultsFieldUpdate::new)
         );
+        namedWriteables.add(new NamedWriteableRegistry.Entry(InferenceConfigUpdate.class, SlimConfigUpdate.NAME, SlimConfigUpdate::new));
         namedWriteables.add(
             new NamedWriteableRegistry.Entry(
                 InferenceConfigUpdate.class,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java
index 1fed8a85cec1..5ca9e7c02826 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java
@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ml.inference;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.Strings;
@@ -246,7 +247,7 @@ public TrainedModelConfig(StreamInput in) throws IOException {
         this.defaultFieldMap = in.readBoolean() ? in.readImmutableMap(StreamInput::readString, StreamInput::readString) : null;
 
         this.inferenceConfig = in.readOptionalNamedWriteable(InferenceConfig.class);
-        if (in.getVersion().onOrAfter(VERSION_3RD_PARTY_CONFIG_ADDED)) {
+        if (in.getTransportVersion().onOrAfter(VERSION_3RD_PARTY_CONFIG_ADDED.transportVersion)) {
             this.modelType = in.readOptionalEnum(TrainedModelType.class);
             this.location = in.readOptionalNamedWriteable(TrainedModelLocation.class);
         } else {
@@ -392,7 +393,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(false);
         }
         out.writeOptionalNamedWriteable(inferenceConfig);
-        if (out.getVersion().onOrAfter(VERSION_3RD_PARTY_CONFIG_ADDED)) {
+        if (out.getTransportVersion().onOrAfter(VERSION_3RD_PARTY_CONFIG_ADDED.transportVersion)) {
             out.writeOptionalEnum(modelType);
             out.writeOptionalNamedWriteable(location);
         }
@@ -894,7 +895,7 @@ public static LazyModelDefinition fromBase64String(String base64String) {
         }
 
         public static LazyModelDefinition fromStreamInput(StreamInput input) throws IOException {
-            if (input.getVersion().onOrAfter(Version.V_8_0_0)) {
+            if (input.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
                 return new LazyModelDefinition(input.readBytesReference(), null);
             } else {
                 return fromBase64String(input.readString());
@@ -958,7 +959,7 @@ private void ensureParsedDefinitionUnsafe(NamedXContentRegistry xContentRegistry
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
                 out.writeBytesReference(getCompressedDefinition());
             } else {
                 out.writeString(getBase64CompressedDefinition());
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStats.java
index 05f62f216c10..c04d8312ac7d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStats.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.inference.assignment;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -163,7 +163,7 @@ public NodeStats(StreamInput in) throws IOException {
             this.pendingCount = in.readOptionalVInt();
             this.routingState = in.readOptionalWriteable(RoutingStateAndReason::new);
             this.startTime = in.readOptionalInstant();
-            if (in.getVersion().onOrAfter(Version.V_8_1_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
                 this.threadsPerAllocation = in.readOptionalVInt();
                 this.numberOfAllocations = in.readOptionalVInt();
                 this.errorCount = in.readVInt();
@@ -176,7 +176,7 @@ public NodeStats(StreamInput in) throws IOException {
                 this.rejectedExecutionCount = 0;
                 this.timeoutCount = 0;
             }
-            if (in.getVersion().onOrAfter(Version.V_8_2_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
                 this.peakThroughput = in.readVLong();
                 this.throughputLastPeriod = in.readVLong();
                 this.avgInferenceTimeLastPeriod = in.readOptionalDouble();
@@ -185,14 +185,14 @@ public NodeStats(StreamInput in) throws IOException {
                 this.throughputLastPeriod = 0;
                 this.avgInferenceTimeLastPeriod = null;
             }
-            if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
                 this.cacheHitCount = in.readOptionalVLong();
                 this.cacheHitCountLastPeriod = in.readOptionalVLong();
             } else {
                 this.cacheHitCount = null;
                 this.cacheHitCountLastPeriod = null;
             }
-            if (in.getVersion().onOrAfter(Version.V_8_5_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_5_0)) {
                 this.avgInferenceTimeExcludingCacheHit = in.readOptionalDouble();
             } else {
                 this.avgInferenceTimeExcludingCacheHit = null;
@@ -342,23 +342,23 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalVInt(pendingCount);
             out.writeOptionalWriteable(routingState);
             out.writeOptionalInstant(startTime);
-            if (out.getVersion().onOrAfter(Version.V_8_1_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
                 out.writeOptionalVInt(threadsPerAllocation);
                 out.writeOptionalVInt(numberOfAllocations);
                 out.writeVInt(errorCount);
                 out.writeVInt(rejectedExecutionCount);
                 out.writeVInt(timeoutCount);
             }
-            if (out.getVersion().onOrAfter(Version.V_8_2_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
                 out.writeVLong(peakThroughput);
                 out.writeVLong(throughputLastPeriod);
                 out.writeOptionalDouble(avgInferenceTimeLastPeriod);
             }
-            if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
                 out.writeOptionalVLong(cacheHitCount);
                 out.writeOptionalVLong(cacheHitCountLastPeriod);
             }
-            if (out.getVersion().onOrAfter(Version.V_8_5_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_5_0)) {
                 out.writeOptionalDouble(avgInferenceTimeExcludingCacheHit);
             }
         }
@@ -461,12 +461,12 @@ public AssignmentStats(StreamInput in) throws IOException {
         state = in.readOptionalEnum(AssignmentState.class);
         reason = in.readOptionalString();
         allocationStatus = in.readOptionalWriteable(AllocationStatus::new);
-        if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             cacheSize = in.readOptionalWriteable(ByteSizeValue::readFrom);
         } else {
             cacheSize = null;
         }
-        if (in.getVersion().onOrAfter(Version.V_8_6_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_6_0)) {
             priority = in.readEnum(Priority.class);
         } else {
             priority = Priority.NORMAL;
@@ -620,17 +620,17 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalVInt(queueCapacity);
         out.writeInstant(startTime);
         out.writeList(nodeStats);
-        if (AssignmentState.FAILED.equals(state) && out.getVersion().before(Version.V_8_4_0)) {
+        if (AssignmentState.FAILED.equals(state) && out.getTransportVersion().before(TransportVersion.V_8_4_0)) {
             out.writeOptionalEnum(AssignmentState.STARTING);
         } else {
             out.writeOptionalEnum(state);
         }
         out.writeOptionalString(reason);
         out.writeOptionalWriteable(allocationStatus);
-        if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             out.writeOptionalWriteable(cacheSize);
         }
-        if (out.getVersion().onOrAfter(Version.V_8_6_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_6_0)) {
             out.writeEnum(priority);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingInfo.java
index ab6ae05f3e0c..14886f42be87 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingInfo.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingInfo.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.inference.assignment;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -67,7 +67,7 @@ public RoutingInfo(int currentAllocations, int targetAllocations, RoutingState s
     }
 
     public RoutingInfo(StreamInput in) throws IOException {
-        if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             this.currentAllocations = in.readVInt();
             this.targetAllocations = in.readVInt();
         } else {
@@ -101,7 +101,7 @@ public boolean isOutdated() {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             out.writeVInt(currentAllocations);
             out.writeVInt(targetAllocations);
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingInfoUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingInfoUpdate.java
index ce08941dfe02..17195c2203da 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingInfoUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingInfoUpdate.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.inference.assignment;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -35,7 +35,7 @@ private RoutingInfoUpdate(Optional<Integer> numberOfAllocations, Optional<Routin
     }
 
     public RoutingInfoUpdate(StreamInput in) throws IOException {
-        if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             numberOfAllocations = Optional.ofNullable(in.readOptionalVInt());
             stateAndReason = Optional.ofNullable(in.readOptionalWriteable(RoutingStateAndReason::new));
         } else {
@@ -46,7 +46,7 @@ public RoutingInfoUpdate(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             out.writeOptionalVInt(numberOfAllocations.orElse(null));
             out.writeOptionalWriteable(stateAndReason.orElse(null));
         } else {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java
index 901416a71b51..9e71070bc4d4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignment.java
@@ -9,11 +9,12 @@
 
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.SimpleDiffable;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -142,7 +143,7 @@ public TrainedModelAssignment(StreamInput in) throws IOException {
         this.assignmentState = in.readEnum(AssignmentState.class);
         this.reason = in.readOptionalString();
         this.startTime = in.readInstant();
-        if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             this.maxAssignedAllocations = in.readVInt();
         } else {
             this.maxAssignedAllocations = totalCurrentAllocations();
@@ -177,7 +178,7 @@ public String[] getStartedNodes() {
             .toArray(String[]::new);
     }
 
-    public Optional<String> selectRandomStartedNodeWeighedOnAllocations() {
+    public List<Tuple<String, Integer>> selectRandomStartedNodesWeighedOnAllocationsForNRequests(int numberOfRequests) {
         List<String> nodeIds = new ArrayList<>(nodeRoutingTable.size());
         List<Integer> cumulativeAllocations = new ArrayList<>(nodeRoutingTable.size());
         int allocationSum = 0;
@@ -189,18 +190,42 @@ public Optional<String> selectRandomStartedNodeWeighedOnAllocations() {
             }
         }
 
+        if (nodeIds.isEmpty()) {
+            return List.of();
+        }
+
         if (allocationSum == 0) {
             // If we are in a mixed cluster where there are assignments prior to introducing allocation distribution
             // we could have a zero-sum of allocations. We fall back to returning a random started node.
-            return nodeIds.isEmpty() ? Optional.empty() : Optional.of(nodeIds.get(Randomness.get().nextInt(nodeIds.size())));
+            int[] counts = new int[nodeIds.size()];
+            for (int i = 0; i < numberOfRequests; i++) {
+                counts[Randomness.get().nextInt(nodeIds.size())]++;
+            }
+
+            var nodeCounts = new ArrayList<Tuple<String, Integer>>();
+            for (int i = 0; i < counts.length; i++) {
+                nodeCounts.add(new Tuple<>(nodeIds.get(i), counts[i]));
+            }
+            return nodeCounts;
+        }
+
+        int[] counts = new int[nodeIds.size()];
+        var randomIter = Randomness.get().ints(numberOfRequests, 1, allocationSum + 1).iterator();
+        for (int i = 0; i < numberOfRequests; i++) {
+            int randomInt = randomIter.nextInt();
+            int nodeIndex = Collections.binarySearch(cumulativeAllocations, randomInt);
+            if (nodeIndex < 0) {
+                nodeIndex = -nodeIndex - 1;
+            }
+
+            counts[nodeIndex]++;
         }
 
-        int randomInt = Randomness.get().ints(1, 1, allocationSum + 1).iterator().nextInt();
-        int nodeIndex = Collections.binarySearch(cumulativeAllocations, randomInt);
-        if (nodeIndex < 0) {
-            nodeIndex = -nodeIndex - 1;
+        var nodeCounts = new ArrayList<Tuple<String, Integer>>();
+        for (int i = 0; i < counts.length; i++) {
+            nodeCounts.add(new Tuple<>(nodeIds.get(i), counts[i]));
         }
-        return Optional.of(nodeIds.get(nodeIndex));
+        return nodeCounts;
     }
 
     public Optional<String> getReason() {
@@ -277,7 +302,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeEnum(assignmentState);
         out.writeOptionalString(reason);
         out.writeInstant(startTime);
-        if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             out.writeVInt(maxAssignedAllocations);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/NlpInferenceResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/NlpInferenceResults.java
index b11efe8ee7ab..33ef519f5e41 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/NlpInferenceResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/NlpInferenceResults.java
@@ -35,6 +35,10 @@ abstract class NlpInferenceResults implements InferenceResults {
 
     abstract void addMapFields(Map<String, Object> map);
 
+    public boolean isTruncated() {
+        return isTruncated;
+    }
+
     @Override
     public final void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(isTruncated);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/SlimResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/SlimResults.java
new file mode 100644
index 000000000000..beb4e2c790ef
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/SlimResults.java
@@ -0,0 +1,128 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.ml.inference.results;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+public class SlimResults extends NlpInferenceResults {
+
+    public static final String NAME = "slim_result";
+
+    public record WeightedToken(int token, float weight) implements Writeable, ToXContentObject {
+
+        public static final String TOKEN = "token";
+        public static final String WEIGHT = "weight";
+
+        public WeightedToken(StreamInput in) throws IOException {
+            this(in.readVInt(), in.readFloat());
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeVInt(token);
+            out.writeFloat(weight);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(TOKEN, token);
+            builder.field(WEIGHT, weight);
+            builder.endObject();
+            return builder;
+        }
+
+        public Map<String, Object> asMap() {
+            return Map.of(TOKEN, token, WEIGHT, weight);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+
+    private final String resultsField;
+    private final List<WeightedToken> weightedTokens;
+
+    public SlimResults(String resultField, List<WeightedToken> weightedTokens, boolean isTruncated) {
+        super(isTruncated);
+        this.resultsField = resultField;
+        this.weightedTokens = weightedTokens;
+    }
+
+    public SlimResults(StreamInput in) throws IOException {
+        super(in);
+        this.resultsField = in.readString();
+        this.weightedTokens = in.readList(WeightedToken::new);
+    }
+
+    public List<WeightedToken> getWeightedTokens() {
+        return weightedTokens;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public String getResultsField() {
+        return resultsField;
+    }
+
+    @Override
+    public Object predictedValue() {
+        throw new UnsupportedOperationException("[" + NAME + "] does not support a single predicted value");
+    }
+
+    @Override
+    void doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        builder.startArray(resultsField);
+        for (var weightedToken : weightedTokens) {
+            weightedToken.toXContent(builder, params);
+        }
+        builder.endArray();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        if (super.equals(o) == false) return false;
+        SlimResults that = (SlimResults) o;
+        return Objects.equals(resultsField, that.resultsField) && Objects.equals(weightedTokens, that.weightedTokens);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), resultsField, weightedTokens);
+    }
+
+    @Override
+    void doWriteTo(StreamOutput out) throws IOException {
+        out.writeString(resultsField);
+        out.writeList(weightedTokens);
+    }
+
+    @Override
+    void addMapFields(Map<String, Object> map) {
+        map.put(resultsField, weightedTokens.stream().map(WeightedToken::asMap).collect(Collectors.toList()));
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/AbstractTokenizationUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/AbstractTokenizationUpdate.java
index b11865768f3f..263083fa26be 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/AbstractTokenizationUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/AbstractTokenizationUpdate.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
@@ -34,7 +34,7 @@ public AbstractTokenizationUpdate(@Nullable Tokenization.Truncate truncate, @Nul
 
     public AbstractTokenizationUpdate(StreamInput in) throws IOException {
         this.truncate = in.readOptionalEnum(Tokenization.Truncate.class);
-        if (in.getVersion().onOrAfter(Version.V_8_2_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
             this.span = in.readOptionalInt();
         } else {
             this.span = null;
@@ -62,7 +62,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalEnum(truncate);
-        if (out.getVersion().onOrAfter(Version.V_8_2_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
             out.writeOptionalInt(span);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfig.java
index f05cef7f4a67..73c4f6521dc5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfig.java
@@ -192,7 +192,7 @@ public boolean isTargetTypeSupported(TargetType targetType) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
+    public Version getMinimalSupportedNodeVersion() {
         return requestingImportance() ? Version.V_7_7_0 : MIN_SUPPORTED_VERSION;
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdate.java
index 9a26d10702e5..07877776fd69 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdate.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.ObjectParser;
@@ -249,8 +249,8 @@ boolean isNoop(ClassificationConfig originalConfig) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_8_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_8_0;
     }
 
     public static class Builder implements InferenceConfigUpdate.Builder<Builder, ClassificationConfigUpdate> {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/EmptyConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/EmptyConfigUpdate.java
index 58fb78d53b8b..6054f829d35b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/EmptyConfigUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/EmptyConfigUpdate.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -66,8 +67,8 @@ public int hashCode() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_9_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_9_0;
     }
 
     public static class Builder implements InferenceConfigUpdate.Builder<Builder, EmptyConfigUpdate> {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/FillMaskConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/FillMaskConfig.java
index deb6287f3274..4b05fc8ffe87 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/FillMaskConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/FillMaskConfig.java
@@ -123,7 +123,7 @@ public boolean isTargetTypeSupported(TargetType targetType) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
+    public Version getMinimalSupportedNodeVersion() {
         return Version.V_8_0_0;
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/FillMaskConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/FillMaskConfigUpdate.java
index da0beaa6785d..1518db7df713 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/FillMaskConfigUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/FillMaskConfigUpdate.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.ObjectParser;
@@ -103,8 +103,8 @@ public String getName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_0_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_0_0;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceConfig.java
index e3fe1a6d6576..3ae273501b1c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceConfig.java
@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.VersionedNamedWriteable;
 import org.elasticsearch.xpack.core.ml.utils.NamedXContentObject;
@@ -17,10 +18,21 @@ public interface InferenceConfig extends NamedXContentObject, VersionedNamedWrit
 
     boolean isTargetTypeSupported(TargetType targetType);
 
+    @Override
+    default TransportVersion getMinimalSupportedVersion() {
+        /*
+         * TODO: This method existed before it inherited from VersionedNamedWriteable,
+         *  so we need to look at this closely when we migrate the bulk of the ML code,
+         *  whether this actually means the transport version or the actual node version
+         *  on the other end
+         */
+        return getMinimalSupportedNodeVersion().transportVersion;
+    }
+
     /**
      * All nodes in the cluster must be at least this version
      */
-    Version getMinimalSupportedVersion();
+    Version getMinimalSupportedNodeVersion();
 
     default boolean requestingImportance() {
         return false;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NerConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NerConfig.java
index 4967b45500f3..cad0ac9eacc5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NerConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NerConfig.java
@@ -156,7 +156,7 @@ public boolean isTargetTypeSupported(TargetType targetType) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
+    public Version getMinimalSupportedNodeVersion() {
         return Version.V_8_0_0;
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NerConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NerConfigUpdate.java
index d52463eb092c..8a685e46733a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NerConfigUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NerConfigUpdate.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.ObjectParser;
@@ -146,8 +146,8 @@ public int hashCode() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_0_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_0_0;
     }
 
     public static class Builder implements InferenceConfigUpdate.Builder<NerConfigUpdate.Builder, NerConfigUpdate> {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NlpConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NlpConfigUpdate.java
index 66cfcd683b56..e15de2c249e0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NlpConfigUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NlpConfigUpdate.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
@@ -78,7 +78,7 @@ public NlpConfigUpdate(@Nullable TokenizationUpdate tokenizationUpdate) {
     }
 
     public NlpConfigUpdate(StreamInput in) throws IOException {
-        if (in.getVersion().onOrAfter(Version.V_8_1_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
             tokenizationUpdate = in.readOptionalNamedWriteable(TokenizationUpdate.class);
         } else {
             tokenizationUpdate = null;
@@ -87,7 +87,7 @@ public NlpConfigUpdate(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getVersion().onOrAfter(Version.V_8_1_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
             out.writeOptionalNamedWriteable(tokenizationUpdate);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NullInferenceConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NullInferenceConfig.java
index 71498ca9fb30..ccacfc2e834d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NullInferenceConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NullInferenceConfig.java
@@ -29,7 +29,7 @@ public boolean isTargetTypeSupported(TargetType targetType) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
+    public Version getMinimalSupportedNodeVersion() {
         return Version.CURRENT;
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PassThroughConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PassThroughConfig.java
index 854544065de6..356fe6d7af8a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PassThroughConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PassThroughConfig.java
@@ -119,7 +119,7 @@ public boolean isTargetTypeSupported(TargetType targetType) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
+    public Version getMinimalSupportedNodeVersion() {
         return Version.V_8_0_0;
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PassThroughConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PassThroughConfigUpdate.java
index 92bfe1bf9ea7..736cfb864461 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PassThroughConfigUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PassThroughConfigUpdate.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.ObjectParser;
@@ -148,8 +148,8 @@ public int hashCode() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_0_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_0_0;
     }
 
     public static class Builder implements InferenceConfigUpdate.Builder<PassThroughConfigUpdate.Builder, PassThroughConfigUpdate> {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/QuestionAnsweringConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/QuestionAnsweringConfig.java
index 0ecbcdf3bee3..c5b52fc9f11f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/QuestionAnsweringConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/QuestionAnsweringConfig.java
@@ -187,7 +187,7 @@ public boolean isTargetTypeSupported(TargetType targetType) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
+    public Version getMinimalSupportedNodeVersion() {
         return Version.V_8_3_0;
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/QuestionAnsweringConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/QuestionAnsweringConfigUpdate.java
index 1ffdd337aaeb..6e980564ba27 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/QuestionAnsweringConfigUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/QuestionAnsweringConfigUpdate.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
@@ -252,7 +252,7 @@ public QuestionAnsweringConfigUpdate build() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_3_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_3_0;
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfig.java
index d50cf82ce7b5..d57cca7e04c8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfig.java
@@ -134,7 +134,7 @@ public boolean isTargetTypeSupported(TargetType targetType) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
+    public Version getMinimalSupportedNodeVersion() {
         return requestingImportance() ? Version.V_7_7_0 : MIN_SUPPORTED_VERSION;
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdate.java
index c655c391317f..4c187ef28416 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdate.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.ObjectParser;
@@ -112,8 +112,8 @@ public String getName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_8_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_8_0;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ResultsFieldUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ResultsFieldUpdate.java
index 0eaf101c25c8..514c48e6c42f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ResultsFieldUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ResultsFieldUpdate.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
@@ -70,8 +70,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_9_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_9_0;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/SlimConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/SlimConfig.java
new file mode 100644
index 000000000000..1c7a96f3e542
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/SlimConfig.java
@@ -0,0 +1,175 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.ml.utils.NamedXContentObjectHelper;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.Optional;
+
+public class SlimConfig implements NlpConfig {
+
+    public static final String NAME = "slim";
+
+    public static SlimConfig fromXContentStrict(XContentParser parser) {
+        return STRICT_PARSER.apply(parser, null);
+    }
+
+    public static SlimConfig fromXContentLenient(XContentParser parser) {
+        return LENIENT_PARSER.apply(parser, null);
+    }
+
+    private static final ConstructingObjectParser<SlimConfig, Void> STRICT_PARSER = createParser(false);
+    private static final ConstructingObjectParser<SlimConfig, Void> LENIENT_PARSER = createParser(true);
+
+    private static ConstructingObjectParser<SlimConfig, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<SlimConfig, Void> parser = new ConstructingObjectParser<>(
+            NAME,
+            ignoreUnknownFields,
+            a -> new SlimConfig((VocabularyConfig) a[0], (Tokenization) a[1], (String) a[2])
+        );
+        parser.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
+            if (ignoreUnknownFields == false) {
+                throw ExceptionsHelper.badRequestException(
+                    "illegal setting [{}] on inference model creation",
+                    VOCABULARY.getPreferredName()
+                );
+            }
+            return VocabularyConfig.fromXContentLenient(p);
+        }, VOCABULARY);
+        parser.declareNamedObject(
+            ConstructingObjectParser.optionalConstructorArg(),
+            (p, c, n) -> p.namedObject(Tokenization.class, n, ignoreUnknownFields),
+            TOKENIZATION
+        );
+        parser.declareString(ConstructingObjectParser.optionalConstructorArg(), RESULTS_FIELD);
+        return parser;
+    }
+
+    private final VocabularyConfig vocabularyConfig;
+    private final Tokenization tokenization;
+    private final String resultsField;
+
+    public SlimConfig(@Nullable VocabularyConfig vocabularyConfig, @Nullable Tokenization tokenization, @Nullable String resultsField) {
+        this.vocabularyConfig = Optional.ofNullable(vocabularyConfig)
+            .orElse(new VocabularyConfig(InferenceIndexConstants.nativeDefinitionStore()));
+        this.tokenization = tokenization == null ? Tokenization.createDefault() : tokenization;
+        if (this.tokenization instanceof BertTokenization == false) {
+            throw ExceptionsHelper.badRequestException(
+                "SLIM must be configured with BERT tokenizer, [{}] given",
+                this.tokenization.getName()
+            );
+        }
+        // TODO support spanning
+        if (this.tokenization.span != -1) {
+            throw ExceptionsHelper.badRequestException(
+                "[{}] does not support windowing long text sequences; configured span [{}]",
+                NAME,
+                this.tokenization.span
+            );
+        }
+        this.resultsField = resultsField;
+    }
+
+    public SlimConfig(StreamInput in) throws IOException {
+        vocabularyConfig = new VocabularyConfig(in);
+        tokenization = in.readNamedWriteable(Tokenization.class);
+        resultsField = in.readOptionalString();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        vocabularyConfig.writeTo(out);
+        out.writeNamedWriteable(tokenization);
+        out.writeOptionalString(resultsField);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(VOCABULARY.getPreferredName(), vocabularyConfig, params);
+        NamedXContentObjectHelper.writeNamedObject(builder, params, TOKENIZATION.getPreferredName(), tokenization);
+        if (resultsField != null) {
+            builder.field(RESULTS_FIELD.getPreferredName(), resultsField);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public boolean isTargetTypeSupported(TargetType targetType) {
+        // TargetType relates to boosted tree models
+        return false;
+    }
+
+    @Override
+    public boolean isAllocateOnly() {
+        return true;
+    }
+
+    @Override
+    public Version getMinimalSupportedNodeVersion() {
+        return Version.V_8_7_0;
+    }
+
+    @Override
+    public String getResultsField() {
+        return resultsField;
+    }
+
+    @Override
+    public VocabularyConfig getVocabularyConfig() {
+        return vocabularyConfig;
+    }
+
+    @Override
+    public Tokenization getTokenization() {
+        return tokenization;
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        SlimConfig that = (SlimConfig) o;
+        return Objects.equals(vocabularyConfig, that.vocabularyConfig)
+            && Objects.equals(tokenization, that.tokenization)
+            && Objects.equals(resultsField, that.resultsField);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(vocabularyConfig, tokenization, resultsField);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/SlimConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/SlimConfigUpdate.java
new file mode 100644
index 000000000000..52a24f558bbc
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/SlimConfigUpdate.java
@@ -0,0 +1,180 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xcontent.ObjectParser;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+
+import static org.elasticsearch.xpack.core.ml.inference.trainedmodel.NlpConfig.RESULTS_FIELD;
+import static org.elasticsearch.xpack.core.ml.inference.trainedmodel.NlpConfig.TOKENIZATION;
+
+public class SlimConfigUpdate extends NlpConfigUpdate {
+
+    public static final String NAME = SlimConfig.NAME;
+
+    public static SlimConfigUpdate fromMap(Map<String, Object> map) {
+        Map<String, Object> options = new HashMap<>(map);
+        String resultsField = (String) options.remove(RESULTS_FIELD.getPreferredName());
+        TokenizationUpdate tokenizationUpdate = NlpConfigUpdate.tokenizationFromMap(options);
+
+        if (options.isEmpty() == false) {
+            throw ExceptionsHelper.badRequestException("Unrecognized fields {}.", options.keySet());
+        }
+        return new SlimConfigUpdate(resultsField, tokenizationUpdate);
+    }
+
+    private static final ObjectParser<SlimConfigUpdate.Builder, Void> STRICT_PARSER = createParser(false);
+
+    private static ObjectParser<SlimConfigUpdate.Builder, Void> createParser(boolean lenient) {
+        ObjectParser<SlimConfigUpdate.Builder, Void> parser = new ObjectParser<>(NAME, lenient, SlimConfigUpdate.Builder::new);
+        parser.declareString(SlimConfigUpdate.Builder::setResultsField, RESULTS_FIELD);
+        parser.declareNamedObject(
+            SlimConfigUpdate.Builder::setTokenizationUpdate,
+            (p, c, n) -> p.namedObject(TokenizationUpdate.class, n, lenient),
+            TOKENIZATION
+        );
+        return parser;
+    }
+
+    public static SlimConfigUpdate fromXContentStrict(XContentParser parser) {
+        return STRICT_PARSER.apply(parser, null).build();
+    }
+
+    private final String resultsField;
+
+    public SlimConfigUpdate(String resultsField, TokenizationUpdate tokenizationUpdate) {
+        super(tokenizationUpdate);
+        this.resultsField = resultsField;
+    }
+
+    public SlimConfigUpdate(StreamInput in) throws IOException {
+        super(in);
+        this.resultsField = in.readOptionalString();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeOptionalString(resultsField);
+    }
+
+    @Override
+    public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        if (resultsField != null) {
+            builder.field(RESULTS_FIELD.getPreferredName(), resultsField);
+        }
+        return builder;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    @Override
+    public InferenceConfig apply(InferenceConfig originalConfig) {
+        if (originalConfig instanceof SlimConfig == false) {
+            throw ExceptionsHelper.badRequestException(
+                "Inference config of type [{}] can not be updated with a request of type [{}]",
+                originalConfig.getName(),
+                getName()
+            );
+        }
+        SlimConfig SlimConfig = (SlimConfig) originalConfig;
+        if (isNoop(SlimConfig)) {
+            return SlimConfig;
+        }
+
+        return new SlimConfig(
+            SlimConfig.getVocabularyConfig(),
+            (tokenizationUpdate == null) ? SlimConfig.getTokenization() : tokenizationUpdate.apply(SlimConfig.getTokenization()),
+            Optional.ofNullable(resultsField).orElse(SlimConfig.getResultsField())
+        );
+    }
+
+    boolean isNoop(SlimConfig originalConfig) {
+        return (this.resultsField == null || this.resultsField.equals(originalConfig.getResultsField())) && super.isNoop();
+    }
+
+    @Override
+    public boolean isSupported(InferenceConfig config) {
+        return config instanceof SlimConfig;
+    }
+
+    @Override
+    public String getResultsField() {
+        return resultsField;
+    }
+
+    @Override
+    public InferenceConfigUpdate.Builder<? extends InferenceConfigUpdate.Builder<?, ?>, ? extends InferenceConfigUpdate> newBuilder() {
+        return new SlimConfigUpdate.Builder().setResultsField(resultsField).setTokenizationUpdate(tokenizationUpdate);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        SlimConfigUpdate that = (SlimConfigUpdate) o;
+        return Objects.equals(resultsField, that.resultsField) && Objects.equals(tokenizationUpdate, that.tokenizationUpdate);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(resultsField, tokenizationUpdate);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_7_0;
+    }
+
+    public static class Builder implements InferenceConfigUpdate.Builder<SlimConfigUpdate.Builder, SlimConfigUpdate> {
+        private String resultsField;
+        private TokenizationUpdate tokenizationUpdate;
+
+        @Override
+        public SlimConfigUpdate.Builder setResultsField(String resultsField) {
+            this.resultsField = resultsField;
+            return this;
+        }
+
+        public SlimConfigUpdate.Builder setTokenizationUpdate(TokenizationUpdate tokenizationUpdate) {
+            this.tokenizationUpdate = tokenizationUpdate;
+            return this;
+        }
+
+        @Override
+        public SlimConfigUpdate build() {
+            return new SlimConfigUpdate(resultsField, tokenizationUpdate);
+        }
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextClassificationConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextClassificationConfig.java
index 3c5a5204ea93..221fcc63b6ed 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextClassificationConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextClassificationConfig.java
@@ -132,7 +132,7 @@ public boolean isTargetTypeSupported(TargetType targetType) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
+    public Version getMinimalSupportedNodeVersion() {
         return Version.V_8_0_0;
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextClassificationConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextClassificationConfigUpdate.java
index a62f9e3a197f..584331b6d6cb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextClassificationConfigUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextClassificationConfigUpdate.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.ObjectParser;
@@ -98,8 +98,8 @@ public String getName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_0_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_0_0;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfig.java
index ba6aeac3ecd5..45aa5967be4d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfig.java
@@ -119,7 +119,7 @@ public boolean isTargetTypeSupported(TargetType targetType) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
+    public Version getMinimalSupportedNodeVersion() {
         return Version.V_8_0_0;
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigUpdate.java
index 589b71bd631d..e9fdf4f7383d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigUpdate.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.ObjectParser;
@@ -28,6 +28,8 @@ public class TextEmbeddingConfigUpdate extends NlpConfigUpdate implements NamedX
 
     public static final String NAME = TextEmbeddingConfig.NAME;
 
+    public static TextEmbeddingConfigUpdate EMPTY_INSTANCE = new TextEmbeddingConfigUpdate(null, null);
+
     public static TextEmbeddingConfigUpdate fromMap(Map<String, Object> map) {
         Map<String, Object> options = new HashMap<>(map);
         String resultsField = (String) options.remove(RESULTS_FIELD.getPreferredName());
@@ -97,8 +99,8 @@ public String getName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_0_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_0_0;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextSimilarityConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextSimilarityConfig.java
index c1eae964898b..aed51739ff60 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextSimilarityConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextSimilarityConfig.java
@@ -148,7 +148,7 @@ public boolean isTargetTypeSupported(TargetType targetType) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
+    public Version getMinimalSupportedNodeVersion() {
         return Version.V_8_5_0;
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextSimilarityConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextSimilarityConfigUpdate.java
index 48ee53c62d49..bd4788393a40 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextSimilarityConfigUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextSimilarityConfigUpdate.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
@@ -210,7 +210,7 @@ public TextSimilarityConfigUpdate build() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_5_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_5_0;
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/Tokenization.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/Tokenization.java
index b4b1eb9e8472..f96b930f4ac8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/Tokenization.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/Tokenization.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -124,7 +124,7 @@ public Tokenization(StreamInput in) throws IOException {
         this.withSpecialTokens = in.readBoolean();
         this.maxSequenceLength = in.readVInt();
         this.truncate = in.readEnum(Truncate.class);
-        if (in.getVersion().onOrAfter(Version.V_8_2_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
             this.span = in.readInt();
         } else {
             this.span = UNSET_SPAN_VALUE;
@@ -137,7 +137,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(withSpecialTokens);
         out.writeVInt(maxSequenceLength);
         out.writeEnum(truncate);
-        if (out.getVersion().onOrAfter(Version.V_8_2_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
             out.writeInt(span);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ZeroShotClassificationConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ZeroShotClassificationConfig.java
index 710a2855167c..d07b19fc1d56 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ZeroShotClassificationConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ZeroShotClassificationConfig.java
@@ -197,7 +197,7 @@ public boolean isTargetTypeSupported(TargetType targetType) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
+    public Version getMinimalSupportedNodeVersion() {
         return Version.V_8_0_0;
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ZeroShotClassificationConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ZeroShotClassificationConfigUpdate.java
index acfd726ca27a..02ed163d389b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ZeroShotClassificationConfigUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ZeroShotClassificationConfigUpdate.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
@@ -243,7 +243,7 @@ public ZeroShotClassificationConfigUpdate build() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_0_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_0_0;
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DataDescription.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DataDescription.java
index 85712f611ca3..4cd8aae3561d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DataDescription.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DataDescription.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ml.job.config;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -131,12 +131,12 @@ public DataDescription(String timeFieldName, String timeFormat) {
     }
 
     public DataDescription(StreamInput in) throws IOException {
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             DataFormat.readFromStream(in);
         }
         timeFieldName = in.readString();
         timeFormat = in.readString();
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             // fieldDelimiter
             if (in.readBoolean()) {
                 in.read();
@@ -150,12 +150,12 @@ public DataDescription(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             DataFormat.XCONTENT.writeTo(out);
         }
         out.writeString(timeFieldName);
         out.writeString(timeFormat);
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             // fieldDelimiter
             out.writeBoolean(false);
             // quoteCharacter
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
index 895bc8e261d8..e80cac65f871 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
@@ -158,6 +158,7 @@ public final class Messages {
     public static final String JOB_AUDIT_DATAFEED_STARTED_FROM_TO = "Datafeed started (from: {0} to: {1}) with frequency [{2}]";
     public static final String JOB_AUDIT_DATAFEED_STARTED_REALTIME = "Datafeed started in real-time";
     public static final String JOB_AUDIT_DATAFEED_STOPPED = "Datafeed stopped";
+    public static final String JOB_AUDIT_DATAFEED_STOPPED_WITH_REASON = "Datafeed stopped with reason [{0}]";
     public static final String JOB_AUDIT_DATAFEED_ISOLATED = "Datafeed isolated";
     public static final String JOB_AUDIT_DELETING = "Deleting job by task with id ''{0}''";
     public static final String JOB_AUDIT_DELETING_FAILED = "Error deleting job: {0}";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecord.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecord.java
index 8f4afffbd41a..2454a169685e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecord.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecord.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ml.job.results;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -217,7 +217,7 @@ public AnomalyRecord(StreamInput in) throws IOException {
             influences = in.readList(Influence::new);
         }
         geoResults = in.readOptionalWriteable(GeoResults::new);
-        if (in.getVersion().onOrAfter(Version.V_8_6_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_6_0)) {
             anomalyScoreExplanation = in.readOptionalWriteable(AnomalyScoreExplanation::new);
         }
     }
@@ -264,7 +264,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeList(influences);
         }
         out.writeOptionalWriteable(geoResults);
-        if (out.getVersion().onOrAfter(Version.V_8_6_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_6_0)) {
             out.writeOptionalWriteable(anomalyScoreExplanation);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyScoreExplanation.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyScoreExplanation.java
index f7affb1d6a73..a8ceb118d352 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyScoreExplanation.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyScoreExplanation.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.job.results;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -31,6 +32,7 @@ public class AnomalyScoreExplanation implements ToXContentObject, Writeable {
     public static final ParseField UPPER_CONFIDENCE_BOUND = new ParseField("upper_confidence_bound");
     public static final ParseField HIGH_VARIANCE_PENALTY = new ParseField("high_variance_penalty");
     public static final ParseField INCOMPLETE_BUCKET_PENALTY = new ParseField("incomplete_bucket_penalty");
+    public static final ParseField MULTIMODAL_DISTRIBUTION = new ParseField("multimodal_distribution");
 
     public static final ObjectParser<AnomalyScoreExplanation, Void> STRICT_PARSER = createParser(false);
     public static final ObjectParser<AnomalyScoreExplanation, Void> LENIENT_PARSER = createParser(true);
@@ -51,6 +53,7 @@ private static ObjectParser<AnomalyScoreExplanation, Void> createParser(Boolean
         parser.declareDouble(AnomalyScoreExplanation::setUpperConfidenceBound, UPPER_CONFIDENCE_BOUND);
         parser.declareBoolean(AnomalyScoreExplanation::setHighVariancePenalty, HIGH_VARIANCE_PENALTY);
         parser.declareBoolean(AnomalyScoreExplanation::setIncompleteBucketPenalty, INCOMPLETE_BUCKET_PENALTY);
+        parser.declareBoolean(AnomalyScoreExplanation::setMultimodalDistribution, MULTIMODAL_DISTRIBUTION);
         return parser;
     }
 
@@ -64,6 +67,7 @@ private static ObjectParser<AnomalyScoreExplanation, Void> createParser(Boolean
     private Double upperConfidenceBound;
     private Boolean highVariancePenalty;
     private Boolean incompleteBucketPenalty;
+    private Boolean multimodalDistribution;
 
     AnomalyScoreExplanation() {}
 
@@ -78,6 +82,9 @@ public AnomalyScoreExplanation(StreamInput in) throws IOException {
         this.upperConfidenceBound = in.readOptionalDouble();
         this.highVariancePenalty = in.readOptionalBoolean();
         this.incompleteBucketPenalty = in.readOptionalBoolean();
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            this.multimodalDistribution = in.readOptionalBoolean();
+        }
     }
 
     @Override
@@ -92,6 +99,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalDouble(upperConfidenceBound);
         out.writeOptionalBoolean(highVariancePenalty);
         out.writeOptionalBoolean(incompleteBucketPenalty);
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            out.writeOptionalBoolean(multimodalDistribution);
+        }
     }
 
     @Override
@@ -127,6 +137,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (incompleteBucketPenalty != null) {
             builder.field(INCOMPLETE_BUCKET_PENALTY.getPreferredName(), incompleteBucketPenalty);
         }
+        if (multimodalDistribution != null) {
+            builder.field(MULTIMODAL_DISTRIBUTION.getPreferredName(), multimodalDistribution);
+        }
         builder.endObject();
         return builder;
     }
@@ -143,7 +156,8 @@ public int hashCode() {
             typicalValue,
             upperConfidenceBound,
             highVariancePenalty,
-            incompleteBucketPenalty
+            incompleteBucketPenalty,
+            multimodalDistribution
         );
     }
 
@@ -166,7 +180,8 @@ public boolean equals(Object other) {
             && Objects.equals(this.typicalValue, that.typicalValue)
             && Objects.equals(this.upperConfidenceBound, that.upperConfidenceBound)
             && Objects.equals(this.highVariancePenalty, that.highVariancePenalty)
-            && Objects.equals(this.incompleteBucketPenalty, that.incompleteBucketPenalty);
+            && Objects.equals(this.incompleteBucketPenalty, that.incompleteBucketPenalty)
+            && Objects.equals(this.multimodalDistribution, that.multimodalDistribution);
     }
 
     public String getAnomalyType() {
@@ -249,4 +264,12 @@ public void setIncompleteBucketPenalty(Boolean incompleteBucketPenalty) {
         this.incompleteBucketPenalty = incompleteBucketPenalty;
     }
 
+    public Boolean isMultimodalDistribution() {
+        return multimodalDistribution;
+    }
+
+    public void setMultimodalDistribution(Boolean multimodalDistribution) {
+        this.multimodalDistribution = multimodalDistribution;
+    }
+
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java
index 749f241580f5..3f4be0d17528 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java
@@ -95,6 +95,7 @@ public final class ReservedFieldNames {
         AnomalyScoreExplanation.UPPER_CONFIDENCE_BOUND.getPreferredName(),
         AnomalyScoreExplanation.HIGH_VARIANCE_PENALTY.getPreferredName(),
         AnomalyScoreExplanation.INCOMPLETE_BUCKET_PENALTY.getPreferredName(),
+        AnomalyScoreExplanation.MULTIMODAL_DISTRIBUTION.getPreferredName(),
 
         GeoResults.TYPICAL_POINT.getPreferredName(),
         GeoResults.ACTUAL_POINT.getPreferredName(),
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/snapshot/upgrade/SnapshotUpgradeTaskParams.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/snapshot/upgrade/SnapshotUpgradeTaskParams.java
index 980a5ad4bde4..a0c707edf3cd 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/snapshot/upgrade/SnapshotUpgradeTaskParams.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/snapshot/upgrade/SnapshotUpgradeTaskParams.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.ml.job.snapshot.upgrade;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.persistent.PersistentTaskParams;
@@ -67,8 +67,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_11_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_11_0;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java
index b227a705e2ed..968bbf7fbb22 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.rest.RestStatus;
@@ -85,13 +86,8 @@ public static ElasticsearchStatusException badRequestException(String msg, Objec
         return new ElasticsearchStatusException(msg, RestStatus.BAD_REQUEST, args);
     }
 
-    public static ElasticsearchStatusException configHasNotBeenMigrated(String verb, String id) {
-        return new ElasticsearchStatusException(
-            "cannot {} as the configuration [{}] is temporarily pending migration",
-            RestStatus.SERVICE_UNAVAILABLE,
-            verb,
-            id
-        );
+    public static ElasticsearchStatusException taskOperationFailureToStatusException(TaskOperationFailure failure) {
+        return new ElasticsearchStatusException(failure.getCause().getMessage(), failure.getStatus(), failure.getCause());
     }
 
     /**
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/MonitoringFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/MonitoringFeatureSetUsage.java
index 6883ceb41374..67615b6b79c6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/MonitoringFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/MonitoringFeatureSetUsage.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.monitoring;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
@@ -38,8 +38,8 @@ public MonitoringFeatureSetUsage(boolean collectionEnabled, Map<String, Object>
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_0_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_0_0;
     }
 
     public Map<String, Object> getExporters() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/exporter/MonitoringTemplateUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/exporter/MonitoringTemplateUtils.java
index 8dca92e9746b..8d43288b8303 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/exporter/MonitoringTemplateUtils.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/exporter/MonitoringTemplateUtils.java
@@ -19,7 +19,7 @@ public final class MonitoringTemplateUtils {
      * <p>
      * It may be possible for this to diverge between templates and pipelines, but for now they're the same.
      */
-    public static final int LAST_UPDATED_VERSION = Version.V_8_1_0.id;
+    public static final int LAST_UPDATED_VERSION = Version.V_8_7_0.id;
 
     /**
      * Current version of templates used in their name to differentiate from breaking changes (separate from product version).
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupFeatureSetUsage.java
index 506de8844661..671349362dd0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/RollupFeatureSetUsage.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.rollup;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
 import org.elasticsearch.xpack.core.XPackField;
@@ -24,8 +24,8 @@ public RollupFeatureSetUsage() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_0_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_0_0;
     }
 
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupShardStatus.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupShardStatus.java
new file mode 100644
index 000000000000..2bbd2d96b9ce
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupShardStatus.java
@@ -0,0 +1,141 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.rollup.action;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Objects;
+
+import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+
+public class RollupShardStatus implements Task.Status {
+    public static final String NAME = "rollup-index-shard";
+    private static final ParseField SHARD_FIELD = new ParseField("shard");
+    private static final ParseField START_TIME_FIELD = new ParseField("start_time");
+    private static final ParseField IN_NUM_DOCS_RECEIVED_FIELD = new ParseField("in_num_docs_received");
+    private static final ParseField OUT_NUM_DOCS_SENT_FIELD = new ParseField("out_num_docs_sent");
+    private static final ParseField OUT_NUM_DOCS_INDEXED_FIELD = new ParseField("out_num_docs_indexed");
+    private static final ParseField OUT_NUM_DOCS_FAILED_FIELD = new ParseField("out_num_docs_failed");
+
+    private final ShardId shardId;
+    private final long rollupStart;
+    private final long numReceived;
+    private final long numSent;
+    private final long numIndexed;
+    private final long numFailed;
+
+    private static final ConstructingObjectParser<RollupShardStatus, Void> PARSER;
+    static {
+        PARSER = new ConstructingObjectParser<>(
+            NAME,
+            args -> new RollupShardStatus(
+                ShardId.fromString((String) args[0]),
+                Instant.parse((String) args[1]).toEpochMilli(),
+                (Long) args[2],
+                (Long) args[3],
+                (Long) args[4],
+                (Long) args[5]
+            )
+        );
+
+        PARSER.declareString(constructorArg(), SHARD_FIELD);
+        PARSER.declareString(constructorArg(), START_TIME_FIELD);
+        PARSER.declareLong(constructorArg(), IN_NUM_DOCS_RECEIVED_FIELD);
+        PARSER.declareLong(constructorArg(), OUT_NUM_DOCS_SENT_FIELD);
+        PARSER.declareLong(constructorArg(), OUT_NUM_DOCS_INDEXED_FIELD);
+        PARSER.declareLong(constructorArg(), OUT_NUM_DOCS_FAILED_FIELD);
+    }
+
+    public RollupShardStatus(StreamInput in) throws IOException {
+        shardId = new ShardId(in);
+        rollupStart = in.readLong();
+        numReceived = in.readLong();
+        numSent = in.readLong();
+        numIndexed = in.readLong();
+        numFailed = in.readLong();
+    }
+
+    public RollupShardStatus(ShardId shardId, long rollupStart, long numReceived, long numSent, long numIndexed, long numFailed) {
+        this.shardId = shardId;
+        this.rollupStart = rollupStart;
+        this.numReceived = numReceived;
+        this.numSent = numSent;
+        this.numIndexed = numIndexed;
+        this.numFailed = numFailed;
+    }
+
+    public static RollupShardStatus fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(SHARD_FIELD.getPreferredName(), shardId);
+        builder.field(START_TIME_FIELD.getPreferredName(), Instant.ofEpochMilli(rollupStart).toString());
+        builder.field(IN_NUM_DOCS_RECEIVED_FIELD.getPreferredName(), numReceived);
+        builder.field(OUT_NUM_DOCS_SENT_FIELD.getPreferredName(), numSent);
+        builder.field(OUT_NUM_DOCS_INDEXED_FIELD.getPreferredName(), numIndexed);
+        builder.field(OUT_NUM_DOCS_FAILED_FIELD.getPreferredName(), numFailed);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        shardId.writeTo(out);
+        out.writeLong(rollupStart);
+        out.writeLong(numReceived);
+        out.writeLong(numSent);
+        out.writeLong(numIndexed);
+        out.writeLong(numFailed);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        RollupShardStatus that = (RollupShardStatus) o;
+        return rollupStart == that.rollupStart
+            && Objects.equals(shardId.getIndexName(), that.shardId.getIndexName())
+            && Objects.equals(shardId.id(), that.shardId.id())
+            && Objects.equals(numReceived, that.numReceived)
+            && Objects.equals(numSent, that.numSent)
+            && Objects.equals(numIndexed, that.numIndexed)
+            && Objects.equals(numFailed, that.numFailed);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(shardId.getIndexName(), shardId.id(), rollupStart, numReceived, numSent, numIndexed, numFailed);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupShardTask.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupShardTask.java
new file mode 100644
index 000000000000..ccf80afd0ae4
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupShardTask.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.rollup.action;
+
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.xpack.core.downsample.DownsampleConfig;
+import org.elasticsearch.xpack.core.rollup.RollupField;
+
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class RollupShardTask extends CancellableTask {
+    private final String rollupIndex;
+    private final DownsampleConfig config;
+    private final ShardId shardId;
+    private final long rollupStartTime;
+    private final AtomicLong numReceived = new AtomicLong(0);
+    private final AtomicLong numSent = new AtomicLong(0);
+    private final AtomicLong numIndexed = new AtomicLong(0);
+    private final AtomicLong numFailed = new AtomicLong(0);
+
+    public RollupShardTask(
+        long id,
+        String type,
+        String action,
+        TaskId parentTask,
+        String rollupIndex,
+        DownsampleConfig config,
+        Map<String, String> headers,
+        ShardId shardId
+    ) {
+        super(id, type, action, RollupField.NAME + "_" + rollupIndex + "[" + shardId.id() + "]", parentTask, headers);
+        this.rollupIndex = rollupIndex;
+        this.config = config;
+        this.shardId = shardId;
+        this.rollupStartTime = System.currentTimeMillis();
+    }
+
+    public String getRollupIndex() {
+        return rollupIndex;
+    }
+
+    public DownsampleConfig config() {
+        return config;
+    }
+
+    @Override
+    public Status getStatus() {
+        return new RollupShardStatus(shardId, rollupStartTime, numReceived.get(), numSent.get(), numIndexed.get(), numFailed.get());
+    }
+
+    public long getNumReceived() {
+        return numReceived.get();
+    }
+
+    public long getNumSent() {
+        return numSent.get();
+    }
+
+    public long getNumIndexed() {
+        return numIndexed.get();
+    }
+
+    public long getNumFailed() {
+        return numFailed.get();
+    }
+
+    public void addNumReceived(long count) {
+        numReceived.addAndGet(count);
+    }
+
+    public void addNumSent(long count) {
+        numSent.addAndGet(count);
+    }
+
+    public void addNumIndexed(long count) {
+        numIndexed.addAndGet(count);
+    }
+
+    public void addNumFailed(long count) {
+        numFailed.addAndGet(count);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJob.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJob.java
index b0c6cd4094c7..b8067c58d2f8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJob.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJob.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.rollup.job;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.SimpleDiffable;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -118,7 +118,7 @@ public int hashCode() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.CURRENT.minimumCompatibilityVersion();
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.CURRENT.minimumCompatibilityVersion();
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatus.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatus.java
index 031a10e0ca96..55e3ea54cea0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatus.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatus.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.rollup.job;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
@@ -75,7 +75,7 @@ public RollupJobStatus(IndexerState state, @Nullable Map<String, Object> positio
     public RollupJobStatus(StreamInput in) throws IOException {
         state = IndexerState.fromStream(in);
         currentPosition = in.readBoolean() ? new TreeMap<>(in.readMap()) : null;
-        if (in.getVersion().before(Version.V_8_0_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             // 7.x nodes serialize `upgradedDocumentID` flag. We don't need it anymore, but
             // we need to pull it off the stream
             // This can go away completely in 9.0
@@ -122,7 +122,7 @@ public void writeTo(StreamOutput out) throws IOException {
         if (currentPosition != null) {
             out.writeGenericMap(currentPosition);
         }
-        if (out.getVersion().before(Version.V_8_0_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_0_0)) {
             // 7.x nodes expect a boolean `upgradedDocumentID` flag. We don't have it anymore,
             // but we need to tell them we are upgraded in case there is a mixed cluster
             // This can go away completely in 9.0
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/MountSearchableSnapshotRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/MountSearchableSnapshotRequest.java
index 7d57a5634fb4..5bd0024f2fb6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/MountSearchableSnapshotRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/MountSearchableSnapshotRequest.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.searchablesnapshots;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -84,7 +84,7 @@ public class MountSearchableSnapshotRequest extends MasterNodeRequest<MountSearc
     /**
      * Searchable snapshots partial storage was introduced in 7.12.0
      */
-    private static final Version SHARED_CACHE_VERSION = Version.V_7_12_0;
+    private static final TransportVersion SHARED_CACHE_VERSION = TransportVersion.V_7_12_0;
 
     private final String mountedIndexName;
     private final String repositoryName;
@@ -127,7 +127,7 @@ public MountSearchableSnapshotRequest(StreamInput in) throws IOException {
         this.indexSettings = readSettingsFromStream(in);
         this.ignoreIndexSettings = in.readStringArray();
         this.waitForCompletion = in.readBoolean();
-        if (in.getVersion().onOrAfter(SHARED_CACHE_VERSION)) {
+        if (in.getTransportVersion().onOrAfter(SHARED_CACHE_VERSION)) {
             this.storage = Storage.readFromStream(in);
         } else {
             this.storage = Storage.FULL_COPY;
@@ -144,11 +144,11 @@ public void writeTo(StreamOutput out) throws IOException {
         indexSettings.writeTo(out);
         out.writeStringArray(ignoreIndexSettings);
         out.writeBoolean(waitForCompletion);
-        if (out.getVersion().onOrAfter(SHARED_CACHE_VERSION)) {
+        if (out.getTransportVersion().onOrAfter(SHARED_CACHE_VERSION)) {
             storage.writeTo(out);
         } else if (storage != Storage.FULL_COPY) {
             throw new UnsupportedOperationException(
-                "storage type [" + storage + "] is not supported on version [" + out.getVersion() + "]"
+                "storage type [" + storage + "] is not supported on version [" + out.getTransportVersion() + "]"
             );
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotFeatureSetUsage.java
index cbf8ccd617fa..4522952699b8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotFeatureSetUsage.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.searchablesnapshots;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.ToXContent;
@@ -27,7 +27,7 @@ public class SearchableSnapshotFeatureSetUsage extends XPackFeatureSet.Usage {
     public SearchableSnapshotFeatureSetUsage(StreamInput input) throws IOException {
         super(input);
         numberOfSearchableSnapshotIndices = input.readVInt();
-        if (input.getVersion().onOrAfter(Version.V_7_13_0)) {
+        if (input.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0)) {
             numberOfFullCopySearchableSnapshotIndices = input.readVInt();
             numberOfSharedCacheSearchableSnapshotIndices = input.readVInt();
         } else {
@@ -37,15 +37,15 @@ public SearchableSnapshotFeatureSetUsage(StreamInput input) throws IOException {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_9_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_9_0;
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeVInt(numberOfSearchableSnapshotIndices);
-        if (out.getVersion().onOrAfter(Version.V_7_13_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0)) {
             out.writeVInt(numberOfFullCopySearchableSnapshotIndices);
             out.writeVInt(numberOfSharedCacheSearchableSnapshotIndices);
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotShardStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotShardStats.java
index 33c84e826534..887ca78e32f1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotShardStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotShardStats.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.searchablesnapshots;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -200,7 +200,7 @@ public CacheIndexInputStats(
         }
 
         CacheIndexInputStats(final StreamInput in) throws IOException {
-            if (in.getVersion().before(Version.V_7_12_0)) {
+            if (in.getTransportVersion().before(TransportVersion.V_7_12_0)) {
                 // This API is currently only used internally for testing, so BWC breaking changes are OK.
                 // We just throw an exception here to get a better error message in case this would be called
                 // in a mixed version cluster
@@ -208,7 +208,7 @@ public CacheIndexInputStats(
             }
             this.fileExt = in.readString();
             this.numFiles = in.readVLong();
-            if (in.getVersion().before(Version.V_7_13_0)) {
+            if (in.getTransportVersion().before(TransportVersion.V_7_13_0)) {
                 this.totalSize = ByteSizeValue.ofBytes(in.readVLong());
                 this.minSize = ByteSizeValue.ZERO;
                 this.maxSize = ByteSizeValue.ZERO;
@@ -231,7 +231,7 @@ public CacheIndexInputStats(
             this.directBytesRead = new TimedCounter(in);
             this.optimizedBytesRead = new TimedCounter(in);
             this.blobStoreBytesRequested = new Counter(in);
-            if (in.getVersion().onOrAfter(Version.V_7_13_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0)) {
                 this.luceneBytesRead = new Counter(in);
             } else {
                 this.luceneBytesRead = new Counter(0, 0, 0, 0);
@@ -273,7 +273,7 @@ public static CacheIndexInputStats combine(CacheIndexInputStats cis1, CacheIndex
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            if (out.getVersion().before(Version.V_7_12_0)) {
+            if (out.getTransportVersion().before(TransportVersion.V_7_12_0)) {
                 // This API is currently only used internally for testing, so BWC breaking changes are OK.
                 // We just throw an exception here to get a better error message in case this would be called
                 // in a mixed version cluster
@@ -281,7 +281,7 @@ public void writeTo(StreamOutput out) throws IOException {
             }
             out.writeString(fileExt);
             out.writeVLong(numFiles);
-            if (out.getVersion().before(Version.V_7_13_0)) {
+            if (out.getTransportVersion().before(TransportVersion.V_7_13_0)) {
                 out.writeVLong(totalSize.getBytes());
             } else {
                 totalSize.writeTo(out);
@@ -303,7 +303,7 @@ public void writeTo(StreamOutput out) throws IOException {
             directBytesRead.writeTo(out);
             optimizedBytesRead.writeTo(out);
             blobStoreBytesRequested.writeTo(out);
-            if (out.getVersion().onOrAfter(Version.V_7_13_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0)) {
                 luceneBytesRead.writeTo(out);
             }
             out.writeVLong(currentIndexCacheFills);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java
index 744c07984509..f64b654cdd28 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java
@@ -9,7 +9,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchSecurityException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext;
@@ -20,6 +20,7 @@
 import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer;
 import org.elasticsearch.xpack.core.security.authc.support.SecondaryAuthentication;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.ParentActionAuthorization;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
@@ -27,6 +28,7 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.function.Consumer;
@@ -86,6 +88,24 @@ public AuthorizationEngine.AuthorizationInfo getAuthorizationInfoFromContext() {
         return Objects.requireNonNull(threadContext.getTransient(AUTHORIZATION_INFO_KEY), "authorization info is missing from context");
     }
 
+    @Nullable
+    public ParentActionAuthorization getParentAuthorization() {
+        try {
+            return ParentActionAuthorization.readFromThreadContext(threadContext);
+        } catch (IOException e) {
+            logger.error("failed to read parent authorization from thread context", e);
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public void setParentAuthorization(ParentActionAuthorization parentAuthorization) {
+        try {
+            parentAuthorization.writeToThreadContext(threadContext);
+        } catch (IOException e) {
+            throw new AssertionError("failed to write parent authorization to the thread context", e);
+        }
+    }
+
     /**
      * Returns the "secondary authentication" (see {@link SecondaryAuthentication}) information,
      * or {@code null} if the current request does not have a secondary authentication context
@@ -128,7 +148,7 @@ public void copyIndicesAccessControlFromReaderContext(ReaderContext readerContex
      * Sets the user forcefully to the provided user. There must not be an existing user in the ThreadContext otherwise an exception
      * will be thrown. This method is package private for testing.
      */
-    public void setInternalUser(User internalUser, Version version) {
+    public void setInternalUser(User internalUser, TransportVersion version) {
         assert User.isInternal(internalUser);
         setAuthentication(Authentication.newInternalAuthentication(internalUser, version, nodeName));
     }
@@ -137,7 +157,7 @@ public void setInternalUser(User internalUser, Version version) {
      * Runs the consumer in a new context as the provided user. The original context is provided to the consumer. When this method
      * returns, the original context is restored.
      */
-    public void executeAsInternalUser(User internalUser, Version version, Consumer<StoredContext> consumer) {
+    public void executeAsInternalUser(User internalUser, TransportVersion version, Consumer<StoredContext> consumer) {
         assert User.isInternal(internalUser);
         final StoredContext original = threadContext.newStoredContextPreservingResponseHeaders();
         try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
@@ -147,10 +167,10 @@ public void executeAsInternalUser(User internalUser, Version version, Consumer<S
     }
 
     public void executeAsSystemUser(Consumer<StoredContext> consumer) {
-        executeAsSystemUser(Version.CURRENT, consumer);
+        executeAsSystemUser(TransportVersion.CURRENT, consumer);
     }
 
-    public void executeAsSystemUser(Version version, Consumer<StoredContext> consumer) {
+    public void executeAsSystemUser(TransportVersion version, Consumer<StoredContext> consumer) {
         executeAsInternalUser(SystemUser.INSTANCE, version, consumer);
     }
 
@@ -170,7 +190,7 @@ public <T> T executeWithAuthentication(Authentication authentication, Function<S
      * Runs the consumer in a new context after setting a new version of the authentication that is compatible with the version provided.
      * The original context is provided to the consumer. When this method returns, the original context is restored.
      */
-    public void executeAfterRewritingAuthentication(Consumer<StoredContext> consumer, Version version) {
+    public void executeAfterRewritingAuthentication(Consumer<StoredContext> consumer, TransportVersion version) {
         // Preserve request headers other than authentication
         final Map<String, String> existingRequestHeaders = threadContext.getRequestHeadersOnly();
         final StoredContext original = threadContext.newStoredContextPreservingResponseHeaders();
@@ -186,6 +206,22 @@ public void executeAfterRewritingAuthentication(Consumer<StoredContext> consumer
         }
     }
 
+    /**
+     * Executes consumer in a new thread context after removing {@link ParentActionAuthorization}.
+     * The original context is provided to the consumer. When this method returns,
+     * the original context is restored preserving response headers.
+     */
+    public void executeAfterRemovingParentAuthorization(Consumer<StoredContext> consumer) {
+        try (
+            ThreadContext.StoredContext original = threadContext.newStoredContextPreservingResponseHeaders(
+                List.of(),
+                List.of(ParentActionAuthorization.THREAD_CONTEXT_KEY)
+            )
+        ) {
+            consumer.accept(original);
+        }
+    }
+
     /**
      * Checks whether the user or API key of the passed in authentication can access the resources owned by the user
      * or API key of this authentication. The rules are as follows:
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityFeatureSetUsage.java
index e7962e51cd0c..d7b6914c5318 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityFeatureSetUsage.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.security;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -52,7 +52,7 @@ public SecurityFeatureSetUsage(StreamInput in) throws IOException {
         realmsUsage = in.readMap();
         rolesStoreUsage = in.readMap();
         sslUsage = in.readMap();
-        if (in.getVersion().onOrAfter(Version.V_7_2_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_2_0)) {
             tokenServiceUsage = in.readMap();
             apiKeyServiceUsage = in.readMap();
         }
@@ -60,16 +60,16 @@ public SecurityFeatureSetUsage(StreamInput in) throws IOException {
         ipFilterUsage = in.readMap();
         anonymousUsage = in.readMap();
         roleMappingStoreUsage = in.readMap();
-        if (in.getVersion().onOrAfter(Version.V_7_5_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_5_0)) {
             fips140Usage = in.readMap();
         }
-        if (in.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             operatorPrivilegesUsage = in.readMap();
         }
-        if (in.getVersion().onOrAfter(Version.V_8_2_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
             domainsUsage = in.readMap();
         }
-        if (in.getVersion().onOrAfter(Version.V_8_5_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_5_0)) {
             userProfileUsage = in.readMap();
         }
     }
@@ -107,8 +107,8 @@ public SecurityFeatureSetUsage(
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_0_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_0_0;
     }
 
     @Override
@@ -117,7 +117,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeGenericMap(realmsUsage);
         out.writeGenericMap(rolesStoreUsage);
         out.writeGenericMap(sslUsage);
-        if (out.getVersion().onOrAfter(Version.V_7_2_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_2_0)) {
             out.writeGenericMap(tokenServiceUsage);
             out.writeGenericMap(apiKeyServiceUsage);
         }
@@ -125,16 +125,16 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeGenericMap(ipFilterUsage);
         out.writeGenericMap(anonymousUsage);
         out.writeGenericMap(roleMappingStoreUsage);
-        if (out.getVersion().onOrAfter(Version.V_7_5_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_5_0)) {
             out.writeGenericMap(fips140Usage);
         }
-        if (out.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             out.writeGenericMap(operatorPrivilegesUsage);
         }
-        if (out.getVersion().onOrAfter(Version.V_8_2_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
             out.writeGenericMap(domainsUsage);
         }
-        if (out.getVersion().onOrAfter(Version.V_8_5_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_5_0)) {
             out.writeGenericMap(userProfileUsage);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/DelegatePkiAuthenticationResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/DelegatePkiAuthenticationResponse.java
index aa8719410de2..bf7a67882ac9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/DelegatePkiAuthenticationResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/DelegatePkiAuthenticationResponse.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.security.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -47,7 +47,7 @@ public DelegatePkiAuthenticationResponse(StreamInput input) throws IOException {
         super(input);
         accessToken = input.readString();
         expiresIn = input.readTimeValue();
-        if (input.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (input.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             authentication = new Authentication(input);
         }
     }
@@ -68,7 +68,7 @@ public Authentication getAuthentication() {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(accessToken);
         out.writeTimeValue(expiresIn);
-        if (out.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             authentication.writeTo(out);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/Grant.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/Grant.java
index f4c361ddef63..86ea99f18344 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/Grant.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/Grant.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.security.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -41,7 +41,7 @@ public Grant(StreamInput in) throws IOException {
         this.username = in.readOptionalString();
         this.password = in.readOptionalSecureString();
         this.accessToken = in.readOptionalSecureString();
-        if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             this.runAsUsername = in.readOptionalString();
         } else {
             this.runAsUsername = null;
@@ -53,7 +53,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(username);
         out.writeOptionalSecureString(password);
         out.writeOptionalSecureString(accessToken);
-        if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             out.writeOptionalString(runAsUsername);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKey.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKey.java
index 5b6a85c9169a..4be0988081ba 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKey.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKey.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.security.action.apikey;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -105,7 +105,7 @@ private ApiKey(
     }
 
     public ApiKey(StreamInput in) throws IOException {
-        if (in.getVersion().onOrAfter(Version.V_7_5_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_5_0)) {
             this.name = in.readOptionalString();
         } else {
             this.name = in.readString();
@@ -116,12 +116,12 @@ public ApiKey(StreamInput in) throws IOException {
         this.invalidated = in.readBoolean();
         this.username = in.readString();
         this.realm = in.readString();
-        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             this.metadata = in.readMap();
         } else {
             this.metadata = Map.of();
         }
-        if (in.getVersion().onOrAfter(Version.V_8_5_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_5_0)) {
             final List<RoleDescriptor> roleDescriptors = in.readOptionalList(RoleDescriptor::new);
             this.roleDescriptors = roleDescriptors != null ? List.copyOf(roleDescriptors) : null;
             this.limitedBy = in.readOptionalWriteable(RoleDescriptorsIntersection::new);
@@ -202,7 +202,7 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getVersion().onOrAfter(Version.V_7_5_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_5_0)) {
             out.writeOptionalString(name);
         } else {
             out.writeString(name);
@@ -213,10 +213,10 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(invalidated);
         out.writeString(username);
         out.writeString(realm);
-        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             out.writeGenericMap(metadata);
         }
-        if (out.getVersion().onOrAfter(Version.V_8_5_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_5_0)) {
             out.writeOptionalCollection(roleDescriptors);
             out.writeOptionalWriteable(limitedBy);
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequest.java
index e9bab89108db..69ad826c933e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequest.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.security.action.apikey;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.WriteRequest;
@@ -74,12 +74,12 @@ public CreateApiKeyRequest(
 
     public CreateApiKeyRequest(StreamInput in) throws IOException {
         super(in);
-        if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             this.id = in.readString();
         } else {
             this.id = UUIDs.base64UUID();
         }
-        if (in.getVersion().onOrAfter(Version.V_7_5_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_5_0)) {
             this.name = in.readOptionalString();
         } else {
             this.name = in.readString();
@@ -87,7 +87,7 @@ public CreateApiKeyRequest(StreamInput in) throws IOException {
         this.expiration = in.readOptionalTimeValue();
         this.roleDescriptors = in.readImmutableList(RoleDescriptor::new);
         this.refreshPolicy = WriteRequest.RefreshPolicy.readFrom(in);
-        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             this.metadata = in.readMap();
         } else {
             this.metadata = null;
@@ -173,10 +173,10 @@ public ActionRequestValidationException validate() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             out.writeString(id);
         }
-        if (out.getVersion().onOrAfter(Version.V_7_5_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_5_0)) {
             out.writeOptionalString(name);
         } else {
             out.writeString(name);
@@ -184,7 +184,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalTimeValue(expiration);
         out.writeList(roleDescriptors);
         refreshPolicy.writeTo(out);
-        if (out.getVersion().onOrAfter(Version.V_7_13_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0)) {
             out.writeGenericMap(metadata);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/GetApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/GetApiKeyRequest.java
index 572b2ab029b8..51ca2060388c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/GetApiKeyRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/GetApiKeyRequest.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.security.action.apikey;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.Strings;
@@ -38,12 +38,12 @@ public GetApiKeyRequest(StreamInput in) throws IOException {
         userName = textOrNull(in.readOptionalString());
         apiKeyId = textOrNull(in.readOptionalString());
         apiKeyName = textOrNull(in.readOptionalString());
-        if (in.getVersion().onOrAfter(Version.V_7_4_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_4_0)) {
             ownedByAuthenticatedUser = in.readOptionalBoolean();
         } else {
             ownedByAuthenticatedUser = false;
         }
-        if (in.getVersion().onOrAfter(Version.V_8_5_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_5_0)) {
             withLimitedBy = in.readBoolean();
         } else {
             withLimitedBy = false;
@@ -126,10 +126,10 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(userName);
         out.writeOptionalString(apiKeyId);
         out.writeOptionalString(apiKeyName);
-        if (out.getVersion().onOrAfter(Version.V_7_4_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_4_0)) {
             out.writeOptionalBoolean(ownedByAuthenticatedUser);
         }
-        if (out.getVersion().onOrAfter(Version.V_8_5_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_5_0)) {
             out.writeBoolean(withLimitedBy);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/InvalidateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/InvalidateApiKeyRequest.java
index 7bf28f30cf0a..2c43767af7b9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/InvalidateApiKeyRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/InvalidateApiKeyRequest.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.security.action.apikey;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.Strings;
@@ -41,7 +41,7 @@ public InvalidateApiKeyRequest(StreamInput in) throws IOException {
         super(in);
         realmName = textOrNull(in.readOptionalString());
         userName = textOrNull(in.readOptionalString());
-        if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             ids = in.readOptionalStringArray();
         } else {
             final String id = in.readOptionalString();
@@ -49,7 +49,7 @@ public InvalidateApiKeyRequest(StreamInput in) throws IOException {
         }
         validateIds(ids);
         name = textOrNull(in.readOptionalString());
-        if (in.getVersion().onOrAfter(Version.V_7_4_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_4_0)) {
             ownedByAuthenticatedUser = in.readOptionalBoolean();
         } else {
             ownedByAuthenticatedUser = false;
@@ -209,7 +209,7 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeOptionalString(realmName);
         out.writeOptionalString(userName);
-        if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_10_0)) {
             out.writeOptionalStringArray(ids);
         } else {
             if (ids != null) {
@@ -223,7 +223,7 @@ public void writeTo(StreamOutput out) throws IOException {
             }
         }
         out.writeOptionalString(name);
-        if (out.getVersion().onOrAfter(Version.V_7_4_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_4_0)) {
             out.writeOptionalBoolean(ownedByAuthenticatedUser);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/QueryApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/QueryApiKeyRequest.java
index 949ceb41b0ae..e315d81975d6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/QueryApiKeyRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/QueryApiKeyRequest.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.security.action.apikey;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -72,7 +72,7 @@ public QueryApiKeyRequest(StreamInput in) throws IOException {
             this.fieldSortBuilders = null;
         }
         this.searchAfterBuilder = in.readOptionalWriteable(SearchAfterBuilder::new);
-        if (in.getVersion().onOrAfter(Version.V_8_5_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_5_0)) {
             this.withLimitedBy = in.readBoolean();
         } else {
             this.withLimitedBy = false;
@@ -136,7 +136,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeList(fieldSortBuilders);
         }
         out.writeOptionalWriteable(searchAfterBuilder);
-        if (out.getVersion().onOrAfter(Version.V_8_5_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_5_0)) {
             out.writeBoolean(withLimitedBy);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectAuthenticateRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectAuthenticateRequest.java
index 7c103d825b17..022c5bc943c2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectAuthenticateRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectAuthenticateRequest.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.security.action.oidc;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.Strings;
@@ -54,7 +54,7 @@ public OpenIdConnectAuthenticateRequest(StreamInput in) throws IOException {
         redirectUri = in.readString();
         state = in.readString();
         nonce = in.readString();
-        if (in.getVersion().onOrAfter(Version.V_7_4_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_4_0)) {
             realm = in.readOptionalString();
         }
 
@@ -113,7 +113,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(redirectUri);
         out.writeString(state);
         out.writeString(nonce);
-        if (out.getVersion().onOrAfter(Version.V_7_4_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_4_0)) {
             out.writeOptionalString(realm);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectAuthenticateResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectAuthenticateResponse.java
index e51a0beb2063..29fcba5be85f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectAuthenticateResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectAuthenticateResponse.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.security.action.oidc;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -42,7 +42,7 @@ public OpenIdConnectAuthenticateResponse(StreamInput in) throws IOException {
         accessTokenString = in.readString();
         refreshTokenString = in.readString();
         expiresIn = in.readTimeValue();
-        if (in.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             authentication = new Authentication(in);
         }
     }
@@ -73,7 +73,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(accessTokenString);
         out.writeString(refreshTokenString);
         out.writeTimeValue(expiresIn);
-        if (out.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             authentication.writeTo(out);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectPrepareAuthenticationResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectPrepareAuthenticationResponse.java
index a0242876be87..00cb814b7e1d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectPrepareAuthenticationResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectPrepareAuthenticationResponse.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.security.action.oidc;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -47,7 +47,7 @@ public OpenIdConnectPrepareAuthenticationResponse(StreamInput in) throws IOExcep
         authenticationRequestUrl = in.readString();
         state = in.readString();
         nonce = in.readString();
-        if (in.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             realmName = in.readString();
         }
     }
@@ -73,7 +73,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(authenticationRequestUrl);
         out.writeString(state);
         out.writeString(nonce);
-        if (out.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             out.writeString(realmName);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequest.java
index e1456accfa50..87f3b348d903 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequest.java
@@ -58,7 +58,7 @@ public PutRoleRequest(StreamInput in) throws IOException {
         runAs = in.readStringArray();
         refreshPolicy = RefreshPolicy.readFrom(in);
         metadata = in.readMap();
-        if (in.getVersion().onOrAfter(RoleDescriptor.VERSION_REMOTE_INDICES)) {
+        if (in.getTransportVersion().onOrAfter(RoleDescriptor.VERSION_REMOTE_INDICES)) {
             remoteIndicesPrivileges = in.readList(RoleDescriptor.RemoteIndicesPrivileges::new);
         }
     }
@@ -213,14 +213,14 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeStringArray(runAs);
         refreshPolicy.writeTo(out);
         out.writeGenericMap(metadata);
-        if (out.getVersion().onOrAfter(RoleDescriptor.VERSION_REMOTE_INDICES)) {
+        if (out.getTransportVersion().onOrAfter(RoleDescriptor.VERSION_REMOTE_INDICES)) {
             out.writeCollection(remoteIndicesPrivileges);
         } else if (hasRemoteIndicesPrivileges()) {
             throw new IllegalArgumentException(
                 "versions of Elasticsearch before ["
                     + RoleDescriptor.VERSION_REMOTE_INDICES
                     + "] can't handle remote indices privileges and attempted to send to ["
-                    + out.getVersion()
+                    + out.getTransportVersion()
                     + "]"
             );
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/PutRoleMappingRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/PutRoleMappingRequest.java
index 3d14f48afcca..89b092b0ea1a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/PutRoleMappingRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/PutRoleMappingRequest.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.security.action.rolemapping;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.WriteRequest;
@@ -47,7 +47,7 @@ public PutRoleMappingRequest(StreamInput in) throws IOException {
         this.name = in.readString();
         this.enabled = in.readBoolean();
         this.roles = in.readStringList();
-        if (in.getVersion().onOrAfter(Version.V_7_2_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_2_0)) {
             this.roleTemplates = in.readList(TemplateRoleName::new);
         }
         this.rules = ExpressionParser.readExpression(in);
@@ -155,7 +155,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(name);
         out.writeBoolean(enabled);
         out.writeStringCollection(roles);
-        if (out.getVersion().onOrAfter(Version.V_7_2_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_2_0)) {
             out.writeList(roleTemplates);
         }
         ExpressionParser.writeExpression(rules, out);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlAuthenticateResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlAuthenticateResponse.java
index caee0ed9deab..707c9f36729a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlAuthenticateResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlAuthenticateResponse.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.security.action.saml;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -31,13 +31,13 @@ public final class SamlAuthenticateResponse extends ActionResponse {
     public SamlAuthenticateResponse(StreamInput in) throws IOException {
         super(in);
         principal = in.readString();
-        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             realm = in.readString();
         }
         tokenString = in.readString();
         refreshToken = in.readString();
         expiresIn = in.readTimeValue();
-        if (in.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             authentication = new Authentication(in);
         }
     }
@@ -78,13 +78,13 @@ public Authentication getAuthentication() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(principal);
-        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             out.writeString(realm);
         }
         out.writeString(tokenString);
         out.writeString(refreshToken);
         out.writeTimeValue(expiresIn);
-        if (out.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             authentication.writeTo(out);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlPrepareAuthenticationRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlPrepareAuthenticationRequest.java
index 6981fc582728..02d55ca18bee 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlPrepareAuthenticationRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlPrepareAuthenticationRequest.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.security.action.saml;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -33,7 +33,7 @@ public SamlPrepareAuthenticationRequest(StreamInput in) throws IOException {
         super(in);
         realmName = in.readOptionalString();
         assertionConsumerServiceURL = in.readOptionalString();
-        if (in.getVersion().onOrAfter(Version.V_7_5_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_5_0)) {
             relayState = in.readOptionalString();
         }
     }
@@ -87,7 +87,7 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeOptionalString(realmName);
         out.writeOptionalString(assertionConsumerServiceURL);
-        if (out.getVersion().onOrAfter(Version.V_7_5_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_5_0)) {
             out.writeOptionalString(relayState);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenResponse.java
index c7c7b8ab1dcc..c5e06ce5d184 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenResponse.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.security.action.token;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -41,7 +41,7 @@ public CreateTokenResponse(StreamInput in) throws IOException {
         scope = in.readOptionalString();
         refreshToken = in.readOptionalString();
         kerberosAuthenticationResponseToken = in.readOptionalString();
-        if (in.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             authentication = new Authentication(in);
         }
     }
@@ -93,7 +93,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(scope);
         out.writeOptionalString(refreshToken);
         out.writeOptionalString(kerberosAuthenticationResponseToken);
-        if (out.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             authentication.writeTo(out);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/AuthenticateRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/AuthenticateRequest.java
index baecb3e2b6bd..73e334f2a249 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/AuthenticateRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/AuthenticateRequest.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.security.action.user;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -20,7 +20,7 @@ public class AuthenticateRequest extends ActionRequest {
 
     public AuthenticateRequest(StreamInput in) throws IOException {
         super(in);
-        if (in.getVersion().before(Version.V_8_5_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_5_0)) {
             // Older versions included the username as a field
             in.readString();
         }
@@ -37,7 +37,7 @@ public ActionRequestValidationException validate() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        if (out.getVersion().before(Version.V_8_5_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_5_0)) {
             throw new IllegalStateException("cannot send authenticate request to a node of earlier version");
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequestBuilder.java
index dcfa395f4574..927f839f267b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequestBuilder.java
@@ -26,6 +26,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.CharBuffer;
+import java.util.Locale;
 
 /**
  * Request to change a user's password.
@@ -70,9 +71,12 @@ public ChangePasswordRequestBuilder password(char[] password, Hasher hasher) {
      */
     public ChangePasswordRequestBuilder passwordHash(char[] passwordHashChars, Hasher configuredHasher) {
         final Hasher resolvedHasher = Hasher.resolveFromHash(passwordHashChars);
-        if (resolvedHasher.equals(configuredHasher) == false) {
+        if (resolvedHasher.equals(configuredHasher) == false
+            && Hasher.getAvailableAlgoStoredHash().contains(resolvedHasher.name().toLowerCase(Locale.ROOT)) == false) {
             throw new IllegalArgumentException(
-                "Provided password hash uses [" + resolvedHasher + "] but the configured hashing algorithm is [" + configuredHasher + "]"
+                "The provided password hash is not a hash or it could not be resolved to a supported hash algorithm. "
+                    + "The supported password hash algorithms are "
+                    + Hasher.getAvailableAlgoStoredHash().toString()
             );
         }
         if (request.passwordHash() != null) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponse.java
index 2b7f85c3797c..325616407e7e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponse.java
@@ -47,7 +47,7 @@ public GetUserPrivilegesResponse(StreamInput in) throws IOException {
         index = Collections.unmodifiableSet(in.readSet(Indices::new));
         application = Collections.unmodifiableSet(in.readSet(RoleDescriptor.ApplicationResourcePrivileges::new));
         runAs = Collections.unmodifiableSet(in.readSet(StreamInput::readString));
-        if (in.getVersion().onOrAfter(RoleDescriptor.VERSION_REMOTE_INDICES)) {
+        if (in.getTransportVersion().onOrAfter(RoleDescriptor.VERSION_REMOTE_INDICES)) {
             remoteIndex = Collections.unmodifiableSet(in.readSet(RemoteIndices::new));
         } else {
             remoteIndex = Collections.emptySet();
@@ -105,14 +105,14 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeCollection(index);
         out.writeCollection(application);
         out.writeCollection(runAs, StreamOutput::writeString);
-        if (out.getVersion().onOrAfter(RoleDescriptor.VERSION_REMOTE_INDICES)) {
+        if (out.getTransportVersion().onOrAfter(RoleDescriptor.VERSION_REMOTE_INDICES)) {
             out.writeCollection(remoteIndex);
         } else if (hasRemoteIndicesPrivileges()) {
             throw new IllegalArgumentException(
                 "versions of Elasticsearch before ["
                     + RoleDescriptor.VERSION_REMOTE_INDICES
                     + "] can't handle remote indices privileges and attempted to send to ["
-                    + out.getVersion()
+                    + out.getTransportVersion()
                     + "]"
             );
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersRequest.java
index 4a668508b200..e6e7c0e6b8a8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersRequest.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.security.action.user;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.Strings;
@@ -28,7 +28,7 @@ public class GetUsersRequest extends ActionRequest implements UserRequest {
     public GetUsersRequest(StreamInput in) throws IOException {
         super(in);
         usernames = in.readStringArray();
-        if (in.getVersion().onOrAfter(Version.V_8_5_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_5_0)) {
             withProfileUid = in.readBoolean();
         } else {
             withProfileUid = false;
@@ -77,7 +77,7 @@ public void setWithProfileUid(boolean withProfileUid) {
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeStringArray(usernames);
-        if (out.getVersion().onOrAfter(Version.V_8_5_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_5_0)) {
             out.writeBoolean(withProfileUid);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersResponse.java
index ca32d3acccd3..d41de69ba3a8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersResponse.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.security.action.user;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -42,7 +42,7 @@ public GetUsersResponse(StreamInput in) throws IOException {
                 users[i] = user;
             }
         }
-        if (in.getVersion().onOrAfter(Version.V_8_5_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_5_0)) {
             if (in.readBoolean()) {
                 profileUidLookup = in.readMap(StreamInput::readString, StreamInput::readString);
             } else {
@@ -82,7 +82,7 @@ public void writeTo(StreamOutput out) throws IOException {
                 Authentication.AuthenticationSerializationHelper.writeUserTo(user, out);
             }
         }
-        if (out.getVersion().onOrAfter(Version.V_8_5_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_5_0)) {
             if (profileUidLookup != null) {
                 out.writeBoolean(true);
                 out.writeMap(profileUidLookup, StreamOutput::writeString, StreamOutput::writeString);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequestBuilder.java
index 552d70bde95c..60a6d04f2c50 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequestBuilder.java
@@ -26,6 +26,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 
@@ -92,9 +93,12 @@ public PutUserRequestBuilder email(String email) {
 
     public PutUserRequestBuilder passwordHash(char[] passwordHash, Hasher configuredHasher) {
         final Hasher resolvedHasher = Hasher.resolveFromHash(passwordHash);
-        if (resolvedHasher.equals(configuredHasher) == false) {
+        if (resolvedHasher.equals(configuredHasher) == false
+            && Hasher.getAvailableAlgoStoredHash().contains(resolvedHasher.name().toLowerCase(Locale.ROOT)) == false) {
             throw new IllegalArgumentException(
-                "Provided password hash uses [" + resolvedHasher + "] but the configured hashing algorithm is [" + configuredHasher + "]"
+                "The provided password hash is not a hash or it could not be resolved to a supported hash algorithm. "
+                    + "The supported password hash algorithms are "
+                    + Hasher.getAvailableAlgoStoredHash().toString()
             );
         }
         if (request.passwordHash() != null) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
index 85ce83baa373..46ad97a0041a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java
@@ -9,7 +9,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Assertions;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -19,6 +19,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -46,12 +47,14 @@
 import java.util.Map;
 import java.util.Objects;
 
+import static org.elasticsearch.common.Strings.EMPTY_ARRAY;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 import static org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef.newAnonymousRealmRef;
 import static org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef.newApiKeyRealmRef;
 import static org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef.newInternalAttachRealmRef;
 import static org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef.newInternalFallbackRealmRef;
+import static org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef.newRemoteAccessRealmRef;
 import static org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef.newServiceAccountRealmRef;
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.ANONYMOUS_REALM_NAME;
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.ANONYMOUS_REALM_TYPE;
@@ -61,6 +64,8 @@
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.ATTACH_REALM_TYPE;
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.FALLBACK_REALM_NAME;
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.FALLBACK_REALM_TYPE;
+import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.REMOTE_ACCESS_REALM_NAME;
+import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.REMOTE_ACCESS_REALM_TYPE;
 import static org.elasticsearch.xpack.core.security.authc.RealmDomain.REALM_DOMAIN_PARSER;
 
 /**
@@ -70,13 +75,31 @@
  * Authentication is serialized and travels across the cluster nodes as the sub-requests are handled,
  * and can also be cached by long-running jobs that continue to act on behalf of the user, beyond
  * the lifetime of the original request.
+ *
+ * The authentication consists of two {@link Subject}s
+ * <ul>
+ *     <li>{@link #authenticatingSubject} performs the authentication, i.e. it provides a credential.</li>
+ *     <li>{@link #effectiveSubject} The subject that {@link #authenticatingSubject} impersonates ({@link #isRunAs()})</li>
+ * </ul>
+ * If {@link #isRunAs()} is {@code false}, the two {@link Subject}s will be the same object.
+ *
+ * Authentication also has a {@link #type} that indicates which mechanism the {@link #authenticatingSubject}
+ * uses to perform the authentication.
+ *
+ * The Authentication's version is its {@link Subject}'s version, i.e. {@code getEffectiveSubject().getTransportVersion()}.
+ * It is guaranteed that the versions are identical for the two Subjects. Hence {@code getAuthenticatingSubject().getTransportVersion()}
+ * will give out the same result. But using {@code getEffectiveSubject()} is more idiomatic since most callers
+ * of this class should just need to know about the {@link #effectiveSubject}. That is, often times, the caller
+ * begins with {@code authentication.getEffectiveSubject()} for interrogating an Authentication object.
  */
 public final class Authentication implements ToXContentObject {
 
     private static final Logger logger = LogManager.getLogger(Authentication.class);
+    private static final TransportVersion VERSION_AUTHENTICATION_TYPE = TransportVersion.fromId(6_07_00_99);
 
-    public static final Version VERSION_API_KEY_ROLES_AS_BYTES = Version.V_7_9_0;
-    public static final Version VERSION_REALM_DOMAINS = Version.V_8_2_0;
+    public static final TransportVersion VERSION_REMOTE_ACCESS_REALM = TransportVersion.V_8_7_0;
+    public static final TransportVersion VERSION_API_KEY_ROLES_AS_BYTES = TransportVersion.V_7_9_0;
+    public static final TransportVersion VERSION_REALM_DOMAINS = TransportVersion.V_8_2_0;
     private final AuthenticationType type;
     private final Subject authenticatingSubject;
     private final Subject effectiveSubject;
@@ -124,9 +147,15 @@ public Authentication(StreamInput in) throws IOException {
         // 4. lookedUpBy != null -> successful run-as -> innerUser must be NOT null
         assert innerUser != null || lookedUpBy == null : "Authentication has no inner-user, but looked-up-by is [" + lookedUpBy + "]";
 
-        final Version version = in.getVersion();
-        type = AuthenticationType.values()[in.readVInt()];
-        final Map<String, Object> metadata = in.readMap();
+        final TransportVersion version = in.getTransportVersion();
+        final Map<String, Object> metadata;
+        if (version.onOrAfter(VERSION_AUTHENTICATION_TYPE)) {
+            type = AuthenticationType.values()[in.readVInt()];
+            metadata = in.readMap();
+        } else {
+            type = AuthenticationType.REALM;
+            metadata = Map.of();
+        }
         if (innerUser != null) {
             authenticatingSubject = new Subject(innerUser, authenticatedBy, version, metadata);
             // The lookup user for run-as currently doesn't have authentication metadata associated with them because
@@ -167,34 +196,43 @@ public boolean isRunAs() {
         return authenticatingSubject != effectiveSubject;
     }
 
-    /**
-     * Get the realm where the effective user comes from.
-     * The effective user is the es-security-runas-user if present or the authenticated user.
-     *
-     * Use {@code getEffectiveSubject().getRealm()} instead.
-     */
-    @Deprecated
-    public RealmRef getSourceRealm() {
-        // TODO: This code retains the existing behaviour which is slightly wrong because
-        // when run-as lookup fails, the effectiveSubject will have a null realm. In this
-        // case, the code returns the authenticatingSubject's realm. This is wrong in theory
-        // because it is not the intention of this method. In practice, it does not matter
-        // because failed lookup will be rejected at authZ time. But fixing it causes test
-        // failures. So leave it for now.
-        final RealmRef sourceRealm = effectiveSubject.getRealm();
-        return sourceRealm == null ? authenticatingSubject.getRealm() : sourceRealm;
+    public boolean isFailedRunAs() {
+        return isRunAs() && effectiveSubject.getRealm() == null;
     }
 
     /**
      * Returns a new {@code Authentication}, like this one, but which is compatible with older version nodes.
      * This is commonly employed when the {@code Authentication} is serialized across cluster nodes with mixed versions.
      */
-    public Authentication maybeRewriteForOlderVersion(Version olderVersion) {
+    public Authentication maybeRewriteForOlderVersion(TransportVersion olderVersion) {
+        return maybeRewriteForOlderVersion(olderVersion, VERSION_REMOTE_ACCESS_REALM);
+    }
+
+    /**
+     * @param remoteAccessRealmVersion using this instead of {@link #VERSION_REMOTE_ACCESS_REALM} enables us to test the new rewriting logic
+     *                                introduced to handle remote access authentication
+     *                                 (see {@link #maybeRewriteMetadataForRemoteAccessAuthentication(TransportVersion, Authentication)}
+     *                                we've introduced for future-proofing, while bypassing the remote access min version check.
+     *                                 This is only necessary in 8.7.
+     * TODO remove this method in 8.8, and inline {@link #VERSION_REMOTE_ACCESS_REALM}
+     */
+    Authentication maybeRewriteForOlderVersion(TransportVersion olderVersion, TransportVersion remoteAccessRealmVersion) {
         // TODO how can this not be true
         // assert olderVersion.onOrBefore(getVersion());
 
-        final Map<String, Object> newMetadata = maybeRewriteMetadataForApiKeyRoleDescriptors(olderVersion, this);
+        // remote access introduced a new synthetic realm and subject type; these cannot be parsed by older versions, so rewriting is not
+        // possible
+        if (isRemoteAccess() && olderVersion.before(remoteAccessRealmVersion)) {
+            throw new IllegalArgumentException(
+                "versions of Elasticsearch before ["
+                    + remoteAccessRealmVersion
+                    + "] can't handle remote access authentication and attempted to rewrite for ["
+                    + olderVersion
+                    + "]"
+            );
+        }
 
+        final Map<String, Object> newMetadata = maybeRewriteMetadata(olderVersion, this);
         final Authentication newAuthentication;
         if (isRunAs()) {
             // The lookup user for run-as currently doesn't have authentication metadata associated with them because
@@ -228,12 +266,19 @@ public Authentication maybeRewriteForOlderVersion(Version olderVersion) {
             );
 
         }
-        if (isAssignedToDomain() && false == newAuthentication.isAssignedToDomain()) {
-            logger.info("Rewriting authentication [" + this + "] without domain");
-        }
         return newAuthentication;
     }
 
+    private static Map<String, Object> maybeRewriteMetadata(TransportVersion olderVersion, Authentication authentication) {
+        if (authentication.isAuthenticatedAsApiKey()) {
+            return maybeRewriteMetadataForApiKeyRoleDescriptors(olderVersion, authentication);
+        } else if (authentication.isRemoteAccess()) {
+            return maybeRewriteMetadataForRemoteAccessAuthentication(olderVersion, authentication);
+        } else {
+            return authentication.getAuthenticatingSubject().getMetadata();
+        }
+    }
+
     /**
      * Returns a new {@code Authentication} that reflects a "run as another user" action under the current {@code Authentication}.
      * The security {@code RealmRef#Domain} of the resulting {@code Authentication} is that of the run-as user's realm.
@@ -250,7 +295,7 @@ public Authentication runAs(User runAs, @Nullable RealmRef lookupRealmRef) {
 
         Objects.requireNonNull(runAs);
         return new Authentication(
-            new Subject(runAs, lookupRealmRef, getEffectiveSubject().getVersion(), Map.of()),
+            new Subject(runAs, lookupRealmRef, getEffectiveSubject().getTransportVersion(), Map.of()),
             authenticatingSubject,
             type
         );
@@ -261,8 +306,8 @@ public Authentication runAs(User runAs, @Nullable RealmRef lookupRealmRef) {
      */
     public Authentication token() {
         assert false == isServiceAccount();
+        assert false == isRemoteAccess();
         final Authentication newTokenAuthentication = new Authentication(effectiveSubject, authenticatingSubject, AuthenticationType.TOKEN);
-        assert Objects.equals(getDomain(), newTokenAuthentication.getDomain());
         return newTokenAuthentication;
     }
 
@@ -286,6 +331,7 @@ public Authentication maybeAddAnonymousRoles(@Nullable AnonymousUser anonymousUs
             && false == anonymousUser.equals(getEffectiveSubject().getUser())
             && false == User.isInternal(getEffectiveSubject().getUser())
             && false == isApiKey()
+            && false == isRemoteAccess()
             && false == isServiceAccount();
 
         if (false == shouldAddAnonymousRoleNames) {
@@ -304,7 +350,7 @@ public Authentication maybeAddAnonymousRoles(@Nullable AnonymousUser anonymousUs
                 new Subject(
                     new User(user.principal(), allRoleNames, user.fullName(), user.email(), user.metadata(), user.enabled()),
                     effectiveSubject.getRealm(),
-                    effectiveSubject.getVersion(),
+                    effectiveSubject.getTransportVersion(),
                     effectiveSubject.getMetadata()
                 ),
                 authenticatingSubject,
@@ -317,7 +363,7 @@ public Authentication maybeAddAnonymousRoles(@Nullable AnonymousUser anonymousUs
                 new Subject(
                     new User(user.principal(), allRoleNames, user.fullName(), user.email(), user.metadata(), user.enabled()),
                     authenticatingSubject.getRealm(),
-                    authenticatingSubject.getVersion(),
+                    authenticatingSubject.getTransportVersion(),
                     authenticatingSubject.getMetadata()
                 ),
                 type
@@ -325,14 +371,15 @@ public Authentication maybeAddAnonymousRoles(@Nullable AnonymousUser anonymousUs
         }
     }
 
+    // Package private for tests
     /**
      * Returns {@code true} if the effective user belongs to a realm under a domain.
-     * See also {@link #getDomain()} and {@link #getSourceRealm()}.
      */
-    public boolean isAssignedToDomain() {
+    boolean isAssignedToDomain() {
         return getDomain() != null;
     }
 
+    // Package private for tests
     /**
      * Returns the {@link RealmDomain} that the effective user belongs to.
      * A user belongs to a realm which in turn belongs to a domain.
@@ -340,8 +387,12 @@ public boolean isAssignedToDomain() {
      * The same username can be authenticated by different realms (e.g. with different credential types),
      * but resources created across realms cannot be accessed unless the realms are also part of the same domain.
      */
-    public @Nullable RealmDomain getDomain() {
-        return getSourceRealm().getDomain();
+    @Nullable
+    RealmDomain getDomain() {
+        if (isFailedRunAs()) {
+            return null;
+        }
+        return getEffectiveSubject().getRealm().getDomain();
     }
 
     public boolean isAuthenticatedWithServiceAccount() {
@@ -379,6 +430,10 @@ public boolean isApiKey() {
         return effectiveSubject.getType() == Subject.Type.API_KEY;
     }
 
+    public boolean isRemoteAccess() {
+        return effectiveSubject.getType() == Subject.Type.REMOTE_ACCESS;
+    }
+
     /**
      * Whether the authentication can run-as another user
      */
@@ -393,6 +448,12 @@ public boolean supportsRunAs(@Nullable AnonymousUser anonymousUser) {
             return false;
         }
 
+        // Real run-as for remote access could happen on the querying cluster side, but not on the fulfilling cluster. Since the
+        // authentication instance corresponds to the fulfilling-cluster-side view, run-as is not supported
+        if (isRemoteAccess()) {
+            return false;
+        }
+
         // There is no reason for internal users to run-as. This check prevents either internal user itself
         // or a token created for it (though no such thing in current code) to run-as.
         if (User.isInternal(getEffectiveSubject().getUser())) {
@@ -437,13 +498,24 @@ public void writeToContext(ThreadContext ctx) throws IOException, IllegalArgumen
 
     public String encode() throws IOException {
         BytesStreamOutput output = new BytesStreamOutput();
-        output.setVersion(getEffectiveSubject().getVersion());
-        Version.writeVersion(getEffectiveSubject().getVersion(), output);
+        output.setTransportVersion(getEffectiveSubject().getTransportVersion());
+        TransportVersion.writeVersion(getEffectiveSubject().getTransportVersion(), output);
         writeTo(output);
         return Base64.getEncoder().encodeToString(BytesReference.toBytes(output.bytes()));
     }
 
     public void writeTo(StreamOutput out) throws IOException {
+        // remote access introduced a new synthetic realm and subject type; these cannot be parsed by older versions, so rewriting we should
+        // not send them across the wire to older nodes
+        if (isRemoteAccess() && out.getTransportVersion().before(VERSION_REMOTE_ACCESS_REALM)) {
+            throw new IllegalArgumentException(
+                "versions of Elasticsearch before ["
+                    + VERSION_REMOTE_ACCESS_REALM
+                    + "] can't handle remote access authentication and attempted to send to ["
+                    + out.getTransportVersion()
+                    + "]"
+            );
+        }
         if (isRunAs()) {
             final User outerUser = effectiveSubject.getUser();
             final User innerUser = authenticatingSubject.getUser();
@@ -466,8 +538,20 @@ public void writeTo(StreamOutput out) throws IOException {
         } else {
             out.writeBoolean(false);
         }
-        out.writeVInt(type.ordinal());
-        out.writeGenericMap(getAuthenticatingSubject().getMetadata());
+        final Map<String, Object> metadata = getAuthenticatingSubject().getMetadata();
+        if (out.getTransportVersion().onOrAfter(VERSION_AUTHENTICATION_TYPE)) {
+            out.writeVInt(type.ordinal());
+            out.writeGenericMap(metadata);
+        } else {
+            assert type == AuthenticationType.REALM && metadata.isEmpty()
+                : Strings.format(
+                    "authentication with version [%s] must have authentication type %s and empty metadata, but got [%s] and [%s]",
+                    out.getTransportVersion(),
+                    AuthenticationType.REALM,
+                    type,
+                    metadata
+                );
+        }
     }
 
     /**
@@ -569,7 +653,7 @@ public void toXContentFragment(XContentBuilder builder) throws IOException {
         }
         builder.endObject();
         builder.field(User.Fields.AUTHENTICATION_TYPE.getPreferredName(), getAuthenticationType().name().toLowerCase(Locale.ROOT));
-        if (isApiKey()) {
+        if (isApiKey() || isRemoteAccess()) {
             final String apiKeyId = (String) getAuthenticatingSubject().getMetadata().get(AuthenticationField.API_KEY_ID_KEY);
             final String apiKeyName = (String) getAuthenticatingSubject().getMetadata().get(AuthenticationField.API_KEY_NAME_KEY);
             if (apiKeyName == null) {
@@ -588,7 +672,7 @@ private void assertInternalConsistency() {
         assert effectiveSubject != null;
         assert authenticatingSubject != null;
         assert type != null;
-        assert effectiveSubject.getVersion().equals(authenticatingSubject.getVersion());
+        assert effectiveSubject.getTransportVersion().equals(authenticatingSubject.getTransportVersion());
 
         if (isRunAs()) {
             assert authenticatingSubject != effectiveSubject : "isRunAs logic does not hold";
@@ -599,13 +683,23 @@ private void assertInternalConsistency() {
         }
 
         // Assert API key metadata
-        assert (false == isAuthenticatedAsApiKey())
+        assert (false == (isAuthenticatedAsApiKey() || isRemoteAccess()))
             || (getAuthenticatingSubject().getMetadata().get(AuthenticationField.API_KEY_ID_KEY) != null)
             : "API KEY authentication requires metadata to contain API KEY id, and the value must be non-null.";
 
+        if (isRemoteAccess()) {
+            assert getAuthenticatingSubject().getMetadata().get(AuthenticationField.REMOTE_ACCESS_AUTHENTICATION_KEY) != null
+                : "Remote access authentication requires metadata to contain a serialized remote access authentication, "
+                    + "and the value must be non-null.";
+            assert getAuthenticatingSubject().getMetadata().get(AuthenticationField.REMOTE_ACCESS_ROLE_DESCRIPTORS_KEY) != null
+                : "Remote access authentication requires metadata to contain a serialized remote access role descriptors, "
+                    + "and the value must be non-null.";
+        }
+
         // Assert domain assignment
         if (isAssignedToDomain()) {
             assert false == isApiKey();
+            assert false == isRemoteAccess();
             assert false == isServiceAccount();
             assert false == isAuthenticatedAnonymously();
             assert false == isAuthenticatedInternally();
@@ -616,12 +710,24 @@ private boolean hasSyntheticRealmNameOrType(@Nullable RealmRef realmRef) {
         if (realmRef == null) {
             return false;
         }
-        if (List.of(API_KEY_REALM_NAME, ServiceAccountSettings.REALM_NAME, ANONYMOUS_REALM_NAME, FALLBACK_REALM_NAME, ATTACH_REALM_NAME)
-            .contains(realmRef.getName())) {
+        if (List.of(
+            API_KEY_REALM_NAME,
+            ServiceAccountSettings.REALM_NAME,
+            ANONYMOUS_REALM_NAME,
+            FALLBACK_REALM_NAME,
+            ATTACH_REALM_NAME,
+            REMOTE_ACCESS_REALM_NAME
+        ).contains(realmRef.getName())) {
             return true;
         }
-        if (List.of(API_KEY_REALM_TYPE, ServiceAccountSettings.REALM_TYPE, ANONYMOUS_REALM_TYPE, FALLBACK_REALM_TYPE, ATTACH_REALM_TYPE)
-            .contains(realmRef.getType())) {
+        if (List.of(
+            API_KEY_REALM_TYPE,
+            ServiceAccountSettings.REALM_TYPE,
+            ANONYMOUS_REALM_TYPE,
+            FALLBACK_REALM_TYPE,
+            ATTACH_REALM_TYPE,
+            REMOTE_ACCESS_REALM_TYPE
+        ).contains(realmRef.getType())) {
             return true;
         }
         return false;
@@ -660,7 +766,7 @@ public RealmRef(StreamInput in) throws IOException {
             this.nodeName = in.readString();
             this.name = in.readString();
             this.type = in.readString();
-            if (in.getVersion().onOrAfter(VERSION_REALM_DOMAINS)) {
+            if (in.getTransportVersion().onOrAfter(VERSION_REALM_DOMAINS)) {
                 this.domain = in.readOptionalWriteable(RealmDomain::readFrom);
             } else {
                 this.domain = null;
@@ -672,7 +778,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(nodeName);
             out.writeString(name);
             out.writeString(type);
-            if (out.getVersion().onOrAfter(VERSION_REALM_DOMAINS)) {
+            if (out.getTransportVersion().onOrAfter(VERSION_REALM_DOMAINS)) {
                 out.writeOptionalWriteable(domain);
             }
         }
@@ -769,6 +875,11 @@ static RealmRef newApiKeyRealmRef(String nodeName) {
             // no domain for API Key tokens
             return new RealmRef(API_KEY_REALM_NAME, API_KEY_REALM_TYPE, nodeName, null);
         }
+
+        static RealmRef newRemoteAccessRealmRef(String nodeName) {
+            // no domain for remote access authentication
+            return new RealmRef(REMOTE_ACCESS_REALM_NAME, REMOTE_ACCESS_REALM_TYPE, nodeName, null);
+        }
     }
 
     public static boolean isFileOrNativeRealm(String realmType) {
@@ -789,7 +900,7 @@ public static boolean isFileOrNativeRealm(String realmType) {
     }
 
     // TODO is a newer version than the node's a valid value?
-    public static Authentication newInternalAuthentication(User internalUser, Version version, String nodeName) {
+    public static Authentication newInternalAuthentication(User internalUser, TransportVersion version, String nodeName) {
         // TODO create a system user class, so that the type system guarantees that this is only invoked for internal users
         assert User.isInternal(internalUser);
         final Authentication.RealmRef authenticatedBy = newInternalAttachRealmRef(nodeName);
@@ -805,7 +916,7 @@ public static Authentication newInternalFallbackAuthentication(User fallbackUser
         // TODO assert SystemUser.is(fallbackUser);
         final Authentication.RealmRef authenticatedBy = newInternalFallbackRealmRef(nodeName);
         Authentication authentication = new Authentication(
-            new Subject(fallbackUser, authenticatedBy, Version.CURRENT, Map.of()),
+            new Subject(fallbackUser, authenticatedBy, TransportVersion.CURRENT, Map.of()),
             Authentication.AuthenticationType.INTERNAL
         );
         assert false == authentication.isAssignedToDomain();
@@ -815,7 +926,7 @@ public static Authentication newInternalFallbackAuthentication(User fallbackUser
     public static Authentication newAnonymousAuthentication(AnonymousUser anonymousUser, String nodeName) {
         final Authentication.RealmRef authenticatedBy = newAnonymousRealmRef(nodeName);
         Authentication authentication = new Authentication(
-            new Subject(anonymousUser, authenticatedBy, Version.CURRENT, Map.of()),
+            new Subject(anonymousUser, authenticatedBy, TransportVersion.CURRENT, Map.of()),
             Authentication.AuthenticationType.ANONYMOUS
         );
         assert false == authentication.isAssignedToDomain();
@@ -826,7 +937,7 @@ public static Authentication newServiceAccountAuthentication(User serviceAccount
         // TODO make the service account user a separate class/interface
         final Authentication.RealmRef authenticatedBy = newServiceAccountRealmRef(nodeName);
         Authentication authentication = new Authentication(
-            new Subject(serviceAccountUser, authenticatedBy, Version.CURRENT, metadata),
+            new Subject(serviceAccountUser, authenticatedBy, TransportVersion.CURRENT, metadata),
             AuthenticationType.TOKEN
         );
         assert false == authentication.isAssignedToDomain();
@@ -836,11 +947,12 @@ public static Authentication newServiceAccountAuthentication(User serviceAccount
     public static Authentication newRealmAuthentication(User user, RealmRef realmRef) {
         // TODO make the type system ensure that this is not a run-as user
         Authentication authentication = new Authentication(
-            new Subject(user, realmRef, Version.CURRENT, Map.of()),
+            new Subject(user, realmRef, TransportVersion.CURRENT, Map.of()),
             AuthenticationType.REALM
         );
         assert false == authentication.isServiceAccount();
         assert false == authentication.isApiKey();
+        assert false == authentication.isRemoteAccess();
         assert false == authentication.isAuthenticatedInternally();
         assert false == authentication.isAuthenticatedAnonymously();
         return authentication;
@@ -852,23 +964,56 @@ public static Authentication newApiKeyAuthentication(AuthenticationResult<User>
         assert apiKeyUser.roles().length == 0 : "The user associated to an API key authentication must have no role";
         final Authentication.RealmRef authenticatedBy = newApiKeyRealmRef(nodeName);
         Authentication authentication = new Authentication(
-            new Subject(apiKeyUser, authenticatedBy, Version.CURRENT, authResult.getMetadata()),
+            new Subject(apiKeyUser, authenticatedBy, TransportVersion.CURRENT, authResult.getMetadata()),
             AuthenticationType.API_KEY
         );
         assert false == authentication.isAssignedToDomain();
         return authentication;
     }
 
-    private static RealmRef maybeRewriteRealmRef(Version streamVersion, RealmRef realmRef) {
+    public Authentication toRemoteAccess(RemoteAccessAuthentication remoteAccessAuthentication) {
+        assert isApiKey() : "can only convert API key authentication to remote access";
+        assert false == isRunAs() : "remote access does not support authentication with run-as";
+        final Map<String, Object> metadata = new HashMap<>(getAuthenticatingSubject().getMetadata());
+        final Authentication.RealmRef authenticatedBy = newRemoteAccessRealmRef(getAuthenticatingSubject().getRealm().getNodeName());
+        final User userFromRemoteCluster = remoteAccessAuthentication.getAuthentication().getEffectiveSubject().getUser();
+        assert userFromRemoteCluster.enabled() : "the user received from a remote cluster must be enabled";
+        final User userWithoutRoles = new User(
+            userFromRemoteCluster.principal(),
+            EMPTY_ARRAY,
+            userFromRemoteCluster.fullName(),
+            userFromRemoteCluster.email(),
+            userFromRemoteCluster.metadata(),
+            userFromRemoteCluster.enabled()
+        );
+        final Authentication authentication = new Authentication(
+            new Subject(
+                userWithoutRoles,
+                authenticatedBy,
+                TransportVersion.CURRENT,
+                remoteAccessAuthentication.copyWithRemoteAccessEntries(metadata)
+            ),
+            getAuthenticationType()
+        );
+        assert false == authentication.isAssignedToDomain();
+        return authentication;
+    }
+
+    // pkg-private for testing
+    static RealmRef maybeRewriteRealmRef(TransportVersion streamVersion, RealmRef realmRef) {
         if (realmRef != null && realmRef.getDomain() != null && streamVersion.before(VERSION_REALM_DOMAINS)) {
+            logger.info("Rewriting realm [" + realmRef + "] without domain");
             // security domain erasure
-            new RealmRef(realmRef.getName(), realmRef.getType(), realmRef.getNodeName(), null);
+            return new RealmRef(realmRef.getName(), realmRef.getType(), realmRef.getNodeName(), null);
         }
         return realmRef;
     }
 
     @SuppressWarnings("unchecked")
-    private static Map<String, Object> maybeRewriteMetadataForApiKeyRoleDescriptors(Version streamVersion, Authentication authentication) {
+    private static Map<String, Object> maybeRewriteMetadataForApiKeyRoleDescriptors(
+        TransportVersion streamVersion,
+        Authentication authentication
+    ) {
         Map<String, Object> metadata = authentication.getAuthenticatingSubject().getMetadata();
         // If authentication user is an API key or a token created by an API key,
         // regardless whether it has run-as, the metadata must contain API key role descriptors
@@ -877,7 +1022,7 @@ private static Map<String, Object> maybeRewriteMetadataForApiKeyRoleDescriptors(
                 : "metadata must contain role descriptor for API key authentication";
             assert metadata.containsKey(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY)
                 : "metadata must contain limited role descriptor for API key authentication";
-            if (authentication.getEffectiveSubject().getVersion().onOrAfter(VERSION_API_KEY_ROLES_AS_BYTES)
+            if (authentication.getEffectiveSubject().getTransportVersion().onOrAfter(VERSION_API_KEY_ROLES_AS_BYTES)
                 && streamVersion.before(VERSION_API_KEY_ROLES_AS_BYTES)) {
                 metadata = new HashMap<>(metadata);
                 metadata.put(
@@ -890,7 +1035,7 @@ private static Map<String, Object> maybeRewriteMetadataForApiKeyRoleDescriptors(
                         (BytesReference) metadata.get(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY)
                     )
                 );
-            } else if (authentication.getEffectiveSubject().getVersion().before(VERSION_API_KEY_ROLES_AS_BYTES)
+            } else if (authentication.getEffectiveSubject().getTransportVersion().before(VERSION_API_KEY_ROLES_AS_BYTES)
                 && streamVersion.onOrAfter(VERSION_API_KEY_ROLES_AS_BYTES)) {
                     metadata = new HashMap<>(metadata);
                     metadata.put(
@@ -910,6 +1055,37 @@ private static Map<String, Object> maybeRewriteMetadataForApiKeyRoleDescriptors(
         return metadata;
     }
 
+    // pkg-private for testing
+    static Map<String, Object> maybeRewriteMetadataForRemoteAccessAuthentication(
+        final TransportVersion olderVersion,
+        final Authentication authentication
+    ) {
+        assert authentication.isRemoteAccess() : "authentication must be remote access";
+        final Map<String, Object> metadata = authentication.getAuthenticatingSubject().getMetadata();
+        assert metadata.containsKey(AuthenticationField.REMOTE_ACCESS_AUTHENTICATION_KEY)
+            : "metadata must contain authentication object for remote access authentication";
+        try {
+            final Authentication authenticationFromMetadata = AuthenticationContextSerializer.decode(
+                (String) metadata.get(AuthenticationField.REMOTE_ACCESS_AUTHENTICATION_KEY)
+            );
+            if (authenticationFromMetadata.getEffectiveSubject().getTransportVersion().after(olderVersion)) {
+                final Map<String, Object> rewrittenMetadata = new HashMap<>(metadata);
+                rewrittenMetadata.put(
+                    AuthenticationField.REMOTE_ACCESS_AUTHENTICATION_KEY,
+                    authenticationFromMetadata.maybeRewriteForOlderVersion(olderVersion).encode()
+                );
+                return rewrittenMetadata;
+            } else {
+                return metadata;
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException(
+                "failed serialization while rewriting [" + AuthenticationField.REMOTE_ACCESS_AUTHENTICATION_KEY + "] metadata field",
+                e
+            );
+        }
+    }
+
     private static Map<String, Object> convertRoleDescriptorsBytesToMap(BytesReference roleDescriptorsBytes) {
         return XContentHelper.convertToMap(roleDescriptorsBytes, false, XContentType.JSON).v2();
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationField.java
index 8a955452deda..a3737d72649e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationField.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationField.java
@@ -33,5 +33,10 @@ public final class AuthenticationField {
     public static final String ATTACH_REALM_NAME = "__attach";
     public static final String ATTACH_REALM_TYPE = "__attach";
 
+    public static final String REMOTE_ACCESS_REALM_NAME = "_es_remote_access";
+    public static final String REMOTE_ACCESS_REALM_TYPE = "_es_remote_access";
+    public static final String REMOTE_ACCESS_AUTHENTICATION_KEY = "_security_remote_access_authentication";
+    public static final String REMOTE_ACCESS_ROLE_DESCRIPTORS_KEY = "_security_remote_access_role_descriptors";
+
     private AuthenticationField() {}
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RemoteAccessAuthentication.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RemoteAccessAuthentication.java
new file mode 100644
index 000000000000..d6c50bf66bd1
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RemoteAccessAuthentication.java
@@ -0,0 +1,214 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.authc;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.common.bytes.AbstractBytesReference;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+public final class RemoteAccessAuthentication {
+    public static final String REMOTE_ACCESS_AUTHENTICATION_HEADER_KEY = "_remote_access_authentication";
+    private final Authentication authentication;
+    private final List<RoleDescriptorsBytes> roleDescriptorsBytesList;
+
+    public RemoteAccessAuthentication(Authentication authentication, RoleDescriptorsIntersection roleDescriptorsIntersection)
+        throws IOException {
+        this(authentication, toRoleDescriptorsBytesList(roleDescriptorsIntersection));
+    }
+
+    private RemoteAccessAuthentication(Authentication authentication, List<RoleDescriptorsBytes> roleDescriptorsBytesList) {
+        this.authentication = authentication;
+        this.roleDescriptorsBytesList = roleDescriptorsBytesList;
+    }
+
+    public void writeToContext(final ThreadContext ctx) throws IOException {
+        ctx.putHeader(REMOTE_ACCESS_AUTHENTICATION_HEADER_KEY, encode());
+    }
+
+    public static RemoteAccessAuthentication readFromContext(final ThreadContext ctx) throws IOException {
+        return decode(ctx.getHeader(REMOTE_ACCESS_AUTHENTICATION_HEADER_KEY));
+    }
+
+    public Authentication getAuthentication() {
+        return authentication;
+    }
+
+    public List<RoleDescriptorsBytes> getRoleDescriptorsBytesList() {
+        return roleDescriptorsBytesList;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        RemoteAccessAuthentication that = (RemoteAccessAuthentication) o;
+
+        if (false == authentication.equals(that.authentication)) return false;
+        return roleDescriptorsBytesList.equals(that.roleDescriptorsBytesList);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = authentication.hashCode();
+        result = 31 * result + roleDescriptorsBytesList.hashCode();
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return "RemoteAccessAuthentication{"
+            + "authentication="
+            + authentication
+            + ", roleDescriptorsBytesList="
+            + roleDescriptorsBytesList
+            + '}';
+    }
+
+    private static List<RoleDescriptorsBytes> toRoleDescriptorsBytesList(final RoleDescriptorsIntersection roleDescriptorsIntersection)
+        throws IOException {
+        // If we ever lift this restriction, we need to ensure that the serialization of each set of role descriptors to raw bytes is
+        // deterministic. We can do so by sorting the role descriptors before serializing.
+        assert roleDescriptorsIntersection.roleDescriptorsList().stream().noneMatch(rds -> rds.size() > 1)
+            : "sets with more than one role descriptor are not supported for remote access authentication";
+        final List<RoleDescriptorsBytes> roleDescriptorsBytesList = new ArrayList<>();
+        for (Set<RoleDescriptor> roleDescriptors : roleDescriptorsIntersection.roleDescriptorsList()) {
+            roleDescriptorsBytesList.add(RoleDescriptorsBytes.fromRoleDescriptors(roleDescriptors));
+        }
+        return roleDescriptorsBytesList;
+    }
+
+    public String encode() throws IOException {
+        final BytesStreamOutput out = new BytesStreamOutput();
+        out.setTransportVersion(authentication.getEffectiveSubject().getTransportVersion());
+        TransportVersion.writeVersion(authentication.getEffectiveSubject().getTransportVersion(), out);
+        authentication.writeTo(out);
+        out.writeCollection(roleDescriptorsBytesList, StreamOutput::writeBytesReference);
+        return Base64.getEncoder().encodeToString(BytesReference.toBytes(out.bytes()));
+    }
+
+    public static RemoteAccessAuthentication decode(final String header) throws IOException {
+        Objects.requireNonNull(header);
+        final byte[] bytes = Base64.getDecoder().decode(header);
+        final StreamInput in = StreamInput.wrap(bytes);
+        final TransportVersion version = TransportVersion.readVersion(in);
+        in.setTransportVersion(version);
+        final Authentication authentication = new Authentication(in);
+        final List<RoleDescriptorsBytes> roleDescriptorsBytesList = in.readImmutableList(RoleDescriptorsBytes::new);
+        return new RemoteAccessAuthentication(authentication, roleDescriptorsBytesList);
+    }
+
+    /**
+     * Returns a copy of the passed-in metadata map, with the relevant remote access fields included. Does not modify the original map.
+     */
+    public Map<String, Object> copyWithRemoteAccessEntries(final Map<String, Object> authenticationMetadata) {
+        assert false == authenticationMetadata.containsKey(AuthenticationField.REMOTE_ACCESS_AUTHENTICATION_KEY)
+            : "metadata already contains [" + AuthenticationField.REMOTE_ACCESS_AUTHENTICATION_KEY + "] entry";
+        assert false == authenticationMetadata.containsKey(AuthenticationField.REMOTE_ACCESS_ROLE_DESCRIPTORS_KEY)
+            : "metadata already contains [" + AuthenticationField.REMOTE_ACCESS_ROLE_DESCRIPTORS_KEY + "] entry";
+        assert false == getAuthentication().isRemoteAccess()
+            : "authentication included in remote access header cannot itself be remote access";
+        final Map<String, Object> copy = new HashMap<>(authenticationMetadata);
+        try {
+            copy.put(AuthenticationField.REMOTE_ACCESS_AUTHENTICATION_KEY, getAuthentication().encode());
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+        copy.put(AuthenticationField.REMOTE_ACCESS_ROLE_DESCRIPTORS_KEY, getRoleDescriptorsBytesList());
+        return Collections.unmodifiableMap(copy);
+    }
+
+    public static final class RoleDescriptorsBytes extends AbstractBytesReference {
+
+        public static final RoleDescriptorsBytes EMPTY = new RoleDescriptorsBytes(new BytesArray("{}"));
+        private final BytesReference rawBytes;
+
+        public RoleDescriptorsBytes(BytesReference rawBytes) {
+            this.rawBytes = rawBytes;
+        }
+
+        public RoleDescriptorsBytes(StreamInput streamInput) throws IOException {
+            this(streamInput.readBytesReference());
+        }
+
+        public static RoleDescriptorsBytes fromRoleDescriptors(final Set<RoleDescriptor> roleDescriptors) throws IOException {
+            final XContentBuilder builder = XContentFactory.jsonBuilder();
+            builder.startObject();
+            for (RoleDescriptor roleDescriptor : roleDescriptors) {
+                builder.field(roleDescriptor.getName(), roleDescriptor);
+            }
+            builder.endObject();
+            return new RoleDescriptorsBytes(BytesReference.bytes(builder));
+        }
+
+        public Set<RoleDescriptor> toRoleDescriptors() {
+            try (XContentParser parser = XContentHelper.createParser(XContentParserConfiguration.EMPTY, rawBytes, XContentType.JSON)) {
+                final List<RoleDescriptor> roleDescriptors = new ArrayList<>();
+                parser.nextToken();
+                while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                    parser.nextToken();
+                    final String roleName = parser.currentName();
+                    roleDescriptors.add(RoleDescriptor.parse(roleName, parser, false));
+                }
+                return Set.copyOf(roleDescriptors);
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        }
+
+        @Override
+        public byte get(int index) {
+            return rawBytes.get(index);
+        }
+
+        @Override
+        public int length() {
+            return rawBytes.length();
+        }
+
+        @Override
+        public BytesReference slice(int from, int length) {
+            return rawBytes.slice(from, length);
+        }
+
+        @Override
+        public long ramBytesUsed() {
+            return rawBytes.ramBytesUsed();
+        }
+
+        @Override
+        public BytesRef toBytesRef() {
+            return rawBytes.toBytesRef();
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Subject.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Subject.java
index 2a8fa800b2ae..5ff7d9749d2b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Subject.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Subject.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.xpack.core.security.authc;
 
 import org.elasticsearch.ElasticsearchSecurityException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.util.ArrayUtils;
@@ -19,6 +19,8 @@
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.security.user.User;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
@@ -26,10 +28,11 @@
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY;
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY;
 import static org.elasticsearch.xpack.core.security.authc.Subject.Type.API_KEY;
+import static org.elasticsearch.xpack.core.security.authc.Subject.Type.REMOTE_ACCESS;
 
 /**
  * A subject is a more generic concept similar to user and associated to the current authentication.
- * It is more generic than user because it can also represent API keys and service accounts.
+ * It is more generic than user because it can also represent API keys, service accounts, or remote access users.
  * It also contains authentication level information, e.g. realm and metadata so that it can answer
  * queries in a better encapsulated way.
  */
@@ -39,19 +42,20 @@ public enum Type {
         USER,
         API_KEY,
         SERVICE_ACCOUNT,
+        REMOTE_ACCESS,
     }
 
-    private final Version version;
+    private final TransportVersion version;
     private final User user;
     private final Authentication.RealmRef realm;
     private final Type type;
     private final Map<String, Object> metadata;
 
     public Subject(User user, Authentication.RealmRef realm) {
-        this(user, realm, Version.CURRENT, Map.of());
+        this(user, realm, TransportVersion.CURRENT, Map.of());
     }
 
-    public Subject(User user, Authentication.RealmRef realm, Version version, Map<String, Object> metadata) {
+    public Subject(User user, Authentication.RealmRef realm, TransportVersion version, Map<String, Object> metadata) {
         this.version = version;
         this.user = user;
         this.realm = realm;
@@ -65,6 +69,9 @@ public Subject(User user, Authentication.RealmRef realm, Version version, Map<St
         } else if (ServiceAccountSettings.REALM_TYPE.equals(realm.getType())) {
             assert ServiceAccountSettings.REALM_NAME.equals(realm.getName()) : "service account realm name mismatch";
             this.type = Type.SERVICE_ACCOUNT;
+        } else if (AuthenticationField.REMOTE_ACCESS_REALM_TYPE.equals(realm.getType())) {
+            assert AuthenticationField.REMOTE_ACCESS_REALM_NAME.equals(realm.getName()) : "remote access realm name mismatch";
+            this.type = Type.REMOTE_ACCESS;
         } else {
             this.type = Type.USER;
         }
@@ -87,7 +94,7 @@ public Map<String, Object> getMetadata() {
         return metadata;
     }
 
-    public Version getVersion() {
+    public TransportVersion getTransportVersion() {
         return version;
     }
 
@@ -99,6 +106,8 @@ public RoleReferenceIntersection getRoleReferenceIntersection(@Nullable Anonymou
                 return buildRoleReferencesForApiKey();
             case SERVICE_ACCOUNT:
                 return new RoleReferenceIntersection(new RoleReference.ServiceAccountRoleReference(user.principal()));
+            case REMOTE_ACCESS:
+                return buildRoleReferencesForRemoteAccess();
             default:
                 assert false : "unknown subject type: [" + type + "]";
                 throw new IllegalStateException("unknown subject type: [" + type + "]");
@@ -116,6 +125,9 @@ public boolean canAccessResourcesOf(Subject resourceCreatorSubject) {
             || (false == API_KEY.equals(getType()) && API_KEY.equals(resourceCreatorSubject.getType()))) {
                 // an API Key cannot access resources created by non-API Keys or vice-versa
                 return false;
+            } else if (REMOTE_ACCESS.equals(getType()) || REMOTE_ACCESS.equals(resourceCreatorSubject.getType())) {
+                // TODO implement this once remote authentication is fully supported
+                return false;
             } else {
                 if (false == getUser().principal().equals(resourceCreatorSubject.getUser().principal())) {
                     return false;
@@ -220,6 +232,27 @@ private RoleReferenceIntersection buildRoleReferencesForApiKey() {
         );
     }
 
+    private RoleReferenceIntersection buildRoleReferencesForRemoteAccess() {
+        final List<RoleReference> roleReferences = new ArrayList<>(4);
+        @SuppressWarnings("unchecked")
+        final List<RemoteAccessAuthentication.RoleDescriptorsBytes> remoteAccessRoleDescriptorsBytes = (List<
+            RemoteAccessAuthentication.RoleDescriptorsBytes>) metadata.get(AuthenticationField.REMOTE_ACCESS_ROLE_DESCRIPTORS_KEY);
+        if (remoteAccessRoleDescriptorsBytes.isEmpty()) {
+            // If the remote access role descriptors are empty, the remote user has no privileges. We need to add an empty role to restrict
+            // access of the overall intersection accordingly
+            roleReferences.add(new RoleReference.RemoteAccessRoleReference(RemoteAccessAuthentication.RoleDescriptorsBytes.EMPTY));
+        } else {
+            // TODO handle this once we support API keys as querying subjects
+            assert remoteAccessRoleDescriptorsBytes.size() == 1
+                : "only a singleton list of remote access role descriptors bytes is supported";
+            for (RemoteAccessAuthentication.RoleDescriptorsBytes roleDescriptorsBytes : remoteAccessRoleDescriptorsBytes) {
+                roleReferences.add(new RoleReference.RemoteAccessRoleReference(roleDescriptorsBytes));
+            }
+        }
+        roleReferences.addAll(buildRoleReferencesForApiKey().getRoleReferences());
+        return new RoleReferenceIntersection(List.copyOf(roleReferences));
+    }
+
     private static boolean isEmptyRoleDescriptorsBytes(BytesReference roleDescriptorsBytes) {
         return roleDescriptorsBytes == null || (roleDescriptorsBytes.length() == 2 && "{}".equals(roleDescriptorsBytes.utf8ToString()));
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/TokenMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/TokenMetadata.java
index 58366184a491..7d00dadb8466 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/TokenMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/TokenMetadata.java
@@ -6,16 +6,19 @@
  */
 package org.elasticsearch.xpack.core.security.authc;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AbstractNamedDiffable;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 
 public final class TokenMetadata extends AbstractNamedDiffable<ClusterState.Custom> implements ClusterState.Custom {
@@ -63,9 +66,9 @@ public String getWriteableName() {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
         // never render this to the user
-        return builder;
+        return Collections.emptyIterator();
     }
 
     @Override
@@ -90,8 +93,8 @@ public String toString() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.CURRENT.minimumIndexCompatibilityVersion();
+    public TransportVersion getMinimalSupportedVersion() {
+        return Version.CURRENT.minimumIndexCompatibilityVersion().transportVersion;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/jwt/JwtRealmSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/jwt/JwtRealmSettings.java
index 42d84d8ce936..5462c2c42361 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/jwt/JwtRealmSettings.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/jwt/JwtRealmSettings.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
@@ -19,7 +20,9 @@
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -155,6 +158,10 @@ private static Set<Setting.AffixSetting<?>> getNonSecureSettings() {
         // JWT End-user settings
         set.addAll(
             List.of(
+                ALLOWED_SUBJECTS,
+                FALLBACK_SUB_CLAIM,
+                FALLBACK_AUD_CLAIM,
+                REQUIRED_CLAIMS,
                 CLAIMS_PRINCIPAL.getClaim(),
                 CLAIMS_PRINCIPAL.getPattern(),
                 CLAIMS_GROUPS.getClaim(),
@@ -245,6 +252,78 @@ private static Set<Setting.AffixSetting<SecureString>> getSecureSettings() {
 
     // JWT end-user settings
 
+    public static final Setting.AffixSetting<List<String>> ALLOWED_SUBJECTS = Setting.affixKeySetting(
+        RealmSettings.realmSettingPrefix(TYPE),
+        "allowed_subjects",
+        key -> Setting.stringListSetting(key, values -> verifyNonNullNotEmpty(key, values, null), Setting.Property.NodeScope)
+    );
+
+    // Registered claim names from the JWT spec https://www.rfc-editor.org/rfc/rfc7519#section-4.1.
+    // Being registered means they have prescribed meanings when they present in a JWT.
+    public static final List<String> REGISTERED_CLAIM_NAMES = List.of("iss", "sub", "aud", "exp", "nbf", "iat", "jti");
+
+    public static final Setting.AffixSetting<String> FALLBACK_SUB_CLAIM = Setting.affixKeySetting(
+        RealmSettings.realmSettingPrefix(TYPE),
+        "fallback_claims.sub",
+        key -> Setting.simpleString(key, "sub", new Setting.Validator<>() {
+            @Override
+            public void validate(String value) {}
+
+            @Override
+            public void validate(String value, Map<Setting<?>, Object> settings, boolean isPresent) {
+                validateFallbackClaimSetting(FALLBACK_SUB_CLAIM, key, value, settings, isPresent);
+            }
+
+            @Override
+            public Iterator<Setting<?>> settings() {
+                final String namespace = FALLBACK_SUB_CLAIM.getNamespace(FALLBACK_SUB_CLAIM.getConcreteSetting(key));
+                final List<Setting<?>> settings = List.of(TOKEN_TYPE.getConcreteSettingForNamespace(namespace));
+                return settings.iterator();
+            }
+        }, Setting.Property.NodeScope)
+    );
+
+    public static final Setting.AffixSetting<String> FALLBACK_AUD_CLAIM = Setting.affixKeySetting(
+        RealmSettings.realmSettingPrefix(TYPE),
+        "fallback_claims.aud",
+        key -> Setting.simpleString(key, "aud", new Setting.Validator<>() {
+            @Override
+            public void validate(String value) {}
+
+            @Override
+            public void validate(String value, Map<Setting<?>, Object> settings, boolean isPresent) {
+                validateFallbackClaimSetting(FALLBACK_AUD_CLAIM, key, value, settings, isPresent);
+            }
+
+            @Override
+            public Iterator<Setting<?>> settings() {
+                final String namespace = FALLBACK_AUD_CLAIM.getNamespace(FALLBACK_AUD_CLAIM.getConcreteSetting(key));
+                final List<Setting<?>> settings = List.of(TOKEN_TYPE.getConcreteSettingForNamespace(namespace));
+                return settings.iterator();
+            }
+        }, Setting.Property.NodeScope)
+    );
+
+    public static final Setting.AffixSetting<Settings> REQUIRED_CLAIMS = Setting.affixKeySetting(
+        RealmSettings.realmSettingPrefix(TYPE),
+        "required_claims",
+        key -> Setting.groupSetting(key + ".", settings -> {
+            final List<String> invalidRequiredClaims = List.of("iss", "sub", "aud", "exp", "nbf", "iat");
+            for (String name : settings.names()) {
+                final String fullName = key + "." + name;
+                if (invalidRequiredClaims.contains(name)) {
+                    throw new IllegalArgumentException(
+                        Strings.format("required claim [%s] cannot be one of [%s]", fullName, String.join(",", invalidRequiredClaims))
+                    );
+                }
+                final List<String> values = settings.getAsList(name);
+                if (values.isEmpty()) {
+                    throw new IllegalArgumentException(Strings.format("required claim [%s] cannot be empty", fullName));
+                }
+            }
+        }, Setting.Property.NodeScope)
+    );
+
     // Note: ClaimSetting is a wrapper for two individual settings: getClaim(), getPattern()
     public static final ClaimSetting CLAIMS_PRINCIPAL = new ClaimSetting(TYPE, "principal");
     public static final ClaimSetting CLAIMS_GROUPS = new ClaimSetting(TYPE, "groups");
@@ -357,4 +436,48 @@ private static void verifyNonNullNotEmpty(final String key, final List<String> v
         }
     }
 
+    private static void validateFallbackClaimSetting(
+        Setting.AffixSetting<String> setting,
+        String key,
+        String value,
+        Map<Setting<?>, Object> settings,
+        boolean isPresent
+    ) {
+        if (false == isPresent) {
+            return;
+        }
+        final String namespace = setting.getNamespace(setting.getConcreteSetting(key));
+        final TokenType tokenType = (TokenType) settings.get(TOKEN_TYPE.getConcreteSettingForNamespace(namespace));
+        if (tokenType == TokenType.ID_TOKEN) {
+            throw new IllegalArgumentException(
+                Strings.format(
+                    "fallback claim setting [%s] is not allowed when JWT realm [%s] is [%s] type",
+                    key,
+                    namespace,
+                    JwtRealmSettings.TokenType.ID_TOKEN.value()
+                )
+            );
+        }
+        verifyFallbackClaimName(key, value);
+    }
+
+    private static void verifyFallbackClaimName(String key, String fallbackClaimName) {
+        final String claimName = key.substring(key.lastIndexOf(".") + 1);
+        verifyNonNullNotEmpty(key, fallbackClaimName, null);
+        if (claimName.equals(fallbackClaimName)) {
+            return;
+        }
+        // Registered claims have prescribed meanings and should not be used for something else.
+        if (REGISTERED_CLAIM_NAMES.contains(fallbackClaimName)) {
+            throw new IllegalArgumentException(
+                Strings.format(
+                    "Invalid fallback claims setting [%s]. Claim [%s] cannot fallback to a registered claim [%s]",
+                    key,
+                    claimName,
+                    fallbackClaimName
+                )
+            );
+        }
+    }
+
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/jwt/JwtRealmsServiceSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/jwt/JwtRealmsServiceSettings.java
deleted file mode 100644
index e5b98619b0c2..000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/jwt/JwtRealmsServiceSettings.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.core.security.authc.jwt;
-
-import org.elasticsearch.common.settings.Setting;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.function.Function;
-
-/**
- * Settings used by JwtRealmsService for common handling of JWT realm instances.
- */
-public class JwtRealmsServiceSettings {
-
-    public static final List<String> DEFAULT_PRINCIPAL_CLAIMS = List.of("sub", "oid", "client_id", "appid", "azp", "email");
-
-    public static final Setting<List<String>> PRINCIPAL_CLAIMS_SETTING = Setting.listSetting(
-        "xpack.security.authc.jwt.principal_claims",
-        DEFAULT_PRINCIPAL_CLAIMS,
-        Function.identity(),
-        Setting.Property.NodeScope
-    );
-
-    /**
-     * Get all settings shared by all JWT Realms.
-     * @return All settings shared by all JWT Realms.
-     */
-    public static Collection<Setting<?>> getSettings() {
-        return List.of(PRINCIPAL_CLAIMS_SETTING);
-    }
-
-    private JwtRealmsServiceSettings() {}
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/AuthenticationContextSerializer.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/AuthenticationContextSerializer.java
index b18b6091adde..4cd0fd3ee5db 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/AuthenticationContextSerializer.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/AuthenticationContextSerializer.java
@@ -7,7 +7,9 @@
 
 package org.elasticsearch.xpack.core.security.authc.support;
 
-import org.elasticsearch.Version;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Nullable;
@@ -23,6 +25,8 @@
  */
 public class AuthenticationContextSerializer {
 
+    private static final Logger logger = LogManager.getLogger(AuthenticationContextSerializer.class);
+
     private final String contextKey;
 
     public AuthenticationContextSerializer() {
@@ -57,11 +61,16 @@ Authentication deserializeHeaderAndPutInContext(String headerValue, ThreadContex
     }
 
     public static Authentication decode(String header) throws IOException {
-        byte[] bytes = Base64.getDecoder().decode(header);
-        StreamInput input = StreamInput.wrap(bytes);
-        Version version = Version.readVersion(input);
-        input.setVersion(version);
-        return new Authentication(input);
+        try {
+            byte[] bytes = Base64.getDecoder().decode(header);
+            StreamInput input = StreamInput.wrap(bytes);
+            TransportVersion version = TransportVersion.readVersion(input);
+            input.setTransportVersion(version);
+            return new Authentication(input);
+        } catch (IOException | RuntimeException e) {
+            logger.warn("Failed to decode authentication [" + header + "]", e);
+            throw e;
+        }
     }
 
     public Authentication getAuthentication(ThreadContext context) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java
index 6e00fdb7b188..783c588565ec 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java
@@ -192,7 +192,7 @@ public char[] hash(SecureString data) {
 
         @Override
         public boolean verify(SecureString data, char[] hash) {
-            return verifyPbkdf2Hash(data, hash, PBKDF2_PREFIX);
+            return verifyPbkdf2Hash(data, hash, PBKDF2_DEFAULT_COST, PBKDF2_PREFIX);
         }
 
     },
@@ -205,7 +205,7 @@ public char[] hash(SecureString data) {
 
         @Override
         public boolean verify(SecureString data, char[] hash) {
-            return verifyPbkdf2Hash(data, hash, PBKDF2_PREFIX);
+            return verifyPbkdf2Hash(data, hash, 1000, PBKDF2_PREFIX);
         }
 
     },
@@ -218,7 +218,7 @@ public char[] hash(SecureString data) {
 
         @Override
         public boolean verify(SecureString data, char[] hash) {
-            return verifyPbkdf2Hash(data, hash, PBKDF2_PREFIX);
+            return verifyPbkdf2Hash(data, hash, 10000, PBKDF2_PREFIX);
         }
 
     },
@@ -231,7 +231,7 @@ public char[] hash(SecureString data) {
 
         @Override
         public boolean verify(SecureString data, char[] hash) {
-            return verifyPbkdf2Hash(data, hash, PBKDF2_PREFIX);
+            return verifyPbkdf2Hash(data, hash, 50000, PBKDF2_PREFIX);
         }
 
     },
@@ -244,7 +244,7 @@ public char[] hash(SecureString data) {
 
         @Override
         public boolean verify(SecureString data, char[] hash) {
-            return verifyPbkdf2Hash(data, hash, PBKDF2_PREFIX);
+            return verifyPbkdf2Hash(data, hash, 100000, PBKDF2_PREFIX);
         }
 
     },
@@ -257,7 +257,7 @@ public char[] hash(SecureString data) {
 
         @Override
         public boolean verify(SecureString data, char[] hash) {
-            return verifyPbkdf2Hash(data, hash, PBKDF2_PREFIX);
+            return verifyPbkdf2Hash(data, hash, 500000, PBKDF2_PREFIX);
         }
 
     },
@@ -270,7 +270,7 @@ public char[] hash(SecureString data) {
 
         @Override
         public boolean verify(SecureString data, char[] hash) {
-            return verifyPbkdf2Hash(data, hash, PBKDF2_PREFIX);
+            return verifyPbkdf2Hash(data, hash, 1000000, PBKDF2_PREFIX);
         }
 
     },
@@ -283,7 +283,7 @@ public char[] hash(SecureString data) {
 
         @Override
         public boolean verify(SecureString data, char[] hash) {
-            return verifyPbkdf2Hash(new SecureString(hashSha512(data)), hash, PBKDF2_STRETCH_PREFIX);
+            return verifyPbkdf2Hash(new SecureString(hashSha512(data)), hash, PBKDF2_DEFAULT_COST, PBKDF2_STRETCH_PREFIX);
         }
 
     },
@@ -296,7 +296,7 @@ public char[] hash(SecureString data) {
 
         @Override
         public boolean verify(SecureString data, char[] hash) {
-            return verifyPbkdf2Hash(new SecureString(hashSha512(data)), hash, PBKDF2_STRETCH_PREFIX);
+            return verifyPbkdf2Hash(new SecureString(hashSha512(data)), hash, 1000, PBKDF2_STRETCH_PREFIX);
         }
 
     },
@@ -309,7 +309,7 @@ public char[] hash(SecureString data) {
 
         @Override
         public boolean verify(SecureString data, char[] hash) {
-            return verifyPbkdf2Hash(new SecureString(hashSha512(data)), hash, PBKDF2_STRETCH_PREFIX);
+            return verifyPbkdf2Hash(new SecureString(hashSha512(data)), hash, 10000, PBKDF2_STRETCH_PREFIX);
         }
 
     },
@@ -322,7 +322,7 @@ public char[] hash(SecureString data) {
 
         @Override
         public boolean verify(SecureString data, char[] hash) {
-            return verifyPbkdf2Hash(new SecureString(hashSha512(data)), hash, PBKDF2_STRETCH_PREFIX);
+            return verifyPbkdf2Hash(new SecureString(hashSha512(data)), hash, 50000, PBKDF2_STRETCH_PREFIX);
         }
 
     },
@@ -335,7 +335,7 @@ public char[] hash(SecureString data) {
 
         @Override
         public boolean verify(SecureString data, char[] hash) {
-            return verifyPbkdf2Hash(new SecureString(hashSha512(data)), hash, PBKDF2_STRETCH_PREFIX);
+            return verifyPbkdf2Hash(new SecureString(hashSha512(data)), hash, 100000, PBKDF2_STRETCH_PREFIX);
         }
 
     },
@@ -348,7 +348,7 @@ public char[] hash(SecureString data) {
 
         @Override
         public boolean verify(SecureString data, char[] hash) {
-            return verifyPbkdf2Hash(new SecureString(hashSha512(data)), hash, PBKDF2_STRETCH_PREFIX);
+            return verifyPbkdf2Hash(new SecureString(hashSha512(data)), hash, 500000, PBKDF2_STRETCH_PREFIX);
         }
 
     },
@@ -361,7 +361,7 @@ public char[] hash(SecureString data) {
 
         @Override
         public boolean verify(SecureString data, char[] hash) {
-            return verifyPbkdf2Hash(new SecureString(hashSha512(data)), hash, PBKDF2_STRETCH_PREFIX);
+            return verifyPbkdf2Hash(new SecureString(hashSha512(data)), hash, 1000000, PBKDF2_STRETCH_PREFIX);
         }
 
     },
@@ -486,11 +486,13 @@ public boolean verify(SecureString text, char[] hash) {
     private static final int PBKDF2_KEY_LENGTH = 256;
     private static final int BCRYPT_DEFAULT_COST = 10;
     private static final SecureRandom SECURE_RANDOM = new SecureRandom();
+    private static final int HMAC_SHA512_BLOCK_SIZE_IN_BITS = 128;
+    private static final int PBKDF2_MIN_SALT_LENGTH_IN_BYTES = 8;
 
     /**
      * Returns a {@link Hasher} instance of the appropriate algorithm and associated cost as
      * indicated by the {@code name}. Name identifiers for the default costs for
-     * BCRYPT and PBKDF2 return the he default BCRYPT and PBKDF2 Hasher instead of the specific
+     * BCRYPT and PBKDF2 return the default BCRYPT and PBKDF2 Hasher instead of the specific
      * instances for the associated cost.
      *
      * @param name The name of the algorithm and cost combination identifier
@@ -545,10 +547,10 @@ public static Hasher resolveFromHash(char[] hash) {
             int cost = Integer.parseInt(new String(Arrays.copyOfRange(hash, BCRYPT_PREFIX_LENGTH, hash.length - 54)));
             return cost == BCRYPT_DEFAULT_COST ? Hasher.BCRYPT : resolve("bcrypt" + cost);
         } else if (CharArrays.charsBeginsWith(PBKDF2_STRETCH_PREFIX, hash)) {
-            int cost = Integer.parseInt(new String(Arrays.copyOfRange(hash, PBKDF2_STRETCH_PREFIX.length(), hash.length - 90)));
+            int cost = parsePbkdf2Iterations(hash, PBKDF2_STRETCH_PREFIX);
             return cost == PBKDF2_DEFAULT_COST ? Hasher.PBKDF2_STRETCH : resolve("pbkdf2_stretch_" + cost);
         } else if (CharArrays.charsBeginsWith(PBKDF2_PREFIX, hash)) {
-            int cost = Integer.parseInt(new String(Arrays.copyOfRange(hash, PBKDF2_PREFIX.length(), hash.length - 90)));
+            int cost = parsePbkdf2Iterations(hash, PBKDF2_PREFIX);
             return cost == PBKDF2_DEFAULT_COST ? Hasher.PBKDF2 : resolve("pbkdf2_" + cost);
         } else if (CharArrays.charsBeginsWith(SHA1_PREFIX, hash)) {
             return Hasher.SHA1;
@@ -610,27 +612,96 @@ private static char[] getPbkdf2Hash(SecureString data, int cost, String prefix)
         }
     }
 
-    private static boolean verifyPbkdf2Hash(SecureString data, char[] hash, String prefix) {
-        // Base64 string length : (4*(n/3)) rounded up to the next multiple of 4 because of padding.
-        // n is 32 (PBKDF2_KEY_LENGTH in bytes), so tokenLength is 44
-        final int tokenLength = 44;
+    private static int parsePbkdf2Iterations(char[] hash, String prefix) {
+        int separator = -1;
+        for (int i = prefix.length(); i < hash.length; i++) {
+            if (hash[i] == '$') {
+                separator = i;
+                break;
+            }
+        }
+        if (separator == -1) {
+            throw new IllegalArgumentException("The number of iterations could not be determined from the provided PBKDF2 hash.");
+        }
+        try {
+            return Integer.parseInt(new String(hash, prefix.length(), separator - prefix.length()));
+        } catch (NumberFormatException e) {
+            throw new IllegalArgumentException("The number of iterations could not be determined from the provided PBKDF2 hash.", e);
+        }
+    }
+
+    /**
+     * Hashes the given clear text password {@code data} and compares it against the given {@code hash}.
+     *
+     * <p>
+     * <b>Note:</b> When verifying the password hashes we dynamically determine the key and salt lengths for user provided hashes,
+     * but for hashes we create we always use the get method to create hashes with fixed key {@link #PBKDF2_KEY_LENGTH}
+     * and fixed salt lengths.
+     *
+     * @param data the clear text password to hash and verify
+     * @param hash the stored hash against to verify password
+     * @param iterations the number of iterations for PBKDF2 function
+     * @param prefix the PBKDF2 hash prefix
+     * @return {@code true} if password data matches given hash after hashing it, otherwise {@code false}
+     */
+    private static boolean verifyPbkdf2Hash(final SecureString data, final char[] hash, final int iterations, final String prefix) {
+        if (CharArrays.charsBeginsWith(prefix, hash) == false) {
+            return false;
+        }
+
+        int separator1 = -1, separator2 = -1;
+        for (int i = prefix.length() + String.valueOf(iterations).length(); i < hash.length; i++) {
+            if (hash[i] == '$') {
+                separator1 = i;
+                break;
+            }
+        }
+        if (separator1 == -1) {
+            return false;
+        }
+        for (int i = separator1 + 1; i < hash.length; i++) {
+            if (hash[i] == '$') {
+                separator2 = i;
+                break;
+            }
+        }
+        if (separator2 == -1) {
+            return false;
+        }
+
         char[] hashChars = null;
         char[] saltChars = null;
-        char[] computedPwdHash = null;
         try {
-            if (CharArrays.charsBeginsWith(prefix, hash) == false) {
-                return false;
+            hashChars = Arrays.copyOfRange(hash, separator2 + 1, hash.length);
+            saltChars = Arrays.copyOfRange(hash, separator1 + 1, separator2);
+            // Convert from base64 (n * 3/4) to the nearest multiple of 16 bytes (/16 * 16) to bits (*8)
+            // (n * 3/4) / 16 * 16 * 8 ==> n * 3 / 64 * 128
+            final int keySize = (hashChars.length * 3) / 64 * 128;
+            return verifyPbkdf2Hash(data, iterations, keySize, saltChars, hashChars);
+        } finally {
+            if (null != hashChars) {
+                Arrays.fill(hashChars, '\u0000');
             }
-            hashChars = Arrays.copyOfRange(hash, hash.length - tokenLength, hash.length);
-            saltChars = Arrays.copyOfRange(hash, hash.length - (2 * tokenLength + 1), hash.length - (tokenLength + 1));
-            int cost = Integer.parseInt(new String(Arrays.copyOfRange(hash, prefix.length(), hash.length - (2 * tokenLength + 2))));
-            SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2withHMACSHA512");
-            PBEKeySpec keySpec = new PBEKeySpec(
-                data.getChars(),
-                Base64.getDecoder().decode(CharArrays.toUtf8Bytes(saltChars)),
-                cost,
-                PBKDF2_KEY_LENGTH
+            if (null != saltChars) {
+                Arrays.fill(saltChars, '\u0000');
+            }
+        }
+    }
+
+    private static boolean verifyPbkdf2Hash(SecureString data, int iterations, int keyLength, char[] saltChars, char[] hashChars) {
+        if (keyLength <= 0 || keyLength % HMAC_SHA512_BLOCK_SIZE_IN_BITS != 0) {
+            throw new ElasticsearchException(
+                "PBKDF2 key length must be positive and multiple of [" + HMAC_SHA512_BLOCK_SIZE_IN_BITS + " bits]"
             );
+        }
+        final byte[] saltBytes = Base64.getDecoder().decode(CharArrays.toUtf8Bytes(saltChars));
+        if (saltBytes.length < PBKDF2_MIN_SALT_LENGTH_IN_BYTES) {
+            throw new ElasticsearchException("PBKDF2 salt must be at least [" + PBKDF2_MIN_SALT_LENGTH_IN_BYTES + " bytes] long");
+        }
+        char[] computedPwdHash = null;
+        try {
+            SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2withHMACSHA512");
+            PBEKeySpec keySpec = new PBEKeySpec(data.getChars(), saltBytes, iterations, keyLength);
             computedPwdHash = CharArrays.utf8BytesToChars(
                 Base64.getEncoder().encode(secretKeyFactory.generateSecret(keySpec).getEncoded())
             );
@@ -643,12 +714,6 @@ private static boolean verifyPbkdf2Hash(SecureString data, char[] hash, String p
             // salt, iv, or password length is not met. We catch this because we don't want the JVM to exit.
             throw new ElasticsearchException("Error using PBKDF2 implementation from the selected Security Provider", e);
         } finally {
-            if (null != hashChars) {
-                Arrays.fill(hashChars, '\u0000');
-            }
-            if (null != saltChars) {
-                Arrays.fill(saltChars, '\u0000');
-            }
             if (null != computedPwdHash) {
                 Arrays.fill(computedPwdHash, '\u0000');
             }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/TokensInvalidationResult.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/TokensInvalidationResult.java
index 96e805f86ad4..9f1229cbf926 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/TokensInvalidationResult.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/TokensInvalidationResult.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.xpack.core.security.authc.support;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -59,10 +59,10 @@ public TokensInvalidationResult(StreamInput in) throws IOException {
         this.invalidatedTokens = in.readStringList();
         this.previouslyInvalidatedTokens = in.readStringList();
         this.errors = in.readList(StreamInput::readException);
-        if (in.getVersion().before(Version.V_7_2_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_7_2_0)) {
             in.readVInt();
         }
-        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             this.restStatus = RestStatus.readFrom(in);
         }
     }
@@ -111,10 +111,10 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeStringCollection(invalidatedTokens);
         out.writeStringCollection(previouslyInvalidatedTokens);
         out.writeCollection(errors, StreamOutput::writeException);
-        if (out.getVersion().before(Version.V_7_2_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_7_2_0)) {
             out.writeVInt(5);
         }
-        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             RestStatus.writeTo(out, restStatus);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/UserRoleMapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/UserRoleMapper.java
index 638495ac2146..1325eeef5dfb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/UserRoleMapper.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/UserRoleMapper.java
@@ -20,7 +20,9 @@
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
 
+import java.lang.ref.SoftReference;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
@@ -74,15 +76,16 @@ public UserData(String username, @Nullable String dn, Collection<String> groups,
          */
         public ExpressionModel asModel() {
             final ExpressionModel model = new ExpressionModel();
+            final DistinguishedNameNormalizer dnNormalizer = getDnNormalizer();
             model.defineField("username", username);
             if (dn != null) {
                 // null dn fields get the default NULL_PREDICATE
-                model.defineField("dn", dn, new DistinguishedNamePredicate(dn));
+                model.defineField("dn", dn, new DistinguishedNamePredicate(dn, dnNormalizer));
             }
             model.defineField(
                 "groups",
                 groups,
-                groups.stream().<Predicate<FieldExpression.FieldValue>>map(DistinguishedNamePredicate::new)
+                groups.stream().<Predicate<FieldExpression.FieldValue>>map(g -> new DistinguishedNamePredicate(g, dnNormalizer))
                     .reduce(Predicate::or)
                     .orElse(fieldValue -> false)
             );
@@ -145,6 +148,67 @@ public Map<String, Object> getMetadata() {
         public RealmConfig getRealm() {
             return realm;
         }
+
+        // Package private for testing
+        DistinguishedNameNormalizer getDnNormalizer() {
+            return new DistinguishedNameNormalizer();
+        }
+    }
+
+    /**
+     * This class parse the given string into a DN and return its normalized format.
+     * If the input string is not a valid DN, {@code null} is returned.
+     * The DN parsing and normalization are cached internally so that the same
+     * input string will only be processed once (as long as the cache entry is not GC'd).
+     * The cache works regardless of whether the input string is a valid DN.
+     *
+     * The cache uses {@link SoftReference} for its values so that they free for GC.
+     * This is to prevent potential memory pressure when there are many concurrent role
+     * mapping processes coupled with large number of groups and role mappings, which
+     * in theory is unbounded.
+     */
+    class DistinguishedNameNormalizer {
+        private static final Logger LOGGER = LogManager.getLogger(DistinguishedNameNormalizer.class);
+        private static final SoftReference<String> NULL_REF = new SoftReference<>(null);
+        private final Map<String, SoftReference<String>> cache = new HashMap<>();
+
+        /**
+         * Parse the input string to a DN and returns its normalized form.
+         * @param str String that may represent a DN
+         * @return The normalized DN form of the input string or {@code null} if input string is not a DN
+         */
+        public String normalize(String str) {
+            final SoftReference<String> normalizedDnRef = cache.get(str);
+            if (normalizedDnRef == NULL_REF) {
+                return null;
+            }
+            if (normalizedDnRef != null) {
+                final String normalizedDn = normalizedDnRef.get();
+                if (normalizedDn != null) {
+                    return normalizedDn;
+                }
+            }
+            final String normalizedDn = doNormalize(str);
+            if (normalizedDn == null) {
+                cache.put(str, NULL_REF);
+            } else {
+                cache.put(str, new SoftReference<>(normalizedDn));
+            }
+            return normalizedDn;
+        }
+
+        String doNormalize(String str) {
+            final DN dn;
+            try {
+                dn = new DN(str);
+            } catch (LDAPException | LDAPSDKUsageException e) {
+                if (LOGGER.isTraceEnabled()) {
+                    LOGGER.trace(() -> "failed to parse [" + str + "] as a DN", e);
+                }
+                return null;
+            }
+            return dn.toNormalizedString();
+        }
     }
 
     /**
@@ -165,26 +229,16 @@ public RealmConfig getRealm() {
      *
      */
     class DistinguishedNamePredicate implements Predicate<FieldExpression.FieldValue> {
-        private static final Logger LOGGER = LogManager.getLogger(DistinguishedNamePredicate.class);
 
         private final String string;
-        private final DN dn;
+        private final DistinguishedNameNormalizer dnNormalizer;
+        private final String normalizedDn;
 
-        public DistinguishedNamePredicate(String string) {
+        public DistinguishedNamePredicate(String string, DistinguishedNameNormalizer dnNormalizer) {
             assert string != null : "DN string should not be null. Use the dedicated NULL_PREDICATE for every user null field.";
             this.string = string;
-            this.dn = parseDn(string);
-        }
-
-        private static DN parseDn(String string) {
-            try {
-                return new DN(string);
-            } catch (LDAPException | LDAPSDKUsageException e) {
-                if (LOGGER.isTraceEnabled()) {
-                    LOGGER.trace(() -> "failed to parse [" + string + "] as a DN", e);
-                }
-                return null;
-            }
+            this.dnNormalizer = dnNormalizer;
+            this.normalizedDn = dnNormalizer.normalize(string);
         }
 
         @Override
@@ -199,13 +253,13 @@ public boolean test(FieldExpression.FieldValue fieldValue) {
                 if (automaton.run(string)) {
                     return true;
                 }
-                if (dn != null && automaton.run(dn.toNormalizedString())) {
+                if (normalizedDn != null && automaton.run(normalizedDn)) {
                     return true;
                 }
                 if (automaton.run(string.toLowerCase(Locale.ROOT)) || automaton.run(string.toUpperCase(Locale.ROOT))) {
                     return true;
                 }
-                if (dn == null) {
+                if (normalizedDn == null) {
                     return false;
                 }
 
@@ -217,36 +271,38 @@ public boolean test(FieldExpression.FieldValue fieldValue) {
                 String pattern = (String) fieldValue.getValue();
 
                 // If the pattern is "*,dc=example,dc=com" then the rule is actually trying to express a DN sub-tree match.
-                // We can use dn.isDescendantOf for that
                 if (pattern.startsWith("*,")) {
                     final String suffix = pattern.substring(2);
                     // if the suffix has a wildcard, then it's not a pure sub-tree match
                     if (suffix.indexOf('*') == -1) {
-                        final DN dnSuffix = parseDn(suffix);
-                        if (dnSuffix != null && dn.isDescendantOf(dnSuffix, false)) {
-                            return true;
-                        }
+                        return isDescendantOf(dnNormalizer.normalize(suffix));
                     }
                 }
 
                 return false;
             }
-            if (fieldValue.getValue() instanceof String) {
-                final String testString = (String) fieldValue.getValue();
+            if (fieldValue.getValue()instanceof final String testString) {
                 if (testString.equalsIgnoreCase(string)) {
                     return true;
                 }
-                if (dn == null) {
+                if (normalizedDn == null) {
                     return false;
                 }
 
-                final DN testDn = parseDn(testString);
-                if (testDn != null) {
-                    return dn.equals(testDn);
+                final String testNormalizedDn = dnNormalizer.normalize(testString);
+                if (testNormalizedDn != null) {
+                    return normalizedDn.equals(testNormalizedDn);
                 }
-                return testString.equalsIgnoreCase(dn.toNormalizedString());
+                return testString.equalsIgnoreCase(normalizedDn);
             }
             return false;
         }
+
+        private boolean isDescendantOf(String normalizedDnSuffix) {
+            if (normalizedDnSuffix == null) {
+                return false;
+            }
+            return normalizedDn.endsWith("," + normalizedDnSuffix) || (normalizedDnSuffix.isEmpty() && false == normalizedDn.isEmpty());
+        }
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java
index 679d3ef49632..cbe1e71d8fc0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.security.authc.support.mapper;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -97,7 +97,7 @@ public ExpressionRoleMapping(StreamInput in) throws IOException {
         this.name = in.readString();
         this.enabled = in.readBoolean();
         this.roles = in.readStringList();
-        if (in.getVersion().onOrAfter(Version.V_7_2_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_2_0)) {
             this.roleTemplates = in.readList(TemplateRoleName::new);
         } else {
             this.roleTemplates = Collections.emptyList();
@@ -111,7 +111,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(name);
         out.writeBoolean(enabled);
         out.writeStringCollection(roles);
-        if (out.getVersion().onOrAfter(Version.V_7_2_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_2_0)) {
             out.writeList(roleTemplates);
         }
         ExpressionParser.writeExpression(expression, out);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java
index 4c5bd39cc774..f6964fdaf252 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java
@@ -12,8 +12,11 @@
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesResponse;
@@ -26,12 +29,14 @@
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Base64;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
@@ -179,7 +184,7 @@ void loadAuthorizedIndices(
         RequestInfo requestInfo,
         AuthorizationInfo authorizationInfo,
         Map<String, IndexAbstraction> indicesLookup,
-        ActionListener<Set<String>> listener
+        ActionListener<AuthorizationEngine.AuthorizedIndices> listener
     );
 
     /**
@@ -235,6 +240,18 @@ void checkPrivileges(
      */
     void getUserPrivileges(AuthorizationInfo authorizationInfo, ActionListener<GetUserPrivilegesResponse> listener);
 
+    /**
+     * Retrieve remote access privileges for a given target cluster, from the provided authorization information, to be sent together
+     * with a cross-cluster request (e.g. CCS) from an originating cluster to the target cluster.
+     */
+    default void getRemoteAccessRoleDescriptorsIntersection(
+        final String remoteClusterAlias,
+        final AuthorizationInfo authorizationInfo,
+        final ActionListener<RoleDescriptorsIntersection> listener
+    ) {
+        throw new UnsupportedOperationException("retrieving remote access role descriptors is not supported by this authorization engine");
+    }
+
     /**
      * Interface for objects that contains the information needed to authorize a request
      */
@@ -257,6 +274,25 @@ default AuthorizationInfo getAuthenticatedUserAuthorizationInfo() {
         }
     }
 
+    /**
+     * Used to retrieve index-like resources that the user has access to, for a specific access action type,
+     * at a specific point in time (for a fixed cluster state view).
+     * It can also be used to check if a specific resource name is authorized (access to the resource name
+     * can be authorized even if it doesn't exist).
+     */
+    interface AuthorizedIndices {
+        /**
+         * Returns all the index-like resource names that are available and accessible for an action type by a user,
+         * at a fixed point in time (for a single cluster state view).
+         */
+        Supplier<Set<String>> all();
+
+        /**
+         * Checks if an index-like resource name is authorized, for an action by a user. The resource might or might not exist.
+         */
+        boolean check(String name);
+    }
+
     /**
      * This encapsulates the privileges that can be checked for access. It's intentional that the privileges to be checked are specified
      * in the same manner that they are granted in the {@link RoleDescriptor}. The privilege check can be detailed or not, per the
@@ -448,17 +484,30 @@ final class RequestInfo {
         private final String action;
         @Nullable
         private final AuthorizationContext originatingAuthorizationContext;
+        @Nullable
+        private final ParentActionAuthorization parentAuthorization;
 
         public RequestInfo(
             Authentication authentication,
             TransportRequest request,
             String action,
-            AuthorizationContext originatingContext
+            AuthorizationContext originatingContext,
+            ParentActionAuthorization parentAuthorization
         ) {
             this.authentication = Objects.requireNonNull(authentication);
             this.request = Objects.requireNonNull(request);
             this.action = Objects.requireNonNull(action);
             this.originatingAuthorizationContext = originatingContext;
+            this.parentAuthorization = parentAuthorization;
+        }
+
+        public RequestInfo(
+            Authentication authentication,
+            TransportRequest request,
+            String action,
+            AuthorizationContext originatingContext
+        ) {
+            this(authentication, request, action, originatingContext, null);
         }
 
         public String getAction() {
@@ -478,6 +527,11 @@ public AuthorizationContext getOriginatingAuthorizationContext() {
             return originatingAuthorizationContext;
         }
 
+        @Nullable
+        public ParentActionAuthorization getParentAuthorization() {
+            return parentAuthorization;
+        }
+
         @Override
         public String toString() {
             return getClass().getSimpleName()
@@ -489,8 +543,10 @@ public String toString() {
                 + "], action=["
                 + action
                 + ']'
-                + ", parent=["
+                + ", originating=["
                 + originatingAuthorizationContext
+                + "], parent=["
+                + parentAuthorization
                 + "]}";
         }
 
@@ -635,6 +691,69 @@ public IndicesAccessControl getIndicesAccessControl() {
         }
     }
 
+    /**
+     * Holds information about authorization of a parent action which is used to pre-authorize its child actions.
+     *
+     *  @param action the parent action
+     */
+    record ParentActionAuthorization(String action) implements Writeable {
+
+        public static final String THREAD_CONTEXT_KEY = "_xpack_security_parent_action_authz";
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(action);
+        }
+
+        /**
+         * Reads an {@link ParentActionAuthorization} from a {@link StreamInput}
+         *
+         * @param in the {@link StreamInput} to read from
+         * @return {@link ParentActionAuthorization}
+         * @throws IOException if I/O operation fails
+         */
+        public static ParentActionAuthorization readFrom(StreamInput in) throws IOException {
+            String action = in.readString();
+            return new ParentActionAuthorization(action);
+        }
+
+        /**
+         * Read and deserialize parent authorization from thread context.
+         *
+         * @param context the thread context to read from
+         * @return {@link ParentActionAuthorization} or null
+         * @throws IOException if reading fails due to I/O exception
+         */
+        @Nullable
+        public static ParentActionAuthorization readFromThreadContext(ThreadContext context) throws IOException {
+            final String header = context.getHeader(THREAD_CONTEXT_KEY);
+            if (header == null) {
+                return null;
+            }
+
+            byte[] bytes = Base64.getDecoder().decode(header);
+            StreamInput input = StreamInput.wrap(bytes);
+            return readFrom(input);
+        }
+
+        /**
+         * Writes the authorization to the context. There must not be an existing authorization in the context and if there is an
+         * {@link IllegalStateException} will be thrown.
+         */
+        public void writeToThreadContext(ThreadContext context) throws IOException {
+            String header = this.encode();
+            assert header != null : "parent authorization object encoded to null";
+            context.putHeader(THREAD_CONTEXT_KEY, header);
+        }
+
+        private String encode() throws IOException {
+            BytesStreamOutput output = new BytesStreamOutput();
+            writeTo(output);
+            return Base64.getEncoder().encodeToString(BytesReference.toBytes(output.bytes()));
+        }
+
+    }
+
     @FunctionalInterface
     interface AsyncSupplier<V> {
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java
index 847b5f36747b..ed213ce714da 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java
@@ -8,7 +8,7 @@
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ElasticsearchSecurityException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -53,7 +53,7 @@
 public class RoleDescriptor implements ToXContentObject, Writeable {
 
     public static final String ROLE_TYPE = "role";
-    public static final Version VERSION_REMOTE_INDICES = Version.V_8_6_0;
+    public static final TransportVersion VERSION_REMOTE_INDICES = TransportVersion.V_8_6_0;
 
     private final String name;
     private final String[] clusterPrivileges;
@@ -166,7 +166,7 @@ public RoleDescriptor(StreamInput in) throws IOException {
 
         this.applicationPrivileges = in.readArray(ApplicationResourcePrivileges::new, ApplicationResourcePrivileges[]::new);
         this.configurableClusterPrivileges = ConfigurableClusterPrivileges.readArray(in);
-        if (in.getVersion().onOrAfter(VERSION_REMOTE_INDICES)) {
+        if (in.getTransportVersion().onOrAfter(VERSION_REMOTE_INDICES)) {
             this.remoteIndicesPrivileges = in.readArray(RemoteIndicesPrivileges::new, RemoteIndicesPrivileges[]::new);
         } else {
             this.remoteIndicesPrivileges = RemoteIndicesPrivileges.NONE;
@@ -337,14 +337,14 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeGenericMap(transientMetadata);
         out.writeArray(ApplicationResourcePrivileges::write, applicationPrivileges);
         ConfigurableClusterPrivileges.writeArray(out, getConditionalClusterPrivileges());
-        if (out.getVersion().onOrAfter(VERSION_REMOTE_INDICES)) {
+        if (out.getTransportVersion().onOrAfter(VERSION_REMOTE_INDICES)) {
             out.writeArray(remoteIndicesPrivileges);
         } else if (hasRemoteIndicesPrivileges()) {
             throw new IllegalArgumentException(
                 "versions of Elasticsearch before ["
                     + VERSION_REMOTE_INDICES
                     + "] can't handle remote indices privileges and attempted to send to ["
-                    + out.getVersion()
+                    + out.getTransportVersion()
                     + "]"
             );
         }
@@ -1034,7 +1034,7 @@ public RemoteIndicesPrivileges build() {
      * A class representing permissions for a group of indices mapped to
      * privileges, field permissions, and a query.
      */
-    public static class IndicesPrivileges implements ToXContentObject, Writeable {
+    public static class IndicesPrivileges implements ToXContentObject, Writeable, Comparable<IndicesPrivileges> {
 
         private static final IndicesPrivileges[] NONE = new IndicesPrivileges[0];
 
@@ -1214,6 +1214,35 @@ public static void write(StreamOutput out, IndicesPrivileges privileges) throws
             privileges.writeTo(out);
         }
 
+        @Override
+        public int compareTo(IndicesPrivileges o) {
+            if (this == o) {
+                return 0;
+            }
+            int cmp = Boolean.compare(allowRestrictedIndices, o.allowRestrictedIndices);
+            if (cmp != 0) {
+                return cmp;
+            }
+            cmp = Arrays.compare(indices, o.indices);
+            if (cmp != 0) {
+                return cmp;
+            }
+            cmp = Arrays.compare(privileges, o.privileges);
+            if (cmp != 0) {
+                return cmp;
+            }
+            cmp = Objects.compare(query, o.query, Comparator.nullsFirst(BytesReference::compareTo));
+            if (cmp != 0) {
+                return cmp;
+            }
+            cmp = Arrays.compare(grantedFields, o.grantedFields);
+            if (cmp != 0) {
+                return cmp;
+            }
+            cmp = Arrays.compare(deniedFields, o.deniedFields);
+            return cmp;
+        }
+
         public static class Builder {
 
             private IndicesPrivileges indicesPrivileges = new IndicesPrivileges();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorsIntersection.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorsIntersection.java
index 9ba992d69f44..30139ae1b1db 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorsIntersection.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorsIntersection.java
@@ -18,11 +18,18 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 
 public record RoleDescriptorsIntersection(Collection<Set<RoleDescriptor>> roleDescriptorsList) implements ToXContentObject, Writeable {
 
+    public static RoleDescriptorsIntersection EMPTY = new RoleDescriptorsIntersection(Collections.emptyList());
+
+    public RoleDescriptorsIntersection(RoleDescriptor roleDescriptor) {
+        this(List.of(Set.of(roleDescriptor)));
+    }
+
     public RoleDescriptorsIntersection(StreamInput in) throws IOException {
         this(in.readImmutableList(inner -> inner.readSet(RoleDescriptor::new)));
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCache.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCache.java
index 20a5ffdebb24..417a270cda8b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCache.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCache.java
@@ -14,7 +14,6 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.ReaderUtil;
 import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
@@ -35,6 +34,8 @@
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.lucene.util.BitSets;
+import org.elasticsearch.lucene.util.MatchAllBitSet;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.Closeable;
@@ -277,14 +278,14 @@ private BitSet computeBitSet(Query query, LeafReaderContext context) throws IOEx
         searcher.setQueryCache(null);
         final Query rewrittenQuery = searcher.rewrite(query);
         if (isEffectiveMatchAllDocsQuery(rewrittenQuery)) {
-            return new MatchAllRoleBitSet(context.reader().maxDoc());
+            return new MatchAllBitSet(context.reader().maxDoc());
         }
         final Weight weight = searcher.createWeight(rewrittenQuery, ScoreMode.COMPLETE_NO_SCORES, 1f);
         final Scorer s = weight.scorer(context);
         if (s == null) {
             return null;
         } else {
-            return bitSetFromDocIterator(s.iterator(), context.reader().maxDoc());
+            return BitSets.of(s.iterator(), context.reader().maxDoc());
         }
     }
 
@@ -380,13 +381,4 @@ void verifyInternalConsistency() {
         });
     }
 
-    static BitSet bitSetFromDocIterator(DocIdSetIterator iter, int maxDoc) throws IOException {
-        final BitSet set = BitSet.of(iter, maxDoc);
-        if (set.cardinality() == maxDoc) {
-            return new MatchAllRoleBitSet(maxDoc);
-        } else {
-            return set;
-        }
-    }
-
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReader.java
index c64cb3d15541..50089309cadd 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReader.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReader.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader;
 import org.elasticsearch.lucene.util.CombinedBitSet;
+import org.elasticsearch.lucene.util.MatchAllBitSet;
 import org.elasticsearch.transport.Transports;
 
 import java.io.IOException;
@@ -59,7 +60,7 @@ private static int computeNumDocs(LeafReader reader, BitSet roleQueryBits) {
         final Bits liveDocs = reader.getLiveDocs();
         if (roleQueryBits == null) {
             return 0;
-        } else if (roleQueryBits instanceof MatchAllRoleBitSet) {
+        } else if (roleQueryBits instanceof MatchAllBitSet) {
             return reader.numDocs();
         } else if (liveDocs == null) {
             // slow
@@ -197,7 +198,7 @@ public Bits getLiveDocs() {
             // If we would return a <code>null</code> liveDocs then that would mean that no docs are marked as deleted,
             // but that isn't the case. No docs match with the role query and therefore all docs are marked as deleted
             return new Bits.MatchNoBits(in.maxDoc());
-        } else if (roleQueryBits instanceof MatchAllRoleBitSet) {
+        } else if (roleQueryBits instanceof MatchAllBitSet) {
             return actualLiveDocs;
         } else if (actualLiveDocs == null) {
             return roleQueryBits;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java
index 38e4fd274060..84e346611910 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java
@@ -8,12 +8,14 @@
 
 import org.apache.lucene.codecs.StoredFieldsReader;
 import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.FilterDirectoryReader;
 import org.apache.lucene.index.FilterLeafReader;
+import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.PointValues;
@@ -21,10 +23,11 @@
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.index.TermState;
+import org.apache.lucene.index.TermVectors;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.VectorValues;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
@@ -162,6 +165,34 @@ public Fields getTermVectors(int docID) throws IOException {
         return f.iterator().hasNext() ? f : null;
     }
 
+    @Override
+    public TermVectors termVectors() throws IOException {
+        TermVectors termVectors = super.termVectors();
+        return new TermVectors() {
+            @Override
+            public Fields get(int doc) throws IOException {
+                Fields f = termVectors.get(doc);
+                if (f == null) {
+                    return null;
+                }
+                f = new FieldFilterFields(f);
+                // we need to check for emptyness, so we can return null:
+                return f.iterator().hasNext() ? f : null;
+            }
+        };
+    }
+
+    @Override
+    public StoredFields storedFields() throws IOException {
+        StoredFields storedFields = super.storedFields();
+        return new StoredFields() {
+            @Override
+            public void document(int docID, StoredFieldVisitor visitor) throws IOException {
+                storedFields.document(docID, new FieldSubsetStoredFieldVisitor(visitor));
+            }
+        };
+    }
+
     /** Filter a map by a {@link CharacterRunAutomaton} that defines the fields to retain. */
     @SuppressWarnings("unchecked")
     static Map<String, Object> filter(Map<String, ?> map, CharacterRunAutomaton includeAutomaton, int initialState) {
@@ -278,8 +309,8 @@ public NumericDocValues getNormValues(String field) throws IOException {
     }
 
     @Override
-    public VectorValues getVectorValues(String field) throws IOException {
-        return hasField(field) ? super.getVectorValues(field) : null;
+    public FloatVectorValues getFloatVectorValues(String field) throws IOException {
+        return hasField(field) ? super.getFloatVectorValues(field) : null;
     }
 
     @Override
@@ -287,6 +318,16 @@ public TopDocs searchNearestVectors(String field, float[] target, int k, Bits ac
         return hasField(field) ? super.searchNearestVectors(field, target, k, acceptDocs, visitedLimit) : null;
     }
 
+    @Override
+    public ByteVectorValues getByteVectorValues(String field) throws IOException {
+        return hasField(field) ? super.getByteVectorValues(field) : null;
+    }
+
+    @Override
+    public TopDocs searchNearestVectors(String field, byte[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
+        return hasField(field) ? super.searchNearestVectors(field, target, k, acceptDocs, visitedLimit) : null;
+    }
+
     // we share core cache keys (for e.g. fielddata)
 
     @Override
@@ -310,8 +351,8 @@ class FieldSubsetStoredFieldsReader extends StoredFieldsReader {
         }
 
         @Override
-        public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
-            reader.visitDocument(docID, new FieldSubsetStoredFieldVisitor(visitor));
+        public void document(int docID, StoredFieldVisitor visitor) throws IOException {
+            reader.document(docID, new FieldSubsetStoredFieldVisitor(visitor));
         }
 
         @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java
index 4c3f1fac6d63..2751a6f60d3e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.core.security.authz.accesscontrol;
 
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.util.CachedSupplier;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.Nullable;
@@ -23,6 +24,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.function.Predicate;
+import java.util.function.Supplier;
 
 /**
  * Encapsulates the field and document permissions per concrete index based on the current request.
@@ -36,11 +38,19 @@ public class IndicesAccessControl {
     public static final IndicesAccessControl DENIED = new IndicesAccessControl(false, Collections.emptyMap());
 
     private final boolean granted;
-    private final Map<String, IndexAccessControl> indexPermissions;
+    private final CachedSupplier<Map<String, IndexAccessControl>> indexPermissionsSupplier;
 
     public IndicesAccessControl(boolean granted, Map<String, IndexAccessControl> indexPermissions) {
+        this(granted, () -> Objects.requireNonNull(indexPermissions));
+    }
+
+    public IndicesAccessControl(boolean granted, Supplier<Map<String, IndexAccessControl>> indexPermissionsSupplier) {
         this.granted = granted;
-        this.indexPermissions = Objects.requireNonNull(indexPermissions);
+        this.indexPermissionsSupplier = new CachedSupplier<>(Objects.requireNonNull(indexPermissionsSupplier));
+    }
+
+    protected IndicesAccessControl(IndicesAccessControl copy) {
+        this(copy.granted, copy.indexPermissionsSupplier);
     }
 
     /**
@@ -49,7 +59,7 @@ public IndicesAccessControl(boolean granted, Map<String, IndexAccessControl> ind
      */
     @Nullable
     public IndexAccessControl getIndexPermissions(String index) {
-        return indexPermissions.get(index);
+        return this.getAllIndexPermissions().get(index);
     }
 
     public boolean hasIndexPermissions(String index) {
@@ -66,7 +76,7 @@ public boolean isGranted() {
     public DlsFlsUsage getFieldAndDocumentLevelSecurityUsage() {
         boolean hasFls = false;
         boolean hasDls = false;
-        for (IndexAccessControl iac : indexPermissions.values()) {
+        for (IndexAccessControl iac : this.getAllIndexPermissions().values()) {
             if (iac.fieldPermissions.hasFieldLevelSecurity()) {
                 hasFls = true;
             }
@@ -99,7 +109,16 @@ public List<String> getIndicesWithDocumentLevelSecurity() {
     }
 
     private List<String> getIndexNames(Predicate<IndexAccessControl> predicate) {
-        return indexPermissions.entrySet().stream().filter(entry -> predicate.test(entry.getValue())).map(Map.Entry::getKey).toList();
+        return this.getAllIndexPermissions()
+            .entrySet()
+            .stream()
+            .filter(entry -> predicate.test(entry.getValue()))
+            .map(Map.Entry::getKey)
+            .toList();
+    }
+
+    private Map<String, IndexAccessControl> getAllIndexPermissions() {
+        return this.indexPermissionsSupplier.get();
     }
 
     public enum DlsFlsUsage {
@@ -246,22 +265,26 @@ public IndicesAccessControl limitIndicesAccessControl(IndicesAccessControl limit
         } else {
             isGranted = false;
         }
-        Set<String> indexes = indexPermissions.keySet();
-        Set<String> otherIndexes = limitedByIndicesAccessControl.indexPermissions.keySet();
-        Set<String> commonIndexes = Sets.intersection(indexes, otherIndexes);
-
-        Map<String, IndexAccessControl> indexPermissionsMap = Maps.newMapWithExpectedSize(commonIndexes.size());
-        for (String index : commonIndexes) {
-            IndexAccessControl indexAccessControl = getIndexPermissions(index);
-            IndexAccessControl limitedByIndexAccessControl = limitedByIndicesAccessControl.getIndexPermissions(index);
-            indexPermissionsMap.put(index, indexAccessControl.limitIndexAccessControl(limitedByIndexAccessControl));
-        }
-        return new IndicesAccessControl(isGranted, indexPermissionsMap);
+
+        final Supplier<Map<String, IndexAccessControl>> limitedIndexPermissions = () -> {
+            Set<String> indexes = this.getAllIndexPermissions().keySet();
+            Set<String> otherIndexes = limitedByIndicesAccessControl.getAllIndexPermissions().keySet();
+            Set<String> commonIndexes = Sets.intersection(indexes, otherIndexes);
+
+            Map<String, IndexAccessControl> indexPermissionsMap = Maps.newMapWithExpectedSize(commonIndexes.size());
+            for (String index : commonIndexes) {
+                IndexAccessControl indexAccessControl = getIndexPermissions(index);
+                IndexAccessControl limitedByIndexAccessControl = limitedByIndicesAccessControl.getIndexPermissions(index);
+                indexPermissionsMap.put(index, indexAccessControl.limitIndexAccessControl(limitedByIndexAccessControl));
+            }
+            return indexPermissionsMap;
+        };
+        return new IndicesAccessControl(isGranted, limitedIndexPermissions);
     }
 
     @Override
     public String toString() {
-        return "IndicesAccessControl{" + "granted=" + granted + ", indexPermissions=" + indexPermissions + '}';
+        return "IndicesAccessControl{" + "granted=" + granted + ", indexPermissions=" + indexPermissionsSupplier.get() + '}';
     }
 
     public static IndicesAccessControl allowAll() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/MatchAllRoleBitSet.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/MatchAllRoleBitSet.java
deleted file mode 100644
index 4c0f21715fed..000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/MatchAllRoleBitSet.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.core.security.authz.accesscontrol;
-
-import org.apache.lucene.util.BitSet;
-import org.apache.lucene.util.RamUsageEstimator;
-
-final class MatchAllRoleBitSet extends BitSet {
-    private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(MatchAllRoleBitSet.class);
-    private final int length;
-
-    MatchAllRoleBitSet(int length) {
-        this.length = length;
-    }
-
-    @Override
-    public void set(int i) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public boolean getAndSet(int i) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void clear(int i) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void clear(int startIndex, int endIndex) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public int cardinality() {
-        return length();
-    }
-
-    @Override
-    public int approximateCardinality() {
-        return length();
-    }
-
-    @Override
-    public int prevSetBit(int index) {
-        return index;
-    }
-
-    @Override
-    public int nextSetBit(int index) {
-        return index;
-    }
-
-    @Override
-    public long ramBytesUsed() {
-        return BASE_RAM_BYTES_USED;
-    }
-
-    @Override
-    public boolean get(int index) {
-        assert 0 <= index && index < this.length();
-        return true;
-    }
-
-    @Override
-    public int length() {
-        return length;
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java
index 70a418e1e1c2..70820a45c165 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java
@@ -37,6 +37,8 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiPredicate;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 
@@ -54,7 +56,7 @@ public final class IndicesPermission {
 
     private static final Set<String> PRIVILEGE_NAME_SET_BWC_ALLOW_MAPPING_UPDATE = Set.of("create", "create_doc", "index", "write");
 
-    private final Map<String, Predicate<IndexAbstraction>> allowedIndicesMatchersForAction = new ConcurrentHashMap<>();
+    private final Map<String, IsResourceAuthorizedPredicate> allowedIndicesMatchersForAction = new ConcurrentHashMap<>();
 
     private final RestrictedIndices restrictedIndices;
     private final Group[] groups;
@@ -126,7 +128,7 @@ public Group[] groups() {
      * @return A predicate that will match all the indices that this permission
      * has the privilege for executing the given action on.
      */
-    public Predicate<IndexAbstraction> allowedIndicesMatcher(String action) {
+    public IsResourceAuthorizedPredicate allowedIndicesMatcher(String action) {
         return allowedIndicesMatchersForAction.computeIfAbsent(action, this::buildIndexMatcherPredicateForAction);
     }
 
@@ -134,7 +136,7 @@ public boolean hasFieldOrDocumentLevelSecurity() {
         return hasFieldOrDocumentLevelSecurity;
     }
 
-    private Predicate<IndexAbstraction> buildIndexMatcherPredicateForAction(String action) {
+    private IsResourceAuthorizedPredicate buildIndexMatcherPredicateForAction(String action) {
         final Set<String> ordinaryIndices = new HashSet<>();
         final Set<String> restrictedIndices = new HashSet<>();
         final Set<String> grantMappingUpdatesOnIndices = new HashSet<>();
@@ -159,10 +161,64 @@ private Predicate<IndexAbstraction> buildIndexMatcherPredicateForAction(String a
         }
         final StringMatcher nameMatcher = indexMatcher(ordinaryIndices, restrictedIndices);
         final StringMatcher bwcSpecialCaseMatcher = indexMatcher(grantMappingUpdatesOnIndices, grantMappingUpdatesOnRestrictedIndices);
-        return indexAbstraction -> nameMatcher.test(indexAbstraction.getName())
-            || (indexAbstraction.getType() != IndexAbstraction.Type.DATA_STREAM
-                && (indexAbstraction.getParentDataStream() == null)
-                && bwcSpecialCaseMatcher.test(indexAbstraction.getName()));
+        return new IsResourceAuthorizedPredicate(nameMatcher, bwcSpecialCaseMatcher);
+    }
+
+    /**
+     * This encapsulates the authorization test for resources.
+     * There is an additional test for resources that are missing or that are not a datastream or a backing index.
+     */
+    public static class IsResourceAuthorizedPredicate implements BiPredicate<String, IndexAbstraction> {
+
+        private final BiPredicate<String, IndexAbstraction> biPredicate;
+
+        // public for tests
+        public IsResourceAuthorizedPredicate(StringMatcher resourceNameMatcher, StringMatcher additionalNonDatastreamNameMatcher) {
+            this((String name, @Nullable IndexAbstraction indexAbstraction) -> {
+                assert indexAbstraction == null || name.equals(indexAbstraction.getName());
+                return resourceNameMatcher.test(name)
+                    || (isPartOfDatastream(indexAbstraction) == false && additionalNonDatastreamNameMatcher.test(name));
+            });
+        }
+
+        private IsResourceAuthorizedPredicate(BiPredicate<String, IndexAbstraction> biPredicate) {
+            this.biPredicate = biPredicate;
+        }
+
+        /**
+        * Given another {@link IsResourceAuthorizedPredicate} instance in {@param other},
+        * return a new {@link IsResourceAuthorizedPredicate} instance that is equivalent to the conjunction of
+        * authorization tests of that other instance and this one.
+        */
+        @Override
+        public final IsResourceAuthorizedPredicate and(BiPredicate<? super String, ? super IndexAbstraction> other) {
+            return new IsResourceAuthorizedPredicate(this.biPredicate.and(other));
+        }
+
+        /**
+         * Verifies if access is authorized to the given {@param indexAbstraction} resource.
+         * The resource must exist. Otherwise, use the {@link #test(String, IndexAbstraction)} method.
+         * Returns {@code true} if access to the given resource is authorized or {@code false} otherwise.
+         */
+        public final boolean test(IndexAbstraction indexAbstraction) {
+            return test(indexAbstraction.getName(), indexAbstraction);
+        }
+
+        /**
+         * Verifies if access is authorized to the resource with the given {@param name}.
+         * The {@param indexAbstraction}, which is the resource to be accessed, must be supplied if the resource exists or be {@code null}
+         * if it doesn't.
+         * Returns {@code true} if access to the given resource is authorized or {@code false} otherwise.
+         */
+        @Override
+        public boolean test(String name, @Nullable IndexAbstraction indexAbstraction) {
+            return biPredicate.test(name, indexAbstraction);
+        }
+
+        private static boolean isPartOfDatastream(IndexAbstraction indexAbstraction) {
+            return indexAbstraction != null
+                && (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM || indexAbstraction.getParentDataStream() != null);
+        }
     }
 
     /**
@@ -373,20 +429,20 @@ public IndicesAccessControl authorize(
         }
 
         final Map<String, IndexResource> resources = Maps.newMapWithExpectedSize(requestedIndicesOrAliases.size());
-        int totalResourceCount = 0;
+        final AtomicInteger totalResourceCountHolder = new AtomicInteger(0);
 
         for (String indexOrAlias : requestedIndicesOrAliases) {
             final IndexResource resource = new IndexResource(indexOrAlias, lookup.get(indexOrAlias));
             resources.put(resource.name, resource);
-            totalResourceCount += resource.size();
+            totalResourceCountHolder.getAndAdd(resource.size());
         }
 
         final boolean overallGranted = isActionGranted(action, resources);
 
-        final Map<String, IndicesAccessControl.IndexAccessControl> indexPermissions = buildIndicesAccessControl(
+        final Supplier<Map<String, IndicesAccessControl.IndexAccessControl>> indexPermissions = () -> buildIndicesAccessControl(
             action,
             resources,
-            totalResourceCount,
+            totalResourceCountHolder.get(),
             fieldPermissionsCache
         );
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java
index 5cac22fa6b4b..ed2cef378f11 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
+import org.elasticsearch.xpack.core.security.authz.permission.IndicesPermission.IsResourceAuthorizedPredicate;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
 import org.elasticsearch.xpack.core.security.support.Automatons;
@@ -21,7 +22,6 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
-import java.util.function.Predicate;
 
 /**
  * A {@link Role} limited by another role.<br>
@@ -123,10 +123,8 @@ public IndicesAccessControl authorize(
      * action on.
      */
     @Override
-    public Predicate<IndexAbstraction> allowedIndicesMatcher(String action) {
-        Predicate<IndexAbstraction> predicate = baseRole.indices().allowedIndicesMatcher(action);
-        predicate = predicate.and(limitedByRole.indices().allowedIndicesMatcher(action));
-        return predicate;
+    public IsResourceAuthorizedPredicate allowedIndicesMatcher(String action) {
+        return baseRole.allowedIndicesMatcher(action).and(limitedByRole.allowedIndicesMatcher(action));
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteIndicesPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteIndicesPermission.java
index 0a9344f3cc41..2abc93997fa9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteIndicesPermission.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/RemoteIndicesPermission.java
@@ -51,6 +51,11 @@ public Builder addGroup(
             final boolean allowRestrictedIndices,
             final String... indices
         ) {
+            assert query == null || query.size() <= 1 : "remote indices groups only support up to one DLS query";
+            assert fieldPermissions.getFieldPermissionsDefinitions()
+                .stream()
+                .noneMatch(groups -> groups.getFieldGrantExcludeGroups().size() > 1)
+                : "remote indices groups only support up to one FLS field-grant-exclude group";
             remoteIndicesGroups.computeIfAbsent(remoteClusterAliases, k -> new ArrayList<>())
                 .add(
                     new IndicesPermission.Group(
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java
index d654746c6a13..bb6d844f62f8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.xpack.core.security.authz.RestrictedIndices;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
+import org.elasticsearch.xpack.core.security.authz.permission.IndicesPermission.IsResourceAuthorizedPredicate;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
@@ -36,7 +37,6 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
-import java.util.function.Predicate;
 
 public interface Role {
 
@@ -64,7 +64,7 @@ public interface Role {
      * @return A predicate that will match all the indices that this role
      * has the privilege for executing the given action on.
      */
-    Predicate<IndexAbstraction> allowedIndicesMatcher(String action);
+    IsResourceAuthorizedPredicate allowedIndicesMatcher(String action);
 
     /**
      * Returns an {@link Automaton} that matches all action names allowed for the given index
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/SimpleRole.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/SimpleRole.java
index cbad92d231fe..7fa0b74f862e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/SimpleRole.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/SimpleRole.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.PrivilegesCheckResult;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.PrivilegesToCheck;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
+import org.elasticsearch.xpack.core.security.authz.permission.IndicesPermission.IsResourceAuthorizedPredicate;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
 
@@ -28,7 +29,6 @@
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Predicate;
 
 public class SimpleRole implements Role {
 
@@ -97,7 +97,7 @@ public boolean hasFieldOrDocumentLevelSecurity() {
     }
 
     @Override
-    public Predicate<IndexAbstraction> allowedIndicesMatcher(String action) {
+    public IsResourceAuthorizedPredicate allowedIndicesMatcher(String action) {
         return indices.allowedIndicesMatcher(action);
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java
index 9368666019d7..14b563fcc478 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ManageOwnApiKeyClusterPrivilege.java
@@ -145,11 +145,11 @@ private static boolean checkIfUserIsOwnerOfApiKeys(
                     if (false == username.equals(authentication.getEffectiveSubject().getUser().principal())) {
                         return false;
                     }
-                    RealmDomain domain = authentication.getSourceRealm().getDomain();
+                    RealmDomain domain = authentication.getEffectiveSubject().getRealm().getDomain();
                     if (domain != null) {
                         return domain.realms().stream().anyMatch(realmIdentifier -> realmName.equals(realmIdentifier.getName()));
                     } else {
-                        return realmName.equals(authentication.getSourceRealm().getName());
+                        return realmName.equals(authentication.getEffectiveSubject().getRealm().getName());
                     }
                 }
             }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
index bbeab238dc0c..8512d29f3901 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
@@ -694,24 +694,35 @@ public static RoleDescriptor kibanaSystemRoleDescriptor(String name) {
                     .indices(".ml-annotations*", ".ml-notifications*")
                     .privileges("read", "write")
                     .build(),
+
                 // APM agent configuration - system index defined in KibanaPlugin
                 RoleDescriptor.IndicesPrivileges.builder()
                     .indices(".apm-agent-configuration")
                     .privileges("all")
                     .allowRestrictedIndices(true)
                     .build(),
+
                 // APM custom link index creation - system index defined in KibanaPlugin
                 RoleDescriptor.IndicesPrivileges.builder()
                     .indices(".apm-custom-link")
                     .privileges("all")
                     .allowRestrictedIndices(true)
                     .build(),
+
+                // APM source map index creation - system index defined in KibanaPlugin
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(".apm-source-map")
+                    .privileges("all")
+                    .allowRestrictedIndices(true)
+                    .build(),
+
                 // APM telemetry queries APM indices in kibana task runner
                 RoleDescriptor.IndicesPrivileges.builder().indices("apm-*").privileges("read", "read_cross_cluster").build(),
                 RoleDescriptor.IndicesPrivileges.builder().indices("logs-apm.*").privileges("read", "read_cross_cluster").build(),
                 RoleDescriptor.IndicesPrivileges.builder().indices("metrics-apm.*").privileges("read", "read_cross_cluster").build(),
                 RoleDescriptor.IndicesPrivileges.builder().indices("traces-apm.*").privileges("read", "read_cross_cluster").build(),
                 RoleDescriptor.IndicesPrivileges.builder().indices("traces-apm-*").privileges("read", "read_cross_cluster").build(),
+
                 // Data telemetry reads mappings, metadata and stats of indices
                 RoleDescriptor.IndicesPrivileges.builder().indices("*").privileges("view_index_metadata", "monitor").build(),
                 // Endpoint diagnostic information. Kibana reads from these indices to send telemetry
@@ -719,6 +730,8 @@ public static RoleDescriptor kibanaSystemRoleDescriptor(String name) {
                 // Fleet Server indices. Kibana create this indice before Fleet Server use them.
                 // Fleet Server indices. Kibana read and write to this indice to manage Elastic Agents
                 RoleDescriptor.IndicesPrivileges.builder().indices(".fleet*").allowRestrictedIndices(true).privileges("all").build(),
+                // Fleet telemetry queries Agent Logs indices in kibana task runner
+                RoleDescriptor.IndicesPrivileges.builder().indices("logs-elastic_agent*").privileges("read").build(),
                 // Legacy "Alerts as data" used in Security Solution.
                 // Kibana user creates these indices; reads / writes to them.
                 RoleDescriptor.IndicesPrivileges.builder().indices(ReservedRolesStore.ALERTS_LEGACY_INDEX).privileges("all").build(),
@@ -755,7 +768,7 @@ public static RoleDescriptor kibanaSystemRoleDescriptor(String name) {
                         "logs-*",
                         "synthetics-*",
                         "traces-*",
-                        "/metrics-.*&~(metrics-endpoint\\.metadata_current_default)/",
+                        "/metrics-.*&~(metrics-endpoint\\.metadata_current_default.*)/",
                         ".logs-endpoint.action.responses-*",
                         ".logs-endpoint.diagnostic.collection-*",
                         ".logs-endpoint.actions-*",
@@ -807,16 +820,24 @@ public static RoleDescriptor kibanaSystemRoleDescriptor(String name) {
                     .build(),
                 RoleDescriptor.IndicesPrivileges.builder()
                     .indices(
-                        "metrics-endpoint.metadata_current_default",
-                        ".metrics-endpoint.metadata_current_default",
-                        ".metrics-endpoint.metadata_united_default"
+                        "metrics-endpoint.metadata_current_default*",
+                        ".metrics-endpoint.metadata_current_default*",
+                        ".metrics-endpoint.metadata_united_default*"
                     )
-                    .privileges("create_index", "delete_index", "read", "index")
+                    .privileges("create_index", "delete_index", "read", "index", IndicesAliasesAction.NAME, UpdateSettingsAction.NAME)
                     .build(),
                 // For src/dest indices of the example transform package
                 RoleDescriptor.IndicesPrivileges.builder()
                     .indices("kibana_sample_data_*")
-                    .privileges("create_index", "delete_index", "read", "index", "view_index_metadata")
+                    .privileges(
+                        "create_index",
+                        "delete_index",
+                        "read",
+                        "index",
+                        "view_index_metadata",
+                        IndicesAliasesAction.NAME,
+                        UpdateSettingsAction.NAME
+                    )
                     .build(),
                 // For src/dest indices of the Cloud Security Posture packages that ships a transform
                 RoleDescriptor.IndicesPrivileges.builder()
@@ -824,8 +845,8 @@ public static RoleDescriptor kibanaSystemRoleDescriptor(String name) {
                     .privileges("read", "view_index_metadata")
                     .build(),
                 RoleDescriptor.IndicesPrivileges.builder()
-                    .indices("logs-cloud_security_posture.findings_latest-default", "logs-cloud_security_posture.scores-default")
-                    .privileges("create_index", "read", "index", "delete")
+                    .indices("logs-cloud_security_posture.findings_latest-default*", "logs-cloud_security_posture.scores-default*")
+                    .privileges("create_index", "read", "index", "delete", IndicesAliasesAction.NAME, UpdateSettingsAction.NAME)
                     .build() },
             null,
             new ConfigurableClusterPrivilege[] {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/RoleReference.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/RoleReference.java
index 9ceeb724b420..3a4a37771329 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/RoleReference.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/RoleReference.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.xpack.core.security.authc.RemoteAccessAuthentication;
 
 import java.util.HashSet;
 import java.util.List;
@@ -116,6 +117,38 @@ public ApiKeyRoleType getRoleType() {
         }
     }
 
+    final class RemoteAccessRoleReference implements RoleReference {
+
+        private final RemoteAccessAuthentication.RoleDescriptorsBytes roleDescriptorsBytes;
+        private RoleKey id = null;
+
+        public RemoteAccessRoleReference(RemoteAccessAuthentication.RoleDescriptorsBytes roleDescriptorsBytes) {
+            this.roleDescriptorsBytes = roleDescriptorsBytes;
+        }
+
+        @Override
+        public RoleKey id() {
+            // Hashing can be expensive. memorize the result in case the method is called multiple times.
+            if (id == null) {
+                final String roleDescriptorsHash = MessageDigests.toHexString(
+                    MessageDigests.digest(roleDescriptorsBytes, MessageDigests.sha256())
+                );
+                id = new RoleKey(Set.of("remote_access:" + roleDescriptorsHash), "remote_access");
+            }
+            return id;
+        }
+
+        @Override
+        public void resolve(RoleReferenceResolver resolver, ActionListener<RolesRetrievalResult> listener) {
+            resolver.resolveRemoteAccessRoleReference(this, listener);
+        }
+
+        public RemoteAccessAuthentication.RoleDescriptorsBytes getRoleDescriptorsBytes() {
+            return roleDescriptorsBytes;
+        }
+
+    }
+
     /**
      * Same as {@link ApiKeyRoleReference} but for BWC purpose (prior to v7.9.0)
      */
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/RoleReferenceIntersection.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/RoleReferenceIntersection.java
index 7c3f1df38c18..26f58693bc25 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/RoleReferenceIntersection.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/RoleReferenceIntersection.java
@@ -37,15 +37,18 @@ public List<RoleReference> getRoleReferences() {
     }
 
     public void buildRole(BiConsumer<RoleReference, ActionListener<Role>> singleRoleBuilder, ActionListener<Role> roleActionListener) {
-        final GroupedActionListener<Role> roleGroupedActionListener = new GroupedActionListener<>(ActionListener.wrap(roles -> {
-            assert false == roles.isEmpty();
-            final Iterator<Role> iterator = roles.stream().iterator();
-            Role finalRole = iterator.next();
-            while (iterator.hasNext()) {
-                finalRole = finalRole.limitedBy(iterator.next());
-            }
-            roleActionListener.onResponse(finalRole);
-        }, roleActionListener::onFailure), roleReferences.size());
+        final GroupedActionListener<Role> roleGroupedActionListener = new GroupedActionListener<>(
+            roleReferences.size(),
+            ActionListener.wrap(roles -> {
+                assert false == roles.isEmpty();
+                final Iterator<Role> iterator = roles.stream().iterator();
+                Role finalRole = iterator.next();
+                while (iterator.hasNext()) {
+                    finalRole = finalRole.limitedBy(iterator.next());
+                }
+                roleActionListener.onResponse(finalRole);
+            }, roleActionListener::onFailure)
+        );
 
         roleReferences.forEach(roleReference -> singleRoleBuilder.accept(roleReference, roleGroupedActionListener));
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/RoleReferenceResolver.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/RoleReferenceResolver.java
index 44522b588452..e39b26afdaca 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/RoleReferenceResolver.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/RoleReferenceResolver.java
@@ -25,4 +25,9 @@ void resolveBwcApiKeyRoleReference(
     );
 
     void resolveServiceAccountRoleReference(ServiceAccountRoleReference roleReference, ActionListener<RolesRetrievalResult> listener);
+
+    void resolveRemoteAccessRoleReference(
+        RoleReference.RemoteAccessRoleReference remoteAccessRoleReference,
+        ActionListener<RolesRetrievalResult> listener
+    );
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/support/DLSRoleQueryValidator.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/support/DLSRoleQueryValidator.java
index 21954e867c0a..7b2e7e1e854b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/support/DLSRoleQueryValidator.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/support/DLSRoleQueryValidator.java
@@ -13,13 +13,9 @@
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.BoostingQueryBuilder;
-import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
 import org.elasticsearch.index.query.GeoShapeQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.TermsQueryBuilder;
-import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptType;
@@ -33,8 +29,7 @@
 import org.elasticsearch.xpack.core.security.user.User;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.Collections;
 
 /**
  * This class helps in evaluating the query field if it is template,
@@ -156,10 +151,38 @@ public static QueryBuilder evaluateAndVerifyRoleQuery(
     @Nullable
     public static QueryBuilder evaluateAndVerifyRoleQuery(String query, NamedXContentRegistry xContentRegistry) throws IOException {
         if (query != null) {
-            try (XContentParser parser = XContentFactory.xContent(query).createParser(parserConfig(xContentRegistry), query)) {
-                QueryBuilder queryBuilder = AbstractQueryBuilder.parseTopLevelQuery(parser);
-                verifyRoleQuery(queryBuilder);
-                return queryBuilder;
+            NamedXContentRegistry registryWrapper = new NamedXContentRegistry(Collections.emptyList()) {
+                @Override
+                public <T, C> T parseNamedObject(Class<T> categoryClass, String name, XContentParser parser, C context) throws IOException {
+                    T namedObject = xContentRegistry.parseNamedObject(categoryClass, name, parser, context);
+                    if (categoryClass.equals(QueryBuilder.class)) {
+                        if (namedObject instanceof TermsQueryBuilder termsQueryBuilder) {
+                            if (termsQueryBuilder.termsLookup() != null) {
+                                throw new IllegalArgumentException("terms query with terms lookup isn't supported as part of a role query");
+                            }
+                        } else if (namedObject instanceof GeoShapeQueryBuilder geoShapeQueryBuilder) {
+                            if (geoShapeQueryBuilder.shape() == null) {
+                                throw new IllegalArgumentException(
+                                    "geoshape query referring to indexed shapes isn't supported as part of a role query"
+                                );
+                            }
+                        }
+                    }
+                    return namedObject;
+                }
+            };
+            try (XContentParser parser = XContentFactory.xContent(query).createParser(parserConfig(registryWrapper), query)) {
+                return AbstractQueryBuilder.parseTopLevelQuery(parser, queryName -> {
+                    switch (queryName) {
+                        // actually only if percolate query is referring to an existing document then this is problematic,
+                        // a normal percolate query does work. However we can't check that here as this query builder is inside
+                        // another module. So we don't allow the entire percolate query. I don't think users would ever use
+                        // a percolate query as role query, so this restriction shouldn't prohibit anyone from using dls.
+                        case "percolate", "has_child", "has_parent" -> throw new IllegalArgumentException(
+                            queryName + " query isn't supported as part of a role query"
+                        );
+                    }
+                });
             }
         }
         return null;
@@ -168,48 +191,4 @@ public static QueryBuilder evaluateAndVerifyRoleQuery(String query, NamedXConten
     private static XContentParserConfiguration parserConfig(NamedXContentRegistry xContentRegistry) {
         return XContentParserConfiguration.EMPTY.withRegistry(xContentRegistry).withDeprecationHandler(LoggingDeprecationHandler.INSTANCE);
     }
-
-    /**
-     * Checks whether the role query contains queries we know can't be used as DLS role query.
-     *
-     * @param queryBuilder {@link QueryBuilder} for given query
-     */
-    // pkg protected for testing
-    static void verifyRoleQuery(QueryBuilder queryBuilder) {
-        if (queryBuilder instanceof TermsQueryBuilder termsQueryBuilder) {
-            if (termsQueryBuilder.termsLookup() != null) {
-                throw new IllegalArgumentException("terms query with terms lookup isn't supported as part of a role query");
-            }
-        } else if (queryBuilder instanceof GeoShapeQueryBuilder geoShapeQueryBuilder) {
-            if (geoShapeQueryBuilder.shape() == null) {
-                throw new IllegalArgumentException("geoshape query referring to indexed shapes isn't supported as part of a role query");
-            }
-        } else if (queryBuilder.getName().equals("percolate")) {
-            // actually only if percolate query is referring to an existing document then this is problematic,
-            // a normal percolate query does work. However we can't check that here as this query builder is inside
-            // another module. So we don't allow the entire percolate query. I don't think users would ever use
-            // a percolate query as role query, so this restriction shouldn't prohibit anyone from using dls.
-            throw new IllegalArgumentException("percolate query isn't supported as part of a role query");
-        } else if (queryBuilder.getName().equals("has_child")) {
-            throw new IllegalArgumentException("has_child query isn't supported as part of a role query");
-        } else if (queryBuilder.getName().equals("has_parent")) {
-            throw new IllegalArgumentException("has_parent query isn't supported as part of a role query");
-        } else if (queryBuilder instanceof BoolQueryBuilder boolQueryBuilder) {
-            List<QueryBuilder> clauses = new ArrayList<>();
-            clauses.addAll(boolQueryBuilder.filter());
-            clauses.addAll(boolQueryBuilder.must());
-            clauses.addAll(boolQueryBuilder.mustNot());
-            clauses.addAll(boolQueryBuilder.should());
-            for (QueryBuilder clause : clauses) {
-                verifyRoleQuery(clause);
-            }
-        } else if (queryBuilder instanceof ConstantScoreQueryBuilder) {
-            verifyRoleQuery(((ConstantScoreQueryBuilder) queryBuilder).innerQuery());
-        } else if (queryBuilder instanceof FunctionScoreQueryBuilder) {
-            verifyRoleQuery(((FunctionScoreQueryBuilder) queryBuilder).query());
-        } else if (queryBuilder instanceof BoostingQueryBuilder) {
-            verifyRoleQuery(((BoostingQueryBuilder) queryBuilder).negativeQuery());
-            verifyRoleQuery(((BoostingQueryBuilder) queryBuilder).positiveQuery());
-        }
-    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/xcontent/XContentUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/xcontent/XContentUtils.java
index ed92541f7c8a..979e7d99a6af 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/xcontent/XContentUtils.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/xcontent/XContentUtils.java
@@ -94,8 +94,14 @@ public static void addAuthorizationInfo(final XContentBuilder builder, final Map
         if (authKey == null) {
             return;
         }
+        Subject authenticationSubject;
+        try {
+            authenticationSubject = AuthenticationContextSerializer.decode(authKey).getEffectiveSubject();
+        } catch (Exception e) {
+            // The exception will have been logged by AuthenticationContextSerializer.decode() so don't log it again here.
+            return;
+        }
         builder.startObject("authorization");
-        Subject authenticationSubject = AuthenticationContextSerializer.decode(authKey).getEffectiveSubject();
         switch (authenticationSubject.getType()) {
             case USER -> builder.array(User.Fields.ROLES.getPreferredName(), authenticationSubject.getUser().roles());
             case API_KEY -> {
@@ -103,12 +109,18 @@ public static void addAuthorizationInfo(final XContentBuilder builder, final Map
                 Map<String, Object> metadata = authenticationSubject.getMetadata();
                 builder.field("id", metadata.get(AuthenticationField.API_KEY_ID_KEY));
                 Object name = metadata.get(AuthenticationField.API_KEY_NAME_KEY);
-                if (name != null) {
+                if (name instanceof String) {
                     builder.field("name", name);
                 }
                 builder.endObject();
             }
             case SERVICE_ACCOUNT -> builder.field("service_account", authenticationSubject.getUser().principal());
+            case REMOTE_ACCESS -> {
+                // TODO handle remote access authentication
+                final String message = "remote access authentication is not yet supported";
+                assert false : message;
+                throw new UnsupportedOperationException(message);
+            }
         }
         builder.endObject();
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SLMFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SLMFeatureSetUsage.java
index 93f154d08894..ffb564b9e0d5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SLMFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SLMFeatureSetUsage.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.slm;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
@@ -39,8 +39,8 @@ public SLMFeatureSetUsage(@Nullable SnapshotLifecycleStats slmStats) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_5_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_5_0;
     }
 
     public SnapshotLifecycleStats getStats() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java
index 9fcd82cba7df..1c0abff4eb2f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java
@@ -7,24 +7,28 @@
 
 package org.elasticsearch.xpack.core.slm;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.DiffableUtils;
 import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.SimpleDiffable;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata;
 import org.elasticsearch.xpack.core.ilm.OperationMode;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -96,6 +100,10 @@ public Map<String, SnapshotLifecyclePolicyMetadata> getSnapshotConfigurations()
         return Collections.unmodifiableMap(this.snapshotConfigurations);
     }
 
+    /**
+     * @deprecated use {@link LifecycleOperationMetadata#getSLMOperationMode()} instead. This may be incorrect.
+     */
+    @Deprecated(since = "8.7.0")
     public OperationMode getOperationMode() {
         return operationMode;
     }
@@ -120,8 +128,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_4_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_4_0;
     }
 
     @Override
@@ -132,11 +140,15 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(POLICIES_FIELD.getPreferredName(), this.snapshotConfigurations);
-        builder.field(OPERATION_MODE_FIELD.getPreferredName(), operationMode);
-        builder.field(STATS_FIELD.getPreferredName(), this.slmStats);
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return Iterators.concat(
+            ChunkedToXContentHelper.xContentValuesMap(POLICIES_FIELD.getPreferredName(), this.snapshotConfigurations),
+            Iterators.single((builder, params) -> {
+                builder.field(OPERATION_MODE_FIELD.getPreferredName(), operationMode);
+                builder.field(STATS_FIELD.getPreferredName(), this.slmStats);
+                return builder;
+            })
+        );
     }
 
     @Override
@@ -215,8 +227,8 @@ static Diff<SnapshotLifecyclePolicyMetadata> readLifecyclePolicyDiffFrom(StreamI
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.V_7_4_0;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.V_7_4_0;
         }
 
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java
index 59c65da8cfea..d3bd79c66d79 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.slm;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.SimpleDiffable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -116,7 +116,7 @@ public static SnapshotLifecyclePolicyMetadata parse(XContentParser parser, Strin
         this.modifiedDate = in.readVLong();
         this.lastSuccess = in.readOptionalWriteable(SnapshotInvocationRecord::new);
         this.lastFailure = in.readOptionalWriteable(SnapshotInvocationRecord::new);
-        this.invocationsSinceLastSuccess = in.getVersion().onOrAfter(Version.V_8_4_0) ? in.readVLong() : 0L;
+        this.invocationsSinceLastSuccess = in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0) ? in.readVLong() : 0L;
     }
 
     @Override
@@ -127,7 +127,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(this.modifiedDate);
         out.writeOptionalWriteable(this.lastSuccess);
         out.writeOptionalWriteable(this.lastFailure);
-        if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             out.writeVLong(this.invocationsSinceLastSuccess);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/GetSLMStatusAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/GetSLMStatusAction.java
index a2a4831cde94..3008e8f2763d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/GetSLMStatusAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/action/GetSLMStatusAction.java
@@ -29,7 +29,7 @@ protected GetSLMStatusAction() {
 
     public static class Response extends ActionResponse implements ToXContentObject {
 
-        private OperationMode mode;
+        private final OperationMode mode;
 
         public Response(StreamInput in) throws IOException {
             super(in);
@@ -45,6 +45,10 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeEnum(this.mode);
         }
 
+        public OperationMode getOperationMode() {
+            return this.mode;
+        }
+
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/spatial/SpatialFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/spatial/SpatialFeatureSetUsage.java
index 447e2483e81b..0c68b5a15099 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/spatial/SpatialFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/spatial/SpatialFeatureSetUsage.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.spatial;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
@@ -28,7 +28,7 @@ public SpatialFeatureSetUsage(SpatialStatsAction.Response statsResponse) {
 
     public SpatialFeatureSetUsage(StreamInput input) throws IOException {
         super(input);
-        if (input.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (input.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             this.statsResponse = new SpatialStatsAction.Response(input);
         } else {
             this.statsResponse = null;
@@ -36,8 +36,8 @@ public SpatialFeatureSetUsage(StreamInput input) throws IOException {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_4_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_4_0;
     }
 
     SpatialStatsAction.Response statsResponse() {
@@ -47,7 +47,7 @@ SpatialStatsAction.Response statsResponse() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        if (out.getVersion().onOrAfter(Version.V_7_11_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_11_0)) {
             this.statsResponse.writeTo(out);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/sql/SqlFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/sql/SqlFeatureSetUsage.java
index a6792fddec52..1cc55f267ce0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/sql/SqlFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/sql/SqlFeatureSetUsage.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.sql;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -32,8 +32,8 @@ public SqlFeatureSetUsage(Map<String, Object> stats) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_0_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_0_0;
     }
 
     public Map<String, Object> stats() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/RestrictedTrustConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/RestrictedTrustConfig.java
index db8de5d36455..8629ff93170f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/RestrictedTrustConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/RestrictedTrustConfig.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.ssl.SslTrustConfig;
 import org.elasticsearch.common.ssl.StoredCertificate;
+import org.elasticsearch.common.ssl.X509Field;
 import org.elasticsearch.common.util.CollectionUtils;
 
 import java.io.IOException;
@@ -17,6 +18,7 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
+import java.util.Set;
 
 import javax.net.ssl.X509ExtendedTrustManager;
 
@@ -30,8 +32,10 @@ public final class RestrictedTrustConfig implements SslTrustConfig {
     private static final String RESTRICTIONS_KEY_SUBJECT_NAME = "trust.subject_name";
     private final Path groupConfigPath;
     private final SslTrustConfig delegate;
+    private final Set<X509Field> configuredX509Fields;
 
-    RestrictedTrustConfig(Path groupConfigPath, SslTrustConfig delegate) {
+    RestrictedTrustConfig(Path groupConfigPath, Set<X509Field> configuredX509Fields, SslTrustConfig delegate) {
+        this.configuredX509Fields = configuredX509Fields;
         this.groupConfigPath = Objects.requireNonNull(groupConfigPath);
         this.delegate = Objects.requireNonNull(delegate);
     }
@@ -41,7 +45,7 @@ public RestrictedTrustManager createTrustManager() {
         try {
             final X509ExtendedTrustManager delegateTrustManager = delegate.createTrustManager();
             final CertificateTrustRestrictions trustGroupConfig = readTrustGroup(groupConfigPath);
-            return new RestrictedTrustManager(delegateTrustManager, trustGroupConfig);
+            return new RestrictedTrustManager(delegateTrustManager, trustGroupConfig, configuredX509Fields);
         } catch (IOException e) {
             throw new ElasticsearchException("failed to initialize TrustManager for {}", e, toString());
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/RestrictedTrustManager.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/RestrictedTrustManager.java
index 7c387017bcb6..f79184c58ce3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/RestrictedTrustManager.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/RestrictedTrustManager.java
@@ -9,6 +9,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.ssl.DerParser;
+import org.elasticsearch.common.ssl.X509Field;
 
 import java.io.IOException;
 import java.net.Socket;
@@ -17,6 +18,7 @@
 import java.security.cert.X509Certificate;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
@@ -31,23 +33,34 @@
 
 /**
  * An X509 trust manager that only trusts connections from a restricted set of predefined network entities (nodes, clients, etc).
- * The trusted entities are defined as a list of predicates on {@link CertificateTrustRestrictions} that are applied to the
- * common-names of the certificate.
- * The common-names are read as subject-alternative-names with type 'Other' and a 'cn' OID.
- * The underlying certificate validation is delegated to another TrustManager.
+ * The trusted entities are defined as a list of predicates on {@link CertificateTrustRestrictions} that built from the
+ * configured restricted trust file. The values in the restricted trust file are compared to value(s) read from the X509 certificate.
+ * If the value(s) read from the X509 certificate match values configured in restricted trust file then restricted trust is established.
+ * If there is no match, then restricted trust is not established and the connection should be terminated. Restricted trust should be used
+ * in conjunction with additional trust models and is intended to restrict, not provide trust.
+ * The values read from the X509 certificate are configurable and the following are supported:
+ * <ul>
+ *     <li>subjectAltName.otherName.commonName</li>
+ *     <li>subjectAltName.dnsName</li>
+ * </ul>
+ * see also: {@link RestrictedTrustConfig}
  */
 public final class RestrictedTrustManager extends X509ExtendedTrustManager {
     private static final Logger logger = LogManager.getLogger(RestrictedTrustManager.class);
     private static final String CN_OID = "2.5.4.3";
     private static final int SAN_CODE_OTHERNAME = 0;
+    private static final int SAN_CODE_DNS = 2;
 
     private final X509ExtendedTrustManager delegate;
     private final CertificateTrustRestrictions trustRestrictions;
+    private final Set<X509Field> x509Fields;
 
-    public RestrictedTrustManager(X509ExtendedTrustManager delegate, CertificateTrustRestrictions restrictions) {
+    public RestrictedTrustManager(X509ExtendedTrustManager delegate, CertificateTrustRestrictions restrictions, Set<X509Field> x509Fields) {
         this.delegate = delegate;
         this.trustRestrictions = restrictions;
+        this.x509Fields = x509Fields;
         logger.debug("Configured with trust restrictions: [{}]", restrictions);
+        logger.debug("Configured with x509 fields: [{}]", x509Fields);
     }
 
     @Override
@@ -96,28 +109,32 @@ private void verifyTrust(X509Certificate[] chain) throws CertificateException {
             throw new CertificateException("No certificate presented");
         }
         final X509Certificate certificate = chain[0];
-        Set<String> names = readCommonNames(certificate);
-        if (verifyCertificateNames(names)) {
+        Set<String> values = readX509Certificate(certificate);
+        if (verifyCertificateNames(values)) {
             logger.debug(
                 () -> format(
-                    "Trusting certificate [%s] [%s] with common-names [%s]",
+                    "Trusting certificate [%s] [%s] with fields [%s] with values [%s]",
                     certificate.getSubjectX500Principal(),
                     certificate.getSerialNumber().toString(16),
-                    names
+                    x509Fields,
+                    values
                 )
             );
         } else {
             logger.info(
-                "Rejecting certificate [{}] [{}] with common-names [{}]",
+                "Rejecting certificate [{}] [{}] for fields [{}] with values [{}]",
                 certificate.getSubjectX500Principal(),
                 certificate.getSerialNumber().toString(16),
-                names
+                x509Fields,
+                values
             );
             throw new CertificateException(
                 "Certificate for "
                     + certificate.getSubjectX500Principal()
-                    + " with common-names "
-                    + names
+                    + " with fields "
+                    + x509Fields
+                    + " with values "
+                    + values
                     + " does not match the trusted names "
                     + trustRestrictions.getTrustedNames()
             );
@@ -135,13 +152,28 @@ private boolean verifyCertificateNames(Set<String> names) {
         return false;
     }
 
-    private static Set<String> readCommonNames(X509Certificate certificate) throws CertificateParsingException {
-        return getSubjectAlternativeNames(certificate).stream()
-            .filter(pair -> ((Integer) pair.get(0)).intValue() == SAN_CODE_OTHERNAME)
-            .map(pair -> pair.get(1))
-            .map(value -> decodeDerValue((byte[]) value, certificate))
-            .filter(Objects::nonNull)
-            .collect(Collectors.toSet());
+    private Set<String> readX509Certificate(X509Certificate certificate) throws CertificateParsingException {
+        Collection<List<?>> sans = getSubjectAlternativeNames(certificate);
+        Set<String> values = new HashSet<>();
+        if (x509Fields.contains(X509Field.SAN_DNS)) {
+            Set<String> dnsNames = sans.stream()
+                .filter(pair -> ((Integer) pair.get(0)).intValue() == SAN_CODE_DNS)
+                .map(pair -> pair.get(1))
+                .map(Object::toString)
+                .filter(Objects::nonNull)
+                .collect(Collectors.toSet());
+            values.addAll(dnsNames);
+        }
+        if (x509Fields.contains(X509Field.SAN_OTHERNAME_COMMONNAME)) {
+            Set<String> otherNames = sans.stream()
+                .filter(pair -> ((Integer) pair.get(0)).intValue() == SAN_CODE_OTHERNAME)
+                .map(pair -> pair.get(1))
+                .map(value -> decodeDerValue((byte[]) value, certificate))
+                .filter(Objects::nonNull)
+                .collect(Collectors.toSet());
+            values.addAll(otherNames);
+        }
+        return values;
     }
 
     /**
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloader.java
index d2517d3650f5..cb32ef2d8b18 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloader.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloader.java
@@ -9,6 +9,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.ssl.SslConfiguration;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.watcher.FileChangesListener;
@@ -26,8 +27,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
 import java.util.function.Consumer;
 
 import javax.net.ssl.SSLContext;
@@ -40,7 +41,12 @@ public final class SSLConfigurationReloader {
 
     private static final Logger logger = LogManager.getLogger(SSLConfigurationReloader.class);
 
-    private final CompletableFuture<SSLService> sslServiceFuture = new CompletableFuture<>();
+    private final PlainActionFuture<SSLService> sslServiceFuture = new PlainActionFuture<>() {
+        @Override
+        protected boolean blockingAllowed() {
+            return true; // waits on the scheduler thread, once, and not for long
+        }
+    };
 
     public SSLConfigurationReloader(ResourceWatcherService resourceWatcherService, Collection<SslConfiguration> sslConfigurations) {
         startWatching(reloadConsumer(sslServiceFuture), resourceWatcherService, sslConfigurations);
@@ -56,13 +62,11 @@ public SSLConfigurationReloader(ResourceWatcherService resourceWatcherService, C
     }
 
     public void setSSLService(SSLService sslService) {
-        final boolean completed = sslServiceFuture.complete(sslService);
-        if (completed == false) {
-            throw new IllegalStateException("ssl service future was already completed!");
-        }
+        assert sslServiceFuture.isDone() == false : "ssl service future was already completed!";
+        sslServiceFuture.onResponse(sslService);
     }
 
-    private static Consumer<SslConfiguration> reloadConsumer(CompletableFuture<SSLService> future) {
+    private static Consumer<SslConfiguration> reloadConsumer(Future<SSLService> future) {
         return sslConfiguration -> {
             try {
                 final SSLService sslService = future.get();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationSettings.java
index 545fc0147047..9ceec66c6422 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationSettings.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationSettings.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.ssl.SslClientAuthenticationMode;
 import org.elasticsearch.common.ssl.SslConfigurationKeys;
 import org.elasticsearch.common.ssl.SslVerificationMode;
+import org.elasticsearch.common.ssl.X509Field;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 
@@ -28,6 +29,8 @@
 
 import javax.net.ssl.TrustManagerFactory;
 
+import static org.elasticsearch.common.ssl.SslConfigurationLoader.GLOBAL_DEFAULT_RESTRICTED_TRUST_FIELDS;
+
 /**
  * Bridges SSLConfiguration into the {@link Settings} framework, using {@link Setting} objects.
  */
@@ -43,6 +46,7 @@ public class SSLConfigurationSettings {
     final Setting<String> truststoreAlgorithm;
     final Setting<Optional<String>> truststoreType;
     final Setting<Optional<String>> trustRestrictionsPath;
+    final Setting<List<X509Field>> trustRestrictionsX509Fields;
     final Setting<List<String>> caPaths;
     final Setting<Optional<SslClientAuthenticationMode>> clientAuth;
     final Setting<Optional<SslVerificationMode>> verificationMode;
@@ -143,16 +147,29 @@ public class SSLConfigurationSettings {
         TRUST_STORE_TYPE_TEMPLATE
     );
 
-    private static final Function<String, Setting<Optional<String>>> TRUST_RESTRICTIONS_TEMPLATE = key -> new Setting<>(
+    private static final Function<String, Setting<Optional<String>>> TRUST_RESTRICTIONS_PATH_TEMPLATE = key -> new Setting<>(
         key,
         s -> null,
         Optional::ofNullable,
         Property.NodeScope,
         Property.Filtered
     );
-    private static final SslSetting<Optional<String>> TRUST_RESTRICTIONS = SslSetting.setting(
+    private static final SslSetting<Optional<String>> TRUST_RESTRICTIONS_PATH = SslSetting.setting(
         "trust_restrictions.path",
-        TRUST_RESTRICTIONS_TEMPLATE
+        TRUST_RESTRICTIONS_PATH_TEMPLATE
+    );
+
+    public static final Function<String, Setting<List<X509Field>>> TRUST_RESTRICTIONS_X509_FIELDS_TEMPLATE = key -> Setting.listSetting(
+        key,
+        GLOBAL_DEFAULT_RESTRICTED_TRUST_FIELDS.stream().map(X509Field::toString).collect(Collectors.toList()),
+        X509Field::parseForRestrictedTrust,
+        Property.NodeScope,
+        Property.Filtered
+    );
+
+    public static final SslSetting<List<X509Field>> TRUST_RESTRICTIONS_X509_FIELDS = SslSetting.setting(
+        SslConfigurationKeys.TRUST_RESTRICTIONS_X509_FIELDS,
+        TRUST_RESTRICTIONS_X509_FIELDS_TEMPLATE
     );
 
     private static final SslSetting<SecureString> LEGACY_KEY_PASSWORD = SslSetting.setting(
@@ -228,7 +245,8 @@ private SSLConfigurationSettings(String prefix, boolean acceptNonSecurePasswords
         truststorePassword = TRUSTSTORE_PASSWORD.withPrefix(prefix);
         truststoreAlgorithm = TRUSTSTORE_ALGORITHM.withPrefix(prefix);
         truststoreType = TRUSTSTORE_TYPE.withPrefix(prefix);
-        trustRestrictionsPath = TRUST_RESTRICTIONS.withPrefix(prefix);
+        trustRestrictionsPath = TRUST_RESTRICTIONS_PATH.withPrefix(prefix);
+        trustRestrictionsX509Fields = TRUST_RESTRICTIONS_X509_FIELDS.withPrefix(prefix);
         caPaths = CERT_AUTH_PATH.withPrefix(prefix);
         clientAuth = CLIENT_AUTH_SETTING.withPrefix(prefix);
         verificationMode = VERIFICATION_MODE.withPrefix(prefix);
@@ -241,6 +259,7 @@ private SSLConfigurationSettings(String prefix, boolean acceptNonSecurePasswords
             truststoreAlgorithm,
             truststoreType,
             trustRestrictionsPath,
+            trustRestrictionsX509Fields,
             caPaths,
             clientAuth,
             verificationMode
@@ -304,7 +323,8 @@ private static Collection<SslSetting<?>> settings() {
             TRUSTSTORE_ALGORITHM,
             KEY_STORE_TYPE,
             TRUSTSTORE_TYPE,
-            TRUST_RESTRICTIONS,
+            TRUST_RESTRICTIONS_PATH,
+            TRUST_RESTRICTIONS_X509_FIELDS,
             KEY_PATH,
             LEGACY_KEY_PASSWORD,
             KEY_PASSWORD,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLService.java
index c8bf0d00c572..101fe1db6d19 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLService.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLService.java
@@ -78,7 +78,9 @@
 import javax.net.ssl.X509ExtendedTrustManager;
 import javax.security.auth.x500.X500Principal;
 
+import static org.elasticsearch.transport.RemoteClusterPortSettings.REMOTE_CLUSTER_PORT_ENABLED;
 import static org.elasticsearch.xpack.core.XPackSettings.DEFAULT_SUPPORTED_PROTOCOLS;
+import static org.elasticsearch.xpack.core.XPackSettings.REMOTE_CLUSTER_SSL_ENABLED;
 
 /**
  * Provides access to {@link SSLEngine} and {@link SSLSocketFactory} objects based on a provided configuration. All
@@ -590,6 +592,8 @@ static Map<String, Settings> getSSLSettingsMap(Settings settings) {
         sslSettingsMap.put(WatcherField.EMAIL_NOTIFICATION_SSL_PREFIX, settings.getByPrefix(WatcherField.EMAIL_NOTIFICATION_SSL_PREFIX));
         sslSettingsMap.put(XPackSettings.TRANSPORT_SSL_PREFIX, settings.getByPrefix(XPackSettings.TRANSPORT_SSL_PREFIX));
         sslSettingsMap.putAll(getTransportProfileSSLSettings(settings));
+        // We build the ssl settings even when remote cluster port is not enabled because we may need it for the client side
+        sslSettingsMap.put(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX, getRemoteClusterSslSettings(settings));
         return Collections.unmodifiableMap(sslSettingsMap);
     }
 
@@ -611,6 +615,7 @@ Map<SslConfiguration, SSLContextHolder> loadSslConfigurations(Map<String, SslCon
         for (String context : List.of("xpack.security.transport.ssl", "xpack.security.http.ssl")) {
             validateServerConfiguration(context);
         }
+        maybeValidateRemoteClusterServerConfiguration();
 
         return Collections.unmodifiableMap(sslContextHolders);
     }
@@ -623,18 +628,7 @@ private void validateServerConfiguration(String prefix) {
             // Client Authentication _should_ be required, but if someone turns it off, then this check is no longer relevant
             final SSLConfigurationSettings configurationSettings = SSLConfigurationSettings.withPrefix(prefix + ".", true);
             if (isConfigurationValidForServerUsage(configuration) == false) {
-                throw new ElasticsearchSecurityException(
-                    "invalid SSL configuration for "
-                        + prefix
-                        + " - server ssl configuration requires a key and certificate, but these have not been configured; "
-                        + "you must set either ["
-                        + configurationSettings.x509KeyPair.keystorePath.getKey()
-                        + "], or both ["
-                        + configurationSettings.x509KeyPair.keyPath.getKey()
-                        + "] and ["
-                        + configurationSettings.x509KeyPair.certificatePath.getKey()
-                        + "]"
-                );
+                throwExceptionForMissingKeyMaterial(prefix, configurationSettings);
             }
         } else if (settings.hasValue(enabledSetting) == false) {
             final List<String> sslSettingNames = settings.keySet().stream().filter(s -> s.startsWith(prefix)).sorted().toList();
@@ -652,6 +646,35 @@ private void validateServerConfiguration(String prefix) {
         }
     }
 
+    private void maybeValidateRemoteClusterServerConfiguration() {
+        if (REMOTE_CLUSTER_PORT_ENABLED.get(settings) == false) {
+            return;
+        }
+        final String prefix = "xpack.security.remote_cluster.ssl";
+        final SslConfiguration sslConfiguration = getSSLConfiguration(prefix);
+        if (REMOTE_CLUSTER_SSL_ENABLED.get(settings)) {
+            if (isConfigurationValidForServerUsage(sslConfiguration) == false) {
+                final SSLConfigurationSettings configurationSettings = SSLConfigurationSettings.withPrefix(prefix + ".", false);
+                throwExceptionForMissingKeyMaterial(prefix, configurationSettings);
+            }
+        }
+    }
+
+    private static void throwExceptionForMissingKeyMaterial(String prefix, SSLConfigurationSettings configurationSettings) {
+        throw new ElasticsearchSecurityException(
+            "invalid SSL configuration for "
+                + prefix
+                + " - server ssl configuration requires a key and certificate, but these have not been configured; "
+                + "you must set either ["
+                + configurationSettings.x509KeyPair.keystorePath.getKey()
+                + "], or both ["
+                + configurationSettings.x509KeyPair.keyPath.getKey()
+                + "] and ["
+                + configurationSettings.x509KeyPair.certificatePath.getKey()
+                + "]"
+        );
+    }
+
     /**
      * Returns information about each certificate that is referenced by any SSL configuration.
      * This includes certificates used for identity (with a private key) and those used for trust, but excludes
@@ -877,6 +900,15 @@ private static Settings getHttpTransportSSLSettings(Settings settings) {
         return builder.build();
     }
 
+    private static Settings getRemoteClusterSslSettings(Settings settings) {
+        final Settings remoteClusterSslSettings = settings.getByPrefix(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX);
+        final Settings.Builder builder = Settings.builder().put(remoteClusterSslSettings);
+        if (builder.get("client_authentication") == null) {
+            builder.put("client_authentication", XPackSettings.REMOTE_CLUSTER_CLIENT_AUTH_DEFAULT);
+        }
+        return builder.build();
+    }
+
     public SslConfiguration getHttpTransportSSLConfiguration() {
         return getSSLConfiguration(XPackSettings.HTTP_SSL_PREFIX);
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SslSettingsLoader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SslSettingsLoader.java
index 3c851dbb9da9..2196558cf990 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SslSettingsLoader.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SslSettingsLoader.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.ssl.SslKeyConfig;
 import org.elasticsearch.common.ssl.SslTrustConfig;
 import org.elasticsearch.common.ssl.SslVerificationMode;
+import org.elasticsearch.common.ssl.X509Field;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.env.Environment;
 
@@ -25,6 +26,7 @@
 import java.security.KeyStore;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -111,13 +113,18 @@ protected char[] getSecureSetting(String key) {
     }
 
     @Override
-    protected SslTrustConfig buildTrustConfig(Path basePath, SslVerificationMode verificationMode, SslKeyConfig keyConfig) {
-        final SslTrustConfig trustConfig = super.buildTrustConfig(basePath, verificationMode, keyConfig);
+    protected SslTrustConfig buildTrustConfig(
+        Path basePath,
+        SslVerificationMode verificationMode,
+        SslKeyConfig keyConfig,
+        Set<X509Field> restrictedTrustFields
+    ) {
+        final SslTrustConfig trustConfig = super.buildTrustConfig(basePath, verificationMode, keyConfig, null);
         final Path trustRestrictions = super.resolvePath("trust_restrictions.path", basePath);
         if (trustRestrictions == null) {
             return trustConfig;
         }
-        return new RestrictedTrustConfig(trustRestrictions, trustConfig);
+        return new RestrictedTrustConfig(trustRestrictions, restrictedTrustFields, trustConfig);
     }
 
     public SslConfiguration load(Environment env) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/cert/CertificateInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/cert/CertificateInfo.java
index 6a6c30a95a53..3588d21110bc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/cert/CertificateInfo.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/cert/CertificateInfo.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ssl.cert;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -56,7 +56,7 @@ public CertificateInfo(String path, String format, String alias, boolean hasPriv
     }
 
     public CertificateInfo(StreamInput in) throws IOException {
-        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             this.path = in.readOptionalString();
         } else {
             this.path = in.readString();
@@ -67,7 +67,7 @@ public CertificateInfo(StreamInput in) throws IOException {
         this.serialNumber = in.readString();
         this.hasPrivateKey = in.readBoolean();
         this.expiry = Instant.ofEpochMilli(in.readLong()).atZone(ZoneOffset.UTC);
-        if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             this.issuer = in.readString();
         } else {
             this.issuer = "";
@@ -76,7 +76,7 @@ public CertificateInfo(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             out.writeOptionalString(this.path);
         } else {
             out.writeString(this.path == null ? "" : this.path);
@@ -87,7 +87,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(serialNumber);
         out.writeBoolean(hasPrivateKey);
         out.writeLong(expiry.toInstant().toEpochMilli());
-        if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             out.writeString(issuer);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumRequest.java
index 5fc0ee5b4cac..40de316c8c61 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumRequest.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.termsenum.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -78,7 +78,7 @@ public NodeTermsEnumRequest(StreamInput in) throws IOException {
         for (int i = 0; i < numShards; i++) {
             shardIds.add(new ShardId(in));
         }
-        if (in.getVersion().onOrAfter(Version.V_7_15_1)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_15_1)) {
             originalIndices = OriginalIndices.readOriginalIndices(in);
         } else {
             String[] indicesNames = shardIds.stream().map(ShardId::getIndexName).distinct().toArray(String[]::new);
@@ -106,7 +106,7 @@ public void writeTo(StreamOutput out) throws IOException {
         for (ShardId shardId : shardIds) {
             shardId.writeTo(out);
         }
-        if (out.getVersion().onOrAfter(Version.V_7_15_1)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_15_1)) {
             OriginalIndices.writeOriginalIndices(originalIndices, out);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumResponse.java
index 710e64485b54..53b44e937804 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumResponse.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.termsenum.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -31,7 +31,7 @@ class NodeTermsEnumResponse extends TransportResponse {
 
     NodeTermsEnumResponse(StreamInput in) throws IOException {
         super(in);
-        if (in.getVersion().before(Version.V_8_2_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_2_0)) {
             terms = in.readList(r -> {
                 String term = r.readString();
                 in.readLong(); // obsolete docCount field
@@ -54,7 +54,7 @@ class NodeTermsEnumResponse extends TransportResponse {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getVersion().before(Version.V_8_2_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_2_0)) {
             out.writeCollection(terms.stream().map(term -> (Writeable) out1 -> {
                 out1.writeString(term);
                 out1.writeLong(1); // obsolete docCount field
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/textstructure/action/FindStructureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/textstructure/action/FindStructureAction.java
index 431a830e5f2b..dab0709e574e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/textstructure/action/FindStructureAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/textstructure/action/FindStructureAction.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.textstructure.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
@@ -148,7 +148,7 @@ public Request(StreamInput in) throws IOException {
             quote = in.readBoolean() ? (char) in.readVInt() : null;
             shouldTrimFields = in.readOptionalBoolean();
             grokPattern = in.readOptionalString();
-            if (in.getVersion().onOrAfter(Version.V_8_5_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_5_0)) {
                 ecsCompatibility = in.readOptionalString();
             } else {
                 ecsCompatibility = null;
@@ -410,7 +410,7 @@ public void writeTo(StreamOutput out) throws IOException {
             }
             out.writeOptionalBoolean(shouldTrimFields);
             out.writeOptionalString(grokPattern);
-            if (out.getVersion().onOrAfter(Version.V_8_5_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_5_0)) {
                 out.writeOptionalString(ecsCompatibility);
             }
             out.writeOptionalString(timestampFormat);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/textstructure/structurefinder/TextStructure.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/textstructure/structurefinder/TextStructure.java
index d67303d71ccf..fc8c4a491661 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/textstructure/structurefinder/TextStructure.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/textstructure/structurefinder/TextStructure.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.textstructure.structurefinder;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -224,7 +224,7 @@ public TextStructure(StreamInput in) throws IOException {
         quote = in.readBoolean() ? (char) in.readVInt() : null;
         shouldTrimFields = in.readOptionalBoolean();
         grokPattern = in.readOptionalString();
-        if (in.getVersion().onOrAfter(Version.V_8_5_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_5_0)) {
             ecsCompatibility = getNonNullEcsCompatibilityString(in.readString());
         } else {
             ecsCompatibility = getNonNullEcsCompatibilityString(null);
@@ -270,7 +270,7 @@ public void writeTo(StreamOutput out) throws IOException {
         }
         out.writeOptionalBoolean(shouldTrimFields);
         out.writeOptionalString(grokPattern);
-        if (out.getVersion().onOrAfter(Version.V_8_5_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_5_0)) {
             out.writeString(ecsCompatibility);
         }
         if (jodaTimestampFormats == null) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsage.java
index b81b173c0d70..0941c4667bb7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsage.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.transform;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -48,8 +48,8 @@ public TransformFeatureSetUsage(
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_5_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_5_0;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java
index f100bac64525..709ec58935f5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java
@@ -22,6 +22,7 @@ public final class TransformField {
     public static final ParseField COUNT = new ParseField("count");
     public static final ParseField GROUP_BY = new ParseField("group_by");
     public static final ParseField TIMEOUT = new ParseField("timeout");
+    public static final ParseField FROM = new ParseField("from");
     public static final ParseField WAIT_FOR_COMPLETION = new ParseField("wait_for_completion");
     public static final ParseField WAIT_FOR_CHECKPOINT = new ParseField("wait_for_checkpoint");
     public static final ParseField STATS_FIELD = new ParseField("stats");
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMessages.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMessages.java
index 957349f7b9f3..5dd36d648f7b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMessages.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMessages.java
@@ -75,6 +75,8 @@ public class TransformMessages {
 
     public static final String FAILED_TO_PARSE_TRANSFORM_CHECKPOINTS = "Failed to parse transform checkpoints for [{0}]";
 
+    public static final String FAILED_TO_PARSE_DATE = "Failed to parse date for [{0}]";
+
     public static final String ID_TOO_LONG = "The id cannot contain more than {0} characters.";
     public static final String INVALID_ID = "Invalid {0}; ''{1}'' can contain lowercase alphanumeric (a-z and 0-9), hyphens or "
         + "underscores; must start and end with alphanumeric";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMetadata.java
index 5a3df91cd25a..eed093ed725f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMetadata.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.transform;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.NamedDiff;
@@ -15,14 +15,15 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.EnumSet;
+import java.util.Iterator;
 import java.util.Objects;
 
 public class TransformMetadata implements Metadata.Custom {
@@ -52,8 +53,8 @@ public boolean isResetMode() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.CURRENT.minimumCompatibilityVersion();
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.CURRENT.minimumCompatibilityVersion();
     }
 
     @Override
@@ -81,9 +82,8 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
-        builder.field(RESET_MODE.getPreferredName(), resetMode);
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return ChunkedToXContentHelper.field(RESET_MODE.getPreferredName(), resetMode);
     }
 
     public static class TransformMetadataDiff implements NamedDiff<Metadata.Custom> {
@@ -119,8 +119,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT.minimumCompatibilityVersion();
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT.minimumCompatibilityVersion();
         }
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetTransformAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetTransformAction.java
index f2bf8224abee..aed2daf0c2a4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetTransformAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetTransformAction.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.transform.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.ValidationException;
@@ -136,7 +136,7 @@ public Response(List<TransformConfig> transformConfigs, long count, List<Error>
 
         public Response(StreamInput in) throws IOException {
             super(in);
-            if (in.getVersion().onOrAfter(Version.V_8_1_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
                 if (in.readBoolean()) {
                     this.errors = in.readList(Error::new);
                 } else {
@@ -197,7 +197,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            if (out.getVersion().onOrAfter(Version.V_8_1_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
                 if (errors != null) {
                     out.writeBoolean(true);
                     out.writeList(errors);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetTransformStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetTransformStatsAction.java
index e961ffb9619c..f2470c3829ca 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetTransformStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetTransformStatsAction.java
@@ -17,7 +17,11 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.action.util.PageParams;
@@ -29,9 +33,11 @@
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
+import static org.elasticsearch.core.Strings.format;
 
 public class GetTransformStatsAction extends ActionType<GetTransformStatsAction.Response> {
 
@@ -51,7 +57,8 @@ public static class Request extends BaseTasksRequest<Request> {
         // used internally to expand the queried id expression
         private List<String> expandedIds;
 
-        public Request(String id) {
+        public Request(String id, @Nullable TimeValue timeout) {
+            setTimeout(timeout);
             if (Strings.isNullOrEmpty(id) || id.equals("*")) {
                 this.id = Metadata.ALL;
             } else {
@@ -140,6 +147,11 @@ public boolean equals(Object obj) {
             Request other = (Request) obj;
             return Objects.equals(id, other.id) && Objects.equals(pageParams, other.pageParams) && allowNoMatch == other.allowNoMatch;
         }
+
+        @Override
+        public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+            return new CancellableTask(id, type, action, format("get_transform_stats[%s]", this.id), parentTaskId, headers);
+        }
     }
 
     public static class Response extends BaseTasksResponse implements ToXContentObject {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/ScheduleNowTransformAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/ScheduleNowTransformAction.java
new file mode 100644
index 000000000000..96d3f498e8de
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/ScheduleNowTransformAction.java
@@ -0,0 +1,156 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.transform.action;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.TaskOperationFailure;
+import org.elasticsearch.action.support.tasks.BaseTasksRequest;
+import org.elasticsearch.action.support.tasks.BaseTasksResponse;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.transform.TransformField;
+import org.elasticsearch.xpack.core.transform.utils.ExceptionsHelper;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+public class ScheduleNowTransformAction extends ActionType<ScheduleNowTransformAction.Response> {
+
+    public static final ScheduleNowTransformAction INSTANCE = new ScheduleNowTransformAction();
+    public static final String NAME = "cluster:admin/transform/schedule_now";
+
+    private ScheduleNowTransformAction() {
+        super(NAME, ScheduleNowTransformAction.Response::new);
+    }
+
+    public static class Request extends BaseTasksRequest<Request> {
+
+        private final String id;
+
+        public Request(String id, TimeValue timeout) {
+            this.id = ExceptionsHelper.requireNonNull(id, TransformField.ID.getPreferredName());
+            this.setTimeout(ExceptionsHelper.requireNonNull(timeout, TransformField.TIMEOUT.getPreferredName()));
+        }
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            this.id = in.readString();
+        }
+
+        public static Request fromXContent(final String id, final TimeValue timeout) {
+            return new Request(id, timeout);
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            if (Metadata.ALL.equals(id)) {
+                ActionRequestValidationException e = new ActionRequestValidationException();
+                e.addValidationError("_schedule_now API does not support _all wildcard");
+                return e;
+            }
+            return null;
+        }
+
+        public String getId() {
+            return id;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(id);
+        }
+
+        @Override
+        public int hashCode() {
+            // the base class does not implement hashCode, therefore we need to hash timeout ourselves
+            return Objects.hash(getTimeout(), id);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            Request other = (Request) obj;
+
+            // the base class does not implement equals, therefore we need to check timeout ourselves
+            return this.id.equals(other.id) && getTimeout().equals(other.getTimeout());
+        }
+    }
+
+    public static class Response extends BaseTasksResponse implements Writeable, ToXContentObject {
+
+        public static final Response TRUE = new Response(true);
+
+        private final boolean acknowledged;
+
+        public Response(StreamInput in) throws IOException {
+            super(in);
+            acknowledged = in.readBoolean();
+        }
+
+        public Response(boolean acknowledged) {
+            super(Collections.emptyList(), Collections.emptyList());
+            this.acknowledged = acknowledged;
+        }
+
+        public Response(
+            List<TaskOperationFailure> taskFailures,
+            List<? extends ElasticsearchException> nodeFailures,
+            boolean acknowledged
+        ) {
+            super(taskFailures, nodeFailures);
+            this.acknowledged = acknowledged;
+        }
+
+        public boolean isAcknowledged() {
+            return acknowledged;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeBoolean(acknowledged);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            toXContentCommon(builder, params);
+            builder.field("acknowledged", acknowledged);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            ScheduleNowTransformAction.Response response = (ScheduleNowTransformAction.Response) o;
+            return acknowledged == response.acknowledged;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(acknowledged);
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/StartTransformAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/StartTransformAction.java
index 8979c16c234a..12979a38f860 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/StartTransformAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/StartTransformAction.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.core.transform.action;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
@@ -20,6 +21,7 @@
 import org.elasticsearch.xpack.core.transform.utils.ExceptionsHelper;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.util.Collections;
 import java.util.Objects;
 
@@ -35,25 +37,39 @@ private StartTransformAction() {
     public static class Request extends AcknowledgedRequest<Request> {
 
         private final String id;
+        private final Instant from;
 
-        public Request(String id, TimeValue timeout) {
+        public Request(String id, Instant from, TimeValue timeout) {
             super(timeout);
             this.id = ExceptionsHelper.requireNonNull(id, TransformField.ID.getPreferredName());
+            this.from = from;
         }
 
         public Request(StreamInput in) throws IOException {
             super(in);
             id = in.readString();
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+                from = in.readOptionalInstant();
+            } else {
+                from = null;
+            }
         }
 
         public String getId() {
             return id;
         }
 
+        public Instant from() {
+            return from;
+        }
+
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             out.writeString(id);
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+                out.writeOptionalInstant(from);
+            }
         }
 
         @Override
@@ -64,7 +80,7 @@ public ActionRequestValidationException validate() {
         @Override
         public int hashCode() {
             // the base class does not implement hashCode, therefore we need to hash timeout ourselves
-            return Objects.hash(timeout(), id);
+            return Objects.hash(timeout(), id, from);
         }
 
         @Override
@@ -77,7 +93,7 @@ public boolean equals(Object obj) {
             }
             Request other = (Request) obj;
             // the base class does not implement equals, therefore we need to check timeout ourselves
-            return Objects.equals(id, other.id) && timeout().equals(other.timeout());
+            return Objects.equals(id, other.id) && Objects.equals(from, other.from) && timeout().equals(other.timeout());
         }
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java
index 0bfec00f84fb..441882cd53f9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.transform.transforms;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -163,17 +163,17 @@ public SettingsConfig(final StreamInput in) throws IOException {
         this.alignCheckpoints = in.readOptionalInt();
         this.usePit = in.readOptionalInt();
 
-        if (in.getVersion().onOrAfter(Version.V_8_1_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
             deduceMappings = in.readOptionalInt();
         } else {
             deduceMappings = DEFAULT_DEDUCE_MAPPINGS;
         }
-        if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             numFailureRetries = in.readOptionalInt();
         } else {
             numFailureRetries = null;
         }
-        if (in.getVersion().onOrAfter(Version.V_8_5_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_5_0)) {
             unattended = in.readOptionalInt();
         } else {
             unattended = DEFAULT_UNATTENDED;
@@ -279,13 +279,13 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalInt(alignCheckpoints);
         out.writeOptionalInt(usePit);
 
-        if (out.getVersion().onOrAfter(Version.V_8_1_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_1_0)) {
             out.writeOptionalInt(deduceMappings);
         }
-        if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_4_0)) {
             out.writeOptionalInt(numFailureRetries);
         }
-        if (out.getVersion().onOrAfter(Version.V_8_5_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_5_0)) {
             out.writeOptionalInt(unattended);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpoint.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpoint.java
index 14d4f4df4759..e6e73fec5893 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpoint.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpoint.java
@@ -46,7 +46,12 @@
  */
 public class TransformCheckpoint implements Writeable, ToXContentObject {
 
-    public static TransformCheckpoint EMPTY = new TransformCheckpoint("empty", 0L, -1L, Collections.emptyMap(), 0L);
+    public static String EMPTY_NAME = "_empty";
+    public static TransformCheckpoint EMPTY = createEmpty(0);
+
+    public static TransformCheckpoint createEmpty(long timestampMillis) {
+        return new TransformCheckpoint(EMPTY_NAME, timestampMillis, -1L, Collections.emptyMap(), timestampMillis);
+    }
 
     // the own checkpoint
     public static final ParseField CHECKPOINT = new ParseField("checkpoint");
@@ -128,7 +133,7 @@ public TransformCheckpoint(StreamInput in) throws IOException {
     }
 
     public boolean isEmpty() {
-        return this.equals(EMPTY);
+        return EMPTY_NAME.equals(transformId) && checkpoint == -1;
     }
 
     /**
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformState.java
index 751c0a77db42..50f4ba56aaab 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformState.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformState.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.transform.transforms;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -146,7 +146,7 @@ public TransformState(
     public TransformState(StreamInput in) throws IOException {
         taskState = TransformTaskState.fromStream(in);
         indexerState = IndexerState.fromStream(in);
-        if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_3_0)) {
             position = in.readOptionalWriteable(TransformIndexerPosition::new);
         } else {
             Map<String, Object> pos = in.readMap();
@@ -155,12 +155,12 @@ public TransformState(StreamInput in) throws IOException {
         checkpoint = in.readLong();
         reason = in.readOptionalString();
         progress = in.readOptionalWriteable(TransformProgress::new);
-        if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_3_0)) {
             node = in.readOptionalWriteable(NodeAttributes::new);
         } else {
             node = null;
         }
-        if (in.getVersion().onOrAfter(Version.V_7_6_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_6_0)) {
             shouldStopAtNextCheckpoint = in.readBoolean();
         } else {
             shouldStopAtNextCheckpoint = false;
@@ -244,7 +244,7 @@ public String getWriteableName() {
     public void writeTo(StreamOutput out) throws IOException {
         taskState.writeTo(out);
         indexerState.writeTo(out);
-        if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_3_0)) {
             out.writeOptionalWriteable(position);
         } else {
             out.writeGenericMap(position != null ? position.getIndexerPosition() : null);
@@ -252,10 +252,10 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeLong(checkpoint);
         out.writeOptionalString(reason);
         out.writeOptionalWriteable(progress);
-        if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_3_0)) {
             out.writeOptionalWriteable(node);
         }
-        if (out.getVersion().onOrAfter(Version.V_7_6_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_6_0)) {
             out.writeBoolean(shouldStopAtNextCheckpoint);
         }
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformStats.java
index cd003247f6b0..d79c9219ead5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformStats.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.transform.transforms;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -94,7 +94,7 @@ public TransformStats(StreamInput in) throws IOException {
         this.indexerStats = new TransformIndexerStats(in);
         this.checkpointingInfo = new TransformCheckpointingInfo(in);
 
-        if (in.getVersion().onOrAfter(Version.V_8_6_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_6_0)) {
             if (in.readBoolean()) {
                 this.health = new TransformHealth(in);
             } else {
@@ -138,7 +138,7 @@ public void writeTo(StreamOutput out) throws IOException {
         }
         indexerStats.writeTo(out);
         checkpointingInfo.writeTo(out);
-        if (out.getVersion().onOrAfter(Version.V_8_6_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_6_0)) {
             if (health != null) {
                 out.writeBoolean(true);
                 health.writeTo(out);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformTaskParams.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformTaskParams.java
index 089165a4aa71..06b631c262ba 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformTaskParams.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformTaskParams.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.core.transform.transforms;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.SimpleDiffable;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -20,44 +21,54 @@
 import org.elasticsearch.xpack.core.transform.TransformField;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.util.Objects;
 
 public class TransformTaskParams implements SimpleDiffable<TransformTaskParams>, PersistentTaskParams {
 
     public static final String NAME = TransformField.TASK_NAME;
+    public static final ParseField FROM = TransformField.FROM;
     public static final ParseField FREQUENCY = TransformField.FREQUENCY;
     public static final ParseField REQUIRES_REMOTE = new ParseField("requires_remote");
 
     private final String transformId;
     private final Version version;
+    private final Instant from;
     private final TimeValue frequency;
     private final Boolean requiresRemote;
 
     public static final ConstructingObjectParser<TransformTaskParams, Void> PARSER = new ConstructingObjectParser<>(
         NAME,
         true,
-        a -> new TransformTaskParams((String) a[0], (String) a[1], (String) a[2], (Boolean) a[3])
+        a -> new TransformTaskParams((String) a[0], (String) a[1], (Long) a[2], (String) a[3], (Boolean) a[4])
     );
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), TransformField.ID);
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TransformField.VERSION);
+        PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), FROM);
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), FREQUENCY);
         PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), REQUIRES_REMOTE);
     }
 
-    private TransformTaskParams(String transformId, String version, String frequency, Boolean remote) {
+    private TransformTaskParams(String transformId, String version, Long from, String frequency, Boolean remote) {
         this(
             transformId,
             version == null ? null : Version.fromString(version),
+            from == null ? null : Instant.ofEpochMilli(from),
             frequency == null ? null : TimeValue.parseTimeValue(frequency, FREQUENCY.getPreferredName()),
             remote == null ? false : remote.booleanValue()
         );
     }
 
     public TransformTaskParams(String transformId, Version version, TimeValue frequency, boolean remote) {
+        this(transformId, version, null, frequency, remote);
+    }
+
+    public TransformTaskParams(String transformId, Version version, Instant from, TimeValue frequency, boolean remote) {
         this.transformId = transformId;
         this.version = version == null ? Version.V_7_2_0 : version;
+        this.from = from;
         this.frequency = frequency;
         this.requiresRemote = remote;
     }
@@ -65,6 +76,11 @@ public TransformTaskParams(String transformId, Version version, TimeValue freque
     public TransformTaskParams(StreamInput in) throws IOException {
         this.transformId = in.readString();
         this.version = Version.readVersion(in);
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            this.from = in.readOptionalInstant();
+        } else {
+            this.from = null;
+        }
         this.frequency = in.readOptionalTimeValue();
         this.requiresRemote = in.readBoolean();
     }
@@ -75,14 +91,17 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_17_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_17_0;
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(transformId);
         Version.writeVersion(version, out);
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            out.writeOptionalInstant(from);
+        }
         out.writeOptionalTimeValue(frequency);
         out.writeBoolean(requiresRemote);
     }
@@ -92,6 +111,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         builder.field(TransformField.ID.getPreferredName(), transformId);
         builder.field(TransformField.VERSION.getPreferredName(), version);
+        if (from != null) {
+            builder.field(FROM.getPreferredName(), from.toEpochMilli());
+        }
         if (frequency != null) {
             builder.field(FREQUENCY.getPreferredName(), frequency.getStringRep());
         }
@@ -108,6 +130,10 @@ public Version getVersion() {
         return version;
     }
 
+    public Instant from() {
+        return from;
+    }
+
     public TimeValue getFrequency() {
         return frequency;
     }
@@ -134,12 +160,13 @@ public boolean equals(Object other) {
 
         return Objects.equals(this.transformId, that.transformId)
             && Objects.equals(this.version, that.version)
+            && Objects.equals(this.from, that.from)
             && Objects.equals(this.frequency, that.frequency)
             && this.requiresRemote == that.requiresRemote;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(transformId, version, frequency, requiresRemote);
+        return Objects.hash(transformId, version, from, frequency, requiresRemote);
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/DateHistogramGroupSource.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/DateHistogramGroupSource.java
index 7a375b56ab3f..b36a6a447b6b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/DateHistogramGroupSource.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/DateHistogramGroupSource.java
@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.xpack.core.transform.transforms.pivot;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Rounding;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -13,6 +14,7 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
+import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -197,6 +199,7 @@ private void writeInterval(Interval anInterval, StreamOutput out) throws IOExcep
 
     private static final String NAME = "data_frame_date_histogram_group";
     private static final ParseField TIME_ZONE = new ParseField("time_zone");
+    private static final ParseField OFFSET = Histogram.OFFSET_FIELD;
 
     private static final ConstructingObjectParser<DateHistogramGroupSource, Void> STRICT_PARSER = createParser(false);
     private static final ConstructingObjectParser<DateHistogramGroupSource, Void> LENIENT_PARSER = createParser(true);
@@ -204,19 +207,33 @@ private void writeInterval(Interval anInterval, StreamOutput out) throws IOExcep
     private final Interval interval;
     private final ZoneId timeZone;
     private final Rounding.Prepared rounding;
-
-    public DateHistogramGroupSource(String field, ScriptConfig scriptConfig, boolean missingBucket, Interval interval, ZoneId timeZone) {
+    private final long offset;
+
+    public DateHistogramGroupSource(
+        String field,
+        ScriptConfig scriptConfig,
+        boolean missingBucket,
+        Interval interval,
+        ZoneId timeZone,
+        Long offset
+    ) {
         super(field, scriptConfig, missingBucket);
         this.interval = interval;
         this.timeZone = timeZone;
-        rounding = buildRounding();
+        this.offset = offset != null ? offset : 0;
+        this.rounding = buildRounding();
     }
 
     public DateHistogramGroupSource(StreamInput in) throws IOException {
         super(in);
         this.interval = readInterval(in);
         this.timeZone = in.readOptionalZoneId();
-        rounding = buildRounding();
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            this.offset = in.readLong();
+        } else {
+            this.offset = 0;
+        }
+        this.rounding = buildRounding();
     }
 
     private Rounding.Prepared buildRounding() {
@@ -231,6 +248,7 @@ private Rounding.Prepared buildRounding() {
         if (timeZone != null) {
             roundingBuilder.timeZone(timeZone);
         }
+        roundingBuilder.offset(offset);
         return roundingBuilder.build().prepareForUnknown();
     }
 
@@ -242,6 +260,7 @@ private static ConstructingObjectParser<DateHistogramGroupSource, Void> createPa
             String fixedInterval = (String) args[3];
             String calendarInterval = (String) args[4];
             ZoneId zoneId = (ZoneId) args[5];
+            Long offset = (Long) args[6];
 
             Interval interval = null;
 
@@ -255,7 +274,7 @@ private static ConstructingObjectParser<DateHistogramGroupSource, Void> createPa
                 throw new IllegalArgumentException("You must specify either fixed_interval or calendar_interval, found none");
             }
 
-            return new DateHistogramGroupSource(field, scriptConfig, missingBucket, interval, zoneId);
+            return new DateHistogramGroupSource(field, scriptConfig, missingBucket, interval, zoneId, offset);
         });
 
         declareValuesSourceFields(parser, lenient);
@@ -271,6 +290,14 @@ private static ConstructingObjectParser<DateHistogramGroupSource, Void> createPa
             }
         }, TIME_ZONE, ObjectParser.ValueType.LONG);
 
+        parser.declareField(optionalConstructorArg(), p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_NUMBER) {
+                return p.longValue();
+            } else {
+                return DateHistogramAggregationBuilder.parseStringOffset(p.text());
+            }
+        }, OFFSET, ObjectParser.ValueType.LONG);
+
         return parser;
     }
 
@@ -295,11 +322,18 @@ public Rounding.Prepared getRounding() {
         return rounding;
     }
 
+    public long getOffset() {
+        return offset;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         writeInterval(interval, out);
         out.writeOptionalZoneId(timeZone);
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            out.writeLong(offset);
+        }
     }
 
     @Override
@@ -310,6 +344,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (timeZone != null) {
             builder.field(TIME_ZONE.getPreferredName(), timeZone.toString());
         }
+        if (offset != 0) {
+            builder.field(OFFSET.getPreferredName(), offset);
+        }
         builder.endObject();
         return builder;
     }
@@ -330,11 +367,12 @@ public boolean equals(Object other) {
             && Objects.equals(this.field, that.field)
             && Objects.equals(this.scriptConfig, that.scriptConfig)
             && Objects.equals(this.interval, that.interval)
-            && Objects.equals(this.timeZone, that.timeZone);
+            && Objects.equals(this.timeZone, that.timeZone)
+            && this.offset == that.offset;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(field, scriptConfig, missingBucket, interval, timeZone);
+        return Objects.hash(field, scriptConfig, missingBucket, interval, timeZone, offset);
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/votingonly/VotingOnlyNodeFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/votingonly/VotingOnlyNodeFeatureSetUsage.java
index 8698c82fc2a5..78ca1035b57a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/votingonly/VotingOnlyNodeFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/votingonly/VotingOnlyNodeFeatureSetUsage.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.votingonly;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
 import org.elasticsearch.xpack.core.XPackField;
@@ -23,8 +23,8 @@ public VotingOnlyNodeFeatureSetUsage() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_3_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_3_0;
     }
 
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/WatcherFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/WatcherFeatureSetUsage.java
index ab3a09bbce57..b7db037fde49 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/WatcherFeatureSetUsage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/WatcherFeatureSetUsage.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.watcher;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -31,8 +31,8 @@ public WatcherFeatureSetUsage(boolean available, boolean enabled, Map<String, Ob
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_0_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_0_0;
     }
 
     public Map<String, Object> stats() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/WatcherMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/WatcherMetadata.java
index ea3e3b4df25b..de78d8604717 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/WatcherMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/WatcherMetadata.java
@@ -6,18 +6,20 @@
  */
 package org.elasticsearch.xpack.core.watcher;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.AbstractNamedDiffable;
 import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.EnumSet;
+import java.util.Iterator;
 import java.util.Objects;
 
 public class WatcherMetadata extends AbstractNamedDiffable<Metadata.Custom> implements Metadata.Custom {
@@ -40,8 +42,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.CURRENT.minimumCompatibilityVersion();
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.CURRENT.minimumCompatibilityVersion();
     }
 
     @Override
@@ -105,9 +107,8 @@ public static Metadata.Custom fromXContent(XContentParser parser) throws IOExcep
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(Field.MANUALLY_STOPPED.getPreferredName(), manuallyStopped);
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return ChunkedToXContentHelper.field(Field.MANUALLY_STOPPED.getPreferredName(), manuallyStopped);
     }
 
     interface Field {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/crypto/CryptoService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/crypto/CryptoService.java
index 7dde4f7270ad..c65710d38fd3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/crypto/CryptoService.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/crypto/CryptoService.java
@@ -9,11 +9,11 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.CharArrays;
+import org.elasticsearch.core.Streams;
 import org.elasticsearch.xpack.core.security.SecurityField;
 import org.elasticsearch.xpack.core.watcher.WatcherField;
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/stats/WatcherStatsResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/stats/WatcherStatsResponse.java
index 7a12b5803b1d..c8838875cc2d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/stats/WatcherStatsResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/stats/WatcherStatsResponse.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -28,7 +29,7 @@
 
 public class WatcherStatsResponse extends BaseNodesResponse<WatcherStatsResponse.Node> implements ToXContentObject {
 
-    private WatcherMetadata watcherMetadata;
+    private final WatcherMetadata watcherMetadata;
 
     public WatcherStatsResponse(StreamInput in) throws IOException {
         super(in);
@@ -63,7 +64,7 @@ protected void writeNodesTo(StreamOutput out, List<Node> nodes) throws IOExcepti
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        watcherMetadata.toXContent(builder, params);
+        ChunkedToXContent.wrapAsToXContent(watcherMetadata).toXContent(builder, params);
         builder.startArray("stats");
         for (Node node : getNodes()) {
             node.toXContent(builder, params);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/Payload.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/Payload.java
index efe22fcab9d7..46803619f562 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/Payload.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/Payload.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.core.watcher.watch;
 
 import org.elasticsearch.common.collect.MapBuilder;
+import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -37,6 +38,7 @@ public Simple(String key, Object value) {
         }
 
         public Simple(Map<String, Object> data) {
+            CollectionUtils.ensureNoSelfReferences(data, "watcher action payload");
             this.data = data;
         }
 
diff --git a/x-pack/plugin/core/src/main/resources/fleet-actions.json b/x-pack/plugin/core/src/main/resources/fleet-actions.json
index fc9fc8768a77..2f41bfabb613 100644
--- a/x-pack/plugin/core/src/main/resources/fleet-actions.json
+++ b/x-pack/plugin/core/src/main/resources/fleet-actions.json
@@ -28,6 +28,9 @@
         "minimum_execution_duration": {
           "type": "integer"
         },
+        "rollout_duration_seconds": {
+          "type": "integer"
+        },
         "input_type": {
           "type": "keyword"
         },
diff --git a/x-pack/plugin/core/src/main/resources/fleet-file-data-ilm-policy.json b/x-pack/plugin/core/src/main/resources/fleet-file-data-ilm-policy.json
new file mode 100644
index 000000000000..7ea1aff57d7e
--- /dev/null
+++ b/x-pack/plugin/core/src/main/resources/fleet-file-data-ilm-policy.json
@@ -0,0 +1,25 @@
+{
+  "phases": {
+    "hot": {
+      "min_age": "0ms",
+      "actions": {
+        "rollover": {
+          "max_size": "10gb",
+          "max_age": "7d"
+        }
+      }
+    },
+    "delete": {
+      "min_age": "7d",
+      "actions": {
+        "delete": {
+          "delete_searchable_snapshot": true
+        }
+      }
+    }
+  },
+  "_meta": {
+    "description": "policy for fleet uploaded files",
+    "managed": true
+  }
+}
\ No newline at end of file
diff --git a/x-pack/plugin/core/src/main/resources/fleet-file-data.json b/x-pack/plugin/core/src/main/resources/fleet-file-data.json
new file mode 100644
index 000000000000..413872f6c0b2
--- /dev/null
+++ b/x-pack/plugin/core/src/main/resources/fleet-file-data.json
@@ -0,0 +1,41 @@
+{
+  "index_patterns": [
+    ".fleet-file-data-*-*"
+  ],
+  "priority": 200,
+  "composed_of": [],
+  "_meta": {
+    "description": "fleet file data index template",
+    "managed": true
+  },
+  "template" : {
+    "settings": {
+      "index.lifecycle.name": ".fleet-file-data-ilm-policy",
+      "auto_expand_replicas": "0-1"
+    },
+    "mappings": {
+      "_doc": {
+        "_meta": {
+          "version": "${xpack.fleet.template.version}"
+        },
+        "properties": {
+          "data": {
+            "type": "binary"
+          },
+          "bid": {
+            "type": "keyword"
+          },
+          "sha2": {
+            "type": "keyword",
+            "index": false
+          },
+          "last": {
+            "type": "boolean",
+            "index": false
+          }
+        }
+      }
+    }
+  },
+  "version": ${xpack.fleet.template.version}
+}
\ No newline at end of file
diff --git a/x-pack/plugin/core/src/main/resources/fleet-files-ilm-policy.json b/x-pack/plugin/core/src/main/resources/fleet-files-ilm-policy.json
new file mode 100644
index 000000000000..98722614ebc2
--- /dev/null
+++ b/x-pack/plugin/core/src/main/resources/fleet-files-ilm-policy.json
@@ -0,0 +1,25 @@
+{
+  "phases": {
+    "hot": {
+      "min_age": "0ms",
+      "actions": {
+        "rollover": {
+          "max_size": "10gb",
+          "max_age": "30d"
+        }
+      }
+    },
+    "delete": {
+      "min_age": "90d",
+      "actions": {
+        "delete": {
+          "delete_searchable_snapshot": true
+        }
+      }
+    }
+  },
+  "_meta": {
+    "description": "policy for fleet uploaded file metadata",
+    "managed": true
+  }
+}
\ No newline at end of file
diff --git a/x-pack/plugin/core/src/main/resources/fleet-files.json b/x-pack/plugin/core/src/main/resources/fleet-files.json
new file mode 100644
index 000000000000..2aa75d78ca05
--- /dev/null
+++ b/x-pack/plugin/core/src/main/resources/fleet-files.json
@@ -0,0 +1,59 @@
+{
+  "index_patterns": [
+    ".fleet-files-*-*"
+  ],
+  "priority": 200,
+  "composed_of": [],
+  "_meta": {
+    "description": "fleet files index template",
+    "managed": true
+  },
+  "template": {
+    "settings": {
+      "index.lifecycle.name": ".fleet-files-ilm-policy",
+      "index.auto_expand_replicas": "0-1"
+    },
+    "mappings": {
+      "_doc": {
+        "_meta": {
+          "version": "${xpack.fleet.template.version}"
+        },
+        "dynamic": false,
+        "properties": {
+          "upload_start": {
+            "type": "date"
+          },
+          "agent_id": {
+            "type": "keyword"
+          },
+          "action_id": {
+            "type": "keyword"
+          },
+          "source": {
+            "type": "keyword"
+          },
+          "upload_id": {
+            "type": "keyword"
+          },
+          "file": {
+            "properties": {
+              "Status": {
+                "type": "keyword"
+              },
+              "ChunkSize": {
+                "type": "integer"
+              },
+              "Compression": {
+                "type": "keyword"
+              },
+              "name": {
+                "type": "keyword"
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "version": ${xpack.fleet.template.version}
+}
\ No newline at end of file
diff --git a/x-pack/plugin/core/src/main/resources/monitoring-es-mb.json b/x-pack/plugin/core/src/main/resources/monitoring-es-mb.json
index d64a24f07389..89fc282c89e1 100644
--- a/x-pack/plugin/core/src/main/resources/monitoring-es-mb.json
+++ b/x-pack/plugin/core/src/main/resources/monitoring-es-mb.json
@@ -305,6 +305,10 @@
                     }
                   }
                 },
+                "roles": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
                 "version": {
                   "ignore_above": 1024,
                   "type": "keyword"
@@ -1860,6 +1864,74 @@
                 }
               }
             },
+            "ingest_pipeline": {
+              "properties": {
+                "name": {
+                  "ignore_above": 1024,
+                  "type": "wildcard"
+                },
+                "processor": {
+                  "properties": {
+                    "count": {
+                      "type": "long"
+                    },
+                    "failed": {
+                      "type": "long"
+                    },
+                    "order_index": {
+                      "type": "long"
+                    },
+                    "time": {
+                      "properties": {
+                        "total": {
+                          "properties": {
+                            "ms": {
+                              "type": "long"
+                            }
+                          }
+                        }
+                      }
+                    },
+                    "type": {
+                      "ignore_above": 1024,
+                      "type": "keyword"
+                    },
+                    "type_tag": {
+                      "ignore_above": 1024,
+                      "type": "keyword"
+                    }
+                  }
+                },
+                "total": {
+                  "properties": {
+                    "count": {
+                      "type": "long"
+                    },
+                    "failed": {
+                      "type": "long"
+                    },
+                    "time": {
+                      "properties": {
+                        "total": {
+                          "properties": {
+                            "ms": {
+                              "type": "long"
+                            }
+                          }
+                        },
+                        "self": {
+                          "properties": {
+                            "ms": {
+                              "type": "long"
+                            }
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            },
             "shard": {
               "properties": {
                 "number": {
diff --git a/x-pack/plugin/core/src/main/resources/monitoring-kibana-mb.json b/x-pack/plugin/core/src/main/resources/monitoring-kibana-mb.json
index ff8abc5d8995..cf7dffbc761d 100644
--- a/x-pack/plugin/core/src/main/resources/monitoring-kibana-mb.json
+++ b/x-pack/plugin/core/src/main/resources/monitoring-kibana-mb.json
@@ -219,6 +219,19 @@
                     }
                   }
                 },
+                "elasticsearch_client": {
+                  "properties": {
+                    "total_active_sockets": {
+                      "type": "integer"
+                    },
+                    "total_idle_sockets": {
+                      "type": "integer"
+                    },
+                    "total_queued_requests": {
+                      "type": "integer"
+                    }
+                  }
+                },
                 "kibana": {
                   "properties": {
                     "status": {
diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_mappings.json
index 01703315357d..fad3fa7b158f 100644
--- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_mappings.json
+++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_mappings.json
@@ -271,6 +271,9 @@
         },
         "incomplete_bucket_penalty": {
           "type": "boolean"
+        },
+        "multimodal_distribution": {
+          "type": "boolean"
         }
       }
     },
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseOperationModeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseOperationModeTests.java
index df1fc9f63d82..86f76b9ca1c4 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseOperationModeTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseOperationModeTests.java
@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.license;
 
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 
 import static org.elasticsearch.license.License.OperationMode;
@@ -59,7 +60,7 @@ public void testResolveUnknown() {
                 final License.LicenseType licenseType = License.LicenseType.resolve(type);
                 OperationMode.resolve(licenseType);
 
-                fail(formatted("[%s] should not be recognized as an operation mode", type));
+                fail(Strings.format("[%s] should not be recognized as an operation mode", type));
             } catch (IllegalArgumentException e) {
                 assertThat(e.getMessage(), equalTo("unknown license type [" + type + "]"));
             }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseTests.java
index 6d31034c0c2e..366db2d7e127 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicenseTests.java
@@ -252,7 +252,7 @@ public void testNotEnoughBytesFromXContent() throws Exception {
 
     public void testMalformedSignatureFromXContent() throws Exception {
 
-        String licenseString = formatted("""
+        String licenseString = Strings.format("""
             {
               "license": {
                 "uid": "4056779d-b823-4c12-a9cb-efa4a8d8c422",
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetadataSerializationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetadataSerializationTests.java
index bb0aa97a4895..bd792c89d9eb 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetadataSerializationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetadataSerializationTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.cluster.metadata.RepositoriesMetadata;
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -38,7 +39,7 @@ public void testXContentSerializationOneSignedLicense() throws Exception {
         XContentBuilder builder = XContentFactory.jsonBuilder();
         builder.startObject();
         builder.startObject("licenses");
-        licensesMetadata.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        ChunkedToXContent.wrapAsToXContent(licensesMetadata).toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
         builder.endObject();
         LicensesMetadata licensesMetadataFromXContent = getLicensesMetadataFromXContent(createParser(builder));
@@ -52,7 +53,7 @@ public void testXContentSerializationOneSignedLicenseWithUsedTrial() throws Exce
         XContentBuilder builder = XContentFactory.jsonBuilder();
         builder.startObject();
         builder.startObject("licenses");
-        licensesMetadata.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        ChunkedToXContent.wrapAsToXContent(licensesMetadata).toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
         builder.endObject();
         LicensesMetadata licensesMetadataFromXContent = getLicensesMetadataFromXContent(createParser(builder));
@@ -78,7 +79,7 @@ public void testLicenseMetadataParsingDoesNotSwallowOtherMetadata() throws Excep
         XContentBuilder builder = XContentFactory.jsonBuilder();
         Params params = new ToXContent.MapParams(Collections.singletonMap(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY));
         builder.startObject();
-        builder = metadataBuilder.build().toXContent(builder, params);
+        builder = ChunkedToXContent.wrapAsToXContent(metadataBuilder.build()).toXContent(builder, params);
         builder.endObject();
         // deserialize metadata again
         Metadata metadata = Metadata.Builder.fromXContent(createParser(builder));
@@ -101,7 +102,7 @@ public void testXContentSerializationOneTrial() throws Exception {
         XContentBuilder builder = XContentFactory.jsonBuilder();
         builder.startObject();
         builder.startObject("licenses");
-        licensesMetadata.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        ChunkedToXContent.wrapAsToXContent(licensesMetadata).toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
         builder.endObject();
         LicensesMetadata licensesMetadataFromXContent = getLicensesMetadataFromXContent(createParser(builder));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java
index d9c25639d811..450ded1eefd8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotShardTests.java
@@ -133,7 +133,6 @@ public void testSourceIncomplete() throws IOException {
                         null,
                         indexShardSnapshotStatus,
                         Version.CURRENT,
-                        Collections.emptyMap(),
                         randomMillisUpToYear9999(),
                         future
                     )
@@ -176,7 +175,6 @@ public void testIncrementalSnapshot() throws IOException {
                         null,
                         indexShardSnapshotStatus,
                         Version.CURRENT,
-                        Collections.emptyMap(),
                         randomMillisUpToYear9999(),
                         future
                     )
@@ -208,7 +206,6 @@ public void testIncrementalSnapshot() throws IOException {
                         null,
                         indexShardSnapshotStatus,
                         Version.CURRENT,
-                        Collections.emptyMap(),
                         randomMillisUpToYear9999(),
                         future
                     )
@@ -240,7 +237,6 @@ public void testIncrementalSnapshot() throws IOException {
                         null,
                         indexShardSnapshotStatus,
                         Version.CURRENT,
-                        Collections.emptyMap(),
                         randomMillisUpToYear9999(),
                         future
                     )
@@ -302,7 +298,6 @@ public void testRestoreMinmal() throws IOException {
                         null,
                         indexShardSnapshotStatus,
                         Version.CURRENT,
-                        Collections.emptyMap(),
                         randomMillisUpToYear9999(),
                         future
                     )
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersRequestTests.java
index 16aa64513de4..414ab0732520 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersRequestTests.java
@@ -10,8 +10,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
-
 public class MigrateToDataTiersRequestTests extends AbstractWireSerializingTestCase<MigrateToDataTiersRequest> {
 
     @Override
@@ -25,7 +23,7 @@ protected MigrateToDataTiersRequest createTestInstance() {
     }
 
     @Override
-    protected MigrateToDataTiersRequest mutateInstance(MigrateToDataTiersRequest instance) throws IOException {
+    protected MigrateToDataTiersRequest mutateInstance(MigrateToDataTiersRequest instance) {
         return randomValueOtherThan(instance, this::createTestInstance);
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersResponseTests.java
index ae69c535e4f9..bb16ef36bbcd 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/action/MigrateToDataTiersResponseTests.java
@@ -10,8 +10,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
-
 public class MigrateToDataTiersResponseTests extends AbstractWireSerializingTestCase<MigrateToDataTiersResponse> {
 
     @Override
@@ -34,7 +32,7 @@ protected MigrateToDataTiersResponse createTestInstance() {
     }
 
     @Override
-    protected MigrateToDataTiersResponse mutateInstance(MigrateToDataTiersResponse instance) throws IOException {
+    protected MigrateToDataTiersResponse mutateInstance(MigrateToDataTiersResponse instance) {
         int i = randomIntBetween(0, 6);
         return switch (i) {
             case 0 -> new MigrateToDataTiersResponse(
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderTests.java
index abefc970a9fe..f897caf2a529 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderTests.java
@@ -51,6 +51,7 @@
 
 import static org.elasticsearch.cluster.routing.allocation.DataTier.DATA_COLD;
 import static org.elasticsearch.cluster.routing.allocation.DataTier.DATA_FROZEN;
+import static org.elasticsearch.common.settings.ClusterSettings.createBuiltInClusterSettings;
 import static org.elasticsearch.node.Node.NODE_EXTERNAL_ID_SETTING;
 import static org.elasticsearch.node.NodeRoleSettings.NODE_ROLES_SETTING;
 import static org.hamcrest.Matchers.containsString;
@@ -73,11 +74,11 @@ public class DataTierAllocationDeciderTests extends ESAllocationTestCase {
     private static final DesiredNode CONTENT_DESIRED_NODE = newDesiredNode("node-content", DiscoveryNodeRole.DATA_CONTENT_NODE_ROLE);
     private static final DesiredNode DATA_DESIRED_NODE = newDesiredNode("node-data", DiscoveryNodeRole.DATA_ROLE);
 
-    private final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+    private final ClusterSettings clusterSettings = createBuiltInClusterSettings();
     private final AllocationDeciders allocationDeciders = new AllocationDeciders(
         Arrays.asList(
             DataTierAllocationDecider.INSTANCE,
-            new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
+            new SameShardAllocationDecider(clusterSettings),
             new ReplicaAfterPrimaryActiveAllocationDecider()
         )
     );
@@ -86,7 +87,8 @@ public class DataTierAllocationDeciderTests extends ESAllocationTestCase {
         new ShardId("myindex", "myindex", 0),
         true,
         RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-        new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "index created")
+        new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "index created"),
+        ShardRouting.Role.DEFAULT
     );
 
     public void testIndexPrefer() {
@@ -186,7 +188,7 @@ public void testIndexPrefer() {
                     clusterState,
                     node,
                     Decision.Type.NO,
-                    formatted(
+                    org.elasticsearch.core.Strings.format(
                         "index has a preference for tiers [%s], but no nodes for any of those tiers are available in the cluster",
                         tierPreference
                     )
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java
index 5d8a357dc8ca..cbc9a762b37d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java
@@ -20,7 +20,8 @@
 import org.elasticsearch.index.mapper.ValueFetcher;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.query.SearchExecutionContext;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -88,14 +89,14 @@ public void testRegexpQuery() {
 
     public void testFetchValue() throws IOException {
         MappedFieldType ft = DataTierFieldMapper.DataTierFieldType.INSTANCE;
-        SourceLookup lookup = new SourceLookup(new SourceLookup.NullSourceProvider());
+        Source source = Source.empty(XContentType.JSON);
 
         List<Object> ignoredValues = new ArrayList<>();
         ValueFetcher valueFetcher = ft.valueFetcher(createContext(), null);
-        assertEquals(singletonList("data_warm"), valueFetcher.fetchValues(lookup, -1, ignoredValues));
+        assertEquals(singletonList("data_warm"), valueFetcher.fetchValues(source, -1, ignoredValues));
 
         ValueFetcher emptyValueFetcher = ft.valueFetcher(createContextWithoutSetting(), null);
-        assertTrue(emptyValueFetcher.fetchValues(lookup, -1, ignoredValues).isEmpty());
+        assertTrue(emptyValueFetcher.fetchValues(source, -1, ignoredValues).isEmpty());
     }
 
     private SearchExecutionContext createContext() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ClientHelperTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ClientHelperTests.java
index ae3fdaeefaec..0b3bd0f7f09c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ClientHelperTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ClientHelperTests.java
@@ -455,14 +455,14 @@ public void testGetPersistableSafeSecurityHeaders() throws IOException {
             final Authentication rewrittenAuth = AuthenticationContextSerializer.decode(
                 headers2.get(AuthenticationField.AUTHENTICATION_KEY)
             );
-            assertThat(rewrittenAuth.getEffectiveSubject().getVersion(), equalTo(previousVersion));
+            assertThat(rewrittenAuth.getEffectiveSubject().getTransportVersion(), equalTo(previousVersion.transportVersion));
             assertThat(rewrittenAuth.getEffectiveSubject().getUser(), equalTo(authentication.getEffectiveSubject().getUser()));
         }
         if (hasSecondaryAuthHeader) {
             final Authentication rewrittenSecondaryAuth = AuthenticationContextSerializer.decode(
                 headers2.get(SecondaryAuthentication.THREAD_CTX_KEY)
             );
-            assertThat(rewrittenSecondaryAuth.getEffectiveSubject().getVersion(), equalTo(previousVersion));
+            assertThat(rewrittenSecondaryAuth.getEffectiveSubject().getTransportVersion(), equalTo(previousVersion.transportVersion));
             assertThat(rewrittenSecondaryAuth.getEffectiveSubject().getUser(), equalTo(authentication.getEffectiveSubject().getUser()));
         }
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/DataTiersFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/DataTiersFeatureSetUsageTests.java
index 830d4ef9e082..e5f37dfb5764 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/DataTiersFeatureSetUsageTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/DataTiersFeatureSetUsageTests.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -23,7 +22,7 @@ protected Writeable.Reader<DataTiersFeatureSetUsage> instanceReader() {
     }
 
     @Override
-    protected DataTiersFeatureSetUsage mutateInstance(DataTiersFeatureSetUsage instance) throws IOException {
+    protected DataTiersFeatureSetUsage mutateInstance(DataTiersFeatureSetUsage instance) {
         return randomValueOtherThan(instance, DataTiersFeatureSetUsageTests::randomUsage);
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/HealthApiFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/HealthApiFeatureSetUsageTests.java
new file mode 100644
index 000000000000..f66912736f2b
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/HealthApiFeatureSetUsageTests.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.metrics.Counters;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.health.HealthStatus;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.hamcrest.Matchers;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.elasticsearch.xpack.core.HealthApiFeatureSetUsage.enrichUsageStatsWithValues;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class HealthApiFeatureSetUsageTests extends AbstractWireSerializingTestCase<HealthApiFeatureSetUsage> {
+
+    @Override
+    protected HealthApiFeatureSetUsage createTestInstance() {
+        return new HealthApiFeatureSetUsage(true, true, randomCounters());
+    }
+
+    @Override
+    protected HealthApiFeatureSetUsage mutateInstance(HealthApiFeatureSetUsage instance) {
+        return randomValueOtherThan(instance, this::createTestInstance);
+    }
+
+    @Override
+    protected Writeable.Reader<HealthApiFeatureSetUsage> instanceReader() {
+        return HealthApiFeatureSetUsage::new;
+    }
+
+    private Counters randomCounters() {
+        if (rarely()) {
+            return null;
+        }
+        Counters counters = new Counters();
+        for (int i = 0; i < randomInt(20); i++) {
+            if (randomBoolean()) {
+                counters.inc(randomAlphaOfLength(10), randomInt(20));
+            } else {
+                counters.inc(randomAlphaOfLength(10) + "." + randomAlphaOfLength(10), randomInt(20));
+            }
+        }
+        return counters;
+    }
+
+    public void testEnrichingValuesEmptyStats() {
+        Map<String, Object> noRequestStats = Map.of("invocations", Map.of("total", 0));
+        enrichUsageStatsWithValues(noRequestStats);
+        assertThat(noRequestStats.keySet(), containsInAnyOrder("invocations"));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testEnrichValuesGreenStats() {
+        Map<String, Object> greenStats = new HashMap<>();
+        int greenCounter = randomInt();
+        greenStats.put("statuses", new HashMap<>(Map.of("green", greenCounter)));
+
+        enrichUsageStatsWithValues(greenStats);
+
+        assertThat(greenStats.keySet(), containsInAnyOrder("statuses"));
+        Map<String, Object> enriched = (Map<String, Object>) greenStats.get("statuses");
+        assertThat(enriched.get("green"), equalTo(greenCounter));
+        assertThat((List<String>) enriched.get("values"), containsInAnyOrder("green"));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testEnrichingValuesUnhealthyStats() {
+        Map<String, Object> stats = new HashMap<>();
+        Map<String, Object> statuses = generateStatsMap(
+            Arrays.stream(HealthStatus.values()).map(HealthStatus::xContentValue).collect(Collectors.toList())
+        );
+        stats.put("statuses", statuses);
+        List<String> indicatorLabels = List.of("ilm", "slm", "disk", "master_stability", "shards_availability");
+        Map<String, Object> redIndicators = generateStatsMap(indicatorLabels);
+        Map<String, Object> yellowIndicators = generateStatsMap(indicatorLabels);
+        stats.put("indicators", Map.of("red", redIndicators, "yellow", yellowIndicators));
+        List<String> diagnosisLabels = IntStream.of(randomIntBetween(1, 50)).mapToObj(i -> "diagnosis:" + i).toList();
+        Map<String, Object> redDiagnoses = generateStatsMap(diagnosisLabels);
+        Map<String, Object> yellowDiagnoses = generateStatsMap(diagnosisLabels);
+        stats.put("diagnoses", Map.of("red", redDiagnoses, "yellow", yellowDiagnoses));
+
+        enrichUsageStatsWithValues(stats);
+
+        assertThat(stats.keySet(), containsInAnyOrder("statuses", "indicators", "diagnoses"));
+        assertValues((Map<String, Object>) stats.get("statuses"));
+        assertValues(((Map<String, Map<String, Object>>) stats.get("indicators")).get("red"));
+        assertValues(((Map<String, Map<String, Object>>) stats.get("indicators")).get("yellow"));
+        assertValues(((Map<String, Map<String, Object>>) stats.get("diagnoses")).get("red"));
+        assertValues(((Map<String, Map<String, Object>>) stats.get("diagnoses")).get("yellow"));
+
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void assertValues(Map<String, Object> map) {
+        Set<String> expectedValues = Sets.difference(map.keySet(), Set.of("values"));
+        assertThat(map.get("values"), notNullValue());
+        assertThat(
+            (List<String>) map.get("values"),
+            containsInAnyOrder(expectedValues.stream().map(Matchers::equalTo).collect(Collectors.toList()))
+        );
+    }
+
+    private static Map<String, Object> generateStatsMap(List<String> statLabels) {
+        Map<String, Object> statsMap = new HashMap<>();
+        for (String stat : randomNonEmptySubsetOf(statLabels)) {
+            statsMap.put(stat, randomIntBetween(1, 1_000));
+        }
+        return statsMap;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java
index 1a1d31e7c16e..937ee93f817c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java
@@ -26,9 +26,9 @@
 import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -201,7 +201,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver expressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         List<Object> components = new ArrayList<>();
         components.addAll(
@@ -218,7 +218,7 @@ public Collection<Object> createComponents(
                 expressionResolver,
                 repositoriesServiceSupplier,
                 tracer,
-                allocationDeciders
+                allocationService
             )
         );
 
@@ -238,7 +238,7 @@ public Collection<Object> createComponents(
                         expressionResolver,
                         repositoriesServiceSupplier,
                         tracer,
-                        allocationDeciders
+                        allocationService
                     )
                 )
             );
@@ -729,6 +729,7 @@ public void cleanUpFeature(
         List<SystemIndexPlugin> systemPlugins = filterPlugins(SystemIndexPlugin.class);
 
         GroupedActionListener<ResetFeatureStateResponse.ResetFeatureStateStatus> allListeners = new GroupedActionListener<>(
+            systemPlugins.size(),
             ActionListener.wrap(listenerResults -> {
                 // If the clean-up produced only one result, use that to pass along. In most
                 // cases it should be 1-1 mapping of feature to response. Passing back success
@@ -738,8 +739,7 @@ public void cleanUpFeature(
                 } else {
                     finalListener.onResponse(ResetFeatureStateStatus.success(getFeatureName()));
                 }
-            }, finalListener::onFailure),
-            systemPlugins.size()
+            }, finalListener::onFailure)
         );
         systemPlugins.forEach(plugin -> plugin.cleanUpFeature(clusterService, client, allListeners));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/ReloadDetailsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/ReloadDetailsTests.java
index 4c1564516f89..9f34f16712c0 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/ReloadDetailsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/ReloadDetailsTests.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.action.ReloadAnalyzersResponse.ReloadDetails;
 
-import java.io.IOException;
 import java.util.HashSet;
 import java.util.Set;
 
@@ -32,7 +31,7 @@ protected Reader<ReloadDetails> instanceReader() {
     }
 
     @Override
-    protected ReloadDetails mutateInstance(ReloadDetails instance) throws IOException {
+    protected ReloadDetails mutateInstance(ReloadDetails instance) {
         String indexName = instance.getIndexName();
         Set<String> reloadedAnalyzers = new HashSet<>(instance.getReloadedAnalyzers());
         Set<String> reloadedIndicesNodes = new HashSet<>(instance.getReloadedIndicesNodes());
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/SetResetModeActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/SetResetModeActionRequestTests.java
index 32d2bca43402..f7d7b3a1fa21 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/SetResetModeActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/SetResetModeActionRequestTests.java
@@ -18,6 +18,11 @@ protected SetResetModeActionRequest createTestInstance() {
         return new SetResetModeActionRequest(enabled, enabled == false && randomBoolean());
     }
 
+    @Override
+    protected SetResetModeActionRequest mutateInstance(SetResetModeActionRequest instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected boolean supportsUnknownFields() {
         return false;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/XPackUsageResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/XPackUsageResponseTests.java
index a5501cebfe6d..b4f3886733f1 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/XPackUsageResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/XPackUsageResponseTests.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.core.action;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
@@ -49,8 +50,8 @@ public OldUsage(final StreamInput in) throws IOException {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return oldVersion;
+        public TransportVersion getMinimalSupportedVersion() {
+            return oldVersion.transportVersion;
         }
 
     }
@@ -66,8 +67,8 @@ public NewUsage(final StreamInput in) throws IOException {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return newVersion;
+        public TransportVersion getMinimalSupportedVersion() {
+            return newVersion.transportVersion;
         }
 
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/util/QueryPageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/util/QueryPageTests.java
index 0ec71d0746ac..68679fae130e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/util/QueryPageTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/util/QueryPageTests.java
@@ -88,8 +88,7 @@ protected Reader<QueryPage<QueryPageTests.QueryPageTester>> instanceReader() {
     }
 
     @Override
-    protected QueryPage<QueryPageTests.QueryPageTester> mutateInstance(QueryPage<QueryPageTests.QueryPageTester> instance)
-        throws IOException {
+    protected QueryPage<QueryPageTests.QueryPageTester> mutateInstance(QueryPage<QueryPageTester> instance) {
         ParseField resultsField = instance.getResultsField();
         List<QueryPageTests.QueryPageTester> page = instance.results();
         long count = instance.count();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/archive/ArchiveFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/archive/ArchiveFeatureSetUsageTests.java
index db644c2261a1..6b949f16518e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/archive/ArchiveFeatureSetUsageTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/archive/ArchiveFeatureSetUsageTests.java
@@ -9,8 +9,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
-
 public class ArchiveFeatureSetUsageTests extends AbstractWireSerializingTestCase<ArchiveFeatureSetUsage> {
 
     @Override
@@ -20,7 +18,7 @@ protected ArchiveFeatureSetUsage createTestInstance() {
     }
 
     @Override
-    protected ArchiveFeatureSetUsage mutateInstance(ArchiveFeatureSetUsage instance) throws IOException {
+    protected ArchiveFeatureSetUsage mutateInstance(ArchiveFeatureSetUsage instance) {
         boolean available = instance.available();
         int numArchiveIndices = instance.getNumberOfArchiveIndices();
         switch (between(0, 1)) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/GetAsyncResultRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/GetAsyncResultRequestTests.java
index e827d228a4e8..09afab08b3f3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/GetAsyncResultRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/async/GetAsyncResultRequestTests.java
@@ -30,6 +30,11 @@ protected GetAsyncResultRequest createTestInstance() {
         return req;
     }
 
+    @Override
+    protected GetAsyncResultRequest mutateInstance(GetAsyncResultRequest instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static String randomSearchId() {
         return randomAsyncId().getEncoded();
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/ShardFollowTaskTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/ShardFollowTaskTests.java
index 6977f7112455..2361eda1e7b7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/ShardFollowTaskTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/ShardFollowTaskTests.java
@@ -43,6 +43,11 @@ protected ShardFollowTask createTestInstance() {
         );
     }
 
+    @Override
+    protected ShardFollowTask mutateInstance(ShardFollowTask instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<ShardFollowTask> instanceReader() {
         return ShardFollowTask::readFrom;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/stats/EnumCountersTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/stats/EnumCountersTests.java
index f3304467dad6..d3b2265b7e93 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/stats/EnumCountersTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/stats/EnumCountersTests.java
@@ -44,6 +44,11 @@ protected EnumCounters<TestV2> createTestInstance() {
         return inst;
     }
 
+    @Override
+    protected EnumCounters<TestV2> mutateInstance(EnumCounters<TestV2> instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected EnumCounters<TestV2> copyInstance(EnumCounters<TestV2> instance, Version version) throws IOException {
         return serialize(instance, in -> new EnumCounters<>(in, TestV2.class));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/frozen/FrozenIndicesFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/frozen/FrozenIndicesFeatureSetUsageTests.java
index 4b1130c1f083..013098bd5f34 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/frozen/FrozenIndicesFeatureSetUsageTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/frozen/FrozenIndicesFeatureSetUsageTests.java
@@ -9,8 +9,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
-
 public class FrozenIndicesFeatureSetUsageTests extends AbstractWireSerializingTestCase<FrozenIndicesFeatureSetUsage> {
 
     @Override
@@ -21,7 +19,7 @@ protected FrozenIndicesFeatureSetUsage createTestInstance() {
     }
 
     @Override
-    protected FrozenIndicesFeatureSetUsage mutateInstance(FrozenIndicesFeatureSetUsage instance) throws IOException {
+    protected FrozenIndicesFeatureSetUsage mutateInstance(FrozenIndicesFeatureSetUsage instance) {
         boolean available = instance.available();
         boolean enabled = instance.enabled();
         int numFrozenIndices = instance.getNumberOfFrozenIndices();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ActionConfigStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ActionConfigStatsTests.java
index 2a503515bf8e..f8565cdabf53 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ActionConfigStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ActionConfigStatsTests.java
@@ -13,8 +13,6 @@
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleFeatureSetUsage.ActionConfigStats;
 
-import java.io.IOException;
-
 public class ActionConfigStatsTests extends AbstractWireSerializingTestCase<ActionConfigStats> {
 
     @Override
@@ -80,7 +78,7 @@ protected Writeable.Reader<ActionConfigStats> instanceReader() {
     }
 
     @Override
-    protected ActionConfigStats mutateInstance(ActionConfigStats instance) throws IOException {
+    protected ActionConfigStats mutateInstance(ActionConfigStats instance) {
         ActionConfigStats.Builder builder = ActionConfigStats.builder(instance);
         switch (between(0, 14)) {
             case 0 -> {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DataTierMigrationRoutedStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DataTierMigrationRoutedStepTests.java
index 61d76a68ed67..4d7d5deaee56 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DataTierMigrationRoutedStepTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DataTierMigrationRoutedStepTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.allocation.DataTier;
@@ -237,7 +238,7 @@ public void testExecuteForIndexWithoutTierRoutingInformationWaitsForReplicasToBe
         {
             IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
                 .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", true, ShardRoutingState.STARTED))
-                .addReplica();
+                .addReplica(ShardRouting.Role.DEFAULT);
 
             ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE)
                 .metadata(Metadata.builder().put(indexMetadata, true).build())
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DeleteActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DeleteActionTests.java
index 6bb85eeb3501..966f260aea55 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DeleteActionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DeleteActionTests.java
@@ -27,6 +27,11 @@ protected DeleteAction createTestInstance() {
         return DeleteAction.WITH_SNAPSHOT_DELETE;
     }
 
+    @Override
+    protected DeleteAction mutateInstance(DeleteAction instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<DeleteAction> instanceReader() {
         return DeleteAction::readFrom;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleActionTests.java
index 82bd02fbe5b0..256bf79e6006 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleActionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DownsampleActionTests.java
@@ -35,6 +35,11 @@ protected DownsampleAction createTestInstance() {
         return randomInstance();
     }
 
+    @Override
+    protected DownsampleAction mutateInstance(DownsampleAction instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<DownsampleAction> instanceReader() {
         return DownsampleAction::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequestTests.java
index 99d82da67d4b..25de084a34b0 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequestTests.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.common.io.stream.Writeable.Reader;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.Arrays;
 
 public class ExplainLifecycleRequestTests extends AbstractWireSerializingTestCase<ExplainLifecycleRequest> {
@@ -45,7 +44,7 @@ protected ExplainLifecycleRequest createTestInstance() {
     }
 
     @Override
-    protected ExplainLifecycleRequest mutateInstance(ExplainLifecycleRequest instance) throws IOException {
+    protected ExplainLifecycleRequest mutateInstance(ExplainLifecycleRequest instance) {
         String[] indices = instance.indices();
         IndicesOptions indicesOptions = instance.indicesOptions();
         boolean onlyErrors = instance.onlyErrors();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeStepTests.java
index e1f6b3584276..eca0804cb265 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeStepTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeStepTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.core.ilm.Step.StepKey;
@@ -169,7 +170,7 @@ public void onFailure(Exception e) {
 
         ElasticsearchException stepException = failedStep.get();
         assertThat(stepException, notNullValue());
-        assertThat(stepException.getMessage(), is(formatted("""
+        assertThat(stepException.getMessage(), is(Strings.format("""
             index [%s] in policy [ilmPolicy] encountered failures [{"shard":0,"index":"%s","status":"BAD_REQUEST",\
             "reason":{"type":"illegal_argument_exception","reason":"couldn't merge"}}] on step [forcemerge]\
             """, index.getName(), index.getName())));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeActionTests.java
index 2ddb127bcc33..858ba31d27a7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeActionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/FreezeActionTests.java
@@ -27,6 +27,11 @@ protected FreezeAction createTestInstance() {
         return FreezeAction.INSTANCE;
     }
 
+    @Override
+    protected FreezeAction mutateInstance(FreezeAction instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<FreezeAction> instanceReader() {
         return in -> FreezeAction.INSTANCE;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/GenerateUniqueIndexNameStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/GenerateUniqueIndexNameStepTests.java
index dd563798100d..bc4501f492c7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/GenerateUniqueIndexNameStepTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/GenerateUniqueIndexNameStepTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.AliasMetadata;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.LifecycleExecutionState;
@@ -191,7 +192,7 @@ public void testValidateGeneratedIndexName() {
                 .numberOfReplicas(randomIntBetween(1, 5))
                 .build();
             ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
-                .routingTable(RoutingTable.builder().addAsNew(indexMetadata).build())
+                .routingTable(RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY).addAsNew(indexMetadata).build())
                 .metadata(Metadata.builder().put(indexMetadata, false))
                 .build();
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponseTests.java
index 92647e830e40..a12b4ff75ee3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponseTests.java
@@ -172,7 +172,7 @@ protected boolean assertToXContentEquivalence() {
     }
 
     @Override
-    protected IndexLifecycleExplainResponse mutateInstance(IndexLifecycleExplainResponse instance) throws IOException {
+    protected IndexLifecycleExplainResponse mutateInstance(IndexLifecycleExplainResponse instance) {
         String index = instance.getIndex();
         Long indexCreationDate = instance.getIndexCreationDate();
         String policy = instance.getPolicyName();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleFeatureSetUsageTests.java
index b01bf25a0d7b..46bb6297847c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleFeatureSetUsageTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleFeatureSetUsageTests.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleFeatureSetUsage.PolicyStats;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -32,7 +31,7 @@ protected IndexLifecycleFeatureSetUsage createTestInstance() {
     }
 
     @Override
-    protected IndexLifecycleFeatureSetUsage mutateInstance(IndexLifecycleFeatureSetUsage instance) throws IOException {
+    protected IndexLifecycleFeatureSetUsage mutateInstance(IndexLifecycleFeatureSetUsage instance) {
         List<PolicyStats> policyStats = instance.getPolicyStats();
         if (policyStats == null) {
             policyStats = new ArrayList<>();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecycleOperationMetadataTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecycleOperationMetadataTests.java
new file mode 100644
index 000000000000..f4b38b0b280f
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecycleOperationMetadataTests.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.ilm;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
+import org.elasticsearch.test.TransportVersionUtils;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+
+public class LifecycleOperationMetadataTests extends AbstractChunkedSerializingTestCase<Metadata.Custom> {
+
+    @Override
+    protected LifecycleOperationMetadata createTestInstance() {
+        return new LifecycleOperationMetadata(randomFrom(OperationMode.values()), randomFrom(OperationMode.values()));
+    }
+
+    @Override
+    protected LifecycleOperationMetadata doParseInstance(XContentParser parser) throws IOException {
+        return LifecycleOperationMetadata.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected Writeable.Reader<Metadata.Custom> instanceReader() {
+        return LifecycleOperationMetadata::new;
+    }
+
+    @Override
+    protected Metadata.Custom mutateInstance(Metadata.Custom instance) {
+        LifecycleOperationMetadata metadata = (LifecycleOperationMetadata) instance;
+        if (randomBoolean()) {
+            return new LifecycleOperationMetadata(
+                randomValueOtherThan(metadata.getILMOperationMode(), () -> randomFrom(OperationMode.values())),
+                metadata.getSLMOperationMode()
+            );
+        } else {
+            return new LifecycleOperationMetadata(
+                metadata.getILMOperationMode(),
+                randomValueOtherThan(metadata.getSLMOperationMode(), () -> randomFrom(OperationMode.values()))
+            );
+        }
+    }
+
+    public void testMinimumSupportedVersion() {
+        TransportVersion min = createTestInstance().getMinimalSupportedVersion();
+        assertTrue(
+            min.onOrBefore(TransportVersionUtils.randomVersionBetween(random(), TransportVersion.V_8_7_0, TransportVersion.CURRENT))
+        );
+    }
+
+    public void testcontext() {
+        assertThat(createTestInstance().context(), containsInAnyOrder(Metadata.XContentContext.API, Metadata.XContentContext.GATEWAY));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyMetadataTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyMetadataTests.java
index 818d89161f6e..de8fc83b9a5c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyMetadataTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyMetadataTests.java
@@ -127,7 +127,7 @@ protected Reader<LifecyclePolicyMetadata> instanceReader() {
     }
 
     @Override
-    protected LifecyclePolicyMetadata mutateInstance(LifecyclePolicyMetadata instance) throws IOException {
+    protected LifecyclePolicyMetadata mutateInstance(LifecyclePolicyMetadata instance) {
         LifecyclePolicy policy = instance.getPolicy();
         Map<String, String> headers = instance.getHeaders();
         long version = instance.getVersion();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java
index 28c75b3eabc0..fa8e799f6a49 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyTests.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.ilm.Step.StepKey;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -281,7 +280,7 @@ public static LifecyclePolicy randomTestLifecyclePolicy(@Nullable String lifecyc
     }
 
     @Override
-    protected LifecyclePolicy mutateInstance(LifecyclePolicy instance) throws IOException {
+    protected LifecyclePolicy mutateInstance(LifecyclePolicy instance) {
         String name = instance.getName();
         Map<String, Phase> phases = instance.getPhases();
         switch (between(0, 1)) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MigrateActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MigrateActionTests.java
index 5be2e2e73d0f..2dd4c8e6a540 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MigrateActionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MigrateActionTests.java
@@ -45,7 +45,7 @@ protected MigrateAction createTestInstance() {
     }
 
     @Override
-    protected MigrateAction mutateInstance(MigrateAction instance) throws IOException {
+    protected MigrateAction mutateInstance(MigrateAction instance) {
         return instance.isEnabled() == false ? MigrateAction.ENABLED : MigrateAction.DISABLED;
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MockActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MockActionTests.java
index b8ac0ec2d44b..306eb6460662 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MockActionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MockActionTests.java
@@ -31,7 +31,7 @@ protected Reader<MockAction> instanceReader() {
     }
 
     @Override
-    protected MockAction mutateInstance(MockAction instance) throws IOException {
+    protected MockAction mutateInstance(MockAction instance) {
         List<Step> steps = instance.getSteps();
         boolean safe = instance.isSafeAction();
         if (randomBoolean()) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PhaseCacheManagementTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PhaseCacheManagementTests.java
index 049a04836f57..8c51d3339682 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PhaseCacheManagementTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PhaseCacheManagementTests.java
@@ -62,21 +62,21 @@ public void testRefreshPhaseJson() throws IOException {
             .setStep("check-rollover-ready")
             .setPhaseDefinition("""
                 {
-                        "policy" : "my-policy",
-                        "phase_definition" : {
-                          "min_age" : "20m",
-                          "actions" : {
-                            "rollover" : {
-                              "max_age" : "5s"
-                            },
-                            "set_priority" : {
-                              "priority" : 150
-                            }
-                          }
-                        },
-                        "version" : 1,
-                        "modified_date_in_millis" : 1578521007076
-                      }""");
+                  "policy" : "my-policy",
+                  "phase_definition" : {
+                    "min_age" : "20m",
+                    "actions" : {
+                      "rollover" : {
+                        "max_age" : "5s"
+                      },
+                      "set_priority" : {
+                        "priority" : 150
+                      }
+                    }
+                  },
+                  "version" : 1,
+                  "modified_date_in_millis" : 1578521007076
+                }""");
 
         IndexMetadata meta = buildIndexMetadata("my-policy", exState);
         String indexName = meta.getIndex().getName();
@@ -216,17 +216,17 @@ public void testReadStepKeys() {
         assertThat(
             readStepKeys(REGISTRY, client, """
                 {
-                        "policy": "my_lifecycle3",
-                        "phase_definition": {
-                          "min_age": "0ms",
-                          "actions": {
-                            "rollover": {
-                              "max_age": "30s"
-                            }
-                          }
-                        },
-                        "version": 3,
-                        "modified_date_in_millis": 1539609701576
+                  "policy": "my_lifecycle3",
+                  "phase_definition": {
+                    "min_age": "0ms",
+                    "actions": {
+                      "rollover": {
+                        "max_age": "30s"
+                      }
+                    }
+                  },
+                  "version": 3,
+                  "modified_date_in_millis": 1539609701576
                 }""", "phase", null),
             contains(
                 new Step.StepKey("phase", "rollover", WaitForRolloverReadyStep.NAME),
@@ -240,20 +240,20 @@ public void testReadStepKeys() {
         assertThat(
             readStepKeys(REGISTRY, client, """
                 {
-                        "policy" : "my_lifecycle3",
-                        "phase_definition" : {
-                          "min_age" : "20m",
-                          "actions" : {
-                            "rollover" : {
-                              "max_age" : "5s"
-                            },
-                            "set_priority" : {
-                              "priority" : 150
-                            }
-                          }
-                        },
-                        "version" : 1,
-                        "modified_date_in_millis" : 1578521007076
+                  "policy" : "my_lifecycle3",
+                  "phase_definition" : {
+                    "min_age" : "20m",
+                    "actions" : {
+                      "rollover" : {
+                        "max_age" : "5s"
+                      },
+                      "set_priority" : {
+                        "priority" : 150
+                      }
+                    }
+                  },
+                  "version" : 1,
+                  "modified_date_in_millis" : 1578521007076
                 }""", "phase", null),
             containsInAnyOrder(
                 new Step.StepKey("phase", "rollover", WaitForRolloverReadyStep.NAME),
@@ -298,21 +298,21 @@ public void testIndexCanBeSafelyUpdated() {
                 .setStep("check-rollover-ready")
                 .setPhaseDefinition("""
                     {
-                            "policy" : "my-policy",
-                            "phase_definition" : {
-                              "min_age" : "20m",
-                              "actions" : {
-                                "rollover" : {
-                                  "max_age" : "5s"
-                                },
-                                "set_priority" : {
-                                  "priority" : 150
-                                }
-                              }
-                            },
-                            "version" : 1,
-                            "modified_date_in_millis" : 1578521007076
-                          }""")
+                      "policy" : "my-policy",
+                      "phase_definition" : {
+                        "min_age" : "20m",
+                        "actions" : {
+                          "rollover" : {
+                            "max_age" : "5s"
+                          },
+                          "set_priority" : {
+                            "priority" : 150
+                          }
+                        }
+                      },
+                      "version" : 1,
+                      "modified_date_in_millis" : 1578521007076
+                    }""")
                 .build();
 
             IndexMetadata meta = mkMeta().putCustom(ILM_CUSTOM_METADATA_KEY, exState.asMap()).build();
@@ -335,21 +335,21 @@ public void testIndexCanBeSafelyUpdated() {
                 .setStep("check-rollover-ready")
                 .setPhaseDefinition("""
                     {
-                            "policy" : "my-policy",
-                            "phase_definition" : {
-                              "min_age" : "20m",
-                              "actions" : {
-                                "rollover" : {
-                                  "max_age" : "5s"
-                                },
-                                "set_priority" : {
-                                  "priority" : 150
-                                }
-                              }
-                            },
-                            "version" : 1,
-                            "modified_date_in_millis" : 1578521007076
-                          }""")
+                      "policy" : "my-policy",
+                      "phase_definition" : {
+                        "min_age" : "20m",
+                        "actions" : {
+                          "rollover" : {
+                            "max_age" : "5s"
+                          },
+                          "set_priority" : {
+                            "priority" : 150
+                          }
+                        }
+                      },
+                      "version" : 1,
+                      "modified_date_in_millis" : 1578521007076
+                    }""")
                 .build();
 
             IndexMetadata meta = mkMeta().putCustom(ILM_CUSTOM_METADATA_KEY, exState.asMap()).build();
@@ -371,21 +371,21 @@ public void testIndexCanBeSafelyUpdated() {
                 .setStep("check-rollover-ready")
                 .setPhaseDefinition("""
                     {
-                            "policy" : "my-policy",
-                            "phase_definition" : {
-                              "min_age" : "20m",
-                              "actions" : {
-                                "rollover" : {
-                                  "max_age" : "5s"
-                                },
-                                "set_priority" : {
-                                  "priority" : 150
-                                }
-                              }
-                            },
-                            "version" : 1,
-                            "modified_date_in_millis" : 1578521007076
-                          }""")
+                      "policy" : "my-policy",
+                      "phase_definition" : {
+                        "min_age" : "20m",
+                        "actions" : {
+                          "rollover" : {
+                            "max_age" : "5s"
+                          },
+                          "set_priority" : {
+                            "priority" : 150
+                          }
+                        }
+                      },
+                      "version" : 1,
+                      "modified_date_in_millis" : 1578521007076
+                    }""")
                 .build();
 
             IndexMetadata meta = mkMeta().putCustom(ILM_CUSTOM_METADATA_KEY, exState.asMap()).build();
@@ -406,21 +406,21 @@ public void testIndexCanBeSafelyUpdated() {
         {
             LifecycleExecutionState exState = LifecycleExecutionState.builder().setPhaseDefinition("""
                 {
-                        "policy" : "my-policy",
-                        "phase_definition" : {
-                          "min_age" : "20m",
-                          "actions" : {
-                            "rollover" : {
-                              "max_age" : "5s"
-                            },
-                            "set_priority" : {
-                              "priority" : 150
-                            }
-                          }
-                        },
-                        "version" : 1,
-                        "modified_date_in_millis" : 1578521007076
-                      }""").build();
+                  "policy" : "my-policy",
+                  "phase_definition" : {
+                    "min_age" : "20m",
+                    "actions" : {
+                      "rollover" : {
+                        "max_age" : "5s"
+                      },
+                      "set_priority" : {
+                        "priority" : 150
+                      }
+                    }
+                  },
+                  "version" : 1,
+                  "modified_date_in_millis" : 1578521007076
+                }""").build();
 
             IndexMetadata meta = mkMeta().putCustom(ILM_CUSTOM_METADATA_KEY, exState.asMap()).build();
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PhaseExecutionInfoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PhaseExecutionInfoTests.java
index 6e6cb7104700..7622118d2b99 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PhaseExecutionInfoTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PhaseExecutionInfoTests.java
@@ -54,7 +54,7 @@ protected PhaseExecutionInfo doParseInstance(XContentParser parser) throws IOExc
     }
 
     @Override
-    protected PhaseExecutionInfo mutateInstance(PhaseExecutionInfo instance) throws IOException {
+    protected PhaseExecutionInfo mutateInstance(PhaseExecutionInfo instance) {
         String policyName = instance.getPolicyName();
         Phase phase = instance.getPhase();
         long version = instance.getVersion();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PhaseStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PhaseStatsTests.java
index 71b697e944e9..3848999d2122 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PhaseStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PhaseStatsTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleFeatureSetUsage.PhaseStats;
 
-import java.io.IOException;
 import java.util.Arrays;
 
 public class PhaseStatsTests extends AbstractWireSerializingTestCase<PhaseStats> {
@@ -29,7 +28,7 @@ public static PhaseStats createRandomInstance() {
     }
 
     @Override
-    protected PhaseStats mutateInstance(PhaseStats instance) throws IOException {
+    protected PhaseStats mutateInstance(PhaseStats instance) {
         TimeValue after = instance.getAfter();
         String[] actionNames = instance.getActionNames();
         switch (between(0, 1)) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PhaseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PhaseTests.java
index cdc97b3cf86e..8258c2b4c161 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PhaseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PhaseTests.java
@@ -75,7 +75,7 @@ protected NamedXContentRegistry xContentRegistry() {
     }
 
     @Override
-    protected Phase mutateInstance(Phase instance) throws IOException {
+    protected Phase mutateInstance(Phase instance) {
         String name = instance.getName();
         TimeValue after = instance.getMinimumAge();
         Map<String, LifecycleAction> actions = instance.getActions();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PolicyStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PolicyStatsTests.java
index 682f813456ec..2e012a03e6b7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PolicyStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/PolicyStatsTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleFeatureSetUsage.PhaseStats;
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleFeatureSetUsage.PolicyStats;
 
-import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -34,7 +33,7 @@ public static PolicyStats createRandomInstance() {
     }
 
     @Override
-    protected PolicyStats mutateInstance(PolicyStats instance) throws IOException {
+    protected PolicyStats mutateInstance(PolicyStats instance) {
         Map<String, PhaseStats> phaseStats = instance.getPhaseStats();
         int indicesManaged = instance.getIndicesManaged();
         switch (between(0, 1)) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ReadOnlyActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ReadOnlyActionTests.java
index b2180773e9d6..8de13780245e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ReadOnlyActionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ReadOnlyActionTests.java
@@ -26,6 +26,11 @@ protected ReadOnlyAction createTestInstance() {
         return new ReadOnlyAction();
     }
 
+    @Override
+    protected ReadOnlyAction mutateInstance(ReadOnlyAction instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<ReadOnlyAction> instanceReader() {
         return ReadOnlyAction::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RolloverActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RolloverActionTests.java
index dd28da61f9c8..928fa730b46c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RolloverActionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RolloverActionTests.java
@@ -79,7 +79,7 @@ protected Reader<RolloverAction> instanceReader() {
     }
 
     @Override
-    protected RolloverAction mutateInstance(RolloverAction instance) throws IOException {
+    protected RolloverAction mutateInstance(RolloverAction instance) {
         ByteSizeValue maxSize = instance.getMaxSize();
         ByteSizeValue maxPrimaryShardSize = instance.getMaxPrimaryShardSize();
         TimeValue maxAge = instance.getMaxAge();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java
index da628bedfe74..b6e8401b9f4c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java
@@ -158,7 +158,7 @@ protected Writeable.Reader<SearchableSnapshotAction> instanceReader() {
     }
 
     @Override
-    protected SearchableSnapshotAction mutateInstance(SearchableSnapshotAction instance) throws IOException {
+    protected SearchableSnapshotAction mutateInstance(SearchableSnapshotAction instance) {
         return randomInstance();
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SetPriorityActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SetPriorityActionTests.java
index adde6a736cd7..869af19c34b3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SetPriorityActionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SetPriorityActionTests.java
@@ -34,6 +34,11 @@ protected SetPriorityAction createTestInstance() {
         return new SetPriorityAction(priority);
     }
 
+    @Override
+    protected SetPriorityAction mutateInstance(SetPriorityAction instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<SetPriorityAction> instanceReader() {
         return SetPriorityAction::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/StartILMRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/StartILMRequestTests.java
index 8982f4352ff2..308cb1a94891 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/StartILMRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/StartILMRequestTests.java
@@ -17,6 +17,11 @@ protected StartILMRequest createTestInstance() {
         return new StartILMRequest();
     }
 
+    @Override
+    protected StartILMRequest mutateInstance(StartILMRequest instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<StartILMRequest> instanceReader() {
         return StartILMRequest::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/StopILMRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/StopILMRequestTests.java
index 5f77498ecc6d..cb40127272b0 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/StopILMRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/StopILMRequestTests.java
@@ -17,6 +17,11 @@ protected StopILMRequest createTestInstance() {
         return new StopILMRequest();
     }
 
+    @Override
+    protected StopILMRequest mutateInstance(StopILMRequest instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<StopILMRequest> instanceReader() {
         return StopILMRequest::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/UnfollowActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/UnfollowActionTests.java
index 2bdc5e6a6e3a..66aacfdc6866 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/UnfollowActionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/UnfollowActionTests.java
@@ -31,6 +31,11 @@ protected UnfollowAction createTestInstance() {
         return UnfollowAction.INSTANCE;
     }
 
+    @Override
+    protected UnfollowAction mutateInstance(UnfollowAction instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<UnfollowAction> instanceReader() {
         return in -> UnfollowAction.INSTANCE;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsTests.java
index 21439d210ab7..6f56fa4023f7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsTests.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRoutingRoleStrategy;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.Strings;
@@ -107,7 +108,10 @@ public void testResultEvaluatedOnWriteIndexAliasWhenExists() {
             .numberOfShards(1)
             .numberOfReplicas(1)
             .build();
-        IndexRoutingTable.Builder routingTable = new IndexRoutingTable.Builder(rolledIndex.getIndex());
+        IndexRoutingTable.Builder routingTable = new IndexRoutingTable.Builder(
+            ShardRoutingRoleStrategy.NO_SHARD_CREATION,
+            rolledIndex.getIndex()
+        );
         routingTable.addShard(
             TestShardRouting.newShardRouting(rolledIndex.getIndex().getName(), 0, "node", null, true, ShardRoutingState.STARTED)
         );
@@ -142,7 +146,10 @@ public void testResultEvaluatedOnOnlyIndexTheAliasPointsToIfWriteIndexIsNull() {
             .numberOfShards(1)
             .numberOfReplicas(1)
             .build();
-        IndexRoutingTable.Builder routingTable = new IndexRoutingTable.Builder(rolledIndex.getIndex());
+        IndexRoutingTable.Builder routingTable = new IndexRoutingTable.Builder(
+            ShardRoutingRoleStrategy.NO_SHARD_CREATION,
+            rolledIndex.getIndex()
+        );
         routingTable.addShard(
             TestShardRouting.newShardRouting(rolledIndex.getIndex().getName(), 0, "node", null, true, ShardRoutingState.STARTED)
         );
@@ -175,7 +182,10 @@ public void testResultEvaluatedOnDataStream() throws IOException {
             .numberOfReplicas(3)
             .build();
 
-        IndexRoutingTable.Builder routingTable = new IndexRoutingTable.Builder(rolledIndexMeta.getIndex());
+        IndexRoutingTable.Builder routingTable = new IndexRoutingTable.Builder(
+            ShardRoutingRoleStrategy.NO_SHARD_CREATION,
+            rolledIndexMeta.getIndex()
+        );
         routingTable.addShard(
             TestShardRouting.newShardRouting(rolledIndexMeta.getIndex().getName(), 0, "node", null, true, ShardRoutingState.STARTED)
         );
@@ -225,7 +235,10 @@ public void testResultReportsMeaningfulMessage() throws IOException {
             .numberOfShards(1)
             .numberOfReplicas(2)
             .build();
-        IndexRoutingTable.Builder routingTable = new IndexRoutingTable.Builder(rolledIndex.getIndex());
+        IndexRoutingTable.Builder routingTable = new IndexRoutingTable.Builder(
+            ShardRoutingRoleStrategy.NO_SHARD_CREATION,
+            rolledIndex.getIndex()
+        );
         routingTable.addShard(
             TestShardRouting.newShardRouting(rolledIndex.getIndex().getName(), 0, "node", null, true, ShardRoutingState.STARTED)
         );
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotActionTests.java
index bda54bb361cd..72f0ba3893d4 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotActionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForSnapshotActionTests.java
@@ -46,7 +46,7 @@ protected Writeable.Reader<WaitForSnapshotAction> instanceReader() {
     }
 
     @Override
-    protected WaitForSnapshotAction mutateInstance(WaitForSnapshotAction instance) throws IOException {
+    protected WaitForSnapshotAction mutateInstance(WaitForSnapshotAction instance) {
         return randomInstance();
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/RemoveIndexLifecyclePolicyRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/RemoveIndexLifecyclePolicyRequestTests.java
index 5ad081a936ca..4bb58540a14e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/RemoveIndexLifecyclePolicyRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/RemoveIndexLifecyclePolicyRequestTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.ilm.action.RemoveIndexLifecyclePolicyAction.Request;
 
-import java.io.IOException;
 import java.util.Arrays;
 
 public class RemoveIndexLifecyclePolicyRequestTests extends AbstractWireSerializingTestCase<Request> {
@@ -45,7 +44,7 @@ protected Writeable.Reader<Request> instanceReader() {
     }
 
     @Override
-    protected Request mutateInstance(Request instance) throws IOException {
+    protected Request mutateInstance(Request instance) {
         String[] indices = instance.indices();
         IndicesOptions indicesOptions = instance.indicesOptions();
         switch (between(0, 1)) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/RemoveIndexLifecyclePolicyResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/RemoveIndexLifecyclePolicyResponseTests.java
index c2a0fc26a35b..0ab431b57449 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/RemoveIndexLifecyclePolicyResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/RemoveIndexLifecyclePolicyResponseTests.java
@@ -32,7 +32,7 @@ protected Writeable.Reader<Response> instanceReader() {
     }
 
     @Override
-    protected Response mutateInstance(Response instance) throws IOException {
+    protected Response mutateInstance(Response instance) {
         List<String> failedIndices = randomValueOtherThan(
             instance.getFailedIndexes(),
             () -> Arrays.asList(generateRandomStringArray(20, 20, false))
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/RetryRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/RetryRequestTests.java
index 6f4e9ad4d1de..fb1c4fb40bf6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/RetryRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/action/RetryRequestTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.ilm.action.RetryAction.Request;
 
-import java.io.IOException;
 import java.util.Arrays;
 
 public class RetryRequestTests extends AbstractWireSerializingTestCase<Request> {
@@ -45,7 +44,7 @@ protected Writeable.Reader<Request> instanceReader() {
     }
 
     @Override
-    protected Request mutateInstance(Request instance) throws IOException {
+    protected Request mutateInstance(Request instance) {
         String[] indices = instance.indices();
         IndicesOptions indicesOptions = instance.indicesOptions();
         switch (between(0, 1)) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CacheInfoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CacheInfoTests.java
index eb8c65c884d8..d269825ef036 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CacheInfoTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CacheInfoTests.java
@@ -34,6 +34,11 @@ protected CacheInfo createTestInstance() {
         return createTestInstance(node);
     }
 
+    @Override
+    protected CacheInfo mutateInstance(CacheInfo instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     static CacheInfo createTestInstance(DiscoveryNode node) {
         return new CacheInfo(node, ByteSizeValue.ofMb(randomLongBetween(1000, 30000)), ByteSizeValue.ofMb(randomLongBetween(0, 1000)));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CancelJobModelSnapshotUpgradeActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CancelJobModelSnapshotUpgradeActionRequestTests.java
index e380b6f45ebe..172ddf52bb3f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CancelJobModelSnapshotUpgradeActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CancelJobModelSnapshotUpgradeActionRequestTests.java
@@ -23,6 +23,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected boolean supportsUnknownFields() {
         return false;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CancelJobModelSnapshotUpgradeActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CancelJobModelSnapshotUpgradeActionResponseTests.java
index d87ea69ff89a..6c8b0685b30d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CancelJobModelSnapshotUpgradeActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CancelJobModelSnapshotUpgradeActionResponseTests.java
@@ -18,6 +18,11 @@ protected Response createTestInstance() {
         return new Response(randomBoolean());
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return CancelJobModelSnapshotUpgradeAction.Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ClearDeploymentCacheActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ClearDeploymentCacheActionRequestTests.java
index b6b6ec36da49..321b2bcadf25 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ClearDeploymentCacheActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ClearDeploymentCacheActionRequestTests.java
@@ -20,4 +20,9 @@ protected Writeable.Reader<ClearDeploymentCacheAction.Request> instanceReader()
     protected ClearDeploymentCacheAction.Request createTestInstance() {
         return new ClearDeploymentCacheAction.Request(randomAlphaOfLength(5));
     }
+
+    @Override
+    protected ClearDeploymentCacheAction.Request mutateInstance(ClearDeploymentCacheAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ClearDeploymentCacheActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ClearDeploymentCacheActionResponseTests.java
index 90455b406952..9360a74e081c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ClearDeploymentCacheActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ClearDeploymentCacheActionResponseTests.java
@@ -20,4 +20,9 @@ protected Writeable.Reader<ClearDeploymentCacheAction.Response> instanceReader()
     protected ClearDeploymentCacheAction.Response createTestInstance() {
         return new ClearDeploymentCacheAction.Response(randomBoolean());
     }
+
+    @Override
+    protected ClearDeploymentCacheAction.Response mutateInstance(ClearDeploymentCacheAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CloseJobActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CloseJobActionRequestTests.java
index 72caf6a0129a..bbca67383ddb 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CloseJobActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CloseJobActionRequestTests.java
@@ -29,6 +29,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected boolean supportsUnknownFields() {
         return false;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CloseJobActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CloseJobActionResponseTests.java
index 6350482fcca2..bf708a2c2aa0 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CloseJobActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CloseJobActionResponseTests.java
@@ -17,6 +17,11 @@ protected Response createTestInstance() {
         return new Response(randomBoolean());
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CreateTrainedModelAssignmentActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CreateTrainedModelAssignmentActionRequestTests.java
index 9e8cb1358f38..71a68a65b797 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CreateTrainedModelAssignmentActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CreateTrainedModelAssignmentActionRequestTests.java
@@ -17,6 +17,11 @@ protected Request createTestInstance() {
         return new Request(StartTrainedModelDeploymentTaskParamsTests.createRandom());
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CreateTrainedModelAssignmentActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CreateTrainedModelAssignmentActionResponseTests.java
index 05a0dc9cb465..7d5c92498db5 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CreateTrainedModelAssignmentActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/CreateTrainedModelAssignmentActionResponseTests.java
@@ -21,6 +21,11 @@ protected Response createTestInstance() {
         return new Response(TrainedModelAssignmentTests.randomInstance());
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DatafeedParamsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DatafeedParamsTests.java
index 2fc067fecc96..ddd47572664d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DatafeedParamsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DatafeedParamsTests.java
@@ -47,6 +47,11 @@ protected StartDatafeedAction.DatafeedParams createTestInstance() {
         return createDatafeedParams();
     }
 
+    @Override
+    protected StartDatafeedAction.DatafeedParams mutateInstance(StartDatafeedAction.DatafeedParams instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<StartDatafeedAction.DatafeedParams> instanceReader() {
         return StartDatafeedAction.DatafeedParams::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteCalendarEventActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteCalendarEventActionRequestTests.java
index cd8e5be946e0..04da4515fe97 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteCalendarEventActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteCalendarEventActionRequestTests.java
@@ -17,6 +17,11 @@ protected Request createTestInstance() {
         return new Request(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteDataFrameAnalyticsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteDataFrameAnalyticsActionRequestTests.java
index dee4a035f7fc..306d5ff67c26 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteDataFrameAnalyticsActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteDataFrameAnalyticsActionRequestTests.java
@@ -26,6 +26,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteDatafeedRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteDatafeedRequestTests.java
index 2eaaea99ea4a..b1b7a8aba8a6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteDatafeedRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteDatafeedRequestTests.java
@@ -17,6 +17,11 @@ protected Request createTestInstance() {
         return new Request(randomAlphaOfLengthBetween(1, 20));
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteExpiredDataActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteExpiredDataActionRequestTests.java
index 7254d792c849..beb5130f7fe1 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteExpiredDataActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteExpiredDataActionRequestTests.java
@@ -29,6 +29,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteExpiredDataActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteExpiredDataActionResponseTests.java
index 00e567a27092..e4ad2a366d23 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteExpiredDataActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteExpiredDataActionResponseTests.java
@@ -17,6 +17,11 @@ protected Response createTestInstance() {
         return new Response(randomBoolean());
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteJobRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteJobRequestTests.java
index 5e9fabada201..10236c9027d0 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteJobRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteJobRequestTests.java
@@ -18,6 +18,11 @@ protected DeleteJobAction.Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected DeleteJobAction.Request mutateInstance(DeleteJobAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<DeleteJobAction.Request> instanceReader() {
         return DeleteJobAction.Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteTrainedModelAliasActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteTrainedModelAliasActionRequestTests.java
index c668881ded5e..011d5b7fe3af 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteTrainedModelAliasActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteTrainedModelAliasActionRequestTests.java
@@ -17,6 +17,11 @@ protected Request createTestInstance() {
         return new Request(randomAlphaOfLength(10), randomAlphaOfLength(10));
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteTrainedModelAssignmentActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteTrainedModelAssignmentActionRequestTests.java
index a77cad4e6850..d8cb3b64daca 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteTrainedModelAssignmentActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteTrainedModelAssignmentActionRequestTests.java
@@ -17,6 +17,11 @@ protected Request createTestInstance() {
         return new Request(randomAlphaOfLength(10));
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteTrainedModelsRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteTrainedModelsRequestTests.java
index 71129120e761..f18fe2aa059e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteTrainedModelsRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/DeleteTrainedModelsRequestTests.java
@@ -17,6 +17,11 @@ protected Request createTestInstance() {
         return new Request(randomAlphaOfLengthBetween(1, 20));
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/EvaluateDataFrameActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/EvaluateDataFrameActionRequestTests.java
index ef3bfbcfdb3f..4cb4879c7f8f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/EvaluateDataFrameActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/EvaluateDataFrameActionRequestTests.java
@@ -70,6 +70,11 @@ protected Request createTestInstance() {
         return new Request().setIndices(indices).setQueryProvider(queryProvider).setEvaluation(evaluation);
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/EvaluateDataFrameActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/EvaluateDataFrameActionResponseTests.java
index 7e0c1bb8753e..c80bfc8fb4f6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/EvaluateDataFrameActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/EvaluateDataFrameActionResponseTests.java
@@ -62,6 +62,11 @@ protected Response createTestInstance() {
         return new Response(evaluationName, metrics);
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ExplainDataFrameAnalyticsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ExplainDataFrameAnalyticsActionResponseTests.java
index bcd76f4b43e5..7940967d4446 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ExplainDataFrameAnalyticsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ExplainDataFrameAnalyticsActionResponseTests.java
@@ -31,6 +31,11 @@ protected Response createTestInstance() {
         return new Response(fieldSelection, memoryEstimation);
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/FlushJobActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/FlushJobActionRequestTests.java
index 665f5b9268f0..4c17a994b92d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/FlushJobActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/FlushJobActionRequestTests.java
@@ -37,6 +37,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ForecastJobActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ForecastJobActionRequestTests.java
index fe8b1ecd04f9..e8d8cd0300af 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ForecastJobActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ForecastJobActionRequestTests.java
@@ -45,6 +45,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ForecastJobActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ForecastJobActionResponseTests.java
index 769c672f4e62..4ea6eed9df93 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ForecastJobActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ForecastJobActionResponseTests.java
@@ -17,6 +17,11 @@ protected Response createTestInstance() {
         return new Response(randomBoolean(), randomAlphaOfLength(20));
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetBucketActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetBucketActionRequestTests.java
index 317fab12c998..890693e33c48 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetBucketActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetBucketActionRequestTests.java
@@ -52,6 +52,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetBucketActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetBucketActionResponseTests.java
index 79480c19e997..3be29ed8ada5 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetBucketActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetBucketActionResponseTests.java
@@ -76,6 +76,11 @@ protected Response createTestInstance() {
         return new Response(buckets);
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetCalendarEventsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetCalendarEventsActionRequestTests.java
index 4aee5003f63f..428a3d33aa74 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetCalendarEventsActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetCalendarEventsActionRequestTests.java
@@ -33,6 +33,11 @@ protected GetCalendarEventsAction.Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected GetCalendarEventsAction.Request mutateInstance(GetCalendarEventsAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<GetCalendarEventsAction.Request> instanceReader() {
         return GetCalendarEventsAction.Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetCalendarsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetCalendarsActionRequestTests.java
index 38a4c3805497..1f330312a35d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetCalendarsActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetCalendarsActionRequestTests.java
@@ -25,6 +25,11 @@ protected GetCalendarsAction.Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected GetCalendarsAction.Request mutateInstance(GetCalendarsAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<GetCalendarsAction.Request> instanceReader() {
         return GetCalendarsAction.Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetCategoriesRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetCategoriesRequestTests.java
index 4e6e84ed8b75..eea1fce83dee 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetCategoriesRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetCategoriesRequestTests.java
@@ -30,6 +30,11 @@ protected GetCategoriesAction.Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected GetCategoriesAction.Request mutateInstance(GetCategoriesAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<GetCategoriesAction.Request> instanceReader() {
         return GetCategoriesAction.Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetCategoriesResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetCategoriesResponseTests.java
index 8aa0229eec96..e17532ba857d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetCategoriesResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetCategoriesResponseTests.java
@@ -26,6 +26,11 @@ protected GetCategoriesAction.Response createTestInstance() {
         return new GetCategoriesAction.Response(queryPage);
     }
 
+    @Override
+    protected GetCategoriesAction.Response mutateInstance(GetCategoriesAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<GetCategoriesAction.Response> instanceReader() {
         return GetCategoriesAction.Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsActionResponseTests.java
index d3531fef674b..15d135f694e4 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsActionResponseTests.java
@@ -53,6 +53,11 @@ protected Response createTestInstance() {
         return new Response(new QueryPage<>(analytics, analytics.size(), Response.RESULTS_FIELD));
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsRequestTests.java
index 19fd64c8d0d5..a5aa0b717d2c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsRequestTests.java
@@ -21,6 +21,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java
index 5ceb9a1f3383..7f5d07240d79 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java
@@ -91,6 +91,11 @@ protected Response createTestInstance() {
         return randomResponse(randomInt(10));
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsRequestTests.java
index 9c6d5a2d8e1e..7f7a276b5c24 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsRequestTests.java
@@ -20,6 +20,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedRunningStateActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedRunningStateActionResponseTests.java
index 26608fae2471..17f402e9fb06 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedRunningStateActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedRunningStateActionResponseTests.java
@@ -31,6 +31,11 @@ protected Response createTestInstance() {
         );
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedStatsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedStatsActionRequestTests.java
index 3f3401f6538a..7736975dffd1 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedStatsActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedStatsActionRequestTests.java
@@ -20,6 +20,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedStatsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedStatsActionResponseTests.java
index 68ed8d19e83c..3d4523745038 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedStatsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedStatsActionResponseTests.java
@@ -70,6 +70,11 @@ protected Response createTestInstance() {
         return result;
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsActionRequestTests.java
index d058193838e1..5a069f9ce0f7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsActionRequestTests.java
@@ -20,6 +20,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsActionResponseTests.java
index 3e5916c8abf6..7ad4b49cab3a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsActionResponseTests.java
@@ -32,6 +32,11 @@ protected Response createTestInstance() {
         return new Response(new QueryPage<>(datafeedList, datafeedList.size(), DatafeedConfig.RESULTS_FIELD));
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDeploymentStatsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDeploymentStatsActionRequestTests.java
index 186754fd0f71..16af1ebfa59b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDeploymentStatsActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDeploymentStatsActionRequestTests.java
@@ -20,4 +20,9 @@ protected Writeable.Reader<GetDeploymentStatsAction.Request> instanceReader() {
     protected GetDeploymentStatsAction.Request createTestInstance() {
         return new GetDeploymentStatsAction.Request(randomAlphaOfLength(5));
     }
+
+    @Override
+    protected GetDeploymentStatsAction.Request mutateInstance(GetDeploymentStatsAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDeploymentStatsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDeploymentStatsActionResponseTests.java
index 4e66fa2dd221..53e956bb9301 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDeploymentStatsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDeploymentStatsActionResponseTests.java
@@ -28,6 +28,11 @@ protected GetDeploymentStatsAction.Response createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected GetDeploymentStatsAction.Response mutateInstance(GetDeploymentStatsAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static GetDeploymentStatsAction.Response createRandom() {
         int numStats = randomIntBetween(0, 2);
         var stats = new ArrayList<AssignmentStats>(numStats);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetFiltersActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetFiltersActionRequestTests.java
index be05071abd49..966e6651eb39 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetFiltersActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetFiltersActionRequestTests.java
@@ -27,6 +27,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetFiltersActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetFiltersActionResponseTests.java
index b5c63df46c0d..5d083c2b0a44 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetFiltersActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetFiltersActionResponseTests.java
@@ -25,6 +25,11 @@ protected Response createTestInstance() {
         return new Response(result);
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetInfluencersActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetInfluencersActionRequestTests.java
index 3eb363effe23..d747c0cfb75d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetInfluencersActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetInfluencersActionRequestTests.java
@@ -50,6 +50,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetInfluencersActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetInfluencersActionResponseTests.java
index 8590a0de81b9..d0f357cbd684 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetInfluencersActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetInfluencersActionResponseTests.java
@@ -40,6 +40,11 @@ protected Response createTestInstance() {
         return new Response(buckets);
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobModelSnapshotsUpgradeStatsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobModelSnapshotsUpgradeStatsActionRequestTests.java
index dfb377b5df06..badbfd8fe959 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobModelSnapshotsUpgradeStatsActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobModelSnapshotsUpgradeStatsActionRequestTests.java
@@ -20,6 +20,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobModelSnapshotsUpgradeStatsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobModelSnapshotsUpgradeStatsActionResponseTests.java
index 8331f73e27b9..e75f97aac6aa 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobModelSnapshotsUpgradeStatsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobModelSnapshotsUpgradeStatsActionResponseTests.java
@@ -34,6 +34,11 @@ protected Response createTestInstance() {
         return new Response(new QueryPage<>(statsList, statsList.size(), GetJobModelSnapshotsUpgradeStatsAction.RESULTS_FIELD));
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobStatsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobStatsActionRequestTests.java
index 0076165b9cf9..452d735206df 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobStatsActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobStatsActionRequestTests.java
@@ -24,6 +24,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobStatsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobStatsActionResponseTests.java
index a432482b8c0d..ac41e1f7195a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobStatsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobStatsActionResponseTests.java
@@ -70,6 +70,11 @@ protected Response createTestInstance() {
         return result;
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobsActionRequestTests.java
index 75c767fe09b4..e1db7657fde4 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobsActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobsActionRequestTests.java
@@ -20,6 +20,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobsActionResponseTests.java
index 46407321d7a4..dc1d0d10a6fe 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobsActionResponseTests.java
@@ -33,6 +33,11 @@ protected Response createTestInstance() {
         return result;
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetModelSnapshotsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetModelSnapshotsActionRequestTests.java
index 6a411130f273..e6d948313e18 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetModelSnapshotsActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetModelSnapshotsActionRequestTests.java
@@ -42,6 +42,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetModelSnapshotsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetModelSnapshotsActionResponseTests.java
index a0cf005428ea..ac42bba9ab17 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetModelSnapshotsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetModelSnapshotsActionResponseTests.java
@@ -29,6 +29,11 @@ protected Response createTestInstance() {
         return new Response(snapshots);
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsActionRequestTests.java
index 19973b31f121..7eb956a027cc 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsActionRequestTests.java
@@ -40,6 +40,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsActionResponseTests.java
index 1d8a85e36c8a..7d17745a23ce 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsActionResponseTests.java
@@ -35,6 +35,11 @@ protected Response createTestInstance() {
         return new Response(snapshots);
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetRecordsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetRecordsActionRequestTests.java
index 9f14ff29b82b..c64a3bb4abf4 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetRecordsActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetRecordsActionRequestTests.java
@@ -50,6 +50,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetRecordsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetRecordsActionResponseTests.java
index a96ce6b5e33b..f874ca296506 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetRecordsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetRecordsActionResponseTests.java
@@ -31,6 +31,11 @@ protected Response createTestInstance() {
         return new Response(snapshots);
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsRequestTests.java
index 34d00f338c85..cb419966f8f3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsRequestTests.java
@@ -38,6 +38,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsActionResponseTests.java
index 97d567939d59..9a71a0b282eb 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetTrainedModelsStatsActionResponseTests.java
@@ -47,6 +47,11 @@ protected Response createTestInstance() {
         return new Response(new QueryPage<>(trainedModelStats, randomLongBetween(listSize, 1000), RESULTS_FIELD));
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     private IngestStats randomIngestStats() {
         List<String> pipelineIds = Stream.generate(() -> randomAlphaOfLength(10)).limit(randomIntBetween(0, 10)).toList();
         return new IngestStats(
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferModelActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferModelActionRequestTests.java
index ad3c92654352..7e4e0c8b1456 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferModelActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferModelActionRequestTests.java
@@ -27,6 +27,8 @@
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.QuestionAnsweringConfigUpdateTests;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RegressionConfigUpdateTests;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ResultsFieldUpdateTests;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.SlimConfigUpdate;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.SlimConfigUpdateTests;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextClassificationConfigUpdate;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextClassificationConfigUpdateTests;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfigUpdate;
@@ -35,6 +37,7 @@
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ZeroShotClassificationConfigUpdateTests;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
@@ -46,14 +49,22 @@ public class InferModelActionRequestTests extends AbstractBWCWireSerializationTe
     @Override
     protected Request createTestInstance() {
         return randomBoolean()
-            ? new Request(
+            ? Request.forDocs(
                 randomAlphaOfLength(10),
                 Stream.generate(InferModelActionRequestTests::randomMap).limit(randomInt(10)).collect(Collectors.toList()),
                 randomInferenceConfigUpdate(),
-                TimeValue.parseTimeValue(randomTimeValue(), null, "test"),
                 randomBoolean()
             )
-            : new Request(randomAlphaOfLength(10), randomMap(), randomInferenceConfigUpdate(), randomBoolean());
+            : Request.forTextInput(
+                randomAlphaOfLength(10),
+                randomInferenceConfigUpdate(),
+                Arrays.asList(generateRandomStringArray(3, 5, false))
+            );
+    }
+
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
     }
 
     private static InferenceConfigUpdate randomInferenceConfigUpdate() {
@@ -109,26 +120,35 @@ protected Request mutateInstanceForVersion(Request instance, Version version) {
                 adjustedUpdate = PassThroughConfigUpdateTests.mutateForVersion(update, version);
             } else if (nlpConfigUpdate instanceof QuestionAnsweringConfigUpdate update) {
                 adjustedUpdate = QuestionAnsweringConfigUpdateTests.mutateForVersion(update, version);
+            } else if (nlpConfigUpdate instanceof SlimConfigUpdate update) {
+                adjustedUpdate = SlimConfigUpdateTests.mutateForVersion(update, version);
             } else {
                 throw new IllegalArgumentException("Unknown update [" + currentUpdate.getName() + "]");
             }
         } else {
             adjustedUpdate = currentUpdate;
         }
-        return version.before(Version.V_8_3_0)
-            ? new Request(
+
+        if (version.before(Version.V_8_3_0)) {
+            return new Request(
                 instance.getModelId(),
-                instance.getObjectsToInfer(),
                 adjustedUpdate,
+                instance.getObjectsToInfer(),
+                null,
                 TimeValue.MAX_VALUE,
                 instance.isPreviouslyLicensed()
-            )
-            : new Request(
+            );
+        } else if (version.before(Version.V_8_7_0)) {
+            return new Request(
                 instance.getModelId(),
-                instance.getObjectsToInfer(),
                 adjustedUpdate,
-                instance.getTimeout(),
+                instance.getObjectsToInfer(),
+                null,
+                instance.getInferenceTimeout(),
                 instance.isPreviouslyLicensed()
             );
+        }
+
+        return instance;
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferModelActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferModelActionResponseTests.java
index 6e592fce9ad5..67d5878223e5 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferModelActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferModelActionResponseTests.java
@@ -53,6 +53,11 @@ protected Response createTestInstance() {
         );
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     private static InferenceResults randomInferenceResult(String resultType) {
         switch (resultType) {
             case ClassificationInferenceResults.NAME:
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferTrainedModelDeploymentRequestsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferTrainedModelDeploymentRequestsTests.java
index 0e2cec9ad8b2..e417235e4a09 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferTrainedModelDeploymentRequestsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferTrainedModelDeploymentRequestsTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ZeroShotClassificationConfigUpdateTests;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -37,10 +38,10 @@ protected InferTrainedModelDeploymentAction.Request createTestInstance() {
         boolean createQueryStringRequest = randomBoolean();
 
         if (createQueryStringRequest) {
-            return new InferTrainedModelDeploymentAction.Request(
+            return InferTrainedModelDeploymentAction.Request.forTextInput(
                 randomAlphaOfLength(4),
                 randomBoolean() ? null : randomInferenceConfigUpdate(),
-                randomAlphaOfLength(6),
+                Arrays.asList(generateRandomStringArray(4, 7, false)),
                 randomBoolean() ? null : TimeValue.parseTimeValue(randomTimeValue(), "timeout")
             );
         } else {
@@ -49,7 +50,7 @@ protected InferTrainedModelDeploymentAction.Request createTestInstance() {
                 () -> randomMap(1, 3, () -> Tuple.tuple(randomAlphaOfLength(7), randomAlphaOfLength(7)))
             );
 
-            return new InferTrainedModelDeploymentAction.Request(
+            return InferTrainedModelDeploymentAction.Request.forDocs(
                 randomAlphaOfLength(4),
                 randomBoolean() ? null : randomInferenceConfigUpdate(),
                 docs,
@@ -58,6 +59,11 @@ protected InferTrainedModelDeploymentAction.Request createTestInstance() {
         }
     }
 
+    @Override
+    protected InferTrainedModelDeploymentAction.Request mutateInstance(InferTrainedModelDeploymentAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
         List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferTrainedModelDeploymentResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferTrainedModelDeploymentResponseTests.java
index 003a40077c7a..75673281a213 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferTrainedModelDeploymentResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferTrainedModelDeploymentResponseTests.java
@@ -16,6 +16,8 @@
 import org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingResultsTests;
 import org.junit.Before;
 
+import java.util.List;
+
 public class InferTrainedModelDeploymentResponseTests extends AbstractBWCWireSerializationTestCase<
     InferTrainedModelDeploymentAction.Response> {
 
@@ -45,7 +47,19 @@ protected Writeable.Reader<InferTrainedModelDeploymentAction.Response> instanceR
 
     @Override
     protected InferTrainedModelDeploymentAction.Response createTestInstance() {
-        return new InferTrainedModelDeploymentAction.Response(TextEmbeddingResultsTests.createRandomResults(), randomLongBetween(1, 200));
+        return new InferTrainedModelDeploymentAction.Response(
+            List.of(
+                TextEmbeddingResultsTests.createRandomResults(),
+                TextEmbeddingResultsTests.createRandomResults(),
+                TextEmbeddingResultsTests.createRandomResults(),
+                TextEmbeddingResultsTests.createRandomResults()
+            )
+        );
+    }
+
+    @Override
+    protected InferTrainedModelDeploymentAction.Response mutateInstance(InferTrainedModelDeploymentAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
     }
 
     @Override
@@ -53,8 +67,8 @@ protected InferTrainedModelDeploymentAction.Response mutateInstanceForVersion(
         InferTrainedModelDeploymentAction.Response instance,
         Version version
     ) {
-        if (version.before(Version.V_8_6_0)) {
-            return new InferTrainedModelDeploymentAction.Response(instance.getResults(), 0);
+        if (version.before(Version.V_8_6_1)) {
+            return new InferTrainedModelDeploymentAction.Response(instance.getResults().subList(0, 1));
         }
 
         return instance;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/JobParamsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/JobParamsTests.java
index 641413618afc..291e9856b0a1 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/JobParamsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/JobParamsTests.java
@@ -39,6 +39,11 @@ protected OpenJobAction.JobParams createTestInstance() {
         return createJobParams();
     }
 
+    @Override
+    protected OpenJobAction.JobParams mutateInstance(OpenJobAction.JobParams instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<OpenJobAction.JobParams> instanceReader() {
         return OpenJobAction.JobParams::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/MlMemoryActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/MlMemoryActionRequestTests.java
index eb4716bc0517..c8d38db87f61 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/MlMemoryActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/MlMemoryActionRequestTests.java
@@ -21,4 +21,9 @@ protected Writeable.Reader<MlMemoryAction.Request> instanceReader() {
     protected MlMemoryAction.Request createTestInstance() {
         return new MlMemoryAction.Request(randomAlphaOfLength(20));
     }
+
+    @Override
+    protected MlMemoryAction.Request mutateInstance(MlMemoryAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/MlMemoryActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/MlMemoryActionResponseTests.java
index 2875ab7c8020..b49a05050c73 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/MlMemoryActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/MlMemoryActionResponseTests.java
@@ -52,4 +52,9 @@ protected MlMemoryAction.Response createTestInstance() {
         }
         return new MlMemoryAction.Response(ClusterName.DEFAULT, nodes, failures);
     }
+
+    @Override
+    protected MlMemoryAction.Response mutateInstance(MlMemoryAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/MlMemoryStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/MlMemoryStatsTests.java
index d5093e7f4adc..99934672a068 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/MlMemoryStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/MlMemoryStatsTests.java
@@ -34,6 +34,11 @@ protected MlMemoryStats createTestInstance() {
         return createTestInstance(node);
     }
 
+    @Override
+    protected MlMemoryStats mutateInstance(MlMemoryStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     static MlMemoryStats createTestInstance(DiscoveryNode node) {
         return new MlMemoryStats(
             node,
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/OpenJobActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/OpenJobActionRequestTests.java
index 2c2b3d5fc170..a80d07d7ee50 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/OpenJobActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/OpenJobActionRequestTests.java
@@ -18,6 +18,11 @@ protected Request createTestInstance() {
         return new Request(JobParamsTests.createJobParams());
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PersistJobActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PersistJobActionRequestTests.java
index 2830abf077f0..c73d03ada9df 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PersistJobActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PersistJobActionRequestTests.java
@@ -20,4 +20,9 @@ protected Writeable.Reader<PersistJobAction.Request> instanceReader() {
     protected PersistJobAction.Request createTestInstance() {
         return new PersistJobAction.Request(randomAlphaOfLength(10));
     }
+
+    @Override
+    protected PersistJobAction.Request mutateInstance(PersistJobAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PersistJobActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PersistJobActionResponseTests.java
index 4d7c90fb9699..06516d13df2c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PersistJobActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PersistJobActionResponseTests.java
@@ -19,4 +19,9 @@ protected Writeable.Reader<PersistJobAction.Response> instanceReader() {
     protected PersistJobAction.Response createTestInstance() {
         return new PersistJobAction.Response(randomBoolean());
     }
+
+    @Override
+    protected PersistJobAction.Response mutateInstance(PersistJobAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PostCalendarEventActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PostCalendarEventActionRequestTests.java
index d86ac5c7f66f..757e56606216 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PostCalendarEventActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PostCalendarEventActionRequestTests.java
@@ -27,6 +27,11 @@ protected PostCalendarEventsAction.Request createTestInstance() {
         return createTestInstance(id);
     }
 
+    @Override
+    protected PostCalendarEventsAction.Request mutateInstance(PostCalendarEventsAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<PostCalendarEventsAction.Request> instanceReader() {
         return PostCalendarEventsAction.Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PostDataActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PostDataActionRequestTests.java
index aaf4c123a64b..54bb58808f31 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PostDataActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PostDataActionRequestTests.java
@@ -31,6 +31,11 @@ protected PostDataAction.Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected PostDataAction.Request mutateInstance(PostDataAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<PostDataAction.Request> instanceReader() {
         return PostDataAction.Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PostDataActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PostDataActionResponseTests.java
index 07ae7a6726f2..f94ec42b2abb 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PostDataActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PostDataActionResponseTests.java
@@ -19,6 +19,11 @@ protected PostDataAction.Response createTestInstance() {
         return new PostDataAction.Response(counts);
     }
 
+    @Override
+    protected PostDataAction.Response mutateInstance(PostDataAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<PostDataAction.Response> instanceReader() {
         return PostDataAction.Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PostDataFlushRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PostDataFlushRequestTests.java
index 6e88959c167d..5b818d9f474a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PostDataFlushRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PostDataFlushRequestTests.java
@@ -28,6 +28,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PostDataFlushResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PostDataFlushResponseTests.java
index 79973e2c0425..193c6ed5af0b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PostDataFlushResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PostDataFlushResponseTests.java
@@ -19,6 +19,11 @@ protected Response createTestInstance() {
         return new Response(randomBoolean(), Instant.now());
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PreviewDataFrameAnalyticsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PreviewDataFrameAnalyticsActionRequestTests.java
index dd34dbfb7a22..13f62b331af1 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PreviewDataFrameAnalyticsActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PreviewDataFrameAnalyticsActionRequestTests.java
@@ -41,4 +41,9 @@ protected Writeable.Reader<Request> instanceReader() {
     protected Request createTestInstance() {
         return new Request(DataFrameAnalyticsConfigTests.createRandom(randomAlphaOfLength(10)));
     }
+
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PreviewDataFrameAnalyticsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PreviewDataFrameAnalyticsActionResponseTests.java
index 4fff8daff671..4d62514a82ba 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PreviewDataFrameAnalyticsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PreviewDataFrameAnalyticsActionResponseTests.java
@@ -25,6 +25,11 @@ protected Response createTestInstance() {
         );
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     private static Map<String, Object> randomHashMap(String... keys) {
         return Arrays.stream(keys).collect(Collectors.toMap(Function.identity(), k -> randomAlphaOfLength(10)));
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PreviewDatafeedActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PreviewDatafeedActionRequestTests.java
index fc112c18b4d6..0a8f9bd84925 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PreviewDatafeedActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PreviewDatafeedActionRequestTests.java
@@ -49,6 +49,11 @@ protected Request createTestInstance() {
         };
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutCalendarActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutCalendarActionRequestTests.java
index c847b39c704b..e087fbf5ce04 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutCalendarActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutCalendarActionRequestTests.java
@@ -21,6 +21,11 @@ protected PutCalendarAction.Request createTestInstance() {
         return new PutCalendarAction.Request(CalendarTests.testInstance(calendarId));
     }
 
+    @Override
+    protected PutCalendarAction.Request mutateInstance(PutCalendarAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<PutCalendarAction.Request> instanceReader() {
         return PutCalendarAction.Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionRequestTests.java
index 03268257a868..8a97e0806e62 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionRequestTests.java
@@ -64,6 +64,11 @@ protected Request createTestInstance() {
         return new Request(DataFrameAnalyticsConfigTests.createRandom(id));
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionResponseTests.java
index 6542d92594f2..4f3d5ab1a938 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionResponseTests.java
@@ -36,6 +36,11 @@ protected Response createTestInstance() {
         return new Response(DataFrameAnalyticsConfigTests.createRandom(DataFrameAnalyticsConfigTests.randomValidId()));
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionRequestTests.java
index 869b0dc8394c..27ea6847d4c7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionRequestTests.java
@@ -34,6 +34,11 @@ protected Request createTestInstance() {
         return new Request(DatafeedConfigTests.createRandomizedDatafeedConfig(randomAlphaOfLength(10), datafeedId, 3600));
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionResponseTests.java
index 2a661263c6c2..873cfe0c6913 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionResponseTests.java
@@ -30,6 +30,11 @@ protected Response createTestInstance() {
         return new Response(datafeedConfig.build());
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutFilterActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutFilterActionRequestTests.java
index 1d4dd5e524b0..e59ccfb9eb81 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutFilterActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutFilterActionRequestTests.java
@@ -21,6 +21,11 @@ protected Request createTestInstance() {
         return new PutFilterAction.Request(MlFilterTests.createRandom(filterId));
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutFilterActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutFilterActionResponseTests.java
index 7982851fe262..ef7aeb59b73e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutFilterActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutFilterActionResponseTests.java
@@ -21,6 +21,11 @@ protected PutFilterAction.Response createTestInstance() {
         return new PutFilterAction.Response(MlFilterTests.createRandom());
     }
 
+    @Override
+    protected PutFilterAction.Response mutateInstance(PutFilterAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<PutFilterAction.Response> instanceReader() {
         return PutFilterAction.Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutJobActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutJobActionRequestTests.java
index be08fcf55d63..60e2f9f1d463 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutJobActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutJobActionRequestTests.java
@@ -24,6 +24,11 @@ protected Request createTestInstance() {
         return new Request(jobConfiguration);
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutJobActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutJobActionResponseTests.java
index 1cad68adb87e..bf902d70e50d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutJobActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutJobActionResponseTests.java
@@ -22,6 +22,11 @@ protected Response createTestInstance() {
         return new Response(builder.build());
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelActionRequestTests.java
index 90eb05a53a50..bb77e7f2dd5a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelActionRequestTests.java
@@ -28,6 +28,11 @@ protected Request createTestInstance() {
         );
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return (in) -> {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelActionResponseTests.java
index 5d48a952c34f..4bdfdfd7e687 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelActionResponseTests.java
@@ -27,6 +27,11 @@ protected Response createTestInstance() {
         );
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return (in) -> {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelAliasActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelAliasActionRequestTests.java
index 9cbbe7df6b98..03f77b96ceb7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelAliasActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelAliasActionRequestTests.java
@@ -30,6 +30,11 @@ protected Request createTestInstance() {
         return new Request(modelAlias, randomAlphaOfLength(10), randomBoolean());
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelDefinitionPartActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelDefinitionPartActionRequestTests.java
index ab24553b21cf..a1ce5c258e26 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelDefinitionPartActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutTrainedModelDefinitionPartActionRequestTests.java
@@ -29,6 +29,11 @@ protected Request createTestInstance() {
         );
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testValidate() {
         Request badRequest = new Request(randomAlphaOfLength(10), new BytesArray(randomAlphaOfLength(10)), -1, -1, -1);
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ResetJobRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ResetJobRequestTests.java
index fba559fec8c2..50f899a1e3f9 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ResetJobRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ResetJobRequestTests.java
@@ -20,6 +20,11 @@ protected ResetJobAction.Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected ResetJobAction.Request mutateInstance(ResetJobAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<ResetJobAction.Request> instanceReader() {
         return ResetJobAction.Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotActionRequestTests.java
index 0bef9bc06ed2..13ce3620b369 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotActionRequestTests.java
@@ -28,6 +28,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotActionResponseTests.java
index 5d621092dec7..c96cf6f47a6c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotActionResponseTests.java
@@ -18,6 +18,11 @@ protected Response createTestInstance() {
         return new RevertModelSnapshotAction.Response(ModelSnapshotTests.createRandomized());
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/SemanticSearchActionKnnQueryOptionsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/SemanticSearchActionKnnQueryOptionsTests.java
deleted file mode 100644
index f24ec35ca352..000000000000
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/SemanticSearchActionKnnQueryOptionsTests.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.core.ml.action;
-
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.test.AbstractWireSerializingTestCase;
-import org.elasticsearch.xcontent.NamedXContentRegistry;
-import org.junit.Before;
-
-import java.util.List;
-
-import static java.util.Collections.emptyList;
-import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.sameInstance;
-
-public class SemanticSearchActionKnnQueryOptionsTests extends AbstractWireSerializingTestCase<SemanticSearchAction.KnnQueryOptions> {
-
-    private NamedWriteableRegistry namedWriteableRegistry;
-    private NamedXContentRegistry namedXContentRegistry;
-
-    @Before
-    public void registerNamedXContents() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
-        namedXContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
-        namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
-    }
-
-    @Override
-    protected NamedXContentRegistry xContentRegistry() {
-        return namedXContentRegistry;
-    }
-
-    @Override
-    protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        return namedWriteableRegistry;
-    }
-
-    public static SemanticSearchAction.KnnQueryOptions randomInstance() {
-        int k = randomIntBetween(1, 10);
-        var knnOptions = new SemanticSearchAction.KnnQueryOptions(randomAlphaOfLength(5), k, randomIntBetween(k, 100));
-        if (randomBoolean()) {
-            knnOptions.boost(randomFloat());
-        }
-
-        int numFilters = randomIntBetween(0, 3);
-        for (int i = 0; i < numFilters; i++) {
-            knnOptions.addFilterQueries(List.of(QueryBuilders.termQuery(randomAlphaOfLength(5), randomAlphaOfLength(10))));
-        }
-        return knnOptions;
-    }
-
-    @Override
-    protected Writeable.Reader<SemanticSearchAction.KnnQueryOptions> instanceReader() {
-        return SemanticSearchAction.KnnQueryOptions::new;
-    }
-
-    @Override
-    protected SemanticSearchAction.KnnQueryOptions createTestInstance() {
-        return randomInstance();
-    }
-
-    public void testToKnnSearchBuilder() {
-        var knnOptions = new SemanticSearchAction.KnnQueryOptions("foo", 5, 100);
-        knnOptions.boost(20.0f);
-        var termsQuery = QueryBuilders.termQuery("foo", "bar");
-        knnOptions.addFilterQueries(List.of(termsQuery));
-
-        var knnSearch = knnOptions.toKnnSearchBuilder(new float[] { 0.1f, 0.2f });
-        assertEquals(5, knnSearch.k());
-        var knnQuery = knnSearch.toQueryBuilder();
-        assertEquals(100, knnQuery.numCands());
-        assertEquals("foo", knnQuery.getFieldName());
-        assertThat(knnQuery.filterQueries(), contains(sameInstance(termsQuery)));
-        assertEquals(20.0f, knnQuery.boost(), 0.001);
-    }
-}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/SemanticSearchActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/SemanticSearchActionRequestTests.java
deleted file mode 100644
index c74ac85f550e..000000000000
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/SemanticSearchActionRequestTests.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.core.ml.action;
-
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.index.query.TermsQueryBuilder;
-import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.search.fetch.StoredFieldsContext;
-import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
-import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
-import org.elasticsearch.test.AbstractWireSerializingTestCase;
-import org.elasticsearch.xcontent.NamedXContentRegistry;
-import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider;
-import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfigUpdateTests;
-import org.junit.Before;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static java.util.Collections.emptyList;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.hasSize;
-
-public class SemanticSearchActionRequestTests extends AbstractWireSerializingTestCase<SemanticSearchAction.Request> {
-    private NamedWriteableRegistry namedWriteableRegistry;
-    private NamedXContentRegistry namedXContentRegistry;
-
-    @Before
-    public void registerNamedXContents() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
-        var xContents = new ArrayList<>(searchModule.getNamedXContents());
-        xContents.addAll(new MlInferenceNamedXContentProvider().getNamedXContentParsers());
-        namedXContentRegistry = new NamedXContentRegistry(xContents);
-
-        var writeables = new ArrayList<>(searchModule.getNamedWriteables());
-        writeables.addAll(new MlInferenceNamedXContentProvider().getNamedWriteables());
-        namedWriteableRegistry = new NamedWriteableRegistry(writeables);
-    }
-
-    @Override
-    protected NamedXContentRegistry xContentRegistry() {
-        return namedXContentRegistry;
-    }
-
-    @Override
-    protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        return namedWriteableRegistry;
-    }
-
-    @Override
-    protected Writeable.Reader<SemanticSearchAction.Request> instanceReader() {
-        return SemanticSearchAction.Request::new;
-    }
-
-    @Override
-    protected SemanticSearchAction.Request createTestInstance() {
-        return new SemanticSearchAction.Request(
-            generateRandomStringArray(1, 5, false),
-            randomBoolean() ? null : randomAlphaOfLength(5),
-            randomAlphaOfLength(5),
-            randomAlphaOfLength(5),
-            randomBoolean() ? null : new TermsQueryBuilder("foo", "bar"),
-            SemanticSearchActionKnnQueryOptionsTests.randomInstance(),
-            TextEmbeddingConfigUpdateTests.randomUpdate(),
-            randomBoolean() ? null : TimeValue.timeValueSeconds(randomIntBetween(1, 10)),
-            randomBoolean() ? null : FetchSourceContext.of(randomBoolean()),
-            randomBoolean() ? null : List.of(new FieldAndFormat("foo", null)),
-            randomBoolean() ? null : List.of(new FieldAndFormat("foo", null)),
-            randomBoolean() ? null : StoredFieldsContext.fromList(List.of("A", "B")),
-            randomBoolean() ? -1 : randomIntBetween(1, 10)
-        );
-    }
-
-    public void testValidate() {
-        var validAction = createTestInstance();
-        assertNull(validAction.validate());
-
-        var action = new SemanticSearchAction.Request(
-            new String[] { "foo" },
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            -1
-        );
-        var validation = action.validate();
-        assertNotNull(validation);
-        assertThat(validation.validationErrors(), hasSize(3));
-        assertThat(validation.validationErrors().get(0), containsString("query_string cannot be null"));
-        assertThat(validation.validationErrors().get(1), containsString("model_id cannot be null"));
-        assertThat(validation.validationErrors().get(2), containsString("knn cannot be null"));
-    }
-}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/SetUpgradeModeActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/SetUpgradeModeActionRequestTests.java
index c358fa41229a..896ddcc8f884 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/SetUpgradeModeActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/SetUpgradeModeActionRequestTests.java
@@ -18,6 +18,11 @@ protected Request createTestInstance() {
         return new Request(randomBoolean());
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected boolean supportsUnknownFields() {
         return false;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsActionTaskParamsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsActionTaskParamsTests.java
index 7024322718b7..d46a5c33f32f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsActionTaskParamsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsActionTaskParamsTests.java
@@ -28,6 +28,11 @@ protected StartDataFrameAnalyticsAction.TaskParams createTestInstance() {
         return new StartDataFrameAnalyticsAction.TaskParams(randomAlphaOfLength(10), randomVersion(random()), randomBoolean());
     }
 
+    @Override
+    protected StartDataFrameAnalyticsAction.TaskParams mutateInstance(StartDataFrameAnalyticsAction.TaskParams instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<StartDataFrameAnalyticsAction.TaskParams> instanceReader() {
         return StartDataFrameAnalyticsAction.TaskParams::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsRequestTests.java
index cef0cec8b32e..4bfe7062e930 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsRequestTests.java
@@ -29,6 +29,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedActionRequestTests.java
index 9cd3508fab65..9cd123fd5c66 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedActionRequestTests.java
@@ -21,6 +21,11 @@ protected Request createTestInstance() {
         return new Request(DatafeedParamsTests.createDatafeedParams());
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentRequestTests.java
index ea59544a1c9e..04e672420194 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentRequestTests.java
@@ -45,6 +45,11 @@ protected Request createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static Request createRandom() {
         Request request = new Request(randomAlphaOfLength(10));
         if (randomBoolean()) {
@@ -221,7 +226,7 @@ public void testValidate_GivenLowPriorityAndMultipleAllocations() {
 
     public void testDefaults() {
         Request request = new Request(randomAlphaOfLength(10));
-        assertThat(request.getTimeout(), equalTo(TimeValue.timeValueSeconds(20)));
+        assertThat(request.getTimeout(), equalTo(TimeValue.timeValueSeconds(30)));
         assertThat(request.getWaitForState(), equalTo(AllocationStatus.State.STARTED));
         assertThat(request.getNumberOfAllocations(), equalTo(1));
         assertThat(request.getThreadsPerAllocation(), equalTo(1));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentTaskParamsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentTaskParamsTests.java
index 3b8ad17daa58..c9b66e304e77 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentTaskParamsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentTaskParamsTests.java
@@ -33,6 +33,11 @@ protected TaskParams createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected TaskParams mutateInstance(TaskParams instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static StartTrainedModelDeploymentAction.TaskParams createRandom() {
         return new TaskParams(
             randomAlphaOfLength(10),
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopDataFrameAnalyticsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopDataFrameAnalyticsActionResponseTests.java
index e4a839f6259b..373006207543 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopDataFrameAnalyticsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopDataFrameAnalyticsActionResponseTests.java
@@ -17,6 +17,11 @@ protected Response createTestInstance() {
         return new Response(randomBoolean());
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopDataFrameAnalyticsRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopDataFrameAnalyticsRequestTests.java
index 921c2821ece9..38e83038b6b3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopDataFrameAnalyticsRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopDataFrameAnalyticsRequestTests.java
@@ -43,6 +43,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedActionRequestTests.java
index 6d9b5f9bc8ed..183f2613770b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedActionRequestTests.java
@@ -32,6 +32,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected boolean supportsUnknownFields() {
         return false;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopTrainedModelDeploymentRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopTrainedModelDeploymentRequestTests.java
index 95bd2b020a94..07249aeb3ccb 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopTrainedModelDeploymentRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopTrainedModelDeploymentRequestTests.java
@@ -37,4 +37,9 @@ protected Request createTestInstance() {
         }
         return request;
     }
+
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/TrainedModelCacheInfoRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/TrainedModelCacheInfoRequestTests.java
index e879dc62a4c6..1d656398d36f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/TrainedModelCacheInfoRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/TrainedModelCacheInfoRequestTests.java
@@ -35,4 +35,9 @@ protected TrainedModelCacheInfoAction.Request createTestInstance() {
         }
         return new TrainedModelCacheInfoAction.Request(nodes);
     }
+
+    @Override
+    protected TrainedModelCacheInfoAction.Request mutateInstance(TrainedModelCacheInfoAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/TrainedModelCacheInfoResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/TrainedModelCacheInfoResponseTests.java
index 2964a47e9c3f..ec83c8fc2be7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/TrainedModelCacheInfoResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/TrainedModelCacheInfoResponseTests.java
@@ -53,4 +53,9 @@ protected TrainedModelCacheInfoAction.Response createTestInstance() {
         }
         return new TrainedModelCacheInfoAction.Response(ClusterName.DEFAULT, nodes, failures);
     }
+
+    @Override
+    protected TrainedModelCacheInfoAction.Response mutateInstance(TrainedModelCacheInfoAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateCalendarJobActionResquestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateCalendarJobActionResquestTests.java
index b49abe4900ac..f5810487ccc0 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateCalendarJobActionResquestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateCalendarJobActionResquestTests.java
@@ -20,6 +20,11 @@ protected UpdateCalendarJobAction.Request createTestInstance() {
         );
     }
 
+    @Override
+    protected UpdateCalendarJobAction.Request mutateInstance(UpdateCalendarJobAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<UpdateCalendarJobAction.Request> instanceReader() {
         return UpdateCalendarJobAction.Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateDataFrameAnalyticsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateDataFrameAnalyticsActionRequestTests.java
index 02ed1945e4a3..5597bf72b965 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateDataFrameAnalyticsActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateDataFrameAnalyticsActionRequestTests.java
@@ -29,6 +29,11 @@ protected Request createTestInstance() {
         return new Request(DataFrameAnalyticsConfigUpdateTests.randomUpdate(randomValidId()));
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Request doParseInstance(XContentParser parser) {
         return Request.parseRequest(null, parser);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateDatafeedActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateDatafeedActionRequestTests.java
index df9351f8e863..bfc78fb69ccb 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateDatafeedActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateDatafeedActionRequestTests.java
@@ -34,6 +34,11 @@ protected Request createTestInstance() {
         return new Request(DatafeedUpdateTests.createRandomized(datafeedId));
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateFilterActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateFilterActionRequestTests.java
index ceff685104fc..f5c1453e90f7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateFilterActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateFilterActionRequestTests.java
@@ -34,6 +34,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateJobActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateJobActionRequestTests.java
index c4af615665c4..01fba1287293 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateJobActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateJobActionRequestTests.java
@@ -30,6 +30,11 @@ protected UpdateJobAction.Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected UpdateJobAction.Request mutateInstance(UpdateJobAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<UpdateJobAction.Request> instanceReader() {
         return UpdateJobAction.Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateModelSnapshotActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateModelSnapshotActionRequestTests.java
index ceaacf2aed73..137225983a7f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateModelSnapshotActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateModelSnapshotActionRequestTests.java
@@ -30,6 +30,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateModelSnapshotActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateModelSnapshotActionResponseTests.java
index c4325d82195f..c7c3c31ff609 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateModelSnapshotActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateModelSnapshotActionResponseTests.java
@@ -18,6 +18,11 @@ protected Response createTestInstance() {
         return new Response(ModelSnapshotTests.createRandomized());
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateProcessActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateProcessActionRequestTests.java
index 6a7864636643..d9c03ae7840a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateProcessActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateProcessActionRequestTests.java
@@ -52,6 +52,11 @@ protected UpdateProcessAction.Request createTestInstance() {
         );
     }
 
+    @Override
+    protected UpdateProcessAction.Request mutateInstance(UpdateProcessAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<UpdateProcessAction.Request> instanceReader() {
         return UpdateProcessAction.Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelAssignmentRoutingInfoActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelAssignmentRoutingInfoActionRequestTests.java
index 866e6526034d..2057501c9563 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelAssignmentRoutingInfoActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelAssignmentRoutingInfoActionRequestTests.java
@@ -18,6 +18,11 @@ protected Request createTestInstance() {
         return new Request(randomAlphaOfLength(10), randomAlphaOfLength(10), RoutingInfoUpdateTests.randomInstance());
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelDeploymentRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelDeploymentRequestTests.java
index 5abda31b483d..092e758d4f3f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelDeploymentRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateTrainedModelDeploymentRequestTests.java
@@ -37,6 +37,11 @@ protected Request createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static Request createRandom() {
         Request request = new Request(randomAlphaOfLength(10));
         if (randomBoolean()) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpgradeJobModelSnapshotRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpgradeJobModelSnapshotRequestTests.java
index 306841b2c63f..ec9ad15e9e99 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpgradeJobModelSnapshotRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpgradeJobModelSnapshotRequestTests.java
@@ -23,6 +23,11 @@ protected Request createTestInstance() {
         );
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpgradeJobModelSnapshotResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpgradeJobModelSnapshotResponseTests.java
index 40a4a3956dc3..adf9036c0cb8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpgradeJobModelSnapshotResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpgradeJobModelSnapshotResponseTests.java
@@ -18,6 +18,11 @@ protected Response createTestInstance() {
         return new Response(randomBoolean(), randomBoolean() ? null : randomAlphaOfLength(10));
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ValidateDetectorActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ValidateDetectorActionRequestTests.java
index 739324d7ec5c..eea7cd334afb 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ValidateDetectorActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ValidateDetectorActionRequestTests.java
@@ -25,6 +25,11 @@ protected Request createTestInstance() {
         return new Request(detector.build());
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ValidateJobConfigActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ValidateJobConfigActionRequestTests.java
index b16c63f95db0..40785c0043cc 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ValidateJobConfigActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/ValidateJobConfigActionRequestTests.java
@@ -32,6 +32,11 @@ protected Request createTestInstance() {
         return new Request(buildJobBuilder(randomValidJobId(), new Date()).build());
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationTests.java
index 3a91a8c62512..480300721ea8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationTests.java
@@ -26,6 +26,11 @@ protected Annotation createTestInstance() {
         return randomAnnotation(randomBoolean() ? randomAlphaOfLengthBetween(10, 30) : null);
     }
 
+    @Override
+    protected Annotation mutateInstance(Annotation instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static Annotation randomAnnotation(String jobId) {
         return new Annotation.Builder().setAnnotation(randomAlphaOfLengthBetween(100, 1000))
             .setCreateTime(new Date(randomNonNegativeLong()))
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/calendars/CalendarTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/calendars/CalendarTests.java
index bdf6f115c1c0..4d946cd44243 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/calendars/CalendarTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/calendars/CalendarTests.java
@@ -44,6 +44,11 @@ protected Calendar createTestInstance() {
         return testInstance();
     }
 
+    @Override
+    protected Calendar mutateInstance(Calendar instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Calendar> instanceReader() {
         return Calendar::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEventTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEventTests.java
index f16e389cceac..0a78b7b6d6f2 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEventTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEventTests.java
@@ -36,6 +36,11 @@ protected ScheduledEvent createTestInstance() {
         return createScheduledEvent(randomAlphaOfLengthBetween(1, 20));
     }
 
+    @Override
+    protected ScheduledEvent mutateInstance(ScheduledEvent instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<ScheduledEvent> instanceReader() {
         return ScheduledEvent::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/AggProviderWireSerializationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/AggProviderWireSerializationTests.java
index 69743f7e005a..3d82f19fc63a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/AggProviderWireSerializationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/AggProviderWireSerializationTests.java
@@ -39,6 +39,11 @@ protected AggProvider createTestInstance() {
         return createRandomValidAggProvider();
     }
 
+    @Override
+    protected AggProvider mutateInstance(AggProvider instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<AggProvider> instanceReader() {
         return AggProvider::fromStream;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/ChunkingConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/ChunkingConfigTests.java
index 1bc5765a7d8c..c0054e118407 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/ChunkingConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/ChunkingConfigTests.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
 import org.elasticsearch.xcontent.XContentParser;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -69,7 +68,7 @@ private static TimeValue randomPositiveSecondsMinutesHours() {
     }
 
     @Override
-    protected ChunkingConfig mutateInstance(ChunkingConfig instance) throws IOException {
+    protected ChunkingConfig mutateInstance(ChunkingConfig instance) {
         ChunkingConfig.Mode mode = instance.getMode();
         TimeValue timeSpan = instance.getTimeSpan();
         switch (between(0, 1)) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigBuilderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigBuilderTests.java
index 2430a95c1228..6a722896970e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigBuilderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigBuilderTests.java
@@ -132,6 +132,11 @@ protected DatafeedConfig.Builder createTestInstance() {
         );
     }
 
+    @Override
+    protected DatafeedConfig.Builder mutateInstance(DatafeedConfig.Builder instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
         SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java
index fb08c0286fe5..00cdb5c96b30 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java
@@ -953,7 +953,7 @@ private static DatafeedConfig createDatafeedWithComposite(CompositeAggregationBu
     }
 
     @Override
-    protected DatafeedConfig mutateInstance(DatafeedConfig instance) throws IOException {
+    protected DatafeedConfig mutateInstance(DatafeedConfig instance) {
         DatafeedConfig.Builder builder = new DatafeedConfig.Builder(instance);
         switch (between(0, 12)) {
             case 0:
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedTimingStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedTimingStatsTests.java
index 6b8bbb9d32c2..9aa1d34d1051 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedTimingStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedTimingStatsTests.java
@@ -60,7 +60,7 @@ protected ToXContent.Params getToXContentParams() {
     }
 
     @Override
-    protected DatafeedTimingStats mutateInstance(DatafeedTimingStats instance) throws IOException {
+    protected DatafeedTimingStats mutateInstance(DatafeedTimingStats instance) {
         String jobId = instance.getJobId();
         long searchCount = instance.getSearchCount();
         long bucketCount = instance.getBucketCount();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DelayedDataCheckConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DelayedDataCheckConfigTests.java
index 9893c7a3d7ab..a98dbb8e37d8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DelayedDataCheckConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DelayedDataCheckConfigTests.java
@@ -11,8 +11,6 @@
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
 import org.elasticsearch.xcontent.XContentParser;
 
-import java.io.IOException;
-
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.core.Is.is;
@@ -68,7 +66,7 @@ public static DelayedDataCheckConfig createRandomizedConfig(long bucketSpanMilli
     }
 
     @Override
-    protected DelayedDataCheckConfig mutateInstance(DelayedDataCheckConfig instance) throws IOException {
+    protected DelayedDataCheckConfig mutateInstance(DelayedDataCheckConfig instance) {
         boolean enabled = instance.isEnabled();
         TimeValue timeWindow = instance.getCheckWindow();
         switch (between(0, 1)) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/SearchIntervalTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/SearchIntervalTests.java
index 4681c040435d..a00e74d4af54 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/SearchIntervalTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/SearchIntervalTests.java
@@ -22,6 +22,11 @@ protected SearchInterval createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected SearchInterval mutateInstance(SearchInterval instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static SearchInterval createRandom() {
         long start = randomNonNegativeLong();
         return new SearchInterval(start, randomLongBetween(start, Long.MAX_VALUE));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java
index 4decfe8f82c7..aac6d48d4b1e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java
@@ -95,6 +95,11 @@ protected DataFrameAnalyticsConfig createTestInstance() {
         return createRandom(randomValidId(), lenient);
     }
 
+    @Override
+    protected DataFrameAnalyticsConfig mutateInstance(DataFrameAnalyticsConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected DataFrameAnalyticsConfig mutateInstanceForVersion(DataFrameAnalyticsConfig instance, Version version) {
         DataFrameAnalyticsConfig.Builder builder = new DataFrameAnalyticsConfig.Builder(instance).setSource(
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigUpdateTests.java
index fc12e16afb16..ff0349a29fd7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigUpdateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigUpdateTests.java
@@ -35,6 +35,11 @@ protected DataFrameAnalyticsConfigUpdate createTestInstance() {
         return randomUpdate(randomValidId());
     }
 
+    @Override
+    protected DataFrameAnalyticsConfigUpdate mutateInstance(DataFrameAnalyticsConfigUpdate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<DataFrameAnalyticsConfigUpdate> instanceReader() {
         return DataFrameAnalyticsConfigUpdate::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsDestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsDestTests.java
index b06ed310c349..43daa56bd696 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsDestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsDestTests.java
@@ -25,6 +25,11 @@ protected DataFrameAnalyticsDest createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected DataFrameAnalyticsDest mutateInstance(DataFrameAnalyticsDest instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static DataFrameAnalyticsDest createRandom() {
         String index = randomAlphaOfLength(10);
         String resultsField = randomBoolean() ? null : randomAlphaOfLength(10);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsSourceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsSourceTests.java
index 208c6a3d609e..3779996fc308 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsSourceTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsSourceTests.java
@@ -54,6 +54,11 @@ protected DataFrameAnalyticsSource createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected DataFrameAnalyticsSource mutateInstance(DataFrameAnalyticsSource instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static DataFrameAnalyticsSource createRandom() {
         String[] index = generateRandomStringArray(10, 10, false, false);
         QueryProvider queryProvider = null;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskStateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskStateTests.java
index 63b2da3542bc..7c7d686f27ca 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskStateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskStateTests.java
@@ -19,6 +19,11 @@ protected DataFrameAnalyticsTaskState createTestInstance() {
         return new DataFrameAnalyticsTaskState(randomFrom(DataFrameAnalyticsState.values()), randomLong(), randomAlphaOfLength(10));
     }
 
+    @Override
+    protected DataFrameAnalyticsTaskState mutateInstance(DataFrameAnalyticsTaskState instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<DataFrameAnalyticsTaskState> instanceReader() {
         return DataFrameAnalyticsTaskState::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParamsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParamsTests.java
index 04b26555b236..c2ee3b735a5a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParamsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParamsTests.java
@@ -52,6 +52,11 @@ protected BoostedTreeParams createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected BoostedTreeParams mutateInstance(BoostedTreeParams instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static BoostedTreeParams createRandom() {
         return BoostedTreeParams.builder()
             .setLambda(randomBoolean() ? null : randomDoubleBetween(0.0, Double.MAX_VALUE, true))
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
index 578d35a2a8bf..51670461bdff 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java
@@ -74,6 +74,11 @@ protected Classification createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Classification mutateInstance(Classification instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected NamedXContentRegistry xContentRegistry() {
         List<NamedXContentRegistry.Entry> namedXContent = new ArrayList<>();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetectionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetectionTests.java
index 62e65842f5f6..3402556d0a6e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetectionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetectionTests.java
@@ -35,6 +35,11 @@ protected OutlierDetection createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected OutlierDetection mutateInstance(OutlierDetection instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static OutlierDetection createRandom() {
         Integer numberNeighbors = randomBoolean() ? null : randomIntBetween(1, 20);
         OutlierDetection.Method method = randomBoolean() ? null : randomFrom(OutlierDetection.Method.values());
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/RegressionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/RegressionTests.java
index 1671ffb4e410..80fc0ef208d3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/RegressionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/RegressionTests.java
@@ -63,6 +63,11 @@ protected Regression createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Regression mutateInstance(Regression instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected NamedXContentRegistry xContentRegistry() {
         List<NamedXContentRegistry.Entry> namedXContent = new ArrayList<>();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyResultTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyResultTests.java
index 33fb1a6aa438..50b9a41070cc 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyResultTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyResultTests.java
@@ -41,6 +41,11 @@ protected Result createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Result mutateInstance(Result instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Result> instanceReader() {
         return Result::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyTests.java
index 6b82f4daf697..18663b77d9ca 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AccuracyTests.java
@@ -44,6 +44,11 @@ protected Accuracy createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Accuracy mutateInstance(Accuracy instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Accuracy> instanceReader() {
         return Accuracy::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AucRocResultTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AucRocResultTests.java
index 376b7c975113..68cbe71adefc 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AucRocResultTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AucRocResultTests.java
@@ -37,6 +37,11 @@ protected Result createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Result mutateInstance(Result instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Result> instanceReader() {
         return Result::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AucRocTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AucRocTests.java
index 300997cfdd79..16bf33c7bebb 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AucRocTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/AucRocTests.java
@@ -24,6 +24,11 @@ protected AucRoc createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected AucRoc mutateInstance(AucRoc instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<AucRoc> instanceReader() {
         return AucRoc::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/ClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/ClassificationTests.java
index 46747b900e99..bf00f8763d92 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/ClassificationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/ClassificationTests.java
@@ -92,6 +92,11 @@ protected Classification createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Classification mutateInstance(Classification instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Classification> instanceReader() {
         return Classification::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrixResultTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrixResultTests.java
index 2809ef78e39e..0c3041269fcd 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrixResultTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrixResultTests.java
@@ -50,6 +50,11 @@ protected Result createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Result mutateInstance(Result instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Result> instanceReader() {
         return Result::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrixTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrixTests.java
index 94a51829cc5d..cb3455b3d5b2 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrixTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/MulticlassConfusionMatrixTests.java
@@ -49,6 +49,11 @@ protected MulticlassConfusionMatrix createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected MulticlassConfusionMatrix mutateInstance(MulticlassConfusionMatrix instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<MulticlassConfusionMatrix> instanceReader() {
         return MulticlassConfusionMatrix::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PerClassSingleValueTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PerClassSingleValueTests.java
index 057f3e4deb23..8236f42c3481 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PerClassSingleValueTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PerClassSingleValueTests.java
@@ -28,4 +28,9 @@ protected Writeable.Reader<PerClassSingleValue> instanceReader() {
     protected PerClassSingleValue createTestInstance() {
         return new PerClassSingleValue(randomAlphaOfLength(10), randomDouble());
     }
+
+    @Override
+    protected PerClassSingleValue mutateInstance(PerClassSingleValue instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PrecisionResultTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PrecisionResultTests.java
index cafd1781b84f..054e76eb8e43 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PrecisionResultTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PrecisionResultTests.java
@@ -41,6 +41,11 @@ protected Result createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Result mutateInstance(Result instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Result> instanceReader() {
         return Result::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PrecisionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PrecisionTests.java
index a8202aeb66a4..d4261d81fea2 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PrecisionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/PrecisionTests.java
@@ -43,6 +43,11 @@ protected Precision createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Precision mutateInstance(Precision instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Precision> instanceReader() {
         return Precision::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/RecallResultTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/RecallResultTests.java
index b56f4c6b6313..cd1595ff68dd 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/RecallResultTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/RecallResultTests.java
@@ -41,6 +41,11 @@ protected Result createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Result mutateInstance(Result instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Result> instanceReader() {
         return Result::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/RecallTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/RecallTests.java
index ee6fa980b38f..5f446083612d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/RecallTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/classification/RecallTests.java
@@ -42,6 +42,11 @@ protected Recall createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Recall mutateInstance(Recall instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Recall> instanceReader() {
         return Recall::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/AucRocTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/AucRocTests.java
index d1c04b2f0da6..70152f5ba0cc 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/AucRocTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/AucRocTests.java
@@ -24,6 +24,11 @@ protected AucRoc createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected AucRoc mutateInstance(AucRoc instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<AucRoc> instanceReader() {
         return AucRoc::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/ConfusionMatrixTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/ConfusionMatrixTests.java
index df3c21b1a41b..acbd647f7bfa 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/ConfusionMatrixTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/ConfusionMatrixTests.java
@@ -33,6 +33,11 @@ protected ConfusionMatrix createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected ConfusionMatrix mutateInstance(ConfusionMatrix instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<ConfusionMatrix> instanceReader() {
         return ConfusionMatrix::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/OutlierDetectionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/OutlierDetectionTests.java
index 5da426d5ed51..cb1ccac87487 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/OutlierDetectionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/OutlierDetectionTests.java
@@ -80,6 +80,11 @@ protected OutlierDetection createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected OutlierDetection mutateInstance(OutlierDetection instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<OutlierDetection> instanceReader() {
         return OutlierDetection::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/PrecisionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/PrecisionTests.java
index 213af2ea8d4d..299aa76f05fd 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/PrecisionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/PrecisionTests.java
@@ -33,6 +33,11 @@ protected Precision createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Precision mutateInstance(Precision instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Precision> instanceReader() {
         return Precision::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/RecallTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/RecallTests.java
index d255594be30d..fb4ab46675ec 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/RecallTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/outlierdetection/RecallTests.java
@@ -33,6 +33,11 @@ protected Recall createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Recall mutateInstance(Recall instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Recall> instanceReader() {
         return Recall::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/HuberTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/HuberTests.java
index 1cd2f417332f..8e7f4ddd3625 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/HuberTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/HuberTests.java
@@ -32,6 +32,11 @@ protected Huber createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Huber mutateInstance(Huber instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Huber> instanceReader() {
         return Huber::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/MeanSquaredErrorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/MeanSquaredErrorTests.java
index 740531bc6832..c6c0d00dd240 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/MeanSquaredErrorTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/MeanSquaredErrorTests.java
@@ -32,6 +32,11 @@ protected MeanSquaredError createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected MeanSquaredError mutateInstance(MeanSquaredError instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<MeanSquaredError> instanceReader() {
         return MeanSquaredError::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/MeanSquaredLogarithmicErrorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/MeanSquaredLogarithmicErrorTests.java
index c7f23991b25c..beb39e46fa5f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/MeanSquaredLogarithmicErrorTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/MeanSquaredLogarithmicErrorTests.java
@@ -32,6 +32,11 @@ protected MeanSquaredLogarithmicError createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected MeanSquaredLogarithmicError mutateInstance(MeanSquaredLogarithmicError instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<MeanSquaredLogarithmicError> instanceReader() {
         return MeanSquaredLogarithmicError::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RSquaredTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RSquaredTests.java
index 222921a0d4b5..644979379703 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RSquaredTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RSquaredTests.java
@@ -33,6 +33,11 @@ protected RSquared createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected RSquared mutateInstance(RSquared instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<RSquared> instanceReader() {
         return RSquared::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RegressionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RegressionTests.java
index 653ff9f5c717..06b86dd81fde 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RegressionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/regression/RegressionTests.java
@@ -67,6 +67,11 @@ protected Regression createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Regression mutateInstance(Regression instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Regression> instanceReader() {
         return Regression::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/explain/FieldSelectionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/explain/FieldSelectionTests.java
index caaed7e2c4f3..a3b050f1ecc8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/explain/FieldSelectionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/explain/FieldSelectionTests.java
@@ -29,6 +29,11 @@ protected FieldSelection createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected FieldSelection mutateInstance(FieldSelection instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected FieldSelection doParseInstance(XContentParser parser) throws IOException {
         return FieldSelection.PARSER.apply(parser, null);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/explain/MemoryEstimationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/explain/MemoryEstimationTests.java
index 259f2815ef90..ad26ae421aca 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/explain/MemoryEstimationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/explain/MemoryEstimationTests.java
@@ -31,6 +31,11 @@ protected MemoryEstimation createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected MemoryEstimation mutateInstance(MemoryEstimation instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<MemoryEstimation> instanceReader() {
         return MemoryEstimation::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStatsTests.java
index 6b9135d06617..d4f1d8d32740 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStatsTests.java
@@ -57,6 +57,11 @@ protected ClassificationStats createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected ClassificationStats mutateInstance(ClassificationStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static ClassificationStats createRandom() {
         return new ClassificationStats(
             randomAlphaOfLength(10),
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/HyperparametersTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/HyperparametersTests.java
index b903ab8f845c..c451ed1507ea 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/HyperparametersTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/HyperparametersTests.java
@@ -48,6 +48,11 @@ protected Hyperparameters createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Hyperparameters mutateInstance(Hyperparameters instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static Hyperparameters createRandom() {
         return new Hyperparameters(
             randomAlphaOfLength(10),
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/TimingStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/TimingStatsTests.java
index e241d80e38f4..99b1e19605f6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/TimingStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/TimingStatsTests.java
@@ -49,6 +49,11 @@ protected TimingStats createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected TimingStats mutateInstance(TimingStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static TimingStats createRandom() {
         return new TimingStats(TimeValue.timeValueMillis(randomNonNegativeLong()), TimeValue.timeValueMillis(randomNonNegativeLong()));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ValidationLossTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ValidationLossTests.java
index c8bc41ebd3ee..4c30708302f9 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ValidationLossTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ValidationLossTests.java
@@ -51,6 +51,11 @@ protected ValidationLoss createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected ValidationLoss mutateInstance(ValidationLoss instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static ValidationLoss createRandom() {
         return new ValidationLoss(randomAlphaOfLength(10), randomList(5, FoldValuesTests::createRandom));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCountsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCountsTests.java
index 0d764fc841f0..3d1384f2a55e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCountsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCountsTests.java
@@ -56,6 +56,11 @@ protected DataCounts createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected DataCounts mutateInstance(DataCounts instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static DataCounts createRandom() {
         return new DataCounts(randomAlphaOfLength(10), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/FoldValuesTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/FoldValuesTests.java
index 5bfbd085a187..5b18fa5c02c9 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/FoldValuesTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/FoldValuesTests.java
@@ -43,6 +43,11 @@ protected FoldValues createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected FoldValues mutateInstance(FoldValues instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static FoldValues createRandom() {
         int valuesSize = randomIntBetween(0, 10);
         double[] values = new double[valuesSize];
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/MemoryUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/MemoryUsageTests.java
index 0f2a08c48bca..c185e49c133f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/MemoryUsageTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/MemoryUsageTests.java
@@ -64,6 +64,11 @@ protected MemoryUsage createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected MemoryUsage mutateInstance(MemoryUsage instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testZeroUsage() {
         MemoryUsage memoryUsage = new MemoryUsage("zero_usage_job");
         String asJson = Strings.toString(memoryUsage);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStatsTests.java
index f29356d30e87..7a749c3754d0 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStatsTests.java
@@ -57,6 +57,11 @@ protected OutlierDetectionStats createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected OutlierDetectionStats mutateInstance(OutlierDetectionStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static OutlierDetectionStats createRandom() {
         return new OutlierDetectionStats(
             randomAlphaOfLength(10),
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/ParametersTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/ParametersTests.java
index c8aab4756d1c..884516b12da8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/ParametersTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/ParametersTests.java
@@ -48,6 +48,11 @@ protected Parameters createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Parameters mutateInstance(Parameters instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static Parameters createRandom() {
 
         return new Parameters(
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/TimingStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/TimingStatsTests.java
index c33b30b77da6..659357c0e22c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/TimingStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/TimingStatsTests.java
@@ -49,6 +49,11 @@ protected TimingStats createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected TimingStats mutateInstance(TimingStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static TimingStats createRandom() {
         return new TimingStats(TimeValue.timeValueMillis(randomNonNegativeLong()));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/HyperparametersTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/HyperparametersTests.java
index 61a0aba7b88f..9e1bb5ba3380 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/HyperparametersTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/HyperparametersTests.java
@@ -48,6 +48,11 @@ protected Hyperparameters createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Hyperparameters mutateInstance(Hyperparameters instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static Hyperparameters createRandom() {
         return new Hyperparameters(
             randomDouble(),
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStatsTests.java
index e35e48c03a04..0d5215cac91a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStatsTests.java
@@ -57,6 +57,11 @@ protected RegressionStats createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected RegressionStats mutateInstance(RegressionStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static RegressionStats createRandom() {
         return new RegressionStats(
             randomAlphaOfLength(10),
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/TimingStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/TimingStatsTests.java
index 868019e1fead..279abab379b3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/TimingStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/TimingStatsTests.java
@@ -49,6 +49,11 @@ protected TimingStats createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected TimingStats mutateInstance(TimingStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static TimingStats createRandom() {
         return new TimingStats(TimeValue.timeValueMillis(randomNonNegativeLong()), TimeValue.timeValueMillis(randomNonNegativeLong()));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/ValidationLossTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/ValidationLossTests.java
index caaaed42d5c5..d08337c6a6a7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/ValidationLossTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/ValidationLossTests.java
@@ -51,6 +51,11 @@ protected ValidationLoss createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected ValidationLoss mutateInstance(ValidationLoss instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static ValidationLoss createRandom() {
         return new ValidationLoss(randomAlphaOfLength(10), randomList(5, FoldValuesTests::createRandom));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/InferenceConfigItemTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/InferenceConfigItemTestCase.java
index 37b37940a578..fc4015d3f557 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/InferenceConfigItemTestCase.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/InferenceConfigItemTestCase.java
@@ -84,7 +84,7 @@ protected NamedWriteableRegistry getNamedWriteableRegistry() {
     protected List<Version> bwcVersions() {
         T obj = createTestInstance();
         return getAllBWCVersions(Version.CURRENT).stream()
-            .filter(v -> v.onOrAfter(obj.getMinimalSupportedVersion()))
+            .filter(v -> v.transportVersion.onOrAfter(obj.getMinimalSupportedVersion()))
             .collect(Collectors.toList());
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfigTests.java
index bbff114de5d4..d6e1e1eebb8f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfigTests.java
@@ -127,6 +127,11 @@ protected TrainedModelConfig createTestInstance() {
         return createTestInstance(randomAlphaOfLength(10), lenient).build();
     }
 
+    @Override
+    protected TrainedModelConfig mutateInstance(TrainedModelConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<TrainedModelConfig> instanceReader() {
         return TrainedModelConfig::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelDefinitionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelDefinitionTests.java
index 83d9247bd22f..5f4c0a7be7a6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelDefinitionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelDefinitionTests.java
@@ -314,6 +314,11 @@ protected TrainedModelDefinition createTestInstance() {
         return createRandomBuilder().build();
     }
 
+    @Override
+    protected TrainedModelDefinition mutateInstance(TrainedModelDefinition instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<TrainedModelDefinition> instanceReader() {
         return TrainedModelDefinition::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelInputTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelInputTests.java
index d338de484d4a..549834327634 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelInputTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelInputTests.java
@@ -49,6 +49,11 @@ protected TrainedModelInput createTestInstance() {
         return createRandomInput();
     }
 
+    @Override
+    protected TrainedModelInput mutateInstance(TrainedModelInput instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<TrainedModelInput> instanceReader() {
         return TrainedModelInput::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/AllocationStatusTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/AllocationStatusTests.java
index 42f0f6be91e0..9f9266130d07 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/AllocationStatusTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/AllocationStatusTests.java
@@ -36,6 +36,11 @@ protected AllocationStatus createTestInstance() {
         return randomInstance();
     }
 
+    @Override
+    protected AllocationStatus mutateInstance(AllocationStatus instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testCalculateState() {
         int targetAllocation = randomIntBetween(2, 10);
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStatsTests.java
index 0b31dff8c207..61f14c3a2ba0 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/AssignmentStatsTests.java
@@ -212,4 +212,9 @@ protected Writeable.Reader<AssignmentStats> instanceReader() {
     protected AssignmentStats createTestInstance() {
         return randomDeploymentStats();
     }
+
+    @Override
+    protected AssignmentStats mutateInstance(AssignmentStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingInfoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingInfoTests.java
index b20ff44b2555..28ebf8b2445c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingInfoTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingInfoTests.java
@@ -32,6 +32,11 @@ protected RoutingInfo createTestInstance() {
         return randomInstance();
     }
 
+    @Override
+    protected RoutingInfo mutateInstance(RoutingInfo instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static RoutingInfo randomInstance() {
         return new RoutingInfo(
             randomIntBetween(1, 10),
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingInfoUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingInfoUpdateTests.java
index 3adb67943125..3af84814bf7c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingInfoUpdateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingInfoUpdateTests.java
@@ -24,6 +24,11 @@ protected RoutingInfoUpdate createTestInstance() {
         return randomInstance();
     }
 
+    @Override
+    protected RoutingInfoUpdate mutateInstance(RoutingInfoUpdate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static RoutingInfoUpdate randomInstance() {
         if (randomBoolean()) {
             return RoutingInfoUpdate.updateNumberOfAllocations(randomIntBetween(1, Integer.MAX_VALUE));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingStateAndReasonTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingStateAndReasonTests.java
index 254c551432ef..b7b042b7efb0 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingStateAndReasonTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/RoutingStateAndReasonTests.java
@@ -33,4 +33,9 @@ protected Writeable.Reader<RoutingStateAndReason> instanceReader() {
     protected RoutingStateAndReason createTestInstance() {
         return randomInstance();
     }
+
+    @Override
+    protected RoutingStateAndReason mutateInstance(RoutingStateAndReason instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java
index bb6d1904a99a..124373a3c21d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/TrainedModelAssignmentTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction;
@@ -19,17 +20,16 @@
 import org.elasticsearch.xpack.core.ml.stats.CountAccumulator;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
 import java.util.stream.Stream;
 
 import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
@@ -63,6 +63,11 @@ protected TrainedModelAssignment createTestInstance() {
         return randomInstance();
     }
 
+    @Override
+    protected TrainedModelAssignment mutateInstance(TrainedModelAssignment instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testBuilderAddingExistingRoute() {
         TrainedModelAssignment.Builder assignment = TrainedModelAssignment.Builder.empty(randomParams());
         String addingNode = "new-node";
@@ -157,62 +162,62 @@ public void testCalculateAndSetAssignmentState_GivenStoppingAssignment() {
         );
     }
 
-    public void testSelectRandomStartedNodeWeighedOnAllocations_GivenNoStartedAllocations() {
+    public void testselectRandomStartedNodeWeighedOnAllocationsForNRequests_GivenNoStartedAllocations() {
         TrainedModelAssignment.Builder builder = TrainedModelAssignment.Builder.empty(randomTaskParams(5));
         builder.addRoutingEntry("node-1", new RoutingInfo(4, 4, RoutingState.STARTING, ""));
         builder.addRoutingEntry("node-2", new RoutingInfo(1, 1, RoutingState.STOPPED, ""));
         TrainedModelAssignment assignment = builder.build();
 
-        assertThat(assignment.selectRandomStartedNodeWeighedOnAllocations().isEmpty(), is(true));
+        assertThat(assignment.selectRandomStartedNodesWeighedOnAllocationsForNRequests(1).isEmpty(), is(true));
     }
 
-    public void testSelectRandomStartedNodeWeighedOnAllocations_GivenSingleStartedNode() {
+    public void testselectRandomStartedNodeWeighedOnAllocationsForNRequests_GivenSingleStartedNode() {
         TrainedModelAssignment.Builder builder = TrainedModelAssignment.Builder.empty(randomTaskParams(5));
         builder.addRoutingEntry("node-1", new RoutingInfo(4, 4, RoutingState.STARTED, ""));
         TrainedModelAssignment assignment = builder.build();
 
-        Optional<String> node = assignment.selectRandomStartedNodeWeighedOnAllocations();
+        var nodes = assignment.selectRandomStartedNodesWeighedOnAllocationsForNRequests(1);
 
-        assertThat(node.isPresent(), is(true));
-        assertThat(node.get(), equalTo("node-1"));
+        assertThat(nodes, hasSize(1));
+        assertThat(nodes.get(0), equalTo(new Tuple<>("node-1", 1)));
     }
 
-    public void testSelectRandomStartedNodeWeighedOnAllocations_GivenMultipleStartedNodes() {
+    public void testSelectRandomStartedNodeWeighedOnAllocationsForNRequests_GivenMultipleStartedNodes() {
         TrainedModelAssignment.Builder builder = TrainedModelAssignment.Builder.empty(randomTaskParams(6));
         builder.addRoutingEntry("node-1", new RoutingInfo(1, 1, RoutingState.STARTED, ""));
         builder.addRoutingEntry("node-2", new RoutingInfo(2, 2, RoutingState.STARTED, ""));
         builder.addRoutingEntry("node-3", new RoutingInfo(3, 3, RoutingState.STARTED, ""));
         TrainedModelAssignment assignment = builder.build();
 
-        final long selectionCount = 10000;
+        final int selectionCount = 10000;
         final CountAccumulator countsPerNodeAccumulator = new CountAccumulator();
-        for (int i = 0; i < selectionCount; i++) {
-            Optional<String> node = assignment.selectRandomStartedNodeWeighedOnAllocations();
-            assertThat(node.isPresent(), is(true));
-            countsPerNodeAccumulator.add(node.get(), 1L);
-        }
+        var nodes = assignment.selectRandomStartedNodesWeighedOnAllocationsForNRequests(selectionCount);
 
-        Map<String, Long> countsPerNode = countsPerNodeAccumulator.asMap();
-        assertThat(countsPerNode.keySet(), contains("node-1", "node-2", "node-3"));
-        assertThat(countsPerNode.get("node-1") + countsPerNode.get("node-2") + countsPerNode.get("node-3"), equalTo(selectionCount));
+        assertThat(nodes, hasSize(3));
+        assertThat(nodes.stream().mapToInt(Tuple::v2).sum(), equalTo(selectionCount));
+        var asMap = new HashMap<String, Integer>();
+        for (var node : nodes) {
+            asMap.put(node.v1(), node.v2());
+        }
 
-        assertValueWithinPercentageOfExpectedRatio(countsPerNode.get("node-1"), selectionCount, 1.0 / 6.0, 0.2);
-        assertValueWithinPercentageOfExpectedRatio(countsPerNode.get("node-2"), selectionCount, 2.0 / 6.0, 0.2);
-        assertValueWithinPercentageOfExpectedRatio(countsPerNode.get("node-3"), selectionCount, 3.0 / 6.0, 0.2);
+        assertValueWithinPercentageOfExpectedRatio(asMap.get("node-1"), selectionCount, 1.0 / 6.0, 0.2);
+        assertValueWithinPercentageOfExpectedRatio(asMap.get("node-2"), selectionCount, 2.0 / 6.0, 0.2);
+        assertValueWithinPercentageOfExpectedRatio(asMap.get("node-3"), selectionCount, 3.0 / 6.0, 0.2);
     }
 
-    public void testSelectRandomStartedNodeWeighedOnAllocations_GivenMultipleStartedNodesWithZeroAllocations() {
+    public void testselectRandomStartedNodeWeighedOnAllocationsForNRequests_GivenMultipleStartedNodesWithZeroAllocations() {
         TrainedModelAssignment.Builder builder = TrainedModelAssignment.Builder.empty(randomTaskParams(6));
         builder.addRoutingEntry("node-1", new RoutingInfo(0, 0, RoutingState.STARTED, ""));
         builder.addRoutingEntry("node-2", new RoutingInfo(0, 0, RoutingState.STARTED, ""));
         builder.addRoutingEntry("node-3", new RoutingInfo(0, 0, RoutingState.STARTED, ""));
         TrainedModelAssignment assignment = builder.build();
-        final long selectionCount = 1000;
-        Set<String> selectedNodes = new HashSet<>();
-        for (int i = 0; i < selectionCount; i++) {
-            Optional<String> selectedNode = assignment.selectRandomStartedNodeWeighedOnAllocations();
-            assertThat(selectedNode.isPresent(), is(true));
-            selectedNodes.add(selectedNode.get());
+        final int selectionCount = 1000;
+        var nodeCounts = assignment.selectRandomStartedNodesWeighedOnAllocationsForNRequests(selectionCount);
+        assertThat(nodeCounts, hasSize(3));
+
+        var selectedNodes = new HashSet<String>();
+        for (var node : nodeCounts) {
+            selectedNodes.add(node.v1());
         }
 
         assertThat(selectedNodes, contains("node-1", "node-2", "node-3"));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/CustomWordEmbeddingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/CustomWordEmbeddingTests.java
index 27876b8b3ce9..be746aa21024 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/CustomWordEmbeddingTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/CustomWordEmbeddingTests.java
@@ -23,6 +23,11 @@ protected CustomWordEmbedding createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected CustomWordEmbedding mutateInstance(CustomWordEmbedding instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static CustomWordEmbedding createRandom() {
         int quantileSize = randomIntBetween(1, 10);
         int internalQuantSize = randomIntBetween(1, 10);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/FrequencyEncodingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/FrequencyEncodingTests.java
index b4821dad4a5d..ec138ae5a946 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/FrequencyEncodingTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/FrequencyEncodingTests.java
@@ -35,6 +35,11 @@ protected FrequencyEncoding createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected FrequencyEncoding mutateInstance(FrequencyEncoding instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static FrequencyEncoding createRandom() {
         return createRandom(randomBoolean() ? null : randomBoolean());
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/MultiTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/MultiTests.java
index 8edb1e25c4d1..1eb8cacc4015 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/MultiTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/MultiTests.java
@@ -41,6 +41,11 @@ protected Multi createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Multi mutateInstance(Multi instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static Multi createRandom() {
         return createRandom(randomBoolean() ? null : randomBoolean());
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/NGramTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/NGramTests.java
index 932aee55932b..5361d46e6791 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/NGramTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/NGramTests.java
@@ -37,6 +37,11 @@ protected NGram createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected NGram mutateInstance(NGram instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static NGram createRandom() {
         return createRandom(randomBoolean() ? randomBoolean() : null);
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/OneHotEncodingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/OneHotEncodingTests.java
index c0c625683118..56310ed19377 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/OneHotEncodingTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/OneHotEncodingTests.java
@@ -35,6 +35,11 @@ protected OneHotEncoding createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected OneHotEncoding mutateInstance(OneHotEncoding instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static OneHotEncoding createRandom() {
         return createRandom(randomBoolean() ? randomBoolean() : null);
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/PreProcessingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/PreProcessingTests.java
index 7b847c113e3a..265ce3804a26 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/PreProcessingTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/PreProcessingTests.java
@@ -38,6 +38,11 @@ protected boolean supportsUnknownFields() {
         return lenient;
     }
 
+    @Override
+    protected T mutateInstance(T instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
         List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/TargetMeanEncodingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/TargetMeanEncodingTests.java
index 3a581b4b61d0..af423be4c0fc 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/TargetMeanEncodingTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/preprocessing/TargetMeanEncodingTests.java
@@ -35,6 +35,11 @@ protected TargetMeanEncoding createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected TargetMeanEncoding mutateInstance(TargetMeanEncoding instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static TargetMeanEncoding createRandom() {
         return createRandom(randomBoolean() ? randomBoolean() : null);
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationFeatureImportanceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationFeatureImportanceTests.java
index 8af64360e2c8..de160d458466 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationFeatureImportanceTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationFeatureImportanceTests.java
@@ -37,6 +37,11 @@ protected ClassificationFeatureImportance createTestInstance() {
         return createRandomInstance();
     }
 
+    @Override
+    protected ClassificationFeatureImportance mutateInstance(ClassificationFeatureImportance instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @SuppressWarnings("unchecked")
     public static ClassificationFeatureImportance createRandomInstance() {
         Supplier<Object> classNameGenerator = randomFrom(
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResultsTests.java
index 22c7b3f64af2..492678f03556 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResultsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResultsTests.java
@@ -132,6 +132,11 @@ protected ClassificationInferenceResults createTestInstance() {
         return createRandomResults();
     }
 
+    @Override
+    protected ClassificationInferenceResults mutateInstance(ClassificationInferenceResults instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<ClassificationInferenceResults> instanceReader() {
         return ClassificationInferenceResults::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/FillMaskResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/FillMaskResultsTests.java
index ae7238504ef4..4119d56be7b5 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/FillMaskResultsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/FillMaskResultsTests.java
@@ -52,6 +52,11 @@ protected FillMaskResults createTestInstance() {
         return createRandomResults();
     }
 
+    @Override
+    protected FillMaskResults mutateInstance(FillMaskResults instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @SuppressWarnings("unchecked")
     public void testAsMap() {
         FillMaskResults testInstance = createTestInstance();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/NerResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/NerResultsTests.java
index c4b4eb9e9c0f..68c69ff67fa4 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/NerResultsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/NerResultsTests.java
@@ -53,6 +53,11 @@ protected NerResults createTestInstance() {
         return createRandomResults();
     }
 
+    @Override
+    protected NerResults mutateInstance(NerResults instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @SuppressWarnings("unchecked")
     public void testAsMap() {
         NerResults testInstance = createTestInstance();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/NlpClassificationInferenceResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/NlpClassificationInferenceResultsTests.java
index 50085dd968f1..d47730bd579a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/NlpClassificationInferenceResultsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/NlpClassificationInferenceResultsTests.java
@@ -68,6 +68,11 @@ protected NlpClassificationInferenceResults createTestInstance() {
         return createRandomResults();
     }
 
+    @Override
+    protected NlpClassificationInferenceResults mutateInstance(NlpClassificationInferenceResults instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<NlpClassificationInferenceResults> instanceReader() {
         return NlpClassificationInferenceResults::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/PyTorchPassThroughResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/PyTorchPassThroughResultsTests.java
index 517bd9c210b0..c2386010a8f6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/PyTorchPassThroughResultsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/PyTorchPassThroughResultsTests.java
@@ -41,6 +41,11 @@ protected PyTorchPassThroughResults createTestInstance() {
         return createRandomResults();
     }
 
+    @Override
+    protected PyTorchPassThroughResults mutateInstance(PyTorchPassThroughResults instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testAsMap() {
         PyTorchPassThroughResults testInstance = createTestInstance();
         Map<String, Object> asMap = testInstance.asMap();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/QuestionAnsweringInferenceResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/QuestionAnsweringInferenceResultsTests.java
index fb1688dcaf7c..947f8732aba6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/QuestionAnsweringInferenceResultsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/QuestionAnsweringInferenceResultsTests.java
@@ -72,6 +72,11 @@ protected QuestionAnsweringInferenceResults createTestInstance() {
         return createRandomResults();
     }
 
+    @Override
+    protected QuestionAnsweringInferenceResults mutateInstance(QuestionAnsweringInferenceResults instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<QuestionAnsweringInferenceResults> instanceReader() {
         return QuestionAnsweringInferenceResults::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionFeatureImportanceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionFeatureImportanceTests.java
index 3a9e7a26026c..d9da9ac9f33f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionFeatureImportanceTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionFeatureImportanceTests.java
@@ -29,6 +29,11 @@ protected RegressionFeatureImportance createTestInstance() {
         return createRandomInstance();
     }
 
+    @Override
+    protected RegressionFeatureImportance mutateInstance(RegressionFeatureImportance instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static RegressionFeatureImportance createRandomInstance() {
         return new RegressionFeatureImportance(randomAlphaOfLength(10), randomDoubleBetween(-10.0, 10.0, false));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResultsTests.java
index ba11768b0f77..6ea00fce676e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResultsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResultsTests.java
@@ -68,6 +68,11 @@ protected RegressionInferenceResults createTestInstance() {
         return createRandomResults();
     }
 
+    @Override
+    protected RegressionInferenceResults mutateInstance(RegressionInferenceResults instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<RegressionInferenceResults> instanceReader() {
         return RegressionInferenceResults::new;
@@ -83,7 +88,7 @@ public void testToXContent() {
         RegressionFeatureImportance fi = new RegressionFeatureImportance("foo", 1.0);
         result = new RegressionInferenceResults(1.0, resultsField, Collections.singletonList(fi));
         stringRep = Strings.toString(result);
-        expected = formatted("""
+        expected = Strings.format("""
             {"%s":1.0,"feature_importance":[{"feature_name":"foo","importance":1.0}]}\
             """, resultsField);
         assertEquals(expected, stringRep);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/SlimResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/SlimResultsTests.java
new file mode 100644
index 000000000000..0b4589456b52
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/SlimResultsTests.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.ml.inference.results;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.ingest.IngestDocument;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class SlimResultsTests extends InferenceResultsTestCase<SlimResults> {
+    @Override
+    protected Writeable.Reader<SlimResults> instanceReader() {
+        return SlimResults::new;
+    }
+
+    @Override
+    protected SlimResults createTestInstance() {
+        int numTokens = randomIntBetween(0, 20);
+        List<SlimResults.WeightedToken> tokenList = new ArrayList<>();
+        for (int i = 0; i < numTokens; i++) {
+            tokenList.add(new SlimResults.WeightedToken(i, (float) randomDoubleBetween(0.0, 5.0, false)));
+        }
+        return new SlimResults(randomAlphaOfLength(4), tokenList, randomBoolean());
+    }
+
+    @Override
+    protected SlimResults mutateInstance(SlimResults instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    void assertFieldValues(SlimResults createdInstance, IngestDocument document, String resultsField) {
+        var ingestedTokens = (List<Map<String, Object>>) document.getFieldValue(
+            resultsField + '.' + createdInstance.getResultsField(),
+            List.class
+        );
+        var originalTokens = createdInstance.getWeightedTokens();
+        assertEquals(originalTokens.size(), ingestedTokens.size());
+        for (int i = 0; i < createdInstance.getWeightedTokens().size(); i++) {
+            assertEquals(originalTokens.get(i).token(), (int) ingestedTokens.get(i).get("token"));
+            assertEquals(originalTokens.get(i).weight(), (float) ingestedTokens.get(i).get("weight"), 0.0001);
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/TextEmbeddingResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/TextEmbeddingResultsTests.java
index 90a8cce88237..d29e79698e2c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/TextEmbeddingResultsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/TextEmbeddingResultsTests.java
@@ -38,6 +38,11 @@ protected TextEmbeddingResults createTestInstance() {
         return createRandomResults();
     }
 
+    @Override
+    protected TextEmbeddingResults mutateInstance(TextEmbeddingResults instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testAsMap() {
         TextEmbeddingResults testInstance = createTestInstance();
         Map<String, Object> asMap = testInstance.asMap();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/TextSimilarityInferenceResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/TextSimilarityInferenceResultsTests.java
index ac80970ae13c..e543e04a0108 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/TextSimilarityInferenceResultsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/TextSimilarityInferenceResultsTests.java
@@ -22,6 +22,11 @@ protected TextSimilarityInferenceResults createTestInstance() {
         return createRandomResults();
     }
 
+    @Override
+    protected TextSimilarityInferenceResults mutateInstance(TextSimilarityInferenceResults instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<TextSimilarityInferenceResults> instanceReader() {
         return TextSimilarityInferenceResults::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/TopAnswerEntryTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/TopAnswerEntryTests.java
index da6264c71e9a..82ec96a25da0 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/TopAnswerEntryTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/TopAnswerEntryTests.java
@@ -33,4 +33,9 @@ protected Writeable.Reader<QuestionAnsweringInferenceResults.TopAnswerEntry> ins
     protected QuestionAnsweringInferenceResults.TopAnswerEntry createTestInstance() {
         return createRandomTopAnswerEntry();
     }
+
+    @Override
+    protected QuestionAnsweringInferenceResults.TopAnswerEntry mutateInstance(QuestionAnsweringInferenceResults.TopAnswerEntry instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/TopClassEntryTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/TopClassEntryTests.java
index 8de72a002a49..2056432e58f7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/TopClassEntryTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/TopClassEntryTests.java
@@ -41,4 +41,9 @@ protected Writeable.Reader<TopClassEntry> instanceReader() {
     protected TopClassEntry createTestInstance() {
         return createRandomTopClassEntry();
     }
+
+    @Override
+    protected TopClassEntry mutateInstance(TopClassEntry instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/WarningInferenceResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/WarningInferenceResultsTests.java
index 96ed7b1b8fec..68379f888d11 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/WarningInferenceResultsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/WarningInferenceResultsTests.java
@@ -22,6 +22,11 @@ protected WarningInferenceResults createTestInstance() {
         return createRandomResults();
     }
 
+    @Override
+    protected WarningInferenceResults mutateInstance(WarningInferenceResults instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<WarningInferenceResults> instanceReader() {
         return WarningInferenceResults::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/AbstractNlpConfigUpdateTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/AbstractNlpConfigUpdateTestCase.java
index c58146368252..dc3335d8407f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/AbstractNlpConfigUpdateTestCase.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/AbstractNlpConfigUpdateTestCase.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase;
 import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider;
 
@@ -29,6 +30,16 @@ protected NamedWriteableRegistry writableRegistry() {
         return new NamedWriteableRegistry(new MlInferenceNamedXContentProvider().getNamedWriteables());
     }
 
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return new NamedXContentRegistry(new MlInferenceNamedXContentProvider().getNamedXContentParsers());
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(new MlInferenceNamedXContentProvider().getNamedWriteables());
+    }
+
     /**
      * @param expectedTokenization The tokenization update that will be provided
      * @return A map and expected resulting object. Note: `tokenization` will be overwritten if provided in the returned map
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertTokenizationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertTokenizationTests.java
index 952e6b437253..b584a2caca4b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertTokenizationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertTokenizationTests.java
@@ -52,6 +52,11 @@ protected BertTokenization createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected BertTokenization mutateInstance(BertTokenization instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected BertTokenization mutateInstanceForVersion(BertTokenization instance, Version version) {
         return mutateForVersion(instance, version);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertTokenizationUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertTokenizationUpdateTests.java
index 4c34a978204b..fa2ea2566f77 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertTokenizationUpdateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertTokenizationUpdateTests.java
@@ -67,6 +67,11 @@ protected BertTokenizationUpdate createTestInstance() {
         return randomInstance();
     }
 
+    @Override
+    protected BertTokenizationUpdate mutateInstance(BertTokenizationUpdate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected BertTokenizationUpdate mutateInstanceForVersion(BertTokenizationUpdate instance, Version version) {
         if (version.before(Version.V_8_2_0)) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigTests.java
index 2880e4952add..ec95d31f2a9c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigTests.java
@@ -38,6 +38,11 @@ protected ClassificationConfig createTestInstance() {
         return randomClassificationConfig();
     }
 
+    @Override
+    protected ClassificationConfig mutateInstance(ClassificationConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<ClassificationConfig> instanceReader() {
         return ClassificationConfig::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdateTests.java
index 5f35e674646b..3c051d9c1bf1 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdateTests.java
@@ -115,6 +115,11 @@ protected ClassificationConfigUpdate createTestInstance() {
         return randomClassificationConfigUpdate();
     }
 
+    @Override
+    protected ClassificationConfigUpdate mutateInstance(ClassificationConfigUpdate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<ClassificationConfigUpdate> instanceReader() {
         return ClassificationConfigUpdate::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/EmptyConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/EmptyConfigUpdateTests.java
index daff31e54f56..b844557039ab 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/EmptyConfigUpdateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/EmptyConfigUpdateTests.java
@@ -26,4 +26,9 @@ protected EmptyConfigUpdate createTestInstance() {
         return new EmptyConfigUpdate();
     }
 
+    @Override
+    protected EmptyConfigUpdate mutateInstance(EmptyConfigUpdate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/FillMaskConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/FillMaskConfigTests.java
index 59394d48ec0b..aa0f99365b23 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/FillMaskConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/FillMaskConfigTests.java
@@ -51,6 +51,11 @@ protected FillMaskConfig createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected FillMaskConfig mutateInstance(FillMaskConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected FillMaskConfig mutateInstanceForVersion(FillMaskConfig instance, Version version) {
         return mutateForVersion(instance, version);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/FillMaskConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/FillMaskConfigUpdateTests.java
index ed174655e6c9..ac461001638c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/FillMaskConfigUpdateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/FillMaskConfigUpdateTests.java
@@ -8,12 +8,9 @@
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -128,17 +125,12 @@ protected FillMaskConfigUpdate createTestInstance() {
     }
 
     @Override
-    protected FillMaskConfigUpdate mutateInstanceForVersion(FillMaskConfigUpdate instance, Version version) {
-        return mutateForVersion(instance, version);
+    protected FillMaskConfigUpdate mutateInstance(FillMaskConfigUpdate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
     }
 
     @Override
-    protected NamedXContentRegistry xContentRegistry() {
-        return new NamedXContentRegistry(new MlInferenceNamedXContentProvider().getNamedXContentParsers());
-    }
-
-    @Override
-    protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        return new NamedWriteableRegistry(new MlInferenceNamedXContentProvider().getNamedWriteables());
+    protected FillMaskConfigUpdate mutateInstanceForVersion(FillMaskConfigUpdate instance, Version version) {
+        return mutateForVersion(instance, version);
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/IndexLocationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/IndexLocationTests.java
index cac2ecac76b8..69a94c67e683 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/IndexLocationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/IndexLocationTests.java
@@ -36,6 +36,11 @@ protected IndexLocation createTestInstance() {
         return randomInstance();
     }
 
+    @Override
+    protected IndexLocation mutateInstance(IndexLocation instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected boolean supportsUnknownFields() {
         return lenient;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceStatsTests.java
index b311219ad806..65e20240a9a9 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/InferenceStatsTests.java
@@ -47,6 +47,11 @@ protected InferenceStats createTestInstance() {
         return createTestInstance(randomAlphaOfLength(10), randomAlphaOfLength(10));
     }
 
+    @Override
+    protected InferenceStats mutateInstance(InferenceStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<InferenceStats> instanceReader() {
         return InferenceStats::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/MPNetTokenizationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/MPNetTokenizationTests.java
index dead82c73644..2045f9a9c7ca 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/MPNetTokenizationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/MPNetTokenizationTests.java
@@ -52,6 +52,11 @@ protected MPNetTokenization createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected MPNetTokenization mutateInstance(MPNetTokenization instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected MPNetTokenization mutateInstanceForVersion(MPNetTokenization instance, Version version) {
         return mutateForVersion(instance, version);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/MPNetTokenizationUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/MPNetTokenizationUpdateTests.java
index 891eba711685..648b681ecc24 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/MPNetTokenizationUpdateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/MPNetTokenizationUpdateTests.java
@@ -60,6 +60,11 @@ protected MPNetTokenizationUpdate createTestInstance() {
         return randomInstance();
     }
 
+    @Override
+    protected MPNetTokenizationUpdate mutateInstance(MPNetTokenizationUpdate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected MPNetTokenizationUpdate mutateInstanceForVersion(MPNetTokenizationUpdate instance, Version version) {
         if (version.before(Version.V_8_2_0)) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NerConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NerConfigTests.java
index 91e670a0049e..cbd5cedf894d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NerConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NerConfigTests.java
@@ -55,6 +55,11 @@ protected NerConfig createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected NerConfig mutateInstance(NerConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected NerConfig mutateInstanceForVersion(NerConfig instance, Version version) {
         return mutateForVersion(instance, version);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NerConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NerConfigUpdateTests.java
index d1089330287f..3371b64ad326 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NerConfigUpdateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NerConfigUpdateTests.java
@@ -8,12 +8,9 @@
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -108,17 +105,12 @@ protected NerConfigUpdate createTestInstance() {
     }
 
     @Override
-    protected NerConfigUpdate mutateInstanceForVersion(NerConfigUpdate instance, Version version) {
-        return mutateForVersion(instance, version);
+    protected NerConfigUpdate mutateInstance(NerConfigUpdate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
     }
 
     @Override
-    protected NamedXContentRegistry xContentRegistry() {
-        return new NamedXContentRegistry(new MlInferenceNamedXContentProvider().getNamedXContentParsers());
-    }
-
-    @Override
-    protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        return new NamedWriteableRegistry(new MlInferenceNamedXContentProvider().getNamedWriteables());
+    protected NerConfigUpdate mutateInstanceForVersion(NerConfigUpdate instance, Version version) {
+        return mutateForVersion(instance, version);
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PassThroughConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PassThroughConfigTests.java
index 28e107101d28..9cbf0f659d27 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PassThroughConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PassThroughConfigTests.java
@@ -50,6 +50,11 @@ protected PassThroughConfig createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected PassThroughConfig mutateInstance(PassThroughConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected PassThroughConfig mutateInstanceForVersion(PassThroughConfig instance, Version version) {
         return mutateForVersion(instance, version);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PassThroughConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PassThroughConfigUpdateTests.java
index 002b9da8a522..3c704b8ed426 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PassThroughConfigUpdateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/PassThroughConfigUpdateTests.java
@@ -8,12 +8,9 @@
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -98,17 +95,12 @@ protected PassThroughConfigUpdate createTestInstance() {
     }
 
     @Override
-    protected PassThroughConfigUpdate mutateInstanceForVersion(PassThroughConfigUpdate instance, Version version) {
-        return mutateForVersion(instance, version);
+    protected PassThroughConfigUpdate mutateInstance(PassThroughConfigUpdate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
     }
 
     @Override
-    protected NamedXContentRegistry xContentRegistry() {
-        return new NamedXContentRegistry(new MlInferenceNamedXContentProvider().getNamedXContentParsers());
-    }
-
-    @Override
-    protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        return new NamedWriteableRegistry(new MlInferenceNamedXContentProvider().getNamedWriteables());
+    protected PassThroughConfigUpdate mutateInstanceForVersion(PassThroughConfigUpdate instance, Version version) {
+        return mutateForVersion(instance, version);
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/QuestionAnsweringConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/QuestionAnsweringConfigTests.java
index 0f8f2f078366..b792f5523ea2 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/QuestionAnsweringConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/QuestionAnsweringConfigTests.java
@@ -52,6 +52,11 @@ protected QuestionAnsweringConfig createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected QuestionAnsweringConfig mutateInstance(QuestionAnsweringConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected QuestionAnsweringConfig mutateInstanceForVersion(QuestionAnsweringConfig instance, Version version) {
         return mutateForVersion(instance, version);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/QuestionAnsweringConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/QuestionAnsweringConfigUpdateTests.java
index c6786b4e4d09..344cf5ea1ff2 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/QuestionAnsweringConfigUpdateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/QuestionAnsweringConfigUpdateTests.java
@@ -8,12 +8,9 @@
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -70,6 +67,11 @@ protected QuestionAnsweringConfigUpdate createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected QuestionAnsweringConfigUpdate mutateInstance(QuestionAnsweringConfigUpdate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected QuestionAnsweringConfigUpdate mutateInstanceForVersion(QuestionAnsweringConfigUpdate instance, Version version) {
         return mutateForVersion(instance, version);
@@ -171,14 +173,4 @@ public void testApply() {
     public static QuestionAnsweringConfigUpdate createRandom() {
         return randomUpdate();
     }
-
-    @Override
-    protected NamedXContentRegistry xContentRegistry() {
-        return new NamedXContentRegistry(new MlInferenceNamedXContentProvider().getNamedXContentParsers());
-    }
-
-    @Override
-    protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        return new NamedWriteableRegistry(new MlInferenceNamedXContentProvider().getNamedWriteables());
-    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigTests.java
index 17a56b97a753..028e233cd827 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigTests.java
@@ -31,6 +31,11 @@ protected RegressionConfig createTestInstance() {
         return randomRegressionConfig();
     }
 
+    @Override
+    protected RegressionConfig mutateInstance(RegressionConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<RegressionConfig> instanceReader() {
         return RegressionConfig::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdateTests.java
index 3918ddbf4f7e..3d3a892e41c3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdateTests.java
@@ -96,6 +96,11 @@ protected RegressionConfigUpdate createTestInstance() {
         return randomRegressionConfigUpdate();
     }
 
+    @Override
+    protected RegressionConfigUpdate mutateInstance(RegressionConfigUpdate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<RegressionConfigUpdate> instanceReader() {
         return RegressionConfigUpdate::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ResultsFieldUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ResultsFieldUpdateTests.java
index 78d305a06153..4237458d01f6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ResultsFieldUpdateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ResultsFieldUpdateTests.java
@@ -29,6 +29,11 @@ protected ResultsFieldUpdate createTestInstance() {
         return randomUpdate();
     }
 
+    @Override
+    protected ResultsFieldUpdate mutateInstance(ResultsFieldUpdate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testIsSupported() {
         ResultsFieldUpdate update = new ResultsFieldUpdate("foo");
         assertTrue(update.isSupported(mock(InferenceConfig.class)));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RobertaTokenizationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RobertaTokenizationTests.java
index 920933be7450..6f4a7ba5517b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RobertaTokenizationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RobertaTokenizationTests.java
@@ -52,6 +52,11 @@ protected RobertaTokenization createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected RobertaTokenization mutateInstance(RobertaTokenization instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected RobertaTokenization mutateInstanceForVersion(RobertaTokenization instance, Version version) {
         return mutateForVersion(instance, version);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RobertaTokenizationUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RobertaTokenizationUpdateTests.java
index 5addb4eec5ce..e82a883cc6f6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RobertaTokenizationUpdateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RobertaTokenizationUpdateTests.java
@@ -60,6 +60,11 @@ protected RobertaTokenizationUpdate createTestInstance() {
         return randomInstance();
     }
 
+    @Override
+    protected RobertaTokenizationUpdate mutateInstance(RobertaTokenizationUpdate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected RobertaTokenizationUpdate mutateInstanceForVersion(RobertaTokenizationUpdate instance, Version version) {
         if (version.before(Version.V_8_2_0)) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/SlimConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/SlimConfigTests.java
new file mode 100644
index 000000000000..4ed618d94e04
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/SlimConfigTests.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
+
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.inference.InferenceConfigItemTestCase;
+
+import java.io.IOException;
+
+public class SlimConfigTests extends InferenceConfigItemTestCase<SlimConfig> {
+
+    public static SlimConfig createRandom() {
+        // create a tokenization config with a no span setting.
+        var tokenization = new BertTokenization(
+            randomBoolean() ? null : randomBoolean(),
+            randomBoolean() ? null : randomBoolean(),
+            randomBoolean() ? null : randomIntBetween(1, 1024),
+            randomBoolean() ? null : randomFrom(Tokenization.Truncate.FIRST),
+            null
+        );
+
+        return new SlimConfig(
+            randomBoolean() ? null : VocabularyConfigTests.createRandom(),
+            randomBoolean() ? null : tokenization,
+            randomBoolean() ? null : randomAlphaOfLength(5)
+        );
+    }
+
+    @Override
+    protected Writeable.Reader<SlimConfig> instanceReader() {
+        return SlimConfig::new;
+    }
+
+    @Override
+    protected SlimConfig createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected SlimConfig mutateInstance(SlimConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
+    @Override
+    protected SlimConfig doParseInstance(XContentParser parser) throws IOException {
+        return SlimConfig.fromXContentLenient(parser);
+    }
+
+    @Override
+    protected SlimConfig mutateInstanceForVersion(SlimConfig instance, Version version) {
+        return instance;
+    }
+
+    public void testBertTokenizationOnly() {
+        ElasticsearchStatusException e = expectThrows(
+            ElasticsearchStatusException.class,
+            () -> new SlimConfig(null, RobertaTokenizationTests.createRandom(), null)
+        );
+        assertEquals(RestStatus.BAD_REQUEST, e.status());
+        assertEquals("SLIM must be configured with BERT tokenizer, [roberta] given", e.getMessage());
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/SlimConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/SlimConfigUpdateTests.java
new file mode 100644
index 000000000000..9af359cd98be
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/SlimConfigUpdateTests.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class SlimConfigUpdateTests extends AbstractNlpConfigUpdateTestCase<SlimConfigUpdate> {
+
+    public static SlimConfigUpdate randomUpdate() {
+        SlimConfigUpdate.Builder builder = new SlimConfigUpdate.Builder();
+        if (randomBoolean()) {
+            builder.setResultsField(randomAlphaOfLength(8));
+        }
+        if (randomBoolean()) {
+            builder.setTokenizationUpdate(new BertTokenizationUpdate(randomFrom(Tokenization.Truncate.values()), null));
+        }
+        return builder.build();
+    }
+
+    public static SlimConfigUpdate mutateForVersion(SlimConfigUpdate instance, Version version) {
+        if (version.before(Version.V_8_1_0)) {
+            return new SlimConfigUpdate(instance.getResultsField(), null);
+        }
+        return instance;
+    }
+
+    @Override
+    protected Writeable.Reader<SlimConfigUpdate> instanceReader() {
+        return SlimConfigUpdate::new;
+    }
+
+    @Override
+    protected SlimConfigUpdate createTestInstance() {
+        return randomUpdate();
+    }
+
+    @Override
+    protected SlimConfigUpdate mutateInstance(SlimConfigUpdate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
+    @Override
+    protected SlimConfigUpdate doParseInstance(XContentParser parser) throws IOException {
+        return SlimConfigUpdate.fromXContentStrict(parser);
+    }
+
+    @Override
+    protected SlimConfigUpdate mutateInstanceForVersion(SlimConfigUpdate instance, Version version) {
+        return mutateForVersion(instance, version);
+    }
+
+    @Override
+    Tuple<Map<String, Object>, SlimConfigUpdate> fromMapTestInstances(TokenizationUpdate expectedTokenization) {
+        SlimConfigUpdate expected = new SlimConfigUpdate("ml-results", expectedTokenization);
+        Map<String, Object> config = new HashMap<>() {
+            {
+                put(NlpConfig.RESULTS_FIELD.getPreferredName(), "ml-results");
+            }
+        };
+        return Tuple.tuple(config, expected);
+    }
+
+    @Override
+    SlimConfigUpdate fromMap(Map<String, Object> map) {
+        return SlimConfigUpdate.fromMap(map);
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextClassificationConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextClassificationConfigTests.java
index 62e97886b944..328707c62353 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextClassificationConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextClassificationConfigTests.java
@@ -56,6 +56,11 @@ protected TextClassificationConfig createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected TextClassificationConfig mutateInstance(TextClassificationConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected TextClassificationConfig mutateInstanceForVersion(TextClassificationConfig instance, Version version) {
         return mutateForVersion(instance, version);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextClassificationConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextClassificationConfigUpdateTests.java
index a8c2c871e488..481c82501900 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextClassificationConfigUpdateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextClassificationConfigUpdateTests.java
@@ -9,12 +9,9 @@
 
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.Version;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -192,17 +189,12 @@ protected TextClassificationConfigUpdate createTestInstance() {
     }
 
     @Override
-    protected TextClassificationConfigUpdate mutateInstanceForVersion(TextClassificationConfigUpdate instance, Version version) {
-        return mutateForVersion(instance, version);
+    protected TextClassificationConfigUpdate mutateInstance(TextClassificationConfigUpdate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
     }
 
     @Override
-    protected NamedXContentRegistry xContentRegistry() {
-        return new NamedXContentRegistry(new MlInferenceNamedXContentProvider().getNamedXContentParsers());
-    }
-
-    @Override
-    protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        return new NamedWriteableRegistry(new MlInferenceNamedXContentProvider().getNamedWriteables());
+    protected TextClassificationConfigUpdate mutateInstanceForVersion(TextClassificationConfigUpdate instance, Version version) {
+        return mutateForVersion(instance, version);
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigTests.java
index d60a8b28107d..b72b5c6ba3fb 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigTests.java
@@ -50,6 +50,11 @@ protected TextEmbeddingConfig createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected TextEmbeddingConfig mutateInstance(TextEmbeddingConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected TextEmbeddingConfig mutateInstanceForVersion(TextEmbeddingConfig instance, Version version) {
         return mutateForVersion(instance, version);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigUpdateTests.java
index 987722e291af..a49da1c08307 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigUpdateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextEmbeddingConfigUpdateTests.java
@@ -8,12 +8,9 @@
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -98,17 +95,12 @@ protected TextEmbeddingConfigUpdate createTestInstance() {
     }
 
     @Override
-    protected TextEmbeddingConfigUpdate mutateInstanceForVersion(TextEmbeddingConfigUpdate instance, Version version) {
-        return mutateForVersion(instance, version);
+    protected TextEmbeddingConfigUpdate mutateInstance(TextEmbeddingConfigUpdate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
     }
 
     @Override
-    protected NamedXContentRegistry xContentRegistry() {
-        return new NamedXContentRegistry(new MlInferenceNamedXContentProvider().getNamedXContentParsers());
-    }
-
-    @Override
-    protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        return new NamedWriteableRegistry(new MlInferenceNamedXContentProvider().getNamedWriteables());
+    protected TextEmbeddingConfigUpdate mutateInstanceForVersion(TextEmbeddingConfigUpdate instance, Version version) {
+        return mutateForVersion(instance, version);
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextSimilarityConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextSimilarityConfigTests.java
index e8976ce1dd7c..91004da995e8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextSimilarityConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextSimilarityConfigTests.java
@@ -52,6 +52,11 @@ protected TextSimilarityConfig createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected TextSimilarityConfig mutateInstance(TextSimilarityConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected TextSimilarityConfig mutateInstanceForVersion(TextSimilarityConfig instance, Version version) {
         return mutateForVersion(instance, version);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextSimilarityConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextSimilarityConfigUpdateTests.java
index 5c8d21ae99fe..1789d8e2fced 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextSimilarityConfigUpdateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TextSimilarityConfigUpdateTests.java
@@ -8,12 +8,9 @@
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -69,6 +66,11 @@ protected TextSimilarityConfigUpdate createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected TextSimilarityConfigUpdate mutateInstance(TextSimilarityConfigUpdate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected TextSimilarityConfigUpdate mutateInstanceForVersion(TextSimilarityConfigUpdate instance, Version version) {
         return mutateForVersion(instance, version);
@@ -169,14 +171,4 @@ public void testApply() {
     public static TextSimilarityConfigUpdate createRandom() {
         return randomUpdate();
     }
-
-    @Override
-    protected NamedXContentRegistry xContentRegistry() {
-        return new NamedXContentRegistry(new MlInferenceNamedXContentProvider().getNamedXContentParsers());
-    }
-
-    @Override
-    protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        return new NamedWriteableRegistry(new MlInferenceNamedXContentProvider().getNamedWriteables());
-    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TrainedModelSizeStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TrainedModelSizeStatsTests.java
index d195532057c4..d65dc8751dde 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TrainedModelSizeStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/TrainedModelSizeStatsTests.java
@@ -22,6 +22,11 @@ protected TrainedModelSizeStats createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected TrainedModelSizeStats mutateInstance(TrainedModelSizeStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static TrainedModelSizeStats createRandom() {
         return new TrainedModelSizeStats(randomNonNegativeLong(), randomNonNegativeLong());
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/VocabularyConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/VocabularyConfigTests.java
index 6c0f07e8a0fd..3e48b3c9e5dc 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/VocabularyConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/VocabularyConfigTests.java
@@ -31,6 +31,11 @@ protected VocabularyConfig createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected VocabularyConfig mutateInstance(VocabularyConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected VocabularyConfig mutateInstanceForVersion(VocabularyConfig instance, Version version) {
         return instance;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ZeroShotClassificationConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ZeroShotClassificationConfigTests.java
index 48e4b25ea731..6e33e34345b4 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ZeroShotClassificationConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ZeroShotClassificationConfigTests.java
@@ -55,6 +55,11 @@ protected ZeroShotClassificationConfig createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected ZeroShotClassificationConfig mutateInstance(ZeroShotClassificationConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected ZeroShotClassificationConfig mutateInstanceForVersion(ZeroShotClassificationConfig instance, Version version) {
         return mutateForVersion(instance, version);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ZeroShotClassificationConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ZeroShotClassificationConfigUpdateTests.java
index 2d424edac4c9..27b34a30fad8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ZeroShotClassificationConfigUpdateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ZeroShotClassificationConfigUpdateTests.java
@@ -8,12 +8,9 @@
 package org.elasticsearch.xpack.core.ml.inference.trainedmodel;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -63,6 +60,11 @@ protected ZeroShotClassificationConfigUpdate createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected ZeroShotClassificationConfigUpdate mutateInstance(ZeroShotClassificationConfigUpdate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected ZeroShotClassificationConfigUpdate mutateInstanceForVersion(ZeroShotClassificationConfigUpdate instance, Version version) {
         return mutateForVersion(instance, version);
@@ -212,14 +214,4 @@ public void testIsNoop() {
     public static ZeroShotClassificationConfigUpdate createRandom() {
         return randomUpdate();
     }
-
-    @Override
-    protected NamedXContentRegistry xContentRegistry() {
-        return new NamedXContentRegistry(new MlInferenceNamedXContentProvider().getNamedXContentParsers());
-    }
-
-    @Override
-    protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        return new NamedWriteableRegistry(new MlInferenceNamedXContentProvider().getNamedWriteables());
-    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/EnsembleTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/EnsembleTests.java
index cec70a2895a9..e1f07d17e5be 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/EnsembleTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/EnsembleTests.java
@@ -113,6 +113,11 @@ protected Ensemble createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Ensemble mutateInstance(Ensemble instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Ensemble> instanceReader() {
         return Ensemble::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/ExponentTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/ExponentTests.java
index 0488f0c083f2..71b22272f045 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/ExponentTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/ExponentTests.java
@@ -35,6 +35,11 @@ protected Exponent createTestInstance() {
         return randomBoolean() ? new Exponent() : createTestInstance(randomIntBetween(1, 100));
     }
 
+    @Override
+    protected Exponent mutateInstance(Exponent instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Exponent> instanceReader() {
         return Exponent::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/LogisticRegressionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/LogisticRegressionTests.java
index af8e36699797..392b90168855 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/LogisticRegressionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/LogisticRegressionTests.java
@@ -36,6 +36,11 @@ protected LogisticRegression createTestInstance() {
         return randomBoolean() ? new LogisticRegression() : createTestInstance(randomIntBetween(1, 100));
     }
 
+    @Override
+    protected LogisticRegression mutateInstance(LogisticRegression instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<LogisticRegression> instanceReader() {
         return LogisticRegression::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/WeightedAggregatorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/WeightedAggregatorTests.java
index 485f7f0bc6fa..5d68a6a57575 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/WeightedAggregatorTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/WeightedAggregatorTests.java
@@ -46,4 +46,9 @@ public void testWithValuesOfWrongLength() {
     }
 
     abstract T createTestInstance(int numberOfWeights);
+
+    @Override
+    protected T mutateInstance(T instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/WeightedModeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/WeightedModeTests.java
index 3d59d49f73fe..74ccecb7af17 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/WeightedModeTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/WeightedModeTests.java
@@ -36,6 +36,11 @@ protected WeightedMode createTestInstance() {
         return randomBoolean() ? new WeightedMode(randomIntBetween(2, 10)) : createTestInstance(randomIntBetween(1, 100));
     }
 
+    @Override
+    protected WeightedMode mutateInstance(WeightedMode instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<WeightedMode> instanceReader() {
         return WeightedMode::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/WeightedSumTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/WeightedSumTests.java
index 5e496931c475..aa78add6ddab 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/WeightedSumTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ensemble/WeightedSumTests.java
@@ -35,6 +35,11 @@ protected WeightedSum createTestInstance() {
         return randomBoolean() ? new WeightedSum() : createTestInstance(randomIntBetween(1, 100));
     }
 
+    @Override
+    protected WeightedSum mutateInstance(WeightedSum instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<WeightedSum> instanceReader() {
         return WeightedSum::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/inference/InferenceDefinitionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/inference/InferenceDefinitionTests.java
index 5a4e335f12e8..14ec7a2053a1 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/inference/InferenceDefinitionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/inference/InferenceDefinitionTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -197,7 +198,7 @@ public void testComplexInferenceDefinitionInferWithCustomPreProcessor() throws I
     }
 
     public static String getClassificationDefinition(boolean customPreprocessor) {
-        return formatted("""
+        return Strings.format("""
             {
                 "preprocessors": [
                     {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/langident/LangIdentNeuralNetworkTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/langident/LangIdentNeuralNetworkTests.java
index 58a143ffd193..ba5a4b252845 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/langident/LangIdentNeuralNetworkTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/langident/LangIdentNeuralNetworkTests.java
@@ -44,6 +44,11 @@ protected LangIdentNeuralNetwork createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected LangIdentNeuralNetwork mutateInstance(LangIdentNeuralNetwork instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static LangIdentNeuralNetwork createRandom() {
         return new LangIdentNeuralNetwork(randomAlphaOfLength(10), LangNetLayerTests.createRandom(), LangNetLayerTests.createRandom());
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/langident/LangNetLayerTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/langident/LangNetLayerTests.java
index a799abd7b118..1dd1e446ecf4 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/langident/LangNetLayerTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/langident/LangNetLayerTests.java
@@ -39,6 +39,11 @@ protected LangNetLayer createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected LangNetLayer mutateInstance(LangNetLayer instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static LangNetLayer createRandom() {
         int numWeights = randomIntBetween(1, 1000);
         return new LangNetLayer(
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/FeatureImportanceBaselineTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/FeatureImportanceBaselineTests.java
index 0f2ff7954c10..7b991674d3f8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/FeatureImportanceBaselineTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/FeatureImportanceBaselineTests.java
@@ -49,6 +49,11 @@ protected FeatureImportanceBaseline createTestInstance() {
         return randomInstance();
     }
 
+    @Override
+    protected FeatureImportanceBaseline mutateInstance(FeatureImportanceBaseline instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<FeatureImportanceBaseline> instanceReader() {
         return FeatureImportanceBaseline::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/HyperparametersTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/HyperparametersTests.java
index 72eaa83ee4c5..b7fc070b7516 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/HyperparametersTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/HyperparametersTests.java
@@ -41,6 +41,11 @@ protected Hyperparameters createTestInstance() {
         return randomInstance();
     }
 
+    @Override
+    protected Hyperparameters mutateInstance(Hyperparameters instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Hyperparameters> instanceReader() {
         return Hyperparameters::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/TotalFeatureImportanceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/TotalFeatureImportanceTests.java
index b14623627189..bee41f1177f0 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/TotalFeatureImportanceTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/TotalFeatureImportanceTests.java
@@ -54,6 +54,11 @@ protected TotalFeatureImportance createTestInstance() {
         return randomInstance();
     }
 
+    @Override
+    protected TotalFeatureImportance mutateInstance(TotalFeatureImportance instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<TotalFeatureImportance> instanceReader() {
         return TotalFeatureImportance::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/TrainedModelMetadataTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/TrainedModelMetadataTests.java
index 98199c57698b..0d8df3d151d8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/TrainedModelMetadataTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/TrainedModelMetadataTests.java
@@ -41,6 +41,11 @@ protected TrainedModelMetadata createTestInstance() {
         return randomInstance();
     }
 
+    @Override
+    protected TrainedModelMetadata mutateInstance(TrainedModelMetadata instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<TrainedModelMetadata> instanceReader() {
         return TrainedModelMetadata::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/TreeNodeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/TreeNodeTests.java
index 0e74e579c4d2..265ca228e945 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/TreeNodeTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/TreeNodeTests.java
@@ -44,6 +44,11 @@ protected TreeNode createTestInstance() {
             .build();
     }
 
+    @Override
+    protected TreeNode mutateInstance(TreeNode instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static TreeNode createRandomLeafNode(double internalValue) {
         return TreeNode.builder(randomInt(100))
             .setDefaultLeft(randomBoolean() ? null : randomBoolean())
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/TreeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/TreeTests.java
index 3c001b70549f..7b855a703f19 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/TreeTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/tree/TreeTests.java
@@ -53,6 +53,11 @@ protected Tree createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Tree mutateInstance(Tree instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static Tree createRandom() {
         return createRandom(randomFrom(TargetType.values()));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisLimitsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisLimitsTests.java
index 8b096824283d..cf061b0b3429 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisLimitsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisLimitsTests.java
@@ -172,7 +172,7 @@ public void testVerify_GivenValid() {
     }
 
     @Override
-    protected AnalysisLimits mutateInstance(AnalysisLimits instance) throws IOException {
+    protected AnalysisLimits mutateInstance(AnalysisLimits instance) {
         Long memoryModelLimit = instance.getModelMemoryLimit();
         Long categorizationExamplesLimit = instance.getCategorizationExamplesLimit();
         switch (between(0, 1)) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/BlockedTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/BlockedTests.java
index cc7062f8e666..5460f0eaf8c8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/BlockedTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/BlockedTests.java
@@ -37,6 +37,11 @@ protected Blocked createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected Blocked mutateInstance(Blocked instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static Blocked createRandom() {
         Blocked.Reason reason = randomFrom(Blocked.Reason.values());
         TaskId taskId = (reason != Blocked.Reason.NONE && randomBoolean())
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DataDescriptionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DataDescriptionTests.java
index 843b4f87cc27..11235227dcd9 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DataDescriptionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DataDescriptionTests.java
@@ -125,7 +125,7 @@ protected DataDescription doParseInstance(XContentParser parser) {
     }
 
     @Override
-    protected DataDescription mutateInstance(DataDescription instance) throws java.io.IOException {
+    protected DataDescription mutateInstance(DataDescription instance) {
         String timeField = instance.getTimeField();
         String timeFormat = instance.getTimeFormat();
         switch (between(0, 1)) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectorTests.java
index d44c35363f14..40640f7221cf 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectorTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectorTests.java
@@ -180,6 +180,11 @@ protected Detector createTestInstance() {
         return detector.build();
     }
 
+    @Override
+    protected Detector mutateInstance(Detector instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<Detector> instanceReader() {
         return Detector::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/FilterRefTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/FilterRefTests.java
index 6720566f091f..f9e6252b3b2c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/FilterRefTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/FilterRefTests.java
@@ -19,6 +19,11 @@ protected FilterRef createTestInstance() {
         return new FilterRef(randomAlphaOfLength(20), randomFrom(FilterRef.FilterType.values()));
     }
 
+    @Override
+    protected FilterRef mutateInstance(FilterRef instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected FilterRef doParseInstance(XContentParser parser) throws IOException {
         return FilterRef.STRICT_PARSER.parse(parser, null);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java
index 05ee0be82e27..de131e9b6af3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java
@@ -76,6 +76,11 @@ protected Job createTestInstance() {
         return createRandomizedJob(new DatafeedConfig.Builder().setIndices(Arrays.asList("airline_data")));
     }
 
+    @Override
+    protected Job mutateInstance(Job instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
         SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java
index 744ad660704d..3dcab054d590 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java
@@ -42,6 +42,11 @@ protected JobUpdate createTestInstance() {
         return createRandom(randomAlphaOfLength(4), null);
     }
 
+    @Override
+    protected JobUpdate mutateInstance(JobUpdate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     /**
      * Creates a completely random update when the job is null
      * or a random update that is is valid for the given job
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/MlFilterTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/MlFilterTests.java
index d1ce7a221395..af88d0aff8e6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/MlFilterTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/MlFilterTests.java
@@ -36,6 +36,11 @@ protected MlFilter createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected MlFilter mutateInstance(MlFilter instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static MlFilter createRandom() {
         return createRandom(randomValidFilterId());
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/ModelPlotConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/ModelPlotConfigTests.java
index a7cdcf5a9a93..0f1dc0ac05fa 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/ModelPlotConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/ModelPlotConfigTests.java
@@ -35,6 +35,11 @@ protected ModelPlotConfig createTestInstance() {
         return createRandomized();
     }
 
+    @Override
+    protected ModelPlotConfig mutateInstance(ModelPlotConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static ModelPlotConfig createRandomized() {
         return new ModelPlotConfig(randomBoolean(), randomAlphaOfLengthBetween(1, 30), randomBoolean() ? randomBoolean() : null);
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/PerPartitionCategorizationConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/PerPartitionCategorizationConfigTests.java
index 1acd199834e4..cc7d9e1d79c4 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/PerPartitionCategorizationConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/PerPartitionCategorizationConfigTests.java
@@ -36,6 +36,11 @@ protected PerPartitionCategorizationConfig createTestInstance() {
         return new PerPartitionCategorizationConfig(enabled, randomBoolean() ? null : enabled && randomBoolean());
     }
 
+    @Override
+    protected PerPartitionCategorizationConfig mutateInstance(PerPartitionCategorizationConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<PerPartitionCategorizationConfig> instanceReader() {
         return PerPartitionCategorizationConfig::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleConditionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleConditionTests.java
index 0fea18cac722..de989eac2a66 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleConditionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleConditionTests.java
@@ -17,6 +17,11 @@ protected RuleCondition createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected RuleCondition mutateInstance(RuleCondition instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static RuleCondition createRandom() {
         RuleCondition.AppliesTo appliesTo = randomFrom(RuleCondition.AppliesTo.values());
         Operator operator = randomFrom(Operator.LT, Operator.LTE, Operator.GT, Operator.GTE);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleScopeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleScopeTests.java
index da90cdfd70b1..104f7c2de2d3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleScopeTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleScopeTests.java
@@ -35,6 +35,11 @@ protected RuleScope createTestInstance() {
         return scope.build();
     }
 
+    @Override
+    protected RuleScope mutateInstance(RuleScope instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<RuleScope> instanceReader() {
         return RuleScope::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/CategorizerStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/CategorizerStatsTests.java
index 87d2f8165dc1..1c84b7695c2e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/CategorizerStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/CategorizerStatsTests.java
@@ -31,6 +31,11 @@ protected CategorizerStats createTestInstance() {
         return createRandomized("foo");
     }
 
+    @Override
+    protected CategorizerStats mutateInstance(CategorizerStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static CategorizerStats createRandomized(String jobId) {
         CategorizerStats.Builder stats = new CategorizerStats.Builder(jobId);
         if (randomBoolean()) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/DataCountsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/DataCountsTests.java
index a0a08ef5085c..7d6f5389ff11 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/DataCountsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/DataCountsTests.java
@@ -49,6 +49,11 @@ public DataCounts createTestInstance() {
         return createTestInstance(randomAlphaOfLength(10));
     }
 
+    @Override
+    protected DataCounts mutateInstance(DataCounts instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<DataCounts> instanceReader() {
         return DataCounts::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStatsTests.java
index 3bfb4e7f248d..8aa5088f41de 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStatsTests.java
@@ -62,6 +62,11 @@ protected ModelSizeStats createTestInstance() {
         return createRandomized();
     }
 
+    @Override
+    protected ModelSizeStats mutateInstance(ModelSizeStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static ModelSizeStats createRandomized() {
         ModelSizeStats.Builder stats = new ModelSizeStats.Builder("foo");
         if (randomBoolean()) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshotTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshotTests.java
index d02f66313cbb..192e2eb9a9c1 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshotTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshotTests.java
@@ -151,6 +151,11 @@ protected ModelSnapshot createTestInstance() {
         return createRandomized();
     }
 
+    @Override
+    protected ModelSnapshot mutateInstance(ModelSnapshot instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static ModelSnapshot createRandomized() {
         ModelSnapshot.Builder modelSnapshot = new ModelSnapshot.Builder(randomAlphaOfLengthBetween(1, 20));
         modelSnapshot.setMinVersion(Version.CURRENT);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/QuantilesTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/QuantilesTests.java
index 8550087bd953..9598eacab194 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/QuantilesTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/QuantilesTests.java
@@ -90,6 +90,11 @@ protected Quantiles createTestInstance() {
         return createRandomized();
     }
 
+    @Override
+    protected Quantiles mutateInstance(Quantiles instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static Quantiles createRandomized() {
         return new Quantiles(
             randomAlphaOfLengthBetween(1, 20),
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/TimingStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/TimingStatsTests.java
index 76ed3e518e95..02eb85419b84 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/TimingStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/TimingStatsTests.java
@@ -44,6 +44,11 @@ public TimingStats createTestInstance() {
         return createTestInstance(randomAlphaOfLength(10));
     }
 
+    @Override
+    protected TimingStats mutateInstance(TimingStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<TimingStats> instanceReader() {
         return TimingStats::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyCauseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyCauseTests.java
index f7c04fba61e4..24a98720f3df 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyCauseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyCauseTests.java
@@ -89,6 +89,11 @@ protected AnomalyCause createTestInstance() {
         return anomalyCause;
     }
 
+    @Override
+    protected AnomalyCause mutateInstance(AnomalyCause instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<AnomalyCause> instanceReader() {
         return AnomalyCause::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecordTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecordTests.java
index 6f0acaffd862..775430483f44 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecordTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecordTests.java
@@ -37,6 +37,11 @@ protected AnomalyRecord createTestInstance() {
         return createTestInstance("foo");
     }
 
+    @Override
+    protected AnomalyRecord mutateInstance(AnomalyRecord instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public AnomalyRecord createTestInstance(String jobId) {
         AnomalyRecord anomalyRecord = new AnomalyRecord(jobId, new Date(randomNonNegativeLong()), randomNonNegativeLong());
         anomalyRecord.setActual(Collections.singletonList(randomDouble()));
@@ -100,6 +105,7 @@ public AnomalyRecord createTestInstance(String jobId) {
             anomalyScoreExplanation.setUpperConfidenceBound(randomDouble());
             anomalyScoreExplanation.setHighVariancePenalty(randomBoolean());
             anomalyScoreExplanation.setIncompleteBucketPenalty(randomBoolean());
+            anomalyScoreExplanation.setMultimodalDistribution(randomBoolean());
         }
 
         return anomalyRecord;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencerTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencerTests.java
index ebf987f0b643..11773845cbbc 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencerTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencerTests.java
@@ -46,6 +46,11 @@ protected BucketInfluencer createTestInstance() {
         return bucketInfluencer;
     }
 
+    @Override
+    protected BucketInfluencer mutateInstance(BucketInfluencer instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<BucketInfluencer> instanceReader() {
         return BucketInfluencer::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/GeoResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/GeoResultsTests.java
index e4cf313ceeb0..e38c96cca957 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/GeoResultsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/GeoResultsTests.java
@@ -41,6 +41,11 @@ protected GeoResults createTestInstance() {
         return createTestGeoResults();
     }
 
+    @Override
+    protected GeoResults mutateInstance(GeoResults instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<GeoResults> instanceReader() {
         return GeoResults::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/InfluencerTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/InfluencerTests.java
index 2b7ae92a03df..32a6bfc1e753 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/InfluencerTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/InfluencerTests.java
@@ -40,6 +40,11 @@ protected Influencer createTestInstance() {
         return createTestInstance(randomAlphaOfLengthBetween(1, 20));
     }
 
+    @Override
+    protected Influencer mutateInstance(Influencer instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<Influencer> instanceReader() {
         return Influencer::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/snapshot/upgrade/SnapshotUpgradeTaskParamsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/snapshot/upgrade/SnapshotUpgradeTaskParamsTests.java
index a133be53f06a..b8f357a82aeb 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/snapshot/upgrade/SnapshotUpgradeTaskParamsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/snapshot/upgrade/SnapshotUpgradeTaskParamsTests.java
@@ -25,6 +25,11 @@ protected SnapshotUpgradeTaskParams createTestInstance() {
         return new SnapshotUpgradeTaskParams(randomAlphaOfLength(10), randomAlphaOfLength(20));
     }
 
+    @Override
+    protected SnapshotUpgradeTaskParams mutateInstance(SnapshotUpgradeTaskParams instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<SnapshotUpgradeTaskParams> instanceReader() {
         return SnapshotUpgradeTaskParams::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/stats/CountAccumulatorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/stats/CountAccumulatorTests.java
index bdfbe4b12178..0c888fde13ea 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/stats/CountAccumulatorTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/stats/CountAccumulatorTests.java
@@ -93,6 +93,11 @@ public CountAccumulator createTestInstance() {
         return accumulator;
     }
 
+    @Override
+    protected CountAccumulator mutateInstance(CountAccumulator instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<CountAccumulator> instanceReader() {
         return CountAccumulator::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/stats/ForecastStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/stats/ForecastStatsTests.java
index 67e053c9cb9c..1b30e5763d57 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/stats/ForecastStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/stats/ForecastStatsTests.java
@@ -231,6 +231,11 @@ public ForecastStats createTestInstance() {
         return createForecastStats(1, 22);
     }
 
+    @Override
+    protected ForecastStats mutateInstance(ForecastStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<ForecastStats> instanceReader() {
         return ForecastStats::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/stats/StatsAccumulatorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/stats/StatsAccumulatorTests.java
index f265ac4300d9..ccbd620970f5 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/stats/StatsAccumulatorTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/stats/StatsAccumulatorTests.java
@@ -155,6 +155,11 @@ public StatsAccumulator createTestInstance() {
         return accumulator;
     }
 
+    @Override
+    protected StatsAccumulator mutateInstance(StatsAccumulator instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<StatsAccumulator> instanceReader() {
         return StatsAccumulator::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelperTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelperTests.java
index e5bd6672ae2a..771a679b9369 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelperTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelperTests.java
@@ -8,9 +8,11 @@
 package org.elasticsearch.xpack.core.ml.utils;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.indices.IndexCreationException;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -46,4 +48,12 @@ public void testFindSearchExceptionRootCause_GivenWrapperException() {
 
         assertThat(rootCauseException.getMessage(), equalTo("cause"));
     }
+
+    public void testTaskOperationFailureToStatusException() {
+        var rootCause = new IllegalArgumentException("bah");
+        var failure = new TaskOperationFailure("foo", 0L, rootCause);
+        var convertedException = ExceptionsHelper.taskOperationFailureToStatusException(failure);
+        assertThat(convertedException.status(), equalTo(RestStatus.BAD_REQUEST));
+        assertThat(convertedException.getCause(), sameInstance(rootCause));
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/ExponentialAverageCalculationContextTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/ExponentialAverageCalculationContextTests.java
index 1d81bf15fa07..4fc448c376e7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/ExponentialAverageCalculationContextTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/ExponentialAverageCalculationContextTests.java
@@ -31,6 +31,11 @@ public ExponentialAverageCalculationContext createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected ExponentialAverageCalculationContext mutateInstance(ExponentialAverageCalculationContext instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<ExponentialAverageCalculationContext> instanceReader() {
         return ExponentialAverageCalculationContext::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/PhaseProgressTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/PhaseProgressTests.java
index 9222464e4ffb..7989b58065bc 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/PhaseProgressTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/PhaseProgressTests.java
@@ -19,6 +19,11 @@ protected PhaseProgress createTestInstance() {
         return createRandom();
     }
 
+    @Override
+    protected PhaseProgress mutateInstance(PhaseProgress instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static PhaseProgress createRandom() {
         return new PhaseProgress(randomAlphaOfLength(10), randomIntBetween(0, 100));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/DownsampleActionConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/DownsampleActionConfigTests.java
index f1a44b5e89c5..67f081bf7bf0 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/DownsampleActionConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/DownsampleActionConfigTests.java
@@ -23,6 +23,11 @@ protected DownsampleConfig createTestInstance() {
         return randomConfig();
     }
 
+    @Override
+    protected DownsampleConfig mutateInstance(DownsampleConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static DownsampleConfig randomConfig() {
         return new DownsampleConfig(ConfigTestHelpers.randomInterval());
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/action/RollupShardStatusSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/action/RollupShardStatusSerializingTests.java
new file mode 100644
index 000000000000..2802b1765abb
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/action/RollupShardStatusSerializingTests.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.rollup.action;
+
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+
+public class RollupShardStatusSerializingTests extends AbstractXContentSerializingTestCase<RollupShardStatus> {
+    @Override
+    protected RollupShardStatus doParseInstance(XContentParser parser) throws IOException {
+        return RollupShardStatus.fromXContent(parser);
+    }
+
+    @Override
+    protected Reader<RollupShardStatus> instanceReader() {
+        return RollupShardStatus::new;
+    }
+
+    @Override
+    protected RollupShardStatus createTestInstance() {
+        RollupShardStatus rollupShardStatus = new RollupShardStatus(
+            new ShardId(randomAlphaOfLength(5), randomAlphaOfLength(5), randomInt(5)),
+            randomMillisUpToYear9999(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong()
+        );
+        return rollupShardStatus;
+    }
+
+    @Override
+    protected RollupShardStatus mutateInstance(RollupShardStatus instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfigSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfigSerializingTests.java
index 7c82407a37e0..f56cf1c77760 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfigSerializingTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/DateHistogramGroupConfigSerializingTests.java
@@ -65,6 +65,11 @@ protected DateHistogramGroupConfig createTestInstance() {
         return config;
     }
 
+    @Override
+    protected DateHistogramGroupConfig mutateInstance(DateHistogramGroupConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testValidateNoMapping() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/GroupConfigSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/GroupConfigSerializingTests.java
index 80f3a07e4784..0421ff170a64 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/GroupConfigSerializingTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/GroupConfigSerializingTests.java
@@ -30,4 +30,9 @@ protected Writeable.Reader<GroupConfig> instanceReader() {
     protected GroupConfig createTestInstance() {
         return randomGroupConfig(random());
     }
+
+    @Override
+    protected GroupConfig mutateInstance(GroupConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/HistogramGroupConfigSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/HistogramGroupConfigSerializingTests.java
index 1a365a227cad..d5adf441b6cd 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/HistogramGroupConfigSerializingTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/HistogramGroupConfigSerializingTests.java
@@ -40,6 +40,11 @@ protected HistogramGroupConfig createTestInstance() {
         return randomHistogramGroupConfig(random());
     }
 
+    @Override
+    protected HistogramGroupConfig mutateInstance(HistogramGroupConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testValidateNoMapping() throws IOException {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/JobWrapperSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/JobWrapperSerializingTests.java
index 83effa0bcbe7..f6a14b9e5bc8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/JobWrapperSerializingTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/JobWrapperSerializingTests.java
@@ -60,4 +60,9 @@ protected GetRollupJobsAction.JobWrapper createTestInstance() {
             new RollupJobStatus(state, Collections.emptyMap())
         );
     }
+
+    @Override
+    protected GetRollupJobsAction.JobWrapper mutateInstance(GetRollupJobsAction.JobWrapper instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricConfigSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricConfigSerializingTests.java
index 1d74eabc9321..f5d14ee8cb77 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricConfigSerializingTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricConfigSerializingTests.java
@@ -45,6 +45,11 @@ protected MetricConfig createTestInstance() {
         return ConfigTestHelpers.randomMetricConfig(random());
     }
 
+    @Override
+    protected MetricConfig mutateInstance(MetricConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testValidateNoMapping() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupIndexerJobStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupIndexerJobStatsTests.java
index e1353f5cbd60..980c18676bb3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupIndexerJobStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupIndexerJobStatsTests.java
@@ -17,6 +17,11 @@ protected RollupIndexerJobStats createTestInstance() {
         return randomStats();
     }
 
+    @Override
+    protected RollupIndexerJobStats mutateInstance(RollupIndexerJobStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<RollupIndexerJobStats> instanceReader() {
         return RollupIndexerJobStats::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfigTests.java
index 387c1c8c0956..4c50abcfcfe1 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfigTests.java
@@ -32,6 +32,11 @@ protected RollupJobConfig createTestInstance() {
         return randomRollupJobConfig(random(), jobId);
     }
 
+    @Override
+    protected RollupJobConfig mutateInstance(RollupJobConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<RollupJobConfig> instanceReader() {
         return RollupJobConfig::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatusTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatusTests.java
index 3fc87ec1fe0e..b0c61b155f61 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatusTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobStatusTests.java
@@ -38,6 +38,11 @@ protected RollupJobStatus createTestInstance() {
         return new RollupJobStatus(randomFrom(IndexerState.values()), randomPosition());
     }
 
+    @Override
+    protected RollupJobStatus mutateInstance(RollupJobStatus instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<RollupJobStatus> instanceReader() {
         return RollupJobStatus::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobTests.java
index a6cc077858d1..64489fde9e7a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobTests.java
@@ -47,6 +47,11 @@ protected RollupJob createTestInstance() {
         return new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), headers);
     }
 
+    @Override
+    protected RollupJob mutateInstance(RollupJob instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected RollupJob makeTestChanges(RollupJob other) {
         if (randomBoolean()) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfigSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfigSerializingTests.java
index becffa1b9354..e715d8ecaca0 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfigSerializingTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/TermsGroupConfigSerializingTests.java
@@ -39,6 +39,11 @@ protected TermsGroupConfig createTestInstance() {
         return randomTermsGroupConfig(random());
     }
 
+    @Override
+    protected TermsGroupConfig mutateInstance(TermsGroupConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testValidateNoMapping() {
         ActionRequestValidationException e = new ActionRequestValidationException();
         Map<String, Map<String, FieldCapabilities>> responseMap = new HashMap<>();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/DataStreamFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/DataStreamFeatureSetUsageTests.java
index a2b6884a5b75..76cf0ed99ebc 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/DataStreamFeatureSetUsageTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/DataStreamFeatureSetUsageTests.java
@@ -11,8 +11,6 @@
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.datastreams.DataStreamFeatureSetUsage;
 
-import java.io.IOException;
-
 public class DataStreamFeatureSetUsageTests extends AbstractWireSerializingTestCase<DataStreamFeatureSetUsage> {
 
     @Override
@@ -23,7 +21,7 @@ protected DataStreamFeatureSetUsage createTestInstance() {
     }
 
     @Override
-    protected DataStreamFeatureSetUsage mutateInstance(DataStreamFeatureSetUsage instance) throws IOException {
+    protected DataStreamFeatureSetUsage mutateInstance(DataStreamFeatureSetUsage instance) {
         return randomValueOtherThan(instance, this::createTestInstance);
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotShardStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotShardStatsTests.java
index c422bd3647a2..10611e7502ee 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotShardStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotShardStatsTests.java
@@ -47,6 +47,11 @@ protected SearchableSnapshotShardStats createTestInstance() {
         return new SearchableSnapshotShardStats(shardRouting, snapshotId, indexId, inputStats);
     }
 
+    @Override
+    protected SearchableSnapshotShardStats mutateInstance(SearchableSnapshotShardStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     private CacheIndexInputStats randomCacheIndexInputStats() {
         return new CacheIndexInputStats(
             randomAlphaOfLength(10),
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotsFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotsFeatureSetUsageTests.java
index dc4564ac2912..aa6546c0cf73 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotsFeatureSetUsageTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/SearchableSnapshotsFeatureSetUsageTests.java
@@ -9,8 +9,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
-
 public class SearchableSnapshotsFeatureSetUsageTests extends AbstractWireSerializingTestCase<SearchableSnapshotFeatureSetUsage> {
 
     @Override
@@ -20,7 +18,7 @@ protected SearchableSnapshotFeatureSetUsage createTestInstance() {
     }
 
     @Override
-    protected SearchableSnapshotFeatureSetUsage mutateInstance(SearchableSnapshotFeatureSetUsage instance) throws IOException {
+    protected SearchableSnapshotFeatureSetUsage mutateInstance(SearchableSnapshotFeatureSetUsage instance) {
         boolean available = instance.available();
         int numFullCopySearchableSnapshotIndices = instance.getNumberOfFullCopySearchableSnapshotIndices();
         int numSharedCacheSearchableSnapshotIndices = instance.getNumberOfSharedCacheSearchableSnapshotIndices();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequestBuilderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequestBuilderTests.java
index e143c620a4df..da0eecbccf48 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequestBuilderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequestBuilderTests.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentType;
@@ -27,7 +28,9 @@ public class CreateApiKeyRequestBuilderTests extends ESTestCase {
 
     public void testParserAndCreateApiRequestBuilder() throws IOException {
         boolean withExpiration = randomBoolean();
-        final String json = formatted("""
+        Object[] args = new Object[] { withExpiration ? """
+            "expiration": "1d",""" : "" };
+        final String json = Strings.format("""
             {
               "name": "my-api-key",
               %s
@@ -51,8 +54,7 @@ public void testParserAndCreateApiRequestBuilder() throws IOException {
                   ]
                 }
               }
-            }""", withExpiration ? """
-            "expiration": "1d",""" : "");
+            }""", args);
         final BytesArray source = new BytesArray(json);
         final NodeClient mockClient = mock(NodeClient.class);
         final CreateApiKeyRequest request = new CreateApiKeyRequestBuilder(mockClient).source(source, XContentType.JSON).request();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/GetApiKeyRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/GetApiKeyRequestTests.java
index a94351c08f4e..2807e285d9a3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/GetApiKeyRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/GetApiKeyRequestTests.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.security.action.apikey;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
@@ -20,7 +20,7 @@
 import java.io.IOException;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
+import static org.elasticsearch.test.TransportVersionUtils.randomVersionBetween;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
@@ -127,11 +127,11 @@ public void testSerialization() throws IOException {
             final GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.builder().ownedByAuthenticatedUser(true).apiKeyId(apiKeyId).build();
             ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
             OutputStreamStreamOutput out = new OutputStreamStreamOutput(outBuffer);
-            out.setVersion(randomVersionBetween(random(), Version.V_7_0_0, Version.V_7_3_0));
+            out.setTransportVersion(randomVersionBetween(random(), TransportVersion.V_7_0_0, TransportVersion.V_7_3_0));
             getApiKeyRequest.writeTo(out);
 
             InputStreamStreamInput inputStreamStreamInput = new InputStreamStreamInput(new ByteArrayInputStream(outBuffer.toByteArray()));
-            inputStreamStreamInput.setVersion(randomVersionBetween(random(), Version.V_7_0_0, Version.V_7_3_0));
+            inputStreamStreamInput.setTransportVersion(randomVersionBetween(random(), TransportVersion.V_7_0_0, TransportVersion.V_7_3_0));
             GetApiKeyRequest requestFromInputStream = new GetApiKeyRequest(inputStreamStreamInput);
 
             assertThat(requestFromInputStream.getApiKeyId(), equalTo(getApiKeyRequest.getApiKeyId()));
@@ -146,11 +146,11 @@ public void testSerialization() throws IOException {
                 .build();
             ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
             OutputStreamStreamOutput out = new OutputStreamStreamOutput(outBuffer);
-            out.setVersion(randomVersionBetween(random(), Version.V_7_4_0, Version.V_8_4_0));
+            out.setTransportVersion(randomVersionBetween(random(), TransportVersion.V_7_4_0, TransportVersion.V_8_4_0));
             getApiKeyRequest.writeTo(out);
 
             InputStreamStreamInput inputStreamStreamInput = new InputStreamStreamInput(new ByteArrayInputStream(outBuffer.toByteArray()));
-            inputStreamStreamInput.setVersion(randomVersionBetween(random(), Version.V_7_4_0, Version.V_8_4_0));
+            inputStreamStreamInput.setTransportVersion(randomVersionBetween(random(), TransportVersion.V_7_4_0, TransportVersion.V_8_4_0));
             GetApiKeyRequest requestFromInputStream = new GetApiKeyRequest(inputStreamStreamInput);
 
             assertThat(requestFromInputStream.getApiKeyId(), equalTo(getApiKeyRequest.getApiKeyId()));
@@ -162,11 +162,11 @@ public void testSerialization() throws IOException {
             final GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.builder().apiKeyId(apiKeyId).withLimitedBy(randomBoolean()).build();
             ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
             OutputStreamStreamOutput out = new OutputStreamStreamOutput(outBuffer);
-            out.setVersion(randomVersionBetween(random(), Version.V_8_5_0, Version.CURRENT));
+            out.setTransportVersion(randomVersionBetween(random(), TransportVersion.V_8_5_0, TransportVersion.CURRENT));
             getApiKeyRequest.writeTo(out);
 
             InputStreamStreamInput inputStreamStreamInput = new InputStreamStreamInput(new ByteArrayInputStream(outBuffer.toByteArray()));
-            inputStreamStreamInput.setVersion(randomVersionBetween(random(), Version.V_8_5_0, Version.CURRENT));
+            inputStreamStreamInput.setTransportVersion(randomVersionBetween(random(), TransportVersion.V_8_5_0, TransportVersion.CURRENT));
             GetApiKeyRequest requestFromInputStream = new GetApiKeyRequest(inputStreamStreamInput);
 
             assertThat(requestFromInputStream, equalTo(getApiKeyRequest));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/QueryApiKeyResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/QueryApiKeyResponseTests.java
index 6fd3c2594ab9..a1364b4b60ef 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/QueryApiKeyResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/QueryApiKeyResponseTests.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges;
 
-import java.io.IOException;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -38,7 +37,7 @@ protected QueryApiKeyResponse createTestInstance() {
     }
 
     @Override
-    protected QueryApiKeyResponse mutateInstance(QueryApiKeyResponse instance) throws IOException {
+    protected QueryApiKeyResponse mutateInstance(QueryApiKeyResponse instance) {
         final List<QueryApiKeyResponse.Item> items = Arrays.stream(instance.getItems()).collect(Collectors.toCollection(ArrayList::new));
         switch (randomIntBetween(0, 3)) {
             case 0:
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/enrollment/KibanaEnrollmentResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/enrollment/KibanaEnrollmentResponseTests.java
index 3333495769c5..fe14c967b818 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/enrollment/KibanaEnrollmentResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/enrollment/KibanaEnrollmentResponseTests.java
@@ -38,7 +38,7 @@ protected KibanaEnrollmentResponse createTestInstance() {
     }
 
     @Override
-    protected KibanaEnrollmentResponse mutateInstance(KibanaEnrollmentResponse instance) throws IOException {
+    protected KibanaEnrollmentResponse mutateInstance(KibanaEnrollmentResponse instance) {
         return switch (randomIntBetween(0, 3)) {
             case 0 -> new KibanaEnrollmentResponse(
                 randomAlphaOfLengthBetween(14, 20),
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/profile/ProfileHasPrivilegesRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/profile/ProfileHasPrivilegesRequestTests.java
index 9737e0255340..0d84eb041ee0 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/profile/ProfileHasPrivilegesRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/profile/ProfileHasPrivilegesRequestTests.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilegeResolver;
 import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
 
-import java.io.IOException;
 import java.util.List;
 import java.util.Locale;
 
@@ -40,7 +39,7 @@ protected ProfileHasPrivilegesRequest createTestInstance() {
     }
 
     @Override
-    protected ProfileHasPrivilegesRequest mutateInstance(ProfileHasPrivilegesRequest instance) throws IOException {
+    protected ProfileHasPrivilegesRequest mutateInstance(ProfileHasPrivilegesRequest instance) {
         if (randomBoolean()) {
             if (instance.profileUids() == null || instance.profileUids().isEmpty()) {
                 return new ProfileHasPrivilegesRequest(
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/profile/ProfileHasPrivilegesResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/profile/ProfileHasPrivilegesResponseTests.java
index e4724d798850..943c0a8191f9 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/profile/ProfileHasPrivilegesResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/profile/ProfileHasPrivilegesResponseTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -48,7 +49,7 @@ protected ProfileHasPrivilegesResponse createTestInstance() {
     }
 
     @Override
-    protected ProfileHasPrivilegesResponse mutateInstance(ProfileHasPrivilegesResponse instance) throws IOException {
+    protected ProfileHasPrivilegesResponse mutateInstance(ProfileHasPrivilegesResponse instance) {
         return randomFrom(
             new ProfileHasPrivilegesResponse(newMutatedSet(instance.hasPrivilegeUids()), instance.errors()),
             new ProfileHasPrivilegesResponse(instance.hasPrivilegeUids(), randomValueOtherThan(instance.errors(), this::randomErrors)),
@@ -81,19 +82,19 @@ public void testToXContent() throws IOException {
                 final String errorString;
                 final Exception e = response.errors().get(k);
                 if (e instanceof IllegalArgumentException illegalArgumentException) {
-                    errorString = formatted("""
+                    errorString = Strings.format("""
                         {
                           "type": "illegal_argument_exception",
                           "reason": "%s"
                         }""", illegalArgumentException.getMessage());
                 } else if (e instanceof ResourceNotFoundException resourceNotFoundException) {
-                    errorString = formatted("""
+                    errorString = Strings.format("""
                         {
                           "type": "resource_not_found_exception",
                           "reason": "%s"
                         }""", resourceNotFoundException.getMessage());
                 } else if (e instanceof ElasticsearchException elasticsearchException) {
-                    errorString = formatted("""
+                    errorString = Strings.format("""
                         {
                           "type": "exception",
                           "reason": "%s",
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestTests.java
index 2fa810a1a613..df26cbc1c559 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestTests.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.security.action.role;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.Strings;
@@ -18,7 +18,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.xpack.core.XPackClientPlugin;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.ApplicationResourcePrivileges;
@@ -160,17 +160,17 @@ public void testValidationOfApplicationPrivileges() {
     public void testSerialization() throws IOException {
         final BytesStreamOutput out = new BytesStreamOutput();
         if (randomBoolean()) {
-            final Version version = VersionUtils.randomCompatibleVersion(random(), Version.CURRENT);
+            final TransportVersion version = TransportVersionUtils.randomCompatibleVersion(random(), TransportVersion.CURRENT);
             logger.info("Serializing with version {}", version);
-            out.setVersion(version);
+            out.setTransportVersion(version);
         }
-        final boolean mayIncludeRemoteIndices = out.getVersion().onOrAfter(Version.V_8_6_0);
+        final boolean mayIncludeRemoteIndices = out.getTransportVersion().onOrAfter(TransportVersion.V_8_6_0);
         final PutRoleRequest original = buildRandomRequest(mayIncludeRemoteIndices);
         original.writeTo(out);
 
         final NamedWriteableRegistry registry = new NamedWriteableRegistry(new XPackClientPlugin().getNamedWriteables());
         StreamInput in = new NamedWriteableAwareStreamInput(ByteBufferStreamInput.wrap(BytesReference.toBytes(out.bytes())), registry);
-        in.setVersion(out.getVersion());
+        in.setTransportVersion(out.getTransportVersion());
         final PutRoleRequest copy = new PutRoleRequest(in);
 
         final RoleDescriptor actual = copy.roleDescriptor();
@@ -180,13 +180,13 @@ public void testSerialization() throws IOException {
 
     public void testSerializationWithRemoteIndicesThrowsOnUnsupportedVersions() throws IOException {
         final BytesStreamOutput out = new BytesStreamOutput();
-        final Version versionBeforeRemoteIndices = VersionUtils.getPreviousVersion(Version.V_8_6_0);
-        final Version version = VersionUtils.randomVersionBetween(
+        final TransportVersion versionBeforeRemoteIndices = TransportVersionUtils.getPreviousVersion(TransportVersion.V_8_6_0);
+        final TransportVersion version = TransportVersionUtils.randomVersionBetween(
             random(),
-            versionBeforeRemoteIndices.minimumCompatibilityVersion(),
+            versionBeforeRemoteIndices.calculateMinimumCompatVersion(),
             versionBeforeRemoteIndices
         );
-        out.setVersion(version);
+        out.setTransportVersion(version);
 
         final PutRoleRequest original = buildRandomRequest(randomBoolean());
         if (original.hasRemoteIndicesPrivileges()) {
@@ -194,7 +194,7 @@ public void testSerializationWithRemoteIndicesThrowsOnUnsupportedVersions() thro
             assertThat(
                 ex.getMessage(),
                 containsString(
-                    "versions of Elasticsearch before [8.6.0] can't handle remote indices privileges and attempted to send to ["
+                    "versions of Elasticsearch before [8060099] can't handle remote indices privileges and attempted to send to ["
                         + version
                         + "]"
                 )
@@ -203,7 +203,7 @@ public void testSerializationWithRemoteIndicesThrowsOnUnsupportedVersions() thro
             original.writeTo(out);
             final NamedWriteableRegistry registry = new NamedWriteableRegistry(new XPackClientPlugin().getNamedWriteables());
             StreamInput in = new NamedWriteableAwareStreamInput(ByteBufferStreamInput.wrap(BytesReference.toBytes(out.bytes())), registry);
-            in.setVersion(out.getVersion());
+            in.setTransportVersion(out.getTransportVersion());
             final PutRoleRequest copy = new PutRoleRequest(in);
             assertThat(copy.roleDescriptor(), equalTo(original.roleDescriptor()));
         }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/CreateServiceAccountTokenResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/CreateServiceAccountTokenResponseTests.java
index a43c47fcb132..b7429d6a0928 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/CreateServiceAccountTokenResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/CreateServiceAccountTokenResponseTests.java
@@ -37,7 +37,7 @@ protected CreateServiceAccountTokenResponse createTestInstance() {
     }
 
     @Override
-    protected CreateServiceAccountTokenResponse mutateInstance(CreateServiceAccountTokenResponse instance) throws IOException {
+    protected CreateServiceAccountTokenResponse mutateInstance(CreateServiceAccountTokenResponse instance) {
         if (randomBoolean()) {
             return CreateServiceAccountTokenResponse.created(
                 randomValueOtherThan(instance.getName(), () -> randomAlphaOfLengthBetween(3, 8)),
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/DeleteServiceAccountTokenRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/DeleteServiceAccountTokenRequestTests.java
index 29b0eaaedd77..a5fbd1f5279e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/DeleteServiceAccountTokenRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/DeleteServiceAccountTokenRequestTests.java
@@ -13,8 +13,6 @@
 import org.elasticsearch.xpack.core.security.support.Validation;
 import org.elasticsearch.xpack.core.security.support.ValidationTests;
 
-import java.io.IOException;
-
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.nullValue;
@@ -36,7 +34,7 @@ protected DeleteServiceAccountTokenRequest createTestInstance() {
     }
 
     @Override
-    protected DeleteServiceAccountTokenRequest mutateInstance(DeleteServiceAccountTokenRequest instance) throws IOException {
+    protected DeleteServiceAccountTokenRequest mutateInstance(DeleteServiceAccountTokenRequest instance) {
         DeleteServiceAccountTokenRequest newInstance = instance;
         if (randomBoolean()) {
             newInstance = new DeleteServiceAccountTokenRequest(
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/DeleteServiceAccountTokenResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/DeleteServiceAccountTokenResponseTests.java
index 8a0b1db5a467..384264dbbace 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/DeleteServiceAccountTokenResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/DeleteServiceAccountTokenResponseTests.java
@@ -33,7 +33,7 @@ protected DeleteServiceAccountTokenResponse createTestInstance() {
     }
 
     @Override
-    protected DeleteServiceAccountTokenResponse mutateInstance(DeleteServiceAccountTokenResponse instance) throws IOException {
+    protected DeleteServiceAccountTokenResponse mutateInstance(DeleteServiceAccountTokenResponse instance) {
         return new DeleteServiceAccountTokenResponse(false == instance.found());
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/GetServiceAccountCredentialsRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/GetServiceAccountCredentialsRequestTests.java
index 82a07bcc2617..68d9801d4437 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/GetServiceAccountCredentialsRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/GetServiceAccountCredentialsRequestTests.java
@@ -11,8 +11,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
-
 import static org.hamcrest.Matchers.containsString;
 
 public class GetServiceAccountCredentialsRequestTests extends AbstractWireSerializingTestCase<GetServiceAccountCredentialsRequest> {
@@ -30,7 +28,7 @@ protected GetServiceAccountCredentialsRequest createTestInstance() {
     }
 
     @Override
-    protected GetServiceAccountCredentialsRequest mutateInstance(GetServiceAccountCredentialsRequest instance) throws IOException {
+    protected GetServiceAccountCredentialsRequest mutateInstance(GetServiceAccountCredentialsRequest instance) {
         return switch (randomIntBetween(0, 2)) {
             case 0 -> new GetServiceAccountCredentialsRequest(
                 randomValueOtherThan(instance.getNamespace(), () -> randomAlphaOfLengthBetween(3, 8)),
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/GetServiceAccountRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/GetServiceAccountRequestTests.java
index fa3fe264575b..01a106efb971 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/GetServiceAccountRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/GetServiceAccountRequestTests.java
@@ -10,8 +10,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
-
 public class GetServiceAccountRequestTests extends AbstractWireSerializingTestCase<GetServiceAccountRequest> {
 
     @Override
@@ -28,7 +26,7 @@ protected GetServiceAccountRequest createTestInstance() {
     }
 
     @Override
-    protected GetServiceAccountRequest mutateInstance(GetServiceAccountRequest instance) throws IOException {
+    protected GetServiceAccountRequest mutateInstance(GetServiceAccountRequest instance) {
         if (randomBoolean()) {
             return new GetServiceAccountRequest(
                 randomValueOtherThan(instance.getNamespace(), () -> randomFrom(randomAlphaOfLengthBetween(3, 8), null)),
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/GetServiceAccountResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/GetServiceAccountResponseTests.java
index 06ae2384fc91..b473f64a3fbf 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/GetServiceAccountResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/service/GetServiceAccountResponseTests.java
@@ -42,7 +42,7 @@ protected GetServiceAccountResponse createTestInstance() {
     }
 
     @Override
-    protected GetServiceAccountResponse mutateInstance(GetServiceAccountResponse instance) throws IOException {
+    protected GetServiceAccountResponse mutateInstance(GetServiceAccountResponse instance) {
         if (instance.getServiceAccountInfos().length == 0) {
             final String principal = randomPrincipal();
             return new GetServiceAccountResponse(
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenResponseTests.java
index 34b7b41d314d..5b923802f79c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenResponseTests.java
@@ -90,7 +90,7 @@ public void testToXContent() throws IOException {
         InvalidateTokenResponse response = new InvalidateTokenResponse(result);
         XContentBuilder builder = XContentFactory.jsonBuilder();
         response.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        assertThat(Strings.toString(builder), equalTo(XContentHelper.stripWhitespace(formatted("""
+        assertThat(Strings.toString(builder), equalTo(XContentHelper.stripWhitespace(Strings.format("""
             {
               "invalidated_tokens": %s,
               "previously_invalidated_tokens": %s,
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponseTests.java
index 622dc2655255..abc6f7619ae6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponseTests.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.security.action.user;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
@@ -19,7 +19,7 @@
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.EqualsHashCodeTestUtils;
-import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.xpack.core.XPackClientPlugin;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.ApplicationResourcePrivileges;
@@ -65,31 +65,31 @@ public void testSerialization() throws IOException {
     }
 
     public void testSerializationForCurrentVersion() throws Exception {
-        final Version version = VersionUtils.randomCompatibleVersion(random(), Version.CURRENT);
+        final TransportVersion version = TransportVersionUtils.randomCompatibleVersion(random(), TransportVersion.CURRENT);
         final boolean canIncludeRemoteIndices = version.onOrAfter(RoleDescriptor.VERSION_REMOTE_INDICES);
 
         final GetUserPrivilegesResponse original = randomResponse(canIncludeRemoteIndices);
 
         final BytesStreamOutput out = new BytesStreamOutput();
-        out.setVersion(version);
+        out.setTransportVersion(version);
         original.writeTo(out);
 
         final NamedWriteableRegistry registry = new NamedWriteableRegistry(new XPackClientPlugin().getNamedWriteables());
         StreamInput in = new NamedWriteableAwareStreamInput(ByteBufferStreamInput.wrap(BytesReference.toBytes(out.bytes())), registry);
-        in.setVersion(version);
+        in.setTransportVersion(version);
         final GetUserPrivilegesResponse copy = new GetUserPrivilegesResponse(in);
         assertThat(copy, equalTo(original));
     }
 
     public void testSerializationWithRemoteIndicesThrowsOnUnsupportedVersions() throws IOException {
         final BytesStreamOutput out = new BytesStreamOutput();
-        final Version versionBeforeRemoteIndices = VersionUtils.getPreviousVersion(RoleDescriptor.VERSION_REMOTE_INDICES);
-        final Version version = VersionUtils.randomVersionBetween(
+        final TransportVersion versionBeforeRemoteIndices = TransportVersionUtils.getPreviousVersion(RoleDescriptor.VERSION_REMOTE_INDICES);
+        final TransportVersion version = TransportVersionUtils.randomVersionBetween(
             random(),
-            versionBeforeRemoteIndices.minimumCompatibilityVersion(),
+            versionBeforeRemoteIndices.calculateMinimumCompatVersion(),
             versionBeforeRemoteIndices
         );
-        out.setVersion(version);
+        out.setTransportVersion(version);
 
         final GetUserPrivilegesResponse original = randomResponse();
         if (original.hasRemoteIndicesPrivileges()) {
@@ -97,7 +97,7 @@ public void testSerializationWithRemoteIndicesThrowsOnUnsupportedVersions() thro
             assertThat(
                 ex.getMessage(),
                 containsString(
-                    "versions of Elasticsearch before [8.6.0] can't handle remote indices privileges and attempted to send to ["
+                    "versions of Elasticsearch before [8060099] can't handle remote indices privileges and attempted to send to ["
                         + version
                         + "]"
                 )
@@ -106,7 +106,7 @@ public void testSerializationWithRemoteIndicesThrowsOnUnsupportedVersions() thro
             original.writeTo(out);
             final NamedWriteableRegistry registry = new NamedWriteableRegistry(new XPackClientPlugin().getNamedWriteables());
             StreamInput in = new NamedWriteableAwareStreamInput(ByteBufferStreamInput.wrap(BytesReference.toBytes(out.bytes())), registry);
-            in.setVersion(out.getVersion());
+            in.setTransportVersion(out.getTransportVersion());
             final GetUserPrivilegesResponse copy = new GetUserPrivilegesResponse(in);
             assertThat(copy, equalTo(original));
         }
@@ -198,7 +198,7 @@ private GetUserPrivilegesResponse randomResponse(boolean allowRemoteIndices) {
             randomArray(3, ConfigurableClusterPrivilege[]::new, () -> new ManageApplicationPrivileges(randomStringSet(3)))
         );
         final Set<GetUserPrivilegesResponse.Indices> index = Sets.newHashSet(
-            randomArray(5, GetUserPrivilegesResponse.Indices[]::new, this::randomIndices)
+            randomArray(5, GetUserPrivilegesResponse.Indices[]::new, () -> randomIndices(true))
         );
         final Set<ApplicationResourcePrivileges> application = Sets.newHashSet(
             randomArray(
@@ -217,7 +217,7 @@ private GetUserPrivilegesResponse randomResponse(boolean allowRemoteIndices) {
                 randomArray(
                     5,
                     GetUserPrivilegesResponse.RemoteIndices[]::new,
-                    () -> new GetUserPrivilegesResponse.RemoteIndices(randomIndices(), randomStringSet(6))
+                    () -> new GetUserPrivilegesResponse.RemoteIndices(randomIndices(false), randomStringSet(6))
                 )
             )
             : Set.of();
@@ -225,13 +225,13 @@ private GetUserPrivilegesResponse randomResponse(boolean allowRemoteIndices) {
         return new GetUserPrivilegesResponse(cluster, conditionalCluster, index, application, runAs, remoteIndex);
     }
 
-    private GetUserPrivilegesResponse.Indices randomIndices() {
+    private GetUserPrivilegesResponse.Indices randomIndices(boolean allowMultipleFlsDlsDefinitions) {
         return new GetUserPrivilegesResponse.Indices(
             randomStringSet(6),
             randomStringSet(8),
             Sets.newHashSet(
                 randomArray(
-                    3,
+                    allowMultipleFlsDlsDefinitions ? 3 : 1,
                     FieldGrantExcludeGroup[]::new,
                     () -> new FieldGrantExcludeGroup(
                         generateRandomStringArray(3, 5, false, false),
@@ -239,7 +239,7 @@ private GetUserPrivilegesResponse.Indices randomIndices() {
                     )
                 )
             ),
-            randomStringSet(3).stream().map(BytesArray::new).collect(Collectors.toSet()),
+            randomStringSet(allowMultipleFlsDlsDefinitions ? 3 : 1).stream().map(BytesArray::new).collect(Collectors.toSet()),
             randomBoolean()
         );
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/GetUsersResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/GetUsersResponseTests.java
index 472de63601c6..d1901b9665ce 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/GetUsersResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/GetUsersResponseTests.java
@@ -66,7 +66,7 @@ public void testToXContent() throws IOException {
             Strings.toString(builder),
             equalTo(
                 XContentHelper.stripWhitespace(
-                    formatted(
+                    Strings.format(
                         """
                             {
                               "%s": {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesRequestTests.java
index 55ca8e420e3b..66eae470af5b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesRequestTests.java
@@ -7,13 +7,13 @@
 
 package org.elasticsearch.xpack.core.security.action.user;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.ApplicationResourcePrivileges;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilegeResolver;
@@ -31,7 +31,7 @@ public class HasPrivilegesRequestTests extends ESTestCase {
 
     public void testSerializationCurrentVersion() throws IOException {
         final HasPrivilegesRequest original = randomRequest();
-        final Version version = VersionUtils.randomCompatibleVersion(random(), Version.CURRENT);
+        final TransportVersion version = TransportVersionUtils.randomCompatibleVersion(random(), TransportVersion.CURRENT);
         final HasPrivilegesRequest copy = serializeAndDeserialize(original, version);
 
         assertThat(copy.username(), equalTo(original.username()));
@@ -72,13 +72,13 @@ public void testValidateNoWildcardApplicationPrivileges() {
         assertThat(exception.validationErrors(), hasItem("Application names may not contain '*' (found '*')"));
     }
 
-    private HasPrivilegesRequest serializeAndDeserialize(HasPrivilegesRequest original, Version version) throws IOException {
+    private HasPrivilegesRequest serializeAndDeserialize(HasPrivilegesRequest original, TransportVersion version) throws IOException {
         final BytesStreamOutput out = new BytesStreamOutput();
-        out.setVersion(version);
+        out.setTransportVersion(version);
         original.writeTo(out);
 
         final StreamInput in = out.bytes().streamInput();
-        in.setVersion(version);
+        in.setTransportVersion(version);
         final HasPrivilegesRequest copy = new HasPrivilegesRequest(in);
         assertThat(in.read(), equalTo(-1));
         return copy;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationSerializationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationSerializationTests.java
index 783da11cb510..ce9dd27b2ab0 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationSerializationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationSerializationTests.java
@@ -6,8 +6,11 @@
  */
 package org.elasticsearch.xpack.core.security.authc;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.xpack.core.security.user.AsyncSearchUser;
 import org.elasticsearch.xpack.core.security.user.ElasticUser;
 import org.elasticsearch.xpack.core.security.user.KibanaSystemUser;
@@ -19,6 +22,7 @@
 import java.util.Arrays;
 
 import static org.elasticsearch.xpack.core.security.authc.Authentication.AuthenticationSerializationHelper;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
@@ -59,6 +63,52 @@ public void testWriteToAndReadFromWithRunAs() throws Exception {
         assertThat(readFromAuthenticatingUser, equalTo(authentication.getAuthenticatingSubject().getUser()));
     }
 
+    public void testWriteToAndReadFromWithRemoteAccess() throws Exception {
+        final Authentication authentication = AuthenticationTestHelper.builder().remoteAccess().build();
+        assertThat(authentication.isRemoteAccess(), is(true));
+
+        BytesStreamOutput output = new BytesStreamOutput();
+        authentication.writeTo(output);
+        final Authentication readFrom = new Authentication(output.bytes().streamInput());
+        assertThat(readFrom.isRemoteAccess(), is(true));
+
+        assertThat(readFrom, not(sameInstance(authentication)));
+        assertThat(readFrom, equalTo(authentication));
+    }
+
+    public void testWriteToWithRemoteAccessThrowsOnUnsupportedVersion() throws Exception {
+        final Authentication authentication = randomBoolean()
+            ? AuthenticationTestHelper.builder().remoteAccess().build()
+            : AuthenticationTestHelper.builder().build();
+
+        final BytesStreamOutput out = new BytesStreamOutput();
+        final TransportVersion version = TransportVersionUtils.randomPreviousCompatibleVersion(
+            random(),
+            Authentication.VERSION_REMOTE_ACCESS_REALM
+        );
+        out.setTransportVersion(version);
+
+        if (authentication.isRemoteAccess()) {
+            final var ex = expectThrows(IllegalArgumentException.class, () -> authentication.writeTo(out));
+            assertThat(
+                ex.getMessage(),
+                containsString(
+                    "versions of Elasticsearch before ["
+                        + Authentication.VERSION_REMOTE_ACCESS_REALM
+                        + "] can't handle remote access authentication and attempted to send to ["
+                        + out.getTransportVersion()
+                        + "]"
+                )
+            );
+        } else {
+            authentication.writeTo(out);
+            final StreamInput in = out.bytes().streamInput();
+            in.setTransportVersion(out.getTransportVersion());
+            final Authentication readFrom = new Authentication(in);
+            assertThat(readFrom, equalTo(authentication.maybeRewriteForOlderVersion(out.getTransportVersion())));
+        }
+    }
+
     public void testSystemUserReadAndWrite() throws Exception {
         BytesStreamOutput output = new BytesStreamOutput();
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTestHelper.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTestHelper.java
index 3b86b6b32f46..69f2b7d9ced1 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTestHelper.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTestHelper.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.security.authc;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -26,6 +26,8 @@
 import org.elasticsearch.xpack.core.security.authc.pki.PkiRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.saml.SamlRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountSettings;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.security.user.AsyncSearchUser;
 import org.elasticsearch.xpack.core.security.user.SecurityProfileUser;
@@ -36,6 +38,7 @@
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -68,7 +71,8 @@ public class AuthenticationTestHelper {
         AuthenticationField.ANONYMOUS_REALM_TYPE,
         AuthenticationField.ATTACH_REALM_TYPE,
         AuthenticationField.FALLBACK_REALM_TYPE,
-        ServiceAccountSettings.REALM_TYPE
+        ServiceAccountSettings.REALM_TYPE,
+        AuthenticationField.REMOTE_ACCESS_REALM_TYPE
     );
 
     private static final Set<User> INTERNAL_USERS = Set.of(
@@ -90,6 +94,10 @@ public static User randomUser() {
         );
     }
 
+    public static User randomInternalUser() {
+        return ESTestCase.randomFrom(INTERNAL_USERS);
+    }
+
     public static User userWithRandomMetadataAndDetails(final String username, final String... roles) {
         return new User(
             username,
@@ -200,7 +208,7 @@ private static AnonymousUser randomAnonymousUser() {
         );
     }
 
-    private static User stripRoles(User user) {
+    static User stripRoles(User user) {
         if (user.roles() != null || user.roles().length == 0) {
             return new User(user.principal(), Strings.EMPTY_ARRAY, user.fullName(), user.email(), user.metadata(), user.enabled());
         } else {
@@ -229,8 +237,48 @@ public static String randomInternalRoleName() {
         );
     }
 
+    public static RemoteAccessAuthentication randomRemoteAccessAuthentication(RoleDescriptorsIntersection roleDescriptorsIntersection) {
+        try {
+            // TODO add apikey() once we have querying-cluster-side API key support
+            final Authentication authentication = ESTestCase.randomFrom(
+                AuthenticationTestHelper.builder().realm(),
+                AuthenticationTestHelper.builder().internal(SystemUser.INSTANCE)
+            ).build();
+            return new RemoteAccessAuthentication(authentication, roleDescriptorsIntersection);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    public static RemoteAccessAuthentication randomRemoteAccessAuthentication() {
+        return randomRemoteAccessAuthentication(
+            new RoleDescriptorsIntersection(
+                List.of(
+                    // TODO randomize to add a second set once we have querying-cluster-side API key support
+                    Set.of(
+                        new RoleDescriptor(
+                            "_remote_user",
+                            null,
+                            new RoleDescriptor.IndicesPrivileges[] {
+                                RoleDescriptor.IndicesPrivileges.builder()
+                                    .indices("index1")
+                                    .privileges("read", "read_cross_cluster")
+                                    .build() },
+                            null,
+                            null,
+                            null,
+                            null,
+                            null,
+                            null
+                        )
+                    )
+                )
+            )
+        );
+    }
+
     public static class AuthenticationTestBuilder {
-        private Version version;
+        private TransportVersion transportVersion;
         private Authentication authenticatingAuthentication;
         private User user;
         private Authentication.RealmRef realmRef;
@@ -238,13 +286,14 @@ public static class AuthenticationTestBuilder {
         private final Map<String, Object> metadata = new HashMap<>();
         private Boolean isServiceAccount;
         private Boolean isRealmUnderDomain;
+        private RemoteAccessAuthentication remoteAccessAuthentication;
 
         private AuthenticationTestBuilder() {}
 
         private AuthenticationTestBuilder(Authentication authentication) {
             assert false == authentication.isRunAs() : "authenticating authentication cannot itself be run-as";
             this.authenticatingAuthentication = authentication;
-            this.version = authentication.getEffectiveSubject().getVersion();
+            this.transportVersion = authentication.getEffectiveSubject().getTransportVersion();
         }
 
         public AuthenticationTestBuilder realm() {
@@ -331,6 +380,22 @@ public AuthenticationTestBuilder user(User user) {
             }
         }
 
+        public AuthenticationTestBuilder remoteAccess() {
+            return remoteAccess(ESTestCase.randomAlphaOfLength(20), randomRemoteAccessAuthentication());
+        }
+
+        public AuthenticationTestBuilder remoteAccess(
+            final String remoteAccessApiKeyId,
+            final RemoteAccessAuthentication remoteAccessAuthentication
+        ) {
+            if (authenticatingAuthentication != null) {
+                throw new IllegalArgumentException("cannot use remote access authentication as run-as target");
+            }
+            apiKey(remoteAccessApiKeyId);
+            this.remoteAccessAuthentication = Objects.requireNonNull(remoteAccessAuthentication);
+            return this;
+        }
+
         public AuthenticationTestBuilder realmRef(Authentication.RealmRef realmRef) {
             assert false == SYNTHETIC_REALM_TYPES.contains(realmRef.getType()) : "use dedicate methods for synthetic realms";
             resetShortcutRelatedVariables();
@@ -342,11 +407,11 @@ public AuthenticationTestBuilder realmRef(Authentication.RealmRef realmRef) {
             return this;
         }
 
-        public AuthenticationTestBuilder version(Version version) {
+        public AuthenticationTestBuilder transportVersion(TransportVersion version) {
             if (authenticatingAuthentication != null) {
                 throw new IllegalArgumentException("cannot set version for run-as authentication");
             }
-            this.version = Objects.requireNonNull(version);
+            this.transportVersion = Objects.requireNonNull(version);
             return this;
         }
 
@@ -359,6 +424,9 @@ public AuthenticationTestBuilder metadata(Map<String, Object> metadata) {
         }
 
         public AuthenticationTestBuilder runAs() {
+            if (remoteAccessAuthentication != null) {
+                throw new IllegalArgumentException("cannot convert to run-as for remote access authentication");
+            }
             if (authenticatingAuthentication != null) {
                 throw new IllegalArgumentException("cannot convert to run-as again for run-as authentication");
             }
@@ -373,7 +441,12 @@ public Authentication build() {
             return build(ESTestCase.randomBoolean());
         }
 
-        public Authentication build(boolean runAsIfNotAlready) {
+        /**
+         * @param maybeRunAsIfNotAlready If the authentication is *not* run-as and the subject is a realm user, it will be transformed
+         *                               into a run-as authentication by moving the realm user to be the run-as user. The authenticating
+         *                               subject can be either a realm user or an API key (in general any subject type that can run-as).
+         */
+        public Authentication build(boolean maybeRunAsIfNotAlready) {
             if (authenticatingAuthentication != null) {
                 if (user == null) {
                     user = randomUser();
@@ -398,7 +471,7 @@ public Authentication build(boolean runAsIfNotAlready) {
                             realmRef = randomRealmRef(isRealmUnderDomain == null ? ESTestCase.randomBoolean() : isRealmUnderDomain);
                         }
                         assert false == SYNTHETIC_REALM_TYPES.contains(realmRef.getType()) : "use dedicate methods for synthetic realms";
-                        if (runAsIfNotAlready) {
+                        if (maybeRunAsIfNotAlready) {
                             authentication = builder().runAs().user(user).realmRef(realmRef).build();
                         } else {
                             authentication = Authentication.newRealmAuthentication(user, realmRef);
@@ -412,10 +485,16 @@ public Authentication build(boolean runAsIfNotAlready) {
                         // User associated to API key authentication has empty roles
                         user = stripRoles(user);
                         prepareApiKeyMetadata();
-                        authentication = Authentication.newApiKeyAuthentication(
+                        final Authentication apiKeyAuthentication = Authentication.newApiKeyAuthentication(
                             AuthenticationResult.success(user, metadata),
                             ESTestCase.randomAlphaOfLengthBetween(3, 8)
                         );
+                        // Remote access is authenticated via API key, but the underlying authentication instance has a different structure,
+                        // and a different subject type. If remoteAccessAuthentication is set, we transform the API key authentication
+                        // instance into a remote access authentication instance.
+                        authentication = remoteAccessAuthentication != null
+                            ? apiKeyAuthentication.toRemoteAccess(remoteAccessAuthentication)
+                            : apiKeyAuthentication;
                     }
                     case TOKEN -> {
                         if (isServiceAccount != null && isServiceAccount) {
@@ -490,20 +569,20 @@ public Authentication build(boolean runAsIfNotAlready) {
                         String nodeName = ESTestCase.randomAlphaOfLengthBetween(3, 8);
                         if (user == SystemUser.INSTANCE) {
                             authentication = ESTestCase.randomFrom(
-                                Authentication.newInternalAuthentication(user, Version.CURRENT, nodeName),
+                                Authentication.newInternalAuthentication(user, TransportVersion.CURRENT, nodeName),
                                 Authentication.newInternalFallbackAuthentication(user, nodeName)
                             );
                         } else {
-                            authentication = Authentication.newInternalAuthentication(user, Version.CURRENT, nodeName);
+                            authentication = Authentication.newInternalAuthentication(user, TransportVersion.CURRENT, nodeName);
                         }
                     }
                     default -> throw new IllegalArgumentException("unknown authentication type [" + authenticationType + "]");
                 }
-                if (version == null) {
-                    version = Version.CURRENT;
+                if (transportVersion == null) {
+                    transportVersion = TransportVersion.CURRENT;
                 }
-                if (version.before(authentication.getEffectiveSubject().getVersion())) {
-                    return authentication.maybeRewriteForOlderVersion(version);
+                if (transportVersion.before(authentication.getEffectiveSubject().getTransportVersion())) {
+                    return authentication.maybeRewriteForOlderVersion(transportVersion);
                 } else {
                     return authentication;
                 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java
index 8f8f82fc84d0..cb2dde215c7b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTests.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.core.security.authc;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -14,6 +15,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -22,6 +24,8 @@
 import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountSettings;
+import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.security.user.User;
 
@@ -32,36 +36,26 @@
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.sameInstance;
 
 public class AuthenticationTests extends ESTestCase {
 
-    public void testWillGetLookedUpByWhenItExists() {
-        final RealmRef authenticatedBy = new RealmRef("auth_by", "auth_by_type", "node");
-        final RealmRef lookedUpBy = new RealmRef("lookup_by", "lookup_by_type", "node");
-        final Authentication authentication = AuthenticationTestHelper.builder()
-            .user(new User("not-user"))
-            .realmRef(authenticatedBy)
-            .runAs()
-            .user(new User("user"))
-            .realmRef(lookedUpBy)
-            .build();
-
-        assertEquals(lookedUpBy, authentication.getSourceRealm());
-    }
-
-    public void testWillGetAuthenticateByWhenLookupIsNull() {
-        final RealmRef authenticatedBy = new RealmRef("auth_by", "auth_by_type", "node");
-        final Authentication authentication = AuthenticationTestHelper.builder()
-            .user(new User("user"))
-            .realmRef(authenticatedBy)
-            .build(false);
+    public void testIsFailedRunAs() {
+        final Authentication failedAuthentication = randomRealmAuthentication(randomBoolean()).runAs(randomUser(), null);
+        assertTrue(failedAuthentication.isRunAs());
+        assertTrue(failedAuthentication.isFailedRunAs());
 
-        assertEquals(authenticatedBy, authentication.getSourceRealm());
+        final Authentication authentication = AuthenticationTestHelper.builder().realm().runAs().build();
+        assertTrue(authentication.isRunAs());
+        assertFalse(authentication.isFailedRunAs());
     }
 
     public void testCanAccessResourcesOf() {
@@ -136,6 +130,14 @@ public void testCanAccessResourcesOf() {
             randomApiKeyAuthentication(user1, apiKeyId1).runAs(user3, realm2),
             randomApiKeyAuthentication(user2, apiKeyId2).runAs(user3, realm2)
         );
+
+        // No resource sharing for remote access authentication for now
+        final RemoteAccessAuthentication randomRemoteAccessAuthentication = AuthenticationTestHelper.randomRemoteAccessAuthentication();
+        final Authentication remoteAccessAuthentication = AuthenticationTestHelper.builder()
+            .remoteAccess(apiKeyId1, randomRemoteAccessAuthentication)
+            .build(false);
+        assertCannotAccessResources(remoteAccessAuthentication, remoteAccessAuthentication);
+        assertCannotAccessResources(remoteAccessAuthentication, randomRemoteAccessAuthentication.getAuthentication());
     }
 
     public void testTokenAccessResourceOf() {
@@ -238,6 +240,28 @@ public void testIsServiceAccount() {
         }
     }
 
+    public void testIsRemoteAccess() {
+        final boolean isRemoteAccess = randomBoolean();
+        final Authentication authentication;
+        if (isRemoteAccess) {
+            authentication = AuthenticationTestHelper.builder().remoteAccess().build();
+        } else {
+            authentication = randomValueOtherThanMany(
+                authc -> AuthenticationField.REMOTE_ACCESS_REALM_TYPE.equals(authc.getAuthenticatingSubject().getRealm().getType()),
+                () -> AuthenticationTestHelper.builder().build()
+            );
+        }
+
+        if (isRemoteAccess) {
+            assertThat(authentication.isRemoteAccess(), is(true));
+            // Also validate that this does not clash with API keys
+            assertThat(authentication.isApiKey(), is(false));
+            assertThat(authentication.isAuthenticatedAsApiKey(), is(false));
+        } else {
+            assertThat(authentication.isRemoteAccess(), is(false));
+        }
+    }
+
     public void testNonRealmAuthenticationsNoDomain() {
         final String apiKeyId = randomAlphaOfLengthBetween(10, 20);
         Authentication apiAuthentication = randomApiKeyAuthentication(randomUser(), apiKeyId);
@@ -255,6 +279,9 @@ public void testNonRealmAuthenticationsNoDomain() {
         Authentication internalAuthentication = randomInternalAuthentication();
         assertThat(internalAuthentication.isAssignedToDomain(), is(false));
         assertThat(internalAuthentication.getDomain(), nullValue());
+        Authentication remoteAccessAuthentication = AuthenticationTestHelper.builder().remoteAccess().build();
+        assertThat(remoteAccessAuthentication.isAssignedToDomain(), is(false));
+        assertThat(remoteAccessAuthentication.getDomain(), nullValue());
     }
 
     public void testRealmAuthenticationIsAssignedToDomain() {
@@ -301,7 +328,7 @@ public void testRunAsAuthenticationWithDomain() {
         assertThat(test.isAssignedToDomain(), is(true));
         assertThat(test.getDomain(), is(lookupRealmRef.getDomain()));
         // api key run-as
-        test = randomApiKeyAuthentication(randomUser(), randomAlphaOfLengthBetween(10, 20), Version.CURRENT);
+        test = randomApiKeyAuthentication(randomUser(), randomAlphaOfLengthBetween(10, 20), TransportVersion.CURRENT);
         assertThat(test.isAssignedToDomain(), is(false));
         assertThat(test.getDomain(), nullValue());
         if (randomBoolean()) {
@@ -409,16 +436,54 @@ public void testDomainSerialize() throws Exception {
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
-            out.setVersion(Version.V_8_0_0);
+            out.setTransportVersion(TransportVersion.V_8_0_0);
             test.writeTo(out);
             StreamInput in = out.bytes().streamInput();
-            in.setVersion(Version.V_8_0_0);
+            in.setTransportVersion(TransportVersion.V_8_0_0);
             Authentication testBack = new Authentication(in);
             assertThat(testBack.getDomain(), nullValue());
             assertThat(testBack.isAssignedToDomain(), is(false));
         }
     }
 
+    public void testRemoteAccessAuthentication() throws IOException {
+        final String remoteAccessApiKeyId = ESTestCase.randomAlphaOfLength(20);
+        final RemoteAccessAuthentication remoteAccessAuthentication = AuthenticationTestHelper.randomRemoteAccessAuthentication();
+        final Authentication authentication = AuthenticationTestHelper.builder()
+            .remoteAccess(remoteAccessApiKeyId, remoteAccessAuthentication)
+            .build(false);
+
+        assertThat(authentication.getAuthenticationType(), equalTo(Authentication.AuthenticationType.API_KEY));
+        assertThat(authentication.getEffectiveSubject(), sameInstance(authentication.getAuthenticatingSubject()));
+        assertThat(
+            authentication.getEffectiveSubject().getUser(),
+            equalTo(AuthenticationTestHelper.stripRoles(authentication.getEffectiveSubject().getUser()))
+        );
+        assertThat(authentication.isRemoteAccess(), is(true));
+        assertThat(
+            authentication.getAuthenticatingSubject().getRealm(),
+            equalTo(
+                new RealmRef(
+                    AuthenticationField.REMOTE_ACCESS_REALM_NAME,
+                    AuthenticationField.REMOTE_ACCESS_REALM_TYPE,
+                    authentication.getAuthenticatingSubject().getRealm().getNodeName()
+                )
+            )
+        );
+        assertThat(
+            authentication.getAuthenticatingSubject().getMetadata(),
+            hasEntry(AuthenticationField.API_KEY_ID_KEY, remoteAccessApiKeyId)
+        );
+        assertThat(
+            authentication.getAuthenticatingSubject().getMetadata(),
+            hasEntry(AuthenticationField.REMOTE_ACCESS_AUTHENTICATION_KEY, remoteAccessAuthentication.getAuthentication().encode())
+        );
+        assertThat(
+            authentication.getAuthenticatingSubject().getMetadata(),
+            hasEntry(AuthenticationField.REMOTE_ACCESS_ROLE_DESCRIPTORS_KEY, remoteAccessAuthentication.getRoleDescriptorsBytesList())
+        );
+    }
+
     public void testSupportsRunAs() {
         final Settings.Builder settingsBuilder = Settings.builder();
         if (randomBoolean()) {
@@ -449,6 +514,9 @@ public void testSupportsRunAs() {
         // Neither anonymous user nor its token can run-as
         assertThat(AuthenticationTestHelper.builder().anonymous(anonymousUser).build().supportsRunAs(anonymousUser), is(false));
         assertThat(AuthenticationTestHelper.builder().anonymous(anonymousUser).build().token().supportsRunAs(anonymousUser), is(false));
+
+        // Remote access cannot run-as
+        assertThat(AuthenticationTestHelper.builder().remoteAccess().build().supportsRunAs(anonymousUser), is(false));
     }
 
     private void assertCanAccessResources(Authentication authentication0, Authentication authentication1) {
@@ -474,6 +542,23 @@ public void testToXContentWithApiKey() throws IOException {
         runWithAuthenticationToXContent(authentication2, m -> assertThat(m, not(hasKey("api_key"))));
     }
 
+    public void testToXContentWithRemoteAccess() throws IOException {
+        final String apiKeyId = randomAlphaOfLength(20);
+        final Authentication authentication = AuthenticationTestHelper.builder()
+            .remoteAccess(apiKeyId, AuthenticationTestHelper.randomRemoteAccessAuthentication())
+            .build(false);
+        final String apiKeyName = (String) authentication.getAuthenticatingSubject()
+            .getMetadata()
+            .get(AuthenticationField.API_KEY_NAME_KEY);
+        runWithAuthenticationToXContent(
+            authentication,
+            m -> assertThat(
+                m,
+                hasEntry("api_key", apiKeyName != null ? Map.of("id", apiKeyId, "name", apiKeyName) : Map.of("id", apiKeyId))
+            )
+        );
+    }
+
     public void testToXContentWithServiceAccount() throws IOException {
         final Authentication authentication1 = randomServiceAccountAuthentication();
         final String tokenName = (String) authentication1.getAuthenticatingSubject()
@@ -488,6 +573,190 @@ public void testToXContentWithServiceAccount() throws IOException {
         );
     }
 
+    public void testBwcWithStoredAuthenticationHeaders() throws IOException {
+        // Version 6.6.1
+        final String headerV6 = "p/HxAgANZWxhc3RpYy1hZG1pbgEJc3VwZXJ1c2VyCgAAAAEABG5vZGUFZmlsZTEEZmlsZQA=";
+        final Authentication authenticationV6 = AuthenticationContextSerializer.decode(headerV6);
+        assertThat(authenticationV6.getEffectiveSubject().getTransportVersion(), equalTo(TransportVersion.fromId(6_06_01_99)));
+        assertThat(authenticationV6.getEffectiveSubject().getUser(), equalTo(new User("elastic-admin", "superuser")));
+        assertThat(authenticationV6.getAuthenticationType(), equalTo(Authentication.AuthenticationType.REALM));
+        assertThat(authenticationV6.isRunAs(), is(false));
+        assertThat(authenticationV6.encode(), equalTo(headerV6));
+
+        // Rewrite for a different version
+        final Version nodeVersion = VersionUtils.randomIndexCompatibleVersion(random());// TODO are we going to use transport version for
+                                                                                        // index compatibility?
+        final Authentication rewrittenAuthentication = authenticationV6.maybeRewriteForOlderVersion(nodeVersion.transportVersion);
+        assertThat(rewrittenAuthentication.getEffectiveSubject().getTransportVersion(), equalTo(nodeVersion.transportVersion));
+        assertThat(rewrittenAuthentication.getEffectiveSubject().getUser(), equalTo(authenticationV6.getEffectiveSubject().getUser()));
+        assertThat(rewrittenAuthentication.getAuthenticationType(), equalTo(Authentication.AuthenticationType.REALM));
+        assertThat(rewrittenAuthentication.isRunAs(), is(false));
+
+        // Version 7.2.1
+        final String headerV7 = "p72sAwANZWxhc3RpYy1hZG1pbgENX2VzX3Rlc3Rfcm9vdAoAAAABAARub2RlBWZpbGUxBGZpbGUAAAoA";
+        final Authentication authenticationV7 = AuthenticationContextSerializer.decode(headerV7);
+        assertThat(authenticationV7.getEffectiveSubject().getTransportVersion(), equalTo(TransportVersion.fromId(7_02_01_99)));
+        assertThat(authenticationV7.getEffectiveSubject().getUser(), equalTo(new User("elastic-admin", "_es_test_root")));
+        assertThat(authenticationV7.getAuthenticationType(), equalTo(Authentication.AuthenticationType.REALM));
+        assertThat(authenticationV7.isRunAs(), is(false));
+        assertThat(authenticationV7.encode(), equalTo(headerV7));
+    }
+
+    public void testMaybeRewriteForOlderVersionWithRemoteAccessThrowsOnUnsupportedVersion() {
+        final Authentication authentication = randomBoolean()
+            ? AuthenticationTestHelper.builder().remoteAccess().build()
+            : AuthenticationTestHelper.builder().build();
+
+        final Version versionBeforeRemoteAccessRealm = VersionUtils.getPreviousVersion(Version.V_8_7_0);
+        final Version version = VersionUtils.randomVersionBetween(
+            random(),
+            versionBeforeRemoteAccessRealm.minimumCompatibilityVersion(),
+            versionBeforeRemoteAccessRealm
+        );
+
+        if (authentication.isRemoteAccess()) {
+            final var ex = expectThrows(
+                IllegalArgumentException.class,
+                () -> authentication.maybeRewriteForOlderVersion(version.transportVersion)
+            );
+            assertThat(
+                ex.getMessage(),
+                containsString(
+                    "versions of Elasticsearch before ["
+                        + Authentication.VERSION_REMOTE_ACCESS_REALM
+                        + "] can't handle remote access authentication and attempted to rewrite for ["
+                        + version.transportVersion
+                        + "]"
+                )
+            );
+        } else {
+            // Assert that rewriting took place; the details of rewriting logic are checked in other tests
+            assertThat(authentication.maybeRewriteForOlderVersion(version.transportVersion), not(equalTo(authentication)));
+        }
+    }
+
+    public void testMaybeRewriteForOlderVersionWithRemoteAccessRewritesAuthenticationInMetadata() throws IOException {
+        // This is a hack for 8.7, to enable testing a scenario where a remote access authentication is rewritten for a node on an older
+        // version. There are two factors why we need this hack:
+        // We introduced a new rewrite mechanism for remote access authentication in 8.7, to stay future-proof.
+        // Rewriting a remote access authentication is only valid if it's towards a node on version 8.7 or later (it's prevented by an
+        // internal validation check).
+        // To test the new rewrite mechanism in 8.7, we call authentication.maybeRewriteForOlderVersion(version, remoteAccessRealmVersion)
+        // with a version that precedes 8.7, thus circumventing the internal validation error
+        // TODO in 8.8, simply set remoteAccessRealmVersion to Authentication.VERSION_REMOTE_ACCESS_REALM
+        final TransportVersion remoteAccessRealmVersion = TransportVersion.CURRENT.equals(Authentication.VERSION_REMOTE_ACCESS_REALM)
+            ? TransportVersionUtils.getPreviousVersion(Authentication.VERSION_REMOTE_ACCESS_REALM)
+            : Authentication.VERSION_REMOTE_ACCESS_REALM;
+        final TransportVersion version = TransportVersionUtils.randomVersionBetween(
+            random(),
+            remoteAccessRealmVersion,
+            TransportVersion.CURRENT
+        );
+        final Authentication innerAuthentication = AuthenticationTestHelper.builder().transportVersion(version).build();
+        final Authentication authentication = AuthenticationTestHelper.builder()
+            .remoteAccess(randomAlphaOfLength(20), new RemoteAccessAuthentication(innerAuthentication, RoleDescriptorsIntersection.EMPTY))
+            .build();
+        final TransportVersion maybeOldVersion = TransportVersionUtils.randomVersionBetween(random(), remoteAccessRealmVersion, version);
+
+        final Authentication actual = authentication.maybeRewriteForOlderVersion(maybeOldVersion, remoteAccessRealmVersion);
+
+        final Authentication innerActualAuthentication = AuthenticationContextSerializer.decode(
+            (String) actual.getAuthenticatingSubject().getMetadata().get(AuthenticationField.REMOTE_ACCESS_AUTHENTICATION_KEY)
+        );
+        assertThat(innerActualAuthentication, equalTo(innerAuthentication.maybeRewriteForOlderVersion(maybeOldVersion)));
+    }
+
+    public void testMaybeRewriteMetadataForRemoteAccessAuthentication() throws IOException {
+        final Authentication innerAuthentication = AuthenticationTestHelper.builder().build();
+        final Authentication authentication = AuthenticationTestHelper.builder()
+            .remoteAccess(randomAlphaOfLength(20), new RemoteAccessAuthentication(innerAuthentication, RoleDescriptorsIntersection.EMPTY))
+            .build();
+        // pick a version before that of the authentication instance to force a rewrite
+        final TransportVersion olderVersion = TransportVersionUtils.randomPreviousCompatibleVersion(
+            random(),
+            authentication.getEffectiveSubject().getTransportVersion()
+        );
+
+        final Map<String, Object> rewrittenMetadata = Authentication.maybeRewriteMetadataForRemoteAccessAuthentication(
+            olderVersion,
+            authentication
+        );
+
+        final Authentication innerRewritten = AuthenticationContextSerializer.decode(
+            (String) rewrittenMetadata.get(AuthenticationField.REMOTE_ACCESS_AUTHENTICATION_KEY)
+        );
+        assertThat(innerRewritten, equalTo(innerAuthentication.maybeRewriteForOlderVersion(olderVersion)));
+    }
+
+    public void testMaybeRewriteForOlderVersionErasesDomainForVersionsBeforeDomains() {
+        final TransportVersion olderVersion = TransportVersionUtils.randomPreviousCompatibleVersion(
+            random(),
+            Authentication.VERSION_REALM_DOMAINS
+        );
+        final Authentication authentication = AuthenticationTestHelper.builder()
+            .realm() // randomize to test both when realm is null on the original auth and non-null, instead of setting `underDomain`
+            .transportVersion(TransportVersionUtils.randomVersionBetween(random(), Authentication.VERSION_REALM_DOMAINS, null))
+            .build();
+        assertThat(authentication.getEffectiveSubject().getTransportVersion().after(olderVersion), is(true));
+
+        final Authentication actual = authentication.maybeRewriteForOlderVersion(olderVersion);
+
+        assertThat(actual.getEffectiveSubject().getTransportVersion(), equalTo(olderVersion));
+        assertThat(actual.getAuthenticatingSubject().getRealm().getDomain(), nullValue());
+        assertThat(actual.getEffectiveSubject().getRealm().getDomain(), nullValue());
+    }
+
+    public void testMaybeRewriteForOlderVersionDoesNotEraseDomainForVersionsAfterDomains() {
+        final TransportVersion olderVersion = TransportVersionUtils.randomVersionBetween(
+            random(),
+            Authentication.VERSION_REALM_DOMAINS,
+            // Don't include CURRENT, so we always have at least one newer version available below
+            TransportVersionUtils.getPreviousVersion()
+        );
+        TransportVersion transportVersion = TransportVersionUtils.randomVersionBetween(random(), olderVersion, null);
+        final Authentication authentication = AuthenticationTestHelper.builder()
+            .realm() // randomize to test both when realm is null on the original auth and non-null, instead of setting `underDomain`
+            // Use CURRENT to force newer version in case randomVersionBetween above picks olderVersion
+            .transportVersion(transportVersion.equals(olderVersion) ? TransportVersion.CURRENT : transportVersion)
+            .build();
+
+        final Authentication actual = authentication.maybeRewriteForOlderVersion(olderVersion);
+
+        assertThat(
+            actual.getAuthenticatingSubject().getRealm().getDomain(),
+            equalTo(authentication.getAuthenticatingSubject().getRealm().getDomain())
+        );
+        assertThat(
+            actual.getEffectiveSubject().getRealm().getDomain(),
+            equalTo(authentication.getEffectiveSubject().getRealm().getDomain())
+        );
+    }
+
+    public void testMaybeRewriteRealmRef() {
+        final RealmRef realmRefWithDomain = AuthenticationTests.randomRealmRef(true);
+        assertThat(realmRefWithDomain.getDomain(), notNullValue());
+
+        assertThat(
+            Authentication.maybeRewriteRealmRef(
+                TransportVersionUtils.randomVersionBetween(
+                    random(),
+                    null,
+                    TransportVersionUtils.getPreviousVersion(Authentication.VERSION_REALM_DOMAINS)
+                ),
+                realmRefWithDomain
+            ).getDomain(),
+            nullValue()
+        );
+
+        assertThat(
+            Authentication.maybeRewriteRealmRef(
+                TransportVersionUtils.randomVersionBetween(random(), Authentication.VERSION_REALM_DOMAINS, null),
+                realmRefWithDomain
+            ),
+            equalTo(realmRefWithDomain)
+        );
+    }
+
     private void runWithAuthenticationToXContent(Authentication authentication, Consumer<Map<String, Object>> consumer) throws IOException {
         try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
             authentication.toXContent(builder, ToXContent.EMPTY_PARAMS);
@@ -526,7 +795,7 @@ public static RealmRef randomRealmRef(boolean underDomain, boolean includeIntern
 
     /**
      * Randomly create an authentication that has either realm or token authentication type.
-     * The authentication can have any version from 7.0.0 to current and random metadata.
+     * The authentication can have any version from 7.0.0 (or from 8.2.0, if the realm has a domain) to current and random metadata.
      */
     public static Authentication randomAuthentication(User user, RealmRef realmRef) {
         return randomAuthentication(user, realmRef, randomBoolean());
@@ -539,7 +808,9 @@ public static Authentication randomAuthentication(User user, RealmRef realmRef,
         if (realmRef == null) {
             realmRef = randomRealmRef(false);
         }
-        final Version version = VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.CURRENT);
+        // If the realm is expected to have a domain, we need a version that's at least compatible with domains
+        final TransportVersion minVersion = realmRef.getDomain() != null ? Authentication.VERSION_REALM_DOMAINS : TransportVersion.V_7_0_0;
+        final TransportVersion version = TransportVersionUtils.randomVersionBetween(random(), minVersion, TransportVersion.CURRENT);
         final Map<String, Object> metadata;
         if (randomBoolean()) {
             metadata = Map.of(randomAlphaOfLengthBetween(3, 8), randomAlphaOfLengthBetween(3, 8));
@@ -548,14 +819,18 @@ public static Authentication randomAuthentication(User user, RealmRef realmRef,
                 .distinct()
                 .collect(Collectors.toMap(s -> s, s -> randomAlphaOfLengthBetween(3, 8)));
         }
-        return AuthenticationTestHelper.builder().user(user).realmRef(realmRef).version(version).metadata(metadata).build(isRunAs);
+        return AuthenticationTestHelper.builder().user(user).realmRef(realmRef).transportVersion(version).metadata(metadata).build(isRunAs);
     }
 
     public static Authentication randomApiKeyAuthentication(User user, String apiKeyId) {
-        return randomApiKeyAuthentication(user, apiKeyId, VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.CURRENT));
+        return randomApiKeyAuthentication(
+            user,
+            apiKeyId,
+            TransportVersionUtils.randomVersionBetween(random(), TransportVersion.V_7_0_0, TransportVersion.CURRENT)
+        );
     }
 
-    public static Authentication randomApiKeyAuthentication(User user, String apiKeyId, Version version) {
+    public static Authentication randomApiKeyAuthentication(User user, String apiKeyId, TransportVersion version) {
         return randomApiKeyAuthentication(
             user,
             apiKeyId,
@@ -570,12 +845,12 @@ public static Authentication randomApiKeyAuthentication(
         String apiKeyId,
         String creatorRealmName,
         String creatorRealmType,
-        Version version
+        TransportVersion version
     ) {
         return AuthenticationTestHelper.builder()
             .apiKey(apiKeyId)
             .user(user)
-            .version(version)
+            .transportVersion(version)
             .metadata(
                 Map.of(
                     AuthenticationField.API_KEY_CREATOR_REALM_NAME,
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/RealmDomainTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/RealmDomainTests.java
index a152629325aa..dc420b33cf63 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/RealmDomainTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/RealmDomainTests.java
@@ -30,4 +30,9 @@ protected RealmDomain createTestInstance() {
         return AuthenticationTestHelper.randomDomain(randomBoolean());
     }
 
+    @Override
+    protected RealmDomain mutateInstance(RealmDomain instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/RemoteAccessAuthenticationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/RemoteAccessAuthenticationTests.java
new file mode 100644
index 000000000000..90615a9109ff
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/RemoteAccessAuthenticationTests.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.authc;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTests.randomUniquelyNamedRoleDescriptors;
+import static org.hamcrest.Matchers.equalTo;
+
+public class RemoteAccessAuthenticationTests extends ESTestCase {
+
+    public void testWriteReadContextRoundtrip() throws IOException {
+        final ThreadContext ctx = new ThreadContext(Settings.EMPTY);
+        final RoleDescriptorsIntersection expectedRoleDescriptorsIntersection = randomRoleDescriptorsIntersection();
+        final var expectedRemoteAccessAuthentication = new RemoteAccessAuthentication(
+            AuthenticationTestHelper.builder().build(),
+            expectedRoleDescriptorsIntersection
+        );
+
+        expectedRemoteAccessAuthentication.writeToContext(ctx);
+        final RemoteAccessAuthentication actual = RemoteAccessAuthentication.readFromContext(ctx);
+
+        assertThat(actual.getAuthentication(), equalTo(expectedRemoteAccessAuthentication.getAuthentication()));
+        final List<Set<RoleDescriptor>> roleDescriptorsList = new ArrayList<>();
+        for (BytesReference bytesReference : actual.getRoleDescriptorsBytesList()) {
+            Set<RoleDescriptor> roleDescriptors = new RemoteAccessAuthentication.RoleDescriptorsBytes(bytesReference).toRoleDescriptors();
+            roleDescriptorsList.add(roleDescriptors);
+        }
+        final var actualRoleDescriptorsIntersection = new RoleDescriptorsIntersection(roleDescriptorsList);
+        assertThat(actualRoleDescriptorsIntersection, equalTo(expectedRoleDescriptorsIntersection));
+    }
+
+    public void testRoleDescriptorsBytesToRoleDescriptors() throws IOException {
+        final Set<RoleDescriptor> expectedRoleDescriptors = Set.copyOf(randomUniquelyNamedRoleDescriptors(0, 3));
+        final XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.map(expectedRoleDescriptors.stream().collect(Collectors.toMap(RoleDescriptor::getName, Function.identity())));
+        final Set<RoleDescriptor> actualRoleDescriptors = new RemoteAccessAuthentication.RoleDescriptorsBytes(BytesReference.bytes(builder))
+            .toRoleDescriptors();
+        assertThat(actualRoleDescriptors, equalTo(expectedRoleDescriptors));
+    }
+
+    private RoleDescriptorsIntersection randomRoleDescriptorsIntersection() {
+        return new RoleDescriptorsIntersection(randomList(0, 3, () -> Set.copyOf(randomUniquelyNamedRoleDescriptors(0, 1))));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/SubjectTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/SubjectTests.java
index 3ebc0b2a22e6..7f575f29457b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/SubjectTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/SubjectTests.java
@@ -7,15 +7,16 @@
 
 package org.elasticsearch.xpack.core.security.authc;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.ArrayUtils;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountSettings;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection;
 import org.elasticsearch.xpack.core.security.authz.store.RoleReference;
 import org.elasticsearch.xpack.core.security.authz.store.RoleReference.ApiKeyRoleReference;
 import org.elasticsearch.xpack.core.security.authz.store.RoleReference.BwcApiKeyRoleReference;
@@ -35,7 +36,10 @@
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.API_KEY_REALM_NAME;
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.API_KEY_REALM_TYPE;
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY;
+import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.REMOTE_ACCESS_REALM_NAME;
+import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.REMOTE_ACCESS_REALM_TYPE;
 import static org.elasticsearch.xpack.core.security.authc.Subject.FLEET_SERVER_ROLE_DESCRIPTOR_BYTES_V_7_14;
+import static org.elasticsearch.xpack.core.security.authz.store.RoleReference.RemoteAccessRoleReference;
 import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
@@ -50,7 +54,7 @@ public void testGetRoleReferencesForRegularUser() {
         final Subject subject = new Subject(
             user,
             new Authentication.RealmRef(randomAlphaOfLength(5), randomAlphaOfLength(5), "node"),
-            Version.CURRENT,
+            TransportVersion.CURRENT,
             Map.of()
         );
 
@@ -73,7 +77,7 @@ public void testGetRoleReferencesForAnonymousUser() {
         final Subject subject = new Subject(
             anonymousUser,
             new Authentication.RealmRef(randomAlphaOfLength(5), randomAlphaOfLength(5), "node"),
-            Version.CURRENT,
+            TransportVersion.CURRENT,
             Map.of()
         );
 
@@ -91,7 +95,7 @@ public void testGetRoleReferencesForServiceAccount() {
         final Subject subject = new Subject(
             serviceUser,
             new Authentication.RealmRef(ServiceAccountSettings.REALM_NAME, ServiceAccountSettings.REALM_TYPE, "node"),
-            Version.CURRENT,
+            TransportVersion.CURRENT,
             Map.of()
         );
 
@@ -122,7 +126,7 @@ public void testGetRoleReferencesForApiKey() {
         final Subject subject = new Subject(
             new User("joe"),
             new Authentication.RealmRef(API_KEY_REALM_NAME, API_KEY_REALM_TYPE, "node"),
-            Version.CURRENT,
+            TransportVersion.CURRENT,
             authMetadata
         );
 
@@ -145,6 +149,81 @@ public void testGetRoleReferencesForApiKey() {
         }
     }
 
+    public void testGetRoleReferencesForRemoteAccess() {
+        Map<String, Object> authMetadata = new HashMap<>();
+        final String apiKeyId = randomAlphaOfLength(12);
+        authMetadata.put(AuthenticationField.API_KEY_ID_KEY, apiKeyId);
+        authMetadata.put(AuthenticationField.API_KEY_NAME_KEY, randomBoolean() ? null : randomAlphaOfLength(12));
+        final BytesReference roleBytes = new BytesArray("""
+            {"role":{"indices":[{"names":["index*"],"privileges":["read"]}]}}""");
+        final BytesReference limitedByRoleBytes = new BytesArray("""
+            {"limited-by-role":{"indices":[{"names":["*"],"privileges":["all"]}]}}""");
+
+        final boolean emptyRoleBytes = randomBoolean();
+
+        authMetadata.put(
+            AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY,
+            emptyRoleBytes ? randomFrom(Arrays.asList(null, new BytesArray("{}"))) : roleBytes
+        );
+        authMetadata.put(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, limitedByRoleBytes);
+
+        final RemoteAccessAuthentication remoteAccessAuthentication = randomBoolean()
+            ? AuthenticationTestHelper.randomRemoteAccessAuthentication(RoleDescriptorsIntersection.EMPTY)
+            : AuthenticationTestHelper.randomRemoteAccessAuthentication();
+        authMetadata = remoteAccessAuthentication.copyWithRemoteAccessEntries(authMetadata);
+
+        final Subject subject = new Subject(
+            new User("joe"),
+            new Authentication.RealmRef(REMOTE_ACCESS_REALM_NAME, REMOTE_ACCESS_REALM_TYPE, "node"),
+            TransportVersion.CURRENT,
+            authMetadata
+        );
+
+        final RoleReferenceIntersection roleReferenceIntersection = subject.getRoleReferenceIntersection(getAnonymousUser());
+        final List<RoleReference> roleReferences = roleReferenceIntersection.getRoleReferences();
+        if (emptyRoleBytes) {
+            assertThat(roleReferences, contains(isA(RemoteAccessRoleReference.class), isA(ApiKeyRoleReference.class)));
+
+            final RemoteAccessRoleReference remoteAccessRoleReference = (RemoteAccessRoleReference) roleReferences.get(0);
+            assertThat(
+                remoteAccessRoleReference.getRoleDescriptorsBytes(),
+                equalTo(
+                    remoteAccessAuthentication.getRoleDescriptorsBytesList().isEmpty()
+                        ? RemoteAccessAuthentication.RoleDescriptorsBytes.EMPTY
+                        : remoteAccessAuthentication.getRoleDescriptorsBytesList().get(0)
+                )
+            );
+
+            final ApiKeyRoleReference roleReference = (ApiKeyRoleReference) roleReferences.get(1);
+            assertThat(roleReference.getApiKeyId(), equalTo(apiKeyId));
+            assertThat(roleReference.getRoleDescriptorsBytes(), equalTo(authMetadata.get(API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY)));
+
+        } else {
+            assertThat(
+                roleReferences,
+                contains(isA(RemoteAccessRoleReference.class), isA(ApiKeyRoleReference.class), isA(ApiKeyRoleReference.class))
+            );
+
+            final RemoteAccessRoleReference remoteAccessRoleReference = (RemoteAccessRoleReference) roleReferences.get(0);
+            assertThat(
+                remoteAccessRoleReference.getRoleDescriptorsBytes(),
+                equalTo(
+                    remoteAccessAuthentication.getRoleDescriptorsBytesList().isEmpty()
+                        ? RemoteAccessAuthentication.RoleDescriptorsBytes.EMPTY
+                        : remoteAccessAuthentication.getRoleDescriptorsBytesList().get(0)
+                )
+            );
+
+            final ApiKeyRoleReference roleReference = (ApiKeyRoleReference) roleReferences.get(1);
+            assertThat(roleReference.getApiKeyId(), equalTo(apiKeyId));
+            assertThat(roleReference.getRoleDescriptorsBytes(), equalTo(authMetadata.get(API_KEY_ROLE_DESCRIPTORS_KEY)));
+
+            final ApiKeyRoleReference limitedByRoleReference = (ApiKeyRoleReference) roleReferences.get(2);
+            assertThat(limitedByRoleReference.getApiKeyId(), equalTo(apiKeyId));
+            assertThat(limitedByRoleReference.getRoleDescriptorsBytes(), equalTo(authMetadata.get(API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY)));
+        }
+    }
+
     public void testGetRoleReferencesForApiKeyBwc() {
         Map<String, Object> authMetadata = new HashMap<>();
         final String apiKeyId = randomAlphaOfLength(12);
@@ -165,7 +244,7 @@ public void testGetRoleReferencesForApiKeyBwc() {
         final Subject subject = new Subject(
             new User("joe"),
             new Authentication.RealmRef(API_KEY_REALM_NAME, API_KEY_REALM_TYPE, "node"),
-            VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.V_7_8_1),
+            TransportVersionUtils.randomVersionBetween(random(), TransportVersion.V_7_0_0, TransportVersion.V_7_8_1),
             authMetadata
         );
 
@@ -195,7 +274,7 @@ public void testGetFleetApiKeyRoleReferenceBwcBugFix() {
         final Subject subject = new Subject(
             new User("elastic/fleet-server"),
             new Authentication.RealmRef(API_KEY_REALM_NAME, API_KEY_REALM_TYPE, "node"),
-            Version.CURRENT,
+            TransportVersion.CURRENT,
             Map.of(
                 AuthenticationField.API_KEY_CREATOR_REALM_NAME,
                 ServiceAccountSettings.REALM_NAME,
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/DistinguishedNameNormalizerTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/DistinguishedNameNormalizerTests.java
new file mode 100644
index 000000000000..f5fb9c96a6e8
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/DistinguishedNameNormalizerTests.java
@@ -0,0 +1,170 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.authc.support;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.authc.RealmConfig;
+import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.ExpressionModel;
+import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression.FieldValue;
+import org.junit.Before;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.function.Predicate;
+import java.util.stream.IntStream;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class DistinguishedNameNormalizerTests extends ESTestCase {
+
+    private UserRoleMapper.DistinguishedNameNormalizer dnNormalizer;
+
+    @Before
+    public void init() {
+        dnNormalizer = getDnNormalizer();
+    }
+
+    public void testDnNormalizingIsCached() {
+        // Parse same DN multiple times, only 1st time DN parsing is performed, 2nd time reads from the cache
+        Mockito.clearInvocations(dnNormalizer);
+        final String dn = randomDn();
+        parseDnMultipleTimes(dn);
+        verify(dnNormalizer, times(1)).doNormalize(dn);
+
+        // The cache is keyed by the literal string form.
+        // Therefore if the literal string changes, it needs to be parsed again even though it is still the same DN
+        Mockito.clearInvocations(dnNormalizer);
+        final String mutatedDn = mutateDn(dn);
+        parseDnMultipleTimes(mutatedDn);
+        verify(dnNormalizer, times(1)).doNormalize(mutatedDn);
+
+        // Invalid DNs should also be cached
+        Mockito.clearInvocations(dnNormalizer);
+        final String invalidDn = randomFrom(
+            "",
+            randomAlphaOfLengthBetween(1, 8),
+            randomAlphaOfLengthBetween(1, 8) + "*",
+            randomAlphaOfLengthBetween(1, 8) + "=",
+            "=" + randomAlphaOfLengthBetween(1, 8)
+        );
+        parseDnMultipleTimes(invalidDn);
+        verify(dnNormalizer, times(1)).doNormalize(invalidDn);
+    }
+
+    public void testDnNormalizingIsCachedForDnPredicate() {
+        final String dn = randomDn();
+        final Predicate<FieldValue> predicate = new UserRoleMapper.DistinguishedNamePredicate(dn, dnNormalizer);
+        verify(dnNormalizer, times(1)).doNormalize(dn);
+
+        // Same DN, it's cached
+        runPredicateMultipleTimes(predicate, dn);
+        verify(dnNormalizer, times(1)).doNormalize(dn);
+
+        // Predicate short-circuits for case differences
+        Mockito.clearInvocations(dnNormalizer);
+        final String casedDn = randomFrom(dn.toLowerCase(Locale.ENGLISH), dn.toUpperCase(Locale.ENGLISH));
+        runPredicateMultipleTimes(predicate, casedDn);
+        verify(dnNormalizer, never()).doNormalize(anyString());
+
+        // Literal string form changes, it will be parsed again
+        Mockito.clearInvocations(dnNormalizer);
+        final String mutatedDn = randomFrom(dn.replace(" ", ""), dn.replace(",", " ,"));
+        runPredicateMultipleTimes(predicate, mutatedDn);
+        verify(dnNormalizer, times(1)).doNormalize(mutatedDn);
+
+        // Subtree DN is also cached
+        Mockito.clearInvocations(dnNormalizer);
+        final String subtreeDn = "*," + randomDn();
+        runPredicateMultipleTimes(predicate, subtreeDn);
+        verify(dnNormalizer, times(1)).doNormalize(subtreeDn.substring(2));
+
+        // Subtree DN is also keyed by the literal form, so they are space sensitive
+        Mockito.clearInvocations(dnNormalizer);
+        final String mutatedSubtreeDn = "*, " + subtreeDn.substring(2);
+        runPredicateMultipleTimes(predicate, mutatedSubtreeDn);
+        verify(dnNormalizer, times(1)).doNormalize(mutatedSubtreeDn.substring(2));
+    }
+
+    public void testUserDataUsesCachedDnNormalizer() {
+        final String userDn = "uid=foo," + randomDn();
+        final List<String> groups = IntStream.range(0, randomIntBetween(50, 100))
+            .mapToObj(i -> "gid=g" + i + "," + randomDn())
+            .distinct()
+            .toList();
+        final RealmConfig realmConfig = mock(RealmConfig.class);
+        when(realmConfig.name()).thenReturn(randomAlphaOfLengthBetween(3, 8));
+        final UserRoleMapper.UserData userData = new UserRoleMapper.UserData(
+            randomAlphaOfLengthBetween(5, 8),
+            userDn,
+            groups,
+            Map.of(),
+            realmConfig
+        );
+        UserRoleMapper.UserData spyUserdata = spy(userData);
+        final UserRoleMapper.DistinguishedNameNormalizer spyDnNormalizer = spy(userData.getDnNormalizer());
+        when(spyUserdata.getDnNormalizer()).thenReturn(spyDnNormalizer);
+
+        final ExpressionModel expressionModel = spyUserdata.asModel();
+
+        // All DNs to be tested should only be parsed once no matter how many groups the userData may have
+        Mockito.clearInvocations(spyDnNormalizer);
+        final List<String> dnList = randomList(100, 200, DistinguishedNameNormalizerTests::randomDn).stream().distinct().toList();
+        final List<FieldValue> fieldValues = dnList.stream()
+            .map(dn -> randomBoolean() ? new FieldValue(dn) : new FieldValue("*," + dn))
+            .toList();
+        expressionModel.test("groups", fieldValues);
+        // Also does not matter how many times the model is tested
+        expressionModel.test("groups", randomNonEmptySubsetOf(fieldValues));
+
+        final ArgumentCaptor<String> argumentCaptor = ArgumentCaptor.forClass(String.class);
+        verify(spyDnNormalizer, times(dnList.size())).doNormalize(argumentCaptor.capture());
+        assertThat(argumentCaptor.getAllValues(), equalTo(dnList));
+    }
+
+    private void parseDnMultipleTimes(String dn) {
+        IntStream.range(0, randomIntBetween(3, 5)).forEach(i -> dnNormalizer.normalize(dn));
+    }
+
+    private void runPredicateMultipleTimes(Predicate<FieldValue> predicate, Object value) {
+        IntStream.range(0, randomIntBetween(3, 5)).forEach(i -> predicate.test(new FieldValue(value)));
+    }
+
+    private UserRoleMapper.DistinguishedNameNormalizer getDnNormalizer() {
+        return spy(new UserRoleMapper.DistinguishedNameNormalizer());
+    }
+
+    private static String randomDn() {
+        return "CN="
+            + randomAlphaOfLengthBetween(3, 12)
+            + ",OU="
+            + randomAlphaOfLength(4)
+            + ", O="
+            + randomAlphaOfLengthBetween(2, 6)
+            + ",dc="
+            + randomAlphaOfLength(3);
+    }
+
+    private static String mutateDn(String dn) {
+        return switch (randomIntBetween(1, 4)) {
+            case 1 -> dn.toLowerCase(Locale.ENGLISH);
+            case 2 -> dn.toUpperCase(Locale.ENGLISH);
+            case 3 -> dn.replace(" ", "");
+            default -> dn.replace(",", " ,");
+        };
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java
index 1ec63c96a579..195e12666248 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java
@@ -72,7 +72,7 @@ public void testParseRoles() throws Exception {
     }
 
     public void testToXContent() throws Exception {
-        final String json = formatted("""
+        final String json = Strings.format("""
             {"template":"{\\"source\\":\\"%s\\"}","format":"%s"}\
             """, randomAlphaOfLengthBetween(8, 24), randomFrom(Format.values()).formatName());
         assertThat(Strings.toString(parse(json)), equalTo(json));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/ParentActionAuthorizationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/ParentActionAuthorizationTests.java
new file mode 100644
index 000000000000..815c9e6d5400
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/ParentActionAuthorizationTests.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.authz;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.ParentActionAuthorization;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * Tests for the {@link ParentActionAuthorization} class.
+ */
+public class ParentActionAuthorizationTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        ParentActionAuthorization authorization = createRandom();
+        final BytesStreamOutput out = new BytesStreamOutput();
+        authorization.writeTo(out);
+        assertThat(ParentActionAuthorization.readFrom(out.bytes().streamInput()), equalTo(authorization));
+    }
+
+    private static ParentActionAuthorization createRandom() {
+        String action = randomAlphaOfLengthBetween(5, 20);
+        return new ParentActionAuthorization(action);
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorTests.java
index b0002578d524..5e73e4c358eb 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptorTests.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.core.security.authz;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -20,7 +20,7 @@
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TestMatchers;
-import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
@@ -48,7 +48,9 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.emptyArray;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.core.Is.is;
 
@@ -393,11 +395,11 @@ public void testParse() throws Exception {
     }
 
     public void testSerializationForCurrentVersion() throws Exception {
-        final Version version = VersionUtils.randomCompatibleVersion(random(), Version.CURRENT);
-        final boolean canIncludeRemoteIndices = version.onOrAfter(Version.V_8_6_0);
+        final TransportVersion version = TransportVersionUtils.randomCompatibleVersion(random(), TransportVersion.CURRENT);
+        final boolean canIncludeRemoteIndices = version.onOrAfter(TransportVersion.V_8_6_0);
         logger.info("Testing serialization with version {}", version);
         BytesStreamOutput output = new BytesStreamOutput();
-        output.setVersion(version);
+        output.setTransportVersion(version);
 
         final RoleDescriptor descriptor = randomRoleDescriptor(true, canIncludeRemoteIndices);
         descriptor.writeTo(output);
@@ -406,21 +408,21 @@ public void testSerializationForCurrentVersion() throws Exception {
             ByteBufferStreamInput.wrap(BytesReference.toBytes(output.bytes())),
             registry
         );
-        streamInput.setVersion(version);
+        streamInput.setTransportVersion(version);
         final RoleDescriptor serialized = new RoleDescriptor(streamInput);
 
         assertThat(serialized, equalTo(descriptor));
     }
 
     public void testSerializationWithRemoteIndicesThrowsOnUnsupportedVersions() throws IOException {
-        final Version versionBeforeRemoteIndices = VersionUtils.getPreviousVersion(Version.V_8_6_0);
-        final Version version = VersionUtils.randomVersionBetween(
+        final TransportVersion versionBeforeRemoteIndices = TransportVersionUtils.getPreviousVersion(TransportVersion.V_8_6_0);
+        final TransportVersion version = TransportVersionUtils.randomVersionBetween(
             random(),
-            versionBeforeRemoteIndices.minimumCompatibilityVersion(),
+            versionBeforeRemoteIndices.calculateMinimumCompatVersion(),
             versionBeforeRemoteIndices
         );
         final BytesStreamOutput output = new BytesStreamOutput();
-        output.setVersion(version);
+        output.setTransportVersion(version);
 
         final RoleDescriptor descriptor = randomRoleDescriptor(true, true);
         if (descriptor.hasRemoteIndicesPrivileges()) {
@@ -428,7 +430,7 @@ public void testSerializationWithRemoteIndicesThrowsOnUnsupportedVersions() thro
             assertThat(
                 ex.getMessage(),
                 containsString(
-                    "versions of Elasticsearch before [8.6.0] can't handle remote indices privileges and attempted to send to ["
+                    "versions of Elasticsearch before [8060099] can't handle remote indices privileges and attempted to send to ["
                         + version
                         + "]"
                 )
@@ -440,7 +442,7 @@ public void testSerializationWithRemoteIndicesThrowsOnUnsupportedVersions() thro
                 ByteBufferStreamInput.wrap(BytesReference.toBytes(output.bytes())),
                 registry
             );
-            streamInput.setVersion(version);
+            streamInput.setTransportVersion(version);
             final RoleDescriptor serialized = new RoleDescriptor(streamInput);
             assertThat(descriptor, equalTo(serialized));
         }
@@ -536,7 +538,7 @@ public void testParseIndicesPrivilegesSucceedsWhenExceptFieldsIsSubsetOfGrantedF
         final String grant = grantAll ? "\"*\"" : "\"f1\",\"f2\"";
         final String except = grantAll ? "\"_fx\",\"f8\"" : "\"f1\"";
 
-        final String json = formatted("""
+        final String json = Strings.format("""
             {
               "indices": [
                 {
@@ -655,6 +657,79 @@ public void testParseIndicesPrivilegesFailsWhenClustersFieldPresent() {
         );
     }
 
+    public void testIndicesPrivilegesCompareTo() {
+        final RoleDescriptor.IndicesPrivileges indexPrivilege = randomIndicesPrivilegesBuilder().build();
+        @SuppressWarnings({ "EqualsWithItself" })
+        final int actual = indexPrivilege.compareTo(indexPrivilege);
+        assertThat(actual, equalTo(0));
+        assertThat(
+            indexPrivilege.compareTo(
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(indexPrivilege.getIndices().clone())
+                    .privileges(indexPrivilege.getPrivileges().clone())
+                    // test for both cases when the query is the same instance or a copy
+                    .query(
+                        (indexPrivilege.getQuery() == null || randomBoolean())
+                            ? indexPrivilege.getQuery()
+                            : new BytesArray(indexPrivilege.getQuery().toBytesRef())
+                    )
+                    .grantedFields(indexPrivilege.getGrantedFields() == null ? null : indexPrivilege.getGrantedFields().clone())
+                    .deniedFields(indexPrivilege.getDeniedFields() == null ? null : indexPrivilege.getDeniedFields().clone())
+                    .allowRestrictedIndices(indexPrivilege.allowRestrictedIndices())
+                    .build()
+            ),
+            equalTo(0)
+        );
+
+        RoleDescriptor.IndicesPrivileges first = randomIndicesPrivilegesBuilder().allowRestrictedIndices(false).build();
+        RoleDescriptor.IndicesPrivileges second = randomIndicesPrivilegesBuilder().allowRestrictedIndices(true).build();
+        assertThat(first.compareTo(second), lessThan(0));
+        assertThat(second.compareTo(first), greaterThan(0));
+
+        first = randomIndicesPrivilegesBuilder().indices("a", "b").build();
+        second = randomIndicesPrivilegesBuilder().indices("b", "a").allowRestrictedIndices(first.allowRestrictedIndices()).build();
+        assertThat(first.compareTo(second), lessThan(0));
+        assertThat(second.compareTo(first), greaterThan(0));
+
+        first = randomIndicesPrivilegesBuilder().privileges("read", "write").build();
+        second = randomIndicesPrivilegesBuilder().allowRestrictedIndices(first.allowRestrictedIndices())
+            .privileges("write", "read")
+            .indices(first.getIndices())
+            .build();
+        assertThat(first.compareTo(second), lessThan(0));
+        assertThat(second.compareTo(first), greaterThan(0));
+
+        first = randomIndicesPrivilegesBuilder().query(randomBoolean() ? null : "{\"match\":{\"field-a\":\"a\"}}").build();
+        second = randomIndicesPrivilegesBuilder().allowRestrictedIndices(first.allowRestrictedIndices())
+            .query("{\"match\":{\"field-b\":\"b\"}}")
+            .indices(first.getIndices())
+            .privileges(first.getPrivileges())
+            .build();
+        assertThat(first.compareTo(second), lessThan(0));
+        assertThat(second.compareTo(first), greaterThan(0));
+
+        first = randomIndicesPrivilegesBuilder().grantedFields(randomBoolean() ? null : new String[] { "a", "b" }).build();
+        second = randomIndicesPrivilegesBuilder().allowRestrictedIndices(first.allowRestrictedIndices())
+            .grantedFields("b", "a")
+            .indices(first.getIndices())
+            .privileges(first.getPrivileges())
+            .query(first.getQuery())
+            .build();
+        assertThat(first.compareTo(second), lessThan(0));
+        assertThat(second.compareTo(first), greaterThan(0));
+
+        first = randomIndicesPrivilegesBuilder().deniedFields(randomBoolean() ? null : new String[] { "a", "b" }).build();
+        second = randomIndicesPrivilegesBuilder().allowRestrictedIndices(first.allowRestrictedIndices())
+            .deniedFields("b", "a")
+            .indices(first.getIndices())
+            .privileges(first.getPrivileges())
+            .query(first.getQuery())
+            .grantedFields(first.getGrantedFields())
+            .build();
+        assertThat(first.compareTo(second), lessThan(0));
+        assertThat(second.compareTo(first), greaterThan(0));
+    }
+
     public void testGlobalPrivilegesOrdering() throws IOException {
         final String roleName = randomAlphaOfLengthBetween(3, 30);
         final String[] applicationNames = generateRandomStringArray(3, randomIntBetween(0, 3), false, true);
@@ -701,7 +776,7 @@ public void testGlobalPrivilegesOrdering() throws IOException {
             }
             profileNamesString.append("\"" + profileNames[i] + "\"");
         }
-        String json = formatted("""
+        String json = Strings.format("""
             {
               "global": {
                 "profile": {
@@ -718,7 +793,7 @@ public void testGlobalPrivilegesOrdering() throws IOException {
             }""", profileNamesString, applicationNamesString);
         RoleDescriptor role3 = RoleDescriptor.parse(roleName, new BytesArray(json), false, XContentType.JSON);
         assertThat(role3, is(role1));
-        json = formatted("""
+        json = Strings.format("""
             {
               "global": {
                 "application": {
@@ -892,30 +967,34 @@ public static RoleDescriptor randomRoleDescriptor(boolean allowReservedMetadata,
         );
     }
 
-    private static RoleDescriptor.IndicesPrivileges[] randomIndicesPriveleges() {
+    public static RoleDescriptor.IndicesPrivileges[] randomIndicesPriveleges() {
         final RoleDescriptor.IndicesPrivileges[] indexPrivileges = new RoleDescriptor.IndicesPrivileges[randomIntBetween(0, 3)];
         for (int i = 0; i < indexPrivileges.length; i++) {
-            final RoleDescriptor.IndicesPrivileges.Builder builder = RoleDescriptor.IndicesPrivileges.builder()
-                .privileges(randomSubsetOf(randomIntBetween(1, 4), IndexPrivilege.names()))
-                .indices(generateRandomStringArray(5, randomIntBetween(3, 9), false, false))
-                .allowRestrictedIndices(randomBoolean());
-            if (randomBoolean()) {
-                builder.query(
-                    randomBoolean()
-                        ? "{ \"term\": { \"" + randomAlphaOfLengthBetween(3, 24) + "\" : \"" + randomAlphaOfLengthBetween(3, 24) + "\" }"
-                        : "{ \"match_all\": {} }"
-                );
-            }
+            indexPrivileges[i] = randomIndicesPrivilegesBuilder().build();
+        }
+        return indexPrivileges;
+    }
+
+    private static RoleDescriptor.IndicesPrivileges.Builder randomIndicesPrivilegesBuilder() {
+        final RoleDescriptor.IndicesPrivileges.Builder builder = RoleDescriptor.IndicesPrivileges.builder()
+            .privileges(randomSubsetOf(randomIntBetween(1, 4), IndexPrivilege.names()))
+            .indices(generateRandomStringArray(5, randomIntBetween(3, 9), false, false))
+            .allowRestrictedIndices(randomBoolean());
+        if (randomBoolean()) {
+            builder.query(
+                randomBoolean()
+                    ? "{ \"term\": { \"" + randomAlphaOfLengthBetween(3, 24) + "\" : \"" + randomAlphaOfLengthBetween(3, 24) + "\" }"
+                    : "{ \"match_all\": {} }"
+            );
+        }
+        if (randomBoolean()) {
             if (randomBoolean()) {
-                if (randomBoolean()) {
-                    builder.grantedFields("*");
-                    builder.deniedFields(generateRandomStringArray(4, randomIntBetween(4, 9), false, false));
-                } else {
-                    builder.grantedFields(generateRandomStringArray(4, randomIntBetween(4, 9), false, false));
-                }
+                builder.grantedFields("*");
+                builder.deniedFields(generateRandomStringArray(4, randomIntBetween(4, 9), false, false));
+            } else {
+                builder.grantedFields(generateRandomStringArray(4, randomIntBetween(4, 9), false, false));
             }
-            indexPrivileges[i] = builder.build();
         }
-        return indexPrivileges;
+        return builder;
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java
index f999db8a1b59..021eab54436c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java
@@ -22,15 +22,12 @@
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BitSet;
-import org.apache.lucene.util.BitSetIterator;
-import org.apache.lucene.util.FixedBitSet;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.logging.Loggers;
@@ -71,7 +68,6 @@
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
@@ -533,43 +529,6 @@ public void testCacheClearEntriesWhenIndexIsClosed() throws Exception {
         }
     }
 
-    public void testRoleBitSets() throws Exception {
-        int maxDocs = randomIntBetween(1, 1024);
-        int numDocs = 0;
-        FixedBitSet matches = new FixedBitSet(maxDocs);
-        for (int i = 0; i < maxDocs; i++) {
-            if (numDocs < maxDocs && randomBoolean()) {
-                numDocs++;
-                matches.set(i);
-            }
-        }
-        DocIdSetIterator it = new BitSetIterator(matches, randomIntBetween(0, numDocs));
-        BitSet bitSet = DocumentSubsetBitsetCache.bitSetFromDocIterator(it, maxDocs);
-        assertThat(bitSet.cardinality(), equalTo(numDocs));
-        assertThat(bitSet.length(), equalTo(maxDocs));
-        for (int i = 0; i < maxDocs; i++) {
-            assertThat(bitSet.get(i), equalTo(matches.get(i)));
-            assertThat(bitSet.nextSetBit(i), equalTo(matches.nextSetBit(i)));
-            assertThat(bitSet.prevSetBit(i), equalTo(matches.prevSetBit(i)));
-        }
-    }
-
-    public void testMatchAllRoleBitSet() throws Exception {
-        int maxDocs = randomIntBetween(1, 128);
-        FixedBitSet matches = new FixedBitSet(maxDocs);
-        for (int i = 0; i < maxDocs; i++) {
-            matches.set(i);
-        }
-        DocIdSetIterator it = new BitSetIterator(matches, randomNonNegativeLong());
-        BitSet bitSet = DocumentSubsetBitsetCache.bitSetFromDocIterator(it, maxDocs);
-        assertThat(bitSet, instanceOf(MatchAllRoleBitSet.class));
-        for (int i = 0; i < maxDocs; i++) {
-            assertTrue(bitSet.get(i));
-            assertThat(bitSet.nextSetBit(i), equalTo(matches.nextSetBit(i)));
-            assertThat(bitSet.prevSetBit(i), equalTo(matches.prevSetBit(i)));
-        }
-    }
-
     public void testEquivalentMatchAllDocsQuery() {
         assertTrue(DocumentSubsetBitsetCache.isEffectiveMatchAllDocsQuery(new MatchAllDocsQuery()));
         assertTrue(DocumentSubsetBitsetCache.isEffectiveMatchAllDocsQuery(new ConstantScoreQuery(new MatchAllDocsQuery())));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java
index ef061b25212b..a64110bb3a17 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java
@@ -11,7 +11,8 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.IntPoint;
-import org.apache.lucene.document.KnnVectorField;
+import org.apache.lucene.document.KnnByteVectorField;
+import org.apache.lucene.document.KnnFloatVectorField;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.SortedNumericDocValuesField;
@@ -20,11 +21,13 @@
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.FilterDirectoryReader;
+import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.LeafReader;
@@ -42,7 +45,6 @@
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.TermsEnum.SeekStatus;
-import org.apache.lucene.index.VectorValues;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.analysis.MockAnalyzer;
@@ -188,32 +190,64 @@ public void testKnnVectors() throws Exception {
         IndexWriter iw = new IndexWriter(dir, iwc);
 
         Document doc = new Document();
-        doc.add(new KnnVectorField("fieldA", new float[] { 0.1f, 0.2f, 0.3f }));
-        doc.add(new KnnVectorField("fieldB", new float[] { 3.0f, 2.0f, 1.0f }));
+        doc.add(new KnnFloatVectorField("fieldA", new float[] { 0.1f, 0.2f, 0.3f }));
+        doc.add(new KnnFloatVectorField("fieldB", new float[] { 3.0f, 2.0f, 1.0f }));
         iw.addDocument(doc);
 
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
         LeafReader leafReader = ir.leaves().get(0).reader();
 
         // Check that fieldA behaves as normal
-        VectorValues vectorValues = leafReader.getVectorValues("fieldA");
+        FloatVectorValues vectorValues = leafReader.getFloatVectorValues("fieldA");
         assertEquals(3, vectorValues.dimension());
         assertEquals(1, vectorValues.size());
         assertEquals(0, vectorValues.nextDoc());
-        assertNotNull(vectorValues.binaryValue());
+        assertNotNull(vectorValues.vectorValue());
 
         TopDocs topDocs = leafReader.searchNearestVectors("fieldA", new float[] { 1.0f, 1.0f, 1.0f }, 5, null, Integer.MAX_VALUE);
         assertNotNull(topDocs);
         assertEquals(1, topDocs.scoreDocs.length);
 
         // Check that we can't see fieldB
-        assertNull(leafReader.getVectorValues("fieldB"));
+        assertNull(leafReader.getFloatVectorValues("fieldB"));
         assertNull(leafReader.searchNearestVectors("fieldB", new float[] { 1.0f, 1.0f, 1.0f }, 5, null, Integer.MAX_VALUE));
 
         TestUtil.checkReader(ir);
         IOUtils.close(ir, iw, dir);
     }
 
+    public void testKnnByteVectors() throws Exception {
+        Directory dir = newDirectory();
+        IndexWriterConfig iwc = new IndexWriterConfig(null);
+        IndexWriter iw = new IndexWriter(dir, iwc);
+
+        Document doc = new Document();
+        doc.add(new KnnByteVectorField("fieldA", new byte[] { 1, 2, 3 }));
+        doc.add(new KnnByteVectorField("fieldB", new byte[] { 3, 2, 1 }));
+        iw.addDocument(doc);
+
+        DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
+        LeafReader leafReader = ir.leaves().get(0).reader();
+
+        // Check that fieldA behaves as normal
+        ByteVectorValues vectorValues = leafReader.getByteVectorValues("fieldA");
+        assertEquals(3, vectorValues.dimension());
+        assertEquals(1, vectorValues.size());
+        assertEquals(0, vectorValues.nextDoc());
+        assertNotNull(vectorValues.vectorValue());
+
+        TopDocs topDocs = leafReader.searchNearestVectors("fieldA", new byte[] { 1, 1, 1 }, 5, null, Integer.MAX_VALUE);
+        assertNotNull(topDocs);
+        assertEquals(1, topDocs.scoreDocs.length);
+
+        // Check that we can't see fieldB
+        assertNull(leafReader.getByteVectorValues("fieldB"));
+        assertNull(leafReader.searchNearestVectors("fieldB", new byte[] { 1, 1, 1 }, 5, null, Integer.MAX_VALUE));
+
+        TestUtil.checkReader(ir);
+        IOUtils.close(ir, iw, dir);
+    }
+
     /**
      * test filtering two stored fields (string)
      */
@@ -232,10 +266,16 @@ public void testStoredFieldsString() throws Exception {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
 
         // see only one field
-        Document d2 = ir.document(0);
-        assertEquals(1, d2.getFields().size());
-        assertEquals("testA", d2.get("fieldA"));
-
+        {
+            Document d2 = ir.document(0);
+            assertEquals(1, d2.getFields().size());
+            assertEquals("testA", d2.get("fieldA"));
+        }
+        {
+            Document d2 = ir.storedFields().document(0);
+            assertEquals(1, d2.getFields().size());
+            assertEquals("testA", d2.get("fieldA"));
+        }
         TestUtil.checkReader(ir);
         IOUtils.close(ir, iw, dir);
     }
@@ -258,10 +298,16 @@ public void testStoredFieldsBinary() throws Exception {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
 
         // see only one field
-        Document d2 = ir.document(0);
-        assertEquals(1, d2.getFields().size());
-        assertEquals(new BytesRef("testA"), d2.getBinaryValue("fieldA"));
-
+        {
+            Document d2 = ir.document(0);
+            assertEquals(1, d2.getFields().size());
+            assertEquals(new BytesRef("testA"), d2.getBinaryValue("fieldA"));
+        }
+        {
+            Document d2 = ir.storedFields().document(0);
+            assertEquals(1, d2.getFields().size());
+            assertEquals(new BytesRef("testA"), d2.getBinaryValue("fieldA"));
+        }
         TestUtil.checkReader(ir);
         IOUtils.close(ir, iw, dir);
     }
@@ -284,10 +330,16 @@ public void testStoredFieldsInt() throws Exception {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
 
         // see only one field
-        Document d2 = ir.document(0);
-        assertEquals(1, d2.getFields().size());
-        assertEquals(1, d2.getField("fieldA").numericValue());
-
+        {
+            Document d2 = ir.document(0);
+            assertEquals(1, d2.getFields().size());
+            assertEquals(1, d2.getField("fieldA").numericValue());
+        }
+        {
+            Document d2 = ir.storedFields().document(0);
+            assertEquals(1, d2.getFields().size());
+            assertEquals(1, d2.getField("fieldA").numericValue());
+        }
         TestUtil.checkReader(ir);
         IOUtils.close(ir, iw, dir);
     }
@@ -310,10 +362,16 @@ public void testStoredFieldsLong() throws Exception {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
 
         // see only one field
-        Document d2 = ir.document(0);
-        assertEquals(1, d2.getFields().size());
-        assertEquals(1L, d2.getField("fieldA").numericValue());
-
+        {
+            Document d2 = ir.document(0);
+            assertEquals(1, d2.getFields().size());
+            assertEquals(1L, d2.getField("fieldA").numericValue());
+        }
+        {
+            Document d2 = ir.storedFields().document(0);
+            assertEquals(1, d2.getFields().size());
+            assertEquals(1L, d2.getField("fieldA").numericValue());
+        }
         TestUtil.checkReader(ir);
         IOUtils.close(ir, iw, dir);
     }
@@ -336,10 +394,16 @@ public void testStoredFieldsFloat() throws Exception {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
 
         // see only one field
-        Document d2 = ir.document(0);
-        assertEquals(1, d2.getFields().size());
-        assertEquals(1F, d2.getField("fieldA").numericValue());
-
+        {
+            Document d2 = ir.document(0);
+            assertEquals(1, d2.getFields().size());
+            assertEquals(1F, d2.getField("fieldA").numericValue());
+        }
+        {
+            Document d2 = ir.storedFields().document(0);
+            assertEquals(1, d2.getFields().size());
+            assertEquals(1F, d2.getField("fieldA").numericValue());
+        }
         TestUtil.checkReader(ir);
         IOUtils.close(ir, iw, dir);
     }
@@ -362,10 +426,16 @@ public void testStoredFieldsDouble() throws Exception {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")));
 
         // see only one field
-        Document d2 = ir.document(0);
-        assertEquals(1, d2.getFields().size());
-        assertEquals(1D, d2.getField("fieldA").numericValue());
-
+        {
+            Document d2 = ir.document(0);
+            assertEquals(1, d2.getFields().size());
+            assertEquals(1D, d2.getField("fieldA").numericValue());
+        }
+        {
+            Document d2 = ir.storedFields().document(0);
+            assertEquals(1, d2.getFields().size());
+            assertEquals(1D, d2.getField("fieldA").numericValue());
+        }
         TestUtil.checkReader(ir);
         IOUtils.close(ir, iw, dir);
     }
@@ -624,10 +694,16 @@ public void testSourceFilteringIntegration() throws Exception {
         DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(automaton));
 
         // see only one field
-        Document d2 = ir.document(0);
-        assertEquals(1, d2.getFields().size());
-        assertEquals("{\"fieldA\":\"testA\"}", d2.getBinaryValue(SourceFieldMapper.NAME).utf8ToString());
-
+        {
+            Document d2 = ir.document(0);
+            assertEquals(1, d2.getFields().size());
+            assertEquals("{\"fieldA\":\"testA\"}", d2.getBinaryValue(SourceFieldMapper.NAME).utf8ToString());
+        }
+        {
+            Document d2 = ir.storedFields().document(0);
+            assertEquals(1, d2.getFields().size());
+            assertEquals("{\"fieldA\":\"testA\"}", d2.getBinaryValue(SourceFieldMapper.NAME).utf8ToString());
+        }
         TestUtil.checkReader(ir);
         IOUtils.close(ir, iw, dir);
     }
@@ -1025,11 +1101,17 @@ public void testEmpty() throws Exception {
 
         // see no vectors
         assertNull(segmentReader.getTermVectors(0));
+        assertNull(segmentReader.termVectors().get(0));
 
         // see no stored fields
-        Document document = segmentReader.document(0);
-        assertEquals(0, document.getFields().size());
-
+        {
+            Document document = segmentReader.document(0);
+            assertEquals(0, document.getFields().size());
+        }
+        {
+            Document document = segmentReader.storedFields().document(0);
+            assertEquals(0, document.getFields().size());
+        }
         TestUtil.checkReader(ir);
         IOUtils.close(ir, iw, dir);
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRoleTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRoleTests.java
index 1bf6ef61b3a1..af67b32140ef 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRoleTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRoleTests.java
@@ -305,6 +305,32 @@ public void testAllowedIndicesMatcher() {
         }
     }
 
+    public void testAllowedIndicesMatcherWithNestedRole() {
+        Role role = Role.builder(EMPTY_RESTRICTED_INDICES, "a-role").add(IndexPrivilege.READ, "ind-1*").build();
+        assertThat(role.allowedIndicesMatcher(SearchAction.NAME).test(mockIndexAbstraction("ind-1")), is(true));
+        assertThat(role.allowedIndicesMatcher(SearchAction.NAME).test(mockIndexAbstraction("ind-11")), is(true));
+        assertThat(role.allowedIndicesMatcher(SearchAction.NAME).test(mockIndexAbstraction("ind-2")), is(false));
+
+        final int depth = randomIntBetween(2, 4);
+        boolean index11Excluded = false;
+        for (int i = 0; i < depth; i++) {
+            final boolean excludeIndex11 = randomBoolean();
+            final String[] indexNames = excludeIndex11 ? new String[] { "ind-1", "ind-2" } : new String[] { "ind-*" };
+            index11Excluded = index11Excluded || excludeIndex11;
+            final Role limitedByRole = Role.builder(EMPTY_RESTRICTED_INDICES, "limited-role-" + i)
+                .add(IndexPrivilege.READ, indexNames)
+                .build();
+            if (randomBoolean()) {
+                role = limitedByRole.limitedBy(role);
+            } else {
+                role = role.limitedBy(limitedByRole);
+            }
+            assertThat(role.allowedIndicesMatcher(SearchAction.NAME).test(mockIndexAbstraction("ind-1")), is(true));
+            assertThat(role.allowedIndicesMatcher(SearchAction.NAME).test(mockIndexAbstraction("ind-11")), is(false == index11Excluded));
+            assertThat(role.allowedIndicesMatcher(SearchAction.NAME).test(mockIndexAbstraction("ind-2")), is(false));
+        }
+    }
+
     public void testAllowedActionsMatcher() {
         Role fromRole = Role.builder(EMPTY_RESTRICTED_INDICES, "fromRole")
             .add(IndexPrivilege.WRITE, "ind*")
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilegeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilegeTests.java
index 24953bce5673..48a9472236c5 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilegeTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilegeTests.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.action.update.UpdateAction;
 import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.ml.action.SemanticSearchAction;
 import org.elasticsearch.xpack.core.rollup.action.GetRollupIndexCapsAction;
 import org.elasticsearch.xpack.core.transform.action.GetCheckpointAction;
 
@@ -81,10 +80,6 @@ public void testPrivilegesForGetCheckPointAction() {
         );
     }
 
-    public void testPrivilegesForSemanticSearchAction() {
-        assertThat(findPrivilegesThatGrant(SemanticSearchAction.NAME), equalTo(List.of("read", "all")));
-    }
-
     public void testRelationshipBetweenPrivileges() {
         assertThat(
             Operations.subsetOf(
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
index 8c562e267a62..e3ac7164e956 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
@@ -565,6 +565,7 @@ public void testKibanaSystemRole() {
             ".reporting-" + randomAlphaOfLength(randomIntBetween(0, 13)),
             ".apm-agent-configuration",
             ".apm-custom-link",
+            ".apm-source-map",
             ReservedRolesStore.ALERTS_LEGACY_INDEX + randomAlphaOfLength(randomIntBetween(0, 13)),
             ReservedRolesStore.ALERTS_BACKING_INDEX + randomAlphaOfLength(randomIntBetween(0, 13)),
             ReservedRolesStore.ALERTS_INDEX_ALIAS + randomAlphaOfLength(randomIntBetween(0, 13)),
@@ -732,6 +733,19 @@ public void testKibanaSystemRole() {
             ".fleet-servers"
         ).forEach(index -> assertAllIndicesAccessAllowed(kibanaRole, index));
 
+        // read-only indices for Fleet telemetry
+        Arrays.asList("logs-elastic_agent-default", "logs-elastic_agent.fleet_server-default").forEach((index) -> {
+            assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:foo").test(mockIndexAbstraction(index)), is(false));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:bar").test(mockIndexAbstraction(index)), is(false));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(GetIndexAction.NAME).test(mockIndexAbstraction(index)), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(CreateIndexAction.NAME).test(mockIndexAbstraction(index)), is(false));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(IndexAction.NAME).test(mockIndexAbstraction(index)), is(false));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(DeleteAction.NAME).test(mockIndexAbstraction(index)), is(false));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(SearchAction.NAME).test(mockIndexAbstraction(index)), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(MultiSearchAction.NAME).test(mockIndexAbstraction(index)), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(GetAction.NAME).test(mockIndexAbstraction(index)), is(true));
+        });
+
         // read-only index for Endpoint and Osquery manager specific action responses
         Arrays.asList(".logs-endpoint.action.responses-" + randomAlphaOfLength(randomIntBetween(0, 13))).forEach((index) -> {
             final IndexAbstraction indexAbstraction = mockIndexAbstraction(index);
@@ -983,7 +997,10 @@ public void testKibanaSystemRole() {
         Arrays.asList(
             "metrics-endpoint.metadata_current_default",
             ".metrics-endpoint.metadata_current_default",
-            ".metrics-endpoint.metadata_united_default"
+            ".metrics-endpoint.metadata_united_default",
+            "metrics-endpoint.metadata_current_default-" + Version.CURRENT,
+            ".metrics-endpoint.metadata_current_default-" + Version.CURRENT,
+            ".metrics-endpoint.metadata_united_default-" + Version.CURRENT
         ).forEach(indexName -> {
             logger.info("index name [{}]", indexName);
             final IndexAbstraction indexAbstraction = mockIndexAbstraction(indexName);
@@ -993,12 +1010,15 @@ public void testKibanaSystemRole() {
             assertThat(kibanaRole.indices().allowedIndicesMatcher(IndexAction.NAME).test(indexAbstraction), is(true));
             assertThat(kibanaRole.indices().allowedIndicesMatcher(UpdateAction.NAME).test(indexAbstraction), is(true));
             assertThat(kibanaRole.indices().allowedIndicesMatcher(BulkAction.NAME).test(indexAbstraction), is(true));
-            // Allow create and delete index
+            // Allow create and delete index, modifying aliases, and updating index settings
             assertThat(kibanaRole.indices().allowedIndicesMatcher(CreateIndexAction.NAME).test(indexAbstraction), is(true));
             assertThat(kibanaRole.indices().allowedIndicesMatcher(AutoCreateAction.NAME).test(indexAbstraction), is(true));
             assertThat(kibanaRole.indices().allowedIndicesMatcher(CreateDataStreamAction.NAME).test(indexAbstraction), is(true));
             assertThat(kibanaRole.indices().allowedIndicesMatcher(DeleteIndexAction.NAME).test(indexAbstraction), is(true));
             assertThat(kibanaRole.indices().allowedIndicesMatcher(DeleteDataStreamAction.NAME).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(GetAliasesAction.NAME).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(IndicesAliasesAction.NAME).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(UpdateSettingsAction.NAME).test(indexAbstraction), is(true));
 
             // Implied by the overall view_index_metadata and monitor privilege
             assertViewIndexMetadata(kibanaRole, indexName);
@@ -1013,9 +1033,8 @@ public void testKibanaSystemRole() {
                 is(indexAbstraction.getType() != IndexAbstraction.Type.DATA_STREAM)
             );
 
-            // Deny deleting documents and modifying the index settings
+            // Deny deleting documents and rollover
             assertThat(kibanaRole.indices().allowedIndicesMatcher(DeleteAction.NAME).test(indexAbstraction), is(false));
-            assertThat(kibanaRole.indices().allowedIndicesMatcher(UpdateSettingsAction.NAME).test(indexAbstraction), is(false));
             assertThat(kibanaRole.indices().allowedIndicesMatcher(RolloverAction.NAME).test(indexAbstraction), is(false));
         });
 
@@ -1063,30 +1082,35 @@ public void testKibanaSystemRole() {
             assertThat(kibanaRole.indices().allowedIndicesMatcher(RolloverAction.NAME).test(indexAbstraction), is(true));
         });
 
-        Arrays.asList("logs-cloud_security_posture.findings_latest-default", "logs-cloud_security_posture.scores-default")
-            .forEach(indexName -> {
-                logger.info("index name [{}]", indexName);
-                final IndexAbstraction indexAbstraction = mockIndexAbstraction(indexName);
-                // Allow indexing
-                assertThat(kibanaRole.indices().allowedIndicesMatcher(SearchAction.NAME).test(indexAbstraction), is(true));
-                assertThat(kibanaRole.indices().allowedIndicesMatcher(GetAction.NAME).test(indexAbstraction), is(true));
-                assertThat(kibanaRole.indices().allowedIndicesMatcher(IndexAction.NAME).test(indexAbstraction), is(true));
-                assertThat(kibanaRole.indices().allowedIndicesMatcher(UpdateAction.NAME).test(indexAbstraction), is(true));
-                assertThat(kibanaRole.indices().allowedIndicesMatcher(BulkAction.NAME).test(indexAbstraction), is(true));
-                // Allow create and delete index
-                assertThat(kibanaRole.indices().allowedIndicesMatcher(CreateIndexAction.NAME).test(indexAbstraction), is(true));
-                assertThat(kibanaRole.indices().allowedIndicesMatcher(AutoCreateAction.NAME).test(indexAbstraction), is(true));
-                assertThat(kibanaRole.indices().allowedIndicesMatcher(CreateDataStreamAction.NAME).test(indexAbstraction), is(true));
-
-                // Implied by the overall view_index_metadata and monitor privilege
-                assertViewIndexMetadata(kibanaRole, indexName);
-                assertThat(
-                    kibanaRole.indices()
-                        .allowedIndicesMatcher("indices:monitor/" + randomAlphaOfLengthBetween(3, 8))
-                        .test(indexAbstraction),
-                    is(true)
-                );
-            });
+        Arrays.asList(
+            "logs-cloud_security_posture.findings_latest-default",
+            "logs-cloud_security_posture.scores-default",
+            "logs-cloud_security_posture.findings_latest-default-" + Version.CURRENT,
+            "logs-cloud_security_posture.scores-default-" + Version.CURRENT
+        ).forEach(indexName -> {
+            logger.info("index name [{}]", indexName);
+            final IndexAbstraction indexAbstraction = mockIndexAbstraction(indexName);
+            // Allow indexing
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(SearchAction.NAME).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(GetAction.NAME).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(IndexAction.NAME).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(UpdateAction.NAME).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(BulkAction.NAME).test(indexAbstraction), is(true));
+            // Allow create and delete index, modifying aliases, and updating index settings
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(CreateIndexAction.NAME).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(AutoCreateAction.NAME).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(CreateDataStreamAction.NAME).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(GetAliasesAction.NAME).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(IndicesAliasesAction.NAME).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(UpdateSettingsAction.NAME).test(indexAbstraction), is(true));
+
+            // Implied by the overall view_index_metadata and monitor privilege
+            assertViewIndexMetadata(kibanaRole, indexName);
+            assertThat(
+                kibanaRole.indices().allowedIndicesMatcher("indices:monitor/" + randomAlphaOfLengthBetween(3, 8)).test(indexAbstraction),
+                is(true)
+            );
+        });
 
         // Ensure privileges necessary for ILM policies in APM & Endpoint packages
         Arrays.asList(
@@ -1121,9 +1145,12 @@ public void testKibanaSystemRole() {
             assertThat(kibanaRole.indices().allowedIndicesMatcher(IndexAction.NAME).test(indexAbstraction), is(true));
             assertThat(kibanaRole.indices().allowedIndicesMatcher(UpdateAction.NAME).test(indexAbstraction), is(true));
             assertThat(kibanaRole.indices().allowedIndicesMatcher(BulkAction.NAME).test(indexAbstraction), is(true));
-            // Allow create and delete index
+            // Allow create and delete index, modifying aliases, and updating index settings
             assertThat(kibanaRole.indices().allowedIndicesMatcher(CreateIndexAction.NAME).test(indexAbstraction), is(true));
             assertThat(kibanaRole.indices().allowedIndicesMatcher(DeleteIndexAction.NAME).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(GetAliasesAction.NAME).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(IndicesAliasesAction.NAME).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(UpdateSettingsAction.NAME).test(indexAbstraction), is(true));
 
             // Implied by the overall view_index_metadata and monitor privilege
             assertViewIndexMetadata(kibanaRole, indexName);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/RoleReferenceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/RoleReferenceTests.java
index 35ba3e171d6c..c10b01f59c2a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/RoleReferenceTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/RoleReferenceTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.hash.MessageDigests;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.authc.RemoteAccessAuthentication;
 
 import java.util.Set;
 
@@ -65,6 +66,18 @@ public void testApiKeyRoleReference() {
         assertThat(roleKey.getSource(), equalTo("apikey_" + apiKeyRoleType));
     }
 
+    public void testRemoteAccessRoleReference() {
+        final var roleDescriptorsBytes = new RemoteAccessAuthentication.RoleDescriptorsBytes(new BytesArray(randomAlphaOfLength(50)));
+        final var remoteAccessRoleReference = new RoleReference.RemoteAccessRoleReference(roleDescriptorsBytes);
+
+        final RoleKey roleKey = remoteAccessRoleReference.id();
+        assertThat(
+            roleKey.getNames(),
+            hasItem("remote_access:" + MessageDigests.toHexString(MessageDigests.digest(roleDescriptorsBytes, MessageDigests.sha256())))
+        );
+        assertThat(roleKey.getSource(), equalTo("remote_access"));
+    }
+
     public void testServiceAccountRoleReference() {
         final String principal = randomAlphaOfLength(8) + "/" + randomAlphaOfLength(8);
         final RoleReference.ServiceAccountRoleReference serviceAccountRoleReference = new RoleReference.ServiceAccountRoleReference(
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/support/DLSRoleQueryValidatorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/support/DLSRoleQueryValidatorTests.java
index 1ec6847ca0d7..9f8f3703a0e1 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/support/DLSRoleQueryValidatorTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/support/DLSRoleQueryValidatorTests.java
@@ -7,7 +7,9 @@
 package org.elasticsearch.xpack.core.security.authz.support;
 
 import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.BoostingQueryBuilder;
 import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
@@ -17,52 +19,86 @@
 import org.elasticsearch.index.query.TermsQueryBuilder;
 import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
 import org.elasticsearch.indices.TermsLookup;
+import org.elasticsearch.join.ParentJoinPlugin;
 import org.elasticsearch.join.query.HasChildQueryBuilder;
 import org.elasticsearch.join.query.HasParentQueryBuilder;
+import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
 import java.io.IOException;
+import java.util.List;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
 public class DLSRoleQueryValidatorTests extends ESTestCase {
 
+    private static final NamedXContentRegistry XCONTENT_REGISTRY = new NamedXContentRegistry(
+        new SearchModule(Settings.EMPTY, List.of(new ParentJoinPlugin())).getNamedXContents()
+    );
+
     public void testVerifyRoleQuery() throws Exception {
         QueryBuilder queryBuilder1 = new TermsQueryBuilder("field", "val1", "val2");
-        DLSRoleQueryValidator.verifyRoleQuery(queryBuilder1);
+        DLSRoleQueryValidator.evaluateAndVerifyRoleQuery(Strings.toString(queryBuilder1), XCONTENT_REGISTRY);
 
         QueryBuilder queryBuilder2 = new TermsQueryBuilder("field", new TermsLookup("_index", "_id", "_path"));
-        Exception e = expectThrows(IllegalArgumentException.class, () -> DLSRoleQueryValidator.verifyRoleQuery(queryBuilder2));
+        Exception e = expectThrows(
+            IllegalArgumentException.class,
+            () -> DLSRoleQueryValidator.evaluateAndVerifyRoleQuery(Strings.toString(queryBuilder2), XCONTENT_REGISTRY)
+        );
         assertThat(e.getMessage(), equalTo("terms query with terms lookup isn't supported as part of a role query"));
 
         QueryBuilder queryBuilder3 = new GeoShapeQueryBuilder("field", "_id");
-        e = expectThrows(IllegalArgumentException.class, () -> DLSRoleQueryValidator.verifyRoleQuery(queryBuilder3));
+        e = expectThrows(
+            IllegalArgumentException.class,
+            () -> DLSRoleQueryValidator.evaluateAndVerifyRoleQuery(Strings.toString(queryBuilder3), XCONTENT_REGISTRY)
+        );
         assertThat(e.getMessage(), equalTo("geoshape query referring to indexed shapes isn't supported as part of a role query"));
 
         QueryBuilder queryBuilder4 = new HasChildQueryBuilder("_type", new MatchAllQueryBuilder(), ScoreMode.None);
-        e = expectThrows(IllegalArgumentException.class, () -> DLSRoleQueryValidator.verifyRoleQuery(queryBuilder4));
+        e = expectThrows(
+            IllegalArgumentException.class,
+            () -> DLSRoleQueryValidator.evaluateAndVerifyRoleQuery(Strings.toString(queryBuilder4), XCONTENT_REGISTRY)
+        );
         assertThat(e.getMessage(), equalTo("has_child query isn't supported as part of a role query"));
 
         QueryBuilder queryBuilder5 = new HasParentQueryBuilder("_type", new MatchAllQueryBuilder(), false);
-        e = expectThrows(IllegalArgumentException.class, () -> DLSRoleQueryValidator.verifyRoleQuery(queryBuilder5));
+        e = expectThrows(
+            IllegalArgumentException.class,
+            () -> DLSRoleQueryValidator.evaluateAndVerifyRoleQuery(Strings.toString(queryBuilder5), XCONTENT_REGISTRY)
+        );
         assertThat(e.getMessage(), equalTo("has_parent query isn't supported as part of a role query"));
 
         QueryBuilder queryBuilder6 = new BoolQueryBuilder().must(new GeoShapeQueryBuilder("field", "_id"));
-        e = expectThrows(IllegalArgumentException.class, () -> DLSRoleQueryValidator.verifyRoleQuery(queryBuilder6));
-        assertThat(e.getMessage(), equalTo("geoshape query referring to indexed shapes isn't supported as part of a role query"));
+        e = expectThrows(
+            IllegalArgumentException.class,
+            () -> DLSRoleQueryValidator.evaluateAndVerifyRoleQuery(Strings.toString(queryBuilder6), XCONTENT_REGISTRY)
+        );
+        assertThat(
+            e.getCause().getMessage(),
+            equalTo("geoshape query referring to indexed shapes isn't supported as part of a role query")
+        );
 
         QueryBuilder queryBuilder7 = new ConstantScoreQueryBuilder(new GeoShapeQueryBuilder("field", "_id"));
-        e = expectThrows(IllegalArgumentException.class, () -> DLSRoleQueryValidator.verifyRoleQuery(queryBuilder7));
+        e = expectThrows(
+            IllegalArgumentException.class,
+            () -> DLSRoleQueryValidator.evaluateAndVerifyRoleQuery(Strings.toString(queryBuilder7), XCONTENT_REGISTRY)
+        );
         assertThat(e.getMessage(), equalTo("geoshape query referring to indexed shapes isn't supported as part of a role query"));
 
         QueryBuilder queryBuilder8 = new FunctionScoreQueryBuilder(new GeoShapeQueryBuilder("field", "_id"));
-        e = expectThrows(IllegalArgumentException.class, () -> DLSRoleQueryValidator.verifyRoleQuery(queryBuilder8));
+        e = expectThrows(
+            IllegalArgumentException.class,
+            () -> DLSRoleQueryValidator.evaluateAndVerifyRoleQuery(Strings.toString(queryBuilder8), XCONTENT_REGISTRY)
+        );
         assertThat(e.getMessage(), equalTo("geoshape query referring to indexed shapes isn't supported as part of a role query"));
 
         QueryBuilder queryBuilder9 = new BoostingQueryBuilder(new GeoShapeQueryBuilder("field", "_id"), new MatchAllQueryBuilder());
-        e = expectThrows(IllegalArgumentException.class, () -> DLSRoleQueryValidator.verifyRoleQuery(queryBuilder9));
+        e = expectThrows(
+            IllegalArgumentException.class,
+            () -> DLSRoleQueryValidator.evaluateAndVerifyRoleQuery(Strings.toString(queryBuilder9), XCONTENT_REGISTRY)
+        );
         assertThat(e.getMessage(), equalTo("geoshape query referring to indexed shapes isn't supported as part of a role query"));
     }
 
@@ -76,5 +112,4 @@ public void testHasStoredScript() throws IOException {
             is(false)
         );
     }
-
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonsTests.java
index 09923c937cfc..0b2e48bd20df 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/AutomatonsTests.java
@@ -12,13 +12,13 @@
 import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
 import org.apache.lucene.util.automaton.Transition;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.Locale;
 
 import static org.apache.lucene.util.automaton.Operations.DEFAULT_DETERMINIZE_WORK_LIMIT;
 import static org.elasticsearch.xpack.core.security.support.Automatons.pattern;
@@ -223,13 +223,14 @@ public void testDisableCache() {
     // (and it is annoying to have to rewrite it each time it's needed)
     public static <A extends Appendable> A debug(Automaton a, A out) throws IOException {
         out.append("Automaton {");
-        out.append(String.format(Locale.ROOT, "States:%d  Deterministic:%s", a.getNumStates(), a.isDeterministic()));
+        out.append(Strings.format("States:%d  Deterministic:%s", a.getNumStates(), a.isDeterministic()));
         for (int s = 0; s < a.getNumStates(); s++) {
-            out.append(String.format(Locale.ROOT, " [State#%d %s", s, a.isAccept(s) ? "(accept)" : ""));
+            Object[] args = new Object[] { s, a.isAccept(s) ? "(accept)" : "" };
+            out.append(Strings.format(" [State#%d %s", args));
             for (int t = 0; t < a.getNumTransitions(s); t++) {
                 Transition transition = new Transition();
                 a.getTransition(s, t, transition);
-                out.append(String.format(Locale.ROOT, " (%05d - %05d => %s)", transition.min, transition.max, transition.dest));
+                out.append(Strings.format(" (%05d - %05d => %s)", transition.min, transition.max, transition.dest));
             }
             out.append("]");
         }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/StringMatcherTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/StringMatcherTests.java
index c624b182cc65..1582cf3404bd 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/StringMatcherTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/StringMatcherTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.List;
-import java.util.Locale;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -209,13 +208,13 @@ public void testInvalidRegexPatterns() {
 
     private void assertMatch(StringMatcher matcher, String str) {
         if (matcher.test(str) == false) {
-            fail(String.format(Locale.ROOT, "Matcher [%s] failed to match [%s] but should", matcher, str));
+            fail(Strings.format("Matcher [%s] failed to match [%s] but should", matcher, str));
         }
     }
 
     private void assertNoMatch(StringMatcher matcher, String str) {
         if (matcher.test(str)) {
-            fail(String.format(Locale.ROOT, "Matcher [%s] matched [%s] but should not", matcher, str));
+            fail(Strings.format("Matcher [%s] matched [%s] but should not", matcher, str));
         }
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/xcontent/XContentUtilsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/xcontent/XContentUtilsTests.java
index 3134e4e1748e..03cb4e109a85 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/xcontent/XContentUtilsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/xcontent/XContentUtilsTests.java
@@ -67,6 +67,11 @@ public void testAddAuthorizationInfoWithServiceAccount() throws IOException {
         assertThat(json, equalTo("{\"authorization\":{\"service_account\":\"" + account + "\"}}"));
     }
 
+    public void testAddAuthorizationInfoWithCorruptData() throws IOException {
+        String json = generateJson(Map.of(AuthenticationField.AUTHENTICATION_KEY, "corrupt"));
+        assertThat(json, equalTo("{}"));
+    }
+
     private String generateJson(Map<String, String> headers) throws IOException {
         try (XContentBuilder builder = JsonXContent.contentBuilder()) {
             builder.startObject();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadataTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadataTests.java
index 2f62452b7b8d..587c2b088cf6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadataTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadataTests.java
@@ -9,14 +9,14 @@
 
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.ilm.OperationMode;
 
 import java.io.IOException;
 import java.util.Map;
 
-public class SnapshotLifecycleMetadataTests extends AbstractXContentSerializingTestCase<SnapshotLifecycleMetadata> {
+public class SnapshotLifecycleMetadataTests extends AbstractChunkedSerializingTestCase<SnapshotLifecycleMetadata> {
     @Override
     protected SnapshotLifecycleMetadata doParseInstance(XContentParser parser) throws IOException {
         return SnapshotLifecycleMetadata.PARSER.apply(parser, null);
@@ -37,6 +37,11 @@ protected SnapshotLifecycleMetadata createTestInstance() {
         );
     }
 
+    @Override
+    protected SnapshotLifecycleMetadata mutateInstance(SnapshotLifecycleMetadata instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<SnapshotLifecycleMetadata> instanceReader() {
         return SnapshotLifecycleMetadata::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadataTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadataTests.java
index d9b587d567a7..d80bfb6950e5 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadataTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadataTests.java
@@ -48,7 +48,7 @@ protected Writeable.Reader<SnapshotLifecyclePolicyMetadata> instanceReader() {
     }
 
     @Override
-    protected SnapshotLifecyclePolicyMetadata mutateInstance(SnapshotLifecyclePolicyMetadata instance) throws IOException {
+    protected SnapshotLifecyclePolicyMetadata mutateInstance(SnapshotLifecyclePolicyMetadata instance) {
         return switch (between(0, 5)) {
             case 0 -> SnapshotLifecyclePolicyMetadata.builder(instance)
                 .setPolicy(randomValueOtherThan(instance.getPolicy(), () -> randomSnapshotLifecyclePolicy(randomAlphaOfLength(10))))
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleStatsTests.java
index e9ee616b557e..156df48e7982 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleStatsTests.java
@@ -53,7 +53,7 @@ protected SnapshotLifecycleStats createTestInstance() {
     }
 
     @Override
-    protected SnapshotLifecycleStats mutateInstance(SnapshotLifecycleStats instance) throws IOException {
+    protected SnapshotLifecycleStats mutateInstance(SnapshotLifecycleStats instance) {
         return randomValueOtherThan(instance, () -> instance.merge(createTestInstance()));
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/spatial/SpatialFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/spatial/SpatialFeatureSetUsageTests.java
index 5313c3b44135..abe14351387c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/spatial/SpatialFeatureSetUsageTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/spatial/SpatialFeatureSetUsageTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.xpack.core.common.stats.EnumCounters;
 import org.elasticsearch.xpack.core.spatial.action.SpatialStatsAction;
 
-import java.io.IOException;
 import java.net.InetAddress;
 import java.util.List;
 
@@ -30,7 +29,7 @@ protected SpatialFeatureSetUsage createTestInstance() {
     }
 
     @Override
-    protected SpatialFeatureSetUsage mutateInstance(SpatialFeatureSetUsage instance) throws IOException {
+    protected SpatialFeatureSetUsage mutateInstance(SpatialFeatureSetUsage instance) {
         return null; // no mutations
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/RestrictedTrustConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/RestrictedTrustConfigTests.java
index 090ad015620f..f6033ff6d929 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/RestrictedTrustConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/RestrictedTrustConfigTests.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.ssl.SslTrustConfig;
 import org.elasticsearch.common.ssl.StoredCertificate;
+import org.elasticsearch.common.ssl.X509Field;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.test.ESTestCase;
@@ -19,6 +20,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Set;
 
 import javax.net.ssl.X509ExtendedTrustManager;
 
@@ -68,7 +70,11 @@ public int hashCode() {
             }
         };
 
-        final RestrictedTrustConfig restrictedTrustConfig = new RestrictedTrustConfig(groupConfigPath, delegate);
+        final RestrictedTrustConfig restrictedTrustConfig = new RestrictedTrustConfig(
+            groupConfigPath,
+            Set.of(X509Field.SAN_OTHERNAME_COMMONNAME),
+            delegate
+        );
         Collection<Path> filesToMonitor = restrictedTrustConfig.getDependentFiles();
         List<Path> expectedPathList = new ArrayList<>(otherFiles);
         expectedPathList.add(groupConfigPath);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/RestrictedTrustManagerTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/RestrictedTrustManagerTests.java
index 1dd71e730c0d..66fb5c7642f2 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/RestrictedTrustManagerTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/RestrictedTrustManagerTests.java
@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.xpack.core.ssl;
 
+import org.elasticsearch.common.ssl.X509Field;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Description;
 import org.hamcrest.TypeSafeMatcher;
@@ -28,6 +29,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
@@ -39,6 +41,7 @@ public class RestrictedTrustManagerTests extends ESTestCase {
     private Map<String, X509Certificate[]> certificates;
     private int numberOfClusters;
     private int numberOfNodes;
+    private List<X509Field> fields;
 
     @Before
     public void readCertificates() throws GeneralSecurityException, IOException {
@@ -92,6 +95,85 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
 
         numberOfClusters = scaledRandomIntBetween(2, 8);
         numberOfNodes = scaledRandomIntBetween(2, 8);
+        fields = randomNonEmptySubsetOf(Set.of(X509Field.SAN_OTHERNAME_COMMONNAME, X509Field.SAN_DNS));
+    }
+
+    public void testTrustsOnlyNameDns() throws Exception {
+        final Path cert = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.crt");
+        baseTrustManager = CertParsingUtils.getTrustManagerFromPEM(List.of(cert));
+        X509Certificate[] certs = CertParsingUtils.readX509Certificates(Collections.singletonList(cert));
+        assertTrue(certs[0].getSubjectAlternativeNames().stream().filter(pair -> (Integer) pair.get(0) == 0).findAny().isEmpty());
+        certificates.put("onlyDns", certs);
+        List<String> validDnsNames = randomNonEmptySubsetOf(
+            List.of("localhost", "localhost.localdomain", "localhost4", "localhost4.localdomain4", "localhost6", "localhost6.localdomain6")
+        );
+        final CertificateTrustRestrictions restrictions = new CertificateTrustRestrictions(validDnsNames);
+        final RestrictedTrustManager trustManager = new RestrictedTrustManager(baseTrustManager, restrictions, Set.of(X509Field.SAN_DNS));
+        assertTrusted(trustManager, "onlyDns");
+    }
+
+    public void testTrustsOnlyNameOther() throws Exception {
+        final Path cert = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/trusted.crt");
+        baseTrustManager = CertParsingUtils.getTrustManagerFromPEM(List.of(cert));
+        X509Certificate[] certs = CertParsingUtils.readX509Certificates(Collections.singletonList(cert));
+        assertTrue(certs[0].getSubjectAlternativeNames().stream().filter(pair -> (Integer) pair.get(0) == 2).findAny().isEmpty());
+        certificates.put("onlyOtherName", certs);
+        final CertificateTrustRestrictions restrictions = new CertificateTrustRestrictions(List.of("node.trusted"));
+        final RestrictedTrustManager trustManager = new RestrictedTrustManager(
+            baseTrustManager,
+            restrictions,
+            Set.of(X509Field.SAN_OTHERNAME_COMMONNAME)
+        );
+        assertTrusted(trustManager, "onlyOtherName");
+    }
+
+    public void testTrustWithVariedFields() throws Exception {
+        final Path cert = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/restricted.trust.crt");
+        baseTrustManager = CertParsingUtils.getTrustManagerFromPEM(List.of(cert));
+        X509Certificate[] certs = CertParsingUtils.readX509Certificates(Collections.singletonList(cert));
+        assertTrue(certs[0].getSubjectAlternativeNames().stream().anyMatch(pair -> (Integer) pair.get(0) == 0)); // othername
+        assertTrue(certs[0].getSubjectAlternativeNames().stream().anyMatch(pair -> (Integer) pair.get(0) == 2)); // dns
+        assertTrue(certs[0].getSubjectAlternativeNames().stream().anyMatch(pair -> (Integer) pair.get(0) == 7)); // ip
+        certificates.put("varied", certs);
+        // othername/common name -> "instance03.cluster02.elasticsearch"
+        // dns -> "search.example.com"
+        // ip -> 50.100.150.200
+        String failureMatchDns = ".*subjectAltName\\.dnsName.*search\\.example\\.com.*does not match.*";
+        String failureMatchCommon = ".*subjectAltName\\.otherName\\.commonName.*instance03\\.cluster02\\.elasticsearch.*does not match.*";
+
+        // instance03.cluster02.elasticsearch -> *.cluster02.elasticsearch
+        CertificateTrustRestrictions restrictions = new CertificateTrustRestrictions(List.of("*.cluster02.elasticsearch"));
+        RestrictedTrustManager trustManager = new RestrictedTrustManager(
+            baseTrustManager,
+            restrictions,
+            Set.of(X509Field.SAN_OTHERNAME_COMMONNAME)
+        );
+        assertTrusted(trustManager, "varied");
+
+        // search.example.com -> *.cluster02.elasticsearch
+        restrictions = new CertificateTrustRestrictions(List.of("*.cluster02.elasticsearch"));
+        trustManager = new RestrictedTrustManager(baseTrustManager, restrictions, Set.of(X509Field.SAN_DNS));
+        assertNotValid(trustManager, "varied", failureMatchDns); // <-- this one
+
+        // search.example.com -> *.example.com
+        restrictions = new CertificateTrustRestrictions(List.of("*.example.com"));
+        trustManager = new RestrictedTrustManager(baseTrustManager, restrictions, Set.of(X509Field.SAN_DNS));
+        assertTrusted(trustManager, "varied");
+
+        // instance03.cluster02.elasticsearch -> *.example.com
+        restrictions = new CertificateTrustRestrictions(List.of("*.example.com"));
+        trustManager = new RestrictedTrustManager(baseTrustManager, restrictions, Set.of(X509Field.SAN_OTHERNAME_COMMONNAME));
+        assertNotValid(trustManager, "varied", failureMatchCommon);
+
+        // instance03.cluster02.elasticsearch or search.example.com -> *.150.200
+        restrictions = new CertificateTrustRestrictions(List.of("*.150.200"));
+        trustManager = new RestrictedTrustManager(
+            baseTrustManager,
+            restrictions,
+            Set.of(X509Field.SAN_DNS, X509Field.SAN_OTHERNAME_COMMONNAME)
+        );
+        assertNotValid(trustManager, "varied", failureMatchDns);
+        assertNotValid(trustManager, "varied", failureMatchCommon);
     }
 
     public void testTrustsExplicitCertificateName() throws Exception {
@@ -101,7 +183,7 @@ public void testTrustsExplicitCertificateName() throws Exception {
             trustedNames.add("node" + node + ".cluster" + trustedCluster + ".elasticsearch");
         }
         final CertificateTrustRestrictions restrictions = new CertificateTrustRestrictions(trustedNames);
-        final RestrictedTrustManager trustManager = new RestrictedTrustManager(baseTrustManager, restrictions);
+        final RestrictedTrustManager trustManager = new RestrictedTrustManager(baseTrustManager, restrictions, Set.copyOf(fields));
         assertSingleClusterIsTrusted(trustedCluster, trustManager, trustedNames);
     }
 
@@ -109,7 +191,7 @@ public void testTrustsWildcardCertificateName() throws Exception {
         final int trustedCluster = randomIntBetween(1, numberOfClusters);
         final List<String> trustedNames = Collections.singletonList("*.cluster" + trustedCluster + ".elasticsearch");
         final CertificateTrustRestrictions restrictions = new CertificateTrustRestrictions(trustedNames);
-        final RestrictedTrustManager trustManager = new RestrictedTrustManager(baseTrustManager, restrictions);
+        final RestrictedTrustManager trustManager = new RestrictedTrustManager(baseTrustManager, restrictions, Set.copyOf(fields));
         assertSingleClusterIsTrusted(trustedCluster, trustManager, trustedNames);
     }
 
@@ -117,7 +199,7 @@ public void testTrustWithRegexCertificateName() throws Exception {
         final int trustedNode = randomIntBetween(1, numberOfNodes);
         final List<String> trustedNames = Collections.singletonList("/node" + trustedNode + ".cluster[0-9].elasticsearch/");
         final CertificateTrustRestrictions restrictions = new CertificateTrustRestrictions(trustedNames);
-        final RestrictedTrustManager trustManager = new RestrictedTrustManager(baseTrustManager, restrictions);
+        final RestrictedTrustManager trustManager = new RestrictedTrustManager(baseTrustManager, restrictions, Set.copyOf(fields));
         for (int cluster = 1; cluster <= numberOfClusters; cluster++) {
             for (int node = 1; node <= numberOfNodes; node++) {
                 if (node == trustedNode) {
@@ -131,7 +213,7 @@ public void testTrustWithRegexCertificateName() throws Exception {
 
     public void testThatDelegateTrustManagerIsRespected() throws Exception {
         final CertificateTrustRestrictions restrictions = new CertificateTrustRestrictions(Collections.singletonList("*.elasticsearch"));
-        final RestrictedTrustManager trustManager = new RestrictedTrustManager(baseTrustManager, restrictions);
+        final RestrictedTrustManager trustManager = new RestrictedTrustManager(baseTrustManager, restrictions, Set.copyOf(fields));
         for (String cert : certificates.keySet()) {
             if (cert.endsWith("/ca")) {
                 assertTrusted(trustManager, cert);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationSettingsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationSettingsTests.java
index 305ad60217a7..a800266a33a1 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationSettingsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationSettingsTests.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.ssl.SslClientAuthenticationMode;
+import org.elasticsearch.common.ssl.X509Field;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Arrays;
@@ -16,6 +17,9 @@
 import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.TrustManagerFactory;
 
+import static org.elasticsearch.common.ssl.SslConfigurationLoader.GLOBAL_DEFAULT_RESTRICTED_TRUST_FIELDS;
+import static org.elasticsearch.test.TestMatchers.throwableWithMessage;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.startsWith;
@@ -73,6 +77,29 @@ public void testParseProtocolsListWithPrefix() {
         assertThat(ssl.supportedProtocols.get(settings), is(Arrays.asList("SSLv3", "SSLv2Hello", "SSLv2")));
     }
 
+    public void testParseTrustRestrictionsListWithPrefix() {
+        final SSLConfigurationSettings ssl = SSLConfigurationSettings.withPrefix("ssl.", true);
+        assertThat(ssl.trustRestrictionsX509Fields.match("ssl.trust_restrictions.x509_fields"), is(true));
+
+        // explicit configuration
+        Settings settings = Settings.builder()
+            .putList("ssl.trust_restrictions.x509_fields", "subjectAltName.otherName.commonName", "subjectAltName.dnsName")
+            .build();
+        assertThat(ssl.trustRestrictionsX509Fields.get(settings), is(Arrays.asList(X509Field.SAN_OTHERNAME_COMMONNAME, X509Field.SAN_DNS)));
+
+        // implicit configuration
+        settings = Settings.builder().build();
+        assertThat(ssl.trustRestrictionsX509Fields.get(settings), is(GLOBAL_DEFAULT_RESTRICTED_TRUST_FIELDS));
+
+        // invalid configuration
+        final Settings invalid = Settings.builder().putList("ssl.trust_restrictions.x509_fields", "foo.bar").build();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ssl.trustRestrictionsX509Fields.get(invalid));
+        assertThat(e.getCause(), throwableWithMessage(containsString("foo.bar is not a supported x509 field for trust restrictions.")));
+        assertThat(e.getCause(), throwableWithMessage(containsString("Recognised values are")));
+        assertThat(e.getCause(), throwableWithMessage(containsString("subjectAltName.otherName.commonName")));
+        assertThat(e.getCause(), throwableWithMessage(containsString("subjectAltName.dnsName")));
+    }
+
     public void testEmptySettingsParsesToDefaults() {
         final SSLConfigurationSettings ssl = SSLConfigurationSettings.withoutPrefix(true);
         final Settings settings = Settings.EMPTY;
@@ -93,6 +120,7 @@ public void testEmptySettingsParsesToDefaults() {
         assertThat(ssl.truststorePassword.exists(settings), is(false));
         assertThat(ssl.truststorePath.get(settings).isPresent(), is(false));
         assertThat(ssl.trustRestrictionsPath.get(settings).isPresent(), is(false));
+        assertThat(ssl.trustRestrictionsX509Fields.get(settings), is(GLOBAL_DEFAULT_RESTRICTED_TRUST_FIELDS));
         assertThat(ssl.verificationMode.get(settings).isPresent(), is(false));
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SslSettingsLoaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SslSettingsLoaderTests.java
index 97114eecc7c0..73ee311979db 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SslSettingsLoaderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SslSettingsLoaderTests.java
@@ -15,18 +15,23 @@
 import org.elasticsearch.common.ssl.KeyStoreUtil;
 import org.elasticsearch.common.ssl.PemKeyConfig;
 import org.elasticsearch.common.ssl.PemTrustConfig;
+import org.elasticsearch.common.ssl.SslClientAuthenticationMode;
 import org.elasticsearch.common.ssl.SslConfiguration;
+import org.elasticsearch.common.ssl.SslConfigurationKeys;
 import org.elasticsearch.common.ssl.SslKeyConfig;
 import org.elasticsearch.common.ssl.SslVerificationMode;
 import org.elasticsearch.common.ssl.StoreKeyConfig;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.RemoteClusterPortSettings;
+import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.junit.Before;
 
 import java.nio.file.Path;
 import java.security.cert.X509Certificate;
 import java.util.Arrays;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 import javax.net.ssl.KeyManager;
@@ -39,6 +44,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.everyItem;
 import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.in;
 import static org.hamcrest.Matchers.instanceOf;
@@ -70,6 +76,63 @@ public void testThatSslConfigurationHasCorrectDefaults() {
         assertThat(globalConfig.supportedProtocols(), not(hasItem("TLSv1")));
     }
 
+    /**
+     * Tests that the Remote Cluster port is configured if enabled and properly uses the default settings.
+     */
+    public void testRemoteClusterPortConfigurationIsInjectedWithDefaults() {
+        assumeTrue("tests Remote Cluster Security 2.0 functionality", TcpTransport.isUntrustedRemoteClusterEnabled());
+        // Remote cluster ssl is always enabled regardless whether the port is enabled
+        Settings testSettings = Settings.builder()
+            .put(RemoteClusterPortSettings.REMOTE_CLUSTER_PORT_ENABLED.getKey(), randomBoolean())
+            .build();
+        Map<String, Settings> settingsMap = SSLService.getSSLSettingsMap(testSettings);
+        assertThat(settingsMap, hasKey(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX));
+        SslConfiguration sslConfiguration = getSslConfiguration(settingsMap.get(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX));
+        assertThat(sslConfiguration.keyConfig(), sameInstance(EmptyKeyConfig.INSTANCE));
+        assertThat(sslConfiguration.trustConfig().getClass().getSimpleName(), is("DefaultJdkTrustConfig"));
+        assertThat(sslConfiguration.supportedProtocols(), equalTo(XPackSettings.DEFAULT_SUPPORTED_PROTOCOLS));
+        assertThat(sslConfiguration.supportedProtocols(), not(hasItem("TLSv1")));
+        assertThat(sslConfiguration.clientAuth(), is(SslClientAuthenticationMode.NONE));
+    }
+
+    /**
+     * Tests that settings are correctly read from the `remote_cluster.ssl.*` settings to set up the {@link SslConfiguration} object used by
+     * all the SSL infrastructure to define per-profile settings. Makes sure to use both regular and secure settings to be sure both are
+     * covered.
+     */
+    public void testRemoteClusterPortConfigurationIsInjectedWithItsSettings() {
+        assumeTrue("tests Remote Cluster Security 2.0 functionality", TcpTransport.isUntrustedRemoteClusterEnabled());
+        final Path path = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks");
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX + SslConfigurationKeys.KEYSTORE_SECURE_PASSWORD, "testnode");
+        // Remote cluster ssl is always enabled regardless whether the port is enabled
+        Settings testSettings = Settings.builder()
+            .put(RemoteClusterPortSettings.REMOTE_CLUSTER_PORT_ENABLED.getKey(), randomBoolean())
+            .put(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX + SslConfigurationKeys.KEYSTORE_PATH, path)
+            .putList(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX + SslConfigurationKeys.PROTOCOLS, "TLSv1.3", "TLSv1.2")
+            .put(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX + SslConfigurationKeys.CLIENT_AUTH, "required")
+            .setSecureSettings(secureSettings)
+            .build();
+        Map<String, Settings> settingsMap = SSLService.getSSLSettingsMap(testSettings);
+        assertThat(settingsMap, hasKey(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX));
+        SslConfiguration sslConfiguration = getSslConfiguration(settingsMap.get(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX));
+        assertThat(sslConfiguration.supportedProtocols(), contains("TLSv1.3", "TLSv1.2"));
+        assertThat(sslConfiguration.clientAuth(), is(SslClientAuthenticationMode.REQUIRED));
+
+        SslKeyConfig keyStore = sslConfiguration.keyConfig();
+        assertThat(keyStore.getDependentFiles(), contains(path));
+        assertThat(keyStore.hasKeyMaterial(), is(true));
+        final X509ExtendedKeyManager keyManager = keyStore.createKeyManager();
+        assertThat(keyManager, notNullValue());
+        assertThat(keyStore.getKeys(), hasSize(3)); // testnode_ec, testnode_rsa, testnode_dsa
+        assertThat(
+            keyStore.getKeys().stream().map(t -> t.v1().getAlgorithm()).collect(Collectors.toUnmodifiableSet()),
+            containsInAnyOrder("RSA", "DSA", "EC")
+        );
+
+        assertCombiningTrustConfigContainsCorrectIssuers(sslConfiguration);
+    }
+
     public void testThatOnlyKeystoreInSettingsSetsTruststoreSettings() {
         final Path path = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks");
         MockSecureSettings secureSettings = new MockSecureSettings();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/TemplateUtilsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/TemplateUtilsTests.java
index 96da45da7443..08b3b91efd95 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/TemplateUtilsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/TemplateUtilsTests.java
@@ -15,7 +15,6 @@
 
 import java.io.IOException;
 import java.util.HashMap;
-import java.util.Locale;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -30,12 +29,12 @@ public class TemplateUtilsTests extends ESTestCase {
 
     public void testLoadTemplate() throws IOException {
         final int version = randomIntBetween(0, 10_000);
-        String resource = String.format(Locale.ROOT, SIMPLE_TEST_TEMPLATE, "test");
+        String resource = Strings.format(SIMPLE_TEST_TEMPLATE, "test");
         String source = TemplateUtils.loadTemplate(resource, String.valueOf(version), "monitoring.template.version");
 
         assertThat(source, notNullValue());
         assertThat(source.length(), greaterThan(0));
-        assertTemplate(XContentHelper.stripWhitespace(source), equalTo(XContentHelper.stripWhitespace(formatted("""
+        assertTemplate(XContentHelper.stripWhitespace(source), equalTo(XContentHelper.stripWhitespace(Strings.format("""
             {
               "index_patterns": ".monitoring-data-%s",
               "mappings": {
@@ -64,7 +63,7 @@ public void testLoadTemplate_GivenTemplateWithVariables() throws IOException {
 
         assertThat(source, notNullValue());
         assertThat(source.length(), greaterThan(0));
-        assertTemplate(XContentHelper.stripWhitespace(source), equalTo(XContentHelper.stripWhitespace(formatted("""
+        assertTemplate(XContentHelper.stripWhitespace(source), equalTo(XContentHelper.stripWhitespace(Strings.format("""
             {
               "index_patterns": ".test-%s",
               "mappings": {
@@ -86,7 +85,7 @@ public void testLoadTemplate_GivenTemplateWithVariables() throws IOException {
     }
 
     public void testLoad() throws IOException {
-        String resource = String.format(Locale.ROOT, SIMPLE_TEST_TEMPLATE, "test");
+        String resource = Strings.format(SIMPLE_TEST_TEMPLATE, "test");
         String source = TemplateUtils.load(resource);
         assertThat(source, notNullValue());
         assertThat(source.length(), greaterThan(0));
@@ -111,7 +110,7 @@ public void testValidateInvalidSource() {
     }
 
     public void testValidate() throws IOException {
-        String resource = String.format(Locale.ROOT, SIMPLE_TEST_TEMPLATE, "test");
+        String resource = Strings.format(SIMPLE_TEST_TEMPLATE, "test");
         TemplateUtils.validate(TemplateUtils.load(resource));
     }
 
@@ -134,7 +133,7 @@ public void testReplaceVariable() {
         final int version = randomIntBetween(0, 100);
         assertTemplate(TemplateUtils.replaceVariable("""
             {"foo-${monitoring.template.version}": "bar-${monitoring.template.version}"}
-            """, "monitoring.template.version", String.valueOf(version)), equalTo(formatted("""
+            """, "monitoring.template.version", String.valueOf(version)), equalTo(Strings.format("""
             {"foo-%s": "bar-%s"}
             """, version, version)));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TermsEnumResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TermsEnumResponseTests.java
index 63785014dfab..a31c44a165cd 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TermsEnumResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TermsEnumResponseTests.java
@@ -82,7 +82,7 @@ public void testToXContent() {
         TermsEnumResponse response = new TermsEnumResponse(terms, 10, 10, 0, new ArrayList<>(), true);
 
         String output = Strings.toString(response);
-        assertEquals(formatted("""
+        assertEquals(Strings.format("""
             {"_shards":{"total":10,"successful":10,"failed":0},"terms":["%s"],"complete":true}""", s), output);
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TransportTermsEnumActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TransportTermsEnumActionTests.java
index 20e88cb69d1c..be2c1728db72 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TransportTermsEnumActionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TransportTermsEnumActionTests.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.termsenum;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
@@ -70,8 +70,8 @@ public void testCCSCheckCompatibility() throws Exception {
         TermsEnumRequest request = new TermsEnumRequest().field("field").timeout(TimeValue.timeValueSeconds(5));
         request.indexFilter(new DummyQueryBuilder() {
             @Override
-            public Version getMinimalSupportedVersion() {
-                return Version.CURRENT;
+            public TransportVersion getMinimalSupportedVersion() {
+                return TransportVersion.CURRENT;
             }
         });
         ExecutionException ex = expectThrows(ExecutionException.class, () -> client().execute(TermsEnumAction.INSTANCE, request).get());
@@ -80,7 +80,9 @@ public Version getMinimalSupportedVersion() {
         assertThat(
             ex.getCause().getCause().getMessage(),
             containsString(
-                "was released first in version " + Version.CURRENT + ", failed compatibility check trying to send it to node with version"
+                "was released first in version "
+                    + TransportVersion.CURRENT
+                    + ", failed compatibility check trying to send it to node with version"
             )
         );
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/textstructure/action/FindTextStructureActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/textstructure/action/FindTextStructureActionRequestTests.java
index 99f61ce77c17..107f03e522b3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/textstructure/action/FindTextStructureActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/textstructure/action/FindTextStructureActionRequestTests.java
@@ -77,6 +77,11 @@ protected FindStructureAction.Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected FindStructureAction.Request mutateInstance(FindStructureAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<FindStructureAction.Request> instanceReader() {
         return FindStructureAction.Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/textstructure/action/FindTextStructureActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/textstructure/action/FindTextStructureActionResponseTests.java
index 8e5742538d9a..31dbfc7dccff 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/textstructure/action/FindTextStructureActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/textstructure/action/FindTextStructureActionResponseTests.java
@@ -17,6 +17,11 @@ protected FindStructureAction.Response createTestInstance() {
         return new FindStructureAction.Response(TextStructureTests.createTestFileStructure());
     }
 
+    @Override
+    protected FindStructureAction.Response mutateInstance(FindStructureAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<FindStructureAction.Response> instanceReader() {
         return FindStructureAction.Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/textstructure/structurefinder/FieldStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/textstructure/structurefinder/FieldStatsTests.java
index 04e56d65aa71..f360f8bbe9fe 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/textstructure/structurefinder/FieldStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/textstructure/structurefinder/FieldStatsTests.java
@@ -22,6 +22,11 @@ protected FieldStats createTestInstance() {
         return createTestFieldStats();
     }
 
+    @Override
+    protected FieldStats mutateInstance(FieldStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     static FieldStats createTestFieldStats() {
 
         long count = randomIntBetween(1, 100000);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/textstructure/structurefinder/TextStructureTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/textstructure/structurefinder/TextStructureTests.java
index 6197a0ea86b0..4132e43a096c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/textstructure/structurefinder/TextStructureTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/textstructure/structurefinder/TextStructureTests.java
@@ -27,6 +27,11 @@ protected TextStructure createTestInstance() {
         return createTestFileStructure();
     }
 
+    @Override
+    protected TextStructure mutateInstance(TextStructure instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static TextStructure createTestFileStructure() {
 
         TextStructure.Format format = randomFrom(EnumSet.allOf(TextStructure.Format.class));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedAggregationBuilder.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedAggregationBuilder.java
index a96d662c8d6c..dc5e863346b5 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedAggregationBuilder.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedAggregationBuilder.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.core.transform;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.DeprecationCategory;
@@ -103,7 +103,7 @@ public static MockDeprecatedAggregationBuilder fromXContent(XContentParser p) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedQueryBuilder.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedQueryBuilder.java
index c5c438964144..5f55c3bc40c9 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedQueryBuilder.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedQueryBuilder.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.xpack.core.transform;
 
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -86,7 +86,7 @@ protected int doHashCode() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsageTests.java
index 5747dfd4dfa7..9e3fc6f37e48 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsageTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/TransformFeatureSetUsageTests.java
@@ -30,6 +30,11 @@ protected TransformFeatureSetUsage createTestInstance() {
         return new TransformFeatureSetUsage(transformCountByState, transformCountByFeature, accumulatedStats);
     }
 
+    @Override
+    protected TransformFeatureSetUsage mutateInstance(TransformFeatureSetUsage instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<TransformFeatureSetUsage> instanceReader() {
         return TransformFeatureSetUsage::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/DeleteTransformActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/DeleteTransformActionRequestTests.java
index 2b5beea82f14..e0fcd471e28e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/DeleteTransformActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/DeleteTransformActionRequestTests.java
@@ -12,8 +12,6 @@
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.transform.action.DeleteTransformAction.Request;
 
-import java.io.IOException;
-
 public class DeleteTransformActionRequestTests extends AbstractWireSerializingTestCase<Request> {
     @Override
     protected Request createTestInstance() {
@@ -26,7 +24,7 @@ protected Writeable.Reader<Request> instanceReader() {
     }
 
     @Override
-    protected Request mutateInstance(Request instance) throws IOException {
+    protected Request mutateInstance(Request instance) {
         String id = instance.getId();
         boolean force = instance.isForce();
         TimeValue timeout = instance.timeout();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointActionRequestTests.java
index 48704068e3d8..0bda09772da6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointActionRequestTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.transform.action.GetCheckpointAction.Request;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -41,7 +40,7 @@ protected Reader<Request> instanceReader() {
     }
 
     @Override
-    protected Request mutateInstance(Request instance) throws IOException {
+    protected Request mutateInstance(Request instance) {
         List<String> indices = instance.indices() != null ? new ArrayList<>(Arrays.asList(instance.indices())) : new ArrayList<>();
         IndicesOptions indicesOptions = instance.indicesOptions();
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointActionResponseTests.java
index fdb7a59a4792..4f15d31460ec 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointActionResponseTests.java
@@ -41,4 +41,9 @@ protected Reader<Response> instanceReader() {
     protected Response createTestInstance() {
         return randomCheckpointResponse();
     }
+
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointNodeActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointNodeActionRequestTests.java
index fd3573f1acae..ac5d1e385944 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointNodeActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointNodeActionRequestTests.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.transform.action.GetCheckpointNodeAction.Request;
 
-import java.io.IOException;
 import java.util.HashSet;
 import java.util.Set;
 
@@ -39,7 +38,7 @@ protected Request createTestInstance() {
     }
 
     @Override
-    protected Request mutateInstance(Request instance) throws IOException {
+    protected Request mutateInstance(Request instance) {
 
         switch (random().nextInt(1)) {
             case 0 -> {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointNodeActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointNodeActionResponseTests.java
index f189a4f0faae..fdb83c6d321b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointNodeActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointNodeActionResponseTests.java
@@ -41,4 +41,9 @@ protected Reader<Response> instanceReader() {
     protected Response createTestInstance() {
         return randomNodeCheckpointResponse();
     }
+
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetTransformActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetTransformActionRequestTests.java
index 4d3cffd10235..3d564f0cf2fa 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetTransformActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetTransformActionRequestTests.java
@@ -22,6 +22,11 @@ protected Request createTestInstance() {
         return new Request(randomAlphaOfLengthBetween(1, 20));
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetTransformActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetTransformActionResponseTests.java
index 6aca03a0d5fd..0bb92737d6e9 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetTransformActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetTransformActionResponseTests.java
@@ -145,6 +145,11 @@ protected Response createTestInstance() {
         return randomTransformResponse();
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetTransformStatsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetTransformStatsActionRequestTests.java
index 7c320c00f0d2..7e835dac0df5 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetTransformStatsActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetTransformStatsActionRequestTests.java
@@ -9,17 +9,42 @@
 
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.transform.action.GetTransformStatsAction.Request;
 
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+
 public class GetTransformStatsActionRequestTests extends AbstractWireSerializingTestCase<Request> {
     @Override
     protected Request createTestInstance() {
-        return new Request(randomBoolean() ? randomAlphaOfLengthBetween(1, 20) : randomBoolean() ? Metadata.ALL : null);
+        return new Request(
+            randomBoolean() ? randomAlphaOfLengthBetween(1, 20) : randomBoolean() ? Metadata.ALL : null,
+            randomBoolean() ? TimeValue.parseTimeValue(randomTimeValue(), "timeout") : null
+        );
+    }
+
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
     }
 
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
     }
+
+    public void testCreateTask() {
+        Request request = new Request("some-transform", null);
+        Task task = request.createTask(123, "type", "action", TaskId.EMPTY_TASK_ID, Map.of());
+        assertThat(task, is(instanceOf(CancellableTask.class)));
+        assertThat(task.getDescription(), is(equalTo("get_transform_stats[some-transform]")));
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetTransformStatsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetTransformStatsActionResponseTests.java
index bf33b5b40001..ce96896e8854 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetTransformStatsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetTransformStatsActionResponseTests.java
@@ -41,6 +41,11 @@ protected Response createTestInstance() {
         return randomTransformStatsResponse();
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformActionRequestTests.java
index 94225a71fe9b..09bd3478857a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformActionRequestTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
@@ -70,6 +71,11 @@ protected Request createTestInstance() {
         return new Request(config, TimeValue.parseTimeValue(randomTimeValue(), "timeout"));
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testParsingOverwritesIdField() throws IOException {
         testParsingOverwrites("", """
             "dest": {"index": "bar","pipeline": "baz"},""", "transform-preview", "bar", "baz");
@@ -95,7 +101,7 @@ private void testParsingOverwrites(
         String expectedDestIndex,
         String expectedDestPipeline
     ) throws IOException {
-        BytesArray json = new BytesArray(formatted("""
+        BytesArray json = new BytesArray(Strings.format("""
             {
               %s
               "source": {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformsActionResponseTests.java
index 0fccc70fe2ba..5e570abafb7e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformsActionResponseTests.java
@@ -45,6 +45,11 @@ protected Response createTestInstance() {
         return randomPreviewResponse();
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected boolean supportsUnknownFields() {
         return false;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformsActionResponseWireTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformsActionResponseWireTests.java
index 576fe003d401..cf8808368fd5 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformsActionResponseWireTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformsActionResponseWireTests.java
@@ -17,6 +17,11 @@ protected Response createTestInstance() {
         return PreviewTransformsActionResponseTests.randomPreviewResponse();
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PutTransformActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PutTransformActionRequestTests.java
index 6ef4dcf2539e..51f070a19dd6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PutTransformActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PutTransformActionRequestTests.java
@@ -14,8 +14,6 @@
 import org.elasticsearch.xpack.core.transform.transforms.TransformConfigTests;
 import org.junit.Before;
 
-import java.io.IOException;
-
 public class PutTransformActionRequestTests extends AbstractWireSerializingTransformTestCase<Request> {
     private String transformId;
 
@@ -36,7 +34,7 @@ protected Request createTestInstance() {
     }
 
     @Override
-    protected Request mutateInstance(Request instance) throws IOException {
+    protected Request mutateInstance(Request instance) {
         TransformConfig config = instance.getConfig();
         boolean deferValidation = instance.isDeferValidation();
         TimeValue timeout = instance.timeout();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/ScheduleNowTransformActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/ScheduleNowTransformActionRequestTests.java
new file mode 100644
index 000000000000..e98e14e341cf
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/ScheduleNowTransformActionRequestTests.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.transform.action;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.transform.action.ScheduleNowTransformAction.Request;
+
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class ScheduleNowTransformActionRequestTests extends AbstractWireSerializingTestCase<Request> {
+
+    @Override
+    protected Request createTestInstance() {
+        return new Request(randomAlphaOfLengthBetween(1, 20), TimeValue.parseTimeValue(randomTimeValue(), "timeout"));
+    }
+
+    @Override
+    protected Writeable.Reader<Request> instanceReader() {
+        return Request::new;
+    }
+
+    @Override
+    protected Request mutateInstance(Request instance) {
+        String id = instance.getId();
+        TimeValue timeout = instance.getTimeout();
+
+        switch (between(0, 1)) {
+            case 0 -> id += randomAlphaOfLengthBetween(1, 5);
+            case 1 -> timeout = new TimeValue(timeout.duration() + randomLongBetween(1, 5), timeout.timeUnit());
+            default -> throw new AssertionError("Illegal randomization branch");
+        }
+
+        return new ScheduleNowTransformAction.Request(id, timeout);
+    }
+
+    public void testValidationSuccess() {
+        Request request = new Request("id", TimeValue.ZERO);
+        assertThat(request.validate(), is(nullValue()));
+    }
+
+    public void testValidationFailure() {
+        Request request = new Request("_all", TimeValue.ZERO);
+        ActionRequestValidationException e = request.validate();
+        assertThat(e, is(notNullValue()));
+        assertThat(e.validationErrors(), contains("_schedule_now API does not support _all wildcard"));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/ScheduleNowTransformActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/ScheduleNowTransformActionResponseTests.java
new file mode 100644
index 000000000000..07d36604c4a8
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/ScheduleNowTransformActionResponseTests.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.transform.action;
+
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.xpack.core.transform.action.ScheduleNowTransformAction.Response;
+
+public class ScheduleNowTransformActionResponseTests extends AbstractWireSerializingTransformTestCase<Response> {
+
+    @Override
+    protected Response createTestInstance() {
+        return new Response(randomBoolean());
+    }
+
+    @Override
+    protected Reader<Response> instanceReader() {
+        return Response::new;
+    }
+
+    @Override
+    protected Response mutateInstance(Response instance) {
+        boolean acknowledged = instance.isAcknowledged();
+        return new Response(acknowledged == false);
+    }
+
+    public void testResponseTrue() {
+        assertTrue(Response.TRUE.isAcknowledged());
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/StartTransformActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/StartTransformActionRequestTests.java
index 09010bf1102b..85cd9b34827b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/StartTransformActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/StartTransformActionRequestTests.java
@@ -12,12 +12,17 @@
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.transform.action.StartTransformAction.Request;
 
-import java.io.IOException;
+import java.time.Duration;
+import java.time.Instant;
 
 public class StartTransformActionRequestTests extends AbstractWireSerializingTestCase<Request> {
     @Override
     protected Request createTestInstance() {
-        return new Request(randomAlphaOfLengthBetween(1, 20), TimeValue.parseTimeValue(randomTimeValue(), "timeout"));
+        return new Request(
+            randomAlphaOfLengthBetween(1, 20),
+            randomBoolean() ? Instant.ofEpochMilli(randomNonNegativeLong()) : null,
+            TimeValue.parseTimeValue(randomTimeValue(), "timeout")
+        );
     }
 
     @Override
@@ -26,16 +31,18 @@ protected Writeable.Reader<Request> instanceReader() {
     }
 
     @Override
-    protected Request mutateInstance(Request instance) throws IOException {
+    protected Request mutateInstance(Request instance) {
         String id = instance.getId();
+        Instant from = instance.from();
         TimeValue timeout = instance.timeout();
 
-        switch (between(0, 1)) {
+        switch (between(0, 2)) {
             case 0 -> id += randomAlphaOfLengthBetween(1, 5);
-            case 1 -> timeout = new TimeValue(timeout.duration() + randomLongBetween(1, 5), timeout.timeUnit());
+            case 1 -> from = from != null ? from.plus(Duration.ofDays(1)) : Instant.ofEpochMilli(randomNonNegativeLong());
+            case 2 -> timeout = new TimeValue(timeout.duration() + randomLongBetween(1, 5), timeout.timeUnit());
             default -> throw new AssertionError("Illegal randomization branch");
         }
 
-        return new Request(id, timeout);
+        return new Request(id, from, timeout);
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/StartTransformActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/StartTransformActionResponseTests.java
index dc39fcff78c1..a1cfd1bc9d79 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/StartTransformActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/StartTransformActionResponseTests.java
@@ -17,6 +17,11 @@ protected Response createTestInstance() {
         return new Response(randomBoolean());
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/StopTransformActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/StopTransformActionRequestTests.java
index e90afaadca24..4797b736ba88 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/StopTransformActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/StopTransformActionRequestTests.java
@@ -39,6 +39,11 @@ protected Request createTestInstance() {
         return request;
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/StopTransformActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/StopTransformActionResponseTests.java
index ea390a4c5b25..4304d22fee15 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/StopTransformActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/StopTransformActionResponseTests.java
@@ -17,6 +17,11 @@ protected Response createTestInstance() {
         return new Response(randomBoolean());
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformActionRequestTests.java
index 8a5364de28b4..555383a9fdc0 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformActionRequestTests.java
@@ -13,8 +13,6 @@
 import org.elasticsearch.xpack.core.transform.transforms.TransformConfigTests;
 import org.elasticsearch.xpack.core.transform.transforms.TransformConfigUpdate;
 
-import java.io.IOException;
-
 import static org.elasticsearch.xpack.core.transform.transforms.TransformConfigUpdateTests.randomTransformConfigUpdate;
 
 public class UpdateTransformActionRequestTests extends AbstractWireSerializingTransformTestCase<Request> {
@@ -39,7 +37,7 @@ protected Request createTestInstance() {
     }
 
     @Override
-    protected Request mutateInstance(Request instance) throws IOException {
+    protected Request mutateInstance(Request instance) {
         String id = instance.getId();
         TransformConfigUpdate update = instance.getUpdate();
         boolean deferValidation = instance.isDeferValidation();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java
index f2528afd7f26..ff1963c4ebb2 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java
@@ -23,6 +23,11 @@ protected Response createTestInstance() {
         return new Response(TransformConfigTests.randomTransformConfigWithoutHeaders());
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<Response> instanceReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpgradeTransformsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpgradeTransformsActionRequestTests.java
index fcf130ae4358..27d36f4e0f0d 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpgradeTransformsActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpgradeTransformsActionRequestTests.java
@@ -12,8 +12,6 @@
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.transform.action.UpgradeTransformsAction.Request;
 
-import java.io.IOException;
-
 public class UpgradeTransformsActionRequestTests extends AbstractWireSerializingTestCase<Request> {
 
     @Override
@@ -27,7 +25,7 @@ protected Request createTestInstance() {
     }
 
     @Override
-    protected Request mutateInstance(Request instance) throws IOException {
+    protected Request mutateInstance(Request instance) {
         boolean dryRun = instance.isDryRun();
         TimeValue timeout = instance.timeout();
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpgradeTransformsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpgradeTransformsActionResponseTests.java
index 5bd856720f2b..a6a668a48f48 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpgradeTransformsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpgradeTransformsActionResponseTests.java
@@ -26,4 +26,9 @@ protected Response createTestInstance() {
         return randomUpgradeResponse();
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/ValidateTransformActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/ValidateTransformActionRequestTests.java
index a92be05c5d96..c780d2bfc8a7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/ValidateTransformActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/ValidateTransformActionRequestTests.java
@@ -12,8 +12,6 @@
 import org.elasticsearch.xpack.core.transform.action.ValidateTransformAction.Request;
 import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
 
-import java.io.IOException;
-
 import static org.elasticsearch.xpack.core.transform.transforms.TransformConfigTests.randomTransformConfig;
 
 public class ValidateTransformActionRequestTests extends AbstractWireSerializingTransformTestCase<Request> {
@@ -29,7 +27,7 @@ protected Reader<Request> instanceReader() {
     }
 
     @Override
-    protected Request mutateInstance(Request instance) throws IOException {
+    protected Request mutateInstance(Request instance) {
         TransformConfig config = instance.getConfig();
         boolean deferValidation = instance.isDeferValidation();
         TimeValue timeout = instance.timeout();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/ValidateTransformActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/ValidateTransformActionResponseTests.java
index 82ce31806eba..91bbe344f186 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/ValidateTransformActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/ValidateTransformActionResponseTests.java
@@ -22,6 +22,11 @@ protected Response createTestInstance() {
         return new Response(randomDestIndexMappings());
     }
 
+    @Override
+    protected Response mutateInstance(Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     private Map<String, String> randomDestIndexMappings() {
         return randomList(1, 20, () -> randomAlphaOfLengthBetween(1, 20)).stream()
             .distinct()
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/DestConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/DestConfigTests.java
index 8776e700b5e8..5429c057e560 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/DestConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/DestConfigTests.java
@@ -49,6 +49,11 @@ protected DestConfig createTestInstance() {
         return randomDestConfig();
     }
 
+    @Override
+    protected DestConfig mutateInstance(DestConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<DestConfig> instanceReader() {
         return DestConfig::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/NodeAttributeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/NodeAttributeTests.java
index a1a6cf11af7c..7a5ef8f28b02 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/NodeAttributeTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/NodeAttributeTests.java
@@ -37,6 +37,11 @@ protected NodeAttributes createTestInstance() {
         return randomNodeAttributes();
     }
 
+    @Override
+    protected NodeAttributes mutateInstance(NodeAttributes instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<NodeAttributes> instanceReader() {
         return NodeAttributes::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/QueryConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/QueryConfigTests.java
index 223e0eb98ea7..24e33eaa167e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/QueryConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/QueryConfigTests.java
@@ -79,6 +79,11 @@ protected QueryConfig createTestInstance() {
         return lenient ? randomBoolean() ? randomQueryConfig() : randomInvalidQueryConfig() : randomQueryConfig();
     }
 
+    @Override
+    protected QueryConfig mutateInstance(QueryConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<QueryConfig> instanceReader() {
         return QueryConfig::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java
index 18594bab4d71..62b9e2e48a90 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java
@@ -83,6 +83,11 @@ protected SettingsConfig createTestInstance() {
         return randomSettingsConfig();
     }
 
+    @Override
+    protected SettingsConfig mutateInstance(SettingsConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<SettingsConfig> instanceReader() {
         return SettingsConfig::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java
index ebba7b5b6518..a22c2231985e 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java
@@ -66,6 +66,11 @@ protected SourceConfig createTestInstance() {
         return lenient ? randomBoolean() ? randomSourceConfig() : randomInvalidSourceConfig() : randomSourceConfig();
     }
 
+    @Override
+    protected SourceConfig mutateInstance(SourceConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected boolean supportsUnknownFields() {
         return lenient;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java
index 0737dde81928..4966bde80e93 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java
@@ -31,6 +31,11 @@ protected TimeRetentionPolicyConfig createTestInstance() {
         return randomTimeRetentionPolicyConfig();
     }
 
+    @Override
+    protected TimeRetentionPolicyConfig mutateInstance(TimeRetentionPolicyConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<TimeRetentionPolicyConfig> instanceReader() {
         return TimeRetentionPolicyConfig::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeSyncConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeSyncConfigTests.java
index 1a9b930c0c0d..e9eab5699ec8 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeSyncConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeSyncConfigTests.java
@@ -32,6 +32,11 @@ protected TimeSyncConfig createTestInstance() {
         return randomTimeSyncConfig();
     }
 
+    @Override
+    protected TimeSyncConfig mutateInstance(TimeSyncConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<TimeSyncConfig> instanceReader() {
         return TimeSyncConfig::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointStatsTests.java
index b7e707b9f7fb..bba1656c3606 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointStatsTests.java
@@ -26,6 +26,11 @@ protected TransformCheckpointStats createTestInstance() {
         return randomTransformCheckpointStats();
     }
 
+    @Override
+    protected TransformCheckpointStats mutateInstance(TransformCheckpointStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<TransformCheckpointStats> instanceReader() {
         return TransformCheckpointStats::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointTests.java
index d7b182008634..5e8e0d0cdb56 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointTests.java
@@ -53,6 +53,11 @@ protected TransformCheckpoint createTestInstance() {
         return randomTransformCheckpoint();
     }
 
+    @Override
+    protected TransformCheckpoint mutateInstance(TransformCheckpoint instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<TransformCheckpoint> instanceReader() {
         return TransformCheckpoint::new;
@@ -112,7 +117,18 @@ public void testMatches() throws IOException {
 
     public void testEmpty() {
         assertTrue(TransformCheckpoint.EMPTY.isEmpty());
+        assertTrue(new TransformCheckpoint("_empty", 123L, -1, Collections.emptyMap(), 456L).isEmpty());
         assertFalse(new TransformCheckpoint("some_id", 0L, -1, Collections.emptyMap(), 0L).isEmpty());
+        assertFalse(new TransformCheckpoint("some_id", 0L, 0, Collections.emptyMap(), 0L).isEmpty());
+        assertFalse(new TransformCheckpoint("some_id", 0L, 1, Collections.emptyMap(), 0L).isEmpty());
+    }
+
+    public void testTransient() {
+        assertTrue(TransformCheckpoint.EMPTY.isTransient());
+        assertTrue(new TransformCheckpoint("_empty", 123L, -1, Collections.emptyMap(), 456L).isTransient());
+        assertTrue(new TransformCheckpoint("some_id", 0L, -1, Collections.emptyMap(), 0L).isTransient());
+        assertFalse(new TransformCheckpoint("some_id", 0L, 0, Collections.emptyMap(), 0L).isTransient());
+        assertFalse(new TransformCheckpoint("some_id", 0L, 1, Collections.emptyMap(), 0L).isTransient());
     }
 
     public void testGetBehind() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointingInfoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointingInfoTests.java
index 9af2d7db8a55..ee374cb46d87 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointingInfoTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointingInfoTests.java
@@ -29,6 +29,11 @@ protected TransformCheckpointingInfo createTestInstance() {
         return randomTransformCheckpointingInfo();
     }
 
+    @Override
+    protected TransformCheckpointingInfo mutateInstance(TransformCheckpointingInfo instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<TransformCheckpointingInfo> instanceReader() {
         return TransformCheckpointingInfo::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java
index 01515ce4d5fb..133eb09a1f7f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java
@@ -259,6 +259,11 @@ protected TransformConfig createTestInstance() {
         return runWithHeaders ? randomTransformConfig(transformId) : randomTransformConfigWithoutHeaders(transformId);
     }
 
+    @Override
+    protected TransformConfig mutateInstance(TransformConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<TransformConfig> instanceReader() {
         return TransformConfig::new;
@@ -402,7 +407,7 @@ public void testPreventHeaderInjection() {
     }
 
     public void testPreventCreateTimeInjection() {
-        String pivotTransform = formatted("""
+        String pivotTransform = Strings.format("""
             {
               "create_time": %s,
               "source": {
@@ -565,7 +570,7 @@ public void testSetIdInBody() throws IOException {
     }
 
     public void testRewriteForUpdate() throws IOException {
-        String pivotTransform = formatted("""
+        String pivotTransform = Strings.format("""
             {
               "id": "body_id",
               "source": {
@@ -608,7 +613,7 @@ public void testRewriteForUpdate() throws IOException {
     }
 
     public void testRewriteForUpdateAlignCheckpoints() throws IOException {
-        String pivotTransform = formatted("""
+        String pivotTransform = Strings.format("""
             {
               "id": "body_id",
               "source": {
@@ -652,7 +657,7 @@ public void testRewriteForUpdateAlignCheckpoints() throws IOException {
     }
 
     public void testRewriteForUpdateMaxPageSizeSearchConflicting() throws IOException {
-        String pivotTransform = formatted("""
+        String pivotTransform = Strings.format("""
             {
               "id": "body_id",
               "source": {
@@ -695,7 +700,7 @@ public void testRewriteForUpdateMaxPageSizeSearchConflicting() throws IOExceptio
     }
 
     public void testRewriteForBWCOfDateNormalization() throws IOException {
-        String pivotTransform = formatted("""
+        String pivotTransform = Strings.format("""
             {
               "id": "body_id",
               "source": {
@@ -822,7 +827,7 @@ public void testGetAdditionalSourceDestValidations_WithRuntimeMappings() throws
     }
 
     public void testGroupByStayInOrder() throws IOException {
-        String json = formatted("""
+        String json = Strings.format("""
             {
               "id": "%s",
               "source": {
@@ -958,7 +963,7 @@ public void testCheckForDeprecations() {
     }
 
     public void testSerializingMetadataPreservesOrder() throws IOException {
-        String json = formatted("""
+        String json = Strings.format("""
             {
               "id": "%s",
               "_meta": {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java
index 92660ed1a3ff..c0858574be98 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java
@@ -54,6 +54,11 @@ protected TransformConfigUpdate createTestInstance() {
         return randomTransformConfigUpdate();
     }
 
+    @Override
+    protected TransformConfigUpdate mutateInstance(TransformConfigUpdate instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<TransformConfigUpdate> instanceReader() {
         return TransformConfigUpdate::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformDestIndexSettingsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformDestIndexSettingsTests.java
index c4b9ff794fc4..0506500c654b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformDestIndexSettingsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformDestIndexSettingsTests.java
@@ -74,4 +74,9 @@ protected Reader<TransformDestIndexSettings> instanceReader() {
     protected TransformDestIndexSettings createTestInstance() {
         return randomDestIndexSettings();
     }
+
+    @Override
+    protected TransformDestIndexSettings mutateInstance(TransformDestIndexSettings instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformHealthIssueTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformHealthIssueTests.java
index 9ea715e2adff..487de07bf096 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformHealthIssueTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformHealthIssueTests.java
@@ -32,4 +32,9 @@ protected Writeable.Reader<TransformHealthIssue> instanceReader() {
     protected TransformHealthIssue createTestInstance() {
         return randomTransformHealthIssue();
     }
+
+    @Override
+    protected TransformHealthIssue mutateInstance(TransformHealthIssue instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformHealthTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformHealthTests.java
index cc26b5f2c07b..da887db8e4fa 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformHealthTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformHealthTests.java
@@ -29,4 +29,9 @@ protected Writeable.Reader<TransformHealth> instanceReader() {
     protected TransformHealth createTestInstance() {
         return randomTransformHealth();
     }
+
+    @Override
+    protected TransformHealth mutateInstance(TransformHealth instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerPositionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerPositionTests.java
index 04cc33587d5a..c99bb2fcba6c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerPositionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerPositionTests.java
@@ -27,6 +27,11 @@ protected TransformIndexerPosition createTestInstance() {
         return randomTransformIndexerPosition();
     }
 
+    @Override
+    protected TransformIndexerPosition mutateInstance(TransformIndexerPosition instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<TransformIndexerPosition> instanceReader() {
         return TransformIndexerPosition::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStatsTests.java
index a47445a1ca6b..98399286229a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStatsTests.java
@@ -52,6 +52,11 @@ protected TransformIndexerStats createTestInstance() {
         return randomStats();
     }
 
+    @Override
+    protected TransformIndexerStats mutateInstance(TransformIndexerStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<TransformIndexerStats> instanceReader() {
         return TransformIndexerStats::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformProgressTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformProgressTests.java
index 95c6c5e80728..19048f84e361 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformProgressTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformProgressTests.java
@@ -38,6 +38,11 @@ protected TransformProgress createTestInstance() {
         return randomTransformProgress();
     }
 
+    @Override
+    protected TransformProgress mutateInstance(TransformProgress instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<TransformProgress> instanceReader() {
         return TransformProgress::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStateTests.java
index ccb0156b9e84..1ce30f0166d4 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStateTests.java
@@ -46,6 +46,11 @@ protected TransformState createTestInstance() {
         return randomTransformState();
     }
 
+    @Override
+    protected TransformState mutateInstance(TransformState instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<TransformState> instanceReader() {
         return TransformState::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java
index bccd89f36fe8..c684a566a39a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java
@@ -29,6 +29,11 @@ protected TransformStats createTestInstance() {
         return randomTransformStats();
     }
 
+    @Override
+    protected TransformStats mutateInstance(TransformStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<TransformStats> instanceReader() {
         return TransformStats::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStoredDocTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStoredDocTests.java
index 83f80c4aa876..671dc6f0fa69 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStoredDocTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStoredDocTests.java
@@ -47,6 +47,11 @@ protected TransformStoredDoc createTestInstance() {
         return randomTransformStoredDoc();
     }
 
+    @Override
+    protected TransformStoredDoc mutateInstance(TransformStoredDoc instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<TransformStoredDoc> instanceReader() {
         return TransformStoredDoc::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformTaskParamsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformTaskParamsTests.java
index d06887209625..fa250bff6058 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformTaskParamsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformTaskParamsTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase;
 
 import java.io.IOException;
+import java.time.Instant;
 
 public class TransformTaskParamsTests extends AbstractSerializingTransformTestCase<TransformTaskParams> {
 
@@ -21,6 +22,7 @@ private static TransformTaskParams randomTransformTaskParams() {
         return new TransformTaskParams(
             randomAlphaOfLengthBetween(1, 10),
             randomBoolean() ? VersionUtils.randomVersion(random()) : null,
+            randomBoolean() ? Instant.ofEpochMilli(randomLongBetween(0, 1_000_000_000_000L)) : null,
             randomBoolean() ? TimeValue.timeValueSeconds(randomLongBetween(1, 24 * 60 * 60)) : null,
             randomBoolean()
         );
@@ -36,6 +38,11 @@ protected TransformTaskParams createTestInstance() {
         return randomTransformTaskParams();
     }
 
+    @Override
+    protected TransformTaskParams mutateInstance(TransformTaskParams instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<TransformTaskParams> instanceReader() {
         return TransformTaskParams::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformTests.java
index ae67ab50e1c7..b4c95e39dfea 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase;
 
 import java.io.IOException;
+import java.time.Instant;
 
 public class TransformTests extends AbstractSerializingTransformTestCase<TransformTaskParams> {
 
@@ -27,11 +28,17 @@ protected TransformTaskParams createTestInstance() {
         return new TransformTaskParams(
             randomAlphaOfLength(10),
             randomBoolean() ? null : Version.CURRENT,
+            randomBoolean() ? Instant.ofEpochMilli(randomLongBetween(0, 1_000_000_000_000L)) : null,
             randomBoolean() ? null : TimeValue.timeValueMillis(randomIntBetween(1_000, 3_600_000)),
             randomBoolean()
         );
     }
 
+    @Override
+    protected TransformTaskParams mutateInstance(TransformTaskParams instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<TransformTaskParams> instanceReader() {
         return TransformTaskParams::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java
index 731a48e45cca..d2718f2c47ac 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java
@@ -43,6 +43,11 @@ protected LatestConfig createTestInstance() {
         return randomLatestConfig();
     }
 
+    @Override
+    protected LatestConfig mutateInstance(LatestConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<LatestConfig> instanceReader() {
         return LatestConfig::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/AggregationConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/AggregationConfigTests.java
index 357564dc3357..bbebcfe5183a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/AggregationConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/AggregationConfigTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.Writeable.Reader;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
@@ -88,6 +89,11 @@ protected AggregationConfig createTestInstance() {
         return lenient ? randomBoolean() ? randomAggregationConfig() : randomInvalidAggregationConfig() : randomAggregationConfig();
     }
 
+    @Override
+    protected AggregationConfig mutateInstance(AggregationConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<AggregationConfig> instanceReader() {
         return AggregationConfig::new;
@@ -134,7 +140,7 @@ public void testFailOnStrictPassOnLenient() throws IOException {
     }
 
     public void testDeprecation() throws IOException {
-        String source = formatted("""
+        String source = Strings.format("""
             {"dep_agg": {"%s" : {}}}
             """, MockDeprecatedAggregationBuilder.NAME);
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/DateHistogramGroupSourceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/DateHistogramGroupSourceTests.java
index f8973345c503..d7c9544b07b6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/DateHistogramGroupSourceTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/DateHistogramGroupSourceTests.java
@@ -8,11 +8,13 @@
 package org.elasticsearch.xpack.core.transform.transforms.pivot;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.common.Rounding;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable.Reader;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.DateFormatters;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
 import org.elasticsearch.test.VersionUtils;
@@ -23,6 +25,7 @@
 import java.time.temporal.TemporalAccessor;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
 
 public class DateHistogramGroupSourceTests extends AbstractXContentSerializingTestCase<DateHistogramGroupSource> {
 
@@ -58,6 +61,7 @@ public static DateHistogramGroupSource randomDateHistogramGroupSource(Version ve
             field = fieldPrefix + randomAlphaOfLengthBetween(1, 20);
         }
         boolean missingBucket = version.onOrAfter(Version.V_7_10_0) ? randomBoolean() : false;
+        Long offset = version.onOrAfter(Version.V_8_7_0) ? randomOffset() : null;
 
         DateHistogramGroupSource dateHistogramGroupSource;
         if (randomBoolean()) {
@@ -66,7 +70,8 @@ public static DateHistogramGroupSource randomDateHistogramGroupSource(Version ve
                 scriptConfig,
                 missingBucket,
                 new DateHistogramGroupSource.FixedInterval(new DateHistogramInterval(randomTimeValue(1, 100, "d", "h", "ms", "s", "m"))),
-                randomBoolean() ? randomZone() : null
+                randomBoolean() ? randomZone() : null,
+                randomBoolean() ? offset : null
             );
         } else {
             dateHistogramGroupSource = new DateHistogramGroupSource(
@@ -76,7 +81,8 @@ public static DateHistogramGroupSource randomDateHistogramGroupSource(Version ve
                 new DateHistogramGroupSource.CalendarInterval(
                     new DateHistogramInterval(randomTimeValue(1, 1, "m", "h", "d", "w", "M", "q", "y"))
                 ),
-                randomBoolean() ? randomZone() : null
+                randomBoolean() ? randomZone() : null,
+                randomBoolean() ? offset : null
             );
         }
 
@@ -110,63 +116,171 @@ protected DateHistogramGroupSource createTestInstance() {
         return randomDateHistogramGroupSource();
     }
 
+    @Override
+    protected DateHistogramGroupSource mutateInstance(DateHistogramGroupSource instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<DateHistogramGroupSource> instanceReader() {
         return DateHistogramGroupSource::new;
     }
 
+    public void testOffset() {
+        {
+            DateHistogramGroupSource dateHistogramGroupSource = new DateHistogramGroupSource(
+                null,
+                null,
+                false,
+                new DateHistogramGroupSource.FixedInterval(new DateHistogramInterval("1d")),
+                null,
+                null
+            );
+            assertThat(dateHistogramGroupSource.getOffset(), equalTo(0L));
+        }
+        {
+            DateHistogramGroupSource dateHistogramGroupSource = new DateHistogramGroupSource(
+                null,
+                null,
+                false,
+                new DateHistogramGroupSource.FixedInterval(new DateHistogramInterval("1d")),
+                null,
+                0L
+            );
+            assertThat(dateHistogramGroupSource.getOffset(), equalTo(0L));
+        }
+        {
+            DateHistogramGroupSource dateHistogramGroupSource = new DateHistogramGroupSource(
+                null,
+                null,
+                false,
+                new DateHistogramGroupSource.FixedInterval(new DateHistogramInterval("1d")),
+                null,
+                DateHistogramAggregationBuilder.parseStringOffset("-1h")
+            );
+            assertThat(dateHistogramGroupSource.getOffset(), equalTo(-3_600_000L));
+        }
+        {
+            DateHistogramGroupSource dateHistogramGroupSource = new DateHistogramGroupSource(
+                null,
+                null,
+                false,
+                new DateHistogramGroupSource.FixedInterval(new DateHistogramInterval("1d")),
+                null,
+                DateHistogramAggregationBuilder.parseStringOffset("+1h")
+            );
+            assertThat(dateHistogramGroupSource.getOffset(), equalTo(3_600_000L));
+        }
+    }
+
     public void testRoundingDateHistogramFixedInterval() {
-        String field = randomBoolean() ? null : randomAlphaOfLengthBetween(1, 20);
         DateHistogramGroupSource dateHistogramGroupSource = new DateHistogramGroupSource(
-            field,
+            randomBoolean() ? null : randomAlphaOfLengthBetween(1, 20),
             null,
             randomBoolean(),
             new DateHistogramGroupSource.FixedInterval(new DateHistogramInterval("1d")),
+            null,
             null
         );
 
-        // not meant to be complete rounding tests, see {@link RoundingTests} for more
-        assertNotNull(dateHistogramGroupSource.getRounding());
+        Rounding.Prepared rounding = dateHistogramGroupSource.getRounding();
+        assertThat(rounding, notNullValue());
 
-        assertThat(
-            dateHistogramGroupSource.getRounding().round(time("2020-03-26T23:59:59.000Z")),
-            equalTo(time("2020-03-26T00:00:00.000Z"))
-        );
-        assertThat(
-            dateHistogramGroupSource.getRounding().round(time("2020-03-26T00:00:01.000Z")),
-            equalTo(time("2020-03-26T00:00:00.000Z"))
-        );
+        // not meant to be complete rounding tests, see {@link RoundingTests} for more
+        assertThat(rounding.round(time("2020-03-25T23:59:59.000Z")), equalTo(time("2020-03-25T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-26T00:00:00.000Z")), equalTo(time("2020-03-26T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-26T00:00:01.000Z")), equalTo(time("2020-03-26T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-26T23:59:59.000Z")), equalTo(time("2020-03-26T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-27T00:00:00.000Z")), equalTo(time("2020-03-27T00:00:00.000Z")));
     }
 
     public void testRoundingDateHistogramCalendarInterval() {
-        String field = randomBoolean() ? null : randomAlphaOfLengthBetween(1, 20);
         DateHistogramGroupSource dateHistogramGroupSource = new DateHistogramGroupSource(
-            field,
+            randomBoolean() ? null : randomAlphaOfLengthBetween(1, 20),
             null,
             randomBoolean(),
             new DateHistogramGroupSource.CalendarInterval(new DateHistogramInterval("1w")),
+            null,
             null
         );
 
+        Rounding.Prepared rounding = dateHistogramGroupSource.getRounding();
+        assertThat(rounding, notNullValue());
+
         // not meant to be complete rounding tests, see {@link RoundingTests} for more
-        assertNotNull(dateHistogramGroupSource.getRounding());
+        assertThat(rounding.round(time("2020-03-21T23:59:59.000Z")), equalTo(time("2020-03-16T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-22T00:00:00.000Z")), equalTo(time("2020-03-16T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-22T23:59:59.000Z")), equalTo(time("2020-03-16T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-23T00:00:00.000Z")), equalTo(time("2020-03-23T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-23T00:00:01.000Z")), equalTo(time("2020-03-23T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-24T00:00:00.000Z")), equalTo(time("2020-03-23T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-26T23:59:59.000Z")), equalTo(time("2020-03-23T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-28T23:59:59.000Z")), equalTo(time("2020-03-23T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-29T00:00:00.000Z")), equalTo(time("2020-03-23T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-29T23:59:59.000Z")), equalTo(time("2020-03-23T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-30T00:00:00.000Z")), equalTo(time("2020-03-30T00:00:00.000Z")));
+    }
 
-        assertThat(
-            dateHistogramGroupSource.getRounding().round(time("2020-03-26T23:59:59.000Z")),
-            equalTo(time("2020-03-23T00:00:00.000Z"))
-        );
-        assertThat(
-            dateHistogramGroupSource.getRounding().round(time("2020-03-29T23:59:59.000Z")),
-            equalTo(time("2020-03-23T00:00:00.000Z"))
+    public void testRoundingDateHistogramCalendarIntervalWithNegativeOffset() {
+        DateHistogramGroupSource dateHistogramGroupSource = new DateHistogramGroupSource(
+            randomBoolean() ? null : randomAlphaOfLengthBetween(1, 20),
+            null,
+            randomBoolean(),
+            new DateHistogramGroupSource.CalendarInterval(new DateHistogramInterval("1w")),
+            null,
+            DateHistogramAggregationBuilder.parseStringOffset("-1d")
         );
-        assertThat(
-            dateHistogramGroupSource.getRounding().round(time("2020-03-23T00:00:01.000Z")),
-            equalTo(time("2020-03-23T00:00:00.000Z"))
+
+        Rounding.Prepared rounding = dateHistogramGroupSource.getRounding();
+        assertThat(rounding, notNullValue());
+
+        // not meant to be complete rounding tests, see {@link RoundingTests} for more
+        assertThat(rounding.round(time("2020-03-21T23:59:59.000Z")), equalTo(time("2020-03-15T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-22T00:00:00.000Z")), equalTo(time("2020-03-22T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-22T23:59:59.000Z")), equalTo(time("2020-03-22T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-23T00:00:00.000Z")), equalTo(time("2020-03-22T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-23T00:00:01.000Z")), equalTo(time("2020-03-22T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-24T00:00:00.000Z")), equalTo(time("2020-03-22T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-26T23:59:59.000Z")), equalTo(time("2020-03-22T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-28T23:59:59.000Z")), equalTo(time("2020-03-22T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-29T00:00:00.000Z")), equalTo(time("2020-03-29T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-29T23:59:59.000Z")), equalTo(time("2020-03-29T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-30T00:00:00.000Z")), equalTo(time("2020-03-29T00:00:00.000Z")));
+    }
+
+    public void testRoundingDateHistogramCalendarIntervalWithPositiveOffset() {
+        DateHistogramGroupSource dateHistogramGroupSource = new DateHistogramGroupSource(
+            randomBoolean() ? null : randomAlphaOfLengthBetween(1, 20),
+            null,
+            randomBoolean(),
+            new DateHistogramGroupSource.CalendarInterval(new DateHistogramInterval("1w")),
+            null,
+            DateHistogramAggregationBuilder.parseStringOffset("+1d")
         );
+
+        Rounding.Prepared rounding = dateHistogramGroupSource.getRounding();
+        assertThat(rounding, notNullValue());
+
+        // not meant to be complete rounding tests, see {@link RoundingTests} for more
+        assertThat(rounding.round(time("2020-03-21T23:59:59.000Z")), equalTo(time("2020-03-17T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-22T00:00:00.000Z")), equalTo(time("2020-03-17T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-22T23:59:59.000Z")), equalTo(time("2020-03-17T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-23T00:00:00.000Z")), equalTo(time("2020-03-17T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-23T00:00:01.000Z")), equalTo(time("2020-03-17T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-24T00:00:00.000Z")), equalTo(time("2020-03-24T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-26T23:59:59.000Z")), equalTo(time("2020-03-24T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-28T23:59:59.000Z")), equalTo(time("2020-03-24T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-29T00:00:00.000Z")), equalTo(time("2020-03-24T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-29T23:59:59.000Z")), equalTo(time("2020-03-24T00:00:00.000Z")));
+        assertThat(rounding.round(time("2020-03-30T00:00:00.000Z")), equalTo(time("2020-03-24T00:00:00.000Z")));
     }
 
     private static long time(String time) {
         TemporalAccessor accessor = DateFormatter.forPattern("date_optional_time").withZone(ZoneOffset.UTC).parse(time);
         return DateFormatters.from(accessor).toInstant().toEpochMilli();
     }
+
+    private static long randomOffset() {
+        return randomLongBetween(-1_000_000, 1_000_000);
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GeoTileGroupSourceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GeoTileGroupSourceTests.java
index c08416b000a8..bca6ef1c03d1 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GeoTileGroupSourceTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GeoTileGroupSourceTests.java
@@ -51,6 +51,11 @@ protected GeoTileGroupSource createTestInstance() {
         return randomGeoTileGroupSource();
     }
 
+    @Override
+    protected GeoTileGroupSource mutateInstance(GeoTileGroupSource instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<GeoTileGroupSource> instanceReader() {
         return GeoTileGroupSource::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GroupConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GroupConfigTests.java
index b297566b65bd..eb211e6dee33 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GroupConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/GroupConfigTests.java
@@ -93,6 +93,11 @@ protected GroupConfig createTestInstance() {
         return randomGroupConfig();
     }
 
+    @Override
+    protected GroupConfig mutateInstance(GroupConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<GroupConfig> instanceReader() {
         return GroupConfig::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/HistogramGroupSourceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/HistogramGroupSourceTests.java
index 653de12db8e4..ef91ec61e924 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/HistogramGroupSourceTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/HistogramGroupSourceTests.java
@@ -63,6 +63,11 @@ protected HistogramGroupSource createTestInstance() {
         return randomHistogramGroupSource();
     }
 
+    @Override
+    protected HistogramGroupSource mutateInstance(HistogramGroupSource instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<HistogramGroupSource> instanceReader() {
         return HistogramGroupSource::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfigTests.java
index ef1f7ee9afe0..e83fa7f3e496 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfigTests.java
@@ -71,6 +71,11 @@ protected PivotConfig createTestInstance() {
         return randomPivotConfig();
     }
 
+    @Override
+    protected PivotConfig mutateInstance(PivotConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<PivotConfig> instanceReader() {
         return PivotConfig::new;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/ScriptConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/ScriptConfigTests.java
index 3be52dffc52d..a392c706cf42 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/ScriptConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/ScriptConfigTests.java
@@ -88,6 +88,11 @@ protected ScriptConfig createTestInstance() {
         return lenient ? randomBoolean() ? randomScriptConfig() : randomInvalidScriptConfig() : randomScriptConfig();
     }
 
+    @Override
+    protected ScriptConfig mutateInstance(ScriptConfig instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testFailOnStrictPassOnLenient() throws IOException {
         // use a wrong syntax to trigger a parsing exception for strict parsing
         String source = """
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/TermsGroupSourceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/TermsGroupSourceTests.java
index e1eb7d5c897a..9929e7462f25 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/TermsGroupSourceTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/TermsGroupSourceTests.java
@@ -69,6 +69,11 @@ protected TermsGroupSource createTestInstance() {
         return randomTermsGroupSource();
     }
 
+    @Override
+    protected TermsGroupSource mutateInstance(TermsGroupSource instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<TermsGroupSource> instanceReader() {
         return TermsGroupSource::new;
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c1.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c1.crt
index 532b19692cf8..28b34827f64b 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c1.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c1.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAOycKYSPiBuHXuHsqaE5KyTLYTqlMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxM1oXDTQ1MDkw
-NDEzMjIxM1owEDEOMAwGA1UEAxMFbjEuYzEwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAM/q6yVK17PHtdsO5pM6DNU6pnOY/FQO+c1JpD2cpOk6B8yokEtFR+a0
-QsdsqQewAGBG77u9jQVerJr6fkPW+AeJT7eEBl5rqYDx82XgeJS6dAJRvclrxsOL
-BDDWsImDIMes0AZaE54P6LDGBooH3XhidTyFj2Gp9fozVY8PWFl5AgMBAAGjfzB9
-MB0GA1UdDgQWBBRLyFFKNa/ll8mohWv8TJyzCzyOmjAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMS5jbHVz
-dGVyMS5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AFmuRJmgPi2dwMwhfzGdDB6d+FLxQOcWSqj10e7Iq+bEydUjYZubp1K0/9dbUB5o
-9iUvvFbZirwhRhcYjB5s/sEJavXEOXQshU8zek+jBuO+uKdndmiA5oGbDcKAEcbH
-aPbq4eJcLAT7RJWlW66nCZNvJNnSh+DmV2w9XUZBG2ryOlq5OYCmLJ57TBKgvK3C
-6gOECdImj2mvZ86xBmjZXZ6JVK0ZbnClIFt+3b+R6l/IUkMf6t0VVN/cs4AB7bTX
-c1E4L4WHUmO7enzMZi1LyOWs7BfdJjjx/YnsVyh6PqcJTt7896ohbnfORmtuMZNb
-eO5el0aBOXZaJb/MiUC6E5I=
+MIIDUDCCAjigAwIBAgIUYEfQjQ92+JVhB2oUCAsgYfZNyQwwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE0WhcNNDIxMTIx
+MjEyMzE0WjAQMQ4wDAYDVQQDEwVuMS5jMTCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAK7bV6N225wvhfQWP4JuX54H5L10N+OO952cdU57jcCdtyAIHt74
+XzvpHxnIvc7OcjfMG90iD8H60EIeXr1PrYqY7Jwra9KHAdxtqQzvFKKzKS+rrq8n
+v6B5rmTCr2MofYf9GqG76JVHCqSUZcNqbwaE9ZA8c6pPrHOCdpL0Wukk0TxIwE5K
+5oCLp0k5FB1QlOnfS4iJ0B4OkHo8t2cte13LPxAq/ip125aTCZD6wErS4HU04NUI
+fM9kqCj2JvMDcRBALsITtCTLPgZrFzRyOoe2IomIrpgCpyX38mlCkEKfUImZ84yw
+ECQYtXfNr08qtIqAxPKJXSDU4OHlH9QU+O8CAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+eAvx39OQ/kxxF3cxHvBTj/+sR7IwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTEuY2x1c3RlcjEuZWxhc3Rp
+Y3NlYXJjaIIcbm9kZTEuY2x1c3RlcjEuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQCjCq8Pw9Uo/13KhyhowLdjavXYJQ7LtYMlymDY
+as5ikj9Op2i5SnfGddH+8R9bhGTGhFaSL4DeDPqeiMNiliG89G7zV101r+Xv4nqX
+wYUuOSef09UQqR0iHgGFL2jSJiJ3YPNsC5t1SSyGGicMyzt1iwFu/dRktJjhJXuF
+FvxFBLLofh3g5xJPsS6M0h8HC2mI5nOZ7J2o2EkFsiKg/ryJ0BfAHX58lXGrSrnL
+93St6aQ1xNBOCiNf/TfrTLO2hajDWXeVh2qXMT+KbXbcAC4EIbXE1hB1Zx8Ebl+o
+4+G6mqRSGO8zJmrqLqxdf2hXRDanMgL1dlea5bFJ7d/LALOU
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c1.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c1.key
new file mode 100644
index 000000000000..8c7b981da54c
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c1.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEArttXo3bbnC+F9BY/gm5fngfkvXQ34473nZx1TnuNwJ23IAge
+3vhfO+kfGci9zs5yN8wb3SIPwfrQQh5evU+tipjsnCtr0ocB3G2pDO8UorMpL6uu
+rye/oHmuZMKvYyh9h/0aobvolUcKpJRlw2pvBoT1kDxzqk+sc4J2kvRa6STRPEjA
+TkrmgIunSTkUHVCU6d9LiInQHg6Qejy3Zy17Xcs/ECr+KnXblpMJkPrAStLgdTTg
+1Qh8z2SoKPYm8wNxEEAuwhO0JMs+BmsXNHI6h7YiiYiumAKnJffyaUKQQp9QiZnz
+jLAQJBi1d82vTyq0ioDE8oldINTg4eUf1BT47wIDAQABAoIBABBpKqj/WJjTMasZ
+GXMoNmt9CdVUHoNfdtLr4mtFbPuVHHZHp39oEnwWIuKcdLVU/+LyZ50R+3uftW23
+LvoX4SLpu685OLXku40mHPXHTImr704iOqiwE4FABQOU7WQM4qHhjzR7D9b5eMC9
+4sJ770P5NO40FkD3mFOrp22fEcHhbc6A795Qi/xUF3+vgxGcLuPbY8yMqyd467R4
+qLg4mNfGYNaAGcZqOO6dhrXzdX4zXc0PZNLP/vlNxKoY0UCVGJIySvTC8OrE3v6R
+PwtvRhgfb2Fy3n4ZN5DjrAXJJvPqiXEsRQxibzYF8xJL1P2mGUnyhqfOd5YtzorN
+BZ4GmWUCgYEAy3BgQ2hDwjzf/b1ywcux/G5ewP7+IiJNBU2flGLnAnfUIr2aqsdx
+I5FYShgwe6sMDZLeGthR0O2K2tpuhqfZGdeD7wBi4Bq7da/MactEqBjse9JDyo1O
+1vSKMkppYwxya8tVHaLPJa9bc0uGm/j4rRO3FkxRF4+MsAQ8+/aBmVsCgYEA3AiE
+oc4rSIxyjzPfPBFRxXcnrp4kYhxMz9o2DI26LmHheGRPe8ytxfaL0VsNXo+UC/2H
+nqd9Tn6TfFU+y/iv4x3I7hMI/JbxCrm1OyCtCkaoUfJsDRLrrogZpXNWXiIvI9aC
+yQaR0bw6vzYONcyWecBsBnDJu4N5s/cuaSzjXv0CgYBM3z3Autk/wDjzbG68g/fR
+ZvMvAiuNQmDQLDCspWKdMCJcrkCPX6WExxsRhGptkIiRaHm8sxjXVasuX3N8Nlh2
+x1T3LrIt9Gv5YGBhaYmb6y8Q6rbrBtJPrz+5GR0RbFagHMkg1bBokQaBXnyS21U8
+MULxjgzmVjvXFwLkQpVeGQKBgFtpYojCWXMTfkiGGVkgLOOnpCHEa4MjFLpJgqjm
+fJHmckCNFARvvUYALog0lrJRXgU5OA2usb1ZkrZVx/f/gjsN1bsqx1IA93UoP+Wn
+ppwdTLJ2z3MDCE0DF/yHlTsYYpM1/DG9+/ixhYqIz5CERYOhcH+gff7cGUSuDWb+
+khuVAoGAFCieBT66xG+4ktuB6cmH1GKVfTY71Zj5K43CE3X+PDaSwtlZKBMZzx/X
+HSWRZndTdSdekIv6tE8zSxwuNMRCv/+I0RbP+fTBv4ICOH9Jrb0Q9Iy3N0HURLPn
+C0BNpXcWLOlksQbQ3CcAEXOqhIB3KIyz/6DClLvNLBPEhiRbwNo=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c2.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c2.crt
index 680547f5beed..7fbd6cbf84df 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c2.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c2.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUV4XHn9tdtxmUFg8gitI7WNN145EwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0
-MTMyMjE0WjAQMQ4wDAYDVQQDEwVuMS5jMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAlC1TOrM0KvZxZ40BCef2HCQOGsFI/IUmFCVno62Rq/DZnQbpqs9eyBjD
-dJ9lMET4arkqJ1Vvg0rodZWCg8TOU3+Wr1xV1u2v4oBTdwyfoMrZxKW3NtMGQRdr
-suTkfQm32ouq4eJ8Xcf6ziCwApuoIKDPRZipu+9Jqmywqw45a1kCAwEAAaN/MH0w
-HQYDVR0OBBYEFG4GgmhNwmrhBpwtZAMtv3bsH2PrMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGUxLmNsdXN0
-ZXIyLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-ZNWqkJ8Fl5DwSRgQfUAjOjEY7+PCAvX2qKY8I9Z3DiB5D8xFeIP0qrz78oN8K8JY
-iJXFPaKFpYihejHywSK7/A+myvZlOUDbPx5rULtsYVeK0e7n2x8htTVD5E6M0nRP
-Jdb9WittiGe4muO3e17wdwSLEOrRfp+YmBnRy+7Q9BOix4v5y6HWpYwGdX5Wnj8N
-devs0ceL9Ymtdd6lSEmDQRxBGQ3xXfBU4+Wl0Ec786OjxfsE30hBRFH9S/eQwfE/
-vGxB9Bo49I1Jar9DRMxnWjdPuoJptvrFHeByejovBeGfjOmXDpl/+eNk6JcnXjH2
-9UTKG8M6OF9J2SVOWCzU4g==
+MIIDUDCCAjigAwIBAgIUcyiHMSNG1nFi/Vd1b5ZdP7frbmkwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE0WhcNNDIxMTIx
+MjEyMzE0WjAQMQ4wDAYDVQQDEwVuMS5jMjCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAI146YTocHeUH0uaiLFjSrxlvCtWJGBTmYqpxY4b0K0p/ZXmQBRU
+BBdZ+EUpRx/IgQftzQPQ4OyvS13+ozBLoTwXX/+IgvVx25tu3gfuHphw3bpK18uL
+YPRCM9gMXWV3Ffynws/DftH443YSsDYEMn1/yVobASdNhl5p7AIPfGQoSVYRbexf
++cErQ0zMbXpcdF0uf5iMpqcloPMp4XgqXwsZ5CCjE0JuUYshcU+4G/kAZ0AXDVHW
+Z6CxQsrcC/PC0AgJgk6fITHHnu+wU2AcSC/9Nz8IaM/CeIuLvnIYMhmf2GZt/kgc
+4gFASV5m3SJEJOdR6gB1fyt3emUYYoT3uMkCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+z9mVG2EsrlH6LJGbpSzRvjxrAaQwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTEuY2x1c3RlcjIuZWxhc3Rp
+Y3NlYXJjaIIcbm9kZTEuY2x1c3RlcjIuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQBuYovGH3pKwxCgNxbwX8PRVHoXIQyxZx2+qY0b
+BMnpWByGMzr72QdflvFm+OvIqthkt9kHKIA1rIGlB0dwUpo7RNLotXg9qspngZ2v
+Bz/OT+v6hk3g53Tv8FncNs4tn/n/7jdEi7j4Ky7WiKlXGHkQv5LuJOpXJRcEV7YG
+dGOiNGqFqKYDt1w3YPi9UtHceebhJPkYzD7nJlEhqpozAcBeCr9VVqcgdIlMUMvo
+H7Cuj14HoUoYOc+RemZ2jZREdHTRIQ5PXntSnFvnkibz8fDhswRAAqDxQ1iVh8hS
+NbH2Ho+pZWLDj7H7KrG9PGtzxlA+bvMQfbGKTtDxnk/j/72P
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c2.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c2.key
new file mode 100644
index 000000000000..ce7bf1476abc
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c2.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAjXjphOhwd5QfS5qIsWNKvGW8K1YkYFOZiqnFjhvQrSn9leZA
+FFQEF1n4RSlHH8iBB+3NA9Dg7K9LXf6jMEuhPBdf/4iC9XHbm27eB+4emHDdukrX
+y4tg9EIz2AxdZXcV/KfCz8N+0fjjdhKwNgQyfX/JWhsBJ02GXmnsAg98ZChJVhFt
+7F/5wStDTMxtelx0XS5/mIympyWg8ynheCpfCxnkIKMTQm5RiyFxT7gb+QBnQBcN
+UdZnoLFCytwL88LQCAmCTp8hMcee77BTYBxIL/03Pwhoz8J4i4u+chgyGZ/YZm3+
+SBziAUBJXmbdIkQk51HqAHV/K3d6ZRhihPe4yQIDAQABAoIBAA9bI4XWBncvBc1F
+YzcAqcO/YJMXpkUqbeJ7lm0GyeTcnPZLaC3iB2ZS0WcNxExH4vGYYgYwWGnN2+bC
+VWD6+e+1iH8KiKSEYlZXIhScR907dbswlTHhLs9J7FS0KuMbmlsICVZXSF8Znp8E
+sknQCib4hZjukMvSV2vMyt2tDqR/AvvGeauDwkRkUmjEOkYDF+uoP3oZna4mPcLy
+/BIAMQ3C2gnR+xOSZBfAgyil7l0T+ZeQbFM1/8ntvK9yn8961Dqm3qiOMKrN0+n1
+e+oZKtCpq1uouV3Uz/aaJpmQVwKVAO8LoNuD2e5P4JeL7iXlWbcM0ThI1EDK8MX7
+GHysrm0CgYEAuhyi42tPkCHskJB13ptiJbMqyOqU0NKbX/4Csm6eD14WulJ0POtk
+iEsEKIZG9sGT1/HzagJNSWh9Y/RzQp8ubp11Iwzl+qd6db9fbgzURJ7bTHVOpv0x
+AIx6zoIlRTddEOCbZvTbUzZWGsmOIDRnKp8a6E1brUv/ioHthXHqvocCgYEAwpj5
+E0FgJM8xyTZiIerjZd9s3OvZu+dskU9Vc04nbO9dKYRuyj2N93oII55Tg/5+CZHj
+67rWXsuAtGoEYp6bIfEbkHjD0ZZEbRqgHrEVq9AFPBU8SxTKPV7Y/nyBLG2EsGrS
+92ri1fUJ3bQHVAh5xnt3HSdx/iaPWzQhjNHM0i8CgYEAmzNeMsA+bTedC0EsyiJR
+ypaKy1m0GCBJKdetcBcHAFG6Kk6wWE0EXRQRt6mibAwpuh8umeypftoAFua3Z4tP
+MTdlg4EyF8CqQp8AecmBCmhfAHeiy2bBAnAjySqG4h21ImnrrgPrEo1xaI1EVF7e
+2ZAs5CP+m5mQ1HRdGrdQxukCgYEAjOOmSdumWWAtoomeSQI/+2zk65pSvSnTv+0Z
+Z+8oukUfRUTd1cz1MT0IEYm627Dw3crjorNWK7HZEEZFwIUmuk9Pbq0Q0XijN/7z
++OSrVQa6E4MocQ+vUPmPz8ii2WC9leDxtzKK4u+TYhHK0HuGdz7CwbfiX2jZl+XJ
+eKQZT/8CgYA/sAlesy6Uc2hGRmre4o6Nnz51EWVA1AKBldpuY/wmr3GMvqMJWoFx
+CKkf6SbSODAi8kTxtHx2SgWCly96z38QcphhlY8ClQg2jPjumtnmCkXCBcQJLcez
+4gQ/MmRdAsu1PGb+4Yi6I4HCc3+3eF7/aoGpxo3Nf9IkR2gN+BJaTw==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c3.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c3.crt
index a29a86e3f674..ca978d7ec4a1 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c3.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c3.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUGqZyGvYnT/TnwhpRix5jqfGzMHEwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0
-MTMyMjE0WjAQMQ4wDAYDVQQDEwVuMS5jMzCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAhXKEuM1xYezN98jNJftpBGSAQ4jM0W4nrP86NhflKhCCNnp6PSh651t4
-4+v8tPo2ZwhaBAkchrfDtHKBuT//6PIteyEbtsj/ej2OztQ9XO6h4+3bL6ccVgfM
-vm6YD1y6TjTM9fCvpbtqrBMouEOYkPPLA6lx23JyW5mJ3Lwwno0CAwEAAaN/MH0w
-HQYDVR0OBBYEFKEfz2T5jwYqWOgbnZOJL2mLkqtAMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGUxLmNsdXN0
-ZXIzLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-Bd8aXS7DvR5mNVz7aSR1qoTBHKVLqIrUju5gKFsIrkfm0mEgvhx8T9m60k5h+sgD
-TMaoHatMIVssOaNoxYwvKtnnuHKMvrkJLb/qyaiDfn0zO2g0jvWMtmaZnShrE+jj
-la1hFKCkxblUgo1d0IQaONaDpgS2PXPu1Iah7h4MPB4BZdFNT5afnKro3zAfiKA7
-lMhGmTwPKTDwkOETNJQ4ETFjZzY2UxFjLsRzbrXDeto+UrU/qJa8jzCWI8cI9jFc
-vffGGldbRWoUq0ALZSbz6W2psf/xAo6tSkaf0MDSMYRZ9mFVmWBSREITn+12gp7g
-5A8wNJujWbyAR0j5GXXZAg==
+MIIDUDCCAjigAwIBAgIUerW/NwMch3xS2m7baNp82ZJH8jowDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE1WhcNNDIxMTIx
+MjEyMzE1WjAQMQ4wDAYDVQQDEwVuMS5jMzCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAKtV/1tTyEHagrRwCVYybv/K4k22CGzJWuUM9ptrP0khaBFviw5l
+iUNik2zBwKLhVGCRh9+1hzfMMe7dpyRCgNlBSdUVqaaQR9VRY7X23XWDj4RfIKtM
+V+cLBX8RiHfjh5gVeWqAJ7O+JibXI3pp1pnyfhg9oj3oRpTRVCyiOXU62uemDxam
+0KC0EQWwJeXG86c32jy1v8CqYztEDYBdXwsJK/kgOXPLmDcFB/bwft3Shzcmboff
+qHWEic7JNPa1uMG6kBgRhm59uNjyKII0HCPhrABV1PnnrcVLpeFznp4qYLKza8T8
+cis5rHgU275NM+u/2J1bt/f6bGntgXVjIrcCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+grh2iD1+KGmc/z4w1S+M0rrZY50wHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRYIcbm9kZTEuY2x1c3RlcjMuZWxhc3RpY3NlYXJjaKAl
+BgNVBAOgHgwcbm9kZTEuY2x1c3RlcjMuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQAfpAHAPMaYP7ufwL7tnpvUbJmLxBRpHYSJPEBW
+0TxD5pRkjuPie0meGR0QHFo3EWwSX+RVEzXoq+2Xe1E0cqvxRpiVgBndNI6adE7N
+G6NZXJoe1UJBCqxMSGhNGAdakkbPPeNvVRyml1SxbiPmYtORAAzlZ846fIkH90wl
+EyvwkM+Q0Ge3YJxajbl/+qZOsDzQpPZkX157cirLg8m8ojdDIw1Du7Q6M7Ea3tGu
+KbHm53Yfh8fxLTRYu/uVNTg709we5B7hx8AL383QMCtfzLC5UinFcDuQlqqBHEkU
+S+wbBCnc77lf7xxxHOXLir7qi5eQ9RtsBQHh3I+oqcpyQiiD
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c3.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c3.key
new file mode 100644
index 000000000000..750979d85dd6
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c3.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAq1X/W1PIQdqCtHAJVjJu/8riTbYIbMla5Qz2m2s/SSFoEW+L
+DmWJQ2KTbMHAouFUYJGH37WHN8wx7t2nJEKA2UFJ1RWpppBH1VFjtfbddYOPhF8g
+q0xX5wsFfxGId+OHmBV5aoAns74mJtcjemnWmfJ+GD2iPehGlNFULKI5dTra56YP
+FqbQoLQRBbAl5cbzpzfaPLW/wKpjO0QNgF1fCwkr+SA5c8uYNwUH9vB+3dKHNyZu
+h9+odYSJzsk09rW4wbqQGBGGbn242PIogjQcI+GsAFXU+eetxUul4XOenipgsrNr
+xPxyKzmseBTbvk0z67/YnVu39/psae2BdWMitwIDAQABAoIBAFDu8wxDc9Wzr8kn
+ISnv8eHvht3ZjnpA1ShcasM4snDLkHqn+4JF8UR89JHLpkDqeq6RICNK/wAa+z1P
+w6vLpEy72/IFZRmSjvQTmauzXKItjqYjP23bRqyTVrnS4Ols97id8DggKGDuAdyY
+BKDHDQG0e97cgl6G8YxLo1zgFo8qgvarVkaAQ4ZEGiIvB8j1pA+AC4Hg1xWr7JXG
+mL6i+bqYEufm5CeFXNg5MTAKvUpwheVXlAJnAeVhymJ1bYMbpL1y+KdkfDqGm8hH
+2Q9jZROPRwW7PXsl1ajYcdZfjj+kNsyAlwEjm2C00Vxk0ogzLGROzIW4Jb3Ddvtc
+pQTu6kUCgYEA5B4hBkCDsWt/tnxDS2ZJAxZg+UhxzxLUJa/vj3l9Lk3pupv92H9U
+1TsNbUwmx150ncu+SL76xGlC5o1qbBNfW8yMR1n1m4au97D8xT9wM8J6C+NlKzmE
+NqHSW/oHBMfo92yKuRx1tk1VWQ8zL0J1LhNt+NQVE4Cjw7yVvSIfvbMCgYEAwEcm
+ap6N2Lo7uxV3VuDPntzoohDlM7oty3IOqDLpoYqFFQDIhd/ZIJbKb/uRdiBdiGM0
+K+d9oej9cBDRaUkCI/j3aFU15iilLXP9kDg2wOPtFJuw+XtceaamgzVUxYoB/3DC
+fO8qr9+geY4AWYdxsT67C7rMiFv08hPoPQujbO0CgYEAsydubEiSpO52OM1S1I6a
+XNBgjKb2qthwwMBx9k5Fn+4U2DfWnwcxit176BDci8uE7sCqytsiesrofcUmhYaZ
+Z7E3onnmSKT4GRNpEUiHSD8+NsP6Gn5rdbSVTyMIDBssvdUDBepyECCgHPexDTa2
+bN9ZiQiXHN/0va8IMqfwIb0CgYBznBQ4NT9amZeChISt4xxTC8aAc6IsePxCAEFZ
+1srtC2m7WC45eH+H/cKYlG1A1nMGp4deAqFiccG42RPgtzonQPIZdGqlCblPXBeQ
+2IvrbOBG7Nn7F7ZhmtcxuyZOZA2Rdk4qam+DMfW+QKhrLGoqxWoXwmhqcMgVqeyQ
+j+fifQKBgQDSxH4PN8BJGcm74pOtqy3KFAuA8AO26hW7r4GNEe+ficWIFZzN0C1c
+ilMCwUJLig7k7QxwtarT+1kMIQBpyb7NV2emaV93k5D+G52KSTWjGiuwR0c4LImI
+RwjYOqWYr7kzJergNipzjZF5ng0ZWe/NImFpFRyMPcTD+M962S6nww==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c4.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c4.crt
index e1838cc5b4fd..3b213d1af469 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c4.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c4.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVANq892kwQ089Hnpd0M4dSCmHpvARMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjEuYzQwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAJVhOQxK4DAxYsFDcm42dl0DdpRCqxYFecP9QmC2cniv+sMUSRLsti8B
-eOKawlWL+NQK7CJhAQPuYnLEBiyhUIFz9dAVozHyOqBKFbf2L3A2nIPuom22UiJN
-79k2YXZgagSCdCdRX8WTvTRbbN8WYRfE/wLO4SBxwW2f501ET6pTAgMBAAGjfzB9
-MB0GA1UdDgQWBBQIGCed24kXamJPxK4inO7BWteawjAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMS5jbHVz
-dGVyNC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AE2ckMOf2Nrh+TfHQwvZWIWhENeIih4A4ivgimfcGWumpCho4tusNu/RIh2DxVqy
-4kJSqQ6NggGADDDPmh2p2UTa5NR2RF2FQGmfNtkFwp48tE/YXWurts0IneidoGYC
-RXM+mVXTNGlPE0BrgwS8tSKRRXh+lsvS9HH1wkLUSa8tk4PWqln1RpttiHYnImqU
-t4d38vpk0AfRMalEV5atb8eoNkKVOfrDiJ8/iA/zJ7qh3PUaU24taJtP1MNR2mo3
-aXZH1GCA+WUqWAUDN8TXHv9TABh8IggmpK39weV17BAKFHsVslq2auyDaFcmoJk1
-tLXpMUUJweWmFTRqQguKu8k=
+MIIDUDCCAjigAwIBAgIUSM8isttfREy53csauHlJheZFo6owDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE1WhcNNDIxMTIx
+MjEyMzE1WjAQMQ4wDAYDVQQDEwVuMS5jNDCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAKv8CweNke4mqGmNcnObtTN0667yWfN0nVWzvANEUZyAWlxRMYoM
+9hfCPv9kOxEddM+/pZ9AZ1v36L85rKGquPsO+m2nnBGJtNV2gup0EeC7DTRTfPlY
+YRbkItMfPzktDd6PAQDUV8FtgzTBlnVgCzWS7A7MLjSXUk/EOCYl5Nuzfxqf/PM+
+25zkwkdBlJPoogs0vnCJIcl+oAu/KXpa7FMxnG4LNdkhxtDXtW21RIK09ZFWsGeR
+ZgQl8JYk9UJPjKwMr4Yn1W5BKZPEUIZOzhg0IjfoykfiZPaRFPwZ054pZcygsrds
+mjXklhely6PSOhbncXjPsKqJuC5Cih+hfPMCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+JL313utz//DGTX5n4betjYFR7PUwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRYIcbm9kZTEuY2x1c3RlcjQuZWxhc3RpY3NlYXJjaKAl
+BgNVBAOgHgwcbm9kZTEuY2x1c3RlcjQuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQA/1fqWtbsrOs7T9UnGBlM03a9kRVWaevA0NvwH
+z11+JW5TlU2ZVsdQPUpWf+gcbh3geXVnbyGuYnbtpn3lR3lAapj/7i5UE9ZZOWwi
+XZH3QZ9yc7lCLGNgHR5fx72a5EXLsD0EewlJ153fCIn6cRyZPOGZ1R9orqRQv4bx
+/2UN8VUazDiQVenJlrjSP4EsZKrto2GqS7bxrvJF61yhty+CYtbqxzlEgQVkaj/0
+yN8nRA7kJ567duvdD7JNmP1H6+QL9Lj3EFx1y54BcEBXtjubh8tUFxdPG8PHhSs6
+TsHmQdoPeRJcJztemEtkHAK7QpbLGXfZBHn1DSPtScfXhekS
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c4.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c4.key
new file mode 100644
index 000000000000..6468d6c3d889
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c4.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAq/wLB42R7iaoaY1yc5u1M3TrrvJZ83SdVbO8A0RRnIBaXFEx
+igz2F8I+/2Q7ER10z7+ln0BnW/fovzmsoaq4+w76baecEYm01XaC6nQR4LsNNFN8
++VhhFuQi0x8/OS0N3o8BANRXwW2DNMGWdWALNZLsDswuNJdST8Q4JiXk27N/Gp/8
+8z7bnOTCR0GUk+iiCzS+cIkhyX6gC78pelrsUzGcbgs12SHG0Ne1bbVEgrT1kVaw
+Z5FmBCXwliT1Qk+MrAyvhifVbkEpk8RQhk7OGDQiN+jKR+Jk9pEU/BnTnillzKCy
+t2yaNeSWF6XLo9I6FudxeM+wqom4LkKKH6F88wIDAQABAoIBACb4S3+Aoqp18/9G
+38I2bA6m1aiknaE+sU/0FAwhdOSjaNY2R6ViXnjvuNqh2Yh9RjS01lCJhWIfgIuk
+A+v2BUhOEoy6R/DpZSJhYjTB7DMh71IGAPF3BzjqpMF1+Rt5jAT8HlwTwXbA29Qe
+cXtTMfcHvZl0xpuAFlSgv6BVQdBOMnvBiZWd5/JPx9vKb0ItUoJCxbYbYWQk0oZO
+OuG2obU4tRgxb047aanBhfoqNqZnZXFFazC/Yl79+Yi4aRAwAabPGyBLgDIIH++C
+IwlL5YDPW5VFM5WFDUi+WZJF8YDMONdu+WYhI+RrO9hvXa/Cr5g80VwTgkkGIp8B
+xrvdQqECgYEA47xCTCbjQ4nRJc5LGd5+bxNn0XsyX2XXEALN2yfXb0Edy6AnGypc
+TQ5i1pauuRjFBRNOQpGKyKFwSakDSmN1l3SwIIbkyLKrxGjiddh660zFB4qK2WlZ
+mv8G21CyvfvBZhrMcD/1ue3AZoDf1rUSfx6wZ/Dhyg2Sq7s09IWd8EkCgYEAwVRv
+mJv1lLEhBTCfZ2zCarEZKD7cpYE7q0mlH5IrKmUGS3Y9EQAGH7eTUbn1DNWCGES/
++4nI6gYhP8pPQFKlGw8lotUVmWCFd4QlE8AEG2VCsIrxu90twLr33DkduAxLE/Bx
+z227c+Pk4SguWrohARUIFcFa2G72x2eQo+G6e1sCgYEAwiS16DGmwOb2k+JAB/ez
+lYHXmbALC2TIaO/bCp/2evPqMLOBYxlDwrUm8YVXwpgpLeB9cSaeR9fI4CLjmUEc
++FcT50bYN3iKElDP7mL0ty+lMH6dTCcRqp8K+2DaYDgFccQa0P7VvwG+aiQnruSC
+tjg1bwbdleDoTtqzlFlsgXECgYBaR5/FWC8aPIz663XEylkPkZv520EWWH8QwddG
+Iu/dYEfMPW7O3X/+l69H3tq8H4gc5R3xzKnx/uLP8x2HIBDRzgT5QSBQ+23YHfFi
+g6xnTqYR4xB3dhXDhovwXwutwq/co8/B2bBLsgRT0Hu1Cm4XH0cjQaO2pjq/a3Fz
+1hlLOQKBgQDHRJ3o/NQxHT3AjkIg//+/JyXWfkmcKWI7+pUMBhu17utJzvA/SAcA
+H1/gMuwlNrg7KTfteFGDIgWkDsDBWrF1HTbftPbvEWn7S9/Qm5UVBAJBiBVpKgNp
+kKhIUploX0oSvopmUkdPVk2oPqFgUL3CyvVyom9cdXvM6Rd2FBn2UA==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c5.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c5.crt
index a844d630a2cd..8c5c3b941450 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c5.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c5.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAJ+D63xrhVvDF8SCF0IN6olyWI9hMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjEuYzUwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBALGUvDdOnTe62apHl5sXV6Ys8GYqKDJ0e1cCQCKgxu3MyCueXC1xdhYJ
-ceA3PTmnTRg7KqYhJLZi1sujBOfuy7vsg5r/7L6EhWDCM/d2QfF9ZUft5ljsEGYN
-OmGakQnU+mFOuDe18hlp72tavC2tdPYHaTmd0t2f4J9ovxQznO+jAgMBAAGjfzB9
-MB0GA1UdDgQWBBTgPsBAr0vFIhwmOhBkC1XvlAGtvjAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMS5jbHVz
-dGVyNS5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AB6nwwv8Pvzmj+/qxqyUu+doqR/DNu7J2ZoVJ4Y/kbp8Sya6GBVocN/Yaj+5aTv3
-phegvSvXTuw38gUuadGHSJ0R+GgaZNQ99HgMGCaePsFSf1Qya3vnK0308n7MSr8F
-4pDnJCmOeOZQkY0PrMHx5KW+BdNWU3bvGALQubVo+nJEuZYTBPt52AkweMbtubiS
-kS3v/SKi/mD0aKWS8amgAlCAFct419gE7frWAsEDXmQU0KDaSE37yx+0CbvqzlCe
-EZRRpgg6R1px1F0WPU1SkCfQyhTE8MMz8Wj1dL4PkCJn7ku5CRF65VgA48t6Ecpt
-4UfKfyiO1rcbZKA+UKovHjI=
+MIIDUDCCAjigAwIBAgIUcfjfdI64Fb8CJevbBBisnOZsuGIwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE1WhcNNDIxMTIx
+MjEyMzE1WjAQMQ4wDAYDVQQDEwVuMS5jNTCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAKIYqHbLrUjbQ8smJzK7O9f6qeLxNvirAiFR9WUIyT77SPqeCiWz
+ezUmBCb8jyLprtG76HiXMPvCnDbtpsbN1RKcZG0j+T7c3TtkygJk3ycfuWffRN01
+KPDqqUB8G89CQ/2W1sQxzkl/FBLjQj/DPCHy+HyYuyfVk1aFH2zH0O8y6ghwD3q8
+ZLa50Psf8fAmVLbUxTiZcD+n6bliYrGPPIqEfgX39U8tUXWrJ2a+/ESJZUdU6Nyd
+tNiM0q8GBc+2Rz7aOwnxP+uy2sdrVdBbocFF0LDwZFnJFlt3vkSrtaNp7cpkduc/
+P1sM/5tv5zL8hOdcw76gBLY8Qw+UaNU2n3kCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+RSkxevULQPv85eNElA6ZZ615LiwwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRYIcbm9kZTEuY2x1c3RlcjUuZWxhc3RpY3NlYXJjaKAl
+BgNVBAOgHgwcbm9kZTEuY2x1c3RlcjUuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQBMj1+uT/fYnNzJgDV0Fa8QhJiF8ekc9LiRPi1x
+XXC3TtvEMAVYfOMWOWsMqIe16rW+2vxplfrEMSkp3EcejB57fX6XFQ3M87QnFIvI
+OcGBWrGm5u/0mGHo+erkWXH6niw9QU74rBSgEiuh2gkguKe1+3Kkrr4uZnMB80zC
+R8fXskMgfP4em8VI7s5UY2pEpFanoSyGysyYPo9A9BlSmbx5R9VBySMsd1x1GweC
+WsgGYghTKf6yvmuRtak49boOR+a1dxz9XttD0xgZ2KFjV2QE5aLkc4S5gjx1iDHe
+JZgHKM80i63atwvdp1UI01uYSE2Fi6zbU7iwj75w3HeyjX9E
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c5.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c5.key
new file mode 100644
index 000000000000..add0c0189cbc
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c5.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAohiodsutSNtDyyYnMrs71/qp4vE2+KsCIVH1ZQjJPvtI+p4K
+JbN7NSYEJvyPIumu0bvoeJcw+8KcNu2mxs3VEpxkbSP5PtzdO2TKAmTfJx+5Z99E
+3TUo8OqpQHwbz0JD/ZbWxDHOSX8UEuNCP8M8IfL4fJi7J9WTVoUfbMfQ7zLqCHAP
+erxktrnQ+x/x8CZUttTFOJlwP6fpuWJisY88ioR+Bff1Ty1RdasnZr78RIllR1To
+3J202IzSrwYFz7ZHPto7CfE/67Lax2tV0FuhwUXQsPBkWckWW3e+RKu1o2ntymR2
+5z8/Wwz/m2/nMvyE51zDvqAEtjxDD5Ro1TafeQIDAQABAoIBABaDMjJ9iy2JNlsq
+i1siYedXKu0X7ziOF5elGF3V4I1cvQABnwspaD0zcHQxs3d32Q3L+Td2WBk/KA8A
+5p7Jy4PwMIpv6pfueNNc5il69PssDsX8XkYSsgLztFn+YqZgX60CnAVXnXzbp2AQ
+LA1LAjj808IKPEQGB5aGmpyzC7OMuA9FjURlojCkGG6EmACOjyFEZcxD37xXV4JD
+XXDciF+K+kFu7n2gpB2GO9YJXin45Cr6AvbSx0urGMS2zX02MQNigi2tK+hxHuHS
+Ci4Lo9VlPHRMYNq0l5DPW2AlbbcwqhcohUN7O831g4UNQMvyrgTuuWw2Mi71EnGq
+ZQ7KT1kCgYEA0/sFii83tRDyjwYmVlhL0jIfxuXyIALCWzatV+tkJgCYis1FDDai
++Y+vlx0SwxLjVA4LYf8X5RVVtd7LIQypMa4JoCBOJbsvwtu0itQ1XaFcpgoL7POF
+DDaiV7smD3p3dhcru/CLxwh63MgQsRfZ0BJJLSBK4qU0K0+ES3D3kAcCgYEAw8HE
+2GE0vIV7Zo8v/Pz6TPjSuHClEDe/XcByVhabx/QWFokpQ/t7nNufYrdxCpSCSZdk
+BduBQHpNbpu0DagA0KtvakAAUpByr+KwGVc5u4Rm5IUBda5zrUool1I0UD8n3tBX
+toUPLgmE/OV2j1fYdJqqq2e5IwvvBYEJrosFdH8CgYEAqIp+JhdSuIEIChV+p8o+
+RpqZz1+GelMXSsSXYD8E+n4gysNYcdSDEd9fYcu43icHXg5omHrk+bxT0G2aneVe
+JOTWPF1TJQbGe6yJyJPe9lnUuyNjxfr8vvA1dYGUSlw53Ueg+yXPTOl+HkpnkxZ+
+lMPlMJ7rS3bEmub3LbBc9UMCgYBTlNJFSQvJSnzrx0tIC4ObhcoC8iWoDQVqNBcd
+Zr3Q3AjqSloSKrgWuzcMYSKkz8/pN/h+/7/qFc4l4dk403i1n90MUpNQrVgqfPGP
+gEyoIdoEzD1ZT8kYPkeihHjNoomVewNYbOuUToA3somynmFPOBKS6NCFhDzj7rqB
+id4kZQKBgQC7kcU0i8Q4zvFjLOU+c0MRcC+BL9Zj+FNwxyvWg4N/Kjc01Nodf8ck
++WdW5q12UTdbAf5yFHO5IeHhBTy4ioVRv4hQ3ZBk6t++knuWVER9SZef3hEGoDjz
+Arxf64h8HUv0YTHxcYnSnu0DWu18YIHSMf5O2W0JYZqq2I5aAWaWmA==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c6.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c6.crt
index 9e3093aae54c..8b7aee7deebc 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c6.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c6.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAJHJ9P/gVQNv5i8q8tW/UzufE1cdMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjEuYzYwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAJGKOOMmMfmt+9VRZfYsorzOWLxWRtqD9VjRimbBVWBoIFYf+EVyEC0E
-/sg8G9eWAcSEUJ2oDp5dnMuXzEZrA3aaYeDZjtOYuHF+i+PFQQFlpY9A+NapdhJe
-9CU6cvL4ImRyatPxw4A6QLLetWHbDDIYA+6iYtmkKTkMlY6bboSzAgMBAAGjfzB9
-MB0GA1UdDgQWBBRs7k4fmkFw8RmVU1f+p9SY5qNwnDAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMS5jbHVz
-dGVyNi5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AKvKMX7eiwShOetFoe8CQ0jls5aJ2119gp917eq75XZE8HGJhiWlBA5qcCCxCrtR
-Cy1BbuHiMqPni/7KaWosw41dxyysJOrsPGw7QYnhda+ji3mtFVxT0Zert2DPKkxR
-Fgt2VWbkF/keMKDGc2vrpyq7YVc0RwDjTzEURSLxI4p+t/HbWQXLAIGRjVXneoUt
-bc+I/2/C5chV2qTQd1xC6OfnixJ1234enIL+0enypLx/8Ca6i8+Gl13FpyFpO/ik
-yY3Xe7S4rRAsvWpHRDWIxqA/M+fzhOb76nLC4H4TGkuAJ6EavsnaHHHOngbL8Ag0
-zFhXT0Vj0GucUzjx2pjFhfY=
+MIIDUTCCAjmgAwIBAgIVAPWiIWzwBeBTxx67S5Q4inI1C6ASMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxNVoXDTQyMTEy
+MTIxMjMxNVowEDEOMAwGA1UEAxMFbjEuYzYwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQCnyRePFOVECgYCEEbWK/hBjwYt1s/Y1uAXwlR9GRFW9k0//xBA
+qS+0BDp/htkEfchZNaXdYb3DvARY+j03F17haagXhQz18GHDntspbYUjo6KZ/HLb
+iT086jiPfueMwszObgEI7OQHk19IdCkF3oodBQBerJk8owCfEumQhY9uH7MvNSHe
+WNaUJhVtqV5lOF5szFUJStWOkY+E66GduL+/dSVKjQnB60SYoQrz72XwwDetmCpN
+ZiBwrZhIGJZ4D9AQ2EaHgJCZ6rMeabdQxcciu2WsruJxKGxxsaJ+XdUH3mlNYJiX
+4V39g7hrT9YIvTL8I0AnZ/zaMzIkJ8mRQjstAgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FMHhSPHivE+Usi8rs6yyBPRQudd4MB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWCHG5vZGUxLmNsdXN0ZXI2LmVsYXN0aWNzZWFyY2ig
+JQYDVQQDoB4MHG5vZGUxLmNsdXN0ZXI2LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEAGSQSbldCPH7pMwHn/GP8tG9eM7W5uss9zU8h
+c+hXxPA3eBB2564xnnzw+pH7DJ/2KSTiAlkjDq3CBO7ClKm1MUt/t9V5dGwAU/C8
+y5sZQ3QHsKkmwVo8EH/rfWjGUi8J2c99nlPeqAZdaPjVZZKiZzRSIDoKbxTEjuzh
+jjfpqePNOgwpg1vA0fkRprCDgj1VLOvEXmw01ASvIzAF5bb5xJOkYdlhRdn2AwVw
+7tvZ7WRfjPX2Riup2NJKk7YBVAGBu27JbjCuTTfYFlMMDFRzftioGjCPSEWOkNSE
+VEFuDfhl6eOvxlfjaqOg1MR9iAcShFj5Zv8ve89uNJrxpAhBcg==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c6.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c6.key
new file mode 100644
index 000000000000..d3a1c2b4c3ee
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c6.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAp8kXjxTlRAoGAhBG1iv4QY8GLdbP2NbgF8JUfRkRVvZNP/8Q
+QKkvtAQ6f4bZBH3IWTWl3WG9w7wEWPo9Nxde4WmoF4UM9fBhw57bKW2FI6Oimfxy
+24k9POo4j37njMLMzm4BCOzkB5NfSHQpBd6KHQUAXqyZPKMAnxLpkIWPbh+zLzUh
+3ljWlCYVbaleZThebMxVCUrVjpGPhOuhnbi/v3UlSo0JwetEmKEK8+9l8MA3rZgq
+TWYgcK2YSBiWeA/QENhGh4CQmeqzHmm3UMXHIrtlrK7icShscbGifl3VB95pTWCY
+l+Fd/YO4a0/WCL0y/CNAJ2f82jMyJCfJkUI7LQIDAQABAoIBAEj/e2l73NeCSOTi
+8f7COItFT5nn6JvduLd9i8a1fk7IcAmUzhxjv2ZhpJXRNF/43Y66gYRUvvwGtBmG
+HdX3LAeEMWayM1ZZFB/I4G8gwBkmW1sFc29CQUtcV+lOfC1In1R8laStc/Q70Ouz
+5hkwngRMJcIK88xm2qFa2BVRlnqBSacl6nudtBQAIpJnMTLnalFIzO4aGAPcHRRX
+2uEPqMSk4CuMBtvVc5dOAeOQoheg0o3tNSEnu5CUF9h5QyGs+DwGgdp0XkWQp5V7
+HSqLI6ORDYYmPD/A9g1790e6RTGKvDYjDJIvcB1OY3iI6reDvQ04zlXnn3iCI2fM
+5Ifqnt0CgYEA2W5qso41FHOxIVbhDde6256pEFarbx4+WFjb08d2MO3EoQNhXzM5
+O71oM+QkHKQs10u3Rje0i81xweAc72tWMg4K0oWwYo4izWJmk1U9RO2zRCAK5X11
+3Iu8NmgToUD6OIqQBa3DWGV01gw5YGux0UN0f/kpFDU+QF3/MPmViVMCgYEAxYxA
+1fU20nYJYzfjR756zCk/xdEBVcZ9PmeSJl+R+RfeGf0GhzFhbE2Hdjvh0zjsPh+H
+0O4+dmwzD9XuWaOK6+zaT65UlhHlUXJUXvboGX7Dy8JjH/VrYiZHjKU2IJnPyO1q
+K1jWnT/wo173sDYafLqK2/QMYlhI/dwG9cxuGX8CgYAbRXokgymtMwWYhOkdb12E
+hlX2Mqi1+zkeo6aO3Ym5gVkUSPRBmI8pKwyS1Z6h6QWmy1xldj8y+cW254E5lmDt
+Tk2VMid5dRJwaFMtT5eruZSuisZTGnVaADzPJiawJxI7XUXLMIw73h3VaxsXOOrI
+sIlAM+QvIK56LFABuIxE+QKBgQDDWbaj3iGgbelKnHG8+LgmPIQ78jsHv4kKtRDp
+sYiAcwHHYVebSVG179UNLppabefwZu2/xlkuckDsX/mb2wX91/Lq6NHstTW4HP8i
+CbfrvleQn2G2sNLH9GK8yoOEKZY4c86RnwBb8Gbep4L4VghLX40BhYzZk8RahcK8
+PrSBsQKBgBU9PQDais52wBvXJW51bih6tob0iWKOMmUAiSw5UPs/QXezEK5bGJFy
+0MkKCqBK3DWnbSk8MWcFI7KBBRheUjO4aQ9w3YYWYgTXZgsUuU1PE11eOnbJkQf6
+u33lKFoAqVdk3MFfM7eNDmc3DllK5ueC1cf2ZfLrFPsMVuB0XSnB
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c7.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c7.crt
index 5301576723ef..cfefc3ef028f 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c7.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c7.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAKo0NbCgIbl24/edkcXtukrKEYx6MA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjEuYzcwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBANHSoRZvMsCdrQREwlLHR8xFRPrlW/qC1VcbMDkwSdvFOrPh/ykpPn+I
-kKrTQ/kOBCOruPSh7x9/5KB4EbkMJnIo+XQlLgLbAUeMGSS11IKpQTS7S6noyxde
-7xU0Z4u/MYk0PS+QgYqDU2ExoY8UN7VGxQg9zSzy3dublTUfVXPbAgMBAAGjfzB9
-MB0GA1UdDgQWBBTxpLGy+DAZqPn1o0uRboFaeT0D8jAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMS5jbHVz
-dGVyNy5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AGPN3G8zC9wT+FPDUJzR5aiXWkBN6Bn8pYaDZyWuWFQr4l0m57sX91vOSBBvXtYv
-0gcz/Nggkz2TpMokfXBOoioeHywYIycy+9F9obbgKy/f5ejCGv8F26FS4UH4WVqs
-dIHaWy77od7SUuBqjlOYBAK13GNZt/4rkWSNsry3p+YoVxmEoUW0Y9BtXDXY2M/U
-ewTEbC1oZHXeyNzgI6Njw5d/hELKMNFwq2RON6tU/XHXABXCdzFkIjHvrM+N8a4N
-ju2xbQALjSgPWjYprjCiYHW5z+aB+lAKbCCUcsrMQnE85Bx1DpUs1oAWYo77h7Xd
-uAX6yQ0JODD/TpNGoWktcUQ=
+MIIDUTCCAjmgAwIBAgIVAJUhyR6QNxUckaeIj/pRCKLuAQxsMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxNVoXDTQyMTEy
+MTIxMjMxNVowEDEOMAwGA1UEAxMFbjEuYzcwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQCtAxYfnr4JQm+PhChak9VoPngeIoiYy8g4dfzj1SmtTd+SMANU
+KEFnfGTRj1Efzwu0UgQdf0XkWau+lduN+MhDHVlxnFQv7zPFJa+wkf+brdfBw5vP
+7hmscniIi+em8grcj6zAgyQR4IWrNl1TQucpY0mMVDsawuPk6fexWbYuTa3eMKYr
+5SCs7jso0SDr/KEBoBbcVvlJnflRM2nPx0Otm6ck3IYUZ0rTlQcrVyqty83U+j2p
+zf9Gu9PCZgQSUpnxEPtLjJmHRmsGCEmDAkQWv+3pkHvu81FE40CMlgmwWwtRaep1
+EQqritU1fIigH0jjxsrcf5dLkU4/LEKAqGrFAgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FP/Ec5xkXZ5pOoPxxsqKHzofGSzdMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGUxLmNsdXN0ZXI3LmVsYXN0
+aWNzZWFyY2iCHG5vZGUxLmNsdXN0ZXI3LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEAYyUAgy4qhWOIFHHH761LTxnQDeCnFEf8b74H
+ns4Y6Ba7poxnkTUUOCi07jJP4ZuSMHyCGFBwaICPJGYi+vaz9fB7tEh0jW0FPtBu
+VPSKhMxk6AXhWO/2CvHZMNDi0wD/ywnq70BxkFLIKCe7MPiNCGN0RO6AQO6bzKlu
+naH3xUqU4k5wxOOAwBLP/a6RhFVEZdzlOrnMyorBSDcW/1GoIHQ4YI8Hm0Nuhjc9
+UACItH9HIVLljoewwBM8mfsn5Xt+0gczdbgnZvEkLYYQDIShr4R6gygWWwATSOL9
+O8tDNQSr/xiYUP/tXd3FTfOzSk9fJAydDWlAOq7itRsp312fwA==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c7.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c7.key
new file mode 100644
index 000000000000..97717ee17647
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c7.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEArQMWH56+CUJvj4QoWpPVaD54HiKImMvIOHX849UprU3fkjAD
+VChBZ3xk0Y9RH88LtFIEHX9F5FmrvpXbjfjIQx1ZcZxUL+8zxSWvsJH/m63XwcOb
+z+4ZrHJ4iIvnpvIK3I+swIMkEeCFqzZdU0LnKWNJjFQ7GsLj5On3sVm2Lk2t3jCm
+K+UgrO47KNEg6/yhAaAW3Fb5SZ35UTNpz8dDrZunJNyGFGdK05UHK1cqrcvN1Po9
+qc3/RrvTwmYEElKZ8RD7S4yZh0ZrBghJgwJEFr/t6ZB77vNRRONAjJYJsFsLUWnq
+dREKq4rVNXyIoB9I48bK3H+XS5FOPyxCgKhqxQIDAQABAoIBAAf8hLysbf7M7Yvt
+MJSvztHbUl2nv/D3tTQgBC67NAuPNPMRWtH7crt1c0z2JetHOjXWdGCtM5t9JS+s
+yMAGvC3W8L5NFljJHLVszi2jK66yyWslHa/tQIuklBpPhP/AnA4+5p1TYGqf1+PF
+pGy3waZ4Mhizea/8T5tNTpUF/GIqOdinX3Tzu5iqm1jA4YPNIaM96E3A90f/Pj/x
+QlPmoCprKwnecM4S5hIXgfBxDIChwECYb3UBKALk2tLU7Xi9MmXD6myAPqgZ/R58
+XCUVrVgVyLNBb3vlrsfL3CuWjrXvYaANblM2hq9w89v3my5KT/1k/fQh2Mp0rxPR
+MeluQpkCgYEA3QqHPMc97JcUZUnIwZ2HXQ47EIw2g7Q61Mvwbb/tZe8X2Z4BobPM
+ofEB5Kb6nhk7wy+e5rUKCVj41NTBfqN/EZHGIqt/DDXTGZCXbfL4w7dXg5OoKEBU
++CoLIMI57FKJGRYNwmg9A8zmBr9MbA04pcNyw0Zuh9Qe6U7NL/KFFgkCgYEAyF/3
+Vp9CI/6lxWybD7nfR5mmbtPb6crodqaaBAYfoXKqhUdDmJOn3ple8S/ui+NYOP0t
+eHid4TwRlyVYKtT4ivo/2cDkj5hEGiPN7gkv8lXK4+P0J0//f8y0towfdVoWx8dh
+y0sO+VqvyCSducmWf5w3D0wf6v2NYrexyphyfd0CgYEAnQCbampC68er7v3noM6Z
+lVlETs6o8geUOg3ZarsGjn6zPBllI0wor2YWaCjKnn1h25P5k/lHdbhCljnAbokG
++p3GVF3n0e5xURlI1gnc9HdHq+e3a0+2isaiPNR4fAlr4+usPkf3/rkNMgGy8JS7
+hdI7s8RVQhlDE4iracEjErECgYEAwhqkwVbZfYDrGQqSU3OdZgDkByxYNmyHMJlH
+TWqZLY6kvSNBygYPckVkgYQLM5vyFJK9f5xoZI4KBD7639MbSZYOUGTZhHjeT+wr
+StG/jqOpPYXPOF34hdbJoKSwvL+enbz5cTOut8Mv0VQXLqinj8rGU51MthkU43JL
+WpS/JEkCgYAkeUZsKWDX1HGAnUEFroijGe1FyY467d+91vy1IANoN+WskiRZ6qSZ
+Yfq1x9kAMzpfHhx4uYANCYDWB/2Ca+sB1sf81MFjjD5pUh2njmnQvWjsHAee+t7I
+fV5yNhVO/sARO2f6RiL5EsZt8eTzgYPNbFkG+jJQDxjoJjjMyRsoOg==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c8.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c8.crt
index 327d80885fab..3891086de81c 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c8.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c8.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUYMQ3IxuiB6pn0PD2FbQpl52431IwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE1WhcNNDUwOTA0
-MTMyMjE1WjAQMQ4wDAYDVQQDEwVuMS5jODCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEA0ExMUocWZ0c7F0gpF9RXw1Uogl79PxBKIc2Jd7YcDVNDWqnyMWNcueeh
-5bBoM1SrPt0q2PDld/rpBeOoBqZqRMfNu1t2J+ij7bvq/eh/7111hiOObqfeWe8r
-vOKcuMuP2XySO4SnStOCpRTbpWJwlWzvp2UewDQ7FKF1tt0BYnsCAwEAAaN/MH0w
-HQYDVR0OBBYEFMZDNcpbk5GD5ynwpCQOY6VDM4gSMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGUxLmNsdXN0
-ZXI4LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-PbfEs/JWjdw5FUuj6+vb6hdpnkv+qCxX9ayZygmgzAd0mgYRpm51DQFyIbeBUYdA
-vqHDiKMT56ODM4n4ij6gWwyzmSxLRbtCTcOC2oVyhyGlwmkYXw/9/JOUMj3oz7SN
-8iRtsICqJwcbTGenvHN8Nfr1qMvl74K0foNE/hczXMcYNQq+LbYgen5dejFbOSmH
-V2oE0e86w7pthriazNSmivWQEeDAJwTR51oOkfm2KMVA1adbqBB7BP8eA0uOO43L
-SNAArh9Bzlm9/5raX/YTL6In7G3/wSFvIuJUYwV3JW57itzaBufZ4G5CCVBulGN1
-ltKJJgSsxo6InUTglO9VDA==
+MIIDUDCCAjigAwIBAgIUbdKoOzTyTVynOAVFoXMXK7VeHdAwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE1WhcNNDIxMTIx
+MjEyMzE1WjAQMQ4wDAYDVQQDEwVuMS5jODCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBANXbcH3OeBdsH37mxPgUxSfPKXNuMo7FdXlqCiHlRtGTv9S8VcOQ
+sh4wBuTmXPry/cgdpESkNIF1lJG/JCWRJ8Qfx9CBiGTjSdUAnX5v0u/ubSh0BApC
+PBt6e4U5gXqtXPHg1MBZj8AbhRsISKq6sVRtwRdNsBS58LXmgx92SThEUF3TNuHl
+sYNGAPhhOhCPA2M1PLeYmkjh/dyfHg32LAOfRE8jdb9xvATAhgb7Qk8ts5/IRfiq
+gYngjm1qLCFsUryEnSrOoPfVp5OtB5kbrsAQ3FDTBNVWOV6+qUBMMrZbhKcPJvhQ
+JUd8B2rud9ehTIeyj6YhCIjHxxm47N6iSIcCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+fTBsfcn65Kl89gCzyClsha6qPpIwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTEuY2x1c3RlcjguZWxhc3Rp
+Y3NlYXJjaIIcbm9kZTEuY2x1c3RlcjguZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQA8m89wtFp3D0gfSKa6yGoMyvPtUj+CcASi3KVd
+4w7b7DR0e5mV2FLz9GKRxXKNQss00H9hpLSL4/uiyuHSYYfTjL+HzXR0sfyV5TDu
+3zsTLE+xLZ7oQv4oDIbXXSJPoNZA90v92XYOMMyz/9sorwOI6rtVX2eFeVQNVTyw
+zdrmL5l7PIZhk3DSU0t+ndS/QizabFJJ+qWKn4fp51xw0eT13Gda/WVF3gX+vYG2
+VBnRMpvxly0ZCFWVtNYMg2yDeT5oaZeFsTPPvOgW72cvQ9goq/Cqp/aMOrXJ3wTT
+XxQcMO1IHVH7hYE3IQUlV8G3oFMCAUs08g7V9wRzGSlNHNhF
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c8.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c8.key
new file mode 100644
index 000000000000..a21c17145c0b
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n1.c8.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA1dtwfc54F2wffubE+BTFJ88pc24yjsV1eWoKIeVG0ZO/1LxV
+w5CyHjAG5OZc+vL9yB2kRKQ0gXWUkb8kJZEnxB/H0IGIZONJ1QCdfm/S7+5tKHQE
+CkI8G3p7hTmBeq1c8eDUwFmPwBuFGwhIqrqxVG3BF02wFLnwteaDH3ZJOERQXdM2
+4eWxg0YA+GE6EI8DYzU8t5iaSOH93J8eDfYsA59ETyN1v3G8BMCGBvtCTy2zn8hF
++KqBieCObWosIWxSvISdKs6g99Wnk60HmRuuwBDcUNME1VY5Xr6pQEwytluEpw8m
++FAlR3wHau5316FMh7KPpiEIiMfHGbjs3qJIhwIDAQABAoIBACwFGWisAUhA91K1
+ycGO9O421D9wZXPB7WZqj19exG8LTKdLhRPWSvOvyxt+15DIR7jTcSa37h7fxw8h
+Gx2ofVj/ea/PmfW6W04+7CxEdFTy7QEwsEw0Sg8V7SsFJUPNP56COwjE5rQSdhmU
+YPDTsQDB/sb/NMlvZL7sjf0B6hxsgxJmaacrV13kDWLtsq0qn9Wo3DQ5ZYlO9dNR
+VSWBD7h4Jo9gd5d9nPGdoapWtoksjT1JJeY+3cW8SRSnhHULVPZD2fUbCgKG8OCh
+Ac/eOcVo/6r6OMSq2VMMYkHeZyuhD/ea3OHeh3Y4jjWsZRSIrhIpefAaLct6AG+j
+ukNiAMECgYEA6G1z6Jj76v1b5fCYqpHehIPZDbTKZSoJWlQvJKR/zSNBfaH3CTbU
+tzIHimLeCDo+bSy0RCAZfdJQoV5JGYeJsn4eu/tuQ9f1Rj1QVwHYimtpaNnf5W6r
+x18BhlhkX8wmZhD0SyPG9dfDy156YtxQKSc6E6nJYYbeibpLbqh4oBcCgYEA64vX
+gVdHZd5leE2FuyAowMFc+7bnreB6dAwT9wFGTCyK4kcr6YRgqeZfMoXu+3WyY5nv
+jIDqgoKTyCZiN/mltahP8B7sZJwU/u5BnsYYysQcSoM/t/9S/9XOHW+Kb+4cwrmT
+gV8xcsCs4M2DoPn+PC7MIhSheZVApkjXidem8RECgYEAhaO5S3n7DyhSGKn3/csV
+eHge6ySSYwyCiC9zU8xJt3eYBlaEJgYi1Jqtf0JNi8mBb2CO3cTfI1s0BTSjL0yh
+PcIKheeRhOy9GH/Um9Jx4VTVLPZpKBA0MNRNmiNkwzgss+73p+SQV3aSG3LdR86J
+QMBPQlMTeUIE3ogBK0Uu5k0CgYAbUKw9I5H0DkZWHSdU+/W95K6TwjI7x7xrmLr3
++f3c941jkD1XZzKOgVio0z9TieKWClPtR0XqvJnGrCzXsOu2lT4v6mYfhLoqlham
+AvL9EKD3QyEwhTtqz7CHWVgEguhy7HJenltto0ppB2wsTtFcyVCC40GWh+hnYCse
+oROLAQKBgHMrkYEyEis3sdi1qU1jFDfHlaznGboySgeIWKPaDPzaWBpCRKjh7W4b
+gvSvraHjPQ3JR+aPI2+hzZcBFYY9rGIgB7rhwmD8oqdgCENLGvX04YHTmygRIm+x
+GUYI685djyaznlsSJoBzXe64xWR0AikVHxSLwh9eKKZmdqQHo+uk
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c1.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c1.crt
index 5153e2a762df..1fa6cac88da7 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c1.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c1.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUNzsUuUWlDaQigjdV2GBNk6NJfAIwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjEzWhcNNDUwOTA0
-MTMyMjEzWjAQMQ4wDAYDVQQDEwVuMi5jMTCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAm8aLIeL8o3DodD9VWa0KvotRwSUtWX+qzAFgM5SqHHymlDnwRJq6gutq
-nCDCos6TGihA6bnlVw4eRnjq8usw7cyJW2iIIFPr4d36DN2Ui/NGdICP4RMClrgf
-5sC+GE1jln0W7Jj3f76jB6HE3BbU6PCHdlAtgVcQ4j59y9dchI8CAwEAAaN/MH0w
-HQYDVR0OBBYEFOMdmsBGJx8YHfiziaJxU93Fc6C5MB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGUyLmNsdXN0
-ZXIxLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-g/ZYdzwpQQQ70gIZ1jUSRkUUXAU8LNTLRx7fGuz7t0RNZ5MoUIlX/9GiJd3BPZOK
-C3fxU6clHn2PA2p6ZyPfR+C7jW8akyHzqAsrawzXg0fqtlOwm1U0njNwt7xLS3Tn
-/naLQoWYuj94rVzoV3wYjehcl2zJovudA2IrDeOGU/vAj2zwTnhagQ5eDQVf6A8H
-kSItIg5kM+7XRYxp1VFgS61B1MGT8JWMATqQ6UyRXlI30HXWOxfL1rmWKmyNB73y
-uMEZLibNAJkAl1vOuiM6W/OHKLPQ0htJnny70vX8nGNocWsAaQyDd7NfXpghS16P
-fgttR3nz3R5lcceu3hKehw==
+MIIDUDCCAjigAwIBAgIUC/58xkjfCrUBQCuvw5F69fSCeKUwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE1WhcNNDIxMTIx
+MjEyMzE1WjAQMQ4wDAYDVQQDEwVuMi5jMTCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAKkiRq1V2jX/FMHln7+vAWaB5ce1n90pKZnbPCeiTy169Zab7uvz
++MH+NIJnLrT3K9MlWw4ScGuTUBsI1TX1TR1YPaveJ4Pom/aTk/c8/Ag/RHNvdRsc
+XI2sR1d9PmVGRYZh6Xb5BgILFszD1/48KdxscXgwW6brM4aw7mJA0X0yhDe53nsE
+WqA+Hd4WBro/zxAK6g8jUAJiFp3gb6N8x/KdSr0ZeeNXkTOfH1jHMpf4QWnhpYV7
+nRx0BY5GeBH6jYKaCamNiAhFJSqwn9cap8mWCBywkArDXHK4ZMqIhaoYKUwIwfC8
+KADPm1dkb/Vvc6pKWfZ5ANpSabUdu7knE1MCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+vkfq/X/jJDH0OX4JOwp1EKAkL38wHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTIuY2x1c3RlcjEuZWxhc3Rp
+Y3NlYXJjaIIcbm9kZTIuY2x1c3RlcjEuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQA9zWccNFBXjV8vsT117tu+Qpvd0gZkGgO1Vx3j
+SKQ0CiKwxttSZvOBhqtUJc8AS1qO+64mDUW5P1zSUrLZ6kVQ/+YSvScHFHo9zDMP
+4VA3PSLiyS85Z9a+5I0hrv6WTeJsh3Za9Vy8jy206nZMLrnz3spRuRdiH56D7aDQ
+aQ4N2e8DAJhW4TC2bv2rG+vkTAu0fd9aGWjo4+7+6h/fMBy8PvwcpN2+MN9dvWbt
+nauei+QnA0PHXOiDlNvHT9ARwZuczRm839xlglU/hbqkFjoE9lYys1usuU/GFjq9
+Va1T965h/cRNTwZ6DIDwERTUH91A35a0bv7Nd9/cW/1Sh/pb
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c1.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c1.key
new file mode 100644
index 000000000000..967d61f2fada
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c1.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAqSJGrVXaNf8UweWfv68BZoHlx7Wf3Skpmds8J6JPLXr1lpvu
+6/P4wf40gmcutPcr0yVbDhJwa5NQGwjVNfVNHVg9q94ng+ib9pOT9zz8CD9Ec291
+GxxcjaxHV30+ZUZFhmHpdvkGAgsWzMPX/jwp3GxxeDBbpuszhrDuYkDRfTKEN7ne
+ewRaoD4d3hYGuj/PEArqDyNQAmIWneBvo3zH8p1KvRl541eRM58fWMcyl/hBaeGl
+hXudHHQFjkZ4EfqNgpoJqY2ICEUlKrCf1xqnyZYIHLCQCsNccrhkyoiFqhgpTAjB
+8LwoAM+bV2Rv9W9zqkpZ9nkA2lJptR27uScTUwIDAQABAoIBABegbSUboB744ipv
+7E/3n4d1T+cFDS706BXRFWcn+k2NJq+CWu7lJvPakfVBernW6W1Wg5DErKWqb39H
+TZl7Cem2gO74WqbHoLniRcI856wzQlmp0T8XSCqWTWUAXu4A0geCt/1hriAs8mNq
+XErslqBvGa7Deponswv2vWZGTb1iTYk57NuLptB4xndraaf5swpEc5kixSz/nVtu
+CCW19TIW5AKbds6VIubBv9AueNmdn+O4iRqIUTxDZnlUa/Z+wGl367cmzfswM/eO
+qSphWJModroIVG9KiCe3FgHfSFihugS+HmIplnqjK+DNSO2AMgrg7wQ+60sggU5z
+lSS+nxECgYEA7Tm8QF4UIvQTKOmQMkt6M1X0Nm2d+7dx8lhCQPvTsyiEpZWVxUTp
+2iUUE1HMvqc5zf7CqCqUMAnL0RArpXkcYyzCgjiAma8nt/qxmoVc7bVVHXHM3R4t
+CPRwV2OPlGs9AbMJGDnoX4RLfb14nzvbSRol/WI3QhMiD6UZeQ87mysCgYEAtoT7
+hKAuiWBmYTGnKNBTC0WNvkvcOqhBiBkePGt7QLCXe10pdTKiIcQdo+xn17zGkuxF
+BilXkEozlE0IajbPYChdcCWCHqPh6s2pRDTlqLphPrXD8rCp6mbl05qq63zR2lK5
+Y4T/jhf4raqqleWlEY/rQ6/p9GjZCiVa/AxiNHkCgYBlRNVZQRAlj74yI+rbnZ7w
+FUxog8kCcOBizUyTQy2veKePrFjAWtfduSBl0vCtnuOtTYk5ktup2jS47cdEH3x/
+1da/EnHLqZsyDgINbcsq6tMH+9GxvzUIcpkWpTjg6hqkBBLo1aeeRvKXHfBFQZ8u
+CMyY63mdal+LADbsmBUg+wKBgQCnYhFfsZ678aIyZFzOOy7NWX4CFIPlb0SZ0Z8+
+/brMuqZRRkCs8EBsAvTDzhv1Z3QcbX1nhyZ8dvASgwrCjX1ky9/U5zELdHPM58rh
+eDldoX9pCTyoJofNJYyDoIY66/9v3wmRUPkkkKtIaQmD6lSVZIz3SC6gLD9O5K8q
+ExlE0QKBgDHiBZZRz5AQcyd1hiyWloaeUbNXlKR3XzP9hkvZX3+iZOQb76A9hIlO
+Ddfz3dgFtspXEiS/nyC1RhwqLkW5yGq0SGsPU0rsCKZCeKqa9uFkkr8ZukXRacd6
+XbYOlhlZYgq2ShRq5q3zJ5ixuWdFhJo9tCprvjAAjMsDDFUB+bta
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c2.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c2.crt
index fc88a125990d..56c860523b81 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c2.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c2.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUBC0p+NZGKecWH4gpX0rdVorODmIwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0
-MTMyMjE0WjAQMQ4wDAYDVQQDEwVuMi5jMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAwD1plbaw0HGBzX3FhRKZIbz+RncMdhwHTZFLrfLbRm83AEhvMyhrBDAP
-1nyB+p/3Mnsk+eaTK+Fo/KBjdakbMLfS/Ewb9cbr+k9ZbTLGXIvULrRqefWvgFN1
-Fu+azjNrDqHS/qh4833+hnqyCM0TEEGxkRX1TIvrdbQsRw8j71MCAwEAAaN/MH0w
-HQYDVR0OBBYEFLwdJ8K91/Xq8cEGOZvM9/4G1gvPMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGUyLmNsdXN0
-ZXIyLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-FdgI/IHCkZKV72pjgI+0Pz9nOvplX9SdBHYOQHl6gKdaa7RsBlqbY6VIFXG4ZNh5
-H/DWTixiEhgquNQJbum7LNan8ugMdsQXXo+WBWOgz3mYYHcPWvU7w4pyPUraXkC1
-QbsoCqY8senXl1C2RBq4psw0jpobVZWqQJxR4RRSV4mb8E6wMa46xwXTDqoTgYLN
-qCh7ywhYYFkWIhfC/aS5D/kigcdU6IpV2hrNvFrFsNwCiNoyHmRy/AUDhGOE0BRp
-8WjHLE2gTsR1hIF1jNIN7vebkdqq2igxs3wkEp9tN7af3mjwwIasi3ZCjKoQHVUC
-EIwmFsFPsgQINx86HRWoeQ==
+MIIDUTCCAjmgAwIBAgIVAPA4rJtPIHtQKLq3nLkTxzcOQUMAMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxNVoXDTQyMTEy
+MTIxMjMxNVowEDEOMAwGA1UEAxMFbjIuYzIwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQCrbO05PwPelGHOKJ0oSLhAtqB/1KSUPrpvTn6M1F4jxhK97SNR
+eEk3sqn6Hq3wVyzxJLKMc3NdVb6dH1b9SsEFaDtwhFt73BZVo618q8Cn8P1jBphJ
+DK0fKDlr70cOBJNmqFmbvqEXrkMsCMYpEFqsT12jZoIkTlmn/DaepRRWegxK98Rb
+gp5JiqdlFC/JCzErenT2OPxiiTrPrsq0O7KKBvNAgF1YA/9k32YRHdcMlV8O2VAV
+V91cnfpgMYdtr6qv/ERxPYmWSRx+ijutDz7vEc5cgpPCDyqOMD92jWPcQZUdtQKV
+cjYObV/zMdkKi0DYJEjRmYV0g8vQ1xfmZEc3AgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FIwNH6lBMg95FcJBG64veXRzzp/rMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGUyLmNsdXN0ZXIyLmVsYXN0
+aWNzZWFyY2iCHG5vZGUyLmNsdXN0ZXIyLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEASS9J/2w53YlEkKVEvwrlkZtlBmVKBntydqLe
+J0nQc5gT41mEAVjrybHAvuyh9Fa+YSVkh3wgkS+3/bJZsYJQq9ePI1iCnLA9Vnxm
+xE39o+U5iDoYB+Y+3R+arcUlQGUafJai+nlljxKUAPB6xKMHHbGtdLBpntONDfKR
++MbcyBc/TQ28UHT0TreFYJNyVBfdD4meNKtz5MbyVH19yeIo3VZU9c8L3NXeWoXv
+Hs0pWH93Q/DeRpuhT+86xF1BEzpzIPTyEFWR3aChgqaGSljAcN23BFZxViXNmAaD
+TINrJowOD3wbN47rxTcgsPa4NpzH5+5sOVHG2VgQCmNnEYMDZg==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c2.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c2.key
new file mode 100644
index 000000000000..2726de654f47
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c2.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAq2ztOT8D3pRhziidKEi4QLagf9SklD66b05+jNReI8YSve0j
+UXhJN7Kp+h6t8Fcs8SSyjHNzXVW+nR9W/UrBBWg7cIRbe9wWVaOtfKvAp/D9YwaY
+SQytHyg5a+9HDgSTZqhZm76hF65DLAjGKRBarE9do2aCJE5Zp/w2nqUUVnoMSvfE
+W4KeSYqnZRQvyQsxK3p09jj8Yok6z67KtDuyigbzQIBdWAP/ZN9mER3XDJVfDtlQ
+FVfdXJ36YDGHba+qr/xEcT2Jlkkcfoo7rQ8+7xHOXIKTwg8qjjA/do1j3EGVHbUC
+lXI2Dm1f8zHZCotA2CRI0ZmFdIPL0NcX5mRHNwIDAQABAoIBACgtCbXqcl42eWun
+MhnwXRpxktESxZWm0+vlQhyAYyXQNLFFhAsCfWpr2ZFwokRbMgG8H4pyPl3oDBnn
+7+vCGtXJFXEr3AL91FAy4aR/3lGd1czhDUzFh0dvlnGY7Ra4dNFQ8FtjhH63zS9j
+SQnztsR+f0wd6K9Ym4OWxThOKSOQ6RglGdZE2PWyUi11WWcl0jebXfBPwmCLJu1z
+6od1OMcMIy8t15iQJ7TFG03olH1aK8dU924rlmRupInjtBY8Y1fyEjXQT3y9B2l/
+3SF3DN6tNCygkfq0wfxzrrvzIpcAkP28Cf+hEiHBD3PSXKFYsCNcPyy6FJ2crvgq
+DS+UWu0CgYEA7cTgYsS8E2PO57SN4AiY0DleW4b68zXE4ef7vrxGrWkU/L+nHcit
+ijNOBH4/1bGw7LyK/kssJFTtqKBUPz+RhcO98g1HHsXWFAGDA+oUv2R9SnAJUE/M
+o1lbCPkEHXMZVJo7DjkOeudYdHJCi0pLiG59afd/7AIQ1AT7JI8R2QUCgYEAuJHO
+HMn1HYKAH7rMrsvrHkLtevwX9vxj0dLtI/8YRo0kS0R4odkYInV3Ayudo2LufqWY
+oltTQaA1tR5r+GdPy2XOodyVM2L6lcaeHt0Z8ZMYie0+oF6nwNRrKANAR8RQ4OFl
+RmTjgYDytyMig7dG0iNCgiH5G/s/obAO1GeSZAsCgYB3zs9+FUHrx0wXpvwiWlKe
+AXpRRlenO33Ekz9f4cD/WkbQAMZ1lIwVajqyuubxh8Lt1yd2cWyBtiW25WruH9r0
++yf4Esa2c9umYwigbVAdDMaxPnkC9eLYoIrln4x6RmgKIFkHlqp8NZx1uKldRIeA
+7KPXRHY59uSB4SynREwq/QKBgGj8z1CC9MH/fKAP4uQ/LJP/IBUIFx2wPZuaLrh+
+H7HqEU4bOb9evSLEzJjqE3ZgDSfPTH52EmrPFh3bmzEmmzYoyBw8XEZps/7Ehycp
+P9uxf5DdefVBQo0mh3mwN71bB2KT88bSFrOQxP+1palk9I0N74QUjR4EkhKIEjdy
+Xsw3AoGAdWS4ZGXZo/bePtTMaakba5ju8rCe9gYBldL7venkwpYEAbP9WKSqkS3f
+xiaW8zMwXdjqVwAbdMNDz5Ci52g/mRkIyMl2Mi/eW0FslKM6rNVA9EBLT2YXPWms
++U/dnBqbaJ3qRMbqRNm9g+SfJAv2S41WjMUzln1dQlxTaHID1S4=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c3.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c3.crt
index 14b9e54e29ea..b1e4a5d21624 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c3.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c3.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUJOYUNCOnak39xRysYs4KSzQALNMwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0
-MTMyMjE0WjAQMQ4wDAYDVQQDEwVuMi5jMzCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAqWUXhz1gV5BnMS4T9gqVyod8OJL62oQ0nR0WPdyv0yODkm7cmMcvuWVk
-2sPtyGHv1gyKcFteLsBH/1TPV1zJGk/ecrCJlmyfyhnJYKuahf0SN4U9EGmtar6o
-5ja5RVWiEmAbNnpJQPGVrBPLI5zmhT/C5hvieY2sGwZP22e0c7kCAwEAAaN/MH0w
-HQYDVR0OBBYEFIx2Qg1dSxouV+t8f1G7G4qw4OfxMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGUyLmNsdXN0
-ZXIzLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-diaaL/4cK+ycmIuyb9pf70PZ12/mNSOovYX0G/ZmhcuISxIOwig2LWz2DPlTs2wc
-QBHsmiIVy0cvMM+cHgOgKz/gr1ZgkS8CoRblLh6gKHVrSkixk6oz8FABVrqmkSMw
-McjMft3+7thITso2jRZJWRl2sbWN+cs0Rwjw7HajjHSLbmLu0/6rwioGTY8TpQJT
-K3t0OkvBFCUQEsN1pkkXgncAoxeMQQg0fl8IJ0mrGyEjKgtqUuDkDLVdOY+y8oL/
-vbzpNHb93LbFl30Dl0Z4ugcUDPbp5tDzmeHMFbPGcKZd9s/TEInZbb/AfpGMq93t
-t4MVFO6u5XaqOV792rc+kQ==
+MIIDUDCCAjigAwIBAgIUNir0RrTNgwE1Tjq+NC31my7H20MwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE1WhcNNDIxMTIx
+MjEyMzE1WjAQMQ4wDAYDVQQDEwVuMi5jMzCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAOj3BiCHYQRsBQ5WcuztaVmL7768COUGxdH29ksxhQ3Sq31G8MQc
+Zq2l7O6uogfgiGxNSUvg69O0S+S42o1DWx/lazryNcg6B5NX/sm0YF4Msx3mZ8VO
+VorCYHX3BFK80DdJ2DXgKAVMPsmBkrbBrvegAQOO4slf4Yex3T7diwKKx/OyzKRN
+DzzxDwaC7j9FQOOfODBr+TCWRtwHgUh39BM5930qBHreecLrl8jZuUmzsBk7wr3N
+cS0feXZxsnYuZ3QPkRtQcO/PRvr5RFf8Y4bj2s2v8j44H/8KY4/LtmxnNMi4CIyE
+7IKUOVzeikVbgAtVJcrT9EpNGBQgzTkPEt8CAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+aRREEMpKtOXrMdf6ANSJL0ShhMwwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRYIcbm9kZTIuY2x1c3RlcjMuZWxhc3RpY3NlYXJjaKAl
+BgNVBAOgHgwcbm9kZTIuY2x1c3RlcjMuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQAKuIGVAE6yViFDE6CMy8foDRZVxJZMxHu48YVs
+upPUqPIa9puTS8HWB8iNGEcgUyEMMTUmnUNXMWvHQtRoCgTbJdm/Hx/IuFID5+67
+BuzMet0861vjEc0fhXsP/wh/j9TzvmQghSwgOsqSYgOR9VoQOWE+gST+u/u2XDQU
+d181YRNAFhNF7lc3JkL6tHN4rjzT6o/iQlsCaId/7/uRvXXvNah+aJ7YYZ8ivVWt
+XGI+jPmEWvV4F6zNGAsh4csg4TvE7WzDAKuoVZR7DsQ772F4I935iSSI1BxANzpF
+2DPT8LPUjUlVnDhFn3uvCWu9E6K9zowPWVmfuUA9ii4NOAi+
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c3.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c3.key
new file mode 100644
index 000000000000..38e15acff655
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c3.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA6PcGIIdhBGwFDlZy7O1pWYvvvrwI5QbF0fb2SzGFDdKrfUbw
+xBxmraXs7q6iB+CIbE1JS+Dr07RL5LjajUNbH+VrOvI1yDoHk1f+ybRgXgyzHeZn
+xU5WisJgdfcEUrzQN0nYNeAoBUw+yYGStsGu96ABA47iyV/hh7HdPt2LAorH87LM
+pE0PPPEPBoLuP0VA4584MGv5MJZG3AeBSHf0Ezn3fSoEet55wuuXyNm5SbOwGTvC
+vc1xLR95dnGydi5ndA+RG1Bw789G+vlEV/xjhuPaza/yPjgf/wpjj8u2bGc0yLgI
+jITsgpQ5XN6KRVuAC1UlytP0Sk0YFCDNOQ8S3wIDAQABAoIBADhIkgzi2EAivtf3
+b92pACZXxizdLgLzL3DqSxfgl2Y/CxqBvKKMwAvdBhCmVVQ03ZDY3Uf9R8yGhz1z
+oa22Y7ohidYcdXzx9CmXfWGK+CtQwyeKpnCHXCY7DuYBL2Sz5FQvPZLKnLCm+pZh
+fSo1QgnmeMg8ybLgPNM1WCAbkz2iBt3U9zolD+dFZqsa/ybJmPYhhZd3fofulPoF
+OuZzbBZzrUISRtp74Z7+y/ViQfK5Kvj821VTwcR9dNYTWHRhtJACcKA6l2WurGhw
+CpzJfuXUNL9+cLRuug++S/Y9MbHohRLnR16V1aVKxEYpqr9XBJduzSyWzw+L3p3m
+mW/cJQECgYEA7QzlfFyMpu9CwoqsZJqPJuggLHOos0XmEMsmM3HHEyow0y7/566P
+hjwOuA3DmOhTk7LuUHX+Kwusyh+vGtB6thnBXLvXTmalqOLvJlq5yZzBGjlp0Cqa
+1Iy46HWxPa94kxtiVF83neKd95hbvsGfkjCjpjp4Oj9gj6IDOqdrqQECgYEA+5aF
+cdwlr0CevOCSWAruWtqT9saAtV2eVrMsdJMotXwCSHg951VjGnIeyNmNUD8tfjkd
+ch2oVtpLxU+gglusSeN+ONtd8in+vvhiBerkKuXsGCCFvF0819YBhHiMN2Jg3l2o
+90Tx2xOO6p0iVByoPtFNI+ZXYhTmoQJREB2z298CgYATf98UxswvIzlbpWqe7/Pc
+oHWdEnqLHLHbKcrfqnlUw0C27oPQfBSEV0Zir4cqedn+KVbfOYWJ5vD0w3x0+TdO
+6Xl1dmI0qc7wwJIplob35bQgwx5IKGkKGPVQ8fV9fISvnS4OgLKjgzO5XUgWvfix
+XtznXKHtMK9xwUMAvSrnAQKBgQCSdOpUAOrzHXQK0dR/akWdIBb4lk/AxLHzGLSt
+XClblMgjddPbgiqB76MaGwMXjt+l1gvdLYScotFvGE6A9GrUTJli8di7q1S/yQoP
+FdEjY9tPpEcWA1Ty9ILHogFGEIABkpZU846zkKwmybh9LH2IrADzABkoYLBtVFLq
+RdA5EwKBgQCf9s9l92hfpFP58TCTmQrCUZsDjyuPnG3tgDi/oLfEQRnN8aaUW8sE
+d5Qr432e50q+AOJQmyEzYLVJLB/aiK2Nk8v8Ltx2OOxKaOcimY7FX8bZXNRnY+zL
+3vAgn/ZkCjFf4c6Pz8DnZjAPQtweAoyxSTwKl0Xf3SD8rPInQpshkw==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c4.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c4.crt
index b94e8b5922ad..a225b3d73cff 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c4.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c4.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVALfaOUWIcNx5DoNpszbsieO27BDlMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjIuYzQwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAKMaYHZ4RiQN/cACOxN9GEyNtxwXC1PD5rD1nPCH0Efyg5SLcDzbBj5Y
-VS+BGl5HIQ3hwX62vnI9pRM8GJOZXX4UGHQ4gdd3B+qbW3n9K29cSqnxulIZiE+A
-x6LOXwwEzSag1nTEpJQo09LZPusyzXnicEJ/RpC0krAa9g6gnvNRAgMBAAGjfzB9
-MB0GA1UdDgQWBBRsXxUJ6YradZoLR5+cQBXgj69aOzAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMi5jbHVz
-dGVyNC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AHe8DhnLXUPvfbKWV2h2NVI1x6MuMBFcDQ7HgukHwMMydnHclVZGffXu8ZcFfy4T
-NxQ4N26irZlwjfpXSpZeE6b1fZ8x8B+3czXSZRGJIi8muBA2/YJjH1ul46z6wdgr
-aR3RYQTYzEicM+qY2SfISEUqx0RGHxVhoFBwNaS/EZWXyi1XZfjD4XQri6KXJIgX
-yf+zr5V0ZVbNSEci3gUR+TfjG5RWE/r8XiagZ+h4TJidAjgt0CPc2vONbgcgcb7k
-mhRA92z/eT1etHaK29Ne13p7Y3NI2MxWHvyygGICBcP5DsthRVCNAbFFq7N8ym8Y
-2ensWXgNScolc5SjzQLUwss=
+MIIDUTCCAjmgAwIBAgIVALlGWZ7VBq1IWG4oiSXuXQqrfBqXMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxNVoXDTQyMTEy
+MTIxMjMxNVowEDEOMAwGA1UEAxMFbjIuYzQwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQC0wzWsb0meqjHmuAYqRLX0HcK5yVRAYRacVIN2pqsDYxlAd/0/
+8yS69w0OScV+iiWESC4TaBdUN+BoSacPhYSe/3H7G2zuAJMg1eOh/spy3V7f+nwP
+LY/LaWAHv0qZYKqBDbZYz/59ZVmcBoDJr6OELlS0hfRfPp9l8pit0zMV7ATK+pzi
+OXm+huTsYsNbJrBzLAgkeR9WwcdpntToB3CaZbTQE216sVyqvedPJcKQv87GcDwQ
+6uSD8CrmGthA7Sf+YtrI4GPMI6sO2+v5Q2/GScFZqPG3xkAgf41zwLuVYkhIpDp7
+lQD+UgJodNL6PhAUuidpZjzIJOE3tYzrzMqjAgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FGqBnUVwwE9FUmwTszJocWx8+6sUMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWCHG5vZGUyLmNsdXN0ZXI0LmVsYXN0aWNzZWFyY2ig
+JQYDVQQDoB4MHG5vZGUyLmNsdXN0ZXI0LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEAVohcUq2ocIp5HveC8YBgGLssx2bNfGDSqbue
+nZbjsmzWVEu2IQc5gyHzD1GcQWhOOy8k+coN90KBeVHLjm4uvQQHm5I1NjnzvJaL
+iuRUpPJg+L3rvWUDmaSeBLCzcNjuNfxCmmHT2emyeotcPaQxjwxV1BLBXfmZylWq
+pb8WGuR+uTlkMql29cLtqdEYirBpHiJTaqBTjMF90XaQLvng3W9td43D62S21AbQ
+l5xyVgtJlWyTh+3brZOVnomaVscGltN1W2+eH8nN4H4AQhkimC+jp9e3JBCvVSHA
+fJaGVIXZQ5ZX9qjVDBU96bqqGt2bsLy7dRqI9VlmoFBfX3eDGQ==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c4.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c4.key
new file mode 100644
index 000000000000..d5d3271eaa1b
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c4.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAtMM1rG9Jnqox5rgGKkS19B3CuclUQGEWnFSDdqarA2MZQHf9
+P/MkuvcNDknFfoolhEguE2gXVDfgaEmnD4WEnv9x+xts7gCTINXjof7Kct1e3/p8
+Dy2Py2lgB79KmWCqgQ22WM/+fWVZnAaAya+jhC5UtIX0Xz6fZfKYrdMzFewEyvqc
+4jl5vobk7GLDWyawcywIJHkfVsHHaZ7U6AdwmmW00BNterFcqr3nTyXCkL/OxnA8
+EOrkg/Aq5hrYQO0n/mLayOBjzCOrDtvr+UNvxknBWajxt8ZAIH+Nc8C7lWJISKQ6
+e5UA/lICaHTS+j4QFLonaWY8yCThN7WM68zKowIDAQABAoIBADNzdO2V56CP1RBU
+xPx2/ktzxR9YnPCOMf9hVuII4pheNqvQ+3aeYJAPDdpJSTuRWshiOysu76T7GpbU
+kAas/Gi9UNYmokTESRyJhPdxLHKxa/XYuEMhttIxNjI+Up+BkkaVfpec7FUI6g6t
+Ey9VHPfcsCOKFVqe2oQMH0Uq3lmYj6U6LwDICDhaCmhg2JXBmnyWf2YNoJUqk22r
+1XJvsZXD0qCnckaGjFceiWpQ6l49XcBI5+ylbNjpsIASgxYnW3Cv3BF9AnVcjSGI
+uAlWZrc9+ixVmXlhNsRvQXhYZ1WorkQytBxGoxNmxw63MnKVZ9cTpydjPDzDSb6y
+9F6afqECgYEAyCZdEbOEfFdGa2w8Hq+HU0ly1MXPgvxza2Rqx5Mg5DPBO1u3o08l
+1fdilmWouxiU+ZKjDV5PYjOrmTksYQZtKLUiL2wSNyb3abmtxRflM5wZEd+/BItJ
+1QSVL/61AjY7UBYS+IOWYfmBWv9+LhrqMW2jwV6WK8naM+woU5+csjsCgYEA5zPr
+aIDepBW65/hFJrwBgwKR+pLvdIQXnyAVkgH3ibfy/tdjE/LT64TLUhuoRs061ax6
+UQNIoB3XCQvwlzkKCORdx86IIT/g/Pzwie++ku4nVYACmqdwpDtChtph8K1LT4W/
+HiL+fin7Bq+p99Igpb9+uOQbW/OfCurtjH8CGrkCgYEAhLlD8FAN4/JwBR5CA8s8
+PtOu7T+7as4Dr8xoLq33GrzjP3yDmy0KFwg0fS0updsYNyOxMlU+6Q0Altvd/P0o
+99ydrbrJnX8ehCKYA+HjoMS7rymjXt0K+dJPKAOMfsn9rQ7mR5vi5ldxGynfv60j
+beGZvMn/ZUw2gUXyO/nVDC8CgYEAy59PafdtnkfBGAHxLT8Receut2fnWs/nWe7Q
+8Imo8HLAoS8jBHOtrtSCmSrglpKziXJhO65PrSz6sO4CE71ZmCFAcvPzsozm4wkF
+9AXg6tiybbvTD0/+hbXnTotRDL4jAMODYdBiIM1qVyYo5Kj8td2khsPnUKGMZfoo
+Ar/tI4kCgYAdVk33vS/Y+GjwhRPQhZA7KYK9DiNsDZFo7yxhlPl6bpXW41opl19I
+m8lPeOST1dI9zwwK07Rnsf6xJ+Ynobs852FWMBV66AIb78fvHaZ21eLl6NejPldT
+rXx5LrIal64JYemAAt2XvvVl3xFDa+eoIFr3PZ5FtXZkM2OBcau+Mg==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c5.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c5.crt
index c7fcbf3c7d79..b99d8bc89d24 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c5.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c5.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAK7v7QnhK4+lR3ne9GSfZiGiriXiMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjIuYzUwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBANLAWJxq07CPGG6VYspUbzyEMn6pBE0z9Or69wrDOKM7CVM8CzrN7aw8
-zYDLuOZy4cEJxLJKvtw5wQfh/JELcMH2WNLIFNABl5+m+FpNLBGgFMC48NUCfPex
-D7JUPAc7oiaZQ/CFCP+5YK+a5F7M9/SwVfQKKnsMPK3v6zyZkFO7AgMBAAGjfzB9
-MB0GA1UdDgQWBBTKJgGOBVnP8ZNUNlZXD0iNkRRDEDAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMi5jbHVz
-dGVyNS5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AI60ZuBWX9IFBBfGSx66cDVV6lCiwtl2h39SHDQoCPdXz0uLdpJqLEhmpQwkf25T
-j4g3bmfRMTZbsztEZFNDUkyX7hlgmTEFmxyx12Z9NtOTzp9xaZdhAr7fAwBRcWjJ
-CHPjrfjR/5CVXtb5XcGdGg18x097khfFOdubyoty4sJ/4wT4zoaZzTLGskzzvvsn
-BSMJdCS2kM7ITGxsUYC2r+LzKR2/nX00o0xefGzgJkPr4fdDwCxsobYWspgt2SkZ
-pjHV1/Qf9qdpi4wxcvZtVh3+YT5WGJRwTsbBicZ8i/gj88qRi5cqe1XiJ/05MGrt
-XEM5DWun3J9yjav9v0jfZyc=
+MIIDUTCCAjmgAwIBAgIVAKBUQjzcNrPi1ZjAItNKdgM+K8OvMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxNVoXDTQyMTEy
+MTIxMjMxNVowEDEOMAwGA1UEAxMFbjIuYzUwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQDSlxfS27E0yYRF4iEY+t1vDkvWN7sgS0QaoJcBqg/XbDeZlq8L
+6jsIFiMewikjNCx8Yk12SoDDORR46AJ/durbCLDRwXrMZJ4T610SkLINrzrUl2sS
+jcji32xd2sJE/69eOB+pQ4Tnc8Xiqi2Y/3eSsLIH98yob384yBgM6kyb58RCWygN
+EiK5YBMAFXlyeUBBGMSoqNcSxBXjBIdA18gnrMMd5rA29QBCtidVKjQGqpUKruMp
+gW8W9B1XCuctYx0ZHzCIZpvkWzo8GdBusIaGCsiH3JJ5foNw8RpJN0g/Ja0OFYgI
+3ZfJGotHPlAg87Edyja9PvnnmuQ5f7MUYBz7AgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FLJa8AzfXWJwVAZH3w2939hbVbm3MB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGUyLmNsdXN0ZXI1LmVsYXN0
+aWNzZWFyY2iCHG5vZGUyLmNsdXN0ZXI1LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEAdB5GmXYTCXhO6Vr1eRL7IVQqwWJYfNozY//F
+a8LBgDOPPhlU1zC+siroaPirhb2oRyR5c76Ar7N9uC+5EJt9QiBTCiel/DJ2OUfA
+XLa9seTj2z1csAgDC/dlsStI6EnE5JFmE4DmN/jM7SEtyS60F1lp3XjAcP4reCoV
+M1Uwrj1ndJvZkDnYmhXJ8MOdI6cmQXY0xEsH+zo9+gGC6dyn45O4HWO3LTmPGJD5
+RDHZANZWbtHt6fx6oY/fFzRFROdRrc4G4NMlTeLCNIXbJhFLnFg2yKO5LfD7QaZ9
+/7f6k68enrdSUlRSMh46VV6xUxk8PG1zwqD5zdxn2UvDMyuaOQ==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c5.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c5.key
new file mode 100644
index 000000000000..472e85b80c43
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c5.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA0pcX0tuxNMmEReIhGPrdbw5L1je7IEtEGqCXAaoP12w3mZav
+C+o7CBYjHsIpIzQsfGJNdkqAwzkUeOgCf3bq2wiw0cF6zGSeE+tdEpCyDa861Jdr
+Eo3I4t9sXdrCRP+vXjgfqUOE53PF4qotmP93krCyB/fMqG9/OMgYDOpMm+fEQlso
+DRIiuWATABV5cnlAQRjEqKjXEsQV4wSHQNfIJ6zDHeawNvUAQrYnVSo0BqqVCq7j
+KYFvFvQdVwrnLWMdGR8wiGab5Fs6PBnQbrCGhgrIh9ySeX6DcPEaSTdIPyWtDhWI
+CN2XyRqLRz5QIPOxHco2vT7555rkOX+zFGAc+wIDAQABAoIBAE4swa7IWEWgwyYQ
+FjpIMjheRb+9i40m856w263F/EB7jz5IRi31ozlSwnHSIORNI3B/K4NMj/UotNVX
+R0obEn8CAGW1XckPVvFND41rQnW7BWtS1HAcSyURTJCguzpS8j5XNkWm52jmPODy
+TJ/9wd0mE8jKb+JdFYy2FtuAhLTgzTg+i1vV5tg2MPxXFCLS0GtMiWxtuzN/quaz
+x8+zJoKZCxttwJQUtjpuDL7gKGGNVOwUqOHNAQ9SrrcS23d4RXVQtwYc+sG4asdc
+4nslvuw/EcqHhrFoq1PTpm0zhHnSojC+JYg0VAJOAj02kN0PyECj+Rx2hrEuOgaz
+yGRGGxECgYEA+1P4M+Udvvj5W9DR0lq4VYj/yUE01PWjK7xZE9L3Q+9yFTFmR3JC
+LS4cnk8ChUXJ/s5ABmVEt3C0FkXrvaJm1QoobjNvESnNQHFxrnAlL0YkPYFYuuJk
+blOFiPXcQDDYmjLEGhCHxpuHuY7jh7v/2t73sQtjvspaWvtq/kRfGgsCgYEA1oFC
+PN5nZ8vcKXMaYL6l/f16o+tU7aO684s7kERGsxnD/2aiUtn45ZPpBCJqIWEr8eu1
+MNkdg5MfZtoM/T/rc4bMlK++57nHg3eFXMSm0I9XXyUfPPX0UyQB8awlbWKiM6Ah
+RRNNC1Mg7U7jZbuVdhh5uWy5+PTNzjwkNza1ztECgYEAhhB5sr0CDA/lfCu88TfB
+Yqcswp9M3liVjMvwvFlp2sxMj6+FPpFdaZpSW8QLtSnQIWLF1mOIr02oUsVDhimu
+LvA3Numq0n345epd2M4rl7cTt9dajH4dUpis/60eqwMjV8XCw6CTvMbUNJBbINmR
+FqCfgcpwj6KDZkr8/Ntd25cCgYA8FvOuR4NiMVqYhfUSuvWXJEYR21/sPlyl0xA5
+htbjX0lnp6G7YfJ2pau2C8n9VdJQr/4PFBNPg4xdbKIM47Yao911nZH2KjX43yeN
+6Ezuyejo72ZD1oa+L60XvWiyrtnPaoFHb3O6w2vK7rmHCziAX7Thu9KRKkrpBjID
+AaGGsQKBgQDmPuOg9nfdmf0HA16EKscnhZJAZsosjDfmoBARGwr99YK2iAlMk5pz
+HwJoOF9etmx5c/nQwxcFRsarbtPTJbRmvbnfvUVCs+Ae4W+uSwCOCINY/GyCaGPb
+NkScTn1tu5Ge8lEjdPiSo63+DBELPNO9XL2Jn9ckmDxH2HNH6nyTkA==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c6.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c6.crt
index 649acff7ce63..14f8dcabadc3 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c6.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c6.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAIrh/Q7uY55aIFW6/o/ToRCGAKLAMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjIuYzYwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAICwXz/hU7DraQWX8TJWd8hva4KzTKOpq8JjnbCd1n3ovAqbDP6GkgVA
-JxY0eMFwP+u4jN2JZuTISWIUOmLGosyHpXsXQwbsvDLUC0JPb6v0/omkLFsrIXgB
-t2xIy7Bt3eifKUmUGO3ePi+LbgKYq7YNy1uVaFiYLR3PRul+TCXpAgMBAAGjfzB9
-MB0GA1UdDgQWBBR8+r3Wp4Dc7QfmCXkTGgz3gyfP0zAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMi5jbHVz
-dGVyNi5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AK3Q82RwW4pqllyno6FO3OGOM3Dr2I6RN9N8A4eJShZXpyIvrgSH0/WSYolkGz2/
-NobQQBHFUYcjWtwUdoCTNrYXrENbG6j+EmW0ey5VBZNHyLc3clufGMc76uJUvARV
-OLeyeSuSjJYxUafwacIzjsJtJbmixmgweKDI8jmBIuvSohnn7icwjGx46nbagBpJ
-2et0zT16kVl6Ti/GjEYw3Q2q4iZfWaQ7WvqVYvk0gidbDwoJGP197MexeU23M8tS
-CcmTLGoj9LbW6mc5maU1pxD7h/cFoMjgHQCDsV7tyoex5ZdxrUVaBKqGT4CVrPXK
-OKyAMVQkOvxq5/QsYT+MHiU=
+MIIDUDCCAjigAwIBAgIUNahDnbmVJBMJpw9oTkc6UAzlebswDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE2WhcNNDIxMTIx
+MjEyMzE2WjAQMQ4wDAYDVQQDEwVuMi5jNjCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAOFeQkB6+nO/5i846Io3oJrNUgx7cKSg/UFMYHGzRiBB9/OYN5J1
+WgkTMRGzOgGCSE+tvEtKi7jWhzufpOLAn9qKuiv+ASpoFzOLA03ncmHxVyMJIlIm
+z1w8DeEVt7VlKNOp8ZOfJ+vViN+KJdvzakI2Rrwa3ZgDJ/CB9AqA+xQoFiq5RAyx
+pzzlBQ1b41Zh2xg9/S5UutYPTOWamrap9/KYAkD5y6/7nrxaivrVAfOQH7Pbzcvr
+soGep7iN3SanADbUBOrGW2p5W12c7ln7uNq8C/aGU/d24dGNvHyvPYCeYWTAzctr
+I8k1ckHKTf6QY9E827pBdeQmxuJ+vvMBhBkCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+Fvb/fRTXlJ7vEYmB9oQ7en9NloUwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTIuY2x1c3RlcjYuZWxhc3Rp
+Y3NlYXJjaIIcbm9kZTIuY2x1c3RlcjYuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQB8BYgOfCZPrzQG1oIe7uHoSu7Zht+u0VINvGvB
+RQ6sL1ggr/NXaWYkTYH9RkDeJTdCVx4ke5sjCO2mKaaBDWxLvDIIRvmXn/+R2Jmz
+q06g4mo5mYw/4TyLTqBNmgu67rw0apoDth6pWmxasfckW7LQ0Ao7sutaBH6c7S0q
+VS2IixcCEOvRO69aul1k+un/tgBXVrizHOfm3k8K7p5m08qE1+1OVVeF8RwUiQXT
+2TYhCPk6JJCRlGuscUDpXOyv0j9Lb0mkKB3VFmh4JqUiTIfKonihUPvkPpKXFJjL
+jAdTIaJpkHaaCt3RTBxaHfQZDXuPoMjYo2abYnWLTUW+Q1pW
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c6.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c6.key
new file mode 100644
index 000000000000..474b03448708
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c6.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA4V5CQHr6c7/mLzjoijegms1SDHtwpKD9QUxgcbNGIEH385g3
+knVaCRMxEbM6AYJIT628S0qLuNaHO5+k4sCf2oq6K/4BKmgXM4sDTedyYfFXIwki
+UibPXDwN4RW3tWUo06nxk58n69WI34ol2/NqQjZGvBrdmAMn8IH0CoD7FCgWKrlE
+DLGnPOUFDVvjVmHbGD39LlS61g9M5Zqatqn38pgCQPnLr/uevFqK+tUB85Afs9vN
+y+uygZ6nuI3dJqcANtQE6sZbanlbXZzuWfu42rwL9oZT93bh0Y28fK89gJ5hZMDN
+y2sjyTVyQcpN/pBj0TzbukF15CbG4n6+8wGEGQIDAQABAoIBAAa1anm2p3SxYFHk
+yuhNZFyjUsdQF9/wsmVCl6be0C425IGF9RaJI4FcZsQo/oeruAkWKgIgQZVCnpMv
+6wW8d+22njjfxmtbnqAwScyEgJ9Jgr9mXtUM9nR/jaa3AalWze6yEwDC1rqCmGDW
+IQ8heXbn3HZuQs0q/+H9DlWPNhopEoXlPd+rvN6oCXjnD4kgWTBnKgPaBviJJMhq
+iWt7SHtHGpgits5nCL8POZPQp5AVhiTK6ueXWzTcapw6I6AO74CbntbP4ow65SeV
+6shUk2wVE37uMej9A8Tp0+5Bm2/gSc5eduzMPGogpD7aq1hw5yA3lQ/jUP0K4Ebb
+GcwmbmECgYEA8bzvkjCZBU1GH6RSzlPSq8IXW0emwuncfYcpWMB7RshLs+AxBxC9
+jN3Q0LqTTLcBzTJTVoejmuoLK3q+sxQjnForLzmGN+tOIp2cMJSeQRdkNvtcV8hS
+VbF5Akbel00Fba8V05XuUzETt4jRRpAv3QHtstfmRObTzkt0qSQbFGECgYEA7qoV
+PnXgCYjBA4BKW3f8iNBrw9YsNkc4MpUJ2+JO6YSAtrwTOnpbXLx5E7k6+33cYvc7
+cd53p3++M8VhFmVOnS8y/KRHfTjO0qkxhj9bCjQplPnkcPtFMCno872Tq7Le61LS
+Qay7XS9b7VeqYj1Q85P8RaRzhEToYD4FQk8kCrkCgYBgkg7OwvVOZelemKGv5VKi
+5P25rG0VLqGzz/wXhYV56Gg9qpFLdkeS78YMrIxNuEpS9AjQ54zXRSpDK1zlY2U6
+f9cCnaNd0XW8ZGRSD/0MmdCiGlZwhwlFTIm//BnOLam+gJRim2HDADcVOeYIn29P
+hga0TIWj/uacYg5vx4m94QKBgQCKE+TOiivf2Dqy59odKUH1dX+jgZl6naGpwgg+
+KqQ0/e/pyM3nGzsYYAI8owG41oxWxWqi6m6XfZRxJ1SQPHRLBC+fcvvZOrCNuE1q
+ueTV9m/IPPspUnYo81ZPDfMNdyL5SqVSUbjkpK2ulvWAeLEI+ykMsaQTdjpq0/b+
+8gKMIQKBgHisg1KyXQPEC8PRZFMzvjcdKeBbC80qHJlCLzbY9oC7g3BGE9A87rJ9
+9w0lYJNRO+KdHt4vuL+86V8+fxweNn4VUKTDzu1UbFFxalT5UXi3G98kreD2BBkZ
+sjtJH5fo79Fv/GK1zkKm5sLOSdQe6EY+MAz6McDKU6K/T7mNcIo1
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c7.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c7.crt
index 88c2c0403ed3..5b2888c9546e 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c7.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c7.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUe1qpohqiGcra4yw7HWh348nvcGkwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0
-MTMyMjE0WjAQMQ4wDAYDVQQDEwVuMi5jNzCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAhbZCMJKpb1FJW1h/xfK694X12jOwAp0Oo3qv2f06lSuR2jMH+2C44+/L
-7JJoc3/Xf0is/VBgKpqJpQF69RDNTU3v/JicVel0aI1SuRZfLrBNK5mGYGbevbLZ
-fAmuzMbPn+5WlBzWePNmyw431SDabPEq9m+ziFRedfYY7K6PHQ8CAwEAAaN/MH0w
-HQYDVR0OBBYEFItxCmYpuU0EV0BsB9Gbc4CDQJKSMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGUyLmNsdXN0
-ZXI3LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-JorZihqwU6V6qWUU/GaXGHBMCIUtQVO6NsLbV+6YQ0ljSDlp4N6f8M2UFZZ47wWT
-t6nWpFjrm00nng8usq5OF5KM2vyHyQ4gGs2Dl2+R0+BGy8mvj6yLfkxMMytOPBHK
-NYucWh0Adjk1HqSf6Hrb1IskZqLk1/sJOoHoFnUixJ2Hz1L3Fcwl/eG4eeJ85KNS
-vTx+Yqnx3S/51euuRcIJfM+1+9nBkAE8aaS2MLpTuU0kWvtk0PBPuro4t/8iAZ/G
-di7yoPhtcwwM3u897hlRBkx7mBe5nae5HSAt3tqIGkvFQz+BxXfM8cvYOHaFyPRh
-2GrBdAXu9kFDKfrAWxIUzQ==
+MIIDUTCCAjmgAwIBAgIVAIt129VlGKEzeEzFC9tYrxj87xTnMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxNloXDTQyMTEy
+MTIxMjMxNlowEDEOMAwGA1UEAxMFbjIuYzcwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQCTn/hDW1OGbbzoG2Y6RS5O0CBHutfWamZ/xvWcUnJvIJUWBTcx
+62hYOOagq44thDXgzxWr65oKbafpTQGiOvzBoEo1VSmA1assCdKzWxnUKI6q03CI
+eqLLJASUHH7lMAuWvafjn0AJW0ZyA5bipSTD/nAO1/RRJVlp9FbUfkyJYXWhs8FK
+quC29MWMDg0Tays3jeCPYPCO9Hdl4xxDelsUC6Bcc+1Xvf4M3J53OYneRUr25ylp
+tAg+WrPTvETnizT47CUepcuUkGOLReg5gO1p8nPqBQnlHeL5j7J5Lxj7+dwTq4Qd
+ROM1U19vmQEGfB4h/rCwPUCDC2QFrMCYIMn3AgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FOSnZLFIIGB/ex0NPJtq3KQf1NxJMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGUyLmNsdXN0ZXI3LmVsYXN0
+aWNzZWFyY2iCHG5vZGUyLmNsdXN0ZXI3LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEALRz6bKbaTIgnZ8cCaTw0zc89E0xz8kooR83y
+iWdNDoW8K1S3EPNxjvBAOlm2PpuqxqwpCHc8+YjaLm9rjeMpmv9OrFMBHJmFHNAg
+2+A3/3gsGVzSWlzbb78aE5M67zXq7qm469QtwDo4hMEb8kRejDPhWzSKxOLwEX2X
+zcNj92glYFXPVt97Gs6/s73E0kgmGpuGMsF4m/IGBr7FWGriRwtJoQ+5QgpW70D/
+66flNUCvV7MddcjjFJgFZyCLoWprScplUMjYm4N0ZuQ8tQCNxJPU586eTFuU7+40
+pu3KW/oHTcZIeVbnbWf/jtqLLWE3VpDM6QGsv8lo7umF5wfLhw==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c7.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c7.key
new file mode 100644
index 000000000000..6e74922e1078
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c7.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAk5/4Q1tThm286BtmOkUuTtAgR7rX1mpmf8b1nFJybyCVFgU3
+MetoWDjmoKuOLYQ14M8Vq+uaCm2n6U0Bojr8waBKNVUpgNWrLAnSs1sZ1CiOqtNw
+iHqiyyQElBx+5TALlr2n459ACVtGcgOW4qUkw/5wDtf0USVZafRW1H5MiWF1obPB
+SqrgtvTFjA4NE2srN43gj2DwjvR3ZeMcQ3pbFAugXHPtV73+DNyedzmJ3kVK9ucp
+abQIPlqz07xE54s0+OwlHqXLlJBji0XoOYDtafJz6gUJ5R3i+Y+yeS8Y+/ncE6uE
+HUTjNVNfb5kBBnweIf6wsD1AgwtkBazAmCDJ9wIDAQABAoIBAAGqJR+FURzL8fxg
+CrgPfaUv3Z1Rcwuc6Yj5Ul2v5FByepvj9Spf8C5GfftD0/eZbQ4a57CnmHYTpHgS
+ATm2FpGMYCnXyv8D6ck2VyTkmxb0Wh99AODZGSrXPiH5bEjjUM4jV1hOvGcem12/
+wpzNpet73AGe18dud2KlebEtt2k5JrjTKJs2o6JDpo05MBTjFMqh9o1SnxRQXcSN
+Z9K0ye0N+asaTAVS48JyCXlsB2ksmedFcupi/s90FQPt78gU8v8lydgu7G5ppfZP
+dgKpanemGCytpJsWmE02rj27L2nHsa+c8UiH+tQc/ZuRkrB3SMzQah5OSsojiZD4
+fflDjNECgYEAzwr9qjdZN/E0xyjw3eCwYZG1N5r9856A4v5PvxyyONapPZ7rmtPc
+t2VxVeFrD2dAeOET/iOmgdYJ05HI4RZ6BDhJCbH2gDGwnEu9HluoYczl0WsMwJ9U
+hjTNYkcLMnKmJvjttbasp8FkDanJVcftgsq5ljB/HTCdg8WqI9BuGYkCgYEAtog0
+CRcLY96X35mXXOYdoIYPrQM47sI5WpTlVX3dvSQhWxect8yob2osnEIlfHkX2LYX
+sb1ACIDy1dx30fhOIdK5oHUUPAcL/U2XOg/tocL11Sayjv6zv+jK0OtS9nvHmD7l
+rgUpQU9HDxFEUjPTzCayl/9tY/O+sU7iwvhcZ38CgYBKjA2LtUQVhW7YauBXM2iu
+qNUUAWWAy0hi8uhpyn5NQtVLD6aLZ88n5YmqP3DZ6fZnz7rwHYv9YcDGAc7PZNVo
+IZACwqYbwEhz7j1Y6e210R3Kin3b+UJo1ChDQ4s/jdJPY//LFswR1usclE6e6U6W
+ckHiVvHd47xDTu+U155c6QKBgCUjNSMcUy7wnQnbYBaAo0qDvDDBVRYO4khxvu9T
++od/EGJOPY6Pzz1L5uOr+aCHqrhm5A8/qAWS5tVoU+Cempb7HtVsvL8x19ijrr8b
+eofM+ONzSiUZ4UyMeqdjGbAYoxVwxp5PYoNtM9H2+2WWMeN5hCC00XbiosvEdxKg
+IXzVAoGAQc4YVnpIuLd6AxVZUt67gopnhk3liKm23VRx73Mh6OeX/o/REJSCJftw
+aj99anNXBi0uPMXY/uMr2uVolgshuanaAg5nugqwBpn7yPrqXDFTN3QKT7ighEiO
+N34RoRmXLM8D0ch/TTcbt68TxZTmJTfKbIrIEzNA/ck5h1iyoyE=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c8.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c8.crt
index 250c5f5beec3..0132f2a30f78 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c8.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c8.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAJC+xJO13Zt3e/q5XOGbqT8JJmGsMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNVoXDTQ1MDkw
-NDEzMjIxNVowEDEOMAwGA1UEAxMFbjIuYzgwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAK9dg+Pl7I8v6JIQCdZA1W/1ieuKHHdhnVkNYkI9y1Kw7cPyhZVsApOz
-mjX5YccUCgu/zsNgo6K/Q/Mo9/2qxx/JCmBvaTL7PAUzFS4VPJFwiyM4rhebP38w
-TUb4YXKk3Y3GZ9Xy1vFFbDMmx4HREFi+2PacMZyrs7pCIGtvpTlNAgMBAAGjfzB9
-MB0GA1UdDgQWBBTCTGp5uFyY09VJmeqTkdEVrElUdTAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMi5jbHVz
-dGVyOC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AKG8Jr8rNglt/TS2LX3/J0Fsgob00H7fmV4DAuOM/Ui3VeZi1xdHm8su9s0R3dDb
-JkU8t/cI+jrg2OI54nDuxaNRX0dLPWIfb5Q2Z3cXiDdvvy/9RIK1SvN1o4XERIfd
-Vu0ZP5akVexuPgZWfEpwgeOrjlkhO5VrHap9+igvSsNU2mJtzK5iFy21nYobC2OT
-/a+qxa658RdvojerNAiYmpC0mvKv3Au94loJ8emoZOnVIS6lQivCz97tKO0L6tnI
-jOsCu7cW7IAxKvYOqtOSZxZzBHSa4QFu6HixpTITbeyJ6a8MaiejHG5fRZ3BpSIG
-BmlsawZmp/Rrd6nLq2Aztqc=
+MIIDUDCCAjigAwIBAgIULAq6TsXXCvEpwqFtAUo2u1hLBR0wDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE2WhcNNDIxMTIx
+MjEyMzE2WjAQMQ4wDAYDVQQDEwVuMi5jODCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBALSKQJMbcL+N7OmSZYnajjrWaSIV2A89E6iLmyvyzj6ki6oGVJa6
+6Rnaome4tA6RX/hSF1oC1pH4ly/KzwCMlAldqwxFrMAmkAhcP/zpgs2w1wQO/QRz
+/6WgDLChPlFd/PoAiNusAjf6/+cUXmfc6Xq/9d6DE+sirYiIuwHAcubgRzhplQQd
+2iXtSII9X2eNf41rQEX+fJHCbd+d6df+6BORDgo77Ih8Drx4gc7MqqK/bPkO+lR4
+LeClwTaosp7U61M46bc30DdiJfDVMELd+hxMKn0sxtjX7WzOI3YNJIyY9D0PXTEv
+Zxn3xNlLLrAJ2WOU+iRgbbU9xdaKA3Gt4UsCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+qrgI4mZy+JxtQsBRrd2WS7ClufcwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTIuY2x1c3RlcjguZWxhc3Rp
+Y3NlYXJjaIIcbm9kZTIuY2x1c3RlcjguZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQBEK2gXBnPrGkEhmkWd+evq+8LtIV3vuFmfOUMW
+bjhoyzohDY6LyCoC9jLEJCwevlZ47wrARDjoLn2DLxCQqjjC+gMVZ7Y8LrTeILKl
+w43m3hAb+vKyXoNgEZnyRCEGp5n1sFveQDUIe8uh5yKfLzow77nvQiRJjlvEvg52
+g5JoNi7Guh/qdS5r+1I/pt6O+qzpCWR4j4l5fmu3NubiaC3iu2MELTnPCMFb05Bd
+KYbZdrVCwyBw5/WCea9ZBE2GMU/MwzOzBffm58zWwYYjEY3rdvYXvC5N1+P8g8RQ
+HdOlplW6aKnPRguaupNW5jWwKH4NtkFLrKg3AAJKFpZD7kD2
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c8.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c8.key
new file mode 100644
index 000000000000..c7d6027bbb76
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n2.c8.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEoQIBAAKCAQEAtIpAkxtwv43s6ZJlidqOOtZpIhXYDz0TqIubK/LOPqSLqgZU
+lrrpGdqiZ7i0DpFf+FIXWgLWkfiXL8rPAIyUCV2rDEWswCaQCFw//OmCzbDXBA79
+BHP/paAMsKE+UV38+gCI26wCN/r/5xReZ9zper/13oMT6yKtiIi7AcBy5uBHOGmV
+BB3aJe1Igj1fZ41/jWtARf58kcJt353p1/7oE5EOCjvsiHwOvHiBzsyqor9s+Q76
+VHgt4KXBNqiyntTrUzjptzfQN2Il8NUwQt36HEwqfSzG2NftbM4jdg0kjJj0PQ9d
+MS9nGffE2UsusAnZY5T6JGBttT3F1ooDca3hSwIDAQABAoIBAA47L6fo69RNrbUj
+sxlQvxFoQVVMf4GEAP6yMoMeWp+ZfODkKtrO1P0SvOaMKrVTREtgvTJ9MsjC3bgI
+tpSGgEzUs1z+SQXEhGoES472ITdxjyFkxREKy8YUfauWqkDjtcNHM4KNZodfI7jl
+ZCOvyD/CzfDCSFmqnB89F/ClaOlF9kvTaUoXNpLL9+CL36lTXZuJRBIkcGGbjRsb
+rX/DRFmXcJdd4pQtx4T81Qj0hZ1nas2SRfS3lZEUsgJDBwvi6z15D5XwmQOGjfG4
+8COxAWFt/HslClOva+1SJo/xnjMXHmI3EG3FUehm/Pzwiwb0i2fZowtbaTM5Jb5t
+jTSQENECgYEAuFeZZbAToFWwKCzA1E7Xl8LTmNWLV9QtdsY8OuADn1bk/Eqmb8GO
+t3TcQWOHdawK6D/4TsjLUrS9tyZ26cs5p2MF04wi6IJ1tYHEMsmFAoaUtKMtMChQ
+Pzb7r/g6N7RtQ1U4sjfkh5jpmvVYeFfqvEUuEfDH+qTciY3O5ocxXh8CgYEA+rhK
+umdXZjbKUvQS+zqnfQF0sBdTNExm8WrPkN36Ftej7uvDYLnTr78eewDZP5J0SQKT
+E3Zjsv/KJ0DOUBDboXpxRmKS7Nq0qrJRaw7Y3Uya69XCqZVi7hlfsCRRrd/vXABx
+8M3+ebt8TfhJ7iry5JtT1WurQK9Vl3xm41PgP1UCgYB7+37SY8Bvf/JAswjmQjjC
+e2ixTD6xzaTeU6nCt1P4YQsVnGegloAUZ2aUHArJW/LiDdRFuMr3T1Sh0B7+U47t
+Q65xf7kixEvCzgpVVjA9aKKvc2g7EYIXYjWZPsx5FsK+HJ1LUEhF4tSLQiREnGzm
+P2ki69y0eRlS8quLZLYRZQKBgQCI+vNpEFkB8WFtHQjZ1dv4QjlclAHTiwKbV737
+6itfG4XnXUwrpptvY73lejs3rm52DUAJ3B7whywNcizGNc+pU9pbmDOp0Hvw3uf4
+0+3MSrRBsVJfkMyyRY6wD8fIdZay71NRO1qTa8moV8UcqL54BB0zd1XjS2g6Ea+s
++ACG2QJ/cH9g96IaFeJCKU5Ik9gox6UppiNU8tUFHo/nWorIn1CrX5b/p2ZRmlFS
+nCGRoghnv1qKly/OjE0U/cVBg/ODEC68ujjrqXSukv7LsEat8uZxU7+J+8wrDcqC
+mcfC+0EKO1A3zlOnpN3IjOwnuF3PuzGjEDTwNlsodkja0edllw==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c1.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c1.crt
index d1fa1d536013..1a34cab66f87 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c1.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c1.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAKd2+KsSsV/wqMN6BTbenGb0t7J5MA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxM1oXDTQ1MDkw
-NDEzMjIxM1owEDEOMAwGA1UEAxMFbjMuYzEwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAIzKcPP+NTeJoVHNK38SvvMpHADXXN4iavSOL8iker0xzCV9EPpBCi6b
-zfiCjUqtS5n0ySTCce75Y4yuhLstAZ3DhUyKvdV3oVZMv2T2Tqy2ybrT20YwhRjs
-MXWKlFjIdWLp4kAVfMt4Mf+qp9Ou+RCUKCtYl1+caHt8K6lE4uPjAgMBAAGjfzB9
-MB0GA1UdDgQWBBRJ7Oy8e2RR2rHLjBcETcdR4DmY1jAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMy5jbHVz
-dGVyMS5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-ACJxPS9L9+bStPqXjVGavLMH1OTT5Jz521oB3KgOnYb0pEewnnUpYi6Tty8cuoh0
-a8MoAF0O/pqqXYJQ9gaUBBdokg0raDcE1c0jObd7OR7IFZoG4ojVh0M8fK28ykFM
-F+Jy2nGLiW5C2Je9pPELYHmwfuw16iU65WlSYkgPAwwc2oNNnq5mmvLgGOE1kcri
-lFjQuKgQEzt/U4oRX74zHHsajC+ZYAYf5mTrz7qJO4IH8/+//HahimRUphE0o/KZ
-NY6iyRppdhWWLQFQ8+VpTY1crmwP5qyUcrlSY1zl5zkldU9dfNmz2NdjrnWtMbl2
-VdTOSAtHe6OtvsPCAlrGFyU=
+MIIDUDCCAjigAwIBAgIUBJaojgaSWGBz/bLezkR6z9wUVtcwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE2WhcNNDIxMTIx
+MjEyMzE2WjAQMQ4wDAYDVQQDEwVuMy5jMTCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBALVxnIZebpQ7pS6Vmwn07m1162CGGlQEy/IJyU3FR8CDs6g69KD0
+UoxN+W0jHTBiAlJhXx2gKQCd7d85kkSJkM0u1zoD6/WfX2dr0IBJ6G+wU0TACSw2
+b044SMHPlRSy8kQ+3uo2M9IKUBmgf7dAUqmIYsbwbB9M+iKFKslk6/SmU/47E4Tf
+bWTtuqS3Qy7NOrTPu9rmAM+N19RmjlpnMDxcNpAisAJeCkkEg89YaOKAu1nh2/L1
+aDhJTUmFlKW0P8SVwY7iGPrp6sRlJJd/nP9mJrt1Xvv3CXEKfhqNP9tQhUvFEifn
+zkdXlExhttZo95YLlXFo5PuIHeNP1cnyIdUCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+lK8wV3Krq7wwonIlfdDgcKEItPQwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTMuY2x1c3RlcjEuZWxhc3Rp
+Y3NlYXJjaIIcbm9kZTMuY2x1c3RlcjEuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQCp45/Rp/TyZtRAmcSrTHVGUtp3f00Ivbhl5VK/
+Kv0jWX68sqZ4qvJZkARE5T3wjvRvDJRz5wzGObHJcOf0w2n0V62VOJ3gXornFyiT
+p3gTkEOam9i3Ts3Q8D7sp+GjbKQp9WO7gUkdUomNKhYzDMMJnM/SLDDUEzjD2MKV
+Q6k0ETVFTIehvpihtobRSeYPlSQMH5kfWVRt0iKUaIBrFB82q7d29fs1lIplAjMs
+QsqpQFIg+5UTdjX8K7l6ytBsnYAY53NIUJ+Wz2mTVzksIZfIlco/8wBcFFRmmQMo
+wHxeAXzGaM/HjiS8McvH41j0kDZofVoPuF+gLa1ySSOGEbNZ
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c1.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c1.key
new file mode 100644
index 000000000000..db8d8ca60403
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c1.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAtXGchl5ulDulLpWbCfTubXXrYIYaVATL8gnJTcVHwIOzqDr0
+oPRSjE35bSMdMGICUmFfHaApAJ3t3zmSRImQzS7XOgPr9Z9fZ2vQgEnob7BTRMAJ
+LDZvTjhIwc+VFLLyRD7e6jYz0gpQGaB/t0BSqYhixvBsH0z6IoUqyWTr9KZT/jsT
+hN9tZO26pLdDLs06tM+72uYAz43X1GaOWmcwPFw2kCKwAl4KSQSDz1ho4oC7WeHb
+8vVoOElNSYWUpbQ/xJXBjuIY+unqxGUkl3+c/2Ymu3Ve+/cJcQp+Go0/21CFS8US
+J+fOR1eUTGG21mj3lguVcWjk+4gd40/VyfIh1QIDAQABAoIBAEijlDNjCMgFXOra
+b/g7e1yNEc9RwKaUSQW8D0A4U7Jpx3n5KEhrp8SNzuZ3YU8mXaGXY9nbba7Wiw2L
+4dfYbwUF1JpgSY3K4Qnpas6BSDG2mI90hfeST1s0Q0R8E0X3IfBabym6y2IySMEJ
+tTeq460Fce4NAGxDh/xb7bY01hXZ+CU+lomBZkHnXCqZnWAzejmeU1fh79YBHbaw
+bMmU2VjmDH0u4m6upclDW8ua4blEIWvswaLAtvb3Bf3x5ll2VR50YrYVgMxfvHdU
+ZM0uDKnwgwdyrnZ1D2/j8HYzFhtZAvPVX5Hr4Oea3Cd2c+gk4zD9sbbhvrz9JGsN
+TYtpV/0CgYEA36N2DkdNGKVQtj7AKiN0x0ZJ5URfEYxOL4qcfB6CzDe0ZAgWASW/
+4tjQx+JDixuiA4tQhvGuL6gGJ4DfoVhTxjjIBr+0mDEKDbr9RMw8sO9vF/0OPT2c
+o3Bwb4uHFRAYcjsHQv1VnpwFKSRELwm1gEEofcHKjWKfj1fYDDkyj+sCgYEAz7MT
+MohXfNQCeNTeKxtRytX+ICLGR/xQbsp3U04DzMOrLNKKzvycZKfqE4lWMAnYTQEA
+QZ9IeP5X4dC6cAtw0nw06R9ISavYp4cM/oYLkitVFBJT1Tnk0zcfFVGvVbEqh035
+evzppE1fPe3KYrnkIcCUHFql8l5qQ+Z05zXrZT8CgYEAvF/w2jge7aqaYNhcf2Ic
+RJqlNfQwbBSV6x/Djh0ZKbGmRjg1aMQGbKsaubOSF0OwZXHc474Bbe0HOhEozbwb
+3LUJb/3cvfFYuZYPLfsnCBa2qisi6lzSay0T2DFcGFbVcKreBARmMVLUWgJNWJ3B
+/4Vi+QLafdtb8Z8ZwboQJ/8CgYEAxQkBAUNOm+EUp0Rpr0MviNkRR7M1nzzBoRj5
+slGkUTCextrFTC9gvix9ZyM2XEQ6qnJ0p8xPmFjRTiwRchGoakXie/VtQlaAJg18
+Skkn2Pa6s1vfo7xUU/tKxZLEXuq/ibRzJDWiAHhWDg3znUz0GwDygSONoivna5rI
+9D62nUkCgYEAjL6sErhbM5HtRHw8F+eU73pb9bcJogFxk3Gf1cUp79kbwnI/y+qv
+sIK8rC4obeuQDAes4em+Exyqzo1Cz24nLPK7rdmaz4bOBqPgz9JqN/WzpjiXwqTl
+KB1FvTtU22eLhTpCC9S+Yx9RIUlSlnZMxyuf+Yien7s8V2XTIczSfRM=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c2.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c2.crt
index a6b388dfe60f..96838dd096f9 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c2.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c2.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAJfn9TW5KLMUqkzAi+zsAHqbcTZWMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjMuYzIwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBALEnh4sMjt8W7vwIeW/PirZad6tqKbiCeGyhSWNL07VGDKUsAfF6RY6v
-oyGjBow/fPxQ7OXjlw5BFEgO3DxlTbxLcNk06JV7d4RrfHhCGF3R3x1L/icTg+rB
-+QdAaMdHAiU6Fme5qGUZg7k2C0JO9mTnJO4tIsL+frPGRFmlNZyzAgMBAAGjfzB9
-MB0GA1UdDgQWBBT9XQWzCcEEuYHJXQVr447M/HqFtTAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMy5jbHVz
-dGVyMi5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-ABd6sS7YCm+KYQKxvhM0I9k4BS3rkd3L9fRajM6gciPQhXpfO4X9lKxXtGfG1LcC
-13U8k7nV8o9VXCcNSUXtLt3jk1+qprqPfAPDWiYkaYgRImotgU5q5IxM2uCQXZP7
-BTlzBODi/e79jgzuxhbOCzO/6D7Eji4vQP3MCnikzjXJ3MMDIV+u4M+e8UuS76vt
-5HwsWQrAMl1hfxuA4a/9OhbfHRrVaYOr3czULqr+66IHetsDsUZs7JSL7qPXFldi
-oMrUQnqECwZKhqxokiY0i/mrC8frajxQbk9Y9GVSBsGess9YXwNXW2VdbyyZFwkw
-rIdSfF/kY4BNf3/c1FqVjNc=
+MIIDUDCCAjigAwIBAgIUXLvr7qOPTXFguQJGr8lY7+4BwNEwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE2WhcNNDIxMTIx
+MjEyMzE2WjAQMQ4wDAYDVQQDEwVuMy5jMjCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAKyLoDjJimn98kRd/QVZmQNfGthB4aKi7Cku5engQ3+VHrO10Bdy
+iOivVQXUupIR5knYoAZyrLW8gjSjsNiO9Gu/fcuYOcw57Oh7ODgjnVPyNtudDHv8
+rbkBgy6qUjtnRL87wqxNEjDXWA0uY8/NS0Fp8hH7ZtmovhOXADbkZ8KJ0WY5GFft
+HrugaETe3zuOUQisZTtORFpIBNmA41Jzt9Q4RLkyck3ujJOpzXkJo/JoavFKtukf
+k2DIVnjmL2ntXgI15th8DKe7kvwwhisrlMScU0ksX3dXTdnptiDdHldPJWKPFy9g
+4Qg9tsalpnpULrK9oDCmqqQZmaMupj7MOS8CAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+WU2KLwZebRk8cUl9ygl909D1eF8wHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTMuY2x1c3RlcjIuZWxhc3Rp
+Y3NlYXJjaIIcbm9kZTMuY2x1c3RlcjIuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQACnbIqetl/9+thIq8QhzAMGI5faSrx5eAiK14B
+2Voc74zEt6UlCypZ2r5A2v7yF/Qy9eW0+JJAfS18azfIYjrqftgf7Wo7nNlbXWyi
+EN2hzVgxYz2bc3YRILVYjk2qh1uI5bQToRmIimdc4DKTIUSBGowyx3aLrBSm4uri
+kbz16klJ2G5JKDvYq4kDR2EIaqSHhyCplrdje9zzaiu/HOawB8vNYkfwzuLi1D3N
+/9e11DEwQJTfBc9rnBk8rgWT3IjW3nc7xL9LK5pdGFE7FBBc7IDn//YEI8fv+dnT
+XqG6+CBRyOaa4ztWcGptGe/9VItqF5V59HTd6UAbaO2EORoW
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c2.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c2.key
new file mode 100644
index 000000000000..b2943ab38048
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c2.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEArIugOMmKaf3yRF39BVmZA18a2EHhoqLsKS7l6eBDf5Ues7XQ
+F3KI6K9VBdS6khHmSdigBnKstbyCNKOw2I70a799y5g5zDns6Hs4OCOdU/I2250M
+e/ytuQGDLqpSO2dEvzvCrE0SMNdYDS5jz81LQWnyEftm2ai+E5cANuRnwonRZjkY
+V+0eu6BoRN7fO45RCKxlO05EWkgE2YDjUnO31DhEuTJyTe6Mk6nNeQmj8mhq8Uq2
+6R+TYMhWeOYvae1eAjXm2HwMp7uS/DCGKyuUxJxTSSxfd1dN2em2IN0eV08lYo8X
+L2DhCD22xqWmelQusr2gMKaqpBmZoy6mPsw5LwIDAQABAoIBAALXk+6Lh0+iorR0
+ld/aL3aKQP9x6l0K8PfmsVvOWV5HHwfXxVM1jELE7JIZhG82g0QRIOWqSuT9c2mZ
+jqaqiCUkNNa/LaiLBbd4nB70mzlQQo4Qgq2iJlxnRXllzc/zrgK2YckDxfpu+aDR
+9EvudkBgFeAs828/Y84fzR2aWokuRAP15VG1jTDwq6YAnJ1JwNRUblkrUrQZEQF5
+koQY9Eng61qYOSwHWzFLX7l1uDsCU9nBi20pUZTB49XOc4zLqEa0WXU9vQFH3kJU
+WB8b0vmGJmiAYpsOwjEw/0gUrWNNaUiOTLzUF2pjz8lmwOV/si878/SZBaO6GxXA
+HQ4Fk80CgYEA5szrHqBEMzocE9w8f+BrTSpNmiMrGQcF0F/OLlbXdex0GpTJJfZB
++QMM0gWW+VZSGTL3l0xCMKWye4hK8b7slXcoNk7aVradljR0b4G8ahKvpIO2PSds
+wnXQR2jRN3PlVeNfTyOM6lHOWO5YfHfsaQwsBgWqAGIkA3N4mOTU6RUCgYEAv2Jt
+GAwdYfZ2jeA2j8H4qBK3zLsEQHM1nJKy0+PnGrJ6PP35qEuxB5zPfXJk3eXTeig8
+UGqIEumxC77bL54l4T09UU07NHSGy//p/PFNeN7z7ockDkltRZAp661EHPZuxTBQ
+yzefQl3s5Nl6RRpV0vKKrhODyFD1v/VTcfz5IjMCgYEAzzZVNYbgFDnssWv+qTR8
+QahkSamfbPXgI2eQfgwbn1TuHFtOiJWiP4Wcggbgt/Tm+pbGsYCWwIRTTRNdbdNj
+JwH3CWMcJdQIga2Qh/grh/RE22ghOqBlxeTuXN5fT/DuioD7tdqODmXp4/ABY6Zr
+ixiRgcqeCYATI0INmP+Bo80CgYEAmb1Rca3ZtKYuGk5wxg7F1w4iQ/4/rh5ehMZ7
+jSdYsOOKupAzC3SyYzEepOFOEcrblRfZS6goYzkDFjJdFOVv/H4bUcI+uyaYWivc
+JWsaFuzS+1voJOnrM6RZaI8pVjLITPNBE948uy2EbVoelIBRTUG1EZ1vkW0n3OoP
+/FOBNm0CgYEAsmx3NSEBSpMSvihuNmcNvvYK8Y4cUWYToVzPOPw8AHWMxDeGpMjW
+ihjSNKI5MUCZROE3YoopKbtQelmzCHEAurvsQ8nnbWrNgb6HM7muKUUPgRhxLkec
+Qu/4XpbcU3Os73fdor8hbe2Is0F1H1CKfG7IdcckViuiNpDnqzL7ys0=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c3.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c3.crt
index 60abe22ce3a2..9bd11c1ca822 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c3.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c3.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUHMEbpWTGffkry4rCbr+g0f66LGwwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0
-MTMyMjE0WjAQMQ4wDAYDVQQDEwVuMy5jMzCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAlMorfk1gUwZci3ta792qR6j50VUKQ5AqZrvOXk74eYzfPZiE8TJwnphG
-dfMm8Ru4v04+4pf5tTzrl0L+dKebSLvVWYE+VPz/So9jyb2gHT7/I1ypEjum7iUp
-SiVzM09exmSyMrcyRpWSRQcrMomC4EK5OwDctUv1EUqAlgbmuqECAwEAAaN/MH0w
-HQYDVR0OBBYEFOkY81GYZIxcO6x/wKKrfPah0ThRMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGUzLmNsdXN0
-ZXIzLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-nKuZp21gDBXwZV+u3/QXpsR2jLp1T8oWh1d1GuJhXkHqT59DTLgaDxNt+8gxA7mA
-Kooisz68hOjBP3KGGvz+WUWM5KhwPJplFyQi605RJOdczYf5+6lRqbeCQnC0kaR7
-iSvfqyLQXc41aTL8MxVBkJJhr3N2Qui6H4H+8nHv+R9ShE2HYC9UDUcifoabc1ME
-B6wKWM9TbGSqozeH65v5yDGcC/17LMe6oKuI/kti8L3Tbcp7E2bRyt0CeYXRwq7e
-/vA83WufkhfXcZZzOvvgJBxKH1MCj7D9tIOMv9Wv1vu2ECBaYPIcClTqcSjGrguw
-XHBKbhh0L6a0jcsolV7v0w==
+MIIDUTCCAjmgAwIBAgIVAM29K8aZdBEwDdTEdx4fbHG1OYmNMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxNloXDTQyMTEy
+MTIxMjMxNlowEDEOMAwGA1UEAxMFbjMuYzMwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQDqYHMyjLjGNFPYUwDI51w+jLytzTK6gWKQHX6ZziyfTUilDea8
+ZyvHaZ5Q0gZCxKgFHn6QB6QgREAlqVnWRXbD7ypFlsKC/kOEBE6h487GuyCF1B7N
+lbT3yN/6LEVYNtiwr2IRAZrgVMnjMIaZA9BPULsUFmbm4ArBLxwvaS261VMcXXCi
+Pfx+BxtvM12b0kZ69TsV85lh0p6deDGvevDPPLsxGuRevxGj7iklRppbZQFg6JdO
+lddi+QiYgIjAlN2NrN8n2GXsa5OQO6JVe1F5Mab1hKizhzPpq93wi8wMGXLFkhkO
+Q2FoZgIejo9Gt+64Xh2w8rmHhAKPbeGxi0jxAgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FB362r6yAWeg2JFXTAk2CoJrWERvMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWCHG5vZGUzLmNsdXN0ZXIzLmVsYXN0aWNzZWFyY2ig
+JQYDVQQDoB4MHG5vZGUzLmNsdXN0ZXIzLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEAVny3LlCab5oI86tfO6oOOsonw6qhq8P9t99j
+vz7wUEt7v6+SK5JouApSxpBTVs4Wo+1+0RGg5z5KJefdovHKdIc/omxhlG6lWPBr
+2M3YXkNuRTbg9Lon/EqzNeI/VL7Xev/uDMuUG27oSn3XtGVVuJkjit1uZEZVS2Ez
+W0/MhguBauyweFZJbL6i5xTZQiZZTor4WA7MKawy9/lAAPWQ5cG0t0PLdT8MmhdB
+h4o8MzW+kAR/uCnPYLWNh1bPNtAHMXWYkycgdbs0pGx++r+Bsz028fwsVLb7EDBB
+FvQvhWx2zMxhQ271j+EapduogbU/C8fWAICiezR9KuYu5zfWvQ==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c3.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c3.key
new file mode 100644
index 000000000000..c645b339e6f5
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c3.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEA6mBzMoy4xjRT2FMAyOdcPoy8rc0yuoFikB1+mc4sn01IpQ3m
+vGcrx2meUNIGQsSoBR5+kAekIERAJalZ1kV2w+8qRZbCgv5DhAROoePOxrsghdQe
+zZW098jf+ixFWDbYsK9iEQGa4FTJ4zCGmQPQT1C7FBZm5uAKwS8cL2ktutVTHF1w
+oj38fgcbbzNdm9JGevU7FfOZYdKenXgxr3rwzzy7MRrkXr8Ro+4pJUaaW2UBYOiX
+TpXXYvkImICIwJTdjazfJ9hl7GuTkDuiVXtReTGm9YSos4cz6avd8IvMDBlyxZIZ
+DkNhaGYCHo6PRrfuuF4dsPK5h4QCj23hsYtI8QIDAQABAoIBACYG9KHmwaMNAe6X
+FwlBuo5+hK15XenzTLPfyTb4w/Yr0qXPEdkPL0P5T8HCtdfW8wDzn+rMBpXipPrH
+VadlmV20fAoS6ifZS8iLb1uma9oEKgR0tHO8PsK2QZrkXHLx4SoXpqRmO4P61c/K
+A4innQCFc3kQom0CvFP4Sg0qNkx5+S8Xn5g6dwdwAd3Aw9qUrXgCvmew3zZbzObO
+jxXYUtw0wNEY75mPeP3wwxmI5gfpdscfp0MQXWf9C7sCIRSfnzsm6s3050wjBIGM
+2wSz2Ks2t4s9DNWSH3JVeRy9nghw1U8Er/g8uAnZF7oCrpqxbbNOOysDkaxSbTWH
+U6TSwa0CgYEA7d+zWPqGl1jAGX2U5ZKOLMk00BNiFs+A5CCFKvtzTdYWc8G0Oe8E
+cXh1PA6/W6tuuxchahjAUZk+trg5/dDkTp3ll8gS4j0SM1N8DSW+/dsocF/vs7vK
+gJ6B1iMfEB71elkOgT6XXQ7afUvuzlhvecFN/cps3oYR77B03dSCsuUCgYEA/DyH
+1zJfD+vELjG3d/lEY1k7tEqKjfjEEYSl/MfMRhJMsmTR3/lomXRx45z6QGhD+TJS
+/1RPfUr74MGjDnE0Xmfw+n4F+GCoM5HaMMKtX3R49xMMdaRVuSzRrMrNDy2A3AMv
+JIo3QCVAASVQMxyLDVUzFS/hiUWvW2GX0xCfoR0CgYAU9+AmWsccdj1QweorJnUT
+65SVNvAxlI8HDPrQJCQw20tVXuDVLoWUvwqOhbpGiWEUauCOKqphRwHpN2odPpmx
+eGexrnhDqyB8l8JaolYoqHXzGfHWRRp/pflJ+ASX9jKrW0hVCkBrepC19MiTNeZ1
+tB183qgkIWtZVpOmrkHrJQKBgAfc7zMjK9l4NQ5nmFuSFrKuJnFRXY5dcedF5lLB
++7k3etKY1/4ObVdTTPWWgCqf1XagPfwdqbizLLTt9GJP3x0OSnHUmeULRlMNOesJ
+NpRiMh8ppdrI+68hGYw0aj4N4hV5t++Aq4ZBYwRmIGDng+IvhL8lVfeMh5Hmse/v
+dbp1AoGAAVCFNQGtHnBqce7hjAQ+5rbSYjlt1QbfOL8DNUZpylvSGiIlp/D0v0DD
+3FsmRvZtiAzpnEdX8CfPWgQt/Zr5J4YPNa6R7WXxUUiZgCeKXqG5NKx0+nJuOi78
+nix4yDoZBGSyZ9tXLJIbvP7TJl2nSCP4OtLCC4lwbxSFoTu3SZc=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c4.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c4.crt
index db1714af7132..2b70efda5a59 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c4.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c4.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAOcJa1E/SlLyIMeH6LDKqwCVLxS7MA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjMuYzQwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAIwFiuTI0adVuF91ltRwyCzA2Eb9fV8JUKmlVXfjElP+XGG2CUerk8H1
-Ps0nPDFBCZB+0VM5BzjrtsEYmuEk7LJDXzoEj6L2MiYKIVia+qxzLvSfbcdJsGNX
-DrKn47sKDrUJbBDVodI43xGCplWyyhf0MS6hMe+7ttwb+LIZzTFLAgMBAAGjfzB9
-MB0GA1UdDgQWBBQJKDvefckZz3in61z5pdM6CpIwYTAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMy5jbHVz
-dGVyNC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AG6ie4xkvfaHLWhq5BtFYVQO2+z6siRz63hBySKwyCgoNvyk1h4xHbmyB8gWG7G7
-TUjCMkRovx0DIhCwtK3Ahj4APh/QyqY3d7KV4i9jQSLXfyaY7ZdvLt18RE9i3MjL
-QR4kGSPaTKlhAOyZ/qr3NdyQH56cvUEto3hgBOb71JcqpT4DBc7w+W/wNeSSPDb9
-/7m5Oq5ntS9DzJBpGcfZ4IHtrgZu9x34uYLkYCblYiUQ4QQyyQae6YyihfzH9QsD
-vt5ja0lueTLuA+LAUv7OxDQm5USXQeRmXYT+Oq+MBKoJkMznftKYnWK3tchwr5qn
-2bQoMq2HNQtemjZAZ3IoQYU=
+MIIDUDCCAjigAwIBAgIUceiv+ma8IfROaLErkki3f8MBciAwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE2WhcNNDIxMTIx
+MjEyMzE2WjAQMQ4wDAYDVQQDEwVuMy5jNDCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAMvbCax0XEJ5GCy0rDwuYcRbXMGQCmPboDv0T/+KqmtOyNhDjNaf
++Q6sZLyehbuuQL64wbK7RMQfMlZMy9o77LyZQhF2cJTPAAFI1nbSmZsRD2G+eEqG
+8uOtTTkpI/uaWV5t2DjY7WBfudqE+c/vtfGaqp97wky+9hPTUSCtOelWqQqT/l3i
+ancEUAdpGubYRPjWAQIIgvso/Z4Is+n3wOpvJXSq7R9GCi2+PLoAqNBxlxHrkYAB
+ZfH5B++ZF3MtVC8W6lAKRn8Zk79Ek5RF7BNqv3AqtzexuNc+Pc2+fPA6O5UuQotB
+NI8dd543pi64DDeoy3RW0C6gV6+sVY3z4hkCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+zhMgSZVQezwsPcFm8+ujjyi9/NswHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRYIcbm9kZTMuY2x1c3RlcjQuZWxhc3RpY3NlYXJjaKAl
+BgNVBAOgHgwcbm9kZTMuY2x1c3RlcjQuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQCVnfWFIhsxteeJRKVzT5cOUPxNOofxY8ulM2rM
+KJ+ii3sTybPtPonsUdTrdATUtPzXohQgQV62cP633tm5FIIXTBwvxbKOjFz05tJr
+oZ7cNp1nmlJXSmOuGtimPHdt9Jm6Is6GuAACdl/RmXDtYwTHKB/h2GeaHtnPugBT
+6RNImWjan92jey/Or97WtcL+otEcu0O1bMZ4GHBm9N50WdlnC/GeLW62qaKZuXje
+FBgB9ZrO//SXtrEyk1VMreS2MarO4UUxcOtfkc+e66nPwwbofN8T2eNm30CMjha9
+lGC0rPdli+H8S06YJQDVUa/nnTZxnIsqu/Ck3zP7YdXuUBTv
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c4.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c4.key
new file mode 100644
index 000000000000..2cb84a699356
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c4.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAy9sJrHRcQnkYLLSsPC5hxFtcwZAKY9ugO/RP/4qqa07I2EOM
+1p/5DqxkvJ6Fu65AvrjBsrtExB8yVkzL2jvsvJlCEXZwlM8AAUjWdtKZmxEPYb54
+Soby461NOSkj+5pZXm3YONjtYF+52oT5z++18Zqqn3vCTL72E9NRIK056VapCpP+
+XeJqdwRQB2ka5thE+NYBAgiC+yj9ngiz6ffA6m8ldKrtH0YKLb48ugCo0HGXEeuR
+gAFl8fkH75kXcy1ULxbqUApGfxmTv0STlEXsE2q/cCq3N7G41z49zb588Do7lS5C
+i0E0jx13njemLrgMN6jLdFbQLqBXr6xVjfPiGQIDAQABAoIBAGMAmY4bZMMTtF3h
+0NPZYrnN9Ro8Lg+ZFhSH7mIYofNDa987zyZcwsQRjUrdYe1oDyGQ8OEweZEV5Yhr
++4SdHbfqZl+8uahLb6EwEpfaZ6cbffDmY90t9MAF5j8gb809hLGLfzr8fywS0VwC
+wJ/vttkDWGHj5NUI2NGACyJnfnyHL8545kttAHRDSXr7xmeVgZKBrx4ngX5h33YL
+L9xjAHABrx0YRD48j2ATaBsIgogPWgywj8pB/9aClGi5QG1gUQINgzQInMcA3nge
+SBXYFiLhJI9M2NXW7Ho3xIVD0mk94Vir2Ybg4I0WwCLJ/gI8LGIVmWDoO8BeTIzU
+2zhVRxECgYEA6v6qg9LVlziTj1C1Fo/JISqc0Cm9d+uEoPtOQNQy10Dt6i2Uvji8
+HauZvWDwIoCbXMyR6FLEcq4PtNrfqpyEezlUPDf7/BbFPKPh+13subx/JC79rmkD
+JvAruAjaNsZKqF9QfZXPCZ0SKcYfhCAJU/BlbK6kIXb6DHL82BjlNHcCgYEA3hPS
+JasZmUpQeCU9QWjRvp8g0qTT2GfdhLEOX5d3qK5IETzL2ly0FvB5LNY+lW212D1Z
+sGzUXxKaYYAB90HrCktR99qjoYC0CvpWZj8plLNTqIGbYu8tst7mAV6DwgVikovj
+F6dHUWiTKRcampuEndii/hKLz/J6NafwUW9kEe8CgYEAhTRYOaMgObjzQ/CuxZw5
+mVhLTV/rq3dQVeQ95leifgurEOxHfVAxqevkmMnDNPuLlZ3MM1H/UzZBZvYOhOp1
+2leT3RKq/ql1ojsPVOpXzqz/sOYGHIAgLklDru0trJDYLvrgaF+r3Tow2ocYxE9Y
+mkERbPrNxL5pXPctRd1Ox18CgYBdaNzltzhHP/jk8FT5pEtE8M3vjCEHT3jfjVHM
+Z3uPWe86QOZF7ORbGb7+Wwdr9v+b+wI0sTIK41pIVH1/MRzZ8aCCrlZRYj9Ytr+M
+tliZEWzI3essDAqeyB7ayJSHEZjKp0hX9KaM2X28tNkd2vhHJI46bDaeuxEobIEq
+372pxQKBgQCOSvmHsolD39e+VV9vZdbJbMumZqbsdUB/EG3lI3hu+Vx3vIqax62j
+woY5KSYL5bwKPQVINuEzd4fI7q7/yzzFEL3l96lD5Oxz/++aHprC7dK5SNVVsW3w
+kTfU9kt/VsiBZGMXxbksiy3uwTKjSqIiAedYytWkRdsSRzOZrVzXBA==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c5.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c5.crt
index 7ee49509b0c6..857d55f2dd01 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c5.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c5.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUVSG6oHWDOX3AILqaPWoYx+i69V0wDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0
-MTMyMjE0WjAQMQ4wDAYDVQQDEwVuMy5jNTCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAm+gpSNeO4opovT+vEOBd34pKfiCaIqoe0D+vR60gB35glziDShITOGMs
-04qUkM44nnv+d/Qib2BdkNlze+l/Sv92r+0DQp+MRzdNE9GkuV4B8yF99hpi71hJ
-MED1lkrYRLUiEFUjmHccUoO6fF8BbdQwvD3rBTpAss1nDuO4BncCAwEAAaN/MH0w
-HQYDVR0OBBYEFFT81Tir0l9q5Cr5n47pDeY9pr4EMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGUzLmNsdXN0
-ZXI1LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-BMhWQlbyMG7OYN7Xtt6d9sOduI+peTlDKju1DQHS0P+jvCuT46Tm7u/4A7G6ommW
-Bc7hLaaGi1gFyiR424FSjiQJg6ye5zdix09H6DQipzSB5GwfyQsB3YWsUniPQn1R
-ZSaqQvcuD4vCXANyo9LE/3VOJKqXB9sLdU42nNU4M4vyB9myTLTHodFoKuZwGifP
-/lv63dd1PQXfeQ/u5FtWopSDdk6fMigoKvo7f1SViTUtI+PmK19sOqOK3qyrCagx
-I0+krf0TsAhdGo1gwpu40JLb15pWU99IhKIAkAu1kjmAxrpkbztAH9wjCm1/jS+V
-nHe7wxiLzStz64Ld9voqPQ==
+MIIDUTCCAjmgAwIBAgIVAOAwJDjlILMIPxSxVK6E36unw3/tMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxNloXDTQyMTEy
+MTIxMjMxNlowEDEOMAwGA1UEAxMFbjMuYzUwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQDda8Hyy0SR0qK89aVAmERnJzLRXr9JuPzPSeNN8u3ivvyU2PHj
+Q8W3xCRnAThkvW6XQB21Cy2gwyn6kbEAU0BwchH0DoNJ21O+h+RW9GOzQVE2fnvc
+VOUxlHxuroLlmNuMXj5II6qT3yGdAB0Alih8AZXNw55PLU3Mf60ejxPJ1iauZggf
+Za90ZNQCvjWwKBn5o5Wj0SmD/IDdO6qPtgopN6zgmpQOQKrpnjwFIm5gpPFQmTPX
+WOKs/VYdtHhlcJ+U5jR0iwteRzmCB2a6zYGi0pSdzyRL0YkwU50YBxTXPgguuKK/
+KqQA9C0bR+KUoDXN4QGuRFpiR2RBXtVvN9A9AgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FJGYg2/QdFmOMA/chIJ+7wtY+odDMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGUzLmNsdXN0ZXI1LmVsYXN0
+aWNzZWFyY2iCHG5vZGUzLmNsdXN0ZXI1LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEAI7fEpgzcOdzyC/9hMez2oM6B46xrxzFSxzic
+Qw0pl02VZPZ1fv129lB5Q3h+FJndpYV/96VS75H0oiwEwLta0OqjnrQtCS0wCcZc
+g7nqtgcRbwr5hJmMXlr12xEo5aaW1ekbykbIo6ACc55+7Kq1/6v1LGDDCp6cA20R
+txNuZlzCPiTSS9+oxUGYgDVdQV3udOll7FGxOrr1cH5AgJQyeMyTccqD0U/GG4pc
+qPKbL5i0fghwWnnt/2gnwdTp5ZGKGGoOss/jWzL13LdMQzsD9OVPSH4AmQpRqM7v
+r/nYgnSrPbV5O0+38Bs+j7IeXqJNms6TCG/zBcCGBrQ4kRS2pQ==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c5.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c5.key
new file mode 100644
index 000000000000..45679e7ec2fd
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c5.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA3WvB8stEkdKivPWlQJhEZycy0V6/Sbj8z0njTfLt4r78lNjx
+40PFt8QkZwE4ZL1ul0AdtQstoMMp+pGxAFNAcHIR9A6DSdtTvofkVvRjs0FRNn57
+3FTlMZR8bq6C5ZjbjF4+SCOqk98hnQAdAJYofAGVzcOeTy1NzH+tHo8TydYmrmYI
+H2WvdGTUAr41sCgZ+aOVo9Epg/yA3Tuqj7YKKTes4JqUDkCq6Z48BSJuYKTxUJkz
+11jirP1WHbR4ZXCflOY0dIsLXkc5ggdmus2BotKUnc8kS9GJMFOdGAcU1z4ILrii
+vyqkAPQtG0filKA1zeEBrkRaYkdkQV7VbzfQPQIDAQABAoIBAA87BlBJIyJUttUA
+oKlcz/e4fopgVhAgJWOs1XHbL/nU4VLryVe0pbkKSOa88sAbN6w3wNNfPXg9dHnD
+48T99QTHE+0Rwn4mFBCbEF3a2bDO4WM6OBLj1BAhAp4qat6YMSHqqaYUrfsV8h32
+kAgmTAqboU5SEmWn3B4g+iuL8/SCIbiDb6Nma/bPKMfgwiHIu4fXAfVvP4AGeNY+
+HZb+mn8PX3n/k9vBOeoP62G1E1LmT97J6pfw2pNmBNSjzMMk2jGY/4A0I6iTz9OM
+mNu52dzO+QbtbIB99tDX/yJ6AHYV80DDPOqXUtvDq3xi1QSZer/40a82h60fZbLD
+7V1aeBECgYEA3aHUbQFu6GubBOrr6NHRPFVRcDcoCOd2BoGq4udeg7Ac4HySJDbO
+csPIHOFwRWQ6FemchfUAFmBzlq9Fpa7/FrkEu7Q/5KbTYs/ZLLZ5Asi4IYTJDhOt
+TTM9pvNUicUD+4dWdzEn6bYbflFFY/DSawYysG2Yo44D3SeiBLfYE0sCgYEA/8GL
+AomEVEC3qPx50viGuFtC0yugcUqjgSGziywUNCZQt2o4Qq0iZO/lto6oj6j1t3Mm
+2dicWKoXQBNPK1ygeqLpXA/RLbe1sJIgzTFhIDSwfh7KYNfqk7ykYwklxP1rQ+zA
+HZg4BqrJu7vLcbl39zglQDbkHGK9kz5xnuW47ZcCgYEA2rvkktBfTTMmA2x294hc
+IFdz0HQfJuqJKVK//MCsHvupBETtVqBgkBN5ZEd6LoIEcavltq9eLrdyXL0O174m
+M57WdQooJNogtdy8UcylEYJXd309kI8K8q4noqV0QaQXkh63z1rMMXRxHfHHsiLz
+CH1NeaIwivqWBaYf2Ng4gJECgYBUego/cUrLRYEkiRh4LezZ0hMVizB4m028ZnRW
+gN2/h/RbUPN6WpOkwXSfwyk8InPe/2MSy7CAvNXnLTmWOgcr2sm3xeWy8x5JT0sU
+W74YFC9DyUnRiSALYmDt8WSPV6Fpkza1z8SZtynhH5uIWdmdR+dN+ZOJKLgzfYms
+EXbuywKBgQCTIvVwnNcdtcUJktSGgNKPZTZj5tE3rJMvdQ2ceiJEh43+Ma7amRtf
+0+MqRpTYk2sY3CGi9OTpBoKn5t84ZI1phuianSnt9irQRdWsx+6MOlsK1nvsuO8O
+qTLjo5E0j5LRHPlf9t/hBT/wHALQYno3jUnGiZ06HrBIFT4pDLI70w==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c6.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c6.crt
index 35613a36735e..fe08e8790a2f 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c6.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c6.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAMK8wla5/wXpbdq+VeKNBkL0pv6/MA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjMuYzYwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAI9iaWrpHwmXPBC3YhfzkVe10A7AKjRzdo5uv4ra8xQmVzsdk3lQ+oQD
-YMDFoP7rfxtRLzB2Z5l5rChFbkLav+3G8DJ09Vdzs28J9Kxz7N/H9tEiTmgvzdPV
-BWRyA7D2WPPiaHBeIIrtgzfZbHenyxuDxfNtvgvSgUZMEjM7Vg83AgMBAAGjfzB9
-MB0GA1UdDgQWBBSADI6qBveWvxd9C8xucsFtogIDRTAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMy5jbHVz
-dGVyNi5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AEb1kIeYEWrqqCHC3g7eM0r7sCIvgL4fLNeWNLSMnECM2nBaDTpC7JQhVo+nnMKE
-hgYVZ1/Waj7LRQTR0tGZwRVXWKLB1suCszp+6Y3ZCeGR7ife9Nu+ZFucszw3IfgD
-q4DmNB/mVc4Uab7ud5BdpEVMLlbExlasiJ6pDMf0ql1MpWaRfV6pExxbeNf+FQhO
-pHWIzNk1Py/i2sTOOU6aWajQEY+2xFjlcdKzNFb0Dtq6ALsD6NuHnSIqAaDH9cAY
-LmMPzzBy7s/4HAO2LM0xyec5T4WI5+W2f2V9xpkgwG8gp/ypdABJfw4/WyeI+bpW
-ki8XWEjW63yhAUDTsAD6pXs=
+MIIDUDCCAjigAwIBAgIUSnQ4R5z71NqC7t4zA/TxyVfhtNIwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE2WhcNNDIxMTIx
+MjEyMzE2WjAQMQ4wDAYDVQQDEwVuMy5jNjCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBALasyt5jgxI+cYn4cd7KFSUkbpldhOZKtYWrSqCq2VcW9l4nRugN
+s3W/TXir9oit69uv3Vj3fKKl8w9iNqnPBbQSlJ16WgWnDBLaAeu9C6rMTNHdodo5
+nNkyP8tuZpNuaflK1wsHilj6himInmGDvmfk/qlTzB4GUQjnsLGCSbEFvKi76pZ+
+2eTSTagggP2+TkquvtBnAEUVQWIhpYspTXC5608gP14tZIZ3SFUXOWBjq0U0WoBE
+xBOsvjaF4X7CJ2QUOzSVGNu4BVBeibcF12t/8djdurzYB+VxF+1yY7F642Iw+fFg
+0j2/b/Ki3tr1Z0gsaE7vc3yVoI26BYtFRTsCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+mrhKR4ll3OD2TS4pu6aaAILHSQAwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTMuY2x1c3RlcjYuZWxhc3Rp
+Y3NlYXJjaIIcbm9kZTMuY2x1c3RlcjYuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQBooCTFaJJN38wbSRlwF+rQeNH4YSpaQpF4lLv+
+0DrfpKkSF8a9YivfoolrYAYe8sLtHADtcxu92b6KIWmd9uPShN/7fnNmnKLRgZ+x
+q1eZp153S3HKa7aGaPLMMDnu7iMJetuZGQx+Loa4cIUL01ldHVx9xsrfqg4aVzhQ
+dfhNDdik9EWtFDWuN3PdxZcueeaV5JnDB3OGm50u8+hhSQagmY9C8zgOjuep/WWl
+xTK93NoFm1vczdY8WKEaA82ExFcXlYgnUDNA4sEOFes+eAkhgunvKZtUjejthKmI
+/XVqn7b7WsbF1gMSvDLr9EWDkvStJdEeNrzN2J3hPdp/iwuy
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c6.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c6.key
new file mode 100644
index 000000000000..1753b6c42d66
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c6.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAtqzK3mODEj5xifhx3soVJSRumV2E5kq1hatKoKrZVxb2XidG
+6A2zdb9NeKv2iK3r26/dWPd8oqXzD2I2qc8FtBKUnXpaBacMEtoB670LqsxM0d2h
+2jmc2TI/y25mk25p+UrXCweKWPqGKYieYYO+Z+T+qVPMHgZRCOewsYJJsQW8qLvq
+ln7Z5NJNqCCA/b5OSq6+0GcARRVBYiGliylNcLnrTyA/Xi1khndIVRc5YGOrRTRa
+gETEE6y+NoXhfsInZBQ7NJUY27gFUF6JtwXXa3/x2N26vNgH5XEX7XJjsXrjYjD5
+8WDSPb9v8qLe2vVnSCxoTu9zfJWgjboFi0VFOwIDAQABAoIBAATYxHsuYxG5WWmC
+j4GMlWS1DkArHzW9rApIt1lYSchJ7Nh7rKHHN0iQ/lJiJvo3ucchkpBj5J6AzlhN
+T/ZPxAQj7WXY6LQ3W0Emyp/sFff+pVfOIcIN/u+I/0X7qboQrGbNCbPvtnJ87HwD
+9XEl6IfjfK7rxmx82L/mkdAWjQj8JVSCkNamfIKqFKxBUv7CxW2qZpDfUG9yXXKS
+HCXAx2ff7gIsaPGW77EUcDNS7yL3KVtIkEwcoWPixBKAr2rbhihzNp+c0MA4KbV9
+HTQEYyDqGX4+Y80jznCX4xpeLXvAEtuzktfRTWQ3IXp4AF76XDd8PlPVGEzjsn0s
+N2jF/80CgYEA4YAWH3FlWZ09RjDysh9axpVaR2+hvTojTsNCPZ0TdHhJOlEGpZDD
+Kk9H4cVHPZ09r/ijPQQ+dWL9sfhRgp3M4nUnzNwzSeXIu0mP9udwK128RDQswQ/V
+cMcPBtbforg7IYLq3Q6jDyW/NaZd+02vqZmm0Rh4Gz0hWLrD1xQZymcCgYEAz2Hi
+bGw8MrG+GrokbCRIEb2/ODPouO6Czu2OWddTSmjMFhi178gufHbqHU2datvDwnQo
+WkM7HKGYQjXlw5+fjrJojjh3vZU+ukm45zNdjs9IZA3HOWU41L9GzRsxrkMZPfFF
+GamrjItS8JKSc3erfnJcl2wkJJlDDTkbDKE3Ug0CgYAok9TnSY90RcAt7kbt2IiM
+LXn2Wtce5uRNmwNDdQ14KuD6dnQcvOTxqxyDOu2r36ynCrXNGANpOivup4F7YIvq
+wCwuR/2OsaOiWAVo55Keqxpip4TaJNBCn/9LOsU5Ua+KQzWiG60p35/9tnc47mzK
+txR2kdGjDSgAAHOXM03ExwKBgDnJluSLjIK70LN15yxTN9npu90XaeQOYljzDVWv
+kMANT8FghWIZsApywG4F8LWUI4qN/6iXT2N9DAZ3Kl1Gn5mN0WDAD22+psoWPPX/
+9DQVQqDOzhMKR7nQyxPgXgmnHOOL1at/AhSZxjTPQwv17s2/ZFm1gqTIKE0I+qac
+TeQJAoGAWvzYUHkXVcqSIp2cudsXMIiQQ4rt18GvYwYLaEI7qtshfISbQTwSPzBR
+iViFqw4Ne53mlKrYusMbZPJ5gsmsrKuKJVaMyil69CXh6NXxVrmaiAFYJabWKZtd
+R2ncCgdPUBuHBoA8msR6h9XRZQe8CPuIlJMgpPXRfQYyxbMsZcM=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c7.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c7.crt
index 8b9597217242..6736eeb6ea86 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c7.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c7.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUGKYfqn6eHAmevff9PPzxeULwjFIwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE1WhcNNDUwOTA0
-MTMyMjE1WjAQMQ4wDAYDVQQDEwVuMy5jNzCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAxbb/mneg3oKdKk7tMpcpC+KDBsWWvgx9tKPqHFBWIrklLEeHVA87wuWo
-XVoEmoPRmaJkHCuNSUOmNUsw4ZAigdCHLI+Fxgu3RXE5YcOlR/VcfIdmeTOkZTmf
-kAVl6sRIPMARTeQkqaaG93WtXpSFQ+hO3RBIkJ3t3vTL7m7dUH0CAwEAAaN/MH0w
-HQYDVR0OBBYEFL/uRR/fiAlWZMNJZ8ZcDCDLLkCaMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGUzLmNsdXN0
-ZXI3LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-r0hXtDPClFrZFUu8WF6h7kLwc2A1Dp3lFtYUrih9e7e2t9ANPMAfYbUN+bS/QCJ7
-1ric6GBn1cyLagWV9EyxoILtkd8lGkXUC9tuS3uk2roq9KBOFgmoIZ3zvWaN8dOT
-juqD2Z1hOQDUTYdiggONDTRXUc+PELS34571oFXq/c5/ZZlwww3J+Vxwr0a3N9LM
-l7hcobhUZIkFz1GFjU6H/64WBDtb81A29ZVGDsRSqsgZJECjigMZhmq9XRPjVSyq
-VK2bCg90lAIlH3408XZVXNahcpaWz6VXq4Rg2QAN9yES9u7t3qkL35AxLmCpmd27
-xp9v2+CIHghzMzboSiZ1Ew==
+MIIDUDCCAjigAwIBAgIUTcFvgBXm16tCqDZXc34HoyfR0AYwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE2WhcNNDIxMTIx
+MjEyMzE2WjAQMQ4wDAYDVQQDEwVuMy5jNzCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBALT4AM8T4lJiPu2zNijce7y2o7mNH8WV4UwegMfyb/JBuDenTACZ
+V8DoYtug0BL8ugum/I0esLPHSdcuyo7Pn2FXHgTJ/9b3btQ/aKArvtUscHIxMS4f
+PhOU0LyfRXErRq8NNqL36B3RV7rG6vxyhw7VLbouEn7ZyH2P22zHjZMWIesvk+yw
+Yybq4d5byDuMHhL0uTRXG3O05Eej4fz1FYADAh+ZbOnVs2U6Y4ra2yZIHtNnM5SM
+MHG4TnFwA/nEbPOa0Khgc37IoBZ+eDehTHLVLESxN7a1JZEbarzisW6bpUbCKRf/
+ycKT0TUwOHPKIu980FNk89dz840Z7L1sRI0CAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+bc4DH6455vgp/RtqE115RYRhjl4wHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTMuY2x1c3RlcjcuZWxhc3Rp
+Y3NlYXJjaIIcbm9kZTMuY2x1c3RlcjcuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQCoEhsgNWIwwj+ACbJj+lEPCOpGmPmYWPyMKtnI
+W5lxvdp9DGDFYtzpuUAsyKMxT1fCGGrED3f8sWnIujGQYdBNkrhhsgSnCKqmlGXk
+M+YCbeoEhNcaPkcf1ZWfZvCkOoIWt6Q1E1ymziSaltCEG4A47jfSAbr4cRbIoxpK
+Zb52G9aAPdjbzIfxW9NVAlFelD8peahpnVWN3+ghNU+7SXsiP2OjUY2K6guur4mH
+LxXinkDmtsZT/k3HJRujjA3hlDcvV6RzO4GkgxV0kBodWQ0V9LJFzp3RLSI2PPBY
+edguNIMyNxzEU3DlW38U92iHp/qwxkvGFCIM0g7T+xSu4y/o
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c7.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c7.key
new file mode 100644
index 000000000000..73824963559b
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c7.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAtPgAzxPiUmI+7bM2KNx7vLajuY0fxZXhTB6Ax/Jv8kG4N6dM
+AJlXwOhi26DQEvy6C6b8jR6ws8dJ1y7Kjs+fYVceBMn/1vdu1D9ooCu+1SxwcjEx
+Lh8+E5TQvJ9FcStGrw02ovfoHdFXusbq/HKHDtUtui4SftnIfY/bbMeNkxYh6y+T
+7LBjJurh3lvIO4weEvS5NFcbc7TkR6Ph/PUVgAMCH5ls6dWzZTpjitrbJkge02cz
+lIwwcbhOcXAD+cRs85rQqGBzfsigFn54N6FMctUsRLE3trUlkRtqvOKxbpulRsIp
+F//JwpPRNTA4c8oi73zQU2Tz13PzjRnsvWxEjQIDAQABAoIBAETz9gLw9ZwkWQvi
+6/D3yaHnCU0q638yk4yYVsNYqbTpu5JhCUF1LA2RvhEnHgEOEKld7/6pQfTOK2F1
+L7LogzbopcR3Xyb+UllG68uTv3ukGhg4jB+hwdpeKoakPSZYse8DHt1LjkLGQCIn
+puOXk8RgD7NP2LZCrvTAun+hLV1sPNhBRlxcZjvE0HINJ3Pw5djU9EJPMwkYVTWw
+t+2YbTbMLc1ZC8z0XYZGDtNdtCMu1KuBzwz121pOOgDlTfNEIq5s48kTR4EsOuaB
+7HIngX8P0stA0Luoe+UU+MMOftVtENr61BkGVCy1ARaBJQug74ndddrHd8I6VwsA
+tloJd5UCgYEAu2K8H2dw01/fZ5PQvysqGhsH/TYf9yi/+1RUW3NFR6yFuLDbThVb
+xan0AToqTIYOAUHzab0KPW1pU0kSvI4fZ+EdXblfICrwOOyUY40DA5oYjPQgDGCH
+N29rJbOupQjSghDbKU0Afaph5p/r/LfK0ZdrJ8NWbXP2vw7D68Pd6mcCgYEA9zvB
+X5xBiL41DANRKyaLOvO+d3TYeOYNxIZ08EhrZ8Qopr1qz+FR+NvJuFMATSsxFciV
+AlUFwTDjV+KBO+Q5+vfmqXa6tFtnWChbnvQVd2UH2ijoEICpOVqrgQ98Lo7GFwAF
+ie2vbtZ6gHiEkY4mrE0Ed5ZEHcgMg3M3FJl6KOsCgYEAqHmGIEoVc8YwC2Un+qjp
+ap1BGpUOOl7l9ScFzU8fS+i+Naa1Fkg3wcki+/N60mB4uQK9wiTXzxDO8LWzZdn2
+PKR2jay77SUN5HA4niTLP/LM/fYD1nz3NY00bapWbEU5Vv2mvkRgG5wT95FzUD1C
+2hxkmFSVIUVw5Kh22EO7AyMCgYA69odh1BRVFnf0S346OlEOWD9eogFWpBc4fZC6
+Fat37fJ8AcTNm0E/hruPOExkGx7zwQi4q6pZrt0pgp2ruPyS/DiIxgY1KW92OPnQ
+6QmNlgzPXN4rwWOaiCNK/nRxbh+UMK8hfAxQ9Nn2dce4JxzWDvA3zSClsaY6W2oR
+gaD+zQKBgQCDYbtHAkMjS46xItF7iQGQtwwtI+LVRXWb/WJerRdwU/2/brUlcrwZ
+GRWOfgyVSn1WOkM+totVPYioRhkRnM6Zm2WJTy80nvH4EjMZFzGpQ3f4dTn1LkU7
+etQA5wLzKfGOTEo8nU/c2VzHhK3E1e7FyutFS+o5gN+sS19eIJ/OhA==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c8.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c8.crt
index 7da427b21a97..7ba0586ce650 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c8.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c8.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAO4k/NUJiy5g41bmdZi+G1mdw9DWMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNVoXDTQ1MDkw
-NDEzMjIxNVowEDEOMAwGA1UEAxMFbjMuYzgwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAIr6JyOARsRn4zJ//TSQ3eMigJDnDTz4qloEcGRhdsEmEkiVxmawSx5X
-YZeqvQIz69axBViHQZEa1emNrgktmzR4swV8wUip8tJhcic7iegSYk40VG8w5YSD
-fVhCa4JEjTF+H/0B61j+Gq81Xzqw6g4m8kJWqWCvICNDo5SKkBSzAgMBAAGjfzB9
-MB0GA1UdDgQWBBRYXtwMkHRZ58jOv4Naa7AkihBRijAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMy5jbHVz
-dGVyOC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-ADGVl3fvVGFLIRBK1+qQK1LHGkGrNpSQ3R9q7uDSHBIP7YYp90SF9mwNah+Ma+0e
-ZqkmpD85QyzYas9lbXJebE/Mp47W422xwbVqqCqU+w7XMpLLZUbY8LM0PKyuepJv
-GETzjI58H2Z9kU99xW3aQ2dti8UcfBeoOiFJgWtyuI4uiqa0u/5+by1uBVW0n1Cv
-pyZD+bWfFt6Aoe+8xkQMnVZSTlfTfgpUzWM8vDmNi4l2cSOUHa39zWmWRH+QEl64
-EF1rbGwxaYEZ4/8JAdmWD8yzjjUtmJjxjn6PpLOQCDkhTySlNFApB23bShr/ABD/
-KyiL0JK8z8EBOkUxZiXztSw=
+MIIDUDCCAjigAwIBAgIUftkt4nygxwtYAIlYKj2Tox8h0BAwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE2WhcNNDIxMTIx
+MjEyMzE2WjAQMQ4wDAYDVQQDEwVuMy5jODCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBALvDJBvmBX5New94tLP1NZPKJHG/GYmX9fIO+568xGcaU77Ohfgg
+K9rtXJOLLN2L+fJlmbhVAJ1VHdNchQbHzEmZpZnA7KLe4vQ3Dxg64z5vEu1z8x4F
+VKw3ngkTAom/ROTd9/UOhFe/Wi2kEGgWOsuwKQFAbMJSlsFoMs3YaAWNg8N00/AP
+/6P6H8Zd+sXwmVOEshzaQ8mr9vYMVv19AjHnLRFuy2QcEHK3BH3iuOVp+xd9nPbd
+pbX8tuDeVKiWNSxwkDbEcgfvChjQZoHShKRuQkk0W+5UsZVLuSOGiWLg/4w83Ral
+F+0BgDbP8y+feR+lnBqz3d6xxMy1pLLQleECAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+ajzsQjTUk2i4HNwpR2liVoPjiTEwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTMuY2x1c3RlcjguZWxhc3Rp
+Y3NlYXJjaIIcbm9kZTMuY2x1c3RlcjguZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQAqYrP6cjb8xGFMzMRrvaH4BfAXDD9QoC+x96ZZ
+/JubC7nrhr0KVpJDaFyMkXB3mEMyvLgJ7FlVrl8gVkrfZMGXiBXck7tnPXpKhzcH
+tvqtmVSlSBwLW77t2IfHLcwVse+wseLkiF9RI1AGIz+USI3junl0Yq8z7PE1U68R
+GkEaFxC5ZBKphfB7ctc+8bAKXlNNxxoCoN8AaXN39mU4PSqDKt4D7bPfNJyftiam
+3iEvCxqcmMqjXiGiPHbtYmYo37cwtaIl9lSwSaCHpESj8OO3LXuya3hgi7aDO9eL
+wQH6p6Rg1SQz658KOMfBwNoM6uVrnWT212QWO36MedEZLo2m
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c8.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c8.key
new file mode 100644
index 000000000000..546cad7dacb5
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n3.c8.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAu8MkG+YFfk17D3i0s/U1k8okcb8ZiZf18g77nrzEZxpTvs6F
++CAr2u1ck4ss3Yv58mWZuFUAnVUd01yFBsfMSZmlmcDsot7i9DcPGDrjPm8S7XPz
+HgVUrDeeCRMCib9E5N339Q6EV79aLaQQaBY6y7ApAUBswlKWwWgyzdhoBY2Dw3TT
+8A//o/ofxl36xfCZU4SyHNpDyav29gxW/X0CMectEW7LZBwQcrcEfeK45Wn7F32c
+9t2ltfy24N5UqJY1LHCQNsRyB+8KGNBmgdKEpG5CSTRb7lSxlUu5I4aJYuD/jDzd
+FqUX7QGANs/zL595H6WcGrPd3rHEzLWkstCV4QIDAQABAoIBAAbQ7xgErg0fXLDr
+d7rjM1RXxaIr9u496h+lD9kbvvEDvYfLGUWwTEtEq+ROzHyH8vVL3yxYCJ9NillE
+je84tFe1KFP4Mxb4CqrlL0r4w0xeTaR5XblopeHhHH91lVVhEEwTK0fWBLOnRouq
+3a4bpXuEu/5Xw4L76s1fXh3HJgKSiB4uJ+71kseKj2cUgF0H53Tgy07sJ0XQ3Dsu
+NWh8yVt5k6GqBntJgILeBLxaw/F0DFHluX8mBqvbsNv6Ssuf1PI7OBAvgN22Mi3u
+0/VH2CuH08f57uFLaV8ZOITCNkXWF1l7TvsOzTjUdQmJ5T2JXgk/lsZIIamdjTE9
+n/pk+SECgYEAvNy/vaOIWA6FdXPChFlbDJaz3Fk0hGsP1a4RvPnpP0suZEDuUzU/
+A+tYb9SHuqOcISNFN4JSu1Tmq2x3rB4x/YHwpuz67577fZAS9oqCDjDIO8AoeDd2
+lql/nZpkVlrlNOWREECGGyTLy7Zlf2xs+IPITUchfAzgfncSPoBUZ3ECgYEA/oJI
+4WZeA32xW/WfP4sZ4Blk7g8igVOX3HcFa2oWisWWWoXV7jrWBc3i5NbRQbosBy4J
+17q5Vqqze/m0TfmgGqkK7dTNcd6s5IaWTwO5KlKrK5yzgmrGTp9/REdnaUxMhwBl
+B1HAwFJ5KMe1tCEtMfPNfOTuziYZ35U81JT5PXECgYEAnQfCmVAOfoDccJLmihqt
+vKLVO4IkS3GlQdqwvvrf+BNvqEXgouij04NsDdedXw7IMzlbRNMkLMRj4uaV/lNA
+53KSAIkMAA6lNnqDeeAiUOA4rI2SWJptDMoz5HvJr492AmxaGDQ+DPWkSWYHsFIb
+rOoNsktRSQ5/3kbssXEMPNECgYEAxHeQiSc7xdgetM1sTGI17E/OIRNf4JPGNDNQ
+5LV1h1S2F1UfaxLSKpLf5yWJTgLIgBZjFLyY3DVT7o05ScQ8ergprqTKn2906ciX
+eZF3PBuq9TS3QiLoyMLJPM0eICP9I24Dj13lvNEtpHReaRn/SINbCmGzk0HIUVdk
+KPqDWJECgYAwTeSbX3vfNtTc/XbKJXjN3Xe31xwd3CKV8LCtA6GKCPJbnS5fZnt3
+MkPgvP+hf0szCK3LLZUP2I0njfAP5wZeu6KXpSygeG9yGVzbg1Z0FClwUujjzL/U
+K8F2jO3nh2O8spC5RbRU8JNyBCGLwMs7DdF0K/tDitvJerA0i5Wy8w==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c1.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c1.crt
index 0229d50acb14..01a67be3e16c 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c1.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c1.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAO20mZ8U6BS1TKy9C/jR5Fiz5Ey4MA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxM1oXDTQ1MDkw
-NDEzMjIxM1owEDEOMAwGA1UEAxMFbjQuYzEwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAJbrB0ic/oGENs1JDhRT4o23j3AwbeUE/CEKZqwHUIavjvIHcD8Vay/0
-TpAJrG6EoriqlqNN+m+Bm66NENnyuLsj4X3q4LyX3WwFimQt4LQkssfv9+q1kvac
-UgKyytoZ3ummHI4bhpYc3Bq4SrOQchLd9EZBCvURMKdDPjxsaAJrAgMBAAGjfzB9
-MB0GA1UdDgQWBBTLdAFCf9QVjWvJ/FUvgZl8vxkmGjAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNC5jbHVz
-dGVyMS5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AJtTgYL5eBaKTYNEWgehivxg5a4uV//zasVQHt6DX1WfJO6mYuDQl+OTdwjgKPKz
-m2ICAKjZ2PV066hzazrVULpjgMaUeRNxzNJi/5QqiJorNSsFVwSYkMjDB99KgWPf
-22QTx0mV8141lUZY49iOnLk5u+h2tAhTF9U+S8MvXha+vyKc0SQpJcFurNxEZZvc
-x0LCfLDixW/xs7QcfiNO2/Jn5lQf6hIOkyOUHRfsGs9QXlPyjnhvys45+CQb5Cve
-v3vEDnE2lRe6lQn2k2ZOWrL9cyYfROcJfQLfzoKpNdr2PaFEd1g5Sthznv7KrAQ6
-dHr6b9xgGbf9I349zliuhnY=
+MIIDUDCCAjigAwIBAgIUKdYDDACrr711ztIDYAPYtIQDDgAwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE2WhcNNDIxMTIx
+MjEyMzE2WjAQMQ4wDAYDVQQDEwVuNC5jMTCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAPPiQKn+A39l9TS5P8KJfu5hoh5pFsaDkMT5qSrVJkBSa1DLOYzt
+umxQr9Xwg7rg67zvP3IIsDFXm+vgXj2z27JMcVJwMECen5WFeRdedI7Ph+Uk/J14
+q2Ew/Y+LLG4SL2nHZZpMxKvti502M2fpPzjpZ+/0ivXXAQS52LePYZrK+nMJq5oc
+u6aOuJo/pAV91Jr0mUIPFPl8trNaZfol3OKBDHkxYKlO6R6bR2bLIUkyVD26XnYQ
+0pbcEjGL1b0uJ0JavvYclUIiHdxplIZynO/qOSD1IXd9JEBPYJjlDQErfh7D22l0
+DMQrOILqkDojlosM5891H7JQFOxFTGkEpwECAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+NgJLwqdmEv4JM43jmYqNHe0yGB0wHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTQuY2x1c3RlcjEuZWxhc3Rp
+Y3NlYXJjaIIcbm9kZTQuY2x1c3RlcjEuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQBbPl4px26ZSsGn/DTwYhf6XYug7XD47S/SlFLP
+ji492jLW2nwecIVAwWnYkwiy0+T96zSrQvyIn6YVGmrw61/VOYI/3YT6qSQvSLid
+TxyISqCl5hvgx2zJ11retwpg3w5Ro6Ex0OKKpTQBg3QDsIk1hdQ8Jr/jwriG9zHr
+E0Gz6hUTpUc046NXCrFMkK/WkvCW7vqJG6skixcGsUmjV2CKmQv7QLqurIMNOXEQ
+uXmjs1NMxPObVxVvi/eWhZQountdpOr//uIWZvGcPJvX9tQurt86+0t5t6fC6lgm
+cSrGJifA40LRM7RDGC1m4EOm2ozinWOzsAun0H1XsB8xQDFC
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c1.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c1.key
new file mode 100644
index 000000000000..009119dcfa82
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c1.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA8+JAqf4Df2X1NLk/wol+7mGiHmkWxoOQxPmpKtUmQFJrUMs5
+jO26bFCv1fCDuuDrvO8/cgiwMVeb6+BePbPbskxxUnAwQJ6flYV5F150js+H5ST8
+nXirYTD9j4ssbhIvacdlmkzEq+2LnTYzZ+k/OOln7/SK9dcBBLnYt49hmsr6cwmr
+mhy7po64mj+kBX3UmvSZQg8U+Xy2s1pl+iXc4oEMeTFgqU7pHptHZsshSTJUPbpe
+dhDSltwSMYvVvS4nQlq+9hyVQiId3GmUhnKc7+o5IPUhd30kQE9gmOUNASt+HsPb
+aXQMxCs4guqQOiOWiwznz3UfslAU7EVMaQSnAQIDAQABAoIBADuTFCqyUxZmyIw1
+iF60CZ6fmOOG1jpt3wpO1kURRkPGHlLhIdDMLZLR7vH6y6L12IARjVUEzE1xUjQ/
+XaYC7nLSe4ggwZi3ZtI2hYrw1tYf2dlevJqW5H5cYkQTQiotJQnY3HfhstUwtL8V
+g18pufjkAzC5csx01dnasBa+3p/Z5VcqpOhQxOsJVi3ccpDi4zEe1LYziIlelEbE
+acnXLX/8rrbVTbzw/spGCDJyKhmT1G5VOsJcwDDPD8DY+ocMZa0cbDDsU8YxwhHa
+rZs2O3ZdTIRFNA6ocTERGuCUYTbLFWJ+p5tqWc23LSPI8IqeO7KKbZ0+Ryy0Dwnd
+lH5lFXUCgYEA/b5DG/A6t5lwIm33Fvbfd3j64L6Hucf1Z0tE0ER9NlfeDe1Te2up
+0lZX4LxT/dNcI+ne6sY8EuudNQucubUzrECpv+q+ubd4DZktLwDpTZId8AzkgUPu
+ZAnU8c37vDgEN6W6hJ83mqAWkOM2fsz31mzN2q5J2FXItJWhKuSt6rsCgYEA9g2K
+vNa+KQaLCL7QoKw3/PZylV1zQn0qBjnoQeWNaZTJvsxHPC4S62T2RNZWw5UVj52I
+gU09BGfCbcnMUrQ4AJZbsga0eD7DMnk92Cmz3gdSjD9baRUmlnvwPUodoB1G/6Ov
+EXOtb085iEg3IAiG8S6ik6rKFCiWF5i42XpVz3MCgYAD2TjnWZNqaXdb7lMLFB97
+jmgvtHQWi0Pblzap+f/0LQz/JRRZEN74g7q9e93CQcxp9wgcg3kxyEUQIubzoh3w
+hOKOOJ8/7dpySlp3Fgpz1L2DgI6Or1sURVnnePpSdn8J4QedL1UGkVsGQ4v2DGHN
+7MSKho40K8jphEN3hRRmNQKBgGK1Dgoxj7m/kTkh62n5jMBjUZZQFfP/aEN1KZJ9
+91v6v3BWXOFsjlR2fUEiR05Q2YRyqxdMGAVTZrFKejhXafv6VbC2deGjNAVklAXK
+vQh+IYR3ykD+pitKbxb3S6k29YuKa6nxoQr9WlQsvEiSgyfwEHJy2nx5x7SRpN5B
+8Hi3AoGBANjS14zMpLAMVJZr++kWrqBPZ5mzjhb9DdPSy/CfQL7Qxq8lPPY7Pvvs
+tG4rsmnIWDoO3ReX/QD0x4FQ2IjXrdDN28okYZewnWy6r9bHpUYraXXyhJSw6mkZ
+MPm4GPHHANNoanWZvg6fmPRH/VRiGW1/0OxEnUG2a9P8InvrgYep
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c2.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c2.crt
index 38c9f20e3512..255f5cce3918 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c2.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c2.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAIIj3+gGiW2qnF1Y0DJuJBBFfM5MMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjQuYzIwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAKj0cnL86BiN2VBhvrH3XDplcwd8lFAGxAzAfP+Sx1Swqd6y4hZoX2Vk
-J43AuYvJY7aoMK4xIQJ4lWbrvCa+5epbOY1GGg82Z+/Od7mfZppfajCKTAmAIXfa
-SW48V/fRZyYl3KDfP4h5AqJV02wAJ4g5r7Olm74dwAMclLF6FqwfAgMBAAGjfzB9
-MB0GA1UdDgQWBBQ9SpGlYooWMcJYNuTND5SQk6EPfjAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNC5jbHVz
-dGVyMi5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AGtA9aht6J9uVrrow+ks4IxvSTeDK055rdM1ryCkszKf/QPzsDfj0VONEjJM2GhE
-Wib6KjYz6Lki7mEa6DY1IxPZHzMjjkfeLYmnE2+zgdGH0X5D1bgO7cK7tWmcuVAO
-eN0YxO67av/PZT/00wLYva1ubH+zY+C5AZYOxJpPcJ8JNpwO73UDFZwbnUzvpK6b
-mFN/eBqvB7XWViJcHgoXEHkYH9Fetnpt8T6EYxWDL+FYi6iN4/L0a60l+emeleha
-fcbjwU5Ax630EZAEecRB1D2wYxwOIFk+CGLeXpT2bSgvtmfU/pIHPeSJ2lSAydui
-bVeYBdRoOZeUpX3LXO2YuLg=
+MIIDUDCCAjigAwIBAgIUUW4V10o1g9DeT8YquG+IwcE2/qYwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE3WhcNNDIxMTIx
+MjEyMzE3WjAQMQ4wDAYDVQQDEwVuNC5jMjCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAMEYP3CxCDOvSxjCS/Qz+9icBl7mrBi2xP5Q5166pSOtaoq0Gfoh
+QvTpa/s9eP5Z1SOJhms2PnL5mzK8u8531EYRkkAiCyo8akNcepVT7FQf8AaNeEX3
+iD0PSP3eg4EpuJW6E0YTG/UMWWbrYoAS15b4a1Mm6komtK46bnOuvSHvSEsAntiv
+ollE+CNZBN4XbAMdWq5bLdI+Py5ym2HlaVodygr6VRzLbSaL4sLmeHBCcXswcSuB
+VGxZfV9oBSns5pWRrGqXKTHrR3+E/0UH93KmsS7MWsj/gWyRLuVgqf67fyjH1iP8
+zI3J0xw+WS0SH5WFCR+8tKQsM7WSALnW7fECAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+1t7irljwvS+iZyIUmPEsB7i1d4UwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRYIcbm9kZTQuY2x1c3RlcjIuZWxhc3RpY3NlYXJjaKAl
+BgNVBAOgHgwcbm9kZTQuY2x1c3RlcjIuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQAVIuSIk8J4Hl3YQK3pBe3zMgHOxmZyGgO1LSLK
+cii74UlCxgF3Xte0mPLlD3kbjgQAcqRhnF927U/4GyEiJedUFpXHyrGpVqX5/QCs
+6wvLQZKk/qA3DwJKCRRr0MexTjPnAV6ok3ZJUUTmMxP1lFHsWYcNssnclBKH+BAd
+/1S6QovlCeKN7bVgSDpaFF3EE2D7tPywQ5O1t6w+c24Vy2Fi52r1fMK6/LGWqSQN
+8lNBh1oRzdXu9DffS9dTjd33o+UYackt0Cm+fn9oaptRC740KWkDOP2oWwr+Z3P2
+9JI21Vu5nG5+NmH2wVikLTUoDA+3xPD/uLcNBx/9cpQykzrk
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c2.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c2.key
new file mode 100644
index 000000000000..c5eb9e9406d5
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c2.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAwRg/cLEIM69LGMJL9DP72JwGXuasGLbE/lDnXrqlI61qirQZ
++iFC9Olr+z14/lnVI4mGazY+cvmbMry7znfURhGSQCILKjxqQ1x6lVPsVB/wBo14
+RfeIPQ9I/d6DgSm4lboTRhMb9QxZZutigBLXlvhrUybqSia0rjpuc669Ie9ISwCe
+2K+iWUT4I1kE3hdsAx1arlst0j4/LnKbYeVpWh3KCvpVHMttJoviwuZ4cEJxezBx
+K4FUbFl9X2gFKezmlZGsapcpMetHf4T/RQf3cqaxLsxayP+BbJEu5WCp/rt/KMfW
+I/zMjcnTHD5ZLRIflYUJH7y0pCwztZIAudbt8QIDAQABAoIBADohRBv3V5pDp85s
+07PRIHy9v9c7B00rXJOZJIYql0Nt1fiqrnVUJRWY3daVU4Ly5Uzc/nfxLQys4xF7
+quhFv8RWTtp6xTSyWmUsBoVE7g0HTlmSpkYd9kg3fUhb/+fW+QguK9Y5M33qeVGY
+BB+9TGTZJWozeW3YSCV5Vr4BuVNo8PWkb9WROmQTUpVzCCnNJFsqNG2QUf9kh6B8
+BSxb03j2imvhXT7Qb8fvnJkFryaXnBDSoK9CSxMbgYr2qqGsE41+qRyf9YWLEHIh
+G7IsVlq1/CKyvuP+6bk0wgiATXvxafmT3KpQQEwEF3D+LOiQVr8J+y3L5rcN6Coi
+nKcBzRcCgYEA72NpKoiD848Ay6MXwzvJ11OgI+wYgwzrF0yjQiFAAwjFPtAmo4OF
+2s0eDK0DJjO1N2rD7WCueM1ojQGaWixglDq6N/Y2XET5z1exQ69bruHMVDKz7/hT
+r9d0tjWhpvqXbBXzb6OXlNtwY1BhKOHEo7TdpEcNy2VgYu2WgMpeuy8CgYEAzn51
+gsE8/8XQHMpm4Jr8xhyWFjMoLuxmcZAviljMy6O/H4VwVQ/JnZTOaZUq3AAQIm/I
+5nWbfl4GEvWgfhWWyj2yJiMmjt7HhQz65su/d6WTsbZcMY7AcL0C13trCNoaBQIb
+PKjn7L2CfeFRRnJbDwjCGDxJ4JWxAwc6VhnUIN8CgYEAhWH6ovxVqUuwwcYp41pY
+6GVsARf3wG1eQCkvfhVQuWAVpx4LqWgLGRLY/28XJh4BhtMyP/1yuVSaQJb4De5U
+EcN3Jt9FArxcBK9z0abPiu29D90AbWEMIBoIqc3QOF3AKTVB4p5gQ0zRnkSXHlRN
+YHnSVUpR4VanEfgXt8MKbXkCgYEAmekIDX2aycnXZJ44MBFVdxS3ibR3rwHlGaAu
+zx+BBgf23LfJEp3B2QKhF3FnkLM7U2+efqgKThYm0Nrj1RaiXfu+XU613qeOb01N
+ddlHzRZND5is/wM3zNAU3ifPj7mAy1rWklATDhHywYC/45I47OvKP+Yv6oWoemlY
+wfReeGECgYEAhx0cP2atYt3kzz0UGzmh61ShdJ4xlJfNTOeN1xQA/4zc91r8/Ht4
+3CxjN0CZZ5BTiTygQ88FEv4y+bmmpkIq1ztDJF8dH7C/8WhtGmgFvphuKKkzHtUS
+sSSpR19bUsBoRQin95/WoeeDF5zPXLP1ax9i+u3PfZCP29e2ZUaTpvI=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c3.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c3.crt
index 2a75d1584f92..eccca7f58adf 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c3.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c3.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUf9E+9p3WuX54FsOh8veaBlZx8ZAwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0
-MTMyMjE0WjAQMQ4wDAYDVQQDEwVuNC5jMzCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAlLG3qMUCPi2usnc7jsxRFHVGjbf5+HWY9nWaboIQ1vHfn0Z3m/hOOCvx
-waNY4RF75V1ZSeRsAKzGeDD4NC1KC2Dxa81elMfbXq6nLDIfh0klkK8lSw6snzpe
-8NDMSJ8agNdr6zLUgL9e3qh2pU1Fc9ApR6+HRXA9rcnqp/xm4XMCAwEAAaN/MH0w
-HQYDVR0OBBYEFCz38GxHKudMx4dyBlrAcIB+8XC0MB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU0LmNsdXN0
-ZXIzLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-PvCi4fiQ+711I/HQqE1YLGv6Z/Fys39CV2Ch2f7VVgb0o2O9UVpvs3UHxHwNOJpt
-7l3yap/x3Av8lheE+w+qwlf8M+JZMrdDkMxevc/sTT1C+Pq28uet+RfaOdDulL89
-/mFD9LP/kJUhZ/nH+/OONWJ3j3oIkniMNZ+fUktMp9b/efDGD1xb6BDqfOS6JJ1s
-NA2+gncc7uOkPZzOOq/cxZptlU5jN+EFu/o5RHpkjA3S6yQBav8SgahDgRGeorUn
-WVaOiv1OdX+LuJAfZbGgOOxBi43zKT/TDhITcTTHCjmNKX9rfP1lrUR6lmqmFlwK
-L9pdLHQ/cPPNgWa8C2Cffg==
+MIIDUTCCAjmgAwIBAgIVALtUuorY5wbj/mFSIND3oGNl8ChaMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxN1oXDTQyMTEy
+MTIxMjMxN1owEDEOMAwGA1UEAxMFbjQuYzMwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQC/J5elXGQOlkp/bulJ+IF2eIk0ZaQRQT8QYttSbacCMSLFnmaq
+BAeEosGmVdpQc4u6mjREh84XJmFGTQEyXf2BQWajfoo7FrVkaN4wTfwZs+1+wRdS
+z3fBKkfQoFHIehQSOKE+7wIUkkBllsw7Ku0TwEGtPtnRK1KbY1VQGSURn8ISjJA2
+d6xuB+BCYU9rPhDRdOrObvp028yr18trWniDuXpALPaEMR8fsXeII3BnB/OLMvyk
+jigfe3GZv51lutASe7XEjUaBC1AXDvns13RLUvSRvaAu1B6ftQPohkU/EPnTfvhy
+xNuOap/5of263E7/qM8UhbM64edMVaEb4aSpAgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FINeMlqF+VVaJAOG4bEiybLgcK0NMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGU0LmNsdXN0ZXIzLmVsYXN0
+aWNzZWFyY2iCHG5vZGU0LmNsdXN0ZXIzLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEAmJHlZEe0RmQERvMDnG+lP9QTjz5KUbKoxGXO
+/4xoKQk6JsHbFGFt7yzFHZHjVgPxuPxD11j4vW8Acdw/x3ieWEMBVObQfYrOZuyg
+pc59al6EyDgBGZdd8FmqWfaty5fKNX3mtfhVYzKcxOnOihRkhmmB5EYDipNL6lwa
+pBBJXy9mTeTXUnn/kmiMgCP3CUhW8Bp9RAoRhpMko+3yjfmWUaGd4gVaotS5OABr
+Mg/T9UCrxFI4oQPshttCNYtzc2aFC/1h49nMAA0wx2IfP/w4czq+FSBcIeE8QtKe
+TGoLpdXRVwiS3/2rV1SKGHOa6cLLziC+uS2+6r6rqHhekP6w7A==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c3.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c3.key
new file mode 100644
index 000000000000..90c454c17471
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c3.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAvyeXpVxkDpZKf27pSfiBdniJNGWkEUE/EGLbUm2nAjEixZ5m
+qgQHhKLBplXaUHOLupo0RIfOFyZhRk0BMl39gUFmo36KOxa1ZGjeME38GbPtfsEX
+Us93wSpH0KBRyHoUEjihPu8CFJJAZZbMOyrtE8BBrT7Z0StSm2NVUBklEZ/CEoyQ
+NnesbgfgQmFPaz4Q0XTqzm76dNvMq9fLa1p4g7l6QCz2hDEfH7F3iCNwZwfzizL8
+pI4oH3txmb+dZbrQEnu1xI1GgQtQFw757Nd0S1L0kb2gLtQen7UD6IZFPxD50374
+csTbjmqf+aH9utxO/6jPFIWzOuHnTFWhG+GkqQIDAQABAoIBACiiFucss4wyQFXE
+UszJiWOQjcHYlT6huvEVwn9MFcMuhLCzUilzv9Fe84ETINwXI62f19p1b8aV/vl9
+pvJ47klKiLHaVTHoSti0Pf7QpPUlvSdDjlypA1DJVwiu+5N898VDymrhTXTn/c5a
+dsFhB5I6PPvBr3V6Gw4y2FC1d56+E8H2x01sfH/TgGOHhyrhgSquI7wQYU0hk5sU
+OtCKqAxe8wv3OA/WXlbdZUpdbFShIo/Yp1DzV5cIpquFtzv8Gyl3/wgpb9sIZ9MS
+dHVa6zE0hOCJdmOPLzKEgbvZSU/+oPoanZBU7KP+JrmINVirf5EfBlAjud70vBEq
+oEc7LnECgYEA0IhiKPSuBs2si1bfh1I0Ags+I1XFjvomJcgATTmoAXEgsWYYZ6jk
+GuXzJtKEJK6d5WspLbAAriIh89RB8LRHvIurQwiloyUBVqYMyfXIcszGHWp/TCcW
+VDwitYyMHTHlnbUZ0eCfKLH+HdQpmkO8dWD75FidOZ5y0LzNbPcOL+UCgYEA6qqN
+nKxRogTWmItKTyfuG0RJvGL+ehjzMuxx5HZcUxwY7t+8Pbls6hvLga/IqpidYGob
+8Vzq9xHx6N2AashulvrgPhn249657g7IPLjBjuESGo508aSqDxXA3fHtYcjY8VbM
+puls9mCEM1rMqhZuZ4KV65OI4aUiF1Df4xtorXUCgYEArDLTDArw4gFZKD72mqIQ
+/FcYWslrOCPs+GnthdfEmlw2qQcrw/29T1DvtvFD2Z6wGFCjUmzyQtlAvD5crU3R
+pyqWo+YyEURvlIPOL7/cZKAgQQ4+TD5CvBBkgRKsaTdY6yAvUBZyl4JT24HlUNP6
+fds2ZlueiYQEP8aIxCBHANkCgYB9MrNM9G61Jymqxx3FL4OMjplcZeBpGeiesaFu
+B2vod2QiBcIzmDswh9XCghtoUPYd216/HhjJ6owK9iouTnMaIA0FdOrQ1iKSjNZt
+9xBlf1UeUAS6a2TDmZKFly5T7kk4MVwMbRtSM1o03+uzjzYmCXic7+CmqmiO0fow
++42UhQKBgH+Jkyv0MJNq6BLfOnVR7nwSiVzWmkxLS8YWak06ZQOdBSm66RXuy2Gw
+Pocfh/9iwU4P4yygweC+mJHIDWICUsimfX5Nnu2tpE52skhQIwJnhQFp5fi7okXI
+nZEu92304WhSaT08BfCgzfvTv6ALllZJJFDdG1bxoCw01t24dcza
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c4.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c4.crt
index 141a3c9215a8..a1004a51976b 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c4.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c4.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUBKLSOiGqfamUa60JiTDMc33jgg4wDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0
-MTMyMjE0WjAQMQ4wDAYDVQQDEwVuNC5jNDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAkPA4JMal7nY+H0Rcyr5Kmvu4uL6HG0kWw1XLXq/ORJ+NoQqGJ+X3mJ4F
-HPHWb9l6yfumutK7HTSUx6voZzgFVG1OcZ6dUakffCkbjeoGVe+4G2ojlwfvjQrr
-JhoB6aXJLWRTDY4VrNOgijlBzYwyZHFF4IHLAWD/8u0RCGLdiZsCAwEAAaN/MH0w
-HQYDVR0OBBYEFANGShToR+wz3wlqm3RqJGVNkBXvMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU0LmNsdXN0
-ZXI0LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-Tots0b/0b0tceD8D6XlqY++Qg53berDvwBjEu2ntMqlrXh4r7h6fOr9pmJ/Tf66L
-VJVnYWG19uh9owI5pO30/C603N33/hAljDvdidQZsaKU1KJ4Vun09qWstRF8PbZO
-4OxTzAblcZtHs7DGhKJL+fovS1k2hz6691jfxFXXls3S7asOnFrXoCDkG3aGvvNp
-+HfygNewAQSnimKKzzYHbwqqhnXAPQbI2nF5h8rFtoew3sd+Kdh6GTqCPZWntlCX
-hEG1+MS17pms3DuiJt5TPsZh0piJwkFksbuzkYEwH8Hg9Xvi/0ztY7J9wrCqicUL
-r9j1FaOwi9QlwLVdFAEQAQ==
+MIIDUDCCAjigAwIBAgIUAKKEtzuqjiVlpL/KVrKMXSx21H8wDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE3WhcNNDIxMTIx
+MjEyMzE3WjAQMQ4wDAYDVQQDEwVuNC5jNDCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAK47THKQz2mSn6XosohLlef99gQe7q67qZpyBOR7ee26BHMD4c5a
+peuCqqbs2+CTIrshp0tuiT5L6t2MTRPquQaDXieZOp08YC9BJim/cLqlTwmnIM/S
+9Q0wvzDCKs9L3nU1EvQZ4CFtJJZ0G+yivkdKmZVU8qlNLCq5p6+6qZYqXAxcW41+
+UuP29L3JF2Si3glhx76SfD4ThtNjN3DoBSafCbbNet5XhXDq+YJGbcp6QU+Hshcw
+lOKxDs4liws/l+AUFTskoCtLoPp4GaH/e83+yGEwBLDGIDp1Ykhdnoq2YGYiQ2ae
+7mGqEaOJO87ZpsAHzFFRiqHI6iPFb0PDTJsCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+NJd/ROxy5v0Vv/84Vi6fTCP9fEAwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRYIcbm9kZTQuY2x1c3RlcjQuZWxhc3RpY3NlYXJjaKAl
+BgNVBAOgHgwcbm9kZTQuY2x1c3RlcjQuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQATIYFvnBCSu2xoEQHmCWNKCC2n0KcVnZt1jsr5
+i3mQxoeAdP9S0dC1gKk6jRWSVAb4seXKQazapQ0cUtfpnc/J5gUsuUm4RFz/IADy
+Xi+3lUBNd1alK7LKbDmRt+wV1+HOmJW07K386HZtUeSD1o+0FnEzpRiXpkrwitGn
+NDzSRjRAJz9rnwIVacD+T14LtjzxxMn04ntnmH7K7ytP0Skdtoel5wzsLJMLWhDD
+MtBmwYU291lTJ4sA8Tw9cIU98W9y069N+Ymir4a6DxpzxD/tXgoLm8LzxIo1mQK1
+xFxiE5I3kaCcDlzSL74v8HrhRg5S9YDrK0FacLujsuD7zIkT
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c4.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c4.key
new file mode 100644
index 000000000000..be8d3c5d10d0
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c4.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEArjtMcpDPaZKfpeiyiEuV5/32BB7urrupmnIE5Ht57boEcwPh
+zlql64Kqpuzb4JMiuyGnS26JPkvq3YxNE+q5BoNeJ5k6nTxgL0EmKb9wuqVPCacg
+z9L1DTC/MMIqz0vedTUS9BngIW0klnQb7KK+R0qZlVTyqU0sKrmnr7qplipcDFxb
+jX5S4/b0vckXZKLeCWHHvpJ8PhOG02M3cOgFJp8Jts163leFcOr5gkZtynpBT4ey
+FzCU4rEOziWLCz+X4BQVOySgK0ug+ngZof97zf7IYTAEsMYgOnViSF2eirZgZiJD
+Zp7uYaoRo4k7ztmmwAfMUVGKocjqI8VvQ8NMmwIDAQABAoIBAAL6l5GD4U/0Rk9/
+XYIQDMWVJX4nizwokDtIRMYlSE3ktsvCnrs4+IpAxysodI/v5BGf/yy+fJE4mhDk
+o7K5+iTe3R6/M23ZyOk163GeSM6gIexP07NJ74+krDgosbOun+SQHj1XLepUY+JA
+pPOUSr2MTHjtpWJ4o88tMm1CO0EKfCRgt1T/PrDu8HuhqBBiTANO/3WZneiqk8tm
+6jSpbnL62AxsWo8YjAIcsCvpI+iiRpKfxuTPRcwfQj856YxHXopqsQyIg5hfTK61
+SdDFSMZFglnkaUrR4M8YhH5W8/Bei2PWkdk9EdGaXx/QbbmnOOmQkrTz3E8t8s0P
+1IZNjwECgYEAz8QgHCFfp8gvsQKpbtvQkP80U51Rf7DAVLF0hZsKsIsebRRV8blE
+IR4mtnuHCSgnYZhYSWSq+KLWJYkJidoF2qdHHUVraTabsUOmZ1Vodm1pFbD/d4pd
+ymAZJDO+l6VXyBf69/01QM9E2LSh0wvIrgwmb4rsPDVs6UaWPJ0XI5sCgYEA1q4q
+Q9QriK7iA75KVmP0mEvcK+lC7S4Ob3s3UsZMHOGTVwEwmZsrlfqllQY4LjIaZQzl
+RDyX+knlG6ahI8MCCKppP+BY4q0zSzbiBtHDymtfS1QOd58AH7YwA2WC+cyy10aC
+TAbY0Ld6awoxpx2NW623CJQAuE9I7j3MBbn1iwECgYEArKCjoVvlc630WloA15iS
+ifdRzeuXzMXaeERdsakpz5g39p//7PFndzDrL5Ihl+0iE4gWSxz2pdH+pn9fSoe5
+d3ynrN2CKpBx+m5gM9mWC1hLCPfgu71ewojoOtk3kh6SejHal/RuK+RLqtci4g0X
+d+2mfeVq9Xfaf1Tj/Yb0s+MCgYEAsWfwCWl2KFfM8qb4K3gikxBte+QhqHrc9wdt
+6ntwPpWy+r1C+k0esawV1Zb0KkjBQyxq8f7Y4L7cFAOZOkJWle6tV1oAWiIjN069
+bkE+eTJN56Qo6HV+1rXp64ShveiJChl8Z3NpDOHyyi38Zkunm1Fs5ivYhHcQn8B1
+r23DGgECgYBwFa9bJtVx71JAw5ZKExPwjrXBSh6dFAxclANmEinXMkUfrXwLiVup
+N2pTFWcJlGBxXuXaGnOBqE9ZW+8Q19Y8cufKbdeqLaRhmDwG200abBjSaDQn6fkO
+ttvOKRmgotRwxTeFzl6E6VPC/axP96TX4fKGZEwrexZRcCD0JzLnCg==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c5.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c5.crt
index f0e9b704cb1b..ed9852ec3bf1 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c5.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c5.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVANsQ0wa8/+5Cod0nsGdhVVTxFa4WMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjQuYzUwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAKCP9rT1CKEdFPbGE2ywJgZiX1qf5H36DwokH8diz/XVk/t2QOU7Hqui
-5HJkQ6Du6ywPXOqAIg/Iv3Rj1Vo+B9jcPs/DY0Sj3mKdZHhySaoTHPWjv7mVa/PL
-fMElbmB4sYkgfotHVxUir3Xw5byGkJ+SjG+xYrMOSktBNph4Zh4jAgMBAAGjfzB9
-MB0GA1UdDgQWBBRrw5xcRVozWKOs6ZHd4Q524MCFyzAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNC5jbHVz
-dGVyNS5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-ABG0KkEjBMa7QdsiR3Fi51gX5w4r169yXmkh52nYOKw792zI63TDZ00WNIMAZg6y
-PG47XTbF2DHGsAuPbWYTDO75SLJT0f8XKAHpw8TIdv+zly3CKzXN/94R1W1HNUuM
-lTSy3gbGy3u6AkZXivbtMNaR2loBGsMAQCHDM3gmi8PYsIaL+EDekMUF0NbiFyBY
-TVEGYERzvU+ftXryGn2BML6ODKGGDcSlzTQb+tSSC/+bfoaouYUcvU7DTHUpHE3I
-f1RzSC/NRv/gx3HKTdlpNNUNqIANkyWgtKMBbzOhn0SkAlIc5Ap9QBYDIZW96Aj+
-/Vn5JlqUvvdbtbBbEz8+zyA=
+MIIDUTCCAjmgAwIBAgIVAPlbDG1QA/jcCcRyqUXdE797CG4bMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxN1oXDTQyMTEy
+MTIxMjMxN1owEDEOMAwGA1UEAxMFbjQuYzUwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQC72VZVHfUqYAOhdg9pa49bk4NAgF4MM3UZiF7h2Dzn+kiQptK/
+dEuL3pXYMGWfno8d1h3zfkKz8AvECOKFtrPsRn7ewRYmt/lMwwaVWFmtIKxA8qrq
+zjv6civxQ3ryDU1C1nsjx2Uk2b5JFM8NqauBmm9LFRllXQmRWBqi8m5sBjGRPty1
+eU801JlVBClziuyfEvXdWNOCjovSICKiqSirsKR1NjylPYCEf6U9B7coIrccTXPv
+VKjsbzkHbL8KdpKXgCPzJlSfevuIcuIUJzdJLcSfxjbio63ulVi2I3fa0tsWhSvR
+w5GzEbZ/tOZCeveIT4ZT7OL9dyEK43fjC34XAgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FG8zhw68o4y6xzfClDg8BBDIAXxBMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGU0LmNsdXN0ZXI1LmVsYXN0
+aWNzZWFyY2iCHG5vZGU0LmNsdXN0ZXI1LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEAQEfkYC+lPm1NlaU5nM7Wh1CSOsO8GKmkg/Cw
+ZD/bHOhshxgRnGyAMJGpE3ms2sitXZ0YYAgcII01iIzbg0G1xuWLCvB0mgHI+U73
+ajZwiKMdqe2e55CfY1h97uMYJ5Wv9DeRWNCww2eUZlxpxDSzp1M2gCuPGh1xUAUx
+ClfqMRN3I9MVtMVGr075+ah9RyQvj6NqpvRdjhNGvuYFnRvSZV8sROijaBq2ZryT
+iW4PyvN51gzm5qL8ngfNRfAvmjeRcBhPhcCx/AYj2uj5BhDezroefH81zL/rNGjq
+/JT70GOKCjmCSSs1kKX9/CSxrPJmeDC8M3NG0uMP8Rkay/goqQ==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c5.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c5.key
new file mode 100644
index 000000000000..a988edf3a051
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c5.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAu9lWVR31KmADoXYPaWuPW5ODQIBeDDN1GYhe4dg85/pIkKbS
+v3RLi96V2DBln56PHdYd835Cs/ALxAjihbaz7EZ+3sEWJrf5TMMGlVhZrSCsQPKq
+6s47+nIr8UN68g1NQtZ7I8dlJNm+SRTPDamrgZpvSxUZZV0JkVgaovJubAYxkT7c
+tXlPNNSZVQQpc4rsnxL13VjTgo6L0iAioqkoq7CkdTY8pT2AhH+lPQe3KCK3HE1z
+71So7G85B2y/CnaSl4Aj8yZUn3r7iHLiFCc3SS3En8Y24qOt7pVYtiN32tLbFoUr
+0cORsxG2f7TmQnr3iE+GU+zi/XchCuN34wt+FwIDAQABAoIBAFaOItMOfjh7zEB6
+C0jqeEBcfJZcb7lUfoj4xqtuvUWwk/4e+C2ikurMcaQPTMwbPdukHOHk1CxgF7PI
+2kN9d3MP2oaeVqwZikQd+usWrh9xVh2JE0slIwrXz7qX+s8MZQS+mjfWuRqdQCoh
+bfRgMTgDJnPksXJ8x5tSitd5OHsKJg6tLIFCnY0VHUTMWADIIkDIRrhNW7sbV6Fh
+0Ycedtfk2j7erY22eszqK2vGoDPtNSkmEy45ffVjr6mTCIbr+EB1DMahXzqColui
+lL3idBGC/7BGVfTzSfYm2+0GIzg/5z77NB6APAnuSMZuwPL1lNfx28h7w/FhNDYH
+4wEKy50CgYEAwueLIspSHkTfVij0VFUDvc15Dv3aaV+a2Pr2hRgoZuN1sLxmuksT
+7752ydL88fjUqC7jKzeoWZHkpk2Hsc9Th19zIdSfTgKvOILBUkn7RIGFcOmjsqOp
+GWKZ6m1EzwotPfs9GITTHpOshcs/M+iyh/8fb9Hrzl8wNaX4FaWExWsCgYEA9ruc
+9D6WfWt8pCbHhnA0HqyMx1PWswFgR+uI08Sdz0vo9M0Iy/R5ZN15puRm6aIp1DEt
+ohBUXekcXoCdXFOg5CR2ABvJtyhsAmGG4fRE90cMd05nVcyqc5LwgcEX3Rr35G/7
+HIFh6IWvwtNuoJ9WJuEJ6E+Xm6n6oBHGfCtgqQUCgYEAwQZhZHYs969L5F+xAmgj
+yXDaKL+99Efr6mI5jTyMnsE8SGl975f1SaU+rcjYG6hMq7er4ZxAjVyQrybvcWic
+F+Cu/F0/3BmKgRP189IzF/iSOG4bYgr1qJ9fQa0vzUAN6GmOa174WzrzTJoqoPUf
+AtLmxm6G856P90HEvWNsZ7sCgYBMHNoxOnGAJT1hAhmA7nIVmFMhVkqgE+eYqB8g
+fMHV5evqQKnbCQK9y9iwVDsvDXhr3fPez7zG75dxy6k/hwjV+C1yoR/b4ZtYLK/I
+WmtAAOnmkAgunqFiX8zTqjZrXdtk2+qSA5OpgDwVFNZGtICGI9vDj3IMciY/ZNxf
+B56c5QKBgQCizpxvD87D6xJCBWBP3f7P5ctXyinRp3KwaV2vMjsnoR916C65Ao4H
+f+yFsL3GpqfZfejyNHvvQtNr/jyQa1Zull4s6S+NqMLBAhdyaxWPeQrsHLKffRr+
+uir3KIi13uwntmF/44TAYv59aGtPn0h6Cro0bDcKD8PTKLWJSpKT1g==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c6.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c6.crt
index 797186f47235..d5c03b9a3079 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c6.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c6.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVANcbAqZT0PhoeOyE4LXgFN/20v+RMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjQuYzYwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAKiyEVPmc9H8quSkeyFE0GpA1yuRJfDd1OJ3I+ogifTkmZcKu3u1wCbx
-v675xRdpRzCbB8+hUr6ZC18qWUDxXp1ml4iRr8n8bVAmENgJzzp3RLBZROFuwVe2
-dttun1SMiBlnlcUnmSCX7OCTJWrr3tI0/F8RIdaB+63gf2Eby2eLAgMBAAGjfzB9
-MB0GA1UdDgQWBBSg537AtIe7R0hQVRHdpCNBBw4I2zAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNC5jbHVz
-dGVyNi5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AC94yZdlaV+HvgSBn/uc6cDYBXMqe9tW1ENkEpUTYl0Umyy50qxonlsTvZijKB6y
-S9vcnvUMs8+AOGPgwvEszM3qY1R5co+Rl+CXI0sx2iZKEeyGDxAFDBldpGaOqV7b
-AXKeyR9SB5/Kh8XZVyxamtY0iWcRqdc79xljkZZ1jl3GbhxkXPGtPYYKCwh3u6hB
-X4cWgBD0cGvDi4yRiHI/o2ssKa4DGZIMKx3Y6aJ10l/7cJ4cWzkOaEc1RsUTSPtk
-MTkO7l3wt0VgCZz3ow5huwjfKwQgKGsLVEZQiLJLCn6P+veo0jm6Bf+pA4vVKWE7
-fzmJuag5fMuoc0oQbt7BxWc=
+MIIDUTCCAjmgAwIBAgIVAIE2Pyz8wcBcRBAD6YhRUv1qvIalMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxN1oXDTQyMTEy
+MTIxMjMxN1owEDEOMAwGA1UEAxMFbjQuYzYwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQDxwut+bfxCJvxtkF2ygK5wIaqpmYUTcmz05eZK8cKnPqNhwg8f
+IPuRPOK9xva3VM/9SY6q3lzGnn6tUhu7GEJGACzLK1wPH1hO70lKQOm1O3+Z3V4Z
+HyO+2bYCEe/fSq2xWjiJAdGA+d7HJFb+2jnaStH1n2ctAC2ulqJ0P8SnuCxDdX2x
+kD+a5/eCpWaNfLPrEpcZuqdhWF4EwKR0tFvgrPmkBpfODWvRNj7NcbN2HcJeFfqu
+AsUJKENCH09hd4Bbxn38YWm6T1x2XMSv/8VCYXKDeuvrZr4gcnpgoCZnY5R/CGxY
+aZtUEjMKOwERS2TMwArBuOsk3S7D57u3PW1fAgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FMw0NbjiJCX5d1BUpW4Bk6h6dEx7MB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWCHG5vZGU0LmNsdXN0ZXI2LmVsYXN0aWNzZWFyY2ig
+JQYDVQQDoB4MHG5vZGU0LmNsdXN0ZXI2LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEAJpn/P+RJrCDIkpRUV1TKr6Kimtt6E1qq8kMP
+qC6lKiLpo9ODgzmSkHymtYMM7cZXq7CIt1JH0J/jlPMCIXXkSfMjgPCrkqWaJf/W
+np0A8qFkGPFFHQQ3jbgcHVNdUbffAljm+T+k7dBfTAE820RcZP9V+22DdLR/BWDN
+BF2zI9xtvZ4qGI0l5mF8CUMEKBsSFh3AGa3+BBhDv8xT3MMJCANa5Vrl1kIzFwoe
+8sSFhKynKt89H6027hJXUXI/L6QAPuSP5FVhI0lewEhLMMEWxy8FakuMe4rgyJcj
+98Q2Kd6TtNs9yOAt2u52sEap+K8BVsKSzEAr1bSCzI58/jto9w==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c6.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c6.key
new file mode 100644
index 000000000000..b760e718ac9e
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c6.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA8cLrfm38Qib8bZBdsoCucCGqqZmFE3Js9OXmSvHCpz6jYcIP
+HyD7kTzivcb2t1TP/UmOqt5cxp5+rVIbuxhCRgAsyytcDx9YTu9JSkDptTt/md1e
+GR8jvtm2AhHv30qtsVo4iQHRgPnexyRW/to52krR9Z9nLQAtrpaidD/Ep7gsQ3V9
+sZA/muf3gqVmjXyz6xKXGbqnYVheBMCkdLRb4Kz5pAaXzg1r0TY+zXGzdh3CXhX6
+rgLFCShDQh9PYXeAW8Z9/GFpuk9cdlzEr//FQmFyg3rr62a+IHJ6YKAmZ2OUfwhs
+WGmbVBIzCjsBEUtkzMAKwbjrJN0uw+e7tz1tXwIDAQABAoIBADCCafo+W2VyqSfa
+1dIhW98IAlRxh3A1f4NMNf9Hr0UbnnDZHWujlJa53s0xhQOYMvuhrMzAPWFlIAnq
+9cF+xp2BH3vMwXYPVgrQnKYXjE1fGOVos57azrNWv9x+eOEW/O8fKUTkZoM4n+jZ
+c9NDOmSAqsxcJbmz1Xa4pvQob28vc2Vr1d+n1x2n4f7DrmdKvtig6bJ6vGerltEe
+gPzUs0p0W1PmDUexW6yejxH2N4jN3BBIS/iAju3JTKVFnB7UDpz+aNIrFUufZwVD
+n/VECHW/GVEzq0UTU4wSQWOF7DqJNGVZqJ22UafQbkPwcD92kmyFzx1g5+2ocabo
+K9B6tAECgYEA/ogxqC2t4isCFiTmsUX1me+H7E7lwKrSA2QsZEaUO+cCmXkeSnjd
+GL4LeZ0ZPOaY78esZjRLwU5Dbq2ToQ9FYWEGqwvykpdmqk+P96WA8iYcGbhFgSZV
+jXm6bZnbNPKRx8L5zBm4rta9ZQAIM8tEHaJTOSt7C5QlfD5zuK0x2l8CgYEA8yfe
+42zNF1z+sF9243oLIOzeI2MPe1JEMX9rXdKsIlPsr9MK2o5pTJyupn9l8c9idvhJ
+9aR+m7ocWAPRYxbkzmbaV7KRCTuApemypsoExIOTMmXNQXvbaH8MMyHQRzjUx5WW
+gtJuvvi3O1z2GRUmCgQRt44Qmt9SIOBy1GvDTQECgYEA/K3vvek7CZ9wftM7QRkW
+bC7iXLEmXFI4Eq3Lc/CtD5qRdhJLA9tggkOH2L8gYtj5UnCCsSORYOF1B9bBC87B
+LZ7Io91EfkpW3RQG8+4VSE6U7uOD/GY14DDL5ioq9zEnj4v7t6aBNPBxLGcPvXs4
+r8KJo+OKT0BOxjypXNhy+TkCgYBli2wInAJWVG7EZ0m2G9yT5neqXY80mBmEdkr3
+Rqi+OUAp+T+aIEkXRoOlYqTHRC1Bs9TnMcfx7toxY37JG+F31PowD5GEQoYqQ5tp
+DgrJqo0JSfxa8YeibxRAMknRxIkPxIezoH/BXwreTqczuGJHYoWnmzuSAdtBgZpS
+3aIVAQKBgFeIJHpLZrPNEhrkidaUQkEtAPASHQ6dzDvMBtHGx8N/hb6zyjNFV1t+
+sLRlLXjCNbrft0lrAYpsRyOVpTCJ0xMZ0p2OCorKArGzw2ClwEsdZFYd3FU0I/Vl
+o0kunMpKxrJNkIylVhqqOfCpeZTXP2ybBRA/2krmdQzoYKFwUGTb
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c7.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c7.crt
index 435d12979dd2..43d8ef599f5d 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c7.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c7.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUU67LewM5UhpCYrMabvxZrdBHhCcwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE1WhcNNDUwOTA0
-MTMyMjE1WjAQMQ4wDAYDVQQDEwVuNC5jNzCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAjvRiPU8N+9dppeWCZf3gRL7r2afS6iS7YZn3yqYeikXec14GkDotYxR+
-ZPeG3iTwhE73I+iXIScqZmtqk6sL3Iv5Z700w8CBIhLFZP0L6MPDpT0WwBweRzC1
-BA1uXam3Cwp0hux5xAH49h0EZvFT6bxSVdhrunsWlW1N5vZIOlUCAwEAAaN/MH0w
-HQYDVR0OBBYEFEMbNKUNe9yyOtm8R/QonVKFl7C9MB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU0LmNsdXN0
-ZXI3LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-kIqfKAq278AizLJjj0tTC7rQTlHRMpVSdzQUfdYR+IZ6d0AuPe3drBKEMLsXcY+r
-OndzUU4YVEsq+OVZcjdPsVW5FH7kHYzQsbxDTPujQgn5ED3ggDGobWG6di18hpVv
-S3Nt7iWN7jl2ilXA/S7fskRH6w7mVjiRMsV8QiwwOtTkj8e1vQBa6l9S7vHS2vFr
-U+nbTpdVrTKTZ5+n36KtmqUdhzmp2V3krVCihOF016TBzU7tp3+I4FNUaCLj/bsC
-A4KWacL0cbZul47Zd4cKpFDDo3PxSxDeBgLh1CDx4SJaKcrko9YRJ4ZgA33GAMAv
-n6V8wxMzyGROsR663RIySA==
+MIIDUTCCAjmgAwIBAgIVAMtrnCz4z5HLifxa72xqgDKRBGtmMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxN1oXDTQyMTEy
+MTIxMjMxN1owEDEOMAwGA1UEAxMFbjQuYzcwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQDVl76qwTsREi33MP/QrPXBkqC+Trbf6a0e0qOMhDLCMMnlK5l5
+cioG1JIWpOMtwVjMFKGO33LYNN/dpp0NHApUpph5YsIn+ZhOEMriANXLipfag+l3
+ktL1dMZcjCH1mj2sAZBmxCtNs4WuF/LS2MfB1TsnpXlA3YURKJiBAtAE8Imrnw+b
+fE7EqFoC6Vg3mt2SnUn6MC/+JKZ2LX2l32zxJR191EMgy5T5kXyDL/hDFMwGBamp
+5Q6OZ/Qf0161GNq0kzlGL0nqe3vgYYo9+0GfyUHOAVZQpM1PqdJHNg6Zs3/oaOf0
+KByBUCkqRu3VEjrSRaiVsl5AWh26S3FhUaLhAgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FILhvS2rgpcUzFi/SCbdBlp4oyq4MB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWCHG5vZGU0LmNsdXN0ZXI3LmVsYXN0aWNzZWFyY2ig
+JQYDVQQDoB4MHG5vZGU0LmNsdXN0ZXI3LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEAYXJlXcIE5oZIblhsSM+UBTA2gba1rvaN2ii0
+I/dUl25Vi3VkJhD01UPs/ojC80a5jqWu53/Hj1vFsxmMmzvgIHZcXBX0Iu3tAe9S
+V+T3JGPHangqadTEKh1T0TOfgVKIekpj9+qBR7mEfRvyNSCghoIH3DJkeEKDU1HH
+SddZuzrWa/pMGmkKbqerXHGEUCciXFAjbvrkIY/IPlXaLbJGi8MRs0i1QP/m2P42
+2vInwp/4nriK44jsxEb3/snpPsnhXtggFRTD/k6GpTdwKMWMtJwGiXF1P1Yq1HEo
+gHkr9SvUesSbeiNfwbgQnMQH+DVk57OBW3HY0uKgL217AyBYZw==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c7.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c7.key
new file mode 100644
index 000000000000..0741214e6007
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c7.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEA1Ze+qsE7ERIt9zD/0Kz1wZKgvk623+mtHtKjjIQywjDJ5SuZ
+eXIqBtSSFqTjLcFYzBShjt9y2DTf3aadDRwKVKaYeWLCJ/mYThDK4gDVy4qX2oPp
+d5LS9XTGXIwh9Zo9rAGQZsQrTbOFrhfy0tjHwdU7J6V5QN2FESiYgQLQBPCJq58P
+m3xOxKhaAulYN5rdkp1J+jAv/iSmdi19pd9s8SUdfdRDIMuU+ZF8gy/4QxTMBgWp
+qeUOjmf0H9NetRjatJM5Ri9J6nt74GGKPftBn8lBzgFWUKTNT6nSRzYOmbN/6Gjn
+9CgcgVApKkbt1RI60kWolbJeQFoduktxYVGi4QIDAQABAoIBAB0r2F03KXtaDEUS
+ashz7CyVJgxXxnTWU/qTB3CNguaFW02dTXIKEYtVROPVfowlN/IcxI6KyK3sABck
+RkF1bu5uF6zxu//8VWd0C6wfiRN/c4s4hjj6fdgK1tHqTEkK2RlrgplGeX48jSlg
+bp2KqzLh4xkLFIBeFhUTLPY8YsS0WrhxzFZT099HbdBg0mELPBM1/2AZHbiB6hcS
+vbkfnogOA4mWxuG+J/6r3cqONq3Blvi1P2uT3605aRyJ7l2scaJapyhD4gm56FLY
+unaVNuHR2Q1aW++NdpP5G1YXxIOovmYbzIOpfDUscVdsa3yzL0pQVfzTv//++oqT
+gkCXhTECgYEA9mmghMTj69bXPxYG1S/rfJIERDMmkayQo7OTNKROlt8UTL+YnrPj
+OizhjZHfJVqQ716NGTCNNtda16xyNqSQybr+duXLiK0hgjBaDc8NJZ7/s3omh6DH
+zzABqsQ2o8FAXVIGiNvgHOWv2HCw5thFqVYmGIT9uN7/cSl0+QFm8pcCgYEA3ec3
+3/1zN9ytExM2GeBvqVnGNvzogN7DpLDsI17rfi3GMmsIVlkphAm1B7zD/okX/tVq
+1QRdgghS6Fc5pFHQgSBHpTIykjYbfWJbOyp8sTb0y46ZcgChl/7LJNO3iN/GGJc8
+lLpfuN4nM/maNxTwOYp5BM0ecmavJY/Johdo3UcCgYB2fVBcx7i5vXN/uH3MoeBg
+5G38xYP0ZMCgl0df9up7jgXa6f7fwaywWsiTJOPiuGwdTqAWzsv45RGfyCKbuAKP
+nsWiMtnufSIx0kJbhYbS4mQez8wbR+sM7A7BKN0FXXYv6bOOt/xKqQAn2fiW6MSG
+qxBilgDTlxfKOJ8w5cW1PQKBgHyrbwT1DEqSPNq+rGKVaaanvWkfZjEg9qIMlF66
+8WuOpA+neqOorF52ZbLXJvZWZ5jQppjPPwnTS2ElwEjFfnNNg3MYZLkolQE4xTDU
+igG2con6r9MQ77jJebBTEoZf0pf2QB7/JtBiX3wf+7h+lwh898juxPBZKIXd5tae
+eFf5AoGAX110qVZXUMjtpGcqotOs81nG1V8IQWFLIrqH7+aq8bmy0Uali4Nk6j+M
+9JaeeGJQPu/Vhu4KUC1u5IxKuUkQlHOMd/meGrNy8A4vhhtV71x6D3An4F5J96di
+ONWacNfep9+s0QWud8UMAYz/xSQaVBFtjpqAISlslyNqc44GBJk=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c8.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c8.crt
index bab6fccac648..233e8bda4b03 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c8.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c8.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUZPtamUFGkwEsTdz8nVPsnuVKmagwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE1WhcNNDUwOTA0
-MTMyMjE1WjAQMQ4wDAYDVQQDEwVuNC5jODCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAnUV+seRzu1C0Cunk2tXNWXVKIc3Ia1PjZ1ihx5diN8KsqgXzJPtVl2BJ
-4+DIIW8yUVqera1P9AcbeQo1btzPvAr5jUFIFAxZI/N73x0KBFK2rHp67RPUgZBn
-f8fkNrh6OiW76Dyj8bd1R7PyGrG6hzjW/gkEA5UMY2Uh1hKUak0CAwEAAaN/MH0w
-HQYDVR0OBBYEFJ9cx8fxd/vOjAM6gIQw/vqIC1gGMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU0LmNsdXN0
-ZXI4LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-KDBvftp+20KL77/gc1sPeViQP7SzYbrL+lNggjuBIti3s0oD5Cv30QmXe0EqpWnH
-uK1BtlNu6JZ5izuDneeVVfscCPUqgzjrDPCdFFHcPBPlqcNlUBdZtsAvq/pXCSzJ
-oyy1qkqu82oOhnaYKZVEUdtm7nC0lu2A9TMMQHgGq20GOvLMTxZ07FsYZRrghNRV
-xUI/YLZFaTyEGIdd0hfafFXtmnLjBJvevgcZ36S7BARfw6ZZ+Yum1tlqKBftVTD6
-P28A0nZJbPdVDOBfh2zY7/PokAaIziq3cVuoyOx8ti0kPhZJiFSxjWSo2Guz9jLp
-fUXqHq3QqRdgaYLt1YCn/A==
+MIIDUDCCAjigAwIBAgIUMsIBuaaDLkTz7XiLK/ndWoawaBQwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE3WhcNNDIxMTIx
+MjEyMzE3WjAQMQ4wDAYDVQQDEwVuNC5jODCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBANdG+0IQOjtBwn+7RL0iTnmKrHMouVZbJAeaiB5OhRf00T2Gn0nx
+p0immg20rw09yWOhtoEemsUZ5q/iu+lhnIsdfHMrlWiEDTMoK9jlUH3jQBxkCWia
+KDfzoRHDROjw7BgF73LRX04o3y8wIQiqz1ATvd1XnyB1DPMl6ScgAa4gGlJlxqNk
+5aIgEiNObMV9LcggzojXXeIbaahmL3Y3tOc/hAMU2UJYq9zh9vEL+hjV+jum1ysx
+Is2qw64ttWGjGNc1bKsmgBzfQQrUodUcWyp2iFXyq5lnpVTdyNtdGi7ehklBN7to
+kJoz8PKXCnLybOa4qRm0Rqo2WqPJZT9xxJ0CAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+qPpqydscQtoCgHccHnepKHEo4ecwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRYIcbm9kZTQuY2x1c3RlcjguZWxhc3RpY3NlYXJjaKAl
+BgNVBAOgHgwcbm9kZTQuY2x1c3RlcjguZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQAhemLND4r2/e2DeTSY5M8Icwl2h+vMj/lUKC8R
+YYIveIfIM/inPMiGCeOA3iD3CiLDp+4Ed3m2zliNNbu5+0vE2f45ht80MF3fccKb
+l5BVjG/2g2xbHIRUkshoTR3aw8GTXPp6qPIjaLIxJcr2Hb234UU9BzG7/deK3drT
+eBJbk0O1UXlxcfVFhlc81CmZFzbXfB9KbgN53WYJ9BpB+wv9cVq2nmKcMeZxfx3a
+W0GIQluO1O2hE5EFkLCoMLqzyXpBxSQZcejwijeW+LD1dC8oc+I9eo4q6qnKYZ/A
+qmlE1T6tZ26P038xXyJRIfZ2qVvtraVhVXOLK1ZNwi3eCg5U
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c8.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c8.key
new file mode 100644
index 000000000000..45eac0c441d1
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n4.c8.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEA10b7QhA6O0HCf7tEvSJOeYqscyi5VlskB5qIHk6FF/TRPYaf
+SfGnSKaaDbSvDT3JY6G2gR6axRnmr+K76WGcix18cyuVaIQNMygr2OVQfeNAHGQJ
+aJooN/OhEcNE6PDsGAXvctFfTijfLzAhCKrPUBO93VefIHUM8yXpJyABriAaUmXG
+o2TloiASI05sxX0tyCDOiNdd4htpqGYvdje05z+EAxTZQlir3OH28Qv6GNX6O6bX
+KzEizarDri21YaMY1zVsqyaAHN9BCtSh1RxbKnaIVfKrmWelVN3I210aLt6GSUE3
+u2iQmjPw8pcKcvJs5ripGbRGqjZao8llP3HEnQIDAQABAoIBAAMztgG7njNBlBHP
+31UVITD4CdouEvVcFBxT/CmbPPMfDkxAekkONCxqP62ipyZ5Dickk9W2P+TvDDD+
+0v7XKelqYFxIwFCEKAStRltmLA9kbrUvrt9JEf43Dq4KmWy4s4JJZOKU9My6mVRg
+IvWl+MD/Fr8VbaEhdbZZkKXd+ThskKq97mUy5ockL83P688MaYDe+B9S9CfBE0fR
+FFDKngokaJRjhHPnEalAigiagc8BV8+IhyKlaL7+F3zh9f3ExjOFRdl9xLndCLeb
+dncY60IdgpHmIKBBuw1uuIGNIwZYKRTzr7fVBxmWbYE0WP2GuC4G5UgyomuW/CNR
+A1dZoC8CgYEA2MDK26tuKYNsl/YjWaZuA/O4C/OLL6I5g3n0AYH1voD/U0OIOgBV
+/5UY+s/RqaD0+ZJZNSdhfKCzE1alCwhquSDs0/XFkMeGsP2+XuykeVCkiq3O3Ngo
+tXtxhV2JdT5n+i7AyVNwpN41wAoMSEDf+Eg02X9fdbET+iGDjtRr/yMCgYEA/kHH
+o9uUSTFw4SSq8JNmIkVJmogIsTl2cK+/v9/dpdZw/6KKqBOiGu2/OviyxKzZ52Np
+yGyt6Wn1B77VLYFef9ywf18FJdSOMQBNcv1W8T4leYaQ9T2Y71sy6mj3kOQlb23x
+6g8Bow01+VRBeTN67MdtM1sZUfliYM4OSga/ST8CgYEAqaxRoB/UPn74tQQiq/0S
+Rdm1CvX80K7m4rrGfZ+kJRqJxD4rF2xH5ahrQ3sHHwg2fF9JochQEeUiCVNcIwTV
+tlHF0i6kFy95inc3XMtV/PlZtI1WT56OT3JwbPvJrJPLhRawJ2k/1m90F/5FmxHd
+wVeUt2Vz7odNohI2D8rNN40CgYEAyZpk7zND0AmI5KyLcjieClyAMSzeH5Bdrppc
+Y4rhmAaCEODUKtNufQW41MywCyy3lzaHgSlTeLnQVXTmWa+zu5TEyioQzd9NY+NG
+hgAE+UJ6unKN7DcMVfYtWBsk+dk9Ka623A1MEfp+qpU5WR4NwUOb/dWlLQ+eBv47
+7EyPTkcCgYEAg3DpbUeTeLduJt6Z380yL+M5SjBC7bW40KaKg/hTNEAvc9eHTZv1
+wgPV+gei9VDN7Hubzj9zbV5wS7OTrdjDK4B7PpN9vD2yElNGmYzsAxzapB4KQkYE
+Gn0cFUQ7jhphm5uyUtH+K+FWRZTN/o5+PRGDoB7+5Mq//HovdKQTZ7s=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c1.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c1.crt
index ede8be0a2738..a44fbdb545ef 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c1.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c1.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUKdCyNvk9xgVTDO9On1JBBAJgLEUwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjEzWhcNNDUwOTA0
-MTMyMjEzWjAQMQ4wDAYDVQQDEwVuNS5jMTCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAk2afL9SKZO/18apqEJtcnVn6j5KFVgiwLRnqDe0O0waKAuAatUkRu16E
-TUQiz9gUJ3LlFA/7+A+6pBwgVj5rh3iheQ5/nn34OSP4aVjwMMcjFJgUxPVvRxR1
-pjspxKBsMdwTt6EXBg7yJTWI4Lk92I0Wy6vb0JAf58syQYd564ECAwEAAaN/MH0w
-HQYDVR0OBBYEFHs9eLWpbASF22YffoTchou3BPfeMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU1LmNsdXN0
-ZXIxLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-Oz+MEYQX8+5uRpJQWkdqw7fsmRUuTmfXSayz09SPQ+jF69iXTa64PvEiKkeaLSYX
-mQKNNKpeskYKdGQ/r/fBFo6FRLLvtFfLfGC1OoOIZxFy2y9+0MYroEijQB9dASeh
-L02IksNsvPZnOMvMitETDoQPRK3mG4OcFbdvWuZa8oBJ1cbHqJPngLaUn6PRc75A
-j2BvbeRLT2OZf9whBfLpE5gI8oWhTUXnGps2xKa/368aWr72Bi4/sMVMLCQkwxxP
-cwbG3GKF015K1Q2BgfTDSPnbiCXViuSGw2opCoflVlH8qTcukxCyrTiOmaXfdds3
-qB2TSCnpAV5UehB2XEoNyg==
+MIIDUDCCAjigAwIBAgIUKYcI+BWm+vfX221XUUgOr/3reeEwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE3WhcNNDIxMTIx
+MjEyMzE3WjAQMQ4wDAYDVQQDEwVuNS5jMTCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAK4ABXJPmbySrBkCM04UlFehN5IYtxkomJ9jM/hirkGbOba7uyhs
+3ZdKiR1PGtFTOwE9N2mKKQep88NzOPL2ji6uvvCSpYO6kEeDHiWWr4vbjn4y1T/+
+4Q1lebO2Jd/eskoKKn5h9V7xNPEzKAi0Yq8SBPhUq+zNL9fhmeUIrsH79aj6+4V/
+NmNg6Ozg4DD91Zefp0+FTFrke1hR76pKZz5/OqF+K0rXUjPILuen7DmArKP6V2D2
+KVEwny+zaKy6zAR+lmxRdYPwhyCtNLHPMPB36XnUNjKhtcGKxaoCMApsilHroPLj
+dlXAz1Jq5dAT2+SXhhk6VDAZHpoyx6I+uNUCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+HtB/V6LPv04eTSww/tLDQJeZoNUwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTUuY2x1c3RlcjEuZWxhc3Rp
+Y3NlYXJjaIIcbm9kZTUuY2x1c3RlcjEuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQAB6LAOJk+yOJYclLfAXhqkUl68SQBi0mheKA3I
+nSxPPHd23GNS5VFUpXG7ePpva9kREUw6zgCIKbMNoq3oh38zXCqzVfZzVnlUrjV8
+JF3xSAGxgfYqob7Pm0NWpLq0tFA8cymza67GTeGsEYNnCw6v89ZNJzwJkgiuySJl
+NZv17o+Ldh10J/p6ne8iSmwRVHmiUM+DuxPggJj7swSeT7pQ6wwJMFND39ekG5ST
+UaJs8pv1SVOrAgm9ExAd6pTej1lBIdwlDV8hxmVDD1BxyCf+63kp0blZ52SqErPf
+F5Qzfow0kvKZynrC+nETIpNBAwIf9vy4xi4XrMoncPTb1n6C
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c1.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c1.key
new file mode 100644
index 000000000000..e26205f66a50
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c1.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEArgAFck+ZvJKsGQIzThSUV6E3khi3GSiYn2Mz+GKuQZs5tru7
+KGzdl0qJHU8a0VM7AT03aYopB6nzw3M48vaOLq6+8JKlg7qQR4MeJZavi9uOfjLV
+P/7hDWV5s7Yl396ySgoqfmH1XvE08TMoCLRirxIE+FSr7M0v1+GZ5Qiuwfv1qPr7
+hX82Y2Do7ODgMP3Vl5+nT4VMWuR7WFHvqkpnPn86oX4rStdSM8gu56fsOYCso/pX
+YPYpUTCfL7NorLrMBH6WbFF1g/CHIK00sc8w8HfpedQ2MqG1wYrFqgIwCmyKUeug
+8uN2VcDPUmrl0BPb5JeGGTpUMBkemjLHoj641QIDAQABAoIBABVFAmVuo+PVvWd8
+id9QYQZ0j3GdR3sx/QogGyEQla2GYd/bre1V9XZs0ZBSyaaZ49DAdhlsHpzbDRBb
+cYmIUxQTdLAStQctEz2dTQjvlRDlwKgzjYwk8Mdtm/ANuC6Cr0z2q4AceF09OKld
+WqufD/XHoEeSOMaZxHCWxB79AzkfYIGyl6/Zbm6O0DIPbUbXh4Gh8rBuuj+t+gIx
+2KOONpPGijms8/G7SDi9OWEyY2hdorMVsThTUp1PYyGmueVxNg6h6U6VL+SA5EJE
+yKRLogHgHjbHeZmCUR09xOGvsyzApPaMUD9Tt7bJtdk4WUDxCOkwD94UzyFj1TKr
+n6TOttcCgYEA04IUx6K932erz0/DOEPtaECBOClKgbRaC/x/84gJG4hsDksl5P1Q
+8cJkQ1h8SID6p59av0Zx3K4HV+daHWKO2zoUsgDzpH8T6ecMulinuhYiJPlpj7QT
+FWwUuJ4qvlXJFPn0FHoQheHeMbLm5SbwNQeeAwTEn8z2PK/8DCuMMesCgYEA0poY
+YxHb6ZAfKvru11g2I6rT1wpG0+IgAx7KQyhminS9BB6u+JpLvBh8n4o20hG92VKa
+df2Z4xxukfzekAPca2tPYGVX01yTWRit8zIZ6SxL7uC1STupQzQLeyp9UYMXlSz6
+Z5SOm06uT59JICd0fjbRcC3WXkbudPPE0pMvUD8CgYEAgf1SdBdrBmCa0UC+RFhL
+EmWChVerOOf056T5plyW7Fr9vhMlaQTeuJa2NEWvrIfIhTiE90V+ORij3ryLP8AQ
++F9L1NKM6y5GuNyNkAFopA9yULMQ04biq6nPWl+q+WuTt9ae0e7rNL1KZ9UB3xf9
+TTovPL+zcFrxgv7Ns1UYRp0CgYEAs6V+6ANz8+hcre0fj9JPXF18FxNNF7DsXb0G
+DAOKdNDf64PA/Y+gyvKYW/8qhEdeCuPLnpqVioXWS/HIaFDi1T7UlwXbxRBTIM1M
+lQdH0v9WRo6oFbAu0oSyxn2YKNAKbdXQREsU2qZjSsg1OJhvZrqcLXTsR7956YmH
+e5jR40ECgYA/42ug3MZ5n5C+u9NNfEm+3x3wo75vRmVDtd3b5cJHq5EHuOKkC/DG
+rW9yTphoe4vL63875oq9zIZm0z69l8Qm3kw5LEJt0yUHaD+vQwtcxVuJzSKaOmT2
+7ttwZlWmGwQCTCAYVMf6p4hGP4qO4bwoxagUnwftBPVQUNLt3ogizg==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c2.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c2.crt
index 27b65a7a5923..7bf8c7226c28 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c2.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c2.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUDQRkdVBMn9YP54iLThtM+V9kTEAwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0
-MTMyMjE0WjAQMQ4wDAYDVQQDEwVuNS5jMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAwIe2m8DEVRgwTgfukT218TRvgaJ88GO5v12NUIg/uzgq2ADAIS4lynhz
-5mzsdhcArzWFCz7MxP/fMxfwzRP7dNm2pjz57+DdUUSpPrBCDL/3+JwHWevd6LgM
-mgLOSohsJOZmWe5PUVMiP8/sluBb1+63v0e3iDNUXreVal0xxF8CAwEAAaN/MH0w
-HQYDVR0OBBYEFOOs6r9usaTHZ735J+HCRsZb8mDmMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU1LmNsdXN0
-ZXIyLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-PsRSE3bUNMgAx8goznM29Uu8WzrsA+8yDd2Ts/Cjj928vkT4xLiOOAGxnNpdhB+P
-9ii99A1bbfacXdyK6Chm95DoiLXfKZusHpu9C5EvV9BDQJDEqHQbf3LUGo7Pl55l
-wdnGBvAmNcAHZ0mZhvm7F0bXBPJ3beBbnjrcJGcL+99x0SwsXYeeLGFNdUft3nA/
-wYk386vX4h1Rq0c+aJkNey9eaf+qUMfClEtAX5okjIvxT4sETd/w2gB9tKldMX88
-0KkwjIk26HqQb/TYZVX5rxrP11Pf3mzbWI28Fnscw6ewNRQr/BWAY4p650MJ1dar
-RLQGA5ZR/1IBQggTer5Rcw==
+MIIDUTCCAjmgAwIBAgIVAN00xIIASDUxm27YBQBd5uNc+b3YMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxN1oXDTQyMTEy
+MTIxMjMxN1owEDEOMAwGA1UEAxMFbjUuYzIwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQDNSvOBvU5gTGzSQ7hGHdD27WoQuquJy5DyIYbX00MgwqerCdnw
+DLHFKxb5WkZODGXHrE+5CcrxEzPr5h0fNjgbAGbtoBs5QKbEAvXiGmFl18wsYtkb
+FzMt+0o48opv1rlwW5d5k+nXEt+xwBXvmdekl74ewSchLyeKKN5bzK1D5f47Dwwc
+rOqd43iFJUfBKGQLRtGHYRqLxCf/LXv3b7NVFnygvNCgu+zldK8s1udw8RWSyyNK
+oJXfcqXvSR4wEq0nsyhnC03u5BzFQI6BtL+0REt1dHG8a9TQKW32oEN3GgQ8bxAD
+ZdCtjqA2qmA2MoAoccJ+GBWN544VhP/tpTkpAgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FFgMdBSaU8g5EQJB4ZGlPKcFaP/rMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWCHG5vZGU1LmNsdXN0ZXIyLmVsYXN0aWNzZWFyY2ig
+JQYDVQQDoB4MHG5vZGU1LmNsdXN0ZXIyLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEABpPG43dhFMhrzIgL2MYSqQjLk4WNJgdhC/+z
+Kz8qVGFi1iGTq2yuOf9y1dzab0+wMCmqygTJzZ0nxt3B2s48zj4d8JAMxicFMH84
+fvE2oML0TmIVDm3SNWSVl8eQsYUOEIG0WpZPg8LZVyciOI50acLl3qv/zDqLxmUF
+QAM6ZFxZBcLnPLWInX4p3zSemEDAc2BlIwMLAjSb54p4aAXhwQErGZ795HJiU1YZ
+iBGLGV2wHPqVGFooa2GdeX0OSlMPzZ8UnmdN0424Wr3onm1rVrw91n/kPVa5MXdD
+7P8eLr59C6Sz2PsA5SSwISt8LD27kGCsb7gI01clq+GgMXZP2A==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c2.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c2.key
new file mode 100644
index 000000000000..12c34e88055c
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c2.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAzUrzgb1OYExs0kO4Rh3Q9u1qELqricuQ8iGG19NDIMKnqwnZ
+8AyxxSsW+VpGTgxlx6xPuQnK8RMz6+YdHzY4GwBm7aAbOUCmxAL14hphZdfMLGLZ
+GxczLftKOPKKb9a5cFuXeZPp1xLfscAV75nXpJe+HsEnIS8niijeW8ytQ+X+Ow8M
+HKzqneN4hSVHwShkC0bRh2Eai8Qn/y1792+zVRZ8oLzQoLvs5XSvLNbncPEVkssj
+SqCV33Kl70keMBKtJ7MoZwtN7uQcxUCOgbS/tERLdXRxvGvU0Clt9qBDdxoEPG8Q
+A2XQrY6gNqpgNjKAKHHCfhgVjeeOFYT/7aU5KQIDAQABAoIBAFLuPLlD6oXvSaSV
+E7tLby9ZEBhK+GHVWKvfjFPidYMl70qiU6Zr0mJ2ejTn2W/NBU6olpoG0TMa0UYx
+8sC5i+eofWvv2h+CNPwIwCCDDKZyCvpBQjf14shn839Wx0LPqKCxd1r3PCcUakYp
+dRMSADFu1TAiUfbPUDc5ND48TCAZbqn+wxVrofyLG5tXdOiXf9eOwhECY5kMC8MN
+t1X9X2gdIJXNiFbIwE3p91vVrNmosqvX12KR3QG54DhHVyM7mWrmRgZEBjIcdshe
+1olIk+varEdRe3/uLTnplUuH3RusnB7bAoUaWqBc45IaGMdahdW3rP4rU5ymquVL
+hJfFGysCgYEA9nvW5Ez7KyRXGhtqgCRryujFDxLUaKWbWryId1pqqKCK6gpn6l4i
+zKNqbJqpQvaryBX19f8wwwMRm3P2g5uYhodhjr7Hyv8ljunz8u9WnHx2HvIHT0u8
+PFCNHAjG8ZVC9AiwjKoOwV17vlaIaIVUPUCTxUjNH33vHe/sLqUKxusCgYEA1Tf+
+kSQyEgN010yM/I2kqpaITMhGY/ULYjYODRnx9HIednukZbpIpPaNFBrzZnP66R7q
+crHhPlQux0kd+jZiTw9eTVWHib4eNSiCr7F0Y+7YwY1iMLGyxUrcCVF/j0eLL/F1
+DvzewJNHJIxN+bUetTzb1KopjzOMH2fG7u2V4zsCgYEAz3S8EbuZS15y0U5x/Riq
+gUYZVe6LSOjaMSytz4cSbotxYDsR63j/4S1Jzc7+15W2T9nsAviemgRjNWwM+Ahm
+ABTCT9rTXlAR1I+k+7vmMh+va6xZ6qvb6wgBPK9Ggt8WUz9dGYeok6eTF0jN1ush
+U8yek1YLD453S8M8DqZqw7ECgYEA0jJBZBSqx9dgeF+fHdsFWqbL9+X4ftI0Rr3D
+XGcXgESAg42gpCjhNL9alVd8XKQ511gDCM8Tnr6e3xfV1bWHdSlRbivDUqd6YC1v
+qITomRv909Ht2tNTyeviv8z4U+fT74+6NuxTLMYEfKNUu7cybrlwflbCYcoPeWZ1
+V1YhTcECgYEAlPOplx1qLzfKGCShDfl31YYjLX1+1Mfsvh7St229o2KAJcnX1AF+
+/E+EdjoI6KJKl9XUZo5kuEwsveOjpbjN+BHrZyUlOBQSJnC+qlQSN6ArtQn/65J3
+f9+I0w0VDZEmYghqIHPy/6mJEMLS5P/uwVhxUSz4thXqrsWZmgstKVc=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c3.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c3.crt
index 89c96600569b..96316e57eff2 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c3.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c3.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUJr1hlTc7CXDlsSooLb5/05DMugYwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0
-MTMyMjE0WjAQMQ4wDAYDVQQDEwVuNS5jMzCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAk4EUO9srNajozMBBRVv23wKGKGQQaiQ3w/8x1SSDTsIQciJNIsyPQW98
-f8lXsThi2IQtRjSCtNxZGWftDRu3t+4a0L3tPEvrbt3H+OK5McIbS2SjhsX8rIqJ
-EUY7UJtdDm1DO8Oibn0U1DOTTsdPloW3zk7GGrDo6bxHpitotyUCAwEAAaN/MH0w
-HQYDVR0OBBYEFOUchU2n+E9rdI9UjdT1y6xrSUopMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU1LmNsdXN0
-ZXIzLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-DLA+tUvoBH8QkdHqpcFWMxAU1pwBN4dFh48iIKeU6tlxBZJ3vuivUwvUMZ+DdgFA
-QRAkmTuBm3t0WnIukYO4huGT67C7nya90wSyghff+9ef9DnqVvd8rYg0S/axjaeG
-tYrDI+qK0GQWF5wddskzpFfxIaZzgeppWB+CBjgFpntDOFql6L4jbaU8rv8GtpW8
-zHCFAF+38sqI2iXD6njZPjDSN7aYcYmDutqyeAB8ulRJ/q3lAK95kawp2wMXX7Yx
-Lz9CrwenfQrVVVpQt6l19zhZQs/Hu1WeDvGGsicxwjq+s7fhBr5rjLYqpUSf+SiK
-YCNk5fKpE2CNh8b4yalySg==
+MIIDUDCCAjigAwIBAgIUJW7IQ8FRtq/JCv6zcHnGMm2CfYgwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE3WhcNNDIxMTIx
+MjEyMzE3WjAQMQ4wDAYDVQQDEwVuNS5jMzCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBALnoNNARck+Diqji7giUyUGlBwxxHwvxRk/vGn1WF3+hKb7chpf8
+IDXdZVDRn5WQKqV8N6yC9QnHTzzHiKxZYrws81Sr1CAtpgiD1WwFKP5NwpPnpPAv
+hDU0DHas0QkconuW7iVKZ4t3iB6lvcdjqHFuLprGV56HwyhiYp4kEBHvIvr54hJf
+MDjs+JJMTNSjIU5llip4J9lNq02iHFMBaXOiXW8YLG5VkBSLzruX5wT1FhzhcOz2
+TDNq467T6+eXYWUuJbxqEaKo2rFUaR1KB/X4VODA+vvGmQBNZnFiNJThjVu+jvGy
+anjSoHLS0LbbwLPOeLXeNmtmQx+Ni67WSykCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+Zj64nueP9XeqNYcpIPNZfK7IF1gwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTUuY2x1c3RlcjMuZWxhc3Rp
+Y3NlYXJjaIIcbm9kZTUuY2x1c3RlcjMuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQBz78FyYAjOGlMJVIKdMKqIA7+QGzdx8D3a1nwS
+DgqC3JZaADGmqSmIcxLZ6CNMMilSR5LQ4KXDB4HchGi0nHhp/XxuT0gkPxDK2XQZ
+SVCq6GPReH5x4zmKsmFdg8Y2JYy1/BewZB16XUZue5zDajR6PPjQFqvoMOsleWfU
+BiDU8VFQl4S9V26PszTEhqieaTdhdbwj8GAxIdswvDEVgebsnZqN0DoWZKW8ZvLc
+BmvVhM5HpCD42ustpolJN+ClpvKYEElEStyj0HQGwRhRDWmJZXP/FhLeoS3Nj1oI
+ItJ3V7PgqqW1VOyvfYeR+XOki5yzzzMckRNIu4gIzjTnZUkG
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c3.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c3.key
new file mode 100644
index 000000000000..1a61dbc3b722
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c3.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAueg00BFyT4OKqOLuCJTJQaUHDHEfC/FGT+8afVYXf6EpvtyG
+l/wgNd1lUNGflZAqpXw3rIL1CcdPPMeIrFlivCzzVKvUIC2mCIPVbAUo/k3Ck+ek
+8C+ENTQMdqzRCRyie5buJUpni3eIHqW9x2OocW4umsZXnofDKGJiniQQEe8i+vni
+El8wOOz4kkxM1KMhTmWWKngn2U2rTaIcUwFpc6JdbxgsblWQFIvOu5fnBPUWHOFw
+7PZMM2rjrtPr55dhZS4lvGoRoqjasVRpHUoH9fhU4MD6+8aZAE1mcWI0lOGNW76O
+8bJqeNKgctLQttvAs854td42a2ZDH42LrtZLKQIDAQABAoIBAApGIhkM8HIi1Q26
+piiqPv8WOtqoq8G8s+6PSmRP1tzyT/zLcigXF8PyTXrsbXMrXvf7R+u25wotzhwP
+85EsIBqXKgbCt5E5r0R8M1YUvkG46WkZcvzlG1uCBLL69P3Ea9mvAHqQxJfUDh08
+z/f4yCJ7Ui8H3r1qR6eahv8HWNisyWvNck04JTBEt4pZAW1GYXmNgd5w39AOY2t5
+cSfozD33A4j5x/N1NwZMxYtvOqpjDhqz76+ykE4AYpVm53lz5Gw7PfPYADcZ/DSg
+2VXwoFPdZZrQhv/N8KxCHl4NaEeuG21uxSNKD4AQmcEJAZA0OPGtznE+YDUNFY0Y
+bK9F+EkCgYEA+RGW/HlBkdzn6f2djryJvGeZbdfat81D1Qe8ImgwlR5Qc+jUvinj
+9K/nJrokZ6zSO2bH6riRgl/P2mE3eabF2MnGw80rbSLc0PrjnYQKalx2xCZyCPHQ
+dnCkOGOegOQzPte5F4VZXwvnIxvXprDZFa+0+ObUzqbwRIS13m334U0CgYEAvxSk
+PERA30CE6VOauFNSNIYtBYPpG96BNaUnFtA1AtXllGBd9yOHhRF24Hdjljm0G3w+
+RnCM44PZws//XQLvBFn5wLy4yhhTzBTSRNiADBy8Wgo6pjJfXvgGxC7WEVCMiVUc
+DmuXddc+IVNAfCE9AYPFDHmbNwN0JkrI2VhiI00CgYB6efbYMjxXINz+CFPJE9db
+SUp+ZE3OYma/w+z0sm1pdp2zUcQDuw+4jCNLvO8X+DjtMu6N8rLC/KB2T6X68VKK
+9zYBpMt9AE67iRQSqhKBGPTiIdGjql2LUHrAq+QOl6jHd0cyVMWM4oqHjjJuarpl
+WqDQYJo3AIpX9zaDqWniZQKBgQCsVsyuQo0UDpgR7cLdPuEk2GOg1tGXvj5hiPxR
+eiZl0U/IXIZDkbr4DMFgQK4EXzUPmSZ916NqzI/KQlk6pZu4HALDkDNGq1xg8+KW
+D4Kl6gEDYLh3D3WvLRrFNM0EPtcSQj4Uz3uC4Eh8PB8sFbp1NEi0MLYKj0XiB49R
+YILa/QKBgBRl8LBvTaBWXJLk7YXh17/VAvClzjORH+T86pTW/e9Hrp4cZAItWE3q
+CG4OfqUMpZkTp4VIVHwRaxZ5xfD8G1ZvnjnMbVSn+450JwoK8ZrE0SSp4dx61OZP
+EJtCUGutPN43xSp+ZHjXGl0fKxBMkDineci5B1oB1nBLUvqkCPNz
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c4.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c4.crt
index 4f6ac740cf26..3552e6422529 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c4.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c4.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUaagjAMKpb7/8mdQov867WsLAx3YwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0
-MTMyMjE0WjAQMQ4wDAYDVQQDEwVuNS5jNDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAiGKK1GCMeyJzrzzi4pNaCWqd2qiFPISAANTh+laKkKQDLbrLo6BNfv9E
-f2ZnpFO2JXWbDzNd2UDA5wKjlNPacFvoWbQM+pydH5KOmQ0tG5AkrWgnMSVyZ4UY
-7tUGAnkMe7ZwtWWETVTOzW478mqV3TAM9FjIPmUFuIyR40hF7EkCAwEAAaN/MH0w
-HQYDVR0OBBYEFI+J3qFQpd1bJy1q/oww35Dt/lB+MB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU1LmNsdXN0
-ZXI0LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-huoeSFpQjEl5VhTgPQe2Kh+O3zjNTIGUXV4XjSf4m1hR1X5InTjEzSVrSwrioAve
-wRRJZfQ33tN9RlII+80LNWozQHMs7MTTPMpOOAL3XcOTUpIaPPEAye5GLfs8hsM1
-gYluFjoKbswkvkgHR78XqHsifsNdABSaSn87JUlP8TV67GkYTzitXwmtA8ZhpqFi
-J3GdoM3t3OwhkJYdYEXjtfWvlzQovlj79zP9NCc32/K37JRLkynzUUpeic5dIHGN
-+Cs9gV23e/kzpHOVgMq8it5h6Yap6HI2XeiRUZfltB8Gm4nQxHFPkuA7M2X5nfXC
-jkQ5EpEs7EsqXazTAWauCA==
+MIIDUDCCAjigAwIBAgIUfpXPSMAftNz+iMtcsu6U9U5hdH4wDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE3WhcNNDIxMTIx
+MjEyMzE3WjAQMQ4wDAYDVQQDEwVuNS5jNDCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAKVK2WJI7BTYWhcQq0JZPP8+iimUCbmEYfAriKmXgYIySi+0YAyo
+S5clXq8eXTaN5fOB1P+emO5NOmyp71qpHjsnXy+qmB84UiGtWIcNzorABfz4NPaD
+cdz4Gpesv3gcKXyj/UUMQEFSfGXyUfA6HsGpJUHUVBrkFZKP6BtqImOlaRbf44LA
+puhk1p8u6tHBdE2Dq2WvoxBRlcx71IYCyxY7uZWAa1hw2p675pe+hhR+B1+Rbc49
+AjUqPFd5Qr/yDP+GXAeGEQ1w5/VlKWuZtAT+Bbr6nMBqQE2/gFVhCTuxSknJcbzA
+jZhS1m1eazP4OKpxGDpglCq7oyeLphl6JycCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+nd8Na7PynFgVH02pQoq5ZEzl2o8wHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRYIcbm9kZTUuY2x1c3RlcjQuZWxhc3RpY3NlYXJjaKAl
+BgNVBAOgHgwcbm9kZTUuY2x1c3RlcjQuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQAgTJNGl6gmpbZuYNtfUdmmUB0FdAGIquIgX5BF
+GaAUBIgaKitLocrixI/eyyAG89HEP1J0OWmoQoHRWUbzLoPw7/Em/D9JL9dDUQsb
+BmE+56NbBsD9Ax4DBwW34rg4k3XmzD9iG3boovG9/8XPEImfQK+cKdDq8EVxxQW6
+IXQFMKkn7b/zX4xEdSF4jgtm7HuXGlvM80EelSjGHpU8+IOEzIzpjCw3Iff3tqEB
+GdyDGo7f0rs9346htO6405F7HMTTiBs8hJQnnkr9lGSTaSsK8/f+OwcM3KMGtIUj
+kXQLuAJp2+JzZUpNCOsWtzLHs5E99viitXxN0EPXazuDIP5J
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c4.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c4.key
new file mode 100644
index 000000000000..d6ffd242f465
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c4.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEApUrZYkjsFNhaFxCrQlk8/z6KKZQJuYRh8CuIqZeBgjJKL7Rg
+DKhLlyVerx5dNo3l84HU/56Y7k06bKnvWqkeOydfL6qYHzhSIa1Yhw3OisAF/Pg0
+9oNx3Pgal6y/eBwpfKP9RQxAQVJ8ZfJR8DoewaklQdRUGuQVko/oG2oiY6VpFt/j
+gsCm6GTWny7q0cF0TYOrZa+jEFGVzHvUhgLLFju5lYBrWHDanrvml76GFH4HX5Ft
+zj0CNSo8V3lCv/IM/4ZcB4YRDXDn9WUpa5m0BP4FuvqcwGpATb+AVWEJO7FKSclx
+vMCNmFLWbV5rM/g4qnEYOmCUKrujJ4umGXonJwIDAQABAoIBAAznUnVvREApAaNy
+UYRM9iDmRMqT0XcyQ/kH/6kJovpt1Z0dbjAexUPn5Bk0psnhpaAZtG3ozNSG9+F6
+xKzlre6n7LDfwZ9npKsXXrqwU72WLwF9bkOzpY8UA/Nvm2+Tdz5RM647vhpmB5W9
+JzNjPiUX1h4/pqSGG2mvzUqyluZYvQuEcek/nG5fssdRh90QtwXjGgnC+eG8ljze
+G3xJbQ2uuRIGieP4qNooqPVHb8OpaDCWaAOWaOT29MCB+7yCemEF6SyOOeQUWSQm
+R27hMVUMpzlN1m7pcNw75NTbltOdVQqYxWQqJ+/bhYL1RzG2gpAln6MWyoJ9LyRy
+2PTx6sUCgYEA2mO1uH0vYVlMDr1ulrv6mFgibbd2NIfSa7zj1tfb56i1BLKlSv8U
+d7tkzT22Sa2R6+nfL3ptmSsqpRJjTpJki9+27qC5p6D4QCuTZarwYMYMDMxXBUBt
+eYPYohAcUjunauLyd5Nv/GNgMoxVQOkQDahZc1q3v2Kv8OGDp4llB6MCgYEAwcI1
+tq7tLnnMpCM32br2lQq/eTtjPT51pEvfIGUX4pjHMTVe4ROBjmea9e2feNWDYMDq
+GMyyOQJmcuC1CSTcS0Ai+uR7HKjHja+DE5zeuKJ6kb+tAO7eTogU3bdGP5m+5oCT
+pOAw4UyoyHc9+v/S5YgZmzbR7HXMpqQ8UDNz6q0CgYAZPSjFN4KAR6eolG/xb6wY
+aycfcvJwQ7onxV63d57seOzirBq6PhyJcdjGd3lXrn2MvCyU+hssdnapOSTM8dI4
+8BPeJCkQnGPUVrTkLQs2w+yklgEAlF+SiZBBuobrYgWJLs1C0IFCs+FsNhesNsMq
+MOiVkkhmD+uxdP6YICLS7QKBgQCUWBzHwSXWb2o+QLiB+qLPeWsfZliUiz2GjJBI
+PR4MXuwiwLnjmQuX+7qoj0WJPe9YJFsWoaHSBARr756rYY1ID9K4RDX9uzsNP1Mb
+sP7UXZF6BUZvcHLDrWxMVAV0rEF8OPTRRtQlqouO77yEb1kIYys3qsEOSSJmc1e6
+D6qwAQKBgDUWZzZrb7NjMPf60otdqL8O3f9Q/5+BIpKyRMQIMDiepHuIWDui75/h
+XlZaqrc5Z4hUfDingSgYIoom3/6h/bEIGzDScNRq78Dd/lEZS7WCH9hsjeAkCMIe
+UICm+R9PtXUOb9ywSkIykf+q1pqJ4jJ4rYTMwDALF2nem9jXJWAE
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c5.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c5.crt
index 8253f01573a2..98b3af5b0081 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c5.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c5.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVALC2GLx1iArQPLmEOTSnU9hhJiiFMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjUuYzUwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBALttwAoG7pVbxG29TtVi8YYvfyrw18qj3pIwvTH3uD0NsqlGqEdl8Sop
-mwobhEc279RoLf5rA54KEcUaoEjPJo0eCPfGIQxWOtSCHi/Dpp28o32wFzj0uY8v
-qgOlhY5akgRQyJXdgHBSVnaxYbHatPgO0P9seGLkpOPEhVK/mhGnAgMBAAGjfzB9
-MB0GA1UdDgQWBBQd5etOJglTebjjwgVm7DnZovWhBzAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNS5jbHVz
-dGVyNS5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AK3qV6LeIWj+CMsy4qjHmC+EbXAUUS0Pra6aFrXBxOZ5t7h9aFEN3zJjqTTTl7cw
-AbS1it93Tmr1wyuIiZ+dCtCr6M9wJE5vrKpe5rEp0tdeSs5pm0OkXPJVU/2xihW+
-NYRw+oV7qG86ldnLO067Tgy0DLPYP041gJYr4HbCCOjPLSi3B6Kn0PMrYcvPdqmi
-lHGqvzqaMXq9dWYD80PKkYVOUwwxYzZ8sjOWAE1JXCVXI1ImGh4yFZAArKsY9uEl
-7scQIEt0emutzHaMLwiQx2yTUjttHaogaa8S3PAHNZq4yHHCrD6a15Zpouuz4EdA
-s5Dd5xbtE6zBBjGsauInD+s=
+MIIDUDCCAjigAwIBAgIUNnRV9Ti0oaTYgZ7ndGhGbRB9jzgwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE4WhcNNDIxMTIx
+MjEyMzE4WjAQMQ4wDAYDVQQDEwVuNS5jNTCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBANwrg7auNvNIWi2EhJfgklukXf+S7lswqeZF2kCnXPMDuR6GHUnt
+Sh/TTJfmI0jCNKxU2mDFLHJNaZsa8odAehj0Z63f8A59Q3IzbcoDp7YJ5gnGFC5b
+g7cnd4ziw8wFghe/FezJ87TgcdYuK0R22nywcd8GgPyGL5+sgir4z+LabYTdo0fZ
+W0B3wH74Drnfu6eEkVNni97XEkRwlUE0i7+YjaO1pSqD+tmm7brfmduMIp7nCvPE
+k9nuG7yKQx3HUXT8gT1P7ogRNE4E8DeuU1E9pkL5SB/CnfWLbUTMyHDF69NG478D
+ERkvoP60ub5ocj8P3bguim6oOZIXwOyY8FcCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+kwWKwYb9wFOIcKGCrVxLo9nwsCkwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTUuY2x1c3RlcjUuZWxhc3Rp
+Y3NlYXJjaIIcbm9kZTUuY2x1c3RlcjUuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQB1xlhjMOhhrCniGx1uO09pPG3PBKZqNY9Ke55R
+On4SAt+ugpym2Yu1MRbs7lt9zSDAPaATvzZ4I4vDQQV+RoFb4wZfXs7tdkcsqc/H
+puxW10hOY6YU5+13K2W8i5UJG6Swl3S43ULD9aHAY0vtETqjEd+7wGB4Lf6JMo0T
+mAZkeohI4FK+ciBdygwVu4NGYM6UKyNpVqWRu1ZOFYIf8hYDCJCXg8xSrpj6a0WZ
+P6xpz/S2meQ5hmYWDnHjR8ITRpdOwYUyTQ9l+WWfOoufjNm1BJCaWpsQIkNkxTY8
+K543KU1Z5ZavsAGjWJsw81lsoCWJ/eDj2d+NjfIgI9h6mszA
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c5.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c5.key
new file mode 100644
index 000000000000..1f7b3192ba5e
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c5.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA3CuDtq4280haLYSEl+CSW6Rd/5LuWzCp5kXaQKdc8wO5HoYd
+Se1KH9NMl+YjSMI0rFTaYMUsck1pmxryh0B6GPRnrd/wDn1DcjNtygOntgnmCcYU
+LluDtyd3jOLDzAWCF78V7MnztOBx1i4rRHbafLBx3waA/IYvn6yCKvjP4tpthN2j
+R9lbQHfAfvgOud+7p4SRU2eL3tcSRHCVQTSLv5iNo7WlKoP62abtut+Z24winucK
+88ST2e4bvIpDHcdRdPyBPU/uiBE0TgTwN65TUT2mQvlIH8Kd9YttRMzIcMXr00bj
+vwMRGS+g/rS5vmhyPw/duC6Kbqg5khfA7JjwVwIDAQABAoIBABwzDpm1UqByYep2
+kK0LPHO0Z722SgUwBHLT+5JQXV1a2rkIIHQ5ikbtP2AtP9Ov3CM+cwH1/y1csOwG
+1Y1xOrI7i5P/nLWeYacc4zD+qtpsBiS2VBtj6UjqKyvGV6GoCvOTq/E6JJbEMJBo
+NW59cpqJ1uvj3114qlzfMYTjfWzmpFMCl5wTFbBtwrN5epfQgmomFxF07SRvRWbc
+zxGwYvf9Z77gmbhVhwETpuwT8PQ/+nUKXiMou13Zns0C286o7cT3nhf9vbECYH1x
+dlkNDWl7llLArsXCSh4UYhKFplL+7QOme02lnZCK2SGbrLRiyWbm2ZJSoT3D2Jf1
+mKZqQlECgYEA65KKUgsc0wnhMw5BBwz8ofPiMCYPMLIOhpd7IxLH9cF5b701Ibl1
+bcIPojBYezDFOmCDsaR2uWkUeUq8LnhHWKtDPDikQY+3Rs1Y1gUGteh4VZlIsUru
+H8ZJDh65BjFR0MxVEFrADpXYiyrP2Rd4TUu7BqnVdkucSKDQKLOiKfsCgYEA70MO
+X31uliKZJBAH6JbF4FhuEab4op7k6ekIijLVHOd7C7cJXsEq35nrWPftn1zbUbxR
+zp6ShMjl75XNkVY+Y1ngNrI1iUO9fQgVsVSGmhliclJ+8/VuNNsxC1ANBf6jQVCk
+owSXkwH0hphl/xZVCcF9cu0JvHlDAnRLDYRzAFUCgYEA0GFMIc7arIOsxXDybL48
+KzaE+npwLW8UElKlBh1y7B08wm7oNPDMw05Kbfm+CxrVHBKL6/PDYbdl+PT3CVQ0
+KGc2jmq8QZJTWToEPy0rBnahG3cAKnRRt6G6tdbPbH7sFHrrNBBSm5RJfLFeMnbk
+X9uBkR9iR8YSdNxrxdVAuUkCgYAHDpZE8MErQnf73VEuZgwOEu2kLaWVvlQIJixv
+TxKEGLvDW7G2DsKB99s+M4nZsnZxkblj+79clURa7bMzfzS133fChT0ZAFt9bZie
+M10I/Iiv0IkYDvwiKPjoWYMY0yrjiUph2IfT+THKKoKPpIv7jq5xjlZELdtxHhb8
+nhR6RQKBgFQervbCSk2aEQWG4QKzCXeObXY2MKdwahnUhvc2ps0px3fuFh2/txQF
+G1DbTpc/+vggdjRby4tLggvwAQIwr0HVEdVYuYi6AUOcbmulApmLSHlhwOpzAMKk
+/T0j3nRnOWnlTeVb6clv20peetQ4Whsft5bc7SIa6IAfjwm2Ezvf
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c6.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c6.crt
index 8e4cd6f50fd8..0da8153b780b 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c6.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c6.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUOLkd6HCk8li8bYemSJ2y1s/7htMwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0
-MTMyMjE0WjAQMQ4wDAYDVQQDEwVuNS5jNjCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAxUpbKRNQ/WxzJj7oYFBzeP3/JwtVTaTqlJexfKpEXAGlTXrUYRVyZP80
-4p+2Dc5sAFEFXwUcmYgVlLCj+mc2OGg9Uv6EyaT1Bz0r4oEbIB7ePBiefx41i2Rh
-d6zy3Jg164hGnrHzmOdqhz0hjKxUAgBFzEtfx0URtIpscMefimkCAwEAAaN/MH0w
-HQYDVR0OBBYEFEyrP19EhCIvYsJFVyZPZ/2Uis/gMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU1LmNsdXN0
-ZXI2LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-pZRghboAl/dWnrUXB5Wp7Bigbpom1KPFZQ2AYyEs0RF8xH6CoAMEMZsXDm4+sjFi
-uDfFoXVNORrwdy6nsu9/Lm6dC4Fl2got2B1ZRT+jjpGXbJmPeH6gjvylqo6xopJ8
-kqozuLyLaJPenuPe/jU7VZY7WTJ36LIQMq/hCf9RVfD6513fGM1DifQwhKK1o6Ms
-u1GSveA6wXUZCqfkbr3nBikhgLQP372dF611mrLQBL34xDqDtIfGeZ01wc/omKCq
-UfDYh0uwJXG90TlM77bjL6LrRP6rvTff9pnFEENqytn39slXXahGoDos4WnU2y8F
-hUcosMm84zADrczmErWI/A==
+MIIDUTCCAjmgAwIBAgIVANgEngU9m51QUEzETj3aIZGGuo1OMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxOFoXDTQyMTEy
+MTIxMjMxOFowEDEOMAwGA1UEAxMFbjUuYzYwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQCaWNR2ssNRqK1/+UFbIe4zSAKRPSNrOt4eQUvoVEXazXiztPPs
+7fY79OrhJW8Yz5cMMQXb2vr3VpF1+6Z7Os0ot1R8dGdyol5BfEHMIfb91YGjo0mO
+LxPXgYQJQ2J8kYeJego2pCjZvQA8RANHZRK5UwA3ilkGzM1+Y05ftxM2bnbIERR7
+poTWzcDpJG+S66osWBRLj86WYtRbakRZp39C2HVS1ChVaOl8JYLGZPvvfRX9k6GJ
+lgYgYdDE4B3ASoJcS5sPjM5d53E6T2w1spdep0Q0rPI25nKyOO1jQV9plvJ+0hov
+pfI422jh9GTdsdqYqPusvV9ABaLdau8M6laHAgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FCQNnF91ymRMvM8ZwKm8nRCFeIRAMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWCHG5vZGU1LmNsdXN0ZXI2LmVsYXN0aWNzZWFyY2ig
+JQYDVQQDoB4MHG5vZGU1LmNsdXN0ZXI2LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEAhNkKQpi40qq74fFZEFGUbSPUPiTdOfYVKdQw
+ZplXSp5Baim7EUT7P71UQg1g1U3eikeXVK6Wk7JLJ0CBS81kXv3q4KtI18YTs3Kt
+I+mfs/adAHWgKzgralAFtqgIR5NnFtr/A82d5PUfKfheiJEaiWHSzFD9w0myRY3P
+eMKcgpRWHTrYReZxIyTCasVopYjiNdSbQdJDUo+5/W0hrDexKWl6tDl3YTBgVfuP
+Ju+0EM6ftRWyKPDvBFGg42AfL9lXYJnt6RqAS7I0NTljIit+BTTUkJRnsqGpFc17
+PlYL/jPnMfpRyqnPZN0yk88unE/vvbJlc35dhUCoLVTQwkdjpw==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c6.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c6.key
new file mode 100644
index 000000000000..531a4fcba5a8
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c6.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAmljUdrLDUaitf/lBWyHuM0gCkT0jazreHkFL6FRF2s14s7Tz
+7O32O/Tq4SVvGM+XDDEF29r691aRdfumezrNKLdUfHRncqJeQXxBzCH2/dWBo6NJ
+ji8T14GECUNifJGHiXoKNqQo2b0APEQDR2USuVMAN4pZBszNfmNOX7cTNm52yBEU
+e6aE1s3A6SRvkuuqLFgUS4/OlmLUW2pEWad/Qth1UtQoVWjpfCWCxmT7730V/ZOh
+iZYGIGHQxOAdwEqCXEubD4zOXedxOk9sNbKXXqdENKzyNuZysjjtY0FfaZbyftIa
+L6XyONto4fRk3bHamKj7rL1fQAWi3WrvDOpWhwIDAQABAoIBAAgywkFoAtWUDZSC
+iKpV0jayUzV5ZZmAjiOrJ3KG5vXcrceqvWgg1MIKowmrP/buk9nZUPr2TsKhvB9y
+I721XXoHO1CLcvhU+oezJ25wgWjjMWauHUqVKa6zBH7IWo/Mht6nsBsVApVb+NFl
+VW8HqWfG8XgP/B4IEhJt0AjZmU21C1rLZgpM4+tfR9bEC6KF4HMrnyIn6x5v+bSw
+hSYSNK/54KgpBE1Ju2ZB7a5eeKjsEvlOeYLjBfzt7sbHpA+4I3YuisRlBSb+JMrx
+GqqkYXA3OJB6Sh4lc9lXbzoM62nYAQjixS7xkJ3F2NjcxpH0J+xUKNhL3dO6dSBc
+olPao5ECgYEAwXzYo4gb2lp5IG2fM9vZVUvVmGkXgxzwG1pDQIwPQ4xK7TByxpiz
+Gnua7l5CgUw5GV2qhXY70TeS5nUKXI38NiUludBcmhphGjnVgZW6RUR9wMvVvOmO
+Y66C+zhS8Ya1KpfQzrER6ziAjMHRxWmh3kgL4UkXgABCrfIIX4QUEhECgYEAzDaz
+RVGbCHfOK59hUNplGH4KlOvXsaZRR5WIfjN/DQ3qKLZR1r4qj7qmNgpCwNL0u9PZ
+wMXBiF6bMEln/okr653GJ15giqYc+b/xiPswoG/Fwk0OmRG1mmCd80SQ4j7ReM4R
+kBJsEd5n/pOIRxfrYzuBZNxf0s0p9Am65eJ6RxcCgYAnv1TlqzVFkuZlIf/dcGFi
+I1T6nUQmPBCbHdzI6rzL/Ir71MMPRxkr7I50P0rcMuNb8aDxD8vT+92imBKL1uMj
+4FeypzoOsjKWZ6EToMWGKz7GF1mOGFCW66eeogBv+2NGH6/WdgufTZki0UIdksic
+2CIQoqpAMk2AXEMaFBDncQKBgB5CcLfxyqYrkno2SKZBZj+F1a2RcMmbGmogBzSk
+DsxxZmQX9/J5BTd88coKJTALUkZOsvJ0myPI2PgDuIwn6UElhMoqmuWQjbSYM4MX
+rXHofgY3gtU9oYo5jM4SN+G+ay2aY1HU2Pyg1woK9GOMcM2ocJjwaqaCzheiJXaQ
+44uLAoGAJ6neGpLb+Gk/X+lR2/EB/1bcFKNHB9VoBt0NBEs9iLEVoyhipWsXy+VJ
+XqHOBTLANo2/PGmdscrt26PRDbfZwpSPsccgBBUa1LPbh657fyYhJCInahM106DI
+H90IoVoqOPQlGgyBSgBINtxvRPQDMFb4OjLE0dG9StaGE54wt5I=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c7.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c7.crt
index a344076342fb..ad15521455db 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c7.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c7.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUXDFTEo12oo6+EKR1kSq5mxjGqq0wDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE1WhcNNDUwOTA0
-MTMyMjE1WjAQMQ4wDAYDVQQDEwVuNS5jNzCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAg6EDuwYFA0XwT9/7mMHsHDejJrfDq8dZsYm094CQDcddBa3DRxHuHvlP
-HI+2yNhrc+LsIXa2xXutpmFfu1Um3BWrnvndOkK0FU5VdyNeFdH8eWjT4QVVr6Xa
-I586qIafTcLAgt2k01pvocYhAYd7fGyTYYsbW4wjxyXP26jspG8CAwEAAaN/MH0w
-HQYDVR0OBBYEFCRFQmSe0o8k8hohFXJ1FtB4A9QIMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU1LmNsdXN0
-ZXI3LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-CQJWjpCXPAEQcDcv2pYKzSbgwO80qZetTcWXSzx+dosP3LZZqkqv1EfPm8As5wm7
-2qeaEmOzrjsAajInZUzLwcfTBo7MqlodlDN72S6/kOkUe/Q/4M9WRl8elXf6D3Ex
-tj3X9dUf5bUj1cqeNInAluOsj3ZoYlFceo3ng3berAjgMxMjqnJi9O7KxLvsTYdR
-fX5a3A7KbSPjlnP0/n6FHXK+cxqysmv4m2TkL5Dsn3jK1gGJJz2qdUjiKu3jR0hR
-hDsyiSlVVgqGVcTWyTa92lAe0KsosXR9eTV6qKwXFKA0FKDlPvVtp4pelDVWdaeh
-QQsTs7Luvaoo7xg7W4GhvQ==
+MIIDUDCCAjigAwIBAgIUG8p1q2vEJG9geHFk5Kfxpb0U3UowDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE4WhcNNDIxMTIx
+MjEyMzE4WjAQMQ4wDAYDVQQDEwVuNS5jNzCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBALXBWOmOzymaMiMUfuArblb7MQVSpva0cXis/OsJPSJhoHkXJbYD
+XT5TKz+T0Aqfm8LtchU/E9SNiJ/MWJd0QfpVEHIRp5Vsm/9pmKMjOSYPnJrfvtbM
+Xl5aeXyS01gQqFTv5PErtNFOeriIW+ZbqO/RMdy0oqjBVTZkW1d2er8FCzLv7eZD
+KMrPQE9G+R0xZAMhg9Z5ObPprSi/q9ze5Qfsu5MVxZdcdzvZMVQ9fJZG/cdmRToc
+2ZOslG/0F8WslqEt/PJPPOANI/PSG5tp5nNwrfmwW3Cq6Ny7SwipVXHOuEzmoodZ
+kIGOTNJgRBNqkjKeR+Qrlpa9p6NdHFDFpnECAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+wAE2HlPlKFrjXRp7KzhXQxXrNPwwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTUuY2x1c3RlcjcuZWxhc3Rp
+Y3NlYXJjaIIcbm9kZTUuY2x1c3RlcjcuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQCMvM87RVOyuk+2eVsgqQModpJwzKc+E90FfY2p
+OGdj1o1iL17/uix7OnZhLcDrGwM+XjhEMWXinxpC1uyqN/ND6CyaJdaupTV+LT3H
+rX+ZOxMOR9YceGPy1IayrWaA9olG48y3pv/VvNWrB2zHdy2de1zeF/UjC1UnSgVs
+n/ttmzmaHOg1qwauZJkHwIiv/2LqHKDJHWSwvGZ8J3jeUuWVCNiYH3/Oda8AmUiv
+5MA+jWv2KBLGt5iEl3UlBFKLWg9V5RV8oM09sFy4FDO0El5OmFeyg1cMlh805Jad
+xGlgUbUyv7JIx8it+00QWh0aguNcLa59InX1GoIF4g6Tdy9j
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c7.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c7.key
new file mode 100644
index 000000000000..70700fd5d4cd
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c7.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAtcFY6Y7PKZoyIxR+4CtuVvsxBVKm9rRxeKz86wk9ImGgeRcl
+tgNdPlMrP5PQCp+bwu1yFT8T1I2In8xYl3RB+lUQchGnlWyb/2mYoyM5Jg+cmt++
+1sxeXlp5fJLTWBCoVO/k8Su00U56uIhb5luo79Ex3LSiqMFVNmRbV3Z6vwULMu/t
+5kMoys9AT0b5HTFkAyGD1nk5s+mtKL+r3N7lB+y7kxXFl1x3O9kxVD18lkb9x2ZF
+OhzZk6yUb/QXxayWoS388k884A0j89Ibm2nmc3Ct+bBbcKro3LtLCKlVcc64TOai
+h1mQgY5M0mBEE2qSMp5H5CuWlr2no10cUMWmcQIDAQABAoIBACtphpimpSGdNGt/
+mjxl80FwVwmb78hOWioGO5TSDJmZMvCpUDcUv6FC0Toqzo6zB8u71jtv6QckVSVB
+9EkcmiO2TKrKP7pnii0r5irVH3HRWcj+N0Fr6ABmYM0Z7sQCmopLEGZwV4yEz7gi
+zbyMWSIK1/uM46smHnnS56RQ5smX/jkd+Vd1pExpe5z5t4z8JGk/LgYt42nXvkos
+0TlmDQ0wl+bkh1MqImm6kU5NPwR3l+36wXqV3geL7WoJ8aVsJdG040udGZXVrzO0
+iXOXzg+r9hQHnLzFWf1VocA4NGkUQZml3J7rh5tbOjtWp+lAbVgAH1l5WL2J6g11
+EVfm7IcCgYEA3j7lMkgnnJgm5j8OjIRYoLdCxL3glpo0YVDKlHWg4kBg9hIr/x7I
+MpV+4sYE2gndPttoal4ISxe8g4FGiCD6ZDeZPaA1ZfoqxWgFoxcnldYj7dnQGbjy
+3dnGtEJm0mzNSTtTWXw+mHf2wJfDduS0lMxe4dOxITLao5KZD9kdU/cCgYEA0Vwm
+O6v+GQXThKqp7g2pUjWHYMiswmgdngp9cipXzVzhuCaOumHSsNW7KfLpuqJ7MG9Y
+ZGzmJD2c21UEOhz7/9TUC88jcbtuvqc2ay+UxSV/OTovzqS420Ebgq+Pmm0IIMIe
+hTSs2P/GJcTdQ/sM+/pPiNcF39VK0HZGMLVwbtcCgYBqv/0pHL1iGzc8Cve7COrl
+L4FzrNyOmj0krUDQ9Hs1kcTBDVNytF9m5KXQ2VqFPI233n2V8dq77wfOhq7RPj3n
+G0mQ7z4W4IXg0ik7ACk0cGpLmf66DDsj0rH97ZMj6bzlOwi7B9GtPJoaQgtSNWKD
+wnKkuCDxmFKqlQTONvQ+WwKBgQCEr1Ev2lZFB5NBbmQaEiBwkKpG5YfQAepbYiyU
+8U8KQUZwzkcAysSb/CkNcf09ooJOdRLhOr800rOZsizmo732Kq1k+pdtGiwauqDb
+l6FCvPv/iegIoA7kLkRANAHgDh/rmvt60qKFOL0c7MKKtg5JmB9WC2iWrngm2/6q
+GiOyqwKBgAmeqd5nc6uEn/0zlZjR19jlXN6WWvdBfeYFgh2rz9imL0/bdmS8ZWkQ
+yfn+Fj6uKpTLYT8kzVattqddktJCYfW5Lppq2QoNOPvVQWk+HYRsVCS7Y85eP4af
+vTra8g3qkrXUFEit/8gM6SHAL4XE8pqgsJ0F9cYc4KiWQKDfIS5Z
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c8.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c8.crt
index d07bc610f1e2..bfb74f744658 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c8.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c8.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAKEAVAjsgY4Ft4ATYFfKD8C11yUNMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNVoXDTQ1MDkw
-NDEzMjIxNVowEDEOMAwGA1UEAxMFbjUuYzgwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAIJQgqB0qJj7Db0GiPnvl4AsKtFa5DV52/DLfqri1FwhOcIVhtnLXFA2
-wsUT/g7aJ/b3tZszfTwG0mKSmNS9voTlpaLAb6Gn+vq6bBpgvgW3kvcP+r5lxMIG
-U0zs8yC3hqXwUOX/FbWZzd87eEiM6fDfhwg4JbP7nSk/JX8MUEqLAgMBAAGjfzB9
-MB0GA1UdDgQWBBR9W6ltaRBtK+Q3dWeXPFp5cwdSFjAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNS5jbHVz
-dGVyOC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AKopACutziODzeAqmRV1qNb816EqKoKdV7KtsiopOH4K68+VALvTJHZBHHk5xsM1
-LvWFvjDtd5zH2KcLz+Hqq9fGBn8OWgUu7CK+S/rbMLpM6eM4O2UpsCRnvXvWJrnE
-Xm2UdW+dVIM1do1/tebMRJA2KQzZExRlWIz+idn+lUFxSW7BZM3tLOQ7RgAuCTcA
-F7zFjCQROi9PnIyCNHr2TGGJ0GiJuJ4Lovj4WB9jA7e/bYPJzbY2zp4cAuIFBZZU
-rbUGeIJCvY4jEyV51KfFXXJVMmCGtrtPA3GNXeqhS9ph/FDftakMPTedx4Ah+hL6
-fhSR5Ce/PvqTW9Hx7LMz7qw=
+MIIDUDCCAjigAwIBAgIUeppoImu0Az/EBkGkblU6YMOzZlYwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE4WhcNNDIxMTIx
+MjEyMzE4WjAQMQ4wDAYDVQQDEwVuNS5jODCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAN33AnMUUC9RLgDwI40dc1G9cKBeVwBIes/rmLQRMvMoeH0+aPLt
+6KUapUSO7w2lBiNWKxMiqFaiVcvg7LF7wgePaUrSBeDN8TS8/gqYA3TENJCNVVx2
+n9/vxsXdzwQkTfmk/wJe729EOKxg5npkID8a9gSQgdEs8oyEQRCXaVmmReV1QeOy
+j6nKTz4k6Mf3oHbnH07cSLzSq1NF5vmyP/qNSuJxtQhof7F/ab8tZc76JIHuZ3Co
+J2FPPra9xh2LyFxJiO4FXi49kAypjVgQ9km9kCIc0RecZJWiXh9cO+Txuf5aHfpO
+v5MxFmr3Ox+Ltfy6iinQvbc0nnFetV4ZEJMCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+B35kkSeyeJdqmCS3e57y9o3ydewwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRYIcbm9kZTUuY2x1c3RlcjguZWxhc3RpY3NlYXJjaKAl
+BgNVBAOgHgwcbm9kZTUuY2x1c3RlcjguZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQBvfDxlJJ8kdfTpkBgEd5QcFJGlABmpfYSilyUQ
+vFKrrhVxUCVWARoR1aPUvVAm/JqZcognc051rS/nQh2+e312LiyByMrRFEnvyFR2
+FNrNAQhs6H2Rhff01UMn3CRLF9ON4JDuYbS3u6skRNWh7y+XdTyXhqfa3zcviJ9G
+wO9PXrC265CVMr9P2Irucl8m1KBbWBVM+8XtwTl4VR9p5l5zY+JuejFYIzjfkaKR
+4Im1EALvqt8B+vMTjKUuytB0C65vdKGEL68HnbI3hRaxlMVQlFf4Or6rFTl3VGN0
+ynellyamTboyHDzUGY+875PpxQ+NEaJiDgDq+oIrth9Dry9w
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c8.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c8.key
new file mode 100644
index 000000000000..57d3dfe919cf
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n5.c8.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA3fcCcxRQL1EuAPAjjR1zUb1woF5XAEh6z+uYtBEy8yh4fT5o
+8u3opRqlRI7vDaUGI1YrEyKoVqJVy+DssXvCB49pStIF4M3xNLz+CpgDdMQ0kI1V
+XHaf3+/Gxd3PBCRN+aT/Al7vb0Q4rGDmemQgPxr2BJCB0SzyjIRBEJdpWaZF5XVB
+47KPqcpPPiTox/egducfTtxIvNKrU0Xm+bI/+o1K4nG1CGh/sX9pvy1lzvokge5n
+cKgnYU8+tr3GHYvIXEmI7gVeLj2QDKmNWBD2Sb2QIhzRF5xklaJeH1w75PG5/lod
++k6/kzEWavc7H4u1/LqKKdC9tzSecV61XhkQkwIDAQABAoIBADPW3qGV2uxSH1Zk
+v0jT2fz7K3So09TjGJu6wF09rxKPHyTREJ275pu4z2RlpXsgUYCrPUAo1fv1yefl
+ziHdb4FogwTQbEMGlhDlGsfKnPQGZuhjuc3wfoJEc73+gbzL5dIqMMyS84SfAEVy
+FCH9hHJ4j4CdZ1Q9FSCwOqnvUtF2zl6U/2lv40a1i55lSgGHoxMDs44UGq/F8U0K
+muuwbiKkiRL4ICO7ZERCCxrWR6pRfdrW8BWJyzjsF0daG6RRJa2RMMTlyOS2iEG3
+7N5iyQ+PWKgjCAuy1L5w7TWHy+3tv3E/el0xsGcUavVMPPKQC4LojFb3cAVFvlWZ
+AZkshSECgYEA+iWrST6iWnxRdLWUmDLLrh28dxAGrZ7oNMwlWQd1fvJ+rrSlnhle
+wuWLYLO/vy7dHQtA1l9eBnbueT2O8i6GVnArC1Vhj57OEOe3mfJo0ZBpbNka4tE+
+P7XzWTAUyRnR1lZg8AnFNDQO6GwJrIQ5lz3u2r1IG6UlyhOiafJP6HsCgYEA4yiI
+zxdpCp44sHQCeeGDPlaC0whqWXRo0poP3PPSA9k27FXrFH6EVc6fyHg4ltwsAyST
+fjF4rfdVwz3RWW3jMPts07seV1U1Ahm8a2FhgRwRqs22bc+ivpfNmWptua46dbg/
+ojQsl0z9SFwivWy0//RZfw/g3DJGbnxSGfGwGMkCgYEAyilJXd+NzH0uQ5lAZySe
+MbB8kOjVzNlhJtJZ2LLLhXYrZbw/IR+10uM6mkOK0tLoBkIC+32FesoBfuB+N0Hc
+dd0fHIC7YRvhxA3Q90zO45bHbR2kueyLV2gQLGNMWR5R1NxoaXYr9z2fUr2Fr/QI
+qy91aziQHy9/8+mcV+1qISMCgYEA1ONsNAmQnC7kmhpfJ7K9uXKCcRz17pg2lQll
+zIGFVa4A10U+Z5qak4BuJwbkoh398YQZ5q6elwuyCKSqytE9lu5Lc7NuIBLu9uT/
+vPMq2kFgHV9llUx/DXhfa2RfKSGsEB95I58aG0JALcyLklOK3kY2ieprpnJHp8xD
+daUYkGECgYAPuE7wJck5U7DyK/IZ96OMUjja+kW0inv4LnrxawSV/+lvXrj1mRWA
+8+WniM04zGml0YXIM0tg9LFH4+qK/DXFOX9a8zJC5fKNuh2UevKvwnfdvR+qNByl
+ll97AoIYNLFuFnXJYqdbfJDr1QTmyEQKvjapxh55LWo/j12hQb6Bkg==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c1.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c1.crt
index 1c8ba3354b14..6b8811f62055 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c1.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c1.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUFpfeijJ00JLPz16Tv/3R4i0TQFgwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjEzWhcNNDUwOTA0
-MTMyMjEzWjAQMQ4wDAYDVQQDEwVuNi5jMTCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAkUdwEbxEusj+J+9qllbp/jYccER/dTYUWoDClkxfwgNUN10Xs5A08ERS
-s1DZ0IVfymRwQlyeciUY/Q0M3gkRXY3vU2M39ZPAEWgcMBezp7VopotG/4fPnGoh
-21vlB+qNu5CwcWgTcdWtsNmfrSkQ03okWfd123WcZhqYxHJxL/sCAwEAAaN/MH0w
-HQYDVR0OBBYEFMsR/lNScpkcl9QxX8JAMVP7mT0WMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU2LmNsdXN0
-ZXIxLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-hFQp+02dHbsLFZs0PeqWDb1Q4iB8+0esIl/GaH9YuEADxjDaOGT4eEvWIC3M8kh6
-6v5OEJO8z43y91EWJdqzu58mF51c/18u6paC1EdkO0V5xR9TT/AXgNFzoswmlpgJ
-G4CI/pKuTJlPlu2zdf74/Npufl3blx6UNdE705xmPmfS61Ws+wm5Q2qGKF0IWlUM
-98OVPftL8MLv29ODtD4RlVe9d9WLlNZKq+0WoIG0UOOHaBaAyC4ZeHbc1LKW8m+S
-hlyZjc1E3p9epCeZQi9rT1xPs3kRxsiNnhM4HVVtmiXmhMDoA7sO9A7OfYrROpLm
-0V84QIeyRw2ArSh3si31cQ==
+MIIDUTCCAjmgAwIBAgIVAMiSK7pGFp+kv+pz3h+rgyPZ4/qCMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxOFoXDTQyMTEy
+MTIxMjMxOFowEDEOMAwGA1UEAxMFbjYuYzEwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQC1vrEJFi3Nq75o9gsikD2jy2VYs+bGi2ozRa6RbuBHKzVV8g3Z
+Swhp8GZ4l+o1sOu9xwVxqdps1LtNaE71yTuqYKl1iI+bSfqdozRQY8UmIXjzEBwi
+8/3Omu8ei5TtGypfXoM0dQf0D+A21LkY/3OUdMPxJWBq+s9XiwjiMBqr4ZJC7SvL
+0RnYMRYzf/3J9FHQTv0EV58ycuKDazi9KmHWurV4RDHDZTzib16BaJxwoiXX2bP5
+tRl/tDNO5CBBjc1FUi0hBKQD5WC4QxBfARztm0loRbBZLa9ws1JF2BNhb7pzJNq3
+amhrK7bViD2NkQgFEW7mNtOaMOk4+qbzZqaJAgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FDZ3xKPK9V5104JEDAHiVwsT4izeMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWCHG5vZGU2LmNsdXN0ZXIxLmVsYXN0aWNzZWFyY2ig
+JQYDVQQDoB4MHG5vZGU2LmNsdXN0ZXIxLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEApr+mTLOkDbHrhxeb1ZsHWlFLsyb83edr7U4K
+G7ZXUioXEYhPqsk5iLjZmLH2rTyQ/7pN08zQminUbqjOOu38E8HzeAIypje289Cz
+5wNG5qJIktnmBjnVS5hG01aFh5EzzoAg4xUFG8utMY2lr/S2TisxubX/d1N+wtMP
+O3SaECk7ar/F1mr1l+HGYj9tQEJ+qfb3Din2Kie0MPKD8BL+TpMSS8Cti1vkxdmP
+ByYroCQmhohv0IAoCNlsnHm6SsdE5yseI5HmjRxgauFOHfzeebLYVx7yUkJARdzf
+XHJOeP9ljd6edIhkHuswgl3JrU2WhMhGuDMRC89j2tMIOZ0h2g==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c1.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c1.key
new file mode 100644
index 000000000000..45c1d482100c
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c1.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAtb6xCRYtzau+aPYLIpA9o8tlWLPmxotqM0WukW7gRys1VfIN
+2UsIafBmeJfqNbDrvccFcanabNS7TWhO9ck7qmCpdYiPm0n6naM0UGPFJiF48xAc
+IvP9zprvHouU7RsqX16DNHUH9A/gNtS5GP9zlHTD8SVgavrPV4sI4jAaq+GSQu0r
+y9EZ2DEWM3/9yfRR0E79BFefMnLig2s4vSph1rq1eEQxw2U84m9egWiccKIl19mz
++bUZf7QzTuQgQY3NRVItIQSkA+VguEMQXwEc7ZtJaEWwWS2vcLNSRdgTYW+6cyTa
+t2poayu21Yg9jZEIBRFu5jbTmjDpOPqm82amiQIDAQABAoIBAAMeOkqT/jVt/cJu
+fTaqt5oCrHKTBJGxtODcEFCHsNKbwcqfkGAC3uo1YQau4YpqBoNfIZ9nePs+w5di
+g59JJGxKBAJDlWYmROXVNdmuuoRK1PjDlR3+5mL36sQRBD00pvuToyfgXaJw7sl2
+CRZuFIkunyl5GRBAzROdC6oYxe5etPtC++C8xAiR/xA+STxmqKoW0p8olKDczgPx
+eHkGn7L8BeTLCL6pkWF+P8kRftrqvjunqXsuNOpkS0fvF0uJZtp53aoHRsOuGE6Z
+ts1EEMvgWTS1FyUSz608csvuLGaXCUNSGcujHAtw64ia5sAxym3cnXJalBiAvnru
+h0H11HECgYEA/o4cstxP9B4FbYE0NNVHw+RhVEZZEO93yIeGg5Klrc6vcZzRcJR5
+q5HyhAx3FJUAMe8wWLtu/EZgYPNSEAIx11ibrOc1aXBKwV6d20peBvOKw1Dfg8aQ
+aUpclZI2s+Jzx6ab/1FnwRm0/73DqlG3WFl/S0kfW8F5xR2LyKcXexkCgYEAtsbH
+2HuyCY95LsZsWSZVKp/M0y5Aocxfreu7LMHDPHH6uptX/ky0LJRkirIgQFt0J48X
+sb3KO7X+FqrVQ034sBUiKZ3q8HnZ3TTtQzOTOCRv9fS5VxVe6uU4wtHcqm3Hs07p
+4Urhg03xU+czde0l9cL0g03aQvoxUyPw1OimZPECgYAqZjlZgWC/5JA64R50/mFp
+u4yKoP0M2RY2jHxXRgHapWaLofcwrUpmzXR7kifzga36o0CGRxIBayd6fozYpcTv
+fY7QieJXoaCR/NSGRijLN52RUtMn+9bp8rzhHO2HoNxwEMxDBqglfcasIkuI/vtu
+99lADPnwKpXBw14ZGdvLYQKBgQCtNHiDnMydpLmo6t4YKWxBxUdbxJP4/+yXqzEn
++8JhEH4Swk1Drjp7mHAiwyQu+Lt9FWvVws3XgeJ0ZpVMf6IwcwZ5u629Dyu3JYXF
+ziUv6c+zm6LbdGQLYwdED432b6v0Ls8R8Vz9iiosYYdxtDbpy0Lc/zTbja6aZA/A
+I2wzcQKBgEkXDsmRmUswCZg0MSATbjdv/eJkcNOMkEmtQdZHFpE9eS+1fEgB3m9I
+AYQbqom6tS5sOfbURES2EoxLOrVExP0v5C1YeZQJJxAbysnVnRoR3fgOcpNaZI9n
+ksLkojWPmhi4dwVkUFwAs0Uy0615i7K29Nc5KPK0536sLSmVxRQq
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c2.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c2.crt
index 0380402e43f0..c186b2b30e51 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c2.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c2.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAPzUcy0n1Ww7M2AliNfADZ8oKBhpMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjYuYzIwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAMJ7+brk5AZGhtD9dMKVhkHCmsAhULpd94aF8nK50Vw/Jgf/SdE6yYeP
-rBxBC+Or+o5+N8Dwe70escEOH6o793gg8yBWyL1AagoJZDV/+A41cxropz0jVW5X
-4SJu+y9tEgVA7m6cdefow1SG1EiMLMtEHyw5wKNmUKrl544O+MQvAgMBAAGjfzB9
-MB0GA1UdDgQWBBQGIpG1JOyVo9JCyp1/h9yMhHE3lTAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNi5jbHVz
-dGVyMi5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-ADM6lIy99jbMevqPYHqxVhbCnfXP/3+amzXOn6OUGme6pXkfZod52CGjXeLSnEsY
-OTwMAlzpp+ghdNHoig+ZewyBsSimdiHU25TZNAuOIFHcAvrX0DPm3zVqnmwAa38a
-Yft8UdcGJgN4oFoY9vCgxiSmOgF/hBHABmj7/WO4tHiM2EZzBaOJcOeGaYe1e8t2
-z9LY3mbtdUJXZktB0e0MVFsHT1Q09+U+qnJzXWoZ3zVeJkcGt7E9RUyzHa54qFfa
-II1g9+BjWg47iuWm15W0B0IJSZqeTcsvTtxSPbX5Y3ym5pcWZ00IMTFvRlLaLPQB
-e2O1dfbp2RJd8ozmBcI97FE=
+MIIDUDCCAjigAwIBAgIUMo7Is0KW3xIjT15ACPBe2oDcco8wDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE4WhcNNDIxMTIx
+MjEyMzE4WjAQMQ4wDAYDVQQDEwVuNi5jMjCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBANHVNgKap/Tr0PvyGN10vaveCsjbdSsbsfFT74HFEGg7Qkx0QrVG
+VKHDFbzbFHtPDQyVFXK9M1xJrHmC2esDBbwdz4hI+sBBLDK3NyJzSnenMI6nh7+o
+UGv9OTBx9t8eZnmlnRDdJNnHhUubjHj4Rbb4bSQNJsR75eWJLfiuvt7SwLuu12aG
+AVxGYQ/1/CycQnKargRqxAmSGMfWdKHNnuOnbfv+icGF87/dETXumHE2as6n5scE
+sbUe2dzcmV1TAheKp5Xh3tXfxy2tj+q7Ea6axU2LLMoyqveHZtsBEm1TOpR/hDbN
+geOmlYB1VVx93WAtwZBX7ZGICytj9vtINNMCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+q+V4VDqbFBgI8marci1dbccWz0YwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRYIcbm9kZTYuY2x1c3RlcjIuZWxhc3RpY3NlYXJjaKAl
+BgNVBAOgHgwcbm9kZTYuY2x1c3RlcjIuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQCF42NFkQxSPLUJG/SHDlOiv0ZlmGfSU5N2t5k3
+ogqMaWy3A6GwOxHVnyt+bV3YU4W7MS6SzzMoAXwFlw84VzO8NXriAdVOvRNTddYd
+7XxH5uZq69/VOhjs+DjlF/I1+cEChXlA4qDbBQ87sQVBU4jtnnfJn+prbpaBtCgc
+UzwiwpN+lt8W/QMETzMxCpXE1/fGc+qE0et3+ENGRblbg/mbftoJ6SBmUwz00N5b
+IdxTEC6rjPep1WNlMPBkzJrxXAwzz06Vku6uWXuvjMgka2XFBBCVA0226Uc9ErEt
+C69e3OuXFZ+ZdsimRzBaWfNiQtVXWOgfVAjCOqYQYg9Pou57
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c2.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c2.key
new file mode 100644
index 000000000000..38ca2accfe88
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c2.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA0dU2Apqn9OvQ+/IY3XS9q94KyNt1Kxux8VPvgcUQaDtCTHRC
+tUZUocMVvNsUe08NDJUVcr0zXEmseYLZ6wMFvB3PiEj6wEEsMrc3InNKd6cwjqeH
+v6hQa/05MHH23x5meaWdEN0k2ceFS5uMePhFtvhtJA0mxHvl5Ykt+K6+3tLAu67X
+ZoYBXEZhD/X8LJxCcpquBGrECZIYx9Z0oc2e46dt+/6JwYXzv90RNe6YcTZqzqfm
+xwSxtR7Z3NyZXVMCF4qnleHe1d/HLa2P6rsRrprFTYssyjKq94dm2wESbVM6lH+E
+Ns2B46aVgHVVXH3dYC3BkFftkYgLK2P2+0g00wIDAQABAoIBAAy8FnWhbc5fNOUs
+Zj+jnm4hRbNt12QDOyNvMF9q+zDKhZ6xF6EN8s2IwJwgLopTAHLiLx7Eaxi0MRhf
+8hE8/Eqv3sMYrQ8zHDvfuiQIQF2XzpN50kVIVhJOmESEIY6dAEhIAtQCU0ulctgc
+JOjs3QLaoFUTgYqIVwWGYYZwyR+Xl0cbJ51TEtB4ysGfZgXODzxukckeE3Y2qhsJ
+NPcUkeZqcWAYD/YTfOJ5AYFGVJjSGbBbt6MhKJhg4nPc9k53JiYWvzY+3SQLVEm/
+S4SiGXVfSeNh8jk37inmvqI6HoqB4rDW7ietVSmNSaC/NgqbbkyJBenIBDMP9k2L
+2gApXIECgYEA5nRmiHlvp+PyzHMMTjE3CrrGafcF22hLzNLr7GNirKzEJliryaS7
++xjm9tv83xvtE9/bw3/As1lp1n03mwVjVE9C1aTEThMfjzYQdIKiO9FZxshSpITm
+xG+6Fwvx8wi8PEQKdzqhSpn1/XCr8qTr8BwhOvSnMreE3YCaGtstgDMCgYEA6Reg
+OATNXv9sLh67i3q8vWDEeUE+N3vcvSnLGsCzTQNuK7AtSZoJ1NBKIDx+sAcMrTk4
+mWm/9NgI+zKEgbyGZXNM3fW5882gJPTv8ZypDufacmeNBUM6GjvxLHA/5bOZsW+0
++2u4MqblQlPiU4CVKQPzNh7sF6YpfN2SAN3jWOECgYAzWLgHrFbuZ2GPoAqb2WdU
+Iq1Wqs1JR4GsD0q1jqdLperGSHduJCAmIDRFyFodM9Gs3fHPK0W6MC/etGORTLAG
+9uoDRAThCG29sOQVsY2IGmsQWU3gjiQEnIgv3l94MPgPzXEvg4awYoFmiJYnKAvR
+odLdmeku6ajW2XJ8jV0j9QKBgEu2yOxpYuyAu3sfsHvWGojJbgiVEMilC0bG1JqU
+Danue1Lln1BRIPBn2CwwelZjc9Kwy5nJp/i4U/KErlYWDvHVShqvPv/awzI9/dLV
+uHSHC4Yt7LjQ5YawAd8Vdyv98uaYConAq8yIOeAZr2LAWbNA/TePQ9FKwB83Ob3e
+VYMBAoGBAOZX/KESQzOzBmerbTWfuSAjPCzPNQQ0b1H18vqj2q3HZ1jIlAav9SvJ
+GOTk2b5rM3kgxzLEaOI4s4Vlel5micMcVF+oN13Xly4jCaM1Mu8jZ8nQznWdlIxn
+9a1u0mD6JWX1QOML7Pr8MX0v308cqLhXWULlZcJFb/dR4hNpDl1c
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c3.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c3.crt
index 8ea16b43baa9..c042ee500770 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c3.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c3.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAMdfglCcL1l4qotlEg6YabyONoRUMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjYuYzMwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBANBW+tmJaaum3CPc+7wj+qrDkbvuDue24I2Yodp3HSsChx94KmFDYhGY
-uRWwsDF3Cu8iPyUi1xk4fCQallLCzkwny9j6dljAHtAYkVfNP07WBdPJ79IWUn4q
-bldqLTam9ZiaQZ0jPtHa6PUmj8doc0QWM1ySfJcHqCP53GkqpH+vAgMBAAGjfzB9
-MB0GA1UdDgQWBBRiy14zafEFR9kFLpd7PeVFrLdKUjAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNi5jbHVz
-dGVyMy5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AJOOREHVw/nFyC65uY+iZIzMdpcmKKWdLwaOw0MAn0ru9B9m2qGT4kwCX3m3D3HJ
-53btTyvHa/URXjtoIPL48VOt9qB30MPAyjQjR6C1po5DwBZHVWMZ2QJXq5RBol3M
-wamYq6dgzyUxWYI03/51az1IuU9N4IS2E7/HN4YVWqE+CPNroh5kvrg85QQuiENv
-D8NfxYYshGJa+kTTuLnkyz2EEfVwWeI8+7x93/tXryZE1b1EvNMxkl91ZJbtg0bC
-Hb/v98RCddP50shEgUxdMQZD2ukkzCabnRGmjuLsvH9OtsrSqCoi/0umapW9wea4
-fhJ7Rt3VHnAkpbsvV3jOx0M=
+MIIDUDCCAjigAwIBAgIUddNCAkMn5XshcN6RxKcVQhoT1U8wDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE4WhcNNDIxMTIx
+MjEyMzE4WjAQMQ4wDAYDVQQDEwVuNi5jMzCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAOMYtGJ3IhRYSxbHAsVqc72B29mI4pLEdK4SptLpTfb2hVFcjozE
+5CXsznkXhRNH32hLliW+lo4XzGrhAlYilKvlt8J7h2ZYtSwVlgOr0b/FEUAMxwBV
+MG8NfQ9cQDGtFailzq5y0akTSRjJ9z8odW6XNHFLlzqz/MYx5ThWk4gg8EnimLzz
+89NeA0sf2pUEE/JVjYYuYNQbhhAYCVYHsuc7Ls9HWTnzUBmuAoXAfDtLlJDbMWWZ
+y0sC5MdMsO+1IknD2k0FS1S0aWlMniLfM8PpwWyruvrOKsbjmL6bGIO6kr1NMt9L
+YRNw1Bg538XR9kdVIs6iIVpIPdQ3hqKEZlsCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+rQkS5bztbA8O7x98RE9lGj105ogwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTYuY2x1c3RlcjMuZWxhc3Rp
+Y3NlYXJjaIIcbm9kZTYuY2x1c3RlcjMuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQBI1Wc2Pa7Dhh4Bp6JqyjULaPnbJmEsyre53o45
+hzQcePWr/l0a19GawnDIKyj0W80OH2IHEAuRTTirUKy0vK6NPkuCjNFkqYBIdpRQ
+AQmG4pYGVZJVSvI3l8KBPJ0+n5cskcYovDpP4hCEDUf8JOpPMm4fuiqApf33scZ6
+TOVl38WUk8PMj8QvjMkoGZepN8UHPrcSDj8ANDbW9Itg42Ngl18bBdSwh2Co67gA
+exMMCDqJCHkai22Iw89zrNLCy0zJDwpe79xmDC5wwjB3F9uv6tjG3kN/go/xfUls
+CUtzq5TUvrQFC1JwW5Kr4vTaceVsDC7zU3qjjnlb1XQObn4u
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c3.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c3.key
new file mode 100644
index 000000000000..202f4ccec11e
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c3.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA4xi0YnciFFhLFscCxWpzvYHb2YjiksR0rhKm0ulN9vaFUVyO
+jMTkJezOeReFE0ffaEuWJb6WjhfMauECViKUq+W3wnuHZli1LBWWA6vRv8URQAzH
+AFUwbw19D1xAMa0VqKXOrnLRqRNJGMn3Pyh1bpc0cUuXOrP8xjHlOFaTiCDwSeKY
+vPPz014DSx/alQQT8lWNhi5g1BuGEBgJVgey5zsuz0dZOfNQGa4ChcB8O0uUkNsx
+ZZnLSwLkx0yw77UiScPaTQVLVLRpaUyeIt8zw+nBbKu6+s4qxuOYvpsYg7qSvU0y
+30thE3DUGDnfxdH2R1UizqIhWkg91DeGooRmWwIDAQABAoIBAFvvLV9LQUWhNMzE
+xg9rd+14+6qF6stlsqKhtF1fy4s142S7po3AS2g7BJ8q8VmL2BpdtywSitp9Mej2
+rD54g11PguO9pGO/d6CgIHnd/ey3d7NJOOqekhvzBvTH8MjrfAApF5Gxs8TRpN7f
+LDOqSlf56GYRJujB4ovNuTDkg5cwkzZgGIVutbNW6tuQ4PXACI3Nj9mLy21L01zG
+e94fiziFdZUpPy9nF69m0pPTxIPjuW8TbC2Hvx5Z2e09cPJWcpaCwnTPgleYC+yQ
+5hJ+qIa818cqbs8atyaiHSVngptVBTBPVkFosvozj0V0Vym4wjV4RDukN+EUNGMd
+F11FXXkCgYEA7SJ6s/MIW0+4mOtxSmT4p2tMAQd+U3eyZody04BCRWXfCu7qgZdz
+h8dg6KxN/ktjMj+L1hMVuex4T9j3P+omSmbei11TvNZtrHkbr8nCOHPI30K7+G+v
+9omk/0lzw4+LR9yoo3TnM5rQZzD/UaithV0OcWZIe0fcAAQrmG0GKH0CgYEA9SnJ
+Q3/GQxwTlkDOrhMRh2Ri5KDtFTvXr4zfhe+ZDFuO0O++ta7Y6QlqrPZ6Xze+L3p+
+YkoXzH4/n/zrbNYNPMmI2eCwiEc78gMRA+3nnzzRKtc2f2Zeg0WFWURSqnJA8f8e
+hO3yi9SinHalmxddATSWvqWlgT0Ps8KU3aNSWbcCgYEAs9eNZYIw2Ud5wYf2MKyp
+JBes2a78MIM76qRZAmxoVkdvQ9R0R3o8JUKhjXR35h74iThGbkGlHY9oo23x/igv
+iAfCScGq6zFYlhB3XfdO9A9jzWbINh9GPVygpPfo8uyffg2sW+MiITINUeWIDT82
+8kjj1j7zQ4P5Jo4DHiO5IuUCgYBUQEn0MmL2CR4KErd0RwVNHAiIPW18UJZss+LU
+6mQURRqtXtaRJUdRvnS0f/fJ5SQsuNHaAVBwbCp9VrOpMV5XN4MaLQc9rg1BvCAK
+06U3qfTP3inSWmG8hxWuebZW8PC0uVHJqOQjfa1sS7HZvIuvsFzlo5Ne+Jb+1a3H
+BSt1vQKBgFJxydI5NAP+d65nMQI5lCm+l5kCy3fmElmES4yz27wEqHexluqgpjM5
+0uIIokkdznHwbXP6kONwP9wA2ikbHOe8JCEGhtKqcKHkMM1msnBS/SDsBUjWR6uv
+/bbgJOrd4sJeAhGCXTf6X1I4MW2+pqJf59Eva8PDBsEwJsZbWA03
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c4.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c4.crt
index 45168d7a670d..21d7df35e672 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c4.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c4.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAIwepFjviZuQeudI1+ErlWyWbb21MA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjYuYzQwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAJDY20sl1QeuOSxilJdb2r8PygM/o6EPlIJk/NHc6Lh1VG2bxqsS091j
-Cp1UY5uHdynMKXxVhgbIG+rKHgP67rh+3dTWzvThWDMZ1ljMMpdf4NNH3caM7Wdg
-Pzj6FpJbuEevBnNp9ENRjKFv8DxUDiJPFsXcCZ/sGuK0HUZSzla7AgMBAAGjfzB9
-MB0GA1UdDgQWBBTPFm83nd3zU/X8KaQpSIF8w8njkjAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNi5jbHVz
-dGVyNC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AIsgxyXgff4XN3ZuhybqDrxKHvMOOOa7npzqODO97/oZ78+Xs2XMbDSp+ruYZylt
-AMsos5FzLydhYESKE2nrrMEUAY+V+mryNGX5lY9Z0TvdjyA24uVVBMSE+R96Cm8G
-UUP+ekbou3e2aVYKOSyaHL3mwqbmo/CB7VkzmOxzcCqVMB4Oi+fD5EQ/qqi3Bi5C
-FNsYdelnWQ2WUX7pfC8P+F011WiM+HYo5I9hhbuqf5/ALG3CRERKZiXcMLTgFsys
-ckjB1QHLVK7yFpiSdUqqAZb4ZPMtP/zr9YjRwyMRH7vDyyQSF6pQm65vAmT7+h00
-FZUilhJHYxVDYGCJ/AQXVyY=
+MIIDUDCCAjigAwIBAgIUXVOLQNWQeU65Ho4uzZHeY8+jGXswDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE4WhcNNDIxMTIx
+MjEyMzE4WjAQMQ4wDAYDVQQDEwVuNi5jNDCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBANMHcSUqk5S1+RwGryfgSkX6yHvKfLuImonuwSlCN5vgHHKbo/M3
+ErSOmmyQIiCTnF54i293DaYJ45UdjfnSQtXkoCOIi4nBBLg/hhUqChJ7dT9sBczM
+aB7/zI+XdVridPSzOKoruaN6S8cVBOpXsr7MGRWvDQsrxfOG7kYDveTQ3dXcU4Xd
+C37/+elin3jNvoEfOuTepxyEF9InpWmvdrh0+xwI/yxCkK11daEGPzcBuZkGL6sF
+/uGDkiHKYxaTm4utGLHCdbgHCnUx504ty/ds0DXJQNcg5vyFlruwP/hLrPqPbq9P
+08+LUygUuRWcRtrOZEx5gWX2cRx2wHl5sCcCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+T+7pAVYShdWkgXnJEPzcM9Ec+REwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTYuY2x1c3RlcjQuZWxhc3Rp
+Y3NlYXJjaIIcbm9kZTYuY2x1c3RlcjQuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQAkwdv5IzAS9C0730Q0GQm9bMv4rmH9eQMyB/uR
++e7my1xkCND+IAriB8BYmNATFdykP1ciNZTi73xGJYbcugNiNGDJ2ipOPVj6nXpf
+3UfDtsS5LwSjowB9LyHqvH/kJX50zGKzIdH55MW6SBGs1yvYgJ0LhCsohzHc828A
+no7K4QWLIO81XnVsTRngq6Z7yMvS8BKQHYox+HHTOORIRZL6J9bAhnXG6UGJEWOs
+f8Vr/bSWf7RFk2JI8tkfPJz8ofTYqdEh6oOZKqjzfJ9OQkruGJ2DTn1m+7kETc0z
+7JeG5sTTLTYzMEfIg03nGuh4HNDQQ4VwwnOFoujCoqOhqkyO
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c4.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c4.key
new file mode 100644
index 000000000000..95c40f19276f
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c4.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA0wdxJSqTlLX5HAavJ+BKRfrIe8p8u4iaie7BKUI3m+Accpuj
+8zcStI6abJAiIJOcXniLb3cNpgnjlR2N+dJC1eSgI4iLicEEuD+GFSoKEnt1P2wF
+zMxoHv/Mj5d1WuJ09LM4qiu5o3pLxxUE6leyvswZFa8NCyvF84buRgO95NDd1dxT
+hd0Lfv/56WKfeM2+gR865N6nHIQX0ielaa92uHT7HAj/LEKQrXV1oQY/NwG5mQYv
+qwX+4YOSIcpjFpObi60YscJ1uAcKdTHnTi3L92zQNclA1yDm/IWWu7A/+Eus+o9u
+r0/Tz4tTKBS5FZxG2s5kTHmBZfZxHHbAeXmwJwIDAQABAoIBACe9V4fFuEwBZrtp
+4NM0EBiUjOuO6qVQKLX/4Z047Irsjb7hY2L6RIn+7sKx89NJqOE7dxqTA114sERU
+7/cTKuXnq2Z1oL1cvVxLDh9vLr1jRWWfnIoAFj18M1P5FC78xUSNeYbGPCu2bI29
+fhL5V95JRhIIyH6JjDn7d2R08O4lNYkjtFkXvwWN2rd0QChGi3veWwOPzvQKkzdk
+qSgAE2HYYRB5fGmMJgVeWumGswJ20DuOLNJ6RzGW/YAStU2JkdyP6qFszT6ThQbc
+DkaMCUG8woKhSDzLdfN4yXV0Ij3ehoKeoGwyHKfkp4HV0vkPkLWkg1oSC4CEIqJp
+hOu8NAUCgYEA4zrGKQOHCxBTRK+eNGKqgUD4ON7pN+megrfgO8aQEGt2zT8SQtTw
+qI3USpCp3Q6e/nToBCqzbp+ZHwDF4jGgrG1+Xe3rSKTkIhcsWYICSWsVAJLG0z6E
+HCZju/QcKGIKP5lv11mz2pWSCXIyQoyi/+MuG7GIb+Rpqn+R2bG/Cw0CgYEA7b+O
+5qxb/nnrgHWZ5zZQ9j0YmrQ5e0Y9NLsyC9rKx+ifUrylJ99KCIi/VUdGlO7ruWbB
+uIenMQTFnl3SSYavWFLigVyqiJhyP2MCOMpehWezOxaAJNlI/ES6pi5BKFDtRnuz
+5yLxl+u8K7IPn+N+2KbOgnIHOG3XWAfk0m0PCwMCgYAR5AbGI+j2DgsAVXN3XG1g
+T+U7VRG3vUUltaSKEFA3er6X2mdtq6ZnHwOf+1Pke8o47qt16IbIJvi/YCg0bwPf
+j59rddgYmCL+LCTpxzq23R+CNAxDhaCRNtSTb9znsRBtgX80zXNDupMM+1jJtJkA
+3Lo8o/XCGZHYQsMRCNWp4QKBgQCCXHnfs/yRUYq3GkS0Sc0k56U5Fr85/+6Jnmc3
+ptVvvfa2aCfq7/1pJoo7GWTzPvpT46svInqjXyl4x5sJo+c7d98o7YW4dlaWuRQ7
+zYL8hTr6mEDVc6sYYBs0CcLCd3VTI3smrrtoAONh1jXJBvXDl9s+HYYmA9yDMXjf
+hNstvwKBgQCNva7FlgRpnCQlo6QWuoyDuZP+ANmLPIpLNM7KX5eEAwMHlXM8rd1K
+UPSoMPeYJQkQDfGpBz1t/J2EYmxXWiaT+j/9jPEeiA3zkdV+58dTI7ciDm+QavNi
+DnCLak+O3+0q746CUIXsB5F5b5leliHRa12KkJgyluE/6yCrssbgoQ==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c5.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c5.crt
index b4a07a31d651..ad8d8d1ffa9c 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c5.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c5.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUAOAVgCzC25AbALSA7HijoxKx2BgwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0
-MTMyMjE0WjAQMQ4wDAYDVQQDEwVuNi5jNTCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAgsoBkvyRXFYOH/3/8AB4blT80XSJAFKu7L/zH4IDbLNXqah7xXxaii7s
-AfeSyNOmDw3JZR6O8LZZA/52q+vf63VeEC5R2lh9xFWDfNLiYVyDYoau+dVy1el3
-tev8vTcUgSlpCFSzD7Eqt3353dXFAzEXI1YAeEHj+tEipY2N+HkCAwEAAaN/MH0w
-HQYDVR0OBBYEFH7CQtBFpWcATKjoiCf5b/jy0mlZMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU2LmNsdXN0
-ZXI1LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-DMqOMi8l1GzyZK67fA+Q0H05+dBGByALXCagAbso4BayA1C2QH8/cTShEHYSyEUC
-WSm7icucMdlyuxDZ7J+Hz84p+el4IejcsCUKNPB/pkWsSt6xNhOxeeuaoX9DYkt1
-KQL2nhHRP/FgrtH/nippMOaz80GXnaIXowcW3qbL3RCa28pY58N8NkrX6AtQfi7h
-UofAwoC/S/N5qVuaImZiqXT62HpL/reKbVaZ/l0uiHdHrlnIAcgtgGM69c9hEOaz
-/W37Ql43n4F3NqtYErCFZKtjuDIdptt8YUGhwhG++ZGMhQZBGKsc8g1OEG4u+oYR
-OVGs/kC5nfp0Nhfiuz2q1w==
+MIIDUTCCAjmgAwIBAgIVANJiQf0TsVbGEGFepXJB+JECC6PIMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxOFoXDTQyMTEy
+MTIxMjMxOFowEDEOMAwGA1UEAxMFbjYuYzUwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQCgaL8ljNM/40ZyW6VppL87aeZkDimhzBNO3xT6ZCJ1ogawyKw1
+j/UPoLtF5U0JImFn+bmookEOnmXPDmUGsUiViEvObNgX+Lw02HryfbJejhAycH+d
+XYVm8QzS5AAPgUFn3Em2xgkvn2LHxcWCE1W0KP/q1+4C8BaIFjw8d+D+1RYmrSxs
+ptXYe+eo6ZJ4oS8sEvGw7uBq2Z8qRQcaRtLmx1GiMkN1AU8TFXfmXO9DZmFXrl+H
+VtbSYWYYQ40KLF8bjXfvpO59kjTcSgYf67Y1ZBgIv4yIflJ1a/qSbdRdNXEMqEEN
+4uLT8CPuc8ZNmQgOo9SmfpCRlg3S+EpxyjPzAgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FJW5RGrISKuTHImvLdge7aUjxAOPMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWCHG5vZGU2LmNsdXN0ZXI1LmVsYXN0aWNzZWFyY2ig
+JQYDVQQDoB4MHG5vZGU2LmNsdXN0ZXI1LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEAdlCxTJpm6SJNerOZMcsezkKeSTWXJzvmROlq
+LQoxXEd/FSdlVb1+JOaZxrKHRhEC6N6pJ5yXXumCZ71qlUXuN/FTDIlrsArVswfw
+up0aWvpya8JVP04ZVTi+czhDWqbqcdxwUpJA6a7ZDa27/k26k0dqUa8EX0Q8iCi9
+NT9igjqpYIJ8O/y+2RwCGOFVqg59YhkkN72n5sJMLYc9YqqBz9gynrur9ftMyW7H
+ocFKUGPikSkA/HHOVXPij+EZx1NQWmhPQqWQs0qOCls/5j53Mla+ofkntL/wPFzh
+Zd/rsOF/tqAIXD4ig6ciQtz3R1xqFCFmcZ+vkEgtOHsmu1na4g==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c5.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c5.key
new file mode 100644
index 000000000000..3976640a233e
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c5.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAoGi/JYzTP+NGclulaaS/O2nmZA4pocwTTt8U+mQidaIGsMis
+NY/1D6C7ReVNCSJhZ/m5qKJBDp5lzw5lBrFIlYhLzmzYF/i8NNh68n2yXo4QMnB/
+nV2FZvEM0uQAD4FBZ9xJtsYJL59ix8XFghNVtCj/6tfuAvAWiBY8PHfg/tUWJq0s
+bKbV2HvnqOmSeKEvLBLxsO7gatmfKkUHGkbS5sdRojJDdQFPExV35lzvQ2ZhV65f
+h1bW0mFmGEONCixfG41376TufZI03EoGH+u2NWQYCL+MiH5SdWv6km3UXTVxDKhB
+DeLi0/Aj7nPGTZkIDqPUpn6QkZYN0vhKccoz8wIDAQABAoIBAEr8zKbUL4lFSRB6
+0VgSoSc4OXD6wfGWQkxzsw3es9FD/la0aPTdKFW28PUa3alwfhKqRmWT/E/GMpfa
+BdZcGC93OOD2Nwr1x534uXkU08aqKN47gxzgtFVMSI9p8LFO3nGcJIWiDHdLzlTP
+mqZVPZJsPD7f1K9g51eU9XHbJL5bxvpa1I6dunBaGeWKAAPI0NBbiyvHPCeHavd8
+hrKyToG/Kj2qnTqtOWAc6lBAISVAIY8aS3D1fZ05hQM8WrrqgoJ4U0sqwSDIMdZE
+WR7ck7aNh2Bgby46S/wLQ14mQvfI11GL3YmdXqREf2YwaZFHD/uz0oY9SH56Ytwf
+lZJr0nkCgYEA2dJzeSKWvBLB2QPqbfyXgVBPTMQ0LK+sHjINT3Pw0uSmnL0PTgw0
+5TxpkWGHWZvM4wHuatBVOzcfNgK8Oth7dXTDFXEGZsBiUFgZYDrfOY0+4WSq4LWH
+Ff+X8SPPlK9aqGmhHbVOFOibCCilXMZu/0N3j1swPJr4SvHuihGQGBcCgYEAvIY2
+SJSjEoKqhYCB25S9QmRDbT8hw7FjYOilfutsLMdhTmcjlhDmHaNhZ2pvLfrQU5nT
+nwXcj7F+FLy8UqjnH04TxtQIcFzw5o9I/nDHYPbY//Gmdd9v6RLkaTZiz+sfKupx
+vffWWrYFRUMr7NxDjRYcQCEzwAP+vLMkyzJd0IUCgYBDHifasZ/Ev/rfReeckHKW
+tApbWzHAezkL7NxRGSahtTbpoQ/wOq6p3B2jYGv2Xp2ZbWTroNpCbs+9MsaRU1dn
+b90U5EjWrH0eyKa5Eirqbu1rQjutFgJT8vf8o844wHes4S92GgG/S5VJxEx/890L
+5twWXkDUy+OqJDYecR73bQKBgHpzkn7CTMmRJcFhHvfJPMUK6uDiyMObTLl9Y0F1
+GOZc+s86HtsyTsWhgXrynxR2fSpQGCyKWiDbYNGHZasrGjQzOpVC09hoBfE70GW0
+S0yoCXr1MvINF1uEcegE81mplq0Baq/BcvGomPPMFZPb3eSeH+tnpX1rGuLmWVfC
+dz0hAoGAZKpoJ2OC7KvHJRerXhPylPlFH/Gou9bMPkdX1zqhose9VApFP6kfMD8p
+1Ieu+OPz9L0bVxS2TeXYj4VADIwAETrJARw1Hu2a8pjFwXueqS4QS3yg4/DeN0sn
+05zFmmewXv6VwrCNRJUjxahbH61r0QzQqyY9BHKymlSbPfRP/c4=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c6.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c6.crt
index 0b4eb6773362..7212bb7bf638 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c6.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c6.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAKtpr++icq5uRPMlFd/HaLlFjtAWMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjYuYzYwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBALAFA8o7mX4GYsexgC1JaQzogoJzVK0XGeeWUMKIU3kjhfjezxwY8Pqk
-1bCohm+WTdhRUxtO4Ku2AMz7FaOYnhf1gAALT87M8iEQD703Fd83jcmrAuWG0zib
-kzos0xxzOrg+yMU7Tj3YGBqDKuOiXR9vNR5vpscU81bg4ZTT561DAgMBAAGjfzB9
-MB0GA1UdDgQWBBSQZ4R1AxlxN/+tAONyY27r6AKU7zAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNi5jbHVz
-dGVyNi5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AIcqrPNc1Yr5f9aeW24lw5pmyH7sUlfCW8O5xyL4Kg1Nr8bWKFQBoWjeCL/5kMD3
-/yrQrYZRCMMNd2Befr+otqe1ykuq1l2fpn5BNLTDlOui07MLuvHDtpV3qUOjPL9p
-ocemCoHz8e3LITaP2/s1eYf37p9zJpyI5Lw2UGHtvbWZ8uOS2yy50Vlf28WHAETK
-/BoE50TjQ0e6/eMxammm9TkC3C7z7m7NFJ/421uyy9Aagc2LibEihD7+H31mDt4r
-X3dDcaCDM8K5n2WiHlJ3ioB8cNdnZjrAQUDJg//iYYLUzN9tH0i0HmPj4Jah5JYS
-bLzwggw9Hfc7gftcBi5UAFs=
+MIIDUTCCAjmgAwIBAgIVAKf9iYYay+CmjihS6ubgY0qjssKyMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxOFoXDTQyMTEy
+MTIxMjMxOFowEDEOMAwGA1UEAxMFbjYuYzYwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQCljTBXLxWO1EyJXNuUXJb4Fv5JUaq6XqlOsIaIj/A0Dx3Z8jtJ
++yMQ41cAZ1bzOBrkpV389WYKLpq+PJnePNyU0liDM3QxaEUWtxKEnVtTZrVqfc2t
+5Nsrb/v8YKIs1cAkivWC30i+37eP4P1O3G4i5Umm4bgjnK3gpo8NIgnSuuGNmxZw
+wnmaHdiWDuxokpakQypE4Dj5X9U5P7Tks0bpxyBZ8lC2XuHiyXxhHBKEZMppCtv1
+S3YWQaSWi2hV2LTeg+E2LmIGuZEXuNUPZ9Kpx86BBGjy/ToDTNJOA/utqVPpL9P0
+ZXYMfb6rqkxcQrvYAjzIvA2BjwKiekC1Fv8XAgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FAWrmjKn6jqSFXt+uYidVlAbkQIWMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWCHG5vZGU2LmNsdXN0ZXI2LmVsYXN0aWNzZWFyY2ig
+JQYDVQQDoB4MHG5vZGU2LmNsdXN0ZXI2LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEAC6nrmnVC7VYAbdPOfdi0bBTPtVWt5AaVy+7X
+AZDiKbOQX8txOHcP7iiLHdV9rLD47mYS5022nstvIX9Nyzoonm9wY0fCP6PA3ZtX
+koIKoFSJUSrM9c9I94i48vo0tZBn5MiJvdZAiviJn1vf23gzsI2MAmcTENWE0jp0
+Ii6Shmy23V/yd8MxNozdysQiBSSW/LGIybQl1JNxmhZhODVJ7JxSGMueI2+xJ9ct
+A84FcdGLMYkWyVNrOMtd+EsAsdKmUjY4EQ63Vu8u9fecVDfN4/cuCfCJKOMi0qiM
+cATs6V51nSdBvpXQWjKoxhQfcxq6I77/r67h9EKbv/J2xrxmPQ==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c6.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c6.key
new file mode 100644
index 000000000000..3441aac51709
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c6.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEApY0wVy8VjtRMiVzblFyW+Bb+SVGqul6pTrCGiI/wNA8d2fI7
+SfsjEONXAGdW8zga5KVd/PVmCi6avjyZ3jzclNJYgzN0MWhFFrcShJ1bU2a1an3N
+reTbK2/7/GCiLNXAJIr1gt9Ivt+3j+D9TtxuIuVJpuG4I5yt4KaPDSIJ0rrhjZsW
+cMJ5mh3Ylg7saJKWpEMqROA4+V/VOT+05LNG6ccgWfJQtl7h4sl8YRwShGTKaQrb
+9Ut2FkGklotoVdi03oPhNi5iBrmRF7jVD2fSqcfOgQRo8v06A0zSTgP7ralT6S/T
+9GV2DH2+q6pMXEK72AI8yLwNgY8ConpAtRb/FwIDAQABAoIBAASY0g6c5eO/oDLX
+dQvYMlaU1N5jIFmCejMPX2Ne+jpUo4b2XSnlvLu7hFWKvEH1FquvFKTXvGVsYkTS
+cl29O00xAt78+8bJCjoVchpkM2TMElEkFlgz/z39vKjtlszZHQSNJnOwdyWnTEKR
+6pa/ZwBP25nnSwsTtUq3dUg8RlRanpxbEWpKUBpYeNgOd2ZPSDoOBVhP+02TdOU0
+LLITFtlHL1/Ve1P6qj061mqg5P6zf1yct53EwwGpDq1ivw0nsc/n+ptVpAZAELgl
+ZZyzZ32COWT15tfVcqj9ngp94dPRxya8BzGGODQSpxkKxMwp6QTBpE4gQaEv8RJK
+GoZQ0VkCgYEA5M0idLTHWJhKNoRIyVUpicgy8+DRI92E1Uf12G351yzwnj9mlk8m
+6Q7NmbvXgHLI/d/x04rk648kqy3Lxlqy72rKb9SvafxA5Av9Db+7yuBUPcE9M5HQ
+9Fr2qH0rEi3MWdzsllq1YoWQ2sC6aB4/PosuxW0cuHwy3r+8Hr8Km0UCgYEAuTs9
+uKRbk5e0hGzcBKMcuqFIXrCDDXZJcpWAoD9/YG/T6ZTXMQL5ibWzLZZ7O8eLW/ry
+5++RABKRtwhiDPbLUm+smPNlrm1TNFhiilQ6QP9euyaR3Ta0WQJNIKgXL6ddKer0
+KdQCfmYVfkC2EZPA+xw6fOB7KCO4uVzEfTpsqKsCgYAkOlrvUcSSwaJWmYGQ53wp
+RdrThNB4tlWXE8rYlylNKd97nxxWRngpxN9HLVgFDafHkxMOZNq0dbjfpeOzJ59X
+i3g8Vtfjlz6LHV7XM3m/uN9928AeD0OfvVt6+D6+pSPLQtDvSbOYwv9Zs9SQMeHf
+vFgOiKVYjMndPUzDLYq3ZQKBgDpplyYyFnrtvUyBc8S2l2m3ULYjncBSDAlpYf2m
+6IlF2qLvgAkBsiWeOhnnOYd5ftYLxiwkHSrTsEXzC9Cgxy+A7gHxx6AA89svACw1
+MYWrwnjsSdVj2e2zzxS8TUuG0NdWM5c5N2wazy5YQ9I5MQM/PKaCt/PJiAhUW9ZN
+5VQvAoGABE1JVEb8WGJ0+6SDKCiTLYfokIkCdC7Zco+qHmmf2eQW/JLXuGHWGm2d
+LBFSpOTEaih4rFjuoaFUVNexgX1TWVtAyuEpDuIqPjN6ZYGIM7k0D6JUpW3mjVpS
+l53w0ayMJ8NWcVDkhWK/tTVT8YMx7l61KgBkOG2vtkzPIzZzuIk=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c7.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c7.crt
index 1cd5275e2681..fd18aebd605e 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c7.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c7.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAJkbTDOQxMn77czLVIYYso4B2oguMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNVoXDTQ1MDkw
-NDEzMjIxNVowEDEOMAwGA1UEAxMFbjYuYzcwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAJy8A2khQkFtzq1bi9E8sSKSrJ/v/sWAVASrv8qwuiyRh0F0sxp6KgtJ
-4ogpiI5WbhbIoqWqjoDwGgGvcvf24m+/sr4JDdKb/ZyzkKGOLhLXugQwZABOCPf5
-4DJIAs/JqoxQ/XhbMPhqLTz6x7qPLj3Tdwybsbj85nZyedkx3gyBAgMBAAGjfzB9
-MB0GA1UdDgQWBBRq79jvLNS4zDUaRG5JNpPgOqXlWTAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNi5jbHVz
-dGVyNy5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AAU1INEj5h7Z4j7GFgwGkmaIOJhLrpFcurqAnrxovGRm0MuSvFSsdORSvRW31Ck1
-f4U/ee5emLhkv5rieUIPoWmJH0B3f5YCNb/TTXJXhD3XeLhz0vsd+UtduT+EGa4M
-V9t2l10rTAvYa1RY2FOsof1UtNblkf6iKynBYO9haz2EoYCE/RGsJMFIComKoXhu
-s6VVjehIlT5kDiTOBu/Dnz8Ol5k/Cx+KAvZJXaPEpLsXrzgVtvH2OT5j6IG7B+nF
-LHzvHuKm6EkgisXqP2taQx8SuWL1twjQbxl2Vjl6UEI2QtY08GE56a5JCqqFxN2k
-hTzjW7Iyyv82mMfiuLb297k=
+MIIDUTCCAjmgAwIBAgIVAO9ZEHuT1ILucRh9xuUHD1io6S5+MA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxOFoXDTQyMTEy
+MTIxMjMxOFowEDEOMAwGA1UEAxMFbjYuYzcwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQCwY2yEute8sYbBquatA01i6B9hwLCZBtvsC8qXVhC2KUCHa1e3
+uOpyCsFacsSjlpUfkoPVITYM/rZY7RFFD5lmWvLErby8t9NFGyKAtWlTU+l0gXEN
+l8nuOgN0XVycfJGhR4uiSo1Ar5EjZydNEsToDRY6Dhy5XB1ScR2cJ0oPPNIq5fOO
+Q50QQpnSCbM26rwg4s5IfmH11gqmR0Ph+6fvPuAGHIxuZSv4wwkcmDAVOjIjeVGr
+ddIJipER5vgnSvEEs0vCIhOoJFrIQChQxYo/KQYn6c0CuzyOKZGBJ8q414N1COso
+wUjWrCg8SmEcg+7O8UJw5nnJL0mHit2C6ENHAgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FAR5b2xJjYmwIX8k+UJXQLLrHv7GMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWCHG5vZGU2LmNsdXN0ZXI3LmVsYXN0aWNzZWFyY2ig
+JQYDVQQDoB4MHG5vZGU2LmNsdXN0ZXI3LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEAftJgZ5C1imUUCITbF8k02/T1AOi0juJ9IkEM
+CiWHSf/Lgzqm879lxfbjFUMMmSTrFdEGPVmSB3ODUCJ6rq9+RieJbfifkJagOQmu
+n/1NrqP+7awLznJ4sNMxjYbnN7Fl8xuIfIAXg0jhUyiEeOZR08wTCmnYFyhEfaid
+f3S8HjHquHMoyTmnQnEO4fzksHcZrt/4lyPc8ZJyWj000V+zz3nxhR2lSSUGvKf3
+tFp6c75kQ3aNZAvNlUjxxDhdOQ6v+mSniucDLBUAX6hyLzqgrsU5/th4SnBsNQen
+98NfLWQ57Ngy5Z3KnAc7NmyU/VJIZGtOcCSZJLqurDwRH+xOrg==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c7.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c7.key
new file mode 100644
index 000000000000..e64cfc4d74e8
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c7.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAsGNshLrXvLGGwarmrQNNYugfYcCwmQbb7AvKl1YQtilAh2tX
+t7jqcgrBWnLEo5aVH5KD1SE2DP62WO0RRQ+ZZlryxK28vLfTRRsigLVpU1PpdIFx
+DZfJ7joDdF1cnHyRoUeLokqNQK+RI2cnTRLE6A0WOg4cuVwdUnEdnCdKDzzSKuXz
+jkOdEEKZ0gmzNuq8IOLOSH5h9dYKpkdD4fun7z7gBhyMbmUr+MMJHJgwFToyI3lR
+q3XSCYqREeb4J0rxBLNLwiITqCRayEAoUMWKPykGJ+nNArs8jimRgSfKuNeDdQjr
+KMFI1qwoPEphHIPuzvFCcOZ5yS9Jh4rdguhDRwIDAQABAoIBAArFD0+tXtXJGxrB
+FhOTQ4rZ9GySCNZXGl93utdxrFSLAUMjdTglBL+Q04xzvHjmjVVfq0f8nqG9faHB
+W9AVKz8aJ3+FwiPdr3slNH6aVQN++J6h9qanUW+HwEU/HBP78SJhTW/L0IMWFS0N
+s9MbZcVrYyOJUAyrSKkLegzaQ0XZzVD1dCdW8sgKX7pKKNxI8SGVDJHKR+uvYR+X
+5wVH9W+1BvueJ/tqwVRvadysO6MwffSJljudKdbDbscczRn4uuWoYuEghJmDdZvr
+oZbqmDWFVDkFTjFDbc3iC/u7NTpSc9/CoMlZHbFDxg2lvMYx3U3AcFNxJdqMJ+h7
+0BMFJlUCgYEAxbZELCjQDYsGM8IhNCTrhxJl4A0hMS9HNJWUgk4e8+ljOdHLowW1
+tZBwXXJAgaOiw0wLWa1LeJkylnWsuusJFY5WjHxne7mOaY7+D8JuoenihvB7uWNy
+4WxqR9DrE0A87kezudk1QXcXli564HMPwry94btJi/Z+lYDwvhYRD1MCgYEA5GPS
+K2dtmmWnYaMn+6KYZzZVYjsNwDZF2hDCtfx21ulDV5m9uoKEOx8eakbB/VzzdAWN
+tywbWZp8ieGzTNP8tSXcwsIa1LUJF1xt+5qUQKUE37vj+IYh3r4VBjU7JuInF+4w
+NiR3I1aoaiuCQa6QIThAO2MV67MFgZqQfZvI4b0CgYAjno2zovqwpe5NHtlaiphA
+MmlWLydetBTQ/W3O+AINkWuF/C0CxsGQ31/PBglnEbL+GlIlKtpg9MCvaK3g0v1E
+++rNzo/aM3jbvZ45+p5x3zAnfWXc5XTJ9f1DyPjA9B7GEvbnMaGtlKYUMXWDwFcD
+j3tqEpLSe2scxikZJeKW7wKBgQCb6v2/PfPYVQ+pG16fbohG5hPyZMhlcWfMkGat
+IPbsjwC3XDDnlFS7m+oCud2Tn5Jgoa323/in+kyNQPVMPOinrnqAx0zT6+eR+Drp
+asVNQJsMNtRShRe4XeIyg8KnL/tqXeSyAcEH8hA1sxBQ0OB6nVFUebesWmIm8aM0
+7VZ9rQKBgBxwhvO+TzSeJs/9T2HZ6L+/Rn85WrTIjbcqZecfdDuvzd5NKAVX+l5S
+qA+DVcqIz/64LI5c+pIY3CNHo+XlYwF5l6YLdpDAC0A5qXWer9W9kV17lh3O6yjK
+9Ki13Rk0lhzF1u3B8MpvE6j8qiLuu6IONb7LIEfj+hS3R5DE7PXu
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c8.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c8.crt
index 5bd70c6b909f..ac26166a7082 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c8.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c8.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAMmpIJnFnhv3d1sLi5WuoNu42eN9MA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNVoXDTQ1MDkw
-NDEzMjIxNVowEDEOMAwGA1UEAxMFbjYuYzgwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAIMcNI3+aYOuGJx1YVq3VntERfFqmnAsssk9FiJMCzbBG2yD+hPlrlGc
-CUpGfTBzGEpvdd7ftSoy+2SJv2tercjCkg1enCS/hhnUp96NmUPm6yq6XOV6ICte
-f0hUVcuNnXwJitsPFpVJbNmECNE7EI3au1JOKXBBlfRet7IlXax5AgMBAAGjfzB9
-MB0GA1UdDgQWBBRzmA0meyNb9xdcsfCBrjVMIEyM7TAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNi5jbHVz
-dGVyOC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AGgndV+7vrrWgSHSMdpGymWF78PexgQBDcyUv+qM5OawYbreSUfOT9lcLef0TFef
-oX4wfin9jP6GmazZpeVIjAnkHN1OjVE/m+4VD+eXnS0pb6u66S7KoEJflliXh0Eq
-pY1axZraGEzSaL22ac13jimgrzwHuDP0MDl71S0e7x93/k2X82MrK5jt6mNw5Fz9
-PQHAeSBJwyFTRHDMW0L0RviIoUg/rskOvAMu1EzydVsktms0B2BJPLW/KGWp6u3S
-NvRctRml1QO9ZxfgxPJZoWM8NsQplu82xqvA1wOhZ+4fLD1lk3ylEwe5xIYVd5WZ
-oN3FzFHLoKVW5so8AyZQG9s=
+MIIDUTCCAjmgAwIBAgIVAKXQ5jJ0cJrnz6O+vO0F5rBp4sCbMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxOFoXDTQyMTEy
+MTIxMjMxOFowEDEOMAwGA1UEAxMFbjYuYzgwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQDAEH3VKJtGbw5xSAkYHr+e4o0/4k7nDDwJIWZ6/j0LI7icgaSu
+5kkhJmxQNks4nNpXSSUQ/kw5Zyzg4NMgkOHTnwx/U2erW5k9x0yPavMvMMPgc7/G
+NKmbhnIkeD4EJUsdBykeq6GGP25PbbjbCl/Hjl7TOABu0ramDaZftJoOtn3mFWIz
+hOXRMhCE+1vory4ctsEbYs/MvXAy9DBt398dlqkG7oEsjD/4c/MrU+aDiilaqGM5
+nwevWwJwjUKmJ6PJN0k6Eb1n0WDh6JkY7A0p3bIFxhpn0him/+NDHaPKxfKkZd+w
+/3Xx8EKoew5rAAV15dQT2qVty0N4mHkQZorRAgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FBdgvcuOzAHU9MhbtwIG3Dm73IVWMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWCHG5vZGU2LmNsdXN0ZXI4LmVsYXN0aWNzZWFyY2ig
+JQYDVQQDoB4MHG5vZGU2LmNsdXN0ZXI4LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEAcfYTZl+5BKZz7GyUuBKUl8Hix2irM8glCicA
+6rk3mJB/XCTLj+fS0VG/qjxut8VpgKDrO/4Y9ErY0R6flAC79ZuSzajilQ9JlKad
+CXGM6hrx9uwONsq2ZreMhl9UiwO4JmOJcegbUmBbLS80mjrHevQsELFy1tr6SdOD
+Wcca4XcBymfdobugCVROq27j14nnC12I/o5yzlcK4AWaEkPDtBftruVuCTvVxH6o
+7F7vKK6BJoW8XoBgvUpoVWlei0LGdLd+xEplPVLdEH6NVlfbFMDD0sYRG++Tj3Gi
+qvl8/QtcFQRTF3v+1C0VEdoLhY/cvWOw82+XjY/owQ1UzxXi1A==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c8.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c8.key
new file mode 100644
index 000000000000..3909fa3a75d9
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n6.c8.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAwBB91SibRm8OcUgJGB6/nuKNP+JO5ww8CSFmev49CyO4nIGk
+ruZJISZsUDZLOJzaV0klEP5MOWcs4ODTIJDh058Mf1Nnq1uZPcdMj2rzLzDD4HO/
+xjSpm4ZyJHg+BCVLHQcpHquhhj9uT2242wpfx45e0zgAbtK2pg2mX7SaDrZ95hVi
+M4Tl0TIQhPtb6K8uHLbBG2LPzL1wMvQwbd/fHZapBu6BLIw/+HPzK1Pmg4opWqhj
+OZ8Hr1sCcI1CpiejyTdJOhG9Z9Fg4eiZGOwNKd2yBcYaZ9IYpv/jQx2jysXypGXf
+sP918fBCqHsOawAFdeXUE9qlbctDeJh5EGaK0QIDAQABAoIBAEQmdefpeTNkW70R
+gPyYiIjhvZoR/MpaGoFo4xq4VMO7oJmUHPHcj81d+2f44/cKHSWEtEzT1ws1U4TV
+8cRn9GGpWX9ignIY1gbITZ/EJNc1WAOwPrefXyFq8tyC+ApzizQZItuZrf5bSpKP
+jdzXUQYXvAj5oZmxVhMU29iw1ktqkgq0NZESBEuoB/UB9R7hKo4u8sAHTM5P2+Yo
+HVxlCsFwGgRUZBJRp2NuTy+HMc95Lv99y/YJ1H89YI1/1Nebpkbm6veG8CX+tUsz
+iWC5wXRfyxPu1q+Lue1v6Owl6+BECSWOfxljna0AfMyhypeMk/rJnq7KXnEUaG4y
+X5oi+MsCgYEAwor8Bc/Y+s1mLXbPgWj2TyQS/fkolXU4yV6NKq0vTTe2pgHHaEY9
+WpuAlQMNmJsbXJhWOQYiq6SbOtXSoclAIvJcukGc3/my3X1EhoNG6joDUb1bJCBS
+KdtM8N6b3PV/eTzgmGoxOyEwYJ0wjm2tWSWjTaT/okBR5/Dyt5tJvecCgYEA/L0R
+SeF6J6/bi6KttbW7mKkshejVXPwZy0585uPRviROd6E02ocp55o3ivx4/sxShkZB
+jEBQKNcSD3hVd2Tjbm1BxAhe2e4eNnzmeGDeRlVur05H5IuxkMIcvys5rOqwIs3V
+SvoLrHTFTSmFCpWXouRA0EkNJKOPR/X9MwtYqocCgYAYxk5WByQgvGuYupquZkmH
+xpZMlaBfeDAlJM9hcfCmKsQt2+gjFV7W3RrA30Sp/Ia9ggDo/ikIJsxF79kFFXX/
+fVHlvllWBJyMrFk/sBQl3AF0NWMX9K3B+tWPjynWN8VmESShBFUErLNWem4mlT0Q
+L0TerHaJmRtVKFAgpbv8XQKBgQC06aG2gPAv+HHvQkOWDsB6Rp7YPBJ+vPa50TUX
+EXzR3qtMdtielKIqZvnbQ15ZBWkNNL0259J5aXpEGJ8VF+K06QBqkpNS7GA0ILPY
+934kWoMgeah8sIIqXMi3Dvn6DQSH5U8InHQ+vld9JHtXWD1zHHBIqAIAeBPYLv1P
+zI9rgQKBgBYKCAjxg6OAx0Sx2K6mIlSWmohsWvSZHSjSVLZIjZhJ1kEhxyx9jz3S
+4uMj7IbgQTU6tudeIUnafunTqWunQos8yY9tpoxbcJ9Xo7C9Da04nobpNZqO0STE
+byCuxgPN442v5wQxezGYATP+kKTkfGNnPyUaoK3IQhfKTGFczNJC
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c1.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c1.crt
index 7d49edd4147e..ea4d18fc84a0 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c1.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c1.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUdTJuIxtwEbOb8dF6m54R36J4KrgwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjEzWhcNNDUwOTA0
-MTMyMjEzWjAQMQ4wDAYDVQQDEwVuNy5jMTCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAtut2Om5X3xl8drxF/Jf/qq+xPYYGs+soYtJM5qAwkC0NGQwPCIDw0pL8
-hpuJXT4WrtGmADZ0ERvPULGTMcBVzmDgt5KANUT8kt3NqzdDS9PXj3VAYKKuDZ6S
-dqqW7dEKhRmZ4NPERtHA12dYvBgHmMoiWOI+jvas2PvmJ0/dId8CAwEAAaN/MH0w
-HQYDVR0OBBYEFMcRlGJupnRk/6cjwiEb8f08SMxPMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU3LmNsdXN0
-ZXIxLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-JhvxXe5MYkolRzbhyFiCZkh4STPxEobkgB2wPqvxd53/YOzfOdE7dmhyspdqj34r
-7j/JqZvWy3A2kjsoIefWV1WGNEjfNsApJAPa69+iCzIpLb5m3hHckDGtrk+JkQN9
-9UE+2GvFsJ1Y+mb8TGZOt7l9vzvZeH6waTV3cUifKFTO3dbEgERA3oGFjPqF+utJ
-HEKtOkId9jHnzkEOs1YD3vDndIfuVv1Esav8vXeXzDtx73AREfoVg1VEgF0TnVFZ
-lPC9kt5AIrjLYOx9Vl0EabpbmGgB0fV9cR167zOYPlYLp4s5mL8jamkbo8arx+2m
-/Ayu7j2j3mCnDM8hdJUoSg==
+MIIDUTCCAjmgAwIBAgIVANCoYPN3J/AJvnD0isLsBrvUshfyMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxOVoXDTQyMTEy
+MTIxMjMxOVowEDEOMAwGA1UEAxMFbjcuYzEwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQCN4MpSJsAGSUT70inndC2BBNn6dJUWzM2h40/w1Fn+cXZde6P6
+ZQ4fmHIGAZSAe2b/sIjpcYroaUWeLeb4iy8qc0QvuNvshaPSXRRFWKDqxq9kDbD4
+cGcNl7TIF9FAfX8zz8P3ZenlXJ2jW5/dCwYKQMH/jd36KxEbCSDFcyrtZrlxOmdv
+QPXAiNnuRzEJLbtKldPcJv/Ix/1JSqJh40P0iocJgLj48DDqmpHaoB2/4QkRSVFk
+dVJepdFgDf2wG313QQ5WUcXGz485dY6SA3qQuJ2r+oeKXMDKbBWF583sTv/IaA/i
+09ZXrbg7Fi6Qj+JLIOjZjve/b4hyVOnlyLk3AgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FCFzm4dxywbXTaTFgrp9HVpaQDJOMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWCHG5vZGU3LmNsdXN0ZXIxLmVsYXN0aWNzZWFyY2ig
+JQYDVQQDoB4MHG5vZGU3LmNsdXN0ZXIxLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEATXeD5PicG822oBYRQhxAWuq/xtGCsZsBGIg4
+zPpML5uzgXDIN8mkMaatDLd9aSmCjA53nP8SPXBo771aRETap7VWZ6nLuKwBeuRq
+GJLhcgFPZzNT/ddFos4BLiSsOMfo5TK+hlkl9rpC0S04nvRIrKqa1J1PNGk76EOz
+tuox9MAMELlhwNug3lSY5yS8933N9PRR0EQcVJYR8UWePJgVFlx+UnW6ftH8duQU
+zK0YknF4aF6e/Fqz0aaQfLoppRdnWn8RYMgZPygIi7qw32IbblqTv7cj8d/i8i68
+HcIycfEA+njKDkM/4FGyvBizMvQ3UwcVnIgMVdjipMrTkt3JPA==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c1.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c1.key
new file mode 100644
index 000000000000..db540b483322
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c1.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAjeDKUibABklE+9Ip53QtgQTZ+nSVFszNoeNP8NRZ/nF2XXuj
++mUOH5hyBgGUgHtm/7CI6XGK6GlFni3m+IsvKnNEL7jb7IWj0l0URVig6savZA2w
++HBnDZe0yBfRQH1/M8/D92Xp5Vydo1uf3QsGCkDB/43d+isRGwkgxXMq7Wa5cTpn
+b0D1wIjZ7kcxCS27SpXT3Cb/yMf9SUqiYeND9IqHCYC4+PAw6pqR2qAdv+EJEUlR
+ZHVSXqXRYA39sBt9d0EOVlHFxs+POXWOkgN6kLidq/qHilzAymwVhefN7E7/yGgP
+4tPWV624OxYukI/iSyDo2Y73v2+IclTp5ci5NwIDAQABAoIBABVkWu0nFZX9Dy8v
+AY15+XB2JE28rUwaRjkOFpwX0JfqULAIt/FFsmr2J98a9uKmeKG94u7b5VdkpBAV
+GLnJ9qseZR4S7+oKPmi2CxvnoJ+1pHVgERpsjEVC5gmzeaQRKBBOV9xiG/Gu1L/w
+TTrdciQynm2jXOBfBP3AExZiZosT1AV8u71kSK/asynhH63sb/ck7+20cqn84bJv
+glMONb4T/QSDpHayk2Es39cAxc0ATo8uzHQIZjDFLj/o32W9dmwwE29blrwoYKhq
+T3/fFVb6V35MIpE9+gmHfvNxrV5nhtAKg10gnEBeH2PiK7lcdWvWGuub6N+oRHDQ
+t/ICxj0CgYEAtrirvehVaBR3P9yYI7EslqvDf9eKKMSQjGoChW3gSXzVAV87Fvfo
+qnxWy/qTEKQM1V978ABAkzytc3zxWusMLjyJ1KlQKlCguuo5shXJlxyk3TZhBr3R
+eQGc4cYdx17PqVokWDCy+a4NL5T82jHpb2x3pUjd4Bgnbg7Zuc1z20UCgYEAxsbj
+4+5dvIjJTtH1RYJIGu3XKHBS/tMBSl29yfVbn9m1kktywByycc1gLfkCa+UzgyAr
+dUrYz1H3WxmNYAfJcnIuwcr65RveQ2rCxjAEv22b0OtvLLHzCyzB+iWCea5qOxI4
+slRVLy1Ne3L0+rEbgI3RVdzqIImt45568q0zTEsCgYAHUkr/ZwnVC8zjcDFgbL7+
+oo9I1hghHg80sSK8ufRcvXXDgk9Z0kEuY6EOt1GhPxGMkzDiIwBzZeEFWaJKJFTu
+nVpgfRBLQcPO+YeqTRkwxuMMQhuNK5nzIU9p9IhI7tGfVY9TQVBKlODOYUfobkuO
+PWhl5Q+O5SjfThUHTuYviQKBgQC0tkqmhKVpar4BT3X5nGjrOoFZ/Uwm6TQKJNEP
+U2XaHHYrKzaGm1BLj9g502E1MCm1s0BvIP2c9bq4crGJtWqaBl6lxmB0e/KEPgIX
+Zb5bPk3jELeA3cLEzQsI3KMS9tMTIwyya2MKQdhoO+mmI++J3uSKeA3OSPzyq7jA
++V9iowKBgQCQUAMRGQ83Xof0Ku6GW1S6UzkWsEtqZ7aBc3sSZ3E5s9MCiPzNjQ9n
+9C4JIRPSXvU5wnd7nNwtiiYF6gMWdN+dNjiHHJ5Cjv+qXn92UCJVA1VaW3C8FbrO
+UuEhu0m9r7OCMaf00dSJftmQXQhrsfePPtD3qk7lzdw8G0TsPmMpJA==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c2.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c2.crt
index 91d98d23d79c..77d4f9773a33 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c2.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c2.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIURYFhQEC5s1rDqkRC5KMkc6adOgkwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0
-MTMyMjE0WjAQMQ4wDAYDVQQDEwVuNy5jMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAimtML3GHPYlipJXpvuNCtUF4NL0qORN1iKA8wSeQEJb4a/zhPF5TqBd/
-Ixd0OYvUD2dsqGmQNiHvoUp13siT9++3sjFrdE0x7y92AwkKzl2iDsjN0i6dxIIa
-EYpVf3YVDONrcj0kbb+OPBG268pstENfluTbhuTI8RaJ3Sne2ycCAwEAAaN/MH0w
-HQYDVR0OBBYEFPaU9fqpl9MA7iS5rdP2JqcNeQI6MB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU3LmNsdXN0
-ZXIyLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-cCkIr6yLbz7Soe0pKkZjK26vBXg/GSn6Pk9m8YRzAlVmMOOTwEjaEB0mPPDo/0K8
-TwTvxdgghla/wVbm6j8X7J3MpUYyYqjUOW0SqBM7THrenNEWMgFqFVo/9Kj51U6b
-kM0yRpxaOhL8QFKklldQpmdzLUlme7ZpodmOOYdNUcGOzqhsKptJ4vYc1YgzQgcQ
-fBAGpgS4tb0KFUWT3IbZYjj+NI6gz6xncX2KSRiglkvX2+IT/Vdo+8k+1s0VCs9b
-U1PL6ufye0ZPcl8uqEqrCLd6pJu1A+5FajnUqAGfRJIbzvslpSqyuFAkCAnFKpA+
-kGj9+HvreJYiuATpOqEV3A==
+MIIDUTCCAjmgAwIBAgIVAN/FuIXocylN1Xj26AzVyyo176BJMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxOVoXDTQyMTEy
+MTIxMjMxOVowEDEOMAwGA1UEAxMFbjcuYzIwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQDaCHi4FmMM7zsQjPw13Eyl0DK6jlswixFRB8buv6T2F3M/mrYW
+m1jENeXqzgpksivxUF0IMqzqlgbaWS+jHixicShzE9R03LVdU54RSNX9GR9jv/Qz
+HsWpexNVh4XFcQjfoo6D7YInQLu+Gdtqblt/qTuyRv6wlSku7n8H2WcgjIjV8oaG
+HCpULX456JDRot//PCEfkcBxZERHBc/tWk0e8p+/Ttx4vbe+C2avFhDYhVh5+3eX
+8gJ3rW8CRfa5TsW3ixqWg1MS75mNWZkBJrz9AFezCCyOzIi3s+LlXsnWGfewWz+1
+byH1cYw62Ni4+W8vDhLU9iuZ5lX+DoVqhVUjAgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FB7ZM/wiSNsJdPh4Ad/KoiQuutS5MB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWCHG5vZGU3LmNsdXN0ZXIyLmVsYXN0aWNzZWFyY2ig
+JQYDVQQDoB4MHG5vZGU3LmNsdXN0ZXIyLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEAEelZZs2sJ3Kj/NqZcaTLUgDylkmp6RFfteqT
+eip6LU5oAxSvmSj/DgMljHPaxiR81NDZIpHSBaBHMew7XXGYylqsdF39XezPT6wA
+3ZglgPeeDrnOxmr2oipTB6Twa/aRpcWos3wM70CzXVF21QzsDSOm0JGBeBKwEwuv
+RdPOkzYAs9btjNt6mYRhfEQ+qg009nZD2ASjJ6ibgbno3WpHjtsZDzcCEqs0a7uC
+5tn8qSf9LtaorivZYqljzxbEFp3i9KX8VgpHMWc4/f2MfCwCyaim9+CyfInjJPXo
+5SBt9F+5sNSbzEjAognV8HKIG8ye1lHt1gLGuR5oqohGNSyaDA==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c2.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c2.key
new file mode 100644
index 000000000000..ad6bc7d43989
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c2.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEA2gh4uBZjDO87EIz8NdxMpdAyuo5bMIsRUQfG7r+k9hdzP5q2
+FptYxDXl6s4KZLIr8VBdCDKs6pYG2lkvox4sYnEocxPUdNy1XVOeEUjV/RkfY7/0
+Mx7FqXsTVYeFxXEI36KOg+2CJ0C7vhnbam5bf6k7skb+sJUpLu5/B9lnIIyI1fKG
+hhwqVC1+OeiQ0aLf/zwhH5HAcWRERwXP7VpNHvKfv07ceL23vgtmrxYQ2IVYeft3
+l/ICd61vAkX2uU7Ft4saloNTEu+ZjVmZASa8/QBXswgsjsyIt7Pi5V7J1hn3sFs/
+tW8h9XGMOtjYuPlvLw4S1PYrmeZV/g6FaoVVIwIDAQABAoIBAAQq5eMGHoX4eEV0
+atYjhcPxN/I3I8jiQTlhlizDG59oMSSo/uXT7PCKY5otrqb5466Jp2jJAZaJIDMC
+Ds+/e5noHGqF5jY6CWELmgZhJ4hed6o7vyqClV5yqU4MHwNeo6ZIRt2ztDZId2Fs
+87PUAMpVM3xqMGN/5MxUwVENMUdYdbTO08vhVTg/NWmBOXNf1Y7Y6vg7X4XYs3ha
+ayRMLxzeVc7LuTtnN4nNv09tFb5XVlT0sp6LQLop80w3UDH1c9JFPcUmeblu6PgS
+qkVMfV1SqCxOTzv8wfF3cTUrj9slHJSW2GzNmtXPRk+eoReJyPWYVKx+I+zHSJZJ
+3hUoDskCgYEA8X7oQzhM9ZwB9Rcfm1k/C40rpj530cNWcvqnVGZUmw1vN8avJ0F3
+6pqbFKwpyIQ3gqwg9mPrUNj+Q6li5V5OhZTp+qhjEu5B0lih8PD93MbMLBsbeIcz
+11VilHfmaexfG7fZue76jAnrANDti0fMaatDRiFGf98hQT1iab8Kd3sCgYEA5yDR
+H5SdGOPQtOggabrpaZcZXbpDsSn43g9EpxKs9gHElLgpqieWnd227GOq1Sr8lnRR
+Z9V0Zj++bB2EwQcfW6MjQIG0VX85G/9vJCL4ADD3w0FsVl9uchwVIMV5JVepN8ny
+odLf0z69Bbk49Na/LeZwJwtFe4R9kAcFlQiT1HkCgYBJF9w83EGO5VVCVWlfn7eR
+S302qBmkB7eWvX7/8KkS9RIAk8ZX31zoSTs/GozbAAm9J1Qiv0NuCP+MgyxHuC+U
+Bup6z5SNGkgZ3nHFd2ue8xzBDMNXy7RJLLLVtSSsUrrGrPLdcBOtZyMV8Ca1zVl1
+cjy+cTvhYCu0j1vGHeiRGQKBgGGh9zTD7mipM2x38dk334l/DnQNTrLz6/DBulpp
+BS8WKBg6gz8hYfplwfOxVmSdt/1GPW0DOhxHKQU6ou13OMPY8fc50xeammE9UZGO
+Q/ywsriEVmNWBK2nNzWUtUwU76JlaFz7Wxk9E1Prnieu3ytggFe95yf6G6BPlwgR
+i4DhAoGAc8RXYihM2L50+IatqZmKyM7xq51CEpzHP5cQA/c8b82tHxAQpOEk3xtj
+OJ0nhZ42fENIrsw0VNXzUa+xAenOAeAtAqfGQ9v0MyQ9kMlsOJM1ZA1WOM4tXCDp
+SwqUy/xsT64xNfL2XptC8kttpW6HeVEmg/StTEHWUXV7z8jZYLg=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c3.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c3.crt
index 3a04cd5cf84e..6983ffbe53a3 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c3.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c3.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAM/neT1zlMH5MrdlYtC52vVP/WAQMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjcuYzMwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAIznxxhfyJLtn7A5vqmMhlRHfI8NXkNTxlaGhuXE1svx+B6PgkQeHZXZ
-Ium+ofQO0Vtam72SYLHXiwY+NrW5VNZzmJxd8uUjSHA4c8bO/UozcYzSB4HXI3lE
-eqSzLwwT5PtXVy4KBzHdPFJ/IWpyCvzQPGYD/I9o+e8UutoRRohxAgMBAAGjfzB9
-MB0GA1UdDgQWBBTj5a/lgJ5f8S8/AdEvVUD5dfl61zAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNy5jbHVz
-dGVyMy5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AKhwKCJguaevAFQ1cMHUrotPajE5Muu2CsIhXUR+j0YQPiax2wUdNmHvzjNTm/9g
-uTNsYa/+FFQom9LToLZXU27tGDjfPERPCN2v8rJVGWR6bCWujEMRpmjJOT73dth6
-qTdU5DY9Rn6v/QSakmBaAm2yF3UTNtd3NXM9PWZlDPq/dvF4Cpv77QMIIpwYCTO4
-+GmYJ+r6RHqocGwPJf8ZXFoMuJpSxzbilkYVzRNWHRUR/c2Z5IsuflMXKdj1iKT0
-81FfQAissG0ied7anNO7QePQKcz9DR2SdtItF2ujMXp1dRezFNMbOQUW0v/iD6C6
-G5sL1K02vl70+3CpIk4Pfa0=
+MIIDUTCCAjmgAwIBAgIVAJF9AIeo2QpwpRD6hU7pzjm07Hv3MA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxOVoXDTQyMTEy
+MTIxMjMxOVowEDEOMAwGA1UEAxMFbjcuYzMwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQCZtqPId1uAsdvYvnGW9kJgUGecqhC1nBWl+F4IsiPpdCCDtVYZ
+Q5/uWsrrXQ60E/Ec93nOymEx1trYlW1Motz8+UkWs1/SYhoZPojPLPktGr+JdFi0
+h+9RNuHLXzzit93tAIiACb/H2uULCshVLld506pzKyNiLYoWi4T/sghtnl0UOKby
+ldUPNXbTbhYelNbOZTQiQtHRZWGCdTsPjX4HlE6v2qmWcwQ3q2/OKdpdw8a8u70z
+L3TP8jsTveSOuNFI+iIBH4COKyJlH5mNKStu1edwNJdIDb3vH4000t7j0Mug3ouq
+kGVREl2yGDDqD5ik+7FvGS93MV8XJvTTO0DFAgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FKqZEBxXoGlmT2YmFgWWibHuC+5kMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGU3LmNsdXN0ZXIzLmVsYXN0
+aWNzZWFyY2iCHG5vZGU3LmNsdXN0ZXIzLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEAV2IYJ3hj3ZLHkQJ9LRN1CjVh78smseyMw3Sh
+jcxEz78+tDtlsSsXfvGfuocXtzXqtxy2YQWrArsmle3vQFm/5Wf8ULv0iojwrFbw
+yxSiWsv3Uj78tz2caj+CIBWGdPCN0tITWQvsjnx/FNQnjN8GyWDoxnebIpRKb3Mm
+YVCdLzfztOrnyhqUCuraHeHtrSRtUdkw1amOTZnO0AN9xJHkQwI1o11x2DQEIlxd
+lwlt4ojpqTpsxVpeJVa93V+D2t9wWNdhVHCH31uomD1z838joluAcDf/xuD2iltu
+V3D5T7ARmhiDzqKhyqmD+M+pjd+PFv7j//mou/F6xhJJzNW0cg==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c3.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c3.key
new file mode 100644
index 000000000000..0cb82d7836fb
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c3.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAmbajyHdbgLHb2L5xlvZCYFBnnKoQtZwVpfheCLIj6XQgg7VW
+GUOf7lrK610OtBPxHPd5zsphMdba2JVtTKLc/PlJFrNf0mIaGT6Izyz5LRq/iXRY
+tIfvUTbhy1884rfd7QCIgAm/x9rlCwrIVS5XedOqcysjYi2KFouE/7IIbZ5dFDim
+8pXVDzV2024WHpTWzmU0IkLR0WVhgnU7D41+B5ROr9qplnMEN6tvzinaXcPGvLu9
+My90z/I7E73kjrjRSPoiAR+AjisiZR+ZjSkrbtXncDSXSA297x+NNNLe49DLoN6L
+qpBlURJdshgw6g+YpPuxbxkvdzFfFyb00ztAxQIDAQABAoIBABr4FvtkqarGSe7Y
+1yAn+9RfNxafJghfJtM0ZPc8nh1nMQ86wSz0lvpgdqrd7Gb++rZc20PDcVMnwcTv
+Trp79q7x4/d2KFFGwhJ8zM8gsmVOBJ4OykN+eCg1K1+dyZZzsR3Ukp54V2wuxTHv
+Rd/gHhvzXbwQLQicOdajywV7zIvC0DDyin2ZOSO8Ije/rxg0lzaE7n0YD/1uOWfU
++ayf6TDhW8uziQn7iclkiAhTxpteXPP/X0PJ2qG880d4rbuUYeG7DFwOdhsuw4P5
+G4OXkC5q5L8lfCaArfLmt3ydp1SCHtsuo2B1dpk249uAjJTH2kqRI6CPEAH+pySb
+9vigNXECgYEAxmral/XAai7V6sLJGIsLXvDSuJwiCLP2n2e8wd3p6y3XhPQuz91N
+2IyxodXVJ7BwK0aGYZ78ZeV0VC4Se0JSWg2aYWxzXNbzoFtf+lMr3L8ijzpHvSVq
+gpcmQJHyF6N9RgRWFaJgkXzddULO1FE2umF0QaXu3Ld1MYxC8mRYkakCgYEAxlKR
+Ig2uvXFN53XechijB2Lwq1LdVewf793GsyfJDDQxrhdeT8EUnIkd3MdSSKdcTiSM
+fgdM7jg2ergea+LsTicau4kh9K4lsUmkj4avu/rIkcXSaxssAqiFPaUzlBi+C8Qm
+zLmWj/cJ4bDxy+Ll3mB8ERJAyTDjGmD0VpO6X70CgYBHwA4qE2kwPJ2IlkWIBYfo
+HiflMDiycvmE7Ut5w5i6PcqIvpBLBJkI7N+zSRMalhgJ3ifsF0ToSBBH/J0cS4Bi
+JjckccZJ1UUmQMS1Tew3mDqUFpwojuY0MRpZES29VJaRgjHrAtvu4Hto0u7eIjwr
+K+CReMKmuAS+FR2Eb7sDeQKBgE+T5+qLJPedFE5w7F5J5njbSYjlSizk/3enPMi8
+iR6dfuXFOLG13whkuJfTXwAKL66nWbxkP4lDAxFA+ev+dSVy6XYgs4BTtIor650v
+xSofmgVsJF+X2pQCfNxcgpLFTtS3033IZlLEJLHr50oRdH1Dp+TNtzNDq4z1JHXs
+9xbdAoGAataIfRZ02MyocBNDyyoVGu1hpev1YbZmeTgMjo/R4fSZl1xCJuDjSf0P
+xCI4GyQ2Fo2tEiRrv603EHk70iijHnoZatojxRSeULVKcn7t1iWz9DmemEUfQG2f
+zK284fSPh+bZq5r57T9JFZAFmCgx7f35G6NvZ6ENxhDc4WZFveM=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c4.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c4.crt
index ec17b1d016d7..4f3816fc622c 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c4.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c4.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAMPIOHRQfF2kDqKTYZFJL47S67T+MA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjcuYzQwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAIjxhvo7UMgEzsm2jc6fAwUSuKyruhbOGFUVWp3wgdSNxRPGEXYdLskE
-BMOADYMcGidTKFxzHl4XK40Qt7GlIB3nsSsttrJ9GChA01ngClYYNQvAaaV0WUxq
-eFMIUcRnYENaBS40J/jXE1KOEsRBKe2mphRWT6tdWLWLKdrGwIFbAgMBAAGjfzB9
-MB0GA1UdDgQWBBQyBHPb5d1Ym8c1rB/MH85v12IeBTAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNy5jbHVz
-dGVyNC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AAyj9Olk0h2NKZ5ayWC/srj9uZb0lcR67shQpgclJIvAsvjzitcQhjmL8intD32j
-UkPFrm/KGEuxitT9nX7gEKGQ0Qh/4c96Co8c4UDxd/RDWQnL3u/LLPTW7H+ZJh0S
-JH7pzT7/8omnrAh8OqqHl3961uLznvRNNIiu0ejpNRHwXFoPk9oL65U1syx8YYx/
-s48dbT1stzImO5TLXv+Bpc5gDJ7WzN+9G5E/f/DJ1etIsgS6r87roaq7WtEjFrvS
-9I3Ci2+nIL5U4q+6hzdilvMiyjTtw2fQKEVQlp5YdPArJYaIjqzggQOUhtBnCSWP
-87bX959dMVWMEJ6ji2zi+3I=
+MIIDUTCCAjmgAwIBAgIVAKXXkae5wC2nLtWYyJO1qdq9ItiNMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxOVoXDTQyMTEy
+MTIxMjMxOVowEDEOMAwGA1UEAxMFbjcuYzQwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQC4WUXxCl/3HDBtMxme0gPH5P5Ep4uAn2Tq7ZWQsUU3mHAFp6iO
+DgZmk/RqJ0H0ATzV+dxQJ/Ce2ljfiqv4cVtaCl5Sg8bSxkgTHAx/G2KIKSqi1uhu
+TsLXiu+jNOt4Oti7/UYPaWc7+9whJd8uJfXWETER5jNk51EFF0fyRE5QGyD5Aqp2
+NMNKoDsVMShu3krhvJaG/LSbNdNAODwRYwEeqfEIjeBXc+O81nE5UnXsyAaXNadK
+SSnywtyj9TMGJs3/rL2s1Y4feSb+78KXJ5mmsOmtCf/rkrmhrruIwEtZaOn7GvRO
+IU+kY4r2NPM4csAgMy22ubQTufOo+d/BpnYdAgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FC9G996atrciQ2PRBrlzXuUXbKYcMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGU3LmNsdXN0ZXI0LmVsYXN0
+aWNzZWFyY2iCHG5vZGU3LmNsdXN0ZXI0LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEAkCunGbfmtPWrCNTB02KCyMAdt8/bCePEQBA+
+ddpvjAXrZWOr0GfKIYpkhaQuUGAWsTXYbz6dxd42wJFwevPHeuMI01+EXza7C13J
+8yFAcD2iw5nC0d64pmGreo8Y43HQPUIAEoRDKMoFoP5B4egXHv9wm1+pJKFZzneM
+wmYRzPZVDFvDz9WbUpP91PspYOZ1D0awefaM8lCEmtLz2pwTMqSwWmhraBn567Vu
+AI662JQ8/4apDft0++eBKhX1BtFaRT+XuK7lxDSqGEKIPmgccWt8cXPmAlxt9fV3
+WmPRvM04mN38btI/TAmmWioesV+5AkW1n/k2/fIoGwYxWKAmwQ==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c4.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c4.key
new file mode 100644
index 000000000000..2c108b158274
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c4.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAuFlF8Qpf9xwwbTMZntIDx+T+RKeLgJ9k6u2VkLFFN5hwBaeo
+jg4GZpP0aidB9AE81fncUCfwntpY34qr+HFbWgpeUoPG0sZIExwMfxtiiCkqotbo
+bk7C14rvozTreDrYu/1GD2lnO/vcISXfLiX11hExEeYzZOdRBRdH8kROUBsg+QKq
+djTDSqA7FTEobt5K4byWhvy0mzXTQDg8EWMBHqnxCI3gV3PjvNZxOVJ17MgGlzWn
+Skkp8sLco/UzBibN/6y9rNWOH3km/u/ClyeZprDprQn/65K5oa67iMBLWWjp+xr0
+TiFPpGOK9jTzOHLAIDMttrm0E7nzqPnfwaZ2HQIDAQABAoIBABhphLWS0M/XJSKQ
+2HJhEAHHSqn/dfMmfdAOfkvsQw+bBC/7hSVVU3ZUtCff4v/+Wfa2tnzMN9+8EnYn
+1aDPghq+XMHt6W+9X4tdn4SwNQV1o3ZvodVZthD3uRw9Gcw7dBAQ+Nd5oc4eYDGT
+jlEnfb1HXDQpQ+JiL3x5Ghye/FqRwuimZnMWINBvi3EaiLLq4efy6bzOmM6LKfx1
+GnqS/Urye72UtWTYz89kI8WWaVYKqomlzUoCQAK8/pn2r8jTd0BZUqqaM9mj4Wus
+ixRPDexbnEHRDvaOsUzRa/U1Ct89EBjgy6eNULGtIXiIPGaWhgf+KzhnEZvVKWfh
+yaKrIRECgYEAuk+e7CXETRhQJBNCAA6mXmN7HogisrIQFjEoeD6bnPp/EJ8weO1q
+td8+kV5tk2DpJt8YthxNUmRdzhs1D+2azOOicwF6rm1vEWjnK9Yh/H3xPj6ghKuX
+tuE2VH4bfmODLxfPPZhETfAqFEZc13ydBmWmYmdsGuj81/e94DOhF68CgYEA/U3A
+RkYoNxMdCv4junHPtsNpz6GhFqS2u7GFoSTHPN86c2Lw1/h2xd7kg6dOotuDnGBv
+pT0ZolA5NBX8iTYbH98FrgUAievItLqhnrZ/Rl/2XBl/rhNo8VfPpzrBHbJytXtR
+33I55bD8Ijw96BauD/vzO63JCSILLmwJfi82dfMCgYBXwt+yj0u7oSNHCwdA7aWy
+upupUyk9EaGAQqG0mpFEzb6C/umfXP6yMznO0jM9hB3voJGMXyKRGBQT3T8OhLcx
+olzIXb8DYSs7gTW7sX+K52x3DJsiQ44QqF+mR0z+KTQHP+jwM8ShDetUZ5yeKOXy
+uHk6y0riFjPcK6f2b3uHowKBgQDEAhtMBdPusjqiHABXsOQTv+ksAx1dIT5JxVfr
+V0NZRj+7tA88gAagnRblDV0bRpP0OOeLzNj+Mefh9zWAk76vjeueOj6teyfHOjgA
+UaXxNia0JAMaF+j4AIQuGpf/O+vppTC4oHJO3AJvF4CsDIjK06LnRlYvXAmc9Xrv
+CuewowKBgQCqCguVTvfQGhShqH7NDpCeBCWc1fLrlsj97yejaPvElEyWvvq2wBNS
+ZeqqyCD3ethOVXDX/H5DwnFo8440tJ5MFzAdnJvTD7u/94TjYi8VWCrS1hrk7La7
+hKpTUXXzw3P+oXOsc5/tJdxx1ImorGUH8s90jx1HG44XXIMU4MwQKQ==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c5.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c5.crt
index f2e2cd8620c8..d5fa9b7b9c48 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c5.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c5.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUIXgmPdXIhG5qFawuVil1bGWGKJgwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0
-MTMyMjE0WjAQMQ4wDAYDVQQDEwVuNy5jNTCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAjEj++bXevuo7QkfAhTmoUwf0WcA24+pQWcNrEzq6KtxprsjgvDEoynRJ
-JgkuHj5OGKEYE6Twah9oWPzRTBAwjSzfKrGfUlor4M++9K1Xk/HkJjJYc6TMt/pC
-GHVwY9GuxmutianrjY0BBab/y/hJXgGYDNa49SGN8z6TcWddsXUCAwEAAaN/MH0w
-HQYDVR0OBBYEFJfy5R84iMmkiqjDjTzeqVWDwWtdMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU3LmNsdXN0
-ZXI1LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-kHXlE5bnKp87EkelJJZFeGsoRWXj60OrQ+AutCsY5k0iP6lbRSfwfaR5A82kxrSE
-e5RZnFKkX5BgIpzNy9zuQzI5O6Iyt7ipghF4EEwptjZOYO8fNHVRJZmX7fTzDQIf
-cGaXaiHTsJi3HP3H5xbnRI/3c+f+BlJsWY0qaH0OqO5HzJG++IakShu6q9cWCDA7
-tixV2vaOEkJ71IzQ0UGSTXp/3TvZK6gKtOWD2oNk2IUeF+UEo9yzEuFox8y3BToc
-EzsUbctLmubnVThHfvKdJI40lt8jRAKKmFxe+xxatEqPrGBVt3E7bvhKhvhwxxai
-mlPaR93bxokwBI9cbGUdzw==
+MIIDUDCCAjigAwIBAgIUeS2chjgBZxYHcSNG0J0torah8NcwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE5WhcNNDIxMTIx
+MjEyMzE5WjAQMQ4wDAYDVQQDEwVuNy5jNTCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAKmS3aN9YPDs6zwaHjUcKgd8ls/NYg2EnWpJioYmbY5fcEmtBZbS
+h/luVMjw9wu1eLCBwvW9HV9ySv0tAUJu2ImxDuXSmQeEidjPxH3SRuZuAG1IIo3o
+NxrFTa4ovuAEQJ6/s57/zTsaOqyt+MMua1jLiTQJI3bO7N+lol+/cBDKcL52Co/2
+34s7WrgNwCQq2Fv7IUkhZn9GNSPFLBY60MghgoUjJKp5VjhU3JCqj+5eTG777xOy
+VzjwKAJf71PReiXvDDaH3BduGM4T9ToNuZEhxoADZRKfXcRiuZXxv7tEelfhJ4ap
+bJOOd8+71AUUYNKH76jzcZckcRqrfVoSNakCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+eulqyzbun/5ZacQTBzDO/Wln644wHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRYIcbm9kZTcuY2x1c3RlcjUuZWxhc3RpY3NlYXJjaKAl
+BgNVBAOgHgwcbm9kZTcuY2x1c3RlcjUuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQBNKHByRqQr1WwmVrnXmR6J5PBbVRzywtEO392/
+ucc7aclt4T5ZxKNAqlxi6Igj4LK1oU3V/pcW6nGyjTn27RTVitberG5OhlsqfWW7
+cGCI6OTwkSC1huMV+19Wxx8XBKblOymjTN+976tp6lzGsCGYrAWZFFKCG1AWFSE/
+UltdCqVZcPIHBnNbtZRq5L/1Rm6h/bcMIVVScs3AvwEfVpKVRraVrkCoby6kwkyu
+KzrMpJ5h/b2bMJdU98iGJ3lBvnb7zWv8JhhlMBDrWoQhGmAafRMKsEkaXJJqu/Q8
+02a+LkfJh5uunObInUWBcSq3SOlzS4npek4FZUSSQSLEhgS1
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c5.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c5.key
new file mode 100644
index 000000000000..9892779aa4af
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c5.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEoQIBAAKCAQEAqZLdo31g8OzrPBoeNRwqB3yWz81iDYSdakmKhiZtjl9wSa0F
+ltKH+W5UyPD3C7V4sIHC9b0dX3JK/S0BQm7YibEO5dKZB4SJ2M/EfdJG5m4AbUgi
+jeg3GsVNrii+4ARAnr+znv/NOxo6rK34wy5rWMuJNAkjds7s36WiX79wEMpwvnYK
+j/bfiztauA3AJCrYW/shSSFmf0Y1I8UsFjrQyCGChSMkqnlWOFTckKqP7l5Mbvvv
+E7JXOPAoAl/vU9F6Je8MNofcF24YzhP1Og25kSHGgANlEp9dxGK5lfG/u0R6V+En
+hqlsk453z7vUBRRg0ofvqPNxlyRxGqt9WhI1qQIDAQABAoH+KmSNBFJT6edQ57Wf
+16cy2qxLB6hQ2xBMSkYYW3E/8lfpuoGG5C290J3ElHepS0QS6A2Ir+aXcUUheinG
+JtWPTNz23H3JxBdeXtWSSeJH11Oox6lA+jkTjZ3FzcRh858ciJNkVFue8yebg4/6
+oND6BTjdFCU0uH8CuOavsUAqxAdvrlht3GJWoLGQPhsVyM5nFQ1tEC/1YFCK1VNm
+d1C4R0dmsl79LnbAV/N+B5nHnc3hohDM+115i6LOC9IClcGo7vsmVddN90hsUsjj
+HjLkW15KX/T3zcCuDfuySmzGL9gRZj5sJNMlIM5NaS3z+bubl2D5kUCcSCne7DRx
+YOECgYEAwSP08wXzoAspGjKrZwEUcrgCHkOmZUgvuI9rpfa6kw17cuLJqTbXhNFq
+FfHwMy2yiZjIIB219pcUNmfeCelMmFCiXv3t9LNLqjUumcuE1ekBTAsQkrHwd5bc
+ko89aXwTgyxEO/71UoPawjjkdOvFTwHifk5LXEvRCakG9FtPk9kCgYEA4MNgprbC
+c3PxfMa1CZOFphYm+o8G6vX+z/zglvSjf1qnV7tCvT3SFiwbbrJbH+JVxRZqEsVS
+jP1qiuABq8tz5xK8V/SjWCnI4CwQg9Iw2W8V43v1c1XM9vLfJe44E8y38jJdofq2
+8j/l/Ljp6nqQvISzLZG/Xyg6yxlXEql5HlECgYAz1SaDofFD3jrloPuG5S9Lqj5H
+SPh67eiAq1zEaXagyFl894gPNXyZVGoYAZno4jar3qi2lcKjrt/pye440rePmPQ6
+cgNOTF/eCe+7rnazwZhrg/yqob5bGc2QN4jPqlDCPzsArue/gUtQHFyTVELbdTIH
+m5N17ZLn+Mi5A1hT4QKBgQCuuPLvZm8u1NjLCKU/NoDUpa/TY1AbYvOct5XXg4F9
+xPkFiDrPZbllJxCEMv4HgSjvtwbqZJ5Rn8YtAwoJhHUrEjN7pF8+b+0J5vdHoiP7
+QxGe74PVDsPwzMiPceoB6kisbF7BRmblVqrwlMayUmklQ3sOV5iO3nUl4SMBDOFs
+oQKBgQCcTHiWP7Uia5WpdAc2LViTBeG54iYtKNkB25cQzicABlSdFinc5m1xtgEV
+xGQZgQoUcaVE8ZTv1nsO4HV5geneYXb2cilRAPXPaBriHIkWii7MzuGf5mvVPlpd
+VS9FqqlZq9YsRaIknV8vlyXINNlKQj50q0JcK3sCZwEqM5pq4A==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c6.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c6.crt
index 4df4def31075..54f7ac2af6f4 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c6.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c6.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUS2riIiKq6tZ9TLvmAmtxF0HCSIQwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0
-MTMyMjE0WjAQMQ4wDAYDVQQDEwVuNy5jNjCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAi7y7aEYewkqr2jpU4pwCkxpIN/6uOvfwl/AI2APaimOrbSo5E8hyuWuC
-AnZt/ETmjeFvdt1MISlj2BbxfwhHDAB6qAH1o0t8h/n5GjGlrA1EskNAPxGao43K
-CsFiki3ulWQMsdGceqYmqdMYkwZs996dTEfOn6uu4rNKF03wA48CAwEAAaN/MH0w
-HQYDVR0OBBYEFDxU26jYM4oNL4Ey5G4IPGiFFIWyMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU3LmNsdXN0
-ZXI2LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-n2m5XJSzf4OFXLVy2CeCMsxkxBCg6UYjRm1i8BmJ8XdSJrm8Ke0V2KRhe+0aBoy8
-rgVlQd5xD85GJNrvT0N7bA2tGSkDw/ucuzcqywzfgM3Sfzg6r+/cs5wSeFQTHTkP
-2CPWSqRJlD052MeLIVUXs6lG2nZeO4TQA9W0qP5T/Wn7tQXh8CdEy2C7Xs+uhUL/
-8eaR4OTSqV5EJCu3EPsV3YY6pZTlr5AzjWIUN+1GS+xg9dz4BatX9n8oB59PqxvN
-5+pgTekpJY6vsnyLbxb0WKDFM6EnHJ3hN8GIoI9Hgbak761X5KVo+DIIw6yq5Mfq
-FSJ/tFhB4tQ6sbSodVshgQ==
+MIIDUDCCAjigAwIBAgIUD2dmtYQ4bszgnVm0eFN5KgkI1UMwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE5WhcNNDIxMTIx
+MjEyMzE5WjAQMQ4wDAYDVQQDEwVuNy5jNjCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAOCHCnhqzTDE/msQvKb25wFGpbRCDNlBrnlRtt5MjJYcQ4/ULPJ8
+PweU2mv0CHK775NZGqewkY2YFhXixES+zEuqYcvzvY4C9wvnL6UIvI00p7f7U/du
+VkyQiS3hHLt9dWMh9RLFiqTrt/dqz2Vjmr7/T4DemVgusAb1g7eBO4vmptskQgA7
+o4Hru2XuQnc93FLMT1oxenr5IAENskOV3aRCmhVWdt0+iu1I/VsgiX10rOz0PKet
+ZD0H8glnY5JifoMHHHiBDAV0hTB+9+kbOUptr9X+J5QrPQsNowfwOR2SVNI1YL3k
+oa/vHFlaWSqoy4VU/0ShYI4beyxbWrR4nd0CAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+7EPqyRJ1Ul9ecuASOt4bym3sZcUwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRYIcbm9kZTcuY2x1c3RlcjYuZWxhc3RpY3NlYXJjaKAl
+BgNVBAOgHgwcbm9kZTcuY2x1c3RlcjYuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQA6K8hKkVTCuy1YAYb8nLo4M/OROfCfVD/CjOaR
+6NgtVlttfkTNKcS+L3RlvNvAfus2HldZXhTT7bJpKdN9GJGRgroq36Fz3qm13Q8H
+rRuRDAt495f11Lf7pc71/MiW/KBwiwo8pdqRnc22jBphU4rgSbw9Pq2VFSo8W7W9
+msrUobaWb4NnyMY7Vi0+WdvxSfhQiH8ECTxCVw6jAxCfs54Zc8M2ELFVHvIvkijc
+hz0maJNxjtp9NwZkmWBeAjR6zhGIhWQ5BqkUWgT7uhspQnU8pxsulmga50DC3B63
+PPYlSn1CXlaFgrziTYtBtA18XMcawjzP54f5oROhW3toABwP
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c6.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c6.key
new file mode 100644
index 000000000000..06a993ffc9e4
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c6.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA4IcKeGrNMMT+axC8pvbnAUaltEIM2UGueVG23kyMlhxDj9Qs
+8nw/B5Taa/QIcrvvk1kap7CRjZgWFeLERL7MS6phy/O9jgL3C+cvpQi8jTSnt/tT
+925WTJCJLeEcu311YyH1EsWKpOu392rPZWOavv9PgN6ZWC6wBvWDt4E7i+am2yRC
+ADujgeu7Ze5Cdz3cUsxPWjF6evkgAQ2yQ5XdpEKaFVZ23T6K7Uj9WyCJfXSs7PQ8
+p61kPQfyCWdjkmJ+gwcceIEMBXSFMH736Rs5Sm2v1f4nlCs9Cw2jB/A5HZJU0jVg
+veShr+8cWVpZKqjLhVT/RKFgjht7LFtatHid3QIDAQABAoIBAC88Sbgeo+9kKjAl
+E8M+g0fR8I91K9XTCx0r00B+Mz9rT81k8lXLTMlQCAmZC+PzZMoHf2OS8nQq8E6H
+aAocZ8dtX2i0bQcA2a9v4iaiqWQ9lSFQJxCRbmdqKunzh4NsN/N0k+XwmCPtEUVk
+e6BVbZtMBBJVAka6f77fDOhgsEHUje2lMpANHoCdkeB0JZHxXjhOITIU+OxHYCYQ
+HKu27OoW4B2vCM6NCOBkLdkmQxCPiR3BR7dymDn/EYbBW6biJbfrJyNRcrce70to
+H07CbNaa4BniwLc7++fP4guYLJKwRadquhjBnKVrej8ilwi5PFf1acKM64C133Ox
+BYPLSzMCgYEA4gf3NMeF3PPNW2YRj4uHDkU9BJ2infYFaV7W95/gUw9Tsb2guhVB
+pacLWGnxrYdAmgV2oDsAwkct6kILXJ/C9filu6dd8aoC5KOSK/Sy5bYMGc4v6W0l
+aWJgQ18zuvlT7LsMupsiGrNLF4tvpvp7O2lzMegOOCkmkxDhYxAvzWMCgYEA/kwK
+AGVhVFUiLnX4RXHiYUDT1+ISFmFjvfWt6OdbsyuOmmw+KaHAPv5+qdQdDuoTsrNO
++qtMAQGEEk9QLGmd9Abebfaa1cuWyqokfs8LmS3zClUt5qAR9UUaeqzhz3XvPy7C
+pd3qtZJ/Ph7KbopyyA8ALLTltUXuv56elocSa78CgYBVFaaK6aiMWgItq6hknqNC
+TfIAMM6sI5Y56geFw1sQKS7aM4WzRpkloFH3ADqHlehNkkzrWAqbF2xDSIU6YrFj
+IixRDVmhxNu98YzumvhU3EiF1kXW2s1AdFXERqwUcBKVhHvwKsGVx3Q28rge5R9K
+aBGC8dWYcnUg2zPctJUp/QKBgHf04KbtK+t/OV+XUKVZooNj9EqbHxMln8plEUOT
+l6M+QcfAldZHsgtFO/Ta1VZKvz4F7ZS7p1+2GloUxeodbGBHuTdAJCwEHL6/YfXb
+r0nz7fkK01s+WRpNHb+Ou/qEVHaGlVVvCJ/cWl+wYKeicaDDyr5MGyvWsFf9+Ctj
+ziG7AoGBANgyU3slzoX3WyNWkookz+r28zZOFfnvrLOUbW3Be0wAQ6j75Nu37Ypy
+zR5DfdIV/no9OxS51m0iAwuU9y0iTBxhIdPtxNl9h+MtOTslog8IIQMsgkxGL1BI
+rMQxE/QnP8fSicIYGSyhWiOd/j85OWjn39nl+ynVhlxXwVFYeCU9
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c7.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c7.crt
index 38ad5dec9beb..d54471078e33 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c7.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c7.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUJTRrnhJztqw6LFCszsDD1c/yPR0wDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE1WhcNNDUwOTA0
-MTMyMjE1WjAQMQ4wDAYDVQQDEwVuNy5jNzCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAoN6Ca8GpbkzxmORqC/sl3GucqCU/ha1ToDUuJxwzS6QTlJJ83DFbqyzP
-pIxx5YAhYiEfruoIVW4hF6L2KrQbKfAVWsfWHFqBpMycmy64yFCOK91EvHJ9Rs75
-+KloOtSHgB5d4akgKOOBVyLHGeH3Dyl7nrUfi6FtCyfVCAjA8VkCAwEAAaN/MH0w
-HQYDVR0OBBYEFL4wJd/SFWld5TliWX3H9JPUcTD4MB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU3LmNsdXN0
-ZXI3LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-gte0h+EhO4I0QoohsX1/q1IsTb/vKAGDaptx0LgXK7WkhOhyY9eJZZWp7JoZvhLw
-4eKqbjyYAxsATDxq46iAWy1dz7FL46A8+e42nO5a6DSm0suufkz7sFQFM6gBrjPT
-3zpD+DhH7vNm+lYNMzq/TpH2YoQlUrivrGS9PSWafmI2jZDVSBpcVbfJoaKUYPuc
-JRj2g2j7rEqWg5upfSCnxocQJXY6v8QI94ZtgbXwVmSZ4+TF1nJ28j1ogAKNjUjs
-ZvUWAbqqO0qzhtK1+X+w6wKNpZguyJ1tQRJ8L+hOJLbVMQIT3s5mNhQyf2qOxdFA
-pKlth68w2dZS2wGBAuob8g==
+MIIDUDCCAjigAwIBAgIUbpWMIAefnXl5wozU/jwyoNcXyP4wDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE5WhcNNDIxMTIx
+MjEyMzE5WjAQMQ4wDAYDVQQDEwVuNy5jNzCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAL9eD9JtlM24/qZIH2+X3Wx9AMcZQk3FZIBKar3UzJEm2tXViktj
+eIR/iwU/WUoXnx3eemyId6g9OOlHPDYLnigTZSfZMOBncgZfwLc1y1lxPnBHve06
+4D85p83ntBtVQQkkSNGwraimYJEU9t+PQKUDr1QQ374Z03h0WRazg89stG32lraP
+NJP28EcuhQSZWpWU22W6ud2BE+vjKYdhWlWgMITHLCnD4WubL/7wXJpYleu72tn8
+lujUpvEaOOFk1QzFjbHi7roHqGZRiowB7My5sOcU40g11VldQvVs7/qPxpcuN7YT
+lRwm7Qd9V2rO72vaNcSUzEXzFXa1ZGeQCUMCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+DawjJcqikrmkIQt+4lnATdNReWEwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRYIcbm9kZTcuY2x1c3RlcjcuZWxhc3RpY3NlYXJjaKAl
+BgNVBAOgHgwcbm9kZTcuY2x1c3RlcjcuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQBTbx7HdSGs63pFzH9erp9H5lfOHrdcfRLjeFgX
+Umcfzdf9WUG8FzpM/kHjPjwHK3oY0yzHbWFu3hZKruvTMRU4vHxDzfyPTLsj0HfM
+QYu9TEpUFjvp/J/8RWmwqMOyeubGYyNIRzfju0JGBjCJQzhwcGMMcxtMEnOoMkq/
+vjxeVJUdzt8feYB6Wgldf4sdkJSiYkBAZ5Gd9o0waw41fuTm9kLI8DIdC02p+9un
+TddPV5rmxOv2SqhMbOcPGhKJQ5Dny5NTTxCHUCXt2U3XUXBvock0TIenj0Y2fkER
+nK9gAXJS/mm3FuRmwywxwt7bVoNsLVVoMnyRgxGR/cjwFngC
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c7.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c7.key
new file mode 100644
index 000000000000..65aafaed4227
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c7.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAv14P0m2Uzbj+pkgfb5fdbH0AxxlCTcVkgEpqvdTMkSba1dWK
+S2N4hH+LBT9ZShefHd56bIh3qD046Uc8NgueKBNlJ9kw4GdyBl/AtzXLWXE+cEe9
+7TrgPzmnzee0G1VBCSRI0bCtqKZgkRT2349ApQOvVBDfvhnTeHRZFrODz2y0bfaW
+to80k/bwRy6FBJlalZTbZbq53YET6+Mph2FaVaAwhMcsKcPha5sv/vBcmliV67va
+2fyW6NSm8Ro44WTVDMWNseLuugeoZlGKjAHszLmw5xTjSDXVWV1C9Wzv+o/Gly43
+thOVHCbtB31Xas7va9o1xJTMRfMVdrVkZ5AJQwIDAQABAoIBAEClSt0ADaONWeDc
+PlpZSACNHcUkYESbaP75pKNqDoz+PjC8SVJ8UMsZtuHX65be8e6KKHsfgqR/wgjz
++oAbXpsVKjfS4/L6nwdOM3DwqDgXzErrwVoxaG2TUX96RVpya1zmN/82GwP86GQ6
+W8dbwi+8Ve8lICA84zYuv6WukNlxvcDniGulHh1BtQnloySzfysCjI9ptzKvYftm
+N/h3GqZqABUEqI8+fJemvD/e/UZQTZbbLX/HikgrHnZ77BT8UflFUEdGbOvo+ZoO
+hVud1d+iw/LMBLt6Gzc5GI/7TNs56clkY3c1yAJu3+aeTOv9D3n/McLnm1HeSzyy
+QAOT54ECgYEA+eZq+EBx8gC0ELd0rl5vhQ0cqxgdrCo6K+6ui7RnvoikBPkU0NKi
+e35vvgazM7eITRAe0al1wStMBlmb/xLE9u52kmfb2OQvHWNewxiAlv4mEphJs7fY
+yp7Qlr4Bx66Gn+r1s5ZdliiF/fDTgYT+30ROhbtzFzBhdhyVhwZRP0ECgYEAxAni
+NntqZtJmIFruf0650me2w0AF/rifwzJ5sP/uqxrDjOiTf8zyzMfLFt3w8byxvEfr
+A8RRHHEYeqVFps3MgKJBCp2EpiFDUAnN0WfV6kNEIB9chNN0onVlt+rq1170uuko
+SGplPDkgHWfKC261/nk4mLYaVsS5AYBhvofs64MCgYAvta32d1dJvJikGGJOZ/ki
+jb5La+8+T1GL0aXBRlAIME4gJO20cIJU0W7D+MN9vCxpigpYPOIeXwXEsJb3u72e
+5yf8C5gNcIMOOUp+NM9YBG2AX1Ky3QamqGtqXBw64zcQkOsXvj1fzZzHPpMixyFh
+vsi+/2qZosL4qbxiavJYAQKBgAzzwjhrSwpkGE9G8r8CipB2PcMEteCwfg29hgDF
+yBUR/Z2PuD8XYWtaP/bQzxyJpQyVi9eB3wIMDtv7k3CD5GETV0zGP39ibzoYi7qs
+K5wAub2NlOVJu0kIjKHsKXv1UCIRSZIPRYUHGTIYK2c591hK1gxw66iT8s1AmDSo
+cmTVAoGBANudebyYdAU6VxsRSYKFeK/SDcYDWtqbUj9xNxPPu0WRaGm5XcR4MtFd
+D01bISxvEOMh+i3SEjAe8T3++oeAfyL4kVvbjeyXkInskZSJUDRHg1Jv+eqhbssx
++wMWPXMBepr+DnXJrHPJHvLGTkZILIxJOxt9tjI3Hu6wYcp6JzpE
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c8.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c8.crt
index 63d393928ee9..a8c704de6635 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c8.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c8.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAPbvvAh3R1XWARmT+6y4h/F8+sqHMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNVoXDTQ1MDkw
-NDEzMjIxNVowEDEOMAwGA1UEAxMFbjcuYzgwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAJEYBwUOXOWE0g3IFtEJu6bR7NCyKdZKgZqhW9CAUUjsJMppUfetDm1K
-RmNqn8cRGmjFkbGtxxJsf+cVR4q39aM4YdCxkMCOmagY8Ay1L6TBg/i65eGcxiDq
-tBrjKUJABe/4Fd99ehLugyXm2e/kRBQoYAQnlq9eI2K1lILJfl3hAgMBAAGjfzB9
-MB0GA1UdDgQWBBTUO6itQmiX4Bhq2pTA1j1OYwkVmTAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNy5jbHVz
-dGVyOC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AAgAeinlmvG7soZGQ8ai3gyEROpwIMpN3yBpgbVV9TKPAcb8Y6t+QRFHztuCFvWy
-N/mijTdB6nuhX4+pHA4pX5V8bL1bjmoaAznR371y6k3b8nRGinZEbAQbahMqefZN
-js6O4XFMJ3hloSmhJT/FnoaRNEZtsdJqdsTlZOS3fA3asxN+INgUaZ4UU6NqjPk3
-xQJD9JV1KFIEakQgWUGhm3RJ33Mb2rlMUZYng+p5vMviMKNsiFrGYJbALHWFhou+
-rR+iUylabeSK4UzswgYJxVwcnly8dsiJayn+7j/CDvwHb62hThcN+Ejlufk0v1Pb
-xmXE9ccZAVthc5SfF6Fjrsc=
+MIIDUDCCAjigAwIBAgIUd0dYYAaYciFT4f6HIPdzY5E1UkUwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE5WhcNNDIxMTIx
+MjEyMzE5WjAQMQ4wDAYDVQQDEwVuNy5jODCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAMitRPTUaCrKC89lmDdq6Rtja5r9lPcx54aO3GCKHAI8Hsqqy34Q
+Bd7k+YSn3ZIqvn2k6vlc3k7weyq1/C/sgqXPxHLV7tNm7xM8mKli/WhXBRv24Xj6
+thj+bb7Kv08iBKl4dXs9z6dX+35csx/NLE5iRcai1KiINGH1GAebjCQwxYQDguwd
+iLmoVq6KMvbzNsjMSAHBk3OyttedufuFqwbYv0N7UrI85m8gSPfgD7XH8rLd0nSH
+JhT7vNu1u7O6Qf32F5F7X/H5NN8SC+Xknd+mRUixwHymLzZG2eghEJnSkhFe+ivy
+7arJVjqUU4VN6HZWtsW3x6t712ptpQsT+gsCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+iNiPJ1drgAGqO8KPGFlhw/5hoNYwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRYIcbm9kZTcuY2x1c3RlcjguZWxhc3RpY3NlYXJjaKAl
+BgNVBAOgHgwcbm9kZTcuY2x1c3RlcjguZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQCjNIgDJJyGlxdDqSTc5YipGyfy/dGz0c0KW1SK
+oA/YoK1mLTBNe1Dh32F+kweOMD16EBvzSzgMmiNaFNBxzx5mtBXKvbdU/mbsQOuP
+J0SESSq5Rgz6MX1wp/k3y39J+KGYmCRSzrDvojJUSDuKYUetZIKcuujAe/28vN2o
+VKrTfSvnXpkYlKkJFEwihnq9MC65BAjLQU9b2TtSpWEaDoFEG/0L2twT1VlKwlZz
+LObLrFL8I+zFzkhyz/ljEXZ7PlI1afOjqj6M3isuAanqfwl+FCr9k8gUck9AETHr
+xN7Vgd5J7mKEyityAHw4yN5C85QgbRliMFHeLk7uXwvpEalA
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c8.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c8.key
new file mode 100644
index 000000000000..d0842904b645
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n7.c8.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEoAIBAAKCAQEAyK1E9NRoKsoLz2WYN2rpG2Nrmv2U9zHnho7cYIocAjweyqrL
+fhAF3uT5hKfdkiq+faTq+VzeTvB7KrX8L+yCpc/EctXu02bvEzyYqWL9aFcFG/bh
+ePq2GP5tvsq/TyIEqXh1ez3Pp1f7flyzH80sTmJFxqLUqIg0YfUYB5uMJDDFhAOC
+7B2IuahWrooy9vM2yMxIAcGTc7K21525+4WrBti/Q3tSsjzmbyBI9+APtcfyst3S
+dIcmFPu827W7s7pB/fYXkXtf8fk03xIL5eSd36ZFSLHAfKYvNkbZ6CEQmdKSEV76
+K/LtqslWOpRThU3odla2xbfHq3vXam2lCxP6CwIDAQABAoIBAATeX0vpU1WmbT7s
+4hKGH32wnc221URiC5HyKVqWPzO2rbCREDrQvLO5rmwPfoqrSs/VjnXaUPJtVy+k
+Fqg6/rhzH/sBQbgTST2YPMPvSOKdKdyR0sqSmpRkp9j3JkkPNDnAWGlTCJSwx8M7
+EIV2/tov/C1jifFEzLCLt7eHiJDgKwmASOKMLNGqVNh4jiY2UMldCMI8PT6uDKoN
+nSbrrtpfwf1fWW67qQh5P1yPlsMWLYvVGrT+eWDZYUP95vm30X5ccfat6LmR9POU
+/z+VHPW3BV016XdyvGY1NlJyaLt7Sd4cth6ycKYtFrVMlLHEjEpLJPxyJI2n3quO
+x5He+mECgYEA6mU1Hyr+MLGaTDdO5itvhnxhtw3hUxUn/VBFWYu988XPhhMQAecu
+4DRBmAVkTIiZj6FPO3AS6TaFEBC95o6E/bVY7eLT7DZyp3ntWJP+5iiDmZIzvDLY
+3I/DQPe+oJImnVFLGUrl2Pov8CCeg7zMbWPMcR41CRpkuSxQAB1m4YkCgYEA2yxw
+nzFVi0HU9l8tWoSYx9trMMnaifZSzNU6PEkBEg1qaaGNUb724IhuA2nMDXPfZFrf
+AZZ1JNO3Dxua7+77bnA7N/3jw6zs3ktkANTl88xkMAXS60Y6MAsLZsKKf9OAltGx
+8wHOZMWSfoMM/2KMvephK6MkjkwcijWCkxzsffMCf1JnQkWy16pGxfgFebdxjcO5
+9qBkQNL0nRpxHfBdMGqDIHICQZ5pqBEqTeUkAB7n5xSXoUNEI2HZyO3c9OYZNKf9
+GUaId/iTK7u1IbwCS5qeo6VZwNZpCdSAtiQg9ON+fhTQf5ZkvmmJi1DgJtophbTv
+YPBjrkGFBeQTsAxaSdECgYBI/7AzFXdsfmyz8ldkwq0wQ8Cb619CfCxv1MUktxwI
+Kze6Hvi2tWf8uPQLZv1zrZhGEI4eNUKnFWzvNz6Pk4Vi7zaKG6H01m++SlXwgPhm
+Dzl/VTE+85EF12fgvgLrOEKn+QfP7FYV8kgsovF9ThGaX6olopzWWDtZPSFUZlnO
+SQKBgCgya4UeVNv6qQNeFO9Zv67MoJE15VQwxn/fOFSbqhGu7JbemLE2m5LCCEgY
+c3yOkNYj57GIoaJKAB9F5BQcrMsxaKp691AeoJeeuMzDr6cQonOT3q7spxregi9Z
+/N2ejgsLA/5kL/Y1f6W3NvJe1LWVdcYG9XpWA0p8XXD2MBU2
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c1.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c1.crt
index f2d915e5cfb0..f339123eab4e 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c1.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c1.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUE4wv8m8ca7CSrCJ0W242hYULIcswDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjEzWhcNNDUwOTA0
-MTMyMjEzWjAQMQ4wDAYDVQQDEwVuOC5jMTCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAgXuRBm6AhPl/n0w+6htbZqVgDzHBd9Cnp13KVWFj/aRR2v6ppf2/ZWSP
-DAp3zcZALQrPOnwlNcAlTdoFYbTNhdiciWb3+jMNSuwZaiNODykWoVUXeF+dHsWF
-mSjeQydkxPK8AfPwS9/8/wTvR2qjFPzIKYk/JPubG07c/BxC+QcCAwEAAaN/MH0w
-HQYDVR0OBBYEFMLuWtBmdv4emh//95cK1tzKVxLVMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU4LmNsdXN0
-ZXIxLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-bdgDse0+Su/oHRZcyP4IaDBUz6L8qp3CFoJsztJN8ja5h19AJ+PXI0amhZ4clwa5
-qho2GqLSLPtButFx/9y28kSCpme1qgXK+2ySwUxOOJDbKCAwDQ6DLZ/umcyxFzxB
-fEdO4QL7lWMfI7hriyNHc5+ZjqqK2i7dP8zamTJ0zsAa6Y7L/jxvRIUd69O1skHv
-BXyeY2bY57flre4JCO3kEvhVtmQgpe6Iggl0hDq4SZ/3GlA9QoLz2ajpCxvQtyzN
-xWuoYf5ZT22yVdIG7Nrpe+eLUjhT1xHQXv1bfpyNwfGlr8OOs0jcdKvENC9GqhS9
-BchqHw44qkBP+5rBljxhAw==
+MIIDUTCCAjmgAwIBAgIVAO7Ic7nbHteW8IZw0EotVbQjYrU4MA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxOVoXDTQyMTEy
+MTIxMjMxOVowEDEOMAwGA1UEAxMFbjguYzEwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQCfpWzZKC5fG0Jxo5JpNNnbNriA+SGxLV4lATgxD0Q7cQ7+Bx85
+ceV33mHoi4Bjzx+HJhoMXJxQsPCqD87Q1UY93ZPp4031v4hvrvnrkItx7FpKojH6
+aT7Z7ASADIcNuXjLdNlivCOa+s98wDn5gPUXdva731nIjiTEM4ZYxfK1MAqZDqiv
+2TBDUvVOtAArvsCGQb4NTKmoOKwt2FwXprL40ctttJnaKFhqVV4l62aY+dkjL8N1
+xDjXshzVI28y/8DT4QUnLsKm1lnvduIF4NFR3LLfTnQLTwy9vsTDsb9+KszEsXbn
+r84aCQpuDdWR05rRXqHTgaKzjp/S3Z+JLdkLAgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FB6m8leBeukPyTIpY5ALAw6uLLgCMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGU4LmNsdXN0ZXIxLmVsYXN0
+aWNzZWFyY2iCHG5vZGU4LmNsdXN0ZXIxLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEApK1WrJI64DE4e1T7N4Mshz/xIc2BeT9V0noc
+VCqR+s40yqphq8RvrzAMv2J0xLSR2gJlL4FRmhXnoPzulLIpnc0Hcsol0kA5hNw/
+zAAup/okb1OKaTpcEB7AlQNc3q4th8GgULqLHKunq1H3V0J1T4Ix3SHSmya9ZOUc
+ZOFotU5mSOAUAu+LO9oDoSwfaCOCrtC6hSN+roTtlyBY3mRKAGhJZor9CRiVBlsF
+mvpOys4eHr2c6i0IZhx0noYYw/ib3kyIt8vKnRvYLNYAOcIG7pXpXg4XWGdDncMY
+ag4JW01WI8vdT96rb7Ku7JKSwvW5BSgL07PPD7TOv9mm/mCRhg==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c1.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c1.key
new file mode 100644
index 000000000000..f091a69eb774
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c1.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAn6Vs2SguXxtCcaOSaTTZ2za4gPkhsS1eJQE4MQ9EO3EO/gcf
+OXHld95h6IuAY88fhyYaDFycULDwqg/O0NVGPd2T6eNN9b+Ib67565CLcexaSqIx
++mk+2ewEgAyHDbl4y3TZYrwjmvrPfMA5+YD1F3b2u99ZyI4kxDOGWMXytTAKmQ6o
+r9kwQ1L1TrQAK77AhkG+DUypqDisLdhcF6ay+NHLbbSZ2ihYalVeJetmmPnZIy/D
+dcQ417Ic1SNvMv/A0+EFJy7CptZZ73biBeDRUdyy3050C08Mvb7Ew7G/firMxLF2
+56/OGgkKbg3VkdOa0V6h04Gis46f0t2fiS3ZCwIDAQABAoIBAAwPRwWwegWdMuEQ
+ENikXBX/MDjYmP71WmND5abEbHcWYfeL5NlGDVDcxQX4DnQmX/n+8Ct2431aSxPN
+qzst9pG90eqdInUp9CMw/e1u4dnH7O+Ds8V4JdNNzO6PWQm5tQee2r0RUUSSexIF
+bgZhriII+TfVS3GnAIgBtxMFq6vxcnwnpHMATRmk9l+3kxG4cx0GpBxxKJKBjP1q
+8pYGSdIh9yaMoiRoZpiYIFR9C53JH6pVsLnbjmF/MTw5gQjwCs6ywcOukNLHRmdb
+JRtEMEkNVterFhCeuwBYwNdZooLZh9sL6saqnqCsYDJHfnu8RC2+kvenyV8eujih
+CuwCySECgYEA2UTbnHpX+g4FZvbD+ERhgaR4Q3V0YAu3Oy8vBD9fcKWLqDpfhsN9
+d/QVx/CbB7QG+5SfN3eruR6eZf4pRJQP0YUWJR+kBmeQ3ufTwSXzzfzxN2GdSoVe
+k2MySOmMXgpSGkpphlWYOn+aANT6otpUjqh8/X6RgIYjZ6tHeZ+K+1UCgYEAvBru
+sKYOaOvn27xcjBO41T920ldQVzYUHhrdMOiS1Tq9axXnNNGWy7nzjzZ73uuZKM5W
+4wboBFKKXlePO8zegSByvKvk9fSHIDqCSScmZmnLtAtY47H0j72AFLPAnzQ0zuxD
+FpCScSqVmlF0gwLINGPB9V33LpNROde3DxsNQt8CgYEArHDRWJVlpApi0UDDcxVa
+ylluNiRPtDc4O1oMl/NwXWiWunNyA/S+bWklrGXTtb512FEyOQTbH3ckwtCBknNt
+Rp65X8eIGKk0OyaGU4M6yN+b15e7Dj1iPdp6O5JZnP3WAhXTQRJIWw2Rfl2Le5uL
+hxpE8bD17KsKCKrRdL9iwG0CgYA9292C2hkDCBFHgbotlo7CEgzRP0V8aJxVCiCT
+ZOhkltyTyrAt/n7IoG4oOpKhzHihormQD0Vdpoc6XzELSWQisrkIsr6o47FMN/GA
+fQ9zvnbuzox81JjIbFkidbpyoIh9Q9UMJp8B+C8agI8ARnnwQJ4YQr524ptjmjzp
+CsxhoQKBgC+toxkdegZqwuDmywpchfKzo4chhwl9H1casyChBHoGt3dp4JKQvH75
+XrQwZd9PEo9/KEQQkISAT85MkPAv8yLJTuyjK/g4xOWJslyBpkbZNca+/Ofhj94K
+hcHT4BVicrlNVno99/kopLEN3CQ3IZp1H60j6CaFkEg6IbDpISt6
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c2.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c2.crt
index 9a6cadda0968..acf4ea01c4c0 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c2.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c2.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUePVFoDIs9eiIrAMKc06YoRk7oIkwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0
-MTMyMjE0WjAQMQ4wDAYDVQQDEwVuOC5jMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAtfvCH7wJN5iSoyeWtfeJFIs3BrCtS/cb4LI41nam3g9T0Ag9WPhf1aPP
-5QBgDQ1umY0kuZEhojduZHgZVoG7mNVNTgqSJCJKooKR+SrQ4gqaIlgIwSpKwo4u
-/za04p1OqRhOJajOMCuC4zEz9yPt1YAfXCkYryBlne5m7Y+mVncCAwEAAaN/MH0w
-HQYDVR0OBBYEFLpfZ8SVBLNw/xipDeUe27M9REyEMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU4LmNsdXN0
-ZXIyLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-dc4bAOe9GrWL3GWSbNyvJJTxEMtlAXTrAlP2kBW2TpiGBTXZ5X8+E48NHAn7SsIX
-W2DvqglKam+ooe98XoRwX0UPTikPNaC2Ud9Snxs4OqpQAxE7XPUYV94FsyRXFA8M
-fEr8REStbQcpFLlUXrIAZjmGlRi6DufLxoGqgmnGS7KBEt3JwEvh7DHkSka8YMx6
-Kd8YrtTPf6ZJuUk/gcmZ5oK/ImZK43asW3b7obgS/wRrzV5OVqx7bvSBd9pfOXbe
-TXwxWpciRGOEHoanIqOWSH2xoWnuzCsL1jsku6d6aX7i2HyddeC9Ler9YOJyLlaN
-tjym0gGIwk4PRCcZ2RJD6Q==
+MIIDUTCCAjmgAwIBAgIVAPY1UvKtFS9Hcdc14T7g6xd7qYD0MA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxOVoXDTQyMTEy
+MTIxMjMxOVowEDEOMAwGA1UEAxMFbjguYzIwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQCrWZeeupWzTp2TobV5Ahw4YoE0htlSq7bHOuNuuvMx14kROFMW
+FqcdBNzC2+cuw4NkLkJwAvTOG52a0hKtj5qf3QCiBO+tWWphX7wmy630/GVGXUZB
+NfFr8IhI3ZdAczTuU3z4J5YTMadCFxxM12peUAQ0TCfnU8wzTo1K3B1Vp1fHNTBN
+tVUZgUKpIOlzhWj2V7eRkyFoy0pmsyemjrp1DwlYMv2YnGVr45YzCjU6DkOzD3if
+Pk7bkZgAD0ZtLaT8UXUQaPjrvXN7Zcl/4roYIF5eImn/PiOWdtQW4+QRMG21/Kr2
+ck8f4KfFrPXFmWcwal+jaf3zpyGaZ8xNUJglAgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FNPs2ipWYi7kSy4yEtEWRSaWmn1GMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWCHG5vZGU4LmNsdXN0ZXIyLmVsYXN0aWNzZWFyY2ig
+JQYDVQQDoB4MHG5vZGU4LmNsdXN0ZXIyLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEArqadNmn2Bz+wf6RbZARIfaqoufbG9xa2Xb97
+gRIi/d5bpT3KmUB19uNBJP9IZ7iYDxSua3obhrYF3u/AZG4XMZGdeHACuLAbZQD9
+YXagUAKkmtyw2kZFVnmCddVnK4e4zc1zXPeBgPPQofEoAKUVoyb24OHCtvf4XisN
+FwybC1LqszEh91VNp20y7vEEyIQWsTFeRK4Ftsn7Z3zWXI2YMJspBH389s83IRtI
+Sbau69ESCgcx8qjTjsz3wshE7V/13R81cffxm4XACT0aIJxTWqA9yUXqnWL6FcNI
+r3l7ZhXZ0mX9k3UE+h5Nvs9lQP1TJvaw7WNjhxf986aBWtBDcg==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c2.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c2.key
new file mode 100644
index 000000000000..85a9d658b6f7
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c2.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAq1mXnrqVs06dk6G1eQIcOGKBNIbZUqu2xzrjbrrzMdeJEThT
+FhanHQTcwtvnLsODZC5CcAL0zhudmtISrY+an90AogTvrVlqYV+8Jsut9PxlRl1G
+QTXxa/CISN2XQHM07lN8+CeWEzGnQhccTNdqXlAENEwn51PMM06NStwdVadXxzUw
+TbVVGYFCqSDpc4Vo9le3kZMhaMtKZrMnpo66dQ8JWDL9mJxla+OWMwo1Og5Dsw94
+nz5O25GYAA9GbS2k/FF1EGj4671ze2XJf+K6GCBeXiJp/z4jlnbUFuPkETBttfyq
+9nJPH+Cnxaz1xZlnMGpfo2n986chmmfMTVCYJQIDAQABAoIBAB6aHx6NkzvVUAY0
+TL1CV7joOswfQ0Pk//0/8qttfw6JQGUPaqNYw1rG6ghxDq5G47nqGxp3JLKEsxaZ
+6xUNPx/K8Bh/ijCq9flTCD7jhHtq8klpUilGUYomCKj+zksH6E2iIoENTNu7s/P1
+PawpkskqQD341e2WBTOqlUhn3GNV9wy08wmvoFIb+7I0WJELVFgFcU1WXljJphOA
+y11xTKRlnn3aGbFCmcmu9DGr6p3Hl/fVKxoOSjdLM2pkBxRR3FLeri9BJycxE/Px
+m6u7ALkdfu+U5y/iPdIE3NRbsjSM/djEq3bA0YwZ8Lc1LK+UKGXFF9lHu1/y8fF+
+Al5RokECgYEAvnUKVYIs53ZwU+qZkojKHX+Hu5iNQAO2sNpdiXa6IpMkmLEJezeW
+WHyit0+OkIC6zS3YowXi14ch9ZwzeF8Ww43DGgiL4F+xSV9sG/QECci0oNPYLfgr
+dN3Rz0aaCAjst+n8k7jN6P384ntl5CACGTd5fyOtbQMAqHd/MGUwfIUCgYEA5lE8
+5MbQ4P7NIkU53Pu6d1ISgCvnfw4jv8QcVpV6BRAXt6ZfbDFm65WpIwchvle0/tPt
+pDJG+iTdz0IiIn61kbm61LpBMd4B7N4N9iz2uhs1MNCDaasGymfKf+o6Yjuowr1H
+uiqeTRGjn9H+znCWfCJ8os/c1bZdk48cL6ItzyECgYBNkRdDWf/1yLvYwWwITOOb
+/euG0It+bcTC5DxCFvOZzzmRYlDNOQdnRGN0qtcSLQtnMuvPAT+wrZ18DUC9HmUE
+AcFa4e4t1Gs8ybsa61eVsIM4GO8rUa5JK2oes89ZxGC8SnSOqS5bXM3YY49MyGhy
+YrB9wkcIZben5PsM97k2jQKBgE6axTyTOjC/5YTfuXrZEu+sPBIwacasLblCFiXa
+yBU4Q8/y4ThMuAKJRntk8LhEuMoLzFwwqJawxF77BClfs1oz2DbIBn4ddezAcGVn
+PSGRLZsmXuNOcnfb6nmzmA90qwkjjnpUF0jYT065XcxZEpL1NjJe3NC3d8JnokgD
+grzhAoGAD8e59av/gr76UjoSWe+GyZDPaAXddcNVnykdmv0v1Y4qVML4/6H9UVQF
+n+1a0npSwIet3HoCSa/nHrIf9g2pEMpEfS6UMMIwI+XkyPsNFD3HVgzcvRSd66bP
+nQ85u7DuxfL18bJ45WIXaDxe8fmDYm/7Kph2fu7c7WZ/rDheHKc=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c3.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c3.crt
index 8b94517e3ae7..70538a32dbfa 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c3.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c3.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVANUiRixRh0fDx5Rw/fcEb1n1Sg0cMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjguYzMwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBALo0zeVKn2186RU4y35d1zlrEBBw/neVSSJME3AOj8VC/QgiQmqYMpWy
-UPp83aXzmlD2ZEle+aS8X7opg89GPCrZVcxK8tM4U4HTEqLP2VXUAp/I+/Vtv9fR
-1QQDB0XfkQx+qDUdwTzS4lv4cnFzsmVDgVLR0NzjD+3fu3bKhjDZAgMBAAGjfzB9
-MB0GA1UdDgQWBBRaBpO1OJaW85kYzyBLyLGRaE2HezAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlOC5jbHVz
-dGVyMy5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AE88FTjwLDZ8bsEI/tQtTfg9G7iSTlQZsKmIF+nRQXUU6b8EMtMDar53ug/yzzZJ
-fZshpjfUmp0w+pQVBt/xrrWLtaXqe+rS9rNzsByGbmTvkl4t36nBv0GOxtoqSdvA
-Cp7QcxVBvJkOK7z63oyxAhq+A1rnoxeUGoEyjuOjeIoLT+oTPLUSSqSmMwqxSIOx
-Q948cGwSiA9QtA8FcgC1TLcHFgTT83A0dBkYTPjJ+7gQwHEGL3kucZVQBBLBpju9
-lFleod93BW+Mdw34Gnc6WAqBEyNYIiOMkkkDXzgDdDMZazgVsPxiOej/7FcS4thD
-ibJVseFjLtnYOLPbv0Wwk50=
+MIIDUDCCAjigAwIBAgIUW2M0qJ2l4fpYp/3ECOSNaqQB+A4wDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE5WhcNNDIxMTIx
+MjEyMzE5WjAQMQ4wDAYDVQQDEwVuOC5jMzCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAKbE6mxo9qIB7bxzHAuqj88uTN/zmUaaPn6FoyLzWi+k71ddfMaS
+tWR0h4yYZB1TV75tB+9RopZUPvBILEGbRqwzBizx5fwewxZtBDEqFxL44c+7+4r9
+vOdCdNmpFa7T1JZh5H++VGCuCRdfQHqkkMPTXZIoPeebgcGgd22+XPSk8saW69z8
+Rd5G5oAQp1AglS118jxTmYerO3kK2NzbzolQccsJ8KLWLNBYw2t/ArIVyE+Pm1rd
+8WHSTfb+7RVCsfnqp8s0NljAiio1W0gDrscS3adWZQplpvIPEkOWEYLDidkXfAhR
+f2G03BIpO8BbETDmDzBKDvKD8MFoV9K2Qm0CAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+pTeuFmtG3bh3rS5eS4kPEXcHFCcwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTguY2x1c3RlcjMuZWxhc3Rp
+Y3NlYXJjaIIcbm9kZTguY2x1c3RlcjMuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQCtDUmfijBjwIxU8FLNS1CNwmS2g5sr+bSfPcn2
+sQ16DUvmVSbTwz34bahSd+fIUvYgWkD/N1eJBVDWSyMdR/HzV8IM9OTtI1jxOhB0
+ecUVio5/XYasCNe8fiOgoxukFeeL9gK4ZGpIlZDCpCIQ7k55LK1gzT+fTyh9JoNf
+t0mytq794dqZw4FUJy6McSSO4vLOuvaZpCK+MRN+pl771TuDt1bh/41BWB36XqTo
+ojS07eUKkXNab6Q8flkgd0v3if8BLkmV366yqU6lDZYEkA5KQpkjYvsztjGZ8LwS
+gvoR2KIQnHcSE/h91yFqMDM/XWvBtGBpP5DAuHE2XqEcqjNS
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c3.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c3.key
new file mode 100644
index 000000000000..e3e2b611f67c
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c3.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEApsTqbGj2ogHtvHMcC6qPzy5M3/OZRpo+foWjIvNaL6TvV118
+xpK1ZHSHjJhkHVNXvm0H71GillQ+8EgsQZtGrDMGLPHl/B7DFm0EMSoXEvjhz7v7
+iv2850J02akVrtPUlmHkf75UYK4JF19AeqSQw9Ndkig955uBwaB3bb5c9KTyxpbr
+3PxF3kbmgBCnUCCVLXXyPFOZh6s7eQrY3NvOiVBxywnwotYs0FjDa38CshXIT4+b
+Wt3xYdJN9v7tFUKx+eqnyzQ2WMCKKjVbSAOuxxLdp1ZlCmWm8g8SQ5YRgsOJ2Rd8
+CFF/YbTcEik7wFsRMOYPMEoO8oPwwWhX0rZCbQIDAQABAoIBADTEwCx3dfrDQ3dK
+sQgb+w8iZl7ClLBbZATiwkDv4YyL0tPBGBc5Gr5jhPR+z8bbtTrjAwy9twV8OMIN
+BDvqJw2bw4t7KzuAfFhfPGFWISD2sXTq1Wgga6J9S74Hm33LN0wyQ3wMPGSvDN9h
+W2SzJ79e5Q+3PgdIFGGxJbkK8BzUhbowDBEnfiQDGlfGXIBPugya3AI3yRqqdAfE
+7MARckbMNYz6TYRtEMX9Sbnm7IaGfJw5FZ1+o5up5DpbWzgggQqPTLcwyCFqkZrX
++qab0dPvvSH4vz3G6QkVxXFpkZySQ7TAy0PbXswdVjyHvvOQcunSpdk7CetmRGUF
+k9eIw10CgYEAxS9P41YMMpmgvIFQVaW+vaxvE3WX34zSmiLJtkwbEijBICMQHcX8
+ugioPvuL14XaAY6Iya1JdB0Brz0XYYS0RuOITaxCFa9tnqLfB6YPbe/3sI20rD14
+FAwUs9KzuugdWIHi81uREbhfshZWHwYTlrNMCV9bPlRrLZ4OxxsjjjMCgYEA2IMe
+6JzRl3P9TbkqaMRqSAmmqEvm+sxfV/FSU0d7fO7LPNo1t0ZX4l3XjlXxb1+zM3fp
+FO1KNSt23DnwwoTecwyaSMH03T4S0vgdXEEs2aBDJhEm8ogwUEvzZlbsGlu9fAc3
+GffpvUGug2zrDkSKCYrHfmRfIO3knoG0JJKhDN8CgYBiduu3pAJXSivftds9yZRj
+9rVIzFHYieOooHKFm59xA/TcbWXg2DsJUsVhHg+IwghI+roHq4agetOahDEmuzOi
+qoAn+TrOxVlB/CbPFPdjeI8BFOIBmHhb2mSEH1Usojf1OHbx+umIJb9zmH/M4iXp
+/QKdM4bGqzUSBHlwp3arzQKBgAlA6WRFt4uCkJPu0woxQt2bU/0jaVbUv0px7PPU
+PUnkptkN9nfD0OPC4QCbWbtJ2s3bVSHo1mAZYHYfH9Qge6VtJ3bumzmNep57rj2Q
+J4uRlu0clI6PFYmJBXkdQ/AIdbvETy2T8/B7yyqg0BcHBONNhCM5xPsST9AaeQmL
+Hsj/AoGBAIYMVeBFz2q0adylh4JBuHWbZ5IEzhEJqDo5EQdVxUlftgCVqaVKPSrc
+YHZYB62Xoz45WTY03g9WugSlfSyS9L9GPTSzIdLo7Wa3fn1AKvPC8/y8vCj+4rMK
+MHlF10cclTiyQdBSYel/OvtxFzWCPEGI6wxv3rPzAG9ViysJ2SXn
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c4.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c4.crt
index 1cd6fc2f321e..9b07a9754ad3 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c4.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c4.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVANfDmHfIieWZe2vAwvx+WCiN/UuUMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjguYzQwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAIhWp/YCB3cVIrTK0QP6LC5zBZ8avn9isi/QeLXwxw69k+qhHEOReoyb
-nPgXyr2wyXLuL6PwyBMfjINDIEZ+N+4TsbYMsE8HiPNYQyZm1mpBI7ux2FbgCbpu
-JRfKA/Pztz4arlBXcHG/tJKistvF+SSrw21t2bLtVZHXBFR0c0NzAgMBAAGjfzB9
-MB0GA1UdDgQWBBSapaWx6oupzaeRg6DMkAo0HwOYrjAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlOC5jbHVz
-dGVyNC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AGkVwo1x2FNSMDj6bUxzyCN/MvAIKSe4oRXBlv1ZAWZjG+DRmJk+QpPmkxgtooue
-6TKRTYARolTc+gJtG1MqyWoJ3QSekLvVg2OAKk4qq+ijIL7zHaKTECPlzjJppddG
-+/OoCh8HDLnOcbC9UU36+3M6cJARFTv0mJZob0kaG5mo67eJYnknJzz8Cvq3r0bg
-xk4CE0LR9+evM/wMz/M9m+jB7HdgRcLoTU2xh0v6coVnJPYa00JL5YfRJuy6MCJs
-LHveTDYh7HMdxXwnHhr5FkdW06e08OJzzNEc10nTC+zCpgaGCTJxFM0mZHRskqMH
-QoDJUQE0/8szz0KfXFb2Vqc=
+MIIDUDCCAjigAwIBAgIUMP7z94WOPxFCwz4QHTE1TxRbSwUwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE5WhcNNDIxMTIx
+MjEyMzE5WjAQMQ4wDAYDVQQDEwVuOC5jNDCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAMGX8ioBo+vf/fyopLs96819Xpr3llStyAufqsiJo8c32cIR2Ulw
+uoXksKRI+cADl9nqk0oL70XApIC9FF8fCwwDA9KAVMHr41BTzmcYObEHIg27UMKQ
+ZbWrHOCI0S28LZ1LS24KO5PPOZ2zr7s54O4I3CyWUNExHabvusDOcG50GxD1FneC
+e4ZRaLCpRFdzdhwN0MHvpXBvdDlyodE3DZ/P2FdUvBLcNlwA1w0zUUL9anByhD/a
+XcuDd8H9isnsFfLmhLAoZHhY+ZvBeDmdsVRDaOQAA9//bk8G4LigcHss3XCUy5Ps
+p/aU3FBRmEHtlB3bNZYxm+ARrpiA/zCm45UCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+zBU28yVxkwV8wCPE+vC7vzQxlMcwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRYIcbm9kZTguY2x1c3RlcjQuZWxhc3RpY3NlYXJjaKAl
+BgNVBAOgHgwcbm9kZTguY2x1c3RlcjQuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQCaMPliOGCffwkT2tjztg0frDi5u7qSqggVc5/a
+6nnOUC96s72CtmVIXO9iXA5LX2QJR2FU4U+1HC4hrfTHLCO0T2kjqkb6UT6fnDY8
+hniHUe90BiGzOIBlQFF/rHU31N7R4KCcRg2MLXLDuRtGTSmyYyFxMoRS3TIL1M+P
+xawNm5eThjg2HPTQivUJLHEQGrGCjONzCA3wyLo5jzZtpFQAosDyNUb5D+m7b7ko
+ssTs0xG542V0Lzw8CkOGVF+fCN41fxH/SvWsuuXRSp8hU+A0sO8y6vyF7ampa1xm
+eXeO7a0U2czhiBNQAtqRdbRv2weEgWtCIBmo/xmB9brsX85P
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c4.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c4.key
new file mode 100644
index 000000000000..f9ad911aa4bf
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c4.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAwZfyKgGj69/9/Kikuz3rzX1emveWVK3IC5+qyImjxzfZwhHZ
+SXC6heSwpEj5wAOX2eqTSgvvRcCkgL0UXx8LDAMD0oBUwevjUFPOZxg5sQciDbtQ
+wpBltasc4IjRLbwtnUtLbgo7k885nbOvuzng7gjcLJZQ0TEdpu+6wM5wbnQbEPUW
+d4J7hlFosKlEV3N2HA3Qwe+lcG90OXKh0TcNn8/YV1S8Etw2XADXDTNRQv1qcHKE
+P9pdy4N3wf2KyewV8uaEsChkeFj5m8F4OZ2xVENo5AAD3/9uTwbguKBweyzdcJTL
+k+yn9pTcUFGYQe2UHds1ljGb4BGumID/MKbjlQIDAQABAoIBAC0/rJRw6bV6F010
+xXwWTwUqpnAmKncEX1uIKEzTDvrDbRxVO6pUtOQSvc7IV8evj1CZ6W9HNtfg3RJS
+H27UPKrEEaKj6CujTPA7hmTGXsWn2yfGfmvbgEpkAk3g7LN55ld7f9fqgF5d8yOP
+GiXys0Yvv/aT87PRfRLFuarTFahbBsqFa25iERaGjj5xukAxA5UuLrO30WIilC3v
+Yx+uswTZDKxLViol7OKEkjq7rz6HAF7qrt5cYMKTnOc11wprLCa//s0MUUhTxYdP
+esFYsCaQk5jlkY6/c1JxXroQVloppxsphCuDnya2dJt8XVZuM6d2R3U9B3lxb5He
+3l7fTUkCgYEA1guPRcDKNeoDEcvOw6rnSh8Wi6hJzIA3pJWjFPV2hmA+HC6uVhwC
+ByUHECcJYtMZViyPNU4HnGWFyXclTUHXMmcgXd1x7hoJyCzZDQ4s/ozCkuTcLehy
+eaqJStHFpdLY98JXI1Sc+Y9VqFNNokirVxSpAt1bkkryOultUA6EDO8CgYEA54oo
+P2t0qZuCgKh9K08BB0jYDZ3mmyuUugFoQAfPDmzs7PhIxJkpSJOjXI2fhRy54dQh
+pipshh7SzjuOoolqPv9SJFUM7TdI+n+R6hHQEUiT0YbKnhvmXZkUx4wFmgyzvPSo
+Bp8cKaJ7aZG0snFC30BywmO6hFf5TB7gEoJnn7sCgYEAwQ0aArqYZroSXKmqahYp
+l1OkvP9i2FxIU1xXA19EsHv7aOW9fZKzTYha73mV4Jku/+cenuAqwydZ3FRmw2B/
+3EjhC2fgCvVXH0E73qeLc2a3U0+xOoKtRQjH3UCH+EDMSeHxP79f+UIK6s9/dMAA
+MYFD0n6dybto8DN60TrWZOcCgYBpHb7h8+3V5mTb9iJFXCuyW+7CV9JO1f1CaMsk
+bqcg/HCfsKSQpJwqd+Rta1ple0IkdUMFhjMYvLulp0I9JjeW/CEVn4sr0rKODwZ2
+hrm4bwO01wX7u9zc11eotDgC1QtwmH4D645ElKCEsAKiFGtwoXi4KcCOvsXOOiIu
+JQfYawKBgQColuiDPPQ0nJQBWP3jMeQ7M8R9UI0Y7XqNRvFkzf61WumpWPnu8xPb
+lt9YrTPkoSBqrGdzvw2K2f9DwtDorgtURSkOZG0PpqWAIn7IDt8JmxqZ4+WMFIXG
+F6QQrGobV4I4V2ZJ2FyRf+9DDQ1JFDdGWrSEGELbIvorbUkbMXjz6g==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c5.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c5.crt
index b80546ca1cae..2b6583ef9069 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c5.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c5.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAPqm3dwF6v7a1hqPfWXePuwO/oYGMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjguYzUwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAJHAR+FS1lWCkwSuqgxJOTy58WVcjIkF2R0mCs2jmCTjcptZV2JL6VyA
-3G88MjchazG5Ebtjl+4CbwyfUV3GR15sCTFq2EiGM9sCzuemScmnv+dR4u2P9zr9
-EKSK/CcNBFbf9VGT4WTt5ebYj6eIVjzxSTt7OuIFCz036nL975DxAgMBAAGjfzB9
-MB0GA1UdDgQWBBQrezQ7l6dt6YqmK7niD66JMi+bqjAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlOC5jbHVz
-dGVyNS5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AFEcqSWmXLaIGLekW9XTyVwTAbU7be8A271Rp8IIfWFNrnIEXz/Cp6JChCpvCRxa
-GjHaDFimp+VRGkWm62Ha1Acbbf+ln8YHFKnaq2adRCZW2pF5wuNMqtHlMS8B+2W3
-ZZ9fMXrd7kXxlzWpHGH/mBLASfMIac+wjgctbZZS0cJx6VbZmH+XSNlqJ3C2tSYa
-7CaJHZUVClXL/A5m4u2LyDwqMQfixfon4/9U/NIwbO/pUWZ/wdqq1BA56Hd5DjVr
-C4S5MFB6N7jETb6NtHOHtezewOxAofSMIv4e2N42r7ydGI0OKEu79LEVe8pyXqlP
-W0zPUJ5D9UaWxdAo81onXzI=
+MIIDUTCCAjmgAwIBAgIVAI1BC3zd4ooAZPipb/bZxsw8FPh8MA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxOVoXDTQyMTEy
+MTIxMjMxOVowEDEOMAwGA1UEAxMFbjguYzUwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQC8tusYf2XErNSFHOJsYmEvxyTqHpHbxx86SleofuBgYNBMdBLO
+P2vKBwAoNwfP617tGGlincHC3tStQAs8dzwETJSq72OVjTISG3Y1E4hQCbFeNpfh
+4A1pZNtejXMkBkeA1m9ro41Gffgt/sVnR8yg3j9Gw4Mt7wh1ppuvXd0HxWKnfJa2
+GHeQ/EFOqAoEmdihCKFw31tmLqJ8aeFSy56mL8DWPmqHOWyQkBn6iZ+KRDa5GNUX
+8vRp5PuPyI8wYfYqAL9tSenjC6CmCvnrtOHKVyW2AryAEdsUxYwElT4fA7OtTE/d
+Rz+0duHZvyUo9eYq3qsNkkTzg/2314j/WDP9AgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FMRGtEPGaO03ND11stIioToFgGSoMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWCHG5vZGU4LmNsdXN0ZXI1LmVsYXN0aWNzZWFyY2ig
+JQYDVQQDoB4MHG5vZGU4LmNsdXN0ZXI1LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEAOwzjwhLdzS+wcNicpI0bDrmbL9Zs05DFBbdy
+AxT1Qs0JIsxs7nPFPrltvKg3h384Fwj2M6bThGSIRpPp4M5NdkyvA59tHQNX4fea
+YVfnaN5+Jymn8dyq3SA10F//y1lNBk8qvQhZXHE5KaMcJNrbSQpSfTPs8MTEPyJV
+fPCYP8FXAGxJvyhzFgo+CBTkpuWpWzQmgdczGcoblGNZtoINLYgpa585b7lsDzYU
+CUDS10Py2hKY/IMuRqWZ7EAG1l/i7wY/NMwxxZapbXzoqo0zJcvADsDpr+bGopDO
+oZk6xnlUm/nq5lV/vqA7i8RiDqi8ZmSKOC3l3MKG15fOdwTCTA==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c5.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c5.key
new file mode 100644
index 000000000000..38755fab0be0
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c5.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAvLbrGH9lxKzUhRzibGJhL8ck6h6R28cfOkpXqH7gYGDQTHQS
+zj9rygcAKDcHz+te7RhpYp3Bwt7UrUALPHc8BEyUqu9jlY0yEht2NROIUAmxXjaX
+4eANaWTbXo1zJAZHgNZva6ONRn34Lf7FZ0fMoN4/RsODLe8Idaabr13dB8Vip3yW
+thh3kPxBTqgKBJnYoQihcN9bZi6ifGnhUsuepi/A1j5qhzlskJAZ+omfikQ2uRjV
+F/L0aeT7j8iPMGH2KgC/bUnp4wugpgr567ThylcltgK8gBHbFMWMBJU+HwOzrUxP
+3Uc/tHbh2b8lKPXmKt6rDZJE84P9t9eI/1gz/QIDAQABAoIBAA7SgTGp4/joKEMS
+Szh/G6oQhrY603MDN1ExQKRT3tTDFelDXqzDb3htR1ZhSCRNFOYjejeMM4lWTjcT
+Zi6IIPcneq/YYKaA8hNZA9ZUY+sBb+R5/kD4A6WpDc/l5UrJkXktno/CR1jnkQQv
+1BpyGcDy495vgdCsSypOfUDvauputHIgkuNq/WB36UjGoKs9sdxquOvPdauJxYlM
+T1XZAMCfFo9Vqcg3IZUKhDrXgj4nHbY6HV/TRLjt4tYMxsDfItfz5p1onSclaWWH
+LAVG+/DgB+JE7ZW/5mfZgIf7zrE52QBOgah91s/w8Ik0/fQvuivp9YcsEplI0h1Y
+ggvpBFMCgYEA2Ckb5jSQRrzNu6yvbD9g7Wtms3fn3wEgPASHicrGSeIkAsRRyL8l
+yij0NtvxrxReZ3GznXiIrMMLQEy/oK01kdeWczPsSSoxPix3BfUvZqcLybu+xgKu
+c3oxQcTz7ewps/HuIQrerW1slsnEzVgaTPeVf9ElxeH9ifNK7NCnJl8CgYEA337Z
+Y/FQDXIrM8NYpznnB02TJGDwcx6jLCcs4obikBdIIk97iJkO/pN36rX38cU6iVAB
+/sGTFVZNfxaXzJZTeKImVZuzfRNWvVcCmzE7GDnmPU1nQW2sZ6Qh72mDY+QltLof
+y3SpgebFCpOR2KPXLeUGTlDopBoIlVRFbD50KyMCgYEAtFPvK2f0aJ6/fu571fE1
+mjs8DUJdlIOq8YbRz5jzPNXENu6yT6dvDiyGN3HbOLTkYKXRMvBIDOAV9clRH+j/
+xLA2mdraJttlAzoy/SxjI979jYjriOLZyApLJ1P4LycST0Tn5Hbd945OxaiDXs89
++bGd7Wh8e42hZsEpY1BZ8T8CgYA4DOQQQJDHDn7vOGqfZdIe7sGcOgOTm3dot1p2
+ZUWkxXH9yNk+ZMRx7CIC5ygLpl8L2NjnPzzX+RDucogXcwv0ERZ0Gdxl3de/HHc2
+3J3pcXvgm+ztZ8f5JhysolWlU65xKMyYheWNAocgI+pL3qs0g1+VfI0buru0V7Ih
+d3hC6QKBgQCNdD52ua61G3DKWSI2H6ovkyIU7v1cB38oWuVg4HhtRPzAGcv01e1i
+cRpKEQI7liztUIoDYyq8UMDdMAjPg3dvrGRPG96xeLMbovuPsg7eqji/IDnn3pXi
+7/QEM6Tf//BTYcOZQGUhRT0/fBUp8mZVbM+lkRKW4Frj1QVTMcaSlA==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c6.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c6.crt
index 361a392effa4..8b757353b96a 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c6.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c6.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrTCCAZWgAwIBAgIVAKbC2vR+wY4bWUJqzy62moQV3NghMA0GCSqGSIb3DQEB
-CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkw
-NDEzMjIxNFowEDEOMAwGA1UEAxMFbjguYzYwgZ8wDQYJKoZIhvcNAQEBBQADgY0A
-MIGJAoGBAKPAUxo+4TTrLmVCZ1WoriuC80jKaYeDelj1F1AvT7eajep1MiDoQgjK
-Wl56n2JeJiTPU49Dbw8IdgK+Sqezv1CErVx8VTsb/ljquF0yzWTzjoMPKa/YtgQ8
-KfdrBMcna50VF/7uIVvvxbLIsN1qQQhog29CSXqwyw4RjAVRCVvLAgMBAAGjfzB9
-MB0GA1UdDgQWBBQ8LqFuicvzgfROGbcN315QOD464zAfBgNVHSMEGDAWgBSrRunq
-ZcqAopdSp1f7svo6+5eoXzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlOC5jbHVz
-dGVyNi5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEB
-AH8G0DRTzIWEbC9VZmnVxT6L9LHsc6z+9y8evNN5gRMAFMxfjaiTRjibqwM8+hjz
-+oaSmBzkzY0t4WcljtEgvkzvIXEEVVeY/MBdppXOKbAh1wMxPDzD2SCUu+ILITIe
-uCALZM8NucxqoOQeFaBXrtrn2y4mTzXVfoip3FZw1dP+M6sAEwfFdRBo051r5bzn
-ZEC2LgpT5eOPWwFaMTj4I/qq6Ce96e/+PoY5ivqo16STPcurYVHd4yEr30lyDEPq
-cc1jYEUNpVuklY735gfdPi9ec9y3ioLUCI025wLWlr0PKOTLhUOflkuZnC3hM6Jp
-R2Ebzpapg4YR64kp6W8X7lo=
+MIIDUTCCAjmgAwIBAgIVAORDwVyeob4Cwqck1NBejd+AsIMOMA0GCSqGSIb3DQEB
+CwUAMBMxETAPBgNVBAMMCGNlcnRBdXRoMB4XDTIyMTEyNjIxMjMxOVoXDTQyMTEy
+MTIxMjMxOVowEDEOMAwGA1UEAxMFbjguYzYwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQDLGEkkMrmyYmJs78VjjYlHCCCOFqcQTM9QSH4O2HieIK5fzZ51
+gTE0XM4Uzwc5LJ+7sJaVfqNVgy2ha2NijrBsUXrCynnN9zEHvTsGhISnFTC9BTNa
+2QBk0ZpajSVZhObB0mzCeBguSmqLwuiA674f2lFAA1n59cDwCpRHPsn1hQ8J8HQV
+VrzMEpIwV9mLXEBdEbrw0lYzCT0z1EghSnxTgPX3LnEBbQyI2rl8wntdzzf9Ku+N
+ZLDcBBkjekfAvHk3R2279BMG9YDhSk9QZGZ7Or70Ez0mjkTeX87wGoiuzRtkTXb+
+wrkGLULCNTm/RA1irHFMuqqdasJT2Vd2G4K3AgMBAAGjgZ4wgZswHQYDVR0OBBYE
+FO8OjgrcTrFlCq2hz718QKZ8E7stMB8GA1UdIwQYMBaAFKtG6eplyoCil1KnV/uy
++jr7l6hfME4GA1UdEQRHMEWCHG5vZGU4LmNsdXN0ZXI2LmVsYXN0aWNzZWFyY2ig
+JQYDVQQDoB4MHG5vZGU4LmNsdXN0ZXI2LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEAeeFm9bxWvvMIfR4GSQ9oyrnf1dqf7vZrjcx+
+2ZWU4mdxLLb+xc+aue6e5pZ8lX1teiekXm6IT+S7Edtbx13fPBkh1cx3R+boVhan
+vSmnRReJ4MfaRnsdiG9nwRgQ88MpjXyhHcsrxxupgqw6Qkw4QreOLAnehDJJ4o9r
+eg2TXaPWNMP3GNm5gzlnhzc44e+bgYYixEb7XeWEX9/oSY2sxq88uAiNG/nIvfQy
+idzWoV62cH6OLbylzCeuH5RJtqyKUaH9dxkG8iifrCyW/xP06FmL0ISiEGWNPPrC
+KDZCC6XKKtoFmjmuw8jeo2usgEeOGR1skWZLnaQoA4uOtfc59Q==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c6.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c6.key
new file mode 100644
index 000000000000..3cb600ffb244
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c6.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAyxhJJDK5smJibO/FY42JRwggjhanEEzPUEh+Dth4niCuX82e
+dYExNFzOFM8HOSyfu7CWlX6jVYMtoWtjYo6wbFF6wsp5zfcxB707BoSEpxUwvQUz
+WtkAZNGaWo0lWYTmwdJswngYLkpqi8LogOu+H9pRQANZ+fXA8AqURz7J9YUPCfB0
+FVa8zBKSMFfZi1xAXRG68NJWMwk9M9RIIUp8U4D19y5xAW0MiNq5fMJ7Xc83/Srv
+jWSw3AQZI3pHwLx5N0dtu/QTBvWA4UpPUGRmezq+9BM9Jo5E3l/O8BqIrs0bZE12
+/sK5Bi1CwjU5v0QNYqxxTLqqnWrCU9lXdhuCtwIDAQABAoIBAGH8nCFaVezCehFs
+F5SiD4cVXTugo0cCgA2QhD1eZMeKJgdz+c1goSBN3GH+xtHnfUO9fdGv0VcrHKNi
+YAeHp4gDQdLkhKMCbgX5XxaUbjn8x9CSx0XAJ+f8lte0fL41mG4QO8gR0DzZS5GV
+3r6AvvC4UPYtB8bJMqa9u++p1B/fkAATPq3974Aync4lLwdpkA4gkJZv5S1J6ZUX
+8iGOCAx+si2ZCJfMXauvBVlrq573V7XxT+tVpObw8NgCugVH10q1VIV+PPiV6cez
+di47sQw2Q/Mh+Mp8Olgp6FQ65Qeah9q1/GnJcIg35ZFofrFEfTO1dZHdscIrcHZh
+xG29EckCgYEA7UdWpo9DTiAW33eIRday10moKPNDnLuxqYxhpbsBaM/r5Gb0jvLv
+9RQWqDvSPHGyOul5+RLzO62ODQCE4h0UFK6MZdoQ1U2gSGyo5ATwQwSIf7E6nSlU
+gYkG++Zv06psMEBds8kQ5Irl/m5c/mwiyKmSFPsJH+qPv0DP27OApb8CgYEA2x58
+x5lx/y5ZL4xPTm3gFbcQu3KIVem3pe1lMVfeZRd/yUHcUBo6gx+7t0CASDpV7JYo
+jc3yllfWoCDwIEhUNQVPmoPtqEGPyTZAOW+zoqL324ct6WPqHgpzTJoloXLMkBRl
+prysPKi2D+GnhALLEl7pzARESisetTjaVRFsEQkCgYEAmJoJ44M98S/4Je51aR0k
+1gscKIITmOip4XZUkscLqAIZH0MLxCgAJamuzLH0i0VAMF1aQdNrNZVwR3JvBBFZ
+idecmiC+NMEA9bNUh5mZgDCZLS6pmMws+kZLhJR2lTYE8J9k5CzLPYELXDBUhJi9
+YjqUpgOxoaI0ovcm95Chi3cCgYEAuLA2zmo4mrSvOZOTGPElqkf2E3jTD08CmxnA
+bFm6uwyWicU4dE/XN8RcjXYRak+5MsPSBMwtB6mb3rZEj4FHcLU8RmmivGoeTqOq
+eH79Mz0XmvKSsSClKZL3Yf6fbzh60YpPr71hk3NEmZhCgNN15NMZTvvdr4IuzOLR
+rFgV9TkCgYEAkBBUE11TJKHaCDr5l5iJ1o7ptMPi4NT/lOWDAe/ys7WouT2/Gsu7
+I2zHBIEkVBATOiBc2LL8q/ji0M2yr/TdgXSoz2wJWzIq/Nrrx4mLcesZ+U3la5nj
+R7HD/7YavsGxBPMOsBpNvIz0WGKv9A5O2VkhaCo6TnXIg5cLJxBKoxQ=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c7.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c7.crt
index 7cfb81868ccb..bde366c585e8 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c7.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c7.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUTU/oQlfBcPG83Lt2OtobAowVER8wDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE1WhcNNDUwOTA0
-MTMyMjE1WjAQMQ4wDAYDVQQDEwVuOC5jNzCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAuA+oN7YU2z47w0L21xSoUQc1p7LlrHTbYZDmoCLKQqg0IUP5gvENBbzE
-0+rb+ckhpCHPKZSL/Up0+zXiAPDodeZJRc72O4u/jXc9DcHhpV+w4obT0mZQDEtt
-ygBDY+t0WJyFZZpgS250XhHMA60w0ZuP6fhAvsmQQDGDRmX702UCAwEAAaN/MH0w
-HQYDVR0OBBYEFHaUc5wM5Ve1XrEx3aYe73WSwSWeMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU4LmNsdXN0
-ZXI3LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-WSFyo3bRteiJe+iOL07z/2PTuB6LfM6klZTllhGZuKDuXM8teKluOoJ/AgkVEb6/
-uuqzkAHncMVy+BiYqmweR7KKT2siQgyHMl/tSMysn5I2YVL/sneIsdreGZkAUQCI
-bGCCPuWbCX8OJAayVDS0U0zSfwsAJoBiAjUDK6ZwzVdimX9WTCJcx/3eJgrdPese
-Hv3XoeTWocfWQeA3qSRb6YE6yOhuDvPgM4H2jsCe2gYeOYeNnD1BlP92T7epaxx7
-00kMsZgY30VRZ6EQ7Hkzghc/9VXLd1jFZ/ICC9KJ2mSjCl0in/5JJ5+uCSBybbII
-uBwjvpppSFUVLimeeIwCEw==
+MIIDUDCCAjigAwIBAgIUF9P+tf5HB68Yd9n7HzLLzw772OMwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzE5WhcNNDIxMTIx
+MjEyMzE5WjAQMQ4wDAYDVQQDEwVuOC5jNzCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAPQ6GN4QL6hINd7MjNW0oZxbD8QfBMTa/Pt2cBAQXEI/jtJfq5DB
+LcsZzazdlwbS01Kh5zfhHtn/Eat2FIRssjTYEA+LvC3CeYGvbZuIn85ZkBg5VH0T
+WYbz7yNCfHv2xOsmxCzVeWCElGNHA0Tt1NfpJxj8gTHkZHAcd5OEVo7lJvmyJQjr
+/LduCVSQwRRM+/Ha/J2wpdVWL5l6wM0hT45M4dilvwt3avWf/Xny+q35zZFQGUOr
+OPRju3zAuv9r0nKQSJCyQHSqs+sffBB8AL56jPWNPzZyKnPDgeFt2PQR1x/Dh1IB
+IdxRyqPtDdKnDgi2lSTp/QsrXjoQ5lPDoNsCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+DMml0S9PDSZEJDX26+niPflDYhYwHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRYIcbm9kZTguY2x1c3RlcjcuZWxhc3RpY3NlYXJjaKAl
+BgNVBAOgHgwcbm9kZTguY2x1c3RlcjcuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQBkNZbH+EJBklVCzsWUcfM+qyZwAGdtBMW02s/f
+wR1htso4FkE1W011LgT+ankjWIJAVzqM16n5tle3d97iq8owAoU7utcZXZUoY65u
+4lIWgd1bJMemaVq3SMDHpDZdcyXRMccyDlTHF5sD5aa1hXDOrRB0u+3bfaniAA3K
+S6QkBOZnlbAlSbKQ7Z44LrN3D2Qx8Gpcx6F7DROFLv32RDsUae9YRfsXlAVqqfs3
+HOyHhecSS5LAIvM2uDLqB0drZJaiQE/ISQC4YZeFBfmIbjzKvPPK0BSIEQuaG92M
+/FOHbxaNrKBl86euughfedJlMIA+7ggnufSxiiYWje4Ds8XD
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c7.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c7.key
new file mode 100644
index 000000000000..d8c55aaa4b13
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c7.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA9DoY3hAvqEg13syM1bShnFsPxB8ExNr8+3ZwEBBcQj+O0l+r
+kMEtyxnNrN2XBtLTUqHnN+Ee2f8Rq3YUhGyyNNgQD4u8LcJ5ga9tm4ifzlmQGDlU
+fRNZhvPvI0J8e/bE6ybELNV5YISUY0cDRO3U1+knGPyBMeRkcBx3k4RWjuUm+bIl
+COv8t24JVJDBFEz78dr8nbCl1VYvmXrAzSFPjkzh2KW/C3dq9Z/9efL6rfnNkVAZ
+Q6s49GO7fMC6/2vScpBIkLJAdKqz6x98EHwAvnqM9Y0/NnIqc8OB4W3Y9BHXH8OH
+UgEh3FHKo+0N0qcOCLaVJOn9CyteOhDmU8Og2wIDAQABAoIBADN2OH42V+x0RtGr
+NXPki3dIYuKXDKi0YkFUfBmOcPpOrTgcC6lFY3Jb/hBSYvv/maLcVb3bFGaoDh+c
+kocy5jwnWP8FmUHlLDhM8GIpIE9ZZPkNv4VZefBrXzUFUfjHD0hjk0vFMb3KQgBH
+WuN/3+0dBm2H62tJbnaHMRUm+hxCuJkvXOx0JcB0GCpyP3rxrudktWIujfqwlKs6
+alKPgp3gp6rBRLhOx2/OcKwgWwwVk4iq6Jrdq0sz4K3hucOOyQpH20Bcx9U3iag7
+MahpWAFTL5dpwoIHTMDKxnm0dOVVkjIN+dzw5o6w65aylZoDf82PHTUZX63KlI6E
+CCpBofkCgYEA/MpVsQS1J8DzCKQTmmV72FmbbOSrBaqQkTWi9i81UPf1y4KiQqrd
+FgUopF7+yjqpp/Dfb13MTkF6b2QB2VR72ysQ9TBCnMHVgZuGRUrjC23y1SaQz04b
+Zi+5trsQJeZezkotN0Qhv/a6pvO/Hx27ScSkO/LQqCJf0aPIUWwY/CkCgYEA91Pt
+kPxE9Dmn2AQwer1dwJdf+quXpW1lckXwjEiOwSePwck4D1qrtOPmVdKlYZPAkXnf
+eCMPO6gSnH2yJKr9+DR4VhalJ6k95roHFywFC58zo/+3Q6Ix4VKEEKPJLkBdPYHz
+MH6uM4NeDksPlusbBfj+lP7GbLyiBIagZuNs1WMCgYBmynvHzGtBbAMnO1jBSozu
+b5l00RL97dp1QYMvagjbEMkA5PDH/SSDZLufAwXGA86i/J5Ocay3EYcQrnDWB7WJ
+bFyr5uDfTy5bUntpx8bFCgNZSyJNOayIg9WXiT9Kuf9JEP6L34d9wH0rkCfiJjvX
+Bj/mJJlWFQwzec7l1Pff6QKBgQCbVBtkLfsEQt11nhPwUdN6a9c8b926aD0JCoGr
+KLp29mijdML/aZti7KoxBi0VXhzXgGnnYmVjuweNPqfNvZHo+tMEhg5NHU4iBH0F
+gYDdrqtY+PVYxZl8AD2u0Z48rPGh+mYH89dlxMAiS+PLjXYYRytj2ao2ijVakE5W
+r8gfLwKBgQCD7/WHRDpte9jsTh0VFGyabnZvFjhkQ4bIWaCkuYKefQ8fUxgBNqAC
+eIH3E6DFGGgOldz5+ak2BlQ0fw7Gvmf1v7RBTaf+SDdiJoGbZ13Zn2UDRfkD5svj
+h808VYsP+YFGm6FzhDFFf6aNrx8SdsFBE9pkrd3BoeIpRj3xj0YKEw==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c8.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c8.crt
index b85a2bc8ff35..423dbe572edf 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c8.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c8.crt
@@ -1,17 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICrDCCAZSgAwIBAgIUX5JaUQz9mUV4BlmMMbsDzYG8Z9UwDQYJKoZIhvcNAQEL
-BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMTgwNDE5MTMyMjE1WhcNNDUwOTA0
-MTMyMjE1WjAQMQ4wDAYDVQQDEwVuOC5jODCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
-gYkCgYEAwvRhMUXpnmopSSajgAAcImV5vnRLUB88duRl/j6oM0Oq4xxPoKg1HfSj
-TnJuLd3mrMuTQPU67KodJoZ1keFoeKQGH1S6pDT+GBI+8IcZMZRJXyeourDB5Akl
-qRVKPiciepoVCHuePYZKVlsUnWbH1Vg9yMcYW6Jx6zbnS/meIC0CAwEAAaN/MH0w
-HQYDVR0OBBYEFO5ttLIFOaMQELSrMcKok9nD4PqyMB8GA1UdIwQYMBaAFKtG6epl
-yoCil1KnV/uy+jr7l6hfMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU4LmNsdXN0
-ZXI4LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEA
-g2NQZrSAadjMDCpeTeSTUJFP9SL81fOJuxK8X3VCTZUtFJAUJTKgrjAZWkDXWcYf
-47lzAoHuCI07/AVGkjWepQTzW5LdZWOdc9osIKRCnb9dofIrWrMYoa+1BVOXvasM
-aErtrlFze3z6WAznQPjQ+R/phpwZSJksAGJ9VPG7AFla0k2oCjOvu8FJUHev+ft0
-9D/ze7TXrIKOdev75iO+F+xUZJnUzDk+rJeWQYC2onyHMUA/oskpkT9GqV4eIb3d
-sJkJVbBH0V9iMahX+9FeYf12colMBUgK0lactFMTrqzP+v3v3vY39UBlDhR3x5JJ
-3pPRrvAcnxZ6pobwgNS6HA==
+MIIDUDCCAjigAwIBAgIUceWfs3lGqwxm9derhIa59A3ssmUwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIY2VydEF1dGgwHhcNMjIxMTI2MjEyMzIwWhcNNDIxMTIx
+MjEyMzIwWjAQMQ4wDAYDVQQDEwVuOC5jODCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAMC9+zC4FlWRP5eEix6jRVO0+QW4AsHQGjaT//KvtygjU9FcE2g8
+lFDTESry9Yiu9wBR2KVR9nYkxNj7KvT1hnbhkgMdiIPEqLQyvuwFSX0BYtclWLJG
+rzNux9QcBuF75ei8kGupH8fJUowY3ydHT1fcj7Ti/8jtV1Xn9OfQbpcsiENaMKeN
+XAkvoFAmwsGK7292tBdgnutJ+/ltr6mJeYk0WV2lUgkn3WxeEw/572mZUimebLxO
+GfwJko29JGQ2izGPVLa17s7mr47roRJ0S5zOsTrG8FhjXrqy+bfr4gaVEuzC9q5r
+ZFH0Kzebzv+KP8twvY02d3i9YgqJo1IUh90CAwEAAaOBnjCBmzAdBgNVHQ4EFgQU
+1LOn90joG1Q7EP0SpFuI6/y9JZ0wHwYDVR0jBBgwFoAUq0bp6mXKgKKXUqdX+7L6
+OvuXqF8wTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTguY2x1c3RlcjguZWxhc3Rp
+Y3NlYXJjaIIcbm9kZTguY2x1c3RlcjguZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAA
+MA0GCSqGSIb3DQEBCwUAA4IBAQBMJqKwX1e+pMKe+CWgiZhV2J/ayMF8haK7u769
+V/UEZSNRebpbLqu6jlwkCFyskiei7igTz5r2I7x0tCTGIbf2qYNM0vPoFfRDS8h7
+DEnloaARBO4G2QWBccpT/ouU0BEUkjqFaYBuK71kVaUbCJEzE4XbLSPluForyeBH
+nKqasqdxigxztgI2Ou3j1qfj6u2M1zBtYO4l5DD/MNxtYJQs+vpYqtNQyRUnFs8w
+Yk2STZNEtZIIbSEsJJo9HVP6yycQHMK+ORczKmT/zoo6fgmsyRR2tkBfadj9aFaM
+YiQzbOunQf175TPv5YVTCH59GA+nofrKJ4iAWo1Q0qizdLUu
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c8.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c8.key
new file mode 100644
index 000000000000..4f046382a60f
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed/n8.c8.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAwL37MLgWVZE/l4SLHqNFU7T5BbgCwdAaNpP/8q+3KCNT0VwT
+aDyUUNMRKvL1iK73AFHYpVH2diTE2Psq9PWGduGSAx2Ig8SotDK+7AVJfQFi1yVY
+skavM27H1BwG4Xvl6LyQa6kfx8lSjBjfJ0dPV9yPtOL/yO1XVef059BulyyIQ1ow
+p41cCS+gUCbCwYrvb3a0F2Ce60n7+W2vqYl5iTRZXaVSCSfdbF4TD/nvaZlSKZ5s
+vE4Z/AmSjb0kZDaLMY9UtrXuzuavjuuhEnRLnM6xOsbwWGNeurL5t+viBpUS7ML2
+rmtkUfQrN5vO/4o/y3C9jTZ3eL1iComjUhSH3QIDAQABAoIBAFIz9KDnsmbWJugJ
+WIg9nEDCK3775vkgNz3Vy1CHDreosYPWIFTRmXQLQQQvUPb1oFTbk2mW1mW28tmT
+7dNBsBKdy/dWLjX8tvL7vZ/N9XTCFZSq0fRe3zQxaDqXpXufqwOjAHlaTTqZK4Ki
+zPHkoPtZD1VXhz37s8bowfPoWnJQhll0LItHSJP6fVBWxc39plgw5Q/ETmL467dt
+zHiCKzC6m1BTikfqm0CGUYC4rHF1wKIsQG9CnPMa+nEhO641lRPy+TlEp5HjTOjD
+QBQcXpx3xW/zofNeRnJPCO8ZJGEvNydejWH49fkReznUxnsrxVe9KshDt6jpnlNJ
+2z4UI0UCgYEA9tJyWPg4NXUvkecy3Bvdn8pbGEZCjxYsQsU7bBvZo0NKm2z1xSrC
+DWr1E28ZX9qFP0MnGxBHJgE6zAJg5ZED7GYHMmii883Q9UzhXQ7Ela1wPemIcxGB
+GFaqGivpMLyi2gdgBpXT7++riUC+X2QEwEhHC06c924aJYCyYk9Z3a8CgYEAx+i8
+VnxKrvOXRE1rmzUlksfyMJsmYG/xuh3YDL5vCmgKebUEVnFFR1OGA348lnamXist
+Vk+Tsho5LbDVEjeYU7utYbEAGKo2ywJNTQ9gzzcNUml6VZwYmRcTydR5lGO7bkJv
+LSmmxycKNlEr7LyXhvjgmoThXIkzpHcU7yYuAjMCgYB9JTT/MVPmnvUwqCYcfjB/
+zW0kwLMMs7RbYn9/kBtocT0J01RF1Skw/nMmmGUoObbn3ZedXEKG3Ya/W9FfdIil
+uxGZGH7O2SNCzOlTcqJwNdE8QwWBOnXXLlHvFSaWJH75x1WD4UHlXQL/g4lC+oaC
+K+OBsylZN0UgkMFhoVQyVwKBgQCn0RCJ6b0oMQUDKZrzqCHqnj8J3rTQsCjUfMkp
+qZO9wKlKklMZ3eyye8M8m3RxOxleZBCjLBpooOrvGEtGB7WwVRSvtc7G5d0pfr01
+u3sheK2OSU2RIIDlwyGxhNSrZS2q2cpKt3RyKqu//VRKX+c0PykNjg9Bh0v6/Jxu
+RjbJUQKBgQDPeTNf4e1cMs7CpNB1NmvLyWRlbyRvOGtmtprNmavu7Y4x6ZXtlSMS
+u5Abfd6uyrt1ViKS0bOz6yQik2F60VdWFX9RoyXUIvtVX6kVnET7m5kwymmtsRYx
+P3e2FLa8R0zjoj7XXby5WMHjGIDChuhJ8+9CKtAUwoQAWL1U7T5OJw==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/readme.md b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/readme.md
new file mode 100644
index 000000000000..c9ef227a2e85
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/readme.md
@@ -0,0 +1,58 @@
+# Create the nodes certificates
+
+cd <your_source_root>
+export SOURCE_ROOT=$PWD
+./gradlew localDistro
+cd $SOURCE_ROOT/build/distribution/local/elasticsearch-<version>-SNAPSHOT
+
+### Create instance.yml
+
+```bash
+rm instances.yml
+echo 'instances:'                                   >> instances.yml
+for n in {1..8}
+do
+for c in {1..8}
+do
+echo "  - name: \"n$n.c$c\""                        >> instances.yml
+echo "    cn:"                                      >> instances.yml
+echo "      - \"node$n.cluster$c.elasticsearch\""   >> instances.yml
+echo "    dns: "                                    >> instances.yml
+echo "      - \"node$n.cluster$c.elasticsearch\""   >> instances.yml
+done
+done
+cat instances.yml
+```
+
+### Create the self signed certificates
+
+```bash
+rm -rf /tmp/certs; mkdir /tmp/certs; rm -rf local-self
+bin/elasticsearch-certutil cert --pem --silent --in instances.yml --out /tmp/certs/self.zip --days 7300 --self-signed
+unzip /tmp/certs/self.zip -d ./local-self
+cp -r ./local-self/n*/*.crt $SOURCE_ROOT/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed
+cp -r ./local-self/n*/*.key $SOURCE_ROOT/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed
+```
+
+### Create the ca signed certificates
+
+```bash
+
+rm -rf /tmp/certs; mkdir /tmp/certs; rm -rf local-ca
+cp $SOURCE_ROOT/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca.crt .
+cp $SOURCE_ROOT/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca.key .
+bin/elasticsearch-certutil cert --pem --silent --in instances.yml --out /tmp/certs/ca.zip --days 7300 --ca-key ca.key --ca-cert ca.crt
+unzip /tmp/certs/ca.zip -d ./local-ca
+cp -r ./local-ca/n*/*.crt $SOURCE_ROOT/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed
+cp -r ./local-ca/n*/*.key $SOURCE_ROOT/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/ca-signed
+```
+
+### Read the certificates
+
+```bash
+export CERT_PATH=$SOURCE_ROOT/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes
+export CERT=ca-signed/n1.c1.crt
+openssl x509 -in $CERT_PATH/$CERT -text
+openssl asn1parse -in $CERT_PATH/$CERT
+openssl asn1parse -in $CERT_PATH/$CERT -strparse 492 # location for SAN OCTET STRING
+```
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/restricted.trust.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/restricted.trust.crt
new file mode 100644
index 000000000000..344c186e0662
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/restricted.trust.crt
@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDZTCCAk2gAwIBAgIUJYSa8ZSlu1fwxC/oYnmjO2KpFUcwDQYJKoZIhvcNAQEL
+BQAwGzEZMBcGA1UEAxMQcmVzdHJpY3RlZC50cnVzdDAeFw0yMjExMjgyMjUzMDla
+Fw00MjExMjMyMjUzMDlaMBsxGTAXBgNVBAMTEHJlc3RyaWN0ZWQudHJ1c3QwggEi
+MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDBrbfvxy+OqGW0XjRpfyPJDB9I
+aNozRT3DCioIBNlKyawh7/rdGXyRsenq2c6uB+J2+fCamEHUCGZWKuCdxhewV8rM
+rLSBRwTOX0q+WjPVe2GN79sHmJHYgkNivdbURgePmL/4U1j+gbwDLGQy1aOMnay9
+TxpRnMbUTsBz8lj5OfA+F9eoSTJjBbf2S/zeQsQMydG/51Mrj2VYfz3dbJjHAxNr
+PlEP8OvM/y9N8qMdMUE9YkaAw2gJKaPqmj/2nucwxrHRuioKTAatW2F3T/5AWJEB
+262WxpZpSqAErR4LXqaUVBdOXwB+DwQKG/uD7gb9QRhuIx0yC4v5MC98mkHJAgMB
+AAGjgaAwgZ0wHQYDVR0OBBYEFPSCxeMJ4lTTDuQ/k3cz6Poc8o96MB8GA1UdIwQY
+MBaAFPSCxeMJ4lTTDuQ/k3cz6Poc8o96MFAGA1UdEQRJMEeCEnNlYXJjaC5leGFt
+cGxlLmNvbYcEMmSWyKArBgNVBAOgJAwiaW5zdGFuY2UwMy5jbHVzdGVyMDIuZWxh
+c3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0GCSqGSIb3DQEBCwUAA4IBAQAq3ZPsnZ8k
+2IdQEBHhbIPZKnK02sEMo7gK5sC4LbXnQGCks5wu9KUMpLNI+t4Q41j8+P6kuwZ9
+9XumWKUo2veMssxzlupwqQDX5/oFNKc3D5OAxTq7scs9qOW4oP7u8Scg0inqtJw1
+nPARV6mwVqzuz9dWJxaCugo8NuZ88d9fUuI4WRKixffTuMlRknHKa8DVL0NhKE5Z
+W0TXtRg0VewaSEotKC5CqxI2UUOo7A2x0wteW8PHjz+He14rdDXA+Mw8RGHfjNXf
+vZigDHFcEWYe+twHFYo91w1xNdu0oZFUCwvJnIdUS5el1CncBgKxHo0+M3DyqLX+
+5ZYk1NOCQkpA
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c1.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c1.crt
index bca151e28b18..c4f91b15c6d0 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c1.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c1.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVALishM0WVKx4AixPiFxwW8AT9G75MA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW4xLmMxMB4XDTE4MDQxOTEzMjIxM1oXDTQ1MDkwNDEz
-MjIxM1owEDEOMAwGA1UEAxMFbjEuYzEwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAM/q6yVK17PHtdsO5pM6DNU6pnOY/FQO+c1JpD2cpOk6B8yokEtFR+a0Qsds
-qQewAGBG77u9jQVerJr6fkPW+AeJT7eEBl5rqYDx82XgeJS6dAJRvclrxsOLBDDW
-sImDIMes0AZaE54P6LDGBooH3XhidTyFj2Gp9fozVY8PWFl5AgMBAAGjfzB9MB0G
-A1UdDgQWBBRLyFFKNa/ll8mohWv8TJyzCzyOmjAfBgNVHSMEGDAWgBRLyFFKNa/l
-l8mohWv8TJyzCzyOmjAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMS5jbHVzdGVy
-MS5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAv/Gm
-maKqfIIGPLWfRv4/GQRssogAOHdqlZcYojxQ6Ovo6ped9qxscYXGb7+8sP7vIeR+
-ibXEoFvbObfM4EyRONo414PTyZnpsg/Jqe3GAwnd4UNrXR1LLcFyxaUDoDvAtKkh
-nI0iE+0yGwrH8UpMuLXyqqp/ivTlTkMBdFVvenA=
+MIIDTTCCAjWgAwIBAgIUBtD5eWhDMSkYinFLJGVs93evIBYwDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjEuYzEwHhcNMjIxMTI2MjEyMjU2WhcNNDIxMTIxMjEy
+MjU2WjAQMQ4wDAYDVQQDEwVuMS5jMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAJz9boSX6TZSTf8SmP7/YGDwrTKqNjU3GS7ldaaUOtOTOBuBYHzc7WrO
+7DlQhNVqBV2euMyXITSNNqcIWpjNivS8oz6PExrEapvjzJyQH1J/zJZ2knW/08U8
+zeTq7Shh9bl7NJn4l2d5J9iDzHvWP8zxYzbNv70Z/12usEqrXdtoJPuvD7LEiyXa
+o/hBKDVQ7Mv2bQaxVJYnsMIEaX6gXCy3cH+9M9v9uGjhJPDvpPZV++d2RWNvvTdc
+bCMqA//15EmwKykjK4k0UWZjkISGNfAGXva9V6v6JQGBZshlsQv1CdBCtbC6LMnQ
+IeBQuaJLhZ1npDtGHBwcqep6GWSfL/MCAwEAAaOBnjCBmzAdBgNVHQ4EFgQUImFC
+TUp1Jq11LbYbLcGBdVXzQzQwHwYDVR0jBBgwFoAUImFCTUp1Jq11LbYbLcGBdVXz
+QzQwTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTEuY2x1c3RlcjEuZWxhc3RpY3Nl
+YXJjaIIcbm9kZTEuY2x1c3RlcjEuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQBcI0A1lAzDbo1J2Vg3uAmb3Takkq1yqBNCkDeetV8i
+Z9Ko9l+3VZpzgE1t+acUQOQwvSRTq4CCPP2Kbno1oN96PFmBv7vTKFyiI9R59Eww
+eJL5R7jNj2c4runmvnqRHEOKXbnfMYU7aXghE35YYJcbbiGyaW64+htAdT6zAbO0
+2/I4ZtGcx9MxfN9ZwaRFPKwFMufmyP+nfqMLw3fM1FWBtxjlOY07TKSvwi+HTosm
+Y9lWWxLBrkJKeS+XA3Dlj1nTcQnBtSVp8qQ6ac4eKmFaLN/Ig/WA6uT2/IWazXzR
+i7YduY2GO0tJrHAN/G3vVsOeyy6l6zGwkz+/p53u/96p
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c1.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c1.key
new file mode 100644
index 000000000000..da1100be6c0f
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c1.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAnP1uhJfpNlJN/xKY/v9gYPCtMqo2NTcZLuV1ppQ605M4G4Fg
+fNztas7sOVCE1WoFXZ64zJchNI02pwhamM2K9LyjPo8TGsRqm+PMnJAfUn/MlnaS
+db/TxTzN5OrtKGH1uXs0mfiXZ3kn2IPMe9Y/zPFjNs2/vRn/Xa6wSqtd22gk+68P
+ssSLJdqj+EEoNVDsy/ZtBrFUliewwgRpfqBcLLdwf70z2/24aOEk8O+k9lX753ZF
+Y2+9N1xsIyoD//XkSbArKSMriTRRZmOQhIY18AZe9r1Xq/olAYFmyGWxC/UJ0EK1
+sLosydAh4FC5okuFnWekO0YcHByp6noZZJ8v8wIDAQABAoIBADdDR0pJp4DFzEsE
+IvNwtl6pmm6BWH+3G5hUkevHtkWQR2n3Sx3pL3Iv0RwD6h6uKaFhJDBw6vFy3FBE
+MpAjmsy4Msbs2o7LobOqpvZeDYifEEfPCZOFqWh/FQOGZ2GVykg2xLeGuNVl4s7m
+5m6ZZvAf8QrztpE8u5YgzlIarPl0zEbXnlF6rY9YZPb5CyFnJg/u0AIZTpBjoB8N
+QG1wv+fiI7R4RUT+p9y3QoJK4MS45iUK6U1jehhrOyyzOuJy6GnS3qCHUSGdH2K9
+56i0qOKngBX1PjWgxkOOh4YTdgABOk3STBm0tjTh3Oyjs67cBItdGS7Bzd/JS/xu
+jY8kiwECgYEAuy6ZMOi/QMIqpsFpIfwrwuSax6XLWAYR2IQTGPSlkVTj8hzzN1+V
+eYVIJAMSXhnY7bNL8reBsyMNsQBRilnD0alhGAqAENcXVc5IlWAzT7H2pJ5yKOBJ
+nbGADMWdfIDMkvlEXxoNtnGT3j4TLO0c9HYWDw+EssqpAG0RqAbYY0ECgYEA1rUv
+M8FU/1p/+0Srkecun70A4D9OJ5lcK2/t0Fe4jXMGCE+GhX5Wv3CUVFYuBssowXAN
+quMBSWXlzjNxFMMMmdHf4ZGTtLUEiW7md5hkMUkhskAWlS+dfOZx7SVZK9PZipSK
+Pu4x9MMpfaQ7GtPbPNkXV2LlHQtUgVKuCoxK6jMCgYBoYUnn7kKImD40k3JPE1JL
+8jMl1hyYz3mwJaaIqhSSAKbEYsR2QoUWKO7btiLN+tfvZaaaiR54iLmf1K+j2P7L
+sgqE34Ye3fMTVF9BfU3fyTvi/MBobvpeR5dCiAiUXza1Vu97oQm4jYmJy0/iI6+b
+2yXd7CdxlHW92RSZgIe2gQKBgDSOQTGf/1hKJM2yr3SVbDo5n39Ev7K9vWv8h+/Z
+qRkmsLFUFg+TL1TJD+Xc+oM7M4y2CwJrcqMuyCPDP8jfPbRhRTYH1moDbUre1Jct
+vEi8J/1j5qM59ojeN8DexYcO4k8jsPtafuv04bEZhEY62Y15blc7o+2KGnNVc7it
+QqE1AoGAWeWK+I9g27UzLcQx0voBX5cMiM6i8SLfyfX7Ruok9dpwlvOuXHr77W7F
+9RB1r9ZxdTg1cUAsLctN7wbxs7uZTuYA2LtTa4KinDjKnWG2vWI0TSDGuTWccsBF
+EswgAhdEuEOpNCCNCokzmqYX88/WPwD39FBidy3K+R93iSXeOrE=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c2.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c2.crt
index 848b7c5ef185..3a6dcd481b33 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c2.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c2.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAK+lmh0imRaWbjXWB5mEQMqEUG5pMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW4xLmMyMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjEuYzIwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAJQtUzqzNCr2cWeNAQnn9hwkDhrBSPyFJhQlZ6Otkavw2Z0G6arPXsgYw3Sf
-ZTBE+Gq5KidVb4NK6HWVgoPEzlN/lq9cVdbtr+KAU3cMn6DK2cSltzbTBkEXa7Lk
-5H0Jt9qLquHifF3H+s4gsAKbqCCgz0WYqbvvSapssKsOOWtZAgMBAAGjfzB9MB0G
-A1UdDgQWBBRuBoJoTcJq4QacLWQDLb927B9j6zAfBgNVHSMEGDAWgBRuBoJoTcJq
-4QacLWQDLb927B9j6zAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMS5jbHVzdGVy
-Mi5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAVjwG
-V7GyCbpBNfXgjVNeO8EygBUjND27dO15a2ElwcNAG7mi+4f4ewIVlf06kzXLzFu7
-ig2rV8YiCDua3FctIWDhrWrvvk/xwoTw+o8z3RBE/4q8m/60BWQATHdmdq8jBBxw
-oFhMS1yKN5mnGdEkfPuq0t+wK29BPU6jVUsHJyg=
+MIIDTTCCAjWgAwIBAgIUV9sRC/CyQ3RlbBT1rkZbXzqtlxQwDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjEuYzIwHhcNMjIxMTI2MjEyMjU2WhcNNDIxMTIxMjEy
+MjU2WjAQMQ4wDAYDVQQDEwVuMS5jMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAMkSIfiyeCl7wpDKvW5alZ+nYO8yy4qS3btwtHnKoQsqbtoUL8TqVUpS
+4umNva+56qUKzItdbWwMFMpy96dsbqfFoy5Qx8aSSd3ZXAy5vK08l4fN3+IkPeGL
+dhyBpXLxbhtqSM5doLUZklfq0xuni2KRTWhGeJl05ZUxij1Wsz80OjopuDUKwxBa
+8Q57TIdnoLDqFBNxEK5jo2l2NQaDxsZTBP4hXhoPw97ujHulEOOXVQnibwfuDPtn
++FBTAh9kYsKeuG1H/2i6NxpdtxlGrZ5fAE3Bp0p7S3sK9xL8NsK8rVlgKVRKGjJq
+rPucPoBf0sy+xik0rP0Vw3w4yJhN3jkCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU6ZSh
++ZcljelRUKQDb2DNfZUoXOcwHwYDVR0jBBgwFoAU6ZSh+ZcljelRUKQDb2DNfZUo
+XOcwTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTEuY2x1c3RlcjIuZWxhc3RpY3Nl
+YXJjaIIcbm9kZTEuY2x1c3RlcjIuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQAZVr5t+GL9R861bBgV8Ns/dMzNFXzh9j7VVvvCLwKF
+lDcL1MmTFVwFdUA2eX3XhjxRoXs1cqF932DzEfqH2OrU3m97GMZjpZWor3adTl43
+JevnOQxkBQsaBwIHgTnuzHGqdSNFHXrG8qhJvtIPBZE87x9P6AWKoOCvDjNNtmCZ
+MQoZ2SsKtMlEkrMYRpidWPAojBu0KSpMadSCGMG7Qm1UtPj25l4FHfyr0Q5Ym/16
+fmz447yOcgSsJ45cb+BcrIF1eobVkVVuZAAdb/ELANAJcV+vSzvOeoVMgG+fOFUj
+F1aBQqTtydi9dnovBShipLYEQqU+A0h61KKfiFFxkH7P
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c2.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c2.key
new file mode 100644
index 000000000000..0df30bd8abb3
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c2.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAyRIh+LJ4KXvCkMq9blqVn6dg7zLLipLdu3C0ecqhCypu2hQv
+xOpVSlLi6Y29r7nqpQrMi11tbAwUynL3p2xup8WjLlDHxpJJ3dlcDLm8rTyXh83f
+4iQ94Yt2HIGlcvFuG2pIzl2gtRmSV+rTG6eLYpFNaEZ4mXTllTGKPVazPzQ6Oim4
+NQrDEFrxDntMh2egsOoUE3EQrmOjaXY1BoPGxlME/iFeGg/D3u6Me6UQ45dVCeJv
+B+4M+2f4UFMCH2Riwp64bUf/aLo3Gl23GUatnl8ATcGnSntLewr3Evw2wrytWWAp
+VEoaMmqs+5w+gF/SzL7GKTSs/RXDfDjImE3eOQIDAQABAoIBAAJOovUnlxliwijH
+KsVJZM4gLYyRGKTOapzr/iYnnRFzuzHlaWbpCXGA4fmsFrEdgQPDy/UNbWP//30T
+NKRHKj3ilqzBYIl2xEI3bEb5GqEiQNreEeOWZt9fqktJcYyQGKNX5N35VQ6+7F71
+4tY+ZesLn8fChz+7h6bemqd5hzUCbzOxdX1LuK7yDJ01OPUqeVolqolHRVpvLjPh
+dLvCzIcDC00d6aWIREpQ6Fm2FNVFajfyfjkhI51ASJ2zMI2BV40xvU+9kcoBys1s
++RrCtKCUymoEulyLHVsaCBhU+AyrCtFk4azDdRACFl7MQ8ImJmlCBqCIYPHzaU06
+xplRCwECgYEA4tfX5xUFJIUrYL+JzEliVSBb8HaeXqisJarH40sOoY8BKHqtB80U
+Dz7KDxal7Aq+uZsYcGP9ol9Qy+x+qvC+vOfPlvYNCvPHCzbJUZVb2lp6VgjHkPtm
+8KyAPgWhXT6UMPt8Ymx5sD978jYjXcdh06upOcKvzwmq5aKn2OBq8Q0CgYEA4upD
+zBoptiV9VKV3kn420ilmasfIRIFzMHKKtLjktPCoOQ3vT65f2d+8F7yt7/YJzVof
+CwYy4vzOXhkAgoUmplpenj0O3BcijjKig5A8vFvr0M1wbQZoIjj2kh6x3mffIPvz
+EpnPys/RUvsLEkiu08Gdwf9ePHoax9gtly0dXt0CgYEAtVnuB5YseFfmqAA5ABMj
+A/9MwV9zmSWsCqkQCMy2s76KJHx+3J+9i7qc+5cVqzGj7OrrjN7DpQ4ioAU/9xVk
+x81jjsLDBqCB3ps8M3b+6gAvkZyPP39desfN9se1i8A95TMI6/7hAmCejLNvbse5
+k+nG7qCQwSpvcAqruZ0gofkCgYEAloAABYAYyDlnPdwdSWky3QMebZYiCuW8y5Y8
+zw6Oij/Poz/NyJCD/FJ7Q1ceGjiaWbv7OOmcmJZKHXkBsI+1icIIXXVrxKgHgD6v
+MQlabyecaTnI5lbIfGJBj7qSCEvmXs8ccerYOdDKKfpdZnwUkFgrVP8VtpDZ8FA+
+NLV0dHUCgYEA2gvZSxYorAlIfKM9AgABN7qvbQJ0gIt7GrQmn7fGJK4LmKt/Ima4
+0+zL1w2WK3pR7CYS73Ij8w1mVJrqHunKq96/aRR3TMLiGAIRB1VkAIX59+U1S5J0
+DhFct+iEMetHo11OwDnx+LPpy091LEcYf7Yf1xIOUBqeuz8yXBM9xeg=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c3.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c3.crt
index b655bf344d89..0814f70c7aa3 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c3.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c3.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAIxnFnx8Ow1xmNEHSzoEFNU/as1+MA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW4xLmMzMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjEuYzMwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAIVyhLjNcWHszffIzSX7aQRkgEOIzNFuJ6z/OjYX5SoQgjZ6ej0oeudbeOPr
-/LT6NmcIWgQJHIa3w7Rygbk//+jyLXshG7bI/3o9js7UPVzuoePt2y+nHFYHzL5u
-mA9cuk40zPXwr6W7aqwTKLhDmJDzywOpcdtycluZidy8MJ6NAgMBAAGjfzB9MB0G
-A1UdDgQWBBShH89k+Y8GKljoG52TiS9pi5KrQDAfBgNVHSMEGDAWgBShH89k+Y8G
-KljoG52TiS9pi5KrQDAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMS5jbHVzdGVy
-My5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAb/80
-ShzdMSxzpLi3kQeOyisoFclkGcfdZAk+wluPJaZjXzu6KdaUV6J23pC5NJ1QP6+N
-2xStsISJt+Cf08S9Oxae4NCAD7mw18oxQveJNlzX6UF+/IilLOGYI1Xfc44BImfJ
-bRInBM8Yip1D3tWqZcPGS/wSlMq2IdyVm1tOPpI=
+MIIDTTCCAjWgAwIBAgIUczFiOGoNnJjSSvoHyZAEUBe2CGcwDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjEuYzMwHhcNMjIxMTI2MjEyMjU2WhcNNDIxMTIxMjEy
+MjU2WjAQMQ4wDAYDVQQDEwVuMS5jMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBALxS8oBTVBW6XRUbwMmaCUt9x8JvEeNimlhT3sxpb+UfXXAmdTlwmvnA
+iWit990KJE4DWNZTU6yI7k6ZbHgz8nUtFFdlikmLHK1QAB/9RxG+8SWU3VLnqm3S
+WnHzXrQ6RNYnudq1Qfm4875VHvO+KOmyVAuOCFtDXFwzr6YOfmXSJ8gFB3QPy6AK
+um7XqOzFGmzHMTW55IbXkGH/+lD3XuV9y/j55ef7kr/xfyEi1fX2YC6Xn1b7WfY0
+9jLJbeInlRXWkiGt35VH9YWlk98TCqA7PNxyI3Mwc4bnkfEjafAJPAK8WonPSlR8
+Nua6e1XEqMBeo96hcA8xuzI6vXAgcpUCAwEAAaOBnjCBmzAdBgNVHQ4EFgQUxO5g
+H5XCLBK0sP4Tum21Zp8IFpcwHwYDVR0jBBgwFoAUxO5gH5XCLBK0sP4Tum21Zp8I
+FpcwTgYDVR0RBEcwRYIcbm9kZTEuY2x1c3RlcjMuZWxhc3RpY3NlYXJjaKAlBgNV
+BAOgHgwcbm9kZTEuY2x1c3RlcjMuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQCFKO57CPesXV0wGekL2d2qMclsCqitHVMmLgOazu7f
+NSZ05opiwfKpbANwJmiUd5E+Svkwqxftn9/wJlhKSMGd7kgXx3MAS18U6whb1fEO
+7lUVj+jRAaFY8x62ghV/PCK0J/jrKCvrYW+cayRBAHo0tS4bSw/P6tn9S+4tFWOd
+Zpa2dfrG3q5wbJlMgpIqj/Rl3US/Z5ramUWY4IewcXES/Nv7M+w+P2DhuUz09moA
+NooXais2eLMvhILWSJVQy8yTi+M5+Xi1raXwpSt86s108KgwNv1md2/jRDwEzDgi
+QAqrOri+6VfPEeEi1mFJqzwYKuHFuiv2g0PUoGNzI9/V
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c3.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c3.key
new file mode 100644
index 000000000000..c58c24a261bd
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c3.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAvFLygFNUFbpdFRvAyZoJS33Hwm8R42KaWFPezGlv5R9dcCZ1
+OXCa+cCJaK333QokTgNY1lNTrIjuTplseDPydS0UV2WKSYscrVAAH/1HEb7xJZTd
+UueqbdJacfNetDpE1ie52rVB+bjzvlUe874o6bJUC44IW0NcXDOvpg5+ZdInyAUH
+dA/LoAq6bteo7MUabMcxNbnkhteQYf/6UPde5X3L+Pnl5/uSv/F/ISLV9fZgLpef
+VvtZ9jT2Mslt4ieVFdaSIa3flUf1haWT3xMKoDs83HIjczBzhueR8SNp8Ak8Arxa
+ic9KVHw25rp7VcSowF6j3qFwDzG7Mjq9cCBylQIDAQABAoIBABe4QNf95tzXvMqg
+VL9vudT60jLAp5mivTr70Vegvhm3Qh8SkPg7oSFs6UG4i7wY/2qeDE3ub98O3ikj
+NODM9jBhDatWXD0YGvQPW/yre4yWJqxrnBbJnBM6B+9LTnXX0vzAQy3zX4xzCaxC
+72OW43NP5TsRWCcajGMGlYyOTchIIO1X4kZj0hOdJxOgudjyWBLEXtHGvLnV1R50
+HoY7swwttb4+hG1hlUg9Ve/blu+ylQcj2twEmsbwQN5z+79ZwRG+JFApdGMiEkvu
+hs31TvomNdg1n7GtWsEXLeSNTmuC1ew0wNxti65HIulHtiOxmlRDthClx6ckwYYl
+B/KW5AECgYEA8eqADRBBdYrMKn3zDXJM4uLGMPt4fi/+uMbzjfsteVUk/qQXPyyE
+2AP5j0tn/yzFUqlAlg1gyc9VVZ5cDWj6UpRewllCSolGRwrzGFb9Jw+qvuqMAspX
+Jw+S5fhiTvG7scTgo2nbtIuTDjHTWhZO4ylACem9dCNf6tTUwjmO0gECgYEAx0m3
+K2ndvEYtKGjb4LnW6mu8kTHxLv5dMXk8joA9Mh/Kk727yUTSXnGkKgp/kDS+1sp0
++s7mgXc2eIEMaENKfg0xOTQXJZEKLnC/VIxYqan9VWm0YpWuWOd6Kt8NtZnLzCy/
+uUJVLkxxuHjHk4UH8+W8unqAREccPai6aZIQOJUCgYAmM36nEXMUyac16o+wBuWn
+SmI7p+o+GvwhV6FsQUbMCyr9XxIgsroDlpUZELJKdFpwlDOTvmcTVz9Fdu1YXMRy
+t+VC0W+Lb6P+YfdDV7FAMtQms8kKQ8OBco8i65cymi+mgc1rYLMGDqhZLHsQlcc9
+Te1D43o8Vb22yocSaJ5qAQKBgFq+tEtub2tVwSZPaR/RP42K9VhgF2TAqormm65/
+sl3qoAHaOXIShoA+2vPRx/hQLD39/npeUBPPxtvkV1P6oCb5ttpHX+rZOnufF2BW
+b+EB6LtF4L+rTLJq5PQ0kTeKCoS0M7EWBVeJs5a+WJJR1yc2C1QWh+WPQ8rH0+pS
+elRpAoGAG3iu+AKV1kHAk+CxL9Dx/L5awRx1ZsfMA201UYVrXpV5CM5AXzlt2URV
+ZsugHlA1oCOFbdFhjUCmeZiMikb0tCMZZHGv81G9n10BEtFor3vi1LNsI/+jXZSV
+GBHonPDTsCVHWEbkSLVQKwqG/fTJN4sIXuSsf5kVYjkCEOXBKgg=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c4.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c4.crt
index 3b7b3bdbf720..dded07f82572 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c4.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c4.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAMllmXfU0dH+YEE+7NFAwve6CVRiMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW4xLmM0MB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjEuYzQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAJVhOQxK4DAxYsFDcm42dl0DdpRCqxYFecP9QmC2cniv+sMUSRLsti8BeOKa
-wlWL+NQK7CJhAQPuYnLEBiyhUIFz9dAVozHyOqBKFbf2L3A2nIPuom22UiJN79k2
-YXZgagSCdCdRX8WTvTRbbN8WYRfE/wLO4SBxwW2f501ET6pTAgMBAAGjfzB9MB0G
-A1UdDgQWBBQIGCed24kXamJPxK4inO7BWteawjAfBgNVHSMEGDAWgBQIGCed24kX
-amJPxK4inO7BWteawjAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMS5jbHVzdGVy
-NC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAMNvY
-Z5SwaZ484mzdmNvseZs8cxZA7EXeuOw5r9jRyeeYAT8aJFBBXgAYx4h8in9JbBnP
-of/88YKTHuVYGmh9ad0wP7TruZQWF+siQZht2grar9sWo8XBO3FiUBHsEfCvSQKv
-ppkkIu7YiXoBdpdpx+6+lYW14L+fgB7BrtRODHo=
+MIIDTTCCAjWgAwIBAgIUU81CxPo5hUWGwt283nbyrHql0uswDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjEuYzQwHhcNMjIxMTI2MjEyMjU2WhcNNDIxMTIxMjEy
+MjU2WjAQMQ4wDAYDVQQDEwVuMS5jNDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAOG6Z3WA1GJ/J8SivSFSMBJrb8LqYWrlfPl8YQsFwBS4XqewQJ/WhPS6
+QmrtP2dKFeSx4CzoaYLHjUpgZiJT2w+12d36ct8gnCvVolnI9upykhY/OpHblcA6
+Uy6cyQGY0XPW2n4hhFdMZm+RxG3whlMhOVB5U7zqJ0A2l5VXvl2s10d/mzq8VuZB
+Jqjfpgjhs8hw+0CU6Ac0JuoIdO2ly774ltksbidGlvH4ONlMl9/scLJTwSx40sk/
+0rue45eRv8LQIXngMLZ546N2y6KtpxeUy4kk2nZ4h0xLo9xkvpb/Be96wICktVQL
+YF0Z0yVY12HYA+5GRCVqw5ir4IDYMlkCAwEAAaOBnjCBmzAdBgNVHQ4EFgQUlyZi
+9Z5kb9/9Pb1aE6VJhHL5GIMwHwYDVR0jBBgwFoAUlyZi9Z5kb9/9Pb1aE6VJhHL5
+GIMwTgYDVR0RBEcwRYIcbm9kZTEuY2x1c3RlcjQuZWxhc3RpY3NlYXJjaKAlBgNV
+BAOgHgwcbm9kZTEuY2x1c3RlcjQuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQCLgXZPcA73Ohm7gjuXIPg1zSs7iVbdeth0bulbi1/g
+zHfrpSnHU9+i66Qoft4D+Aw8Up2dwsQVfPAdpGQnQLcHB0c1WZtTNMu+o7/RMpGe
+cOgKDy8M7MegROrlZA7U199x3lFaJm8Cbe0/Tc/WwqXJ9uN7tatUj0iJPGoe2NeR
+T9Anq4qBBNOFhZ2332ItSh0B6dTnZ9vZdMRH5u+YbZBRGxmQ4xEhbLg9eDPeiGaB
+blvCvnoBcZ1iH0iZYFA4nSGPf9376dS4caHvlCx4MRwXVgCOV8jNyx8eGy/4NPMg
+/nDwIPvQ7wlitnAt/ixiMScT31SydRRvOXa6yEFGH9G+
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c4.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c4.key
new file mode 100644
index 000000000000..40778de1b613
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c4.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA4bpndYDUYn8nxKK9IVIwEmtvwuphauV8+XxhCwXAFLhep7BA
+n9aE9LpCau0/Z0oV5LHgLOhpgseNSmBmIlPbD7XZ3fpy3yCcK9WiWcj26nKSFj86
+kduVwDpTLpzJAZjRc9bafiGEV0xmb5HEbfCGUyE5UHlTvOonQDaXlVe+XazXR3+b
+OrxW5kEmqN+mCOGzyHD7QJToBzQm6gh07aXLvviW2SxuJ0aW8fg42UyX3+xwslPB
+LHjSyT/Su57jl5G/wtAheeAwtnnjo3bLoq2nF5TLiSTadniHTEuj3GS+lv8F73rA
+gKS1VAtgXRnTJVjXYdgD7kZEJWrDmKvggNgyWQIDAQABAoIBAAnSIFozvAnPr8fr
+BKuEdeRMmaozTwBlnPsFzFBnHvpHzeZ3JqNNpASPGKtU8yM9TUKL8qLsNwWiRPrC
+wu7PaIKAePV4G14AfzWbr9qULG9QLhxFrU4B7bv/+EqDi8NEBijcui4uqyFZM5kQ
+ZdQarjLDs/NVCVhOH7MNglQpuHXpeusGK+g9VdlrUmAFmn11wQ/6ntSMvvtkwirh
+ErR0LVKBDrh+9kT/iA18qUwxk5FtAVShbAESDQ9q+2sKD9I8Sbo9jdHiqc4ajUL2
+wdGAeFdENdH+AGb76UMwZp7Y6LVoOnVRabRn/sO8Yt1tdHCXQGCLSSomDBNZMJJi
+odtBw9kCgYEA5O4erm9wX+UEZigMT9MWND04DKj9FSN+gS1s4RfXrSBY+Fh2TfiC
+mw4Xly2VeARbjIcsiYQefu/GJ3HpjW2h5asZBvxZBXsPc3kqhQPi6KjfI36oPzhH
+NKYrj3aFp5bL+fKgFl272XRHg1FLLZ0AI1Vf7RlkyCXlU2akUi4ckM8CgYEA/Gtb
+ZQGIlwyf1J0UO8dH1ICJbJoGelnyUemRMzY2WOrI7cp7ODAo/BkVl7fJlMjHpozy
+2NzmjmAlMsw9MKr5fe2kTbd78aR9OtFpWuNM0q/m8HKH4jPPQhcKydOCHW/OBgr9
+08/MQ0STzkdVkl/42S1cGmnuRfGFcgd788mURFcCgYAN0obNt3LMh3JljKN5qmAb
+1VpuAUIOpa78s7ZwPNVL4bkdfZHfEVMpc4dfHGR686Bncr3wq6Z/uZB4Tztvy5/L
+0zHWpyPk3dJIAZfcoihwLcoTRZF5B7fc6O+WYOye+s7kD4806oHuFkQ/HIhyf5lD
+iOMa2L6qCwe2qmrff2NAJwKBgQDE1W66j/yUVTrZj/f0OYzwz57aOi2tATDt4SXh
++ndc1REEJO37Pcrn8yOcYKvnwr21SZohs/hxs9WuDAf3SIwvP73tAJS9NRiWYjA2
+uKLydEqw247MvLv0/ITkJ3QCQmo02iNfJV5SYkqkA4peD0q9CBsXWd19Wz5FUkK9
+M0bUFwKBgFC4Hfo4iDGJPg4hEZxYjHzFLZCayx2wioMrrXwzxPIQoSekUAZ1VW4g
+d3ga/h5bzlhV72ByUNaODNznfTRsynEiLz5fWGon8k8+PQcgyOp74HULKXTpeeNz
+fWUd3F+jydavZi2VeG8sNhBBeReZKQz+2q1VRAcMY5KEvUxBNaOG
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c5.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c5.crt
index eb405cef94c5..ad238530aa4a 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c5.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c5.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAPqYnJv7Ib8hs7fwymtTbafjPHznMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW4xLmM1MB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjEuYzUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBALGUvDdOnTe62apHl5sXV6Ys8GYqKDJ0e1cCQCKgxu3MyCueXC1xdhYJceA3
-PTmnTRg7KqYhJLZi1sujBOfuy7vsg5r/7L6EhWDCM/d2QfF9ZUft5ljsEGYNOmGa
-kQnU+mFOuDe18hlp72tavC2tdPYHaTmd0t2f4J9ovxQznO+jAgMBAAGjfzB9MB0G
-A1UdDgQWBBTgPsBAr0vFIhwmOhBkC1XvlAGtvjAfBgNVHSMEGDAWgBTgPsBAr0vF
-IhwmOhBkC1XvlAGtvjAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMS5jbHVzdGVy
-NS5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAFbiX
-wn34AQsvyD0AQkMy0G6ZXH0UngtGfW6qyz0rFNNRtV7VnxYjZvuXKiZ4ixaHA+yv
-heUYsReWoWzZAb4SYyxp4t5fuKxIF+diW+WelUV9FvTjD/Ynzp4jnxI2B1+2InNd
-WFcODD+1pYaOoUoM4554Ir84GoybcvL2iWAJX84=
+MIIDTjCCAjagAwIBAgIVAOT2bsITKOYGtEmGTQTFuQc7T0fYMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW4xLmM1MB4XDTIyMTEyNjIxMjI1NloXDTQyMTEyMTIx
+MjI1NlowEDEOMAwGA1UEAxMFbjEuYzUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQDWDtiPIFavsh880iSg5HWxldIEAMO7AJkKxERF1evr892lfGwHn9YX
+coUiTXgiNktRUv1wSfz4GEK0mNW6Gst068yn+167A4gsxXKUkrZiZyguQHxPji4Q
+dcbx+7liBtKVl71IbbUUPLTxH9FcGRchq59hHjxi+lSTYAwOK11t1Lo+ycV9msGk
+TJj1C519SAFaKMUzs//5de6arMzv4N6HBs69ahxbKUcfLJFAsuDhpzxfkrPUdBAa
+dIQphnxJEmatUjMN1HTew8Z8cG4PmMPeWm7oXk1R1Bnm5EvXS3hXft2HMXNvKC8J
+oGtbjVMOzvKf70ijg5v2wniwR9eTV9NxAgMBAAGjgZ4wgZswHQYDVR0OBBYEFPRS
+GtVqDrC4WAsAjv/FKTQhE7yZMB8GA1UdIwQYMBaAFPRSGtVqDrC4WAsAjv/FKTQh
+E7yZME4GA1UdEQRHMEWCHG5vZGUxLmNsdXN0ZXI1LmVsYXN0aWNzZWFyY2igJQYD
+VQQDoB4MHG5vZGUxLmNsdXN0ZXI1LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAak0s3TOk0eS9ZBdDRXmUN7Oy5ZP3P/GbMUtqC7ho
+Tkd7pF6tRbrx/wXuvcmpsgUVrjRFQWbx292SnkrMkEX4QkJV+sr9XLxWf8F/t7op
+8aUcS43USf3DQ/TF+2WLQlcJOnN9I9QP+MBJx9m0xqKl3YbW6uxgcen3SgRs+c/K
+5CBZXURTuUD3nOKiqouueVuTPULd0OuDzlK+WphjviP62SklM6m9QoYYp2aznYa9
+MMeXUQsmhzg876yHSGgngqSatc+AxfOntdWY58UJtpY1DHutqJp+HQvDWjLQAfd6
+xx68PdjrvzCBMPjyKytfqvULyUiqDOf0uMxe8ADRvZdVtA==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c5.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c5.key
new file mode 100644
index 000000000000..29e389caa8ec
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c5.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA1g7YjyBWr7IfPNIkoOR1sZXSBADDuwCZCsRERdXr6/PdpXxs
+B5/WF3KFIk14IjZLUVL9cEn8+BhCtJjVuhrLdOvMp/teuwOILMVylJK2YmcoLkB8
+T44uEHXG8fu5YgbSlZe9SG21FDy08R/RXBkXIaufYR48YvpUk2AMDitdbdS6PsnF
+fZrBpEyY9QudfUgBWijFM7P/+XXumqzM7+DehwbOvWocWylHHyyRQLLg4ac8X5Kz
+1HQQGnSEKYZ8SRJmrVIzDdR03sPGfHBuD5jD3lpu6F5NUdQZ5uRL10t4V37dhzFz
+bygvCaBrW41TDs7yn+9Io4Ob9sJ4sEfXk1fTcQIDAQABAoIBACmhKoGvVtyFhii0
+b7YwqCj4y6bGpUo1hA4BfIueXmr9euInr/TNo3vc5EJ3EYGJ0J0IyEwxv3ZadYKc
+5Jf0hRQAlcUnkFVLiwy1+6IkZD7/8+MG3IY2TwQZe67s9zE2xXadU40F9PzyD3iD
+aRqBsKt2LqFbGYBgUdl1iPKqgVmN9dqJ0GxKoqlLdhLQiP1ptkpawjrYKKQpvxMM
+vmqErkNKAYdhkVJbv5+QIPdGybAzh2vgfFuUu6xWKcMgy/96kVGqRw18egX7joQi
+ACUz9bF2mp1WyN8MNeNzxaYRkvsVt5sfpKIEkbTV7PAWn6yvGZ0PgE5GZAFIH3ZK
+9LSjI+kCgYEA9s9/o7ryCz/rPNksKZAfxqh4nKaotNMHI9vOKHKojnyRoir3NVY+
+YJWX9Zqg6pZaGQkDcQxxm0rumGRK1BiJjYL2BsiLklTKnWyr/oaOhR5Gz8l6ti/I
+cq3bRIlpkUPVHSp8ewKRVSYUwKv5kQK9sVcls83hRYzhH+Za8sGBTh0CgYEA3gcp
+sc0z5/RcZCrsvktIrGB2FudcY54sxB8X0RBN5iFo+c7k7xMybOnQsXk5YXDJXVEe
+SP8VD3e+HkeYz6RdzyDJkHjVZyysp5i4WUGiRkYchJQrn3Omvq3uBYKAhFpww9SA
+gw0b0qPPwt5YVvmflcfoCKHMrBW9TGhhthQsamUCgYEA2E1p87h3QFy4cQ9zoMlq
+z9P+QGvfFvqLG2Yb+17hUl7h01oMZXY8HLF+CAWIWY+DTsWTepQJTKwTBwN+gPsw
+MhFARRmlGI2vNOB0rBCI/ylVre1J2jluVvL/Y8V5+5GTSIhfp/7MF+/T6DfP9MKO
+NMnHhRk+AJJDCo37zxy/khECgYBF4XVDC/YBVIzbzhLTVfiaoIWqcclHZVpro+mo
+GHycKPnUXkT55k3hr7nKkf/4ZjpKj6R7jyV+x85j018KQ45vCb/HI/Du10XEPjIP
+jbPYcGq7Y7NTLWBW9YFL5KY0gg6p5DFKjg6qvno+uCKPxWNwDgo+5UX5B6Kb8OKI
+H1NXbQKBgQDtjEtI3/9q/ud7OCtID+ahuel5/zCAYrCrGKfiTDB2ulyuTjenV79H
+dhXI590j55276cpb6Nxzm+6sTy7ZZWy97FQQgNx0Un6zYPqxxYERi1VtrVRfjATI
+fwGGBhcEYeEs79ntN4xGi2u0TgZINR2OLbr5OSENelhNCTHeeebMIA==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c6.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c6.crt
index cebc88f99790..4250e14a619e 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c6.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c6.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAPQ03RwYzZEGZE9BmLB3yibXdFlrMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW4xLmM2MB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjEuYzYwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAJGKOOMmMfmt+9VRZfYsorzOWLxWRtqD9VjRimbBVWBoIFYf+EVyEC0E/sg8
-G9eWAcSEUJ2oDp5dnMuXzEZrA3aaYeDZjtOYuHF+i+PFQQFlpY9A+NapdhJe9CU6
-cvL4ImRyatPxw4A6QLLetWHbDDIYA+6iYtmkKTkMlY6bboSzAgMBAAGjfzB9MB0G
-A1UdDgQWBBRs7k4fmkFw8RmVU1f+p9SY5qNwnDAfBgNVHSMEGDAWgBRs7k4fmkFw
-8RmVU1f+p9SY5qNwnDAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMS5jbHVzdGVy
-Ni5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAEXhO
-vittRGJEOl5Yt9pEmw1UJ/TRum2osbMj+3+7QGlCpU+GC6ziMXbwkPI+AksCySBX
-mXZlgMK3j9oESC8K5CiWA8IpY2HcUJnKRFac6NIqYdbStVNAKJBMDvmyhORW2ejU
-K2MBzTHUyJtCjFmIlFW0YzWFgMUJ1fqHWBwpOZ4=
+MIIDTTCCAjWgAwIBAgIUBB/n0nN8jZBmOmWXXA1IHlR8F90wDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjEuYzYwHhcNMjIxMTI2MjEyMjU2WhcNNDIxMTIxMjEy
+MjU2WjAQMQ4wDAYDVQQDEwVuMS5jNjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAKDzXqsQ7aKfuKTuRtruWq8lt68cLo9l7/9tVbXgUuuaHNBJoqv+gvRU
+F/yGTuEccGJUfozY9aVNXW8DpQ9mJdcMNbqhjzAoSBB6cKvxyOMqR7WQ/C1lPunB
+FboEv+VHok7KRG33V+NksMUrPhR58SqdEB/ZmmMxC5VvvX/PGjmlVpOHxbxWKD86
+Vg3prR5v/QkkINte4lSLSbFi4+JyjUpAphC4gElM/EtbGYPwWOR7H4lc38Y3qK4l
+2gbB5VDRLUsjivrKsfPXg69j+hN8pIzeMViX7eraEyHfFExD/g8HGKV9o6oWl0yq
+Ts6+HZMVg7bQzSgAz1Q7uuguyj9w5XcCAwEAAaOBnjCBmzAdBgNVHQ4EFgQUTc+9
+x9MZ7tk6jyBR6uVXyCZrquowHwYDVR0jBBgwFoAUTc+9x9MZ7tk6jyBR6uVXyCZr
+quowTgYDVR0RBEcwRYIcbm9kZTEuY2x1c3RlcjYuZWxhc3RpY3NlYXJjaKAlBgNV
+BAOgHgwcbm9kZTEuY2x1c3RlcjYuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQA0IuVSgXSUrYWvHL+gLy7GXTrmbp+VjjqtT0DAh71X
+gvyNFxmSJ+3T6pHC/QkUxKgxOvOg8yXEPbNnJxDkbEqhYm5p/UDsQC59Ra6zyI9g
+p8KMsgGISBvYjottv7nmNFx9dJNQG80IPvT3b6tcHIpzpaHnlTrdF5XOlV/vpJV3
+FCXreCuqFQyUvUCOEO/tX/1OD48eg72u3gqZbr8GbLZR5OiZKrGqCYY7+DzCuxM1
+emNs0mYeX/KunXz4dfRmG1EkYaKQKLd9R2vQDRUnSFJuAGv2q7JwxbCN+9xGZnyP
+rLEK8qG99Tp5pAR7ak4dCtLbzM5LIzQFihp7HT5aNcud
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c6.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c6.key
new file mode 100644
index 000000000000..f21807aad5af
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c6.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAoPNeqxDtop+4pO5G2u5aryW3rxwuj2Xv/21VteBS65oc0Emi
+q/6C9FQX/IZO4RxwYlR+jNj1pU1dbwOlD2Yl1ww1uqGPMChIEHpwq/HI4ypHtZD8
+LWU+6cEVugS/5UeiTspEbfdX42SwxSs+FHnxKp0QH9maYzELlW+9f88aOaVWk4fF
+vFYoPzpWDemtHm/9CSQg217iVItJsWLj4nKNSkCmELiASUz8S1sZg/BY5HsfiVzf
+xjeoriXaBsHlUNEtSyOK+sqx89eDr2P6E3ykjN4xWJft6toTId8UTEP+DwcYpX2j
+qhaXTKpOzr4dkxWDttDNKADPVDu66C7KP3DldwIDAQABAoIBAAgmCibzBnTzTS/c
+eG3hH2o6qKAO9zR/XrU53dsFbm1LMiZBdCDtE8g28bfxPNEc1XvE91s+KMGjtxVF
+w/eRnlzS222FfHBdubhL2qR560fE/jhtDLdnptXC1szCLMz44XjHuMsweJyknmZ9
+F/mFavZTWVujcrOO6h04uQldfOBqL/LOj8+9/pSwGtBs/dQFh/rIcx+NgUuSIFbg
+bCuqJjMLoyVHbdVz3Ov+wYZeL+Ed9aVAtEIKAH8BdDRKmQ8hiO5XvlMVc/uGLeH0
+YlEV7tNTb0bSeD1dxpYnVR2ZvlWDLwQS8T11fWqI/Irlqw26NfKmJO4IwDEO+YzN
+IiINrxECgYEAwgyZ1b+p0oHN2rJhurouGxNzvxH0sMTjufDppwZvJLC4vZ6VZsWc
+z2VfJhcE7Q2DxgINPCQWpmce5i74ajk7+VRQbd17YnZQU2umZ/rwtvfI6uOXMJIZ
+pkP3skjk6zsTOez5xUKYwFKfHeR2qgqQ5z1Ztn88sHvw0r1nPYHRY/MCgYEA1FWq
+TVItASAhEBI/tdLyabpmoQ25wIX6R02buOr06fk3MHAWNMJcgrShQPTjiElSfO0g
+68sAQQD5byf3XGfHMdAx9nkkLQD80YIPpH/qN9zRLZflf8orDpbF9DoVyAuKlwYi
+3F4FuBzi/MR0+fvYHi7FqippNu6T2TdhUxdyDW0CgYBtdkA9FD4O0kw2JtP8SlQB
+0ApF6TIXmSlA1YZoQrflgY7k4BU1ARpA39U5nliYBDymMLgYid8BBUCn5i0Zdqm/
+LvGj2AFm/Ii5VgtrsmGhRMtgvPunWXMAmGycdIQ1sYBZ97S5sqSKMNEzQpIxv34Y
+A0KdMM8iWvfxWQho6NceQwKBgQC4Zz1jsT5Mf+622mJiMjU9BRkkPZtYTAKz8f6P
+dd4FZN49PtrKLxHQobZ5hzQjpCnzPK0emko6KfJ2SqtUoPwpkroRZ9u3CeiGuwGC
+x2RN54PyqLwzYwxJuz9nfxpzhTJzPm0AHNeyscOf6a39fG6Qp+YahoKhpkjYmHP5
+iZ8PcQKBgQCGZ9R+gkboujCDNsD3kgjLQgRuFuomF4BXLx5ZSUH3cINCBr+CC4f7
+gEk1j5wxNx9ntXWjVeLQtLszccFX2x6cXcquF1UH0J7buWgExg08b+ZfZ+VrhrMJ
+OT7bh6kzQkH8HyOtWlUecJsMdgP5gJfVHZfUkOjOjZ89Byj492HQUQ==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c7.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c7.crt
index 08c2d4e7af7b..1201b1a904b3 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c7.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c7.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUccM78mwTgQTTvYEwDeDeYmTLBRkwDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjEuYzcwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0MTMy
-MjE0WjAQMQ4wDAYDVQQDEwVuMS5jNzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEA0dKhFm8ywJ2tBETCUsdHzEVE+uVb+oLVVxswOTBJ28U6s+H/KSk+f4iQqtND
-+Q4EI6u49KHvH3/koHgRuQwmcij5dCUuAtsBR4wZJLXUgqlBNLtLqejLF17vFTRn
-i78xiTQ9L5CBioNTYTGhjxQ3tUbFCD3NLPLd25uVNR9Vc9sCAwEAAaN/MH0wHQYD
-VR0OBBYEFPGksbL4MBmo+fWjS5FugVp5PQPyMB8GA1UdIwQYMBaAFPGksbL4MBmo
-+fWjS5FugVp5PQPyMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGUxLmNsdXN0ZXI3
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQBfWuwf
-la/PtbA9kFYzzHozSWubOKUoSTzT4yyWYMrONU6rDll/x+nRUujrko3ecGcQHvns
-qvfLRxifj+TS0SQmGn2edgkfhNBke/rYfGA3CzuTmlqOiGdIz3onNEf/W97mfNu0
-ssOitDOTm3FipxkxvPlvfIOBuA4vUUX4MlfPaA==
+MIIDTjCCAjagAwIBAgIVAMjbD93GO6JCsPWwTeWoG8PeBC+HMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW4xLmM3MB4XDTIyMTEyNjIxMjI1NloXDTQyMTEyMTIx
+MjI1NlowEDEOMAwGA1UEAxMFbjEuYzcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQCZ3K6CqfRGpCWj3is7VzxG77lNl0Brjbiazgtd1nNNWwGcf8MX2qwr
+mnuZ5tXlwSVt4EtpuAZvg3WZbuu022sT8jElLATaAtU6G8iR5jZd5Xudko3Vc3aC
+3XQJtGDyZWW7r5v0YeJrGFZwQcrKYNdgjcdiGD78hTe7BTqJ+s/6UOKaej4H+c9y
+erX52DSIJwkKBdMbGxhkBQlZgwlTiiiUuy96ZaJg5qiurq/D9iSCZhaVxDvYEvzI
+1UD/oZGOeKDH/8GSBAFxiaGR666asGTy9fD2nGECISS2thjDN4ZDCuk+l6oUUhrE
+OfSv/1DDfl1pscf9q1MZ2/jRr7BzbTKBAgMBAAGjgZ4wgZswHQYDVR0OBBYEFHTP
+8Vh2iWOK+DPOmQb35601GP8IMB8GA1UdIwQYMBaAFHTP8Vh2iWOK+DPOmQb35601
+GP8IME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGUxLmNsdXN0ZXI3LmVsYXN0aWNz
+ZWFyY2iCHG5vZGUxLmNsdXN0ZXI3LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEADxOvuT37Z+4r57P5++++ZtD7L3L/WGgjqChdi6YY
+Dw7ON53MDnTlSEy3XhK4BZ6SY5pJdN6/q6dFxScHxjC3xsUVTZD582n8JmTmTDu5
+xcJ3mHTTZj99PcpGO5n9h2qKH5T7Nef3/OtGQGYzWGxbYe7wDqpMvPUM9iPiNiDH
+LOPDBSuUCJm5c+ZW/vRcjUapdE+VOec2vqT/c27jmMgEa2A92c8EW7YSn4PJvt6A
+sdi7OQPOFFXiyFlcP28+uatNpCfHvzzd/jDuu3As/gyH60f2bNbsYm/T4rDaepdU
+T4OcJRrN8kUR82Lb2agYi5gdqBVGxETwdthBtyIVUcmW4g==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c7.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c7.key
new file mode 100644
index 000000000000..f68c2be6664a
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c7.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAmdyugqn0RqQlo94rO1c8Ru+5TZdAa424ms4LXdZzTVsBnH/D
+F9qsK5p7mebV5cElbeBLabgGb4N1mW7rtNtrE/IxJSwE2gLVOhvIkeY2XeV7nZKN
+1XN2gt10CbRg8mVlu6+b9GHiaxhWcEHKymDXYI3HYhg+/IU3uwU6ifrP+lDimno+
+B/nPcnq1+dg0iCcJCgXTGxsYZAUJWYMJU4oolLsvemWiYOaorq6vw/YkgmYWlcQ7
+2BL8yNVA/6GRjnigx//BkgQBcYmhkeuumrBk8vXw9pxhAiEktrYYwzeGQwrpPpeq
+FFIaxDn0r/9Qw35dabHH/atTGdv40a+wc20ygQIDAQABAoIBADAMqVj332fSKkkP
+ADxDZXkXsyxGgGE2PIDnuMgAyThOo+WuRoxkB2GEmHX7DzB1jY1PPu3QgP4BNVAJ
+wkVUQo/TkkJ0CT3WJaXEeHILzU2sdVrDolSW/tsmcm73bFE2ljszP8ZlbTKZwqDD
+AyQJcyldjXcWcFEm/z0CCIJ/l8SpUrvyKgeyncXYf1dbKcYKSTnzo+UPDQaNtAPW
+qKXXnByF74qY8hawo5cykp9HlW905iEXE2aVZMu9lCR9cZVOX9JWJKFOuQET2j8Q
+Tr6hkzHjkLcdB7fiD5EDAg3Ej1tasJ7qhthZZ7ytrK5BIlQwWFHXhrVSeeYMNBbR
+Rnw7tesCgYEA0dwWsIdRqT0B/1RS/+ShNzXwtfNrocizBP6YrI4MNeliiabwNYyP
+BsPfWw1m12lOCLY4E3JxiQ/xmwwRVvsH4WEDKONYhF+ZRo+Hl4I1JPaujz043lLV
+IMdx+l+zrE8/lOjwxkKDY08Q+QMjqOLU2qMcM+7PtzbVXWIagO+WxMcCgYEAu7DG
+YXTdsPA0KOFEEtUqzmq+9xvUCU7xMjKEltdJugFbz8DT7C4y3gPf1nm6xvJTpHQx
+YrVcXT/eBvod29d0XGc0vxCBRPK1JMfoctgo+Q7TR2X5cuzqUgKb8cy1PRZl0a9D
+8K+TVsL5mO4FpRjCa8CAPrKJ1vtSYWKFA9E8dncCgYEAvLjnzqhiT4eFnFAAtA6u
+zC+94GKfkEYkd4FdWxAC+xh+fLubmQxmMvdCjWdn7J/OhRcZJqPsZg3ogD5B+a6W
+T6Amb0bD+4bdGA9LKHz3vd347OdplMV41JmHbllSAsrhqtAwdpqp/UzFPqoMD3yt
+XlEAs9M3CNRO3EO1Ae5z71ECgYBGrkOG3+O8hjieTHa+QNhl830GwcwP1VKvfKbB
+x0FUcr6Idi1Ye+1b/kmFE0JrOUaWX8tZSMMHus16AybLa0++nH8YlLaZqGze9Qls
+dkq3wAWAlb5isSVszN69KEH4Rz8Av+ft87Cgs7wbA0fM7I4K1et1sPaHpw1W7yr4
+TQtf8QKBgQC7v4jANltbKnXLZ08fAZXHLfiosSw+a6OXVwZazr8oL5/y3DZeHirt
+oOrNrLw6KKseLy0TxcbpU0LhFyxhrssefSSMQ0g+uCMW7L671e0jxhvbCwUNemZb
+6OEtEOL7xoyJfVDcD77aOpXYpIXLauwoYumUwM5yMMcnG306yCrsXQ==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c8.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c8.crt
index feba2f614aa2..69ee5e9852c4 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c8.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c8.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAMUWjLPS533+7BzHoMpXAZQIJc6xMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW4xLmM4MB4XDTE4MDQxOTEzMjIxNVoXDTQ1MDkwNDEz
-MjIxNVowEDEOMAwGA1UEAxMFbjEuYzgwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBANBMTFKHFmdHOxdIKRfUV8NVKIJe/T8QSiHNiXe2HA1TQ1qp8jFjXLnnoeWw
-aDNUqz7dKtjw5Xf66QXjqAamakTHzbtbdifoo+276v3of+9ddYYjjm6n3lnvK7zi
-nLjLj9l8kjuEp0rTgqUU26VicJVs76dlHsA0OxShdbbdAWJ7AgMBAAGjfzB9MB0G
-A1UdDgQWBBTGQzXKW5ORg+cp8KQkDmOlQzOIEjAfBgNVHSMEGDAWgBTGQzXKW5OR
-g+cp8KQkDmOlQzOIEjAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMS5jbHVzdGVy
-OC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAmbRL
-wgD19qMd5bFbK1sYiasX4DAN04U2Bx21FRbuOXA03vGT+FVnVLnbkJumtIqBCGCZ
-sebCxLEhE/fmSX12hKQnqFR/1EMKvI5bps2996M6NeMTmdGCfJECBIuMN22DHIcc
-5Ek5UCEQJM/L7tQpdWBRl3q9ca4x8HeHLlDYSvc=
+MIIDTjCCAjagAwIBAgIVAKN8FU4SXaTfBwYMLhpElFUOF6OUMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW4xLmM4MB4XDTIyMTEyNjIxMjI1NloXDTQyMTEyMTIx
+MjI1NlowEDEOMAwGA1UEAxMFbjEuYzgwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQC5WPnWSCxq0yjLmWa0z/3Ud7vRtyPmNIpQNnAgO3ct7chNN/C9oVkF
+xVUTnVZBDBYpVvelFGIuWpM+aRcEm7Nf/2NWUc74Ik76n3tZ1U4xuVRfILLvu7Eb
+OusH1bro6K5yfz33AJ+iXrnyovMYW3wZ1pt8a/SLplogLijn4/rlhcamCZdNfz4h
+FEZWJ0MK/8SYwqAmBvmNnk2QLbp31qse/U9LEUQ94BcnozAb2vNkAHOf9NYuvskR
+jKKJU1KI7QX5JEFCuFQKKKWOPgcA17fNh83qs9QcZOQqWtAR8/cCpsFm/3r0SDGf
+IglezQ/IIqHMj6IHKlISqYck7W/q1nFjAgMBAAGjgZ4wgZswHQYDVR0OBBYEFHKw
+SgT+VNu7YKA9acIRNJOkKXZ7MB8GA1UdIwQYMBaAFHKwSgT+VNu7YKA9acIRNJOk
+KXZ7ME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGUxLmNsdXN0ZXI4LmVsYXN0aWNz
+ZWFyY2iCHG5vZGUxLmNsdXN0ZXI4LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEACaRtc3Urh84Lqb3kZ1eacjxOK8KNFa8O1gDIZLHS
+c2pRnWfs7oRxoD59TBPp9e69lQaNVwynj1Jcp7qYZUrSEi7UMRR5AUzsP7gY76I8
+zTkzp7aNAKbbg/Y1c+jMYDhsWPgpwp1tC0t/VhiD3BbmT5hBjveFTQaBq3laXCYG
+lClJ/5XzdBEEZ/Il06OA2e88jaxRVXXNWkeNYXKWBIr9YioIeYsYKDqtU+V6sa6u
+YQfR1RF8wGWjGBbST6WZlGS6wFYoTunS9hl7zWPLqmMPocQrkT+NIdKxk/qk0+zC
+RKz4G417FTsFh+ViD+Zk7EHb0m0sK5aRMGpYJ/TCveCuGA==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c8.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c8.key
new file mode 100644
index 000000000000..49da22c6359f
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n1.c8.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAuVj51kgsatMoy5lmtM/91He70bcj5jSKUDZwIDt3Le3ITTfw
+vaFZBcVVE51WQQwWKVb3pRRiLlqTPmkXBJuzX/9jVlHO+CJO+p97WdVOMblUXyCy
+77uxGzrrB9W66Oiucn899wCfol658qLzGFt8GdabfGv0i6ZaIC4o5+P65YXGpgmX
+TX8+IRRGVidDCv/EmMKgJgb5jZ5NkC26d9arHv1PSxFEPeAXJ6MwG9rzZABzn/TW
+Lr7JEYyiiVNSiO0F+SRBQrhUCiiljj4HANe3zYfN6rPUHGTkKlrQEfP3AqbBZv96
+9EgxnyIJXs0PyCKhzI+iBypSEqmHJO1v6tZxYwIDAQABAoIBAFhA3xlinU+yQuML
+JIXk4YqZBoP5jazwCIngGeynE/z5E159S/KnUJG4X+WqjacEX8sIX3uppIRI2YKm
+nHCzyKnlm4G/C2tIPs4EQUJJmGWBxwRSuBEBkHeUStQct1ZucJ4RK0qg7+Hq8+Ru
+JY1MBl/xOXD5oGQcbh6TjCO6gL1ncJomz2vM9qewMftn5KaufU2xXl8sRxtLsfAz
+7Z++VxL2c+257nJascYkU/z8pI1AivnM4EPTf6+oCEV8Q7M0/iKbx6QVmVL4oxko
+8xaOctIuHA1lniDw4n8H+jjk44udD8UxZ8rw7YUPtLlWv2XGlm2Z1ENSvDHdzT9e
+2v7QXnUCgYEAxrxhFWM7lA0pjYZPXwxbvXQgWHbd3sZxNrrtQJcDPFrsmQcsYjSl
+Z5FmAon4L3jgVmwpTcJkVLaC+MPik4eVXHDxnAuHiATxlQkMZ+bsuyTpLxtW5Cit
++06UvMekctLC9Si0dcNhYGlHVqRwfaDFhWVmTOR3RPBAkAOopp8gCRUCgYEA7sEE
+lcNTlHKSe1IDHYbIpjYEHuY22PrUAOSihhigrZESK0TNPhyhOgzKH9i8Xyt+ryrp
+Y8FXYMa4uep73VvuFQc0LnfpXkVcKTgZbk9Oo1klongDT9cbryRlpKHBwMzJ2bx+
+MekZ5+441DcMsqs61ZQ8BpiqDkpRHcatvs1GPpcCgYBLTdw3n3dW3Tnuj4MJ0AP/
+IL6U5s2heW4yv9fgjphCSW23epldyPcX7GdL9P2e912CQc/Q0CT2WIX+PzNelZSK
+B531dATyDHWB9IDH6h+vtgfJusFNCQEL5bXEZp2wiJ4pCTiAGv2N67aFZR/e6omN
+x2NhJXzwGV1NdKV2JXudFQKBgDY3OnVPr5EMG1tJmk+AJdwoc/MYT+ghvTaetJFC
+0mnXgYq1yu2JzsrB0Lw+mTf7Rn88fVcbtl3Rgm0lZA82KvA3fI/4xtOTaIQrqwqz
+k6XgPn2QRyBANpck6rfhy3+egU+iT/3BcxKJs/nsrkN8QpXxXVgaHrO0CIt8bEPh
+lUvtAoGBAL7QLM80kA3UKRsMqGEMGh9PtP4cr5bDmw1Yd2hdqARIPY5k5KO0O+R9
+6/b6exAm1I1QTXz3yKSfVi/dBgTFpueEG1YGO0IxS9jRvQqfzFrhf6LgeDhV+3/o
+FBZACFarJ4U2LTbBCO24SrIbmUBrcwa1CpC7yS4iykhE4fUalFL/
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c1.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c1.crt
index be43ffd28eb4..6b5b3801fba0 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c1.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c1.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUGY+OrC0XCiA+1oCJX+CHvv7UH9QwDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjIuYzEwHhcNMTgwNDE5MTMyMjEzWhcNNDUwOTA0MTMy
-MjEzWjAQMQ4wDAYDVQQDEwVuMi5jMTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAm8aLIeL8o3DodD9VWa0KvotRwSUtWX+qzAFgM5SqHHymlDnwRJq6gutqnCDC
-os6TGihA6bnlVw4eRnjq8usw7cyJW2iIIFPr4d36DN2Ui/NGdICP4RMClrgf5sC+
-GE1jln0W7Jj3f76jB6HE3BbU6PCHdlAtgVcQ4j59y9dchI8CAwEAAaN/MH0wHQYD
-VR0OBBYEFOMdmsBGJx8YHfiziaJxU93Fc6C5MB8GA1UdIwQYMBaAFOMdmsBGJx8Y
-HfiziaJxU93Fc6C5MDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGUyLmNsdXN0ZXIx
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQBqgc5U
-S4wYk1VmWTI+8ezAgtlnsiILn70rfsi0KjI1hzo8PvTWLIVvV7NBefMr34ugqstL
-5by7pBAmKVf8Ut8148Jpbd48qxOLdwQVZ/Cthho+5u8i/eFmAJbATHkmyEj4Pfp2
-u9/QIikh8Gcm5sWf4CyC/7tDISxVHm1FsM3nNA==
+MIIDTjCCAjagAwIBAgIVANHgGmrRMgAIpk80q1mNdbLG3L6wMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW4yLmMxMB4XDTIyMTEyNjIxMjI1NloXDTQyMTEyMTIx
+MjI1NlowEDEOMAwGA1UEAxMFbjIuYzEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQCmijIm44nzN9i2+hEVqhgkIblphxe/5C4hYf2mRjKzo/BiZO8iiL+E
+ayXF7xz7pRcwaZRCnxQWljeB+JTereVQEHNVSg0MAljWe1W9dFU088R7XVMjnXYW
+4W8s0++A94BMj+qG9CZHOox2qIQaSAlutrNjvtxjWmnCoaghnKLAFTUsRihP3Vb9
+1C9eWLS5O5lgjEBHQSxnz1NhzqPT/WcjnD9a12iJfZq8zikWTRwo+wojOQ4hOTkL
+9+7jjdlrIns89UXfykYV3HySdNu/Sk5dJ9y2bGZ2IwElpeW1C6qv3KYza251C4E2
+6WVCpyKw6A99fqa0j3HD3jpsdOYbR8FtAgMBAAGjgZ4wgZswHQYDVR0OBBYEFHLC
+6+syAACyhQrRBjIn8UDNEQvSMB8GA1UdIwQYMBaAFHLC6+syAACyhQrRBjIn8UDN
+EQvSME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGUyLmNsdXN0ZXIxLmVsYXN0aWNz
+ZWFyY2iCHG5vZGUyLmNsdXN0ZXIxLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAMxH/Ravx0TxZovtApi5+UewvdUhQF815B6qt/wGs
+Wlb9/WsBFNbzjN1xLJQZiUEcRn+jF9dX4WSN0WaUue9bxoTWovW3yNQXBXvwSuN4
+7VHwDz3BjLhtE1HjEr31WjScM8rHWMStgu1LZQFx2LODlPlbgYn/JcGNE0lEahKY
+Aiwuk2BN4sFiOCSD/Q4+eLksbmR7sdBoqEPOQji9xyYgdMsd2RAeqx88JV9u65/O
+PYo7dqZjwGfDvYvjUA4JeR11/Z5xXPAXBUBZaW/bSQDQ7cU7j7xGhWDmXT1m/Q2c
+sANccCjZ1wE1YZGE3JP38RDwL22BUPHuh+N1Rqw3H8fwgg==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c1.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c1.key
new file mode 100644
index 000000000000..10865550f4db
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c1.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEApooyJuOJ8zfYtvoRFaoYJCG5aYcXv+QuIWH9pkYys6PwYmTv
+Ioi/hGslxe8c+6UXMGmUQp8UFpY3gfiU3q3lUBBzVUoNDAJY1ntVvXRVNPPEe11T
+I512FuFvLNPvgPeATI/qhvQmRzqMdqiEGkgJbrazY77cY1ppwqGoIZyiwBU1LEYo
+T91W/dQvXli0uTuZYIxAR0EsZ89TYc6j0/1nI5w/WtdoiX2avM4pFk0cKPsKIzkO
+ITk5C/fu443ZayJ7PPVF38pGFdx8knTbv0pOXSfctmxmdiMBJaXltQuqr9ymM2tu
+dQuBNullQqcisOgPfX6mtI9xw946bHTmG0fBbQIDAQABAoIBABKSSB/ykUxcTrAu
+6C9SicEjup2kOTeGnkdOOl8rplVGg5SxXUJsXfEKEssFDGYahaC6MVSRkUeohjWZ
+7x365d69CuN4r8ZGT7pRRCEtox/VzDQWluIamdPY3VEyLBvlXbsL9HMKsGcShh3o
+Ulzs6hJc0Nnfa4uvxin0ZX+kDDkfpfV0L4n1FpNRP7VA6oPcvXjb0T9+GMUKBmlK
+vH+f3qeaH3HnC+bOEeTMCUUoT6HdZFVwD2WMBDpRgE6M1TWeDNLOTojmd56eFSor
+b/6ti+BDubFa1vtU8xkfTwzKQEEi9q/OsGTCdPE0AGAPjKuDJkYne0HVlkdIDAt1
+tnP7BMECgYEA53ZJ4OhCbjR7mlBrmcnI2tACau3je+jmsKukPoS3aNfiPfzvgFna
+D4cjDkCwq8cPR60YDTW53cw45EOskUkMgglIG8roUmhXrsbDMxiVYJiscvb27qe6
+6uo6ntycM7DtnnfznqHz2bHws0uij/sPXFiCsC4D02pEggJAYueRJC0CgYEAuDH3
+GEtpQTPo/O66hdeyzPDFKyr1Rs8LZ0za0396URZt3A9lzIk1cOPcXuFKVq7BiLjl
+0b2xBI7pja7yQMCnELPZ9FTK9GbVBNovAQQuEMtV31HRNuRGmgadwZDVpICYJa5o
+FebWIDA60Xii3pAyXR7PaMmdGlAqkiv8GfxlGkECgYEA2nC+RG2t1Aff5sCi696N
+mkPZJEUh/VzN6knylbX2yhGDyPCer0kdDK6CnP7cPSF+AC0Rl9Xuda44uuRLRknk
+bgyIawJRwwQKTD31FpHK2AD+Lvl4Db9qHi2z6R1WpUoC1x1pY3qAgiD2gp+M6ckM
+tCAsJk0m+y09Pl0VmsHkLnkCgYEAk95ow3Qmt1V5ATSVQMV8oNHZN2JxLbshKHim
+Y4u/IkGjSDqEOsMbx9jlHLY3+4anj57/wHVMGqCwFjIQALOoatw3aNJ5a+Lxt8eA
+XIyCfrD96x2/mxV/KD+PfclL+ZfSjM2Keaqq04Rkc60o8wtHo0IMt87C9FDPHDF7
+XVElyQECgYATbD2NsseqHG65WuxgPHW86kfBaTdoX/WR5JShyzowCoNMhovnaU+S
+pG41rQ4LRxKPLNZcD5VZ1Y8h0XiWEemmQ297aOx2prUMiyYLELaSZCzs4//seA/S
+4PLhR/M/kPJ/ch9pqhDoO7QrdXBKG4d16tOtiwwayq9PzrPmi+rDXw==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c2.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c2.crt
index f77708c26628..e77c575cff92 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c2.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c2.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAI/exdUNVY6z8nhKNGOdfOBbf0KFMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW4yLmMyMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjIuYzIwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAMA9aZW2sNBxgc19xYUSmSG8/kZ3DHYcB02RS63y20ZvNwBIbzMoawQwD9Z8
-gfqf9zJ7JPnmkyvhaPygY3WpGzC30vxMG/XG6/pPWW0yxlyL1C60ann1r4BTdRbv
-ms4zaw6h0v6oePN9/oZ6sgjNExBBsZEV9UyL63W0LEcPI+9TAgMBAAGjfzB9MB0G
-A1UdDgQWBBS8HSfCvdf16vHBBjmbzPf+BtYLzzAfBgNVHSMEGDAWgBS8HSfCvdf1
-6vHBBjmbzPf+BtYLzzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMi5jbHVzdGVy
-Mi5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAIAYW
-v7RbhTCsN6uXSIR94OHlbQCejHT2WBaJNecKNRXc1aQb3qUDiNkjp5TX/bokHhlb
-9lXlWP0CtX/w93PLgwQ/YG4HP+y18kPGvbm6s4E5hM5eLVPOj2LBewmOTYyJQivd
-6XLTaFmQnsMx/X/YMZhhVvymzM2HV7117aAR4Dc=
+MIIDTjCCAjagAwIBAgIVAKIsbqGxRYFv8xbGfaCCFvVawEFpMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW4yLmMyMB4XDTIyMTEyNjIxMjI1NloXDTQyMTEyMTIx
+MjI1NlowEDEOMAwGA1UEAxMFbjIuYzIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQDhDACgCB/IUiLQX3sA72b4G90qkJMjVEpwv8kEr6PgLJe6s7doND9a
+FA3ieSxKD6IMs//EtRgXje560Q9Koep659ybBzu+YfUnEz2SWLr74l+i/+rBgPSr
+L9wRrR9kD3E31F1eIxZqqyKFX36Q2n5SM4iTqCh1CpO/9FeMG1FUqKetXaIfu2+D
+GrPOJw49IXXNiLqORrL9Vw1C7x9yAYzqPcGyeb1UQD/O4HCRtaF77wnvgDgfUdnO
+QKG+u4rGMLNqxHzyuJj+em5hWzaj1gCh5fJfwEb+L4qKKdJkU2v1zHU4Rlx0JCL5
+BtZ1ku0vEqIImAMLUN4ltWrVXOFJ1QozAgMBAAGjgZ4wgZswHQYDVR0OBBYEFGTF
+ctJNLIn2CiBsSWeD55xzf+i2MB8GA1UdIwQYMBaAFGTFctJNLIn2CiBsSWeD55xz
+f+i2ME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGUyLmNsdXN0ZXIyLmVsYXN0aWNz
+ZWFyY2iCHG5vZGUyLmNsdXN0ZXIyLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEA1/VPtBHI7VU2CmgMImtG5VCQSRariQZlgQeyu+pr
+BOkK0CAehOw1Nyg+RRQItnRft/KzMCGdbl06r2pysf3ZixUct3lhvxikDCb3DE7O
+zybADmuemM3UvO2iKYNoa6OnI68Km5smfkVca1WGXgh/Uu/uMuiH8r7dTZcCm2dJ
+BwRN8k2nO7SUEx3ccuO2kShEvdZfZ4IG58hdy6x8aoINbERe15wqEYGdMW0LYr5L
+IhMJHD3aVbNnyQq2vkZF3gUdwZ2/Ep+R3DvIrbKzgG+5UYnl3pC/LX3WkkPGIOwQ
+A1wTzkPsuguJEHmywALQLyfwQNicaToeFM1Ka5hzH3UzJA==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c2.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c2.key
new file mode 100644
index 000000000000..a49aa7400efa
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c2.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA4QwAoAgfyFIi0F97AO9m+BvdKpCTI1RKcL/JBK+j4CyXurO3
+aDQ/WhQN4nksSg+iDLP/xLUYF43uetEPSqHqeufcmwc7vmH1JxM9kli6++Jfov/q
+wYD0qy/cEa0fZA9xN9RdXiMWaqsihV9+kNp+UjOIk6godQqTv/RXjBtRVKinrV2i
+H7tvgxqzzicOPSF1zYi6jkay/VcNQu8fcgGM6j3Bsnm9VEA/zuBwkbWhe+8J74A4
+H1HZzkChvruKxjCzasR88riY/npuYVs2o9YAoeXyX8BG/i+KiinSZFNr9cx1OEZc
+dCQi+QbWdZLtLxKiCJgDC1DeJbVq1VzhSdUKMwIDAQABAoIBADmiaUaSzhDveBsF
+WOsUZLqF6E5OWGBj5eyaMVER6lYvFWyrOf1l5BOdrzCg98a/2jCGqKon9tORddHe
+WJw0e2iemNDbOWG4lKA9PTeNodWAVB3ajLSh4gc1apbcT33Atq4lpQPG2KbYzS57
+Fl7ilhdXO1MVbccLvmsamJE5LcsQK264kN+OSvJfuf0qMiSkDgcYGn/xze5Te0VH
+VbdlVM/MAkJER/iH6mHroog1/tQR/vodEY/LKI6mZbQrajhSqdNlUSdvIbQEYJc3
+aboHXDuR6wGkT7l5o+/S9/PTKaKxkoj5jLZnSqB2LKqHUMwNS6dssNPxOO8V8xkP
+82PRywECgYEA8hoFx6qMzP8ADKw3kFSlOxWiq7+Np+Zi240iw0CJpC25u0XJfvAh
+WL0rNPJ7Wm3JKOilMmJX6LO/Df+8/lDUcnbyvOp8QWOpRVuINHtJDTn4/OLkJIDW
+rLED6loJlmC6HDHKB4rVNrROJYhJw/nzWMcDtT5LkutGlugfQ4zcz8sCgYEA7fdW
+4iSpOkzJr7iwiiDDsWWQIXnaQ/MO5HWTVKW1OLQ0rhuRGCAI5PeA8KR3bLpt0yz8
+han9uBeX9mwAxQTnkaNMbw9O/ns+LT0ZhazOA7ZKkhu92kl3p8VGfUpNSiUhJ5Fn
+vtOHCW9qpDnarmcuDiBDJ/b28WqGtey0w0NbkjkCgYBUTbV3iq8O7zjwMMQEcUU8
+JtOHZOOJ9NK1yxaw0PddAUMssCdzGWEzSmQSQuGrYpFqumBUBCTCdlzwHf6MvE7O
+MRdjq29rQa3+5dmvFMMiZO6L+OIix+55IgbQA1xVTgdgrAHC0JhgPTb9INE2+gHA
+B0UGRtQALAdfoyLH3fRCMwKBgD1CSfyHYYUflWAGFHRKgPYe5h6oBrctT+0iE15c
+Fi3VviX/9LcYMBH5YRkRzaiQyYrubsdd/nGwqS8HmJ8fZxmHxzpj33sFcPkmXHac
+CKDOAZ5hQ9t4xDPNRhsk0IbAzisodGOcgou8QeCrAA+bgjCiv4b/PYziVnFo0RPf
+OCiJAoGBAJO7z1UiSW90y8xfljY9MdVcOQhYkk36ESShLsMUQv3853g2/B44vUJp
+wdsRnG5skEl2jfwUHLLXItXQxvnt4ydo764PuoiVu62Ki8m5qyK//R5Hv6v5kM7s
+4YVSjQQJkxjtB5i7YVrlFxHS0xs7fMrcaDjMTCpCCWhcM9P8qqEC
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c3.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c3.crt
index 9ab1ccdd5a8a..3c1ec05bf906 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c3.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c3.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAI32UnAStCMbefX4MfICKcGW3TPkMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW4yLmMzMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjIuYzMwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAKllF4c9YFeQZzEuE/YKlcqHfDiS+tqENJ0dFj3cr9Mjg5Ju3JjHL7llZNrD
-7chh79YMinBbXi7AR/9Uz1dcyRpP3nKwiZZsn8oZyWCrmoX9EjeFPRBprWq+qOY2
-uUVVohJgGzZ6SUDxlawTyyOc5oU/wuYb4nmNrBsGT9tntHO5AgMBAAGjfzB9MB0G
-A1UdDgQWBBSMdkINXUsaLlfrfH9RuxuKsODn8TAfBgNVHSMEGDAWgBSMdkINXUsa
-LlfrfH9RuxuKsODn8TAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMi5jbHVzdGVy
-My5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAOy2M
-3ILUvZgwjWCpWc9q/2A2wy4ahFm++rjs9JzKVi+mSGlk60YEOFdJao/UxhFX8+qo
-LNZQxFUStfQDmyiWM48Fa0JZyMKOSiRGdBbreSaHaDI3lpBhetrOCf+d2UsUFfMs
-SW23cfeIDQwGD39xPjNRl+zPr36C5T5+1l1gBwA=
+MIIDTTCCAjWgAwIBAgIUe615Wc5eU32waFUzoL1YQZdQmlowDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjIuYzMwHhcNMjIxMTI2MjEyMjU2WhcNNDIxMTIxMjEy
+MjU2WjAQMQ4wDAYDVQQDEwVuMi5jMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAMwxAzcdhHMJXtF4ZOFQcNgk6cyLzbDBKqlDB2FhM42YIikhIamyeTm5
+S1Oi8VtbkFYt2Jx6WeXchB148AYCvuy3otdo662odmy6GI++E9ocW3p1VdlM8klf
+0ey+L8o3xwtbSxFXPN0p8ajwzJjHziyhJVEIIlGbbzIMvR/9JBgYSYUt30tQDL8q
+gXYre72/hzagAvw8xMmYSEqt0TK3dU34GLfGBnckDXh9G2eMBwhrKYnNlx7JOJd/
+PIQxS2u1hHqXuRjd/irsG1AOTskBp93N+0RxkAmAYhzWI9yW3VJNEEOKIC1nPI5c
+XZxw1AjmjuMREMcIyw2DgkO2WQ2v2xkCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU9I+1
+tg9yxpvUzjDHVneiWOexSfUwHwYDVR0jBBgwFoAU9I+1tg9yxpvUzjDHVneiWOex
+SfUwTgYDVR0RBEcwRYIcbm9kZTIuY2x1c3RlcjMuZWxhc3RpY3NlYXJjaKAlBgNV
+BAOgHgwcbm9kZTIuY2x1c3RlcjMuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQA+xr75jIozxoRBzzjkvwmycHCRRgMaMEOHBAWDefGO
+4b0zAe+eZSOQpP4/MippIDAl2ukAMD2LC5UaGS2cfu6k/I0+8Z3KUez8qi3l7j8X
+sSBujQsVO4npjYN8ppR7cCRLyjjr6EFYbpeYTQb4+L+Lo3g9V40B4MTGIv0x1NNM
+9IJNu/ml1T3t94e4uuAZxTSsKDxkBa50RwOeIQZZml+cosxUVfTra50BGVOROCj1
+WWmOvhna+CAmva6SdCs7Zoz34jucYbUjzM9xpJhP38MNpkD602fUdAw1dzReeiRF
++RVsGvVXubMkaKJyfWgVd3DyCau3smoBAxxgY1Hz4Nmm
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c3.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c3.key
new file mode 100644
index 000000000000..90024890d892
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c3.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAzDEDNx2Ecwle0Xhk4VBw2CTpzIvNsMEqqUMHYWEzjZgiKSEh
+qbJ5OblLU6LxW1uQVi3YnHpZ5dyEHXjwBgK+7Lei12jrrah2bLoYj74T2hxbenVV
+2UzySV/R7L4vyjfHC1tLEVc83SnxqPDMmMfOLKElUQgiUZtvMgy9H/0kGBhJhS3f
+S1AMvyqBdit7vb+HNqAC/DzEyZhISq3RMrd1TfgYt8YGdyQNeH0bZ4wHCGspic2X
+Hsk4l388hDFLa7WEepe5GN3+KuwbUA5OyQGn3c37RHGQCYBiHNYj3JbdUk0QQ4og
+LWc8jlxdnHDUCOaO4xEQxwjLDYOCQ7ZZDa/bGQIDAQABAoIBAFtaWDZp0c6HtDCi
+HPwC+aFyDlmxeHJqUZRNhTlHfAGOvUXKaV2bGAo/wJgoD7RQSEiOFVmYkay5li/7
+YeNGijWsyMF30u0sL+Nf9Momf4o24qRsZ16jQsiaAnjY89eUUcxeaTzWW0gVUmVD
+aj22MqMb626P5Y6+LCbntRthLStLsRQnqngk6Is9D3Ix4mvFppdtFjPNtyz9kpST
+SeM8f867iXDHwM8BCemiOunoupMkRSwpDfd11gWMbGN19uYrus5XtpQI+38GkEO/
+Ct4n9cWVWnpaZvsytrM8my4eRp58ZQKQEHdPbEer7qBVz34m5mtnbH+JY7KRw47H
+YRytunsCgYEA+N+jG/fHLN3RhRa4DZwtfHYYr7BtWY/MnEB5YFrXjZhNQy2xsxpZ
+DgrJP1tVJA4TlSL6CPH06jhtJ5pNhyMikOiof3uPWcR5PjzdkDEYM+bS2Z+/sx2R
+CQKpa2tHzpOIzGrPa1nQ8cOKq/MEL0LUQqbivPa1+HHHe7kxls6Aju8CgYEA0gnV
+fuXlIUZdjPLj1VF0nXOClVE3Ko6f7bs3PmmJU8SabCt2laarBnW9AUWBr6CKMOuS
+ant2mHFOVeQ1AaPbQhjMk49eEvmcWMmX5aH8GDEex6nLdsex4c4uUse5pilwTFMq
+L0f7dES731Tyf0Yk+jMO74QDxy8euZ6qh8qpNncCgYEArQ28+p6hxeX8YgH5O6WE
+z43C/q+8adGfImuGuuSQDGqCFxt3/CVvFV+UrtSQcIDFRyfKG1w7xH42EJqwHC7n
+Q0I0oGjHtlDzSb7DBhfNT5HKvf7iKud9SXmrWwF/LfrS3enLj0cmDma9T3GtUDu1
+Ix+ufF1FDZv4bWTKQaYYKMsCgYEArj/eUPKkzqAzyFVRUo3LMzy7HulL3RCJ+Y4R
+3MNtXIOEoSzFqo2k4S0Yak+hsXR8cO1egNyAnSOTG4rBtuSbaBBu2BRetVtDrB1C
+2Q0zuVY29OoWlCpDltOYdrCfggxqz8dG/ga3w9OqETPGInStBK0LPQXGpJ6g3NmX
+HCSzkS0CgYAR/omLieExU85fGvetxP0mbKjxaq1BDGNLSkvvQwf8lBeU0qFAv4Jd
+naXvXIHNqT0YkJT/TYN5gQrGjubYFd75FzJPLgqWpy5KID1xIvNt+2tF/3o3gNjv
+oojlJpP1B1UzSvotprxsdz6GW3+DxO3B6jFb854hPj1iCgCH6cAIyA==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c4.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c4.crt
index 8e111943b69c..ebe2b268a585 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c4.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c4.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVANl52f14Y2XwcvVk/EWwSt6iKbJCMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW4yLmM0MB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjIuYzQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAKMaYHZ4RiQN/cACOxN9GEyNtxwXC1PD5rD1nPCH0Efyg5SLcDzbBj5YVS+B
-Gl5HIQ3hwX62vnI9pRM8GJOZXX4UGHQ4gdd3B+qbW3n9K29cSqnxulIZiE+Ax6LO
-XwwEzSag1nTEpJQo09LZPusyzXnicEJ/RpC0krAa9g6gnvNRAgMBAAGjfzB9MB0G
-A1UdDgQWBBRsXxUJ6YradZoLR5+cQBXgj69aOzAfBgNVHSMEGDAWgBRsXxUJ6Yra
-dZoLR5+cQBXgj69aOzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMi5jbHVzdGVy
-NC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAlOSo
-5cafQgnZyRFGdBdyovfVfJ88JzH4G1YC0As+6cnZmafZ/pIdUiaZOuDxg9AI11BL
-LWlaX3c68kk3PWpL/vtJJg00QNhxzyHaiLdLbWuPoIcE7fDHvwbuOsXWQv446MKr
-Q6mIe90sGfGCIcyBXXzP8mOTAyTcacHTS/ZL3Y8=
+MIIDTjCCAjagAwIBAgIVAP7+3MOgJEogl1ohtIvDi3QqGQ02MA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW4yLmM0MB4XDTIyMTEyNjIxMjI1NloXDTQyMTEyMTIx
+MjI1NlowEDEOMAwGA1UEAxMFbjIuYzQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQDl6OPoUSVF4fSMHxGZw/wsKEdGZvccwB3grBiMhbfjoRi19OaxjLa8
+eN7pDoGOvsUtdw78U3be/ZbFmsQbD2xvxLQqirQsxWHDbHebKxbj4D2Ta/FtRBlz
+n8OQIQwHFu7g5y76m8z8uh7VwoKEImECvfq52oqY2RE6KLpzW8zkCwRe7E/pzaUg
+YjgyvxumCpd4koityR7qlSqbdksE/CU4u2fAiaPqhPaClQABdgCC0WwDBtIrWP4w
+riHsXkhi/fVkSK2guWVqSKkpKLfpgXHx2NOeVDF1eAejCo2QOzgJegbpTKPaRitz
+SMkJlYRqxETZgjEYQOQO3dpy8rqIgtQZAgMBAAGjgZ4wgZswHQYDVR0OBBYEFLZR
+JfioFbITDCyh12oLrsKu0ndHMB8GA1UdIwQYMBaAFLZRJfioFbITDCyh12oLrsKu
+0ndHME4GA1UdEQRHMEWCHG5vZGUyLmNsdXN0ZXI0LmVsYXN0aWNzZWFyY2igJQYD
+VQQDoB4MHG5vZGUyLmNsdXN0ZXI0LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAB8h4bKPCDWFfRxTQ/it9RqTgwup6AaLvbfUPrcK3
+uZX7UeDOb2glqg21D7KN4MFiT+k/y3MWJk+ztjzyKgf/kDNo/k83rLboDtpQaRa1
+2EhjxiXQOh/JUHn34UL1M04pqthxZGeTiLVl1ccm673lurQ4Zyl/k+7XnMtyX7J/
+DaP7FAOiwmnW4TmT9oqvGSIDD/XVdocccFTHDIROQBBhNjschxKhucfakHVKlQjS
+52WPrfbRpKggXKs+uy2+X+HGfzltM1xf/pa/G1voSYCOLwm+PTWkSjh3Yeg5Z8qX
+vBOr0CcyXr9h0M8ESiM0nSwph8fRRlSiVm0lDDt9/nNhqg==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c4.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c4.key
new file mode 100644
index 000000000000..c027d3635cd0
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c4.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEA5ejj6FElReH0jB8RmcP8LChHRmb3HMAd4KwYjIW346EYtfTm
+sYy2vHje6Q6Bjr7FLXcO/FN23v2WxZrEGw9sb8S0Koq0LMVhw2x3mysW4+A9k2vx
+bUQZc5/DkCEMBxbu4Ocu+pvM/Loe1cKChCJhAr36udqKmNkROii6c1vM5AsEXuxP
+6c2lIGI4Mr8bpgqXeJKIrcke6pUqm3ZLBPwlOLtnwImj6oT2gpUAAXYAgtFsAwbS
+K1j+MK4h7F5IYv31ZEitoLllakipKSi36YFx8djTnlQxdXgHowqNkDs4CXoG6Uyj
+2kYrc0jJCZWEasRE2YIxGEDkDt3acvK6iILUGQIDAQABAoIBAAHxFRets8Ri06dp
+TcNAMf2He46fKQAfqUDwAycQrAxqnGRRBK/EMfjmhm9udW6oss6e/kvoQo1AlAjo
+NTbLSw2omiLv1/S22AwH+vyfAQr8plh75WaYWCSvWT364Xif23WfzjhEn5cU1VTV
+1zdZePtGcnsmWtxOnQQiOuzlfhYA87atzEX8deZ8AUn6jvkiRK4WyZOmHBg1vzjW
+51lyou/8pSCcEAOE6qHh7x/61JYn5ChL4AXMqMJeit1p4o84vYkmYqKEGXNJoFb8
+kYHLGEZH7mEsuSxQUQNpko1IVSJQ+edx1jyAakveBOQK6FNLV8pmTAAiXr9t/v5X
+GQ5OrbkCgYEA/rbOA2zy85TBxgxDmAwZ46k75AqwJwRgycguqJuwQ87KoFD4VO+z
+B2DPi1cy8GB5Zzo/J/G/8+szX2taH3UbY0WLEpGu4t4OUS/eiuQM/jIjCa883ODx
+WerY+ZjF2Uo20fwMmCZO/YU+XtQNz/PJE2+uZjWQ0MQQsH3YVbK1qR0CgYEA5xIH
+Mf4/zgPsZw6700uqBjPzIGNqfS+V/+OkDKBXfxK4EY0Umss2sQRfAYojWa4IUmZA
+AGcSMOC+aMwPCpXehThyUN8QE121xnY11A1FPp4VOR5zcMa2oOUEf+oZXnJvRk66
+hwl1xQpW11VOLu/ZnyXMNRBOE9jpLm1QUxKDYi0CgYEAhiIeOw+UoVEk+fZbYiZ9
+O7ZTI7O9L87OaXRRndje6dZsPiBlpdTUon4IzYgN+yfSxtmViKN2Qi0J0KXnv8kb
+sxtcMVsmjbICOB7NQrb1YKdmbCMsGeUbNA7IVChviA3c8JeaykTiM/+G6MkivO/U
+Y7fcT1bF/CpD361lzztdDsECgYEA26vn6z8qqVBRjigTp+xm1z7MBB5/PkPe9kYT
+z20fwU6qbwUQcZh/hgoByBTrRnaMsslCTx+ty4ndewODySxxP3VKOFfXkJGmHeaD
+hfSXv8/JnWO1m6FOPed7P5/uPjvMTarK6VW1AH2z0ULn8PdUjFgCqu+1SmkMRXbm
+xhLzlIECgYEA99Z0+JqSvpmYnS6WaWZXgCjOk4M7PaPViBPYUm67d78EfIScVzJc
+iPsgY1DAHVzqeeyWSWWvZGgnb0jVK73MUC8uWw5unjwxsRWhx3HvWIyoUwe8JYMU
+P7fwfvowBvf61ShGp2n90khWoW72kaFrQTRPo55SfS0UpBF7BANAM1M=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c5.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c5.crt
index 31d960f5e03c..151b471dc280 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c5.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c5.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAI/+n5L43cIAcNrDP3rvwdoNitatMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW4yLmM1MB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjIuYzUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBANLAWJxq07CPGG6VYspUbzyEMn6pBE0z9Or69wrDOKM7CVM8CzrN7aw8zYDL
-uOZy4cEJxLJKvtw5wQfh/JELcMH2WNLIFNABl5+m+FpNLBGgFMC48NUCfPexD7JU
-PAc7oiaZQ/CFCP+5YK+a5F7M9/SwVfQKKnsMPK3v6zyZkFO7AgMBAAGjfzB9MB0G
-A1UdDgQWBBTKJgGOBVnP8ZNUNlZXD0iNkRRDEDAfBgNVHSMEGDAWgBTKJgGOBVnP
-8ZNUNlZXD0iNkRRDEDAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMi5jbHVzdGVy
-NS5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAbcaY
-9U5E3lyLp1n1UT9ejqHbJwQD+lfTFx9OPou9HifYG9qbNTg92lo3zl2lcm/pD421
-zvqQxz/Eyr18WE0qzzlX7aHuzpy7S9UUB8GnrbZ+oVHWiVJstZlTAOXO+gbPi1gx
-UhiXctx+YQO4V07f7pFFLkzeOT94Aaq6yLDqc4Y=
+MIIDTjCCAjagAwIBAgIVAJcoStt1AtuPNLvpl0+wWmEGynjPMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW4yLmM1MB4XDTIyMTEyNjIxMjI1N1oXDTQyMTEyMTIx
+MjI1N1owEDEOMAwGA1UEAxMFbjIuYzUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQCZHYIY6C3Z4MGWABTJce8rvYEs+ShnRBMOScqujpur3iDYehJb7DdI
+HLkI6JI6QRL6Kee+iuw/zubnfVxt49moRy0V5v4mK0w7QgS9WftICcDVXbFnPr94
+no13rw4PMfyRkW5cwSCel5AHjI5TH5zST92UdELS/OjsacQ03nG2qyvvBj1G+D7J
+NC0alfSXuV4M9QvOO0wNYQihafURUuzBbqxT7fYawK7YbmhHWyK8FaxAPF9fru+p
+BnfsFbdUaqtLKhNlwHrUCKbeOcWalzFh0qt9mXYy5wux3Bqooc/e7wtr06HzBpfz
+gvXHtG5IQiRs3q/dJNMUJf1qPJYc4ljfAgMBAAGjgZ4wgZswHQYDVR0OBBYEFJeV
+l64BRDPdj9N2YXawoUZtU0yBMB8GA1UdIwQYMBaAFJeVl64BRDPdj9N2YXawoUZt
+U0yBME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGUyLmNsdXN0ZXI1LmVsYXN0aWNz
+ZWFyY2iCHG5vZGUyLmNsdXN0ZXI1LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAkJebbN2CJjyyw5GKKtvW46NBpYxgEjbwEhYyNDaq
+bUniBTwXMdJIngsMIWmZk3ST7SeIZOV9Vafy75jXJ6eTjp/6lejwEs9ui2Vnk543
+m685ztyiqh1fIlFHK0Yc70DcaJKEA8XYWGinc1ijpAoeqMR8NwwS060dF/IZzXAV
+LRGtR2IS9W5UTM/2QCDj3ms5HiVt5KIK7UB9E4PudZebTU5veT9t05rxtV+Rk7OY
+ErnfdhFuhx5I8JTf8lJ/01BOrZ9b4k2/OG9Cx3KngYlctC4cUO7m7FURUNBeE0pm
+LdyC28Bf0bgRJWTuh1GFW8dEaXMQ9M5HiJ3VkUrm5LW4dQ==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c5.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c5.key
new file mode 100644
index 000000000000..8677818ee4c7
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c5.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAmR2CGOgt2eDBlgAUyXHvK72BLPkoZ0QTDknKro6bq94g2HoS
+W+w3SBy5COiSOkES+innvorsP87m531cbePZqEctFeb+JitMO0IEvVn7SAnA1V2x
+Zz6/eJ6Nd68ODzH8kZFuXMEgnpeQB4yOUx+c0k/dlHRC0vzo7GnENN5xtqsr7wY9
+Rvg+yTQtGpX0l7leDPULzjtMDWEIoWn1EVLswW6sU+32GsCu2G5oR1sivBWsQDxf
+X67vqQZ37BW3VGqrSyoTZcB61Aim3jnFmpcxYdKrfZl2MucLsdwaqKHP3u8La9Oh
+8waX84L1x7RuSEIkbN6v3STTFCX9ajyWHOJY3wIDAQABAoIBADUAI1/2wqRYKfp/
+1bIzZ5xG8Td9mbBBPq1j+q44vAcekVUN0FwUZMBn/N7qDm/uxaBlsAvjb8rYllIM
+MVnylI2s8ZzSxCe3NiCPBi/7PUGe6qqQz49TB5k7nPGQXXgCEBDdfAvMaDutcBRE
+4aFBYJnmPmecZOM+22u2Ys4AL/GQPsTb9bv1Uroac9m1HP9QexpXgMcqoFgyRzvH
+CcGaPNzp9HHKyg4LyyiAIqYeu2XePhlw0z4bCGwEjPpnC7rurnMvk8pQvbvK1wG+
+J01sLLYXQRKlK17KlqKMHaZZ8C3ztKUGG1jkJ/yD+GWs8ZPBJAzf9cU9BBo3cmMR
++8nBTd0CgYEAwxFWEy032KrK0P7fs7zBHh+BhdzRZOkF9meAhxMvZu07uDCJTkIA
++PjP75SIcAm+wLyclfYoqgrSRrW85VHPonpbiGxIMYk8v3ppDxi6H9vTuizUOHo8
+yo7FoAoEq4j5TLBbxKk8hP6DbSiXgqzGaBP/lRvnNHT6U6BjtpbGRaMCgYEAyPFt
+n3ftfwwwuOJKOOIs3xQIWFQD44Kk3lmBjC73yJORV0FHNyos9hvMGBB83EYt6V40
+btYEvtDG2UmqiY0U58OeyrB5YtFgo5caRaF70KZsR1WnbqnpJSeu+o+KxSCuKjMv
+aYSiM2eEJydQ4juvSWDGvqhnx/0u5ShEnxdH+5UCgYEAtlu3nxfPtJL+JeGPmcTk
+uNd1S4JCktaxR0g9RIlv8AXppWhihDF6HUblhTDTWAIUPLRWom7MVyLkr0vo+my/
+8kH23hN0bjLBArRaEM3p+iyUw+ByyRWlTSupntKeyQm1HbdlIvWfaSlSDjM7aONC
+XDwN3ASI4WZWOuWfMivD0YcCgYAVH1O/pJKhfT2N4bOdoSDIXkm2RzaVbsE34j68
+5p8DRYfMnT8ILHXHdQqSKQGHAhWHZ3CpifO7vyR36Md1aBAfgK+rbLuFyK6Y46RO
+V3g1aZkOMpHEDzZDwpUhH3+d0x6brv+33spSDvqURwy4iz+UvgiGOBuYqSOJ0R7N
+flwBtQKBgQC7upWH47PQ3a53plWK/r8RNbqSnpzhwni+1hy6AhJFXYeJ1z4MnpJ8
+sE4tJJvg/A+8t64/6x1qteiwXpN3YsOZFxEP2LL59cqUXwsP2VAkYiZdl0XrGuE1
+wB1x+o8XRlTfSbdZC+fWNZpysBxjJOD76EVlV4Qv1nF1aM9ma1Amvg==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c6.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c6.crt
index e8ffcdd543f7..b8d44d75425c 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c6.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c6.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVALQXuI2M4/7RVxeg9uI+ABZLvK7UMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW4yLmM2MB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjIuYzYwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAICwXz/hU7DraQWX8TJWd8hva4KzTKOpq8JjnbCd1n3ovAqbDP6GkgVAJxY0
-eMFwP+u4jN2JZuTISWIUOmLGosyHpXsXQwbsvDLUC0JPb6v0/omkLFsrIXgBt2xI
-y7Bt3eifKUmUGO3ePi+LbgKYq7YNy1uVaFiYLR3PRul+TCXpAgMBAAGjfzB9MB0G
-A1UdDgQWBBR8+r3Wp4Dc7QfmCXkTGgz3gyfP0zAfBgNVHSMEGDAWgBR8+r3Wp4Dc
-7QfmCXkTGgz3gyfP0zAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMi5jbHVzdGVy
-Ni5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAc3WH
-mYDHi4OWsar/4jcqq1S0RZ5lGQWtJIpVXjw2SzH7mzfR9BrD48fgwIrPJkpTXsi5
-ATFoEZfVQluifS7qjg5lPVmRkQ0FvNMxOXq97tyEi8zqeWa3TC/5brUx64aeyAB0
-rokzCaTAHY38r7b8eDIFpOlQXj8CowjbXx/rvgc=
+MIIDTTCCAjWgAwIBAgIUR9zy9zaKszvdno5SeBTB8Hhcy5QwDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjIuYzYwHhcNMjIxMTI2MjEyMjU3WhcNNDIxMTIxMjEy
+MjU3WjAQMQ4wDAYDVQQDEwVuMi5jNjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAJGNujkGJ+9CoUSssDOkZunkTx2LetJD5Fr+VJ+aC7Oim4919ZbOF6aH
+Uwe5RuHAmc0+l0Qa/bPQhgIN/lDDerXVDwlWHJcNkRrN75tnuI5x/jQZXPTNPWWG
+m/cSNjPing115PtjYGrammxmdkfrizH/MwmlTuBF9r6nvTQNcWMs2I36FVf8XzTB
+EOvV3jYG5HueTLZwI8RPfObjDzoeEd+eybPfFL4bW5nPH94Lpssjzz0UQUrCwx5I
+udxsO4t5LN8x+wPut5zCiex3AWlbl6Dqh2EwaqT3Q5uKmbWRv9QlFpQY40xpjLeu
+WxMHA2hqe4YQWOvdQRliCw6pMZN0GhkCAwEAAaOBnjCBmzAdBgNVHQ4EFgQUI+K4
+6UV5R1OqMR/VU2R2T8NCQqMwHwYDVR0jBBgwFoAUI+K46UV5R1OqMR/VU2R2T8NC
+QqMwTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTIuY2x1c3RlcjYuZWxhc3RpY3Nl
+YXJjaIIcbm9kZTIuY2x1c3RlcjYuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQB9ri2rP34uKk07vLx7Rad3axdNAiMUzFyDVQA3hIur
+TCZE6ukwkZnj4cCurD5OHW5R8bNgm94iSVaXk6JHCLKVfaxg2jJKKABn+46qeOLk
+mKHU8uu5Zb85l7mnRJPHf9S+G0xgACxY2SYk7ota0F3Iy6pXCX8N+53oBm1mbWTk
+55qO/h6P0031iO5yFzaADFpJj2z6G9OFSrU7LucebJTpC22utr6QaSBGiY7Fb+/O
+OKJcFYfIU+YvrsJ5v3yup452YiM5/30we5euWVb6bCVlGp/fg4halvZb5CnCVM5m
+RCTDtSnG4E6YuiZ3CKiLnrBzf0WrnyAWGQFAbDVTOFDI
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c6.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c6.key
new file mode 100644
index 000000000000..7f92d17f5b37
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c6.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAkY26OQYn70KhRKywM6Rm6eRPHYt60kPkWv5Un5oLs6Kbj3X1
+ls4XpodTB7lG4cCZzT6XRBr9s9CGAg3+UMN6tdUPCVYclw2RGs3vm2e4jnH+NBlc
+9M09ZYab9xI2M+KeDXXk+2NgatqabGZ2R+uLMf8zCaVO4EX2vqe9NA1xYyzYjfoV
+V/xfNMEQ69XeNgbke55MtnAjxE985uMPOh4R357Js98Uvhtbmc8f3gumyyPPPRRB
+SsLDHki53Gw7i3ks3zH7A+63nMKJ7HcBaVuXoOqHYTBqpPdDm4qZtZG/1CUWlBjj
+TGmMt65bEwcDaGp7hhBY691BGWILDqkxk3QaGQIDAQABAoH/Ao77NwVkrk/9hbeD
+YWUCahoZSeupvQh0eyT9V9DdY8r38Q2hw4Tb5ZF7ZCddl5pLGdB/2DKhQwfnUyHr
+1n+ZKJkWKpPY7M7QPihgjdrE47wdO802VRdGfQJgu3cWfNL+9D2eSMz2ZOpyzHUA
+n0MEr1xNCV5ehJK2Q6OvL7sMwzF/07IAZDI0hGEs6hhe2FbwsBBCOXNtjUzE7kte
+ZarbnvfNUaJTMbxZbMTEaY4wPX6uAXv7LIU+z7BPloSXY6YJQTWOJ5dXE0IA5dNP
++0QTM3Qzv/v9q2vV9bwV95Xs0EAFh82oB9tfyW5w+CrmMbXoRXHETDgLdLlr07pC
+yj1rAoGBAMF2uX7MtrCGpykBcFYrXp7ORmDozh4Wg6Mmc7Y/AvnlM3lpbbEBDYNA
+F5yOS/91cGBvtvYs81Rm0XFjUbPdWiyKXv72kRep2SPg3kmMMfcxtOjZZZx97VT5
+hf/Pn/gjJQM+u6kEZKId1zNXja1cMoBrQpujQtL6ROBiS9UO+U6HAoGBAMCaZsqA
+B003gLq+jjYbJkQe6TDBtcyg1gWD7Ib5DJXLb7G/2vu5mZZTAerLnXoSYqenluzI
+Yj9l4hxJ8+JLIz64pOdaXevQ5OtqqTSf0RQYOyEn2YyBXvVlHnH4opSKl64J/UO5
+ClQzH4K1R+wiaxiVBHaev9XzXQtOkT8Sp9pfAoGBAKnOfoQoRNXuanxYcfSAPp3e
+6fEXhgrssgAauQemiQM0RRj1C72H8gdY0yCr2VLRDqDmjU3IKI/Va1I93bGkwQhF
+L5pUawe9IgUW06T0c8rHi0Wxwzk3Yn/qP85Hu949YrhVcLuqSnXtAT1kRMt8GCuV
+ebJH+1hZKOlV/b91VCezAoGBAKKY9tRXt1JR3ARLYMV+sBRsQBMgRbllwqjPUbw0
+vpIrN7VIsbc3XdkRZJnyoZRvtwewrPJizMaW3YdiS1ksc6m8ucJygL3XEZtI5WX+
+0cR0HF8GRFug0Ci84DH5ZE+gnSKZXk1xBq5EpoJzRi+scttC2ozFwk/97vvfrTEh
+L8FjAoGAeIgcBtnefZbvrhvou2siMLeYpSiWlpGYlcnBXw7whR1axfj1qS5l4JIp
+I/IMu0fp80cGoMqqCvQ7wU5hz3LBN7lvG0aMlqrX/sVaBfmZOwToOOslFvFArCgs
+tizcy4BvdZlFckoiAN8yXIBVnhLk3DdiT/C9iFJLekOU+YDJd7w=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c7.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c7.crt
index 242a81a0d29e..de496e58e218 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c7.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c7.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAKWasxMN2xGHyCWP+sEXohZanUU4MA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW4yLmM3MB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjIuYzcwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAIW2QjCSqW9RSVtYf8XyuveF9dozsAKdDqN6r9n9OpUrkdozB/tguOPvy+yS
-aHN/139IrP1QYCqaiaUBevUQzU1N7/yYnFXpdGiNUrkWXy6wTSuZhmBm3r2y2XwJ
-rszGz5/uVpQc1njzZssON9Ug2mzxKvZvs4hUXnX2GOyujx0PAgMBAAGjfzB9MB0G
-A1UdDgQWBBSLcQpmKblNBFdAbAfRm3OAg0CSkjAfBgNVHSMEGDAWgBSLcQpmKblN
-BFdAbAfRm3OAg0CSkjAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMi5jbHVzdGVy
-Ny5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAHVpk
-qkgLywJFDpCTTt+vtpsaYlevMdrNAKI9VU3F5NQks4goc6HvzsDwttFh1BrWXitp
-NaxEsbwk21lLFTZ7GlzuChFHQWsZIHZDiKStCEiN726fqcjfzYGF5wtca6idD6Fm
-xAhf41BB/O9kzIJasldR58vtfepGXPLJWKTrshg=
+MIIDTjCCAjagAwIBAgIVAJvOpFC8dTItEeqG65GtohGlA2Z5MA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW4yLmM3MB4XDTIyMTEyNjIxMjI1N1oXDTQyMTEyMTIx
+MjI1N1owEDEOMAwGA1UEAxMFbjIuYzcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQClr0f07HkjIoauVMeOH7bXPqLHBMPtsS7CahwEsJlwpbEluObR4rW6
+dnOCZ6dbkhIiIDCePL4/vHsxjr/8wgp7M/S7itL54VK/wdpFz2EydhYfMj45AlFM
+d9xBXMzZ3RxQD48En/YodVLyhR6/QC5Rf2pkcR64bsntJkltkxEBY/GVuRlcAvtf
+/jz5Jijz8tqz3MheqRm0cDni7M+4zT8Ctu2RQxHPXbqAt91QXe3k+/F2fo+xNA0e
+g4d6rAzAZ6qqXrHY4zK67S+7o4fsCdnQgVIabO3IwjeoIUj0+GOQg1rmaPuh1Xvs
+b8AdxUT3VZmWIoLfNKVmEzEKF5jJcDAnAgMBAAGjgZ4wgZswHQYDVR0OBBYEFPPa
+yf6p+k7sbzAaWZQYJ4pf1VoFMB8GA1UdIwQYMBaAFPPayf6p+k7sbzAaWZQYJ4pf
+1VoFME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGUyLmNsdXN0ZXI3LmVsYXN0aWNz
+ZWFyY2iCHG5vZGUyLmNsdXN0ZXI3LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAlY8eVTJG0WIT2qlwQg+win+bT4eqH/un/8kKKDwL
+puglPK1qbWLkZ6R7qcuaxLHC1zMdUM/ffaA1xYUZS7udD098psLHVUuziQYJT6ll
+fldju4I0hDoAudkMiD78weanL0FRsecHu/aEcEa6GPaqJ15gS4uNxxQc6sniaQ6u
+FGHZxWOM7bf392Ipcp/Z7BV/y+P3ncb+R/a68/BswigjX59KDGaCfmWHxoQCLXvC
+k+gt2ejm4I8ulidK4vko7MhzEL0RxncdGRWF/A3oqYnrysZvDfN3z6cEpEeM4S4g
+wlqP0mpPl0hq4QVDYoM9i37k/FoTy8i62nnctrdJpMN+eA==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c7.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c7.key
new file mode 100644
index 000000000000..fc447af514a7
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c7.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEApa9H9Ox5IyKGrlTHjh+21z6ixwTD7bEuwmocBLCZcKWxJbjm
+0eK1unZzgmenW5ISIiAwnjy+P7x7MY6//MIKezP0u4rS+eFSv8HaRc9hMnYWHzI+
+OQJRTHfcQVzM2d0cUA+PBJ/2KHVS8oUev0AuUX9qZHEeuG7J7SZJbZMRAWPxlbkZ
+XAL7X/48+SYo8/Las9zIXqkZtHA54uzPuM0/ArbtkUMRz126gLfdUF3t5Pvxdn6P
+sTQNHoOHeqwMwGeqql6x2OMyuu0vu6OH7AnZ0IFSGmztyMI3qCFI9PhjkINa5mj7
+odV77G/AHcVE91WZliKC3zSlZhMxCheYyXAwJwIDAQABAoIBAAK5rV9T/HIdgUWg
+0199Gg7UUu0fIPHp9jDweeGB3iRCNXW/YtyU9BRLGn9MoS4aav3wtZwEt8Ron0Yj
+aPkJYA7CvgXPtMSzdh79W0/Gp3yjAJ4mv2tEpH2zgwX4Ns5mmq6ecuXKk8v6khC3
+K9hNf/WLOP7J9Y1nFR83gAGzZ5Y0PdtDA/EmpIqFofIewvUzzSYngZM1nmPgGWB9
+BGXbQL9eAhiV2nq6MO+ow14AGlmccf0Kt9D/3AJbnyX0AYLEGUYQeAbE4W3rqmX7
+Lk0p9Z8qZqRR1GMx3u1ZmE75HgiZl1s20eGVEBUKSoWFzlYUaOZGAAmZSZd9k4kO
+2HYjgnECgYEAvnLol1H43VP6WRtg2xLgYrjD0IdM+hoKzhh5PO8voLAuorXCpAtO
+iEfHXBUfCs9SUUF0DUSRwm7ej/xcXos+WQd8b/BPS9eGao95PG2DZZRKym8k59Hh
+YD+gQYWydY+Yg17CUzbFC8ltIdRsH54xfOsQme/DGVayQ8FTyCfKsOUCgYEA3rZR
+aNPBhckBZ3y2Uxy8ayrpiXtpCtNGQG43hM5yuqwegVYNPLUtMrufhNmH8lnd9z7Y
+6RTYMmlwXTwEw99JmzchZ5xeqPQ9X53Hb/vZ2/udKmgzbq+S64qToiJ3j44RUtOa
+DIm2RsqUiHudkp1bcvKTBgCzwgUgkISEwp/A6BsCgYBkINXYeOh2tOgyYQqTI2GF
+hc6tm/UEqRx1LrjPKJHJ0dczBIzI4ojvbY2LBC/F5ypIPjldoGdh6hhpayzFKEAn
+XqA12gpWvYGWFMiQ1KL4jPalZ7nnh5kOmWUQDdK9rDlrGC7iwTDn+Ya++oe+N9Wc
+75lS1ZOlhsdCiCocWY9fYQKBgQCGe6VRHX1gYe49JWubaGDGfdUjtafKZ6WmRme0
+BE33NvP8uY+otuaiGfXLbOfQij+kiwjERlf+qPi29AUUmgKCfZnoTN5EO6fL0ySZ
+FIWLdraC4wUbs12OadGYOCPmRa1kTG5bX9T3jmB+wdZ2sBqGoL/zAf2KDTqKydq5
+A2qjPQKBgCF7YjFnV/rMo+eKwJVSaKIVenGe5Jehf2C0TZr6f88pC+IuQBTLslvH
+Yih1Gq5CNo9IM1P2NdJUOvvGaHH36wpOc7zB+gUh8F8KELoe2XdP52RiNmkGVvsu
+JHXCmwL2Q7WZUQh0DkOtFFczGb+5R5g42JvYGmIZSBBYpXv5CWQT
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c8.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c8.crt
index 8beeada43809..8a9668e282b0 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c8.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c8.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUXWB0aBshcvW7Us5kC8nYweCwm7IwDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjIuYzgwHhcNMTgwNDE5MTMyMjE1WhcNNDUwOTA0MTMy
-MjE1WjAQMQ4wDAYDVQQDEwVuMi5jODCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAr12D4+Xsjy/okhAJ1kDVb/WJ64ocd2GdWQ1iQj3LUrDtw/KFlWwCk7OaNflh
-xxQKC7/Ow2Cjor9D8yj3/arHH8kKYG9pMvs8BTMVLhU8kXCLIziuF5s/fzBNRvhh
-cqTdjcZn1fLW8UVsMybHgdEQWL7Y9pwxnKuzukIga2+lOU0CAwEAAaN/MH0wHQYD
-VR0OBBYEFMJManm4XJjT1UmZ6pOR0RWsSVR1MB8GA1UdIwQYMBaAFMJManm4XJjT
-1UmZ6pOR0RWsSVR1MDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGUyLmNsdXN0ZXI4
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQCA1rWP
-pNBDZ/vvZHe4FpXS5jWj1qxlcr9bGr9CCpWtb8cA1CKEi5bBAN6gvt+hoFw6/6XW
-wz+Az3BteusKEFHm7nohr3xswCdffJDXY7olHhNu3j+r1RtW3YXxfMlFsCZxUikL
-zOD5OVDBhNPja0rt4zUzoa54jg0xy+14G/dfWg==
+MIIDTTCCAjWgAwIBAgIUB4t0CwwWjeH5jVZnP4k/zauS2wgwDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjIuYzgwHhcNMjIxMTI2MjEyMjU3WhcNNDIxMTIxMjEy
+MjU3WjAQMQ4wDAYDVQQDEwVuMi5jODCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBALLm191VZfD/2gOTWLWMMaDnkXcyvCXTGauIZ/a83mTOaf8WBPE7Et5K
+u8jGL4yJiprdFUoTXYEEez1X6akWVSPX8dfGv4Wpx8+1Z+TEzPulBrVH2oPaChh8
+kzwFfVRli/+YUux4J1lvRpcS0AjtScjiUizbrwG25oCgajw97j5L+x6iNeOW1fDc
+XMIvVQnNv2hUHX/othDXn4t2o6vmD6IelAtFbPnjJRjnTsHnuMZQW9u3DKUiR62T
+uXksw4G3Y0R/vFfLD76eJLu/tt3RsL27qz0UtLIQNNK1UZc0CQ+nRxjj/hER8IRv
+PsYBF0ZN/i8+2MVIZq4sVWAdyoyGu0sCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU2pjc
+P2VyYCZHArlSAXLbq75OYzowHwYDVR0jBBgwFoAU2pjcP2VyYCZHArlSAXLbq75O
+YzowTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTIuY2x1c3RlcjguZWxhc3RpY3Nl
+YXJjaIIcbm9kZTIuY2x1c3RlcjguZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQALlCYwKm1wbih7kpcif2t4OEucbKAS0ebDjXxJ1kxU
+hN0jbMNNIe4e993O2hAaLyBES3V4Atlt4st97ta/ktB9oulXw+NeGqZJj8b5vM9d
+pBtfaC0rwkDhwr/ZnVRCYhOSrILHQvTpYDqt+5BiuqoZbAIIpvtBpL2VBsqBapfY
+jh3JLMtD+Wr6TrMPVhezYzR8j1bcz3QpVQYtrvfA/SH7BRPZDtpdDwUA1YrCevJs
+6wI/gGQfbCwpyNJRKT1CdmhoArdCmqcG5eeWJXH4MjxyL0CYslzihICfnl0G3HU3
+zAzVRLC93zxEmol9zs07NidxQu/p4kr+abp2sSP8aGz3
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c8.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c8.key
new file mode 100644
index 000000000000..f81b716b3f69
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n2.c8.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAsubX3VVl8P/aA5NYtYwxoOeRdzK8JdMZq4hn9rzeZM5p/xYE
+8TsS3kq7yMYvjImKmt0VShNdgQR7PVfpqRZVI9fx18a/hanHz7Vn5MTM+6UGtUfa
+g9oKGHyTPAV9VGWL/5hS7HgnWW9GlxLQCO1JyOJSLNuvAbbmgKBqPD3uPkv7HqI1
+45bV8Nxcwi9VCc2/aFQdf+i2ENefi3ajq+YPoh6UC0Vs+eMlGOdOwee4xlBb27cM
+pSJHrZO5eSzDgbdjRH+8V8sPvp4ku7+23dGwvburPRS0shA00rVRlzQJD6dHGOP+
+ERHwhG8+xgEXRk3+Lz7YxUhmrixVYB3KjIa7SwIDAQABAoIBAAKZoUyJt7QTe98u
+bo0g7v24L+CMH7jrAwX/swcegprzdw3qi4LYCXOqoWUPIFTKEPYkPXF4q5S69HGV
+TAOVnAIr2st4i4MNTyahK9Pi2lnT8a+DzlS5QAlQEjG+oK4v1aOO2rGuI2BE4kFC
+2Dww+rANPgWB4d7EAkCfVDvw6HwGR2MdyMbDNoyKB50UsCqHcnlKWZsoOy9oc7J2
+IHnEb9tthsnJRcZXnsaukpl6mq1DK9RO2/1swCP+tsUbt0XaFTKsQKfLlRMWdPZQ
+OtFVRz9BL5DYj0XNjF7/BLCD1B2+2bAmw1saEJLpp/PzfhyWChEqQ6cTZL+3rBTB
+9NV8utECgYEAuDUW2nV51dLCpCyJUnugD9DhwDD7DYWHfoFy9cFx3kqmwxGqtrMU
+CXahQrzvH9dS97MdNG303vNpyEQxNxsy3TOc08m7FmTQyKgHwEstY3oDku3uhkt2
+sweeCl+oWObFSdnlYen9PQWxh+mSQKHs9Nj1jAQaplCM4fa4/je15xECgYEA+KBk
+0WfvOfgMgqrMx8WDjlXH3CoERvHPfv8K0Y5KXbl287I7A8jrkLwdqAZCmcaU0QO6
+sDo0HFTgxDyDCeVqCWyMi9FTEPTfOB/C8oZAs8pBNs4972l4RICsH/fX2SBs9ZX0
+IEmjgNazRL2sgOJsavqMRYp46zRBCJiNW3bulJsCgYABjqFU021mO1Pd42p9wgHG
+YHOoFkHpCVnt3dZE8Y+u216xg3K3/KeMtO4EDGjREhbndSIoEU3inJgmUz0/RAGY
+ERmgSLJGLC//ojv+iiQmFR/2nABKLlUgik3xtIFNzxWX3DmuLDYxz9yCU50i4ruI
+DlV1C4vWIBQwzM8fhwUIoQKBgC/J/Lb3iFcXI6h2FhiMEYZpXJooBVxxnFWnxjeT
+Zx3uyTt8VbWbiBPw5K5oTaLrNa2bTuYSufN8KRvvGazsSN542+cAHDxVJRbPnBQr
+1AH67WAQcn0zQ6e/J9aZOAcHIYpvAVa3cNMR8bFT1/WKEBbcA9xWCnoDOEDjucAL
+8z3jAoGACSyWqYVf0ElY62Kw+ij0gW2MBVQmPr8VR30Yv7nbKpBZjeL4ekhIEgR8
+/mzZiUqO8Y2taQ1p6B0dRYek1kpdBhf6HTPO8YYpCN8Ff06PwEtzouak/55emHLr
+/lW+E4AHdbX93QuWtVpSiq2mikssO5tCbZVwNraE61ybJk9myaQ=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c1.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c1.crt
index ce72a9419b3f..3fff07781559 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c1.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c1.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUeWtVaCz/N70J5xpIy86NU8PUhSgwDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjMuYzEwHhcNMTgwNDE5MTMyMjEzWhcNNDUwOTA0MTMy
-MjEzWjAQMQ4wDAYDVQQDEwVuMy5jMTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAjMpw8/41N4mhUc0rfxK+8ykcANdc3iJq9I4vyKR6vTHMJX0Q+kEKLpvN+IKN
-Sq1LmfTJJMJx7vljjK6Euy0BncOFTIq91XehVky/ZPZOrLbJutPbRjCFGOwxdYqU
-WMh1YuniQBV8y3gx/6qn0675EJQoK1iXX5xoe3wrqUTi4+MCAwEAAaN/MH0wHQYD
-VR0OBBYEFEns7Lx7ZFHascuMFwRNx1HgOZjWMB8GA1UdIwQYMBaAFEns7Lx7ZFHa
-scuMFwRNx1HgOZjWMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGUzLmNsdXN0ZXIx
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQBzLfJj
-Y0VUqP0O+FmSmE8xeWchvSG7XAKJHkHPnUeFJbq/duSHVAqkIDMNYd9YvHHX9/R9
-8OlIfahOMy/j42E52PSmC6KSeotaUhsaN/RJ2WdgTS8+lnIimYemi1CbXpVgdZgf
-lBAfF56ypKp+xZ2gSQEZZdD7bATa45LF5ZyjFA==
+MIIDTTCCAjWgAwIBAgIUTtnXtmROb8uKWlmJr9q41UElqK0wDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjMuYzEwHhcNMjIxMTI2MjEyMjU3WhcNNDIxMTIxMjEy
+MjU3WjAQMQ4wDAYDVQQDEwVuMy5jMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAJauAZu24APbVHtZ3qot6TmqQ4s7QEnKFz65mQMYBiTIT26Y5jL+Vy8V
+veC57X7myrgk7drsG08+jfLTNuNjlqngNFEF18aGa8hKi6N5HbBL82/zCp9F3gyA
+WbjYAJJn2oxEdOXHlK7A5c8gwByk76jTnW4vmBZTtvymAZdYAouaR7CPlrzkh9VH
+S4AOJQd37JLupDRAVae+I4j9E55YeCG0ekyzzQWwOQ9fN1B97wJqLbE+XnTlnNrJ
+QEJCXeVf58Z4tb+2BcckgOFNebx0qEBWt8RPE4fSSsCbQy1sVdy6Kmv57TJgojLs
+IdSPUqT/7n6majR8F2PWEW58cGCz1j0CAwEAAaOBnjCBmzAdBgNVHQ4EFgQUak87
+JMKh7rAqm6fvREPMB8elvz0wHwYDVR0jBBgwFoAUak87JMKh7rAqm6fvREPMB8el
+vz0wTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTMuY2x1c3RlcjEuZWxhc3RpY3Nl
+YXJjaIIcbm9kZTMuY2x1c3RlcjEuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQABPvkNaD7cYchWR24StYKjuqGORp/kxjoboVQuM9xG
+klCJ11L/P7RdsUl1Ku5YDp7OQ0UNROyRZgA6q7PwVWJBuZg0eSv3OpZhVoagKr9A
+wurgMkw8jKlS5cG3bpZwMBvXhi5isFQjN27+biNs8WT+egXzNCBi9AvKvdZzuJ6n
+2e1NUoFVApQVlnmijZxuDjsD825whGd1hBAb8NRPKcu0Oi4+Clho5jCUo+YQqioG
+HC2ZoiweQcPsLun9pZOIdjKexReTTkEA8n/LjGvNg4XrBGJglRDX1vxjKZfeqh1l
++O6NyMKHu/d2nNmTWDeHbVMPf8q028GuedBRlnVqMe3V
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c1.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c1.key
new file mode 100644
index 000000000000..2044add5984e
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c1.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAlq4Bm7bgA9tUe1neqi3pOapDiztAScoXPrmZAxgGJMhPbpjm
+Mv5XLxW94LntfubKuCTt2uwbTz6N8tM242OWqeA0UQXXxoZryEqLo3kdsEvzb/MK
+n0XeDIBZuNgAkmfajER05ceUrsDlzyDAHKTvqNOdbi+YFlO2/KYBl1gCi5pHsI+W
+vOSH1UdLgA4lB3fsku6kNEBVp74jiP0Tnlh4IbR6TLPNBbA5D183UH3vAmotsT5e
+dOWc2slAQkJd5V/nxni1v7YFxySA4U15vHSoQFa3xE8Th9JKwJtDLWxV3Loqa/nt
+MmCiMuwh1I9SpP/ufqZqNHwXY9YRbnxwYLPWPQIDAQABAoIBAAG8u9ppv1qloR6G
+KLtzKajQ8VQEbBDtulrhlOwLAncfs03yuvGEXfwpY5w/SIezOkvpN3zT4A5Uo4C7
+NeGYeL8Fc+6QAxVDRrBzSGARuc/m8EWU9CuZIjhB4PokM2cWUqS8Kh53SjWOYv5P
+KFsSlvJhWwO1zl3jiGaQgdAwfMcGc2hZlEcpa3yFKh3Qb/mnorbnz2FQieNQmo1M
+5/Y9+W/MZJgyu4Tq649lmCyfq4AsnSMaaehPmiX8iPY9Uw+4zqYupI0mTMxNVoNA
+QXzl9jJafciQfmPWPPs5K0hxsTWMONmCKyKX4MhY/APFMe3YSOik9h8ImpOOhny6
+ljMkb8ECgYEAuvbM+e/W3iHM1XsrIft1DtM9sZ0dH7wuaeSeurv6ZDjKy76/dezf
+32HKsU8SzvWLywh89mmeAd2XFEhE7bPpZygh5IoTSvd6TfLsn+8Us/7k9HfoGNf+
+DqZ5Oj/4vlcl+DIunLJKpFqiua97unw0PIiOxTG3U0Ww3KftW8aKogUCgYEAzlFY
+bAiDnjO7bl6MwY8OiA0q0fSKEg5+sdB2Ctdpb1QIT+VSn5V68ghCaECfzQdGBYQu
+bghUMFZomeSdudVaUuI0SCHVBqJ9eMvUvgkw1AfADEICPAgcmNh9Vx5zcDUANSm6
+kBKl2xMulhoYkzMF2X1jAZLacjVaUM9pF0TWgNkCgYEAthsSCBgeN3rAouc3WgAt
+QmVq4AdiiLiolKgEG2phNyFCFjYTEu/IFzKPVSjp7v+oS/hWj0TMjJCjSTQjP0xP
++RRT/uCmbgyi6i3FYPpkNfX+WEg6n4gyg16PEbehtvP3ncpzMFlQ7skBWs+cOkYk
+Mxnnxhbos4lb/XDrdsrp2WkCgYEAvULj4R6iVoLQQnvblUe9JdMmX7l62vuO/2Di
+mClpH+YvRdNxStFeGh63hFkb/oHYmwDZcriTOlBZ3ipZAfsUw4mcOjtG5rIydZhO
+DRHzR93gL55x3E34pcGaoUtezCTijuI7sSRaHzBmgO1pc8bPa+bY5r2nRDNrKUxK
+9DyCpWkCgYAkXW9PcgfnV/7F7tGDJaCOC5QOQ8mDqYauoNKlUHGepTfaiV10jVSw
+7m5PMn9lNwMjfeBoP5PLtn6CT9HPrMSZCh+O1HvTM3J8GcYRsET/mxZzhldJgN+g
+7rfjCQDVAITIDCJ/yJnviU//ejYFvXsUf+R8BCmLOeU/cqkdHru6tw==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c2.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c2.crt
index 219424ef4c86..ddeeb7588604 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c2.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c2.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAMevBYYMfrY3DQ3H2TBDSH5DKJYdMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW4zLmMyMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjMuYzIwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBALEnh4sMjt8W7vwIeW/PirZad6tqKbiCeGyhSWNL07VGDKUsAfF6RY6voyGj
-Bow/fPxQ7OXjlw5BFEgO3DxlTbxLcNk06JV7d4RrfHhCGF3R3x1L/icTg+rB+QdA
-aMdHAiU6Fme5qGUZg7k2C0JO9mTnJO4tIsL+frPGRFmlNZyzAgMBAAGjfzB9MB0G
-A1UdDgQWBBT9XQWzCcEEuYHJXQVr447M/HqFtTAfBgNVHSMEGDAWgBT9XQWzCcEE
-uYHJXQVr447M/HqFtTAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMy5jbHVzdGVy
-Mi5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEABxKm
-Id8s0A6ShwMzQT6C0QfgyYiI/dJ7ghAAg2nJ6P6Gl4WBQNSDkY/UGFXJALIud1Lo
-Eb0x7omu7k8fYd9GpbWzWgNLwy9lRfvndzHbCLmCJjdH88jqvkN+lzINcEu2hjX6
-dEaR79lYBj3Kam9tLQCqRd66zN9/yt1N3oNCwo4=
+MIIDTTCCAjWgAwIBAgIUD81S15BOvBDd3oEF6AYLabSHYhgwDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjMuYzIwHhcNMjIxMTI2MjEyMjU3WhcNNDIxMTIxMjEy
+MjU3WjAQMQ4wDAYDVQQDEwVuMy5jMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBALHIlrzwnyeSPpiJVTHprJZXBOaBEpz1wZ/xCWzY8/eHKMij2NTmYXr4
+i1rmdkCwP9WsomRN/E990K8yhjRCHvSIU8C3rUP+jfgnWIOJOPcGsJdcDFiWPyDD
+nJ9dncGAYgUAI5iSESOlLx4VGEtdY0nnJZJMurK0VPo8DZ416R10t1WDv0zBXfqK
+NsQy7TXlPYov9t/MWSCJc5NW0WNP9q6bbqtXIS0ieb86tcAqccgkpBlD6+9WvDtl
+zVcAPQsRUd4C4hfBGUMOx3gcUsJ6Ue8Mno4txKwcyrxTBVNAozFnGjYi19ytUvO/
+btMkHNlN0JVXQB/8lygEDndn+APfPmkCAwEAAaOBnjCBmzAdBgNVHQ4EFgQUZsmn
+44JkgHLJ9z4OIYuQf4zuMoowHwYDVR0jBBgwFoAUZsmn44JkgHLJ9z4OIYuQf4zu
+MoowTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTMuY2x1c3RlcjIuZWxhc3RpY3Nl
+YXJjaIIcbm9kZTMuY2x1c3RlcjIuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQCSJ5dLXlYQcET/luNsb0oW3Y83I/287zX/Ddu0bxLl
+pUcIj9LqroIqnsZ69k3SVTYMxVE1O+UiirQfU95qzveybYZVls1H7ejvM0rgNa5A
+FTtZfFIyy8yLAT00qqjQcOXTEaW7iu0RJgFMiExvJ3by/nO23MsFv/Cei8luvZlU
+Lb8P8Hsfck0UdS0ANMHd+wmFTatyqbk8oqg6vllouaJ1bNvg3XqJvBfRfxwB1Vkd
+EJNXbCfBprgEsj4cDmC2k2AlVa+hZCWsSu4AIN3+7Bnnt/1QCFEi+F5C5aNfAZp8
+aq8VPwBOt68KP7WdZN8RBg7jfGKhUj3rWU6tY49AhVcH
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c2.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c2.key
new file mode 100644
index 000000000000..b0be430a7d61
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c2.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAsciWvPCfJ5I+mIlVMemsllcE5oESnPXBn/EJbNjz94coyKPY
+1OZheviLWuZ2QLA/1ayiZE38T33QrzKGNEIe9IhTwLetQ/6N+CdYg4k49wawl1wM
+WJY/IMOcn12dwYBiBQAjmJIRI6UvHhUYS11jSeclkky6srRU+jwNnjXpHXS3VYO/
+TMFd+oo2xDLtNeU9ii/238xZIIlzk1bRY0/2rptuq1chLSJ5vzq1wCpxyCSkGUPr
+71a8O2XNVwA9CxFR3gLiF8EZQw7HeBxSwnpR7wyeji3ErBzKvFMFU0CjMWcaNiLX
+3K1S879u0yQc2U3QlVdAH/yXKAQOd2f4A98+aQIDAQABAoIBAFZ79uA4yaONYAgC
+HsCmJQAJmueq+GwEnUl6DxS1QZEzXeKk2hFPdTJGADybCgGT+6mtTl4vFcNxt/fS
+QzjexYI9Y0jH0XDXPybttQBf26dHyNYIwDTD+jetEN7xwpFkK8q085v/ynilnZMr
+veLDrk5ANKXDz7qpQQFt/03sedeKo9ZlZKWcLwSMR5YjAPjf7MteB+OhaWyyaeY5
+TlGaQs66PkPLKAhvbV4Q28eKE5IerXeb9RQE/XQSWOkWxSS3Sm9QSPFNfJ4PL/tm
+zczg/zy5mXZdPBM6d8WhCZ5SFoiOruxDJEWRUiEXEniX52OHYp34ibJ1UYhbHzbF
+8rUX22UCgYEA7tHwOg+0s7jBT2bME0LWmwZfk0t9ZuIBOS1sNT/uIvagjVg1EFxb
+XMzSqlrYEfKD143GXtQWB2uDJ0qGm4YnfojttJmKvSdGqBnVzjayIUmLxd3mHsZn
+llVjk13JTewadW1y/5CZXTdLtwYX/4HzQ7Z89wF3e0v0Kto9w/olMBMCgYEAvpKd
+RLPUGxAHhQD1DzA6vtmVFh4z6Ma0LcUiBQZl0IfpsDSuMjkZGXto/DlkPWq7jL7O
+TU80ld9cSPy3DritrDChIMsH09A0zF/6wSA3MaLrQ8JRP4uDFx/bRavZydO1DVdY
+SoIbLFPvCTwGZ9CQ96w/S8DHe9XLJBIJxluYPxMCgYBK/LxzXq8cp2x7ZAm8GoaM
+Eso7075CacB8lZied75sWAg9mW6//l/pmZ6kQvYJXJ6tNiUNbYalOcrqsWnmt477
+Gj+ZL4Tys7xp2ALcRyxTa2Zp8KzrCN1JxPIJOa+VugbszDsJp/uf21b41v7+CW26
+2zgNzvYXTtgSBIjUWl0G5QKBgH/OdAqPT8CWj3Rzn+X/wmYcJhAfUVDbchsACkC0
+ebPQcrjpm6mzAVJFobMGCmPmIMHTdSgD5pBcq9eHJH2K/W5tbqI/h5BWIhKui8zZ
+fiqbyIw41aR4PAJWKwgeqIDiqpm36k6N0wt3fugCiYk2JwLzUb/MGfzC9OnL7yJI
+a2JhAoGAMyETGPuKusdSFbiwTyhghPkXHBGj2GdBauKDQuiVzGdaay53SNxEMuzw
+r/6BD2lVTYah6vSnikXspACFpJgs+OlFUcGLdPjNgdr7hCxBYiv5n7Ljjb4PcWGO
+5QFOUPiCbL96rM3ni9mpuwOOApNIp104Ity/Rc4qcExA0QNoaU8=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c3.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c3.crt
index 27727392f100..91527a277275 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c3.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c3.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUUzIsCAQr4iJUUIBY0ZWZuZfi8ZAwDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjMuYzMwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0MTMy
-MjE0WjAQMQ4wDAYDVQQDEwVuMy5jMzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAlMorfk1gUwZci3ta792qR6j50VUKQ5AqZrvOXk74eYzfPZiE8TJwnphGdfMm
-8Ru4v04+4pf5tTzrl0L+dKebSLvVWYE+VPz/So9jyb2gHT7/I1ypEjum7iUpSiVz
-M09exmSyMrcyRpWSRQcrMomC4EK5OwDctUv1EUqAlgbmuqECAwEAAaN/MH0wHQYD
-VR0OBBYEFOkY81GYZIxcO6x/wKKrfPah0ThRMB8GA1UdIwQYMBaAFOkY81GYZIxc
-O6x/wKKrfPah0ThRMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGUzLmNsdXN0ZXIz
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQAGGvtY
-Byl4t9Jv1ykFwOnKZIBeUK31TxYQ3YP6w7Dodu7HJ9AZl+QF9di3FXuoAvwK8coD
-djuyqvvh6fYOdXKZDG2ZEbGHrQlGV+VXQw2OwdY0r9TwCjLT6MPqJeqiONQ/M2py
-JM+mshS35mZpsD1ZCtbw6M/CFJrMVQMP6y05xQ==
+MIIDTTCCAjWgAwIBAgIUM2bQVJR8CQZ4l4oZ1n4h6UwYV0swDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjMuYzMwHhcNMjIxMTI2MjEyMjU3WhcNNDIxMTIxMjEy
+MjU3WjAQMQ4wDAYDVQQDEwVuMy5jMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAJ6NE/DRhhGMmi3e/8iIuzZCcAi+IFKX2RZ6UtjVrX8Wq7p7QlQswpgX
+C1g4PxCa5bbZMxP5086WSqd/1v6FX42tOV2F4Gny1mBrZEtjxFsxNiUTae/D4W38
+juiWq0IxkWmXTFPR3zdxuoOgA56XdRhAT2Zv+Gl3rBYztH9mft2GAwaIIWS6ol34
+cOyj3sgsAaeHMhUNRxf/EAdNvkjjFATRFmQZZTKIoXCXF8ld9C3d4zxVSmGiLCQ2
+Vhgdk5xQuzIqfmEJyzuzWT7n3lL5ZYNb6CQ5qqki74zyef0cButKcHFMslFnzEio
+hcB/YC5fWT5PyuM6c2Z+mGVqKM14iBUCAwEAAaOBnjCBmzAdBgNVHQ4EFgQUa0ER
+qF7M233CRElTl8TWZpD8p4owHwYDVR0jBBgwFoAUa0ERqF7M233CRElTl8TWZpD8
+p4owTgYDVR0RBEcwRYIcbm9kZTMuY2x1c3RlcjMuZWxhc3RpY3NlYXJjaKAlBgNV
+BAOgHgwcbm9kZTMuY2x1c3RlcjMuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQB4vTz4ZViLlUVJIyVgJMxWvU7DKBB/ax0evMjAG5an
+lW3Wmmpz2mP1WnFHBU0NTr2drv1JZhLmrsgZFNp0Iqf+khfx2ZpUdvt9gX44S6I9
+x+1ca8mpOoO+5FvXngB9Qb955CY5k1JvouD6W5LFvQ0ubvv09a4y427bxZI9xwex
+C+mPcMEUnbVm4UBm0SU8zCl/4WEr03LLVi2zMLoKyzkXKz6h2vA/Yqno9bN35h65
+xAdtFybVYzyGE2p/9wNcuwmQoBQbWb9M09nGul+SddZcFJVju7pQ0k7kwdEhSX7c
+PQQ142Hofw+D+Az4CpPLN/HOCOTHnaJckqdJLM8dx54M
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c3.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c3.key
new file mode 100644
index 000000000000..301f4ed6313c
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c3.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAno0T8NGGEYyaLd7/yIi7NkJwCL4gUpfZFnpS2NWtfxaruntC
+VCzCmBcLWDg/EJrlttkzE/nTzpZKp3/W/oVfja05XYXgafLWYGtkS2PEWzE2JRNp
+78PhbfyO6JarQjGRaZdMU9HfN3G6g6ADnpd1GEBPZm/4aXesFjO0f2Z+3YYDBogh
+ZLqiXfhw7KPeyCwBp4cyFQ1HF/8QB02+SOMUBNEWZBllMoihcJcXyV30Ld3jPFVK
+YaIsJDZWGB2TnFC7Mip+YQnLO7NZPufeUvllg1voJDmqqSLvjPJ5/RwG60pwcUyy
+UWfMSKiFwH9gLl9ZPk/K4zpzZn6YZWoozXiIFQIDAQABAoIBAA9fEpaXnaWgacc2
+NjqpJOtVBBsY4bXYMmwHNnZ+utgWKNaGwi66ezA3Fbq7E0QnHEu+oYDXiPUfpYO5
+NW0OHa4049mhE9l5uVZ2Ou5DMSN9MShlijD8XuPkpy8+AkMD3rrrt/LtZzRvInvF
+3Ov4d5n/DCOcieshts2dPCvY0kZkI1/URrcciCEapcfPCXLnIsFXWDY5wwFp4sSc
+KTPkwyEceFEFQoJ+/MW3QYn9xMn14xJfATllP2M6hzzgPy8eu+jXcPkYnlHYk8JU
+0ijv+bQuE/3vrRKodZ8SSBA90EhMiPWHPWx/cfTHNpsUCfVeXesSRPGGW8mTCWwK
+fefiUOkCgYEAxpA3+2cub8EImcyWxXXtTvkAQFgofBg3CRS2ZQa6IhvpBD200+D+
+O7/MLJ3qyP9AVS6ZChtWr4K+SHbWkbsuEmVgE5y7+o1RF4BmtNNpaqYZP/BJXX0X
+viothVR58rxgVWfflgIvnqV0ZQi3XHLqv0XALmODlH6XwjvzYHom8i0CgYEAzGnq
+3/BNUt7nCpa/CNMP++wEercFuMAQtICCdxoXf3pX88p8UW0zW4TanzX+Z1ZvqXYd
+2kLpWzr2J+dYJhizQMubrTxT4eMPAD6WQT/lnPWgC4e1ErTH0yx7HeArXCYZGtzv
+zpaZvDBRDSzNGZBPiDcQrvh5ldnBqwmG0g+PZokCgYAz3GUSv49MVvGFIf9bEYLq
+tnRPWfcw+i4drBA7cLdaw0ln9ynskP414tFT8ebpmmlWFjaYgi1w58gFKyy9oVLW
+MS4X0T7zoMdGSR4KKpGE7l736S1wQoTaATAke7ziHjtW8ytdhbKkT/iHjvA9MDKJ
+xJD+TLWIKkw+yeO2FhFn/QKBgAtXRQSvnFqgq0jXHDotG5HqgkUcqmCZp1OcXaCl
+/e/ApeiMr2EFIfhN4qM6p6asEP3h474cQD4jg0yrdSSH8koCAd0Wn8DEjCyDTGrK
+VL7kkcCSAuXKydtZeGsr6LvwESyAXX2ktVrPg2+rsKVecEsqYu6SMoyxzCDlImm9
+G0OhAoGAHXnN0xmk8aUoHpIo6GCezBOYlnNMGW3P/GUN52QIiDExPXPnq7/N5rik
+oY/rDX0KQ3JP5gJl1kKg87Nlfw9Ix2p7L6cL0xBRhlDlwDtB6GUFqZmSbdVRcwEv
+gct7CVZ+baR5O52Ln+U0Rhkg08W4/PLNRzgbygun643oClXFIRU=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c4.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c4.crt
index 911e756f2860..0c6b287b2832 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c4.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c4.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAMasz2/hWiW7UURuyjgybK8+NYpbMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW4zLmM0MB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjMuYzQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAIwFiuTI0adVuF91ltRwyCzA2Eb9fV8JUKmlVXfjElP+XGG2CUerk8H1Ps0n
-PDFBCZB+0VM5BzjrtsEYmuEk7LJDXzoEj6L2MiYKIVia+qxzLvSfbcdJsGNXDrKn
-47sKDrUJbBDVodI43xGCplWyyhf0MS6hMe+7ttwb+LIZzTFLAgMBAAGjfzB9MB0G
-A1UdDgQWBBQJKDvefckZz3in61z5pdM6CpIwYTAfBgNVHSMEGDAWgBQJKDvefckZ
-z3in61z5pdM6CpIwYTAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMy5jbHVzdGVy
-NC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAS/6P
-0hx5NljLIKafQ4/MfompNgk4/yTY2eJfBuw7qdpqdfe1+bd2jB8ptuQ7t+vR8j+F
-42EmDGwZHwFVqr5PkXfhn9Drrx9muSXeXLheZ9FeknvR1n0upZ4KnKq9sqaN+JJh
-BnxpDpp3wLmP6LwWxHrAxWtiCvjngDh/hXx/P4M=
+MIIDTjCCAjagAwIBAgIVAOK+JDapeK8JH+vKe+rcKZSaox9XMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW4zLmM0MB4XDTIyMTEyNjIxMjI1N1oXDTQyMTEyMTIx
+MjI1N1owEDEOMAwGA1UEAxMFbjMuYzQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQDoA1LW6tTsYF3DsZuh9QRZUwyB6bflTIukLVOJP7bLfNktvRQg2Aup
+z3LxZsiiqGc0ujtBFh3tV3dpOBTB94irZaKn5sOdiVG5WnlOUCkdxz4pOdnYZuec
+pyfP2U+8aktje7QErbGHwRqDk9GljUiC+UbPkkkDBxLhnRmNwZTSRls4XdbIo+R3
+aOalyykDMd29Sk5WkEVhZsqaErp1emv4Yc35OvLhdRSAxpwuqnS6kStmsuQZBUQY
+NuD3sn3n36l8e3hX993hoqFHsz4TIXnsTQqWWN2ybqOGtt31Ss2nudSwIsyqwSg0
+tQ6mp+scv5djZCaDAH3dvsPo+8hgMTITAgMBAAGjgZ4wgZswHQYDVR0OBBYEFLX4
+zI0P/tAGizJCmdIm3QvGVgLpMB8GA1UdIwQYMBaAFLX4zI0P/tAGizJCmdIm3QvG
+VgLpME4GA1UdEQRHMEWCHG5vZGUzLmNsdXN0ZXI0LmVsYXN0aWNzZWFyY2igJQYD
+VQQDoB4MHG5vZGUzLmNsdXN0ZXI0LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAvd7wKrRc42DfsRqCeJwgqe/CZkn2o54ZPi+PbZsz
+Aw0XwVnUOkA4wpCxctdmO36XzxgoPFMZahZ/7Ik5M8UkU9/qWZIMR4eoMxcGL9vU
+qSAmaT/TAhSysyrFE/WQjKhKb5HcY6P9G+t5Uq8dLqclq13d4II0HbGx0BN3H/D3
+rBFaqVmVgIvZ7foYHHKkD3rRdhEYzWqH/d2BTO2+LPuhbQ0wCTzy7rFMpnXq9/tJ
+1g3FMtAWtaqgLMjULGrJpR70ikRjN0s3tTzW3xJ3zHrntcfmKgE6iNyENrePpWpb
+I1KbR6SCTuWPTo7XJWhwg8s/z2YKzfjosPKQNlfa3t5tyw==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c4.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c4.key
new file mode 100644
index 000000000000..3859cc57e80a
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c4.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA6ANS1urU7GBdw7GbofUEWVMMgem35UyLpC1TiT+2y3zZLb0U
+INgLqc9y8WbIoqhnNLo7QRYd7Vd3aTgUwfeIq2Wip+bDnYlRuVp5TlApHcc+KTnZ
+2GbnnKcnz9lPvGpLY3u0BK2xh8Eag5PRpY1IgvlGz5JJAwcS4Z0ZjcGU0kZbOF3W
+yKPkd2jmpcspAzHdvUpOVpBFYWbKmhK6dXpr+GHN+Try4XUUgMacLqp0upErZrLk
+GQVEGDbg97J959+pfHt4V/fd4aKhR7M+EyF57E0Klljdsm6jhrbd9UrNp7nUsCLM
+qsEoNLUOpqfrHL+XY2QmgwB93b7D6PvIYDEyEwIDAQABAoIBADFQk4kO/RcJHKT9
+2WDBXZ1G47+xV98rN/s7jyf7HPV/ogHw0Jhyk3gSmhHpkPfR563zb6jT9KSAFOeO
+fgsEfE5CGiKqECYqC+7j5ijZLQQhV3sXytAOf2hytymcvTAFP+TzZD4iVTvPtb0F
+7GfN8f200nJIW7CDDmZaKBZTcTiISDwF+ia+vnYaS3dGrQ71XUebDEn7m4yvd2Om
+WRaxLQUrbUBuQb8FikqKV1Y0+we87BIA/IWZCnUmnbTGeo1yquSQdvxefX/WNaWU
+uYHHNtQKzHPaGk6JZeAvJyifWZAHE6xAPcsApjqQOyJSsJKc7MQzzZXwe4tofvoP
+iKrKYfECgYEA8ffvhkh4XSw9hNiHD+K3X6/wGWX3gqWFrUsInnRjK/O7fON0SaAw
+ajjbXMrVnT9h8QgUkD8oOO49trRWrzzbTGPtNFltOsrRVLWlo8zAFESn5F7ZyG3z
+CQd5fciB4xhQZdppH3ARZUyoYct6+TQ/vTP7KANRWtEjedVKLtU02SUCgYEA9XeY
+ugxf61Njg+G8bbUSbxyZibcTpFT7nRYJeJJORyRvSVUT2AVVZsKYn8nueV/pujlU
+Svt3Ow1ueNH3JatAfSkbfMQJceeXrSrDoZR41fSndK1cMxBxYMMk+3hFSt7KlRwA
+uUMYauRm/QJStnzjsNoEb5AAjyX3ibChC+tNRNcCgYEAnRhAjWk9pBhV9Irz3U9v
+iJNcidDWyh6W1HSzqtSbkmySnDm8RRTYeg5h9TPHTmlZODjryN8ISTX8IUOiR/F1
+23nBBkzNCu+kyp+IY/lJ956/dhhSwrsHoCQXL5/fHEar5ATSXVPQJ3mGlL+cHnFa
+tIcsF8VXq2O+6ZmdqRxJTHUCgYEAwUvkoN2NyypnezsPvvSQB59l6OSaOxYQ7aqk
+2qWbrroqSH0Om1XX4ovq1CmX+UxmKZdroYBUa9Qv4kS1cpxgtC7izmUxatkMv/il
+BWB9pz55f4/kwvF31tLVFJGpP0QiEzlx+HG1kLc7wpFJwRqn0mRcid3EEvW68Fp+
+93vb5RECgYATHqozTc9BwKYJpES7x5Cy06QNFHs3DH607qFj++aV6MMdpXdaXia0
+zZxnQ2udXYXcsAztCtq0nC77Z++O0f4s7EfppleLJB1/mJH8hE6Z/Zg/O+L4O5hn
+CTeYio03rg/vavaWKHjvleuWzz1HQQOFqLFM/Xu5xASfUJ2Cat0PFw==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c5.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c5.crt
index 1132dabfb9e6..c4b447e2fc8d 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c5.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c5.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUCiG69NDv2GoMmFE3qaYNb/8lYRwwDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjMuYzUwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0MTMy
-MjE0WjAQMQ4wDAYDVQQDEwVuMy5jNTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAm+gpSNeO4opovT+vEOBd34pKfiCaIqoe0D+vR60gB35glziDShITOGMs04qU
-kM44nnv+d/Qib2BdkNlze+l/Sv92r+0DQp+MRzdNE9GkuV4B8yF99hpi71hJMED1
-lkrYRLUiEFUjmHccUoO6fF8BbdQwvD3rBTpAss1nDuO4BncCAwEAAaN/MH0wHQYD
-VR0OBBYEFFT81Tir0l9q5Cr5n47pDeY9pr4EMB8GA1UdIwQYMBaAFFT81Tir0l9q
-5Cr5n47pDeY9pr4EMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGUzLmNsdXN0ZXI1
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQBeApiQ
-M3HD7EzECcaSFMOhCpgmCQg/fV3gDaKSe8pbbzbSkKqoYsLwQM7HNUV+sEhrl+e0
-G9vR3cEeDMhuGuthVeqJ6jhgqn9vjXL2cLsDzgQYvj5AEOQe1/RfYWVOqYAuBtMk
-M+k7Rwrddd5+Fko2LtNXGskYwv1ExHSOl3MYSA==
+MIIDTjCCAjagAwIBAgIVAPVrbCMKMdUaDmrid+l9dmN7OS1lMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW4zLmM1MB4XDTIyMTEyNjIxMjI1N1oXDTQyMTEyMTIx
+MjI1N1owEDEOMAwGA1UEAxMFbjMuYzUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQDscm/Dykq174Z2bmcnZAh6IChHnJ9GQnKXVv5YTLtNTKEjSVP+LbwN
+Q6mY+vBW98nRLVy4N285q0DQHr/y4Fkn3LWJ6tWf+2j5JW8h9McKmb/6g4UUm/I3
+KsOWmMTl3ODmarWUwqh1s/eo/UDsjHEfe2NR0RxNFSwWxJfnupZUlE2fGA93mEJp
+RCUgD0HQlX2gk28IvUUagS9JJFwGbz3RFMG9l26eXVVnabxyjkBvPutr22cDdFir
+kNOfNMM3S53nPsLNEviXJ/JblfFEl23oDfWheO7eP3uAzdbT0ZnmPkATCBvwm7t2
+7KmqFhn9oEdycnD0CSKpkFKcSbRnSr0BAgMBAAGjgZ4wgZswHQYDVR0OBBYEFGME
+6PtWgSt6pf0KfCyk9N4sne6UMB8GA1UdIwQYMBaAFGME6PtWgSt6pf0KfCyk9N4s
+ne6UME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGUzLmNsdXN0ZXI1LmVsYXN0aWNz
+ZWFyY2iCHG5vZGUzLmNsdXN0ZXI1LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEABSDAns/uTOizogdOmNEbi3M2byGrpAYAYfnMI2eD
+kM3H3p7JWpbbEm7a3NZbFgc/mviff6JWFboSiyGv0lv7zc9US8c0iLE37/Nd0j/c
+BYgP4avub7MlixNv/duffrTb6vwipQQlpfKpQ9mSUUIZLvDEcaIfQ8PlRTG9Y6Vb
+zO97nOwkq0a6u/8mENY1M98EsJKLJc5ElDt6zTTUOJSlVJ7fWeWDrv5OYgXLB+d1
+d+PDXl9A0KkinkwhBajjykjHvFlb54sxXYtSjT9SLzS/8m/6HofwmaeLeiJcOnCx
+tw1qqw5qELrKYCtvLUAxeNrN/Wp1kHZgkaMXyZ12TbQVuw==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c5.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c5.key
new file mode 100644
index 000000000000..127a90999459
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c5.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEoQIBAAKCAQEA7HJvw8pKte+Gdm5nJ2QIeiAoR5yfRkJyl1b+WEy7TUyhI0lT
+/i28DUOpmPrwVvfJ0S1cuDdvOatA0B6/8uBZJ9y1ierVn/to+SVvIfTHCpm/+oOF
+FJvyNyrDlpjE5dzg5mq1lMKodbP3qP1A7IxxH3tjUdEcTRUsFsSX57qWVJRNnxgP
+d5hCaUQlIA9B0JV9oJNvCL1FGoEvSSRcBm890RTBvZdunl1VZ2m8co5Abz7ra9tn
+A3RYq5DTnzTDN0ud5z7CzRL4lyfyW5XxRJdt6A31oXju3j97gM3W09GZ5j5AEwgb
+8Ju7duypqhYZ/aBHcnJw9AkiqZBSnEm0Z0q9AQIDAQABAoH/Lg3YIR2c0bUWzqiQ
+ZQbB8++hiNlD2K6GnV2vasl/Tf4YAMR+nKqRz0+XkBmIiJf48kpQnf0GKAhoNmFW
+59q04uPFZFieRWv1tVmKPOXtgu4Ri5zkZfA+DGrC7+lPqKoOEDNGrPQBVfO40GyL
+ocd1HTmuTbhaN9Vg4gpmwBcCwvREBqQZXnklPZjMsuTDCnObETjWpldqKsrex/kv
+tOCnTvyHFYblc7kQrRD4ywzC9j3imEz7Fs4WHEou+lEP2Z1E0m7CjjhBPcio5AIx
+1WuAs4wS7cCcCc7ZYS6uiMBVJsOaF7j3Kp+Si5VzhNHKZyzMsHFQENVWfA0f4yFN
+UWIBAoGBAO1U7s1nHsLNOtIrTtXzlQ9EKitNrTrbmSg07Ep34So1wctXqwSYGNMy
+vZQWWxzHD2bqooOCk2E3NShaRophQshPxDZjIqQ9HypOTmh/j5FBJi0PUdUu95rR
+rBAzgF6a7lT29vEoNxZXOAhtMOkyT732aRXvlqSLzRzwegLcBoYBAoGBAP8LsBYn
+IFICjlJhcItTNkSXF/Hc/oB8Ig4KiH6xqe3UpOKVNpPObLDFPB01kCIh+1FxAaE5
++z16fSzfpx/QarF1eV2Q7d+hu3oBThNGrbFkh0UijowhGB+TUIyouRPYKYlLsjWu
+SXei54bmg0xpD9fGFDcOXcUFuYEcwaBIejcBAoGAVL1GpsBzE0sD/ZbJB0t5pYHJ
+LL0HZUSTYu/CjrRPCSs4jso2neVQt9geKIny7kuFDFIqxJoS1VtI96eEGFSx66PP
+gzvSWqIYRpq/FEC3mj2Qy/X0WhHzBqxmq0I4S482GdA0S9b4FFn//8I8/IaiQkmB
+K/vn5cik6/hRGERHcAECgYAFxnH6TnCA6uXmH6a4wO2dp/42z9ZDeowa3Uox9Dvw
+0rvZFZ/iwO3dIto1nxzgaPyT34c162YxAjELbxUNt15zHkHdSIbaZYMlRBpMfXE5
+bxlRC3NG4Qj2zhMo3Dk0pdRq83fZl1S6jMnJWDbo6AgfzrBufrjI2VRWBByuUHXU
+AQKBgQDbjFt4jwddHTkSzgkZq/gTJn4PaK9ZY7UOpukvNYI4HdOIcze6KtvVc710
+elYVTEz4+wIZTJlFVJv6XEAOoEWm0+z3DELy6Sbu5zydgJ1dY27U9UuFJc1iK0oP
+nyvqQ/EflTMlu2rKIvMWVCXHgtj1+hzQnhjdsVaP8yBk28i8Eg==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c6.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c6.crt
index ab338792607e..a6fb40f71fc8 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c6.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c6.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVANXn8r0H2YaPh9YLfjlI7c/QtXNVMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW4zLmM2MB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjMuYzYwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAI9iaWrpHwmXPBC3YhfzkVe10A7AKjRzdo5uv4ra8xQmVzsdk3lQ+oQDYMDF
-oP7rfxtRLzB2Z5l5rChFbkLav+3G8DJ09Vdzs28J9Kxz7N/H9tEiTmgvzdPVBWRy
-A7D2WPPiaHBeIIrtgzfZbHenyxuDxfNtvgvSgUZMEjM7Vg83AgMBAAGjfzB9MB0G
-A1UdDgQWBBSADI6qBveWvxd9C8xucsFtogIDRTAfBgNVHSMEGDAWgBSADI6qBveW
-vxd9C8xucsFtogIDRTAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMy5jbHVzdGVy
-Ni5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEABgqe
-FBbsrBz4276uUtr0YkmUlVUoOYewotLq267YGFH/kQkLJbN/Yvdow++7xn0+W2Wo
-LqkwlMfzZicpnm5TgqKO9SQCXxMonhCyvNIuJgShkADIgwKnLNQq4PpV7gyN+ePV
-RUCAQJdcLMPt5SH+q9E528K28WJ1aVWarDDkxEo=
+MIIDTTCCAjWgAwIBAgIUB8643/7eGO7b2AKA6fGzsqpWsp8wDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjMuYzYwHhcNMjIxMTI2MjEyMjU3WhcNNDIxMTIxMjEy
+MjU3WjAQMQ4wDAYDVQQDEwVuMy5jNjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBALyp8UaakM8Z5UlHPlPP458akRIOzuYUZrsnKHFZDBpX0y00B9VEkbwS
+XH857PsIOuhJAzEQ+OPn8MMookyFmyeWB5gokSYud4PyOy0HPbPcdw0eVaYi8EQ0
+PfkQ2qvGeFT2HNBIDwX8Tj8BogAOQZ4JK2YSgsRftpOPYuvn9rfrIWmjYvs1eFbw
+zYyETsY0Tk3auVP2ecHkznfliSRgyy7FpKf10f1Sd61MidSPo8A0tBv/xe3gOCz/
+aUU5HY8Io94ToJljF7q381bB3nO67k8IaU7ApAunt7bToHMQtKfnPR8/qFLW964S
+jtoZEtJzFdt+VoxwB0p/4TWfhe1wIfMCAwEAAaOBnjCBmzAdBgNVHQ4EFgQUkYUe
+kyCWC/5aEzYV+yx4s18doHEwHwYDVR0jBBgwFoAUkYUekyCWC/5aEzYV+yx4s18d
+oHEwTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTMuY2x1c3RlcjYuZWxhc3RpY3Nl
+YXJjaIIcbm9kZTMuY2x1c3RlcjYuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQAiJbE3qFFQ8M22RgzLoyQnV8gcQ2xKTBi9S+I0nZEV
+xXL0VY+W72LiJvaG4buXNjpKCo+bHnL03xHNOlSFjyjlsZCDvm+LvPT6e+67y/L4
+mlMpyIe3k6txSG5RUV7lHi2tpvpC5Wfu6nJLn5+70JdgxmOLXRzvqYFKmEZTrh8C
+dfqOSQvnHKoz9TWdoyElO66e/+7vUFIBn+y+LQjSyK12HN85PEZqvyroHRfq5gni
+l+ntElEbun164qaou1m4eBfR8O76vl5CPongnw1VgJ0J0xn9T+gBey7o6sUINWNR
+/79I/gpYVd+6Yx4c/CxfsnrZIZQHY5qwDBonb9AYQRm5
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c6.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c6.key
new file mode 100644
index 000000000000..dc4e1c7700c1
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c6.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAvKnxRpqQzxnlSUc+U8/jnxqREg7O5hRmuycocVkMGlfTLTQH
+1USRvBJcfzns+wg66EkDMRD44+fwwyiiTIWbJ5YHmCiRJi53g/I7LQc9s9x3DR5V
+piLwRDQ9+RDaq8Z4VPYc0EgPBfxOPwGiAA5BngkrZhKCxF+2k49i6+f2t+shaaNi
++zV4VvDNjIROxjROTdq5U/Z5weTOd+WJJGDLLsWkp/XR/VJ3rUyJ1I+jwDS0G//F
+7eA4LP9pRTkdjwij3hOgmWMXurfzVsHec7ruTwhpTsCkC6e3ttOgcxC0p+c9Hz+o
+Utb3rhKO2hkS0nMV235WjHAHSn/hNZ+F7XAh8wIDAQABAoIBAFg2kcT0cm8TaZz4
+vdDMbF6xOlGaMO1tbpdleMh+3WTX2brSEDOMyyOH5k3qlL7Pm1OPB/oCOI2/nlhy
+AGkNM32v7b8cgLm8kk9UjIzCTPxzQgsVOvDidzs8vhld4xDcTNlU9iYA+11OGnvl
+VIz4HYA6Q1uHnjMZmbGnIpaxMC/P555xIqTGc+7LMLfzmA9hqpgOBItvYg0ocW+r
+1rDSavJu/CGkuVocCBqzwVeGjV8Hd5F/ThLBDaxFYBz08MdLy+S2H/hpCzobcSok
+OurEFWIcStSD79Wgg7FzaymmMdJMs5742fh3lerYUnLxM/4ccVqMKrU13Lo1chBI
+XL9+MXECgYEAxzT9uiFZYFdOmzYweqy+VBkoqnsFHs3Ir/am0f2ibf/IWoU0fRu1
+DSELlWK9L/iasLQY7NabRPBoKBK/UNfFztKB8NcS4AYnIZ3yYKDec/ocprEz4htY
+62ThIjIDr+qqlvcd4Ft6Mwp4KojaiaQQhI8Nl5D4M0Kxrp8byxthxAkCgYEA8nN3
+BnplB/TQUxwe4Qg2KMZfS6mqpAHU74+tynPaRw3oNQCnfekY3ZJco/t1qYa3H3n9
+ZDSs4dC4VZqYOmfOn2gRRG+Bc84B2auZXz0cYwWMuPWhgbTSjPpxksFCyoF7lz9C
+0HwmHp6OG870NVwyO/WztjUIWW5RveGKbfzMDRsCgYBoHBa6zzERu69WSOTfSCaU
+dy2co3ySk8Pdys92kdLxICvKuzEqbW+a8/lE0nzDdG8MCKO6K5KsXeRhhSK9kndL
+1xKYFx9DO5P56r1AqPJ5tG6C47uU4OwA4hETOQZaSw9qpK3Vkplz0XXp7Ooh4z/K
+t6TH4LkGuUV5WKhC5wWaWQKBgCsZP3VVWJRbfpTnkM2r6gTBjrsihAT3lmSng9tT
+UOAEptyVKzEEuLjahcFYuAfUvMTJqLOhcYIK2lnNS3ZGccKAe2FQ+RAe9NE+4dq+
+QEeR+vv+HiGyhSMuefX+iOL6tX+jbTsmdphG0zg6R8scgomKGdIwBzXumHVKnJME
+tzk1AoGAbyNDAz+wTMijCoP5bdFeqHtiFzDlrGkGppPrBh1xTkxLTEmEz+uNsgsx
+MiUrpmnr4erxtRRPihL0gbtMkLlBI5rynGChbV6IHXcqFzxUXLiuO/jNeKq3bmqc
+0P+qag+WFLjpxYMY3aLVxFqylv86npsaNuHvVm1ZGNFX1vINJTQ=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c7.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c7.crt
index 5aced504e995..d04fea93e805 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c7.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c7.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUXRJ0mqzIAtG7P2+PpLY08BWDIigwDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjMuYzcwHhcNMTgwNDE5MTMyMjE1WhcNNDUwOTA0MTMy
-MjE1WjAQMQ4wDAYDVQQDEwVuMy5jNzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAxbb/mneg3oKdKk7tMpcpC+KDBsWWvgx9tKPqHFBWIrklLEeHVA87wuWoXVoE
-moPRmaJkHCuNSUOmNUsw4ZAigdCHLI+Fxgu3RXE5YcOlR/VcfIdmeTOkZTmfkAVl
-6sRIPMARTeQkqaaG93WtXpSFQ+hO3RBIkJ3t3vTL7m7dUH0CAwEAAaN/MH0wHQYD
-VR0OBBYEFL/uRR/fiAlWZMNJZ8ZcDCDLLkCaMB8GA1UdIwQYMBaAFL/uRR/fiAlW
-ZMNJZ8ZcDCDLLkCaMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGUzLmNsdXN0ZXI3
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQA01y/O
-PSsN1yMbI8uj8G8HH+ZTXqvf70DdL4Ujy3t0l+bMUW61ZtPKVE1eoQ8e2Zc7c52n
-uMuLF+ndIAaAg07aAA8ay6vCivSymgzskMfy4qludDAnenbfrK+7+elrTh+Bq/9X
-h6iBhL+cb6ooOJaoynlZyHKP1HY8xq6GYY7K8A==
+MIIDTjCCAjagAwIBAgIVAN6pghHjL+hceM5qySvpjDokFK61MA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW4zLmM3MB4XDTIyMTEyNjIxMjI1N1oXDTQyMTEyMTIx
+MjI1N1owEDEOMAwGA1UEAxMFbjMuYzcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQD6fM4sO1MvlU/wpFm6TJ2lDtzWrfpj95cvFiALz+y8x/cMM4qObLOu
+/d5Y4M81JiLtf0bWU6eouqp05wMMo47ZAlKOrrQvqKGVY+GcMuD9UFhpr69GBUon
+j68jPVVztDjqAjuU9ge1n5fJZGnSx6lQ+2kyw2CN23r2X7hnGKEC7iapIBaTdJqf
+aycmap/cmKy1C/3zLf5f1pf/tEztlnRSJUV3MReFCZDk/yjjAY3gjf0uoxwSwL1R
+XhLUXJmIASgHJxTesXYLd52EEXJXnYBx/zQUkYYaqHNBgkLPnFeCtcNaWKNIszrb
+1g4qdM9xq2Qv+IApkK6nJYrCQgcmN2FvAgMBAAGjgZ4wgZswHQYDVR0OBBYEFOjo
+/YNHG2YQwcYhT5U/k9g/PWQ6MB8GA1UdIwQYMBaAFOjo/YNHG2YQwcYhT5U/k9g/
+PWQ6ME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGUzLmNsdXN0ZXI3LmVsYXN0aWNz
+ZWFyY2iCHG5vZGUzLmNsdXN0ZXI3LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAp84lG6fryTC+fSgnYaT0wddnwi6MI0FHI3N1i5ic
+mT/iP/iG5YbkksIzRMb7z5uzId5Ak465++VI1ogCI9FKyV2U7Vao5aJ0K0zIV927
+miANMYOWABfneFc1yP5Vr0944BE8GNdrqWozsmw2EaP/CmVK33i4bwVafvgLVrrK
+hXeKUZgJL0WmMA5UXwuU8ppSxUom+/RW4lGYSxJtHccEt1ir7PKUOGWHkCOCGBju
+FDSoJT6X4YyqQLCwnbDDUwpWg2//xycZ6v5L8RhN8ENIWdKtsIbuInl5oCeoQd02
+QVzsXUGXeho5C/ZaIG9AoPzZm5IgujW3sjYHQvjB2/24kw==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c7.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c7.key
new file mode 100644
index 000000000000..814bbd213bdb
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c7.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEA+nzOLDtTL5VP8KRZukydpQ7c1q36Y/eXLxYgC8/svMf3DDOK
+jmyzrv3eWODPNSYi7X9G1lOnqLqqdOcDDKOO2QJSjq60L6ihlWPhnDLg/VBYaa+v
+RgVKJ4+vIz1Vc7Q46gI7lPYHtZ+XyWRp0sepUPtpMsNgjdt69l+4ZxihAu4mqSAW
+k3San2snJmqf3JistQv98y3+X9aX/7RM7ZZ0UiVFdzEXhQmQ5P8o4wGN4I39LqMc
+EsC9UV4S1FyZiAEoBycU3rF2C3edhBFyV52Acf80FJGGGqhzQYJCz5xXgrXDWlij
+SLM629YOKnTPcatkL/iAKZCupyWKwkIHJjdhbwIDAQABAoIBABC2xTcUWk+hge0H
+umkA9b/uxvOYNk33VmIw8sB3Bk/uc+Cfi85HGJJDpFN/5LOiMNByHrLscBKFKhyU
+0ZJFU6HNUt7sb1+CvA1w1bwLdRHSC6Q5sdNVRQZwwvnx/MK0stuCfKFK7WbEGGe6
+yIjlA7pAnFtjyAyPKZxI9NAER7IfOi/MyTCNB3hXWKCpkv28STSyIBD3J9gWhVeI
+wnUD41evcQJRLunJhYv8QZFbDLrn/aTpwC1tbYMbZPi2iS/1SuEZJCmRFHxka3Px
+pPN6j1+vKSzdem4sx4PIpPUQWvE8Kffo7hbZO7KL6m265BhKvz6E0dAp6mw/G89R
+ymkXyeECgYEA/FpiIV32plbKS7LVGgTYwNTwZBqw599VOHwbwlhunysgh3RtYvZI
+6Fi4fB4eaF8VrirCTknx3ohMFw7rxTtMkGePAkGwds4pD4j4KGthyS37ut36lO8x
+/Y7T/hK+NEGtNB8Pidr44TotHkHE8x/VPrmHol2uGosv6BZ1bNcA/NECgYEA/huF
+LHO1VTS8Mp5doutnPDHeym3uUUNPAKygj91RBksXqzu2shnDGPnCZfUHycZFo2/c
+/k30GxwlPSfCjvVGi2yLh5hsarF1M2WFTZqJ4HbavLWk24enESZoNcSxpWoue3C9
+TakC2btwzDg5vCuAWjFM2B5xnlgqyZNpN0PJCj8CgYEA7A+4oDH/2tZxYqKlijNi
+d/A598qoFh/q3m3qocCLqsVzJoMZGGv/A6qK2UBrfYMFCung3c96PIsPcrp4fwXb
+t3HIa3n7kweDjY65ZUYeMU2cn8+q6ste3ixPzurFlqihV5J+vJxkGHyNXDJ2BGCQ
+4J45bP9pMFdntxnbOrddqyECgYEA+HdMZsTVYHVqOEyl3nIzxaY15NZNG2SnUU96
+yHFtkqvGfwaqg8ew/zDopu1LKj8XRNJS0s7Ci+efegbwtPyceCQw4utlx4WxxgnU
+qKcdfkB5beBnTMl90MUvTLQByG7k+Iqqb899HA2uI+IughI/NPHT49cQ3fX0EJp1
+XTaI090CgYEA+SpKH72fBMP6VyjjILmmSaVEZTlSVTUynd/61MdojgYOVHE5CvGm
+e6XgX15k9bOZUgruZ9keANXVNT1npMIOntKk9z04T3t9Q93KoO9JDF5jnUJvafYk
+7ctrTGwhGpGaJto57kjPy1GlgKz9jfBjlhLHVw2SookBWxjARg+HogA=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c8.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c8.crt
index 87817130c15f..df0d2c8e1a5d 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c8.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c8.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAPLfYPUyngardZMqXbQSc9jFW/ZNMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW4zLmM4MB4XDTE4MDQxOTEzMjIxNVoXDTQ1MDkwNDEz
-MjIxNVowEDEOMAwGA1UEAxMFbjMuYzgwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAIr6JyOARsRn4zJ//TSQ3eMigJDnDTz4qloEcGRhdsEmEkiVxmawSx5XYZeq
-vQIz69axBViHQZEa1emNrgktmzR4swV8wUip8tJhcic7iegSYk40VG8w5YSDfVhC
-a4JEjTF+H/0B61j+Gq81Xzqw6g4m8kJWqWCvICNDo5SKkBSzAgMBAAGjfzB9MB0G
-A1UdDgQWBBRYXtwMkHRZ58jOv4Naa7AkihBRijAfBgNVHSMEGDAWgBRYXtwMkHRZ
-58jOv4Naa7AkihBRijAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlMy5jbHVzdGVy
-OC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAiP9A
-lTjp4exnE8d71b2a2bP5+4zF/yT6Jq6eVKKUi1ChFnGpUQLHaCtEeNMrMusQfDvz
-NYi9E+3TG/5/CDgkimAWxJc/gzTge2Y2HhCWV4Sm3qV2TY5gnqbN4Ib5YQaHcL/x
-1fKEKHxHf+6F8CdyGlMBei1LwIx9iHqWMjuagg4=
+MIIDTTCCAjWgAwIBAgIUWkcRSOeqE9XdcFAqtBIinaP0XQowDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjMuYzgwHhcNMjIxMTI2MjEyMjU3WhcNNDIxMTIxMjEy
+MjU3WjAQMQ4wDAYDVQQDEwVuMy5jODCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBALNIneMayIua6przMCfviWOO9Fsr9b0Nm0AFy7Y5BGa3IwPJGt755NKy
+mxr3GRRdXrHxgxuPr2qxva10UdQ0+YE8+hN+ndg1jtCV2h2nD2NScebwLu4f310w
+Swix+tcjecMXv7E7xC0JIfJYhsI0DK9ar2WZ3CNh85ejNlPiVSIKTQZSoDN1gZY7
+uksEabGvQocdp3thk72HFnWlncdjjGVBiyC30wbTLTSxvbzsxEVVCqTt7QtOAhrA
+U5f/coRHLT4iMtMTL0Zg6p+/Kef8t04Cadto0EzdFXzznLKhDxuOSRCjJNx50f/8
+vRAuaGYwS6+E4hOBLf0DbX4p0wPojK0CAwEAAaOBnjCBmzAdBgNVHQ4EFgQUUL6D
+UvnCMaHFwlqp8/V74qi1h+owHwYDVR0jBBgwFoAUUL6DUvnCMaHFwlqp8/V74qi1
+h+owTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTMuY2x1c3RlcjguZWxhc3RpY3Nl
+YXJjaIIcbm9kZTMuY2x1c3RlcjguZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQB/DKPXEp6C3Alj/glG+cylPxzTTMUQxynJ+38hiqRM
+n0mQ4E4szHXxRE4ctz8TLcyd41Ga/hi3sMHkeWqXjixnoyI7PIULfTw2+pYyoOt5
+tg575Tjnb3Fs4bm63PNaW3hYYDxRLSinRkj5Vt2xR0D+KpQZm7Rcaq3AcInEPWsb
+kMZ3nXUvSXAtNmw7we8ZBLp7I3bKnThXNlAIP5CBcbw3Y++CmH6FuVrYb6l8HyzG
+EbgDB9K4WSgR2OP8S496dzdvWPBz56RQs7nE5lKYrejx+hdcR9sliEHq9yPqPBYl
+4Sku/7Fes7UhZKhuM3adUtG3knF0cnRXDk3YsmeqgsdL
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c8.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c8.key
new file mode 100644
index 000000000000..699c9773a3b0
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n3.c8.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAs0id4xrIi5rqmvMwJ++JY470Wyv1vQ2bQAXLtjkEZrcjA8ka
+3vnk0rKbGvcZFF1esfGDG4+varG9rXRR1DT5gTz6E36d2DWO0JXaHacPY1Jx5vAu
+7h/fXTBLCLH61yN5wxe/sTvELQkh8liGwjQMr1qvZZncI2Hzl6M2U+JVIgpNBlKg
+M3WBlju6SwRpsa9Chx2ne2GTvYcWdaWdx2OMZUGLILfTBtMtNLG9vOzERVUKpO3t
+C04CGsBTl/9yhEctPiIy0xMvRmDqn78p5/y3TgJp22jQTN0VfPOcsqEPG45JEKMk
+3HnR//y9EC5oZjBLr4TiE4Et/QNtfinTA+iMrQIDAQABAoIBACuGYSJeAbr2yKcG
+jDXdzu53KbaqpPWZRIwV/B9Il1tycZM9MwNYCqfmSYaR36pUgqVTmz9JMk6GSSS/
+nRAWLk6kZZfvANYZJotYHgRd5vv3a4Sj4djYrwr6KvU49EtfpvNopBacimNemLmz
+qT+jkELQEs3pPQrwAEnbi1hxFbb1/QzLjp7X9NbT3zw/rvRqRclQNFvt+HZ7LJlo
+4P1SO13pASONVDYtPu1DApUudmMkxjUNcac/ntKXIpbwtikwqI5+14BpLgO2DE6F
+M3PEB/AECAmD3u/Qn/MBQ3oCauloTWG1Aqw9wIoxVVX5ncTz14K3kwELI+HlLjX7
+pzBNa0ECgYEA+OGtEdwjQdnmgnSDUSnJMddol06njpSeaW730w0UDvtvcQOl7Dcv
+z7hIyvspjBSGxsdC9CRaX7hE9vnSb+LAxj1hHh5QzJmQ3LzpgFBzfc/DH6e07zZs
+ZGI4AbohUUktAWF1zx0R3zWFCzXJ+DUHmUWtRx5D8c4qMq9iwiuNyM0CgYEAuGlX
+XCZh3pJVpsrpqVbcWgPh3O8WrRmY2BLTwgUV0f8Ijs4uYG1ingn4h6ijJqisdFDQ
+Q/25Al3AibFbudly34fbBqvqObpNnL+EVgNQsGfTiMRmyvX+ZGjKQW74rNuSbinT
+KyMmMSw3Y/0VSZ+I3ANhNBWqkV729lu0PutpU2ECgYEA+GRD1w9eINrYtFqUtOx/
+YL8cI+bcU/EAKToXHQxweIN5NdHuR1LM5QOHow4I6UWTPdChggjGKcI/Ej9DR2eH
+/lJEG0smWQzrB2ODqAyjN22j0HQQoK4wf/+G5jJeDkJ6KGacrZjVvXAs2INWj+Jd
+MxvXVKNgLHUWqH9ikWfmmgECgYEAh3BuxfI2rSYinAapiEKQQdhaGU7MA8Qp+oAR
+YI0lkn7RxThu3UlZRaMjEldrxCfbyO3VmCoPqEo876QAL0YiVUPNPGAk/dLzz6zl
+P9rdd0I6HVfJ+0SiWxKPgXFEWIhHSnuTaTo+gZzs7ZXh/ZmbBaPBz2QeSvhJP3To
+s9FoIOECgYBJqfEwXcpoyGRAPrfZ68g42eUbyJ89AgKbsx/EHa53JaNmcAKvPxXq
+lsYCAU9aIGeE2fWQq8jwi8vL1Y3xm1sCLuyrAWpXYQoevROfW6LJ3KFtrVWizVuI
+DxD122FwR0OLaNStTyrTNeNovnA3bjN0dTstTicjRYC5NdbrKqPAYQ==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c1.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c1.crt
index 3083fbc5ee2c..dd3982013a03 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c1.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c1.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUP6Apm4lhwWUMZcX1Sgop10fSuPEwDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjQuYzEwHhcNMTgwNDE5MTMyMjEzWhcNNDUwOTA0MTMy
-MjEzWjAQMQ4wDAYDVQQDEwVuNC5jMTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAlusHSJz+gYQ2zUkOFFPijbePcDBt5QT8IQpmrAdQhq+O8gdwPxVrL/ROkAms
-boSiuKqWo036b4Gbro0Q2fK4uyPhfergvJfdbAWKZC3gtCSyx+/36rWS9pxSArLK
-2hne6aYcjhuGlhzcGrhKs5ByEt30RkEK9REwp0M+PGxoAmsCAwEAAaN/MH0wHQYD
-VR0OBBYEFMt0AUJ/1BWNa8n8VS+BmXy/GSYaMB8GA1UdIwQYMBaAFMt0AUJ/1BWN
-a8n8VS+BmXy/GSYaMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU0LmNsdXN0ZXIx
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQBhVNAp
-rAoxevBNuG6HP/1CougyBIXz4/hxdEfMr/D/u6kEBXTTkulFbI7IHUvwATz4oKUd
-CNzBQYRp+dcWpvqSL1rgkdxnPkNbdBHr9BEwiyCpk4nQAF12DpPPMxgeeOE2EiLQ
-kTkE00nh6i2cJlYQsriPlQ1S9KUcY70ch+rjaQ==
+MIIDTTCCAjWgAwIBAgIUZYBYKhCsZvpfySUytHi3Pr7smTAwDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjQuYzEwHhcNMjIxMTI2MjEyMjU3WhcNNDIxMTIxMjEy
+MjU3WjAQMQ4wDAYDVQQDEwVuNC5jMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBALpTlDyrKSPySpN1XZiWWvfVeu9ix8CWtKJZ5fQTAi7kkkXA4Zv+0FL3
+iioCpSFg7JYH3tejh1WxCncOZunQDtSjkMHnGCybHzFQLk8+/A+VIwilV0SR/WVs
+45ro97LaG+CQcD5QbMci+/7xCBwX+hPmgtQzW34++ENBX21RwdYEnYoyx1zaWrWM
+frV3jxLue0CECa022dwwlvyNRRJ4YyrpzyZR1tDdTA7qs5tB5JqqNFG/RdHU8h1x
+aK7BLmabBOKEB6uLkCdW60HZn2SGWdz6xV8iq1gndbq44/d1RFY+3YiHPbK09+Sq
+uODoC1uUAQotReXKIPgCcKCYZ0kC2fECAwEAAaOBnjCBmzAdBgNVHQ4EFgQUL5GO
+2RzM9WZcOfuTAO9loD1/0lYwHwYDVR0jBBgwFoAUL5GO2RzM9WZcOfuTAO9loD1/
+0lYwTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTQuY2x1c3RlcjEuZWxhc3RpY3Nl
+YXJjaIIcbm9kZTQuY2x1c3RlcjEuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQCfBGwRW2y3aAddyauaJnmnYb1Jpv8NhrGFVhCJYruE
+M3FOmUJGSErX0nTLOYs8n6h5u4mPYYfnQp2K6ow4x/UDWr7m6VnxCbN1BqjDxk3k
+PcqiLb+VgtsnWARTHejsaw8ByQ/BFl5Uy0OXM+M/vJLVuHTEhskI9nCQs32LA9Ld
+ngnaknOzBs1hkQyGeaG7ZZ6l1MFeOhQBZU3LmXvzPlXHoIw0FB+H3/Vh7K8d4N9E
+RpOOWJjBj1YVt012DFhApMq7XSLvtNrccj/4lp8nrozRZrp3haanj9I2fhd6PwQ0
+17D1AFMScje0n37uPKcS3xvYLdqejF7A7im7LYFRcOAp
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c1.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c1.key
new file mode 100644
index 000000000000..2b7db0bcffd6
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c1.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAulOUPKspI/JKk3VdmJZa99V672LHwJa0olnl9BMCLuSSRcDh
+m/7QUveKKgKlIWDslgfe16OHVbEKdw5m6dAO1KOQwecYLJsfMVAuTz78D5UjCKVX
+RJH9ZWzjmuj3stob4JBwPlBsxyL7/vEIHBf6E+aC1DNbfj74Q0FfbVHB1gSdijLH
+XNpatYx+tXePEu57QIQJrTbZ3DCW/I1FEnhjKunPJlHW0N1MDuqzm0Hkmqo0Ub9F
+0dTyHXForsEuZpsE4oQHq4uQJ1brQdmfZIZZ3PrFXyKrWCd1urjj93VEVj7diIc9
+srT35Kq44OgLW5QBCi1F5cog+AJwoJhnSQLZ8QIDAQABAoIBABd0kgKZIVKvG4g4
+jBG7S7RLIXClJLQvxIlze6kgA3RXvboeL8htaVgoZTIopxTumJnCX+ERauX0pXab
+T4U7dcJ63KwsJTToSiLxDwNW0HA9u+1IsxWksxtje7tyfEv9fOjmBdsyUJ843jQX
+r4ep1QdQS0PfOzNHRgVhY1vloNsJ/jUfM8ccAupzeTRZwzCr0CRdPWhCnLB9V1+W
+46MWu9ZGRPvkrlsaHnaWzDxKfjoXnxXdZ/CEbPEcjShn0ztlLPDttWZCZ38KTe0W
+CBm7YAwlfxEWEKRYsd8iujwV+7v64fAJc+VD1eWRvpvryNpmfR0dDWnRd7t4BZxC
+AGQiwGECgYEA+g6u+b8qbOBrgslX1TcyMpns0QoXsrMdfOwcmhgi3y6k/5rr40tR
+mDjKRPwNKweUTCfUUTKQfDZtf5Ci0a0w1l8eYI6EC6wSnmxNeJKUNInM92UM8p7I
+J86vEZ8BWu7KE2EiRUSQfKlra3qWrGiQ3Sbi1byK4lja4NclLWIdnMUCgYEAvsEq
+SKwnZRlCYiRCPvcC0kuw6+I8mi/GtZ8MF3XinNcnOxcrS9cz5nS250vGHeiMOEzH
+1derT8/CHjY6zBXPdEQjHRdzM0LsBRxsB4yELmaOVVAgtvlA02tuIZJZOzN+Cgw7
+FOTPi+HUPgTe2FWF2trYapxXlhkcezx13+jocz0CgYEApfDGl0AQRG5tij5ZRko5
+yrCxIkvjTBhnMjZyCKSJx7fpkSJa9dxOyoitWjg1+N8vv5YS5Mt8bKGl6qiuFB0t
+sqUuwH7zC9fmBzHx+1iyd2cXGPn7LKrIQvp63WO6f4DFozSrjHIivEel3I5enQ1L
+TSd4EsTrGahDoD/GESbyZc0CgYBysxXo0tNxYAnhl2naEI86wlakz8me8OFsLjGI
+HdNgeaLWTi0zkjhb94xPcemz7DP8WnbC58bT29oebn9WXjr670mp4YLcrP3caOWw
+eFa2y+YD+IVjYGcReYSkXSqYhXLhOc+A1cDSnL0Reb5BmNM+8N625EFqOLk+hPFf
+7N1xBQKBgARJ89KPeuIKJ3ree7LtI7+FBlQAjZI0SyKC9jsAcxxU6U3PwVvVmgS0
+jmAVp1t1m2YmXvDv0SkhsU+KTSiqIMSvrcfXuUTqdd/THHMEAAUBSfIV3u6BksXM
+ZStOlMSErFxe7mIXVghlgCNZ2Zu+q8+FFtftvBu6fIYOedgpsYCb
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c2.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c2.crt
index d0bb18e84d3e..2983bc1cb140 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c2.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c2.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVALfLHibW9Z409nWoaPHhXA+o+2ksMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW40LmMyMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjQuYzIwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAKj0cnL86BiN2VBhvrH3XDplcwd8lFAGxAzAfP+Sx1Swqd6y4hZoX2VkJ43A
-uYvJY7aoMK4xIQJ4lWbrvCa+5epbOY1GGg82Z+/Od7mfZppfajCKTAmAIXfaSW48
-V/fRZyYl3KDfP4h5AqJV02wAJ4g5r7Olm74dwAMclLF6FqwfAgMBAAGjfzB9MB0G
-A1UdDgQWBBQ9SpGlYooWMcJYNuTND5SQk6EPfjAfBgNVHSMEGDAWgBQ9SpGlYooW
-McJYNuTND5SQk6EPfjAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNC5jbHVzdGVy
-Mi5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAO/iR
-mu5N94XQp0J3K5v0ZZzP2jyhpkki746RtHO2NGIOaq96wdivsvqYpDMCI1DXUktQ
-Pbt1Wti4TkmyOXmDQdHtf8pWFHeAPdZgi/g0ThPI8Fz6crrZeJZL9JCj29O8MxQT
-LNf0T/FXeo7EChM4VVekMcv7eJXwKTWXOng6Axw=
+MIIDTTCCAjWgAwIBAgIUGkJg/U5N2vJRe6qhupaa/Aq8tDwwDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjQuYzIwHhcNMjIxMTI2MjEyMjU3WhcNNDIxMTIxMjEy
+MjU3WjAQMQ4wDAYDVQQDEwVuNC5jMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAIVaLTgKx51KbttfwBSJRgmFiGU+RGwUjOXm2g0sm6ZnLGmQZH/sv0Pw
+RYtufLYtcqLvTzBRbU6XPdOcnT8el2tu4gixiPiTe67WQi59hXirS6HLqkZ9CfrD
+r1VegU4LkWWOkBU8w6PSIiu1NhX1Xwtnw4cCHQJLTLi4pBD3VPV3Xo30wmPndO2L
+XNtgbUgh4036+oAc31zc3qEYsB9fEjS0QItE5h/RjTPKmnB4XzDBNOkENdfL6cMq
+n62og8HeLQYNqeTgw198WkPQiKZX1GqjTUz1GXFgUsofddVAs8dCbATOBh6jc3r2
+v9tuYmz0BNYAppkz3nI3wHO2Zhp+x2ECAwEAAaOBnjCBmzAdBgNVHQ4EFgQUtmtn
+oGqsCEDnKwVDkohALut3QrUwHwYDVR0jBBgwFoAUtmtnoGqsCEDnKwVDkohALut3
+QrUwTgYDVR0RBEcwRYIcbm9kZTQuY2x1c3RlcjIuZWxhc3RpY3NlYXJjaKAlBgNV
+BAOgHgwcbm9kZTQuY2x1c3RlcjIuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQADFtnXrG+jl9AO8Z/EPLMFRqbpMx+QqWn3G1fLtxc1
+zldw9tS+I3SM6lO0XtbIEjUtFRTj1Jx8TPKebHU1uYRcR4XWZaFpXBWUwsKwD15R
+YF7sOh91CAGHtlC3dHOqr/El3VFXxmcYJSYg6VRHYUrBf4le6N2wmlqqD3HMWvHl
+JGOxxg2PC121Ie1Iz28fx2zffa65izZInCluE0S6/Z+FCI84An9QRVnCgwLI/4jq
+lcLIBb6I/3NUHVbkxsiwUXQxhRZJuBk1AELyltwKqtIIYhm/XY4lgqTV6qPK5wig
+JgJZDh0LDx/DofhP1yRFO3pCrq8kzZSvqh+mWwTV6ndn
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c2.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c2.key
new file mode 100644
index 000000000000..2ccf72c9bf5b
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c2.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAhVotOArHnUpu21/AFIlGCYWIZT5EbBSM5ebaDSybpmcsaZBk
+f+y/Q/BFi258ti1you9PMFFtTpc905ydPx6Xa27iCLGI+JN7rtZCLn2FeKtLocuq
+Rn0J+sOvVV6BTguRZY6QFTzDo9IiK7U2FfVfC2fDhwIdAktMuLikEPdU9XdejfTC
+Y+d07Ytc22BtSCHjTfr6gBzfXNzeoRiwH18SNLRAi0TmH9GNM8qacHhfMME06QQ1
+18vpwyqfraiDwd4tBg2p5ODDX3xaQ9CIplfUaqNNTPUZcWBSyh911UCzx0JsBM4G
+HqNzeva/225ibPQE1gCmmTPecjfAc7ZmGn7HYQIDAQABAoIBAAgPiuMz15wc5zWQ
+hKJZJ+gkb7m6+VTVKy0sdqrMMv5shyU8aq7G50rah8GlZJl4htDtiUpg6awR/VVA
+xsqNdVazpasj3CU2eQf5AJgh00MXi6NUlc7b+RH06TDOXR+UGG5cnz4BZcwNxC4D
+LqlGI0AjvhB3orxEQ/d4hcZWxQZvYetFaBHENe9f1+TG6NizRL0vwbNkmhjFpMSO
+fQoXyLjrON9UTQMHkDhhi6gmNibSuwfeBML5b5VuBka7fSEnfwjjzl/YQZB24vMh
+12w7Q/rY+BLbQ3jjfjXS9cVDxavH+oZDZC1+ssET4mts3ZpXos84L7ul7VBVTUUK
+Ohw1ykkCgYEAteRqIPGInIlvGCMXprBSVKV2U3C+xrruGxeltpmJdtmi3UQQ6LKv
+H/txcJIQ96MJ0yJzSSy27cBG0y+3dfL5jT4EYjDuxSQpPm+xpGXuC2yH3nnIHfFb
+62F7iWoXSrMysVFPuna8woYH5NTDAftbkMsqYlJaPRWDtzuWSRTBpz0CgYEAu677
+JdiVmXGZMp4wkRx45zF8GPJVLdYyQkeuACxw+FVqxgRnsqMnUl+Oga1t0KBumD7j
+mZjgzGTAumGWdnyOeIfowhZS5k6unnXylvWBEVRhfP0HanI4//X1JbED7xGQVvEE
+16JqkZCheI3v6BJkU2rl1/sil0vtJ7slkHD8QvUCgYEAqZjfx/XN7j9LIdNesyLG
+sXxSNvzyNTSI3SQQdJD2lqXY8L0NkxtApO57WH4pae1QNNhzbDBQJKDaqQFd65Fx
+VFwWFEQR7imId1oH5gTQj0BT84HTsO+iSHftizQroZAy3Ri24WLjH/F/HMOxTIDF
+YN2+mEy3LSuJU+kweNXstvUCgYAW5tI0mwoeMDNubskkW+IQ7APlk2R7dGCrn82G
+reWXF+0wfocc0DXTIl/kSQJWa80y/LGPI42Cw2SVGaVfjfFgmcz4KfsxqTY7lc7l
+Rs3aDH7c3o9SGOAFs8CXOdNuD6maQkzB90qjuo9Jg6BbpCrj1u52/WkKJzSoHXkG
++BKAJQKBgQCWan8uDctb5gJ9BJHf0HmYpyVw6YTWBFieG7MjMqHpT1V/LV1X2L1s
+wGUCASP0/1HD1RwRomTySKib3x7ippJojl8CoK4Fv54qUdTRQQ/hKao1tK9wDU4K
+GN0QZpdBwQynGi1KyqaOWgfgLD0iGkjJrxcMhL9+EHnbair3dNVsyQ==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c3.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c3.crt
index e68c6097b31d..ba49233bbdc4 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c3.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c3.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIULVXMkTsJtk6I/NjAm5UONplDpOowDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjQuYzMwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0MTMy
-MjE0WjAQMQ4wDAYDVQQDEwVuNC5jMzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAlLG3qMUCPi2usnc7jsxRFHVGjbf5+HWY9nWaboIQ1vHfn0Z3m/hOOCvxwaNY
-4RF75V1ZSeRsAKzGeDD4NC1KC2Dxa81elMfbXq6nLDIfh0klkK8lSw6snzpe8NDM
-SJ8agNdr6zLUgL9e3qh2pU1Fc9ApR6+HRXA9rcnqp/xm4XMCAwEAAaN/MH0wHQYD
-VR0OBBYEFCz38GxHKudMx4dyBlrAcIB+8XC0MB8GA1UdIwQYMBaAFCz38GxHKudM
-x4dyBlrAcIB+8XC0MDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU0LmNsdXN0ZXIz
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQCNoVNn
-+rZHQclIPn3eB5sFMypQemZSz+k6VCm1o85lCBJnMn7NrR90SFY6PDN2U7WjKLXB
-6Q2+xqneaTLi2zDU5LvbBmRakXHn6kFWU8xVnTWTanQqvh5lqwptQjb33yN+iSBT
-/D7jEcYAcntrwtcel1EjfTbmfpB3UsyG09s/sg==
+MIIDTjCCAjagAwIBAgIVAI5MDCx4+sc2UxgCqPOc6VYX727ZMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW40LmMzMB4XDTIyMTEyNjIxMjI1N1oXDTQyMTEyMTIx
+MjI1N1owEDEOMAwGA1UEAxMFbjQuYzMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQCY/ApQs6XT/wALVRAlXbDFDci1gEM7HVX5MmHQr9rBnnXKY+JbFAB7
+nVqLG+AXse82I5JTci1ESz13wxnl2L5Dh1Fi5dkatEycnQ37Zo+dyjqHJz2s1M5X
+WnMpEWK71vJea4o8cugqYo7SlTp8rhNr4PTHsyQ7yRl7t4gmdbUoot3v+n/3ndcN
+D/Vk/EIqXeBWbW1AlzunUR2+O1OnsvhRa5uVhSa9w9dz5IzIKXsvQ5emax6cO+yg
+ImRGXtpske9cYkg1IaKlINcf/WAT9ecR8gAX/8k5IQg38ctR0H8uVfD94x+BoaUn
+w8uBH8/xI+L2TjI08vOp9Q3TZNTKsmgXAgMBAAGjgZ4wgZswHQYDVR0OBBYEFEuk
+04CXzEzB3+hXzJTq+ak4WEkvMB8GA1UdIwQYMBaAFEuk04CXzEzB3+hXzJTq+ak4
+WEkvME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGU0LmNsdXN0ZXIzLmVsYXN0aWNz
+ZWFyY2iCHG5vZGU0LmNsdXN0ZXIzLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAHq66M2T/xkruS/uPs8iqtv+UbEwMAfscd7AkhPXu
+y4ir/U/J+1p+aK9w2LY+HHi8gvUM4VOFER/XL8ATvImP/CfoFQ/m8CqAS5xF5KCh
+iUuXSG0MVC4jia94f9gI9JTvchIeWn53OPkztISPTmA0AAqz9G3fA2HUtC44BLpT
+1cPBxcJu2cUe+WQsMv6+7xq2Bf7QfSAP4OgjRuhVD1RYksRSQdEHhpmPw450vaJl
+87uG1FsX5T6yxZ7+cu/kY3c4KGmFqsDhrs3MXQLz5qjp462poP23NPx/ZuCnWePT
+dXEkrjL0XqTTc5nRRz0CSiyEdTrdaPRxMkWQ26J7+67prQ==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c3.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c3.key
new file mode 100644
index 000000000000..8ab1a5a75dc0
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c3.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAmPwKULOl0/8AC1UQJV2wxQ3ItYBDOx1V+TJh0K/awZ51ymPi
+WxQAe51aixvgF7HvNiOSU3ItREs9d8MZ5di+Q4dRYuXZGrRMnJ0N+2aPnco6hyc9
+rNTOV1pzKRFiu9byXmuKPHLoKmKO0pU6fK4Ta+D0x7MkO8kZe7eIJnW1KKLd7/p/
+953XDQ/1ZPxCKl3gVm1tQJc7p1EdvjtTp7L4UWublYUmvcPXc+SMyCl7L0OXpmse
+nDvsoCJkRl7abJHvXGJINSGipSDXH/1gE/XnEfIAF//JOSEIN/HLUdB/LlXw/eMf
+gaGlJ8PLgR/P8SPi9k4yNPLzqfUN02TUyrJoFwIDAQABAoIBAANWWYyXJAKdk80A
+0bWu9zXlQ2qfiK1CdOdXGlvTakHtCK7Il7MwRYPCKVZ9iNmbemzDlo6WyrasWYPs
++qSmoh1BAh0r3HJBBVF+pO5wt1NUNQtREimyCWpNwSi1MLfeTDKTxVPk7HxNjXC5
+6/5focbUWnf8yB6RwtTjFRJEYRZTe+rzFUQwZJPVh/AQJ3i9fQ3LGb97gLtwYme4
+akZ+FIWfBS200lzDasVMRwNZEycNPToXVYRDfW+QmCMmQB+QEvkezRKgMjf12F1m
+mBk3HlS07U75fs+kT0P0JLaVt7HpNMyeZrvDthif8kX9c2w0S1jI2UscU1n+VkWV
+/VDV4KECgYEAvcHMwGusxnTyWwQoDXUtQFTRaQdklSq3oJI85O+62Kxk3ZdXqufg
+kiNl5zdC6pKTMKivOtBM3wlPb7SC5gJF8FZ7FT7MV5B8H0Zuj5UjEOLYWYWDFdiK
+DJTfACtUiq4VG8g7b2xidtcDvxorXgl+2jhmHkR2LWfFOde5OuideucCgYEAzmP1
+AaG6nKPqEuaDO8US7/8xvEFUg7MDhaD0OgfyVNHvBboDsKxxxMxlwtIBCUcWLe4s
+pnM8pByE0WZlmurUG3WPL10+/3z3RH1JyTAd02jzcqlh8HkKQZp3AkpmDzArk4Zg
+W5hrjcFfff++pvgcAx338Nmx/iDv2uuv20W5s1ECgYBTEovj1X+/JiWPqQMr2Rel
+aTyAQYdMMnZPapYUF8tx3g8OX5o2rOrIr4yGK9A0x1R+4epM0jJFFltbaOhBb89U
+h1e9Fk/5KbzdKEVwwHaohtb2YhfgMXtGRMpX1aBeUhXVotQ6VHXK1xjByeAgfAG3
+4Sb7qzcda6eRShrCamG/EwKBgG57H4pSEt0K+aBIePOj2TFTwEHwDte5GieMM3Bl
+FJnHD1ZXbuZCSVx5tfeGBzv6K6tGlew8P7fx9D9LuxQkpIpWceF+3txQ/uRQ1RC3
+ix/CTJ+SgHUDtjPMBezZ+qwisxDzm90sSFbHruB56IrkwcoBXto1DozlAMCF7ctJ
+FdEhAoGAaDMkD3P0+EfZV3KgnarvfJlk7ALI1oA4bS/tWAY8vGJk2akS5AJOsCDy
+krepJ9suoKVeGwEoEsChlxs9uoMskwt5kZ5oy20X9VYrsf/gj8qmmGUKdN9QPSvv
+uNKkdOccrYt4XQyha6s0sS/MkeeNgIbYK82s4FCrN3cT02tFTY4=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c4.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c4.crt
index 29fc38bbe7eb..9aefe5b2e80a 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c4.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c4.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAKdwxZ0iNx1vHQXPhBF0+HxwMwMFMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW40LmM0MB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjQuYzQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAJDwOCTGpe52Ph9EXMq+Spr7uLi+hxtJFsNVy16vzkSfjaEKhifl95ieBRzx
-1m/Zesn7prrSux00lMer6Gc4BVRtTnGenVGpH3wpG43qBlXvuBtqI5cH740K6yYa
-AemlyS1kUw2OFazToIo5Qc2MMmRxReCBywFg//LtEQhi3YmbAgMBAAGjfzB9MB0G
-A1UdDgQWBBQDRkoU6EfsM98Japt0aiRlTZAV7zAfBgNVHSMEGDAWgBQDRkoU6Efs
-M98Japt0aiRlTZAV7zAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNC5jbHVzdGVy
-NC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAKfU5
-VHQXHnNTvfvzPUD0PDswSvqV3Sp+dujKXX9YcSEtp3kIJzFKDNb5Pjw50+/H5vl1
-Pu+n2nGaFxSHnqbsGGMylzAO7qngaJMUXuFyYHpyxf4dXqu3hjNfQFQaAJ0ggMnp
-WZfyNLA2TO20ptEDa1A5wbLqRu2vLnLeGtyBfqE=
+MIIDTTCCAjWgAwIBAgIUMGFzK+eHNmepSg75gEXfiCojpKEwDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjQuYzQwHhcNMjIxMTI2MjEyMjU4WhcNNDIxMTIxMjEy
+MjU4WjAQMQ4wDAYDVQQDEwVuNC5jNDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBALeJc6AJEFfUDid3pYAIMHtjdGGw0OKTLeIKBej/YYME2rzi0e7F5dDn
+gF2AobXfS/vnYm00HeIb5h+oFdPfxZpxYKJ2jqot6MlnXkzn5n9g7sOxw2qzO8FB
+4uNIhwxrcX+j5oKXS8828m9+nrs8WGt+lKezurXG0NbNeDhyXdkps+JloHvW/Ix6
+BBMOTRy6c/hsL1Ro3QqTpsFEAs4+VwxIUYVH2TPQYua7VUYaxFXQQKxCr+lESEWY
+t3D/+62fX66QpmMvOP+e4W9zrNKthmVGKeiLt7WF5iCYTz262CKR+xTvdJK9tTk9
+UTyKVSOTpZU2tIfdgA4eV8wSaVdnYZcCAwEAAaOBnjCBmzAdBgNVHQ4EFgQUm4AC
+GOQjiaPZBvc9/DdeF4qGUbcwHwYDVR0jBBgwFoAUm4ACGOQjiaPZBvc9/DdeF4qG
+UbcwTgYDVR0RBEcwRYIcbm9kZTQuY2x1c3RlcjQuZWxhc3RpY3NlYXJjaKAlBgNV
+BAOgHgwcbm9kZTQuY2x1c3RlcjQuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQB0Io+OWf734N5XdxSwI1Qv6DLODKedfngEHWlXcv8a
+oUwWf6IW0RX9/DJb8jN7rugiy+1J9IXtDfpC4CfHOHnNcv0FzsKmfMTUP0zWLFzZ
+V58fgNQz24zSs1uyJsS1nu7RCb4qYVpAegyMcD3UctmK+STw3An4JNeNssx3sS6V
+37NjYE3+62KXKMtFQhICETfMjrUnz1ZNfxXeREWMN2qgrVTl2QwLOhT4rgDvI3nH
+LzLB3EU/5M8Z5gUiZEc4S62B+SsxaMxh1u/j6aWZV/dSuyVZmqqc+MpE8LuyLEDj
+Ps59DYN6nb9YzQ53jRpNqXqObXEkJJaVftwWJO1k63ll
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c4.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c4.key
new file mode 100644
index 000000000000..f835559dab42
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c4.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAt4lzoAkQV9QOJ3elgAgwe2N0YbDQ4pMt4goF6P9hgwTavOLR
+7sXl0OeAXYChtd9L++dibTQd4hvmH6gV09/FmnFgonaOqi3oyWdeTOfmf2Duw7HD
+arM7wUHi40iHDGtxf6PmgpdLzzbyb36euzxYa36Up7O6tcbQ1s14OHJd2Smz4mWg
+e9b8jHoEEw5NHLpz+GwvVGjdCpOmwUQCzj5XDEhRhUfZM9Bi5rtVRhrEVdBArEKv
+6URIRZi3cP/7rZ9frpCmYy84/57hb3Os0q2GZUYp6Iu3tYXmIJhPPbrYIpH7FO90
+kr21OT1RPIpVI5OllTa0h92ADh5XzBJpV2dhlwIDAQABAoIBABWzfeVtTO5ll2rZ
+PcterITaZJdsdbZkmqdAciRzlc8NEuQEmbf21E6YINhtbjF3/p8BY+TvpZznmZUi
+pboQmmfBgiCTjhjXJtAxlbfYL6veBT1EVovWN3mJVc9z8uvoBFqIQagtRXxxzcab
+HR+cagQDyoXtvWTi5iyPzOg3tZsInP1YJvJLt26HxBb4hs1+LRQX3kD5LgUTAvcW
++QB3CVxHwV65nSeoh8aVVvchwPQZ/0i+wGR/teolw5SDSa6nNyn1wz/9BXyuoAtV
+DsqWdrTAETMzkNGA0qbxpiYclXPqJJycoP9kB5shdvUXw9MNtDVIBW/cGWLsPjBv
+hjWuIvECgYEA0Keii2jkx8M6/YZkBU100lsg1Nl3vBwfwIxabhRKxNpGoSSrA4Z1
+8yOgXXM2YrpNQ7XCEavkv3ricTOQjoR1BqTNjFeARNzpgVYV1GgEl9DLU/wydbHN
+96syW5fBpuaHF662n2nCVVEKglrloS7qJS9tfkOnJfqgLdu7M0rrWukCgYEA4S7D
+3ZdEvYyVqO7DZU+FPaMT1zLGnGFrNZcQ5DKh8Y+xwIymnhFVCYsayOV3yZQDlilp
+F3FFKFRzrobyJT4DzNu+oocZUMefg6XVobhJi7pH9gae1QZXGKdn1O6pnyniw7my
+hOF0+3AK1L5WJRQpWCBm1qmmVmiKhApAMkaeCH8CgYEAigAlWFEC50Avdth4tK0a
+87nDyCMgtvBe68moUZ/oJ7Q5/IA0/zriLfJFMSlvMfMIAR1ogqmOvDn5JGe3Vxsy
+DzfWntiMhH8bjSgzfe1iBEra/+wQL5luoTzALKX9NXbIqJgsPtkC58yWx8p4j5do
+0zlAEL+MAGl3oXzvnigR62kCgYAd/FM6s8UElK7tMvY0DHXKsmeW9Xz7IAHdvgMS
+0Cc+eosJ//DEBbzUK4DilaPw5NPhcBw9cejqqOxKpk+iY/Kvmed1zxWw6xQzvKaJ
+BXhNsf2ZJmKKLXD1qH0sHqfZuQMoWxcJmNgyD0Zx8/KWt2uV/6KSHT84LG2djZvk
+1gB/UwKBgCzCSqYubsoshymflu1KkH/UmKvVE6YqA+N+BEDmyF0fWL4OkYkDwJ04
+xQE6ZUnyd7ze0Qv4/B1wqHXnWRbNOYLx0Eira5IpXxWP+JV/Izo8tnMNBTTlC/jX
+PCoZKWMfCNcRDlqvj28ubsSUI99M2zUZyBABeiqFyM7W1Y70Dj1Z
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c5.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c5.crt
index 2ff79753fdad..dab6ebbbf5e7 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c5.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c5.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAON+/VKvMR2xMgnemknulon7vQlmMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW40LmM1MB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjQuYzUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAKCP9rT1CKEdFPbGE2ywJgZiX1qf5H36DwokH8diz/XVk/t2QOU7Hqui5HJk
-Q6Du6ywPXOqAIg/Iv3Rj1Vo+B9jcPs/DY0Sj3mKdZHhySaoTHPWjv7mVa/PLfMEl
-bmB4sYkgfotHVxUir3Xw5byGkJ+SjG+xYrMOSktBNph4Zh4jAgMBAAGjfzB9MB0G
-A1UdDgQWBBRrw5xcRVozWKOs6ZHd4Q524MCFyzAfBgNVHSMEGDAWgBRrw5xcRVoz
-WKOs6ZHd4Q524MCFyzAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNC5jbHVzdGVy
-NS5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEADiK8
-asbnbUu8Aj6OO+lizquvCx+p1jTHN9+teu2WNtQzcSqDvE7AzvtFQoU7381BCIoA
-IrtCwnqK8TIliZfLYK4kNWAW2Z1d6fn7KlL6HWwtHBy+ht/7jr7M+wcLd78UbxZL
-mdCX4+s0FJ2mfldNe8xq0yRjM0laiF+P3F2WREU=
+MIIDTjCCAjagAwIBAgIVAIMscC0h1NnPF1npd3hcWxu/jSAlMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW40LmM1MB4XDTIyMTEyNjIxMjI1OFoXDTQyMTEyMTIx
+MjI1OFowEDEOMAwGA1UEAxMFbjQuYzUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQDss810Ct63IBqOAbs2IB3zvJ6HJuz0qsevoilua+yV2BYcocbCahkc
+ETDmKb1jwt4uSvjQBayBgu21fwdm5C75GH70PoFsQ4xQDE/Yf6+lQP/db0DUvbyF
+NmaaTZ3g6yxel67Auixx2PFFXjkYxaIbmvRjHWpzKmGij2lzDArse6tkxpa6+x3j
+3ZrZeBntbZWq2AO14FUdei5buOqNhgv2ZWGpG9h4RKY57KoPM6TvdPFtzAa8aV6W
+Ja4G+U9esXUX8+g7NlmjFF94UCkJubZq/S6fTM/J9ieJz3NO4z8tPfYAQNVnbXWO
+r+/R0CcqT9hUyytoLrSw3jC1R3KN5xAtAgMBAAGjgZ4wgZswHQYDVR0OBBYEFBjX
+1TgKnV5XM8N3vikUG/+KOduAMB8GA1UdIwQYMBaAFBjX1TgKnV5XM8N3vikUG/+K
+OduAME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGU0LmNsdXN0ZXI1LmVsYXN0aWNz
+ZWFyY2iCHG5vZGU0LmNsdXN0ZXI1LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAZulsFv45x9YamlzsBrG+pHSmWO0uOogfurGFr+/m
+gfRtQ4ozZkL9tghMqxv8QEtTe/4c5KO4JV65neSanBfdD4W8mhQkfRuTSIHwIIOB
+lVeKNwiAoHsgS/Pq6y9pEeZBrd/0mX18cNCCEnDfkcg9cgTfDB4VkPqhChfbm7n+
+0vJWNFFihjjX7dFo5clC7zjRLTOiNgNtW9jhATR+KAwey+5T+JmWQ3utArsXow9O
+lt+zoRD8s35Wna0+UxTR9NKt15hx2OYJ30MvQA7ZsCu+HaubaadtMeNx0aOrdz63
+j6RiglJmrA9Da8ApXq1/Yf5kEuzbRM6ap68IjIsQx32BRw==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c5.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c5.key
new file mode 100644
index 000000000000..5490b20eacef
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c5.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA7LPNdAretyAajgG7NiAd87yehybs9KrHr6IpbmvsldgWHKHG
+wmoZHBEw5im9Y8LeLkr40AWsgYLttX8HZuQu+Rh+9D6BbEOMUAxP2H+vpUD/3W9A
+1L28hTZmmk2d4OssXpeuwLoscdjxRV45GMWiG5r0Yx1qcyphoo9pcwwK7HurZMaW
+uvsd492a2XgZ7W2VqtgDteBVHXouW7jqjYYL9mVhqRvYeESmOeyqDzOk73TxbcwG
+vGleliWuBvlPXrF1F/PoOzZZoxRfeFApCbm2av0un0zPyfYnic9zTuM/LT32AEDV
+Z211jq/v0dAnKk/YVMsraC60sN4wtUdyjecQLQIDAQABAoIBAAl2seDxG1D2iWFC
+m/XQyGnJkYwFSCWLG3M/+EiYK4wdKNsps6zruYfH/yzAn/gYXzI6IKMOiwRdk/v0
+8XqRZnhinJbO1fVaaQZ9S4b0KfVhOSmlDFJFKL+pumXycvy+YlaA2YWOm/7MRe84
+Um+VXUMdyaYZ07r5h5BMjBKDuX8VagSz3JAb/47kIt+yuOu2lYSm/UBv+01V2xYb
+K6fahrkIKyA2d3YfPOWnQxB76759fsprcJUVWUH5qCO2o7RpWWP/waF1nmF/5mkN
+mCsxOXV35Xyler1RTQu09bTQNVOpDx8xPhS3b6BqTWP0kEWCn/0aQzuteibba0wD
+IEs0/XkCgYEA/+dDdfk375wJcFv7du/fKc+U/UYuR7sihjdMJOvTCbeyxe2tf7XM
+mt2AC7H0PzJG175LBZUS5+6k0B70+LWKTwbUtJROC3Ltlm6tCUu7RwxwlCl9J90Q
+QubyZqLf+RStFiZAW04mCoflXXPOAdI+WIb5BYhnVyPVoRbua8woYlkCgYEA7Mqu
+2PMk47Js8Z7E2akFOiDS7E796vZqqs7vqkwrRU+JiMPyRjyZgUWYfUTAKz/bDULv
+oDU+Dco4SdqOTL05oilWV1DoE6dkXrHzUIP+nZyabq+nOBP5bBKoJzIEezvGEbpD
+LgS2Go9oeZMvubg3QrgIlU64wF3pD9t3jWeQWfUCgYEA3NPKiUF4nZJ/KrCVfXtd
+jo0NOsqXIVtBXaTWVn4SbS5SXS7PK4fMek9uMals5iQSshYKU1TjLbXoD5jHOOAf
+7DYu022vM34HvGrhc0Yg5By+AX0T/ZRsde8pepxOvnizLIvxuVjkxu2iZlSKj1PI
+bUh8F+QvmSjZLtvE/sz8hHECgYAdBE1e2pTXrkK5bSpUPekdEW6ndJqbE21I1yj5
+ZnIbAT8qWlCzGLP66mtgVm9yHVYbjutMZ5neGLzDGBw9/SY13/jrpEd8ORQMoqPd
+Zv9pkwo9rBrCvXfgLX0bqH5yF4YTIA8hcYRbRt2XoFjF2nQzIdgza70qHZf00IEa
+++jtjQKBgCSvTCcQR7aRE/kRi85TV8GEqDVoBTbeBXq8R4nN9IniIABfkgj3JPmE
+FOM+/MiRDCUbbqpP3MLwt3DLsccrMBdLwXl6tsTgwYt7C5yZ6gy7/sTwTyoTZIMs
+o4mXUVDB3BPyEd2NrItkUqxSPkBQS6mEJ6ak/hkg977vdMaF4Gb/
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c6.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c6.crt
index e22f9161b0d7..87402f0f2c8e 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c6.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c6.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAJ+HlcBtFkTPyQrCCvbdHlBLtP4pMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW40LmM2MB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjQuYzYwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAKiyEVPmc9H8quSkeyFE0GpA1yuRJfDd1OJ3I+ogifTkmZcKu3u1wCbxv675
-xRdpRzCbB8+hUr6ZC18qWUDxXp1ml4iRr8n8bVAmENgJzzp3RLBZROFuwVe2dttu
-n1SMiBlnlcUnmSCX7OCTJWrr3tI0/F8RIdaB+63gf2Eby2eLAgMBAAGjfzB9MB0G
-A1UdDgQWBBSg537AtIe7R0hQVRHdpCNBBw4I2zAfBgNVHSMEGDAWgBSg537AtIe7
-R0hQVRHdpCNBBw4I2zAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNC5jbHVzdGVy
-Ni5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAUdzF
-snAZflUna0f2B2WW03U3sBl5aybZaajWNxH/u/CCnSmZcdSytawJ+iuyuEaLMPci
-zZqgxRVfOwgKlKou6u2hu2A6zOWuHI83LfrQXHNipy7X0pEVX31tgb0yRu4azsRr
-3tm4FOLGjYqkU3LviuukIBSzigcOp3s3DkLI60Q=
+MIIDTTCCAjWgAwIBAgIUQ6I6ZODwCiYVpokq22Uvm7drWJ4wDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjQuYzYwHhcNMjIxMTI2MjEyMjU4WhcNNDIxMTIxMjEy
+MjU4WjAQMQ4wDAYDVQQDEwVuNC5jNjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAI4L6UeuVhkmz3m6v556y/yu+L7fLZa5LyruWVOfQiqcoWtgz+GN+MgH
+GyDeCT5vDlSFuIxeW2HmEu9fzlIirvmarnlLjeLfkZ03bGHT4UxlA/D/Cnfeo/UD
+Q0OoeNWKU4nf9usYiPg/QuF/dcyBoQn1TbbaywrZHEebt3KmftPzyLBuo9E8IQWD
+yLsDUpssMqjcwIz4YjsSes9gPdhIQtQsOVoW2Z4BXYZ6+KHAzS17knS7KTrZnIqS
+M4n38PFWyysZqy6hEab5Gx1pVuw6jzCUAMgzCh5TbciwTie33E3bNvAgza24+dLx
+QVitADXThri4zD7jBVUofQkmCvaksS8CAwEAAaOBnjCBmzAdBgNVHQ4EFgQUA0A9
+/zFtZkQEa024Tm32dS7pPnMwHwYDVR0jBBgwFoAUA0A9/zFtZkQEa024Tm32dS7p
+PnMwTgYDVR0RBEcwRYIcbm9kZTQuY2x1c3RlcjYuZWxhc3RpY3NlYXJjaKAlBgNV
+BAOgHgwcbm9kZTQuY2x1c3RlcjYuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQCMGviMfMk8DZfI1DSJFSQiNgYkHYxd/96kGpSehP5C
+kwxRLos1hJO1p7LGd44VYizwJQoNNPU4zzatyFPLhZd1Q29deP4QFquV56MwV3qh
+BO4fcTPkcnAUU0/W6DU6VZLT78kYzLLhchyZ7Hid/pmvQ4f+sxqyK8xNI7h//V53
+Pn1RCcq6saqb+Ebj20eX1xdSX+ehCD4FXEPY8Iz2Mjq89EbOPjrZth2al4B1xqRB
+UNsA3ElwbXUJ9wSBbi6BKvtCg64jysNDj5dNhK4RJLyLaIwwlQfgZ5edzxjkKRMe
+9vi25UXCPAIB0FWEQ+TKPwN7yfuWyy+5s4bGkwnScrnq
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c6.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c6.key
new file mode 100644
index 000000000000..326fbcb0aea3
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c6.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAjgvpR65WGSbPebq/nnrL/K74vt8tlrkvKu5ZU59CKpyha2DP
+4Y34yAcbIN4JPm8OVIW4jF5bYeYS71/OUiKu+ZqueUuN4t+RnTdsYdPhTGUD8P8K
+d96j9QNDQ6h41YpTid/26xiI+D9C4X91zIGhCfVNttrLCtkcR5u3cqZ+0/PIsG6j
+0TwhBYPIuwNSmywyqNzAjPhiOxJ6z2A92EhC1Cw5WhbZngFdhnr4ocDNLXuSdLsp
+OtmcipIziffw8VbLKxmrLqERpvkbHWlW7DqPMJQAyDMKHlNtyLBOJ7fcTds28CDN
+rbj50vFBWK0ANdOGuLjMPuMFVSh9CSYK9qSxLwIDAQABAoIBAAKepEjBewOvvDfU
+gorbsj4QxAmmeA1wO4x2ejND8VEeiQ4/6irmFrvUVbhqH+WE4CYdk+US1cCdIPrv
+22WXD38O+T9yGJsh4GpJMWMvKIKOLQpuybSi6Icw7jal5BnxbnpRJRlPidx6M4jm
+dDe3VE7+CoGK0Mp8bTxs2hnv3tBDMbGpM3ByZXqboN1kYcLV7nir2X7CdSrg2ucx
+zTa3Ei5w6jEOWyqzWRHxLcg1mbRAPL3ncU4V0snSslsEB1yiMYrHoFux30YYFv0m
+YyWuQWNv8uAfD+7LgO7SiU3LuYDMapCHXjJ/jm/okQ4Unv0vyIeB3devZfA0FOne
+UcKe47ECgYEAv0xuz4JUssn45qs+o3+ihkgBlzQ/9xCAYCiGihvjB4v3A+c4pHVd
+c8AgyRRVyHt91GtMNwuFkqGcvDKQN3SJdzz3zHBa2j+2nOTSOnolry4fHKBhoVFR
+lafbRODsg9uj/wJlJEJUBIuzZy8Z8ZwAK05LKnqX+lbCGgcPlOH693MCgYEAvhb+
+4+pC8gtMlVBQ76lewN3AlIgIfu8epZWdmsZcSYFx7tqJKBXkfcOd4VCMpJFixjJX
+0K2/cy5B0aF0fnc/hf37XPsVcAhdhAe5pIzlNF9d8z/V6piZaX9otQyu+Jmg1OME
+FghSi31kVmy21k1FgkFlTCbfPnnOf67MsOd7WFUCgYEAmrWr4HKdIUMq4xKXRA9T
+V5ogfzKokJ1t1VaxCDzpbQ8LJML5EjhA7f3yBe1GOy90YjismM/epOnDIokaDV9Y
+VXCfT12GfH+OpnCAfP6QP47jZnlmu9pSSuBH+vqVDUay6Of8+EgUPuDkpIx5ROdP
+U7asH5yqELHdpMMVHOlt2HUCgYBOaXn3VjkLUzHiHeFlGz6k3FSGcJ7gjY7cyxza
+LZ0ADU+MMcWGLZG1dy33NZzufbJ7A2VKBYIF5zzrVmiRims/wIch0ckED89TmUQw
+5pXdxsEgiGKbhRszc4UjufmrPuQjLQoQdJpoQ+9R30KEoBdeJc+9XAOW5/QQo3B4
+QOSWiQKBgQCIAjRqbgwX9KrBBnBcpWMeLqDQarRe4NP70r8HcHnbY6rpt1v3CsAQ
+1HKKHzq+seclgdcqZuGIoaLw2T2/mvWWCzRCxqDTsSl4o2iLvclPpHVZLQQbLS0w
+XQLgu5BMjeLVL5ykjUGTtYIeMt/61j86PkR6bKcRzgmKQeaMQEteNA==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c7.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c7.crt
index b0ec3d41aa8a..8f2439ac1dd3 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c7.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c7.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUb4zqML7sOuozKkgpS93QUD40XSIwDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjQuYzcwHhcNMTgwNDE5MTMyMjE1WhcNNDUwOTA0MTMy
-MjE1WjAQMQ4wDAYDVQQDEwVuNC5jNzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAjvRiPU8N+9dppeWCZf3gRL7r2afS6iS7YZn3yqYeikXec14GkDotYxR+ZPeG
-3iTwhE73I+iXIScqZmtqk6sL3Iv5Z700w8CBIhLFZP0L6MPDpT0WwBweRzC1BA1u
-Xam3Cwp0hux5xAH49h0EZvFT6bxSVdhrunsWlW1N5vZIOlUCAwEAAaN/MH0wHQYD
-VR0OBBYEFEMbNKUNe9yyOtm8R/QonVKFl7C9MB8GA1UdIwQYMBaAFEMbNKUNe9yy
-Otm8R/QonVKFl7C9MDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU0LmNsdXN0ZXI3
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQAKn5SX
-5Cwk9cXIijfveeJ+vzzqs3K3FBukA1rT+0PrFRT7kZ4Zg5IeUwCGnBVDKf4XoNyO
-Aq3ads+iVOOVn8GvlMacvh76YqDmlgF3TqX3KJRm+NEVHwonwdaxXLMi+iK7sbtA
-af0IE1zsgddYjz6yQ4nbFGKjji7U6MxwcEKRRQ==
+MIIDTjCCAjagAwIBAgIVAMIJinHuSd71CpBcnBwTrF5CS67XMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW40LmM3MB4XDTIyMTEyNjIxMjI1OFoXDTQyMTEyMTIx
+MjI1OFowEDEOMAwGA1UEAxMFbjQuYzcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQCjgq/KdqWYn8l9dXrr9d2usP9ov4zL5jlRMswF8zSIe0+W4H/eWihi
+laPQCCF9h7Lt7Z10dnrw7xrLJoNrGYcZz+qiOFeuuMjkGb9nN7C2rOOoXe3u1/95
+70q1MqqfJY0oUZEU2f8sEu9wVWJXeG49e2dRjMC6snnDaZPXYliSh3J+azU6o4y0
+XkxzieeE7z/dbrihNAe0WqJkPLy+JGsRxeqFP8fo7k4244v5NHcCZ1Kz32ScbslV
+HtgtTSXWqATNMSZxuzvGzKjfUTQMcz2mxwMijxvB9+tY5l2DQ27+kuTmNJyibxRk
+FM2c+JX2aR+7PvbGCYjsVQCtmu+UdpVPAgMBAAGjgZ4wgZswHQYDVR0OBBYEFKk2
+LFGc2F7aV757ATsMm8DeJEG1MB8GA1UdIwQYMBaAFKk2LFGc2F7aV757ATsMm8De
+JEG1ME4GA1UdEQRHMEWCHG5vZGU0LmNsdXN0ZXI3LmVsYXN0aWNzZWFyY2igJQYD
+VQQDoB4MHG5vZGU0LmNsdXN0ZXI3LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAmoJsZszCCBZpi0Tu/Bx3Tdkyn5gL5AXJ5B1zjMCB
+M/jjxqrpdg1FrXpbfPRU7AtnYsE49M6jCiJH+SLCso5vZDhueDnZzhPlL6WQYAjz
+pykjWW3+xojS3A8OScvGUnCU7UzZSpuLQtMYJhchgdIExpU1rn/ZjlkKECWjX8Qw
+L5JESdvqINOmB6Tx7JkQ7bpjxVymMAzG2pNjkRTPQlD21JdR01hCcr2Qe/6ParRE
+WkIMnv+I+Vti/2pIcb5YwFnZJaa6SpE3yOVcvvL5PQWG0HPHPAlxOUe1Ht4RCgfc
+i+RQM0H81JRvtFjBAwihhbBmpdAR6+mdT5quvhdmj8Sy6w==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c7.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c7.key
new file mode 100644
index 000000000000..1d83dfa11bcc
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c7.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAo4KvynalmJ/JfXV66/XdrrD/aL+My+Y5UTLMBfM0iHtPluB/
+3looYpWj0AghfYey7e2ddHZ68O8ayyaDaxmHGc/qojhXrrjI5Bm/ZzewtqzjqF3t
+7tf/ee9KtTKqnyWNKFGRFNn/LBLvcFViV3huPXtnUYzAurJ5w2mT12JYkodyfms1
+OqOMtF5Mc4nnhO8/3W64oTQHtFqiZDy8viRrEcXqhT/H6O5ONuOL+TR3AmdSs99k
+nG7JVR7YLU0l1qgEzTEmcbs7xsyo31E0DHM9pscDIo8bwffrWOZdg0Nu/pLk5jSc
+om8UZBTNnPiV9mkfuz72xgmI7FUArZrvlHaVTwIDAQABAoIBABPEceIlM+Het2o+
+cKzkHXfIGCiet7PS0m0ZmgbCH0C8v6N64pqMBcPdWV+GDB5z8FLJXja9OCG2lmOm
+9U5WWLnrI++WpisOVeMVpckht3BuOFubd29fl5gzMLH1PSFeVdFhAwwlSq8WXvIS
+sH/opFM0fwoh3Q71ErloIyJv1ktoeMNpEA+UE7V31zK9PtdSZa2wovR15qoDwGqv
+Oh48+5n/1KXhcb749gXy9pl54HS9Bf9qMhjHr6fbfg1TxbHxEbMNk2anFQpSgGxP
+7DdEbIG/IcGjE+iXNfv/hIMf2WdjrcFds5naMLPLgws0drgGr0fmHXNFG4fyzEYZ
++jrZMOECgYEA5kOCDj/0RE//38KGKwrP6ln1++T/2FJCR//EUdKl6dcrdKBJq8Yx
+lJ7amjHWLOUpeMA+PvDPEHnsDSKSnQ1i/rHKA31XhOncii+Eos3BPW3c9le2xz/a
+WPTUsCPPHH1IX7TyPnqVOZfIPMN5nfsegG/dTzUm5nigkuwTLpWRH38CgYEAtcku
+gjyxDPRcduIMYHgky2nfA1TgwQs60KL4qpb/ZgYNPZXs+ltz1N4FX10MvR8Zo6ex
+QPlWv/J1IWo+4SI6um5FjP3wHZEN4vMfwEiYGEGV1M4Ml3eklLXlhiBnacLhNs3z
+6vL28u9ROUvDeG8tkJL4RvGS9kwUtVP85yDXcjECgYEA1Wftqk1FkuZd68C6Xjvo
+FHeJzgofwBoEyfvWtEq5cC72XGGNxcCrBOXxDgVwTS2aMQpi5iyCi++jgUzwetQE
+Uq7t3iwVp96iggOmNsjg9pDSwBDtGl8oe1d1o0Wc1veZ0vG9rYAouzbb4sz78xng
+3AJRJu7I/cMAdwqKgJsn4/sCgYEAn6BFYr+AiOuathnQ3HUjeitBVV+W7Gy7/LDe
+wo1NP31Y6xdXvI/JqsY62X6PzRIeJYL/Nxq7flmdWPY+uNqhz0s9b40c6BmTpBHx
+c5tLUyFkPDWjAO5iauM+5QRXSpuBv+ohodIv3ysem71Xq82UqTslIMbIUdaE6KUT
+krFX/kECgYEA4x+Uucmn3tMrOqoywd98429sNY7vaNgjLgSfdp1F4gBGOdGvavNC
+cxJMFnKfda+C09J5B5Bn1hHmne1nksc3GI/GGCdlwD26gXLLCM2UVxi9Vmp07GNf
+2HvnR5Pra4lwv0K3Njklaf91Z/uIEanQtc1ZBJo4jtmEG3Pz+B3Kgko=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c8.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c8.crt
index cbd83c42b525..12d8d54e4679 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c8.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c8.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAP9uvL1aQzoHJdqBJ277FRrW7RTYMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW40LmM4MB4XDTE4MDQxOTEzMjIxNVoXDTQ1MDkwNDEz
-MjIxNVowEDEOMAwGA1UEAxMFbjQuYzgwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAJ1FfrHkc7tQtArp5NrVzVl1SiHNyGtT42dYoceXYjfCrKoF8yT7VZdgSePg
-yCFvMlFanq2tT/QHG3kKNW7cz7wK+Y1BSBQMWSPze98dCgRStqx6eu0T1IGQZ3/H
-5Da4ejolu+g8o/G3dUez8hqxuoc41v4JBAOVDGNlIdYSlGpNAgMBAAGjfzB9MB0G
-A1UdDgQWBBSfXMfH8Xf7zowDOoCEMP76iAtYBjAfBgNVHSMEGDAWgBSfXMfH8Xf7
-zowDOoCEMP76iAtYBjAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNC5jbHVzdGVy
-OC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEARVu0
-SuTODyHIHTuHOByZ1h4UefWpYn64DwsrZC1wElUimOjcxxGhRKlgfM6aElDNLAOO
-hcEO4KX9GROFzzQ+4ZvifXWU6Wdsrmu8Y1RXxqQUhbzlQok3rEJE15TSvTspMs4X
-+7iDYZ8wvWLnZ8mf8IoPEb1r5uoJOkf00PxYLv4=
+MIIDTjCCAjagAwIBAgIVALs/XE/mtZilJ7UcwM7smjr+NPryMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW40LmM4MB4XDTIyMTEyNjIxMjI1OFoXDTQyMTEyMTIx
+MjI1OFowEDEOMAwGA1UEAxMFbjQuYzgwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQCcg8THAP0R8k9QBd5RCDndp922O1vuuc+gf/vgAbYdu/yL6Sir/eyC
+vODll7vK0nk4ya7D2eRTPIvLKQBPRJGXjbj+xeMMlru7ttQi0unZMjh/3/PN1Zd/
+UGinXY2Esgzf2JnUd3/6naH9ai1kWNGDdvVN7mpWCiyOhvZwfkXQhDAYtcYSWcET
+wCzg/Q31PIuht0hqf7i6PRIjtoop5oMlsQm5hPzcEAhXIhCKCNOl3KitQ62bw/Qr
+BTZu6pBd3O+hWv+s6Cc7xQD1irydIs5xO3sfbL2J/NqoYO0ruTXLrEFCiYwBjA3X
+AJ9I5gZTXDTcDxglL8SHq3QTfCeLRQcFAgMBAAGjgZ4wgZswHQYDVR0OBBYEFHIm
+GGPn/dSb2dsMCkiBKQcHfpnNMB8GA1UdIwQYMBaAFHImGGPn/dSb2dsMCkiBKQcH
+fpnNME4GA1UdEQRHMEWCHG5vZGU0LmNsdXN0ZXI4LmVsYXN0aWNzZWFyY2igJQYD
+VQQDoB4MHG5vZGU0LmNsdXN0ZXI4LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAZ65V6X3biHSSt4DeN2A70GUmTpP4fVykMs63YE+6
+DmYh96ZF/bXfCeLgF1TXOLNi1KDoAn+Anvm+ZuHfPsxnGbVTA28NNVDUFKI2XKoc
+wzjgxUvC5KIDagUACBnMhhDzt8R6d6vjENf98CmCHgPosLGw/ixalo+8vi9EukBb
+af+pTWpT8IOQNhw5Adntv56WktZtQz1bxJkZva0XFE+3UkB0iHbCtVq79qo4V/cJ
+7qmYGhIycl7HbNB6+ZlSqU0bWi+UBtziQm99OP+bpyoSZCizLl4dYKUcGbPqeN9m
+X3/aGHwbJeg16AHN12pviRhjVBo+eqgPNk5WHjW980vpow==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c8.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c8.key
new file mode 100644
index 000000000000..6154d76e0aa8
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n4.c8.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAnIPExwD9EfJPUAXeUQg53afdtjtb7rnPoH/74AG2Hbv8i+ko
+q/3sgrzg5Ze7ytJ5OMmuw9nkUzyLyykAT0SRl424/sXjDJa7u7bUItLp2TI4f9/z
+zdWXf1Bop12NhLIM39iZ1Hd/+p2h/WotZFjRg3b1Te5qVgosjob2cH5F0IQwGLXG
+ElnBE8As4P0N9TyLobdIan+4uj0SI7aKKeaDJbEJuYT83BAIVyIQigjTpdyorUOt
+m8P0KwU2buqQXdzvoVr/rOgnO8UA9Yq8nSLOcTt7H2y9ifzaqGDtK7k1y6xBQomM
+AYwN1wCfSOYGU1w03A8YJS/Eh6t0E3wni0UHBQIDAQABAoIBAAKDeTsK8IWDwzNR
+TM83VxqQshZVoLXWRRfaRlTEIkUa+kTS80TD2LMGzoPRaoIo3CqFzqk1jB2sGMCk
+AW3Ed6Qb8g8hZXKTzyMGHiULBxdmX5LqKVOybPLt+yhI79dkNqG3rEzFuIfxVSMp
+O95jMMYwOGNOg2WrVyjY0Qdgd8WP+3mXVasju8eQnSTvwxn75bsrdXCgS/dNiMZf
+Ap1hNrR/Pz0u5HMuSJGE/ICPbLpMUBXnX/0aV/H53iU96mPGskf8VgNK5ea98BIx
+kTutbfpdUrFVf9GTxaDb5vtyZCA8nxaVX9pVBzkb6SljGK+goqpiTn4MvWWyEu6u
+7oxdWZcCgYEAxMG0QX3cD4pE/1RU84J4xZtR8w4rddZA98ed2PxLrMl3E0SYKZhT
+Btmfq4757/w5QQXs8zKpImra3qFo//xqPtlA/idhfGOkfYYTGyXW/wnbax7ltuv5
+5HBHhrM4AKlfmW3XQTICiKBrpdp1+vdK8mh/HsRmju50EQ/EBg53tdsCgYEAy6Qo
+Abp1cX6u72RNnMKLadMHLTT9Z9ZdIgtlb0oawYNGy+iMmGKSYT/WTb2pBv+cQgWZ
+pIZ0cCM0ppUu9IuBg38anUeGIjx/qXgRfeUTOKap3ndroBO2iZW8SBqG5+CRKuHX
+CRaW4nVsp2PqfjtGVYpPq3UpLTkGloRMsDEPfJ8CgYEAxJFIxo05BwV6fSQBYu4c
+Pj84Ff04Ches/F6fIiKcb9kubz9+TT/y4ssUK89BgDKZ+I409ZrNQqdCVl0ni2bf
+tj5SIhex2xWpRgZmnB6ntlBgt5o8oQ0LsN9Ddh0ie/a6nYwRGv8dbOXn5eBlMf9R
+DvC630ATvgQYTNtVWT+m9TUCgYAbhv4tdaIIjIJjM+DV7upfjLosnzwd/Dksr/Ov
+PootXGyGhdHzi2eUI8f5TEj7/DOILS+esNouimPEnOfzAKSqPNVBK3jzSdBL/UII
+DZ4yizztKKFwyxlisc34xTU1RQL6k/q4bMhKpmrgmb4KAEM6ru6kE7tUNKAyWoHv
+U0JvqwKBgCyNzCz6AoqyabT+K5IYPgTvethTKjFKli0ijlizx4U4muIfk5umuYLw
+1CJXFJlFc+myk4b0+gvDgMm4Akv8QNBR07L6v/GpuSPNjHNX9DiZIl3phf9AJKmP
+sFw4h/BOZ+TKjre6POipP7e6/LTUTpE1cQiqsPM7IU8xUypSZ6D6
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c1.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c1.crt
index dd3475e13b3e..a4e2eecc3844 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c1.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c1.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAPJ+UHse1NzYUAWE4qVvMbBC3/WNMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW41LmMxMB4XDTE4MDQxOTEzMjIxM1oXDTQ1MDkwNDEz
-MjIxM1owEDEOMAwGA1UEAxMFbjUuYzEwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAJNmny/UimTv9fGqahCbXJ1Z+o+ShVYIsC0Z6g3tDtMGigLgGrVJEbtehE1E
-Is/YFCdy5RQP+/gPuqQcIFY+a4d4oXkOf559+Dkj+GlY8DDHIxSYFMT1b0cUdaY7
-KcSgbDHcE7ehFwYO8iU1iOC5PdiNFsur29CQH+fLMkGHeeuBAgMBAAGjfzB9MB0G
-A1UdDgQWBBR7PXi1qWwEhdtmH36E3IaLtwT33jAfBgNVHSMEGDAWgBR7PXi1qWwE
-hdtmH36E3IaLtwT33jAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNS5jbHVzdGVy
-MS5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAi6Um
-1XXvrakSEQLnFHYwVoTGEvQOX48J/W/sB5CYFHunn4S3I1h+uY1U3onR8vkHGrj8
-ClfYuDeEXv3UjmOulAb/YT5pcSjXroYRSQZgkFTlbMmoaIyNVOPw2jXRm5l27CjN
-lWKnJAu9OQEPPkjb9SSkqcazgPi5wmAKtnRmEaM=
+MIIDTTCCAjWgAwIBAgIUR3i95m/a7j/DtAaMXZdVWTq5fw0wDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjUuYzEwHhcNMjIxMTI2MjEyMjU4WhcNNDIxMTIxMjEy
+MjU4WjAQMQ4wDAYDVQQDEwVuNS5jMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBANQHW7eBKa6mNFCBJwtqazIJ2jf24DgSVMAZgTonthGfvd7mX1U31gEp
+CrgsPuMqklve/5rNjlTKsRT3Ohrt90LpzuxOezRM+ZpYbAejD1NUY3lYwx9zKKk4
+8RYz7KSl2wEarCJY93bGcJnyqP4MEkNlzltRuTImJavHQ2Ff5mYtibSgdY5Yxm4u
+9TDvTlYdhv+27pZa+jJtQXhQP9cMOTXTlZVESbAbnnPErnnCwj5XmVuQX0QMWU+b
+i3g32ruyHnONZQ4gJY2whccbskDyVnFjDHAh51xWAJjtZK/3pPuublaWhfND2AOc
+uHuiRQMXGoJhyhRVGC2nU40qYxe9BGsCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU7vbc
+7x6/7U1EAIy3rwQb+bduvNEwHwYDVR0jBBgwFoAU7vbc7x6/7U1EAIy3rwQb+bdu
+vNEwTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTUuY2x1c3RlcjEuZWxhc3RpY3Nl
+YXJjaIIcbm9kZTUuY2x1c3RlcjEuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQBdmOtcMoKaP/ilMHfhR8ZWaMg2Er0Dx/omM97ovYBm
+JeW1dKxayvN0DyDGa+tKEIgY5f9C/XCu04MgrqCFBDdZ9URK+5t5V26HeC5qRkA/
+rCuogwf8cIJsaBAYRJjufbbTX26bnehOq5iF08oVtmIIsZZC91FEnsnfjWazZ5JL
+BPEPwR47YSsmS1sjzi6zsXK6cimIC2+riQe9wZXRDFYsoCZmn8TUxbvCjTiL0Xyu
+D10ECj+EIoIl8QZ8ryo1Q9nd4FIpQaoTqMxkt6bvAnfzgwIn0up1IhwuUK6kARSF
+1KUFUTYZ3F34Qcl6UNz8MGRpV+8VsDzK/cn24D6Ahvqw
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c1.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c1.key
new file mode 100644
index 000000000000..0505508998e7
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c1.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA1Adbt4EprqY0UIEnC2prMgnaN/bgOBJUwBmBOie2EZ+93uZf
+VTfWASkKuCw+4yqSW97/ms2OVMqxFPc6Gu33QunO7E57NEz5mlhsB6MPU1RjeVjD
+H3MoqTjxFjPspKXbARqsIlj3dsZwmfKo/gwSQ2XOW1G5MiYlq8dDYV/mZi2JtKB1
+jljGbi71MO9OVh2G/7bullr6Mm1BeFA/1ww5NdOVlURJsBuec8SuecLCPleZW5Bf
+RAxZT5uLeDfau7Iec41lDiAljbCFxxuyQPJWcWMMcCHnXFYAmO1kr/ek+65uVpaF
+80PYA5y4e6JFAxcagmHKFFUYLadTjSpjF70EawIDAQABAoIBAAyZ71qFo6uMlzld
+o6trBLnxmfWPh53tgPn1yhdhutFrVAaGXI4dBmwFO2zcvFQV+KLbKwfRRa9GaKXd
+V6e4EuT4LmMe+vWNyZvhu3HswAFs14ijvN0GpBkCmFmAdoUGknCLXEWiUfrOKm0L
+3f1TTFaTG2OUBNS1LbH64sdLws65tGqrNLhaUS9/mE4cyzHuXeM/FfZSfMbtwjtS
+6hwQ7QxGISfkRAL7shukci5qvAp6Eq/76fwWdz9xB2D7U1VarItKSlBBtg13QCA8
+rNn6KCzMnPHVZulYF8RImxEUp9yM8eXMXQWA8e/afCBlDk76TDauQDT8AXjSGPMb
+YlSvUqECgYEA8T5q4FnNJmHbg2Q1J7tRljU4+K9WlE/XvdjhvDugp5nitNkELzSd
+mijb2i8cotNRG9pkfOETDPqfTc0pWEKba9bvWjEj/jZn+auj2/mg97LwK4ZIXgWR
+CgGKZE2UHH8kaNjfBwjjw2d1rCh9kkS7h79p5mCfo0W2lGRTHtp30fECgYEA4P93
+8JeoHVUp/BreNrm3YWdhSmX4CPznmDB6UO8m4NxXDW1aM+bXRAYf2ljqtbfI9ejY
+rZvtk7c59/+qIF2/F7MbNZ1LqTPWaMfg7QrZm848mzcesjycRaJWn0NRqOAgLJ1Q
+QNBZrRsbJzbelr4OEhKpnjcDkvQxkSNt8iTX4BsCgYBA4cKo2D4epo79RYQuTWjh
+9yp9Z9rnRJoHiz8a+wZ4TSD19q/zajlA7uBCTA1tlVvpG9GO7JHNGdKigB0wial3
+VXVGUpUVl9hbA9isHvU16UBC1Hi+IM0jXcq0s1C0YVWPhVRmdfOb6EVzqegr4KR3
+G/oFZdNJfERgoKLiV2uGYQKBgQCeK/otcPAGbvBtApZk8nowR3Tl0HlmGCTXoHzc
+mMNEIKIyFkXrZsS+VW/fF9MonGLyPRbWNV/3ED/4UAtRZZYJGyhtsH6YHJEGaD1/
+ZrlEBuAkXcN+mY1ihpVOsQyfI7B6rLzWU6I3R273SmOApWtO2T/UsegZTwHpfna0
+W6JHMQKBgQCdLxJLH8qLp6rtmyZrXghhzLRpQB1k58FecyamxmQGcF3lj4uF2/w3
+ixTmLNB6V2gIh+TMoVDUVG11Qz00cAbdQLouDcXR0kAd2xg0hZr1QGYnLrs2IuoL
+V50fUOi705ehD7p3eef6eKItO7SLCvtEXsdNkKQuP71ooeAIaXDy3w==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c2.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c2.crt
index 82d7e9c42a1b..248155d6b8f7 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c2.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c2.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUAjNAqngaZ7ks/XGNv6f5elfJU/QwDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjUuYzIwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0MTMy
-MjE0WjAQMQ4wDAYDVQQDEwVuNS5jMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAwIe2m8DEVRgwTgfukT218TRvgaJ88GO5v12NUIg/uzgq2ADAIS4lynhz5mzs
-dhcArzWFCz7MxP/fMxfwzRP7dNm2pjz57+DdUUSpPrBCDL/3+JwHWevd6LgMmgLO
-SohsJOZmWe5PUVMiP8/sluBb1+63v0e3iDNUXreVal0xxF8CAwEAAaN/MH0wHQYD
-VR0OBBYEFOOs6r9usaTHZ735J+HCRsZb8mDmMB8GA1UdIwQYMBaAFOOs6r9usaTH
-Z735J+HCRsZb8mDmMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU1LmNsdXN0ZXIy
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQA0lOcK
-U1O9pvw+GRqtodI1fNWObmMewUc482JWTCA+WVam4kwnXF0zsCYePdAnlu3eCBV7
-TAZeSIVRez6HGomSFz1OqtQOSOB/2mg4P+hNy+VIi+RoB7BPtQ6H7JBrTZ22wMcp
-7CzrUiOStlsuqx2K/euX109WmRLGy2Brj3MxFg==
+MIIDTjCCAjagAwIBAgIVAPAeByZhvQ2JsDNknuBtrbIlsVzbMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW41LmMyMB4XDTIyMTEyNjIxMjI1OFoXDTQyMTEyMTIx
+MjI1OFowEDEOMAwGA1UEAxMFbjUuYzIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQCQhsQ9/Xmz6+79dUjeEE+yPh01xxpGkrDI5QW72l3B60FC+i00HxHP
+vdjPlcndeBJBnOjz15m2vXCJ/19evk8f8XJ6rHJo7sFtkimaGFUA5bI9mHCUcvIb
+B43+nOJrYxRIkPYh8YwaWoqPFeKMtGztc2XdArcbOR1eSDXXZ679+GptH78Z7STa
+PrkvHFEOnYwo3qWP8FJhx6jO0GOVn0UPoZXKTt5IZsvMZHbZEybt12PUkZ3W6MVZ
+m2Yaqa09i1jX70iflxcCAbaMoR0CpObSqy83Xu36Erjp3qwb2VR9VkIJ0dBmrbSJ
++twXt5DDGFXJXTND4hNwFKts15g8jqWpAgMBAAGjgZ4wgZswHQYDVR0OBBYEFOPc
+MuJlv+W9jJOSYRRGTIDkHdExMB8GA1UdIwQYMBaAFOPcMuJlv+W9jJOSYRRGTIDk
+HdExME4GA1UdEQRHMEWCHG5vZGU1LmNsdXN0ZXIyLmVsYXN0aWNzZWFyY2igJQYD
+VQQDoB4MHG5vZGU1LmNsdXN0ZXIyLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAP+/MnGk8S/ViffzUikr73810VMfXOdUHnr4An+C4
+WbhMDOmjBeW6MpkEXdDkYnzq5sI8aYh0gpGfiipY+nc37RFm5YtAHUF+2+s1tWH+
+R+FVMhS5dpWFGzLfrNyXyYwswCO9kx19ZKSSXoz9RoDBP4KlBlRjaqPCJO05SH8+
+NjE9XCDhtP5h4GWBhhnRmyfeP2j/xCwl68BO/HUMLReW0oiURf6OA7Dlt/3Jp1c5
+fTCePCSjZe9EJB60DNw1JI96NJ0Z1sOaxO06IgaFoPX7vrMAttgZhdEuPn5qmNYo
+zF0fR5zt1SwNQjNfiJL45VwI+zAB8qI4sRYpeeSuaYH62Q==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c2.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c2.key
new file mode 100644
index 000000000000..f62314131820
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c2.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAkIbEPf15s+vu/XVI3hBPsj4dNccaRpKwyOUFu9pdwetBQvot
+NB8Rz73Yz5XJ3XgSQZzo89eZtr1wif9fXr5PH/FyeqxyaO7BbZIpmhhVAOWyPZhw
+lHLyGweN/pzia2MUSJD2IfGMGlqKjxXijLRs7XNl3QK3GzkdXkg112eu/fhqbR+/
+Ge0k2j65LxxRDp2MKN6lj/BSYceoztBjlZ9FD6GVyk7eSGbLzGR22RMm7ddj1JGd
+1ujFWZtmGqmtPYtY1+9In5cXAgG2jKEdAqTm0qsvN17t+hK46d6sG9lUfVZCCdHQ
+Zq20ifrcF7eQwxhVyV0zQ+ITcBSrbNeYPI6lqQIDAQABAoIBAC72U6svA85mklxy
+sXBTDYqGK3ivj0ayYNaOsMvHW6LpHmY0o96lX+TItDxT2GnqG8iNQ5cEjlK2gOye
+ei9vmIdxLflVssGNH3ZKwdDsKibl/IKOV7qtFSWaUVYURXCOm+cuj4QdGKDxncd7
+3YdKE4Ee99EWkkg6yWO7BB/P9alZg6Y+dUg6WuYVoxleqooTS49oGot7C3MHPL2x
+QrrU9z3pE5J7jM1Ad8oV7O7HWN8lRQn2iErRfmewfDUWKKyzNNTSu4xLaXsA1l1M
+Vhfx4qeHIZTHFQzjMwp1hpjDR6gyzZrT3hTPsXBuSS8N1KfuoKfQUGRZ8DH4/CIf
+DYIOmt8CgYEAx+kqeN+7nkiT7pP1SEzWXPd9cRzCHtuIzKx1GqMHsTeroOhXcIii
+Axf7Vt04XyhLwwMi0DsyYQ+Q+M4k62EeDILmol608PmIsenj3QIWCx59MVn483mr
+8YmM0eq6fyPjnXg/G0zmnuoFQNJbmeKK1RsYC0Y0LqayPpHuZ3y+8sMCgYEAuRON
+LOb0yVL5vT6Je+BO5bv0GA33RUwKxjheGtnLuBbgY/ocIFoq0gnkQ91v83s0MeaN
+rUXPiaStQcfmp99ly+bseqC0r8fIPX0of9/DnlwuH2Ghgpnuh8meo0fxxDUPTRBb
+Mzhf6JLOLNtnNTFI1fkDV9DV9c9WwFFfbK+tZyMCgYBIgPdneXWZdac5ey4Vt9Bz
+TUuQr6Zz/JlD6jRaodBTZOZ5aGr2JIfl6Ve97GPal0nLA6MntKCT9UpWAqX/PGzi
+isNNFl43m7kJFdQPvwbxp3AuavxQictjwchyBdj37bNK1GbG5wApmKSpGuIgELrb
+iYUR6bdZSBoTeWfm3qZ+cwKBgQCrZZzniZgG3k4kYUcHrsrCXRcIM97ckTilK8yS
+x0Kv4Je8XwMvPvHLBJgd9XPZJC9JSz/HyRATY6uEcPF+ZBgtBlXYlv3ekIEz2LXp
+/rt1695W956QiafqnuDITDdtQspBzzT2sW24wEQqLKJfE55lcrAS4ndTpsmrX3Cl
+AejvywKBgHN4OFMEYja4iEmq0OKCHSeDapDA2hrTWxmV0J6kI+xtBwTHhvTmW8Y9
+xuR/vTzgrBcm8fiVoycVtCzY8/1p+6CcnHPy8koLQQtFLCQxcIPVE000G0Hm2x8H
+HtkeUP2NNQXjbbMaamfE45of1NmIB7KA5RBYs5grFRkbaSekfaiv
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c3.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c3.crt
index 40166d0c2717..b834c8352251 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c3.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c3.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAIyTRMpfs9Nfo0QL/yDgL0L8eeN9MA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW41LmMzMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjUuYzMwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAJOBFDvbKzWo6MzAQUVb9t8ChihkEGokN8P/MdUkg07CEHIiTSLMj0FvfH/J
-V7E4YtiELUY0grTcWRln7Q0bt7fuGtC97TxL627dx/jiuTHCG0tko4bF/KyKiRFG
-O1CbXQ5tQzvDom59FNQzk07HT5aFt85Oxhqw6Om8R6YraLclAgMBAAGjfzB9MB0G
-A1UdDgQWBBTlHIVNp/hPa3SPVI3U9cusa0lKKTAfBgNVHSMEGDAWgBTlHIVNp/hP
-a3SPVI3U9cusa0lKKTAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNS5jbHVzdGVy
-My5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAcAdT
-R+l3uo8EKz1p9PxiCB26VI+1ZWVmyD4bKAjAgzYCz2HvNAbkkVMW5tGfmzLDuZm1
-pxpIrxi0VFeFhZyzlgF/umcPjr88MXWox0QNYLMiYFIxQQMJljglDxwyO9keVLoD
-VZdIp8pZoaGs6tKQDyzmBh//6deecyNDNubfJb0=
+MIIDTTCCAjWgAwIBAgIUd9cnNGI4x0z4dz8DLl0C+a8habkwDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjUuYzMwHhcNMjIxMTI2MjEyMjU4WhcNNDIxMTIxMjEy
+MjU4WjAQMQ4wDAYDVQQDEwVuNS5jMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBALN1NeNFFTeJmH2NkEUG5oK1OutyqBeCezfgdPfLeaa8igBksqDrKwSF
+a3eUc8DuMK2xfCQGf6/20YJ57i1pc/A0Bd0vnhSquWQtyqTzPiI9JDLxjSlVzvUs
+Es0fBYc54LjH0yWcmX9yIguFj1EBhB3m7wCp0Z0qBKsXpA9hHV9U1gk90d+mZ4r9
+w8upEdVtOjulSrdfdLD0mGSYhPJ05lJpUpMpbNuCf3WncWAzfuDUZN9VBEfu9Cas
+jhswDkezjqZG4VD9JvyPv1XW3UZ38SxonleRKGrq6gSyJq+FRFzsYFE9DFqJ+Im8
+wXHA7wHyLRuMFUUCsVs54SU32+dDWmUCAwEAAaOBnjCBmzAdBgNVHQ4EFgQUlE1q
+MDtKj+TFW/tEJoKc4ucVTMUwHwYDVR0jBBgwFoAUlE1qMDtKj+TFW/tEJoKc4ucV
+TMUwTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTUuY2x1c3RlcjMuZWxhc3RpY3Nl
+YXJjaIIcbm9kZTUuY2x1c3RlcjMuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQCoc4hSkEuhKvFNNnmO/oL2Dfh/K+n8iJ40kFYdjepd
+/CKlfQR52q+fQY8PESeu541izzhm5kj6fZJZ8J0avcugaLMMiNLX877YsqaOn9D2
+ZEI3KbS4BbOqBakNwTrz9W2zhM+lvhY8fu9l5gAbIiMIkfvysn2qdoK8e37oC5EL
+7T2NrvrGcOwRFrOSBCx2uFK3T3ubP/RriP5lw+ODMghUi4s+3gd1/nP60iXC7vz6
+vsSW+kTz51gdFwqt7POFvDURamOmkJ/s3yL3o+8d3TCedQP6AGDNGu9NXtFCyl/4
+22nVZm/e+9JcU3m/Fo561tycBp7jJJXc8a8olb5rVN4J
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c3.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c3.key
new file mode 100644
index 000000000000..60dc9c8c6246
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c3.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAs3U140UVN4mYfY2QRQbmgrU663KoF4J7N+B098t5pryKAGSy
+oOsrBIVrd5RzwO4wrbF8JAZ/r/bRgnnuLWlz8DQF3S+eFKq5ZC3KpPM+Ij0kMvGN
+KVXO9SwSzR8FhznguMfTJZyZf3IiC4WPUQGEHebvAKnRnSoEqxekD2EdX1TWCT3R
+36Zniv3Dy6kR1W06O6VKt190sPSYZJiE8nTmUmlSkyls24J/dadxYDN+4NRk31UE
+R+70JqyOGzAOR7OOpkbhUP0m/I+/VdbdRnfxLGieV5EoaurqBLImr4VEXOxgUT0M
+Won4ibzBccDvAfItG4wVRQKxWznhJTfb50NaZQIDAQABAoIBAD8lrmmjhHy/cSY0
+6yAu9mvmA66X8Ogc5Ojbla0gL1VpoJHU3Tpcpg14OUGhvj2UUxK7knhGxSWRD1X/
+O71FDyPl2m5jdBfHn2bN6jGy2hlINHWso8R4ylNhR9hf4x0bU6URw43HlgfPjIVL
+YQvbIbc37L/qAbVw9KWTswoqXSCO7MDJwtf4cCaiD6GSg61ReinB+y+Yxo1SYfTp
+M/HcfhJwhOdyXmi55PRNp9ZWo03B1IHmlDjzd7OIvV4eB7+6BAhKzGiuVfqQVyYy
+dKJR/RQduU+oBNW3lhDg13oGn6sp78FCBvffWtJVs1PHKhqIaPb5CS2pCAHBQAdF
+6ut+Wf0CgYEAvn4hMdrwkKn+cIAnFn9DrOC8KTjba97HDzsX8kaZPydRcWiMKVuE
+lvYU3xNefX531PC/5rlbu74kosSqLso/d21bHOATDoVdZOfeHIhNYsmLFGgdoggF
+scDcnyj6dtlc1pKufdfWLjFP3TN2BU9chtqv+fFXnYiA3XVssqG+3gsCgYEA8Suj
+N7SRsXTnlqjrxX+lH4IYd1IXCxLMJ6AjA+8dNIHGPTGuv9R9PXlnm+y8kKNeFIOc
+wz6VsyE5b7Uy6rGTQ9V1O/ijVJyl3Lz4grOuE4qb8DRfqOnP61z9bvuUL7+7go9T
+Fg4G/BH5AD7uV8/L8zCp1vs8RWFeHFu/ouDcn08CgYEAp8zvbL1R3/z+9xbMV9Lv
+SGAHH8DH+HObod0KMQ7je3AlG+NeZCgL2fcXV5H2UFGROXCBerqZBf5uwzq+Ns1N
+ruQTuJHh946fQH3HeBbRJE4pR2aXg93RNk2PilxMlVjPoOxA7PWt467ojRABzJem
+B9yn22IvLcPK0EnZ4Fj+ixUCgYEAkSOy8aZg/AwgB+KXKd+87y5rLPwhVA+2/633
+BLoVuHDVbiSFR3tI+AMlYIhjnyRCTVQ7tqDmlMxM9bZ18xp1RyCikjn7icNQanzr
+jjU2cRo86J2MaGp2L+5hszYCo43e+h7pYN+GQpNKaGR3Ki+rTSsfO3stdY+4hJaO
+ySZMgC8CgYBIzDSvGKpVpRz7oSNNz2tJE9Ujd5qOSofOgCu2nmZFtWm+SYpFjv03
+DMUKM2bW0BDS1AW5u8p7rGDYvY8bG+sCirilpoZ4GVOwBYNqLcYVfpJNIdtsZ4Lz
+qTrTuoQkkhh426F5/zKLJ/vqPPR/J953iXLCIF37jwSF91DAKGiJaw==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c4.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c4.crt
index 5f9167b441ce..6ad4a1da686c 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c4.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c4.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUVxoYfxLb1GFz1joNjsIRlzhQ58EwDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjUuYzQwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0MTMy
-MjE0WjAQMQ4wDAYDVQQDEwVuNS5jNDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAiGKK1GCMeyJzrzzi4pNaCWqd2qiFPISAANTh+laKkKQDLbrLo6BNfv9Ef2Zn
-pFO2JXWbDzNd2UDA5wKjlNPacFvoWbQM+pydH5KOmQ0tG5AkrWgnMSVyZ4UY7tUG
-AnkMe7ZwtWWETVTOzW478mqV3TAM9FjIPmUFuIyR40hF7EkCAwEAAaN/MH0wHQYD
-VR0OBBYEFI+J3qFQpd1bJy1q/oww35Dt/lB+MB8GA1UdIwQYMBaAFI+J3qFQpd1b
-Jy1q/oww35Dt/lB+MDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU1LmNsdXN0ZXI0
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQAqxVEf
-8W8V3ZHi5DkIxGqZoRN2ukHG1UVOvxItbhlbvdYN0mrFeGl8xzWtP4wYHgiGBs6D
-3lzwD0cj1hbUi5Wg5kt8KuGqpAiELHKLbdp7AFRrAPTkLC1XNWEMYjZcbR4XHVP8
-rMVVYK0QV8ycLsotazvpHbmT+DfXRDyqFkxAwg==
+MIIDTjCCAjagAwIBAgIVANt0GDxh1IVmxmAE9bhsPGp8A83MMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW41LmM0MB4XDTIyMTEyNjIxMjI1OFoXDTQyMTEyMTIx
+MjI1OFowEDEOMAwGA1UEAxMFbjUuYzQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQC4yOwUlBFVyCLf8RCgYi308AO2kl1ao/WtIiMMtDVTknK5vaWUZcrI
+dFPSmiuJV4xWUOnMed62x+RD7l2gWcd+QCK8L3PymqIlGeeo0ogDeQWJhdc3LTRx
+k/nIGkbdCykFGe3DumO7JgkZYmAXHjU42sqRor6KjHRB3Dmy2j/T4ze5U8O2rWPz
+r8mH2Uj1k4XGnkSFn6dejihk6TZS4430yuNbD4Ca73GyefvuSbQ5UrWVTgMs3uBf
+Eb1RXTFN+k41L5Jj7a2HwhDqefVXEVq/5G0LpHVQ9PQVBit3IulubS240228pF9j
+sYxMjZt26mzAZFdqxzYu9aZ3oad0zut3AgMBAAGjgZ4wgZswHQYDVR0OBBYEFOWS
+Q9VXQJ+HQlOa/1D9jWspp2HYMB8GA1UdIwQYMBaAFOWSQ9VXQJ+HQlOa/1D9jWsp
+p2HYME4GA1UdEQRHMEWCHG5vZGU1LmNsdXN0ZXI0LmVsYXN0aWNzZWFyY2igJQYD
+VQQDoB4MHG5vZGU1LmNsdXN0ZXI0LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEATav2pGjvOkAkZWvjKDiYySE+s4BLIFV2XI3sunO7
+eznvU77WPtMJMXGn8el4XRDUHYsfdRTsrzfpXQFPBIJXzUsFqP3rPF1djIWg3Z1n
+Sm+plHHRVUTO1h+8pCc2pZ1LwMcnpHFlV4RLw+fB+JeiVycQ8fz7Ane/HleAH2Gk
+DTGzcOh+CQJIMxyHMRSOuOKjLm7ejPToRgIY2GXw9ZsWipJZN5qpHuit3f/kdRWb
+P4M9d7HBbPQUQKfzC9JvEmHgPEIwLsEVOSNwwoQylGMwrOQvhA17jVTcnR8CfRm/
+q3Vrn9FKd9fmcq1nuk5QQ+BYinLfoJLQftwWtpe0noTecg==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c4.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c4.key
new file mode 100644
index 000000000000..c39099defddd
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c4.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEoQIBAAKCAQEAuMjsFJQRVcgi3/EQoGIt9PADtpJdWqP1rSIjDLQ1U5Jyub2l
+lGXKyHRT0poriVeMVlDpzHnetsfkQ+5doFnHfkAivC9z8pqiJRnnqNKIA3kFiYXX
+Ny00cZP5yBpG3QspBRntw7pjuyYJGWJgFx41ONrKkaK+iox0Qdw5sto/0+M3uVPD
+tq1j86/Jh9lI9ZOFxp5EhZ+nXo4oZOk2UuON9MrjWw+Amu9xsnn77km0OVK1lU4D
+LN7gXxG9UV0xTfpONS+SY+2th8IQ6nn1VxFav+RtC6R1UPT0FQYrdyLpbm0tuNNt
+vKRfY7GMTI2bdupswGRXasc2LvWmd6GndM7rdwIDAQABAoH/c5vsGEk15KAn3El2
+SDck3u9LY26EpFubLuGstsYIlxYzsQHLa+JD0iZK9fh+70QBs/9A8i1NVLi7Qm+1
+JB2yMnWYmyRqkTPYk7MhZ6j7GoZycG13DjkPa//+HAj/TPcuiC0wv28IoTkAIww+
+vOKpab4j/qSo5wX5C+Y83dQDxbcmuvUQxlytZLSpVKPIpyvrfW568utssgFc1XbW
+O7wY2BtSfg82owZdQAm/a1V1Oo6GIsoMjf9rDN9UQgslUyaYFgYZ41PhUhCRt291
+1oGLX0gID5lqrlh6t9CsQBiTa+ds/n314VfwNRGGdXmLfGvexfHhHu2cweEstUyp
+V0Y9AoGBAPD3scFP9WG9Q+gBbqu6776eKhJRJrvCtX7HtOFj7lU4qGnoOvmTQB29
+T3oaDOoWJYGvXuRy6e8Frj1itBvLzJk6VDdiDvW0c2PB2wwoTRKBtgsrFUCt3MnR
+kgdgY97fBx+vOjTk6TityagEgEFdyP3fbHy3Y2w1mg+VW/aGq24rAoGBAMRP+j6A
+JD01ivX6nQuUN8YZxgZxGkm6D3IyIFidD92A79xjCTW76nbyqIM6qrz7/nJ1SJj7
+ae5Xl6NriL0C5z5THA4paM/UdgME6tDFVJY5Ib/l8c4pd88OgRogtKnVOYurbZtY
+LXSub6Uaf+Su5TFMWiykx5QiINN69GJjL53lAoGAN72UaNtOL+XQJ4LtTtxXSQjd
+Qo9zHkvUIBeiR+EN6LjKfHUz7zeYxUhdcilEg4HL+0vN75qFcr0lqR2D/EoQcgGE
+KauNOlkbuB/Hw/Auo/FNYxf8OpH9WNIZrYsmjw+YXkPyevnY2TuaQjuIT5aaP9/H
+ALKLVSSxFpm+RDCyxzECgYBeg+cGqmlIMvwpe7Sau4yyyvvlZBT1BCiloW6jW1Ml
+IQvavq/TeBrlk+T/8YH8pRKXhtIiHYGd5/wxCPf3kGtHPXpXcyZsUW1T/M0VILyd
+6vnNy2bFDIALZ39O1gpFbkxba3jE/wo7kDReeEi1EGN2fb04m0kfj+LgDuWUbQ6i
+5QKBgQDmSL8X2kkeUsOuvT8T3TOmCxa8FIWHOrpYD2vv2tdHNaJq4Eqs78zjyZex
+uSVrPJxFDpeDPDtKU3lI1Rpx64fttrSYTXc+/cTa7ikf+mK2valG8cGK/q13mQsQ
+QY6iK4VTgEn2TMv22ip8tJKQntv+RyFqEOW/uGJet8DefOqheQ==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c5.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c5.crt
index 0c6f69a17e88..ad7f47cb838a 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c5.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c5.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUNJzE2lRQpXQl/GqdwDQi0JvuPqUwDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjUuYzUwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0MTMy
-MjE0WjAQMQ4wDAYDVQQDEwVuNS5jNTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAu23ACgbulVvEbb1O1WLxhi9/KvDXyqPekjC9Mfe4PQ2yqUaoR2XxKimbChuE
-Rzbv1Ggt/msDngoRxRqgSM8mjR4I98YhDFY61IIeL8OmnbyjfbAXOPS5jy+qA6WF
-jlqSBFDIld2AcFJWdrFhsdq0+A7Q/2x4YuSk48SFUr+aEacCAwEAAaN/MH0wHQYD
-VR0OBBYEFB3l604mCVN5uOPCBWbsOdmi9aEHMB8GA1UdIwQYMBaAFB3l604mCVN5
-uOPCBWbsOdmi9aEHMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU1LmNsdXN0ZXI1
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQAt5yP6
-J/JgBzJpYmX7YQp5gkaCZZDK07zGl0/53HDb/yiSxeiZcDuqwqlpmIeGdCbf6J8+
-2aEdC47+82TQn9ugEudvLUfTY7RDVsKW6KAeD8UJhQH3cjARxlJlwTELkS5QqFyC
-mM35puJoLctM3pxK0iqHKSh8wSkckARDYP/2Hw==
+MIIDTjCCAjagAwIBAgIVALLrYKwnxwvD+geAEyldHa7i61nfMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW41LmM1MB4XDTIyMTEyNjIxMjI1OFoXDTQyMTEyMTIx
+MjI1OFowEDEOMAwGA1UEAxMFbjUuYzUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQCf1BHcvQiwsNS/HPFvAp7fyzvrE11S4HvL2JKXbFjF6HLDYt0ux0vZ
+J5olFKpuBgZ+fT9TzZhj+a0nB2KYrFcZdE2FbTGaDB9SgLOuI7T3OvxDzuSkNlDB
+hdv6p/N4+JXUJUzmxUeoEivRJSm01c0Ncb46mqOliIUVmFnb7/+nSG7k6fj1BNs2
+4oEuoxT/5VxGkzNvXEIOvFg9EbiMWg1KidntqS+fVBDH2yB/ITpkBgPwketmdpqX
+BlI50iV5bq0NxJGEO8AEw+GCeXPlhLGq3QN3QpUL/7jzgTc6VnlU9ZJoS+R3D8Do
+7jmO4y5nB7SNOCs4hUjMIh1SwCQi/rKHAgMBAAGjgZ4wgZswHQYDVR0OBBYEFJU8
+XRWMR90LxftEfCy0cUNEvGblMB8GA1UdIwQYMBaAFJU8XRWMR90LxftEfCy0cUNE
+vGblME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGU1LmNsdXN0ZXI1LmVsYXN0aWNz
+ZWFyY2iCHG5vZGU1LmNsdXN0ZXI1LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAeziF15irMtCGrFQmHtDCjlwhTEKcR9usL0Pp6piQ
+x0zlrOfoFB60MJM+/nZ0Us6r5iPid1SRakMcqzqEq+7CE3EIUjUp78fA8kQDFB8V
+TibegrUQU01hsdGDae06L9iytR8O9ViXOkemEQI2X/VT+3eZ0fTSQaPHeomKGNxY
+kFFpxWYH/OHKwLe2/FHkPbY1KkdwFjIMkc1y5mQYvt0kihqboacBPpzU9uvOC7r4
+dmykTRdWPp0320uYAdG//UaIGkEVJI/IZq/QhyWa0ByxlQ5kgngHOituQKOQbIwn
+f+GEPajEMAJUh6FwSQ2Qey3rMGDhl7oGvnrf8KG7IezyuQ==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c5.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c5.key
new file mode 100644
index 000000000000..81a2e58bdcf0
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c5.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAn9QR3L0IsLDUvxzxbwKe38s76xNdUuB7y9iSl2xYxehyw2Ld
+LsdL2SeaJRSqbgYGfn0/U82YY/mtJwdimKxXGXRNhW0xmgwfUoCzriO09zr8Q87k
+pDZQwYXb+qfzePiV1CVM5sVHqBIr0SUptNXNDXG+OpqjpYiFFZhZ2+//p0hu5On4
+9QTbNuKBLqMU/+VcRpMzb1xCDrxYPRG4jFoNSonZ7akvn1QQx9sgfyE6ZAYD8JHr
+ZnaalwZSOdIleW6tDcSRhDvABMPhgnlz5YSxqt0Dd0KVC/+484E3OlZ5VPWSaEvk
+dw/A6O45juMuZwe0jTgrOIVIzCIdUsAkIv6yhwIDAQABAoIBABrkEO9NRNGJpvnS
+slJlo/puVNH765c+msWU1jEw8wy1S93Q2/r7YCkUrywu+m/RhrUweH+ul0W24+mt
+peqfEHaBaWuaEB7ljD2S1uhlLJTuMTRtcm2oj9THo1fceyvu0zKi5fnr1l8SU5n4
+QTiiQkJRUVrF2uPGgPG9tNy62A46bweBuI7oVuEd5lBYIexUCJEdO3G1xteFVA15
+Tvf9lPKygxLUaIguL1Kg4oxO9AFtkU2NMpnnW0s6v1T3Bjtal7uFpySvkoiRvJoO
+eXPU7xDHwo8t1xtAYqbhjx8XfJhI9SBHq3hTtfoJgRf/4dK+F46LbOoWWuWvq2xI
+1INf3BUCgYEA0QyKwjwK7avyPOnVB2VvjYql25nsnFeh3uYqE2Kjh0BJ7P088uuX
+Hc/QO9xPCVN0RDtScCAUI7lsJAoNgzPvY748WcardJZ2XH4XRvXWAbWnrYCLATmg
+qGvWR/XncEeLDC9O3vT3o/8UhKD4iphOnt4l2wxaUaXec+0Y80COwDUCgYEAw7mL
+uv9NZbDPbZnBozabEqyzmgDA56fYlUAq0AyNMADzypf0o5XUbszO3KsNJ7AaEpDj
+9RdTPbojPUXH7RQtXSiKynxpjWXqT20LBPwSOFn/GubQ4Ioe8ShqeSAJ9OrJe6pB
+mR7idqb5kZrb6bXcGayKk76l4CEJ+h95dWR9N0sCgYEAufOmGlC+h9vDhluZmxKn
+wAfssQKibg+ldVjXZJcBT3XoTF6WsJ7PtTz9xcGarHkHmwoJsqNWRtDRWN14V5UD
+BH5bwJ/wM60DbBLdMcnLq4mitbr56wUP28DQwwIB9OYu0Y/jYIAXKFHmNnPZoa8J
+gFeq5MaAMNCSqBvnbbmnQWECgYB5yd8UwQAcIuUvwdkoAjaEbEplrcpbW6n9VSxO
+wUCpei45Bk0H7Zd+Spr93MIWzBGtM9NlEtqNMmHzIpTBBSklKrESRlA32A7w1V1i
+AKglwHpo9u9KiSev0AJQ1Irdh6tP9CVVbk5knon0/HOS/cKzhxXD0YHN8UEQDr25
+Oopf+QKBgQC8PK1uPIIFdUAcvpv7KCoA+YTHFCLrDLeZ3H2I07eVKmwzhEivvYGs
+dNSQC6KxhFSTeVB2D02AiiOcdAWwY/jZSlUOYy34m9dd3rzuQ+A33dTcEIb7BMCg
+ezoMD4v+gLfbQ3gve+zm9GR8xvkgGrKZAjFr9/42XTGDJ9XSCNVHSQ==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c6.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c6.crt
index 5cf212017fa8..820afacfb918 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c6.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c6.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUDJQ6adYh3ugguVhpwXAztpir0/MwDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjUuYzYwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0MTMy
-MjE0WjAQMQ4wDAYDVQQDEwVuNS5jNjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAxUpbKRNQ/WxzJj7oYFBzeP3/JwtVTaTqlJexfKpEXAGlTXrUYRVyZP804p+2
-Dc5sAFEFXwUcmYgVlLCj+mc2OGg9Uv6EyaT1Bz0r4oEbIB7ePBiefx41i2Rhd6zy
-3Jg164hGnrHzmOdqhz0hjKxUAgBFzEtfx0URtIpscMefimkCAwEAAaN/MH0wHQYD
-VR0OBBYEFEyrP19EhCIvYsJFVyZPZ/2Uis/gMB8GA1UdIwQYMBaAFEyrP19EhCIv
-YsJFVyZPZ/2Uis/gMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU1LmNsdXN0ZXI2
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQB5b061
-fk5o+h+m/mOR7pi9TKwKnseiK5P/N4Gv9WuAyN+edee74+/MfWmLtGtTyeZQ3XXb
-Twxfr+UvHzHDN+0QHuQIj77DBDTxfULKF6AXKoVX4NGABDdhKCkxRXjdqmq19G+q
-upeZXyel+qvteAH16mzrVMGu59gG2IGz5+RMsg==
+MIIDTjCCAjagAwIBAgIVAPIQAY+4wdoyfeEuTqz3wwylFojmMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW41LmM2MB4XDTIyMTEyNjIxMjI1OFoXDTQyMTEyMTIx
+MjI1OFowEDEOMAwGA1UEAxMFbjUuYzYwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQDTnSgWiqKP8NtGJt9Ip0cw7FxNRHbysm/qnlDTpvAMf3GSkvdG4IO2
+B9I/9lc9Ch/Ai+lSzZh7HVElJsReqsRB3BeXzkwW8vVvgtPq3Kvb0bg/Q3gCKWx3
+TeMnZy1tX5R9IcJP/Cp8eCby+AI/3Xhg8nv/7A4iP02QoPxiGw6VcRjYl5HHFlzt
+ECQ1ppRta4/xIiAuG3883j02DIbpKAfWfC5zPtugkJxM2r0jKPDvuFNg28D4GPPP
+Wykc0+AH/38tWotQ0aOiCY1m1FNO7KxhbP7vXfpY2knbYphvPo5SpalQECq37oAx
+yRGGP756k16M8wlyzOhL3moOAFLx3zCXAgMBAAGjgZ4wgZswHQYDVR0OBBYEFC5Y
+lfZOKmazs3GfLZ5slVeXX/whMB8GA1UdIwQYMBaAFC5YlfZOKmazs3GfLZ5slVeX
+X/whME4GA1UdEQRHMEWCHG5vZGU1LmNsdXN0ZXI2LmVsYXN0aWNzZWFyY2igJQYD
+VQQDoB4MHG5vZGU1LmNsdXN0ZXI2LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAA2wsusK//Vz00qcuekznVPnJElKunw1a6a8ca3pa
+mnjx39D5FKDoI0hwfpPkiPKADc0uY0CZgORsPUlhNF7K/fMirwEfQfBGlEveqCaX
+RLlB9QVqSXAwMYQuAPnsqsGNqkI41Uv0TIIhF+xqzt8BQ0YnJHOHTnHEFbQVXfhH
+g6Itzy48SlL5tkDTh0XeXieSom6wCh6Ql9Xk8NSU1mM2cOShJQF99XHkii8KmXGr
+Ac3yck8FtlCZ090+X/xuM2TzqFMdFeUvGNbneRagY/CiqoPZHifEgXkQT6H9P98P
+oxu8gvqtDWCQlxSIxeDZpE2wniJu+mpSv90XSJyfg7ANqA==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c6.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c6.key
new file mode 100644
index 000000000000..6ba4955d917b
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c6.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA050oFoqij/DbRibfSKdHMOxcTUR28rJv6p5Q06bwDH9xkpL3
+RuCDtgfSP/ZXPQofwIvpUs2Yex1RJSbEXqrEQdwXl85MFvL1b4LT6tyr29G4P0N4
+Ailsd03jJ2ctbV+UfSHCT/wqfHgm8vgCP914YPJ7/+wOIj9NkKD8YhsOlXEY2JeR
+xxZc7RAkNaaUbWuP8SIgLht/PN49NgyG6SgH1nwucz7boJCcTNq9Iyjw77hTYNvA
++Bjzz1spHNPgB/9/LVqLUNGjogmNZtRTTuysYWz+7136WNpJ22KYbz6OUqWpUBAq
+t+6AMckRhj++epNejPMJcszoS95qDgBS8d8wlwIDAQABAoIBABnyCjjpT+aHc3sg
+clP9c98YZnYaz4gQ6g6W0UuCC6tHR5eJvsPJQfI5oe+xtnSHdJPZE7HRVMFo5iAV
+/BGsrQTI2s5ZaZQ5acT9rMSpFcaNP3fnlSPpYaEnGD4mimKp9hIkPNItwCUkCN0q
+x1Z+mFnaoUAkTdSpH6DR/DUoeBtJBNaL1FPyJxBZ5LsMlAscE4mYy7vHafAPkjBX
+gFF5ncODUQzHx5CU7bpA/BbQRZu6RlZYxJok6l7xwacOArVH5275vTbfA6nGdYEL
+FX8UKXTKabNoO6on1qW0Q3B2h6QQcHdaLW/RnW7rqBDxvOsPJV3rYEKuVWX51jM+
+AKEqifECgYEA8B7/5P+rxcINlkNbd3dWf+HuMW3qrq0p8KhvfUoMpHPqkD/g02n/
+/l5gw/bant8GUETSEtH5L4FdPuffSqIhE6apVId/9WpYqzwz4VxgPhjcqz4xL68/
+wy0SwUuE0WtgnSDSkYlBJhewJxovk7xs5yk5wtcGWeRAkf2LfB2i8RsCgYEA4ZuP
+TMiRuVj7+m3doYxejVcsSVcaetAY0VhwhQ5G2OjquJkpxZgv8iwcsxw8w2Y7DCi0
+LqPmEFi8KNZt04vN6lcvPkknCSlJYo3WzRTBTNfka3cAg/x5jtn4+g4wjGp2tb8N
+3oeNY5ml/+sqraNHvDFiNZKUyh7U9Hg57MDkMjUCgYBCOoqiAtLB7NAKjfGAde3p
+v8sO3x5Hp4NV2V9G01HBWidsOfIs3aO56RYPdtq9zw3SveijQD7Wl3u8if/Qef9d
+iC5UHjtw2TAzqps0oWH6Pqk+ohy25kNT9e6iWc38ZP102US613ycoRP49QQCX4o6
+B+jikKQzzvzwR1pdlV8H2QKBgQCAD/5S1pMpg+JcsBqbhgNx9aa05tWoGxHLW4lb
+bdQ3clcye/ajLBUjQXj1DrVLMW+ZIk3NH8oLQ2NjRvzGlVhIij7N+1ZGNAJtCBiD
+ByIGjZIikkVfpmQV+5MB1iYmvpQt7Iph484VWHVLBOGP+NxSbjHSOSduZ/v33Xr7
+8i5WdQKBgHqQM48U7idLfyRliXo/xlIaakpsSHcouVHUKQHzXwHj//WkN9+bXsDG
+fqFhOVNWI0TtCpnLoOjVSXOdr/NFHbDvZ1bzfngScn9q0Tv7l/UmB4r7ZhiqHjUX
+H31Ig+97+yY3hlUjtqP1DUvUry7Jx0zydidfGXDWtJ6OUog70jqg
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c7.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c7.crt
index e928f5fdb5de..e666e854dc14 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c7.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c7.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVALd4gLmbZw/Ym+O+RgzBT3rore8kMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW41LmM3MB4XDTE4MDQxOTEzMjIxNVoXDTQ1MDkwNDEz
-MjIxNVowEDEOMAwGA1UEAxMFbjUuYzcwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAIOhA7sGBQNF8E/f+5jB7Bw3oya3w6vHWbGJtPeAkA3HXQWtw0cR7h75TxyP
-tsjYa3Pi7CF2tsV7raZhX7tVJtwVq5753TpCtBVOVXcjXhXR/Hlo0+EFVa+l2iOf
-OqiGn03CwILdpNNab6HGIQGHe3xsk2GLG1uMI8clz9uo7KRvAgMBAAGjfzB9MB0G
-A1UdDgQWBBQkRUJkntKPJPIaIRVydRbQeAPUCDAfBgNVHSMEGDAWgBQkRUJkntKP
-JPIaIRVydRbQeAPUCDAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNS5jbHVzdGVy
-Ny5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAQJlL
-BeTrtZr94sm0hYdpl54dPJciBtyWi8XMw8SmngyuDaQaLe9kpWAbrsDX46m4vXAd
-FkWyRZexDS4R1mBuMHjcj6hSTAFOiy6ypdQvfCuzvLxLEq9NXq/4y+2pkp8G7c43
-Kcnk1o3P/BqFoHcwKV2oozDgIYawQxvE1fIXzHc=
+MIIDTTCCAjWgAwIBAgIUb9QulilVWkQvUt5SwBauRXnkkHowDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjUuYzcwHhcNMjIxMTI2MjEyMjU4WhcNNDIxMTIxMjEy
+MjU4WjAQMQ4wDAYDVQQDEwVuNS5jNzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAKHj9oMTSE8PaKWuudm4HBlJ9AyedCQbIO3KHO/G9CoMQWMa1Oih7Sfp
+cXfBwkyR7gJ82ogyS5sB5rUTl7WmgJ2HxdPv0O2vs6dwCL4OwpKxciay0283OE0R
+w9xkLjMKCW6eZ7btU7qD0bRJKq6fG3d4DUc4EL53nyGZnNgTNPRwzwV1YD2vomum
+SYxmLD9I3wBWphnTSQyDQGqt1tMqDlPtVnNXqyuwmGh8tPsX/uHCwOll/1aGVeSy
+NrECLuc22FzuMbXf9FP8pQ7sYgqUXK5jkyZA6uduLOg+yi4zsFA+hT0G/+T8cam3
+SnUanLFQ80Esvho/RoxLz/31wJnv3tkCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU7+Hw
+R8ErGM/gjrllcPIGK+6xKRwwHwYDVR0jBBgwFoAU7+HwR8ErGM/gjrllcPIGK+6x
+KRwwTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTUuY2x1c3RlcjcuZWxhc3RpY3Nl
+YXJjaIIcbm9kZTUuY2x1c3RlcjcuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQCHXwx17N7nu4ssT7Os0WJo1NTG0vZY2hZbWuvS7YfJ
+Zv8e1CUg8Namrzi6bWBVJLnAwtIdar5CtI5E9x/3Tz/sAdDEC/X5ORYlt8W/smIY
+kgm1GWDmB5sTme1Nr5kHpEv6TPdn2DVW6reTlA9KX0LYFOrG3BHrhbwAMkCTk5iE
+v2VyxZUdJF6PJ99DlaFHGZE8klHLn7M+Far5jvBs6hze0DcVwZsH/SWX6qVFRApd
+WNPIyDT/SaYN/OpsfmgmTrko/ZkUujDZ0DIQGfvSHy3PG8WpKiBQVJJMIkKyzGQm
+J8b1oiYSEQngs3OofkRJHfHX9vIqMrn2mdd3earap5fk
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c7.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c7.key
new file mode 100644
index 000000000000..b222d127c8f6
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c7.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAoeP2gxNITw9opa652bgcGUn0DJ50JBsg7coc78b0KgxBYxrU
+6KHtJ+lxd8HCTJHuAnzaiDJLmwHmtROXtaaAnYfF0+/Q7a+zp3AIvg7CkrFyJrLT
+bzc4TRHD3GQuMwoJbp5ntu1TuoPRtEkqrp8bd3gNRzgQvnefIZmc2BM09HDPBXVg
+Pa+ia6ZJjGYsP0jfAFamGdNJDINAaq3W0yoOU+1Wc1erK7CYaHy0+xf+4cLA6WX/
+VoZV5LI2sQIu5zbYXO4xtd/0U/ylDuxiCpRcrmOTJkDq524s6D7KLjOwUD6FPQb/
+5PxxqbdKdRqcsVDzQSy+Gj9GjEvP/fXAme/e2QIDAQABAoIBAB+TBItAAYRZ7wER
+ePH7tI9CGzUy2lgIan22Q7qIRSrGRSJ9IzVUD5sPLrA6vBiPBaq6iyyxEoni7wo2
+DgrBtKgEyRKy3T7eW922Ph5DZrORpD/j9NTmIjb11gtwBoqx1HEcuAG3e/MOrqHI
++2gfZA4C/iP7zVg4oHK/MRX7eQKLhkjWLVmxzzlWTL7k76M0PyFi+rooG2wMnRYA
+kpsnju7M+tGAcYT05r03aE/B73FfWFT74EERCruJY9Amva/sVszJgiFUHrUiVDRm
+HsUqQhr5468VQeYSkhYv7KKPkTsmDY/zmvsRZxbwnHIwnBctHdr1q8fmfd+Qb1gN
+IDeqYuECgYEAu7VqYOasRp32CGIVwC2NgfitlyFYthOyf0WYUm6WY/DZgXsx9Ixf
+f5GS/gls5ZYhMfHn7qUZE9iZdbLGdD4x81ki5ATyJyPARJZtIRxL3VIPJlb39rCR
+jztfDsKg4Pp5cHxXAD2bIAT4g+CA814pIRfhytGrU0zc5afrZ0SfkRUCgYEA3Mns
+1M15/K/Q5hflBT3qlUa/NRTcrjUE9xaha/bHdOeV5wZsEmdFAZB9sTFQG1on342z
+EzKfX1tdn6zoaks/KX0UG4YaizDGn92DPGDxoUN+thfjZDmoBM4e2CvCeymP3zzC
+NKKGkXJHvZeA0iWB2M8irctbPCYxDNrbmQwJ37UCgYBHaA0OzdGtj3u/1qhVkj6p
+udM3kdzwbu/RUMVQXndhj4b+pcHsONY/bj6P20AM2p7IJDFJ7I5sqVI/nwG1yM8x
+tq/6NBN2ZjhHHz5mgSAvEn9m42Cy/Jpy9XAE6m9PBpnZ+8MQ+V6gwKNVb7s+WYyl
+5TLk9/wH1U6XItj/UPCuvQKBgD2r4Y1nf8WHTrCWyA/4xYfA5TPsid/XS3XBSoGf
+wwLylTqc3p8dXXccZx+RiLC7NBwLtpOMNlwqZzJs19apAhGSsIUnpk5u52xMMZYx
+9gx3j/NJJOkf+cCt2ovDXWWNrHRMwmp1Y2VJKKTaps8MTwLCiQtGnEtm15ihoWlU
+pYVpAoGADCkaxfyiGQAvZlHXViu+ks5/fOl4isdVYXQS9SLo1vxOK8L8kGEZlW+I
+o7idusBYoWEfTmBPPSmEtYiMYgEr/XYfGwU7gd3uepRjw2ykmBOVnDrvol9v4vHi
+Mr7o2Kd2AbRVRTN53Sc7awhXgiHKKwAGnOg29YzSwENL5FIWqkQ=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c8.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c8.crt
index 5cb3781514f7..b2fe0e6acd0c 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c8.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c8.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAIQ/OUvFomO/a5eOYDI6WJ3HM5VuMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW41LmM4MB4XDTE4MDQxOTEzMjIxNVoXDTQ1MDkwNDEz
-MjIxNVowEDEOMAwGA1UEAxMFbjUuYzgwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAIJQgqB0qJj7Db0GiPnvl4AsKtFa5DV52/DLfqri1FwhOcIVhtnLXFA2wsUT
-/g7aJ/b3tZszfTwG0mKSmNS9voTlpaLAb6Gn+vq6bBpgvgW3kvcP+r5lxMIGU0zs
-8yC3hqXwUOX/FbWZzd87eEiM6fDfhwg4JbP7nSk/JX8MUEqLAgMBAAGjfzB9MB0G
-A1UdDgQWBBR9W6ltaRBtK+Q3dWeXPFp5cwdSFjAfBgNVHSMEGDAWgBR9W6ltaRBt
-K+Q3dWeXPFp5cwdSFjAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNS5jbHVzdGVy
-OC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAW0cU
-1UW5gZip0ShYRi8iQ4CiXreN0BcR+Snts9b5BCf/P4mjg8x+55Ewz/+sVigyI8cV
-h1zgMKb84g3bnxG9vahb1sA4N7q4o+DDgDLCoMNUUgyuLWa+TSdXVfqDONikIosr
-fkJ03rnOg57QvOto+SR2hiIDC2zyHif6wUn6SOI=
+MIIDTjCCAjagAwIBAgIVAJ3az2MqXVBVL/PtjC6PanAQkDfLMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW41LmM4MB4XDTIyMTEyNjIxMjI1OVoXDTQyMTEyMTIx
+MjI1OVowEDEOMAwGA1UEAxMFbjUuYzgwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQCyda/Ce1pnhuky/aB0eyiMiNixT02IKKe44PBTJ/utWTzTHLMrb4xC
+ojL70dCZGgWtCbiSTdr0y++ZcSyvdrMOlu72DajFJy9XDd8EOoHaAhwDwt0Foj4T
+feJ750f+nxOrE/TyDWYIYSBhnNTlnnjIzw6/nWQUIARDidJBD8wT6TUTZO2K+Kdn
+oiUoSJ0kiQvhIA7BI6MdAheHhzB1vxA1vAt4JfBCuUJ4AOm9x0kfXs6zFEtdJzWQ
+FflLxoycWnB3FN9NXwJ4QuXFHEgnJ7ryy0DwTooPKaevSchImpjqs5FWwOcQLD6f
+jdVn1ByJDRX1yl22X0aAWJh4sNe0JJmlAgMBAAGjgZ4wgZswHQYDVR0OBBYEFEyl
+MlqF6EtNYPt4hiOgMcAazTp1MB8GA1UdIwQYMBaAFEylMlqF6EtNYPt4hiOgMcAa
+zTp1ME4GA1UdEQRHMEWCHG5vZGU1LmNsdXN0ZXI4LmVsYXN0aWNzZWFyY2igJQYD
+VQQDoB4MHG5vZGU1LmNsdXN0ZXI4LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAGMcdCezwP1qmr07WtnyCxU8qBi1IwP7O4KxAxXaP
+ElcddnxBBAQXH6S+VNcwLG1+WcDy9vgY0FwjjI+pdsND6LpVU7v4gBVuuaeF9M6Q
+Il1IAyUybspQzxLjI5CaqP6jxau98z3uW5dl3TG1dMHbpgpM0PsATAlS/1dE64v2
+DvgUD3iRzvFZRhe6oqxczATalBWl1P8R+JiRWROfzCuxSvvlAoAtDLukPxqJ4e0/
+TDcGVEmon1FIl5P2LP4E8Kox1vRAGN1Yw86UQEs2ERmu99dUaCBURDUpQeE1nz4O
+g6GYIoqJI9Hu/aRgXtci9r5V291ND7mDuyHSrvUzzrKJOQ==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c8.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c8.key
new file mode 100644
index 000000000000..7585be097ebe
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n5.c8.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAsnWvwntaZ4bpMv2gdHsojIjYsU9NiCinuODwUyf7rVk80xyz
+K2+MQqIy+9HQmRoFrQm4kk3a9MvvmXEsr3azDpbu9g2oxScvVw3fBDqB2gIcA8Ld
+BaI+E33ie+dH/p8TqxP08g1mCGEgYZzU5Z54yM8Ov51kFCAEQ4nSQQ/ME+k1E2Tt
+ivinZ6IlKEidJIkL4SAOwSOjHQIXh4cwdb8QNbwLeCXwQrlCeADpvcdJH17OsxRL
+XSc1kBX5S8aMnFpwdxTfTV8CeELlxRxIJye68stA8E6KDymnr0nISJqY6rORVsDn
+ECw+n43VZ9QciQ0V9cpdtl9GgFiYeLDXtCSZpQIDAQABAoIBACxrwOEfBUTJvxhT
+K3JzCYfZtF6GR91BiWYqj4eTZ0Eh+lLuTObd8ELRzvEBq/PAF3OX/NpmCL9wV+aQ
+1jeiyzUxeq/CfKT+ePkpWb7atWHHrqN9CLkTRnJhnlOLIU+O1z15TTpG/YPOcVn6
+yL+TL6uEZr5pdfTQd188WhqZzQmTU0PG+kM3C/28qYo5QLpj6KAdRunRRMez05Lr
+GhyWbFsL+3+RpowhNdwDyama4oUJAL1Z9RT0zEMnjS1xJCQcvShEeqrd4FZAw79+
+68GKgNrM7H8snBLcXKYLgI9fYE0zrF4V6rdzz32+FjVDPQbJETOUZNuxZey8bgRI
+2KlNhi0CgYEAxUZAquV7WQAfP/UY5JQQnugBcBNXIlCvNVku/i7N0u/CbIsegaGl
+zI8Ml9olYJKZpkD9/F5qgSTLgPaFM94K5YjHoIV1o8NVjM7wWDrFMedNKs4YcKHF
+a0uWul7Wsd7xK+R6svMluSs/j0Po7gmXDSHdZhf8q2U7QYbqBUm3v0sCgYEA55Wd
+sTNddkCVGEhP/I1SYrWkJrRTA6GVCngPRUE8xUr91WtG5JDKXNxDpzCxNQXtehM6
+SSygF828bW0unaG4ThLDJdbbxs/m3cR2HTTFzZZQ+uecqjgjanEpYsRmtPXiOnuT
+YocKBZA27G8AsTReWpwQnWn3bbw+rOpLEVejRM8CgYB0T6dxGzxOjTPfkbLLkkD6
+b+/pFH4bTVcALUT3uMDBzXrkZGHoLFaE9RKHtaaz+bt9unQFh+Vjnz4dGzwOCKyE
+CNEqY4WouFKIYrpySrRZZ/andjwSrE6ppDeKSsBqvveS2XM+bv2e31DqNy/uqNjW
+dVEEEMB4jYcSa2kvFPexYQKBgG/E5/yz+7Gs7chwbhpH4rE9VWJQuZgRDiulUkls
+ePYHgYbVZbu+ncw/im9QstYSbFD0orbaUNHsO7rcbRkrCha0O3cxwZSslpIF1bO2
+2pikqaTOz+/fdEhVIVJH1AIWgf8T0UhGQHH67mciE8Vyru8wT1yJ3h8aMW2nDl/q
+WiRFAoGABUZtvfO/A5TQEsATIt+RqRTuVM7a62E6a4A78zrrAZsqASbcLAUDzXuw
+DVOxvRd1lmvYJSTj/Kl/OpGX97HywtRZiP++IusnEPpTu2K5FhC2670QT2OyJs7/
+D4GWM97PdR+6TxO9BCZYyjtHJLq4/EBv0uOvMAOuhA+33Iadcq8=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c1.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c1.crt
index 5c822ab56b20..93c946a51491 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c1.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c1.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAPfDYqSl581XnG/kfEiDs6IEfKWnMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW42LmMxMB4XDTE4MDQxOTEzMjIxM1oXDTQ1MDkwNDEz
-MjIxM1owEDEOMAwGA1UEAxMFbjYuYzEwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAJFHcBG8RLrI/ifvapZW6f42HHBEf3U2FFqAwpZMX8IDVDddF7OQNPBEUrNQ
-2dCFX8pkcEJcnnIlGP0NDN4JEV2N71NjN/WTwBFoHDAXs6e1aKaLRv+Hz5xqIdtb
-5QfqjbuQsHFoE3HVrbDZn60pENN6JFn3ddt1nGYamMRycS/7AgMBAAGjfzB9MB0G
-A1UdDgQWBBTLEf5TUnKZHJfUMV/CQDFT+5k9FjAfBgNVHSMEGDAWgBTLEf5TUnKZ
-HJfUMV/CQDFT+5k9FjAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNi5jbHVzdGVy
-MS5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAcDU1
-Z/0qJCuqIXugAbeBQLBiDwWpsPC2AatBslCaHm6prnA4QCGwZ7wI/FnunMzmGf13
-VvFEEWnW786+mh6WkI5iEba7sHf5vWmKO9rBJHYm3bcFnh7A2D8hvf7DwFNG3Kpm
-+/iqy81ag/BOveDlpu5fJPzAvJwTpVJ6ie0XQ8U=
+MIIDTjCCAjagAwIBAgIVANLIvdFnuHRekTC0k1oKx8PcCjk4MA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW42LmMxMB4XDTIyMTEyNjIxMjI1OVoXDTQyMTEyMTIx
+MjI1OVowEDEOMAwGA1UEAxMFbjYuYzEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQCc6djaJFv/WVs7h/ovZq5KT6fIdV+PBqb5nazw4WQ7QqtADqFl8pCe
+lXFYG581/Xc1SQIvngWIIq1Cdm6yu7mW+sLkuDM/milX07HdbE1Oyi0qiqOEoplJ
+OyqdTkY0Pi+aNUUV63sUmWO8gv468dmgMLZ5STS2QUoZNYdDdaF18TABWjVLXS0Q
+ZkIpgHz8HolmQZfcrsEh5t/8lMBVylAG5gJSr+SAVuH6Fchgicy7+6OQmf101ysq
+Ln7IhExKz/ZM1iXoBfZA8mOmfD7JndHsi9IZNLPFo19olMTOCZHMr7NPjKv+9mc/
+w12+rKxJrQf0i8dKUb5HRD0ndb46X7Q/AgMBAAGjgZ4wgZswHQYDVR0OBBYEFEoU
+y3Rv8fC7nUGGgcbTgUzNudj7MB8GA1UdIwQYMBaAFEoUy3Rv8fC7nUGGgcbTgUzN
+udj7ME4GA1UdEQRHMEWCHG5vZGU2LmNsdXN0ZXIxLmVsYXN0aWNzZWFyY2igJQYD
+VQQDoB4MHG5vZGU2LmNsdXN0ZXIxLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAfuNjWsPsYZdd3ORwNr9qKt3TPGbNRSONzKGYB/Si
+I0ULuKYBKQqXkBFOj0QAWm+nmfBWTkDdZYkeDNAQ54jpxznL0uqi0AGuxstBPYLv
+TLVjG025AHVbKQIBZiTdsunb+e/uR8GzJHMpl1yuUVHuoFiWUzCvb57mxnkYKerV
+Pzb8suDF+gV33BepudoGKpmkNTE453Ic2QgTz9+LHHBZWzozDWhj71t8sYtpm3AG
+s9+rgkL0WIns2qqqZ3goC3SbRez0M2Jv18fzcDFLEkcICuJIq+sgnzGcm+9Ya3jY
+DuinoRoK+fR77kaNKC6KSakkjNM5nIssL5g/9szbDFxVKw==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c1.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c1.key
new file mode 100644
index 000000000000..5e044be083be
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c1.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAnOnY2iRb/1lbO4f6L2auSk+nyHVfjwam+Z2s8OFkO0KrQA6h
+ZfKQnpVxWBufNf13NUkCL54FiCKtQnZusru5lvrC5LgzP5opV9Ox3WxNTsotKoqj
+hKKZSTsqnU5GND4vmjVFFet7FJljvIL+OvHZoDC2eUk0tkFKGTWHQ3WhdfEwAVo1
+S10tEGZCKYB8/B6JZkGX3K7BIebf/JTAVcpQBuYCUq/kgFbh+hXIYInMu/ujkJn9
+dNcrKi5+yIRMSs/2TNYl6AX2QPJjpnw+yZ3R7IvSGTSzxaNfaJTEzgmRzK+zT4yr
+/vZnP8NdvqysSa0H9IvHSlG+R0Q9J3W+Ol+0PwIDAQABAoIBAC16EOFYPFmLWkes
+pzPpvIXJBEWyztkzAl3B9wdxK42JSvyrwNc9L/DmOw1gJAqPxWbqG/oVAGFuO2zR
+yk9+3NEHKEkFocTDN0w/DIOyCc3WBrUXrUCZp3dz8Gva7bnidAVrfvRVZTyZIDDY
+fjozMfGdbKVGNELGNWWNAADrLLpzkf2auItVP2MdwbhGnSUGjRiF+4KwCPhJR5d6
+OuTatvyw5bDAB7tG3LJSqfJbwRoGGNErxQB2OKGtApzwhthRGVM+1Beyt8kigdzd
+EDvfFvK4d8VdOTxVFjroq1obv0DHMGPQ37AXgLOI2AoxKBCOWYNhirpI/uv1uFpZ
+31oKdSECgYEAuSW2tbvAq46Q5YAhthin+cocYUDFK5cWXBwiqlVh2RkdBXdFYax7
+dIL1VKK5/99GHX5zg7+iR3bsKrz4TjBthWvhDTdjZzYHUcHdgD9/abhs4NgnIgqV
+J/alo2emQdTx8rxBBu6p3JsIWLGmGkrKiQBAm8XhmACkAE9DHEGZ3JkCgYEA2PYn
+VmYMVgykDGL0+558tI1OGtzGP+YqJSTBFrAVtk9k12uNjBUwsH0XtvuLcok15zrR
+nbDZp/DAEtbp4gDwXlQVerivEsBGAW51wa5GllPh/RAkQg4aLWri8WGMSz5ORiDS
+452BOslDC3q4UFyonuaPEgT3qtvmIUtuXvXrBpcCgYB4aRygqJCvgjKd72z9Pp/Q
+Bi/a2ALjTImQMgQhfxH53SlQqrcGxOQtbBYdiw8oZ470E48QugrSK3eAfBGI2FoC
+EZz1uXX+Y9byxwBFLgaxPO4idLFOpE9JBFHJKdmoqtMHqIMp/QGfpXwSIscCBm4e
+F3ntVQ5YiuIeCW596/Qi0QKBgQDFrYGp9tDGlj3iC0eq+nwlBZpkX0NCdTNxJEvJ
+oCYoNa5rmFMfjy8Nnuw8jHRBp0K1q6HRS5EEsxZOFAKFKNXZHEN1zU+2hflC1KH2
+FhO6ZDNDn7Fpfe/b8cdUpsxswhgPTVjYiml0ZmvvyaCll8cLAIbapCwEQNcs/s1R
+OD8rgQKBgBb0NoHC9OQUbVBFkNdmeHG0x1VbB+ObgGg5yXSLeQd+Y+Vxgv363i3i
+sBv7sMyS9E/Y/sSY7JUmnZACpl+IIsBa4NK+ulzobeos8sXXm/ni45E+GPelhd0W
+CBmjvolZzi/TQejB30DjizY9fOFafdUpaVN7ZD9Xwasw9WOkSxS+
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c2.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c2.crt
index 9cf522d3ec27..bb152cab30ad 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c2.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c2.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAKSYs2CPma3+QQ1KFujo0UdYTPChMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW42LmMyMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjYuYzIwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAMJ7+brk5AZGhtD9dMKVhkHCmsAhULpd94aF8nK50Vw/Jgf/SdE6yYePrBxB
-C+Or+o5+N8Dwe70escEOH6o793gg8yBWyL1AagoJZDV/+A41cxropz0jVW5X4SJu
-+y9tEgVA7m6cdefow1SG1EiMLMtEHyw5wKNmUKrl544O+MQvAgMBAAGjfzB9MB0G
-A1UdDgQWBBQGIpG1JOyVo9JCyp1/h9yMhHE3lTAfBgNVHSMEGDAWgBQGIpG1JOyV
-o9JCyp1/h9yMhHE3lTAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNi5jbHVzdGVy
-Mi5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEADfpT
-pfnIHFoNw5xvdxOelm5S3CKOk7lBroCjnowGaR59D9yoPRlyaX+YaRO51Mv0rzvS
-3qJjjrKlCXeGNF9rKU0OlM6MVuGmlMhgj6nv+0eX9ntHVYzSvbBxXUx0/jvQZYV7
-C2t1qljMjpFoi++p349dWsMfosx8X97HhXA7P8w=
+MIIDTjCCAjagAwIBAgIVAP7ji9cnys138Y380I6ku9lV4ZwKMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW42LmMyMB4XDTIyMTEyNjIxMjI1OVoXDTQyMTEyMTIx
+MjI1OVowEDEOMAwGA1UEAxMFbjYuYzIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQDWfZjEJbSQJbOa3fZmzSZW+KtZJuMAc6aRdWOR5pfluo/W7VlD0wOz
+gdvDmbQDJJW9UP59pB2u3a73dGxVqKS9aHaLrWcbqjHzVWQcrdeYHzOBRzNqOWP/
+TkX8lWhaE/tMphI0qgJSrqFyfq7h/j3rYhUgNwn2OuQOL+pKhgK8dMigsXmpCYwR
+vvX2sJu8ti8GUEPwwekUinlBzB0GbtjdFEHApcJmo7NMwZuCGnSwKczWypTQgBIZ
+s34Pe3Sen1/nQj5iWxrNMY5dib+7Gy9Gfva9H3cbHFfhX8TWrJwtreLYt8ByGFHB
+ec8w+Vm9uJQNY/m5+Bfrf3W5YUpQVQpvAgMBAAGjgZ4wgZswHQYDVR0OBBYEFLrN
+K93Y6ZEYk8ybn4MjszXVvGkgMB8GA1UdIwQYMBaAFLrNK93Y6ZEYk8ybn4MjszXV
+vGkgME4GA1UdEQRHMEWCHG5vZGU2LmNsdXN0ZXIyLmVsYXN0aWNzZWFyY2igJQYD
+VQQDoB4MHG5vZGU2LmNsdXN0ZXIyLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAgi3FqDRIZLBXLzIkXvquNA6pL0Ed3IHpZS7wQHOT
+AboqH9nS2rVPeRaW/9K+X+dPexLfuxl6fe2vBdBX+ZAKNurYGW3ppTWQQI1dzYDw
+UlZE3nvcZ/xgiUcqe1g3+trx/HNcae/YTYGOUEQeTJGPE2CHbjKhsbbbX38cymKY
+l2TcN/jFm/yZ8mBo4nCTKQey6bdo/aPtauDhsyNPn6K1+JHkZ6TWSDkrHhE5IiH5
+ddRa7Rx7ChUDIXVFO4N1mRSrA1bcqftSZ1xm81ZY1NwKAiPX3fblcYJ20DWbQh4i
+wHL1hkP24ZgGAshGF4FlqUj0iq5OY6wce+ey9FDo2nyj+g==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c2.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c2.key
new file mode 100644
index 000000000000..f43065033ebc
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c2.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA1n2YxCW0kCWzmt32Zs0mVvirWSbjAHOmkXVjkeaX5bqP1u1Z
+Q9MDs4Hbw5m0AySVvVD+faQdrt2u93RsVaikvWh2i61nG6ox81VkHK3XmB8zgUcz
+ajlj/05F/JVoWhP7TKYSNKoCUq6hcn6u4f4962IVIDcJ9jrkDi/qSoYCvHTIoLF5
+qQmMEb719rCbvLYvBlBD8MHpFIp5QcwdBm7Y3RRBwKXCZqOzTMGbghp0sCnM1sqU
+0IASGbN+D3t0np9f50I+YlsazTGOXYm/uxsvRn72vR93GxxX4V/E1qycLa3i2LfA
+chhRwXnPMPlZvbiUDWP5ufgX6391uWFKUFUKbwIDAQABAoIBACjQTF3iiNm2+taY
+rta/6bA58CzuK9q9pu/dxP0RwkyZ893jWw1/XSBusNkm6ayBbSbRfyN8M5O/Jhh7
+JFQmW3aZJpmMfOa0O34AfSTB5ZjSMxRB7qQjWIZPqByYd3tQewyKO0OfKAMArMLj
+HTbukGy2RKvVHUWnzhZvAqd8mY+vVfQ4k04s6IuGgu3WtfuKn2/L6BIRYneMYkaz
+cDCXQw0Q4CFMFq5w4+kJinLsla47gX6ck65QQJUyrTlcxEi0hX1T4lGnDrsOADXn
+QSus7K3ChgJYX/2UZI5U2ixLWl6AulVpq6G1CInTddVOSw1Fk5oeWYvX76z2gqWN
+CQMBL8ECgYEA8q71tx38PS5OyT89iUmaJeJ+PbysvxbHWdEBkJmBCp7XOkOBV7hX
+tdyXu6NB+rRYezIMFo72+lSR/aeMSc53zjnLJeEumcsF4tEkTtR6/8v0S5DqbHoN
+Rr5NLHD3KT2038L52GERwrGJiLKt0o3lHaMhlX7YU9SvcGiKg/Efx0ECgYEA4kKa
+ymvDImT2FSnM4z/jjVhDd5fFDRFr1ubOCmXRGpzLPw5IvgP16DFTtjVvFNAeeG37
+ptUtb3qWNRi5tUnPZTdQ+01XkJO2Z/sjPtqdWGNZw60Tr6iDWPjBUBU0uNRW5FnY
+SN70DsMphqOnelhONGmAMwUyOx14YfBZ/c7jla8CgYBsvve+OYKJC9IkHeF+PmwD
+o+17zqr96cAkwCJaYF7RmoHJSuyPmTw8cv/PBgYPyHaJpdCC30dOv0y4BDc573oC
+ACXG8JPIOJCHpb4GUPJY41Sx89sIBt0iIS5Fxs0KPutpBV6dlkm2G4YMqjx3yDut
+vybSbYeUcQ64aTAAlbDuwQKBgCovVuLkwqYYAtWhTIUWNVv2F+YlUuUD2YJg2AuS
+PjFjCHYCOMfMvz2CbOQtBD/yKaw0fVX2qxj81yKhMIa/cz6g2KVyHyBimgC7DzA5
+3FpNc8AK9f+rQIoHHyvKOGE5ndXHdo7nmsNh2c0zmBc0fahB/1gD4QCqlvZTtMOg
+nkgbAoGBAMZJAsDOj5pH+Nxv2wrZ29hQfgUyeSmZxJvIvkgY/AzZl4GaFyEvcp2d
+ytMAREFv+zU6pPLQcg7XgQ2wUeOQo1yu5mjmkikd2SajvcZnnxx5ABIx1K354Q9M
+OPoLY6Hn6f+2TVaupSOYvADRiT0ljYytQh+WF8ocLQIUB1gQNFG0
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c3.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c3.crt
index e6347d23ae61..a76eee10ff29 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c3.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c3.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVANfjBjDTVL4+UG3xEGlDqhK/fhH4MA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW42LmMzMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjYuYzMwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBANBW+tmJaaum3CPc+7wj+qrDkbvuDue24I2Yodp3HSsChx94KmFDYhGYuRWw
-sDF3Cu8iPyUi1xk4fCQallLCzkwny9j6dljAHtAYkVfNP07WBdPJ79IWUn4qbldq
-LTam9ZiaQZ0jPtHa6PUmj8doc0QWM1ySfJcHqCP53GkqpH+vAgMBAAGjfzB9MB0G
-A1UdDgQWBBRiy14zafEFR9kFLpd7PeVFrLdKUjAfBgNVHSMEGDAWgBRiy14zafEF
-R9kFLpd7PeVFrLdKUjAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNi5jbHVzdGVy
-My5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAZueI
-4pUaJEsWQSdPFg4L64T/ec0ETzolzIDb/uxwCBT0RGHRP/94x6o5EGUdA59V2PMT
-5jWOAfa5TgtZYuuAXRyB93Z8wSdTpeHRyxNiVDCRy6Jou28VDIRpFlaJABe7ZfUF
-+TEJ8FiDyjABXkUw6TkIIuGkKaGO4SNCX89P+i4=
+MIIDTjCCAjagAwIBAgIVAJJJhjqi04Rj8I1hF2Y25dKTOJIYMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW42LmMzMB4XDTIyMTEyNjIxMjI1OVoXDTQyMTEyMTIx
+MjI1OVowEDEOMAwGA1UEAxMFbjYuYzMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQCuyLsP+wqu28kvKImCzxCPChhZRmClv9Q6p2yvDiJ7qD1rZ2JL2W14
+OLuMMCXFEXBi8mZp/OJFj6iz77YV206lAxuz7k3xpVO2C2qY+nl4BojHrqbcnED6
+Nt3JpikOYVKoClG/bB80bGBhd84pEwMygc42LlWG6sxM4A/L1CaQxG5BBF1X7kFM
+75GQkHvkwOM5vRsgTudjAlk+TAF1UxeKkdRc3oMlbFZRGVUfL17DdvPn2R0eAmer
+bhdJD3ozyZfwR8ZmVjrjzVyFN2IFwKeosaSOgLpG9delea5KOv9kNe2X7aEDqmH0
+BWVGsVonNEMeHcvc9nI+YH0oGL24Ury3AgMBAAGjgZ4wgZswHQYDVR0OBBYEFPc6
+vuQtIXXwdTvCsxr5r+9T51DcMB8GA1UdIwQYMBaAFPc6vuQtIXXwdTvCsxr5r+9T
+51DcME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGU2LmNsdXN0ZXIzLmVsYXN0aWNz
+ZWFyY2iCHG5vZGU2LmNsdXN0ZXIzLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAnCPSWqjvHEEBipVBCAlPc/KD3Ja3dWt4o0orKgls
+9Y69rQzEABHltW26s9+bf5GsnyI6Dfi8uGRaWi8phGbXbcDcsEnMnjlhSCba0212
+xh9UQPWm8GQ9+dE21bUh20b+V+68JZeJcLSzr+mu3FCE3BJailIygoc4UHIfHGuX
+7l4xWj4CrNAdEMAnNwyu056f3xbD+o+sqpUCAcDAqf1SUkSQ/AIGG/KVqv99lXDh
+XCV32ppcF8WKBbo8Dls71JUNxxxBc3d3fjLN3QuEhFr8J14X1zW10R/QEDLUerKx
+uH86es0jQDT32v494jt750kF7pb6jTVK9tRukLzFlk9vzw==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c3.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c3.key
new file mode 100644
index 000000000000..d8df3a4890e4
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c3.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEArsi7D/sKrtvJLyiJgs8QjwoYWUZgpb/UOqdsrw4ie6g9a2di
+S9lteDi7jDAlxRFwYvJmafziRY+os++2FdtOpQMbs+5N8aVTtgtqmPp5eAaIx66m
+3JxA+jbdyaYpDmFSqApRv2wfNGxgYXfOKRMDMoHONi5VhurMTOAPy9QmkMRuQQRd
+V+5BTO+RkJB75MDjOb0bIE7nYwJZPkwBdVMXipHUXN6DJWxWURlVHy9ew3bz59kd
+HgJnq24XSQ96M8mX8EfGZlY6481chTdiBcCnqLGkjoC6RvXXpXmuSjr/ZDXtl+2h
+A6ph9AVlRrFaJzRDHh3L3PZyPmB9KBi9uFK8twIDAQABAoIBAA4MVTiyGUg3oyJ2
+tCK+6NpOWemnBfKcif7S4SHq1Wux+InO6/AjwDA3Ax//6LE/txRE3mVrbXZg1xLr
+U+vKzjXmG45pO3uOKb3Kih1UmHnEsK64A3Jc3r4dtdLU15zvxph6B9sGYjWw9kgv
+PHNBs3KWS1BAhiBLc/ND2REtcpIVDWP7LtInJmaMKms8vEIqGL/rKa9on8Yd/8lO
+/K1GQp0vv7mak1grlb/uwvAmwK7gS5RYZfsSUL2rXOF3UKE8XYGt2AYKyu7Pf8XB
+fR62ssBvTMsHRKoOeV6ajXRc2QS0rjrOfN20Yg2NCmwspU5z9tw93zsraiJZouIj
+twaLtLUCgYEA8ljsOn15TMUsjuzNv5I5lWsTbihQFyKRmVlnn9UUVZefuHfNaoFd
+TbuQX/TblAtRSSG3v2Rq43JH8N2UMEpxyiGdmba7juGsd2sc9W4CysuGoJpbaoFC
+2W5QtAgCjNF/hE+SkvHNrgNed8k9TM64hLE0j+ZwdfKRE3POz8+XPs0CgYEAuKFt
+K9ow43AXI2zPnWS2QXAXdTJlvTzIO+4virRYIdbWXNsQnN6HILaO3mzBDXDT7Jh/
+AJWvunYjCYkX/pRE4wTcZIhkmxD0bAb3afuZV3NjSHaunwwW5Ms/bYqxTAfgkybu
+xBdj8aQN1ts5OQ5z2v3+qwk6LeElAqJ/FvBmYZMCgYEAqp6oUdWc3MvEvytA/6q3
+F9c26ss+98rCrzAeoIynsEr7jCfsWxuQ0PVIhAYjwWrmY5J1hy3vgwomVBHsDsOa
+sTBRgLMCAWoovjqPPMui85uuwMtrqOQwaFwePWGWSgYEvOY4z3Ol1ulQTQ1FNxoj
+FmicFOOsMZ4BPay+4dfJJ9ECgYEAryL5oypzS5utZz41ZvD87e/9YH+xAKRisG7N
+jD5sh5T6fNSU2qk/9ykVNypkqnJHpVw2xQA41KBxtf7k0uokulCHG5pCDNI1eS/W
+jCNNHyZmnHtwQ5C9OsUN4sQ4UI38WroIrdweQz7CnfgQPO5XuflMFURIevLiH7Cy
+67iY9Z8CgYBAjrKAQVyS0UawQAg843jSxtgI+XkiaIbd2YTMpXUig+80dBOphSIi
+C8IpcIe0T4UtF5LB/pziVKBQ4HPevTXDsQnkI/6cTkzKxDKoqogk9izi7Sxx19eW
+tKuoONkdlLjgudlSo3mmXo+iWQJ/0tRz65ckIBCQ7MnhnLuiOVXxlg==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c4.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c4.crt
index ba5d4620dd15..2b0dfe3a64f7 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c4.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c4.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAIXNMMJ8xgAMFoITRuJVgQZCsNEAMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW42LmM0MB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjYuYzQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAJDY20sl1QeuOSxilJdb2r8PygM/o6EPlIJk/NHc6Lh1VG2bxqsS091jCp1U
-Y5uHdynMKXxVhgbIG+rKHgP67rh+3dTWzvThWDMZ1ljMMpdf4NNH3caM7WdgPzj6
-FpJbuEevBnNp9ENRjKFv8DxUDiJPFsXcCZ/sGuK0HUZSzla7AgMBAAGjfzB9MB0G
-A1UdDgQWBBTPFm83nd3zU/X8KaQpSIF8w8njkjAfBgNVHSMEGDAWgBTPFm83nd3z
-U/X8KaQpSIF8w8njkjAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNi5jbHVzdGVy
-NC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAKAQP
-iW5nHwgAXy92DaO4eLdzMuODEMgR+w/qSkMw6kaTyD+ZjeyTjlDbldXF5bomQmnR
-w1/90j3MPHoelX2POi8D2BPT350hSXouts7meM55U8fnOHXMLb7CpbNO4TTdYLIO
-2+emw36SIIxVBRz+T+7lB7532vM6TuJN5AsW5kA=
+MIIDTjCCAjagAwIBAgIVAIe0NImvOxe806rwkTiIZ/RB/LIxMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW42LmM0MB4XDTIyMTEyNjIxMjI1OVoXDTQyMTEyMTIx
+MjI1OVowEDEOMAwGA1UEAxMFbjYuYzQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQCefHL87xAOATBaSF5UeFUEvr2DNqUSqZ6DMUOObhBhPhhAipEsyvmk
+fetlBL8fdKml+Sf8GYsGcnmcNqHDTmqKfbP+s25pNXf+EOzlGpc1zyRCvhsWzai0
+vODEOkFclBwSKadjIsfYR2Tewn7EYrzyDDd+sgr6fEI9STkQojxR70mVIRKRVNWj
+GqMrO4xxnwP6X7fpcY1O2z6PaNlh3lPxemHMiWE/lJRFpHjtbK4SILwN8JHQrBs2
+E02PFL8TtiO2HQbOUomw28ShDYozbjbNWTzknmC47+TPqR/KQRZtb0TxEnreBA3J
+8kOp9bfdO9OULqtaEdwYEaYsOjzEW7/7AgMBAAGjgZ4wgZswHQYDVR0OBBYEFDjV
+0vJkaBa2bYNqN6Eu9tjW1UlnMB8GA1UdIwQYMBaAFDjV0vJkaBa2bYNqN6Eu9tjW
+1UlnME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGU2LmNsdXN0ZXI0LmVsYXN0aWNz
+ZWFyY2iCHG5vZGU2LmNsdXN0ZXI0LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAnlglJ56A2OELsMWc6wXeAbaH2L9grrXFD6+qqbe8
+epdPchhqk4XNCzia23fG3xnevssq4ZgPRzSTX+4/mfTpJyvEhazMAwGxIgo4RXU2
+lfVV95wXNnkPqIe/f25Z+WSkgoqOsg/2fKsKUtGC0qCvjFmzHEJG5KAfuXLTd7iL
+DcxCOJPliYFmzsbKQ2qlCKJMzA48Pj+CyBiuMomoRMZvaO94IkGobfghElg+cnO6
+jSX1QS+Da+BpDW6hd8fXfL17PGdtxkORJd3FW1Vet4Sc0L1R8Lel6VN6RvnppDAG
+qXn0Lzz+67liQerOPXqlPVLR79N77PyM1yR4N1MRbhOvgA==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c4.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c4.key
new file mode 100644
index 000000000000..03d4a2105dcb
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c4.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAnnxy/O8QDgEwWkheVHhVBL69gzalEqmegzFDjm4QYT4YQIqR
+LMr5pH3rZQS/H3Sppfkn/BmLBnJ5nDahw05qin2z/rNuaTV3/hDs5RqXNc8kQr4b
+Fs2otLzgxDpBXJQcEimnYyLH2Edk3sJ+xGK88gw3frIK+nxCPUk5EKI8Ue9JlSES
+kVTVoxqjKzuMcZ8D+l+36XGNTts+j2jZYd5T8XphzIlhP5SURaR47WyuEiC8DfCR
+0KwbNhNNjxS/E7Yjth0GzlKJsNvEoQ2KM242zVk85J5guO/kz6kfykEWbW9E8RJ6
+3gQNyfJDqfW33TvTlC6rWhHcGBGmLDo8xFu/+wIDAQABAoIBAECpMLb+iHPWS1aC
+cS4QfEt30NVAjb/cAWawSkpJoJg0H+vxPQLj+09/6nLoKVwZmn+zdHtYJo8mVHSG
+G6CWFUTUvYO8zF7TaFX5rNVacHgNP9OVQXMfzGWVLDUTOYieiwdkBD2fup6EnmR+
+QP8587Hjhx7DYFRLLzFW9xfQwUCB9hZeyO2L40ht+2IQ+jCi96Bo2/DBs8YY6fjY
+G8jegUE0IvIywOqRjrVmeeiP31yio7Ov4GvXhpjOoJr+HW5HRM3sfNDsYLB5/rzp
+nJe57SZKOtvm8qEG4Myl9hhE/F2MurlNqP8Y06gFofqyQxp+cHjt0wa7+TOuZLRm
+dVW2FNUCgYEA2RPO7Zv4tFDVFy2kYYUHShKacrXvBgrbUKecEe37cAOHxJDvX4jS
+0oPOTG7FaI2tuGkRJ5YbGcP6zGNi5cqL/gqk4egGkqbCZmjvKdqB8T54tLdGt/5d
+gKC0KMCtkXZpEldUHAa41izXGIfxUb7uNX1VFu9oENAAkQUVewqFQW0CgYEAuuc1
++WSCNpUoyuDZnXHr7wKH1/fLTnMLq+JQKPOs3xjogdZ6FJTiMQ7JnUEvLZ2DuQis
+a/M12aW1SHiXuLN0ojp5+uc2ySgM3LDMWC/YOD91zVDl6ZM9rPTs1FGxdiEdxry5
+X+ks43U1JyMlBmImgdi6x6q1vkh1Vw3duTLPDgcCgYEAk3P+Gq7Y6HsD1EO/pjJ7
+uhga65Kjs0IKqaXAwPkb04HCZoZg408IjdDUgFy6IYQjneFYssAm1oCmStGz6w2D
+vGO0jnNSi6lU4cWjt5VisOFTksXv9qmFEAuZdRk0jVuWkcvaKVNzmKBK17H3/SQp
+oRHqi7LByLHRwLBoX/TMgZ0CgYEAp5t+cJasmuwqN71qCWJHFQBs98S0YjaTSeHa
+xs62wR6iKwU+V0SojxvWu5ngCAUQQhfAH9JsATE0H7hdPvpXZsAP8Wv1WnD7D9HQ
+fmjxeoeWSbf5Zfxvu7GszD4yaWALRtSpXJRLd/L85osCwfW02I6V+UZiUzDBYX8C
+zfj9a3MCgYBFwdBbDrqkDZVmCdFtOmYSgwrEAHEJ80s39XLnrJ8xndzoATrV0xlC
+5ng0mDeEBPtDxBqgkJ7gLvlfVFeslOTt6zbOoWrjFbDSLQLd+Fmdfi4Zlr283mcH
+KdIcY34lChJkc+zFGN1Z7VmYRE4pKWucRXbkI/ulOBHqf68rUfAtpg==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c5.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c5.crt
index 026d31155ac9..914d1ee1d7f5 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c5.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c5.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVANAqTNvSXNb+QuG2Dty261T05fD7MA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW42LmM1MB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjYuYzUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAILKAZL8kVxWDh/9//AAeG5U/NF0iQBSruy/8x+CA2yzV6moe8V8Woou7AH3
-ksjTpg8NyWUejvC2WQP+dqvr3+t1XhAuUdpYfcRVg3zS4mFcg2KGrvnVctXpd7Xr
-/L03FIEpaQhUsw+xKrd9+d3VxQMxFyNWAHhB4/rRIqWNjfh5AgMBAAGjfzB9MB0G
-A1UdDgQWBBR+wkLQRaVnAEyo6Ign+W/48tJpWTAfBgNVHSMEGDAWgBR+wkLQRaVn
-AEyo6Ign+W/48tJpWTAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNi5jbHVzdGVy
-NS5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEABjLS
-Ws3YyRfF4XhvveU82BiIKS0B0hH5P/ozXiJYUjBZxYc/U1XZR6m8LRQLCfc/kk+S
-KpZgizFPsJvaqxoEFj9YgDFraMD/3dqd/R8GiCeOQYf+LRXSbG+eHnHk9ouzqciU
-kZXeeU0dnM9aVlBFrjlQLk4E4GQhdTZGCxXuuWI=
+MIIDTjCCAjagAwIBAgIVANgI4RhooF1HoE1oK495zN9pe8fkMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW42LmM1MB4XDTIyMTEyNjIxMjI1OVoXDTQyMTEyMTIx
+MjI1OVowEDEOMAwGA1UEAxMFbjYuYzUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQCmu4VjyAtgsJ4COjFr3uGQ+kUTJROqQaHEnXZXexzpPD85W6aVzn16
+rjeXIhWo6L3m2G2CHFgibWozo2MkJiYy/owfTWGmfzSmHirnmibgB9CwaE4SdZnc
+9/wQWg3GbpHEOwEKyA6wLmRgrgUJHy25e3rc0zj/Qtbu5vG11KzZYf04kzT1qFxk
+yu6Hsw3yUXIEWxJoB7ddcW8czsSC381LfrPQrtfB/QLn9UHWIEPKSIWfbx5uwrMK
+js1Qy1Bury4MmX97gDSVVKHuPjXy3cixZGyaejmztDEySq9YAjm+hjZLj+CR4/a2
+/vLbJSiGnnJcq/J+3FRm+WeSukKBc3hJAgMBAAGjgZ4wgZswHQYDVR0OBBYEFAyg
+VeLvqzbOQC79xMR45fLR+k+VMB8GA1UdIwQYMBaAFAygVeLvqzbOQC79xMR45fLR
++k+VME4GA1UdEQRHMEWCHG5vZGU2LmNsdXN0ZXI1LmVsYXN0aWNzZWFyY2igJQYD
+VQQDoB4MHG5vZGU2LmNsdXN0ZXI1LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAAM/YxLaCsiL1b2CSk5dmaAPxE7TedMFHkI/xvgGU
+3IYLO078ACl4XnTHmNoZuAYoqOQ3xAMtoIbw0eMyx68bSDQ4Tjd3GICZn3tg+K4/
+lDSFjecyfX75JRiLbFQJBUdD2pkh6lKy2wgvIosMWdL+JGVKvbBSlwJG8n8/iH9u
+46An1iCbYEbX7UY4ft3bA9LNaOpTYwcIgOn4QkLANQYa874CCzTkIYavClKk7Url
+DiOCQdFHMp1KCXavoTYSrRZt6otR20gpPRXlmED5j9XKhILnFsPT/HhmkU7S0cpy
+V2hS733rAC7kQZQv1YWdLSQZMVBA3lzO5pqPk+7O6wEYGg==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c5.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c5.key
new file mode 100644
index 000000000000..49b7e20b94a2
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c5.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEApruFY8gLYLCeAjoxa97hkPpFEyUTqkGhxJ12V3sc6Tw/OVum
+lc59eq43lyIVqOi95thtghxYIm1qM6NjJCYmMv6MH01hpn80ph4q55om4AfQsGhO
+EnWZ3Pf8EFoNxm6RxDsBCsgOsC5kYK4FCR8tuXt63NM4/0LW7ubxtdSs2WH9OJM0
+9ahcZMruh7MN8lFyBFsSaAe3XXFvHM7Egt/NS36z0K7Xwf0C5/VB1iBDykiFn28e
+bsKzCo7NUMtQbq8uDJl/e4A0lVSh7j418t3IsWRsmno5s7QxMkqvWAI5voY2S4/g
+keP2tv7y2yUohp5yXKvyftxUZvlnkrpCgXN4SQIDAQABAoIBADIRCj+JwPziXDhA
+ADorneFafaZ/kjWkaq6vN+uSAP76g+O15UKD1IMj0HaWGhnz9Z5drBiPaVnBMExc
+Sjd/KyZR8IKvylxkQ0uEsI6GaBkKz7xBPfxmCvNjFfJVEGDfL+9UZn41z89QxvyP
+FN0F20Wg0iaAWNOx3vezKq7Ws6a8vVGQY/YCWvRCivW1oxRbP3d7HCZ3YuigOeOR
+HEuiq+qIR7IFqfk7O5lWUmnaqQ0XOVrs2n0+b4F8aW0H7lPAiGnN/yRuDixDckMF
+ubl/KkmS2FJOS/1v3Y7Oa8dF0RrZsQDEzeyy5JvKE/qdw3hRF0nNMD6GQ9cUrS7F
+cxKtofcCgYEAxt0Dr3o1cViVHvHTC0q5GWeWt0JbPof3UtVzSmFcanxciEwuA1rh
+WHE8L2Ha+/2iVB/k8NrX6cagxXS3bou71A5xO3YEQPUFUaU9u//2xiClhIqQ6YaA
+mftEGYEmRMAAR/Izec9S0dOCXAN1+oeymlPKyy3ARcKYvwgi22cJQQcCgYEA1qMw
+YjsIjGc0VOE+q083xgyw2SBpTEXz8OXxiA2jAJsIM79973SJX/uf+bq0N8UMdccV
+ZUxby0AKld4GOAUI2jTQQfXlwTAx4bNdnseG8FgNmkAFiabCzrGceWTOaQi2IRoi
+xXsozkT4Oy0F7Ot/tAQCFXq+OPDs9tVea9SBOC8CgYAf+3OYSSNHM2PoMiQq8L2M
+JGjM2FMsX1mbbNC56aafSZjoIrhTuc38F1oClM5PMJ9mwCplwgxvgI5isz68/KQc
+mqYDVhuhBsbwNo8TKTta6e00v8RGimcsXIi7QVYSQH7LE1zL/lfnfwi8G6JhxGkp
+AMjgnF8L2RwPvRM+DfgjCQKBgByTKzzYqSGwF7p5JlO+PymhxOWTOBGMepaGtmor
+jpjlCumAw+5eMG/T2w9vOND37dm8QG/1A3+CriYXofJRUUVjpJozd3wJw4pNy9f4
+rlFPF7iCb3pM9kB4SuGqdwRqwXyUsy5hYCRd8EJIkzJgg+beZbygU7TW9mVCMMjg
+SLfHAoGBALK8J/1XrtF8jZO9VQ9QSle4bQoH3IdNqdIB+3Nn/R3m4qqO9bcdPSmK
+8rKFPqyFX3hD1KLBn2nCO6t2I50BVjv0Xkmff8JLaff6GInq2rUhHk1TACugaJyh
+/Ag4UPyP3fIM8b2DdPcPFYyL0PqGhjcwNmTP9OnrCC8vLO2+KvPt
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c6.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c6.crt
index ec90f2789849..0f128dbfffc0 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c6.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c6.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAJBP/I3D0hBi9f12S8zW653VRl2iMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW42LmM2MB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjYuYzYwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBALAFA8o7mX4GYsexgC1JaQzogoJzVK0XGeeWUMKIU3kjhfjezxwY8Pqk1bCo
-hm+WTdhRUxtO4Ku2AMz7FaOYnhf1gAALT87M8iEQD703Fd83jcmrAuWG0zibkzos
-0xxzOrg+yMU7Tj3YGBqDKuOiXR9vNR5vpscU81bg4ZTT561DAgMBAAGjfzB9MB0G
-A1UdDgQWBBSQZ4R1AxlxN/+tAONyY27r6AKU7zAfBgNVHSMEGDAWgBSQZ4R1Axlx
-N/+tAONyY27r6AKU7zAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNi5jbHVzdGVy
-Ni5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAXwKj
-9JQDAud4CMyHcSz7SWNvode5Pr+1VO8OUAP2UARgtiEzV+ACIgXsWMKY04Y/uP0+
-0bWJv9MRUfw5jIK1Q1Rh6fjrfspL6uU9PidXDxqlfno7YybPCj9IUeJDImQC6LrT
-0OQbhodDwWhFXP4ibbk/tCbiWX+fS7mBThIWZT4=
+MIIDTTCCAjWgAwIBAgIUZRawUqAmCNbgJ4EEmOCAJ8ukyp4wDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjYuYzYwHhcNMjIxMTI2MjEyMjU5WhcNNDIxMTIxMjEy
+MjU5WjAQMQ4wDAYDVQQDEwVuNi5jNjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAIkxQCz3mjLK5bUEw2CALimAYLDLZr4IwPAh4tDxBH8aDaAdhQa9XbMV
+HbZL3SZ7cTMxlwLyZ/3XIpIKJ87OS0l+s3XpzSrFwYg410RPAoBU7ljof9nF0Rzg
+UAEg2JTfg7KJ+7T7yOp2uyqJnDZ9dJvb4SgMuPeZIu6ODQgoVc0n3xSKj3BCSgdi
+OevcuaZwEIEF+tIzH11JHgxrI8shGh+a59CtkJ5FPXp7/Vx5jkZi/iXOJygOAf4Q
+/p08x1Kn0ulvH7wosYilj8yQ/FdApe0SNxFsPCFxFPzO/G6Ozdz4lC1bWG+FUsW3
+jdB5mhTbaA/ADeApY7ivi1M8hgg9TmsCAwEAAaOBnjCBmzAdBgNVHQ4EFgQUVQxa
+VrpLCplK6OTZFaa+rQxPpUEwHwYDVR0jBBgwFoAUVQxaVrpLCplK6OTZFaa+rQxP
+pUEwTgYDVR0RBEcwRYIcbm9kZTYuY2x1c3RlcjYuZWxhc3RpY3NlYXJjaKAlBgNV
+BAOgHgwcbm9kZTYuY2x1c3RlcjYuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQBmOf2DFuSZYXre2Htmj4+HRSXjjNjCKjaO3N1cP4RB
+u8BsKlYPVUTjgaDEOwobuk5S39uEpj+hZ0sl+K8w1tp/e/fPLcSxl7eNod5FQhoc
+hf9i9IRDsv89R8/KmQ5GHKLwLeRNOmhabEKp3INtJ8x2H8GLjzmOUcLdI1v9GuEK
+Nun7AO380aJMY7Tt+QQed3zwgIHX5npeYWd0OC1wsjwRXCa7/E9sNGPiGjglKw0y
+yCBfUmnR/1xzO4P/GW9kYJ0aOoMT3TqkBRQuCHczxIlkovhInrXzXo4VpjEnf/tv
+vM2UY6wq36Oz9SLRjN/5TM3TAQ8TVszSwdnoceK5+D9b
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c6.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c6.key
new file mode 100644
index 000000000000..27da0af46900
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c6.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAiTFALPeaMsrltQTDYIAuKYBgsMtmvgjA8CHi0PEEfxoNoB2F
+Br1dsxUdtkvdJntxMzGXAvJn/dcikgonzs5LSX6zdenNKsXBiDjXRE8CgFTuWOh/
+2cXRHOBQASDYlN+Dson7tPvI6na7KomcNn10m9vhKAy495ki7o4NCChVzSffFIqP
+cEJKB2I569y5pnAQgQX60jMfXUkeDGsjyyEaH5rn0K2QnkU9env9XHmORmL+Jc4n
+KA4B/hD+nTzHUqfS6W8fvCixiKWPzJD8V0Cl7RI3EWw8IXEU/M78bo7N3PiULVtY
+b4VSxbeN0HmaFNtoD8AN4CljuK+LUzyGCD1OawIDAQABAoIBAACMxKlnmImYOhCW
+tETx1hultgyPWzRyqvIgJ7P1c9CpTrBhGiU/F/DI9Z33/gzWWPEg2gUCkCFCH+26
+zVZ0p92YSAc2ZsZSfuKkgcxfu6M3RsDvHs4urR36e5iMnuN93fg8UFiiU/PbcxaK
+AANQ9P/XCr6arFUeMZI1Q5s5rPw+5bijYPQhIfCUti38EZGyet9L8v9SHe1E/Fp2
+WzDjAWqbIu2onepPdUJII1jxvR/0NB0abeC0SbWvwTfvwqcg/YfwXd346bX9wYGX
+qPNuy4xhjKqXCSk1Uoip5b1Y4XbiqPlzH/yqkqYm7vloUwOzEyujUe+PtKTjtPO0
+g2dYfjECgYEAtT0YjD+BpB4odNBr37LhFAhRq7QY0G2Xs096TCATQmRLu4SPUcwd
+YA6aMu8kiW4xChH/VrpdF604ckXWfHGMqr2U2QzsYx0a/XQng/Cu9ZzZ8IIAexpA
+PZtVEF5IhF4YA67AX0gHlr3MHhOAKyq2v1bmi0h/sJNs/lzVu0ZVIfsCgYEAwcjY
+Bl09yTBztFG/MhBnegJSvHdlI3o58IHU44cW8vf38WuMIRTAYH+WtTimNu8JJ9x5
+Cm3uT9czcu4xop0eTCgN1NgJ2cr315Ls99ZZWm+KWjPe/106b+qk5eRFubbwEKju
+eflXPxgBf9vtabtctavnFRiZl0b4lFuQDGSBSlECgYEAhValsMkoP+5QJWERfErG
+fU6Bh6nAd9epPCrE9yzB/p57eUJVjwB0vC8CE2cNom6Hdbk3+j6yLrEqTihDXNrN
+evBviiCP6XuIV+AVEmUI03rKfL+oSulNSgX1100GXRw83ri6n18G9kg83APMz/Vk
+7fCpPN/h0a8lCR6TwGxGRfECgYEAuioo3nRVFyLlBSZDd+Uqg5wXv9cz4pFZeCRC
+URNGiMSP/e5EAMPezNXGAvMZO00IjrLP1aUmfK1RT+hk5sPqh18zanbhf8ntr7wG
+y1GxJaBiWaXbZZg6u6nAINrb4T5OArNH5wYTOxnk1uCBhEHrHty2cx3e7+XOiTgt
+erpTOwECgYEAjQWwnt3t2Vio5MEix9OVzdWHDFQw+Dnx1AcSB9dA1lYW02ej1UFc
+j8sJDfOJ0J2vrW+QXz//bdafI/BpnuyJfaQdyHIzerKIQzCtNiIR0bTjrR0xYgtt
+l8BBOauhN2+nRxSLJf8IHXpc9ncExvOy8WAl6DOT4S2+OYa+ahd2PEs=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c7.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c7.crt
index ba90a4453daa..c8324e452b94 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c7.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c7.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVALhovlP3SbGFE87OOAIBX/CaSQaRMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW42LmM3MB4XDTE4MDQxOTEzMjIxNVoXDTQ1MDkwNDEz
-MjIxNVowEDEOMAwGA1UEAxMFbjYuYzcwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAJy8A2khQkFtzq1bi9E8sSKSrJ/v/sWAVASrv8qwuiyRh0F0sxp6KgtJ4ogp
-iI5WbhbIoqWqjoDwGgGvcvf24m+/sr4JDdKb/ZyzkKGOLhLXugQwZABOCPf54DJI
-As/JqoxQ/XhbMPhqLTz6x7qPLj3Tdwybsbj85nZyedkx3gyBAgMBAAGjfzB9MB0G
-A1UdDgQWBBRq79jvLNS4zDUaRG5JNpPgOqXlWTAfBgNVHSMEGDAWgBRq79jvLNS4
-zDUaRG5JNpPgOqXlWTAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNi5jbHVzdGVy
-Ny5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEATtla
-YQx5JvRrZoMhpXYTb40/mFNgnCdT4YktqAL89IrjnU4Uc0xZLNVKt+4wCAx4Uosw
-wT/aymJY/FtwEpTT40l9jYxMSYVFLTiheDSgBzWUHYKZQ6by72ibUzWf7gXALRMV
-pu55YQZJF2ImsjH6gJRniJ07ma2nDHszMi7NWJE=
+MIIDTjCCAjagAwIBAgIVAMysDPIJK/5wTrXLAgzKRLkztMGcMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW42LmM3MB4XDTIyMTEyNjIxMjI1OVoXDTQyMTEyMTIx
+MjI1OVowEDEOMAwGA1UEAxMFbjYuYzcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQDUhxxCMek8pcb96Lw3XNtvSXRRck3jYcnCaHPOzwxyCv/0/aKHgEqu
+OKs1mXmLuWu8GniBqqEZVst1jnHSfCapQ7ROMxb5Q7cHQmmm9eTukhllyd62zsst
+9OSZJJd8Wzcq9rDpaGkZ5fe/mJl1lYizCoqUVwkuMSyz47UBtIyD8TM6j5XtdeKG
+3ey6IDtniho0A94PK2pDIMlKbbhByAtGO539Du/LnKAa/0XnTqIZb0TWFsQezWzz
+AXv611u9RmFFsb1DrXqjVQVMBP3Q2zle3+cjGZcBIOaIFQysZfbHUhE/hv0+IYMf
+p1WNiFnPZgkZelD4N11WtmsD9orELjx5AgMBAAGjgZ4wgZswHQYDVR0OBBYEFI3n
+ewtwdui3X/peOJ/2Xo2OpGqsMB8GA1UdIwQYMBaAFI3newtwdui3X/peOJ/2Xo2O
+pGqsME4GA1UdEQRHMEWCHG5vZGU2LmNsdXN0ZXI3LmVsYXN0aWNzZWFyY2igJQYD
+VQQDoB4MHG5vZGU2LmNsdXN0ZXI3LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAVhqMmErj1b2afa1o1eaJXEyIV4LTH277Nn7KnrHx
+aU+KzhLwQT5oywjP3lPUv5u1vRJ51+MEzruA7GK0WWei0lbuD2lIU5xUeW0ykIYF
+s00tcEVKqwCBkj16NY71glIn2tYnp29NJ8bknKbpj7MdYGx2IM8OS04GW9NsX4SF
+pGDOQsfcbapjzpZZXMAVc5BimVrBH+SyqVLhFheY7Ij4BxxQ+Agqw70IAs93S7YH
+4MY5cU7n8fT+n5CMNZSlLVzZWG2vwYSdSuOWvZgKf1OKIrIrYqp8HZFQxpepcpJ0
+TfkqOOzGtVuEM9BzDwNcvAi35GbvAtmsPTjDOPkXok+fuQ==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c7.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c7.key
new file mode 100644
index 000000000000..5ac9601f01fe
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c7.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA1IccQjHpPKXG/ei8N1zbb0l0UXJN42HJwmhzzs8Mcgr/9P2i
+h4BKrjirNZl5i7lrvBp4gaqhGVbLdY5x0nwmqUO0TjMW+UO3B0JppvXk7pIZZcne
+ts7LLfTkmSSXfFs3Kvaw6WhpGeX3v5iZdZWIswqKlFcJLjEss+O1AbSMg/EzOo+V
+7XXiht3suiA7Z4oaNAPeDytqQyDJSm24QcgLRjud/Q7vy5ygGv9F506iGW9E1hbE
+Hs1s8wF7+tdbvUZhRbG9Q616o1UFTAT90Ns5Xt/nIxmXASDmiBUMrGX2x1IRP4b9
+PiGDH6dVjYhZz2YJGXpQ+DddVrZrA/aKxC48eQIDAQABAoIBAA4TXrtBm/1wy1Og
+uOp/w2bE+ie0xtTS8q5mBZmCXAQnl1kt/Tg1Nn0ActJrdkHbR3ffVvDbkKzhczs3
+cnFq2JKoUcZGGvTqV7Jq4x10NrPYSEdWDeKUop9KE2PTKRdK4UX+ravfxuo4sWZm
+oG9JtpKJFbGKbCi7iAboA2J4B0Brt53ERtUsR6A8Qu852qA+vfrOjdH8fZuaHZT6
+zf4DOz5Bfzi1NZPj/fGnZH2+qUte928ytBlmlYY2OdOSfquO+A1LMMuT20vpLqHY
+x+por6ygDDnTHLIyxK3+bIo8iEt0noKR75mQG4D6zyYkL3gvSryUbtaZz9BoWuMr
+e8AL4+kCgYEA1J7mlqI6MIIwGS7MyW1B1KGfevQRY4LhCG2QN+ipy0p71AbHW2ga
+hXVTTnBBROZM5aSpQffYwotbymHLS/bI3rqnaSkNomxOi6Lsvq98FKu5myWmcA1D
+BdOuzhjSWMGdLIcjqVrHqwTqhpY5IHE/2xPd/61Ug9a+erbsL/WVgT0CgYEA/+Nb
+G+5sKKeHGKs3f2tLafWQYNUxWN/RvvhEkcmMldQhXyNnJXFOJyvyrKRAO0FVOTrq
+Uht5Ml1IF1ljsNC+llCyB/ms6nuXD4aCKHiYnSWWAzeN1s+RJxxQoGJiMNKbJqXm
+sblvEqLnB7UGwupT5LfWF7oGH9HhglUZR/2nY+0CgYEAmFhaxGfs5cECkai010bP
+unkr+j9TDkhC0XUfts+giacO3rcUtOv8xM9yJ2vLeIDDbG/1Tu01hP+xf2te7Ukg
+WGbJeVIPEOglx2uv0lmOwIji7sWlYfg+o7gvMkEaxrnD0pnp9Xi0RFe7az/MIvDC
+Xkj38jDRs710shzLUvnWa8kCgYEA55Yr0gahatXb5c6nprFDin1NkuXACJBgcvac
+kFkZ3Ewle8Y9xtxaAAe9VNQ+saI6cFZK1c65jIRyB3fGSrNRmhckWSV1gq4iFFIH
+IHusm6T1tKmlvEEVPJFawxzngwpIktzZAgYyaWp3rzj+CpArJzfeEpEwk8HO17cO
+72G5Ci0CgYApNFK2yRMhF4wc9A0JhP/gbPtZw6s7Pt9svhYowEeoBdk4oC2cy8zh
+CeqcK8X6jc2PuMW5taZc9H5WfXv7tW3/5Yi7GqfU7zGe5bWqRzQyF1Ud+jxqcwoW
+RkFGVRA3rUqlhZHkKSPDeo2TObS4Txo3AVC4zIMwA9HalV5a2GM1hw==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c8.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c8.crt
index 74dd306fe341..2dbe23b5ac84 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c8.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c8.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAKF/pqC6jeqYbOiIgDBmcXZgUzERMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW42LmM4MB4XDTE4MDQxOTEzMjIxNVoXDTQ1MDkwNDEz
-MjIxNVowEDEOMAwGA1UEAxMFbjYuYzgwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAIMcNI3+aYOuGJx1YVq3VntERfFqmnAsssk9FiJMCzbBG2yD+hPlrlGcCUpG
-fTBzGEpvdd7ftSoy+2SJv2tercjCkg1enCS/hhnUp96NmUPm6yq6XOV6ICtef0hU
-VcuNnXwJitsPFpVJbNmECNE7EI3au1JOKXBBlfRet7IlXax5AgMBAAGjfzB9MB0G
-A1UdDgQWBBRzmA0meyNb9xdcsfCBrjVMIEyM7TAfBgNVHSMEGDAWgBRzmA0meyNb
-9xdcsfCBrjVMIEyM7TAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNi5jbHVzdGVy
-OC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAeNbr
-Fh4rvhqkDxGYUdKzTBi9BcqpAPtl5YiAE7++vH+/kcCtmNppimggFzAxIUPhAdDZ
-QkpmuHX55AGsZf7PdHi/N3BjyB9d3EVEWItV38xGSsJylNK6VF4HDSWtvSDZlgSj
-sQ+m2kbZOICzFodFo2tZ+CfLJWHm4BekVeYC7FQ=
+MIIDTTCCAjWgAwIBAgIUelGjzYJW3Fp+xMI740rHUhd848gwDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjYuYzgwHhcNMjIxMTI2MjEyMjU5WhcNNDIxMTIxMjEy
+MjU5WjAQMQ4wDAYDVQQDEwVuNi5jODCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAMHpSRtMeXjvtTMtEQbHwmnae96EH1TToIw7ZLy4DJiSySirw/9HJbex
+w9/JWizH81HtlqZkzeIdFcwVUpzeHOr15VNrM3Q5KhXBeULzLTbTtGuk83X0wCyb
+oIR3v742dFr71oNf0B5dWDChWx/FTQnPoyb3lmLlYBdgvEEpb6mKwANyvik/Lp1A
+PTrDGQ04OpjYONSRKdDghlBXfkghaN0ba/sYZTlvQFC+w0oNbI9BjHeEOH2g3Vvm
+J+fmp0TwSPvFrbgxt5J75FWPcBwWIeqGhc9dE0KqF7lq5bHMomvnrQ2fFQNLLXLV
+rl1rEeeu77hwDzjcCeaHqpKucG86OOkCAwEAAaOBnjCBmzAdBgNVHQ4EFgQUPmdf
+teaBulwCla6KMWbLLeIPGCQwHwYDVR0jBBgwFoAUPmdfteaBulwCla6KMWbLLeIP
+GCQwTgYDVR0RBEcwRYIcbm9kZTYuY2x1c3RlcjguZWxhc3RpY3NlYXJjaKAlBgNV
+BAOgHgwcbm9kZTYuY2x1c3RlcjguZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQBa8i9wzlqtfEjU+kPVO4gEsFz0WGKdRZHeLSyaQeRI
+fXx2KDwpzZJpvHZbh5Dbdkml60wb1uax3MViiHoTipo4q3StlVZ1l1pMflODkdb1
+VApjK2gi8OjJpm5f4f9iMJuwhXojU5r56S1M5JmMMIxTgXreqijL+AymSQX3M9ds
+KlaEMwEf9PGlxDTSSZKmgn42IQVlTpivTPS6ZV+XTcXny7c1hl4wK5MZcYbRIWTz
+QKeUpCtKzN26ZUgazqhq9SzciXW4ZvWXAff15nzr2KmN9zHpH0vxL9TpYhyA1+fA
++/3jT41scb9XK8hU5U5OUfb61aGuOYvEOu7cIpnUlfFZ
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c8.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c8.key
new file mode 100644
index 000000000000..f06d30472dec
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n6.c8.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAwelJG0x5eO+1My0RBsfCadp73oQfVNOgjDtkvLgMmJLJKKvD
+/0clt7HD38laLMfzUe2WpmTN4h0VzBVSnN4c6vXlU2szdDkqFcF5QvMtNtO0a6Tz
+dfTALJughHe/vjZ0WvvWg1/QHl1YMKFbH8VNCc+jJveWYuVgF2C8QSlvqYrAA3K+
+KT8unUA9OsMZDTg6mNg41JEp0OCGUFd+SCFo3Rtr+xhlOW9AUL7DSg1sj0GMd4Q4
+faDdW+Yn5+anRPBI+8WtuDG3knvkVY9wHBYh6oaFz10TQqoXuWrlscyia+etDZ8V
+A0stctWuXWsR567vuHAPONwJ5oeqkq5wbzo46QIDAQABAoIBAAW7wdVcjL/qaINn
+mvgGGoW6N8zjv7BopHfY/rjEwVyWhHufHoezARa+GxJIGBI4ZEWhHoHze5Hqtb0H
+yV5Jg/XfiOUPR6FxWkDz7EOyQH09Q7mmEivD7ZW0EUoxZzo2znhtN1HBmLv5GxUv
+BCx7jzRRzxtlGS155j4ytkY/9SUAX89H5A37OJpB/mIOp6a1/EHi59QkyLs+GmIC
+bu90htiF2Gks3KixUHwAZ+WYFtAFQcMvn/7FiwasN8YF23ezcCYCOTvVVZ7It1ET
+/AoKXd7nSdzEoEh+kPRGaF483BpHYjVMqvr9r58IFdJPZt7awIjKpRafzhvuCN7+
+hiEPHo0CgYEA5zX7AcJYP66wItyShVQ+ST7zkBTjVWneo7OEayqakmLKZ8yjKuyp
+16sj1xgMe5jYzxEz55BavoCoSNZaOIJ3oeFxsyi2cBmmRlm5tAIIx8beqWwi2lxP
+x0Znx0j6E87XBtlTRS+fTphw9rpIohNMZUTgFerBGPmNg7nH8GUzTLUCgYEA1rOM
+Glm4c6yX4QZMGEgaBGRoWK6wDx9kOduRp0stKPmlrQSTThwGe8te5CLx3Bt9iwod
+TATzLlFqqhfGCOTETode0Byq0AbxSAGKvuflIr+orzsZlXYcS2JJBSgoFnB6o0YM
+bVlyH21XOnEiNvzT+UEryGNbNz6JjXYgpWp4D+UCgYAN4QGmQ2JEwFm2XgM1ma77
+epnoRji3Qdmpffcu76nDq8Z2oKQSyY4Qcid+m+Tz7mLlWyUAe3vW/6YaaTOVPM+t
+mfjXRlRdOkzu4CfzlEOuqlNnXCRjczMexS0pm0nXsozZzmXfO1KWxzPQu7HseLgX
+RCEnb6YTt7c09wv3dDu2GQKBgQCwJ9J80D5ZkjNu6mbGyWmnTuaaVIGZds0Z93/y
+/ehIpUP8yDNANBhRu9PYZJ+CU3/W826EL8+CokRc2HKgW2qBb+on4E15ryiKRtGa
+vtMuhogVde0Qxz6Bz1tQAtmZR6InklKh6XCMPn8CMU68RQZ6jLEqkfKGSiY7zb8N
+WWS2zQKBgQCuPcci8Lnt+HzU2dUtVPKE2/P8kRV0r3Eg71MvN2szQ6NbLLsGzQEW
+bhuz6b4FhUdTr7AQIMEDEvH5pMg2+JY47sj6GycBODyFV62hAYogK11l4u3EJJoq
+ZRRqbfFa4n9CjHwqeRwwg9cV8UHBFHjf0obmUUp4Ljq7hwcyvZcK5Q==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c1.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c1.crt
index 3f67ed8fecc8..a31de7858b98 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c1.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c1.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUHFRaZgHge/5D2Djq/MM3MhXtgSMwDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjcuYzEwHhcNMTgwNDE5MTMyMjEzWhcNNDUwOTA0MTMy
-MjEzWjAQMQ4wDAYDVQQDEwVuNy5jMTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAtut2Om5X3xl8drxF/Jf/qq+xPYYGs+soYtJM5qAwkC0NGQwPCIDw0pL8hpuJ
-XT4WrtGmADZ0ERvPULGTMcBVzmDgt5KANUT8kt3NqzdDS9PXj3VAYKKuDZ6SdqqW
-7dEKhRmZ4NPERtHA12dYvBgHmMoiWOI+jvas2PvmJ0/dId8CAwEAAaN/MH0wHQYD
-VR0OBBYEFMcRlGJupnRk/6cjwiEb8f08SMxPMB8GA1UdIwQYMBaAFMcRlGJupnRk
-/6cjwiEb8f08SMxPMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU3LmNsdXN0ZXIx
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQCWG5eE
-eptDX5VWR8tbmvciCAt0pEXCO0y9UIflCbnCl5LaaWCn3Dg5LzrPTQxZryksuVSV
-w8RAHUqG17Td3UhB5mgNdOD8MZMoSanO3yFDX+6wZ2pfPSYnLUVu8UdlgTI7p/RL
-r7+PWUgXplJVWpME5FZzVBuC7Rz+n5eaDQzVlA==
+MIIDTTCCAjWgAwIBAgIUIZKOqIZkq+GTCtbKwpkI8f1YdUwwDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjcuYzEwHhcNMjIxMTI2MjEyMjU5WhcNNDIxMTIxMjEy
+MjU5WjAQMQ4wDAYDVQQDEwVuNy5jMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAJ3XPmT1asE8rLqrdksISEXya3I33ZHSMnvPDJwgCYwwbnksGYqWGMQQ
+srh73viNWg6E9uuEbmd4LfOF2mya38pIlM8jMkk0xw+mxVQW9GF1WI0wQ6gjmCgq
+MOt0BO6qq/driQiGPLxKyCOeJrpumrJEIkeilEzIj1p+1PACAxqTsmmSQcPHjZJJ
+kObqfAEhxkCJG8gxuw8nreKp/9LXMk93W0xhUJYiyLWWZiKplIr9HC50F5wVECE9
+1g372Ci0xzmNmIf3SUOz14qOEx+9LjlvT1waQO3uO+kNVjYFK09+zwi6m9oNBMum
+XTDo3RrPIsVtN1v4pro2jVbbbOWkuvECAwEAAaOBnjCBmzAdBgNVHQ4EFgQUhR/3
+omrrcRjaDwKtG4jwzC/JB90wHwYDVR0jBBgwFoAUhR/3omrrcRjaDwKtG4jwzC/J
+B90wTgYDVR0RBEcwRYIcbm9kZTcuY2x1c3RlcjEuZWxhc3RpY3NlYXJjaKAlBgNV
+BAOgHgwcbm9kZTcuY2x1c3RlcjEuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQA8b1/phwUIjOk1tsA4YuxUB8O5eZKWtqJosQJKonOd
+wMYnFVVNf01ulWNiCZt3/x29wfneul6msczghijz+inChSJzRdXLAlpB7AE4dlaO
+IdzsJz5dyfQ5omZj8VpZuXBK5w4l4NtCyc4ivPFvGwK8kSr75iEwP4awxo2GJJV1
+B5miof8K7a21hTPqb3GRhue56+afa43/dNyPGCaIaQlKtH2RQaRsvktBESVDCEHe
+TVz1zzjxRwEVHV8y1JL1ac5RKDxbgXG5Cb3PwhPfR+Jp5cdqI677dU0TK5wXxzhb
+zOEryab77X525Z6BN47ub4vnrfLz/GGmQ830MivxY9YK
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c1.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c1.key
new file mode 100644
index 000000000000..baf1be28aa36
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c1.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAndc+ZPVqwTysuqt2SwhIRfJrcjfdkdIye88MnCAJjDBueSwZ
+ipYYxBCyuHve+I1aDoT264RuZ3gt84XabJrfykiUzyMySTTHD6bFVBb0YXVYjTBD
+qCOYKCow63QE7qqr92uJCIY8vErII54mum6askQiR6KUTMiPWn7U8AIDGpOyaZJB
+w8eNkkmQ5up8ASHGQIkbyDG7Dyet4qn/0tcyT3dbTGFQliLItZZmIqmUiv0cLnQX
+nBUQIT3WDfvYKLTHOY2Yh/dJQ7PXio4TH70uOW9PXBpA7e476Q1WNgUrT37PCLqb
+2g0Ey6ZdMOjdGs8ixW03W/imujaNVtts5aS68QIDAQABAoIBADgqFzGlgk8wX+Ys
+jPM2B0GceU8zQcxZl00m573GSJ9xvF2xq4T80+Mwbefna/fVlG4GMjar8P0wbmlw
+/8AVe/xmx0zNoV8bQJn5Jc4+PMdREilrOyfhQotIOBAQQJEMehSYVzKJUaNOmNjT
+tcQxByrXw7rpxwtrclcpOmwMyz3xMUR50TOl7cA/RlYGq22vWeYAnU1vv9I6n2jl
+FqkOnGlaUYht7RXGqNgSMvu0lUgJRB4CuW6mYKYVIMO1ZxhNR0bhqv0AgLxMwmKe
+vB5h8kiNsUTavarVypOvn1rskPYp/JFZsW8viHOXq0Mnt0k9woVQoLIBPHyd4Fx9
+KPLMauECgYEAyKDU/uMWDNoKBv9LgZJTKvElUCGgzPpzJtu0Lud+WK3enTp+NobC
+acdSIl4OfPdQztQ5BGdQvfRyWaqjwiC3JdEG1FOafBVwYgGsJGQTwhY7rvJowpUC
+VybMvUOJuJIzkRRey0kBbVPh1bZz6ICw47XVk+TazomsoN3Ln1AZAAMCgYEAyWdP
+r3P60Yke4nVl4NQ147RE7xBsbSMUww79jJ1NuX0qJLQDAiHHIyNdhhin5zloi80z
+tEGAyztlzRZBv6PTexKJ6FBW2r2HLxvLuj9T4HlEYWq0Az+Ax5X3UiKQvwbEq1f3
+quGnp5e7wnal2GaZazOQi/zvfcHFD5NY/JG16PsCgYBReS7GC4YK1gi4arhI+Bka
+GS+j6OQaGaLlptb3LT8b2A/NHdHfZr5QoobyOev8R5rjQz+cUDBz028Uhr1Vul9l
+1grF8uWLelFXG4LCjHmRmeHG2947FrQKnFEMkv68m3Khefo853rS4PuXBC7xvkEX
+V+JR2uAYe8RE1403NxnP7QKBgAnpLJGsumX/QmbQch/7MV+wKkg9XaiNNmNW5IE7
+6Gw9TYqa1R8QvHTtM5f6tKB7QbrHcUjohEb7+El8U+XzUN/YjLLC6VNASlu2GXNj
+1GPSc9v7XTpfsJ2hspb3HnpYAeuQm55sFkz3+YRbRFP+0IPm+LD61Goy09itXo6n
+OPVrAoGBALrYTRX6B3yPgriZAHMQg1mwErT7Xl9C2OsYcxp65iUhF5liUq0s37xu
+zlFy+CZ+OOnEhAYv8r1AfpkZxbPRhicvk5jR1qsaaW+NrOgDXJtG7yxn5FeDd5UA
+kmWVkf/rX9DQ/U/K5w5+yfge+Ibgd0+DJmj5Eah92oJAjr2BliHv
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c2.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c2.crt
index 12ae16246e3e..176083d09d78 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c2.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c2.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVANLcg7s67r1TLdqNOBRZ45hhVL29MA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW43LmMyMB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjcuYzIwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAIprTC9xhz2JYqSV6b7jQrVBeDS9KjkTdYigPMEnkBCW+Gv84TxeU6gXfyMX
-dDmL1A9nbKhpkDYh76FKdd7Ik/fvt7Ixa3RNMe8vdgMJCs5dog7IzdIuncSCGhGK
-VX92FQzja3I9JG2/jjwRtuvKbLRDX5bk24bkyPEWid0p3tsnAgMBAAGjfzB9MB0G
-A1UdDgQWBBT2lPX6qZfTAO4kua3T9ianDXkCOjAfBgNVHSMEGDAWgBT2lPX6qZfT
-AO4kua3T9ianDXkCOjAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNy5jbHVzdGVy
-Mi5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAaeyI
-QHWlllHZjZ5M3zy1fG/v7gH0uVWDvdmcrchvYRPJw99CDXUVigDLYD8470BwUqaB
-tygQozU65Y/jazIge9q53REAw7812BXO4lGUj820CxV+KHjjbI2+VnUVDZguUBwP
-ftjrG9lhWAt637WSdShW/Eb5wFSj/MnYY0Ilvl8=
+MIIDTjCCAjagAwIBAgIVAMxKU8gjY6x2PZ3flfJA8HJ61WsdMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW43LmMyMB4XDTIyMTEyNjIxMjI1OVoXDTQyMTEyMTIx
+MjI1OVowEDEOMAwGA1UEAxMFbjcuYzIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQC6pkFGrd+lACEiziZrWNh/JFM21T5Nkl4XNGuc0y4oHHRhUnLysm31
+3DIeK3OYebIqmE3cTZULx0lR/sX5u1qkbzomA21XcKxOyFukiKa5proUxN/jAIr9
+03Zd9o2dhTxHFR/iGixda1hjQFCM/4HWo+UtZzHduwFd21LPilyLdcpvhj/BqoRl
+ZuPYFn/V5687BXqUvGsgcCMWgxj/0AXz5eZ5YmRGXtCuw4XhjaVDFmUnF0F3j4uE
+gyGWnw0+8FP1YlT3kvt6eDcgHQiRSp695f+Ce0Ek/igHJNiNXP5O6r00bwjzltJI
+AmAcK93crqn16GP2ICVXP4Vmneoy+20nAgMBAAGjgZ4wgZswHQYDVR0OBBYEFCIm
+M3doAK3dJlsGyxaLHzU7eQg5MB8GA1UdIwQYMBaAFCImM3doAK3dJlsGyxaLHzU7
+eQg5ME4GA1UdEQRHMEWCHG5vZGU3LmNsdXN0ZXIyLmVsYXN0aWNzZWFyY2igJQYD
+VQQDoB4MHG5vZGU3LmNsdXN0ZXIyLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAFIj09y8KcXFcq+arj0ZT7kZyQuoPKFVGRcIYXoF4
+dj4Cu5uUf7Y1m5r/pr0MSwUMxB0tKxoU/9LfIugKQX1+uajs+NM1cyeWSd4OnLJw
+KFiLpZNtpaob3RPfz246CTH02saZgxvtuD2yjjsHWPU/yLVjhIZqEE5zb5yknAEa
+4f+AxdR1UyevCRk9JZr7bCy7A5NyRjCgltbXIaQeHLb7jRlVoStjVTc63C4QMFaV
+QcsBZggsqHfoJ0iAo3uY/3qfheaDHo3IjGd5ry3skssaxy0pcTm8REKyuo4Pgm0F
+P9UKkfIvA5XOPoRkxe9yN9WMkda5uZk+EjXebkc4tyHBtw==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c2.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c2.key
new file mode 100644
index 000000000000..b284fea1d7c0
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c2.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAuqZBRq3fpQAhIs4ma1jYfyRTNtU+TZJeFzRrnNMuKBx0YVJy
+8rJt9dwyHitzmHmyKphN3E2VC8dJUf7F+btapG86JgNtV3CsTshbpIimuaa6FMTf
+4wCK/dN2XfaNnYU8RxUf4hosXWtYY0BQjP+B1qPlLWcx3bsBXdtSz4pci3XKb4Y/
+waqEZWbj2BZ/1eevOwV6lLxrIHAjFoMY/9AF8+XmeWJkRl7QrsOF4Y2lQxZlJxdB
+d4+LhIMhlp8NPvBT9WJU95L7eng3IB0IkUqeveX/gntBJP4oByTYjVz+Tuq9NG8I
+85bSSAJgHCvd3K6p9ehj9iAlVz+FZp3qMvttJwIDAQABAoIBAAGbpKmFaybSjUNU
+GVQ70lAOGEI7RhHvAxNNTU879oRvJYWGuBzLjFNdthGfRQJF3ux4N9UZ1s3gkqSI
+ZSlvA5HlFdAo0v+5pcWSYYLvKaVYM8Qxo5tWCg+zvni19g33jUeWsM0G3EwvVfYz
+VXGEuTHUyaJ0s1ZHFSghpC3+vo6PZz6SSF0NdlNdAmp+uTu2WNql5G0eG8lzOwmb
+mDSqGtFRJpUCmqH4tix6OzZOdg0VNpvZxnZTlLD/MyWAoZvMdPTOe/PqZOfjp2ud
+lvHjHzTxJ3mg4mUo2It80J/7EI37RHZIRQgYaG3tRVFHUKl7s6c4SdtogKbkI+37
+ZJmHSckCgYEAyT78KDIbm8FpAuyrqPaToEIfOZuDSsBhU34/z3uBkhLKUN5Jklo5
+rlYdq6gCPNnOtRPuQG5p2AZ59wujO2ofhEGGiG1LIJLPOtfQt8rwgkH4EcwuvtsZ
+WvkFJF6SYOuAJEr2vxceqEf5aCi4NZFbQ/Hm6LpmRqX7eaW6v8JSVSMCgYEA7W6Z
+oOKc4kf6K4FBERM1VPt2f3M45c/0HDR/CqVyU37SLSfvW66YmWs8WMG33LFvXZ1H
+VPZyaVK9BIAFYbuTcF2CZz8gwr7HrXJe7HGfiEcrUEsJ2IJHx7wp/aN18hDiXuWx
+L6tkP25aBQhLVIDYddSDk9r8IeRS/9GEgBTCEi0CgYEAkaELlgbUpbyrQhuECSwO
+Hj1bHTPlKXXXuDS29mz4kX0ZfLWu5cJ21SqqI+rbW699/tliYqvD2jitj9GTGM6t
+6eYE9bqpT/QNFBbSif3IV2pnmnF4LlKcoZFsYCozoQEqZJj+v796Rup2mCJHUzRR
+p/RoVWd11ZANT7VXU6bKL6MCgYEAgzLYZZSTSF6pmJylzb+ler4xbp/G0wakR2mi
+wclGcWjkztsAYtMJs80T3Z+974+EkGA2aHZ+JkD3/aKYVrcF89NYGaFCgxpU0D0g
+wtrOYsmg3493c1IaQ/5v3Z4TsJ0v2WxnCoZUsNBTofBBYAKKmiHCS1N4e2nli0LF
+StktnEkCgYB0FSjtEmA1OTZK/T+He0e9nQAsSRh4Ugn2iF05/J1fphyYiaPrRy7m
+wD0Y/qVj/Lkcchl3i3UO4n8nsD6Ahh0cWi8bKX6ZD15bDrCIZI+flAZWLCUoOQPF
+ObFdE27lM9173IDp/6qNvsSsj5kPEe56/zDzlbwuLhdnapeqxfJaYQ==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c3.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c3.crt
index 6ab5f2e7808f..51736b559407 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c3.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c3.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUAcrmViyO3Zxe2UJugNb8VtZNomkwDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjcuYzMwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0MTMy
-MjE0WjAQMQ4wDAYDVQQDEwVuNy5jMzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAjOfHGF/Iku2fsDm+qYyGVEd8jw1eQ1PGVoaG5cTWy/H4Ho+CRB4dldki6b6h
-9A7RW1qbvZJgsdeLBj42tblU1nOYnF3y5SNIcDhzxs79SjNxjNIHgdcjeUR6pLMv
-DBPk+1dXLgoHMd08Un8hanIK/NA8ZgP8j2j57xS62hFGiHECAwEAAaN/MH0wHQYD
-VR0OBBYEFOPlr+WAnl/xLz8B0S9VQPl1+XrXMB8GA1UdIwQYMBaAFOPlr+WAnl/x
-Lz8B0S9VQPl1+XrXMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU3LmNsdXN0ZXIz
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQAoWtAw
-kGlOpCoGhgVmws1eG/47m0o780t+Mq5Lh+IhooviwvaHGlSJfWieOePuiXeLyeb+
-dXTKkgxnwPozNh5lT6X15C4XDIMJ/T3fr8Ggzi+4n1tY7MNW7ll3pdGFvoiKaSNN
-2gQ8u/UpR2Dv2b6OHExiQP3jy/2mtIAQOQ5fPQ==
+MIIDTjCCAjagAwIBAgIVAN6fQcwJXH9Cgrh1eMh/vzkI8+YpMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW43LmMzMB4XDTIyMTEyNjIxMjI1OVoXDTQyMTEyMTIx
+MjI1OVowEDEOMAwGA1UEAxMFbjcuYzMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQC0g1U2ybKwsvOAnOC1nxzkSUZKAHTgjmKUA9HtmkcV7vrdtxKp+5mX
+JVoHRmjpGgYE5rMkX8ofip83G8MY9tCduXk6kKEqW32/vKG9ysK5midA7yInt3AY
+DfpnWLktFfri6GIB3G6vdPUNe+VXy2Z4mV2lKsysMWG2mkz8ORFnyrOfY1ZkOKy6
+bsD977pNwKbcfUT6yw5B+TyT7uQwWJ5A8dcdX66NP1eahxWer+YrMeo3un4rQedF
+bwsxr2t2CTKqpG3fmtl1ZVRFbzH4g80xiUbo040mqF06SGFg33bkPVVqW2icmzdE
+pXDPiKg1GFvlPhX9ji/Hvssas2cDPz6LAgMBAAGjgZ4wgZswHQYDVR0OBBYEFOrm
+zHIXmeTefGMUUVvML+hWLedsMB8GA1UdIwQYMBaAFOrmzHIXmeTefGMUUVvML+hW
+LedsME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGU3LmNsdXN0ZXIzLmVsYXN0aWNz
+ZWFyY2iCHG5vZGU3LmNsdXN0ZXIzLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAjuoPQDRuum2vaFMYbWIbo3GKqLHcPU/6APHWSMhB
+Z1CI0dRRE9a9upW54OnkoMThYrXEVy1xrC4CY/ujL0Dcvhx+b4xVTO7S3WL5k9yk
+RYDXHI0EvHgTfE28pZhqcMgSnL7/kqB2sfvQvTdBm55yZgd/mvcfysi2v0V6zOF3
+kzq7SrdeWr2W3+ZGNoplWDMnRSqqftRZ794i94N+l3NpJDg0uCnA7cP/pS5ssyq7
+F2NC8D+NXQXNsAgsshEspEBnVOTzJf7AkMfyw97i9Ge5bjqeJOGbNZjTQOfPnJUk
+pUJhQTxOuMVds+8CFqpLY1ICZB8fIV+khJc6fN5sHkvPTQ==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c3.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c3.key
new file mode 100644
index 000000000000..5c679c74d723
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c3.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAtINVNsmysLLzgJzgtZ8c5ElGSgB04I5ilAPR7ZpHFe763bcS
+qfuZlyVaB0Zo6RoGBOazJF/KH4qfNxvDGPbQnbl5OpChKlt9v7yhvcrCuZonQO8i
+J7dwGA36Z1i5LRX64uhiAdxur3T1DXvlV8tmeJldpSrMrDFhtppM/DkRZ8qzn2NW
+ZDisum7A/e+6TcCm3H1E+ssOQfk8k+7kMFieQPHXHV+ujT9XmocVnq/mKzHqN7p+
+K0HnRW8LMa9rdgkyqqRt35rZdWVURW8x+IPNMYlG6NONJqhdOkhhYN925D1Valto
+nJs3RKVwz4ioNRhb5T4V/Y4vx77LGrNnAz8+iwIDAQABAoIBAAb0EinAnkIBOrEH
+Gf7VCLtgeTUrjFUEFxcFiKYi7b8SDBbTUQQ8PV/dn/isxSjJiFbS/i6WBAawA88P
+IjZJd6LDvMA9RwAHMzJaFU7myxZ0MjPlokauYcKDc1RczhFHCfmIoxHYSGk406G3
+1o9JtgJKQhm1KtotMsriabnDhwCc9TucNoqZ55vs9TtA6ndEOBGsGEi7N0Da2Evn
+VchKaNm1qzT2T9q9KLCh1T+JoJVgtBYJr98uOKw4H994rJOyP9AJX2Txzg6VCFgA
+7vcxnnlMlEjzuXVO/eBFknEf6QRb6xxdXxXMCcVRGVAFb9rM7KqEoe5DWmaYHssE
+h8S0xwUCgYEA103MS27LI9/g0fGUXvl7VGBz1JHtwGLLTtmaJvvSD32Ded0HJ2vq
+31tRIKzttafpldLZyL8/O6uBfoC8AG4JbjWMJIZU9POgZ5lj1kAZTpDvsVdJcVzc
+LL35Kpfoc3mXsg15M6KZAXgV7/uULhd0hyvcGuuZvXSNN5mn4jwHYm0CgYEA1qIP
+s2nMBhuMjIK70pB8wePpMkPm431P/FsY4/r1R2l7WkwbZgQSqg79Yl/y9B/6DDgU
+SGx1Ekf2KOXUVQoLeprMmAH+JcX0scBZI9Un9IU3AkXWuuQrFj7P030G0X9XJxyL
+Daojg+WFC5cWAR00vmus39pn+3tzJZDU4eVUydcCgYEAtCAtU8qMfyX1JA+1bsPQ
+L2ZGYcf4naTJrN0WP5kcQ8NhNJn7OzrpFBaEyA6FDR4+qQ7vDykZUOWKf/hZ0Q4+
+K4r3BTOxbgFI6IOG6B9WTMmrDrYVBpeGyJPjGoRegAVxKRBmnxFgar85sz3J2cnb
+vp6uFYOYas1Qu/x8XSSVlP0CgYEAhysuyiWxuJL287KNiLE38H+Q5/f9W30at2nD
+higQ+nVCmv5dLuUXwAOKrb6Rag8NUU6r72RmYinxq5X6H8ggrIj6mix/GtjNHnht
+avSdog+XMhY7gh2ZyPurtJwDhEI258nBBM0GatVL8Z2n3PHAur9i8uNAIw9+18Sb
+XNwphqcCgYEAij3cAWoMjv14P6I4Y7M52t1qUXmfW6d0Y7iAAa9C5kO9sSncuPDi
+MOUwnhK9Y+L+5slkuiCMYy0GCxrsBUhwn4h1XslvRrRtuWl2KsUJZjWH+qgpSwNV
+o3YAyrkh+bdm1UIF+Zh69M1mE4i0S7UAVnEMxRv9NOUh4G/XOH9//4g=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c4.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c4.crt
index e5014272b86f..e1c34264a003 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c4.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c4.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUCFu9yiqlax/R6J1jzPe0n1gF424wDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjcuYzQwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0MTMy
-MjE0WjAQMQ4wDAYDVQQDEwVuNy5jNDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAiPGG+jtQyATOybaNzp8DBRK4rKu6Fs4YVRVanfCB1I3FE8YRdh0uyQQEw4AN
-gxwaJ1MoXHMeXhcrjRC3saUgHeexKy22sn0YKEDTWeAKVhg1C8BppXRZTGp4UwhR
-xGdgQ1oFLjQn+NcTUo4SxEEp7aamFFZPq11YtYsp2sbAgVsCAwEAAaN/MH0wHQYD
-VR0OBBYEFDIEc9vl3VibxzWsH8wfzm/XYh4FMB8GA1UdIwQYMBaAFDIEc9vl3Vib
-xzWsH8wfzm/XYh4FMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU3LmNsdXN0ZXI0
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQAfV2qT
-G96FWbvq+o4zxrW6TVAlmFQ2AWcu35JcD9O9kfl/bd25k8mhsac1P/cIPRxKcFIN
-9EicJpp+EQ6tespPU5xb74eoEro9V9cflQig7P1xmiqVitmxN5zjJBWowOoPeB56
-zx/f/byUY8MJ8t1pAFLoeYCAOrD4eKv2eGpB9Q==
+MIIDTTCCAjWgAwIBAgIUB3fQByCC6qJge9B5NdJS4Wd8ZfkwDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjcuYzQwHhcNMjIxMTI2MjEyMzAwWhcNNDIxMTIxMjEy
+MzAwWjAQMQ4wDAYDVQQDEwVuNy5jNDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAJNX2kBvqwsvejGbLK3yWotQS03cPKzVVk9dSqeZPWYNTA3GwdFD5P2L
+zgH5KrbBnJd6p5UTPQc1nee67EBk42wRmGtyEi6w0j/JPUuWDVKUZsLP1mUeBSbj
+3hEQ5XPVKv9XsCfWaKvVWWv0c68CKmhZHr+xXQPWcx6IOva7OvpmY33VattYVwFD
+MTA0hsqghpBnKp/Jp2kRHnjMUcZQIWNOc4rrkgja1q9xHjPuvOHkMTg3lK1gh/HG
+za1YZ7xQcAQZkHi13xsUzf0ceKy2KQVFKm8y1jsCkRF/vKE83Wa5QjvkL4qjra47
+iFAQfYDz1BDqrTgV/LrlxbNOPhzEYOMCAwEAAaOBnjCBmzAdBgNVHQ4EFgQUe6PW
+NifbX/5GSPe4xzqJ1H3AV7MwHwYDVR0jBBgwFoAUe6PWNifbX/5GSPe4xzqJ1H3A
+V7MwTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTcuY2x1c3RlcjQuZWxhc3RpY3Nl
+YXJjaIIcbm9kZTcuY2x1c3RlcjQuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQBWJ45t1hEWIlpijAMK8058IbiafCFAwfPBtTeJp1+F
+147MEfklLNkglAlk8JwubIgl79yJCncAqgx5ahZdTk48F05xrwjDGap7dOizn9mO
+3kNtScMRyZScZrpq2iiPV7DfiXEH1Rf0vw30sYqlC1+HUeQxEyxjdGvYbU4wFM6k
+7fSDJscM/S0aA2Re8JEWKUfWcYXprmWGgnqp8p1E/3qaUZwZ7hLbNlir8kvPPyFq
+cnTqVdRDXCjw47wH7w/wFQiaXob+msqYR02K1MpBLFyF9Tjky2iwbtVahag3blou
+hnSTKK2ACT5J7Qp2Li6XG/ShMIrKwGVkkzOGwPhTpvdM
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c4.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c4.key
new file mode 100644
index 000000000000..1a2d54782ff5
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c4.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAk1faQG+rCy96MZssrfJai1BLTdw8rNVWT11Kp5k9Zg1MDcbB
+0UPk/YvOAfkqtsGcl3qnlRM9BzWd57rsQGTjbBGYa3ISLrDSP8k9S5YNUpRmws/W
+ZR4FJuPeERDlc9Uq/1ewJ9Zoq9VZa/RzrwIqaFkev7FdA9ZzHog69rs6+mZjfdVq
+21hXAUMxMDSGyqCGkGcqn8mnaREeeMxRxlAhY05ziuuSCNrWr3EeM+684eQxODeU
+rWCH8cbNrVhnvFBwBBmQeLXfGxTN/Rx4rLYpBUUqbzLWOwKREX+8oTzdZrlCO+Qv
+iqOtrjuIUBB9gPPUEOqtOBX8uuXFs04+HMRg4wIDAQABAoIBADgr11wWVj5K6Mot
+jrB3wrX2s+aobgczw5Y5CiNvw5z6t8JOwgf92rggSras33BRylljXKzz6yGq+ppj
+w/pem+bQpGtdzvm21kiT9wawWz7erXhaVLEy6yLi3v9hkiABJcW3j8P4NOpBSThj
+NpEDAdjLFPJecwfqAvQiWlxCj57ywlg0B08UCq35953mml6O1YHsW3dDHccUZt2A
+Yu3Z4e77pNB9vmZMj874Ro4/O/n7gMFGsdV3SCjcukS6XDgidO/67czZdO8uuFF+
+07o14rccPS3T7zkcFUFJHidPnBbaU3tj9Fpho4cMm0ctFQkcgp+LsB02+ReZAd0C
+wrfhi3kCgYEAtfNnp5y/bjZinc/rr3pRqZ0Tr3ez6470LFU91++CFrAlMe2s9ShR
+2fD1sq1Bhj89wZDemPKrBPGZSqExtCuZAR9EgV+ZLty1yLbJHRq59lBf2rBjtEop
+SZ6L951F9JWPGgFKH57uKixAWhCGjfGBeOG1SfoZaVA3lKNOMuwEVBsCgYEAz07X
+p0qXmazSvK41/sk/sVRCCv0QkwidQvFPnDKidL2ghAOr8Ufuc2BCyxVVZ66zN05f
+uBtcl7O0jWgwDpAukmiLnYjwKPb5FwJJ0Ok97y7mSC1/0fZjjhVnggU7eqNuUpxY
++EDbN1GtscMqNlHVvQAqEvU0tRSUTfzedC2gotkCgYEAmGTs/YMZBwuAH6OlSI1P
+1ptcgIvm4zYRpwwiRzkwoSYZkS77vT5vB/Z9I5Ffnzxa0kQvXLVrrjFY7hwC0F/v
+v/PbYeybFZgAncMSwVDATuDU2AsDB19SI3oqy1Mr44ZcnDBWUBPCcQotQAI1YK/V
+fFS3jeVu4TkxxEnfVmE94t8CgYA2wV5Yg20zW+B1kZYVa9Z04gpnmJvuGhJfghlC
+6mTabNOeytF5c0RaGQzLUyExt4UCkKLbn+5+6yo/R0uaLR7/8LkfrPGyADmeiC4z
+DcgcCsksO0hNjsSQ3LpwLy2coNoc28rsF0VPgTVdhGz71/WabBaw1y9KvNrHFokz
+KnIJUQKBgGmYad8p87Mo3q639xryMuVyqil/oJjY9+UNikXBUefHDMvmcr2uV7zD
+RVufuvqNqyIf8FcshtrPx57BAcIX2lQoo6lXAi8UQzYJ29oF3gVaxjRo3QCKflMe
+KqLGiX7BTnVR0+88srvnm3kKSD3HG0UeW/4ZoclVGAiiETTJGw9e
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c5.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c5.crt
index 01bd3a9713f1..5f2cfb4cf8fc 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c5.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c5.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAKI2zR76R0ojhtQ6/nGpY/yzXCoIMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW43LmM1MB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjcuYzUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAIxI/vm13r7qO0JHwIU5qFMH9FnANuPqUFnDaxM6uircaa7I4LwxKMp0SSYJ
-Lh4+ThihGBOk8GofaFj80UwQMI0s3yqxn1JaK+DPvvStV5Px5CYyWHOkzLf6Qhh1
-cGPRrsZrrYmp642NAQWm/8v4SV4BmAzWuPUhjfM+k3FnXbF1AgMBAAGjfzB9MB0G
-A1UdDgQWBBSX8uUfOIjJpIqow4083qlVg8FrXTAfBgNVHSMEGDAWgBSX8uUfOIjJ
-pIqow4083qlVg8FrXTAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNy5jbHVzdGVy
-NS5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAiXUe
-wizWQNa6GwXFLFZOBPsYCuVJPepjy1Zu7AWuH9WEOQk/QhlVc47CigYnt5LN0H9k
-OYrGMugwt9MydZvujO7K7Vya2AmvGXtc4rzBZhSzzKw2OqCyebV4q1BH2NPTzq5E
-InDLZDgZ8ClZVJEoXKlgHGetW9nwwplBHCN+Qxs=
+MIIDTTCCAjWgAwIBAgIUBDVFOnWfggQ/dqS99z+vrPZaOXswDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjcuYzUwHhcNMjIxMTI2MjEyMzAwWhcNNDIxMTIxMjEy
+MzAwWjAQMQ4wDAYDVQQDEwVuNy5jNTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAL+TgD9d8kTxbabIe6rOyNncdONrXa/6OvDy2yUk9uxGjvJNj22F0A5O
+VYj7YYIGqZ4W1Bg8oMlIEwDVXE8CQWkQR25tRl1Ap8OuxmbqXQjJPFTN8bAc9Skh
+LWpo3vLtXvDga+vh23HAHjme3kj8BzaNHgx0f3lkhXHTNQOPRqDb/aGlftqD/ynH
+kfz1O48Fsa4GNMNS5TMEVoTZNrVZIOZyCHi2JYH68FL9WZDkX3LIMj+QuKs+JVKE
+z0umfAsChKs8WGE7zMF2uh92ug+tSKhcn/2D/Gi1FmNzIi6NfgMv37fm1YQJzeIX
+kdkCGF9Go4tRd01BA0dqcvwdw13cXkcCAwEAAaOBnjCBmzAdBgNVHQ4EFgQUY43r
+gQTGh+XRtrm1v8yHMQk6LywwHwYDVR0jBBgwFoAUY43rgQTGh+XRtrm1v8yHMQk6
+LywwTgYDVR0RBEcwRYIcbm9kZTcuY2x1c3RlcjUuZWxhc3RpY3NlYXJjaKAlBgNV
+BAOgHgwcbm9kZTcuY2x1c3RlcjUuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQB8gY1bsZydbsLimr1P2tDhuECn0vp1qGhoqS+wFrgV
+8mGS5efggjELlK8x8JWHl253/58ZUXR8Curc1Ep+nkEsqwvTTz0Tm732yIsWeQqa
+boZVS/zLVSuTmNR5mZ4Vadw+XZYkvpLPdbWXNmkUpsQzWJX7H5GCdJPMHkbx5Shu
+kQ7PIu7F1Xx80pCnw3Lv+q3NVkC0cw4NgRzJUXfs8pBI/IPqmDvdmhCYHoIGS+fv
+BAGBPHeU0dvU3pTHXjrAmIVln6UL/lphs5MnhRLVMNpzjeoBGkyujSn69UMAONTd
+hyacN6SzdRtm3HmqkRLlGhMn2152wUgQaJ/DIe1WRhTB
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c5.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c5.key
new file mode 100644
index 000000000000..fbf35def1279
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c5.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAv5OAP13yRPFtpsh7qs7I2dx042tdr/o68PLbJST27EaO8k2P
+bYXQDk5ViPthggapnhbUGDygyUgTANVcTwJBaRBHbm1GXUCnw67GZupdCMk8VM3x
+sBz1KSEtamje8u1e8OBr6+HbccAeOZ7eSPwHNo0eDHR/eWSFcdM1A49GoNv9oaV+
+2oP/KceR/PU7jwWxrgY0w1LlMwRWhNk2tVkg5nIIeLYlgfrwUv1ZkORfcsgyP5C4
+qz4lUoTPS6Z8CwKEqzxYYTvMwXa6H3a6D61IqFyf/YP8aLUWY3MiLo1+Ay/ft+bV
+hAnN4heR2QIYX0aji1F3TUEDR2py/B3DXdxeRwIDAQABAoIBABwgFgx3iOlpPm1l
+co51lFQzwu9eZF9BD+Uu6lBuOFgg1byp+VwdcJmhE3qsWTjjLIS6mkHCqOJUFAIj
+0M0CqkK38py6K0Hmi5q1bohOcQEothRHNevinHWA2u+L3OtyQOWBNSbovjRZfbZ8
+5ep2IcL2/cnbUpmqUdMPrHFM2BQmZrXnjP2rHhI6l1nVlagU4eahkccxxgGbICUU
+thVHUyAs+/+ZAz/tZSxvNPB5XYdVidc0S1wCW+iaS2gx03NMig5yMMZOJtILomro
+e9txf1Q2+074bKkEo6zm6ZR+0+SGvzDkge0jtJgTFOTkdqk4c8a1ttf2QemVzMAq
+zZDpZekCgYEA8XBkU9R3aHoEAcNF1g5ERKsCE72JzC4h6NgJEkP/HR96waNktoM2
+c4H61wlvIaL8ji2gaevrBmQqF0uJtaRNhIKrThT+OtrBY9lwtDEsLVs5naLZ4XhQ
+aLdKwYhkPg3kjGRePAHlVl5em+3ibmj5nyUOCTd6Qja/P74+G7kHy4kCgYEAyyFF
+IqL2Jwi4abNElSbczhL0XeEXzzbZXK5R95wxBAsTvducQbmPBJjON48SorNPkUQF
+3vaD6DbyRbyT0xN+YVCJgSpAfkzC0IFcJJouZCwmJfFBHDe7cF5iBbH6JR2S2lOw
+HbA8VyqLUSehyM7hI41SpdlLufOwIHilsszBV08CgYEAvz6PufdqPk+oLN1IzC2w
+1dO/aXORWzKPmajGRrmlsqPk0M1kUU3OK+ChmQPYC4TXGKf7kOtW11EwJhOnmUH0
+RSpcFxV565Sr6zOpmpH4tFS2Zq9AduCekNPfWz12adEHYE2Ad3fo+Wj29/yOh3aP
+hAgN/i77oMdM5mkDOWJvUkECgYBbxa8/g9KkNgtjLev5WvJ9ZBDlbTHdm5E+jz9X
+GEkI45xKtbJ1+0jDMepRLaGkwlBbeBkS79sOieyRVy6OOYtqmgFVyuoSZcnKe/dO
+Th5RgaTGlaXuArv7e5jbq9ow9JiJbS/vadUJKJ6Czt4IJHeRIWNKrM7Kt29GbFuo
+WVusfwKBgHm1jMIYaLUNk9RVdV2koEJU1BEQFkuW5+pcKD99MVbQeSKCyY/kvLRH
+KX9ZQ+5yXjvSOwxNYhc+gn/GqI28Mzk91R/LF4ThuXy4kMwCZVYaM+52vQoeDqpT
+dspZbTGe4AC05kpwEVHO+uyVN4/F8hokUoz5mrHeuW3nSEejBMMR
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c6.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c6.crt
index a46ad1772f9f..eb4df3893e0f 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c6.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c6.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAL0x6vL81rrpGoM8JmeGDFzfPEQZMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW43LmM2MB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjcuYzYwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAIu8u2hGHsJKq9o6VOKcApMaSDf+rjr38JfwCNgD2opjq20qORPIcrlrggJ2
-bfxE5o3hb3bdTCEpY9gW8X8IRwwAeqgB9aNLfIf5+RoxpawNRLJDQD8RmqONygrB
-YpIt7pVkDLHRnHqmJqnTGJMGbPfenUxHzp+rruKzShdN8AOPAgMBAAGjfzB9MB0G
-A1UdDgQWBBQ8VNuo2DOKDS+BMuRuCDxohRSFsjAfBgNVHSMEGDAWgBQ8VNuo2DOK
-DS+BMuRuCDxohRSFsjAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlNy5jbHVzdGVy
-Ni5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAgzC/
-i2TQMQEMbDRJew02HbwJyAVR4v00Ep8/sK6fgzj0pB3blUgI1CFhu/iN+rbMkLe0
-1IFgRbJT6xhhu29TCWWC64uggySFWQTwujyphu4ATGgS7zJF07u3fUDB0JEtFecb
-UeQC0xwkQUk0RBnnSsakhfAGQ1iDi5/TPA31irk=
+MIIDTTCCAjWgAwIBAgIUQ2Pmb8vrcM/5MQN6cm64pgoPFSAwDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjcuYzYwHhcNMjIxMTI2MjEyMzAwWhcNNDIxMTIxMjEy
+MzAwWjAQMQ4wDAYDVQQDEwVuNy5jNjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBALgUib2HKHUQOJfvFPysBK/dL1QtwBIIbena5gG/ZAljXvSdm3lcYH6H
+OLgvzI4pV6y+dDT2u9SnBRt7ELSP3kJM/c4m+Z9pzmwSSlVh4ay910OzCfxj1wPR
+PTbVPRJm5stUqqfmTjNdWWjp2Zt7x3pUqSU6JzKBRBKY8b9ygk8hcjX/Gmdb3pY9
+zkYvVa/+Uns/1M+wce9GE7b9oUgNNPFVIh/uNDy3ukd+XQBjUKw2FM9h1IeXlpAY
+Lknc6Aa2Lk5hwB3QCO7doeLUhuQwfSZd9YdwMHMbiJykL0adf/OPLeNmozSY6JhY
+oegi6q/eQVTI4trbpx60vFOsrRWvJTcCAwEAAaOBnjCBmzAdBgNVHQ4EFgQUEZ3K
+nDLsBuvS0s6KybVwfdUbfU8wHwYDVR0jBBgwFoAUEZ3KnDLsBuvS0s6KybVwfdUb
+fU8wTgYDVR0RBEcwRYIcbm9kZTcuY2x1c3RlcjYuZWxhc3RpY3NlYXJjaKAlBgNV
+BAOgHgwcbm9kZTcuY2x1c3RlcjYuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQCMhle9VxOKcdcZOT8Uj781z72hVc4uj31Rp+/W1VQC
+ePKMw/0ZkIJidl6sDgF9xO+sbrH4QojperCalwpvYOMrfSsILIyAMeMMnpHT9Z9p
+iDTuTJeQDXZxQl0RQRn8okDT6RNGiqpGXkpTnIZl2ZxOls6ZdAduzyWdXoWo1gqU
+1UR8F8Yp+osL4AzCfviaExoKXlL0fbzhEwh2rhMMcNEVmyx8sIlEdRDNLUZRQz/y
+XoBaDjZ92X9cRFZ1ETGU+lwa5CwZ9ln7EORT8+7yNE03m6Lq1B6B3eVH7kcgO5Hg
+j63lsCnfFcJfO/70XOSrknDe+uqV2cxZAE4cb8829dRq
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c6.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c6.key
new file mode 100644
index 000000000000..409d71e1cd7b
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c6.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAuBSJvYcodRA4l+8U/KwEr90vVC3AEght6drmAb9kCWNe9J2b
+eVxgfoc4uC/MjilXrL50NPa71KcFG3sQtI/eQkz9zib5n2nObBJKVWHhrL3XQ7MJ
+/GPXA9E9NtU9Embmy1Sqp+ZOM11ZaOnZm3vHelSpJTonMoFEEpjxv3KCTyFyNf8a
+Z1velj3ORi9Vr/5Sez/Uz7Bx70YTtv2hSA008VUiH+40PLe6R35dAGNQrDYUz2HU
+h5eWkBguSdzoBrYuTmHAHdAI7t2h4tSG5DB9Jl31h3AwcxuInKQvRp1/848t42aj
+NJjomFih6CLqr95BVMji2tunHrS8U6ytFa8lNwIDAQABAoIBAAgqSQ3SH/ZJWOlM
+Xpbynmaang3S1bE3FIEKB6fOMIbGFKEvGIVilpkmMFrjK+SmatArxoXO5/f/zJcy
+d0Q7GXXBaoSpb+slS1W53fg5lkQOS60iFPBQEWqG7nsvLD1TuSsNv2AhNBjNLDke
+HlvqOqfvd21CThjMaROWGysNzj/qPXQh89CE6QqDwCtZpwFzzjMXmUJX0eRnXlNj
+6WyPHC6M4Ud5d9qqYXkD/wnWUhjxNe5bhaanqgCfN8kr9u7ivL/i3re7z3ZjRZ4h
+veyNvXoI59ZOvYKLvSoVIN2o6DG0gQ37uq8/NNk9tYHkJsbRriL5bXqp0pqrfMTl
+6gM4YiECgYEAvh8jEjMxBQ/BWGk5CM9NEHnqLGAIWKkftHiQnSfzO4PLUJ8I7ZKM
+sLai3vQ5xdGowVW6uafxenELyW48mMg0Vx9iEmoAkpQCU60z5XN0SqrpbQyEiDs3
+S4/EmT+ZyyGOGTC7nWXUNpXvSd2ssBSKsgUVQRUXSWQT7B6z0Cm6BscCgYEA991+
+mf68uKPq62szNXGSuwNguRXTOqFcSmTjekPO4RBzM3yaTiNpHBy/qNxmfOqj+DEM
+dw9wgmz3C/e9iY4lOueGXLgUOPUUMaQ0Cy2pFbSTGjTFdCXREPJAouc+Xo1t8sAp
+KJhESmhW9fk7QXpwRbiJDrLlGiQetIVm3KVLvhECgYEAh1qfm1t6sMHQwuLWyiDc
+GrjRMRqFlqmBevjC7D/++QYRf6pZrtnA9CZyugxh/SYQ4OPp9jdMwgEthRDIb3VJ
+XiVnqvhww+o/5dbPtbGGv4oCebKJwHt4GIGYVAplsaqIAgMh7yMJ/HquS4ZYZVua
+irb3tMgRNhzb2iTZm3+5gw0CgYBiY/MgcAU8Ch1zrA8c1OOZBuM7NVWu/WSkR14h
+J4+UubHxKzIZeGb9+M9/2VV5mq7SE65OCLvxmg2LG6GKW/mdY1Brt8qXPXnxpqve
+EBcBfbey/GWxEF3oKzHLOJ4CbIPLi8oKyj65hqeWZoPGHzM4CeOmCQ92jAYARMAr
+MBZb8QKBgHFCxFA4GAIQevsSZqFRtYXPI57JXHURS/xZagW2TqN51SlzMHGjqSBd
+qLI1Bb/BxyDHyKYkjF41Q0oLeFCj1xRZ8Uz3ijNMaEAQGANHwbou5sMF7yirt5Fm
+mNESeEq+Lm+th6hOV0ngyDfQlDbUMONDr+thpqamxmvxV3R0ufu6
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c7.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c7.crt
index c1281ce2b0cc..a648e6794e04 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c7.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c7.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUSDyZ1S/oPMJsyaBKt48INtI6P4UwDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjcuYzcwHhcNMTgwNDE5MTMyMjE1WhcNNDUwOTA0MTMy
-MjE1WjAQMQ4wDAYDVQQDEwVuNy5jNzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAoN6Ca8GpbkzxmORqC/sl3GucqCU/ha1ToDUuJxwzS6QTlJJ83DFbqyzPpIxx
-5YAhYiEfruoIVW4hF6L2KrQbKfAVWsfWHFqBpMycmy64yFCOK91EvHJ9Rs75+Klo
-OtSHgB5d4akgKOOBVyLHGeH3Dyl7nrUfi6FtCyfVCAjA8VkCAwEAAaN/MH0wHQYD
-VR0OBBYEFL4wJd/SFWld5TliWX3H9JPUcTD4MB8GA1UdIwQYMBaAFL4wJd/SFWld
-5TliWX3H9JPUcTD4MDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU3LmNsdXN0ZXI3
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQAk4x86
-/2n5y+Blg4FkJ9tpWX+nfkJa4SLXINfJxX3D1MNBXIkNlzPSC96XZ4b1zkdiZVTi
-EZiIsEwxU1EVjvBYmwGqfLjnqSzNYxl5yR3FPUC1Nhyn9pZ2nieB7tuCxwsrJ1OS
-s4xZjO8As7AYNUpytRRYvx8/KmBfZEW/aRHEcw==
+MIIDTTCCAjWgAwIBAgIUZqDTTVpePnrL7QjGOwjcDTjTRAcwDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjcuYzcwHhcNMjIxMTI2MjEyMzAwWhcNNDIxMTIxMjEy
+MzAwWjAQMQ4wDAYDVQQDEwVuNy5jNzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAJ3advzj9X4BP9SkDPahhW53YVJcyQ9LFIUYQsc4c0W4/+eYaGjp5+Yk
+GiOcs2UzydpAKgDKpKb5ftyjRX5Izg1kBLKKWF4Y+TRSQ1rVkesI2c9dKx7KCOE4
+oa7fkuAu2IdBuqmh83fOALW/t5iKXAB+e0bpoUFIdoQ7/xwpvBNr6uHnnS5cUrt7
+oLJlp1an6oO6eOWFhpA7yKhpW98XnqjGJ4Zj21dl0RMvRssw7I8rYEI82v7IvJST
+Rr5Y/yr3wSyajuB3ukasuQbJbofIUb04xFp72jMHR54K2IXyrHviQco5+QZwSurZ
+9Sn5cIC3WZ0CyBD9jajf0ZM0xP7/ViECAwEAAaOBnjCBmzAdBgNVHQ4EFgQUcG/5
+VZ/teprNolivxZ8yRRYCuiEwHwYDVR0jBBgwFoAUcG/5VZ/teprNolivxZ8yRRYC
+uiEwTgYDVR0RBEcwRYIcbm9kZTcuY2x1c3RlcjcuZWxhc3RpY3NlYXJjaKAlBgNV
+BAOgHgwcbm9kZTcuY2x1c3RlcjcuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQB6l3m2Er/auUmVdUYtk9gWyxSKn2TcXSAN2CJ+HDZO
+lPgTSn9h5j6xQ4PuIqFDre4PPN1ORuTCmvTSdGX4OzlJp3a0yrzKUA7aTt5SOJum
+yySQKQ2lmkhQOL2VOxkcXaBmAHKTLtc7Pm69jOf+j3t2xpIRQ3lk7ErKzOQcMY3W
+wqZgwydM5L46a/tLxST8/uYbwmgjhyha0ChnNFrY4L5Eb82z+K16luJf9P/xrSBL
+VOxKRx0bIAdZ1EnugwwOQJUHXohcp5gIO2bYk9hsgmirTziSSPiuSuShRwCJx2/y
+qDTXDRRG3ZDDPXaujVkim4KfEWTL4OOntwGpc+IwGQsJ
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c7.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c7.key
new file mode 100644
index 000000000000..dc66a5d8e472
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c7.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAndp2/OP1fgE/1KQM9qGFbndhUlzJD0sUhRhCxzhzRbj/55ho
+aOnn5iQaI5yzZTPJ2kAqAMqkpvl+3KNFfkjODWQEsopYXhj5NFJDWtWR6wjZz10r
+HsoI4Tihrt+S4C7Yh0G6qaHzd84Atb+3mIpcAH57RumhQUh2hDv/HCm8E2vq4eed
+LlxSu3ugsmWnVqfqg7p45YWGkDvIqGlb3xeeqMYnhmPbV2XREy9GyzDsjytgQjza
+/si8lJNGvlj/KvfBLJqO4He6Rqy5Bsluh8hRvTjEWnvaMwdHngrYhfKse+JByjn5
+BnBK6tn1KflwgLdZnQLIEP2NqN/RkzTE/v9WIQIDAQABAoIBAE6+/KvUtpNY4TRd
+cGAanJtKb6DsS5rW/BIXU7zZgtvN2c95fuCTi02KRgXs9qtuReycJLgHX1My1KAR
+zSmd/Ti95AADNukeMTdyMEqgjy3+WNXy5v9xjzN3kmTQAI/Ynsj+t7VG5tOlz0Hb
+taWhpBrU/f+WrgyIryLxYSb2BO50lnST3w0jLUPvgJHUTKfVnlcNn8XbBi1zg7R2
+t4WsOISNWcNLiG6MP2DgnZAC2LawHB4sefJzHID6vrBBzj28VFqf8Q7p7MdmQfd3
+/HKgPU/9wBGX4iY/Hz92l4pS4FzcR/tL4zX6YpfwI6BpkFa7D++CKSDSV+MFokNE
+k6fkne8CgYEA3sL1w8RtbGkRNe0/iYOzhSv/FQhEwp6kP//5tlKZILkAHVmHU3x6
+vw1OhFpOzRKVxcLm7Ln55UgdSqZmuKoLmVjGDH4+Huuj0s54lG9FORYf0okRh7W2
+3oSI3ltPmahu0lPvBqKBStwFRcNRcckTtoxSDoZfVWBhyMRMNPcHP3sCgYEAtWgk
+zM+UmvZUSODCctgr4kociMDDlwV+u+lPj+cdwD/S5Qby32e2qKpOeWsg9JuO/AiJ
+mRlic04Ubfo5/T8zw1NtLuDpYEX/lRCg2NKL//VDUFvA72pj2m1MEMf3z8XzSjFy
+Iy+aWaVT0EOuQdwhEr2MI/mZ31X92aGliiye4BMCgYEA3hXM4+IfnaeaSwM/ByBQ
+iAJ3C0b1FnsHEe9ny8fJdoA6hwhNDITb80GW+Aj46ycOPt2zUGuRom8lFNmVK6IM
+clO5D5eE41rWN0M1n/bXMrTGASY6XexiZrIHqneqgm0UH4kikqDM+g8lCopR1nk7
+uBFpdaWtwfaXRou91kg556UCgYEAnbRR+bOkTytCjrDGmqyPSgR/1esunkXqrVXh
+Ewsq9A0IPI7T6Z/yTaLw4eFeHwh1P2n0pSJKUk9Yt1tHun+Ncos+afihKk7wKBjy
+nEw95WDFCkEZ84kcEbHe5N/S8c/291MRFMG3f6OYv393KE5EelRKVqQRPvU04FQE
+46By5f0CgYAQroAYIFryyLdlo+UV8qJe9LfGBJ3jl9Is3dy5EOTmhwFlNBipRGi8
+ErDclUd0jaLBfg2Zdw0jdeHa/peRwI+UYFG8HDHf9sTD1dfb42RNnemtNaf5inFc
+Hw7w3YcdeNS04aYVeQaXKuUajvizCgb8eUvTYDMvzQKBBLVRnoZh4g==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c8.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c8.crt
index 3e32ad646696..da894f4eaeb7 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c8.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c8.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUKgOBruyfZwrIdrRexXAJ8NwVvJswDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjcuYzgwHhcNMTgwNDE5MTMyMjE1WhcNNDUwOTA0MTMy
-MjE1WjAQMQ4wDAYDVQQDEwVuNy5jODCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAkRgHBQ5c5YTSDcgW0Qm7ptHs0LIp1kqBmqFb0IBRSOwkymlR960ObUpGY2qf
-xxEaaMWRsa3HEmx/5xVHirf1ozhh0LGQwI6ZqBjwDLUvpMGD+Lrl4ZzGIOq0GuMp
-QkAF7/gV3316Eu6DJebZ7+REFChgBCeWr14jYrWUgsl+XeECAwEAAaN/MH0wHQYD
-VR0OBBYEFNQ7qK1CaJfgGGralMDWPU5jCRWZMB8GA1UdIwQYMBaAFNQ7qK1CaJfg
-GGralMDWPU5jCRWZMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU3LmNsdXN0ZXI4
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQBXqo24
-seeiTtrmmzceAHa7ufi/F3OZ/aikFAa7ZXRgGf7bwJ7dSJNDFBsbhkRnvypWMKQN
-Di9cgsNiIfLzrVAt8VKFfUQwIvw8MT9Uh82lJ9ZqdSR9Uli3D4c2WoIhz81w7CCV
-cAu+0WiDoKqszaxiyxAt9vhi4Q+rV7KrTSgOnw==
+MIIDTjCCAjagAwIBAgIVAOqM+FqobIhuJ2Jdykg46NR9hRJxMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW43LmM4MB4XDTIyMTEyNjIxMjMwMFoXDTQyMTEyMTIx
+MjMwMFowEDEOMAwGA1UEAxMFbjcuYzgwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQC+/XP0g05K0aqH2B5hgHhpb1Kf3dJx1U5dlJMeqUZtz6v9Sl7g4EDu
+6biRBFmkHJp86VCwA4VQGCqN4lJB1pUnVYU0C0Tn42k8E4CuMNOfoN6/Ffa1pN6G
+jcfCWZc8vVOMpv0LO6I1OIILgjzVjmQ3k4FLQjgUzS8a0YVHZFmZ2IkiZZ4Z51Na
+VPYOGayGX8Up1ymr3UfziaGi6p+4gLVTcW0UUl7JS+fQ2XbpHBnM/t4T5XVZxhjK
+6pYwSxMGIuwHrzlzORVObem47SwxgAi1zK9Clhpe0fFxUP/Xfd8PMEsIGnqEWzKp
+LNMaxWBHMuTEf4sdyISgioG+EIEiyY3ZAgMBAAGjgZ4wgZswHQYDVR0OBBYEFM1U
+cXKwu2p0MMbF0WRcI0bTTOd0MB8GA1UdIwQYMBaAFM1UcXKwu2p0MMbF0WRcI0bT
+TOd0ME4GA1UdEQRHMEWCHG5vZGU3LmNsdXN0ZXI4LmVsYXN0aWNzZWFyY2igJQYD
+VQQDoB4MHG5vZGU3LmNsdXN0ZXI4LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAf6g+NPWoc8BeOQs5R4UUgT+dFj5AzJxblwjrzkJ0
+qJvCTyrT3B9WQQP7aYnglArqy9mwuMmPE21UYh5Kl9a32T/ZvK3kZmBVWJZjx4L5
+CG6DUPi6Y80rExtkUv+D4mlZFSlgJg/n5pF4docadrUWxWaC8IHMuSUursVU9gi8
+zHQY09m/CcVzerxaZ1RolHlEROkV0Q+gFzD4EWHFFMXTJzG4AmCOCSWJ1Hc0ntuI
+lOjrT5vC+jTKYZnWWPX33bW9b5SfM0QufgNA9y3UCXLHAO9s/e+RN1qHipwdI/8c
+zqa+EtHwsrvZhU/BAVvv9ASrdrDeanvKCHxMbIH29ybbBQ==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c8.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c8.key
new file mode 100644
index 000000000000..de6ffa27d752
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n7.c8.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAvv1z9INOStGqh9geYYB4aW9Sn93ScdVOXZSTHqlGbc+r/Upe
+4OBA7um4kQRZpByafOlQsAOFUBgqjeJSQdaVJ1WFNAtE5+NpPBOArjDTn6DevxX2
+taTeho3HwlmXPL1TjKb9CzuiNTiCC4I81Y5kN5OBS0I4FM0vGtGFR2RZmdiJImWe
+GedTWlT2Dhmshl/FKdcpq91H84mhouqfuIC1U3FtFFJeyUvn0Nl26RwZzP7eE+V1
+WcYYyuqWMEsTBiLsB685czkVTm3puO0sMYAItcyvQpYaXtHxcVD/133fDzBLCBp6
+hFsyqSzTGsVgRzLkxH+LHciEoIqBvhCBIsmN2QIDAQABAoIBAD6GvIEGwcl11cXu
+Z5fVFtaxttOWq/BPfH3fh8Coe2G09OISCHjHggqVYrK7QVucb6lIOv280Va06q2T
+tAlQ5ni/o0G0FMVz6n5cBitA2e9vG61kU+YIyT80yHqkmQIuG123R613QvX4f36k
+DyV9Hesh89i/H5wTxoGZqXADduAJ9pRWI6hs5AwgE4jiF0SQC2LWF94wVSoFkOFU
+7Am7dI+Mdqmemv9N/7ybgKmwTVBkqcYdrmg6xjKr/5IPPl3KzI2kpYKRH3Awi+EK
+oApBYLIblLOB43DpunDlwPIm5J9z1mS/PbyWMMsV6VGklY3c+lLO0sNaJDMz0sdB
+Lfh3VecCgYEA1i04YvexGObrEs8IjMb8nuCWl99t6Dyv12NuQIb99h1O+5owkRY3
+0ql6+6IFe7jOLy2kc4TpRK3qUYGWl9A3A+I2D/yqMsvnBrdQQ/ht/hesKj2z+jjh
+MoWtbHYDtOc7sqgbXOtYryzkdKr7VltQICvFETsPwppqPeS6pMYYtF8CgYEA5Ekg
+FRUwFXZjnBYanSXMkQW4HdHBEmGAMLfqIHwSY4pWCIZsp8y1LXght16I1VmaWqP9
+7tfil6mAB52OMYUrcd/WcSPdNdAmCVqInchb4Zc+wDL5arSkJWyI1S3ZzLoZ3BME
+i3LdeSGnOhF7gIv3eqcRi/EvHQ75HIoFJXQpqMcCgYAv22LiGl6kJdUZGtHaQwxn
+o9+CiBZS8lyIUFtuZ7uPpvTjG40GnmRS2jP0zjtTbfQEbrp0ZZqKEiJtm/s+aPQi
+FgXmhJqzRwSUZ5mz1msdf8Skm6z/mH4Zf+1EcwpY/eJFBXM3zA7A9dMGjr4iibzZ
+s24Rb5UR8tcShW1/LBSRGQKBgD7HmWoS3JlaaIRZU9JZvcs4Hm0c7RLH+3/qZBmw
+5UG4nnZTuXfgBQdPHT6XYdwCez9oEAJqHmeE9q+hdaRKsirr2sZcjY4qAvvZiO6T
+VilQ9bHXkt30dZvRIEQ4CsiReWEbXcmXui6AwcNnYWZ1l4aEgX/RwRolrv5ZOpcV
+PgurAoGBAMZ7r6975ROMV4prT6yaJDvClDrziBG66k9NQoNVIYrdVyoVntgKLFO4
+us9U76gELoMkm7cQjH4rBSoWO6w1B8E4o7yuIijaY7pcpSC0VL3n2QKEmGawSxaR
+BV7QZKG+V754hvp7l2Paf5aX2dP9coxcD9dHhySeEmhl4Q9de1R/
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c1.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c1.crt
index 1b172d4bcc67..3663a6c30d17 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c1.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c1.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUQK7PM3T7+mYy8ElPam9Rp233WEEwDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjguYzEwHhcNMTgwNDE5MTMyMjEzWhcNNDUwOTA0MTMy
-MjEzWjAQMQ4wDAYDVQQDEwVuOC5jMTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAgXuRBm6AhPl/n0w+6htbZqVgDzHBd9Cnp13KVWFj/aRR2v6ppf2/ZWSPDAp3
-zcZALQrPOnwlNcAlTdoFYbTNhdiciWb3+jMNSuwZaiNODykWoVUXeF+dHsWFmSje
-QydkxPK8AfPwS9/8/wTvR2qjFPzIKYk/JPubG07c/BxC+QcCAwEAAaN/MH0wHQYD
-VR0OBBYEFMLuWtBmdv4emh//95cK1tzKVxLVMB8GA1UdIwQYMBaAFMLuWtBmdv4e
-mh//95cK1tzKVxLVMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU4LmNsdXN0ZXIx
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQBqyT1Q
-NDOp5FDMKVowZqJKG5UqenIZ0FHk5YGSV+TJnLE6LdGLsrEie+zBirY/rWYH+vJs
-Ao3HrA/IM1hGVPAZI2g7nFsokqOtTNgrm0/SSRuMSf1J82KhrG0cHcuXuMb0Z9PG
-d8yP0524XXfbbwN0fT/AuEWa3hmS2NWXiJOoKA==
+MIIDTTCCAjWgAwIBAgIUTAE53k1+p/Ugrtc+HOoZqiqc/skwDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjguYzEwHhcNMjIxMTI2MjEyMzAwWhcNNDIxMTIxMjEy
+MzAwWjAQMQ4wDAYDVQQDEwVuOC5jMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAL5JyZGkRUqy3lFTMVs6tPbgmdiziLz9I0apRDt0+4yLeVk5iqy15TWm
+2TwnJEFj+WE7MSr8KNZvDponEzvBFegTCuOPVF6BVvD8leICw/we/BGkCXK9897t
+upNlnlaow6v7sRs7RTLsxZ0kTPQH1ruyQahcM6CJRBEDZRPN+lI/UevQUogCmn0z
+wHLy+HF43b+BI2P9iw+0PGkaRCOR95oMxpKMbknoKeWbHgFECPMJ2evC+wH4jzp0
+KJzk5jYDHgsZBumkQhCuN1dp4dKLPGJbe59g4RPEOS6bBDSDVjE+0qgT0DW8bE/R
+Z8s/nGTaZovlC8WyIxEjjga87YtGdhsCAwEAAaOBnjCBmzAdBgNVHQ4EFgQUpfvx
+Xkzh2FhJKu+Xwag7JqZ8H0QwHwYDVR0jBBgwFoAUpfvxXkzh2FhJKu+Xwag7JqZ8
+H0QwTgYDVR0RBEcwRaAlBgNVBAOgHgwcbm9kZTguY2x1c3RlcjEuZWxhc3RpY3Nl
+YXJjaIIcbm9kZTguY2x1c3RlcjEuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQABwI2KYvdWalV4Bl4VSKqeFi9VENM/3ZEvEDMA/y5g
+DqbFEaRMFar7e02ZYQGVH+WC/j3r7MIod+xUhp4Llq/k4hRDgGWkaFxVnOJGNJe7
+0H+QooyBrNQy0elaJMmv6SlaK3I4QIBFq14Ru31KCRb9LlLM/ao8TlBNG+S3WP+b
+Pa03Vg+cXUiKeBCE+pGematEoH9w0UvdDezkn+Vd8SIHGVkTnroNMUz+TL/FB1Yf
+XVLbaZfxtgb8WtQzM8du+saV0VsYn+hEdb4WL9T904W8ntZILaVsUsCyf14RIvJE
+P//GZcJuUIVlQgULqy46mk3LaNpOV8XWKAXnTdQeKVFn
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c1.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c1.key
new file mode 100644
index 000000000000..aeba4b96eb0e
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c1.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAvknJkaRFSrLeUVMxWzq09uCZ2LOIvP0jRqlEO3T7jIt5WTmK
+rLXlNabZPCckQWP5YTsxKvwo1m8OmicTO8EV6BMK449UXoFW8PyV4gLD/B78EaQJ
+cr3z3u26k2WeVqjDq/uxGztFMuzFnSRM9AfWu7JBqFwzoIlEEQNlE836Uj9R69BS
+iAKafTPAcvL4cXjdv4EjY/2LD7Q8aRpEI5H3mgzGkoxuSegp5ZseAUQI8wnZ68L7
+AfiPOnQonOTmNgMeCxkG6aRCEK43V2nh0os8Ylt7n2DhE8Q5LpsENINWMT7SqBPQ
+NbxsT9Fnyz+cZNpmi+ULxbIjESOOBrzti0Z2GwIDAQABAoIBAD32zzGi+YFqn6SB
+37eidV1TJCYByIwqLjuLURG/dU59+IUKjNLS5JHFBEZnRJIamRGFcexacn3paire
+gI/bv2N0F/6kJMglfAc7aR9Q4TsPZ+viq8T2gBxe83ueAqKbkCRH8JM6iL0dhvg1
+4u0wREeYs3K8m38yU7zqixaY42O2dOFR/dgvTxJzleyOfcG/OAAhdcb1ygPB5u1+
+KeU8HU9Pe21eXe3vDMjVC2WvsOcI+P5bVSh4X+UOBFdNCncX3Tz2KTsP7N6djxAG
+pJslZ5HprBGlenZIYKpEjqNTgdqyXt5UZGYJzn9kQT7WefNS2iLK02+eomOXySev
+LV7db/ECgYEA0Kyyw/dF5BB5cJe17o8UOrP5UXMVSgMdBDOpZWUb5DaOrj9OWImc
+U7E9TgsjW7PPjP3YHIghI8IFOcivTeTsEyLoMc490cuwyxl7u0ayLlAEXmfhg7Wk
+j6PEqbejL+mrmJJyieBwY5GmJEf/gXI7vxcfwuy624GJ/Tmqmvq/StMCgYEA6XGb
+WozJ6rFb9hpa0fUkheEbt046Fsgk8L7l/ci3qWCtEsLODbwDlw+HnVSzWlwswcym
+RYJgQyLC3Kz3hSViwWMzpi1eKY93TYm16D5CdukAD8cdhs27xlkmBJmdkkAWTAic
+jTwLKsKfU4zBHASLLfrlGCnmZshyAcJCbLDPipkCgYEAgMyHjoNahTlR65gIeFWh
+09XwJkDVPBiN2WEVxnrQ61xtVsnGTsmpTtMQFtKzIU/r5Idt07aGVe7UkOT4pyiM
+OgeKr+svpwWFeuagw2gQZJuJBsgxnr0H9IzNvz+UnucovgRuZAG+/QYSOUSjRT+/
+9uOL/CFn+EKNbFoinoP0fHcCgYEAlwafqjpRW1u5EgPE4/aD1XVoN/QNErYTaAko
+xEI5yAO6lNug8TPE4tLyOrAUhI4DirG6lFefrW9sv9XLIMGqHVDvlj/dnmepmGlS
+XVkCKOne11ZLXO324IDGXs1/KqH5iuE7XwgMdXKKH0R9noG2BV4/Hp3k6HQfhdHG
+pYjSDGkCgYA+Ot4GzSsyrothknGda2XnLQDvyeOsDLpkboLvajizKiZshiiTjisq
+bvP1h5KVd8XmBzopmHAt+8YNOoHTnVQa6FjmJkJdXGMjEo3q/u+b9rQRWfY6DvS2
+41xB9cVG2zcbP9sAd71sCjkHU0r66LxamLyQIlu93HCbWZd+k1I5OA==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c2.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c2.crt
index 8c3fe5249b55..259cd2e81286 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c2.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c2.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIURuq55mbrZcWnTGXp61EVqnrGxngwDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjguYzIwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0MTMy
-MjE0WjAQMQ4wDAYDVQQDEwVuOC5jMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAtfvCH7wJN5iSoyeWtfeJFIs3BrCtS/cb4LI41nam3g9T0Ag9WPhf1aPP5QBg
-DQ1umY0kuZEhojduZHgZVoG7mNVNTgqSJCJKooKR+SrQ4gqaIlgIwSpKwo4u/za0
-4p1OqRhOJajOMCuC4zEz9yPt1YAfXCkYryBlne5m7Y+mVncCAwEAAaN/MH0wHQYD
-VR0OBBYEFLpfZ8SVBLNw/xipDeUe27M9REyEMB8GA1UdIwQYMBaAFLpfZ8SVBLNw
-/xipDeUe27M9REyEMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU4LmNsdXN0ZXIy
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQBODSa5
-2GRYIcw6npTyw+Ds3UhiluYMpYmopi+igLRRndBVk1Z9gCb4hSBi6MjcMtfX89dz
-55MBnIMNCSqkW9orrPGKVCGZNpyxIkDqvgiwnm4vIURoHkujhX1UUCcf0Q+EMclY
-EFdEaR/DV72n2uBT7xETbOJEngsN+cgy5t6Bew==
+MIIDTjCCAjagAwIBAgIVAKfPuXVkRZmYoULIbNNTWluGX37IMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW44LmMyMB4XDTIyMTEyNjIxMjMwMFoXDTQyMTEyMTIx
+MjMwMFowEDEOMAwGA1UEAxMFbjguYzIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQDR2fxTbZWOLAn0LHE3DQ14SbI1Dgs1f3vl8zgFr/Q9gwVmNuohKjyb
+P08y9l8aRKc+RL0to6BnwZVIELwZi202YvpVcQ97oPLznu6j+6AokqeFP763vhfE
+Qosb0Joq9uBYwbY5DKrH7ZZl/97rvh0BZjoBfE7IvQICU6kXweE7KoQuEK+bonW/
+27Co6J06GaUwOwZ/Yu+HnOu0qZH3SiqkShYrsUA4FGEQpiueFrhkMi1HuebCTQQR
+JDJVQpxqyLuVVhpucEME2/kRVgy+cr4TD1JIwQKjtwe2PPe+GXGCxVGeN4OPjIRY
+S9R9BNEvv6aQNSMdrbVAmJGxFcGgmIqTAgMBAAGjgZ4wgZswHQYDVR0OBBYEFGXW
+KB1bRypP4kIuLkSbv6M2w5xpMB8GA1UdIwQYMBaAFGXWKB1bRypP4kIuLkSbv6M2
+w5xpME4GA1UdEQRHMEWCHG5vZGU4LmNsdXN0ZXIyLmVsYXN0aWNzZWFyY2igJQYD
+VQQDoB4MHG5vZGU4LmNsdXN0ZXIyLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAW1IS2Xan05vs11P/9/NLnMnPoMc/MHM6hGXLu4r9
+qgSL2uGq0eEHmTOOioePcKseihyY9Z3tcaVJWrxT/4Ghe315bhbqnUal5wK6lRy2
++lRr9OWHspWs3fJYAHDlDxuy91uXXLJI7TKtMmknZ5OvRNyjcgNfcTcBRFMyvKr1
+j5Oc6PFnRLt9n3hRYDdkhuBA9IY8d1A7MR+uXTJXTmWMK4P2u5+8IT+nGUknOAuB
+sxsU695kyeG3Lmu2/SK0gSPH3cDE/D2EhCAGqvFvpLsQ2tYmDQTZTx0lhzS1eeZO
+lMhOzKDReKhKI7ayLeWkCb1o62Poblc7q8GPiSs91gTWxw==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c2.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c2.key
new file mode 100644
index 000000000000..9252495f56e0
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c2.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA0dn8U22VjiwJ9CxxNw0NeEmyNQ4LNX975fM4Ba/0PYMFZjbq
+ISo8mz9PMvZfGkSnPkS9LaOgZ8GVSBC8GYttNmL6VXEPe6Dy857uo/ugKJKnhT++
+t74XxEKLG9CaKvbgWMG2OQyqx+2WZf/e674dAWY6AXxOyL0CAlOpF8HhOyqELhCv
+m6J1v9uwqOidOhmlMDsGf2Lvh5zrtKmR90oqpEoWK7FAOBRhEKYrnha4ZDItR7nm
+wk0EESQyVUKcasi7lVYabnBDBNv5EVYMvnK+Ew9SSMECo7cHtjz3vhlxgsVRnjeD
+j4yEWEvUfQTRL7+mkDUjHa21QJiRsRXBoJiKkwIDAQABAoIBAEZEwQpKoo6lznt1
+uj9d3onN25+KYVR7qMg6JiM5ZQaH7fHpJ4MNepfdni6JcxT/siTWraYvqVG078De
+4a2IE3znrb2Yiz+xiEjtiilc46dSXEfxLXKnc5vjUGNy+XyziDIQQ21dAX7K3tau
+1D2KPnouefPvZQ4wxBB1ZSSwZjQCD61aAWSlYTiFsAX3yioBW+JG+0e5ytBQKHYp
+l2Eov1KhoSnj+0G5VdX0QTIZJUB81+GD36cOe55FEkrJMgwUnoNerp/bHisdDXpx
+XVTAORCdjwmofsArhZiahYD/G09vwQQ/PRSOWxQGVTb1bPlQdVY5VxlATlXUF2C3
+R4gWHpkCgYEA0sFCwjZDcp9ODVoKIIUu/40AMpNUAhJMfW9AEmm3po+XUkSn131S
+LBLZJp9RIa2MBukuJzkiHdNe8N2/XETdLLh3OX43iq18wlLhz/bPctctMZZ194e/
+/F0qiwu8M8CNroqk0feE+wfMLwvMc0vNd4AO2riyTJ5TtKwampjot/cCgYEA/ucT
+EmcxkJuaMgE8uD/8pfH2RlFTbtPmCzQofdpOgw/Ar0KWDAbqe0XYmgSosGq6iXO+
+uX5vB6SAM29UWG66hVsB9/i0lZlBmY6Oppuv9vzD+fPk00UwFG8VXPE2AanG/Uk1
+SLYlDe91wEf/l2eNLRVqGjcku5vz9eSCG6ndc0UCgYBHgPBAah6qFehqWUEUVXtj
+kpgdkKSw6zpuWD9Zv+piJVeGvupfDxLBB0gLdYQWNAZNFfHlttmfWuZcPC+/g1Gy
+6Ybxx6Padg/c8jfQ0gECCoU3zL+LUtBNM+LKOBoLSOoI7NsMrbi9XqDk2VDtUSuN
+vrHLfVxo6FKn+rgv+W8DcwKBgQCG3tTkg91h2jL4CleKVbDkTOY1xooSteOeQfa6
+wGuSRR96nAywSGiA2TAGLjjjP1V7rNZZZ2Bv9sYucNV8BPqD3OdpwRornW9sRFWO
+Kgos8Qjwk++bKZyK80umZaphrucu1TFwS4/Hk9AF8RJ3lb7I/++n901kRtwKWuHM
+qVAW0QKBgQCqqW44MgNrH8m77s4PdkcdJH2pT4RgMFFpryqIX3MYAUwVr5YioZMn
+jpwedzpza/rwDFMUvLmhQyp4yXI87ysITeqrwnX1XYOvkOteAdjrO4oe3fNkZZ4O
+LqYNCWbZfF9W2OpqgQqMuSP7cZhNG66mM77hi7tUolUhjUh1sX6gzg==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c3.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c3.crt
index 14582bd47bbd..b755e49d05e9 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c3.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c3.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUPGx8GdQgamiFY9WkFeBWsOMKRGUwDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjguYzMwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0MTMy
-MjE0WjAQMQ4wDAYDVQQDEwVuOC5jMzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAujTN5UqfbXzpFTjLfl3XOWsQEHD+d5VJIkwTcA6PxUL9CCJCapgylbJQ+nzd
-pfOaUPZkSV75pLxfuimDz0Y8KtlVzEry0zhTgdMSos/ZVdQCn8j79W2/19HVBAMH
-Rd+RDH6oNR3BPNLiW/hycXOyZUOBUtHQ3OMP7d+7dsqGMNkCAwEAAaN/MH0wHQYD
-VR0OBBYEFFoGk7U4lpbzmRjPIEvIsZFoTYd7MB8GA1UdIwQYMBaAFFoGk7U4lpbz
-mRjPIEvIsZFoTYd7MDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU4LmNsdXN0ZXIz
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQBsqZfT
-V6gBfsknJp0g4vbZ4ipMOxBQpmEYUUFTUn4vWX1UAhw9tdfFx6p3yLwrPBBeuPT4
-WfL6goaR46aCAELLA9RvcvGGiWysAfPChdHY5ELdDN5ACOyLa9aCY6CMzoMqho+5
-v8F4i6Rg/hFT5Xl6X+DtJrevjyyeECIS7MkVwQ==
+MIIDTjCCAjagAwIBAgIVAJ+XUJpxk3mz3LBHWK12tjA88jfdMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW44LmMzMB4XDTIyMTEyNjIxMjMwMFoXDTQyMTEyMTIx
+MjMwMFowEDEOMAwGA1UEAxMFbjguYzMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQCmC1dZeWOzmCFeXyfjInFjPU8WTIgIEqrI/9+3M5M2KXBX+OtnF2zk
+1/aoQJIFgkaNmPXKT2CLZCjS2vmG7rihTSfRi9iEy1d8Yqh5J300Jx2Tgfq02BZO
+xF01Kljk7LnWGLlSrqUM7E1hl2Fm0pL8cXefATtATMyUYgkvjgZBlWv2VugfZErO
+w9EITpGbXbxtJ1a4tYS9CLXyb6XI1AWk2+D3UV2R33dykm4Cqcc8x51pOTKqAxQb
+wPo4M9SDY68RZGpaWKGQLzktEhy1/oFQd7LoBbL6GBLImamINvbCVrWndcMxv4B/
+3QSs6WQX5UfEkvFsqs/yEAr42Y9T8TwfAgMBAAGjgZ4wgZswHQYDVR0OBBYEFM2z
+9IgObMoSrNSJ8uy8ZdT3rCsFMB8GA1UdIwQYMBaAFM2z9IgObMoSrNSJ8uy8ZdT3
+rCsFME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGU4LmNsdXN0ZXIzLmVsYXN0aWNz
+ZWFyY2iCHG5vZGU4LmNsdXN0ZXIzLmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAFAonW4Cn2JduGxqsI467CKuzECoJNrjq9LGD4tP6
+qG1oEEago81jlSzfa9ZddfTUKvUsDJhRrtLTjw5PcMMMR+KgBkQj1gLtA13gv9vE
+nhAIBGIfHzHWjndelKGIGSkTvcZV+MRxEDkGdYVvsRLzEE58ORAgz4dqYE3VOF+d
+kCHSv0Dm0FqptEuamrDAgSYMJDnDSZHIlG4i99imZ5EVjZbRW6FYgCst3DQntb+H
+C5qSk72n/gwpc/g+yWDKPB+r7RXcRA0ePUQ3AeYVMAGeoQOnx2clqJezOT0ty8Ks
+jHV7O4RlgfY7caDWDUdOSC+2kSMqjbI6qV+46PvFy/OFww==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c3.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c3.key
new file mode 100644
index 000000000000..61c0582d3bf9
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c3.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEApgtXWXljs5ghXl8n4yJxYz1PFkyICBKqyP/ftzOTNilwV/jr
+Zxds5Nf2qECSBYJGjZj1yk9gi2Qo0tr5hu64oU0n0YvYhMtXfGKoeSd9NCcdk4H6
+tNgWTsRdNSpY5Oy51hi5Uq6lDOxNYZdhZtKS/HF3nwE7QEzMlGIJL44GQZVr9lbo
+H2RKzsPRCE6Rm128bSdWuLWEvQi18m+lyNQFpNvg91Fdkd93cpJuAqnHPMedaTky
+qgMUG8D6ODPUg2OvEWRqWlihkC85LRIctf6BUHey6AWy+hgSyJmpiDb2wla1p3XD
+Mb+Af90ErOlkF+VHxJLxbKrP8hAK+NmPU/E8HwIDAQABAoIBABODMq0fkEFbJ9S0
+vvW4pvWAD2qx7KxyMSK0uwAG7g5H2V1lMZioTpIn9OL5z48ZK5xZNM0k/dXzm76H
+oBoktIq0C/Fai0Ozt6C04VJ3ewWlcRrUA/e7iwePmNp2EwM39HC98A1Q27GElWVa
+CkawiThkRyfm16UrQUFQsf6mJIMRUSauDxsW73Qx95Xtxm4E1RjwUBrW8F8lQvef
+vjHwmHbzYr3FPDeC6Bv9my+CBKbzEZqvYX+aWDRw6MTsqf2QHBreD4+0haA/8si4
+GtQhsonYfj59YA2CDhSZTsTjvn3uWZXftE/C+5sg1v9K32QgyclHX11FL68J1gto
+UijLLm0CgYEA1Oo9OZjnaVrvgp8FLNv+jltoZejftPvyiQs0RwlsfHDkrwYqnMFB
+UMLcdMDejHFjhznvkXMNJPjV+YyQqb52Z92Bk6X4mIcQITVse7rW9D+UE7XNoQdc
+XzPuWoGQ+9q+9IUqAmNBPQTBY5waagpuMfm2E6QtlF0DZODrOl4+UrMCgYEAx6UI
+J5Tc5M4t1GPBvqLXyddDOAprc8aM2ycQGOddv/b9xTD09azloVzvdPT4A/+8r9RX
+u/C+7YgH9uuCwrg5yqsyUY1D4uqnfnVMhtRpfn0aEQLuRnjBIiH+cm/aeG5FYmzL
+Wz34qKCa0aRVS2y+EpUf4/6NhaJUB4//1KYCtuUCgYBfwfXFSMqTLK3ZvnFSaqMD
+HYNHikPKnyHGlv5tEIT1NsTzHr/dfmFE1E7YR8IPwhN0Rw9DCE6aGTwLHzgUz69r
+KUlUwWmcF+qeNOvUfgSdF1cewj54NNpLiwyhMfKmAwys5pMtPbyil9fuV8/db1F+
+MoO8M7O4LVkQbsP8nVREVwKBgH/pH4lkmgZKhxht9aLzaHgcYCfWnTOnQXXj2C9s
+hSdJ0xt1G20QImu5RH5VuVWukzDARpnWTC2Zmyi7iaLCL9zGxM11Scpn93pRIv+7
+tf9hncLKculZ0rvQtdqSCZJhbDj8ZWoHEXrQs4Cf3jod6MW1H6KO5BN7hoT14L5Y
+wqJ1AoGBAMWkikWfPYXHA4cAAjHi5OWM35ILvkNvs0QPSH7AlV3grvZXynMJWpPl
+SeplNpjnp9gUfjnHtsKXTVw+e5Tq214vpqLIgM/+hh4jO3q/qNS+bLZwTMe1Hudx
+yC9lUs48+l+1FuRiApXRxyUsCjc7uA4JOFVx3Ddppj7cJ4q7+Zzy
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c4.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c4.crt
index bad3f9a24f20..328ebea2ada5 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c4.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c4.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVALsTUZNLfQ0QhhChFKeFVPkVAJ9rMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW44LmM0MB4XDTE4MDQxOTEzMjIxNFoXDTQ1MDkwNDEz
-MjIxNFowEDEOMAwGA1UEAxMFbjguYzQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBAIhWp/YCB3cVIrTK0QP6LC5zBZ8avn9isi/QeLXwxw69k+qhHEOReoybnPgX
-yr2wyXLuL6PwyBMfjINDIEZ+N+4TsbYMsE8HiPNYQyZm1mpBI7ux2FbgCbpuJRfK
-A/Pztz4arlBXcHG/tJKistvF+SSrw21t2bLtVZHXBFR0c0NzAgMBAAGjfzB9MB0G
-A1UdDgQWBBSapaWx6oupzaeRg6DMkAo0HwOYrjAfBgNVHSMEGDAWgBSapaWx6oup
-zaeRg6DMkAo0HwOYrjAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlOC5jbHVzdGVy
-NC5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEACX/P
-iZsjozczvsvkMhB8RLFVo29jjrk99GWKjZeYlmN6ntnmuQZNCgaCPaPLMZnEz33s
-tN7i6DkigTkcDfsLnBzFHo9d6tpi5CeE0gCnXvSfFkAT7ZxR3hC8EJOquGPBy2Jh
-A2L/CTwXmbxmEj/qXqMzsjJr2BDgNjATKv6za5s=
+MIIDTTCCAjWgAwIBAgIULGlImaXPFbF0CMBnnucF4yXruGAwDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjguYzQwHhcNMjIxMTI2MjEyMzAwWhcNNDIxMTIxMjEy
+MzAwWjAQMQ4wDAYDVQQDEwVuOC5jNDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAJ7hLWxAt9veXNo/CU12KKHIG8oHWdnPx44kl/yHweHRg3vRiE5PO+8s
+KFwROVfD424Su6+GTBJy08M8epsaSrgshV/lRDHMd2234QZQbhazxgAxGJ4CM7pl
+WbAa3XMAUnUT8WCug8rNq71A5oIFnfwA+TcjLtiJCT5TaY1cA391rv6d/ikQVL30
+Pg8/2spxE6hkFcOySwutUXrW8AHUN3cSnTDbdNGp3//RrHTMgLQzySwJKFBrl1C8
+A5XI1J+EL3FY4TM4XXkIJ7kcgtGPWGTxrMb0aXpsGgb5rO1rxRHxSj5JVH9sZHXo
+aZLhSPHpCq721eP9dBSU7l9mlnB7FBcCAwEAAaOBnjCBmzAdBgNVHQ4EFgQUQ1uP
+F6SOx+aUAQlLKu7vG9wwioowHwYDVR0jBBgwFoAUQ1uPF6SOx+aUAQlLKu7vG9ww
+ioowTgYDVR0RBEcwRYIcbm9kZTguY2x1c3RlcjQuZWxhc3RpY3NlYXJjaKAlBgNV
+BAOgHgwcbm9kZTguY2x1c3RlcjQuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQB2I8SRunyypVG9WOt4ibEXe5MKkEpuIHOqrcF71iwu
+oaperrQdVtbB8mX6qIxIYVzJtjaBP4zJXf604bzabe8lzA5w6xGx+lxirNWj3KfZ
+LEq6DtV05ugGQZYljOm1aSNNlTdv5eQYd0IgSz7QwliaCB8qspH+6tdEYWtmdQnf
+j+9BNtfaIsgii8Q345ok8mNzhwHr+2UDAOjl37DHknFCcCIt+371ogk3kjJ7p0U6
+VUxtlvSSvnhzJQlmbzn/dOdUL7jy9SqKxC2nVeYVrPnOwhUwtYjFbx17/rHU6xQI
+kl4zCvGmLbzG4YBmZ82ZWU/yJDuGv9SWPCi54XBW5E8s
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c4.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c4.key
new file mode 100644
index 000000000000..6ecd96a040aa
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c4.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAnuEtbEC3295c2j8JTXYoocgbygdZ2c/HjiSX/IfB4dGDe9GI
+Tk877ywoXBE5V8PjbhK7r4ZMEnLTwzx6mxpKuCyFX+VEMcx3bbfhBlBuFrPGADEY
+ngIzumVZsBrdcwBSdRPxYK6Dys2rvUDmggWd/AD5NyMu2IkJPlNpjVwDf3Wu/p3+
+KRBUvfQ+Dz/aynETqGQVw7JLC61RetbwAdQ3dxKdMNt00anf/9GsdMyAtDPJLAko
+UGuXULwDlcjUn4QvcVjhMzhdeQgnuRyC0Y9YZPGsxvRpemwaBvms7WvFEfFKPklU
+f2xkdehpkuFI8ekKrvbV4/10FJTuX2aWcHsUFwIDAQABAoIBABh3cWjv7SBfyt6/
+wa1Qb+C8vILNGlTE1DnwgAifLgfvcsG45Qu3f0b3/t3Gt+pAZp/xzz4FfruB5q3J
+0iZb0yaE50563kM+SySgoiI4dUDrvacT6HLqFeXeFloHQ75M+LsXhMlbC+f0P5Pt
+EFJKOSeworA6ASHOteyyKFlg0m0TMkQu9Fs15FvrXHFqSKQ9X+WetMPxBiXNUdXC
+bDjQ2YgUVeDd/gvRSbV295hjqhYC7ewnN+WvtzGXaLr3jwwn6AB5FL0YGJecZ+Wc
+C/uiVfA0yVNrzZertSHGyDPWqufSAOQAyNgNrX/soHqoe2SNuUkEcYZiY3/0CvWI
+2LFdKaECgYEAtbicAp0nm+4iF8AZse8xIopXXdecRA0uZG+w/LOG3WZMFMUSNJna
+G+gu42YuLg+9uzCF4c0vtnhOG56SrGtzQ1mnZ9f0+fbu1higKOYTn+K+DXdHGbjd
+gmcxmefbog5B3QmH5t4kuZsL4GEwzzcW22DQFLhpF9QgFr0HJq7nwScCgYEA39Jr
+L6tFlIBQkESLbUWIMnOK3qGB7DQsxFRWw1vFAqX6/7YRo3iAtnxSJ4tI4yJZ5ngL
++pY3owVuCqxmh3wJ2YG4XBagqqqq1FpzxWLJlEXDD5EGQrS/hPn53ANRNfNEd2T+
+HZRl1rJ1gbq1N7XN2fBl2dSEk8tP2CTTVeUGC5ECgYAboNG3ZAVdo7rzOXWuo9so
+kTfvQJS1k/t1sbWK9hCPILp4fe3iqXGLxKU8VZC32BTUnO/+AKA8cdYORPGv3TZN
+Cxh0pVTNnEl7q4rOLQzWxjZ2/aeFtaGX7fCQobu2y8wfuw86fpO+1F/d4Wj2WD3V
+ZMa+H7t6r/mg+2RW8UizCQKBgQDEAHFq8jJhhCiI3e23SC+n9rTCl26+GUamuzCr
+rX6N/ioaqicyH0GgDKipBnlSpm9RuwjKbafVM9kUsbqrpLy/Y7C9u/cA3anxgjBl
+x0e9d4YE6LwxyDT7Tyk2ORVyCFGJuMAryRuRcwAiFg3B9oyI8NNQeaGMB3wpTL5h
+NskfcQKBgQCD8q3U1wG+VibYn3ehoa5+4F0OkinFrWL+axtrTMNI7rcqKOZlqSmE
+tQfTEXc0A2zpzbOKiegfRn0d85tD6CwjmYjXoXNUGFPg+bzNBX1cffEafgfv/RAH
+adMmqkSHGMaobLdETVgQsM4yd5GHXx2ngP+ywwDWOQUrhp1VXiyzPQ==
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c5.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c5.crt
index 8124b32c7cb0..cc36cd053562 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c5.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c5.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUDXZKYe8N8urRIiGxYze5nudE6ZswDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjguYzUwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0MTMy
-MjE0WjAQMQ4wDAYDVQQDEwVuOC5jNTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAkcBH4VLWVYKTBK6qDEk5PLnxZVyMiQXZHSYKzaOYJONym1lXYkvpXIDcbzwy
-NyFrMbkRu2OX7gJvDJ9RXcZHXmwJMWrYSIYz2wLO56ZJyae/51Hi7Y/3Ov0QpIr8
-Jw0EVt/1UZPhZO3l5tiPp4hWPPFJO3s64gULPTfqcv3vkPECAwEAAaN/MH0wHQYD
-VR0OBBYEFCt7NDuXp23piqYrueIProkyL5uqMB8GA1UdIwQYMBaAFCt7NDuXp23p
-iqYrueIProkyL5uqMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU4LmNsdXN0ZXI1
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQAz0nxM
-esENmZpDl9A5KJascywHWPlNHziEzwV92IWXZTNts8CUixW3VvhPypWMOWOdvo7g
-bI6KQY82WgFT3GN+X5M+qgxLh7R13mRyOW/ncxbc3dpZhKurlW1bPKhchSR57WHr
-1KSKw70C05tqEbMl3CweZ51EwMDq/1IqO5gpuw==
+MIIDTjCCAjagAwIBAgIVAJ8C8GcCcxCloFQSscgM8PaIdkuiMA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW44LmM1MB4XDTIyMTEyNjIxMjMwMVoXDTQyMTEyMTIx
+MjMwMVowEDEOMAwGA1UEAxMFbjguYzUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQDkCUBbRjghReUrCo9FcCXqKWLfLqI/Gi31YuFiaa83Uqk8S/2T0Y1G
+lgjHSJvwvE8hXCwlxGibs+gOXnKtS+S45JBf93H+2O8H6VXsVHNQip8Na3QiyAYB
++t15jbP+9VxrPz9BPsiinnkK8tvyAWRXC36F4STRxWT2lLQZ7j0gk54PCg0uevpF
+kmDeEdcbpppon0se099e8HbCZZM0x2N2Vra1pQwH2yttO3wRaIjLHpbPvHJU8xsw
+ZrxuxDb2fQxRJdXb9+PWPtTE2TJuC1meuUBZwYedBL1rT2mtKHKAVcXWjht69B22
+DLcpniRbgTqFQL65goXuguh0w10Vpj7XAgMBAAGjgZ4wgZswHQYDVR0OBBYEFIi9
+2+E2kl2kLgEleXT1tAHTuBQiMB8GA1UdIwQYMBaAFIi92+E2kl2kLgEleXT1tAHT
+uBQiME4GA1UdEQRHMEWCHG5vZGU4LmNsdXN0ZXI1LmVsYXN0aWNzZWFyY2igJQYD
+VQQDoB4MHG5vZGU4LmNsdXN0ZXI1LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAOAirQCQ1iJpJlvosQ0bY2qUo7G5Wzid4K4GX1nKQ
+K5opmr2lpi5E2pK/CDPzOfw8i3a7K/oqpS69O9gIZHxGtWypkfr+v6vWMSsQBUHm
+0jC2q+n1LK98nIoS5uFax6W7da0e+lARd59i8C7BItqgb7+qd5MfyUwt4Gv+04Wb
+fm3CpD2kM838U+/LuK+TfAcq4pFApftSyFrPODaXKxiN5QIBPGorTAVHvuumyOc/
+6s6NOqkkZDLZo51W6bApfgo693dW+RH07kucQQYyG0hyb5W5M93KbqiW0QoAAXXp
+eNvVk45ZS21JnkUq+ZgQeSh/tTua3FgD3qGebersyIz4ww==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c5.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c5.key
new file mode 100644
index 000000000000..132e7eb70824
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c5.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA5AlAW0Y4IUXlKwqPRXAl6ili3y6iPxot9WLhYmmvN1KpPEv9
+k9GNRpYIx0ib8LxPIVwsJcRom7PoDl5yrUvkuOSQX/dx/tjvB+lV7FRzUIqfDWt0
+IsgGAfrdeY2z/vVcaz8/QT7Iop55CvLb8gFkVwt+heEk0cVk9pS0Ge49IJOeDwoN
+Lnr6RZJg3hHXG6aaaJ9LHtPfXvB2wmWTNMdjdla2taUMB9srbTt8EWiIyx6Wz7xy
+VPMbMGa8bsQ29n0MUSXV2/fj1j7UxNkybgtZnrlAWcGHnQS9a09prShygFXF1o4b
+evQdtgy3KZ4kW4E6hUC+uYKF7oLodMNdFaY+1wIDAQABAoIBADIzVS8izL2KcRJi
+HztGnponpUQDtKUWotJ4A60SHggcUvRauEa/5Y2J/wG8GtRK6FKsQ23HCM5qMyZi
+pXVAFWicrKnrw/U6FZkUxZ0i1EYBVlVyd90mUqhB7VrSlrMEmeVkRY0cALhIEdaW
+s1PZ+ZhuB9r6PwxNG9Zq0yE4QM24Ny1+Co47qfXEt2gc9yKBfeNsE7r9cyAlTLmY
+xlCXUALm+mu6BHrR0fAfoiTfWeqZZQRcVLmpos9gccJXZ+NDh/PSgcN1s6Q+NyEA
+J28M7ySMEUrLrSFtBQFCuiVkMhVtBJ0MLTrmlaKhhJ/0TvOWGki6Y70vmxvrd4GV
+Blq/RgkCgYEA+cwSpg6y9mnS4KAv6pCkS6M6jyK9u07/9Z5VLFqqc1wKa8xLz+3W
+4ssChDSzf/gwf3jx7j5tPcpqwpZrlPwxcwYH3eaEStvb8CrY0W3Q+cNhmdEr6ou0
+ewvyJ1Zq1o4EjlGSMX5FQLIOu3dwt4bLsqoA3un0opyhrUuCFBh2KB8CgYEA6bLY
+vZOOajukS+QywsGzyviRRxPwoNOlpTB+YUYgcMTMnFft+mhi3OuebpGUYW3PmufL
+/v28tjFp2kPIVdgzrRtp0DaLruAA8KngkvtXiHkQaWGzAfLfByEq9zoRwEryOWh0
+oTD8iV33+zDX6PydOU6nF0/zqWT60bDKi0sJckkCgYEAqLPnYFkLCpp7vhLRaaSY
+ITEzKlo3M1puSY7OTS9KcLCXpSgLh3lEqFSc9Xa2R9XxwxWGpvVoR9P9+D8oly0f
+pGRNSdMWQ4xq5K+7UO3Ote/7aau38ia6FGQD/gdCDndJomh6yIxT7xaSFo/bA0e5
+KIfiIKp2+p+4HVaimjnPCtcCgYA+4ZKUH/n396hTi0llQf2ApbVhbKoYfBpcAlpf
+DiF4dlufzcEI2cGQNA3jLmTAem2b/rbTG68FTvJuekKTZqMw9ulXzvVyzm8yUmNM
+6Tg6TmjuEg26foEoR/Jpp0RTe1cFv2dyX93zkEdDXXcbB62+V7159ldXwwYCGhHF
+vSEDkQKBgANovyaW5eX4S1qpyUByLuENM3ss6RkMDG/L/H8kYTgAKG6VzaLMwb9b
+WKManFbKmrXWUXmOL56p6/64TZMGC3nNLYQbHLKP/kYMstmB8L8KJs3OxmzW1Gg0
+owFZjrriIU0amWhMXPtsSCKlIlOOCiwyPhsSldsQCQiFVHV/mc92
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c6.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c6.crt
index dc1bd9c514ef..bcfbb3a90f3c 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c6.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c6.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUBntPQqqFKoP/ZsDltxJ/FYW7x0QwDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjguYzYwHhcNMTgwNDE5MTMyMjE0WhcNNDUwOTA0MTMy
-MjE0WjAQMQ4wDAYDVQQDEwVuOC5jNjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAo8BTGj7hNOsuZUJnVaiuK4LzSMpph4N6WPUXUC9Pt5qN6nUyIOhCCMpaXnqf
-Yl4mJM9Tj0NvDwh2Ar5Kp7O/UIStXHxVOxv+WOq4XTLNZPOOgw8pr9i2BDwp92sE
-xydrnRUX/u4hW+/Fssiw3WpBCGiDb0JJerDLDhGMBVEJW8sCAwEAAaN/MH0wHQYD
-VR0OBBYEFDwuoW6Jy/OB9E4Ztw3fXlA4PjrjMB8GA1UdIwQYMBaAFDwuoW6Jy/OB
-9E4Ztw3fXlA4PjrjMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU4LmNsdXN0ZXI2
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQBJ5uWB
-ctpUMiTKasyyio9jZMMsezuZGNnpOb6ZjwSfX2xbyC3KvNS2Dcb5ajlLaNuRLdqd
-dz/IGzI69cl808fUqsFa9rGJBQvyQQBbtNGE4knQ5pdw3cqJEayMfaliViY0CYya
-t3CmJIB1UZ3/zvbMDhrb8RG+/xYvcagCRpchnQ==
+MIIDTjCCAjagAwIBAgIVAL4ywdFEQ+TioL+txBKJkXuWgMk1MA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW44LmM2MB4XDTIyMTEyNjIxMjMwMVoXDTQyMTEyMTIx
+MjMwMVowEDEOMAwGA1UEAxMFbjguYzYwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQDML8+2lJviVFFKUv+xjzZVxxZMvpUOgA7OboKqs/8mKh3w93eHiSQy
+MJm6TY45xvS7nXByIKFPVU1PjpaRkMMO+h6HaAUlixjHEUaXuwOmmpVHbLg40oWM
+RDBOSCc2nGY0nVSeNQdQMMNJ/8uvWR1NI1pmhEz1ErvYEITYd6pPs52xQsCfYyYV
+3tTtJXh9aOjZxKwfl+H2cd4x8R7IrDp5j8vJ2akfJJg730SvIGIuOPMYnsYZycy9
+jKP+ZifYu5VwcIiCVC97TmyAcB5rJ1pewJADJyRTKJpB4btujdJJObyJXafV7nUn
+T+7OSMvPefUGbWJFEAr5GnRqqzsn/bn9AgMBAAGjgZ4wgZswHQYDVR0OBBYEFHGJ
+dMu/OnZO/kgD5skdYCMn0A7pMB8GA1UdIwQYMBaAFHGJdMu/OnZO/kgD5skdYCMn
+0A7pME4GA1UdEQRHMEWCHG5vZGU4LmNsdXN0ZXI2LmVsYXN0aWNzZWFyY2igJQYD
+VQQDoB4MHG5vZGU4LmNsdXN0ZXI2LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEASW6tzJ2tsGlVz7Y64x1BI3JrW89sBN5eZtYQD16K
+X+j2fkzR/4KP7Szs6+0/fyyvvnamuAujlliZ3SddTc30uNUYJ7sfDz/W475HXaQe
+MqMT78oIO+Osmc8eHEdi8Nv8uR1eF/aqkUfBMFRhcnT3I5hNeBWQGc+vV1AIwB7v
+HKGfOjAUyWdQ/s1roB0g7HZFpwtWnfTw58U9zDkqjsRWL3PfUcboJHOdx6h1bbY7
+AwqEZPcM6XZ0u9/cBghM8n9A4Skv9RQuU/uLd2XtI9fHbPi4DUj7Qi+SxT5zHCw3
+amhvhLk4il8tT8/2Q586JvOpClWqVKiiZMKpQzVGvASBFA==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c6.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c6.key
new file mode 100644
index 000000000000..906b0b5ef9d8
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c6.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAzC/PtpSb4lRRSlL/sY82VccWTL6VDoAOzm6CqrP/Jiod8Pd3
+h4kkMjCZuk2OOcb0u51wciChT1VNT46WkZDDDvoeh2gFJYsYxxFGl7sDppqVR2y4
+ONKFjEQwTkgnNpxmNJ1UnjUHUDDDSf/Lr1kdTSNaZoRM9RK72BCE2HeqT7OdsULA
+n2MmFd7U7SV4fWjo2cSsH5fh9nHeMfEeyKw6eY/LydmpHySYO99EryBiLjjzGJ7G
+GcnMvYyj/mYn2LuVcHCIglQve05sgHAeaydaXsCQAyckUyiaQeG7bo3SSTm8iV2n
+1e51J0/uzkjLz3n1Bm1iRRAK+Rp0aqs7J/25/QIDAQABAoIBABWqDv5exylc6+EY
+bFPBD7FLNdT/tZ37vVZuyENnZK9+cX3TK+YACYyftnfbabc8olPTsQ4S4zqhINpL
+lTMtqlyI8jtIsJ8aoo7giN5pc8o2BZe189fnkk7fH0b3Bd4X+wojFUrR8owyGE5e
+KxTYPjEkip9s8yNfkPmsSwQSsknRYt3BfEoqwp/Blqv8Crf8QQTyQdA+H2yUgZQ8
++MvAYg8pFv/uoO2jSl/Q4QwfcYKzG4qBD1Jf6oYSaH7FeD3nWhXcEq4h7zlFaajm
++fGyJD5/NXQ2TRy7EY7f4F4j3lVRFfOdsqfT0+eY+Gdfriff0jKqKrXDheYwrVBp
+kEBuetECgYEA7xnc1UDC/ouXUwbZDqk4HQF1+1ZF8cVscfPu/tQpSMX/RjpSLdA1
+6H7xmCGvEyRw/6L1UkEOjpiw76LPLhpHOdH1ktDJ/bQyHGmVoS83F5r/oF5LHahX
+yqs0Y/i4sBL1E9SFyx5w1Q5/l95xztdPHvpiI84GhMAsmNyVaglrALkCgYEA2p47
+onMvfJ8N2zGwlQzCZ0Gb/p2MVzaszQXAaDLCwNObfKs1ysjET5IaImaCkg5zzJii
+JqVYj6xmF+bPivgkSJPQ9d66K8TqPuJ1lQw4yl8LlRRdn/MPdgkoqpJf6Zm/fFXx
+yf6xhwJz5QP5nM3UbSPxRmILI2HKxZdLMatneWUCgYBeFzFeAjQ58zjDhYhZSqI4
+V+DT3Wl88RHx7ep5AYiViMUiH8VDHhJ4wavVEDxTLh4Bm2Eq00slNlU6mDkrjU/G
+IyeedOdoHUczAyHM4kWt1mMN+s1YiGyz5pehV7lmIK+s9A8KlZoVXbU8UF885GHQ
+cPu511OdXfqu1O8/YSFokQKBgQDXBM5Ohs/r2Fw+fG7bCkOiXytYkAhaNxJaTA/j
+7BkUoovJYHN10WUThfViZESD9ub6Eh1ZuUKTfNKlK4RxgIHfSFM7oCanqfNiikVv
+pSeuEzFHHVLAYMWSKIvwyCrBjVCCr90x+OkOiuEGHJTDHsaR2coSmZeu4x4WnSl9
+3sHU6QKBgAywmj7z0V0Jbb8Vuo5XzjCyA/ICK9F/TrMSgeq0/Vwf8Ya9pVy0eUz4
+ZQKFgO7TVjLMhbbFZ0KEvXeYr7GDgNrBbl2JdnHBoH/Le5/iDlw7iue3zrtH187x
+mblWcwwAQ2K2gB+CC5DRQrAgvsUwZi2von2AjD7G0XeWy4zNJdlR
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c7.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c7.crt
index 6bdf74aade91..e854f4056d8b 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c7.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c7.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKTCCAZKgAwIBAgIVAITsoW3WKpWQHwqyFSUAIXdectrZMA0GCSqGSIb3DQEB
-CwUAMBAxDjAMBgNVBAMTBW44LmM3MB4XDTE4MDQxOTEzMjIxNVoXDTQ1MDkwNDEz
-MjIxNVowEDEOMAwGA1UEAxMFbjguYzcwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ
-AoGBALgPqDe2FNs+O8NC9tcUqFEHNaey5ax022GQ5qAiykKoNCFD+YLxDQW8xNPq
-2/nJIaQhzymUi/1KdPs14gDw6HXmSUXO9juLv413PQ3B4aVfsOKG09JmUAxLbcoA
-Q2PrdFichWWaYEtudF4RzAOtMNGbj+n4QL7JkEAxg0Zl+9NlAgMBAAGjfzB9MB0G
-A1UdDgQWBBR2lHOcDOVXtV6xMd2mHu91ksElnjAfBgNVHSMEGDAWgBR2lHOcDOVX
-tV6xMd2mHu91ksElnjAwBgNVHREEKTAnoCUGA1UEA6AeDBxub2RlOC5jbHVzdGVy
-Ny5lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADgYEAC8Ta
-BWlhJWv75fq+NPy5glqnSgfAGCvNpjRYcxFbkxSUTZkFq5mTtu0J+Ckk4fj2IzgP
-lF+NeNxaWmzoCfsYOLKs2+pChor3eaFEckVJThDdkg17rYMa5Ym/T/Ph+hXWoI95
-3xm1ZZbCVJhdaZzFcCnSghji/g4aeXlE6hLwBmk=
+MIIDTTCCAjWgAwIBAgIUAcdRKrzoeK5IlqNcp3OX2Me5uswwDQYJKoZIhvcNAQEL
+BQAwEDEOMAwGA1UEAxMFbjguYzcwHhcNMjIxMTI2MjEyMzAxWhcNNDIxMTIxMjEy
+MzAxWjAQMQ4wDAYDVQQDEwVuOC5jNzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBALeGCDYi+SHv46zoXEVNRijcQFPsv+LLnenMBOrSOF+0s+gIUXMQ8ohG
+TNdOgUXt5opDi4XPDp/00Q51+nE8j9t7oLBG4XLMNXY00lFSgRUulblJ2ZJPnEnx
+jvjtZinBd46CeD+iZSNjEvBLgqvNVSiINx892VX4RARnsFTfrnlFv6rzpHgdkBAi
+9hOWvWXRhe4/r3wMu3U/0fDn1SadFEQlNzjm/skQeDLkhMHXWH2ELppJCe72Q4zf
+bBBIVvfi8mmE0ThqAk3Tjq/d3keMrOmAcF1IrcMlQ+bM6u3VQ+Rk/1gxaRyIbkUr
+Fi42PgqCN0AzPygyA6ve/h3JT9lfthcCAwEAAaOBnjCBmzAdBgNVHQ4EFgQU5MLK
+D9bKWe6x1NxJlhtvHcwC4mkwHwYDVR0jBBgwFoAU5MLKD9bKWe6x1NxJlhtvHcwC
+4mkwTgYDVR0RBEcwRYIcbm9kZTguY2x1c3RlcjcuZWxhc3RpY3NlYXJjaKAlBgNV
+BAOgHgwcbm9kZTguY2x1c3RlcjcuZWxhc3RpY3NlYXJjaDAJBgNVHRMEAjAAMA0G
+CSqGSIb3DQEBCwUAA4IBAQChQWJ/f1A+QDqAKvhcRCHgyhuhSONo4e8qwJuRmeHn
+F8KnBnOhirw++eTEuAhRf2H8k9R/gmsPB/3pDs5IjvM9/TUOIcGqHfnZCd3P9SDf
+nkZvrDf7uXt3+ni2whGbfcEZZMzFSNCtEZOAjxe8xfT8BuKpxAiJd3PzVE221XEZ
+DO/Wq8hETKWavUqKaHSR2fkhIeRZXG/hzHrB12g/X5tKLxjN5fezC4KsI27sJc+y
+V0TVku8jYhccqODC1EpiZL3E3XmKNFkG5dQAmbyh15uBxZj1vgZ14JAjBaKLi4sw
+s4+EGZMilaMhfrqhTDkCQ4W8N1GH0STZIywTYrD/CsGE
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c7.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c7.key
new file mode 100644
index 000000000000..c3e48622415b
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c7.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAt4YINiL5Ie/jrOhcRU1GKNxAU+y/4sud6cwE6tI4X7Sz6AhR
+cxDyiEZM106BRe3mikOLhc8On/TRDnX6cTyP23ugsEbhcsw1djTSUVKBFS6VuUnZ
+kk+cSfGO+O1mKcF3joJ4P6JlI2MS8EuCq81VKIg3Hz3ZVfhEBGewVN+ueUW/qvOk
+eB2QECL2E5a9ZdGF7j+vfAy7dT/R8OfVJp0URCU3OOb+yRB4MuSEwddYfYQumkkJ
+7vZDjN9sEEhW9+LyaYTROGoCTdOOr93eR4ys6YBwXUitwyVD5szq7dVD5GT/WDFp
+HIhuRSsWLjY+CoI3QDM/KDIDq97+HclP2V+2FwIDAQABAoIBAEPICA5eml8TZ6Z7
+52BdKvV6a3PTtAdCpt8r0AEehV2rsuqW7+YsszbrqAO5Zft64zPbwJM6Jy+izvel
+UUszT/qDTZTtiOTKUVCIL8mDTFOTcg8OHIJyTsnsop6Dp8DySILk4x61jt8j/2Gq
+sedcHgCrVph+Ul/zIsPxu+mM97aoEEkMkkNXed3A+bxiK374ibN8q0Mo97CHRIS9
+jEbrpSU0wUFjeR1tueoy9EYJY+zsFOctnP98LYVSiN0mde/ykVWcJpqsBiMDcSBq
+2RAEclqMmQplhT9eQONUgK1kUXCb8v8V4qXtFVMM8JG6wgPSCbjOMAcmSD/DBLpG
+p0oaHgECgYEAxN9nGNRHOyiK+PcYk416Rk+vLGrDhvsad8uVJPhThRQWrG/6Gr9b
+MjfYoLxH5Af/JtUqHTbe8paOlGCZXDXo4vb38aN0dAtoX/rhQn54fYoAh5NdFu85
+V3lb+Ju4myuMzjLcMfE0tQY4UwVb8k1aXQMGrTGWsoAvMfKcXxWMAQECgYEA7qRH
+1p3ScqyyINLh8RoCRWzvu7oKFkl8E9hIyeBXM7xdCOU9tbGI8cBm7/Own2f+EVQI
+gvcifBu5QrjW1IfSo67yUbk1TR6UeJAeMaHd9M/c8WtJtjAUEY4IhsK32OiXrjia
+rE3XJyoL8tdWZrPQ85+xjE/eHOmdEn+EicksnxcCgYBHaGzBscWAbK0zrpSIeImA
+VUjJiJQRMqUgDIS1WykYHPg4ZYlHCF9wsJbvfjtpN4lRsYHMBOoK/ZcLtJoLuzYU
+lNXMHKiRqs2Z+qqX7r2Pc23LvyTFu9GFq+wzGXB1P++bnxEFQGMIwg/N9SSrl2kO
+UZmrWlA5wIrdyAz15B/bAQKBgCFL2pofOxW08sWRpuS09Ewrfud6dZ3WcvFx4/YG
+zyPwzCpJkeqf8rOIXNRt897J67EPQIDFapJWa+rbT14n9UPRTQ4KAV3ZPm3z7cKP
+CyD1P6aPiaFZ4/LvFlF3KZ+iFEfCrhIMalCUlUY0tWFvBWB4j5U8L4lpLvk9wYhY
+AVnRAoGAfSJeiGIcZ33paTjlyZanTrc/0NDA9ZihatBoka4u+HuRfUxs+xScNyxf
+j1LcImZpM/raVLbTLIF/ao1CXUwOHw9lKBIReDIs1zTFd0Ir6E7lRd+N/DNKWSux
+q5xg4jEnemiWPJd6jOtM4hC11tLmSvYV9Tw9kn+d0fgu0vBEqv8=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c8.crt b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c8.crt
index 80beac386fcd..44ad4dfb481d 100644
--- a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c8.crt
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c8.crt
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICKDCCAZGgAwIBAgIUWHiDGtjFMmyQe5o/E90cc8TTAJQwDQYJKoZIhvcNAQEL
-BQAwEDEOMAwGA1UEAxMFbjguYzgwHhcNMTgwNDE5MTMyMjE1WhcNNDUwOTA0MTMy
-MjE1WjAQMQ4wDAYDVQQDEwVuOC5jODCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
-gYEAwvRhMUXpnmopSSajgAAcImV5vnRLUB88duRl/j6oM0Oq4xxPoKg1HfSjTnJu
-Ld3mrMuTQPU67KodJoZ1keFoeKQGH1S6pDT+GBI+8IcZMZRJXyeourDB5AklqRVK
-PiciepoVCHuePYZKVlsUnWbH1Vg9yMcYW6Jx6zbnS/meIC0CAwEAAaN/MH0wHQYD
-VR0OBBYEFO5ttLIFOaMQELSrMcKok9nD4PqyMB8GA1UdIwQYMBaAFO5ttLIFOaMQ
-ELSrMcKok9nD4PqyMDAGA1UdEQQpMCegJQYDVQQDoB4MHG5vZGU4LmNsdXN0ZXI4
-LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOBgQCoZw5U
-6LVuXhAIKtncNCTbxfjmeL5JG4GsKb1lS1Xwb5WedO9uAusIXQS2SuLhKvc35EW3
-TJgkDlLfSqtRCx53fwm1cw+HmRRzpmMaiKaJ/K1tdhv7r3bvH+tRVzFHxye2Sai9
-U8gIueP6v/5/T/D6MobHsyXDL9l/flHHdQtsFg==
+MIIDTjCCAjagAwIBAgIVAPVxpd2FAVkJ2f31em0UKJkPQL/0MA0GCSqGSIb3DQEB
+CwUAMBAxDjAMBgNVBAMTBW44LmM4MB4XDTIyMTEyNjIxMjMwMVoXDTQyMTEyMTIx
+MjMwMVowEDEOMAwGA1UEAxMFbjguYzgwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQC14yzjbwwpjWZ5LC+ch4pq2t+UTfzuxObdmHVLXL+a5WoDnid1HCUq
+FitVU2AUs9aaoo/h+eODtAY8GgL+r4E8g4OtQLD8ZXDm7KSXfWDXuUrNOCEHPqbJ
+6BFPe7xBKf/zho4om4/32EP6uWfzNUKTNCxyirOm9EnKKIMorfQyhQ4JmWxxEgqH
+cxdRWF/hBF1mHadGuAfhso2LptskKumXktSX5EzF80efx7P+OaPYu1mBlO/+LEoh
+CktkdqRR1En1R8//9oYd7yUOD0pr1KhTa+6m41NIUmIu0sI3JeHwlpotzdU33vTO
+VCJcd9o9L6yoWqSpU7LOvVnbsby5omMRAgMBAAGjgZ4wgZswHQYDVR0OBBYEFDPd
+ku4cLj+JnrvGDuMqfGONuJUsMB8GA1UdIwQYMBaAFDPdku4cLj+JnrvGDuMqfGON
+uJUsME4GA1UdEQRHMEWgJQYDVQQDoB4MHG5vZGU4LmNsdXN0ZXI4LmVsYXN0aWNz
+ZWFyY2iCHG5vZGU4LmNsdXN0ZXI4LmVsYXN0aWNzZWFyY2gwCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEAtEMjPqElf8dXxypYMv5PksEYtsE2Md8VoREFjieU
+wpW1jnI5WxBo/Tw4CjJwqnv0Uy8XTL7JLTQbbIR3n45ZGu4JW8PGlnTHDPBBLkEQ
+uFDOs7rWpXgKW9PuD6dH4yRXilnOi9P8MRtk+9mT225s8cu5fLxwsT4A7Wpx0F1m
+XV1mxv971t0WOKIK45rEDnnZWfsivWvU7WqqCSnl2VULKm07qO/omoUayrVM4YzB
+z3GP0DVTwEKzwrmIZX742SEnZF1oZ82lwuoB2kBESMM5Nr4sBwk4WBACSqKvPtRy
+DDyQDV0GZBeENCe+ivTJPej3vAPVPvPN2I6KTnh/43ytAw==
 -----END CERTIFICATE-----
diff --git a/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c8.key b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c8.key
new file mode 100644
index 000000000000..463f3fbdd303
--- /dev/null
+++ b/x-pack/plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/nodes/self-signed/n8.c8.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAteMs428MKY1meSwvnIeKatrflE387sTm3Zh1S1y/muVqA54n
+dRwlKhYrVVNgFLPWmqKP4fnjg7QGPBoC/q+BPIODrUCw/GVw5uykl31g17lKzTgh
+Bz6myegRT3u8QSn/84aOKJuP99hD+rln8zVCkzQscoqzpvRJyiiDKK30MoUOCZls
+cRIKh3MXUVhf4QRdZh2nRrgH4bKNi6bbJCrpl5LUl+RMxfNHn8ez/jmj2LtZgZTv
+/ixKIQpLZHakUdRJ9UfP//aGHe8lDg9Ka9SoU2vupuNTSFJiLtLCNyXh8JaaLc3V
+N970zlQiXHfaPS+sqFqkqVOyzr1Z27G8uaJjEQIDAQABAoIBACTlY7hyERNHw/r+
+F91mkFFLO3Hj3RRZXGSktdx3AMYs1fPMbAAhWLegHiO1oUap0XV2Vv4V2Q1hIiek
+dimQVATAv+AQAKb+c+fgJqY1UIUKKQ6WnXqQ/rfz67kGJmLIoItXpvQ9VDfIRsfw
+PtIy4mGmgoKsuLM8GFDaJXsFyITLqaMOBtX4f2czK2GNuZ9zTQE7kPXPmt+kFjos
+DJdDCQaNYLpzVgAXixkHdOonwGtQSB/vwVxpvhAgPAac0K18nCgWeR5SK2kQlRAR
+whN4UggocoSwuH/xSNYK/iJUaXsdnXxOH/hZjHVnQLC4r/6k4rtJMMZX1jKvVpuN
+ZW2TgR0CgYEA5K9rmakCOQTOP8SpqL9AvsEeWgWjVYURyjiZoUBjMprG3Hospjs9
+x0JD5UDMcJqsC03riEkZ05EoBj1KzG6mIA1q1mofdLqTDp46rn3mlTbaoLncQ/t+
+Wt16qKVa3M8Y0Y0h9ZXwaIjYTtyQR2+AcGWmzR9MWdxw0XpiYrRPl+0CgYEAy5zN
+mxeG1xEbRH+K8DRhvDjGUXgMD7yFYgMFeGjSC49jZ/c0CKA8ontD2S81rrG5fYUz
+XfNf1gOTX6D2Muqc0qpXVgej4TzY0tk+ZXo4UV47g3c9rVC7WJkD3mhRjxBwGtg0
+cJX/TgLt6ftmBYRwqzVEkAZgA1FFY0UuWLKLyzUCgYBYkXAU9RX9S1Ut8VbcGiZT
+OqwspV5a8syvCzjE8RD6OmOmZFMClpMYpxtwoKYNPno/bpA4Gke8Xs2njPl5L6l7
+ZHbqT/Oa29+0m6eSiQRS6gnGkfUzByYqvviQdGo6dqLgFc7/NHEFq2pLMLVYVWDj
+Jxtw+hwx+WRgXQvZuofRCQKBgAfo9Yva5CAix/5nPgyK3Gklg37SvRU3a68dEu9U
+kyRj06bMUbcsEm71fvNHBDzfJqebrx0tQUHdLbJ/XFEqECGXAIGVFma/qGS4j9X9
+Y4a1tHYtAbZj9oNAnofbYZ+/Mbu5PoBJM3viXT/DxnjcZkwga6N0u/IlDB8JJv3H
+IespAoGAP5bDdT9WASVMRMKeNlSB90JgN5oDtLK7ccvUEBUB0UCKWUNZUkhWAKp9
+3iLJqMfr8LfDN8+W592tUBZ5V7UXbO+Jojlgjp4CQa1yf3tla0BNXPyeyHbJRhN2
+ZmGlYO03zGne46suhP2MO+EMc1w8dUyMdV6JtfLhLldF3UrEf0g=
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/deprecation/qa/early-deprecation-rest/build.gradle b/x-pack/plugin/deprecation/qa/early-deprecation-rest/build.gradle
index 65e571e9c284..3e0b02e8f71b 100644
--- a/x-pack/plugin/deprecation/qa/early-deprecation-rest/build.gradle
+++ b/x-pack/plugin/deprecation/qa/early-deprecation-rest/build.gradle
@@ -2,7 +2,7 @@ import org.elasticsearch.gradle.util.GradleUtils
 import org.elasticsearch.gradle.internal.info.BuildParams
 
 apply plugin: 'elasticsearch.base-internal-es-plugin'
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 esplugin {
   description 'Deprecated query plugin'
diff --git a/x-pack/plugin/deprecation/qa/rest/build.gradle b/x-pack/plugin/deprecation/qa/rest/build.gradle
index 33ada7c6a432..b7d4757f668c 100644
--- a/x-pack/plugin/deprecation/qa/rest/build.gradle
+++ b/x-pack/plugin/deprecation/qa/rest/build.gradle
@@ -2,7 +2,7 @@ import org.elasticsearch.gradle.util.GradleUtils
 import org.elasticsearch.gradle.internal.info.BuildParams
 
 apply plugin: 'elasticsearch.base-internal-es-plugin'
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 esplugin {
   description 'Deprecated query plugin'
diff --git a/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecatedQueryBuilder.java b/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecatedQueryBuilder.java
index 84e694fa7856..a2400cab1fed 100644
--- a/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecatedQueryBuilder.java
+++ b/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecatedQueryBuilder.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.xpack.deprecation;
 
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -90,7 +90,7 @@ protected boolean doEquals(TestDeprecatedQueryBuilder other) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_EMPTY;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.ZERO;
     }
 }
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java
index 04ded0e4fc42..b042915f755f 100644
--- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java
+++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.logging.RateLimitingFilter;
@@ -100,7 +100,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         final DeprecationIndexingTemplateRegistry templateRegistry = new DeprecationIndexingTemplateRegistry(
             environment.settings(),
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java
index 14252b2adaf1..931e039ca968 100644
--- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java
+++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationInfoAction.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.deprecation;
 
 import org.elasticsearch.ElasticsearchStatusException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
@@ -154,7 +154,7 @@ public Response(StreamInput in) throws IOException {
             clusterSettingsIssues = in.readList(DeprecationIssue::new);
             nodeSettingsIssues = in.readList(DeprecationIssue::new);
             indexSettingsIssues = in.readMapOfLists(StreamInput::readString, DeprecationIssue::new);
-            if (in.getVersion().before(Version.V_7_11_0)) {
+            if (in.getTransportVersion().before(TransportVersion.V_7_11_0)) {
                 List<DeprecationIssue> mlIssues = in.readList(DeprecationIssue::new);
                 pluginSettingsIssues = new HashMap<>();
                 pluginSettingsIssues.put("ml_settings", mlIssues);
@@ -204,7 +204,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeList(clusterSettingsIssues);
             out.writeList(nodeSettingsIssues);
             out.writeMapOfLists(indexSettingsIssues, StreamOutput::writeString, (o, v) -> v.writeTo(o));
-            if (out.getVersion().before(Version.V_7_11_0)) {
+            if (out.getTransportVersion().before(TransportVersion.V_7_11_0)) {
                 out.writeList(pluginSettingsIssues.getOrDefault("ml_settings", Collections.emptyList()));
             } else {
                 out.writeMapOfLists(pluginSettingsIssues, StreamOutput::writeString, (o, v) -> v.writeTo(o));
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java
index b87c4810f20c..0ffcd0c01d18 100644
--- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java
+++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java
@@ -126,9 +126,7 @@ protected final void masterOperation(
                     PLUGIN_CHECKERS,
                     components,
                     new ThreadedActionListener<>(
-                        logger,
-                        client.threadPool(),
-                        ThreadPool.Names.GENERIC,
+                        client.threadPool().generic(),
                         listener.map(
                             deprecationIssues -> DeprecationInfoAction.Response.from(
                                 state,
@@ -140,8 +138,7 @@ protected final void masterOperation(
                                 deprecationIssues,
                                 skipTheseDeprecations
                             )
-                        ),
-                        false
+                        )
                     )
                 );
 
@@ -162,6 +159,7 @@ static void pluginSettingIssues(
             return;
         }
         GroupedActionListener<DeprecationChecker.CheckResult> groupedActionListener = new GroupedActionListener<>(
+            enabledCheckers.size(),
             ActionListener.wrap(
                 checkResults -> listener.onResponse(
                     checkResults.stream()
@@ -170,8 +168,7 @@ static void pluginSettingIssues(
                         )
                 ),
                 listener::onFailure
-            ),
-            enabledCheckers.size()
+            )
         );
         for (DeprecationChecker checker : checkers) {
             checker.check(components, groupedActionListener);
diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionRequestTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionRequestTests.java
index 34703fe6d361..3bba7fc38570 100644
--- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionRequestTests.java
+++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionRequestTests.java
@@ -16,6 +16,11 @@ protected DeprecationInfoAction.Request createTestInstance() {
         return new DeprecationInfoAction.Request(randomAlphaOfLength(10));
     }
 
+    @Override
+    protected DeprecationInfoAction.Request mutateInstance(DeprecationInfoAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<DeprecationInfoAction.Request> instanceReader() {
         return DeprecationInfoAction.Request::new;
diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java
index f2686a37dd23..17b65c0004a2 100644
--- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java
+++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DeprecationInfoActionResponseTests.java
@@ -72,6 +72,11 @@ protected DeprecationInfoAction.Response createTestInstance() {
         return new DeprecationInfoAction.Response(clusterIssues, nodeIssues, indexIssues, pluginIssues);
     }
 
+    @Override
+    protected DeprecationInfoAction.Response mutateInstance(DeprecationInfoAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<DeprecationInfoAction.Response> instanceReader() {
         return DeprecationInfoAction.Response::new;
diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckRequestTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckRequestTests.java
index ea3d5dd4d0f6..a9e6e80df504 100644
--- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckRequestTests.java
+++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckRequestTests.java
@@ -10,8 +10,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
-
 public class NodesDeprecationCheckRequestTests extends AbstractWireSerializingTestCase<NodesDeprecationCheckRequest> {
 
     @Override
@@ -20,7 +18,7 @@ protected Writeable.Reader<NodesDeprecationCheckRequest> instanceReader() {
     }
 
     @Override
-    protected NodesDeprecationCheckRequest mutateInstance(NodesDeprecationCheckRequest instance) throws IOException {
+    protected NodesDeprecationCheckRequest mutateInstance(NodesDeprecationCheckRequest instance) {
         int newSize = randomValueOtherThan(instance.nodesIds().length, () -> randomIntBetween(0, 10));
         String[] newNodeIds = randomArray(newSize, newSize, String[]::new, () -> randomAlphaOfLengthBetween(5, 10));
         return new NodesDeprecationCheckRequest(newNodeIds);
diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckResponseTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckResponseTests.java
index 9ba330cd4ea5..04e9c8efe75d 100644
--- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckResponseTests.java
+++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckResponseTests.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue.Level;
 
-import java.io.IOException;
 import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -42,27 +41,25 @@ protected NodesDeprecationCheckResponse createTestInstance() {
     }
 
     @Override
-    protected NodesDeprecationCheckResponse mutateInstance(NodesDeprecationCheckResponse instance) throws IOException {
+    protected NodesDeprecationCheckResponse mutateInstance(NodesDeprecationCheckResponse instance) {
         int mutate = randomIntBetween(1, 3);
-        switch (mutate) {
+        return switch (mutate) {
             case 1 -> {
                 List<NodesDeprecationCheckAction.NodeResponse> responses = new ArrayList<>(instance.getNodes());
                 responses.add(randomNodeResponse());
-                return new NodesDeprecationCheckResponse(instance.getClusterName(), responses, instance.failures());
+                yield new NodesDeprecationCheckResponse(instance.getClusterName(), responses, instance.failures());
             }
             case 2 -> {
                 ArrayList<FailedNodeException> failures = new ArrayList<>(instance.failures());
                 failures.add(new FailedNodeException("test node", "test failure", new RuntimeException(randomAlphaOfLength(10))));
-                return new NodesDeprecationCheckResponse(instance.getClusterName(), instance.getNodes(), failures);
+                yield new NodesDeprecationCheckResponse(instance.getClusterName(), instance.getNodes(), failures);
             }
             case 3 -> {
                 String clusterName = randomValueOtherThan(instance.getClusterName().value(), () -> randomAlphaOfLengthBetween(5, 15));
-                return new NodesDeprecationCheckResponse(new ClusterName(clusterName), instance.getNodes(), instance.failures());
+                yield new NodesDeprecationCheckResponse(new ClusterName(clusterName), instance.getNodes(), instance.failures());
             }
-            default -> fail("invalid mutation");
-        }
-
-        return super.mutateInstance(instance);
+            default -> throw new AssertionError("invalid mutation");
+        };
     }
 
     private static DiscoveryNode randomDiscoveryNode() throws Exception {
diff --git a/x-pack/plugin/enrich/qa/rest-with-advanced-security/build.gradle b/x-pack/plugin/enrich/qa/rest-with-advanced-security/build.gradle
index b2ddbc8aebf4..2e649e718b08 100644
--- a/x-pack/plugin/enrich/qa/rest-with-advanced-security/build.gradle
+++ b/x-pack/plugin/enrich/qa/rest-with-advanced-security/build.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 import org.elasticsearch.gradle.internal.info.BuildParams
 
diff --git a/x-pack/plugin/enrich/qa/rest-with-security/build.gradle b/x-pack/plugin/enrich/qa/rest-with-security/build.gradle
index a016c47de614..69fec4ad32c7 100644
--- a/x-pack/plugin/enrich/qa/rest-with-security/build.gradle
+++ b/x-pack/plugin/enrich/qa/rest-with-security/build.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 import org.elasticsearch.gradle.internal.info.BuildParams
 
diff --git a/x-pack/plugin/enrich/qa/rest-with-security/src/javaRestTest/java/org/elasticsearch/xpack/enrich/EnrichSecurityIT.java b/x-pack/plugin/enrich/qa/rest-with-security/src/javaRestTest/java/org/elasticsearch/xpack/enrich/EnrichSecurityIT.java
index 07257464a51c..1d3a1c8b9356 100644
--- a/x-pack/plugin/enrich/qa/rest-with-security/src/javaRestTest/java/org/elasticsearch/xpack/enrich/EnrichSecurityIT.java
+++ b/x-pack/plugin/enrich/qa/rest-with-security/src/javaRestTest/java/org/elasticsearch/xpack/enrich/EnrichSecurityIT.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.enrich.CommonEnrichRestTestCase;
 
 import static org.hamcrest.CoreMatchers.containsString;
@@ -33,7 +34,7 @@ public void testInsufficientPermissionsOnNonExistentIndex() throws Exception {
         // This test is here because it requires a valid user that has permission to execute policy PUTs but should fail if the user
         // does not have access to read the backing indices used to enrich the data.
         Request request = new Request("PUT", "/some-other-index");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {
              "mappings" : {%s}
             }""", createSourceIndexMapping()));
diff --git a/x-pack/plugin/enrich/qa/rest/build.gradle b/x-pack/plugin/enrich/qa/rest/build.gradle
index 697144fc81be..e8473d15ed9e 100644
--- a/x-pack/plugin/enrich/qa/rest/build.gradle
+++ b/x-pack/plugin/enrich/qa/rest/build.gradle
@@ -1,6 +1,6 @@
-apply plugin: 'elasticsearch.internal-java-rest-test'
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 
 import org.elasticsearch.gradle.Version
diff --git a/x-pack/plugin/enrich/src/internalClusterTest/java/org/elasticsearch/xpack/enrich/EnrichMultiNodeIT.java b/x-pack/plugin/enrich/src/internalClusterTest/java/org/elasticsearch/xpack/enrich/EnrichMultiNodeIT.java
index 705313fce285..401857f50b93 100644
--- a/x-pack/plugin/enrich/src/internalClusterTest/java/org/elasticsearch/xpack/enrich/EnrichMultiNodeIT.java
+++ b/x-pack/plugin/enrich/src/internalClusterTest/java/org/elasticsearch/xpack/enrich/EnrichMultiNodeIT.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.ingest.common.IngestCommonPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.reindex.ReindexPlugin;
@@ -313,7 +314,7 @@ private static void createAndExecutePolicy() {
     }
 
     private static void createPipeline() {
-        String pipelineBody = formatted("""
+        String pipelineBody = Strings.format("""
             {
               "processors": [ { "enrich": { "policy_name": "%s", "field": "%s", "target_field": "user" } } ]
             }""", POLICY_NAME, MATCH_FIELD);
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichMetadata.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichMetadata.java
index 83b4aff08879..b2b3cc780e4e 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichMetadata.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichMetadata.java
@@ -7,14 +7,15 @@
 package org.elasticsearch.xpack.enrich;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.AbstractNamedDiffable;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.enrich.EnrichPolicy;
 
@@ -22,6 +23,7 @@
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Objects;
 
@@ -81,8 +83,8 @@ public EnumSet<Metadata.XContentContext> context() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_5_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_5_0;
     }
 
     @Override
@@ -96,15 +98,8 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(POLICIES.getPreferredName());
-        for (Map.Entry<String, EnrichPolicy> entry : policies.entrySet()) {
-            builder.startObject(entry.getKey());
-            builder.value(entry.getValue());
-            builder.endObject();
-        }
-        builder.endObject();
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return ChunkedToXContentHelper.xContentFragmentValuesMap(POLICIES.getPreferredName(), policies);
     }
 
     @Override
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java
index 5e4bc9879402..6bd1e7ad6ff9 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -201,7 +201,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver expressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         EnrichPolicyLocks enrichPolicyLocks = new EnrichPolicyLocks();
         EnrichPolicyExecutor enrichPolicyExecutor = new EnrichPolicyExecutor(
diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichCoordinatorStatsAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichCoordinatorStatsAction.java
index ab5f2289511d..19c5bbc8270e 100644
--- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichCoordinatorStatsAction.java
+++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichCoordinatorStatsAction.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.enrich.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.ActionFilters;
@@ -101,7 +101,9 @@ public static class NodeResponse extends BaseNodeResponse {
 
         NodeResponse(StreamInput in) throws IOException {
             super(in);
-            this.cacheStats = in.getVersion().onOrAfter(Version.V_7_16_0) ? new EnrichStatsAction.Response.CacheStats(in) : null;
+            this.cacheStats = in.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)
+                ? new EnrichStatsAction.Response.CacheStats(in)
+                : null;
             this.coordinatorStats = new CoordinatorStats(in);
         }
 
@@ -116,7 +118,7 @@ public EnrichStatsAction.Response.CacheStats getCacheStats() {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            if (out.getVersion().onOrAfter(Version.V_7_16_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
                 cacheStats.writeTo(out);
             }
             coordinatorStats.writeTo(out);
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java
index 5343c049bc8c..da9102c20e04 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.ingest.common.IngestCommonPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.reindex.ReindexPlugin;
@@ -97,7 +98,7 @@ public void testIngestDataWithMatchProcessor() {
         client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(policyName)).actionGet();
 
         String pipelineName = "my-pipeline";
-        String pipelineBody = formatted("""
+        String pipelineBody = Strings.format("""
             {
               "processors": [
                 {
@@ -190,7 +191,7 @@ public void testIngestDataWithGeoMatchProcessor() {
         client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(policyName)).actionGet();
 
         String pipelineName = "my-pipeline";
-        String pipelineBody = formatted("""
+        String pipelineBody = Strings.format("""
             {
               "processors": [
                 {
@@ -250,7 +251,7 @@ public void testMultiplePolicies() {
             client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(policyName)).actionGet();
 
             String pipelineName = "pipeline" + i;
-            String pipelineBody = formatted("""
+            String pipelineBody = Strings.format("""
                 {
                   "processors": [ { "enrich": { "policy_name": "%s", "field": "key", "target_field": "target" } } ]
                 }""", policyName);
@@ -308,7 +309,7 @@ public void testAsyncTaskExecute() throws Exception {
         });
 
         String pipelineName = "test-pipeline";
-        String pipelineBody = formatted("""
+        String pipelineBody = Strings.format("""
             {
               "processors": [ { "enrich": { "policy_name": "%s", "field": "key", "target_field": "target" } } ]
             }""", policyName);
@@ -350,7 +351,7 @@ public void testTemplating() throws Exception {
         client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(policyName)).actionGet();
 
         String pipelineName = "my-pipeline";
-        String pipelineBody = formatted(
+        String pipelineBody = Strings.format(
             """
                 {"processors": [{"enrich": {"policy_name":"%s", "field": "{{indirection1}}", "target_field": "{{indirection2}}"}}]}""",
             policyName
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichMetadataTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichMetadataTests.java
index 6057255bd456..4d863de41147 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichMetadataTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichMetadataTests.java
@@ -8,7 +8,7 @@
 
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.enrich.EnrichPolicy;
@@ -19,7 +19,7 @@
 import static org.elasticsearch.xpack.enrich.EnrichPolicyTests.randomEnrichPolicy;
 import static org.hamcrest.Matchers.equalTo;
 
-public class EnrichMetadataTests extends AbstractXContentSerializingTestCase<EnrichMetadata> {
+public class EnrichMetadataTests extends AbstractChunkedSerializingTestCase<EnrichMetadata> {
 
     @Override
     protected EnrichMetadata doParseInstance(XContentParser parser) throws IOException {
@@ -31,6 +31,11 @@ protected EnrichMetadata createTestInstance() {
         return randomEnrichMetadata(randomFrom(XContentType.values()));
     }
 
+    @Override
+    protected EnrichMetadata mutateInstance(EnrichMetadata instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected EnrichMetadata createXContextTestInstance(XContentType xContentType) {
         return randomEnrichMetadata(xContentType);
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java
index 90a9462f770d..5792e6e123ef 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java
@@ -38,6 +38,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.engine.Segment;
 import org.elasticsearch.index.mapper.MapperService;
@@ -429,7 +430,7 @@ public void testRunnerMultiSource() throws Exception {
         for (int idx = 0; idx < numberOfSourceIndices; idx++) {
             final String sourceIndex = baseSourceName + idx;
             IndexResponse indexRequest = client().index(
-                new IndexRequest().index(sourceIndex).id(randomAlphaOfLength(10)).source(formatted("""
+                new IndexRequest().index(sourceIndex).id(randomAlphaOfLength(10)).source(Strings.format("""
                     {
                       "idx": %s,
                       "key": "key%s",
@@ -518,7 +519,7 @@ public void testRunnerMultiSourceDocIdCollisions() throws Exception {
         for (int idx = 0; idx < numberOfSourceIndices; idx++) {
             final String sourceIndex = baseSourceName + idx;
             IndexResponse indexRequest = client().index(
-                new IndexRequest().index(sourceIndex).id(collidingDocId).routing(collidingDocId + idx).source(formatted("""
+                new IndexRequest().index(sourceIndex).id(collidingDocId).routing(collidingDocId + idx).source(Strings.format("""
                     {
                       "idx": %s,
                       "key": "key%s",
@@ -622,7 +623,7 @@ public void testRunnerMultiSourceEnrichKeyCollisions() throws Exception {
         for (int idx = 0; idx < numberOfSourceIndices; idx++) {
             final String sourceIndex = baseSourceName + idx;
             IndexResponse indexRequest = client().index(
-                new IndexRequest().index(sourceIndex).id(randomAlphaOfLength(10)).source(formatted("""
+                new IndexRequest().index(sourceIndex).id(randomAlphaOfLength(10)).source(Strings.format("""
                     {
                       "idx": %s,
                       "key": "key",
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyTests.java
index a321c9f86a7f..5c8796865aee 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyTests.java
@@ -41,6 +41,11 @@ protected EnrichPolicy createTestInstance() {
         return randomEnrichPolicy(randomFrom(XContentType.values()));
     }
 
+    @Override
+    protected EnrichPolicy mutateInstance(EnrichPolicy instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected EnrichPolicy createXContextTestInstance(XContentType xContentType) {
         return randomEnrichPolicy(xContentType);
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java
index f664a5ebd8b7..5a8649f57e4c 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.ingest.IngestDocument;
@@ -335,7 +336,7 @@ static Metadata createMetadata(String name, EnrichPolicy policy) throws IOExcept
             .build();
         IndexMetadata.Builder builder = IndexMetadata.builder(EnrichPolicy.getBaseName(name) + "-1");
         builder.settings(settings);
-        builder.putMapping(formatted("""
+        builder.putMapping(Strings.format("""
             {"_meta": {"enrich_match_field": "%s", "enrich_policy_type": "%s"}}
             """, policy.getMatchField(), policy.getType()));
         builder.putAlias(AliasMetadata.builder(EnrichPolicy.getBaseName(name)).build());
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichResiliencyTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichResiliencyTests.java
index dc664835fdb4..049684a2c778 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichResiliencyTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichResiliencyTests.java
@@ -52,7 +52,7 @@ protected Settings nodeSettings() {
             .put(EnrichPlugin.CACHE_SIZE.getKey(), 0)
             .put(EnrichPlugin.COORDINATOR_PROXY_MAX_CONCURRENT_REQUESTS.getKey(), 1)
             .put(EnrichPlugin.COORDINATOR_PROXY_MAX_LOOKUPS_PER_REQUEST.getKey(), 1)
-            .put(EnrichPlugin.COORDINATOR_PROXY_QUEUE_CAPACITY.getKey(), 10)
+            .put(EnrichPlugin.COORDINATOR_PROXY_QUEUE_CAPACITY.getKey(), 2)
             // TODO Fix the test so that it runs with security enabled
             // https://github.com/elastic/elasticsearch/issues/75940
             .put(XPackSettings.SECURITY_ENABLED.getKey(), false)
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/DeleteEnrichPolicyActionRequestTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/DeleteEnrichPolicyActionRequestTests.java
index c59d03ef705f..2e778d6b6221 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/DeleteEnrichPolicyActionRequestTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/DeleteEnrichPolicyActionRequestTests.java
@@ -16,6 +16,11 @@ protected DeleteEnrichPolicyAction.Request createTestInstance() {
         return new DeleteEnrichPolicyAction.Request(randomAlphaOfLength(4));
     }
 
+    @Override
+    protected DeleteEnrichPolicyAction.Request mutateInstance(DeleteEnrichPolicyAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<DeleteEnrichPolicyAction.Request> instanceReader() {
         return DeleteEnrichPolicyAction.Request::new;
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/EnrichStatsResponseTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/EnrichStatsResponseTests.java
index f7cacd1fc374..bcaeb0e626bc 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/EnrichStatsResponseTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/EnrichStatsResponseTests.java
@@ -50,6 +50,11 @@ protected EnrichStatsAction.Response createTestInstance() {
         return new EnrichStatsAction.Response(executingPolicies, coordinatorStats, cacheStats);
     }
 
+    @Override
+    protected EnrichStatsAction.Response mutateInstance(EnrichStatsAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<EnrichStatsAction.Response> instanceReader() {
         return EnrichStatsAction.Response::new;
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/ExecuteEnrichPolicyActionRequestTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/ExecuteEnrichPolicyActionRequestTests.java
index ecc0157a3eeb..a945f49ac97d 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/ExecuteEnrichPolicyActionRequestTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/ExecuteEnrichPolicyActionRequestTests.java
@@ -17,6 +17,11 @@ protected ExecuteEnrichPolicyAction.Request createTestInstance() {
         return new ExecuteEnrichPolicyAction.Request(randomAlphaOfLength(3));
     }
 
+    @Override
+    protected ExecuteEnrichPolicyAction.Request mutateInstance(ExecuteEnrichPolicyAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<ExecuteEnrichPolicyAction.Request> instanceReader() {
         return ExecuteEnrichPolicyAction.Request::new;
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/GetEnrichPolicyActionRequestTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/GetEnrichPolicyActionRequestTests.java
index 24864466b987..f84b72727bca 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/GetEnrichPolicyActionRequestTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/GetEnrichPolicyActionRequestTests.java
@@ -17,6 +17,11 @@ protected GetEnrichPolicyAction.Request createTestInstance() {
         return new GetEnrichPolicyAction.Request(generateRandomStringArray(0, 4, false));
     }
 
+    @Override
+    protected GetEnrichPolicyAction.Request mutateInstance(GetEnrichPolicyAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<GetEnrichPolicyAction.Request> instanceReader() {
         return GetEnrichPolicyAction.Request::new;
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/GetEnrichPolicyActionResponseTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/GetEnrichPolicyActionResponseTests.java
index 026a9f2882fb..c46005163fa1 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/GetEnrichPolicyActionResponseTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/GetEnrichPolicyActionResponseTests.java
@@ -55,6 +55,11 @@ protected GetEnrichPolicyAction.Response createTestInstance() {
         return new GetEnrichPolicyAction.Response(items);
     }
 
+    @Override
+    protected GetEnrichPolicyAction.Response mutateInstance(GetEnrichPolicyAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<GetEnrichPolicyAction.Response> instanceReader() {
         return GetEnrichPolicyAction.Response::new;
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/PutEnrichPolicyActionRequestTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/PutEnrichPolicyActionRequestTests.java
index 8ae0870bd308..7675524435d2 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/PutEnrichPolicyActionRequestTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/PutEnrichPolicyActionRequestTests.java
@@ -22,6 +22,11 @@ protected PutEnrichPolicyAction.Request createTestInstance() {
         return new PutEnrichPolicyAction.Request(randomAlphaOfLength(3), policy);
     }
 
+    @Override
+    protected PutEnrichPolicyAction.Request mutateInstance(PutEnrichPolicyAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<PutEnrichPolicyAction.Request> instanceReader() {
         return PutEnrichPolicyAction.Request::new;
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/monitoring/collector/enrich/EnrichCoordinatorDocTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/monitoring/collector/enrich/EnrichCoordinatorDocTests.java
index 948cb5c62658..d5560f910177 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/monitoring/collector/enrich/EnrichCoordinatorDocTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/monitoring/collector/enrich/EnrichCoordinatorDocTests.java
@@ -83,7 +83,7 @@ public void testToXContent() throws IOException {
             xContent.utf8ToString(),
             equalTo(
                 XContentHelper.stripWhitespace(
-                    formatted(
+                    Strings.format(
                         """
                             {
                               "cluster_uuid": "_cluster",
diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/monitoring/collector/enrich/ExecutingPolicyDocTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/monitoring/collector/enrich/ExecutingPolicyDocTests.java
index 84880b1cb12a..f56024c22e33 100644
--- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/monitoring/collector/enrich/ExecutingPolicyDocTests.java
+++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/monitoring/collector/enrich/ExecutingPolicyDocTests.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.xpack.monitoring.exporter.BaseMonitoringDocTestCase;
 
 import java.io.IOException;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 
@@ -74,62 +73,54 @@ public void testToXContent() throws IOException {
         final ExecutingPolicyDoc document = new ExecutingPolicyDoc("_cluster", timestamp, intervalMillis, node, executingPolicy);
         final BytesReference xContent = XContentHelper.toXContent(document, XContentType.JSON, false);
         Optional<Map.Entry<String, String>> header = executingPolicy.getTaskInfo().headers().entrySet().stream().findAny();
-        assertThat(
-            xContent.utf8ToString(),
-            equalTo(
-                XContentHelper.stripWhitespace(
-                    formatted(
-                        """
-                            {
-                              "cluster_uuid": "_cluster",
-                              "timestamp": "%s",
-                              "interval_ms": %s,
-                              "type": "enrich_executing_policy_stats",
-                              "source_node": {
-                                "uuid": "_uuid",
-                                "host": "_host",
-                                "transport_address": "_addr",
-                                "ip": "_ip",
-                                "name": "_name",
-                                "timestamp": "%s"
-                              },
-                              "enrich_executing_policy_stats": {
-                                "name": "%s",
-                                "task": {
-                                  "node": "%s",
-                                  "id": %s,
-                                  "type": "%s",
-                                  "action": "%s",
-                                  "description": "%s",
-                                  "start_time_in_millis": %s,
-                                  "running_time_in_nanos": %s,
-                                  "cancellable": %s,
-                                  %s
-                                  "headers": %s
-                                }
-                              }
-                            }""",
-                        DATE_TIME_FORMATTER.formatMillis(timestamp),
-                        intervalMillis,
-                        DATE_TIME_FORMATTER.formatMillis(nodeTimestamp),
-                        executingPolicy.getName(),
-                        executingPolicy.getTaskInfo().taskId().getNodeId(),
-                        executingPolicy.getTaskInfo().taskId().getId(),
-                        executingPolicy.getTaskInfo().type(),
-                        executingPolicy.getTaskInfo().action(),
-                        executingPolicy.getTaskInfo().description(),
-                        executingPolicy.getTaskInfo().startTime(),
-                        executingPolicy.getTaskInfo().runningTimeNanos(),
-                        executingPolicy.getTaskInfo().cancellable(),
-                        executingPolicy.getTaskInfo().cancellable()
-                            ? formatted("\"cancelled\": %s,", executingPolicy.getTaskInfo().cancelled())
-                            : "",
-                        header.map(entry -> String.format(Locale.ROOT, """
-                            {"%s":"%s"}""", entry.getKey(), entry.getValue())).orElse("{}")
-                    )
-                )
-            )
-        );
+        Object[] args = new Object[] {
+            DATE_TIME_FORMATTER.formatMillis(timestamp),
+            intervalMillis,
+            DATE_TIME_FORMATTER.formatMillis(nodeTimestamp),
+            executingPolicy.getName(),
+            executingPolicy.getTaskInfo().taskId().getNodeId(),
+            executingPolicy.getTaskInfo().taskId().getId(),
+            executingPolicy.getTaskInfo().type(),
+            executingPolicy.getTaskInfo().action(),
+            executingPolicy.getTaskInfo().description(),
+            executingPolicy.getTaskInfo().startTime(),
+            executingPolicy.getTaskInfo().runningTimeNanos(),
+            executingPolicy.getTaskInfo().cancellable(),
+            executingPolicy.getTaskInfo().cancellable()
+                ? Strings.format("\"cancelled\": %s,", executingPolicy.getTaskInfo().cancelled())
+                : "",
+            header.map(entry -> { return Strings.format("""
+                {"%s":"%s"}""", entry.getKey(), entry.getValue()); }).orElse("{}") };
+        assertThat(xContent.utf8ToString(), equalTo(XContentHelper.stripWhitespace(Strings.format("""
+            {
+              "cluster_uuid": "_cluster",
+              "timestamp": "%s",
+              "interval_ms": %s,
+              "type": "enrich_executing_policy_stats",
+              "source_node": {
+                "uuid": "_uuid",
+                "host": "_host",
+                "transport_address": "_addr",
+                "ip": "_ip",
+                "name": "_name",
+                "timestamp": "%s"
+              },
+              "enrich_executing_policy_stats": {
+                "name": "%s",
+                "task": {
+                  "node": "%s",
+                  "id": %s,
+                  "type": "%s",
+                  "action": "%s",
+                  "description": "%s",
+                  "start_time_in_millis": %s,
+                  "running_time_in_nanos": %s,
+                  "cancellable": %s,
+                  %s
+                  "headers": %s
+                }
+              }
+            }""", args))));
     }
 
     public void testEnrichCoordinatorStatsFieldsMapped() throws IOException {
diff --git a/x-pack/plugin/eql/build.gradle b/x-pack/plugin/eql/build.gradle
index e23d7406662f..ac149581b359 100644
--- a/x-pack/plugin/eql/build.gradle
+++ b/x-pack/plugin/eql/build.gradle
@@ -8,11 +8,6 @@ esplugin {
   extendedPlugins = ['x-pack-ql', 'lang-painless']
 }
 
-ext {
-  // EQL dependency versions
-  antlrVersion = "4.9.2"
-}
-
 archivesBaseName = 'x-pack-eql'
 
 dependencies {
@@ -51,7 +46,7 @@ configurations {
 }
 
 dependencies {
-  regenerate "org.antlr:antlr4:${antlrVersion}"
+  regenerate "org.antlr:antlr4:${versions.antlr4}"
 }
 
 String grammarPath = 'src/main/antlr'
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java
index 78cd15289964..3227ef989222 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java
@@ -51,6 +51,7 @@ public abstract class BaseEqlSpecTestCase extends RemoteClusterAwareEqlRestTestC
      * any negative value means undefined (ie. no "size" will be passed to the query)
      */
     private final int size;
+    private final int maxSamplesPerKey;
 
     @Before
     public void setup() throws Exception {
@@ -102,13 +103,23 @@ protected static List<Object[]> asArray(List<EqlSpec> specs) {
                 name = "" + (counter);
             }
 
-            results.add(new Object[] { spec.query(), name, spec.expectedEventIds(), spec.joinKeys(), spec.size() });
+            results.add(
+                new Object[] { spec.query(), name, spec.expectedEventIds(), spec.joinKeys(), spec.size(), spec.maxSamplesPerKey() }
+            );
         }
 
         return results;
     }
 
-    BaseEqlSpecTestCase(String index, String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size) {
+    BaseEqlSpecTestCase(
+        String index,
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey
+    ) {
         this.index = index;
 
         this.query = query;
@@ -116,6 +127,7 @@ protected static List<Object[]> asArray(List<EqlSpec> specs) {
         this.eventIds = eventIds;
         this.joinKeys = joinKeys;
         this.size = size == null ? -1 : size;
+        this.maxSamplesPerKey = maxSamplesPerKey == null ? -1 : maxSamplesPerKey;
     }
 
     public void test() throws Exception {
@@ -148,6 +160,9 @@ protected ObjectPath runQuery(String index, String query) throws Exception {
         builder.field("size", this.size < 0 ? requestSize() : this.size);
         builder.field("fetch_size", requestFetchSize());
         builder.field("result_position", requestResultPosition());
+        if (maxSamplesPerKey > 0) {
+            builder.field("max_samples_per_key", maxSamplesPerKey);
+        }
         builder.endObject();
 
         Request request = new Request("POST", "/" + index + "/_eql/search");
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/DataLoader.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/DataLoader.java
index d630f24257ee..c2517f549e50 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/DataLoader.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/DataLoader.java
@@ -58,6 +58,7 @@ public class DataLoader {
     public static final String TEST_EXTRA_INDEX = "extra";
     public static final String TEST_NANOS_INDEX = "endgame-140-nanos";
     public static final String TEST_SAMPLE = "sample1,sample2,sample3";
+    public static final String TEST_SAMPLE_MULTI = "sample-multi";
 
     private static final Map<String, String[]> replacementPatterns = Collections.unmodifiableMap(getReplacementPatterns());
 
@@ -102,6 +103,7 @@ public static void loadDatasetIntoEs(
         // There are mixed values with and without nanos precision so that the filtering is properly tested for both cases.
         load(client, TEST_NANOS_INDEX, TEST_INDEX, DataLoader::timestampToUnixNanos, p);
         load(client, TEST_SAMPLE, null, null, p);
+        load(client, TEST_SAMPLE_MULTI, null, null, p);
     }
 
     private static void load(
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlDateNanosSpecTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlDateNanosSpecTestCase.java
index 8036db997198..a368ea0d22aa 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlDateNanosSpecTestCase.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlDateNanosSpecTestCase.java
@@ -21,13 +21,28 @@ public static List<Object[]> readTestSpecs() throws Exception {
     }
 
     // constructor for "local" rest tests
-    public EqlDateNanosSpecTestCase(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size) {
-        this(TEST_NANOS_INDEX, query, name, eventIds, joinKeys, size);
+    public EqlDateNanosSpecTestCase(
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey
+    ) {
+        this(TEST_NANOS_INDEX, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
     }
 
     // constructor for multi-cluster tests
-    public EqlDateNanosSpecTestCase(String index, String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size) {
-        super(index, query, name, eventIds, joinKeys, size);
+    public EqlDateNanosSpecTestCase(
+        String index,
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey
+    ) {
+        super(index, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
     }
 
     @Override
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlExtraSpecTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlExtraSpecTestCase.java
index c14aaa497336..292fe6c895ce 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlExtraSpecTestCase.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlExtraSpecTestCase.java
@@ -21,13 +21,28 @@ public static List<Object[]> readTestSpecs() throws Exception {
     }
 
     // constructor for "local" rest tests
-    public EqlExtraSpecTestCase(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size) {
-        this(TEST_EXTRA_INDEX, query, name, eventIds, joinKeys, size);
+    public EqlExtraSpecTestCase(
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey
+    ) {
+        this(TEST_EXTRA_INDEX, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
     }
 
     // constructor for multi-cluster tests
-    public EqlExtraSpecTestCase(String index, String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size) {
-        super(index, query, name, eventIds, joinKeys, size);
+    public EqlExtraSpecTestCase(
+        String index,
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey
+    ) {
+        super(index, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
     }
 
     @Override
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlRestTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlRestTestCase.java
index 4dca5bd453c3..074fce78e6b9 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlRestTestCase.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlRestTestCase.java
@@ -58,7 +58,10 @@ public void checkSearchContent() throws Exception {
             """, validQuery), "filter doesn't support values of type: VALUE_NULL" },
         { String.format(Locale.ROOT, """
             {"query": "%s", "filter": {}}
-            """, validQuery), "query malformed, empty clause found" } };
+            """, validQuery), "query malformed, empty clause found" },
+        { String.format(Locale.ROOT, """
+            {"query": "%s", "max_samples_per_key": 0}
+            """, validQuery), "max_samples_per_key must be greater than 0" } };
 
     public void testBadRequests() throws Exception {
         createIndex(defaultValidationIndexName, (String) null);
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlRestValidationTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlRestValidationTestCase.java
index ad509c262ab4..a011231729f3 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlRestValidationTestCase.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlRestValidationTestCase.java
@@ -54,10 +54,14 @@ public void prepareIndices() throws IOException {
 
     protected abstract String getInexistentIndexErrorMessage();
 
+    protected String getInexistentWildcardErrorMessage() {
+        return getInexistentIndexErrorMessage();
+    }
+
     protected abstract void assertErrorMessageWhenAllowNoIndicesIsFalse(String reqParameter) throws IOException;
 
     public void testDefaultIndicesOptions() throws IOException {
-        assertErrorMessages(inexistentIndexNameWithWildcard, EMPTY, getInexistentIndexErrorMessage());
+        assertErrorMessages(inexistentIndexNameWithWildcard, EMPTY, getInexistentWildcardErrorMessage());
         assertErrorMessages(inexistentIndexNameWithoutWildcard, EMPTY, getInexistentIndexErrorMessage());
         assertValidRequestOnIndices(existentIndexWithWildcard, EMPTY);
         assertValidRequestOnIndices(existentIndexWithoutWildcard, EMPTY);
@@ -70,7 +74,7 @@ public void testAllowNoIndicesOption() throws IOException {
         String reqParameter = setAllowNoIndices ? "?allow_no_indices=" + allowNoIndices : EMPTY;
 
         if (isAllowNoIndices) {
-            assertErrorMessages(inexistentIndexNameWithWildcard, reqParameter, getInexistentIndexErrorMessage());
+            assertErrorMessages(inexistentIndexNameWithWildcard, reqParameter, getInexistentWildcardErrorMessage());
             assertErrorMessages(inexistentIndexNameWithoutWildcard, reqParameter, getInexistentIndexErrorMessage());
             assertValidRequestOnIndices(existentIndexWithWildcard, reqParameter);
             assertValidRequestOnIndices(existentIndexWithoutWildcard, reqParameter);
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleMultipleEntriesTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleMultipleEntriesTestCase.java
new file mode 100644
index 000000000000..6471e264a92f
--- /dev/null
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleMultipleEntriesTestCase.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.test.eql;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import java.util.List;
+
+import static org.elasticsearch.test.eql.DataLoader.TEST_SAMPLE_MULTI;
+
+public abstract class EqlSampleMultipleEntriesTestCase extends BaseEqlSpecTestCase {
+
+    public EqlSampleMultipleEntriesTestCase(
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey
+    ) {
+        this(TEST_SAMPLE_MULTI, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+    }
+
+    public EqlSampleMultipleEntriesTestCase(
+        String index,
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey
+    ) {
+        super(index, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+    }
+
+    @ParametersFactory(shuffle = false, argumentFormatting = PARAM_FORMATTING)
+    public static List<Object[]> readTestSpecs() throws Exception {
+        return asArray(EqlSpecLoader.load("/test_sample_multi.toml"));
+    }
+
+    @Override
+    protected String tiebreaker() {
+        return null;
+    }
+
+    @Override
+    protected String idField() {
+        return "id";
+    }
+
+    @Override
+    protected int requestFetchSize() {
+        // a more relevant fetch_size value for Samples, from algorithm point of view, so we'll mostly test this value
+        return frequently() ? 2 : super.requestFetchSize();
+    }
+}
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleTestCase.java
index 60e8fad41829..dfae73b3602a 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleTestCase.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSampleTestCase.java
@@ -15,12 +15,20 @@
 
 public abstract class EqlSampleTestCase extends BaseEqlSpecTestCase {
 
-    public EqlSampleTestCase(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size) {
-        this(TEST_SAMPLE, query, name, eventIds, joinKeys, size);
+    public EqlSampleTestCase(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) {
+        this(TEST_SAMPLE, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
     }
 
-    public EqlSampleTestCase(String index, String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size) {
-        super(index, query, name, eventIds, joinKeys, size);
+    public EqlSampleTestCase(
+        String index,
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey
+    ) {
+        super(index, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
     }
 
     @ParametersFactory(shuffle = false, argumentFormatting = PARAM_FORMATTING)
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java
index 5e2f4c1fd077..db7ee05ff223 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java
@@ -29,6 +29,7 @@ public class EqlSpec {
     private String[] joinKeys;
 
     private Integer size;
+    private Integer maxSamplesPerKey;
 
     public String name() {
         return name;
@@ -94,6 +95,14 @@ public void size(Integer size) {
         this.size = size;
     }
 
+    public Integer maxSamplesPerKey() {
+        return maxSamplesPerKey;
+    }
+
+    public void maxSamplesPerKey(Integer maxSamplesPerKey) {
+        this.maxSamplesPerKey = maxSamplesPerKey;
+    }
+
     @Override
     public String toString() {
         String str = "";
@@ -120,6 +129,9 @@ public String toString() {
         if (size != null) {
             str = appendWithComma(str, "size", "" + size);
         }
+        if (maxSamplesPerKey != null) {
+            str = appendWithComma(str, "max_samples_per_key", "" + maxSamplesPerKey);
+        }
 
         return str;
     }
@@ -136,12 +148,14 @@ public boolean equals(Object other) {
 
         EqlSpec that = (EqlSpec) other;
 
-        return Objects.equals(this.query(), that.query()) && Objects.equals(size, that.size);
+        return Objects.equals(this.query(), that.query())
+            && Objects.equals(size, that.size)
+            && Objects.equals(maxSamplesPerKey, that.maxSamplesPerKey);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(this.query, size);
+        return Objects.hash(this.query, size, maxSamplesPerKey);
     }
 
     private static String appendWithComma(String str, String name, String append) {
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java
index 3e78ca386be4..a1f555563e29 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java
@@ -121,6 +121,7 @@ private static List<EqlSpec> readFromStream(InputStream is, Set<String> uniqueTe
 
             arr = table.getList("join_keys");
             spec.joinKeys(arr != null ? arr.toArray(new String[0]) : new String[0]);
+            spec.maxSamplesPerKey(getInteger(table, "max_samples_per_key"));
             validateAndAddSpec(testSpecs, spec, uniqueTestNames);
         }
 
diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecTestCase.java
index 472d8071ff94..7113924f7902 100644
--- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecTestCase.java
+++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecTestCase.java
@@ -28,12 +28,20 @@ protected String tiebreaker() {
     }
 
     // constructor for "local" rest tests
-    public EqlSpecTestCase(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size) {
-        this(TEST_INDEX, query, name, eventIds, joinKeys, size);
+    public EqlSpecTestCase(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) {
+        this(TEST_INDEX, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
     }
 
     // constructor for multi-cluster tests
-    public EqlSpecTestCase(String index, String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size) {
-        super(index, query, name, eventIds, joinKeys, size);
+    public EqlSpecTestCase(
+        String index,
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey
+    ) {
+        super(index, query, name, eventIds, joinKeys, size, maxSamplesPerKey);
     }
 }
diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/data/sample-multi.data b/x-pack/plugin/eql/qa/common/src/main/resources/data/sample-multi.data
new file mode 100644
index 000000000000..47f72cdea960
--- /dev/null
+++ b/x-pack/plugin/eql/qa/common/src/main/resources/data/sample-multi.data
@@ -0,0 +1,73 @@
+[
+  {
+    "host_ip": "10.0.0.1",
+    "event_type": "alert_1",
+    "host": "doom",
+    "uptime": 0,
+    "port": 12,
+    "os": "win10",
+    "id": 1
+  },
+  {
+    "event_type": "alert_1",
+    "host": "CS",
+    "os": "win10",
+    "id": 2
+  },
+  {
+    "event_type": "alert_2",
+    "host": "farcry",
+    "bool": true,
+    "os": "win10",
+    "id": 3
+  },
+  {
+    "event_type": "alert_1",
+    "host": "GTA",
+    "os": "win10",
+    "bool": true,
+    "id": 4
+  },
+  {
+    "event_type": "alert_1",
+    "host": "GTA",
+    "os": "fedora",
+    "bool": true,
+    "id": 5
+  },
+  {
+    "event_type": "alert_2",
+    "host": "farcry",
+    "bool": true,
+    "os": "redhat",
+    "id": 6
+  },
+  {
+    "event_type": "alert_1",
+    "host": "farcry",
+    "bool": true,
+    "os": "redhat",
+    "id": 7
+  },
+  {
+    "event_type": "alert_1",
+    "host": "farcry",
+    "bool": true,
+    "os": null,
+    "id": 8
+  },
+  {
+    "event_type": "alert_2",
+    "host": "farcry",
+    "bool": true,
+    "os": null,
+    "id": 9
+  },
+  {
+    "event_type": "alert_1",
+    "host": "GTA",
+    "bool": true,
+    "id": 10
+  }
+
+]
diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/data/sample-multi.mapping b/x-pack/plugin/eql/qa/common/src/main/resources/data/sample-multi.mapping
new file mode 100644
index 000000000000..77abf7d406e1
--- /dev/null
+++ b/x-pack/plugin/eql/qa/common/src/main/resources/data/sample-multi.mapping
@@ -0,0 +1,39 @@
+{
+	"properties": {
+		"host_ip": {
+			"type":"ip"
+		},
+	    "date": {
+	      "type": "date"
+	    },
+	    "event_type": {
+	      "type": "keyword"
+	    },
+	    "event": {
+	      "properties": {
+	        "category": {
+	          "type": "alias",
+	          "path": "event_type"
+	        }
+	      }
+	    },
+	    "missing_keyword": {
+			"type": "keyword"
+		},
+	    "host": {
+	      "type": "keyword"
+	    },
+	    "os": {
+	      "type": "keyword"
+	    },
+	    "bool": {
+	      "type": "boolean"
+	    },
+	    "uptime" : {
+	      "type" : "long"
+	    },
+	    "port" : {
+	      "type" : "long"
+	    }
+	}
+}
diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_sample_multi.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_sample_multi.toml
new file mode 100644
index 000000000000..d6ac01cfca19
--- /dev/null
+++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_sample_multi.toml
@@ -0,0 +1,367 @@
+# Test syntax:
+#
+# each test starts with a [[queries]] tag and has a set of mandatory attributes:
+#
+# - name: string, the test name, reported in the logs in case of failure
+# - query: string, the query to execute
+# - expected_event_ids: the expected result of the query. Can be one of the following:
+#    - an array of event IDs, (eg [1, 2, 3]): the expected event IDs returned by the query
+#    - an array of arrays of event IDs (eg. [[1, 2, 3], [1, 4, 5]]): each array of IDs is a possible valid result for the query;
+#      this is useful when the query is not deterministic, ie. it can return different valid results based on how the documents are indexed.
+#    - an empty array [], if the query is expected to return no results
+#   If the query returns multiple samples, the expected_event_ids will contain a flattened list of IDs for all the events
+#   in the returned samples, eg. if the query resturns sample1=[event1(id=1), event2(id=2)] and sample2=[event3(id=3), event4(id=4)],
+#   the expected_event_ids will be [1, 2, 3, 4]
+# - join_keys: array, a set of join keys returned in the results. This is flattened sequence of composite join keys for all the events returned
+#   eg. if the test returns two events (e1, e2) with two join keys each (eg. e1 with [joinkey1, joinkey2] and e2 with [joinkey3, joinkey4],
+#   the expected join_keys value will be [joinkey1, joinkey2, joinkey3, joinkey4]
+#
+# in addition, the spec syntax accepts the following optional attributes (currently ignored):
+# - note: string
+# - description: string
+# - tags: array of strings
+#
+# Strings can be delimited by single or double quotes. Triple single quotes can be used for multi-line strings
+
+[[queries]]
+name = "oneSamplePerKey"
+max_samples_per_key = 1
+query = '''
+  sample by os
+    [alert_1 where true]
+    [alert_2 where true]
+'''
+# non-deterministic, can return any of the following
+expected_event_ids  = [
+    [7, 6,
+     1, 3],
+    [7, 6,
+     2, 3],
+    [7, 6,
+     4, 3]
+]
+join_keys = ["redhat",
+             "win10"]
+
+[[queries]]
+name = "oneSamplePerKey_optional"
+max_samples_per_key = 1
+query = '''
+  sample by ?os
+    [alert_1 where true]
+    [alert_2 where true]
+'''
+# non-deterministic, can return any of the following
+expected_event_ids  = [
+    [8, 9,
+    7, 6,
+    1, 3],
+    [8, 9,
+    7, 6,
+    2, 3],
+    [8, 9,
+    7, 6,
+    4, 3],
+    [10, 9,
+    7, 6,
+    1, 3],
+    [10, 9,
+    7, 6,
+    2, 3],
+    [10, 9,
+    7, 6,
+    4, 3]
+]
+join_keys = ["null",
+             "redhat",
+             "win10"]
+
+
+[[queries]]
+name = "twoSamplesPerKeyOutOfThree"
+max_samples_per_key = 2
+query = '''
+  sample by os
+    [alert_1 where true]
+    [alert_2 where true]
+'''
+# non-deterministic, can return any of the following
+expected_event_ids  = [
+    [7, 6,
+     1, 3,
+     2, 3],
+    [7, 6,
+     1, 3,
+     4, 3],
+    [7, 6,
+     2, 3,
+     1, 3],
+    [7, 6,
+     2, 3,
+     4, 3],
+    [7, 6,
+     4, 3,
+     1, 3],
+    [7, 6,
+     4, 3,
+     2, 3]
+]
+join_keys = ["redhat",
+             "win10",
+             "win10"]
+
+
+
+[[queries]]
+name = "twoSamplesPerKeyOutOfThree_optional"
+max_samples_per_key = 2
+query = '''
+  sample by ?os
+    [alert_1 where true]
+    [alert_2 where true]
+'''
+# non-deterministic, can return any of the following
+expected_event_ids  = [
+    [8, 9,
+     10, 9,
+     7, 6,
+     1, 3,
+     2, 3],
+    [8, 9,
+     10, 9,
+     7, 6,
+     1, 3,
+     4, 3],
+    [8, 9,
+     10, 9,
+     7, 6,
+     2, 3,
+     1, 3],
+    [8, 9,
+     10, 9,
+     7, 6,
+     2, 3,
+     4, 3],
+    [8, 9,
+     10, 9,
+     7, 6,
+     4, 3,
+     1, 3],
+    [8, 9,
+     10, 9,
+     7, 6,
+     4, 3,
+     2, 3],
+    [10, 9,
+     8, 9,
+     7, 6,
+     1, 3,
+     2, 3],
+    [10, 9,
+     8, 9,
+     7, 6,
+     1, 3,
+     4, 3],
+    [10, 9,
+     8, 9,
+     7, 6,
+     2, 3,
+     1, 3],
+    [10, 9,
+     8, 9,
+     7, 6,
+     2, 3,
+     4, 3],
+    [10, 9,
+     8, 9,
+     7, 6,
+     4, 3,
+     1, 3],
+    [10, 9,
+     8, 9,
+     7, 6,
+     4, 3,
+     2, 3]
+]
+join_keys = ["null",
+             "null",
+             "redhat",
+             "win10",
+             "win10"]
+
+
+[[queries]]
+name = "twoSamplesPerKeyOutOfThree_withCondition"
+max_samples_per_key = 2
+query = '''
+  sample by os
+    [alert_1 where host != "farcry"]
+    [alert_2 where true]
+'''
+# non-deterministic, can return any of the following
+expected_event_ids  = [
+    [1, 3,
+     2, 3],
+    [1, 3,
+     4, 3],
+    [2, 3,
+     1, 3],
+    [2, 3,
+     4, 3],
+    [4, 3,
+     1, 3],
+    [4, 3,
+     2, 3]
+]
+join_keys = ["win10",
+             "win10"]
+
+
+[[queries]]
+name = "twoSamplesPerKeyOutOfThree2"
+max_samples_per_key = 2
+query = '''
+  sample by os
+    [alert_2 where true]
+    [alert_1 where true]
+'''
+# non-deterministic, can return any of the following
+expected_event_ids  = [
+    [6, 7,
+     3, 1,
+     3, 2],
+    [6, 7,
+     3, 1,
+     3, 4],
+    [6, 7,
+     3, 2,
+     3, 1],
+    [6, 7,
+     3, 2,
+     3, 4],
+    [6, 7,
+     3, 4,
+     3, 1],
+    [6, 7,
+     3, 4,
+     3, 2]
+]
+join_keys = ["redhat",
+             "win10",
+             "win10"]
+
+
+[[queries]]
+name = "threeSamplesPerKey"
+max_samples_per_key = 3
+query = '''
+  sample by os
+    [alert_1 where true]
+    [alert_2 where true]
+'''
+# non-deterministic, can return any of the following
+expected_event_ids  = [
+    [7, 6,
+     1, 3,
+     2, 3,
+     4, 3],
+    [7, 6,
+     1, 3,
+     4, 3,
+     2, 3],
+    [7, 6,
+     2, 3,
+     1, 3,
+     4, 3],
+    [7, 6,
+     2, 3,
+     4, 3,
+     1, 3],
+    [7, 6,
+     4, 3,
+     1, 3,
+     2, 3],
+    [7, 6,
+     4, 3,
+     2, 3,
+     1, 3]
+]
+join_keys = ["redhat",
+             "win10",
+             "win10",
+             "win10"]
+
+
+[[queries]]
+name = "moreSamplesPerKey"
+max_samples_per_key = 10
+query = '''
+  sample by os
+    [alert_1 where true]
+    [alert_2 where true]
+'''
+# non-deterministic, can return any of the following
+expected_event_ids  = [
+    [7, 6,
+     1, 3,
+     2, 3,
+     4, 3],
+    [7, 6,
+     1, 3,
+     4, 3,
+     2, 3],
+    [7, 6,
+     2, 3,
+     1, 3,
+     4, 3],
+    [7, 6,
+     2, 3,
+     4, 3,
+     1, 3],
+    [7, 6,
+     4, 3,
+     1, 3,
+     2, 3],
+    [7, 6,
+     4, 3,
+     2, 3,
+     1, 3]
+]
+join_keys = ["redhat",
+             "win10",
+             "win10",
+             "win10"]
+
+
+[[queries]]
+name = "twoSamplesPerKeyOutOfThree_inlineByKey"
+max_samples_per_key = 2
+query = '''
+  sample
+    [alert_1 where true] by os
+    [alert_2 where true] by os
+'''
+# non-deterministic, can return any of the following
+expected_event_ids  = [
+    [7, 6,
+     1, 3,
+     2, 3],
+    [7, 6,
+     1, 3,
+     4, 3],
+    [7, 6,
+     2, 3,
+     1, 3],
+    [7, 6,
+     2, 3,
+     4, 3],
+    [7, 6,
+     4, 3,
+     1, 3],
+    [7, 6,
+     4, 3,
+     2, 3]
+]
+join_keys = ["redhat",
+             "win10",
+             "win10"]
diff --git a/x-pack/plugin/eql/qa/correctness/build.gradle b/x-pack/plugin/eql/qa/correctness/build.gradle
index 890584d0055c..aeddca54d145 100644
--- a/x-pack/plugin/eql/qa/correctness/build.gradle
+++ b/x-pack/plugin/eql/qa/correctness/build.gradle
@@ -1,5 +1,5 @@
 apply plugin: 'elasticsearch.java'
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.internal-testclusters'
 
 import org.elasticsearch.gradle.testclusters.RunTask
diff --git a/x-pack/plugin/eql/qa/mixed-node/build.gradle b/x-pack/plugin/eql/qa/mixed-node/build.gradle
index 9aa0a93ff153..1e93ea035058 100644
--- a/x-pack/plugin/eql/qa/mixed-node/build.gradle
+++ b/x-pack/plugin/eql/qa/mixed-node/build.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.bwc-test'
 
 import org.elasticsearch.gradle.VersionProperties
diff --git a/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle b/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle
index b637517b29dc..5fc247693c45 100644
--- a/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle
+++ b/x-pack/plugin/eql/qa/multi-cluster-with-security/build.gradle
@@ -1,6 +1,6 @@
 import org.elasticsearch.gradle.testclusters.DefaultTestClustersTask
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 dependencies {
   javaRestTestImplementation project(path: xpackModule('eql:qa:common'))
diff --git a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java
index 6632d0a31290..ad8e94e50e8f 100644
--- a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java
+++ b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java
@@ -16,7 +16,7 @@
 
 public class EqlDateNanosIT extends EqlDateNanosSpecTestCase {
 
-    public EqlDateNanosIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size) {
-        super(remoteClusterIndex(TEST_NANOS_INDEX), query, name, eventIds, joinKeys, size);
+    public EqlDateNanosIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) {
+        super(remoteClusterIndex(TEST_NANOS_INDEX), query, name, eventIds, joinKeys, size, maxSamplesPerKey);
     }
 }
diff --git a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java
index 979f38702122..7e79b55deb41 100644
--- a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java
+++ b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java
@@ -16,7 +16,7 @@
 
 public class EqlExtraIT extends EqlExtraSpecTestCase {
 
-    public EqlExtraIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size) {
-        super(remoteClusterIndex(TEST_EXTRA_INDEX), query, name, eventIds, joinKeys, size);
+    public EqlExtraIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) {
+        super(remoteClusterIndex(TEST_EXTRA_INDEX), query, name, eventIds, joinKeys, size, maxSamplesPerKey);
     }
 }
diff --git a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlRestValidationIT.java b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlRestValidationIT.java
index 66617341aa27..a9f1d22ce096 100644
--- a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlRestValidationIT.java
+++ b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlRestValidationIT.java
@@ -37,7 +37,8 @@ protected void assertErrorMessageWhenAllowNoIndicesIsFalse(String reqParameter)
         assertErrorMessage(
             "inexistent1,inexistent2",
             reqParameter,
-            getInexistentIndexErrorMessage() + "[" + indexPattern("inexistent1") + "," + indexPattern("inexistent2") + "]\""
+            getInexistentIndexErrorMessage()
+                + "[null]\",\"resource.type\":\"index_expression\",\"resource.id\":[\"inexistent1\",\"inexistent2\"]}]"
         );
     }
 
diff --git a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java
index 0198ae1476b8..0d36c6938666 100644
--- a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java
+++ b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java
@@ -16,7 +16,7 @@
 
 public class EqlSampleIT extends EqlSampleTestCase {
 
-    public EqlSampleIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size) {
-        super(remoteClusterPattern(TEST_SAMPLE), query, name, eventIds, joinKeys, size);
+    public EqlSampleIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) {
+        super(remoteClusterPattern(TEST_SAMPLE), query, name, eventIds, joinKeys, size, maxSamplesPerKey);
     }
 }
diff --git a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java
new file mode 100644
index 000000000000..15bac1be1706
--- /dev/null
+++ b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.eql;
+
+import org.elasticsearch.test.eql.EqlSampleMultipleEntriesTestCase;
+
+import java.util.List;
+
+import static org.elasticsearch.test.eql.DataLoader.TEST_SAMPLE_MULTI;
+import static org.elasticsearch.xpack.eql.RemoteClusterTestUtils.remoteClusterPattern;
+
+public class EqlSampleMultipleEntriesIT extends EqlSampleMultipleEntriesTestCase {
+
+    public EqlSampleMultipleEntriesIT(
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey
+    ) {
+        super(remoteClusterPattern(TEST_SAMPLE_MULTI), query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+    }
+}
diff --git a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java
index 3fdd24bd729a..ffa83ae362f5 100644
--- a/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java
+++ b/x-pack/plugin/eql/qa/multi-cluster-with-security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java
@@ -16,7 +16,7 @@
 
 public class EqlSpecIT extends EqlSpecTestCase {
 
-    public EqlSpecIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size) {
-        super(remoteClusterIndex(TEST_INDEX), query, name, eventIds, joinKeys, size);
+    public EqlSpecIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) {
+        super(remoteClusterIndex(TEST_INDEX), query, name, eventIds, joinKeys, size, maxSamplesPerKey);
     }
 }
diff --git a/x-pack/plugin/eql/qa/rest/build.gradle b/x-pack/plugin/eql/qa/rest/build.gradle
index 01a9b176a007..a49317ad8c4e 100644
--- a/x-pack/plugin/eql/qa/rest/build.gradle
+++ b/x-pack/plugin/eql/qa/rest/build.gradle
@@ -1,6 +1,6 @@
-apply plugin: 'elasticsearch.internal-java-rest-test'
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 import org.elasticsearch.gradle.internal.info.BuildParams
 
diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java
index 6b52b1773282..e196813f32e6 100644
--- a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java
+++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlDateNanosIT.java
@@ -13,7 +13,7 @@
 
 public class EqlDateNanosIT extends EqlDateNanosSpecTestCase {
 
-    public EqlDateNanosIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size) {
-        super(query, name, eventIds, joinKeys, size);
+    public EqlDateNanosIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) {
+        super(query, name, eventIds, joinKeys, size, maxSamplesPerKey);
     }
 }
diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java
index 6627473ec1e4..c0a3c86106d2 100644
--- a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java
+++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java
@@ -13,7 +13,7 @@
 
 public class EqlExtraIT extends EqlExtraSpecTestCase {
 
-    public EqlExtraIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size) {
-        super(query, name, eventIds, joinKeys, size);
+    public EqlExtraIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) {
+        super(query, name, eventIds, joinKeys, size, maxSamplesPerKey);
     }
 }
diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlRestValidationIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlRestValidationIT.java
index ebcf49801947..0bcf90a35f54 100644
--- a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlRestValidationIT.java
+++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlRestValidationIT.java
@@ -30,7 +30,11 @@ protected void assertErrorMessageWhenAllowNoIndicesIsFalse(String reqParameter)
         // TODO: revisit after
         // https://github.com/elastic/elasticsearch/issues/64197
         // is closed
-        assertErrorMessage("inexistent1,inexistent2", reqParameter, """
-            "root_cause":[{"type":"index_not_found_exception","reason":"no such index [null]\"""");
+        assertErrorMessage(
+            "inexistent1,inexistent2",
+            reqParameter,
+            "\"root_cause\":[{\"type\":\"index_not_found_exception\",\"reason\":\"no such index [null]\","
+                + "\"resource.type\":\"index_expression\",\"resource.id\":[\"inexistent1\",\"inexistent2\"]}]"
+        );
     }
 }
diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java
index d93da0916b93..bd40074b26de 100644
--- a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java
+++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleIT.java
@@ -13,8 +13,8 @@
 
 public class EqlSampleIT extends EqlSampleTestCase {
 
-    public EqlSampleIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size) {
-        super(query, name, eventIds, joinKeys, size);
+    public EqlSampleIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) {
+        super(query, name, eventIds, joinKeys, size, maxSamplesPerKey);
     }
 
 }
diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java
new file mode 100644
index 000000000000..077f66c6dd24
--- /dev/null
+++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSampleMultipleEntriesIT.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.eql;
+
+import org.elasticsearch.test.eql.EqlSampleMultipleEntriesTestCase;
+
+import java.util.List;
+
+public class EqlSampleMultipleEntriesIT extends EqlSampleMultipleEntriesTestCase {
+
+    public EqlSampleMultipleEntriesIT(
+        String query,
+        String name,
+        List<long[]> eventIds,
+        String[] joinKeys,
+        Integer size,
+        Integer maxSamplesPerKey
+    ) {
+        super(query, name, eventIds, joinKeys, size, maxSamplesPerKey);
+    }
+
+}
diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java
index 141e26d39695..df966f3bbf77 100644
--- a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java
+++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java
@@ -13,7 +13,7 @@
 
 public class EqlSpecIT extends EqlSpecTestCase {
 
-    public EqlSpecIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size) {
-        super(query, name, eventIds, joinKeys, size);
+    public EqlSpecIT(String query, String name, List<long[]> eventIds, String[] joinKeys, Integer size, Integer maxSamplesPerKey) {
+        super(query, name, eventIds, joinKeys, size, maxSamplesPerKey);
     }
 }
diff --git a/x-pack/plugin/eql/qa/security/build.gradle b/x-pack/plugin/eql/qa/security/build.gradle
index d13c3570ee61..bcc9d0cb4ef8 100644
--- a/x-pack/plugin/eql/qa/security/build.gradle
+++ b/x-pack/plugin/eql/qa/security/build.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 import org.elasticsearch.gradle.internal.info.BuildParams
 
diff --git a/x-pack/plugin/eql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlRestValidationIT.java b/x-pack/plugin/eql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlRestValidationIT.java
index 3cb5c5caa284..406345d1144f 100644
--- a/x-pack/plugin/eql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlRestValidationIT.java
+++ b/x-pack/plugin/eql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlRestValidationIT.java
@@ -23,6 +23,11 @@ protected Settings restClientSettings() {
 
     @Override
     protected String getInexistentIndexErrorMessage() {
+        return "\"root_cause\":[{\"type\":\"verification_exception\",\"reason\":\"Found 1 problem\\nline -1:-1: Unknown index ";
+    }
+
+    @Override
+    protected String getInexistentWildcardErrorMessage() {
         return """
             "root_cause":[{"type":"verification_exception","reason":"Found 1 problem\\nline -1:-1: Unknown index [*,-*]"}],\
             "type":"index_not_found_exception","reason":"no such index\s""";
@@ -38,9 +43,9 @@ protected void assertErrorMessageWhenAllowNoIndicesIsFalse(String reqParameter)
             "root_cause":[{"type":"index_not_found_exception","reason":"no such index [inexistent*]\"""");
         // TODO: revisit the next two tests when https://github.com/elastic/elasticsearch/issues/64190 is closed
         assertErrorMessage("inexistent", reqParameter, """
-            "root_cause":[{"type":"index_not_found_exception","reason":"no such index [[inexistent]]\"""");
+            "root_cause":[{"type":"index_not_found_exception","reason":"no such index [inexistent]\"""");
         assertErrorMessage("inexistent1,inexistent2", reqParameter, """
-            "root_cause":[{"type":"index_not_found_exception","reason":"no such index [[inexistent1, inexistent2]]\"""");
+            "root_cause":[{"type":"index_not_found_exception","reason":"no such index [null]\"""");
     }
 
 }
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java
index b0fc5be6f2ae..6edc7087f82a 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.eql.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
@@ -62,6 +62,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re
     private String resultPosition = "tail";
     private List<FieldAndFormat> fetchFields;
     private Map<String, Object> runtimeMappings = emptyMap();
+    private int maxSamplesPerKey = RequestDefaults.MAX_SAMPLES_PER_KEY;
 
     // Async settings
     private TimeValue waitForCompletionTimeout = null;
@@ -81,6 +82,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re
     static final String KEY_RESULT_POSITION = "result_position";
     static final String KEY_FETCH_FIELDS = "fields";
     static final String KEY_RUNTIME_MAPPINGS = "runtime_mappings";
+    static final String KEY_MAX_SAMPLES_PER_KEY = "max_samples_per_key";
 
     static final ParseField FILTER = new ParseField(KEY_FILTER);
     static final ParseField TIMESTAMP_FIELD = new ParseField(KEY_TIMESTAMP_FIELD);
@@ -94,6 +96,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re
     static final ParseField KEEP_ON_COMPLETION = new ParseField(KEY_KEEP_ON_COMPLETION);
     static final ParseField RESULT_POSITION = new ParseField(KEY_RESULT_POSITION);
     static final ParseField FETCH_FIELDS_FIELD = SearchSourceBuilder.FETCH_FIELDS_FIELD;
+    static final ParseField MAX_SAMPLES_PER_KEY = new ParseField(KEY_MAX_SAMPLES_PER_KEY);
 
     private static final ObjectParser<EqlSearchRequest, Void> PARSER = objectParser(EqlSearchRequest::new);
 
@@ -112,16 +115,16 @@ public EqlSearchRequest(StreamInput in) throws IOException {
         size = in.readVInt();
         fetchSize = in.readVInt();
         query = in.readString();
-        if (in.getVersion().onOrAfter(Version.V_7_15_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_15_0)) {
             this.ccsMinimizeRoundtrips = in.readBoolean();
         }
         this.waitForCompletionTimeout = in.readOptionalTimeValue();
         this.keepAlive = in.readOptionalTimeValue();
         this.keepOnCompletion = in.readBoolean();
-        if (in.getVersion().onOrAfter(Version.V_7_17_8)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_17_8)) {
             resultPosition = in.readString();
         }
-        if (in.getVersion().onOrAfter(Version.V_7_13_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0)) {
             if (in.readBoolean()) {
                 fetchFields = in.readList(FieldAndFormat::new);
             }
@@ -129,6 +132,9 @@ public EqlSearchRequest(StreamInput in) throws IOException {
         } else {
             runtimeMappings = emptyMap();
         }
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            maxSamplesPerKey = in.readInt();
+        }
     }
 
     @Override
@@ -181,6 +187,10 @@ public ActionRequestValidationException validate() {
             validationException = validateRuntimeMappings(runtimeMappings, validationException);
         }
 
+        if (maxSamplesPerKey < 1) {
+            validationException = addValidationError("max_samples_per_key must be greater than 0", validationException);
+        }
+
         return validationException;
     }
 
@@ -234,6 +244,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (runtimeMappings != null) {
             builder.field(KEY_RUNTIME_MAPPINGS, runtimeMappings);
         }
+        builder.field(KEY_MAX_SAMPLES_PER_KEY, maxSamplesPerKey);
 
         return builder;
     }
@@ -267,6 +278,7 @@ protected static <R extends EqlSearchRequest> ObjectParser<R, Void> objectParser
         parser.declareString(EqlSearchRequest::resultPosition, RESULT_POSITION);
         parser.declareField(EqlSearchRequest::fetchFields, EqlSearchRequest::parseFetchFields, FETCH_FIELDS_FIELD, ValueType.VALUE_ARRAY);
         parser.declareObject(EqlSearchRequest::runtimeMappings, (p, c) -> p.map(), SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD);
+        parser.declareInt(EqlSearchRequest::maxSamplesPerKey, MAX_SAMPLES_PER_KEY);
         return parser;
     }
 
@@ -406,6 +418,15 @@ public EqlSearchRequest fetchFields(List<FieldAndFormat> fetchFields) {
         return this;
     }
 
+    public int maxSamplesPerKey() {
+        return maxSamplesPerKey;
+    }
+
+    public EqlSearchRequest maxSamplesPerKey(int maxSamplesPerKey) {
+        this.maxSamplesPerKey = maxSamplesPerKey;
+        return this;
+    }
+
     private static List<FieldAndFormat> parseFetchFields(XContentParser parser) throws IOException {
         List<FieldAndFormat> result = new ArrayList<>();
         Token token = parser.currentToken();
@@ -430,22 +451,25 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVInt(size);
         out.writeVInt(fetchSize);
         out.writeString(query);
-        if (out.getVersion().onOrAfter(Version.V_7_15_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_15_0)) {
             out.writeBoolean(ccsMinimizeRoundtrips);
         }
         out.writeOptionalTimeValue(waitForCompletionTimeout);
         out.writeOptionalTimeValue(keepAlive);
         out.writeBoolean(keepOnCompletion);
-        if (out.getVersion().onOrAfter(Version.V_7_17_8)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_17_8)) {
             out.writeString(resultPosition);
         }
-        if (out.getVersion().onOrAfter(Version.V_7_13_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0)) {
             out.writeBoolean(fetchFields != null);
             if (fetchFields != null) {
                 out.writeList(fetchFields);
             }
             out.writeGenericMap(runtimeMappings);
         }
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            out.writeInt(maxSamplesPerKey);
+        }
     }
 
     @Override
@@ -471,7 +495,8 @@ public boolean equals(Object o) {
             && Objects.equals(keepAlive, that.keepAlive)
             && Objects.equals(resultPosition, that.resultPosition)
             && Objects.equals(fetchFields, that.fetchFields)
-            && Objects.equals(runtimeMappings, that.runtimeMappings);
+            && Objects.equals(runtimeMappings, that.runtimeMappings)
+            && Objects.equals(maxSamplesPerKey, that.maxSamplesPerKey);
     }
 
     @Override
@@ -491,7 +516,8 @@ public int hashCode() {
             keepAlive,
             resultPosition,
             fetchFields,
-            runtimeMappings
+            runtimeMappings,
+            maxSamplesPerKey
         );
     }
 
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestBuilder.java
index 9cc7c98bf94c..db7c45c07bfb 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestBuilder.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestBuilder.java
@@ -55,4 +55,9 @@ public EqlSearchRequestBuilder query(String query) {
         request.query(query);
         return this;
     }
+
+    public EqlSearchRequestBuilder maxSamplesPerKey(int maxSamplesPerKey) {
+        request.maxSamplesPerKey(maxSamplesPerKey);
+        return this;
+    }
 }
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java
index cedcb20bcd20..e631d8cdf94a 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.eql.action;
 
 import org.apache.lucene.search.TotalHits;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -256,7 +256,7 @@ public Event(StreamInput in) throws IOException {
             index = in.readString();
             id = in.readString();
             source = in.readBytesReference();
-            if (in.getVersion().onOrAfter(Version.V_7_13_0) && in.readBoolean()) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0) && in.readBoolean()) {
                 fetchFields = in.readMap(StreamInput::readString, DocumentField::new);
             } else {
                 fetchFields = null;
@@ -268,7 +268,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(index);
             out.writeString(id);
             out.writeBytesReference(source);
-            if (out.getVersion().onOrAfter(Version.V_7_13_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0)) {
                 out.writeBoolean(fetchFields != null);
                 if (fetchFields != null) {
                     out.writeMap(fetchFields, StreamOutput::writeString, (stream, documentField) -> documentField.writeTo(stream));
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java
index 74fb806d27eb..58bfb9513add 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java
@@ -17,4 +17,5 @@ private RequestDefaults() {}
     public static int SIZE = 10;
     public static int FETCH_SIZE = 1000;
     public static boolean CCS_MINIMIZE_ROUNDTRIPS = true;
+    public static int MAX_SAMPLES_PER_KEY = 1;
 }
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java
index b4664ee3a3c3..22c589e25a82 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java
@@ -9,20 +9,19 @@
 
 import org.elasticsearch.xpack.eql.expression.OptionalMissingAttribute;
 import org.elasticsearch.xpack.eql.expression.OptionalUnresolvedAttribute;
+import org.elasticsearch.xpack.ql.analyzer.AnalyzerRules;
+import org.elasticsearch.xpack.ql.analyzer.AnalyzerRules.AnalyzerRule;
 import org.elasticsearch.xpack.ql.common.Failure;
 import org.elasticsearch.xpack.ql.expression.Attribute;
 import org.elasticsearch.xpack.ql.expression.Expression;
 import org.elasticsearch.xpack.ql.expression.Literal;
 import org.elasticsearch.xpack.ql.expression.NamedExpression;
 import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute;
-import org.elasticsearch.xpack.ql.expression.function.Function;
-import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition;
-import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
 import org.elasticsearch.xpack.ql.expression.function.UnresolvedFunction;
 import org.elasticsearch.xpack.ql.plan.logical.Filter;
 import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.ql.rule.ParameterizedRuleExecutor;
 import org.elasticsearch.xpack.ql.rule.RuleExecutor;
-import org.elasticsearch.xpack.ql.session.Configuration;
 import org.elasticsearch.xpack.ql.type.DataTypes;
 
 import java.util.Collection;
@@ -31,28 +30,30 @@
 import static java.util.Arrays.asList;
 import static org.elasticsearch.xpack.eql.analysis.AnalysisUtils.resolveAgainstList;
 import static org.elasticsearch.xpack.ql.analyzer.AnalyzerRules.AddMissingEqualsToBoolField;
+import static org.elasticsearch.xpack.ql.analyzer.AnalyzerRules.resolveFunction;
 
-public class Analyzer extends RuleExecutor<LogicalPlan> {
+public class Analyzer extends ParameterizedRuleExecutor<LogicalPlan, AnalyzerContext> {
+
+    private static final Iterable<RuleExecutor.Batch<LogicalPlan>> rules;
+
+    static {
+        var optional = new Batch<>("Optional", Limiter.ONCE, new ResolveOrReplaceOptionalRefs());
+        var resolution = new Batch<>("Resolution", new ResolveRefs(), new ResolveFunctions());
+        var cleanup = new Batch<>("Finish Analysis", Limiter.ONCE, new AddMissingEqualsToBoolField());
+
+        rules = asList(optional, resolution, cleanup);
+    }
 
-    private final Configuration configuration;
-    private final FunctionRegistry functionRegistry;
     private final Verifier verifier;
 
-    public Analyzer(Configuration configuration, FunctionRegistry functionRegistry, Verifier verifier) {
-        this.configuration = configuration;
-        this.functionRegistry = functionRegistry;
+    public Analyzer(AnalyzerContext context, Verifier verifier) {
+        super(context);
         this.verifier = verifier;
     }
 
     @Override
-    protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
-        Batch optional = new Batch("Optional", Limiter.ONCE, new ResolveOrReplaceOptionalRefs());
-
-        Batch resolution = new Batch("Resolution", new ResolveRefs(), new ResolveFunctions());
-
-        Batch cleanup = new Batch("Finish Analysis", Limiter.ONCE, new AddMissingEqualsToBoolField());
-
-        return asList(optional, resolution, cleanup);
+    protected Iterable<RuleExecutor.Batch<LogicalPlan>> batches() {
+        return rules;
     }
 
     public LogicalPlan analyze(LogicalPlan plan) {
@@ -99,29 +100,14 @@ protected LogicalPlan rule(LogicalPlan plan) {
         }
     }
 
-    private class ResolveFunctions extends AnalyzerRule<LogicalPlan> {
+    private static class ResolveFunctions extends AnalyzerRules.ParameterizedAnalyzerRule<LogicalPlan, AnalyzerContext> {
 
         @Override
-        protected LogicalPlan rule(LogicalPlan plan) {
-            return plan.transformExpressionsUp(UnresolvedFunction.class, uf -> {
-                if (uf.analyzed()) {
-                    return uf;
-                }
-
-                String name = uf.name();
-
-                if (uf.childrenResolved() == false) {
-                    return uf;
-                }
-
-                String functionName = functionRegistry.resolveAlias(name);
-                if (functionRegistry.functionExists(functionName) == false) {
-                    return uf.missing(functionName, functionRegistry.listFunctions());
-                }
-                FunctionDefinition def = functionRegistry.resolveFunction(functionName);
-                Function f = uf.buildResolved(configuration, def);
-                return f;
-            });
+        protected LogicalPlan rule(LogicalPlan plan, AnalyzerContext context) {
+            return plan.transformExpressionsUp(
+                UnresolvedFunction.class,
+                uf -> resolveFunction(uf, context.configuration(), context.functionRegistry())
+            );
         }
     }
 
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalyzerContext.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalyzerContext.java
new file mode 100644
index 000000000000..7a09e363734f
--- /dev/null
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalyzerContext.java
@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.eql.analysis;
+
+import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
+import org.elasticsearch.xpack.ql.session.Configuration;
+
+public record AnalyzerContext(Configuration configuration, FunctionRegistry functionRegistry) {}
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalyzerRule.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalyzerRule.java
deleted file mode 100644
index 2df98c739824..000000000000
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalyzerRule.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.eql.analysis;
-
-import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
-import org.elasticsearch.xpack.ql.rule.Rule;
-
-public abstract class AnalyzerRule<SubPlan extends LogicalPlan> extends Rule<SubPlan, LogicalPlan> {
-
-    // transformUp (post-order) - that is first children and then the node
-    // but with a twist; only if the tree is not resolved or analyzed
-    @Override
-    public final LogicalPlan apply(LogicalPlan plan) {
-        return plan.transformUp(typeToken(), t -> t.analyzed() || skipResolved() && t.resolved() ? t : rule(t));
-    }
-
-    @Override
-    protected abstract LogicalPlan rule(SubPlan plan);
-
-    protected boolean skipResolved() {
-        return true;
-    }
-}
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java
index 4f4437c730a6..cccf49a56815 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java
@@ -211,7 +211,14 @@ public Executable assemble(List<List<Attribute>> listOfKeys, List<PhysicalPlan>
             }
         }
 
-        return new SampleIterator(new PITAwareQueryClient(session), criteria, cfg.fetchSize(), limit, session.circuitBreaker());
+        return new SampleIterator(
+            new PITAwareQueryClient(session),
+            criteria,
+            cfg.fetchSize(),
+            limit,
+            session.circuitBreaker(),
+            cfg.maxSamplesPerKey()
+        );
     }
 
     private HitExtractor timestampExtractor(HitExtractor hitExtractor) {
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/SampleQueryRequest.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/SampleQueryRequest.java
index 166b3c84e0ac..9c91adfc8dc3 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/SampleQueryRequest.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/SampleQueryRequest.java
@@ -146,7 +146,7 @@ public void multipleKeyPairs(List<Map<String, Object>> values, List<String> prev
      * Sets keys / terms to filter on in the final stage filtering (where actual events are gathered).
      * Can be removed through null.
      */
-    public void singleKeyPair(final List<Object> compositeKeyValues, int maxStages) {
+    public void singleKeyPair(final List<Object> compositeKeyValues, int maxStages, int maxSamplesPerKey) {
         List<QueryBuilder> newFilters = new ArrayList<>();
         if (compositeKeyValues.isEmpty()) {
             // no keys have been specified and none have been set
@@ -169,8 +169,10 @@ public void singleKeyPair(final List<Object> compositeKeyValues, int maxStages)
 
         SearchSourceBuilder newSource = copySource(searchSource);
         RuntimeUtils.replaceFilter(singleKeyPairFilters, newFilters, newSource);
-        newSource.size(maxStages) // ask for exactly number of filters/stages documents
-            .terminateAfter(maxStages) // no need to ask for more from each shard since we don't need sorting or more docs
+        // ask for the minimum needed to get at least N samplese per key
+        int minResultsNeeded = maxStages + maxSamplesPerKey - 1;
+        newSource.size(minResultsNeeded)
+            .terminateAfter(minResultsNeeded) // no need to ask for more from each shard since we don't need sorting or more docs
             .fetchSource(FetchSourceContext.DO_NOT_FETCH_SOURCE) // we'll get the source in a separate step
             .trackTotalHits(false)
             .trackScores(false);
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SampleIterator.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SampleIterator.java
index cc888ae7da83..5d9feef7b0b4 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SampleIterator.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sample/SampleIterator.java
@@ -57,6 +57,7 @@ public class SampleIterator implements Executable {
     final List<Sample> samples;
     private final int fetchSize;
     private final Limit limit;
+    private final int maxSamplesPerKey;
     private long startTime;
 
     // ---------- CIRCUIT BREAKER -----------
@@ -84,7 +85,14 @@ public class SampleIterator implements Executable {
      */
     private long previousTotalPageSize = 0;
 
-    public SampleIterator(QueryClient client, List<SampleCriterion> criteria, int fetchSize, Limit limit, CircuitBreaker circuitBreaker) {
+    public SampleIterator(
+        QueryClient client,
+        List<SampleCriterion> criteria,
+        int fetchSize,
+        Limit limit,
+        CircuitBreaker circuitBreaker,
+        int maxSamplesPerKey
+    ) {
         this.client = client;
         this.criteria = criteria;
         this.maxCriteria = criteria.size();
@@ -92,6 +100,7 @@ public SampleIterator(QueryClient client, List<SampleCriterion> criteria, int fe
         this.samples = new ArrayList<>();
         this.limit = limit;
         this.circuitBreaker = circuitBreaker;
+        this.maxSamplesPerKey = maxSamplesPerKey;
     }
 
     @Override
@@ -194,7 +203,7 @@ private void finalStep(ActionListener<Payload> listener) {
             // take each filter query and add to it a filter by join keys with the corresponding values
             for (SampleCriterion criterion : criteria) {
                 SampleQueryRequest r = criterion.finalQuery();
-                r.singleKeyPair(compositeKeyValues, maxCriteria);
+                r.singleKeyPair(compositeKeyValues, maxCriteria, maxSamplesPerKey);
                 searches.add(prepareRequest(r.searchSource(), false, EMPTY_ARRAY));
             }
             sampleKeys.add(new SequenceKey(compositeKeyValues.toArray()));
@@ -213,8 +222,8 @@ private void finalStep(ActionListener<Payload> listener) {
                     sample.add(hits);
                 }
                 if (docGroupsCounter == maxCriteria) {
-                    List<SearchHit> match = matchSample(sample, maxCriteria);
-                    if (match != null) {
+                    List<List<SearchHit>> matches = matchSamples(sample, maxCriteria, maxSamplesPerKey);
+                    for (List<SearchHit> match : matches) {
                         if (samples.size() < limit.limit()) {
                             samples.add(new Sample(sampleKeys.get(responseIndex / maxCriteria), match));
                         }
@@ -314,34 +323,40 @@ public List<HitReference> next() {
      * [doc1, doc1, doc3]
      * there is no solution.
      */
-    static List<SearchHit> matchSample(List<List<SearchHit>> hits, int hitsCount) {
+    static List<List<SearchHit>> matchSamples(List<List<SearchHit>> hits, int hitsCount, int maxSamplesPerKey) {
         if (hits.size() < hitsCount) {
             return null;
         }
-        List<SearchHit> result = new ArrayList<>(hitsCount);
-        if (match(0, hits, result, hitsCount)) {
-            return result;
-        }
-        return null;
+        List<List<SearchHit>> result = new ArrayList<>(maxSamplesPerKey);
+        match(0, hits, result, new ArrayList<>(hitsCount), hitsCount, maxSamplesPerKey);
+        return result;
     }
 
-    private static boolean match(int currentCriterion, List<List<SearchHit>> hits, List<SearchHit> result, int hitsCount) {
+    private static void match(
+        int currentCriterion,
+        List<List<SearchHit>> hits,
+        List<List<SearchHit>> result,
+        List<SearchHit> partial,
+        int hitsCount,
+        int maxSamplesPerKey
+    ) {
         for (SearchHit o : hits.get(currentCriterion)) {
-            if (result.contains(o) == false) {
-                result.add(o);
+            if (partial.contains(o) == false) {
+                partial.add(o);
                 if (currentCriterion == hitsCount - 1) {
-                    return true;
+                    result.add(new ArrayList<>(partial));
+                    if (maxSamplesPerKey == result.size()) {
+                        return;
+                    }
                 } else {
-                    if (match(currentCriterion + 1, hits, result, hitsCount)) {
-                        return true;
+                    match(currentCriterion + 1, hits, result, partial, hitsCount, maxSamplesPerKey);
+                    if (maxSamplesPerKey == result.size()) {
+                        return;
                     }
                 }
+                partial.remove(partial.size() - 1);
             }
         }
-        if (result.size() > 0) {
-            result.remove(result.get(result.size() - 1));
-        }
-        return false;
     }
 
     private void addMemory(long bytes, String label) {
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java
index 53fe7391018f..b090e768a358 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java
@@ -78,8 +78,8 @@ public LogicalPlan optimize(LogicalPlan verified) {
     }
 
     @Override
-    protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
-        Batch substitutions = new Batch(
+    protected Iterable<RuleExecutor.Batch<LogicalPlan>> batches() {
+        var substitutions = new Batch<>(
             "Substitution",
             Limiter.ONCE,
             new ReplaceWildcards(),
@@ -89,7 +89,7 @@ protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
             new AddMandatoryJoinKeyFilter()
         );
 
-        Batch operators = new Batch(
+        var operators = new Batch<>(
             "Operator Optimization",
             new ConstantFolding(),
             // boolean
@@ -107,13 +107,13 @@ protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
             new PushDownAndCombineFilters()
         );
 
-        Batch constraints = new Batch("Infer constraints", Limiter.ONCE, new PropagateJoinKeyConstraints());
+        var constraints = new Batch<>("Infer constraints", Limiter.ONCE, new PropagateJoinKeyConstraints());
 
-        Batch ordering = new Batch("Implicit Order", new SortByLimit(), new PushDownOrderBy());
+        var ordering = new Batch<>("Implicit Order", new SortByLimit(), new PushDownOrderBy());
 
-        Batch local = new Batch("Skip Elasticsearch", new SkipEmptyFilter(), new SkipEmptyJoin(), new SkipQueryOnLimitZero());
+        var local = new Batch<>("Skip Elasticsearch", new SkipEmptyFilter(), new SkipEmptyJoin(), new SkipQueryOnLimitZero());
 
-        Batch label = new Batch("Set as Optimized", Limiter.ONCE, new SetAsOptimized());
+        var label = new Batch<>("Set as Optimized", Limiter.ONCE, new SetAsOptimized());
 
         return asList(substitutions, operators, constraints, operators, ordering, local, label);
     }
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseBaseListener.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseBaseListener.java
index 2430245f16e7..ec9dd7d693b6 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseBaseListener.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseBaseListener.java
@@ -10,6 +10,7 @@
  * which can be extended to create a listener which only needs to handle a subset
  * of the available methods.
  */
+@SuppressWarnings("CheckReturnValue")
 class EqlBaseBaseListener implements EqlBaseListener {
     /**
      * {@inheritDoc}
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseBaseVisitor.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseBaseVisitor.java
index be6972314aa0..00fc279a6018 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseBaseVisitor.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseBaseVisitor.java
@@ -11,6 +11,7 @@
  * @param <T> The return type of the visit operation. Use {@link Void} for
  * operations with no return type.
  */
+@SuppressWarnings("CheckReturnValue")
 class EqlBaseBaseVisitor<T> extends AbstractParseTreeVisitor<T> implements EqlBaseVisitor<T> {
     /**
      * {@inheritDoc}
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseLexer.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseLexer.java
index e6a12bc7bd71..056140428e4f 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseLexer.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseLexer.java
@@ -8,10 +8,10 @@
 import org.antlr.v4.runtime.dfa.DFA;
 import org.antlr.v4.runtime.misc.*;
 
-@SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast" })
+@SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue" })
 class EqlBaseLexer extends Lexer {
     static {
-        RuntimeMetaData.checkVersion("4.9.2", RuntimeMetaData.VERSION);
+        RuntimeMetaData.checkVersion("4.11.1", RuntimeMetaData.VERSION);
     }
 
     protected static final DFA[] _decisionToDFA;
@@ -267,186 +267,323 @@ public ATN getATN() {
         return _ATN;
     }
 
-    public static final String _serializedATN = "\3\u608b\ua72a\u8133\ub9ed\u417c\u3be7\u7786\u5964\2\66\u01f4\b\1\4\2"
-        + "\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4"
-        + "\13\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22"
-        + "\t\22\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31"
-        + "\t\31\4\32\t\32\4\33\t\33\4\34\t\34\4\35\t\35\4\36\t\36\4\37\t\37\4 \t"
-        + " \4!\t!\4\"\t\"\4#\t#\4$\t$\4%\t%\4&\t&\4\'\t\'\4(\t(\4)\t)\4*\t*\4+\t"
-        + "+\4,\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\4\61\t\61\4\62\t\62\4\63\t\63\4\64"
-        + "\t\64\4\65\t\65\4\66\t\66\4\67\t\67\48\t8\49\t9\4:\t:\4;\t;\4<\t<\3\2"
-        + "\3\2\3\2\3\2\3\3\3\3\3\3\3\3\3\4\3\4\3\4\3\5\3\5\3\5\3\5\3\5\3\5\3\6\3"
-        + "\6\3\6\3\7\3\7\3\7\3\7\3\b\3\b\3\b\3\b\3\b\3\t\3\t\3\t\3\t\3\t\3\n\3\n"
-        + "\3\n\3\n\3\n\3\n\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\13\3\f\3\f\3\f\3"
-        + "\f\3\r\3\r\3\r\3\r\3\r\3\16\3\16\3\16\3\17\3\17\3\17\3\20\3\20\3\20\3"
-        + "\20\3\20\3\20\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\22\3"
-        + "\22\3\22\3\22\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\24\3\24\3"
-        + "\24\3\24\3\24\3\25\3\25\3\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\3\26\3"
-        + "\26\3\27\3\27\3\27\3\27\3\27\3\30\3\30\3\31\3\31\3\32\3\32\3\32\3\33\3"
-        + "\33\3\33\3\34\3\34\3\35\3\35\3\35\3\36\3\36\3\37\3\37\3\37\3 \3 \3!\3"
-        + "!\3\"\3\"\3#\3#\3$\3$\3%\3%\3&\3&\3\'\3\'\3(\3(\3)\3)\3*\3*\3+\3+\3,\3"
-        + ",\3-\3-\3-\3.\3.\3/\3/\3/\3/\3/\6/\u0124\n/\r/\16/\u0125\3/\3/\3\60\3"
-        + "\60\3\61\3\61\3\61\3\61\7\61\u0130\n\61\f\61\16\61\u0133\13\61\3\61\3"
-        + "\61\3\61\3\61\3\61\3\61\7\61\u013b\n\61\f\61\16\61\u013e\13\61\3\61\3"
-        + "\61\3\61\3\61\3\61\5\61\u0145\n\61\3\61\5\61\u0148\n\61\3\61\3\61\3\61"
-        + "\3\61\7\61\u014e\n\61\f\61\16\61\u0151\13\61\3\61\3\61\3\61\3\61\3\61"
-        + "\3\61\3\61\7\61\u015a\n\61\f\61\16\61\u015d\13\61\3\61\3\61\3\61\3\61"
-        + "\3\61\3\61\3\61\7\61\u0166\n\61\f\61\16\61\u0169\13\61\3\61\5\61\u016c"
-        + "\n\61\3\62\6\62\u016f\n\62\r\62\16\62\u0170\3\63\6\63\u0174\n\63\r\63"
-        + "\16\63\u0175\3\63\3\63\7\63\u017a\n\63\f\63\16\63\u017d\13\63\3\63\3\63"
-        + "\6\63\u0181\n\63\r\63\16\63\u0182\3\63\6\63\u0186\n\63\r\63\16\63\u0187"
-        + "\3\63\3\63\7\63\u018c\n\63\f\63\16\63\u018f\13\63\5\63\u0191\n\63\3\63"
-        + "\3\63\3\63\3\63\6\63\u0197\n\63\r\63\16\63\u0198\3\63\3\63\5\63\u019d"
-        + "\n\63\3\64\3\64\5\64\u01a1\n\64\3\64\3\64\3\64\7\64\u01a6\n\64\f\64\16"
-        + "\64\u01a9\13\64\3\65\3\65\3\65\3\65\7\65\u01af\n\65\f\65\16\65\u01b2\13"
-        + "\65\3\65\3\65\3\66\3\66\3\66\3\66\7\66\u01ba\n\66\f\66\16\66\u01bd\13"
-        + "\66\3\66\3\66\3\67\3\67\5\67\u01c3\n\67\3\67\6\67\u01c6\n\67\r\67\16\67"
-        + "\u01c7\38\38\39\39\3:\3:\3:\3:\7:\u01d2\n:\f:\16:\u01d5\13:\3:\5:\u01d8"
-        + "\n:\3:\5:\u01db\n:\3:\3:\3;\3;\3;\3;\3;\7;\u01e4\n;\f;\16;\u01e7\13;\3"
-        + ";\3;\3;\3;\3;\3<\6<\u01ef\n<\r<\16<\u01f0\3<\3<\4\u013c\u01e5\2=\3\3\5"
-        + "\4\7\5\t\6\13\7\r\b\17\t\21\n\23\13\25\f\27\r\31\16\33\17\35\20\37\21"
-        + "!\22#\23%\24\'\25)\26+\27-\30/\31\61\32\63\33\65\34\67\359\36;\37= ?!"
-        + "A\"C#E$G%I&K\'M(O)Q*S+U,W-Y\2[\2]\2_\2a.c/e\60g\61i\62k\63m\2o\2q\2s\64"
-        + "u\65w\66\3\2\20\n\2$$))^^ddhhppttvv\5\2\62;CHch\6\2\f\f\17\17$$^^\4\2"
-        + "\f\f\17\17\6\2\f\f\17\17))^^\5\2\f\f\17\17$$\5\2\f\f\17\17))\4\2BBaa\3"
-        + "\2bb\4\2GGgg\4\2--//\3\2\62;\4\2C\\c|\5\2\13\f\17\17\"\"\2\u0219\2\3\3"
-        + "\2\2\2\2\5\3\2\2\2\2\7\3\2\2\2\2\t\3\2\2\2\2\13\3\2\2\2\2\r\3\2\2\2\2"
-        + "\17\3\2\2\2\2\21\3\2\2\2\2\23\3\2\2\2\2\25\3\2\2\2\2\27\3\2\2\2\2\31\3"
-        + "\2\2\2\2\33\3\2\2\2\2\35\3\2\2\2\2\37\3\2\2\2\2!\3\2\2\2\2#\3\2\2\2\2"
-        + "%\3\2\2\2\2\'\3\2\2\2\2)\3\2\2\2\2+\3\2\2\2\2-\3\2\2\2\2/\3\2\2\2\2\61"
-        + "\3\2\2\2\2\63\3\2\2\2\2\65\3\2\2\2\2\67\3\2\2\2\29\3\2\2\2\2;\3\2\2\2"
-        + "\2=\3\2\2\2\2?\3\2\2\2\2A\3\2\2\2\2C\3\2\2\2\2E\3\2\2\2\2G\3\2\2\2\2I"
-        + "\3\2\2\2\2K\3\2\2\2\2M\3\2\2\2\2O\3\2\2\2\2Q\3\2\2\2\2S\3\2\2\2\2U\3\2"
-        + "\2\2\2W\3\2\2\2\2a\3\2\2\2\2c\3\2\2\2\2e\3\2\2\2\2g\3\2\2\2\2i\3\2\2\2"
-        + "\2k\3\2\2\2\2s\3\2\2\2\2u\3\2\2\2\2w\3\2\2\2\3y\3\2\2\2\5}\3\2\2\2\7\u0081"
-        + "\3\2\2\2\t\u0084\3\2\2\2\13\u008a\3\2\2\2\r\u008d\3\2\2\2\17\u0091\3\2"
-        + "\2\2\21\u0096\3\2\2\2\23\u009b\3\2\2\2\25\u00a1\3\2\2\2\27\u00a9\3\2\2"
-        + "\2\31\u00ad\3\2\2\2\33\u00b2\3\2\2\2\35\u00b5\3\2\2\2\37\u00b8\3\2\2\2"
-        + "!\u00be\3\2\2\2#\u00c5\3\2\2\2%\u00cc\3\2\2\2\'\u00d5\3\2\2\2)\u00da\3"
-        + "\2\2\2+\u00e0\3\2\2\2-\u00e6\3\2\2\2/\u00eb\3\2\2\2\61\u00ed\3\2\2\2\63"
-        + "\u00ef\3\2\2\2\65\u00f2\3\2\2\2\67\u00f5\3\2\2\29\u00f7\3\2\2\2;\u00fa"
-        + "\3\2\2\2=\u00fc\3\2\2\2?\u00ff\3\2\2\2A\u0101\3\2\2\2C\u0103\3\2\2\2E"
-        + "\u0105\3\2\2\2G\u0107\3\2\2\2I\u0109\3\2\2\2K\u010b\3\2\2\2M\u010d\3\2"
-        + "\2\2O\u010f\3\2\2\2Q\u0111\3\2\2\2S\u0113\3\2\2\2U\u0115\3\2\2\2W\u0117"
-        + "\3\2\2\2Y\u0119\3\2\2\2[\u011c\3\2\2\2]\u011e\3\2\2\2_\u0129\3\2\2\2a"
-        + "\u016b\3\2\2\2c\u016e\3\2\2\2e\u019c\3\2\2\2g\u01a0\3\2\2\2i\u01aa\3\2"
-        + "\2\2k\u01b5\3\2\2\2m\u01c0\3\2\2\2o\u01c9\3\2\2\2q\u01cb\3\2\2\2s\u01cd"
-        + "\3\2\2\2u\u01de\3\2\2\2w\u01ee\3\2\2\2yz\7c\2\2z{\7p\2\2{|\7f\2\2|\4\3"
-        + "\2\2\2}~\7c\2\2~\177\7p\2\2\177\u0080\7{\2\2\u0080\6\3\2\2\2\u0081\u0082"
-        + "\7d\2\2\u0082\u0083\7{\2\2\u0083\b\3\2\2\2\u0084\u0085\7h\2\2\u0085\u0086"
-        + "\7c\2\2\u0086\u0087\7n\2\2\u0087\u0088\7u\2\2\u0088\u0089\7g\2\2\u0089"
-        + "\n\3\2\2\2\u008a\u008b\7k\2\2\u008b\u008c\7p\2\2\u008c\f\3\2\2\2\u008d"
-        + "\u008e\7k\2\2\u008e\u008f\7p\2\2\u008f\u0090\7\u0080\2\2\u0090\16\3\2"
-        + "\2\2\u0091\u0092\7l\2\2\u0092\u0093\7q\2\2\u0093\u0094\7k\2\2\u0094\u0095"
-        + "\7p\2\2\u0095\20\3\2\2\2\u0096\u0097\7n\2\2\u0097\u0098\7k\2\2\u0098\u0099"
-        + "\7m\2\2\u0099\u009a\7g\2\2\u009a\22\3\2\2\2\u009b\u009c\7n\2\2\u009c\u009d"
-        + "\7k\2\2\u009d\u009e\7m\2\2\u009e\u009f\7g\2\2\u009f\u00a0\7\u0080\2\2"
-        + "\u00a0\24\3\2\2\2\u00a1\u00a2\7o\2\2\u00a2\u00a3\7c\2\2\u00a3\u00a4\7"
-        + "z\2\2\u00a4\u00a5\7u\2\2\u00a5\u00a6\7r\2\2\u00a6\u00a7\7c\2\2\u00a7\u00a8"
-        + "\7p\2\2\u00a8\26\3\2\2\2\u00a9\u00aa\7p\2\2\u00aa\u00ab\7q\2\2\u00ab\u00ac"
-        + "\7v\2\2\u00ac\30\3\2\2\2\u00ad\u00ae\7p\2\2\u00ae\u00af\7w\2\2\u00af\u00b0"
-        + "\7n\2\2\u00b0\u00b1\7n\2\2\u00b1\32\3\2\2\2\u00b2\u00b3\7q\2\2\u00b3\u00b4"
-        + "\7h\2\2\u00b4\34\3\2\2\2\u00b5\u00b6\7q\2\2\u00b6\u00b7\7t\2\2\u00b7\36"
-        + "\3\2\2\2\u00b8\u00b9\7t\2\2\u00b9\u00ba\7g\2\2\u00ba\u00bb\7i\2\2\u00bb"
-        + "\u00bc\7g\2\2\u00bc\u00bd\7z\2\2\u00bd \3\2\2\2\u00be\u00bf\7t\2\2\u00bf"
-        + "\u00c0\7g\2\2\u00c0\u00c1\7i\2\2\u00c1\u00c2\7g\2\2\u00c2\u00c3\7z\2\2"
-        + "\u00c3\u00c4\7\u0080\2\2\u00c4\"\3\2\2\2\u00c5\u00c6\7u\2\2\u00c6\u00c7"
-        + "\7c\2\2\u00c7\u00c8\7o\2\2\u00c8\u00c9\7r\2\2\u00c9\u00ca\7n\2\2\u00ca"
-        + "\u00cb\7g\2\2\u00cb$\3\2\2\2\u00cc\u00cd\7u\2\2\u00cd\u00ce\7g\2\2\u00ce"
-        + "\u00cf\7s\2\2\u00cf\u00d0\7w\2\2\u00d0\u00d1\7g\2\2\u00d1\u00d2\7p\2\2"
-        + "\u00d2\u00d3\7e\2\2\u00d3\u00d4\7g\2\2\u00d4&\3\2\2\2\u00d5\u00d6\7v\2"
-        + "\2\u00d6\u00d7\7t\2\2\u00d7\u00d8\7w\2\2\u00d8\u00d9\7g\2\2\u00d9(\3\2"
-        + "\2\2\u00da\u00db\7w\2\2\u00db\u00dc\7p\2\2\u00dc\u00dd\7v\2\2\u00dd\u00de"
-        + "\7k\2\2\u00de\u00df\7n\2\2\u00df*\3\2\2\2\u00e0\u00e1\7y\2\2\u00e1\u00e2"
-        + "\7j\2\2\u00e2\u00e3\7g\2\2\u00e3\u00e4\7t\2\2\u00e4\u00e5\7g\2\2\u00e5"
-        + ",\3\2\2\2\u00e6\u00e7\7y\2\2\u00e7\u00e8\7k\2\2\u00e8\u00e9\7v\2\2\u00e9"
-        + "\u00ea\7j\2\2\u00ea.\3\2\2\2\u00eb\u00ec\7<\2\2\u00ec\60\3\2\2\2\u00ed"
-        + "\u00ee\7?\2\2\u00ee\62\3\2\2\2\u00ef\u00f0\7?\2\2\u00f0\u00f1\7?\2\2\u00f1"
-        + "\64\3\2\2\2\u00f2\u00f3\7#\2\2\u00f3\u00f4\7?\2\2\u00f4\66\3\2\2\2\u00f5"
-        + "\u00f6\7>\2\2\u00f68\3\2\2\2\u00f7\u00f8\7>\2\2\u00f8\u00f9\7?\2\2\u00f9"
-        + ":\3\2\2\2\u00fa\u00fb\7@\2\2\u00fb<\3\2\2\2\u00fc\u00fd\7@\2\2\u00fd\u00fe"
-        + "\7?\2\2\u00fe>\3\2\2\2\u00ff\u0100\7-\2\2\u0100@\3\2\2\2\u0101\u0102\7"
-        + "/\2\2\u0102B\3\2\2\2\u0103\u0104\7,\2\2\u0104D\3\2\2\2\u0105\u0106\7\61"
-        + "\2\2\u0106F\3\2\2\2\u0107\u0108\7\'\2\2\u0108H\3\2\2\2\u0109\u010a\7\60"
-        + "\2\2\u010aJ\3\2\2\2\u010b\u010c\7.\2\2\u010cL\3\2\2\2\u010d\u010e\7]\2"
-        + "\2\u010eN\3\2\2\2\u010f\u0110\7_\2\2\u0110P\3\2\2\2\u0111\u0112\7*\2\2"
-        + "\u0112R\3\2\2\2\u0113\u0114\7+\2\2\u0114T\3\2\2\2\u0115\u0116\7~\2\2\u0116"
-        + "V\3\2\2\2\u0117\u0118\7A\2\2\u0118X\3\2\2\2\u0119\u011a\7^\2\2\u011a\u011b"
-        + "\t\2\2\2\u011bZ\3\2\2\2\u011c\u011d\t\3\2\2\u011d\\\3\2\2\2\u011e\u011f"
-        + "\7^\2\2\u011f\u0120\7w\2\2\u0120\u0121\3\2\2\2\u0121\u0123\7}\2\2\u0122"
-        + "\u0124\5[.\2\u0123\u0122\3\2\2\2\u0124\u0125\3\2\2\2\u0125\u0123\3\2\2"
-        + "\2\u0125\u0126\3\2\2\2\u0126\u0127\3\2\2\2\u0127\u0128\7\177\2\2\u0128"
-        + "^\3\2\2\2\u0129\u012a\n\4\2\2\u012a`\3\2\2\2\u012b\u0131\7$\2\2\u012c"
-        + "\u0130\5Y-\2\u012d\u0130\5]/\2\u012e\u0130\5_\60\2\u012f\u012c\3\2\2\2"
-        + "\u012f\u012d\3\2\2\2\u012f\u012e\3\2\2\2\u0130\u0133\3\2\2\2\u0131\u012f"
-        + "\3\2\2\2\u0131\u0132\3\2\2\2\u0132\u0134\3\2\2\2\u0133\u0131\3\2\2\2\u0134"
-        + "\u016c\7$\2\2\u0135\u0136\7$\2\2\u0136\u0137\7$\2\2\u0137\u0138\7$\2\2"
-        + "\u0138\u013c\3\2\2\2\u0139\u013b\n\5\2\2\u013a\u0139\3\2\2\2\u013b\u013e"
-        + "\3\2\2\2\u013c\u013d\3\2\2\2\u013c\u013a\3\2\2\2\u013d\u013f\3\2\2\2\u013e"
-        + "\u013c\3\2\2\2\u013f\u0140\7$\2\2\u0140\u0141\7$\2\2\u0141\u0142\7$\2"
-        + "\2\u0142\u0144\3\2\2\2\u0143\u0145\7$\2\2\u0144\u0143\3\2\2\2\u0144\u0145"
-        + "\3\2\2\2\u0145\u0147\3\2\2\2\u0146\u0148\7$\2\2\u0147\u0146\3\2\2\2\u0147"
-        + "\u0148\3\2\2\2\u0148\u016c\3\2\2\2\u0149\u014f\7)\2\2\u014a\u014b\7^\2"
-        + "\2\u014b\u014e\t\2\2\2\u014c\u014e\n\6\2\2\u014d\u014a\3\2\2\2\u014d\u014c"
-        + "\3\2\2\2\u014e\u0151\3\2\2\2\u014f\u014d\3\2\2\2\u014f\u0150\3\2\2\2\u0150"
-        + "\u0152\3\2\2\2\u0151\u014f\3\2\2\2\u0152\u016c\7)\2\2\u0153\u0154\7A\2"
-        + "\2\u0154\u0155\7$\2\2\u0155\u015b\3\2\2\2\u0156\u0157\7^\2\2\u0157\u015a"
-        + "\7$\2\2\u0158\u015a\n\7\2\2\u0159\u0156\3\2\2\2\u0159\u0158\3\2\2\2\u015a"
-        + "\u015d\3\2\2\2\u015b\u0159\3\2\2\2\u015b\u015c\3\2\2\2\u015c\u015e\3\2"
-        + "\2\2\u015d\u015b\3\2\2\2\u015e\u016c\7$\2\2\u015f\u0160\7A\2\2\u0160\u0161"
-        + "\7)\2\2\u0161\u0167\3\2\2\2\u0162\u0163\7^\2\2\u0163\u0166\7)\2\2\u0164"
-        + "\u0166\n\b\2\2\u0165\u0162\3\2\2\2\u0165\u0164\3\2\2\2\u0166\u0169\3\2"
-        + "\2\2\u0167\u0165\3\2\2\2\u0167\u0168\3\2\2\2\u0168\u016a\3\2\2\2\u0169"
-        + "\u0167\3\2\2\2\u016a\u016c\7)\2\2\u016b\u012b\3\2\2\2\u016b\u0135\3\2"
-        + "\2\2\u016b\u0149\3\2\2\2\u016b\u0153\3\2\2\2\u016b\u015f\3\2\2\2\u016c"
-        + "b\3\2\2\2\u016d\u016f\5o8\2\u016e\u016d\3\2\2\2\u016f\u0170\3\2\2\2\u0170"
-        + "\u016e\3\2\2\2\u0170\u0171\3\2\2\2\u0171d\3\2\2\2\u0172\u0174\5o8\2\u0173"
-        + "\u0172\3\2\2\2\u0174\u0175\3\2\2\2\u0175\u0173\3\2\2\2\u0175\u0176\3\2"
-        + "\2\2\u0176\u0177\3\2\2\2\u0177\u017b\5I%\2\u0178\u017a\5o8\2\u0179\u0178"
-        + "\3\2\2\2\u017a\u017d\3\2\2\2\u017b\u0179\3\2\2\2\u017b\u017c\3\2\2\2\u017c"
-        + "\u019d\3\2\2\2\u017d\u017b\3\2\2\2\u017e\u0180\5I%\2\u017f\u0181\5o8\2"
-        + "\u0180\u017f\3\2\2\2\u0181\u0182\3\2\2\2\u0182\u0180\3\2\2\2\u0182\u0183"
-        + "\3\2\2\2\u0183\u019d\3\2\2\2\u0184\u0186\5o8\2\u0185\u0184\3\2\2\2\u0186"
-        + "\u0187\3\2\2\2\u0187\u0185\3\2\2\2\u0187\u0188\3\2\2\2\u0188\u0190\3\2"
-        + "\2\2\u0189\u018d\5I%\2\u018a\u018c\5o8\2\u018b\u018a\3\2\2\2\u018c\u018f"
-        + "\3\2\2\2\u018d\u018b\3\2\2\2\u018d\u018e\3\2\2\2\u018e\u0191\3\2\2\2\u018f"
-        + "\u018d\3\2\2\2\u0190\u0189\3\2\2\2\u0190\u0191\3\2\2\2\u0191\u0192\3\2"
-        + "\2\2\u0192\u0193\5m\67\2\u0193\u019d\3\2\2\2\u0194\u0196\5I%\2\u0195\u0197"
-        + "\5o8\2\u0196\u0195\3\2\2\2\u0197\u0198\3\2\2\2\u0198\u0196\3\2\2\2\u0198"
-        + "\u0199\3\2\2\2\u0199\u019a\3\2\2\2\u019a\u019b\5m\67\2\u019b\u019d\3\2"
-        + "\2\2\u019c\u0173\3\2\2\2\u019c\u017e\3\2\2\2\u019c\u0185\3\2\2\2\u019c"
-        + "\u0194\3\2\2\2\u019df\3\2\2\2\u019e\u01a1\5q9\2\u019f\u01a1\t\t\2\2\u01a0"
-        + "\u019e\3\2\2\2\u01a0\u019f\3\2\2\2\u01a1\u01a7\3\2\2\2\u01a2\u01a6\5q"
-        + "9\2\u01a3\u01a6\5o8\2\u01a4\u01a6\7a\2\2\u01a5\u01a2\3\2\2\2\u01a5\u01a3"
-        + "\3\2\2\2\u01a5\u01a4\3\2\2\2\u01a6\u01a9\3\2\2\2\u01a7\u01a5\3\2\2\2\u01a7"
-        + "\u01a8\3\2\2\2\u01a8h\3\2\2\2\u01a9\u01a7\3\2\2\2\u01aa\u01b0\7b\2\2\u01ab"
-        + "\u01af\n\n\2\2\u01ac\u01ad\7b\2\2\u01ad\u01af\7b\2\2\u01ae\u01ab\3\2\2"
-        + "\2\u01ae\u01ac\3\2\2\2\u01af\u01b2\3\2\2\2\u01b0\u01ae\3\2\2\2\u01b0\u01b1"
-        + "\3\2\2\2\u01b1\u01b3\3\2\2\2\u01b2\u01b0\3\2\2\2\u01b3\u01b4\7b\2\2\u01b4"
-        + "j\3\2\2\2\u01b5\u01bb\5q9\2\u01b6\u01ba\5q9\2\u01b7\u01ba\5o8\2\u01b8"
-        + "\u01ba\7a\2\2\u01b9\u01b6\3\2\2\2\u01b9\u01b7\3\2\2\2\u01b9\u01b8\3\2"
-        + "\2\2\u01ba\u01bd\3\2\2\2\u01bb\u01b9\3\2\2\2\u01bb\u01bc\3\2\2\2\u01bc"
-        + "\u01be\3\2\2\2\u01bd\u01bb\3\2\2\2\u01be\u01bf\7\u0080\2\2\u01bfl\3\2"
-        + "\2\2\u01c0\u01c2\t\13\2\2\u01c1\u01c3\t\f\2\2\u01c2\u01c1\3\2\2\2\u01c2"
-        + "\u01c3\3\2\2\2\u01c3\u01c5\3\2\2\2\u01c4\u01c6\5o8\2\u01c5\u01c4\3\2\2"
-        + "\2\u01c6\u01c7\3\2\2\2\u01c7\u01c5\3\2\2\2\u01c7\u01c8\3\2\2\2\u01c8n"
-        + "\3\2\2\2\u01c9\u01ca\t\r\2\2\u01cap\3\2\2\2\u01cb\u01cc\t\16\2\2\u01cc"
-        + "r\3\2\2\2\u01cd\u01ce\7\61\2\2\u01ce\u01cf\7\61\2\2\u01cf\u01d3\3\2\2"
-        + "\2\u01d0\u01d2\n\5\2\2\u01d1\u01d0\3\2\2\2\u01d2\u01d5\3\2\2\2\u01d3\u01d1"
-        + "\3\2\2\2\u01d3\u01d4\3\2\2\2\u01d4\u01d7\3\2\2\2\u01d5\u01d3\3\2\2\2\u01d6"
-        + "\u01d8\7\17\2\2\u01d7\u01d6\3\2\2\2\u01d7\u01d8\3\2\2\2\u01d8\u01da\3"
-        + "\2\2\2\u01d9\u01db\7\f\2\2\u01da\u01d9\3\2\2\2\u01da\u01db\3\2\2\2\u01db"
-        + "\u01dc\3\2\2\2\u01dc\u01dd\b:\2\2\u01ddt\3\2\2\2\u01de\u01df\7\61\2\2"
-        + "\u01df\u01e0\7,\2\2\u01e0\u01e5\3\2\2\2\u01e1\u01e4\5u;\2\u01e2\u01e4"
-        + "\13\2\2\2\u01e3\u01e1\3\2\2\2\u01e3\u01e2\3\2\2\2\u01e4\u01e7\3\2\2\2"
-        + "\u01e5\u01e6\3\2\2\2\u01e5\u01e3\3\2\2\2\u01e6\u01e8\3\2\2\2\u01e7\u01e5"
-        + "\3\2\2\2\u01e8\u01e9\7,\2\2\u01e9\u01ea\7\61\2\2\u01ea\u01eb\3\2\2\2\u01eb"
-        + "\u01ec\b;\2\2\u01ecv\3\2\2\2\u01ed\u01ef\t\17\2\2\u01ee\u01ed\3\2\2\2"
-        + "\u01ef\u01f0\3\2\2\2\u01f0\u01ee\3\2\2\2\u01f0\u01f1\3\2\2\2\u01f1\u01f2"
-        + "\3\2\2\2\u01f2\u01f3\b<\2\2\u01f3x\3\2\2\2(\2\u0125\u012f\u0131\u013c"
-        + "\u0144\u0147\u014d\u014f\u0159\u015b\u0165\u0167\u016b\u0170\u0175\u017b"
-        + "\u0182\u0187\u018d\u0190\u0198\u019c\u01a0\u01a5\u01a7\u01ae\u01b0\u01b9"
-        + "\u01bb\u01c2\u01c7\u01d3\u01d7\u01da\u01e3\u01e5\u01f0\3\2\3\2";
+    public static final String _serializedATN = "\u0004\u00004\u01f2\u0006\uffff\uffff\u0002\u0000\u0007\u0000\u0002\u0001"
+        + "\u0007\u0001\u0002\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004"
+        + "\u0007\u0004\u0002\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007"
+        + "\u0007\u0007\u0002\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b"
+        + "\u0007\u000b\u0002\f\u0007\f\u0002\r\u0007\r\u0002\u000e\u0007\u000e\u0002"
+        + "\u000f\u0007\u000f\u0002\u0010\u0007\u0010\u0002\u0011\u0007\u0011\u0002"
+        + "\u0012\u0007\u0012\u0002\u0013\u0007\u0013\u0002\u0014\u0007\u0014\u0002"
+        + "\u0015\u0007\u0015\u0002\u0016\u0007\u0016\u0002\u0017\u0007\u0017\u0002"
+        + "\u0018\u0007\u0018\u0002\u0019\u0007\u0019\u0002\u001a\u0007\u001a\u0002"
+        + "\u001b\u0007\u001b\u0002\u001c\u0007\u001c\u0002\u001d\u0007\u001d\u0002"
+        + "\u001e\u0007\u001e\u0002\u001f\u0007\u001f\u0002 \u0007 \u0002!\u0007"
+        + "!\u0002\"\u0007\"\u0002#\u0007#\u0002$\u0007$\u0002%\u0007%\u0002&\u0007"
+        + "&\u0002\'\u0007\'\u0002(\u0007(\u0002)\u0007)\u0002*\u0007*\u0002+\u0007"
+        + "+\u0002,\u0007,\u0002-\u0007-\u0002.\u0007.\u0002/\u0007/\u00020\u0007"
+        + "0\u00021\u00071\u00022\u00072\u00023\u00073\u00024\u00074\u00025\u0007"
+        + "5\u00026\u00076\u00027\u00077\u00028\u00078\u00029\u00079\u0002:\u0007"
+        + ":\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001"
+        + "\u0001\u0001\u0001\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0003"
+        + "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0004"
+        + "\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"
+        + "\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0007"
+        + "\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\b\u0001\b\u0001"
+        + "\b\u0001\b\u0001\b\u0001\b\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001"
+        + "\t\u0001\t\u0001\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\u000b\u0001\u000b"
+        + "\u0001\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\f\u0001\r\u0001"
+        + "\r\u0001\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e"
+        + "\u0001\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f"
+        + "\u0001\u000f\u0001\u000f\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010"
+        + "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0011\u0001\u0011\u0001\u0011"
+        + "\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011"
+        + "\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0013"
+        + "\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0014"
+        + "\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0015"
+        + "\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0016\u0001\u0016"
+        + "\u0001\u0017\u0001\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0019"
+        + "\u0001\u0019\u0001\u0019\u0001\u001a\u0001\u001a\u0001\u001b\u0001\u001b"
+        + "\u0001\u001b\u0001\u001c\u0001\u001c\u0001\u001d\u0001\u001d\u0001\u001d"
+        + "\u0001\u001e\u0001\u001e\u0001\u001f\u0001\u001f\u0001 \u0001 \u0001!"
+        + "\u0001!\u0001\"\u0001\"\u0001#\u0001#\u0001$\u0001$\u0001%\u0001%\u0001"
+        + "&\u0001&\u0001\'\u0001\'\u0001(\u0001(\u0001)\u0001)\u0001*\u0001*\u0001"
+        + "+\u0001+\u0001+\u0001,\u0001,\u0001-\u0001-\u0001-\u0001-\u0001-\u0004"
+        + "-\u0122\b-\u000b-\f-\u0123\u0001-\u0001-\u0001.\u0001.\u0001/\u0001/\u0001"
+        + "/\u0001/\u0005/\u012e\b/\n/\f/\u0131\t/\u0001/\u0001/\u0001/\u0001/\u0001"
+        + "/\u0001/\u0005/\u0139\b/\n/\f/\u013c\t/\u0001/\u0001/\u0001/\u0001/\u0001"
+        + "/\u0003/\u0143\b/\u0001/\u0003/\u0146\b/\u0001/\u0001/\u0001/\u0001/\u0005"
+        + "/\u014c\b/\n/\f/\u014f\t/\u0001/\u0001/\u0001/\u0001/\u0001/\u0001/\u0001"
+        + "/\u0005/\u0158\b/\n/\f/\u015b\t/\u0001/\u0001/\u0001/\u0001/\u0001/\u0001"
+        + "/\u0001/\u0005/\u0164\b/\n/\f/\u0167\t/\u0001/\u0003/\u016a\b/\u00010"
+        + "\u00040\u016d\b0\u000b0\f0\u016e\u00011\u00041\u0172\b1\u000b1\f1\u0173"
+        + "\u00011\u00011\u00051\u0178\b1\n1\f1\u017b\t1\u00011\u00011\u00041\u017f"
+        + "\b1\u000b1\f1\u0180\u00011\u00041\u0184\b1\u000b1\f1\u0185\u00011\u0001"
+        + "1\u00051\u018a\b1\n1\f1\u018d\t1\u00031\u018f\b1\u00011\u00011\u00011"
+        + "\u00011\u00041\u0195\b1\u000b1\f1\u0196\u00011\u00011\u00031\u019b\b1"
+        + "\u00012\u00012\u00032\u019f\b2\u00012\u00012\u00012\u00052\u01a4\b2\n"
+        + "2\f2\u01a7\t2\u00013\u00013\u00013\u00013\u00053\u01ad\b3\n3\f3\u01b0"
+        + "\t3\u00013\u00013\u00014\u00014\u00014\u00014\u00054\u01b8\b4\n4\f4\u01bb"
+        + "\t4\u00014\u00014\u00015\u00015\u00035\u01c1\b5\u00015\u00045\u01c4\b"
+        + "5\u000b5\f5\u01c5\u00016\u00016\u00017\u00017\u00018\u00018\u00018\u0001"
+        + "8\u00058\u01d0\b8\n8\f8\u01d3\t8\u00018\u00038\u01d6\b8\u00018\u00038"
+        + "\u01d9\b8\u00018\u00018\u00019\u00019\u00019\u00019\u00019\u00059\u01e2"
+        + "\b9\n9\f9\u01e5\t9\u00019\u00019\u00019\u00019\u00019\u0001:\u0004:\u01ed"
+        + "\b:\u000b:\f:\u01ee\u0001:\u0001:\u0002\u013a\u01e3\u0000;\u0001\u0001"
+        + "\u0003\u0002\u0005\u0003\u0007\u0004\t\u0005\u000b\u0006\r\u0007\u000f"
+        + "\b\u0011\t\u0013\n\u0015\u000b\u0017\f\u0019\r\u001b\u000e\u001d\u000f"
+        + "\u001f\u0010!\u0011#\u0012%\u0013\'\u0014)\u0015+\u0016-\u0017/\u0018"
+        + "1\u00193\u001a5\u001b7\u001c9\u001d;\u001e=\u001f? A!C\"E#G$I%K&M\'O("
+        + "Q)S*U+W\u0000Y\u0000[\u0000]\u0000_,a-c.e/g0i1k\u0000m\u0000o\u0000q2"
+        + "s3u4\u0001\u0000\u000e\b\u0000\"\"\'\'\\\\bbffnnrrtt\u0003\u000009AFa"
+        + "f\u0004\u0000\n\n\r\r\"\"\\\\\u0002\u0000\n\n\r\r\u0004\u0000\n\n\r\r"
+        + "\'\'\\\\\u0003\u0000\n\n\r\r\"\"\u0003\u0000\n\n\r\r\'\'\u0002\u0000@"
+        + "@__\u0001\u0000``\u0002\u0000EEee\u0002\u0000++--\u0001\u000009\u0002"
+        + "\u0000AZaz\u0003\u0000\t\n\r\r  \u0217\u0000\u0001\u0001\u0000\u0000\u0000"
+        + "\u0000\u0003\u0001\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000"
+        + "\u0000\u0007\u0001\u0000\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0000"
+        + "\u000b\u0001\u0000\u0000\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f"
+        + "\u0001\u0000\u0000\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013"
+        + "\u0001\u0000\u0000\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017"
+        + "\u0001\u0000\u0000\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b"
+        + "\u0001\u0000\u0000\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f"
+        + "\u0001\u0000\u0000\u0000\u0000!\u0001\u0000\u0000\u0000\u0000#\u0001\u0000"
+        + "\u0000\u0000\u0000%\u0001\u0000\u0000\u0000\u0000\'\u0001\u0000\u0000"
+        + "\u0000\u0000)\u0001\u0000\u0000\u0000\u0000+\u0001\u0000\u0000\u0000\u0000"
+        + "-\u0001\u0000\u0000\u0000\u0000/\u0001\u0000\u0000\u0000\u00001\u0001"
+        + "\u0000\u0000\u0000\u00003\u0001\u0000\u0000\u0000\u00005\u0001\u0000\u0000"
+        + "\u0000\u00007\u0001\u0000\u0000\u0000\u00009\u0001\u0000\u0000\u0000\u0000"
+        + ";\u0001\u0000\u0000\u0000\u0000=\u0001\u0000\u0000\u0000\u0000?\u0001"
+        + "\u0000\u0000\u0000\u0000A\u0001\u0000\u0000\u0000\u0000C\u0001\u0000\u0000"
+        + "\u0000\u0000E\u0001\u0000\u0000\u0000\u0000G\u0001\u0000\u0000\u0000\u0000"
+        + "I\u0001\u0000\u0000\u0000\u0000K\u0001\u0000\u0000\u0000\u0000M\u0001"
+        + "\u0000\u0000\u0000\u0000O\u0001\u0000\u0000\u0000\u0000Q\u0001\u0000\u0000"
+        + "\u0000\u0000S\u0001\u0000\u0000\u0000\u0000U\u0001\u0000\u0000\u0000\u0000"
+        + "_\u0001\u0000\u0000\u0000\u0000a\u0001\u0000\u0000\u0000\u0000c\u0001"
+        + "\u0000\u0000\u0000\u0000e\u0001\u0000\u0000\u0000\u0000g\u0001\u0000\u0000"
+        + "\u0000\u0000i\u0001\u0000\u0000\u0000\u0000q\u0001\u0000\u0000\u0000\u0000"
+        + "s\u0001\u0000\u0000\u0000\u0000u\u0001\u0000\u0000\u0000\u0001w\u0001"
+        + "\u0000\u0000\u0000\u0003{\u0001\u0000\u0000\u0000\u0005\u007f\u0001\u0000"
+        + "\u0000\u0000\u0007\u0082\u0001\u0000\u0000\u0000\t\u0088\u0001\u0000\u0000"
+        + "\u0000\u000b\u008b\u0001\u0000\u0000\u0000\r\u008f\u0001\u0000\u0000\u0000"
+        + "\u000f\u0094\u0001\u0000\u0000\u0000\u0011\u0099\u0001\u0000\u0000\u0000"
+        + "\u0013\u009f\u0001\u0000\u0000\u0000\u0015\u00a7\u0001\u0000\u0000\u0000"
+        + "\u0017\u00ab\u0001\u0000\u0000\u0000\u0019\u00b0\u0001\u0000\u0000\u0000"
+        + "\u001b\u00b3\u0001\u0000\u0000\u0000\u001d\u00b6\u0001\u0000\u0000\u0000"
+        + "\u001f\u00bc\u0001\u0000\u0000\u0000!\u00c3\u0001\u0000\u0000\u0000#\u00ca"
+        + "\u0001\u0000\u0000\u0000%\u00d3\u0001\u0000\u0000\u0000\'\u00d8\u0001"
+        + "\u0000\u0000\u0000)\u00de\u0001\u0000\u0000\u0000+\u00e4\u0001\u0000\u0000"
+        + "\u0000-\u00e9\u0001\u0000\u0000\u0000/\u00eb\u0001\u0000\u0000\u00001"
+        + "\u00ed\u0001\u0000\u0000\u00003\u00f0\u0001\u0000\u0000\u00005\u00f3\u0001"
+        + "\u0000\u0000\u00007\u00f5\u0001\u0000\u0000\u00009\u00f8\u0001\u0000\u0000"
+        + "\u0000;\u00fa\u0001\u0000\u0000\u0000=\u00fd\u0001\u0000\u0000\u0000?"
+        + "\u00ff\u0001\u0000\u0000\u0000A\u0101\u0001\u0000\u0000\u0000C\u0103\u0001"
+        + "\u0000\u0000\u0000E\u0105\u0001\u0000\u0000\u0000G\u0107\u0001\u0000\u0000"
+        + "\u0000I\u0109\u0001\u0000\u0000\u0000K\u010b\u0001\u0000\u0000\u0000M"
+        + "\u010d\u0001\u0000\u0000\u0000O\u010f\u0001\u0000\u0000\u0000Q\u0111\u0001"
+        + "\u0000\u0000\u0000S\u0113\u0001\u0000\u0000\u0000U\u0115\u0001\u0000\u0000"
+        + "\u0000W\u0117\u0001\u0000\u0000\u0000Y\u011a\u0001\u0000\u0000\u0000["
+        + "\u011c\u0001\u0000\u0000\u0000]\u0127\u0001\u0000\u0000\u0000_\u0169\u0001"
+        + "\u0000\u0000\u0000a\u016c\u0001\u0000\u0000\u0000c\u019a\u0001\u0000\u0000"
+        + "\u0000e\u019e\u0001\u0000\u0000\u0000g\u01a8\u0001\u0000\u0000\u0000i"
+        + "\u01b3\u0001\u0000\u0000\u0000k\u01be\u0001\u0000\u0000\u0000m\u01c7\u0001"
+        + "\u0000\u0000\u0000o\u01c9\u0001\u0000\u0000\u0000q\u01cb\u0001\u0000\u0000"
+        + "\u0000s\u01dc\u0001\u0000\u0000\u0000u\u01ec\u0001\u0000\u0000\u0000w"
+        + "x\u0005a\u0000\u0000xy\u0005n\u0000\u0000yz\u0005d\u0000\u0000z\u0002"
+        + "\u0001\u0000\u0000\u0000{|\u0005a\u0000\u0000|}\u0005n\u0000\u0000}~\u0005"
+        + "y\u0000\u0000~\u0004\u0001\u0000\u0000\u0000\u007f\u0080\u0005b\u0000"
+        + "\u0000\u0080\u0081\u0005y\u0000\u0000\u0081\u0006\u0001\u0000\u0000\u0000"
+        + "\u0082\u0083\u0005f\u0000\u0000\u0083\u0084\u0005a\u0000\u0000\u0084\u0085"
+        + "\u0005l\u0000\u0000\u0085\u0086\u0005s\u0000\u0000\u0086\u0087\u0005e"
+        + "\u0000\u0000\u0087\b\u0001\u0000\u0000\u0000\u0088\u0089\u0005i\u0000"
+        + "\u0000\u0089\u008a\u0005n\u0000\u0000\u008a\n\u0001\u0000\u0000\u0000"
+        + "\u008b\u008c\u0005i\u0000\u0000\u008c\u008d\u0005n\u0000\u0000\u008d\u008e"
+        + "\u0005~\u0000\u0000\u008e\f\u0001\u0000\u0000\u0000\u008f\u0090\u0005"
+        + "j\u0000\u0000\u0090\u0091\u0005o\u0000\u0000\u0091\u0092\u0005i\u0000"
+        + "\u0000\u0092\u0093\u0005n\u0000\u0000\u0093\u000e\u0001\u0000\u0000\u0000"
+        + "\u0094\u0095\u0005l\u0000\u0000\u0095\u0096\u0005i\u0000\u0000\u0096\u0097"
+        + "\u0005k\u0000\u0000\u0097\u0098\u0005e\u0000\u0000\u0098\u0010\u0001\u0000"
+        + "\u0000\u0000\u0099\u009a\u0005l\u0000\u0000\u009a\u009b\u0005i\u0000\u0000"
+        + "\u009b\u009c\u0005k\u0000\u0000\u009c\u009d\u0005e\u0000\u0000\u009d\u009e"
+        + "\u0005~\u0000\u0000\u009e\u0012\u0001\u0000\u0000\u0000\u009f\u00a0\u0005"
+        + "m\u0000\u0000\u00a0\u00a1\u0005a\u0000\u0000\u00a1\u00a2\u0005x\u0000"
+        + "\u0000\u00a2\u00a3\u0005s\u0000\u0000\u00a3\u00a4\u0005p\u0000\u0000\u00a4"
+        + "\u00a5\u0005a\u0000\u0000\u00a5\u00a6\u0005n\u0000\u0000\u00a6\u0014\u0001"
+        + "\u0000\u0000\u0000\u00a7\u00a8\u0005n\u0000\u0000\u00a8\u00a9\u0005o\u0000"
+        + "\u0000\u00a9\u00aa\u0005t\u0000\u0000\u00aa\u0016\u0001\u0000\u0000\u0000"
+        + "\u00ab\u00ac\u0005n\u0000\u0000\u00ac\u00ad\u0005u\u0000\u0000\u00ad\u00ae"
+        + "\u0005l\u0000\u0000\u00ae\u00af\u0005l\u0000\u0000\u00af\u0018\u0001\u0000"
+        + "\u0000\u0000\u00b0\u00b1\u0005o\u0000\u0000\u00b1\u00b2\u0005f\u0000\u0000"
+        + "\u00b2\u001a\u0001\u0000\u0000\u0000\u00b3\u00b4\u0005o\u0000\u0000\u00b4"
+        + "\u00b5\u0005r\u0000\u0000\u00b5\u001c\u0001\u0000\u0000\u0000\u00b6\u00b7"
+        + "\u0005r\u0000\u0000\u00b7\u00b8\u0005e\u0000\u0000\u00b8\u00b9\u0005g"
+        + "\u0000\u0000\u00b9\u00ba\u0005e\u0000\u0000\u00ba\u00bb\u0005x\u0000\u0000"
+        + "\u00bb\u001e\u0001\u0000\u0000\u0000\u00bc\u00bd\u0005r\u0000\u0000\u00bd"
+        + "\u00be\u0005e\u0000\u0000\u00be\u00bf\u0005g\u0000\u0000\u00bf\u00c0\u0005"
+        + "e\u0000\u0000\u00c0\u00c1\u0005x\u0000\u0000\u00c1\u00c2\u0005~\u0000"
+        + "\u0000\u00c2 \u0001\u0000\u0000\u0000\u00c3\u00c4\u0005s\u0000\u0000\u00c4"
+        + "\u00c5\u0005a\u0000\u0000\u00c5\u00c6\u0005m\u0000\u0000\u00c6\u00c7\u0005"
+        + "p\u0000\u0000\u00c7\u00c8\u0005l\u0000\u0000\u00c8\u00c9\u0005e\u0000"
+        + "\u0000\u00c9\"\u0001\u0000\u0000\u0000\u00ca\u00cb\u0005s\u0000\u0000"
+        + "\u00cb\u00cc\u0005e\u0000\u0000\u00cc\u00cd\u0005q\u0000\u0000\u00cd\u00ce"
+        + "\u0005u\u0000\u0000\u00ce\u00cf\u0005e\u0000\u0000\u00cf\u00d0\u0005n"
+        + "\u0000\u0000\u00d0\u00d1\u0005c\u0000\u0000\u00d1\u00d2\u0005e\u0000\u0000"
+        + "\u00d2$\u0001\u0000\u0000\u0000\u00d3\u00d4\u0005t\u0000\u0000\u00d4\u00d5"
+        + "\u0005r\u0000\u0000\u00d5\u00d6\u0005u\u0000\u0000\u00d6\u00d7\u0005e"
+        + "\u0000\u0000\u00d7&\u0001\u0000\u0000\u0000\u00d8\u00d9\u0005u\u0000\u0000"
+        + "\u00d9\u00da\u0005n\u0000\u0000\u00da\u00db\u0005t\u0000\u0000\u00db\u00dc"
+        + "\u0005i\u0000\u0000\u00dc\u00dd\u0005l\u0000\u0000\u00dd(\u0001\u0000"
+        + "\u0000\u0000\u00de\u00df\u0005w\u0000\u0000\u00df\u00e0\u0005h\u0000\u0000"
+        + "\u00e0\u00e1\u0005e\u0000\u0000\u00e1\u00e2\u0005r\u0000\u0000\u00e2\u00e3"
+        + "\u0005e\u0000\u0000\u00e3*\u0001\u0000\u0000\u0000\u00e4\u00e5\u0005w"
+        + "\u0000\u0000\u00e5\u00e6\u0005i\u0000\u0000\u00e6\u00e7\u0005t\u0000\u0000"
+        + "\u00e7\u00e8\u0005h\u0000\u0000\u00e8,\u0001\u0000\u0000\u0000\u00e9\u00ea"
+        + "\u0005:\u0000\u0000\u00ea.\u0001\u0000\u0000\u0000\u00eb\u00ec\u0005="
+        + "\u0000\u0000\u00ec0\u0001\u0000\u0000\u0000\u00ed\u00ee\u0005=\u0000\u0000"
+        + "\u00ee\u00ef\u0005=\u0000\u0000\u00ef2\u0001\u0000\u0000\u0000\u00f0\u00f1"
+        + "\u0005!\u0000\u0000\u00f1\u00f2\u0005=\u0000\u0000\u00f24\u0001\u0000"
+        + "\u0000\u0000\u00f3\u00f4\u0005<\u0000\u0000\u00f46\u0001\u0000\u0000\u0000"
+        + "\u00f5\u00f6\u0005<\u0000\u0000\u00f6\u00f7\u0005=\u0000\u0000\u00f78"
+        + "\u0001\u0000\u0000\u0000\u00f8\u00f9\u0005>\u0000\u0000\u00f9:\u0001\u0000"
+        + "\u0000\u0000\u00fa\u00fb\u0005>\u0000\u0000\u00fb\u00fc\u0005=\u0000\u0000"
+        + "\u00fc<\u0001\u0000\u0000\u0000\u00fd\u00fe\u0005+\u0000\u0000\u00fe>"
+        + "\u0001\u0000\u0000\u0000\u00ff\u0100\u0005-\u0000\u0000\u0100@\u0001\u0000"
+        + "\u0000\u0000\u0101\u0102\u0005*\u0000\u0000\u0102B\u0001\u0000\u0000\u0000"
+        + "\u0103\u0104\u0005/\u0000\u0000\u0104D\u0001\u0000\u0000\u0000\u0105\u0106"
+        + "\u0005%\u0000\u0000\u0106F\u0001\u0000\u0000\u0000\u0107\u0108\u0005."
+        + "\u0000\u0000\u0108H\u0001\u0000\u0000\u0000\u0109\u010a\u0005,\u0000\u0000"
+        + "\u010aJ\u0001\u0000\u0000\u0000\u010b\u010c\u0005[\u0000\u0000\u010cL"
+        + "\u0001\u0000\u0000\u0000\u010d\u010e\u0005]\u0000\u0000\u010eN\u0001\u0000"
+        + "\u0000\u0000\u010f\u0110\u0005(\u0000\u0000\u0110P\u0001\u0000\u0000\u0000"
+        + "\u0111\u0112\u0005)\u0000\u0000\u0112R\u0001\u0000\u0000\u0000\u0113\u0114"
+        + "\u0005|\u0000\u0000\u0114T\u0001\u0000\u0000\u0000\u0115\u0116\u0005?"
+        + "\u0000\u0000\u0116V\u0001\u0000\u0000\u0000\u0117\u0118\u0005\\\u0000"
+        + "\u0000\u0118\u0119\u0007\u0000\u0000\u0000\u0119X\u0001\u0000\u0000\u0000"
+        + "\u011a\u011b\u0007\u0001\u0000\u0000\u011bZ\u0001\u0000\u0000\u0000\u011c"
+        + "\u011d\u0005\\\u0000\u0000\u011d\u011e\u0005u\u0000\u0000\u011e\u011f"
+        + "\u0001\u0000\u0000\u0000\u011f\u0121\u0005{\u0000\u0000\u0120\u0122\u0003"
+        + "Y,\u0000\u0121\u0120\u0001\u0000\u0000\u0000\u0122\u0123\u0001\u0000\u0000"
+        + "\u0000\u0123\u0121\u0001\u0000\u0000\u0000\u0123\u0124\u0001\u0000\u0000"
+        + "\u0000\u0124\u0125\u0001\u0000\u0000\u0000\u0125\u0126\u0005}\u0000\u0000"
+        + "\u0126\\\u0001\u0000\u0000\u0000\u0127\u0128\b\u0002\u0000\u0000\u0128"
+        + "^\u0001\u0000\u0000\u0000\u0129\u012f\u0005\"\u0000\u0000\u012a\u012e"
+        + "\u0003W+\u0000\u012b\u012e\u0003[-\u0000\u012c\u012e\u0003].\u0000\u012d"
+        + "\u012a\u0001\u0000\u0000\u0000\u012d\u012b\u0001\u0000\u0000\u0000\u012d"
+        + "\u012c\u0001\u0000\u0000\u0000\u012e\u0131\u0001\u0000\u0000\u0000\u012f"
+        + "\u012d\u0001\u0000\u0000\u0000\u012f\u0130\u0001\u0000\u0000\u0000\u0130"
+        + "\u0132\u0001\u0000\u0000\u0000\u0131\u012f\u0001\u0000\u0000\u0000\u0132"
+        + "\u016a\u0005\"\u0000\u0000\u0133\u0134\u0005\"\u0000\u0000\u0134\u0135"
+        + "\u0005\"\u0000\u0000\u0135\u0136\u0005\"\u0000\u0000\u0136\u013a\u0001"
+        + "\u0000\u0000\u0000\u0137\u0139\b\u0003\u0000\u0000\u0138\u0137\u0001\u0000"
+        + "\u0000\u0000\u0139\u013c\u0001\u0000\u0000\u0000\u013a\u013b\u0001\u0000"
+        + "\u0000\u0000\u013a\u0138\u0001\u0000\u0000\u0000\u013b\u013d\u0001\u0000"
+        + "\u0000\u0000\u013c\u013a\u0001\u0000\u0000\u0000\u013d\u013e\u0005\"\u0000"
+        + "\u0000\u013e\u013f\u0005\"\u0000\u0000\u013f\u0140\u0005\"\u0000\u0000"
+        + "\u0140\u0142\u0001\u0000\u0000\u0000\u0141\u0143\u0005\"\u0000\u0000\u0142"
+        + "\u0141\u0001\u0000\u0000\u0000\u0142\u0143\u0001\u0000\u0000\u0000\u0143"
+        + "\u0145\u0001\u0000\u0000\u0000\u0144\u0146\u0005\"\u0000\u0000\u0145\u0144"
+        + "\u0001\u0000\u0000\u0000\u0145\u0146\u0001\u0000\u0000\u0000\u0146\u016a"
+        + "\u0001\u0000\u0000\u0000\u0147\u014d\u0005\'\u0000\u0000\u0148\u0149\u0005"
+        + "\\\u0000\u0000\u0149\u014c\u0007\u0000\u0000\u0000\u014a\u014c\b\u0004"
+        + "\u0000\u0000\u014b\u0148\u0001\u0000\u0000\u0000\u014b\u014a\u0001\u0000"
+        + "\u0000\u0000\u014c\u014f\u0001\u0000\u0000\u0000\u014d\u014b\u0001\u0000"
+        + "\u0000\u0000\u014d\u014e\u0001\u0000\u0000\u0000\u014e\u0150\u0001\u0000"
+        + "\u0000\u0000\u014f\u014d\u0001\u0000\u0000\u0000\u0150\u016a\u0005\'\u0000"
+        + "\u0000\u0151\u0152\u0005?\u0000\u0000\u0152\u0153\u0005\"\u0000\u0000"
+        + "\u0153\u0159\u0001\u0000\u0000\u0000\u0154\u0155\u0005\\\u0000\u0000\u0155"
+        + "\u0158\u0005\"\u0000\u0000\u0156\u0158\b\u0005\u0000\u0000\u0157\u0154"
+        + "\u0001\u0000\u0000\u0000\u0157\u0156\u0001\u0000\u0000\u0000\u0158\u015b"
+        + "\u0001\u0000\u0000\u0000\u0159\u0157\u0001\u0000\u0000\u0000\u0159\u015a"
+        + "\u0001\u0000\u0000\u0000\u015a\u015c\u0001\u0000\u0000\u0000\u015b\u0159"
+        + "\u0001\u0000\u0000\u0000\u015c\u016a\u0005\"\u0000\u0000\u015d\u015e\u0005"
+        + "?\u0000\u0000\u015e\u015f\u0005\'\u0000\u0000\u015f\u0165\u0001\u0000"
+        + "\u0000\u0000\u0160\u0161\u0005\\\u0000\u0000\u0161\u0164\u0005\'\u0000"
+        + "\u0000\u0162\u0164\b\u0006\u0000\u0000\u0163\u0160\u0001\u0000\u0000\u0000"
+        + "\u0163\u0162\u0001\u0000\u0000\u0000\u0164\u0167\u0001\u0000\u0000\u0000"
+        + "\u0165\u0163\u0001\u0000\u0000\u0000\u0165\u0166\u0001\u0000\u0000\u0000"
+        + "\u0166\u0168\u0001\u0000\u0000\u0000\u0167\u0165\u0001\u0000\u0000\u0000"
+        + "\u0168\u016a\u0005\'\u0000\u0000\u0169\u0129\u0001\u0000\u0000\u0000\u0169"
+        + "\u0133\u0001\u0000\u0000\u0000\u0169\u0147\u0001\u0000\u0000\u0000\u0169"
+        + "\u0151\u0001\u0000\u0000\u0000\u0169\u015d\u0001\u0000\u0000\u0000\u016a"
+        + "`\u0001\u0000\u0000\u0000\u016b\u016d\u0003m6\u0000\u016c\u016b\u0001"
+        + "\u0000\u0000\u0000\u016d\u016e\u0001\u0000\u0000\u0000\u016e\u016c\u0001"
+        + "\u0000\u0000\u0000\u016e\u016f\u0001\u0000\u0000\u0000\u016fb\u0001\u0000"
+        + "\u0000\u0000\u0170\u0172\u0003m6\u0000\u0171\u0170\u0001\u0000\u0000\u0000"
+        + "\u0172\u0173\u0001\u0000\u0000\u0000\u0173\u0171\u0001\u0000\u0000\u0000"
+        + "\u0173\u0174\u0001\u0000\u0000\u0000\u0174\u0175\u0001\u0000\u0000\u0000"
+        + "\u0175\u0179\u0003G#\u0000\u0176\u0178\u0003m6\u0000\u0177\u0176\u0001"
+        + "\u0000\u0000\u0000\u0178\u017b\u0001\u0000\u0000\u0000\u0179\u0177\u0001"
+        + "\u0000\u0000\u0000\u0179\u017a\u0001\u0000\u0000\u0000\u017a\u019b\u0001"
+        + "\u0000\u0000\u0000\u017b\u0179\u0001\u0000\u0000\u0000\u017c\u017e\u0003"
+        + "G#\u0000\u017d\u017f\u0003m6\u0000\u017e\u017d\u0001\u0000\u0000\u0000"
+        + "\u017f\u0180\u0001\u0000\u0000\u0000\u0180\u017e\u0001\u0000\u0000\u0000"
+        + "\u0180\u0181\u0001\u0000\u0000\u0000\u0181\u019b\u0001\u0000\u0000\u0000"
+        + "\u0182\u0184\u0003m6\u0000\u0183\u0182\u0001\u0000\u0000\u0000\u0184\u0185"
+        + "\u0001\u0000\u0000\u0000\u0185\u0183\u0001\u0000\u0000\u0000\u0185\u0186"
+        + "\u0001\u0000\u0000\u0000\u0186\u018e\u0001\u0000\u0000\u0000\u0187\u018b"
+        + "\u0003G#\u0000\u0188\u018a\u0003m6\u0000\u0189\u0188\u0001\u0000\u0000"
+        + "\u0000\u018a\u018d\u0001\u0000\u0000\u0000\u018b\u0189\u0001\u0000\u0000"
+        + "\u0000\u018b\u018c\u0001\u0000\u0000\u0000\u018c\u018f\u0001\u0000\u0000"
+        + "\u0000\u018d\u018b\u0001\u0000\u0000\u0000\u018e\u0187\u0001\u0000\u0000"
+        + "\u0000\u018e\u018f\u0001\u0000\u0000\u0000\u018f\u0190\u0001\u0000\u0000"
+        + "\u0000\u0190\u0191\u0003k5\u0000\u0191\u019b\u0001\u0000\u0000\u0000\u0192"
+        + "\u0194\u0003G#\u0000\u0193\u0195\u0003m6\u0000\u0194\u0193\u0001\u0000"
+        + "\u0000\u0000\u0195\u0196\u0001\u0000\u0000\u0000\u0196\u0194\u0001\u0000"
+        + "\u0000\u0000\u0196\u0197\u0001\u0000\u0000\u0000\u0197\u0198\u0001\u0000"
+        + "\u0000\u0000\u0198\u0199\u0003k5\u0000\u0199\u019b\u0001\u0000\u0000\u0000"
+        + "\u019a\u0171\u0001\u0000\u0000\u0000\u019a\u017c\u0001\u0000\u0000\u0000"
+        + "\u019a\u0183\u0001\u0000\u0000\u0000\u019a\u0192\u0001\u0000\u0000\u0000"
+        + "\u019bd\u0001\u0000\u0000\u0000\u019c\u019f\u0003o7\u0000\u019d\u019f"
+        + "\u0007\u0007\u0000\u0000\u019e\u019c\u0001\u0000\u0000\u0000\u019e\u019d"
+        + "\u0001\u0000\u0000\u0000\u019f\u01a5\u0001\u0000\u0000\u0000\u01a0\u01a4"
+        + "\u0003o7\u0000\u01a1\u01a4\u0003m6\u0000\u01a2\u01a4\u0005_\u0000\u0000"
+        + "\u01a3\u01a0\u0001\u0000\u0000\u0000\u01a3\u01a1\u0001\u0000\u0000\u0000"
+        + "\u01a3\u01a2\u0001\u0000\u0000\u0000\u01a4\u01a7\u0001\u0000\u0000\u0000"
+        + "\u01a5\u01a3\u0001\u0000\u0000\u0000\u01a5\u01a6\u0001\u0000\u0000\u0000"
+        + "\u01a6f\u0001\u0000\u0000\u0000\u01a7\u01a5\u0001\u0000\u0000\u0000\u01a8"
+        + "\u01ae\u0005`\u0000\u0000\u01a9\u01ad\b\b\u0000\u0000\u01aa\u01ab\u0005"
+        + "`\u0000\u0000\u01ab\u01ad\u0005`\u0000\u0000\u01ac\u01a9\u0001\u0000\u0000"
+        + "\u0000\u01ac\u01aa\u0001\u0000\u0000\u0000\u01ad\u01b0\u0001\u0000\u0000"
+        + "\u0000\u01ae\u01ac\u0001\u0000\u0000\u0000\u01ae\u01af\u0001\u0000\u0000"
+        + "\u0000\u01af\u01b1\u0001\u0000\u0000\u0000\u01b0\u01ae\u0001\u0000\u0000"
+        + "\u0000\u01b1\u01b2\u0005`\u0000\u0000\u01b2h\u0001\u0000\u0000\u0000\u01b3"
+        + "\u01b9\u0003o7\u0000\u01b4\u01b8\u0003o7\u0000\u01b5\u01b8\u0003m6\u0000"
+        + "\u01b6\u01b8\u0005_\u0000\u0000\u01b7\u01b4\u0001\u0000\u0000\u0000\u01b7"
+        + "\u01b5\u0001\u0000\u0000\u0000\u01b7\u01b6\u0001\u0000\u0000\u0000\u01b8"
+        + "\u01bb\u0001\u0000\u0000\u0000\u01b9\u01b7\u0001\u0000\u0000\u0000\u01b9"
+        + "\u01ba\u0001\u0000\u0000\u0000\u01ba\u01bc\u0001\u0000\u0000\u0000\u01bb"
+        + "\u01b9\u0001\u0000\u0000\u0000\u01bc\u01bd\u0005~\u0000\u0000\u01bdj\u0001"
+        + "\u0000\u0000\u0000\u01be\u01c0\u0007\t\u0000\u0000\u01bf\u01c1\u0007\n"
+        + "\u0000\u0000\u01c0\u01bf\u0001\u0000\u0000\u0000\u01c0\u01c1\u0001\u0000"
+        + "\u0000\u0000\u01c1\u01c3\u0001\u0000\u0000\u0000\u01c2\u01c4\u0003m6\u0000"
+        + "\u01c3\u01c2\u0001\u0000\u0000\u0000\u01c4\u01c5\u0001\u0000\u0000\u0000"
+        + "\u01c5\u01c3\u0001\u0000\u0000\u0000\u01c5\u01c6\u0001\u0000\u0000\u0000"
+        + "\u01c6l\u0001\u0000\u0000\u0000\u01c7\u01c8\u0007\u000b\u0000\u0000\u01c8"
+        + "n\u0001\u0000\u0000\u0000\u01c9\u01ca\u0007\f\u0000\u0000\u01cap\u0001"
+        + "\u0000\u0000\u0000\u01cb\u01cc\u0005/\u0000\u0000\u01cc\u01cd\u0005/\u0000"
+        + "\u0000\u01cd\u01d1\u0001\u0000\u0000\u0000\u01ce\u01d0\b\u0003\u0000\u0000"
+        + "\u01cf\u01ce\u0001\u0000\u0000\u0000\u01d0\u01d3\u0001\u0000\u0000\u0000"
+        + "\u01d1\u01cf\u0001\u0000\u0000\u0000\u01d1\u01d2\u0001\u0000\u0000\u0000"
+        + "\u01d2\u01d5\u0001\u0000\u0000\u0000\u01d3\u01d1\u0001\u0000\u0000\u0000"
+        + "\u01d4\u01d6\u0005\r\u0000\u0000\u01d5\u01d4\u0001\u0000\u0000\u0000\u01d5"
+        + "\u01d6\u0001\u0000\u0000\u0000\u01d6\u01d8\u0001\u0000\u0000\u0000\u01d7"
+        + "\u01d9\u0005\n\u0000\u0000\u01d8\u01d7\u0001\u0000\u0000\u0000\u01d8\u01d9"
+        + "\u0001\u0000\u0000\u0000\u01d9\u01da\u0001\u0000\u0000\u0000\u01da\u01db"
+        + "\u00068\u0000\u0000\u01dbr\u0001\u0000\u0000\u0000\u01dc\u01dd\u0005/"
+        + "\u0000\u0000\u01dd\u01de\u0005*\u0000\u0000\u01de\u01e3\u0001\u0000\u0000"
+        + "\u0000\u01df\u01e2\u0003s9\u0000\u01e0\u01e2\t\u0000\u0000\u0000\u01e1"
+        + "\u01df\u0001\u0000\u0000\u0000\u01e1\u01e0\u0001\u0000\u0000\u0000\u01e2"
+        + "\u01e5\u0001\u0000\u0000\u0000\u01e3\u01e4\u0001\u0000\u0000\u0000\u01e3"
+        + "\u01e1\u0001\u0000\u0000\u0000\u01e4\u01e6\u0001\u0000\u0000\u0000\u01e5"
+        + "\u01e3\u0001\u0000\u0000\u0000\u01e6\u01e7\u0005*\u0000\u0000\u01e7\u01e8"
+        + "\u0005/\u0000\u0000\u01e8\u01e9\u0001\u0000\u0000\u0000\u01e9\u01ea\u0006"
+        + "9\u0000\u0000\u01eat\u0001\u0000\u0000\u0000\u01eb\u01ed\u0007\r\u0000"
+        + "\u0000\u01ec\u01eb\u0001\u0000\u0000\u0000\u01ed\u01ee\u0001\u0000\u0000"
+        + "\u0000\u01ee\u01ec\u0001\u0000\u0000\u0000\u01ee\u01ef\u0001\u0000\u0000"
+        + "\u0000\u01ef\u01f0\u0001\u0000\u0000\u0000\u01f0\u01f1\u0006:\u0000\u0000"
+        + "\u01f1v\u0001\u0000\u0000\u0000&\u0000\u0123\u012d\u012f\u013a\u0142\u0145"
+        + "\u014b\u014d\u0157\u0159\u0163\u0165\u0169\u016e\u0173\u0179\u0180\u0185"
+        + "\u018b\u018e\u0196\u019a\u019e\u01a3\u01a5\u01ac\u01ae\u01b7\u01b9\u01c0"
+        + "\u01c5\u01d1\u01d5\u01d8\u01e1\u01e3\u01ee\u0001\u0000\u0001\u0000";
     public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray());
     static {
         _decisionToDFA = new DFA[_ATN.getNumberOfDecisions()];
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseParser.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseParser.java
index aa4dfdbe3a57..41433931296d 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseParser.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseParser.java
@@ -9,10 +9,10 @@
 
 import java.util.List;
 
-@SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast" })
+@SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue" })
 class EqlBaseParser extends Parser {
     static {
-        RuntimeMetaData.checkVersion("4.9.2", RuntimeMetaData.VERSION);
+        RuntimeMetaData.checkVersion("4.11.1", RuntimeMetaData.VERSION);
     }
 
     protected static final DFA[] _decisionToDFA;
@@ -211,7 +211,7 @@ public Vocabulary getVocabulary() {
 
     @Override
     public String getGrammarFileName() {
-        return "EqlBase.g4";
+        return "java-escape";
     }
 
     @Override
@@ -234,6 +234,7 @@ public EqlBaseParser(TokenStream input) {
         _interp = new ParserATNSimulator(this, _ATN, _decisionToDFA, _sharedContextCache);
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class SingleStatementContext extends ParserRuleContext {
         public StatementContext statement() {
             return getRuleContext(StatementContext.class, 0);
@@ -290,6 +291,7 @@ public final SingleStatementContext singleStatement() throws RecognitionExceptio
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class SingleExpressionContext extends ParserRuleContext {
         public ExpressionContext expression() {
             return getRuleContext(ExpressionContext.class, 0);
@@ -346,6 +348,7 @@ public final SingleExpressionContext singleExpression() throws RecognitionExcept
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class StatementContext extends ParserRuleContext {
         public QueryContext query() {
             return getRuleContext(QueryContext.class, 0);
@@ -419,6 +422,7 @@ public final StatementContext statement() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class QueryContext extends ParserRuleContext {
         public SequenceContext sequence() {
             return getRuleContext(SequenceContext.class, 0);
@@ -508,6 +512,7 @@ public final QueryContext query() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class SequenceParamsContext extends ParserRuleContext {
         public TerminalNode WITH() {
             return getToken(EqlBaseParser.WITH, 0);
@@ -578,6 +583,7 @@ public final SequenceParamsContext sequenceParams() throws RecognitionException
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class SequenceContext extends ParserRuleContext {
         public JoinKeysContext by;
         public JoinKeysContext disallowed;
@@ -717,6 +723,7 @@ public final SequenceContext sequence() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class SampleContext extends ParserRuleContext {
         public JoinKeysContext by;
 
@@ -806,6 +813,7 @@ public final SampleContext sample() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class JoinContext extends ParserRuleContext {
         public JoinKeysContext by;
         public JoinTermContext until;
@@ -914,6 +922,7 @@ public final JoinContext join() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class PipeContext extends ParserRuleContext {
         public Token kind;
 
@@ -981,10 +990,7 @@ public final PipeContext pipe() throws RecognitionException {
                 setState(141);
                 _errHandler.sync(this);
                 _la = _input.LA(1);
-                if ((((_la) & ~0x3f) == 0
-                    && ((1L << _la) & ((1L << FALSE) | (1L << NOT) | (1L << NULL) | (1L << TRUE) | (1L << PLUS) | (1L << MINUS) | (1L << LP)
-                        | (1L << OPTIONAL) | (1L << STRING) | (1L << INTEGER_VALUE) | (1L << DECIMAL_VALUE) | (1L << IDENTIFIER) | (1L
-                            << QUOTED_IDENTIFIER) | (1L << TILDE_IDENTIFIER))) != 0)) {
+                if (((_la) & ~0x3f) == 0 && ((1L << _la) & 1118209768429584L) != 0) {
                     {
                         setState(133);
                         booleanExpression(0);
@@ -1018,6 +1024,7 @@ public final PipeContext pipe() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class JoinKeysContext extends ParserRuleContext {
         public TerminalNode BY() {
             return getToken(EqlBaseParser.BY, 0);
@@ -1103,6 +1110,7 @@ public final JoinKeysContext joinKeys() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class JoinTermContext extends ParserRuleContext {
         public JoinKeysContext by;
 
@@ -1170,6 +1178,7 @@ public final JoinTermContext joinTerm() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class SequenceTermContext extends ParserRuleContext {
         public JoinKeysContext by;
         public Token key;
@@ -1271,6 +1280,7 @@ public final SequenceTermContext sequenceTerm() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class SubqueryContext extends ParserRuleContext {
         public TerminalNode LB() {
             return getToken(EqlBaseParser.LB, 0);
@@ -1333,6 +1343,7 @@ public final SubqueryContext subquery() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class EventQueryContext extends ParserRuleContext {
         public EventFilterContext eventFilter() {
             return getRuleContext(EventFilterContext.class, 0);
@@ -1383,6 +1394,7 @@ public final EventQueryContext eventQuery() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class EventFilterContext extends ParserRuleContext {
         public EventValueContext event;
 
@@ -1466,6 +1478,7 @@ public final EventFilterContext eventFilter() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ExpressionContext extends ParserRuleContext {
         public BooleanExpressionContext booleanExpression() {
             return getRuleContext(BooleanExpressionContext.class, 0);
@@ -1516,6 +1529,7 @@ public final ExpressionContext expression() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class BooleanExpressionContext extends ParserRuleContext {
         public BooleanExpressionContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -1533,6 +1547,7 @@ public void copyFrom(BooleanExpressionContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class LogicalNotContext extends BooleanExpressionContext {
         public TerminalNode NOT() {
             return getToken(EqlBaseParser.NOT, 0);
@@ -1563,6 +1578,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class BooleanDefaultContext extends BooleanExpressionContext {
         public ValueExpressionContext valueExpression() {
             return getRuleContext(ValueExpressionContext.class, 0);
@@ -1589,6 +1605,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ProcessCheckContext extends BooleanExpressionContext {
         public Token relationship;
 
@@ -1625,6 +1642,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class LogicalBinaryContext extends BooleanExpressionContext {
         public BooleanExpressionContext left;
         public Token operator;
@@ -1771,6 +1789,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ValueExpressionContext extends ParserRuleContext {
         public ValueExpressionContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -1788,6 +1807,7 @@ public void copyFrom(ValueExpressionContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ValueExpressionDefaultContext extends ValueExpressionContext {
         public OperatorExpressionContext operatorExpression() {
             return getRuleContext(OperatorExpressionContext.class, 0);
@@ -1814,6 +1834,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ComparisonContext extends ValueExpressionContext {
         public OperatorExpressionContext left;
         public OperatorExpressionContext right;
@@ -1887,6 +1908,7 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class OperatorExpressionContext extends ParserRuleContext {
         public OperatorExpressionContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -1904,6 +1926,7 @@ public void copyFrom(OperatorExpressionContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class OperatorExpressionDefaultContext extends OperatorExpressionContext {
         public PrimaryExpressionContext primaryExpression() {
             return getRuleContext(PrimaryExpressionContext.class, 0);
@@ -1934,6 +1957,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ArithmeticBinaryContext extends OperatorExpressionContext {
         public OperatorExpressionContext left;
         public Token operator;
@@ -1988,6 +2012,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ArithmeticUnaryContext extends OperatorExpressionContext {
         public Token operator;
 
@@ -2114,8 +2139,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
                                     setState(218);
                                     ((ArithmeticBinaryContext) _localctx).operator = _input.LT(1);
                                     _la = _input.LA(1);
-                                    if (!((((_la) & ~0x3f) == 0
-                                        && ((1L << _la) & ((1L << ASTERISK) | (1L << SLASH) | (1L << PERCENT))) != 0))) {
+                                    if (!(((_la) & ~0x3f) == 0 && ((1L << _la) & 60129542144L) != 0)) {
                                         ((ArithmeticBinaryContext) _localctx).operator = (Token) _errHandler.recoverInline(this);
                                     } else {
                                         if (_input.LA(1) == Token.EOF) matchedEOF = true;
@@ -2164,6 +2188,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class PredicateContext extends ParserRuleContext {
         public Token kind;
 
@@ -2316,9 +2341,7 @@ public final PredicateContext predicate() throws RecognitionException {
                     setState(243);
                     ((PredicateContext) _localctx).kind = _input.LT(1);
                     _la = _input.LA(1);
-                    if (!((((_la) & ~0x3f) == 0
-                        && ((1L << _la) & ((1L << LIKE) | (1L << LIKE_INSENSITIVE) | (1L << REGEX) | (1L << REGEX_INSENSITIVE) | (1L
-                            << SEQ))) != 0))) {
+                    if (!(((_la) & ~0x3f) == 0 && ((1L << _la) & 8487680L) != 0)) {
                         ((PredicateContext) _localctx).kind = (Token) _errHandler.recoverInline(this);
                     } else {
                         if (_input.LA(1) == Token.EOF) matchedEOF = true;
@@ -2334,9 +2357,7 @@ public final PredicateContext predicate() throws RecognitionException {
                     setState(245);
                     ((PredicateContext) _localctx).kind = _input.LT(1);
                     _la = _input.LA(1);
-                    if (!((((_la) & ~0x3f) == 0
-                        && ((1L << _la) & ((1L << LIKE) | (1L << LIKE_INSENSITIVE) | (1L << REGEX) | (1L << REGEX_INSENSITIVE) | (1L
-                            << SEQ))) != 0))) {
+                    if (!(((_la) & ~0x3f) == 0 && ((1L << _la) & 8487680L) != 0)) {
                         ((PredicateContext) _localctx).kind = (Token) _errHandler.recoverInline(this);
                     } else {
                         if (_input.LA(1) == Token.EOF) matchedEOF = true;
@@ -2378,6 +2399,7 @@ public final PredicateContext predicate() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class PrimaryExpressionContext extends ParserRuleContext {
         public PrimaryExpressionContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -2395,6 +2417,7 @@ public void copyFrom(PrimaryExpressionContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class DereferenceContext extends PrimaryExpressionContext {
         public QualifiedNameContext qualifiedName() {
             return getRuleContext(QualifiedNameContext.class, 0);
@@ -2421,6 +2444,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ConstantDefaultContext extends PrimaryExpressionContext {
         public ConstantContext constant() {
             return getRuleContext(ConstantContext.class, 0);
@@ -2447,6 +2471,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ParenthesizedExpressionContext extends PrimaryExpressionContext {
         public TerminalNode LP() {
             return getToken(EqlBaseParser.LP, 0);
@@ -2481,6 +2506,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class FunctionContext extends PrimaryExpressionContext {
         public FunctionExpressionContext functionExpression() {
             return getRuleContext(FunctionExpressionContext.class, 0);
@@ -2557,6 +2583,7 @@ public final PrimaryExpressionContext primaryExpression() throws RecognitionExce
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class FunctionExpressionContext extends ParserRuleContext {
         public FunctionNameContext name;
 
@@ -2628,10 +2655,7 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx
                 setState(278);
                 _errHandler.sync(this);
                 _la = _input.LA(1);
-                if ((((_la) & ~0x3f) == 0
-                    && ((1L << _la) & ((1L << FALSE) | (1L << NOT) | (1L << NULL) | (1L << TRUE) | (1L << PLUS) | (1L << MINUS) | (1L << LP)
-                        | (1L << OPTIONAL) | (1L << STRING) | (1L << INTEGER_VALUE) | (1L << DECIMAL_VALUE) | (1L << IDENTIFIER) | (1L
-                            << QUOTED_IDENTIFIER) | (1L << TILDE_IDENTIFIER))) != 0)) {
+                if (((_la) & ~0x3f) == 0 && ((1L << _la) & 1118209768429584L) != 0) {
                     {
                         setState(270);
                         expression();
@@ -2667,6 +2691,7 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class FunctionNameContext extends ParserRuleContext {
         public TerminalNode IDENTIFIER() {
             return getToken(EqlBaseParser.IDENTIFIER, 0);
@@ -2729,6 +2754,7 @@ public final FunctionNameContext functionName() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ConstantContext extends ParserRuleContext {
         public ConstantContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -2746,6 +2772,7 @@ public void copyFrom(ConstantContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class NullLiteralContext extends ConstantContext {
         public TerminalNode NULL() {
             return getToken(EqlBaseParser.NULL, 0);
@@ -2772,6 +2799,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class StringLiteralContext extends ConstantContext {
         public StringContext string() {
             return getRuleContext(StringContext.class, 0);
@@ -2798,6 +2826,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class NumericLiteralContext extends ConstantContext {
         public NumberContext number() {
             return getRuleContext(NumberContext.class, 0);
@@ -2824,6 +2853,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class BooleanLiteralContext extends ConstantContext {
         public BooleanValueContext booleanValue() {
             return getRuleContext(BooleanValueContext.class, 0);
@@ -2900,6 +2930,7 @@ public final ConstantContext constant() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ComparisonOperatorContext extends ParserRuleContext {
         public TerminalNode EQ() {
             return getToken(EqlBaseParser.EQ, 0);
@@ -2960,8 +2991,7 @@ public final ComparisonOperatorContext comparisonOperator() throws RecognitionEx
             {
                 setState(290);
                 _la = _input.LA(1);
-                if (!((((_la) & ~0x3f) == 0
-                    && ((1L << _la) & ((1L << EQ) | (1L << NEQ) | (1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0))) {
+                if (!(((_la) & ~0x3f) == 0 && ((1L << _la) & 2113929216L) != 0)) {
                     _errHandler.recoverInline(this);
                 } else {
                     if (_input.LA(1) == Token.EOF) matchedEOF = true;
@@ -2979,6 +3009,7 @@ public final ComparisonOperatorContext comparisonOperator() throws RecognitionEx
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class BooleanValueContext extends ParserRuleContext {
         public TerminalNode TRUE() {
             return getToken(EqlBaseParser.TRUE, 0);
@@ -3041,6 +3072,7 @@ public final BooleanValueContext booleanValue() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class QualifiedNameContext extends ParserRuleContext {
         public List<IdentifierContext> identifier() {
             return getRuleContexts(IdentifierContext.class);
@@ -3189,6 +3221,7 @@ public final QualifiedNameContext qualifiedName() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class IdentifierContext extends ParserRuleContext {
         public TerminalNode IDENTIFIER() {
             return getToken(EqlBaseParser.IDENTIFIER, 0);
@@ -3251,6 +3284,7 @@ public final IdentifierContext identifier() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class TimeUnitContext extends ParserRuleContext {
         public Token unit;
 
@@ -3318,6 +3352,7 @@ public final TimeUnitContext timeUnit() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class NumberContext extends ParserRuleContext {
         public NumberContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -3335,6 +3370,7 @@ public void copyFrom(NumberContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class DecimalLiteralContext extends NumberContext {
         public TerminalNode DECIMAL_VALUE() {
             return getToken(EqlBaseParser.DECIMAL_VALUE, 0);
@@ -3361,6 +3397,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class IntegerLiteralContext extends NumberContext {
         public TerminalNode INTEGER_VALUE() {
             return getToken(EqlBaseParser.INTEGER_VALUE, 0);
@@ -3421,6 +3458,7 @@ public final NumberContext number() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class StringContext extends ParserRuleContext {
         public TerminalNode STRING() {
             return getToken(EqlBaseParser.STRING, 0);
@@ -3471,6 +3509,7 @@ public final StringContext string() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class EventValueContext extends ParserRuleContext {
         public TerminalNode STRING() {
             return getToken(EqlBaseParser.STRING, 0);
@@ -3563,122 +3602,218 @@ private boolean operatorExpression_sempred(OperatorExpressionContext _localctx,
         return true;
     }
 
-    public static final String _serializedATN = "\3\u608b\ua72a\u8133\ub9ed\u417c\u3be7\u7786\u5964\3\66\u0149\4\2\t\2"
-        + "\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13"
-        + "\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"
-        + "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"
-        + "\4\32\t\32\4\33\t\33\4\34\t\34\4\35\t\35\4\36\t\36\4\37\t\37\4 \t \4!"
-        + "\t!\3\2\3\2\3\2\3\3\3\3\3\3\3\4\3\4\7\4K\n\4\f\4\16\4N\13\4\3\5\3\5\3"
-        + "\5\3\5\5\5T\n\5\3\6\3\6\3\6\3\6\3\6\3\7\3\7\3\7\5\7^\n\7\3\7\3\7\5\7b"
-        + "\n\7\5\7d\n\7\3\7\6\7g\n\7\r\7\16\7h\3\7\3\7\5\7m\n\7\3\b\3\b\5\bq\n\b"
-        + "\3\b\6\bt\n\b\r\b\16\bu\3\t\3\t\5\tz\n\t\3\t\3\t\6\t~\n\t\r\t\16\t\177"
-        + "\3\t\3\t\5\t\u0084\n\t\3\n\3\n\3\n\3\n\3\n\7\n\u008b\n\n\f\n\16\n\u008e"
-        + "\13\n\5\n\u0090\n\n\3\13\3\13\3\13\3\13\7\13\u0096\n\13\f\13\16\13\u0099"
-        + "\13\13\3\f\3\f\5\f\u009d\n\f\3\r\3\r\5\r\u00a1\n\r\3\r\3\r\3\r\3\r\5\r"
-        + "\u00a7\n\r\3\16\3\16\3\16\3\16\3\17\3\17\3\20\3\20\5\20\u00b1\n\20\3\20"
-        + "\3\20\3\20\3\21\3\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\5\22\u00bf\n\22"
-        + "\3\22\3\22\3\22\3\22\3\22\3\22\7\22\u00c7\n\22\f\22\16\22\u00ca\13\22"
-        + "\3\23\3\23\3\23\3\23\3\23\5\23\u00d1\n\23\3\24\3\24\3\24\5\24\u00d6\n"
-        + "\24\3\24\3\24\5\24\u00da\n\24\3\24\3\24\3\24\3\24\3\24\3\24\7\24\u00e2"
-        + "\n\24\f\24\16\24\u00e5\13\24\3\25\5\25\u00e8\n\25\3\25\3\25\3\25\3\25"
-        + "\3\25\7\25\u00ef\n\25\f\25\16\25\u00f2\13\25\3\25\3\25\3\25\3\25\3\25"
-        + "\3\25\3\25\3\25\3\25\7\25\u00fd\n\25\f\25\16\25\u0100\13\25\3\25\3\25"
-        + "\5\25\u0104\n\25\3\26\3\26\3\26\3\26\3\26\3\26\3\26\5\26\u010d\n\26\3"
-        + "\27\3\27\3\27\3\27\3\27\7\27\u0114\n\27\f\27\16\27\u0117\13\27\5\27\u0119"
-        + "\n\27\3\27\3\27\3\30\3\30\3\31\3\31\3\31\3\31\5\31\u0123\n\31\3\32\3\32"
-        + "\3\33\3\33\3\34\5\34\u012a\n\34\3\34\3\34\3\34\3\34\3\34\6\34\u0131\n"
-        + "\34\r\34\16\34\u0132\3\34\7\34\u0136\n\34\f\34\16\34\u0139\13\34\3\35"
-        + "\3\35\3\36\3\36\5\36\u013f\n\36\3\37\3\37\5\37\u0143\n\37\3 \3 \3!\3!"
-        + "\3!\2\4\"&\"\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,.\60\62\64"
-        + "\668:<>@\2\13\3\2!\"\3\2#%\3\2\7\b\5\2\n\13\21\22\31\31\4\2\61\61\63\63"
-        + "\3\2\33 \4\2\6\6\25\25\3\2\61\62\4\2..\61\61\2\u015a\2B\3\2\2\2\4E\3\2"
-        + "\2\2\6H\3\2\2\2\bS\3\2\2\2\nU\3\2\2\2\fZ\3\2\2\2\16n\3\2\2\2\20w\3\2\2"
-        + "\2\22\u0085\3\2\2\2\24\u0091\3\2\2\2\26\u009a\3\2\2\2\30\u009e\3\2\2\2"
-        + "\32\u00a8\3\2\2\2\34\u00ac\3\2\2\2\36\u00b0\3\2\2\2 \u00b5\3\2\2\2\"\u00be"
-        + "\3\2\2\2$\u00d0\3\2\2\2&\u00d9\3\2\2\2(\u0103\3\2\2\2*\u010c\3\2\2\2,"
-        + "\u010e\3\2\2\2.\u011c\3\2\2\2\60\u0122\3\2\2\2\62\u0124\3\2\2\2\64\u0126"
-        + "\3\2\2\2\66\u0129\3\2\2\28\u013a\3\2\2\2:\u013c\3\2\2\2<\u0142\3\2\2\2"
-        + ">\u0144\3\2\2\2@\u0146\3\2\2\2BC\5\6\4\2CD\7\2\2\3D\3\3\2\2\2EF\5 \21"
-        + "\2FG\7\2\2\3G\5\3\2\2\2HL\5\b\5\2IK\5\22\n\2JI\3\2\2\2KN\3\2\2\2LJ\3\2"
-        + "\2\2LM\3\2\2\2M\7\3\2\2\2NL\3\2\2\2OT\5\f\7\2PT\5\20\t\2QT\5\34\17\2R"
-        + "T\5\16\b\2SO\3\2\2\2SP\3\2\2\2SQ\3\2\2\2SR\3\2\2\2T\t\3\2\2\2UV\7\30\2"
-        + "\2VW\7\f\2\2WX\7\32\2\2XY\5:\36\2Y\13\3\2\2\2Zc\7\24\2\2[]\5\24\13\2\\"
-        + "^\5\n\6\2]\\\3\2\2\2]^\3\2\2\2^d\3\2\2\2_a\5\n\6\2`b\5\24\13\2a`\3\2\2"
-        + "\2ab\3\2\2\2bd\3\2\2\2c[\3\2\2\2c_\3\2\2\2cd\3\2\2\2df\3\2\2\2eg\5\30"
-        + "\r\2fe\3\2\2\2gh\3\2\2\2hf\3\2\2\2hi\3\2\2\2il\3\2\2\2jk\7\26\2\2km\5"
-        + "\30\r\2lj\3\2\2\2lm\3\2\2\2m\r\3\2\2\2np\7\23\2\2oq\5\24\13\2po\3\2\2"
-        + "\2pq\3\2\2\2qs\3\2\2\2rt\5\26\f\2sr\3\2\2\2tu\3\2\2\2us\3\2\2\2uv\3\2"
-        + "\2\2v\17\3\2\2\2wy\7\t\2\2xz\5\24\13\2yx\3\2\2\2yz\3\2\2\2z{\3\2\2\2{"
-        + "}\5\26\f\2|~\5\26\f\2}|\3\2\2\2~\177\3\2\2\2\177}\3\2\2\2\177\u0080\3"
-        + "\2\2\2\u0080\u0083\3\2\2\2\u0081\u0082\7\26\2\2\u0082\u0084\5\26\f\2\u0083"
-        + "\u0081\3\2\2\2\u0083\u0084\3\2\2\2\u0084\21\3\2\2\2\u0085\u0086\7,\2\2"
-        + "\u0086\u008f\7\61\2\2\u0087\u008c\5\"\22\2\u0088\u0089\7\'\2\2\u0089\u008b"
-        + "\5\"\22\2\u008a\u0088\3\2\2\2\u008b\u008e\3\2\2\2\u008c\u008a\3\2\2\2"
-        + "\u008c\u008d\3\2\2\2\u008d\u0090\3\2\2\2\u008e\u008c\3\2\2\2\u008f\u0087"
-        + "\3\2\2\2\u008f\u0090\3\2\2\2\u0090\23\3\2\2\2\u0091\u0092\7\5\2\2\u0092"
-        + "\u0097\5 \21\2\u0093\u0094\7\'\2\2\u0094\u0096\5 \21\2\u0095\u0093\3\2"
-        + "\2\2\u0096\u0099\3\2\2\2\u0097\u0095\3\2\2\2\u0097\u0098\3\2\2\2\u0098"
-        + "\25\3\2\2\2\u0099\u0097\3\2\2\2\u009a\u009c\5\32\16\2\u009b\u009d\5\24"
-        + "\13\2\u009c\u009b\3\2\2\2\u009c\u009d\3\2\2\2\u009d\27\3\2\2\2\u009e\u00a0"
-        + "\5\32\16\2\u009f\u00a1\5\24\13\2\u00a0\u009f\3\2\2\2\u00a0\u00a1\3\2\2"
-        + "\2\u00a1\u00a6\3\2\2\2\u00a2\u00a3\7\30\2\2\u00a3\u00a4\7\61\2\2\u00a4"
-        + "\u00a5\7\32\2\2\u00a5\u00a7\5<\37\2\u00a6\u00a2\3\2\2\2\u00a6\u00a7\3"
-        + "\2\2\2\u00a7\31\3\2\2\2\u00a8\u00a9\7(\2\2\u00a9\u00aa\5\36\20\2\u00aa"
-        + "\u00ab\7)\2\2\u00ab\33\3\2\2\2\u00ac\u00ad\5\36\20\2\u00ad\35\3\2\2\2"
-        + "\u00ae\u00b1\7\4\2\2\u00af\u00b1\5@!\2\u00b0\u00ae\3\2\2\2\u00b0\u00af"
-        + "\3\2\2\2\u00b1\u00b2\3\2\2\2\u00b2\u00b3\7\27\2\2\u00b3\u00b4\5 \21\2"
-        + "\u00b4\37\3\2\2\2\u00b5\u00b6\5\"\22\2\u00b6!\3\2\2\2\u00b7\u00b8\b\22"
-        + "\1\2\u00b8\u00b9\7\r\2\2\u00b9\u00bf\5\"\22\7\u00ba\u00bb\7\61\2\2\u00bb"
-        + "\u00bc\7\17\2\2\u00bc\u00bf\5\32\16\2\u00bd\u00bf\5$\23\2\u00be\u00b7"
-        + "\3\2\2\2\u00be\u00ba\3\2\2\2\u00be\u00bd\3\2\2\2\u00bf\u00c8\3\2\2\2\u00c0"
-        + "\u00c1\f\4\2\2\u00c1\u00c2\7\3\2\2\u00c2\u00c7\5\"\22\5\u00c3\u00c4\f"
-        + "\3\2\2\u00c4\u00c5\7\20\2\2\u00c5\u00c7\5\"\22\4\u00c6\u00c0\3\2\2\2\u00c6"
-        + "\u00c3\3\2\2\2\u00c7\u00ca\3\2\2\2\u00c8\u00c6\3\2\2\2\u00c8\u00c9\3\2"
-        + "\2\2\u00c9#\3\2\2\2\u00ca\u00c8\3\2\2\2\u00cb\u00d1\5&\24\2\u00cc\u00cd"
-        + "\5&\24\2\u00cd\u00ce\5\62\32\2\u00ce\u00cf\5&\24\2\u00cf\u00d1\3\2\2\2"
-        + "\u00d0\u00cb\3\2\2\2\u00d0\u00cc\3\2\2\2\u00d1%\3\2\2\2\u00d2\u00d3\b"
-        + "\24\1\2\u00d3\u00d5\5*\26\2\u00d4\u00d6\5(\25\2\u00d5\u00d4\3\2\2\2\u00d5"
-        + "\u00d6\3\2\2\2\u00d6\u00da\3\2\2\2\u00d7\u00d8\t\2\2\2\u00d8\u00da\5&"
-        + "\24\5\u00d9\u00d2\3\2\2\2\u00d9\u00d7\3\2\2\2\u00da\u00e3\3\2\2\2\u00db"
-        + "\u00dc\f\4\2\2\u00dc\u00dd\t\3\2\2\u00dd\u00e2\5&\24\5\u00de\u00df\f\3"
-        + "\2\2\u00df\u00e0\t\2\2\2\u00e0\u00e2\5&\24\4\u00e1\u00db\3\2\2\2\u00e1"
-        + "\u00de\3\2\2\2\u00e2\u00e5\3\2\2\2\u00e3\u00e1\3\2\2\2\u00e3\u00e4\3\2"
-        + "\2\2\u00e4\'\3\2\2\2\u00e5\u00e3\3\2\2\2\u00e6\u00e8\7\r\2\2\u00e7\u00e6"
-        + "\3\2\2\2\u00e7\u00e8\3\2\2\2\u00e8\u00e9\3\2\2\2\u00e9\u00ea\t\4\2\2\u00ea"
-        + "\u00eb\7*\2\2\u00eb\u00f0\5 \21\2\u00ec\u00ed\7\'\2\2\u00ed\u00ef\5 \21"
-        + "\2\u00ee\u00ec\3\2\2\2\u00ef\u00f2\3\2\2\2\u00f0\u00ee\3\2\2\2\u00f0\u00f1"
-        + "\3\2\2\2\u00f1\u00f3\3\2\2\2\u00f2\u00f0\3\2\2\2\u00f3\u00f4\7+\2\2\u00f4"
-        + "\u0104\3\2\2\2\u00f5\u00f6\t\5\2\2\u00f6\u0104\5\60\31\2\u00f7\u00f8\t"
-        + "\5\2\2\u00f8\u00f9\7*\2\2\u00f9\u00fe\5\60\31\2\u00fa\u00fb\7\'\2\2\u00fb"
-        + "\u00fd\5\60\31\2\u00fc\u00fa\3\2\2\2\u00fd\u0100\3\2\2\2\u00fe\u00fc\3"
-        + "\2\2\2\u00fe\u00ff\3\2\2\2\u00ff\u0101\3\2\2\2\u0100\u00fe\3\2\2\2\u0101"
-        + "\u0102\7+\2\2\u0102\u0104\3\2\2\2\u0103\u00e7\3\2\2\2\u0103\u00f5\3\2"
-        + "\2\2\u0103\u00f7\3\2\2\2\u0104)\3\2\2\2\u0105\u010d\5\60\31\2\u0106\u010d"
-        + "\5,\27\2\u0107\u010d\5\66\34\2\u0108\u0109\7*\2\2\u0109\u010a\5 \21\2"
-        + "\u010a\u010b\7+\2\2\u010b\u010d\3\2\2\2\u010c\u0105\3\2\2\2\u010c\u0106"
-        + "\3\2\2\2\u010c\u0107\3\2\2\2\u010c\u0108\3\2\2\2\u010d+\3\2\2\2\u010e"
-        + "\u010f\5.\30\2\u010f\u0118\7*\2\2\u0110\u0115\5 \21\2\u0111\u0112\7\'"
-        + "\2\2\u0112\u0114\5 \21\2\u0113\u0111\3\2\2\2\u0114\u0117\3\2\2\2\u0115"
-        + "\u0113\3\2\2\2\u0115\u0116\3\2\2\2\u0116\u0119\3\2\2\2\u0117\u0115\3\2"
-        + "\2\2\u0118\u0110\3\2\2\2\u0118\u0119\3\2\2\2\u0119\u011a\3\2\2\2\u011a"
-        + "\u011b\7+\2\2\u011b-\3\2\2\2\u011c\u011d\t\6\2\2\u011d/\3\2\2\2\u011e"
-        + "\u0123\7\16\2\2\u011f\u0123\5<\37\2\u0120\u0123\5\64\33\2\u0121\u0123"
-        + "\5> \2\u0122\u011e\3\2\2\2\u0122\u011f\3\2\2\2\u0122\u0120\3\2\2\2\u0122"
-        + "\u0121\3\2\2\2\u0123\61\3\2\2\2\u0124\u0125\t\7\2\2\u0125\63\3\2\2\2\u0126"
-        + "\u0127\t\b\2\2\u0127\65\3\2\2\2\u0128\u012a\7-\2\2\u0129\u0128\3\2\2\2"
-        + "\u0129\u012a\3\2\2\2\u012a\u012b\3\2\2\2\u012b\u0137\58\35\2\u012c\u012d"
-        + "\7&\2\2\u012d\u0136\58\35\2\u012e\u0130\7(\2\2\u012f\u0131\7/\2\2\u0130"
-        + "\u012f\3\2\2\2\u0131\u0132\3\2\2\2\u0132\u0130\3\2\2\2\u0132\u0133\3\2"
-        + "\2\2\u0133\u0134\3\2\2\2\u0134\u0136\7)\2\2\u0135\u012c\3\2\2\2\u0135"
-        + "\u012e\3\2\2\2\u0136\u0139\3\2\2\2\u0137\u0135\3\2\2\2\u0137\u0138\3\2"
-        + "\2\2\u0138\67\3\2\2\2\u0139\u0137\3\2\2\2\u013a\u013b\t\t\2\2\u013b9\3"
-        + "\2\2\2\u013c\u013e\5<\37\2\u013d\u013f\7\61\2\2\u013e\u013d\3\2\2\2\u013e"
-        + "\u013f\3\2\2\2\u013f;\3\2\2\2\u0140\u0143\7\60\2\2\u0141\u0143\7/\2\2"
-        + "\u0142\u0140\3\2\2\2\u0142\u0141\3\2\2\2\u0143=\3\2\2\2\u0144\u0145\7"
-        + ".\2\2\u0145?\3\2\2\2\u0146\u0147\t\n\2\2\u0147A\3\2\2\2+LS]achlpuy\177"
-        + "\u0083\u008c\u008f\u0097\u009c\u00a0\u00a6\u00b0\u00be\u00c6\u00c8\u00d0"
-        + "\u00d5\u00d9\u00e1\u00e3\u00e7\u00f0\u00fe\u0103\u010c\u0115\u0118\u0122"
-        + "\u0129\u0132\u0135\u0137\u013e\u0142";
+    public static final String _serializedATN = "\u0004\u00014\u0147\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"
+        + "\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004\u0002"
+        + "\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007\u0002"
+        + "\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b\u0002"
+        + "\f\u0007\f\u0002\r\u0007\r\u0002\u000e\u0007\u000e\u0002\u000f\u0007\u000f"
+        + "\u0002\u0010\u0007\u0010\u0002\u0011\u0007\u0011\u0002\u0012\u0007\u0012"
+        + "\u0002\u0013\u0007\u0013\u0002\u0014\u0007\u0014\u0002\u0015\u0007\u0015"
+        + "\u0002\u0016\u0007\u0016\u0002\u0017\u0007\u0017\u0002\u0018\u0007\u0018"
+        + "\u0002\u0019\u0007\u0019\u0002\u001a\u0007\u001a\u0002\u001b\u0007\u001b"
+        + "\u0002\u001c\u0007\u001c\u0002\u001d\u0007\u001d\u0002\u001e\u0007\u001e"
+        + "\u0002\u001f\u0007\u001f\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0001"
+        + "\u0001\u0001\u0001\u0001\u0001\u0002\u0001\u0002\u0005\u0002I\b\u0002"
+        + "\n\u0002\f\u0002L\t\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"
+        + "\u0003\u0003R\b\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004"
+        + "\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\\\b\u0005"
+        + "\u0001\u0005\u0001\u0005\u0003\u0005`\b\u0005\u0003\u0005b\b\u0005\u0001"
+        + "\u0005\u0004\u0005e\b\u0005\u000b\u0005\f\u0005f\u0001\u0005\u0001\u0005"
+        + "\u0003\u0005k\b\u0005\u0001\u0006\u0001\u0006\u0003\u0006o\b\u0006\u0001"
+        + "\u0006\u0004\u0006r\b\u0006\u000b\u0006\f\u0006s\u0001\u0007\u0001\u0007"
+        + "\u0003\u0007x\b\u0007\u0001\u0007\u0001\u0007\u0004\u0007|\b\u0007\u000b"
+        + "\u0007\f\u0007}\u0001\u0007\u0001\u0007\u0003\u0007\u0082\b\u0007\u0001"
+        + "\b\u0001\b\u0001\b\u0001\b\u0001\b\u0005\b\u0089\b\b\n\b\f\b\u008c\t\b"
+        + "\u0003\b\u008e\b\b\u0001\t\u0001\t\u0001\t\u0001\t\u0005\t\u0094\b\t\n"
+        + "\t\f\t\u0097\t\t\u0001\n\u0001\n\u0003\n\u009b\b\n\u0001\u000b\u0001\u000b"
+        + "\u0003\u000b\u009f\b\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b"
+        + "\u0003\u000b\u00a5\b\u000b\u0001\f\u0001\f\u0001\f\u0001\f\u0001\r\u0001"
+        + "\r\u0001\u000e\u0001\u000e\u0003\u000e\u00af\b\u000e\u0001\u000e\u0001"
+        + "\u000e\u0001\u000e\u0001\u000f\u0001\u000f\u0001\u0010\u0001\u0010\u0001"
+        + "\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0003\u0010\u00bd"
+        + "\b\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001"
+        + "\u0010\u0005\u0010\u00c5\b\u0010\n\u0010\f\u0010\u00c8\t\u0010\u0001\u0011"
+        + "\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0003\u0011\u00cf\b\u0011"
+        + "\u0001\u0012\u0001\u0012\u0001\u0012\u0003\u0012\u00d4\b\u0012\u0001\u0012"
+        + "\u0001\u0012\u0003\u0012\u00d8\b\u0012\u0001\u0012\u0001\u0012\u0001\u0012"
+        + "\u0001\u0012\u0001\u0012\u0001\u0012\u0005\u0012\u00e0\b\u0012\n\u0012"
+        + "\f\u0012\u00e3\t\u0012\u0001\u0013\u0003\u0013\u00e6\b\u0013\u0001\u0013"
+        + "\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0005\u0013\u00ed\b\u0013"
+        + "\n\u0013\f\u0013\u00f0\t\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001"
+        + "\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0005"
+        + "\u0013\u00fb\b\u0013\n\u0013\f\u0013\u00fe\t\u0013\u0001\u0013\u0001\u0013"
+        + "\u0003\u0013\u0102\b\u0013\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014"
+        + "\u0001\u0014\u0001\u0014\u0001\u0014\u0003\u0014\u010b\b\u0014\u0001\u0015"
+        + "\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0005\u0015\u0112\b\u0015"
+        + "\n\u0015\f\u0015\u0115\t\u0015\u0003\u0015\u0117\b\u0015\u0001\u0015\u0001"
+        + "\u0015\u0001\u0016\u0001\u0016\u0001\u0017\u0001\u0017\u0001\u0017\u0001"
+        + "\u0017\u0003\u0017\u0121\b\u0017\u0001\u0018\u0001\u0018\u0001\u0019\u0001"
+        + "\u0019\u0001\u001a\u0003\u001a\u0128\b\u001a\u0001\u001a\u0001\u001a\u0001"
+        + "\u001a\u0001\u001a\u0001\u001a\u0004\u001a\u012f\b\u001a\u000b\u001a\f"
+        + "\u001a\u0130\u0001\u001a\u0005\u001a\u0134\b\u001a\n\u001a\f\u001a\u0137"
+        + "\t\u001a\u0001\u001b\u0001\u001b\u0001\u001c\u0001\u001c\u0003\u001c\u013d"
+        + "\b\u001c\u0001\u001d\u0001\u001d\u0003\u001d\u0141\b\u001d\u0001\u001e"
+        + "\u0001\u001e\u0001\u001f\u0001\u001f\u0001\u001f\u0000\u0002 $ \u0000"
+        + "\u0002\u0004\u0006\b\n\f\u000e\u0010\u0012\u0014\u0016\u0018\u001a\u001c"
+        + "\u001e \"$&(*,.02468:<>\u0000\t\u0001\u0000\u001f \u0001\u0000!#\u0001"
+        + "\u0000\u0005\u0006\u0003\u0000\b\t\u000f\u0010\u0017\u0017\u0002\u0000"
+        + "//11\u0001\u0000\u0019\u001e\u0002\u0000\u0004\u0004\u0013\u0013\u0001"
+        + "\u0000/0\u0002\u0000,,//\u0158\u0000@\u0001\u0000\u0000\u0000\u0002C\u0001"
+        + "\u0000\u0000\u0000\u0004F\u0001\u0000\u0000\u0000\u0006Q\u0001\u0000\u0000"
+        + "\u0000\bS\u0001\u0000\u0000\u0000\nX\u0001\u0000\u0000\u0000\fl\u0001"
+        + "\u0000\u0000\u0000\u000eu\u0001\u0000\u0000\u0000\u0010\u0083\u0001\u0000"
+        + "\u0000\u0000\u0012\u008f\u0001\u0000\u0000\u0000\u0014\u0098\u0001\u0000"
+        + "\u0000\u0000\u0016\u009c\u0001\u0000\u0000\u0000\u0018\u00a6\u0001\u0000"
+        + "\u0000\u0000\u001a\u00aa\u0001\u0000\u0000\u0000\u001c\u00ae\u0001\u0000"
+        + "\u0000\u0000\u001e\u00b3\u0001\u0000\u0000\u0000 \u00bc\u0001\u0000\u0000"
+        + "\u0000\"\u00ce\u0001\u0000\u0000\u0000$\u00d7\u0001\u0000\u0000\u0000"
+        + "&\u0101\u0001\u0000\u0000\u0000(\u010a\u0001\u0000\u0000\u0000*\u010c"
+        + "\u0001\u0000\u0000\u0000,\u011a\u0001\u0000\u0000\u0000.\u0120\u0001\u0000"
+        + "\u0000\u00000\u0122\u0001\u0000\u0000\u00002\u0124\u0001\u0000\u0000\u0000"
+        + "4\u0127\u0001\u0000\u0000\u00006\u0138\u0001\u0000\u0000\u00008\u013a"
+        + "\u0001\u0000\u0000\u0000:\u0140\u0001\u0000\u0000\u0000<\u0142\u0001\u0000"
+        + "\u0000\u0000>\u0144\u0001\u0000\u0000\u0000@A\u0003\u0004\u0002\u0000"
+        + "AB\u0005\u0000\u0000\u0001B\u0001\u0001\u0000\u0000\u0000CD\u0003\u001e"
+        + "\u000f\u0000DE\u0005\u0000\u0000\u0001E\u0003\u0001\u0000\u0000\u0000"
+        + "FJ\u0003\u0006\u0003\u0000GI\u0003\u0010\b\u0000HG\u0001\u0000\u0000\u0000"
+        + "IL\u0001\u0000\u0000\u0000JH\u0001\u0000\u0000\u0000JK\u0001\u0000\u0000"
+        + "\u0000K\u0005\u0001\u0000\u0000\u0000LJ\u0001\u0000\u0000\u0000MR\u0003"
+        + "\n\u0005\u0000NR\u0003\u000e\u0007\u0000OR\u0003\u001a\r\u0000PR\u0003"
+        + "\f\u0006\u0000QM\u0001\u0000\u0000\u0000QN\u0001\u0000\u0000\u0000QO\u0001"
+        + "\u0000\u0000\u0000QP\u0001\u0000\u0000\u0000R\u0007\u0001\u0000\u0000"
+        + "\u0000ST\u0005\u0016\u0000\u0000TU\u0005\n\u0000\u0000UV\u0005\u0018\u0000"
+        + "\u0000VW\u00038\u001c\u0000W\t\u0001\u0000\u0000\u0000Xa\u0005\u0012\u0000"
+        + "\u0000Y[\u0003\u0012\t\u0000Z\\\u0003\b\u0004\u0000[Z\u0001\u0000\u0000"
+        + "\u0000[\\\u0001\u0000\u0000\u0000\\b\u0001\u0000\u0000\u0000]_\u0003\b"
+        + "\u0004\u0000^`\u0003\u0012\t\u0000_^\u0001\u0000\u0000\u0000_`\u0001\u0000"
+        + "\u0000\u0000`b\u0001\u0000\u0000\u0000aY\u0001\u0000\u0000\u0000a]\u0001"
+        + "\u0000\u0000\u0000ab\u0001\u0000\u0000\u0000bd\u0001\u0000\u0000\u0000"
+        + "ce\u0003\u0016\u000b\u0000dc\u0001\u0000\u0000\u0000ef\u0001\u0000\u0000"
+        + "\u0000fd\u0001\u0000\u0000\u0000fg\u0001\u0000\u0000\u0000gj\u0001\u0000"
+        + "\u0000\u0000hi\u0005\u0014\u0000\u0000ik\u0003\u0016\u000b\u0000jh\u0001"
+        + "\u0000\u0000\u0000jk\u0001\u0000\u0000\u0000k\u000b\u0001\u0000\u0000"
+        + "\u0000ln\u0005\u0011\u0000\u0000mo\u0003\u0012\t\u0000nm\u0001\u0000\u0000"
+        + "\u0000no\u0001\u0000\u0000\u0000oq\u0001\u0000\u0000\u0000pr\u0003\u0014"
+        + "\n\u0000qp\u0001\u0000\u0000\u0000rs\u0001\u0000\u0000\u0000sq\u0001\u0000"
+        + "\u0000\u0000st\u0001\u0000\u0000\u0000t\r\u0001\u0000\u0000\u0000uw\u0005"
+        + "\u0007\u0000\u0000vx\u0003\u0012\t\u0000wv\u0001\u0000\u0000\u0000wx\u0001"
+        + "\u0000\u0000\u0000xy\u0001\u0000\u0000\u0000y{\u0003\u0014\n\u0000z|\u0003"
+        + "\u0014\n\u0000{z\u0001\u0000\u0000\u0000|}\u0001\u0000\u0000\u0000}{\u0001"
+        + "\u0000\u0000\u0000}~\u0001\u0000\u0000\u0000~\u0081\u0001\u0000\u0000"
+        + "\u0000\u007f\u0080\u0005\u0014\u0000\u0000\u0080\u0082\u0003\u0014\n\u0000"
+        + "\u0081\u007f\u0001\u0000\u0000\u0000\u0081\u0082\u0001\u0000\u0000\u0000"
+        + "\u0082\u000f\u0001\u0000\u0000\u0000\u0083\u0084\u0005*\u0000\u0000\u0084"
+        + "\u008d\u0005/\u0000\u0000\u0085\u008a\u0003 \u0010\u0000\u0086\u0087\u0005"
+        + "%\u0000\u0000\u0087\u0089\u0003 \u0010\u0000\u0088\u0086\u0001\u0000\u0000"
+        + "\u0000\u0089\u008c\u0001\u0000\u0000\u0000\u008a\u0088\u0001\u0000\u0000"
+        + "\u0000\u008a\u008b\u0001\u0000\u0000\u0000\u008b\u008e\u0001\u0000\u0000"
+        + "\u0000\u008c\u008a\u0001\u0000\u0000\u0000\u008d\u0085\u0001\u0000\u0000"
+        + "\u0000\u008d\u008e\u0001\u0000\u0000\u0000\u008e\u0011\u0001\u0000\u0000"
+        + "\u0000\u008f\u0090\u0005\u0003\u0000\u0000\u0090\u0095\u0003\u001e\u000f"
+        + "\u0000\u0091\u0092\u0005%\u0000\u0000\u0092\u0094\u0003\u001e\u000f\u0000"
+        + "\u0093\u0091\u0001\u0000\u0000\u0000\u0094\u0097\u0001\u0000\u0000\u0000"
+        + "\u0095\u0093\u0001\u0000\u0000\u0000\u0095\u0096\u0001\u0000\u0000\u0000"
+        + "\u0096\u0013\u0001\u0000\u0000\u0000\u0097\u0095\u0001\u0000\u0000\u0000"
+        + "\u0098\u009a\u0003\u0018\f\u0000\u0099\u009b\u0003\u0012\t\u0000\u009a"
+        + "\u0099\u0001\u0000\u0000\u0000\u009a\u009b\u0001\u0000\u0000\u0000\u009b"
+        + "\u0015\u0001\u0000\u0000\u0000\u009c\u009e\u0003\u0018\f\u0000\u009d\u009f"
+        + "\u0003\u0012\t\u0000\u009e\u009d\u0001\u0000\u0000\u0000\u009e\u009f\u0001"
+        + "\u0000\u0000\u0000\u009f\u00a4\u0001\u0000\u0000\u0000\u00a0\u00a1\u0005"
+        + "\u0016\u0000\u0000\u00a1\u00a2\u0005/\u0000\u0000\u00a2\u00a3\u0005\u0018"
+        + "\u0000\u0000\u00a3\u00a5\u0003:\u001d\u0000\u00a4\u00a0\u0001\u0000\u0000"
+        + "\u0000\u00a4\u00a5\u0001\u0000\u0000\u0000\u00a5\u0017\u0001\u0000\u0000"
+        + "\u0000\u00a6\u00a7\u0005&\u0000\u0000\u00a7\u00a8\u0003\u001c\u000e\u0000"
+        + "\u00a8\u00a9\u0005\'\u0000\u0000\u00a9\u0019\u0001\u0000\u0000\u0000\u00aa"
+        + "\u00ab\u0003\u001c\u000e\u0000\u00ab\u001b\u0001\u0000\u0000\u0000\u00ac"
+        + "\u00af\u0005\u0002\u0000\u0000\u00ad\u00af\u0003>\u001f\u0000\u00ae\u00ac"
+        + "\u0001\u0000\u0000\u0000\u00ae\u00ad\u0001\u0000\u0000\u0000\u00af\u00b0"
+        + "\u0001\u0000\u0000\u0000\u00b0\u00b1\u0005\u0015\u0000\u0000\u00b1\u00b2"
+        + "\u0003\u001e\u000f\u0000\u00b2\u001d\u0001\u0000\u0000\u0000\u00b3\u00b4"
+        + "\u0003 \u0010\u0000\u00b4\u001f\u0001\u0000\u0000\u0000\u00b5\u00b6\u0006"
+        + "\u0010\uffff\uffff\u0000\u00b6\u00b7\u0005\u000b\u0000\u0000\u00b7\u00bd"
+        + "\u0003 \u0010\u0005\u00b8\u00b9\u0005/\u0000\u0000\u00b9\u00ba\u0005\r"
+        + "\u0000\u0000\u00ba\u00bd\u0003\u0018\f\u0000\u00bb\u00bd\u0003\"\u0011"
+        + "\u0000\u00bc\u00b5\u0001\u0000\u0000\u0000\u00bc\u00b8\u0001\u0000\u0000"
+        + "\u0000\u00bc\u00bb\u0001\u0000\u0000\u0000\u00bd\u00c6\u0001\u0000\u0000"
+        + "\u0000\u00be\u00bf\n\u0002\u0000\u0000\u00bf\u00c0\u0005\u0001\u0000\u0000"
+        + "\u00c0\u00c5\u0003 \u0010\u0003\u00c1\u00c2\n\u0001\u0000\u0000\u00c2"
+        + "\u00c3\u0005\u000e\u0000\u0000\u00c3\u00c5\u0003 \u0010\u0002\u00c4\u00be"
+        + "\u0001\u0000\u0000\u0000\u00c4\u00c1\u0001\u0000\u0000\u0000\u00c5\u00c8"
+        + "\u0001\u0000\u0000\u0000\u00c6\u00c4\u0001\u0000\u0000\u0000\u00c6\u00c7"
+        + "\u0001\u0000\u0000\u0000\u00c7!\u0001\u0000\u0000\u0000\u00c8\u00c6\u0001"
+        + "\u0000\u0000\u0000\u00c9\u00cf\u0003$\u0012\u0000\u00ca\u00cb\u0003$\u0012"
+        + "\u0000\u00cb\u00cc\u00030\u0018\u0000\u00cc\u00cd\u0003$\u0012\u0000\u00cd"
+        + "\u00cf\u0001\u0000\u0000\u0000\u00ce\u00c9\u0001\u0000\u0000\u0000\u00ce"
+        + "\u00ca\u0001\u0000\u0000\u0000\u00cf#\u0001\u0000\u0000\u0000\u00d0\u00d1"
+        + "\u0006\u0012\uffff\uffff\u0000\u00d1\u00d3\u0003(\u0014\u0000\u00d2\u00d4"
+        + "\u0003&\u0013\u0000\u00d3\u00d2\u0001\u0000\u0000\u0000\u00d3\u00d4\u0001"
+        + "\u0000\u0000\u0000\u00d4\u00d8\u0001\u0000\u0000\u0000\u00d5\u00d6\u0007"
+        + "\u0000\u0000\u0000\u00d6\u00d8\u0003$\u0012\u0003\u00d7\u00d0\u0001\u0000"
+        + "\u0000\u0000\u00d7\u00d5\u0001\u0000\u0000\u0000\u00d8\u00e1\u0001\u0000"
+        + "\u0000\u0000\u00d9\u00da\n\u0002\u0000\u0000\u00da\u00db\u0007\u0001\u0000"
+        + "\u0000\u00db\u00e0\u0003$\u0012\u0003\u00dc\u00dd\n\u0001\u0000\u0000"
+        + "\u00dd\u00de\u0007\u0000\u0000\u0000\u00de\u00e0\u0003$\u0012\u0002\u00df"
+        + "\u00d9\u0001\u0000\u0000\u0000\u00df\u00dc\u0001\u0000\u0000\u0000\u00e0"
+        + "\u00e3\u0001\u0000\u0000\u0000\u00e1\u00df\u0001\u0000\u0000\u0000\u00e1"
+        + "\u00e2\u0001\u0000\u0000\u0000\u00e2%\u0001\u0000\u0000\u0000\u00e3\u00e1"
+        + "\u0001\u0000\u0000\u0000\u00e4\u00e6\u0005\u000b\u0000\u0000\u00e5\u00e4"
+        + "\u0001\u0000\u0000\u0000\u00e5\u00e6\u0001\u0000\u0000\u0000\u00e6\u00e7"
+        + "\u0001\u0000\u0000\u0000\u00e7\u00e8\u0007\u0002\u0000\u0000\u00e8\u00e9"
+        + "\u0005(\u0000\u0000\u00e9\u00ee\u0003\u001e\u000f\u0000\u00ea\u00eb\u0005"
+        + "%\u0000\u0000\u00eb\u00ed\u0003\u001e\u000f\u0000\u00ec\u00ea\u0001\u0000"
+        + "\u0000\u0000\u00ed\u00f0\u0001\u0000\u0000\u0000\u00ee\u00ec\u0001\u0000"
+        + "\u0000\u0000\u00ee\u00ef\u0001\u0000\u0000\u0000\u00ef\u00f1\u0001\u0000"
+        + "\u0000\u0000\u00f0\u00ee\u0001\u0000\u0000\u0000\u00f1\u00f2\u0005)\u0000"
+        + "\u0000\u00f2\u0102\u0001\u0000\u0000\u0000\u00f3\u00f4\u0007\u0003\u0000"
+        + "\u0000\u00f4\u0102\u0003.\u0017\u0000\u00f5\u00f6\u0007\u0003\u0000\u0000"
+        + "\u00f6\u00f7\u0005(\u0000\u0000\u00f7\u00fc\u0003.\u0017\u0000\u00f8\u00f9"
+        + "\u0005%\u0000\u0000\u00f9\u00fb\u0003.\u0017\u0000\u00fa\u00f8\u0001\u0000"
+        + "\u0000\u0000\u00fb\u00fe\u0001\u0000\u0000\u0000\u00fc\u00fa\u0001\u0000"
+        + "\u0000\u0000\u00fc\u00fd\u0001\u0000\u0000\u0000\u00fd\u00ff\u0001\u0000"
+        + "\u0000\u0000\u00fe\u00fc\u0001\u0000\u0000\u0000\u00ff\u0100\u0005)\u0000"
+        + "\u0000\u0100\u0102\u0001\u0000\u0000\u0000\u0101\u00e5\u0001\u0000\u0000"
+        + "\u0000\u0101\u00f3\u0001\u0000\u0000\u0000\u0101\u00f5\u0001\u0000\u0000"
+        + "\u0000\u0102\'\u0001\u0000\u0000\u0000\u0103\u010b\u0003.\u0017\u0000"
+        + "\u0104\u010b\u0003*\u0015\u0000\u0105\u010b\u00034\u001a\u0000\u0106\u0107"
+        + "\u0005(\u0000\u0000\u0107\u0108\u0003\u001e\u000f\u0000\u0108\u0109\u0005"
+        + ")\u0000\u0000\u0109\u010b\u0001\u0000\u0000\u0000\u010a\u0103\u0001\u0000"
+        + "\u0000\u0000\u010a\u0104\u0001\u0000\u0000\u0000\u010a\u0105\u0001\u0000"
+        + "\u0000\u0000\u010a\u0106\u0001\u0000\u0000\u0000\u010b)\u0001\u0000\u0000"
+        + "\u0000\u010c\u010d\u0003,\u0016\u0000\u010d\u0116\u0005(\u0000\u0000\u010e"
+        + "\u0113\u0003\u001e\u000f\u0000\u010f\u0110\u0005%\u0000\u0000\u0110\u0112"
+        + "\u0003\u001e\u000f\u0000\u0111\u010f\u0001\u0000\u0000\u0000\u0112\u0115"
+        + "\u0001\u0000\u0000\u0000\u0113\u0111\u0001\u0000\u0000\u0000\u0113\u0114"
+        + "\u0001\u0000\u0000\u0000\u0114\u0117\u0001\u0000\u0000\u0000\u0115\u0113"
+        + "\u0001\u0000\u0000\u0000\u0116\u010e\u0001\u0000\u0000\u0000\u0116\u0117"
+        + "\u0001\u0000\u0000\u0000\u0117\u0118\u0001\u0000\u0000\u0000\u0118\u0119"
+        + "\u0005)\u0000\u0000\u0119+\u0001\u0000\u0000\u0000\u011a\u011b\u0007\u0004"
+        + "\u0000\u0000\u011b-\u0001\u0000\u0000\u0000\u011c\u0121\u0005\f\u0000"
+        + "\u0000\u011d\u0121\u0003:\u001d\u0000\u011e\u0121\u00032\u0019\u0000\u011f"
+        + "\u0121\u0003<\u001e\u0000\u0120\u011c\u0001\u0000\u0000\u0000\u0120\u011d"
+        + "\u0001\u0000\u0000\u0000\u0120\u011e\u0001\u0000\u0000\u0000\u0120\u011f"
+        + "\u0001\u0000\u0000\u0000\u0121/\u0001\u0000\u0000\u0000\u0122\u0123\u0007"
+        + "\u0005\u0000\u0000\u01231\u0001\u0000\u0000\u0000\u0124\u0125\u0007\u0006"
+        + "\u0000\u0000\u01253\u0001\u0000\u0000\u0000\u0126\u0128\u0005+\u0000\u0000"
+        + "\u0127\u0126\u0001\u0000\u0000\u0000\u0127\u0128\u0001\u0000\u0000\u0000"
+        + "\u0128\u0129\u0001\u0000\u0000\u0000\u0129\u0135\u00036\u001b\u0000\u012a"
+        + "\u012b\u0005$\u0000\u0000\u012b\u0134\u00036\u001b\u0000\u012c\u012e\u0005"
+        + "&\u0000\u0000\u012d\u012f\u0005-\u0000\u0000\u012e\u012d\u0001\u0000\u0000"
+        + "\u0000\u012f\u0130\u0001\u0000\u0000\u0000\u0130\u012e\u0001\u0000\u0000"
+        + "\u0000\u0130\u0131\u0001\u0000\u0000\u0000\u0131\u0132\u0001\u0000\u0000"
+        + "\u0000\u0132\u0134\u0005\'\u0000\u0000\u0133\u012a\u0001\u0000\u0000\u0000"
+        + "\u0133\u012c\u0001\u0000\u0000\u0000\u0134\u0137\u0001\u0000\u0000\u0000"
+        + "\u0135\u0133\u0001\u0000\u0000\u0000\u0135\u0136\u0001\u0000\u0000\u0000"
+        + "\u01365\u0001\u0000\u0000\u0000\u0137\u0135\u0001\u0000\u0000\u0000\u0138"
+        + "\u0139\u0007\u0007\u0000\u0000\u01397\u0001\u0000\u0000\u0000\u013a\u013c"
+        + "\u0003:\u001d\u0000\u013b\u013d\u0005/\u0000\u0000\u013c\u013b\u0001\u0000"
+        + "\u0000\u0000\u013c\u013d\u0001\u0000\u0000\u0000\u013d9\u0001\u0000\u0000"
+        + "\u0000\u013e\u0141\u0005.\u0000\u0000\u013f\u0141\u0005-\u0000\u0000\u0140"
+        + "\u013e\u0001\u0000\u0000\u0000\u0140\u013f\u0001\u0000\u0000\u0000\u0141"
+        + ";\u0001\u0000\u0000\u0000\u0142\u0143\u0005,\u0000\u0000\u0143=\u0001"
+        + "\u0000\u0000\u0000\u0144\u0145\u0007\b\u0000\u0000\u0145?\u0001\u0000"
+        + "\u0000\u0000)JQ[_afjnsw}\u0081\u008a\u008d\u0095\u009a\u009e\u00a4\u00ae"
+        + "\u00bc\u00c4\u00c6\u00ce\u00d3\u00d7\u00df\u00e1\u00e5\u00ee\u00fc\u0101"
+        + "\u010a\u0113\u0116\u0120\u0127\u0130\u0133\u0135\u013c\u0140";
     public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray());
     static {
         _decisionToDFA = new DFA[_ATN.getNumberOfDecisions()];
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/Mapper.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/Mapper.java
index 721c7910699b..2e20d75ab55a 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/Mapper.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/Mapper.java
@@ -50,8 +50,8 @@ PhysicalPlan map(LogicalPlan plan) {
     }
 
     @Override
-    protected Iterable<RuleExecutor<PhysicalPlan>.Batch> batches() {
-        Batch conversion = new Batch("Mapping", new SimpleExecMapper());
+    protected Iterable<RuleExecutor.Batch<PhysicalPlan>> batches() {
+        var conversion = new Batch<>("Mapping", new SimpleExecMapper());
 
         return Arrays.asList(conversion);
     }
@@ -135,7 +135,6 @@ public final PhysicalPlan apply(PhysicalPlan plan) {
         }
 
         @SuppressWarnings("unchecked")
-        @Override
         protected final PhysicalPlan rule(UnplannedExec plan) {
             LogicalPlan subPlan = plan.plan();
             if (subPlanToken.isInstance(subPlan)) {
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryFolder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryFolder.java
index a6ce44221fec..cd53ce2adfc7 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryFolder.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryFolder.java
@@ -40,9 +40,9 @@ PhysicalPlan fold(PhysicalPlan plan) {
     }
 
     @Override
-    protected Iterable<RuleExecutor<PhysicalPlan>.Batch> batches() {
-        Batch fold = new Batch("Fold queries", new FoldProject(), new FoldFilter(), new FoldOrderBy(), new FoldLimit());
-        Batch finish = new Batch("Finish query", Limiter.ONCE, new PlanOutputToQueryRef());
+    protected Iterable<RuleExecutor.Batch<PhysicalPlan>> batches() {
+        var fold = new Batch<>("Fold queries", new FoldProject(), new FoldFilter(), new FoldOrderBy(), new FoldLimit());
+        var finish = new Batch<>("Finish query", Limiter.ONCE, new PlanOutputToQueryRef());
 
         return Arrays.asList(fold, finish);
     }
@@ -139,7 +139,6 @@ public final PhysicalPlan apply(PhysicalPlan plan) {
             return plan.transformUp(typeToken(), this::rule);
         }
 
-        @Override
         protected abstract PhysicalPlan rule(SubPlan plan);
     }
 
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java
index daaab61df410..8097447df048 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -83,7 +83,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver expressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         return createComponents(client, environment.settings(), clusterService);
     }
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java
index 344d7c3d1527..d0b2b4b445a1 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java
@@ -230,6 +230,7 @@ public static void operation(
                 timeout,
                 request.indicesOptions(),
                 request.fetchSize(),
+                request.maxSamplesPerKey(),
                 clientId,
                 new TaskId(nodeId, task.getId()),
                 task
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java
index 338c2663dfd9..8dd8220fb63b 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java
@@ -29,6 +29,7 @@ public class EqlConfiguration extends org.elasticsearch.xpack.ql.session.Configu
     private final TaskId taskId;
     private final EqlSearchTask task;
     private final int fetchSize;
+    private final int maxSamplesPerKey;
 
     @Nullable
     private final QueryBuilder filter;
@@ -48,6 +49,7 @@ public EqlConfiguration(
         TimeValue requestTimeout,
         IndicesOptions indicesOptions,
         int fetchSize,
+        int maxSamplesPerKey,
         String clientId,
         TaskId taskId,
         EqlSearchTask task
@@ -64,6 +66,7 @@ public EqlConfiguration(
         this.taskId = taskId;
         this.task = task;
         this.fetchSize = fetchSize;
+        this.maxSamplesPerKey = maxSamplesPerKey;
     }
 
     public String[] indices() {
@@ -82,6 +85,10 @@ public int fetchSize() {
         return fetchSize;
     }
 
+    public int maxSamplesPerKey() {
+        return maxSamplesPerKey;
+    }
+
     public QueryBuilder filter() {
         return filter;
     }
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java
index 9ec908c696a0..3c55275b8dee 100644
--- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java
+++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.xpack.eql.analysis.Analyzer;
+import org.elasticsearch.xpack.eql.analysis.AnalyzerContext;
 import org.elasticsearch.xpack.eql.analysis.PostAnalyzer;
 import org.elasticsearch.xpack.eql.analysis.PreAnalyzer;
 import org.elasticsearch.xpack.eql.analysis.Verifier;
@@ -59,7 +60,7 @@ public EqlSession(
         this.indexResolver = indexResolver;
         this.preAnalyzer = preAnalyzer;
         this.postAnalyzer = postAnalyzer;
-        this.analyzer = new Analyzer(cfg, functionRegistry, verifier);
+        this.analyzer = new Analyzer(new AnalyzerContext(cfg, functionRegistry), verifier);
         this.optimizer = optimizer;
         this.planner = planner;
         this.circuitBreaker = circuitBreaker;
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java
index f56187f5ff84..28f4a9fc4671 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java
@@ -53,6 +53,7 @@ private EqlTestUtils() {}
         TimeValue.timeValueSeconds(30),
         null,
         123,
+        1,
         "",
         new TaskId("test", 123),
         null
@@ -70,6 +71,7 @@ public static EqlConfiguration randomConfiguration() {
             new TimeValue(randomNonNegativeLong()),
             randomIndicesOptions(),
             randomIntBetween(1, 1000),
+            randomIntBetween(1, 1000),
             randomAlphaOfLength(16),
             new TaskId(randomAlphaOfLength(10), randomNonNegativeLong()),
             randomTask()
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java
index f59242769f7e..cf50f3e1762b 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java
@@ -82,13 +82,19 @@ protected EqlSearchRequest createTestInstance() {
                 .keepOnCompletion(randomBoolean())
                 .fetchFields(randomFetchFields)
                 .runtimeMappings(randomRuntimeMappings())
-                .resultPosition(randomFrom("tail", "head"));
+                .resultPosition(randomFrom("tail", "head"))
+                .maxSamplesPerKey(randomIntBetween(1, 1000));
         } catch (IOException ex) {
             assertNotNull("unexpected IOException " + ex.getCause().getMessage(), ex);
         }
         return null;
     }
 
+    @Override
+    protected EqlSearchRequest mutateInstance(EqlSearchRequest instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     private TimeValue randomTV() {
         return TimeValue.parseTimeValue(randomTimeValue(), null, "test");
     }
@@ -133,6 +139,7 @@ protected EqlSearchRequest mutateInstanceForVersion(EqlSearchRequest instance, V
         mutatedInstance.fetchFields(version.onOrAfter(Version.V_7_13_0) ? instance.fetchFields() : null);
         mutatedInstance.runtimeMappings(version.onOrAfter(Version.V_7_13_0) ? instance.runtimeMappings() : emptyMap());
         mutatedInstance.resultPosition(version.onOrAfter(Version.V_7_17_8) ? instance.resultPosition() : "tail");
+        mutatedInstance.maxSamplesPerKey(version.onOrAfter(Version.V_8_7_0) ? instance.maxSamplesPerKey() : 1);
 
         return mutatedInstance;
     }
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java
index d9f12004df01..e36dccdea1fb 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java
@@ -157,6 +157,11 @@ protected EqlSearchResponse createTestInstance() {
         return randomEqlSearchResponse(XContentType.JSON);
     }
 
+    @Override
+    protected EqlSearchResponse mutateInstance(EqlSearchResponse instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<EqlSearchResponse> instanceReader() {
         return EqlSearchResponse::new;
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/AnalyzerTestUtils.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/AnalyzerTestUtils.java
new file mode 100644
index 000000000000..c93bb693f736
--- /dev/null
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/AnalyzerTestUtils.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.eql.analysis;
+
+import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry;
+import org.elasticsearch.xpack.eql.session.EqlConfiguration;
+import org.elasticsearch.xpack.eql.stats.Metrics;
+import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
+
+import static org.elasticsearch.xpack.eql.EqlTestUtils.TEST_CFG;
+
+public final class AnalyzerTestUtils {
+
+    private AnalyzerTestUtils() {}
+
+    public static Analyzer analyzer() {
+        return new Analyzer(new AnalyzerContext(TEST_CFG, new EqlFunctionRegistry()), new Verifier(new Metrics()));
+    }
+
+    public static Analyzer analyzer(Verifier verifier) {
+        return analyzer(TEST_CFG, new EqlFunctionRegistry(), verifier);
+    }
+
+    public static Analyzer analyzer(EqlConfiguration configuration) {
+        return analyzer(configuration, new EqlFunctionRegistry());
+    }
+
+    public static Analyzer analyzer(EqlConfiguration configuration, FunctionRegistry registry) {
+        return analyzer(configuration, registry, new Verifier(new Metrics()));
+    }
+
+    public static Analyzer analyzer(EqlConfiguration configuration, FunctionRegistry registry, Verifier verifier) {
+        return new Analyzer(new AnalyzerContext(configuration, registry), verifier);
+    }
+}
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/AnalyzerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/AnalyzerTests.java
index 5d6b2e52b901..eec3af3a23d9 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/AnalyzerTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/AnalyzerTests.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.eql.expression.OptionalMissingAttribute;
 import org.elasticsearch.xpack.eql.expression.OptionalResolvedAttribute;
-import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.Concat;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.ToString;
 import org.elasticsearch.xpack.eql.parser.EqlParser;
@@ -19,7 +18,6 @@
 import org.elasticsearch.xpack.eql.plan.logical.LimitWithOffset;
 import org.elasticsearch.xpack.eql.plan.logical.Sample;
 import org.elasticsearch.xpack.eql.plan.logical.Sequence;
-import org.elasticsearch.xpack.eql.stats.Metrics;
 import org.elasticsearch.xpack.ql.expression.Expression;
 import org.elasticsearch.xpack.ql.expression.FieldAttribute;
 import org.elasticsearch.xpack.ql.expression.Literal;
@@ -41,7 +39,7 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.eql.EqlTestUtils.TEST_CFG;
+import static org.elasticsearch.xpack.eql.analysis.AnalyzerTestUtils.analyzer;
 
 public class AnalyzerTests extends ESTestCase {
 
@@ -265,7 +263,7 @@ public void testOptionalFieldsAsSampleKey() {
 
     private LogicalPlan accept(IndexResolution resolution, String eql) {
         PreAnalyzer preAnalyzer = new PreAnalyzer();
-        Analyzer analyzer = new Analyzer(TEST_CFG, new EqlFunctionRegistry(), new Verifier(new Metrics()));
+        Analyzer analyzer = analyzer();
         EqlParser parser = new EqlParser();
         LogicalPlan plan = parser.createStatement(eql);
         return analyzer.analyze(preAnalyzer.preAnalyze(plan, resolution));
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java
index 4001348c3062..d13852cb3a01 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java
@@ -9,14 +9,11 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.eql.EqlTestUtils;
-import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry;
 import org.elasticsearch.xpack.eql.parser.EqlParser;
 import org.elasticsearch.xpack.eql.parser.ParsingException;
 import org.elasticsearch.xpack.eql.plan.logical.KeyedFilter;
 import org.elasticsearch.xpack.eql.plan.logical.Sample;
 import org.elasticsearch.xpack.eql.session.EqlConfiguration;
-import org.elasticsearch.xpack.eql.stats.Metrics;
 import org.elasticsearch.xpack.ql.expression.EmptyAttribute;
 import org.elasticsearch.xpack.ql.index.EsIndex;
 import org.elasticsearch.xpack.ql.index.IndexResolution;
@@ -29,6 +26,7 @@
 import java.util.function.Function;
 
 import static java.util.Collections.emptyMap;
+import static org.elasticsearch.xpack.eql.analysis.AnalyzerTestUtils.analyzer;
 import static org.hamcrest.Matchers.startsWith;
 
 public class VerifierTests extends ESTestCase {
@@ -48,7 +46,7 @@ private IndexResolution loadIndexResolution(String name) {
     private LogicalPlan accept(IndexResolution resolution, String eql) {
         EqlParser parser = new EqlParser();
         PreAnalyzer preAnalyzer = new PreAnalyzer();
-        Analyzer analyzer = new Analyzer(EqlTestUtils.TEST_CFG, new EqlFunctionRegistry(), new Verifier(new Metrics()));
+        Analyzer analyzer = analyzer();
 
         LogicalPlan plan = parser.createStatement(eql);
         return analyzer.analyze(preAnalyzer.preAnalyze(plan, resolution));
@@ -467,11 +465,12 @@ private LogicalPlan analyzeWithVerifierFunction(Function<String, Collection<Stri
             TimeValue.timeValueSeconds(30),
             null,
             123,
+            1,
             "",
             new TaskId("test", 123),
             null
         );
-        Analyzer analyzer = new Analyzer(eqlConfiguration, new EqlFunctionRegistry(), new Verifier(new Metrics()));
+        Analyzer analyzer = analyzer(eqlConfiguration);
         IndexResolution resolution = IndexResolution.valid(new EsIndex("irrelevant", loadEqlMapping("mapping-default.json")));
         return analyzer.analyze(preAnalyzer.preAnalyze(new EqlParser().createStatement("any where true"), resolution));
     }
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/CircuitBreakerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/CircuitBreakerTests.java
index dbd04989184b..e9d65201aa14 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/CircuitBreakerTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/CircuitBreakerTests.java
@@ -85,7 +85,7 @@ public void query(QueryRequest r, ActionListener<SearchResponse> l) {}
 
             @Override
             public void fetchHits(Iterable<List<HitReference>> refs, ActionListener<List<List<SearchHit>>> listener) {}
-        }, mockCriteria(), randomIntBetween(10, 500), new Limit(1000, 0), CIRCUIT_BREAKER);
+        }, mockCriteria(), randomIntBetween(10, 500), new Limit(1000, 0), CIRCUIT_BREAKER, 1);
 
         CIRCUIT_BREAKER.startBreaking();
         iterator.pushToStack(new SampleIterator.Page(CB_STACK_SIZE_PRECISION - 1));
@@ -137,7 +137,8 @@ public void fetchHits(Iterable<List<HitReference>> refs, ActionListener<List<Lis
                 mockCriteria(),
                 randomIntBetween(10, 500),
                 new Limit(1000, 0),
-                eqlCircuitBreaker
+                eqlCircuitBreaker,
+                1
             );
 
             // unfortunately, mocking an actual result set it extremely complicated
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/SampleIteratorTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/SampleIteratorTests.java
index 6e3d0bbec062..76deb43f2e62 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/SampleIteratorTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/SampleIteratorTests.java
@@ -15,24 +15,56 @@
 
 import static java.util.Arrays.asList;
 import static java.util.Collections.emptyList;
-import static org.elasticsearch.xpack.eql.execution.sample.SampleIterator.matchSample;
+import static org.elasticsearch.xpack.eql.execution.sample.SampleIterator.matchSamples;
 
 public class SampleIteratorTests extends ESTestCase {
 
-    public void testMatchSample() {
+    public void testMatchSamples() {
         assertEquals(
+            List.of(asSearchHitsList(2, 1, 3)),
+            matchSamples(asList(asSearchHitsList(1, 1, 2), asSearchHitsList(1, 1, 1), asSearchHitsList(1, 1, 3)), 3, 1)
+        );
+        assertEquals(
+            List.of(asSearchHitsList(1, 4, 5)),
+            matchSamples(asList(asSearchHitsList(1, 2, 3), asSearchHitsList(4, 5), asSearchHitsList(4, 5, 3)), 3, 1)
+        );
+        assertEquals(
+            List.of(asSearchHitsList(1, 2, 3)),
+            matchSamples(asList(asSearchHitsList(1), asSearchHitsList(2), asSearchHitsList(3)), 3, 1)
+        );
+        assertTrue(matchSamples(asList(asSearchHitsList(3), asSearchHitsList(3), asSearchHitsList(3)), 3, 1).isEmpty());
+        assertTrue(matchSamples(asList(asSearchHitsList(1), asSearchHitsList(1), asSearchHitsList(3)), 3, 1).isEmpty());
+        assertTrue(matchSamples(asList(asSearchHitsList(1), asSearchHitsList(3), asSearchHitsList(3)), 3, 1).isEmpty());
+        assertTrue(matchSamples(asList(asSearchHitsList(1, 1, 1), asSearchHitsList(1, 1, 1), asSearchHitsList(1, 1, 3)), 3, 1).isEmpty());
+
+        // multiple matches per key
+        List<List<SearchHit>> combinations = List.of(
+            asSearchHitsList(1, 3, 4),
+            asSearchHitsList(1, 4, 3),
             asSearchHitsList(2, 1, 3),
-            matchSample(asList(asSearchHitsList(1, 1, 2), asSearchHitsList(1, 1, 1), asSearchHitsList(1, 1, 3)), 3)
+            asSearchHitsList(2, 1, 4),
+            asSearchHitsList(2, 3, 1),
+            asSearchHitsList(2, 3, 4),
+            asSearchHitsList(2, 4, 1),
+            asSearchHitsList(2, 4, 3),
+            asSearchHitsList(3, 1, 4),
+            asSearchHitsList(3, 4, 1)
         );
+        for (int i = 1; i <= combinations.size(); i++) {
+            assertEquals(
+                combinations.subList(0, i),
+                matchSamples(asList(asSearchHitsList(1, 2, 3), asSearchHitsList(1, 3, 4), asSearchHitsList(1, 3, 4)), 3, i)
+            );
+        }
         assertEquals(
-            asSearchHitsList(1, 4, 5),
-            matchSample(asList(asSearchHitsList(1, 2, 3), asSearchHitsList(4, 5), asSearchHitsList(4, 5, 3)), 3)
+            combinations,
+            matchSamples(
+                asList(asSearchHitsList(1, 2, 3), asSearchHitsList(1, 3, 4), asSearchHitsList(1, 3, 4)),
+                3,
+                combinations.size() + 1
+            )
         );
-        assertEquals(asSearchHitsList(1, 2, 3), matchSample(asList(asSearchHitsList(1), asSearchHitsList(2), asSearchHitsList(3)), 3));
-        assertNull(matchSample(asList(asSearchHitsList(3), asSearchHitsList(3), asSearchHitsList(3)), 3));
-        assertNull(matchSample(asList(asSearchHitsList(1), asSearchHitsList(1), asSearchHitsList(3)), 3));
-        assertNull(matchSample(asList(asSearchHitsList(1), asSearchHitsList(3), asSearchHitsList(3)), 3));
-        assertNull(matchSample(asList(asSearchHitsList(1, 1, 1), asSearchHitsList(1, 1, 1), asSearchHitsList(1, 1, 3)), 3));
+
     }
 
     private List<SearchHit> asSearchHitsList(Integer... docIds) {
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java
index 4751cd7e6e9e..8c4a67173d0c 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java
@@ -290,6 +290,7 @@ private QueryClient buildQueryClient(ESMockClient esClient, CircuitBreaker eqlCi
             TimeValue.timeValueSeconds(30),
             null,
             123,
+            1,
             "",
             new TaskId("test", 123),
             new EqlSearchTask(
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/PITFailureTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/PITFailureTests.java
index c32a5c37b03e..5f84b43f6cb6 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/PITFailureTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/PITFailureTests.java
@@ -78,6 +78,7 @@ public void testHandlingPitFailure() {
                 TimeValue.timeValueSeconds(30),
                 null,
                 123,
+                1,
                 "",
                 new TaskId("test", 123),
                 new EqlSearchTask(
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/predicate/operator/comparison/InsensitiveBinaryComparisonProcessorTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/predicate/operator/comparison/InsensitiveBinaryComparisonProcessorTests.java
index 13abf95e5f1e..408c4a79e813 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/predicate/operator/comparison/InsensitiveBinaryComparisonProcessorTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/predicate/operator/comparison/InsensitiveBinaryComparisonProcessorTests.java
@@ -33,6 +33,11 @@ protected InsensitiveBinaryComparisonProcessor createTestInstance() {
         return randomProcessor();
     }
 
+    @Override
+    protected InsensitiveBinaryComparisonProcessor mutateInstance(InsensitiveBinaryComparisonProcessor instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<InsensitiveBinaryComparisonProcessor> instanceReader() {
         return InsensitiveBinaryComparisonProcessor::new;
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java
index 420db2d47d66..79c8ea19fd3b 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java
@@ -12,8 +12,6 @@
 import org.elasticsearch.xpack.eql.analysis.Analyzer;
 import org.elasticsearch.xpack.eql.analysis.PostAnalyzer;
 import org.elasticsearch.xpack.eql.analysis.PreAnalyzer;
-import org.elasticsearch.xpack.eql.analysis.Verifier;
-import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.ToString;
 import org.elasticsearch.xpack.eql.parser.EqlParser;
 import org.elasticsearch.xpack.eql.plan.logical.AbstractJoin;
@@ -23,7 +21,6 @@
 import org.elasticsearch.xpack.eql.plan.logical.Sequence;
 import org.elasticsearch.xpack.eql.plan.logical.Tail;
 import org.elasticsearch.xpack.eql.plan.physical.LocalRelation;
-import org.elasticsearch.xpack.eql.stats.Metrics;
 import org.elasticsearch.xpack.ql.TestUtils;
 import org.elasticsearch.xpack.ql.expression.Attribute;
 import org.elasticsearch.xpack.ql.expression.EmptyAttribute;
@@ -69,6 +66,7 @@
 import static java.util.Collections.singletonList;
 import static java.util.stream.Collectors.toList;
 import static org.elasticsearch.xpack.eql.EqlTestUtils.TEST_CFG;
+import static org.elasticsearch.xpack.eql.analysis.AnalyzerTestUtils.analyzer;
 import static org.elasticsearch.xpack.ql.TestUtils.UTC;
 import static org.elasticsearch.xpack.ql.expression.Literal.TRUE;
 import static org.elasticsearch.xpack.ql.tree.Source.EMPTY;
@@ -92,7 +90,7 @@ public static IndexResolution loadIndexResolution(String name) {
     private LogicalPlan accept(IndexResolution resolution, String eql) {
         PreAnalyzer preAnalyzer = new PreAnalyzer();
         PostAnalyzer postAnalyzer = new PostAnalyzer();
-        Analyzer analyzer = new Analyzer(TEST_CFG, new EqlFunctionRegistry(), new Verifier(new Metrics()));
+        Analyzer analyzer = analyzer();
         return optimizer.optimize(
             postAnalyzer.postAnalyze(analyzer.analyze(preAnalyzer.preAnalyze(parser.createStatement(eql), resolution)), TEST_CFG)
         );
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/TomlFoldTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/TomlFoldTests.java
index dc10a99b9638..f7250e63be34 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/TomlFoldTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/TomlFoldTests.java
@@ -11,11 +11,8 @@
 
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.eql.analysis.Analyzer;
-import org.elasticsearch.xpack.eql.analysis.Verifier;
-import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry;
 import org.elasticsearch.xpack.eql.parser.EqlParser;
 import org.elasticsearch.xpack.eql.plan.physical.LocalRelation;
-import org.elasticsearch.xpack.eql.stats.Metrics;
 import org.elasticsearch.xpack.ql.expression.Alias;
 import org.elasticsearch.xpack.ql.expression.Expression;
 import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
@@ -30,7 +27,7 @@
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
 import static java.util.stream.Collectors.toList;
-import static org.elasticsearch.xpack.eql.EqlTestUtils.TEST_CFG;
+import static org.elasticsearch.xpack.eql.analysis.AnalyzerTestUtils.analyzer;
 import static org.elasticsearch.xpack.ql.tree.Source.EMPTY;
 
 public class TomlFoldTests extends ESTestCase {
@@ -38,8 +35,7 @@ public class TomlFoldTests extends ESTestCase {
     protected static final String PARAM_FORMATTING = "%1$s.test -> %2$s";
 
     private static final EqlParser PARSER = new EqlParser();
-    private static final EqlFunctionRegistry FUNCTION_REGISTRY = new EqlFunctionRegistry();
-    private static final Analyzer ANALYZER = new Analyzer(TEST_CFG, FUNCTION_REGISTRY, new Verifier(new Metrics()));
+    private static final Analyzer ANALYZER = analyzer();
 
     private final int num;
     private final EqlFoldSpec spec;
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/AbstractQueryTranslatorTestCase.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/AbstractQueryTranslatorTestCase.java
index bcca55a5692b..4bdccd6bd34b 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/AbstractQueryTranslatorTestCase.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/AbstractQueryTranslatorTestCase.java
@@ -12,16 +12,14 @@
 import org.elasticsearch.xpack.eql.analysis.Analyzer;
 import org.elasticsearch.xpack.eql.analysis.PostAnalyzer;
 import org.elasticsearch.xpack.eql.analysis.PreAnalyzer;
-import org.elasticsearch.xpack.eql.analysis.Verifier;
-import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry;
 import org.elasticsearch.xpack.eql.optimizer.Optimizer;
 import org.elasticsearch.xpack.eql.parser.EqlParser;
 import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan;
 import org.elasticsearch.xpack.eql.session.EqlConfiguration;
-import org.elasticsearch.xpack.eql.stats.Metrics;
 import org.elasticsearch.xpack.ql.index.EsIndex;
 import org.elasticsearch.xpack.ql.index.IndexResolution;
 
+import static org.elasticsearch.xpack.eql.analysis.AnalyzerTestUtils.analyzer;
 import static org.elasticsearch.xpack.ql.type.TypesTests.loadMapping;
 
 public abstract class AbstractQueryTranslatorTestCase extends ESTestCase {
@@ -29,7 +27,7 @@ public abstract class AbstractQueryTranslatorTestCase extends ESTestCase {
     protected PreAnalyzer preAnalyzer = new PreAnalyzer();
     protected PostAnalyzer postAnalyzer = new PostAnalyzer();
     protected EqlConfiguration configuration = EqlTestUtils.randomConfiguration();
-    protected Analyzer analyzer = new Analyzer(configuration, new EqlFunctionRegistry(), new Verifier(new Metrics()));
+    protected Analyzer analyzer = analyzer(configuration);
     protected Optimizer optimizer = new Optimizer();
     protected Planner planner = new Planner();
 
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/stats/VerifierMetricsTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/stats/VerifierMetricsTests.java
index 9628370c0543..bdb0a4800f97 100644
--- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/stats/VerifierMetricsTests.java
+++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/stats/VerifierMetricsTests.java
@@ -18,8 +18,10 @@
 import org.elasticsearch.xpack.eql.parser.EqlParser;
 import org.elasticsearch.xpack.ql.index.IndexResolution;
 
+import java.util.Arrays;
 import java.util.Set;
 
+import static org.elasticsearch.xpack.eql.analysis.AnalyzerTestUtils.analyzer;
 import static org.elasticsearch.xpack.eql.stats.FeatureMetric.EVENT;
 import static org.elasticsearch.xpack.eql.stats.FeatureMetric.JOIN;
 import static org.elasticsearch.xpack.eql.stats.FeatureMetric.JOIN_KEYS_FIVE_OR_MORE;
@@ -179,7 +181,7 @@ private void assertCounters(Counters actual, Set<FeatureMetric> metrics) {
     private Counters eql(String query) {
         Metrics metrics = new Metrics();
         Verifier verifier = new Verifier(metrics);
-        Analyzer analyzer = new Analyzer(EqlTestUtils.randomConfiguration(), eqlFunctionRegistry, verifier);
+        Analyzer analyzer = analyzer(EqlTestUtils.randomConfiguration(), eqlFunctionRegistry, verifier);
         analyzer.analyze(preAnalyzer.preAnalyze(parser.createStatement(query), index));
         return metrics.stats();
     }
@@ -189,9 +191,7 @@ private static class MetricsHolder {
 
         MetricsHolder() {
             this.metrics = new long[FeatureMetric.values().length];
-            for (int i = 0; i < this.metrics.length; i++) {
-                this.metrics[i] = 0;
-            }
+            Arrays.fill(this.metrics, 0);
         }
 
         void set(Set<FeatureMetric> metricSet) {
diff --git a/x-pack/plugin/fleet/build.gradle b/x-pack/plugin/fleet/build.gradle
index 71b023106fe6..9fef7a928bcf 100644
--- a/x-pack/plugin/fleet/build.gradle
+++ b/x-pack/plugin/fleet/build.gradle
@@ -7,7 +7,7 @@
 
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 esplugin {
   name 'x-pack-fleet'
@@ -21,6 +21,7 @@ dependencies {
   testImplementation(testArtifact(project(xpackModule('core'))))
   javaRestTestImplementation(project(path: xpackModule('core')))
   javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
+  compileOnly project(path: xpackModule('ilm'))
 }
 
 testClusters.configureEach {
diff --git a/x-pack/plugin/fleet/qa/rest/build.gradle b/x-pack/plugin/fleet/qa/rest/build.gradle
index 8a9736c710b3..7dfbfaafa7d3 100644
--- a/x-pack/plugin/fleet/qa/rest/build.gradle
+++ b/x-pack/plugin/fleet/qa/rest/build.gradle
@@ -1,6 +1,6 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 
 dependencies {
   yamlRestTestImplementation(testArtifact(project(xpackModule('core'))))
diff --git a/x-pack/plugin/fleet/src/internalClusterTest/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsActionIT.java b/x-pack/plugin/fleet/src/internalClusterTest/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsActionIT.java
index 989d4d10dc38..7b90e41584cf 100644
--- a/x-pack/plugin/fleet/src/internalClusterTest/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsActionIT.java
+++ b/x-pack/plugin/fleet/src/internalClusterTest/java/org/elasticsearch/xpack/fleet/action/GetGlobalCheckpointsActionIT.java
@@ -23,7 +23,9 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
 import org.elasticsearch.xpack.fleet.Fleet;
+import org.elasticsearch.xpack.ilm.IndexLifecycle;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -43,7 +45,7 @@ public class GetGlobalCheckpointsActionIT extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Stream.of(Fleet.class).collect(Collectors.toList());
+        return Stream.of(Fleet.class, LocalStateCompositeXPackPlugin.class, IndexLifecycle.class).collect(Collectors.toList());
     }
 
     public void testGetGlobalCheckpoints() throws Exception {
diff --git a/x-pack/plugin/fleet/src/internalClusterTest/java/org/elasticsearch/xpack/fleet/action/SearchUsageStatsIT.java b/x-pack/plugin/fleet/src/internalClusterTest/java/org/elasticsearch/xpack/fleet/action/SearchUsageStatsIT.java
index 0dc6a7a51929..0dbd745a5b0b 100644
--- a/x-pack/plugin/fleet/src/internalClusterTest/java/org/elasticsearch/xpack/fleet/action/SearchUsageStatsIT.java
+++ b/x-pack/plugin/fleet/src/internalClusterTest/java/org/elasticsearch/xpack/fleet/action/SearchUsageStatsIT.java
@@ -16,11 +16,14 @@
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
 import org.elasticsearch.xpack.fleet.Fleet;
+import org.elasticsearch.xpack.ilm.IndexLifecycle;
 
 import java.io.IOException;
 import java.util.Collection;
-import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 1)
 public class SearchUsageStatsIT extends ESIntegTestCase {
@@ -32,7 +35,7 @@ protected boolean addMockHttpTransport() {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return List.of(Fleet.class);
+        return Stream.of(Fleet.class, LocalStateCompositeXPackPlugin.class, IndexLifecycle.class).collect(Collectors.toList());
     }
 
     public void testSearchUsageStats() throws IOException {
diff --git a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java
index c40179f3a843..3d37ca7b7359 100644
--- a/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java
+++ b/x-pack/plugin/fleet/src/javaRestTest/java/org/elasticsearch/xpack/fleet/FleetSystemIndicesIT.java
@@ -9,6 +9,7 @@
 
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -17,10 +18,12 @@
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xcontent.XContentType;
 
+import java.util.List;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
 
 public class FleetSystemIndicesIT extends ESRestTestCase {
 
@@ -37,6 +40,22 @@ protected Settings restClientSettings() {
             .build();
     }
 
+    private void expectSystemIndexWarning(Request request, String indexName) {
+        String warningMsg = "index name ["
+            + indexName
+            + "] starts with a dot '.', in the next major version, "
+            + "index names starting with a dot are reserved for hidden indices and system indices";
+
+        List<String> expectedWarnings = List.of(warningMsg);
+
+        logger.info("expecting warnings: " + expectedWarnings.toString());
+        RequestOptions consumeSystemIndicesWarningsOptions = RequestOptions.DEFAULT.toBuilder()
+            .setWarningsHandler(warnings -> expectedWarnings.equals(warnings) == false)
+            .build();
+
+        request.setOptions(consumeSystemIndicesWarningsOptions);
+    }
+
     public void testSearchWithoutIndexCreatedIsAllowed() throws Exception {
         Request request = new Request("GET", ".fleet-agents/_search");
         request.setJsonEntity("{ \"query\": { \"match_all\": {} } }");
@@ -77,6 +96,33 @@ public void testCreationOfFleetActions() throws Exception {
         assertThat(responseBody, containsString("action_id"));
     }
 
+    public void testCreationOfFleetFiles() throws Exception {
+        Request request = new Request("PUT", ".fleet-files-agent-00001");
+        expectSystemIndexWarning(request, ".fleet-files-agent-00001");
+        Response response = client().performRequest(request);
+        assertEquals(200, response.getStatusLine().getStatusCode());
+
+        request = new Request("GET", ".fleet-files-agent-00001/_mapping");
+        response = client().performRequest(request);
+        String responseBody = EntityUtils.toString(response.getEntity());
+        assertThat(responseBody, not(containsString("xpack.fleet.template.version"))); // assert templating worked
+        assertThat(responseBody, containsString("action_id"));
+    }
+
+    public void testCreationOfFleetFileData() throws Exception {
+        Request request = new Request("PUT", ".fleet-file-data-agent-00001");
+        expectSystemIndexWarning(request, ".fleet-file-data-agent-00001");
+        Response response = client().performRequest(request);
+        assertEquals(200, response.getStatusLine().getStatusCode());
+
+        request = new Request("GET", ".fleet-file-data-agent-00001/_mapping");
+        response = client().performRequest(request);
+        String responseBody = EntityUtils.toString(response.getEntity());
+        assertThat(responseBody, not(containsString("xpack.fleet.template.version"))); // assert templating worked
+        assertThat(responseBody, containsString("data"));
+        assertThat(responseBody, containsString("bid"));
+    }
+
     public void testCreationOfFleetArtifacts() throws Exception {
         Request request = new Request("PUT", ".fleet-artifacts");
         Response response = client().performRequest(request);
@@ -165,7 +211,7 @@ public void testCreationOfFleetActionsResults() throws Exception {
     }
 
     @SuppressWarnings("unchecked")
-    public void verifyILMPolicyExists() throws Exception {
+    public void verifyActionsILMPolicyExists() throws Exception {
         assertBusy(() -> {
             Request request = new Request("GET", "_ilm/policy/.fleet-actions-results-ilm-policy");
             Response response = client().performRequest(request);
@@ -178,4 +224,34 @@ public void verifyILMPolicyExists() throws Exception {
             assertThat(policyMap.size(), equalTo(2));
         });
     }
+
+    @SuppressWarnings("unchecked")
+    public void verifyFilesILMPolicyExists() throws Exception {
+        assertBusy(() -> {
+            Request request = new Request("GET", "_ilm/policy/.fleet-files-ilm-policy");
+            Response response = client().performRequest(request);
+            assertEquals(200, response.getStatusLine().getStatusCode());
+            final String responseJson = EntityUtils.toString(response.getEntity());
+            Map<String, Object> responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), responseJson, false);
+            assertNotNull(responseMap.get(".fleet-files-ilm-policy"));
+            Map<String, Object> policyMap = (Map<String, Object>) responseMap.get(".fleet-files-ilm-policy");
+            assertNotNull(policyMap);
+            assertThat(policyMap.size(), equalTo(2));
+        });
+    }
+
+    @SuppressWarnings("unchecked")
+    public void verifyFileDataILMPolicyExists() throws Exception {
+        assertBusy(() -> {
+            Request request = new Request("GET", "_ilm/policy/.fleet-file-data-ilm-policy");
+            Response response = client().performRequest(request);
+            assertEquals(200, response.getStatusLine().getStatusCode());
+            final String responseJson = EntityUtils.toString(response.getEntity());
+            Map<String, Object> responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), responseJson, false);
+            assertNotNull(responseMap.get(".fleet-file-data-ilm-policy"));
+            Map<String, Object> policyMap = (Map<String, Object>) responseMap.get(".fleet-file-data-ilm-policy");
+            assertNotNull(policyMap);
+            assertThat(policyMap.size(), equalTo(2));
+        });
+    }
 }
diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java
index 737d9a86c301..9d7e76a85a99 100644
--- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java
+++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -95,7 +95,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver expressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         FleetTemplateRegistry registry = new FleetTemplateRegistry(
             environment.settings(),
diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/FleetTemplateRegistry.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/FleetTemplateRegistry.java
index 5b5b1e45aedc..398528ba05a4 100644
--- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/FleetTemplateRegistry.java
+++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/FleetTemplateRegistry.java
@@ -8,25 +8,43 @@
 package org.elasticsearch.xpack.fleet;
 
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
+import org.elasticsearch.xpack.core.template.IndexTemplateConfig;
 import org.elasticsearch.xpack.core.template.IndexTemplateRegistry;
 import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig;
 
 import java.util.List;
+import java.util.Map;
 
 public class FleetTemplateRegistry extends IndexTemplateRegistry {
 
+    public static final int INDEX_TEMPLATE_VERSION = 1;
+
+    public static final String TEMPLATE_VERSION_VARIABLE = "xpack.fleet.template.version";
+
     private static final List<LifecyclePolicy> LIFECYCLE_POLICIES = List.of(
         new LifecyclePolicyConfig(".fleet-actions-results-ilm-policy", "/fleet-actions-results-ilm-policy.json").load(
             LifecyclePolicyConfig.DEFAULT_X_CONTENT_REGISTRY
+        ),
+        new LifecyclePolicyConfig(".fleet-file-data-ilm-policy", "/fleet-file-data-ilm-policy.json").load(
+            LifecyclePolicyConfig.DEFAULT_X_CONTENT_REGISTRY
+        ),
+        new LifecyclePolicyConfig(".fleet-files-ilm-policy", "/fleet-files-ilm-policy.json").load(
+            LifecyclePolicyConfig.DEFAULT_X_CONTENT_REGISTRY
         )
     );
 
+    public static final Map<String, ComposableIndexTemplate> COMPOSABLE_INDEX_TEMPLATE_CONFIGS = parseComposableTemplates(
+        new IndexTemplateConfig(".fleet-files", "/fleet-files.json", INDEX_TEMPLATE_VERSION, TEMPLATE_VERSION_VARIABLE),
+        new IndexTemplateConfig(".fleet-file-data", "/fleet-file-data.json", INDEX_TEMPLATE_VERSION, TEMPLATE_VERSION_VARIABLE)
+    );
+
     public FleetTemplateRegistry(
         Settings nodeSettings,
         ClusterService clusterService,
@@ -46,4 +64,9 @@ protected String getOrigin() {
     protected List<LifecyclePolicy> getPolicyConfigs() {
         return LIFECYCLE_POLICIES;
     }
+
+    @Override
+    protected Map<String, ComposableIndexTemplate> getComposableTemplateConfigs() {
+        return COMPOSABLE_INDEX_TEMPLATE_CONFIGS;
+    }
 }
diff --git a/x-pack/plugin/graph/qa/with-security/build.gradle b/x-pack/plugin/graph/qa/with-security/build.gradle
index 75109373b400..6a4264477eaf 100644
--- a/x-pack/plugin/graph/qa/with-security/build.gradle
+++ b/x-pack/plugin/graph/qa/with-security/build.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 
 dependencies {
   yamlRestTestImplementation project(":x-pack:plugin:core")
diff --git a/x-pack/plugin/identity-provider/qa/idp-rest-tests/build.gradle b/x-pack/plugin/identity-provider/qa/idp-rest-tests/build.gradle
index 14e9b91edb1a..d44f4fa3ab3c 100644
--- a/x-pack/plugin/identity-provider/qa/idp-rest-tests/build.gradle
+++ b/x-pack/plugin/identity-provider/qa/idp-rest-tests/build.gradle
@@ -1,5 +1,5 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 dependencies {
   javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
diff --git a/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/idp/IdentityProviderAuthenticationIT.java b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/idp/IdentityProviderAuthenticationIT.java
index 2353a28b7ac9..b8565bc4ff89 100644
--- a/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/idp/IdentityProviderAuthenticationIT.java
+++ b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/idp/IdentityProviderAuthenticationIT.java
@@ -128,11 +128,11 @@ private SamlPrepareAuthenticationResponse generateSamlAuthnRequest(String realmN
     private String generateSamlResponse(String entityId, String acs, @Nullable Map<String, Object> authnState) throws Exception {
         final Request request = new Request("POST", "/_idp/saml/init");
         if (authnState != null && authnState.isEmpty() == false) {
-            request.setJsonEntity(formatted("""
+            request.setJsonEntity(Strings.format("""
                 {"entity_id":"%s", "acs":"%s","authn_state":%s}
                 """, entityId, acs, Strings.toString(JsonXContent.contentBuilder().map(authnState))));
         } else {
-            request.setJsonEntity(formatted("""
+            request.setJsonEntity(Strings.format("""
                 {"entity_id":"%s", "acs":"%s"}
                 """, entityId, acs));
         }
@@ -153,11 +153,11 @@ private void authenticateWithSamlResponse(String samlResponse, @Nullable String
         final String encodedResponse = Base64.getEncoder().encodeToString(samlResponse.getBytes(StandardCharsets.UTF_8));
         final Request request = new Request("POST", "/_security/saml/authenticate");
         if (Strings.hasText(id)) {
-            request.setJsonEntity(formatted("""
+            request.setJsonEntity(Strings.format("""
                 {"content":"%s", "realm":"%s", "ids":["%s"]}
                 """, encodedResponse, REALM_NAME, id));
         } else {
-            request.setJsonEntity(formatted("""
+            request.setJsonEntity(Strings.format("""
                 {"content":"%s", "realm":"%s"}
                 """, encodedResponse, REALM_NAME));
         }
diff --git a/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/idp/IdpRestTestCase.java b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/idp/IdpRestTestCase.java
index 6467256a7fcd..c05840a82738 100644
--- a/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/idp/IdpRestTestCase.java
+++ b/x-pack/plugin/identity-provider/qa/idp-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/idp/IdpRestTestCase.java
@@ -62,7 +62,7 @@ protected User createUser(String username, SecureString password, String role) t
         );
         final String endpoint = "/_security/user/" + username;
         final Request request = new Request(HttpPut.METHOD_NAME, endpoint);
-        final String body = formatted("""
+        final String body = Strings.format("""
             {
                 "username": "%s",
                 "full_name": "%s",
@@ -138,7 +138,7 @@ protected void createApplicationPrivileges(String applicationName, Map<String, C
     protected void setUserPassword(String username, SecureString password) throws IOException {
         final String endpoint = "/_security/user/" + username + "/_password";
         final Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        final String body = formatted("""
+        final String body = Strings.format("""
             {
                 "password": "%s"
             }
diff --git a/x-pack/plugin/identity-provider/src/internalClusterTest/java/org/elasticsearch/xpack/idp/action/SamlIdentityProviderTests.java b/x-pack/plugin/identity-provider/src/internalClusterTest/java/org/elasticsearch/xpack/idp/action/SamlIdentityProviderTests.java
index eeb0ab6ff9e2..551b53948c1e 100644
--- a/x-pack/plugin/identity-provider/src/internalClusterTest/java/org/elasticsearch/xpack/idp/action/SamlIdentityProviderTests.java
+++ b/x-pack/plugin/identity-provider/src/internalClusterTest/java/org/elasticsearch/xpack/idp/action/SamlIdentityProviderTests.java
@@ -97,7 +97,7 @@ public void testIdpInitiatedSsoWithMultipleRoles() throws Exception {
             )
             .build();
         // This role has "editor" on the deployment itself, and "viewer" for the organization that owns the deployment
-        createRole(roleName, formatted("""
+        createRole(roleName, Strings.format("""
             {
               "cluster": [ "manage_own_api_key" ],
               "applications": [
@@ -205,7 +205,7 @@ public void testSpInitiatedSso() throws Exception {
         );
         XContentBuilder authnStateBuilder = jsonBuilder();
         authnStateBuilder.map(authnState);
-        initRequest.setJsonEntity(formatted("""
+        initRequest.setJsonEntity(Strings.format("""
             {"entity_id":"%s","acs":"%s","authn_state":%s}
             """, entityId, serviceProvider.get("acs"), Strings.toString(authnStateBuilder)));
         Response initResponse = getRestClient().performRequest(initRequest);
@@ -271,7 +271,7 @@ public void testSpInitiatedSsoFailsForUserWithNoAccess() throws Exception {
         );
         XContentBuilder authnStateBuilder = jsonBuilder();
         authnStateBuilder.map(authnState);
-        initRequest.setJsonEntity(formatted("""
+        initRequest.setJsonEntity(Strings.format("""
             {"entity_id":"%s", "acs":"%s","authn_state":%s}
             """, entityId, acsUrl, Strings.toString(authnStateBuilder)));
         Response initResponse = getRestClient().performRequest(initRequest);
@@ -468,7 +468,7 @@ private void createRole(String roleName, String roleBody, RequestOptions options
 
     private void createUser(String userName, SecureString password, String roleName, RequestOptions options) throws IOException {
         final Request req = new Request("PUT", "/_security/user/" + userName);
-        final String body = formatted("""
+        final String body = Strings.format("""
             {
               "username": "%s",
               "full_name": "Test User (%s)",
@@ -557,7 +557,7 @@ private byte[] sign(byte[] content, String algo, X509Credential credential) thro
     }
 
     private void assertContainsAttributeWithValues(String message, String attribute, String... values) {
-        final String startAttribute = formatted("""
+        final String startAttribute = Strings.format("""
             <saml2:Attribute FriendlyName="%s" Name="https://saml.elasticsearch.org/attributes/%s" \
             NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:uri">""", attribute, attribute);
         assertThat(message, containsString(startAttribute));
diff --git a/x-pack/plugin/identity-provider/src/internalClusterTest/java/org/elasticsearch/xpack/idp/saml/test/IdentityProviderIntegTestCase.java b/x-pack/plugin/identity-provider/src/internalClusterTest/java/org/elasticsearch/xpack/idp/saml/test/IdentityProviderIntegTestCase.java
index 434715c13605..f02ccae7b8f2 100644
--- a/x-pack/plugin/identity-provider/src/internalClusterTest/java/org/elasticsearch/xpack/idp/saml/test/IdentityProviderIntegTestCase.java
+++ b/x-pack/plugin/identity-provider/src/internalClusterTest/java/org/elasticsearch/xpack/idp/saml/test/IdentityProviderIntegTestCase.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.transport.netty4.Netty4Plugin;
@@ -170,7 +171,7 @@ private String configRoles() {
         // IDP end user doesn't need any privileges on the security cluster
         // Could switch to grant apikey for user and call this as console_user
         // Console user should be able to call all IDP related endpoints and register application privileges
-        return formatted("""
+        return Strings.format("""
             %s:
               cluster: [ ALL ]
               indices:
@@ -261,7 +262,7 @@ private static String writeFile(Path folder, String name, byte[] content) {
                 Files.move(tempFile, path, REPLACE_EXISTING);
             }
         } catch (final IOException e) {
-            throw new UncheckedIOException(formatted("could not write file [%s]", path.toAbsolutePath()), e);
+            throw new UncheckedIOException(Strings.format("could not write file [%s]", path.toAbsolutePath()), e);
         } finally {
             // we are ignoring exceptions here, so we do not need handle whether or not tempFile was initialized nor if the file exists
             IOUtils.deleteFilesIgnoringExceptions(tempFile);
diff --git a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/IdentityProviderPlugin.java b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/IdentityProviderPlugin.java
index ad78de0ad597..28d76d3a2562 100644
--- a/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/IdentityProviderPlugin.java
+++ b/x-pack/plugin/identity-provider/src/main/java/org/elasticsearch/xpack/idp/IdentityProviderPlugin.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -98,7 +98,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         settings = environment.settings();
         enabled = ENABLED_SETTING.get(settings);
diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnActionTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnActionTests.java
index 516f3e0422c1..dc8010b9b992 100644
--- a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnActionTests.java
+++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/TransportSamlInitiateSingleSignOnActionTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.tasks.Task;
@@ -225,7 +226,7 @@ private TransportSamlInitiateSingleSignOnAction setupTransportAction(boolean wit
     }
 
     private void assertContainsAttributeWithValue(String message, String attribute, String value) {
-        assertThat(message, containsString(formatted("""
+        assertThat(message, containsString(Strings.format("""
             <saml2:Attribute FriendlyName="%s" Name="https://saml.elasticsearch.org/attributes/%s" \
             NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:uri"><saml2:AttributeValue \
             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="xsd:string">%s</saml2:AttributeValue>\
diff --git a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ilm/CCRIndexLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ilm/CCRIndexLifecycleIT.java
index 4e4c38d82296..0f55d296fce5 100644
--- a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ilm/CCRIndexLifecycleIT.java
+++ b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ilm/CCRIndexLifecycleIT.java
@@ -212,7 +212,7 @@ public void testCcrAndIlmWithRollover() throws Exception {
             try (RestClient leaderClient = buildLeaderClient()) {
                 // Create an index on the leader using the template set up above
                 Request createIndexRequest = new Request("PUT", "/" + indexName);
-                createIndexRequest.setJsonEntity(formatted("""
+                createIndexRequest.setJsonEntity(Strings.format("""
                     {
                       "mappings": {
                         "properties": {
@@ -558,7 +558,7 @@ private void configureRemoteClusters(String name, String leaderRemoteClusterSeed
             "{\"persistent\": {\"cluster.remote."
                 + name
                 + ".seeds\": "
-                + (leaderRemoteClusterSeed != null ? formatted("\"%s\"", leaderRemoteClusterSeed) : null)
+                + (leaderRemoteClusterSeed != null ? Strings.format("\"%s\"", leaderRemoteClusterSeed) : null)
                 + "}}"
         );
         assertThat(client().performRequest(request).getStatusLine().getStatusCode(), equalTo(200));
diff --git a/x-pack/plugin/ilm/qa/multi-node/build.gradle b/x-pack/plugin/ilm/qa/multi-node/build.gradle
index e60c772e5a59..523bf5d04ae4 100644
--- a/x-pack/plugin/ilm/qa/multi-node/build.gradle
+++ b/x-pack/plugin/ilm/qa/multi-node/build.gradle
@@ -1,6 +1,6 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 dependencies {
   javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java
index f5433bff82ed..6ff4542825a7 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java
@@ -182,7 +182,7 @@ public void testMigrateToDataTiersAction() throws Exception {
         updateIndexSettings(indexWithDataWarmRouting, Settings.builder().putNull(DataTier.TIER_PREFERENCE));
 
         Request migrateRequest = new Request("POST", "_ilm/migrate_to_data_tiers");
-        migrateRequest.setJsonEntity(formatted("""
+        migrateRequest.setJsonEntity(Strings.format("""
             {"legacy_template_to_delete": "%s", "node_attribute": "data"}
             """, templateName));
         Response migrateDeploymentResponse = client().performRequest(migrateRequest);
@@ -473,7 +473,7 @@ private String getCachedPhaseDefAsMap(Response clusterMetadataResponse, String i
 
     private void createLegacyTemplate(String templateName) throws IOException {
         String indexPrefix = randomAlphaOfLengthBetween(5, 15).toLowerCase(Locale.ROOT);
-        final StringEntity template = new StringEntity(formatted("""
+        final StringEntity template = new StringEntity(Strings.format("""
             {
               "index_patterns": "%s*",
               "settings": {
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ChangePolicyForIndexIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ChangePolicyForIndexIT.java
index 0b7f203e6f33..7f75b010346a 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ChangePolicyForIndexIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/ChangePolicyForIndexIT.java
@@ -129,7 +129,7 @@ public void testChangePolicyForIndex() throws Exception {
             .put(LifecycleSettings.LIFECYCLE_NAME, "policy_1")
             .build();
         Request createIndexRequest = new Request("PUT", "/" + indexName);
-        createIndexRequest.setJsonEntity(formatted("""
+        createIndexRequest.setJsonEntity(Strings.format("""
             {
               "settings": %s,
               "aliases": {
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
index f92fab0c9643..975025f2045b 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
@@ -552,7 +552,7 @@ public void testSetNullPriority() throws Exception {
     @SuppressWarnings("unchecked")
     public void testNonexistentPolicy() throws Exception {
         String indexPrefix = randomAlphaOfLengthBetween(5, 15).toLowerCase(Locale.ROOT);
-        final StringEntity template = new StringEntity(formatted("""
+        final StringEntity template = new StringEntity(Strings.format("""
             {
               "index_patterns": "%s*",
               "settings": {
@@ -719,7 +719,7 @@ public void testRemoveAndReaddPolicy() throws Exception {
 
         // Add the policy again
         Request addPolicyRequest = new Request("PUT", "/" + originalIndex + "/_settings");
-        addPolicyRequest.setJsonEntity(formatted("""
+        addPolicyRequest.setJsonEntity(Strings.format("""
             {
               "settings": {
                 "index.lifecycle.name": "%s",
@@ -780,7 +780,7 @@ public void testWaitForActiveShardsStep() throws Exception {
         // update policy on index
         updatePolicy(client(), originalIndex, policy);
         Request createIndexTemplate = new Request("PUT", "_template/rolling_indexes");
-        createIndexTemplate.setJsonEntity(formatted("""
+        createIndexTemplate.setJsonEntity(Strings.format("""
             {"index_patterns": ["%s-*"],
               "settings": {
                 "number_of_shards": 1,
@@ -806,7 +806,7 @@ public void testWaitForActiveShardsStep() throws Exception {
     public void testHistoryIsWrittenWithSuccess() throws Exception {
         createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, null, null, 1L, null, null, null, null, null, null));
         Request createIndexTemplate = new Request("PUT", "_template/rolling_indexes");
-        createIndexTemplate.setJsonEntity(formatted("""
+        createIndexTemplate.setJsonEntity(Strings.format("""
             {
               "index_patterns": [ "%s-*" ],
               "settings": {
@@ -915,7 +915,7 @@ public void testRefreshablePhaseJson() throws Exception {
 
         createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, null, null, 100L, null, null, null, null, null, null));
         Request createIndexTemplate = new Request("PUT", "_template/rolling_indexes");
-        createIndexTemplate.setJsonEntity(formatted("""
+        createIndexTemplate.setJsonEntity(Strings.format("""
             {
               "index_patterns": ["%s-*"],
               "settings": {
@@ -1134,47 +1134,44 @@ private void assertHistoryIsPresent(
             stepName
         );
         final Request historySearchRequest = new Request("GET", "ilm-history*/_search?expand_wildcards=all");
-        historySearchRequest.setJsonEntity(
-            formatted(
-                """
+        Object[] args = new Object[] {
+            policyName,
+            success,
+            indexName,
+            stepName,
+            phase == null ? "" : Strings.format(",{\"term\": {\"state.phase\": \"%s\"}}", phase),
+            action == null ? "" : ",{\"term\": {\"state.action\": \"" + action + "\"}}" };
+        historySearchRequest.setJsonEntity(Strings.format("""
+            {
+              "query": {
+                "bool": {
+                  "must": [
                     {
-                      "query": {
-                        "bool": {
-                          "must": [
-                            {
-                              "term": {
-                                "policy": "%s"
-                              }
-                            },
-                            {
-                              "term": {
-                                "success": %s
-                              }
-                            },
-                            {
-                              "term": {
-                                "index": "%s"
-                              }
-                            },
-                            {
-                              "term": {
-                                "state.step": "%s"
-                              }
-                            }
-                            %s
-                            %s
-                          ]
-                        }
+                      "term": {
+                        "policy": "%s"
                       }
-                    }""",
-                policyName,
-                success,
-                indexName,
-                stepName,
-                phase == null ? "" : formatted(",{\"term\": {\"state.phase\": \"%s\"}}", phase),
-                action == null ? "" : ",{\"term\": {\"state.action\": \"" + action + "\"}}"
-            )
-        );
+                    },
+                    {
+                      "term": {
+                        "success": %s
+                      }
+                    },
+                    {
+                      "term": {
+                        "index": "%s"
+                      }
+                    },
+                    {
+                      "term": {
+                        "state.step": "%s"
+                      }
+                    }
+                    %s
+                    %s
+                  ]
+                }
+              }
+            }""", args));
         Response historyResponse;
         try {
             historyResponse = client().performRequest(historySearchRequest);
@@ -1188,7 +1185,7 @@ private void assertHistoryIsPresent(
             // For a failure, print out whatever history we *do* have for the index
             if (hits == 0) {
                 final Request allResults = new Request("GET", "ilm-history*/_search");
-                allResults.setJsonEntity(formatted("""
+                allResults.setJsonEntity(Strings.format("""
                     {
                       "query": {
                         "bool": {
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java
index 306a7189470b..55cf44ddf85d 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java
@@ -199,7 +199,7 @@ public void testRollupIndexInTheHotPhaseAfterRollover() throws Exception {
 
         // and a template
         Request createTemplateRequest = new Request("PUT", "_template/" + index);
-        createTemplateRequest.setJsonEntity(formatted("""
+        createTemplateRequest.setJsonEntity(Strings.format("""
             {
               "index_patterns": ["%s-*"],
               "settings": {
@@ -249,7 +249,7 @@ public void testTsdbDataStreams() throws Exception {
 
         // Create a template
         Request createIndexTemplateRequest = new Request("POST", "/_index_template/" + dataStream);
-        createIndexTemplateRequest.setJsonEntity(formatted(TEMPLATE, dataStream, policy));
+        createIndexTemplateRequest.setJsonEntity(Strings.format(TEMPLATE, dataStream, policy));
         assertOK(client().performRequest(createIndexTemplateRequest));
 
         String now = DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(Instant.now());
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RolloverActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RolloverActionIT.java
index c813ed4670a1..b12df4b43faa 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RolloverActionIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/RolloverActionIT.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.CheckedRunnable;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction;
 import org.elasticsearch.test.rest.ESRestTestCase;
@@ -98,7 +99,7 @@ public void testRolloverActionWithIndexingComplete() throws Exception {
         );
 
         Request updateSettingsRequest = new Request("PUT", "/" + originalIndex + "/_settings");
-        updateSettingsRequest.setJsonEntity(formatted("""
+        updateSettingsRequest.setJsonEntity(Strings.format("""
             {
               "settings": {
                 "%s": true
@@ -106,7 +107,7 @@ public void testRolloverActionWithIndexingComplete() throws Exception {
             }""", LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE));
         client().performRequest(updateSettingsRequest);
         Request updateAliasRequest = new Request("POST", "/_aliases");
-        updateAliasRequest.setJsonEntity(formatted("""
+        updateAliasRequest.setJsonEntity(Strings.format("""
             {
               "actions": [
                 {
@@ -347,7 +348,7 @@ public void testILMRolloverOnManuallyRolledIndex() throws Exception {
         // Set up a policy with rollover
         createNewSingletonPolicy(client(), policy, "hot", new RolloverAction(null, null, null, 2L, null, null, null, null, null, null));
         Request createIndexTemplate = new Request("PUT", "_template/rolling_indexes");
-        createIndexTemplate.setJsonEntity(formatted("""
+        createIndexTemplate.setJsonEntity(Strings.format("""
             {
               "index_patterns": ["%s-*"],
               "settings": {
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java
index 630157989484..cbacffef5567 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java
@@ -83,11 +83,6 @@ public void refreshIndex() {
         );
     }
 
-    @Override
-    protected boolean waitForAllSnapshotsWiped() {
-        return true;
-    }
-
     public void testSearchableSnapshotAction() throws Exception {
         createSnapshotRepo(client(), snapshotRepo, randomBoolean());
         createNewSingletonPolicy(client(), policy, "cold", new SearchableSnapshotAction(snapshotRepo, true));
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ShrinkActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ShrinkActionIT.java
index bf3a81e5eccf..0d9c135bd5a6 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ShrinkActionIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/ShrinkActionIT.java
@@ -219,7 +219,7 @@ public void testShrinkActionInTheHotPhase() throws Exception {
 
         // and a template
         Request createTemplateRequest = new Request("PUT", "_template/" + index);
-        createTemplateRequest.setJsonEntity(formatted("""
+        createTemplateRequest.setJsonEntity(Strings.format("""
             {
               "index_patterns": ["%s-*"],
               "settings": {
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java
index 02bdf446e6b3..242558c5e79a 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java
@@ -67,11 +67,6 @@
 public class SnapshotLifecycleRestIT extends ESRestTestCase {
     private static final String NEVER_EXECUTE_CRON_SCHEDULE = "* * * 31 FEB ? *";
 
-    @Override
-    protected boolean waitForAllSnapshotsWiped() {
-        return true;
-    }
-
     // as we are testing the SLM history entries we'll preserve the "slm-history-ilm-policy" policy as it'll be associated with the
     // .slm-history-* indices and we won't be able to delete it when we wipe out the cluster
     @Override
@@ -365,9 +360,7 @@ public void testStartStopStatus() throws Exception {
 
             assertBusy(() -> {
                 try {
-                    Map<String, List<Map<?, ?>>> snaps = wipeSnapshots();
-                    logger.info("--> checking for wiped snapshots: {}", snaps);
-                    assertThat(snaps.size(), equalTo(0));
+                    wipeSnapshots();
                 } catch (ResponseException e) {
                     logger.error("got exception wiping snapshots", e);
                     fail("got exception: " + EntityUtils.toString(e.getResponse().getEntity()));
@@ -598,6 +591,74 @@ public void testSLMXpackUsage() throws Exception {
         assertNotNull(slm.get("policy_stats"));
     }
 
+    public void testSnapshotRetentionWithMissingRepo() throws Exception {
+        // Create two snapshot repositories
+        String repo = "test-repo";
+        initializeRepo(repo);
+        String missingRepo = "missing-repo";
+        initializeRepo(missingRepo);
+
+        // Create a policy per repository
+        final String indexName = "test";
+        final String policyName = "policy-1";
+        createSnapshotPolicy(
+            policyName,
+            "snap",
+            NEVER_EXECUTE_CRON_SCHEDULE,
+            repo,
+            indexName,
+            true,
+            new SnapshotRetentionConfiguration(TimeValue.timeValueMillis(1), null, null)
+        );
+        final String policyWithMissingRepo = "policy-2";
+        createSnapshotPolicy(
+            policyWithMissingRepo,
+            "snap",
+            NEVER_EXECUTE_CRON_SCHEDULE,
+            missingRepo,
+            indexName,
+            true,
+            new SnapshotRetentionConfiguration(TimeValue.timeValueMillis(1), null, null)
+        );
+
+        // Delete the repo of one of the policies
+        deleteRepository(missingRepo);
+
+        // Manually create a snapshot based on the "correct" policy
+        final String snapshotName = executePolicy(policyName);
+
+        // Check that the executed snapshot is created
+        assertBusy(() -> {
+            try {
+                Response response = client().performRequest(new Request("GET", "/_snapshot/" + repo + "/" + snapshotName));
+                Map<String, Object> snapshotResponseMap;
+                try (InputStream is = response.getEntity().getContent()) {
+                    snapshotResponseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true);
+                }
+                assertThat(snapshotResponseMap.size(), greaterThan(0));
+                final Map<String, Object> metadata = extractMetadata(snapshotResponseMap, snapshotName);
+                assertNotNull(metadata);
+                assertThat(metadata.get("policy"), equalTo(policyName));
+                assertHistoryIsPresent(policyName, true, repo, CREATE_OPERATION);
+            } catch (ResponseException e) {
+                fail("expected snapshot to exist but it does not: " + EntityUtils.toString(e.getResponse().getEntity()));
+            }
+        }, 60, TimeUnit.SECONDS);
+
+        execute_retention(client());
+
+        // Check that the snapshot created by the policy has been removed by retention
+        assertBusy(() -> {
+            try {
+                Response response = client().performRequest(new Request("GET", "/_snapshot/" + repo + "/" + snapshotName));
+                assertThat(EntityUtils.toString(response.getEntity()), containsString("snapshot_missing_exception"));
+            } catch (ResponseException e) {
+                assertThat(EntityUtils.toString(e.getResponse().getEntity()), containsString("snapshot_missing_exception"));
+            }
+            assertHistoryIsPresent(policyName, true, repo, DELETE_OPERATION);
+        }, 60, TimeUnit.SECONDS);
+    }
+
     public Map<String, Object> getLocation(String path) {
         try {
             Response executeRepsonse = client().performRequest(new Request("GET", path));
@@ -665,7 +726,7 @@ private Map<String, Object> getSLMStats() {
     @SuppressWarnings("unchecked")
     private void assertHistoryIsPresent(String policyName, boolean success, String repository, String operation) throws IOException {
         final Request historySearchRequest = new Request("GET", ".slm-history*/_search");
-        historySearchRequest.setJsonEntity(formatted("""
+        historySearchRequest.setJsonEntity(Strings.format("""
             {
               "query": {
                 "bool": {
@@ -834,6 +895,11 @@ private static void index(RestClient client, String index, String id, Object...
         assertOK(client.performRequest(request));
     }
 
+    private static void execute_retention(RestClient client) throws IOException {
+        final Request request = new Request("POST", "/_slm/_execute_retention");
+        assertOK(client.performRequest(request));
+    }
+
     @SuppressWarnings("unchecked")
     private static Map<String, Object> policyStatsAsMap(Map<String, Object> stats) {
         return ((List<Map<String, Object>>) stats.get(SnapshotLifecycleStats.POLICY_STATS.getPreferredName())).stream()
diff --git a/x-pack/plugin/ilm/qa/rest/build.gradle b/x-pack/plugin/ilm/qa/rest/build.gradle
index 92b5b4e77c20..a17c130d4bbb 100644
--- a/x-pack/plugin/ilm/qa/rest/build.gradle
+++ b/x-pack/plugin/ilm/qa/rest/build.gradle
@@ -1,5 +1,5 @@
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 apply plugin: 'elasticsearch.authenticated-testclusters'
 
 dependencies {
diff --git a/x-pack/plugin/ilm/qa/with-security/build.gradle b/x-pack/plugin/ilm/qa/with-security/build.gradle
index 27a7da011c7e..7aaa43f14fe3 100644
--- a/x-pack/plugin/ilm/qa/with-security/build.gradle
+++ b/x-pack/plugin/ilm/qa/with-security/build.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.authenticated-testclusters'
 
 dependencies {
diff --git a/x-pack/plugin/ilm/qa/with-security/src/javaRestTest/java/org/elasticsearch/xpack/security/PermissionsIT.java b/x-pack/plugin/ilm/qa/with-security/src/javaRestTest/java/org/elasticsearch/xpack/security/PermissionsIT.java
index e14a927b007d..235379f5ac6d 100644
--- a/x-pack/plugin/ilm/qa/with-security/src/javaRestTest/java/org/elasticsearch/xpack/security/PermissionsIT.java
+++ b/x-pack/plugin/ilm/qa/with-security/src/javaRestTest/java/org/elasticsearch/xpack/security/PermissionsIT.java
@@ -339,7 +339,7 @@ private void createNewSingletonPolicy(RestClient client, String policy, String p
 
     private void createIndexAsAdmin(String name, Settings settings, String mapping) throws IOException {
         Request request = new Request("PUT", "/" + name);
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {
              "settings": %s, "mappings" : {%s}
             }""", Strings.toString(settings), mapping));
@@ -354,7 +354,7 @@ private void createIndexAsAdmin(String name, String alias, boolean isWriteIndex)
 
     private void createIndexTemplate(String name, String pattern, String alias, String policy) throws IOException {
         Request request = new Request("PUT", "/_template/" + name);
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {
               "index_patterns": [
                 "%s"
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
index a35eacfcc1d6..aed1a3b253e1 100644
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
+++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java
@@ -478,12 +478,11 @@ public void testCreatePolicyWhenStopped() throws Exception {
         final String node1 = getLocalNodeId(server_1);
 
         assertAcked(client().execute(StopILMAction.INSTANCE, new StopILMRequest()).get());
-        assertBusy(
-            () -> assertThat(
-                client().execute(GetStatusAction.INSTANCE, new GetStatusAction.Request()).get().getMode(),
-                equalTo(OperationMode.STOPPED)
-            )
-        );
+        assertBusy(() -> {
+            OperationMode mode = client().execute(GetStatusAction.INSTANCE, new GetStatusAction.Request()).get().getMode();
+            logger.info("--> waiting for STOPPED, currently: {}", mode);
+            assertThat(mode, equalTo(OperationMode.STOPPED));
+        });
 
         logger.info("Creating lifecycle [test_lifecycle]");
         PutLifecycleAction.Request putLifecycleRequest = new PutLifecycleAction.Request(lifecyclePolicy);
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/UpdateSettingsStepTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/UpdateSettingsStepTests.java
index 94c2ff1d3cf4..4a8b9e239a66 100644
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/UpdateSettingsStepTests.java
+++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/UpdateSettingsStepTests.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -74,7 +74,7 @@ public Collection<Object> createComponents(
             IndexNameExpressionResolver expressionResolver,
             Supplier<RepositoriesService> repositoriesServiceSupplier,
             Tracer tracer,
-            AllocationDeciders allocationDeciders
+            AllocationService allocationService
         ) {
             return List.of(service);
         }
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMFileSettingsIT.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMFileSettingsIT.java
index b8646f4ffbbf..086109e5fdff 100644
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMFileSettingsIT.java
+++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMFileSettingsIT.java
@@ -148,7 +148,6 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     @Override
     protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
         return Settings.builder()
-            .put(super.nodeSettings(nodeOrdinal, otherSettings))
             .put(LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED, false)
             .put(LifecycleSettings.SLM_HISTORY_INDEX_ENABLED, false)
             .build();
diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java
index 9dc38cccedfb..fa312cdac8e1 100644
--- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java
+++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SnapshotLifecycleInitialisationTests.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.ilm.OperationMode;
-import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicy;
 import org.elasticsearch.xpack.core.slm.SnapshotRetentionConfiguration;
 import org.elasticsearch.xpack.core.slm.action.PutSnapshotLifecycleAction;
@@ -31,6 +30,7 @@
 import java.util.Collections;
 import java.util.concurrent.TimeUnit;
 
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentSLMMode;
 import static org.hamcrest.core.Is.is;
 
 public class SnapshotLifecycleInitialisationTests extends ESSingleNodeTestCase {
@@ -82,7 +82,6 @@ public void testSLMIsInRunningModeWhenILMIsDisabled() throws Exception {
         ).get(10, TimeUnit.SECONDS);
 
         ClusterState state = getInstanceFromNode(ClusterService.class).state();
-        SnapshotLifecycleMetadata snapMeta = state.metadata().custom(SnapshotLifecycleMetadata.TYPE);
-        assertThat(snapMeta.getOperationMode(), is(OperationMode.RUNNING));
+        assertThat(currentSLMMode(state), is(OperationMode.RUNNING));
     }
 }
diff --git a/x-pack/plugin/ilm/src/main/java/module-info.java b/x-pack/plugin/ilm/src/main/java/module-info.java
index 7909ffc280e0..80ab9c5386f4 100644
--- a/x-pack/plugin/ilm/src/main/java/module-info.java
+++ b/x-pack/plugin/ilm/src/main/java/module-info.java
@@ -13,7 +13,7 @@
     requires org.apache.logging.log4j;
 
     exports org.elasticsearch.xpack.ilm.action to org.elasticsearch.server;
-    exports org.elasticsearch.xpack.ilm to org.elasticsearch.server;
+    exports org.elasticsearch.xpack.ilm;
     exports org.elasticsearch.xpack.slm.action to org.elasticsearch.server;
     exports org.elasticsearch.xpack.slm to org.elasticsearch.server;
 
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java
index ed29727a9ff1..ff9659235f2d 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java
@@ -51,6 +51,7 @@
 import static org.elasticsearch.cluster.metadata.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
 import static org.elasticsearch.cluster.routing.allocation.DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE;
 import static org.elasticsearch.cluster.routing.allocation.DataTier.TIER_PREFERENCE;
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentILMMode;
 import static org.elasticsearch.xpack.core.ilm.OperationMode.STOPPED;
 import static org.elasticsearch.xpack.core.ilm.PhaseCacheManagement.updateIndicesForPolicy;
 import static org.elasticsearch.xpack.ilm.IndexLifecycleTransition.moveStateToNextActionAndUpdateCachedPhase;
@@ -182,9 +183,9 @@ public static Tuple<ClusterState, MigratedEntities> migrateToDataTiersRouting(
     ) {
         if (dryRun == false) {
             IndexLifecycleMetadata currentMetadata = currentState.metadata().custom(IndexLifecycleMetadata.TYPE);
-            if (currentMetadata != null && currentMetadata.getOperationMode() != STOPPED) {
+            if (currentMetadata != null && currentILMMode(currentState) != STOPPED) {
                 throw new IllegalStateException(
-                    "stop ILM before migrating to data tiers, current state is [" + currentMetadata.getOperationMode() + "]"
+                    "stop ILM before migrating to data tiers, current state is [" + currentILMMode(currentState) + "]"
                 );
             }
         }
@@ -274,7 +275,7 @@ static List<String> migrateIlmPolicies(
         }
 
         if (migratedPolicies.size() > 0) {
-            IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, currentLifecycleMetadata.getOperationMode());
+            IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, currentILMMode(currentState));
             mb.putCustom(IndexLifecycleMetadata.TYPE, newMetadata);
         }
         return migratedPolicies;
@@ -692,6 +693,9 @@ static List<String> migrateComposableTemplates(Metadata.Builder mb, ClusterState
                     migratedComposableTemplateBuilder.metadata(composableTemplate.metadata());
                     migratedComposableTemplateBuilder.dataStreamTemplate(composableTemplate.getDataStreamTemplate());
                     migratedComposableTemplateBuilder.allowAutoCreate(composableTemplate.getAllowAutoCreate());
+                    migratedComposableTemplateBuilder.ignoreMissingComponentTemplates(
+                        composableTemplate.getIgnoreMissingComponentTemplates()
+                    );
 
                     mb.put(templateEntry.getKey(), migratedComposableTemplateBuilder.build());
                     migratedComposableTemplates.add(templateEntry.getKey());
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IlmHealthIndicatorService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IlmHealthIndicatorService.java
index b6d8a3c22b19..e9e48b0099fc 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IlmHealthIndicatorService.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IlmHealthIndicatorService.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.ilm;
 
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.health.Diagnosis;
 import org.elasticsearch.health.HealthIndicatorDetails;
@@ -25,6 +26,7 @@
 
 import static org.elasticsearch.health.HealthStatus.GREEN;
 import static org.elasticsearch.health.HealthStatus.YELLOW;
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentILMMode;
 
 /**
  * This indicator reports health for index lifecycle management component.
@@ -64,17 +66,19 @@ public String name() {
     }
 
     @Override
-    public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
-        var ilmMetadata = clusterService.state().metadata().custom(IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata.EMPTY);
+    public HealthIndicatorResult calculate(boolean verbose, int maxAffectedResourcesCount, HealthInfo healthInfo) {
+        final ClusterState currentState = clusterService.state();
+        var ilmMetadata = currentState.metadata().custom(IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata.EMPTY);
+        final OperationMode currentMode = currentILMMode(currentState);
         if (ilmMetadata.getPolicyMetadatas().isEmpty()) {
             return createIndicator(
                 GREEN,
                 "No Index Lifecycle Management policies configured",
-                createDetails(verbose, ilmMetadata),
+                createDetails(verbose, ilmMetadata, currentMode),
                 Collections.emptyList(),
                 Collections.emptyList()
             );
-        } else if (ilmMetadata.getOperationMode() != OperationMode.RUNNING) {
+        } else if (currentMode != OperationMode.RUNNING) {
             List<HealthIndicatorImpact> impacts = Collections.singletonList(
                 new HealthIndicatorImpact(
                     NAME,
@@ -88,7 +92,7 @@ public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
             return createIndicator(
                 YELLOW,
                 "Index Lifecycle Management is not running",
-                createDetails(verbose, ilmMetadata),
+                createDetails(verbose, ilmMetadata, currentMode),
                 impacts,
                 List.of(ILM_NOT_RUNNING)
             );
@@ -96,18 +100,16 @@ public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
             return createIndicator(
                 GREEN,
                 "Index Lifecycle Management is running",
-                createDetails(verbose, ilmMetadata),
+                createDetails(verbose, ilmMetadata, currentMode),
                 Collections.emptyList(),
                 Collections.emptyList()
             );
         }
     }
 
-    private static HealthIndicatorDetails createDetails(boolean verbose, IndexLifecycleMetadata metadata) {
+    private static HealthIndicatorDetails createDetails(boolean verbose, IndexLifecycleMetadata metadata, OperationMode mode) {
         if (verbose) {
-            return new SimpleHealthIndicatorDetails(
-                Map.of("ilm_status", metadata.getOperationMode(), "policies", metadata.getPolicies().size())
-            );
+            return new SimpleHealthIndicatorDetails(Map.of("ilm_status", mode, "policies", metadata.getPolicies().size()));
         } else {
             return HealthIndicatorDetails.EMPTY;
         }
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java
index aaf4b4da61ad..fb4ffba26db1 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java
@@ -15,7 +15,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
@@ -54,6 +54,7 @@
 import org.elasticsearch.xpack.core.ilm.FreezeAction;
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.ilm.LifecycleAction;
+import org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata;
 import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
 import org.elasticsearch.xpack.core.ilm.LifecycleType;
 import org.elasticsearch.xpack.core.ilm.MigrateAction;
@@ -213,7 +214,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver expressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         final List<Object> components = new ArrayList<>();
         ILMHistoryTemplateRegistry ilmTemplateRegistry = new ILMHistoryTemplateRegistry(
@@ -304,6 +305,11 @@ private static List<NamedXContentRegistry.Entry> xContentEntries() {
                 new ParseField(SnapshotLifecycleMetadata.TYPE),
                 parser -> SnapshotLifecycleMetadata.PARSER.parse(parser, null)
             ),
+            new NamedXContentRegistry.Entry(
+                Metadata.Custom.class,
+                new ParseField(LifecycleOperationMetadata.TYPE),
+                parser -> LifecycleOperationMetadata.PARSER.parse(parser, null)
+            ),
             // Lifecycle Types
             new NamedXContentRegistry.Entry(
                 LifecycleType.class,
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleService.java
index fac7be2dc999..e57c8b331f31 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleService.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleService.java
@@ -61,6 +61,7 @@
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.xpack.core.ilm.IndexLifecycleOriginationDateParser.parseIndexNameAndExtractDate;
 import static org.elasticsearch.xpack.core.ilm.IndexLifecycleOriginationDateParser.shouldParseIndexName;
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentILMMode;
 
 /**
  * A service which runs the {@link LifecyclePolicy}s associated with indexes.
@@ -167,7 +168,7 @@ void onMaster(ClusterState clusterState) {
 
         final IndexLifecycleMetadata currentMetadata = clusterState.metadata().custom(IndexLifecycleMetadata.TYPE);
         if (currentMetadata != null) {
-            OperationMode currentMode = currentMetadata.getOperationMode();
+            OperationMode currentMode = currentILMMode(clusterState);
             if (OperationMode.STOPPED.equals(currentMode)) {
                 return;
             }
@@ -238,11 +239,15 @@ void onMaster(ClusterState clusterState) {
             }
 
             if (safeToStop && OperationMode.STOPPING == currentMode) {
-                submitUnbatchedTask("ilm_operation_mode_update[stopped]", OperationModeUpdateTask.ilmMode(OperationMode.STOPPED));
+                stopILM();
             }
         }
     }
 
+    private void stopILM() {
+        submitUnbatchedTask("ilm_operation_mode_update[stopped]", OperationModeUpdateTask.ilmMode(OperationMode.STOPPED));
+    }
+
     @Override
     public void beforeIndexAddedToCluster(Index index, Settings indexSettings) {
         if (shouldParseIndexName(indexSettings)) {
@@ -318,10 +323,7 @@ public void clusterChanged(ClusterChangedEvent event) {
                 });
             }
 
-            final IndexLifecycleMetadata lifecycleMetadata = event.state().metadata().custom(IndexLifecycleMetadata.TYPE);
-            if (lifecycleMetadata != null) {
-                triggerPolicies(event.state(), true);
-            }
+            triggerPolicies(event.state(), true);
         }
     }
 
@@ -370,12 +372,15 @@ public boolean policyExists(String policyId) {
     void triggerPolicies(ClusterState clusterState, boolean fromClusterStateChange) {
         IndexLifecycleMetadata currentMetadata = clusterState.metadata().custom(IndexLifecycleMetadata.TYPE);
 
+        OperationMode currentMode = currentILMMode(clusterState);
         if (currentMetadata == null) {
+            if (currentMode == OperationMode.STOPPING) {
+                // There are no policies and ILM is in stopping mode, so stop ILM and get out of here
+                stopILM();
+            }
             return;
         }
 
-        OperationMode currentMode = currentMetadata.getOperationMode();
-
         if (OperationMode.STOPPED.equals(currentMode)) {
             return;
         }
@@ -454,7 +459,7 @@ void triggerPolicies(ClusterState clusterState, boolean fromClusterStateChange)
         }
 
         if (safeToStop && OperationMode.STOPPING == currentMode) {
-            submitUnbatchedTask("ilm_operation_mode_update[stopped]", OperationModeUpdateTask.ilmMode(OperationMode.STOPPED));
+            stopILM();
         }
     }
 
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/OperationModeUpdateTask.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/OperationModeUpdateTask.java
index 80cc3834ce70..3162b232757a 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/OperationModeUpdateTask.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/OperationModeUpdateTask.java
@@ -17,12 +17,14 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata;
 import org.elasticsearch.xpack.core.ilm.OperationMode;
-import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
 
 import java.util.Objects;
 
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentILMMode;
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentSLMMode;
+
 /**
  * This task updates the operation mode state for ILM.
  *
@@ -91,27 +93,26 @@ private ClusterState updateILMState(final ClusterState currentState) {
         if (ilmMode == null) {
             return currentState;
         }
-        IndexLifecycleMetadata currentMetadata = currentState.metadata().custom(IndexLifecycleMetadata.TYPE);
-        if (currentMetadata != null && currentMetadata.getOperationMode().isValidChange(ilmMode) == false) {
+
+        final OperationMode currentMode = currentILMMode(currentState);
+        if (currentMode.equals(ilmMode)) {
+            // No need for a new state
             return currentState;
-        } else if (currentMetadata == null) {
-            currentMetadata = IndexLifecycleMetadata.EMPTY;
         }
 
         final OperationMode newMode;
-        if (currentMetadata.getOperationMode().isValidChange(ilmMode)) {
+        if (currentMode.isValidChange(ilmMode)) {
             newMode = ilmMode;
         } else {
-            newMode = currentMetadata.getOperationMode();
+            // The transition is invalid, return the current state
+            return currentState;
         }
 
-        if (newMode.equals(ilmMode) == false) {
-            logger.info("updating ILM operation mode to {}", newMode);
-        }
+        logger.info("updating ILM operation mode to {}", newMode);
         return ClusterState.builder(currentState)
             .metadata(
                 Metadata.builder(currentState.metadata())
-                    .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata(currentMetadata.getPolicyMetadatas(), newMode))
+                    .putCustom(LifecycleOperationMetadata.TYPE, new LifecycleOperationMetadata(newMode, currentSLMMode(currentState)))
             )
             .build();
     }
@@ -120,30 +121,26 @@ private ClusterState updateSLMState(final ClusterState currentState) {
         if (slmMode == null) {
             return currentState;
         }
-        SnapshotLifecycleMetadata currentMetadata = currentState.metadata().custom(SnapshotLifecycleMetadata.TYPE);
-        if (currentMetadata != null && currentMetadata.getOperationMode().isValidChange(slmMode) == false) {
+
+        final OperationMode currentMode = currentSLMMode(currentState);
+        if (currentMode.equals(slmMode)) {
+            // No need for a new state
             return currentState;
-        } else if (currentMetadata == null) {
-            currentMetadata = SnapshotLifecycleMetadata.EMPTY;
         }
 
         final OperationMode newMode;
-        if (currentMetadata.getOperationMode().isValidChange(slmMode)) {
+        if (currentMode.isValidChange(slmMode)) {
             newMode = slmMode;
         } else {
-            newMode = currentMetadata.getOperationMode();
+            // The transition is invalid, return the current state
+            return currentState;
         }
 
-        if (newMode.equals(slmMode) == false) {
-            logger.info("updating SLM operation mode to {}", newMode);
-        }
+        logger.info("updating SLM operation mode to {}", newMode);
         return ClusterState.builder(currentState)
             .metadata(
                 Metadata.builder(currentState.metadata())
-                    .putCustom(
-                        SnapshotLifecycleMetadata.TYPE,
-                        new SnapshotLifecycleMetadata(currentMetadata.getSnapshotConfigurations(), newMode, currentMetadata.getStats())
-                    )
+                    .putCustom(LifecycleOperationMetadata.TYPE, new LifecycleOperationMetadata(currentILMMode(currentState), newMode))
             )
             .build();
     }
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleAction.java
index 26f1f513ab55..39d87d26c4e9 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportDeleteLifecycleAction.java
@@ -37,6 +37,8 @@
 import java.util.SortedMap;
 import java.util.TreeMap;
 
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentILMMode;
+
 public class TransportDeleteLifecycleAction extends TransportMasterNodeAction<Request, AcknowledgedResponse> {
 
     @Inject
@@ -103,7 +105,7 @@ public ClusterState execute(ClusterState currentState) {
             }
             SortedMap<String, LifecyclePolicyMetadata> newPolicies = new TreeMap<>(currentMetadata.getPolicyMetadatas());
             newPolicies.remove(request.getPolicyName());
-            IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, currentMetadata.getOperationMode());
+            IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, currentILMMode(currentState));
             newState.metadata(Metadata.builder(currentState.getMetadata()).putCustom(IndexLifecycleMetadata.TYPE, newMetadata).build());
             return newState.build();
         }
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetStatusAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetStatusAction.java
index 62c59c80b815..90533405c1fc 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetStatusAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportGetStatusAction.java
@@ -19,12 +19,12 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
-import org.elasticsearch.xpack.core.ilm.OperationMode;
 import org.elasticsearch.xpack.core.ilm.action.GetStatusAction;
 import org.elasticsearch.xpack.core.ilm.action.GetStatusAction.Request;
 import org.elasticsearch.xpack.core.ilm.action.GetStatusAction.Response;
 
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentILMMode;
+
 public class TransportGetStatusAction extends TransportMasterNodeAction<Request, Response> {
 
     @Inject
@@ -50,15 +50,7 @@ public TransportGetStatusAction(
 
     @Override
     protected void masterOperation(Task task, Request request, ClusterState state, ActionListener<Response> listener) {
-        IndexLifecycleMetadata metadata = state.metadata().custom(IndexLifecycleMetadata.TYPE);
-        final Response response;
-        if (metadata == null) {
-            // no need to actually install metadata just yet, but safe to say it is not stopped
-            response = new Response(OperationMode.RUNNING);
-        } else {
-            response = new Response(metadata.getOperationMode());
-        }
-        listener.onResponse(response);
+        listener.onResponse(new Response(currentILMMode(state)));
     }
 
     @Override
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java
index fc6017dbb953..9a80e5d83ebc 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java
@@ -38,6 +38,7 @@
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
 
 import static org.elasticsearch.xpack.cluster.metadata.MetadataMigrateToDataTiersRoutingService.migrateToDataTiersRouting;
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentILMMode;
 import static org.elasticsearch.xpack.core.ilm.OperationMode.STOPPED;
 
 public class TransportMigrateToDataTiersAction extends TransportMasterNodeAction<MigrateToDataTiersRequest, MigrateToDataTiersResponse> {
@@ -107,11 +108,9 @@ protected void masterOperation(
         }
 
         IndexLifecycleMetadata currentMetadata = state.metadata().custom(IndexLifecycleMetadata.TYPE);
-        if (currentMetadata != null && currentMetadata.getOperationMode() != STOPPED) {
+        if (currentMetadata != null && currentILMMode(state) != STOPPED) {
             listener.onFailure(
-                new IllegalStateException(
-                    "stop ILM before migrating to data tiers, current state is [" + currentMetadata.getOperationMode() + "]"
-                )
+                new IllegalStateException("stop ILM before migrating to data tiers, current state is [" + currentILMMode(state) + "]")
             );
             return;
         }
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java
index d4834bef5939..d681ba8e126b 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleAction.java
@@ -52,6 +52,7 @@
 import java.util.SortedMap;
 import java.util.TreeMap;
 
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentILMMode;
 import static org.elasticsearch.xpack.core.ilm.PhaseCacheManagement.updateIndicesForPolicy;
 import static org.elasticsearch.xpack.core.searchablesnapshots.SearchableSnapshotsConstants.SEARCHABLE_SNAPSHOT_FEATURE;
 
@@ -190,7 +191,7 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                     logger.info("updating index lifecycle policy [{}]", request.getPolicy().getName());
                 }
             }
-            IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, currentMetadata.getOperationMode());
+            IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(newPolicies, currentILMMode(currentState));
             stateBuilder.metadata(Metadata.builder(currentState.getMetadata()).putCustom(IndexLifecycleMetadata.TYPE, newMetadata).build());
             ClusterState nonRefreshedState = stateBuilder.build();
             if (oldPolicy == null) {
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java
index eec70e138f07..b8af3e8e0daa 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStore.java
@@ -10,16 +10,17 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteRequest;
-import org.elasticsearch.action.bulk.BackoffPolicy;
 import org.elasticsearch.action.bulk.BulkItemResponse;
-import org.elasticsearch.action.bulk.BulkProcessor;
+import org.elasticsearch.action.bulk.BulkProcessor2;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -28,7 +29,6 @@
 import org.elasticsearch.xcontent.XContentFactory;
 
 import java.io.Closeable;
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.Map;
 import java.util.Objects;
@@ -44,12 +44,18 @@
 
 /**
  * The {@link ILMHistoryStore} handles indexing {@link ILMHistoryItem} documents into the
- * appropriate index. It sets up a {@link BulkProcessor} for indexing in bulk, and handles creation
+ * appropriate index. It sets up a {@link BulkProcessor2} for indexing in bulk, and handles creation
  * of the index/alias as needed for ILM policies.
  */
 public class ILMHistoryStore implements Closeable {
     private static final Logger logger = LogManager.getLogger(ILMHistoryStore.class);
 
+    public static final Setting<ByteSizeValue> MAX_BULK_REQUEST_BYTE_IN_FLIGHT_SETTING = Setting.byteSizeSetting(
+        "es.indices.lifecycle.history.bulk.request.bytes.in.flight",
+        ByteSizeValue.ofMb(100),
+        Setting.Property.NodeScope
+    );
+
     public static final String ILM_HISTORY_DATA_STREAM = "ilm-history-" + INDEX_TEMPLATE_VERSION;
 
     private static int ILM_HISTORY_BULK_SIZE = StrictMath.toIntExact(
@@ -60,82 +66,103 @@ public class ILMHistoryStore implements Closeable {
     );
 
     private volatile boolean ilmHistoryEnabled = true;
-    private final BulkProcessor processor;
-    private final ThreadPool threadPool;
+    private final BulkProcessor2 processor;
 
     public ILMHistoryStore(Client client, ClusterService clusterService, ThreadPool threadPool) {
+        this(client, clusterService, threadPool, ActionListener.noop(), TimeValue.timeValueSeconds(5));
+    }
+
+    /**
+     *  For unit testing, allows a more frequent flushInterval
+     * @param client
+     * @param clusterService
+     * @param threadPool
+     * @param listener
+     * @param flushInterval
+     */
+    ILMHistoryStore(
+        Client client,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        ActionListener<BulkResponse> listener,
+        TimeValue flushInterval
+    ) {
         this.setIlmHistoryEnabled(LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING.get(clusterService.getSettings()));
         clusterService.getClusterSettings().addSettingsUpdateConsumer(LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING, this::setIlmHistoryEnabled);
 
-        this.threadPool = threadPool;
-
-        this.processor = BulkProcessor.builder(new OriginSettingClient(client, INDEX_LIFECYCLE_ORIGIN)::bulk, new BulkProcessor.Listener() {
-            @Override
-            public void beforeBulk(long executionId, BulkRequest request) {
-                if (clusterService.state().getMetadata().templatesV2().containsKey(ILM_TEMPLATE_NAME) == false) {
-                    ElasticsearchException e = new ElasticsearchException("no ILM history template");
-                    logger.warn(
-                        () -> format(
-                            "unable to index the following ILM history items:\n%s",
+        this.processor = BulkProcessor2.builder(
+            new OriginSettingClient(client, INDEX_LIFECYCLE_ORIGIN)::bulk,
+            new BulkProcessor2.Listener() {
+                @Override
+                public void beforeBulk(long executionId, BulkRequest request) {
+                    if (clusterService.state().getMetadata().templatesV2().containsKey(ILM_TEMPLATE_NAME) == false) {
+                        ElasticsearchException e = new ElasticsearchException("no ILM history template");
+                        logger.warn(
+                            () -> format(
+                                "unable to index the following ILM history items:\n%s",
+                                request.requests()
+                                    .stream()
+                                    .filter(dwr -> (dwr instanceof IndexRequest))
+                                    .map(dwr -> ((IndexRequest) dwr))
+                                    .map(IndexRequest::sourceAsMap)
+                                    .map(Object::toString)
+                                    .collect(joining("\n"))
+                            ),
+                            e
+                        );
+                        throw new ElasticsearchException(e);
+                    }
+                    if (logger.isTraceEnabled()) {
+                        logger.info(
+                            "about to index: {}",
                             request.requests()
                                 .stream()
-                                .filter(dwr -> (dwr instanceof IndexRequest))
-                                .map(dwr -> ((IndexRequest) dwr))
-                                .map(IndexRequest::sourceAsMap)
-                                .map(Object::toString)
-                                .collect(joining("\n"))
-                        ),
-                        e
-                    );
-                    throw new ElasticsearchException(e);
-                }
-                if (logger.isTraceEnabled()) {
-                    logger.info(
-                        "about to index: {}",
-                        request.requests()
-                            .stream()
-                            .map(dwr -> ((IndexRequest) dwr).sourceAsMap())
-                            .map(Objects::toString)
-                            .collect(Collectors.joining(","))
-                    );
-                }
-            }
-
-            @Override
-            public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
-                long items = request.numberOfActions();
-                if (logger.isTraceEnabled()) {
-                    logger.trace(
-                        "indexed [{}] items into ILM history index [{}]",
-                        items,
-                        Arrays.stream(response.getItems()).map(BulkItemResponse::getIndex).distinct().collect(Collectors.joining(","))
-                    );
+                                .map(dwr -> ((IndexRequest) dwr).sourceAsMap())
+                                .map(Objects::toString)
+                                .collect(Collectors.joining(","))
+                        );
+                    }
                 }
-                if (response.hasFailures()) {
-                    Map<String, String> failures = Arrays.stream(response.getItems())
-                        .filter(BulkItemResponse::isFailed)
-                        .collect(
-                            Collectors.toMap(
-                                BulkItemResponse::getId,
-                                BulkItemResponse::getFailureMessage,
-                                (msg1, msg2) -> Objects.equals(msg1, msg2) ? msg1 : msg1 + "," + msg2
-                            )
+
+                @Override
+                public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
+                    long items = request.numberOfActions();
+                    if (logger.isTraceEnabled()) {
+                        logger.trace(
+                            "indexed [{}] items into ILM history index [{}]",
+                            items,
+                            Arrays.stream(response.getItems()).map(BulkItemResponse::getIndex).distinct().collect(Collectors.joining(","))
                         );
-                    logger.error("failures: [{}]", failures);
+                    }
+                    if (response.hasFailures()) {
+                        Map<String, String> failures = Arrays.stream(response.getItems())
+                            .filter(BulkItemResponse::isFailed)
+                            .collect(
+                                Collectors.toMap(
+                                    BulkItemResponse::getId,
+                                    BulkItemResponse::getFailureMessage,
+                                    (msg1, msg2) -> Objects.equals(msg1, msg2) ? msg1 : msg1 + "," + msg2
+                                )
+                            );
+                        logger.error("failures: [{}]", failures);
+                    }
+                    listener.onResponse(response);
                 }
-            }
-
-            @Override
-            public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
-                long items = request.numberOfActions();
-                logger.error(() -> "failed to index " + items + " items into ILM history index", failure);
-            }
-        }, "ilm-history-store")
+
+                @Override
+                public void afterBulk(long executionId, BulkRequest request, Exception failure) {
+                    long items = request.numberOfActions();
+                    logger.error(() -> "failed to index " + items + " items into ILM history index", failure);
+                    listener.onFailure(failure);
+                }
+            },
+            threadPool
+        )
             .setBulkActions(-1)
             .setBulkSize(ByteSizeValue.ofBytes(ILM_HISTORY_BULK_SIZE))
-            .setFlushInterval(TimeValue.timeValueSeconds(5))
-            .setConcurrentRequests(1)
-            .setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(1000), 3))
+            .setFlushInterval(flushInterval)
+            .setMaxBytesInFlight(MAX_BULK_REQUEST_BYTE_IN_FLIGHT_SETTING.get(clusterService.getSettings()))
+            .setMaxNumberOfRetries(3)
             .build();
     }
 
@@ -155,21 +182,9 @@ public void putAsync(ILMHistoryItem item) {
         try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
             item.toXContent(builder, ToXContent.EMPTY_PARAMS);
             IndexRequest request = new IndexRequest(ILM_HISTORY_DATA_STREAM).source(builder).opType(DocWriteRequest.OpType.CREATE);
-            // TODO: remove the threadpool wrapping when the .add call is non-blocking
-            // (it can currently execute the bulk request occasionally)
-            // see: https://github.com/elastic/elasticsearch/issues/50440
-            threadPool.executor(ThreadPool.Names.GENERIC).execute(() -> {
-                try {
-                    processor.add(request);
-                } catch (Exception e) {
-                    logger.error(
-                        () -> format("failed add ILM history item to queue for index [%s]: [%s]", ILM_HISTORY_DATA_STREAM, item),
-                        e
-                    );
-                }
-            });
-        } catch (IOException exception) {
-            logger.error(() -> format("failed to queue ILM history item in index [%s]: [%s]", ILM_HISTORY_DATA_STREAM, item), exception);
+            processor.add(request);
+        } catch (Exception e) {
+            logger.error(() -> format("failed to send ILM history item to index [%s]: [%s]", ILM_HISTORY_DATA_STREAM, item), e);
         }
     }
 
@@ -179,6 +194,7 @@ public void close() {
             processor.awaitClose(10, TimeUnit.SECONDS);
         } catch (InterruptedException e) {
             logger.warn("failed to shut down ILM history bulk processor after 10 seconds", e);
+            Thread.currentThread().interrupt();
         }
     }
 
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorService.java
index 45ecd0a9c279..6358744f597a 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorService.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorService.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.slm;
 
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.health.Diagnosis;
@@ -32,6 +33,7 @@
 
 import static org.elasticsearch.health.HealthStatus.GREEN;
 import static org.elasticsearch.health.HealthStatus.YELLOW;
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentSLMMode;
 import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.SLM_HEALTH_FAILED_SNAPSHOT_WARN_THRESHOLD_SETTING;
 
 /**
@@ -97,17 +99,19 @@ public String name() {
     }
 
     @Override
-    public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
-        var slmMetadata = clusterService.state().metadata().custom(SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata.EMPTY);
+    public HealthIndicatorResult calculate(boolean verbose, int maxAffectedResourcesCount, HealthInfo healthInfo) {
+        final ClusterState currentState = clusterService.state();
+        var slmMetadata = currentState.metadata().custom(SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata.EMPTY);
+        final OperationMode currentMode = currentSLMMode(currentState);
         if (slmMetadata.getSnapshotConfigurations().isEmpty()) {
             return createIndicator(
                 GREEN,
                 "No Snapshot Lifecycle Management policies configured",
-                createDetails(verbose, Collections.emptyList(), slmMetadata),
+                createDetails(verbose, Collections.emptyList(), slmMetadata, currentMode),
                 Collections.emptyList(),
                 Collections.emptyList()
             );
-        } else if (slmMetadata.getOperationMode() != OperationMode.RUNNING) {
+        } else if (currentMode != OperationMode.RUNNING) {
             List<HealthIndicatorImpact> impacts = Collections.singletonList(
                 new HealthIndicatorImpact(
                     NAME,
@@ -120,7 +124,7 @@ public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
             return createIndicator(
                 YELLOW,
                 "Snapshot Lifecycle Management is not running",
-                createDetails(verbose, Collections.emptyList(), slmMetadata),
+                createDetails(verbose, Collections.emptyList(), slmMetadata, currentMode),
                 impacts,
                 List.of(SLM_NOT_RUNNING)
             );
@@ -159,7 +163,7 @@ public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
                     : "An automated snapshot policy is unhealthy:\n") + unhealthyPolicyCauses;
 
                 String unhealthyPolicyActions = unhealthyPolicies.stream()
-                    .map(policy -> "- /_slm/policy/" + policy.getPolicy().getId() + "?human")
+                    .map(policy -> "- GET /_slm/policy/" + policy.getPolicy().getId() + "?human")
                     .collect(Collectors.joining("\n"));
                 String action = "Check the snapshot lifecycle "
                     + (unhealthyPolicies.size() > 1 ? "policies" : "policy")
@@ -169,7 +173,7 @@ public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
                 return createIndicator(
                     YELLOW,
                     "Encountered [" + unhealthyPolicies.size() + "] unhealthy snapshot lifecycle management policies.",
-                    createDetails(verbose, unhealthyPolicies, slmMetadata),
+                    createDetails(verbose, unhealthyPolicies, slmMetadata, currentMode),
                     impacts,
                     List.of(
                         new Diagnosis(
@@ -177,7 +181,10 @@ public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
                             List.of(
                                 new Diagnosis.Resource(
                                     Diagnosis.Resource.Type.SLM_POLICY,
-                                    unhealthyPolicies.stream().map(SnapshotLifecyclePolicyMetadata::getName).toList()
+                                    unhealthyPolicies.stream()
+                                        .map(SnapshotLifecyclePolicyMetadata::getName)
+                                        .limit(Math.min(unhealthyPolicies.size(), maxAffectedResourcesCount))
+                                        .toList()
                                 )
                             )
                         )
@@ -188,7 +195,7 @@ public HealthIndicatorResult calculate(boolean verbose, HealthInfo healthInfo) {
             return createIndicator(
                 GREEN,
                 "Snapshot Lifecycle Management is running",
-                createDetails(verbose, Collections.emptyList(), slmMetadata),
+                createDetails(verbose, Collections.emptyList(), slmMetadata, currentMode),
                 Collections.emptyList(),
                 Collections.emptyList()
             );
@@ -215,11 +222,12 @@ static boolean snapshotFailuresExceedWarningCount(long failedSnapshotWarnThresho
     private static HealthIndicatorDetails createDetails(
         boolean verbose,
         Collection<SnapshotLifecyclePolicyMetadata> unhealthyPolicies,
-        SnapshotLifecycleMetadata metadata
+        SnapshotLifecycleMetadata metadata,
+        OperationMode mode
     ) {
         if (verbose) {
             Map<String, Object> details = new LinkedHashMap<>();
-            details.put("slm_status", metadata.getOperationMode());
+            details.put("slm_status", mode);
             details.put("policies", metadata.getSnapshotConfigurations().size());
             if (unhealthyPolicies.size() > 0) {
                 details.put(
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleService.java
index 3f3cb8ef7dba..f47985891ccd 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleService.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleService.java
@@ -31,13 +31,14 @@
 import java.io.Closeable;
 import java.time.Clock;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Supplier;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentSLMMode;
+
 /**
  * {@code SnapshotLifecycleService} manages snapshot policy scheduling and triggering of the
  * {@link SnapshotLifecycleTask}. It reacts to new policies in the cluster state by scheduling a
@@ -121,20 +122,16 @@ SchedulerEngine getScheduler() {
      * Returns true if SLM is in the stopping or stopped state
      */
     static boolean slmStoppedOrStopping(ClusterState state) {
-        return Optional.ofNullable((SnapshotLifecycleMetadata) state.metadata().custom(SnapshotLifecycleMetadata.TYPE))
-            .map(SnapshotLifecycleMetadata::getOperationMode)
-            .map(mode -> OperationMode.STOPPING == mode || OperationMode.STOPPED == mode)
-            .orElse(false);
+        OperationMode mode = currentSLMMode(state);
+        return OperationMode.STOPPING == mode || OperationMode.STOPPED == mode;
     }
 
     /**
      * Returns true if SLM is in the stopping state
      */
     static boolean slmStopping(ClusterState state) {
-        return Optional.ofNullable((SnapshotLifecycleMetadata) state.metadata().custom(SnapshotLifecycleMetadata.TYPE))
-            .map(SnapshotLifecycleMetadata::getOperationMode)
-            .map(mode -> OperationMode.STOPPING == mode)
-            .orElse(false);
+        OperationMode mode = currentSLMMode(state);
+        return OperationMode.STOPPING == mode;
     }
 
     /**
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java
index bd06e35a5b21..58e208d3d2fb 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java
@@ -41,6 +41,7 @@
 import java.util.Optional;
 
 import static org.elasticsearch.core.Strings.format;
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentSLMMode;
 
 public class SnapshotLifecycleTask implements SchedulerEngine.Listener {
 
@@ -284,7 +285,11 @@ public ClusterState execute(ClusterState currentState) throws Exception {
             }
 
             snapLifecycles.put(policyName, newPolicyMetadata.build());
-            SnapshotLifecycleMetadata lifecycleMetadata = new SnapshotLifecycleMetadata(snapLifecycles, snapMeta.getOperationMode(), stats);
+            SnapshotLifecycleMetadata lifecycleMetadata = new SnapshotLifecycleMetadata(
+                snapLifecycles,
+                currentSLMMode(currentState),
+                stats
+            );
             Metadata currentMeta = currentState.metadata();
             return ClusterState.builder(currentState)
                 .metadata(Metadata.builder(currentMeta).putCustom(SnapshotLifecycleMetadata.TYPE, lifecycleMetadata))
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotRetentionTask.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotRetentionTask.java
index a8092cd35dd5..e310b3f27324 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotRetentionTask.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotRetentionTask.java
@@ -10,7 +10,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.GroupedActionListener;
+import org.elasticsearch.action.support.CountDownActionListener;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.OriginSettingClient;
@@ -295,9 +295,14 @@ void getAllRetainableSnapshots(
                         snapshots.computeIfAbsent(info.repository(), repo -> new ArrayList<>()).add(info);
                     }
                 }
+                if (resp.isFailed()) {
+                    for (String repo : resp.getFailures().keySet()) {
+                        logger.debug(() -> "unable to retrieve snapshots for [" + repo + "] repositories: ", resp.getFailures().get(repo));
+                    }
+                }
                 listener.onResponse(snapshots);
             }, e -> {
-                logger.debug(() -> "unable to retrieve snapshots for [" + repositories + "] repositories", e);
+                logger.debug(() -> "unable to retrieve snapshots for [" + repositories + "] repositories: ", e);
                 listener.onFailure(e);
             }));
     }
@@ -328,13 +333,13 @@ void deleteSnapshots(
         long startTime = nowNanoSupplier.getAsLong();
         final AtomicInteger deleted = new AtomicInteger(0);
         final AtomicInteger failed = new AtomicInteger(0);
-        final GroupedActionListener<Void> allDeletesListener = new GroupedActionListener<>(
-            ActionListener.runAfter(listener.map(v -> null), () -> {
+        final CountDownActionListener allDeletesListener = new CountDownActionListener(
+            snapshotsToDelete.size(),
+            ActionListener.runAfter(listener, () -> {
                 TimeValue totalElapsedTime = TimeValue.timeValueNanos(nowNanoSupplier.getAsLong() - startTime);
                 logger.debug("total elapsed time for deletion of [{}] snapshots: {}", deleted, totalElapsedTime);
                 slmStats.deletionTime(totalElapsedTime);
-            }),
-            snapshotsToDelete.size()
+            })
         );
         for (Map.Entry<String, List<Tuple<SnapshotId, String>>> entry : snapshotsToDelete.entrySet()) {
             String repo = entry.getKey();
@@ -354,7 +359,7 @@ private void deleteSnapshots(
         ActionListener<Void> listener
     ) {
 
-        final ActionListener<Void> allDeletesListener = new GroupedActionListener<>(listener.map(v -> null), snapshots.size());
+        final ActionListener<Void> allDeletesListener = new CountDownActionListener(snapshots.size(), listener);
         for (Tuple<SnapshotId, String> info : snapshots) {
             final SnapshotId snapshotId = info.v1();
             if (runningDeletions.add(snapshotId) == false) {
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/UpdateSnapshotLifecycleStatsTask.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/UpdateSnapshotLifecycleStatsTask.java
index f4b31de86828..11478e929c76 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/UpdateSnapshotLifecycleStatsTask.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/UpdateSnapshotLifecycleStatsTask.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.xpack.core.slm.SnapshotLifecycleStats;
 
 import static org.elasticsearch.core.Strings.format;
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentSLMMode;
 
 /**
  * {@link UpdateSnapshotLifecycleStatsTask} is a cluster state update task that retrieves the
@@ -45,7 +46,7 @@ public ClusterState execute(ClusterState currentState) {
         SnapshotLifecycleStats newMetrics = currentSlmMeta.getStats().merge(runStats);
         SnapshotLifecycleMetadata newSlmMeta = new SnapshotLifecycleMetadata(
             currentSlmMeta.getSnapshotConfigurations(),
-            currentSlmMeta.getOperationMode(),
+            currentSLMMode(currentState),
             newMetrics
         );
 
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java
index 4628560f6a73..4ccd025bab5a 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadata;
 import org.elasticsearch.xpack.core.slm.action.DeleteSnapshotLifecycleAction;
@@ -101,6 +102,7 @@ public ClusterState execute(ClusterState currentState) {
             if (snapMeta == null) {
                 throw new ResourceNotFoundException("snapshot lifecycle policy not found: {}", request.getLifecycleId());
             }
+            var currentMode = LifecycleOperationMetadata.currentSLMMode(currentState);
             // Check that the policy exists in the first place
             snapMeta.getSnapshotConfigurations()
                 .entrySet()
@@ -123,7 +125,7 @@ public ClusterState execute(ClusterState currentState) {
                             SnapshotLifecycleMetadata.TYPE,
                             new SnapshotLifecycleMetadata(
                                 newConfigs,
-                                snapMeta.getOperationMode(),
+                                currentMode,
                                 snapMeta.getStats().removePolicy(request.getLifecycleId())
                             )
                         )
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSLMStatusAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSLMStatusAction.java
index fbb182824a1d..57b585395edc 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSLMStatusAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportGetSLMStatusAction.java
@@ -19,10 +19,10 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.ilm.OperationMode;
-import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
 import org.elasticsearch.xpack.core.slm.action.GetSLMStatusAction;
 
+import static org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata.currentSLMMode;
+
 public class TransportGetSLMStatusAction extends TransportMasterNodeAction<GetSLMStatusAction.Request, GetSLMStatusAction.Response> {
 
     @Inject
@@ -53,15 +53,7 @@ protected void masterOperation(
         ClusterState state,
         ActionListener<GetSLMStatusAction.Response> listener
     ) {
-        SnapshotLifecycleMetadata metadata = state.metadata().custom(SnapshotLifecycleMetadata.TYPE);
-        final GetSLMStatusAction.Response response;
-        if (metadata == null) {
-            // no need to actually install metadata just yet, but safe to say it is not stopped
-            response = new GetSLMStatusAction.Response(OperationMode.RUNNING);
-        } else {
-            response = new GetSLMStatusAction.Response(metadata.getOperationMode());
-        }
-        listener.onResponse(response);
+        listener.onResponse(new GetSLMStatusAction.Response(currentSLMMode(state)));
     }
 
     @Override
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java
index 6ea63e6ef2c7..4ce91d2291b7 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java
@@ -28,8 +28,8 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata;
 import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
-import org.elasticsearch.xpack.core.ilm.OperationMode;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadata;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecycleStats;
@@ -129,6 +129,7 @@ public static class UpdateSnapshotPolicyTask extends AckedClusterStateUpdateTask
         @Override
         public ClusterState execute(ClusterState currentState) {
             SnapshotLifecycleMetadata snapMeta = currentState.metadata().custom(SnapshotLifecycleMetadata.TYPE);
+            var currentMode = LifecycleOperationMetadata.currentSLMMode(currentState);
 
             String id = request.getLifecycleId();
             final SnapshotLifecycleMetadata lifecycleMetadata;
@@ -140,7 +141,7 @@ public ClusterState execute(ClusterState currentState) {
                     .build();
                 lifecycleMetadata = new SnapshotLifecycleMetadata(
                     Collections.singletonMap(id, meta),
-                    OperationMode.RUNNING,
+                    currentMode,
                     new SnapshotLifecycleStats()
                 );
                 logger.info("adding new snapshot lifecycle [{}]", id);
@@ -154,7 +155,7 @@ public ClusterState execute(ClusterState currentState) {
                     .setModifiedDate(Instant.now().toEpochMilli())
                     .build();
                 snapLifecycles.put(id, newLifecycle);
-                lifecycleMetadata = new SnapshotLifecycleMetadata(snapLifecycles, snapMeta.getOperationMode(), snapMeta.getStats());
+                lifecycleMetadata = new SnapshotLifecycleMetadata(snapLifecycles, currentMode, snapMeta.getStats());
                 if (oldLifecycle == null) {
                     logger.info("adding new snapshot lifecycle [{}]", id);
                 } else {
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java
index 16ded13369a2..5385a2dae0fb 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.ESTestCase;
@@ -1551,7 +1552,7 @@ public void testMigrateIndexAndComponentTemplates() {
     }
 
     private String getWarmPhaseDef() {
-        return formatted("""
+        return Strings.format("""
             {
               "policy": "%s",
               "phase_definition": {
@@ -1577,7 +1578,7 @@ private String getWarmPhaseDef() {
     }
 
     private String getColdPhaseDefinitionWithTotalShardsPerNode() {
-        return formatted("""
+        return Strings.format("""
             {
               "policy": "%s",
               "phase_definition": {
@@ -1597,7 +1598,7 @@ private String getColdPhaseDefinitionWithTotalShardsPerNode() {
     }
 
     private String getColdPhaseDefinition() {
-        return formatted("""
+        return Strings.format("""
             {
               "policy": "%s",
               "phase_definition": {
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IlmHealthIndicatorServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IlmHealthIndicatorServiceTests.java
index a2ef17299426..01e5e38111f5 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IlmHealthIndicatorServiceTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IlmHealthIndicatorServiceTests.java
@@ -124,6 +124,16 @@ public void testIsGreenWhenNoMetadata() {
         );
     }
 
+    // We expose the indicator name and the diagnoses in the x-pack usage API. In order to index them properly in a telemetry index
+    // they need to be declared in the health-api-indexer.edn in the telemetry repository.
+    public void testMappedFieldsForTelemetry() {
+        assertThat(IlmHealthIndicatorService.NAME, equalTo("ilm"));
+        assertThat(
+            IlmHealthIndicatorService.ILM_NOT_RUNNING.definition().getUniqueId(),
+            equalTo("elasticsearch:health:ilm:diagnosis:ilm_disabled")
+        );
+    }
+
     private static ClusterState createClusterStateWith(IndexLifecycleMetadata metadata) {
         var builder = new ClusterState.Builder(new ClusterName("test-cluster"));
         if (metadata != null) {
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleFeatureSetUsageTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleFeatureSetUsageTests.java
index 8b2f9f1d537f..d57cd78f1eed 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleFeatureSetUsageTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleFeatureSetUsageTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleFeatureSetUsage;
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleFeatureSetUsage.PolicyStats;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -29,7 +28,7 @@ protected IndexLifecycleFeatureSetUsage createTestInstance() {
     }
 
     @Override
-    protected IndexLifecycleFeatureSetUsage mutateInstance(IndexLifecycleFeatureSetUsage instance) throws IOException {
+    protected IndexLifecycleFeatureSetUsage mutateInstance(IndexLifecycleFeatureSetUsage instance) {
         boolean available = instance.available();
         List<PolicyStats> policyStats = instance.getPolicyStats();
         policyStats = new ArrayList<>(policyStats);
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleMetadataTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleMetadataTests.java
index 9c4c0ec2c860..f479c6d0e788 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleMetadataTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleMetadataTests.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.ilm;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.metadata.Metadata;
@@ -15,8 +15,8 @@
 import org.elasticsearch.common.io.stream.Writeable.Reader;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.test.SimpleDiffableSerializationTestCase;
-import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.ChunkedToXContentDiffableSerializationTestCase;
+import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentParser;
@@ -55,7 +55,7 @@
 import static org.elasticsearch.xpack.ilm.LifecyclePolicyTestsUtils.newTestLifecyclePolicy;
 import static org.elasticsearch.xpack.ilm.LifecyclePolicyTestsUtils.randomTimeseriesLifecyclePolicy;
 
-public class IndexLifecycleMetadataTests extends SimpleDiffableSerializationTestCase<Custom> {
+public class IndexLifecycleMetadataTests extends ChunkedToXContentDiffableSerializationTestCase<Custom> {
 
     @Override
     protected IndexLifecycleMetadata createTestInstance() {
@@ -144,7 +144,7 @@ protected NamedXContentRegistry xContentRegistry() {
     }
 
     @Override
-    protected Metadata.Custom mutateInstance(Metadata.Custom instance) {
+    protected Metadata.Custom mutateInstance(Custom instance) {
         IndexLifecycleMetadata metadata = (IndexLifecycleMetadata) instance;
         Map<String, LifecyclePolicyMetadata> policies = metadata.getPolicyMetadatas();
         policies = new TreeMap<>(policies);
@@ -177,8 +177,8 @@ protected Reader<Diff<Custom>> diffReader() {
     }
 
     public void testMinimumSupportedVersion() {
-        Version min = createTestInstance().getMinimalSupportedVersion();
-        assertTrue(min.onOrBefore(VersionUtils.randomCompatibleVersion(random(), Version.CURRENT)));
+        TransportVersion min = createTestInstance().getMinimalSupportedVersion();
+        assertTrue(min.onOrBefore(TransportVersionUtils.randomCompatibleVersion(random(), TransportVersion.CURRENT)));
     }
 
     public void testcontext() {
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunnerTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunnerTests.java
index 83f58493304f..c9b97cd4f95e 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunnerTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunnerTests.java
@@ -370,7 +370,7 @@ public void testRunStateChangePolicyWithNextStep() throws Exception {
             .stream()
             .findFirst()
             .orElseThrow(() -> new AssertionError("failed to register ILM history"));
-        assertThat(historyItem.toString(), containsString(formatted("""
+        assertThat(historyItem.toString(), containsString(Strings.format("""
             {
               "index": "test",
               "policy": "foo",
@@ -1268,7 +1268,7 @@ private static class NoOpHistoryStore extends ILMHistoryStore {
         private final List<ILMHistoryItem> items = new CopyOnWriteArrayList<>();
 
         NoOpHistoryStore(Client noopClient, ClusterService clusterService) {
-            super(noopClient, clusterService, null);
+            super(noopClient, clusterService, clusterService.threadPool());
         }
 
         public List<ILMHistoryItem> getItems() {
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransitionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransitionTests.java
index 6bb56b9394b2..fdb9c930d3de 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransitionTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransitionTests.java
@@ -572,21 +572,21 @@ public void testValidateTransitionToCachedStepMissingFromPolicy() {
             .setStep("check-rollover-ready")
             .setPhaseDefinition("""
                 {
-                        "policy" : "my-policy",
-                        "phase_definition" : {
-                          "min_age" : "20m",
-                          "actions" : {
-                            "rollover" : {
-                              "max_age" : "5s"
-                            },
-                            "set_priority" : {
-                              "priority" : 150
-                            }
-                          }
-                        },
-                        "version" : 1,
-                        "modified_date_in_millis" : 1578521007076
-                      }""");
+                  "policy" : "my-policy",
+                  "phase_definition" : {
+                    "min_age" : "20m",
+                    "actions" : {
+                      "rollover" : {
+                        "max_age" : "5s"
+                      },
+                      "set_priority" : {
+                        "priority" : 150
+                      }
+                    }
+                  },
+                  "version" : 1,
+                  "modified_date_in_millis" : 1578521007076
+                }""");
 
         IndexMetadata meta = buildIndexMetadata("my-policy", executionState);
 
@@ -829,21 +829,21 @@ public void testMoveClusterStateToPreviouslyFailedStepAsAutomaticRetry() {
     public void testMoveToFailedStepDoesntRefreshCachedPhaseWhenUnsafe() {
         String initialPhaseDefinition = """
             {
-                    "policy" : "my-policy",
-                    "phase_definition" : {
-                      "min_age" : "20m",
-                      "actions" : {
-                        "rollover" : {
-                          "max_age" : "5s"
-                        },
-                        "set_priority" : {
-                          "priority" : 150
-                        }
-                      }
-                    },
-                    "version" : 1,
-                    "modified_date_in_millis" : 1578521007076
-                  }""";
+              "policy" : "my-policy",
+              "phase_definition" : {
+                "min_age" : "20m",
+                "actions" : {
+                  "rollover" : {
+                    "max_age" : "5s"
+                  },
+                  "set_priority" : {
+                    "priority" : 150
+                  }
+                }
+              },
+              "version" : 1,
+              "modified_date_in_millis" : 1578521007076
+            }""";
         String failedStep = "check-rollover-ready";
         LifecycleExecutionState.Builder currentExecutionState = LifecycleExecutionState.builder()
             .setPhase("hot")
@@ -900,21 +900,21 @@ public void testRefreshPhaseJson() throws IOException {
             .setStep("check-rollover-ready")
             .setPhaseDefinition("""
                 {
-                        "policy" : "my-policy",
-                        "phase_definition" : {
-                          "min_age" : "20m",
-                          "actions" : {
-                            "rollover" : {
-                              "max_age" : "5s"
-                            },
-                            "set_priority" : {
-                              "priority" : 150
-                            }
-                          }
-                        },
-                        "version" : 1,
-                        "modified_date_in_millis" : 1578521007076
-                      }""");
+                  "policy" : "my-policy",
+                  "phase_definition" : {
+                    "min_age" : "20m",
+                    "actions" : {
+                      "rollover" : {
+                        "max_age" : "5s"
+                      },
+                      "set_priority" : {
+                        "priority" : 150
+                      }
+                    }
+                  },
+                  "version" : 1,
+                  "modified_date_in_millis" : 1578521007076
+                }""");
 
         IndexMetadata meta = buildIndexMetadata("my-policy", exState);
         String index = meta.getIndex().getName();
@@ -1052,21 +1052,21 @@ public void testMoveStateToNextActionAndUpdateCachedPhase() {
             .setStep("check-rollover-ready")
             .setPhaseDefinition("""
                 {
-                        "policy" : "my-policy",
-                        "phase_definition" : {
-                          "min_age" : "20m",
-                          "actions" : {
-                            "rollover" : {
-                              "max_age" : "5s"
-                            },
-                            "set_priority" : {
-                              "priority" : 150
-                            }
-                          }
-                        },
-                        "version" : 1,
-                        "modified_date_in_millis" : 1578521007076
-                      }""");
+                  "policy" : "my-policy",
+                  "phase_definition" : {
+                    "min_age" : "20m",
+                    "actions" : {
+                      "rollover" : {
+                        "max_age" : "5s"
+                      },
+                      "set_priority" : {
+                        "priority" : 150
+                      }
+                    }
+                  },
+                  "version" : 1,
+                  "modified_date_in_millis" : 1578521007076
+                }""");
 
         IndexMetadata meta = buildIndexMetadata("my-policy", currentExecutionState);
 
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/LifecycleOperationSnapshotTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/LifecycleOperationSnapshotTests.java
new file mode 100644
index 000000000000..a0c9517db409
--- /dev/null
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/LifecycleOperationSnapshotTests.java
@@ -0,0 +1,137 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.ilm;
+
+import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsAction;
+import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotAction;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.snapshots.SnapshotState;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
+import org.elasticsearch.xpack.core.ilm.OperationMode;
+import org.elasticsearch.xpack.core.ilm.Phase;
+import org.elasticsearch.xpack.core.ilm.ReadOnlyAction;
+import org.elasticsearch.xpack.core.ilm.StopILMRequest;
+import org.elasticsearch.xpack.core.ilm.action.GetStatusAction;
+import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction;
+import org.elasticsearch.xpack.core.ilm.action.StopILMAction;
+import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicy;
+import org.elasticsearch.xpack.core.slm.action.ExecuteSnapshotLifecycleAction;
+import org.elasticsearch.xpack.core.slm.action.GetSLMStatusAction;
+import org.elasticsearch.xpack.core.slm.action.PutSnapshotLifecycleAction;
+import org.elasticsearch.xpack.core.slm.action.StopSLMAction;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadataTests.randomRetention;
+import static org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicyMetadataTests.randomSchedule;
+import static org.hamcrest.Matchers.equalTo;
+
+public class LifecycleOperationSnapshotTests extends ESSingleNodeTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return List.of(LocalStateCompositeXPackPlugin.class, IndexLifecycle.class);
+    }
+
+    @Override
+    protected Settings nodeSettings() {
+        return Settings.builder().put(super.nodeSettings()).put("slm.history_index_enabled", false).build();
+    }
+
+    public void testModeSnapshotRestore() throws Exception {
+        client().admin()
+            .cluster()
+            .preparePutRepository("repo")
+            .setType("fs")
+            .setSettings(Settings.builder().put("location", "repo").build())
+            .get();
+
+        client().execute(
+            PutSnapshotLifecycleAction.INSTANCE,
+            new PutSnapshotLifecycleAction.Request(
+                "slm-policy",
+                new SnapshotLifecyclePolicy(
+                    "slm-policy",
+                    randomAlphaOfLength(4).toLowerCase(Locale.ROOT),
+                    randomSchedule(),
+                    "repo",
+                    null,
+                    randomRetention()
+                )
+            )
+        ).get();
+
+        client().execute(
+            PutLifecycleAction.INSTANCE,
+            new PutLifecycleAction.Request(
+                new LifecyclePolicy(
+                    "ilm-policy",
+                    Map.of("warm", new Phase("warm", TimeValue.timeValueHours(1), Map.of("readonly", new ReadOnlyAction())))
+                )
+            )
+        );
+
+        assertThat(ilmMode(), equalTo(OperationMode.RUNNING));
+        assertThat(slmMode(), equalTo(OperationMode.RUNNING));
+
+        // Take snapshot
+        ExecuteSnapshotLifecycleAction.Response resp = client().execute(
+            ExecuteSnapshotLifecycleAction.INSTANCE,
+            new ExecuteSnapshotLifecycleAction.Request("slm-policy")
+        ).get();
+        final String snapshotName = resp.getSnapshotName();
+        // Wait for the snapshot to be successful
+        assertBusy(() -> {
+            logger.info("--> checking for snapshot success");
+            try {
+                GetSnapshotsResponse getResp = client().execute(
+                    GetSnapshotsAction.INSTANCE,
+                    new GetSnapshotsRequest(new String[] { "repo" }, new String[] { snapshotName })
+                ).get();
+                assertThat(getResp.getSnapshots().size(), equalTo(1));
+                assertThat(getResp.getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
+            } catch (Exception e) {
+                fail("snapshot does not yet exist");
+            }
+        });
+
+        assertAcked(client().execute(StopILMAction.INSTANCE, new StopILMRequest()).get());
+        assertAcked(client().execute(StopSLMAction.INSTANCE, new StopSLMAction.Request()).get());
+        assertBusy(() -> assertThat(ilmMode(), equalTo(OperationMode.STOPPED)));
+        assertBusy(() -> assertThat(slmMode(), equalTo(OperationMode.STOPPED)));
+
+        // Restore snapshot
+        client().execute(
+            RestoreSnapshotAction.INSTANCE,
+            new RestoreSnapshotRequest("repo", snapshotName).includeGlobalState(true).indices(Strings.EMPTY_ARRAY).waitForCompletion(true)
+        ).get();
+
+        assertBusy(() -> assertThat(ilmMode(), equalTo(OperationMode.STOPPED)));
+        assertBusy(() -> assertThat(slmMode(), equalTo(OperationMode.STOPPED)));
+    }
+
+    private OperationMode ilmMode() throws Exception {
+        return client().execute(GetStatusAction.INSTANCE, new GetStatusAction.Request()).get().getMode();
+    }
+
+    private OperationMode slmMode() throws Exception {
+        return client().execute(GetSLMStatusAction.INSTANCE, new GetSLMStatusAction.Request()).get().getOperationMode();
+    }
+}
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/OperationModeUpdateTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/OperationModeUpdateTaskTests.java
index 632b93da6b92..05f66b821c55 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/OperationModeUpdateTaskTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/OperationModeUpdateTaskTests.java
@@ -12,50 +12,89 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.ilm.LifecycleOperationMetadata;
 import org.elasticsearch.xpack.core.ilm.OperationMode;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecycleStats;
 
 import java.util.Collections;
 import java.util.Map;
+import java.util.Optional;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 
 public class OperationModeUpdateTaskTests extends ESTestCase {
 
-    public void testExecute() {
-        assertMove(OperationMode.RUNNING, OperationMode.STOPPING);
-        assertMove(OperationMode.STOPPING, randomFrom(OperationMode.RUNNING, OperationMode.STOPPED));
-        assertMove(OperationMode.STOPPED, OperationMode.RUNNING);
+    public void testILMExecute() {
+        assertILMMove(OperationMode.RUNNING, OperationMode.STOPPING);
+        assertILMMove(OperationMode.STOPPING, OperationMode.RUNNING);
+        assertILMMove(OperationMode.STOPPING, OperationMode.STOPPED);
+        assertILMMove(OperationMode.STOPPED, OperationMode.RUNNING);
 
         OperationMode mode = randomFrom(OperationMode.values());
-        assertNoMove(mode, mode);
-        assertNoMove(OperationMode.STOPPED, OperationMode.STOPPING);
-        assertNoMove(OperationMode.RUNNING, OperationMode.STOPPED);
+        assertNoILMMove(mode, mode);
+        assertNoILMMove(OperationMode.STOPPED, OperationMode.STOPPING);
+        assertNoILMMove(OperationMode.RUNNING, OperationMode.STOPPED);
+    }
+
+    public void testSLMExecute() {
+        assertSLMMove(OperationMode.RUNNING, OperationMode.STOPPING);
+        assertSLMMove(OperationMode.STOPPING, OperationMode.RUNNING);
+        assertSLMMove(OperationMode.STOPPING, OperationMode.STOPPED);
+        assertSLMMove(OperationMode.STOPPED, OperationMode.RUNNING);
+
+        OperationMode mode = randomFrom(OperationMode.values());
+        assertNoSLMMove(mode, mode);
+        assertNoSLMMove(OperationMode.STOPPED, OperationMode.STOPPING);
+        assertNoSLMMove(OperationMode.RUNNING, OperationMode.STOPPED);
     }
 
     public void testExecuteWithEmptyMetadata() {
         OperationMode requestedMode = OperationMode.STOPPING;
-        OperationMode newMode = executeUpdate(false, IndexLifecycleMetadata.EMPTY.getOperationMode(), requestedMode, false);
+        OperationMode newMode = executeILMUpdate(false, LifecycleOperationMetadata.EMPTY.getILMOperationMode(), requestedMode, false);
+        assertThat(newMode, equalTo(requestedMode));
+
+        newMode = executeSLMUpdate(false, LifecycleOperationMetadata.EMPTY.getSLMOperationMode(), requestedMode, false);
         assertThat(newMode, equalTo(requestedMode));
 
-        requestedMode = randomFrom(OperationMode.RUNNING, OperationMode.STOPPED);
-        newMode = executeUpdate(false, IndexLifecycleMetadata.EMPTY.getOperationMode(), requestedMode, false);
+        requestedMode = OperationMode.RUNNING;
+        newMode = executeILMUpdate(false, LifecycleOperationMetadata.EMPTY.getILMOperationMode(), requestedMode, true);
+        assertThat(newMode, equalTo(OperationMode.RUNNING));
+        requestedMode = OperationMode.STOPPED;
+        newMode = executeILMUpdate(false, LifecycleOperationMetadata.EMPTY.getILMOperationMode(), requestedMode, true);
+        assertThat(newMode, equalTo(OperationMode.RUNNING));
+
+        requestedMode = OperationMode.RUNNING;
+        newMode = executeSLMUpdate(false, LifecycleOperationMetadata.EMPTY.getSLMOperationMode(), requestedMode, true);
+        assertThat(newMode, equalTo(OperationMode.RUNNING));
+        requestedMode = OperationMode.STOPPED;
+        newMode = executeSLMUpdate(false, LifecycleOperationMetadata.EMPTY.getSLMOperationMode(), requestedMode, true);
         assertThat(newMode, equalTo(OperationMode.RUNNING));
     }
 
-    private void assertMove(OperationMode currentMode, OperationMode requestedMode) {
-        OperationMode newMode = executeUpdate(true, currentMode, requestedMode, false);
+    private void assertILMMove(OperationMode currentMode, OperationMode requestedMode) {
+        OperationMode newMode = executeILMUpdate(true, currentMode, requestedMode, false);
         assertThat(newMode, equalTo(requestedMode));
     }
 
-    private void assertNoMove(OperationMode currentMode, OperationMode requestedMode) {
-        OperationMode newMode = executeUpdate(true, currentMode, requestedMode, true);
+    private void assertSLMMove(OperationMode currentMode, OperationMode requestedMode) {
+        OperationMode newMode = executeSLMUpdate(true, currentMode, requestedMode, false);
+        assertThat(newMode, equalTo(requestedMode));
+    }
+
+    private void assertNoILMMove(OperationMode currentMode, OperationMode requestedMode) {
+        OperationMode newMode = executeILMUpdate(true, currentMode, requestedMode, true);
+        assertThat(newMode, equalTo(currentMode));
+    }
+
+    private void assertNoSLMMove(OperationMode currentMode, OperationMode requestedMode) {
+        OperationMode newMode = executeSLMUpdate(true, currentMode, requestedMode, true);
         assertThat(newMode, equalTo(currentMode));
     }
 
-    private OperationMode executeUpdate(
+    @SuppressWarnings("deprecated")
+    private OperationMode executeILMUpdate(
         boolean metadataInstalled,
         OperationMode currentMode,
         OperationMode requestMode,
@@ -76,13 +115,49 @@ private OperationMode executeUpdate(
         ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build();
         OperationModeUpdateTask task = OperationModeUpdateTask.ilmMode(requestMode);
         ClusterState newState = task.execute(state);
+        if (assertSameClusterState) {
+            assertSame("expected the same state instance but they were different", state, newState);
+        } else {
+            assertThat("expected a different state instance but they were the same", state, not(equalTo(newState)));
+        }
+        LifecycleOperationMetadata newMetadata = newState.metadata().custom(LifecycleOperationMetadata.TYPE);
+        IndexLifecycleMetadata oldMetadata = newState.metadata().custom(IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata.EMPTY);
+        return Optional.ofNullable(newMetadata)
+            .map(LifecycleOperationMetadata::getILMOperationMode)
+            .orElseGet(oldMetadata::getOperationMode);
+    }
+
+    @SuppressWarnings("deprecated")
+    private OperationMode executeSLMUpdate(
+        boolean metadataInstalled,
+        OperationMode currentMode,
+        OperationMode requestMode,
+        boolean assertSameClusterState
+    ) {
+        IndexLifecycleMetadata indexLifecycleMetadata = new IndexLifecycleMetadata(Collections.emptyMap(), currentMode);
+        SnapshotLifecycleMetadata snapshotLifecycleMetadata = new SnapshotLifecycleMetadata(
+            Collections.emptyMap(),
+            currentMode,
+            new SnapshotLifecycleStats()
+        );
+        Metadata.Builder metadata = Metadata.builder().persistentSettings(settings(Version.CURRENT).build());
+        if (metadataInstalled) {
+            metadata.customs(
+                Map.of(IndexLifecycleMetadata.TYPE, indexLifecycleMetadata, SnapshotLifecycleMetadata.TYPE, snapshotLifecycleMetadata)
+            );
+        }
+        ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build();
+        OperationModeUpdateTask task = OperationModeUpdateTask.slmMode(requestMode);
+        ClusterState newState = task.execute(state);
         if (assertSameClusterState) {
             assertSame(state, newState);
         } else {
             assertThat(state, not(equalTo(newState)));
         }
-        IndexLifecycleMetadata newMetadata = newState.metadata().custom(IndexLifecycleMetadata.TYPE);
-        assertThat(newMetadata.getPolicyMetadatas(), equalTo(indexLifecycleMetadata.getPolicyMetadatas()));
-        return newMetadata.getOperationMode();
+        LifecycleOperationMetadata newMetadata = newState.metadata().custom(LifecycleOperationMetadata.TYPE);
+        SnapshotLifecycleMetadata oldMetadata = newState.metadata().custom(SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata.EMPTY);
+        return Optional.ofNullable(newMetadata)
+            .map(LifecycleOperationMetadata::getSLMOperationMode)
+            .orElseGet(oldMetadata::getOperationMode);
     }
 }
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/PhaseStatsTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/PhaseStatsTests.java
index d01002be318d..738a7bf46405 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/PhaseStatsTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/PhaseStatsTests.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.core.ilm.ActionConfigStatsTests;
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleFeatureSetUsage.PhaseStats;
 
-import java.io.IOException;
 import java.util.Arrays;
 
 public class PhaseStatsTests extends AbstractWireSerializingTestCase<PhaseStats> {
@@ -30,7 +29,7 @@ static PhaseStats randomPhaseStats() {
     }
 
     @Override
-    protected PhaseStats mutateInstance(PhaseStats instance) throws IOException {
+    protected PhaseStats mutateInstance(PhaseStats instance) {
         TimeValue minimumAge = instance.getAfter();
         String[] actionNames = instance.getActionNames();
         switch (between(0, 1)) {
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/PolicyStatsTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/PolicyStatsTests.java
index 906d48ea03d5..b2bbe2de1f51 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/PolicyStatsTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/PolicyStatsTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleFeatureSetUsage.PhaseStats;
 import org.elasticsearch.xpack.core.ilm.IndexLifecycleFeatureSetUsage.PolicyStats;
 
-import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -34,7 +33,7 @@ static PolicyStats randomPolicyStats() {
     }
 
     @Override
-    protected PolicyStats mutateInstance(PolicyStats instance) throws IOException {
+    protected PolicyStats mutateInstance(PolicyStats instance) {
         Map<String, PhaseStats> phaseStats = instance.getPhaseStats();
         int numberIndicesManaged = instance.getIndicesManaged();
         switch (between(0, 1)) {
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/PolicyStepsRegistryTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/PolicyStepsRegistryTests.java
index e6a4c73d9b58..e2888f36d7ad 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/PolicyStepsRegistryTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/PolicyStepsRegistryTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
@@ -237,7 +238,7 @@ public void testUpdateFromNothingToSomethingToNothing() throws Exception {
             .build();
         try (XContentBuilder builder = JsonXContent.contentBuilder()) {
             builder.startObject();
-            metadata.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            ChunkedToXContent.wrapAsToXContent(metadata).toXContent(builder, ToXContent.EMPTY_PARAMS);
             builder.endObject();
             logger.info("--> metadata: {}", Strings.toString(builder));
         }
@@ -406,7 +407,7 @@ public void testUpdatePolicyButNoPhaseChangeIndexStepsDontChange() throws Except
             .build();
         try (XContentBuilder builder = JsonXContent.contentBuilder()) {
             builder.startObject();
-            metadata.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            ChunkedToXContent.wrapAsToXContent(metadata).toXContent(builder, ToXContent.EMPTY_PARAMS);
             builder.endObject();
             logger.info("--> metadata: {}", Strings.toString(builder));
         }
@@ -447,7 +448,7 @@ public void testUpdatePolicyButNoPhaseChangeIndexStepsDontChange() throws Except
         metadata = Metadata.builder(metadata).putCustom(IndexLifecycleMetadata.TYPE, lifecycleMetadata).build();
         try (XContentBuilder builder = JsonXContent.contentBuilder()) {
             builder.startObject();
-            metadata.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            ChunkedToXContent.wrapAsToXContent(metadata).toXContent(builder, ToXContent.EMPTY_PARAMS);
             builder.endObject();
             logger.info("--> metadata: {}", Strings.toString(builder));
         }
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/ReservedLifecycleStateServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/ReservedLifecycleStateServiceTests.java
index 359380754e05..f0e987893ab1 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/ReservedLifecycleStateServiceTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/ReservedLifecycleStateServiceTests.java
@@ -133,15 +133,15 @@ public void testValidationFails() {
 
         String badPolicyJSON = """
             {
-                "my_timeseries_lifecycle": {
-                    "phase": {
-                        "warm": {
-                            "min_age": "10s",
-                            "actions": {
-                            }
-                        }
+              "my_timeseries_lifecycle": {
+                "phase": {
+                  "warm": {
+                    "min_age": "10s",
+                    "actions": {
                     }
+                  }
                 }
+              }
             }""";
 
         assertEquals(
@@ -169,29 +169,29 @@ public void testActionAddRemove() throws Exception {
 
         String twoPoliciesJSON = """
             {
-                "my_timeseries_lifecycle": {
-                    "phases": {
-                        "warm": {
-                            "min_age": "10s",
-                            "actions": {
-                            }
-                        }
+              "my_timeseries_lifecycle": {
+                "phases": {
+                  "warm": {
+                    "min_age": "10s",
+                    "actions": {
                     }
-                },
-                "my_timeseries_lifecycle1": {
-                    "phases": {
-                        "warm": {
-                            "min_age": "10s",
-                            "actions": {
-                            }
-                        },
-                        "delete": {
-                            "min_age": "30s",
-                            "actions": {
-                            }
-                        }
+                  }
+                }
+              },
+              "my_timeseries_lifecycle1": {
+                "phases": {
+                  "warm": {
+                    "min_age": "10s",
+                    "actions": {
+                    }
+                  },
+                  "delete": {
+                    "min_age": "30s",
+                    "actions": {
                     }
+                  }
                 }
+              }
             }""";
 
         prevState = updatedState;
@@ -204,15 +204,15 @@ public void testActionAddRemove() throws Exception {
 
         String onePolicyRemovedJSON = """
             {
-                "my_timeseries_lifecycle": {
-                    "phases": {
-                        "warm": {
-                            "min_age": "10s",
-                            "actions": {
-                            }
-                        }
+              "my_timeseries_lifecycle": {
+                "phases": {
+                  "warm": {
+                    "min_age": "10s",
+                    "actions": {
                     }
+                  }
                 }
+              }
             }""";
 
         prevState = updatedState;
@@ -223,15 +223,15 @@ public void testActionAddRemove() throws Exception {
 
         String onePolicyRenamedJSON = """
             {
-                "my_timeseries_lifecycle2": {
-                    "phases": {
-                        "warm": {
-                            "min_age": "10s",
-                            "actions": {
-                            }
-                        }
+              "my_timeseries_lifecycle2": {
+                "phases": {
+                  "warm": {
+                    "min_age": "10s",
+                    "actions": {
                     }
+                  }
                 }
+              }
             }""";
 
         prevState = updatedState;
@@ -301,55 +301,55 @@ public void testOperatorControllerFromJSONContent() throws IOException {
 
         String testJSON = """
             {
-                 "metadata": {
-                     "version": "1234",
-                     "compatibility": "8.4.0"
-                 },
-                 "state": {
-                     "cluster_settings": {
-                         "indices.recovery.max_bytes_per_sec": "50mb"
-                     },
-                     "ilm": {
-                         "my_timeseries_lifecycle": {
-                             "phases": {
-                                 "hot": {
-                                     "min_age": "10s",
-                                     "actions": {
-                                        "rollover": {
-                                           "max_primary_shard_size": "50gb",
-                                           "max_age": "30d"
-                                        }
-                                     }
-                                 },
-                                 "delete": {
-                                     "min_age": "30s",
-                                     "actions": {
-                                     }
-                                 }
-                             }
-                         },
-                         "my_timeseries_lifecycle1": {
-                             "phases": {
-                                 "warm": {
-                                     "min_age": "10s",
-                                     "actions": {
-                                        "shrink": {
-                                          "number_of_shards": 1
-                                        },
-                                        "forcemerge": {
-                                          "max_num_segments": 1
-                                        }
-                                     }
-                                 },
-                                 "delete": {
-                                     "min_age": "30s",
-                                     "actions": {
-                                     }
-                                 }
-                             }
-                         }
-                     }
-                 }
+              "metadata": {
+                "version": "1234",
+                "compatibility": "8.4.0"
+              },
+              "state": {
+                "cluster_settings": {
+                  "indices.recovery.max_bytes_per_sec": "50mb"
+                },
+                "ilm": {
+                  "my_timeseries_lifecycle": {
+                    "phases": {
+                      "hot": {
+                        "min_age": "10s",
+                        "actions": {
+                           "rollover": {
+                              "max_primary_shard_size": "50gb",
+                              "max_age": "30d"
+                           }
+                        }
+                      },
+                      "delete": {
+                        "min_age": "30s",
+                        "actions": {
+                        }
+                      }
+                    }
+                  },
+                  "my_timeseries_lifecycle1": {
+                    "phases": {
+                      "warm": {
+                        "min_age": "10s",
+                        "actions": {
+                          "shrink": {
+                            "number_of_shards": 1
+                          },
+                          "forcemerge": {
+                            "max_num_segments": 1
+                          }
+                        }
+                      },
+                      "delete": {
+                        "min_age": "30s",
+                        "actions": {
+                        }
+                      }
+                    }
+                  }
+                }
+              }
             }""";
 
         AtomicReference<Exception> x = new AtomicReference<>();
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleActionTests.java
index d2f88106b2c6..f2f09b5e62c7 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleActionTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleActionTests.java
@@ -37,18 +37,18 @@ public class TransportExplainLifecycleActionTests extends ESTestCase {
 
     public static final String PHASE_DEFINITION = """
         {
-                "policy" : "my-policy",
-                "phase_definition" : {
-                  "min_age" : "20m",
-                  "actions" : {
-                    "rollover" : {
-                      "max_age" : "5s"
-                    }
-                  }
-                },
-                "version" : 1,
-                "modified_date_in_millis" : 1578521007076
-              }""";
+          "policy" : "my-policy",
+          "phase_definition" : {
+            "min_age" : "20m",
+            "actions" : {
+              "rollover" : {
+                "max_age" : "5s"
+              }
+            }
+          },
+          "version" : 1,
+          "modified_date_in_millis" : 1578521007076
+        }""";
 
     private static final NamedXContentRegistry REGISTRY;
 
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java
index d53f91c04aae..ef1d65f2e495 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java
@@ -60,15 +60,15 @@ public void testReservedStateHandler() throws Exception {
 
         String json = """
             {
-                "policy": {
-                    "phases": {
-                        "warm": {
-                            "min_age": "10s",
-                            "actions": {
-                            }
-                        }
+              "policy": {
+                "phases": {
+                  "warm": {
+                    "min_age": "10s",
+                    "actions": {
                     }
+                  }
                 }
+              }
             }""";
 
         try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, json)) {
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStoreTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStoreTests.java
index c0b320100321..8f8e0451cdbb 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStoreTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/history/ILMHistoryStoreTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
@@ -29,7 +30,9 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.client.NoOpClient;
@@ -40,10 +43,12 @@
 import org.junit.After;
 import org.junit.Before;
 
+import java.util.List;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.IntStream;
 
 import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED_SETTING;
@@ -81,7 +86,7 @@ public void setup() {
                 .metadata(Metadata.builder(state.metadata()).indexTemplates(registry.getComposableTemplateConfigs()))
                 .build()
         );
-        historyStore = new ILMHistoryStore(client, clusterService, threadPool);
+        historyStore = new ILMHistoryStore(client, clusterService, threadPool, ActionListener.noop(), TimeValue.timeValueMillis(500));
     }
 
     @After
@@ -109,7 +114,7 @@ public void testNoActionIfDisabled() throws Exception {
             return null;
         });
         historyStore.putAsync(record);
-        latch.await(10, TimeUnit.SECONDS);
+        assertFalse(latch.await(2, TimeUnit.SECONDS));
     }
 
     public void testPut() throws Exception {
@@ -205,6 +210,81 @@ public void testPut() throws Exception {
         }
     }
 
+    /*
+     * This tests that things behave correctly if we throw a lot of data at the ILMHistoryStore quickly -- multiple flushes occur, all
+     * counts add up, and no deadlock occurs.
+     */
+    // @TestLogging(
+    // value = "org.elasticsearch.action.bulk:trace",
+    // reason = "Logging information about locks useful for tracking down deadlock"
+    // )
+    public void testMultipleFlushes() throws Exception {
+        String policyId = randomAlphaOfLength(5);
+        final long timestamp = randomNonNegativeLong();
+        AtomicLong actions = new AtomicLong(0);
+        long numberOfDocs = 400_000;
+        CountDownLatch latch = new CountDownLatch((int) numberOfDocs);
+        client.setVerifier((action, request, listener) -> {
+            assertThat(action, instanceOf(BulkAction.class));
+            assertThat(request, instanceOf(BulkRequest.class));
+            BulkRequest bulkRequest = (BulkRequest) request;
+            List<DocWriteRequest<?>> realRequests = bulkRequest.requests();
+            realRequests.forEach(dwr -> assertEquals(ILM_HISTORY_DATA_STREAM, dwr.index()));
+            assertNotNull(listener);
+
+            // The content of this BulkResponse doesn't matter, so just make it have the same number of responses
+            int responses = bulkRequest.numberOfActions();
+            BulkItemResponse.Failure failure = new BulkItemResponse.Failure(
+                "index",
+                "1",
+                new Exception("message"),
+                RestStatus.TOO_MANY_REQUESTS
+            );
+            DocWriteResponse response = new IndexResponse(new ShardId("index", "indexUUID", 1), "1", 1L, 1L, 1L, true);
+            BulkResponse bulkItemResponse = new BulkResponse(
+                IntStream.range(0, responses)
+                    .mapToObj(
+                        i -> randomBoolean()
+                            ? BulkItemResponse.success(i, DocWriteRequest.OpType.INDEX, response)
+                            : BulkItemResponse.failure(i, DocWriteRequest.OpType.INDEX, failure)
+                    )
+                    .toArray(BulkItemResponse[]::new),
+                1000L
+            );
+            return bulkItemResponse;
+        });
+        try (ILMHistoryStore localHistoryStore = new ILMHistoryStore(client, clusterService, threadPool, new ActionListener<>() {
+            @Override
+            public void onResponse(BulkResponse response) {
+                int itemsInResponse = response.getItems().length;
+                actions.addAndGet(itemsInResponse);
+                for (int i = 0; i < itemsInResponse; i++) {
+                    latch.countDown();
+                }
+                logger.info("cumulative responses: {}", actions.get());
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                logger.error(e);
+                fail(e.getMessage());
+            }
+        }, TimeValue.timeValueMillis(randomIntBetween(50, 1000)))) {
+            for (int i = 0; i < numberOfDocs; i++) {
+                ILMHistoryItem record1 = ILMHistoryItem.success(
+                    "index",
+                    policyId,
+                    timestamp,
+                    10L,
+                    LifecycleExecutionState.builder().setPhase("phase").build()
+                );
+                localHistoryStore.putAsync(record1);
+            }
+            latch.await(5, TimeUnit.SECONDS);
+            assertThat(actions.get(), equalTo(numberOfDocs));
+        }
+    }
+
     /**
      * A client that delegates to a verifying function for action/request/listener
      */
@@ -238,4 +318,5 @@ VerifyingClient setVerifier(TriFunction<ActionType<?>, ActionRequest, ActionList
             return this;
         }
     }
+
 }
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorServiceTests.java
index 089413d7b045..0dfd62cebfba 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorServiceTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SlmHealthIndicatorServiceTests.java
@@ -43,6 +43,7 @@
 import static org.elasticsearch.xpack.core.ilm.OperationMode.STOPPING;
 import static org.elasticsearch.xpack.slm.SlmHealthIndicatorService.NAME;
 import static org.elasticsearch.xpack.slm.SlmHealthIndicatorService.SLM_NOT_RUNNING;
+import static org.elasticsearch.xpack.slm.SlmHealthIndicatorService.checkRecentlyFailedSnapshots;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.mockito.Mockito.mock;
@@ -223,7 +224,7 @@ public void testIsYellowWhenPoliciesHaveFailedForMoreThanWarningThreshold() {
                                     + "] repeated failures without successful execution since ["
                                     + FORMATTER.formatMillis(execTime)
                                     + "]",
-                                "Check the snapshot lifecycle policy for detailed failure info:\n- /_slm/policy/policy-id?human"
+                                "Check the snapshot lifecycle policy for detailed failure info:\n- GET /_slm/policy/policy-id?human"
                             ),
                             List.of(new Diagnosis.Resource(Type.SLM_POLICY, List.of("test-policy")))
                         )
@@ -283,6 +284,17 @@ public void testSnapshotPolicyExceedsWarningThresholdPredicate() {
         assertThat(SlmHealthIndicatorService.snapshotFailuresExceedWarningCount(1L, slmPolicyMetadata), is(false));
     }
 
+    // We expose the indicator name and the diagnoses in the x-pack usage API. In order to index them properly in a telemetry index
+    // they need to be declared in the health-api-indexer.edn in the telemetry repository.
+    public void testMappedFieldsForTelemetry() {
+        assertThat(SlmHealthIndicatorService.NAME, equalTo("slm"));
+        assertThat(
+            checkRecentlyFailedSnapshots("cause", "action").getUniqueId(),
+            equalTo("elasticsearch:health:slm:diagnosis:check_recent_snapshot_failures")
+        );
+        assertThat(SLM_NOT_RUNNING.definition().getUniqueId(), equalTo("elasticsearch:health:slm:diagnosis:slm_disabled"));
+    }
+
     private static ClusterState createClusterStateWith(SnapshotLifecycleMetadata metadata) {
         var builder = new ClusterState.Builder(new ClusterName("test-cluster"));
         if (metadata != null) {
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java
index f42f120f27c5..e6c6fcf9adff 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTaskTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
@@ -146,28 +147,28 @@ public void testCreateSnapshotOnTrigger() {
             Settings.EMPTY,
             Sets.union(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS, Set.of(SLM_HISTORY_INDEX_ENABLED_SETTING))
         );
-        final String createSnapResponse = formatted("""
+        final String createSnapResponse = Strings.format("""
             {
-                "snapshot": {
-                    "snapshot": "snapshot_1",
-                    "uuid": "bcP3ClgCSYO_TP7_FCBbBw",
-                    "version_id": %s,
-                    "version": "%s",
-                    "indices": [],
-                    "include_global_state": true,
-                    "state": "SUCCESS",
-                    "start_time": "2019-03-19T22:19:53.542Z",
-                    "start_time_in_millis": 1553033993542,
-                    "end_time": "2019-03-19T22:19:53.567Z",
-                    "end_time_in_millis": 1553033993567,
-                    "duration_in_millis": 25,
-                    "failures": [],
-                    "shards": {
-                        "total": 0,
-                        "failed": 0,
-                        "successful": 0
-                    }
+              "snapshot": {
+                "snapshot": "snapshot_1",
+                "uuid": "bcP3ClgCSYO_TP7_FCBbBw",
+                "version_id": %s,
+                "version": "%s",
+                "indices": [],
+                "include_global_state": true,
+                "state": "SUCCESS",
+                "start_time": "2019-03-19T22:19:53.542Z",
+                "start_time_in_millis": 1553033993542,
+                "end_time": "2019-03-19T22:19:53.567Z",
+                "end_time_in_millis": 1553033993567,
+                "duration_in_millis": 25,
+                "failures": [],
+                "shards": {
+                  "total": 0,
+                  "failed": 0,
+                  "successful": 0
                 }
+              }
             }""", Version.CURRENT.id, Version.CURRENT);
 
         final AtomicBoolean clientCalled = new AtomicBoolean(false);
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java
index e180c2971a9b..46618be1a4bf 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/action/ReservedSnapshotLifecycleStateServiceTests.java
@@ -81,21 +81,21 @@ public void testValidationFails() {
 
         String badPolicyJSON = """
             {
-                "daily-snapshots": {
-                    "no_schedule": "0 1 2 3 4 ?",
-                    "name": "<production-snap-{now/d}>",
-                    "repository": "repo",
-                    "config": {
-                        "indices": ["foo-*", "important"],
-                        "ignore_unavailable": true,
-                        "include_global_state": false
-                    },
-                    "retention": {
-                        "expire_after": "30d",
-                        "min_count": 1,
-                        "max_count": 50
-                    }
+              "daily-snapshots": {
+                "no_schedule": "0 1 2 3 4 ?",
+                "name": "<production-snap-{now/d}>",
+                "repository": "repo",
+                "config": {
+                  "indices": ["foo-*", "important"],
+                  "ignore_unavailable": true,
+                  "include_global_state": false
+                },
+                "retention": {
+                  "expire_after": "30d",
+                  "min_count": 1,
+                  "max_count": 50
                 }
+              }
             }""";
 
         assertEquals(
@@ -127,36 +127,36 @@ public void testActionAddRemove() throws Exception {
 
         String twoPoliciesJSON = """
             {
-                "daily-snapshots": {
-                    "schedule": "0 1 2 3 4 ?",
-                    "name": "<production-snap-{now/d}>",
-                    "repository": "repo",
-                    "config": {
-                        "indices": ["foo-*", "important"],
-                        "ignore_unavailable": true,
-                        "include_global_state": false
-                    },
-                    "retention": {
-                        "expire_after": "30d",
-                        "min_count": 1,
-                        "max_count": 50
-                    }
+              "daily-snapshots": {
+                "schedule": "0 1 2 3 4 ?",
+                "name": "<production-snap-{now/d}>",
+                "repository": "repo",
+                "config": {
+                  "indices": ["foo-*", "important"],
+                  "ignore_unavailable": true,
+                  "include_global_state": false
                 },
-                "daily-snapshots1": {
-                    "schedule": "0 1 2 3 4 ?",
-                    "name": "<production-snap-{now/d}>",
-                    "repository": "repo",
-                    "config": {
-                        "indices": ["bar-*", "not-important"],
-                        "ignore_unavailable": true,
-                        "include_global_state": false
-                    },
-                    "retention": {
-                        "expire_after": "30d",
-                        "min_count": 1,
-                        "max_count": 50
-                    }
+                "retention": {
+                  "expire_after": "30d",
+                  "min_count": 1,
+                  "max_count": 50
+                }
+              },
+              "daily-snapshots1": {
+                "schedule": "0 1 2 3 4 ?",
+                "name": "<production-snap-{now/d}>",
+                "repository": "repo",
+                "config": {
+                  "indices": ["bar-*", "not-important"],
+                  "ignore_unavailable": true,
+                  "include_global_state": false
+                },
+                "retention": {
+                  "expire_after": "30d",
+                  "min_count": 1,
+                  "max_count": 50
                 }
+              }
             }""";
 
         prevState = updatedState;
@@ -167,21 +167,21 @@ public void testActionAddRemove() throws Exception {
 
         String onePolicyRemovedJSON = """
             {
-                "daily-snapshots": {
-                    "schedule": "0 1 2 3 4 ?",
-                    "name": "<production-snap-{now/d}>",
-                    "repository": "repo",
-                    "config": {
-                        "indices": ["foo-*", "important"],
-                        "ignore_unavailable": true,
-                        "include_global_state": false
-                    },
-                    "retention": {
-                        "expire_after": "30d",
-                        "min_count": 1,
-                        "max_count": 50
-                    }
+              "daily-snapshots": {
+                "schedule": "0 1 2 3 4 ?",
+                "name": "<production-snap-{now/d}>",
+                "repository": "repo",
+                "config": {
+                  "indices": ["foo-*", "important"],
+                  "ignore_unavailable": true,
+                  "include_global_state": false
+                },
+                "retention": {
+                  "expire_after": "30d",
+                  "min_count": 1,
+                  "max_count": 50
                 }
+              }
             }""";
 
         prevState = updatedState;
@@ -192,21 +192,21 @@ public void testActionAddRemove() throws Exception {
 
         String onePolicyRenamedJSON = """
             {
-                "daily-snapshots-2": {
-                    "schedule": "0 1 2 3 4 ?",
-                    "name": "<production-snap-{now/d}>",
-                    "repository": "repo",
-                    "config": {
-                        "indices": ["foo-*", "important"],
-                        "ignore_unavailable": true,
-                        "include_global_state": false
-                    },
-                    "retention": {
-                        "expire_after": "30d",
-                        "min_count": 1,
-                        "max_count": 50
-                    }
+              "daily-snapshots-2": {
+                "schedule": "0 1 2 3 4 ?",
+                "name": "<production-snap-{now/d}>",
+                "repository": "repo",
+                "config": {
+                  "indices": ["foo-*", "important"],
+                  "ignore_unavailable": true,
+                  "include_global_state": false
+                },
+                "retention": {
+                  "expire_after": "30d",
+                  "min_count": 1,
+                  "max_count": 50
                 }
+              }
             }""";
 
         prevState = updatedState;
@@ -278,55 +278,55 @@ public void testOperatorControllerFromJSONContent() throws IOException {
 
         String testJSON = """
             {
-                 "metadata": {
-                     "version": "1234",
-                     "compatibility": "8.4.0"
-                 },
-                 "state": {
-                     "cluster_settings": {
-                         "indices.recovery.max_bytes_per_sec": "50mb"
-                     },
-                     "snapshot_repositories": {
-                        "repo": {
-                           "type": "fs",
-                           "settings": {
-                              "location": "my_backup_location"
-                           }
-                        }
-                     },
-                     "slm": {
-                        "daily-snapshots": {
-                            "schedule": "0 1 2 3 4 ?",
-                            "name": "<production-snap-{now/d}>",
-                            "repository": "repo",
-                            "config": {
-                                "indices": ["foo-*", "important"],
-                                "ignore_unavailable": true,
-                                "include_global_state": false
-                            },
-                            "retention": {
-                                "expire_after": "30d",
-                                "min_count": 1,
-                                "max_count": 50
-                            }
-                        },
-                        "daily-snapshots1": {
-                            "schedule": "0 1 2 3 4 ?",
-                            "name": "<production-snap-{now/d}>",
-                            "repository": "repo",
-                            "config": {
-                                "indices": ["bar-*", "not-important"],
-                                "ignore_unavailable": true,
-                                "include_global_state": false
-                            },
-                            "retention": {
-                                "expire_after": "30d",
-                                "min_count": 1,
-                                "max_count": 50
-                            }
-                        }
+              "metadata": {
+                "version": "1234",
+                "compatibility": "8.4.0"
+              },
+              "state": {
+                "cluster_settings": {
+                  "indices.recovery.max_bytes_per_sec": "50mb"
+                },
+                "snapshot_repositories": {
+                  "repo": {
+                    "type": "fs",
+                    "settings": {
+                      "location": "my_backup_location"
                     }
-                 }
+                  }
+                },
+                "slm": {
+                  "daily-snapshots": {
+                    "schedule": "0 1 2 3 4 ?",
+                    "name": "<production-snap-{now/d}>",
+                    "repository": "repo",
+                    "config": {
+                      "indices": ["foo-*", "important"],
+                      "ignore_unavailable": true,
+                      "include_global_state": false
+                    },
+                    "retention": {
+                      "expire_after": "30d",
+                      "min_count": 1,
+                      "max_count": 50
+                    }
+                  },
+                  "daily-snapshots1": {
+                    "schedule": "0 1 2 3 4 ?",
+                    "name": "<production-snap-{now/d}>",
+                    "repository": "repo",
+                    "config": {
+                      "indices": ["bar-*", "not-important"],
+                      "ignore_unavailable": true,
+                      "include_global_state": false
+                    },
+                    "retention": {
+                      "expire_after": "30d",
+                      "min_count": 1,
+                      "max_count": 50
+                    }
+                  }
+                }
+              }
             }""";
 
         AtomicReference<Exception> x = new AtomicReference<>();
@@ -387,19 +387,19 @@ public void testPutSLMReservedStateHandler() throws Exception {
 
         String json = """
             {
-                "schedule": "0 1 2 3 4 ?",
-                "name": "<production-snap-{now/d}>",
-                "repository": "repo",
-                "config": {
-                    "indices": ["foo-*", "important"],
-                    "ignore_unavailable": true,
-                    "include_global_state": false
-                },
-                "retention": {
-                    "expire_after": "30d",
-                    "min_count": 1,
-                    "max_count": 50
-                }
+              "schedule": "0 1 2 3 4 ?",
+              "name": "<production-snap-{now/d}>",
+              "repository": "repo",
+              "config": {
+                "indices": ["foo-*", "important"],
+                "ignore_unavailable": true,
+                "include_global_state": false
+              },
+              "retention": {
+                "expire_after": "30d",
+                "min_count": 1,
+                "max_count": 50
+              }
             }""";
 
         try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, json)) {
diff --git a/x-pack/plugin/logstash/build.gradle b/x-pack/plugin/logstash/build.gradle
index 5ad1bcae1f1e..b8bfe255e152 100644
--- a/x-pack/plugin/logstash/build.gradle
+++ b/x-pack/plugin/logstash/build.gradle
@@ -1,5 +1,5 @@
 apply plugin: 'elasticsearch.internal-es-plugin'
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 esplugin {
   name 'x-pack-logstash'
diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java
index c1b9412f8ced..11d247c99790 100644
--- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java
+++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.common.time.DateMathParser;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
@@ -158,8 +159,9 @@ public static class Builder extends FieldMapper.Builder {
         }, m -> toType(m).defaultMetric, XContentBuilder::field, Objects::toString);
 
         private final Version indexCreatedVersion;
+        private final IndexMode indexMode;
 
-        public Builder(String name, Boolean ignoreMalformedByDefault, Version indexCreatedVersion) {
+        public Builder(String name, Boolean ignoreMalformedByDefault, Version indexCreatedVersion, IndexMode mode) {
             super(name);
             this.ignoreMalformed = Parameter.boolParam(
                 Names.IGNORE_MALFORMED,
@@ -168,9 +170,9 @@ public Builder(String name, Boolean ignoreMalformedByDefault, Version indexCreat
                 ignoreMalformedByDefault
             );
 
-            this.timeSeriesMetric = TimeSeriesParams.metricParam(m -> toType(m).metricType, MetricType.gauge);
-
+            this.timeSeriesMetric = TimeSeriesParams.metricParam(m -> toType(m).metricType, MetricType.GAUGE);
             this.indexCreatedVersion = Objects.requireNonNull(indexCreatedVersion);
+            this.indexMode = mode;
         }
 
         @Override
@@ -218,7 +220,8 @@ public AggregateDoubleMetricFieldMapper build(MapperBuilderContext context) {
                         ScriptCompiler.NONE,
                         false,
                         false,
-                        indexCreatedVersion
+                        indexCreatedVersion,
+                        indexMode
                     ).allowMultipleValues(false);
                 } else {
                     builder = new NumberFieldMapper.Builder(
@@ -227,7 +230,8 @@ public AggregateDoubleMetricFieldMapper build(MapperBuilderContext context) {
                         ScriptCompiler.NONE,
                         false,
                         true,
-                        indexCreatedVersion
+                        indexCreatedVersion,
+                        indexMode
                     ).allowMultipleValues(false);
                 }
                 NumberFieldMapper fieldMapper = builder.build(context);
@@ -258,7 +262,7 @@ public AggregateDoubleMetricFieldMapper build(MapperBuilderContext context) {
     }
 
     public static final FieldMapper.TypeParser PARSER = new TypeParser(
-        (n, c) -> new Builder(n, IGNORE_MALFORMED_SETTING.get(c.getSettings()), c.indexVersionCreated()),
+        (n, c) -> new Builder(n, IGNORE_MALFORMED_SETTING.get(c.getSettings()), c.indexVersionCreated(), c.getIndexSettings().getMode()),
         notInMultiFields(CONTENT_TYPE)
     );
 
@@ -523,6 +527,8 @@ public MetricType getMetricType() {
     /** The metric type (gauge, counter, summary) if  field is a time series metric */
     private final TimeSeriesParams.MetricType metricType;
 
+    private final IndexMode indexMode;
+
     private AggregateDoubleMetricFieldMapper(
         String simpleName,
         MappedFieldType mappedFieldType,
@@ -537,6 +543,7 @@ private AggregateDoubleMetricFieldMapper(
         this.metricFieldMappers = metricFieldMappers;
         this.metricType = builder.timeSeriesMetric.getValue();
         this.indexCreatedVersion = builder.indexCreatedVersion;
+        this.indexMode = builder.indexMode;
     }
 
     @Override
@@ -661,7 +668,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(simpleName(), ignoreMalformedByDefault, indexCreatedVersion).metric(metricType).init(this);
+        return new Builder(simpleName(), ignoreMalformedByDefault, indexCreatedVersion, indexMode).metric(metricType).init(this);
     }
 
     @Override
diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java
index 40cfc437e633..e87df85acb63 100644
--- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java
+++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java
@@ -110,7 +110,7 @@ public void testParseValue() throws Exception {
         ParsedDocument doc = mapper.parse(
             source(b -> b.startObject("field").field("min", -10.1).field("max", 50.0).field("value_count", 14).endObject())
         );
-        assertEquals(-10.1, doc.rootDoc().getField("field.min").numericValue());
+        assertEquals("DoubleField <field.min:-10.1>", doc.rootDoc().getField("field.min").toString());
 
         Mapper fieldMapper = mapper.mappers().getMapper("field");
         assertThat(fieldMapper, instanceOf(AggregateDoubleMetricFieldMapper.class));
diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java
index 140f6ecc922a..9fcc24a12d8d 100644
--- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java
+++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.script.ScoreScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.lookup.SourceLookup;
 import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.AggregateDoubleMetricFieldType;
 import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Metric;
 
@@ -127,7 +126,7 @@ public void testUsedInScript() throws IOException {
                     (mft, lookupSupplier, fdo) -> mft.fielddataBuilder(
                         new FieldDataContext("test", lookupSupplier, searchExecutionContext::sourcePath, fdo)
                     ).build(null, null),
-                    new SourceLookup.ReaderSourceProvider()
+                    (ctx, doc) -> null
                 );
                 when(searchExecutionContext.lookup()).thenReturn(lookup);
                 IndexSearcher searcher = newSearcher(reader);
diff --git a/x-pack/plugin/mapper-constant-keyword/build.gradle b/x-pack/plugin/mapper-constant-keyword/build.gradle
index fe57cdc4380f..ffcbc8ed45be 100644
--- a/x-pack/plugin/mapper-constant-keyword/build.gradle
+++ b/x-pack/plugin/mapper-constant-keyword/build.gradle
@@ -1,7 +1,7 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 
 apply plugin: 'elasticsearch.internal-es-plugin'
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 
 esplugin {
   name 'constant-keyword'
diff --git a/x-pack/plugin/mapper-unsigned-long/build.gradle b/x-pack/plugin/mapper-unsigned-long/build.gradle
index eaff7fe36255..adead5dc126b 100644
--- a/x-pack/plugin/mapper-unsigned-long/build.gradle
+++ b/x-pack/plugin/mapper-unsigned-long/build.gradle
@@ -11,8 +11,8 @@ import org.elasticsearch.gradle.internal.info.BuildParams
 evaluationDependsOn(xpackModule('core'))
 
 apply plugin: 'elasticsearch.internal-es-plugin'
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 esplugin {
   name 'unsigned-long'
diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/SourceValueFetcherSortedUnsignedLongIndexFieldData.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/SourceValueFetcherSortedUnsignedLongIndexFieldData.java
index 3f9a0654fa9d..b74cb64f708c 100644
--- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/SourceValueFetcherSortedUnsignedLongIndexFieldData.java
+++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/SourceValueFetcherSortedUnsignedLongIndexFieldData.java
@@ -16,7 +16,8 @@
 import org.elasticsearch.script.field.DocValuesScriptFieldFactory;
 import org.elasticsearch.script.field.ToScriptFieldFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.search.lookup.SourceProvider;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -40,10 +41,10 @@ public Builder(
             String fieldName,
             ValuesSourceType valuesSourceType,
             ValueFetcher valueFetcher,
-            SourceLookup sourceLookup,
+            SourceProvider sourceProvider,
             ToScriptFieldFactory<SortedNumericDocValues> toScriptFieldFactory
         ) {
-            super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory);
+            super(fieldName, valuesSourceType, valueFetcher, sourceProvider, toScriptFieldFactory);
         }
 
         @Override
@@ -52,7 +53,7 @@ public SourceValueFetcherSortedUnsignedLongIndexFieldData build(IndexFieldDataCa
                 fieldName,
                 valuesSourceType,
                 valueFetcher,
-                sourceLookup,
+                sourceProvider,
                 toScriptFieldFactory
             );
         }
@@ -62,15 +63,15 @@ protected SourceValueFetcherSortedUnsignedLongIndexFieldData(
         String fieldName,
         ValuesSourceType valuesSourceType,
         ValueFetcher valueFetcher,
-        SourceLookup sourceLookup,
+        SourceProvider sourceProvider,
         ToScriptFieldFactory<SortedNumericDocValues> toScriptFieldFactory
     ) {
-        super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory);
+        super(fieldName, valuesSourceType, valueFetcher, sourceProvider, toScriptFieldFactory);
     }
 
     @Override
     public SourceValueFetcherLeafFieldData<SortedNumericDocValues> loadDirect(LeafReaderContext context) {
-        return new SourceValueFetcherSortedUnsignedLongLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceLookup);
+        return new SourceValueFetcherSortedUnsignedLongLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceProvider);
     }
 
     private static class SourceValueFetcherSortedUnsignedLongLeafFieldData extends SourceValueFetcherLeafFieldData<SortedNumericDocValues> {
@@ -79,15 +80,15 @@ private SourceValueFetcherSortedUnsignedLongLeafFieldData(
             ToScriptFieldFactory<SortedNumericDocValues> toScriptFieldFactory,
             LeafReaderContext leafReaderContext,
             ValueFetcher valueFetcher,
-            SourceLookup sourceLookup
+            SourceProvider sourceProvider
         ) {
-            super(toScriptFieldFactory, leafReaderContext, valueFetcher, sourceLookup);
+            super(toScriptFieldFactory, leafReaderContext, valueFetcher, sourceProvider);
         }
 
         @Override
         public DocValuesScriptFieldFactory getScriptFieldFactory(String name) {
             return toScriptFieldFactory.getScriptFieldFactory(
-                new SourceValueFetcherSortedUnsignedLongDocValues(leafReaderContext, valueFetcher, sourceLookup),
+                new SourceValueFetcherSortedUnsignedLongDocValues(leafReaderContext, valueFetcher, sourceProvider),
                 name
             );
         }
@@ -98,7 +99,7 @@ private static class SourceValueFetcherSortedUnsignedLongDocValues extends Sorte
         private final LeafReaderContext leafReaderContext;
 
         private final ValueFetcher valueFetcher;
-        private final SourceLookup sourceLookup;
+        private final SourceProvider sourceProvider;
 
         private List<Long> values;
         private Iterator<Long> iterator;
@@ -106,21 +107,20 @@ private static class SourceValueFetcherSortedUnsignedLongDocValues extends Sorte
         private SourceValueFetcherSortedUnsignedLongDocValues(
             LeafReaderContext leafReaderContext,
             ValueFetcher valueFetcher,
-            SourceLookup sourceLookup
+            SourceProvider sourceProvider
         ) {
             this.leafReaderContext = leafReaderContext;
             this.valueFetcher = valueFetcher;
-            this.sourceLookup = sourceLookup;
+            this.sourceProvider = sourceProvider;
 
             values = new ArrayList<>();
         }
 
         @Override
         public boolean advanceExact(int doc) throws IOException {
-            sourceLookup.setSegmentAndDocument(leafReaderContext, doc);
             values.clear();
-
-            for (Object value : valueFetcher.fetchValues(sourceLookup, doc, Collections.emptyList())) {
+            Source source = sourceProvider.getSource(leafReaderContext, doc);
+            for (Object value : valueFetcher.fetchValues(source, doc, Collections.emptyList())) {
                 assert value instanceof Number;
                 values.add(((Number) value).longValue() ^ MASK_2_63);
             }
diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java
index 97ff720c6c14..3db75f6e723d 100644
--- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java
+++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java
@@ -8,16 +8,18 @@
 package org.elasticsearch.xpack.unsignedlong;
 
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.LongField;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StoredField;
-import org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
+import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
@@ -29,6 +31,8 @@
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.mapper.SimpleMappedFieldType;
+import org.elasticsearch.index.mapper.SortedNumericDocValuesSyntheticFieldLoader;
+import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.mapper.SourceValueFetcher;
 import org.elasticsearch.index.mapper.TextSearchInfo;
 import org.elasticsearch.index.mapper.TimeSeriesParams;
@@ -70,7 +74,7 @@ private static UnsignedLongFieldMapper toType(FieldMapper in) {
     }
 
     public static class Builder extends FieldMapper.Builder {
-        private final Parameter<Boolean> indexed = Parameter.indexParam(m -> toType(m).indexed, true);
+        private final Parameter<Boolean> indexed;
         private final Parameter<Boolean> hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true);
         private final Parameter<Boolean> stored = Parameter.storeParam(m -> toType(m).stored, false);
         private final Parameter<Explicit<Boolean>> ignoreMalformed;
@@ -89,11 +93,13 @@ public static class Builder extends FieldMapper.Builder {
          */
         private final Parameter<MetricType> metric;
 
-        public Builder(String name, Settings settings) {
-            this(name, IGNORE_MALFORMED_SETTING.get(settings));
+        private final IndexMode indexMode;
+
+        public Builder(String name, Settings settings, IndexMode mode) {
+            this(name, IGNORE_MALFORMED_SETTING.get(settings), mode);
         }
 
-        public Builder(String name, boolean ignoreMalformedByDefault) {
+        public Builder(String name, boolean ignoreMalformedByDefault, IndexMode mode) {
             super(name);
             this.ignoreMalformed = Parameter.explicitBoolParam(
                 "ignore_malformed",
@@ -110,7 +116,15 @@ public Builder(String name, boolean ignoreMalformedByDefault) {
                 XContentBuilder::field,
                 Objects::toString
             ).acceptsNull();
-
+            this.indexMode = mode;
+            this.indexed = Parameter.indexParam(m -> toType(m).indexed, () -> {
+                if (indexMode == IndexMode.TIME_SERIES) {
+                    var metricType = getMetric().getValue();
+                    return metricType != MetricType.COUNTER && metricType != MetricType.GAUGE;
+                } else {
+                    return true;
+                }
+            });
             this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension).addValidator(v -> {
                 if (v && (indexed.getValue() == false || hasDocValues.getValue() == false)) {
                     throw new IllegalArgumentException(
@@ -125,7 +139,7 @@ public Builder(String name, boolean ignoreMalformedByDefault) {
                 }
             });
 
-            this.metric = TimeSeriesParams.metricParam(m -> toType(m).metricType, MetricType.gauge, MetricType.counter).addValidator(v -> {
+            this.metric = TimeSeriesParams.metricParam(m -> toType(m).metricType, MetricType.GAUGE, MetricType.COUNTER).addValidator(v -> {
                 if (v != null && hasDocValues.getValue() == false) {
                     throw new IllegalArgumentException(
                         "Field [" + TimeSeriesParams.TIME_SERIES_METRIC_PARAM + "] requires that [" + hasDocValues.name + "] is true"
@@ -159,6 +173,10 @@ public Builder metric(MetricType metric) {
             return this;
         }
 
+        private Parameter<MetricType> getMetric() {
+            return metric;
+        }
+
         @Override
         protected Parameter<?>[] getParameters() {
             return new Parameter<?>[] { indexed, hasDocValues, stored, ignoreMalformed, nullValue, meta, dimension, metric };
@@ -188,7 +206,7 @@ public UnsignedLongFieldMapper build(MapperBuilderContext context) {
         }
     }
 
-    public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getSettings()));
+    public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getSettings(), c.getIndexSettings().getMode()));
 
     public static final class UnsignedLongFieldType extends SimpleMappedFieldType {
 
@@ -298,7 +316,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext
                 failIfNoDocValues();
             }
 
-            ValuesSourceType valuesSourceType = metricType == TimeSeriesParams.MetricType.counter
+            ValuesSourceType valuesSourceType = metricType == TimeSeriesParams.MetricType.COUNTER
                 ? TimeSeriesValuesSourceType.COUNTER
                 : IndexNumericFieldData.NumericType.LONG.getValuesSourceType();
 
@@ -322,7 +340,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext
                     name(),
                     valuesSourceType,
                     sourceValueFetcher(sourcePaths),
-                    searchLookup.source(),
+                    searchLookup,
                     UnsignedLongDocValuesField::new
                 );
             }
@@ -510,6 +528,7 @@ public MetricType getMetricType() {
     private final Long nullValueIndexed; // null value to use for indexing, represented as shifted to signed long range
     private final boolean dimension;
     private final MetricType metricType;
+    private final IndexMode indexMode;
 
     private UnsignedLongFieldMapper(
         String simpleName,
@@ -533,6 +552,7 @@ private UnsignedLongFieldMapper(
         }
         this.dimension = builder.dimension.getValue();
         this.metricType = builder.metric.getValue();
+        this.indexMode = builder.indexMode;
     }
 
     @Override
@@ -588,11 +608,12 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
         }
 
         List<Field> fields = new ArrayList<>();
-        if (indexed) {
-            fields.add(new LongPoint(fieldType().name(), numericValue));
-        }
-        if (hasDocValues) {
+        if (indexed && hasDocValues) {
+            fields.add(new LongField(fieldType().name(), numericValue));
+        } else if (hasDocValues) {
             fields.add(new SortedNumericDocValuesField(fieldType().name(), numericValue));
+        } else if (indexed) {
+            fields.add(new LongPoint(fieldType().name(), numericValue));
         }
         if (stored) {
             // for stored field, keeping original unsigned_long value in the String form
@@ -608,7 +629,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(simpleName(), ignoreMalformedByDefault).dimension(dimension).metric(metricType).init(this);
+        return new Builder(simpleName(), ignoreMalformedByDefault, indexMode).dimension(dimension).metric(metricType).init(this);
     }
 
     /**
@@ -688,4 +709,29 @@ public void doValidate(MappingLookup lookup) {
             );
         }
     }
+
+    @Override
+    public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+        if (hasDocValues == false) {
+            throw new IllegalArgumentException(
+                "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it doesn't have doc values"
+            );
+        }
+        if (ignoreMalformed.value()) {
+            throw new IllegalArgumentException(
+                "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it ignores malformed numbers"
+            );
+        }
+        if (copyTo.copyToFields().isEmpty() != true) {
+            throw new IllegalArgumentException(
+                "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
+            );
+        }
+        return new SortedNumericDocValuesSyntheticFieldLoader(name(), simpleName(), ignoreMalformed()) {
+            @Override
+            protected void writeValue(XContentBuilder b, long value) throws IOException {
+                b.value(DocValueFormat.UNSIGNED_LONG_SHIFTED.format(value));
+            }
+        };
+    }
 }
diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java
index 76c0df3bb634..5dce9feaf028 100644
--- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java
+++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java
@@ -9,13 +9,18 @@
 
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperTestCase;
 import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.TimeSeriesParams;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.junit.AssumptionViolatedException;
@@ -27,6 +32,8 @@
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.matchesPattern;
 
 public class UnsignedLongFieldMapperTests extends MapperTestCase {
 
@@ -67,26 +74,16 @@ public void testDefaults() throws Exception {
         {
             ParsedDocument doc = mapper.parse(source(b -> b.field("field", "18446744073709551615")));
             IndexableField[] fields = doc.rootDoc().getFields("field");
-            assertEquals(2, fields.length);
-            IndexableField pointField = fields[0];
-            assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
-            assertFalse(pointField.fieldType().stored());
-            assertEquals(9223372036854775807L, pointField.numericValue().longValue());
-            IndexableField dvField = fields[1];
-            assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
-            assertEquals(9223372036854775807L, dvField.numericValue().longValue());
-            assertFalse(dvField.fieldType().stored());
+            assertEquals(1, fields.length);
+            assertEquals("LongField <field:9223372036854775807>", fields[0].toString());
         }
 
         // test indexing values as integer numbers
         {
             ParsedDocument doc = mapper.parse(source(b -> b.field("field", 9223372036854775807L)));
             IndexableField[] fields = doc.rootDoc().getFields("field");
-            assertEquals(2, fields.length);
-            IndexableField pointField = fields[0];
-            assertEquals(-1L, pointField.numericValue().longValue());
-            IndexableField dvField = fields[1];
-            assertEquals(-1L, dvField.numericValue().longValue());
+            assertEquals(1, fields.length);
+            assertEquals("LongField <field:-1>", fields[0].toString());
         }
 
         // test that indexing values as number with decimal is not allowed
@@ -122,14 +119,9 @@ public void testStore() throws Exception {
         DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "unsigned_long").field("store", true)));
         ParsedDocument doc = mapper.parse(source(b -> b.field("field", "18446744073709551615")));
         IndexableField[] fields = doc.rootDoc().getFields("field");
-        assertEquals(3, fields.length);
-        IndexableField pointField = fields[0];
-        assertEquals(1, pointField.fieldType().pointIndexDimensionCount());
-        assertEquals(9223372036854775807L, pointField.numericValue().longValue());
-        IndexableField dvField = fields[1];
-        assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
-        assertEquals(9223372036854775807L, dvField.numericValue().longValue());
-        IndexableField storedField = fields[2];
+        assertEquals(2, fields.length);
+        assertEquals("LongField <field:9223372036854775807>", fields[0].toString());
+        IndexableField storedField = fields[1];
         assertTrue(storedField.fieldType().stored());
         assertEquals("18446744073709551615", storedField.stringValue());
     }
@@ -161,11 +153,8 @@ public void testNullValue() throws IOException {
             ParsedDocument doc = mapper.parse(source(b -> b.nullField("field")));
             ;
             IndexableField[] fields = doc.rootDoc().getFields("field");
-            assertEquals(2, fields.length);
-            IndexableField pointField = fields[0];
-            assertEquals(9223372036854775807L, pointField.numericValue().longValue());
-            IndexableField dvField = fields[1];
-            assertEquals(9223372036854775807L, dvField.numericValue().longValue());
+            assertEquals(1, fields.length);
+            assertEquals("LongField <field:9223372036854775807>", fields[0].toString());
         }
     }
 
@@ -197,11 +186,9 @@ public void testDecimalParts() throws IOException {
         }
         ParsedDocument doc = mapper.parse(source(b -> b.field("field", randomFrom("100.", "100.0", "100.00", 100.0, 100.0f))));
         assertThat(doc.rootDoc().getFields("field")[0].numericValue().longValue(), equalTo(Long.MIN_VALUE + 100L));
-        assertThat(doc.rootDoc().getFields("field")[1].numericValue().longValue(), equalTo(Long.MIN_VALUE + 100L));
 
         doc = mapper.parse(source(b -> b.field("field", randomFrom("0.", "0.0", ".00", 0.0, 0.0f))));
         assertThat(doc.rootDoc().getFields("field")[0].numericValue().longValue(), equalTo(Long.MIN_VALUE));
-        assertThat(doc.rootDoc().getFields("field")[1].numericValue().longValue(), equalTo(Long.MIN_VALUE));
     }
 
     public void testIndexingOutOfRangeValues() throws Exception {
@@ -329,6 +316,19 @@ public void testMetricAndDimension() {
         );
     }
 
+    public void testTimeSeriesIndexDefault() throws Exception {
+        var randomMetricType = randomFrom(TimeSeriesParams.MetricType.values());
+        var indexSettings = getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName())
+            .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension_field");
+        var mapperService = createMapperService(indexSettings.build(), fieldMapping(b -> {
+            minimalMapping(b);
+            b.field("time_series_metric", randomMetricType.toString());
+        }));
+        var ft = (UnsignedLongFieldMapper.UnsignedLongFieldType) mapperService.fieldType("field");
+        assertThat(ft.getMetricType(), equalTo(randomMetricType));
+        assertThat(ft.isIndexed(), is(false));
+    }
+
     @Override
     protected Object generateRandomInputValue(MappedFieldType ft) {
         Number n = randomNumericValue();
@@ -354,11 +354,74 @@ private Number randomNumericValue() {
 
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
-        throw new AssumptionViolatedException("not supported");
+        assumeFalse("unsigned_long doesn't support ignore_malformed with synthetic _source", ignoreMalformed);
+        return new NumberSyntheticSourceSupport();
     }
 
     @Override
     protected IngestScriptSupport ingestScriptSupport() {
         throw new AssumptionViolatedException("not supported");
     }
+
+    final class NumberSyntheticSourceSupport implements SyntheticSourceSupport {
+        private final BigInteger nullValue = usually() ? null : BigInteger.valueOf(randomNonNegativeLong());
+
+        @Override
+        public SyntheticSourceExample example(int maxVals) {
+            if (randomBoolean()) {
+                Tuple<Object, Object> v = generateValue();
+                return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping);
+            }
+            List<Tuple<Object, Object>> values = randomList(1, maxVals, this::generateValue);
+            List<Object> in = values.stream().map(Tuple::v1).toList();
+            List<Object> outList = values.stream().map(Tuple::v2).sorted().toList();
+            Object out = outList.size() == 1 ? outList.get(0) : outList;
+            return new SyntheticSourceExample(in, out, this::mapping);
+        }
+
+        private Tuple<Object, Object> generateValue() {
+            if (nullValue != null && randomBoolean()) {
+                return Tuple.tuple(null, nullValue);
+            }
+            long n = randomNonNegativeLong();
+            BigInteger b = BigInteger.valueOf(n);
+            if (b.signum() < 0) {
+                b = b.add(BigInteger.ONE.shiftLeft(64));
+            }
+            return Tuple.tuple(n, b);
+        }
+
+        private void mapping(XContentBuilder b) throws IOException {
+            minimalMapping(b);
+            if (nullValue != null) {
+                b.field("null_value", nullValue);
+            }
+            if (rarely()) {
+                b.field("index", false);
+            }
+            if (rarely()) {
+                b.field("store", false);
+            }
+        }
+
+        @Override
+        public List<SyntheticSourceInvalidExample> invalidExample() {
+            return List.of(
+                new SyntheticSourceInvalidExample(
+                    matchesPattern("field \\[field] of type \\[.+] doesn't support synthetic source because it doesn't have doc values"),
+                    b -> {
+                        minimalMapping(b);
+                        b.field("doc_values", false);
+                    }
+                ),
+                new SyntheticSourceInvalidExample(
+                    matchesPattern("field \\[field] of type \\[.+] doesn't support synthetic source because it ignores malformed numbers"),
+                    b -> {
+                        minimalMapping(b);
+                        b.field("ignore_malformed", true);
+                    }
+                )
+            );
+        }
+    }
 }
diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java
index 37c435b1a7d9..8fb19ee0d950 100644
--- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java
+++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java
@@ -166,13 +166,14 @@ public void testParseUpperTermForRangeQuery() {
     }
 
     public void testFetchSourceValue() throws IOException {
-        MappedFieldType mapper = new UnsignedLongFieldMapper.Builder("field", false).build(MapperBuilderContext.root(false)).fieldType();
+        MappedFieldType mapper = new UnsignedLongFieldMapper.Builder("field", false, null).build(MapperBuilderContext.root(false))
+            .fieldType();
         assertEquals(List.of(0L), fetchSourceValue(mapper, 0L));
         assertEquals(List.of(9223372036854775807L), fetchSourceValue(mapper, 9223372036854775807L));
         assertEquals(List.of(BIGINTEGER_2_64_MINUS_ONE), fetchSourceValue(mapper, "18446744073709551615"));
         assertEquals(List.of(), fetchSourceValue(mapper, ""));
 
-        MappedFieldType nullValueMapper = new UnsignedLongFieldMapper.Builder("field", false).nullValue("18446744073709551615")
+        MappedFieldType nullValueMapper = new UnsignedLongFieldMapper.Builder("field", false, null).nullValue("18446744073709551615")
             .build(MapperBuilderContext.root(false))
             .fieldType();
         assertEquals(List.of(BIGINTEGER_2_64_MINUS_ONE), fetchSourceValue(nullValueMapper, ""));
diff --git a/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/70_time_series.yml b/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/70_time_series.yml
index 0742558f03e4..150c90faf175 100644
--- a/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/70_time_series.yml
+++ b/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/70_time_series.yml
@@ -71,6 +71,32 @@ fetch the _tsid:
   - match: {hits.hits.1.fields.metricset: [aa]}
   - match: {hits.hits.1.fields.ul: [9223372036854775808]}
 
+---
+fetch the _source:
+  - skip:
+      version: " - 8.6.99"
+      reason: synthetic source support to unsigned long added in 8.7.0
+
+  - do:
+      search:
+        index: test
+        body:
+          fields: [_tsid, metricset, ul]
+          query:
+            query_string:
+              query: '+@timestamp:"2021-04-28T18:35:24.467Z" +metricset:aa'
+          sort:   [ _tsid ]
+
+  - match: {hits.total.value: 2}
+  - match: {hits.hits.0._source.voltage: 7.2}
+  - match: {hits.hits.0._source.metricset: aa}
+  - match: {hits.hits.0._source.@timestamp: 2021-04-28T18:35:24.467Z}
+  - match: {hits.hits.0._source.ul: 9223372036854775807}
+  - match: {hits.hits.1._source.voltage: 3.2}
+  - match: {hits.hits.1._source.metricset: aa}
+  - match: {hits.hits.1._source.@timestamp: 2021-04-28T18:35:24.467Z}
+  - match: {hits.hits.1._source.ul: 9223372036854775808}
+
 ---
 # Sort order is of unsigned_long fields is not the one we would expect.
 # This is caused by the encoding of unsigned_long as a signed long before
diff --git a/x-pack/plugin/mapper-version/build.gradle b/x-pack/plugin/mapper-version/build.gradle
index 0558120c9578..437e03a42a4b 100644
--- a/x-pack/plugin/mapper-version/build.gradle
+++ b/x-pack/plugin/mapper-version/build.gradle
@@ -4,8 +4,8 @@ evaluationDependsOn(xpackModule('core'))
 
 
 apply plugin: 'elasticsearch.internal-es-plugin'
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
diff --git a/x-pack/plugin/ml/build.gradle b/x-pack/plugin/ml/build.gradle
index a7ff0a68b58e..06186540bcab 100644
--- a/x-pack/plugin/ml/build.gradle
+++ b/x-pack/plugin/ml/build.gradle
@@ -1,6 +1,3 @@
-import org.elasticsearch.gradle.VersionProperties
-import org.elasticsearch.gradle.internal.dra.DraResolvePlugin
-
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.internal-test-artifact'
@@ -14,42 +11,27 @@ esplugin {
   extendedPlugins = ['x-pack-autoscaling', 'lang-painless']
 }
 
-def localRepo = providers.systemProperty('build.ml_cpp.repo').orNull
 if (useDra == false) {
   repositories {
     exclusiveContent {
-      filter {
-        includeGroup 'org.elasticsearch.ml'
-      }
       forRepository {
         ivy {
           name "ml-cpp"
+          url providers.systemProperty('build.ml_cpp.repo').orElse('https://prelert-artifacts.s3.amazonaws.com').get()
           metadataSources {
             // no repository metadata, look directly for the artifact
             artifact()
           }
-          if (localRepo) {
-            url localRepo
-            patternLayout {
-              artifact "maven/[orgPath]/[module]/[revision]/[module]-[revision](-[classifier]).[ext]"
-            }
-          } else {
-            url "https://artifacts-snapshot.elastic.co/"
-            patternLayout {
-              if (VersionProperties.isElasticsearchSnapshot()) {
-                artifact '/ml-cpp/[revision]/downloads/ml-cpp/[module]-[revision]-[classifier].[ext]'
-              } else {
-                // When building locally we always use snapshot artifacts even if passing `-Dbuild.snapshot=false`.
-                // Release builds are always done with a local repo.
-                artifact '/ml-cpp/[revision]-SNAPSHOT/downloads/ml-cpp/[module]-[revision]-SNAPSHOT-[classifier].[ext]'
-              }
-            }
+          patternLayout {
+            artifact "maven/org/elasticsearch/ml/ml-cpp/[revision]/[module]-[revision](-[classifier]).[ext]"
           }
         }
       }
+      filter {
+        includeGroup 'org.elasticsearch.ml'
+      }
     }
   }
-
 }
 
 configurations {
@@ -130,4 +112,4 @@ tasks.named("dependencyLicenses").configure {
   mapping from: /lucene-.*/, to: 'lucene'
 }
 
-addQaCheckDependencies(project)
\ No newline at end of file
+addQaCheckDependencies(project)
diff --git a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle
index 2234f5e43d51..72793d1f9a41 100644
--- a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle
+++ b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle
@@ -1,6 +1,6 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 testClusters.configureEach {
   testDistribution = 'DEFAULT'
diff --git a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java
index e872ca36fbe7..76b6dd991a41 100644
--- a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java
+++ b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java
@@ -21,7 +21,6 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
@@ -546,7 +545,7 @@ private void createAndIndexFarequote() throws Exception {
         String dateFormat = datesHaveNanoSecondResolution ? "strict_date_optional_time_nanos" : "strict_date_optional_time";
         String randomNanos = datesHaveNanoSecondResolution ? "," + randomIntBetween(100000000, 999999999) : "";
         Request createAirlineDataRequest = new Request("PUT", "/airline-data");
-        createAirlineDataRequest.setJsonEntity(String.format(Locale.ROOT, """
+        createAirlineDataRequest.setJsonEntity(Strings.format("""
             {
               "mappings": {
                 "properties": {
diff --git a/x-pack/plugin/ml/qa/disabled/build.gradle b/x-pack/plugin/ml/qa/disabled/build.gradle
index b0690460115d..97a7b0eed73a 100644
--- a/x-pack/plugin/ml/qa/disabled/build.gradle
+++ b/x-pack/plugin/ml/qa/disabled/build.gradle
@@ -1,6 +1,6 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 //dependencies {
 //  testImplementation project(":x-pack:plugin:core")
diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle
index 478736168d33..333a5cb7f1e7 100644
--- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle
+++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 
 dependencies {
   yamlRestTestImplementation(testArtifact(project(xpackModule('core'))))
@@ -8,7 +8,7 @@ dependencies {
 // bring in machine learning rest test suite
 restResources {
   restApi {
-    include '_common', 'cluster', 'nodes', 'indices', 'index', 'search', 'get', 'count', 'ingest', 'bulk', 'ml', 'cat', 'semantic_search'
+    include '_common', 'cluster', 'nodes', 'indices', 'index', 'search', 'get', 'count', 'ingest', 'bulk', 'ml', 'cat'
   }
   restTests {
     includeXpack 'ml'
@@ -185,6 +185,7 @@ tasks.named("yamlRestTest").configure {
     'ml/jobs_crud/Test put job with time field in analysis_config',
     'ml/jobs_crud/Test put job with duplicate detector configurations',
     'ml/jobs_crud/Test job with categorization_analyzer and categorization_filters',
+    'ml/jobs_crud/Test job with rule referencing missing filter',
     'ml/jobs_get/Test get job given missing job_id',
     'ml/jobs_get_result_buckets/Test mutually-exclusive params',
     'ml/jobs_get_result_buckets/Test mutually-exclusive params via body',
@@ -197,6 +198,7 @@ tasks.named("yamlRestTest").configure {
     'ml/jobs_get_result_overall_buckets/Test overall buckets given invalid start param',
     'ml/jobs_get_result_overall_buckets/Test overall buckets given invalid end param',
     'ml/jobs_get_result_overall_buckets/Test overall buckets given bucket_span is smaller than max job bucket_span',
+    'ml/jobs_get_stats/Test closed results index',
     'ml/jobs_get_stats/Test get job stats given missing job',
     'ml/jobs_get_stats/Test no exception on get job stats with missing index',
     'ml/job_groups/Test put job with empty group',
diff --git a/x-pack/plugin/ml/qa/ml-with-security/roles.yml b/x-pack/plugin/ml/qa/ml-with-security/roles.yml
index cf1d1b118cf2..538c9bbdb49c 100644
--- a/x-pack/plugin/ml/qa/ml-with-security/roles.yml
+++ b/x-pack/plugin/ml/qa/ml-with-security/roles.yml
@@ -7,7 +7,7 @@ minimal:
     # Give all users involved in these tests access to the indices where the data to
     # be analyzed is stored, because the ML roles alone do not provide access to
     # non-ML indices
-    - names: [ 'airline-data', 'index-*', 'unavailable-data', 'utopia', 'store', 'store-flattened', 'embedded_text', 'embedded_text_10_dims' ]
+    - names: [ 'airline-data', 'index-*', 'unavailable-data', 'utopia', 'store', 'store-flattened' ]
       privileges:
         - create_index
         - indices:admin/refresh
diff --git a/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/build.gradle b/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/build.gradle
new file mode 100644
index 000000000000..368aab4615fc
--- /dev/null
+++ b/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/build.gradle
@@ -0,0 +1,63 @@
+import org.elasticsearch.gradle.internal.test.RestIntegTestTask
+import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionProperties
+
+apply plugin: 'elasticsearch.internal-testclusters'
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-resources'
+
+dependencies {
+  testImplementation project(':x-pack:qa')
+}
+
+Version ccsCompatVersion = new Version(VersionProperties.getElasticsearchVersion().getMajor(), VersionProperties.getElasticsearchVersion().getMinor() - 1, 0)
+
+restResources {
+  restApi {
+    include '_common', 'bulk', 'indices', 'cluster', 'search', 'security', 'ml'
+  }
+}
+
+def remoteCluster = testClusters.register('remote-cluster') {
+    testDistribution = 'DEFAULT'
+    versions = [ccsCompatVersion.toString(), project.version]
+    numberOfNodes = 2
+    setting 'node.roles', '[data,ingest,master]'
+    setting 'xpack.security.enabled', 'true'
+    setting 'xpack.watcher.enabled', 'false'
+    setting 'xpack.license.self_generated.type', 'trial'
+
+    user username: "test_user", password: "x-pack-test-password"
+}
+
+testClusters.register('mixed-cluster') {
+    testDistribution = 'DEFAULT'
+    numberOfNodes = 2
+    setting 'node.roles', '[data,ingest,master]'
+    setting 'xpack.security.enabled', 'true'
+    setting 'xpack.watcher.enabled', 'false'
+    setting 'xpack.license.self_generated.type', 'trial'
+    setting 'cluster.remote.my_remote_cluster.seeds', {
+      remoteCluster.get().getAllTransportPortURI().collect { "\"$it\"" }.toString()
+    }
+    setting 'cluster.remote.connections_per_cluster', "1"
+
+    user username: "test_user", password: "x-pack-test-password"
+}
+
+tasks.register('remote-cluster', RestIntegTestTask) {
+  mustRunAfter("precommit")
+  systemProperty 'tests.rest.suite', 'remote_cluster'
+}
+
+tasks.register('mixed-cluster', RestIntegTestTask) {
+  dependsOn 'remote-cluster'
+  useCluster remoteCluster
+  systemProperty 'tests.rest.suite', 'multi_cluster'
+}
+
+tasks.register("integTest") {
+  dependsOn 'mixed-cluster'
+}
+
+tasks.named("check").configure { dependsOn("integTest") }
diff --git a/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/src/test/java/org/elasticsearch/multi_cluster/MultiClusterYamlTestSuiteIT.java b/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/src/test/java/org/elasticsearch/multi_cluster/MultiClusterYamlTestSuiteIT.java
new file mode 100644
index 000000000000..0fbd3d463f7f
--- /dev/null
+++ b/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/src/test/java/org/elasticsearch/multi_cluster/MultiClusterYamlTestSuiteIT.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.multi_cluster;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+
+import org.apache.lucene.tests.util.TimeUnits;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+
+@TimeoutSuite(millis = 5 * TimeUnits.MINUTE) // to account for slow as hell VMs
+public class MultiClusterYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    private static final String USER = "test_user";
+    private static final String PASS = "x-pack-test-password";
+
+    @Override
+    protected boolean preserveIndicesUponCompletion() {
+        return true;
+    }
+
+    public MultiClusterYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray()));
+        return Settings.builder().put(super.restClientSettings()).put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+}
diff --git a/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/30_jobs.yml b/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/30_jobs.yml
new file mode 100644
index 000000000000..540fed64ec53
--- /dev/null
+++ b/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/30_jobs.yml
@@ -0,0 +1,118 @@
+---
+setup:
+  - skip:
+      features: headers
+
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+
+  - do:
+      security.put_user:
+        username: "joe"
+        body:  >
+          {
+            "password": "joe-password",
+            "roles" : [ "x_cluster_role" ]
+          }
+  - do:
+      security.put_role:
+        name: "x_cluster_role"
+        body:  >
+          {
+            "cluster": ["manage_ml"],
+            "indices": [
+              {
+                "names": ["test_index"],
+                "privileges": ["read", "view_index_metadata"]
+              }
+            ]
+          }
+---
+teardown:
+  - do:
+      security.delete_user:
+        username: "joe"
+        ignore: 404
+
+---
+"Test ML job with local indices only":
+  - do:
+      catch: /Cannot create datafeed \[job-with-local-indices\] because user joe lacks permissions on the indices/
+      headers: { Authorization: "Basic am9lOmpvZS1wYXNzd29yZA==" }  # This is joe
+      ml.put_job:
+        job_id: job-with-local-indices
+        body: >
+          {
+            "description":"Analysis of response time by airline",
+            "analysis_config" : {
+                "detectors" :[{"function":"count"}]
+            },
+            "analysis_limits" : {
+                "model_memory_limit": "20mb"
+            },
+            "data_description" : {
+                "format":"xcontent"
+            },
+            "datafeed_config": {
+              "indexes":["test_index", "test_index-2"]
+            }
+          }
+
+---
+"Test ML job with local and remote index":
+  - do:
+      headers: { Authorization: "Basic am9lOmpvZS1wYXNzd29yZA==" }  # This is joe
+      ml.put_job:
+        job_id: job-with-local-and-remote-index
+        body: >
+          {
+            "description":"Analysis of response time by airline",
+            "analysis_config" : {
+                "detectors" :[{"function":"count"}]
+            },
+            "analysis_limits" : {
+                "model_memory_limit": "20mb"
+            },
+            "data_description" : {
+                "format":"xcontent"
+            },
+            "datafeed_config": {
+              "indexes":["test_index", "my_remote_cluster:remote_test_index"]
+            }
+          }
+  - match: { job_id: "job-with-local-and-remote-index" }
+  - is_true: datafeed_config
+  - match: { datafeed_config.job_id: "job-with-local-and-remote-index" }
+  - match: { datafeed_config.datafeed_id: "job-with-local-and-remote-index" }
+  - is_true: datafeed_config.authorization.roles
+  - is_true: create_time
+
+---
+"Test ML job with remote index only":
+  - do:
+      headers: { Authorization: "Basic am9lOmpvZS1wYXNzd29yZA==" }  # This is joe
+      ml.put_job:
+        job_id: job-with-remote-index-only
+        body: >
+          {
+            "description":"Analysis of response time by airline",
+            "analysis_config" : {
+                "detectors" :[{"function":"count"}]
+            },
+            "analysis_limits" : {
+                "model_memory_limit": "20mb"
+            },
+            "data_description" : {
+                "format":"xcontent"
+            },
+            "datafeed_config": {
+              "indexes":["my_remote_cluster:remote_test_index"]
+            }
+          }
+  - match: { job_id: "job-with-remote-index-only" }
+  - is_true: datafeed_config
+  - match: { datafeed_config.job_id: "job-with-remote-index-only" }
+  - match: { datafeed_config.datafeed_id: "job-with-remote-index-only" }
+  - is_true: datafeed_config.authorization.roles
+  - is_true: create_time
diff --git a/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/remote_cluster/30_jobs.yml b/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/remote_cluster/30_jobs.yml
new file mode 100644
index 000000000000..2e4e7866fb07
--- /dev/null
+++ b/x-pack/plugin/ml/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/remote_cluster/30_jobs.yml
@@ -0,0 +1,97 @@
+---
+setup:
+  - skip:
+      features: headers
+
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+  - do:
+      security.put_user:
+        username: "joe"
+        body:  >
+            {
+              "password": "joe-password",
+              "roles" : [ "x_cluster_role" ]
+            }
+  - do:
+      security.put_role:
+        name: "x_cluster_role"
+        body:  >
+          {
+            "cluster": [],
+            "indices": [
+              {
+                "names": ["remote_test_index"],
+                "privileges": ["read", "view_index_metadata"]
+              }
+            ]
+          }
+---
+teardown:
+  - do:
+      security.delete_user:
+        username: "joe"
+        ignore: 404
+
+---
+"Index data on the remote cluster":
+  - do:
+      indices.create:
+        index: remote_test_index
+        body:
+          settings:
+            index:
+              number_of_shards: 3
+              number_of_replicas: 0
+          aliases:
+            test_alias: {}
+          mappings:
+            properties:
+              time:
+                type: date
+              user:
+                type: keyword
+              stars:
+                type: integer
+              coolness:
+                type: integer
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+            - '{"index": {"_index": "remote_test_index"}}'
+            - '{"user": "a", "stars": 1, "date" : "2018-10-29T12:12:12.123456789Z"}'
+            - '{"index": {"_index": "remote_test_index"}}'
+            - '{"user": "a", "stars": 4, "date" : "2018-10-29T12:14:12.123456789Z"}'
+            - '{"index": {"_index": "remote_test_index"}}'
+            - '{"user": "a", "stars": 5, "date" : "2018-10-29T12:16:12.123456789Z"}'
+            - '{"index": {"_index": "remote_test_index"}}'
+            - '{"user": "b", "stars": 2, "date" : "2018-10-29T12:17:12.123456789Z"}'
+            - '{"index": {"_index": "remote_test_index"}}'
+            - '{"user": "b", "stars": 3, "date" : "2018-10-29T12:22:12.123456789Z"}'
+            - '{"index": {"_index": "remote_test_index"}}'
+            - '{"user": "a", "stars": 5, "date" : "2018-10-29T12:23:12.123456789Z"}'
+            - '{"index": {"_index": "remote_test_index"}}'
+            - '{"user": "b", "stars": 1, "date" : "2018-10-29T12:32:12.123456789Z"}'
+            - '{"index": {"_index": "remote_test_index"}}'
+            - '{"user": "a", "stars": 3, "date" : "2018-10-29T12:34:12.123456789Z"}'
+            - '{"index": {"_index": "remote_test_index"}}'
+            - '{"user": "c", "stars": 4, "date" : "2018-10-29T12:35:12.123456789Z"}'
+  - do:
+      headers: { Authorization: "Basic am9lOmpvZS1wYXNzd29yZA==" }  # This is joe
+      search:
+        rest_total_hits_as_int: true
+        index: remote_test_index
+        body:
+          aggs:
+            user:
+              terms:
+                field: user
+
+  - match: { _shards.total: 3 }
+  - match: { hits.total: 9 }
+  - length: { aggregations.user.buckets: 3 }
+  - match: { aggregations.user.buckets.0.key: "a" }
+  - match: { aggregations.user.buckets.0.doc_count: 5 }
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle
index 9baa7d38c529..2645a3f3fa0d 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle
@@ -1,5 +1,5 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 dependencies {
   javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java
index 8aa53fce41c4..a2fa26914e08 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
@@ -39,7 +40,6 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import java.util.Locale;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
 
@@ -356,7 +356,7 @@ public void testStopOnWarn() throws IOException {
         for (int docNum = 0; docNum < 200; ++docNum) {
             // Two thirds of our messages are "Node 1 started", the rest "Failed to shutdown"
             int partitionNum = (docNum % 3) / 2;
-            json.append(String.format(Locale.ROOT, """
+            json.append(Strings.format("""
                 {"time":1000000,"part":"%s","msg":"%s"}
                 """, partitions[partitionNum], messages[partitionNum]));
         }
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationWithSecurityIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationWithSecurityIT.java
index dd9cfb4dc872..86848c5e94f6 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationWithSecurityIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationWithSecurityIT.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.SecuritySettingsSourceField;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
@@ -35,7 +36,7 @@ protected Settings restClientSettings() {
 
     private static void setupDataAccessRole(String index) throws IOException {
         Request request = new Request("PUT", "/_security/role/test_data_access");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {  "indices" : [    { "names": ["%s"], "privileges": ["read"] }  ]}
             """, index));
         client().performRequest(request);
@@ -45,7 +46,7 @@ private void setupUser(String user, List<String> roles) throws IOException {
         String password = new String(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING.getChars());
 
         Request request = new Request("PUT", "/_security/user/" + user);
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             { "password" : "%s",  "roles" : [ %s ]}
             """, password, roles.stream().map(unquoted -> "\"" + unquoted + "\"").collect(Collectors.joining(", "))));
         client().performRequest(request);
@@ -79,7 +80,7 @@ public void testEvaluate_withSecurity() throws Exception {
 
     private static Request buildRegressionEval(String index, String primaryHeader, String secondaryHeader) {
         Request evaluateRequest = new Request("POST", "_ml/data_frame/_evaluate");
-        evaluateRequest.setJsonEntity(formatted("""
+        evaluateRequest.setJsonEntity(Strings.format("""
             {
               "index": "%s",
               "evaluation": {
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
index 8eb1c3fbb479..5dafd074287a 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.rest.RestStatus;
@@ -1101,7 +1102,7 @@ private void initialize(String jobId, boolean isDatastream) {
     }
 
     static void createIndex(String index, boolean isDatastream) {
-        String mapping = formatted(
+        String mapping = Strings.format(
             """
                 {
                   "properties": {
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DataFrameAnalysisCustomFeatureIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DataFrameAnalysisCustomFeatureIT.java
index 0979454b9f12..929f50fe4b5d 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DataFrameAnalysisCustomFeatureIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DataFrameAnalysisCustomFeatureIT.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchModule;
@@ -199,7 +200,7 @@ private void initialize(String jobId, boolean isDatastream) {
     }
 
     private static void createIndex(String index, boolean isDatastream) {
-        String mapping = formatted(
+        String mapping = Strings.format(
             """
                 {
                   "properties": {
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java
index c3cae87228d0..01a8f409ed6b 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.ml.integration;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.hotthreads.NodeHotThreads;
@@ -64,6 +65,7 @@
 import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.createScheduledJob;
 import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.getDataCounts;
 import static org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase.indexDocs;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasSize;
@@ -828,4 +830,55 @@ public void testStartDatafeed_GivenTimeout_Returns408() throws Exception {
 
         assertThat(e.status(), equalTo(RestStatus.REQUEST_TIMEOUT));
     }
+
+    public void testStart_GivenAggregateMetricDoubleWithoutAggs() throws Exception {
+        final String index = "index-with-aggregate-metric-double";
+        String mapping = """
+            {
+              "properties": {
+                "time": {
+                  "type": "date"
+                },
+                "presum": {
+                  "type": "aggregate_metric_double",
+                  "metrics": [ "min", "max", "sum", "value_count" ],
+                  "default_metric": "max"
+                }
+              }
+            }""";
+        client().admin().indices().prepareCreate(index).setMapping(mapping).get();
+
+        DataDescription.Builder dataDescription = new DataDescription.Builder();
+
+        Detector.Builder d = new Detector.Builder("avg", "presum");
+        AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(d.build()));
+        analysisConfig.setBucketSpan(TimeValue.timeValueHours(1));
+
+        Job.Builder jobBuilder = new Job.Builder();
+        jobBuilder.setId("job-with-aggregate-metric-double");
+        jobBuilder.setAnalysisConfig(analysisConfig);
+        jobBuilder.setDataDescription(dataDescription);
+
+        putJob(jobBuilder);
+        openJob(jobBuilder.getId());
+        assertBusy(() -> assertEquals(getJobStats(jobBuilder.getId()).get(0).getState(), JobState.OPENED));
+
+        DatafeedConfig.Builder dfBuilder = new DatafeedConfig.Builder(jobBuilder.getId() + "-datafeed", jobBuilder.getId());
+        dfBuilder.setIndices(Collections.singletonList(index));
+
+        DatafeedConfig datafeedConfig = dfBuilder.build();
+
+        putDatafeed(datafeedConfig);
+
+        ElasticsearchStatusException e = expectThrows(
+            ElasticsearchStatusException.class,
+            () -> startDatafeed(datafeedConfig.getId(), 0L, null)
+        );
+
+        assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
+        assertThat(
+            e.getMessage(),
+            containsString("field [presum] is of type [aggregate_metric_double] and cannot be used in a datafeed without aggregations")
+        );
+    }
 }
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java
index 33eaad77aa93..1657a6df5074 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.SecuritySettingsSourceField;
 import org.elasticsearch.test.rest.ESRestTestCase;
@@ -30,7 +31,6 @@
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
-import java.util.Locale;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
@@ -66,7 +66,7 @@ protected boolean preserveTemplatesUponCompletion() {
 
     private static void setupDataAccessRole(String index) throws IOException {
         Request request = new Request("PUT", "/_security/role/test_data_access");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {  "indices" : [    { "names": ["%s"], "privileges": ["read"] }  ]}
             """, index));
         client().performRequest(request);
@@ -74,7 +74,7 @@ private static void setupDataAccessRole(String index) throws IOException {
 
     private void setupFullAccessRole(String index) throws IOException {
         Request request = new Request("PUT", "/_security/role/test_data_access");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {  "indices" : [    { "names": ["%s"], "privileges": ["all"] }  ]}
             """, index));
         client().performRequest(request);
@@ -84,7 +84,7 @@ private void setupUser(String user, List<String> roles) throws IOException {
         String password = new String(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING.getChars());
 
         Request request = new Request("PUT", "/_security/user/" + user);
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             { "password" : "%s", "roles" : [ %s ]}
             """, password, roles.stream().map(unquoted -> "\"" + unquoted + "\"").collect(Collectors.joining(", "))));
         client().performRequest(request);
@@ -330,11 +330,11 @@ private void addNetworkData(String index) throws IOException {
         for (int i = 0; i < 120; i++) {
             long byteCount = randomNonNegativeLong();
             bulk.append("{\"index\": {\"_index\": \"").append(index).append("\"}}\n");
-            bulk.append(String.format(Locale.ROOT, docTemplate, date.getTime(), "hostA", byteCount)).append('\n');
+            bulk.append(Strings.format(docTemplate, date.getTime(), "hostA", byteCount)).append('\n');
 
             byteCount = randomNonNegativeLong();
             bulk.append("{\"index\": {\"_index\": \"").append(index).append("\"}}\n");
-            bulk.append(String.format(Locale.ROOT, docTemplate, date.getTime(), "hostB", byteCount)).append('\n');
+            bulk.append(Strings.format(docTemplate, date.getTime(), "hostB", byteCount)).append('\n');
 
             date = new Date(date.getTime() + 10_000);
         }
@@ -1584,7 +1584,7 @@ private void waitUntilJobIsClosed(String jobId) throws Exception {
 
     private Response createJob(String id, String airlineVariant) throws Exception {
         Request request = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + id);
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {
               "description": "Analysis of response time by airline",
               "analysis_config": {
@@ -1667,31 +1667,28 @@ DatafeedBuilder setIndicesOptions(String indicesOptions) {
 
         Response build() throws IOException {
             Request request = new Request("PUT", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId);
-            request.setJsonEntity(
-                formatted(
-                    """
-                        {
-                          "job_id": "%s",
-                          "indexes":["%s"]
-                           %s
-                           %s
-                           %s
-                           %s
-                           %s
-                           %s
-                        }""",
-                    jobId,
-                    index,
-                    source ? ",\"_source\":true" : "",
-                    scriptedFields == null ? "" : ",\"script_fields\":" + scriptedFields,
-                    aggregations == null ? "" : ",\"aggs\":" + aggregations,
-                    frequency == null ? "" : ",\"frequency\":\"" + frequency + "\"",
-                    indicesOptions == null ? "" : ",\"indices_options\":" + indicesOptions,
-                    chunkingTimespan == null ? "" : formatted("""
-                        ,"chunking_config":{"mode":"MANUAL","time_span":"%s"}
-                        """, chunkingTimespan)
-                )
-            );
+            Object[] args = new Object[] {
+                jobId,
+                index,
+                source ? ",\"_source\":true" : "",
+                scriptedFields == null ? "" : ",\"script_fields\":" + scriptedFields,
+                aggregations == null ? "" : ",\"aggs\":" + aggregations,
+                frequency == null ? "" : ",\"frequency\":\"" + frequency + "\"",
+                indicesOptions == null ? "" : ",\"indices_options\":" + indicesOptions,
+                chunkingTimespan == null ? "" : Strings.format("""
+                    ,"chunking_config":{"mode":"MANUAL","time_span":"%s"}
+                    """, chunkingTimespan) };
+            request.setJsonEntity(Strings.format("""
+                {
+                  "job_id": "%s",
+                  "indexes":["%s"]
+                   %s
+                   %s
+                   %s
+                   %s
+                   %s
+                   %s
+                }""", args));
             RequestOptions.Builder options = request.getOptions().toBuilder();
             options.addHeader("Authorization", authHeader);
             if (this.secondaryAuthHeader != null) {
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ExplainDataFrameAnalyticsRestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ExplainDataFrameAnalyticsRestIT.java
index fea744aadb3c..317e933500d6 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ExplainDataFrameAnalyticsRestIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ExplainDataFrameAnalyticsRestIT.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.SecuritySettingsSourceField;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
@@ -56,7 +57,7 @@ private void setupUser(String user, List<String> roles) throws IOException {
         String password = new String(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING.getChars());
 
         Request request = new Request("PUT", "/_security/user/" + user);
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             { "password" : "%s",  "roles" : [ %s ] }
             """, password, roles.stream().map(unquoted -> "\"" + unquoted + "\"").collect(Collectors.joining(", "))));
         client().performRequest(request);
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ForecastIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ForecastIT.java
index fa661ac28d9e..447bca4f4e68 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ForecastIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/ForecastIT.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xpack.core.ml.action.DeleteForecastAction;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
@@ -30,7 +31,6 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
@@ -624,7 +624,7 @@ private void createDataWithLotsOfClientIps(TimeValue bucketSpan, Job.Builder job
             double value = 10.0 + h;
             for (int i = 1; i < 101; i++) {
                 for (int j = 1; j < 81; j++) {
-                    String json = String.format(Locale.ROOT, """
+                    String json = Strings.format("""
                         {"time": %s, "value": %f, "clientIP": "192.168.%d.%d"}
                         """, timestamp, value, i, j);
                     data.add(json);
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceIT.java
index 161d87712af1..c8c580d2933c 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceIT.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.SecuritySettingsSourceField;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -62,7 +63,7 @@ public void testInferRegressionModel() throws Exception {
         }
 
         Request inferRequest = new Request("POST", "_ml/trained_models/" + regressionModelId + "/_infer");
-        inferRequest.setJsonEntity(formatted("""
+        inferRequest.setJsonEntity(Strings.format("""
             {
               "docs": [%s]
             }
@@ -75,7 +76,7 @@ public void testInferRegressionModel() throws Exception {
         putModelAlias("foo_regression", regressionModelId);
 
         inferRequest = new Request("POST", "_ml/trained_models/foo_regression/_infer");
-        inferRequest.setJsonEntity(formatted("""
+        inferRequest.setJsonEntity(Strings.format("""
             {
               "docs": [%s]
             }
@@ -98,7 +99,7 @@ public void testInferClassificationModel() throws Exception {
         }
 
         Request inferRequest = new Request("POST", "_ml/trained_models/" + classificationModelId + "/_infer");
-        inferRequest.setJsonEntity(formatted("""
+        inferRequest.setJsonEntity(Strings.format("""
             {
               "docs": [%s]
             }
@@ -111,7 +112,7 @@ public void testInferClassificationModel() throws Exception {
         putModelAlias("foo_classification", classificationModelId);
 
         inferRequest = new Request("POST", "_ml/trained_models/foo_classification/_infer");
-        inferRequest.setJsonEntity(formatted("""
+        inferRequest.setJsonEntity(Strings.format("""
             {
               "docs": [%s]
             }
@@ -282,7 +283,7 @@ private void putModelAlias(String modelAlias, String newModel) throws IOExceptio
           }
         }""";
 
-    private static final String REGRESSION_CONFIG = formatted("""
+    private static final String REGRESSION_CONFIG = Strings.format("""
         {
             "input": {
                 "field_names": [
@@ -304,7 +305,7 @@ protected NamedXContentRegistry xContentRegistry() {
         return new NamedXContentRegistry(new MlInferenceNamedXContentProvider().getNamedXContentParsers());
     }
 
-    private static final String CLASSIFICATION_CONFIG = formatted("""
+    private static final String CLASSIFICATION_CONFIG = Strings.format("""
         {
           "input": {
             "field_names": [ "col1", "col2", "col3", "col4" ]
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java
index 7d0fbdc5c499..84c5ed9f934b 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ExternalTestCluster;
@@ -290,7 +291,7 @@ public void testSimulate() throws IOException {
         String regressionModelId = "test_regression_simulate";
         putModel(regressionModelId, REGRESSION_CONFIG);
 
-        String source = formatted("""
+        String source = Strings.format("""
             {
               "pipeline": {
                 "processors": [
@@ -390,7 +391,7 @@ public void testSimulate() throws IOException {
     public void testSimulateWithDefaultMappedField() throws IOException {
         String classificationModelId = "test_classification_default_mapped_field";
         putModel(classificationModelId, CLASSIFICATION_CONFIG);
-        String source = formatted("""
+        String source = Strings.format("""
             {
               "pipeline": {
                 "processors": [
@@ -674,7 +675,7 @@ private Map<String, Object> generateSourceDoc() {
           }
         }""";
 
-    private static final String REGRESSION_CONFIG = formatted("""
+    private static final String REGRESSION_CONFIG = Strings.format("""
         {
             "input": {
                 "field_names": [
@@ -696,7 +697,7 @@ protected NamedXContentRegistry xContentRegistry() {
         return new NamedXContentRegistry(new MlInferenceNamedXContentProvider().getNamedXContentParsers());
     }
 
-    private static final String CLASSIFICATION_CONFIG = formatted("""
+    private static final String CLASSIFICATION_CONFIG = Strings.format("""
         {
           "input": {
             "field_names": [ "col1", "col2", "col3", "col4" ]
@@ -712,7 +713,7 @@ protected NamedXContentRegistry xContentRegistry() {
         }""", InferenceDefinitionTests.getClassificationDefinition(false));
 
     private static String pipelineDefinition(String modelId, String inferenceConfig) {
-        return formatted("""
+        return Strings.format("""
             {
               "processors": [
                 {
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java
index 57abc7dc3898..bb322cf79adc 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.SecuritySettingsSourceField;
@@ -31,7 +32,6 @@
 
 import java.io.IOException;
 import java.util.Collections;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -187,11 +187,8 @@ public void testCantCreateJobWithSameID() throws Exception {
               "results_index_name" : "%s"}""";
 
         String jobId = "cant-create-job-with-same-id-job";
-        putJob(jobId, String.format(Locale.ROOT, jobTemplate, "index-1"));
-        ResponseException e = expectThrows(
-            ResponseException.class,
-            () -> putJob(jobId, String.format(Locale.ROOT, jobTemplate, "index-2"))
-        );
+        putJob(jobId, Strings.format(jobTemplate, "index-1"));
+        ResponseException e = expectThrows(ResponseException.class, () -> putJob(jobId, Strings.format(jobTemplate, "index-2")));
 
         assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(400));
         assertThat(e.getMessage(), containsString("The job cannot be created with the Id '" + jobId + "'. The Id is already used."));
@@ -208,10 +205,10 @@ public void testCreateJobsWithIndexNameOption() throws Exception {
 
         String jobId1 = "create-jobs-with-index-name-option-job-1";
         String indexName = "non-default-index";
-        putJob(jobId1, String.format(Locale.ROOT, jobTemplate, indexName));
+        putJob(jobId1, Strings.format(jobTemplate, indexName));
 
         String jobId2 = "create-jobs-with-index-name-option-job-2";
-        putJob(jobId2, String.format(Locale.ROOT, jobTemplate, indexName));
+        putJob(jobId2, Strings.format(jobTemplate, indexName));
 
         // With security enabled GET _aliases throws an index_not_found_exception
         // if no aliases have been created. In multi-node tests the alias may not
@@ -219,12 +216,12 @@ public void testCreateJobsWithIndexNameOption() throws Exception {
         assertBusy(() -> {
             try {
                 String aliasesResponse = getAliases();
-                assertThat(aliasesResponse, containsString(formatted("""
+                assertThat(aliasesResponse, containsString(Strings.format("""
                     "%s":{"aliases":{""", AnomalyDetectorsIndex.jobResultsAliasedName("custom-" + indexName))));
                 assertThat(
                     aliasesResponse,
                     containsString(
-                        formatted(
+                        Strings.format(
                             """
                                 "%s":{"filter":{"term":{"job_id":{"value":"%s"}}},"is_hidden":true}""",
                             AnomalyDetectorsIndex.jobResultsAliasedName(jobId1),
@@ -232,12 +229,12 @@ public void testCreateJobsWithIndexNameOption() throws Exception {
                         )
                     )
                 );
-                assertThat(aliasesResponse, containsString(formatted("""
+                assertThat(aliasesResponse, containsString(Strings.format("""
                     "%s":{"is_hidden":true}""", AnomalyDetectorsIndex.resultsWriteAlias(jobId1))));
                 assertThat(
                     aliasesResponse,
                     containsString(
-                        formatted(
+                        Strings.format(
                             """
                                 "%s":{"filter":{"term":{"job_id":{"value":"%s"}}},"is_hidden":true}""",
                             AnomalyDetectorsIndex.jobResultsAliasedName(jobId2),
@@ -245,7 +242,7 @@ public void testCreateJobsWithIndexNameOption() throws Exception {
                         )
                     )
                 );
-                assertThat(aliasesResponse, containsString(formatted("""
+                assertThat(aliasesResponse, containsString(Strings.format("""
                     "%s":{"is_hidden":true}""", AnomalyDetectorsIndex.resultsWriteAlias(jobId2))));
             } catch (ResponseException e) {
                 throw new AssertionError(e);
@@ -262,15 +259,15 @@ public void testCreateJobsWithIndexNameOption() throws Exception {
         assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))));
 
         { // create jobId1 docs
-            String id = String.format(Locale.ROOT, "%s_bucket_%s_%s", jobId1, "1234", 300);
+            String id = Strings.format("%s_bucket_%s_%s", jobId1, "1234", 300);
             Request createResultRequest = new Request("PUT", AnomalyDetectorsIndex.jobResultsAliasedName(jobId1) + "/_doc/" + id);
-            createResultRequest.setJsonEntity(String.format(Locale.ROOT, """
+            createResultRequest.setJsonEntity(Strings.format("""
                 {"job_id":"%s", "timestamp": "%s", "result_type":"bucket", "bucket_span": "%s"}""", jobId1, "1234", 1));
             client().performRequest(createResultRequest);
 
-            id = String.format(Locale.ROOT, "%s_bucket_%s_%s", jobId1, "1236", 300);
+            id = Strings.format("%s_bucket_%s_%s", jobId1, "1236", 300);
             createResultRequest = new Request("PUT", AnomalyDetectorsIndex.jobResultsAliasedName(jobId1) + "/_doc/" + id);
-            createResultRequest.setJsonEntity(String.format(Locale.ROOT, """
+            createResultRequest.setJsonEntity(Strings.format("""
                 {"job_id":"%s", "timestamp": "%s", "result_type":"bucket", "bucket_span": "%s"}""", jobId1, "1236", 1));
             client().performRequest(createResultRequest);
 
@@ -288,15 +285,15 @@ public void testCreateJobsWithIndexNameOption() throws Exception {
             assertThat(responseAsString, containsString("\"value\":2"));
         }
         { // create jobId2 docs
-            String id = String.format(Locale.ROOT, "%s_bucket_%s_%s", jobId2, "1234", 300);
+            String id = Strings.format("%s_bucket_%s_%s", jobId2, "1234", 300);
             Request createResultRequest = new Request("PUT", AnomalyDetectorsIndex.jobResultsAliasedName(jobId2) + "/_doc/" + id);
-            createResultRequest.setJsonEntity(String.format(Locale.ROOT, """
+            createResultRequest.setJsonEntity(Strings.format("""
                 {"job_id":"%s", "timestamp": "%s", "result_type":"bucket", "bucket_span": "%s"}""", jobId2, "1234", 1));
             client().performRequest(createResultRequest);
 
-            id = String.format(Locale.ROOT, "%s_bucket_%s_%s", jobId2, "1236", 300);
+            id = Strings.format("%s_bucket_%s_%s", jobId2, "1236", 300);
             createResultRequest = new Request("PUT", AnomalyDetectorsIndex.jobResultsAliasedName(jobId2) + "/_doc/" + id);
-            createResultRequest.setJsonEntity(String.format(Locale.ROOT, """
+            createResultRequest.setJsonEntity(Strings.format("""
                 {"job_id":"%s", "timestamp": "%s", "result_type":"bucket", "bucket_span": "%s"}""", jobId2, "1236", 1));
             client().performRequest(createResultRequest);
 
@@ -363,7 +360,7 @@ public void testCreateJobInSharedIndexUpdatesMapping() throws Exception {
         String jobId2 = "create-job-in-shared-index-updates-mapping-job-2";
         String byFieldName2 = "cpu-usage";
 
-        putJob(jobId1, String.format(Locale.ROOT, jobTemplate, byFieldName1));
+        putJob(jobId1, Strings.format(jobTemplate, byFieldName1));
 
         // Check the index mapping contains the first by_field_name
         Request getResultsMappingRequest = new Request(
@@ -375,7 +372,7 @@ public void testCreateJobInSharedIndexUpdatesMapping() throws Exception {
         assertThat(resultsMappingAfterJob1, containsString(byFieldName1));
         assertThat(resultsMappingAfterJob1, not(containsString(byFieldName2)));
 
-        putJob(jobId2, String.format(Locale.ROOT, jobTemplate, byFieldName2));
+        putJob(jobId2, Strings.format(jobTemplate, byFieldName2));
         // Check the index mapping now contains both fields
         String resultsMappingAfterJob2 = EntityUtils.toString(client().performRequest(getResultsMappingRequest).getEntity());
         assertThat(resultsMappingAfterJob2, containsString(byFieldName1));
@@ -396,7 +393,7 @@ public void testCreateJobInCustomSharedIndexUpdatesMapping() throws Exception {
         String jobId2 = "create-job-in-custom-shared-index-updates-mapping-job-2";
         String byFieldName2 = "cpu-usage";
 
-        putJob(jobId1, String.format(Locale.ROOT, jobTemplate, byFieldName1));
+        putJob(jobId1, Strings.format(jobTemplate, byFieldName1));
 
         // Check the index mapping contains the first by_field_name
         Request getResultsMappingRequest = new Request(
@@ -408,7 +405,7 @@ public void testCreateJobInCustomSharedIndexUpdatesMapping() throws Exception {
         assertThat(resultsMappingAfterJob1, containsString(byFieldName1));
         assertThat(resultsMappingAfterJob1, not(containsString(byFieldName2)));
 
-        putJob(jobId2, String.format(Locale.ROOT, jobTemplate, byFieldName2));
+        putJob(jobId2, Strings.format(jobTemplate, byFieldName2));
 
         // Check the index mapping now contains both fields
         String resultsMappingAfterJob2 = EntityUtils.toString(client().performRequest(getResultsMappingRequest).getEntity());
@@ -438,12 +435,9 @@ public void testCreateJob_WithClashingFieldMappingsFails() throws Exception {
             byFieldName2 = "response";
         }
 
-        putJob(jobId1, String.format(Locale.ROOT, jobTemplate, byFieldName1));
+        putJob(jobId1, Strings.format(jobTemplate, byFieldName1));
 
-        ResponseException e = expectThrows(
-            ResponseException.class,
-            () -> putJob(jobId2, String.format(Locale.ROOT, jobTemplate, byFieldName2))
-        );
+        ResponseException e = expectThrows(ResponseException.class, () -> putJob(jobId2, Strings.format(jobTemplate, byFieldName2)));
         assertThat(
             e.getMessage(),
             containsString(
@@ -740,7 +734,7 @@ public void testMultiIndexDelete() throws Exception {
         // Make the job's results span an extra two indices, i.e. three in total.
         // To do this the job's results alias needs to encompass all three indices.
         Request extraIndex1 = new Request("PUT", indexName + "-001");
-        extraIndex1.setJsonEntity(formatted("""
+        extraIndex1.setJsonEntity(Strings.format("""
             {
               "aliases": {
                 "%s": {
@@ -755,7 +749,7 @@ public void testMultiIndexDelete() throws Exception {
             }""", AnomalyDetectorsIndex.jobResultsAliasedName(jobId), Job.ID, jobId));
         client().performRequest(extraIndex1);
         Request extraIndex2 = new Request("PUT", indexName + "-002");
-        extraIndex2.setJsonEntity(formatted("""
+        extraIndex2.setJsonEntity(Strings.format("""
             {
               "aliases": {
                 "%s": {
@@ -781,7 +775,7 @@ public void testMultiIndexDelete() throws Exception {
 
         // Add some documents to each index to make sure the DBQ clears them out
         Request createDoc0 = new Request("PUT", indexName + "/_doc/" + 123);
-        createDoc0.setJsonEntity(String.format(Locale.ROOT, """
+        createDoc0.setJsonEntity(Strings.format("""
             {"job_id":"%s", "timestamp": "%s", "bucket_span":%d, "result_type":"record"}""", jobId, 123, 1));
         client().performRequest(createDoc0);
         Request createDoc1 = new Request("PUT", indexName + "-001/_doc/" + 123);
@@ -921,13 +915,13 @@ public void testDelete_multipleRequest() throws Exception {
             assertNull(recreationException.get().getMessage(), recreationException.get());
         }
 
-        String expectedReadAliasString = formatted(
+        String expectedReadAliasString = Strings.format(
             """
                 "%s":{"filter":{"term":{"job_id":{"value":"%s"}}},"is_hidden":true}""",
             AnomalyDetectorsIndex.jobResultsAliasedName(jobId),
             jobId
         );
-        String expectedWriteAliasString = formatted("""
+        String expectedWriteAliasString = Strings.format("""
             "%s":{"is_hidden":true}""", AnomalyDetectorsIndex.resultsWriteAlias(jobId));
         try {
             // The idea of the code above is that the deletion is sufficiently time-consuming that
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PyTorchModelIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PyTorchModelIT.java
index 99e38e756796..45c1843a639e 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PyTorchModelIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PyTorchModelIT.java
@@ -14,14 +14,17 @@
 import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xpack.core.ml.inference.assignment.AllocationStatus;
 import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentState;
 import org.elasticsearch.xpack.core.ml.inference.assignment.Priority;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Base64;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -309,7 +312,7 @@ public void testFailedDeploymentStats() throws Exception {
         createPassThroughModel(badModel);
         putVocabulary(List.of("once", "twice"), badModel);
         Request request = new Request("PUT", "_ml/trained_models/" + badModel + "/definition/0");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {"total_definition_length":%s,"definition": "%s","total_parts": 1}""", length, poorlyFormattedModelBase64));
         client().performRequest(request);
         startDeployment(badModel, AllocationStatus.State.STARTING.toString());
@@ -418,6 +421,79 @@ public void testInferWithMissingModel() {
         assertThat(ex.getMessage(), containsString("Could not find trained model [missing_model]"));
     }
 
+    @SuppressWarnings("unchecked")
+    public void testInferWithMultipleDocs() throws IOException {
+        String modelId = "infer_multi_docs";
+        // Use the text embedding model from SemanticSearchIT so
+        // that each response can be linked to the originating request.
+        // The test ensures the responses are returned in the same order
+        // as the requests
+        createTextEmbeddingModel(modelId);
+        putModelDefinition(modelId, SemanticSearchIT.BASE_64_ENCODED_MODEL, SemanticSearchIT.RAW_MODEL_SIZE);
+        putVocabulary(
+            List.of("these", "are", "my", "words", "the", "washing", "machine", "is", "leaking", "octopus", "comforter", "smells"),
+            modelId
+        );
+        startDeployment(modelId, AllocationStatus.State.FULLY_ALLOCATED.toString());
+
+        List<String> inputs = List.of(
+            "my words",
+            "the machine is leaking",
+            "washing machine",
+            "these are my words",
+            "the octopus comforter smells",
+            "the octopus comforter is leaking",
+            "washing machine smells"
+        );
+
+        List<List<Double>> expectedEmbeddings = new ArrayList<>();
+
+        // Generate the text embeddings one at a time using the _infer API
+        // then index them for search
+        for (var input : inputs) {
+            Response inference = infer(input, modelId);
+            List<Map<String, Object>> responseMap = (List<Map<String, Object>>) entityAsMap(inference).get("inference_results");
+            Map<String, Object> inferenceResult = responseMap.get(0);
+            List<Double> embedding = (List<Double>) inferenceResult.get("predicted_value");
+            expectedEmbeddings.add(embedding);
+        }
+
+        // Now do the same with all documents sent at once
+        var docsBuilder = new StringBuilder();
+        int numInputs = inputs.size();
+        for (int i = 0; i < numInputs - 1; i++) {
+            docsBuilder.append("{\"input\":\"").append(inputs.get(i)).append("\"},");
+        }
+        docsBuilder.append("{\"input\":\"").append(inputs.get(numInputs - 1)).append("\"}");
+
+        {
+            Request request = new Request("POST", "/_ml/trained_models/" + modelId + "/_infer");
+            request.setJsonEntity(String.format(Locale.ROOT, """
+                {  "docs": [%s] }
+                """, docsBuilder));
+            Response response = client().performRequest(request);
+            var responseMap = entityAsMap(response);
+            List<Map<String, Object>> inferenceResults = (List<Map<String, Object>>) responseMap.get("inference_results");
+            assertThat(inferenceResults, hasSize(numInputs));
+
+            // Check the result order matches the input order by comparing
+            // the to the pre-calculated embeddings
+            for (int i = 0; i < numInputs; i++) {
+                List<Double> embedding = (List<Double>) inferenceResults.get(i).get("predicted_value");
+                assertArrayEquals(expectedEmbeddings.get(i).toArray(), embedding.toArray());
+            }
+        }
+        {
+            // the deprecated deployment/_infer endpoint does not support multiple docs
+            Request request = new Request("POST", "/_ml/trained_models/" + modelId + "/deployment/_infer");
+            request.setJsonEntity(String.format(Locale.ROOT, """
+                {  "docs": [%s] }
+                """, docsBuilder));
+            Exception ex = expectThrows(Exception.class, () -> client().performRequest(request));
+            assertThat(ex.getMessage(), containsString("multiple documents are not supported"));
+        }
+    }
+
     public void testGetPytorchModelWithDefinition() throws IOException {
         String model = "should-fail-get";
         createPassThroughModel(model);
@@ -435,7 +511,7 @@ public void testStartDeploymentWithTruncatedDefinition() throws IOException {
         createPassThroughModel(model);
         putVocabulary(List.of("once", "twice"), model);
         Request request = new Request("PUT", "_ml/trained_models/" + model + "/definition/0");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {"total_definition_length":%s2,"definition": "%s","total_parts": 1}""", RAW_MODEL_SIZE, BASE_64_ENCODED_MODEL));
         client().performRequest(request);
         Exception ex = expectThrows(Exception.class, () -> startDeployment(model));
@@ -475,7 +551,7 @@ public void testInferencePipelineAgainstUnallocatedModel() throws IOException {
         assertThat(
             response,
             allOf(
-                containsString("model [not-deployed] must be deployed to use. Please deploy with the start trained model deployment API."),
+                containsString("Model [not-deployed] must be deployed to use. Please deploy with the start trained model deployment API."),
                 containsString("error"),
                 not(containsString("warning"))
             )
@@ -498,7 +574,7 @@ public void testInferencePipelineAgainstUnallocatedModel() throws IOException {
                   }
             """);
         Exception ex = expectThrows(Exception.class, () -> client().performRequest(request));
-        assertThat(ex.getMessage(), containsString("Trained model [not-deployed] is not deployed."));
+        assertThat(ex.getMessage(), containsString("Model [not-deployed] must be deployed to use."));
     }
 
     public void testTruncation() throws IOException {
@@ -534,8 +610,9 @@ public void testTruncation() throws IOException {
             containsString("Input too large. The tokenized input length [3] exceeds the maximum sequence length [2]")
         );
 
-        request = new Request("POST", "/_ml/trained_models/" + modelId + "/_infer");
-        request.setJsonEntity(formatted("""
+        // We set timeout to 20s as we've seen this test time out on some busy workers.
+        request = new Request("POST", "/_ml/trained_models/" + modelId + "/_infer?timeout=20s");
+        request.setJsonEntity(Strings.format("""
             {
               "docs": [
                 {
@@ -562,7 +639,7 @@ public void testStopUsedDeploymentByIngestProcessor() throws IOException {
         putVocabulary(List.of("these", "are", "my", "words"), modelId);
         startDeployment(modelId);
 
-        client().performRequest(putPipeline("my_pipeline", formatted("""
+        client().performRequest(putPipeline("my_pipeline", Strings.format("""
             {
               "processors": [
                 {
@@ -592,9 +669,9 @@ public void testStopWithModelAliasUsedDeploymentByIngestProcessor() throws IOExc
         putModelDefinition(modelId);
         putVocabulary(List.of("these", "are", "my", "words"), modelId);
         startDeployment(modelId);
-        client().performRequest(new Request("PUT", formatted("_ml/trained_models/%s/model_aliases/%s", modelId, modelAlias)));
+        client().performRequest(new Request("PUT", Strings.format("_ml/trained_models/%s/model_aliases/%s", modelId, modelAlias)));
 
-        client().performRequest(putPipeline("my_pipeline", formatted("""
+        client().performRequest(putPipeline("my_pipeline", Strings.format("""
             {
               "processors": [
                 {
@@ -624,9 +701,9 @@ public void testInferenceProcessorWithModelAlias() throws IOException {
         putModelDefinition(modelId);
         putVocabulary(List.of("these", "are", "my", "words"), modelId);
         startDeployment(modelId);
-        client().performRequest(new Request("PUT", formatted("_ml/trained_models/%s/model_aliases/%s", modelId, modelAlias)));
+        client().performRequest(new Request("PUT", Strings.format("_ml/trained_models/%s/model_aliases/%s", modelId, modelAlias)));
 
-        String source = formatted("""
+        String source = Strings.format("""
             {
               "pipeline": {
                 "processors": [
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PyTorchModelRestTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PyTorchModelRestTestCase.java
index 53bdff094e5a..da5115837cba 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PyTorchModelRestTestCase.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/PyTorchModelRestTestCase.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.SecuritySettingsSourceField;
 import org.elasticsearch.test.rest.ESRestTestCase;
@@ -27,7 +28,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -57,7 +57,7 @@ public void setLogging() throws IOException {
         Request loggingSettings = new Request("PUT", "_cluster/settings");
         loggingSettings.setJsonEntity("""
             {"persistent" : {
-                    "logger.org.elasticsearch.xpack.ml.inference.assignment" : "DEBUG",
+                    "logger.org.elasticsearch.xpack.ml.inference.assignment" : "TRACE",
                     "logger.org.elasticsearch.xpack.ml.inference.deployment" : "DEBUG",
                     "logger.org.elasticsearch.xpack.ml.inference.pytorch" : "DEBUG",
                     "logger.org.elasticsearch.xpack.ml.process.logging" : "DEBUG"
@@ -129,7 +129,7 @@ protected int sumInferenceCountOnNodes(List<Map<String, Object>> nodes) {
 
     protected void putModelDefinition(String modelId, String base64EncodedModel, long unencodedModelSize) throws IOException {
         Request request = new Request("PUT", "_ml/trained_models/" + modelId + "/definition/0");
-        String body = String.format(Locale.ROOT, """
+        String body = Strings.format("""
             {"total_definition_length":%s,"definition": "%s","total_parts": 1}""", unencodedModelSize, base64EncodedModel);
         request.setJsonEntity(body);
         client().performRequest(request);
@@ -143,7 +143,7 @@ protected void putVocabulary(List<String> vocabulary, String modelId) throws IOE
         String quotedWords = vocabularyWithPad.stream().map(s -> "\"" + s + "\"").collect(Collectors.joining(","));
 
         Request request = new Request("PUT", "_ml/trained_models/" + modelId + "/vocabulary");
-        request.setJsonEntity(String.format(Locale.ROOT, """
+        request.setJsonEntity(Strings.format("""
             { "vocabulary": [%s] }
             """, quotedWords));
         client().performRequest(request);
@@ -244,23 +244,23 @@ protected Response getTrainedModelStats(String modelId) throws IOException {
 
     protected Response infer(String input, String modelId, TimeValue timeout) throws IOException {
         Request request = new Request("POST", "/_ml/trained_models/" + modelId + "/_infer?timeout=" + timeout.toString());
-        request.setJsonEntity(String.format(Locale.ROOT, """
+        request.setJsonEntity(Strings.format("""
             {  "docs": [{"input":"%s"}] }
             """, input));
         return client().performRequest(request);
     }
 
     protected Response infer(String input, String modelId) throws IOException {
-        Request request = new Request("POST", "/_ml/trained_models/" + modelId + "/_infer");
-        request.setJsonEntity(String.format(Locale.ROOT, """
+        Request request = new Request("POST", "/_ml/trained_models/" + modelId + "/_infer?timeout=30s");
+        request.setJsonEntity(Strings.format("""
             {  "docs": [{"input":"%s"}] }
             """, input));
         return client().performRequest(request);
     }
 
     protected Response infer(String input, String modelId, String resultsField) throws IOException {
-        Request request = new Request("POST", "/_ml/trained_models/" + modelId + "/_infer");
-        request.setJsonEntity(String.format(Locale.ROOT, """
+        Request request = new Request("POST", "/_ml/trained_models/" + modelId + "/_infer?timeout=30s");
+        request.setJsonEntity(Strings.format("""
             {
               "docs": [ { "input": "%s" } ],
               "inference_config": {
@@ -272,19 +272,19 @@ protected Response infer(String input, String modelId, String resultsField) thro
         return client().performRequest(request);
     }
 
-    protected Response semanticSearch(String index, String queryText, String modelId, String denseVectorFieldName) throws IOException {
+    protected Response semanticSearch(String index, String modelText, String modelId, String denseVectorFieldName) throws IOException {
         Request request = new Request("GET", index + "/_semantic_search?error_trace=true");
 
-        request.setJsonEntity(String.format(Locale.ROOT, """
+        request.setJsonEntity(Strings.format("""
             {
               "model_id": "%s",
-              "query_string": "%s",
+              "model_text": "%s",
               "knn": {
                   "field": "%s",
                   "k": 5,
                   "num_candidates": 10
               }
-            }""", modelId, queryText, denseVectorFieldName));
+            }""", modelId, modelText, denseVectorFieldName));
         return client().performRequest(request);
     }
 
@@ -297,14 +297,14 @@ protected Response semanticSearchWithTermsFilter(
     ) throws IOException {
         Request request = new Request("GET", index + "/_semantic_search?error_trace=true");
 
-        String termsFilter = String.format(Locale.ROOT, """
+        String termsFilter = Strings.format("""
             {"term": {"filter_field": "%s"}}
             """, filter);
 
-        request.setJsonEntity(String.format(Locale.ROOT, """
+        request.setJsonEntity(Strings.format("""
             {
               "model_id": "%s",
-              "query_string": "%s",
+              "model_text": "%s",
               "knn": {
                   "field": "%s",
                   "k": 5,
@@ -319,10 +319,10 @@ protected Response semanticSearchWithQuery(String index, String queryText, Strin
         throws IOException {
         Request request = new Request("GET", index + "/_semantic_search?error_trace=true");
 
-        request.setJsonEntity(String.format(Locale.ROOT, """
+        request.setJsonEntity(Strings.format("""
             {
               "model_id": "%s",
-              "query_string": "%s",
+              "model_text": "%s",
               "knn": {
                   "field": "%s",
                   "k": 5,
@@ -371,7 +371,7 @@ protected void assertThatTrainedModelAssignmentMetadataIsEmpty() throws IOExcept
     protected void assertNotificationsContain(String modelId, String... auditMessages) throws IOException {
         client().performRequest(new Request("POST", ".ml-notifications-*/_refresh"));
         Request search = new Request("POST", ".ml-notifications-*/_search");
-        search.setJsonEntity(String.format(Locale.ROOT, """
+        search.setJsonEntity(Strings.format("""
             {
                 "size": 100,
                 "query": {
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
index 8f9358cde79d..41f81ca15877 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java
@@ -596,7 +596,11 @@ public void testAliasFields() throws Exception {
             fail("Failed to index data: " + bulkResponse.buildFailureMessage());
         }
 
-        long seed = randomLong();
+        // Very infrequently this test may fail as the algorithm underestimates the
+        // required number of trees for this simple problem. This failure is irrelevant
+        // for non-trivial real-world problem and improving estimation of the number of trees
+        // would introduce unnecessary overhead. Hence, to reduce the noise from this test we fix the seed.
+        long seed = 1000L; // fix seed
 
         Regression regression = new Regression("field_2", BoostedTreeParams.builder().build(), null, 90.0, seed, null, null, null, null);
         DataFrameAnalyticsConfig config = new DataFrameAnalyticsConfig.Builder().setId(jobId)
@@ -897,7 +901,7 @@ static void indexData(String sourceIndex, int numTrainingRows, int numNonTrainin
     }
 
     static void indexData(String sourceIndex, int numTrainingRows, int numNonTrainingRows, boolean dataStream) {
-        String mapping = formatted("""
+        String mapping = Strings.format("""
             {
               "properties": {
                 "@timestamp": {
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/SemanticSearchIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/SemanticSearchIT.java
index 57ebbcb8daf6..21b688994562 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/SemanticSearchIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/SemanticSearchIT.java
@@ -8,18 +8,10 @@
 package org.elasticsearch.xpack.ml.integration;
 
 import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.xpack.core.ml.utils.MapHelper;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Base64;
 import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
-import static org.hamcrest.Matchers.closeTo;
-import static org.hamcrest.Matchers.hasSize;
 
 /**
  * This test uses a tiny text embedding model to simulate an trained
@@ -96,187 +88,6 @@ public class SemanticSearchIT extends PyTorchModelRestTestCase {
         RAW_MODEL_SIZE = Base64.getDecoder().decode(BASE_64_ENCODED_MODEL).length;
     }
 
-    @SuppressWarnings("unchecked")
-    public void testSemanticSearch() throws IOException {
-        String modelId = "semantic-search-test";
-        String indexName = modelId + "-index";
-
-        createTextEmbeddingModel(modelId);
-        putModelDefinition(modelId, BASE_64_ENCODED_MODEL, RAW_MODEL_SIZE);
-        putVocabulary(
-            List.of("these", "are", "my", "words", "the", "washing", "machine", "is", "leaking", "octopus", "comforter", "smells"),
-            modelId
-        );
-        startDeployment(modelId);
-
-        List<String> inputs = List.of(
-            "my words",
-            "the machine is leaking",
-            "washing machine",
-            "these are my words",
-            "the octopus comforter smells",
-            "the octopus comforter is leaking",
-            "washing machine smells"
-        );
-        List<String> filters = List.of("foo", "bar", "baz", "foo", "bar", "baz", "foo");
-        List<List<Double>> embeddings = new ArrayList<>();
-
-        // Generate the text embeddings via the inference API
-        // then index them for search
-        for (var input : inputs) {
-            Response inference = infer(input, modelId);
-            List<Map<String, Object>> responseMap = (List<Map<String, Object>>) entityAsMap(inference).get("inference_results");
-            Map<String, Object> inferenceResult = responseMap.get(0);
-            List<Double> embedding = (List<Double>) inferenceResult.get("predicted_value");
-            embeddings.add(embedding);
-        }
-
-        // index dense vectors
-        createVectorSearchIndex(indexName);
-        bulkIndexDocs(inputs, filters, embeddings, indexName);
-        forceMergeIndex(indexName);
-
-        // Test semantic search against the indexed vectors
-        for (int i = 0; i < 5; i++) {
-            int randomInput = randomIntBetween(0, inputs.size() - 1);
-            var semanticSearchResponse = semanticSearch(indexName, inputs.get(randomInput), modelId, "embedding");
-            assertOkWithErrorMessage(semanticSearchResponse);
-
-            Map<String, Object> responseMap = responseAsMap(semanticSearchResponse);
-            List<Map<String, Object>> hits = (List<Map<String, Object>>) MapHelper.dig("hits.hits", responseMap);
-            Map<String, Object> topHit = hits.get(0);
-            String sourceText = (String) MapHelper.dig("_source.source_text", topHit);
-            assertEquals(inputs.get(randomInput), sourceText);
-        }
-
-        // Test semantic search with filters
-        {
-            var semanticSearchResponse = semanticSearchWithTermsFilter(indexName, inputs.get(0), "foo", modelId, "embedding");
-            assertOkWithErrorMessage(semanticSearchResponse);
-
-            Map<String, Object> responseMap = responseAsMap(semanticSearchResponse);
-            List<Map<String, Object>> hits = (List<Map<String, Object>>) MapHelper.dig("hits.hits", responseMap);
-            assertThat(hits, hasSize(3));
-            for (var hit : hits) {
-                String filter = (String) MapHelper.dig("_source.filter_field", hit);
-                assertEquals("foo", filter);
-            }
-        }
-        {
-            var semanticSearchResponse = semanticSearchWithTermsFilter(indexName, inputs.get(2), "baz", modelId, "embedding");
-            assertOkWithErrorMessage(semanticSearchResponse);
-
-            Map<String, Object> responseMap = responseAsMap(semanticSearchResponse);
-            List<Map<String, Object>> hits = (List<Map<String, Object>>) MapHelper.dig("hits.hits", responseMap);
-            assertThat(hits, hasSize(2));
-            for (var hit : hits) {
-                String filter = (String) MapHelper.dig("_source.filter_field", hit);
-                assertEquals("baz", filter);
-            }
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testHybridSearch() throws IOException {
-        String modelId = "hybrid-semantic-search-test";
-        String indexName = modelId + "-index";
-
-        createTextEmbeddingModel(modelId);
-        putModelDefinition(modelId, BASE_64_ENCODED_MODEL, RAW_MODEL_SIZE);
-        putVocabulary(
-            List.of("these", "are", "my", "words", "the", "washing", "machine", "is", "leaking", "octopus", "comforter", "smells"),
-            modelId
-        );
-        startDeployment(modelId);
-
-        List<String> inputs = List.of(
-            "my words",
-            "the machine is leaking",
-            "washing machine",
-            "these are my words",
-            "the octopus comforter smells",
-            "the octopus comforter is leaking",
-            "washing machine smells"
-        );
-        List<String> filters = List.of("foo", "bar", "baz", "foo", "bar", "baz", "foo");
-        List<List<Double>> embeddings = new ArrayList<>();
-
-        // Generate the text embeddings via the inference API
-        // then index them for search
-        for (var input : inputs) {
-            Response inference = infer(input, modelId);
-            List<Map<String, Object>> responseMap = (List<Map<String, Object>>) entityAsMap(inference).get("inference_results");
-            Map<String, Object> inferenceResult = responseMap.get(0);
-            List<Double> embedding = (List<Double>) inferenceResult.get("predicted_value");
-            embeddings.add(embedding);
-        }
-
-        // index dense vectors
-        createVectorSearchIndex(indexName);
-        bulkIndexDocs(inputs, filters, embeddings, indexName);
-        forceMergeIndex(indexName);
-
-        String queryTemplate = """
-            {"match": {"source_text": {"query": "%s"}}}
-            """;
-
-        {
-            // combined query should return size documents where size > k
-            Request request = new Request("GET", indexName + "/_semantic_search");
-            request.setJsonEntity(String.format(Locale.ROOT, """
-                {
-                  "model_id": "%s",
-                  "query_string": "my words",
-                  "knn": {
-                      "field": "embedding",
-                      "k": 3,
-                      "num_candidates": 10,
-                      "boost": 10.0
-                  },
-                  "query": {"match_all": {}},
-                  "size": 7
-                }""", modelId));
-            var semanticSearchResponse = client().performRequest(request);
-            assertOkWithErrorMessage(semanticSearchResponse);
-
-            Map<String, Object> responseMap = responseAsMap(semanticSearchResponse);
-            int hitCount = (Integer) MapHelper.dig("hits.total.value", responseMap);
-            assertEquals(7, hitCount);
-        }
-        {
-            // boost the knn score, as the query is an exact match the unboosted
-            // score should be close to 1.0. Use an unrelated query so scores are
-            // not combined
-            Request request = new Request("GET", indexName + "/_semantic_search");
-            request.setJsonEntity(String.format(Locale.ROOT, """
-                {
-                  "model_id": "%s",
-                  "query_string": "my words",
-                  "knn": {
-                      "field": "embedding",
-                      "k": 3,
-                      "num_candidates": 10,
-                      "boost": 10.0
-                  },
-                  "query": {"match": {"source_text": {"query": "apricot unrelated"}}}
-                }""", modelId));
-            var semanticSearchResponse = client().performRequest(request);
-            assertOkWithErrorMessage(semanticSearchResponse);
-
-            Map<String, Object> responseMap = responseAsMap(semanticSearchResponse);
-            List<Map<String, Object>> hits = (List<Map<String, Object>>) MapHelper.dig("hits.hits", responseMap);
-            boolean found = false;
-            for (var hit : hits) {
-                String source = (String) MapHelper.dig("_source.source_text", hit);
-                if (source.equals("my words")) {
-                    assertThat((Double) MapHelper.dig("_score", hit), closeTo(10.0, 0.01));
-                    found = true;
-                }
-            }
-            assertTrue("should have found hit for string 'my words'", found);
-        }
-    }
-
     private void createVectorSearchIndex(String indexName) throws IOException {
         Request createIndex = new Request("PUT", "/" + indexName);
         createIndex.setJsonEntity("""
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java
index 945cca4bb120..991af48afdf0 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.SecuritySettingsSourceField;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -365,7 +366,7 @@ private void putPyTorchModel(String modelId) throws IOException {
 
     private void putModelDefinitionPart(String modelId, int totalSize, int numParts, int partNumber) throws IOException {
         Request request = new Request("PUT", "_ml/trained_models/" + modelId + "/definition/" + partNumber);
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {
               "total_definition_length": %s,
               "definition": "UEsDBAAACAgAAAAAAAAAAAAAAAAAAAAAAAAUAA4Ac2ltcGxlbW9kZW==",
diff --git a/x-pack/plugin/ml/qa/semantic-search-tests/build.gradle b/x-pack/plugin/ml/qa/semantic-search-tests/build.gradle
deleted file mode 100644
index 52c532a5d654..000000000000
--- a/x-pack/plugin/ml/qa/semantic-search-tests/build.gradle
+++ /dev/null
@@ -1,23 +0,0 @@
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-
-dependencies {
-  yamlRestTestImplementation(testArtifact(project(xpackModule('core'))))
-  yamlRestTestImplementation(testArtifact(project(':x-pack:plugin')))
-}
-
-// bring in machine learning rest test suite
-restResources {
-  restApi {
-    include '_common', 'cluster', 'nodes', 'indices', 'index', 'search', 'get', 'bulk', 'ml', 'semantic_search'
-  }
-}
-
-testClusters.configureEach {
-  testDistribution = 'DEFAULT'
-  rolesFile file('roles.yml')
-  user username: "x_pack_rest_user", password: "x-pack-test-password"
-  user username: "read_index_no_ml", password: "read_index_no_ml_password", role: "all_data"
-  user username: "no_read_index_no_ml", password: "no_read_index_no_ml_password", role: "unrelated_index_only"
-  setting 'xpack.license.self_generated.type', 'trial'
-  setting 'xpack.security.enabled', 'true'
-}
diff --git a/x-pack/plugin/ml/qa/semantic-search-tests/roles.yml b/x-pack/plugin/ml/qa/semantic-search-tests/roles.yml
deleted file mode 100644
index 51bb1b9b5d4b..000000000000
--- a/x-pack/plugin/ml/qa/semantic-search-tests/roles.yml
+++ /dev/null
@@ -1,29 +0,0 @@
-all_data:
-  cluster:
-    # This is always required because the REST client uses it to find the version of
-    # Elasticsearch it's talking to
-    - cluster:monitor/main
-  indices:
-    # User
-    - names: [ 'embedded_text', 'unrelated' ]
-      privileges:
-        - create_index
-        - indices:admin/refresh
-        - read
-        - write
-        - view_index_metadata
-
-unrelated_index_only:
-  cluster:
-    # This is always required because the REST client uses it to find the version of
-    # Elasticsearch it's talking to
-    - cluster:monitor/main
-  indices:
-    #
-    - names: [ 'unrelated' ]
-      privileges:
-        - create_index
-        - indices:admin/refresh
-        - read
-        - write
-        - view_index_metadata
diff --git a/x-pack/plugin/ml/qa/semantic-search-tests/src/yamlRestTest/java/org/elasticsearch/smoketest/AbstractSemanticSearchPermissionsIT.java b/x-pack/plugin/ml/qa/semantic-search-tests/src/yamlRestTest/java/org/elasticsearch/smoketest/AbstractSemanticSearchPermissionsIT.java
deleted file mode 100644
index 645465e1ef56..000000000000
--- a/x-pack/plugin/ml/qa/semantic-search-tests/src/yamlRestTest/java/org/elasticsearch/smoketest/AbstractSemanticSearchPermissionsIT.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.smoketest;
-
-import com.carrotsearch.randomizedtesting.annotations.Name;
-
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.test.SecuritySettingsSourceField;
-import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
-import org.elasticsearch.xpack.test.rest.AbstractXPackRestTest;
-
-import java.util.Collections;
-import java.util.Map;
-
-public abstract class AbstractSemanticSearchPermissionsIT extends AbstractXPackRestTest {
-
-    private static final String TEST_ADMIN_USERNAME = "x_pack_rest_user";
-
-    public AbstractSemanticSearchPermissionsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
-        super(testCandidate);
-    }
-
-    protected abstract String[] getCredentials();
-
-    @Override
-    protected Settings restClientSettings() {
-        String[] creds = getCredentials();
-        String token = basicAuthHeaderValue(creds[0], new SecureString(creds[1].toCharArray()));
-        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
-    }
-
-    @Override
-    protected Settings restAdminSettings() {
-        String token = basicAuthHeaderValue(TEST_ADMIN_USERNAME, SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING);
-        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
-    }
-
-    @Override
-    protected Map<String, String> getApiCallHeaders() {
-        return Collections.singletonMap(
-            "Authorization",
-            basicAuthHeaderValue(TEST_ADMIN_USERNAME, SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)
-        );
-    }
-
-    @Override
-    protected boolean isMachineLearningTest() {
-        return true;
-    }
-}
diff --git a/x-pack/plugin/ml/qa/semantic-search-tests/src/yamlRestTest/java/org/elasticsearch/smoketest/SemanticSearchNoReadPermissionsIT.java b/x-pack/plugin/ml/qa/semantic-search-tests/src/yamlRestTest/java/org/elasticsearch/smoketest/SemanticSearchNoReadPermissionsIT.java
deleted file mode 100644
index 0547c744677d..000000000000
--- a/x-pack/plugin/ml/qa/semantic-search-tests/src/yamlRestTest/java/org/elasticsearch/smoketest/SemanticSearchNoReadPermissionsIT.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.smoketest;
-
-import com.carrotsearch.randomizedtesting.annotations.Name;
-
-import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
-import org.elasticsearch.test.rest.yaml.section.DoSection;
-import org.elasticsearch.test.rest.yaml.section.ExecutableSection;
-
-import java.io.IOException;
-
-import static org.hamcrest.Matchers.containsString;
-
-public class SemanticSearchNoReadPermissionsIT extends AbstractSemanticSearchPermissionsIT {
-
-    private static final String USERNAME = "no_read_index_no_ml";
-
-    private final ClientYamlTestCandidate testCandidate;
-
-    public SemanticSearchNoReadPermissionsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
-        super(testCandidate);
-        this.testCandidate = testCandidate;
-    }
-
-    @Override
-    protected String[] getCredentials() {
-        return new String[] { USERNAME, "no_read_index_no_ml_password" };
-    }
-
-    @Override
-    public void test() throws IOException {
-        try {
-            // Cannot use expectThrows here because blacklisted tests will throw an
-            // InternalAssumptionViolatedException rather than an AssertionError
-            super.test();
-
-            for (ExecutableSection section : testCandidate.getTestSection().getExecutableSections()) {
-                if (section instanceof DoSection doSection) {
-                    String apiName = doSection.getApiCallSection().getApi();
-                    fail("call to semantic_search endpoint [" + apiName + "] should have failed because of missing role");
-                }
-            }
-        } catch (AssertionError ae) {
-            if (ae.getMessage().startsWith("call to")) {
-                // rethrow the fail() from the try section above
-                throw ae;
-            }
-            assertThat(ae.getMessage(), containsString("security_exception"));
-            assertThat(ae.getMessage(), containsString("returned [403 Forbidden]"));
-            assertThat(ae.getMessage(), containsString("is unauthorized for user [" + USERNAME + "]"));
-        }
-    }
-}
diff --git a/x-pack/plugin/ml/qa/semantic-search-tests/src/yamlRestTest/java/org/elasticsearch/smoketest/SemanticSearchReadPermissionsIT.java b/x-pack/plugin/ml/qa/semantic-search-tests/src/yamlRestTest/java/org/elasticsearch/smoketest/SemanticSearchReadPermissionsIT.java
deleted file mode 100644
index e81ccd561412..000000000000
--- a/x-pack/plugin/ml/qa/semantic-search-tests/src/yamlRestTest/java/org/elasticsearch/smoketest/SemanticSearchReadPermissionsIT.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.smoketest;
-
-import com.carrotsearch.randomizedtesting.annotations.Name;
-
-import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
-
-public class SemanticSearchReadPermissionsIT extends AbstractSemanticSearchPermissionsIT {
-
-    public SemanticSearchReadPermissionsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
-        super(testCandidate);
-    }
-
-    @Override
-    protected String[] getCredentials() {
-        return new String[] { "read_index_no_ml", "read_index_no_ml_password" };
-    }
-}
diff --git a/x-pack/plugin/ml/qa/semantic-search-tests/src/yamlRestTest/resources/rest-api-spec/test/semantic_search/10_basic.yml b/x-pack/plugin/ml/qa/semantic-search-tests/src/yamlRestTest/resources/rest-api-spec/test/semantic_search/10_basic.yml
deleted file mode 100644
index 4affee774c14..000000000000
--- a/x-pack/plugin/ml/qa/semantic-search-tests/src/yamlRestTest/resources/rest-api-spec/test/semantic_search/10_basic.yml
+++ /dev/null
@@ -1,126 +0,0 @@
-# This test uses the simple text embedding model from
-# SemanticSearchIT.java to create the dense vectors.
-setup:
-  - skip:
-      features: headers
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-      indices.create:
-        index: embedded_text
-        body:
-          settings:
-            number_of_replicas: 0
-          mappings:
-            properties:
-              source_text:
-                type: keyword
-              embedding:
-                type: dense_vector
-                dims: 100
-                index: true
-                similarity: cosine
-
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-      indices.create:
-        index: unrelated
-        body:
-          settings:
-            number_of_replicas: 0
-          mappings:
-            properties:
-              source_text:
-                type: keyword
-
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-      ml.put_trained_model:
-        model_id: "text_embedding_model"
-        body: >
-          {
-            "description": "simple model for testing",
-            "model_type": "pytorch",
-            "inference_config": {
-              "text_embedding": {
-                "tokenization": {
-                  "bert": {
-                    "with_special_tokens": false
-                  }
-                }
-              }
-            }
-          }
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-      ml.put_trained_model_vocabulary:
-        model_id: "text_embedding_model"
-        body: >
-          { "vocabulary": ["[PAD]", "[UNK]", "these", "are", "my", "words", "the", "washing", "machine", "is", "leaking", "octopus", "comforter", "smells"] }
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-      ml.put_trained_model_definition_part:
-        model_id: "text_embedding_model"
-        part: 0
-        body: >
-          {
-            "total_definition_length":1694,
-            "definition": "UEsDBAAACAgAAAAAAAAAAAAAAAAAAAAAAAAUAA4Ac2ltcGxlbW9kZWwvZGF0YS5wa2xGQgoAWlpaWlpaWlpaWoACY19fdG9yY2hfXwpUaW55VGV4dEVtYmVkZGluZwpxACmBfShYCAAAAHRyYWluaW5ncQGJWBYAAABfaXNfZnVsbF9iYWNrd2FyZF9ob29rcQJOdWJxAy5QSwcIsFTQsFgAAABYAAAAUEsDBBQACAgIAAAAAAAAAAAAAAAAAAAAAAAdAB0Ac2ltcGxlbW9kZWwvY29kZS9fX3RvcmNoX18ucHlGQhkAWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWoWPMWvDMBCF9/yKGy1IQ7Ia0q1j2yWbMYdsnWphWWd0Em3+fS3bBEopXd99j/dd77UI3Fy43+grvUwdGePCR/XKJntS9QEAcdZRT5QoCiJcoWnXtMvW/ohS1C4sZaihY/YFcoI2e4+d7sdPHQ0OzONyf5+T46B9U8DSNWTBcixMJeRtvQwkjv2AePpld1wKAC7MOaEzUsONgnDc4sQjBUz3mbbbY2qD2usbB9rQmcWV47/gOiVIReAvUsHT8y5S7yKL/mnSIWuPQmSqLRm0DJWkWD0eUEqtjUgpx7AXow6mai5HuJzPrTp8A1BLBwiD/6yJ6gAAAKkBAABQSwMEFAAICAgAAAAAAAAAAAAAAAAAAAAAACcAQQBzaW1wbGVtb2RlbC9jb2RlL19fdG9yY2hfXy5weS5kZWJ1Z19wa2xGQj0AWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWo2Qz0rDQBDGk/5RmjfwlmMCbWivBZ9gWL0IFkRCdLcmmOwmuxu0N08O3r2rCOrdx9CDgm/hWUUQMdugzUk6LCwzv++bGeak5YE1saoorNgCCwsbzFc9sm1PvivQo2zqToU8iiT1FEunfadXRcLzUocJVWN3i3ElZF3W4pDxUM9yVrPNXCeCR+lOLdp1190NwVktzoVKDF5COh+nQpbtsX+0/tjpOWYJuR8HMuJUZEEW8TJKQ8UY9eJIxZ7S0vvb3vf9yiCZLiV3Fz5v1HdHw6HvFK3JWnUElWR5ygbz8TThB4NMUJYG+axowyoWHbiHBwQbSWbHHXiEJ4QWkmOTPMMLQhvJaZOgSX49Z3a8uPq5Ia/whtBBctEkl4a8wwdCF8lVk1wb8glfCCtIbprkttntrkF00Q1+AFBLBwi4BIswOAEAAP0BAABQSwMEAAAICAAAAAAAAAAAAAAAAAAAAAAAABkAQQBzaW1wbGVtb2RlbC9jb25zdGFudHMucGtsRkI9AFpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlqAAikuUEsHCG0vCVcEAAAABAAAAFBLAwQAAAgIAAAAAAAAAAAAAAAAAAAAAAAAEwA7AHNpbXBsZW1vZGVsL3ZlcnNpb25GQjcAWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWjMKUEsHCNGeZ1UCAAAAAgAAAFBLAQIAAAAACAgAAAAAAACwVNCwWAAAAFgAAAAUAAAAAAAAAAAAAAAAAAAAAABzaW1wbGVtb2RlbC9kYXRhLnBrbFBLAQIAABQACAgIAAAAAACD/6yJ6gAAAKkBAAAdAAAAAAAAAAAAAAAAAKgAAABzaW1wbGVtb2RlbC9jb2RlL19fdG9yY2hfXy5weVBLAQIAABQACAgIAAAAAAC4BIswOAEAAP0BAAAnAAAAAAAAAAAAAAAAAPoBAABzaW1wbGVtb2RlbC9jb2RlL19fdG9yY2hfXy5weS5kZWJ1Z19wa2xQSwECAAAAAAgIAAAAAAAAbS8JVwQAAAAEAAAAGQAAAAAAAAAAAAAAAADIAwAAc2ltcGxlbW9kZWwvY29uc3RhbnRzLnBrbFBLAQIAAAAACAgAAAAAAADRnmdVAgAAAAIAAAATAAAAAAAAAAAAAAAAAFQEAABzaW1wbGVtb2RlbC92ZXJzaW9uUEsGBiwAAAAAAAAAHgMtAAAAAAAAAAAABQAAAAAAAAAFAAAAAAAAAGoBAAAAAAAA0gQAAAAAAABQSwYHAAAAADwGAAAAAAAAAQAAAFBLBQYAAAAABQAFAGoBAADSBAAAAAA=",
-            "total_parts": 1
-          }
-
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-        Content-Type: application/json
-      bulk:
-        index: embedded_text
-        refresh: true
-        body: |
-          {"index": {}}
-          {"source_text": "the octopus comforter smells", "embedding":[0.3925197124481201, 0.9145996570587158, 0.01372915506362915, 0.9987854957580566, 0.3240084648132324, 0.6188188195228577, 0.926924467086792, 0.12143599987030029, 0.175662100315094, 0.16076070070266724, 0.7671306133270264, 0.9518267512321472, 0.4557478427886963, 0.5410670638084412, 0.7594802975654602, 0.5035953521728516, 0.4115469455718994, 0.038427770137786865, 0.5419668555259705, 0.6362232565879822, 0.17554593086242676, 0.01821446418762207, 0.2931918501853943, 0.294437050819397, 0.6901726722717285, 0.1679999828338623, 0.7995195984840393, 0.8781598210334778, 0.18507736921310425, 0.8614458441734314, 0.690071702003479, 0.7859554886817932, 0.803643524646759, 0.0048452019691467285, 0.19700628519058228, 0.22210919857025146, 0.7043975591659546, 0.6320799589157104, 0.542057991027832, 0.8704766035079956, 0.32195907831192017, 0.7272325158119202, 0.4066658020019531, 0.89588862657547, 0.7947880029678345, 0.06543421745300293, 0.2873639464378357, 0.8773637413978577, 0.36480581760406494, 0.692948043346405, 0.19171112775802612, 0.14275449514389038, 0.17054951190948486, 0.8969640135765076, 0.39838290214538574, 0.26756417751312256, 0.5369327664375305, 0.4736328721046448, 0.21181154251098633, 0.2695402503013611, 0.8651300072669983, 0.8051849603652954, 0.7073134779930115, 0.5963589549064636, 0.09601861238479614, 0.5362404584884644, 0.23020631074905396, 0.8515381813049316, 0.5730932354927063, 0.7235705256462097, 0.08228331804275513, 0.5840849280357361, 0.6030013561248779, 0.2084050178527832, 0.7312950491905212, 0.6159517168998718, 0.6482340693473816, 0.07220339775085449, 0.5136227607727051, 0.9152160286903381, 0.8169018030166626, 0.15515869855880737, 0.7978536486625671, 0.564482569694519, 0.4757157564163208, 0.2718064785003662, 0.6910138726234436, 0.5675734877586365, 0.702862024307251, 0.19079893827438354, 0.8995556235313416, 0.4988499879837036, 0.6378234028816223, 0.2683940529823303, 0.21990180015563965, 0.8442690372467041, 0.8502047061920166, 0.9857811331748962, 0.3549607992172241, 0.7605474591255188]}
-          {"index": {}}
-          {"source_text": "the machine is leaking", "embedding":[0.09775793552398682, 0.9594467282295227, 0.7915146946907043, 0.9140479564666748, 0.5148435235023499, 0.8556410670280457, 0.6022665500640869, 0.05222177505493164, 0.9821935296058655, 0.49276530742645264, 0.23147249221801758, 0.2428399920463562, 0.3865380883216858, 0.5778483748435974, 0.5600519776344299, 0.9427472352981567, 0.48832541704177856, 0.8807493448257446, 0.32909590005874634, 0.8452557325363159, 0.811530590057373, 0.13344216346740723, 0.15256845951080322, 0.5025331974029541, 0.4288772940635681, 0.6590417623519897, 0.9282752871513367, 0.8842046856880188, 0.7873250842094421, 0.356731653213501, 0.9959152936935425, 0.07572609186172485, 0.5062583088874817, 0.36245888471603394, 0.6189196705818176, 0.7766605019569397, 0.5198523998260498, 0.7379586100578308, 0.0553441047668457, 0.5035901665687561, 0.24139636754989624, 0.10798943042755127, 0.272808313369751, 0.38171595335006714, 0.24275553226470947, 0.956981897354126, 0.8182021379470825, 0.9383817315101624, 0.06551980972290039, 0.6892690658569336, 0.7068917751312256, 0.5184322595596313, 0.6103079319000244, 0.7020677328109741, 0.7181660532951355, 0.6477184295654297, 0.26282840967178345, 0.9316624402999878, 0.8318467140197754, 0.1487215757369995, 0.39937925338745117, 0.6842989921569824, 0.3496543765068054, 0.6008991003036499, 0.9530165791511536, 0.4209877848625183, 0.5675879716873169, 0.7883706092834473, 0.9547191858291626, 0.6292906403541565, 0.49566715955734253, 0.6907342672348022, 0.0834314227104187, 0.19785481691360474, 0.4896165728569031, 0.8460168838500977, 0.9680339097976685, 0.43386441469192505, 0.7068926095962524, 0.19123870134353638, 0.5661664009094238, 0.610595166683197, 0.23599380254745483, 0.2831611633300781, 0.7919651865959167, 0.0018386244773864746, 0.15559959411621094, 0.4622604250907898, 0.02038663625717163, 0.42241227626800537, 0.4200526475906372, 0.1223069429397583, 0.7035380005836487, 0.09902423620223999, 0.7804107666015625, 0.05339455604553223, 0.6485095024108887, 0.29347676038742065, 0.9716366529464722, 0.30257928371429443]}
-          {"index": {}}
-          {"source_text": "these are my words", "embedding":[0.7000167369842529, 0.590781033039093, 0.009879708290100098, 0.7874260544776917, 0.797156572341919, 0.1791083812713623, 0.07826781272888184, 0.25102007389068604, 0.09334254264831543, 0.3819708824157715, 0.7312374711036682, 0.02819347381591797, 0.20099765062332153, 0.7702597975730896, 0.9443559050559998, 0.35520339012145996, 0.25699591636657715, 0.5596823692321777, 0.23947590589523315, 0.47478222846984863, 0.23411548137664795, 0.9809996485710144, 0.3806597590446472, 0.5006771087646484, 0.5724453926086426, 0.21510547399520874, 0.07062828540802002, 0.9858258962631226, 0.9636645317077637, 0.36034029722213745, 0.07260054349899292, 0.06882566213607788, 0.18354403972625732, 0.06756395101547241, 0.5749042630195618, 0.05275309085845947, 0.1865217685699463, 0.5852730870246887, 0.1086682677268982, 0.10090464353561401, 0.32582908868789673, 0.5494027733802795, 0.873362123966217, 0.02236837148666382, 0.37973177433013916, 0.5556552410125732, 0.5083678364753723, 0.8081125020980835, 0.09164196252822876, 0.2207810878753662, 0.8086426258087158, 0.271828293800354, 0.5981417298316956, 0.7745779156684875, 0.40872830152511597, 0.6035888195037842, 0.5598325133323669, 0.19086670875549316, 0.02406853437423706, 0.8299782872200012, 0.4994274377822876, 0.0300295352935791, 0.47190529108047485, 0.8889331817626953, 0.34195321798324585, 0.9380808472633362, 0.4418332576751709, 0.5789303779602051, 0.0526617169380188, 0.7349719405174255, 0.44571834802627563, 0.6602563261985779, 0.3819742202758789, 0.16881734132766724, 0.45588219165802, 0.028081774711608887, 0.6681976914405823, 0.8183007836341858, 0.7887755632400513, 0.4506028890609741, 0.8040162324905396, 0.431918203830719, 0.7408918738365173, 0.39756304025650024, 0.7438145875930786, 0.6120601892471313, 0.5724676251411438, 0.08701330423355103, 0.18344634771347046, 0.7226220369338989, 0.3648560643196106, 0.9813777208328247, 0.2615315318107605, 0.9847549796104431, 0.32967478036880493, 0.47099196910858154, 0.3591546416282654, 0.4132147431373596, 0.48631107807159424, 0.04420149326324463]}
-          {"index": {}}
-          {"source_text": "washing machine", "embedding":[0.7044712901115417, 0.12284207344055176, 0.5008929967880249, 0.04643195867538452, 0.3666788339614868, 0.26660799980163574, 0.24114298820495605, 0.0761682391166687, 0.5294214487075806, 0.16935181617736816, 0.6257967948913574, 0.2804388999938965, 0.6417903900146484, 0.169958233833313, 0.4216839075088501, 0.6773303747177124, 0.9472144842147827, 0.21874648332595825, 0.5095921754837036, 0.839306116104126, 0.6176233291625977, 0.5847064852714539, 0.6748610734939575, 0.3264034390449524, 0.4112023115158081, 0.13818275928497314, 0.08356589078903198, 0.4147903323173523, 0.5626787543296814, 0.7167286276817322, 0.6314535737037659, 0.23092854022979736, 0.34547603130340576, 0.7425565719604492, 0.2837678790092468, 0.47037917375564575, 0.1555209755897522, 0.5618507266044617, 0.2076261043548584, 0.3026384711265564, 0.04561811685562134, 0.1691250205039978, 0.2504339814186096, 0.5350574851036072, 0.26857447624206543, 0.23607933521270752, 0.16938960552215576, 0.23708534240722656, 0.026302993297576904, 0.16901731491088867, 0.2847784757614136, 0.944273829460144, 0.28171658515930176, 0.9864799380302429, 0.6811433434486389, 0.9383156895637512, 0.5682582259178162, 0.14361613988876343, 0.7900274395942688, 0.27808505296707153, 0.05677521228790283, 0.08594226837158203, 0.6450491547584534, 0.06500720977783203, 0.36045730113983154, 0.1987738013267517, 0.07287931442260742, 0.5315744280815125, 0.04742676019668579, 0.7842378616333008, 0.0881078839302063, 0.7612627744674683, 0.2528950572013855, 0.27305954694747925, 0.03027820587158203, 0.4686838984489441, 0.13311690092086792, 0.048372089862823486, 0.808062732219696, 0.44010263681411743, 0.5726178288459778, 0.15828031301498413, 0.4597446322441101, 0.6375324130058289, 0.8452948927879333, 0.9763500690460205, 0.5094607472419739, 0.3535742163658142, 0.664739191532135, 0.40749913454055786, 0.8537857532501221, 0.5830079913139343, 0.7949922680854797, 0.6309236288070679, 0.07258343696594238, 0.1224660873413086, 0.24250483512878418, 0.36189037561416626, 0.5156043171882629, 0.1819135546684265]}
-          {"index": {}}
-          {"source_text": "washing machine smells", "embedding":[0.7249823808670044, 0.3981819152832031, 0.4572623372077942, 0.7442894577980042, 0.15898281335830688, 0.6481881737709045, 0.1513708233833313, 0.8945682644844055, 0.7708938121795654, 0.5494217276573181, 0.48253726959228516, 0.39402270317077637, 0.6369197368621826, 0.7152248024940491, 0.6326345205307007, 0.7362181544303894, 0.350342333316803, 0.16101288795471191, 0.4180338382720947, 0.04114532470703125, 0.002633512020111084, 0.20396709442138672, 0.8963556885719299, 0.1552276611328125, 0.7476853728294373, 0.9651047587394714, 0.7527561187744141, 0.7041972279548645, 0.12461084127426147, 0.6282403469085693, 0.9631509184837341, 0.16590750217437744, 0.4101366400718689, 0.31320667266845703, 0.13579899072647095, 0.2895740270614624, 0.9905323386192322, 0.02118372917175293, 0.637545645236969, 0.5133231282234192, 0.679695188999176, 0.04641437530517578, 0.21913814544677734, 0.16534924507141113, 0.02987360954284668, 0.14805591106414795, 0.16874665021896362, 0.9378783702850342, 0.8607399463653564, 0.7287217974662781, 0.5402306318283081, 0.9973209500312805, 0.26169413328170776, 0.3835873603820801, 0.1874808669090271, 0.8038567304611206, 0.18557673692703247, 0.8631893992424011, 0.7676172256469727, 0.3599127531051636, 0.48698097467422485, 0.926689088344574, 0.6542723774909973, 0.49722349643707275, 0.7027173638343811, 0.13385021686553955, 0.9873734712600708, 0.17187494039535522, 0.7995050549507141, 0.5259199142456055, 0.33804380893707275, 0.21665722131729126, 0.952264130115509, 0.8337767720222473, 0.879487156867981, 0.5553549528121948, 0.6160674095153809, 0.1315295696258545, 0.8010737895965576, 0.834412693977356, 0.20340144634246826, 0.8993185758590698, 0.6493895649909973, 0.9454924464225769, 0.38529330492019653, 0.6891772150993347, 0.5530646443367004, 0.18555349111557007, 0.8361382484436035, 0.11815804243087769, 0.38942235708236694, 0.945141613483429, 0.6417409181594849, 0.39776402711868286, 0.5133314728736877, 0.5431299805641174, 0.2615429759025574, 0.8987119793891907, 0.023733675479888916, 0.4941052794456482]}
-          {"index": {}}
-          {"source_text": "my words", "embedding":[0.19087255001068115, 0.5498749017715454, 0.9536173939704895, 0.25011056661605835, 0.37642204761505127, 0.18271470069885254, 0.670674741268158, 0.5553990006446838, 0.3306507468223572, 0.3368762731552124, 0.053364574909210205, 0.047215282917022705, 0.4221981167793274, 0.7591024041175842, 0.998794436454773, 0.6113318204879761, 0.8178470730781555, 0.8554672598838806, 0.40100908279418945, 0.6486459374427795, 0.804382860660553, 0.6775466799736023, 0.2916865944862366, 0.7019925117492676, 0.9812073707580566, 0.4414554834365845, 0.08203905820846558, 0.9167835116386414, 0.3082762360572815, 0.5454868674278259, 0.6665160655975342, 0.06828844547271729, 0.36014634370803833, 0.01810687780380249, 0.2640475630760193, 0.1856365203857422, 0.4734996557235718, 0.8153479695320129, 0.9614933133125305, 0.4851576089859009, 0.003343045711517334, 0.17352384328842163, 0.26423048973083496, 0.24217921495437622, 0.5694647431373596, 0.8538861274719238, 0.06464511156082153, 0.038984060287475586, 0.7695011496543884, 0.008188009262084961, 0.3858819007873535, 0.7950196862220764, 0.7225212454795837, 0.3982154130935669, 0.4996080994606018, 0.28709208965301514, 0.6753579378128052, 0.6779837608337402, 0.4815831184387207, 0.27917128801345825, 0.8400004506111145, 0.9022405743598938, 0.8253144025802612, 0.6251398324966431, 0.25444501638412476, 0.7694959044456482, 0.006821691989898682, 0.7958594560623169, 0.9144708514213562, 0.8688076138496399, 0.9641174077987671, 0.44437146186828613, 0.06135892868041992, 0.2638128399848938, 0.05436718463897705, 0.9926314353942871, 0.8661795854568481, 0.9176243543624878, 0.5521496534347534, 0.6017677783966064, 0.22096896171569824, 0.7030748128890991, 0.16923701763153076, 0.8178754448890686, 0.47008246183395386, 0.28875309228897095, 0.14314061403274536, 0.3431167006492615, 0.9301973581314087, 0.5416158437728882, 0.563427209854126, 0.7897542119026184, 0.2761036157608032, 0.16855067014694214, 0.42684781551361084, 0.7562968730926514, 0.2551668882369995, 0.7754542827606201, 0.218039870262146, 0.7080662846565247]}
-
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-        Content-Type: application/json
-      indices.forcemerge:
-        index: embedded_text
-        max_num_segments: 1
-
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-        Content-Type: application/json
-      ml.start_trained_model_deployment:
-        model_id: text_embedding_model
-        wait_for: started
-
----
-"Test semantic search":
-  - do:
-      semantic_search:
-        index: embedded_text
-        body:
-          model_id: text_embedding_model
-          query_string: "the octopus comforter smells"
-          knn:
-            field: embedding
-            k: 3
-            num_candidates: 10
-  - gte: { inference_took: 0 }
-  - match: { hits.total.value: 3 }
diff --git a/x-pack/plugin/ml/qa/single-node-tests/build.gradle b/x-pack/plugin/ml/qa/single-node-tests/build.gradle
index cf1d2571c2d9..eb86ca600d75 100644
--- a/x-pack/plugin/ml/qa/single-node-tests/build.gradle
+++ b/x-pack/plugin/ml/qa/single-node-tests/build.gradle
@@ -1,6 +1,6 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 testClusters.configureEach {
   testDistribution = 'DEFAULT'
diff --git a/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceIT.java b/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceIT.java
index 2e13fdb90d2f..b974ba4a02a2 100644
--- a/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceIT.java
+++ b/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceIT.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -104,7 +105,7 @@ public void testAllFieldsMissingWarning() throws IOException {
 
     private Response infer(String modelId, String body) throws IOException {
         Request request = new Request("POST", "/_ml/trained_models/" + modelId + "/_infer");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {  "docs": [%s] }
             """, body));
         return client().performRequest(request);
diff --git a/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceProcessorIT.java b/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceProcessorIT.java
index 864dbb616eb7..b62b0e170585 100644
--- a/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceProcessorIT.java
+++ b/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceProcessorIT.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -220,7 +221,7 @@ public void testCreateProcessorWithDeprecatedFields() throws Exception {
 
         createdPipelines.add("regression-model-deprecated-pipeline");
         Request putPipeline = new Request("PUT", "_ingest/pipeline/regression-model-deprecated-pipeline");
-        putPipeline.setJsonEntity(formatted("""
+        putPipeline.setJsonEntity(Strings.format("""
             {
               "processors": [
                 {
@@ -283,7 +284,7 @@ private void infer(String pipelineId) throws IOException {
 
     private void putPipeline(String modelId, String pipelineName) throws IOException {
         Request putPipeline = new Request("PUT", "_ingest/pipeline/" + pipelineName);
-        putPipeline.setJsonEntity(formatted("""
+        putPipeline.setJsonEntity(Strings.format("""
             {
               "processors": [
                 {
diff --git a/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/transforms/PainlessDomainSplitIT.java b/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/transforms/PainlessDomainSplitIT.java
index e045c9f7307a..cbdad9e86f4f 100644
--- a/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/transforms/PainlessDomainSplitIT.java
+++ b/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/transforms/PainlessDomainSplitIT.java
@@ -205,7 +205,7 @@ public void testIsolated() throws Exception {
             logger.info("params={}", mapAsJson);
 
             Request searchRequest = new Request("GET", "/painless/_search");
-            searchRequest.setJsonEntity(formatted("""
+            searchRequest.setJsonEntity(Strings.format("""
                 {
                     "query" : {
                         "match_all": {}
@@ -305,7 +305,7 @@ public void testHRDSplit() throws Exception {
                 // Anomaly has 100 docs, but we don't care about the value
                 for (int j = 0; j < 100; j++) {
                     Request createDocRequest = new Request("POST", "/painless/_doc");
-                    createDocRequest.setJsonEntity(formatted("""
+                    createDocRequest.setJsonEntity(Strings.format("""
                         {"domain": "bar.bar.com", "time": "%s"}
                         """, formattedTime));
                     client().performRequest(createDocRequest);
@@ -313,7 +313,7 @@ public void testHRDSplit() throws Exception {
             } else {
                 // Non-anomalous values will be what's seen when the anomaly is reported
                 Request createDocRequest = new Request("PUT", "/painless/_doc/" + formattedTime);
-                createDocRequest.setJsonEntity(formatted("""
+                createDocRequest.setJsonEntity(Strings.format("""
                     {"domain": "%s", "time": "%s"}
                     """, test.hostName, formattedTime));
                 client().performRequest(createDocRequest);
diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/license/MachineLearningLicensingIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/license/MachineLearningLicensingIT.java
index 4036b22ba196..50da96be5068 100644
--- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/license/MachineLearningLicensingIT.java
+++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/license/MachineLearningLicensingIT.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.license.License.OperationMode;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
@@ -544,7 +545,7 @@ public void testMachineLearningCreateInferenceProcessorRestricted() {
             .execute()
             .actionGet();
 
-        String simulateSource = formatted("""
+        String simulateSource = Strings.format("""
             {
               "pipeline": %s,
               "docs": [
@@ -663,7 +664,7 @@ public void testMachineLearningInferModelRestricted() {
         PlainActionFuture<InferModelAction.Response> inferModelSuccess = PlainActionFuture.newFuture();
         client().execute(
             InferModelAction.INSTANCE,
-            new InferModelAction.Request(
+            InferModelAction.Request.forDocs(
                 modelId,
                 Collections.singletonList(Collections.emptyMap()),
                 RegressionConfigUpdate.EMPTY_PARAMS,
@@ -684,7 +685,7 @@ public void testMachineLearningInferModelRestricted() {
         ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, () -> {
             client().execute(
                 InferModelAction.INSTANCE,
-                new InferModelAction.Request(
+                InferModelAction.Request.forDocs(
                     modelId,
                     Collections.singletonList(Collections.emptyMap()),
                     RegressionConfigUpdate.EMPTY_PARAMS,
@@ -700,7 +701,7 @@ public void testMachineLearningInferModelRestricted() {
         inferModelSuccess = PlainActionFuture.newFuture();
         client().execute(
             InferModelAction.INSTANCE,
-            new InferModelAction.Request(
+            InferModelAction.Request.forDocs(
                 modelId,
                 Collections.singletonList(Collections.emptyMap()),
                 RegressionConfigUpdate.EMPTY_PARAMS,
@@ -720,7 +721,7 @@ public void testMachineLearningInferModelRestricted() {
         PlainActionFuture<InferModelAction.Response> listener = PlainActionFuture.newFuture();
         client().execute(
             InferModelAction.INSTANCE,
-            new InferModelAction.Request(
+            InferModelAction.Request.forDocs(
                 modelId,
                 Collections.singletonList(Collections.emptyMap()),
                 RegressionConfigUpdate.EMPTY_PARAMS,
diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnomalyJobCRUDIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnomalyJobCRUDIT.java
index 1be8afe96194..4a091891de0a 100644
--- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnomalyJobCRUDIT.java
+++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnomalyJobCRUDIT.java
@@ -92,7 +92,7 @@ public void testUpdateModelMemoryLimitOnceEstablished() {
             new ModelSizeStats.Builder(jobId).setTimestamp(new Date()).setLogTime(new Date()).setModelBytes(10000000).build(),
             () -> false
         );
-        jobResultsPersister.commitResultWrites(jobId);
+        jobResultsPersister.commitWrites(jobId, JobResultsPersister.CommitType.RESULTS);
 
         ElasticsearchStatusException iae = expectThrows(
             ElasticsearchStatusException.class,
diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
index bb5ce746d2ff..63f1b4d5fc7a 100644
--- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
+++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
@@ -110,6 +110,8 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.startsWith;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
@@ -372,7 +374,7 @@ public void testParseQuantiles_GivenRenormalizationIsEnabled() throws Exception
         Optional<Quantiles> persistedQuantiles = getQuantiles();
         assertTrue(persistedQuantiles.isPresent());
         assertEquals(quantiles, persistedQuantiles.get());
-        verify(renormalizer).renormalize(quantiles);
+        verify(renormalizer).renormalize(eq(quantiles), any(Runnable.class));
     }
 
     public void testParseQuantiles_GivenRenormalizationIsDisabled() throws Exception {
@@ -389,7 +391,7 @@ public void testParseQuantiles_GivenRenormalizationIsDisabled() throws Exception
         Optional<Quantiles> persistedQuantiles = getQuantiles();
         assertTrue(persistedQuantiles.isPresent());
         assertEquals(quantiles, persistedQuantiles.get());
-        verify(renormalizer, never()).renormalize(quantiles);
+        verify(renormalizer, never()).renormalize(any(), any());
     }
 
     public void testDeleteInterimResults() throws Exception {
diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/EstablishedMemUsageIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/EstablishedMemUsageIT.java
index 5b297846a642..e09df368ecbd 100644
--- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/EstablishedMemUsageIT.java
+++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/EstablishedMemUsageIT.java
@@ -82,7 +82,7 @@ public void testEstablishedMem_givenNoStatsLongHistory() throws Exception {
         initClusterAndJob(jobId);
 
         createBuckets(jobId, 25);
-        jobResultsPersister.commitResultWrites(jobId);
+        jobResultsPersister.commitWrites(jobId, JobResultsPersister.CommitType.RESULTS);
 
         assertThat(queryEstablishedMemoryUsage(jobId), equalTo(0L));
     }
@@ -93,7 +93,7 @@ public void testEstablishedMem_givenNoStatsShortHistory() throws Exception {
         initClusterAndJob(jobId);
 
         createBuckets(jobId, 5);
-        jobResultsPersister.commitResultWrites(jobId);
+        jobResultsPersister.commitWrites(jobId, JobResultsPersister.CommitType.RESULTS);
 
         assertThat(queryEstablishedMemoryUsage(jobId), equalTo(0L));
     }
@@ -106,7 +106,7 @@ public void testEstablishedMem_givenHistoryTooShort() throws Exception {
         createBuckets(jobId, 19);
         createModelSizeStats(jobId, 1, 19000L);
         ModelSizeStats latestModelSizeStats = createModelSizeStats(jobId, 10, 20000L);
-        jobResultsPersister.commitResultWrites(jobId);
+        jobResultsPersister.commitWrites(jobId, JobResultsPersister.CommitType.RESULTS);
 
         assertThat(queryEstablishedMemoryUsage(jobId), equalTo(0L));
         assertThat(queryEstablishedMemoryUsage(jobId, 19, latestModelSizeStats), equalTo(0L));
@@ -120,7 +120,7 @@ public void testEstablishedMem_givenHistoryJustEnoughLowVariation() throws Excep
         createBuckets(jobId, 20);
         createModelSizeStats(jobId, 1, 19000L);
         ModelSizeStats latestModelSizeStats = createModelSizeStats(jobId, 10, 20000L);
-        jobResultsPersister.commitResultWrites(jobId);
+        jobResultsPersister.commitWrites(jobId, JobResultsPersister.CommitType.RESULTS);
 
         assertThat(queryEstablishedMemoryUsage(jobId), equalTo(20000L));
         assertThat(queryEstablishedMemoryUsage(jobId, 20, latestModelSizeStats), equalTo(20000L));
@@ -134,7 +134,7 @@ public void testEstablishedMem_givenHistoryJustEnoughAndUninitialized() throws E
         createBuckets(jobId, 20);
         createModelSizeStats(jobId, 1, 0L);
         ModelSizeStats latestModelSizeStats = createModelSizeStats(jobId, 10, 0L);
-        jobResultsPersister.commitResultWrites(jobId);
+        jobResultsPersister.commitWrites(jobId, JobResultsPersister.CommitType.RESULTS);
 
         assertThat(queryEstablishedMemoryUsage(jobId), equalTo(0L));
         assertThat(queryEstablishedMemoryUsage(jobId, 20, latestModelSizeStats), equalTo(0L));
@@ -148,7 +148,7 @@ public void testEstablishedMem_givenHistoryJustEnoughHighVariation() throws Exce
         createBuckets(jobId, 20);
         createModelSizeStats(jobId, 1, 1000L);
         ModelSizeStats latestModelSizeStats = createModelSizeStats(jobId, 10, 20000L);
-        jobResultsPersister.commitResultWrites(jobId);
+        jobResultsPersister.commitWrites(jobId, JobResultsPersister.CommitType.RESULTS);
 
         assertThat(queryEstablishedMemoryUsage(jobId), equalTo(0L));
         assertThat(queryEstablishedMemoryUsage(jobId, 20, latestModelSizeStats), equalTo(0L));
@@ -162,7 +162,7 @@ public void testEstablishedMem_givenLongEstablished() throws Exception {
         createBuckets(jobId, 25);
         createModelSizeStats(jobId, 1, 10000L);
         ModelSizeStats latestModelSizeStats = createModelSizeStats(jobId, 2, 20000L);
-        jobResultsPersister.commitResultWrites(jobId);
+        jobResultsPersister.commitWrites(jobId, JobResultsPersister.CommitType.RESULTS);
 
         assertThat(queryEstablishedMemoryUsage(jobId), equalTo(20000L));
         assertThat(queryEstablishedMemoryUsage(jobId, 25, latestModelSizeStats), equalTo(20000L));
@@ -176,7 +176,7 @@ public void testEstablishedMem_givenOneRecentChange() throws Exception {
         createBuckets(jobId, 25);
         createModelSizeStats(jobId, 1, 10000L);
         ModelSizeStats latestModelSizeStats = createModelSizeStats(jobId, 10, 20000L);
-        jobResultsPersister.commitResultWrites(jobId);
+        jobResultsPersister.commitWrites(jobId, JobResultsPersister.CommitType.RESULTS);
 
         assertThat(queryEstablishedMemoryUsage(jobId), equalTo(20000L));
         assertThat(queryEstablishedMemoryUsage(jobId, 25, latestModelSizeStats), equalTo(20000L));
@@ -189,7 +189,7 @@ public void testEstablishedMem_givenOneRecentChangeOnlyAndUninitialized() throws
 
         createBuckets(jobId, 25);
         ModelSizeStats latestModelSizeStats = createModelSizeStats(jobId, 10, 0L);
-        jobResultsPersister.commitResultWrites(jobId);
+        jobResultsPersister.commitWrites(jobId, JobResultsPersister.CommitType.RESULTS);
 
         assertThat(queryEstablishedMemoryUsage(jobId), equalTo(0L));
         assertThat(queryEstablishedMemoryUsage(jobId, 25, latestModelSizeStats), equalTo(0L));
@@ -202,7 +202,7 @@ public void testEstablishedMem_givenOneRecentChangeOnly() throws Exception {
 
         createBuckets(jobId, 25);
         ModelSizeStats latestModelSizeStats = createModelSizeStats(jobId, 10, 20000L);
-        jobResultsPersister.commitResultWrites(jobId);
+        jobResultsPersister.commitWrites(jobId, JobResultsPersister.CommitType.RESULTS);
 
         assertThat(queryEstablishedMemoryUsage(jobId), equalTo(20000L));
         assertThat(queryEstablishedMemoryUsage(jobId, 25, latestModelSizeStats), equalTo(20000L));
@@ -220,7 +220,7 @@ public void testEstablishedMem_givenHistoricHighVariationRecentLowVariation() th
         createModelSizeStats(jobId, 19, 9000L);
         createModelSizeStats(jobId, 30, 19000L);
         ModelSizeStats latestModelSizeStats = createModelSizeStats(jobId, 35, 20000L);
-        jobResultsPersister.commitResultWrites(jobId);
+        jobResultsPersister.commitWrites(jobId, JobResultsPersister.CommitType.RESULTS);
 
         assertThat(queryEstablishedMemoryUsage(jobId), equalTo(20000L));
         assertThat(queryEstablishedMemoryUsage(jobId, 40, latestModelSizeStats), equalTo(20000L));
@@ -238,7 +238,7 @@ public void testEstablishedMem_givenHistoricLowVariationRecentHighVariation() th
         createModelSizeStats(jobId, 27, 39000L);
         createModelSizeStats(jobId, 30, 67000L);
         ModelSizeStats latestModelSizeStats = createModelSizeStats(jobId, 35, 95000L);
-        jobResultsPersister.commitResultWrites(jobId);
+        jobResultsPersister.commitWrites(jobId, JobResultsPersister.CommitType.RESULTS);
 
         assertThat(queryEstablishedMemoryUsage(jobId), equalTo(0L));
         assertThat(queryEstablishedMemoryUsage(jobId, 40, latestModelSizeStats), equalTo(0L));
diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java
index 84677093ae58..8f6b82c9d674 100644
--- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java
+++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java
@@ -468,7 +468,7 @@ public void testGetDataCountsModelSizeAndTimingStatsWithSomeDocs() throws Except
 
         ModelSizeStats storedModelSizeStats = new ModelSizeStats.Builder(job.getId()).setModelBytes(10L).build();
         jobResultsPersister.persistModelSizeStats(storedModelSizeStats, () -> false);
-        jobResultsPersister.commitResultWrites(job.getId());
+        jobResultsPersister.commitWrites(job.getId(), JobResultsPersister.CommitType.RESULTS);
 
         setOrThrow.get();
         assertThat(dataCountsAtomicReference.get().getJobId(), equalTo(job.getId()));
@@ -479,7 +479,7 @@ public void testGetDataCountsModelSizeAndTimingStatsWithSomeDocs() throws Except
         storedTimingStats.updateStats(10);
 
         jobResultsPersister.bulkPersisterBuilder(job.getId()).persistTimingStats(storedTimingStats).executeRequest();
-        jobResultsPersister.commitResultWrites(job.getId());
+        jobResultsPersister.commitWrites(job.getId(), JobResultsPersister.CommitType.RESULTS);
 
         setOrThrow.get();
 
@@ -491,8 +491,8 @@ public void testGetDataCountsModelSizeAndTimingStatsWithSomeDocs() throws Except
         storedDataCounts.incrementInputBytes(1L);
         storedDataCounts.incrementMissingFieldCount(1L);
         JobDataCountsPersister jobDataCountsPersister = new JobDataCountsPersister(client(), resultsPersisterService, auditor);
-        jobDataCountsPersister.persistDataCounts(job.getId(), storedDataCounts);
-        jobResultsPersister.commitResultWrites(job.getId());
+        jobDataCountsPersister.persistDataCounts(job.getId(), storedDataCounts, true);
+        jobResultsPersister.commitWrites(job.getId(), JobResultsPersister.CommitType.RESULTS);
 
         setOrThrow.get();
         assertThat(dataCountsAtomicReference.get(), equalTo(storedDataCounts));
@@ -1046,9 +1046,9 @@ private void indexScheduledEvents(List<ScheduledEvent> events) throws IOExceptio
         }
     }
 
-    private void indexDataCounts(DataCounts counts, String jobId) {
+    private void indexDataCounts(DataCounts counts, String jobId) throws InterruptedException {
         JobDataCountsPersister persister = new JobDataCountsPersister(client(), resultsPersisterService, auditor);
-        persister.persistDataCounts(jobId, counts);
+        persister.persistDataCounts(jobId, counts, true);
     }
 
     private void indexFilters(List<MlFilter> filters) throws IOException {
diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/ModelInferenceActionIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/ModelInferenceActionIT.java
index d95c1c5f1725..bff4caaeb0dc 100644
--- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/ModelInferenceActionIT.java
+++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/ModelInferenceActionIT.java
@@ -171,14 +171,14 @@ public void testInferModels() throws Exception {
         });
 
         // Test regression
-        InferModelAction.Request request = new InferModelAction.Request(modelId1, toInfer, RegressionConfigUpdate.EMPTY_PARAMS, true);
+        InferModelAction.Request request = InferModelAction.Request.forDocs(modelId1, toInfer, RegressionConfigUpdate.EMPTY_PARAMS, true);
         InferModelAction.Response response = client().execute(InferModelAction.INSTANCE, request).actionGet();
         assertThat(
             response.getInferenceResults().stream().map(i -> ((SingleValueInferenceResults) i).value()).collect(Collectors.toList()),
             contains(1.3, 1.25)
         );
 
-        request = new InferModelAction.Request(modelId1, toInfer2, RegressionConfigUpdate.EMPTY_PARAMS, true);
+        request = InferModelAction.Request.forDocs(modelId1, toInfer2, RegressionConfigUpdate.EMPTY_PARAMS, true);
         response = client().execute(InferModelAction.INSTANCE, request).actionGet();
         assertThat(
             response.getInferenceResults().stream().map(i -> ((SingleValueInferenceResults) i).value()).collect(Collectors.toList()),
@@ -186,7 +186,7 @@ public void testInferModels() throws Exception {
         );
 
         // Test classification
-        request = new InferModelAction.Request(modelId2, toInfer, ClassificationConfigUpdate.EMPTY_PARAMS, true);
+        request = InferModelAction.Request.forDocs(modelId2, toInfer, ClassificationConfigUpdate.EMPTY_PARAMS, true);
         response = client().execute(InferModelAction.INSTANCE, request).actionGet();
         assertThat(
             response.getInferenceResults()
@@ -197,7 +197,7 @@ public void testInferModels() throws Exception {
         );
 
         // Get top classes
-        request = new InferModelAction.Request(modelId2, toInfer, new ClassificationConfigUpdate(2, null, null, null, null), true);
+        request = InferModelAction.Request.forDocs(modelId2, toInfer, new ClassificationConfigUpdate(2, null, null, null, null), true);
         response = client().execute(InferModelAction.INSTANCE, request).actionGet();
 
         ClassificationInferenceResults classificationInferenceResults = (ClassificationInferenceResults) response.getInferenceResults()
@@ -220,7 +220,7 @@ public void testInferModels() throws Exception {
         );
 
         // Test that top classes restrict the number returned
-        request = new InferModelAction.Request(modelId2, toInfer2, new ClassificationConfigUpdate(1, null, null, null, null), true);
+        request = InferModelAction.Request.forDocs(modelId2, toInfer2, new ClassificationConfigUpdate(1, null, null, null, null), true);
         response = client().execute(InferModelAction.INSTANCE, request).actionGet();
 
         classificationInferenceResults = (ClassificationInferenceResults) response.getInferenceResults().get(0);
@@ -319,7 +319,12 @@ public void testInferModelMultiClassModel() throws Exception {
         });
 
         // Test regression
-        InferModelAction.Request request = new InferModelAction.Request(modelId, toInfer, ClassificationConfigUpdate.EMPTY_PARAMS, true);
+        InferModelAction.Request request = InferModelAction.Request.forDocs(
+            modelId,
+            toInfer,
+            ClassificationConfigUpdate.EMPTY_PARAMS,
+            true
+        );
         InferModelAction.Response response = client().execute(InferModelAction.INSTANCE, request).actionGet();
         assertThat(
             response.getInferenceResults()
@@ -329,7 +334,7 @@ public void testInferModelMultiClassModel() throws Exception {
             contains("option_0", "option_2")
         );
 
-        request = new InferModelAction.Request(modelId, toInfer2, ClassificationConfigUpdate.EMPTY_PARAMS, true);
+        request = InferModelAction.Request.forDocs(modelId, toInfer2, ClassificationConfigUpdate.EMPTY_PARAMS, true);
         response = client().execute(InferModelAction.INSTANCE, request).actionGet();
         assertThat(
             response.getInferenceResults()
@@ -340,7 +345,7 @@ public void testInferModelMultiClassModel() throws Exception {
         );
 
         // Get top classes
-        request = new InferModelAction.Request(modelId, toInfer, new ClassificationConfigUpdate(3, null, null, null, null), true);
+        request = InferModelAction.Request.forDocs(modelId, toInfer, new ClassificationConfigUpdate(3, null, null, null, null), true);
         response = client().execute(InferModelAction.INSTANCE, request).actionGet();
 
         ClassificationInferenceResults classificationInferenceResults = (ClassificationInferenceResults) response.getInferenceResults()
@@ -358,7 +363,7 @@ public void testInferModelMultiClassModel() throws Exception {
 
     public void testInferMissingModel() {
         String model = "test-infer-missing-model";
-        InferModelAction.Request request = new InferModelAction.Request(
+        InferModelAction.Request request = InferModelAction.Request.forDocs(
             model,
             Collections.emptyList(),
             RegressionConfigUpdate.EMPTY_PARAMS,
@@ -404,7 +409,7 @@ public void testInferMissingFields() throws Exception {
             }
         });
 
-        InferModelAction.Request request = new InferModelAction.Request(
+        InferModelAction.Request request = InferModelAction.Request.forDocs(
             modelId,
             toInferMissingField,
             RegressionConfigUpdate.EMPTY_PARAMS,
diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TestFeatureLicenseTrackingIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TestFeatureLicenseTrackingIT.java
index 1d2526ed01e4..4ad4dfdbaccf 100644
--- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TestFeatureLicenseTrackingIT.java
+++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TestFeatureLicenseTrackingIT.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.ingest.PutPipelineAction;
 import org.elasticsearch.action.ingest.PutPipelineRequest;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.license.GetFeatureUsageRequest;
 import org.elasticsearch.license.GetFeatureUsageResponse;
 import org.elasticsearch.license.TransportGetFeatureUsageAction;
@@ -209,7 +210,7 @@ private List<GetJobsStatsAction.Response.JobStats> getJobStats(String jobId) {
     }
 
     private void putTrainedModelIngestPipeline(String pipelineId, String modelId) throws Exception {
-        client().execute(PutPipelineAction.INSTANCE, new PutPipelineRequest(pipelineId, new BytesArray(formatted("""
+        client().execute(PutPipelineAction.INSTANCE, new PutPipelineRequest(pipelineId, new BytesArray(Strings.format("""
             {
                 "processors": [
                   {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
index 60c73ee8352f..c55b6e922f1c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
@@ -29,7 +29,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.collect.MapBuilder;
@@ -160,7 +160,6 @@
 import org.elasticsearch.xpack.core.ml.action.PutTrainedModelVocabularyAction;
 import org.elasticsearch.xpack.core.ml.action.ResetJobAction;
 import org.elasticsearch.xpack.core.ml.action.RevertModelSnapshotAction;
-import org.elasticsearch.xpack.core.ml.action.SemanticSearchAction;
 import org.elasticsearch.xpack.core.ml.action.SetResetModeAction;
 import org.elasticsearch.xpack.core.ml.action.SetUpgradeModeAction;
 import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction;
@@ -261,7 +260,6 @@
 import org.elasticsearch.xpack.ml.action.TransportPutTrainedModelVocabularyAction;
 import org.elasticsearch.xpack.ml.action.TransportResetJobAction;
 import org.elasticsearch.xpack.ml.action.TransportRevertModelSnapshotAction;
-import org.elasticsearch.xpack.ml.action.TransportSemanticSearchAction;
 import org.elasticsearch.xpack.ml.action.TransportSetResetModeAction;
 import org.elasticsearch.xpack.ml.action.TransportSetUpgradeModeAction;
 import org.elasticsearch.xpack.ml.action.TransportStartDataFrameAnalyticsAction;
@@ -414,7 +412,6 @@
 import org.elasticsearch.xpack.ml.rest.inference.RestPutTrainedModelAliasAction;
 import org.elasticsearch.xpack.ml.rest.inference.RestPutTrainedModelDefinitionPartAction;
 import org.elasticsearch.xpack.ml.rest.inference.RestPutTrainedModelVocabularyAction;
-import org.elasticsearch.xpack.ml.rest.inference.RestSemanticSearchAction;
 import org.elasticsearch.xpack.ml.rest.inference.RestStartTrainedModelDeploymentAction;
 import org.elasticsearch.xpack.ml.rest.inference.RestStopTrainedModelDeploymentAction;
 import org.elasticsearch.xpack.ml.rest.inference.RestUpdateTrainedModelDeploymentAction;
@@ -853,7 +850,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         if (enabled == false) {
             // special holder for @link(MachineLearningFeatureSetUsage) which needs access to job manager, empty if ML is disabled
@@ -1325,7 +1322,6 @@ public List<RestHandler> getRestHandlers(
             new RestPutTrainedModelVocabularyAction(),
             new RestInferTrainedModelAction(),
             new RestClearDeploymentCacheAction(),
-            new RestSemanticSearchAction(),
             // CAT Handlers
             new RestCatJobsAction(),
             new RestCatTrainedModelsAction(),
@@ -1432,7 +1428,6 @@ public List<RestHandler> getRestHandlers(
                 TransportUpdateTrainedModelAssignmentStateAction.class
             ),
             new ActionHandler<>(ClearDeploymentCacheAction.INSTANCE, TransportClearDeploymentCacheAction.class),
-            new ActionHandler<>(SemanticSearchAction.INSTANCE, TransportSemanticSearchAction.class),
             usageAction,
             infoAction
         );
@@ -1573,7 +1568,7 @@ public List<AggregationSpec> getAggregations() {
             ).addResultReader(InternalCategorizationAggregation::new)
                 .setAggregatorRegistrar(s -> s.registerUsage(CategorizeTextAggregationBuilder.NAME)),
             new AggregationSpec(
-                FrequentItemSetsAggregationBuilder.NAME,
+                new ParseField(FrequentItemSetsAggregationBuilder.NAME, FrequentItemSetsAggregationBuilder.DEPRECATED_NAME),
                 FrequentItemSetsAggregationBuilder::new,
                 checkAggLicense(FrequentItemSetsAggregationBuilder.PARSER, FREQUENT_ITEM_SETS_AGG_FEATURE)
             ).addResultReader(FrequentItemSetsAggregatorFactory.getResultReader())
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java
index c99af373e525..5129d0d45fde 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java
@@ -20,6 +20,8 @@
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -66,6 +68,8 @@
 
 public class MachineLearningUsageTransportAction extends XPackUsageFeatureTransportAction {
 
+    private static final Logger logger = LogManager.getLogger(MachineLearningUsageTransportAction.class);
+
     private final Client client;
     private final XPackLicenseState licenseState;
     private final JobManagerHolder jobManagerHolder;
@@ -124,8 +128,8 @@ protected void masterOperation(
         int nodeCount = mlNodeCount(state);
 
         // Step 5. return final ML usage
-        ActionListener<Map<String, Object>> inferenceUsageListener = ActionListener.wrap(inferenceUsage -> {
-            listener.onResponse(
+        ActionListener<Map<String, Object>> inferenceUsageListener = ActionListener.wrap(
+            inferenceUsage -> listener.onResponse(
                 new XPackUsageFeatureResponse(
                     new MachineLearningFeatureSetUsage(
                         MachineLearningField.ML_API_FEATURE.checkWithoutTracking(licenseState),
@@ -137,45 +141,76 @@ protected void masterOperation(
                         nodeCount
                     )
                 )
-            );
-        }, listener::onFailure);
+            ),
+            e -> {
+                logger.warn("Failed to get inference usage to include in ML usage", e);
+                listener.onResponse(
+                    new XPackUsageFeatureResponse(
+                        new MachineLearningFeatureSetUsage(
+                            MachineLearningField.ML_API_FEATURE.checkWithoutTracking(licenseState),
+                            enabled,
+                            jobsUsage,
+                            datafeedsUsage,
+                            analyticsUsage,
+                            Collections.emptyMap(),
+                            nodeCount
+                        )
+                    )
+                );
+            }
+        );
 
         // Step 4. Extract usage from data frame analytics configs and then get inference usage
         ActionListener<GetDataFrameAnalyticsAction.Response> dataframeAnalyticsListener = ActionListener.wrap(response -> {
             addDataFrameAnalyticsUsage(response, analyticsUsage);
             addInferenceUsage(inferenceUsageListener);
-        }, listener::onFailure);
+        }, e -> {
+            logger.warn("Failed to get data frame analytics configs to include in ML usage", e);
+            addInferenceUsage(inferenceUsageListener);
+        });
 
         // Step 3. Extract usage from data frame analytics stats and then request data frame analytics configs
+        GetDataFrameAnalyticsAction.Request getDfaRequest = new GetDataFrameAnalyticsAction.Request(Metadata.ALL);
+        getDfaRequest.setPageParams(new PageParams(0, 10_000));
         ActionListener<GetDataFrameAnalyticsStatsAction.Response> dataframeAnalyticsStatsListener = ActionListener.wrap(response -> {
             addDataFrameAnalyticsStatsUsage(response, analyticsUsage);
-            GetDataFrameAnalyticsAction.Request getDfaRequest = new GetDataFrameAnalyticsAction.Request(Metadata.ALL);
-            getDfaRequest.setPageParams(new PageParams(0, 10_000));
             client.execute(GetDataFrameAnalyticsAction.INSTANCE, getDfaRequest, dataframeAnalyticsListener);
-        }, listener::onFailure);
+        }, e -> {
+            logger.warn("Failed to get data frame analytics stats to include in ML usage", e);
+            client.execute(GetDataFrameAnalyticsAction.INSTANCE, getDfaRequest, dataframeAnalyticsListener);
+        });
 
         // Step 2. Extract usage from datafeeds stats and then request stats for data frame analytics
+        GetDataFrameAnalyticsStatsAction.Request dataframeAnalyticsStatsRequest = new GetDataFrameAnalyticsStatsAction.Request(
+            Metadata.ALL
+        );
+        dataframeAnalyticsStatsRequest.setPageParams(new PageParams(0, 10_000));
         ActionListener<GetDatafeedsStatsAction.Response> datafeedStatsListener = ActionListener.wrap(response -> {
             addDatafeedsUsage(response, datafeedsUsage);
-            GetDataFrameAnalyticsStatsAction.Request dataframeAnalyticsStatsRequest = new GetDataFrameAnalyticsStatsAction.Request(
-                Metadata.ALL
-            );
-            dataframeAnalyticsStatsRequest.setPageParams(new PageParams(0, 10_000));
             client.execute(GetDataFrameAnalyticsStatsAction.INSTANCE, dataframeAnalyticsStatsRequest, dataframeAnalyticsStatsListener);
-        }, listener::onFailure);
+        }, e -> {
+            logger.warn("Failed to get datafeed stats to include in ML usage", e);
+            client.execute(GetDataFrameAnalyticsStatsAction.INSTANCE, dataframeAnalyticsStatsRequest, dataframeAnalyticsStatsListener);
+        });
 
         // Step 1. Extract usage from jobs stats and then request stats for all datafeeds
-        GetJobsStatsAction.Request jobStatsRequest = new GetJobsStatsAction.Request(Metadata.ALL);
+        GetDatafeedsStatsAction.Request datafeedStatsRequest = new GetDatafeedsStatsAction.Request(Metadata.ALL);
         ActionListener<GetJobsStatsAction.Response> jobStatsListener = ActionListener.wrap(
             response -> jobManagerHolder.getJobManager().expandJobs(Metadata.ALL, true, ActionListener.wrap(jobs -> {
                 addJobsUsage(response, jobs.results(), jobsUsage);
-                GetDatafeedsStatsAction.Request datafeedStatsRequest = new GetDatafeedsStatsAction.Request(Metadata.ALL);
                 client.execute(GetDatafeedsStatsAction.INSTANCE, datafeedStatsRequest, datafeedStatsListener);
-            }, listener::onFailure)),
-            listener::onFailure
+            }, e -> {
+                logger.warn("Failed to get job configs to include in ML usage", e);
+                client.execute(GetDatafeedsStatsAction.INSTANCE, datafeedStatsRequest, datafeedStatsListener);
+            })),
+            e -> {
+                logger.warn("Failed to get job stats to include in ML usage", e);
+                client.execute(GetDatafeedsStatsAction.INSTANCE, datafeedStatsRequest, datafeedStatsListener);
+            }
         );
 
         // Step 0. Kick off the chain of callbacks by requesting jobs stats
+        GetJobsStatsAction.Request jobStatsRequest = new GetJobsStatsAction.Request(Metadata.ALL);
         client.execute(GetJobsStatsAction.INSTANCE, jobStatsRequest, jobStatsListener);
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportClearDeploymentCacheAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportClearDeploymentCacheAction.java
index 735a413b9d3b..79df84d4ab9b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportClearDeploymentCacheAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportClearDeploymentCacheAction.java
@@ -23,14 +23,13 @@
 import org.elasticsearch.xpack.core.ml.action.ClearDeploymentCacheAction.Request;
 import org.elasticsearch.xpack.core.ml.action.ClearDeploymentCacheAction.Response;
 import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignment;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.inference.assignment.TrainedModelAssignmentMetadata;
 import org.elasticsearch.xpack.ml.inference.deployment.TrainedModelDeploymentTask;
 
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.ExceptionsHelper.convertToElastic;
-
 public class TransportClearDeploymentCacheAction extends TransportTasksAction<TrainedModelDeploymentTask, Request, Response, Response> {
 
     @Inject
@@ -59,9 +58,9 @@ protected Response newResponse(
         List<FailedNodeException> failedNodeExceptions
     ) {
         if (taskOperationFailures.isEmpty() == false) {
-            throw convertToElastic(taskOperationFailures.get(0).getCause());
+            throw ExceptionsHelper.taskOperationFailureToStatusException(taskOperationFailures.get(0));
         } else if (failedNodeExceptions.isEmpty() == false) {
-            throw convertToElastic(failedNodeExceptions.get(0));
+            throw failedNodeExceptions.get(0);
         }
         return new Response(true);
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java
index 0400418039bf..4ad78fd5d5c6 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportCloseJobAction.java
@@ -342,6 +342,7 @@ private void stopDatafeeds(List<String> runningDatafeedIds, boolean isForce, Tim
     void isolateDatafeeds(List<String> openJobs, List<String> runningDatafeedIds, ActionListener<Void> listener) {
 
         GroupedActionListener<IsolateDatafeedAction.Response> groupedListener = new GroupedActionListener<>(
+            runningDatafeedIds.size(),
             ActionListener.wrap(c -> listener.onResponse(null), e -> {
                 // This is deliberately NOT an error. The reasoning is as follows:
                 // - Isolate datafeed just sets a flag on the datafeed, so cannot fail IF it reaches the running datafeed code
@@ -363,8 +364,7 @@ void isolateDatafeeds(List<String> openJobs, List<String> runningDatafeedIds, Ac
                 // race condition easier.
                 logger.info("could not isolate all datafeeds while force closing jobs " + openJobs, e);
                 listener.onResponse(null);
-            }),
-            runningDatafeedIds.size()
+            })
         );
 
         for (String runningDatafeedId : runningDatafeedIds) {
@@ -479,9 +479,9 @@ protected CloseJobAction.Response newResponse(
         // otherwise something went wrong
         if (request.getOpenJobIds().length != tasks.size()) {
             if (taskOperationFailures.isEmpty() == false) {
-                throw org.elasticsearch.ExceptionsHelper.convertToElastic(taskOperationFailures.get(0).getCause());
+                throw ExceptionsHelper.taskOperationFailureToStatusException(taskOperationFailures.get(0));
             } else if (failedNodeExceptions.isEmpty() == false) {
-                throw org.elasticsearch.ExceptionsHelper.convertToElastic(failedNodeExceptions.get(0));
+                throw failedNodeExceptions.get(0);
             } else {
                 // This can happen when the actual task in the node no longer exists,
                 // which means the job(s) have already been closed.
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java
index bdb01a2c70f0..96a106ba525b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteExpiredDataAction.java
@@ -209,7 +209,7 @@ void deleteExpiredData(
             // thread is a disaster.
             remover.remove(
                 requestsPerSecond,
-                new ThreadedActionListener<>(logger, threadPool, executor, nextListener, false),
+                new ThreadedActionListener<>(threadPool.executor(executor), nextListener),
                 isTimedOutSupplier
             );
         } else {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInferTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInferTrainedModelDeploymentAction.java
index 8f42e79c4c6c..de73fde109c3 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInferTrainedModelDeploymentAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInferTrainedModelDeploymentAction.java
@@ -7,38 +7,30 @@
 
 package org.elasticsearch.xpack.ml.action;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.GroupedActionListener;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
-import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction;
 import org.elasticsearch.xpack.core.ml.action.InferTrainedModelDeploymentAction;
-import org.elasticsearch.xpack.core.ml.inference.TrainedModelType;
-import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentState;
-import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignment;
+import org.elasticsearch.xpack.core.ml.inference.results.InferenceResults;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
-import org.elasticsearch.xpack.ml.inference.ModelAliasMetadata;
-import org.elasticsearch.xpack.ml.inference.assignment.TrainedModelAssignmentMetadata;
 import org.elasticsearch.xpack.ml.inference.deployment.NlpInferenceInput;
 import org.elasticsearch.xpack.ml.inference.deployment.TrainedModelDeploymentTask;
 import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider;
 
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
-import java.util.Optional;
-
-import static org.elasticsearch.core.Strings.format;
 
 public class TransportInferTrainedModelDeploymentAction extends TransportTasksAction<
     TrainedModelDeploymentTask,
@@ -46,10 +38,6 @@ public class TransportInferTrainedModelDeploymentAction extends TransportTasksAc
     InferTrainedModelDeploymentAction.Response,
     InferTrainedModelDeploymentAction.Response> {
 
-    private static final Logger logger = LogManager.getLogger(TransportInferTrainedModelDeploymentAction.class);
-
-    private final TrainedModelProvider provider;
-
     @Inject
     public TransportInferTrainedModelDeploymentAction(
         ClusterService clusterService,
@@ -67,63 +55,6 @@ public TransportInferTrainedModelDeploymentAction(
             InferTrainedModelDeploymentAction.Response::new,
             ThreadPool.Names.SAME
         );
-        this.provider = provider;
-    }
-
-    @Override
-    protected void doExecute(
-        Task task,
-        InferTrainedModelDeploymentAction.Request request,
-        ActionListener<InferTrainedModelDeploymentAction.Response> listener
-    ) {
-        TaskId taskId = new TaskId(clusterService.localNode().getId(), task.getId());
-        // Update the requests model ID if it's an alias
-        Optional.ofNullable(ModelAliasMetadata.fromState(clusterService.state()).getModelId(request.getModelId()))
-            .ifPresent(request::setModelId);
-        // We need to check whether there is at least an assigned task here, otherwise we cannot redirect to the
-        // node running the job task.
-        TrainedModelAssignment assignment = TrainedModelAssignmentMetadata.assignmentForModelId(
-            clusterService.state(),
-            request.getModelId()
-        ).orElse(null);
-        if (assignment == null) {
-            // If there is no assignment, verify the model even exists so that we can provide a nicer error message
-            provider.getTrainedModel(request.getModelId(), GetTrainedModelsAction.Includes.empty(), taskId, ActionListener.wrap(config -> {
-                if (config.getModelType() != TrainedModelType.PYTORCH) {
-                    listener.onFailure(
-                        ExceptionsHelper.badRequestException(
-                            "Only [pytorch] models are supported by _infer, provided model [{}] has type [{}]",
-                            config.getModelId(),
-                            config.getModelType()
-                        )
-                    );
-                    return;
-                }
-                String message = "Trained model [" + request.getModelId() + "] is not deployed";
-                listener.onFailure(ExceptionsHelper.conflictStatusException(message));
-            }, listener::onFailure));
-            return;
-        }
-        if (assignment.getAssignmentState() == AssignmentState.STOPPING) {
-            String message = "Trained model [" + request.getModelId() + "] is STOPPING";
-            listener.onFailure(ExceptionsHelper.conflictStatusException(message));
-            return;
-        }
-        logger.trace(() -> format("[%s] selecting node from routing table: %s", assignment.getModelId(), assignment.getNodeRoutingTable()));
-        assignment.selectRandomStartedNodeWeighedOnAllocations().ifPresentOrElse(node -> {
-            logger.trace(() -> format("[%s] selected node [%s]", assignment.getModelId(), node));
-            request.setNodes(node);
-            long start = System.currentTimeMillis();
-            super.doExecute(task, request, ActionListener.wrap(r -> {
-                r.setTookMillis(System.currentTimeMillis() - start);
-                listener.onResponse(r);
-            }, listener::onFailure));
-        }, () -> {
-            logger.trace(() -> format("[%s] model not allocated to any node [%s]", assignment.getModelId()));
-            listener.onFailure(
-                ExceptionsHelper.conflictStatusException("Trained model [" + request.getModelId() + "] is not allocated to any nodes")
-            );
-        });
     }
 
     @Override
@@ -134,9 +65,9 @@ protected InferTrainedModelDeploymentAction.Response newResponse(
         List<FailedNodeException> failedNodeExceptions
     ) {
         if (taskOperationFailures.isEmpty() == false) {
-            throw org.elasticsearch.ExceptionsHelper.convertToElastic(taskOperationFailures.get(0).getCause());
+            throw ExceptionsHelper.taskOperationFailureToStatusException(taskOperationFailures.get(0));
         } else if (failedNodeExceptions.isEmpty() == false) {
-            throw org.elasticsearch.ExceptionsHelper.convertToElastic(failedNodeExceptions.get(0));
+            throw failedNodeExceptions.get(0);
         } else if (tasks.isEmpty()) {
             throw new ElasticsearchStatusException(
                 "Unable to find deployment task for model [{}] please stop and start the deployment or try again momentarily",
@@ -144,6 +75,7 @@ protected InferTrainedModelDeploymentAction.Response newResponse(
                 request.getModelId()
             );
         } else {
+            assert tasks.size() == 1;
             return tasks.get(0);
         }
     }
@@ -157,23 +89,26 @@ protected void taskOperation(
     ) {
         assert actionTask instanceof CancellableTask : "task [" + actionTask + "] not cancellable";
 
-        NlpInferenceInput input;
+        var nlpInputs = new ArrayList<NlpInferenceInput>();
         if (request.getTextInput() != null) {
-            input = NlpInferenceInput.fromText(request.getTextInput());
+            for (var text : request.getTextInput()) {
+                nlpInputs.add(NlpInferenceInput.fromText(text));
+            }
         } else {
-            input = NlpInferenceInput.fromDoc(request.getDocs().get(0));
+            for (var doc : request.getDocs()) {
+                nlpInputs.add(NlpInferenceInput.fromDoc(doc));
+            }
         }
 
-        task.infer(
-            input,
-            request.getUpdate(),
-            request.isSkipQueue(),
-            request.getInferenceTimeout(),
-            actionTask,
-            ActionListener.wrap(
-                pyTorchResult -> listener.onResponse(new InferTrainedModelDeploymentAction.Response(pyTorchResult, 0)),
-                listener::onFailure
-            )
+        // Multiple documents to infer on, wait for all results
+        ActionListener<Collection<InferenceResults>> collectingListener = ActionListener.wrap(
+            pyTorchResults -> { listener.onResponse(new InferTrainedModelDeploymentAction.Response(new ArrayList<>(pyTorchResults))); },
+            listener::onFailure
         );
+
+        GroupedActionListener<InferenceResults> groupedListener = new GroupedActionListener<>(nlpInputs.size(), collectingListener);
+        for (var input : nlpInputs) {
+            task.infer(input, request.getUpdate(), request.isSkipQueue(), request.getInferenceTimeout(), actionTask, groupedListener);
+        }
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java
index 31cfa27ad626..31af0853f4e3 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java
@@ -6,13 +6,15 @@
  */
 package org.elasticsearch.xpack.ml.action;
 
+import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.license.License;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
@@ -28,8 +30,11 @@
 import org.elasticsearch.xpack.core.ml.action.InferModelAction.Request;
 import org.elasticsearch.xpack.core.ml.action.InferModelAction.Response;
 import org.elasticsearch.xpack.core.ml.action.InferTrainedModelDeploymentAction;
+import org.elasticsearch.xpack.core.ml.inference.TrainedModelType;
+import org.elasticsearch.xpack.core.ml.inference.assignment.AssignmentState;
+import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignment;
 import org.elasticsearch.xpack.core.ml.inference.results.InferenceResults;
-import org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfigUpdate;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.ml.inference.ModelAliasMetadata;
 import org.elasticsearch.xpack.ml.inference.assignment.TrainedModelAssignmentMetadata;
@@ -38,9 +43,10 @@
 import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider;
 import org.elasticsearch.xpack.ml.utils.TypedChainTaskExecutor;
 
-import java.util.Collections;
-import java.util.Map;
+import java.util.List;
 import java.util.Optional;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
@@ -132,19 +138,19 @@ private void doInfer(
     ) {
         String concreteModelId = Optional.ofNullable(ModelAliasMetadata.fromState(clusterService.state()).getModelId(request.getModelId()))
             .orElse(request.getModelId());
-        if (isAllocatedModel(concreteModelId)) {
+
+        responseBuilder.setModelId(concreteModelId);
+
+        TrainedModelAssignmentMetadata trainedModelAssignmentMetadata = TrainedModelAssignmentMetadata.fromState(clusterService.state());
+
+        if (trainedModelAssignmentMetadata.isAssigned(concreteModelId)) {
             // It is important to use the resolved model ID here as the alias could change between transport calls.
-            inferAgainstAllocatedModel(request, concreteModelId, responseBuilder, parentTaskId, listener);
+            inferAgainstAllocatedModel(trainedModelAssignmentMetadata, request, concreteModelId, responseBuilder, parentTaskId, listener);
         } else {
             getModelAndInfer(request, responseBuilder, parentTaskId, (CancellableTask) task, listener);
         }
     }
 
-    private boolean isAllocatedModel(String modelId) {
-        TrainedModelAssignmentMetadata trainedModelAssignmentMetadata = TrainedModelAssignmentMetadata.fromState(clusterService.state());
-        return trainedModelAssignmentMetadata.isAssigned(modelId);
-    }
-
     private void getModelAndInfer(
         Request request,
         Response.Builder responseBuilder,
@@ -169,75 +175,153 @@ private void getModelAndInfer(
 
             typedChainTaskExecutor.execute(ActionListener.wrap(inferenceResultsInterfaces -> {
                 model.release();
-                listener.onResponse(responseBuilder.setInferenceResults(inferenceResultsInterfaces).setModelId(model.getModelId()).build());
+                listener.onResponse(responseBuilder.addInferenceResults(inferenceResultsInterfaces).build());
             }, e -> {
                 model.release();
                 listener.onFailure(e);
             }));
-        }, listener::onFailure);
+        }, e -> {
+            if (ExceptionsHelper.unwrapCause(e) instanceof ResourceNotFoundException) {
+                listener.onFailure(e);
+                return;
+            }
 
+            // The model was found, check if a more relevant error message can be returned
+            trainedModelProvider.getTrainedModel(
+                request.getModelId(),
+                GetTrainedModelsAction.Includes.empty(),
+                parentTaskId,
+                ActionListener.wrap(trainedModelConfig -> {
+                    if (trainedModelConfig.getModelType() == TrainedModelType.PYTORCH) {
+                        // The PyTorch model cannot be allocated if we got here
+                        listener.onFailure(
+                            ExceptionsHelper.conflictStatusException(
+                                "Model ["
+                                    + request.getModelId()
+                                    + "] must be deployed to use. Please deploy with the start trained model deployment API.",
+                                request.getModelId()
+                            )
+                        );
+                    } else {
+                        // return the original error
+                        listener.onFailure(e);
+                    }
+                }, listener::onFailure)
+            );
+        });
+
+        // TODO should `getModelForInternalInference` be used here??
         modelLoadingService.getModelForPipeline(request.getModelId(), parentTaskId, getModelListener);
     }
 
     private void inferAgainstAllocatedModel(
+        TrainedModelAssignmentMetadata assignmentMeta,
         Request request,
         String concreteModelId,
         Response.Builder responseBuilder,
         TaskId parentTaskId,
         ActionListener<Response> listener
     ) {
-        TypedChainTaskExecutor<InferenceResults> typedChainTaskExecutor = new TypedChainTaskExecutor<>(
-            client.threadPool().executor(ThreadPool.Names.SAME),
-            // run through all tasks
-            r -> true,
-            // Always fail immediately and return an error
-            ex -> true
-        );
-        request.getObjectsToInfer()
-            .forEach(
-                stringObjectMap -> typedChainTaskExecutor.add(
-                    chainedTask -> inferSingleDocAgainstAllocatedModel(
-                        concreteModelId,
-                        request.getTimeout(),
-                        request.getUpdate(),
-                        stringObjectMap,
-                        parentTaskId,
-                        chainedTask
-                    )
-                )
+        TrainedModelAssignment assignment = assignmentMeta.getModelAssignment(concreteModelId);
+
+        if (assignment.getAssignmentState() == AssignmentState.STOPPING) {
+            String message = "Trained model [" + request.getModelId() + "] is STOPPING";
+            listener.onFailure(ExceptionsHelper.conflictStatusException(message));
+            return;
+        }
+
+        // Get a list of nodes to send the requests to and the number of
+        // documents for each node.
+        var nodes = assignment.selectRandomStartedNodesWeighedOnAllocationsForNRequests(request.numberOfDocuments());
+        if (nodes.isEmpty()) {
+            logger.trace(() -> format("[%s] model not allocated to any node [%s]", assignment.getModelId()));
+            listener.onFailure(
+                ExceptionsHelper.conflictStatusException("Trained model [" + request.getModelId() + "] is not allocated to any nodes")
             );
+            return;
+        }
 
-        typedChainTaskExecutor.execute(
-            ActionListener.wrap(
-                inferenceResults -> listener.onResponse(
-                    responseBuilder.setInferenceResults(inferenceResults).setModelId(concreteModelId).build()
-                ),
-                listener::onFailure
-            )
-        );
+        assert nodes.stream().mapToInt(Tuple::v2).sum() == request.numberOfDocuments()
+            : "mismatch; sum of node requests does not match number of documents in request";
+
+        AtomicInteger count = new AtomicInteger();
+        AtomicArray<List<InferenceResults>> results = new AtomicArray<>(nodes.size());
+        AtomicReference<Exception> failure = new AtomicReference<>();
+
+        int startPos = 0;
+        int slot = 0;
+        for (var node : nodes) {
+            InferTrainedModelDeploymentAction.Request deploymentRequest;
+            if (request.getTextInput() == null) {
+                deploymentRequest = InferTrainedModelDeploymentAction.Request.forDocs(
+                    concreteModelId,
+                    request.getUpdate(),
+                    request.getObjectsToInfer().subList(startPos, startPos + node.v2()),
+                    request.getInferenceTimeout()
+                );
+            } else {
+                deploymentRequest = InferTrainedModelDeploymentAction.Request.forTextInput(
+                    concreteModelId,
+                    request.getUpdate(),
+                    request.getTextInput().subList(startPos, startPos + node.v2()),
+                    request.getInferenceTimeout()
+                );
+            }
+            deploymentRequest.setNodes(node.v1());
+            deploymentRequest.setParentTask(parentTaskId);
+
+            startPos += node.v2();
+
+            executeAsyncWithOrigin(
+                client,
+                ML_ORIGIN,
+                InferTrainedModelDeploymentAction.INSTANCE,
+                deploymentRequest,
+                collectingListener(count, results, failure, slot, nodes.size(), responseBuilder, listener)
+            );
+
+            slot++;
+        }
     }
 
-    private void inferSingleDocAgainstAllocatedModel(
-        String modelId,
-        TimeValue timeValue,
-        InferenceConfigUpdate inferenceConfigUpdate,
-        Map<String, Object> doc,
-        TaskId parentTaskId,
-        ActionListener<InferenceResults> listener
+    private ActionListener<InferTrainedModelDeploymentAction.Response> collectingListener(
+        AtomicInteger count,
+        AtomicArray<List<InferenceResults>> results,
+        AtomicReference<Exception> failure,
+        int slot,
+        int totalNumberOfResponses,
+        Response.Builder responseBuilder,
+        ActionListener<Response> finalListener
     ) {
-        InferTrainedModelDeploymentAction.Request request = new InferTrainedModelDeploymentAction.Request(
-            modelId,
-            inferenceConfigUpdate,
-            Collections.singletonList(doc),
-            timeValue
-        );
-        request.setParentTask(parentTaskId);
-        executeAsyncWithOrigin(
-            client,
-            ML_ORIGIN,
-            InferTrainedModelDeploymentAction.INSTANCE,
-            request,
-            ActionListener.wrap(r -> listener.onResponse(r.getResults()), listener::onFailure)
-        );
+        return new ActionListener<>() {
+            @Override
+            public void onResponse(InferTrainedModelDeploymentAction.Response response) {
+                results.setOnce(slot, response.getResults());
+                if (count.incrementAndGet() == totalNumberOfResponses) {
+                    sendResponse();
+                }
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                failure.set(e);
+                if (count.incrementAndGet() == totalNumberOfResponses) {
+                    sendResponse();
+                }
+            }
+
+            private void sendResponse() {
+                if (results.nonNullLength() > 0) {
+                    for (int i = 0; i < results.length(); i++) {
+                        if (results.get(i) != null) {
+                            responseBuilder.addInferenceResults(results.get(i));
+                        }
+                    }
+                    finalListener.onResponse(responseBuilder.build());
+                } else {
+                    finalListener.onFailure(failure.get());
+                }
+            }
+        };
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportIsolateDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportIsolateDatafeedAction.java
index 9679147b5672..ee6a767501d4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportIsolateDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportIsolateDatafeedAction.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.IsolateDatafeedAction;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.MachineLearning;
 
 import java.util.List;
@@ -71,9 +72,9 @@ protected IsolateDatafeedAction.Response newResponse(
         assert taskOperationFailures.size() <= 1 : "more than 1 item in taskOperationFailures: " + taskOperationFailures.size();
         assert failedNodeExceptions.size() <= 1 : "more than 1 item in failedNodeExceptions: " + failedNodeExceptions.size();
         if (taskOperationFailures.isEmpty() == false) {
-            throw org.elasticsearch.ExceptionsHelper.convertToElastic(taskOperationFailures.get(0).getCause());
+            throw ExceptionsHelper.taskOperationFailureToStatusException(taskOperationFailures.get(0));
         } else if (failedNodeExceptions.isEmpty() == false) {
-            throw org.elasticsearch.ExceptionsHelper.convertToElastic(failedNodeExceptions.get(0));
+            throw failedNodeExceptions.get(0);
         } else if (tasks.isEmpty() == false) {
             return tasks.get(0);
         }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java
index 91494d9e11b5..e337a3606926 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java
@@ -86,9 +86,9 @@ static <Response extends BaseTasksResponse> Response selectFirst(
         // the actionlistener's onFailure
         if (tasks.isEmpty()) {
             if (taskOperationFailures.isEmpty() == false) {
-                throw org.elasticsearch.ExceptionsHelper.convertToElastic(taskOperationFailures.get(0).getCause());
+                throw ExceptionsHelper.taskOperationFailureToStatusException(taskOperationFailures.get(0));
             } else if (failedNodeExceptions.isEmpty() == false) {
-                throw org.elasticsearch.ExceptionsHelper.convertToElastic(failedNodeExceptions.get(0));
+                throw failedNodeExceptions.get(0);
             } else {
                 throw new IllegalStateException("No errors or response");
             }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportKillProcessAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportKillProcessAction.java
index fbe0cb8597ad..6fb9a05ea09c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportKillProcessAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportKillProcessAction.java
@@ -70,9 +70,7 @@ protected KillProcessAction.Response newResponse(
         List<FailedNodeException> failedNodeExceptions
     ) {
         org.elasticsearch.ExceptionsHelper.rethrowAndSuppress(
-            taskOperationFailures.stream()
-                .map(t -> org.elasticsearch.ExceptionsHelper.convertToElastic(t.getCause()))
-                .collect(Collectors.toList())
+            taskOperationFailures.stream().map(ExceptionsHelper::taskOperationFailureToStatusException).collect(Collectors.toList())
         );
         org.elasticsearch.ExceptionsHelper.rethrowAndSuppress(failedNodeExceptions);
         return new KillProcessAction.Response(true);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
index 08d4deaf3e62..ea9a51e66f88 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.core.ml.MlConfigIndex;
 import org.elasticsearch.xpack.core.ml.MlTasks;
+import org.elasticsearch.xpack.core.ml.action.GetFiltersAction;
 import org.elasticsearch.xpack.core.ml.action.GetModelSnapshotsAction;
 import org.elasticsearch.xpack.core.ml.action.NodeAcknowledgedResponse;
 import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
@@ -50,7 +51,9 @@
 import org.elasticsearch.xpack.ml.process.MlMemoryTracker;
 
 import java.util.Optional;
+import java.util.Set;
 import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
@@ -169,7 +172,7 @@ public void onFailure(Exception e) {
             );
 
             // Tell the job tracker to refresh the memory requirement for this job and all other jobs that have persistent tasks
-            ActionListener<Boolean> modelSnapshotValidationListener = ActionListener.wrap(
+            ActionListener<Boolean> referencedRuleFiltersPresentListener = ActionListener.wrap(
                 response -> memoryTracker.refreshAnomalyDetectorJobMemoryAndAllOthers(
                     jobParams.getJobId(),
                     memoryRequirementRefreshListener
@@ -177,6 +180,23 @@ public void onFailure(Exception e) {
                 listener::onFailure
             );
 
+            // Validate referenced rule filters are present
+            ActionListener<Boolean> modelSnapshotValidationListener = ActionListener.wrap(response -> {
+                Set<String> referencedRuleFilters = jobParams.getJob().getAnalysisConfig().extractReferencedFilters();
+                if (referencedRuleFilters.isEmpty()) {
+                    referencedRuleFiltersPresentListener.onResponse(true);
+                } else {
+                    GetFiltersAction.Request getFiltersRequest = new GetFiltersAction.Request();
+                    getFiltersRequest.setResourceId(referencedRuleFilters.stream().collect(Collectors.joining(",")));
+                    getFiltersRequest.setAllowNoResources(false);
+                    client.execute(
+                        GetFiltersAction.INSTANCE,
+                        getFiltersRequest,
+                        ActionListener.wrap(filtersResponse -> referencedRuleFiltersPresentListener.onResponse(true), listener::onFailure)
+                    );
+                }
+            }, listener::onFailure);
+
             // Validate the model snapshot is supported
             ActionListener<Boolean> getJobHandler = ActionListener.wrap(response -> {
                 if (jobParams.getJob().getModelSnapshotId() == null) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java
index 1c9c01e55cbe..e9f857b2650a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java
@@ -140,7 +140,7 @@ private void previewDatafeed(
                 job,
                 xContentRegistry,
                 // Fake DatafeedTimingStatsReporter that does not have access to results index
-                new DatafeedTimingStatsReporter(new DatafeedTimingStats(datafeedConfig.getJobId()), (ts, refreshPolicy) -> {}),
+                new DatafeedTimingStatsReporter(new DatafeedTimingStats(datafeedConfig.getJobId()), (ts, refreshPolicy, listener1) -> {}),
                 listener.delegateFailure(
                     (l, dataExtractorFactory) -> isDateNanos(
                         previewDatafeedConfig,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java
index 802be705e7ea..a4982addd1d0 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.license.License;
 import org.elasticsearch.license.LicenseUtils;
+import org.elasticsearch.license.RemoteClusterLicenseChecker;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -54,10 +55,12 @@
 
 import java.io.IOException;
 import java.time.Instant;
+import java.util.Arrays;
 import java.util.Map;
 import java.util.Objects;
 import java.util.function.Supplier;
 
+import static java.util.function.Predicate.not;
 import static org.elasticsearch.xpack.ml.utils.SecondaryAuthorizationUtils.useSecondaryAuthIfAvailable;
 
 public class TransportPutDataFrameAnalyticsAction extends TransportMasterNodeAction<
@@ -160,10 +163,12 @@ private void putValidatedConfig(
         if (securityContext != null) {
             useSecondaryAuthIfAvailable(securityContext, () -> {
                 final String username = securityContext.getUser().principal();
-                RoleDescriptor.IndicesPrivileges sourceIndexPrivileges = RoleDescriptor.IndicesPrivileges.builder()
-                    .indices(preparedForPutConfig.getSource().getIndex())
-                    .privileges("read")
-                    .build();
+                // DFA doesn't support CCS, but if it did it would need this filter, so it's safest to have the filter
+                // in place even though it's a no-op.
+                // TODO: Remove this filter once https://github.com/elastic/elasticsearch/issues/67798 is fixed.
+                final String[] sourceIndices = Arrays.stream(preparedForPutConfig.getSource().getIndex())
+                    .filter(not(RemoteClusterLicenseChecker::isRemoteIndex))
+                    .toArray(String[]::new);
                 RoleDescriptor.IndicesPrivileges destIndexPrivileges = RoleDescriptor.IndicesPrivileges.builder()
                     .indices(preparedForPutConfig.getDest().getIndex())
                     .privileges("read", "index", "create_index")
@@ -173,7 +178,17 @@ private void putValidatedConfig(
                 privRequest.applicationPrivileges(new RoleDescriptor.ApplicationResourcePrivileges[0]);
                 privRequest.username(username);
                 privRequest.clusterPrivileges(Strings.EMPTY_ARRAY);
-                privRequest.indexPrivileges(sourceIndexPrivileges, destIndexPrivileges);
+                RoleDescriptor.IndicesPrivileges[] indicesPrivileges;
+                if (sourceIndices.length > 0) {
+                    RoleDescriptor.IndicesPrivileges sourceIndexPrivileges = RoleDescriptor.IndicesPrivileges.builder()
+                        .indices(sourceIndices)
+                        .privileges("read")
+                        .build();
+                    indicesPrivileges = new RoleDescriptor.IndicesPrivileges[] { sourceIndexPrivileges, destIndexPrivileges };
+                } else {
+                    indicesPrivileges = new RoleDescriptor.IndicesPrivileges[] { destIndexPrivileges };
+                }
+                privRequest.indexPrivileges(indicesPrivileges);
 
                 ActionListener<HasPrivilegesResponse> privResponseListener = ActionListener.wrap(
                     r -> handlePrivsResponse(username, preparedForPutConfig, r, masterNodeTimeout, listener),
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java
index 2f6b13934a63..c431842a969b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java
@@ -334,7 +334,6 @@ private ActionListener<RevertModelSnapshotAction.Response> wrapRevertDataCountsL
         ModelSnapshot modelSnapshot,
         String jobId
     ) {
-
         return ActionListener.wrap(response -> jobResultsProvider.dataCounts(jobId, counts -> {
             counts.setLatestRecordTimeStamp(modelSnapshot.getLatestRecordTimeStamp());
             jobDataCountsPersister.persistDataCountsAsync(jobId, counts, listener.delegateFailure((l, r) -> l.onResponse(response)));
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSemanticSearchAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSemanticSearchAction.java
deleted file mode 100644
index 8525d8275797..000000000000
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSemanticSearchAction.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.ml.action;
-
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.client.internal.OriginSettingClient;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
-import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.tasks.TaskId;
-import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.ml.action.InferTrainedModelDeploymentAction;
-import org.elasticsearch.xpack.core.ml.action.SemanticSearchAction;
-import org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingResults;
-
-import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
-
-public class TransportSemanticSearchAction extends HandledTransportAction<SemanticSearchAction.Request, SemanticSearchAction.Response> {
-
-    private final Client client;
-    private final ClusterService clusterService;
-
-    @Inject
-    public TransportSemanticSearchAction(
-        TransportService transportService,
-        ActionFilters actionFilters,
-        Client client,
-        ClusterService clusterService
-    ) {
-        super(SemanticSearchAction.NAME, transportService, actionFilters, SemanticSearchAction.Request::new);
-        this.client = client;
-        this.clusterService = clusterService;
-    }
-
-    @Override
-    protected void doExecute(Task task, SemanticSearchAction.Request request, ActionListener<SemanticSearchAction.Response> listener) {
-
-        var parentTask = new TaskId(clusterService.localNode().getId(), task.getId());
-        var originSettingClient = new OriginSettingClient(client, ML_ORIGIN);
-
-        // call inference as ML_ORIGIN
-        originSettingClient.execute(
-            InferTrainedModelDeploymentAction.INSTANCE,
-            toInferenceRequest(request, parentTask),
-            ActionListener.wrap(inferenceResults -> {
-                if (inferenceResults.getResults()instanceof TextEmbeddingResults textEmbeddingResults) {
-
-                    var searchRequestBuilder = buildSearch(client, textEmbeddingResults, request);
-                    searchRequestBuilder.request().setParentTask(parentTask);
-
-                    // execute search using the callers permissions
-                    searchRequestBuilder.execute(ActionListener.wrap(searchResponse -> {
-                        listener.onResponse(
-                            new SemanticSearchAction.Response(
-                                searchResponse.getTook(),
-                                TimeValue.timeValueMillis(inferenceResults.getTookMillis()),
-                                searchResponse
-                            )
-                        );
-                    }, listener::onFailure));
-                } else {
-                    listener.onFailure(
-                        new IllegalArgumentException(
-                            "model ["
-                                + request.getModelId()
-                                + "] must be a text_embedding model; provided ["
-                                + inferenceResults.getResults().getWriteableName()
-                                + "]"
-                        )
-                    );
-                }
-            }, listener::onFailure)
-        );
-    }
-
-    private SearchRequestBuilder buildSearch(Client client, TextEmbeddingResults inferenceResults, SemanticSearchAction.Request request) {
-        var searchBuilder = client.prepareSearch();
-        searchBuilder.setIndices(request.indices());
-        searchBuilder.setIndicesOptions(request.indicesOptions());
-        if (request.getRouting() != null) {
-            searchBuilder.setRouting(request.getRouting());
-        }
-
-        var knnSearchBuilder = request.getKnnQueryOptions().toKnnSearchBuilder(inferenceResults.getInferenceAsFloat());
-
-        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-        sourceBuilder.trackTotalHitsUpTo(SearchContext.TRACK_TOTAL_HITS_ACCURATE);
-        sourceBuilder.knnSearch(knnSearchBuilder);
-        if (request.getSize() != -1) {
-            sourceBuilder.size(request.getSize());
-        }
-        if (request.getQuery() != null) {
-            sourceBuilder.query(request.getQuery());
-        }
-        if (request.getFetchSource() != null) {
-            sourceBuilder.fetchSource(request.getFetchSource());
-        }
-        if (request.getFields() != null) {
-            for (FieldAndFormat field : request.getFields()) {
-                sourceBuilder.fetchField(field);
-            }
-        }
-        if (request.getDocValueFields() != null) {
-            for (FieldAndFormat field : request.getDocValueFields()) {
-                sourceBuilder.docValueField(field.field, field.format);
-            }
-        }
-        if (request.getStoredFields() != null) {
-            sourceBuilder.storedFields(request.getStoredFields());
-        }
-
-        searchBuilder.setSource(sourceBuilder);
-        return searchBuilder;
-    }
-
-    private InferTrainedModelDeploymentAction.Request toInferenceRequest(SemanticSearchAction.Request request, TaskId parentTask) {
-        var inferenceRequest = new InferTrainedModelDeploymentAction.Request(
-            request.getModelId(),
-            request.getEmbeddingConfig(),
-            request.getQueryString(),
-            request.getInferenceTimeout()
-        );
-        inferenceRequest.setParentTask(parentTask);
-        return inferenceRequest;
-    }
-}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
index 4212e80d9dc0..ce5ab42fa2ee 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
@@ -355,7 +355,7 @@ private void createDataExtractor(
             job,
             xContentRegistry,
             // Fake DatafeedTimingStatsReporter that does not have access to results index
-            new DatafeedTimingStatsReporter(new DatafeedTimingStats(job.getId()), (ts, refreshPolicy) -> {}),
+            new DatafeedTimingStatsReporter(new DatafeedTimingStats(job.getId()), (ts, refreshPolicy, listener1) -> {}),
             ActionListener.wrap(
                 unused -> persistentTasksService.sendStartRequest(
                     MlTasks.datafeedTaskId(params.getDatafeedId()),
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsAction.java
index 76fffbd9607f..cc4939234d3a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsAction.java
@@ -356,9 +356,9 @@ protected StopDataFrameAnalyticsAction.Response newResponse(
     ) {
         if (request.getExpandedIds().size() != tasks.size()) {
             if (taskOperationFailures.isEmpty() == false) {
-                throw org.elasticsearch.ExceptionsHelper.convertToElastic(taskOperationFailures.get(0).getCause());
+                throw ExceptionsHelper.taskOperationFailureToStatusException(taskOperationFailures.get(0));
             } else if (failedNodeExceptions.isEmpty() == false) {
-                throw org.elasticsearch.ExceptionsHelper.convertToElastic(failedNodeExceptions.get(0));
+                throw failedNodeExceptions.get(0);
             } else {
                 // This can happen when the actual task in the node no longer exists,
                 // which means the data frame analytic(s) have already been closed.
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java
index fccb8bcc2912..8dab7cd168c2 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDatafeedAction.java
@@ -520,9 +520,9 @@ protected StopDatafeedAction.Response newResponse(
         // tasks, otherwise something went wrong
         if (request.getResolvedStartedDatafeedIds().length != tasks.size()) {
             if (taskOperationFailures.isEmpty() == false) {
-                throw org.elasticsearch.ExceptionsHelper.convertToElastic(taskOperationFailures.get(0).getCause());
+                throw ExceptionsHelper.taskOperationFailureToStatusException(taskOperationFailures.get(0));
             } else if (failedNodeExceptions.isEmpty() == false) {
-                throw org.elasticsearch.ExceptionsHelper.convertToElastic(failedNodeExceptions.get(0));
+                throw failedNodeExceptions.get(0);
             } else {
                 // This can happen when the local task in the node no longer exists,
                 // which means the datafeed(s) have already been stopped. It can
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopTrainedModelDeploymentAction.java
index 2f514df0eefa..d035bb6b3da5 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopTrainedModelDeploymentAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopTrainedModelDeploymentAction.java
@@ -254,9 +254,9 @@ protected StopTrainedModelDeploymentAction.Response newResponse(
         List<FailedNodeException> failedNodeExceptions
     ) {
         if (taskOperationFailures.isEmpty() == false) {
-            throw org.elasticsearch.ExceptionsHelper.convertToElastic(taskOperationFailures.get(0).getCause());
+            throw ExceptionsHelper.taskOperationFailureToStatusException(taskOperationFailures.get(0));
         } else if (failedNodeExceptions.isEmpty() == false) {
-            throw org.elasticsearch.ExceptionsHelper.convertToElastic(failedNodeExceptions.get(0));
+            throw failedNodeExceptions.get(0);
         } else {
             return new StopTrainedModelDeploymentAction.Response(true);
         }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregationBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregationBuilder.java
index fa1b8313d5bb..691539770503 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregationBuilder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregationBuilder.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.xpack.ml.aggs.categorization;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
@@ -54,7 +54,7 @@ public class CategorizeTextAggregationBuilder extends AbstractAggregationBuilder
     // some nodes are pre-8.3 and others are newer, so we throw an error in
     // this situation. The aggregation was experimental at the time this change
     // was made, so this is acceptable.
-    public static final Version ALGORITHM_CHANGED_VERSION = Version.V_8_3_0;
+    public static final TransportVersion ALGORITHM_CHANGED_VERSION = TransportVersion.V_8_3_0;
 
     static final ParseField FIELD_NAME = new ParseField("field");
     static final ParseField SIMILARITY_THRESHOLD = new ParseField("similarity_threshold");
@@ -125,7 +125,7 @@ public CategorizeTextAggregationBuilder setFieldName(String fieldName) {
     public CategorizeTextAggregationBuilder(StreamInput in) throws IOException {
         super(in);
         // Disallow this aggregation in mixed version clusters that cross the algorithm change boundary.
-        if (in.getVersion().before(ALGORITHM_CHANGED_VERSION)) {
+        if (in.getTransportVersion().before(ALGORITHM_CHANGED_VERSION)) {
             throw new ElasticsearchException(
                 "["
                     + NAME
@@ -281,7 +281,7 @@ protected CategorizeTextAggregationBuilder(
     @Override
     protected void doWriteTo(StreamOutput out) throws IOException {
         // Disallow this aggregation in mixed version clusters that cross the algorithm change boundary.
-        if (out.getVersion().before(ALGORITHM_CHANGED_VERSION)) {
+        if (out.getTransportVersion().before(ALGORITHM_CHANGED_VERSION)) {
             throw new ElasticsearchException(
                 "["
                     + NAME
@@ -344,7 +344,7 @@ public String getType() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
+    public TransportVersion getMinimalSupportedVersion() {
         // This isn't strictly true, as the categorize_text aggregation has existed since 7.16.
         // However, the implementation completely changed in 8.3, so it's best that if the
         // coordinating node is on 8.3 or above then it should refuse to use this aggregation
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregator.java
index f0476f8ba745..549709c864b4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregator.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/CategorizeTextAggregator.java
@@ -10,8 +10,10 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.util.BytesRefHash;
 import org.elasticsearch.common.util.ObjectArray;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.search.aggregations.AggregationExecutionContext;
@@ -25,7 +27,9 @@
 import org.elasticsearch.search.aggregations.bucket.terms.LongKeyedBucketOrds;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
-import org.elasticsearch.search.lookup.SourceLookup;
+import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.search.lookup.SourceFilter;
+import org.elasticsearch.search.lookup.SourceProvider;
 import org.elasticsearch.xpack.core.ml.job.config.CategorizationAnalyzerConfig;
 import org.elasticsearch.xpack.ml.aggs.categorization.InternalCategorizationAggregation.Bucket;
 import org.elasticsearch.xpack.ml.job.categorization.CategorizationAnalyzer;
@@ -40,7 +44,8 @@
 public class CategorizeTextAggregator extends DeferableBucketAggregator {
 
     private final TermsAggregator.BucketCountThresholds bucketCountThresholds;
-    private final SourceLookup sourceLookup;
+    private final SourceProvider sourceProvider;
+    private final SourceFilter sourceFilter;
     private final MappedFieldType fieldType;
     private final CategorizationAnalyzer analyzer;
     private final String sourceFieldName;
@@ -63,8 +68,9 @@ protected CategorizeTextAggregator(
         Map<String, Object> metadata
     ) throws IOException {
         super(name, factories, context, parent, metadata);
-        this.sourceLookup = context.lookup().source();
+        this.sourceProvider = context.lookup();
         this.sourceFieldName = sourceFieldName;
+        this.sourceFilter = new SourceFilter(new String[] { sourceFieldName }, Strings.EMPTY_ARRAY);
         this.fieldType = fieldType;
         CategorizationAnalyzerConfig analyzerConfig = Optional.ofNullable(categorizationAnalyzerConfig)
             .orElse(CategorizationAnalyzerConfig.buildStandardCategorizationAnalyzer(List.of()));
@@ -156,8 +162,8 @@ public void collect(int doc, long owningBucketOrd) throws IOException {
             }
 
             private void collectFromSource(int doc, long owningBucketOrd, TokenListCategorizer categorizer) throws IOException {
-                sourceLookup.setSegmentAndDocument(aggCtx.getLeafReaderContext(), doc);
-                Iterator<String> itr = sourceLookup.extractRawValuesWithoutCaching(sourceFieldName).stream().map(obj -> {
+                Source source = sourceProvider.getSource(aggCtx.getLeafReaderContext(), doc).filter(sourceFilter);
+                Iterator<String> itr = XContentMapValues.extractRawValues(sourceFieldName, source.source()).stream().map(obj -> {
                     if (obj instanceof BytesRef) {
                         return fieldType.valueForDisplay(obj).toString();
                     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/InternalCategorizationAggregation.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/InternalCategorizationAggregation.java
index 027a0edade33..9d63eeb8d381 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/InternalCategorizationAggregation.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/categorization/InternalCategorizationAggregation.java
@@ -108,7 +108,7 @@ public Bucket(SerializableTokenListCategory serializableCategory, long bucketOrd
 
         public Bucket(StreamInput in) throws IOException {
             // Disallow this aggregation in mixed version clusters that cross the algorithm change boundary.
-            if (in.getVersion().before(CategorizeTextAggregationBuilder.ALGORITHM_CHANGED_VERSION)) {
+            if (in.getTransportVersion().before(CategorizeTextAggregationBuilder.ALGORITHM_CHANGED_VERSION)) {
                 throw new ElasticsearchException(
                     "["
                         + CategorizeTextAggregationBuilder.NAME
@@ -126,7 +126,7 @@ public Bucket(StreamInput in) throws IOException {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             // Disallow this aggregation in mixed version clusters that cross the algorithm change boundary.
-            if (out.getVersion().before(CategorizeTextAggregationBuilder.ALGORITHM_CHANGED_VERSION)) {
+            if (out.getTransportVersion().before(CategorizeTextAggregationBuilder.ALGORITHM_CHANGED_VERSION)) {
                 throw new ElasticsearchException(
                     "["
                         + CategorizeTextAggregationBuilder.NAME
@@ -238,7 +238,7 @@ protected InternalCategorizationAggregation(
     public InternalCategorizationAggregation(StreamInput in) throws IOException {
         super(in);
         // Disallow this aggregation in mixed version clusters that cross the algorithm change boundary.
-        if (in.getVersion().before(CategorizeTextAggregationBuilder.ALGORITHM_CHANGED_VERSION)) {
+        if (in.getTransportVersion().before(CategorizeTextAggregationBuilder.ALGORITHM_CHANGED_VERSION)) {
             throw new ElasticsearchException(
                 "["
                     + CategorizeTextAggregationBuilder.NAME
@@ -256,7 +256,7 @@ public InternalCategorizationAggregation(StreamInput in) throws IOException {
     @Override
     protected void doWriteTo(StreamOutput out) throws IOException {
         // Disallow this aggregation in mixed version clusters that cross the algorithm change boundary.
-        if (out.getVersion().before(CategorizeTextAggregationBuilder.ALGORITHM_CHANGED_VERSION)) {
+        if (out.getTransportVersion().before(CategorizeTextAggregationBuilder.ALGORITHM_CHANGED_VERSION)) {
             throw new ElasticsearchException(
                 "["
                     + CategorizeTextAggregationBuilder.NAME
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangePointAggregationBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangePointAggregationBuilder.java
index 1c0112e84e35..3aa4c71ad41c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangePointAggregationBuilder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/changepoint/ChangePointAggregationBuilder.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.ml.aggs.changepoint;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers;
@@ -58,8 +58,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_2_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_2_0;
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/correlation/BucketCorrelationAggregationBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/correlation/BucketCorrelationAggregationBuilder.java
index 4c09a2bc2e74..3503cc20d1e8 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/correlation/BucketCorrelationAggregationBuilder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/correlation/BucketCorrelationAggregationBuilder.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.ml.aggs.correlation;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers;
@@ -136,7 +136,7 @@ public int hashCode() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_14_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_14_0;
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/CountingItemSetTraverser.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/CountingItemSetTraverser.java
index 3dd56d624645..35a0bd9e4c43 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/CountingItemSetTraverser.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/CountingItemSetTraverser.java
@@ -261,6 +261,14 @@ public void prune() {
         topItemSetTraverser.prune();
     }
 
+    public void pruneToNextMainBranch() {
+        long thisCount = getCount();
+
+        while (getNumberOfItems() > 1 && getCount() == thisCount) {
+            topItemSetTraverser.prune();
+        }
+    }
+
     /**
      * Return true if the item set tree is on a leaf, which mean no further items can be added to the candidate set.
      */
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/EclatMapReducer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/EclatMapReducer.java
index 7ba54033c2ed..9a8a47991af4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/EclatMapReducer.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/EclatMapReducer.java
@@ -9,10 +9,14 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.io.stream.ByteArrayStreamInput;
+import org.elasticsearch.common.io.stream.BytesRefStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.BytesRefArray;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.core.Tuple;
@@ -90,6 +94,10 @@ FrequentItemSet[] getFrequentItemSets() {
             return frequentItemSets;
         }
 
+        Map<String, Object> getProfilingInfo() {
+            return profilingInfo;
+        }
+
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeArray(frequentItemSets);
@@ -112,6 +120,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     private static final Logger logger = LogManager.getLogger(EclatMapReducer.class);
     private static final int VERSION = 1;
 
+    // named writable for this implementation
     public static final String NAME = "frequent_items-eclat-" + VERSION;
 
     // cache for marking transactions visited, memory usage: ((BITSET_CACHE_TRAVERSAL_DEPTH -2) * BITSET_CACHE_NUMBER_OF_TRANSACTIONS) / 8
@@ -128,6 +137,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     private final Map<String, Object> profilingInfoReduce;
     private final Map<String, Object> profilingInfoMap;
 
+    @Override
+    protected OrdinalOptimization getDefaultOrdinalOptimization() {
+        return OrdinalOptimization.GLOBAL_ORDINALS;
+    }
+
     public EclatMapReducer(String aggregationName, double minimumSupport, int minimumSetSize, int size, boolean profiling) {
         super(aggregationName, NAME);
 
@@ -195,8 +209,10 @@ public HashBasedTransactionStore mapFiltered(HashBasedTransactionStore transacti
     }
 
     @Override
-    protected ImmutableTransactionStore mapFinalize(HashBasedTransactionStore transactionStore) {
+    protected ImmutableTransactionStore mapFinalize(HashBasedTransactionStore transactionStore, List<OrdinalLookupFunction> ordinalLookup)
+        throws IOException {
 
+        final long relativeStartNanos = System.nanoTime();
         // reported by shard in collectDebugInfo
         if (profiling) {
             // note: collect this before creating the immutable transaction store as memory gets freed
@@ -208,15 +224,72 @@ protected ImmutableTransactionStore mapFinalize(HashBasedTransactionStore transa
             profilingInfoMap.put("unique_transactions_after_map", transactionStore.getUniqueTransactionCount());
         }
 
-        ImmutableTransactionStore transactionStoreMapFinalize = transactionStore.createImmutableTransactionStore();
+        ImmutableTransactionStore transactionStoreMapFinalize = rewriteOrdinalItems(
+            transactionStore.createImmutableTransactionStore(),
+            ordinalLookup
+        );
 
         if (profiling) {
             profilingInfoMap.put("ram_bytes_transactionstore_map_finalize", transactionStoreMapFinalize.ramBytesUsed());
+            profilingInfoMap.put("runtime_ms_finalize_map", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - relativeStartNanos));
         }
 
         return transactionStoreMapFinalize;
     }
 
+    /**
+     * rewrites items that use ordinals to their real values
+     */
+    private ImmutableTransactionStore rewriteOrdinalItems(
+        ImmutableTransactionStore transactionStore,
+        List<OrdinalLookupFunction> ordinalLookups
+    ) throws IOException {
+        if (ordinalLookups == null || ordinalLookups.isEmpty()) {
+            return transactionStore;
+        }
+
+        logger.trace("rewrite items based on ordinals");
+
+        BytesRef scratchBytesRef = new BytesRef();
+        ByteArrayStreamInput scratchByteArrayStreamInput = new ByteArrayStreamInput();
+        BytesRefStreamOutput scratchItemBytesStreamOutput = new BytesRefStreamOutput();
+        BytesRefArray items = transactionStore.getItems();
+
+        BytesRefArray rewrittenItems = null;
+        try {
+            rewrittenItems = new BytesRefArray(items.size(), transactionStore.getBigArrays());
+
+            for (long i = 0; i < items.size(); ++i) {
+                items.get(i, scratchBytesRef);
+                scratchByteArrayStreamInput.reset(scratchBytesRef.bytes, scratchBytesRef.offset, scratchBytesRef.length);
+                int fieldId = scratchByteArrayStreamInput.readVInt();
+                Object ordinal = scratchByteArrayStreamInput.readGenericValue();
+                scratchItemBytesStreamOutput.reset();
+                scratchItemBytesStreamOutput.writeVInt(fieldId);
+                scratchItemBytesStreamOutput.writeGenericValue(ordinalLookups.get(fieldId).apply(ordinal));
+
+                rewrittenItems.append(scratchItemBytesStreamOutput.get());
+            }
+
+            // swap items
+            ImmutableTransactionStore rewrittenStore = new ImmutableTransactionStore(
+                transactionStore.getBigArrays(),
+                rewrittenItems,
+                transactionStore.getItemCounts(),
+                transactionStore.getTotalItemCount(),
+                transactionStore.getTransactions(),
+                transactionStore.getTransactionCounts(),
+                transactionStore.getTotalTransactionCount(),
+                transactionStore.getFilteredTransactionCount()
+            );
+
+            rewrittenItems = items;
+            return rewrittenStore;
+        } finally {
+            Releasables.close(rewrittenItems);
+        }
+    }
+
     @Override
     protected void collectDebugInfo(BiConsumer<String, Object> add) {
         profilingInfoMap.forEach(add);
@@ -420,6 +493,22 @@ private static EclatResult eclat(
                         // no need to set visited, as we are on a leaf
                     }
 
+                    /**
+                     * Optimization
+                     *
+                     * a - b - c - d
+                     * |   |    \- h
+                     * |   |\- e - f
+                     * |    \- h - j
+                     *  \- x - y
+                     *
+                     * assume we pruned at d above, if c has the same count as d, we don't need the sub-branches, but go up
+                     * until we find a branch that has a higher count than the pruned one. This allows us to skip over subtrees.
+                     */
+                    if (setTraverser.atLeaf()) {
+                        setTraverser.pruneToNextMainBranch();
+                    }
+
                     continue;
                 }
 
@@ -469,6 +558,16 @@ private static EclatResult eclat(
                     previousMinCount = minCount;
                     logger.debug("adjusting min count to {}", minCount);
                 }
+
+                /**
+                 * Optimization:
+                 *
+                 * If we reached a leaf, go up the branch until the new branch has a higher count than our current leaf.
+                 */
+                if (setTraverser.atLeaf()) {
+                    setTraverser.prune();
+                    setTraverser.pruneToNextMainBranch();
+                }
             }
             FrequentItemSet[] topFrequentItems = collector.finalizeAndGetResults(fields);
             final long eclatRuntimeNanos = System.nanoTime() - relativeStartNanos;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetCollector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetCollector.java
index 1ceb5935ae2c..0b04240e8970 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetCollector.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetCollector.java
@@ -39,7 +39,7 @@
  *
  * Technically this is implemented as priority queue which is implemented as heap with the last item on top.
  *
- * To get to the top-n results pop the collector until it is empty and than reverse the order.
+ * To get to the top-n results pop the collector until it is empty and then reverse the order.
  */
 public final class FrequentItemSetCollector {
 
@@ -177,6 +177,7 @@ FrequentItemSet toFrequentItemSet(List<Field> fields) throws IOException {
             int pos = items.nextSetBit(0);
             while (pos > 0) {
                 Tuple<Integer, Object> item = transactionStore.getItem(topItemIds.getItemIdAt(pos - 1));
+                assert item.v1() < fields.size() : "item id exceed number of given items, did you configure eclat correctly?";
                 final Field field = fields.get(item.v1());
                 Object formattedValue = field.formatValue(item.v2());
                 String fieldName = fields.get(item.v1()).getName();
@@ -288,27 +289,42 @@ public long add(ItemSetBitSet itemSet, long docCount) {
             return queue.top().getDocCount();
         }
 
-        // closed set criteria: don't add if we already store a superset
-        if (hasSuperSet(itemSet, docCount)) {
-            logger.trace("skip itemset with super set");
-            return queue.size() < size ? min : queue.top().getDocCount();
+        /**
+         * Criteria for closed sets
+         *
+         * An item set is called closed if no superset has the same support.
+         * e.g.
+         *
+         * [cat, dog, crocodile] -> 0.2
+         * [cat, dog, crocodile, river] -> 0.2
+         *
+         * [cat, dog, crocodile] gets skipped, because [cat, dog, crocodile, river] has the same support.
+         * if [cat, dog, crocodile] was added first, [cat, dog, crocodile, river] replaces [cat, dog, crocodile]
+         */
+        List<FrequentItemSetCandidate> setsThatShareSameDocCountBits = frequentItemsByCount.get(docCount);
+
+        if (setsThatShareSameDocCountBits != null) {
+            for (FrequentItemSetCandidate otherSet : setsThatShareSameDocCountBits) {
+                ItemSetBitSet.SetRelation relation = itemSet.setRelation(otherSet.getItems());
+
+                if (relation.equals(ItemSetBitSet.SetRelation.SUPER_SET)) {
+                    removeFromFrequentItemsByCount(otherSet);
+                    queue.remove(otherSet);
+                    break;
+                }
+
+                if (relation.equals(ItemSetBitSet.SetRelation.SUB_SET)) {
+                    // closed set criteria: don't add if we already store a superset
+                    return queue.size() < size ? min : queue.top().getDocCount();
+                }
+            }
         }
 
         spareSet.reset(count++, itemSet, docCount);
         FrequentItemSetCandidate newItemSet = spareSet;
         FrequentItemSetCandidate removedItemSet = queue.insertWithOverflow(spareSet);
         if (removedItemSet != null) {
-            // remove item from frequentItemsByCount
-            frequentItemsByCount.compute(removedItemSet.getDocCount(), (k, sets) -> {
-
-                // short cut, if there is only 1, it must be the one we are looking for
-                if (sets.size() == 1) {
-                    return null;
-                }
-
-                sets.remove(removedItemSet);
-                return sets;
-            });
+            removeFromFrequentItemsByCount(removedItemSet);
             spareSet = removedItemSet;
         } else {
             spareSet = new FrequentItemSetCandidate();
@@ -319,6 +335,20 @@ public long add(ItemSetBitSet itemSet, long docCount) {
         return queue.size() < size ? min : queue.top().getDocCount();
     }
 
+    private void removeFromFrequentItemsByCount(FrequentItemSetCandidate removedItemSet) {
+        // remove item from frequentItemsByCount
+        frequentItemsByCount.compute(removedItemSet.getDocCount(), (k, sets) -> {
+
+            // short cut, if there is only 1, it must be the one we are looking for
+            if (sets.size() == 1) {
+                return null;
+            }
+
+            sets.remove(removedItemSet);
+            return sets;
+        });
+    }
+
     // for unit tests
     Map<Long, List<FrequentItemSetCandidate>> getFrequentItemsByCount() {
         return frequentItemsByCount;
@@ -331,32 +361,4 @@ FrequentItemSetPriorityQueue getQueue() {
     FrequentItemSetCandidate getLastSet() {
         return queue.top();
     }
-
-    /**
-     * Criteria for closed sets
-     *
-     * A item set is called closed if no superset has the same support.
-     *
-     * E.g.
-     *
-     * [cat, dog, crocodile] -> 0.2
-     * [cat, dog, crocodile, river] -> 0.2
-     *
-     * [cat, dog, crocodile] gets skipped, because [cat, dog, crocodile, river] has the same support.
-     *
-     */
-    private boolean hasSuperSet(ItemSetBitSet itemSetBitSet, long docCount) {
-        List<FrequentItemSetCandidate> setsThatShareSameDocCountBits = frequentItemsByCount.get(docCount);
-
-        if (setsThatShareSameDocCountBits != null) {
-            for (FrequentItemSetCandidate otherSet : setsThatShareSameDocCountBits) {
-                if (itemSetBitSet.isSubset(otherSet.getItems())) {
-                    return true;
-                }
-            }
-        }
-
-        return false;
-    }
-
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregationBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregationBuilder.java
index 4a10dd2d5a83..e4263da428bc 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregationBuilder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregationBuilder.java
@@ -7,7 +7,8 @@
 
 package org.elasticsearch.xpack.ml.aggs.frequentitemsets;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
@@ -32,17 +33,24 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.common.Strings.format;
+
 public final class FrequentItemSetsAggregationBuilder extends AbstractAggregationBuilder<FrequentItemSetsAggregationBuilder> {
 
-    public static final String NAME = "frequent_items";
+    public static final String NAME = "frequent_item_sets";
+
+    // name used between 8.4 - 8.6, kept for backwards compatibility until 9.0
+    public static final String DEPRECATED_NAME = "frequent_items";
 
     public static final double DEFAULT_MINIMUM_SUPPORT = 0.01;
     public static final int DEFAULT_MINIMUM_SET_SIZE = 1;
     public static final int DEFAULT_SIZE = 10;
+    public static final List<String> EXECUTION_HINT_ALLOWED_MODES = List.of("global_ordinals", "map");
 
     public static final ParseField MINIMUM_SUPPORT = new ParseField("minimum_support");
     public static final ParseField MINIMUM_SET_SIZE = new ParseField("minimum_set_size");
     public static final ParseField FIELDS = new ParseField("fields");
+    public static final ParseField EXECUTION_HINT_FIELD_NAME = new ParseField("execution_hint");
 
     public static final ConstructingObjectParser<FrequentItemSetsAggregationBuilder, String> PARSER = new ConstructingObjectParser<>(
         NAME,
@@ -54,8 +62,9 @@ public final class FrequentItemSetsAggregationBuilder extends AbstractAggregatio
             int minimumSetSize = args[2] == null ? DEFAULT_MINIMUM_SET_SIZE : (int) args[2];
             int size = args[3] == null ? DEFAULT_SIZE : (int) args[3];
             QueryBuilder filter = (QueryBuilder) args[4];
+            String executionHint = (String) args[5];
 
-            return new FrequentItemSetsAggregationBuilder(context, fields, minimumSupport, minimumSetSize, size, filter);
+            return new FrequentItemSetsAggregationBuilder(context, fields, minimumSupport, minimumSetSize, size, filter, executionHint);
         }
     );
 
@@ -64,7 +73,8 @@ public final class FrequentItemSetsAggregationBuilder extends AbstractAggregatio
             false,  // scriptable
             false,  // timezone aware
             false,  // filtered (not defined per field, but for all fields below)
-            false   // format
+            false,  // format
+            true    // includes and excludes
         );
         PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, n) -> fieldsParser.parse(p, null).build(), FIELDS);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), MINIMUM_SUPPORT);
@@ -76,6 +86,7 @@ public final class FrequentItemSetsAggregationBuilder extends AbstractAggregatio
             MultiValuesSourceFieldConfig.FILTER,
             ObjectParser.ValueType.OBJECT
         );
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), EXECUTION_HINT_FIELD_NAME);
     }
 
     static final ValuesSourceRegistry.RegistryKey<ItemSetMapReduceValueSource.ValueSourceSupplier> REGISTRY_KEY =
@@ -103,6 +114,7 @@ public static void registerAggregators(ValuesSourceRegistry.Builder registry) {
     private final int minimumSetSize;
     private final int size;
     private final QueryBuilder filter;
+    private final String executionHint;
 
     public FrequentItemSetsAggregationBuilder(
         String name,
@@ -110,7 +122,8 @@ public FrequentItemSetsAggregationBuilder(
         double minimumSupport,
         int minimumSetSize,
         int size,
-        QueryBuilder filter
+        QueryBuilder filter,
+        String executionHint
     ) {
         super(name);
         this.fields = fields;
@@ -131,6 +144,18 @@ public FrequentItemSetsAggregationBuilder(
         }
         this.size = size;
         this.filter = filter;
+
+        if (executionHint != null && EXECUTION_HINT_ALLOWED_MODES.contains(executionHint) == false) {
+            throw new IllegalArgumentException(
+                format(
+                    "[execution_hint] must be one of [%s]. Found [%s]",
+                    Strings.collectionToCommaDelimitedString(EXECUTION_HINT_ALLOWED_MODES),
+                    executionHint
+                )
+            );
+        }
+
+        this.executionHint = executionHint;
     }
 
     public FrequentItemSetsAggregationBuilder(StreamInput in) throws IOException {
@@ -139,11 +164,16 @@ public FrequentItemSetsAggregationBuilder(StreamInput in) throws IOException {
         this.minimumSupport = in.readDouble();
         this.minimumSetSize = in.readVInt();
         this.size = in.readVInt();
-        if (in.getVersion().onOrAfter(Version.V_8_6_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_6_0)) {
             this.filter = in.readOptionalNamedWriteable(QueryBuilder.class);
         } else {
             this.filter = null;
         }
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            this.executionHint = in.readOptionalString();
+        } else {
+            this.executionHint = null;
+        }
     }
 
     @Override
@@ -153,7 +183,7 @@ public boolean supportsSampling() {
 
     @Override
     protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map<String, Object> metadata) {
-        return new FrequentItemSetsAggregationBuilder(name, fields, minimumSupport, minimumSetSize, size, filter);
+        return new FrequentItemSetsAggregationBuilder(name, fields, minimumSupport, minimumSetSize, size, filter, executionHint);
     }
 
     @Override
@@ -167,9 +197,12 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeDouble(minimumSupport);
         out.writeVInt(minimumSetSize);
         out.writeVInt(size);
-        if (out.getVersion().onOrAfter(Version.V_8_6_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_6_0)) {
             out.writeOptionalNamedWriteable(filter);
         }
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_7_0)) {
+            out.writeOptionalString(executionHint);
+        }
     }
 
     @Override
@@ -186,7 +219,8 @@ protected AggregatorFactory doBuild(AggregationContext context, AggregatorFactor
             minimumSupport,
             minimumSetSize,
             size,
-            filter
+            filter,
+            executionHint
         );
     }
 
@@ -225,8 +259,8 @@ public FrequentItemSetsAggregationBuilder subAggregation(AggregationBuilder aggr
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_4_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_4_0;
     }
 
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregatorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregatorFactory.java
index 5842d282d4d3..e82e8358b8bf 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregatorFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregatorFactory.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.ml.aggs.frequentitemsets;
 
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
@@ -15,10 +16,12 @@
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.CardinalityUpperBound;
+import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.aggregations.support.MultiValuesSourceFieldConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.xpack.ml.aggs.frequentitemsets.mr.AbstractItemSetMapReducer;
 import org.elasticsearch.xpack.ml.aggs.frequentitemsets.mr.InternalItemSetMapReduceAggregation;
 import org.elasticsearch.xpack.ml.aggs.frequentitemsets.mr.ItemSetMapReduceAggregator;
 
@@ -27,6 +30,8 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.core.Tuple.tuple;
+
 /**
  * Factory for frequent items aggregation
  *
@@ -60,7 +65,8 @@ public class FrequentItemSetsAggregatorFactory extends AggregatorFactory {
     private final double minimumSupport;
     private final int minimumSetSize;
     private final int size;
-    private final QueryBuilder filter;
+    private final QueryBuilder documentFilter;
+    private final String executionHint;
 
     public FrequentItemSetsAggregatorFactory(
         String name,
@@ -72,36 +78,54 @@ public FrequentItemSetsAggregatorFactory(
         double minimumSupport,
         int minimumSetSize,
         int size,
-        QueryBuilder filter
+        QueryBuilder documentFilter,
+        String executionHint
     ) throws IOException {
         super(name, context, parent, subFactoriesBuilder, metadata);
         this.fields = fields;
         this.minimumSupport = minimumSupport;
         this.minimumSetSize = minimumSetSize;
         this.size = size;
-        this.filter = filter;
+        this.documentFilter = documentFilter;
+        this.executionHint = executionHint;
     }
 
     @Override
     protected Aggregator createInternal(Aggregator parent, CardinalityUpperBound cardinality, Map<String, Object> metadata)
         throws IOException {
 
-        List<ValuesSourceConfig> configs = new ArrayList<>(fields.size());
+        List<Tuple<ValuesSourceConfig, IncludeExclude>> configsAndFilters = new ArrayList<>(fields.size());
         for (MultiValuesSourceFieldConfig field : fields) {
-            configs.add(
-                ValuesSourceConfig.resolve(
-                    context,
-                    field.getUserValueTypeHint(),
-                    field.getFieldName(),
-                    field.getScript(),
-                    field.getMissing(),
-                    field.getTimeZone(),
-                    field.getFormat(),
-                    CoreValuesSourceType.KEYWORD
+            configsAndFilters.add(
+                tuple(
+                    ValuesSourceConfig.resolve(
+                        context,
+                        field.getUserValueTypeHint(),
+                        field.getFieldName(),
+                        field.getScript(),
+                        field.getMissing(),
+                        field.getTimeZone(),
+                        field.getFormat(),
+                        CoreValuesSourceType.KEYWORD
+                    ),
+                    field.getIncludeExclude()
                 )
             );
         }
 
+        EclatMapReducer eclatMapReducer = new EclatMapReducer(
+            FrequentItemSetsAggregationBuilder.NAME,
+            minimumSupport,
+            minimumSetSize,
+            size,
+            context.profiling()
+        );
+        AbstractItemSetMapReducer.OrdinalOptimization ordinalOptimization = eclatMapReducer.getDefaultOrdinalOptimization();
+
+        if ("map".equals(executionHint)) {
+            ordinalOptimization = AbstractItemSetMapReducer.OrdinalOptimization.NO_ORDINALS;
+        }
+
         return new ItemSetMapReduceAggregator<
             HashBasedTransactionStore,
             ImmutableTransactionStore,
@@ -112,9 +136,10 @@ protected Aggregator createInternal(Aggregator parent, CardinalityUpperBound car
                 context,
                 parent,
                 metadata,
-                new EclatMapReducer(FrequentItemSetsAggregationBuilder.NAME, minimumSupport, minimumSetSize, size, context.profiling()),
-                configs,
-                filter
+                eclatMapReducer,
+                configsAndFilters,
+                documentFilter,
+                ordinalOptimization
             ) {
         };
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/ImmutableTransactionStore.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/ImmutableTransactionStore.java
index b0ad1802d179..878d64616032 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/ImmutableTransactionStore.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/ImmutableTransactionStore.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.xpack.ml.aggs.frequentitemsets;
 
 import org.apache.lucene.util.RamUsageEstimator;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.BigArrays;
@@ -82,7 +82,7 @@ public ImmutableTransactionStore(StreamInput in, BigArrays bigArrays) throws IOE
             }
             this.totalTransactionCount = in.readVLong();
 
-            if (in.getVersion().onOrAfter(Version.V_8_6_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_6_0)) {
                 this.filteredTransactionCount = in.readVLong();
             } else {
                 this.filteredTransactionCount = 0;
@@ -158,7 +158,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeVLong(transactionCounts.get(i));
         }
         out.writeVLong(totalTransactionCount);
-        if (out.getVersion().onOrAfter(Version.V_8_6_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_6_0)) {
             out.writeVLong(filteredTransactionCount);
         }
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/ItemSetBitSet.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/ItemSetBitSet.java
index a2006550011b..ed1e26a37cbb 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/ItemSetBitSet.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/ItemSetBitSet.java
@@ -25,6 +25,13 @@
  */
 class ItemSetBitSet implements Cloneable {
 
+    public enum SetRelation {
+        DISJOINT_OR_INTERSECT, // we don't care if disjoint or intersect
+        SUB_SET,
+        SUPER_SET,
+        EQUAL
+    }
+
     // taken from {@code BitSet}
     private static final int ADDRESS_BITS_PER_WORD = 6;
     private static final int BITS_PER_WORD = 1 << ADDRESS_BITS_PER_WORD;
@@ -110,12 +117,44 @@ boolean isSubset(ItemSetBitSet set) {
             return false;
         }
 
-        for (int i = wordsInUse - 1; i >= 0; i--)
-            if ((words[i] & set.words[i]) != words[i]) return false;
+        for (int i = wordsInUse - 1; i >= 0; i--) {
+            if ((words[i] & set.words[i]) != words[i]) {
+                return false;
+            }
+        }
 
         return true;
     }
 
+    public SetRelation setRelation(ItemSetBitSet set) {
+        // this method is performance critical, change carefully
+
+        if (wordsInUse > set.wordsInUse || cardinality > set.cardinality) {
+            for (int i = set.wordsInUse - 1; i >= 0; i--) {
+                if ((set.words[i] & words[i]) != set.words[i]) {
+                    return SetRelation.DISJOINT_OR_INTERSECT;
+                }
+            }
+            return SetRelation.SUPER_SET;
+        } else if (wordsInUse < set.wordsInUse || cardinality < set.cardinality) {
+            for (int i = wordsInUse - 1; i >= 0; i--) {
+                if ((words[i] & set.words[i]) != words[i]) {
+                    return SetRelation.DISJOINT_OR_INTERSECT;
+                }
+            }
+            return SetRelation.SUB_SET;
+        }
+
+        // both bitsets have the same wordsInUse and cardinality, so they can be either equal or not
+        for (int i = wordsInUse - 1; i >= 0; i--) {
+            if ((words[i] & set.words[i]) != words[i]) {
+                return SetRelation.DISJOINT_OR_INTERSECT;
+            }
+        }
+
+        return SetRelation.EQUAL;
+    }
+
     int nextSetBit(int fromIndex) {
         if (fromIndex < 0) throw new IndexOutOfBoundsException("fromIndex < 0: " + fromIndex);
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/TransactionStore.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/TransactionStore.java
index a7054c0fb83d..4c6aca009165 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/TransactionStore.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/TransactionStore.java
@@ -194,6 +194,10 @@ static Comparator<Long> compareItems(final LongArray counts) {
         this.bigArrays = bigArrays;
     }
 
+    BigArrays getBigArrays() {
+        return bigArrays;
+    }
+
     /**
      * Get the total number of items
      *
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/AbstractItemSetMapReducer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/AbstractItemSetMapReducer.java
index 3566d008502b..9b56480f0889 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/AbstractItemSetMapReducer.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/AbstractItemSetMapReducer.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.search.aggregations.AggregationReduceContext;
 import org.elasticsearch.search.aggregations.InternalAggregation;
@@ -57,6 +58,21 @@ public abstract class AbstractItemSetMapReducer<
     ReduceContext extends Closeable,
     Result extends ToXContent & Writeable> implements NamedWriteable {
 
+    /**
+     * Describes how ordinals are used to optimize.
+     */
+    public enum OrdinalOptimization {
+        // don't use ordinals
+        NO_ORDINALS,
+        // use per shard global ordinals
+        GLOBAL_ORDINALS
+    }
+
+    /**
+     * Interface to lookup the real object from an ordinal
+     */
+    public interface OrdinalLookupFunction extends CheckedFunction<Object, Object, IOException> {}
+
     private final String aggregationName;
     private final String mapReducerName;
 
@@ -65,6 +81,15 @@ protected AbstractItemSetMapReducer(String aggregationName, String mapReducerNam
         this.mapReducerName = mapReducerName;
     }
 
+    /**
+     * Define the ordinal optimization to use per default, default NO_ORDINALS.
+     *
+     * @return ordinal optimization to be used
+     */
+    protected OrdinalOptimization getDefaultOrdinalOptimization() {
+        return OrdinalOptimization.NO_ORDINALS;
+    }
+
     /**
      * Definition of code to execute before the mapper processes any input.
      *
@@ -92,8 +117,10 @@ protected MapContext mapFiltered(MapContext mapContext) {
     /**
      * Definition of code to execute(optional) after the mapper processed all input.
      *
+     * @param mapContext context object returned from map
+     * @param ordinalLookup an ordinal lookup function to remap ordinals to values
      */
-    protected abstract MapFinalContext mapFinalize(MapContext mapContext);
+    protected abstract MapFinalContext mapFinalize(MapContext mapContext, List<OrdinalLookupFunction> ordinalLookup) throws IOException;
 
     /**
      * Definition of code to execute before the reducer processes any input.
@@ -134,8 +161,8 @@ protected abstract MapFinalContext combine(
     /**
      * Definition of code to execute after the reducer processed all input.
      *
-     * @param reduceContext the result object returned from doReduce
-     * @param fields list of fields from the input
+     * @param reduceContext      the result object returned from doReduce
+     * @param fields             list of fields from the input
      * @param isCanceledSupplier supplier to check whether the request has been canceled
      * @throws IOException
      */
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/InternalItemSetMapReduceAggregation.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/InternalItemSetMapReduceAggregation.java
index 61ef26834f86..a4e8d88a299a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/InternalItemSetMapReduceAggregation.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/InternalItemSetMapReduceAggregation.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.search.aggregations.support.SamplingContext;
 import org.elasticsearch.search.profile.SearchProfileResults;
 import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.ToXContent.DelegatingMapParams;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.ml.aggs.frequentitemsets.mr.ItemSetMapReduceValueSource.Field;
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/ItemSetMapReduceAggregator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/ItemSetMapReduceAggregator.java
index d54a910c797d..7afe6265f61d 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/ItemSetMapReduceAggregator.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/ItemSetMapReduceAggregator.java
@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.ml.aggs.frequentitemsets.mr;
 
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Weight;
@@ -17,6 +19,7 @@
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.LongObjectPagedHashMap;
 import org.elasticsearch.core.Releasables;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.aggregations.AggregationExecutionContext;
 import org.elasticsearch.search.aggregations.Aggregator;
@@ -26,6 +29,7 @@
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
+import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
@@ -46,13 +50,14 @@ public abstract class ItemSetMapReduceAggregator<
     ReduceContext extends Closeable,
     Result extends ToXContent & Writeable> extends AggregatorBase {
 
-    private final List<ItemSetMapReduceValueSource> extractors;
-    private final Weight weightFilter;
+    private final List<ItemSetMapReduceValueSource> valueSources;
+    private final Weight weightDocumentFilter;
     private final List<Field> fields;
     private final AbstractItemSetMapReducer<MapContext, MapFinalContext, ReduceContext, Result> mapReducer;
     private final BigArrays bigArraysForMapReduce;
     private final LongObjectPagedHashMap<Object> mapReduceContextByBucketOrdinal;
     private final boolean profiling;
+    private final boolean rewriteBasedOnOrdinals;
     private final DelegatingCircuitBreakerService breakerService;
 
     protected ItemSetMapReduceAggregator(
@@ -62,29 +67,37 @@ protected ItemSetMapReduceAggregator(
         Aggregator parent,
         Map<String, Object> metadata,
         AbstractItemSetMapReducer<MapContext, MapFinalContext, ReduceContext, Result> mapReducer,
-        List<ValuesSourceConfig> configs,
-        QueryBuilder filter
+        List<Tuple<ValuesSourceConfig, IncludeExclude>> configsAndValueFilters,
+        QueryBuilder documentFilter,
+        AbstractItemSetMapReducer.OrdinalOptimization ordinalOptimization
     ) throws IOException {
         super(name, AggregatorFactories.EMPTY, context, parent, CardinalityUpperBound.NONE, metadata);
 
-        List<ItemSetMapReduceValueSource> extractors = new ArrayList<>();
+        List<ItemSetMapReduceValueSource> valueSources = new ArrayList<>();
         List<Field> fields = new ArrayList<>();
         IndexSearcher contextSearcher = context.searcher();
+        LeafReaderContext ctx = getLeafReaderForOrdinals(context);
 
         int id = 0;
-        this.weightFilter = filter != null
-            ? contextSearcher.createWeight(contextSearcher.rewrite(context.buildQuery(filter)), ScoreMode.COMPLETE_NO_SCORES, 1f)
+        this.weightDocumentFilter = documentFilter != null
+            ? contextSearcher.createWeight(contextSearcher.rewrite(context.buildQuery(documentFilter)), ScoreMode.COMPLETE_NO_SCORES, 1f)
             : null;
 
-        for (ValuesSourceConfig c : configs) {
-            ItemSetMapReduceValueSource e = context.getValuesSourceRegistry().getAggregator(registryKey, c).build(c, id++);
+        boolean rewriteBasedOnOrdinals = false;
+
+        for (var c : configsAndValueFilters) {
+            ItemSetMapReduceValueSource e = context.getValuesSourceRegistry()
+                .getAggregator(registryKey, c.v1())
+                .build(c.v1(), id++, c.v2(), ordinalOptimization, ctx);
             if (e.getField().getName() != null) {
                 fields.add(e.getField());
-                extractors.add(e);
+                valueSources.add(e);
             }
+            rewriteBasedOnOrdinals |= e.usesOrdinals();
         }
 
-        this.extractors = Collections.unmodifiableList(extractors);
+        this.rewriteBasedOnOrdinals = rewriteBasedOnOrdinals;
+        this.valueSources = Collections.unmodifiableList(valueSources);
         this.fields = Collections.unmodifiableList(fields);
         this.mapReducer = mapReducer;
         this.profiling = context.profiling();
@@ -115,21 +128,26 @@ public final InternalAggregation[] buildAggregations(long[] owningBucketOrds) th
     @Override
     protected LeafBucketCollector getLeafCollector(AggregationExecutionContext ctx, LeafBucketCollector sub) throws IOException {
 
-        final Bits bits = weightFilter != null
+        final Bits bits = weightDocumentFilter != null
             ? Lucene.asSequentialAccessBits(
                 ctx.getLeafReaderContext().reader().maxDoc(),
-                weightFilter.scorerSupplier(ctx.getLeafReaderContext())
+                weightDocumentFilter.scorerSupplier(ctx.getLeafReaderContext())
             )
             : null;
 
+        List<ItemSetMapReduceValueSource.ValueCollector> valueCollectors = new ArrayList<>(valueSources.size());
+        for (ItemSetMapReduceValueSource valueSource : valueSources) {
+            valueCollectors.add(valueSource.getValueCollector(ctx.getLeafReaderContext()));
+        }
+
         return new LeafBucketCollectorBase(sub, null) {
             @Override
             public void collect(int doc, long owningBucketOrd) throws IOException {
                 SetOnce<IOException> firstException = new SetOnce<>();
                 if (bits == null || bits.get(doc)) {
-                    mapReducer.map(extractors.stream().map(extractor -> {
+                    mapReducer.map(valueCollectors.stream().map(c -> {
                         try {
-                            return extractor.collect(ctx.getLeafReaderContext(), doc);
+                            return c.collect(doc);
                         } catch (IOException e) {
                             firstException.trySet(e);
                             // ignored in AbstractMapReducer
@@ -148,10 +166,20 @@ public void collect(int doc, long owningBucketOrd) throws IOException {
     }
 
     @Override
-    public void doPostCollection() {
+    public void doPostCollection() throws IOException {
+        List<AbstractItemSetMapReducer.OrdinalLookupFunction> ordinalLookupFunctions = null;
+
+        // only rewrite ordinals back to real values when required
+        if (rewriteBasedOnOrdinals) {
+            ordinalLookupFunctions = new ArrayList<>(valueSources.size());
+            for (ItemSetMapReduceValueSource valueSource : valueSources) {
+                ordinalLookupFunctions.add(valueSource::mapOrdinal);
+            }
+        }
+
         for (long ordIdx = 0; ordIdx < mapReduceContextByBucketOrdinal.size(); ordIdx++) {
             MapContext context = getMapReduceContext(ordIdx);
-            mapReduceContextByBucketOrdinal.put(ordIdx, mapReducer.mapFinalize(context));
+            mapReduceContextByBucketOrdinal.put(ordIdx, mapReducer.mapFinalize(context, ordinalLookupFunctions));
         }
     }
 
@@ -192,4 +220,8 @@ private InternalAggregation buildAggregation(long owningBucketOrdinal) throws IO
         return new InternalItemSetMapReduceAggregation<>(name, metadata(), mapReducer, context, null, fields, profiling);
     }
 
+    private static LeafReaderContext getLeafReaderForOrdinals(AggregationContext context) {
+        IndexReader reader = context.searcher().getIndexReader();
+        return reader.leaves().get(0);
+    }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/ItemSetMapReduceValueSource.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/ItemSetMapReduceValueSource.java
index e298388514e7..24e8da132ca4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/ItemSetMapReduceValueSource.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/ItemSetMapReduceValueSource.java
@@ -7,9 +7,13 @@
 
 package org.elasticsearch.xpack.ml.aggs.frequentitemsets.mr;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LongBitSet;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -17,6 +21,7 @@
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
@@ -33,9 +38,25 @@
  */
 public abstract class ItemSetMapReduceValueSource {
 
+    /**
+     * Interface to hook value collection into the {@link org.elasticsearch.search.aggregations.support.ValuesSourceRegistry}
+     */
     @FunctionalInterface
     public interface ValueSourceSupplier {
-        ItemSetMapReduceValueSource build(ValuesSourceConfig config, int id);
+        ItemSetMapReduceValueSource build(
+            ValuesSourceConfig config,
+            int id,
+            IncludeExclude includeExclude,
+            AbstractItemSetMapReducer.OrdinalOptimization ordinalOptimization,
+            LeafReaderContext ctx
+        ) throws IOException;
+    }
+
+    /**
+     * Internal interface for collecting values
+     */
+    interface ValueCollector {
+        Tuple<Field, List<Object>> collect(int doc) throws IOException;
     }
 
     enum ValueFormatter {
@@ -120,15 +141,15 @@ public int hashCode() {
             return Objects.hash(id, valueFormatter, name, format);
         }
 
-    };
+    }
 
     private final Field field;
+    private static final Logger logger = LogManager.getLogger(ItemSetMapReduceValueSource.class);
 
-    abstract Tuple<Field, List<Object>> collect(LeafReaderContext ctx, int doc) throws IOException;
+    abstract ValueCollector getValueCollector(LeafReaderContext ctx) throws IOException;
 
     ItemSetMapReduceValueSource(ValuesSourceConfig config, int id, ValueFormatter valueFormatter) {
         String fieldName = config.fieldContext() != null ? config.fieldContext().field() : null;
-
         if (Strings.isNullOrEmpty(fieldName)) {
             throw new IllegalArgumentException("scripts are not supported");
         }
@@ -140,53 +161,250 @@ Field getField() {
         return field;
     }
 
+    boolean usesOrdinals() {
+        return false;
+    }
+
+    Object mapOrdinal(Object ord) throws IOException {
+        return ord;
+    }
+
     public static class KeywordValueSource extends ItemSetMapReduceValueSource {
-        private final ValuesSource.Bytes source;
 
-        public KeywordValueSource(ValuesSourceConfig config, int id) {
+        interface ExecutionStrategy {
+            ValueCollector getValueCollector(LeafReaderContext ctx) throws IOException;
+
+            Object mapOrdinal(Object ord) throws IOException;
+
+            boolean usesOrdinals();
+        }
+
+        static class GlobalOrdinalsStrategy implements ExecutionStrategy {
+
+            private final Field field;
+            private final Bytes.WithOrdinals source;
+            private final SortedSetDocValues docValues;
+            private final LongBitSet bitSetFilter;
+
+            GlobalOrdinalsStrategy(
+                Field field,
+                Bytes.WithOrdinals source,
+                IncludeExclude.OrdinalsFilter globalOrdinalsFilter,
+                LeafReaderContext ctx
+            ) throws IOException {
+                this.field = field;
+                this.source = source;
+
+                bitSetFilter = globalOrdinalsFilter != null
+                    ? globalOrdinalsFilter.acceptedGlobalOrdinals(source.globalOrdinalsValues(ctx))
+                    : null;
+
+                this.docValues = source.globalOrdinalsValues(ctx);
+            }
+
+            @Override
+            public ValueCollector getValueCollector(LeafReaderContext ctx) throws IOException {
+                final SortedSetDocValues values = source.globalOrdinalsValues(ctx);
+                final Tuple<Field, List<Object>> empty = new Tuple<>(field, Collections.emptyList());
+
+                return doc -> {
+                    if (values.advanceExact(doc)) {
+                        int valuesCount = values.docValueCount();
+
+                        if (valuesCount == 1) {
+                            long v = values.nextOrd();
+                            assert v >= 0;
+                            if (bitSetFilter == null || bitSetFilter.get(v)) {
+                                return new Tuple<>(field, Collections.singletonList(v));
+                            }
+                            return empty;
+                        }
+
+                        if (valuesCount == 0) {
+                            return empty;
+                        }
+
+                        List<Object> objects = new ArrayList<>(valuesCount);
+
+                        for (int i = 0; i < valuesCount; ++i) {
+                            long v = values.nextOrd();
+                            assert v >= 0;
+                            if (bitSetFilter == null || bitSetFilter.get(v)) {
+                                objects.add(v);
+                            }
+                        }
+                        return new Tuple<>(field, objects);
+                    }
+                    return empty;
+                };
+            }
+
+            @Override
+            public boolean usesOrdinals() {
+                return true;
+            }
+
+            @Override
+            public Object mapOrdinal(Object ord) throws IOException {
+                return docValues.lookupOrd((Long) ord);
+            }
+        }
+
+        static class MapStrategy implements ExecutionStrategy {
+
+            private final Field field;
+            private final Bytes source;
+            private final IncludeExclude.StringFilter stringFilter;
+
+            MapStrategy(Field field, Bytes source, IncludeExclude.StringFilter stringFilter) {
+                this.field = field;
+                this.source = source;
+                this.stringFilter = stringFilter;
+            }
+
+            @Override
+            public ValueCollector getValueCollector(LeafReaderContext ctx) throws IOException {
+                final SortedBinaryDocValues values = source.bytesValues(ctx);
+                final Tuple<Field, List<Object>> empty = new Tuple<>(field, Collections.emptyList());
+
+                return doc -> {
+                    if (values.advanceExact(doc)) {
+                        int valuesCount = values.docValueCount();
+
+                        if (valuesCount == 1) {
+                            BytesRef v = values.nextValue();
+                            if (stringFilter == null || stringFilter.accept(v)) {
+                                return new Tuple<>(field, Collections.singletonList(BytesRef.deepCopyOf(v)));
+                            }
+                            return empty;
+                        }
+
+                        if (valuesCount == 0) {
+                            return empty;
+                        }
+
+                        List<Object> objects = new ArrayList<>(valuesCount);
+
+                        for (int i = 0; i < valuesCount; ++i) {
+                            BytesRef v = values.nextValue();
+                            if (stringFilter == null || stringFilter.accept(v)) {
+                                objects.add(BytesRef.deepCopyOf(v));
+                            }
+                        }
+                        return new Tuple<>(field, objects);
+                    }
+                    return empty;
+                };
+            }
+
+            @Override
+            public boolean usesOrdinals() {
+                return false;
+            }
+
+            @Override
+            public Object mapOrdinal(Object ord) {
+                return ord;
+            }
+        }
+
+        private final ExecutionStrategy executionStrategy;
+
+        public KeywordValueSource(
+            ValuesSourceConfig config,
+            int id,
+            IncludeExclude includeExclude,
+            AbstractItemSetMapReducer.OrdinalOptimization ordinalOptimization,
+            LeafReaderContext ctx
+        ) throws IOException {
             super(config, id, ValueFormatter.BYTES_REF);
-            this.source = (Bytes) config.getValuesSource();
+
+            if (AbstractItemSetMapReducer.OrdinalOptimization.GLOBAL_ORDINALS.equals(ordinalOptimization)
+                && config.getValuesSource() instanceof Bytes.WithOrdinals
+                && ((Bytes.WithOrdinals) config.getValuesSource()).supportsGlobalOrdinalsMapping()) {
+                logger.debug("Use ordinals for field [{}]", config.fieldContext().field());
+
+                this.executionStrategy = new GlobalOrdinalsStrategy(
+                    getField(),
+                    (Bytes.WithOrdinals) config.getValuesSource(),
+                    includeExclude == null ? null : includeExclude.convertToOrdinalsFilter(config.format()),
+                    ctx
+                );
+            } else {
+                this.executionStrategy = new MapStrategy(
+                    getField(),
+                    (Bytes) config.getValuesSource(),
+                    includeExclude == null ? null : includeExclude.convertToStringFilter(config.format())
+                );
+            }
         }
 
         @Override
-        public Tuple<Field, List<Object>> collect(LeafReaderContext ctx, int doc) throws IOException {
-            SortedBinaryDocValues values = source.bytesValues(ctx);
+        ValueCollector getValueCollector(LeafReaderContext ctx) throws IOException {
+            return executionStrategy.getValueCollector(ctx);
+        }
 
-            if (values.advanceExact(doc)) {
-                int valuesCount = values.docValueCount();
-                List<Object> objects = new ArrayList<>(valuesCount);
+        @Override
+        public boolean usesOrdinals() {
+            return executionStrategy.usesOrdinals();
+        }
 
-                for (int i = 0; i < valuesCount; ++i) {
-                    objects.add(BytesRef.deepCopyOf(values.nextValue()));
-                }
-                return new Tuple<>(getField(), objects);
-            }
-            return new Tuple<>(getField(), Collections.emptyList());
+        @Override
+        Object mapOrdinal(Object ord) throws IOException {
+            return executionStrategy.mapOrdinal(ord);
         }
     }
 
     public static class NumericValueSource extends ItemSetMapReduceValueSource {
         private final ValuesSource.Numeric source;
-
-        public NumericValueSource(ValuesSourceConfig config, int id) {
+        private final IncludeExclude.LongFilter longFilter;
+
+        public NumericValueSource(
+            ValuesSourceConfig config,
+            int id,
+            IncludeExclude includeExclude,
+            AbstractItemSetMapReducer.OrdinalOptimization unusedOrdinalOptimization,
+            LeafReaderContext unusedCtx
+        ) {
             super(config, id, ValueFormatter.LONG);
             this.source = (Numeric) config.getValuesSource();
+            this.longFilter = includeExclude == null ? null : includeExclude.convertToLongFilter(config.format());
         }
 
         @Override
-        public Tuple<Field, List<Object>> collect(LeafReaderContext ctx, int doc) throws IOException {
-            SortedNumericDocValues values = source.longValues(ctx);
-
-            if (values.advanceExact(doc)) {
-                int valuesCount = values.docValueCount();
-                List<Object> objects = new ArrayList<>(valuesCount);
-
-                for (int i = 0; i < valuesCount; ++i) {
-                    objects.add(values.nextValue());
+        ValueCollector getValueCollector(LeafReaderContext ctx) throws IOException {
+            final SortedNumericDocValues values = source.longValues(ctx);
+            final Field field = getField();
+            final Tuple<Field, List<Object>> empty = new Tuple<>(field, Collections.emptyList());
+
+            return doc -> {
+                if (values.advanceExact(doc)) {
+                    int valuesCount = values.docValueCount();
+
+                    if (valuesCount == 1) {
+                        long v = values.nextValue();
+                        if (longFilter == null || longFilter.accept(v)) {
+                            return new Tuple<>(getField(), Collections.singletonList(v));
+                        }
+                        return empty;
+                    }
+
+                    if (valuesCount == 0) {
+                        return empty;
+                    }
+
+                    List<Object> objects = new ArrayList<>(valuesCount);
+
+                    for (int i = 0; i < valuesCount; ++i) {
+                        long v = values.nextValue();
+                        if (longFilter == null || longFilter.accept(v)) {
+                            objects.add(v);
+                        }
+                    }
+                    return new Tuple<>(field, objects);
                 }
-                return new Tuple<>(getField(), objects);
-            }
-            return new Tuple<>(getField(), Collections.emptyList());
+                return empty;
+            };
         }
 
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/inference/InferencePipelineAggregationBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/inference/InferencePipelineAggregationBuilder.java
index 6b485fd46575..ccff6ae21e0b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/inference/InferencePipelineAggregationBuilder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/inference/InferencePipelineAggregationBuilder.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.xpack.ml.aggs.inference;
 
 import org.apache.lucene.util.SetOnce;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.Strings;
@@ -379,7 +379,7 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_9_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_9_0;
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/kstest/BucketCountKSTestAggregationBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/kstest/BucketCountKSTestAggregationBuilder.java
index 17e8e769082e..147d7831af5d 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/kstest/BucketCountKSTestAggregationBuilder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/aggs/kstest/BucketCountKSTestAggregationBuilder.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.ml.aggs.kstest;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
@@ -184,7 +184,7 @@ public int hashCode() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_14_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_14_0;
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/annotations/AnnotationPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/annotations/AnnotationPersister.java
index 4d45c7a8fa73..18d3c4701243 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/annotations/AnnotationPersister.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/annotations/AnnotationPersister.java
@@ -66,22 +66,22 @@ public Tuple<String, Annotation> persistAnnotation(@Nullable String annotationId
     }
 
     public Builder bulkPersisterBuilder(String jobId) {
-        return new Builder(jobId);
+        return new Builder(jobId, () -> true);
+    }
+
+    public Builder bulkPersisterBuilder(String jobId, Supplier<Boolean> shouldRetry) {
+        return new Builder(jobId, shouldRetry);
     }
 
     public class Builder {
 
         private final String jobId;
         private BulkRequest bulkRequest = new BulkRequest(AnnotationIndex.WRITE_ALIAS_NAME);
-        private Supplier<Boolean> shouldRetry = () -> true;
+        private final Supplier<Boolean> shouldRetry;
 
-        private Builder(String jobId) {
+        private Builder(String jobId, Supplier<Boolean> shouldRetry) {
             this.jobId = Objects.requireNonNull(jobId);
-        }
-
-        public Builder shouldRetry(Supplier<Boolean> shouldRetry) {
             this.shouldRetry = Objects.requireNonNull(shouldRetry);
-            return this;
         }
 
         public Builder persistAnnotation(Annotation annotation) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlScalingReason.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlScalingReason.java
index c5fb0d22e439..d97b5ff6ba69 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlScalingReason.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlScalingReason.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.ml.autoscaling;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
@@ -49,7 +49,7 @@ public MlScalingReason(StreamInput in) throws IOException {
         this.waitingAnalyticsJobs = in.readStringList();
         this.waitingAnomalyJobs = in.readStringList();
         this.waitingSnapshotUpgrades = in.readStringList();
-        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             this.waitingModels = in.readStringList();
         } else {
             this.waitingModels = List.of();
@@ -136,7 +136,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeStringCollection(this.waitingAnalyticsJobs);
         out.writeStringCollection(this.waitingAnomalyJobs);
         out.writeStringCollection(this.waitingSnapshotUpgrades);
-        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_0_0)) {
             out.writeStringCollection(this.waitingModels);
         }
         this.passedConfiguration.writeTo(out);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java
index 0e70fb689563..fc5c8b5b35fe 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java
@@ -137,6 +137,10 @@ public Integer getMaxEmptySearches() {
         return maxEmptySearches;
     }
 
+    public long numberOfSearchesIn24Hours() {
+        return (60_000 * 60 * 24) / frequencyMs;
+    }
+
     public void finishReportingTimingStats() {
         timingStatsReporter.finishReporting();
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java
index 4c6b9a78e306..7fcf7cf98024 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java
@@ -59,6 +59,7 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import static java.util.function.Predicate.not;
 import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
 import static org.elasticsearch.xpack.ml.utils.SecondaryAuthorizationUtils.useSecondaryAuthIfAvailable;
@@ -106,7 +107,12 @@ public void putDatafeed(
     ) {
         if (XPackSettings.SECURITY_ENABLED.get(settings)) {
             useSecondaryAuthIfAvailable(securityContext, () -> {
-                final String[] indices = request.getDatafeed().getIndices().toArray(new String[0]);
+                // TODO: Remove this filter once https://github.com/elastic/elasticsearch/issues/67798 is fixed.
+                final String[] indices = request.getDatafeed()
+                    .getIndices()
+                    .stream()
+                    .filter(not(RemoteClusterLicenseChecker::isRemoteIndex))
+                    .toArray(String[]::new);
 
                 final String username = securityContext.getUser().principal();
                 final HasPrivilegesRequest privRequest = new HasPrivilegesRequest();
@@ -128,8 +134,12 @@ public void putDatafeed(
                     } else {
                         indicesPrivilegesBuilder.privileges(SearchAction.NAME, RollupSearchAction.NAME);
                     }
-                    privRequest.indexPrivileges(indicesPrivilegesBuilder.build());
-                    client.execute(HasPrivilegesAction.INSTANCE, privRequest, privResponseListener);
+                    if (indices.length == 0) {
+                        privResponseListener.onResponse(new HasPrivilegesResponse());
+                    } else {
+                        privRequest.indexPrivileges(indicesPrivilegesBuilder.build());
+                        client.execute(HasPrivilegesAction.INSTANCE, privRequest, privResponseListener);
+                    }
                 }, e -> {
                     if (ExceptionsHelper.unwrapCause(e) instanceof IndexNotFoundException) {
                         indicesPrivilegesBuilder.privileges(SearchAction.NAME);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedRunner.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedRunner.java
index e5bc6fde7847..5341b44b013c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedRunner.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedRunner.java
@@ -96,7 +96,13 @@ public DatafeedRunner(
     public void run(TransportStartDatafeedAction.DatafeedTask task, Consumer<Exception> finishHandler) {
         ActionListener<DatafeedJob> datafeedJobHandler = ActionListener.wrap(datafeedJob -> {
             String jobId = datafeedJob.getJobId();
-            Holder holder = new Holder(task, task.getDatafeedId(), datafeedJob, new ProblemTracker(auditor, jobId), finishHandler);
+            Holder holder = new Holder(
+                task,
+                task.getDatafeedId(),
+                datafeedJob,
+                new ProblemTracker(auditor, jobId, datafeedJob.numberOfSearchesIn24Hours()),
+                finishHandler
+            );
             StoppedOrIsolated stoppedOrIsolated = task.executeIfNotStoppedOrIsolated(
                 () -> runningDatafeedsOnThisNode.put(task.getAllocationId(), holder)
             );
@@ -313,15 +319,15 @@ protected void doRun() {
                     } catch (DatafeedJob.EmptyDataCountException e) {
                         int emptyDataCount = holder.problemTracker.reportEmptyDataCount();
                         if (e.haveEverSeenData == false && holder.shouldStopAfterEmptyData(emptyDataCount)) {
-                            logger.warn(
-                                "Datafeed for ["
-                                    + jobId
-                                    + "] has seen no data in ["
-                                    + emptyDataCount
-                                    + "] attempts, and never seen any data previously, so stopping..."
-                            );
+                            String noDataMessage = "Datafeed for ["
+                                + jobId
+                                + "] has seen no data in ["
+                                + emptyDataCount
+                                + "] attempts, and never seen any data previously, so stopping...";
+                            logger.warn(noDataMessage);
+
                             // In this case we auto-close the job, as though a lookback-only datafeed stopped
-                            holder.stop("no_data", TimeValue.timeValueSeconds(20), e, true);
+                            holder.stop("no_data", TimeValue.timeValueSeconds(20), e, true, noDataMessage);
                             return;
                         }
                         nextDelayInMsSinceEpoch = e.nextDelayInMsSinceEpoch;
@@ -432,10 +438,10 @@ boolean isIsolated() {
         }
 
         public void stop(String source, TimeValue timeout, Exception e) {
-            stop(source, timeout, e, defaultAutoCloseJob);
+            stop(source, timeout, e, defaultAutoCloseJob, null);
         }
 
-        public void stop(String source, TimeValue timeout, Exception e, boolean autoCloseJob) {
+        public void stop(String source, TimeValue timeout, Exception e, boolean autoCloseJob, String stoppedReason) {
             if (isNodeShuttingDown) {
                 return;
             }
@@ -466,10 +472,16 @@ public void stop(String source, TimeValue timeout, Exception e, boolean autoClos
                     if (cancellable != null) {
                         cancellable.cancel();
                     }
-                    auditor.info(
-                        datafeedJob.getJobId(),
-                        Messages.getMessage(isIsolated() ? Messages.JOB_AUDIT_DATAFEED_ISOLATED : Messages.JOB_AUDIT_DATAFEED_STOPPED)
-                    );
+                    String auditMessage;
+                    if (isIsolated()) {
+                        auditMessage = Messages.getMessage(Messages.JOB_AUDIT_DATAFEED_ISOLATED);
+                    } else {
+                        auditMessage = stoppedReason == null
+                            ? Messages.getMessage(Messages.JOB_AUDIT_DATAFEED_STOPPED)
+                            : Messages.getMessage(Messages.JOB_AUDIT_DATAFEED_STOPPED_WITH_REASON, stoppedReason);
+                    }
+                    auditor.info(datafeedJob.getJobId(), auditMessage);
+
                     datafeedJob.finishReportingTimingStats();
                     finishHandler.accept(e);
                     logger.info(
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedTimingStatsReporter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedTimingStatsReporter.java
index 93ba370e93b8..5fa73fab4664 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedTimingStatsReporter.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedTimingStatsReporter.java
@@ -8,38 +8,47 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
 
 import java.util.Objects;
+import java.util.concurrent.CountDownLatch;
 
 /**
- * {@link DatafeedTimingStatsReporter} class handles the logic of persisting {@link DatafeedTimingStats} if they changed significantly
- * since the last time they were persisted.
- *
+ * {@link DatafeedTimingStatsReporter} class handles the logic of persisting {@link DatafeedTimingStats} for a single job
+ * if they changed significantly since the last time they were persisted.
+ * <p>
  * This class is not thread-safe.
  */
 public class DatafeedTimingStatsReporter {
 
-    private static final Logger LOGGER = LogManager.getLogger(DatafeedTimingStatsReporter.class);
+    private static final Logger logger = LogManager.getLogger(DatafeedTimingStatsReporter.class);
 
     /** Interface used for persisting current timing stats to the results index. */
     @FunctionalInterface
     public interface DatafeedTimingStatsPersister {
         /** Does nothing by default. This behavior is useful when creating fake {@link DatafeedTimingStatsReporter} objects. */
-        void persistDatafeedTimingStats(DatafeedTimingStats timingStats, WriteRequest.RefreshPolicy refreshPolicy);
+        void persistDatafeedTimingStats(
+            DatafeedTimingStats timingStats,
+            WriteRequest.RefreshPolicy refreshPolicy,
+            ActionListener<BulkResponse> listener
+        );
     }
 
     /** Persisted timing stats. May be stale. */
-    private DatafeedTimingStats persistedTimingStats;
+    private volatile DatafeedTimingStats persistedTimingStats;
     /** Current timing stats. */
-    private volatile DatafeedTimingStats currentTimingStats;
+    private final DatafeedTimingStats currentTimingStats;
     /** Object used to persist current timing stats. */
     private final DatafeedTimingStatsPersister persister;
     /** Whether or not timing stats will be persisted by the persister object. */
     private volatile boolean allowedPersisting;
+    /** Records whether a persist is currently in progress. */
+    private CountDownLatch persistInProgressLatch;
 
     public DatafeedTimingStatsReporter(DatafeedTimingStats timingStats, DatafeedTimingStatsPersister persister) {
         Objects.requireNonNull(timingStats);
@@ -81,9 +90,11 @@ public void reportDataCounts(DataCounts dataCounts) {
 
     /** Finishes reporting of timing stats. Makes timing stats persisted immediately. */
     public void finishReporting() {
-        // Don't flush if current timing stats are identical to the persisted ones
-        if (currentTimingStats.equals(persistedTimingStats) == false) {
-            flush(WriteRequest.RefreshPolicy.IMMEDIATE);
+        try {
+            flush(WriteRequest.RefreshPolicy.IMMEDIATE, true);
+        } catch (InterruptedException e) {
+            logger.warn("[{}] interrupted while finishing reporting of datafeed timing stats", currentTimingStats.getJobId());
+            Thread.currentThread().interrupt();
         }
     }
 
@@ -94,19 +105,50 @@ public void disallowPersisting() {
 
     private void flushIfDifferSignificantly() {
         if (differSignificantly(currentTimingStats, persistedTimingStats)) {
-            flush(WriteRequest.RefreshPolicy.NONE);
+            try {
+                flush(WriteRequest.RefreshPolicy.NONE, false);
+            } catch (InterruptedException e) {
+                assert false : "This should never happen when flush is called with mustWait set to false";
+                Thread.currentThread().interrupt();
+            }
         }
     }
 
-    private void flush(WriteRequest.RefreshPolicy refreshPolicy) {
-        persistedTimingStats = new DatafeedTimingStats(currentTimingStats);
-        if (allowedPersisting) {
-            try {
-                persister.persistDatafeedTimingStats(persistedTimingStats, refreshPolicy);
-            } catch (Exception ex) {
-                // Since persisting datafeed timing stats is not critical, we just log a warning here.
-                LOGGER.warn(() -> "[" + currentTimingStats.getJobId() + "] failed to report datafeed timing stats", ex);
-            }
+    private synchronized void flush(WriteRequest.RefreshPolicy refreshPolicy, boolean mustWait) throws InterruptedException {
+        String jobId = currentTimingStats.getJobId();
+        if (allowedPersisting && mustWait && persistInProgressLatch != null) {
+            persistInProgressLatch.await();
+            persistInProgressLatch = null;
+        }
+        // Don't persist if:
+        // 1. Persistence is disallowed
+        // 2. There is already a persist in progress
+        // 3. Current timing stats are identical to the persisted ones
+        if (allowedPersisting == false) {
+            logger.trace("[{}] not persisting datafeed timing stats as persistence is disallowed", jobId);
+            return;
+        }
+        if (persistInProgressLatch != null && persistInProgressLatch.getCount() > 0) {
+            logger.trace("[{}] not persisting datafeed timing stats as the previous persist is still in progress", jobId);
+            return;
+        }
+        if (currentTimingStats.equals(persistedTimingStats)) {
+            logger.trace("[{}] not persisting datafeed timing stats as they are identical to latest already persisted", jobId);
+            return;
+        }
+        final CountDownLatch thisPersistLatch = new CountDownLatch(1);
+        final DatafeedTimingStats thisPersistTimingStats = new DatafeedTimingStats(currentTimingStats);
+        persistInProgressLatch = thisPersistLatch;
+        persister.persistDatafeedTimingStats(thisPersistTimingStats, refreshPolicy, ActionListener.wrap(r -> {
+            persistedTimingStats = thisPersistTimingStats;
+            thisPersistLatch.countDown();
+        }, e -> {
+            thisPersistLatch.countDown();
+            // Since persisting datafeed timing stats is not critical, we just log a warning here.
+            logger.warn(() -> "[" + jobId + "] failed to report datafeed timing stats", e);
+        }));
+        if (mustWait) {
+            thisPersistLatch.await();
         }
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/ProblemTracker.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/ProblemTracker.java
index 30873715ad1c..8a5ea60af89e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/ProblemTracker.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/ProblemTracker.java
@@ -35,10 +35,12 @@ class ProblemTracker {
     private volatile boolean hadProblems;
     private volatile String previousProblem;
     private volatile int emptyDataCount;
+    private final long numberOfSearchesInADay;
 
-    ProblemTracker(AnomalyDetectionAuditor auditor, String jobId) {
+    ProblemTracker(AnomalyDetectionAuditor auditor, String jobId, long numberOfSearchesInADay) {
         this.auditor = Objects.requireNonNull(auditor);
         this.jobId = Objects.requireNonNull(jobId);
+        this.numberOfSearchesInADay = Math.max(numberOfSearchesInADay, 1);
     }
 
     /**
@@ -74,10 +76,11 @@ private void reportProblem(String template, String problemMessage) {
 
     /**
      * Updates the tracking of empty data cycles. If the number of consecutive empty data
-     * cycles reaches {@code EMPTY_DATA_WARN_COUNT}, a warning is reported.
+     * cycles reaches {@code EMPTY_DATA_WARN_COUNT} or the 24 hours of empty data counts
+     * have passed a warning is reported.
      */
     public int reportEmptyDataCount() {
-        if (++emptyDataCount == EMPTY_DATA_WARN_COUNT) {
+        if (++emptyDataCount == EMPTY_DATA_WARN_COUNT || (emptyDataCount % numberOfSearchesInADay) == 0) {
             auditor.warning(jobId, Messages.getMessage(Messages.JOB_AUDIT_DATAFEED_NO_DATA));
         }
         return emptyDataCount;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java
index d35db03df6a0..7a485d08d064 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.fieldcaps.FieldCapabilities;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesAction;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
@@ -25,10 +26,16 @@
 import org.elasticsearch.xpack.ml.datafeed.DatafeedTimingStatsReporter;
 import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory;
 
+import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 
 public class ScrollDataExtractorFactory implements DataExtractorFactory {
+
+    // This field type is not supported for scrolling datafeeds.
+    private static final String AGGREGATE_METRIC_DOUBLE = "aggregate_metric_double";
+
     private final Client client;
     private final DatafeedConfig datafeedConfig;
     private final Job job;
@@ -104,6 +111,17 @@ public static void create(
                 );
                 return;
             }
+            Optional<String> optionalAggregatedMetricDouble = findFirstAggregatedMetricDoubleField(fieldCapabilitiesResponse);
+            if (optionalAggregatedMetricDouble.isPresent()) {
+                listener.onFailure(
+                    ExceptionsHelper.badRequestException(
+                        "field [{}] is of type [{}] and cannot be used in a datafeed without aggregations",
+                        optionalAggregatedMetricDouble.get(),
+                        AGGREGATE_METRIC_DOUBLE
+                    )
+                );
+                return;
+            }
             TimeBasedExtractedFields fields = TimeBasedExtractedFields.build(job, datafeed, fieldCapabilitiesResponse);
             listener.onResponse(new ScrollDataExtractorFactory(client, datafeed, job, fields, xContentRegistry, timingStatsReporter));
         }, e -> {
@@ -142,4 +160,16 @@ public static void create(
             return null;
         });
     }
+
+    private static Optional<String> findFirstAggregatedMetricDoubleField(FieldCapabilitiesResponse fieldCapabilitiesResponse) {
+        Map<String, Map<String, FieldCapabilities>> indexTofieldCapsMap = fieldCapabilitiesResponse.get();
+        for (Map.Entry<String, Map<String, FieldCapabilities>> indexToFieldCaps : indexTofieldCapsMap.entrySet()) {
+            for (Map.Entry<String, FieldCapabilities> typeToFieldCaps : indexToFieldCaps.getValue().entrySet()) {
+                if (AGGREGATE_METRIC_DOUBLE.equals(typeToFieldCaps.getKey())) {
+                    return Optional.of(typeToFieldCaps.getValue().getName());
+                }
+            }
+        }
+        return Optional.empty();
+    }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ModelAliasMetadata.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ModelAliasMetadata.java
index daa6a89b6434..755121af95ea 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ModelAliasMetadata.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ModelAliasMetadata.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.ml.inference;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.DiffableUtils;
@@ -16,8 +16,10 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
@@ -26,6 +28,7 @@
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Objects;
 
@@ -88,13 +91,8 @@ public Map<String, ModelAliasEntry> modelAliases() {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(MODEL_ALIASES.getPreferredName());
-        for (Map.Entry<String, ModelAliasEntry> modelAliasEntry : modelAliases.entrySet()) {
-            builder.field(modelAliasEntry.getKey(), modelAliasEntry.getValue());
-        }
-        builder.endObject();
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return ChunkedToXContentHelper.xContentValuesMap(MODEL_ALIASES.getPreferredName(), modelAliases);
     }
 
     @Override
@@ -113,8 +111,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_13_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_13_0;
     }
 
     @Override
@@ -163,8 +161,8 @@ public void writeTo(StreamOutput out) throws IOException {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.V_7_13_0;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.V_7_13_0;
         }
 
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java
index 2d4b73b24baf..da1533486bf7 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java
@@ -809,15 +809,15 @@ static Optional<String> detectReasonIfMlJobsStopped(ClusterChangedEvent event) {
         final PersistentTasksCustomMetadata currentPersistentTasks = event.state().getMetadata().custom(PersistentTasksCustomMetadata.TYPE);
         Set<String> previousMlTaskIds = findMlProcessTaskIds(previousPersistentTasks);
         Set<String> currentMlTaskIds = findMlProcessTaskIds(currentPersistentTasks);
-        previousMlTaskIds.removeAll(currentMlTaskIds);
         Set<String> stoppedTaskTypes = previousMlTaskIds.stream()
+            .filter(id -> currentMlTaskIds.contains(id) == false) // remove the tasks that are still present. Stopped Ids only.
             .map(previousPersistentTasks::getTask)
             .map(PersistentTasksCustomMetadata.PersistentTask::getTaskName)
             .map(MlTasks::prettyPrintTaskName)
             .collect(Collectors.toSet());
-        if (previousMlTaskIds.size() == 1) {
+        if (stoppedTaskTypes.size() == 1) {
             return Optional.of("ML [" + stoppedTaskTypes.iterator().next() + "] job stopped");
-        } else if (previousMlTaskIds.size() > 1) {
+        } else if (stoppedTaskTypes.size() > 1) {
             return Optional.of("ML " + stoppedTaskTypes + " jobs stopped");
         }
         return Optional.empty();
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentMetadata.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentMetadata.java
index 13085e43499c..2951ab85ecba 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentMetadata.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentMetadata.java
@@ -9,7 +9,7 @@
 
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.DiffableUtils;
@@ -19,7 +19,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignment;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
@@ -27,6 +27,7 @@
 import java.io.IOException;
 import java.util.Collections;
 import java.util.EnumSet;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Objects;
@@ -107,9 +108,11 @@ public Map<String, TrainedModelAssignment> modelAssignments() {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.mapContents(modelRoutingEntries);
-        return builder;
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params ignored) {
+        return modelRoutingEntries.entrySet()
+            .stream()
+            .map(entry -> (ToXContent) (builder, params) -> entry.getValue().toXContent(builder.field(entry.getKey()), params))
+            .iterator();
     }
 
     @Override
@@ -128,8 +131,8 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_0_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_0_0;
     }
 
     @Override
@@ -281,8 +284,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.V_8_0_0;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.V_8_0_0;
         }
 
         @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManager.java
index a25222c4c34d..8a85edb0a676 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/deployment/DeploymentManager.java
@@ -64,6 +64,7 @@
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
+import static org.elasticsearch.xpack.ml.MachineLearning.UTILITY_THREAD_POOL_NAME;
 
 public class DeploymentManager {
 
@@ -90,15 +91,11 @@ public DeploymentManager(
         this.xContentRegistry = Objects.requireNonNull(xContentRegistry);
         this.pyTorchProcessFactory = Objects.requireNonNull(pyTorchProcessFactory);
         this.threadPool = Objects.requireNonNull(threadPool);
-        this.executorServiceForDeployment = threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME);
+        this.executorServiceForDeployment = threadPool.executor(UTILITY_THREAD_POOL_NAME);
         this.executorServiceForProcess = threadPool.executor(MachineLearning.NATIVE_INFERENCE_COMMS_THREAD_POOL_NAME);
         this.maxProcesses = maxProcesses;
     }
 
-    public void startDeployment(TrainedModelDeploymentTask task, ActionListener<TrainedModelDeploymentTask> listener) {
-        doStartDeployment(task, listener);
-    }
-
     public Optional<ModelStats> getStats(TrainedModelDeploymentTask task) {
         return Optional.ofNullable(processContextByAllocation.get(task.getId())).map(processContext -> {
             var stats = processContext.getResultProcessor().getResultStats();
@@ -126,19 +123,23 @@ public Optional<ModelStats> getStats(TrainedModelDeploymentTask task) {
 
     // function exposed for testing
     ProcessContext addProcessContext(Long id, ProcessContext processContext) {
-        if (processContextByAllocation.size() >= maxProcesses) {
-            throw ExceptionsHelper.serverError(
-                "[{}] Could not start inference process as the node reached the max number [{}] of processes",
-                processContext.task.getModelId(),
-                maxProcesses
-            );
-        }
         return processContextByAllocation.putIfAbsent(id, processContext);
     }
 
-    private void doStartDeployment(TrainedModelDeploymentTask task, ActionListener<TrainedModelDeploymentTask> finalListener) {
+    public void startDeployment(TrainedModelDeploymentTask task, ActionListener<TrainedModelDeploymentTask> finalListener) {
         logger.info("[{}] Starting model deployment", task.getModelId());
 
+        if (processContextByAllocation.size() >= maxProcesses) {
+            finalListener.onFailure(
+                ExceptionsHelper.serverError(
+                    "[{}] Could not start inference process as the node reached the max number [{}] of processes",
+                    task.getModelId(),
+                    maxProcesses
+                )
+            );
+            return;
+        }
+
         ProcessContext processContext = new ProcessContext(task);
         if (addProcessContext(task.getId(), processContext) != null) {
             finalListener.onFailure(
@@ -147,15 +148,18 @@ private void doStartDeployment(TrainedModelDeploymentTask task, ActionListener<T
             return;
         }
 
-        ActionListener<TrainedModelDeploymentTask> listener = ActionListener.wrap(finalListener::onResponse, failure -> {
-            processContextByAllocation.remove(task.getId());
+        ActionListener<TrainedModelDeploymentTask> failedDeploymentListener = ActionListener.wrap(finalListener::onResponse, failure -> {
+            ProcessContext failedContext = processContextByAllocation.remove(task.getId());
+            if (failedContext != null) {
+                failedContext.stopProcess();
+            }
             finalListener.onFailure(failure);
         });
 
         ActionListener<Boolean> modelLoadedListener = ActionListener.wrap(success -> {
             executorServiceForProcess.execute(() -> processContext.getResultProcessor().process(processContext.process.get()));
-            listener.onResponse(task);
-        }, listener::onFailure);
+            finalListener.onResponse(task);
+        }, failedDeploymentListener::onFailure);
 
         ActionListener<GetTrainedModelsAction.Response> getModelListener = ActionListener.wrap(getModelResponse -> {
             assert getModelResponse.getResources().results().size() == 1;
@@ -168,7 +172,7 @@ private void doStartDeployment(TrainedModelDeploymentTask task, ActionListener<T
                 SearchRequest searchRequest = vocabSearchRequest(nlpConfig.getVocabularyConfig(), modelConfig.getModelId());
                 executeAsyncWithOrigin(client, ML_ORIGIN, SearchAction.INSTANCE, searchRequest, ActionListener.wrap(searchVocabResponse -> {
                     if (searchVocabResponse.getHits().getHits().length == 0) {
-                        listener.onFailure(
+                        failedDeploymentListener.onFailure(
                             new ResourceNotFoundException(
                                 Messages.getMessage(
                                     Messages.VOCABULARY_NOT_FOUND,
@@ -187,12 +191,10 @@ private void doStartDeployment(TrainedModelDeploymentTask task, ActionListener<T
                     // here, we are being called back on the searching thread, which MAY be a network thread
                     // `startAndLoad` creates named pipes, blocking the calling thread, better to execute that in our utility
                     // executor.
-                    executorServiceForDeployment.execute(
-                        () -> startAndLoad(processContext, modelConfig.getLocation(), modelLoadedListener)
-                    );
-                }, listener::onFailure));
+                    executorServiceForDeployment.execute(() -> processContext.startAndLoad(modelConfig.getLocation(), modelLoadedListener));
+                }, failedDeploymentListener::onFailure));
             } else {
-                listener.onFailure(
+                failedDeploymentListener.onFailure(
                     new IllegalArgumentException(
                         format(
                             "[%s] must be a pytorch model; found inference config of kind [%s]",
@@ -202,7 +204,7 @@ private void doStartDeployment(TrainedModelDeploymentTask task, ActionListener<T
                     )
                 );
             }
-        }, listener::onFailure);
+        }, failedDeploymentListener::onFailure);
 
         executeAsyncWithOrigin(
             client,
@@ -238,23 +240,8 @@ Vocabulary parseVocabularyDocLeniently(SearchHit hit) throws IOException {
         }
     }
 
-    private void startAndLoad(ProcessContext processContext, TrainedModelLocation modelLocation, ActionListener<Boolean> loadedListener) {
-        try {
-            processContext.startProcess();
-            processContext.loadModel(modelLocation, ActionListener.wrap(success -> {
-                processContext.startPriorityProcessWorker();
-                loadedListener.onResponse(success);
-            }, loadedListener::onFailure));
-        } catch (Exception e) {
-            loadedListener.onFailure(e);
-        }
-    }
-
     public void stopDeployment(TrainedModelDeploymentTask task) {
-        ProcessContext processContext;
-        synchronized (processContextByAllocation) {
-            processContext = processContextByAllocation.get(task.getId());
-        }
+        ProcessContext processContext = processContextByAllocation.remove(task.getId());
         if (processContext != null) {
             logger.info("[{}] Stopping deployment, reason [{}]", task.getModelId(), task.stoppedReason().orElse("unknown"));
             processContext.stopProcess();
@@ -394,6 +381,7 @@ class ProcessContext {
         private volatile Integer numAllocations;
         private final AtomicInteger rejectedExecutionCount = new AtomicInteger();
         private final AtomicInteger timeoutCount = new AtomicInteger();
+        private volatile boolean isStopped;
 
         ProcessContext(TrainedModelDeploymentTask task) {
             this.task = Objects.requireNonNull(task);
@@ -401,9 +389,12 @@ class ProcessContext {
                 this.numThreadsPerAllocation = threadSettings.numThreadsPerAllocation();
                 this.numAllocations = threadSettings.numAllocations();
             });
-            // We want to use the utility thread pool to load the model and not one of the process
-            // threads that are dedicated to processing done throughout the lifetime of the process.
-            this.stateStreamer = new PyTorchStateStreamer(client, executorServiceForDeployment, xContentRegistry);
+            // We want to use the inference thread pool to load the model as it is a possibly long operation
+            // and knowing it is an inference thread would enable better understanding during debugging.
+            // Even though we account for 3 threads per process in the thread pool, loading the model
+            // happens before we start input/output so it should be ok to use a thread from that pool for loading
+            // the model.
+            this.stateStreamer = new PyTorchStateStreamer(client, executorServiceForProcess, xContentRegistry);
             this.priorityProcessWorker = new PriorityProcessWorkerExecutorService(
                 threadPool.getThreadContext(),
                 "inference process",
@@ -415,9 +406,36 @@ PyTorchResultProcessor getResultProcessor() {
             return resultProcessor;
         }
 
-        synchronized void startProcess() {
-            process.set(pyTorchProcessFactory.createProcess(task, executorServiceForProcess, onProcessCrash()));
+        synchronized void startAndLoad(TrainedModelLocation modelLocation, ActionListener<Boolean> loadedListener) {
+            assert Thread.currentThread().getName().contains(UTILITY_THREAD_POOL_NAME)
+                : format("Must execute from [%s] but thread is [%s]", UTILITY_THREAD_POOL_NAME, Thread.currentThread().getName());
+
+            if (isStopped) {
+                logger.debug("[{}] model stopped before it is started", task.getModelId());
+                loadedListener.onFailure(new IllegalArgumentException("model stopped before it is started"));
+                return;
+            }
+
+            logger.debug("[{}] start and load", task.getModelId());
+            process.set(pyTorchProcessFactory.createProcess(task, executorServiceForProcess, this::onProcessCrash));
             startTime = Instant.now();
+            logger.debug("[{}] process started", task.getModelId());
+            try {
+                loadModel(modelLocation, ActionListener.wrap(success -> {
+                    if (isStopped) {
+                        logger.debug("[{}] model loaded but process is stopped", task.getModelId());
+                        killProcessIfPresent();
+                        loadedListener.onFailure(new IllegalStateException("model loaded but process is stopped"));
+                        return;
+                    }
+
+                    logger.debug("[{}] model loaded, starting priority process worker thread", task.getModelId());
+                    startPriorityProcessWorker();
+                    loadedListener.onResponse(success);
+                }, loadedListener::onFailure));
+            } catch (Exception e) {
+                loadedListener.onFailure(e);
+            }
         }
 
         void startPriorityProcessWorker() {
@@ -425,40 +443,58 @@ void startPriorityProcessWorker() {
         }
 
         synchronized void stopProcess() {
+            isStopped = true;
             resultProcessor.stop();
+            stateStreamer.cancel();
             priorityProcessWorker.shutdown();
+            killProcessIfPresent();
+            if (nlpTaskProcessor.get() != null) {
+                nlpTaskProcessor.get().close();
+            }
+        }
+
+        private void killProcessIfPresent() {
             try {
                 if (process.get() == null) {
                     return;
                 }
-                stateStreamer.cancel();
                 process.get().kill(true);
-                processContextByAllocation.remove(task.getId());
             } catch (IOException e) {
                 logger.error(() -> "[" + task.getModelId() + "] Failed to kill process", e);
-            } finally {
-                if (nlpTaskProcessor.get() != null) {
-                    nlpTaskProcessor.get().close();
-                }
             }
         }
 
-        private Consumer<String> onProcessCrash() {
-            return reason -> {
-                logger.error("[{}] inference process crashed due to reason [{}]", task.getModelId(), reason);
-                resultProcessor.stop();
-                priorityProcessWorker.shutdownWithError(new IllegalStateException(reason));
-                processContextByAllocation.remove(task.getId());
-                if (nlpTaskProcessor.get() != null) {
-                    nlpTaskProcessor.get().close();
-                }
-                task.setFailed("inference process crashed due to reason [" + reason + "]");
-            };
+        private void onProcessCrash(String reason) {
+            logger.error("[{}] inference process crashed due to reason [{}]", task.getModelId(), reason);
+            processContextByAllocation.remove(task.getId());
+            isStopped = true;
+            resultProcessor.stop();
+            stateStreamer.cancel();
+            priorityProcessWorker.shutdownWithError(new IllegalStateException(reason));
+            if (nlpTaskProcessor.get() != null) {
+                nlpTaskProcessor.get().close();
+            }
+            task.setFailed("inference process crashed due to reason [" + reason + "]");
         }
 
         void loadModel(TrainedModelLocation modelLocation, ActionListener<Boolean> listener) {
+            if (isStopped) {
+                listener.onFailure(new IllegalArgumentException("Process has stopped, model loading canceled"));
+                return;
+            }
             if (modelLocation instanceof IndexLocation indexLocation) {
-                process.get().loadModel(task.getModelId(), indexLocation.getIndexName(), stateStreamer, listener);
+                // Loading the model happens on the inference thread pool but when we get the callback
+                // we need to return to the utility thread pool to avoid leaking the thread we used.
+                process.get()
+                    .loadModel(
+                        task.getModelId(),
+                        indexLocation.getIndexName(),
+                        stateStreamer,
+                        ActionListener.wrap(
+                            r -> executorServiceForDeployment.submit(() -> listener.onResponse(r)),
+                            e -> executorServiceForDeployment.submit(() -> listener.onFailure(e))
+                        )
+                    );
             } else {
                 listener.onFailure(
                     new IllegalStateException("unsupported trained model location [" + modelLocation.getClass().getSimpleName() + "]")
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
index 48d98c981e14..0f4797740eee 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
@@ -41,6 +41,8 @@
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.QuestionAnsweringConfigUpdate;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RegressionConfig;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RegressionConfigUpdate;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.SlimConfig;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.SlimConfigUpdate;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextClassificationConfig;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextClassificationConfigUpdate;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfig;
@@ -150,7 +152,7 @@ InferModelAction.Request buildRequest(IngestDocument ingestDocument) {
             fields.put(INGEST_KEY, ingestDocument.getIngestMetadata());
         }
         LocalModel.mapFieldsIfNecessary(fields, fieldMap);
-        return new InferModelAction.Request(modelId, fields, inferenceConfig, previouslyLicensed);
+        return InferModelAction.Request.forDocs(modelId, List.of(fields), inferenceConfig, previouslyLicensed);
     }
 
     void auditWarningAboutLicenseIfNecessary() {
@@ -306,6 +308,9 @@ InferenceConfigUpdate inferenceConfigUpdateFromMap(Map<String, Object> configMap
             } else if (configMap.containsKey(RegressionConfig.NAME.getPreferredName())) {
                 checkSupportedVersion(RegressionConfig.EMPTY_PARAMS);
                 return RegressionConfigUpdate.fromMap(valueMap);
+            } else if (configMap.containsKey(SlimConfig.NAME)) {
+                checkNlpSupported(SlimConfig.NAME);
+                return SlimConfigUpdate.fromMap(valueMap);
             } else if (configMap.containsKey(TextClassificationConfig.NAME)) {
                 checkNlpSupported(TextClassificationConfig.NAME);
                 return TextClassificationConfigUpdate.fromMap(valueMap);
@@ -350,12 +355,12 @@ void checkNlpSupported(String taskType) {
         }
 
         void checkSupportedVersion(InferenceConfig config) {
-            if (config.getMinimalSupportedVersion().after(minNodeVersion)) {
+            if (config.getMinimalSupportedVersion().after(minNodeVersion.transportVersion)) {
                 throw ExceptionsHelper.badRequestException(
                     Messages.getMessage(
                         Messages.INFERENCE_CONFIG_NOT_SUPPORTED_ON_VERSION,
                         config.getName(),
-                        config.getMinimalSupportedVersion(),
+                        config.getMinimalSupportedNodeVersion(),
                         minNodeVersion
                     )
                 );
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java
index d99d81b9c39f..f893341cbec8 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java
@@ -355,10 +355,7 @@ private void loadModel(String modelId, Consumer consumer) {
                     );
                     return;
                 }
-                handleLoadFailure(
-                    modelId,
-                    new ElasticsearchStatusException("Trained model [{}] is not deployed.", RestStatus.BAD_REQUEST, modelId)
-                );
+                handleLoadFailure(modelId, modelMustBeDeployedError(modelId));
                 return;
             }
             auditNewReferencedModel(modelId);
@@ -409,13 +406,7 @@ private void loadWithoutCaching(
                     );
                     return;
                 }
-                modelActionListener.onFailure(
-                    new ElasticsearchStatusException(
-                        "model [{}] must be deployed to use. Please deploy with the start trained model deployment API.",
-                        RestStatus.BAD_REQUEST,
-                        modelId
-                    )
-                );
+                modelActionListener.onFailure(modelMustBeDeployedError(modelId));
                 return;
             }
             // Verify we can pull the model into memory without causing OOM
@@ -483,6 +474,14 @@ private void updateCircuitBreakerEstimate(
         }
     }
 
+    private ElasticsearchStatusException modelMustBeDeployedError(String modelId) {
+        return new ElasticsearchStatusException(
+            "Model [{}] must be deployed to use. Please deploy with the start trained model deployment API.",
+            RestStatus.BAD_REQUEST,
+            modelId
+        );
+    }
+
     private void handleLoadSuccess(
         String modelId,
         Consumer consumer,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/SlimProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/SlimProcessor.java
new file mode 100644
index 000000000000..7edb052ba18a
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/SlimProcessor.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.ml.inference.nlp;
+
+import org.elasticsearch.xpack.core.ml.inference.results.InferenceResults;
+import org.elasticsearch.xpack.core.ml.inference.results.SlimResults;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.NlpConfig;
+import org.elasticsearch.xpack.ml.inference.nlp.tokenizers.NlpTokenizer;
+import org.elasticsearch.xpack.ml.inference.nlp.tokenizers.TokenizationResult;
+import org.elasticsearch.xpack.ml.inference.pytorch.results.PyTorchInferenceResult;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+import static org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfig.DEFAULT_RESULTS_FIELD;
+
+public class SlimProcessor extends NlpTask.Processor {
+
+    private final NlpTask.RequestBuilder requestBuilder;
+
+    public SlimProcessor(NlpTokenizer tokenizer) {
+        super(tokenizer);
+        this.requestBuilder = tokenizer.requestBuilder();
+    }
+
+    @Override
+    public void validateInputs(List<String> inputs) {}
+
+    @Override
+    public NlpTask.RequestBuilder getRequestBuilder(NlpConfig config) {
+        return requestBuilder;
+    }
+
+    @Override
+    public NlpTask.ResultProcessor getResultProcessor(NlpConfig config) {
+        return (tokenization, pyTorchResult) -> processResult(tokenization, pyTorchResult, config.getResultsField());
+    }
+
+    static InferenceResults processResult(TokenizationResult tokenization, PyTorchInferenceResult pyTorchResult, String resultsField) {
+        // Convert the verbose results to the sparse format.
+        // Anything with a score > 0.0 is retained.
+        List<SlimResults.WeightedToken> weightedTokens = new ArrayList<>();
+        double[] weights = pyTorchResult.getInferenceResult()[0][0];
+        for (int i = 0; i < weights.length; i++) {
+            if (weights[i] > 0.0) {
+                weightedTokens.add(new SlimResults.WeightedToken(i, (float) weights[i]));
+            }
+        }
+
+        return new SlimResults(
+            Optional.ofNullable(resultsField).orElse(DEFAULT_RESULTS_FIELD),
+            weightedTokens,
+            tokenization.anyTruncated()
+        );
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TaskType.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TaskType.java
index a1fca461a738..4f86e10e9d7c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TaskType.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TaskType.java
@@ -64,6 +64,12 @@ public NlpTask.Processor createProcessor(NlpTokenizer tokenizer, NlpConfig confi
         public NlpTask.Processor createProcessor(NlpTokenizer tokenizer, NlpConfig config) {
             return new TextSimilarityProcessor(tokenizer);
         }
+    },
+    SLIM {
+        @Override
+        public NlpTask.Processor createProcessor(NlpTokenizer tokenizer, NlpConfig config) {
+            return new SlimProcessor(tokenizer);
+        }
     };
 
     public NlpTask.Processor createProcessor(NlpTokenizer tokenizer, NlpConfig config) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/Vocabulary.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/Vocabulary.java
index 6deb9a8b6d0f..7edf0be8e09a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/Vocabulary.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/Vocabulary.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.ml.inference.nlp;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -60,7 +60,7 @@ public Vocabulary(List<String> vocab, String modelId, @Nullable List<String> mer
     public Vocabulary(StreamInput in) throws IOException {
         vocab = in.readStringList();
         modelId = in.readString();
-        if (in.getVersion().onOrAfter(Version.V_8_2_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
             merges = in.readStringList();
         } else {
             merges = List.of();
@@ -79,7 +79,7 @@ public List<String> merges() {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeStringCollection(vocab);
         out.writeString(modelId);
-        if (out.getVersion().onOrAfter(Version.V_8_2_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_2_0)) {
             out.writeStringCollection(merges);
         }
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BasicTokenFilter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BasicTokenFilter.java
index 3be4eded9989..c5eb8f024f0d 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BasicTokenFilter.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BasicTokenFilter.java
@@ -118,19 +118,19 @@ public boolean incrementToken() throws IOException {
         }
         current = null; // not really needed, but for safety
         if (input.incrementToken()) {
-            if (isStripAccents) {
-                stripAccent();
-            }
             if (neverSplitSet.contains(termAtt)) {
                 return true;
             }
             // split punctuation and maybe cjk chars!!!
             LinkedList<DelimitedToken> splits = split();
-            // There is nothing to merge, nothing to store, simply return
-            if (splits.size() == 1) {
-                return true;
+            LinkedList<DelimitedToken> delimitedTokens = mergeSplits(splits);
+            if (isStripAccents) {
+                for (DelimitedToken token : delimitedTokens) {
+                    tokens.add(stripAccent(token));
+                }
+            } else {
+                tokens.addAll(delimitedTokens);
             }
-            tokens.addAll(mergeSplits(splits));
             this.current = captureState();
             DelimitedToken token = tokens.removeFirst();
             termAtt.setEmpty().append(token.charSequence());
@@ -140,14 +140,14 @@ public boolean incrementToken() throws IOException {
         return false;
     }
 
-    private void stripAccent() {
+    private DelimitedToken stripAccent(DelimitedToken token) {
         accentBuffer.setLength(0);
         boolean changed = false;
-        if (normalizer.quickCheck(termAtt) != Normalizer.YES) {
-            normalizer.normalize(termAtt, accentBuffer);
+        if (normalizer.quickCheck(token.charSequence()) != Normalizer.YES) {
+            normalizer.normalize(token.charSequence(), accentBuffer);
             changed = true;
         } else {
-            accentBuffer.append(termAtt);
+            accentBuffer.append(token.charSequence());
         }
         List<Integer> badIndices = new ArrayList<>();
         List<Integer> charCount = new ArrayList<>();
@@ -172,8 +172,9 @@ private void stripAccent() {
             }
         }
         if (changed) {
-            termAtt.setEmpty().append(accentBuffer);
+            return new DelimitedToken(accentBuffer.toString(), token.startOffset(), token.endOffset());
         }
+        return token;
     }
 
     private LinkedList<DelimitedToken> split() {
@@ -210,6 +211,9 @@ private LinkedList<DelimitedToken> split() {
     }
 
     private LinkedList<DelimitedToken> mergeSplits(LinkedList<DelimitedToken> splits) {
+        if (splits.size() == 1) {
+            return splits;
+        }
         LinkedList<DelimitedToken> mergedTokens = new LinkedList<>();
         List<DelimitedToken> matchingTokens = new ArrayList<>();
         CharSeqTokenTrieNode current = neverSplit;
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/NlpTokenizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/NlpTokenizer.java
index 3cdb3b702ae2..abc370e93c57 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/NlpTokenizer.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/NlpTokenizer.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.ml.inference.nlp.tokenizers;
 
 import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.BertTokenization;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.MPNetTokenization;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RobertaTokenization;
@@ -227,15 +228,16 @@ public TokenizationResult.Tokens tokenize(
      * @return tokenization result for the sequence pair
      */
     public List<TokenizationResult.Tokens> tokenize(String seq1, String seq2, Tokenization.Truncate truncate, int span, int sequenceId) {
+        if (isWithSpecialTokens() == false) {
+            throw new IllegalArgumentException("Unable to do sequence pair tokenization without special tokens");
+        }
+
         var innerResultSeq1 = innerTokenize(seq1);
         List<? extends DelimitedToken.Encoded> tokenIdsSeq1 = innerResultSeq1.tokens;
         List<Integer> tokenPositionMapSeq1 = innerResultSeq1.tokenPositionMap;
         var innerResultSeq2 = innerTokenize(seq2);
         List<? extends DelimitedToken.Encoded> tokenIdsSeq2 = innerResultSeq2.tokens;
         List<Integer> tokenPositionMapSeq2 = innerResultSeq2.tokenPositionMap;
-        if (isWithSpecialTokens() == false) {
-            throw new IllegalArgumentException("Unable to do sequence pair tokenization without special tokens");
-        }
         int extraTokens = getNumExtraTokensForSeqPair();
         int numTokens = tokenIdsSeq1.size() + tokenIdsSeq2.size() + extraTokens;
 
@@ -296,6 +298,30 @@ public List<TokenizationResult.Tokens> tokenize(String seq1, String seq2, Tokeni
         List<Integer> seq1TokenIds = tokenIdsSeq1.stream().map(DelimitedToken.Encoded::getEncoding).collect(Collectors.toList());
 
         final int trueMaxSeqLength = maxSequenceLength() - extraTokens - tokenIdsSeq1.size();
+        if (trueMaxSeqLength <= 0) {
+            throw new IllegalArgumentException(
+                Strings.format(
+                    "Unable to do sequence pair tokenization: the first sequence [%d tokens] "
+                        + "is longer than the max sequence length [%d tokens]",
+                    tokenIdsSeq1.size() + extraTokens,
+                    maxSequenceLength()
+                )
+            );
+        }
+
+        if (span > trueMaxSeqLength) {
+            throw new IllegalArgumentException(
+                Strings.format(
+                    "Unable to do sequence pair tokenization: the combined first sequence and span length [%d + %d = %d tokens] "
+                        + "is longer than the max sequence length [%d tokens]. Reduce the size of the [span] window.",
+                    tokenIdsSeq1.size(),
+                    span,
+                    tokenIdsSeq1.size() + span,
+                    maxSequenceLength()
+                )
+            );
+        }
+
         while (splitEndPos < tokenIdsSeq2.size()) {
             splitEndPos = Math.min(splitStartPos + trueMaxSeqLength, tokenIdsSeq2.size());
             // Make sure we do not end on a word
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
index 2ad36e4123a3..aa07df14a10d 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
@@ -688,7 +688,7 @@ public void revertSnapshot(
         CheckedConsumer<Boolean, Exception> updateHandler = response -> {
             if (response) {
                 ModelSizeStats revertedModelSizeStats = new ModelSizeStats.Builder(modelSizeStats).setLogTime(new Date()).build();
-                jobResultsPersister.persistModelSizeStats(
+                jobResultsPersister.persistModelSizeStatsWithoutRetries(
                     revertedModelSizeStats,
                     WriteRequest.RefreshPolicy.IMMEDIATE,
                     ActionListener.wrap(modelSizeStatsResponseHandler, actionListener::onFailure)
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java
index 04d6429d4ab3..c5c296374f65 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java
@@ -23,14 +23,17 @@
 
 import java.io.IOException;
 import java.time.Instant;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
 
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
 
 /**
- * Update a job's dataCounts
- * i.e. the number of processed records, fields etc.
+ * Updates job data counts, i.e. the number of processed records, fields etc.
+ * One instance of this class handles updates for all jobs.
  */
 public class JobDataCountsPersister {
 
@@ -40,6 +43,8 @@ public class JobDataCountsPersister {
     private final Client client;
     private final AnomalyDetectionAuditor auditor;
 
+    private final Map<String, CountDownLatch> ongoingPersists = new ConcurrentHashMap<>();
+
     public JobDataCountsPersister(Client client, ResultsPersisterService resultsPersisterService, AnomalyDetectionAuditor auditor) {
         this.resultsPersisterService = resultsPersisterService;
         this.client = client;
@@ -52,12 +57,27 @@ private static XContentBuilder serialiseCounts(DataCounts counts) throws IOExcep
     }
 
     /**
-     * Update the job's data counts stats and figures.
-     * NOTE: This call is synchronous and pauses the calling thread.
-     * @param jobId Job to update
-     * @param counts The counts
+     * Update a job's data counts stats and figures.
+     * If the previous call for the same job is still in progress
+     * @param jobId Job to update.
+     * @param counts The counts.
+     * @param mustWait Whether to wait for the counts to be persisted.
+     *                 This will involve waiting for the supplied counts
+     *                 and also potentially the previous counts to be
+     *                 persisted if that previous persist is still ongoing.
+     * @return <code>true</code> if the counts were sent for persistence, or <code>false</code>
+     *         if the previous persist was still in progress.
      */
-    public void persistDataCounts(String jobId, DataCounts counts) {
+    public boolean persistDataCounts(String jobId, DataCounts counts, boolean mustWait) throws InterruptedException {
+        CountDownLatch latch = new CountDownLatch(1);
+        CountDownLatch previousLatch = ongoingPersists.putIfAbsent(jobId, latch);
+        while (previousLatch != null) {
+            if (mustWait == false) {
+                return false;
+            }
+            previousLatch.await();
+            previousLatch = ongoingPersists.putIfAbsent(jobId, latch);
+        }
         counts.setLogTime(Instant.now());
         try {
             resultsPersisterService.indexWithRetry(
@@ -69,21 +89,29 @@ public void persistDataCounts(String jobId, DataCounts counts) {
                 DataCounts.documentId(jobId),
                 true,
                 () -> true,
-                retryMessage -> logger.debug("[{}] Job data_counts {}", jobId, retryMessage)
+                retryMessage -> logger.debug("[{}] Job data_counts {}", jobId, retryMessage),
+                ActionListener.wrap(r -> ongoingPersists.remove(jobId).countDown(), e -> {
+                    ongoingPersists.remove(jobId).countDown();
+                    logger.error(() -> "[" + jobId + "] Failed persisting data_counts stats", e);
+                    auditor.error(jobId, "Failed persisting data_counts stats: " + e.getMessage());
+                })
             );
-        } catch (IOException ioe) {
-            logger.error(() -> "[" + jobId + "] Failed writing data_counts stats", ioe);
-        } catch (Exception ex) {
-            logger.error(() -> "[" + jobId + "] Failed persisting data_counts stats", ex);
-            auditor.error(jobId, "Failed persisting data_counts stats: " + ex.getMessage());
+        } catch (IOException e) {
+            // An exception caught here basically means toXContent() failed, which should never happen
+            logger.error(() -> "[" + jobId + "] Failed writing data_counts stats", e);
+            return false;
+        }
+        if (mustWait) {
+            latch.await();
         }
+        return true;
     }
 
     /**
-     * The same as {@link JobDataCountsPersister#persistDataCounts(String, DataCounts)} but done Asynchronously.
-     *
+     * Very similar to {@link JobDataCountsPersister#persistDataCounts(String, DataCounts, boolean)}.
+     * <p>
      * Two differences are:
-     *  - The listener is notified on persistence failure
+     *  - The caller is notified on persistence failure
      *  - If the persistence fails, it is not automatically retried
      * @param jobId Job to update
      * @param counts The counts
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java
index e65d78749f6b..c2661fc933ca 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -49,9 +50,14 @@
 import org.elasticsearch.xpack.ml.utils.persistence.ResultsPersisterService;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.EnumSet;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.function.Supplier;
 
 import static org.elasticsearch.core.Strings.format;
@@ -82,30 +88,39 @@ public class JobResultsPersister {
     private final OriginSettingClient client;
     private final ResultsPersisterService resultsPersisterService;
 
+    /**
+     * The possible types of data that may be committed.
+     */
+    public enum CommitType {
+        RESULTS,
+        STATE,
+        ANNOTATIONS
+    };
+
     public JobResultsPersister(OriginSettingClient client, ResultsPersisterService resultsPersisterService) {
         this.client = client;
         this.resultsPersisterService = resultsPersisterService;
     }
 
     public Builder bulkPersisterBuilder(String jobId) {
-        return new Builder(jobId);
+        return new Builder(jobId, () -> true);
+    }
+
+    public Builder bulkPersisterBuilder(String jobId, Supplier<Boolean> shouldRetry) {
+        return new Builder(jobId, shouldRetry);
     }
 
     public class Builder {
-        private BulkRequest bulkRequest;
+        private final Map<String, IndexRequest> items;
         private final String jobId;
         private final String indexName;
-        private Supplier<Boolean> shouldRetry = () -> true;
+        private final Supplier<Boolean> shouldRetry;
 
-        private Builder(String jobId) {
-            this.bulkRequest = new BulkRequest();
+        private Builder(String jobId, Supplier<Boolean> shouldRetry) {
+            this.items = new LinkedHashMap<>();
             this.jobId = Objects.requireNonNull(jobId);
             this.indexName = AnomalyDetectorsIndex.resultsWriteAlias(jobId);
-        }
-
-        public Builder shouldRetry(Supplier<Boolean> shouldRetry) {
-            this.shouldRetry = Objects.requireNonNull(shouldRetry);
-            return this;
+            this.shouldRetry = shouldRetry;
         }
 
         /**
@@ -115,7 +130,7 @@ public Builder shouldRetry(Supplier<Boolean> shouldRetry) {
          * @param bucket The bucket to persist
          * @return this
          */
-        public Builder persistBucket(Bucket bucket) {
+        public synchronized Builder persistBucket(Bucket bucket) {
             // If the supplied bucket has records then create a copy with records
             // removed, because we never persist nested records in buckets
             Bucket bucketWithoutRecords = bucket;
@@ -132,7 +147,7 @@ public Builder persistBucket(Bucket bucket) {
             return this;
         }
 
-        private void persistBucketInfluencersStandalone(
+        private synchronized void persistBucketInfluencersStandalone(
             @SuppressWarnings("HiddenField") String jobId,
             List<BucketInfluencer> bucketInfluencers
         ) {
@@ -151,7 +166,7 @@ private void persistBucketInfluencersStandalone(
          * @param timingStats timing stats to persist
          * @return this
          */
-        public Builder persistTimingStats(TimingStats timingStats) {
+        public synchronized Builder persistTimingStats(TimingStats timingStats) {
             indexResult(
                 TimingStats.documentId(timingStats.getJobId()),
                 timingStats,
@@ -167,7 +182,7 @@ public Builder persistTimingStats(TimingStats timingStats) {
          * @param records the records to persist
          * @return this
          */
-        public Builder persistRecords(List<AnomalyRecord> records) {
+        public synchronized Builder persistRecords(List<AnomalyRecord> records) {
             for (AnomalyRecord record : records) {
                 logger.trace("[{}] ES BULK ACTION: index record to index [{}] with ID [{}]", jobId, indexName, record.getId());
                 indexResult(record.getId(), record, "record");
@@ -183,7 +198,7 @@ public Builder persistRecords(List<AnomalyRecord> records) {
          * @param influencers the influencers to persist
          * @return this
          */
-        public Builder persistInfluencers(List<Influencer> influencers) {
+        public synchronized Builder persistInfluencers(List<Influencer> influencers) {
             for (Influencer influencer : influencers) {
                 logger.trace("[{}] ES BULK ACTION: index influencer to index [{}] with ID [{}]", jobId, indexName, influencer.getId());
                 indexResult(influencer.getId(), influencer, "influencer");
@@ -192,13 +207,13 @@ public Builder persistInfluencers(List<Influencer> influencers) {
             return this;
         }
 
-        public Builder persistModelPlot(ModelPlot modelPlot) {
+        public synchronized Builder persistModelPlot(ModelPlot modelPlot) {
             logger.trace("[{}] ES BULK ACTION: index model plot to index [{}] with ID [{}]", jobId, indexName, modelPlot.getId());
             indexResult(modelPlot.getId(), modelPlot, "model plot");
             return this;
         }
 
-        public Builder persistCategorizerStats(CategorizerStats categorizerStats) {
+        public synchronized Builder persistCategorizerStats(CategorizerStats categorizerStats) {
             logger.trace(
                 "[{}] ES BULK ACTION: index categorizer stats to index [{}] with ID [{}]",
                 jobId,
@@ -209,20 +224,42 @@ public Builder persistCategorizerStats(CategorizerStats categorizerStats) {
             return this;
         }
 
-        public Builder persistForecast(Forecast forecast) {
+        public synchronized Builder persistCategoryDefinition(CategoryDefinition categoryDefinition) {
+            logger.trace(
+                "[{}] ES BULK ACTION: index category definition to index [{}] with ID [{}]",
+                jobId,
+                indexName,
+                categoryDefinition.getId()
+            );
+            indexResult(categoryDefinition.getId(), categoryDefinition, "category definition");
+            return this;
+        }
+
+        public synchronized Builder persistModelSizeStats(ModelSizeStats modelSizeStats) {
+            logger.trace(
+                "[{}] ES BULK ACTION: index model size stats to index [{}] with ID [{}]",
+                jobId,
+                indexName,
+                modelSizeStats.getId()
+            );
+            indexResult(modelSizeStats.getId(), modelSizeStats, "model size stats");
+            return this;
+        }
+
+        public synchronized Builder persistForecast(Forecast forecast) {
             logger.trace("[{}] ES BULK ACTION: index forecast to index [{}] with ID [{}]", jobId, indexName, forecast.getId());
             indexResult(forecast.getId(), forecast, Forecast.RESULT_TYPE_VALUE);
             return this;
         }
 
-        public Builder persistForecastRequestStats(ForecastRequestStats forecastRequestStats) {
+        public synchronized Builder persistForecastRequestStats(ForecastRequestStats forecastRequestStats) {
             logger.trace(
                 "[{}] ES BULK ACTION: index forecast request stats to index [{}] with ID [{}]",
                 jobId,
                 indexName,
                 forecastRequestStats.getId()
             );
-            indexResult(forecastRequestStats.getId(), forecastRequestStats, Forecast.RESULT_TYPE_VALUE);
+            indexResult(forecastRequestStats.getId(), forecastRequestStats, "forecast request stats");
             return this;
         }
 
@@ -232,12 +269,12 @@ private void indexResult(String id, ToXContent resultDoc, String resultType) {
 
         private void indexResult(String id, ToXContent resultDoc, ToXContent.Params params, String resultType) {
             try (XContentBuilder content = toXContentBuilder(resultDoc, params)) {
-                bulkRequest.add(new IndexRequest(indexName).id(id).source(content));
+                items.put(id, new IndexRequest(indexName).id(id).source(content));
             } catch (IOException e) {
                 logger.error(() -> format("[%s] Error serialising %s", jobId, resultType), e);
             }
 
-            if (bulkRequest.numberOfActions() >= JobRenormalizedResultsPersister.BULK_LIMIT) {
+            if (items.size() >= JobRenormalizedResultsPersister.BULK_LIMIT) {
                 executeRequest();
             }
         }
@@ -245,45 +282,36 @@ private void indexResult(String id, ToXContent resultDoc, ToXContent.Params para
         /**
          * Execute the bulk action
          */
-        public void executeRequest() {
-            if (bulkRequest.numberOfActions() == 0) {
+        public synchronized void executeRequest() {
+            if (items.isEmpty()) {
                 return;
             }
-            logger.trace("[{}] ES API CALL: bulk request with {} actions", jobId, bulkRequest.numberOfActions());
+            logger.trace("[{}] ES API CALL: bulk request with {} actions", jobId, items.size());
             resultsPersisterService.bulkIndexWithRetry(
-                bulkRequest,
+                buildBulkRequest(),
                 jobId,
                 shouldRetry,
                 retryMessage -> logger.debug("[{}] Bulk indexing of results failed {}", jobId, retryMessage)
             );
-            bulkRequest = new BulkRequest();
+            clear();
         }
 
-        public void clearBulkRequest() {
-            bulkRequest = new BulkRequest();
+        private BulkRequest buildBulkRequest() {
+            BulkRequest bulkRequest = new BulkRequest();
+            for (IndexRequest item : items.values()) {
+                bulkRequest.add(item);
+            }
+            return bulkRequest;
         }
 
-        // for testing
-        BulkRequest getBulkRequest() {
-            return bulkRequest;
+        public synchronized void clear() {
+            items.clear();
         }
-    }
 
-    /**
-     * Persist the category definition
-     *
-     * @param category The category to be persisted
-     */
-    public void persistCategoryDefinition(CategoryDefinition category, Supplier<Boolean> shouldRetry) {
-        Persistable persistable = new Persistable(
-            AnomalyDetectorsIndex.resultsWriteAlias(category.getJobId()),
-            category.getJobId(),
-            category,
-            category.getId()
-        );
-        persistable.persist(shouldRetry, true);
-        // Don't commit as we expect masses of these updates and they're not
-        // read again by this process
+        // for testing
+        synchronized BulkRequest getBulkRequest() {
+            return buildBulkRequest();
+        }
     }
 
     /**
@@ -323,7 +351,7 @@ public void persistQuantiles(Quantiles quantiles, WriteRequest.RefreshPolicy ref
 
             Persistable persistable = new Persistable(indexOrAlias, quantiles.getJobId(), quantiles, quantilesDocId);
             persistable.setRefreshPolicy(refreshPolicy);
-            persistable.persist(listener, AnomalyDetectorsIndex.jobStateIndexWriteAlias().equals(indexOrAlias));
+            persistable.persistWithoutRetries(listener, AnomalyDetectorsIndex.jobStateIndexWriteAlias().equals(indexOrAlias));
         }, listener::onFailure);
 
         // Step 1: Search for existing quantiles document in .ml-state*
@@ -383,7 +411,7 @@ public void persistModelSizeStats(ModelSizeStats modelSizeStats, Supplier<Boolea
     /**
      * Persist the memory usage data
      */
-    public void persistModelSizeStats(
+    public void persistModelSizeStatsWithoutRetries(
         ModelSizeStats modelSizeStats,
         WriteRequest.RefreshPolicy refreshPolicy,
         ActionListener<IndexResponse> listener
@@ -397,7 +425,7 @@ public void persistModelSizeStats(
             modelSizeStats.getId()
         );
         persistable.setRefreshPolicy(refreshPolicy);
-        persistable.persist(listener, true);
+        persistable.persistWithoutRetries(listener, true);
     }
 
     /**
@@ -411,52 +439,47 @@ public void deleteInterimResults(String jobId) {
      * Once all the job data has been written this function will be
      * called to commit the writes to the datastore.
      *
-     * @param jobId The job Id
+     * @param jobId The job ID.
+     * @param commitType Which type of data will be committed?
      */
-    public void commitResultWrites(String jobId) {
-        // We refresh using the read alias in order to ensure all indices will
-        // be refreshed even if a rollover occurs in between.
-        String indexName = AnomalyDetectorsIndex.jobResultsAliasedName(jobId);
-
-        // Refresh should wait for Lucene to make the data searchable
-        logger.trace("[{}] ES API CALL: refresh index {}", jobId, indexName);
-        RefreshRequest refreshRequest = new RefreshRequest(indexName);
-        refreshRequest.indicesOptions(IndicesOptions.lenientExpandOpen());
-        try (ThreadContext.StoredContext ignore = client.threadPool().getThreadContext().stashWithOrigin(ML_ORIGIN)) {
-            client.admin().indices().refresh(refreshRequest).actionGet();
-        }
+    public void commitWrites(String jobId, CommitType commitType) {
+        commitWrites(jobId, EnumSet.of(commitType));
     }
 
     /**
-     * Makes annotations searchable as they are considered part of a job's results
-     * to fulfil the contract that job results are searchable immediately after a
-     * close or flush.
+     * Once all the job data has been written this function will be
+     * called to commit the writes to the datastore.
+     *
+     * @param jobId The job ID.
+     * @param commitTypes Which type(s) of data will be committed?
      */
-    public void commitAnnotationWrites() {
-        // We refresh using the read alias in order to ensure all indices will
-        // be refreshed even if a rollover occurs in between.
-        RefreshRequest refreshRequest = new RefreshRequest(AnnotationIndex.READ_ALIAS_NAME);
-        refreshRequest.indicesOptions(IndicesOptions.lenientExpandOpen());
-        try (ThreadContext.StoredContext ignore = client.threadPool().getThreadContext().stashWithOrigin(ML_ORIGIN)) {
-            client.admin().indices().refresh(refreshRequest).actionGet();
+    public void commitWrites(String jobId, Set<CommitType> commitTypes) {
+        if (commitTypes.isEmpty()) {
+            return;
+        }
+        List<String> indexNames = new ArrayList<>();
+        if (commitTypes.contains(CommitType.RESULTS)) {
+            // We refresh using the read alias in order to ensure all indices will
+            // be refreshed even if a rollover occurs in between.
+            indexNames.add(AnomalyDetectorsIndex.jobResultsAliasedName(jobId));
+        }
+        if (commitTypes.contains(CommitType.STATE)) {
+            indexNames.add(AnomalyDetectorsIndex.jobStateIndexPattern());
+        }
+        if (commitTypes.contains(CommitType.ANNOTATIONS)) {
+            // We refresh using the read alias in order to ensure all indices will
+            // be refreshed even if a rollover occurs in between.
+            indexNames.add(AnnotationIndex.READ_ALIAS_NAME);
         }
-    }
 
-    /**
-     * Once the job state has been written calling this function makes it
-     * immediately searchable.
-     *
-     * @param jobId The job Id
-     * */
-    public void commitStateWrites(String jobId) {
-        String indexName = AnomalyDetectorsIndex.jobStateIndexPattern();
         // Refresh should wait for Lucene to make the data searchable
-        logger.trace("[{}] ES API CALL: refresh index {}", jobId, indexName);
-        RefreshRequest refreshRequest = new RefreshRequest(indexName);
+        logger.trace("[{}] ES API CALL: refresh indices {}", jobId, indexNames);
+        RefreshRequest refreshRequest = new RefreshRequest(indexNames.toArray(String[]::new));
         refreshRequest.indicesOptions(IndicesOptions.lenientExpandOpen());
         try (ThreadContext.StoredContext ignore = client.threadPool().getThreadContext().stashWithOrigin(ML_ORIGIN)) {
             client.admin().indices().refresh(refreshRequest).actionGet();
         }
+        logger.trace("[{}] ES API CALL: finished refresh indices {}", jobId, indexNames);
     }
 
     /**
@@ -464,8 +487,13 @@ public void commitStateWrites(String jobId) {
      *
      * @param timingStats datafeed timing stats to persist
      * @param refreshPolicy refresh policy to apply
+     * @param listener listener for response or error
      */
-    public BulkResponse persistDatafeedTimingStats(DatafeedTimingStats timingStats, WriteRequest.RefreshPolicy refreshPolicy) {
+    public void persistDatafeedTimingStats(
+        DatafeedTimingStats timingStats,
+        WriteRequest.RefreshPolicy refreshPolicy,
+        ActionListener<BulkResponse> listener
+    ) {
         String jobId = timingStats.getJobId();
         logger.trace("[{}] Persisting datafeed timing stats", jobId);
         Persistable persistable = new Persistable(
@@ -476,7 +504,7 @@ public BulkResponse persistDatafeedTimingStats(DatafeedTimingStats timingStats,
             DatafeedTimingStats.documentId(timingStats.getJobId())
         );
         persistable.setRefreshPolicy(refreshPolicy);
-        return persistable.persist(() -> true, true);
+        persistable.persist(() -> true, true, listener);
     }
 
     private static XContentBuilder toXContentBuilder(ToXContent obj, ToXContent.Params params) throws IOException {
@@ -512,9 +540,15 @@ void setRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) {
         }
 
         BulkResponse persist(Supplier<Boolean> shouldRetry, boolean requireAlias) {
+            final PlainActionFuture<BulkResponse> getResponseFuture = PlainActionFuture.newFuture();
+            persist(shouldRetry, requireAlias, getResponseFuture);
+            return getResponseFuture.actionGet();
+        }
+
+        void persist(Supplier<Boolean> shouldRetry, boolean requireAlias, ActionListener<BulkResponse> listener) {
             logCall();
             try {
-                return resultsPersisterService.indexWithRetry(
+                resultsPersisterService.indexWithRetry(
                     jobId,
                     indexName,
                     object,
@@ -523,20 +557,23 @@ BulkResponse persist(Supplier<Boolean> shouldRetry, boolean requireAlias) {
                     id,
                     requireAlias,
                     shouldRetry,
-                    retryMessage -> logger.debug("[{}] {} {}", jobId, id, retryMessage)
+                    retryMessage -> logger.debug("[{}] {} {}", jobId, id, retryMessage),
+                    listener
                 );
             } catch (IOException e) {
                 logger.error(() -> format("[%s] Error writing [%s]", jobId, (id == null) ? "auto-generated ID" : id), e);
                 IndexResponse.Builder notCreatedResponse = new IndexResponse.Builder();
                 notCreatedResponse.setResult(Result.NOOP);
-                return new BulkResponse(
-                    new BulkItemResponse[] { BulkItemResponse.success(0, DocWriteRequest.OpType.INDEX, notCreatedResponse.build()) },
-                    0
+                listener.onResponse(
+                    new BulkResponse(
+                        new BulkItemResponse[] { BulkItemResponse.success(0, DocWriteRequest.OpType.INDEX, notCreatedResponse.build()) },
+                        0
+                    )
                 );
             }
         }
 
-        void persist(ActionListener<IndexResponse> listener, boolean requireAlias) {
+        void persistWithoutRetries(ActionListener<IndexResponse> listener, boolean requireAlias) {
             logCall();
 
             try (XContentBuilder content = toXContentBuilder(object, params)) {
@@ -563,5 +600,4 @@ private void logCall() {
             }
         }
     }
-
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
index 40ffb7d2ea3e..8a6c6265839a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
@@ -95,6 +95,7 @@
 import org.elasticsearch.xpack.core.ml.action.GetBucketsAction;
 import org.elasticsearch.xpack.core.ml.action.GetCategoriesAction;
 import org.elasticsearch.xpack.core.ml.action.GetInfluencersAction;
+import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction;
 import org.elasticsearch.xpack.core.ml.action.GetRecordsAction;
 import org.elasticsearch.xpack.core.ml.calendars.Calendar;
 import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent;
@@ -2010,17 +2011,36 @@ public void onFailure(Exception e) {
     public void getRestartTimeInfo(String jobId, ActionListener<RestartTimeInfo> listener) {
         AtomicReference<Bucket> latestFinalBucketHolder = new AtomicReference<>();
 
-        Consumer<DataCounts> dataCountsHandler = dataCounts -> listener.onResponse(
-            new RestartTimeInfo(
-                latestFinalBucketHolder.get() == null ? null : latestFinalBucketHolder.get().getTimestamp().getTime(),
-                dataCounts.getLatestRecordTimeStamp() == null ? null : dataCounts.getLatestRecordTimeStamp().getTime(),
-                dataCounts.getInputRecordCount() > 0
-            )
-        );
+        ActionListener<GetJobsStatsAction.Response> jobStatsHandler = ActionListener.wrap(r -> {
+            QueryPage<GetJobsStatsAction.Response.JobStats> page = r.getResponse();
+            if (page.count() == 1) {
+                DataCounts dataCounts = page.results().get(0).getDataCounts();
+                listener.onResponse(
+                    new RestartTimeInfo(
+                        latestFinalBucketHolder.get() == null ? null : latestFinalBucketHolder.get().getTimestamp().getTime(),
+                        dataCounts.getLatestRecordTimeStamp() == null ? null : dataCounts.getLatestRecordTimeStamp().getTime(),
+                        dataCounts.getInputRecordCount() > 0
+                    )
+                );
+            } else {
+                listener.onFailure(
+                    new IllegalStateException(
+                        "[" + jobId + "] Incorrect response for job stats for single job: got [" + page.count() + "] stats."
+                    )
+                );
+            }
+        }, listener::onFailure);
 
         ActionListener<Bucket> latestFinalBucketListener = ActionListener.wrap(latestFinalBucket -> {
             latestFinalBucketHolder.set(latestFinalBucket);
-            dataCounts(jobId, dataCountsHandler, listener::onFailure);
+            // If the job is open we must get the data counts from the running job instead of from the index,
+            // as the data counts in the running job might be more recent than those that have been indexed. If
+            // we go to the index it would be more efficient to just get the data counts rather than all stats.
+            // However, we _shouldn't_ ever do this when starting a datafeed, because a precondition for starting
+            // the datafeed is that its job is open. Therefore, it's reasonable to always go via the get job stats
+            // endpoint, on the assumption it will almost always get the stats from memory. (The only edge case
+            // where this won't happen is when the job fails during datafeed startup, and that should be very rare.)
+            client.execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request(jobId), jobStatsHandler);
         }, listener::onFailure);
 
         getLatestFinalBucket(jobId, latestFinalBucketListener);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/DataCountsReporter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/DataCountsReporter.java
index 306425c50d3c..610386e33bc2 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/DataCountsReporter.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/DataCountsReporter.java
@@ -20,7 +20,7 @@
 /**
  * Status reporter for tracking counts of the good/bad records written to the API.
  * Call one of the reportXXX() methods to update the records counts.
- *
+ * <p>
  * Stats are logged at specific stages
  * <ol>
  * <li>Every 10,000 records for the first 100,000 records</li>
@@ -43,6 +43,8 @@ public class DataCountsReporter {
     private final DataCounts totalRecordStats;
     private volatile DataCounts incrementalRecordStats;
 
+    private DataCounts unreportedStats;
+
     private long analyzedFieldsPerRecord = 1;
 
     private long lastRecordCountQuotient = 0;
@@ -236,7 +238,32 @@ public void finishReporting() {
         totalRecordStats.setLastDataTimeStamp(now);
         diagnostics.flush();
         retrieveDiagnosticsIntermediateResults();
-        dataCountsPersister.persistDataCounts(job.getId(), runningTotalStats());
+        synchronized (this) {
+            unreportedStats = null;
+        }
+        DataCounts statsToReport = runningTotalStats();
+        try {
+            if (dataCountsPersister.persistDataCounts(job.getId(), statsToReport, false) == false) {
+                synchronized (this) {
+                    unreportedStats = statsToReport;
+                }
+            }
+        } catch (InterruptedException e) {
+            assert false : "This should never happen when persistDataCounts is called with mustWait set to false";
+            Thread.currentThread().interrupt();
+        }
+    }
+
+    /**
+     * Report the most recent counts, if any, that {@link #finishReporting} failed to
+     * report when the previous persistence was in progress.
+     */
+    public synchronized void writeUnreportedCounts() throws InterruptedException {
+        if (unreportedStats != null) {
+            boolean persisted = dataCountsPersister.persistDataCounts(job.getId(), unreportedStats, true);
+            assert persisted;
+            unreportedStats = null;
+        }
     }
 
     /**
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java
index c21c3ddb6b9c..1136f4c083cc 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java
@@ -180,6 +180,7 @@ public void close() {
         try {
             future.get();
             autodetectWorkerExecutor.shutdown();
+            dataCountsReporter.writeUnreportedCounts();
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
         } catch (ExecutionException e) {
@@ -256,6 +257,11 @@ public void writeUpdateProcessMessage(UpdateProcessMessage update, BiConsumer<Vo
     public void flushJob(FlushJobParams params, BiConsumer<FlushAcknowledgement, Exception> handler) {
         submitOperation(() -> {
             String flushId = autodetectProcess.flushJob(params);
+            try {
+                dataCountsReporter.writeUnreportedCounts();
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
             return waitFlushToCompletion(flushId, params.isWaitForNormalization());
         }, handler);
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessor.java
index f240104c285a..ff32f0cc72d6 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessor.java
@@ -47,6 +47,7 @@
 import java.time.Clock;
 import java.time.Duration;
 import java.util.Date;
+import java.util.EnumSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -68,7 +69,7 @@
  * <p>
  * Has methods to register and remove alert observers.
  * Also has a method to wait for a flush to be complete.
- *
+ * <p>
  * Buckets are the written last after records, influencers etc
  * when the end of bucket is reached. Therefore results aren't persisted
  * until the bucket is read, this means that interim results for all
@@ -80,7 +81,7 @@
  */
 public class AutodetectResultProcessor {
 
-    private static final Logger LOGGER = LogManager.getLogger(AutodetectResultProcessor.class);
+    private static final Logger logger = LogManager.getLogger(AutodetectResultProcessor.class);
 
     private final Client client;
     private final AnomalyDetectionAuditor auditor;
@@ -157,8 +158,8 @@ public AutodetectResultProcessor(
         this.process = Objects.requireNonNull(autodetectProcess);
         this.flushListener = Objects.requireNonNull(flushListener);
         this.latestModelSizeStats = Objects.requireNonNull(latestModelSizeStats);
-        this.bulkResultsPersister = persister.bulkPersisterBuilder(jobId).shouldRetry(this::isAlive);
-        this.bulkAnnotationsPersister = annotationPersister.bulkPersisterBuilder(jobId).shouldRetry(this::isAlive);
+        this.bulkResultsPersister = persister.bulkPersisterBuilder(jobId, this::isAlive);
+        this.bulkAnnotationsPersister = annotationPersister.bulkPersisterBuilder(jobId, this::isAlive);
         this.timingStatsReporter = new TimingStatsReporter(timingStats, bulkResultsPersister);
         this.clock = Objects.requireNonNull(clock);
         this.deleteInterimRequired = true;
@@ -181,9 +182,9 @@ public void process() {
                     bulkAnnotationsPersister.executeRequest();
                 }
             } catch (Exception e) {
-                LOGGER.warn(() -> "[" + jobId + "] Error persisting autodetect results", e);
+                logger.warn(() -> "[" + jobId + "] Error persisting autodetect results", e);
             }
-            LOGGER.info("[{}] {} buckets parsed from autodetect output", jobId, currentRunBucketCount);
+            logger.info("[{}] {} buckets parsed from autodetect output", jobId, currentRunBucketCount);
 
         } catch (Exception e) {
             failed = true;
@@ -193,14 +194,14 @@ public void process() {
                 // that it would have been better to close jobs before shutting down,
                 // but we now fully expect jobs to move between nodes without doing
                 // all their graceful close activities.
-                LOGGER.warn("[{}] some results not processed due to the process being killed", jobId);
+                logger.warn("[{}] some results not processed due to the process being killed", jobId);
             } else if (process.isProcessAliveAfterWaiting() == false) {
                 // Don't log the stack trace to not shadow the root cause.
-                LOGGER.warn("[{}] some results not processed due to the termination of autodetect", jobId);
+                logger.warn("[{}] some results not processed due to the termination of autodetect", jobId);
             } else {
                 // We should only get here if the iterator throws in which
                 // case parsing the autodetect output has failed.
-                LOGGER.error(() -> "[" + jobId + "] error parsing autodetect output", e);
+                logger.error(() -> "[" + jobId + "] error parsing autodetect output", e);
             }
         } finally {
             flushListener.clear();
@@ -218,13 +219,13 @@ private void readResults() {
                     AutodetectResult result = iterator.next();
                     processResult(result);
                     if (result.getBucket() != null) {
-                        LOGGER.trace("[{}] Bucket number {} parsed from output", jobId, currentRunBucketCount);
+                        logger.trace("[{}] Bucket number {} parsed from output", jobId, currentRunBucketCount);
                     }
                 } catch (Exception e) {
                     if (isAlive() == false) {
                         throw e;
                     }
-                    LOGGER.warn(() -> "[" + jobId + "] Error processing autodetect result", e);
+                    logger.warn(() -> "[" + jobId + "] Error processing autodetect result", e);
                 }
             }
         } finally {
@@ -250,9 +251,9 @@ public void setVacating(boolean vacating) {
     void handleOpenForecasts() {
         try {
             if (runningForecasts.isEmpty() == false) {
-                LOGGER.warn("[{}] still had forecasts {} executing. Attempting to set them to failed.", jobId, runningForecasts.keySet());
+                logger.warn("[{}] still had forecasts {} executing. Attempting to set them to failed.", jobId, runningForecasts.keySet());
                 // There may be many docs in the results persistence queue. But we only want to bother updating the running forecasts
-                bulkResultsPersister.clearBulkRequest();
+                bulkResultsPersister.clear();
                 for (ForecastRequestStats forecastRequestStats : runningForecasts.values()) {
                     ForecastRequestStats failedStats = new ForecastRequestStats(forecastRequestStats);
                     failedStats.setStatus(ForecastRequestStats.ForecastRequestStatus.FAILED);
@@ -262,7 +263,7 @@ void handleOpenForecasts() {
                 bulkResultsPersister.executeRequest();
             }
         } catch (Exception ex) {
-            LOGGER.warn(() -> "[" + jobId + "] failure setting running forecasts to failed.", ex);
+            logger.warn(() -> "[" + jobId + "] failure setting running forecasts to failed.", ex);
         }
     }
 
@@ -276,19 +277,23 @@ void processResult(AutodetectResult result) {
             if (deleteInterimRequired) {
                 // Delete any existing interim results generated by a Flush command
                 // which have not been replaced or superseded by new results.
-                LOGGER.trace("[{}] Deleting interim results", jobId);
+                logger.trace("[{}] Deleting interim results", jobId);
                 persister.deleteInterimResults(jobId);
-                deleteInterimRequired = false;
             }
 
             if (bucket.isInterim() == false) {
                 timingStatsReporter.reportBucket(bucket);
                 ++currentRunBucketCount;
             }
-            // persist after deleting interim results in case the new
-            // results are also interim
-            bulkResultsPersister.persistBucket(bucket).executeRequest();
-            bulkAnnotationsPersister.executeRequest();
+            bulkResultsPersister.persistBucket(bucket);
+            if (deleteInterimRequired || bucket.isInterim()) {
+                // Execute the bulk request after deleting interim results in case the new
+                // results are also interim. Also execute the bulk request after creating new
+                // interim results, so that they exist before any subsequent deletion.
+                bulkResultsPersister.executeRequest();
+                bulkAnnotationsPersister.executeRequest();
+                deleteInterimRequired = false;
+            }
         }
         List<AnomalyRecord> records = result.getRecords();
         if (records != null && records.isEmpty() == false) {
@@ -300,7 +305,7 @@ void processResult(AutodetectResult result) {
         }
         CategoryDefinition categoryDefinition = result.getCategoryDefinition();
         if (categoryDefinition != null) {
-            persister.persistCategoryDefinition(categoryDefinition, this::isAlive);
+            bulkResultsPersister.persistCategoryDefinition(categoryDefinition);
         }
         CategorizerStats categorizerStats = result.getCategorizerStats();
         if (categorizerStats != null) {
@@ -321,7 +326,7 @@ void processResult(AutodetectResult result) {
         }
         ForecastRequestStats forecastRequestStats = result.getForecastRequestStats();
         if (forecastRequestStats != null) {
-            LOGGER.trace("Received Forecast Stats [{}]", forecastRequestStats.getId());
+            logger.trace("Received Forecast Stats [{}]", forecastRequestStats.getId());
             bulkResultsPersister.persistForecastRequestStats(forecastRequestStats);
 
             if (forecastRequestStats.getStatus()
@@ -364,21 +369,24 @@ void processResult(AutodetectResult result) {
         }
         Quantiles quantiles = result.getQuantiles();
         if (quantiles != null) {
-            LOGGER.debug("[{}] Parsed Quantiles with timestamp {}", jobId, quantiles.getTimestamp());
+            logger.debug("[{}] Parsed Quantiles with timestamp {}", jobId, quantiles.getTimestamp());
             persister.persistQuantiles(quantiles, this::isAlive);
-            bulkResultsPersister.executeRequest();
 
             // If a node is trying to shut down then don't trigger any further normalizations on the node
             if (vacating == false && processKilled == false && renormalizer.isEnabled()) {
-                // We need to make all results written up to these quantiles available for renormalization
-                persister.commitResultWrites(jobId);
-                LOGGER.debug("[{}] Quantiles queued for renormalization", jobId);
-                renormalizer.renormalize(quantiles);
+                logger.debug("[{}] Quantiles queued for renormalization", jobId);
+                renormalizer.renormalize(quantiles, () -> {
+                    // We need to make all results written up to these quantiles available for renormalization.
+                    // However, this should be done as close to the point of normalization as possible, as many
+                    // quantiles are superseded before they're used.
+                    bulkResultsPersister.executeRequest();
+                    persister.commitWrites(jobId, JobResultsPersister.CommitType.RESULTS);
+                });
             }
         }
         FlushAcknowledgement flushAcknowledgement = result.getFlushAcknowledgement();
         if (flushAcknowledgement != null) {
-            LOGGER.debug("[{}] Flush acknowledgement parsed from output for ID {}", jobId, flushAcknowledgement.getId());
+            logger.debug("[{}] Flush acknowledgement parsed from output for ID {}", jobId, flushAcknowledgement.getId());
             // Commit previous writes here, effectively continuing
             // the flush from the C++ autodetect process right
             // through to the data store
@@ -386,11 +394,12 @@ void processResult(AutodetectResult result) {
             try {
                 bulkResultsPersister.executeRequest();
                 bulkAnnotationsPersister.executeRequest();
-                persister.commitResultWrites(jobId);
-                persister.commitAnnotationWrites();
-                LOGGER.debug("[{}] Flush acknowledgement sent to listener for ID {}", jobId, flushAcknowledgement.getId());
+                persister.commitWrites(
+                    jobId,
+                    EnumSet.of(JobResultsPersister.CommitType.RESULTS, JobResultsPersister.CommitType.ANNOTATIONS)
+                );
             } catch (Exception e) {
-                LOGGER.error(
+                logger.error(
                     "["
                         + jobId
                         + "] failed to bulk persist results and commit writes during flush acknowledgement for ID "
@@ -428,7 +437,7 @@ private Annotation createModelSnapshotAnnotation(ModelSnapshot modelSnapshot) {
     }
 
     private void processModelSizeStats(ModelSizeStats modelSizeStats) {
-        LOGGER.trace(
+        logger.trace(
             "[{}] Parsed ModelSizeStats: {} / {} / {} / {} / {} / {}",
             jobId,
             modelSizeStats.getModelBytes(),
@@ -439,7 +448,7 @@ private void processModelSizeStats(ModelSizeStats modelSizeStats) {
             modelSizeStats.getMemoryStatus()
         );
 
-        persister.persistModelSizeStats(modelSizeStats, this::isAlive);
+        bulkResultsPersister.persistModelSizeStats(modelSizeStats);
         notifyModelMemoryStatusChange(modelSizeStats);
 
         latestModelSizeStats = modelSizeStats;
@@ -491,7 +500,7 @@ protected void updateModelSnapshotOnJob(ModelSnapshot modelSnapshot) {
             updateModelSnapshotSemaphore.acquire();
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
-            LOGGER.info("[{}] Interrupted acquiring update model snapshot semaphore", jobId);
+            logger.info("[{}] Interrupted acquiring update model snapshot semaphore", jobId);
             return;
         }
 
@@ -499,13 +508,13 @@ protected void updateModelSnapshotOnJob(ModelSnapshot modelSnapshot) {
             @Override
             public void onResponse(PutJobAction.Response response) {
                 updateModelSnapshotSemaphore.release();
-                LOGGER.debug("[{}] Updated job with model snapshot id [{}]", jobId, modelSnapshot.getSnapshotId());
+                logger.debug("[{}] Updated job with model snapshot id [{}]", jobId, modelSnapshot.getSnapshotId());
             }
 
             @Override
             public void onFailure(Exception e) {
                 updateModelSnapshotSemaphore.release();
-                LOGGER.error("[" + jobId + "] Failed to update job with new model snapshot id [" + modelSnapshot.getSnapshotId() + "]", e);
+                logger.error("[" + jobId + "] Failed to update job with new model snapshot id [" + modelSnapshot.getSnapshotId() + "]", e);
             }
         });
     }
@@ -525,13 +534,11 @@ public void awaitCompletion() throws TimeoutException {
 
             // These lines ensure that the "completion" we're awaiting includes making the results searchable
             waitUntilRenormalizerIsIdle();
-            persister.commitResultWrites(jobId);
-            persister.commitAnnotationWrites();
-            persister.commitStateWrites(jobId);
+            persister.commitWrites(jobId, EnumSet.allOf(JobResultsPersister.CommitType.class));
 
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
-            LOGGER.info("[{}] Interrupted waiting for results processor to complete", jobId);
+            logger.info("[{}] Interrupted waiting for results processor to complete", jobId);
         }
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/JobSnapshotUpgraderResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/JobSnapshotUpgraderResultProcessor.java
index 12dca226858b..7d5bc1feeaf8 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/JobSnapshotUpgraderResultProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/JobSnapshotUpgraderResultProcessor.java
@@ -69,7 +69,7 @@ public JobSnapshotUpgraderResultProcessor(
         this.snapshotId = Objects.requireNonNull(snapshotId);
         this.persister = Objects.requireNonNull(persister);
         this.process = Objects.requireNonNull(autodetectProcess);
-        this.bulkResultsPersister = persister.bulkPersisterBuilder(jobId).shouldRetry(this::isAlive);
+        this.bulkResultsPersister = persister.bulkPersisterBuilder(jobId, this::isAlive);
         this.flushListener = new FlushListener();
     }
 
@@ -249,7 +249,7 @@ public void awaitCompletion() throws TimeoutException {
             }
 
             // These lines ensure that the "completion" we're awaiting includes making the results searchable
-            persister.commitStateWrites(jobId);
+            persister.commitWrites(jobId, JobResultsPersister.CommitType.STATE);
 
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Renormalizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Renormalizer.java
index a6500503ee76..62d3a7d58524 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Renormalizer.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Renormalizer.java
@@ -20,7 +20,7 @@ public interface Renormalizer {
      * Update the anomaly score field on all previously persisted buckets
      * and all contained records
      */
-    void renormalize(Quantiles quantiles);
+    void renormalize(Quantiles quantiles, Runnable setupStep);
 
     /**
      * Blocks until the renormalizer is idle and no further quantiles updates are pending.
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizer.java
index c3932c0334f9..1371221cb16b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizer.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizer.java
@@ -8,6 +8,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.util.CancellableThreads;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles;
 
 import java.util.Date;
@@ -53,7 +54,7 @@ public boolean isEnabled() {
     }
 
     @Override
-    public void renormalize(Quantiles quantiles) {
+    public void renormalize(Quantiles quantiles, Runnable setupStep) {
         if (isEnabled() == false) {
             return;
         }
@@ -61,8 +62,13 @@ public void renormalize(Quantiles quantiles) {
         // Needed to ensure work is not added while the tryFinishWork() method is running
         synchronized (this) {
             latestQuantilesHolder = (latestQuantilesHolder == null)
-                ? new AugmentedQuantiles(quantiles, null, new CountDownLatch(1))
-                : new AugmentedQuantiles(quantiles, latestQuantilesHolder.getEvictedTimestamp(), latestQuantilesHolder.getLatch());
+                ? new AugmentedQuantiles(quantiles, setupStep, null, new CountDownLatch(1))
+                : new AugmentedQuantiles(
+                    quantiles,
+                    setupStep,
+                    latestQuantilesHolder.getEvictedTimestamp(),
+                    latestQuantilesHolder.getLatch()
+                );
             tryStartWork();
         }
     }
@@ -89,8 +95,15 @@ public void waitUntilIdle() throws InterruptedException {
                 try {
                     taskToWaitFor.get();
                 } catch (ExecutionException e) {
-                    // This shouldn't happen, because we catch normalization errors inside the normalization loop
+                    if (e.getCause() instanceof CancellableThreads.ExecutionCancelledException) {
+                        // This happens if reset mode is enabled while result writes are being retried.
+                        // Reset mode means the job whose results were being normalized will shortly
+                        // cease to exist, so it's fine to consider the wait for renormalization complete.
+                        break;
+                    }
                     logger.error("[" + jobId + "] Error propagated from normalization", e);
+                    // Don't loop again for the same task that caused the error
+                    taskToWaitFor = null;
                 } catch (CancellationException e) {
                     // Convert cancellations to interruptions to simplify the interface
                     throw new InterruptedException("Normalization cancelled");
@@ -153,6 +166,7 @@ private void doRenormalizations() {
             AugmentedQuantiles latestAugmentedQuantiles = getLatestAugmentedQuantilesAndClear();
             assert latestAugmentedQuantiles != null;
             if (latestAugmentedQuantiles != null) { // TODO: remove this if the assert doesn't trip in CI over the next year or so
+                latestAugmentedQuantiles.runSetupStep();
                 Quantiles latestQuantiles = latestAugmentedQuantiles.getQuantiles();
                 CountDownLatch latch = latestAugmentedQuantiles.getLatch();
                 try {
@@ -181,11 +195,13 @@ private void doRenormalizations() {
      */
     private class AugmentedQuantiles {
         private final Quantiles quantiles;
+        private final Runnable setupStep;
         private final Date earliestEvictedTimestamp;
         private final CountDownLatch latch;
 
-        AugmentedQuantiles(Quantiles quantiles, Date earliestEvictedTimestamp, CountDownLatch latch) {
+        AugmentedQuantiles(Quantiles quantiles, Runnable setupStep, Date earliestEvictedTimestamp, CountDownLatch latch) {
             this.quantiles = Objects.requireNonNull(quantiles);
+            this.setupStep = Objects.requireNonNull(setupStep);
             this.earliestEvictedTimestamp = earliestEvictedTimestamp;
             this.latch = Objects.requireNonNull(latch);
         }
@@ -194,6 +210,10 @@ Quantiles getQuantiles() {
             return quantiles;
         }
 
+        void runSetupStep() {
+            setupStep.run();
+        }
+
         Date getEvictedTimestamp() {
             return (earliestEvictedTimestamp != null) ? earliestEvictedTimestamp : quantiles.getTimestamp();
         }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemover.java
index f0a82ca2e774..2ef38e3e3693 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemover.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemover.java
@@ -125,11 +125,8 @@ private static DeleteByQueryRequest createDBQRequest(Job job, float requestsPerS
     @Override
     void calcCutoffEpochMs(String jobId, long retentionDays, ActionListener<CutoffDetails> listener) {
         ThreadedActionListener<CutoffDetails> threadedActionListener = new ThreadedActionListener<>(
-            LOGGER,
-            threadPool,
-            MachineLearning.UTILITY_THREAD_POOL_NAME,
-            listener,
-            false
+            threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME),
+            listener
         );
         latestBucketTime(client, getParentTaskId(), jobId, ActionListener.wrap(latestTime -> {
             if (latestTime == null) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredForecastsRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredForecastsRemover.java
index 5adaa1cb3946..388e971f1e06 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredForecastsRemover.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredForecastsRemover.java
@@ -100,7 +100,7 @@ public void remove(float requestsPerSec, ActionListener<Boolean> listener, Boole
         client.execute(
             SearchAction.INSTANCE,
             searchRequest,
-            new ThreadedActionListener<>(LOGGER, threadPool, MachineLearning.UTILITY_THREAD_POOL_NAME, forecastStatsHandler, false)
+            new ThreadedActionListener<>(threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME), forecastStatsHandler)
         );
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemover.java
index 1e4cedb20076..1854e3b752de 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemover.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemover.java
@@ -100,11 +100,8 @@ Long getRetentionDays(Job job) {
     @Override
     void calcCutoffEpochMs(String jobId, long retentionDays, ActionListener<CutoffDetails> listener) {
         ThreadedActionListener<CutoffDetails> threadedActionListener = new ThreadedActionListener<>(
-            LOGGER,
-            threadPool,
-            MachineLearning.UTILITY_THREAD_POOL_NAME,
-            listener,
-            false
+            threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME),
+            listener
         );
 
         latestSnapshotTimeStamp(jobId, ActionListener.wrap(latestTime -> {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java
index 6faa2c4e8498..0e5ed9bbdd57 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java
@@ -150,11 +150,8 @@ private DeleteByQueryRequest createDBQRequest(Job job, float requestsPerSec, lon
     @Override
     void calcCutoffEpochMs(String jobId, long retentionDays, ActionListener<CutoffDetails> listener) {
         ThreadedActionListener<CutoffDetails> threadedActionListener = new ThreadedActionListener<>(
-            LOGGER,
-            threadPool,
-            MachineLearning.UTILITY_THREAD_POOL_NAME,
-            listener,
-            false
+            threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME),
+            listener
         );
         latestBucketTime(client, getParentTaskId(), jobId, ActionListener.wrap(latestTime -> {
             if (latestTime == null) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java
index 235f5b825501..73701359b8d3 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java
@@ -233,7 +233,7 @@ private GroupedActionListener<ActionResponse> createGroupedListener(
         final List<TrainedModelConfig> configs,
         final ActionListener<Table> listener
     ) {
-        return new GroupedActionListener<>(listener.delegateFailure((l, responses) -> {
+        return new GroupedActionListener<>(size, listener.delegateFailure((l, responses) -> {
             GetTrainedModelsStatsAction.Response statsResponse = extractResponse(responses, GetTrainedModelsStatsAction.Response.class);
             GetDataFrameAnalyticsAction.Response analytics = extractResponse(responses, GetDataFrameAnalyticsAction.Response.class);
             l.onResponse(
@@ -244,7 +244,7 @@ private GroupedActionListener<ActionResponse> createGroupedListener(
                     analytics == null ? Collections.emptyList() : analytics.getResources().results()
                 )
             );
-        }), size);
+        }));
     }
 
     private Table buildTable(
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestInferTrainedModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestInferTrainedModelAction.java
index ce32e74a6272..9c8b3aa634a4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestInferTrainedModelAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestInferTrainedModelAction.java
@@ -14,9 +14,7 @@
 import org.elasticsearch.rest.action.RestCancellableNodeClient;
 import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.xpack.core.ml.action.InferModelAction;
-import org.elasticsearch.xpack.core.ml.action.InferTrainedModelDeploymentAction;
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig;
-import org.elasticsearch.xpack.core.ml.inference.trainedmodel.EmptyConfigUpdate;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
@@ -48,16 +46,13 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
         }
         InferModelAction.Request.Builder request = InferModelAction.Request.parseRequest(modelId, restRequest.contentParser());
 
-        if (restRequest.hasParam(InferTrainedModelDeploymentAction.Request.TIMEOUT.getPreferredName())) {
+        if (restRequest.hasParam(InferModelAction.Request.TIMEOUT.getPreferredName())) {
             TimeValue inferTimeout = restRequest.paramAsTime(
-                InferTrainedModelDeploymentAction.Request.TIMEOUT.getPreferredName(),
-                InferTrainedModelDeploymentAction.Request.DEFAULT_TIMEOUT
+                InferModelAction.Request.TIMEOUT.getPreferredName(),
+                InferModelAction.Request.DEFAULT_TIMEOUT
             );
             request.setInferenceTimeout(inferTimeout);
         }
-        if (request.getUpdate() == null) {
-            request.setUpdate(new EmptyConfigUpdate());
-        }
 
         return channel -> new RestCancellableNodeClient(client, restRequest.getHttpChannel()).execute(
             InferModelAction.EXTERNAL_INSTANCE,
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestInferTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestInferTrainedModelDeploymentAction.java
index 8f736569df42..01f10862bc5f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestInferTrainedModelDeploymentAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestInferTrainedModelDeploymentAction.java
@@ -7,13 +7,16 @@
 
 package org.elasticsearch.xpack.ml.rest.inference;
 
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestCancellableNodeClient;
 import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.ml.action.InferModelAction;
 import org.elasticsearch.xpack.core.ml.action.InferTrainedModelDeploymentAction;
 import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
@@ -60,23 +63,36 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
         if (restRequest.hasContent() == false) {
             throw ExceptionsHelper.badRequestException("requires body");
         }
-        InferTrainedModelDeploymentAction.Request.Builder request = InferTrainedModelDeploymentAction.Request.parseRequest(
-            modelId,
-            restRequest.contentParser()
-        );
+        InferModelAction.Request.Builder requestBuilder = InferModelAction.Request.parseRequest(modelId, restRequest.contentParser());
 
-        if (restRequest.hasParam(InferTrainedModelDeploymentAction.Request.TIMEOUT.getPreferredName())) {
+        if (restRequest.hasParam(InferModelAction.Request.TIMEOUT.getPreferredName())) {
             TimeValue inferTimeout = restRequest.paramAsTime(
-                InferTrainedModelDeploymentAction.Request.TIMEOUT.getPreferredName(),
-                InferTrainedModelDeploymentAction.Request.DEFAULT_TIMEOUT
+                InferModelAction.Request.TIMEOUT.getPreferredName(),
+                InferModelAction.Request.DEFAULT_TIMEOUT
             );
-            request.setInferenceTimeout(inferTimeout);
+            requestBuilder.setInferenceTimeout(inferTimeout);
+        }
+
+        // Unlike the _infer API, deployment/_infer only accepts a single document
+        var request = requestBuilder.build();
+        if (request.getObjectsToInfer() != null && request.getObjectsToInfer().size() > 1) {
+            ValidationException ex = new ValidationException();
+            ex.addValidationError("multiple documents are not supported");
+            throw ex;
         }
 
         return channel -> new RestCancellableNodeClient(client, restRequest.getHttpChannel()).execute(
-            InferTrainedModelDeploymentAction.INSTANCE,
-            request.build(),
-            new RestToXContentListener<>(channel)
+            InferModelAction.EXTERNAL_INSTANCE,
+            request,
+            // This API is deprecated but refactoring makes it simpler to call
+            // the new replacement API and swap in the old response.
+            ActionListener.wrap(response -> {
+                InferTrainedModelDeploymentAction.Response oldResponse = new InferTrainedModelDeploymentAction.Response(
+                    response.getInferenceResults()
+                );
+                new RestToXContentListener<>(channel).onResponse(oldResponse);
+            }, e -> new RestToXContentListener<>(channel).onFailure(e))
+
         );
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestSemanticSearchAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestSemanticSearchAction.java
deleted file mode 100644
index 60aaa7aa324e..000000000000
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestSemanticSearchAction.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.ml.rest.inference;
-
-import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.xpack.core.ml.action.SemanticSearchAction;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.elasticsearch.rest.RestRequest.Method.GET;
-import static org.elasticsearch.rest.RestRequest.Method.POST;
-
-public class RestSemanticSearchAction extends BaseRestHandler {
-
-    public RestSemanticSearchAction() {}
-
-    @Override
-    public List<Route> routes() {
-        String path = "{index}/_semantic_search";
-        return List.of(Route.builder(GET, path).build(), Route.builder(POST, path).build());
-    }
-
-    @Override
-    public String getName() {
-        return "semantic_search_action";
-    }
-
-    @Override
-    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
-        SemanticSearchAction.Request request = SemanticSearchAction.Request.parseRestRequest(restRequest);
-        return channel -> client.execute(SemanticSearchAction.INSTANCE, request, new RestToXContentListener<>(channel));
-    }
-}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/NativeMemoryCalculator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/NativeMemoryCalculator.java
index 6964639f0535..4920e94ee1f0 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/NativeMemoryCalculator.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/NativeMemoryCalculator.java
@@ -40,6 +40,8 @@ public final class NativeMemoryCalculator {
     // Must match the value used in MachineDependentHeap.MachineNodeRole.ML_ONLY.
     public static final long JVM_SIZE_KNOT_POINT = ByteSizeValue.ofGb(16).getBytes();
     private static final long BYTES_IN_4MB = ByteSizeValue.ofMb(4).getBytes();
+    // The minimum automatic node size implicitly defines a minimum JVM size
+    private static final long MINIMUM_AUTOMATIC_JVM_SIZE = dynamicallyCalculateJvmSizeFromNodeSize(MINIMUM_AUTOMATIC_NODE_SIZE);
 
     private NativeMemoryCalculator() {}
 
@@ -165,7 +167,7 @@ public static long dynamicallyCalculateJvmSizeFromNodeSize(long nodeSize) {
         );
     }
 
-    public static long dynamicallyCalculateJvmSizeFromMlNativeMemorySize(long nativeMachineMemory) {
+    public static long dynamicallyCalculateJvmSizeFromMlNativeMemorySize(long mlNativeMemorySize) {
         // For <= 16GB node, the JVM is 0.4 * total_node_size. This means the rest is 0.6 the node size.
         // So, nativeAndOverhead = 0.6 * total_node_size => total_node_size = (nativeAndOverhead / 0.6)
         // Consequently jvmSize = (nativeAndOverhead / 0.6) * 0.4 = nativeAndOverhead * 2 / 3
@@ -177,7 +179,7 @@ public static long dynamicallyCalculateJvmSizeFromMlNativeMemorySize(long native
         //
         // In both cases JVM size is rounded down to the next lower multiple of 4 megabytes to match
         // MachineDependentHeap.MachineNodeRole.ML_ONLY.
-        long nativeAndOverhead = nativeMachineMemory + OS_OVERHEAD;
+        long nativeAndOverhead = mlNativeMemorySize + OS_OVERHEAD;
         long higherAnswer;
         if (nativeAndOverhead <= (JVM_SIZE_KNOT_POINT - dynamicallyCalculateJvmSizeFromNodeSize(JVM_SIZE_KNOT_POINT))) {
             higherAnswer = (nativeAndOverhead * 2 / 3 / BYTES_IN_4MB) * BYTES_IN_4MB;
@@ -196,10 +198,10 @@ public static long dynamicallyCalculateJvmSizeFromMlNativeMemorySize(long native
             long lowerAnswer = higherAnswer - BYTES_IN_4MB;
             long nodeSizeImpliedByLowerAnswer = nativeAndOverhead + lowerAnswer;
             if (dynamicallyCalculateJvmSizeFromNodeSize(nodeSizeImpliedByLowerAnswer) == lowerAnswer) {
-                return Math.min(lowerAnswer, STATIC_JVM_UPPER_THRESHOLD);
+                return Math.max(MINIMUM_AUTOMATIC_JVM_SIZE, Math.min(lowerAnswer, STATIC_JVM_UPPER_THRESHOLD));
             }
         }
-        return Math.min(higherAnswer, STATIC_JVM_UPPER_THRESHOLD);
+        return Math.max(MINIMUM_AUTOMATIC_JVM_SIZE, Math.min(higherAnswer, STATIC_JVM_UPPER_THRESHOLD));
     }
 
     /**
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterService.java
index cf168e370424..de418364732e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterService.java
@@ -41,8 +41,6 @@
 import java.io.IOException;
 import java.time.Duration;
 import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.BiConsumer;
@@ -58,20 +56,16 @@ public class ResultsPersisterService {
     /**
      * List of rest statuses that we consider irrecoverable
      */
-    public static final Set<RestStatus> IRRECOVERABLE_REST_STATUSES = Collections.unmodifiableSet(
-        new HashSet<>(
-            Arrays.asList(
-                RestStatus.GONE,
-                RestStatus.NOT_IMPLEMENTED,
-                // Not found is returned when we require an alias but the index is NOT an alias.
-                RestStatus.NOT_FOUND,
-                RestStatus.BAD_REQUEST,
-                RestStatus.UNAUTHORIZED,
-                RestStatus.FORBIDDEN,
-                RestStatus.METHOD_NOT_ALLOWED,
-                RestStatus.NOT_ACCEPTABLE
-            )
-        )
+    public static final Set<RestStatus> IRRECOVERABLE_REST_STATUSES = Set.of(
+        RestStatus.GONE,
+        RestStatus.NOT_IMPLEMENTED,
+        // Not found is returned when we require an alias but the index is NOT an alias.
+        RestStatus.NOT_FOUND,
+        RestStatus.BAD_REQUEST,
+        RestStatus.UNAUTHORIZED,
+        RestStatus.FORBIDDEN,
+        RestStatus.METHOD_NOT_ALLOWED,
+        RestStatus.NOT_ACCEPTABLE
     );
 
     private static final Logger LOGGER = LogManager.getLogger(ResultsPersisterService.class);
@@ -161,6 +155,25 @@ public BulkResponse indexWithRetry(
         return bulkIndexWithRetry(bulkRequest, jobId, shouldRetry, retryMsgHandler);
     }
 
+    public void indexWithRetry(
+        String jobId,
+        String indexName,
+        ToXContent object,
+        ToXContent.Params params,
+        WriteRequest.RefreshPolicy refreshPolicy,
+        String id,
+        boolean requireAlias,
+        Supplier<Boolean> shouldRetry,
+        Consumer<String> retryMsgHandler,
+        ActionListener<BulkResponse> finalListener
+    ) throws IOException {
+        BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(refreshPolicy);
+        try (XContentBuilder content = object.toXContent(XContentFactory.jsonBuilder(), params)) {
+            bulkRequest.add(new IndexRequest(indexName).id(id).source(content).setRequireAlias(requireAlias));
+        }
+        bulkIndexWithRetry(bulkRequest, jobId, shouldRetry, retryMsgHandler, finalListener);
+    }
+
     public BulkResponse bulkIndexWithRetry(
         BulkRequest bulkRequest,
         String jobId,
@@ -170,6 +183,25 @@ public BulkResponse bulkIndexWithRetry(
         return bulkIndexWithRetry(bulkRequest, jobId, shouldRetry, retryMsgHandler, client::bulk);
     }
 
+    public void bulkIndexWithRetry(
+        BulkRequest bulkRequest,
+        String jobId,
+        Supplier<Boolean> shouldRetry,
+        Consumer<String> retryMsgHandler,
+        ActionListener<BulkResponse> finalListener
+    ) {
+        if (isShutdown || isResetMode) {
+            finalListener.onFailure(
+                new ElasticsearchException(
+                    "Bulk indexing has failed as {}",
+                    isShutdown ? "node is shutting down." : "machine learning feature is being reset."
+                )
+            );
+            return;
+        }
+        bulkIndexWithRetry(bulkRequest, jobId, shouldRetry, retryMsgHandler, client::bulk, finalListener);
+    }
+
     public BulkResponse bulkIndexWithHeadersWithRetry(
         Map<String, String> headers,
         BulkRequest bulkRequest,
@@ -206,10 +238,22 @@ private BulkResponse bulkIndexWithRetry(
                 isShutdown ? "node is shutting down." : "machine learning feature is being reset."
             );
         }
-        final PlainActionFuture<BulkResponse> getResponse = PlainActionFuture.newFuture();
+        final PlainActionFuture<BulkResponse> getResponseFuture = PlainActionFuture.newFuture();
+        bulkIndexWithRetry(bulkRequest, jobId, shouldRetry, retryMsgHandler, actionExecutor, getResponseFuture);
+        return getResponseFuture.actionGet();
+    }
+
+    private void bulkIndexWithRetry(
+        BulkRequest bulkRequest,
+        String jobId,
+        Supplier<Boolean> shouldRetry,
+        Consumer<String> retryMsgHandler,
+        BiConsumer<BulkRequest, ActionListener<BulkResponse>> actionExecutor,
+        ActionListener<BulkResponse> finalListener
+    ) {
         final Object key = new Object();
         final ActionListener<BulkResponse> removeListener = ActionListener.runBefore(
-            getResponse,
+            finalListener,
             () -> onGoingRetryableBulkActions.remove(key)
         );
         BulkRetryableAction bulkRetryableAction = new BulkRetryableAction(
@@ -229,7 +273,6 @@ private BulkResponse bulkIndexWithRetry(
                 )
             );
         }
-        return getResponse.actionGet();
     }
 
     public SearchResponse searchWithRetry(
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigrationEligibilityCheckTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigrationEligibilityCheckTests.java
index 68af65ec440f..f77e8cd9ff1c 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigrationEligibilityCheckTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigrationEligibilityCheckTests.java
@@ -130,7 +130,8 @@ private void addMlConfigIndex(String indexName, String aliasName, Metadata.Build
             shardId,
             true,
             RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""),
+            ShardRouting.Role.DEFAULT
         );
         shardRouting = shardRouting.initialize("node_id", null, 0L);
         shardRouting = shardRouting.moveToStarted(ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java
index d9b749628292..3e6b2f7b76c1 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java
@@ -10,7 +10,7 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
@@ -19,7 +19,7 @@
 
 import static org.hamcrest.Matchers.equalTo;
 
-public class MlMetadataTests extends AbstractXContentSerializingTestCase<MlMetadata> {
+public class MlMetadataTests extends AbstractChunkedSerializingTestCase<MlMetadata> {
 
     @Override
     protected MlMetadata createTestInstance() {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/InternalCategorizationAggregationTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/InternalCategorizationAggregationTests.java
index 8d2414a5c15a..1f9119463994 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/InternalCategorizationAggregationTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/InternalCategorizationAggregationTests.java
@@ -123,4 +123,9 @@ private static Map<Object, Long> toCounts(Stream<? extends InternalCategorizatio
             )
         );
     }
+
+    @Override
+    protected InternalCategorizationAggregation mutateInstance(InternalCategorizationAggregation instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/SerializableTokenListCategoryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/SerializableTokenListCategoryTests.java
index ef3997c40297..3f3fcf697e17 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/SerializableTokenListCategoryTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/categorization/SerializableTokenListCategoryTests.java
@@ -85,6 +85,11 @@ protected SerializableTokenListCategory createTestInstance() {
         return createTestInstance(bytesRefHash);
     }
 
+    @Override
+    protected SerializableTokenListCategory mutateInstance(SerializableTokenListCategory instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static SerializableTokenListCategory createTestInstance(CategorizationBytesRefHash bytesRefHash) {
         return new SerializableTokenListCategory(
             TokenListCategoryTests.createTestInstance(bytesRefHash, randomIntBetween(1, 1000)),
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/InternalChangePointAggregationTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/InternalChangePointAggregationTests.java
index 468c21e54d70..aacb6766fb06 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/InternalChangePointAggregationTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/changepoint/InternalChangePointAggregationTests.java
@@ -49,4 +49,9 @@ protected InternalChangePointAggregation createTestInstance() {
             )
         );
     }
+
+    @Override
+    protected InternalChangePointAggregation mutateInstance(InternalChangePointAggregation instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/correlation/CountCorrelationIndicatorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/correlation/CountCorrelationIndicatorTests.java
index d192863cbb9a..77358da6356b 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/correlation/CountCorrelationIndicatorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/correlation/CountCorrelationIndicatorTests.java
@@ -43,4 +43,9 @@ protected Writeable.Reader<CountCorrelationIndicator> instanceReader() {
     protected CountCorrelationIndicator createTestInstance() {
         return randomInstance();
     }
+
+    @Override
+    protected CountCorrelationIndicator mutateInstance(CountCorrelationIndicator instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/EclatMapReducerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/EclatMapReducerTests.java
new file mode 100644
index 000000000000..4715002cdbcc
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/EclatMapReducerTests.java
@@ -0,0 +1,494 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.ml.aggs.frequentitemsets;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.MockPageCacheRecycler;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.ml.aggs.frequentitemsets.mr.ItemSetMapReduceValueSource.Field;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.core.Tuple.tuple;
+import static org.elasticsearch.xpack.ml.aggs.frequentitemsets.mr.ItemSetMapReduceValueSourceTests.createKeywordFieldTestInstance;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThan;
+
+public class EclatMapReducerTests extends ESTestCase {
+
+    private static Supplier<Boolean> doNotCancelSupplier = () -> false;
+
+    public void testSimple() throws IOException {
+        Field field1 = createKeywordFieldTestInstance("keyword1", 0);
+        Field field2 = createKeywordFieldTestInstance("keyword2", 1);
+        Field field3 = createKeywordFieldTestInstance("keyword3", 2);
+
+        EclatMapReducer eclat = new EclatMapReducer(getTestName(), 0.1, 2, 10, true);
+
+        HashBasedTransactionStore transactionStore = eclat.mapInit(mockBigArrays());
+        eclat.map(mockOneDocument(List.of(tuple(field1, "f1-a"), tuple(field2, "f2-1"), tuple(field3, "f3-A"))), transactionStore);
+        eclat.map(mockOneDocument(List.of(tuple(field1, "f1-a"), tuple(field2, "f2-1"), tuple(field3, "f3-B"))), transactionStore);
+        eclat.map(mockOneDocument(List.of(tuple(field1, "f1-a"), tuple(field2, "f2-1"), tuple(field3, "f3-C"))), transactionStore);
+        eclat.map(mockOneDocument(List.of(tuple(field1, "f1-a"), tuple(field2, "f2-1"), tuple(field3, "f3-D"))), transactionStore);
+        eclat.map(mockOneDocument(List.of(tuple(field1, "f1-a"), tuple(field2, "f2-1"), tuple(field3, "f3-E"))), transactionStore);
+        eclat.map(mockOneDocument(List.of(tuple(field1, "f1-b"), tuple(field2, "f2-1"), tuple(field3, "f3-F"))), transactionStore);
+        eclat.map(mockOneDocument(List.of(tuple(field1, "f1-b"), tuple(field2, "f2-1"), tuple(field3, "f3-G"))), transactionStore);
+        eclat.map(mockOneDocument(List.of(tuple(field1, "f1-c"), tuple(field2, "f2-1"), tuple(field3, "f3-H"))), transactionStore);
+        eclat.map(mockOneDocument(List.of(tuple(field1, "f1-d"), tuple(field2, "f2-1"), tuple(field3, "f3-I"))), transactionStore);
+        eclat.map(mockOneDocument(List.of(tuple(field1, "f1-b"), tuple(field2, "f2-1"), tuple(field3, "f3-J"))), transactionStore);
+        eclat.map(mockOneDocument(List.of(tuple(field1, "f1-f"), tuple(field2, "f2-1"), tuple(field3, "f3-K"))), transactionStore);
+        eclat.map(mockOneDocument(List.of(tuple(field1, "f1-a"), tuple(field2, "f2-1"), tuple(field3, "f3-L"))), transactionStore);
+
+        EclatMapReducer.EclatResult result = runEclat(eclat, List.of(field1, field2, field3), transactionStore);
+        assertThat(result.getFrequentItemSets().length, equalTo(2));
+        assertThat(result.getFrequentItemSets()[0].getSupport(), equalTo(0.5));
+        assertThat(result.getFrequentItemSets()[1].getSupport(), equalTo(0.25));
+        assertThat(result.getProfilingInfo().get("unique_items_after_reduce"), equalTo(18L));
+        assertThat(result.getProfilingInfo().get("total_transactions_after_reduce"), equalTo(12L));
+        assertThat(result.getProfilingInfo().get("total_items_after_reduce"), equalTo(36L));
+    }
+
+    public void testPruneToNextMainBranch() throws IOException {
+        Field field1 = createKeywordFieldTestInstance("keyword1", 0);
+        Field field2 = createKeywordFieldTestInstance("keyword2", 1);
+        Field field3 = createKeywordFieldTestInstance("keyword3", 2);
+
+        // create 3 fields that "follow" field 2
+        Field field2a = createKeywordFieldTestInstance("keyword2a", 3);
+        Field field2b = createKeywordFieldTestInstance("keyword2b", 4);
+        Field field2c = createKeywordFieldTestInstance("keyword2c", 5);
+
+        EclatMapReducer eclat = new EclatMapReducer(getTestName(), 0.1, 2, 10, true);
+
+        HashBasedTransactionStore transactionStore = eclat.mapInit(mockBigArrays());
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-a"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-A"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-a"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-B"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-a"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-C"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-a"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-D"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-a"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-E"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-b"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-F"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-b"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-G"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-c"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-H"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-d"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-I"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-b"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-J"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-f"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-K"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-a"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-L"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1")
+                )
+            ),
+            transactionStore
+        );
+
+        EclatMapReducer.EclatResult result = runEclat(eclat, List.of(field1, field2, field3, field2a, field2b, field2c), transactionStore);
+        assertThat(result.getFrequentItemSets().length, equalTo(3));
+        assertThat(result.getFrequentItemSets()[0].getSupport(), equalTo(1.0));
+        assertThat(result.getFrequentItemSets()[1].getSupport(), equalTo(0.5));
+        assertThat(result.getFrequentItemSets()[2].getSupport(), equalTo(0.25));
+        assertThat(result.getProfilingInfo().get("unique_items_after_reduce"), equalTo(21L));
+        assertThat(result.getProfilingInfo().get("total_transactions_after_reduce"), equalTo(12L));
+        assertThat(result.getProfilingInfo().get("total_items_after_reduce"), equalTo(72L));
+        assertThat(result.getProfilingInfo().get("item_sets_checked_eclat"), equalTo(47L));
+    }
+
+    public void testPruneToNextMainBranchAfterMinCountPrune() throws IOException {
+        Field field1 = createKeywordFieldTestInstance("keyword1", 0);
+        Field field2 = createKeywordFieldTestInstance("keyword2", 1);
+        Field field3 = createKeywordFieldTestInstance("keyword3", 2);
+
+        // create 3 fields that "follow" field 2
+        Field field2a = createKeywordFieldTestInstance("keyword2a", 3);
+        Field field2b = createKeywordFieldTestInstance("keyword2b", 4);
+        Field field2c = createKeywordFieldTestInstance("keyword2c", 5);
+
+        // create another field to enforce min count pruning
+        Field field4 = createKeywordFieldTestInstance("keyword4", 6);
+        Field field4a = createKeywordFieldTestInstance("keyword4a", 7);
+
+        EclatMapReducer eclat = new EclatMapReducer(getTestName(), 0.1, 2, 10, true);
+
+        HashBasedTransactionStore transactionStore = eclat.mapInit(mockBigArrays());
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-a"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-A"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1"),
+                    tuple(field4, "f4-1"),
+                    tuple(field4a, "f4a-1")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-a"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-B"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1"),
+                    tuple(field4, "f4-2"),
+                    tuple(field4a, "f4a-2")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-a"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-C"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1"),
+                    tuple(field4, "f4-3"),
+                    tuple(field4a, "f4a-3")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-a"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-D"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1"),
+                    tuple(field4, "f4-4"),
+                    tuple(field4a, "f4a-4")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-a"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-E"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1"),
+                    tuple(field4, "f4-5"),
+                    tuple(field4a, "f4a-5")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-b"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-F"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1"),
+                    tuple(field4, "f4-1"),
+                    tuple(field4a, "f4a-1")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-b"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-G"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1"),
+                    tuple(field4, "f4-1"),
+                    tuple(field4a, "f4a-1")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-c"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-H"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1"),
+                    tuple(field4, "f4-6"),
+                    tuple(field4a, "f4a-6")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-d"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-I"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1"),
+                    tuple(field4, "f4-7"),
+                    tuple(field4a, "f4a-7")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-b"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-J"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1"),
+                    tuple(field4, "f4-8"),
+                    tuple(field4a, "f4a-8")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-f"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-K"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1"),
+                    tuple(field4, "f4-3"),
+                    tuple(field4a, "f4a-3")
+                )
+            ),
+            transactionStore
+        );
+        eclat.map(
+            mockOneDocument(
+                List.of(
+                    tuple(field1, "f1-a"),
+                    tuple(field2, "f2-1"),
+                    tuple(field3, "f3-L"),
+                    tuple(field2a, "f2a-1"),
+                    tuple(field2b, "f2b-1"),
+                    tuple(field2c, "f2c-1"),
+                    tuple(field4, "f4-10"),
+                    tuple(field4a, "f4a-10")
+                )
+            ),
+            transactionStore
+        );
+
+        EclatMapReducer.EclatResult result = runEclat(
+            eclat,
+            List.of(field1, field2, field3, field2a, field2b, field2c, field4, field4a),
+            transactionStore
+        );
+
+        assertThat(result.getFrequentItemSets().length, equalTo(6));
+        assertThat(result.getFrequentItemSets()[0].getSupport(), equalTo(1.0));
+        assertThat(result.getFrequentItemSets()[1].getSupport(), equalTo(0.5));
+        assertThat(result.getFrequentItemSets()[2].getSupport(), equalTo(0.25));
+        assertThat(result.getProfilingInfo().get("unique_items_after_reduce"), equalTo(39L));
+        assertThat(result.getProfilingInfo().get("unique_items_after_prune"), equalTo(10L));
+        assertThat(result.getProfilingInfo().get("total_transactions_after_reduce"), equalTo(12L));
+        assertThat(result.getProfilingInfo().get("total_items_after_reduce"), equalTo(96L));
+        assertThat(result.getProfilingInfo().get("total_items_after_prune"), equalTo(96L));
+
+        // the number can vary depending on order, so we can only check a range, which is still much lower than without
+        // that optimization
+        assertThat((Long) result.getProfilingInfo().get("item_sets_checked_eclat"), greaterThanOrEqualTo(294L));
+        assertThat((Long) result.getProfilingInfo().get("item_sets_checked_eclat"), lessThan(310L));
+    }
+
+    private static BigArrays mockBigArrays() {
+        return new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
+    }
+
+    private static Tuple<Field, List<Object>> mockOneField(Field field, String... fieldValues) {
+        return tuple(field, Arrays.stream(fieldValues).map(v -> new BytesRef(v)).collect(Collectors.toList()));
+    }
+
+    private static Stream<Tuple<Field, List<Object>>> mockOneDocument(List<Tuple<Field, String>> fieldsAndValues) {
+        return fieldsAndValues.stream().map(fieldAndValue -> mockOneField(fieldAndValue.v1(), fieldAndValue.v2()));
+    }
+
+    private static EclatMapReducer.EclatResult runEclat(
+        EclatMapReducer eclat,
+        List<Field> fields,
+        HashBasedTransactionStore... transactionStores
+    ) throws IOException {
+        HashBasedTransactionStore transactionStoreForReduce = eclat.reduceInit(mockBigArrays());
+
+        for (HashBasedTransactionStore transactionStore : transactionStores) {
+            ImmutableTransactionStore transactionStoreAfterFinalizing = eclat.mapFinalize(transactionStore, null);
+            List<ImmutableTransactionStore> allPartitions = List.of(transactionStoreAfterFinalizing);
+            eclat.reduce(allPartitions.stream(), transactionStoreForReduce, doNotCancelSupplier);
+        }
+        return eclat.reduceFinalize(transactionStoreForReduce, fields, doNotCancelSupplier);
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetCollectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetCollectorTests.java
index 671f7b3a7c07..847494ecb49d 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetCollectorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetCollectorTests.java
@@ -170,6 +170,20 @@ public void testLargerItemSetsPreference() {
         }
     }
 
+    public void testSuperSetAfterSubSet() {
+        transactionStore = new HashBasedTransactionStore(mockBigArrays());
+
+        try (TopItemIds topItemIds = transactionStore.getTopItemIds();) {
+            FrequentItemSetCollector collector = new FrequentItemSetCollector(transactionStore, topItemIds, 5, Long.MAX_VALUE);
+
+            assertEquals(Long.MAX_VALUE, addToCollector(collector, new long[] { 1L, 2L, 3L, 4L, 6L }, 3L));
+            assertEquals(Long.MAX_VALUE, addToCollector(collector, new long[] { 1L, 2L, 3L, 4L, 6L, 8L }, 3L));
+
+            assertEquals(1, collector.size());
+            assertThat(collector.getQueue().pop().getItems(), equalTo(createItemSetBitSet(new long[] { 1L, 2L, 3L, 4L, 6L, 8L })));
+        }
+    }
+
     private static ItemSetBitSet createItemSetBitSet(long[] longs) {
         ItemSetBitSet itemsAsBitVector = new ItemSetBitSet();
         for (int i = 0; i < longs.length; ++i) {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetWireSerializationTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetWireSerializationTests.java
index 8d242bbaddab..f7da422fd834 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetWireSerializationTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetWireSerializationTests.java
@@ -39,4 +39,9 @@ protected FrequentItemSet createTestInstance() {
         return randomFrequentItemSet();
     }
 
+    @Override
+    protected FrequentItemSet mutateInstance(FrequentItemSet instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregationBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregationBuilderTests.java
index 61b60bcea8cd..9534ace3d3b9 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregationBuilderTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregationBuilderTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.BaseAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
 import org.elasticsearch.search.aggregations.support.MultiValuesSourceFieldConfig;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -27,8 +28,10 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.TreeSet;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.xpack.ml.aggs.frequentitemsets.FrequentItemSetsAggregationBuilder.EXECUTION_HINT_ALLOWED_MODES;
 import static org.hamcrest.Matchers.hasSize;
 
 public class FrequentItemSetsAggregationBuilderTests extends AbstractXContentSerializingTestCase<FrequentItemSetsAggregationBuilder> {
@@ -47,6 +50,10 @@ public static FrequentItemSetsAggregationBuilder randomFrequentItemsSetsAggregat
                 field.setMissing(randomAlphaOfLength(5));
             }
 
+            if (randomBoolean()) {
+                field.setIncludeExclude(randomIncludeExclude());
+            }
+
             return field.build();
         }).collect(Collectors.toList());
 
@@ -56,7 +63,8 @@ public static FrequentItemSetsAggregationBuilder randomFrequentItemsSetsAggregat
             randomDoubleBetween(0.0, 1.0, false),
             randomIntBetween(1, 20),
             randomIntBetween(1, 20),
-            randomBoolean() ? QueryBuilders.termQuery(randomAlphaOfLength(10), randomAlphaOfLength(10)) : null
+            randomBoolean() ? QueryBuilders.termQuery(randomAlphaOfLength(10), randomAlphaOfLength(10)) : null,
+            randomFrom(EXECUTION_HINT_ALLOWED_MODES)
         );
     }
 
@@ -83,6 +91,11 @@ protected FrequentItemSetsAggregationBuilder createTestInstance() {
         return randomFrequentItemsSetsAggregationBuilder();
     }
 
+    @Override
+    protected FrequentItemSetsAggregationBuilder mutateInstance(FrequentItemSetsAggregationBuilder instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
         return new NamedWriteableRegistry(new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedWriteables());
@@ -114,7 +127,8 @@ public void testValidation() {
                 1.2,
                 randomIntBetween(1, 20),
                 randomIntBetween(1, 20),
-                null
+                null,
+                randomFrom(EXECUTION_HINT_ALLOWED_MODES)
             )
         );
         assertEquals("[minimum_support] must be greater than 0 and less or equal to 1. Found [1.2] in [fi]", e.getMessage());
@@ -130,7 +144,8 @@ public void testValidation() {
                 randomDoubleBetween(0.0, 1.0, false),
                 -4,
                 randomIntBetween(1, 20),
-                null
+                null,
+                randomFrom(EXECUTION_HINT_ALLOWED_MODES)
             )
         );
 
@@ -147,7 +162,8 @@ public void testValidation() {
                 randomDoubleBetween(0.0, 1.0, false),
                 randomIntBetween(1, 20),
                 -2,
-                null
+                null,
+                randomFrom(EXECUTION_HINT_ALLOWED_MODES)
             )
         );
 
@@ -164,26 +180,72 @@ public void testValidation() {
             randomDoubleBetween(0.0, 1.0, false),
             randomIntBetween(1, 20),
             randomIntBetween(1, 20),
-            null
+            null,
+            randomFrom(EXECUTION_HINT_ALLOWED_MODES)
         ).subAggregation(AggregationBuilders.avg("fieldA")));
 
-        assertEquals("Aggregator [fi] of type [frequent_items] cannot accept sub-aggregations", e.getMessage());
+        assertEquals("Aggregator [fi] of type [frequent_item_sets] cannot accept sub-aggregations", e.getMessage());
 
-        e = expectThrows(IllegalArgumentException.class, () ->
+        e = expectThrows(
+            IllegalArgumentException.class,
+            () -> new FrequentItemSetsAggregationBuilder(
+                "fi",
+                List.of(
+                    new MultiValuesSourceFieldConfig.Builder().setFieldName("fieldA").build(),
+                    new MultiValuesSourceFieldConfig.Builder().setFieldName("fieldB").build()
+                ),
+                randomDoubleBetween(0.0, 1.0, false),
+                randomIntBetween(1, 20),
+                randomIntBetween(1, 20),
+                null,
+                randomFrom(EXECUTION_HINT_ALLOWED_MODES)
+            ).subAggregations(new AggregatorFactories.Builder().addAggregator(AggregationBuilders.avg("fieldA")))
+        );
 
-        new FrequentItemSetsAggregationBuilder(
-            "fi",
-            List.of(
-                new MultiValuesSourceFieldConfig.Builder().setFieldName("fieldA").build(),
-                new MultiValuesSourceFieldConfig.Builder().setFieldName("fieldB").build()
-            ),
-            randomDoubleBetween(0.0, 1.0, false),
-            randomIntBetween(1, 20),
-            randomIntBetween(1, 20),
-            null
-        ).subAggregations(new AggregatorFactories.Builder().addAggregator(AggregationBuilders.avg("fieldA"))));
+        assertEquals("Aggregator [fi] of type [frequent_item_sets] cannot accept sub-aggregations", e.getMessage());
+
+        e = expectThrows(
+            IllegalArgumentException.class,
+            () -> new FrequentItemSetsAggregationBuilder(
+                "fi",
+                List.of(
+                    new MultiValuesSourceFieldConfig.Builder().setFieldName("fieldA").build(),
+                    new MultiValuesSourceFieldConfig.Builder().setFieldName("fieldB").build()
+                ),
+                randomDoubleBetween(0.0, 1.0, false),
+                randomIntBetween(1, 20),
+                randomIntBetween(1, 20),
+                null,
+                "wrong-execution-mode"
+            )
+        );
 
-        assertEquals("Aggregator [fi] of type [frequent_items] cannot accept sub-aggregations", e.getMessage());
+        assertEquals("[execution_hint] must be one of [global_ordinals,map]. Found [wrong-execution-mode]", e.getMessage());
     }
 
+    private static IncludeExclude randomIncludeExclude() {
+        switch (randomInt(7)) {
+            case 0:
+                return new IncludeExclude("incl*de", null, null, null);
+            case 1:
+                return new IncludeExclude("incl*de", "excl*de", null, null);
+            case 2:
+                return new IncludeExclude("incl*de", null, null, new TreeSet<>(Set.of(newBytesRef("exclude"))));
+            case 3:
+                return new IncludeExclude(null, "excl*de", null, null);
+            case 4:
+                return new IncludeExclude(null, "excl*de", new TreeSet<>(Set.of(newBytesRef("include"))), null);
+            case 5:
+                return new IncludeExclude(null, null, new TreeSet<>(Set.of(newBytesRef("include"))), null);
+            case 6:
+                return new IncludeExclude(
+                    null,
+                    null,
+                    new TreeSet<>(Set.of(newBytesRef("include"))),
+                    new TreeSet<>(Set.of(newBytesRef("exclude")))
+                );
+            default:
+                return new IncludeExclude(null, null, null, new TreeSet<>(Set.of(newBytesRef("exclude"))));
+        }
+    }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregatorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregatorTests.java
index baa4199032a6..d102c89cba03 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregatorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/FrequentItemSetsAggregatorTests.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.IpFieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
@@ -27,6 +28,7 @@
 import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.aggregations.support.MultiValuesSourceFieldConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
@@ -39,11 +41,17 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeSet;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.core.Tuple.tuple;
+import static org.elasticsearch.xpack.ml.aggs.frequentitemsets.FrequentItemSetsAggregationBuilder.EXECUTION_HINT_ALLOWED_MODES;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 
 public class FrequentItemSetsAggregatorTests extends AggregatorTestCase {
@@ -80,14 +88,22 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy
             FrequentItemSetsAggregationBuilder.DEFAULT_MINIMUM_SUPPORT,
             FrequentItemSetsAggregationBuilder.DEFAULT_MINIMUM_SET_SIZE,
             FrequentItemSetsAggregationBuilder.DEFAULT_SIZE,
-            null
+            null,
+            randomFrom(EXECUTION_HINT_ALLOWED_MODES)
         );
     }
 
     public void testKeywordsArray() throws IOException {
         List<MultiValuesSourceFieldConfig> fields = new ArrayList<>();
 
-        fields.add(new MultiValuesSourceFieldConfig.Builder().setFieldName(KEYWORD_FIELD1).build());
+        String exclude = randomBoolean() ? randomFrom("item-3", "item-4", "item-5", "item-99") : null;
+        fields.add(
+            new MultiValuesSourceFieldConfig.Builder().setFieldName(KEYWORD_FIELD1)
+                .setIncludeExclude(
+                    exclude != null ? new IncludeExclude(null, null, null, new TreeSet<>(Set.of(new BytesRef(exclude)))) : null
+                )
+                .build()
+        );
 
         double minimumSupport = randomDoubleBetween(0.13, 0.41, true);
         int minimumSetSize = randomIntBetween(2, 5);
@@ -120,7 +136,8 @@ public void testKeywordsArray() throws IOException {
             minimumSupport,
             minimumSetSize,
             size,
-            null
+            null,
+            randomFrom(EXECUTION_HINT_ALLOWED_MODES)
         );
 
         testCase(iw -> {
@@ -205,19 +222,66 @@ public void testKeywordsArray() throws IOException {
             );
         }, (InternalItemSetMapReduceAggregation<?, ?, ?, EclatResult> results) -> {
             assertNotNull(results);
-            assertResults(expectedResults, results.getMapReduceResult().getFrequentItemSets(), minimumSupport, minimumSetSize, size);
+            assertResults(
+                expectedResults,
+                results.getMapReduceResult().getFrequentItemSets(),
+                minimumSupport,
+                minimumSetSize,
+                size,
+                exclude,
+                null
+            );
         }, new AggTestConfig(builder, keywordType).withQuery(query));
     }
 
     public void testMixedSingleValues() throws IOException {
         List<MultiValuesSourceFieldConfig> fields = new ArrayList<>();
 
-        fields.add(new MultiValuesSourceFieldConfig.Builder().setFieldName(KEYWORD_FIELD1).build());
-        fields.add(new MultiValuesSourceFieldConfig.Builder().setFieldName(KEYWORD_FIELD2).build());
-        fields.add(new MultiValuesSourceFieldConfig.Builder().setFieldName(KEYWORD_FIELD3).build());
+        String stringExclude = randomBoolean() ? randomFrom("host-2", "192.168.0.1", "client-2", "127.0.0.1") : null;
+        Integer intExclude = randomBoolean() ? randomIntBetween(0, 10) : null;
+
+        fields.add(
+            new MultiValuesSourceFieldConfig.Builder().setFieldName(KEYWORD_FIELD1)
+                .setIncludeExclude(
+                    stringExclude != null ? new IncludeExclude(null, null, null, new TreeSet<>(Set.of(new BytesRef(stringExclude)))) : null
+                )
+                .build()
+        );
+        fields.add(
+            new MultiValuesSourceFieldConfig.Builder().setFieldName(KEYWORD_FIELD2)
+                .setIncludeExclude(
+                    stringExclude != null ? new IncludeExclude(null, null, null, new TreeSet<>(Set.of(new BytesRef(stringExclude)))) : null
+                )
+                .build()
+        );
+        fields.add(
+            new MultiValuesSourceFieldConfig.Builder().setFieldName(KEYWORD_FIELD3)
+                .setIncludeExclude(
+                    stringExclude != null ? new IncludeExclude(null, null, null, new TreeSet<>(Set.of(new BytesRef(stringExclude)))) : null
+                )
+                .build()
+        );
         fields.add(new MultiValuesSourceFieldConfig.Builder().setFieldName(FLOAT_FIELD).build());
-        fields.add(new MultiValuesSourceFieldConfig.Builder().setFieldName(INT_FIELD).build());
-        fields.add(new MultiValuesSourceFieldConfig.Builder().setFieldName(IP_FIELD).build());
+        fields.add(
+            new MultiValuesSourceFieldConfig.Builder().setFieldName(INT_FIELD)
+                .setIncludeExclude(
+                    intExclude != null
+                        ? new IncludeExclude(null, null, null, new TreeSet<>(Set.of(new BytesRef(String.valueOf(intExclude)))))
+                        : null
+                )
+                .build()
+        );
+        fields.add(
+            new MultiValuesSourceFieldConfig.Builder().setFieldName(IP_FIELD)
+                .setIncludeExclude(
+                    stringExclude != null
+                        ? stringExclude.startsWith("1") // only add exclude if it is an IP
+                            ? new IncludeExclude(null, null, null, new TreeSet<>(Set.of(new BytesRef(stringExclude))))
+                            : null
+                        : null
+                )
+                .build()
+        );
 
         double minimumSupport = randomDoubleBetween(0.13, 0.51, true);
         int minimumSetSize = randomIntBetween(2, 6);
@@ -264,9 +328,9 @@ public void testMixedSingleValues() throws IOException {
                 0.2
             ),
             new FrequentItemSet(Map.of(IP_FIELD, List.of("192.168.0.1"), FLOAT_FIELD, List.of(4.1f), INT_FIELD, List.of(2)), 2, 0.2),
-            new FrequentItemSet(Map.of(FLOAT_FIELD, List.of(5), KEYWORD_FIELD1, List.of("host-2")), 2, 0.2),
+            new FrequentItemSet(Map.of(FLOAT_FIELD, List.of(5.0f), KEYWORD_FIELD1, List.of("host-2")), 2, 0.2),
             new FrequentItemSet(Map.of(INT_FIELD, List.of(5), KEYWORD_FIELD1, List.of("host-2")), 2, 0.2),
-            new FrequentItemSet(Map.of(FLOAT_FIELD, List.of(5), KEYWORD_FIELD2, List.of("client-2")), 2, 0.2),
+            new FrequentItemSet(Map.of(FLOAT_FIELD, List.of(5.0f), KEYWORD_FIELD2, List.of("client-2")), 2, 0.2),
             new FrequentItemSet(Map.of(IP_FIELD, List.of("192.168.0.5"), KEYWORD_FIELD2, List.of("client-2")), 2, 0.2)
         );
 
@@ -276,7 +340,8 @@ public void testMixedSingleValues() throws IOException {
             minimumSupport,
             minimumSetSize,
             size,
-            null
+            null,
+            randomFrom(EXECUTION_HINT_ALLOWED_MODES)
         );
 
         testCase(iw -> {
@@ -382,7 +447,15 @@ public void testMixedSingleValues() throws IOException {
             );
         }, (InternalItemSetMapReduceAggregation<?, ?, ?, EclatResult> results) -> {
             assertNotNull(results);
-            assertResults(expectedResults, results.getMapReduceResult().getFrequentItemSets(), minimumSupport, minimumSetSize, size);
+            assertResults(
+                expectedResults,
+                results.getMapReduceResult().getFrequentItemSets(),
+                minimumSupport,
+                minimumSetSize,
+                size,
+                stringExclude,
+                intExclude
+            );
         }, new AggTestConfig(builder, keywordType1, keywordType2, keywordType3, intType, floatType, ipType).withQuery(query));
 
     }
@@ -390,10 +463,18 @@ public void testMixedSingleValues() throws IOException {
     public void testSingleValueWithDate() throws IOException {
         List<MultiValuesSourceFieldConfig> fields = new ArrayList<>();
 
+        String dateExclude = randomBoolean() ? randomFrom("2022-06-02", "2022-06-03", "1970-01-01") : null;
+
         fields.add(new MultiValuesSourceFieldConfig.Builder().setFieldName(KEYWORD_FIELD1).build());
         fields.add(new MultiValuesSourceFieldConfig.Builder().setFieldName(KEYWORD_FIELD2).build());
         fields.add(new MultiValuesSourceFieldConfig.Builder().setFieldName(KEYWORD_FIELD3).build());
-        fields.add(new MultiValuesSourceFieldConfig.Builder().setFieldName(DATE_FIELD).build());
+        fields.add(
+            new MultiValuesSourceFieldConfig.Builder().setFieldName(DATE_FIELD)
+                .setIncludeExclude(
+                    dateExclude != null ? new IncludeExclude(null, null, null, new TreeSet<>(Set.of(new BytesRef(dateExclude)))) : null
+                )
+                .build()
+        );
 
         double minimumSupport = randomDoubleBetween(0.13, 0.51, true);
         int minimumSetSize = randomIntBetween(2, 6);
@@ -475,7 +556,8 @@ public void testSingleValueWithDate() throws IOException {
             minimumSupport,
             minimumSetSize,
             size,
-            null
+            null,
+            randomFrom(EXECUTION_HINT_ALLOWED_MODES)
         );
 
         testCase(iw -> {
@@ -571,7 +653,15 @@ public void testSingleValueWithDate() throws IOException {
             );
         }, (InternalItemSetMapReduceAggregation<?, ?, ?, EclatResult> results) -> {
             assertNotNull(results);
-            assertResults(expectedResults, results.getMapReduceResult().getFrequentItemSets(), minimumSupport, minimumSetSize, size);
+            assertResults(
+                expectedResults,
+                results.getMapReduceResult().getFrequentItemSets(),
+                minimumSupport,
+                minimumSetSize,
+                size,
+                dateExclude,
+                null
+            );
         }, new AggTestConfig(builder, keywordType1, keywordType2, keywordType3, dateType, ipType).withQuery(query));
 
     }
@@ -594,11 +684,45 @@ private DateFieldMapper.DateFieldType dateFieldType(String name) {
         );
     }
 
-    private void assertResults(List<FrequentItemSet> expected, FrequentItemSet[] actual, double minSupport, int minimumSetSize, int size) {
+    private void assertResults(
+        List<FrequentItemSet> expected,
+        FrequentItemSet[] actual,
+        double minSupport,
+        int minimumSetSize,
+        int size,
+        String stringExclude,
+        Integer intExclude
+    ) {
         // sort the expected results descending by doc count
         expected.get(0).getFields().values().stream().mapToLong(v -> v.stream().count()).sum();
 
-        List<FrequentItemSet> filteredExpected = expected.stream()
+        List<FrequentItemSet> filteredExpectedWithDups = expected.stream()
+            .map(
+                fi -> new FrequentItemSet(
+                    fi.getFields()
+                        .entrySet()
+                        .stream()
+                        .map(
+                            // filter the string exclude from the list of objects
+                            keyValues -> tuple(
+                                keyValues.getKey(),
+                                keyValues.getValue().stream().filter(v -> v.equals(stringExclude) == false).collect(Collectors.toList())
+                            )
+                        )
+                        .map(
+                            // filter the int exclude
+                            keyValues -> tuple(
+                                keyValues.v1(),
+                                keyValues.v2().stream().filter(v -> v.equals(intExclude) == false).collect(Collectors.toList())
+                            )
+                        )
+                        // after filtering out excludes the list of objects might be empty
+                        .filter(t -> t.v2().size() > 0)
+                        .collect(Collectors.toMap(Tuple::v1, Tuple::v2)),
+                    fi.getDocCount(),
+                    fi.getSupport()
+                )
+            )
             .filter(fi -> fi.getSupport() >= minSupport)
             .filter(
                 fi -> {
@@ -638,6 +762,17 @@ private void assertResults(List<FrequentItemSet> expected, FrequentItemSet[] act
             })
             .collect(Collectors.toList());
 
+        // after removing excluded items there might be duplicate entries, which need to be collapsed, we do this by very simple hashing
+        List<FrequentItemSet> filteredExpected = new ArrayList<>();
+        Set<Integer> valuesSeen = new HashSet<>();
+        for (FrequentItemSet fi : filteredExpectedWithDups) {
+            if (valuesSeen.add(
+                fi.getFields().entrySet().stream().mapToInt(v -> Objects.hash(v.getKey(), v.getValue())).reduce(13, (t, s) -> 41 * t + s)
+            )) {
+                filteredExpected.add(fi);
+            }
+        }
+
         // if size applies, cut the list, however if sets have the same number of items it's unclear which ones are returned
         int additionalSetsThatShareTheSameDocCount = 0;
         if (size < filteredExpected.size()) {
@@ -661,7 +796,12 @@ private void assertResults(List<FrequentItemSet> expected, FrequentItemSet[] act
             + " minimum_set_size: "
             + minimumSetSize
             + " size: "
-            + size;
+            + size
+            + " string exclude: ["
+            + stringExclude
+            + "] int exclude: ["
+            + intExclude
+            + "]";
 
         assertEquals(
             "number of results do not match, " + setsAssertMessage,
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/ItemSetBitSetTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/ItemSetBitSetTests.java
index e40decb0883c..49acb01ddb2a 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/ItemSetBitSetTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/ItemSetBitSetTests.java
@@ -44,7 +44,7 @@ public void testBasics() {
         assertTrue(set.get(200));
     }
 
-    public void testIsSubSet() {
+    public void testIsSubSetAndSetRelation() {
         ItemSetBitSet set1 = new ItemSetBitSet();
         set1.set(0);
         set1.set(3);
@@ -60,24 +60,39 @@ public void testIsSubSet() {
 
         assertEquals(3, set2.cardinality());
         assertTrue(set2.isSubset(set1));
+        assertEquals(ItemSetBitSet.SetRelation.SUB_SET, set2.setRelation(set1));
         assertFalse(set1.isSubset(set2));
+        assertEquals(ItemSetBitSet.SetRelation.SUPER_SET, set1.setRelation(set2));
         assertTrue(set1.isSubset(set1));
+        assertEquals(ItemSetBitSet.SetRelation.EQUAL, set1.setRelation(set1));
 
         set2.set(0);
         set2.set(5);
         assertTrue(set2.isSubset(set1));
         assertTrue(set1.isSubset(set2));
+        assertEquals(ItemSetBitSet.SetRelation.EQUAL, set1.setRelation(set2));
+        assertEquals(ItemSetBitSet.SetRelation.EQUAL, set2.setRelation(set1));
 
         set2.set(99);
         assertFalse(set2.isSubset(set1));
         assertTrue(set1.isSubset(set2));
+        assertEquals(ItemSetBitSet.SetRelation.SUPER_SET, set2.setRelation(set1));
+        assertEquals(ItemSetBitSet.SetRelation.SUB_SET, set1.setRelation(set2));
 
         set1.set(999);
         assertFalse(set1.isSubset(set2));
+        assertEquals(ItemSetBitSet.SetRelation.DISJOINT_OR_INTERSECT, set2.setRelation(set1));
+        assertEquals(ItemSetBitSet.SetRelation.DISJOINT_OR_INTERSECT, set1.setRelation(set2));
+
         set2.set(999);
         assertTrue(set1.isSubset(set2));
+        assertEquals(ItemSetBitSet.SetRelation.SUPER_SET, set2.setRelation(set1));
+        assertEquals(ItemSetBitSet.SetRelation.SUB_SET, set1.setRelation(set2));
+
         set2.set(2222);
         assertTrue(set1.isSubset(set2));
+        assertEquals(ItemSetBitSet.SetRelation.SUPER_SET, set2.setRelation(set1));
+        assertEquals(ItemSetBitSet.SetRelation.SUB_SET, set1.setRelation(set2));
     }
 
     public void testClone() {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/InternalItemSetMapReduceAggregationTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/InternalItemSetMapReduceAggregationTests.java
index 270cd1d0d9b3..b7b04b0e88b5 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/InternalItemSetMapReduceAggregationTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/InternalItemSetMapReduceAggregationTests.java
@@ -50,6 +50,13 @@ public class InternalItemSetMapReduceAggregationTests extends InternalAggregatio
 
     private static String[] WORDS = new String[] { "apple", "banana", "orange", "peach", "strawberry" };
 
+    @Override
+    protected InternalItemSetMapReduceAggregation<WordCounts, WordCounts, WordCounts, WordCounts> mutateInstance(
+        InternalItemSetMapReduceAggregation<WordCounts, WordCounts, WordCounts, WordCounts> instance
+    ) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     static class WordCountMapReducer extends AbstractItemSetMapReducer<WordCounts, WordCounts, WordCounts, WordCounts> {
 
         static class WordCounts implements ToXContent, Writeable, Closeable {
@@ -142,7 +149,7 @@ public WordCounts reduceFinalize(WordCounts wordCounts, List<Field> fields, Supp
         }
 
         @Override
-        protected WordCounts mapFinalize(WordCounts mapReduceContext) {
+        protected WordCounts mapFinalize(WordCounts mapReduceContext, List<OrdinalLookupFunction> ordinalLookup) {
             return mapReduceContext;
         }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/ItemSetMapReduceValueSourceSerializationTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/ItemSetMapReduceValueSourceSerializationTests.java
index 87b3813ba48c..86dc60352344 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/ItemSetMapReduceValueSourceSerializationTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/ItemSetMapReduceValueSourceSerializationTests.java
@@ -41,6 +41,11 @@ protected Field createTestInstance() {
         throw new AssertionError("field type missing");
     }
 
+    @Override
+    protected Field mutateInstance(Field instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Before
     public void registerNamedObjects() {
         SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/inference/InternalInferenceAggregationTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/inference/InternalInferenceAggregationTests.java
index ce0f8e2b91a8..5dc7f43118dd 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/inference/InternalInferenceAggregationTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/inference/InternalInferenceAggregationTests.java
@@ -223,4 +223,9 @@ public void testGetProperty_topClasses() {
             expectThrows(InvalidAggregationPathException.class, () -> internalAgg.getProperty(Collections.singletonList("top_classes")));
         }
     }
+
+    @Override
+    protected InternalInferenceAggregation mutateInstance(InternalInferenceAggregation instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/kstest/InternalKSTestAggregationTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/kstest/InternalKSTestAggregationTests.java
index 3a7712cf304b..2085e44925cc 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/kstest/InternalKSTestAggregationTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/kstest/InternalKSTestAggregationTests.java
@@ -68,4 +68,9 @@ protected void assertFromXContent(InternalKSTestAggregation aggregation, ParsedA
         ParsedKSTest ks = (ParsedKSTest) parsedAggregation;
         assertThat(ks.getModes(), equalTo(aggregation.getModeValues()));
     }
+
+    @Override
+    protected InternalKSTestAggregation mutateInstance(InternalKSTestAggregation instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlScalingReasonTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlScalingReasonTests.java
index bd02e2e79755..cc576423f7cf 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlScalingReasonTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlScalingReasonTests.java
@@ -48,6 +48,11 @@ protected MlScalingReason createTestInstance() {
         );
     }
 
+    @Override
+    protected MlScalingReason mutateInstance(MlScalingReason instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     protected static AutoscalingCapacity.AutoscalingResources randomAutoscalingResources() {
         return new AutoscalingCapacity.AutoscalingResources(
             ByteSizeValue.ofBytes(randomLongBetween(10, ByteSizeValue.ofGb(10).getBytes())),
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfigAutoUpdaterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfigAutoUpdaterTests.java
index 4ceefdee4da8..6082ef5789f3 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfigAutoUpdaterTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfigAutoUpdaterTests.java
@@ -177,7 +177,8 @@ public void testIsAbleToRun() {
             shardId,
             true,
             RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""),
+            ShardRouting.Role.DEFAULT
         );
         shardRouting = shardRouting.initialize("node_id", null, 0L);
         shardRouting = shardRouting.moveToStarted(ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
@@ -202,7 +203,8 @@ public void testIsAbleToRun() {
                 shardId,
                 true,
                 RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")
+                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""),
+                ShardRouting.Role.DEFAULT
             );
             shardRouting = shardRouting.initialize("node_id", null, 0L);
             routingTable.add(
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java
index 1dcad2b372bc..0a4bc016ed18 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java
@@ -722,7 +722,8 @@ private static RoutingTable generateRoutingTable(IndexMetadata indexMetadata, Li
                     shardId,
                     true,
                     RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")
+                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""),
+                    ShardRouting.Role.DEFAULT
                 );
             }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedRunnerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedRunnerTests.java
index f7f993a54b63..1d2773397d80 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedRunnerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedRunnerTests.java
@@ -130,6 +130,7 @@ public void setUpTests() {
         when(datafeedJob.stop()).thenReturn(true);
         when(datafeedJob.getJobId()).thenReturn(job.getId());
         when(datafeedJob.getMaxEmptySearches()).thenReturn(null);
+        when(datafeedJob.numberOfSearchesIn24Hours()).thenReturn(24L);
         datafeedJobBuilder = mock(DatafeedJobBuilder.class);
         doAnswer(invocationOnMock -> {
             ActionListener<DatafeedJob> listener = (ActionListener<DatafeedJob>) invocationOnMock.getArguments()[2];
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedTimingStatsReporterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedTimingStatsReporterTests.java
index 35267efcf493..54a791d0e577 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedTimingStatsReporterTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedTimingStatsReporterTests.java
@@ -7,6 +7,8 @@
 package org.elasticsearch.xpack.ml.datafeed;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESTestCase;
@@ -23,6 +25,8 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.inOrder;
 import static org.mockito.Mockito.mock;
@@ -40,6 +44,12 @@ public class DatafeedTimingStatsReporterTests extends ESTestCase {
     @Before
     public void setUpTests() {
         timingStatsPersister = mock(DatafeedTimingStatsPersister.class);
+        doAnswer(invocationOnMock -> {
+            @SuppressWarnings("unchecked")
+            ActionListener<BulkResponse> listener = (ActionListener<BulkResponse>) invocationOnMock.getArguments()[2];
+            listener.onResponse(mock(BulkResponse.class));
+            return Void.TYPE;
+        }).when(timingStatsPersister).persistDatafeedTimingStats(any(), any(), any());
     }
 
     public void testReportSearchDuration_Null() {
@@ -59,7 +69,11 @@ public void testReportSearchDuration_Zero() {
         reporter.reportSearchDuration(TimeValue.ZERO);
         assertThat(reporter.getCurrentTimingStats(), equalTo(createDatafeedTimingStats(JOB_ID, 1, 0, 0.0)));
 
-        verify(timingStatsPersister).persistDatafeedTimingStats(createDatafeedTimingStats(JOB_ID, 1, 0, 0.0), RefreshPolicy.NONE);
+        verify(timingStatsPersister).persistDatafeedTimingStats(
+            eq(createDatafeedTimingStats(JOB_ID, 1, 0, 0.0)),
+            eq(RefreshPolicy.NONE),
+            any()
+        );
         verifyNoMoreInteractions(timingStatsPersister);
     }
 
@@ -81,9 +95,9 @@ public void testReportSearchDuration() {
 
         InOrder inOrder = inOrder(timingStatsPersister);
         inOrder.verify(timingStatsPersister)
-            .persistDatafeedTimingStats(createDatafeedTimingStats(JOB_ID, 15, 10, 12000.0, 12000.0), RefreshPolicy.NONE);
+            .persistDatafeedTimingStats(eq(createDatafeedTimingStats(JOB_ID, 15, 10, 12000.0, 12000.0)), eq(RefreshPolicy.NONE), any());
         inOrder.verify(timingStatsPersister)
-            .persistDatafeedTimingStats(createDatafeedTimingStats(JOB_ID, 17, 10, 14000.0, 14000.0), RefreshPolicy.NONE);
+            .persistDatafeedTimingStats(eq(createDatafeedTimingStats(JOB_ID, 17, 10, 14000.0, 14000.0)), eq(RefreshPolicy.NONE), any());
         verifyNoMoreInteractions(timingStatsPersister);
     }
 
@@ -112,7 +126,7 @@ public void testReportDataCounts() {
 
         InOrder inOrder = inOrder(timingStatsPersister);
         inOrder.verify(timingStatsPersister)
-            .persistDatafeedTimingStats(createDatafeedTimingStats(JOB_ID, 3, 23, 10000.0), RefreshPolicy.NONE);
+            .persistDatafeedTimingStats(eq(createDatafeedTimingStats(JOB_ID, 3, 23, 10000.0)), eq(RefreshPolicy.NONE), any());
         verifyNoMoreInteractions(timingStatsPersister);
     }
 
@@ -130,8 +144,9 @@ public void testFinishReporting_WithChange() {
         reporter.finishReporting();
 
         verify(timingStatsPersister).persistDatafeedTimingStats(
-            new DatafeedTimingStats(JOB_ID, 0, 0, 0.0, new ExponentialAverageCalculationContext(0.0, TIMESTAMP, null)),
-            RefreshPolicy.IMMEDIATE
+            eq(new DatafeedTimingStats(JOB_ID, 0, 0, 0.0, new ExponentialAverageCalculationContext(0.0, TIMESTAMP, null))),
+            eq(RefreshPolicy.IMMEDIATE),
+            any()
         );
         verifyNoMoreInteractions(timingStatsPersister);
     }
@@ -205,11 +220,11 @@ public void testTimingStatsDifferSignificantly() {
     }
 
     public void testFinishReportingTimingStatsException() {
-        doThrow(new ElasticsearchException("BOOM")).when(timingStatsPersister).persistDatafeedTimingStats(any(), any());
+        doThrow(new ElasticsearchException("BOOM")).when(timingStatsPersister).persistDatafeedTimingStats(any(), any(), any());
         DatafeedTimingStatsReporter reporter = createReporter(new DatafeedTimingStats(JOB_ID));
 
         try {
-            reporter.reportDataCounts(createDataCounts(0, TIMESTAMP));
+            reporter.reportDataCounts(createDataCounts(0));
             reporter.finishReporting();
         } catch (ElasticsearchException ex) {
             fail("Should not have failed with: " + ex.getDetailedMessage());
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/ProblemTrackerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/ProblemTrackerTests.java
index 37abb900138f..b54f3e6b0ab8 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/ProblemTrackerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/ProblemTrackerTests.java
@@ -25,10 +25,12 @@ public class ProblemTrackerTests extends ESTestCase {
 
     private ProblemTracker problemTracker;
 
+    private static final long NUM_SEARCHES_IN_DAY = 24L;
+
     @Before
     public void setUpTests() {
         auditor = mock(AnomalyDetectionAuditor.class);
-        problemTracker = new ProblemTracker(auditor, "foo");
+        problemTracker = new ProblemTracker(auditor, "foo", NUM_SEARCHES_IN_DAY);
     }
 
     public void testReportExtractionProblem() {
@@ -118,6 +120,26 @@ public void testUpdateEmptyDataCount_GivenNonEmptyAfterTenEmpty() {
         verify(auditor).info("foo", "Datafeed has started retrieving data again");
     }
 
+    public void testUpdateEmptyDataCount_DailyTrigger() {
+        for (int i = 0; i < NUM_SEARCHES_IN_DAY; i++) {
+            problemTracker.reportEmptyDataCount();
+        }
+        verify(auditor, times(2)).warning("foo", "Datafeed has been retrieving no data for a while");
+
+        for (int i = 0; i < NUM_SEARCHES_IN_DAY; i++) {
+            problemTracker.reportEmptyDataCount();
+        }
+        verify(auditor, times(3)).warning("foo", "Datafeed has been retrieving no data for a while");
+    }
+
+    public void testUpdateEmptyDataCount_NumSearchesInDayIsZero() {
+        auditor = mock(AnomalyDetectionAuditor.class);
+        problemTracker = new ProblemTracker(auditor, "foo", 0);
+
+        problemTracker.reportEmptyDataCount();
+        verify(auditor, times(1)).warning("foo", "Datafeed has been retrieving no data for a while");
+    }
+
     public void testFinishReport_GivenNoProblems() {
         problemTracker.finishReport();
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/traintestsplit/StratifiedTrainTestSplitterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/traintestsplit/StratifiedTrainTestSplitterTests.java
index 3309e444f178..581270538ce8 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/traintestsplit/StratifiedTrainTestSplitterTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/traintestsplit/StratifiedTrainTestSplitterTests.java
@@ -214,7 +214,7 @@ public void testIsTraining_SelectsTrainingRowsUniformly() {
         // should be close to the training percent, which is set to 0.5
         for (int rowTrainingCount : trainingCountPerRow) {
             double meanCount = rowTrainingCount / (double) runCount;
-            assertThat(meanCount, is(closeTo(0.5, 0.13)));
+            assertThat(meanCount, is(closeTo(0.5, 0.14)));
         }
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsServiceTests.java
index 5e672e21e72a..3173cbd0908c 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsServiceTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsServiceTests.java
@@ -123,7 +123,8 @@ public void testVerifyIndicesExistAndPrimaryShardsAreActive() {
                     shardId,
                     true,
                     RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")
+                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""),
+                    ShardRouting.Role.DEFAULT
                 );
                 shardRouting = shardRouting.initialize("node_id", null, 0L);
                 routingTable.add(
@@ -344,7 +345,8 @@ private static void addToRoutingTable(String concreteIndex, RoutingTable.Builder
             shardId,
             true,
             RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""),
+            ShardRouting.Role.DEFAULT
         );
         shardRouting = shardRouting.initialize("node_id", null, 0L);
         shardRouting = shardRouting.moveToStarted(ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentMetadataTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentMetadataTests.java
index d2a5f89350ac..4166876ea997 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentMetadataTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentMetadataTests.java
@@ -9,7 +9,7 @@
 
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction;
 import org.elasticsearch.xpack.core.ml.inference.assignment.Priority;
@@ -25,7 +25,7 @@
 
 import static org.hamcrest.Matchers.is;
 
-public class TrainedModelAssignmentMetadataTests extends AbstractXContentSerializingTestCase<TrainedModelAssignmentMetadata> {
+public class TrainedModelAssignmentMetadataTests extends AbstractChunkedSerializingTestCase<TrainedModelAssignmentMetadata> {
 
     public static TrainedModelAssignmentMetadata randomInstance() {
         LinkedHashMap<String, TrainedModelAssignment> map = Stream.generate(() -> randomAlphaOfLength(10))
@@ -51,6 +51,11 @@ protected TrainedModelAssignmentMetadata createTestInstance() {
         return new TrainedModelAssignmentMetadata(new HashMap<>());
     }
 
+    @Override
+    protected TrainedModelAssignmentMetadata mutateInstance(TrainedModelAssignmentMetadata instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testIsAllocated() {
         String allocatedModelId = "test_model_id";
         TrainedModelAssignmentMetadata metadata = TrainedModelAssignmentMetadata.Builder.empty()
@@ -71,5 +76,4 @@ private static StartTrainedModelDeploymentAction.TaskParams randomParams(String
             randomFrom(Priority.values())
         );
     }
-
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/SlimProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/SlimProcessorTests.java
new file mode 100644
index 000000000000..ad07b25b6541
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/SlimProcessorTests.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.ml.inference.nlp;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.ml.inference.results.SlimResults;
+import org.elasticsearch.xpack.ml.inference.nlp.tokenizers.BertTokenizationResult;
+import org.elasticsearch.xpack.ml.inference.nlp.tokenizers.TokenizationResult;
+import org.elasticsearch.xpack.ml.inference.pytorch.results.PyTorchInferenceResult;
+
+import java.util.List;
+
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class SlimProcessorTests extends ESTestCase {
+
+    public void testProcessResult() {
+        double[][][] pytorchResult = new double[][][] { { { 0.0, 1.0, 0.0, 3.0, 4.0, 0.0, 0.0 } } };
+
+        TokenizationResult tokenizationResult = new BertTokenizationResult(List.of(), List.of(), 0);
+
+        var inferenceResult = SlimProcessor.processResult(tokenizationResult, new PyTorchInferenceResult(pytorchResult), "foo");
+        assertThat(inferenceResult, instanceOf(SlimResults.class));
+        var slimResults = (SlimResults) inferenceResult;
+        assertEquals(slimResults.getResultsField(), "foo");
+
+        var weightedTokens = slimResults.getWeightedTokens();
+        assertThat(weightedTokens, hasSize(3));
+        assertEquals(new SlimResults.WeightedToken(1, 1.0f), weightedTokens.get(0));
+        assertEquals(new SlimResults.WeightedToken(3, 3.0f), weightedTokens.get(1));
+        assertEquals(new SlimResults.WeightedToken(4, 4.0f), weightedTokens.get(2));
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertTokenizerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertTokenizerTests.java
index 6fa7c00425d7..4a0ac4f72ca9 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertTokenizerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertTokenizerTests.java
@@ -19,6 +19,7 @@
 import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 
@@ -66,6 +67,35 @@ public void testTokenize() {
         }
     }
 
+    public void testTokenizeFailureCaseAccentFilter() {
+        List<String> testingVocab = List.of(
+            "[CLS]",
+            "br",
+            "##ᄎ",
+            "##ᅡ",
+            "##ᆼ",
+            "##n",
+            "'",
+            "s",
+            "[SEP]",
+            BertTokenizer.MASK_TOKEN,
+            BertTokenizer.UNKNOWN_TOKEN,
+            BertTokenizer.PAD_TOKEN
+        );
+        try (
+            BertTokenizer tokenizer = BertTokenizer.builder(
+                testingVocab,
+                new BertTokenization(true, true, 512, Tokenization.Truncate.FIRST, -1)
+            ).build()
+        ) {
+            TokenizationResult.Tokens tokenization = tokenizer.tokenize("Br창n's", Tokenization.Truncate.NONE, -1, 0).get(0);
+            assertThat(tokenization.tokenIds(), equalTo(new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8 }));
+
+            tokenization = tokenizer.tokenize("Br창n", Tokenization.Truncate.NONE, -1, 0).get(0);
+            assertThat(tokenization.tokenIds(), equalTo(new int[] { 0, 1, 2, 3, 4, 5, 8 }));
+        }
+    }
+
     public void testTokenizeLargeInputNoTruncation() {
         try (
             BertTokenizer tokenizer = BertTokenizer.builder(
@@ -469,6 +499,50 @@ public void testMultiSeqTokenizationWithSpanOnLongInput() {
         }
     }
 
+    public void testMultiSeqTokenizationWithSpanFirstInputTooLong() {
+        try (
+            BertTokenizer tokenizer = BertTokenizer.builder(TEST_CASED_VOCAB, Tokenization.createDefault())
+                .setDoLowerCase(false)
+                .setWithSpecialTokens(true)
+                .setMaxSequenceLength(3)
+                .build()
+        ) {
+            IllegalArgumentException iae = expectThrows(
+                IllegalArgumentException.class,
+                () -> tokenizer.tokenize("Elasticsearch is fun", "Godzilla my little red car", Tokenization.Truncate.NONE, 2, 0)
+            );
+            assertThat(
+                iae.getMessage(),
+                containsString(
+                    "Unable to do sequence pair tokenization: the first sequence [7 tokens] "
+                        + "is longer than the max sequence length [3 tokens]"
+                )
+            );
+        }
+    }
+
+    public void testMultiSeqTokenizationWithSpanPlusFirstInputTooLong() {
+        try (
+            BertTokenizer tokenizer = BertTokenizer.builder(TEST_CASED_VOCAB, Tokenization.createDefault())
+                .setDoLowerCase(false)
+                .setWithSpecialTokens(true)
+                .setMaxSequenceLength(8)
+                .build()
+        ) {
+            IllegalArgumentException iae = expectThrows(
+                IllegalArgumentException.class,
+                () -> tokenizer.tokenize("Elasticsearch is fun", "Godzilla my little red car", Tokenization.Truncate.NONE, 5, 0)
+            );
+            assertThat(
+                iae.getMessage(),
+                containsString(
+                    "Unable to do sequence pair tokenization: the combined first sequence and span length [4 + 5 = 9 tokens] "
+                        + "is longer than the max sequence length [8 tokens]. Reduce the size of the [span] window."
+                )
+            );
+        }
+    }
+
     public void testTokenizeLargeInputMultiSequenceTruncation() {
         try (
             BertTokenizer tokenizer = BertTokenizer.builder(
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/persistence/TrainedModelDefinitionDocTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/persistence/TrainedModelDefinitionDocTests.java
index 2fd93bab1b34..e844fcf07af4 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/persistence/TrainedModelDefinitionDocTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/persistence/TrainedModelDefinitionDocTests.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.ml.inference.persistence;
 
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.AbstractXContentTestCase;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.json.JsonXContent;
@@ -25,7 +26,7 @@ public void testParsingDocWithCompressedStringDefinition() throws IOException {
 
         // The previous storage format was a base64 encoded string.
         // The new format should parse and decode the string storing the raw bytes.
-        String compressedStringDoc = formatted("""
+        String compressedStringDoc = Strings.format("""
             {
               "doc_type": "trained_model_definition_doc",
               "model_id": "bntHUo",
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobBuilderTests.java
index 999f1bcd3464..80c774f1d524 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobBuilderTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobBuilderTests.java
@@ -76,6 +76,11 @@ protected Job.Builder createTestInstance() {
         return builder;
     }
 
+    @Override
+    protected Job.Builder mutateInstance(Job.Builder instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Job.Builder> instanceReader() {
         return Job.Builder::new;
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTaskStateTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTaskStateTests.java
index 27778eae7d92..8ba8a15cf66d 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTaskStateTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTaskStateTests.java
@@ -19,6 +19,11 @@ protected JobTaskState createTestInstance() {
         return new JobTaskState(randomFrom(JobState.values()), randomLong(), randomAlphaOfLength(10));
     }
 
+    @Override
+    protected JobTaskState mutateInstance(JobTaskState instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<JobTaskState> instanceReader() {
         return JobTaskState::new;
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java
index e73ab615045f..1d556f7399a8 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java
@@ -74,7 +74,6 @@ public class JobResultsPersisterTests extends ESTestCase {
     private static final String JOB_ID = "foo";
 
     private Client client;
-    private OriginSettingClient originSettingClient;
     private ArgumentCaptor<BulkRequest> bulkRequestCaptor;
     private JobResultsPersister persister;
 
@@ -83,7 +82,7 @@ public void setUpTests() {
         bulkRequestCaptor = ArgumentCaptor.forClass(BulkRequest.class);
         client = mock(Client.class);
         doAnswer(withResponse(mock(BulkResponse.class))).when(client).execute(eq(BulkAction.INSTANCE), any(), any());
-        originSettingClient = MockOriginSettingClient.mockOriginSettingClient(client, ClientHelper.ML_ORIGIN);
+        OriginSettingClient originSettingClient = MockOriginSettingClient.mockOriginSettingClient(client, ClientHelper.ML_ORIGIN);
         persister = new JobResultsPersister(originSettingClient, buildResultsPersisterService(originSettingClient));
     }
 
@@ -222,8 +221,8 @@ public void testExecuteRequest_ClearsBulkRequest() {
 
     public void testBulkRequestExecutesWhenReachMaxDocs() {
         JobResultsPersister.Builder bulkBuilder = persister.bulkPersisterBuilder("foo");
-        ModelPlot modelPlot = new ModelPlot("foo", new Date(), 123456, 0);
         for (int i = 0; i <= JobRenormalizedResultsPersister.BULK_LIMIT; i++) {
+            ModelPlot modelPlot = new ModelPlot("foo", new Date(), 123456, i);
             bulkBuilder.persistModelPlot(modelPlot);
         }
 
@@ -282,7 +281,6 @@ public void testPersistTimingStats() {
         );
     }
 
-    @SuppressWarnings("unchecked")
     public void testPersistDatafeedTimingStats() {
         DatafeedTimingStats timingStats = new DatafeedTimingStats(
             "foo",
@@ -291,7 +289,11 @@ public void testPersistDatafeedTimingStats() {
             666.0,
             new ExponentialAverageCalculationContext(600.0, Instant.ofEpochMilli(123456789), 60.0)
         );
-        persister.persistDatafeedTimingStats(timingStats, WriteRequest.RefreshPolicy.IMMEDIATE);
+        persister.persistDatafeedTimingStats(
+            timingStats,
+            WriteRequest.RefreshPolicy.IMMEDIATE,
+            ActionListener.wrap(r -> {}, e -> fail("unexpected exception " + e.getMessage()))
+        );
 
         InOrder inOrder = inOrder(client);
         inOrder.verify(client).settings();
@@ -325,7 +327,6 @@ public void testPersistDatafeedTimingStats() {
         );
     }
 
-    @SuppressWarnings("unchecked")
     private void testPersistQuantilesSync(SearchHits searchHits, String expectedIndexOrAlias) {
         SearchResponse searchResponse = mock(SearchResponse.class);
         when(searchResponse.status()).thenReturn(RestStatus.OK);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/DataCountsReporterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/DataCountsReporterTests.java
index 3784e3ae5623..51cf4f5acc7b 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/DataCountsReporterTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/DataCountsReporterTests.java
@@ -18,8 +18,8 @@
 import org.mockito.Mockito;
 
 import java.time.Instant;
-import java.util.Arrays;
 import java.util.Date;
+import java.util.List;
 import java.util.concurrent.TimeUnit;
 
 import static org.mockito.ArgumentMatchers.any;
@@ -37,10 +37,10 @@ public class DataCountsReporterTests extends ESTestCase {
 
     @Before
     public void setUpMocks() {
-        AnalysisConfig.Builder acBuilder = new AnalysisConfig.Builder(Arrays.asList(new Detector.Builder("metric", "field").build()));
+        AnalysisConfig.Builder acBuilder = new AnalysisConfig.Builder(List.of(new Detector.Builder("metric", "field").build()));
         acBuilder.setBucketSpan(bucketSpan);
         acBuilder.setLatency(TimeValue.ZERO);
-        acBuilder.setDetectors(Arrays.asList(new Detector.Builder("metric", "field").build()));
+        acBuilder.setDetectors(List.of(new Detector.Builder("metric", "field").build()));
 
         Job.Builder builder = new Job.Builder("sr");
         builder.setAnalysisConfig(acBuilder);
@@ -153,7 +153,7 @@ public void testReportLatestTimeIncrementalStats() {
         assertEquals(5001L, dataCountsReporter.incrementalStats().getLatestRecordTimeStamp().getTime());
     }
 
-    public void testReportRecordsWritten() {
+    public void testReportRecordsWritten() throws InterruptedException {
         DataCountsReporter dataCountsReporter = new DataCountsReporter(job, new DataCounts(job.getId()), jobDataCountsPersister);
         dataCountsReporter.setAnalysedFieldsPerRecord(3);
 
@@ -174,7 +174,7 @@ public void testReportRecordsWritten() {
 
         assertEquals(dataCountsReporter.incrementalStats(), dataCountsReporter.runningTotalStats());
 
-        verify(jobDataCountsPersister, never()).persistDataCounts(anyString(), any(DataCounts.class));
+        verify(jobDataCountsPersister, never()).persistDataCounts(anyString(), any(DataCounts.class), eq(false));
     }
 
     public void testReportRecordsWritten_Given9999Records() {
@@ -262,7 +262,7 @@ public void testReportRecordsWritten_Given2_000_000Records() {
         assertEquals(20, dataCountsReporter.getLogStatusCallCount());
     }
 
-    public void testFinishReporting() {
+    public void testFinishReporting() throws InterruptedException {
         DataCountsReporter dataCountsReporter = new DataCountsReporter(job, new DataCounts(job.getId()), jobDataCountsPersister);
 
         dataCountsReporter.setAnalysedFieldsPerRecord(3);
@@ -300,7 +300,7 @@ public void testFinishReporting() {
         );
 
         dc.setLastDataTimeStamp(dataCountsReporter.incrementalStats().getLastDataTimeStamp());
-        verify(jobDataCountsPersister, times(1)).persistDataCounts(eq("sr"), eq(dc));
+        verify(jobDataCountsPersister, times(1)).persistDataCounts(eq("sr"), eq(dc), eq(false));
         assertEquals(dc, dataCountsReporter.incrementalStats());
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java
index 35a7d9c422c1..7a5d02be1a26 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java
@@ -186,13 +186,11 @@ public void setup() throws Exception {
         jobResultsProvider = mock(JobResultsProvider.class);
         jobResultsPersister = mock(JobResultsPersister.class);
         JobResultsPersister.Builder bulkPersisterBuilder = mock(JobResultsPersister.Builder.class);
-        when(bulkPersisterBuilder.shouldRetry(any())).thenReturn(bulkPersisterBuilder);
-        when(jobResultsPersister.bulkPersisterBuilder(any())).thenReturn(bulkPersisterBuilder);
+        when(jobResultsPersister.bulkPersisterBuilder(any(), any())).thenReturn(bulkPersisterBuilder);
         jobDataCountsPersister = mock(JobDataCountsPersister.class);
         annotationPersister = mock(AnnotationPersister.class);
         AnnotationPersister.Builder bulkAnnotationsBuilder = mock(AnnotationPersister.Builder.class);
-        when(bulkAnnotationsBuilder.shouldRetry(any())).thenReturn(bulkAnnotationsBuilder);
-        when(annotationPersister.bulkPersisterBuilder(any())).thenReturn(bulkAnnotationsBuilder);
+        when(annotationPersister.bulkPersisterBuilder(any(), any())).thenReturn(bulkAnnotationsBuilder);
         autodetectCommunicator = mock(AutodetectCommunicator.class);
         autodetectFactory = mock(AutodetectProcessFactory.class);
         normalizerFactory = mock(NormalizerFactory.class);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessorTests.java
index cfd10da57de2..d49596a4811f 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessorTests.java
@@ -58,6 +58,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
+import java.util.EnumSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
@@ -84,7 +85,6 @@ public class AutodetectResultProcessorTests extends ESTestCase {
     private static final long BUCKET_SPAN_MS = 1000;
     private static final Instant CURRENT_TIME = Instant.ofEpochMilli(2000000000);
 
-    private ThreadPool threadPool;
     private Client client;
     private AnomalyDetectionAuditor auditor;
     private Renormalizer renormalizer;
@@ -101,19 +101,17 @@ public class AutodetectResultProcessorTests extends ESTestCase {
     public void setUpMocks() {
         executor = new Scheduler.SafeScheduledThreadPoolExecutor(1);
         client = mock(Client.class);
-        threadPool = mock(ThreadPool.class);
+        ThreadPool threadPool = mock(ThreadPool.class);
         when(client.threadPool()).thenReturn(threadPool);
         when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY));
         auditor = mock(AnomalyDetectionAuditor.class);
         renormalizer = mock(Renormalizer.class);
         persister = mock(JobResultsPersister.class);
         bulkResultsPersister = mock(JobResultsPersister.Builder.class);
-        when(bulkResultsPersister.shouldRetry(any())).thenReturn(bulkResultsPersister);
-        when(persister.bulkPersisterBuilder(eq(JOB_ID))).thenReturn(bulkResultsPersister);
+        when(persister.bulkPersisterBuilder(eq(JOB_ID), any())).thenReturn(bulkResultsPersister);
         annotationPersister = mock(AnnotationPersister.class);
         bulkAnnotationsPersister = mock(AnnotationPersister.Builder.class);
-        when(bulkAnnotationsPersister.shouldRetry(any())).thenReturn(bulkAnnotationsPersister);
-        when(annotationPersister.bulkPersisterBuilder(eq(JOB_ID))).thenReturn(bulkAnnotationsPersister);
+        when(annotationPersister.bulkPersisterBuilder(eq(JOB_ID), any())).thenReturn(bulkAnnotationsPersister);
         process = mock(AutodetectProcess.class);
         flushListener = mock(FlushListener.class);
         processorUnderTest = new AutodetectResultProcessor(
@@ -133,7 +131,7 @@ public void setUpMocks() {
 
     @After
     public void cleanup() {
-        verify(annotationPersister).bulkPersisterBuilder(eq(JOB_ID));
+        verify(annotationPersister).bulkPersisterBuilder(eq(JOB_ID), any());
         verifyNoMoreInteractions(auditor, renormalizer, persister, annotationPersister);
         executor.shutdown();
     }
@@ -147,10 +145,8 @@ public void testProcess() throws Exception {
         assertThat(processorUnderTest.completionLatch.getCount(), is(equalTo(0L)));
 
         verify(renormalizer).waitUntilIdle();
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
-        verify(persister).commitResultWrites(JOB_ID);
-        verify(persister).commitAnnotationWrites();
-        verify(persister).commitStateWrites(JOB_ID);
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
+        verify(persister).commitWrites(JOB_ID, EnumSet.allOf(JobResultsPersister.CommitType.class));
     }
 
     public void testProcessResult_bucket() {
@@ -163,10 +159,10 @@ public void testProcessResult_bucket() {
         processorUnderTest.setDeleteInterimRequired(false);
         processorUnderTest.processResult(result);
 
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
         verify(bulkResultsPersister).persistTimingStats(any(TimingStats.class));
         verify(bulkResultsPersister).persistBucket(bucket);
-        verify(bulkResultsPersister).executeRequest();
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
+        verify(bulkResultsPersister, never()).executeRequest();
         verify(persister, never()).deleteInterimResults(JOB_ID);
     }
 
@@ -184,7 +180,7 @@ public void testProcessResult_bucket_deleteInterimRequired() {
         verify(bulkResultsPersister).persistTimingStats(any(TimingStats.class));
         verify(bulkResultsPersister).persistBucket(bucket);
         verify(bulkResultsPersister).executeRequest();
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
         verify(persister).deleteInterimResults(JOB_ID);
     }
 
@@ -203,7 +199,7 @@ public void testProcessResult_bucket_isInterim() {
         verify(bulkResultsPersister, never()).persistTimingStats(any(TimingStats.class));
         verify(bulkResultsPersister).persistBucket(bucket);
         verify(bulkResultsPersister).executeRequest();
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
     }
 
     public void testProcessResult_records() {
@@ -219,7 +215,7 @@ public void testProcessResult_records() {
 
         verify(bulkResultsPersister).persistRecords(records);
         verify(bulkResultsPersister, never()).executeRequest();
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
     }
 
     public void testProcessResult_influencers() {
@@ -235,7 +231,7 @@ public void testProcessResult_influencers() {
 
         verify(bulkResultsPersister).persistInfluencers(influencers);
         verify(bulkResultsPersister, never()).executeRequest();
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
     }
 
     public void testProcessResult_categoryDefinition() {
@@ -247,9 +243,9 @@ public void testProcessResult_categoryDefinition() {
         processorUnderTest.setDeleteInterimRequired(false);
         processorUnderTest.processResult(result);
 
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
+        verify(bulkResultsPersister).persistCategoryDefinition(eq(categoryDefinition));
         verify(bulkResultsPersister, never()).executeRequest();
-        verify(persister).persistCategoryDefinition(eq(categoryDefinition), any());
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
     }
 
     public void testProcessResult_flushAcknowledgement() {
@@ -262,17 +258,19 @@ public void testProcessResult_flushAcknowledgement() {
         processorUnderTest.processResult(result);
         assertTrue(processorUnderTest.isDeleteInterimRequired());
 
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
         verify(flushListener).acknowledgeFlush(flushAcknowledgement, null);
-        verify(persister).commitResultWrites(JOB_ID);
-        verify(persister).commitAnnotationWrites();
+        verify(persister).commitWrites(
+            JOB_ID,
+            EnumSet.of(JobResultsPersister.CommitType.RESULTS, JobResultsPersister.CommitType.ANNOTATIONS)
+        );
         verify(bulkResultsPersister).executeRequest();
     }
 
     public void testProcessResult_flushAcknowledgementMustBeProcessedLast() {
         AutodetectResult result = mock(AutodetectResult.class);
         FlushAcknowledgement flushAcknowledgement = mock(FlushAcknowledgement.class);
-        when(flushAcknowledgement.getId()).thenReturn(JOB_ID);
+        when(flushAcknowledgement.getId()).thenReturn(Integer.valueOf(randomInt(100)).toString());
         when(result.getFlushAcknowledgement()).thenReturn(flushAcknowledgement);
         CategoryDefinition categoryDefinition = mock(CategoryDefinition.class);
         when(categoryDefinition.getCategoryId()).thenReturn(1L);
@@ -283,11 +281,13 @@ public void testProcessResult_flushAcknowledgementMustBeProcessedLast() {
         assertTrue(processorUnderTest.isDeleteInterimRequired());
 
         InOrder inOrder = inOrder(persister, bulkResultsPersister, flushListener);
-        inOrder.verify(persister).bulkPersisterBuilder(eq(JOB_ID));
-        inOrder.verify(persister).persistCategoryDefinition(eq(categoryDefinition), any());
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
+        inOrder.verify(bulkResultsPersister).persistCategoryDefinition(eq(categoryDefinition));
         inOrder.verify(bulkResultsPersister).executeRequest();
-        inOrder.verify(persister).commitResultWrites(JOB_ID);
-        inOrder.verify(persister).commitAnnotationWrites();
+        verify(persister).commitWrites(
+            JOB_ID,
+            EnumSet.of(JobResultsPersister.CommitType.RESULTS, JobResultsPersister.CommitType.ANNOTATIONS)
+        );
         inOrder.verify(flushListener).acknowledgeFlush(flushAcknowledgement, null);
     }
 
@@ -299,7 +299,7 @@ public void testProcessResult_modelPlot() {
         processorUnderTest.setDeleteInterimRequired(false);
         processorUnderTest.processResult(result);
 
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
         verify(bulkResultsPersister).persistModelPlot(modelPlot);
     }
 
@@ -310,7 +310,7 @@ public void testProcessResult_annotation() {
 
         processorUnderTest.processResult(result);
 
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
         verify(bulkAnnotationsPersister).persistAnnotation(annotation);
         if (annotation.getEvent() == Annotation.Event.CATEGORIZATION_STATUS_CHANGE) {
             verify(auditor).warning(eq(JOB_ID), anyString());
@@ -326,8 +326,8 @@ public void testProcessResult_modelSizeStats() {
         processorUnderTest.processResult(result);
         assertThat(processorUnderTest.modelSizeStats(), is(equalTo(modelSizeStats)));
 
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
-        verify(persister).persistModelSizeStats(eq(modelSizeStats), any());
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
+        verify(bulkResultsPersister).persistModelSizeStats(eq(modelSizeStats));
     }
 
     public void testProcessResult_modelSizeStatsWithMemoryStatusChanges() {
@@ -357,8 +357,8 @@ public void testProcessResult_modelSizeStatsWithMemoryStatusChanges() {
         when(result.getModelSizeStats()).thenReturn(modelSizeStats);
         processorUnderTest.processResult(result);
 
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
-        verify(persister, times(4)).persistModelSizeStats(any(ModelSizeStats.class), any());
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
+        verify(bulkResultsPersister, times(4)).persistModelSizeStats(any(ModelSizeStats.class));
         // We should have only fired two notifications: one for soft_limit and one for hard_limit
         verify(auditor).warning(JOB_ID, Messages.getMessage(Messages.JOB_AUDIT_MEMORY_STATUS_SOFT_LIMIT));
         verify(auditor).error(JOB_ID, Messages.getMessage(Messages.JOB_AUDIT_MEMORY_STATUS_HARD_LIMIT, "512mb", "1kb"));
@@ -381,7 +381,7 @@ public void testProcessResult_categorizationStatusChangeAnnotationCausesNotifica
         when(result.getAnnotation()).thenReturn(annotation);
         processorUnderTest.processResult(result);
 
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
         verify(bulkAnnotationsPersister).persistAnnotation(annotation);
         verify(auditor).warning(JOB_ID, "Categorization status changed to 'warn' for partition 'foo' after 0 buckets");
     }
@@ -419,7 +419,7 @@ public void testProcessResult_modelSnapshot() {
             new JobUpdate.Builder(JOB_ID).setModelSnapshotId("a_snapshot_id").build()
         );
 
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
         verify(persister).persistModelSnapshot(eq(modelSnapshot), eq(WriteRequest.RefreshPolicy.IMMEDIATE), any());
         verify(bulkAnnotationsPersister).persistAnnotation(ModelSnapshot.annotationDocumentId(modelSnapshot), expectedAnnotation);
         verify(client).execute(same(UpdateJobAction.INSTANCE), eq(expectedJobUpdateRequest), any());
@@ -434,12 +434,10 @@ public void testProcessResult_quantiles_givenRenormalizationIsEnabled() {
         processorUnderTest.setDeleteInterimRequired(false);
         processorUnderTest.processResult(result);
 
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
         verify(persister).persistQuantiles(eq(quantiles), any());
-        verify(bulkResultsPersister).executeRequest();
-        verify(persister).commitResultWrites(JOB_ID);
         verify(renormalizer).isEnabled();
-        verify(renormalizer).renormalize(quantiles);
+        verify(renormalizer).renormalize(eq(quantiles), any(Runnable.class));
     }
 
     public void testProcessResult_quantiles_givenRenormalizationIsDisabled() {
@@ -451,9 +449,9 @@ public void testProcessResult_quantiles_givenRenormalizationIsDisabled() {
         processorUnderTest.setDeleteInterimRequired(false);
         processorUnderTest.processResult(result);
 
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
         verify(persister).persistQuantiles(eq(quantiles), any());
-        verify(bulkResultsPersister).executeRequest();
+        verify(bulkResultsPersister, never()).executeRequest();
         verify(renormalizer).isEnabled();
     }
 
@@ -466,10 +464,8 @@ public void testAwaitCompletion() throws Exception {
         assertThat(processorUnderTest.completionLatch.getCount(), is(equalTo(0L)));
         assertThat(processorUnderTest.updateModelSnapshotSemaphore.availablePermits(), is(equalTo(1)));
 
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
-        verify(persister).commitResultWrites(JOB_ID);
-        verify(persister).commitAnnotationWrites();
-        verify(persister).commitStateWrites(JOB_ID);
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
+        verify(persister).commitWrites(JOB_ID, EnumSet.allOf(JobResultsPersister.CommitType.class));
         verify(renormalizer).waitUntilIdle();
     }
 
@@ -486,7 +482,7 @@ public void testPersisterThrowingDoesntBlockProcessing() {
 
         processorUnderTest.process();
 
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
         verify(persister, times(2)).persistModelSnapshot(any(), eq(WriteRequest.RefreshPolicy.IMMEDIATE), any());
     }
 
@@ -507,7 +503,7 @@ public void testParsingErrorSetsFailed() throws Exception {
         );
         assertThat(flushAcknowledgement, is(nullValue()));
 
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
     }
 
     public void testKill() throws Exception {
@@ -520,11 +516,9 @@ public void testKill() throws Exception {
         assertThat(processorUnderTest.completionLatch.getCount(), is(equalTo(0L)));
         assertThat(processorUnderTest.updateModelSnapshotSemaphore.availablePermits(), is(equalTo(1)));
 
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
-        verify(persister).commitResultWrites(JOB_ID);
-        verify(persister).commitAnnotationWrites();
-        verify(persister).commitStateWrites(JOB_ID);
-        verify(renormalizer, never()).renormalize(any());
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
+        verify(persister).commitWrites(JOB_ID, EnumSet.allOf(JobResultsPersister.CommitType.class));
+        verify(renormalizer, never()).renormalize(any(), any());
         verify(renormalizer).shutdown();
         verify(renormalizer).waitUntilIdle();
         verify(flushListener).clear();
@@ -546,7 +540,7 @@ public void testProcessingOpenedForecasts() {
 
         verify(bulkResultsPersister, times(2)).persistForecastRequestStats(argument.capture());
         verify(bulkResultsPersister, times(1)).executeRequest();
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
         verify(persister, never()).deleteInterimResults(JOB_ID);
 
         // Get all values is in reverse call order
@@ -578,7 +572,7 @@ public void testProcessingForecasts() {
 
         verify(bulkResultsPersister, times(2)).persistForecastRequestStats(argument.capture());
         verify(bulkResultsPersister, times(1)).executeRequest();
-        verify(persister).bulkPersisterBuilder(eq(JOB_ID));
+        verify(persister).bulkPersisterBuilder(eq(JOB_ID), any());
         verify(persister, never()).deleteInterimResults(JOB_ID);
 
         List<ForecastRequestStats> stats = argument.getAllValues();
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/FlushAcknowledgementTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/FlushAcknowledgementTests.java
index 40e038c776d7..f9cf683d64a5 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/FlushAcknowledgementTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/FlushAcknowledgementTests.java
@@ -29,6 +29,11 @@ protected FlushAcknowledgement createTestInstance() {
         }
     }
 
+    @Override
+    protected FlushAcknowledgement mutateInstance(FlushAcknowledgement instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<FlushAcknowledgement> instanceReader() {
         return FlushAcknowledgement::new;
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/XContentRecordReaderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/XContentRecordReaderTests.java
index 14f7d3735a6d..05350d9f3146 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/XContentRecordReaderTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/XContentRecordReaderTests.java
@@ -8,6 +8,7 @@
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.DeprecationHandler;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -23,7 +24,6 @@
 import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.Locale;
 import java.util.Map;
 
 import static org.mockito.Mockito.mock;
@@ -190,7 +190,7 @@ public void testRead_HitParseErrorsLimit() throws IOException {
             """;
         StringBuilder builder = new StringBuilder();
         for (int i = 0; i < XContentRecordReader.PARSE_ERRORS_LIMIT; i++) {
-            builder.append(String.format(Locale.ROOT, format, i));
+            builder.append(Strings.format(format, i));
         }
 
         XContentParser parser = createParser(builder.toString());
@@ -213,7 +213,7 @@ private void readUntilError(XContentRecordReader reader) throws IOException {
     public void testRead_givenControlCharacterInData() throws Exception {
         char controlChar = '\u0002';
 
-        String data = formatted("""
+        String data = Strings.format("""
             {"a":10, "%s" : 5, "b":20, "c":30}
             {"b":21, "a":11, "c":31}
             {"c":32, "b":22, "a":12}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerResultTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerResultTests.java
index 15289fc4eb56..2df7ba5e7eec 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerResultTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerResultTests.java
@@ -42,6 +42,11 @@ protected NormalizerResult createTestInstance() {
         return msg;
     }
 
+    @Override
+    protected NormalizerResult mutateInstance(NormalizerResult instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<NormalizerResult> instanceReader() {
         return NormalizerResult::new;
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizerTests.java
index 02c4830a6e60..ae34a675d15b 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizerTests.java
@@ -47,12 +47,12 @@ public void testNormalize() throws InterruptedException {
             // Blast through many sets of quantiles in quick succession, faster than the normalizer can process them
             for (int i = 1; i < TEST_SIZE / 2; ++i) {
                 Quantiles quantiles = new Quantiles(JOB_ID, new Date(), Integer.toString(i));
-                renormalizer.renormalize(quantiles);
+                renormalizer.renormalize(quantiles, () -> {});
             }
             renormalizer.waitUntilIdle();
             for (int i = TEST_SIZE / 2; i <= TEST_SIZE; ++i) {
                 Quantiles quantiles = new Quantiles(JOB_ID, new Date(), Integer.toString(i));
-                renormalizer.renormalize(quantiles);
+                renormalizer.renormalize(quantiles, () -> {});
             }
             renormalizer.waitUntilIdle();
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/AutodetectResultTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/AutodetectResultTests.java
index d701c8b0a896..9d812ac01823 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/AutodetectResultTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/AutodetectResultTests.java
@@ -150,6 +150,11 @@ protected AutodetectResult createTestInstance() {
         );
     }
 
+    @Override
+    protected AutodetectResult mutateInstance(AutodetectResult instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<AutodetectResult> instanceReader() {
         return AutodetectResult::new;
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/BucketTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/BucketTests.java
index 258499629de1..049e531701d8 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/BucketTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/BucketTests.java
@@ -35,6 +35,11 @@ public Bucket createTestInstance() {
         return createTestInstance("foo");
     }
 
+    @Override
+    protected Bucket mutateInstance(Bucket instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public Bucket createTestInstance(String jobId) {
         Bucket bucket = new Bucket(jobId, randomDate(), randomLongBetween(1, MAX_BUCKET_SPAN_SEC));
         if (randomBoolean()) {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/CategoryDefinitionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/CategoryDefinitionTests.java
index 13d2ab999bb5..1789a4276c70 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/CategoryDefinitionTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/CategoryDefinitionTests.java
@@ -50,6 +50,11 @@ protected CategoryDefinition createTestInstance() {
         return createTestInstance(randomAlphaOfLength(10));
     }
 
+    @Override
+    protected CategoryDefinition mutateInstance(CategoryDefinition instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<CategoryDefinition> instanceReader() {
         return CategoryDefinition::new;
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastRequestStatsTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastRequestStatsTests.java
index 385c81f75d70..cd8b1fc6198c 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastRequestStatsTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastRequestStatsTests.java
@@ -28,6 +28,11 @@ protected ForecastRequestStats createTestInstance() {
         return createTestInstance("ForecastRequestStatsTest", randomAlphaOfLength(20));
     }
 
+    @Override
+    protected ForecastRequestStats mutateInstance(ForecastRequestStats instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public ForecastRequestStats createTestInstance(String jobId, String forecastId) {
         ForecastRequestStats forecastRequestStats = new ForecastRequestStats(jobId, forecastId);
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastTests.java
index 94965f52a053..769a94a1250a 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastTests.java
@@ -25,6 +25,11 @@ protected Forecast createTestInstance() {
         return createTestInstance("ForecastTest");
     }
 
+    @Override
+    protected Forecast mutateInstance(Forecast instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public Forecast createTestInstance(String jobId) {
         Forecast forecast = new Forecast(jobId, randomAlphaOfLength(20), randomDate(), randomNonNegativeLong(), randomInt());
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/InfluenceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/InfluenceTests.java
index 661105830efd..86ce08b70e85 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/InfluenceTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/InfluenceTests.java
@@ -30,6 +30,11 @@ protected Influence createTestInstance() {
         return new Influence(randomAlphaOfLengthBetween(1, 30), fieldValues);
     }
 
+    @Override
+    protected Influence mutateInstance(Influence instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<Influence> instanceReader() {
         return Influence::new;
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ModelPlotTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ModelPlotTests.java
index 511e3741babc..c93c4158b8c5 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ModelPlotTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ModelPlotTests.java
@@ -33,6 +33,11 @@ protected ModelPlot createTestInstance() {
         return createTestInstance("foo");
     }
 
+    @Override
+    protected ModelPlot mutateInstance(ModelPlot instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public ModelPlot createTestInstance(String jobId) {
         ModelPlot modelPlot = new ModelPlot(jobId, randomDate(), randomNonNegativeLong(), randomInt());
         if (randomBoolean()) {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/OverallBucketTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/OverallBucketTests.java
index 380357ecc0b5..cb10fea41575 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/OverallBucketTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/OverallBucketTests.java
@@ -36,6 +36,11 @@ protected OverallBucket createTestInstance() {
         );
     }
 
+    @Override
+    protected OverallBucket mutateInstance(OverallBucket instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<OverallBucket> instanceReader() {
         return OverallBucket::new;
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java
index a7502792df8f..fcf87fa4a07e 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutorTests.java
@@ -259,7 +259,8 @@ private void addIndices(Metadata.Builder metadata, RoutingTable.Builder routingT
                 shardId,
                 true,
                 RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")
+                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""),
+                ShardRouting.Role.DEFAULT
             );
             shardRouting = shardRouting.initialize("node_id", null, 0L);
             shardRouting = shardRouting.moveToStarted(ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/logging/CppLogMessageTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/logging/CppLogMessageTests.java
index eb2f742faf23..7ea2c7f7fb24 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/logging/CppLogMessageTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/logging/CppLogMessageTests.java
@@ -72,6 +72,11 @@ protected CppLogMessage createTestInstance() {
         return msg;
     }
 
+    @Override
+    protected CppLogMessage mutateInstance(CppLogMessage instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<CppLogMessage> instanceReader() {
         return CppLogMessage::new;
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutorTests.java
index 9d063bd55638..dec5346b9cf7 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutorTests.java
@@ -84,7 +84,8 @@ public void testVerifyIndicesPrimaryShardsAreActive() {
                 shardId,
                 true,
                 RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")
+                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""),
+                ShardRouting.Role.DEFAULT
             );
             shardRouting = shardRouting.initialize("node_id", null, 0L);
             routingTable.add(
@@ -133,7 +134,8 @@ private void addIndices(Metadata.Builder metadata, RoutingTable.Builder routingT
                 shardId,
                 true,
                 RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")
+                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""),
+                ShardRouting.Role.DEFAULT
             );
             shardRouting = shardRouting.initialize("node_id", null, 0L);
             shardRouting = shardRouting.moveToStarted(ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java
index 7392a9382ed6..85adf1098311 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -130,7 +130,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver expressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         final ClusterSettings clusterSettings = clusterService.getClusterSettings();
         final CleanerService cleanerService = new CleanerService(settings, clusterSettings, threadPool, getLicenseState());
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java
index b488f13cb4c5..68521d90c8a0 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java
@@ -50,7 +50,7 @@ public class MonitoringTemplateRegistry extends IndexTemplateRegistry {
      * continue to use the release version number in this registry, even though this is not standard practice for template
      * registries.
      */
-    public static final int REGISTRY_VERSION = Version.V_8_1_0.id;
+    public static final int REGISTRY_VERSION = Version.V_8_7_0.id;
     private static final String REGISTRY_VERSION_VARIABLE = "xpack.monitoring.template.release.version";
 
     /**
@@ -78,7 +78,7 @@ public class MonitoringTemplateRegistry extends IndexTemplateRegistry {
      * writes monitoring data in ECS format as of 8.0. These templates define the ECS schema as well as alias fields for the old monitoring
      * mappings that point to the corresponding ECS fields.
      */
-    public static final int STACK_MONITORING_REGISTRY_VERSION = Version.V_8_0_0.id + 4;
+    public static final int STACK_MONITORING_REGISTRY_VERSION = Version.V_8_0_0.id + 5;
     private static final String STACK_MONITORING_REGISTRY_VERSION_VARIABLE = "xpack.stack.monitoring.template.release.version";
     private static final String STACK_TEMPLATE_VERSION = "8";
     private static final String STACK_TEMPLATE_VERSION_VARIABLE = "xpack.stack.monitoring.template.version";
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDoc.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDoc.java
index d669f069efd8..9d46803b1332 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDoc.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDoc.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.collect.MapBuilder;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.license.License;
 import org.elasticsearch.xcontent.ToXContent;
@@ -162,7 +163,9 @@ protected void innerToXContent(XContentBuilder builder, Params params) throws IO
             {
                 builder.field("nodes_hash", nodesHash(clusterState.nodes()));
                 builder.field("status", status.name().toLowerCase(Locale.ROOT));
-                clusterState.toXContent(builder, CLUSTER_STATS_PARAMS);
+                // we need the whole doc in memory anyway so no need to preserve chunking here; moreover CLUSTER_STATS_PARAMS doesn't
+                // include anything heavy so this should be fine.
+                ChunkedToXContent.wrapAsToXContent(clusterState).toXContent(builder, CLUSTER_STATS_PARAMS);
             }
             builder.endObject();
         }
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardMonitoringDoc.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardMonitoringDoc.java
index f034dd0fc3a7..37eab685521a 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardMonitoringDoc.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardMonitoringDoc.java
@@ -32,10 +32,20 @@ public class ShardMonitoringDoc extends FilteredMonitoringDoc {
         final long interval,
         final MonitoringDoc.Node node,
         final ShardRouting shardRouting,
-        final String clusterStateUUID
+        final String clusterStateUUID,
+        final int shardIndex
     ) {
 
-        super(cluster, timestamp, interval, node, MonitoredSystem.ES, TYPE, id(clusterStateUUID, shardRouting), XCONTENT_FILTERS);
+        super(
+            cluster,
+            timestamp,
+            interval,
+            node,
+            MonitoredSystem.ES,
+            TYPE,
+            id(clusterStateUUID, shardRouting, shardIndex),
+            XCONTENT_FILTERS
+        );
         this.shardRouting = Objects.requireNonNull(shardRouting);
         this.clusterStateUUID = Objects.requireNonNull(clusterStateUUID);
     }
@@ -61,9 +71,9 @@ protected void innerToXContent(XContentBuilder builder, Params params) throws IO
     /**
      * Compute an id that has the format:
      *
-     * {state_uuid}:{node_id || '_na'}:{index}:{shard}:{'p' || 'r'}
+     * {state_uuid}:{node_id || '_na'}:{index}:s{shard}:{'p' || 'rX'}
      */
-    public static String id(String stateUUID, ShardRouting shardRouting) {
+    public static String id(String stateUUID, ShardRouting shardRouting, int shardIndex) {
         StringBuilder builder = new StringBuilder();
         builder.append(stateUUID);
         builder.append(':');
@@ -74,13 +84,14 @@ public static String id(String stateUUID, ShardRouting shardRouting) {
         }
         builder.append(':');
         builder.append(shardRouting.getIndexName());
-        builder.append(':');
+        builder.append(":s");
         builder.append(Integer.valueOf(shardRouting.id()));
         builder.append(':');
         if (shardRouting.primary()) {
             builder.append("p");
         } else {
             builder.append("r");
+            builder.append(Integer.valueOf(shardIndex));
         }
         return builder.toString();
     }
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsCollector.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsCollector.java
index dde8fb7353a5..276935e599fd 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsCollector.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsCollector.java
@@ -8,6 +8,8 @@
 
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -20,7 +22,9 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 /**
  * Collector for shards.
@@ -46,23 +50,46 @@ protected Collection<MonitoringDoc> doCollect(final MonitoringDoc.Node node, fin
         if (clusterState != null) {
             RoutingTable routingTable = clusterState.routingTable();
             if (routingTable != null) {
-                List<ShardRouting> shards = routingTable.allShards();
-                if (shards != null) {
-                    final String clusterUuid = clusterUuid(clusterState);
-                    final String stateUUID = clusterState.stateUUID();
-                    final long timestamp = timestamp();
+                final String clusterUuid = clusterUuid(clusterState);
+                final String stateUUID = clusterState.stateUUID();
+                final long timestamp = timestamp();
 
-                    final String[] indices = getCollectionIndices();
-                    final boolean isAllIndices = IndexNameExpressionResolver.isAllIndices(Arrays.asList(indices));
+                final String[] indicesToMonitor = getCollectionIndices();
+                final boolean isAllIndices = IndexNameExpressionResolver.isAllIndices(Arrays.asList(indicesToMonitor));
+                final String[] indices = isAllIndices
+                    ? routingTable.indicesRouting().keySet().toArray(new String[0])
+                    : expandIndexPattern(indicesToMonitor, routingTable.indicesRouting().keySet().toArray(new String[0]));
 
-                    for (ShardRouting shard : shards) {
-                        if (isAllIndices || Regex.simpleMatch(indices, shard.getIndexName())) {
-                            MonitoringDoc.Node shardNode = null;
-                            if (shard.assignedToNode()) {
+                for (String index : indices) {
+                    IndexRoutingTable indexRoutingTable = routingTable.index(index);
+                    if (indexRoutingTable != null) {
+                        final int shardCount = indexRoutingTable.size();
+                        for (int i = 0; i < shardCount; i++) {
+                            IndexShardRoutingTable shardRoutingTable = indexRoutingTable.shard(i);
+
+                            ShardRouting primary = shardRoutingTable.primaryShard();
+                            MonitoringDoc.Node primaryShardNode = null;
+                            if (primary.assignedToNode()) {
                                 // If the shard is assigned to a node, the shard monitoring document refers to this node
-                                shardNode = convertNode(node.getTimestamp(), clusterState.getNodes().get(shard.currentNodeId()));
+                                primaryShardNode = convertNode(node.getTimestamp(), clusterState.getNodes().get(primary.currentNodeId()));
+                            }
+                            results.add(new ShardMonitoringDoc(clusterUuid, timestamp, interval, primaryShardNode, primary, stateUUID, 0));
+
+                            List<ShardRouting> replicas = shardRoutingTable.replicaShards();
+                            for (int j = 0; j < replicas.size(); j++) {
+                                ShardRouting replica = replicas.get(j);
+
+                                MonitoringDoc.Node replicaShardNode = null;
+                                if (replica.assignedToNode()) {
+                                    replicaShardNode = convertNode(
+                                        node.getTimestamp(),
+                                        clusterState.getNodes().get(replica.currentNodeId())
+                                    );
+                                }
+                                results.add(
+                                    new ShardMonitoringDoc(clusterUuid, timestamp, interval, replicaShardNode, replica, stateUUID, j + 1)
+                                );
                             }
-                            results.add(new ShardMonitoringDoc(clusterUuid, timestamp, interval, shardNode, shard, stateUUID));
                         }
                     }
                 }
@@ -70,4 +97,22 @@ protected Collection<MonitoringDoc> doCollect(final MonitoringDoc.Node node, fin
         }
         return Collections.unmodifiableCollection(results);
     }
+
+    private String[] expandIndexPattern(String[] indicesToMonitor, String[] indices) {
+        final Set<String> expandedIndices = new HashSet<>();
+
+        for (String indexOrPattern : indicesToMonitor) {
+            if (indexOrPattern.contains("*")) {
+                for (String index : indices) {
+                    if (Regex.simpleMatch(indexOrPattern, index)) {
+                        expandedIndices.add(index);
+                    }
+                }
+            } else {
+                expandedIndices.add(indexOrPattern);
+            }
+        }
+
+        return expandedIndices.toArray(new String[0]);
+    }
 }
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java
index 67a25ecdb1c2..37fe08b302e7 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java
@@ -237,7 +237,7 @@ public void testSerialization() throws IOException {
         originalRequest.writeTo(out);
 
         final StreamInput in = out.bytes().streamInput();
-        in.setVersion(out.getVersion());
+        in.setTransportVersion(out.getTransportVersion());
 
         final MonitoringBulkRequest deserializedRequest = new MonitoringBulkRequest(in);
 
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringMigrateAlertsResponseTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringMigrateAlertsResponseTests.java
index ea421691de5a..f6fa2bade19d 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringMigrateAlertsResponseTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringMigrateAlertsResponseTests.java
@@ -63,4 +63,9 @@ protected MonitoringMigrateAlertsResponse createTestInstance() {
         }
         return new MonitoringMigrateAlertsResponse(results);
     }
+
+    @Override
+    protected MonitoringMigrateAlertsResponse mutateInstance(MonitoringMigrateAlertsResponse instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
 }
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/cleaner/AbstractIndicesCleanerTestCase.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/cleaner/AbstractIndicesCleanerTestCase.java
index 635e70e191e8..58159f8d6c61 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/cleaner/AbstractIndicesCleanerTestCase.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/cleaner/AbstractIndicesCleanerTestCase.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.xpack.core.monitoring.MonitoringField;
@@ -142,7 +143,7 @@ public void testDeleteIndices() throws Exception {
     public void testRetentionAsGlobalSetting() throws Exception {
         final int max = 10;
         final int retention = randomIntBetween(1, max);
-        internalCluster().startNode(Settings.builder().put(MonitoringField.HISTORY_DURATION.getKey(), formatted("%dd", retention)));
+        internalCluster().startNode(Settings.builder().put(MonitoringField.HISTORY_DURATION.getKey(), Strings.format("%dd", retention)));
 
         final ZonedDateTime now = now();
         for (int i = 0; i < max; i++) {
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
index 0b51bc3e63a7..e533460b8448 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
@@ -38,6 +38,7 @@
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.index.shard.ShardId;
@@ -75,6 +76,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -119,6 +121,7 @@ public void setUp() throws Exception {
             .add(masterNode);
 
         when(clusterState.nodes()).thenReturn(builder.build());
+        when(clusterState.toXContentChunked(any())).thenReturn(Collections.emptyIterator());
     }
 
     @Override
@@ -397,7 +400,8 @@ public void testToXContent() throws IOException {
             shardId,
             true,
             RecoverySource.ExistingStoreRecoverySource.INSTANCE,
-            unassignedInfo
+            unassignedInfo,
+            ShardRouting.Role.DEFAULT
         );
 
         final ShardStats mockShardStats = mock(ShardStats.class);
@@ -442,7 +446,13 @@ public void testToXContent() throws IOException {
         );
 
         final BytesReference xContent = XContentHelper.toXContent(doc, XContentType.JSON, false);
-        final String expectedJson = formatted("""
+        Object[] args = new Object[] {
+            needToEnableTLS ? ",\"cluster_needs_tls\": true" : "",
+            mockNodeVersion,
+            Version.CURRENT,
+            Version.CURRENT,
+            apmIndicesExist };
+        final String expectedJson = Strings.format("""
             {
               "cluster_uuid": "_cluster",
               "timestamp": "2017-08-07T12:03:22.133Z",
@@ -729,7 +739,8 @@ public void testToXContent() throws IOException {
                     },
                     "roles": [
                       "master"
-                    ]
+                    ],
+                    "version": "%s"
                   }
                 }
               },
@@ -752,7 +763,7 @@ public void testToXContent() throws IOException {
                   }
                 }
               }
-            }""", needToEnableTLS ? ",\"cluster_needs_tls\": true" : "", mockNodeVersion, Version.CURRENT, apmIndicesExist);
+            }""", args);
         assertEquals(stripWhitespace(expectedJson), xContent.utf8ToString());
     }
 
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryCollectorTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryCollectorTests.java
index dc3e0757e601..c9b1138a18ea 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryCollectorTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryCollectorTests.java
@@ -102,7 +102,7 @@ public void testDoCollect() throws Exception {
             ShardId shardId = new ShardId("_index_" + i, "_uuid_" + i, i);
             RecoverySource source = RecoverySource.PeerRecoverySource.INSTANCE;
             final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "_index_info_" + i);
-            final ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, true, source, unassignedInfo)
+            final ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, false, source, unassignedInfo, ShardRouting.Role.DEFAULT)
                 .initialize(localNode.getId(), "_allocation_id", 10 * i);
 
             final RecoveryState recoveryState = new RecoveryState(shardRouting, localNode, localNode);
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryMonitoringDocTests.java
index 1a75d658f481..99524f84ad1e 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryMonitoringDocTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryMonitoringDocTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.transport.NodeDisconnectedException;
@@ -104,10 +105,14 @@ public void testToXContent() throws IOException {
         );
 
         final ShardId shardId = new ShardId("_index_a", "_uuid_a", 0);
-        final RecoverySource source = RecoverySource.PeerRecoverySource.INSTANCE;
         final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "_index_info_a");
-        final ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, true, source, unassignedInfo)
-            .initialize("_node_id", "_allocation_id", 123L);
+        final ShardRouting shardRouting = ShardRouting.newUnassigned(
+            shardId,
+            false,
+            RecoverySource.PeerRecoverySource.INSTANCE,
+            unassignedInfo,
+            ShardRouting.Role.DEFAULT
+        ).initialize("_node_id", "_allocation_id", 123L);
 
         final Map<String, List<RecoveryState>> shardRecoveryStates = new HashMap<>();
         final RecoveryState recoveryState = new RecoveryState(shardRouting, discoveryNodeOne, discoveryNodeOne);
@@ -131,7 +136,7 @@ public void testToXContent() throws IOException {
         );
 
         final BytesReference xContent = XContentHelper.toXContent(document, XContentType.JSON, false);
-        final String expected = XContentHelper.stripWhitespace(formatted("""
+        final String expected = XContentHelper.stripWhitespace(Strings.format("""
             {
               "cluster_uuid": "_cluster",
               "timestamp": "2017-08-09T08:18:59.402Z",
@@ -152,7 +157,7 @@ public void testToXContent() throws IOException {
                     "id": 0,
                     "type": "PEER",
                     "stage": "INIT",
-                    "primary": true,
+                    "primary": false,
                     "start_time_in_millis": %s,
                     "stop_time_in_millis": %s,
                     "total_time_in_millis": %s,
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java
index e9e2a19df690..9ace219c7c2d 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.cache.query.QueryCacheStats;
 import org.elasticsearch.index.cache.request.RequestCacheStats;
@@ -156,7 +157,7 @@ public void testToXContent() throws IOException {
         }
 
         // indexStatsSummary()
-        final String expected = stripWhitespace(formatted("""
+        final String expected = stripWhitespace(Strings.format("""
             {
               "cluster_uuid": "_cluster",
               "timestamp": "2017-08-09T08:18:59.402Z",
@@ -320,7 +321,7 @@ public void testToXContentWithNullStats() throws IOException {
         );
 
         final BytesReference xContent = XContentHelper.toXContent(document, XContentType.JSON, false);
-        final String expected = stripWhitespace(formatted("""
+        final String expected = stripWhitespace(Strings.format("""
             {
               "cluster_uuid": "_cluster",
               "timestamp": "2017-08-09T08:18:59.402Z",
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsCollectorTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsCollectorTests.java
index 08949d77c57a..8bfec539f128 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsCollectorTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsCollectorTests.java
@@ -7,6 +7,8 @@
 package org.elasticsearch.xpack.monitoring.collector.shards;
 
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
@@ -20,7 +22,10 @@
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.Random;
 import java.util.UUID;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
@@ -30,6 +35,7 @@
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.matchesPattern;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.ArgumentMatchers.eq;
@@ -77,7 +83,12 @@ public void testDoCollect() throws Exception {
         final String stateUUID = UUID.randomUUID().toString();
         when(clusterState.stateUUID()).thenReturn(stateUUID);
 
-        final String[] indices = randomFrom(NONE, Strings.EMPTY_ARRAY, new String[] { "_all" }, new String[] { "_index*" });
+        final String[] indices = randomFrom(
+            NONE,
+            Strings.EMPTY_ARRAY,
+            new String[] { "_all" },
+            new String[] { "_index*", "_does-not-exist" }
+        );
         withCollectionIndices(indices);
 
         final RoutingTable routingTable = mockRoutingTable();
@@ -111,7 +122,11 @@ public void testDoCollect() throws Exception {
             assertThat(document.getIntervalMillis(), equalTo(interval));
             assertThat(document.getSystem(), is(MonitoredSystem.ES));
             assertThat(document.getType(), equalTo(ShardMonitoringDoc.TYPE));
-            assertThat(document.getId(), equalTo(ShardMonitoringDoc.id(stateUUID, document.getShardRouting())));
+            if (document.getShardRouting().primary()) {
+                assertThat(document.getId(), equalTo(ShardMonitoringDoc.id(stateUUID, document.getShardRouting(), 0)));
+            } else {
+                assertThat(document.getId(), matchesPattern("^[\\w-]+:(_current|_na)+:_index:s\\d:r[1-9]+[0-9]*"));
+            }
             assertThat(document.getClusterStateUUID(), equalTo(stateUUID));
 
             if (document.getShardRouting().assignedToNode()) {
@@ -130,15 +145,60 @@ public void testDoCollect() throws Exception {
     private static RoutingTable mockRoutingTable() {
         final List<ShardRouting> allShards = new ArrayList<>();
 
-        final int nbShards = randomIntBetween(0, 10);
-        for (int i = 0; i < nbShards; i++) {
+        final int numberOfPrimaryShards = randomIntBetween(0, 10);
+        for (int i = 0; i < numberOfPrimaryShards; i++) {
             ShardRoutingState state = randomFrom(STARTED, UNASSIGNED);
             ShardId shardId = new ShardId("_index", randomAlphaOfLength(12), i);
-            allShards.add(TestShardRouting.newShardRouting(shardId, state == STARTED ? "_current" : null, true, state));
+            ShardRouting primary = TestShardRouting.newShardRouting(shardId, state == STARTED ? "_current" : null, true, state);
+            allShards.add(primary);
+        }
+
+        final int numberOfReplicaShards = randomIntBetween(0, 3);
+        for (int i = 0; i < numberOfPrimaryShards; i++) {
+            for (int j = 0; j < numberOfReplicaShards; j++) {
+                ShardRoutingState state = randomFrom(STARTED, UNASSIGNED);
+                ShardId shardId = new ShardId("_index", randomAlphaOfLength(12), i);
+                ShardRouting replica = TestShardRouting.newShardRouting(shardId, state == STARTED ? "_current" : null, false, state);
+                allShards.add(replica);
+            }
         }
 
         final RoutingTable routingTable = mock(RoutingTable.class);
+        // _index* matches the test data above
+        when(routingTable.allShards("_index*")).thenReturn(allShards);
+        // _all is reserved to mean all indices in the routing table
+        when(routingTable.allShards("_all")).thenReturn(allShards);
+        // When collection indices is set to [], it's treated the same as "_all", so the key set of the routing table is used to grab the
+        // index names
+        when(routingTable.allShards("_index")).thenReturn(allShards);
+        // This mock routing table only has the index named "_index", so if collection indices is set to ["_none"] no shards should be
+        // found.
+        when(routingTable.allShards("_none")).thenReturn(new ArrayList<>(0));
+
+        final IndexRoutingTable indexRoutingTable = mock(IndexRoutingTable.class);
+        final Map<String, IndexRoutingTable> indicesRouting = Map.of("_index", indexRoutingTable);
+        when(routingTable.indicesRouting()).thenReturn(indicesRouting);
+        when(routingTable.index("_index")).thenReturn(indexRoutingTable);
+
+        when(indexRoutingTable.size()).thenReturn(numberOfPrimaryShards);
+        for (int i = 0; i < numberOfPrimaryShards; i++) {
+            final IndexShardRoutingTable shardRoutingTable = mock(IndexShardRoutingTable.class);
+            when(indexRoutingTable.shard(i)).thenReturn(shardRoutingTable);
+            when(shardRoutingTable.primaryShard()).thenReturn(allShards.get(i));
+            List<ShardRouting> replicas = new ArrayList<>();
+            int replicaIndexStart = numberOfPrimaryShards + i * numberOfReplicaShards;
+            int replicaIndexEnd = replicaIndexStart + numberOfReplicaShards;
+            for (int j = replicaIndexStart; j < replicaIndexEnd; j++) {
+                replicas.add(allShards.get(j));
+            }
+            when(shardRoutingTable.replicaShards()).thenReturn(replicas);
+        }
+
+        // This is only used by the test to decide how many shards should be covered
         when(routingTable.allShards()).thenReturn(allShards);
+
+        Collections.shuffle(allShards, new Random(numberOfPrimaryShards));
+
         return routingTable;
     }
 }
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsMonitoringDocTests.java
index b23130f7b6a3..b001ad42ad3c 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsMonitoringDocTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsMonitoringDocTests.java
@@ -58,14 +58,14 @@ protected ShardMonitoringDoc createMonitoringDoc(
         String type,
         String id
     ) {
-        return new ShardMonitoringDoc(cluster, timestamp, interval, node, shardRouting, stateUuid);
+        return new ShardMonitoringDoc(cluster, timestamp, interval, node, shardRouting, stateUuid, shardRouting.primary() ? 0 : 1);
     }
 
     @Override
     protected void assertFilteredMonitoringDoc(final ShardMonitoringDoc document) {
         assertThat(document.getSystem(), is(MonitoredSystem.ES));
         assertThat(document.getType(), is(ShardMonitoringDoc.TYPE));
-        assertThat(document.getId(), equalTo(ShardMonitoringDoc.id(stateUuid, shardRouting)));
+        assertThat(document.getId(), equalTo(ShardMonitoringDoc.id(stateUuid, shardRouting, shardRouting.primary() ? 0 : 1)));
 
         assertThat(document.getShardRouting(), is(shardRouting));
         if (assignedToNode) {
@@ -82,31 +82,31 @@ protected Set<String> getExpectedXContentFilters() {
     }
 
     public void testConstructorShardRoutingMustNotBeNull() {
-        expectThrows(NullPointerException.class, () -> new ShardMonitoringDoc(cluster, timestamp, interval, node, null, stateUuid));
+        expectThrows(NullPointerException.class, () -> new ShardMonitoringDoc(cluster, timestamp, interval, node, null, stateUuid, 0));
     }
 
     public void testConstructorStateUuidMustNotBeNull() {
-        expectThrows(NullPointerException.class, () -> new ShardMonitoringDoc(cluster, timestamp, interval, node, shardRouting, null));
+        expectThrows(NullPointerException.class, () -> new ShardMonitoringDoc(cluster, timestamp, interval, node, shardRouting, null, 0));
     }
 
     public void testIdWithPrimaryShardAssigned() {
         shardRouting = newShardRouting("_index_0", 123, "_node_0", randomAlphaOfLength(5), true, INITIALIZING);
-        assertEquals("_state_uuid_0:_node_0:_index_0:123:p", ShardMonitoringDoc.id("_state_uuid_0", shardRouting));
+        assertEquals("_state_uuid_0:_node_0:_index_0:s123:p", ShardMonitoringDoc.id("_state_uuid_0", shardRouting, 0));
     }
 
     public void testIdWithReplicaShardAssigned() {
         shardRouting = newShardRouting("_index_1", 456, "_node_1", randomAlphaOfLength(5), false, INITIALIZING);
-        assertEquals("_state_uuid_1:_node_1:_index_1:456:r", ShardMonitoringDoc.id("_state_uuid_1", shardRouting));
+        assertEquals("_state_uuid_1:_node_1:_index_1:s456:r1", ShardMonitoringDoc.id("_state_uuid_1", shardRouting, 1));
     }
 
     public void testIdWithPrimaryShardUnassigned() {
         shardRouting = newShardRouting("_index_2", 789, null, true, UNASSIGNED);
-        assertEquals("_state_uuid_2:_na:_index_2:789:p", ShardMonitoringDoc.id("_state_uuid_2", shardRouting));
+        assertEquals("_state_uuid_2:_na:_index_2:s789:p", ShardMonitoringDoc.id("_state_uuid_2", shardRouting, 0));
     }
 
     public void testIdWithReplicaShardUnassigned() {
         shardRouting = newShardRouting("_index_3", 159, null, false, UNASSIGNED);
-        assertEquals("_state_uuid_3:_na:_index_3:159:r", ShardMonitoringDoc.id("_state_uuid_3", shardRouting));
+        assertEquals("_state_uuid_3:_na:_index_3:s159:r1", ShardMonitoringDoc.id("_state_uuid_3", shardRouting, 1));
     }
 
     @Override
@@ -119,7 +119,8 @@ public void testToXContent() throws IOException {
             1506593717631L,
             node,
             shardRouting,
-            "_state_uuid"
+            "_state_uuid",
+            0
         );
 
         final BytesReference xContent = XContentHelper.toXContent(doc, XContentType.JSON, randomBoolean());
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/ExportersTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/ExportersTests.java
index 84bbe4ae21a1..4f6bbac87069 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/ExportersTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/ExportersTests.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
@@ -42,7 +43,6 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
@@ -154,7 +154,7 @@ public void testExporterIndexPattern() {
         int year = zonedDateTime.getYear();
         int month = zonedDateTime.getMonthValue();
         int day = zonedDateTime.getDayOfMonth();
-        String expecdateDate = String.format(Locale.ROOT, "%02d.%02d.%02d", year, month, day);
+        String expecdateDate = Strings.format("%02d.%02d.%02d", year, month, day);
         String formattedDate = formatter.format(instant);
         assertThat("input date was " + instant, expecdateDate, is(formattedDate));
     }
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulkResponseListenerTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulkResponseListenerTests.java
index c21379a49b2e..0263a65ca26e 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulkResponseListenerTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulkResponseListenerTests.java
@@ -10,6 +10,7 @@
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.DeprecationHandler;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -78,7 +79,7 @@ public void testOnSuccessWithInnerErrors() {
         final String[] expectedErrors = new String[] { randomAlphaOfLengthBetween(4, 10), randomAlphaOfLengthBetween(5, 9) };
         final AtomicInteger counter = new AtomicInteger(0);
         final Response response = mock(Response.class);
-        final StringEntity entity = new StringEntity(formatted("""
+        final StringEntity entity = new StringEntity(Strings.format("""
             {
               "took": 4,
               "errors": true,
diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldLuceneVersions.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldLuceneVersions.java
index 99cac1d84a82..937554f6a06a 100644
--- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldLuceneVersions.java
+++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldLuceneVersions.java
@@ -18,9 +18,11 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.ReferenceDocs;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.lucene.Lucene;
@@ -99,7 +101,7 @@ public Collection<Object> createComponents(
         final IndexNameExpressionResolver resolver,
         final Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         this.failShardsListener.set(new FailShardsOnInvalidLicenseClusterListener(getLicenseState(), clusterService.getRerouteService()));
         if (DiscoveryNode.isMasterNode(environment.settings())) {
@@ -191,7 +193,17 @@ private static void convertToNewFormat(IndexShard indexShard) {
             // clean older segments file
             Lucene.pruneUnreferencedFiles(segmentInfos.getSegmentsFileName(), indexShard.store().directory());
         } catch (IOException e) {
-            throw new UncheckedIOException(e);
+            throw new UncheckedIOException(
+                Strings.format(
+                    """
+                        Elasticsearch version [{}] has limited support for indices created in version [{}] but this index could not be \
+                        read. It may be using an unsupported feature, or it may be damaged or corrupt. See {} for further information.""",
+                    Version.CURRENT,
+                    IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(indexShard.indexSettings().getSettings()),
+                    ReferenceDocs.ARCHIVE_INDICES
+                ),
+                e
+            );
         } finally {
             indexShard.store().decRef();
         }
diff --git a/x-pack/plugin/profiler/build.gradle b/x-pack/plugin/profiler/build.gradle
new file mode 100644
index 000000000000..80624ec9a181
--- /dev/null
+++ b/x-pack/plugin/profiler/build.gradle
@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+apply plugin: 'elasticsearch.internal-cluster-test'
+apply plugin: 'elasticsearch.internal-es-plugin'
+
+esplugin {
+  name 'x-pack-profiling'
+  description 'The profiler plugin adds support for retrieving data from Universal Profiler.'
+  classname 'org.elasticsearch.xpack.profiler.ProfilingPlugin'
+}
diff --git a/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/CancellationIT.java b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/CancellationIT.java
new file mode 100644
index 000000000000..d04289298a35
--- /dev/null
+++ b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/CancellationIT.java
@@ -0,0 +1,233 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiler;
+
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.logging.log4j.LogManager;
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
+import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.client.Cancellable;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.script.MockScriptPlugin;
+import org.elasticsearch.search.lookup.LeafStoredFieldsLookup;
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.tasks.TaskInfo;
+import org.elasticsearch.tasks.TaskManager;
+import org.elasticsearch.transport.TransportService;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+
+import static org.elasticsearch.action.support.ActionTestUtils.wrapAsRestResponseListener;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class CancellationIT extends ProfilingTestCase {
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
+        plugins.add(ScriptedBlockPlugin.class);
+        return plugins;
+    }
+
+    @Override
+    protected boolean useOnlyAllEvents() {
+        // we assume that all indices have been created to simplify the testing logic.
+        return false;
+    }
+
+    public void testAutomaticCancellation() throws Exception {
+        Request restRequest = new Request("POST", "/_profiling/stacktraces");
+        restRequest.setEntity(new StringEntity("""
+                {
+                  "sample_size": 10000,
+                  "query": {
+                    "bool": {
+                      "filter": [
+                        {
+                          "script": {
+                            "script": {
+                              "lang": "mockscript",
+                              "source": "search_block",
+                              "params": {}
+                            }
+                          }
+                        }
+                      ]
+                    }
+                  }
+                }
+            """, ContentType.APPLICATION_JSON.withCharset(StandardCharsets.UTF_8)));
+        verifyCancellation(GetProfilingAction.NAME, restRequest);
+    }
+
+    void verifyCancellation(String action, Request restRequest) throws Exception {
+        Map<String, String> nodeIdToName = readNodesInfo();
+        List<ScriptedBlockPlugin> plugins = initBlockFactory();
+
+        PlainActionFuture<Response> future = PlainActionFuture.newFuture();
+        Cancellable cancellable = getRestClient().performRequestAsync(restRequest, wrapAsRestResponseListener(future));
+
+        awaitForBlock(plugins);
+        Collection<TaskId> profilingTasks = collectProfilingRelatedTasks(action);
+        cancellable.cancel();
+        ensureTasksAreCancelled(profilingTasks, nodeIdToName::get);
+
+        disableBlocks(plugins);
+        expectThrows(CancellationException.class, future::actionGet);
+    }
+
+    private static Map<String, String> readNodesInfo() {
+        Map<String, String> nodeIdToName = new HashMap<>();
+        NodesInfoResponse nodesInfoResponse = client().admin().cluster().prepareNodesInfo().get();
+        assertFalse(nodesInfoResponse.hasFailures());
+        for (NodeInfo node : nodesInfoResponse.getNodes()) {
+            nodeIdToName.put(node.getNode().getId(), node.getNode().getName());
+        }
+        return nodeIdToName;
+    }
+
+    private static Collection<TaskId> collectProfilingRelatedTasks(String transportAction) {
+        SetOnce<TaskInfo> profilingTask = new SetOnce<>();
+        Map<TaskId, Set<TaskId>> taskToParent = new HashMap<>();
+        ListTasksResponse listTasksResponse = client().admin().cluster().prepareListTasks().get();
+        for (TaskInfo task : listTasksResponse.getTasks()) {
+            TaskId parentTaskId = task.parentTaskId();
+            if (parentTaskId != null) {
+                if (taskToParent.containsKey(parentTaskId) == false) {
+                    taskToParent.put(parentTaskId, new HashSet<>());
+                }
+                taskToParent.get(parentTaskId).add(task.taskId());
+            }
+            if (task.action().equals(transportAction)) {
+                profilingTask.set(task);
+            }
+        }
+        assertNotNull(profilingTask.get());
+        Set<TaskId> childTaskIds = taskToParent.get(profilingTask.get().taskId());
+        Set<TaskId> profilingTaskIds = new HashSet<>();
+        profilingTaskIds.add(profilingTask.get().taskId());
+        if (childTaskIds != null) {
+            profilingTaskIds.addAll(childTaskIds);
+        }
+        return profilingTaskIds;
+    }
+
+    private static void ensureTasksAreCancelled(Collection<TaskId> taskIds, Function<String, String> nodeIdToName) throws Exception {
+        assertBusy(() -> {
+            for (TaskId taskId : taskIds) {
+                String nodeName = nodeIdToName.apply(taskId.getNodeId());
+                TaskManager taskManager = internalCluster().getInstance(TransportService.class, nodeName).getTaskManager();
+                Task task = taskManager.getTask(taskId.getId());
+                // as we capture the task hierarchy at the beginning but cancel in the middle of execution, some tasks have been
+                // unregistered already by the time we verify cancellation.
+                if (task != null) {
+                    assertThat(task, instanceOf(CancellableTask.class));
+                    assertTrue(((CancellableTask) task).isCancelled());
+                }
+            }
+        });
+    }
+
+    private static List<ScriptedBlockPlugin> initBlockFactory() {
+        List<ScriptedBlockPlugin> plugins = new ArrayList<>();
+        for (PluginsService pluginsService : internalCluster().getDataNodeInstances(PluginsService.class)) {
+            plugins.addAll(pluginsService.filterPlugins(ScriptedBlockPlugin.class));
+        }
+        for (ScriptedBlockPlugin plugin : plugins) {
+            plugin.reset();
+            plugin.enableBlock();
+            // Allow to execute one search and only block starting with the second one. This
+            // is done so we have at least one child action and can check that all active children
+            // are cancelled with the parent action.
+            plugin.setSlack(1);
+        }
+        return plugins;
+    }
+
+    private void awaitForBlock(List<ScriptedBlockPlugin> plugins) throws Exception {
+        assertBusy(() -> {
+            int numberOfBlockedPlugins = 0;
+            for (ScriptedBlockPlugin plugin : plugins) {
+                numberOfBlockedPlugins += plugin.hits.get();
+            }
+            logger.info("The plugin blocked on {} shards", numberOfBlockedPlugins);
+            assertThat(numberOfBlockedPlugins, greaterThan(0));
+        }, 10, TimeUnit.SECONDS);
+    }
+
+    private static void disableBlocks(List<ScriptedBlockPlugin> plugins) {
+        for (ScriptedBlockPlugin plugin : plugins) {
+            plugin.disableBlock();
+        }
+    }
+
+    public static class ScriptedBlockPlugin extends MockScriptPlugin {
+        static final String SCRIPT_NAME = "search_block";
+
+        private final AtomicInteger hits = new AtomicInteger();
+
+        private final AtomicInteger slack = new AtomicInteger(0);
+
+        private final AtomicBoolean shouldBlock = new AtomicBoolean(true);
+
+        void reset() {
+            hits.set(0);
+        }
+
+        void disableBlock() {
+            shouldBlock.set(false);
+        }
+
+        void enableBlock() {
+            shouldBlock.set(true);
+        }
+
+        void setSlack(int slack) {
+            this.slack.set(slack);
+        }
+
+        @Override
+        public Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
+            return Collections.singletonMap(SCRIPT_NAME, params -> {
+                LeafStoredFieldsLookup fieldsLookup = (LeafStoredFieldsLookup) params.get("_fields");
+                LogManager.getLogger(CancellationIT.class).info("Blocking on the document {}", fieldsLookup.get("_id"));
+                hits.incrementAndGet();
+                if (slack.decrementAndGet() < 0) {
+                    try {
+                        waitUntil(() -> shouldBlock.get() == false);
+                    } catch (Exception e) {
+                        throw new RuntimeException(e);
+                    }
+                }
+                return true;
+            });
+        }
+    }
+}
diff --git a/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/GetProfilingActionIT.java b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/GetProfilingActionIT.java
new file mode 100644
index 000000000000..dae5602ea380
--- /dev/null
+++ b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/GetProfilingActionIT.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiler;
+
+import org.elasticsearch.rest.RestStatus;
+
+import java.util.List;
+
+public class GetProfilingActionIT extends ProfilingTestCase {
+    @Override
+    protected boolean useOnlyAllEvents() {
+        return randomBoolean();
+    }
+
+    public void testGetProfilingDataUnfiltered() throws Exception {
+        GetProfilingRequest request = new GetProfilingRequest(1, null);
+        GetProfilingResponse response = client().execute(GetProfilingAction.INSTANCE, request).get();
+        assertEquals(RestStatus.OK, response.status());
+        assertEquals(1, response.getTotalFrames());
+        assertNotNull(response.getStackTraces());
+        StackTrace stackTrace = response.getStackTraces().get("QjoLteG7HX3VUUXr-J4kHQ");
+        assertArrayEquals(new int[] { 1083999 }, stackTrace.addressOrLines);
+        assertArrayEquals(new String[] { "QCCDqjSg3bMK1C4YRK6Tiw" }, stackTrace.fileIds);
+        assertArrayEquals(new String[] { "QCCDqjSg3bMK1C4YRK6TiwAAAAAAEIpf" }, stackTrace.frameIds);
+        assertArrayEquals(new int[] { 2 }, stackTrace.typeIds);
+
+        assertNotNull(response.getStackFrames());
+        StackFrame stackFrame = response.getStackFrames().get("QCCDqjSg3bMK1C4YRK6TiwAAAAAAEIpf");
+        assertEquals(List.of("_raw_spin_unlock_irqrestore", "inlined_frame_1", "inlined_frame_0"), stackFrame.functionName);
+        assertNotNull(response.getStackTraceEvents());
+        assertEquals(1, (int) response.getStackTraceEvents().get("QjoLteG7HX3VUUXr-J4kHQ"));
+
+        assertNotNull(response.getExecutables());
+        assertNotNull("libc.so.6", response.getExecutables().get("QCCDqjSg3bMK1C4YRK6Tiw"));
+    }
+}
diff --git a/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java
new file mode 100644
index 000000000000..e79a79d4bbc2
--- /dev/null
+++ b/x-pack/plugin/profiler/src/internalClusterTest/java/org/elasticsearch/xpack/profiler/ProfilingTestCase.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiler;
+
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.transport.netty4.Netty4Plugin;
+import org.elasticsearch.xcontent.XContentType;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 1)
+public abstract class ProfilingTestCase extends ESIntegTestCase {
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(ProfilingPlugin.class, getTestTransportPlugin());
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal, otherSettings))
+            .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME)
+            .put(NetworkModule.HTTP_TYPE_KEY, Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME)
+            .build();
+    }
+
+    @Override
+    protected boolean addMockHttpTransport() {
+        return false; // enable http
+    }
+
+    @Override
+    protected boolean ignoreExternalCluster() {
+        return true;
+    }
+
+    private byte[] read(String resource) throws IOException {
+        return GetProfilingAction.class.getClassLoader().getResourceAsStream(resource).readAllBytes();
+    }
+
+    private void createIndex(String name, String bodyFileName) throws Exception {
+        client().admin().indices().prepareCreate(name).setSource(read(bodyFileName), XContentType.JSON).execute().get();
+    }
+
+    private void indexDoc(String index, String id, Map<String, Object> source) {
+        IndexResponse indexResponse = client().prepareIndex(index).setId(id).setSource(source).get();
+        assertEquals(RestStatus.CREATED, indexResponse.status());
+    }
+
+    /**
+     * Only the index "profiling-events-all" is always present. All other indices (e.g. "profiling-events-5pow02") are created on demand
+     * at a later point when there are enough samples. With this flag we simulate that data should be retrieved briefly after cluster
+     * start when only profiling-events-all is present. We expect that also in this case, available data is returned but we rely only
+     * on the single existing index.
+     *
+     * @return <code>true</code> iff this test should rely on only "profiling-events-all" being present.
+     */
+    protected abstract boolean useOnlyAllEvents();
+
+    @Before
+    public void setupData() throws Exception {
+        Collection<String> eventsIndices = useOnlyAllEvents() ? List.of(EventsIndex.FULL_INDEX.getName()) : EventsIndex.indexNames();
+
+        for (String idx : eventsIndices) {
+            createIndex(idx, "events.json");
+        }
+        createIndex("profiling-stackframes", "stackframes.json");
+        createIndex("profiling-stacktraces", "stacktraces.json");
+        createIndex("profiling-executables", "executables.json");
+        ensureGreen();
+
+        // ensure that we have this in every index, so we find an event
+        for (String idx : eventsIndices) {
+            indexDoc(
+                idx,
+                "QjoLteG7HX3VUUXr-J4kHQ",
+                Map.of("@timestamp", 1668761065, "Stacktrace.id", "QjoLteG7HX3VUUXr-J4kHQ", "Stacktrace.count", 1)
+            );
+        }
+
+        indexDoc(
+            "profiling-stacktraces",
+            "QjoLteG7HX3VUUXr-J4kHQ",
+            Map.of("Stacktrace.frame.ids", "QCCDqjSg3bMK1C4YRK6TiwAAAAAAEIpf", "Stacktrace.frame.types", "AQI")
+        );
+        indexDoc(
+            "profiling-stackframes",
+            "QCCDqjSg3bMK1C4YRK6TiwAAAAAAEIpf",
+            Map.of("Stackframe.function.name", List.of("_raw_spin_unlock_irqrestore", "inlined_frame_1", "inlined_frame_0"))
+        );
+        indexDoc("profiling-executables", "QCCDqjSg3bMK1C4YRK6Tiw", Map.of("Executable.file.name", "libc.so.6"));
+
+        refresh();
+    }
+}
diff --git a/x-pack/plugin/profiler/src/internalClusterTest/resources/events.json b/x-pack/plugin/profiler/src/internalClusterTest/resources/events.json
new file mode 100644
index 000000000000..855c754191db
--- /dev/null
+++ b/x-pack/plugin/profiler/src/internalClusterTest/resources/events.json
@@ -0,0 +1,78 @@
+{
+  "settings": {
+    "index": {
+      "number_of_shards": "4",
+      "max_result_window": 150000,
+      "refresh_interval": "10s",
+      "sort": {
+        "field": [
+          "service.name",
+          "@timestamp",
+          "orchestrator.resource.name",
+          "container.name",
+          "process.thread.name",
+          "host.id"
+        ]
+      }
+    },
+    "codec": "best_compression"
+  },
+  "mappings": {
+    "_doc": {
+      "_source": {
+        "enabled": false
+      },
+      "properties": {
+        "ecs.version": {
+          "type": "keyword",
+          "index": true
+        },
+        "service.name": {
+          "type": "keyword"
+        },
+        "@timestamp": {
+          "type": "date",
+          "format": "epoch_second"
+        },
+        "host.id": {
+          "type": "keyword"
+        },
+        "Stacktrace.id": {
+          "type": "keyword",
+          "index": false
+        },
+        "orchestrator.resource.name": {
+          "type": "keyword"
+        },
+        "container.name": {
+          "type": "keyword"
+        },
+        "process.thread.name": {
+          "type": "keyword"
+        },
+        "Stacktrace.count": {
+          "type": "short",
+          "index": false
+        },
+        "agent.version": {
+          "type": "keyword"
+        },
+        "host.ip": {
+          "type": "ip"
+        },
+        "host.ipstring": {
+          "type": "keyword"
+        },
+        "host.name": {
+          "type": "keyword"
+        },
+        "os.kernel": {
+          "type": "keyword"
+        },
+        "tags": {
+          "type": "keyword"
+        }
+      }
+    }
+  }
+}
diff --git a/x-pack/plugin/profiler/src/internalClusterTest/resources/executables.json b/x-pack/plugin/profiler/src/internalClusterTest/resources/executables.json
new file mode 100644
index 000000000000..f5db60318bcf
--- /dev/null
+++ b/x-pack/plugin/profiler/src/internalClusterTest/resources/executables.json
@@ -0,0 +1,32 @@
+{
+  "settings": {
+    "index": {
+      "refresh_interval": "10s"
+    }
+  },
+  "mappings": {
+    "_doc": {
+      "_source": {
+        "mode": "synthetic"
+      },
+      "properties": {
+        "ecs.version": {
+          "type": "keyword",
+          "index": true
+        },
+        "Executable.build.id": {
+          "type": "keyword",
+          "index": true
+        },
+        "Executable.file.name": {
+          "type": "keyword",
+          "index": true
+        },
+        "@timestamp": {
+          "type": "date",
+          "format": "epoch_second"
+        }
+      }
+    }
+  }
+}
diff --git a/x-pack/plugin/profiler/src/internalClusterTest/resources/stackframes.json b/x-pack/plugin/profiler/src/internalClusterTest/resources/stackframes.json
new file mode 100644
index 000000000000..ced7b081a3d3
--- /dev/null
+++ b/x-pack/plugin/profiler/src/internalClusterTest/resources/stackframes.json
@@ -0,0 +1,41 @@
+{
+  "settings": {
+    "index": {
+      "number_of_shards": "16",
+      "refresh_interval": "10s"
+    }
+  },
+  "mappings": {
+    "_doc": {
+      "_source": {
+        "enabled": true
+      },
+      "properties": {
+        "ecs.version": {
+          "type": "keyword",
+          "index": true,
+          "doc_values": false,
+          "store": false
+        },
+        "Stackframe.line.number": {
+          "type": "integer",
+          "index": false,
+          "doc_values": false,
+          "store": false
+        },
+        "Stackframe.file.name": {
+          "type": "keyword",
+          "index": false,
+          "doc_values": false,
+          "store": false
+        },
+        "Stackframe.function.name": {
+          "type": "keyword",
+          "index": false,
+          "doc_values": false,
+          "store": false
+        }
+      }
+    }
+  }
+}
diff --git a/x-pack/plugin/profiler/src/internalClusterTest/resources/stacktraces.json b/x-pack/plugin/profiler/src/internalClusterTest/resources/stacktraces.json
new file mode 100644
index 000000000000..0b72dccdfe5c
--- /dev/null
+++ b/x-pack/plugin/profiler/src/internalClusterTest/resources/stacktraces.json
@@ -0,0 +1,38 @@
+{
+  "settings": {
+    "index": {
+      "number_of_shards": "16",
+      "refresh_interval": "10s",
+      "sort": {
+        "field": [
+          "Stacktrace.frame.ids"
+        ]
+      }
+    }
+  },
+  "mappings": {
+    "_doc": {
+      "_source": {
+        "mode": "synthetic"
+      },
+      "properties": {
+        "ecs.version": {
+          "type": "keyword",
+          "index": true
+        },
+        "Stacktrace.frame.ids": {
+          "type": "keyword",
+          "index": false
+        },
+        "Stacktrace.frame.types": {
+          "type": "keyword",
+          "index": false
+        },
+        "@timestamp": {
+          "type": "date",
+          "format": "epoch_second"
+        }
+      }
+    }
+  }
+}
diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/EventsIndex.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/EventsIndex.java
new file mode 100644
index 000000000000..441a5bbd6cd2
--- /dev/null
+++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/EventsIndex.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiler;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Locale;
+import java.util.Set;
+
+public final class EventsIndex {
+    private static final String PREFIX = "profiling-events";
+    private static final String ALL_EVENTS = PREFIX + "-all";
+
+    private static final int SAMPLING_FACTOR = 5;
+
+    private static final int MIN_EXPONENT = 1;
+
+    private static final int MAX_EXPONENT = 11;
+
+    public static final EventsIndex FULL_INDEX = new EventsIndex(ALL_EVENTS, 1, 1);
+
+    // Start with counting the results in the index down-sampled by 5^6.
+    // That is in the middle of our down-sampled indexes.
+    public static final EventsIndex MEDIUM_DOWNSAMPLED = fromFactorAndExponent(SAMPLING_FACTOR, 6);
+
+    private final String name;
+
+    private final int samplingFactor;
+
+    private final int exponent;
+
+    private EventsIndex(String name, int samplingFactor, int exponent) {
+        this.name = name;
+        this.samplingFactor = samplingFactor;
+        this.exponent = exponent;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public int getExponent() {
+        return exponent;
+    }
+
+    public double getSampleRate() {
+        return Math.pow(1.0d / samplingFactor, exponent);
+    }
+
+    public EventsIndex getResampledIndex(long targetSampleSize, long currentSampleSize) {
+        return EventsIndex.getSampledIndex(targetSampleSize, currentSampleSize, this.getExponent());
+    }
+
+    // Return the index that has between targetSampleSize..targetSampleSize*samplingFactor entries.
+    // The starting point is the number of entries from the profiling-events-5pow<initialExp> index.
+    private static EventsIndex getSampledIndex(long targetSampleSize, long sampleCountFromInitialExp, int initialExp) {
+        if (sampleCountFromInitialExp == 0) {
+            return FULL_INDEX;
+        }
+        int exp = initialExp - (int) Math.round(
+            Math.log((targetSampleSize * SAMPLING_FACTOR) / (double) sampleCountFromInitialExp) / Math.log(SAMPLING_FACTOR)
+        ) + 1;
+
+        if (exp < MIN_EXPONENT) {
+            return FULL_INDEX;
+        }
+        if (exp > MAX_EXPONENT) {
+            exp = MAX_EXPONENT;
+        }
+        return fromFactorAndExponent(SAMPLING_FACTOR, exp);
+    }
+
+    private static EventsIndex fromFactorAndExponent(int factor, int exp) {
+        return new EventsIndex(indexName(factor, exp), factor, exp);
+    }
+
+    private static String indexName(int factor, int exp) {
+        return String.format(Locale.ROOT, "%s-%dpow%02d", PREFIX, factor, exp);
+    }
+
+    public static Collection<String> indexNames() {
+        Set<String> names = new HashSet<>();
+        names.add(EventsIndex.ALL_EVENTS);
+        for (int exp = MIN_EXPONENT; exp <= MAX_EXPONENT; exp++) {
+            names.add(indexName(SAMPLING_FACTOR, exp));
+        }
+        return Collections.unmodifiableSet(names);
+    }
+}
diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/GetProfilingAction.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/GetProfilingAction.java
new file mode 100644
index 000000000000..eaec2e4b7739
--- /dev/null
+++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/GetProfilingAction.java
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.profiler;
+
+import org.elasticsearch.action.ActionType;
+
+public final class GetProfilingAction extends ActionType<GetProfilingResponse> {
+    public static final GetProfilingAction INSTANCE = new GetProfilingAction();
+    public static final String NAME = "indices:data/read/profiling";
+
+    private GetProfilingAction() {
+        super(NAME, GetProfilingResponse::new);
+    }
+}
diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/GetProfilingRequest.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/GetProfilingRequest.java
new file mode 100644
index 000000000000..13c3b2e38ea4
--- /dev/null
+++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/GetProfilingRequest.java
@@ -0,0 +1,183 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.profiler;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+import static org.elasticsearch.index.query.AbstractQueryBuilder.parseTopLevelQuery;
+
+/**
+ * A request to get profiling details
+ */
+public class GetProfilingRequest extends ActionRequest implements IndicesRequest {
+    public static final ParseField QUERY_FIELD = new ParseField("query");
+    public static final ParseField SAMPLE_SIZE_FIELD = new ParseField("sample_size");
+
+    private QueryBuilder query;
+
+    private Integer sampleSize;
+
+    public GetProfilingRequest() {
+        this(null, null);
+    }
+
+    public GetProfilingRequest(Integer sampleSize, QueryBuilder query) {
+        this.sampleSize = sampleSize;
+        this.query = query;
+    }
+
+    public GetProfilingRequest(StreamInput in) throws IOException {
+        this.query = in.readOptionalNamedWriteable(QueryBuilder.class);
+        this.sampleSize = in.readOptionalInt();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalNamedWriteable(query);
+        out.writeOptionalInt(sampleSize);
+    }
+
+    public Integer getSampleSize() {
+        return sampleSize;
+    }
+
+    public QueryBuilder getQuery() {
+        return query;
+    }
+
+    public void parseXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        String currentFieldName = null;
+        if (token != XContentParser.Token.START_OBJECT && (token = parser.nextToken()) != XContentParser.Token.START_OBJECT) {
+            throw new ParsingException(
+                parser.getTokenLocation(),
+                "Expected [" + XContentParser.Token.START_OBJECT + "] but found [" + token + "]",
+                parser.getTokenLocation()
+            );
+        }
+
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (SAMPLE_SIZE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                    this.sampleSize = parser.intValue();
+                } else {
+                    throw new ParsingException(
+                        parser.getTokenLocation(),
+                        "Unknown key for a " + token + " in [" + currentFieldName + "].",
+                        parser.getTokenLocation()
+                    );
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (QUERY_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                    this.query = parseTopLevelQuery(parser);
+                }
+            } else {
+                throw new ParsingException(
+                    parser.getTokenLocation(),
+                    "Unknown key for a " + token + " in [" + currentFieldName + "].",
+                    parser.getTokenLocation()
+                );
+            }
+        }
+
+        token = parser.nextToken();
+        if (token != null) {
+            throw new ParsingException(parser.getTokenLocation(), "Unexpected token [" + token + "] found after the main object.");
+        }
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (sampleSize == null) {
+            validationException = addValidationError("[" + SAMPLE_SIZE_FIELD.getPreferredName() + "] is mandatory", validationException);
+        } else if (sampleSize <= 0) {
+            validationException = addValidationError(
+                "[" + SAMPLE_SIZE_FIELD.getPreferredName() + "] must be greater or equals than 1, got: " + sampleSize,
+                validationException
+            );
+        }
+        return validationException;
+    }
+
+    @Override
+    public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+        return new CancellableTask(id, type, action, null, parentTaskId, headers) {
+            @Override
+            public String getDescription() {
+                // generating description lazily since the query could be large
+                StringBuilder sb = new StringBuilder();
+                sb.append("sample_size[").append(sampleSize).append("]");
+                if (query == null) {
+                    sb.append(", query[]");
+                } else {
+                    sb.append(", query[").append(Strings.toString(query)).append("]");
+                }
+                return sb.toString();
+            }
+        };
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        GetProfilingRequest that = (GetProfilingRequest) o;
+        return Objects.equals(query, that.query) && Objects.equals(sampleSize, that.sampleSize);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(query, sampleSize);
+    }
+
+    @Override
+    public String[] indices() {
+        Set<String> indices = new HashSet<>();
+        indices.add("profiling-stacktraces");
+        indices.add("profiling-stackframes");
+        indices.add("profiling-executables");
+        indices.addAll(EventsIndex.indexNames());
+        return indices.toArray(new String[0]);
+    }
+
+    @Override
+    public IndicesOptions indicesOptions() {
+        return IndicesOptions.STRICT_EXPAND_OPEN;
+    }
+
+    @Override
+    public boolean includeDataStreams() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/GetProfilingResponse.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/GetProfilingResponse.java
new file mode 100644
index 000000000000..1ae5f3dde3a4
--- /dev/null
+++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/GetProfilingResponse.java
@@ -0,0 +1,222 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.profiler;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.StatusToXContentObject;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.rest.RestStatus.OK;
+
+public class GetProfilingResponse extends ActionResponse implements StatusToXContentObject {
+    @Nullable
+    private final Map<String, StackTrace> stackTraces;
+    @Nullable
+    private final Map<String, StackFrame> stackFrames;
+    @Nullable
+    private final Map<String, String> executables;
+    @Nullable
+    private final Map<String, Integer> stackTraceEvents;
+    private final int totalFrames;
+    @Nullable
+    private final Exception error;
+
+    public GetProfilingResponse(StreamInput in) throws IOException {
+        this.stackTraces = in.readBoolean()
+            ? in.readMap(
+                StreamInput::readString,
+                i -> new StackTrace(i.readIntArray(), i.readStringArray(), i.readStringArray(), i.readIntArray())
+            )
+            : null;
+        this.stackFrames = in.readBoolean()
+            ? in.readMap(
+                StreamInput::readString,
+                i -> new StackFrame(
+                    i.readList(StreamInput::readString),
+                    i.readList(StreamInput::readString),
+                    i.readList(StreamInput::readInt),
+                    i.readList(StreamInput::readInt),
+                    i.readList(StreamInput::readInt)
+                )
+            )
+            : null;
+        this.executables = in.readBoolean() ? in.readMap(StreamInput::readString, StreamInput::readString) : null;
+        this.stackTraceEvents = in.readBoolean() ? in.readMap(StreamInput::readString, StreamInput::readInt) : null;
+        this.totalFrames = in.readInt();
+        this.error = in.readBoolean() ? in.readException() : null;
+    }
+
+    public GetProfilingResponse(
+        Map<String, StackTrace> stackTraces,
+        Map<String, StackFrame> stackFrames,
+        Map<String, String> executables,
+        Map<String, Integer> stackTraceEvents,
+        int totalFrames
+    ) {
+        this(stackTraces, stackFrames, executables, stackTraceEvents, totalFrames, null);
+    }
+
+    public GetProfilingResponse(Exception error) {
+        this(null, null, null, null, 0, error);
+    }
+
+    private GetProfilingResponse(
+        Map<String, StackTrace> stackTraces,
+        Map<String, StackFrame> stackFrames,
+        Map<String, String> executables,
+        Map<String, Integer> stackTraceEvents,
+        int totalFrames,
+        Exception error
+    ) {
+        this.stackTraces = stackTraces;
+        this.stackFrames = stackFrames;
+        this.executables = executables;
+        this.stackTraceEvents = stackTraceEvents;
+        this.totalFrames = totalFrames;
+        this.error = error;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        if (stackTraces != null) {
+            out.writeBoolean(true);
+            out.writeMap(stackTraces, StreamOutput::writeString, (o, v) -> {
+                o.writeIntArray(v.addressOrLines);
+                o.writeStringArray(v.fileIds);
+                o.writeStringArray(v.frameIds);
+                o.writeIntArray(v.typeIds);
+            });
+        } else {
+            out.writeBoolean(false);
+        }
+        if (stackFrames != null) {
+            out.writeBoolean(true);
+            out.writeMap(stackFrames, StreamOutput::writeString, (o, v) -> {
+                o.writeCollection(v.fileName, StreamOutput::writeString);
+                o.writeCollection(v.functionName, StreamOutput::writeString);
+                o.writeCollection(v.functionOffset, StreamOutput::writeInt);
+                o.writeCollection(v.lineNumber, StreamOutput::writeInt);
+                o.writeCollection(v.sourceType, StreamOutput::writeInt);
+            });
+        } else {
+            out.writeBoolean(false);
+        }
+        if (executables != null) {
+            out.writeBoolean(true);
+            out.writeMap(executables, StreamOutput::writeString, StreamOutput::writeString);
+        } else {
+            out.writeBoolean(false);
+        }
+        if (stackTraceEvents != null) {
+            out.writeBoolean(true);
+            out.writeMap(stackTraceEvents, StreamOutput::writeString, StreamOutput::writeInt);
+        } else {
+            out.writeBoolean(false);
+        }
+        out.writeInt(totalFrames);
+        if (error != null) {
+            out.writeBoolean(true);
+            out.writeException(error);
+        } else {
+            out.writeBoolean(false);
+        }
+    }
+
+    @Override
+    public RestStatus status() {
+        return error != null ? ExceptionsHelper.status(ExceptionsHelper.unwrapCause(error)) : OK;
+    }
+
+    public Map<String, StackTrace> getStackTraces() {
+        return stackTraces;
+    }
+
+    public Map<String, StackFrame> getStackFrames() {
+        return stackFrames;
+    }
+
+    public Map<String, String> getExecutables() {
+        return executables;
+    }
+
+    public Map<String, Integer> getStackTraceEvents() {
+        return stackTraceEvents;
+    }
+
+    public int getTotalFrames() {
+        return totalFrames;
+    }
+
+    public Exception getError() {
+        return error;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (stackTraces != null) {
+            builder.startObject("stack_traces");
+            builder.mapContents(stackTraces);
+            builder.endObject();
+        }
+        if (stackFrames != null) {
+            builder.startObject("stack_frames");
+            builder.mapContents(stackFrames);
+            builder.endObject();
+        }
+        if (executables != null) {
+            builder.startObject("executables");
+            builder.mapContents(executables);
+            builder.endObject();
+        }
+        if (stackTraceEvents != null) {
+            builder.startObject("stack_trace_events");
+            builder.mapContents(stackTraceEvents);
+            builder.endObject();
+        }
+        builder.field("total_frames", totalFrames);
+        if (error != null) {
+            builder.startObject("error");
+            ElasticsearchException.generateThrowableXContent(builder, params, error);
+            builder.endObject();
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        GetProfilingResponse response = (GetProfilingResponse) o;
+        return totalFrames == response.totalFrames
+            && Objects.equals(stackTraces, response.stackTraces)
+            && Objects.equals(stackFrames, response.stackFrames)
+            && Objects.equals(executables, response.executables)
+            && Objects.equals(stackTraceEvents, response.stackTraceEvents)
+            && Objects.equals(error, response.error);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(stackTraces, stackFrames, executables, stackTraceEvents, totalFrames, error);
+    }
+}
diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java
new file mode 100644
index 000000000000..ec3d70baa41d
--- /dev/null
+++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/ProfilingPlugin.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiler;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestHandler;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.tracing.Tracer;
+import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static java.util.Collections.singletonList;
+
+public class ProfilingPlugin extends Plugin implements ActionPlugin {
+    private static final Logger logger = LogManager.getLogger(ProfilingPlugin.class);
+    public static final Setting<Boolean> PROFILING_ENABLED = Setting.boolSetting(
+        "xpack.profiling.enabled",
+        true,
+        Setting.Property.NodeScope
+    );
+    private final boolean enabled;
+
+    public ProfilingPlugin(Settings settings) {
+        this.enabled = PROFILING_ENABLED.get(settings);
+    }
+
+    @Override
+    public Collection<Object> createComponents(
+        Client client,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        ResourceWatcherService resourceWatcherService,
+        ScriptService scriptService,
+        NamedXContentRegistry xContentRegistry,
+        Environment environment,
+        NodeEnvironment nodeEnvironment,
+        NamedWriteableRegistry namedWriteableRegistry,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        Supplier<RepositoriesService> repositoriesServiceSupplier,
+        Tracer tracer,
+        AllocationService allocationService
+    ) {
+        logger.info("Profiling is {}", enabled ? "enabled" : "disabled");
+        return super.createComponents(
+            client,
+            clusterService,
+            threadPool,
+            resourceWatcherService,
+            scriptService,
+            xContentRegistry,
+            environment,
+            nodeEnvironment,
+            namedWriteableRegistry,
+            indexNameExpressionResolver,
+            repositoriesServiceSupplier,
+            tracer,
+            allocationService
+        );
+    }
+
+    @Override
+    public List<RestHandler> getRestHandlers(
+        final Settings settings,
+        final RestController restController,
+        final ClusterSettings clusterSettings,
+        final IndexScopedSettings indexScopedSettings,
+        final SettingsFilter settingsFilter,
+        final IndexNameExpressionResolver indexNameExpressionResolver,
+        final Supplier<DiscoveryNodes> nodesInCluster
+    ) {
+        if (enabled) {
+            return singletonList(new RestGetProfilingAction());
+        } else {
+            return Collections.emptyList();
+        }
+    }
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        return List.of(
+            PROFILING_ENABLED,
+            TransportGetProfilingAction.PROFILING_MAX_STACKTRACE_QUERY_SLICES,
+            TransportGetProfilingAction.PROFILING_MAX_DETAIL_QUERY_SLICES,
+            TransportGetProfilingAction.PROFILING_QUERY_REALTIME
+        );
+    }
+
+    @Override
+    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        return List.of(new ActionHandler<>(GetProfilingAction.INSTANCE, TransportGetProfilingAction.class));
+    }
+}
diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/RestGetProfilingAction.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/RestGetProfilingAction.java
new file mode 100644
index 000000000000..3b831f257e71
--- /dev/null
+++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/RestGetProfilingAction.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.profiler;
+
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestCancellableNodeClient;
+import org.elasticsearch.rest.action.RestStatusToXContentListener;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+
+public class RestGetProfilingAction extends BaseRestHandler {
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(GET, "/_profiling/stacktraces"), new Route(POST, "/_profiling/stacktraces"));
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        GetProfilingRequest getProfilingRequest = new GetProfilingRequest();
+        request.applyContentParser(getProfilingRequest::parseXContent);
+
+        return channel -> {
+            RestStatusToXContentListener<GetProfilingResponse> listener = new RestStatusToXContentListener<>(channel);
+            RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel());
+            cancelClient.execute(GetProfilingAction.INSTANCE, getProfilingRequest, listener);
+        };
+    }
+
+    @Override
+    public String getName() {
+        return "get_profiling_action";
+    }
+}
diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/StackFrame.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/StackFrame.java
new file mode 100644
index 000000000000..9add1ee22db9
--- /dev/null
+++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/StackFrame.java
@@ -0,0 +1,103 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiler;
+
+import org.elasticsearch.xcontent.ObjectPath;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+final class StackFrame implements ToXContentObject {
+    List<String> fileName;
+    List<String> functionName;
+    List<Integer> functionOffset;
+    List<Integer> lineNumber;
+    List<Integer> sourceType;
+
+    StackFrame(Object fileName, Object functionName, Object functionOffset, Object lineNumber, Object sourceType) {
+        this.fileName = listOf(fileName);
+        this.functionName = listOf(functionName);
+        this.functionOffset = listOf(functionOffset);
+        this.lineNumber = listOf(lineNumber);
+        this.sourceType = listOf(sourceType);
+    }
+
+    @SuppressWarnings("unchecked")
+    private static <T> List<T> listOf(Object o) {
+        if (o instanceof List) {
+            return (List<T>) o;
+        } else if (o != null) {
+            return List.of((T) o);
+        } else {
+            return Collections.emptyList();
+        }
+    }
+
+    public static StackFrame fromSource(Map<String, Object> source) {
+        // stack frames may either be stored with synthetic source or regular one
+        // which results either in a nested or flat document structure.
+
+        if (source.containsKey("Stackframe")) {
+            // synthetic source
+            return new StackFrame(
+                ObjectPath.eval("Stackframe.file.name", source),
+                ObjectPath.eval("Stackframe.function.name", source),
+                ObjectPath.eval("Stackframe.function.offset", source),
+                ObjectPath.eval("Stackframe.line.number", source),
+                ObjectPath.eval("Stackframe.source.type", source)
+            );
+        } else {
+            // regular source
+            return new StackFrame(
+                source.get("Stackframe.file.name"),
+                source.get("Stackframe.function.name"),
+                source.get("Stackframe.function.offset"),
+                source.get("Stackframe.line.number"),
+                source.get("Stackframe.source.type")
+            );
+        }
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field("file_name", this.fileName);
+        builder.field("function_name", this.functionName);
+        builder.field("function_offset", this.functionOffset);
+        builder.field("line_number", this.lineNumber);
+        builder.field("source_type", this.sourceType);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        StackFrame that = (StackFrame) o;
+        return Objects.equals(fileName, that.fileName)
+            && Objects.equals(functionName, that.functionName)
+            && Objects.equals(functionOffset, that.functionOffset)
+            && Objects.equals(lineNumber, that.lineNumber)
+            && Objects.equals(sourceType, that.sourceType);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(fileName, functionName, functionOffset, lineNumber, sourceType);
+    }
+}
diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/StackTrace.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/StackTrace.java
new file mode 100644
index 000000000000..22c4c71887dc
--- /dev/null
+++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/StackTrace.java
@@ -0,0 +1,231 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiler;
+
+import org.elasticsearch.xcontent.ObjectPath;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Map;
+
+final class StackTrace implements ToXContentObject {
+    int[] addressOrLines;
+    String[] fileIds;
+    String[] frameIds;
+    int[] typeIds;
+
+    StackTrace(int[] addressOrLines, String[] fileIds, String[] frameIds, int[] typeIds) {
+        this.addressOrLines = addressOrLines;
+        this.fileIds = fileIds;
+        this.frameIds = frameIds;
+        this.typeIds = typeIds;
+    }
+
+    private static final int BASE64_FRAME_ID_LENGTH = 32;
+
+    private static final String SAFE_BASE64_ENCODER = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz01234456789-_";
+
+    // tag::noformat
+    private static final int[] SAFE_BASE64_DECODER = new int[] {
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0,
+        0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
+        25, 0, 0, 0, 0, 63, 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44,
+        45, 46, 47, 48, 49, 50, 51, 0, 0, 0, 0, 0
+    };
+    // end::noformat
+
+    /**
+     *
+     * runLengthDecodeBase64Url decodes a run-length encoding for the base64-encoded input string.
+     * E.g. the string 'BQADAg' is converted into an int array like [0, 0, 0, 0, 0, 2, 2, 2].
+     * The motivating intent for this method is to unpack a base64-encoded run-length encoding
+     * without using intermediate storage.
+     *
+     * This method relies on these assumptions and details:
+     *  - array encoded using run-length and base64 always returns string of length 0, 3, or 6 (mod 8)
+     *  - since original array is composed of int, we ignore Unicode codepoints
+     *
+     * @param input A base64-encoded string.
+     * @param size Decoded length of the input.
+     * @param capacity Capacity of the underlying array (>= size).
+     *
+     * @return Corresponding numbers that are encoded in the input.
+     */
+    // package-private for testing
+    static int[] runLengthDecodeBase64Url(String input, int size, int capacity) {
+        int[] output = new int[capacity];
+        int multipleOf8 = size / 8;
+        int remainder = size % 8;
+
+        int n;
+        int count;
+        int value;
+        int i;
+        int j = 0;
+
+        for (i = 0; i < multipleOf8 * 8; i += 8) {
+            n = (charCodeAt(input, i) << 26) | (charCodeAt(input, i + 1) << 20) | (charCodeAt(input, i + 2) << 14) | (charCodeAt(
+                input,
+                i + 3
+            ) << 8) | (charCodeAt(input, i + 4) << 2) | (charCodeAt(input, i + 5) >> 4);
+
+            count = (n >> 24) & 0xff;
+            value = (n >> 16) & 0xff;
+
+            Arrays.fill(output, j, j + count, value);
+            j += count;
+
+            count = (n >> 8) & 0xff;
+            value = n & 0xff;
+
+            Arrays.fill(output, j, j + count, value);
+            j += count;
+
+            n = ((charCodeAt(input, i + 5) & 0xf) << 12) | (charCodeAt(input, i + 6) << 6) | charCodeAt(input, i + 7);
+
+            count = (n >> 8) & 0xff;
+            value = n & 0xff;
+
+            Arrays.fill(output, j, j + count, value);
+            j += count;
+        }
+
+        if (remainder == 6) {
+            n = (charCodeAt(input, i) << 26) | (charCodeAt(input, i + 1) << 20) | (charCodeAt(input, i + 2) << 14) | (charCodeAt(
+                input,
+                i + 3
+            ) << 8) | (charCodeAt(input, i + 4) << 2) | (charCodeAt(input, i + 5) >> 4);
+
+            count = (n >> 24) & 0xff;
+            value = (n >> 16) & 0xff;
+
+            Arrays.fill(output, j, j + count, value);
+            j += count;
+
+            count = (n >> 8) & 0xff;
+            value = n & 0xff;
+
+            Arrays.fill(output, j, j + count, value);
+            j += count;
+        } else if (remainder == 3) {
+            n = (charCodeAt(input, i) << 12) | (charCodeAt(input, i + 1) << 6) | charCodeAt(input, i + 2);
+            n >>= 2;
+
+            count = (n >> 8) & 0xff;
+            value = n & 0xff;
+
+            Arrays.fill(output, j, j + count, value);
+            j += count;
+        }
+        return output;
+    }
+
+    // package-private for testing
+    static int getAddressFromStackFrameID(String frameID) {
+        int address = charCodeAt(frameID, 21) & 0xf;
+        address <<= 6;
+        address += charCodeAt(frameID, 22);
+        address <<= 6;
+        address += charCodeAt(frameID, 23);
+        address <<= 6;
+        address += charCodeAt(frameID, 24);
+        address <<= 6;
+        address += charCodeAt(frameID, 25);
+        address <<= 6;
+        address += charCodeAt(frameID, 26);
+        address <<= 6;
+        address += charCodeAt(frameID, 27);
+        address <<= 6;
+        address += charCodeAt(frameID, 28);
+        address <<= 6;
+        address += charCodeAt(frameID, 29);
+        address <<= 6;
+        address += charCodeAt(frameID, 30);
+        address <<= 6;
+        address += charCodeAt(frameID, 31);
+        return address;
+    }
+
+    private static int charCodeAt(String input, int i) {
+        return SAFE_BASE64_DECODER[input.charAt(i) & 0x7f];
+    }
+
+    // package-private for testing
+    static String getFileIDFromStackFrameID(String frameID) {
+        return frameID.substring(0, 21) + SAFE_BASE64_ENCODER.charAt(frameID.charAt(21) & 0x30);
+    }
+
+    public static StackTrace fromSource(Map<String, Object> source) {
+        String inputFrameIDs = ObjectPath.eval("Stacktrace.frame.ids", source);
+        String inputFrameTypes = ObjectPath.eval("Stacktrace.frame.types", source);
+        int countsFrameIDs = inputFrameIDs.length() / BASE64_FRAME_ID_LENGTH;
+
+        String[] fileIDs = new String[countsFrameIDs];
+        String[] frameIDs = new String[countsFrameIDs];
+        int[] addressOrLines = new int[countsFrameIDs];
+
+        // Step 1: Convert the base64-encoded frameID list into two separate
+        // lists (frame IDs and file IDs), both of which are also base64-encoded.
+        //
+        // To get the frame ID, we grab the next 32 bytes.
+        //
+        // To get the file ID, we grab the first 22 bytes of the frame ID.
+        // However, since the file ID is base64-encoded using 21.33 bytes
+        // (16 * 4 / 3), then the 22 bytes have an extra 4 bits from the
+        // address (see diagram in definition of EncodedStackTrace).
+        for (int i = 0, pos = 0; i < countsFrameIDs; i++, pos += BASE64_FRAME_ID_LENGTH) {
+            String frameID = inputFrameIDs.substring(pos, pos + BASE64_FRAME_ID_LENGTH);
+            frameIDs[i] = frameID;
+            fileIDs[i] = getFileIDFromStackFrameID(frameID);
+            addressOrLines[i] = getAddressFromStackFrameID(frameID);
+        }
+
+        // Step 2: Convert the run-length byte encoding into a list of uint8s.
+        int[] typeIDs = runLengthDecodeBase64Url(inputFrameTypes, inputFrameTypes.length(), countsFrameIDs);
+
+        return new StackTrace(addressOrLines, fileIDs, frameIDs, typeIDs);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.array("address_or_lines", this.addressOrLines);
+        builder.array("file_ids", this.fileIds);
+        builder.array("frame_ids", this.frameIds);
+        builder.array("type_ids", this.typeIds);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        StackTrace that = (StackTrace) o;
+        return Arrays.equals(addressOrLines, that.addressOrLines)
+            && Arrays.equals(fileIds, that.fileIds)
+            && Arrays.equals(frameIds, that.frameIds)
+            && Arrays.equals(typeIds, that.typeIds);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = Arrays.hashCode(addressOrLines);
+        result = 31 * result + Arrays.hashCode(fileIds);
+        result = 31 * result + Arrays.hashCode(frameIds);
+        result = 31 * result + Arrays.hashCode(typeIds);
+        return result;
+    }
+}
diff --git a/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/TransportGetProfilingAction.java b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/TransportGetProfilingAction.java
new file mode 100644
index 000000000000..8f8d143bbc7e
--- /dev/null
+++ b/x-pack/plugin/profiler/src/main/java/org/elasticsearch/xpack/profiler/TransportGetProfilingAction.java
@@ -0,0 +1,465 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.profiler;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.get.MultiGetItemResponse;
+import org.elasticsearch.action.get.MultiGetResponse;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.client.internal.ParentTaskAssigningClient;
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.Sum;
+import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xcontent.ObjectPath;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class TransportGetProfilingAction extends HandledTransportAction<GetProfilingRequest, GetProfilingResponse> {
+    private static final Logger log = LogManager.getLogger(TransportGetProfilingAction.class);
+
+    public static final Setting<Integer> PROFILING_MAX_STACKTRACE_QUERY_SLICES = Setting.intSetting(
+        "xpack.profiling.query.stacktrace.max_slices",
+        16,
+        1,
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting<Integer> PROFILING_MAX_DETAIL_QUERY_SLICES = Setting.intSetting(
+        "xpack.profiling.query.details.max_slices",
+        1,
+        1,
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting<Boolean> PROFILING_QUERY_REALTIME = Setting.boolSetting(
+        "xpack.profiling.query.realtime",
+        true,
+        Setting.Property.NodeScope
+    );
+
+    private final NodeClient nodeClient;
+    private final TransportService transportService;
+    private final int desiredSlices;
+    private final int desiredDetailSlices;
+    private final boolean realtime;
+
+    @Inject
+    public TransportGetProfilingAction(
+        Settings settings,
+        TransportService transportService,
+        ActionFilters actionFilters,
+        NodeClient nodeClient
+    ) {
+        super(GetProfilingAction.NAME, transportService, actionFilters, GetProfilingRequest::new);
+        this.nodeClient = nodeClient;
+        this.transportService = transportService;
+        this.desiredSlices = PROFILING_MAX_STACKTRACE_QUERY_SLICES.get(settings);
+        this.desiredDetailSlices = PROFILING_MAX_DETAIL_QUERY_SLICES.get(settings);
+        this.realtime = PROFILING_QUERY_REALTIME.get(settings);
+    }
+
+    @Override
+    protected void doExecute(Task submitTask, GetProfilingRequest request, ActionListener<GetProfilingResponse> submitListener) {
+        long start = System.nanoTime();
+        Client client = new ParentTaskAssigningClient(this.nodeClient, transportService.getLocalNode(), submitTask);
+        EventsIndex mediumDownsampled = EventsIndex.MEDIUM_DOWNSAMPLED;
+        client.prepareSearch(mediumDownsampled.getName())
+            .setSize(0)
+            .setQuery(request.getQuery())
+            .setTrackTotalHits(true)
+            .execute(new ActionListener<>() {
+                @Override
+                public void onResponse(SearchResponse searchResponse) {
+                    long sampleCount = searchResponse.getHits().getTotalHits().value;
+                    EventsIndex resampledIndex = mediumDownsampled.getResampledIndex(request.getSampleSize(), sampleCount);
+                    log.debug("getResampledIndex took [" + (System.nanoTime() - start) / 1_000_000.0d + " ms].");
+                    searchEventGroupByStackTrace(client, request, resampledIndex, submitListener);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // Apart from profiling-events-all, indices are created lazily. In a relatively empty cluster it can happen
+                    // that there are so few data that we need to resort to the full index. As this is an edge case we'd rather
+                    // fail instead of prematurely checking for existence in all cases.
+                    if (e instanceof IndexNotFoundException) {
+                        String missingIndex = ((IndexNotFoundException) e).getIndex().getName();
+                        EventsIndex fullIndex = EventsIndex.FULL_INDEX;
+                        log.debug("Index [{}] does not exist. Using [{}] instead.", missingIndex, fullIndex.getName());
+                        searchEventGroupByStackTrace(client, request, fullIndex, submitListener);
+                    } else {
+                        submitListener.onFailure(e);
+                    }
+                }
+            });
+    }
+
+    private void searchEventGroupByStackTrace(
+        Client client,
+        GetProfilingRequest request,
+        EventsIndex eventsIndex,
+        ActionListener<GetProfilingResponse> submitListener
+    ) {
+        long start = System.nanoTime();
+        GetProfilingResponseBuilder responseBuilder = new GetProfilingResponseBuilder();
+        client.prepareSearch(eventsIndex.getName())
+            .setTrackTotalHits(false)
+            .setQuery(request.getQuery())
+            .addAggregation(
+                new TermsAggregationBuilder("group_by")
+                    // 'size' should be max 100k, but might be slightly more. Better be on the safe side.
+                    .size(150_000)
+                    .field("Stacktrace.id")
+                    // 'execution_hint: map' skips the slow building of ordinals that we don't need.
+                    // Especially with high cardinality fields, this makes aggregations really slow.
+                    .executionHint("map")
+                    .subAggregation(new SumAggregationBuilder("count").field("Stacktrace.count"))
+            )
+            .addAggregation(new SumAggregationBuilder("total_count").field("Stacktrace.count"))
+            .execute(new ActionListener<>() {
+                @Override
+                public void onResponse(SearchResponse searchResponse) {
+                    Sum totalCountAgg = searchResponse.getAggregations().get("total_count");
+                    long totalCount = Math.round(totalCountAgg.value());
+                    Resampler resampler = new Resampler(request, eventsIndex.getSampleRate(), totalCount);
+                    StringTerms stacktraces = searchResponse.getAggregations().get("group_by");
+                    // sort items lexicographically to access Lucene's term dictionary more efficiently when issuing an mget request.
+                    // The term dictionary is lexicographically sorted and using the same order reduces the number of page faults
+                    // needed to load it.
+                    Map<String, Integer> stackTraceEvents = new TreeMap<>();
+                    for (StringTerms.Bucket bucket : stacktraces.getBuckets()) {
+                        Sum count = bucket.getAggregations().get("count");
+                        int finalCount = resampler.adjustSampleCount((int) count.value());
+                        if (finalCount > 0) {
+                            stackTraceEvents.put(bucket.getKeyAsString(), finalCount);
+                        }
+                    }
+                    log.debug("searchEventGroupByStackTrace took [" + (System.nanoTime() - start) / 1_000_000.0d + " ms].");
+                    if (stackTraceEvents.isEmpty() == false) {
+                        responseBuilder.setStackTraceEvents(stackTraceEvents);
+                        retrieveStackTraces(client, responseBuilder, submitListener);
+                    } else {
+                        submitListener.onResponse(responseBuilder.build());
+                    }
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    submitListener.onFailure(e);
+                }
+            });
+    }
+
+    private void retrieveStackTraces(
+        Client client,
+        GetProfilingResponseBuilder responseBuilder,
+        ActionListener<GetProfilingResponse> submitListener
+    ) {
+        List<String> eventIds = new ArrayList<>(responseBuilder.getStackTraceEvents().keySet());
+        List<List<String>> slicedEventIds = sliced(eventIds, desiredSlices);
+        StackTraceHandler handler = new StackTraceHandler(client, responseBuilder, submitListener, slicedEventIds.size());
+        for (List<String> slice : slicedEventIds) {
+            client.prepareMultiGet().setRealtime(realtime).addIds("profiling-stacktraces", slice).execute(new ActionListener<>() {
+                @Override
+                public void onResponse(MultiGetResponse multiGetItemResponses) {
+                    handler.onResponse(multiGetItemResponses);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    submitListener.onFailure(e);
+                }
+            });
+        }
+    }
+
+    // package private for testing
+    static <T> List<List<T>> sliced(List<T> c, int slices) {
+        if (c.size() <= slices) {
+            return List.of(c);
+        }
+        List<List<T>> slicedList = new ArrayList<>();
+        int batchSize = c.size() / slices;
+        if (c.size() % slices != 0) {
+            batchSize += 1;
+        }
+        for (int slice = 0; slice < slices; slice++) {
+            List<T> ids = c.subList(slice * batchSize, Math.min((slice + 1) * batchSize, c.size()));
+            slicedList.add(ids);
+        }
+        return Collections.unmodifiableList(slicedList);
+    }
+
+    private class StackTraceHandler {
+        private final AtomicInteger remainingSlices;
+        private final Client client;
+        private final GetProfilingResponseBuilder responseBuilder;
+        private final ActionListener<GetProfilingResponse> submitListener;
+        private final Map<String, StackTrace> stackTracePerId = new ConcurrentHashMap<>();
+        // sort items lexicographically to access Lucene's term dictionary more efficiently when issuing an mget request.
+        // The term dictionary is lexicographically sorted and using the same order reduces the number of page faults
+        // needed to load it.
+        private final Set<String> stackFrameIds = new ConcurrentSkipListSet<>();
+        private final Set<String> executableIds = new ConcurrentSkipListSet<>();
+        private final AtomicInteger totalFrames = new AtomicInteger();
+        private final long start = System.nanoTime();
+
+        private StackTraceHandler(
+            Client client,
+            GetProfilingResponseBuilder responseBuilder,
+            ActionListener<GetProfilingResponse> submitListener,
+            int slices
+        ) {
+            this.remainingSlices = new AtomicInteger(slices);
+            this.client = client;
+            this.responseBuilder = responseBuilder;
+            this.submitListener = submitListener;
+        }
+
+        public void onResponse(MultiGetResponse multiGetItemResponses) {
+            for (MultiGetItemResponse trace : multiGetItemResponses) {
+                if (trace.isFailed() == false && trace.getResponse().isExists()) {
+                    String id = trace.getId();
+                    StackTrace stacktrace = StackTrace.fromSource(trace.getResponse().getSource());
+                    stackTracePerId.put(id, stacktrace);
+                    totalFrames.addAndGet(stacktrace.frameIds.length);
+                    stackFrameIds.addAll(Arrays.asList(stacktrace.frameIds));
+                    executableIds.addAll(Arrays.asList(stacktrace.fileIds));
+                }
+            }
+            if (this.remainingSlices.decrementAndGet() == 0) {
+                responseBuilder.setStackTraces(stackTracePerId);
+                responseBuilder.setTotalFrames(totalFrames.get());
+                log.debug("retrieveStackTraces took [" + (System.nanoTime() - start) / 1_000_000.0d + " ms].");
+                retrieveStackTraceDetails(client, responseBuilder, stackFrameIds, executableIds, submitListener);
+            }
+        }
+    }
+
+    private void retrieveStackTraceDetails(
+        Client client,
+        GetProfilingResponseBuilder responseBuilder,
+        Set<String> stackFrameIds,
+        Set<String> executableIds,
+        ActionListener<GetProfilingResponse> submitListener
+    ) {
+        List<List<String>> slicedStackFrameIds = sliced(new ArrayList<>(stackFrameIds), desiredDetailSlices);
+        List<List<String>> slicedExecutableIds = sliced(new ArrayList<>(executableIds), desiredDetailSlices);
+        DetailsHandler handler = new DetailsHandler(
+            responseBuilder,
+            submitListener,
+            slicedExecutableIds.size(),
+            slicedStackFrameIds.size()
+        );
+
+        if (stackFrameIds.isEmpty()) {
+            handler.onStackFramesResponse(new MultiGetResponse(new MultiGetItemResponse[0]));
+        } else {
+            for (List<String> slice : slicedStackFrameIds) {
+                client.prepareMultiGet().addIds("profiling-stackframes", slice).setRealtime(realtime).execute(new ActionListener<>() {
+                    @Override
+                    public void onResponse(MultiGetResponse multiGetItemResponses) {
+                        handler.onStackFramesResponse(multiGetItemResponses);
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        submitListener.onFailure(e);
+                    }
+                });
+            }
+        }
+        // no data dependency - we can do this concurrently
+        if (executableIds.isEmpty()) {
+            handler.onExecutableDetailsResponse(new MultiGetResponse(new MultiGetItemResponse[0]));
+        } else {
+            for (List<String> slice : slicedExecutableIds) {
+                client.prepareMultiGet().addIds("profiling-executables", slice).setRealtime(realtime).execute(new ActionListener<>() {
+                    @Override
+                    public void onResponse(MultiGetResponse multiGetItemResponses) {
+                        handler.onExecutableDetailsResponse(multiGetItemResponses);
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        submitListener.onFailure(e);
+                    }
+                });
+            }
+        }
+    }
+
+    private static class Resampler {
+        private final boolean requiresResampling;
+
+        private final Random r;
+
+        private final double sampleRate;
+
+        private final double p;
+
+        Resampler(GetProfilingRequest request, double sampleRate, long totalCount) {
+            // Manually reduce sample count if totalCount exceeds sampleSize by 10%.
+            if (totalCount > request.getSampleSize() * 1.1) {
+                this.requiresResampling = true;
+                // Make the RNG predictable to get reproducible results.
+                this.r = new Random(request.hashCode());
+                this.sampleRate = sampleRate;
+                this.p = (double) request.getSampleSize() / totalCount;
+            } else {
+                this.requiresResampling = false;
+                this.r = null;
+                this.sampleRate = sampleRate;
+                this.p = 1.0d;
+            }
+        }
+
+        public int adjustSampleCount(int originalCount) {
+            if (requiresResampling) {
+                int newCount = 0;
+                for (int i = 0; i < originalCount; i++) {
+                    if (r.nextDouble() < p) {
+                        newCount++;
+                    }
+                }
+                if (newCount > 0) {
+                    // Adjust the sample counts from down-sampled to fully sampled.
+                    // Be aware that downsampling drops entries from stackTraceEvents, so that
+                    // the sum of the upscaled count values is less that totalCount.
+                    return (int) Math.floor(newCount / (sampleRate * p));
+                } else {
+                    return 0;
+                }
+            } else {
+                return originalCount;
+            }
+        }
+    }
+
+    /**
+     * Collects stack trace details which are retrieved concurrently and sends a response only when all details are known.
+     */
+    private static class DetailsHandler {
+        private final GetProfilingResponseBuilder builder;
+        private final ActionListener<GetProfilingResponse> submitListener;
+        private final Map<String, String> executables;
+        private final Map<String, StackFrame> stackFrames;
+        private final AtomicInteger expectedExecutableSlices;
+        private final AtomicInteger expectedStackFrameSlices;
+        private final long start = System.nanoTime();
+
+        private DetailsHandler(
+            GetProfilingResponseBuilder builder,
+            ActionListener<GetProfilingResponse> submitListener,
+            int expectedExecutableSlices,
+            int expectedStackFrameSlices
+        ) {
+            this.builder = builder;
+            this.submitListener = submitListener;
+            this.executables = new ConcurrentHashMap<>();
+            this.stackFrames = new ConcurrentHashMap<>();
+            this.expectedExecutableSlices = new AtomicInteger(expectedExecutableSlices);
+            this.expectedStackFrameSlices = new AtomicInteger(expectedStackFrameSlices);
+        }
+
+        public void onStackFramesResponse(MultiGetResponse multiGetItemResponses) {
+            for (MultiGetItemResponse frame : multiGetItemResponses) {
+                if (frame.isFailed() == false && frame.getResponse().isExists()) {
+                    stackFrames.put(frame.getId(), StackFrame.fromSource(frame.getResponse().getSource()));
+                }
+            }
+            expectedStackFrameSlices.decrementAndGet();
+            mayFinish();
+        }
+
+        public void onExecutableDetailsResponse(MultiGetResponse multiGetItemResponses) {
+            for (MultiGetItemResponse executable : multiGetItemResponses) {
+                if (executable.isFailed() == false && executable.getResponse().isExists()) {
+                    executables.put(executable.getId(), ObjectPath.eval("Executable.file.name", executable.getResponse().getSource()));
+                }
+            }
+            expectedExecutableSlices.decrementAndGet();
+            mayFinish();
+        }
+
+        public void mayFinish() {
+            if (expectedStackFrameSlices.get() == 0 && expectedExecutableSlices.get() == 0) {
+                builder.setExecutables(executables);
+                builder.setStackFrames(stackFrames);
+                log.debug("retrieveStackTraceDetails took [" + (System.nanoTime() - start) / 1_000_000.0d + " ms].");
+                submitListener.onResponse(builder.build());
+            }
+        }
+    }
+
+    private static class GetProfilingResponseBuilder {
+        private Map<String, StackTrace> stackTraces;
+        private int totalFrames;
+        private Map<String, StackFrame> stackFrames;
+        private Map<String, String> executables;
+        private Map<String, Integer> stackTraceEvents;
+        private Exception error;
+
+        public void setStackTraces(Map<String, StackTrace> stackTraces) {
+            this.stackTraces = stackTraces;
+        }
+
+        public void setTotalFrames(int totalFrames) {
+            this.totalFrames = totalFrames;
+        }
+
+        public void setStackFrames(Map<String, StackFrame> stackFrames) {
+            this.stackFrames = stackFrames;
+        }
+
+        public void setExecutables(Map<String, String> executables) {
+            this.executables = executables;
+        }
+
+        public void setStackTraceEvents(Map<String, Integer> stackTraceEvents) {
+            this.stackTraceEvents = stackTraceEvents;
+        }
+
+        public Map<String, Integer> getStackTraceEvents() {
+            return stackTraceEvents;
+        }
+
+        public void setError(Exception error) {
+            this.error = error;
+        }
+
+        public GetProfilingResponse build() {
+            if (error != null) {
+                return new GetProfilingResponse(error);
+            } else {
+                return new GetProfilingResponse(stackTraces, stackFrames, executables, stackTraceEvents, totalFrames);
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/EventsIndexTests.java b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/EventsIndexTests.java
new file mode 100644
index 000000000000..b653fecfe37f
--- /dev/null
+++ b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/EventsIndexTests.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiler;
+
+import org.elasticsearch.test.ESTestCase;
+
+public class EventsIndexTests extends ESTestCase {
+    public void testFullIndex() {
+        EventsIndex idx = EventsIndex.FULL_INDEX;
+        assertEquals("profiling-events-all", idx.getName());
+        assertEquals(1.0d, idx.getSampleRate(), 1e-3);
+    }
+
+    public void testResampledIndexSameSize() {
+        EventsIndex resampledIndex = EventsIndex.MEDIUM_DOWNSAMPLED.getResampledIndex(100, 100);
+        assertEquals("profiling-events-5pow06", resampledIndex.getName());
+        assertEquals(Math.pow(1.0d / 5.0d, 6.0d), resampledIndex.getSampleRate(), 1e-9);
+    }
+
+    public void testResampledIndexDifferentSizes() {
+        assertResampledIndex("profiling-events-5pow01", Math.pow(5.0d, 5));
+        assertResampledIndex("profiling-events-5pow02", Math.pow(5.0d, 4));
+        assertResampledIndex("profiling-events-5pow03", Math.pow(5.0d, 3));
+
+        assertResampledIndex("profiling-events-5pow04", Math.pow(5.0d, 2));
+        assertResampledIndex("profiling-events-5pow05", Math.pow(5.0d, 1));
+
+        assertResampledIndex("profiling-events-5pow06", Math.pow(5.0d, 0));
+        assertResampledIndex("profiling-events-5pow07", Math.pow(5.0d, -1));
+        assertResampledIndex("profiling-events-5pow08", Math.pow(5.0d, -2));
+
+        assertResampledIndex("profiling-events-5pow09", Math.pow(5.0d, -3));
+        assertResampledIndex("profiling-events-5pow10", Math.pow(5.0d, -4));
+        assertResampledIndex("profiling-events-5pow11", Math.pow(5.0d, -5));
+    }
+
+    private void assertResampledIndex(String expectedName, double ratio) {
+        long currentSampleSize = 10_000_000L;
+        long targetSampleSize = (long) (currentSampleSize * ratio);
+        EventsIndex e = EventsIndex.MEDIUM_DOWNSAMPLED;
+        assertEquals(expectedName, e.getResampledIndex(targetSampleSize, currentSampleSize).getName());
+    }
+}
diff --git a/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/GetProfilingRequestTests.java b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/GetProfilingRequestTests.java
new file mode 100644
index 000000000000..d4394b2b1926
--- /dev/null
+++ b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/GetProfilingRequestTests.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiler;
+
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.RangeQueryBuilder;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+
+import static java.util.Collections.emptyList;
+
+public class GetProfilingRequestTests extends ESTestCase {
+    public void testSerialization() throws IOException {
+        Integer sampleSize = randomBoolean() ? randomIntBetween(0, Integer.MAX_VALUE) : null;
+        QueryBuilder query = randomBoolean() ? new BoolQueryBuilder() : null;
+
+        GetProfilingRequest request = new GetProfilingRequest(sampleSize, query);
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            request.writeTo(out);
+            try (NamedWriteableAwareStreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), writableRegistry())) {
+                GetProfilingRequest deserialized = new GetProfilingRequest(in);
+                assertEquals(sampleSize, deserialized.getSampleSize());
+                assertEquals(query, deserialized.getQuery());
+            }
+        }
+    }
+
+    public void testParseValidXContent() throws IOException {
+        try (XContentParser content = createParser(XContentFactory.jsonBuilder()
+        //tag::noformat
+            .startObject()
+                .field("sample_size", 500)
+                .startObject("query")
+                    .startObject("range")
+                        .startObject("@timestamp")
+                            .field("gte", "2022-10-05")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject()
+        //end::noformat
+        )) {
+
+            GetProfilingRequest profilingRequest = new GetProfilingRequest();
+            profilingRequest.parseXContent(content);
+
+            assertEquals(Integer.valueOf(500), profilingRequest.getSampleSize());
+            // a basic check suffices here
+            assertEquals("@timestamp", ((RangeQueryBuilder) profilingRequest.getQuery()).fieldName());
+        }
+    }
+
+    public void testParseXContentUnrecognizedField() throws IOException {
+        try (XContentParser content = createParser(XContentFactory.jsonBuilder()
+        //tag::noformat
+            .startObject()
+                // should be sample_size
+                .field("sample-size", 500)
+                .startObject("query")
+                    .startObject("range")
+                        .startObject("@timestamp")
+                            .field("gte", "2022-10-05")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject()
+        //end::noformat
+        )) {
+
+            GetProfilingRequest profilingRequest = new GetProfilingRequest();
+            ParsingException ex = expectThrows(ParsingException.class, () -> profilingRequest.parseXContent(content));
+            assertEquals("Unknown key for a VALUE_NUMBER in [sample-size].", ex.getMessage());
+        }
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        // to register the query parser
+        return new NamedXContentRegistry(new SearchModule(Settings.EMPTY, emptyList()).getNamedXContents());
+    }
+
+    @Override
+    protected NamedWriteableRegistry writableRegistry() {
+        return new NamedWriteableRegistry(new SearchModule(Settings.EMPTY, emptyList()).getNamedWriteables());
+    }
+}
diff --git a/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/GetProfilingResponseTests.java b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/GetProfilingResponseTests.java
new file mode 100644
index 000000000000..2028f06d2be7
--- /dev/null
+++ b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/GetProfilingResponseTests.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiler;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import java.util.Map;
+import java.util.function.Supplier;
+
+public class GetProfilingResponseTests extends AbstractWireSerializingTestCase<GetProfilingResponse> {
+    private <T> T randomNullable(Supplier<T> v) {
+        return randomBoolean() ? v.get() : null;
+    }
+
+    private <T> T randomNullable(T v) {
+        return randomBoolean() ? v : null;
+    }
+
+    @Override
+    protected GetProfilingResponse createTestInstance() {
+        int totalFrames = randomIntBetween(1, 100);
+
+        Map<String, StackTrace> stackTraces = randomNullable(
+            Map.of(
+                "QjoLteG7HX3VUUXr-J4kHQ",
+                new StackTrace(
+                    new int[] { 1083999 },
+                    new String[] { "QCCDqjSg3bMK1C4YRK6Tiw" },
+                    new String[] { "QCCDqjSg3bMK1C4YRK6TiwAAAAAAEIpf" },
+                    new int[] { 2 }
+                )
+            )
+        );
+        int maxInlined = randomInt(5);
+        Map<String, StackFrame> stackFrames = randomNullable(
+            Map.of(
+                "QCCDqjSg3bMK1C4YRK6TiwAAAAAAEIpf",
+                new StackFrame(
+                    randomList(0, maxInlined, () -> randomAlphaOfLength(20)),
+                    randomList(0, maxInlined, () -> randomAlphaOfLength(20)),
+                    randomList(0, maxInlined, () -> randomIntBetween(1, Integer.MAX_VALUE)),
+                    randomList(0, maxInlined, () -> randomIntBetween(1, 30_000)),
+                    randomList(0, maxInlined, () -> randomIntBetween(1, 10))
+                )
+            )
+        );
+        Map<String, String> executables = randomNullable(Map.of("QCCDqjSg3bMK1C4YRK6Tiw", "libc.so.6"));
+        Map<String, Integer> stackTraceEvents = randomNullable(Map.of(randomAlphaOfLength(12), randomIntBetween(1, 200)));
+
+        return new GetProfilingResponse(stackTraces, stackFrames, executables, stackTraceEvents, totalFrames);
+    }
+
+    @Override
+    protected GetProfilingResponse mutateInstance(GetProfilingResponse instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
+    @Override
+    protected Writeable.Reader<GetProfilingResponse> instanceReader() {
+        return GetProfilingResponse::new;
+    }
+}
diff --git a/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/RestGetProfilingActionTests.java b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/RestGetProfilingActionTests.java
new file mode 100644
index 000000000000..d9a3532b451f
--- /dev/null
+++ b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/RestGetProfilingActionTests.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiler;
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.test.rest.RestActionTestCase;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xcontent.XContentType;
+import org.junit.Before;
+
+import java.util.Collections;
+
+import static java.util.Collections.emptyList;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class RestGetProfilingActionTests extends RestActionTestCase {
+    @Before
+    public void setUpAction() {
+        controller().registerHandler(new RestGetProfilingAction());
+    }
+
+    public void testPrepareEmptyRequest() {
+        SetOnce<Boolean> executeCalled = new SetOnce<>();
+        verifyingClient.setExecuteLocallyVerifier((actionType, request) -> {
+            assertThat(request, instanceOf(GetProfilingRequest.class));
+            GetProfilingRequest profilingRequest = (GetProfilingRequest) request;
+            assertThat(profilingRequest.getSampleSize(), nullValue());
+            assertThat(profilingRequest.getQuery(), nullValue());
+            executeCalled.set(true);
+            return new GetProfilingResponse(
+                Collections.emptyMap(),
+                Collections.emptyMap(),
+                Collections.emptyMap(),
+                Collections.emptyMap(),
+                0
+            );
+        });
+        RestRequest profilingRequest = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.POST)
+            .withPath("/_profiling/stacktraces")
+            .withContent(new BytesArray("{}"), XContentType.JSON)
+            .build();
+        dispatchRequest(profilingRequest);
+        assertThat(executeCalled.get(), equalTo(true));
+    }
+
+    public void testPrepareParameterizedRequest() {
+        SetOnce<Boolean> executeCalled = new SetOnce<>();
+        verifyingClient.setExecuteLocallyVerifier((actionType, request) -> {
+            assertThat(request, instanceOf(GetProfilingRequest.class));
+            GetProfilingRequest profilingRequest = (GetProfilingRequest) request;
+            assertThat(profilingRequest.getSampleSize(), is(10000));
+            assertThat(profilingRequest.getQuery(), notNullValue(QueryBuilder.class));
+            executeCalled.set(true);
+            return new GetProfilingResponse(
+                Collections.emptyMap(),
+                Collections.emptyMap(),
+                Collections.emptyMap(),
+                Collections.emptyMap(),
+                0
+            );
+        });
+        RestRequest profilingRequest = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.POST)
+            .withPath("/_profiling/stacktraces")
+            .withContent(new BytesArray("""
+                            {
+                              "sample_size": 10000,
+                              "query": {
+                                "bool": {
+                                  "filter": [
+                                    {
+                                      "range": {
+                                        "@timestamp": {
+                                          "gte": "2022-10-05",
+                                          "lt": "2022-12-05"
+                                        }
+                                      }
+                                    }
+                                  ]
+                                }
+                              }
+                            }
+                """), XContentType.JSON)
+            .build();
+        dispatchRequest(profilingRequest);
+        assertThat(executeCalled.get(), equalTo(true));
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        // to register the query parser
+        return new NamedXContentRegistry(new SearchModule(Settings.EMPTY, emptyList()).getNamedXContents());
+    }
+}
diff --git a/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/StackFrameTests.java b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/StackFrameTests.java
new file mode 100644
index 000000000000..609af94f012e
--- /dev/null
+++ b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/StackFrameTests.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiler;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class StackFrameTests extends ESTestCase {
+    public void testCreateFromSyntheticSource() {
+        // tag::noformat
+        StackFrame frame = StackFrame.fromSource(
+            Map.of("Stackframe", Map.of(
+                "file", Map.of("name", "Main.java"),
+                "function", Map.of(
+                        "name", "helloWorld",
+                        "offset", 31733
+                    ),
+                "line", Map.of("number", 22),
+                "source", Map.of("type", 3))
+                )
+        );
+        // end::noformat
+        assertEquals(List.of(3), frame.sourceType);
+        assertEquals(List.of("Main.java"), frame.fileName);
+        assertEquals(List.of("helloWorld"), frame.functionName);
+        assertEquals(List.of(31733), frame.functionOffset);
+        assertEquals(List.of(22), frame.lineNumber);
+    }
+
+    public void testCreateFromRegularSource() {
+        // tag::noformat
+        StackFrame frame = StackFrame.fromSource(
+            Map.of(
+                "Stackframe.file.name", "Main.java",
+                "Stackframe.function.name", "helloWorld",
+                "Stackframe.line.number", 22
+            )
+        );
+        // end::noformat
+        assertEquals(Collections.emptyList(), frame.sourceType);
+        assertEquals(List.of("Main.java"), frame.fileName);
+        assertEquals(List.of("helloWorld"), frame.functionName);
+        assertEquals(Collections.emptyList(), frame.functionOffset);
+        assertEquals(List.of(22), frame.lineNumber);
+    }
+
+    public void testToXContent() throws IOException {
+        XContentType contentType = randomFrom(XContentType.values());
+        XContentBuilder expectedRequest = XContentFactory.contentBuilder(contentType)
+            .startObject()
+            .array("file_name", "Main.java")
+            .array("function_name", "helloWorld")
+            .array("function_offset", 31733)
+            .array("line_number", 22)
+            .array("source_type", 3)
+            .endObject();
+
+        XContentBuilder actualRequest = XContentFactory.contentBuilder(contentType);
+        StackFrame stackTrace = new StackFrame("Main.java", "helloWorld", 31733, 22, 3);
+        stackTrace.toXContent(actualRequest, ToXContent.EMPTY_PARAMS);
+
+        assertToXContentEquivalent(BytesReference.bytes(expectedRequest), BytesReference.bytes(actualRequest), contentType);
+    }
+
+    public void testEquality() {
+        StackFrame frame = new StackFrame("Main.java", "helloWorld", 31733, 22, 3);
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(
+            frame,
+            (o -> new StackFrame(o.fileName, o.functionName, o.functionOffset, o.lineNumber, o.sourceType))
+        );
+
+    }
+}
diff --git a/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/StackTraceTests.java b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/StackTraceTests.java
new file mode 100644
index 000000000000..30a525811478
--- /dev/null
+++ b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/StackTraceTests.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiler;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class StackTraceTests extends ESTestCase {
+    public void testDecodeFrameId() {
+        String frameId = "AAAAAAAAAAUAAAAAAAAB3gAAAAAAD67u";
+        // base64 encoded representation of the tuple (5, 478)
+        assertEquals("AAAAAAAAAAUAAAAAAAAB3g", StackTrace.getFileIDFromStackFrameID(frameId));
+        assertEquals(1027822, StackTrace.getAddressFromStackFrameID(frameId));
+    }
+
+    public void testRunlengthDecodeUniqueValues() {
+        // 0 - 9 (reversed)
+        String encodedFrameTypes = "AQkBCAEHAQYBBQEEAQMBAgEBAQA";
+        int[] actual = StackTrace.runLengthDecodeBase64Url(encodedFrameTypes, encodedFrameTypes.length(), 10);
+        assertArrayEquals(new int[] { 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 }, actual);
+    }
+
+    public void testRunlengthDecodeSingleValue() {
+        // "4", repeated ten times
+        String encodedFrameTypes = "CgQ";
+        int[] actual = StackTrace.runLengthDecodeBase64Url(encodedFrameTypes, encodedFrameTypes.length(), 10);
+        assertArrayEquals(new int[] { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 }, actual);
+    }
+
+    public void testRunlengthDecodeFillsGap() {
+        // "2", repeated three times
+        String encodedFrameTypes = "AwI";
+        int[] actual = StackTrace.runLengthDecodeBase64Url(encodedFrameTypes, encodedFrameTypes.length(), 5);
+        // zeroes should be appended for the last two values which are not present in the encoded representation.
+        assertArrayEquals(new int[] { 2, 2, 2, 0, 0 }, actual);
+    }
+
+    public void testRunlengthDecodeMixedValue() {
+        // 4
+        String encodedFrameTypes = "BQADAg";
+        int[] actual = StackTrace.runLengthDecodeBase64Url(encodedFrameTypes, encodedFrameTypes.length(), 8);
+        assertArrayEquals(new int[] { 0, 0, 0, 0, 0, 2, 2, 2 }, actual);
+    }
+
+    public void testCreateFromSource() {
+        String ids = "AAAAAAAAAAUAAAAAAAAB3gAAAAAAD67u";
+        String types = "AQI";
+        // tag::noformat
+        StackTrace stackTrace = StackTrace.fromSource(
+            Map.of("Stacktrace",
+                Map.of("frame",
+                    Map.of(
+                        "ids", ids,
+                        "types", types)
+                )
+            )
+        );
+        // end::noformat
+        assertArrayEquals(new String[] { "AAAAAAAAAAUAAAAAAAAB3gAAAAAAD67u" }, stackTrace.frameIds);
+        assertArrayEquals(new String[] { "AAAAAAAAAAUAAAAAAAAB3g" }, stackTrace.fileIds);
+        assertArrayEquals(new int[] { 1027822 }, stackTrace.addressOrLines);
+        assertArrayEquals(new int[] { 2 }, stackTrace.typeIds);
+    }
+
+    public void testToXContent() throws IOException {
+        XContentType contentType = randomFrom(XContentType.values());
+        XContentBuilder expectedRequest = XContentFactory.contentBuilder(contentType)
+            .startObject()
+            .array("address_or_lines", new int[] { 1027822 })
+            .array("file_ids", "AAAAAAAAAAUAAAAAAAAB3g")
+            .array("frame_ids", "AAAAAAAAAAUAAAAAAAAB3gAAAAAAD67u")
+            .array("type_ids", new int[] { 2 })
+            .endObject();
+
+        XContentBuilder actualRequest = XContentFactory.contentBuilder(contentType);
+        StackTrace stackTrace = new StackTrace(
+            new int[] { 1027822 },
+            new String[] { "AAAAAAAAAAUAAAAAAAAB3g" },
+            new String[] { "AAAAAAAAAAUAAAAAAAAB3gAAAAAAD67u" },
+            new int[] { 2 }
+        );
+        stackTrace.toXContent(actualRequest, ToXContent.EMPTY_PARAMS);
+
+        assertToXContentEquivalent(BytesReference.bytes(expectedRequest), BytesReference.bytes(actualRequest), contentType);
+    }
+
+    public void testEquality() {
+        StackTrace stackTrace = new StackTrace(
+            new int[] { 1027822 },
+            new String[] { "AAAAAAAAAAUAAAAAAAAB3g" },
+            new String[] { "AAAAAAAAAAUAAAAAAAAB3gAAAAAAD67u" },
+            new int[] { 2 }
+        );
+
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(
+            stackTrace,
+            (o -> new StackTrace(o.addressOrLines.clone(), o.fileIds.clone(), o.frameIds.clone(), o.typeIds.clone()))
+        );
+    }
+}
diff --git a/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/TransportGetProfilingActionTests.java b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/TransportGetProfilingActionTests.java
new file mode 100644
index 000000000000..16ba72b3ae61
--- /dev/null
+++ b/x-pack/plugin/profiler/src/test/java/org/elasticsearch/xpack/profiler/TransportGetProfilingActionTests.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.profiler;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Collections;
+import java.util.List;
+
+public class TransportGetProfilingActionTests extends ESTestCase {
+    public void testSliceEmptyList() {
+        assertEquals(List.of(List.of()), TransportGetProfilingAction.sliced(Collections.emptyList(), 4));
+    }
+
+    public void testSliceListSmallerOrEqualToSliceCount() {
+        int slices = 7;
+        List<String> input = randomList(0, slices, () -> randomAlphaOfLength(3));
+        List<List<String>> sliced = TransportGetProfilingAction.sliced(input, slices);
+        assertEquals(1, sliced.size());
+        assertEquals(input, sliced.get(0));
+    }
+
+    public void testSliceListMultipleOfSliceCount() {
+        int slices = 2;
+        List<String> input = List.of("a", "b", "c", "d");
+        List<List<String>> sliced = TransportGetProfilingAction.sliced(input, slices);
+        assertEquals(slices, sliced.size());
+        assertEquals(List.of("a", "b"), sliced.get(0));
+        assertEquals(List.of("c", "d"), sliced.get(1));
+    }
+
+    public void testSliceListGreaterThanSliceCount() {
+        int slices = 3;
+        List<String> input = List.of("a", "b", "c", "d", "e", "f", "g", "h", "i", "j");
+        List<List<String>> sliced = TransportGetProfilingAction.sliced(input, slices);
+        assertEquals(slices, sliced.size());
+        assertEquals(List.of("a", "b", "c", "d"), sliced.get(0));
+        assertEquals(List.of("e", "f", "g", "h"), sliced.get(1));
+        assertEquals(List.of("i", "j"), sliced.get(2));
+    }
+}
diff --git a/x-pack/plugin/ql/build.gradle b/x-pack/plugin/ql/build.gradle
index 153a8bbafe4b..9c1554bfaf12 100644
--- a/x-pack/plugin/ql/build.gradle
+++ b/x-pack/plugin/ql/build.gradle
@@ -8,14 +8,10 @@ esplugin {
   extendedPlugins = ['x-pack-core']
 }
 
-ext {
-  antlrVersion = "4.9.2"
-}
-
 archivesBaseName = 'x-pack-ql'
 
 dependencies {
-  api "org.antlr:antlr4-runtime:${antlrVersion}"
+  api "org.antlr:antlr4-runtime:${versions.antlr4}"
   api project(path: xpackModule('mapper-version'))
   compileOnly project(path: xpackModule('core'))
   testApi(project(xpackModule('ql:test-fixtures'))) {
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/analyzer/AnalyzerRules.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/analyzer/AnalyzerRules.java
index 10f510436c95..62cf5e133a88 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/analyzer/AnalyzerRules.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/analyzer/AnalyzerRules.java
@@ -7,16 +7,35 @@
 
 package org.elasticsearch.xpack.ql.analyzer;
 
+import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.xpack.ql.expression.Attribute;
 import org.elasticsearch.xpack.ql.expression.Expression;
 import org.elasticsearch.xpack.ql.expression.FieldAttribute;
 import org.elasticsearch.xpack.ql.expression.Literal;
+import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute;
+import org.elasticsearch.xpack.ql.expression.function.Function;
+import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition;
+import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
+import org.elasticsearch.xpack.ql.expression.function.UnresolvedFunction;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogic;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.ql.plan.logical.Filter;
 import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.ql.rule.ParameterizedRule;
 import org.elasticsearch.xpack.ql.rule.Rule;
+import org.elasticsearch.xpack.ql.session.Configuration;
+import org.elasticsearch.xpack.ql.type.DataTypes;
+import org.elasticsearch.xpack.ql.type.InvalidMappedField;
+import org.elasticsearch.xpack.ql.type.UnsupportedEsField;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.BiFunction;
 
 import static java.util.Arrays.asList;
+import static java.util.Collections.singletonList;
 import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
 
 public final class AnalyzerRules {
@@ -66,7 +85,6 @@ public final LogicalPlan apply(LogicalPlan plan) {
             return plan.transformUp(typeToken(), t -> t.analyzed() || skipResolved() && t.resolved() ? t : rule(t));
         }
 
-        @Override
         protected abstract LogicalPlan rule(SubPlan plan);
 
         protected boolean skipResolved() {
@@ -74,6 +92,24 @@ protected boolean skipResolved() {
         }
     }
 
+    public abstract static class ParameterizedAnalyzerRule<SubPlan extends LogicalPlan, P> extends ParameterizedRule<
+        SubPlan,
+        LogicalPlan,
+        P> {
+
+        // transformUp (post-order) - that is first children and then the node
+        // but with a twist; only if the tree is not resolved or analyzed
+        public final LogicalPlan apply(LogicalPlan plan, P context) {
+            return plan.transformUp(typeToken(), t -> t.analyzed() || skipResolved() && t.resolved() ? t : rule(t, context));
+        }
+
+        protected abstract LogicalPlan rule(SubPlan plan, P context);
+
+        protected boolean skipResolved() {
+            return true;
+        }
+    }
+
     public abstract static class BaseAnalyzerRule extends AnalyzerRule<LogicalPlan> {
 
         @Override
@@ -86,4 +122,123 @@ protected LogicalPlan rule(LogicalPlan plan) {
 
         protected abstract LogicalPlan doRule(LogicalPlan plan);
     }
+
+    public static Function resolveFunction(UnresolvedFunction uf, Configuration configuration, FunctionRegistry functionRegistry) {
+        Function f = null;
+        if (uf.analyzed()) {
+            f = uf;
+        } else if (uf.childrenResolved() == false) {
+            f = uf;
+        } else {
+            String functionName = functionRegistry.resolveAlias(uf.name());
+            if (functionRegistry.functionExists(functionName) == false) {
+                f = uf.missing(functionName, functionRegistry.listFunctions());
+            } else {
+                FunctionDefinition def = functionRegistry.resolveFunction(functionName);
+                f = uf.buildResolved(configuration, def);
+            }
+        }
+        return f;
+    }
+
+    public static List<Attribute> maybeResolveAgainstList(
+        UnresolvedAttribute u,
+        Collection<Attribute> attrList,
+        boolean allowCompound,
+        boolean acceptPattern
+    ) {
+        List<Attribute> matches = new ArrayList<>();
+
+        // first take into account the qualified version
+        boolean qualified = u.qualifier() != null;
+
+        var name = u.name();
+        var isPattern = acceptPattern && Regex.isSimpleMatchPattern(name);
+        BiFunction<String, String, Boolean> nameMatcher = isPattern ? Regex::simpleMatch : Objects::equals;
+
+        for (Attribute attribute : attrList) {
+            if (attribute.synthetic() == false) {
+                boolean match = qualified ? Objects.equals(u.qualifiedName(), attribute.qualifiedName()) : // TODO: qualified w/ pattern?
+                // if the field is unqualified
+                // first check the names directly
+                    (nameMatcher.apply(name, attribute.name())
+                        // but also if the qualifier might not be quoted and if there's any ambiguity with nested fields
+                        || nameMatcher.apply(name, attribute.qualifiedName()));
+                if (match) {
+                    matches.add(attribute);
+                }
+            }
+        }
+
+        if (matches.isEmpty()) {
+            return matches;
+        }
+
+        // found exact match or multiple if pattern
+        if (matches.size() == 1 || isPattern) {
+            // only add the location if the match is univocal; b/c otherwise adding the location will overwrite any preexisting one
+            matches.replaceAll(e -> handleSpecialFields(u, e.withLocation(u.source()), allowCompound));
+            return matches;
+        }
+
+        // report ambiguity
+        List<String> refs = matches.stream().sorted((a, b) -> {
+            int lineDiff = a.sourceLocation().getLineNumber() - b.sourceLocation().getLineNumber();
+            int colDiff = a.sourceLocation().getColumnNumber() - b.sourceLocation().getColumnNumber();
+            return lineDiff != 0 ? lineDiff : (colDiff != 0 ? colDiff : a.qualifiedName().compareTo(b.qualifiedName()));
+        })
+            .map(
+                a -> "line "
+                    + a.sourceLocation().toString().substring(1)
+                    + " ["
+                    + (a.qualifier() != null ? "\"" + a.qualifier() + "\".\"" + a.name() + "\"" : a.name())
+                    + "]"
+            )
+            .toList();
+
+        return singletonList(
+            u.withUnresolvedMessage(
+                "Reference [" + u.qualifiedName() + "] is ambiguous (to disambiguate use quotes or qualifiers); " + "matches any of " + refs
+            )
+        );
+    }
+
+    private static Attribute handleSpecialFields(UnresolvedAttribute u, Attribute named, boolean allowCompound) {
+        // if it's a object/compound type, keep it unresolved with a nice error message
+        if (named instanceof FieldAttribute fa) {
+
+            // incompatible mappings
+            if (fa.field() instanceof InvalidMappedField) {
+                named = u.withUnresolvedMessage(
+                    "Cannot use field [" + fa.name() + "] due to ambiguities being " + ((InvalidMappedField) fa.field()).errorMessage()
+                );
+            }
+            // unsupported types
+            else if (DataTypes.isUnsupported(fa.dataType())) {
+                UnsupportedEsField unsupportedField = (UnsupportedEsField) fa.field();
+                if (unsupportedField.hasInherited()) {
+                    named = u.withUnresolvedMessage(
+                        "Cannot use field ["
+                            + fa.name()
+                            + "] with unsupported type ["
+                            + unsupportedField.getOriginalType()
+                            + "] in hierarchy (field ["
+                            + unsupportedField.getInherited()
+                            + "])"
+                    );
+                } else {
+                    named = u.withUnresolvedMessage(
+                        "Cannot use field [" + fa.name() + "] with unsupported type [" + unsupportedField.getOriginalType() + "]"
+                    );
+                }
+            }
+            // compound fields
+            else if (allowCompound == false && DataTypes.isPrimitive(fa.dataType()) == false) {
+                named = u.withUnresolvedMessage(
+                    "Cannot use field [" + fa.name() + "] type [" + fa.dataType().typeName() + "] only its subfields"
+                );
+            }
+        }
+        return named;
+    }
 }
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java
index 102c88b1cc3e..f8f20d25df70 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.ql.execution.search.extractor;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.document.DocumentField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -74,7 +74,7 @@ protected AbstractFieldHitExtractor(StreamInput in) throws IOException {
         String typeName = in.readOptionalString();
         dataType = typeName != null ? loadTypeFromName(typeName) : null;
         hitName = in.readOptionalString();
-        if (in.getVersion().before(Version.V_8_6_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_8_6_0)) {
             this.multiValueSupport = in.readBoolean() ? MultiValueSupport.LENIENT : MultiValueSupport.NONE;
         } else {
             this.multiValueSupport = in.readEnum(MultiValueSupport.class);
@@ -93,7 +93,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(fieldName);
         out.writeOptionalString(dataType == null ? null : dataType.typeName());
         out.writeOptionalString(hitName);
-        if (out.getVersion().before(Version.V_8_6_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_8_6_0)) {
             out.writeBoolean(multiValueSupport != MultiValueSupport.NONE);
         } else {
             out.writeEnum(multiValueSupport);
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/EsIndex.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/EsIndex.java
index d37f696fa0de..75dfba526a10 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/EsIndex.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/EsIndex.java
@@ -52,6 +52,6 @@ public boolean equals(Object obj) {
         }
 
         EsIndex other = (EsIndex) obj;
-        return Objects.equals(name, other.name) && mapping == other.mapping;
+        return Objects.equals(name, other.name) && Objects.equals(mapping, other.mapping);
     }
 }
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java
index 89f720111e0c..0a1370844519 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java
@@ -1619,7 +1619,6 @@ public final LogicalPlan apply(LogicalPlan plan) {
             return rule(plan);
         }
 
-        @Override
         protected final LogicalPlan rule(LogicalPlan plan) {
             // eliminate redundant casts
             return plan.transformExpressionsUp(castType, this::maybePruneCast);
@@ -1766,12 +1765,10 @@ public LogicalPlan apply(LogicalPlan plan) {
             return plan;
         }
 
-        @Override
-        protected LogicalPlan rule(LogicalPlan plan) {
+        private void rule(LogicalPlan plan) {
             if (plan.optimized() == false) {
                 plan.setOptimized();
             }
-            return plan;
         }
     }
 
@@ -1794,7 +1791,6 @@ public final LogicalPlan apply(LogicalPlan plan) {
                 : plan.transformUp(typeToken(), this::rule);
         }
 
-        @Override
         protected abstract LogicalPlan rule(SubPlan plan);
     }
 
@@ -1817,7 +1813,6 @@ public final LogicalPlan apply(LogicalPlan plan) {
                 : plan.transformExpressionsUp(expressionTypeToken, this::rule);
         }
 
-        @Override
         protected LogicalPlan rule(LogicalPlan plan) {
             return plan;
         }
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/rule/ParameterizedRule.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/rule/ParameterizedRule.java
new file mode 100644
index 000000000000..0b3fac1d894a
--- /dev/null
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/rule/ParameterizedRule.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.ql.rule;
+
+import org.elasticsearch.xpack.ql.tree.Node;
+
+public abstract class ParameterizedRule<E extends T, T extends Node<T>, P> extends Rule<E, T> {
+
+    public abstract T apply(T t, P p);
+
+    public T apply(T t) {
+        throw new RuleExecutionException("Cannot call parameterized rule without parameter");
+    }
+}
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/rule/ParameterizedRuleExecutor.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/rule/ParameterizedRuleExecutor.java
new file mode 100644
index 000000000000..bff63fe919da
--- /dev/null
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/rule/ParameterizedRuleExecutor.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.ql.rule;
+
+import org.elasticsearch.xpack.ql.tree.Node;
+
+import java.util.function.Function;
+
+public abstract class ParameterizedRuleExecutor<TreeType extends Node<TreeType>, Context> extends RuleExecutor<TreeType> {
+
+    private final Context context;
+
+    protected ParameterizedRuleExecutor(Context context) {
+        this.context = context;
+    }
+
+    protected Context context() {
+        return context;
+    }
+
+    @Override
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    protected Function<TreeType, TreeType> transform(Rule<?, TreeType> rule) {
+        return (rule instanceof ParameterizedRule pr) ? t -> (TreeType) pr.apply(t, context) : t -> rule.apply(t);
+    }
+}
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/rule/Rule.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/rule/Rule.java
index 355f104ef13b..6a0b1b7169a2 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/rule/Rule.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/rule/Rule.java
@@ -11,17 +11,15 @@
 import org.elasticsearch.xpack.ql.tree.Node;
 import org.elasticsearch.xpack.ql.util.ReflectionUtils;
 
-import java.util.function.UnaryOperator;
-
 /**
  * Rules that apply transformation to a tree. In addition, performs
  * type filtering so that a rule that the rule implementation doesn't
  * have to manually filter.
  * <p>
  * Rules <strong>could</strong> could be built as lambdas but most
- * rules are much larger so we keep them as full blown subclasses.
+ * rules are much larger, so we keep them as full-blown subclasses.
  */
-public abstract class Rule<E extends T, T extends Node<T>> implements UnaryOperator<T> {
+public abstract class Rule<E extends T, T extends Node<T>> {
 
     protected Logger log = LogManager.getLogger(getClass());
 
@@ -44,10 +42,10 @@ public String name() {
         return name;
     }
 
-    protected abstract T rule(E e);
-
     @Override
     public String toString() {
         return name();
     }
+
+    public abstract T apply(T t);
 }
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/rule/RuleExecutor.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/rule/RuleExecutor.java
index 9ca63bc62adf..2b71ca1cef85 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/rule/RuleExecutor.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/rule/RuleExecutor.java
@@ -16,6 +16,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 
 public abstract class RuleExecutor<TreeType extends Node<TreeType>> {
 
@@ -45,7 +46,7 @@ boolean reached(int numberOfRuns) {
         }
     }
 
-    public class Batch {
+    public static class Batch<TreeType extends Node<TreeType>> {
         private final String name;
         private final Rule<?, TreeType>[] rules;
         private final Limiter limit;
@@ -68,19 +69,19 @@ public String name() {
         }
     }
 
-    private final Iterable<Batch> batches = batches();
+    private final Iterable<Batch<TreeType>> batches = batches();
 
-    protected abstract Iterable<RuleExecutor<TreeType>.Batch> batches();
+    protected abstract Iterable<RuleExecutor.Batch<TreeType>> batches();
 
     public class Transformation {
         private final TreeType before, after;
-        private final Rule<?, TreeType> rule;
+        private final String name;
         private Boolean lazyHasChanged;
 
-        Transformation(TreeType plan, Rule<?, TreeType> rule) {
-            this.rule = rule;
-            before = plan;
-            after = rule.apply(before);
+        Transformation(String name, TreeType plan, Function<TreeType, TreeType> transform) {
+            this.name = name;
+            this.before = plan;
+            this.after = transform.apply(before);
         }
 
         public boolean hasChanged() {
@@ -90,8 +91,8 @@ public boolean hasChanged() {
             return lazyHasChanged;
         }
 
-        public String ruleName() {
-            return rule.name();
+        public String name() {
+            return name;
         }
 
         public TreeType before() {
@@ -106,9 +107,9 @@ public TreeType after() {
     public class ExecutionInfo {
 
         private final TreeType before, after;
-        private final Map<Batch, List<Transformation>> transformations;
+        private final Map<Batch<TreeType>, List<Transformation>> transformations;
 
-        ExecutionInfo(TreeType before, TreeType after, Map<Batch, List<Transformation>> transformations) {
+        ExecutionInfo(TreeType before, TreeType after, Map<Batch<TreeType>, List<Transformation>> transformations) {
             this.before = before;
             this.after = after;
             this.transformations = transformations;
@@ -122,23 +123,23 @@ public TreeType after() {
             return after;
         }
 
-        public Map<Batch, List<Transformation>> transformations() {
+        public Map<Batch<TreeType>, List<Transformation>> transformations() {
             return transformations;
         }
     }
 
-    protected TreeType execute(TreeType plan) {
+    protected final TreeType execute(TreeType plan) {
         return executeWithInfo(plan).after;
     }
 
-    protected ExecutionInfo executeWithInfo(TreeType plan) {
+    protected final ExecutionInfo executeWithInfo(TreeType plan) {
         TreeType currentPlan = plan;
 
         long totalDuration = 0;
 
-        Map<Batch, List<Transformation>> transformations = new LinkedHashMap<>();
+        Map<Batch<TreeType>, List<Transformation>> transformations = new LinkedHashMap<>();
 
-        for (Batch batch : batches) {
+        for (Batch<TreeType> batch : batches) {
             int batchRuns = 0;
             List<Transformation> tfs = new ArrayList<>();
             transformations.put(batch, tfs);
@@ -156,7 +157,7 @@ protected ExecutionInfo executeWithInfo(TreeType plan) {
                     if (log.isTraceEnabled()) {
                         log.trace("About to apply rule {}", rule);
                     }
-                    Transformation tf = new Transformation(currentPlan, rule);
+                    Transformation tf = new Transformation(rule.name(), currentPlan, transform(rule));
                     tfs.add(tf);
                     currentPlan = tf.after;
 
@@ -198,4 +199,8 @@ protected ExecutionInfo executeWithInfo(TreeType plan) {
 
         return new ExecutionInfo(plan, currentPlan, transformations);
     }
+
+    protected Function<TreeType, TreeType> transform(Rule<?, TreeType> rule) {
+        return rule::apply;
+    }
 }
diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/EsField.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/EsField.java
index 163667749de2..eaf8a5c894db 100644
--- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/EsField.java
+++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/EsField.java
@@ -48,6 +48,13 @@ public DataType getDataType() {
         return esDataType;
     }
 
+    /**
+     * Create a new {@link EsField} replacing the type.
+     */
+    public EsField withType(DataType esDataType) {
+        return new EsField(name, esDataType, properties, aggregatable, isAlias);
+    }
+
     /**
      * This field can be aggregated
      */
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/action/QlStatusResponseTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/action/QlStatusResponseTests.java
index f24d762470a9..1c96dafd6d55 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/action/QlStatusResponseTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/action/QlStatusResponseTests.java
@@ -59,24 +59,23 @@ protected QlStatusResponse mutateInstance(QlStatusResponse instance) {
     public void testToXContent() throws IOException {
         QlStatusResponse response = createTestInstance();
         try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
-            String expectedJson = formatted(
-                """
-                    {
-                      "id" : "%s",
-                      "is_running" : %s,
-                      "is_partial" : %s,
-                      %s
-                      "expiration_time_in_millis" : %s
-                      %s
-                    }
-                    """,
+            Object[] args = new Object[] {
                 response.getId(),
                 response.isRunning(),
                 response.isPartial(),
                 response.getStartTime() != null ? "\"start_time_in_millis\" : " + response.getStartTime() + "," : "",
                 response.getExpirationTime(),
-                response.getCompletionStatus() != null ? ", \"completion_status\" : " + response.getCompletionStatus().getStatus() : ""
-            );
+                response.getCompletionStatus() != null ? ", \"completion_status\" : " + response.getCompletionStatus().getStatus() : "" };
+            String expectedJson = Strings.format("""
+                {
+                  "id" : "%s",
+                  "is_running" : %s,
+                  "is_partial" : %s,
+                  %s
+                  "expiration_time_in_millis" : %s
+                  %s
+                }
+                """, args);
             response.toXContent(builder, ToXContent.EMPTY_PARAMS);
             assertEquals(XContentHelper.stripWhitespace(expectedJson), Strings.toString(builder));
         }
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/async/StoredAsyncResponseTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/async/StoredAsyncResponseTests.java
index 845c6987c014..eb3ef415eb7c 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/async/StoredAsyncResponseTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/async/StoredAsyncResponseTests.java
@@ -73,6 +73,11 @@ protected StoredAsyncResponse<TestResponse> createTestInstance() {
         }
     }
 
+    @Override
+    protected StoredAsyncResponse<TestResponse> mutateInstance(StoredAsyncResponse<TestResponse> instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<StoredAsyncResponse<TestResponse>> instanceReader() {
         return in -> new StoredAsyncResponse<>(TestResponse::new, in);
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/execution/search/extractor/ConstantExtractorTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/execution/search/extractor/ConstantExtractorTests.java
index b116916613fa..e174107fb321 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/execution/search/extractor/ConstantExtractorTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/execution/search/extractor/ConstantExtractorTests.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.function.Supplier;
 
 public class ConstantExtractorTests extends AbstractWireSerializingTestCase<ConstantExtractor> {
@@ -35,7 +34,7 @@ protected Reader<ConstantExtractor> instanceReader() {
     }
 
     @Override
-    protected ConstantExtractor mutateInstance(ConstantExtractor instance) throws IOException {
+    protected ConstantExtractor mutateInstance(ConstantExtractor instance) {
         return new ConstantExtractor(instance.extract((SearchHit) null) + "mutated");
     }
 
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/gen/processor/ChainingProcessorTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/gen/processor/ChainingProcessorTests.java
index f321a323a862..4d0da38a5205 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/gen/processor/ChainingProcessorTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/gen/processor/ChainingProcessorTests.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessorTests;
 import org.elasticsearch.xpack.ql.expression.processor.Processors;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Supplier;
@@ -40,7 +39,7 @@ protected Reader<ChainingProcessor> instanceReader() {
     }
 
     @Override
-    protected ChainingProcessor mutateInstance(ChainingProcessor instance) throws IOException {
+    protected ChainingProcessor mutateInstance(ChainingProcessor instance) {
         @SuppressWarnings("unchecked")
         Supplier<ChainingProcessor> supplier = randomFrom(
             () -> new ChainingProcessor(instance.first(), randomValueOtherThan(instance.second(), () -> randomProcessor())),
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/gen/processor/ConstantProcessorTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/gen/processor/ConstantProcessorTests.java
index d38d02c56ac5..89c1c9b39201 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/gen/processor/ConstantProcessorTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/gen/processor/ConstantProcessorTests.java
@@ -45,7 +45,7 @@ protected Reader<ConstantProcessor> instanceReader() {
     }
 
     @Override
-    protected ConstantProcessor mutateInstance(ConstantProcessor instance) throws IOException {
+    protected ConstantProcessor mutateInstance(ConstantProcessor instance) {
         return new ConstantProcessor(randomValueOtherThan(instance.process(null), () -> randomLong()));
     }
 
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/logical/BinaryLogicProcessorTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/logical/BinaryLogicProcessorTests.java
index 907fcb1815dd..c400053dd05f 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/logical/BinaryLogicProcessorTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/logical/BinaryLogicProcessorTests.java
@@ -32,6 +32,11 @@ protected BinaryLogicProcessor createTestInstance() {
         return randomProcessor();
     }
 
+    @Override
+    protected BinaryLogicProcessor mutateInstance(BinaryLogicProcessor instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<BinaryLogicProcessor> instanceReader() {
         return BinaryLogicProcessor::new;
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/nulls/CheckNullProcessorTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/nulls/CheckNullProcessorTests.java
index 70e38cb27809..9f2cb5d451f0 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/nulls/CheckNullProcessorTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/nulls/CheckNullProcessorTests.java
@@ -28,6 +28,11 @@ protected CheckNullProcessor createTestInstance() {
         return randomProcessor();
     }
 
+    @Override
+    protected CheckNullProcessor mutateInstance(CheckNullProcessor instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<CheckNullProcessor> instanceReader() {
         return CheckNullProcessor::new;
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/BinaryArithmeticProcessorTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/BinaryArithmeticProcessorTests.java
index c47971e67bb6..696bcacf35f1 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/BinaryArithmeticProcessorTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/arithmetic/BinaryArithmeticProcessorTests.java
@@ -35,6 +35,11 @@ protected BinaryArithmeticProcessor createTestInstance() {
         return randomProcessor();
     }
 
+    @Override
+    protected BinaryArithmeticProcessor mutateInstance(BinaryArithmeticProcessor instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<BinaryArithmeticProcessor> instanceReader() {
         return BinaryArithmeticProcessor::new;
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/BinaryComparisonProcessorTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/BinaryComparisonProcessorTests.java
index ddfed79feb3e..45d804ba4283 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/BinaryComparisonProcessorTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/BinaryComparisonProcessorTests.java
@@ -40,6 +40,11 @@ protected BinaryComparisonProcessor createTestInstance() {
         return randomProcessor();
     }
 
+    @Override
+    protected BinaryComparisonProcessor mutateInstance(BinaryComparisonProcessor instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<BinaryComparisonProcessor> instanceReader() {
         return BinaryComparisonProcessor::new;
diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/InProcessorTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/InProcessorTests.java
index f5014b81d349..03436d9b3f5d 100644
--- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/InProcessorTests.java
+++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/operator/comparison/InProcessorTests.java
@@ -34,6 +34,11 @@ protected InProcessor createTestInstance() {
         return randomProcessor();
     }
 
+    @Override
+    protected InProcessor mutateInstance(InProcessor instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<InProcessor> instanceReader() {
         return InProcessor::new;
diff --git a/x-pack/plugin/repositories-metering-api/qa/azure/build.gradle b/x-pack/plugin/repositories-metering-api/qa/azure/build.gradle
index 82785d162121..b917c9907a56 100644
--- a/x-pack/plugin/repositories-metering-api/qa/azure/build.gradle
+++ b/x-pack/plugin/repositories-metering-api/qa/azure/build.gradle
@@ -8,7 +8,7 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 
 final Project fixture = project(':test:fixtures:azure-fixture')
diff --git a/x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle b/x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle
index 0c833fcb6f2d..2db3bf4f64cd 100644
--- a/x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle
+++ b/x-pack/plugin/repositories-metering-api/qa/gcs/build.gradle
@@ -14,7 +14,7 @@ import java.security.KeyPairGenerator
 
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 
 final Project fixture = project(':test:fixtures:gcs-fixture')
diff --git a/x-pack/plugin/repositories-metering-api/qa/s3/build.gradle b/x-pack/plugin/repositories-metering-api/qa/s3/build.gradle
index 83dbde8c0093..05dbe3d6bd8c 100644
--- a/x-pack/plugin/repositories-metering-api/qa/s3/build.gradle
+++ b/x-pack/plugin/repositories-metering-api/qa/s3/build.gradle
@@ -1,7 +1,7 @@
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 
 final Project fixture = project(':test:fixtures:s3-fixture')
diff --git a/x-pack/plugin/rollup/build.gradle b/x-pack/plugin/rollup/build.gradle
index e2da0ba0f866..0e97e4f555e7 100644
--- a/x-pack/plugin/rollup/build.gradle
+++ b/x-pack/plugin/rollup/build.gradle
@@ -10,8 +10,8 @@ archivesBaseName = 'x-pack-rollup'
 
 dependencies {
   compileOnly project(path: xpackModule('core'))
-  compileOnly project(':modules:data-streams')
-  compileOnly project(path: xpackModule('ilm'))
+  testImplementation project(':modules:data-streams')
+  testImplementation project(path: xpackModule('ilm'))
   compileOnly project(path: xpackModule('mapper-aggregate-metric'))
   testImplementation(testArtifact(project(xpackModule('core'))))
 }
diff --git a/x-pack/plugin/rollup/qa/rest/build.gradle b/x-pack/plugin/rollup/qa/rest/build.gradle
index 6e69f1e929d8..e91e56d2d1ba 100644
--- a/x-pack/plugin/rollup/qa/rest/build.gradle
+++ b/x-pack/plugin/rollup/qa/rest/build.gradle
@@ -8,8 +8,8 @@
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 dependencies {
   yamlRestTestImplementation project(path: xpackModule('rollup'))
diff --git a/x-pack/plugin/rollup/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/rollup/10_basic.yml b/x-pack/plugin/rollup/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/rollup/10_basic.yml
index 5c6d9f4d83a1..e63cf9e82287 100644
--- a/x-pack/plugin/rollup/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/rollup/10_basic.yml
+++ b/x-pack/plugin/rollup/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/rollup/10_basic.yml
@@ -86,11 +86,78 @@ setup:
         body:
           index.blocks.write: true
 
+  - do:
+      indices.create:
+        index: test-histogram
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+            index:
+              mode: time_series
+              routing_path: [ metricset, k8s.pod.uid ]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              metricset:
+                type: keyword
+                time_series_dimension: true
+              k8s:
+                properties:
+                  pod:
+                    properties:
+                      uid:
+                        type: keyword
+                        time_series_dimension: true
+                      name:
+                        type: keyword
+                      latency:
+                        type: histogram
+                      empty-histogram:
+                        type: histogram
+                      network:
+                        properties:
+                          tx:
+                            type: long
+                            time_series_metric: gauge
+                          rx:
+                            type: long
+                            time_series_metric: gauge
+  - do:
+      bulk:
+        refresh: true
+        index: test-histogram
+        body:
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "network": {"tx": 2001818691, "rx": 802133794}, "latency": {"counts": [10, 12, 20, 5], "values": [1.0, 10.0, 100.0, 1000.0]}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:55:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "network": {"tx": 2001819988, "rx": 802133911}, "latency": {"counts": [8, 7, 10, 12], "values": [1.0, 2.0, 5.0, 10.0]}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T19:00:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "network": {"tx": 1001818691, "rx": 502133794}, "latency": {"counts": [1, 5, 5, 22], "values": [1.0, 10.0, 100.0, 1000.0]}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T19:05:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "network": {"tx": 1001822087, "rx": 502134222}, "latency": {"counts": [7, 15, 10, 10], "values": [1.0, 2.0, 5.0, 10.0]}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:51:20.467Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "network": {"tx": 1781818691, "rx": 533135238}, "latency": {"counts": [2, 4, 16, 4], "values": [1.0, 2.0, 5.0, 10.0]}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:56:20.467Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "network": {"tx": 1781818691, "rx": 533135567}, "latency": {"counts": [2, 2, 8, 8], "values": [1.0, 10.0, 100.0, 1000.0]}}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T19:01:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "network": {"tx": 1651818691, "rx": 487133866}, "latency": {"counts": [4, 5, 4, 13], "values": [1.0, 10.0, 100.0, 1000.0]}}}}'
+
+  - do:
+      indices.put_settings:
+        index: test-histogram
+        body:
+          index.blocks.write: true
+
 ---
 "Downsample index":
   - skip:
       version: " - 8.4.99"
-      reason: "rollup renamed to downsample in 8.5.0"
+      reason: "Downsampling GA-ed in 8.7.0"
 
   - do:
       indices.downsample:
@@ -110,23 +177,23 @@ setup:
 
   - length: { hits.hits: 4 }
   - match:  { hits.hits.0._source._doc_count: 2 }
-  - match:  { hits.hits.0._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match:  { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   - match:  { hits.hits.0._source.metricset: pod }
   - match:  { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
-  - match:  { hits.hits.0._source.k8s\.pod\.multi-counter: 21 }
-  - match:  { hits.hits.0._source.k8s\.pod\.multi-gauge.min: 90 }
-  - match:  { hits.hits.0._source.k8s\.pod\.multi-gauge.max: 200 }
-  - match:  { hits.hits.0._source.k8s\.pod\.multi-gauge.sum: 726 }
-  - match:  { hits.hits.0._source.k8s\.pod\.multi-gauge.value_count: 6 }
-  - match:  { hits.hits.0._source.k8s\.pod\.network\.tx.min: 2001818691 }
-  - match:  { hits.hits.0._source.k8s\.pod\.network\.tx.max: 2005177954 }
-  - match:  { hits.hits.0._source.k8s\.pod\.network\.tx.value_count: 2 }
-  - match:  { hits.hits.0._source.k8s\.pod\.ip: "10.10.55.26" }
-  - match:  { hits.hits.0._source.k8s\.pod\.created_at: "2021-04-28T19:35:00.000Z" }
-  - match:  { hits.hits.0._source.k8s\.pod\.number_of_containers: 2 }
-  - match:  { hits.hits.0._source.k8s\.pod\.tags: ["backend", "prod", "us-west1"] }
-  - match:  { hits.hits.0._source.k8s\.pod\.values: [1, 1, 3] }
-  - is_true: hits.hits.0._source.k8s\.pod\.running
+  - match:  { hits.hits.0._source.k8s.pod.multi-counter: 21 }
+  - match:  { hits.hits.0._source.k8s.pod.multi-gauge.min: 90 }
+  - match:  { hits.hits.0._source.k8s.pod.multi-gauge.max: 200 }
+  - match:  { hits.hits.0._source.k8s.pod.multi-gauge.sum: 726 }
+  - match:  { hits.hits.0._source.k8s.pod.multi-gauge.value_count: 6 }
+  - match:  { hits.hits.0._source.k8s.pod.network.tx.min: 2001818691 }
+  - match:  { hits.hits.0._source.k8s.pod.network.tx.max: 2005177954 }
+  - match:  { hits.hits.0._source.k8s.pod.network.tx.value_count: 2 }
+  - match:  { hits.hits.0._source.k8s.pod.ip: "10.10.55.26" }
+  - match:  { hits.hits.0._source.k8s.pod.created_at: "2021-04-28T19:35:00.000Z" }
+  - match:  { hits.hits.0._source.k8s.pod.number_of_containers: 2 }
+  - match:  { hits.hits.0._source.k8s.pod.tags: ["backend", "prod", "us-west1"] }
+  - match:  { hits.hits.0._source.k8s.pod.values: [1, 1, 3] }
+  - is_true: hits.hits.0._source.k8s.pod.running
 
   # Assert rollup index settings
   - do:
@@ -345,8 +412,8 @@ setup:
 ---
 "Downsample a downsampled index":
   - skip:
-      version: " - 8.4.99"
-      reason: "Rollup of rollups introduced in 8.5.0"
+      version: " - 8.6.99"
+      reason: "Rollup GA-ed in 8.7.0"
 
   - do:
       indices.downsample:
@@ -398,29 +465,29 @@ setup:
 
   - length: { hits.hits: 3 }
   - match: { hits.hits.0._source._doc_count: 2 }
-  - match: { hits.hits.0._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   - match: { hits.hits.0._source.metricset: pod }
   - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
-  - match: { hits.hits.0._source.k8s\.pod\.multi-counter: 21 }
-  - match: { hits.hits.0._source.k8s\.pod\.multi-gauge.min: 90 }
-  - match: { hits.hits.0._source.k8s\.pod\.multi-gauge.max: 200 }
-  - match: { hits.hits.0._source.k8s\.pod\.multi-gauge.sum: 726 }
-  - match: { hits.hits.0._source.k8s\.pod\.multi-gauge.value_count: 6 }
-  - match: { hits.hits.0._source.k8s\.pod\.network\.tx.min: 2001818691 }
-  - match: { hits.hits.0._source.k8s\.pod\.network\.tx.max: 2005177954 }
-  - match: { hits.hits.0._source.k8s\.pod\.network\.tx.value_count: 2 }
-  - match: { hits.hits.0._source.k8s\.pod\.ip: "10.10.55.26" }
-  - match: { hits.hits.0._source.k8s\.pod\.created_at: "2021-04-28T19:35:00.000Z" }
-  - match: { hits.hits.0._source.k8s\.pod\.number_of_containers: 2 }
-  - match: { hits.hits.0._source.k8s\.pod\.tags: [ "backend", "prod", "us-west1" ] }
-  - match: { hits.hits.0._source.k8s\.pod\.values: [ 1, 1, 3 ] }
-
-  - match: { hits.hits.1._source.k8s\.pod\.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match: { hits.hits.0._source.k8s.pod.multi-counter: 21 }
+  - match: { hits.hits.0._source.k8s.pod.multi-gauge.min: 90 }
+  - match: { hits.hits.0._source.k8s.pod.multi-gauge.max: 200 }
+  - match: { hits.hits.0._source.k8s.pod.multi-gauge.sum: 726 }
+  - match: { hits.hits.0._source.k8s.pod.multi-gauge.value_count: 6 }
+  - match: { hits.hits.0._source.k8s.pod.network.tx.min: 2001818691 }
+  - match: { hits.hits.0._source.k8s.pod.network.tx.max: 2005177954 }
+  - match: { hits.hits.0._source.k8s.pod.network.tx.value_count: 2 }
+  - match: { hits.hits.0._source.k8s.pod.ip: "10.10.55.26" }
+  - match: { hits.hits.0._source.k8s.pod.created_at: "2021-04-28T19:35:00.000Z" }
+  - match: { hits.hits.0._source.k8s.pod.number_of_containers: 2 }
+  - match: { hits.hits.0._source.k8s.pod.tags: [ "backend", "prod", "us-west1" ] }
+  - match: { hits.hits.0._source.k8s.pod.values: [ 1, 1, 3 ] }
+
+  - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
   - match: { hits.hits.1._source.metricset: pod }
   - match: { hits.hits.1._source.@timestamp: 2021-04-28T20:00:00.000Z }
   - match: { hits.hits.1._source._doc_count: 2 }
 
-  - match: { hits.hits.2._source.k8s\.pod\.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match: { hits.hits.2._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
   - match: { hits.hits.2._source.metricset: pod }
   - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:00:00.000Z }
   - match: { hits.hits.2._source._doc_count: 4 }
@@ -480,3 +547,104 @@ setup:
           {
             "fixed_interval": "90m"
           }
+
+---
+"Downsample histogram as label":
+  - skip:
+      version: " - 8.4.99"
+      reason: "rollup renamed to downsample in 8.5.0"
+
+  - do:
+      indices.downsample:
+        index: test-histogram
+        target_index: rollup-test-histogram
+        body:  >
+          {
+            "fixed_interval": "1h"
+          }
+  - is_true: acknowledged
+
+  - do:
+      indices.get_mapping:
+        index: rollup-test-histogram
+
+  - match: { rollup-test-histogram.mappings.properties.@timestamp.type: date }
+  - match: { rollup-test-histogram.mappings.properties.@timestamp.meta.fixed_interval: 1h }
+  - match: { rollup-test-histogram.mappings.properties.@timestamp.meta.time_zone: UTC }
+  - match: { rollup-test-histogram.mappings.properties.k8s.properties.pod.properties.latency.type: histogram }
+  - match: { rollup-test-histogram.mappings.properties.k8s.properties.pod.properties.latency.time_series_metric: null }
+  - match: { rollup-test-histogram.mappings.properties.k8s.properties.pod.properties.empty-histogram.type: histogram }
+  - match: { rollup-test-histogram.mappings.properties.k8s.properties.pod.properties.empty-histogram.time_series_metric: null }
+  - match: { rollup-test-histogram.mappings.properties.k8s.properties.pod.properties.uid.type: keyword }
+  - match: { rollup-test-histogram.mappings.properties.k8s.properties.pod.properties.uid.time_series_dimension: true }
+  - match: { rollup-test-histogram.mappings.properties.k8s.properties.pod.properties.network.properties.tx.type: aggregate_metric_double }
+  - match: { rollup-test-histogram.mappings.properties.k8s.properties.pod.properties.network.properties.tx.metrics: [ "min", "max", "sum", "value_count" ] }
+  - match: { rollup-test-histogram.mappings.properties.k8s.properties.pod.properties.network.properties.tx.default_metric: max }
+  - match: { rollup-test-histogram.mappings.properties.k8s.properties.pod.properties.network.properties.tx.time_series_metric: gauge }
+
+  - do:
+      search:
+        index: rollup-test-histogram
+        body:
+          sort: [ "_tsid", "@timestamp" ]
+
+  - length: { hits.hits: 4 }
+  - match:  { hits.hits.0._source._doc_count: 2 }
+  - match:  { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match:  { hits.hits.0._source.metricset: pod }
+  - match:  { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z }
+  - length: { hits.hits.0._source.k8s.pod.latency.counts: 4 }
+  - match:  { hits.hits.0._source.k8s.pod.latency.counts.0: 8 }
+  - match:  { hits.hits.0._source.k8s.pod.latency.counts.1: 7 }
+  - match:  { hits.hits.0._source.k8s.pod.latency.counts.2: 10 }
+  - match:  { hits.hits.0._source.k8s.pod.latency.counts.3: 12 }
+  - length: { hits.hits.0._source.k8s.pod.latency.values: 4 }
+  - match:  { hits.hits.0._source.k8s.pod.latency.values.0: 1.0 }
+  - match:  { hits.hits.0._source.k8s.pod.latency.values.1: 2.0 }
+  - match:  { hits.hits.0._source.k8s.pod.latency.values.2: 5.0 }
+  - match:  { hits.hits.0._source.k8s.pod.latency.values.3: 10.0 }
+
+  - match:  { hits.hits.1._source._doc_count: 2 }
+  - match:  { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 }
+  - match:  { hits.hits.1._source.metricset: pod }
+  - match:  { hits.hits.1._source.@timestamp: 2021-04-28T19:00:00.000Z }
+  - length: { hits.hits.1._source.k8s.pod.latency.counts: 4 }
+  - match:  { hits.hits.1._source.k8s.pod.latency.counts.0: 7 }
+  - match:  { hits.hits.1._source.k8s.pod.latency.counts.1: 15 }
+  - match:  { hits.hits.1._source.k8s.pod.latency.counts.2: 10 }
+  - match:  { hits.hits.1._source.k8s.pod.latency.counts.3: 10 }
+  - length: { hits.hits.1._source.k8s.pod.latency.values: 4 }
+  - match:  { hits.hits.1._source.k8s.pod.latency.values.0: 1.0 }
+  - match:  { hits.hits.1._source.k8s.pod.latency.values.1: 2.0 }
+  - match:  { hits.hits.1._source.k8s.pod.latency.values.2: 5.0 }
+  - match:  { hits.hits.1._source.k8s.pod.latency.values.3: 10.0 }
+
+  - match:  { hits.hits.2._source._doc_count: 2 }
+  - match:  { hits.hits.2._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match:  { hits.hits.2._source.metricset: pod }
+  - match:  { hits.hits.2._source.@timestamp: 2021-04-28T18:00:00.000Z }
+  - length: { hits.hits.2._source.k8s.pod.latency.counts: 4 }
+  - match:  { hits.hits.2._source.k8s.pod.latency.counts.0: 2 }
+  - match:  { hits.hits.2._source.k8s.pod.latency.counts.1: 2 }
+  - match:  { hits.hits.2._source.k8s.pod.latency.counts.2: 8 }
+  - match:  { hits.hits.2._source.k8s.pod.latency.counts.3: 8 }
+  - length: { hits.hits.2._source.k8s.pod.latency.values: 4 }
+  - match:  { hits.hits.2._source.k8s.pod.latency.values.0: 1.0 }
+  - match:  { hits.hits.2._source.k8s.pod.latency.values.1: 10.0 }
+  - match:  { hits.hits.2._source.k8s.pod.latency.values.2: 100.0 }
+  - match:  { hits.hits.2._source.k8s.pod.latency.values.3: 1000.0 }
+
+  - match:  { hits.hits.3._source._doc_count: 1 }
+  - match:  { hits.hits.3._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 }
+  - match:  { hits.hits.3._source.metricset: pod }
+  - match:  { hits.hits.3._source.@timestamp: 2021-04-28T19:00:00.000Z }
+  - length: { hits.hits.3._source.k8s.pod.latency.counts: 4 }
+  - match:  { hits.hits.3._source.k8s.pod.latency.counts.0: 4 }
+  - match:  { hits.hits.3._source.k8s.pod.latency.counts.1: 5 }
+  - match:  { hits.hits.3._source.k8s.pod.latency.counts.2: 4 }
+  - match:  { hits.hits.3._source.k8s.pod.latency.counts.3: 13 }
+  - length: { hits.hits.3._source.k8s.pod.latency.values: 4 }
+  - match:  { hits.hits.3._source.k8s.pod.latency.values.0: 1.0 }
+  - match:  { hits.hits.3._source.k8s.pod.latency.values.1: 10.0 }
+  - match:  { hits.hits.3._source.k8s.pod.latency.values.2: 100.0 }
+  - match:  { hits.hits.3._source.k8s.pod.latency.values.3: 1000.0 }
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/AbstractRollupFieldProducer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/AbstractRollupFieldProducer.java
index 44bf433ada13..935eb0735775 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/AbstractRollupFieldProducer.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/AbstractRollupFieldProducer.java
@@ -7,16 +7,16 @@
 
 package org.elasticsearch.xpack.downsample;
 
-import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.index.fielddata.FormattedDocValues;
 
 import java.io.IOException;
 
 /**
- * Base class for classes that read metric and label fields.
+ * Base class that reads fields from the source index and produces their downsampled values
  */
-abstract class AbstractRollupFieldProducer<T> {
+abstract class AbstractRollupFieldProducer implements RollupFieldSerializer {
 
-    protected final String name;
+    private final String name;
     protected boolean isEmpty;
 
     AbstractRollupFieldProducer(String name) {
@@ -24,14 +24,6 @@ abstract class AbstractRollupFieldProducer<T> {
         this.isEmpty = true;
     }
 
-    /**
-     * Collect a value for the field applying the specific subclass collection strategy.
-     *
-     * @param field the name of the field to collect
-     * @param value the value to collect.
-     */
-    public abstract void collect(String field, T value);
-
     /**
      * @return the name of the field.
      */
@@ -44,15 +36,12 @@ public String name() {
      */
     public abstract void reset();
 
-    /**
-     * Serialize the downsampled value of the field.
-     */
-    public abstract void write(XContentBuilder builder) throws IOException;
-
     /**
      * @return true if the field has not collected any value.
      */
     public boolean isEmpty() {
         return isEmpty;
     }
+
+    public abstract void collect(FormattedDocValues docValues, int docId) throws IOException;
 }
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldSerializer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldSerializer.java
new file mode 100644
index 000000000000..50014f304a7a
--- /dev/null
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldSerializer.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.downsample;
+
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Collection;
+
+public class AggregateMetricFieldSerializer implements RollupFieldSerializer {
+    private final Collection<AbstractRollupFieldProducer> producers;
+    private final String name;
+
+    /**
+     * @param name the name of the aggregate_metric_double field as it will be serialized
+     *             in the downsampled index
+     * @param producers a collection of {@link AbstractRollupFieldProducer} instances with the subfields
+     *                  of the aggregate_metric_double field.
+     */
+    public AggregateMetricFieldSerializer(String name, Collection<AbstractRollupFieldProducer> producers) {
+        this.name = name;
+        this.producers = producers;
+    }
+
+    @Override
+    public void write(XContentBuilder builder) throws IOException {
+        if (isEmpty()) {
+            return;
+        }
+
+        builder.startObject(name);
+        for (AbstractRollupFieldProducer rollupFieldProducer : producers) {
+            assert name.equals(rollupFieldProducer.name()) : "producer has a different name";
+            if (rollupFieldProducer.isEmpty() == false) {
+                if (rollupFieldProducer instanceof MetricFieldProducer metricFieldProducer) {
+                    for (MetricFieldProducer.Metric metric : metricFieldProducer.metrics()) {
+                        if (metric.get() != null) {
+                            builder.field(metric.name(), metric.get());
+                        }
+                    }
+                } else if (rollupFieldProducer instanceof LabelFieldProducer labelFieldProducer) {
+                    LabelFieldProducer.Label label = labelFieldProducer.label();
+                    if (label.get() != null) {
+                        builder.field(label.name(), label.get());
+                    }
+                }
+            }
+        }
+        builder.endObject();
+    }
+
+    private boolean isEmpty() {
+        for (AbstractRollupFieldProducer p : producers) {
+            if (p.isEmpty() == false) {
+                return false;
+            }
+        }
+        return true;
+    }
+}
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldValueFetcher.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldValueFetcher.java
new file mode 100644
index 000000000000..19abd6add235
--- /dev/null
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldValueFetcher.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.downsample;
+
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper;
+import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.AggregateDoubleMetricFieldType;
+
+import java.util.List;
+
+public class AggregateMetricFieldValueFetcher extends FieldValueFetcher {
+
+    private AggregateDoubleMetricFieldType aggMetricFieldType;
+
+    private final AbstractRollupFieldProducer rollupFieldProducer;
+
+    protected AggregateMetricFieldValueFetcher(
+        MappedFieldType fieldType,
+        AggregateDoubleMetricFieldType aggMetricFieldType,
+        IndexFieldData<?> fieldData
+    ) {
+        super(fieldType, fieldData);
+        this.aggMetricFieldType = aggMetricFieldType;
+        this.rollupFieldProducer = createRollupFieldProducer();
+    }
+
+    public AbstractRollupFieldProducer rollupFieldProducer() {
+        return rollupFieldProducer;
+    }
+
+    private AbstractRollupFieldProducer createRollupFieldProducer() {
+        AggregateDoubleMetricFieldMapper.Metric metric = null;
+        for (var e : aggMetricFieldType.getMetricFields().entrySet()) {
+            NumberFieldMapper.NumberFieldType metricSubField = e.getValue();
+            if (metricSubField.name().equals(name())) {
+                metric = e.getKey();
+                break;
+            }
+        }
+        assert metric != null : "Cannot resolve metric type for field " + name();
+
+        if (aggMetricFieldType.getMetricType() != null) {
+            // If the field is an aggregate_metric_double field, we should use the correct subfields
+            // for each aggregation. This is a rollup-of-rollup case
+            MetricFieldProducer.Metric metricOperation = switch (metric) {
+                case max -> new MetricFieldProducer.Max();
+                case min -> new MetricFieldProducer.Min();
+                case sum -> new MetricFieldProducer.Sum();
+                // To compute value_count summary, we must sum all field values
+                case value_count -> new MetricFieldProducer.Sum(AggregateDoubleMetricFieldMapper.Metric.value_count.name());
+            };
+            return new MetricFieldProducer.GaugeMetricFieldProducer(aggMetricFieldType.name(), List.of(metricOperation));
+        } else {
+            // If field is not a metric, we downsample it as a label
+            return new LabelFieldProducer.AggregateMetricFieldProducer.AggregateMetricFieldProducer(aggMetricFieldType.name(), metric);
+        }
+    }
+}
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java
index 5c99d6135b3e..02bd6b35b1e8 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java
@@ -16,51 +16,62 @@
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper;
 
+import java.util.ArrayList;
 import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.Map;
+import java.util.List;
 
 /**
  * Utility class used for fetching field values by reading field data
  */
 class FieldValueFetcher {
-    private final String name;
+
     private final MappedFieldType fieldType;
-    private final DocValueFormat format;
     private final IndexFieldData<?> fieldData;
+    private final AbstractRollupFieldProducer rollupFieldProducer;
 
-    protected FieldValueFetcher(String name, MappedFieldType fieldType, IndexFieldData<?> fieldData) {
-        this.name = name;
+    protected FieldValueFetcher(MappedFieldType fieldType, IndexFieldData<?> fieldData) {
         this.fieldType = fieldType;
-        this.format = fieldType.docValueFormat(null, null);
         this.fieldData = fieldData;
+        this.rollupFieldProducer = createRollupFieldProducer();
     }
 
     public String name() {
-        return name;
+        return fieldType().name();
     }
 
     public MappedFieldType fieldType() {
         return fieldType;
     }
 
-    public DocValueFormat format() {
-        return format;
+    public FormattedDocValues getLeaf(LeafReaderContext context) {
+        DocValueFormat format = fieldType.docValueFormat(null, null);
+        return fieldData.load(context).getFormattedValues(format);
     }
 
-    public IndexFieldData<?> fieldData() {
-        return fieldData;
+    public AbstractRollupFieldProducer rollupFieldProducer() {
+        return rollupFieldProducer;
     }
 
-    FormattedDocValues getLeaf(LeafReaderContext context) {
-        return fieldData.load(context).getFormattedValues(format);
+    private AbstractRollupFieldProducer createRollupFieldProducer() {
+        if (fieldType.getMetricType() != null) {
+            return switch (fieldType.getMetricType()) {
+                case GAUGE -> new MetricFieldProducer.GaugeMetricFieldProducer(name());
+                case COUNTER -> new MetricFieldProducer.CounterMetricFieldProducer(name());
+            };
+        } else {
+            // If field is not a metric, we downsample it as a label
+            if ("histogram".equals(fieldType.typeName())) {
+                return new LabelFieldProducer.HistogramLastLabelFieldProducer(name());
+            }
+            return new LabelFieldProducer.LabelLastValueFieldProducer(name());
+        }
     }
 
     /**
      * Retrieve field value fetchers for a list of fields.
      */
-    static Map<String, FieldValueFetcher> create(SearchExecutionContext context, String[] fields) {
-        Map<String, FieldValueFetcher> fetchers = new LinkedHashMap<>();
+    static List<FieldValueFetcher> create(SearchExecutionContext context, String[] fields) {
+        List<FieldValueFetcher> fetchers = new ArrayList<>();
         for (String field : fields) {
             MappedFieldType fieldType = context.getFieldType(field);
             assert fieldType != null : "Unknown field type for field: [" + field + "]";
@@ -71,24 +82,16 @@ static Map<String, FieldValueFetcher> create(SearchExecutionContext context, Str
                 for (NumberFieldMapper.NumberFieldType metricSubField : aggMetricFieldType.getMetricFields().values()) {
                     if (context.fieldExistsInIndex(metricSubField.name())) {
                         IndexFieldData<?> fieldData = context.getForField(metricSubField, MappedFieldType.FielddataOperation.SEARCH);
-                        fetchers.put(metricSubField.name(), new FieldValueFetcher(metricSubField.name(), fieldType, fieldData));
+                        fetchers.add(new AggregateMetricFieldValueFetcher(metricSubField, aggMetricFieldType, fieldData));
                     }
                 }
             } else {
                 if (context.fieldExistsInIndex(field)) {
                     IndexFieldData<?> fieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH);
-                    fetchers.put(field, new FieldValueFetcher(field, fieldType, fieldData));
+                    fetchers.add(new FieldValueFetcher(fieldType, fieldData));
                 }
             }
         }
-        return Collections.unmodifiableMap(fetchers);
-    }
-
-    static Map<String, FormattedDocValues> docValuesFetchers(LeafReaderContext ctx, Map<String, FieldValueFetcher> fieldValueFetchers) {
-        final Map<String, FormattedDocValues> docValuesFetchers = new LinkedHashMap<>(fieldValueFetchers.size());
-        for (FieldValueFetcher fetcher : fieldValueFetchers.values()) {
-            docValuesFetchers.put(fetcher.name(), fetcher.getLeaf(ctx));
-        }
-        return Collections.unmodifiableMap(docValuesFetchers);
+        return Collections.unmodifiableList(fetchers);
     }
 }
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/LabelFieldProducer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/LabelFieldProducer.java
index 8affcbdc0c1e..5a364eec11f7 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/LabelFieldProducer.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/LabelFieldProducer.java
@@ -7,58 +7,28 @@
 
 package org.elasticsearch.xpack.downsample;
 
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.NumberFieldMapper;
-import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.index.fielddata.FormattedDocValues;
+import org.elasticsearch.index.fielddata.HistogramValue;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper;
+import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Metric;
 
 import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.Map;
+import java.util.ArrayList;
+import java.util.List;
 
 /**
  * Class that produces values for a label field.
  */
-abstract class LabelFieldProducer extends AbstractRollupFieldProducer<Object> {
+abstract class LabelFieldProducer extends AbstractRollupFieldProducer {
 
-    private final Label label;
-
-    LabelFieldProducer(String name, Label label) {
+    LabelFieldProducer(String name) {
         super(name);
-        this.label = label;
-    }
-
-    public String name() {
-        return name;
-    }
-
-    /** Collect the value of a raw field  */
-    @Override
-    public void collect(String field, Object value) {
-        label.collect(value);
-        isEmpty = false;
-    }
-
-    public Label label() {
-        return this.label;
-    }
-
-    public void reset() {
-        label.reset();
-        isEmpty = true;
     }
 
-    /**
-     * Return the downsampled value as computed after collecting all raw values.
-     * @return
-     */
-    public abstract Object value();
+    abstract Label label();
 
     abstract static class Label {
-        final String name;
+        private final String name;
 
         /**
          * Abstract class that defines how a label is downsampled.
@@ -94,14 +64,7 @@ static class LastValueLabel extends Label {
         }
 
         LastValueLabel() {
-            super("last_value");
-        }
-
-        @Override
-        void collect(Object value) {
-            if (lastValue == null) {
-                lastValue = value;
-            }
+            this("last_value");
         }
 
         @Override
@@ -113,101 +76,95 @@ Object get() {
         void reset() {
             lastValue = null;
         }
+
+        void collect(Object value) {
+            if (lastValue == null) {
+                lastValue = value;
+            }
+        }
     }
 
     /**
      * {@link LabelFieldProducer} implementation for a last value label
      */
     static class LabelLastValueFieldProducer extends LabelFieldProducer {
+        protected final LastValueLabel label;
+
+        LabelLastValueFieldProducer(String name, LastValueLabel label) {
+            super(name);
+            this.label = label;
+        }
 
         LabelLastValueFieldProducer(String name) {
-            super(name, new LastValueLabel());
+            this(name, new LastValueLabel());
         }
 
         @Override
-        public Object value() {
-            return label().get();
+        Label label() {
+            return label;
         }
 
         @Override
         public void write(XContentBuilder builder) throws IOException {
             if (isEmpty() == false) {
-                builder.field(name(), value());
+                builder.field(name(), label.get());
             }
         }
-    }
-
-    static class AggregateMetricFieldProducer extends LabelFieldProducer {
-
-        private Map<String, Label> labelsByField = new LinkedHashMap<>();
-
-        AggregateMetricFieldProducer(String name) {
-            super(name, null);
-        }
-
-        public void addLabel(String field, Label label) {
-            labelsByField.put(field, label);
-        }
-
-        @Override
-        public void collect(String field, Object value) {
-            labelsByField.get(field).collect(value);
-            isEmpty = false;
-        }
 
         @Override
-        public void write(XContentBuilder builder) throws IOException {
+        public void collect(FormattedDocValues docValues, int docId) throws IOException {
             if (isEmpty() == false) {
-                builder.startObject(name());
-                for (Label label : labels()) {
-                    if (label.get() != null) {
-                        builder.field(label.name(), label.get());
-                    }
-                }
-                builder.endObject();
+                return;
+            }
+            if (docValues.advanceExact(docId) == false) {
+                return;
             }
-        }
-
-        public Collection<Label> labels() {
-            return labelsByField.values();
-        }
 
-        @Override
-        public Object value() {
-            return labelsByField;
+            int docValuesCount = docValues.docValueCount();
+            assert docValuesCount > 0;
+            isEmpty = false;
+            if (docValuesCount == 1) {
+                label.collect(docValues.nextValue());
+            } else {
+                Object[] values = new Object[docValuesCount];
+                for (int i = 0; i < docValuesCount; i++) {
+                    values[i] = docValues.nextValue();
+                }
+                label.collect(values);
+            }
         }
 
         @Override
         public void reset() {
-            labels().forEach(Label::reset);
+            label.reset();
             isEmpty = true;
         }
     }
 
-    /**
-     * Create a collection of label field producers.
-     */
-    static Map<String, LabelFieldProducer> createLabelFieldProducers(SearchExecutionContext context, String[] labelFields) {
-        final Map<String, LabelFieldProducer> fields = new LinkedHashMap<>();
-        for (String field : labelFields) {
-            MappedFieldType fieldType = context.getFieldType(field);
-            assert fieldType != null : "Unknown field type for field: [" + field + "]";
-
-            if (fieldType instanceof AggregateDoubleMetricFieldMapper.AggregateDoubleMetricFieldType aggMetricFieldType) {
-                // If the field is an aggregate_metric_double field, we should use the correct subfields
-                // for each aggregation. This is a rollup-of-rollup case
-                AggregateMetricFieldProducer producer = new AggregateMetricFieldProducer(field);
-                for (var e : aggMetricFieldType.getMetricFields().entrySet()) {
-                    AggregateDoubleMetricFieldMapper.Metric metric = e.getKey();
-                    NumberFieldMapper.NumberFieldType metricSubField = e.getValue();
-                    producer.addLabel(metricSubField.name(), new LastValueLabel(metric.name()));
-                    fields.put(metricSubField.name(), producer);
+    static class AggregateMetricFieldProducer extends LabelLastValueFieldProducer {
+
+        AggregateMetricFieldProducer(String name, Metric metric) {
+            super(name, new LastValueLabel(metric.name()));
+        }
+    }
+
+    public static class HistogramLastLabelFieldProducer extends LabelLastValueFieldProducer {
+        HistogramLastLabelFieldProducer(String name) {
+            super(name);
+        }
+
+        @Override
+        public void write(XContentBuilder builder) throws IOException {
+            if (isEmpty() == false) {
+                final HistogramValue histogramValue = (HistogramValue) label.get();
+                final List<Double> values = new ArrayList<>();
+                final List<Integer> counts = new ArrayList<>();
+                while (histogramValue.next()) {
+                    values.add(histogramValue.value());
+                    counts.add(histogramValue.count());
                 }
-            } else {
-                LabelFieldProducer producer = new LabelLastValueFieldProducer(field);
-                fields.put(field, producer);
+                builder.startObject(name()).field("counts", counts).field("values", values).endObject();
             }
         }
-        return Collections.unmodifiableMap(fields);
     }
 }
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/MetricFieldProducer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/MetricFieldProducer.java
index a849ac915eeb..1028a210754c 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/MetricFieldProducer.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/MetricFieldProducer.java
@@ -7,26 +7,21 @@
 
 package org.elasticsearch.xpack.downsample;
 
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.NumberFieldMapper;
-import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.index.fielddata.FormattedDocValues;
 import org.elasticsearch.search.aggregations.metrics.CompensatedSum;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper;
 
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Map;
 
 /**
  * Class that collects all raw values for a metric field and computes its aggregate (downsampled)
  * values. Based on the supported metric types, the subclasses of this class compute values for
  * gauge and metric types.
  */
-abstract class MetricFieldProducer extends AbstractRollupFieldProducer<Number> {
+abstract class MetricFieldProducer extends AbstractRollupFieldProducer {
     /**
      * a list of metrics that will be computed for the field
      */
@@ -41,30 +36,35 @@ abstract class MetricFieldProducer extends AbstractRollupFieldProducer<Number> {
      * Reset all values collected for the field
      */
     public void reset() {
-        for (Metric metric : metrics()) {
-            metric.reset();
-        }
+        metrics().forEach(Metric::reset);
         isEmpty = true;
     }
 
-    public String name() {
-        return name;
-    }
-
     /** return the list of metrics that are computed for the field */
     public Collection<Metric> metrics() {
         return metrics;
     }
 
     /** Collect the value of a raw field and compute all downsampled metrics */
-    @Override
-    public void collect(String field, Number value) {
+    void collect(Number value) {
         for (MetricFieldProducer.Metric metric : metrics()) {
             metric.collect(value);
         }
         isEmpty = false;
     }
 
+    @Override
+    public void collect(FormattedDocValues docValues, int docId) throws IOException {
+        if (docValues.advanceExact(docId) == false) {
+            return;
+        }
+        int docValuesCount = docValues.docValueCount();
+        for (int i = 0; i < docValuesCount; i++) {
+            Number num = (Number) docValues.nextValue();
+            collect(num);
+        }
+    }
+
     abstract static class Metric {
         final String name;
 
@@ -212,7 +212,7 @@ static class LastValue extends Metric {
         @Override
         void collect(Number value) {
             if (lastValue == null) {
-                lastValue = value.doubleValue();
+                lastValue = value;
             }
         }
 
@@ -236,6 +236,17 @@ static class CounterMetricFieldProducer extends MetricFieldProducer {
             super(name, Collections.singletonList(new LastValue()));
         }
 
+        @Override
+        public void collect(FormattedDocValues docValues, int docId) throws IOException {
+            // Counter producers only collect the last_value. Since documents are
+            // collected by descending timestamp order, the producer should only
+            // process the first value for every tsid. So, it will only collect the
+            // field if no value has been set before.
+            if (isEmpty()) {
+                super.collect(docValues, docId);
+            }
+        }
+
         public Object value() {
             assert metrics().size() == 1 : "Single value producers must have only one metric";
             return metrics().iterator().next().get();
@@ -275,80 +286,4 @@ public void write(XContentBuilder builder) throws IOException {
             }
         }
     }
-
-    static class AggregateMetricFieldProducer extends MetricFieldProducer {
-
-        private Map<String, Metric> metricsByField = new LinkedHashMap<>();
-
-        AggregateMetricFieldProducer(String name) {
-            super(name, Collections.emptyList());
-        }
-
-        public void addMetric(String field, Metric metric) {
-            metricsByField.put(field, metric);
-        }
-
-        @Override
-        public void collect(String field, Number value) {
-            metricsByField.get(field).collect(value);
-            isEmpty = false;
-        }
-
-        @Override
-        public void write(XContentBuilder builder) throws IOException {
-            if (isEmpty() == false) {
-                builder.startObject(name());
-                for (MetricFieldProducer.Metric metric : metrics()) {
-                    if (metric.get() != null) {
-                        builder.field(metric.name(), metric.get());
-                    }
-                }
-                builder.endObject();
-            }
-        }
-
-        @Override
-        public Collection<Metric> metrics() {
-            return metricsByField.values();
-        }
-    }
-
-    /**
-     * Create a collection of metric field producers based on the metric_type mapping parameter in the field
-     * mapping.
-     */
-    static Map<String, MetricFieldProducer> createMetricFieldProducers(SearchExecutionContext context, String[] metricFields) {
-        final Map<String, MetricFieldProducer> fields = new LinkedHashMap<>();
-        for (String field : metricFields) {
-            MappedFieldType fieldType = context.getFieldType(field);
-            assert fieldType != null : "Unknown field type for field: [" + field + "]";
-            assert fieldType.getMetricType() != null : "Unknown metric type for metric field: [" + field + "]";
-
-            if (fieldType instanceof AggregateDoubleMetricFieldMapper.AggregateDoubleMetricFieldType aggMetricFieldType) {
-                // If the field is an aggregate_metric_double field, we should use the correct subfields
-                // for each aggregation. This is a rollup-of-rollup case
-                AggregateMetricFieldProducer producer = new AggregateMetricFieldProducer(field);
-                for (var e : aggMetricFieldType.getMetricFields().entrySet()) {
-                    AggregateDoubleMetricFieldMapper.Metric metric = e.getKey();
-                    NumberFieldMapper.NumberFieldType metricSubField = e.getValue();
-                    Metric metricOperation = switch (metric) {
-                        case max -> new Max();
-                        case min -> new Min();
-                        case sum -> new Sum();
-                        // To compute value_count summary, we must sum all field values
-                        case value_count -> new Sum(AggregateDoubleMetricFieldMapper.Metric.value_count.name());
-                    };
-                    producer.addMetric(metricSubField.name(), metricOperation);
-                    fields.put(metricSubField.name(), producer);
-                }
-            } else {
-                MetricFieldProducer producer = switch (fieldType.getMetricType()) {
-                    case gauge -> new GaugeMetricFieldProducer(field);
-                    case counter -> new CounterMetricFieldProducer(field);
-                };
-                fields.put(field, producer);
-            }
-        }
-        return Collections.unmodifiableMap(fields);
-    }
 }
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/RollupFieldSerializer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/RollupFieldSerializer.java
new file mode 100644
index 000000000000..f4e6615ec337
--- /dev/null
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/RollupFieldSerializer.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.downsample;
+
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+public interface RollupFieldSerializer {
+
+    /**
+     * Serialize the downsampled value of the field.
+     */
+    void write(XContentBuilder builder) throws IOException;
+}
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/RollupShardIndexer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/RollupShardIndexer.java
index 479e0e9cddc1..14cac197618c 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/RollupShardIndexer.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/RollupShardIndexer.java
@@ -24,9 +24,9 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.util.ArrayUtils;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.fielddata.FormattedDocValues;
@@ -43,24 +43,25 @@
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.bucket.DocCountProvider;
 import org.elasticsearch.search.aggregations.support.TimeSeriesIndexSearcher;
+import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.downsample.DownsampleConfig;
 import org.elasticsearch.xpack.core.downsample.RollupIndexerAction;
+import org.elasticsearch.xpack.core.rollup.action.RollupShardTask;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static java.util.stream.Collectors.groupingBy;
 import static org.elasticsearch.core.Strings.format;
 
 /**
@@ -89,12 +90,12 @@ class RollupShardIndexer {
     private final String[] dimensionFields;
     private final String[] metricFields;
     private final String[] labelFields;
-    private final Map<String, FieldValueFetcher> fieldValueFetchers;
-    private final AtomicLong numSent = new AtomicLong();
-    private final AtomicLong numIndexed = new AtomicLong();
-    private final AtomicLong numFailed = new AtomicLong();
+    private final List<FieldValueFetcher> fieldValueFetchers;
+    private final RollupShardTask task;
+    private volatile boolean abort = false;
 
     RollupShardIndexer(
+        RollupShardTask task,
         Client client,
         IndexService indexService,
         ShardId shardId,
@@ -104,6 +105,7 @@ class RollupShardIndexer {
         String[] metricFields,
         String[] labelFields
     ) {
+        this.task = task;
         this.client = client;
         this.indexShard = indexService.getShard(shardId.id());
         this.config = config;
@@ -111,7 +113,6 @@ class RollupShardIndexer {
         this.dimensionFields = dimensionFields;
         this.metricFields = metricFields;
         this.labelFields = labelFields;
-
         this.searcher = indexShard.acquireSearcher("downsampling");
         Closeable toClose = searcher;
         try {
@@ -126,7 +127,11 @@ class RollupShardIndexer {
             this.timestampField = searchExecutionContext.getFieldType(DataStreamTimestampFieldMapper.DEFAULT_PATH);
             this.timestampFormat = timestampField.docValueFormat(null, null);
             this.rounding = config.createRounding();
-            this.fieldValueFetchers = FieldValueFetcher.create(searchExecutionContext, ArrayUtils.concat(metricFields, labelFields));
+
+            List<FieldValueFetcher> fetchers = new ArrayList<>(metricFields.length + labelFields.length);
+            fetchers.addAll(FieldValueFetcher.create(searchExecutionContext, metricFields));
+            fetchers.addAll(FieldValueFetcher.create(searchExecutionContext, labelFields));
+            this.fieldValueFetchers = Collections.unmodifiableList(fetchers);
             toClose = null;
         } finally {
             IOUtils.closeWhileHandlingException(toClose);
@@ -137,8 +142,7 @@ public RollupIndexerAction.ShardRollupResponse execute() throws IOException {
         long startTime = System.currentTimeMillis();
         BulkProcessor bulkProcessor = createBulkProcessor();
         try (searcher; bulkProcessor) {
-            // TODO: add cancellations
-            final TimeSeriesIndexSearcher timeSeriesSearcher = new TimeSeriesIndexSearcher(searcher, List.of());
+            final TimeSeriesIndexSearcher timeSeriesSearcher = new TimeSeriesIndexSearcher(searcher, List.of(() -> { checkCancelled(); }));
             TimeSeriesBucketCollector bucketCollector = new TimeSeriesBucketCollector(bulkProcessor);
             bucketCollector.preCollection();
             timeSeriesSearcher.search(new MatchAllDocsQuery(), bucketCollector);
@@ -146,41 +150,60 @@ public RollupIndexerAction.ShardRollupResponse execute() throws IOException {
         }
 
         logger.info(
-            "Shard [{}] successfully sent [{}], indexed [{}], failed [{}], took [{}]",
+            "Shard [{}] successfully sent [{}], received source doc [{}], indexed rollup doc [{}], failed [{}], took [{}]",
             indexShard.shardId(),
-            numSent.get(),
-            numIndexed.get(),
-            numFailed.get(),
+            task.getNumReceived(),
+            task.getNumSent(),
+            task.getNumIndexed(),
+            task.getNumFailed(),
             TimeValue.timeValueMillis(System.currentTimeMillis() - startTime)
         );
 
-        if (numIndexed.get() != numSent.get()) {
+        if (task.getNumIndexed() != task.getNumSent()) {
             throw new ElasticsearchException(
                 "Shard ["
                     + indexShard.shardId()
                     + "] failed to index all rollup documents. Sent ["
-                    + numSent.get()
+                    + task.getNumSent()
                     + "], indexed ["
-                    + numIndexed.get()
+                    + task.getNumIndexed()
                     + "]."
             );
         }
-        return new RollupIndexerAction.ShardRollupResponse(indexShard.shardId(), numIndexed.get());
+
+        return new RollupIndexerAction.ShardRollupResponse(indexShard.shardId(), task.getNumIndexed());
+    }
+
+    private void checkCancelled() {
+        if (task.isCancelled() || abort) {
+            logger.warn(
+                "Shard [{}] rollup abort, sent [{}], indexed [{}], failed[{}]",
+                indexShard.shardId(),
+                task.getNumSent(),
+                task.getNumIndexed(),
+                task.getNumFailed()
+            );
+            throw new TaskCancelledException(format("Shard %s rollup cancelled", indexShard.shardId()));
+        }
     }
 
     private BulkProcessor createBulkProcessor() {
         final BulkProcessor.Listener listener = new BulkProcessor.Listener() {
             @Override
             public void beforeBulk(long executionId, BulkRequest request) {
-                numSent.addAndGet(request.numberOfActions());
+                task.addNumSent(request.numberOfActions());
             }
 
             @Override
             public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
-                numIndexed.addAndGet(request.numberOfActions());
+                task.addNumIndexed(request.numberOfActions());
                 if (response.hasFailures()) {
-                    Map<String, String> failures = Arrays.stream(response.getItems())
+                    List<BulkItemResponse> failedItems = Arrays.stream(response.getItems())
                         .filter(BulkItemResponse::isFailed)
+                        .collect(Collectors.toList());
+                    task.addNumFailed(failedItems.size());
+
+                    Map<String, String> failures = failedItems.stream()
                         .collect(
                             Collectors.toMap(
                                 BulkItemResponse::getId,
@@ -188,8 +211,10 @@ public void afterBulk(long executionId, BulkRequest request, BulkResponse respon
                                 (msg1, msg2) -> Objects.equals(msg1, msg2) ? msg1 : msg1 + "," + msg2
                             )
                         );
-                    numFailed.addAndGet(failures.size());
                     logger.error("Shard [{}] failed to populate rollup index. Failures: [{}]", indexShard.shardId(), failures);
+
+                    // cancel rollup task
+                    abort = true;
                 }
             }
 
@@ -197,8 +222,11 @@ public void afterBulk(long executionId, BulkRequest request, BulkResponse respon
             public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
                 if (failure != null) {
                     long items = request.numberOfActions();
-                    numFailed.addAndGet(items);
+                    task.addNumFailed(items);
                     logger.error(() -> format("Shard [%s] failed to populate rollup index.", indexShard.shardId()), failure);
+
+                    // cancel rollup task
+                    abort = true;
                 }
             }
         };
@@ -214,14 +242,18 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure)
 
     private class TimeSeriesBucketCollector extends BucketCollector {
         private final BulkProcessor bulkProcessor;
+        private final RollupBucketBuilder rollupBucketBuilder;
         private long docsProcessed;
         private long bucketsCreated;
-        private final RollupBucketBuilder rollupBucketBuilder = new RollupBucketBuilder();
         long lastTimestamp = Long.MAX_VALUE;
         long lastHistoTimestamp = Long.MAX_VALUE;
 
         TimeSeriesBucketCollector(BulkProcessor bulkProcessor) {
             this.bulkProcessor = bulkProcessor;
+            List<AbstractRollupFieldProducer> rollupFieldProducers = fieldValueFetchers.stream()
+                .map(FieldValueFetcher::rollupFieldProducer)
+                .toList();
+            this.rollupBucketBuilder = new RollupBucketBuilder(rollupFieldProducers);
         }
 
         @Override
@@ -229,11 +261,16 @@ public LeafBucketCollector getLeafCollector(final AggregationExecutionContext ag
             final LeafReaderContext ctx = aggCtx.getLeafReaderContext();
             final DocCountProvider docCountProvider = new DocCountProvider();
             docCountProvider.setLeafReaderContext(ctx);
-            final Map<String, FormattedDocValues> docValuesFetchers = FieldValueFetcher.docValuesFetchers(ctx, fieldValueFetchers);
+
+            // For each field, return a tuple with the rollup field producer and the field value leaf
+            final List<Tuple<AbstractRollupFieldProducer, FormattedDocValues>> fieldValueTuples = fieldValueFetchers.stream()
+                .map(fetcher -> Tuple.tuple(fetcher.rollupFieldProducer(), fetcher.getLeaf(ctx)))
+                .toList();
 
             return new LeafBucketCollector() {
                 @Override
                 public void collect(int docId, long owningBucketOrd) throws IOException {
+                    task.addNumReceived(1);
                     final BytesRef tsid = aggCtx.getTsid();
                     assert tsid != null : "Document without [" + TimeSeriesIdFieldMapper.NAME + "] field was found.";
                     final int tsidOrd = aggCtx.getTsidOrd();
@@ -290,29 +327,16 @@ public void collect(int docId, long owningBucketOrd) throws IOException {
                         } else {
                             rollupBucketBuilder.resetTimestamp(lastHistoTimestamp);
                         }
-
                         bucketsCreated++;
                     }
 
                     final int docCount = docCountProvider.getDocCount(docId);
                     rollupBucketBuilder.collectDocCount(docCount);
-                    for (Map.Entry<String, FormattedDocValues> e : docValuesFetchers.entrySet()) {
-                        final String fieldName = e.getKey();
-                        final FormattedDocValues leafField = e.getValue();
-
-                        if (leafField.advanceExact(docId)) {
-                            rollupBucketBuilder.collect(fieldName, leafField.docValueCount(), docValueCount -> {
-                                final Object[] values = new Object[docValueCount];
-                                for (int i = 0; i < docValueCount; ++i) {
-                                    try {
-                                        values[i] = leafField.nextValue();
-                                    } catch (IOException ex) {
-                                        throw new ElasticsearchException("Failed to read values for field [" + fieldName + "]");
-                                    }
-                                }
-                                return values;
-                            });
-                        }
+                    // Iterate over all field values and collect the doc_values for this docId
+                    for (Tuple<AbstractRollupFieldProducer, FormattedDocValues> tuple : fieldValueTuples) {
+                        AbstractRollupFieldProducer rollupFieldProducer = tuple.v1();
+                        FormattedDocValues docValues = tuple.v2();
+                        rollupFieldProducer.collect(docValues, docId);
                     }
                     docsProcessed++;
                 }
@@ -330,7 +354,8 @@ private void indexBucket(XContentBuilder doc) {
 
         @Override
         public void preCollection() throws IOException {
-            // no-op
+            // check cancel when start running
+            checkCancelled();
         }
 
         @Override
@@ -341,6 +366,10 @@ public void postCollection() throws IOException {
                 indexBucket(doc);
             }
             bulkProcessor.flush();
+
+            // check cancel after the flush all data
+            checkCancelled();
+
             logger.info("Shard {} processed [{}] docs, created [{}] rollup buckets", indexShard.shardId(), docsProcessed, bucketsCreated);
         }
 
@@ -355,12 +384,10 @@ private class RollupBucketBuilder {
         private int tsidOrd = -1;
         private long timestamp;
         private int docCount;
-        private final Map<String, MetricFieldProducer> metricFieldProducers;
-        private final Map<String, LabelFieldProducer> labelFieldProducers;
+        private final List<AbstractRollupFieldProducer> rollupFieldProducers;
 
-        RollupBucketBuilder() {
-            this.metricFieldProducers = MetricFieldProducer.createMetricFieldProducers(searchExecutionContext, metricFields);
-            this.labelFieldProducers = LabelFieldProducer.createLabelFieldProducers(searchExecutionContext, labelFields);
+        RollupBucketBuilder(List<AbstractRollupFieldProducer> rollupFieldProducers) {
+            this.rollupFieldProducers = rollupFieldProducers;
         }
 
         /**
@@ -378,8 +405,7 @@ public RollupBucketBuilder resetTsid(BytesRef tsid, int tsidOrd, long timestamp)
         public RollupBucketBuilder resetTimestamp(long timestamp) {
             this.timestamp = timestamp;
             this.docCount = 0;
-            this.metricFieldProducers.values().forEach(MetricFieldProducer::reset);
-            this.labelFieldProducers.values().forEach(LabelFieldProducer::reset);
+            this.rollupFieldProducers.forEach(AbstractRollupFieldProducer::reset);
             if (logger.isTraceEnabled()) {
                 logger.trace(
                     "New bucket for _tsid: [{}], @timestamp: [{}]",
@@ -390,46 +416,6 @@ public RollupBucketBuilder resetTimestamp(long timestamp) {
             return this;
         }
 
-        public void collect(final String field, int docValueCount, final Function<Integer, Object[]> fieldValues) {
-            final Object[] values = fieldValues.apply(docValueCount);
-            if (metricFieldProducers.containsKey(field)) {
-                // TODO: missing support for array metrics
-                collectMetric(field, values);
-            } else if (labelFieldProducers.containsKey(field)) {
-                if (values.length == 1) {
-                    collectLabel(field, values[0]);
-                } else {
-                    collectLabel(field, values);
-                }
-            } else {
-                throw new IllegalArgumentException(
-                    "Field '"
-                        + field
-                        + "' is not a label nor a metric, existing labels: [ "
-                        + String.join(",", labelFieldProducers.keySet())
-                        + "], existing metrics: ["
-                        + String.join(", ", metricFieldProducers.keySet())
-                        + "]"
-                );
-            }
-        }
-
-        private void collectLabel(final String field, final Object value) {
-            labelFieldProducers.get(field).collect(field, value);
-        }
-
-        private void collectMetric(final String field, final Object[] values) {
-            for (Object value : values) {
-                if (value instanceof Number number) {
-                    metricFieldProducers.get(field).collect(field, number);
-                } else {
-                    throw new IllegalArgumentException(
-                        "Expected numeric value for field '" + field + "' but got non numeric value: '" + value + "'"
-                    );
-                }
-            }
-        }
-
         public void collectDocCount(int docCount) {
             this.docCount += docCount;
         }
@@ -452,14 +438,28 @@ public XContentBuilder buildRollupDocument() throws IOException {
                 builder.field(e.getKey(), e.getValue());
             }
 
-            // Serialize all metric fields
-            for (var producer : new HashSet<>(metricFieldProducers.values())) {
-                producer.write(builder);
-            }
+            /*
+             * The rollup field producers for aggregate_metric_double all share the same name (this is
+             * the name they will be serialized in the target index). We group all field producers by
+             * name. If grouping yields multiple rollup field producers, we delegate serialization to
+             * the AggregateMetricFieldSerializer class.
+             */
+            List<RollupFieldSerializer> groupedProducers = rollupFieldProducers.stream()
+                .collect(groupingBy(AbstractRollupFieldProducer::name))
+                .entrySet()
+                .stream()
+                .map(e -> {
+                    if (e.getValue().size() == 1) {
+                        return e.getValue().get(0);
+                    } else {
+                        return new AggregateMetricFieldSerializer(e.getKey(), e.getValue());
+                    }
+                })
+                .toList();
 
-            // Serialize all label fields
-            for (var producer : new HashSet<>(labelFieldProducers.values())) {
-                producer.write(builder);
+            // Serialize fields
+            for (RollupFieldSerializer fieldProducer : groupedProducers) {
+                fieldProducer.write(builder);
             }
 
             builder.endObject();
@@ -485,5 +485,6 @@ public int docCount() {
         public boolean isEmpty() {
             return tsid() == null || timestamp() == 0 || docCount() == 0;
         }
+
     }
 }
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TimeseriesFieldTypeHelper.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TimeseriesFieldTypeHelper.java
index 06bd2260198b..691279187e1a 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TimeseriesFieldTypeHelper.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TimeseriesFieldTypeHelper.java
@@ -42,7 +42,7 @@ public boolean isTimeSeriesLabel(final String field, final Map<String, ?> unused
     public boolean isTimeSeriesMetric(final String unused, final Map<String, ?> fieldMapping) {
         final String metricType = (String) fieldMapping.get(TIME_SERIES_METRIC_PARAM);
         return metricType != null
-            && List.of(TimeSeriesParams.MetricType.values()).contains(TimeSeriesParams.MetricType.valueOf(metricType));
+            && List.of(TimeSeriesParams.MetricType.values()).contains(TimeSeriesParams.MetricType.fromString(metricType));
     }
 
     public boolean isTimeSeriesDimension(final String unused, final Map<String, ?> fieldMapping) {
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportRollupAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportRollupAction.java
index a793cddc8e42..1c52ef19dbbf 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportRollupAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportRollupAction.java
@@ -477,10 +477,10 @@ private static void addTimestampField(final DownsampleConfig config, final XCont
 
     private static void addMetricFieldMapping(final XContentBuilder builder, final String field, final Map<String, ?> fieldProperties)
         throws IOException {
-        final TimeSeriesParams.MetricType metricType = TimeSeriesParams.MetricType.valueOf(
+        final TimeSeriesParams.MetricType metricType = TimeSeriesParams.MetricType.fromString(
             fieldProperties.get(TIME_SERIES_METRIC_PARAM).toString()
         );
-        if (TimeSeriesParams.MetricType.counter.equals(metricType)) {
+        if (metricType == TimeSeriesParams.MetricType.COUNTER) {
             // For counters, we keep the same field type, because they store
             // only one value (the last value of the counter)
             builder.startObject(field).field("type", fieldProperties.get("type")).field(TIME_SERIES_METRIC_PARAM, metricType).endObject();
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportRollupIndexerAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportRollupIndexerAction.java
index e7ccf37edde2..be61f9991682 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportRollupIndexerAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportRollupIndexerAction.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.downsample.RollupIndexerAction;
+import org.elasticsearch.xpack.core.rollup.action.RollupShardTask;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -125,6 +126,7 @@ protected RollupIndexerAction.ShardRollupResponse shardOperation(RollupIndexerAc
         throws IOException {
         IndexService indexService = indicesService.indexService(request.shardId().getIndex());
         RollupShardIndexer indexer = new RollupShardIndexer(
+            (RollupShardTask) task,
             client,
             indexService,
             request.shardId(),
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java
index 0fb6be439b53..11c0a33c9298 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -114,7 +114,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver expressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         return emptyList();
     }
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java
index f10679117a2e..d27bd1a29d3b 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java
@@ -39,12 +39,14 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper;
 import org.elasticsearch.index.mapper.TimeSeriesParams;
+import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchHit;
@@ -65,6 +67,9 @@
 import org.elasticsearch.search.aggregations.metrics.ValueCountAggregationBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.tasks.TaskCancelHelper;
+import org.elasticsearch.tasks.TaskCancelledException;
+import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
@@ -75,6 +80,7 @@
 import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
 import org.elasticsearch.xpack.core.ilm.RolloverAction;
 import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
+import org.elasticsearch.xpack.core.rollup.action.RollupShardTask;
 import org.elasticsearch.xpack.ilm.IndexLifecycle;
 import org.elasticsearch.xpack.rollup.Rollup;
 import org.junit.Before;
@@ -96,11 +102,13 @@
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
+import static java.util.Collections.emptyMap;
 import static org.elasticsearch.index.mapper.TimeSeriesParams.TIME_SERIES_METRIC_PARAM;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 
 public class DownsampleActionSingleNodeTests extends ESSingleNodeTestCase {
 
@@ -206,7 +214,7 @@ public void setup() throws IOException {
         mapping.startObject(FIELD_LABEL_DOUBLE).field("type", "double").endObject();
         mapping.startObject(FIELD_LABEL_INTEGER).field("type", "integer").endObject();
         mapping.startObject(FIELD_LABEL_KEYWORD).field("type", "keyword").endObject();
-        mapping.startObject(FIELD_LABEL_TEXT).field("type", "text").endObject();
+        mapping.startObject(FIELD_LABEL_TEXT).field("type", "text").field("store", "true").endObject();
         mapping.startObject(FIELD_LABEL_BOOLEAN).field("type", "boolean").endObject();
         mapping.startObject(FIELD_LABEL_IPv4_ADDRESS).field("type", "ip").endObject();
         mapping.startObject(FIELD_LABEL_IPv6_ADDRESS).field("type", "ip").endObject();
@@ -571,6 +579,78 @@ public void testRollupDatastream() throws Exception {
         assertFalse(indices.stream().filter(i -> i.getName().equals(sourceIndex)).toList().isEmpty());
     }
 
+    public void testCancelRollupIndexer() throws IOException {
+        // create rollup config and index documents into source index
+        DownsampleConfig config = new DownsampleConfig(randomInterval());
+        SourceSupplier sourceSupplier = () -> XContentFactory.jsonBuilder()
+            .startObject()
+            .field(FIELD_TIMESTAMP, randomDateForInterval(config.getInterval()))
+            .field(FIELD_DIMENSION_1, randomAlphaOfLength(1))
+            .field(FIELD_NUMERIC_1, randomDouble())
+            .endObject();
+        bulkIndex(sourceSupplier);
+        prepareSourceIndex(sourceIndex);
+
+        IndicesService indexServices = getInstanceFromNode(IndicesService.class);
+        Index srcIndex = resolveIndex(sourceIndex);
+        IndexService indexService = indexServices.indexServiceSafe(srcIndex);
+        int shardNum = randomIntBetween(0, numOfShards - 1);
+        IndexShard shard = indexService.getShard(shardNum);
+        RollupShardTask task = new RollupShardTask(
+            randomLong(),
+            "rollup",
+            "action",
+            TaskId.EMPTY_TASK_ID,
+            rollupIndex,
+            config,
+            emptyMap(),
+            shard.shardId()
+        );
+        TaskCancelHelper.cancel(task, "test cancel");
+
+        // re-use source index as temp index for test
+        RollupShardIndexer indexer = new RollupShardIndexer(
+            task,
+            client(),
+            indexService,
+            shard.shardId(),
+            rollupIndex,
+            config,
+            new String[] { FIELD_DIMENSION_1, FIELD_DIMENSION_2 },
+            new String[] { FIELD_NUMERIC_1, FIELD_NUMERIC_2 },
+            new String[] {}
+        );
+
+        TaskCancelledException exception = expectThrows(TaskCancelledException.class, () -> indexer.execute());
+        assertThat(exception.getMessage(), equalTo("Shard [" + sourceIndex + "][" + shardNum + "] rollup cancelled"));
+    }
+
+    public void testRollupBulkFailed() throws IOException {
+        // create rollup config and index documents into source index
+        DownsampleConfig config = new DownsampleConfig(randomInterval());
+        SourceSupplier sourceSupplier = () -> XContentFactory.jsonBuilder()
+            .startObject()
+            .field(FIELD_TIMESTAMP, randomDateForInterval(config.getInterval()))
+            .field(FIELD_DIMENSION_1, randomAlphaOfLength(1))
+            .field(FIELD_NUMERIC_1, randomDouble())
+            .endObject();
+        bulkIndex(sourceSupplier);
+        prepareSourceIndex(sourceIndex);
+
+        // block rollup index
+        assertAcked(
+            client().admin()
+                .indices()
+                .preparePutTemplate(rollupIndex)
+                .setPatterns(List.of(rollupIndex))
+                .setSettings(Settings.builder().put("index.blocks.write", "true").build())
+                .get()
+        );
+
+        ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> rollup(sourceIndex, rollupIndex, config));
+        assertThat(exception.getMessage(), equalTo("Unable to rollup index [" + sourceIndex + "]"));
+    }
+
     private DateHistogramInterval randomInterval() {
         return ConfigTestHelpers.randomInterval();
     }
@@ -666,7 +746,7 @@ private void assertRollupIndex(String sourceIndex, String rollupIndex, Downsampl
         Map<String, String> labelFields = new HashMap<>();
         MappingVisitor.visitMapping(sourceIndexMappings, (field, fieldMapping) -> {
             if (helper.isTimeSeriesMetric(field, fieldMapping)) {
-                metricFields.put(field, TimeSeriesParams.MetricType.valueOf(fieldMapping.get(TIME_SERIES_METRIC_PARAM).toString()));
+                metricFields.put(field, TimeSeriesParams.MetricType.fromString(fieldMapping.get(TIME_SERIES_METRIC_PARAM).toString()));
             } else if (helper.isTimeSeriesLabel(field, fieldMapping)) {
                 labelFields.put(field, fieldMapping.get("type").toString());
             }
@@ -831,8 +911,8 @@ private void assertFieldMappings(
 
         metricFields.forEach((field, metricType) -> {
             switch (metricType) {
-                case counter -> assertEquals("double", mappings.get(field).get("type"));
-                case gauge -> assertEquals("aggregate_metric_double", mappings.get(field).get("type"));
+                case COUNTER -> assertEquals("double", mappings.get(field).get("type"));
+                case GAUGE -> assertEquals("aggregate_metric_double", mappings.get(field).get("type"));
                 default -> fail("Unsupported field type");
             }
             assertEquals(metricType.toString(), mappings.get(field).get("time_series_metric"));
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/LabelFieldProducerTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/LabelFieldProducerTests.java
index 641cbe228e02..469e00f7af9a 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/LabelFieldProducerTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/LabelFieldProducerTests.java
@@ -7,8 +7,11 @@
 
 package org.elasticsearch.xpack.downsample;
 
+import org.elasticsearch.index.fielddata.FormattedDocValues;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 
+import java.io.IOException;
+
 public class LabelFieldProducerTests extends AggregatorTestCase {
 
     public void testLastValueKeywordLabel() {
@@ -61,16 +64,33 @@ public void testLastValueBooleanLabel() {
         assertNull(label.get());
     }
 
-    public void testLabelFieldProducer() {
+    public void testLabelFieldProducer() throws IOException {
         final LabelFieldProducer producer = new LabelFieldProducer.LabelLastValueFieldProducer("dummy");
         assertTrue(producer.isEmpty());
         assertEquals("dummy", producer.name());
-        assertEquals("last_value", producer.label().name);
-        producer.collect("dummy", "aaaa");
+        assertEquals("last_value", producer.label().name());
+        FormattedDocValues docValues = new FormattedDocValues() {
+            @Override
+            public boolean advanceExact(int docId) {
+                return true;
+            }
+
+            @Override
+            public int docValueCount() {
+                return 1;
+            }
+
+            @Override
+            public Object nextValue() {
+                return "aaaa";
+            }
+        };
+        producer.collect(docValues, 1);
+        // producer.collect("dummy", "aaaa");
         assertFalse(producer.isEmpty());
-        assertEquals("aaaa", producer.value());
+        assertEquals("aaaa", producer.label().get());
         producer.reset();
         assertTrue(producer.isEmpty());
-        assertNull(producer.value());
+        assertNull(producer.label().get());
     }
 }
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/MetricFieldProducerTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/MetricFieldProducerTests.java
index 0c93f010ec99..34ccbc5c7202 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/MetricFieldProducerTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/MetricFieldProducerTests.java
@@ -8,19 +8,11 @@
 package org.elasticsearch.xpack.downsample;
 
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.NumberFieldMapper;
-import org.elasticsearch.index.mapper.TimeSeriesParams;
-import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.json.JsonXContent;
 
 import java.io.IOException;
-import java.util.Map;
-
-import static java.util.Collections.emptyMap;
 
 public class MetricFieldProducerTests extends AggregatorTestCase {
 
@@ -69,7 +61,7 @@ public void testSummationAccuracy() {
         for (int i = 0; i < values.length; i++) {
             metric.collect(values[i]);
         }
-        assertEquals(metric.get().doubleValue(), 15.3, Double.MIN_NORMAL);
+        assertEquals(15.3, metric.get().doubleValue(), Double.MIN_NORMAL);
 
         // Summing up an array which contains NaN and infinities and expect a result same as naive summation
         metric.reset();
@@ -82,7 +74,7 @@ public void testSummationAccuracy() {
             sum += d;
             metric.collect(d);
         }
-        assertEquals(metric.get().doubleValue(), sum, 1e-10);
+        assertEquals(sum, metric.get().doubleValue(), 1e-10);
 
         // Summing up some big double values and expect infinity result
         metric.reset();
@@ -90,13 +82,13 @@ public void testSummationAccuracy() {
         for (int i = 0; i < n; i++) {
             metric.collect(Double.MAX_VALUE);
         }
-        assertEquals(metric.get().doubleValue(), Double.POSITIVE_INFINITY, 0d);
+        assertEquals(Double.POSITIVE_INFINITY, metric.get().doubleValue(), 0d);
 
         metric.reset();
         for (int i = 0; i < n; i++) {
             metric.collect(-Double.MAX_VALUE);
         }
-        assertEquals(metric.get().doubleValue(), Double.NEGATIVE_INFINITY, 0d);
+        assertEquals(Double.NEGATIVE_INFINITY, metric.get().doubleValue(), 0d);
     }
 
     public void testValueCountMetric() {
@@ -116,7 +108,7 @@ public void testLastValueMetric() {
         metric.collect(40);
         metric.collect(30);
         metric.collect(20);
-        assertEquals(40.0, metric.get());
+        assertEquals(40, metric.get());
         metric.reset();
         assertNull(metric.get());
     }
@@ -125,9 +117,9 @@ public void testCounterMetricFieldProducer() throws IOException {
         final String field = "field";
         var producer = new MetricFieldProducer.CounterMetricFieldProducer(field);
         assertTrue(producer.isEmpty());
-        producer.collect(field, 55.0);
-        producer.collect(field, 12.2);
-        producer.collect(field, 5.5);
+        producer.collect(55.0);
+        producer.collect(12.2);
+        producer.collect(5.5);
 
         assertFalse(producer.isEmpty());
         Object o = producer.value();
@@ -144,9 +136,9 @@ public void testGaugeMetricFieldProducer() throws IOException {
         final String field = "field";
         MetricFieldProducer producer = new MetricFieldProducer.GaugeMetricFieldProducer(field);
         assertTrue(producer.isEmpty());
-        producer.collect(field, 55.0);
-        producer.collect(field, 12.2);
-        producer.collect(field, 5.5);
+        producer.collect(55.0);
+        producer.collect(12.2);
+        producer.collect(5.5);
 
         assertFalse(producer.isEmpty());
 
@@ -157,72 +149,4 @@ public void testGaugeMetricFieldProducer() throws IOException {
 
         assertEquals(field, producer.name());
     }
-
-    public void testBuildMetricProducers() {
-        final Map<String, MappedFieldType> provideMappedFieldType = Map.of(
-            "gauge_field",
-            new NumberFieldMapper.NumberFieldType(
-                "gauge_field",
-                NumberFieldMapper.NumberType.DOUBLE,
-                true,
-                true,
-                true,
-                true,
-                null,
-                emptyMap(),
-                null,
-                false,
-                TimeSeriesParams.MetricType.gauge
-            ),
-            "counter_field",
-            new NumberFieldMapper.NumberFieldType(
-                "counter_field",
-                NumberFieldMapper.NumberType.DOUBLE,
-                true,
-                true,
-                true,
-                true,
-                null,
-                emptyMap(),
-                null,
-                false,
-                TimeSeriesParams.MetricType.counter
-            )
-        );
-
-        IndexSettings settings = createIndexSettings();
-        SearchExecutionContext searchExecutionContext = new SearchExecutionContext(
-            0,
-            0,
-            settings,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            () -> 0L,
-            null,
-            null,
-            () -> true,
-            null,
-            emptyMap()
-        ) {
-            @Override
-            public MappedFieldType getFieldType(String name) {
-                return provideMappedFieldType.get(name);
-            }
-        };
-
-        Map<String, MetricFieldProducer> producers = MetricFieldProducer.createMetricFieldProducers(
-            searchExecutionContext,
-            new String[] { "gauge_field", "counter_field" }
-        );
-        assertTrue(producers.get("gauge_field") instanceof MetricFieldProducer.GaugeMetricFieldProducer);
-        assertTrue(producers.get("counter_field") instanceof MetricFieldProducer.CounterMetricFieldProducer);
-    }
 }
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/DeleteJobActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/DeleteJobActionRequestTests.java
index b1cdd19e809f..d81d2ed2079b 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/DeleteJobActionRequestTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/DeleteJobActionRequestTests.java
@@ -17,6 +17,11 @@ protected Request createTestInstance() {
         return new Request(randomAlphaOfLengthBetween(1, 20));
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetJobsActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetJobsActionRequestTests.java
index e7a5ab17ac91..e31992597317 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetJobsActionRequestTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetJobsActionRequestTests.java
@@ -31,6 +31,11 @@ protected Request createTestInstance() {
         return new Request(randomAlphaOfLengthBetween(1, 20));
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java
index a2af119ebc62..2037be407696 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java
@@ -37,6 +37,11 @@ protected GetRollupCapsAction.Request createTestInstance() {
         return new GetRollupCapsAction.Request(randomAlphaOfLengthBetween(1, 20));
     }
 
+    @Override
+    protected GetRollupCapsAction.Request mutateInstance(GetRollupCapsAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<GetRollupCapsAction.Request> instanceReader() {
         return GetRollupCapsAction.Request::new;
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java
index c40037663804..7ca0d433f600 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java
@@ -36,6 +36,11 @@ protected GetRollupIndexCapsAction.Request createTestInstance() {
         return new GetRollupIndexCapsAction.Request(new String[] { randomAlphaOfLengthBetween(1, 20) });
     }
 
+    @Override
+    protected GetRollupIndexCapsAction.Request mutateInstance(GetRollupIndexCapsAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<GetRollupIndexCapsAction.Request> instanceReader() {
         return GetRollupIndexCapsAction.Request::new;
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobActionRequestTests.java
index 1313a064618c..aee82bc011c2 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobActionRequestTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobActionRequestTests.java
@@ -29,6 +29,11 @@ protected Request createTestInstance() {
         return new Request(ConfigTestHelpers.randomRollupJobConfig(random(), jobId));
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/StartJobActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/StartJobActionRequestTests.java
index 578b14905752..57c9e9dce6a6 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/StartJobActionRequestTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/StartJobActionRequestTests.java
@@ -17,6 +17,11 @@ protected Request createTestInstance() {
         return new Request(randomAlphaOfLengthBetween(1, 20));
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/StopJobActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/StopJobActionRequestTests.java
index 01b0618e2b3a..a018afa505dd 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/StopJobActionRequestTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/StopJobActionRequestTests.java
@@ -17,6 +17,11 @@ protected Request createTestInstance() {
         return new Request(randomAlphaOfLengthBetween(1, 20));
     }
 
+    @Override
+    protected Request mutateInstance(Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java
index 19b3da0160b7..6bd33fe1edc1 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java
@@ -741,7 +741,8 @@ private Map<String, MappedFieldType> createFieldTypes(RollupJobConfig job) {
                     ScriptCompiler.NONE,
                     false,
                     false,
-                    Version.CURRENT
+                    Version.CURRENT,
+                    null
                 ).build(MapperBuilderContext.root(false)).fieldType();
                 fieldTypes.put(ft.name(), ft);
             }
@@ -763,7 +764,8 @@ private Map<String, MappedFieldType> createFieldTypes(RollupJobConfig job) {
                     ScriptCompiler.NONE,
                     false,
                     false,
-                    Version.CURRENT
+                    Version.CURRENT,
+                    null
                 ).build(MapperBuilderContext.root(false)).fieldType();
                 fieldTypes.put(ft.name(), ft);
             }
diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java
index a3c91376f495..62548b7c5e38 100644
--- a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java
+++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java
@@ -13,7 +13,7 @@
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.NumericUtils;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -220,7 +220,7 @@ private PinnedQueryBuilder(QueryBuilder organicQuery, List<String> ids, List<Ite
      */
     public PinnedQueryBuilder(StreamInput in) throws IOException {
         super(in);
-        if (in.getVersion().before(Version.V_7_15_0)) {
+        if (in.getTransportVersion().before(TransportVersion.V_7_15_0)) {
             ids = in.readStringList();
             docs = null;
         } else {
@@ -232,7 +232,7 @@ public PinnedQueryBuilder(StreamInput in) throws IOException {
 
     @Override
     protected void doWriteTo(StreamOutput out) throws IOException {
-        if (out.getVersion().before(Version.V_7_15_0)) {
+        if (out.getTransportVersion().before(TransportVersion.V_7_15_0)) {
             out.writeStringCollection(this.ids);
         } else {
             out.writeOptionalStringCollection(this.ids);
@@ -390,7 +390,7 @@ protected boolean doEquals(PinnedQueryBuilder other) {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_4_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_4_0;
     }
 }
diff --git a/x-pack/plugin/searchable-snapshots/build.gradle b/x-pack/plugin/searchable-snapshots/build.gradle
index 357dc5efda3e..c79f13744a95 100644
--- a/x-pack/plugin/searchable-snapshots/build.gradle
+++ b/x-pack/plugin/searchable-snapshots/build.gradle
@@ -6,13 +6,14 @@ esplugin {
   name 'searchable-snapshots'
   description 'A plugin for the searchable snapshots functionality'
   classname 'org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots'
-  extendedPlugins = ['x-pack-core']
+  extendedPlugins = ['x-pack-core', 'blob-cache']
 }
 archivesBaseName = 'x-pack-searchable-snapshots'
 
 dependencies {
   compileOnly project(path: xpackModule('core'))
-  implementation project(path: 'preallocate')
+  compileOnly project(path: xpackModule('blob-cache'))
+  testImplementation(testArtifact(project(xpackModule('blob-cache'))))
   internalClusterTestImplementation(testArtifact(project(xpackModule('core'))))
   internalClusterTestImplementation(project(path: xpackModule('shutdown')))
   internalClusterTestImplementation(project(path: xpackModule('snapshot-based-recoveries')))
diff --git a/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle b/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle
index b4f60a7e09b0..73032e36679a 100644
--- a/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle
+++ b/x-pack/plugin/searchable-snapshots/qa/azure/build.gradle
@@ -1,7 +1,7 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 
 final Project fixture = project(':test:fixtures:azure-fixture')
diff --git a/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle b/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle
index 5c017b2e34eb..c9ce9c1e05a5 100644
--- a/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle
+++ b/x-pack/plugin/searchable-snapshots/qa/gcs/build.gradle
@@ -7,7 +7,7 @@ import java.security.KeyPairGenerator
 
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 
 final Project fixture = project(':test:fixtures:gcs-fixture')
diff --git a/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle b/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle
index a6ef39dc6fe6..a1d4bd121160 100644
--- a/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle
+++ b/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle
@@ -16,7 +16,7 @@ import java.nio.file.Paths
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 
 apply plugin: 'elasticsearch.test.fixtures'
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 apply plugin: 'elasticsearch.internal-available-ports'
 
diff --git a/x-pack/plugin/searchable-snapshots/qa/minio/build.gradle b/x-pack/plugin/searchable-snapshots/qa/minio/build.gradle
index d00b51c0c2da..860e42378dcd 100644
--- a/x-pack/plugin/searchable-snapshots/qa/minio/build.gradle
+++ b/x-pack/plugin/searchable-snapshots/qa/minio/build.gradle
@@ -1,6 +1,6 @@
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
 apply plugin: 'elasticsearch.rest-resources'
 
diff --git a/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle b/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle
index 75b62df35b55..a15f0b2b3c83 100644
--- a/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle
+++ b/x-pack/plugin/searchable-snapshots/qa/rest/build.gradle
@@ -1,6 +1,6 @@
-apply plugin: 'elasticsearch.internal-java-rest-test'
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 dependencies {
   javaRestTestImplementation(testArtifact(project(xpackModule('searchable-snapshots'))))
diff --git a/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle b/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle
index 2b359f88f2d6..01a2dc065ec6 100644
--- a/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle
+++ b/x-pack/plugin/searchable-snapshots/qa/s3/build.gradle
@@ -1,7 +1,7 @@
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 
 final Project fixture = project(':test:fixtures:s3-fixture')
diff --git a/x-pack/plugin/searchable-snapshots/qa/url/build.gradle b/x-pack/plugin/searchable-snapshots/qa/url/build.gradle
index 0faab0acfe61..160eb8d28cbc 100644
--- a/x-pack/plugin/searchable-snapshots/qa/url/build.gradle
+++ b/x-pack/plugin/searchable-snapshots/qa/url/build.gradle
@@ -1,6 +1,6 @@
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 
 final Project fixture = project(':test:fixtures:nginx-fixture')
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseFrozenSearchableSnapshotsIntegTestCase.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseFrozenSearchableSnapshotsIntegTestCase.java
index 5a00c63f3f3e..a314db8f94b4 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseFrozenSearchableSnapshotsIntegTestCase.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseFrozenSearchableSnapshotsIntegTestCase.java
@@ -7,12 +7,12 @@
 
 package org.elasticsearch.xpack.searchablesnapshots;
 
+import org.elasticsearch.blobcache.shared.SharedBlobCacheService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.RatioValue;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService;
 
 public abstract class BaseFrozenSearchableSnapshotsIntegTestCase extends BaseSearchableSnapshotsIntegTestCase {
     @Override
@@ -25,7 +25,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
         Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings));
         if (DiscoveryNode.canContainData(otherSettings)) {
             builder.put(
-                FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey(),
+                SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(),
                 rarely()
                     ? randomBoolean()
                         ? new ByteSizeValue(randomIntBetween(1, 10), ByteSizeUnit.KB).getStringRep()
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java
index 154a7ba5e798..b73be16c71c7 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java
@@ -11,6 +11,8 @@
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.blobcache.BlobCachePlugin;
+import org.elasticsearch.blobcache.shared.SharedBlobCacheService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -41,7 +43,6 @@
 import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest.Storage;
 import org.elasticsearch.xpack.searchablesnapshots.cache.blob.BlobStoreCacheService;
 import org.elasticsearch.xpack.searchablesnapshots.cache.full.CacheService;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService;
 import org.elasticsearch.xpack.snapshotbasedrecoveries.SnapshotBasedRecoveriesPlugin;
 import org.junit.After;
 
@@ -59,11 +60,11 @@
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Stream;
 
+import static org.elasticsearch.blobcache.shared.SharedBytes.pageAligned;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.license.LicenseService.SELF_GENERATED_LICENSE_TYPE;
 import static org.elasticsearch.test.NodeRoles.addRoles;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.xpack.searchablesnapshots.cache.shared.SharedBytes.pageAligned;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -80,6 +81,7 @@ protected boolean addMockInternalEngine() {
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
+        plugins.add(BlobCachePlugin.class);
         plugins.add(LocalStateSearchableSnapshots.class);
         plugins.add(LicensedSnapshotBasedRecoveriesPlugin.class);
         return Collections.unmodifiableList(plugins);
@@ -106,17 +108,17 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
             );
         }
         if (DiscoveryNode.canContainData(otherSettings) && randomBoolean()) {
-            builder.put(FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ZERO.getStringRep());
+            builder.put(SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ZERO.getStringRep());
         }
         builder.put(
-            FrozenCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(),
+            SharedBlobCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(),
             rarely()
                 ? pageAligned(new ByteSizeValue(randomIntBetween(4, 1024), ByteSizeUnit.KB))
                 : pageAligned(new ByteSizeValue(randomIntBetween(1, 10), ByteSizeUnit.MB))
         );
         if (randomBoolean()) {
             builder.put(
-                FrozenCacheService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(),
+                SharedBlobCacheService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(),
                 rarely()
                     ? pageAligned(new ByteSizeValue(randomIntBetween(4, 1024), ByteSizeUnit.KB))
                     : pageAligned(new ByteSizeValue(randomIntBetween(1, 10), ByteSizeUnit.MB))
@@ -124,7 +126,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
         }
         if (randomBoolean()) {
             builder.put(
-                FrozenCacheService.SHARED_CACHE_RECOVERY_RANGE_SIZE_SETTING.getKey(),
+                SharedBlobCacheService.SHARED_CACHE_RECOVERY_RANGE_SIZE_SETTING.getKey(),
                 rarely()
                     ? pageAligned(new ByteSizeValue(randomIntBetween(4, 1024), ByteSizeUnit.KB))
                     : pageAligned(new ByteSizeValue(randomIntBetween(1, 10), ByteSizeUnit.MB))
@@ -351,6 +353,11 @@ protected void assertExecutorIsIdle(String executorName) throws Exception {
     }
 
     public static class LicensedSnapshotBasedRecoveriesPlugin extends SnapshotBasedRecoveriesPlugin {
+
+        public LicensedSnapshotBasedRecoveriesPlugin(Settings settings) {
+            super(settings);
+        }
+
         @Override
         public boolean isLicenseEnabled() {
             return true;
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/PrevalidateNodeRemovalWithSearchableSnapshotIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/PrevalidateNodeRemovalWithSearchableSnapshotIntegTests.java
index 5d62f984be54..44b5d734ed0c 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/PrevalidateNodeRemovalWithSearchableSnapshotIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/PrevalidateNodeRemovalWithSearchableSnapshotIntegTests.java
@@ -71,6 +71,7 @@ public void testNodeRemovalFromClusterWihRedSearchableSnapshotIndex() throws Exc
         }
         PrevalidateNodeRemovalResponse resp = client().execute(PrevalidateNodeRemovalAction.INSTANCE, req.build()).get();
         assertTrue(resp.getPrevalidation().isSafe());
+        assertThat(resp.getPrevalidation().message(), equalTo("all red indices are searchable snapshot indices"));
         assertThat(resp.getPrevalidation().nodes().size(), equalTo(1));
         NodesRemovalPrevalidation.NodeResult nodeResult = resp.getPrevalidation().nodes().get(0);
         assertNotNull(nodeResult);
@@ -78,5 +79,7 @@ public void testNodeRemovalFromClusterWihRedSearchableSnapshotIndex() throws Exc
         assertThat(nodeResult.Id(), not(emptyString()));
         assertThat(nodeResult.externalId(), not(emptyString()));
         assertTrue(nodeResult.result().isSafe());
+        assertThat(nodeResult.result().reason(), equalTo(NodesRemovalPrevalidation.Reason.NO_RED_SHARDS_EXCEPT_SEARCHABLE_SNAPSHOTS));
+        assertThat(nodeResult.result().message(), equalTo(""));
     }
 }
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java
index 4368efb28023..1531ca226f2f 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.blobcache.shared.SharedBlobCacheService;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -33,7 +34,6 @@
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotAction;
 import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService;
 
 import java.io.IOException;
 import java.time.Instant;
@@ -68,7 +68,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
             return Settings.builder()
                 .put(initialSettings)
                 // Have a shared cache of reasonable size available on each node because tests randomize over frozen and cold allocation
-                .put(FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ofMb(randomLongBetween(1, 10)))
+                .put(SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ofMb(randomLongBetween(1, 10)))
                 .build();
         } else {
             return initialSettings;
@@ -100,10 +100,18 @@ public void testSearchableSnapshotShardsAreSkippedWithoutQueryingAnyNodeWhenThey
 
         // Either add data outside of the range, or documents that don't have timestamp data
         final boolean indexDataWithTimestamp = randomBoolean();
+        // Add enough documents to have non-metadata segment files in all shards,
+        // otherwise the mount operation might go through as the read won't be
+        // blocked
+        final int numberOfDocsInIndexOutsideSearchRange = between(350, 1000);
         if (indexDataWithTimestamp) {
-            indexDocumentsWithTimestampWithinDate(indexOutsideSearchRange, between(1, 1000), "2020-11-26T%02d:%02d:%02d.%09dZ");
+            indexDocumentsWithTimestampWithinDate(
+                indexOutsideSearchRange,
+                numberOfDocsInIndexOutsideSearchRange,
+                "2020-11-26T%02d:%02d:%02d.%09dZ"
+            );
         } else {
-            indexRandomDocs(indexOutsideSearchRange, between(0, 1000));
+            indexRandomDocs(indexOutsideSearchRange, numberOfDocsInIndexOutsideSearchRange);
         }
 
         // Index enough documents to ensure that all shards have at least some documents
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotDiskThresholdIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotDiskThresholdIntegTests.java
index bf0a419e956b..753f9e9296be 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotDiskThresholdIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotDiskThresholdIntegTests.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.searchablesnapshots.allocation;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.blobcache.shared.SharedBlobCacheService;
 import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterInfoServiceUtils;
 import org.elasticsearch.cluster.DiskUsageIntegTestCase;
@@ -47,7 +48,6 @@
 import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest;
 import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest.Storage;
 import org.elasticsearch.xpack.searchablesnapshots.LocalStateSearchableSnapshots;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -223,7 +223,7 @@ public void testHighWatermarkCanNotBeExceededOnColdNode() throws Exception {
         } else {
             otherDataNodeSettings.put(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE.roleName())
                 .put(
-                    FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey(),
+                    SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(),
                     ByteSizeValue.ofBytes(Math.min(indicesStoresSizes.values().stream().mapToLong(value -> value).sum(), 5 * 1024L * 1024L))
                 );
         }
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheIntegTests.java
index 8cf3e142aeb4..83fd5fe99d86 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheIntegTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.blobcache.shared.SharedBlobCacheService;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -48,7 +49,6 @@
 import org.elasticsearch.xpack.searchablesnapshots.action.SearchableSnapshotsStatsAction;
 import org.elasticsearch.xpack.searchablesnapshots.action.SearchableSnapshotsStatsRequest;
 import org.elasticsearch.xpack.searchablesnapshots.cache.full.CacheService;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
@@ -59,10 +59,10 @@
 import java.util.Locale;
 import java.util.concurrent.TimeUnit;
 
+import static org.elasticsearch.blobcache.shared.SharedBytes.pageAligned;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_BLOB_CACHE_INDEX;
-import static org.elasticsearch.xpack.searchablesnapshots.cache.shared.SharedBytes.pageAligned;
 import static org.elasticsearch.xpack.searchablesnapshots.store.SearchableSnapshotDirectory.unwrapDirectory;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -82,11 +82,11 @@ public static void setUpCacheSettings() {
         builder.put(CacheService.SNAPSHOT_CACHE_RECOVERY_RANGE_SIZE_SETTING.getKey(), blobCacheMaxLength);
 
         // Frozen (shared cache) cache should be large enough to not cause direct reads
-        builder.put(FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ofMb(128));
+        builder.put(SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ofMb(128));
         // Align ranges to match the blob cache max length
-        builder.put(FrozenCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(), blobCacheMaxLength);
-        builder.put(FrozenCacheService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(), blobCacheMaxLength);
-        builder.put(FrozenCacheService.SHARED_CACHE_RECOVERY_RANGE_SIZE_SETTING.getKey(), blobCacheMaxLength);
+        builder.put(SharedBlobCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(), blobCacheMaxLength);
+        builder.put(SharedBlobCacheService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(), blobCacheMaxLength);
+        builder.put(SharedBlobCacheService.SHARED_CACHE_RECOVERY_RANGE_SIZE_SETTING.getKey(), blobCacheMaxLength);
         cacheSettings = builder.build();
     }
 
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java
index 5733c438cfb2..d266b2923118 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.blobcache.common.ByteRange;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -35,7 +36,6 @@
 import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest.Storage;
 import org.elasticsearch.xpack.searchablesnapshots.BaseFrozenSearchableSnapshotsIntegTestCase;
 import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.ByteRange;
 
 import java.time.Duration;
 import java.time.Instant;
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/NodesCachesStatsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/NodesCachesStatsIntegTests.java
index 3641e722c97e..3b2c915f9b50 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/NodesCachesStatsIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/NodesCachesStatsIntegTests.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.searchablesnapshots.cache.shared;
 
+import org.elasticsearch.blobcache.shared.SharedBlobCacheService;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
@@ -39,6 +40,7 @@
 
 public class NodesCachesStatsIntegTests extends BaseFrozenSearchableSnapshotsIntegTestCase {
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/93555")
     public void testNodesCachesStats() throws Exception {
         final String[] nodeNames = internalCluster().getNodeNames();
         // here to ensure no shard relocations after the snapshot is mounted,
@@ -93,10 +95,10 @@ public void testNodesCachesStats() throws Exception {
                 .getTotal()
                 .getBytes();
 
-            final long cacheSize = FrozenCacheService.calculateCacheSize(clusterService.getSettings(), totalFsSize);
+            final long cacheSize = SharedBlobCacheService.calculateCacheSize(clusterService.getSettings(), totalFsSize);
             assertThat(nodeCachesStats.getSize(), equalTo(cacheSize));
 
-            final long regionSize = FrozenCacheService.SHARED_CACHE_REGION_SIZE_SETTING.get(clusterService.getSettings()).getBytes();
+            final long regionSize = SharedBlobCacheService.SHARED_CACHE_REGION_SIZE_SETTING.get(clusterService.getSettings()).getBytes();
             assertThat(nodeCachesStats.getRegionSize(), equalTo(regionSize));
 
             assertThat(nodeCachesStats.getNumRegions(), equalTo(Math.toIntExact(cacheSize / regionSize)));
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/PartiallyCachedShardAllocationIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/PartiallyCachedShardAllocationIntegTests.java
index 0b5855939934..c7c9b83aa5de 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/PartiallyCachedShardAllocationIntegTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/PartiallyCachedShardAllocationIntegTests.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.xpack.searchablesnapshots.BaseFrozenSearchableSnapshotsIntegTestCase;
 import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots;
 import org.elasticsearch.xpack.searchablesnapshots.action.cache.FrozenCacheInfoNodeAction;
+import org.junit.After;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -49,14 +50,15 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.blobcache.shared.SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING;
 import static org.elasticsearch.cluster.routing.allocation.DataTier.TIER_PREFERENCE;
 import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING;
 import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING;
 import static org.elasticsearch.test.NodeRoles.onlyRole;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService.SHARED_CACHE_SIZE_SETTING;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 public class PartiallyCachedShardAllocationIntegTests extends BaseFrozenSearchableSnapshotsIntegTestCase {
 
@@ -191,6 +193,7 @@ public void testOnlyPartialSearchableSnapshotAllocatedToDedicatedFrozenNodes() t
         }
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/91800")
     public void testPartialSearchableSnapshotDelaysAllocationUntilNodeCacheStatesKnown() throws Exception {
 
         assertAcked(
@@ -322,11 +325,15 @@ public void onFailure(Exception e) {
             );
         }
 
-        assertTrue(
+        assertThat(
             "balanced across " + newNodes + " in " + state,
-            Math.abs(shardCountsByNodeName.get(newNodes.get(0)) - shardCountsByNodeName.get(newNodes.get(1))) <= 1
+            Math.abs(shardCountsByNodeName.get(newNodes.get(0)) - shardCountsByNodeName.get(newNodes.get(1))),
+            lessThanOrEqualTo(1)
         );
+    }
 
+    @After
+    public void cleanUpSettings() {
         assertAcked(
             client().admin()
                 .cluster()
@@ -334,5 +341,4 @@ public void onFailure(Exception e) {
                 .setPersistentSettings(Settings.builder().putNull(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey()))
         );
     }
-
 }
diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/recovery/SearchableSnapshotRecoveryStateIntegrationTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/recovery/SearchableSnapshotRecoveryStateIntegrationTests.java
index 2e7dac7d5833..1530569b9752 100644
--- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/recovery/SearchableSnapshotRecoveryStateIntegrationTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/recovery/SearchableSnapshotRecoveryStateIntegrationTests.java
@@ -105,6 +105,8 @@ public void testRecoveryStateRecoveredBytesMatchPhysicalCacheState() throws Exce
 
         assertThat("Physical cache size doesn't match with recovery state data", physicalCacheSize, equalTo(recoveredBytes));
         assertThat("Expected to recover 100% of files", recoveryState.getIndex().recoveredBytesPercent(), equalTo(100.0f));
+        assertThat(recoveryState.getIndex().recoveredFromSnapshotBytes(), equalTo(recoveredBytes));
+        assertThat(recoveryState.getIndex().recoveredBytes(), equalTo(recoveredBytes));
     }
 
     public void testFilesStoredInThePersistentCacheAreMarkedAsReusedInRecoveryState() throws Exception {
@@ -198,6 +200,7 @@ public void testFilesStoredInThePersistentCacheAreMarkedAsReusedInRecoveryState(
         assertThat(physicalCacheSize, equalTo(expectedPhysicalCacheSize));
         assertThat(physicalCacheSize + inMemoryCacheSize, equalTo(recoveryState.getIndex().reusedBytes()));
         assertThat("Expected to recover 100% of files", recoveryState.getIndex().recoveredBytesPercent(), equalTo(100.0f));
+        assertThat(recoveryState.getIndex().recoveredFromSnapshotBytes(), equalTo(0L));
 
         for (RecoveryState.FileDetail fileDetail : recoveryState.getIndex().fileDetails()) {
             assertThat(fileDetail.name() + " wasn't mark as reused", fileDetail.reused(), equalTo(true));
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java
index 256e8f4b1907..f4b69b422793 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.blobcache.shared.SharedBlobCacheService;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterStateListener;
@@ -22,10 +23,10 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RerouteService;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -47,6 +48,7 @@
 import org.elasticsearch.index.engine.EngineFactory;
 import org.elasticsearch.index.engine.ReadOnlyEngine;
 import org.elasticsearch.index.engine.frozen.FrozenEngine;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogStats;
 import org.elasticsearch.indices.SystemIndexDescriptor;
@@ -100,10 +102,10 @@
 import org.elasticsearch.xpack.searchablesnapshots.allocation.decider.SearchableSnapshotRepositoryExistsAllocationDecider;
 import org.elasticsearch.xpack.searchablesnapshots.cache.blob.BlobStoreCacheMaintenanceService;
 import org.elasticsearch.xpack.searchablesnapshots.cache.blob.BlobStoreCacheService;
+import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey;
 import org.elasticsearch.xpack.searchablesnapshots.cache.full.CacheService;
 import org.elasticsearch.xpack.searchablesnapshots.cache.full.PersistentCache;
 import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheInfoService;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService;
 import org.elasticsearch.xpack.searchablesnapshots.recovery.SearchableSnapshotRecoveryState;
 import org.elasticsearch.xpack.searchablesnapshots.rest.RestClearSearchableSnapshotsCacheAction;
 import org.elasticsearch.xpack.searchablesnapshots.rest.RestMountSearchableSnapshotAction;
@@ -124,6 +126,7 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
@@ -253,7 +256,7 @@ public class SearchableSnapshots extends Plugin implements IndexStorePlugin, Eng
     private volatile Supplier<RepositoriesService> repositoriesServiceSupplier;
     private final SetOnce<BlobStoreCacheService> blobStoreCacheService = new SetOnce<>();
     private final SetOnce<CacheService> cacheService = new SetOnce<>();
-    private final SetOnce<FrozenCacheService> frozenCacheService = new SetOnce<>();
+    private final SetOnce<SharedBlobCacheService<CacheKey>> frozenCacheService = new SetOnce<>();
     private final SetOnce<ThreadPool> threadPool = new SetOnce<>();
     private final SetOnce<FailShardsOnInvalidLicenseClusterListener> failShardsListener = new SetOnce<>();
     private final SetOnce<SearchableSnapshotAllocator> allocator = new SetOnce<>();
@@ -280,6 +283,12 @@ public static BlobStoreRepository getSearchableRepository(Repository repository)
         return (BlobStoreRepository) repository;
     }
 
+    public static Predicate<CacheKey> forceEvictPredicate(ShardId shardId, Settings indexSettings) {
+        final String snapshotUUID = SNAPSHOT_SNAPSHOT_ID_SETTING.get(indexSettings);
+        final String snapshotIndexName = SNAPSHOT_INDEX_NAME_SETTING.get(indexSettings);
+        return k -> shardId.equals(k.shardId()) && snapshotIndexName.equals(k.snapshotIndexName()) && snapshotUUID.equals(k.snapshotUUID());
+    }
+
     @Override
     public List<Setting<?>> getSettings() {
         return List.of(
@@ -302,14 +311,6 @@ public List<Setting<?>> getSettings() {
             CacheService.SNAPSHOT_CACHE_MAX_FILES_TO_SYNC_AT_ONCE_SETTING,
             CacheService.SNAPSHOT_CACHE_SYNC_SHUTDOWN_TIMEOUT,
             SearchableSnapshotEnableAllocationDecider.SEARCHABLE_SNAPSHOTS_ALLOCATE_ON_ROLLING_RESTART,
-            FrozenCacheService.SHARED_CACHE_SIZE_SETTING,
-            FrozenCacheService.SHARED_CACHE_SIZE_MAX_HEADROOM_SETTING,
-            FrozenCacheService.SHARED_CACHE_REGION_SIZE_SETTING,
-            FrozenCacheService.SHARED_CACHE_RANGE_SIZE_SETTING,
-            FrozenCacheService.SHARED_CACHE_RECOVERY_RANGE_SIZE_SETTING,
-            FrozenCacheService.SHARED_CACHE_MAX_FREQ_SETTING,
-            FrozenCacheService.SHARED_CACHE_DECAY_INTERVAL_SETTING,
-            FrozenCacheService.SHARED_CACHE_MIN_TIME_DELTA_SETTING,
             BlobStoreCacheMaintenanceService.SNAPSHOT_SNAPSHOT_CLEANUP_INTERVAL_SETTING,
             BlobStoreCacheMaintenanceService.SNAPSHOT_SNAPSHOT_CLEANUP_KEEP_ALIVE_SETTING,
             BlobStoreCacheMaintenanceService.SNAPSHOT_SNAPSHOT_CLEANUP_BATCH_SIZE_SETTING,
@@ -331,7 +332,7 @@ public Collection<Object> createComponents(
         final IndexNameExpressionResolver resolver,
         final Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         final List<Object> components = new ArrayList<>();
         this.repositoriesServiceSupplier = repositoriesServiceSupplier;
@@ -340,8 +341,12 @@ public Collection<Object> createComponents(
         if (DiscoveryNode.canContainData(settings)) {
             final CacheService cacheService = new CacheService(settings, clusterService, threadPool, new PersistentCache(nodeEnvironment));
             this.cacheService.set(cacheService);
-            final FrozenCacheService frozenCacheService = new FrozenCacheService(nodeEnvironment, settings, threadPool);
-            this.frozenCacheService.set(frozenCacheService);
+            final SharedBlobCacheService<CacheKey> sharedBlobCacheService = new SharedBlobCacheService<>(
+                nodeEnvironment,
+                settings,
+                threadPool
+            );
+            this.frozenCacheService.set(sharedBlobCacheService);
             components.add(cacheService);
             final BlobStoreCacheService blobStoreCacheService = new BlobStoreCacheService(
                 clusterService,
@@ -714,20 +719,20 @@ public CacheService get() {
     }
 
     /**
-     * Allows to inject the {@link FrozenCacheService} instance to transport actions
+     * Allows to inject the {@link SharedBlobCacheService} instance to transport actions
      */
-    public static final class FrozenCacheServiceSupplier implements Supplier<FrozenCacheService> {
+    public static final class FrozenCacheServiceSupplier implements Supplier<SharedBlobCacheService<CacheKey>> {
 
         @Nullable
-        private final FrozenCacheService frozenCacheService;
+        private final SharedBlobCacheService<CacheKey> sharedBlobCacheService;
 
-        FrozenCacheServiceSupplier(@Nullable FrozenCacheService frozenCacheService) {
-            this.frozenCacheService = frozenCacheService;
+        FrozenCacheServiceSupplier(@Nullable SharedBlobCacheService<CacheKey> sharedBlobCacheService) {
+            this.sharedBlobCacheService = sharedBlobCacheService;
         }
 
         @Override
-        public FrozenCacheService get() {
-            return frozenCacheService;
+        public SharedBlobCacheService<CacheKey> get() {
+            return sharedBlobCacheService;
         }
     }
 
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsUtils.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsUtils.java
deleted file mode 100644
index f0daca83feea..000000000000
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsUtils.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.searchablesnapshots;
-
-import org.elasticsearch.common.unit.ByteSizeUnit;
-
-public class SearchableSnapshotsUtils {
-
-    /**
-     * We use {@code long} to represent offsets and lengths of files since they may be larger than 2GB, but {@code int} to represent
-     * offsets and lengths of arrays in memory which are limited to 2GB in size. We quite often need to convert from the file-based world
-     * of {@code long}s into the memory-based world of {@code int}s, knowing for certain that the result will not overflow. This method
-     * should be used to clarify that we're doing this.
-     */
-    public static int toIntBytes(long l) {
-        return ByteSizeUnit.BYTES.toIntBytes(l);
-    }
-}
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportClearSearchableSnapshotsCacheAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportClearSearchableSnapshotsCacheAction.java
index eaea799278a8..0fab5c4db2e4 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportClearSearchableSnapshotsCacheAction.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportClearSearchableSnapshotsCacheAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.searchablesnapshots.action;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction.EmptyResult;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -22,7 +21,6 @@
 import org.elasticsearch.xpack.searchablesnapshots.store.SearchableSnapshotDirectory;
 
 import java.io.IOException;
-import java.util.List;
 
 public class TransportClearSearchableSnapshotsCacheAction extends AbstractTransportSearchableSnapshotsAction<
     ClearSearchableSnapshotsCacheRequest,
@@ -54,20 +52,20 @@ public TransportClearSearchableSnapshotsCacheAction(
 
     @Override
     protected EmptyResult readShardResult(StreamInput in) {
-        return EmptyResult.readEmptyResultFrom(in);
+        return EmptyResult.INSTANCE;
     }
 
     @Override
-    protected ClearSearchableSnapshotsCacheResponse newResponse(
+    protected ResponseFactory<ClearSearchableSnapshotsCacheResponse, EmptyResult> getResponseFactory(
         ClearSearchableSnapshotsCacheRequest request,
-        int totalShards,
-        int successfulShards,
-        int failedShards,
-        List<EmptyResult> responses,
-        List<DefaultShardOperationFailedException> shardFailures,
         ClusterState clusterState
     ) {
-        return new ClearSearchableSnapshotsCacheResponse(totalShards, successfulShards, failedShards, shardFailures);
+        return (totalShards, successfulShards, failedShards, emptyResults, shardFailures) -> new ClearSearchableSnapshotsCacheResponse(
+            totalShards,
+            successfulShards,
+            failedShards,
+            shardFailures
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportSearchableSnapshotsStatsAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportSearchableSnapshotsStatsAction.java
index fd37a420fb14..3d518c4359b3 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportSearchableSnapshotsStatsAction.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportSearchableSnapshotsStatsAction.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.searchablesnapshots.action;
 
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -27,7 +26,6 @@
 import org.elasticsearch.xpack.searchablesnapshots.store.SearchableSnapshotDirectory;
 
 import java.io.IOException;
-import java.util.List;
 import java.util.stream.Collectors;
 
 public class TransportSearchableSnapshotsStatsAction extends AbstractTransportSearchableSnapshotsAction<
@@ -62,16 +60,17 @@ protected SearchableSnapshotShardStats readShardResult(StreamInput in) throws IO
     }
 
     @Override
-    protected SearchableSnapshotsStatsResponse newResponse(
+    protected ResponseFactory<SearchableSnapshotsStatsResponse, SearchableSnapshotShardStats> getResponseFactory(
         SearchableSnapshotsStatsRequest request,
-        int totalShards,
-        int successfulShards,
-        int failedShards,
-        List<SearchableSnapshotShardStats> shardsStats,
-        List<DefaultShardOperationFailedException> shardFailures,
         ClusterState clusterState
     ) {
-        return new SearchableSnapshotsStatsResponse(shardsStats, totalShards, successfulShards, failedShards, shardFailures);
+        return (totalShards, successfulShards, failedShards, shardsStats, shardFailures) -> new SearchableSnapshotsStatsResponse(
+            shardsStats,
+            totalShards,
+            successfulShards,
+            failedShards,
+            shardFailures
+        );
     }
 
     @Override
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/FrozenCacheInfoNodeAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/FrozenCacheInfoNodeAction.java
index 3cbb60de1338..fc1acb36892d 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/FrozenCacheInfoNodeAction.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/FrozenCacheInfoNodeAction.java
@@ -21,7 +21,7 @@
 
 import java.io.IOException;
 
-import static org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService.SHARED_CACHE_SIZE_SETTING;
+import static org.elasticsearch.blobcache.shared.SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING;
 
 public class FrozenCacheInfoNodeAction extends ActionType<FrozenCacheInfoResponse> {
 
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotsNodeCachesStatsAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotsNodeCachesStatsAction.java
index a79b9fbcd465..889d9cd2d64a 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotsNodeCachesStatsAction.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotsNodeCachesStatsAction.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
+import org.elasticsearch.blobcache.shared.SharedBlobCacheService;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -30,7 +31,7 @@
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService;
+import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -52,7 +53,7 @@ public class TransportSearchableSnapshotsNodeCachesStatsAction extends Transport
 
     public static final ActionType<NodesCachesStatsResponse> TYPE = new ActionType<>(ACTION_NAME, NodesCachesStatsResponse::new);
 
-    private final Supplier<FrozenCacheService> frozenCacheService;
+    private final Supplier<SharedBlobCacheService<CacheKey>> frozenCacheService;
     private final XPackLicenseState licenseState;
 
     @Inject
@@ -119,22 +120,22 @@ protected void resolveRequest(NodesRequest request, ClusterState clusterState) {
     @Override
     protected NodeCachesStatsResponse nodeOperation(NodeRequest request, Task task) {
         SearchableSnapshots.ensureValidLicense(licenseState);
-        final FrozenCacheService.Stats frozenCacheStats;
+        final SharedBlobCacheService.Stats frozenCacheStats;
         if (frozenCacheService.get() != null) {
             frozenCacheStats = frozenCacheService.get().getStats();
         } else {
-            frozenCacheStats = FrozenCacheService.Stats.EMPTY;
+            frozenCacheStats = SharedBlobCacheService.Stats.EMPTY;
         }
         return new NodeCachesStatsResponse(
             clusterService.localNode(),
-            frozenCacheStats.getNumberOfRegions(),
-            frozenCacheStats.getSize(),
-            frozenCacheStats.getRegionSize(),
-            frozenCacheStats.getWriteCount(),
-            frozenCacheStats.getWriteBytes(),
-            frozenCacheStats.getReadCount(),
-            frozenCacheStats.getReadBytes(),
-            frozenCacheStats.getEvictCount()
+            frozenCacheStats.numberOfRegions(),
+            frozenCacheStats.size(),
+            frozenCacheStats.regionSize(),
+            frozenCacheStats.writeCount(),
+            frozenCacheStats.writeBytes(),
+            frozenCacheStats.readCount(),
+            frozenCacheStats.readBytes(),
+            frozenCacheStats.evictCount()
         );
     }
 
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotAllocator.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotAllocator.java
index a82bf62db6c1..7b55bfceb44d 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotAllocator.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotAllocator.java
@@ -65,6 +65,7 @@
 import java.util.stream.StreamSupport;
 
 import static java.util.Collections.emptyList;
+import static org.elasticsearch.gateway.ReplicaShardAllocator.augmentExplanationsWithStoreInfo;
 import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SNAPSHOT_PARTIAL_SETTING;
 import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_INDEX_ID_SETTING;
 import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_INDEX_NAME_SETTING;
@@ -447,28 +448,6 @@ public void onFailure(Exception e) {
         return new AsyncShardFetch.FetchResult<>(shardId, asyncFetch.data(), Collections.emptySet());
     }
 
-    /**
-     * Takes the store info for nodes that have a shard store and adds them to the node decisions,
-     * leaving the node explanations untouched for those nodes that do not have any store information.
-     */
-    private static List<NodeAllocationResult> augmentExplanationsWithStoreInfo(
-        Map<String, NodeAllocationResult> nodeDecisions,
-        Map<String, NodeAllocationResult> withShardStores
-    ) {
-        if (nodeDecisions == null || withShardStores == null) {
-            return null;
-        }
-        List<NodeAllocationResult> augmented = new ArrayList<>();
-        for (Map.Entry<String, NodeAllocationResult> entry : nodeDecisions.entrySet()) {
-            if (withShardStores.containsKey(entry.getKey())) {
-                augmented.add(withShardStores.get(entry.getKey()));
-            } else {
-                augmented.add(entry.getValue());
-            }
-        }
-        return augmented;
-    }
-
     private static MatchingNodes findMatchingNodes(
         ShardRouting shard,
         RoutingAllocation allocation,
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotIndexEventListener.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotIndexEventListener.java
index 60b56fd1e2d5..3e76c334c707 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotIndexEventListener.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotIndexEventListener.java
@@ -8,7 +8,9 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.StepListener;
+import org.elasticsearch.blobcache.shared.SharedBlobCacheService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -21,8 +23,9 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots;
+import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey;
 import org.elasticsearch.xpack.searchablesnapshots.cache.full.CacheService;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService;
 import org.elasticsearch.xpack.searchablesnapshots.store.SearchableSnapshotDirectory;
 
 import static org.elasticsearch.core.Strings.format;
@@ -35,16 +38,16 @@ public class SearchableSnapshotIndexEventListener implements IndexEventListener
     private static final Logger logger = LogManager.getLogger(SearchableSnapshotIndexEventListener.class);
 
     private final @Nullable CacheService cacheService;
-    private final @Nullable FrozenCacheService frozenCacheService;
+    private final @Nullable SharedBlobCacheService<CacheKey> sharedBlobCacheService;
 
     public SearchableSnapshotIndexEventListener(
         Settings settings,
         @Nullable CacheService cacheService,
-        @Nullable FrozenCacheService frozenCacheService
+        @Nullable SharedBlobCacheService<CacheKey> sharedBlobCacheService
     ) {
         assert cacheService != null || DiscoveryNode.canContainData(settings) == false;
         this.cacheService = cacheService;
-        this.frozenCacheService = frozenCacheService;
+        this.sharedBlobCacheService = sharedBlobCacheService;
     }
 
     /**
@@ -55,9 +58,10 @@ public SearchableSnapshotIndexEventListener(
      * @param indexSettings the shard's index settings
      */
     @Override
-    public void beforeIndexShardRecovery(IndexShard indexShard, IndexSettings indexSettings) {
+    public void beforeIndexShardRecovery(IndexShard indexShard, IndexSettings indexSettings, ActionListener<Void> listener) {
         assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.GENERIC);
         ensureSnapshotIsLoaded(indexShard);
+        listener.onResponse(null);
     }
 
     private static void ensureSnapshotIsLoaded(IndexShard indexShard) {
@@ -101,12 +105,8 @@ public void beforeIndexRemoved(IndexService indexService, IndexRemovalReason rea
                             shardId
                         );
                     }
-                    if (frozenCacheService != null) {
-                        frozenCacheService.markShardAsEvictedInCache(
-                            SNAPSHOT_SNAPSHOT_ID_SETTING.get(indexSettings.getSettings()),
-                            SNAPSHOT_INDEX_NAME_SETTING.get(indexSettings.getSettings()),
-                            shardId
-                        );
+                    if (sharedBlobCacheService != null) {
+                        sharedBlobCacheService.forceEvict(SearchableSnapshots.forceEvictPredicate(shardId, indexSettings.getSettings()));
                     }
                 }
             }
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotIndexFoldersDeletionListener.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotIndexFoldersDeletionListener.java
index 328ea6dda8e6..4d7174e0f7ff 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotIndexFoldersDeletionListener.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotIndexFoldersDeletionListener.java
@@ -9,12 +9,14 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.blobcache.shared.SharedBlobCacheService;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.plugins.IndexStorePlugin;
+import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots;
+import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey;
 import org.elasticsearch.xpack.searchablesnapshots.cache.full.CacheService;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService;
 
 import java.nio.file.Path;
 import java.util.Objects;
@@ -33,11 +35,11 @@ public class SearchableSnapshotIndexFoldersDeletionListener implements IndexStor
     private static final Logger logger = LogManager.getLogger(SearchableSnapshotIndexEventListener.class);
 
     private final Supplier<CacheService> cacheServiceSupplier;
-    private final Supplier<FrozenCacheService> frozenCacheServiceSupplier;
+    private final Supplier<SharedBlobCacheService<CacheKey>> frozenCacheServiceSupplier;
 
     public SearchableSnapshotIndexFoldersDeletionListener(
         Supplier<CacheService> cacheServiceSupplier,
-        Supplier<FrozenCacheService> frozenCacheServiceSupplier
+        Supplier<SharedBlobCacheService<CacheKey>> frozenCacheServiceSupplier
     ) {
         this.cacheServiceSupplier = Objects.requireNonNull(cacheServiceSupplier);
         this.frozenCacheServiceSupplier = Objects.requireNonNull(frozenCacheServiceSupplier);
@@ -70,12 +72,8 @@ private void markShardAsEvictedInCache(ShardId shardId, IndexSettings indexSetti
             shardId
         );
 
-        final FrozenCacheService frozenCacheService = this.frozenCacheServiceSupplier.get();
-        assert frozenCacheService != null : "frozen cache service not initialized";
-        frozenCacheService.markShardAsEvictedInCache(
-            SNAPSHOT_SNAPSHOT_ID_SETTING.get(indexSettings.getSettings()),
-            SNAPSHOT_INDEX_NAME_SETTING.get(indexSettings.getSettings()),
-            shardId
-        );
+        final SharedBlobCacheService<CacheKey> sharedBlobCacheService = this.frozenCacheServiceSupplier.get();
+        assert sharedBlobCacheService != null : "frozen cache service not initialized";
+        sharedBlobCacheService.forceEvict(SearchableSnapshots.forceEvictPredicate(shardId, indexSettings.getSettings()));
     }
 }
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/HasFrozenCacheAllocationDecider.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/HasFrozenCacheAllocationDecider.java
index 93518d96173f..801f71b828c9 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/HasFrozenCacheAllocationDecider.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/HasFrozenCacheAllocationDecider.java
@@ -16,7 +16,7 @@
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheInfoService;
 
-import static org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService.SHARED_CACHE_SIZE_SETTING;
+import static org.elasticsearch.blobcache.shared.SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING;
 
 public class HasFrozenCacheAllocationDecider extends AllocationDecider {
 
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheService.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheService.java
index a36fadaa71e5..b8e527f14312 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheService.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheService.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.TransportActions;
+import org.elasticsearch.blobcache.common.ByteRange;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.OriginSettingClient;
 import org.elasticsearch.cluster.block.ClusterBlockException;
@@ -41,7 +42,6 @@
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.ByteRange;
 
 import java.time.Instant;
 import java.util.concurrent.ExecutionException;
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/CacheFile.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/CacheFile.java
index c748ca997000..9377164dd095 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/CacheFile.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/CacheFile.java
@@ -4,6 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+
 package org.elasticsearch.xpack.searchablesnapshots.cache.common;
 
 import org.apache.logging.log4j.LogManager;
@@ -11,6 +12,8 @@
 import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.blobcache.common.ByteRange;
+import org.elasticsearch.blobcache.common.SparseFileTracker;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.core.AbstractRefCounted;
 import org.elasticsearch.core.IOUtils;
@@ -32,7 +35,7 @@
 import java.util.concurrent.Executor;
 import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.Consumer;
+import java.util.function.LongConsumer;
 
 public class CacheFile {
 
@@ -168,7 +171,7 @@ FileChannel getChannel() {
     }
 
     // Only used in tests
-    SortedSet<ByteRange> getCompletedRanges() {
+    public SortedSet<ByteRange> getCompletedRanges() {
         return tracker.getCompletedRanges();
     }
 
@@ -331,7 +334,7 @@ public interface RangeAvailableHandler {
 
     @FunctionalInterface
     public interface RangeMissingHandler {
-        void fillCacheRange(FileChannel channel, long from, long to, Consumer<Long> progressUpdater) throws IOException;
+        void fillCacheRange(FileChannel channel, long from, long to, LongConsumer progressUpdater) throws IOException;
     }
 
     /**
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/CacheKey.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/CacheKey.java
index 314ce8ef894f..c045318561d4 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/CacheKey.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/CacheKey.java
@@ -4,18 +4,14 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+
 package org.elasticsearch.xpack.searchablesnapshots.cache.common;
 
 import org.elasticsearch.index.shard.ShardId;
 
 import java.util.Objects;
 
-public class CacheKey {
-
-    private final String snapshotUUID;
-    private final String snapshotIndexName;
-    private final ShardId shardId;
-    private final String fileName;
+public record CacheKey(String snapshotUUID, String snapshotIndexName, ShardId shardId, String fileName) {
 
     public CacheKey(String snapshotUUID, String snapshotIndexName, ShardId shardId, String fileName) {
         this.snapshotUUID = Objects.requireNonNull(snapshotUUID);
@@ -23,53 +19,4 @@ public CacheKey(String snapshotUUID, String snapshotIndexName, ShardId shardId,
         this.shardId = Objects.requireNonNull(shardId);
         this.fileName = Objects.requireNonNull(fileName);
     }
-
-    public String getSnapshotUUID() {
-        return snapshotUUID;
-    }
-
-    public String getSnapshotIndexName() {
-        return snapshotIndexName;
-    }
-
-    public ShardId getShardId() {
-        return shardId;
-    }
-
-    public String getFileName() {
-        return fileName;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (o == null || getClass() != o.getClass()) {
-            return false;
-        }
-        final CacheKey cacheKey = (CacheKey) o;
-        return Objects.equals(snapshotUUID, cacheKey.snapshotUUID)
-            && Objects.equals(snapshotIndexName, cacheKey.snapshotIndexName)
-            && Objects.equals(shardId, cacheKey.shardId)
-            && Objects.equals(fileName, cacheKey.fileName);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(snapshotUUID, snapshotIndexName, shardId, fileName);
-    }
-
-    @Override
-    public String toString() {
-        return "[snapshotUUID="
-            + snapshotUUID
-            + ", snapshotIndexName="
-            + snapshotIndexName
-            + ", shardId="
-            + shardId
-            + ", fileName='"
-            + fileName
-            + "']";
-    }
 }
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/CacheService.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/CacheService.java
index dfbf658add23..3daa72efdd60 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/CacheService.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/CacheService.java
@@ -9,6 +9,8 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.AlreadyClosedException;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.blobcache.common.ByteRange;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.cache.Cache;
@@ -30,7 +32,6 @@
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.ByteRange;
 import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheFile;
 import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey;
 import org.elasticsearch.xpack.searchablesnapshots.store.SearchableSnapshotDirectory;
@@ -55,8 +56,8 @@
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
+import static org.elasticsearch.blobcache.BlobCacheUtils.toIntBytes;
 import static org.elasticsearch.core.Strings.format;
-import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsUtils.toIntBytes;
 
 /**
  * {@link CacheService} maintains a cache entry for all files read from searchable snapshot directories (see
@@ -140,7 +141,7 @@ public class CacheService extends AbstractLifecycleComponent {
     private final Cache<CacheKey, CacheFile> cache;
     private final ByteSizeValue rangeSize;
     private final ByteSizeValue recoveryRangeSize;
-    private final Map<ShardEviction, Future<?>> pendingShardsEvictions;
+    private final Map<ShardEviction, PlainActionFuture<?>> pendingShardsEvictions;
     private final ReadWriteLock shardsEvictionsLock;
     private final Object shardsEvictionsMutex;
 
@@ -345,18 +346,22 @@ public void markShardAsEvictedInCache(String snapshotUUID, String snapshotIndexN
         synchronized (shardsEvictionsMutex) {
             if (allowShardsEvictions) {
                 final ShardEviction shardEviction = new ShardEviction(snapshotUUID, snapshotIndexName, shardId);
-                pendingShardsEvictions.computeIfAbsent(shardEviction, shard -> threadPool.generic().submit(new AbstractRunnable() {
-                    @Override
-                    protected void doRun() {
-                        processShardEviction(shardEviction);
-                    }
+                pendingShardsEvictions.computeIfAbsent(shardEviction, shard -> {
+                    final PlainActionFuture<?> future = PlainActionFuture.newFuture();
+                    threadPool.generic().execute(new AbstractRunnable() {
+                        @Override
+                        protected void doRun() {
+                            processShardEviction(shard);
+                        }
 
-                    @Override
-                    public void onFailure(Exception e) {
-                        logger.warn(() -> format("failed to evict cache files associated with shard %s", shardEviction), e);
-                        assert false : e;
-                    }
-                }));
+                        @Override
+                        public void onFailure(Exception e) {
+                            logger.warn(() -> format("failed to evict cache files associated with shard %s", shard), e);
+                            assert false : e;
+                        }
+                    });
+                    return future;
+                });
             }
         }
     }
@@ -423,8 +428,8 @@ private void processShardEviction(ShardEviction shardEviction) {
                 }
             } finally {
                 synchronized (shardsEvictionsMutex) {
-                    final Future<?> removedFuture = pendingShardsEvictions.remove(shardEviction);
-                    assert removedFuture != null;
+                    final PlainActionFuture<?> removedFuture = pendingShardsEvictions.remove(shardEviction);
+                    removedFuture.onResponse(null);
                 }
             }
         } finally {
@@ -681,47 +686,10 @@ public void onCacheFileDelete(CacheFile cacheFile) {
     }
 
     /**
-     * Represents the searchable snapshots information of a shard that has been removed from the node. These information are kept around
-     * to evict the cache files associated to that shard.
-     */
-    static class ShardEviction {
-
-        private final String snapshotUUID;
-        private final String snapshotIndexName;
-        private final ShardId shardId;
-
-        ShardEviction(String snapshotUUID, String snapshotIndexName, ShardId shardId) {
-            this.snapshotUUID = snapshotUUID;
-            this.snapshotIndexName = snapshotIndexName;
-            this.shardId = shardId;
-        }
-
-        public String getSnapshotUUID() {
-            return snapshotUUID;
-        }
-
-        public String getSnapshotIndexName() {
-            return snapshotIndexName;
-        }
-
-        public ShardId getShardId() {
-            return shardId;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            ShardEviction that = (ShardEviction) o;
-            return Objects.equals(snapshotUUID, that.snapshotUUID)
-                && Objects.equals(snapshotIndexName, that.snapshotIndexName)
-                && Objects.equals(shardId, that.shardId);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(snapshotUUID, snapshotIndexName, shardId);
-        }
+         * Represents the searchable snapshots information of a shard that has been removed from the node. These information are kept around
+         * to evict the cache files associated to that shard.
+         */
+    record ShardEviction(String snapshotUUID, String snapshotIndexName, ShardId shardId) {
 
         @Override
         public String toString() {
@@ -729,9 +697,9 @@ public String toString() {
         }
 
         boolean matches(CacheKey cacheKey) {
-            return Objects.equals(snapshotUUID, cacheKey.getSnapshotUUID())
-                && Objects.equals(snapshotIndexName, cacheKey.getSnapshotIndexName())
-                && Objects.equals(shardId, cacheKey.getShardId());
+            return Objects.equals(snapshotUUID, cacheKey.snapshotUUID())
+                && Objects.equals(snapshotIndexName, cacheKey.snapshotIndexName())
+                && Objects.equals(shardId, cacheKey.shardId());
         }
     }
 
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCache.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCache.java
index 92e99de0f9a2..da08c6b38819 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCache.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCache.java
@@ -39,6 +39,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
+import org.elasticsearch.blobcache.common.ByteRange;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -51,7 +52,6 @@
 import org.elasticsearch.index.shard.ShardPath;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.snapshots.SnapshotId;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.ByteRange;
 import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheFile;
 import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey;
 
@@ -577,12 +577,12 @@ private static Document buildDocument(NodeEnvironment.DataPath dataPath, CacheFi
         }
 
         final CacheKey cacheKey = cacheFile.getCacheKey();
-        document.add(new StringField(FILE_NAME_FIELD, cacheKey.getFileName(), Field.Store.YES));
+        document.add(new StringField(FILE_NAME_FIELD, cacheKey.fileName(), Field.Store.YES));
         document.add(new StringField(FILE_LENGTH_FIELD, Long.toString(cacheFile.getLength()), Field.Store.YES));
-        document.add(new StringField(SNAPSHOT_ID_FIELD, cacheKey.getSnapshotUUID(), Field.Store.YES));
-        document.add(new StringField(SNAPSHOT_INDEX_NAME_FIELD, cacheKey.getSnapshotIndexName(), Field.Store.YES));
+        document.add(new StringField(SNAPSHOT_ID_FIELD, cacheKey.snapshotUUID(), Field.Store.YES));
+        document.add(new StringField(SNAPSHOT_INDEX_NAME_FIELD, cacheKey.snapshotIndexName(), Field.Store.YES));
 
-        final ShardId shardId = cacheKey.getShardId();
+        final ShardId shardId = cacheKey.shardId();
         document.add(new StringField(SHARD_INDEX_NAME_FIELD, shardId.getIndex().getName(), Field.Store.YES));
         document.add(new StringField(SHARD_INDEX_ID_FIELD, shardId.getIndex().getUUID(), Field.Store.YES));
         document.add(new StringField(SHARD_ID_FIELD, Integer.toString(shardId.getId()), Field.Store.YES));
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/FrozenCacheService.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/FrozenCacheService.java
deleted file mode 100644
index 3d7db97c35b4..000000000000
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/FrozenCacheService.java
+++ /dev/null
@@ -1,999 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.searchablesnapshots.cache.shared;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.apache.lucene.store.AlreadyClosedException;
-import org.elasticsearch.Assertions;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.StepListener;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodeRole;
-import org.elasticsearch.cluster.routing.allocation.DataTier;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.unit.RelativeByteSizeValue;
-import org.elasticsearch.common.util.concurrent.AbstractAsyncTask;
-import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.common.util.concurrent.KeyedLock;
-import org.elasticsearch.core.AbstractRefCounted;
-import org.elasticsearch.core.Releasable;
-import org.elasticsearch.core.Releasables;
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.NodeEnvironment;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.monitor.fs.FsProbe;
-import org.elasticsearch.node.NodeRoleSettings;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.ByteRange;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.SparseFileTracker;
-
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.Executor;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.atomic.LongAdder;
-import java.util.function.Consumer;
-import java.util.function.LongSupplier;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
-
-import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsUtils.toIntBytes;
-
-public class FrozenCacheService implements Releasable {
-
-    private static final String SHARED_CACHE_SETTINGS_PREFIX = "xpack.searchable.snapshot.shared_cache.";
-
-    public static final Setting<ByteSizeValue> SHARED_CACHE_RANGE_SIZE_SETTING = new Setting<>(
-        SHARED_CACHE_SETTINGS_PREFIX + "range_size",
-        ByteSizeValue.ofMb(16).getStringRep(),
-        s -> ByteSizeValue.parseBytesSizeValue(s, SHARED_CACHE_SETTINGS_PREFIX + "range_size"),
-        getPositivePageSizeAlignedByteSizeValueValidator(SHARED_CACHE_SETTINGS_PREFIX + "range_size"),
-        Setting.Property.NodeScope
-    );
-
-    public static final Setting<ByteSizeValue> SHARED_CACHE_RECOVERY_RANGE_SIZE_SETTING = new Setting<>(
-        SHARED_CACHE_SETTINGS_PREFIX + "recovery_range_size",
-        ByteSizeValue.ofKb(128L).getStringRep(),
-        s -> ByteSizeValue.parseBytesSizeValue(s, SHARED_CACHE_SETTINGS_PREFIX + "recovery_range_size"),
-        getPositivePageSizeAlignedByteSizeValueValidator(SHARED_CACHE_SETTINGS_PREFIX + "recovery_range_size"),
-        Setting.Property.NodeScope
-    );
-
-    public static final Setting<ByteSizeValue> SHARED_CACHE_REGION_SIZE_SETTING = new Setting<>(
-        SHARED_CACHE_SETTINGS_PREFIX + "region_size",
-        SHARED_CACHE_RANGE_SIZE_SETTING,
-        s -> ByteSizeValue.parseBytesSizeValue(s, SHARED_CACHE_SETTINGS_PREFIX + "region_size"),
-        getPositivePageSizeAlignedByteSizeValueValidator(SHARED_CACHE_SETTINGS_PREFIX + "region_size"),
-        Setting.Property.NodeScope
-    );
-
-    private static Setting.Validator<ByteSizeValue> getPageSizeAlignedByteSizeValueValidator(String settingName) {
-        return value -> {
-            if (value.getBytes() == -1) {
-                throw new SettingsException("setting [{}] must be non-negative", settingName);
-            }
-            if (value.getBytes() % SharedBytes.PAGE_SIZE != 0L) {
-                throw new SettingsException("setting [{}] must be multiple of {}", settingName, SharedBytes.PAGE_SIZE);
-            }
-        };
-    }
-
-    private static Setting.Validator<ByteSizeValue> getPositivePageSizeAlignedByteSizeValueValidator(String settingName) {
-        return value -> {
-            if (value.getBytes() <= 0L) {
-                throw new SettingsException("setting [{}] must be greater than zero", settingName);
-            }
-            getPageSizeAlignedByteSizeValueValidator(settingName).validate(value);
-        };
-    }
-
-    public static final Setting<RelativeByteSizeValue> SHARED_CACHE_SIZE_SETTING = new Setting<>(
-        new Setting.SimpleKey(SHARED_CACHE_SETTINGS_PREFIX + "size"),
-        (settings) -> {
-            if (DiscoveryNode.isDedicatedFrozenNode(settings)) {
-                return "90%";
-            } else {
-                return ByteSizeValue.ZERO.getStringRep();
-            }
-        },
-        s -> RelativeByteSizeValue.parseRelativeByteSizeValue(s, SHARED_CACHE_SETTINGS_PREFIX + "size"),
-        new Setting.Validator<>() {
-
-            @Override
-            public void validate(final RelativeByteSizeValue value) {
-
-            }
-
-            @Override
-            public void validate(final RelativeByteSizeValue value, final Map<Setting<?>, Object> settings) {
-                if (value.isAbsolute() && value.getAbsolute().getBytes() == -1) {
-                    throw new SettingsException("setting [{}] must be non-negative", SHARED_CACHE_SETTINGS_PREFIX + "size");
-                }
-                if (value.isNonZeroSize()) {
-                    @SuppressWarnings("unchecked")
-                    final List<DiscoveryNodeRole> roles = (List<DiscoveryNodeRole>) settings.get(NodeRoleSettings.NODE_ROLES_SETTING);
-                    if (DataTier.isFrozenNode(Set.of(roles.toArray(DiscoveryNodeRole[]::new))) == false) {
-                        throw new SettingsException(
-                            "setting [{}] to be positive [{}] is only permitted on nodes with the data_frozen role, roles are [{}]",
-                            SHARED_CACHE_SETTINGS_PREFIX + "size",
-                            value.getStringRep(),
-                            roles.stream().map(DiscoveryNodeRole::roleName).collect(Collectors.joining(","))
-                        );
-                    }
-
-                    @SuppressWarnings("unchecked")
-                    final List<String> dataPaths = (List<String>) settings.get(Environment.PATH_DATA_SETTING);
-                    if (dataPaths.size() > 1) {
-                        throw new SettingsException(
-                            "setting [{}={}] is not permitted on nodes with multiple data paths [{}]",
-                            SHARED_CACHE_SIZE_SETTING.getKey(),
-                            value.getStringRep(),
-                            String.join(",", dataPaths)
-                        );
-                    }
-                }
-            }
-
-            @Override
-            public Iterator<Setting<?>> settings() {
-                final List<Setting<?>> settings = List.of(NodeRoleSettings.NODE_ROLES_SETTING, Environment.PATH_DATA_SETTING);
-                return settings.iterator();
-            }
-
-        },
-        Setting.Property.NodeScope
-    );
-
-    public static final Setting<ByteSizeValue> SHARED_CACHE_SIZE_MAX_HEADROOM_SETTING = new Setting<>(
-        new Setting.SimpleKey(SHARED_CACHE_SETTINGS_PREFIX + "size.max_headroom"),
-        (settings) -> {
-            if (SHARED_CACHE_SIZE_SETTING.exists(settings) == false && DiscoveryNode.isDedicatedFrozenNode(settings)) {
-                return "100GB";
-            }
-
-            return "-1";
-        },
-        (s) -> ByteSizeValue.parseBytesSizeValue(s, SHARED_CACHE_SETTINGS_PREFIX + "size.max_headroom"),
-        new Setting.Validator<>() {
-            private final Collection<Setting<?>> dependencies = List.of(SHARED_CACHE_SIZE_SETTING);
-
-            @Override
-            public Iterator<Setting<?>> settings() {
-                return dependencies.iterator();
-            }
-
-            @Override
-            public void validate(ByteSizeValue value) {
-                // ignore
-            }
-
-            @Override
-            public void validate(ByteSizeValue value, Map<Setting<?>, Object> settings, boolean isPresent) {
-                if (isPresent && value.getBytes() != -1) {
-                    RelativeByteSizeValue sizeValue = (RelativeByteSizeValue) settings.get(SHARED_CACHE_SIZE_SETTING);
-                    if (sizeValue.isAbsolute()) {
-                        throw new SettingsException(
-                            "setting [{}] cannot be specified for absolute [{}={}]",
-                            SHARED_CACHE_SIZE_MAX_HEADROOM_SETTING.getKey(),
-                            SHARED_CACHE_SIZE_SETTING.getKey(),
-                            sizeValue.getStringRep()
-                        );
-                    }
-                }
-            }
-        },
-        Setting.Property.NodeScope
-    );
-
-    public static final TimeValue MIN_SHARED_CACHE_DECAY_INTERVAL = TimeValue.timeValueSeconds(1L);
-    public static final Setting<TimeValue> SHARED_CACHE_DECAY_INTERVAL_SETTING = Setting.timeSetting(
-        SHARED_CACHE_SETTINGS_PREFIX + "decay.interval",
-        TimeValue.timeValueSeconds(60L),                        // default
-        MIN_SHARED_CACHE_DECAY_INTERVAL,                      // min
-        Setting.Property.NodeScope,
-        Setting.Property.Dynamic
-    );
-
-    public static final Setting<Integer> SHARED_CACHE_MAX_FREQ_SETTING = Setting.intSetting(
-        SHARED_CACHE_SETTINGS_PREFIX + "max_freq",
-        100,                       // default
-        1,                            // min
-        Setting.Property.NodeScope
-    );
-
-    public static final Setting<TimeValue> SHARED_CACHE_MIN_TIME_DELTA_SETTING = Setting.timeSetting(
-        SHARED_CACHE_SETTINGS_PREFIX + "min_time_delta",
-        TimeValue.timeValueSeconds(60L),                        // default
-        TimeValue.timeValueSeconds(0L),                         // min
-        Setting.Property.NodeScope
-    );
-
-    private static final Logger logger = LogManager.getLogger(FrozenCacheService.class);
-
-    private final ConcurrentHashMap<RegionKey, Entry<CacheFileRegion>> keyMapping;
-
-    private final LongSupplier currentTimeSupplier;
-
-    private final KeyedLock<CacheKey> keyedLock = new KeyedLock<>();
-
-    private final SharedBytes sharedBytes;
-    private final long cacheSize;
-    private final long regionSize;
-    private final ByteSizeValue rangeSize;
-    private final ByteSizeValue recoveryRangeSize;
-
-    private final int numRegions;
-    private final ConcurrentLinkedQueue<Integer> freeRegions = new ConcurrentLinkedQueue<>();
-    private final Entry<CacheFileRegion>[] freqs;
-    private final int maxFreq;
-    private final long minTimeDelta;
-
-    private final AtomicReference<CacheFileRegion>[] regionOwners; // to assert exclusive access of regions
-
-    private final CacheDecayTask decayTask;
-
-    private final LongAdder writeCount = new LongAdder();
-    private final LongAdder writeBytes = new LongAdder();
-
-    private final LongAdder readCount = new LongAdder();
-    private final LongAdder readBytes = new LongAdder();
-
-    private final LongAdder evictCount = new LongAdder();
-
-    @SuppressWarnings({ "unchecked", "rawtypes" })
-    public FrozenCacheService(NodeEnvironment environment, Settings settings, ThreadPool threadPool) {
-        this.currentTimeSupplier = threadPool::relativeTimeInMillis;
-        long totalFsSize;
-        try {
-            totalFsSize = FsProbe.getTotal(Environment.getFileStore(environment.nodeDataPaths()[0]));
-        } catch (IOException e) {
-            throw new IllegalStateException("unable to probe size of filesystem [" + environment.nodeDataPaths()[0] + "]");
-        }
-        this.cacheSize = calculateCacheSize(settings, totalFsSize);
-        final long regionSize = SHARED_CACHE_REGION_SIZE_SETTING.get(settings).getBytes();
-        this.numRegions = Math.toIntExact(cacheSize / regionSize);
-        keyMapping = new ConcurrentHashMap<>();
-        if (Assertions.ENABLED) {
-            regionOwners = new AtomicReference[numRegions];
-            for (int i = 0; i < numRegions; i++) {
-                regionOwners[i] = new AtomicReference<>();
-            }
-        } else {
-            regionOwners = null;
-        }
-        for (int i = 0; i < numRegions; i++) {
-            freeRegions.add(i);
-        }
-        this.regionSize = regionSize;
-        assert regionSize > 0L;
-        this.maxFreq = SHARED_CACHE_MAX_FREQ_SETTING.get(settings);
-        this.minTimeDelta = SHARED_CACHE_MIN_TIME_DELTA_SETTING.get(settings).millis();
-        freqs = new Entry[maxFreq];
-        try {
-            sharedBytes = new SharedBytes(numRegions, regionSize, environment, writeBytes::add, readBytes::add);
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-        decayTask = new CacheDecayTask(threadPool, SHARED_CACHE_DECAY_INTERVAL_SETTING.get(settings));
-        decayTask.rescheduleIfNecessary();
-        this.rangeSize = SHARED_CACHE_RANGE_SIZE_SETTING.get(settings);
-        this.recoveryRangeSize = SHARED_CACHE_RECOVERY_RANGE_SIZE_SETTING.get(settings);
-    }
-
-    static long calculateCacheSize(Settings settings, long totalFsSize) {
-        return SHARED_CACHE_SIZE_SETTING.get(settings)
-            .calculateValue(ByteSizeValue.ofBytes(totalFsSize), SHARED_CACHE_SIZE_MAX_HEADROOM_SETTING.get(settings))
-            .getBytes();
-    }
-
-    public int getRangeSize() {
-        return toIntBytes(rangeSize.getBytes());
-    }
-
-    public int getRecoveryRangeSize() {
-        return toIntBytes(recoveryRangeSize.getBytes());
-    }
-
-    private int getRegion(long position) {
-        return Math.toIntExact(position / regionSize);
-    }
-
-    private long getRegionRelativePosition(long position) {
-        return position % regionSize;
-    }
-
-    private long getRegionStart(int region) {
-        return region * regionSize;
-    }
-
-    private long getRegionEnd(int region) {
-        return (region + 1) * regionSize;
-    }
-
-    private int getEndingRegion(long position) {
-        assert position > 0L;
-        if (position % regionSize == 0L) {
-            return getRegion(position - 1);
-        }
-        return getRegion(position);
-    }
-
-    private ByteRange mapSubRangeToRegion(ByteRange range, int region) {
-        final long regionStart = getRegionStart(region);
-        final long regionEnd = getRegionEnd(region);
-        if (range.start() >= regionEnd || range.end() <= regionStart) {
-            return ByteRange.EMPTY;
-        }
-        final long rangeStart = Math.max(regionStart, range.start());
-        final long rangeEnd = Math.min(regionEnd, range.end());
-        if (rangeStart >= rangeEnd) {
-            return ByteRange.EMPTY;
-        }
-        return ByteRange.of(
-            getRegionRelativePosition(rangeStart),
-            rangeEnd == regionEnd ? regionSize : getRegionRelativePosition(rangeEnd)
-        );
-    }
-
-    private long getRegionSize(long fileLength, int region) {
-        assert fileLength > 0;
-        final int maxRegion = getEndingRegion(fileLength);
-        assert region >= 0 && region <= maxRegion : region + " - " + maxRegion;
-        final long effectiveRegionSize;
-        if (region == maxRegion && (region + 1) * regionSize != fileLength) {
-            assert getRegionRelativePosition(fileLength) != 0L;
-            effectiveRegionSize = getRegionRelativePosition(fileLength);
-        } else {
-            effectiveRegionSize = regionSize;
-        }
-        assert getRegionStart(region) + effectiveRegionSize <= fileLength;
-        return effectiveRegionSize;
-    }
-
-    public CacheFileRegion get(CacheKey cacheKey, long fileLength, int region) {
-        final long effectiveRegionSize = getRegionSize(fileLength, region);
-        try (Releasable ignore = keyedLock.acquire(cacheKey)) {
-            final RegionKey regionKey = new RegionKey(cacheKey, region);
-            final long now = currentTimeSupplier.getAsLong();
-            final Entry<CacheFileRegion> entry = keyMapping.computeIfAbsent(
-                regionKey,
-                key -> new Entry<>(new CacheFileRegion(regionKey, effectiveRegionSize), now)
-            );
-            if (entry.chunk.sharedBytesPos == -1) {
-                // new item
-                assert entry.freq == 0;
-                assert entry.prev == null;
-                assert entry.next == null;
-                final Integer freeSlot = freeRegions.poll();
-                if (freeSlot != null) {
-                    // no need to evict an item, just add
-                    entry.chunk.sharedBytesPos = freeSlot;
-                    assert regionOwners[freeSlot].compareAndSet(null, entry.chunk);
-                    synchronized (this) {
-                        pushEntryToBack(entry);
-                    }
-                } else {
-                    // need to evict something
-                    synchronized (this) {
-                        maybeEvict();
-                    }
-                    final Integer freeSlotRetry = freeRegions.poll();
-                    if (freeSlotRetry != null) {
-                        entry.chunk.sharedBytesPos = freeSlotRetry;
-                        assert regionOwners[freeSlotRetry].compareAndSet(null, entry.chunk);
-                        synchronized (this) {
-                            pushEntryToBack(entry);
-                        }
-                    } else {
-                        boolean removed = keyMapping.remove(regionKey, entry);
-                        assert removed;
-                        throw new AlreadyClosedException("no free region found");
-                    }
-                }
-            } else {
-                // check if we need to promote item
-                synchronized (this) {
-                    if (now - entry.lastAccessed >= minTimeDelta && entry.freq + 1 < maxFreq) {
-                        unlink(entry);
-                        entry.freq++;
-                        entry.lastAccessed = now;
-                        pushEntryToBack(entry);
-                    }
-                }
-            }
-            return entry.chunk;
-        }
-    }
-
-    public void onClose(CacheFileRegion chunk) {
-        assert regionOwners[chunk.sharedBytesPos].compareAndSet(chunk, null);
-        freeRegions.add(chunk.sharedBytesPos);
-    }
-
-    // used by tests
-    int freeRegionCount() {
-        return freeRegions.size();
-    }
-
-    public Stats getStats() {
-        return new Stats(
-            numRegions,
-            cacheSize,
-            regionSize,
-            evictCount.sum(),
-            writeCount.sum(),
-            writeBytes.sum(),
-            readCount.sum(),
-            readBytes.sum()
-        );
-    }
-
-    private synchronized boolean invariant(final Entry<CacheFileRegion> e, boolean present) {
-        boolean found = false;
-        for (int i = 0; i < maxFreq; i++) {
-            assert freqs[i] == null || freqs[i].prev != null;
-            assert freqs[i] == null || freqs[i].prev != freqs[i] || freqs[i].next == null;
-            assert freqs[i] == null || freqs[i].prev.next == null;
-            for (Entry<CacheFileRegion> entry = freqs[i]; entry != null; entry = entry.next) {
-                assert entry.next == null || entry.next.prev == entry;
-                assert entry.prev != null;
-                assert entry.prev.next == null || entry.prev.next == entry;
-                assert entry.freq == i;
-                if (entry == e) {
-                    found = true;
-                }
-            }
-            for (Entry<CacheFileRegion> entry = freqs[i]; entry != null && entry.prev != freqs[i]; entry = entry.prev) {
-                assert entry.next == null || entry.next.prev == entry;
-                assert entry.prev != null;
-                assert entry.prev.next == null || entry.prev.next == entry;
-                assert entry.freq == i;
-                if (entry == e) {
-                    found = true;
-                }
-            }
-        }
-        assert found == present;
-        return true;
-    }
-
-    private void maybeEvict() {
-        assert Thread.holdsLock(this);
-        for (int i = 0; i < maxFreq; i++) {
-            for (Entry<CacheFileRegion> entry = freqs[i]; entry != null; entry = entry.next) {
-                boolean evicted = entry.chunk.tryEvict();
-                if (evicted) {
-                    unlink(entry);
-                    keyMapping.remove(entry.chunk.regionKey, entry);
-                    return;
-                }
-            }
-        }
-    }
-
-    private void pushEntryToBack(final Entry<CacheFileRegion> entry) {
-        assert Thread.holdsLock(this);
-        assert invariant(entry, false);
-        assert entry.prev == null;
-        assert entry.next == null;
-        final Entry<CacheFileRegion> currFront = freqs[entry.freq];
-        if (currFront == null) {
-            freqs[entry.freq] = entry;
-            entry.prev = entry;
-            entry.next = null;
-        } else {
-            assert currFront.freq == entry.freq;
-            final Entry<CacheFileRegion> last = currFront.prev;
-            currFront.prev = entry;
-            last.next = entry;
-            entry.prev = last;
-            entry.next = null;
-        }
-        assert freqs[entry.freq].prev == entry;
-        assert freqs[entry.freq].prev.next == null;
-        assert entry.prev != null;
-        assert entry.prev.next == null || entry.prev.next == entry;
-        assert entry.next == null;
-        assert invariant(entry, true);
-    }
-
-    private void unlink(final Entry<CacheFileRegion> entry) {
-        assert Thread.holdsLock(this);
-        assert invariant(entry, true);
-        assert entry.prev != null;
-        final Entry<CacheFileRegion> currFront = freqs[entry.freq];
-        assert currFront != null;
-        if (currFront == entry) {
-            freqs[entry.freq] = entry.next;
-            if (entry.next != null) {
-                assert entry.prev != entry;
-                entry.next.prev = entry.prev;
-            }
-        } else {
-            if (entry.next != null) {
-                entry.next.prev = entry.prev;
-            }
-            entry.prev.next = entry.next;
-            if (currFront.prev == entry) {
-                currFront.prev = entry.prev;
-            }
-        }
-        entry.next = null;
-        entry.prev = null;
-        assert invariant(entry, false);
-    }
-
-    private void computeDecay() {
-        synchronized (this) {
-            long now = currentTimeSupplier.getAsLong();
-            for (int i = 0; i < maxFreq; i++) {
-                for (Entry<CacheFileRegion> entry = freqs[i]; entry != null; entry = entry.next) {
-                    if (now - entry.lastAccessed >= 2 * minTimeDelta) {
-                        if (entry.freq > 0) {
-                            unlink(entry);
-                            entry.freq--;
-                            pushEntryToBack(entry);
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-    public void removeFromCache(CacheKey cacheKey) {
-        forceEvict(cacheKey::equals);
-    }
-
-    public void markShardAsEvictedInCache(String snapshotUUID, String snapshotIndexName, ShardId shardId) {
-        forceEvict(
-            k -> shardId.equals(k.getShardId())
-                && snapshotIndexName.equals(k.getSnapshotIndexName())
-                && snapshotUUID.equals(k.getSnapshotUUID())
-        );
-    }
-
-    private void forceEvict(Predicate<CacheKey> cacheKeyPredicate) {
-        final List<Entry<CacheFileRegion>> matchingEntries = new ArrayList<>();
-        keyMapping.forEach((key, value) -> {
-            if (cacheKeyPredicate.test(key.file)) {
-                matchingEntries.add(value);
-            }
-        });
-        if (matchingEntries.isEmpty() == false) {
-            synchronized (this) {
-                for (Entry<CacheFileRegion> entry : matchingEntries) {
-                    boolean evicted = entry.chunk.forceEvict();
-                    if (evicted) {
-                        unlink(entry);
-                        keyMapping.remove(entry.chunk.regionKey, entry);
-                    }
-                }
-            }
-        }
-    }
-
-    // used by tests
-    int getFreq(CacheFileRegion cacheFileRegion) {
-        return keyMapping.get(cacheFileRegion.regionKey).freq;
-    }
-
-    @Override
-    public void close() {
-        sharedBytes.decRef();
-        decayTask.close();
-    }
-
-    class CacheDecayTask extends AbstractAsyncTask {
-
-        CacheDecayTask(ThreadPool threadPool, TimeValue interval) {
-            super(logger, Objects.requireNonNull(threadPool), Objects.requireNonNull(interval), true);
-        }
-
-        @Override
-        protected boolean mustReschedule() {
-            return true;
-        }
-
-        @Override
-        public void runInternal() {
-            computeDecay();
-        }
-
-        @Override
-        protected String getThreadPool() {
-            return ThreadPool.Names.GENERIC;
-        }
-
-        @Override
-        public String toString() {
-            return "frozen_cache_decay_task";
-        }
-    }
-
-    private static class RegionKey {
-        RegionKey(CacheKey file, int region) {
-            this.file = file;
-            this.region = region;
-        }
-
-        final CacheKey file;
-        final int region;
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            RegionKey regionKey = (RegionKey) o;
-            return region == regionKey.region && file.equals(regionKey.file);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(file, region);
-        }
-
-        @Override
-        public String toString() {
-            return "Chunk{" + "file=" + file + ", region=" + region + '}';
-        }
-    }
-
-    static class Entry<T> {
-        final T chunk;
-        Entry<T> prev;
-        Entry<T> next;
-        int freq;
-        long lastAccessed;
-
-        Entry(T chunk, long lastAccessed) {
-            this.chunk = chunk;
-            this.lastAccessed = lastAccessed;
-        }
-    }
-
-    class CacheFileRegion extends AbstractRefCounted {
-        final RegionKey regionKey;
-        final SparseFileTracker tracker;
-        volatile int sharedBytesPos = -1;
-
-        CacheFileRegion(RegionKey regionKey, long regionSize) {
-            this.regionKey = regionKey;
-            assert regionSize > 0L;
-            tracker = new SparseFileTracker("file", regionSize);
-        }
-
-        public long physicalStartOffset() {
-            return sharedBytes.getPhysicalOffset(sharedBytesPos);
-        }
-
-        public long physicalEndOffset() {
-            return sharedBytes.getPhysicalOffset(sharedBytesPos + 1);
-        }
-
-        // If true this file region has been evicted from the cache and should not be used any more
-        private final AtomicBoolean evicted = new AtomicBoolean(false);
-
-        // tries to evict this chunk if noone is holding onto its resources anymore
-        public boolean tryEvict() {
-            if (refCount() <= 1 && evicted.compareAndSet(false, true)) {
-                logger.trace("evicted {} with channel offset {}", regionKey, physicalStartOffset());
-                evictCount.increment();
-                decRef();
-                return true;
-            }
-            return false;
-        }
-
-        public boolean forceEvict() {
-            if (evicted.compareAndSet(false, true)) {
-                logger.trace("force evicted {} with channel offset {}", regionKey, physicalStartOffset());
-                evictCount.increment();
-                decRef();
-                return true;
-            }
-            return false;
-        }
-
-        public boolean isEvicted() {
-            return evicted.get();
-        }
-
-        @Override
-        protected void closeInternal() {
-            // now actually free the region associated with this chunk
-            onClose(this);
-            logger.trace("closed {} with channel offset {}", regionKey, physicalStartOffset());
-        }
-
-        private void ensureOpen() {
-            if (evicted.get()) {
-                throwAlreadyEvicted();
-            }
-        }
-
-        private static void throwAlreadyEvicted() {
-            throw new AlreadyClosedException("File chunk is evicted");
-        }
-
-        StepListener<Integer> populateAndRead(
-            final ByteRange rangeToWrite,
-            final ByteRange rangeToRead,
-            final RangeAvailableHandler reader,
-            final RangeMissingHandler writer,
-            final Executor executor
-        ) {
-            assert rangeToRead.length() > 0;
-            final StepListener<Integer> listener = new StepListener<>();
-            Releasable decrementRef = null;
-            try {
-                ensureOpen();
-                incRef();
-                decrementRef = Releasables.releaseOnce(this::decRef);
-                ensureOpen();
-                Releasable finalDecrementRef = decrementRef;
-                listener.whenComplete(integer -> finalDecrementRef.close(), throwable -> finalDecrementRef.close());
-                final SharedBytes.IO fileChannel = sharedBytes.getFileChannel(sharedBytesPos);
-                listener.whenComplete(integer -> fileChannel.decRef(), e -> fileChannel.decRef());
-                final ActionListener<Void> rangeListener = rangeListener(rangeToRead, reader, listener, fileChannel);
-                final List<SparseFileTracker.Gap> gaps = tracker.waitForRange(rangeToWrite, rangeToRead, rangeListener);
-
-                for (SparseFileTracker.Gap gap : gaps) {
-                    executor.execute(new AbstractRunnable() {
-
-                        @Override
-                        protected void doRun() throws Exception {
-                            if (CacheFileRegion.this.tryIncRef() == false) {
-                                throw new AlreadyClosedException("Cache file channel has been released and closed");
-                            }
-                            try {
-                                ensureOpen();
-                                final long start = gap.start();
-                                assert regionOwners[sharedBytesPos].get() == CacheFileRegion.this;
-                                writer.fillCacheRange(
-                                    fileChannel,
-                                    physicalStartOffset() + gap.start(),
-                                    gap.start(),
-                                    gap.end() - gap.start(),
-                                    progress -> gap.onProgress(start + progress)
-                                );
-                                writeCount.increment();
-                            } finally {
-                                decRef();
-                            }
-                            gap.onCompletion();
-                        }
-
-                        @Override
-                        public void onFailure(Exception e) {
-                            gap.onFailure(e);
-                        }
-                    });
-                }
-            } catch (Exception e) {
-                releaseAndFail(listener, decrementRef, e);
-            }
-            return listener;
-        }
-
-        private ActionListener<Void> rangeListener(
-            ByteRange rangeToRead,
-            RangeAvailableHandler reader,
-            ActionListener<Integer> listener,
-            SharedBytes.IO fileChannel
-        ) {
-            return ActionListener.wrap(success -> {
-                final long physicalStartOffset = physicalStartOffset();
-                assert regionOwners[sharedBytesPos].get() == CacheFileRegion.this;
-                final int read = reader.onRangeAvailable(
-                    fileChannel,
-                    physicalStartOffset + rangeToRead.start(),
-                    rangeToRead.start(),
-                    rangeToRead.length()
-                );
-                assert read == rangeToRead.length()
-                    : "partial read ["
-                        + read
-                        + "] does not match the range to read ["
-                        + rangeToRead.end()
-                        + '-'
-                        + rangeToRead.start()
-                        + ']';
-                readCount.increment();
-                listener.onResponse(read);
-            }, listener::onFailure);
-        }
-
-        private static void releaseAndFail(ActionListener<Integer> listener, Releasable decrementRef, Exception e) {
-            try {
-                Releasables.close(decrementRef);
-            } catch (Exception ex) {
-                e.addSuppressed(ex);
-            }
-            listener.onFailure(e);
-        }
-
-        @Override
-        protected void alreadyClosed() {
-            throwAlreadyEvicted();
-        }
-    }
-
-    public class FrozenCacheFile {
-
-        private final CacheKey cacheKey;
-        private final long length;
-
-        private FrozenCacheFile(CacheKey cacheKey, long length) {
-            this.cacheKey = cacheKey;
-            this.length = length;
-        }
-
-        public long getLength() {
-            return length;
-        }
-
-        public CacheKey getCacheKey() {
-            return cacheKey;
-        }
-
-        public StepListener<Integer> populateAndRead(
-            final ByteRange rangeToWrite,
-            final ByteRange rangeToRead,
-            final RangeAvailableHandler reader,
-            final RangeMissingHandler writer,
-            final Executor executor
-        ) {
-            StepListener<Integer> stepListener = null;
-            final long writeStart = rangeToWrite.start();
-            final long readStart = rangeToRead.start();
-            for (int region = getRegion(rangeToWrite.start()); region <= getEndingRegion(rangeToWrite.end()); region++) {
-                final ByteRange subRangeToWrite = mapSubRangeToRegion(rangeToWrite, region);
-                final ByteRange subRangeToRead = mapSubRangeToRegion(rangeToRead, region);
-                if (subRangeToRead.length() == 0L) {
-                    // nothing to read, skip
-                    if (stepListener == null) {
-                        stepListener = new StepListener<>();
-                        stepListener.onResponse(0);
-                    }
-                    continue;
-                }
-                final CacheFileRegion fileRegion = get(cacheKey, length, region);
-                final long regionStart = getRegionStart(region);
-                final long writeOffset = writeStart - regionStart;
-                final long readOffset = readStart - regionStart;
-                final StepListener<Integer> lis = fileRegion.populateAndRead(
-                    subRangeToWrite,
-                    subRangeToRead,
-                    (channel, channelPos, relativePos, len) -> {
-                        assert regionOwners[fileRegion.sharedBytesPos].get() == fileRegion;
-                        assert channelPos >= fileRegion.physicalStartOffset() && channelPos + len <= fileRegion.physicalEndOffset();
-                        return reader.onRangeAvailable(channel, channelPos, relativePos - readOffset, len);
-                    },
-                    (channel, channelPos, relativePos, len, progressUpdater) -> {
-                        assert regionOwners[fileRegion.sharedBytesPos].get() == fileRegion;
-                        assert channelPos >= fileRegion.physicalStartOffset() && channelPos + len <= fileRegion.physicalEndOffset();
-                        writer.fillCacheRange(channel, channelPos, relativePos - writeOffset, len, progressUpdater);
-                    },
-                    executor
-                );
-                assert lis != null;
-                if (stepListener == null) {
-                    stepListener = lis;
-                } else {
-                    stepListener = stepListener.thenCombine(lis, Math::addExact);
-                }
-
-            }
-            return stepListener;
-        }
-
-        @Override
-        public String toString() {
-            return "FrozenCacheFile{" + "cacheKey=" + cacheKey + ", length=" + length + '}';
-        }
-    }
-
-    public FrozenCacheFile getFrozenCacheFile(CacheKey cacheKey, long length) {
-        return new FrozenCacheFile(cacheKey, length);
-    }
-
-    @FunctionalInterface
-    public interface RangeAvailableHandler {
-        // caller that wants to read from x should instead do a positional read from x + relativePos
-        // caller should also only read up to length, further bytes will be offered by another call to this method
-        int onRangeAvailable(SharedBytes.IO channel, long channelPos, long relativePos, long length) throws IOException;
-    }
-
-    @FunctionalInterface
-    public interface RangeMissingHandler {
-        void fillCacheRange(SharedBytes.IO channel, long channelPos, long relativePos, long length, Consumer<Long> progressUpdater)
-            throws IOException;
-    }
-
-    public static class Stats {
-
-        public static final Stats EMPTY = new Stats(0, 0L, 0L, 0L, 0L, 0L, 0L, 0L);
-
-        private final int numberOfRegions;
-        private final long size;
-        private final long regionSize;
-        private final long evictCount;
-        private final long writeCount;
-        private final long writeBytes;
-        private final long readCount;
-        private final long readBytes;
-
-        private Stats(
-            int numberOfRegions,
-            long size,
-            long regionSize,
-            long evictCount,
-            long writeCount,
-            long writeBytes,
-            long readCount,
-            long readBytes
-        ) {
-            this.numberOfRegions = numberOfRegions;
-            this.size = size;
-            this.regionSize = regionSize;
-            this.evictCount = evictCount;
-            this.writeCount = writeCount;
-            this.writeBytes = writeBytes;
-            this.readCount = readCount;
-            this.readBytes = readBytes;
-        }
-
-        public int getNumberOfRegions() {
-            return numberOfRegions;
-        }
-
-        public long getSize() {
-            return size;
-        }
-
-        public long getRegionSize() {
-            return regionSize;
-        }
-
-        public long getEvictCount() {
-            return evictCount;
-        }
-
-        public long getWriteCount() {
-            return writeCount;
-        }
-
-        public long getWriteBytes() {
-            return writeBytes;
-        }
-
-        public long getReadCount() {
-            return readCount;
-        }
-
-        public long getReadBytes() {
-            return readBytes;
-        }
-    }
-}
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectory.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectory.java
index 869622844c03..fab396be75c0 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectory.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectory.java
@@ -20,7 +20,10 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.StepListener;
-import org.elasticsearch.action.support.GroupedActionListener;
+import org.elasticsearch.action.support.CountDownActionListener;
+import org.elasticsearch.blobcache.common.ByteRange;
+import org.elasticsearch.blobcache.shared.SharedBlobCacheService;
+import org.elasticsearch.blobcache.store.InMemoryNoOpCommitDirectory;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.common.blobstore.BlobContainer;
@@ -51,12 +54,9 @@
 import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots;
 import org.elasticsearch.xpack.searchablesnapshots.cache.blob.BlobStoreCacheService;
 import org.elasticsearch.xpack.searchablesnapshots.cache.blob.CachedBlob;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.ByteRange;
 import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheFile;
 import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey;
 import org.elasticsearch.xpack.searchablesnapshots.cache.full.CacheService;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService.FrozenCacheFile;
 import org.elasticsearch.xpack.searchablesnapshots.recovery.SearchableSnapshotRecoveryState;
 import org.elasticsearch.xpack.searchablesnapshots.store.input.CachedBlobContainerIndexInput;
 import org.elasticsearch.xpack.searchablesnapshots.store.input.ChecksumBlobContainerIndexInput;
@@ -135,7 +135,7 @@ public class SearchableSnapshotDirectory extends BaseDirectory {
     private final ShardPath shardPath;
     private final AtomicBoolean closed;
     private final boolean partial;
-    private final FrozenCacheService frozenCacheService;
+    private final SharedBlobCacheService<CacheKey> sharedBlobCacheService;
     private final ByteSizeValue blobStoreCacheMaxLength;
 
     // volatile fields are updated once under `this` lock, all together, iff loaded is not true.
@@ -158,7 +158,7 @@ public SearchableSnapshotDirectory(
         Path cacheDir,
         ShardPath shardPath,
         ThreadPool threadPool,
-        FrozenCacheService frozenCacheService
+        SharedBlobCacheService<CacheKey> sharedBlobCacheService
     ) {
         super(new SingleInstanceLockFactory());
         this.snapshotSupplier = Objects.requireNonNull(snapshot);
@@ -182,7 +182,7 @@ public SearchableSnapshotDirectory(
         this.blobStoreCacheMaxLength = SNAPSHOT_BLOB_CACHE_METADATA_FILES_MAX_LENGTH_SETTING.get(indexSettings);
         this.threadPool = threadPool;
         this.loaded = false;
-        this.frozenCacheService = frozenCacheService;
+        this.sharedBlobCacheService = sharedBlobCacheService;
         assert invariant();
     }
 
@@ -352,7 +352,7 @@ public void clearCache(boolean clearCacheService, boolean clearFrozenCacheServic
                 cacheService.removeFromCache(cacheKey);
             }
             if (clearFrozenCacheService) {
-                frozenCacheService.removeFromCache(cacheKey);
+                sharedBlobCacheService.removeFromCache(cacheKey);
             }
         }
     }
@@ -394,7 +394,7 @@ public IndexInput openInput(final String name, final IOContext context) throws I
         final IndexInputStats inputStats = stats.computeIfAbsent(ext, n -> {
             final IndexInputStats.Counter counter = new IndexInputStats.Counter();
             for (BlobStoreIndexShardSnapshot.FileInfo file : files()) {
-                if (ext.equals(getNonNullFileExt(file.physicalName()))) {
+                if (n.equals(getNonNullFileExt(file.physicalName()))) {
                     counter.add(file.length());
                 }
             }
@@ -408,8 +408,8 @@ public IndexInput openInput(final String name, final IOContext context) throws I
                     fileInfo,
                     context,
                     inputStats,
-                    frozenCacheService.getRangeSize(),
-                    frozenCacheService.getRecoveryRangeSize()
+                    sharedBlobCacheService.getRangeSize(),
+                    sharedBlobCacheService.getRecoveryRangeSize()
                 );
             } else {
                 return new CachedBlobContainerIndexInput(
@@ -423,7 +423,7 @@ public IndexInput openInput(final String name, final IOContext context) throws I
                 );
             }
         } else {
-            return new DirectBlobContainerIndexInput(name, this, fileInfo, context, inputStats, getUncachedChunkSize());
+            return new DirectBlobContainerIndexInput(name, blobContainer(), fileInfo, context, inputStats, getUncachedChunkSize());
         }
     }
 
@@ -484,10 +484,13 @@ private void prewarmCache(ActionListener<Void> listener) {
         final BlockingQueue<Tuple<ActionListener<Void>, CheckedRunnable<Exception>>> queue = new LinkedBlockingQueue<>();
         final Executor executor = prewarmExecutor();
 
-        final GroupedActionListener<Void> completionListener = new GroupedActionListener<>(ActionListener.wrap(voids -> {
-            recoveryState.setPreWarmComplete();
-            listener.onResponse(null);
-        }, listener::onFailure), snapshot().totalFileCount());
+        final CountDownActionListener completionListener = new CountDownActionListener(
+            snapshot().totalFileCount(),
+            ActionListener.wrap(ignored -> {
+                recoveryState.setPreWarmComplete();
+                listener.onResponse(null);
+            }, listener::onFailure)
+        );
 
         for (BlobStoreIndexShardSnapshot.FileInfo file : snapshot().indexFiles()) {
             boolean hashEqualsContents = file.metadata().hashEqualsContents();
@@ -507,9 +510,9 @@ private void prewarmCache(ActionListener<Void> listener) {
                 assert input instanceof CachedBlobContainerIndexInput : "expected cached index input but got " + input.getClass();
 
                 final int numberOfParts = file.numberOfParts();
-                final StepListener<Collection<Void>> fileCompletionListener = new StepListener<>();
-                fileCompletionListener.addListener(completionListener.map(voids -> null));
-                fileCompletionListener.whenComplete(voids -> {
+                final StepListener<Void> fileCompletionListener = new StepListener<>();
+                fileCompletionListener.addListener(completionListener);
+                fileCompletionListener.whenComplete(ignored -> {
                     logger.debug("{} file [{}] prewarmed", shardId, file.physicalName());
                     input.close();
                 }, e -> {
@@ -517,7 +520,7 @@ private void prewarmCache(ActionListener<Void> listener) {
                     IOUtils.closeWhileHandlingException(input);
                 });
 
-                final GroupedActionListener<Void> partsListener = new GroupedActionListener<>(fileCompletionListener, numberOfParts);
+                final CountDownActionListener partsListener = new CountDownActionListener(numberOfParts, fileCompletionListener);
                 submitted = true;
                 for (int p = 0; p < numberOfParts; p++) {
                     final int part = p;
@@ -530,7 +533,7 @@ private void prewarmCache(ActionListener<Void> listener) {
                         if (persistentCacheLength == file.length()) {
                             recoveryState.markIndexFileAsReused(file.physicalName());
                         } else {
-                            recoveryState.getIndex().addRecoveredBytesToFile(file.physicalName(), file.partBytes(part));
+                            recoveryState.getIndex().addRecoveredFromSnapshotBytesToFile(file.physicalName(), file.partBytes(part));
                         }
 
                         logger.trace(
@@ -545,7 +548,7 @@ private void prewarmCache(ActionListener<Void> listener) {
                         );
                     }));
                 }
-            } catch (IOException e) {
+            } catch (Exception e) {
                 logger.warn(() -> format("%s unable to prewarm file [%s]", shardId, file.physicalName()), e);
                 if (submitted == false) {
                     completionListener.onFailure(e);
@@ -583,7 +586,7 @@ public static Directory create(
         LongSupplier currentTimeNanosSupplier,
         ThreadPool threadPool,
         BlobStoreCacheService blobStoreCacheService,
-        FrozenCacheService frozenCacheService
+        SharedBlobCacheService<CacheKey> sharedBlobCacheService
     ) throws IOException {
 
         if (SNAPSHOT_REPOSITORY_NAME_SETTING.exists(indexSettings.getSettings()) == false
@@ -667,7 +670,7 @@ public static Directory create(
                 cacheDir,
                 shardPath,
                 threadPool,
-                frozenCacheService
+                sharedBlobCacheService
             )
         );
     }
@@ -709,8 +712,8 @@ public void putCachedBlob(String name, ByteRange range, BytesReference content,
         );
     }
 
-    public FrozenCacheFile getFrozenCacheFile(String fileName, long length) {
-        return frozenCacheService.getFrozenCacheFile(createCacheKey(fileName), length);
+    public SharedBlobCacheService<CacheKey>.CacheFile getFrozenCacheFile(String fileName, long length) {
+        return sharedBlobCacheService.getCacheFile(createCacheKey(fileName), length);
     }
 
     private static Repository repositoryByUuid(Map<String, Repository> repositories, String repositoryUuid, String originalName) {
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/BaseSearchableSnapshotIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/BaseSearchableSnapshotIndexInput.java
index ec15a12753fe..be009575cd7f 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/BaseSearchableSnapshotIndexInput.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/BaseSearchableSnapshotIndexInput.java
@@ -15,9 +15,7 @@
 import org.elasticsearch.index.snapshots.blobstore.SlicedInputStream;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.ByteRange;
 import org.elasticsearch.xpack.searchablesnapshots.store.IndexInputStats;
-import org.elasticsearch.xpack.searchablesnapshots.store.SearchableSnapshotDirectory;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -26,14 +24,13 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Predicate;
 
-import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsUtils.toIntBytes;
+import static org.elasticsearch.blobcache.BlobCacheUtils.toIntBytes;
 import static org.elasticsearch.xpack.searchablesnapshots.store.input.ChecksumBlobContainerIndexInput.checksumToBytesArray;
 
 public abstract class BaseSearchableSnapshotIndexInput extends BufferedIndexInput {
 
     protected final Logger logger;
     protected final String name;
-    protected final SearchableSnapshotDirectory directory;
     protected final BlobContainer blobContainer;
     protected final FileInfo fileInfo;
     protected final IOContext context;
@@ -41,45 +38,28 @@ public abstract class BaseSearchableSnapshotIndexInput extends BufferedIndexInpu
     protected final long offset;
     protected final long length;
 
-    /**
-     * Range of bytes that should be cached in the blob cache for the current index input's header.
-     */
-    protected final ByteRange headerBlobCacheByteRange;
-
-    /**
-     * Range of bytes that should be cached in the blob cache for the current index input's footer. This footer byte range should only be
-     * required for slices of CFS files; regular files already have their footers extracted from the {@link FileInfo} (see method
-     * {@link BaseSearchableSnapshotIndexInput#maybeReadChecksumFromFileInfo}).
-     */
-    protected final ByteRange footerBlobCacheByteRange;
-
     // the following are only mutable so they can be adjusted after cloning/slicing
     protected volatile boolean isClone;
-    private AtomicBoolean closed;
+    protected AtomicBoolean closed;
 
     public BaseSearchableSnapshotIndexInput(
         Logger logger,
         String name,
-        SearchableSnapshotDirectory directory,
+        BlobContainer blobContainer,
         FileInfo fileInfo,
         IOContext context,
         IndexInputStats stats,
         long offset,
-        long length,
-        ByteRange headerBlobCacheByteRange,
-        ByteRange footerBlobCacheByteRange
+        long length
     ) {
         super(name, context);
         this.name = Objects.requireNonNull(name);
         this.logger = Objects.requireNonNull(logger);
-        this.directory = Objects.requireNonNull(directory);
-        this.blobContainer = Objects.requireNonNull(directory.blobContainer());
+        this.blobContainer = Objects.requireNonNull(blobContainer);
         this.fileInfo = Objects.requireNonNull(fileInfo);
         this.context = Objects.requireNonNull(context);
         assert fileInfo.metadata().hashEqualsContents() == false
             : "this method should only be used with blobs that are NOT stored in metadata's hash field " + "(fileInfo: " + fileInfo + ')';
-        this.headerBlobCacheByteRange = Objects.requireNonNull(headerBlobCacheByteRange);
-        this.footerBlobCacheByteRange = Objects.requireNonNull(footerBlobCacheByteRange);
         this.stats = Objects.requireNonNull(stats);
         this.offset = offset;
         this.length = length;
@@ -155,16 +135,6 @@ private boolean maybeReadChecksumFromFileInfo(ByteBuffer b) throws IOException {
         return success;
     }
 
-    protected ByteRange rangeToReadFromBlobCache(long position, int readLength) {
-        final long end = position + readLength;
-        if (headerBlobCacheByteRange.contains(position, end)) {
-            return headerBlobCacheByteRange;
-        } else if (footerBlobCacheByteRange.contains(position, end)) {
-            return footerBlobCacheByteRange;
-        }
-        return ByteRange.EMPTY;
-    }
-
     /**
      * Opens an {@link InputStream} for the given range of bytes which reads the data directly from the blob store. If the requested range
      * spans multiple blobs then this stream will request them in turn.
@@ -175,10 +145,10 @@ protected ByteRange rangeToReadFromBlobCache(long position, int readLength) {
     protected InputStream openInputStreamFromBlobStore(final long position, final long readLength) throws IOException {
         assert assertCurrentThreadMayAccessBlobStore();
         if (fileInfo.numberOfParts() == 1L) {
-            assert position + readLength <= fileInfo.partBytes(0)
+            assert position + readLength <= fileInfo.length()
                 : "cannot read [" + position + "-" + (position + readLength) + "] from [" + fileInfo + "]";
             stats.addBlobStoreBytesRequested(readLength);
-            return blobContainer.readBlob(fileInfo.partName(0), position, readLength);
+            return blobContainer.readBlob(fileInfo.name(), position, readLength);
         } else {
             final int startPart = getPartNumberForPosition(position);
             final int endPart = getPartNumberForPosition(position + readLength - 1);
@@ -262,12 +232,6 @@ protected String getFullSliceDescription(String sliceDescription) {
         return resourceDesc;
     }
 
-    protected void ensureOpen() throws IOException {
-        if (closed.get()) {
-            throw new IOException(toString() + " is closed");
-        }
-    }
-
     @Override
     public final void close() throws IOException {
         if (closed.compareAndSet(false, true)) {
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInput.java
index c0f112accfd6..b595d0bb042b 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInput.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInput.java
@@ -9,13 +9,10 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
+import org.elasticsearch.blobcache.common.ByteRange;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.ByteRange;
 import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheFile;
 import org.elasticsearch.xpack.searchablesnapshots.store.IndexInputStats;
 import org.elasticsearch.xpack.searchablesnapshots.store.SearchableSnapshotDirectory;
@@ -27,7 +24,8 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.IntStream;
 
-import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsUtils.toIntBytes;
+import static org.elasticsearch.blobcache.BlobCacheUtils.readSafe;
+import static org.elasticsearch.blobcache.BlobCacheUtils.toIntBytes;
 
 public class CachedBlobContainerIndexInput extends MetadataCachingIndexInput {
 
@@ -124,14 +122,7 @@ protected void readWithoutBlobCache(ByteBuffer b) throws Exception {
         assert rangeToRead.isSubRangeOf(rangeToWrite) : rangeToRead + " vs " + rangeToWrite;
         assert rangeToRead.length() == b.remaining() : b.remaining() + " vs " + rangeToRead;
 
-        final Future<Integer> populateCacheFuture = cacheFile.populateAndRead(
-            rangeToWrite,
-            rangeToRead,
-            channel -> readCacheFile(channel, position, b),
-            this::writeCacheFile,
-            directory.cacheFetchAsyncExecutor()
-        );
-
+        final Future<Integer> populateCacheFuture = populateAndRead(b, position, length, cacheFile, rangeToWrite);
         final int bytesRead = populateCacheFuture.get();
         assert bytesRead == length : bytesRead + " vs " + length;
     }
@@ -172,9 +163,7 @@ public Tuple<Long, Long> prefetchPart(final int part) throws IOException {
                 range.end(),
                 cacheFileReference
             );
-
-            final byte[] copyBuffer = new byte[toIntBytes(Math.min(COPY_BUFFER_SIZE, range.length()))];
-
+            final ByteBuffer copyBuffer = writeBuffer.get();
             long totalBytesRead = 0L;
             final AtomicLong totalBytesWritten = new AtomicLong();
             long remainingBytes = range.length();
@@ -182,7 +171,8 @@ public Tuple<Long, Long> prefetchPart(final int part) throws IOException {
             try (InputStream input = openInputStreamFromBlobStore(range.start(), range.length())) {
                 while (remainingBytes > 0L) {
                     assert totalBytesRead + remainingBytes == range.length();
-                    final int bytesRead = readSafe(input, copyBuffer, range.start(), range.end(), remainingBytes, cacheFileReference);
+                    copyBuffer.clear();
+                    final int bytesRead = readSafe(input, copyBuffer, range.start(), remainingBytes, cacheFileReference);
 
                     // The range to prewarm in cache
                     final long readStart = range.start() + totalBytesRead;
@@ -192,8 +182,11 @@ public Tuple<Long, Long> prefetchPart(final int part) throws IOException {
                     // noinspection UnnecessaryLocalVariable
                     final ByteRange rangeToRead = rangeToWrite;
                     cacheFile.populateAndRead(rangeToWrite, rangeToRead, (channel) -> bytesRead, (channel, start, end, progressUpdater) -> {
-                        final ByteBuffer byteBuffer = ByteBuffer.wrap(copyBuffer, toIntBytes(start - readStart), toIntBytes(end - start));
-                        final int writtenBytes = positionalWrite(channel, start, byteBuffer);
+                        final int writtenBytes = positionalWrite(
+                            channel,
+                            start,
+                            copyBuffer.slice(toIntBytes(start - readStart), toIntBytes(end - start))
+                        );
                         logger.trace(
                             "prefetchPart: writing range [{}-{}] of file [{}], [{}] bytes written",
                             start,
@@ -240,47 +233,15 @@ public CachedBlobContainerIndexInput clone() {
     }
 
     @Override
-    public IndexInput slice(String sliceName, long sliceOffset, long sliceLength) {
-        if (sliceOffset < 0 || sliceLength < 0 || sliceOffset + sliceLength > length()) {
-            throw new IllegalArgumentException(
-                "slice() "
-                    + sliceName
-                    + " out of bounds: offset="
-                    + sliceOffset
-                    + ",length="
-                    + sliceLength
-                    + ",fileLength="
-                    + length()
-                    + ": "
-                    + this
-            );
-        }
-
-        // Are we creating a slice from a CFS file?
-        final boolean sliceCompoundFile = IndexFileNames.matchesExtension(name, "cfs")
-            && IndexFileNames.getExtension(sliceName) != null
-            && compoundFileOffset == 0L // not already a compound file
-            && isClone == false; // tests aggressively clone and slice
-
-        final ByteRange sliceHeaderByteRange;
-        final ByteRange sliceFooterByteRange;
-        final long sliceCompoundFileOffset;
-
-        if (sliceCompoundFile) {
-            sliceCompoundFileOffset = this.offset + sliceOffset;
-            sliceHeaderByteRange = directory.getBlobCacheByteRange(sliceName, sliceLength).shift(sliceCompoundFileOffset);
-            if (sliceHeaderByteRange.isEmpty() == false && sliceHeaderByteRange.length() < sliceLength) {
-                sliceFooterByteRange = ByteRange.of(sliceLength - CodecUtil.footerLength(), sliceLength).shift(sliceCompoundFileOffset);
-            } else {
-                sliceFooterByteRange = ByteRange.EMPTY;
-            }
-        } else {
-            sliceCompoundFileOffset = this.compoundFileOffset;
-            sliceHeaderByteRange = ByteRange.EMPTY;
-            sliceFooterByteRange = ByteRange.EMPTY;
-        }
-
-        final CachedBlobContainerIndexInput slice = new CachedBlobContainerIndexInput(
+    protected MetadataCachingIndexInput doSlice(
+        String sliceName,
+        long sliceOffset,
+        long sliceLength,
+        ByteRange sliceHeaderByteRange,
+        ByteRange sliceFooterByteRange,
+        long sliceCompoundFileOffset
+    ) {
+        return new CachedBlobContainerIndexInput(
             sliceName,
             directory,
             fileInfo,
@@ -295,8 +256,6 @@ public IndexInput slice(String sliceName, long sliceOffset, long sliceLength) {
             sliceHeaderByteRange,
             sliceFooterByteRange
         );
-        slice.isClone = true;
-        return slice;
     }
 
     @Override
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/ChecksumBlobContainerIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/ChecksumBlobContainerIndexInput.java
index db02872a89c8..552c65a6f255 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/ChecksumBlobContainerIndexInput.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/ChecksumBlobContainerIndexInput.java
@@ -18,7 +18,7 @@
 import java.util.Arrays;
 import java.util.Objects;
 
-import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsUtils.toIntBytes;
+import static org.elasticsearch.blobcache.BlobCacheUtils.toIntBytes;
 
 /**
  * A {@link IndexInput} that can only be used to verify footer checksums.
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInput.java
index e305e7a026dc..ec3042dfc41c 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInput.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInput.java
@@ -10,15 +10,15 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
+import org.elasticsearch.blobcache.BlobCacheUtils;
 import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.core.CheckedRunnable;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.Streams;
 import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.ByteRange;
 import org.elasticsearch.xpack.searchablesnapshots.store.IndexInputStats;
-import org.elasticsearch.xpack.searchablesnapshots.store.SearchableSnapshotDirectory;
 
 import java.io.Closeable;
 import java.io.EOFException;
@@ -29,7 +29,7 @@
 import java.util.Objects;
 import java.util.concurrent.atomic.LongAdder;
 
-import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsUtils.toIntBytes;
+import static org.elasticsearch.blobcache.BlobCacheUtils.toIntBytes;
 
 /**
  * A {@link DirectBlobContainerIndexInput} instance corresponds to a single file from a Lucene directory that has been snapshotted. Because
@@ -65,23 +65,22 @@ public class DirectBlobContainerIndexInput extends BaseSearchableSnapshotIndexIn
     private StreamForSequentialReads streamForSequentialReads;
     private long sequentialReadSize;
     private static final long NO_SEQUENTIAL_READ_OPTIMIZATION = 0L;
-    private static final int COPY_BUFFER_SIZE = 8192;
 
     public DirectBlobContainerIndexInput(
         String name,
-        SearchableSnapshotDirectory directory,
+        BlobContainer blobContainer,
         FileInfo fileInfo,
         IOContext context,
         IndexInputStats stats,
         long sequentialReadSize
     ) {
-        this(name, directory, fileInfo, context, stats, 0L, 0L, fileInfo.length(), sequentialReadSize);
+        this(name, blobContainer, fileInfo, context, stats, 0L, 0L, fileInfo.length(), sequentialReadSize);
         stats.incrementOpenCount();
     }
 
     private DirectBlobContainerIndexInput(
         String name,
-        SearchableSnapshotDirectory directory,
+        BlobContainer blobContainer,
         FileInfo fileInfo,
         IOContext context,
         IndexInputStats stats,
@@ -90,8 +89,7 @@ private DirectBlobContainerIndexInput(
         long length,
         long sequentialReadSize
     ) {
-        super(logger, name, directory, fileInfo, context, stats, offset, length, ByteRange.EMPTY, ByteRange.EMPTY); // TODO should use blob
-                                                                                                                    // cache
+        super(logger, name, blobContainer, fileInfo, context, stats, offset, length); // TODO should use blob cache
         this.position = position;
         assert sequentialReadSize >= 0;
         this.sequentialReadSize = sequentialReadSize;
@@ -99,7 +97,9 @@ private DirectBlobContainerIndexInput(
 
     @Override
     protected void doReadInternal(ByteBuffer b) throws IOException {
-        ensureOpen();
+        if (closed.get()) {
+            throw new IOException(this + " is closed");
+        }
         if (fileInfo.numberOfParts() == 1) {
             readInternalBytes(0, position, b, b.remaining());
         } else {
@@ -245,11 +245,7 @@ public void close() throws IOException {
 
     @Override
     protected void seekInternal(long pos) throws IOException {
-        if (pos > length()) {
-            throw new EOFException("Reading past end of file [position=" + pos + ", length=" + length() + "] for " + toString());
-        } else if (pos < 0L) {
-            throw new IOException("Seeking to negative position [" + pos + "] for " + toString());
-        }
+        BlobCacheUtils.ensureSeek(pos, this);
         if (position != offset + pos) {
             position = offset + pos;
             closeStreamForSequentialReads();
@@ -268,28 +264,23 @@ public DirectBlobContainerIndexInput clone() {
 
     @Override
     public IndexInput slice(String sliceName, long offset, long length) throws IOException {
-        if ((offset >= 0L) && (length >= 0L) && (offset + length <= length())) {
-            final DirectBlobContainerIndexInput slice = new DirectBlobContainerIndexInput(
-                sliceName,
-                directory,
-                fileInfo,
-                context,
-                stats,
-                position,
-                this.offset + offset,
-                length,
-                // Slices might not be closed when they are no longer needed, but we must always close streamForSequentialReads. The simple
-                // solution: do not optimize sequential reads on slices.
-                NO_SEQUENTIAL_READ_OPTIMIZATION
-            );
-            slice.isClone = true;
-            slice.seek(0L);
-            return slice;
-        } else {
-            throw new IllegalArgumentException(
-                "slice() " + sliceName + " out of bounds: offset=" + offset + ",length=" + length + ",fileLength=" + length() + ": " + this
-            );
-        }
+        BlobCacheUtils.ensureSlice(sliceName, offset, length, this);
+        final DirectBlobContainerIndexInput slice = new DirectBlobContainerIndexInput(
+            sliceName,
+            blobContainer,
+            fileInfo,
+            context,
+            stats,
+            position,
+            this.offset + offset,
+            length,
+            // Slices might not be closed when they are no longer needed, but we must always close streamForSequentialReads. The simple
+            // solution: do not optimize sequential reads on slices.
+            NO_SEQUENTIAL_READ_OPTIMIZATION
+        );
+        slice.isClone = true;
+        slice.seek(0L);
+        return slice;
     }
 
     @Override
@@ -313,17 +304,9 @@ private InputStream openBlobStream(int part, long pos, long length) throws IOExc
      */
     private static int readFully(InputStream inputStream, final ByteBuffer b, int length, CheckedRunnable<IOException> onEOF)
         throws IOException {
-        int totalRead = 0;
-        final byte[] buffer = new byte[Math.min(length, COPY_BUFFER_SIZE)];
-        while (totalRead < length) {
-            final int len = Math.min(length - totalRead, COPY_BUFFER_SIZE);
-            final int read = inputStream.read(buffer, 0, len);
-            if (read == -1) {
-                onEOF.run();
-                break;
-            }
-            b.put(buffer, 0, read);
-            totalRead += read;
+        int totalRead = Streams.read(inputStream, b, length);
+        if (totalRead < length) {
+            onEOF.run();
         }
         return totalRead > 0 ? totalRead : -1;
     }
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInput.java
index 80c05a492b01..e743c1093056 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInput.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInput.java
@@ -9,38 +9,29 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.elasticsearch.action.StepListener;
-import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.blobcache.BlobCacheUtils;
+import org.elasticsearch.blobcache.common.ByteRange;
+import org.elasticsearch.blobcache.shared.SharedBlobCacheService;
+import org.elasticsearch.blobcache.shared.SharedBytes;
 import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.ByteRange;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService.FrozenCacheFile;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.SharedBytes;
+import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey;
 import org.elasticsearch.xpack.searchablesnapshots.store.IndexInputStats;
 import org.elasticsearch.xpack.searchablesnapshots.store.SearchableSnapshotDirectory;
 
-import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
-import java.util.Locale;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.function.Consumer;
-
-import static org.elasticsearch.core.Strings.format;
-import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsUtils.toIntBytes;
+import java.util.concurrent.Semaphore;
+import java.util.function.LongConsumer;
 
 public class FrozenIndexInput extends MetadataCachingIndexInput {
 
     private static final Logger logger = LogManager.getLogger(FrozenIndexInput.class);
 
-    private final FrozenCacheFile frozenCacheFile;
+    private final SharedBlobCacheService<CacheKey>.CacheFile cacheFile;
 
     public FrozenIndexInput(
         String name,
@@ -80,7 +71,7 @@ private FrozenIndexInput(
         long compoundFileOffset,
         long length,
         CacheFileReference cacheFileReference,
-        FrozenCacheFile frozenCacheFile,
+        SharedBlobCacheService<CacheKey>.CacheFile cacheFile,
         int defaultRangeSize,
         int recoveryRangeSize,
         ByteRange headerBlobCacheByteRange,
@@ -102,7 +93,7 @@ private FrozenIndexInput(
             headerBlobCacheByteRange,
             footerBlobCacheByteRange
         );
-        this.frozenCacheFile = frozenCacheFile;
+        this.cacheFile = cacheFile;
     }
 
     @Override
@@ -116,22 +107,12 @@ protected void readWithoutBlobCache(ByteBuffer b) throws Exception {
         final int length = b.remaining();
         final int originalByteBufPosition = b.position();
 
-        final ReentrantReadWriteLock luceneByteBufLock = new ReentrantReadWriteLock();
-        final AtomicBoolean stopAsyncReads = new AtomicBoolean();
-        // Runnable that, when called, ensures that async callbacks (such as those used by readCacheFile) are not
+        // Semaphore that, when all permits are acquired, ensures that async callbacks (such as those used by readCacheFile) are not
         // accessing the byte buffer anymore that was passed to readWithoutBlobCache
-        // In particular, it's important to call this method before adapting the ByteBuffer's offset
-        final Runnable preventAsyncBufferChanges = () -> {
-            luceneByteBufLock.writeLock().lock();
-            try {
-                stopAsyncReads.set(true);
-            } finally {
-                luceneByteBufLock.writeLock().unlock();
-            }
-        };
-
-        logger.trace("readInternal: read [{}-{}] ([{}] bytes) from [{}]", position, position + length, length, this);
-
+        // In particular, it's important to acquire all permits before adapting the ByteBuffer's offset
+        final Semaphore luceneByteBufPermits = new Semaphore(Integer.MAX_VALUE);
+        boolean bufferWriteLocked = false;
+        logger.trace("readInternal: read [{}-{}] from [{}]", position, position + length, this);
         try {
             final ByteRange startRangeToWrite = computeRange(position);
             final ByteRange endRangeToWrite = computeRange(position + length - 1);
@@ -142,7 +123,7 @@ protected void readWithoutBlobCache(ByteBuffer b) throws Exception {
                 : "[" + position + "-" + (position + length) + "] vs " + rangeToWrite;
             final ByteRange rangeToRead = ByteRange.of(position, position + length);
 
-            final StepListener<Integer> populateCacheFuture = frozenCacheFile.populateAndRead(
+            final int bytesRead = cacheFile.populateAndRead(
                 rangeToWrite,
                 rangeToRead,
                 (channel, pos, relativePos, len) -> readCacheFile(
@@ -152,9 +133,7 @@ protected void readWithoutBlobCache(ByteBuffer b) throws Exception {
                     len,
                     b,
                     rangeToRead.start(),
-                    false,
-                    luceneByteBufLock,
-                    stopAsyncReads
+                    luceneByteBufPermits
                 ),
                 (channel, channelPos, relativePos, len, progressUpdater) -> {
                     final long startTimeNanos = stats.currentTimeNanos();
@@ -164,17 +143,18 @@ protected void readWithoutBlobCache(ByteBuffer b) throws Exception {
                         writeCacheFile(channel, input, channelPos, relativePos, len, progressUpdater, startTimeNanos);
                     }
                 },
-                directory.cacheFetchAsyncExecutor()
+                SearchableSnapshots.CACHE_FETCH_ASYNC_THREAD_POOL_NAME
             );
-
-            final int bytesRead = populateCacheFuture.asFuture().get();
             assert bytesRead == length : bytesRead + " vs " + length;
-            assert luceneByteBufLock.getReadHoldCount() == 0;
+            assert luceneByteBufPermits.availablePermits() == Integer.MAX_VALUE;
 
-            preventAsyncBufferChanges.run();
+            luceneByteBufPermits.acquire(Integer.MAX_VALUE);
+            bufferWriteLocked = true;
             b.position(originalByteBufPosition + bytesRead); // mark all bytes as accounted for
         } finally {
-            preventAsyncBufferChanges.run();
+            if (bufferWriteLocked == false) {
+                luceneByteBufPermits.acquire(Integer.MAX_VALUE);
+            }
         }
     }
 
@@ -187,38 +167,6 @@ private static int positionalWrite(SharedBytes.IO fc, long start, ByteBuffer byt
         return written;
     }
 
-    /**
-     * Perform a single {@code read()} from {@code inputStream} into {@code copyBuffer}, handling an EOF by throwing an {@link EOFException}
-     * rather than returning {@code -1}. Returns the number of bytes read, which is always positive.
-     *
-     * Most of its arguments are there simply to make the message of the {@link EOFException} more informative.
-     */
-    private static int readSafe(
-        InputStream inputStream,
-        byte[] copyBuffer,
-        long rangeStart,
-        long rangeEnd,
-        long remaining,
-        FrozenCacheFile frozenCacheFile
-    ) throws IOException {
-        final int len = (remaining < copyBuffer.length) ? toIntBytes(remaining) : copyBuffer.length;
-        final int bytesRead = inputStream.read(copyBuffer, 0, len);
-        if (bytesRead == -1) {
-            throw new EOFException(
-                String.format(
-                    Locale.ROOT,
-                    "unexpected EOF reading [%d-%d] ([%d] bytes remaining) from %s",
-                    rangeStart,
-                    rangeEnd,
-                    remaining,
-                    frozenCacheFile
-                )
-            );
-        }
-        assert bytesRead > 0 : bytesRead;
-        return bytesRead;
-    }
-
     private int readCacheFile(
         final SharedBytes.IO fc,
         long channelPos,
@@ -226,53 +174,32 @@ private int readCacheFile(
         long length,
         final ByteBuffer buffer,
         long logicalPos,
-        boolean cached,
-        ReentrantReadWriteLock luceneByteBufLock,
-        AtomicBoolean stopAsyncReads
+        Semaphore luceneByteBufPermits
     ) throws IOException {
         logger.trace(
             "{}: reading cached {} logical {} channel {} pos {} length {} (details: {})",
             fileInfo.physicalName(),
-            cached,
+            false,
             logicalPos,
             channelPos,
             relativePos,
             length,
-            frozenCacheFile
+            cacheFile
         );
         if (length == 0L) {
             return 0;
         }
         final int bytesRead;
-        if (luceneByteBufLock.readLock().tryLock()) {
+        if (luceneByteBufPermits.tryAcquire()) {
             try {
-                boolean shouldStopReading = stopAsyncReads.get();
-                if (shouldStopReading) {
-                    // return fake response
-                    return Math.toIntExact(length);
-                }
                 // create slice that is positioned to read the given values
-                final ByteBuffer dup = buffer.duplicate();
-                final int newPosition = dup.position() + Math.toIntExact(relativePos);
-                assert newPosition <= dup.limit() : "newpos " + newPosition + " limit " + dup.limit();
-                assert newPosition + length <= buffer.limit()
-                    : "oldpos " + dup.position() + " newpos " + newPosition + " length " + length + " limit " + buffer.limit();
-                dup.position(newPosition);
-                dup.limit(newPosition + Math.toIntExact(length));
+                final ByteBuffer dup = buffer.slice(buffer.position() + Math.toIntExact(relativePos), Math.toIntExact(length));
                 bytesRead = fc.read(dup, channelPos);
                 if (bytesRead == -1) {
-                    throw new EOFException(
-                        String.format(
-                            Locale.ROOT,
-                            "unexpected EOF reading [%d-%d] from %s",
-                            channelPos,
-                            channelPos + dup.remaining(),
-                            this.frozenCacheFile
-                        )
-                    );
+                    BlobCacheUtils.throwEOF(channelPos, dup.remaining(), this.cacheFile);
                 }
             } finally {
-                luceneByteBufLock.readLock().unlock();
+                luceneByteBufPermits.release();
             }
         } else {
             // return fake response
@@ -282,27 +209,13 @@ private int readCacheFile(
         return bytesRead;
     }
 
-    /**
-     * Thread local direct byte buffer to aggregate multiple positional writes to the cache file.
-     */
-    private static final int MAX_BYTES_PER_WRITE = StrictMath.toIntExact(
-        ByteSizeValue.parseBytesSizeValue(
-            System.getProperty("es.searchable.snapshot.shared_cache.write_buffer.size", "2m"),
-            "es.searchable.snapshot.shared_cache.write_buffer.size"
-        ).getBytes()
-    );
-
-    private static final ThreadLocal<ByteBuffer> writeBuffer = ThreadLocal.withInitial(
-        () -> ByteBuffer.allocateDirect(MAX_BYTES_PER_WRITE)
-    );
-
     private void writeCacheFile(
         final SharedBytes.IO fc,
         final InputStream input,
         final long fileChannelPos,
         final long relativePos,
         final long length,
-        final Consumer<Long> progressUpdater,
+        final LongConsumer progressUpdater,
         final long startTimeNanos
     ) throws IOException {
         assert ThreadPool.assertCurrentThreadPool(SearchableSnapshots.CACHE_FETCH_ASYNC_THREAD_POOL_NAME);
@@ -312,31 +225,22 @@ private void writeCacheFile(
             fileChannelPos,
             relativePos,
             length,
-            frozenCacheFile
+            cacheFile
         );
         final long end = relativePos + length;
-        final byte[] copyBuffer = new byte[toIntBytes(Math.min(COPY_BUFFER_SIZE, length))];
-        logger.trace(() -> format("writing range [%s-%s] to cache file [%s]", relativePos, end, frozenCacheFile));
+        logger.trace("writing range [{}-{}] to cache file [{}]", relativePos, end, cacheFile);
 
         long bytesCopied = 0L;
         long remaining = length;
-        final ByteBuffer buf = writeBuffer.get();
-        buf.clear();
+        final ByteBuffer buf = writeBuffer.get().clear();
         while (remaining > 0L) {
-            final int bytesRead = readSafe(input, copyBuffer, relativePos, end, remaining, frozenCacheFile);
-            if (bytesRead > buf.remaining()) {
-                // always fill up buf to the max
-                final int bytesToAdd = buf.remaining();
-                buf.put(copyBuffer, 0, bytesToAdd);
-                assert buf.remaining() == 0;
-                long bytesWritten = positionalWrite(fc, fileChannelPos + bytesCopied, buf);
-                bytesCopied += bytesWritten;
-                progressUpdater.accept(bytesCopied);
-                // add the remaining bytes to buf
-                buf.put(copyBuffer, bytesToAdd, bytesRead - bytesToAdd);
-            } else {
-                buf.put(copyBuffer, 0, bytesRead);
+            final int bytesRead = BlobCacheUtils.readSafe(input, buf, relativePos, remaining, cacheFile);
+            if (buf.hasRemaining()) {
+                break;
             }
+            long bytesWritten = positionalWrite(fc, fileChannelPos + bytesCopied, buf);
+            bytesCopied += bytesWritten;
+            progressUpdater.accept(bytesCopied);
             remaining -= bytesRead;
         }
         // ensure that last write is aligned on 4k boundaries (= page size)
@@ -358,47 +262,15 @@ public FrozenIndexInput clone() {
     }
 
     @Override
-    public IndexInput slice(String sliceName, long sliceOffset, long sliceLength) {
-        if (sliceOffset < 0 || sliceLength < 0 || sliceOffset + sliceLength > length()) {
-            throw new IllegalArgumentException(
-                "slice() "
-                    + sliceName
-                    + " out of bounds: offset="
-                    + sliceOffset
-                    + ",length="
-                    + sliceLength
-                    + ",fileLength="
-                    + length()
-                    + ": "
-                    + this
-            );
-        }
-
-        // Are we creating a slice from a CFS file?
-        final boolean sliceCompoundFile = IndexFileNames.matchesExtension(name, "cfs")
-            && IndexFileNames.getExtension(sliceName) != null
-            && compoundFileOffset == 0L // not already a compound file
-            && isClone == false; // tests aggressively clone and slice
-
-        final ByteRange sliceHeaderByteRange;
-        final ByteRange sliceFooterByteRange;
-        final long sliceCompoundFileOffset;
-
-        if (sliceCompoundFile) {
-            sliceCompoundFileOffset = this.offset + sliceOffset;
-            sliceHeaderByteRange = directory.getBlobCacheByteRange(sliceName, sliceLength).shift(sliceCompoundFileOffset);
-            if (sliceHeaderByteRange.isEmpty() == false && sliceHeaderByteRange.length() < sliceLength) {
-                sliceFooterByteRange = ByteRange.of(sliceLength - CodecUtil.footerLength(), sliceLength).shift(sliceCompoundFileOffset);
-            } else {
-                sliceFooterByteRange = ByteRange.EMPTY;
-            }
-        } else {
-            sliceCompoundFileOffset = this.compoundFileOffset;
-            sliceHeaderByteRange = ByteRange.EMPTY;
-            sliceFooterByteRange = ByteRange.EMPTY;
-        }
-
-        final FrozenIndexInput slice = new FrozenIndexInput(
+    protected MetadataCachingIndexInput doSlice(
+        String sliceName,
+        long sliceOffset,
+        long sliceLength,
+        ByteRange sliceHeaderByteRange,
+        ByteRange sliceFooterByteRange,
+        long sliceCompoundFileOffset
+    ) {
+        return new FrozenIndexInput(
             sliceName,
             directory,
             fileInfo,
@@ -408,14 +280,12 @@ public IndexInput slice(String sliceName, long sliceOffset, long sliceLength) {
             sliceCompoundFileOffset,
             sliceLength,
             cacheFileReference,
-            frozenCacheFile,
+            cacheFile,
             defaultRangeSize,
             recoveryRangeSize,
             sliceHeaderByteRange,
             sliceFooterByteRange
         );
-        slice.isClone = true;
-        return slice;
     }
 
 }
diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java
index f4ef1ffe57b8..3c22956c076d 100644
--- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java
+++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java
@@ -8,39 +8,44 @@
 package org.elasticsearch.xpack.searchablesnapshots.store.input;
 
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefIterator;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.blobcache.BlobCacheUtils;
+import org.elasticsearch.blobcache.common.ByteRange;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Channels;
-import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Streams;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots;
 import org.elasticsearch.xpack.searchablesnapshots.cache.blob.BlobStoreCacheService;
 import org.elasticsearch.xpack.searchablesnapshots.cache.blob.CachedBlob;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.ByteRange;
 import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheFile;
 import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey;
 import org.elasticsearch.xpack.searchablesnapshots.store.IndexInputStats;
 import org.elasticsearch.xpack.searchablesnapshots.store.SearchableSnapshotDirectory;
 
-import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
-import java.util.Locale;
+import java.util.Objects;
 import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Consumer;
+import java.util.function.LongConsumer;
 
+import static org.elasticsearch.blobcache.BlobCacheUtils.throwEOF;
+import static org.elasticsearch.blobcache.BlobCacheUtils.toIntBytes;
 import static org.elasticsearch.core.Strings.format;
-import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsUtils.toIntBytes;
 
 /**
  * Searchable snapshots index input that supports fully caching metadata files as well as header/footer information for each file,
@@ -48,10 +53,24 @@
  */
 public abstract class MetadataCachingIndexInput extends BaseSearchableSnapshotIndexInput {
 
-    protected static final int COPY_BUFFER_SIZE = ByteSizeUnit.KB.toIntBytes(8);
+    /**
+     * Thread local direct byte buffer to aggregate multiple positional writes to the cache file.
+     */
+    protected static final int MAX_BYTES_PER_WRITE = StrictMath.toIntExact(
+        ByteSizeValue.parseBytesSizeValue(
+            System.getProperty("es.searchable.snapshot.shared_cache.write_buffer.size", "2m"),
+            "es.searchable.snapshot.shared_cache.write_buffer.size"
+        ).getBytes()
+    );
+
+    protected static final ThreadLocal<ByteBuffer> writeBuffer = ThreadLocal.withInitial(
+        () -> ByteBuffer.allocateDirect(MAX_BYTES_PER_WRITE)
+    );
 
     protected final CacheFileReference cacheFileReference;
 
+    protected final SearchableSnapshotDirectory directory;
+
     /**
      * If > 0, represents a logical file within a compound (CFS) file or is a slice thereof represents the offset of the logical
      * compound file within the physical CFS file
@@ -62,9 +81,21 @@ public abstract class MetadataCachingIndexInput extends BaseSearchableSnapshotIn
     protected final int recoveryRangeSize;
 
     // last read position is kept around in order to detect (non)contiguous reads for stats
-    protected long lastReadPosition;
+    private long lastReadPosition;
     // last seek position is kept around in order to detect forward/backward seeks for stats
-    protected long lastSeekPosition;
+    private long lastSeekPosition;
+
+    /**
+     * Range of bytes that should be cached in the blob cache for the current index input's header.
+     */
+    protected final ByteRange headerBlobCacheByteRange;
+
+    /**
+     * Range of bytes that should be cached in the blob cache for the current index input's footer. This footer byte range should only be
+     * required for slices of CFS files; regular files already have their footers extracted from the
+     * {@link BlobStoreIndexShardSnapshot.FileInfo} (see method {@link BaseSearchableSnapshotIndexInput#maybeReadChecksumFromFileInfo}).
+     */
+    protected final ByteRange footerBlobCacheByteRange;
 
     public MetadataCachingIndexInput(
         Logger logger,
@@ -82,13 +113,16 @@ public MetadataCachingIndexInput(
         ByteRange headerBlobCacheByteRange,
         ByteRange footerBlobCacheByteRange
     ) {
-        super(logger, name, directory, fileInfo, context, stats, offset, length, headerBlobCacheByteRange, footerBlobCacheByteRange);
+        super(logger, name, directory.blobContainer(), fileInfo, context, stats, offset, length);
+        this.directory = Objects.requireNonNull(directory);
         this.cacheFileReference = cacheFileReference;
         this.compoundFileOffset = compoundFileOffset;
         this.defaultRangeSize = defaultRangeSize;
         this.recoveryRangeSize = recoveryRangeSize;
         this.lastReadPosition = offset;
         this.lastSeekPosition = offset;
+        this.headerBlobCacheByteRange = Objects.requireNonNull(headerBlobCacheByteRange);
+        this.footerBlobCacheByteRange = Objects.requireNonNull(footerBlobCacheByteRange);
         assert offset >= compoundFileOffset;
         assert getBufferSize() <= BlobStoreCacheService.DEFAULT_CACHED_BLOB_SIZE; // must be able to cache at least one buffer's worth
     }
@@ -119,6 +153,16 @@ protected void doReadInternal(ByteBuffer b) throws IOException {
 
     protected abstract void readWithoutBlobCache(ByteBuffer b) throws Exception;
 
+    private ByteRange rangeToReadFromBlobCache(long position, int readLength) {
+        final long end = position + readLength;
+        if (headerBlobCacheByteRange.contains(position, end)) {
+            return headerBlobCacheByteRange;
+        } else if (footerBlobCacheByteRange.contains(position, end)) {
+            return footerBlobCacheByteRange;
+        }
+        return ByteRange.EMPTY;
+    }
+
     private void readWithBlobCache(ByteBuffer b, ByteRange blobCacheByteRange) throws Exception {
         final long position = getAbsolutePosition();
         final int length = b.remaining();
@@ -149,18 +193,7 @@ private void readWithBlobCache(ByteBuffer b, ByteRange blobCacheByteRange) throw
 
             // We must fill in a cache miss even if CACHE_NOT_READY since the cache index is only created on the first put.
             // TODO TBD use a different trigger for creating the cache index and avoid a put in the CACHE_NOT_READY case.
-            final ByteRange rangeToWrite = blobCacheByteRange;
-            final ByteRange rangeToRead = ByteRange.of(position, position + length);
-            assert rangeToRead.isSubRangeOf(rangeToWrite) : rangeToRead + " vs " + rangeToWrite;
-            assert rangeToRead.length() == b.remaining() : b.remaining() + " vs " + rangeToRead;
-
-            final Future<Integer> populateCacheFuture = cacheFile.populateAndRead(
-                rangeToWrite,
-                rangeToRead,
-                channel -> readCacheFile(channel, position, b),
-                this::writeCacheFile,
-                directory.cacheFetchAsyncExecutor()
-            );
+            final Future<Integer> populateCacheFuture = populateAndRead(b, position, length, cacheFile, blobCacheByteRange);
 
             fillIndexCache(cacheFile, blobCacheByteRange);
             if (compoundFileOffset > 0L
@@ -229,6 +262,20 @@ private void readWithBlobCache(ByteBuffer b, ByteRange blobCacheByteRange) throw
         }
     }
 
+    protected Future<Integer> populateAndRead(ByteBuffer b, long position, int length, CacheFile cacheFile, ByteRange rangeToWrite) {
+        final ByteRange rangeToRead = ByteRange.of(position, position + length);
+        assert rangeToRead.isSubRangeOf(rangeToWrite) : rangeToRead + " vs " + rangeToWrite;
+        assert rangeToRead.length() == b.remaining() : b.remaining() + " vs " + rangeToRead;
+
+        return cacheFile.populateAndRead(
+            rangeToWrite,
+            rangeToRead,
+            channel -> readCacheFile(channel, position, b),
+            this::writeCacheFile,
+            directory.cacheFetchAsyncExecutor()
+        );
+    }
+
     private void readComplete(long position, int length) {
         stats.incrementBytesRead(lastReadPosition, position, length);
         lastReadPosition = position + length;
@@ -239,35 +286,28 @@ protected int readCacheFile(final FileChannel fc, final long position, final Byt
         assert assertFileChannelOpen(fc);
         final int bytesRead = Channels.readFromFileChannel(fc, position, buffer);
         if (bytesRead == -1) {
-            throw new EOFException(
-                String.format(
-                    Locale.ROOT,
-                    "unexpected EOF reading [%d-%d] from %s",
-                    position,
-                    position + buffer.remaining(),
-                    cacheFileReference
-                )
-            );
+            throwEOF(position, buffer.remaining(), cacheFileReference);
         }
         stats.addCachedBytesRead(bytesRead);
         return bytesRead;
     }
 
-    protected void writeCacheFile(final FileChannel fc, final long start, final long end, final Consumer<Long> progressUpdater)
+    protected void writeCacheFile(final FileChannel fc, final long start, final long end, final LongConsumer progressUpdater)
         throws IOException {
         assert assertFileChannelOpen(fc);
         assert ThreadPool.assertCurrentThreadPool(SearchableSnapshots.CACHE_FETCH_ASYNC_THREAD_POOL_NAME);
         final long length = end - start;
-        final byte[] copyBuffer = new byte[toIntBytes(Math.min(COPY_BUFFER_SIZE, length))];
-        logger.trace(() -> format("writing range [%s-%s] to cache file [%s]", start, end, cacheFileReference));
+        final ByteBuffer copyBuffer = writeBuffer.get().clear();
+        logger.trace("writing range [{}-{}] to cache file [{}]", start, end, cacheFileReference);
 
         long bytesCopied = 0L;
         long remaining = end - start;
         final long startTimeNanos = stats.currentTimeNanos();
         try (InputStream input = openInputStreamFromBlobStore(start, length)) {
             while (remaining > 0L) {
-                final int bytesRead = readSafe(input, copyBuffer, start, end, remaining, cacheFileReference);
-                positionalWrite(fc, start + bytesCopied, ByteBuffer.wrap(copyBuffer, 0, bytesRead));
+                final int bytesRead = BlobCacheUtils.readSafe(input, copyBuffer, start, remaining, cacheFileReference);
+                positionalWrite(fc, start + bytesCopied, copyBuffer.flip());
+                copyBuffer.clear();
                 bytesCopied += bytesRead;
                 remaining -= bytesRead;
                 progressUpdater.accept(start + bytesCopied);
@@ -292,17 +332,7 @@ private void fillIndexCache(CacheFile cacheFile, ByteRange indexCacheMiss) {
             // NB use Channels.readFromFileChannelWithEofException not readCacheFile() to avoid counting this in the stats
             byteBuffer.flip();
             final BytesReference content = BytesReference.fromByteBuffer(byteBuffer);
-            directory.putCachedBlob(fileInfo.physicalName(), indexCacheMiss, content, new ActionListener<>() {
-                @Override
-                public void onResponse(Void response) {
-                    onCacheFillComplete.close();
-                }
-
-                @Override
-                public void onFailure(Exception e1) {
-                    onCacheFillComplete.close();
-                }
-            });
+            directory.putCachedBlob(fileInfo.physicalName(), indexCacheMiss, content, ActionListener.releasing(onCacheFillComplete));
             return indexCacheMissLength;
         });
 
@@ -314,38 +344,6 @@ public void onFailure(Exception e1) {
         }
     }
 
-    /**
-     * Perform a single {@code read()} from {@code inputStream} into {@code copyBuffer}, handling an EOF by throwing an {@link EOFException}
-     * rather than returning {@code -1}. Returns the number of bytes read, which is always positive.
-     *
-     * Most of its arguments are there simply to make the message of the {@link EOFException} more informative.
-     */
-    protected static int readSafe(
-        InputStream inputStream,
-        byte[] copyBuffer,
-        long rangeStart,
-        long rangeEnd,
-        long remaining,
-        CacheFileReference cacheFileReference
-    ) throws IOException {
-        final int len = (remaining < copyBuffer.length) ? toIntBytes(remaining) : copyBuffer.length;
-        final int bytesRead = inputStream.read(copyBuffer, 0, len);
-        if (bytesRead == -1) {
-            throw new EOFException(
-                String.format(
-                    Locale.ROOT,
-                    "unexpected EOF reading [%d-%d] ([%d] bytes remaining) from %s",
-                    rangeStart,
-                    rangeEnd,
-                    remaining,
-                    cacheFileReference
-                )
-            );
-        }
-        assert bytesRead > 0 : bytesRead;
-        return bytesRead;
-    }
-
     protected static boolean assertFileChannelOpen(FileChannel fileChannel) {
         assert fileChannel != null;
         assert fileChannel.isOpen();
@@ -362,40 +360,19 @@ protected int readDirectlyIfAlreadyClosed(long position, ByteBuffer b, Exception
         if (e instanceof AlreadyClosedException || (e.getCause() != null && e.getCause() instanceof AlreadyClosedException)) {
             try {
                 // cache file was evicted during the range fetching, read bytes directly from blob container
-                final long length = b.remaining();
-                final byte[] copyBuffer = new byte[toIntBytes(Math.min(COPY_BUFFER_SIZE, length))];
-                logger.trace(
-                    () -> format("direct reading of range [%s-%s] for cache file [%s]", position, position + length, cacheFileReference)
-                );
+                final int length = b.remaining();
+                logger.trace("direct reading of range [{}-{}] for cache file [{}]", position, position + length, cacheFileReference);
 
-                int bytesCopied = 0;
                 final long startTimeNanos = stats.currentTimeNanos();
                 try (InputStream input = openInputStreamFromBlobStore(position, length)) {
-                    long remaining = length;
-                    while (remaining > 0) {
-                        final int len = (remaining < copyBuffer.length) ? (int) remaining : copyBuffer.length;
-                        int bytesRead = input.read(copyBuffer, 0, len);
-                        if (bytesRead == -1) {
-                            throw new EOFException(
-                                String.format(
-                                    Locale.ROOT,
-                                    "unexpected EOF reading [%d-%d] ([%d] bytes remaining) from %s",
-                                    position,
-                                    position + length,
-                                    remaining,
-                                    cacheFileReference
-                                )
-                            );
-                        }
-                        b.put(copyBuffer, 0, bytesRead);
-                        bytesCopied += bytesRead;
-                        remaining -= bytesRead;
-                        assert remaining == b.remaining() : remaining + " vs " + b.remaining();
+                    final int bytesRead = Streams.read(input, b, length);
+                    if (bytesRead < length) {
+                        throwEOF(position, length - bytesRead, cacheFileReference);
                     }
                     final long endTimeNanos = stats.currentTimeNanos();
-                    stats.addDirectBytesRead(bytesCopied, endTimeNanos - startTimeNanos);
+                    stats.addDirectBytesRead(bytesRead, endTimeNanos - startTimeNanos);
+                    return bytesRead;
                 }
-                return bytesCopied;
             } catch (Exception inner) {
                 e.addSuppressed(inner);
             }
@@ -406,24 +383,65 @@ protected int readDirectlyIfAlreadyClosed(long position, ByteBuffer b, Exception
     protected abstract long getDefaultRangeSize();
 
     protected ByteRange computeRange(long position) {
-        final long rangeSize = getDefaultRangeSize();
-        long start = (position / rangeSize) * rangeSize;
-        long end = Math.min(start + rangeSize, fileInfo.length());
-        return ByteRange.of(start, end);
+        return BlobCacheUtils.computeRange(getDefaultRangeSize(), position, fileInfo.length());
     }
 
     @Override
     protected void seekInternal(long pos) throws IOException {
-        if (pos > length()) {
-            throw new EOFException("Reading past end of file [position=" + pos + ", length=" + length() + "] for " + toString());
-        } else if (pos < 0L) {
-            throw new IOException("Seeking to negative position [" + pos + "] for " + toString());
-        }
+        BlobCacheUtils.ensureSeek(pos, this);
         final long position = pos + this.offset;
         stats.incrementSeeks(lastSeekPosition, position);
         lastSeekPosition = position;
     }
 
+    @Override
+    public IndexInput slice(String sliceName, long sliceOffset, long sliceLength) {
+        BlobCacheUtils.ensureSlice(sliceName, sliceOffset, sliceLength, this);
+
+        // Are we creating a slice from a CFS file?
+        final boolean sliceCompoundFile = IndexFileNames.matchesExtension(name, "cfs")
+            && IndexFileNames.getExtension(sliceName) != null
+            && compoundFileOffset == 0L // not already a compound file
+            && isClone == false; // tests aggressively clone and slice
+
+        final ByteRange sliceHeaderByteRange;
+        final ByteRange sliceFooterByteRange;
+        final long sliceCompoundFileOffset;
+
+        if (sliceCompoundFile) {
+            sliceCompoundFileOffset = this.offset + sliceOffset;
+            sliceHeaderByteRange = directory.getBlobCacheByteRange(sliceName, sliceLength).shift(sliceCompoundFileOffset);
+            if (sliceHeaderByteRange.isEmpty() == false && sliceHeaderByteRange.length() < sliceLength) {
+                sliceFooterByteRange = ByteRange.of(sliceLength - CodecUtil.footerLength(), sliceLength).shift(sliceCompoundFileOffset);
+            } else {
+                sliceFooterByteRange = ByteRange.EMPTY;
+            }
+        } else {
+            sliceCompoundFileOffset = this.compoundFileOffset;
+            sliceHeaderByteRange = ByteRange.EMPTY;
+            sliceFooterByteRange = ByteRange.EMPTY;
+        }
+        final MetadataCachingIndexInput slice = doSlice(
+            sliceName,
+            sliceOffset,
+            sliceLength,
+            sliceHeaderByteRange,
+            sliceFooterByteRange,
+            sliceCompoundFileOffset
+        );
+        slice.isClone = true;
+        return slice;
+    }
+
+    protected abstract MetadataCachingIndexInput doSlice(
+        String sliceName,
+        long sliceOffset,
+        long sliceLength,
+        ByteRange sliceHeaderByteRange,
+        ByteRange sliceFooterByteRange,
+        long sliceCompoundFileOffset
+    );
+
     @Override
     public void doClose() {
         if (isClone == false) {
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java
index fef5f7406ec3..3d3da1fd8fd7 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsRestTestCase.java
@@ -157,7 +157,7 @@ private void runSearchableSnapshotsTest(
 
                         long n;
                         while ((n = remainingDocs.decrementAndGet()) >= 0) {
-                            bulkBody.append(String.format(Locale.ROOT, """
+                            bulkBody.append(Strings.format("""
                                     {"index": {"_id":"%d"} }
                                     {"field": %d, "text": "Document number %d"}
                                 """, n, n, n));
@@ -253,7 +253,7 @@ public void testSourceOnlyRepository() throws Exception {
                 String sourceOnlySnapshot = "source-only-snap-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
                 final Request request = new Request(HttpPut.METHOD_NAME, "_snapshot/" + WRITE_REPOSITORY_NAME + '/' + sourceOnlySnapshot);
                 request.addParameter("wait_for_completion", "true");
-                request.setJsonEntity(formatted("""
+                request.setJsonEntity(Strings.format("""
                     {
                         "include_global_state": false,
                         "indices" : "%s"
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsTestCase.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsTestCase.java
index 45150b37d11f..a545ae7df2f1 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsTestCase.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsTestCase.java
@@ -15,6 +15,8 @@
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.elasticsearch.Version;
+import org.elasticsearch.blobcache.common.ByteRange;
+import org.elasticsearch.blobcache.shared.SharedBlobCacheService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -44,12 +46,10 @@
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPoolStats;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.ByteRange;
 import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheFile;
 import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey;
 import org.elasticsearch.xpack.searchablesnapshots.cache.full.CacheService;
 import org.elasticsearch.xpack.searchablesnapshots.cache.full.PersistentCache;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService;
 import org.elasticsearch.xpack.searchablesnapshots.recovery.SearchableSnapshotRecoveryState;
 import org.junit.After;
 import org.junit.Before;
@@ -69,9 +69,9 @@
 import java.util.concurrent.TimeUnit;
 
 import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiLettersOfLengthBetween;
+import static org.elasticsearch.blobcache.shared.SharedBytes.PAGE_SIZE;
+import static org.elasticsearch.blobcache.shared.SharedBytes.pageAligned;
 import static org.elasticsearch.xpack.searchablesnapshots.cache.common.TestUtils.randomPopulateAndReads;
-import static org.elasticsearch.xpack.searchablesnapshots.cache.shared.SharedBytes.PAGE_SIZE;
-import static org.elasticsearch.xpack.searchablesnapshots.cache.shared.SharedBytes.pageAligned;
 
 public abstract class AbstractSearchableSnapshotsTestCase extends ESIndexInputTestCase {
 
@@ -135,27 +135,27 @@ protected CacheService randomCacheService() {
     }
 
     /**
-     * @return a new {@link FrozenCacheService} instance configured with default settings
+     * @return a new {@link SharedBlobCacheService} instance configured with default settings
      */
-    protected FrozenCacheService defaultFrozenCacheService() {
-        return new FrozenCacheService(nodeEnvironment, Settings.EMPTY, threadPool);
+    protected SharedBlobCacheService<CacheKey> defaultFrozenCacheService() {
+        return new SharedBlobCacheService<>(nodeEnvironment, Settings.EMPTY, threadPool);
     }
 
-    protected FrozenCacheService randomFrozenCacheService() {
+    protected SharedBlobCacheService<CacheKey> randomFrozenCacheService() {
         final Settings.Builder cacheSettings = Settings.builder();
         if (randomBoolean()) {
-            cacheSettings.put(FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), randomFrozenCacheSize());
+            cacheSettings.put(SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), randomFrozenCacheSize());
         }
         if (randomBoolean()) {
-            cacheSettings.put(FrozenCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(), pageAligned(randomFrozenCacheSize()));
+            cacheSettings.put(SharedBlobCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(), pageAligned(randomFrozenCacheSize()));
         }
         if (randomBoolean()) {
-            cacheSettings.put(FrozenCacheService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(), randomFrozenCacheRangeSize());
+            cacheSettings.put(SharedBlobCacheService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(), randomFrozenCacheRangeSize());
         }
         if (randomBoolean()) {
-            cacheSettings.put(FrozenCacheService.SHARED_CACHE_RECOVERY_RANGE_SIZE_SETTING.getKey(), randomFrozenCacheRangeSize());
+            cacheSettings.put(SharedBlobCacheService.SHARED_CACHE_RECOVERY_RANGE_SIZE_SETTING.getKey(), randomFrozenCacheRangeSize());
         }
-        return new FrozenCacheService(singlePathNodeEnvironment, cacheSettings.build(), threadPool);
+        return new SharedBlobCacheService<>(singlePathNodeEnvironment, cacheSettings.build(), threadPool);
     }
 
     /**
@@ -170,12 +170,12 @@ protected CacheService createCacheService(final ByteSizeValue cacheRangeSize) {
         );
     }
 
-    protected FrozenCacheService createFrozenCacheService(final ByteSizeValue cacheSize, final ByteSizeValue cacheRangeSize) {
-        return new FrozenCacheService(
+    protected SharedBlobCacheService<CacheKey> createFrozenCacheService(final ByteSizeValue cacheSize, final ByteSizeValue cacheRangeSize) {
+        return new SharedBlobCacheService<>(
             singlePathNodeEnvironment,
             Settings.builder()
-                .put(FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), cacheSize)
-                .put(FrozenCacheService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(), cacheRangeSize)
+                .put(SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), cacheSize)
+                .put(SharedBlobCacheService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(), cacheRangeSize)
                 .build(),
             threadPool
         );
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotAllocatorTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotAllocatorTests.java
index dbcc00eee11b..beebf6890f56 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotAllocatorTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/allocation/SearchableSnapshotAllocatorTests.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -63,7 +64,7 @@ public void testAllocateToNodeWithLargestCache() {
         final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue();
 
         final Metadata metadata = buildSingleShardIndexMetadata(shardId);
-        final RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        final RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         routingTableBuilder.addAsRestore(metadata.index(shardId.getIndex()), randomSnapshotSource(shardId));
 
         final ClusterState state = buildClusterState(nodes, metadata, routingTableBuilder);
@@ -137,7 +138,7 @@ public void testNoFetchesOnDeciderNo() {
         final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue();
 
         final Metadata metadata = buildSingleShardIndexMetadata(shardId);
-        final RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        final RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         routingTableBuilder.addAsRestore(metadata.index(shardId.getIndex()), randomSnapshotSource(shardId));
 
         final ClusterState state = buildClusterState(nodes, metadata, routingTableBuilder);
@@ -176,7 +177,7 @@ public void testNoFetchesForPartialIndex() {
         final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue();
 
         final Metadata metadata = buildSingleShardIndexMetadata(shardId, builder -> builder.put(SNAPSHOT_PARTIAL_SETTING.getKey(), true));
-        final RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        final RoutingTable.Builder routingTableBuilder = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         routingTableBuilder.addAsRestore(metadata.index(shardId.getIndex()), randomSnapshotSource(shardId));
 
         final ClusterState state = buildClusterState(nodes, metadata, routingTableBuilder);
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheServiceTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheServiceTests.java
index cf20915739b9..828b349466a3 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheServiceTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/BlobStoreCacheServiceTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.blobcache.common.ByteRange;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -24,7 +25,6 @@
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.ByteRange;
 import org.junit.After;
 import org.junit.Before;
 
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/CacheFileTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/CacheFileTests.java
index 4d69291b5502..92ec94963c0c 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/CacheFileTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/CacheFileTests.java
@@ -4,12 +4,15 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+
 package org.elasticsearch.xpack.searchablesnapshots.cache.common;
 
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.blobcache.BlobCacheTestUtils;
+import org.elasticsearch.blobcache.common.ByteRange;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.filesystem.FileSystemNatives;
 import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
@@ -19,7 +22,6 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheFile.EvictionListener;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.TestUtils.FSyncTrackingFileSystemProvider;
 import org.hamcrest.Matcher;
 
 import java.io.IOException;
@@ -241,7 +243,7 @@ public void testConcurrentAccess() throws Exception {
     }
 
     public void testFSync() throws Exception {
-        final FSyncTrackingFileSystemProvider fileSystem = setupFSyncCountingFileSystem();
+        final BlobCacheTestUtils.FSyncTrackingFileSystemProvider fileSystem = setupFSyncCountingFileSystem();
         try {
             final TestCacheFileModificationListener updatesListener = new TestCacheFileModificationListener();
             final CacheFile cacheFile = new CacheFile(CACHE_KEY, randomLongBetween(0L, 1000L), fileSystem.resolve("test"), updatesListener);
@@ -284,7 +286,7 @@ public void testFSync() throws Exception {
     }
 
     public void testFSyncOnEvictedFile() throws Exception {
-        final FSyncTrackingFileSystemProvider fileSystem = setupFSyncCountingFileSystem();
+        final BlobCacheTestUtils.FSyncTrackingFileSystemProvider fileSystem = setupFSyncCountingFileSystem();
         try {
             final TestCacheFileModificationListener updatesListener = new TestCacheFileModificationListener();
             final CacheFile cacheFile = new CacheFile(CACHE_KEY, randomLongBetween(0L, 1000L), fileSystem.resolve("test"), updatesListener);
@@ -342,7 +344,7 @@ public void testFSyncOnEvictedFile() throws Exception {
     }
 
     public void testFSyncFailure() throws Exception {
-        final FSyncTrackingFileSystemProvider fileSystem = setupFSyncCountingFileSystem();
+        final BlobCacheTestUtils.FSyncTrackingFileSystemProvider fileSystem = setupFSyncCountingFileSystem();
         try {
             fileSystem.failFSyncs(true);
 
@@ -526,15 +528,20 @@ synchronized void reset() {
     }
 
     public static void assertNumberOfFSyncs(final Path path, final Matcher<Integer> matcher) {
-        final FSyncTrackingFileSystemProvider provider = (FSyncTrackingFileSystemProvider) path.getFileSystem().provider();
+        final BlobCacheTestUtils.FSyncTrackingFileSystemProvider provider = (BlobCacheTestUtils.FSyncTrackingFileSystemProvider) path
+            .getFileSystem()
+            .provider();
         final Integer fsyncCount = provider.getNumberOfFSyncs(path);
         assertThat("File [" + path + "] was never fsynced", fsyncCount, notNullValue());
         assertThat("Mismatching number of fsync for [" + path + "]", fsyncCount, matcher);
     }
 
-    private static FSyncTrackingFileSystemProvider setupFSyncCountingFileSystem() {
+    private static BlobCacheTestUtils.FSyncTrackingFileSystemProvider setupFSyncCountingFileSystem() {
         final FileSystem defaultFileSystem = PathUtils.getDefaultFileSystem();
-        final FSyncTrackingFileSystemProvider provider = new FSyncTrackingFileSystemProvider(defaultFileSystem, createTempDir());
+        final BlobCacheTestUtils.FSyncTrackingFileSystemProvider provider = new BlobCacheTestUtils.FSyncTrackingFileSystemProvider(
+            defaultFileSystem,
+            createTempDir()
+        );
         PathUtilsForTesting.installMock(provider.getFileSystem(null));
         return provider;
     }
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/CacheKeyTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/CacheKeyTests.java
index 76f4ac4c42d3..fd2d7b2742d1 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/CacheKeyTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/CacheKeyTests.java
@@ -4,6 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+
 package org.elasticsearch.xpack.searchablesnapshots.cache.common;
 
 import org.elasticsearch.common.UUIDs;
@@ -37,24 +38,24 @@ private CacheKey createInstance() {
     }
 
     private CacheKey copy(final CacheKey origin) {
-        ShardId shardId = origin.getShardId();
+        ShardId shardId = origin.shardId();
         if (randomBoolean()) {
             shardId = new ShardId(new Index(shardId.getIndex().getName(), shardId.getIndex().getUUID()), shardId.id());
         }
-        return new CacheKey(origin.getSnapshotUUID(), origin.getSnapshotIndexName(), shardId, origin.getFileName());
+        return new CacheKey(origin.snapshotUUID(), origin.snapshotIndexName(), shardId, origin.fileName());
     }
 
     private CacheKey mutate(CacheKey origin) {
-        String snapshotUUID = origin.getSnapshotUUID();
-        String snapshotIndexName = origin.getSnapshotIndexName();
-        ShardId shardId = origin.getShardId();
-        String fileName = origin.getFileName();
+        String snapshotUUID = origin.snapshotUUID();
+        String snapshotIndexName = origin.snapshotIndexName();
+        ShardId shardId = origin.shardId();
+        String fileName = origin.fileName();
 
         switch (randomInt(3)) {
             case 0 -> snapshotUUID = randomValueOtherThan(snapshotUUID, this::randomSnapshotUUID);
             case 1 -> snapshotIndexName = randomValueOtherThan(snapshotIndexName, this::randomSnapshotIndexName);
-            case 2 -> shardId = randomValueOtherThan(origin.getShardId(), this::randomShardId);
-            case 3 -> fileName = randomValueOtherThan(origin.getFileName(), () -> randomAlphaOfLengthBetween(5, 10));
+            case 2 -> shardId = randomValueOtherThan(origin.shardId(), this::randomShardId);
+            case 3 -> fileName = randomValueOtherThan(origin.fileName(), () -> randomAlphaOfLengthBetween(5, 10));
             default -> throw new AssertionError("Unsupported mutation");
         }
         return new CacheKey(snapshotUUID, snapshotIndexName, shardId, fileName);
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/TestUtils.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/TestUtils.java
index 670ac0276fff..6415ff468198 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/TestUtils.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/common/TestUtils.java
@@ -6,14 +6,12 @@
  */
 package org.elasticsearch.xpack.searchablesnapshots.cache.common;
 
-import org.apache.lucene.tests.mockfile.FilterFileChannel;
-import org.apache.lucene.tests.mockfile.FilterFileSystemProvider;
-import org.apache.lucene.tests.mockfile.FilterPath;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.blobcache.common.ByteRange;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.TriConsumer;
 import org.elasticsearch.common.blobstore.BlobContainer;
@@ -26,13 +24,12 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
 import org.elasticsearch.core.CheckedConsumer;
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.PathUtilsForTesting;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.xpack.searchablesnapshots.cache.blob.BlobStoreCacheService;
 import org.elasticsearch.xpack.searchablesnapshots.store.IndexInputStats;
+import org.hamcrest.MatcherAssert;
 
 import java.io.ByteArrayInputStream;
 import java.io.FileNotFoundException;
@@ -40,30 +37,23 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.channels.FileChannel;
-import java.nio.file.FileSystem;
-import java.nio.file.OpenOption;
-import java.nio.file.Path;
-import java.nio.file.attribute.FileAttribute;
-import java.nio.file.spi.FileSystemProvider;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
 
 import static java.util.Collections.synchronizedNavigableSet;
+import static org.elasticsearch.blobcache.BlobCacheTestUtils.mergeContiguousRanges;
+import static org.elasticsearch.blobcache.BlobCacheUtils.toIntBytes;
 import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM;
 import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
 import static org.elasticsearch.test.ESTestCase.between;
 import static org.elasticsearch.test.ESTestCase.randomLongBetween;
 import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_BLOB_CACHE_INDEX;
-import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsUtils.toIntBytes;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.equalTo;
 import static org.junit.Assert.assertTrue;
@@ -97,68 +87,15 @@ public static SortedSet<ByteRange> randomPopulateAndReads(CacheFile cacheFile, T
         return mergeContiguousRanges(ranges);
     }
 
-    public static long numberOfRanges(long fileSize, long rangeSize) {
-        return numberOfRanges(toIntBytes(fileSize), toIntBytes(rangeSize));
-    }
-
-    static long numberOfRanges(int fileSize, int rangeSize) {
-        long numberOfRanges = fileSize / rangeSize;
-        if (fileSize % rangeSize > 0) {
-            numberOfRanges++;
-        }
-        if (numberOfRanges == 0) {
-            numberOfRanges++;
-        }
-        return numberOfRanges;
-    }
-
-    /**
-     * Generates a sorted set of non-empty and non-contiguous random ranges that could fit into a file of a given maximum length.
-     */
-    public static SortedSet<ByteRange> randomRanges(long length) {
-        final SortedSet<ByteRange> randomRanges = new TreeSet<>();
-        for (long i = 0L; i < length;) {
-            long start = randomLongBetween(i, Math.max(0L, length - 1L));
-            long end = randomLongBetween(start + 1L, length); // +1 for non empty ranges
-            randomRanges.add(ByteRange.of(start, end));
-            i = end + 1L + randomLongBetween(0L, Math.max(0L, length - end)); // +1 for non contiguous ranges
-        }
-        return randomRanges;
-    }
-
-    public static SortedSet<ByteRange> mergeContiguousRanges(final SortedSet<ByteRange> ranges) {
-        return ranges.stream().collect(TreeSet::new, (gaps, gap) -> {
-            if (gaps.isEmpty()) {
-                gaps.add(gap);
-            } else {
-                final ByteRange previous = gaps.pollLast();
-                if (previous.end() == gap.start()) {
-                    gaps.add(ByteRange.of(previous.start(), gap.end()));
-                } else {
-                    gaps.add(previous);
-                    gaps.add(gap);
-                }
-            }
-        }, (gaps1, gaps2) -> {
-            if (gaps1.isEmpty() == false && gaps2.isEmpty() == false) {
-                final ByteRange last = gaps1.pollLast();
-                final ByteRange first = gaps2.pollFirst();
-                if (last.end() == first.start()) {
-                    gaps1.add(ByteRange.of(last.start(), first.end()));
-                } else {
-                    gaps1.add(last);
-                    gaps2.add(first);
-                }
-            }
-            gaps1.addAll(gaps2);
-        });
+    public static void assertCacheFileEquals(CacheFile expected, CacheFile actual) {
+        MatcherAssert.assertThat(actual.getLength(), equalTo(expected.getLength()));
+        MatcherAssert.assertThat(actual.getFile(), equalTo(expected.getFile()));
+        MatcherAssert.assertThat(actual.getCacheKey(), equalTo(expected.getCacheKey()));
+        MatcherAssert.assertThat(actual.getCompletedRanges(), equalTo(expected.getCompletedRanges()));
     }
 
-    public static void assertCacheFileEquals(CacheFile expected, CacheFile actual) {
-        assertThat(actual.getLength(), equalTo(expected.getLength()));
-        assertThat(actual.getFile(), equalTo(expected.getFile()));
-        assertThat(actual.getCacheKey(), equalTo(expected.getCacheKey()));
-        assertThat(actual.getCompletedRanges(), equalTo(expected.getCompletedRanges()));
+    public static long sumOfCompletedRangesLengths(CacheFile cacheFile) {
+        return cacheFile.getCompletedRanges().stream().mapToLong(ByteRange::length).sum();
     }
 
     public static void assertCounter(IndexInputStats.Counter counter, long total, long count, long min, long max) {
@@ -168,22 +105,6 @@ public static void assertCounter(IndexInputStats.Counter counter, long total, lo
         assertThat(counter.max(), equalTo(max));
     }
 
-    public static void assertCounter(
-        IndexInputStats.TimedCounter timedCounter,
-        long total,
-        long count,
-        long min,
-        long max,
-        long totalNanoseconds
-    ) {
-        assertCounter(timedCounter, total, count, min, max);
-        assertThat(timedCounter.totalNanoseconds(), equalTo(totalNanoseconds));
-    }
-
-    public static long sumOfCompletedRangesLengths(CacheFile cacheFile) {
-        return cacheFile.getCompletedRanges().stream().mapToLong(ByteRange::length).sum();
-    }
-
     /**
      * A {@link BlobContainer} that can read a single in-memory blob.
      * Any attempt to read a different blob will throw a {@link FileNotFoundException}
@@ -388,56 +309,4 @@ protected void innerPut(IndexRequest request, ActionListener<IndexResponse> list
             );
         }
     }
-
-    /**
-     * A {@link FileSystemProvider} that counts the number of times the method {@link FileChannel#force(boolean)} is executed on every
-     * files.
-     */
-    public static class FSyncTrackingFileSystemProvider extends FilterFileSystemProvider {
-
-        private final Map<Path, AtomicInteger> files = new ConcurrentHashMap<>();
-        private final AtomicBoolean failFSyncs = new AtomicBoolean();
-        private final FileSystem delegateInstance;
-        private final Path rootDir;
-
-        public FSyncTrackingFileSystemProvider(FileSystem delegate, Path rootDir) {
-            super("fsynccounting://", delegate);
-            this.rootDir = new FilterPath(rootDir, this.fileSystem);
-            this.delegateInstance = delegate;
-        }
-
-        public void failFSyncs(boolean shouldFail) {
-            failFSyncs.set(shouldFail);
-        }
-
-        public Path resolve(String other) {
-            return rootDir.resolve(other);
-        }
-
-        @Nullable
-        public Integer getNumberOfFSyncs(Path path) {
-            final AtomicInteger counter = files.get(path);
-            return counter != null ? counter.get() : null;
-        }
-
-        @Override
-        public FileChannel newFileChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
-            final AtomicInteger counter = files.computeIfAbsent(path, p -> new AtomicInteger(0));
-            return new FilterFileChannel(super.newFileChannel(path, options, attrs)) {
-
-                @Override
-                public void force(boolean metaData) throws IOException {
-                    if (failFSyncs.get()) {
-                        throw new IOException("simulated");
-                    }
-                    super.force(metaData);
-                    counter.incrementAndGet();
-                }
-            };
-        }
-
-        public void tearDown() {
-            PathUtilsForTesting.installMock(delegateInstance);
-        }
-    }
 }
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/CacheServiceTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/CacheServiceTests.java
index c57ea1673c64..3d854d8ffa83 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/CacheServiceTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/CacheServiceTests.java
@@ -9,21 +9,22 @@
 
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.apache.lucene.util.Constants;
+import org.elasticsearch.blobcache.BlobCacheTestUtils.FSyncTrackingFileSystemProvider;
+import org.elasticsearch.blobcache.common.ByteRange;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.core.PathUtilsForTesting;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.xpack.searchablesnapshots.AbstractSearchableSnapshotsTestCase;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.ByteRange;
 import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheFile;
 import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.TestUtils.FSyncTrackingFileSystemProvider;
 import org.elasticsearch.xpack.searchablesnapshots.cache.full.CacheService.ShardEviction;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -42,8 +43,8 @@
 import java.util.stream.Collectors;
 
 import static java.util.Collections.emptySortedSet;
+import static org.elasticsearch.blobcache.BlobCacheTestUtils.randomRanges;
 import static org.elasticsearch.xpack.searchablesnapshots.cache.common.TestUtils.randomPopulateAndReads;
-import static org.elasticsearch.xpack.searchablesnapshots.cache.common.TestUtils.randomRanges;
 import static org.elasticsearch.xpack.searchablesnapshots.cache.full.CacheService.resolveSnapshotCache;
 import static org.hamcrest.Matchers.aMapWithSize;
 import static org.hamcrest.Matchers.containsString;
@@ -117,7 +118,7 @@ public void testCacheSynchronization() throws Exception {
                 logger.trace("--> creating new cache files and randomly read/write them");
                 for (int i = 0; i < between(1, 25); i++) {
                     final ShardId shardId = new ShardId(index, randomIntBetween(0, numShards - 1));
-                    final String fileName = String.format(Locale.ROOT, "file_%d_%d", iteration, i);
+                    final String fileName = Strings.format("file_%d_%d", iteration, i);
                     final CacheKey cacheKey = new CacheKey(snapshotUUID, snapshotIndexName, shardId, fileName);
                     final CacheFile cacheFile = cacheService.get(cacheKey, randomIntBetween(0, 10_000), shardsCacheDirs[shardId.id()]);
 
@@ -154,7 +155,7 @@ public void testCacheSynchronization() throws Exception {
                         .stream()
                         .filter(update -> update.getValue().v2() != null)
                         .filter(update -> update.getValue().v2() > 0)
-                        .anyMatch(update -> update.getKey().getShardId().equals(shardId))) {
+                        .anyMatch(update -> update.getKey().shardId().equals(shardId))) {
                         cacheDirFSyncs.put(shardCacheDir, numberOfFSyncs == null ? 1 : numberOfFSyncs + 1);
                     } else {
                         cacheDirFSyncs.put(shardCacheDir, numberOfFSyncs);
@@ -196,7 +197,7 @@ public void testPut() throws Exception {
             );
 
             final Path cacheDir = Files.createDirectories(
-                resolveSnapshotCache(randomShardPath(cacheKey.getShardId())).resolve(cacheKey.getSnapshotUUID())
+                resolveSnapshotCache(randomShardPath(cacheKey.shardId())).resolve(cacheKey.snapshotUUID())
             );
             final String cacheFileUuid = UUIDs.randomBase64UUID(random());
             final SortedSet<ByteRange> cacheFileRanges = randomBoolean() ? randomRanges(fileLength) : emptySortedSet();
@@ -252,7 +253,7 @@ public void testMarkShardAsEvictedInCache() throws Exception {
         }
 
         for (int i = 0; i < between(1, 3); i++) {
-            cacheService.markShardAsEvictedInCache(shard.getSnapshotUUID(), shard.getSnapshotIndexName(), shard.getShardId());
+            cacheService.markShardAsEvictedInCache(shard.snapshotUUID(), shard.snapshotIndexName(), shard.shardId());
         }
 
         blockingListener.waitForBlock();
@@ -276,7 +277,7 @@ public void testMarkShardAsEvictedInCache() throws Exception {
 
         if (randomBoolean()) {
             // mark shard as evicted after cache service is stopped should have no effect
-            cacheService.markShardAsEvictedInCache(shard.getSnapshotUUID(), shard.getSnapshotIndexName(), shard.getShardId());
+            cacheService.markShardAsEvictedInCache(shard.snapshotUUID(), shard.snapshotIndexName(), shard.shardId());
             assertThat(cacheService.pendingShardsEvictions(), aMapWithSize(0));
         }
     }
@@ -297,11 +298,11 @@ public void testProcessShardEviction() throws Exception {
         assertTrue(Files.exists(randomCacheFile.getFile()));
         randomCacheFile.acquire(blockingListener);
 
-        cacheService.markShardAsEvictedInCache(shard.getSnapshotUUID(), shard.getSnapshotIndexName(), shard.getShardId());
+        cacheService.markShardAsEvictedInCache(shard.snapshotUUID(), shard.snapshotIndexName(), shard.shardId());
 
         final Map<CacheFile, Boolean> afterShardRecoveryCacheFiles = ConcurrentCollections.newConcurrentMap();
         final Future<?> waitForShardEvictionFuture = threadPool.generic().submit(() -> {
-            cacheService.waitForCacheFilesEvictionIfNeeded(shard.getSnapshotUUID(), shard.getSnapshotIndexName(), shard.getShardId());
+            cacheService.waitForCacheFilesEvictionIfNeeded(shard.snapshotUUID(), shard.snapshotIndexName(), shard.shardId());
             for (CacheFile cacheFile : cacheFilesAssociatedWithShard) {
                 afterShardRecoveryCacheFiles.put(cacheFile, Files.exists(cacheFile.getFile()));
             }
@@ -378,7 +379,7 @@ private static ShardEviction randomShardEvictionFrom(List<CacheFile> cacheFiles)
     private static Set<ShardEviction> listOfShardEvictions(List<CacheFile> cacheFiles) {
         return cacheFiles.stream()
             .map(CacheFile::getCacheKey)
-            .map(cacheKey -> new ShardEviction(cacheKey.getSnapshotUUID(), cacheKey.getSnapshotIndexName(), cacheKey.getShardId()))
+            .map(cacheKey -> new ShardEviction(cacheKey.snapshotUUID(), cacheKey.snapshotIndexName(), cacheKey.shardId()))
             .collect(Collectors.toSet());
     }
 
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCacheTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCacheTests.java
index 7a1362edb1a2..1cd301400916 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCacheTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCacheTests.java
@@ -13,6 +13,7 @@
 import org.apache.lucene.tests.mockfile.FilterFileChannel;
 import org.apache.lucene.tests.mockfile.FilterFileSystemProvider;
 import org.apache.lucene.tests.mockfile.FilterPath;
+import org.elasticsearch.blobcache.common.ByteRange;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
@@ -24,7 +25,6 @@
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.xpack.searchablesnapshots.AbstractSearchableSnapshotsTestCase;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.ByteRange;
 import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheFile;
 import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey;
 
@@ -229,13 +229,13 @@ public void testFSyncDoesNotAddDocumentsBackInPersistentCacheWhenShardIsEvicted(
                 final CacheFile randomCacheFile = randomFrom(cacheFiles);
                 final CacheKey cacheKey = randomCacheFile.getCacheKey();
 
-                final SnapshotId snapshotId = new SnapshotId("_ignored_", cacheKey.getSnapshotUUID());
+                final SnapshotId snapshotId = new SnapshotId("_ignored_", cacheKey.snapshotUUID());
                 if (randomBoolean()) {
                     final ByteRange absent = randomCacheFile.getAbsentRangeWithin(ByteRange.of(0L, randomCacheFile.getLength()));
                     if (absent != null) {
                         assertThat(
                             "Persistent cache should not contain any cached data",
-                            persistentCache.getCacheSize(cacheKey.getShardId(), snapshotId),
+                            persistentCache.getCacheSize(cacheKey.shardId(), snapshotId),
                             equalTo(0L)
                         );
 
@@ -244,7 +244,7 @@ public void testFSyncDoesNotAddDocumentsBackInPersistentCacheWhenShardIsEvicted(
                         assertThat(cacheService.getCacheFilesEventsQueueSize(), equalTo(0L));
                         assertThat(cacheService.getNumberOfCacheFilesEvents(), equalTo(0L));
 
-                        final long sizeInCache = persistentCache.getCacheSize(cacheKey.getShardId(), snapshotId);
+                        final long sizeInCache = persistentCache.getCacheSize(cacheKey.shardId(), snapshotId);
                         final long sizeInCacheFile = sumOfCompletedRangesLengths(randomCacheFile);
                         assertThat(
                             "Persistent cache should contain cached data for at least 1 cache file",
@@ -268,7 +268,7 @@ public void testFSyncDoesNotAddDocumentsBackInPersistentCacheWhenShardIsEvicted(
 
                         assertThat(
                             "Persistent cache should contain cached data for at least 1 cache file",
-                            persistentCache.getCacheSize(cacheKey.getShardId(), snapshotId),
+                            persistentCache.getCacheSize(cacheKey.shardId(), snapshotId),
                             equalTo(sizeInCache)
                         );
                     }
@@ -286,14 +286,10 @@ public void testFSyncDoesNotAddDocumentsBackInPersistentCacheWhenShardIsEvicted(
                 fileSystem.waitForBlock();
 
                 logger.debug("starting eviction of shard [{}]", cacheKey);
-                cacheService.markShardAsEvictedInCache(cacheKey.getSnapshotUUID(), cacheKey.getSnapshotIndexName(), cacheKey.getShardId());
+                cacheService.markShardAsEvictedInCache(cacheKey.snapshotUUID(), cacheKey.snapshotIndexName(), cacheKey.shardId());
 
                 logger.debug("waiting for shard eviction to be processed");
-                cacheService.waitForCacheFilesEvictionIfNeeded(
-                    cacheKey.getSnapshotUUID(),
-                    cacheKey.getSnapshotIndexName(),
-                    cacheKey.getShardId()
-                );
+                cacheService.waitForCacheFilesEvictionIfNeeded(cacheKey.snapshotUUID(), cacheKey.snapshotIndexName(), cacheKey.shardId());
 
                 logger.debug("unblocking synchronization of cache files");
                 fileSystem.unblock();
@@ -301,7 +297,7 @@ public void testFSyncDoesNotAddDocumentsBackInPersistentCacheWhenShardIsEvicted(
 
                 assertThat(
                     "Persistent cache should not report any cached data for the evicted shard",
-                    persistentCache.getCacheSize(cacheKey.getShardId(), new SnapshotId("_ignored_", cacheKey.getSnapshotUUID())),
+                    persistentCache.getCacheSize(cacheKey.shardId(), new SnapshotId("_ignored_", cacheKey.snapshotUUID())),
                     equalTo(0L)
                 );
 
@@ -310,11 +306,7 @@ public void testFSyncDoesNotAddDocumentsBackInPersistentCacheWhenShardIsEvicted(
 
                 assertThat(
                     "Persistent cache should not report any cached data for the evicted shard (ignoring deleted files)",
-                    persistentCache.getCacheSize(
-                        cacheKey.getShardId(),
-                        new SnapshotId("_ignored_", cacheKey.getSnapshotUUID()),
-                        path -> true
-                    ),
+                    persistentCache.getCacheSize(cacheKey.shardId(), new SnapshotId("_ignored_", cacheKey.snapshotUUID()), path -> true),
                     equalTo(0L)
                 );
 
@@ -335,7 +327,7 @@ public void testGetCacheSizeIgnoresDeletedCacheFiles() throws Exception {
             final long sizeOfCacheFileInCache = sumOfCompletedRangesLengths(cacheFile);
 
             final Supplier<Long> cacheSizeSupplier = () -> cacheService.getPersistentCache()
-                .getCacheSize(cacheFile.getCacheKey().getShardId(), new SnapshotId("_ignored_", cacheFile.getCacheKey().getSnapshotUUID()));
+                .getCacheSize(cacheFile.getCacheKey().shardId(), new SnapshotId("_ignored_", cacheFile.getCacheKey().snapshotUUID()));
             assertThat(cacheSizeSupplier.get(), equalTo(0L));
 
             cacheService.synchronizeCache();
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/FrozenCacheServiceTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/FrozenCacheServiceTests.java
deleted file mode 100644
index 73bdd783dac0..000000000000
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/FrozenCacheServiceTests.java
+++ /dev/null
@@ -1,405 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.searchablesnapshots.cache.shared;
-
-import org.elasticsearch.cluster.node.DiscoveryNodeRole;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
-import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.unit.RatioValue;
-import org.elasticsearch.common.unit.RelativeByteSizeValue;
-import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.NodeEnvironment;
-import org.elasticsearch.env.TestEnvironment;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.node.NodeRoleSettings;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.ByteRange;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService.CacheFileRegion;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import static org.elasticsearch.node.Node.NODE_NAME_SETTING;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
-
-public class FrozenCacheServiceTests extends ESTestCase {
-
-    private static long size(long numPages) {
-        return numPages * SharedBytes.PAGE_SIZE;
-    }
-
-    public void testBasicEviction() throws IOException {
-        Settings settings = Settings.builder()
-            .put(NODE_NAME_SETTING.getKey(), "node")
-            .put(FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(500)).getStringRep())
-            .put(FrozenCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(100)).getStringRep())
-            .put("path.home", createTempDir())
-            .build();
-        final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue();
-        try (
-            NodeEnvironment environment = new NodeEnvironment(settings, TestEnvironment.newEnvironment(settings));
-            FrozenCacheService cacheService = new FrozenCacheService(environment, settings, taskQueue.getThreadPool())
-        ) {
-            final CacheKey cacheKey = generateCacheKey();
-            assertEquals(5, cacheService.freeRegionCount());
-            final CacheFileRegion region0 = cacheService.get(cacheKey, size(250), 0);
-            assertEquals(size(100), region0.tracker.getLength());
-            assertEquals(4, cacheService.freeRegionCount());
-            final CacheFileRegion region1 = cacheService.get(cacheKey, size(250), 1);
-            assertEquals(size(100), region1.tracker.getLength());
-            assertEquals(3, cacheService.freeRegionCount());
-            final CacheFileRegion region2 = cacheService.get(cacheKey, size(250), 2);
-            assertEquals(size(50), region2.tracker.getLength());
-            assertEquals(2, cacheService.freeRegionCount());
-
-            assertTrue(region1.tryEvict());
-            assertEquals(3, cacheService.freeRegionCount());
-            assertFalse(region1.tryEvict());
-            assertEquals(3, cacheService.freeRegionCount());
-            region0.populateAndRead(
-                ByteRange.of(0L, 1L),
-                ByteRange.of(0L, 1L),
-                (channel, channelPos, relativePos, length) -> 1,
-                (channel, channelPos, relativePos, length, progressUpdater) -> progressUpdater.accept(length),
-                taskQueue.getThreadPool().executor(ThreadPool.Names.GENERIC)
-            );
-            assertFalse(region0.tryEvict());
-            assertEquals(3, cacheService.freeRegionCount());
-            taskQueue.runAllRunnableTasks();
-            assertTrue(region0.tryEvict());
-            assertEquals(4, cacheService.freeRegionCount());
-            assertTrue(region2.tryEvict());
-            assertEquals(5, cacheService.freeRegionCount());
-        }
-    }
-
-    public void testAutoEviction() throws IOException {
-        Settings settings = Settings.builder()
-            .put(NODE_NAME_SETTING.getKey(), "node")
-            .put(FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(200)).getStringRep())
-            .put(FrozenCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(100)).getStringRep())
-            .put("path.home", createTempDir())
-            .build();
-        final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue();
-        try (
-            NodeEnvironment environment = new NodeEnvironment(settings, TestEnvironment.newEnvironment(settings));
-            FrozenCacheService cacheService = new FrozenCacheService(environment, settings, taskQueue.getThreadPool())
-        ) {
-            final CacheKey cacheKey = generateCacheKey();
-            assertEquals(2, cacheService.freeRegionCount());
-            final CacheFileRegion region0 = cacheService.get(cacheKey, size(250), 0);
-            assertEquals(size(100), region0.tracker.getLength());
-            assertEquals(1, cacheService.freeRegionCount());
-            final CacheFileRegion region1 = cacheService.get(cacheKey, size(250), 1);
-            assertEquals(size(100), region1.tracker.getLength());
-            assertEquals(0, cacheService.freeRegionCount());
-            assertFalse(region0.isEvicted());
-            assertFalse(region1.isEvicted());
-
-            // acquire region 2, which should evict region 0 (oldest)
-            final CacheFileRegion region2 = cacheService.get(cacheKey, size(250), 2);
-            assertEquals(size(50), region2.tracker.getLength());
-            assertEquals(0, cacheService.freeRegionCount());
-            assertTrue(region0.isEvicted());
-            assertFalse(region1.isEvicted());
-
-            // explicitly evict region 1
-            assertTrue(region1.tryEvict());
-            assertEquals(1, cacheService.freeRegionCount());
-        }
-    }
-
-    public void testForceEviction() throws IOException {
-        Settings settings = Settings.builder()
-            .put(NODE_NAME_SETTING.getKey(), "node")
-            .put(FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(500)).getStringRep())
-            .put(FrozenCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(100)).getStringRep())
-            .put("path.home", createTempDir())
-            .build();
-        final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue();
-        try (
-            NodeEnvironment environment = new NodeEnvironment(settings, TestEnvironment.newEnvironment(settings));
-            FrozenCacheService cacheService = new FrozenCacheService(environment, settings, taskQueue.getThreadPool())
-        ) {
-            final CacheKey cacheKey1 = generateCacheKey();
-            final CacheKey cacheKey2 = generateCacheKey();
-            assertEquals(5, cacheService.freeRegionCount());
-            final CacheFileRegion region0 = cacheService.get(cacheKey1, size(250), 0);
-            assertEquals(4, cacheService.freeRegionCount());
-            final CacheFileRegion region1 = cacheService.get(cacheKey2, size(250), 1);
-            assertEquals(3, cacheService.freeRegionCount());
-            assertFalse(region0.isEvicted());
-            assertFalse(region1.isEvicted());
-            cacheService.removeFromCache(cacheKey1);
-            assertTrue(region0.isEvicted());
-            assertFalse(region1.isEvicted());
-            assertEquals(4, cacheService.freeRegionCount());
-        }
-    }
-
-    public void testDecay() throws IOException {
-        Settings settings = Settings.builder()
-            .put(NODE_NAME_SETTING.getKey(), "node")
-            .put(FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(500)).getStringRep())
-            .put(FrozenCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(100)).getStringRep())
-            .put("path.home", createTempDir())
-            .build();
-        final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue();
-        try (
-            NodeEnvironment environment = new NodeEnvironment(settings, TestEnvironment.newEnvironment(settings));
-            FrozenCacheService cacheService = new FrozenCacheService(environment, settings, taskQueue.getThreadPool())
-        ) {
-            final CacheKey cacheKey1 = generateCacheKey();
-            final CacheKey cacheKey2 = generateCacheKey();
-            assertEquals(5, cacheService.freeRegionCount());
-            final CacheFileRegion region0 = cacheService.get(cacheKey1, size(250), 0);
-            assertEquals(4, cacheService.freeRegionCount());
-            final CacheFileRegion region1 = cacheService.get(cacheKey2, size(250), 1);
-            assertEquals(3, cacheService.freeRegionCount());
-
-            assertEquals(0, cacheService.getFreq(region0));
-            assertEquals(0, cacheService.getFreq(region1));
-
-            taskQueue.advanceTime();
-            taskQueue.runAllRunnableTasks();
-
-            final CacheFileRegion region0Again = cacheService.get(cacheKey1, size(250), 0);
-            assertSame(region0Again, region0);
-            assertEquals(1, cacheService.getFreq(region0));
-            assertEquals(0, cacheService.getFreq(region1));
-
-            taskQueue.advanceTime();
-            taskQueue.runAllRunnableTasks();
-            cacheService.get(cacheKey1, size(250), 0);
-            assertEquals(2, cacheService.getFreq(region0));
-            cacheService.get(cacheKey1, size(250), 0);
-            assertEquals(2, cacheService.getFreq(region0));
-
-            // advance 2 ticks (decay only starts after 2 ticks)
-            taskQueue.advanceTime();
-            taskQueue.runAllRunnableTasks();
-            taskQueue.advanceTime();
-            taskQueue.runAllRunnableTasks();
-            assertEquals(1, cacheService.getFreq(region0));
-            assertEquals(0, cacheService.getFreq(region1));
-
-            // advance another tick
-            taskQueue.advanceTime();
-            taskQueue.runAllRunnableTasks();
-            assertEquals(0, cacheService.getFreq(region0));
-            assertEquals(0, cacheService.getFreq(region1));
-        }
-    }
-
-    public void testCacheSizeRejectedOnNonFrozenNodes() {
-        String cacheSize = randomBoolean()
-            ? ByteSizeValue.ofBytes(size(500)).getStringRep()
-            : (new RatioValue(between(1, 100))).formatNoTrailingZerosPercent();
-        final Settings settings = Settings.builder()
-            .put(FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), cacheSize)
-            .put(FrozenCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(100)).getStringRep())
-            .putList(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), DiscoveryNodeRole.DATA_HOT_NODE_ROLE.roleName())
-            .build();
-        final IllegalArgumentException e = expectThrows(
-            IllegalArgumentException.class,
-            () -> FrozenCacheService.SHARED_CACHE_SIZE_SETTING.get(settings)
-        );
-        assertThat(e.getCause(), notNullValue());
-        assertThat(e.getCause(), instanceOf(SettingsException.class));
-        assertThat(
-            e.getCause().getMessage(),
-            is(
-                "setting ["
-                    + FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey()
-                    + "] to be positive ["
-                    + cacheSize
-                    + "] is only permitted on nodes with the data_frozen role, roles are [data_hot]"
-            )
-        );
-    }
-
-    public void testMultipleDataPathsRejectedOnFrozenNodes() {
-        final Settings settings = Settings.builder()
-            .put(FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(500)).getStringRep())
-            .putList(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE.roleName())
-            .putList(Environment.PATH_DATA_SETTING.getKey(), List.of("a", "b"))
-            .build();
-        final IllegalArgumentException e = expectThrows(
-            IllegalArgumentException.class,
-            () -> FrozenCacheService.SHARED_CACHE_SIZE_SETTING.get(settings)
-        );
-        assertThat(e.getCause(), notNullValue());
-        assertThat(e.getCause(), instanceOf(SettingsException.class));
-        assertThat(
-            e.getCause().getMessage(),
-            is(
-                "setting ["
-                    + FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey()
-                    + "="
-                    + ByteSizeValue.ofBytes(size(500)).getStringRep()
-                    + "] is not permitted on nodes with multiple data paths [a,b]"
-            )
-        );
-    }
-
-    public void testDedicateFrozenCacheSizeDefaults() {
-        final Settings settings = Settings.builder()
-            .putList(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE.roleName())
-            .build();
-
-        RelativeByteSizeValue relativeCacheSize = FrozenCacheService.SHARED_CACHE_SIZE_SETTING.get(settings);
-        assertThat(relativeCacheSize.isAbsolute(), is(false));
-        assertThat(relativeCacheSize.isNonZeroSize(), is(true));
-        assertThat(relativeCacheSize.calculateValue(ByteSizeValue.ofBytes(10000), null), equalTo(ByteSizeValue.ofBytes(9000)));
-        assertThat(FrozenCacheService.SHARED_CACHE_SIZE_MAX_HEADROOM_SETTING.get(settings), equalTo(ByteSizeValue.ofGb(100)));
-    }
-
-    public void testNotDedicatedFrozenCacheSizeDefaults() {
-        final Settings settings = Settings.builder()
-            .putList(
-                NodeRoleSettings.NODE_ROLES_SETTING.getKey(),
-                Sets.union(
-                    Set.of(
-                        randomFrom(
-                            DiscoveryNodeRole.DATA_HOT_NODE_ROLE,
-                            DiscoveryNodeRole.DATA_COLD_NODE_ROLE,
-                            DiscoveryNodeRole.DATA_WARM_NODE_ROLE,
-                            DiscoveryNodeRole.DATA_CONTENT_NODE_ROLE
-                        )
-                    ),
-                    new HashSet<>(
-                        randomSubsetOf(
-                            between(0, 3),
-                            DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE,
-                            DiscoveryNodeRole.INGEST_ROLE,
-                            DiscoveryNodeRole.MASTER_ROLE
-                        )
-                    )
-                ).stream().map(DiscoveryNodeRole::roleName).collect(Collectors.toList())
-            )
-            .build();
-
-        RelativeByteSizeValue relativeCacheSize = FrozenCacheService.SHARED_CACHE_SIZE_SETTING.get(settings);
-        assertThat(relativeCacheSize.isNonZeroSize(), is(false));
-        assertThat(relativeCacheSize.isAbsolute(), is(true));
-        assertThat(relativeCacheSize.getAbsolute(), equalTo(ByteSizeValue.ZERO));
-        assertThat(FrozenCacheService.SHARED_CACHE_SIZE_MAX_HEADROOM_SETTING.get(settings), equalTo(ByteSizeValue.ofBytes(-1)));
-    }
-
-    public void testMaxHeadroomRejectedForAbsoluteCacheSize() {
-        String cacheSize = ByteSizeValue.ofBytes(size(500)).getStringRep();
-        final Settings settings = Settings.builder()
-            .put(FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), cacheSize)
-            .put(FrozenCacheService.SHARED_CACHE_SIZE_MAX_HEADROOM_SETTING.getKey(), ByteSizeValue.ofBytes(size(100)).getStringRep())
-            .putList(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE.roleName())
-            .build();
-        final IllegalArgumentException e = expectThrows(
-            IllegalArgumentException.class,
-            () -> FrozenCacheService.SHARED_CACHE_SIZE_MAX_HEADROOM_SETTING.get(settings)
-        );
-        assertThat(e.getCause(), notNullValue());
-        assertThat(e.getCause(), instanceOf(SettingsException.class));
-        assertThat(
-            e.getCause().getMessage(),
-            is(
-                "setting ["
-                    + FrozenCacheService.SHARED_CACHE_SIZE_MAX_HEADROOM_SETTING.getKey()
-                    + "] cannot be specified for absolute ["
-                    + FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey()
-                    + "="
-                    + cacheSize
-                    + "]"
-            )
-        );
-    }
-
-    public void testCalculateCacheSize() {
-        long smallSize = 10000;
-        long largeSize = ByteSizeValue.ofTb(10).getBytes();
-        assertThat(FrozenCacheService.calculateCacheSize(Settings.EMPTY, smallSize), equalTo(0L));
-        final Settings settings = Settings.builder()
-            .putList(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE.roleName())
-            .build();
-        assertThat(FrozenCacheService.calculateCacheSize(settings, smallSize), equalTo(9000L));
-        assertThat(FrozenCacheService.calculateCacheSize(settings, largeSize), equalTo(largeSize - ByteSizeValue.ofGb(100).getBytes()));
-    }
-
-    private static CacheKey generateCacheKey() {
-        return new CacheKey(
-            randomAlphaOfLength(10),
-            randomAlphaOfLength(10),
-            new ShardId(randomAlphaOfLength(10), randomAlphaOfLength(10), randomInt(10)),
-            randomAlphaOfLength(10)
-        );
-    }
-
-    public void testCacheSizeChanges() throws IOException {
-        ByteSizeValue val1 = new ByteSizeValue(randomIntBetween(1, 5), ByteSizeUnit.MB);
-        Settings settings = Settings.builder()
-            .put(NODE_NAME_SETTING.getKey(), "node")
-            .put(FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), val1.getStringRep())
-            .put(FrozenCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(100)).getStringRep())
-            .put("path.home", createTempDir())
-            .build();
-        final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue();
-        try (
-            NodeEnvironment environment = new NodeEnvironment(settings, TestEnvironment.newEnvironment(settings));
-            FrozenCacheService cacheService = new FrozenCacheService(environment, settings, taskQueue.getThreadPool())
-        ) {
-            assertEquals(val1.getBytes(), cacheService.getStats().getSize());
-        }
-
-        ByteSizeValue val2 = new ByteSizeValue(randomIntBetween(1, 5), ByteSizeUnit.MB);
-        settings = Settings.builder().put(settings).put(FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), val2.getStringRep()).build();
-        try (
-            NodeEnvironment environment = new NodeEnvironment(settings, TestEnvironment.newEnvironment(settings));
-            FrozenCacheService cacheService = new FrozenCacheService(environment, settings, taskQueue.getThreadPool())
-        ) {
-            assertEquals(val2.getBytes(), cacheService.getStats().getSize());
-        }
-    }
-
-    private void assertThatNonPositiveRecoveryRangeSizeRejected(Setting<ByteSizeValue> setting) {
-        final String value = randomFrom(ByteSizeValue.MINUS_ONE, ByteSizeValue.ZERO).getStringRep();
-        final Settings settings = Settings.builder()
-            .put(FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), ByteSizeValue.ofBytes(size(100)).getStringRep())
-            .putList(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE.roleName())
-            .put(setting.getKey(), value)
-            .build();
-        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> setting.get(settings));
-        assertThat(e.getCause(), notNullValue());
-        assertThat(e.getCause(), instanceOf(SettingsException.class));
-        assertThat(e.getCause().getMessage(), is("setting [" + setting.getKey() + "] must be greater than zero"));
-    }
-
-    public void testNonPositiveRegionSizeRejected() {
-        assertThatNonPositiveRecoveryRangeSizeRejected(FrozenCacheService.SHARED_CACHE_REGION_SIZE_SETTING);
-    }
-
-    public void testNonPositiveRangeSizeRejected() {
-        assertThatNonPositiveRecoveryRangeSizeRejected(FrozenCacheService.SHARED_CACHE_RANGE_SIZE_SETTING);
-    }
-
-    public void testNonPositiveRecoveryRangeSizeRejected() {
-        assertThatNonPositiveRecoveryRangeSizeRejected(FrozenCacheService.SHARED_CACHE_RECOVERY_RANGE_SIZE_SETTING);
-    }
-
-}
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/IndexInputStatsTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/IndexInputStatsTests.java
index 2df88235d324..96a0b902bcf0 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/IndexInputStatsTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/IndexInputStatsTests.java
@@ -10,7 +10,7 @@
 
 import java.util.function.LongSupplier;
 
-import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsUtils.toIntBytes;
+import static org.elasticsearch.blobcache.BlobCacheUtils.toIntBytes;
 import static org.elasticsearch.xpack.searchablesnapshots.cache.common.TestUtils.assertCounter;
 import static org.elasticsearch.xpack.searchablesnapshots.store.IndexInputStats.SEEKING_THRESHOLD;
 
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryStatsTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryStatsTests.java
index 5aa60360bfa3..f862f79d9b1a 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryStatsTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryStatsTests.java
@@ -11,6 +11,9 @@
 import org.apache.lucene.store.IndexInput;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.blobcache.BlobCacheTestUtils;
+import org.elasticsearch.blobcache.shared.SharedBlobCacheService;
+import org.elasticsearch.blobcache.shared.SharedBytes;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -35,11 +38,9 @@
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.xpack.searchablesnapshots.AbstractSearchableSnapshotsTestCase;
+import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey;
 import org.elasticsearch.xpack.searchablesnapshots.cache.common.TestUtils;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.TestUtils.NoopBlobStoreCacheService;
 import org.elasticsearch.xpack.searchablesnapshots.cache.full.CacheService;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.SharedBytes;
 import org.elasticsearch.xpack.searchablesnapshots.recovery.SearchableSnapshotRecoveryState;
 
 import java.io.IOException;
@@ -49,10 +50,10 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.LongSupplier;
 
+import static org.elasticsearch.blobcache.BlobCacheUtils.toIntBytes;
 import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING;
 import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_CACHE_PREWARM_ENABLED_SETTING;
 import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_UNCACHED_CHUNK_SIZE_SETTING;
-import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsUtils.toIntBytes;
 import static org.elasticsearch.xpack.searchablesnapshots.cache.common.TestUtils.assertCounter;
 import static org.elasticsearch.xpack.searchablesnapshots.cache.common.TestUtils.singleBlobContainer;
 import static org.elasticsearch.xpack.searchablesnapshots.cache.full.CacheService.resolveSnapshotCache;
@@ -126,7 +127,7 @@ public void testCachedBytesReadsAndWrites() throws Exception {
                 final byte[] result = randomReadAndSlice(input, toIntBytes(length));
                 assertArrayEquals(fileContent, result);
 
-                final long cachedBytesWriteCount = TestUtils.numberOfRanges(length, rangeSize.getBytes());
+                final long cachedBytesWriteCount = BlobCacheTestUtils.numberOfRanges(length, rangeSize.getBytes());
 
                 // cache writes are executed in a different thread pool and can take some time to be processed
                 assertBusy(() -> {
@@ -607,7 +608,7 @@ private void executeTestCaseWithCache(
 
     private void executeTestCase(
         final CacheService cacheService,
-        final FrozenCacheService frozenCacheService,
+        final SharedBlobCacheService<CacheKey> sharedBlobCacheService,
         final Settings indexSettings,
         final TriConsumer<String, byte[], SearchableSnapshotDirectory> test
     ) throws Exception {
@@ -649,11 +650,11 @@ private void executeTestCase(
 
         try (
             CacheService ignored = cacheService;
-            FrozenCacheService ignored2 = frozenCacheService;
+            var ignored2 = sharedBlobCacheService;
             SearchableSnapshotDirectory directory = new SearchableSnapshotDirectory(
                 () -> blobContainer,
                 () -> snapshot,
-                new NoopBlobStoreCacheService(),
+                new TestUtils.NoopBlobStoreCacheService(),
                 "_repo",
                 snapshotId,
                 indexId,
@@ -664,7 +665,7 @@ private void executeTestCase(
                 cacheDir,
                 shardPath,
                 threadPool,
-                frozenCacheService
+                sharedBlobCacheService
             ) {
                 @Override
                 protected IndexInputStats createIndexInputStats(long numFiles, long totalSize, long minSize, long maxSize) {
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryTests.java
index 612af98a488b..c9ee774930e2 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/SearchableSnapshotDirectoryTests.java
@@ -40,6 +40,7 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.blobcache.shared.SharedBlobCacheService;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
 import org.elasticsearch.common.CheckedBiConsumer;
@@ -90,9 +91,9 @@
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xpack.searchablesnapshots.AbstractSearchableSnapshotsTestCase;
+import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey;
 import org.elasticsearch.xpack.searchablesnapshots.cache.common.TestUtils;
 import org.elasticsearch.xpack.searchablesnapshots.cache.full.CacheService;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService;
 import org.elasticsearch.xpack.searchablesnapshots.recovery.SearchableSnapshotRecoveryState;
 import org.elasticsearch.xpack.searchablesnapshots.store.input.ChecksumBlobContainerIndexInput;
 import org.hamcrest.Matcher;
@@ -122,7 +123,7 @@
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
-import static java.util.Collections.emptyMap;
+import static org.elasticsearch.blobcache.BlobCacheUtils.toIntBytes;
 import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING;
 import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_CACHE_EXCLUDED_FILE_TYPES_SETTING;
 import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_CACHE_PREWARM_ENABLED_SETTING;
@@ -131,7 +132,6 @@
 import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_REPOSITORY_NAME_SETTING;
 import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_SNAPSHOT_ID_SETTING;
 import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_SNAPSHOT_NAME_SETTING;
-import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsUtils.toIntBytes;
 import static org.elasticsearch.xpack.searchablesnapshots.cache.full.CacheService.resolveSnapshotCache;
 import static org.elasticsearch.xpack.searchablesnapshots.store.SearchableSnapshotDirectory.getNonNullFileExt;
 import static org.hamcrest.Matchers.allOf;
@@ -631,7 +631,6 @@ protected void assertSnapshotOrGenericThread() {
                             null,
                             snapshotStatus,
                             Version.CURRENT,
-                            emptyMap(),
                             randomMillisUpToYear9999(),
                             future
                         )
@@ -649,8 +648,8 @@ protected void assertSnapshotOrGenericThread() {
                 final CacheService cacheService = defaultCacheService();
                 releasables.add(cacheService);
                 cacheService.start();
-                final FrozenCacheService frozenCacheService = defaultFrozenCacheService();
-                releasables.add(frozenCacheService);
+                final SharedBlobCacheService<CacheKey> sharedBlobCacheService = defaultFrozenCacheService();
+                releasables.add(sharedBlobCacheService);
 
                 try (
                     SearchableSnapshotDirectory snapshotDirectory = new SearchableSnapshotDirectory(
@@ -671,7 +670,7 @@ protected void assertSnapshotOrGenericThread() {
                         cacheDir,
                         shardPath,
                         threadPool,
-                        frozenCacheService
+                        sharedBlobCacheService
                     )
                 ) {
                     final PlainActionFuture<Void> f = PlainActionFuture.newFuture();
@@ -754,7 +753,7 @@ public void testClearCache() throws Exception {
             final Path shardDir = randomShardPath(shardId);
             final ShardPath shardPath = new ShardPath(false, shardDir, shardDir, shardId);
             final Path cacheDir = Files.createDirectories(resolveSnapshotCache(shardDir).resolve(snapshotId.getUUID()));
-            final FrozenCacheService frozenCacheService = defaultFrozenCacheService();
+            final SharedBlobCacheService<CacheKey> sharedBlobCacheService = defaultFrozenCacheService();
             try (
                 SearchableSnapshotDirectory directory = new SearchableSnapshotDirectory(
                     () -> blobContainer,
@@ -775,7 +774,7 @@ public void testClearCache() throws Exception {
                     cacheDir,
                     shardPath,
                     threadPool,
-                    frozenCacheService
+                    sharedBlobCacheService
                 )
             ) {
                 final RecoveryState recoveryState = createRecoveryState(randomBoolean());
@@ -809,7 +808,7 @@ public void testClearCache() throws Exception {
                     }
                 }
             } finally {
-                frozenCacheService.close();
+                sharedBlobCacheService.close();
                 assertThreadPoolNotBusy(threadPool);
             }
         }
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInputTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInputTests.java
index 2d1898604b57..7e240858fe90 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInputTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInputTests.java
@@ -10,6 +10,8 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.blobcache.BlobCacheTestUtils;
+import org.elasticsearch.blobcache.shared.SharedBlobCacheService;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.support.FilterBlobContainer;
 import org.elasticsearch.common.settings.Settings;
@@ -23,10 +25,9 @@
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.xpack.searchablesnapshots.AbstractSearchableSnapshotsTestCase;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.TestUtils;
+import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey;
 import org.elasticsearch.xpack.searchablesnapshots.cache.common.TestUtils.NoopBlobStoreCacheService;
 import org.elasticsearch.xpack.searchablesnapshots.cache.full.CacheService;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService;
 import org.elasticsearch.xpack.searchablesnapshots.store.SearchableSnapshotDirectory;
 
 import java.io.EOFException;
@@ -102,7 +103,7 @@ public void testRandomReads() throws Exception {
                 final Path shardDir = randomShardPath(shardId);
                 final ShardPath shardPath = new ShardPath(false, shardDir, shardDir, shardId);
                 final Path cacheDir = Files.createDirectories(resolveSnapshotCache(shardDir).resolve(snapshotId.getUUID()));
-                final FrozenCacheService frozenCacheService = defaultFrozenCacheService();
+                final SharedBlobCacheService<CacheKey> sharedBlobCacheService = defaultFrozenCacheService();
                 try (
                     SearchableSnapshotDirectory directory = new SearchableSnapshotDirectory(
                         () -> blobContainer,
@@ -121,7 +122,7 @@ public void testRandomReads() throws Exception {
                         cacheDir,
                         shardPath,
                         threadPool,
-                        frozenCacheService
+                        sharedBlobCacheService
                     )
                 ) {
                     RecoveryState recoveryState = createRecoveryState(recoveryFinalizedDone);
@@ -143,11 +144,11 @@ public void testRandomReads() throws Exception {
                         assertArrayEquals(input, output);
                     }
                 } finally {
-                    frozenCacheService.close();
+                    sharedBlobCacheService.close();
                 }
 
                 if (blobContainer instanceof CountingBlobContainer) {
-                    long numberOfRanges = TestUtils.numberOfRanges(
+                    long numberOfRanges = BlobCacheTestUtils.numberOfRanges(
                         input.length,
                         recoveryFinalizedDone ? cacheService.getRangeSize() : cacheService.getRecoveryRangeSize()
                     );
@@ -211,7 +212,7 @@ public void testThrowsEOFException() throws Exception {
             final Path shardDir = randomShardPath(shardId);
             final ShardPath shardPath = new ShardPath(false, shardDir, shardDir, shardId);
             final Path cacheDir = Files.createDirectories(resolveSnapshotCache(shardDir).resolve(snapshotId.getUUID()));
-            final FrozenCacheService frozenCacheService = defaultFrozenCacheService();
+            final SharedBlobCacheService<CacheKey> sharedBlobCacheService = defaultFrozenCacheService();
             try (
                 SearchableSnapshotDirectory searchableSnapshotDirectory = new SearchableSnapshotDirectory(
                     () -> blobContainer,
@@ -227,7 +228,7 @@ public void testThrowsEOFException() throws Exception {
                     cacheDir,
                     shardPath,
                     threadPool,
-                    frozenCacheService
+                    sharedBlobCacheService
                 )
             ) {
                 RecoveryState recoveryState = createRecoveryState(randomBoolean());
@@ -251,7 +252,7 @@ public void testThrowsEOFException() throws Exception {
                     }
                 }
             } finally {
-                frozenCacheService.close();
+                sharedBlobCacheService.close();
                 assertThreadPoolNotBusy(threadPool);
             }
         }
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInputTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInputTests.java
index a700cb134ceb..ab7cb4327ea9 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInputTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInputTests.java
@@ -15,10 +15,7 @@
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo;
 import org.elasticsearch.index.store.StoreFileMetadata;
-import org.elasticsearch.xpack.searchablesnapshots.cache.blob.CachedBlob;
-import org.elasticsearch.xpack.searchablesnapshots.cache.common.ByteRange;
 import org.elasticsearch.xpack.searchablesnapshots.store.IndexInputStats;
-import org.elasticsearch.xpack.searchablesnapshots.store.SearchableSnapshotDirectory;
 
 import java.io.ByteArrayInputStream;
 import java.io.EOFException;
@@ -27,9 +24,9 @@
 import java.io.InputStream;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import static org.elasticsearch.blobcache.BlobCacheUtils.toIntBytes;
 import static org.elasticsearch.xpack.searchablesnapshots.AbstractSearchableSnapshotsTestCase.randomChecksumBytes;
 import static org.elasticsearch.xpack.searchablesnapshots.AbstractSearchableSnapshotsTestCase.randomIOContext;
-import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsUtils.toIntBytes;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -37,7 +34,6 @@
 import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.startsWith;
-import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
@@ -122,13 +118,9 @@ public int read(byte[] b, int off, int len) throws IOException {
             }
         });
 
-        final SearchableSnapshotDirectory directory = mock(SearchableSnapshotDirectory.class);
-        when(directory.getCachedBlob(anyString(), any(ByteRange.class))).thenReturn(CachedBlob.CACHE_NOT_READY);
-        when(directory.blobContainer()).thenReturn(blobContainer);
-
         final DirectBlobContainerIndexInput indexInput = new DirectBlobContainerIndexInput(
             fileName,
-            directory,
+            blobContainer,
             fileInfo,
             randomIOContext(),
             new IndexInputStats(1L, fileInfo.length(), fileInfo.length(), fileInfo.length(), () -> 0L),
diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInputTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInputTests.java
index ff52975f7eed..f0868a550f6c 100644
--- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInputTests.java
+++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/store/input/FrozenIndexInputTests.java
@@ -10,6 +10,8 @@
 import org.apache.lucene.store.IndexInput;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.blobcache.shared.SharedBlobCacheService;
+import org.elasticsearch.blobcache.shared.SharedBytes;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.Tuple;
@@ -27,10 +29,9 @@
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.xpack.searchablesnapshots.AbstractSearchableSnapshotsTestCase;
 import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots;
+import org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheKey;
 import org.elasticsearch.xpack.searchablesnapshots.cache.common.TestUtils;
 import org.elasticsearch.xpack.searchablesnapshots.cache.full.CacheService;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService;
-import org.elasticsearch.xpack.searchablesnapshots.cache.shared.SharedBytes;
 import org.elasticsearch.xpack.searchablesnapshots.store.SearchableSnapshotDirectory;
 
 import java.io.IOException;
@@ -60,7 +61,7 @@ public void testRandomReads() throws IOException {
 
         final ByteSizeValue rangeSize;
         if (rarely()) {
-            rangeSize = FrozenCacheService.SHARED_CACHE_RANGE_SIZE_SETTING.get(Settings.EMPTY);
+            rangeSize = SharedBlobCacheService.SHARED_CACHE_RANGE_SIZE_SETTING.get(Settings.EMPTY);
         } else if (randomBoolean()) {
             rangeSize = ByteSizeValue.ofBytes(randomIntBetween(1, 16) * SharedBytes.PAGE_SIZE);
         } else {
@@ -69,7 +70,7 @@ public void testRandomReads() throws IOException {
 
         final ByteSizeValue regionSize;
         if (rarely()) {
-            regionSize = FrozenCacheService.SHARED_CACHE_REGION_SIZE_SETTING.get(Settings.EMPTY);
+            regionSize = SharedBlobCacheService.SHARED_CACHE_REGION_SIZE_SETTING.get(Settings.EMPTY);
         } else {
             regionSize = ByteSizeValue.ofBytes(randomIntBetween(1, 16) * SharedBytes.PAGE_SIZE);
         }
@@ -82,9 +83,9 @@ public void testRandomReads() throws IOException {
         }
 
         final Settings settings = Settings.builder()
-            .put(FrozenCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(), regionSize)
-            .put(FrozenCacheService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(), rangeSize)
-            .put(FrozenCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), cacheSize)
+            .put(SharedBlobCacheService.SHARED_CACHE_REGION_SIZE_SETTING.getKey(), regionSize)
+            .put(SharedBlobCacheService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(), rangeSize)
+            .put(SharedBlobCacheService.SHARED_CACHE_SIZE_SETTING.getKey(), cacheSize)
             .put("path.home", createTempDir())
             .build();
         final Environment environment = TestEnvironment.newEnvironment(settings);
@@ -97,10 +98,10 @@ public void testRandomReads() throws IOException {
         final Path cacheDir = Files.createDirectories(resolveSnapshotCache(shardDir).resolve(snapshotId.getUUID()));
         try (
             NodeEnvironment nodeEnvironment = new NodeEnvironment(settings, environment);
-            FrozenCacheService frozenCacheService = new FrozenCacheService(nodeEnvironment, settings, threadPool);
+            SharedBlobCacheService<CacheKey> sharedBlobCacheService = new SharedBlobCacheService<>(nodeEnvironment, settings, threadPool);
             CacheService cacheService = randomCacheService();
             TestSearchableSnapshotDirectory directory = new TestSearchableSnapshotDirectory(
-                frozenCacheService,
+                sharedBlobCacheService,
                 cacheService,
                 fileInfo,
                 snapshotId,
@@ -127,7 +128,7 @@ public void testRandomReads() throws IOException {
     private class TestSearchableSnapshotDirectory extends SearchableSnapshotDirectory {
 
         TestSearchableSnapshotDirectory(
-            FrozenCacheService service,
+            SharedBlobCacheService<CacheKey> service,
             CacheService cacheService,
             FileInfo fileInfo,
             SnapshotId snapshotId,
diff --git a/x-pack/plugin/security/qa/basic-enable-security/build.gradle b/x-pack/plugin/security/qa/basic-enable-security/build.gradle
index 1ed92dff3381..aa4a632255d5 100644
--- a/x-pack/plugin/security/qa/basic-enable-security/build.gradle
+++ b/x-pack/plugin/security/qa/basic-enable-security/build.gradle
@@ -5,7 +5,6 @@
  * 2.0.
  */
 
-import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 import org.elasticsearch.gradle.internal.info.BuildParams
 
 apply plugin: 'elasticsearch.internal-java-rest-test'
@@ -16,57 +15,8 @@ dependencies {
 }
 
 tasks.named("javaRestTest").configure {
-  description = "Run tests against a cluster that has security disabled"
-  systemProperty 'tests.has_security', 'false'
-}
-
-if (BuildParams.inFipsJvm){
   // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC
-  tasks.named("javaRestTest").configure{enabled = false }
-}
-
-def cluster = testClusters.register("javaRestTest") {
-    testDistribution = 'DEFAULT'
-    numberOfNodes = 2
-    setting 'xpack.ml.enabled', 'false'
-    setting 'xpack.license.self_generated.type', 'basic'
-    setting 'xpack.security.enabled', 'false'
-}
-
-tasks.register("javaRestTestWithSecurityEnabled", StandaloneRestIntegTestTask) {
-  mustRunAfter("javaRestTest")
-  description = "Run tests against a cluster that has security enabled"
-  dependsOn "javaRestTest"
-  useCluster cluster
-  systemProperty 'tests.has_security', 'true'
-  testClassesDirs = sourceSets.javaRestTest.output.classesDirs
-  classpath = sourceSets.javaRestTest.runtimeClasspath
-  onlyIf {
-    BuildParams.inFipsJvm == false
-  }
-  doFirst {
-    cluster.configure {
-      // TODO Rene: revisit if using dedicated new cluster definitions would be more efficient.
-      // Reconfigure cluster to enable security
-      setting 'xpack.security.enabled', 'true'
-      setting 'xpack.security.authc.anonymous.roles', 'anonymous'
-      setting 'xpack.security.transport.ssl.enabled', 'true'
-      setting 'xpack.security.transport.ssl.certificate', 'transport.crt'
-      setting 'xpack.security.transport.ssl.key', 'transport.key'
-      setting 'xpack.security.transport.ssl.key_passphrase', 'transport-password'
-      setting 'xpack.security.transport.ssl.certificate_authorities', 'ca.crt'
-
-      extraConfigFile 'transport.key', file('src/javaRestTest/resources/ssl/transport.key')
-      extraConfigFile 'transport.crt', file('src/javaRestTest/resources/ssl/transport.crt')
-      extraConfigFile 'ca.crt', file('src/javaRestTest/resources/ssl/ca.crt')
-      rolesFile file('src/javaRestTest/resources/roles.yml')
-
-      user username: "admin_user", password: "admin-password"
-      user username: "security_test_user", password: "security-test-password", role: "security_test_role"
+  onlyIf { BuildParams.inFipsJvm == false }
 
-      restart()
-    }
-    nonInputProperties.systemProperty 'tests.rest.cluster', cluster.map(c->c.getAllHttpSocketURI().join(","))
-  }
+  usesDefaultDistribution()
 }
-tasks.named("check").configure { dependsOn("javaRestTestWithSecurityEnabled") }
diff --git a/x-pack/plugin/security/qa/basic-enable-security/src/javaRestTest/java/org/elasticsearch/xpack/security/EnableSecurityOnBasicLicenseIT.java b/x-pack/plugin/security/qa/basic-enable-security/src/javaRestTest/java/org/elasticsearch/xpack/security/EnableSecurityOnBasicLicenseIT.java
index 7c2c9fe7cbc0..ec2a3e2b0180 100644
--- a/x-pack/plugin/security/qa/basic-enable-security/src/javaRestTest/java/org/elasticsearch/xpack/security/EnableSecurityOnBasicLicenseIT.java
+++ b/x-pack/plugin/security/qa/basic-enable-security/src/javaRestTest/java/org/elasticsearch/xpack/security/EnableSecurityOnBasicLicenseIT.java
@@ -6,6 +6,8 @@
  */
 package org.elasticsearch.xpack.security;
 
+import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
+
 import org.apache.http.HttpHost;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
@@ -16,12 +18,17 @@
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.core.Booleans;
+import org.elasticsearch.test.AnnotationTestOrdering;
+import org.elasticsearch.test.AnnotationTestOrdering.Order;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.MutableSettingsProvider;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.cluster.util.resource.Resource;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.ObjectPath;
 import org.elasticsearch.xpack.security.authc.InternalRealms;
 import org.hamcrest.Matchers;
-import org.junit.BeforeClass;
+import org.junit.ClassRule;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -34,14 +41,37 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
+@TestCaseOrdering(AnnotationTestOrdering.class)
 public class EnableSecurityOnBasicLicenseIT extends ESRestTestCase {
+    private static MutableSettingsProvider clusterSettings = new MutableSettingsProvider() {
+        {
+            put("xpack.ml.enabled", "false");
+            put("xpack.security.enabled", "false");
+            put("xpack.license.self_generated.type", "basic");
+        }
+    };
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .nodes(2)
+        .distribution(DistributionType.DEFAULT)
+        .settings(clusterSettings)
+        .configFile("transport.key", Resource.fromClasspath("ssl/transport.key"))
+        .configFile("transport.crt", Resource.fromClasspath("ssl/transport.crt"))
+        .configFile("ca.crt", Resource.fromClasspath("ssl/ca.crt"))
+        .rolesFile(Resource.fromClasspath("roles.yml"))
+        .user("admin_user", "admin-password")
+        .user("security_test_user", "security-test-password", "security_test_role")
+        .build();
 
-    private static boolean securityEnabled;
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
 
-    @BeforeClass
-    public static void checkTestMode() {
-        final String hasSecurity = System.getProperty("tests.has_security");
-        securityEnabled = Booleans.parseBoolean(hasSecurity);
+    @Override
+    protected boolean preserveClusterUponCompletion() {
+        return true;
     }
 
     @Override
@@ -56,13 +86,6 @@ protected Settings restClientSettings() {
         return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
     }
 
-    @Override
-    protected boolean preserveClusterUponCompletion() {
-        // If this is the first run (security is disabled), then don't clean up afterwards because we want to test restart
-        // with data
-        return securityEnabled == false;
-    }
-
     @Override
     protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException {
         RestClientBuilder builder = RestClient.builder(hosts);
@@ -70,7 +93,30 @@ protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOE
         return builder.build();
     }
 
-    public void testSecuritySetup() throws Exception {
+    @Order(1)
+    public void testSecurityDisabledSetup() throws Exception {
+        checkSecuritySetup(false);
+    }
+
+    @Order(2)
+    public void testEnableSecurityAndRestartCluster() throws IOException {
+        clusterSettings.put("xpack.security.enabled", "true");
+        clusterSettings.put("xpack.security.authc.anonymous.roles", "anonymous");
+        clusterSettings.put("xpack.security.transport.ssl.enabled", "true");
+        clusterSettings.put("xpack.security.transport.ssl.certificate", "transport.crt");
+        clusterSettings.put("xpack.security.transport.ssl.key", "transport.key");
+        clusterSettings.put("xpack.security.transport.ssl.key_passphrase", "transport-password");
+        clusterSettings.put("xpack.security.transport.ssl.certificate_authorities", "ca.crt");
+        cluster.restart(false);
+        closeClients();
+    }
+
+    @Order(3)
+    public void testSecurityEnabledSetup() throws Exception {
+        checkSecuritySetup(true);
+    }
+
+    public void checkSecuritySetup(boolean securityEnabled) throws Exception {
         logger.info("Security status: {}", securityEnabled);
         logger.info("Cluster:\n{}", getClusterInfo());
         logger.info("Indices:\n{}", getIndices());
diff --git a/x-pack/plugin/security/qa/jwt-realm/build.gradle b/x-pack/plugin/security/qa/jwt-realm/build.gradle
index 9f4fd09f7da6..d561f0c8ceee 100644
--- a/x-pack/plugin/security/qa/jwt-realm/build.gradle
+++ b/x-pack/plugin/security/qa/jwt-realm/build.gradle
@@ -1,7 +1,7 @@
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 dependencies {
   javaRestTestImplementation project(path: xpackModule('core'))
@@ -10,7 +10,13 @@ dependencies {
   javaRestTestImplementation project(":client:rest")
 }
 
-boolean explicitIdTokenType = (new Random(Long.parseUnsignedLong(BuildParams.testSeed.tokenize(':').get(0), 16))).nextBoolean()
+def random = BuildParams.random
+boolean explicitIdTokenType = random.nextBoolean()
+def serviceSubject = 'service_' + random.nextInt(1, 9) + '@app' + random.nextInt(1, 9) + '.example.com'
+
+tasks.named("javaRestTest").configure {
+  systemProperty 'jwt2.service_subject', serviceSubject
+}
 
 testClusters.matching { it.name == 'javaRestTest' }.configureEach {
   testDistribution = 'DEFAULT'
@@ -40,8 +46,6 @@ testClusters.matching { it.name == 'javaRestTest' }.configureEach {
   setting 'xpack.security.http.ssl.certificate_authorities', 'ca.crt'
   setting 'xpack.security.http.ssl.client_authentication', 'optional'
 
-  setting 'xpack.security.authc.jwt.principal_claims', 'sub,oid,client_id,azp,appid,email'
-
   setting 'xpack.security.authc.realms.file.admin_file.order', '0'
 
   // These realm settings are generated by JwtRealmGenerateTests
@@ -56,6 +60,8 @@ testClusters.matching { it.name == 'javaRestTest' }.configureEach {
   setting 'xpack.security.authc.realms.jwt.jwt1.claims.dn', 'dn'
   setting 'xpack.security.authc.realms.jwt.jwt1.claims.name', 'name'
   setting 'xpack.security.authc.realms.jwt.jwt1.claims.mail', 'mail'
+  setting 'xpack.security.authc.realms.jwt.jwt1.required_claims.token_use', 'id'
+  setting 'xpack.security.authc.realms.jwt.jwt1.required_claims.version', '2.0'
   setting 'xpack.security.authc.realms.jwt.jwt1.client_authentication.type', 'NONE'
   // Use default value (RS256) for signature algorithm
   setting 'xpack.security.authc.realms.jwt.jwt1.pkc_jwkset_path', 'rsa.jwkset'
@@ -64,20 +70,27 @@ testClusters.matching { it.name == 'javaRestTest' }.configureEach {
   setting 'xpack.security.authc.realms.native.lookup_native.order', '2'
 
   setting 'xpack.security.authc.realms.jwt.jwt2.order', '3'
-  if (explicitIdTokenType) {
-    setting 'xpack.security.authc.realms.jwt.jwt2.token_type', 'id_token'
-  }
+  setting 'xpack.security.authc.realms.jwt.jwt2.token_type', 'access_token'
+  setting 'xpack.security.authc.realms.jwt.jwt2.fallback_claims.sub', 'email'
+  setting 'xpack.security.authc.realms.jwt.jwt2.fallback_claims.aud', 'scope'
   setting 'xpack.security.authc.realms.jwt.jwt2.allowed_issuer', 'my-issuer'
+  setting 'xpack.security.authc.realms.jwt.jwt2.allowed_subjects', serviceSubject
   setting 'xpack.security.authc.realms.jwt.jwt2.allowed_audiences', 'es01,es02,es03'
   setting 'xpack.security.authc.realms.jwt.jwt2.allowed_signature_algorithms', 'HS256,HS384'
-  setting 'xpack.security.authc.realms.jwt.jwt2.claims.principal', 'email'
+  // Both email or sub works because of fallback
+  if (random.nextBoolean()) {
+    setting 'xpack.security.authc.realms.jwt.jwt2.claims.principal', 'email'
+  } else {
+    setting 'xpack.security.authc.realms.jwt.jwt2.claims.principal', 'sub'
+  }
   setting 'xpack.security.authc.realms.jwt.jwt2.claim_patterns.principal', '^(.*)@[^.]*[.]example[.]com$'
+  setting 'xpack.security.authc.realms.jwt.jwt2.required_claims.token_use', 'access'
   setting 'xpack.security.authc.realms.jwt.jwt2.authorization_realms', 'lookup_native'
   setting 'xpack.security.authc.realms.jwt.jwt2.client_authentication.type', 'shared_secret'
   keystore 'xpack.security.authc.realms.jwt.jwt2.client_authentication.shared_secret', 'test-secret'
   keystore 'xpack.security.authc.realms.jwt.jwt2.hmac_key', 'test-HMAC/secret passphrase-value'
 
-  // Place PKI realm after JWT realm to verify realm chain fall-throug
+  // Place PKI realm after JWT realm to verify realm chain fall-through
   setting 'xpack.security.authc.realms.pki.pki_realm.order', '4'
 
   setting 'xpack.security.authc.realms.jwt.jwt3.order', '5'
diff --git a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRestIT.java b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRestIT.java
index eb3567d45243..79da6b75d90e 100644
--- a/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRestIT.java
+++ b/x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRestIT.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.PathUtils;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.TestSecurityClient;
 import org.elasticsearch.test.rest.ESRestTestCase;
@@ -126,7 +127,7 @@ public void testAuthenticateWithRsaSignedJWTAndRoleMappingByPrincipal() throws E
         final String dn = randomDn();
         final String name = randomName();
         final String mail = randomMail();
-        final String rules = formatted("""
+        final String rules = Strings.format("""
             { "all": [
                 { "field": { "realm.name": "jwt1" } },
                 { "field": { "username": "%s" } }
@@ -142,7 +143,7 @@ public void testAuthenticateWithRsaSignedJWTAndRoleMappingByDn() throws Exceptio
         final String name = randomName();
         final String mail = randomMail();
 
-        final String rules = formatted("""
+        final String rules = Strings.format("""
             { "all": [
                 { "field": { "realm.name": "jwt1" } },
                 { "field": { "dn": "%s" } }
@@ -160,7 +161,7 @@ public void testAuthenticateWithRsaSignedJWTAndRoleMappingByGroups() throws Exce
         final List<String> groups = randomList(1, 12, () -> randomAlphaOfLengthBetween(4, 12));
         final String mappedGroup = randomFrom(groups);
 
-        final String rules = formatted("""
+        final String rules = Strings.format("""
             { "all": [
                 { "field": { "realm.name": "jwt1" } },
                 { "field": { "groups": "%s" } }
@@ -175,7 +176,7 @@ public void testAuthenticateWithRsaSignedJWTAndRoleMappingByMetadata() throws Ex
         final String dn = randomDn();
         final String name = randomName();
         final String mail = randomMail();
-        final String rules = formatted("""
+        final String rules = Strings.format("""
             { "all": [
                 { "field": { "realm.name": "jwt1" } },
                 { "field": { "metadata.jwt_claim_sub": "%s" } }
@@ -281,10 +282,11 @@ public void testFailureOnNonMatchingRsaSignature() throws Exception {
      * - uses a shared-secret for client authentication
      */
     public void testAuthenticateWithHmacSignedJWTAndDelegatedAuthorization() throws Exception {
-        final String principal = randomPrincipal();
+        final String principal = System.getProperty("jwt2.service_subject");
+        final String username = getUsernameFromPrincipal(principal);
         final List<String> roles = randomRoles();
         final String randomMetadata = randomAlphaOfLengthBetween(6, 18);
-        createUser(principal, roles, Map.of("test_key", randomMetadata));
+        createUser(username, roles, Map.of("test_key", randomMetadata));
 
         try {
             final SignedJWT jwt = buildAndSignJwtForRealm2(principal);
@@ -292,7 +294,7 @@ public void testAuthenticateWithHmacSignedJWTAndDelegatedAuthorization() throws
 
             final Map<String, Object> response = client.authenticate();
 
-            assertThat(response.get(User.Fields.USERNAME.getPreferredName()), is(principal));
+            assertThat(response.get(User.Fields.USERNAME.getPreferredName()), is(username));
             assertThat(assertMap(response, User.Fields.AUTHENTICATION_REALM), hasEntry(User.Fields.REALM_NAME.getPreferredName(), "jwt2"));
             assertThat(assertList(response, User.Fields.ROLES), Matchers.containsInAnyOrder(roles.toArray(String[]::new)));
             assertThat(assertMap(response, User.Fields.METADATA), hasEntry("test_key", randomMetadata));
@@ -304,14 +306,15 @@ public void testAuthenticateWithHmacSignedJWTAndDelegatedAuthorization() throws
             );
             assertThat(exception.getResponse(), hasStatusCode(RestStatus.FORBIDDEN));
         } finally {
-            deleteUser(principal);
+            deleteUser(username);
         }
     }
 
     public void testFailureOnInvalidHMACSignature() throws Exception {
-        final String principal = randomPrincipal();
+        final String principal = System.getProperty("jwt2.service_subject");
+        final String username = getUsernameFromPrincipal(principal);
         final List<String> roles = randomRoles();
-        createUser(principal, roles, Map.of());
+        createUser(username, roles, Map.of());
 
         try {
             final JWTClaimsSet claimsSet = buildJwtForRealm2(principal, Instant.now());
@@ -320,7 +323,7 @@ public void testFailureOnInvalidHMACSignature() throws Exception {
                 // This is the correct HMAC passphrase (from build.gradle)
                 final SignedJWT jwt = signHmacJwt(claimsSet, "test-HMAC/secret passphrase-value");
                 final TestSecurityClient client = getSecurityClient(jwt, VALID_SHARED_SECRET);
-                assertThat(client.authenticate(), hasEntry(User.Fields.USERNAME.getPreferredName(), principal));
+                assertThat(client.authenticate(), hasEntry(User.Fields.USERNAME.getPreferredName(), username));
             }
             {
                 // This is not the correct HMAC passphrase
@@ -331,13 +334,36 @@ public void testFailureOnInvalidHMACSignature() throws Exception {
                 assertThat(exception.getResponse(), hasStatusCode(RestStatus.UNAUTHORIZED));
             }
         } finally {
-            deleteUser(principal);
+            deleteUser(username);
         }
 
     }
 
+    public void testFailureOnRequiredClaims() throws JOSEException, IOException {
+        final String principal = System.getProperty("jwt2.service_subject");
+        final String username = getUsernameFromPrincipal(principal);
+        final List<String> roles = randomRoles();
+        createUser(username, roles, Map.of());
+        try {
+            final String audience = "es0" + randomIntBetween(1, 3);
+            final Map<String, Object> data = new HashMap<>(Map.of("iss", "my-issuer", "aud", audience, "email", principal));
+            // The required claim is either missing or mismatching
+            if (randomBoolean()) {
+                data.put("token_use", randomValueOtherThan("access", () -> randomAlphaOfLengthBetween(3, 10)));
+            }
+            final JWTClaimsSet claimsSet = buildJwt(data, Instant.now(), false, false);
+            final SignedJWT jwt = signHmacJwt(claimsSet, "test-HMAC/secret passphrase-value");
+            final TestSecurityClient client = getSecurityClient(jwt, VALID_SHARED_SECRET);
+            final ResponseException exception = expectThrows(ResponseException.class, client::authenticate);
+            assertThat(exception.getResponse(), hasStatusCode(RestStatus.UNAUTHORIZED));
+        } finally {
+            deleteUser(username);
+        }
+    }
+
     public void testAuthenticationFailureIfDelegatedAuthorizationFails() throws Exception {
-        final String principal = randomPrincipal();
+        final String principal = System.getProperty("jwt2.service_subject");
+        final String username = getUsernameFromPrincipal(principal);
         final SignedJWT jwt = buildAndSignJwtForRealm2(principal);
         final TestSecurityClient client = getSecurityClient(jwt, VALID_SHARED_SECRET);
 
@@ -345,19 +371,20 @@ public void testAuthenticationFailureIfDelegatedAuthorizationFails() throws Exce
         final ResponseException exception = expectThrows(ResponseException.class, client::authenticate);
         assertThat(exception.getResponse(), hasStatusCode(RestStatus.UNAUTHORIZED));
 
-        createUser(principal, List.of(), Map.of());
+        createUser(username, List.of(), Map.of());
         try {
             // Now it works
-            assertThat(client.authenticate(), hasEntry(User.Fields.USERNAME.getPreferredName(), principal));
+            assertThat(client.authenticate(), hasEntry(User.Fields.USERNAME.getPreferredName(), username));
         } finally {
-            deleteUser(principal);
+            deleteUser(username);
         }
     }
 
     public void testFailureOnInvalidClientAuthentication() throws Exception {
-        final String principal = randomPrincipal();
+        final String principal = System.getProperty("jwt2.service_subject");
+        final String username = getUsernameFromPrincipal(principal);
         final List<String> roles = randomRoles();
-        createUser(principal, roles, Map.of());
+        createUser(username, roles, Map.of());
 
         try {
             final SignedJWT jwt = buildAndSignJwtForRealm2(principal);
@@ -368,7 +395,7 @@ public void testFailureOnInvalidClientAuthentication() throws Exception {
             assertThat(exception.getResponse(), hasStatusCode(RestStatus.UNAUTHORIZED));
 
         } finally {
-            deleteUser(principal);
+            deleteUser(username);
         }
     }
 
@@ -482,7 +509,9 @@ private SignedJWT buildAndSignJwtForRealm1(
                 Map.entry("dn", dn),
                 Map.entry("name", name),
                 Map.entry("mail", mail),
-                Map.entry("roles", groups) // Realm realm config has `claim.groups: "roles"`
+                Map.entry("roles", groups), // Realm realm config has `claim.groups: "roles"`
+                Map.entry("token_use", "id"),
+                Map.entry("version", "2.0")
             ),
             issueTime
         );
@@ -499,14 +528,18 @@ private SignedJWT buildAndSignJwtForRealm2(String principal, Instant issueTime)
     }
 
     private JWTClaimsSet buildJwtForRealm2(String principal, Instant issueTime) {
-        final String emailAddress = principal + "@" + randomAlphaOfLengthBetween(3, 6) + ".example.com";
         // The "jwt2" realm, supports 3 audiences (es01/02/03)
         final String audience = "es0" + randomIntBetween(1, 3);
-        final JWTClaimsSet claimsSet = buildJwt(
-            Map.ofEntries(Map.entry("iss", "my-issuer"), Map.entry("aud", audience), Map.entry("email", emailAddress)),
-            issueTime,
-            false
-        );
+        final Map<String, Object> data = new HashMap<>(Map.of("iss", "my-issuer", "email", principal, "token_use", "access"));
+        if (randomBoolean()) {
+            data.put("aud", audience);
+            // scope (fallback audience) is ignored since aud exists
+            data.put("scope", randomAlphaOfLength(20));
+        } else {
+            data.put("scope", audience);
+        }
+
+        final JWTClaimsSet claimsSet = buildJwt(data, issueTime, false, false);
         return claimsSet;
     }
 
@@ -566,16 +599,18 @@ private SignedJWT signHmacJwt(JWTClaimsSet claimsSet, String hmacPassphrase) thr
 
     // JWT construction
     private JWTClaimsSet buildJwt(Map<String, Object> claims, Instant issueTime) {
-        return buildJwt(claims, issueTime, true);
+        return buildJwt(claims, issueTime, true, true);
     }
 
-    private JWTClaimsSet buildJwt(Map<String, Object> claims, Instant issueTime, boolean includeSub) {
+    private JWTClaimsSet buildJwt(Map<String, Object> claims, Instant issueTime, boolean includeSub, boolean includeAud) {
         final JWTClaimsSet.Builder builder = new JWTClaimsSet.Builder();
         builder.issuer(randomAlphaOfLengthBetween(4, 24));
         if (includeSub) {
             builder.subject(randomAlphaOfLengthBetween(4, 24));
         }
-        builder.audience(randomList(1, 6, () -> randomAlphaOfLengthBetween(4, 12)));
+        if (includeAud) {
+            builder.audience(randomList(1, 6, () -> randomAlphaOfLengthBetween(4, 12)));
+        }
         if (randomBoolean()) {
             builder.jwtID(UUIDs.randomBase64UUID(random()));
         }
@@ -645,16 +680,38 @@ private void createUser(String principal, List<String> roles, Map<String, Object
         createUser(principal, new SecureString(randomAlphaOfLength(12).toCharArray()), roles, metadata);
     }
 
-    private void createUser(String username, SecureString password, List<String> roles, Map<String, Object> metadata) throws IOException {
+    private void createUser(String principal, SecureString password, List<String> roles, Map<String, Object> metadata) throws IOException {
+        final String username;
+        if (principal.contains("@")) {
+            username = principal.substring(0, principal.indexOf("@"));
+        } else {
+            username = principal;
+        }
         final String realName = randomAlphaOfLengthBetween(6, 18);
         final User user = new User(username, roles.toArray(String[]::new), realName, null, metadata, true);
         getAdminSecurityClient().putUser(user, password);
     }
 
-    private void deleteUser(String username) throws IOException {
+    private void deleteUser(String principal) throws IOException {
+        final String username;
+        if (principal.contains("@")) {
+            username = principal.substring(0, principal.indexOf("@"));
+        } else {
+            username = principal;
+        }
         getAdminSecurityClient().deleteUser(username);
     }
 
+    private String getUsernameFromPrincipal(String principal) {
+        final String username;
+        if (principal.contains("@")) {
+            username = principal.substring(0, principal.indexOf("@"));
+        } else {
+            username = principal;
+        }
+        return username;
+    }
+
     private String createRoleMapping(List<String> roles, String rules) throws IOException {
         Map<String, Object> mapping = new HashMap<>();
         mapping.put("enabled", true);
diff --git a/x-pack/plugin/security/qa/multi-cluster/build.gradle b/x-pack/plugin/security/qa/multi-cluster/build.gradle
new file mode 100644
index 000000000000..f55a11f37e26
--- /dev/null
+++ b/x-pack/plugin/security/qa/multi-cluster/build.gradle
@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.internal.info.BuildParams
+import org.elasticsearch.gradle.internal.test.RestIntegTestTask
+
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-resources'
+
+def fulfillingCluster = testClusters.register('fulfilling-cluster') {
+  requiresFeature 'es.untrusted_remote_cluster_feature_flag_registered', Version.fromString("8.5.0")
+  setting 'xpack.security.enabled', 'true'
+  setting 'xpack.license.self_generated.type', 'trial'
+  setting 'remote_cluster.enabled', 'true'
+  setting 'remote_cluster.port', '0'
+
+  extraConfigFile 'transport.key', file("src/test/resources/ssl/transport.key")
+  extraConfigFile 'transport.cert', file("src/test/resources/ssl/transport.crt")
+  extraConfigFile 'transport.ca', file("src/test/resources/ssl/transport-ca.crt")
+
+  // Transport SSL can be enabled or disabled. It is independent from the remote cluster server SSL
+  if (BuildParams.random.nextBoolean()) {
+    setting 'xpack.security.transport.ssl.enabled', 'true'
+  } else {
+    setting 'xpack.security.transport.ssl.enabled', 'false'
+  }
+  setting 'xpack.security.transport.ssl.key', 'transport.key'
+  setting 'xpack.security.transport.ssl.certificate', 'transport.cert'
+  setting 'xpack.security.transport.ssl.key_passphrase', 'transport-password'
+  setting 'xpack.security.transport.ssl.certificate_authorities', 'transport.ca'
+  setting 'xpack.security.transport.ssl.client_authentication', 'required'
+  setting 'xpack.security.transport.ssl.verification_mode', 'certificate'
+
+  // Server side SSL configuration for remote cluster
+  extraConfigFile 'remote-cluster.key', file("src/test/resources/ssl/remote_cluster.key")
+  extraConfigFile 'remote-cluster.cert', file("src/test/resources/ssl/remote_cluster.crt")
+  extraConfigFile 'remote-cluster.ca', file("src/test/resources/ssl/remote-cluster-ca.crt")
+
+  setting 'xpack.security.remote_cluster.ssl.enabled', 'true'
+  setting 'xpack.security.remote_cluster.ssl.key', 'remote-cluster.key'
+  setting 'xpack.security.remote_cluster.ssl.certificate', 'remote-cluster.cert'
+  keystore 'xpack.security.remote_cluster.ssl.secure_key_passphrase', 'remote-cluster-password'
+  // client auth defaults to none for remote_cluster
+  if (BuildParams.random.nextBoolean()) {
+    setting 'xpack.security.remote_cluster.ssl.client_authentication', 'none'
+  }
+
+  user username: "test_user", password: "x-pack-test-password"
+}
+
+def queryingCluster = testClusters.register('querying-cluster') {
+  requiresFeature 'es.untrusted_remote_cluster_feature_flag_registered', Version.fromString("8.5.0")
+  setting 'xpack.security.enabled', 'true'
+  setting 'xpack.license.self_generated.type', 'trial'
+  setting 'cluster.remote.connections_per_cluster', "1"
+
+  extraConfigFile 'transport.key', file("src/test/resources/ssl/transport.key")
+  extraConfigFile 'transport.cert', file("src/test/resources/ssl/transport.crt")
+  extraConfigFile 'transport.ca', file("src/test/resources/ssl/transport-ca.crt")
+
+  setting 'xpack.security.transport.ssl.enabled', 'true'
+  setting 'xpack.security.transport.ssl.key', 'transport.key'
+  setting 'xpack.security.transport.ssl.certificate', 'transport.cert'
+  setting 'xpack.security.transport.ssl.key_passphrase', 'transport-password'
+  setting 'xpack.security.transport.ssl.certificate_authorities', 'transport.ca'
+  setting 'xpack.security.transport.ssl.client_authentication', 'required'
+  setting 'xpack.security.transport.ssl.verification_mode', 'certificate'
+
+  // Client (QC) side ssl configuration for remote cluster
+  extraConfigFile 'remote-cluster.ca', file("src/test/resources/ssl/remote-cluster-ca.crt")
+  setting 'xpack.security.remote_cluster.ssl.certificate_authorities', 'remote-cluster.ca'
+
+  setting 'cluster.remote.my_remote_cluster.mode', 'proxy'
+  setting 'cluster.remote.my_remote_cluster.proxy_address', {
+    "\"${fulfillingCluster.get().getAllRemoteAccessPortURI().get(0)}\""
+  }
+  setting 'cluster.remote.my_remote_cluster.authorization', 'foobar'
+
+  user username: "test_user", password: "x-pack-test-password"
+}
+
+tasks.register('fulfilling-cluster', RestIntegTestTask) {
+  useCluster fulfillingCluster
+  systemProperty 'tests.rest.suite', 'fulfilling_cluster'
+}
+
+tasks.register('querying-cluster', RestIntegTestTask) {
+  dependsOn 'fulfilling-cluster'
+  useCluster queryingCluster
+  useCluster fulfillingCluster
+  systemProperty 'tests.rest.suite', 'querying_cluster'
+}
+
+// runs the fulfilling-cluster cluster tests then the querying-cluster tests
+tasks.register("integTest") {
+  dependsOn 'querying-cluster'
+}
+
+tasks.named("check").configure { dependsOn("integTest") }
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecuritySmokeIT.java b/x-pack/plugin/security/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecuritySmokeIT.java
new file mode 100644
index 000000000000..d019d70bfa4c
--- /dev/null
+++ b/x-pack/plugin/security/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecuritySmokeIT.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.remotecluster;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.rest.ESRestTestCase;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * This test suite will be run twice: Once against the fulfilling cluster, then again against the querying cluster. The typical usage is to
+ * conditionalize on whether the test is running against the fulfilling or the querying cluster.
+ */
+public class RemoteClusterSecuritySmokeIT extends ESRestTestCase {
+    @Override
+    protected boolean preserveIndicesUponCompletion() {
+        return true;
+    }
+
+    @Override
+    protected boolean preserveDataStreamsUponCompletion() {
+        return true;
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("test_user", new SecureString("x-pack-test-password".toCharArray()));
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+
+    private boolean isFulfillingCluster() {
+        return "fulfilling_cluster".equals(System.getProperty("tests.rest.suite"));
+    }
+
+    /**
+     * This test really depends on the local build.gradle, which configures cross-cluster search using the `remote_cluster.*` settings.
+     */
+    public void testRemoteAccessPortFunctions() throws Exception {
+        if (isFulfillingCluster()) {
+            // Index some documents, so we can search them from the querying cluster
+            Request indexDocRequest = new Request("POST", "/test_idx/_doc");
+            indexDocRequest.setJsonEntity("{\"foo\": \"bar\"}");
+            Response response = client().performRequest(indexDocRequest);
+            assertOK(response);
+        } else {
+            // Check that we can search the fulfilling cluster from the querying cluster
+            Request searchRequest = new Request("GET", "/my_remote_cluster:test_idx/_search");
+
+            // Since authentication is not wired up on the FC side, getting a 401 response is sufficient
+            // for testing connection (port, SSL) is working
+            final ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(searchRequest));
+            assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(401));
+            assertThat(e.getMessage(), containsString("missing authentication credentials"));
+
+            // TODO: uncomment once authentication is wired up on the FC side
+            // Response response = client().performRequest(searchRequest);
+            // assertOK(response);
+            // ObjectPath responseObj = ObjectPath.createFromResponse(response);
+            // int totalHits = responseObj.evaluate("hits.total.value");
+            // assertThat(totalHits, equalTo(1));
+        }
+    }
+
+}
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/README.asciidoc b/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/README.asciidoc
new file mode 100644
index 000000000000..04c2ce9bc88d
--- /dev/null
+++ b/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/README.asciidoc
@@ -0,0 +1,55 @@
+= Keystore Details
+This document details the steps used to create the certificate and keystore files in this directory.
+
+== Instructions on generating certificates
+The certificates in this directory have been generated using elasticsearch-certutil (8.7.0 SNAPSHOT)
+
+[source,shell]
+-----------------------------------------------------------------------------------------------------------
+elasticsearch-certutil ca --pem --ca-dn 'CN=Elastic Auto Transport CA' --out=ca.zip --pass="transport-ca-password" --days=3500
+unzip ca.zip
+mv ca/ca.crt ./transport-ca.crt
+mv ca/ca.key ./transport-ca.key
+
+rm ca.zip
+rmdir ca
+-----------------------------------------------------------------------------------------------------------
+
+[source,shell]
+-----------------------------------------------------------------------------------------------------------
+elasticsearch-certutil cert --pem --name=transport --out=transport.zip --pass="transport-password" --days=3500 \
+    --ca-cert=transport-ca.crt --ca-key=transport-ca.key --ca-pass="transport-ca-password" \
+    --dns=localhost --dns=localhost.localdomain --dns=localhost4 --dns=localhost4.localdomain4 --dns=localhost6 --dns=localhost6.localdomain6 \
+    --ip=127.0.0.1 --ip=0:0:0:0:0:0:0:1
+
+unzip transport.zip
+mv transport/transport.* ./
+
+rm transport.zip
+rmdir transport
+-----------------------------------------------------------------------------------------------------------
+
+[source,shell]
+-----------------------------------------------------------------------------------------------------------
+elasticsearch-certutil ca --pem --ca-dn 'CN=Elastic Auto RemoteCluster CA' --out=ca.zip --pass="remote-cluster-ca-password" --days=3500
+unzip ca.zip
+mv ca/ca.crt ./remote-cluster-ca.crt
+mv ca/ca.key ./remote-cluster-ca.key
+
+rm ca.zip
+rmdir ca
+-----------------------------------------------------------------------------------------------------------
+
+[source,shell]
+-----------------------------------------------------------------------------------------------------------
+elasticsearch-certutil cert --pem --name=remote_cluster --out=remote_cluster.zip --pass="remote-cluster-password" --days=3500 \
+    --ca-cert=remote-cluster-ca.crt --ca-key=remote-cluster-ca.key --ca-pass="remote-cluster-ca-password" \
+    --dns=localhost --dns=localhost.localdomain --dns=localhost4 --dns=localhost4.localdomain4 --dns=localhost6 --dns=localhost6.localdomain6 \
+    --ip=127.0.0.1 --ip=0:0:0:0:0:0:0:1
+
+unzip remote_cluster.zip
+mv remote_cluster/remote_cluster.* ./
+
+rm remote_cluster.zip
+rmdir remote_cluster
+-----------------------------------------------------------------------------------------------------------
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/remote-cluster-ca.crt b/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/remote-cluster-ca.crt
new file mode 100644
index 000000000000..b0683c9c8cf8
--- /dev/null
+++ b/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/remote-cluster-ca.crt
@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDMjCCAhqgAwIBAgIVAM6zI4OMux23oGuXq3gxYWB40sUJMA0GCSqGSIb3DQEB
+CwUAMCgxJjAkBgNVBAMTHUVsYXN0aWMgQXV0byBSZW1vdGVDbHVzdGVyIENBMB4X
+DTIzMDEyOTEyMDcxOFoXDTMyMDgyOTEyMDcxOFowKDEmMCQGA1UEAxMdRWxhc3Rp
+YyBBdXRvIFJlbW90ZUNsdXN0ZXIgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQC98BMOj4AAVx0IytNIg5r6xY1E3YOKYVpyXo0qLca8JIYLWDPiKpou
+IiroziGXtT8d/tw2KNgHlICqq2iKZ+6uWkYKZjnzIw1CQBwwdEJNm04QK4Ti2QI1
+d5RRHjKGUWwHNXZzFokDeDfiqhkWF32E3hKmV2zeCQpLmCULzIUq8G4T2P2KsWI8
+WVscWYjiOd1DcX6pK90OFlJ+nN6tptb1nWvG0cbrHnTqw+rPIvEfz9zQlP3kQjG+
+kECrdMqhTpJ5h5enj/Wr/fzqGqPfK5PHYZO5Y64/AXTZDNiYugEuGpwHZDMOo19v
+pIZYQ6CT2pHYpl0elFzEredB9s19878jAgMBAAGjUzBRMB0GA1UdDgQWBBQenD5A
+nV/NdQ+nFuQuninbD1ArjzAfBgNVHSMEGDAWgBQenD5AnV/NdQ+nFuQuninbD1Ar
+jzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAw5ZbObu/D6n8N
+3Q21ohkPmppr1gtHbFqDpXHcVWMRYRO4ktk6mCXtPVnaZi4sONolwlhV5SXJwL0p
+1g3yFbXrTlEZ5xs9QdIREUUDW12QKO1WuEYTTNgIu1my6UkQ9x7rlogX4gQYJyon
+KVNrpNWWanNY5VjdUxb7DZ5dDDcRgKJQ/63xvtS1rQfMJSdXG1W3YCZjFDmMTN8K
+af8r5N9X8wEZJBq2hG6n45XyvnbgErgQ2FMdIWeeHDEeM9Nw1u0VzQ/Y/U1j0mch
+UltZAcAn9NC8IXGdHS58ZiM6ncWzXlM0qlw1hodv5GLY6pqg2jrW5uTGj43PPx3S
+izlLT1CM
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/remote-cluster-ca.key b/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/remote-cluster-ca.key
new file mode 100644
index 000000000000..344c54e135aa
--- /dev/null
+++ b/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/remote-cluster-ca.key
@@ -0,0 +1,30 @@
+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: AES-128-CBC,DD3FAB8EEE02EB0A2B4F399D5B0569BD
+
+q4gCQQKkPiXLN6CtzmNK3GNPgPj3qwaC+cKbcJClwI2B7vN6OvwilwOcwWHvcxlb
+cH4ivLkyu4zoc5pYBGHDXwDiq8KXvh+9aSjDcvdg/4KU5fLvlRxPBBKCXyasL7vd
+xqu9hlcs5KzXnhizn+pN79FWEaTUCLMZmxKkQ/Bzlh1UPFRvFfnmBoSMw6pzIRVQ
+KKVwXnKIBN1sgVm9nsIMVSsKRoctIkOueWlgFxOpVR0/YAPmjueqIY2pL+Nkt0Ue
+xxJMWj7RABbf8DecJHVRcIFMMJrPwCqkBAhH6t6DjbnVhW0edu1GEmvQN8fp2PY2
+vpuUF9jdpZxqYbxlEvwtqUuiLNt5NDyXcCwmP+k6gc9g4pp+rZ9ZqzS95HADTkUq
+YZnBmofBCY7kFVB58VyP1iOiT+wsVPoie4cSZnLZzNYro/v22jjEoIYGnCDl24qy
+pBd4radE7qnLZFSk2z5yBh1b/gw1+S/IdkMivb5rWOUQXmE28C4VvUU/Dc+JSX+h
+NfJZK7YwKr4n5414ecebQ91FxIA1zbHWk2d4k9kQn1tbRINQP76/TB42BMLuZBYg
+xMjx5GAnPASRGmkZObZemapeRNkSfBQuw/Wmzs8hgntzGZvkdyIsH4SIMAoCrieO
+OZv9BR33c/aLSPJE0LWnJLSmtaF2KQ2lya2c/sc14c2h+UwU4jo4pcdlSKqIFgEl
+2l+XKbYFjJiEQBa+2j9eUXERGXHuwCZkYuP50MiXpN7PUO13NOPNUnOWMwtEm9TX
+ddIi2x1De2FZ8FuCgy8cdrX2ybiasce1OC9gxr83Zz8+2D5q2d4eFfmGj2PIolzN
+HnC+SPbShUfQiGLs8PSlWo2y2XKlA/S53o5ZBO4NNQlGNvjwIH1sBpEogPWVkIgH
+tW9i9LWaNturH0EzYG4T53ItMQtvXRLFTASwYZfcNgi7r5TGUfpa7oVpdK2bmz+T
+XLKwNU7HeKD9W39NQt8yS8Y2iEKFCT/pUyMxRNki6toPXt/7nX+Wx6/SB9938NaT
+0pD3lC/zIXkDx8/tSg+K0C/+d4Tzjflwu/7rrDsE8cE+MAWN/03uUMjPmgMltfvO
+n+fczxmjOYqo53K9RnMYy54JSMb6tbgxTwl0Zr/Dr+8Qo9u37+3zAlCZWqJhMdKs
+Wbokpi4qr5TNEdkgmcA5pBOdU79Gii2kyuoKtZCvuGhWMFTliiUmt0y08HiKfUay
+Uj6X+N59rufUseLSSDgphMkiPGzwc/octyTdxsQIhu+vDh63TARWdDiMau5KGuqh
+MwnJfyphJ/jOkX1U1fSJNesRTOpiQD0HwkoI/RV/CvBc0ehheXneFH84pXCY/qen
+f3Y1aO+gaC9oonyh6ZQylj6pnC1bXE6V88GagpIM7moODdHJedOFT8AKUJUUX0vq
+7rcgAo4lScllamoHetbgIt5OIfGqdDtO0K6sKpTDXU1Wa6tIYPHkbigrLPqgbIf7
+ica+Wvw1x4WTwkuiOAps/KvHFQyL3X08HYf4r69/yx7yVXoidd+9U8vxRyJzMkI0
+DJRDUcLvq4id08a6nPMo3cfj7RoDOQtTw7uZTNvCBWKCy6vGlRW0UXb7G8IZTba6
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/remote_cluster.crt b/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/remote_cluster.crt
new file mode 100644
index 000000000000..604ac9a1c29a
--- /dev/null
+++ b/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/remote_cluster.crt
@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDsTCCApmgAwIBAgIVAIjtnAa2buuAJoEm0ngGn6XfziRVMA0GCSqGSIb3DQEB
+CwUAMCgxJjAkBgNVBAMTHUVsYXN0aWMgQXV0byBSZW1vdGVDbHVzdGVyIENBMB4X
+DTIzMDEyOTEyMDgzN1oXDTMyMDgyOTEyMDgzN1owGTEXMBUGA1UEAwwOcmVtb3Rl
+X2NsdXN0ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCus8b7ZOw7
+2wTGq7DkaHpHdSs77t4hGig93xyR8+/ega3zpn7bzBU9MD8L0NqLCEmAIAzc7vzj
+3UG8KJiBFA+1a9O3Xpk/iYFyGxaBXaC/dbBeZC8nNuaXLlRby6lWlJ0uUQ4uhp1T
+4zzSNFO7cb8OjzkHnq8saPyuh51I1X7S/ND1CEXP/DV2MUmfFOYCxwN2SnZVKSuR
+hZrq2sYkbAiHq1XVvRwxQJ/BQDh+hfmd9Fqx9Q9uzA0Ck3/wfixIn2bci3Hcl90G
+sgRbtnJc/oi//bAkxi5sZdJ6Blq6EJYPtLQpX+FgmkeGPWsJKInrV/j3xTunr0Uw
+xGR0p2MrwbM7AgMBAAGjgeAwgd0wHQYDVR0OBBYEFARRmEVNytzppHf98wEcc0XB
+5zz3MB8GA1UdIwQYMBaAFB6cPkCdX811D6cW5C6eKdsPUCuPMIGPBgNVHREEgYcw
+gYSCCWxvY2FsaG9zdIIXbG9jYWxob3N0Ni5sb2NhbGRvbWFpbjaHBH8AAAGHEAAA
+AAAAAAAAAAAAAAAAAAGCCmxvY2FsaG9zdDSCCmxvY2FsaG9zdDaCFWxvY2FsaG9z
+dC5sb2NhbGRvbWFpboIXbG9jYWxob3N0NC5sb2NhbGRvbWFpbjQwCQYDVR0TBAIw
+ADANBgkqhkiG9w0BAQsFAAOCAQEARQjnYcx+494l9G/r92IkBPVtuvPBF7+rjXZA
+jVyagFXm4h+oNyn+9VL629UO/1sNGlWGL03B3CC4uEXKElkdJKFUL9Rim6UV9ofj
+ytNu43i/vp3j6VDBznxBZ4Jc/5nLmCb1prekZ78OG3rf2iub8L8HSUkq6ypC39c7
+gDNwAdYwNl8DbvHzHL/ukUzaLqBjD1AodlDi/p7/LQyhzCQCbWLWorzzQS65old4
+Ku6Fyu5yBAWvme9W5Gjk4n6FJWpTRCnAQDB5usLOoN07wKC5taP6NFmZQ5aI9Ht5
+BbwIk+vPkx1aWENdj7btmwbrBRfVryXLR4xY1KmAlJa4Crg/VA==
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/remote_cluster.key b/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/remote_cluster.key
new file mode 100644
index 000000000000..47eed51430e6
--- /dev/null
+++ b/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/remote_cluster.key
@@ -0,0 +1,30 @@
+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: AES-128-CBC,A85335FAC92C3F890E39FCF60C5FD295
+
+vJivcDLeMyb7q8G9iqBiH6KnW8NUFBf3TpRw301Vsv6FTHZi1H65eA8x8VVf/pPO
+IIfdW9QIzBXmPAx0Zj9Lj6BpwflDswrRkcRCbyrv6khXC9GZjTsGrKBdvKahca3L
+DlR1mmOjPSXlm+VIMOx6FsWhuLOYihui253CcJMxcWsuQLJu6juz6Gij7B2cuCBm
+NaW6EXj0c9ZxM1FD3TFxp4hawEmD4uKEckjkEfV3i+cGa+yAkjMIx1D90gyqkRl1
+Jrh8+VFkRm9OfGNKvvYkX9UAc9/pt37LweE/ECtrBLGYm+WbLD6QmdIXPWzJSpVS
+MFsljPHk35XDOYke9qEdQlmS1Q8SOAAEGyS9eUroAgPXbK7BhgSkR+v52Qjpo/ud
+YH1QrZv+YQzY2ew1wVShJsMgiUjxBn05+4BwXXWJ1xkvQ4cZuTcUrkKEXLn1kjuh
+Nv1aCtOI8awz7Xvhd8Dp7FCjBFwFKMW+J5pVbbTdI+JuWEkh/p9RxdtDrn8IHcGE
+9r84Y0LDCjz3nmXlhHVC57XVSZdofAK0V/AbtkLeqmTnvv7/I9kA9155Uvmx0OWR
+tHnCmKnMVR3d0zxbpIzSNjdsGqA4n5van5DyKVH3eF+ip6tMNs5BrPRBX3XO2tV1
+7/icQCYDoyFBEcevAalytSMoh7/ZFiGX3K8WGps6/aP3nfPKEHHqPrOPQJbrmsAX
+VtEJE8clO83P0HnoGQlopBYRJrr5sFs1KNipcSejyUdB7sbnrH5HqRIGWSc8+kPc
+9f2HS0vm4G94oOSTe14blWsGq7nagJYWRFnK8FBIwggA3tNN3YfUzAo3fIZOasX6
+IEs1WmoPfGK7VUsaN0uWEVi5S49QlvrG804/CWWfUKeXOHGcqG2UQ5QMHW72jqzV
+qk8+1g6UTYQybXWz7Xfwb1yKFMx2H6b+yUfTUsOZb88hjvNlhZvXkWsqd78Gb2XK
+IfEpWA/3f0YVP1TpsbBCnCpVTnncx0yo/jPIodUuiPgKOiaI+/X/7Jfv4ylXVxsS
+aN+AnNPZKgO7SvvEdKpoDvJ1tzsa4lY3z3h1KU8i4PFTS1Rh1gdwJTC1+nVnL8O/
+KjIqT1FD5LU1XX2UXJviQkXhSVMPM3n1luC0vIdwp/v04yY+SzQL0Vb7CfoEZ5bp
+vbAkMlz0/Cq0eS3zgeJTt8zqMGyr7Z7vADqAlYjEYD9l7aMxdlKzN0UVJuApwuWA
+6VHLGY6WbAHVaqhd+1UOKQAlgdJ9C7knWTIZMtXdDsIEllFbVMnc3lYg1QcGZqcp
+ejRNYfzkK8HLLVFRHH15KyLqvhJc8LE6kjYZ7snwHRFfw61OQJ2d/HIrp6abmmrt
+N/NDFB3TyMtK6sS+tosSXeCErc6iUVVXcn/EtK7JXf9N0RlNCDJ1GldfS+avXXw9
+84peAvhSt+0gOfKAfG2IDu1KnNvcz67qXH0a6OkvMXUevqW/VVeEq/3+wMkOCUeo
+3Zzo1+Est2fpCLsyOfLEQsKeRFDtYYsT6iFOYPslnRSzzPsL1I0TJUrCifDM/2eN
+gg4ydl3a8Ixme64smmywc3Ptf57lFPhN4q/83lTNl63QtiE87GZtMT68ru81/x6i
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/transport-ca.crt b/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/transport-ca.crt
new file mode 100644
index 000000000000..d5b219f6de4a
--- /dev/null
+++ b/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/transport-ca.crt
@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDKTCCAhGgAwIBAgIURWCLvSo2ZzpMSlw9yblrKk+FCIIwDQYJKoZIhvcNAQEL
+BQAwJDEiMCAGA1UEAxMZRWxhc3RpYyBBdXRvIFRyYW5zcG9ydCBDQTAeFw0yMzAx
+MjkxMjAzMzFaFw0zMjA4MjkxMjAzMzFaMCQxIjAgBgNVBAMTGUVsYXN0aWMgQXV0
+byBUcmFuc3BvcnQgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQD5
+zm51wptQfKbbFmdTcbpk+tGT8/MXMkNpKdtxUQgrPu6urCxTs6UkjsYhw10Tw8qa
+r7+txjDDZPZOIAFA9rFHotp+/dGvsEuBjT084KIS48rQiLhJzD4q+U+aborIuUwb
+iLjQOIflP5olOmFWXjZUoRUaYelzmZp4/laVzvRC2l+Gj5+422S1EQEidr8EZZyU
+9NHz2kGOz9ObVw68aBXZgWZwE4ctumvyAprBF8Y3w836edL3EjVcMHOjg2FXjMDh
+y7H+I/RcKDUw0AHHdvuRui8E4+kBVCiErnKqNQ1YOqDAXdMDnB4ZT052LflJkVEF
+u8y/Jjs9OHtxyHjMsY+PAgMBAAGjUzBRMB0GA1UdDgQWBBRTknQaF13CHJCK2nyw
+j6/ohmCkHDAfBgNVHSMEGDAWgBRTknQaF13CHJCK2nywj6/ohmCkHDAPBgNVHRMB
+Af8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCCsKTXLjs/g48iRe7hDzln6GXr
+wp3uuwGs0E5LjEgqQld+D1jALRPuCiStYL8wpKINy2VKCB2bbtEb3lGBog1ns5Pg
+ntVu5z54wq76f+kFe+NGt54TCEELkxccvkhTPm7yDTmYOOgUwYXJ/sqSjNABNmAu
+1wFl3B48iWrEd0QkSHjutEaRuODaoApjdpfaLUUMC6C6C0WYMtK2qE4DklwyIDSH
+/JZQsmLNRVrNNvmwphVhiXMwehnPPGpMa3x1qTZWBtgOoiWnwt0qNCPoTcImZr2h
+8rDuVFNOjGk9zwPz3mv7x1H2rqF57/1ItBPmARzR6TkIollRicODPo4WXqVC
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/transport-ca.key b/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/transport-ca.key
new file mode 100644
index 000000000000..29ca330770ad
--- /dev/null
+++ b/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/transport-ca.key
@@ -0,0 +1,30 @@
+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: AES-128-CBC,5985A282ABF260D0DEB6BCE172A06F98
+
+f1P2c6P2PHhdpNYS2LZjfFN4xCY8Wf0zFcShcyK8ssIfEm09xjXCVKQxmyGNb9bQ
+VO3widBXK/r/l3hqWo3bdouxY4itGloBhhdJGaBN4A+raMv+0kttMdGrgaWkk/ek
+/ljCzsj6lTfWPO7Hk8YOkDxadD6mVNSz/TxSSdED8rM9xsuYhQMcOhSa5sXwdf6w
+JImlSO0JKbK1iButZ0m4Os/xvr1D2f6XGjX5OqIl4I/iDt6INbciT8jlY6C7YTGo
+dyjr5aFVT5fc+6HHjqi2NdMIn9jOPjct0+hgGVC3180hJDJcfBmcDbMYPf5kMqIx
+xJkB0B9QTXG9sbc4w4tqG9JDwzM3H/nzMyNeFDaHdJO9DqnE7SNU3GmzrtkoU3hK
+1hgcrLDJhHNLlN6D/ApgjDythjxDLsuR9Cr6CLmcxzVxv/al0qY3/PLpJA+KQjr9
++4zAxSHQBmu/aBZTqlNth2A8NzVVTRz8F9iEMsllytAzT6C24yXAgoM55Ja8LNSZ
+r55PWqC2wLETvmaoBnbr3qcX9xbeeGnhYQbFOGJfTgILhhkkjjB77FTGlXJMtjOg
+wX5BlmoWYdkUsqHiCAtIv3I8bS2pJZQt6O8fPvn+9MSgpAyn5xaPlGB1tYn4SRFd
+4OAv6KJwQ65jVw7miPOWIYeBm3iBrcZxtzHrylVM1SdcbGz2+A7CQcUu0KMkdSBS
+ybNNz2UYQG66a6TOKGXI9lkCBQl9DHiS+hcv80q8IgEE8h8PNGhyddZ99iRlQMDK
+3IoO04bUdWZ9JQOgGa3PXhhYlyZ5W0lKrpsvfp8jl/adTixSJ5gw6d7tzXnCrMuj
+5YrArkpO+rJXEDXE4IpgQ1zNQUkBCdZu6adqIwVDXAlAMSEZw0/z/mwB1+5BogBK
+YPWDtKKN77K36WUmQddm9q1qfBhsxj33OwQlHoYISk0VGmFMs0u/jgIvH2qJwhE8
+wcP7MgZUvr0VT1ezM9x8pTJmmGhY7d1AbC+1qb4uNAk+lWp9Y5y/thz9qFDDb3PY
+ppKPHM/VG+Uq6rXAki/LJjfi0B3diWOAdrcznbmQGQ8LkH9uc9juBgQik7gRNrN8
+mYZ9E1itBMifGXuKuPtU2qYTz3RFcI5pPgMVnJQkO8lJqGF8D4qbeKk/u2wUj1/c
+i+7EkFAXFSjv9kjVFVsbyhZJia7TLE2QXyMq2OiP2Gwhcu/QxB7kYpdCivmY7q7e
+8jTWt08vn9xaT044015Qoc+3w0ixcI+WkEQ+72MDJOX0akfl4xpA9cAfHYs68qkJ
+fB7ynFOS2i6/BXvulo9RXb9tIXixLJqDL9pCztCLh/zegMfT0vUbDjjO7Z2edEXk
+oJgL5QpshqCxoFq3Rc18NHBRIUa7QpzhT72FtIlgavY7P3bxjtERB7Yku1gEs0/5
+n+8bP4bRGBGgmXqfNMFCrBd3K3ifil/LgHnOEq8qrQ+j/pvGorG0dI3OTcZ+dmYC
+w4E7WzSesZhE4xx9xU51tqGLgU7+BUdc9t85zahLo/tXjZSQHsgEqVC24//fqPX1
+kOeEPdqXlMhwq0UlPOsuhfK9fHdFuAFiqh62xH8aPKGi9nnWB/d/JtFsdSugyyzb
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/transport.crt b/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/transport.crt
new file mode 100644
index 000000000000..c76e45e70d1f
--- /dev/null
+++ b/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/transport.crt
@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDpzCCAo+gAwIBAgIUBJisi0qK38LiU3acHwcS9B6/SykwDQYJKoZIhvcNAQEL
+BQAwJDEiMCAGA1UEAxMZRWxhc3RpYyBBdXRvIFRyYW5zcG9ydCBDQTAeFw0yMzAx
+MjkxMjA1NTNaFw0zMjA4MjkxMjA1NTNaMBQxEjAQBgNVBAMTCXRyYW5zcG9ydDCC
+ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMKL79IsmRLzmDBhAsjny/Ho
+6hP4WekmTXyAvEpCo/h3JqGsCxT2C/qazff67jj57F/OvH2RyGQMHVcHhiZwrs7d
+Alo8297UENJfh3YWq3GLxSlP7L5U5+D4sGCHqRQhIVIw1D7FzeskU30trjVp68Di
+4SdDLqNrfKw6pmbaWwU0zEh4puM5YeP4exEhfc13v2Yt8JeJMvBV3aUqjwcif57z
+suJq/XxGJL9iUZdBVUjc0R2MX52UQllg6jkLoExqkBmZjVE+3LbyRmR5f5kMKY8M
+5hvxfKIA/Eycq7tstpM57H/cFG2N38Fbu3VOPo5lm8jsJ0RtMdVv9as1cBVbZ2EC
+AwEAAaOB4DCB3TAdBgNVHQ4EFgQUIn0c4Ani9eLCCm128j9PZNsnzb0wHwYDVR0j
+BBgwFoAUU5J0GhddwhyQitp8sI+v6IZgpBwwgY8GA1UdEQSBhzCBhIIJbG9jYWxo
+b3N0ghdsb2NhbGhvc3Q2LmxvY2FsZG9tYWluNocEfwAAAYcQAAAAAAAAAAAAAAAA
+AAAAAYIKbG9jYWxob3N0NIIKbG9jYWxob3N0NoIVbG9jYWxob3N0LmxvY2FsZG9t
+YWlughdsb2NhbGhvc3Q0LmxvY2FsZG9tYWluNDAJBgNVHRMEAjAAMA0GCSqGSIb3
+DQEBCwUAA4IBAQBzm07DRNr9Oyg8k6gZ6UvMRo9Elbp+A/B22l3YJHPQnj0dxA6P
+iEnQ+TdqwCefowMdakndWJ7nEUBjOg9OLnJnrhd3ibkix2dshUnBChhDimt30JVN
+HvC2IvBOZvUlH3MMWTt93FGMEqKjASA+eCfWxLbcvOI8f6Vx5VXnM1NQlmfYiqHD
+vGXCIHvPmNBq0UxXU550Pp3sxd9Y2OnnzEAZ1W02UJAICQcTS2fkUzYdEexjIjkY
+EX3n5354+tjGSIxPWth4hb19Fn0+1v3DPCIHxfzDUeqh+voOm8+PXWJCqxcOLFPQ
+POG10udYr1g2WusMiaNHHrfDZ/o03I3Cz7M2
+-----END CERTIFICATE-----
diff --git a/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/transport.key b/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/transport.key
new file mode 100644
index 000000000000..4c85df651b29
--- /dev/null
+++ b/x-pack/plugin/security/qa/multi-cluster/src/test/resources/ssl/transport.key
@@ -0,0 +1,30 @@
+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: AES-128-CBC,FF3713FD7246D42335C7FFF4FCACB10C
+
+KTez5b5wnuCGPIs/pmGyIs58B9PiBsm6SgVNT7UoIy+FdNstN+VZY/KU9qLestRa
+CuA+pJrTSqO26XxqpJb4yXn6S+ZhdkVcNl9HqT6MgqUfXPHLLufvwV55ktDHCKxV
+Taj09/RXYYJ1CfvwMem8jjBayI2/OPHUM77dkBmqlGp+eC6NwDxkfKxX/vxunrL1
+Skn+WEgbWUm0jM4JNtxpc5KwGzpQdnmggTEbN3pwF6yE8K9iPQ06Dme6GNTkrBRm
+HQUcMm5ZWJBCFfYz91CmrrLaCu0Q8kKNoE7wieUhy8r5JJjtCVMebf0Yu2epbJOi
+oOCkDqgFbb+g3yaelGAFloKPAoZf0liBpe4vFWhFnfcyY4k0eVUxuaXhCnjtKkll
+DKTLC36vgFx1qQc5TaxWtzhsk/ccNYoDcrQySJ9isuuv1J9Nc2ltQ2EgdPekvl21
+tIas4bzbi3o8KQxMIPWJ5CpgXm72bVJ1GJe7C70pGGLOsSk9nTe0sN9o7B6+5KiD
+OrwoiKY8bvr27KFc0oGBXWl0XFEuXn1uxnWc2KIyLSQDvqo7Jdur0gxgtIZAK4J7
+9DwJFwK0Db174JebSMtVLo17VDDV2DbVkg0PKiyP7bgAUCBr75AvqJz7Yok+KIxV
+hDoBgD5RAfmyKJ0DwJJmBM7qgqF5yNNOrceyxRTIGQUDXhbQDPIKGaLz1PUIY1qJ
+bycTkFgxOOSHVPlAhZari4+XF8wth37sPxJPic2MTQtKjt02WFg5Mq59VggkcFTp
+Y0eelUcV6dWWHNbtDR/HKI9jrBTjRunJwJiZ3Bw+nnvkVKp19Lfkp8iCgHuNMbp/
+4e9AQjBXy+ududxIBzvkEWtlWeyX/AdzHwFU9FdYGOlYKzK4kmpscG9BfIMp83zP
+wNg04iaZdict3cjKwUZKhUyXtb5v8gng60zAFOpGTBFRHr8qYWMPzxIhkxttz72M
+DNv1kZJMRfphq5zmNL28S2VYrIE600s7daH50iebfx48pUvTrjjiGPL83Y2RKa1w
+ctBYfiF09NFSYXa+rhRgXhZwDrQJ1g2+KYSSAjhCftmZA1eycMmuPppv3sAI+8DX
+NEQL7f+jxy+asQwPA7ApujvtWkO4kUBxM8wolu3RA4tZejwB1wcVVqnCbTbwE9qn
+DeLztHx7EuxbS2+eCkfhxYJlxJsqRM9DBqBmLranq5w/44SknkdDfZIcG6Z/M7Yv
+Vd9nE2+7vo8QrTA/Iy83PDZBsHUxFoYBKFaHbEoDdyypoxsLURqbdBqUp/JC/Lmf
+wc5uZcbgtcWqMriazZSsjSPm+PxnYQZWZres7bnppmCDTjZ/mP1Gy/f99JiIa/Pw
+TnotriAFM5No5vo8pIZlNRR8+HzTrAevdMypMZdbv8TsfP93fd3TJvpwR/cvw9Nc
+GCQQtz1OeuoJCkuZev00Q9FgsmZofKFCgw03nC+y+QHF5mD/wvVkL83H4YOfmQRe
+xdwzRvQq8P7Bl9my4QqbTfCIW1ftsbu+4M0aRv/iv21FdQhMVAICfRwC3aRgf9rI
+qoeDcHZSSR/ElXFni/Bvr+4Fd+E/V8pPmM/l2KRWN+GHFL+qLjJi3GFCS4qMhiHh
+-----END RSA PRIVATE KEY-----
diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/build.gradle b/x-pack/plugin/security/qa/operator-privileges-tests/build.gradle
index ab3a80956d52..c42b8583e648 100644
--- a/x-pack/plugin/security/qa/operator-privileges-tests/build.gradle
+++ b/x-pack/plugin/security/qa/operator-privileges-tests/build.gradle
@@ -1,7 +1,7 @@
 import org.elasticsearch.gradle.Version
 
 apply plugin: 'elasticsearch.base-internal-es-plugin'
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 esplugin {
   name 'operator-privileges-test'
diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java
index f92f7f89061a..c0db57f84b96 100644
--- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java
+++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java
@@ -88,6 +88,7 @@ public class Constants {
         "cluster:admin/transform/reset",
         "cluster:admin/transform/start",
         "cluster:admin/transform/stop",
+        "cluster:admin/transform/schedule_now",
         "cluster:admin/transform/update",
         "cluster:admin/transform/upgrade",
         "cluster:admin/transform/validate",
@@ -347,6 +348,7 @@ public class Constants {
         "cluster:monitor/xpack/usage/eql",
         "cluster:monitor/xpack/usage/frozen_indices",
         "cluster:monitor/xpack/usage/graph",
+        "cluster:monitor/xpack/usage/health_api",
         "cluster:monitor/xpack/usage/ilm",
         "cluster:monitor/xpack/usage/logstash",
         "cluster:monitor/xpack/usage/ml",
@@ -437,12 +439,12 @@ public class Constants {
         "indices:data/read/mtv",
         "indices:data/read/mtv[shard]",
         "indices:data/read/open_point_in_time",
+        "indices:data/read/profiling",
         "indices:data/read/rank_eval",
         "indices:data/read/scroll",
         "indices:data/read/scroll/clear",
         "indices:data/read/search",
         "indices:data/read/search/template",
-        "indices:data/read/semantic_search",
         "indices:data/read/shard_multi_search",
         "indices:data/read/sql",
         "indices:data/read/sql/close_cursor",
@@ -488,6 +490,7 @@ public class Constants {
         "internal:cluster/master_history/get",
         "internal:cluster/coordination_diagnostics/info",
         "internal:cluster/formation/info",
-        "internal:gateway/local/started_shards"
+        "internal:gateway/local/started_shards",
+        "internal:admin/indices/prevalidate_shard_path"
     );
 }
diff --git a/x-pack/plugin/security/qa/profile/build.gradle b/x-pack/plugin/security/qa/profile/build.gradle
index 14e13abb207a..eae33a5ba587 100644
--- a/x-pack/plugin/security/qa/profile/build.gradle
+++ b/x-pack/plugin/security/qa/profile/build.gradle
@@ -1,7 +1,7 @@
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 dependencies {
   javaRestTestImplementation project(':x-pack:plugin:core')
@@ -9,7 +9,7 @@ dependencies {
   javaRestTestImplementation project(':x-pack:plugin:security')
 }
 
-boolean literalUsername = (new Random(Long.parseUnsignedLong(BuildParams.testSeed.tokenize(':').get(0), 16))).nextBoolean()
+boolean literalUsername = BuildParams.random.nextBoolean()
 
 testClusters.matching { it.name == 'javaRestTest' }.configureEach {
   testDistribution = 'DEFAULT'
diff --git a/x-pack/plugin/security/qa/profile/src/javaRestTest/java/org/elasticsearch/xpack/security/profile/ProfileIT.java b/x-pack/plugin/security/qa/profile/src/javaRestTest/java/org/elasticsearch/xpack/security/profile/ProfileIT.java
index 01437d8e5a0e..fc25bda6d89b 100644
--- a/x-pack/plugin/security/qa/profile/src/javaRestTest/java/org/elasticsearch/xpack/security/profile/ProfileIT.java
+++ b/x-pack/plugin/security/qa/profile/src/javaRestTest/java/org/elasticsearch/xpack/security/profile/ProfileIT.java
@@ -102,7 +102,7 @@ public void testProfileHasPrivileges() throws IOException {
         final Map<String, Object> activateProfileMap = doActivateProfile();
         final String profileUid = (String) activateProfileMap.get("uid");
         final Request profileHasPrivilegesRequest = new Request("POST", "_security/profile/_has_privileges");
-        profileHasPrivilegesRequest.setJsonEntity(formatted("""
+        profileHasPrivilegesRequest.setJsonEntity(Strings.format("""
             {
               "uids": ["some_missing_profile", "%s"],
               "privileges": {
@@ -152,7 +152,7 @@ public void testGetProfiles() throws IOException {
 
         // Create the profile documents
         for (String uid : uids) {
-            final String source = String.format(Locale.ROOT, SAMPLE_PROFILE_DOCUMENT_TEMPLATE, uid, uid, Instant.now().toEpochMilli());
+            final String source = Strings.format(SAMPLE_PROFILE_DOCUMENT_TEMPLATE, uid, uid, Instant.now().toEpochMilli());
             final Request indexRequest = new Request("PUT", ".security-profile/_doc/profile_" + uid);
             indexRequest.setJsonEntity(source);
             indexRequest.addParameter("refresh", "wait_for");
@@ -264,7 +264,7 @@ public void testSuggestProfileWithHint() throws IOException {
         final String payload;
         switch (randomIntBetween(0, 2)) {
             case 0 -> {
-                payload = formatted("""
+                payload = Strings.format("""
                     {
                       "name": "rac",
                       "hint": {
@@ -274,7 +274,8 @@ public void testSuggestProfileWithHint() throws IOException {
                     """, "not-" + uid);
             }
             case 1 -> {
-                payload = formatted("""
+                Object[] args = new Object[] { randomBoolean() ? "\"demo\"" : "[\"demo\"]" };
+                payload = Strings.format("""
                     {
                       "name": "rac",
                       "hint": {
@@ -283,10 +284,11 @@ public void testSuggestProfileWithHint() throws IOException {
                         }
                       }
                     }
-                    """, randomBoolean() ? "\"demo\"" : "[\"demo\"]");
+                    """, args);
             }
             default -> {
-                payload = String.format(Locale.ROOT, """
+                Object[] args = new Object[] { "not-" + uid, randomBoolean() ? "\"demo\"" : "[\"demo\"]" };
+                payload = Strings.format("""
                     {
                       "name": "rac",
                       "hint": {
@@ -295,7 +297,7 @@ public void testSuggestProfileWithHint() throws IOException {
                           "kibana.spaces": %s
                         }
                       }
-                    }""", "not-" + uid, randomBoolean() ? "\"demo\"" : "[\"demo\"]");
+                    }""", args);
             }
         }
         suggestProfilesRequest1.setJsonEntity(payload);
@@ -463,7 +465,7 @@ private Map<String, Object> doActivateProfile() throws IOException {
 
     private Map<String, Object> doActivateProfile(String username, String password) throws IOException {
         final Request activateProfileRequest = new Request("POST", "_security/profile/_activate");
-        activateProfileRequest.setJsonEntity(String.format(Locale.ROOT, """
+        activateProfileRequest.setJsonEntity(Strings.format("""
             {
               "grant_type": "password",
               "username": "%s",
diff --git a/x-pack/plugin/security/qa/security-basic/build.gradle b/x-pack/plugin/security/qa/security-basic/build.gradle
index b6aed6bb26e9..10a808178965 100644
--- a/x-pack/plugin/security/qa/security-basic/build.gradle
+++ b/x-pack/plugin/security/qa/security-basic/build.gradle
@@ -1,5 +1,5 @@
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 import org.elasticsearch.gradle.internal.info.BuildParams
 
diff --git a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java
index 184606ed9521..f04641ae70b2 100644
--- a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java
+++ b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.XContentTestUtils;
 import org.elasticsearch.xcontent.XContentType;
@@ -21,7 +22,6 @@
 import java.util.ArrayList;
 import java.util.Base64;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Consumer;
@@ -113,8 +113,8 @@ public void testQuery() throws IOException {
                 var subset = randomSubsetOf(randomIntBetween(1, 5), apiKeys);
                 assertQuery(
                     API_KEY_ADMIN_AUTH_HEADER,
-                    formatted("""
-                        { "query": { "ids": { "values": [%s] } } }""",
+                        Strings.format("""
+                                { "query": { "ids": { "values": [%s] } } }""",
                             subset.stream().map(m -> "\"" + m.get("id") + "\"").collect(Collectors.joining(","))),
                     keys -> {
                         assertThat(keys, hasSize(subset.size()));
@@ -132,8 +132,8 @@ public void testQuery() throws IOException {
 
         // Search for fields that are not allowed in Query DSL but used internally by the service itself
         final String fieldName = randomFrom("doc_type", "api_key_invalidated");
-        assertQueryError(API_KEY_ADMIN_AUTH_HEADER, 400, String.format(Locale.ROOT, """
-            { "query": { "term": {"%s": "%s"} } }""", fieldName, randomAlphaOfLengthBetween(3, 8)));
+        assertQueryError(API_KEY_ADMIN_AUTH_HEADER, 400, Strings.format("""
+                { "query": { "term": {"%s": "%s"} } }""", fieldName, randomAlphaOfLengthBetween(3, 8)));
 
         // Search for api keys won't return other entities
         assertQuery(API_KEY_ADMIN_AUTH_HEADER, """
@@ -164,10 +164,10 @@ public void testQuery() throws IOException {
         final String authHeader = randomFrom(API_KEY_ADMIN_AUTH_HEADER, API_KEY_USER_AUTH_HEADER);
         final String invalidatedApiKeyId1 = createAndInvalidateApiKey("temporary-key-1", authHeader);
         final String queryString = randomFrom("""
-            {"query": { "term": {"name": "temporary-key-1"} } }""", formatted("""
-            {"query":{"bool":{"must":[{"term":{"name":{"value":"temporary-key-1"}}},\
-            {"term":{"invalidated":{"value":"%s"}}}]}}}
-            """, randomBoolean()));
+            {"query": { "term": {"name": "temporary-key-1"} } }""", Strings.format("""
+                {"query":{"bool":{"must":[{"term":{"name":{"value":"temporary-key-1"}}},\
+                {"term":{"invalidated":{"value":"%s"}}}]}}}
+                """, randomBoolean()));
 
         assertQuery(authHeader, queryString, apiKeys -> {
             if (queryString.contains("""
@@ -230,7 +230,7 @@ public void testPagination() throws IOException, InterruptedException {
         final String authHeader = randomFrom(API_KEY_ADMIN_AUTH_HEADER, API_KEY_USER_AUTH_HEADER);
         final int total = randomIntBetween(8, 12);
         final List<String> apiKeyNames = IntStream.range(0, total)
-            .mapToObj(i -> formatted("k-%02d", i))
+            .mapToObj(i -> Strings.format("k-%02d", i))
             .toList();
         final List<String> apiKeyIds = new ArrayList<>(total);
         for (int i = 0; i < total; i++) {
@@ -260,9 +260,9 @@ public void testPagination() throws IOException, InterruptedException {
             } else {
                 searchAfter.append(sortValues.get(0));
             }
-            request2.setJsonEntity(String.format(Locale.ROOT, """
-                {"size":%s,"sort":["%s"],"search_after":[%s]}
-                """, size, sortField, searchAfter));
+            request2.setJsonEntity(Strings.format("""
+                    {"size":%s,"sort":["%s"],"search_after":[%s]}
+                    """, size, sortField, searchAfter));
             actualSize = collectApiKeys(apiKeyInfos, request2, total, size);
             if (actualSize == 0 && apiKeyInfos.size() < remaining) {
                 fail("fail to retrieve all API keys, expect [" + remaining + "] keys, got [" + apiKeyInfos + "]");
@@ -599,10 +599,10 @@ private Tuple<String, String> createApiKey(
         final String metadataString = XContentTestUtils.convertToXContent(metadata == null ? Map.of() : metadata, XContentType.JSON)
             .utf8ToString();
         if (expiration == null) {
-            request.setJsonEntity(String.format(Locale.ROOT, """
+            request.setJsonEntity(Strings.format("""
                 {"name":"%s", "role_descriptors":%s, "metadata":%s}""", name, roleDescriptorsString, metadataString));
         } else {
-            request.setJsonEntity(String.format(Locale.ROOT, """
+            request.setJsonEntity(Strings.format("""
                 {"name":"%s", "expiration": "%s", "role_descriptors":%s,\
                 "metadata":%s}""", name, expiration, roleDescriptorsString, metadataString));
         }
@@ -617,7 +617,7 @@ private String createAndInvalidateApiKey(String name, String authHeader) throws
         final Tuple<String, String> tuple = createApiKey(name, null, authHeader);
         final Request request = new Request("DELETE", "/_security/api_key");
         request.setOptions(request.getOptions().toBuilder().addHeader(HttpHeaders.AUTHORIZATION, authHeader));
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {"ids": ["%s"],"owner":true}""", tuple.v1()));
         assertOK(client().performRequest(request));
         return tuple.v1();
diff --git a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java
index e6f7fe6ca8a2..e3de90ce2a05 100644
--- a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java
+++ b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.rest.ObjectPath;
 import org.elasticsearch.xpack.security.authc.InternalRealms;
@@ -512,7 +513,7 @@ private void assertUserProfileFeatures(boolean clusterHasTrialLicense) throws IO
         // Profile hasPrivileges
         final Request hasPrivilegesRequest = new Request("POST", "_security/profile/_has_privileges");
         hasPrivilegesRequest.setOptions(requestOptions);
-        hasPrivilegesRequest.setJsonEntity(formatted("""
+        hasPrivilegesRequest.setJsonEntity(Strings.format("""
             {
               "uids": [
                 "%s"
diff --git a/x-pack/plugin/security/qa/security-disabled/build.gradle b/x-pack/plugin/security/qa/security-disabled/build.gradle
index e1d850c4adb9..e3d9c2f4f65b 100644
--- a/x-pack/plugin/security/qa/security-disabled/build.gradle
+++ b/x-pack/plugin/security/qa/security-disabled/build.gradle
@@ -7,7 +7,7 @@
  */
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 dependencies {
   javaRestTestImplementation(testArtifact(project(xpackModule('security'))))
diff --git a/x-pack/plugin/security/qa/security-disabled/src/javaRestTest/java/org/elasticsearch/xpack/security/SetSecurityUserProcessorWithSecurityDisabledIT.java b/x-pack/plugin/security/qa/security-disabled/src/javaRestTest/java/org/elasticsearch/xpack/security/SetSecurityUserProcessorWithSecurityDisabledIT.java
index 4e0d1b83a875..58251fcdbabf 100644
--- a/x-pack/plugin/security/qa/security-disabled/src/javaRestTest/java/org/elasticsearch/xpack/security/SetSecurityUserProcessorWithSecurityDisabledIT.java
+++ b/x-pack/plugin/security/qa/security-disabled/src/javaRestTest/java/org/elasticsearch/xpack/security/SetSecurityUserProcessorWithSecurityDisabledIT.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.rest.ESRestTestCase;
 
 import static org.hamcrest.Matchers.containsString;
@@ -26,7 +27,7 @@ public void testDefineAndUseProcessor() throws Exception {
         final String index = "index-" + getTestName();
         {
             final Request putPipeline = new Request("PUT", "/_ingest/pipeline/" + pipeline);
-            putPipeline.setJsonEntity(formatted("""
+            putPipeline.setJsonEntity(Strings.format("""
                 {
                   "description": "Test pipeline (%s)",
                   "processors": [ { "set_security_user": { "field": "user" } } ]
diff --git a/x-pack/plugin/security/qa/security-trial/build.gradle b/x-pack/plugin/security/qa/security-trial/build.gradle
index b0f5a6da10d7..838c8366729b 100644
--- a/x-pack/plugin/security/qa/security-trial/build.gradle
+++ b/x-pack/plugin/security/qa/security-trial/build.gradle
@@ -1,4 +1,6 @@
-apply plugin: 'elasticsearch.internal-java-rest-test'
+import org.elasticsearch.gradle.Version
+
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 dependencies {
   javaRestTestImplementation project(path: xpackModule('core'))
@@ -8,6 +10,7 @@ dependencies {
 }
 
 testClusters.matching { it.name == 'javaRestTest' }.configureEach {
+  requiresFeature 'es.untrusted_remote_cluster_feature_flag_registered', Version.fromString("8.5.0")
   testDistribution = 'DEFAULT'
   numberOfNodes = 2
 
@@ -19,6 +22,7 @@ testClusters.matching { it.name == 'javaRestTest' }.configureEach {
   setting 'xpack.security.transport.ssl.enabled', 'false'
   setting 'xpack.security.authc.token.enabled', 'true'
   setting 'xpack.security.authc.api_key.enabled', 'true'
+  setting 'xpack.security.remote_cluster.ssl.enabled', 'false'
 
   rolesFile file('src/javaRestTest/resources/roles.yml')
   user username: "admin_user", password: "admin-password"
diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/remoteaccess/RemoteAccessHeadersForCcsRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/remoteaccess/RemoteAccessHeadersForCcsRestIT.java
new file mode 100644
index 000000000000..ffc3e7429c29
--- /dev/null
+++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/remoteaccess/RemoteAccessHeadersForCcsRestIT.java
@@ -0,0 +1,521 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.remoteaccess;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.entity.ContentType;
+import org.apache.http.nio.entity.NStringEntity;
+import org.apache.lucene.search.TotalHits;
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsAction;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.internal.InternalSearchResponse;
+import org.elasticsearch.test.rest.ObjectPath;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TcpTransport;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.json.JsonXContent;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.RemoteAccessAuthentication;
+import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection;
+import org.elasticsearch.xpack.core.security.user.SystemUser;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.security.SecurityOnTrialLicenseRestTestCase;
+import org.elasticsearch.xpack.security.authz.RBACEngine;
+import org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.common.UUIDs.randomBase64UUID;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasKey;
+
+public class RemoteAccessHeadersForCcsRestIT extends SecurityOnTrialLicenseRestTestCase {
+    @BeforeClass
+    public static void checkFeatureFlag() {
+        assumeTrue("untrusted remote cluster feature flag must be enabled", TcpTransport.isUntrustedRemoteClusterEnabled());
+    }
+
+    private static final String CLUSTER_A = "my_remote_cluster_a";
+    private static final String CLUSTER_B = "my_remote_cluster_b";
+    private static final String REMOTE_SEARCH_USER = "remote_search_user";
+    private static final SecureString PASSWORD = new SecureString("super-secret-password".toCharArray());
+    private static final String REMOTE_SEARCH_ROLE = "remote_search";
+
+    private final ThreadPool threadPool = new TestThreadPool(getClass().getName());
+
+    @Before
+    public void setup() throws IOException {
+        createUser(REMOTE_SEARCH_USER, PASSWORD, List.of(REMOTE_SEARCH_ROLE));
+
+        final var putRoleRequest = new Request("PUT", "/_security/role/" + REMOTE_SEARCH_ROLE);
+        putRoleRequest.setJsonEntity("""
+            {
+              "indices": [
+                {
+                  "names": ["index-a"],
+                  "privileges": ["read"]
+                }
+              ],
+              "remote_indices": [
+                {
+                  "names": ["index-a"],
+                  "privileges": ["read", "read_cross_cluster"],
+                  "clusters": ["my_remote_cluster*"]
+                },
+                {
+                  "names": ["index-b"],
+                  "privileges": ["read", "read_cross_cluster"],
+                  "clusters": ["my_remote_cluster_b"]
+                }
+              ]
+            }""");
+        assertOK(adminClient().performRequest(putRoleRequest));
+
+        final var indexDocRequest = new Request("POST", "/index-a/_doc?refresh=true");
+        indexDocRequest.setJsonEntity("{\"foo\": \"bar\"}");
+        assertOK(adminClient().performRequest(indexDocRequest));
+    }
+
+    @After
+    public void cleanup() throws IOException {
+        deleteUser(REMOTE_SEARCH_USER);
+        deleteRole(REMOTE_SEARCH_ROLE);
+        deleteIndex(adminClient(), "index-a");
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
+    }
+
+    public void testRemoteAccessHeadersSentSingleRemote() throws Exception {
+        final BlockingQueue<CapturedActionWithHeaders> capturedHeaders = ConcurrentCollections.newBlockingQueue();
+        try (MockTransportService remoteTransport = startTransport("remoteNodeA", threadPool, capturedHeaders)) {
+            final String clusterCredential = randomBase64UUID(random());
+            final TransportAddress remoteAddress = remoteTransport.getOriginalTransport()
+                .profileBoundAddresses()
+                .get("_remote_cluster")
+                .publishAddress();
+            final boolean useProxyMode = randomBoolean();
+            setupClusterSettings(CLUSTER_A, clusterCredential, remoteAddress, useProxyMode);
+            final boolean alsoSearchLocally = randomBoolean();
+            final boolean minimizeRoundtrips = randomBoolean();
+            final Request searchRequest = new Request(
+                "GET",
+                String.format(
+                    Locale.ROOT,
+                    "/%s%s:index-a/_search?ccs_minimize_roundtrips=%s",
+                    alsoSearchLocally ? "index-a," : "",
+                    CLUSTER_A,
+                    minimizeRoundtrips
+                )
+            );
+            searchRequest.setOptions(
+                searchRequest.getOptions()
+                    .toBuilder()
+                    .addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(REMOTE_SEARCH_USER, PASSWORD))
+            );
+
+            final Response response = client().performRequest(searchRequest);
+            assertOK(response);
+            assertThat(ObjectPath.createFromResponse(response).evaluate("hits.total.value"), equalTo(alsoSearchLocally ? 1 : 0));
+
+            expectActionsAndHeadersForCluster(
+                List.copyOf(capturedHeaders),
+                useProxyMode,
+                minimizeRoundtrips,
+                clusterCredential,
+                new RoleDescriptorsIntersection(
+                    List.of(
+                        Set.of(
+                            new RoleDescriptor(
+                                RBACEngine.REMOTE_USER_ROLE_NAME,
+                                null,
+                                new RoleDescriptor.IndicesPrivileges[] {
+                                    RoleDescriptor.IndicesPrivileges.builder()
+                                        .indices("index-a")
+                                        .privileges("read", "read_cross_cluster")
+                                        .build() },
+                                null,
+                                null,
+                                null,
+                                null,
+                                null,
+                                null
+                            )
+                        )
+                    )
+                )
+            );
+        }
+    }
+
+    public void testRemoteAccessHeadersSentMultipleRemotes() throws Exception {
+        final Map<String, BlockingQueue<CapturedActionWithHeaders>> capturedHeadersByCluster = Map.of(
+            CLUSTER_A,
+            ConcurrentCollections.newBlockingQueue(),
+            CLUSTER_B,
+            ConcurrentCollections.newBlockingQueue()
+        );
+        try (
+            MockTransportService remoteTransportA = startTransport("remoteNodeA", threadPool, capturedHeadersByCluster.get(CLUSTER_A));
+            MockTransportService remoteTransportB = startTransport("remoteNodeB", threadPool, capturedHeadersByCluster.get(CLUSTER_B))
+        ) {
+            final String clusterCredentialA = randomBase64UUID(random());
+            final boolean useProxyModeA = randomBoolean();
+            setupClusterSettings(
+                CLUSTER_A,
+                clusterCredentialA,
+                remoteTransportA.getOriginalTransport().profileBoundAddresses().get("_remote_cluster").publishAddress(),
+                useProxyModeA
+            );
+
+            final String clusterCredentialB = randomBase64UUID(random());
+            final boolean useProxyModeB = randomBoolean();
+            setupClusterSettings(
+                CLUSTER_B,
+                clusterCredentialB,
+                remoteTransportB.getOriginalTransport().profileBoundAddresses().get("_remote_cluster").publishAddress(),
+                useProxyModeB
+            );
+
+            final boolean minimizeRoundtrips = randomBoolean();
+            final Request searchRequest = new Request(
+                "GET",
+                String.format(
+                    Locale.ROOT,
+                    "/%s:index-a,%s:index-*/_search?ccs_minimize_roundtrips=%s",
+                    CLUSTER_A,
+                    CLUSTER_B,
+                    minimizeRoundtrips
+                )
+            );
+            searchRequest.setOptions(
+                searchRequest.getOptions()
+                    .toBuilder()
+                    .addHeader("Authorization", UsernamePasswordToken.basicAuthHeaderValue(REMOTE_SEARCH_USER, PASSWORD))
+            );
+
+            final Response response = client().performRequest(searchRequest);
+            assertOK(response);
+            assertThat(ObjectPath.createFromResponse(response).evaluate("hits.total.value"), equalTo(0));
+
+            expectActionsAndHeadersForCluster(
+                List.copyOf(capturedHeadersByCluster.get(CLUSTER_A)),
+                useProxyModeA,
+                minimizeRoundtrips,
+                clusterCredentialA,
+                new RoleDescriptorsIntersection(
+                    List.of(
+                        Set.of(
+                            new RoleDescriptor(
+                                RBACEngine.REMOTE_USER_ROLE_NAME,
+                                null,
+                                new RoleDescriptor.IndicesPrivileges[] {
+                                    RoleDescriptor.IndicesPrivileges.builder()
+                                        .indices("index-a")
+                                        .privileges("read", "read_cross_cluster")
+                                        .build() },
+                                null,
+                                null,
+                                null,
+                                null,
+                                null,
+                                null
+                            )
+                        )
+                    )
+                )
+            );
+            expectActionsAndHeadersForCluster(
+                List.copyOf(capturedHeadersByCluster.get(CLUSTER_B)),
+                useProxyModeB,
+                minimizeRoundtrips,
+                clusterCredentialB,
+                new RoleDescriptorsIntersection(
+                    List.of(
+                        Set.of(
+                            new RoleDescriptor(
+                                RBACEngine.REMOTE_USER_ROLE_NAME,
+                                null,
+                                new RoleDescriptor.IndicesPrivileges[] {
+                                    RoleDescriptor.IndicesPrivileges.builder()
+                                        .indices("index-a")
+                                        .privileges("read", "read_cross_cluster")
+                                        .build(),
+                                    RoleDescriptor.IndicesPrivileges.builder()
+                                        .indices("index-b")
+                                        .privileges("read", "read_cross_cluster")
+                                        .build() },
+                                null,
+                                null,
+                                null,
+                                null,
+                                null,
+                                null
+                            )
+                        )
+                    )
+                )
+            );
+        }
+    }
+
+    private void setupClusterSettings(
+        final String clusterAlias,
+        final String clusterCredential,
+        final TransportAddress remoteAddress,
+        boolean useProxyMode
+    ) throws IOException {
+        if (useProxyMode) {
+            updateRemoteClusterSettings(
+                clusterAlias,
+                Map.of("mode", "proxy", "proxy_address", remoteAddress.toString(), "authorization", clusterCredential)
+            );
+        } else {
+            updateRemoteClusterSettings(clusterAlias, Map.of("seeds", remoteAddress.toString(), "authorization", clusterCredential));
+        }
+    }
+
+    private void expectActionsAndHeadersForCluster(
+        final List<CapturedActionWithHeaders> actualActionsWithHeaders,
+        boolean useProxyMode,
+        boolean minimizeRoundtrips,
+        final String clusterCredential,
+        final RoleDescriptorsIntersection expectedRoleDescriptorsIntersection
+    ) throws IOException {
+        final Set<String> expectedActions = new HashSet<>();
+        if (minimizeRoundtrips) {
+            expectedActions.add(SearchAction.NAME);
+        } else {
+            expectedActions.add(ClusterSearchShardsAction.NAME);
+        }
+        if (false == useProxyMode) {
+            expectedActions.add(ClusterStateAction.NAME);
+        }
+        assertThat(
+            actualActionsWithHeaders.stream().map(CapturedActionWithHeaders::action).collect(Collectors.toUnmodifiableSet()),
+            equalTo(expectedActions)
+        );
+        for (CapturedActionWithHeaders actual : actualActionsWithHeaders) {
+            switch (actual.action) {
+                // the cluster state action is run by the system user, so we expect a remote access authentication header with an internal
+                // user authentication and empty role descriptors intersection
+                case ClusterStateAction.NAME -> {
+                    assertContainsRemoteAccessHeaders(actual.headers());
+                    assertContainsRemoteClusterCredential(clusterCredential, actual);
+                    final var actualRemoteAccessAuthentication = RemoteAccessAuthentication.decode(
+                        actual.headers().get(RemoteAccessAuthentication.REMOTE_ACCESS_AUTHENTICATION_HEADER_KEY)
+                    );
+                    final var expectedRemoteAccessAuthentication = new RemoteAccessAuthentication(
+                        Authentication.newInternalAuthentication(
+                            SystemUser.INSTANCE,
+                            TransportVersion.CURRENT,
+                            // Since we are running on a multi-node cluster the actual node name may be different between runs
+                            // so just copy the one from the actual result
+                            actualRemoteAccessAuthentication.getAuthentication().getEffectiveSubject().getRealm().getNodeName()
+                        ),
+                        RoleDescriptorsIntersection.EMPTY
+                    );
+                    assertThat(actualRemoteAccessAuthentication, equalTo(expectedRemoteAccessAuthentication));
+                }
+                case SearchAction.NAME, ClusterSearchShardsAction.NAME -> {
+                    assertContainsRemoteAccessHeaders(actual.headers());
+                    assertContainsRemoteClusterCredential(clusterCredential, actual);
+                    final var actualRemoteAccessAuthentication = RemoteAccessAuthentication.decode(
+                        actual.headers().get(RemoteAccessAuthentication.REMOTE_ACCESS_AUTHENTICATION_HEADER_KEY)
+                    );
+                    final var expectedRemoteAccessAuthentication = new RemoteAccessAuthentication(
+                        Authentication.newRealmAuthentication(
+                            new User(REMOTE_SEARCH_USER, REMOTE_SEARCH_ROLE),
+                            new Authentication.RealmRef(
+                                "default_native",
+                                "native",
+                                // Since we are running on a multi-node cluster the actual node name may be different between runs
+                                // so just copy the one from the actual result
+                                actualRemoteAccessAuthentication.getAuthentication().getEffectiveSubject().getRealm().getNodeName()
+                            )
+                        ),
+                        expectedRoleDescriptorsIntersection
+                    );
+                    assertThat(actualRemoteAccessAuthentication, equalTo(expectedRemoteAccessAuthentication));
+                }
+                default -> fail("Unexpected action [" + actual.action + "]");
+            }
+        }
+    }
+
+    private void assertContainsRemoteClusterCredential(String clusterCredential, CapturedActionWithHeaders actual) {
+        assertThat(actual.headers(), hasKey(SecurityServerTransportInterceptor.REMOTE_ACCESS_CLUSTER_CREDENTIAL_HEADER_KEY));
+        assertThat(
+            actual.headers().get(SecurityServerTransportInterceptor.REMOTE_ACCESS_CLUSTER_CREDENTIAL_HEADER_KEY),
+            equalTo("ApiKey " + clusterCredential)
+        );
+    }
+
+    private static MockTransportService startTransport(
+        final String nodeName,
+        final ThreadPool threadPool,
+        final BlockingQueue<CapturedActionWithHeaders> capturedHeaders
+    ) {
+        boolean success = false;
+        final Settings settings = Settings.builder()
+            .put("node.name", nodeName)
+            .put("remote_cluster.enabled", "true")
+            .put("remote_cluster.port", "0")
+            .put("xpack.security.remote_cluster.ssl.enabled", "false")
+            .build();
+        final ClusterName clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
+        final MockTransportService service = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null);
+        try {
+            service.registerRequestHandler(
+                ClusterStateAction.NAME,
+                ThreadPool.Names.SAME,
+                ClusterStateRequest::new,
+                (request, channel, task) -> {
+                    capturedHeaders.add(
+                        new CapturedActionWithHeaders(task.getAction(), Map.copyOf(threadPool.getThreadContext().getHeaders()))
+                    );
+                    channel.sendResponse(new ClusterStateResponse(clusterName, ClusterState.builder(clusterName).build(), false));
+                }
+            );
+            service.registerRequestHandler(
+                ClusterSearchShardsAction.NAME,
+                ThreadPool.Names.SAME,
+                ClusterSearchShardsRequest::new,
+                (request, channel, task) -> {
+                    capturedHeaders.add(
+                        new CapturedActionWithHeaders(task.getAction(), Map.copyOf(threadPool.getThreadContext().getHeaders()))
+                    );
+                    channel.sendResponse(
+                        new ClusterSearchShardsResponse(new ClusterSearchShardsGroup[0], new DiscoveryNode[0], Collections.emptyMap())
+                    );
+                }
+            );
+            service.registerRequestHandler(SearchAction.NAME, ThreadPool.Names.SAME, SearchRequest::new, (request, channel, task) -> {
+                capturedHeaders.add(
+                    new CapturedActionWithHeaders(task.getAction(), Map.copyOf(threadPool.getThreadContext().getHeaders()))
+                );
+                channel.sendResponse(
+                    new SearchResponse(
+                        new InternalSearchResponse(
+                            new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN),
+                            InternalAggregations.EMPTY,
+                            null,
+                            null,
+                            false,
+                            null,
+                            1
+                        ),
+                        null,
+                        1,
+                        1,
+                        0,
+                        100,
+                        ShardSearchFailure.EMPTY_ARRAY,
+                        SearchResponse.Clusters.EMPTY
+                    )
+                );
+            });
+            service.start();
+            service.acceptIncomingRequests();
+            success = true;
+            return service;
+        } finally {
+            if (success == false) {
+                service.close();
+            }
+        }
+    }
+
+    private void assertContainsRemoteAccessHeaders(final Map<String, String> actualHeaders) {
+        assertThat(
+            actualHeaders.keySet(),
+            containsInAnyOrder(
+                RemoteAccessAuthentication.REMOTE_ACCESS_AUTHENTICATION_HEADER_KEY,
+                SecurityServerTransportInterceptor.REMOTE_ACCESS_CLUSTER_CREDENTIAL_HEADER_KEY
+            )
+        );
+    }
+
+    private record CapturedActionWithHeaders(String action, Map<String, String> headers) {}
+
+    private static void updateRemoteClusterSettings(final String clusterAlias, final Map<String, Object> settings) throws IOException {
+        final Request request = new Request("PUT", "/_cluster/settings");
+        request.setEntity(buildUpdateSettingsRequestBody(clusterAlias, settings));
+        final Response response = adminClient().performRequest(request);
+        assertOK(response);
+        assertEquals(200, response.getStatusLine().getStatusCode());
+    }
+
+    private static HttpEntity buildUpdateSettingsRequestBody(final String clusterAlias, final Map<String, Object> settings)
+        throws IOException {
+        final String requestBody;
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            builder.startObject();
+            {
+                builder.startObject("persistent");
+                {
+                    builder.startObject("cluster.remote." + clusterAlias);
+                    {
+                        for (Map.Entry<String, Object> entry : settings.entrySet()) {
+                            builder.field(entry.getKey(), entry.getValue());
+                        }
+                    }
+                    builder.endObject();
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+            requestBody = Strings.toString(builder);
+        }
+        return new NStringEntity(requestBody, ContentType.APPLICATION_JSON);
+    }
+}
diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/role/RoleWithRemoteIndicesPrivilegesRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/role/RoleWithRemoteIndicesPrivilegesRestIT.java
index 20a2cf00bc87..d0530929e095 100644
--- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/role/RoleWithRemoteIndicesPrivilegesRestIT.java
+++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/role/RoleWithRemoteIndicesPrivilegesRestIT.java
@@ -28,8 +28,10 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
 
 public class RoleWithRemoteIndicesPrivilegesRestIT extends SecurityOnTrialLicenseRestTestCase {
 
@@ -55,21 +57,25 @@ public void cleanup() throws IOException {
     }
 
     public void testRemoteIndexPrivileges() throws IOException {
-        var putRoleRequest = new Request("PUT", "_security/role/" + REMOTE_SEARCH_ROLE);
+        var putRoleRequest = new Request("PUT", "/_security/role/" + REMOTE_SEARCH_ROLE);
         putRoleRequest.setJsonEntity("""
             {
               "remote_indices": [
                 {
                   "names": ["index-a", "*"],
                   "privileges": ["read"],
-                  "clusters": ["remote-a", "*"]
+                  "clusters": ["remote-a", "*"],
+                  "query": "{\\"match\\":{\\"field\\":\\"a\\"}}",
+                  "field_security" : {
+                    "grant": ["field"]
+                  }
                 }
               ]
             }""");
         final Response putRoleResponse1 = adminClient().performRequest(putRoleRequest);
         assertOK(putRoleResponse1);
 
-        final Response getRoleResponse = adminClient().performRequest(new Request("GET", "_security/role/" + REMOTE_SEARCH_ROLE));
+        final Response getRoleResponse = adminClient().performRequest(new Request("GET", "/_security/role/" + REMOTE_SEARCH_ROLE));
         assertOK(getRoleResponse);
         expectRoleDescriptorInResponse(
             getRoleResponse,
@@ -83,7 +89,12 @@ public void testRemoteIndexPrivileges() throws IOException {
                 null,
                 null,
                 new RoleDescriptor.RemoteIndicesPrivileges[] {
-                    RoleDescriptor.RemoteIndicesPrivileges.builder("remote-a", "*").indices("index-a", "*").privileges("read").build() }
+                    RoleDescriptor.RemoteIndicesPrivileges.builder("remote-a", "*")
+                        .indices("index-a", "*")
+                        .query("{\"match\":{\"field\":\"a\"}}")
+                        .privileges("read")
+                        .grantedFields("field")
+                        .build() }
             )
         );
 
@@ -113,7 +124,11 @@ public void testRemoteIndexPrivileges() throws IOException {
                 {
                   "names": ["index-a", "*"],
                   "privileges": ["read"],
-                  "clusters": ["remote-a", "*"]
+                  "clusters": ["remote-a", "*"],
+                  "query": "{\\"match\\":{\\"field\\":\\"a\\"}}",
+                  "field_security" : {
+                    "grant": ["field"]
+                  }
                 }
               ]
             }""");
@@ -146,7 +161,12 @@ public void testRemoteIndexPrivileges() throws IOException {
                 null,
                 null,
                 new RoleDescriptor.RemoteIndicesPrivileges[] {
-                    RoleDescriptor.RemoteIndicesPrivileges.builder("remote-a", "*").indices("index-a", "*").privileges("read").build() }
+                    RoleDescriptor.RemoteIndicesPrivileges.builder("remote-a", "*")
+                        .indices("index-a", "*")
+                        .privileges("read")
+                        .query("{\"match\":{\"field\":\"a\"}}")
+                        .grantedFields("field")
+                        .build() }
             )
         );
     }
@@ -159,7 +179,11 @@ public void testGetUserPrivileges() throws IOException {
                 {
                   "names": ["index-a", "*"],
                   "privileges": ["read"],
-                  "clusters": ["remote-a", "*"]
+                  "clusters": ["remote-a", "*"],
+                  "query": "{\\"match\\":{\\"field\\":\\"a\\"}}",
+                  "field_security": {
+                    "grant": ["field"]
+                  }
                 }
               ]
             }""");
@@ -168,7 +192,7 @@ public void testGetUserPrivileges() throws IOException {
 
         final Response getUserPrivilegesResponse1 = executeAsRemoteSearchUser(new Request("GET", "/_security/user/_privileges"));
         assertOK(getUserPrivilegesResponse1);
-        assertThat(responseAsMap(getUserPrivilegesResponse1), equalTo(XContentHelper.convertToMap(JsonXContent.jsonXContent, """
+        assertThat(responseAsMap(getUserPrivilegesResponse1), equalTo(mapFromJson("""
             {
               "cluster": [],
               "global": [],
@@ -180,10 +204,12 @@ public void testGetUserPrivileges() throws IOException {
                   "names": ["*", "index-a"],
                   "privileges": ["read"],
                   "allow_restricted_indices": false,
-                  "clusters": ["remote-a", "*"]
+                  "clusters": ["remote-a", "*"],
+                  "query": ["{\\"match\\":{\\"field\\":\\"a\\"}}"],
+                  "field_security": [{"grant": ["field"]}]
                 }
               ]
-            }""", false)));
+            }""")));
 
         final var putRoleRequest2 = new Request("PUT", "/_security/role/" + REMOTE_SEARCH_ROLE);
         putRoleRequest2.setJsonEntity("""
@@ -208,7 +234,7 @@ public void testGetUserPrivileges() throws IOException {
 
         final Response getUserPrivilegesResponse2 = executeAsRemoteSearchUser(new Request("GET", "/_security/user/_privileges"));
         assertOK(getUserPrivilegesResponse2);
-        assertThat(responseAsMap(getUserPrivilegesResponse2), equalTo(XContentHelper.convertToMap(JsonXContent.jsonXContent, """
+        assertThat(responseAsMap(getUserPrivilegesResponse2), equalTo(mapFromJson("""
             {
               "cluster": ["all"],
               "global": [],
@@ -229,7 +255,65 @@ public void testGetUserPrivileges() throws IOException {
                   "clusters": ["remote-a", "*"]
                 }
               ]
-            }""", false)));
+            }""")));
+    }
+
+    public void testGetUserPrivilegesWithMultipleFlsDlsDefinitionsPreservesGroupPerIndexPrivilege() throws IOException {
+        final var putRoleRequest = new Request("PUT", "/_security/role/" + REMOTE_SEARCH_ROLE);
+        putRoleRequest.setJsonEntity("""
+            {
+              "remote_indices": [
+                {
+                  "names": ["index-a", "*"],
+                  "privileges": ["read"],
+                  "clusters": ["remote-a", "*"],
+                  "query": "{\\"match\\":{\\"field-a\\":\\"a\\"}}",
+                  "field_security": {
+                    "grant": ["field-a"]
+                  }
+                },
+                {
+                  "names": ["index-a", "*"],
+                  "privileges": ["read"],
+                  "clusters": ["remote-a", "*"],
+                  "query": "{\\"match\\":{\\"field-b\\":\\"b\\"}}",
+                  "field_security": {
+                    "grant": ["field-b"]
+                  }
+                }
+              ]
+            }""");
+        final Response putRoleResponse1 = adminClient().performRequest(putRoleRequest);
+        assertOK(putRoleResponse1);
+
+        final Response getUserPrivilegesResponse1 = executeAsRemoteSearchUser(new Request("GET", "/_security/user/_privileges"));
+        assertOK(getUserPrivilegesResponse1);
+        final Map<String, Object> actual = responseAsMap(getUserPrivilegesResponse1);
+        // The order of remote indices is not deterministic, so manually extract remote indices from map response to compare ignoring order
+        @SuppressWarnings("unchecked")
+        final List<Object> rawRemoteIndices = (List<Object>) actual.get("remote_indices");
+        assertThat(rawRemoteIndices, notNullValue());
+        assertThat(rawRemoteIndices, containsInAnyOrder(mapFromJson("""
+            {
+              "names": ["*", "index-a"],
+              "privileges": ["read"],
+              "allow_restricted_indices": false,
+              "clusters": ["remote-a", "*"],
+              "query": ["{\\"match\\":{\\"field-a\\":\\"a\\"}}"],
+              "field_security": [{"grant": ["field-a"]}]
+            }"""), mapFromJson("""
+            {
+              "names": ["*", "index-a"],
+              "privileges": ["read"],
+              "allow_restricted_indices": false,
+              "clusters": ["remote-a", "*"],
+              "query": ["{\\"match\\":{\\"field-b\\":\\"b\\"}}"],
+              "field_security": [{"grant": ["field-b"]}]
+            }""")));
+    }
+
+    private static Map<String, Object> mapFromJson(String json) {
+        return XContentHelper.convertToMap(JsonXContent.jsonXContent, json, false);
     }
 
     private Response executeAsRemoteSearchUser(final Request request) throws IOException {
diff --git a/x-pack/plugin/security/qa/service-account/build.gradle b/x-pack/plugin/security/qa/service-account/build.gradle
index 4c9724d14704..8953e7f0afbd 100644
--- a/x-pack/plugin/security/qa/service-account/build.gradle
+++ b/x-pack/plugin/security/qa/service-account/build.gradle
@@ -4,41 +4,5 @@ dependencies {
   javaRestTestImplementation project(':x-pack:plugin:core')
   javaRestTestImplementation project(':client:rest-high-level')
   javaRestTestImplementation project(':x-pack:plugin:security')
-}
-
-testClusters.matching { it.name == 'javaRestTest' }.configureEach {
-  testDistribution = 'DEFAULT'
-  numberOfNodes = 2
-
-  extraConfigFile 'node.key', file('src/javaRestTest/resources/ssl/node.key')
-  extraConfigFile 'node.crt', file('src/javaRestTest/resources/ssl/node.crt')
-  extraConfigFile 'ca.crt', file('src/javaRestTest/resources/ssl/ca.crt')
-  extraConfigFile 'service_tokens', file('src/javaRestTest/resources/service_tokens')
-
-  setting 'xpack.ml.enabled', 'false'
-  setting 'xpack.license.self_generated.type', 'trial'
-
-  setting 'xpack.security.enabled', 'true'
-  setting 'xpack.security.authc.token.enabled', 'true'
-  setting 'xpack.security.authc.api_key.enabled', 'true'
-
-  setting 'xpack.security.http.ssl.enabled', 'true'
-  setting 'xpack.security.http.ssl.certificate', 'node.crt'
-  setting 'xpack.security.http.ssl.key', 'node.key'
-  setting 'xpack.security.http.ssl.certificate_authorities', 'ca.crt'
-
-  setting 'xpack.security.transport.ssl.enabled', 'true'
-  setting 'xpack.security.transport.ssl.certificate', 'node.crt'
-  setting 'xpack.security.transport.ssl.key', 'node.key'
-  setting 'xpack.security.transport.ssl.certificate_authorities', 'ca.crt'
-  setting 'xpack.security.transport.ssl.verification_mode', 'certificate'
-
-  keystore 'bootstrap.password', 'x-pack-test-password'
-  keystore 'xpack.security.transport.ssl.secure_key_passphrase', 'node-password'
-  keystore 'xpack.security.http.ssl.secure_key_passphrase', 'node-password'
-
-  rolesFile file('src/javaRestTest/resources/roles.yml')
-  user username: "test_admin", password: 'x-pack-test-password'
-  user username: "elastic/fleet-server", password: 'x-pack-test-password', role: "superuser"
-  user username: "service_account_manager", password: 'x-pack-test-password', role: "service_account_manager"
+  clusterModules(project(":modules:analysis-common"))
 }
diff --git a/x-pack/plugin/security/qa/service-account/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountIT.java b/x-pack/plugin/security/qa/service-account/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountIT.java
index 1a06da6efb46..7732a5fa1e93 100644
--- a/x-pack/plugin/security/qa/service-account/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountIT.java
+++ b/x-pack/plugin/security/qa/service-account/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountIT.java
@@ -19,6 +19,8 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.PathUtils;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.util.resource.Resource;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentType;
@@ -26,6 +28,7 @@
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.core.security.user.KibanaSystemUser;
 import org.junit.BeforeClass;
+import org.junit.ClassRule;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -35,7 +38,6 @@
 import java.nio.file.Path;
 import java.util.Base64;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.anEmptyMap;
@@ -197,6 +199,36 @@ public class ServiceAccountIT extends ESRestTestCase {
             }
         }""";
 
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .nodes(2)
+        .module("analysis-common")
+        .setting("xpack.license.self_generated.type", "trial")
+        .setting("xpack.security.enabled", "true")
+        .setting("xpack.security.authc.token.enabled", "true")
+        .setting("xpack.security.authc.api_key.enabled", "true")
+        .setting("xpack.security.http.ssl.enabled", "true")
+        .setting("xpack.security.http.ssl.certificate", "node.crt")
+        .setting("xpack.security.http.ssl.key", "node.key")
+        .setting("xpack.security.http.ssl.certificate_authorities", "ca.crt")
+        .setting("xpack.security.transport.ssl.enabled", "true")
+        .setting("xpack.security.transport.ssl.certificate", "node.crt")
+        .setting("xpack.security.transport.ssl.key", "node.key")
+        .setting("xpack.security.transport.ssl.certificate_authorities", "ca.crt")
+        .setting("xpack.security.transport.ssl.verification_mode", "certificate")
+        .keystore("bootstrap.password", "x-pack-test-password")
+        .keystore("xpack.security.transport.ssl.secure_key_passphrase", "node-password")
+        .keystore("xpack.security.http.ssl.secure_key_passphrase", "node-password")
+        .configFile("node.key", Resource.fromClasspath("ssl/node.key"))
+        .configFile("node.crt", Resource.fromClasspath("ssl/node.crt"))
+        .configFile("ca.crt", Resource.fromClasspath("ssl/ca.crt"))
+        .configFile("service_tokens", Resource.fromClasspath("service_tokens"))
+        .rolesFile(Resource.fromClasspath("roles.yml"))
+        .user("test_admin", "x-pack-test-password")
+        .user("elastic/fleet-server", "x-pack-test-password", "superuser")
+        .user("service_account_manager", "x-pack-test-password", "service_account_manager")
+        .build();
+
     @BeforeClass
     public static void init() throws URISyntaxException, FileNotFoundException {
         URL resource = ServiceAccountIT.class.getResource("/ssl/ca.crt");
@@ -206,6 +238,11 @@ public static void init() throws URISyntaxException, FileNotFoundException {
         caPath = PathUtils.get(resource.toURI());
     }
 
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
     @Override
     protected String getProtocol() {
         // Because http.ssl.enabled = true
@@ -281,7 +318,7 @@ public void testAuthenticate() throws IOException {
             responseAsMap(response),
             equalTo(
                 XContentHelper.convertToMap(
-                    new BytesArray(String.format(Locale.ROOT, AUTHENTICATE_RESPONSE, "token1", "file")),
+                    new BytesArray(Strings.format(AUTHENTICATE_RESPONSE, "token1", "file")),
                     false,
                     XContentType.JSON
                 ).v2()
@@ -328,7 +365,7 @@ public void testAuthenticateShouldWorkWithOAuthBearerToken() throws IOException
 
         final String refreshToken = (String) oauthTokenResponseMap.get("refresh_token");
         final Request refreshTokenRequest = new Request("POST", "_security/oauth2/token");
-        refreshTokenRequest.setJsonEntity(formatted("""
+        refreshTokenRequest.setJsonEntity(Strings.format("""
             {"grant_type":"refresh_token","refresh_token":"%s"}
             """, refreshToken));
         final Response refreshTokenResponse = adminClient().performRequest(refreshTokenRequest);
@@ -373,7 +410,7 @@ public void testCreateApiServiceAccountTokenAndAuthenticateWithIt() throws IOExc
             responseAsMap(response),
             equalTo(
                 XContentHelper.convertToMap(
-                    new BytesArray(String.format(Locale.ROOT, AUTHENTICATE_RESPONSE, "api-token-1", "index")),
+                    new BytesArray(Strings.format(AUTHENTICATE_RESPONSE, "api-token-1", "index")),
                     false,
                     XContentType.JSON
                 ).v2()
@@ -520,7 +557,7 @@ public void testManageOwnApiKey() throws IOException {
         assertThat(e.getMessage(), containsString("is unauthorized for API key"));
 
         final Request invalidateApiKeysRequest = new Request("DELETE", "_security/api_key");
-        invalidateApiKeysRequest.setJsonEntity(formatted("""
+        invalidateApiKeysRequest.setJsonEntity(Strings.format("""
             {"ids":["%s"],"owner":true}""", apiKeyId1));
         invalidateApiKeysRequest.setOptions(requestOptions);
         final Response invalidateApiKeysResponse = client().performRequest(invalidateApiKeysRequest);
@@ -574,6 +611,6 @@ private void assertNodesCredentials(Map<String, Object> responseMap) {
         final Map<String, Object> fileTokens = (Map<String, Object>) nodes.get("file_tokens");
         assertThat(fileTokens, hasKey("token1"));
         final Map<String, Object> token1 = (Map<String, Object>) fileTokens.get("token1");
-        assertThat((List<String>) token1.get("nodes"), equalTo(List.of("javaRestTest-0", "javaRestTest-1")));
+        assertThat((List<String>) token1.get("nodes"), equalTo(List.of("test-cluster-0", "test-cluster-1")));
     }
 }
diff --git a/x-pack/plugin/security/qa/smoke-test-all-realms/build.gradle b/x-pack/plugin/security/qa/smoke-test-all-realms/build.gradle
index 02e40b7c8cf4..ca33b61de4ca 100644
--- a/x-pack/plugin/security/qa/smoke-test-all-realms/build.gradle
+++ b/x-pack/plugin/security/qa/smoke-test-all-realms/build.gradle
@@ -6,7 +6,7 @@
  * This test is also intended to work correctly on FIPS mode because we also want to know if a realm breaks on FIPS.
  */
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 dependencies {
   javaRestTestImplementation(testArtifact(project(xpackModule('security'))))
diff --git a/x-pack/plugin/security/qa/tls-basic/build.gradle b/x-pack/plugin/security/qa/tls-basic/build.gradle
index a4c3554afd93..fbe91009011e 100644
--- a/x-pack/plugin/security/qa/tls-basic/build.gradle
+++ b/x-pack/plugin/security/qa/tls-basic/build.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 import org.elasticsearch.gradle.internal.info.BuildParams
 
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/AbstractDocumentAndFieldLevelSecurityTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/AbstractDocumentAndFieldLevelSecurityTests.java
new file mode 100644
index 000000000000..152ff43e963b
--- /dev/null
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/AbstractDocumentAndFieldLevelSecurityTests.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.integration;
+
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.license.GetFeatureUsageRequest;
+import org.elasticsearch.license.GetFeatureUsageResponse;
+import org.elasticsearch.license.TransportGetFeatureUsageAction;
+import org.elasticsearch.test.SecurityIntegTestCase;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.elasticsearch.xpack.core.security.SecurityField.DOCUMENT_LEVEL_SECURITY_FEATURE;
+import static org.elasticsearch.xpack.core.security.SecurityField.FIELD_LEVEL_SECURITY_FEATURE;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.everyItem;
+import static org.hamcrest.Matchers.in;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+
+abstract class AbstractDocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
+
+    private static final Set<String> DLS_FLS_FEATURE_NAMES = Set.of(
+        DOCUMENT_LEVEL_SECURITY_FEATURE.getName(),
+        FIELD_LEVEL_SECURITY_FEATURE.getName()
+    );
+
+    protected static void assertOnlyDlsTracked() {
+        Set<String> features = fetchFeatureUsageFromAllNodes();
+        assertThat(DOCUMENT_LEVEL_SECURITY_FEATURE.getName(), is(in(features)));
+        assertThat(FIELD_LEVEL_SECURITY_FEATURE.getName(), not(is(in(features))));
+    }
+
+    protected static void assertOnlyFlsTracked() {
+        Set<String> features = fetchFeatureUsageFromAllNodes();
+        assertThat(FIELD_LEVEL_SECURITY_FEATURE.getName(), is(in(features)));
+        assertThat(DOCUMENT_LEVEL_SECURITY_FEATURE.getName(), not(is(in(features))));
+    }
+
+    protected static void assertDlsFlsTracked() {
+        assertThat(DLS_FLS_FEATURE_NAMES, everyItem(is(in(fetchFeatureUsageFromAllNodes()))));
+    }
+
+    protected static void assertDlsFlsNotTrackedAcrossAllNodes() {
+        assertThat(fetchFeatureUsageFromAllNodes(), not(containsInAnyOrder(DLS_FLS_FEATURE_NAMES)));
+    }
+
+    protected static void assertDlsFlsNotTrackedOnCoordOnlyNode() {
+        assertThat(fetchFeatureUsageFromCoordOnlyNode(), not(containsInAnyOrder(DLS_FLS_FEATURE_NAMES)));
+    }
+
+    protected static Set<String> fetchFeatureUsageFromAllNodes() {
+        final Set<String> result = new HashSet<>();
+        // Nodes are chosen at random when test is executed,
+        // hence we have to aggregate feature usage across all nodes in the cluster.
+        Set.of(internalCluster().getNodeNames()).stream().forEach(node -> { result.addAll(fetchFeatureUsageFromNode(client(node))); });
+        return result;
+    }
+
+    protected static Set<String> fetchFeatureUsageFromCoordOnlyNode() {
+        return fetchFeatureUsageFromNode(internalCluster().coordOnlyNodeClient());
+    }
+
+    protected static Set<String> fetchFeatureUsageFromNode(Client client) {
+        final Set<String> result = new HashSet<>();
+        PlainActionFuture<GetFeatureUsageResponse> listener = new PlainActionFuture<>();
+        client.execute(TransportGetFeatureUsageAction.TYPE, new GetFeatureUsageRequest(), listener);
+        GetFeatureUsageResponse response = listener.actionGet();
+        for (var feature : response.getFeatures()) {
+            result.add(feature.getName());
+        }
+        return result;
+    }
+}
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java
index 7d80212305e4..f57ecc461fed 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.cache.request.RequestCacheStats;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.mustache.MustachePlugin;
@@ -36,7 +37,6 @@
 import java.util.Base64;
 import java.util.Collection;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
@@ -97,7 +97,7 @@ protected String configUsers() {
 
     @Override
     protected String configRoles() {
-        return String.format(Locale.ROOT, """
+        return Strings.format("""
             %s%s:
               cluster: [ "manage_own_api_key" ]
               indices:
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java
index 0f4fe241d2b8..caa23c0377cf 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.sort.SortOrder;
@@ -29,7 +30,6 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
@@ -49,7 +49,7 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
     @Override
     protected String configUsers() {
         final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(USERS_PASSWD));
-        return super.configUsers() + String.format(Locale.ROOT, """
+        return super.configUsers() + Strings.format("""
             user1:%s
             user2:%s
             user3:%s
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityFeatureUsageTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityFeatureUsageTests.java
new file mode 100644
index 000000000000..de4679386042
--- /dev/null
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityFeatureUsageTests.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.integration;
+
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.SecuritySettingsSourceField;
+import org.elasticsearch.xpack.core.XPackSettings;
+
+import java.util.Collections;
+
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER;
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+
+@ESIntegTestCase.ClusterScope(numClientNodes = 1)
+public class DocumentLevelSecurityFeatureUsageTests extends AbstractDocumentAndFieldLevelSecurityTests {
+
+    protected static final SecureString USERS_PASSWD = SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING;
+
+    @Override
+    protected String configUsers() {
+        final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(USERS_PASSWD));
+        return super.configUsers() + Strings.format("""
+            user1:%s
+            user2:%s
+            """, usersPasswdHashed, usersPasswdHashed);
+    }
+
+    @Override
+    protected String configUsersRoles() {
+        return super.configUsersRoles() + """
+            role1:user1
+            role2:user1
+            role3:user2
+            """;
+    }
+
+    @Override
+    protected String configRoles() {
+        return super.configRoles() + """
+
+            role1:
+              cluster: [ none ]
+              indices:
+                - names: '*'
+                  privileges: [ none ]
+            role2:
+              cluster:
+                - all
+              indices:
+                - names: '*'
+                  privileges:
+                    - all
+                  query:
+                    term:
+                      field1: value1
+            role3:
+              cluster: [ all ]
+              indices:
+                - names: '*'
+                  privileges: [ ALL ]
+            """;
+    }
+
+    @Override
+    public Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal, otherSettings))
+            .put(XPackSettings.DLS_FLS_ENABLED.getKey(), true)
+            .put(XPackSettings.AUDIT_ENABLED.getKey(), false) // Just to make logs less noisy
+            .build();
+    }
+
+    public void testDlsFeatureUsageTracking() throws Exception {
+        assertAcked(
+            client().admin().indices().prepareCreate("test").setMapping("field1", "type=text", "field2", "type=text", "field3", "type=text")
+        );
+        client().prepareIndex("test").setId("1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get();
+        client().prepareIndex("test").setId("2").setSource("field2", "value2").setRefreshPolicy(IMMEDIATE).get();
+        client().prepareIndex("test").setId("3").setSource("field3", "value3").setRefreshPolicy(IMMEDIATE).get();
+
+        SearchResponse response = internalCluster().coordOnlyNodeClient()
+            .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+            .prepareSearch("test")
+            .setQuery(randomBoolean() ? QueryBuilders.termQuery("field1", "value1") : QueryBuilders.matchAllQuery())
+            .get();
+        assertHitCount(response, 1);
+        assertSearchHits(response, "1");
+
+        // coordinating only node should not tack DLS/FLS feature usage
+        assertDlsFlsNotTrackedOnCoordOnlyNode();
+        // only DLS feature should have been tracked across all data nodes
+        assertOnlyDlsTracked();
+    }
+
+    public void testDlsFlsFeatureUsageNotTracked() {
+        assertAcked(
+            client().admin().indices().prepareCreate("test").setMapping("id", "type=keyword", "field1", "type=text", "field2", "type=text")
+        );
+        client().prepareIndex("test").setId("1").setSource("id", "1", "field1", "value1").setRefreshPolicy(IMMEDIATE).get();
+        client().prepareIndex("test").setId("2").setSource("id", "2", "field2", "value2").setRefreshPolicy(IMMEDIATE).get();
+
+        // Running a search with user2 (which has role3 without DLS/FLS) should not trigger feature tracking.
+        SearchResponse response = internalCluster().coordOnlyNodeClient()
+            .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
+            .prepareSearch("test")
+            .get();
+        assertHitCount(response, 2);
+        assertSearchHits(response, "1", "2");
+
+        assertDlsFlsNotTrackedAcrossAllNodes();
+    }
+
+}
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
index ad586e3e83f0..19a940ca19e1 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
@@ -81,7 +81,6 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 
 import static java.util.stream.Collectors.toList;
@@ -125,7 +124,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     @Override
     protected String configUsers() {
         final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(USERS_PASSWD));
-        return super.configUsers() + String.format(Locale.ROOT, """
+        return super.configUsers() + Strings.format("""
             user1:%s
             user2:%s
             user3:%s
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityFeatureUsageTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityFeatureUsageTests.java
new file mode 100644
index 000000000000..de38722fffd3
--- /dev/null
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityFeatureUsageTests.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.integration;
+
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.SecuritySettingsSourceField;
+import org.elasticsearch.xpack.core.XPackSettings;
+
+import java.util.Collections;
+
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER;
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+
+@ESIntegTestCase.ClusterScope(numClientNodes = 1)
+public class FieldLevelSecurityFeatureUsageTests extends AbstractDocumentAndFieldLevelSecurityTests {
+
+    protected static final SecureString USERS_PASSWD = SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING;
+
+    @Override
+    protected String configUsers() {
+        final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(USERS_PASSWD));
+        return super.configUsers() + Strings.format("""
+            user1:%s
+            """, usersPasswdHashed);
+    }
+
+    @Override
+    protected String configUsersRoles() {
+        return super.configUsersRoles() + """
+            role1:user1
+            role2:user1
+            """;
+    }
+
+    @Override
+    protected String configRoles() {
+        return super.configRoles() + """
+
+            role1:
+              cluster: [ none ]
+              indices:
+                - names: '*'
+                  privileges: [ none ]
+            role2:
+              cluster: [ all ]
+              indices:
+                  - names: '*'
+                    privileges: [ ALL ]
+                    field_security:
+                       grant: [ field1 ]
+            """;
+    }
+
+    @Override
+    public Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal, otherSettings))
+            .put(XPackSettings.DLS_FLS_ENABLED.getKey(), true)
+            .put(XPackSettings.AUDIT_ENABLED.getKey(), false) // Just to make logs less noisy
+            .build();
+    }
+
+    public void testFlsFeatureUsageTracking() throws Exception {
+        assertAcked(client().admin().indices().prepareCreate("test").setMapping("field1", "type=text", "field2", "type=text"));
+        client().prepareIndex("test").setId("1").setSource("field1", "value1", "field2", "value1").setRefreshPolicy(IMMEDIATE).get();
+        client().prepareIndex("test").setId("2").setSource("field1", "value2", "field2", "value2").setRefreshPolicy(IMMEDIATE).get();
+
+        SearchResponse response = internalCluster().coordOnlyNodeClient()
+            .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+            .prepareSearch("test")
+            .setQuery(QueryBuilders.termQuery("field1", "value1"))
+            .get();
+        assertHitCount(response, 1);
+
+        // coordinating only node should not tack DLS/FLS feature usage
+        assertDlsFlsNotTrackedOnCoordOnlyNode();
+        // only FLS feature should have been tracked across all data nodes
+        assertOnlyFlsTracked();
+    }
+
+}
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java
index 042c8c7603b9..f05692d97a61 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.SecurityIntegTestCase;
@@ -21,7 +22,6 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 
@@ -92,7 +92,7 @@ protected String configRoles() {
             roleFields.append("          - ").append(field).append('\n');
         }
 
-        return String.format(Locale.ROOT, """
+        return Strings.format("""
             %s
             role1:
               cluster: [ none ]
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/IndexPrivilegeIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/IndexPrivilegeIntegTests.java
index 5e4be993d7cc..10ea09285b2e 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/IndexPrivilegeIntegTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/IndexPrivilegeIntegTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.SecuritySettingsSourceField;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
@@ -127,7 +128,7 @@ protected String configUsers() {
         final Hasher passwdHasher = getFastStoredHashAlgoForTests();
         final String usersPasswdHashed = new String(passwdHasher.hash(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING));
 
-        return super.configUsers() + formatted("""
+        return super.configUsers() + Strings.format("""
             admin:%1$s
             u1:%1$s
             u2:%1$s
@@ -660,26 +661,26 @@ private void assertUserExecutes(String user, String action, String index, boolea
                 if (userIsAllowed) {
                     assertAccessIsAllowed(user, "PUT", "/" + index + "/_doc/321", "{ \"foo\" : \"bar\" }");
                     // test auto mapping update is allowed but deprecated
-                    Response response = assertAccessIsAllowed(user, "PUT", "/" + index + "/_doc/4321", formatted("""
+                    Response response = assertAccessIsAllowed(user, "PUT", "/" + index + "/_doc/4321", Strings.format("""
                         { "%s" : "foo" }""", UUIDs.randomBase64UUID()));
                     String warningHeader = response.getHeader("Warning");
-                    assertThat(warningHeader, containsString(formatted("""
+                    assertThat(warningHeader, containsString(Strings.format("""
                         the index privilege [index] allowed the update mapping action [indices:admin/mapping/auto_put] on index [%s], \
                         this privilege will not permit mapping updates in the next major release - users who require access to update \
                         mappings must be granted explicit privileges""", index)));
                     assertAccessIsAllowed(user, "POST", "/" + index + "/_update/321", """
                         { "doc" : { "foo" : "baz" } }
                         """);
-                    response = assertAccessIsAllowed(user, "POST", "/" + index + "/_update/321", formatted("""
+                    response = assertAccessIsAllowed(user, "POST", "/" + index + "/_update/321", Strings.format("""
                         { "doc" : { "%s" : "baz" } }
                         """, UUIDs.randomBase64UUID()));
                     warningHeader = response.getHeader("Warning");
-                    assertThat(warningHeader, containsString(formatted("""
+                    assertThat(warningHeader, containsString(Strings.format("""
                         the index privilege [index] allowed the update mapping action [indices:admin/mapping/auto_put] on index [%s], \
                         this privilege will not permit mapping updates in the next major release - users who require access to update \
                         mappings must be granted explicit privileges\
                         """, index)));
-                    assertThat(warningHeader, containsString(formatted("""
+                    assertThat(warningHeader, containsString(Strings.format("""
                         the index privilege [index] allowed the update mapping action [indices:admin/mapping/auto_put] on index [%s], \
                         this privilege will not permit mapping updates in the next major release - users who require access to update \
                         mappings must be granted explicit privileges\
@@ -710,7 +711,7 @@ private void assertUserExecutes(String user, String action, String index, boolea
 
                     assertAccessIsAllowed(user, "PUT", "/" + index + "/_doc/321", "{ \"foo\" : \"bar\" }");
                     // test auto mapping update is allowed but deprecated
-                    Response response = assertAccessIsAllowed(user, "PUT", "/" + index + "/_doc/4321", formatted("""
+                    Response response = assertAccessIsAllowed(user, "PUT", "/" + index + "/_doc/4321", Strings.format("""
                         { "%s" : "foo" }""", UUIDs.randomBase64UUID()));
                     String warningHeader = response.getHeader("Warning");
                     assertThat(
@@ -725,7 +726,7 @@ private void assertUserExecutes(String user, String action, String index, boolea
                     assertAccessIsAllowed(user, "POST", "/" + index + "/_update/321", """
                         { "doc" : { "foo" : "baz" } }
                         """);
-                    response = assertAccessIsAllowed(user, "POST", "/" + index + "/_update/321", formatted("""
+                    response = assertAccessIsAllowed(user, "POST", "/" + index + "/_update/321", Strings.format("""
                         { "doc" : { "%s" : "baz" } }
                         """, UUIDs.randomBase64UUID()));
                     warningHeader = response.getHeader("Warning");
@@ -745,7 +746,7 @@ private void assertUserExecutes(String user, String action, String index, boolea
                 break;
 
             default:
-                fail(formatted("Unknown action %s to execute", action));
+                fail(Strings.format("Unknown action %s to execute", action));
         }
 
     }
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java
index b2c5d0345097..3372434c4534 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.datastreams.DataStreamsPlugin;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -55,7 +56,7 @@ protected String configUsersRoles() {
 
     @Override
     protected String configRoles() {
-        return formatted("""
+        return Strings.format("""
             %s
             role1:
               cluster: [ all ]
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/InternalUserAndRoleIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/InternalUserAndRoleIntegTests.java
index c934746e2b46..e678bc7865e8 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/InternalUserAndRoleIntegTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/InternalUserAndRoleIntegTests.java
@@ -23,7 +23,6 @@
 import org.junit.BeforeClass;
 
 import java.nio.file.Path;
-import java.util.Locale;
 
 public class InternalUserAndRoleIntegTests extends AbstractPrivilegeTestCase {
     private static final String[] INTERNAL_USERNAMES = new String[] {
@@ -53,7 +52,7 @@ protected String configRoles() {
     }
 
     private String defaultRole(String roleName) {
-        return formatted("""
+        return org.elasticsearch.core.Strings.format("""
             %s:
               cluster: [ none ]
               indices:
@@ -68,7 +67,7 @@ protected String configUsers() {
         final String usersPasswdHashed = new String(passwdHasher.hash(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING));
         StringBuilder builder = new StringBuilder(super.configUsers());
         for (String username : INTERNAL_USERNAMES) {
-            builder.append(String.format(Locale.ROOT, "%s:%s\n", username, usersPasswdHashed));
+            builder.append(org.elasticsearch.core.Strings.format("%s:%s\n", username, usersPasswdHashed));
         }
         return builder + "user:" + usersPasswdHashed + "\n";
     }
@@ -78,10 +77,10 @@ protected String configUsersRoles() {
         StringBuilder builder = new StringBuilder(super.configUsersRoles());
         // non-internal username maps to all internal role names
         for (String roleName : INTERNAL_ROLE_NAMES) {
-            builder.append(String.format(Locale.ROOT, "%s:%s\n", roleName, NON_INTERNAL_USERNAME));
+            builder.append(org.elasticsearch.core.Strings.format("%s:%s\n", roleName, NON_INTERNAL_USERNAME));
         }
         // all internal usernames are mapped to custom role
-        return builder + String.format(Locale.ROOT, "%s:%s\n", NON_INTERNAL_ROLE_NAME, Strings.join(INTERNAL_USERNAMES, ","));
+        return builder + org.elasticsearch.core.Strings.format("%s:%s\n", NON_INTERNAL_ROLE_NAME, Strings.join(INTERNAL_USERNAMES, ","));
     }
 
     private static Path repositoryLocation;
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java
index 3a6d1c88f39e..960d5157fac5 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.NativeRealmIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSourceField;
@@ -37,7 +38,7 @@ public class KibanaUserRoleIntegTests extends NativeRealmIntegTestCase {
 
     @Override
     public String configRoles() {
-        return formatted("""
+        return Strings.format("""
             %s
             my_kibana_user:
               indices:
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java
index 590df3c15749..dbbbc90d0f5d 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.SecurityIntegTestCase;
@@ -75,7 +76,7 @@ protected boolean addMockHttpTransport() {
     @Override
     protected String configRoles() {
         // The definition of TEST_ROLE here is intentionally different than the definition in the superclass.
-        return formatted("""
+        return Strings.format("""
             %s:
               cluster: [ all ]
               indices:
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java
index 5e6f63f04ff5..47603fb93662 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java
@@ -35,7 +35,6 @@
 import org.junit.After;
 
 import java.io.ByteArrayInputStream;
-import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -138,7 +137,7 @@ public class RoleMappingFileSettingsIT extends NativeRealmIntegTestCase {
         }""";
 
     @After
-    public void cleanUp() throws IOException {
+    public void cleanUp() {
         ClusterUpdateSettingsResponse settingsResponse = client().admin()
             .cluster()
             .prepareUpdateSettings()
@@ -164,7 +163,7 @@ private void writeJSONFile(String node, String json) throws Exception {
         Files.move(tempFilePath, fileSettingsService.operatorSettingsFile(), StandardCopyOption.ATOMIC_MOVE);
     }
 
-    private Tuple<CountDownLatch, AtomicLong> setupClusterStateListener(String node) {
+    private Tuple<CountDownLatch, AtomicLong> setupClusterStateListener(String node, String expectedKey) {
         ClusterService clusterService = internalCluster().clusterService(node);
         CountDownLatch savedClusterState = new CountDownLatch(1);
         AtomicLong metadataVersion = new AtomicLong(-1);
@@ -174,7 +173,7 @@ public void clusterChanged(ClusterChangedEvent event) {
                 ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
                 if (reservedState != null) {
                     ReservedStateHandlerMetadata handlerMetadata = reservedState.handlers().get(ReservedRoleMappingAction.NAME);
-                    if (handlerMetadata != null && handlerMetadata.keys().contains("everyone_kibana")) {
+                    if (handlerMetadata != null && handlerMetadata.keys().contains(expectedKey)) {
                         clusterService.removeListener(this);
                         metadataVersion.set(event.state().metadata().version());
                         savedClusterState.countDown();
@@ -280,7 +279,7 @@ private void assertRoleMappingsSaveOK(CountDownLatch savedClusterState, AtomicLo
     public void testRoleMappingsApplied() throws Exception {
         ensureGreen();
 
-        var savedClusterState = setupClusterStateListener(internalCluster().getMasterName());
+        var savedClusterState = setupClusterStateListener(internalCluster().getMasterName(), "everyone_kibana");
         writeJSONFile(internalCluster().getMasterName(), testJSON);
 
         assertRoleMappingsSaveOK(savedClusterState.v1(), savedClusterState.v2());
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityClearScrollTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityClearScrollTests.java
index fe1c13653c25..83cda0e51851 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityClearScrollTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityClearScrollTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.action.search.MultiSearchRequestBuilder;
 import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSourceField;
 import org.elasticsearch.xcontent.XContentType;
@@ -51,7 +52,7 @@ protected String configUsersRoles() {
 
     @Override
     protected String configRoles() {
-        return formatted("""
+        return Strings.format("""
             %s
             allowed_role:
               cluster:
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureStateIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureStateIntegTests.java
index b8b7a293a377..670ff9c4cfe7 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureStateIntegTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/SecurityFeatureStateIntegTests.java
@@ -17,13 +17,13 @@
 import org.elasticsearch.cluster.SnapshotsInProgress;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.SecuritySettingsSourceField;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 import java.nio.file.Path;
-import java.util.Locale;
 
 import static org.elasticsearch.test.SecuritySettingsSource.ES_TEST_ROOT_USER;
 import static org.hamcrest.Matchers.containsString;
@@ -93,7 +93,7 @@ public void testSecurityFeatureStateSnapshotAndRestore() throws Exception {
         // create a test user
         final Request createUserRequest = new Request("PUT", "/_security/user/" + LOCAL_TEST_USER_NAME);
         createUserRequest.addParameter("refresh", "wait_for");
-        createUserRequest.setJsonEntity(String.format(Locale.ROOT, """
+        createUserRequest.setJsonEntity(Strings.format("""
             {  "password": "%s",  "roles": [ "%s" ]}
             """, LOCAL_TEST_USER_PASSWORD, roleName));
         performSuperuserRequest(createUserRequest);
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/license/LicensingTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/license/LicensingTests.java
index 2072ab0d99fa..be9c62219c1f 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/license/LicensingTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/license/LicensingTests.java
@@ -71,7 +71,7 @@ public class LicensingTests extends SecurityIntegTestCase {
 
     private static final SecureString HASH_PASSWD = new SecureString(Hasher.BCRYPT4.hash(new SecureString("passwd".toCharArray())));
 
-    private static final String ROLES = formatted("""
+    private static final String ROLES = Strings.format("""
         %s:
           cluster: [ all ]
           indices:
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsRestartIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsRestartIT.java
new file mode 100644
index 000000000000..5ef6a2d2c97b
--- /dev/null
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsRestartIT.java
@@ -0,0 +1,154 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata;
+import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.reservedstate.service.FileSettingsService;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.SecurityIntegTestCase;
+import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsAction;
+import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsRequest;
+import org.elasticsearch.xpack.security.action.rolemapping.ReservedRoleMappingAction;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.Arrays;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.notNullValue;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
+public class FileSettingsRoleMappingsRestartIT extends SecurityIntegTestCase {
+
+    private static AtomicLong versionCounter = new AtomicLong(1);
+
+    private static String testJSONOnlyRoleMappings = """
+        {
+             "metadata": {
+                 "version": "%s",
+                 "compatibility": "8.4.0"
+             },
+             "state": {
+                 "role_mappings": {
+                       "everyone_kibana_alone": {
+                          "enabled": true,
+                          "roles": [ "kibana_user" ],
+                          "rules": { "field": { "username": "*" } },
+                          "metadata": {
+                             "uuid" : "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7",
+                             "_foo": "something"
+                          }
+                       },
+                       "everyone_fleet_alone": {
+                          "enabled": true,
+                          "roles": [ "fleet_user" ],
+                          "rules": { "field": { "username": "*" } },
+                          "metadata": {
+                             "uuid" : "b9a59ba9-6b92-4be3-bb8d-02bb270cb3a7",
+                             "_foo": "something_else"
+                          }
+                       }
+                 }
+             }
+        }""";
+
+    private void writeJSONFile(String node, String json) throws Exception {
+        long version = versionCounter.incrementAndGet();
+
+        FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, node);
+
+        Files.deleteIfExists(fileSettingsService.operatorSettingsFile());
+
+        Files.createDirectories(fileSettingsService.operatorSettingsDir());
+        Path tempFilePath = createTempFile();
+
+        logger.info("--> writing JSON config to node {} with path {}", node, tempFilePath);
+        logger.info(Strings.format(json, version));
+        Files.write(tempFilePath, Strings.format(json, version).getBytes(StandardCharsets.UTF_8));
+        Files.move(tempFilePath, fileSettingsService.operatorSettingsFile(), StandardCopyOption.ATOMIC_MOVE);
+    }
+
+    private Tuple<CountDownLatch, AtomicLong> setupClusterStateListener(String node, String expectedKey) {
+        ClusterService clusterService = internalCluster().clusterService(node);
+        CountDownLatch savedClusterState = new CountDownLatch(1);
+        AtomicLong metadataVersion = new AtomicLong(-1);
+        clusterService.addListener(new ClusterStateListener() {
+            @Override
+            public void clusterChanged(ClusterChangedEvent event) {
+                ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
+                if (reservedState != null) {
+                    ReservedStateHandlerMetadata handlerMetadata = reservedState.handlers().get(ReservedRoleMappingAction.NAME);
+                    if (handlerMetadata != null && handlerMetadata.keys().contains(expectedKey)) {
+                        clusterService.removeListener(this);
+                        metadataVersion.set(event.state().metadata().version());
+                        savedClusterState.countDown();
+                    }
+                }
+            }
+        });
+
+        return new Tuple<>(savedClusterState, metadataVersion);
+    }
+
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/93048")
+    public void testReservedStatePersistsOnRestart() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+
+        final String masterNode = internalCluster().getMasterName();
+        var savedClusterState = setupClusterStateListener(masterNode, "everyone_kibana_alone");
+
+        FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
+
+        assertTrue(masterFileSettingsService.watching());
+
+        logger.info("--> write some role mappings, no other file settings");
+        writeJSONFile(masterNode, testJSONOnlyRoleMappings);
+        boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
+        assertTrue(awaitSuccessful);
+
+        logger.info("--> restart master");
+        internalCluster().restartNode(masterNode);
+
+        ensureGreen();
+
+        var clusterStateResponse = client().admin().cluster().state(new ClusterStateRequest()).actionGet();
+        assertThat(
+            clusterStateResponse.getState()
+                .metadata()
+                .reservedStateMetadata()
+                .get(FileSettingsService.NAMESPACE)
+                .handlers()
+                .get(ReservedRoleMappingAction.NAME)
+                .keys(),
+            containsInAnyOrder("everyone_fleet_alone", "everyone_kibana_alone")
+        );
+
+        var request = new GetRoleMappingsRequest();
+        request.setNames("everyone_kibana_alone", "everyone_fleet_alone");
+        var response = client().execute(GetRoleMappingsAction.INSTANCE, request).get();
+        assertTrue(response.hasMappings());
+        assertThat(
+            Arrays.stream(response.mappings()).map(r -> r.getName()).collect(Collectors.toSet()),
+            allOf(notNullValue(), containsInAnyOrder("everyone_kibana_alone", "everyone_fleet_alone"))
+        );
+    }
+}
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsStartupIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsStartupIT.java
new file mode 100644
index 000000000000..cf62dccaadf2
--- /dev/null
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsStartupIT.java
@@ -0,0 +1,140 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.metadata.ReservedStateErrorMetadata;
+import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.reindex.ReindexPlugin;
+import org.elasticsearch.reservedstate.service.FileSettingsService;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
+import org.elasticsearch.test.SecurityIntegTestCase;
+import org.elasticsearch.transport.netty4.Netty4Plugin;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.notNullValue;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
+public class FileSettingsRoleMappingsStartupIT extends SecurityIntegTestCase {
+
+    private static AtomicLong versionCounter = new AtomicLong(1);
+    private static String testJSONForFailedCase = """
+        {
+             "metadata": {
+                 "version": "%s",
+                 "compatibility": "8.4.0"
+             },
+             "state": {
+                 "role_mappings": {
+                       "everyone_kibana_2": {
+                          "enabled": true,
+                          "roles": [ "kibana_user" ],
+                          "rules": { "field": { "username": "*" } },
+                          "metadata": {
+                             "uuid" : "b9a59ba9-6b92-4be2-bb8d-02bb270cb3a7",
+                             "_foo": "something"
+                          }
+                       }
+                 }
+             }
+        }""";
+
+    @Override
+    protected void doAssertXPackIsInstalled() {}
+
+    @Override
+    protected Path nodeConfigPath(int nodeOrdinal) {
+        return null;
+    }
+
+    private void writeJSONFile(String node, String json) throws Exception {
+        long version = versionCounter.incrementAndGet();
+
+        FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, node);
+
+        Files.deleteIfExists(fileSettingsService.operatorSettingsFile());
+
+        Files.createDirectories(fileSettingsService.operatorSettingsDir());
+        Path tempFilePath = createTempFile();
+
+        logger.info("--> writing JSON config to node {} with path {}", node, tempFilePath);
+        logger.info(Strings.format(json, version));
+        Files.write(tempFilePath, Strings.format(json, version).getBytes(StandardCharsets.UTF_8));
+        Files.move(tempFilePath, fileSettingsService.operatorSettingsFile(), StandardCopyOption.ATOMIC_MOVE);
+    }
+
+    private Tuple<CountDownLatch, AtomicLong> setupClusterStateListenerForError(String node) {
+        ClusterService clusterService = internalCluster().clusterService(node);
+        CountDownLatch savedClusterState = new CountDownLatch(1);
+        AtomicLong metadataVersion = new AtomicLong(-1);
+        clusterService.addListener(new ClusterStateListener() {
+            @Override
+            public void clusterChanged(ClusterChangedEvent event) {
+                ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
+                if (reservedState != null && reservedState.errorMetadata() != null) {
+                    assertEquals(ReservedStateErrorMetadata.ErrorKind.VALIDATION, reservedState.errorMetadata().errorKind());
+                    assertThat(reservedState.errorMetadata().errors(), allOf(notNullValue(), hasSize(1)));
+                    assertThat(reservedState.errorMetadata().errors().get(0), containsString("Fake exception"));
+                    clusterService.removeListener(this);
+                    metadataVersion.set(event.state().metadata().version());
+                    savedClusterState.countDown();
+                }
+            }
+        });
+
+        return new Tuple<>(savedClusterState, metadataVersion);
+    }
+
+    public void testFailsOnStartMasterNodeWithError() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+
+        internalCluster().startMasterOnlyNode();
+        logger.info("--> write some role mappings, no other file settings");
+        writeJSONFile(internalCluster().getMasterName(), testJSONForFailedCase);
+        var savedClusterState = setupClusterStateListenerForError(internalCluster().getMasterName());
+
+        boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
+        assertTrue(awaitSuccessful);
+    }
+
+    public Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(
+            UnstableLocalStateSecurity.class,
+            Netty4Plugin.class,
+            ReindexPlugin.class,
+            CommonAnalysisPlugin.class,
+            InternalSettingsPlugin.class,
+            MapperExtrasPlugin.class
+        );
+    }
+
+    @Override
+    protected boolean addMockTransportService() {
+        return false; // security has its own transport service
+    }
+}
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java
index ad3e2c691429..79f674014f68 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java
@@ -96,6 +96,7 @@
 import org.elasticsearch.xpack.security.transport.filter.IPFilter;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -149,6 +150,7 @@
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 public class ApiKeyIntegTests extends SecurityIntegTestCase {
@@ -162,6 +164,13 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase {
         null
     );
 
+    private static long deleteRetentionPeriodDays;
+
+    @BeforeClass
+    public static void randomDeleteRetentionPeriod() {
+        deleteRetentionPeriodDays = randomLongBetween(0, 7);
+    }
+
     @Override
     public Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
         return Settings.builder()
@@ -170,6 +179,7 @@ public Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
             .put(XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey(), true)
             .put(ApiKeyService.DELETE_INTERVAL.getKey(), TimeValue.timeValueMillis(DELETE_INTERVAL_MILLIS))
             .put(ApiKeyService.DELETE_TIMEOUT.getKey(), TimeValue.timeValueSeconds(5L))
+            .put(ApiKeyService.DELETE_RETENTION_PERIOD.getKey(), TimeValue.timeValueDays(deleteRetentionPeriodDays))
             .put("xpack.security.crypto.thread_pool.queue_size", CRYPTO_THREAD_POOL_QUEUE_SIZE)
             .build();
     }
@@ -459,12 +469,48 @@ private void verifyInvalidateResponse(
         assertThat(invalidateResponse.getErrors().size(), equalTo(0));
     }
 
-    public void testInvalidatedApiKeysDeletedByRemover() throws Exception {
+    public void testApiKeyRemover() throws Exception {
+        final String namePrefix = randomAlphaOfLength(10);
+        try {
+            if (deleteRetentionPeriodDays == 0) {
+                doTestInvalidKeysImmediatelyDeletedByRemover(namePrefix);
+                // Change the setting dynamically and test the other behaviour
+                deleteRetentionPeriodDays = randomIntBetween(1, 7);
+                setRetentionPeriod(false);
+                doTestDeletionBehaviorWhenKeysBecomeInvalidBeforeAndAfterRetentionPeriod("not-" + namePrefix);
+            } else {
+                doTestDeletionBehaviorWhenKeysBecomeInvalidBeforeAndAfterRetentionPeriod(namePrefix);
+                // Change the setting dynamically and test the other behaviour
+                deleteRetentionPeriodDays = 0;
+                setRetentionPeriod(false);
+                doTestInvalidKeysImmediatelyDeletedByRemover("not-" + namePrefix);
+            }
+        } finally {
+            setRetentionPeriod(true);
+        }
+    }
+
+    private void setRetentionPeriod(boolean clear) {
+        final Settings.Builder builder = Settings.builder();
+        if (clear) {
+            builder.putNull(ApiKeyService.DELETE_RETENTION_PERIOD.getKey());
+        } else {
+            builder.put(ApiKeyService.DELETE_RETENTION_PERIOD.getKey(), TimeValue.timeValueDays(deleteRetentionPeriodDays));
+        }
+        client().admin().cluster().prepareUpdateSettings().setPersistentSettings(builder).get();
+    }
+
+    private void doTestInvalidKeysImmediatelyDeletedByRemover(String namePrefix) throws Exception {
+        assertThat(deleteRetentionPeriodDays, equalTo(0L));
         Client client = waitForExpiredApiKeysRemoverTriggerReadyAndGetClient().filterWithHeader(
             Collections.singletonMap("Authorization", basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING))
         );
 
-        List<CreateApiKeyResponse> createdApiKeys = createApiKeys(2, null).v1();
+        // Create a very short-lived key (1ms expiration)
+        createApiKeys(1, TimeValue.timeValueMillis(1));
+        // Create keys that will not expire during this test
+        final CreateApiKeyResponse nonExpiringKey = createApiKeys(1, namePrefix, TimeValue.timeValueDays(1)).v1().get(0);
+        List<CreateApiKeyResponse> createdApiKeys = createApiKeys(2, namePrefix, randomBoolean() ? TimeValue.timeValueDays(1) : null).v1();
 
         PlainActionFuture<InvalidateApiKeyResponse> listener = new PlainActionFuture<>();
         client.execute(
@@ -481,13 +527,21 @@ public void testInvalidatedApiKeysDeletedByRemover() throws Exception {
         refreshSecurityIndex();
 
         PlainActionFuture<GetApiKeyResponse> getApiKeyResponseListener = new PlainActionFuture<>();
-        client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.builder().realmName("file").build(), getApiKeyResponseListener);
-        Set<String> expectedKeyIds = Sets.newHashSet(createdApiKeys.get(0).getId(), createdApiKeys.get(1).getId());
+        client.execute(
+            GetApiKeyAction.INSTANCE,
+            GetApiKeyRequest.builder().apiKeyName(namePrefix + "*").build(),
+            getApiKeyResponseListener
+        );
+        // The first API key with 1ms expiration should already be deleted
+        Set<String> expectedKeyIds = Sets.newHashSet(nonExpiringKey.getId(), createdApiKeys.get(0).getId(), createdApiKeys.get(1).getId());
         boolean apiKeyInvalidatedButNotYetDeletedByExpiredApiKeysRemover = false;
         for (ApiKey apiKey : getApiKeyResponseListener.get().getApiKeyInfos()) {
             assertThat(apiKey.getId(), is(in(expectedKeyIds)));
-            if (apiKey.getId().equals(createdApiKeys.get(0).getId())) {
-                // has been invalidated but not yet deleted by ExpiredApiKeysRemover
+            if (apiKey.getId().equals(nonExpiringKey.getId())) {
+                assertThat(apiKey.isInvalidated(), is(false));
+                assertThat(apiKey.getExpiration(), notNullValue());
+            } else if (apiKey.getId().equals(createdApiKeys.get(0).getId())) {
+                // has been invalidated but not yet deleted by InactiveApiKeysRemover
                 assertThat(apiKey.isInvalidated(), is(true));
                 apiKeyInvalidatedButNotYetDeletedByExpiredApiKeysRemover = true;
             } else if (apiKey.getId().equals(createdApiKeys.get(1).getId())) {
@@ -497,7 +551,7 @@ public void testInvalidatedApiKeysDeletedByRemover() throws Exception {
         }
         assertThat(
             getApiKeyResponseListener.get().getApiKeyInfos().length,
-            is((apiKeyInvalidatedButNotYetDeletedByExpiredApiKeysRemover) ? 2 : 1)
+            is((apiKeyInvalidatedButNotYetDeletedByExpiredApiKeysRemover) ? 3 : 2)
         );
 
         client = waitForExpiredApiKeysRemoverTriggerReadyAndGetClient().filterWithHeader(
@@ -517,22 +571,29 @@ public void testInvalidatedApiKeysDeletedByRemover() throws Exception {
         refreshSecurityIndex();
 
         // Verify that 1st invalidated API key is deleted whereas the next one may be or may not be as it depends on whether update was
-        // indexed before ExpiredApiKeysRemover ran
+        // indexed before InactiveApiKeysRemover ran
         getApiKeyResponseListener = new PlainActionFuture<>();
-        client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.builder().realmName("file").build(), getApiKeyResponseListener);
-        expectedKeyIds = Sets.newHashSet(createdApiKeys.get(1).getId());
+        client.execute(
+            GetApiKeyAction.INSTANCE,
+            GetApiKeyRequest.builder().apiKeyName(namePrefix + "*").build(),
+            getApiKeyResponseListener
+        );
+        expectedKeyIds = Sets.newHashSet(nonExpiringKey.getId(), createdApiKeys.get(1).getId());
         apiKeyInvalidatedButNotYetDeletedByExpiredApiKeysRemover = false;
         for (ApiKey apiKey : getApiKeyResponseListener.get().getApiKeyInfos()) {
             assertThat(apiKey.getId(), is(in(expectedKeyIds)));
-            if (apiKey.getId().equals(createdApiKeys.get(1).getId())) {
-                // has been invalidated but not yet deleted by ExpiredApiKeysRemover
+            if (apiKey.getId().equals(nonExpiringKey.getId())) {
+                assertThat(apiKey.isInvalidated(), is(false));
+                assertThat(apiKey.getExpiration(), notNullValue());
+            } else if (apiKey.getId().equals(createdApiKeys.get(1).getId())) {
+                // has been invalidated but not yet deleted by InactiveApiKeysRemover
                 assertThat(apiKey.isInvalidated(), is(true));
                 apiKeyInvalidatedButNotYetDeletedByExpiredApiKeysRemover = true;
             }
         }
         assertThat(
             getApiKeyResponseListener.get().getApiKeyInfos().length,
-            is((apiKeyInvalidatedButNotYetDeletedByExpiredApiKeysRemover) ? 1 : 0)
+            is((apiKeyInvalidatedButNotYetDeletedByExpiredApiKeysRemover) ? 2 : 1)
         );
     }
 
@@ -554,44 +615,99 @@ private Client waitForExpiredApiKeysRemoverTriggerReadyAndGetClient() throws Exc
         return internalCluster().client(nodeWithMostRecentRun);
     }
 
-    public void testExpiredApiKeysBehaviorWhenKeysExpired1WeekBeforeAnd1DayBefore() throws Exception {
+    private void doTestDeletionBehaviorWhenKeysBecomeInvalidBeforeAndAfterRetentionPeriod(String namePrefix) throws Exception {
+        assertThat(deleteRetentionPeriodDays, greaterThan(0L));
         Client client = waitForExpiredApiKeysRemoverTriggerReadyAndGetClient().filterWithHeader(
             Collections.singletonMap("Authorization", basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING))
         );
 
-        int noOfKeys = 4;
-        List<CreateApiKeyResponse> createdApiKeys = createApiKeys(noOfKeys, null).v1();
+        int noOfKeys = 9;
+        List<CreateApiKeyResponse> createdApiKeys = createApiKeys(noOfKeys, namePrefix, null).v1();
         Instant created = Instant.now();
 
         PlainActionFuture<GetApiKeyResponse> getApiKeyResponseListener = new PlainActionFuture<>();
-        client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.builder().realmName("file").build(), getApiKeyResponseListener);
+        client.execute(
+            GetApiKeyAction.INSTANCE,
+            GetApiKeyRequest.builder().apiKeyName(namePrefix + "*").build(),
+            getApiKeyResponseListener
+        );
         assertThat(getApiKeyResponseListener.get().getApiKeyInfos().length, is(noOfKeys));
 
         // Expire the 1st key such that it cannot be deleted by the remover
-        // hack doc to modify the expiration time to a day before
-        Instant dayBefore = created.minus(1L, ChronoUnit.DAYS);
-        assertTrue(Instant.now().isAfter(dayBefore));
+        // hack doc to modify the expiration time
+        Instant withinRetention = created.minus(deleteRetentionPeriodDays - 1, ChronoUnit.DAYS);
+        assertFalse(created.isBefore(withinRetention));
         UpdateResponse expirationDateUpdatedResponse = client.prepareUpdate(SECURITY_MAIN_ALIAS, createdApiKeys.get(0).getId())
-            .setDoc("expiration_time", dayBefore.toEpochMilli())
+            .setDoc("expiration_time", withinRetention.toEpochMilli())
             .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
             .get();
         assertThat(expirationDateUpdatedResponse.getResult(), is(DocWriteResponse.Result.UPDATED));
 
         // Expire the 2nd key such that it can be deleted by the remover
-        // hack doc to modify the expiration time to the week before
-        Instant weekBefore = created.minus(8L, ChronoUnit.DAYS);
-        assertTrue(Instant.now().isAfter(weekBefore));
+        // hack doc to modify the expiration time
+        Instant outsideRetention = created.minus(deleteRetentionPeriodDays + 1, ChronoUnit.DAYS);
+        assertTrue(Instant.now().isAfter(outsideRetention));
         expirationDateUpdatedResponse = client.prepareUpdate(SECURITY_MAIN_ALIAS, createdApiKeys.get(1).getId())
-            .setDoc("expiration_time", weekBefore.toEpochMilli())
+            .setDoc("expiration_time", outsideRetention.toEpochMilli())
             .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
             .get();
         assertThat(expirationDateUpdatedResponse.getResult(), is(DocWriteResponse.Result.UPDATED));
 
+        // Invalidate the 3rd key such that it cannot be deleted by the remover
+        UpdateResponse invalidateUpdateResponse = client.prepareUpdate(SECURITY_MAIN_ALIAS, createdApiKeys.get(2).getId())
+            .setDoc("invalidation_time", withinRetention.toEpochMilli(), "api_key_invalidated", true)
+            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .get();
+        assertThat(invalidateUpdateResponse.getResult(), is(DocWriteResponse.Result.UPDATED));
+
+        // Invalidate the 4th key such that it will be deleted by the remover
+        invalidateUpdateResponse = client.prepareUpdate(SECURITY_MAIN_ALIAS, createdApiKeys.get(3).getId())
+            .setDoc("invalidation_time", outsideRetention.toEpochMilli(), "api_key_invalidated", true)
+            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .get();
+        assertThat(invalidateUpdateResponse.getResult(), is(DocWriteResponse.Result.UPDATED));
+
+        // 5th key will be deleted because its expiration is outside of retention even though its invalidation time is not
+        UpdateResponse updateResponse = client.prepareUpdate(SECURITY_MAIN_ALIAS, createdApiKeys.get(4).getId())
+            .setDoc(
+                "expiration_time",
+                outsideRetention.toEpochMilli(),
+                "invalidation_time",
+                withinRetention.toEpochMilli(),
+                "api_key_invalidated",
+                true
+            )
+            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .get();
+        assertThat(updateResponse.getResult(), is(DocWriteResponse.Result.UPDATED));
+
+        // 6th key will be deleted because its invalidation time is outside of retention even though its expiration is not
+        updateResponse = client.prepareUpdate(SECURITY_MAIN_ALIAS, createdApiKeys.get(5).getId())
+            .setDoc(
+                "expiration_time",
+                withinRetention.toEpochMilli(),
+                "invalidation_time",
+                outsideRetention.toEpochMilli(),
+                "api_key_invalidated",
+                true
+            )
+            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .get();
+        assertThat(updateResponse.getResult(), is(DocWriteResponse.Result.UPDATED));
+
+        // 7th key will be deleted because it has old style invalidation (no invalidation time)
+        // It does not matter whether it has an expiration time or whether the expiration time is still within retention period
+        updateResponse = client.prepareUpdate(SECURITY_MAIN_ALIAS, createdApiKeys.get(6).getId())
+            .setDoc("api_key_invalidated", true, "expiration_time", randomBoolean() ? withinRetention.toEpochMilli() : null)
+            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .get();
+        assertThat(updateResponse.getResult(), is(DocWriteResponse.Result.UPDATED));
+
         // Invalidate to trigger the remover
         PlainActionFuture<InvalidateApiKeyResponse> listener = new PlainActionFuture<>();
         client.execute(
             InvalidateApiKeyAction.INSTANCE,
-            InvalidateApiKeyRequest.usingApiKeyId(createdApiKeys.get(2).getId(), false),
+            InvalidateApiKeyRequest.usingApiKeyId(createdApiKeys.get(7).getId(), false),
             listener
         );
         assertThat(listener.get().getInvalidatedApiKeys().size(), is(1));
@@ -600,16 +716,20 @@ public void testExpiredApiKeysBehaviorWhenKeysExpired1WeekBeforeAnd1DayBefore()
 
         refreshSecurityIndex();
 
-        // Verify get API keys does not return api keys deleted by ExpiredApiKeysRemover
+        // Verify get API keys does not return api keys deleted by InactiveApiKeysRemover
         getApiKeyResponseListener = new PlainActionFuture<>();
-        client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.builder().realmName("file").build(), getApiKeyResponseListener);
+        client.execute(
+            GetApiKeyAction.INSTANCE,
+            GetApiKeyRequest.builder().apiKeyName(namePrefix + "*").build(),
+            getApiKeyResponseListener
+        );
 
         Set<String> expectedKeyIds = Sets.newHashSet(
             createdApiKeys.get(0).getId(),
             createdApiKeys.get(2).getId(),
-            createdApiKeys.get(3).getId()
+            createdApiKeys.get(7).getId(),
+            createdApiKeys.get(8).getId()
         );
-        boolean apiKeyInvalidatedButNotYetDeletedByExpiredApiKeysRemover = false;
         for (ApiKey apiKey : getApiKeyResponseListener.get().getApiKeyInfos()) {
             assertThat(apiKey.getId(), is(in(expectedKeyIds)));
             if (apiKey.getId().equals(createdApiKeys.get(0).getId())) {
@@ -617,11 +737,14 @@ public void testExpiredApiKeysBehaviorWhenKeysExpired1WeekBeforeAnd1DayBefore()
                 assertTrue(apiKey.getExpiration().isBefore(Instant.now()));
                 assertThat(apiKey.isInvalidated(), is(false));
             } else if (apiKey.getId().equals(createdApiKeys.get(2).getId())) {
-                // has not been expired as no expiration, is invalidated but not yet deleted by ExpiredApiKeysRemover
+                // has been invalidated, not expired
+                assertThat(apiKey.getExpiration(), nullValue());
+                assertThat(apiKey.isInvalidated(), is(true));
+            } else if (apiKey.getId().equals(createdApiKeys.get(7).getId())) {
+                // has not been expired as no expiration, is invalidated but not yet deleted by InactiveApiKeysRemover
                 assertThat(apiKey.getExpiration(), is(nullValue()));
                 assertThat(apiKey.isInvalidated(), is(true));
-                apiKeyInvalidatedButNotYetDeletedByExpiredApiKeysRemover = true;
-            } else if (apiKey.getId().equals(createdApiKeys.get(3).getId())) {
+            } else if (apiKey.getId().equals(createdApiKeys.get(8).getId())) {
                 // has not been expired as no expiration, not invalidated
                 assertThat(apiKey.getExpiration(), is(nullValue()));
                 assertThat(apiKey.isInvalidated(), is(false));
@@ -629,10 +752,7 @@ public void testExpiredApiKeysBehaviorWhenKeysExpired1WeekBeforeAnd1DayBefore()
                 fail("unexpected API key " + apiKey);
             }
         }
-        assertThat(
-            getApiKeyResponseListener.get().getApiKeyInfos().length,
-            is((apiKeyInvalidatedButNotYetDeletedByExpiredApiKeysRemover) ? 3 : 2)
-        );
+        assertThat(getApiKeyResponseListener.get().getApiKeyInfos().length, is(4));
     }
 
     private void refreshSecurityIndex() throws Exception {
@@ -2894,6 +3014,18 @@ private Tuple<List<CreateApiKeyResponse>, List<Map<String, Object>>> createApiKe
         return createApiKeys(ES_TEST_ROOT_USER, noOfApiKeys, expiration, DEFAULT_API_KEY_ROLE_DESCRIPTOR.getClusterPrivileges());
     }
 
+    private Tuple<List<CreateApiKeyResponse>, List<Map<String, Object>>> createApiKeys(
+        int noOfApiKeys,
+        String namePrefix,
+        TimeValue expiration
+    ) {
+        final Map<String, String> headers = Collections.singletonMap(
+            "Authorization",
+            basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING)
+        );
+        return createApiKeys(headers, noOfApiKeys, namePrefix, expiration, DEFAULT_API_KEY_ROLE_DESCRIPTOR.getClusterPrivileges());
+    }
+
     private Tuple<List<CreateApiKeyResponse>, List<Map<String, Object>>> createApiKeys(
         String user,
         int noOfApiKeys,
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java
index 16ef1853adee..d185683fa3eb 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.security.authc;
 
 import org.apache.http.client.methods.HttpPost;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.WriteRequest;
@@ -19,6 +19,7 @@
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.SecurityIntegTestCase;
@@ -44,7 +45,6 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -150,7 +150,7 @@ public void testExpiredTokensDeletedAfterExpiration() throws Exception {
         Request updateRequest = new Request(HttpPost.METHOD_NAME, SecuritySystemIndices.SECURITY_TOKENS_ALIAS + "/_update/" + docId.get());
         updateRequest.addParameter("refresh", WriteRequest.RefreshPolicy.IMMEDIATE.getValue());
         updateRequest.setOptions(SECURITY_REQUEST_OPTIONS);
-        updateRequest.setJsonEntity(formatted("""
+        updateRequest.setJsonEntity(Strings.format("""
             {
               "doc": {
                 "creation_time": %s
@@ -309,7 +309,10 @@ public void testInvalidateMultipleTimes() throws IOException {
 
     public void testInvalidateNotValidAccessTokens() throws Exception {
         // Perform a request to invalidate a token, before the tokens index is created
-        ResponseException e = expectThrows(ResponseException.class, () -> invalidateAccessToken(generateAccessToken(Version.CURRENT)));
+        ResponseException e = expectThrows(
+            ResponseException.class,
+            () -> invalidateAccessToken(generateAccessToken(TransportVersion.CURRENT))
+        );
         assertThat(e.getResponse(), hasStatusCode(RestStatus.BAD_REQUEST));
         // Create a token to trigger index creation
         createToken(TEST_USER_NAME, SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING);
@@ -323,14 +326,14 @@ public void testInvalidateNotValidAccessTokens() throws Exception {
         assertThat(invalidateResponse.previouslyInvalidated(), equalTo(0));
         assertThat(invalidateResponse.errors(), empty());
 
-        invalidateResponse = invalidateAccessToken(generateInvalidShortAccessToken(Version.CURRENT));
+        invalidateResponse = invalidateAccessToken(generateInvalidShortAccessToken(TransportVersion.CURRENT));
         assertThat(invalidateResponse.invalidated(), equalTo(0));
         assertThat(invalidateResponse.previouslyInvalidated(), equalTo(0));
         assertThat(invalidateResponse.errors(), empty());
 
         // Generate a token that could be a valid token string for the version we are on, and should decode fine, but is not found in our
         // tokens index
-        invalidateResponse = invalidateAccessToken(generateAccessToken(Version.CURRENT));
+        invalidateResponse = invalidateAccessToken(generateAccessToken(TransportVersion.CURRENT));
         assertThat(invalidateResponse.invalidated(), equalTo(0));
         assertThat(invalidateResponse.previouslyInvalidated(), equalTo(0));
         assertThat(invalidateResponse.errors(), empty());
@@ -340,7 +343,9 @@ public void testInvalidateNotValidRefreshTokens() throws Exception {
         // Perform a request to invalidate a refresh token, before the tokens index is created
         ResponseException e = expectThrows(
             ResponseException.class,
-            () -> invalidateRefreshToken(TokenService.prependVersionAndEncodeRefreshToken(Version.CURRENT, UUIDs.randomBase64UUID()))
+            () -> invalidateRefreshToken(
+                TokenService.prependVersionAndEncodeRefreshToken(TransportVersion.CURRENT, UUIDs.randomBase64UUID())
+            )
         );
         assertThat(e.getResponse(), hasStatusCode(RestStatus.BAD_REQUEST));
         // Create a token to trigger index creation
@@ -356,7 +361,7 @@ public void testInvalidateNotValidRefreshTokens() throws Exception {
         assertThat(invalidateResponse.errors(), empty());
 
         invalidateResponse = invalidateRefreshToken(
-            TokenService.prependVersionAndEncodeRefreshToken(Version.CURRENT, randomAlphaOfLength(32))
+            TokenService.prependVersionAndEncodeRefreshToken(TransportVersion.CURRENT, randomAlphaOfLength(32))
         );
         assertThat(invalidateResponse.invalidated(), equalTo(0));
         assertThat(invalidateResponse.previouslyInvalidated(), equalTo(0));
@@ -365,7 +370,7 @@ public void testInvalidateNotValidRefreshTokens() throws Exception {
         // Generate a token that could be a valid token string for the version we are on, and should decode fine, but is not found in our
         // tokens index
         invalidateResponse = invalidateRefreshToken(
-            TokenService.prependVersionAndEncodeRefreshToken(Version.CURRENT, UUIDs.randomBase64UUID())
+            TokenService.prependVersionAndEncodeRefreshToken(TransportVersion.CURRENT, UUIDs.randomBase64UUID())
         );
         assertThat(invalidateResponse.invalidated(), equalTo(0));
         assertThat(invalidateResponse.previouslyInvalidated(), equalTo(0));
@@ -443,7 +448,7 @@ public void testRefreshingMultipleTimesFails() throws Exception {
         Request updateRequest = new Request(HttpPost.METHOD_NAME, SecuritySystemIndices.SECURITY_TOKENS_ALIAS + "/_update/" + docId.get());
         updateRequest.addParameter("refresh", WriteRequest.RefreshPolicy.IMMEDIATE.getValue());
         updateRequest.setOptions(SECURITY_REQUEST_OPTIONS);
-        updateRequest.setJsonEntity(formatted("""
+        updateRequest.setJsonEntity(Strings.format("""
             {
               "doc": {
                 "refresh_token": {
@@ -628,9 +633,9 @@ public void testAuthenticateWithWrongToken() throws Exception {
         // Now attempt to authenticate with an invalid access token string
         assertUnauthorizedToken(randomAlphaOfLengthBetween(0, 128));
         // Now attempt to authenticate with an invalid access token with valid structure (pre 7.2)
-        assertUnauthorizedToken(generateAccessToken(Version.V_7_1_0));
+        assertUnauthorizedToken(generateAccessToken(TransportVersion.V_7_1_0));
         // Now attempt to authenticate with an invalid access token with valid structure (after 7.2)
-        assertUnauthorizedToken(generateAccessToken(Version.V_7_4_0));
+        assertUnauthorizedToken(generateAccessToken(TransportVersion.V_7_4_0));
     }
 
     @Before
@@ -675,7 +680,7 @@ public void testCreatorRealmCaptureWillWorkWithClientRunAs() throws IOException
         assertThat(ObjectPath.evaluate(authenticateMap, "authentication_realm.name"), equalTo("file"));
         assertThat(ObjectPath.evaluate(authenticateMap, "authentication_type"), is("token"));
 
-        final TokenInvalidation tokenInvalidation = getSecurityClient().invalidateTokens(String.format(Locale.ROOT, """
+        final TokenInvalidation tokenInvalidation = getSecurityClient().invalidateTokens(Strings.format("""
             {
               "realm_name":"%s",
               "username":"%s"
@@ -743,13 +748,13 @@ private TokenInvalidation invalidateRefreshToken(String refreshToken, RequestOpt
         return super.getSecurityClient(options).invalidateRefreshToken(refreshToken);
     }
 
-    private String generateAccessToken(Version version) throws Exception {
+    private String generateAccessToken(TransportVersion version) throws Exception {
         TokenService tokenService = internalCluster().getInstance(TokenService.class);
         String accessTokenString = UUIDs.randomBase64UUID();
         return tokenService.prependVersionAndEncodeAccessToken(version, accessTokenString);
     }
 
-    private String generateInvalidShortAccessToken(Version version) throws Exception {
+    private String generateInvalidShortAccessToken(TransportVersion version) throws Exception {
         TokenService tokenService = internalCluster().getInstance(TokenService.class);
         String accessTokenString = randomAlphaOfLength(32); // UUIDs are 36
         return tokenService.prependVersionAndEncodeAccessToken(version, accessTokenString);
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/apikey/ApiKeySingleNodeTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/apikey/ApiKeySingleNodeTests.java
index eab4f59d7cd9..5eb6614b5b43 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/apikey/ApiKeySingleNodeTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/apikey/ApiKeySingleNodeTests.java
@@ -41,6 +41,8 @@
 import org.elasticsearch.xpack.core.security.action.apikey.GetApiKeyResponse;
 import org.elasticsearch.xpack.core.security.action.apikey.GrantApiKeyAction;
 import org.elasticsearch.xpack.core.security.action.apikey.GrantApiKeyRequest;
+import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyAction;
+import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.apikey.QueryApiKeyAction;
 import org.elasticsearch.xpack.core.security.action.apikey.QueryApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.apikey.QueryApiKeyResponse;
@@ -74,7 +76,12 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.emptyArray;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.hamcrest.Matchers.nullValue;
 
 public class ApiKeySingleNodeTests extends SecuritySingleNodeTestCase {
 
@@ -376,6 +383,33 @@ public void testGrantApiKeyForUserWithRunAs() throws IOException {
         );
     }
 
+    public void testInvalidateApiKeyWillRecordTimestamp() {
+        final String apiKeyId = client().execute(
+            CreateApiKeyAction.INSTANCE,
+            new CreateApiKeyRequest(randomAlphaOfLengthBetween(3, 8), null, TimeValue.timeValueMillis(randomLongBetween(1, 1000)), null)
+        ).actionGet().getId();
+        assertThat(getApiKeyDocument(apiKeyId).get("invalidation_time"), nullValue());
+
+        final long start = Instant.now().toEpochMilli();
+        final List<String> invalidatedApiKeys = client().execute(
+            InvalidateApiKeyAction.INSTANCE,
+            InvalidateApiKeyRequest.usingApiKeyId(apiKeyId, true)
+        ).actionGet().getInvalidatedApiKeys();
+        final long finish = Instant.now().toEpochMilli();
+
+        assertThat(invalidatedApiKeys, equalTo(List.of(apiKeyId)));
+        final Map<String, Object> apiKeyDocument = getApiKeyDocument(apiKeyId);
+        assertThat(apiKeyDocument.get("api_key_invalidated"), is(true));
+        assertThat(apiKeyDocument.get("invalidation_time"), instanceOf(Long.class));
+        final long invalidationTime = (long) apiKeyDocument.get("invalidation_time");
+        assertThat(invalidationTime, greaterThanOrEqualTo(start));
+        assertThat(invalidationTime, lessThanOrEqualTo(finish));
+
+        // Invalidate it again won't change the timestamp
+        client().execute(InvalidateApiKeyAction.INSTANCE, InvalidateApiKeyRequest.usingApiKeyId(apiKeyId, true)).actionGet();
+        assertThat((long) getApiKeyDocument(apiKeyId).get("invalidation_time"), equalTo(invalidationTime));
+    }
+
     private GrantApiKeyRequest buildGrantApiKeyRequest(String username, SecureString password, String runAsUsername) throws IOException {
         final SecureString clonedPassword = password.clone();
         final GrantApiKeyRequest grantApiKeyRequest = new GrantApiKeyRequest();
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSingleNodeTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSingleNodeTests.java
new file mode 100644
index 000000000000..26e08fd34bde
--- /dev/null
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSingleNodeTests.java
@@ -0,0 +1,224 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authc.jwt;
+
+import com.nimbusds.jose.JWSHeader;
+import com.nimbusds.jose.util.Base64URL;
+import com.nimbusds.jwt.JWTClaimsSet;
+import com.nimbusds.jwt.SignedJWT;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.test.SecuritySettingsSource;
+import org.elasticsearch.test.SecuritySingleNodeTestCase;
+import org.elasticsearch.xpack.security.authc.Realms;
+
+import java.text.ParseException;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+public class JwtRealmSingleNodeTests extends SecuritySingleNodeTestCase {
+
+    @Override
+    protected Settings nodeSettings() {
+        final Settings.Builder builder = Settings.builder()
+            .put(super.nodeSettings())
+            // 1st JWT realm
+            .put("xpack.security.authc.realms.jwt.jwt0.order", 10)
+            .put(
+                randomBoolean()
+                    ? Settings.builder().put("xpack.security.authc.realms.jwt.jwt0.token_type", "id_token").build()
+                    : Settings.EMPTY
+            )
+            .put("xpack.security.authc.realms.jwt.jwt0.allowed_issuer", "my-issuer-01")
+            .put("xpack.security.authc.realms.jwt.jwt0.allowed_audiences", "es-01")
+            .put("xpack.security.authc.realms.jwt.jwt0.claims.principal", "sub")
+            .put("xpack.security.authc.realms.jwt.jwt0.claims.groups", "groups")
+            .put("xpack.security.authc.realms.jwt.jwt0.client_authentication.type", "shared_secret")
+            .putList("xpack.security.authc.realms.jwt.jwt0.allowed_signature_algorithms", "HS256", "HS384")
+            // 2nd JWT realm
+            .put("xpack.security.authc.realms.jwt.jwt1.order", 20)
+            .put("xpack.security.authc.realms.jwt.jwt1.token_type", "access_token")
+            .put("xpack.security.authc.realms.jwt.jwt1.allowed_issuer", "my-issuer-02")
+            .put("xpack.security.authc.realms.jwt.jwt1.allowed_subjects", "user-02")
+            .put("xpack.security.authc.realms.jwt.jwt1.allowed_audiences", "es-02")
+            .put("xpack.security.authc.realms.jwt.jwt1.fallback_claims.sub", "client_id")
+            .put("xpack.security.authc.realms.jwt.jwt1.claims.principal", "appid")
+            .put("xpack.security.authc.realms.jwt.jwt1.claims.groups", "groups")
+            .put("xpack.security.authc.realms.jwt.jwt1.client_authentication.type", "shared_secret")
+            .putList("xpack.security.authc.realms.jwt.jwt1.allowed_signature_algorithms", "HS256", "HS384")
+            // 3rd JWT realm
+            .put("xpack.security.authc.realms.jwt.jwt2.order", 30)
+            .put("xpack.security.authc.realms.jwt.jwt2.token_type", "access_token")
+            .put("xpack.security.authc.realms.jwt.jwt2.allowed_issuer", "my-issuer-03")
+            .put("xpack.security.authc.realms.jwt.jwt2.allowed_subjects", "user-03")
+            .put("xpack.security.authc.realms.jwt.jwt2.allowed_audiences", "es-03")
+            .put("xpack.security.authc.realms.jwt.jwt2.fallback_claims.sub", "oid")
+            .put("xpack.security.authc.realms.jwt.jwt2.claims.principal", "email")
+            .put("xpack.security.authc.realms.jwt.jwt2.claims.groups", "groups")
+            .put("xpack.security.authc.realms.jwt.jwt2.client_authentication.type", "shared_secret")
+            .putList("xpack.security.authc.realms.jwt.jwt2.allowed_signature_algorithms", "HS256", "HS384");
+
+        SecuritySettingsSource.addSecureSettings(builder, secureSettings -> {
+            secureSettings.setString("xpack.security.authc.realms.jwt.jwt0.hmac_key", "jwt0_hmac_key");
+            secureSettings.setString("xpack.security.authc.realms.jwt.jwt0.client_authentication.shared_secret", "jwt0_shared_secret");
+            secureSettings.setString("xpack.security.authc.realms.jwt.jwt1.hmac_key", "jwt1_hmac_key");
+            secureSettings.setString("xpack.security.authc.realms.jwt.jwt1.client_authentication.shared_secret", "jwt1_shared_secret");
+            secureSettings.setString("xpack.security.authc.realms.jwt.jwt2.hmac_key", "jwt2_hmac_key");
+            secureSettings.setString("xpack.security.authc.realms.jwt.jwt2.client_authentication.shared_secret", "jwt2_shared_secret");
+        });
+
+        return builder.build();
+    }
+
+    public void testAnyJwtRealmWillExtractTheToken() throws ParseException {
+        final List<JwtRealm> jwtRealms = getJwtRealms();
+        final JwtRealm jwtRealm = randomFrom(jwtRealms);
+
+        final String sharedSecret = randomBoolean() ? randomAlphaOfLengthBetween(10, 20) : null;
+        final String iss = randomAlphaOfLengthBetween(5, 18);
+        final String aud = randomAlphaOfLengthBetween(5, 18);
+        final String sub = randomAlphaOfLengthBetween(5, 18);
+
+        // Realm 1 will extract the token because the JWT has all iss, sub, aud, principal claims.
+        // Their values do not match what realm 1 expects but that does not matter when extracting the token
+        final SignedJWT signedJWT1 = getSignedJWT(Map.of("iss", iss, "aud", aud, "sub", sub));
+        final ThreadContext threadContext1 = prepareThreadContext(signedJWT1, sharedSecret);
+        final var token1 = (JwtAuthenticationToken) jwtRealm.token(threadContext1);
+        final String principal1 = Strings.format("%s/%s/%s/%s", iss, aud, sub, sub);
+        assertJwtToken(token1, principal1, sharedSecret, signedJWT1);
+
+        // Realm 2 for extracting the token from the following JWT
+        // Because it does not have the sub claim but client_id, which is configured as fallback by realm 2
+        final String appId = randomAlphaOfLengthBetween(5, 18);
+        final SignedJWT signedJWT2 = getSignedJWT(Map.of("iss", iss, "aud", aud, "client_id", sub, "appid", appId));
+        final ThreadContext threadContext2 = prepareThreadContext(signedJWT2, sharedSecret);
+        final var token2 = (JwtAuthenticationToken) jwtRealm.token(threadContext2);
+        final String principal2 = Strings.format("%s/%s/%s/%s", iss, aud, sub, appId);
+        assertJwtToken(token2, principal2, sharedSecret, signedJWT2);
+
+        // Realm 3 will extract the token from the following JWT
+        // Because it has the oid claim which is configured as a fallback by realm 3
+        final String email = randomAlphaOfLengthBetween(5, 18) + "@example.com";
+        final SignedJWT signedJWT3 = getSignedJWT(Map.of("iss", iss, "aud", aud, "oid", sub, "email", email));
+        final ThreadContext threadContext3 = prepareThreadContext(signedJWT3, sharedSecret);
+        final var token3 = (JwtAuthenticationToken) jwtRealm.token(threadContext3);
+        final String principal3 = Strings.format("%s/%s/%s/%s", iss, aud, sub, email);
+        assertJwtToken(token3, principal3, sharedSecret, signedJWT3);
+
+        // The JWT does not match any realm's configuration, a token with generic token principal will be extracted
+        final SignedJWT signedJWT4 = getSignedJWT(Map.of("iss", iss, "aud", aud, "azp", sub, "email", email));
+        final ThreadContext threadContext4 = prepareThreadContext(signedJWT4, sharedSecret);
+        final var token4 = (JwtAuthenticationToken) jwtRealm.token(threadContext4);
+        final String principal4 = Strings.format("<unrecognized-jwt> by %s", iss);
+        assertJwtToken(token4, principal4, sharedSecret, signedJWT4);
+
+        // The JWT does not have an issuer, a token with generic token principal will be extracted
+        final SignedJWT signedJWT5 = getSignedJWT(Map.of("aud", aud, "sub", sub));
+        final ThreadContext threadContext5 = prepareThreadContext(signedJWT5, sharedSecret);
+        final var token5 = (JwtAuthenticationToken) jwtRealm.token(threadContext5);
+        final String principal5 = "<unrecognized-jwt>";
+        assertJwtToken(token5, principal5, sharedSecret, signedJWT5);
+    }
+
+    public void testJwtRealmReturnsNullTokenWhenJwtCredentialIsAbsent() {
+        final List<JwtRealm> jwtRealms = getJwtRealms();
+        final JwtRealm jwtRealm = randomFrom(jwtRealms);
+        final String sharedSecret = randomBoolean() ? randomAlphaOfLengthBetween(10, 20) : null;
+
+        // Authorization header is absent
+        final ThreadContext threadContext1 = prepareThreadContext(null, sharedSecret);
+        assertThat(jwtRealm.token(threadContext1), nullValue());
+
+        // Scheme is not Bearer
+        final ThreadContext threadContext2 = prepareThreadContext(null, sharedSecret);
+        threadContext2.putHeader("Authorization", "Basic foobar");
+        assertThat(jwtRealm.token(threadContext2), nullValue());
+    }
+
+    public void testJwtRealmThrowsErrorOnJwtParsingFailure() throws ParseException {
+        final List<JwtRealm> jwtRealms = getJwtRealms();
+        final JwtRealm jwtRealm = randomFrom(jwtRealms);
+        final String sharedSecret = randomBoolean() ? randomAlphaOfLengthBetween(10, 20) : null;
+
+        // Not a JWT
+        final ThreadContext threadContext1 = prepareThreadContext(null, sharedSecret);
+        threadContext1.putHeader("Authorization", "Bearer " + randomAlphaOfLengthBetween(40, 60));
+        final IllegalArgumentException e1 = expectThrows(IllegalArgumentException.class, () -> jwtRealm.token(threadContext1));
+        assertThat(e1.getMessage(), containsString("Failed to parse JWT bearer token"));
+
+        // Payload is not JSON
+        final SignedJWT signedJWT2 = new SignedJWT(
+            JWSHeader.parse(Map.of("alg", randomAlphaOfLengthBetween(5, 10))).toBase64URL(),
+            Base64URL.encode("payload"),
+            Base64URL.encode("signature")
+        );
+        final ThreadContext threadContext2 = prepareThreadContext(null, sharedSecret);
+        threadContext2.putHeader("Authorization", "Bearer " + signedJWT2.serialize());
+        final IllegalArgumentException e2 = expectThrows(IllegalArgumentException.class, () -> jwtRealm.token(threadContext2));
+        assertThat(e2.getMessage(), containsString("Failed to parse JWT claims set"));
+    }
+
+    private void assertJwtToken(JwtAuthenticationToken token, String tokenPrincipal, String sharedSecret, SignedJWT signedJWT)
+        throws ParseException {
+        assertThat(token.principal(), equalTo(tokenPrincipal));
+        assertThat(token.getClientAuthenticationSharedSecret(), equalTo(sharedSecret));
+        assertThat(token.getJWTClaimsSet(), equalTo(signedJWT.getJWTClaimsSet()));
+        assertThat(token.getSignedJWT().getHeader().toJSONObject(), equalTo(signedJWT.getHeader().toJSONObject()));
+        assertThat(token.getSignedJWT().getSignature(), equalTo(signedJWT.getSignature()));
+        assertThat(token.getSignedJWT().getJWTClaimsSet(), equalTo(token.getJWTClaimsSet()));
+    }
+
+    private List<JwtRealm> getJwtRealms() {
+        final Realms realms = getInstanceFromNode(Realms.class);
+        final List<JwtRealm> jwtRealms = realms.getActiveRealms()
+            .stream()
+            .filter(realm -> realm instanceof JwtRealm)
+            .map(JwtRealm.class::cast)
+            .toList();
+        return jwtRealms;
+    }
+
+    private SignedJWT getSignedJWT(Map<String, Object> m) throws ParseException {
+        final HashMap<String, Object> claimsMap = new HashMap<>(m);
+        final Instant now = Instant.now();
+        // timestamp does not matter for tokenExtraction
+        claimsMap.put("iat", now.minus(randomIntBetween(-1, 1), ChronoUnit.DAYS).getEpochSecond());
+        claimsMap.put("exp", now.plus(randomIntBetween(-1, 1), ChronoUnit.DAYS).getEpochSecond());
+
+        final JWTClaimsSet claimsSet = JWTClaimsSet.parse(claimsMap);
+        final SignedJWT signedJWT = new SignedJWT(
+            JWSHeader.parse(Map.of("alg", randomAlphaOfLengthBetween(5, 10))).toBase64URL(),
+            claimsSet.toPayload().toBase64URL(),
+            Base64URL.encode("signature")
+        );
+        return signedJWT;
+    }
+
+    private ThreadContext prepareThreadContext(SignedJWT signedJWT, String clientSecret) {
+        final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        if (signedJWT != null) {
+            threadContext.putHeader("Authorization", "Bearer " + signedJWT.serialize());
+        }
+        if (clientSecret != null) {
+            threadContext.putHeader(
+                JwtRealm.HEADER_CLIENT_AUTHENTICATION,
+                JwtRealm.HEADER_SHARED_SECRET_AUTHENTICATION_SCHEME + " " + clientSecret
+            );
+        }
+        return threadContext;
+    }
+}
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/pki/PkiAuthDelegationIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/pki/PkiAuthDelegationIntegTests.java
index df7dbff845bc..9115c3555158 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/pki/PkiAuthDelegationIntegTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/pki/PkiAuthDelegationIntegTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
@@ -105,7 +106,7 @@ protected String configUsers() {
 
     @Override
     protected String configRoles() {
-        return formatted("""
+        return Strings.format("""
             %s
             role_manage:
               cluster: [ manage ]
@@ -123,7 +124,7 @@ protected String configRoles() {
 
     @Override
     protected String configUsersRoles() {
-        return formatted("""
+        return Strings.format("""
             %s
             role_manage:user_manage
             role_manage_security:user_manage_security
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/pki/PkiAuthenticationTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/pki/PkiAuthenticationTests.java
index c0bbe037ca2f..5b90b5855329 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/pki/PkiAuthenticationTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/pki/PkiAuthenticationTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.ssl.SslClientAuthenticationMode;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.test.SecuritySingleNodeTestCase;
 import org.elasticsearch.xpack.core.common.socket.SocketAccess;
@@ -141,6 +142,6 @@ private String getNodeUrl() {
             node().injector().getInstance(HttpServerTransport.class).boundAddress().boundAddresses()
         );
         final InetSocketAddress inetSocketAddress = transportAddress.address();
-        return formatted("https://%s/", NetworkAddress.format(inetSocketAddress));
+        return Strings.format("https://%s/", NetworkAddress.format(inetSocketAddress));
     }
 }
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/AnalyzeTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/AnalyzeTests.java
index 5f66d475f530..a77e3d14f288 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/AnalyzeTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/AnalyzeTests.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction;
 import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSourceField;
 
@@ -35,7 +36,7 @@ protected String configUsersRoles() {
     @Override
     protected String configRoles() {
         // role that has analyze indices privileges only
-        return formatted("""
+        return Strings.format("""
             %s
             analyze_indices:
               indices:
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/IndexAliasesTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/IndexAliasesTests.java
index 148ac1ea3ee2..36cdcd40ebea 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/IndexAliasesTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/IndexAliasesTests.java
@@ -769,14 +769,16 @@ public void testGetAliasesAliasesOnlyPermissionIgnoreUnavailable() {
             .get();
         assertEquals(0, getAliasesResponse.getAliases().size());
 
-        // no manage_aliases privilege on non_authorized alias
-        getAliasesResponse = client.admin()
-            .indices()
-            .prepareGetAliases("non_authorized")
-            .addIndices("test_1")
-            .setIndicesOptions(IndicesOptions.lenientExpandOpen())
-            .get();
-        assertEquals(0, getAliasesResponse.getAliases().size());
+        // no manage_aliases privilege on `non_authorized` alias and ignore_unavailable doesn't influence aliases
+        assertThrowsAuthorizationException(
+            client.admin()
+                .indices()
+                .prepareGetAliases("non_authorized")
+                .addIndices("test_1")
+                .setIndicesOptions(IndicesOptions.lenientExpandOpen())::get,
+            GetAliasesAction.NAME,
+            "aliases_only"
+        );
 
         // no manage_aliases privilege on non_authorized index
         getAliasesResponse = client.admin()
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/ReadActionsTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/ReadActionsTests.java
index b674141668ea..48a0fabd1e31 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/ReadActionsTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/ReadActionsTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.action.termvectors.MultiTermVectorsResponse;
 import org.elasticsearch.action.termvectors.TermVectorsAction;
 import org.elasticsearch.client.internal.Requests;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.search.SearchHit;
@@ -39,7 +40,7 @@ public class ReadActionsTests extends SecurityIntegTestCase {
 
     @Override
     protected String configRoles() {
-        return formatted("""
+        return Strings.format("""
             %s:
               cluster: [ ALL ]
               indices:
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/WriteActionsTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/WriteActionsTests.java
index 924b5d1f5b99..c55e5b409279 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/WriteActionsTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/WriteActionsTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.action.update.UpdateAction;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.client.internal.Requests;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.engine.DocumentMissingException;
 import org.elasticsearch.rest.RestStatus;
@@ -33,7 +34,7 @@ public class WriteActionsTests extends SecurityIntegTestCase {
 
     @Override
     protected String configRoles() {
-        return formatted("""
+        return Strings.format("""
             %s:
               cluster: [ ALL ]
               indices:
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileCancellationIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileCancellationIntegTests.java
index 58be2dffcc3d..749c99207a93 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileCancellationIntegTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileCancellationIntegTests.java
@@ -417,7 +417,7 @@ public void loadAuthorizedIndices(
                     RequestInfo requestInfo,
                     AuthorizationInfo authorizationInfo,
                     Map<String, IndexAbstraction> indicesLookup,
-                    ActionListener<Set<String>> listener
+                    ActionListener<AuthorizationEngine.AuthorizedIndices> listener
                 ) {
                     listener.onFailure(new UnsupportedOperationException("not implemented"));
                 }
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileDomainIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileDomainIntegTests.java
index fde9e781d9ef..9c271a50fe49 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileDomainIntegTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileDomainIntegTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.xcontent.XContentType;
@@ -275,7 +276,7 @@ public void testConcurrentCreationOfNewProfiles() throws InterruptedException {
             existingUid = future.actionGet().uid();
             assertThat(existingUid, endsWith("_0"));
             final UpdateRequest updateRequest = client().prepareUpdate(SECURITY_PROFILE_ALIAS, "profile_" + existingUid)
-                .setDoc(formatted("""
+                .setDoc(Strings.format("""
                     {
                       "user_profile": {
                         "user": { "username": "%s" }
@@ -357,13 +358,16 @@ public void testDifferentiator() {
             }
             final String newUsername = i == otherRacUserIndex ? OTHER_RAC_USER_NAME : "some-other-name-" + randomAlphaOfLength(8);
             // Manually update the username to create hash collision
-            final UpdateRequest updateRequest = client().prepareUpdate(SECURITY_PROFILE_ALIAS, "profile_" + currentUid).setDoc(formatted("""
-                {
-                  "user_profile": {
-                    "user": { "username": "%s" }
-                  }
-                }
-                """, newUsername), XContentType.JSON).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).request();
+            final UpdateRequest updateRequest = client().prepareUpdate(SECURITY_PROFILE_ALIAS, "profile_" + currentUid)
+                .setDoc(Strings.format("""
+                    {
+                      "user_profile": {
+                        "user": { "username": "%s" }
+                      }
+                    }
+                    """, newUsername), XContentType.JSON)
+                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+                .request();
             client().update(updateRequest).actionGet();
             if (newUsername.equals(OTHER_RAC_USER_NAME)) {
                 // The manually updated profile document can still be activated by the other rac user
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/filter/IpFilteringUpdateTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/filter/IpFilteringUpdateTests.java
index bc206f4557f6..6b94591cab19 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/filter/IpFilteringUpdateTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/filter/IpFilteringUpdateTests.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.junit.BeforeClass;
@@ -16,7 +17,6 @@
 import java.net.InetSocketAddress;
 import java.net.UnknownHostException;
 import java.util.Arrays;
-import java.util.Locale;
 
 import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -184,7 +184,7 @@ private void assertConnectionAccepted(String profile, String host) throws Unknow
         }
 
         IPFilter ipFilter = internalCluster().getDataNodeInstance(IPFilter.class);
-        String message = String.format(Locale.ROOT, "Expected allowed connection for profile %s against host %s", profile, host);
+        String message = Strings.format("Expected allowed connection for profile %s against host %s", profile, host);
         assertThat(message, ipFilter.accept(profile, new InetSocketAddress(InetAddress.getByName(host), 0)), is(true));
     }
 
@@ -195,7 +195,7 @@ private void assertConnectionRejected(String profile, String host) throws Unknow
         }
 
         IPFilter ipFilter = internalCluster().getDataNodeInstance(IPFilter.class);
-        String message = String.format(Locale.ROOT, "Expected rejection for profile %s against host %s", profile, host);
+        String message = Strings.format("Expected rejection for profile %s against host %s", profile, host);
         assertThat(message, ipFilter.accept(profile, new InetSocketAddress(InetAddress.getByName(host), 0)), is(false));
     }
 }
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/ssl/SslIntegrationTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/ssl/SslIntegrationTests.java
index 7139c345f059..a4453521ffb4 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/ssl/SslIntegrationTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/transport/ssl/SslIntegrationTests.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.ssl.SslConfiguration;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.test.SecurityIntegTestCase;
@@ -121,6 +122,6 @@ private String getNodeUrl() {
             internalCluster().getInstance(HttpServerTransport.class).boundAddress().boundAddresses()
         );
         final InetSocketAddress inetSocketAddress = transportAddress.address();
-        return formatted("https://%s/", NetworkAddress.format(inetSocketAddress));
+        return Strings.format("https://%s/", NetworkAddress.format(inetSocketAddress));
     }
 }
diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java
index 18dd2a33ae70..71a4d7d9231c 100644
--- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java
+++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.TestSecurityClient;
 import org.elasticsearch.xpack.core.XPackSettings;
@@ -71,7 +72,7 @@ public Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
 
     @Override
     public String configRoles() {
-        return formatted("""
+        return Strings.format("""
             %s
             anonymous:
               cluster: [ manage_token ]
diff --git a/x-pack/plugin/security/src/main/java/module-info.java b/x-pack/plugin/security/src/main/java/module-info.java
index 9d87b5fba927..6242f7bd7be2 100644
--- a/x-pack/plugin/security/src/main/java/module-info.java
+++ b/x-pack/plugin/security/src/main/java/module-info.java
@@ -39,6 +39,7 @@
     requires org.opensaml.xmlsec;
 
     requires com.nimbusds.jose.jwt;
+    requires io.netty.common;
     requires io.netty.handler;
     requires io.netty.transport;
     requires jopt.simple;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/ProfileConfigurations.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/ProfileConfigurations.java
index 8569ecaa81c0..bdf7c165b55a 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/ProfileConfigurations.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/ProfileConfigurations.java
@@ -8,26 +8,115 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.ssl.SslConfiguration;
-import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.transport.TransportSettings;
+import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.transport.RemoteClusterPortSettings.REMOTE_CLUSTER_PORT_ENABLED;
+import static org.elasticsearch.transport.RemoteClusterPortSettings.REMOTE_CLUSTER_PROFILE;
+import static org.elasticsearch.xpack.core.XPackSettings.REMOTE_CLUSTER_SSL_ENABLED;
 import static org.elasticsearch.xpack.core.security.SecurityField.setting;
 
+/**
+ * Settings for a transport profile usually begin with "transport.profiles.NAME."
+ * The settings can be either of the two categories:
+ *   1. Networking - e.g. `transport.profiles.NAME.tcp.keep_alive: true`
+ *   2. SSL - e.g. `transport.profiles.NAME.xpack.security.ssl.client_authentication: none`
+ * This class is responsible for building SSL configuration for transport profiles.
+ *
+ * Among the transport profiles, two of them are special: "default" and "_remote_cluster".
+ *
+ * The "default" profile has dedicated settings for both networking (e.g. `transport.tcp.keep_alive`)
+ * and SSL (e.g. `xpack.security.transport.ssl.client_authentication`).
+ * It also accepts networking settings specified with its transport profile name,
+ * e.g. `transport.profiles.default.tcp.keep_alive` is valid configuration.
+ * But it does *not* allow SSL settings to be specified with its transport profile name,
+ * e.g. `transport.profiles.default.xpack.security.ssl.client_authentication` is NOT valid configuration.
+ *
+ * The "_remote_cluster" profile also has dedicated settings for both networking (e.g. `remote_cluster.tcp.keep_alive`)
+ * and SSL (e.g. `xpack.security.remote_cluster.ssl.client_authentication`).
+ * This profile is completely synthetic in that it does NOT accept either networking or SSL settings
+ * with its transport profile name.
+ * NOTE the "_remote_cluster" profile name is special ONLY when the remote cluster port is enabled.
+ * If the remote cluster port is not enabled, this profile name will be treated just as a normal profile (for BWC).
+ *
+ * When building SSL configurations for the transport profiles, assuming SSL is enabled,
+ * this class builds a map that contains a configuration for each of the configured transport profiles
+ * (keyed by its name).
+ * The map also contains an entry that has the special key "default" and value being the SSL
+ * configuration for the "default" profile.
+ * If remote cluster is enabled, the map will also contain an entry that has the special key
+ * "_remote_cluster" with the value being the SSL configuration of the synthetic "_remote_cluster" profile.
+ *
+ * NOTE the "_remote_cluster" profile only applies to the new remote cluster model.
+ * The legacy remote cluster model mostly just uses the "default" transport profile.
+ */
 public final class ProfileConfigurations {
 
     private ProfileConfigurations() {}
 
-    public static Map<String, SslConfiguration> get(Settings settings, SSLService sslService, SslConfiguration defaultConfiguration) {
+    /**
+     * Builds SSL configuration for transport profiles.
+     *
+     * @param settings Settings of the ES node
+     * @param sslService For resolving the SSL configuration based on its prefix
+     * @param sslEnabledOnly If true, only include the SSL configuration if SSL is enabled for the profile.
+     *                       If false, SSL configuration is included for a profile regardless whether SSL is actually enabled for it.
+     * @return A map that contains {@link SslConfiguration} for each named transport profile as well
+     *         as an entry for the "default" profile. If the remote_cluster feature is enabled, it also
+     *         contains an entry for the synthetic "_remote_cluster" profile.
+     */
+    public static Map<String, SslConfiguration> get(Settings settings, SSLService sslService, boolean sslEnabledOnly) {
+        final boolean transportSslEnabled = XPackSettings.TRANSPORT_SSL_ENABLED.get(settings);
+        final boolean remoteClusterPortEnabled = REMOTE_CLUSTER_PORT_ENABLED.get(settings);
+        final boolean remoteClusterSslEnabled = remoteClusterPortEnabled && REMOTE_CLUSTER_SSL_ENABLED.get(settings);
+
+        final Map<String, SslConfiguration> profileConfigurations = new HashMap<>();
+
+        if (sslEnabledOnly) {
+            if (transportSslEnabled == false && remoteClusterSslEnabled == false) {
+                return profileConfigurations;
+            } else if (transportSslEnabled == false) {
+                // The single TRANSPORT_SSL_ENABLED setting determines whether SSL is enabled for both
+                // the default transport profile and any custom transport profiles. That is, SSL is
+                // always either enabled or disabled together for default and custom transport profiles.
+                profileConfigurations.put(REMOTE_CLUSTER_PROFILE, sslService.getSSLConfiguration(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX));
+                return profileConfigurations;
+            } else if (remoteClusterSslEnabled == false) {
+                populateFromTransportProfiles(settings, sslService, profileConfigurations);
+                return profileConfigurations;
+            }
+        }
+
+        // At this point, either SSL is enabled for both transport and remote cluster, or sslEnabledOnly is false.
+        // In both case, we need to include all configurations
+        populateFromTransportProfiles(settings, sslService, profileConfigurations);
+        if (remoteClusterPortEnabled) {
+            assert profileConfigurations.containsKey(REMOTE_CLUSTER_PROFILE) == false;
+            profileConfigurations.put(REMOTE_CLUSTER_PROFILE, sslService.getSSLConfiguration(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX));
+        }
+
+        return profileConfigurations;
+    }
+
+    private static void populateFromTransportProfiles(
+        Settings settings,
+        SSLService sslService,
+        Map<String, SslConfiguration> profileConfigurations
+    ) {
+        final SslConfiguration defaultConfiguration = sslService.getSSLConfiguration(setting("transport.ssl."));
+
         Set<String> profileNames = settings.getGroups("transport.profiles.", true).keySet();
-        Map<String, SslConfiguration> profileConfiguration = Maps.newMapWithExpectedSize(profileNames.size() + 1);
         for (String profileName : profileNames) {
             if (profileName.equals(TransportSettings.DEFAULT_PROFILE)) {
                 // don't attempt to parse ssl settings from the profile;
                 // profiles need to be killed with fire
+                // We don't need to check _remote_cluster profile here; when remote access settings are validated, we check that there are
+                // no direct usages of the profile, so we can just add it after all the profiles are in place.
                 if (settings.getByPrefix("transport.profiles.default.xpack.security.ssl.").isEmpty()) {
                     continue;
                 } else {
@@ -37,12 +126,12 @@ public static Map<String, SslConfiguration> get(Settings settings, SSLService ss
                     );
                 }
             }
+
             SslConfiguration configuration = sslService.getSSLConfiguration("transport.profiles." + profileName + "." + setting("ssl"));
-            profileConfiguration.put(profileName, configuration);
+            profileConfigurations.put(profileName, configuration);
         }
 
-        assert profileConfiguration.containsKey(TransportSettings.DEFAULT_PROFILE) == false;
-        profileConfiguration.put(TransportSettings.DEFAULT_PROFILE, defaultConfiguration);
-        return profileConfiguration;
+        assert profileConfigurations.containsKey(TransportSettings.DEFAULT_PROFILE) == false;
+        profileConfigurations.put(TransportSettings.DEFAULT_PROFILE, defaultConfiguration);
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/SecurityTransportExceptionHandler.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/SecurityTransportExceptionHandler.java
index 051801479e3f..d8f5f773bd0c 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/SecurityTransportExceptionHandler.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/SecurityTransportExceptionHandler.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.network.CloseableChannel;
 import org.elasticsearch.transport.TcpChannel;
+import org.elasticsearch.transport.netty4.SSLExceptionHelper;
 
 import java.util.function.BiConsumer;
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/netty4/SecurityNetty4Transport.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/netty4/SecurityNetty4Transport.java
index 611f5a3ce763..fecccaadcac6 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/netty4/SecurityNetty4Transport.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/netty4/SecurityNetty4Transport.java
@@ -12,10 +12,11 @@
 import io.netty.channel.ChannelOutboundHandlerAdapter;
 import io.netty.channel.ChannelPromise;
 import io.netty.handler.ssl.SslHandler;
+import io.netty.util.concurrent.Future;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
@@ -25,7 +26,9 @@
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.ConnectionProfile;
 import org.elasticsearch.transport.TcpChannel;
+import org.elasticsearch.transport.TransportSettings;
 import org.elasticsearch.transport.netty4.Netty4Transport;
 import org.elasticsearch.transport.netty4.SharedGroupFactory;
 import org.elasticsearch.xpack.core.XPackSettings;
@@ -43,7 +46,9 @@
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLParameters;
 
-import static org.elasticsearch.xpack.core.security.SecurityField.setting;
+import static org.elasticsearch.transport.RemoteClusterPortSettings.REMOTE_CLUSTER_PORT_ENABLED;
+import static org.elasticsearch.transport.RemoteClusterPortSettings.REMOTE_CLUSTER_PROFILE;
+import static org.elasticsearch.xpack.core.XPackSettings.REMOTE_CLUSTER_SSL_ENABLED;
 
 /**
  * Implementation of a transport that extends the {@link Netty4Transport} to add SSL and IP Filtering
@@ -53,13 +58,16 @@ public class SecurityNetty4Transport extends Netty4Transport {
 
     private final SecurityTransportExceptionHandler exceptionHandler;
     private final SSLService sslService;
-    private final SslConfiguration sslConfiguration;
-    private final Map<String, SslConfiguration> profileConfiguration;
-    private final boolean sslEnabled;
+    private final SslConfiguration defaultSslConfiguration;
+    private final Map<String, SslConfiguration> profileConfigurations;
+    private final boolean transportSslEnabled;
+    private final boolean remoteClusterPortEnabled;
+    private final boolean remoteClusterSslEnabled;
+    private final SslConfiguration remoteClusterClientSslConfiguration;
 
     public SecurityNetty4Transport(
         final Settings settings,
-        final Version version,
+        final TransportVersion version,
         final ThreadPool threadPool,
         final NetworkService networkService,
         final PageCacheRecycler pageCacheRecycler,
@@ -80,14 +88,20 @@ public SecurityNetty4Transport(
         );
         this.exceptionHandler = new SecurityTransportExceptionHandler(logger, lifecycle, (c, e) -> super.onException(c, e));
         this.sslService = sslService;
-        this.sslEnabled = XPackSettings.TRANSPORT_SSL_ENABLED.get(settings);
-        if (sslEnabled) {
-            this.sslConfiguration = sslService.getSSLConfiguration(setting("transport.ssl."));
-            Map<String, SslConfiguration> profileConfiguration = ProfileConfigurations.get(settings, sslService, sslConfiguration);
-            this.profileConfiguration = Collections.unmodifiableMap(profileConfiguration);
+        this.transportSslEnabled = XPackSettings.TRANSPORT_SSL_ENABLED.get(settings);
+        this.remoteClusterPortEnabled = REMOTE_CLUSTER_PORT_ENABLED.get(settings);
+        this.remoteClusterSslEnabled = REMOTE_CLUSTER_SSL_ENABLED.get(settings);
+
+        this.profileConfigurations = Collections.unmodifiableMap(ProfileConfigurations.get(settings, sslService, true));
+        this.defaultSslConfiguration = this.profileConfigurations.get(TransportSettings.DEFAULT_PROFILE);
+        assert this.transportSslEnabled == false || this.defaultSslConfiguration != null;
+
+        // Client configuration does not depend on whether the remote access port is enabled
+        if (remoteClusterSslEnabled) {
+            this.remoteClusterClientSslConfiguration = sslService.getSSLConfiguration(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX);
+            assert this.remoteClusterClientSslConfiguration != null;
         } else {
-            this.profileConfiguration = Collections.emptyMap();
-            this.sslConfiguration = null;
+            this.remoteClusterClientSslConfiguration = null;
         }
     }
 
@@ -98,8 +112,18 @@ protected void doStart() {
 
     @Override
     public final ChannelHandler getServerChannelInitializer(String name) {
-        if (sslEnabled) {
-            SslConfiguration configuration = profileConfiguration.get(name);
+        if (remoteClusterPortEnabled && REMOTE_CLUSTER_PROFILE.equals(name)) {
+            if (remoteClusterSslEnabled) {
+                final SslConfiguration remoteClusterSslConfiguration = profileConfigurations.get(name);
+                if (remoteClusterSslConfiguration == null) {
+                    throw new IllegalStateException("remote cluster SSL is enabled but no configuration is found");
+                }
+                return getSslChannelInitializer(name, remoteClusterSslConfiguration);
+            } else {
+                return getNoSslChannelInitializer(name);
+            }
+        } else if (transportSslEnabled) {
+            SslConfiguration configuration = profileConfigurations.get(name);
             if (configuration == null) {
                 throw new IllegalStateException("unknown profile: " + name);
             }
@@ -114,8 +138,8 @@ protected ChannelHandler getNoSslChannelInitializer(final String name) {
     }
 
     @Override
-    protected ChannelHandler getClientChannelInitializer(DiscoveryNode node) {
-        return new SecurityClientChannelInitializer(node);
+    protected ChannelHandler getClientChannelInitializer(DiscoveryNode node, ConnectionProfile connectionProfile) {
+        return new SecurityClientChannelInitializer(node, connectionProfile);
     }
 
     @Override
@@ -143,21 +167,36 @@ protected void initChannel(Channel ch) throws Exception {
     }
 
     protected ServerChannelInitializer getSslChannelInitializer(final String name, final SslConfiguration configuration) {
-        return new SslChannelInitializer(name, sslConfiguration);
+        return new SslChannelInitializer(name, configuration);
     }
 
     @Override
     public boolean isSecure() {
-        return this.sslEnabled;
+        return this.transportSslEnabled;
     }
 
     private class SecurityClientChannelInitializer extends ClientChannelInitializer {
 
         private final boolean hostnameVerificationEnabled;
         private final SNIHostName serverName;
+        private final SslConfiguration channelSslConfiguration;
 
-        SecurityClientChannelInitializer(DiscoveryNode node) {
-            this.hostnameVerificationEnabled = sslEnabled && sslConfiguration.verificationMode().isHostnameVerificationEnabled();
+        SecurityClientChannelInitializer(DiscoveryNode node, ConnectionProfile connectionProfile) {
+            final String transportProfile = connectionProfile.getTransportProfile();
+            logger.trace("initiating security client channel with transport profile [{}]", transportProfile);
+            // Only client connections to a new RCS remote cluster can have transport profile of _remote_cluster
+            // All other client connections use the default transport profile regardless of the transport profile used on the server side.
+            if (REMOTE_CLUSTER_PROFILE.equals(transportProfile)) {
+                this.channelSslConfiguration = remoteClusterClientSslConfiguration;
+            } else {
+                assert TransportSettings.DEFAULT_PROFILE.equals(transportProfile);
+                this.channelSslConfiguration = defaultSslConfiguration;
+            }
+            if (this.channelSslConfiguration != null) {
+                this.hostnameVerificationEnabled = this.channelSslConfiguration.verificationMode().isHostnameVerificationEnabled();
+            } else {
+                this.hostnameVerificationEnabled = false;
+            }
             String configuredServerName = node.getAttributes().get("server_name");
             if (configuredServerName != null) {
                 try {
@@ -173,9 +212,11 @@ private class SecurityClientChannelInitializer extends ClientChannelInitializer
         @Override
         protected void initChannel(Channel ch) throws Exception {
             super.initChannel(ch);
-            if (sslEnabled) {
+            if (channelSslConfiguration != null) {
                 ch.pipeline()
-                    .addFirst(new ClientSslHandlerInitializer(sslConfiguration, sslService, hostnameVerificationEnabled, serverName));
+                    .addFirst(
+                        new ClientSslHandlerInitializer(channelSslConfiguration, sslService, hostnameVerificationEnabled, serverName)
+                    );
             }
         }
     }
@@ -217,8 +258,24 @@ public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, Sock
                 sslParameters.setServerNames(Collections.singletonList(serverName));
                 sslEngine.setSSLParameters(sslParameters);
             }
-            ctx.pipeline().replace(this, "ssl", new SslHandler(sslEngine));
-            super.connect(ctx, remoteAddress, localAddress, promise);
+            final ChannelPromise connectPromise = ctx.newPromise();
+            final SslHandler sslHandler = new SslHandler(sslEngine);
+            ctx.pipeline().replace(this, "ssl", sslHandler);
+            final Future<?> handshakePromise = sslHandler.handshakeFuture();
+            connectPromise.addListener(result -> {
+                if (result.isSuccess() == false) {
+                    promise.tryFailure(result.cause());
+                } else {
+                    handshakePromise.addListener(handshakeResult -> {
+                        if (handshakeResult.isSuccess()) {
+                            promise.setSuccess();
+                        } else {
+                            promise.tryFailure(handshakeResult.cause());
+                        }
+                    });
+                }
+            });
+            super.connect(ctx, remoteAddress, localAddress, connectPromise);
         }
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/InitialNodeSecurityAutoConfiguration.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/InitialNodeSecurityAutoConfiguration.java
index 3585fe01d70c..866a3bf1bad4 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/InitialNodeSecurityAutoConfiguration.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/InitialNodeSecurityAutoConfiguration.java
@@ -120,6 +120,7 @@ protected void doRun() {
                         }
                         final String httpsCaFingerprint = fingerprint;
                         GroupedActionListener<Map<String, String>> groupedActionListener = new GroupedActionListener<>(
+                            3,
                             ActionListener.wrap(results -> {
                                 final Map<String, String> allResultsMap = new HashMap<>();
                                 for (Map<String, String> result : results) {
@@ -135,8 +136,7 @@ protected void doRun() {
                                     httpsCaFingerprint,
                                     console
                                 );
-                            }, e -> LOGGER.error("Unexpected exception during security auto-configuration", e)),
-                            3
+                            }, e -> LOGGER.error("Unexpected exception during security auto-configuration", e))
                         );
                         // we only generate the elastic user password if the node has been auto-configured in a specific way, such that the
                         // first time a node starts it will form a cluster by itself and can hold the .security index (which we assume
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
index 3c9e48d78de2..edabbd2b2723 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -9,7 +9,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.SetOnce;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -22,7 +22,7 @@
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.Strings;
@@ -38,6 +38,7 @@
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.ssl.KeyStoreUtil;
 import org.elasticsearch.common.ssl.SslConfiguration;
+import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -49,6 +50,7 @@
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.NodeMetadata;
 import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.http.netty4.Netty4HttpServerTransport;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
@@ -82,7 +84,9 @@
 import org.elasticsearch.transport.TransportInterceptor;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestHandler;
+import org.elasticsearch.transport.netty4.AcceptChannelHandler;
 import org.elasticsearch.transport.netty4.SharedGroupFactory;
+import org.elasticsearch.transport.netty4.TLSConfig;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xpack.core.XPackField;
@@ -156,7 +160,6 @@
 import org.elasticsearch.xpack.core.security.authc.Realm;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
-import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmsServiceSettings;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
@@ -260,6 +263,7 @@
 import org.elasticsearch.xpack.security.authz.interceptor.IndicesAliasesRequestInterceptor;
 import org.elasticsearch.xpack.security.authz.interceptor.RequestInterceptor;
 import org.elasticsearch.xpack.security.authz.interceptor.ResizeRequestInterceptor;
+import org.elasticsearch.xpack.security.authz.interceptor.SearchRequestCacheDisablingInterceptor;
 import org.elasticsearch.xpack.security.authz.interceptor.SearchRequestInterceptor;
 import org.elasticsearch.xpack.security.authz.interceptor.ShardSearchRequestInterceptor;
 import org.elasticsearch.xpack.security.authz.interceptor.UpdateRequestInterceptor;
@@ -338,10 +342,10 @@
 import org.elasticsearch.xpack.security.transport.SecurityHttpSettings;
 import org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor;
 import org.elasticsearch.xpack.security.transport.filter.IPFilter;
-import org.elasticsearch.xpack.security.transport.netty4.SecurityNetty4HttpServerTransport;
 import org.elasticsearch.xpack.security.transport.netty4.SecurityNetty4ServerTransport;
 
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.time.Clock;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -560,7 +564,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver expressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         try {
             return createComponents(
@@ -865,7 +869,8 @@ Collection<Object> createComponents(
                     new ShardSearchRequestInterceptor(threadPool, getLicenseState(), clusterService),
                     new UpdateRequestInterceptor(threadPool, getLicenseState()),
                     new BulkShardRequestInterceptor(threadPool, getLicenseState()),
-                    new DlsFlsLicenseRequestInterceptor(threadPool.getThreadContext(), getLicenseState())
+                    new DlsFlsLicenseRequestInterceptor(threadPool.getThreadContext(), getLicenseState()),
+                    new SearchRequestCacheDisablingInterceptor(threadPool, getLicenseState())
                 )
             );
         }
@@ -933,6 +938,7 @@ Collection<Object> createComponents(
         components.add(new SecurityUsageServices(realms, allRolesStore, nativeRoleMappingStore, ipFilter.get(), profileService));
 
         reservedRoleMappingAction.set(new ReservedRoleMappingAction(nativeRoleMappingStore));
+        systemIndices.getMainIndexManager().onStateRecovered(state -> reservedRoleMappingAction.get().securityIndexRecovered());
 
         cacheInvalidatorRegistry.validate();
 
@@ -1089,7 +1095,6 @@ public static List<Setting<?>> getSettings(List<SecurityExtension> securityExten
         // authentication and authorization settings
         AnonymousUser.addSettings(settingsList);
         settingsList.addAll(InternalRealmsSettings.getSettings());
-        settingsList.addAll(JwtRealmsServiceSettings.getSettings());
         ReservedRealm.addSettings(settingsList);
         AuthenticationService.addSettings(settingsList);
         AuthorizationService.addSettings(settingsList);
@@ -1104,6 +1109,7 @@ public static List<Setting<?>> getSettings(List<SecurityExtension> securityExten
         settingsList.add(ApiKeyService.PASSWORD_HASHING_ALGORITHM);
         settingsList.add(ApiKeyService.DELETE_TIMEOUT);
         settingsList.add(ApiKeyService.DELETE_INTERVAL);
+        settingsList.add(ApiKeyService.DELETE_RETENTION_PERIOD);
         settingsList.add(ApiKeyService.CACHE_HASH_ALGO_SETTING);
         settingsList.add(ApiKeyService.CACHE_MAX_KEYS_SETTING);
         settingsList.add(ApiKeyService.CACHE_TTL_SETTING);
@@ -1519,7 +1525,7 @@ public Map<String, Supplier<Transport>> getTransports(
                 transportReference.set(
                     new SecurityNetty4ServerTransport(
                         settings,
-                        Version.CURRENT,
+                        TransportVersion.CURRENT,
                         threadPool,
                         networkService,
                         pageCacheRecycler,
@@ -1552,24 +1558,48 @@ public Map<String, Supplier<HttpServerTransport>> getHttpTransports(
             return Collections.emptyMap();
         }
 
+        final IPFilter ipFilter = this.ipFilter.get();
+        final AcceptChannelHandler.AcceptPredicate acceptPredicate = new AcceptChannelHandler.AcceptPredicate() {
+            @Override
+            public void setBoundAddress(BoundTransportAddress boundHttpTransportAddress) {
+                ipFilter.setBoundHttpTransportAddress(boundHttpTransportAddress);
+            }
+
+            @Override
+            public boolean test(String profile, InetSocketAddress peerAddress) {
+                return ipFilter.accept(profile, peerAddress);
+            }
+        };
+
         Map<String, Supplier<HttpServerTransport>> httpTransports = new HashMap<>();
-        httpTransports.put(
-            SecurityField.NAME4,
-            () -> new SecurityNetty4HttpServerTransport(
+        httpTransports.put(SecurityField.NAME4, () -> {
+            final boolean ssl = HTTP_SSL_ENABLED.get(settings);
+            SSLService sslService = getSslService();
+            final SslConfiguration sslConfiguration;
+            if (ssl) {
+                sslConfiguration = sslService.getHttpTransportSSLConfiguration();
+                if (SSLService.isConfigurationValidForServerUsage(sslConfiguration) == false) {
+                    throw new IllegalArgumentException(
+                        "a key must be provided to run as a server. the key should be configured using the "
+                            + "[xpack.security.http.ssl.key] or [xpack.security.http.ssl.keystore.path] setting"
+                    );
+                }
+            } else {
+                sslConfiguration = null;
+            }
+            return new Netty4HttpServerTransport(
                 settings,
                 networkService,
-                pageCacheRecycler,
-                ipFilter.get(),
-                getSslService(),
                 threadPool,
                 xContentRegistry,
                 dispatcher,
                 clusterSettings,
                 getNettySharedGroupFactory(settings),
-                tracer
-            )
-        );
-
+                tracer,
+                new TLSConfig(sslConfiguration, sslService::createSSLEngine),
+                acceptPredicate
+            );
+        });
         return httpTransports;
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageTransportAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageTransportAction.java
index f6a02be8ebd1..09068c50b016 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageTransportAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageTransportAction.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.RemoteClusterPortSettings;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
@@ -44,6 +45,7 @@
 import static org.elasticsearch.xpack.core.XPackSettings.API_KEY_SERVICE_ENABLED_SETTING;
 import static org.elasticsearch.xpack.core.XPackSettings.FIPS_MODE_ENABLED;
 import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED;
+import static org.elasticsearch.xpack.core.XPackSettings.REMOTE_CLUSTER_SSL_ENABLED;
 import static org.elasticsearch.xpack.core.XPackSettings.TOKEN_SERVICE_ENABLED_SETTING;
 import static org.elasticsearch.xpack.core.XPackSettings.TRANSPORT_SSL_ENABLED;
 
@@ -188,6 +190,9 @@ static Map<String, Object> sslUsage(Settings settings) {
             Map<String, Object> map = Maps.newMapWithExpectedSize(2);
             map.put("http", singletonMap("enabled", HTTP_SSL_ENABLED.get(settings)));
             map.put("transport", singletonMap("enabled", TRANSPORT_SSL_ENABLED.get(settings)));
+            if (RemoteClusterPortSettings.REMOTE_CLUSTER_PORT_ENABLED.get(settings)) {
+                map.put("remote_cluster", singletonMap("enabled", REMOTE_CLUSTER_SSL_ENABLED.get(settings)));
+            }
             return map;
         } else {
             return Collections.emptyMap();
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilter.java
index 313a04301146..e8cb854c80ce 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilter.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilter.java
@@ -8,7 +8,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -106,7 +106,7 @@ operations are blocked on license expiration. All data operations (read and writ
                 AuthorizationUtils.switchUserBasedOnActionOriginAndExecute(
                     threadContext,
                     securityContext,
-                    Version.CURRENT, // current version since this is on the same node
+                    TransportVersion.CURRENT, // current version since this is on the same node
                     (original) -> { applyInternal(task, chain, action, request, contextPreservingListener); }
                 );
             } else {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/ReservedRoleMappingAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/ReservedRoleMappingAction.java
index 74e0cc081ace..2328c8478deb 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/ReservedRoleMappingAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/ReservedRoleMappingAction.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.GroupedActionListener;
+import org.elasticsearch.common.util.concurrent.ListenableFuture;
 import org.elasticsearch.reservedstate.NonStateTransformResult;
 import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
 import org.elasticsearch.reservedstate.TransformState;
@@ -41,6 +42,7 @@ public class ReservedRoleMappingAction implements ReservedClusterStateHandler<Li
     public static final String NAME = "role_mappings";
 
     private final NativeRoleMappingStore roleMappingStore;
+    private final ListenableFuture<Void> securityIndexRecoveryListener = new ListenableFuture<>();
 
     /**
      * Creates a ReservedRoleMappingAction
@@ -84,10 +86,17 @@ public TransformState transform(Object source, TransformState prevState) throws
         // non cluster state transform call.
         @SuppressWarnings("unchecked")
         var requests = prepare((List<ExpressionRoleMapping>) source);
-        return new TransformState(prevState.state(), prevState.keys(), l -> nonStateTransform(requests, prevState, l));
+        return new TransformState(
+            prevState.state(),
+            prevState.keys(),
+            l -> securityIndexRecoveryListener.addListener(
+                ActionListener.wrap(ignored -> nonStateTransform(requests, prevState, l), l::onFailure)
+            )
+        );
     }
 
-    private void nonStateTransform(
+    // Exposed for testing purposes
+    protected void nonStateTransform(
         Collection<PutRoleMappingRequest> requests,
         TransformState prevState,
         ActionListener<NonStateTransformResult> listener
@@ -104,7 +113,7 @@ private void nonStateTransform(
             return;
         }
 
-        GroupedActionListener<Boolean> taskListener = new GroupedActionListener<>(new ActionListener<>() {
+        GroupedActionListener<Boolean> taskListener = new GroupedActionListener<>(tasksCount, new ActionListener<>() {
             @Override
             public void onResponse(Collection<Boolean> booleans) {
                 listener.onResponse(new NonStateTransformResult(ReservedRoleMappingAction.NAME, Collections.unmodifiableSet(entities)));
@@ -114,7 +123,7 @@ public void onResponse(Collection<Boolean> booleans) {
             public void onFailure(Exception e) {
                 listener.onFailure(e);
             }
-        }, tasksCount);
+        });
 
         for (var request : requests) {
             roleMappingStore.putRoleMapping(request, taskListener);
@@ -144,4 +153,8 @@ public List<ExpressionRoleMapping> fromXContent(XContentParser parser) throws IO
 
         return result;
     }
+
+    public void securityIndexRecovered() {
+        securityIndexRecoveryListener.onResponse(null);
+    }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionAction.java
index 6806accc046b..d92eaf115c87 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionAction.java
@@ -113,8 +113,8 @@ private void findAndInvalidateTokens(SamlRealm realm, SamlLogoutRequestHandler.R
                 listener.onResponse(0);
             } else {
                 GroupedActionListener<TokensInvalidationResult> groupedListener = new GroupedActionListener<>(
-                    ActionListener.wrap(collection -> listener.onResponse(collection.size()), listener::onFailure),
-                    tokens.size()
+                    tokens.size(),
+                    ActionListener.wrap(collection -> listener.onResponse(collection.size()), listener::onFailure)
                 );
                 tokens.forEach(tuple -> invalidateTokenPair(tuple, groupedListener));
             }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordAction.java
index d734721a3314..89c06c1b5de5 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordAction.java
@@ -21,6 +21,8 @@
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
 
+import java.util.Locale;
+
 public class TransportChangePasswordAction extends HandledTransportAction<ChangePasswordRequest, ActionResponse.Empty> {
 
     private final Settings settings;
@@ -45,17 +47,15 @@ protected void doExecute(Task task, ChangePasswordRequest request, ActionListene
             listener.onFailure(new IllegalArgumentException("user [" + username + "] is anonymous and cannot be modified via the API"));
             return;
         }
-        final String requestPwdHashAlgo = Hasher.resolveFromHash(request.passwordHash()).name();
-        final String configPwdHashAlgo = Hasher.resolve(XPackSettings.PASSWORD_HASHING_ALGORITHM.get(settings)).name();
-        if (requestPwdHashAlgo.equalsIgnoreCase(configPwdHashAlgo) == false) {
+        final Hasher requestPwdHashAlgo = Hasher.resolveFromHash(request.passwordHash());
+        final Hasher configPwdHashAlgo = Hasher.resolve(XPackSettings.PASSWORD_HASHING_ALGORITHM.get(settings));
+        if (requestPwdHashAlgo.equals(configPwdHashAlgo) == false
+            && Hasher.getAvailableAlgoStoredHash().contains(requestPwdHashAlgo.name().toLowerCase(Locale.ROOT)) == false) {
             listener.onFailure(
                 new IllegalArgumentException(
-                    "incorrect password hashing algorithm ["
-                        + requestPwdHashAlgo
-                        + "] used while"
-                        + " ["
-                        + configPwdHashAlgo
-                        + "] is configured."
+                    "The provided password hash is not a hash or it could not be resolved to a supported hash algorithm. "
+                        + "The supported password hash algorithms are "
+                        + Hasher.getAvailableAlgoStoredHash().toString()
                 )
             );
             return;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersAction.java
index ea646fcb68ce..2cfb1597f3b6 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersAction.java
@@ -103,7 +103,7 @@ protected void doExecute(Task task, final GetUsersRequest request, final ActionL
             }
 
         }, listener::onFailure);
-        final GroupedActionListener<Collection<User>> groupListener = new GroupedActionListener<>(sendingListener, 2);
+        final GroupedActionListener<Collection<User>> groupListener = new GroupedActionListener<>(2, sendingListener);
         // We have two sources for the users object, the reservedRealm and the usersStore, we query both at the same time with a
         // GroupedActionListener
         if (realmLookup.isEmpty()) {
@@ -117,7 +117,7 @@ protected void doExecute(Task task, final GetUsersRequest request, final ActionL
         } else {
             // nested group listener action here - for each of the users we got and fetch it concurrently - once we are done we notify
             // the "global" group listener.
-            GroupedActionListener<User> realmGroupListener = new GroupedActionListener<>(groupListener, realmLookup.size());
+            GroupedActionListener<User> realmGroupListener = new GroupedActionListener<>(realmLookup.size(), groupListener);
             for (String user : realmLookup) {
                 reservedRealm.lookupUser(user, realmGroupListener);
             }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java
index ae200aa7c4ae..163e8dd48c10 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java
@@ -1606,6 +1606,8 @@ private void setThreadContextField(ThreadContext threadContext, String threadCon
         }
 
         LogEntryBuilder withAuthentication(Authentication authentication) {
+            // TODO auditing for remote access
+            assert false == authentication.isRemoteAccess() : "remote access authentication not supported here yet";
             logEntry.with(PRINCIPAL_FIELD_NAME, authentication.getEffectiveSubject().getUser().principal());
             logEntry.with(AUTHENTICATION_TYPE_FIELD_NAME, authentication.getAuthenticationType().toString());
             if (authentication.isApiKey()) {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
index 64dcfc683ac6..805922e48138 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
@@ -163,6 +163,12 @@ public class ApiKeyService {
         TimeValue.timeValueHours(24L),
         Property.NodeScope
     );
+    public static final Setting<TimeValue> DELETE_RETENTION_PERIOD = Setting.positiveTimeSetting(
+        "xpack.security.authc.api_key.delete.retention_period",
+        TimeValue.timeValueDays(7),
+        Property.NodeScope,
+        Property.Dynamic
+    );
     public static final Setting<String> CACHE_HASH_ALGO_SETTING = Setting.simpleString(
         "xpack.security.authc.api_key.cache.hash_algo",
         "ssha256",
@@ -193,7 +199,7 @@ public class ApiKeyService {
     private final Hasher hasher;
     private final boolean enabled;
     private final Settings settings;
-    private final ExpiredApiKeysRemover expiredApiKeysRemover;
+    private final InactiveApiKeysRemover inactiveApiKeysRemover;
     private final TimeValue deleteInterval;
     private final Cache<String, ListenableFuture<CachedApiKeyHashResult>> apiKeyAuthCache;
     private final Hasher cacheHasher;
@@ -225,7 +231,7 @@ public ApiKeyService(
         this.hasher = Hasher.resolve(PASSWORD_HASHING_ALGORITHM.get(settings));
         this.settings = settings;
         this.deleteInterval = DELETE_INTERVAL.get(settings);
-        this.expiredApiKeysRemover = new ExpiredApiKeysRemover(settings, client);
+        this.inactiveApiKeysRemover = new InactiveApiKeysRemover(settings, client, clusterService);
         this.threadPool = threadPool;
         this.cacheHasher = Hasher.resolve(CACHE_HASH_ALGO_SETTING.get(settings));
         final TimeValue ttl = CACHE_TTL_SETTING.get(settings);
@@ -1363,9 +1369,10 @@ private void indexInvalidation(Collection<String> apiKeyIds, ActionListener<Inva
             listener.onFailure(new ElasticsearchSecurityException("No api key ids provided for invalidation"));
         } else {
             BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
+            final long invalidationTime = clock.instant().toEpochMilli();
             for (String apiKeyId : apiKeyIds) {
                 UpdateRequest request = client.prepareUpdate(SECURITY_MAIN_ALIAS, apiKeyId)
-                    .setDoc(Collections.singletonMap("api_key_invalidated", true))
+                    .setDoc(Map.of("api_key_invalidated", true, "invalidation_time", invalidationTime))
                     .request();
                 bulkRequestBuilder.add(request);
             }
@@ -1553,7 +1560,7 @@ private static <E extends Throwable> E traceLog(String action, E exception) {
 
     // pkg scoped for testing
     boolean isExpirationInProgress() {
-        return expiredApiKeysRemover.isExpirationInProgress();
+        return inactiveApiKeysRemover.isExpirationInProgress();
     }
 
     // pkg scoped for testing
@@ -1564,7 +1571,7 @@ long lastTimeWhenApiKeysRemoverWasTriggered() {
     private void maybeStartApiKeyRemover() {
         if (securityIndex.isAvailable()) {
             if (client.threadPool().relativeTimeInMillis() - lastExpirationRunMs > deleteInterval.getMillis()) {
-                expiredApiKeysRemover.submit(client.threadPool());
+                inactiveApiKeysRemover.submit(client.threadPool());
                 lastExpirationRunMs = client.threadPool().relativeTimeInMillis();
             }
         }
@@ -1747,7 +1754,11 @@ public static String getCreatorRealmName(final Authentication authentication) {
         } else {
             // TODO we should use the effective subject realm here but need to handle the failed lookup scenario, in which the realm may be
             // `null`. Since this method is used in audit logging, this requires some care.
-            return authentication.getSourceRealm().getName();
+            if (authentication.isFailedRunAs()) {
+                return authentication.getAuthenticatingSubject().getRealm().getName();
+            } else {
+                return authentication.getEffectiveSubject().getRealm().getName();
+            }
         }
     }
 
@@ -1790,7 +1801,11 @@ public static String getCreatorRealmType(final Authentication authentication) {
         } else {
             // TODO we should use the effective subject realm here but need to handle the failed lookup scenario, in which the realm may be
             // `null`. Since this method is used in audit logging, this requires some care.
-            return authentication.getSourceRealm().getType();
+            if (authentication.isFailedRunAs()) {
+                return authentication.getAuthenticatingSubject().getRealm().getType();
+            } else {
+                return authentication.getEffectiveSubject().getRealm().getType();
+            }
         }
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredApiKeysRemover.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredApiKeysRemover.java
deleted file mode 100644
index bec3e94a3c73..000000000000
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredApiKeysRemover.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.security.authc;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.bulk.BulkItemResponse;
-import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.reindex.BulkByScrollResponse;
-import org.elasticsearch.index.reindex.DeleteByQueryAction;
-import org.elasticsearch.index.reindex.DeleteByQueryRequest;
-import org.elasticsearch.index.reindex.ScrollableHitSource;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.threadpool.ThreadPool.Names;
-import org.elasticsearch.xpack.security.support.SecuritySystemIndices;
-
-import java.time.Duration;
-import java.time.Instant;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import static org.elasticsearch.action.support.TransportActions.isShardNotAvailableException;
-import static org.elasticsearch.core.Strings.format;
-import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN;
-import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
-
-/**
- * Responsible for cleaning the invalidated and expired API keys from the security index.
- */
-public final class ExpiredApiKeysRemover extends AbstractRunnable {
-    public static final Duration EXPIRED_API_KEYS_RETENTION_PERIOD = Duration.ofDays(7L);
-
-    private static final Logger logger = LogManager.getLogger(ExpiredApiKeysRemover.class);
-
-    private final Client client;
-    private final AtomicBoolean inProgress = new AtomicBoolean(false);
-    private final TimeValue timeout;
-
-    ExpiredApiKeysRemover(Settings settings, Client client) {
-        this.client = client;
-        this.timeout = ApiKeyService.DELETE_TIMEOUT.get(settings);
-    }
-
-    @Override
-    public void doRun() {
-        DeleteByQueryRequest expiredDbq = new DeleteByQueryRequest(SecuritySystemIndices.SECURITY_MAIN_ALIAS);
-        if (timeout != TimeValue.MINUS_ONE) {
-            expiredDbq.setTimeout(timeout);
-            expiredDbq.getSearchRequest().source().timeout(timeout);
-        }
-        final Instant now = Instant.now();
-        expiredDbq.setQuery(
-            QueryBuilders.boolQuery()
-                .filter(QueryBuilders.termsQuery("doc_type", "api_key"))
-                .should(QueryBuilders.termsQuery("api_key_invalidated", true))
-                .should(QueryBuilders.rangeQuery("expiration_time").lte(now.minus(EXPIRED_API_KEYS_RETENTION_PERIOD).toEpochMilli()))
-                .minimumShouldMatch(1)
-        );
-
-        executeAsyncWithOrigin(client, SECURITY_ORIGIN, DeleteByQueryAction.INSTANCE, expiredDbq, ActionListener.wrap(r -> {
-            debugDbqResponse(r);
-            markComplete();
-        }, this::onFailure));
-    }
-
-    void submit(ThreadPool threadPool) {
-        if (inProgress.compareAndSet(false, true)) {
-            threadPool.executor(Names.GENERIC).submit(this);
-        }
-    }
-
-    private static void debugDbqResponse(BulkByScrollResponse response) {
-        if (logger.isDebugEnabled()) {
-            logger.debug(
-                "delete by query of api keys finished with [{}] deletions, [{}] bulk failures, [{}] search failures",
-                response.getDeleted(),
-                response.getBulkFailures().size(),
-                response.getSearchFailures().size()
-            );
-            for (BulkItemResponse.Failure failure : response.getBulkFailures()) {
-                logger.debug(
-                    () -> format("deletion failed for index [%s], id [%s]", failure.getIndex(), failure.getId()),
-                    failure.getCause()
-                );
-            }
-            for (ScrollableHitSource.SearchFailure failure : response.getSearchFailures()) {
-                logger.debug(
-                    () -> format(
-                        "search failed for index [%s], shard [%s] on node [%s]",
-                        failure.getIndex(),
-                        failure.getShardId(),
-                        failure.getNodeId()
-                    ),
-                    failure.getReason()
-                );
-            }
-        }
-    }
-
-    boolean isExpirationInProgress() {
-        return inProgress.get();
-    }
-
-    @Override
-    public void onFailure(Exception e) {
-        if (isShardNotAvailableException(e)) {
-            logger.debug("failed to delete expired or invalidated api keys", e);
-        } else {
-            logger.error("failed to delete expired or invalidated api keys", e);
-        }
-        markComplete();
-    }
-
-    private void markComplete() {
-        if (inProgress.compareAndSet(true, false) == false) {
-            throw new IllegalStateException("in progress was set to false but should have been true!");
-        }
-    }
-
-}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InactiveApiKeysRemover.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InactiveApiKeysRemover.java
new file mode 100644
index 000000000000..1d34371f6186
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InactiveApiKeysRemover.java
@@ -0,0 +1,142 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authc;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.reindex.BulkByScrollResponse;
+import org.elasticsearch.index.reindex.DeleteByQueryAction;
+import org.elasticsearch.index.reindex.DeleteByQueryRequest;
+import org.elasticsearch.index.reindex.ScrollableHitSource;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.threadpool.ThreadPool.Names;
+import org.elasticsearch.xpack.security.support.SecuritySystemIndices;
+
+import java.time.Instant;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.elasticsearch.action.support.TransportActions.isShardNotAvailableException;
+import static org.elasticsearch.core.Strings.format;
+import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN;
+import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
+
+/**
+ * Responsible for cleaning the invalidated and expired API keys from the security index.
+ */
+public final class InactiveApiKeysRemover extends AbstractRunnable {
+    private static final Logger logger = LogManager.getLogger(InactiveApiKeysRemover.class);
+
+    private final Client client;
+    private final AtomicBoolean inProgress = new AtomicBoolean(false);
+    private final TimeValue timeout;
+    private final AtomicLong retentionPeriodInMs;
+
+    InactiveApiKeysRemover(Settings settings, Client client, ClusterService clusterService) {
+        this.client = client;
+        this.timeout = ApiKeyService.DELETE_TIMEOUT.get(settings);
+        this.retentionPeriodInMs = new AtomicLong(ApiKeyService.DELETE_RETENTION_PERIOD.get(settings).getMillis());
+        clusterService.getClusterSettings()
+            .addSettingsUpdateConsumer(
+                ApiKeyService.DELETE_RETENTION_PERIOD,
+                newRetentionPeriod -> this.retentionPeriodInMs.set(newRetentionPeriod.getMillis())
+            );
+    }
+
+    @Override
+    public void doRun() {
+        DeleteByQueryRequest expiredDbq = new DeleteByQueryRequest(SecuritySystemIndices.SECURITY_MAIN_ALIAS);
+        if (timeout != TimeValue.MINUS_ONE) {
+            expiredDbq.setTimeout(timeout);
+            expiredDbq.getSearchRequest().source().timeout(timeout);
+        }
+        final Instant now = Instant.now();
+        final long cutoffTimestamp = now.minusMillis(retentionPeriodInMs.get()).toEpochMilli();
+        expiredDbq.setQuery(
+            QueryBuilders.boolQuery()
+                .filter(QueryBuilders.termsQuery("doc_type", "api_key"))
+                .should(QueryBuilders.rangeQuery("expiration_time").lte(cutoffTimestamp))
+                .should(
+                    QueryBuilders.boolQuery()
+                        .filter(QueryBuilders.termsQuery("api_key_invalidated", true))
+                        .should(QueryBuilders.rangeQuery("invalidation_time").lte(cutoffTimestamp))
+                        .should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("invalidation_time")))
+                        .minimumShouldMatch(1)
+                )
+                .minimumShouldMatch(1)
+        );
+
+        executeAsyncWithOrigin(client, SECURITY_ORIGIN, DeleteByQueryAction.INSTANCE, expiredDbq, ActionListener.wrap(r -> {
+            debugDbqResponse(r);
+            markComplete();
+        }, this::onFailure));
+    }
+
+    void submit(ThreadPool threadPool) {
+        if (inProgress.compareAndSet(false, true)) {
+            threadPool.executor(Names.GENERIC).submit(this);
+        }
+    }
+
+    private static void debugDbqResponse(BulkByScrollResponse response) {
+        if (logger.isDebugEnabled()) {
+            logger.debug(
+                "delete by query of api keys finished with [{}] deletions, [{}] bulk failures, [{}] search failures",
+                response.getDeleted(),
+                response.getBulkFailures().size(),
+                response.getSearchFailures().size()
+            );
+            for (BulkItemResponse.Failure failure : response.getBulkFailures()) {
+                logger.debug(
+                    () -> format("deletion failed for index [%s], id [%s]", failure.getIndex(), failure.getId()),
+                    failure.getCause()
+                );
+            }
+            for (ScrollableHitSource.SearchFailure failure : response.getSearchFailures()) {
+                logger.debug(
+                    () -> format(
+                        "search failed for index [%s], shard [%s] on node [%s]",
+                        failure.getIndex(),
+                        failure.getShardId(),
+                        failure.getNodeId()
+                    ),
+                    failure.getReason()
+                );
+            }
+        }
+    }
+
+    boolean isExpirationInProgress() {
+        return inProgress.get();
+    }
+
+    @Override
+    public void onFailure(Exception e) {
+        if (isShardNotAvailableException(e)) {
+            logger.debug("failed to delete expired or invalidated api keys", e);
+        } else {
+            logger.error("failed to delete expired or invalidated api keys", e);
+        }
+        markComplete();
+    }
+
+    private void markComplete() {
+        if (inProgress.compareAndSet(true, false) == false) {
+            throw new IllegalStateException("in progress was set to false but should have been true!");
+        }
+    }
+
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InternalRealms.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InternalRealms.java
index 9e697baa76de..2cc45791286f 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InternalRealms.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InternalRealms.java
@@ -32,7 +32,7 @@
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
 import org.elasticsearch.xpack.security.authc.file.FileRealm;
-import org.elasticsearch.xpack.security.authc.jwt.JwtRealmsService;
+import org.elasticsearch.xpack.security.authc.jwt.JwtRealm;
 import org.elasticsearch.xpack.security.authc.kerberos.KerberosRealm;
 import org.elasticsearch.xpack.security.authc.ldap.LdapRealm;
 import org.elasticsearch.xpack.security.authc.oidc.OpenIdConnectRealm;
@@ -137,7 +137,6 @@ public static Map<String, Realm.Factory> getFactories(
         NativeRoleMappingStore nativeRoleMappingStore,
         SecurityIndexManager securityIndex
     ) {
-        final JwtRealmsService jwtRealmsService = new JwtRealmsService(settings); // parse shared settings needed by all JwtRealm instances
         return Map.of(
             // file realm
             FileRealmSettings.TYPE,
@@ -169,7 +168,7 @@ public static Map<String, Realm.Factory> getFactories(
             config -> new OpenIdConnectRealm(config, sslService, nativeRoleMappingStore, resourceWatcherService),
             // JWT realm
             JwtRealmSettings.TYPE,
-            config -> jwtRealmsService.createJwtRealm(config, sslService, nativeRoleMappingStore)
+            config -> new JwtRealm(config, sslService, nativeRoleMappingStore)
         );
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java
index 6db4f703cec0..cce3f8501be1 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Realms.java
@@ -9,6 +9,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
@@ -16,7 +17,6 @@
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.Maps;
-import org.elasticsearch.common.util.concurrent.CountDown;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Nullable;
@@ -309,9 +309,8 @@ public void usageStats(ActionListener<Map<String, Object>> listener) {
         Map<String, Object> realmMap = new HashMap<>();
         final AtomicBoolean failed = new AtomicBoolean(false);
         final List<Realm> realmList = getActiveRealms().stream().filter(r -> ReservedRealm.TYPE.equals(r.type()) == false).toList();
-        final CountDown countDown = new CountDown(realmList.size());
-        final Runnable doCountDown = () -> {
-            if ((realmList.isEmpty() || countDown.countDown()) && failed.get() == false) {
+        try (var refs = new RefCountingRunnable(() -> {
+            if (failed.get() == false) {
                 // iterate over the factories so we can add enabled & available info
                 for (String type : factories.keySet()) {
                     assert ReservedRealm.TYPE.equals(type) == false;
@@ -333,12 +332,9 @@ public void usageStats(ActionListener<Map<String, Object>> listener) {
                 }
                 listener.onResponse(realmMap);
             }
-        };
-
-        if (realmList.isEmpty()) {
-            doCountDown.run();
-        } else {
+        })) {
             for (Realm realm : realmList) {
+                final var ref = refs.acquire();
                 realm.usageStats(ActionListener.wrap(stats -> {
                     if (failed.get() == false) {
                         synchronized (realmMap) {
@@ -352,7 +348,7 @@ public void usageStats(ActionListener<Map<String, Object>> listener) {
                                 return value;
                             });
                         }
-                        doCountDown.run();
+                        ref.close();
                     }
                 }, e -> {
                     if (failed.compareAndSet(false, true)) {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
index 993dd666702b..a0df6288f2f4 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
@@ -15,7 +15,7 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteRequest.OpType;
 import org.elasticsearch.action.DocWriteResponse;
@@ -203,11 +203,11 @@ public final class TokenService {
     static final int MINIMUM_BYTES = VERSION_BYTES + TOKEN_LENGTH + 1;
     static final int LEGACY_MINIMUM_BASE64_BYTES = Double.valueOf(Math.ceil((4 * LEGACY_MINIMUM_BYTES) / 3)).intValue();
     public static final int MINIMUM_BASE64_BYTES = Double.valueOf(Math.ceil((4 * MINIMUM_BYTES) / 3)).intValue();
-    static final Version VERSION_HASHED_TOKENS = Version.V_7_2_0;
-    static final Version VERSION_TOKENS_INDEX_INTRODUCED = Version.V_7_2_0;
-    static final Version VERSION_ACCESS_TOKENS_AS_UUIDS = Version.V_7_2_0;
-    static final Version VERSION_MULTIPLE_CONCURRENT_REFRESHES = Version.V_7_2_0;
-    static final Version VERSION_CLIENT_AUTH_FOR_REFRESH = Version.V_8_2_0;
+    static final TransportVersion VERSION_HASHED_TOKENS = TransportVersion.V_7_2_0;
+    static final TransportVersion VERSION_TOKENS_INDEX_INTRODUCED = TransportVersion.V_7_2_0;
+    static final TransportVersion VERSION_ACCESS_TOKENS_AS_UUIDS = TransportVersion.V_7_2_0;
+    static final TransportVersion VERSION_MULTIPLE_CONCURRENT_REFRESHES = TransportVersion.V_7_2_0;
+    static final TransportVersion VERSION_CLIENT_AUTH_FOR_REFRESH = TransportVersion.V_8_2_0;
 
     private static final Logger logger = LogManager.getLogger(TokenService.class);
 
@@ -280,7 +280,7 @@ public void createOAuth2Tokens(
         ActionListener<CreateTokenResult> listener
     ) {
         // the created token is compatible with the oldest node version in the cluster
-        final Version tokenVersion = getTokenVersionCompatibility();
+        final TransportVersion tokenVersion = getTokenVersionCompatibility();
         // tokens moved to a separate index in newer versions
         final SecurityIndexManager tokensIndex = getTokensIndexForVersion(tokenVersion);
         // the id of the created tokens ought be unguessable
@@ -304,7 +304,7 @@ public void createOAuth2Tokens(
         ActionListener<CreateTokenResult> listener
     ) {
         // the created token is compatible with the oldest node version in the cluster
-        final Version tokenVersion = getTokenVersionCompatibility();
+        final TransportVersion tokenVersion = getTokenVersionCompatibility();
         // tokens moved to a separate index in newer versions
         final SecurityIndexManager tokensIndex = getTokensIndexForVersion(tokenVersion);
         createOAuth2Tokens(accessToken, refreshToken, tokenVersion, tokensIndex, authentication, originatingClientAuth, metadata, listener);
@@ -342,7 +342,7 @@ public void createOAuth2Tokens(
     private void createOAuth2Tokens(
         String accessToken,
         String refreshToken,
-        Version tokenVersion,
+        TransportVersion tokenVersion,
         SecurityIndexManager tokensIndex,
         Authentication authentication,
         Authentication originatingClientAuth,
@@ -451,7 +451,7 @@ public void getAuthenticationAndMetadata(String token, ActionListener<Tuple<Auth
      * Gets the {@link UserToken} with the given {@code userTokenId} and {@code tokenVersion} by fetching and parsing the corresponding
      * token document.
      */
-    private void getUserTokenFromId(String userTokenId, Version tokenVersion, ActionListener<UserToken> listener) {
+    private void getUserTokenFromId(String userTokenId, TransportVersion tokenVersion, ActionListener<UserToken> listener) {
         final SecurityIndexManager tokensIndex = getTokensIndexForVersion(tokenVersion);
         final SecurityIndexManager frozenTokensIndex = tokensIndex.freeze();
         if (frozenTokensIndex.isAvailable() == false) {
@@ -533,8 +533,8 @@ private void decodeAndValidateToken(SecureString tokenString, ActionListener<Use
     void decodeToken(String token, ActionListener<UserToken> listener) {
         final byte[] bytes = token.getBytes(StandardCharsets.UTF_8);
         try (StreamInput in = new InputStreamStreamInput(Base64.getDecoder().wrap(new ByteArrayInputStream(bytes)), bytes.length)) {
-            final Version version = Version.readVersion(in);
-            in.setVersion(version);
+            final TransportVersion version = TransportVersion.readVersion(in);
+            in.setTransportVersion(version);
             if (version.onOrAfter(VERSION_ACCESS_TOKENS_AS_UUIDS)) {
                 // The token was created in a > VERSION_ACCESS_TOKENS_UUIDS cluster
                 if (in.available() < MINIMUM_BYTES) {
@@ -777,7 +777,7 @@ private void indexInvalidation(
         final Set<String> idsOfRecentTokens = new HashSet<>();
         final Set<String> idsOfOlderTokens = new HashSet<>();
         for (UserToken userToken : userTokens) {
-            if (userToken.getVersion().onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) {
+            if (userToken.getTransportVersion().onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) {
                 idsOfRecentTokens.add(userToken.getId());
             } else {
                 idsOfOlderTokens.add(userToken.getId());
@@ -997,9 +997,9 @@ private void findTokenFromRefreshToken(String refreshToken, Iterator<TimeValue>
                 findTokenFromRefreshToken(refreshToken, securityTokensIndex, backoff, listener);
             } else {
                 logger.debug("Assuming a refresh token [{}] provided from a client", refreshToken);
-                final Version refreshTokenVersion;
+                final TransportVersion refreshTokenVersion;
                 final String unencodedRefreshToken;
-                final Tuple<Version, String> versionAndRefreshTokenTuple;
+                final Tuple<TransportVersion, String> versionAndRefreshTokenTuple;
                 try {
                     versionAndRefreshTokenTuple = unpackVersionAndPayload(refreshToken);
                     refreshTokenVersion = versionAndRefreshTokenTuple.v1();
@@ -1129,7 +1129,7 @@ private void innerRefresh(
             return;
         }
         final RefreshTokenStatus refreshTokenStatus = checkRefreshResult.v1();
-        final SecurityIndexManager refreshedTokenIndex = getTokensIndexForVersion(refreshTokenStatus.getVersion());
+        final SecurityIndexManager refreshedTokenIndex = getTokensIndexForVersion(refreshTokenStatus.getTransportVersion());
         if (refreshTokenStatus.isRefreshed()) {
             logger.debug(
                 "Token document [{}] was recently refreshed, when a new token document was generated. Reusing that result.",
@@ -1141,7 +1141,7 @@ private void innerRefresh(
         } else {
             final String newAccessTokenString = UUIDs.randomBase64UUID();
             final String newRefreshTokenString = UUIDs.randomBase64UUID();
-            final Version newTokenVersion = getTokenVersionCompatibility();
+            final TransportVersion newTokenVersion = getTokenVersionCompatibility();
             final Map<String, Object> updateMap = new HashMap<>();
             updateMap.put("refreshed", true);
             if (newTokenVersion.onOrAfter(VERSION_MULTIPLE_CONCURRENT_REFRESHES)) {
@@ -1367,8 +1367,8 @@ public void onResponse(GetResponse response) {
                                 );
                                 listener.onResponse(
                                     new CreateTokenResult(
-                                        prependVersionAndEncodeAccessToken(refreshTokenStatus.getVersion(), decryptedTokens[0]),
-                                        prependVersionAndEncodeRefreshToken(refreshTokenStatus.getVersion(), decryptedTokens[1]),
+                                        prependVersionAndEncodeAccessToken(refreshTokenStatus.getTransportVersion(), decryptedTokens[0]),
+                                        prependVersionAndEncodeRefreshToken(refreshTokenStatus.getTransportVersion(), decryptedTokens[1]),
                                         authentication
                                     )
                                 );
@@ -1427,9 +1427,9 @@ private void getTokenDocAsync(
         );
     }
 
-    Version getTokenVersionCompatibility() {
+    TransportVersion getTokenVersionCompatibility() {
         // newly minted tokens are compatible with the min node version in the cluster
-        return clusterService.state().nodes().getMinNodeVersion();
+        return clusterService.state().nodes().getMinNodeVersion().transportVersion;// should there be getMinNodeTransportVersion ?
     }
 
     public static Boolean isTokenServiceEnabled(Settings settings) {
@@ -1480,7 +1480,7 @@ private static Tuple<RefreshTokenStatus, Optional<ElasticsearchSecurityException
     private static Tuple<UserToken, RefreshTokenStatus> parseTokenAndRefreshStatus(Map<String, Object> source) {
         final RefreshTokenStatus refreshTokenStatus = RefreshTokenStatus.fromSourceMap(getRefreshTokenSourceMap(source));
         final UserToken userToken = UserToken.fromSourceMap(getUserTokenSourceMap(source));
-        refreshTokenStatus.setVersion(userToken.getVersion());
+        refreshTokenStatus.setTransportVersion(userToken.getTransportVersion());
         return new Tuple<>(userToken, refreshTokenStatus);
     }
 
@@ -1565,7 +1565,7 @@ private static Optional<ElasticsearchSecurityException> checkMultipleRefreshes(
         RefreshTokenStatus refreshTokenStatus
     ) {
         if (refreshTokenStatus.isRefreshed()) {
-            if (refreshTokenStatus.getVersion().onOrAfter(VERSION_MULTIPLE_CONCURRENT_REFRESHES)) {
+            if (refreshTokenStatus.getTransportVersion().onOrAfter(VERSION_MULTIPLE_CONCURRENT_REFRESHES)) {
                 if (refreshRequested.isAfter(refreshTokenStatus.getRefreshInstant().plus(30L, ChronoUnit.SECONDS))) {
                     return Optional.of(invalidGrantException("token has already been refreshed more than 30 seconds in the past"));
                 }
@@ -1794,8 +1794,11 @@ static BytesReference createTokenDocument(
                     .field("refreshed", false)
                     .startObject("client")
                     .field("type", "unassociated_client");
-                if (userToken.getVersion().onOrAfter(VERSION_CLIENT_AUTH_FOR_REFRESH)) {
-                    builder.field("authentication", originatingClientAuth.maybeRewriteForOlderVersion(userToken.getVersion()).encode());
+                if (userToken.getTransportVersion().onOrAfter(VERSION_CLIENT_AUTH_FOR_REFRESH)) {
+                    builder.field(
+                        "authentication",
+                        originatingClientAuth.maybeRewriteForOlderVersion(userToken.getTransportVersion()).encode()
+                    );
                 } else {
                     builder.field("user", originatingClientAuth.getEffectiveSubject().getUser().principal())
                         .field("realm", originatingClientAuth.getAuthenticatingSubject().getRealm().getName());
@@ -1805,12 +1808,13 @@ static BytesReference createTokenDocument(
                 }
                 builder.endObject().endObject();
             }
+            final Authentication.RealmRef userTokenEffectiveRealm = userToken.getAuthentication().getEffectiveSubject().getRealm();
             builder.startObject("access_token")
                 .field("invalidated", false)
                 .field("user_token", userToken)
-                .field("realm", userToken.getAuthentication().getSourceRealm().getName());
-            if (userToken.getAuthentication().getSourceRealm().getDomain() != null) {
-                builder.field("realm_domain", userToken.getAuthentication().getSourceRealm().getDomain());
+                .field("realm", userTokenEffectiveRealm.getName());
+            if (userTokenEffectiveRealm.getDomain() != null) {
+                builder.field("realm_domain", userTokenEffectiveRealm.getDomain());
             }
             builder.endObject().endObject();
             return BytesReference.bytes(builder);
@@ -1823,9 +1827,9 @@ private static Predicate<Map<String, Object>> isOfUser(String username) {
         return source -> {
             String auth = (String) source.get("authentication");
             Integer version = (Integer) source.get("version");
-            Version authVersion = Version.fromId(version);
+            TransportVersion authVersion = TransportVersion.fromId(version);
             try (StreamInput in = StreamInput.wrap(Base64.getDecoder().decode(auth))) {
-                in.setVersion(authVersion);
+                in.setTransportVersion(authVersion);
                 Authentication authentication = new Authentication(in);
                 return authentication.getEffectiveSubject().getUser().principal().equals(username);
             } catch (IOException e) {
@@ -1905,7 +1909,7 @@ private void ensureEnabled() {
      * removed automatically by the {@code ExpiredTokenRemover} periodic job. Therefore, in general, when searching for a token we need to
      * consider both the new and the old indices.
      */
-    private SecurityIndexManager getTokensIndexForVersion(Version version) {
+    private SecurityIndexManager getTokensIndexForVersion(TransportVersion version) {
         if (version.onOrAfter(VERSION_TOKENS_INDEX_INTRODUCED)) {
             return securityTokensIndex;
         } else {
@@ -1921,7 +1925,7 @@ private void checkIfTokenIsValid(UserToken userToken, ActionListener<UserToken>
             listener.onFailure(traceLog("validate token", userToken.getId(), expiredTokenException()));
             return;
         }
-        final SecurityIndexManager tokensIndex = getTokensIndexForVersion(userToken.getVersion());
+        final SecurityIndexManager tokensIndex = getTokensIndexForVersion(userToken.getTransportVersion());
         if (tokensIndex.indexExists() == false) {
             // index doesn't exist so the token is considered invalid as we cannot verify its validity
             logger.warn("failed to validate access token because the index [" + tokensIndex.aliasName() + "] doesn't exist");
@@ -2008,11 +2012,11 @@ private void maybeStartTokenRemover() {
         }
     }
 
-    String prependVersionAndEncodeAccessToken(Version version, String accessToken) throws IOException, GeneralSecurityException {
+    String prependVersionAndEncodeAccessToken(TransportVersion version, String accessToken) throws IOException, GeneralSecurityException {
         if (version.onOrAfter(VERSION_ACCESS_TOKENS_AS_UUIDS)) {
             try (BytesStreamOutput out = new BytesStreamOutput(MINIMUM_BASE64_BYTES)) {
-                out.setVersion(version);
-                Version.writeVersion(version, out);
+                out.setTransportVersion(version);
+                TransportVersion.writeVersion(version, out);
                 out.writeString(accessToken);
                 return Base64.getEncoder().encodeToString(out.bytes().toBytesRef().bytes);
             }
@@ -2023,9 +2027,9 @@ String prependVersionAndEncodeAccessToken(Version version, String accessToken) t
                 OutputStream base64 = Base64.getEncoder().wrap(os);
                 StreamOutput out = new OutputStreamStreamOutput(base64)
             ) {
-                out.setVersion(version);
+                out.setTransportVersion(version);
                 KeyAndCache keyAndCache = keyCache.activeKeyCache;
-                Version.writeVersion(version, out);
+                TransportVersion.writeVersion(version, out);
                 out.writeByteArray(keyAndCache.getSalt().bytes);
                 out.writeByteArray(keyAndCache.getKeyHash().bytes);
                 final byte[] initializationVector = getRandomBytes(IV_BYTES);
@@ -2037,7 +2041,7 @@ String prependVersionAndEncodeAccessToken(Version version, String accessToken) t
                     );
                     StreamOutput encryptedStreamOutput = new OutputStreamStreamOutput(encryptedOutput)
                 ) {
-                    encryptedStreamOutput.setVersion(version);
+                    encryptedStreamOutput.setTransportVersion(version);
                     encryptedStreamOutput.writeString(accessToken);
                     // StreamOutput needs to be closed explicitly because it wraps CipherOutputStream
                     encryptedStreamOutput.close();
@@ -2047,10 +2051,10 @@ String prependVersionAndEncodeAccessToken(Version version, String accessToken) t
         }
     }
 
-    public static String prependVersionAndEncodeRefreshToken(Version version, String payload) {
+    public static String prependVersionAndEncodeRefreshToken(TransportVersion version, String payload) {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
-            out.setVersion(version);
-            Version.writeVersion(version, out);
+            out.setTransportVersion(version);
+            TransportVersion.writeVersion(version, out);
             out.writeString(payload);
             return Base64.getEncoder().encodeToString(out.bytes().toBytesRef().bytes);
 
@@ -2063,11 +2067,11 @@ public static String prependVersionAndEncodeRefreshToken(Version version, String
     /**
      * Unpacks a base64 encoded pair of a version tag and String payload.
      */
-    public static Tuple<Version, String> unpackVersionAndPayload(String encodedPack) throws IOException {
+    public static Tuple<TransportVersion, String> unpackVersionAndPayload(String encodedPack) throws IOException {
         final byte[] bytes = encodedPack.getBytes(StandardCharsets.UTF_8);
         try (StreamInput in = new InputStreamStreamInput(Base64.getDecoder().wrap(new ByteArrayInputStream(bytes)), bytes.length)) {
-            final Version version = Version.readVersion(in);
-            in.setVersion(version);
+            final TransportVersion version = TransportVersion.readVersion(in);
+            in.setTransportVersion(version);
             final String payload = in.readString();
             return new Tuple<>(version, payload);
         }
@@ -2079,7 +2083,7 @@ private static void ensureEncryptionCiphersSupported() throws NoSuchPaddingExcep
     }
 
     // Package private for testing
-    Cipher getEncryptionCipher(byte[] iv, KeyAndCache keyAndCache, Version version) throws GeneralSecurityException {
+    Cipher getEncryptionCipher(byte[] iv, KeyAndCache keyAndCache, TransportVersion version) throws GeneralSecurityException {
         Cipher cipher = Cipher.getInstance(ENCRYPTION_CIPHER);
         BytesKey salt = keyAndCache.getSalt();
         try {
@@ -2117,18 +2121,18 @@ private void getKeyAsync(BytesKey decodedSalt, KeyAndCache keyAndCache, ActionLi
         }
     }
 
-    private static String decryptTokenId(byte[] encryptedTokenId, Cipher cipher, Version version) throws IOException {
+    private static String decryptTokenId(byte[] encryptedTokenId, Cipher cipher, TransportVersion version) throws IOException {
         try (
             ByteArrayInputStream bais = new ByteArrayInputStream(encryptedTokenId);
             CipherInputStream cis = new CipherInputStream(bais, cipher);
             StreamInput decryptedInput = new InputStreamStreamInput(cis)
         ) {
-            decryptedInput.setVersion(version);
+            decryptedInput.setTransportVersion(version);
             return decryptedInput.readString();
         }
     }
 
-    private Cipher getDecryptionCipher(byte[] iv, SecretKey key, Version version, BytesKey salt) throws GeneralSecurityException {
+    private Cipher getDecryptionCipher(byte[] iv, SecretKey key, TransportVersion version, BytesKey salt) throws GeneralSecurityException {
         Cipher cipher = Cipher.getInstance(ENCRYPTION_CIPHER);
         cipher.init(Cipher.DECRYPT_MODE, key, new GCMParameterSpec(128, iv), secureRandom);
         cipher.updateAAD(ByteBuffer.allocate(4).putInt(version.id).array());
@@ -2533,7 +2537,7 @@ static final class RefreshTokenStatus {
         private final String iv;
         @Nullable
         private final String salt;
-        private Version version;
+        private TransportVersion version;
 
         // pkg-private for testing
         RefreshTokenStatus(
@@ -2545,7 +2549,7 @@ static final class RefreshTokenStatus {
             String iv,
             String salt
         ) {
-            assert associatedAuthentication.getEffectiveSubject().getVersion().onOrAfter(VERSION_CLIENT_AUTH_FOR_REFRESH);
+            assert associatedAuthentication.getEffectiveSubject().getTransportVersion().onOrAfter(VERSION_CLIENT_AUTH_FOR_REFRESH);
             this.invalidated = invalidated;
             // not used, filled-in for consistency's sake
             this.associatedUser = associatedAuthentication.getEffectiveSubject().getUser().principal();
@@ -2620,11 +2624,11 @@ String getSalt() {
             return salt;
         }
 
-        Version getVersion() {
+        TransportVersion getTransportVersion() {
             return version;
         }
 
-        void setVersion(Version version) {
+        void setTransportVersion(TransportVersion version) {
             this.version = version;
         }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/UserToken.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/UserToken.java
index f6fb96ebacca..68e7a619abe3 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/UserToken.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/UserToken.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.security.authc;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -35,7 +35,7 @@
  */
 public final class UserToken implements Writeable, ToXContentObject {
 
-    private final Version version;
+    private final TransportVersion version;
     private final String id;
     private final Authentication authentication;
     private final Instant expirationTime;
@@ -45,20 +45,20 @@ public final class UserToken implements Writeable, ToXContentObject {
      * Create a new token with an autogenerated id
      */
     UserToken(Authentication authentication, Instant expirationTime) {
-        this(Version.CURRENT, authentication, expirationTime, Collections.emptyMap());
+        this(TransportVersion.CURRENT, authentication, expirationTime, Collections.emptyMap());
     }
 
     /**
      * Create a new token with an autogenerated id
      */
-    private UserToken(Version version, Authentication authentication, Instant expirationTime, Map<String, Object> metadata) {
+    private UserToken(TransportVersion version, Authentication authentication, Instant expirationTime, Map<String, Object> metadata) {
         this(UUIDs.randomBase64UUID(), version, authentication, expirationTime, metadata);
     }
 
     /**
      * Create a new token from an existing id
      */
-    UserToken(String id, Version version, Authentication authentication, Instant expirationTime, Map<String, Object> metadata) {
+    UserToken(String id, TransportVersion version, Authentication authentication, Instant expirationTime, Map<String, Object> metadata) {
         this.version = Objects.requireNonNull(version);
         this.id = Objects.requireNonNull(id);
         this.authentication = Objects.requireNonNull(authentication);
@@ -70,7 +70,7 @@ private UserToken(Version version, Authentication authentication, Instant expira
      * Creates a new token based on the values from the stream
      */
     UserToken(StreamInput input) throws IOException {
-        this.version = input.getVersion();
+        this.version = input.getTransportVersion();
         this.id = input.readString();
         this.authentication = new Authentication(input);
         this.expirationTime = Instant.ofEpochSecond(input.readLong(), input.readInt());
@@ -108,9 +108,9 @@ public String getId() {
     }
 
     /**
-     * The version of the node this token was created on
+     * The transport version of the node this token was created on
      */
-    Version getVersion() {
+    TransportVersion getTransportVersion() {
         return version;
     }
 
@@ -129,7 +129,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field("version", version.id);
         builder.field("metadata", metadata);
         try (BytesStreamOutput output = new BytesStreamOutput()) {
-            output.setVersion(version);
+            output.setTransportVersion(version);
             authentication.writeTo(output);
             builder.field("authentication", output.bytes().toBytesRef().bytes);
         }
@@ -155,9 +155,9 @@ static UserToken fromSourceMap(Map<String, Object> source) throws IllegalStateEx
         if (authString == null) {
             throw new IllegalStateException("user token source document does not have the \"authentication\" field");
         }
-        final Version version = Version.fromId(versionId);
+        final TransportVersion version = TransportVersion.fromId(versionId);
         try (StreamInput in = StreamInput.wrap(Base64.getDecoder().decode(authString))) {
-            in.setVersion(version);
+            in.setTransportVersion(version);
             Authentication authentication = new Authentication(in);
             return new UserToken(id, version, authentication, Instant.ofEpochMilli(expirationEpochMilli), metadata);
         } catch (IOException e) {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/FallbackableClaim.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/FallbackableClaim.java
new file mode 100644
index 000000000000..94bba5d03dc6
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/FallbackableClaim.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authc.jwt;
+
+import com.nimbusds.jwt.JWTClaimsSet;
+
+import org.elasticsearch.core.Nullable;
+
+import java.text.ParseException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.core.Strings.format;
+
+/**
+ * A JWT claim that can optionally fallback to another claim (if configured) for retrieving the associated value
+ * from a {@link JWTClaimsSet}. The fallback behaviour happens only when:
+ * 1. The fallback is configured (it can be null)
+ * 2. The original claim does not exist in the {@link JWTClaimsSet}
+ * In any other cases, the original claim will be used for retrieving the value.
+ */
+public class FallbackableClaim {
+    private final String name;
+    private final JWTClaimsSet claimsSet;
+    private final String actualName;
+
+    public FallbackableClaim(String name, @Nullable Map<String, String> fallbackClaimNames, JWTClaimsSet claimsSet) {
+        this.name = Objects.requireNonNull(name);
+        this.claimsSet = Objects.requireNonNull(claimsSet);
+        final String fallbackName;
+        if (fallbackClaimNames != null) {
+            fallbackName = fallbackClaimNames.getOrDefault(name, name);
+        } else {
+            fallbackName = null;
+        }
+        if (fallbackName == null) {
+            this.actualName = name;
+        } else {
+            this.actualName = claimsSet.getClaim(name) != null ? name : fallbackName;
+        }
+    }
+
+    public String getActualName() {
+        return actualName;
+    }
+
+    public String getStringClaimValue() {
+        try {
+            return claimsSet.getStringClaim(actualName);
+        } catch (ParseException e) {
+            throw new IllegalArgumentException(format("cannot parse string claim [%s] as string", this), e);
+        }
+    }
+
+    public List<String> getStringListClaimValue() {
+        final Object claimValue = claimsSet.getClaim(actualName);
+        if (claimValue instanceof String) {
+            return List.of((String) claimValue);
+        } else {
+            try {
+                return claimsSet.getStringListClaim(actualName);
+            } catch (ParseException e) {
+                throw new IllegalArgumentException(format("cannot parse string claim [%s] as string array", this), e);
+            }
+        }
+    }
+
+    @Override
+    public String toString() {
+        if (name.equals(actualName)) {
+            return name;
+        } else {
+            return format("%s (fallback of %s)", actualName, name);
+        }
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwkSetLoader.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwkSetLoader.java
index a6f25ef69097..168813956f56 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwkSetLoader.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwkSetLoader.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.common.util.concurrent.ListenableFuture;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
 import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings;
@@ -42,7 +43,7 @@ public class JwkSetLoader implements Releasable {
 
     private static final Logger logger = LogManager.getLogger(JwkSetLoader.class);
 
-    private final AtomicReference<ListenableFuture<Boolean>> reloadFutureRef = new AtomicReference<>();
+    private final AtomicReference<ListenableFuture<Tuple<Boolean, JwksAlgs>>> reloadFutureRef = new AtomicReference<>();
     private final RealmConfig realmConfig;
     private final List<String> allowedJwksAlgsPkc;
     private final String jwkSetPath;
@@ -67,10 +68,10 @@ public JwkSetLoader(final RealmConfig realmConfig, List<String> allowedJwksAlgsP
 
         // Any exception during loading requires closing JwkSetLoader's HTTP client to avoid a thread pool leak
         try {
-            final PlainActionFuture<Boolean> future = new PlainActionFuture<>();
+            final PlainActionFuture<Tuple<Boolean, JwksAlgs>> future = new PlainActionFuture<>();
             reload(future);
             // ASSUME: Blocking read operations are OK during startup
-            final Boolean isUpdated = future.actionGet();
+            final Boolean isUpdated = future.actionGet().v1();
             assert isUpdated : "initial reload should have updated the JWK set";
         } catch (Throwable t) {
             close();
@@ -83,8 +84,8 @@ public JwkSetLoader(final RealmConfig realmConfig, List<String> allowedJwksAlgsP
      * they are different. The listener is called with false if the reloaded content is the same
      * as the existing one or true if they are different.
      */
-    void reload(final ActionListener<Boolean> listener) {
-        final ListenableFuture<Boolean> future = this.getFuture();
+    void reload(final ActionListener<Tuple<Boolean, JwksAlgs>> listener) {
+        final ListenableFuture<Tuple<Boolean, JwksAlgs>> future = getFuture();
         future.addListener(listener);
     }
 
@@ -92,17 +93,18 @@ ContentAndJwksAlgs getContentAndJwksAlgs() {
         return contentAndJwksAlgs;
     }
 
-    private ListenableFuture<Boolean> getFuture() {
+    // Package private for testing
+    ListenableFuture<Tuple<Boolean, JwksAlgs>> getFuture() {
         for (;;) {
-            final ListenableFuture<Boolean> existingFuture = this.reloadFutureRef.get();
+            final ListenableFuture<Tuple<Boolean, JwksAlgs>> existingFuture = reloadFutureRef.get();
             if (existingFuture != null) {
                 return existingFuture;
             }
 
-            final ListenableFuture<Boolean> newFuture = new ListenableFuture<>();
-            if (this.reloadFutureRef.compareAndSet(null, newFuture)) {
-                loadInternal(ActionListener.runAfter(newFuture, () -> {
-                    final ListenableFuture<Boolean> oldValue = this.reloadFutureRef.getAndSet(null);
+            final ListenableFuture<Tuple<Boolean, JwksAlgs>> newFuture = new ListenableFuture<>();
+            if (reloadFutureRef.compareAndSet(null, newFuture)) {
+                loadInternal(ActionListener.runBefore(newFuture, () -> {
+                    final ListenableFuture<Tuple<Boolean, JwksAlgs>> oldValue = reloadFutureRef.getAndSet(null);
                     assert oldValue == newFuture : "future reference changed unexpectedly";
                 }));
                 return newFuture;
@@ -111,7 +113,8 @@ private ListenableFuture<Boolean> getFuture() {
         }
     }
 
-    private void loadInternal(final ActionListener<Boolean> listener) {
+    // Package private for testing
+    void loadInternal(final ActionListener<Tuple<Boolean, JwksAlgs>> listener) {
         // PKC JWKSet get contents from local file or remote HTTPS URL
         if (httpClient == null) {
             logger.trace("Loading PKC JWKs from path [{}]", jwkSetPath);
@@ -135,16 +138,18 @@ private void loadInternal(final ActionListener<Boolean> listener) {
         }
     }
 
-    private Boolean handleReloadedContentAndJwksAlgs(byte[] bytes) {
+    private Tuple<Boolean, JwksAlgs> handleReloadedContentAndJwksAlgs(byte[] bytes) {
         final ContentAndJwksAlgs newContentAndJwksAlgs = parseContent(bytes);
+        final boolean isUpdated;
         if (contentAndJwksAlgs != null && Arrays.equals(contentAndJwksAlgs.sha256, newContentAndJwksAlgs.sha256)) {
             logger.debug("No change in reloaded JWK set");
-            return false;
+            isUpdated = false;
         } else {
             logger.debug("Reloaded JWK set is different from the existing set");
             contentAndJwksAlgs = newContentAndJwksAlgs;
-            return true;
+            isUpdated = true;
         }
+        return new Tuple<>(isUpdated, contentAndJwksAlgs.jwksAlgs);
     }
 
     private ContentAndJwksAlgs parseContent(final byte[] jwkSetContentBytesPkc) {
@@ -186,7 +191,7 @@ record JwksAlgs(List<JWK> jwks, List<String> algs) {
         }
 
         boolean isEmpty() {
-            return this.jwks.isEmpty() && this.algs.isEmpty();
+            return jwks.isEmpty() && algs.isEmpty();
         }
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtAlgorithmValidator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtAlgorithmValidator.java
index 5f2fe02c263b..79082d140bfb 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtAlgorithmValidator.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtAlgorithmValidator.java
@@ -11,12 +11,12 @@
 import com.nimbusds.jose.JWSHeader;
 import com.nimbusds.jwt.JWTClaimsSet;
 
-import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.rest.RestStatus;
 
 import java.util.List;
 
+import static org.elasticsearch.core.Strings.format;
+
 public class JwtAlgorithmValidator implements JwtFieldValidator {
 
     private final List<String> allowedAlgorithms;
@@ -28,15 +28,16 @@ public JwtAlgorithmValidator(List<String> allowedAlgorithms) {
     public void validate(JWSHeader jwsHeader, JWTClaimsSet jwtClaimsSet) {
         final JWSAlgorithm algorithm = jwsHeader.getAlgorithm();
         if (algorithm == null) {
-            throw new ElasticsearchSecurityException("missing JWT algorithm header", RestStatus.BAD_REQUEST);
+            throw new IllegalArgumentException("missing JWT algorithm header");
         }
 
         if (false == allowedAlgorithms.contains(algorithm.getName())) {
-            throw new ElasticsearchSecurityException(
-                "invalid JWT algorithm [{}], allowed algorithms are [{}]",
-                RestStatus.BAD_REQUEST,
-                algorithm,
-                Strings.collectionToCommaDelimitedString(allowedAlgorithms)
+            throw new IllegalArgumentException(
+                format(
+                    "invalid JWT algorithm [%s], allowed algorithms are [%s]",
+                    algorithm,
+                    Strings.collectionToCommaDelimitedString(allowedAlgorithms)
+                )
             );
         }
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticationToken.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticationToken.java
index 4adc33450463..a80865428e59 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticationToken.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticationToken.java
@@ -9,121 +9,51 @@
 import com.nimbusds.jwt.JWTClaimsSet;
 import com.nimbusds.jwt.SignedJWT;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
-import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmsServiceSettings;
 
 import java.text.ParseException;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeSet;
-import java.util.stream.Collectors;
+import java.util.Arrays;
+import java.util.Objects;
 
 /**
  * An {@link AuthenticationToken} to hold JWT authentication related content.
  */
 public class JwtAuthenticationToken implements AuthenticationToken {
-    private static final Logger LOGGER = LogManager.getLogger(JwtAuthenticationToken.class);
-
-    // Stored members
-    protected SecureString endUserSignedJwt; // required
-    protected SecureString clientAuthenticationSharedSecret; // optional, nullable
-    protected String principal; // Defaults to "iss/aud/sub", with an ordered "aud" list
+    private final String principal;
+    private SignedJWT signedJWT;
+    private final byte[] userCredentialsHash;
+    @Nullable
+    private final SecureString clientAuthenticationSharedSecret;
 
     /**
-     * Store a mandatory JWT and optional Shared Secret. Parse the JWT, and extract the header, claims set, and signature.
-     * Compute a token principal, for use as a realm order cache key. For OIDC ID Tokens, cache key is iss/aud/sub.
-     * For other JWTs, {@link JwtRealmsServiceSettings#PRINCIPAL_CLAIMS_SETTING} supports alternative claims for sub.
-     * Throws IllegalArgumentException if principalClaimNames is empty, JWT is missing, or if JWT parsing fails.
-     * @param principalClaimNames Ordered list of string claims to use for principalClaimValue. The first one found is used (ex: sub).
-     * @param endUserSignedJwt Base64Url-encoded JWT for End-user authentication. Required by all JWT realms.
+     * Store a mandatory JWT and optional Shared Secret.
+     * @param principal The token's principal, useful as a realm order cache key
+     * @param signedJWT The JWT parsed from the end-user credentials
+     * @param userCredentialsHash The hash of the end-user credentials is used to compute the key for user cache at the realm level.
+     *                            See also {@link JwtRealm#authenticate}.
      * @param clientAuthenticationSharedSecret URL-safe Shared Secret for Client authentication. Required by some JWT realms.
      */
     public JwtAuthenticationToken(
-        final List<String> principalClaimNames,
-        final SecureString endUserSignedJwt,
+        String principal,
+        SignedJWT signedJWT,
+        byte[] userCredentialsHash,
         @Nullable final SecureString clientAuthenticationSharedSecret
     ) {
-        if (principalClaimNames.isEmpty()) {
-            throw new IllegalArgumentException("JWT token principal claim names list must be non-empty");
-        } else if (endUserSignedJwt.isEmpty()) {
-            throw new IllegalArgumentException("JWT bearer token must be non-empty");
-        } else if ((clientAuthenticationSharedSecret != null) && (clientAuthenticationSharedSecret.isEmpty())) {
-            throw new IllegalArgumentException("Client shared secret must be non-empty");
-        }
-        this.endUserSignedJwt = endUserSignedJwt; // required
-        this.clientAuthenticationSharedSecret = clientAuthenticationSharedSecret; // optional, nullable
-
-        JWTClaimsSet jwtClaimsSet;
-        try {
-            jwtClaimsSet = SignedJWT.parse(this.endUserSignedJwt.toString()).getJWTClaimsSet();
-        } catch (ParseException e) {
-            throw new IllegalArgumentException("Failed to parse JWT bearer token", e);
-        }
-
-        // get and validate iss and aud claims
-        final String issuer = jwtClaimsSet.getIssuer();
-        final List<String> audiences = jwtClaimsSet.getAudience();
-        if (Strings.hasText(issuer) == false) {
-            throw new IllegalArgumentException("Issuer claim 'iss' is missing.");
-        } else if ((audiences == null) || (audiences.isEmpty())) {
-            throw new IllegalArgumentException("Audiences claim 'aud' is missing.");
-        }
-
-        // get and validate sub claim, or the first configured backup claim (if sub is absent)
-        final String principalClaimValue = this.resolvePrincipalClaimName(jwtClaimsSet, principalClaimNames);
-        this.principal = issuer + "/" + String.join(",", new TreeSet<>(audiences)) + "/" + principalClaimValue;
-    }
+        this.principal = Objects.requireNonNull(principal);
+        this.signedJWT = Objects.requireNonNull(signedJWT);
+        this.userCredentialsHash = Objects.requireNonNull(userCredentialsHash);
 
-    private String resolvePrincipalClaimName(final JWTClaimsSet jwtClaimsSet, final List<String> principalClaimNames) {
-        for (final String principalClaimName : principalClaimNames) {
-            final Object claimValue = jwtClaimsSet.getClaim(principalClaimName);
-            if (claimValue instanceof String principalClaimValue) {
-                // found an allowed string claim name
-                if (principalClaimValue.isEmpty()) {
-                    throw new IllegalArgumentException(
-                        "Allowed principal claim name '"
-                            + principalClaimName
-                            + "' exists but cannot be used because the value of that claim is an empty string"
-                    );
-                }
-                LOGGER.trace("Found allowed principal claim name [{}] with value [{}]", principalClaimName, principalClaimValue);
-                return principalClaimValue;
-            } else if (claimValue != null) {
-                throw new IllegalArgumentException(
-                    "Allowed principal claim name '"
-                        + principalClaimName
-                        + "' exists but cannot be used because the value of that claim must be a string, but instead it was a ["
-                        + claimValue.getClass().getSimpleName()
-                        + "]"
-                );
-            }
+        if ((clientAuthenticationSharedSecret != null) && (clientAuthenticationSharedSecret.isEmpty())) {
+            throw new IllegalArgumentException("Client shared secret must be non-empty");
         }
-
-        // at this point, none of the principalClaimNames were found
-        // throw an exception with a detailed log message about which string claims were available in the JWT
-        final String allClaimNamesWithStringValues = jwtClaimsSet.getClaims()
-            .entrySet()
-            .stream()
-            .filter(e -> e.getValue() instanceof String)
-            .map(Map.Entry::getKey)
-            .collect(Collectors.joining(","));
-        throw new IllegalArgumentException(
-            "None of these configured principal claim names were found in the JWT Claims Set ["
-                + String.join(",", principalClaimNames)
-                + "] - available claims in the JWT with potential compatible string values are ["
-                + allClaimNamesWithStringValues
-                + "]"
-        );
+        this.clientAuthenticationSharedSecret = clientAuthenticationSharedSecret;
     }
 
     @Override
     public String principal() {
-        return this.principal;
+        return principal;
     }
 
     @Override
@@ -131,23 +61,34 @@ public SecureString credentials() {
         return null;
     }
 
-    public SecureString getEndUserSignedJwt() {
-        return this.endUserSignedJwt;
+    public SignedJWT getSignedJWT() {
+        return signedJWT;
+    }
+
+    public JWTClaimsSet getJWTClaimsSet() {
+        try {
+            return signedJWT.getJWTClaimsSet();
+        } catch (ParseException e) {
+            assert false : "The JWT claims set should have already been successfully parsed before building the JWT authentication token";
+            throw new IllegalArgumentException(e);
+        }
+    }
+
+    public byte[] getUserCredentialsHash() {
+        return userCredentialsHash;
     }
 
     public SecureString getClientAuthenticationSharedSecret() {
-        return this.clientAuthenticationSharedSecret;
+        return clientAuthenticationSharedSecret;
     }
 
     @Override
     public void clearCredentials() {
-        this.endUserSignedJwt.close();
-        this.endUserSignedJwt = null;
-        if (this.clientAuthenticationSharedSecret != null) {
-            this.clientAuthenticationSharedSecret.close();
-            this.clientAuthenticationSharedSecret = null;
+        signedJWT = null;
+        Arrays.fill(userCredentialsHash, (byte) 0);
+        if (clientAuthenticationSharedSecret != null) {
+            clientAuthenticationSharedSecret.close();
         }
-        this.principal = null;
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticator.java
index 7b769f15fe72..17d6a22bdb43 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticator.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticator.java
@@ -14,16 +14,17 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 
-import java.text.ParseException;
 import java.time.Clock;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
 /**
  * This class performs validations of header, claims and signatures against the incoming {@link JwtAuthenticationToken}.
@@ -37,6 +38,7 @@ public class JwtAuthenticator implements Releasable {
     private final List<JwtFieldValidator> jwtFieldValidators;
     private final JwtSignatureValidator jwtSignatureValidator;
     private final JwtRealmSettings.TokenType tokenType;
+    private final Map<String, String> fallbackClaimNames;
 
     public JwtAuthenticator(
         final RealmConfig realmConfig,
@@ -45,38 +47,29 @@ public JwtAuthenticator(
     ) {
         this.realmConfig = realmConfig;
         this.tokenType = realmConfig.getSetting(JwtRealmSettings.TOKEN_TYPE);
+        final List<JwtFieldValidator> jwtFieldValidators = new ArrayList<>();
         if (tokenType == JwtRealmSettings.TokenType.ID_TOKEN) {
-            this.jwtFieldValidators = configureFieldValidatorsForIdToken(realmConfig);
+            this.fallbackClaimNames = Map.of();
+            jwtFieldValidators.addAll(configureFieldValidatorsForIdToken(realmConfig));
         } else {
-            this.jwtFieldValidators = configureFieldValidatorsForAccessToken(realmConfig);
+            this.fallbackClaimNames = Map.ofEntries(
+                Map.entry("sub", realmConfig.getSetting(JwtRealmSettings.FALLBACK_SUB_CLAIM)),
+                Map.entry("aud", realmConfig.getSetting(JwtRealmSettings.FALLBACK_AUD_CLAIM))
+            );
+            jwtFieldValidators.addAll(configureFieldValidatorsForAccessToken(realmConfig, fallbackClaimNames));
         }
+        jwtFieldValidators.addAll(getRequireClaimsValidators());
+        this.jwtFieldValidators = List.copyOf(jwtFieldValidators);
         this.jwtSignatureValidator = new JwtSignatureValidator.DelegatingJwtSignatureValidator(realmConfig, sslService, reloadNotifier);
     }
 
     public void authenticate(JwtAuthenticationToken jwtAuthenticationToken, ActionListener<JWTClaimsSet> listener) {
         final String tokenPrincipal = jwtAuthenticationToken.principal();
-
         // JWT cache
-        final SecureString serializedJwt = jwtAuthenticationToken.getEndUserSignedJwt();
-        final SignedJWT signedJWT;
-        try {
-            signedJWT = SignedJWT.parse(serializedJwt.toString());
-        } catch (ParseException e) {
-            // TODO: No point to continue to another realm since parsing failed
-            listener.onFailure(e);
-            return;
-        }
-
-        final JWTClaimsSet jwtClaimsSet;
-        try {
-            jwtClaimsSet = signedJWT.getJWTClaimsSet();
-        } catch (ParseException e) {
-            // TODO: No point to continue to another realm since get claimset failed
-            listener.onFailure(e);
-            return;
-        }
-
+        final SignedJWT signedJWT = jwtAuthenticationToken.getSignedJWT();
+        final JWTClaimsSet jwtClaimsSet = jwtAuthenticationToken.getJWTClaimsSet();
         final JWSHeader jwsHeader = signedJWT.getHeader();
+
         if (logger.isDebugEnabled()) {
             logger.debug(
                 "Realm [{}] successfully parsed JWT token [{}] with header [{}] and claimSet [{}]",
@@ -97,12 +90,17 @@ public void authenticate(JwtAuthenticationToken jwtAuthenticationToken, ActionLi
         }
 
         try {
-            jwtSignatureValidator.validate(tokenPrincipal, signedJWT, listener.map(ignored -> jwtClaimsSet));
+            validateSignature(tokenPrincipal, signedJWT, listener.map(ignored -> jwtClaimsSet));
         } catch (Exception e) {
             listener.onFailure(e);
         }
     }
 
+    // Package private for testing
+    void validateSignature(String tokenPrincipal, SignedJWT signedJWT, ActionListener<Void> listener) {
+        jwtSignatureValidator.validate(tokenPrincipal, signedJWT, listener);
+    }
+
     @Override
     public void close() {
         jwtSignatureValidator.close();
@@ -112,6 +110,10 @@ public JwtRealmSettings.TokenType getTokenType() {
         return tokenType;
     }
 
+    public Map<String, String> getFallbackClaimNames() {
+        return fallbackClaimNames;
+    }
+
     // Package private for testing
     JwtSignatureValidator.DelegatingJwtSignatureValidator getJwtSignatureValidator() {
         assert jwtSignatureValidator instanceof JwtSignatureValidator.DelegatingJwtSignatureValidator;
@@ -122,10 +124,19 @@ private static List<JwtFieldValidator> configureFieldValidatorsForIdToken(RealmC
         assert realmConfig.getSetting(JwtRealmSettings.TOKEN_TYPE) == JwtRealmSettings.TokenType.ID_TOKEN;
         final TimeValue allowedClockSkew = realmConfig.getSetting(JwtRealmSettings.ALLOWED_CLOCK_SKEW);
         final Clock clock = Clock.systemUTC();
+
+        final JwtStringClaimValidator subjectClaimValidator;
+        if (realmConfig.hasSetting(JwtRealmSettings.ALLOWED_SUBJECTS)) {
+            subjectClaimValidator = new JwtStringClaimValidator("sub", realmConfig.getSetting(JwtRealmSettings.ALLOWED_SUBJECTS), true);
+        } else {
+            // Allow any value for the sub claim as long as there is a non-null value
+            subjectClaimValidator = JwtStringClaimValidator.ALLOW_ALL_SUBJECTS;
+        }
+
         return List.of(
             JwtTypeValidator.INSTANCE,
-            // TODO: mandate "sub" claim once access token support is in place
             new JwtStringClaimValidator("iss", List.of(realmConfig.getSetting(JwtRealmSettings.ALLOWED_ISSUER)), true),
+            subjectClaimValidator,
             new JwtStringClaimValidator("aud", realmConfig.getSetting(JwtRealmSettings.ALLOWED_AUDIENCES), false),
             new JwtAlgorithmValidator(realmConfig.getSetting(JwtRealmSettings.ALLOWED_SIGNATURE_ALGORITHMS)),
             new JwtDateClaimValidator(clock, "iat", allowedClockSkew, JwtDateClaimValidator.Relationship.BEFORE_NOW, false),
@@ -135,8 +146,30 @@ private static List<JwtFieldValidator> configureFieldValidatorsForIdToken(RealmC
         );
     }
 
-    private static List<JwtFieldValidator> configureFieldValidatorsForAccessToken(RealmConfig realmConfig) {
+    private static List<JwtFieldValidator> configureFieldValidatorsForAccessToken(
+        RealmConfig realmConfig,
+        Map<String, String> fallbackClaimLookup
+    ) {
         assert realmConfig.getSetting(JwtRealmSettings.TOKEN_TYPE) == JwtRealmSettings.TokenType.ACCESS_TOKEN;
-        throw new UnsupportedOperationException("NYI");
+        final TimeValue allowedClockSkew = realmConfig.getSetting(JwtRealmSettings.ALLOWED_CLOCK_SKEW);
+        final Clock clock = Clock.systemUTC();
+
+        return List.of(
+            JwtTypeValidator.INSTANCE,
+            new JwtStringClaimValidator("iss", List.of(realmConfig.getSetting(JwtRealmSettings.ALLOWED_ISSUER)), true),
+            new JwtStringClaimValidator("sub", fallbackClaimLookup, realmConfig.getSetting(JwtRealmSettings.ALLOWED_SUBJECTS), true),
+            new JwtStringClaimValidator("aud", fallbackClaimLookup, realmConfig.getSetting(JwtRealmSettings.ALLOWED_AUDIENCES), false),
+            new JwtAlgorithmValidator(realmConfig.getSetting(JwtRealmSettings.ALLOWED_SIGNATURE_ALGORITHMS)),
+            new JwtDateClaimValidator(clock, "iat", allowedClockSkew, JwtDateClaimValidator.Relationship.BEFORE_NOW, false),
+            new JwtDateClaimValidator(clock, "exp", allowedClockSkew, JwtDateClaimValidator.Relationship.AFTER_NOW, false)
+        );
+    }
+
+    private List<JwtStringClaimValidator> getRequireClaimsValidators() {
+        final Settings requiredClaims = realmConfig.getSetting(JwtRealmSettings.REQUIRED_CLAIMS);
+        return requiredClaims.names().stream().map(name -> {
+            final List<String> allowedValues = requiredClaims.getAsList(name);
+            return new JwtStringClaimValidator(name, allowedValues, false);
+        }).toList();
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtDateClaimValidator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtDateClaimValidator.java
index 07f63c5ee435..8aabededde50 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtDateClaimValidator.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtDateClaimValidator.java
@@ -10,10 +10,8 @@
 import com.nimbusds.jose.JWSHeader;
 import com.nimbusds.jwt.JWTClaimsSet;
 
-import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.rest.RestStatus;
 
 import java.text.ParseException;
 import java.time.Clock;
@@ -47,14 +45,14 @@ public void validate(JWSHeader jwsHeader, JWTClaimsSet jwtClaimsSet) {
         try {
             claimValue = jwtClaimsSet.getDateClaim(claimName);
         } catch (ParseException e) {
-            throw new ElasticsearchSecurityException("cannot parse date claim [" + claimName + "]", RestStatus.BAD_REQUEST, e);
+            throw new IllegalArgumentException("cannot parse date claim [" + claimName + "]", e);
         }
 
         if (claimValue == null) {
             if (allowNull) {
                 return;
             } else {
-                throw new ElasticsearchSecurityException("missing required date claim [" + claimName + "]");
+                throw new IllegalArgumentException("missing required date claim [" + claimName + "]");
             }
         }
 
@@ -64,27 +62,25 @@ public void validate(JWSHeader jwsHeader, JWTClaimsSet jwtClaimsSet) {
         switch (relationship) {
             case BEFORE_NOW:
                 if (false == claimInstant.isBefore(now.plusSeconds(allowedClockSkewSeconds))) {
-                    throw new ElasticsearchSecurityException(
+                    throw new IllegalArgumentException(
                         Strings.format(
                             "date claim [%s] value [%s] must be before now [%s]",
                             claimName,
                             claimInstant.toEpochMilli(),
                             now.toEpochMilli()
-                        ),
-                        RestStatus.BAD_REQUEST
+                        )
                     );
                 }
                 break;
             case AFTER_NOW:
                 if (false == claimInstant.isAfter(now.minusSeconds(allowedClockSkewSeconds))) {
-                    throw new ElasticsearchSecurityException(
+                    throw new IllegalArgumentException(
                         Strings.format(
                             "date claim [%s] value [%s] must be after now [%s]",
                             claimName,
                             claimInstant.toEpochMilli(),
                             now.toEpochMilli()
-                        ),
-                        RestStatus.BAD_REQUEST
+                        )
                     );
                 }
                 break;
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java
index 847dc7ffaf6d..9d5c8daed681 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.security.authc.jwt;
 
 import com.nimbusds.jwt.JWTClaimsSet;
+import com.nimbusds.jwt.SignedJWT;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.Strings;
@@ -34,6 +35,8 @@
 import org.elasticsearch.xpack.security.authc.support.ClaimParser;
 import org.elasticsearch.xpack.security.authc.support.DelegatedAuthorizationSupport;
 
+import java.text.ParseException;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
@@ -41,6 +44,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.TreeSet;
+import java.util.function.Function;
 
 import static java.lang.String.join;
 import static org.elasticsearch.core.Strings.format;
@@ -58,7 +63,6 @@ public class JwtRealm extends Realm implements CachingRealm, Releasable {
 
     private final Cache<BytesArray, ExpiringUser> jwtCache;
     private final CacheIteratorHelper<BytesArray, ExpiringUser> jwtCacheHelper;
-    private final JwtRealmsService jwtRealmsService;
     private final UserRoleMapper userRoleMapper;
     private final Boolean populateUserMetadata;
     private final ClaimParser claimParserPrincipal;
@@ -71,23 +75,15 @@ public class JwtRealm extends Realm implements CachingRealm, Releasable {
     private final JwtAuthenticator jwtAuthenticator;
     private final TimeValue allowedClockSkew;
     DelegatedAuthorizationSupport delegatedAuthorizationSupport = null;
+    private List<Function<JWTClaimsSet, String>> tokenPrincipalFunctions;
 
-    JwtRealm(
-        final RealmConfig realmConfig,
-        final JwtRealmsService jwtRealmsService,
-        final SSLService sslService,
-        final UserRoleMapper userRoleMapper
-    ) throws SettingsException {
+    public JwtRealm(final RealmConfig realmConfig, final SSLService sslService, final UserRoleMapper userRoleMapper)
+        throws SettingsException {
         super(realmConfig);
-        this.jwtRealmsService = jwtRealmsService; // common configuration settings shared by all JwtRealm instances
         this.userRoleMapper = userRoleMapper;
         this.userRoleMapper.refreshRealmOnChange(this);
         this.allowedClockSkew = realmConfig.getSetting(JwtRealmSettings.ALLOWED_CLOCK_SKEW);
-        this.claimParserPrincipal = ClaimParser.forSetting(logger, JwtRealmSettings.CLAIMS_PRINCIPAL, realmConfig, true);
-        this.claimParserGroups = ClaimParser.forSetting(logger, JwtRealmSettings.CLAIMS_GROUPS, realmConfig, false);
-        this.claimParserDn = ClaimParser.forSetting(logger, JwtRealmSettings.CLAIMS_DN, realmConfig, false);
-        this.claimParserMail = ClaimParser.forSetting(logger, JwtRealmSettings.CLAIMS_MAIL, realmConfig, false);
-        this.claimParserName = ClaimParser.forSetting(logger, JwtRealmSettings.CLAIMS_NAME, realmConfig, false);
+
         this.populateUserMetadata = realmConfig.getSetting(JwtRealmSettings.POPULATE_USER_METADATA);
         this.clientAuthenticationType = realmConfig.getSetting(JwtRealmSettings.CLIENT_AUTHENTICATION_TYPE);
         final SecureString sharedSecret = realmConfig.getSetting(JwtRealmSettings.CLIENT_AUTHENTICATION_SHARED_SECRET);
@@ -115,6 +111,20 @@ public class JwtRealm extends Realm implements CachingRealm, Releasable {
             this.jwtCacheHelper = null;
         }
         jwtAuthenticator = new JwtAuthenticator(realmConfig, sslService, this::expireAll);
+
+        final Map<String, String> fallbackClaimNames = jwtAuthenticator.getFallbackClaimNames();
+
+        this.claimParserPrincipal = ClaimParser.forSetting(
+            logger,
+            JwtRealmSettings.CLAIMS_PRINCIPAL,
+            fallbackClaimNames,
+            realmConfig,
+            true
+        );
+        this.claimParserGroups = ClaimParser.forSetting(logger, JwtRealmSettings.CLAIMS_GROUPS, fallbackClaimNames, realmConfig, false);
+        this.claimParserDn = ClaimParser.forSetting(logger, JwtRealmSettings.CLAIMS_DN, fallbackClaimNames, realmConfig, false);
+        this.claimParserMail = ClaimParser.forSetting(logger, JwtRealmSettings.CLAIMS_MAIL, fallbackClaimNames, realmConfig, false);
+        this.claimParserName = ClaimParser.forSetting(logger, JwtRealmSettings.CLAIMS_NAME, fallbackClaimNames, realmConfig, false);
     }
 
     /**
@@ -130,6 +140,14 @@ public void initialize(final Iterable<Realm> allRealms, final XPackLicenseState
         }
         // extract list of realms referenced by config.settings() value for DelegatedAuthorizationSettings.ROLES_REALMS
         delegatedAuthorizationSupport = new DelegatedAuthorizationSupport(allRealms, config, xpackLicenseState);
+
+        final List<Function<JWTClaimsSet, String>> tokenPrincipalFunctions = new ArrayList<>();
+        for (var realm : allRealms) {
+            if (realm instanceof final JwtRealm jwtRealm) {
+                tokenPrincipalFunctions.add(jwtRealm::buildTokenPrincipal);
+            }
+        }
+        this.tokenPrincipalFunctions = List.copyOf(tokenPrincipalFunctions);
     }
 
     /**
@@ -166,10 +184,66 @@ public void expireAll() {
     @Override
     public AuthenticationToken token(final ThreadContext threadContext) {
         ensureInitialized();
-        // Token parsing is common code for all realms
-        // First JWT realm will parse in a way that is compatible with all JWT realms,
-        // taking into consideration each JWT realm might have a different principal claim name
-        return jwtRealmsService.token(threadContext);
+
+        final SecureString userCredentials = JwtUtil.getHeaderValue(
+            threadContext,
+            JwtRealm.HEADER_END_USER_AUTHENTICATION,
+            JwtRealm.HEADER_END_USER_AUTHENTICATION_SCHEME,
+            false
+        );
+        if (userCredentials == null) {
+            return null;
+        }
+        if (userCredentials.isEmpty()) {
+            throw new IllegalArgumentException("JWT bearer token must be non-empty");
+        }
+
+        final SecureString clientCredentials = JwtUtil.getHeaderValue(
+            threadContext,
+            JwtRealm.HEADER_CLIENT_AUTHENTICATION,
+            JwtRealm.HEADER_SHARED_SECRET_AUTHENTICATION_SCHEME,
+            true
+        );
+
+        // No point to fall through the realm chain if JWT parsing fails, so we throw error here on failure.
+        final SignedJWT signedJWT;
+        try {
+            signedJWT = SignedJWT.parse(userCredentials.toString());
+        } catch (ParseException e) {
+            throw new IllegalArgumentException("Failed to parse JWT bearer token", e);
+        }
+
+        final JWTClaimsSet jwtClaimsSet;
+        try {
+            jwtClaimsSet = signedJWT.getJWTClaimsSet();
+        } catch (ParseException e) {
+            throw new IllegalArgumentException("Failed to parse JWT claims set", e);
+        }
+
+        // If Issuer is not found, still return a JWT token since it is after still a JWT, authentication
+        // will fail later because issuer is mandated
+        final String issuer = jwtClaimsSet.getIssuer();
+        if (Strings.hasText(issuer) == false) {
+            logger.warn("Issuer claim 'iss' is missing.");
+            return new JwtAuthenticationToken("<unrecognized-jwt>", signedJWT, JwtUtil.sha256(userCredentials), clientCredentials);
+        }
+
+        // Try all known extraction functions to build the token principal
+        for (Function<JWTClaimsSet, String> func : tokenPrincipalFunctions) {
+            final String tokenPrincipalSuffix = func.apply(jwtClaimsSet);
+            if (tokenPrincipalSuffix != null) {
+                return new JwtAuthenticationToken(
+                    issuer + "/" + tokenPrincipalSuffix,
+                    signedJWT,
+                    JwtUtil.sha256(userCredentials),
+                    clientCredentials
+                );
+            }
+        }
+
+        // Token principal cannot be extracted even after trying all functions, but this is
+        // still a JWT token so that we should return as one.
+        return new JwtAuthenticationToken("<unrecognized-jwt> by " + issuer, signedJWT, JwtUtil.sha256(userCredentials), clientCredentials);
     }
 
     @Override
@@ -195,8 +269,7 @@ public void authenticate(final AuthenticationToken authenticationToken, final Ac
                 return; // FAILED (secret is missing or mismatched)
             }
 
-            final SecureString serializedJwt = jwtAuthenticationToken.getEndUserSignedJwt();
-            final BytesArray jwtCacheKey = isCacheEnabled() ? new BytesArray(JwtUtil.sha256(serializedJwt)) : null;
+            final BytesArray jwtCacheKey = isCacheEnabled() ? new BytesArray(jwtAuthenticationToken.getUserCredentialsHash()) : null;
             if (jwtCacheKey != null) {
                 final User cachedUser = tryAuthenticateWithCache(tokenPrincipal, jwtCacheKey);
                 if (cachedUser != null) {
@@ -254,40 +327,21 @@ private User tryAuthenticateWithCache(final String tokenPrincipal, final BytesAr
             final Date exp = expiringUser.exp; // claimsSet.getExpirationTime().getTime() + allowedClockSkew.getMillis()
             final String principal = user.principal();
             final Date now = new Date();
-            if (now.getTime() < exp.getTime()) {
-                logger.trace(
-                    "Realm ["
-                        + name()
-                        + "] JWT cache hit token=["
-                        + tokenPrincipal
-                        + "] key=["
-                        + jwtCacheKey
-                        + "] principal=["
-                        + principal
-                        + "] exp=["
-                        + exp
-                        + "] now=["
-                        + now
-                        + "]."
-                );
+            final boolean cacheEntryNotExpired = now.getTime() < exp.getTime();
+            logger.trace(
+                "Realm [{}] JWT cache {} token=[{}] key=[{}] principal=[{}] exp=[{}] now=[{}].",
+                name(),
+                cacheEntryNotExpired ? "hit" : "exp",
+                tokenPrincipal,
+                jwtCacheKey,
+                principal,
+                exp,
+                now
+            );
+            if (cacheEntryNotExpired) {
                 return user;
             }
             // TODO: evict the entry
-            logger.trace(
-                "Realm ["
-                    + name()
-                    + "] JWT cache exp token=["
-                    + tokenPrincipal
-                    + "] key=["
-                    + jwtCacheKey
-                    + "] principal=["
-                    + principal
-                    + "] exp=["
-                    + exp
-                    + "] now=["
-                    + now
-                    + "]."
-            );
         }
         return null;
     }
@@ -400,6 +454,30 @@ private Map<String, Object> buildUserMetadata(JWTClaimsSet claimsSet) {
         return Map.copyOf(metadata);
     }
 
+    private String buildTokenPrincipal(JWTClaimsSet jwtClaimsSet) {
+        final Map<String, String> fallbackClaimNames = jwtAuthenticator.getFallbackClaimNames();
+        final FallbackableClaim subClaim = new FallbackableClaim("sub", fallbackClaimNames, jwtClaimsSet);
+        final String subject = subClaim.getStringClaimValue();
+        if (false == Strings.hasText(subject)) {
+            logger.debug("claim [{}] is missing for building token principal for realm [{}]", subClaim, name());
+            return null;
+        }
+
+        final FallbackableClaim audClaim = new FallbackableClaim("aud", fallbackClaimNames, jwtClaimsSet);
+        final List<String> audiences = audClaim.getStringListClaimValue();
+        if (audiences == null || audiences.isEmpty()) {
+            logger.debug("claim [{}] is missing for building token principal for realm [{}]", audClaim, name());
+            return null;
+        }
+
+        final String userPrincipal = claimParserPrincipal.getClaimValue(jwtClaimsSet);
+        if (false == Strings.hasText(userPrincipal)) {
+            logger.debug("No user principal can be extracted with [{}] for realm [{}]", claimParserPrincipal, name());
+            return null;
+        }
+        return String.join(",", new TreeSet<>(audiences)) + "/" + subject + "/" + userPrincipal;
+    }
+
     /**
      * JWTClaimsSet values are only allowed to be String, Boolean, Number, or Collection.
      * Collections are only allowed to contain String, Boolean, or Number.
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmsService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmsService.java
deleted file mode 100644
index 1be9e86de3dc..000000000000
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmsService.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.security.authc.jwt;
-
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
-import org.elasticsearch.xpack.core.security.authc.RealmConfig;
-import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmsServiceSettings;
-import org.elasticsearch.xpack.core.ssl.SSLService;
-import org.elasticsearch.xpack.security.authc.InternalRealms;
-import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
-
-import java.util.Collections;
-import java.util.List;
-
-/**
- * Parse common settings shared by all JwtRealm instances on behalf of InternalRealms.
- * Construct JwtRealm instances on behalf of lambda defined in InternalRealms.
- * Construct AuthenticationToken instances on behalf of JwtRealm instances.
- * @see InternalRealms
- * @see JwtRealm
- */
-public class JwtRealmsService {
-
-    private final List<String> principalClaimNames;
-
-    /**
-     * Parse all xpack settings passed in from {@link InternalRealms#getFactories}
-     * @param settings All xpack settings
-     */
-    public JwtRealmsService(final Settings settings) {
-        this.principalClaimNames = Collections.unmodifiableList(JwtRealmsServiceSettings.PRINCIPAL_CLAIMS_SETTING.get(settings));
-    }
-
-    /**
-     * Return prioritized list of principal claim names to use for computing realm cache keys for all JWT realms.
-     * @return Prioritized list of principal claim names (ex: sub, oid, client_id, azp, appid, client_id, email).
-     */
-    public List<String> getPrincipalClaimNames() {
-        return this.principalClaimNames;
-    }
-
-    /**
-     * Construct JwtRealm instance using settings passed in via lambda defined in {@link InternalRealms#getFactories}
-     * @param config Realm config
-     * @param sslService SSL service settings
-     * @param nativeRoleMappingStore Native role mapping store
-     */
-    public JwtRealm createJwtRealm(
-        final RealmConfig config,
-        final SSLService sslService,
-        final NativeRoleMappingStore nativeRoleMappingStore
-    ) {
-        return new JwtRealm(config, this, sslService, nativeRoleMappingStore);
-    }
-
-    /**
-     * Construct JwtAuthenticationToken instance using request passed in via JwtRealm.token.
-     * @param threadContext Request headers and parameters
-     * @return JwtAuthenticationToken contains mandatory JWT header, optional client secret, and a realm order cache key
-     */
-    AuthenticationToken token(final ThreadContext threadContext) {
-        // extract value from Authorization header with Bearer scheme prefix
-        final SecureString authenticationParameterValue = JwtUtil.getHeaderValue(
-            threadContext,
-            JwtRealm.HEADER_END_USER_AUTHENTICATION,
-            JwtRealm.HEADER_END_USER_AUTHENTICATION_SCHEME,
-            false
-        );
-        if (authenticationParameterValue == null) {
-            return null;
-        }
-        // extract value from ES-Client-Authentication header with SharedSecret scheme prefix
-        final SecureString clientAuthenticationSharedSecretValue = JwtUtil.getHeaderValue(
-            threadContext,
-            JwtRealm.HEADER_CLIENT_AUTHENTICATION,
-            JwtRealm.HEADER_SHARED_SECRET_AUTHENTICATION_SCHEME,
-            true
-        );
-        return new JwtAuthenticationToken(this.principalClaimNames, authenticationParameterValue, clientAuthenticationSharedSecretValue);
-    }
-}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtSignatureValidator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtSignatureValidator.java
index 93b87ea48aa9..685cea2c54ee 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtSignatureValidator.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtSignatureValidator.java
@@ -254,23 +254,25 @@ class PkcJwtSignatureValidator implements JwtSignatureValidator {
 
         public void validate(String tokenPrincipal, SignedJWT signedJWT, ActionListener<Void> listener) {
             // TODO: assert algorithm?
+            final JwkSetLoader.ContentAndJwksAlgs contentAndJwksAlgs = jwkSetLoader.getContentAndJwksAlgs();
+            final JwkSetLoader.JwksAlgs jwksAlgs = contentAndJwksAlgs.jwksAlgs();
             try {
-                JwtValidateUtil.validateSignature(signedJWT, jwkSetLoader.getContentAndJwksAlgs().jwksAlgs().jwks());
+                JwtValidateUtil.validateSignature(signedJWT, jwksAlgs.jwks());
                 listener.onResponse(null);
             } catch (Exception primaryException) {
                 logger.debug(
                     () -> org.elasticsearch.core.Strings.format(
                         "Signature verification failed for JWT [%s] reloading JWKSet (was: #[%s] JWKs, #[%s] algs, sha256=[%s])",
                         tokenPrincipal,
-                        jwkSetLoader.getContentAndJwksAlgs().jwksAlgs().jwks().size(),
-                        jwkSetLoader.getContentAndJwksAlgs().jwksAlgs().algs().size(),
-                        MessageDigests.toHexString(jwkSetLoader.getContentAndJwksAlgs().sha256())
+                        jwksAlgs.jwks().size(),
+                        jwksAlgs.algs().size(),
+                        MessageDigests.toHexString(contentAndJwksAlgs.sha256())
                     ),
                     primaryException
                 );
 
-                jwkSetLoader.reload(ActionListener.wrap(isUpdated -> {
-                    if (false == isUpdated) {
+                jwkSetLoader.reload(ActionListener.wrap(reloadResult -> {
+                    if (false == reloadResult.v1()) {
                         // No change in JWKSet
                         logger.debug("Reloaded same PKC JWKs, can't retry verify JWT token [{}]", tokenPrincipal);
                         listener.onFailure(primaryException);
@@ -281,13 +283,14 @@ public void validate(String tokenPrincipal, SignedJWT signedJWT, ActionListener<
                     // Enhancement idea: When some JWKs are retained (ex: rotation), only invalidate for removed JWKs.
                     reloadNotifier.reloaded();
 
-                    if (jwkSetLoader.getContentAndJwksAlgs().jwksAlgs().isEmpty()) {
+                    final JwkSetLoader.JwksAlgs reloadedJwksAlgs = reloadResult.v2();
+                    if (reloadedJwksAlgs.isEmpty()) {
                         logger.debug("Reloaded empty PKC JWKs, signature verification will fail for JWT [{}]", tokenPrincipal);
                         // fall through and let try/catch below handle empty JWKs failure log and response
                     }
 
                     try {
-                        JwtValidateUtil.validateSignature(signedJWT, jwkSetLoader.getContentAndJwksAlgs().jwksAlgs().jwks());
+                        JwtValidateUtil.validateSignature(signedJWT, reloadedJwksAlgs.jwks());
                         listener.onResponse(null);
                     } catch (Exception secondaryException) {
                         logger.debug(
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtStringClaimValidator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtStringClaimValidator.java
index add8d0d1336c..30ea0979a624 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtStringClaimValidator.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtStringClaimValidator.java
@@ -10,13 +10,11 @@
 import com.nimbusds.jose.JWSHeader;
 import com.nimbusds.jwt.JWTClaimsSet;
 
-import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.xpack.core.security.support.StringMatcher;
+import org.elasticsearch.core.Nullable;
 
-import java.text.ParseException;
 import java.util.List;
+import java.util.Map;
 
 /**
  * Validates a string claim against a list of allowed values. The validation is successful
@@ -26,62 +24,65 @@
  * values.
  * Whether a claim's value can be an array of strings is customised with the {@link #singleValuedClaim}
  * field, which enforces the claim's value to be a single string if it is configured to {@code true}.
+ *
+ * NOTE the allowed values can be null which means skipping the actual value check, i.e. the validator
+ * succeeds as long as there is a (non-null) value.
  */
 public class JwtStringClaimValidator implements JwtFieldValidator {
 
+    public static JwtStringClaimValidator ALLOW_ALL_SUBJECTS = new JwtStringClaimValidator("sub", null, true);
+
     private final String claimName;
+    @Nullable
+    private final Map<String, String> fallbackClaimNames;
+    @Nullable
     private final List<String> allowedClaimValues;
     // Whether the claim should be a single string
     private final boolean singleValuedClaim;
-    private final StringMatcher claimValueMatcher;
 
     public JwtStringClaimValidator(String claimName, List<String> allowedClaimValues, boolean singleValuedClaim) {
+        this(claimName, null, allowedClaimValues, singleValuedClaim);
+    }
+
+    public JwtStringClaimValidator(
+        String claimName,
+        Map<String, String> fallbackClaimNames,
+        List<String> allowedClaimValues,
+        boolean singleValuedClaim
+    ) {
         this.claimName = claimName;
+        this.fallbackClaimNames = fallbackClaimNames;
         this.allowedClaimValues = allowedClaimValues;
         this.singleValuedClaim = singleValuedClaim;
-        this.claimValueMatcher = StringMatcher.of(allowedClaimValues);
     }
 
     @Override
     public void validate(JWSHeader jwsHeader, JWTClaimsSet jwtClaimsSet) {
-        final List<String> claimValues;
-        try {
-            claimValues = getStringClaimValues(jwtClaimsSet);
-        } catch (ParseException e) {
-            throw new ElasticsearchSecurityException("cannot parse string claim [" + claimName + "]", RestStatus.BAD_REQUEST, e);
-        }
+        final FallbackableClaim fallbackableClaim = new FallbackableClaim(claimName, fallbackClaimNames, jwtClaimsSet);
+        final List<String> claimValues = getStringClaimValues(fallbackableClaim);
         if (claimValues == null) {
-            throw new ElasticsearchSecurityException("missing required string claim [" + claimName + "]", RestStatus.BAD_REQUEST);
+            throw new IllegalArgumentException("missing required string claim [" + fallbackableClaim + "]");
         }
 
-        if (false == claimValues.stream().anyMatch(claimValueMatcher)) {
-            throw new ElasticsearchSecurityException(
+        if (allowedClaimValues != null && false == claimValues.stream().anyMatch(allowedClaimValues::contains)) {
+            throw new IllegalArgumentException(
                 "string claim ["
-                    + claimName
+                    + fallbackableClaim
                     + "] has value ["
                     + Strings.collectionToCommaDelimitedString(claimValues)
                     + "] which does not match allowed claim values ["
                     + Strings.collectionToCommaDelimitedString(allowedClaimValues)
-                    + "]",
-                RestStatus.BAD_REQUEST
+                    + "]"
             );
         }
     }
 
-    private List<String> getStringClaimValues(JWTClaimsSet claimsSet) throws ParseException {
-        // TODO: fallback claims
-        final String actualClaimName = claimName;
-
+    private List<String> getStringClaimValues(FallbackableClaim fallbackableClaim) {
         if (singleValuedClaim) {
-            final String claimValue = claimsSet.getStringClaim(actualClaimName);
+            final String claimValue = fallbackableClaim.getStringClaimValue();
             return claimValue != null ? List.of(claimValue) : null;
         } else {
-            final Object claimValue = claimsSet.getClaim(actualClaimName);
-            if (claimValue instanceof String) {
-                return List.of((String) claimValue);
-            } else {
-                return claimsSet.getStringListClaim(actualClaimName);
-            }
+            return fallbackableClaim.getStringListClaimValue();
         }
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtTypeValidator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtTypeValidator.java
index 5a163c7c566e..8d0dd06d59c2 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtTypeValidator.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtTypeValidator.java
@@ -15,9 +15,6 @@
 import com.nimbusds.jose.proc.SecurityContext;
 import com.nimbusds.jwt.JWTClaimsSet;
 
-import org.elasticsearch.ElasticsearchSecurityException;
-import org.elasticsearch.rest.RestStatus;
-
 public class JwtTypeValidator implements JwtFieldValidator {
 
     private static final JOSEObjectTypeVerifier<SecurityContext> JWT_HEADER_TYPE_VERIFIER = new DefaultJOSEObjectTypeVerifier<>(
@@ -34,7 +31,7 @@ public void validate(JWSHeader jwsHeader, JWTClaimsSet jwtClaimsSet) {
         try {
             JWT_HEADER_TYPE_VERIFIER.verify(jwtHeaderType, null);
         } catch (BadJOSEException e) {
-            throw new ElasticsearchSecurityException("invalid jwt typ header", RestStatus.BAD_REQUEST, e);
+            throw new IllegalArgumentException("invalid jwt typ header", e);
         }
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java
index b4a3f910467c..5b0be7de76fa 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java
@@ -251,7 +251,7 @@ public static CloseableHttpAsyncClient createHttpClient(final RealmConfig realmC
                 final RequestConfig requestConfig = RequestConfig.custom()
                     .setConnectTimeout(Math.toIntExact(realmConfig.getSetting(JwtRealmSettings.HTTP_CONNECT_TIMEOUT).getMillis()))
                     .setConnectionRequestTimeout(
-                        Math.toIntExact(realmConfig.getSetting(JwtRealmSettings.HTTP_CONNECTION_READ_TIMEOUT).getSeconds())
+                        Math.toIntExact(realmConfig.getSetting(JwtRealmSettings.HTTP_CONNECTION_READ_TIMEOUT).getMillis())
                     )
                     .setSocketTimeout(Math.toIntExact(realmConfig.getSetting(JwtRealmSettings.HTTP_SOCKET_TIMEOUT).getMillis()))
                     .build();
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtValidateUtil.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtValidateUtil.java
index 0a80dc705252..65d3244aeb17 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtValidateUtil.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtValidateUtil.java
@@ -9,27 +9,19 @@
 
 import com.nimbusds.jose.JOSEException;
 import com.nimbusds.jose.JWSAlgorithm;
-import com.nimbusds.jose.JWSHeader;
-import com.nimbusds.jose.JWSSigner;
 import com.nimbusds.jose.JWSVerifier;
-import com.nimbusds.jose.crypto.ECDSASigner;
 import com.nimbusds.jose.crypto.ECDSAVerifier;
-import com.nimbusds.jose.crypto.MACSigner;
 import com.nimbusds.jose.crypto.MACVerifier;
-import com.nimbusds.jose.crypto.RSASSASigner;
 import com.nimbusds.jose.crypto.RSASSAVerifier;
 import com.nimbusds.jose.jwk.ECKey;
 import com.nimbusds.jose.jwk.JWK;
 import com.nimbusds.jose.jwk.OctetSequenceKey;
 import com.nimbusds.jose.jwk.RSAKey;
-import com.nimbusds.jose.util.Base64URL;
-import com.nimbusds.jwt.JWTClaimsSet;
 import com.nimbusds.jwt.SignedJWT;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.settings.SecureString;
 
 import java.util.List;
 
@@ -94,7 +86,7 @@ public static void validateSignature(final SignedJWT jwt, final List<JWK> jwks)
         }
 
         for (final JWK jwk : jwksStrength) {
-            if (jwt.verify(JwtValidateUtil.createJwsVerifier(jwk))) {
+            if (jwt.verify(createJwsVerifier(jwk))) {
                 LOGGER.trace(
                     "JWT signature validation succeeded with JWK kty=[{}], jwtAlg=[{}], jwtKid=[{}], use=[{}], ops=[{}]",
                     jwk.getKeyType(),
@@ -127,22 +119,7 @@ public static JWSVerifier createJwsVerifier(final JWK jwk) throws JOSEException
         } else if (jwk instanceof OctetSequenceKey octetSequenceKey) {
             return new MACVerifier(octetSequenceKey);
         }
-        throw JwtValidateUtil.createExceptionInvalidJwkClass(jwk);
-    }
-
-    public static JWSSigner createJwsSigner(final JWK jwk) throws JOSEException {
-        if (jwk instanceof RSAKey rsaKey) {
-            return new RSASSASigner(rsaKey);
-        } else if (jwk instanceof ECKey ecKey) {
-            return new ECDSASigner(ecKey);
-        } else if (jwk instanceof OctetSequenceKey octetSequenceKey) {
-            return new MACSigner(octetSequenceKey);
-        }
-        throw JwtValidateUtil.createExceptionInvalidJwkClass(jwk);
-    }
-
-    private static JOSEException createExceptionInvalidJwkClass(final JWK jwk) {
-        return new JOSEException(
+        throw new JOSEException(
             "Unsupported JWK class ["
                 + (jwk == null ? "null" : jwk.getClass().getCanonicalName())
                 + "]. Supported classes are ["
@@ -154,27 +131,4 @@ private static JOSEException createExceptionInvalidJwkClass(final JWK jwk) {
                 + "]."
         );
     }
-
-    // Build from Base64 components. Signature may or may not be valid. Useful for negative test cases.
-    public static SecureString buildJwt(final JWSHeader header, final JWTClaimsSet claims, final Base64URL signature) throws Exception {
-        final SignedJWT signedJwt = new SignedJWT(header.toBase64URL(), claims.toPayload().toBase64URL(), signature);
-        return new SecureString(signedJwt.serialize().toCharArray());
-    }
-
-    public static SignedJWT buildUnsignedJwt(final JWSHeader jwtHeader, final JWTClaimsSet jwtClaimsSet) {
-        return new SignedJWT(jwtHeader, jwtClaimsSet);
-    }
-
-    // Convenience method to construct JWSVerifier from JWK, and verify the signed JWT
-    public static boolean verifyJwt(final JWK jwk, final SignedJWT signedJwt) throws Exception {
-        return signedJwt.verify(JwtValidateUtil.createJwsVerifier(jwk));
-    }
-
-    // Convenience method to construct JWSSigner from JWK, sign the JWT, and return serialized SecureString
-    public static SecureString signJwt(final JWK jwk, final SignedJWT unsignedJwt) throws Exception {
-        // Copy the header and claims set to a new unsigned JWT, in case JWT is being re-signing
-        final SignedJWT signedJwt = new SignedJWT(unsignedJwt.getHeader(), unsignedJwt.getJWTClaimsSet());
-        signedJwt.sign(JwtValidateUtil.createJwsSigner(jwk));
-        return new SecureString(signedJwt.serialize().toCharArray());
-    }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStore.java
index 134931160c2f..36a0b7667a7a 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStore.java
@@ -47,6 +47,7 @@
 import org.elasticsearch.xpack.core.security.action.service.TokenInfo;
 import org.elasticsearch.xpack.core.security.action.service.TokenInfo.TokenSource;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.Subject;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.security.authc.service.ServiceAccount.ServiceAccountId;
 import org.elasticsearch.xpack.security.authc.service.ServiceAccountToken.ServiceAccountTokenId;
@@ -269,15 +270,16 @@ private XContentBuilder newDocument(Authentication authentication, ServiceAccoun
             .field("creation_time", clock.instant().toEpochMilli())
             .field("enabled", true);
         {
+            final Subject effectiveSubject = authentication.getEffectiveSubject();
             builder.startObject("creator")
-                .field("principal", authentication.getEffectiveSubject().getUser().principal())
-                .field("full_name", authentication.getEffectiveSubject().getUser().fullName())
-                .field("email", authentication.getEffectiveSubject().getUser().email())
-                .field("metadata", authentication.getEffectiveSubject().getUser().metadata())
-                .field("realm", authentication.getSourceRealm().getName())
-                .field("realm_type", authentication.getSourceRealm().getType());
-            if (authentication.getSourceRealm().getDomain() != null) {
-                builder.field("realm_domain", authentication.getSourceRealm().getDomain());
+                .field("principal", effectiveSubject.getUser().principal())
+                .field("full_name", effectiveSubject.getUser().fullName())
+                .field("email", effectiveSubject.getUser().email())
+                .field("metadata", effectiveSubject.getUser().metadata())
+                .field("realm", effectiveSubject.getRealm().getName())
+                .field("realm_type", effectiveSubject.getRealm().getType());
+            if (effectiveSubject.getRealm().getDomain() != null) {
+                builder.field("realm_domain", effectiveSubject.getRealm().getDomain());
             }
             builder.endObject();
         }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ClaimParser.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ClaimParser.java
index 25a42b215d23..0b9ea1d29033 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ClaimParser.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ClaimParser.java
@@ -15,9 +15,11 @@
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
 import org.elasticsearch.xpack.core.security.authc.support.ClaimSetting;
+import org.elasticsearch.xpack.security.authc.jwt.FallbackableClaim;
 
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.function.Function;
 import java.util.regex.Matcher;
@@ -80,9 +82,9 @@ public String toString() {
     }
 
     @SuppressWarnings("unchecked")
-    private static Collection<String> parseClaimValues(JWTClaimsSet claimsSet, String claimName, String settingKey) {
+    private static Collection<String> parseClaimValues(JWTClaimsSet claimsSet, FallbackableClaim fallbackableClaim, String settingKey) {
         Collection<String> values;
-        final Object claimValueObject = claimsSet.getClaim(claimName);
+        final Object claimValueObject = claimsSet.getClaim(fallbackableClaim.getActualName());
         if (claimValueObject == null) {
             values = List.of();
         } else if (claimValueObject instanceof String) {
@@ -91,50 +93,67 @@ private static Collection<String> parseClaimValues(JWTClaimsSet claimsSet, Strin
             && ((Collection<?>) claimValueObject).stream().allMatch(c -> c instanceof String)) {
                 values = (Collection<String>) claimValueObject;
             } else {
-                throw new SettingsException("Setting [ " + settingKey + " expects a claim with String or a String Array value");
+                throw new SettingsException(
+                    "Setting [ " + settingKey + "] expects claim [" + fallbackableClaim + "] with String or a String Array value"
+                );
             }
         return values;
     }
 
     public static ClaimParser forSetting(Logger logger, ClaimSetting setting, RealmConfig realmConfig, boolean required) {
+        return forSetting(logger, setting, Map.of(), realmConfig, required);
+    }
+
+    public static ClaimParser forSetting(
+        Logger logger,
+        ClaimSetting setting,
+        Map<String, String> fallbackClaimNames,
+        RealmConfig realmConfig,
+        boolean required
+    ) {
 
         if (realmConfig.hasSetting(setting.getClaim())) {
-            String claimName = realmConfig.getSetting(setting.getClaim());
+            final String claimName = realmConfig.getSetting(setting.getClaim());
             if (realmConfig.hasSetting(setting.getPattern())) {
                 Pattern regex = Pattern.compile(realmConfig.getSetting(setting.getPattern()));
                 return new ClaimParser(setting.name(realmConfig), claimName, regex.pattern(), claims -> {
+                    final FallbackableClaim fallbackableClaim = new FallbackableClaim(claimName, fallbackClaimNames, claims);
                     Collection<String> values = parseClaimValues(
                         claims,
-                        claimName,
+                        fallbackableClaim,
                         RealmSettings.getFullSettingKey(realmConfig, setting.getClaim())
                     );
                     return values.stream().map(s -> {
                         if (s == null) {
-                            logger.debug("Claim [{}] is null", claimName);
+                            logger.debug("Claim [{}] is null", fallbackableClaim);
                             return null;
                         }
                         final Matcher matcher = regex.matcher(s);
                         if (matcher.find() == false) {
-                            logger.debug("Claim [{}] is [{}], which does not match [{}]", claimName, s, regex.pattern());
+                            logger.debug("Claim [{}] is [{}], which does not match [{}]", fallbackableClaim, s, regex.pattern());
                             return null;
                         }
                         final String value = matcher.group(1);
                         if (Strings.isNullOrEmpty(value)) {
-                            logger.debug("Claim [{}] is [{}], which does match [{}] but group(1) is empty", claimName, s, regex.pattern());
+                            logger.debug(
+                                "Claim [{}] is [{}], which does match [{}] but group(1) is empty",
+                                fallbackableClaim,
+                                s,
+                                regex.pattern()
+                            );
                             return null;
                         }
                         return value;
                     }).filter(Objects::nonNull).toList();
                 });
             } else {
-                return new ClaimParser(
-                    setting.name(realmConfig),
-                    claimName,
-                    null,
-                    claims -> parseClaimValues(claims, claimName, RealmSettings.getFullSettingKey(realmConfig, setting.getClaim())).stream()
+                return new ClaimParser(setting.name(realmConfig), claimName, null, claims -> {
+                    final FallbackableClaim fallbackableClaim = new FallbackableClaim(claimName, fallbackClaimNames, claims);
+                    return parseClaimValues(claims, fallbackableClaim, RealmSettings.getFullSettingKey(realmConfig, setting.getClaim()))
+                        .stream()
                         .filter(Objects::nonNull)
-                        .toList()
-                );
+                        .toList();
+                });
             }
         } else if (required) {
             throw new SettingsException("Setting [" + RealmSettings.getFullSettingKey(realmConfig, setting.getClaim()) + "] is required");
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/CompositeRoleMapper.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/CompositeRoleMapper.java
index d8f0e574e94c..260550f9ff08 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/CompositeRoleMapper.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/CompositeRoleMapper.java
@@ -44,11 +44,11 @@ private CompositeRoleMapper(UserRoleMapper... delegates) {
     @Override
     public void resolveRoles(UserData user, ActionListener<Set<String>> listener) {
         GroupedActionListener<Set<String>> groupListener = new GroupedActionListener<>(
+            delegates.size(),
             ActionListener.wrap(
                 composite -> listener.onResponse(composite.stream().flatMap(Set::stream).collect(Collectors.toSet())),
                 listener::onFailure
-            ),
-            delegates.size()
+            )
         );
         this.delegates.forEach(mapper -> mapper.resolveRoles(user, groupListener));
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
index 6c2fc43d6cfa..56cde154604d 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
@@ -57,10 +57,12 @@
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationResult;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.EmptyAuthorizationInfo;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.IndexAuthorizationResult;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.ParentActionAuthorization;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.RequestInfo;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
 import org.elasticsearch.xpack.core.security.authz.ResolvedIndices;
 import org.elasticsearch.xpack.core.security.authz.RestrictedIndices;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilegeResolver;
@@ -73,6 +75,7 @@
 import org.elasticsearch.xpack.security.audit.AuditTrail;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.elasticsearch.xpack.security.audit.AuditUtil;
+import org.elasticsearch.xpack.security.authz.accesscontrol.wrapper.DlsFlsFeatureTrackingIndicesAccessControlWrapper;
 import org.elasticsearch.xpack.security.authz.interceptor.RequestInterceptor;
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
 import org.elasticsearch.xpack.security.operator.OperatorPrivileges.OperatorPrivilegesService;
@@ -128,6 +131,7 @@ public class AuthorizationService {
     private final XPackLicenseState licenseState;
     private final OperatorPrivilegesService operatorPrivilegesService;
     private final RestrictedIndices restrictedIndices;
+    private final DlsFlsFeatureTrackingIndicesAccessControlWrapper indicesAccessControlWrapper;
 
     private final boolean isAnonymousEnabled;
     private final boolean anonymousAuthzExceptionEnabled;
@@ -167,6 +171,7 @@ public AuthorizationService(
         this.settings = settings;
         this.licenseState = licenseState;
         this.operatorPrivilegesService = operatorPrivilegesService;
+        this.indicesAccessControlWrapper = new DlsFlsFeatureTrackingIndicesAccessControlWrapper(settings, licenseState);
     }
 
     public void checkPrivileges(
@@ -202,6 +207,38 @@ public void retrieveUserPrivileges(
         authorizationEngine.getUserPrivileges(authorizationInfo, wrapPreservingContext(listener, threadContext));
     }
 
+    public void retrieveRemoteAccessRoleDescriptorsIntersection(
+        final String remoteClusterAlias,
+        final Subject subject,
+        final ActionListener<RoleDescriptorsIntersection> listener
+    ) {
+        if (isInternal(subject.getUser())) {
+            final String message = "the user ["
+                + subject.getUser().principal()
+                + "] is an internal user and we should never try to retrieve its remote access roles descriptors";
+            assert false : message;
+            listener.onFailure(new IllegalArgumentException(message));
+            return;
+        }
+
+        final AuthorizationEngine authorizationEngine = getAuthorizationEngineForSubject(subject);
+        final AuthorizationInfo authorizationInfo = threadContext.getTransient(AUTHORIZATION_INFO_KEY);
+        assert authorizationInfo != null : "authorization info must be available in thread context";
+        authorizationEngine.resolveAuthorizationInfo(
+            subject,
+            wrapPreservingContext(
+                listener.delegateFailure(
+                    (delegatedLister, resolvedAuthzInfo) -> authorizationEngine.getRemoteAccessRoleDescriptorsIntersection(
+                        remoteClusterAlias,
+                        resolvedAuthzInfo,
+                        wrapPreservingContext(delegatedLister, threadContext)
+                    )
+                ),
+                threadContext
+            )
+        );
+    }
+
     /**
      * Verifies that the given user can execute the given request (and action). If the user doesn't
      * have the appropriate privileges for this action/request, an {@link ElasticsearchSecurityException}
@@ -220,6 +257,7 @@ public void authorize(
     ) {
 
         final AuthorizationContext enclosingContext = extractAuthorizationContext(threadContext, action);
+        final ParentActionAuthorization parentAuthorization = securityContext.getParentAuthorization();
 
         /* authorization fills in certain transient headers, which must be observed in the listener (action handler execution)
          * as well, but which must not bleed across different action context (eg parent-child action contexts).
@@ -228,7 +266,12 @@ public void authorize(
          * previous parent action that ran under the same thread context (also on the same node).
          * When the returned {@code StoredContext} is closed, ALL the original headers are restored.
          */
-        try (ThreadContext.StoredContext ignore = threadContext.newStoredContext(ACTION_SCOPE_AUTHORIZATION_KEYS)) {
+        try (
+            ThreadContext.StoredContext ignore = threadContext.newStoredContext(
+                ACTION_SCOPE_AUTHORIZATION_KEYS,
+                List.of(ParentActionAuthorization.THREAD_CONTEXT_KEY)
+            )
+        ) {
             // this does not clear {@code AuthorizationServiceField.ORIGINATING_ACTION_KEY}
             // prior to doing any authorization lets set the originating action in the thread context
             // the originating action is the current action if no originating action has yet been set in the current thread context
@@ -258,7 +301,13 @@ public void authorize(
                 // this never goes async so no need to wrap the listener
                 authorizeSystemUser(authentication, action, auditId, unwrappedRequest, listener);
             } else {
-                final RequestInfo requestInfo = new RequestInfo(authentication, unwrappedRequest, action, enclosingContext);
+                final RequestInfo requestInfo = new RequestInfo(
+                    authentication,
+                    unwrappedRequest,
+                    action,
+                    enclosingContext,
+                    parentAuthorization
+                );
                 final AuthorizationEngine engine = getAuthorizationEngine(authentication);
                 final ActionListener<AuthorizationInfo> authzInfoListener = wrapPreservingContext(ActionListener.wrap(authorizationInfo -> {
                     threadContext.putTransient(AUTHORIZATION_INFO_KEY, authorizationInfo);
@@ -466,10 +515,15 @@ private void handleIndexActionAuthorizationResult(
         final Metadata metadata,
         final ActionListener<Void> listener
     ) {
+        final IndicesAccessControl indicesAccessControl = indicesAccessControlWrapper.wrap(result.getIndicesAccessControl());
         final Authentication authentication = requestInfo.getAuthentication();
         final TransportRequest request = requestInfo.getRequest();
         final String action = requestInfo.getAction();
-        securityContext.putIndicesAccessControl(result.getIndicesAccessControl());
+        securityContext.putIndicesAccessControl(indicesAccessControl);
+
+        final AuthorizationContext authzContext = new AuthorizationContext(action, authzInfo, indicesAccessControl);
+        PreAuthorizationUtils.maybeSkipChildrenActionAuthorization(securityContext, authzContext);
+
         // if we are creating an index we need to authorize potential aliases created at the same time
         if (IndexPrivilege.CREATE_INDEX_MATCHER.test(action)) {
             assert (request instanceof CreateIndexRequest)
@@ -481,12 +535,7 @@ private void handleIndexActionAuthorizationResult(
                 runRequestInterceptors(requestInfo, authzInfo, authorizationEngine, listener);
             } else {
                 Set<Alias> aliases = ((CreateIndexRequest) request).aliases();
-                final AuthorizationContext parentContext = new AuthorizationContext(
-                    requestInfo.getAction(),
-                    authzInfo,
-                    result.getIndicesAccessControl()
-                );
-                final RequestInfo aliasesRequestInfo = new RequestInfo(authentication, request, IndicesAliasesAction.NAME, parentContext);
+                final RequestInfo aliasesRequestInfo = new RequestInfo(authentication, request, IndicesAliasesAction.NAME, authzContext);
                 authzEngine.authorizeIndexAction(aliasesRequestInfo, authzInfo, ril -> {
                     resolvedIndicesAsyncSupplier.getAsync(ActionListener.wrap(resolvedIndices -> {
                         List<String> aliasesAndIndices = new ArrayList<>(resolvedIndices.getLocal());
@@ -514,11 +563,6 @@ private void handleIndexActionAuthorizationResult(
             // if this is performing multiple actions on the index, then check each of those actions.
             assert request instanceof BulkShardRequest
                 : "Action " + action + " requires " + BulkShardRequest.class + " but was " + request.getClass();
-            final AuthorizationContext authzContext = new AuthorizationContext(
-                requestInfo.getAction(),
-                authzInfo,
-                result.getIndicesAccessControl()
-            );
             authorizeBulkItems(
                 requestInfo,
                 authzContext,
@@ -787,7 +831,7 @@ private void authorizeBulkItems(
                 listener::onFailure
             );
             final ActionListener<Tuple<String, IndexAuthorizationResult>> groupedActionListener = wrapPreservingContext(
-                new GroupedActionListener<>(bulkAuthzListener, actionToIndicesMap.size()),
+                new GroupedActionListener<>(actionToIndicesMap.size(), bulkAuthzListener),
                 threadContext
             );
 
@@ -796,7 +840,8 @@ private void authorizeBulkItems(
                     requestInfo.getAuthentication(),
                     requestInfo.getRequest(),
                     bulkItemAction,
-                    bulkAuthzContext
+                    bulkAuthzContext,
+                    requestInfo.getParentAuthorization()
                 );
                 authzEngine.authorizeIndexAction(
                     bulkItemInfo,
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java
index fef9512bf0ab..98847df0e9bd 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.security.authz;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.security.SecurityContext;
@@ -107,7 +107,7 @@ public static boolean shouldSetUserBasedOnActionOrigin(ThreadContext context) {
     public static void switchUserBasedOnActionOriginAndExecute(
         ThreadContext threadContext,
         SecurityContext securityContext,
-        Version version,
+        TransportVersion version,
         Consumer<ThreadContext.StoredContext> consumer
     ) {
         final String actionOrigin = threadContext.getTransient(ClientHelper.ACTION_ORIGIN_TRANSIENT_NAME);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
index 3927be8a6ba6..6a693fe357f9 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.transport.RemoteClusterAware;
 import org.elasticsearch.transport.RemoteConnectionStrategy;
 import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
 import org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField;
 import org.elasticsearch.xpack.core.security.authz.ResolvedIndices;
 
@@ -41,6 +42,8 @@
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.concurrent.CopyOnWriteArraySet;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
 import java.util.stream.Stream;
 
 import static org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField.NO_INDEX_PLACEHOLDER;
@@ -87,7 +90,12 @@ class IndicesAndAliasesResolver {
      * Otherwise, <em>N</em> will be added to the <em>local</em> index list.
      */
 
-    ResolvedIndices resolve(String action, TransportRequest request, Metadata metadata, Set<String> authorizedIndices) {
+    ResolvedIndices resolve(
+        String action,
+        TransportRequest request,
+        Metadata metadata,
+        AuthorizationEngine.AuthorizedIndices authorizedIndices
+    ) {
         if (request instanceof IndicesAliasesRequest indicesAliasesRequest) {
             ResolvedIndices.Builder resolvedIndicesBuilder = new ResolvedIndices.Builder();
             for (IndicesRequest indicesRequest : indicesAliasesRequest.getAliasActions()) {
@@ -181,7 +189,7 @@ ResolvedIndices resolveIndicesAndAliases(
         String action,
         IndicesRequest indicesRequest,
         Metadata metadata,
-        Set<String> authorizedIndices
+        AuthorizationEngine.AuthorizedIndices authorizedIndices
     ) {
         final ResolvedIndices.Builder resolvedIndicesBuilder = new ResolvedIndices.Builder();
         boolean indicesReplacedWithNoIndices = false;
@@ -193,14 +201,16 @@ ResolvedIndices resolveIndicesAndAliases(
              */
             assert indicesRequest.indices() == null || indicesRequest.indices().length == 0
                 : "indices are: " + Arrays.toString(indicesRequest.indices()); // Arrays.toString() can handle null values - all good
-            resolvedIndicesBuilder.addLocal(getPutMappingIndexOrAlias((PutMappingRequest) indicesRequest, authorizedIndices, metadata));
+            resolvedIndicesBuilder.addLocal(
+                getPutMappingIndexOrAlias((PutMappingRequest) indicesRequest, authorizedIndices::check, metadata)
+            );
         } else if (indicesRequest instanceof final IndicesRequest.Replaceable replaceable) {
             final IndicesOptions indicesOptions = indicesRequest.indicesOptions();
 
             // check for all and return list of authorized indices
             if (IndexNameExpressionResolver.isAllIndices(indicesList(indicesRequest.indices()))) {
                 if (indicesOptions.expandWildcardExpressions()) {
-                    for (String authorizedIndex : authorizedIndices) {
+                    for (String authorizedIndex : authorizedIndices.all().get()) {
                         if (IndexAbstractionResolver.isIndexVisible(
                             "*",
                             authorizedIndex,
@@ -226,7 +236,8 @@ ResolvedIndices resolveIndicesAndAliases(
                     split.getLocal(),
                     indicesOptions,
                     metadata,
-                    authorizedIndices,
+                    authorizedIndices.all(),
+                    authorizedIndices::check,
                     indicesRequest.includeDataStreams()
                 );
                 resolvedIndicesBuilder.addLocal(replaced);
@@ -262,7 +273,7 @@ ResolvedIndices resolveIndicesAndAliases(
             if (aliasesRequest.expandAliasesWildcards()) {
                 List<String> aliases = replaceWildcardsWithAuthorizedAliases(
                     aliasesRequest.aliases(),
-                    loadAuthorizedAliases(authorizedIndices, metadata)
+                    loadAuthorizedAliases(authorizedIndices.all(), metadata)
                 );
                 aliasesRequest.replaceAliases(aliases.toArray(new String[aliases.size()]));
             }
@@ -304,7 +315,7 @@ ResolvedIndices resolveIndicesAndAliases(
      * request's concrete index is not in the list of authorized indices, then we need to look to
      * see if this can be authorized against an alias
      */
-    static String getPutMappingIndexOrAlias(PutMappingRequest request, Set<String> authorizedIndicesList, Metadata metadata) {
+    static String getPutMappingIndexOrAlias(PutMappingRequest request, Predicate<String> isAuthorized, Metadata metadata) {
         final String concreteIndexName = request.getConcreteIndex().getName();
 
         // validate that the concrete index exists, otherwise there is no remapping that we could do
@@ -320,7 +331,7 @@ static String getPutMappingIndexOrAlias(PutMappingRequest request, Set<String> a
                     + indexAbstraction.getType().getDisplayName()
                     + "], but a concrete index is expected"
             );
-        } else if (authorizedIndicesList.contains(concreteIndexName)) {
+        } else if (isAuthorized.test(concreteIndexName)) {
             // user is authorized to put mappings for this index
             resolvedAliasOrIndex = concreteIndexName;
         } else {
@@ -329,21 +340,17 @@ static String getPutMappingIndexOrAlias(PutMappingRequest request, Set<String> a
             Map<String, List<AliasMetadata>> foundAliases = metadata.findAllAliases(new String[] { concreteIndexName });
             List<AliasMetadata> aliasMetadata = foundAliases.get(concreteIndexName);
             if (aliasMetadata != null) {
-                Optional<String> foundAlias = aliasMetadata.stream()
-                    .map(AliasMetadata::alias)
-                    .filter(authorizedIndicesList::contains)
-                    .filter(aliasName -> {
-                        IndexAbstraction alias = metadata.getIndicesLookup().get(aliasName);
-                        List<Index> indices = alias.getIndices();
-                        if (indices.size() == 1) {
-                            return true;
-                        } else {
-                            assert alias.getType() == IndexAbstraction.Type.ALIAS;
-                            Index writeIndex = alias.getWriteIndex();
-                            return writeIndex != null && writeIndex.getName().equals(concreteIndexName);
-                        }
-                    })
-                    .findFirst();
+                Optional<String> foundAlias = aliasMetadata.stream().map(AliasMetadata::alias).filter(isAuthorized).filter(aliasName -> {
+                    IndexAbstraction alias = metadata.getIndicesLookup().get(aliasName);
+                    List<Index> indices = alias.getIndices();
+                    if (indices.size() == 1) {
+                        return true;
+                    } else {
+                        assert alias.getType() == IndexAbstraction.Type.ALIAS;
+                        Index writeIndex = alias.getWriteIndex();
+                        return writeIndex != null && writeIndex.getName().equals(concreteIndexName);
+                    }
+                }).findFirst();
                 resolvedAliasOrIndex = foundAlias.orElse(concreteIndexName);
             } else {
                 resolvedAliasOrIndex = concreteIndexName;
@@ -353,10 +360,10 @@ static String getPutMappingIndexOrAlias(PutMappingRequest request, Set<String> a
         return resolvedAliasOrIndex;
     }
 
-    private static List<String> loadAuthorizedAliases(Set<String> authorizedIndices, Metadata metadata) {
+    private static List<String> loadAuthorizedAliases(Supplier<Set<String>> authorizedIndices, Metadata metadata) {
         List<String> authorizedAliases = new ArrayList<>();
         SortedMap<String, IndexAbstraction> existingAliases = metadata.getIndicesLookup();
-        for (String authorizedIndex : authorizedIndices) {
+        for (String authorizedIndex : authorizedIndices.get()) {
             IndexAbstraction indexAbstraction = existingAliases.get(authorizedIndex);
             if (indexAbstraction != null && indexAbstraction.getType() == IndexAbstraction.Type.ALIAS) {
                 authorizedAliases.add(authorizedIndex);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/PreAuthorizationUtils.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/PreAuthorizationUtils.java
new file mode 100644
index 000000000000..67a4d14177fa
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/PreAuthorizationUtils.java
@@ -0,0 +1,198 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authz;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchTransportService;
+import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationContext;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.ParentActionAuthorization;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.RequestInfo;
+import org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
+import org.elasticsearch.xpack.core.security.authz.permission.Role;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+public final class PreAuthorizationUtils {
+
+    private static final Logger logger = LogManager.getLogger(PreAuthorizationUtils.class);
+
+    /**
+     * This map holds parent-child action relationships for which we can optimize authorization
+     * and skip authorization for child actions if the parent action is successfully authorized.
+     * Normally every action would be authorized on a local node on which it's being executed.
+     * Here we define all child actions for which the authorization can be safely skipped
+     * on a remote node as they only access a subset of resources.
+     */
+    public static final Map<String, Set<String>> CHILD_ACTIONS_PRE_AUTHORIZED_BY_PARENT = Map.of(
+        SearchAction.NAME,
+        Set.of(
+            SearchTransportService.FREE_CONTEXT_ACTION_NAME,
+            SearchTransportService.DFS_ACTION_NAME,
+            SearchTransportService.QUERY_ACTION_NAME,
+            SearchTransportService.QUERY_ID_ACTION_NAME,
+            SearchTransportService.FETCH_ID_ACTION_NAME,
+            SearchTransportService.QUERY_CAN_MATCH_NAME,
+            SearchTransportService.QUERY_CAN_MATCH_NODE_NAME
+        )
+    );
+
+    /**
+     * This method sets {@link ParentActionAuthorization} as a header in the thread context,
+     * which will be used for skipping authorization of child actions if the following conditions are met:
+     *
+     * <ul>
+     * <li>parent action is one of the white listed in {@link #CHILD_ACTIONS_PRE_AUTHORIZED_BY_PARENT}</li>
+     * <li>FLS and DLS are not configured</li>
+     * <li>RBACEngine was used to authorize parent request and not a custom authorization engine</li>
+     * </ul>
+     */
+    public static void maybeSkipChildrenActionAuthorization(
+        SecurityContext securityContext,
+        AuthorizationContext parentAuthorizationContext
+    ) {
+        final String parentAction = parentAuthorizationContext.getAction();
+        if (CHILD_ACTIONS_PRE_AUTHORIZED_BY_PARENT.containsKey(parentAction) == false) {
+            // This is not one of the white listed parent actions.
+            return;
+        }
+
+        final IndicesAccessControl indicesAccessControl = parentAuthorizationContext.getIndicesAccessControl();
+        if (indicesAccessControl == null) {
+            // This can happen if the parent request was authorized by index name only - e.g. bulk request
+            // A missing IAC is not an error, but it means we can't safely tie authz of the child action to the parent authz
+            return;
+        }
+
+        // Just a sanity check. If we ended up here, the authz should have been granted.
+        if (indicesAccessControl.isGranted() == false) {
+            return;
+        }
+
+        final Role role = RBACEngine.maybeGetRBACEngineRole(parentAuthorizationContext.getAuthorizationInfo());
+        if (role == null) {
+            // If role is null, it means a custom authorization engine is in use, hence we cannot do the optimization here.
+            return;
+        }
+
+        // We can't safely pre-authorize actions if DLS or FLS is configured without passing IAC as well with the authorization result.
+        // For simplicity, we only pre-authorize actions when FLS and DLS are not configured, otherwise we would have to compute and send
+        // indices access control as well, which we want to avoid with this optimization.
+        if (role.hasFieldOrDocumentLevelSecurity()) {
+            return;
+        }
+
+        final ParentActionAuthorization existingParentAuthorization = securityContext.getParentAuthorization();
+        if (existingParentAuthorization != null) {
+            throw new AssertionError(
+                "found parent authorization for action ["
+                    + existingParentAuthorization.action()
+                    + "] while attempting to set authorization for new parent action ["
+                    + parentAction
+                    + "]"
+            );
+        } else {
+            if (logger.isDebugEnabled()) {
+                logger.debug("adding authorization for parent action [" + parentAction + "] to the thread context");
+            }
+            securityContext.setParentAuthorization(new ParentActionAuthorization(parentAction));
+        }
+    }
+
+    private static boolean shouldPreAuthorizeChildActionOfParent(final String parent, final String child) {
+        final Set<String> children = CHILD_ACTIONS_PRE_AUTHORIZED_BY_PARENT.get(parent);
+        return children != null && children.contains(child);
+    }
+
+    public static boolean shouldRemoveParentAuthorizationFromThreadContext(
+        Optional<String> remoteClusterAlias,
+        String childAction,
+        SecurityContext securityContext
+    ) {
+        final ParentActionAuthorization parentAuthorization = securityContext.getParentAuthorization();
+        if (parentAuthorization == null) {
+            // Nothing to remove.
+            return false;
+        }
+
+        if (remoteClusterAlias.isPresent()) {
+            // We never want to send the parent authorization header to remote clusters.
+            return true;
+        }
+
+        if (shouldPreAuthorizeChildActionOfParent(parentAuthorization.action(), childAction) == false) {
+            // We want to remove the parent authorization header if the child action is not one of the white listed.
+            return true;
+        }
+
+        return false;
+    }
+
+    public static boolean shouldPreAuthorizeChildByParentAction(RequestInfo childRequestInfo, AuthorizationInfo childAuthorizationInfo) {
+
+        final ParentActionAuthorization parentAuthorization = childRequestInfo.getParentAuthorization();
+        if (parentAuthorization == null) {
+            return false;
+        }
+
+        Role role = RBACEngine.maybeGetRBACEngineRole(childAuthorizationInfo);
+        if (role == null) {
+            // If role is null, it means a custom authorization engine is in use.
+            return false;
+        }
+        if (role.hasFieldOrDocumentLevelSecurity()) {
+            // We can't safely pre-authorize actions if DLS or FLS is configured
+            // without sending IAC as well with authorization result.
+            return false;
+        }
+
+        final String parentAction = parentAuthorization.action();
+        final String childAction = childRequestInfo.getAction();
+        if (shouldPreAuthorizeChildActionOfParent(parentAction, childAction) == false) {
+            // We only pre-authorize explicitly allowed child actions.
+            return false;
+        }
+
+        final IndicesRequest indicesRequest;
+        if (childRequestInfo.getRequest() instanceof IndicesRequest) {
+            indicesRequest = (IndicesRequest) childRequestInfo.getRequest();
+        } else {
+            // Can only handle indices request here
+            return false;
+        }
+
+        final String[] indices = indicesRequest.indices();
+        if (indices == null || indices.length == 0) {
+            // No indices to check
+            return false;
+        }
+
+        if (Arrays.equals(IndicesAndAliasesResolverField.NO_INDICES_OR_ALIASES_ARRAY, indices)) {
+            // Special placeholder for no indices.
+            // We probably can short circuit this, but it's safer not to and just fall through to the regular authorization
+            return false;
+        }
+
+        if (logger.isDebugEnabled()) {
+            logger.debug("pre-authorizing child action [" + childAction + "] of parent action [" + parentAction + "]");
+        }
+        return true;
+    }
+
+    private PreAuthorizationUtils() {
+        throw new IllegalAccessError();
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
index a2a1564e8667..fce05a60356b 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.CachedSupplier;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.transport.TransportActionProxy;
@@ -60,10 +61,12 @@
 import org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField;
 import org.elasticsearch.xpack.core.security.authz.ResolvedIndices;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCache;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition;
 import org.elasticsearch.xpack.core.security.authz.permission.IndicesPermission;
+import org.elasticsearch.xpack.core.security.authz.permission.IndicesPermission.IsResourceAuthorizedPredicate;
 import org.elasticsearch.xpack.core.security.authz.permission.RemoteIndicesPermission;
 import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges;
 import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivilegesMap;
@@ -81,12 +84,12 @@
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -118,6 +121,8 @@ public class RBACEngine implements AuthorizationEngine {
     private static final String DELETE_SUB_REQUEST_REPLICA = DeleteAction.NAME + "[r]";
 
     private static final Logger logger = LogManager.getLogger(RBACEngine.class);
+    // TODO move once we have a dedicated class for RCS 2.0 constants
+    public static final String REMOTE_USER_ROLE_NAME = "_remote_user";
 
     private final Settings settings;
     private final CompositeRolesStore rolesStore;
@@ -351,8 +356,12 @@ public void authorizeIndexAction(
                     )
                 );
             }
-        } else if (isChildActionAuthorizedByParent(requestInfo, authorizationInfo)) {
+        } else if (isChildActionAuthorizedByParentOnLocalNode(requestInfo, authorizationInfo)) {
             listener.onResponse(new IndexAuthorizationResult(requestInfo.getOriginatingAuthorizationContext().getIndicesAccessControl()));
+        } else if (PreAuthorizationUtils.shouldPreAuthorizeChildByParentAction(requestInfo, authorizationInfo)) {
+            // We only pre-authorize child actions if DLS/FLS is not configured,
+            // hence we can allow here access for all requested indices.
+            listener.onResponse(new IndexAuthorizationResult(IndicesAccessControl.allowAll()));
         } else if (allowsRemoteIndices(request) || role.checkIndicesAction(action)) {
             indicesAsyncSupplier.getAsync(ActionListener.wrap(resolvedIndices -> {
                 assert resolvedIndices.isEmpty() == false
@@ -386,7 +395,7 @@ private static boolean allowsRemoteIndices(TransportRequest transportRequest) {
         return transportRequest instanceof IndicesRequest.Replaceable replaceable && replaceable.allowsRemoteIndices();
     }
 
-    private static boolean isChildActionAuthorizedByParent(RequestInfo requestInfo, AuthorizationInfo authorizationInfo) {
+    private static boolean isChildActionAuthorizedByParentOnLocalNode(RequestInfo requestInfo, AuthorizationInfo authorizationInfo) {
         final AuthorizationContext parent = requestInfo.getOriginatingAuthorizationContext();
         if (parent == null) {
             return false;
@@ -455,7 +464,7 @@ public void loadAuthorizedIndices(
         RequestInfo requestInfo,
         AuthorizationInfo authorizationInfo,
         Map<String, IndexAbstraction> indicesLookup,
-        ActionListener<Set<String>> listener
+        ActionListener<AuthorizationEngine.AuthorizedIndices> listener
     ) {
         if (authorizationInfo instanceof RBACAuthorizationInfo) {
             final Role role = ((RBACAuthorizationInfo) authorizationInfo).getRole();
@@ -657,6 +666,94 @@ public void getUserPrivileges(AuthorizationInfo authorizationInfo, ActionListene
         }
     }
 
+    @Override
+    public void getRemoteAccessRoleDescriptorsIntersection(
+        final String remoteClusterAlias,
+        final AuthorizationInfo authorizationInfo,
+        final ActionListener<RoleDescriptorsIntersection> listener
+    ) {
+        if (authorizationInfo instanceof RBACAuthorizationInfo == false) {
+            listener.onFailure(
+                new IllegalArgumentException("unsupported authorization info:" + authorizationInfo.getClass().getSimpleName())
+            );
+            return;
+        }
+
+        final Role role = ((RBACAuthorizationInfo) authorizationInfo).getRole();
+        final RemoteIndicesPermission remoteIndicesPermission;
+        try {
+            remoteIndicesPermission = role.remoteIndices().forCluster(remoteClusterAlias);
+        } catch (UnsupportedOperationException e) {
+            // TODO we will need to implement this to support API keys with assigned role descriptors
+            listener.onFailure(
+                new IllegalArgumentException(
+                    "cannot retrieve remote access role descriptors for API keys with assigned role descriptors.",
+                    e
+                )
+            );
+            return;
+        }
+
+        if (remoteIndicesPermission.remoteIndicesGroups().isEmpty()) {
+            listener.onResponse(RoleDescriptorsIntersection.EMPTY);
+            return;
+        }
+
+        final List<RoleDescriptor.IndicesPrivileges> indicesPrivileges = new ArrayList<>();
+        for (RemoteIndicesPermission.RemoteIndicesGroup remoteIndicesGroup : remoteIndicesPermission.remoteIndicesGroups()) {
+            for (IndicesPermission.Group indicesGroup : remoteIndicesGroup.indicesPermissionGroups()) {
+                indicesPrivileges.add(toIndicesPrivileges(indicesGroup));
+            }
+        }
+
+        listener.onResponse(
+            new RoleDescriptorsIntersection(
+                List.of(
+                    Set.of(
+                        new RoleDescriptor(
+                            REMOTE_USER_ROLE_NAME,
+                            null,
+                            // The role descriptors constructed here may be cached in raw byte form, using a hash of their content as a
+                            // cache key; we therefore need deterministic order when constructing them here, to ensure cache hits for
+                            // equivalent role descriptors
+                            indicesPrivileges.stream().sorted().toArray(RoleDescriptor.IndicesPrivileges[]::new),
+                            null,
+                            null,
+                            null,
+                            null,
+                            null
+                        )
+                    )
+                )
+            )
+        );
+    }
+
+    private static RoleDescriptor.IndicesPrivileges toIndicesPrivileges(final IndicesPermission.Group indicesGroup) {
+        final Set<BytesReference> queries = indicesGroup.getQuery();
+        final Set<FieldPermissionsDefinition.FieldGrantExcludeGroup> fieldGrantExcludeGroups = getFieldGrantExcludeGroups(indicesGroup);
+        assert queries == null || queries.size() <= 1
+            : "translation from an indices permission group to indices privileges supports up to one DLS query but multiple queries found";
+        assert fieldGrantExcludeGroups.size() <= 1
+            : "translation from an indices permission group to indices privileges supports up to one FLS field-grant-exclude group"
+                + " but multiple groups found";
+
+        final BytesReference query = (queries == null || false == queries.iterator().hasNext()) ? null : queries.iterator().next();
+        final RoleDescriptor.IndicesPrivileges.Builder builder = RoleDescriptor.IndicesPrivileges.builder()
+            // Sort because these index privileges will be part of role descriptors that may be cached in raw byte form;
+            // we need deterministic order to ensure cache hits for equivalent role descriptors
+            .indices(Arrays.stream(indicesGroup.indices()).sorted().collect(Collectors.toList()))
+            .privileges(indicesGroup.privilege().name().stream().sorted().collect(Collectors.toList()))
+            .allowRestrictedIndices(indicesGroup.allowRestrictedIndices())
+            .query(query);
+        if (false == fieldGrantExcludeGroups.isEmpty()) {
+            final FieldPermissionsDefinition.FieldGrantExcludeGroup fieldGrantExcludeGroup = fieldGrantExcludeGroups.iterator().next();
+            builder.grantedFields(fieldGrantExcludeGroup.getGrantedFields()).deniedFields(fieldGrantExcludeGroup.getExcludedFields());
+        }
+
+        return builder.build();
+    }
+
     static GetUserPrivilegesResponse buildUserPrivilegesResponseObject(Role userRole) {
         logger.trace(() -> "List privileges for role [" + arrayToCommaDelimitedString(userRole.names()) + "]");
 
@@ -721,17 +818,7 @@ static GetUserPrivilegesResponse buildUserPrivilegesResponseObject(Role userRole
 
     private static GetUserPrivilegesResponse.Indices toIndices(final IndicesPermission.Group group) {
         final Set<BytesReference> queries = group.getQuery() == null ? Collections.emptySet() : group.getQuery();
-        final Set<FieldPermissionsDefinition.FieldGrantExcludeGroup> fieldSecurity;
-        if (group.getFieldPermissions().hasFieldLevelSecurity()) {
-            final List<FieldPermissionsDefinition> fieldPermissionsDefinitions = group.getFieldPermissions()
-                .getFieldPermissionsDefinitions();
-            assert fieldPermissionsDefinitions.size() == 1
-                : "limited-by field must not exist since we do not support reporting user privileges for limited roles";
-            final FieldPermissionsDefinition definition = fieldPermissionsDefinitions.get(0);
-            fieldSecurity = definition.getFieldGrantExcludeGroups();
-        } else {
-            fieldSecurity = Collections.emptySet();
-        }
+        final Set<FieldPermissionsDefinition.FieldGrantExcludeGroup> fieldSecurity = getFieldGrantExcludeGroups(group);
         return new GetUserPrivilegesResponse.Indices(
             Arrays.asList(group.indices()),
             group.privilege().name(),
@@ -741,23 +828,32 @@ private static GetUserPrivilegesResponse.Indices toIndices(final IndicesPermissi
         );
     }
 
-    static Set<String> resolveAuthorizedIndicesFromRole(Role role, RequestInfo requestInfo, Map<String, IndexAbstraction> lookup) {
-        return resolveAuthorizedIndicesFromRole(role, requestInfo, lookup, () -> LoadAuthorizedIndicesTimeChecker.NO_OP_CONSUMER);
+    private static Set<FieldPermissionsDefinition.FieldGrantExcludeGroup> getFieldGrantExcludeGroups(IndicesPermission.Group group) {
+        if (group.getFieldPermissions().hasFieldLevelSecurity()) {
+            final List<FieldPermissionsDefinition> fieldPermissionsDefinitions = group.getFieldPermissions()
+                .getFieldPermissionsDefinitions();
+            assert fieldPermissionsDefinitions.size() == 1
+                : "limited-by field must not exist since we do not support reporting user privileges for limited roles";
+            final FieldPermissionsDefinition definition = fieldPermissionsDefinitions.get(0);
+            return definition.getFieldGrantExcludeGroups();
+        } else {
+            return Collections.emptySet();
+        }
     }
 
-    static Set<String> resolveAuthorizedIndicesFromRole(
+    static AuthorizedIndices resolveAuthorizedIndicesFromRole(
         Role role,
         RequestInfo requestInfo,
         Map<String, IndexAbstraction> lookup,
         Supplier<Consumer<Collection<String>>> timerSupplier
     ) {
-        Predicate<IndexAbstraction> predicate = role.allowedIndicesMatcher(requestInfo.getAction());
+        IsResourceAuthorizedPredicate predicate = role.allowedIndicesMatcher(requestInfo.getAction());
 
         // do not include data streams for actions that do not operate on data streams
         TransportRequest request = requestInfo.getRequest();
         final boolean includeDataStreams = (request instanceof IndicesRequest) && ((IndicesRequest) request).includeDataStreams();
 
-        return new AuthorizedIndicesSet(() -> {
+        return new AuthorizedIndices(() -> {
             Consumer<Collection<String>> timeChecker = timerSupplier.get();
             Set<String> indicesAndAliases = new HashSet<>();
             // TODO: can this be done smarter? I think there are usually more indices/aliases in the cluster then indices defined a roles?
@@ -785,19 +881,15 @@ static Set<String> resolveAuthorizedIndicesFromRole(
         }, name -> {
             final IndexAbstraction indexAbstraction = lookup.get(name);
             if (indexAbstraction == null) {
-                return false;
-            }
-            if (includeDataStreams) {
+                // test access (by name) to a resource that does not currently exist
+                // the action handler must handle the case of accessing resources that do not exist
+                return predicate.test(name, null);
+            } else {
                 // We check the parent data stream first if there is one. For testing requested indices, this is most likely
                 // more efficient than checking the index name first because we recommend grant privileges over data stream
                 // instead of backing indices.
-                if (indexAbstraction.getParentDataStream() != null && predicate.test(indexAbstraction.getParentDataStream())) {
-                    return true;
-                } else {
-                    return predicate.test(indexAbstraction);
-                }
-            } else {
-                return indexAbstraction.getType() != IndexAbstraction.Type.DATA_STREAM && predicate.test(indexAbstraction);
+                return (indexAbstraction.getParentDataStream() != null && predicate.test(indexAbstraction.getParentDataStream()))
+                    || predicate.test(indexAbstraction);
             }
         });
     }
@@ -931,103 +1023,24 @@ private static boolean isAsyncRelatedAction(String action) {
             || action.equals(SqlAsyncActionNames.SQL_ASYNC_GET_RESULT_ACTION_NAME);
     }
 
-    /**
-     * A lazily loaded Set for authorized indices. It avoids loading the set if only contains check is required.
-     * It only loads the set if iterating through it is necessary, i.e. when expanding wildcards.
-     * <p>
-     * NOTE that the lazy loading is NOT thread-safe and must NOT be used by multi-threads.
-     * The current usage has it wrapped inside a CachingAsyncSupplier which guarantees it to be accessed
-     * from a single thread. Extra caution is needed if moving or using this class in other places.
-     */
-    static class AuthorizedIndicesSet implements Set<String> {
-
-        private final Supplier<Set<String>> supplier;
-        private final Predicate<String> predicate;
-        private Set<String> authorizedIndices = null;
-
-        AuthorizedIndicesSet(Supplier<Set<String>> supplier, Predicate<String> predicate) {
-            this.supplier = Objects.requireNonNull(supplier);
-            this.predicate = Objects.requireNonNull(predicate);
-        }
-
-        private Set<String> getAuthorizedIndices() {
-            if (authorizedIndices == null) {
-                authorizedIndices = supplier.get();
-            }
-            return authorizedIndices;
-        }
-
-        @Override
-        public int size() {
-            return getAuthorizedIndices().size();
-        }
-
-        @Override
-        public boolean isEmpty() {
-            return getAuthorizedIndices().isEmpty();
-        }
-
-        @Override
-        public boolean contains(Object o) {
-            if (authorizedIndices == null) {
-                return predicate.test((String) o);
-            } else {
-                return authorizedIndices.contains(o);
-            }
-        }
-
-        @Override
-        public Iterator<String> iterator() {
-            return getAuthorizedIndices().iterator();
-        }
-
-        @Override
-        public Object[] toArray() {
-            return getAuthorizedIndices().toArray();
-        }
-
-        @Override
-        public <T> T[] toArray(T[] a) {
-            return getAuthorizedIndices().toArray(a);
-        }
+    static final class AuthorizedIndices implements AuthorizationEngine.AuthorizedIndices {
 
-        @Override
-        public boolean add(String s) {
-            throw new UnsupportedOperationException();
-        }
+        private final CachedSupplier<Set<String>> allAuthorizedAndAvailableSupplier;
+        private final Predicate<String> isAuthorizedPredicate;
 
-        @Override
-        public boolean remove(Object o) {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public boolean containsAll(Collection<?> c) {
-            if (authorizedIndices == null) {
-                return c.stream().allMatch(this::contains);
-            } else {
-                return authorizedIndices.containsAll(c);
-            }
-        }
-
-        @Override
-        public boolean addAll(Collection<? extends String> c) {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public boolean retainAll(Collection<?> c) {
-            throw new UnsupportedOperationException();
+        AuthorizedIndices(Supplier<Set<String>> allAuthorizedAndAvailableSupplier, Predicate<String> isAuthorizedPredicate) {
+            this.allAuthorizedAndAvailableSupplier = new CachedSupplier<>(allAuthorizedAndAvailableSupplier);
+            this.isAuthorizedPredicate = Objects.requireNonNull(isAuthorizedPredicate);
         }
 
         @Override
-        public boolean removeAll(Collection<?> c) {
-            throw new UnsupportedOperationException();
+        public Supplier<Set<String>> all() {
+            return allAuthorizedAndAvailableSupplier;
         }
 
         @Override
-        public void clear() {
-            throw new UnsupportedOperationException();
+        public boolean check(String name) {
+            return this.isAuthorizedPredicate.test(name);
         }
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractor.java
index bc5fde79f080..bf802caa2746 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractor.java
@@ -8,7 +8,6 @@
 
 import org.apache.lucene.index.PrefixCodedTerms.TermIterator;
 import org.apache.lucene.queries.spans.SpanTermQuery;
-import org.apache.lucene.sandbox.search.DocValuesNumbersQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.DisjunctionMaxQuery;
@@ -35,8 +34,8 @@
  * such as highlighting, not security. For example terms in a Boolean {@code MUST_NOT} clause
  * are not included, TermsQuery doesn't implement the method as it could be terribly slow, etc.
  *
- * TODO reimplement this using QueryVisitors
  */
+// TODO this should be rewritten using the Lucene QueryVisitor API
 class FieldExtractor {
 
     /**
@@ -76,8 +75,6 @@ static void extractFields(Query query, Set<String> fields) throws UnsupportedOpe
             fields.add(pointInSetQuery.getField());
         } else if (query instanceof FieldExistsQuery fieldExistsQuery) {
             fields.add(fieldExistsQuery.getField());
-        } else if (query instanceof DocValuesNumbersQuery docValuesNumbersQuery) {
-            fields.add(docValuesNumbersQuery.getField());
         } else if (query instanceof IndexOrDocValuesQuery indexOrDocValuesQuery) {
             // Both queries are supposed to be equivalent, so if any of them can be extracted, we are good
             try {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/wrapper/DlsFlsFeatureTrackingIndicesAccessControlWrapper.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/wrapper/DlsFlsFeatureTrackingIndicesAccessControlWrapper.java
new file mode 100644
index 000000000000..01215c981de0
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/wrapper/DlsFlsFeatureTrackingIndicesAccessControlWrapper.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authz.accesscontrol.wrapper;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
+
+import static org.elasticsearch.xpack.core.security.SecurityField.DOCUMENT_LEVEL_SECURITY_FEATURE;
+import static org.elasticsearch.xpack.core.security.SecurityField.FIELD_LEVEL_SECURITY_FEATURE;
+
+/**
+ * The wrapper of {@link IndicesAccessControl} which adds ability to track actual Document and Field Level Security feature usage.
+ */
+public class DlsFlsFeatureTrackingIndicesAccessControlWrapper {
+
+    private final XPackLicenseState licenseState;
+    private final boolean isDlsFlsEnabled;
+
+    public DlsFlsFeatureTrackingIndicesAccessControlWrapper(Settings settings, XPackLicenseState licenseState) {
+        this.licenseState = licenseState;
+        this.isDlsFlsEnabled = XPackSettings.DLS_FLS_ENABLED.get(settings);
+    }
+
+    public IndicesAccessControl wrap(IndicesAccessControl indicesAccessControl) {
+        if (isDlsFlsEnabled == false
+            || indicesAccessControl == null
+            || indicesAccessControl == IndicesAccessControl.ALLOW_NO_INDICES
+            || indicesAccessControl == IndicesAccessControl.DENIED
+            || indicesAccessControl == IndicesAccessControl.allowAll()
+            || indicesAccessControl instanceof DlsFlsFeatureUsageTracker) {
+            // Wrap only if it's not one of the statically defined nor already wrapped.
+            return indicesAccessControl;
+        } else {
+            return new DlsFlsFeatureUsageTracker(indicesAccessControl, licenseState);
+        }
+    }
+
+    private static class DlsFlsFeatureUsageTracker extends IndicesAccessControl {
+
+        private final XPackLicenseState licenseState;
+
+        private DlsFlsFeatureUsageTracker(IndicesAccessControl wrapped, XPackLicenseState licenseState) {
+            super(wrapped);
+            this.licenseState = licenseState;
+        }
+
+        @Override
+        public IndexAccessControl getIndexPermissions(String index) {
+            final IndexAccessControl iac = super.getIndexPermissions(index);
+            if (iac != null) {
+                trackUsage(iac);
+            }
+            return iac;
+        }
+
+        private void trackUsage(IndexAccessControl iac) {
+            if (iac.getDocumentPermissions().hasDocumentLevelPermissions()) {
+                boolean dlsLicensed = DOCUMENT_LEVEL_SECURITY_FEATURE.check(licenseState);
+                assert dlsLicensed : "DLS feature should be licensed before usage";
+            }
+            if (iac.getFieldPermissions().hasFieldLevelSecurity()) {
+                boolean flsLicensed = FIELD_LEVEL_SECURITY_FEATURE.check(licenseState);
+                assert flsLicensed : "FLS feature should be licensed before usage";
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/BulkShardRequestInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/BulkShardRequestInterceptor.java
index bce7ed956106..d948a5c3faa5 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/BulkShardRequestInterceptor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/BulkShardRequestInterceptor.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
 
 import static org.elasticsearch.xpack.core.security.SecurityField.DOCUMENT_LEVEL_SECURITY_FEATURE;
+import static org.elasticsearch.xpack.core.security.SecurityField.FIELD_LEVEL_SECURITY_FEATURE;
 
 /**
  * Similar to {@link UpdateRequestInterceptor}, but checks if there are update requests embedded in a bulk request.
@@ -47,7 +48,9 @@ public void intercept(
         AuthorizationInfo authorizationInfo,
         ActionListener<Void> listener
     ) {
-        if (requestInfo.getRequest()instanceof BulkShardRequest bulkShardRequest) {
+        final boolean isDlsLicensed = DOCUMENT_LEVEL_SECURITY_FEATURE.checkWithoutTracking(licenseState);
+        final boolean isFlsLicensed = FIELD_LEVEL_SECURITY_FEATURE.checkWithoutTracking(licenseState);
+        if (requestInfo.getRequest()instanceof BulkShardRequest bulkShardRequest && (isDlsLicensed || isFlsLicensed)) {
             IndicesAccessControl indicesAccessControl = threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY);
             // this uses the {@code BulkShardRequest#index()} because the {@code bulkItemRequest#index()}
             // can still be an unresolved date math expression
@@ -55,10 +58,7 @@ public void intercept(
             // TODO replace if condition with assertion
             if (indexAccessControl != null
                 && (indexAccessControl.getFieldPermissions().hasFieldLevelSecurity()
-                    || indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions())
-                && DOCUMENT_LEVEL_SECURITY_FEATURE.checkWithoutTracking(licenseState) // the feature usage checker is a "last-ditch"
-                                                                                      // verification, it doesn't have practical importance
-            ) {
+                    || indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions())) {
                 for (BulkItemRequest bulkItemRequest : bulkShardRequest.items()) {
                     boolean found = false;
                     if (bulkItemRequest.request() instanceof UpdateRequest) {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/DlsFlsLicenseRequestInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/DlsFlsLicenseRequestInterceptor.java
index 7fcf855d6084..73cb2bea3441 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/DlsFlsLicenseRequestInterceptor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/DlsFlsLicenseRequestInterceptor.java
@@ -56,46 +56,52 @@ public void intercept(
                 final IndicesAccessControl indicesAccessControl = threadContext.getTransient(INDICES_PERMISSIONS_KEY);
                 if (indicesAccessControl != null) {
                     final XPackLicenseState frozenLicenseState = licenseState.copyCurrentLicenseState();
-                    final IndicesAccessControl.DlsFlsUsage dlsFlsUsage = indicesAccessControl.getFieldAndDocumentLevelSecurityUsage();
-                    boolean incompatibleLicense = false;
-                    if (dlsFlsUsage.hasFieldLevelSecurity()) {
-                        logger.debug(
-                            () -> format(
-                                "User [%s] has field level security on [%s]",
-                                requestInfo.getAuthentication(),
-                                indicesAccessControl.getIndicesWithFieldLevelSecurity()
-                            )
-                        );
-                        if (false == FIELD_LEVEL_SECURITY_FEATURE.check(frozenLicenseState)) {
-                            incompatibleLicense = true;
+                    if (logger.isDebugEnabled()) {
+                        final IndicesAccessControl.DlsFlsUsage dlsFlsUsage = indicesAccessControl.getFieldAndDocumentLevelSecurityUsage();
+                        if (dlsFlsUsage.hasFieldLevelSecurity()) {
+                            logger.debug(
+                                () -> format(
+                                    "User [%s] has field level security on [%s]",
+                                    requestInfo.getAuthentication(),
+                                    indicesAccessControl.getIndicesWithFieldLevelSecurity()
+                                )
+                            );
+                        }
+                        if (dlsFlsUsage.hasDocumentLevelSecurity()) {
+                            logger.debug(
+                                () -> format(
+                                    "User [%s] has document level security on [%s]",
+                                    requestInfo.getAuthentication(),
+                                    indicesAccessControl.getIndicesWithDocumentLevelSecurity()
+                                )
+                            );
                         }
                     }
-                    if (dlsFlsUsage.hasDocumentLevelSecurity()) {
-                        logger.debug(
-                            () -> format(
-                                "User [%s] has document level security on [%s]",
-                                requestInfo.getAuthentication(),
-                                indicesAccessControl.getIndicesWithDocumentLevelSecurity()
-                            )
-                        );
-                        if (false == DOCUMENT_LEVEL_SECURITY_FEATURE.check(frozenLicenseState)) {
+                    if (false == DOCUMENT_LEVEL_SECURITY_FEATURE.checkWithoutTracking(frozenLicenseState)
+                        || false == FIELD_LEVEL_SECURITY_FEATURE.checkWithoutTracking(frozenLicenseState)) {
+                        boolean incompatibleLicense = false;
+                        IndicesAccessControl.DlsFlsUsage dlsFlsUsage = indicesAccessControl.getFieldAndDocumentLevelSecurityUsage();
+                        if (dlsFlsUsage.hasDocumentLevelSecurity() && false == DOCUMENT_LEVEL_SECURITY_FEATURE.check(frozenLicenseState)) {
                             incompatibleLicense = true;
                         }
-                    }
-                    if (incompatibleLicense) {
-                        final ElasticsearchSecurityException licenseException = LicenseUtils.newComplianceException(
-                            "field and document level security"
-                        );
-                        licenseException.addMetadata(
-                            "es.indices_with_dls_or_fls",
-                            indicesAccessControl.getIndicesWithFieldOrDocumentLevelSecurity()
-                        );
-                        listener.onFailure(licenseException);
-                        return;
+                        if (dlsFlsUsage.hasFieldLevelSecurity() && false == FIELD_LEVEL_SECURITY_FEATURE.check(frozenLicenseState)) {
+                            incompatibleLicense = true;
+                        }
+
+                        if (incompatibleLicense) {
+                            final ElasticsearchSecurityException licenseException = LicenseUtils.newComplianceException(
+                                "field and document level security"
+                            );
+                            licenseException.addMetadata(
+                                "es.indices_with_dls_or_fls",
+                                indicesAccessControl.getIndicesWithFieldOrDocumentLevelSecurity()
+                            );
+                            listener.onFailure(licenseException);
+                            return;
+                        }
                     }
                 }
             }
-
         }
         listener.onResponse(null);
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/FieldAndDocumentLevelSecurityRequestInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/FieldAndDocumentLevelSecurityRequestInterceptor.java
index c63a0197f4fa..8c1e4ff0bb78 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/FieldAndDocumentLevelSecurityRequestInterceptor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/FieldAndDocumentLevelSecurityRequestInterceptor.java
@@ -23,6 +23,7 @@
 import java.util.Map;
 
 import static org.elasticsearch.xpack.core.security.SecurityField.DOCUMENT_LEVEL_SECURITY_FEATURE;
+import static org.elasticsearch.xpack.core.security.SecurityField.FIELD_LEVEL_SECURITY_FEATURE;
 
 /**
  * Base class for interceptors that disables features when field level security is configured for indices a request
@@ -47,36 +48,33 @@ public void intercept(
         AuthorizationInfo authorizationInfo,
         ActionListener<Void> listener
     ) {
+        final boolean isDlsLicensed = DOCUMENT_LEVEL_SECURITY_FEATURE.checkWithoutTracking(licenseState);
+        final boolean isFlsLicensed = FIELD_LEVEL_SECURITY_FEATURE.checkWithoutTracking(licenseState);
         if (requestInfo.getRequest()instanceof IndicesRequest indicesRequest
-            && false == TransportActionProxy.isProxyAction(requestInfo.getAction())) {
-            // TODO: should we check is DLS/FLS feature allowed here
-            if (supports(indicesRequest)) {
-                final boolean isDlsLicensed = DOCUMENT_LEVEL_SECURITY_FEATURE.checkWithoutTracking(licenseState);
-                final IndicesAccessControl indicesAccessControl = threadContext.getTransient(
-                    AuthorizationServiceField.INDICES_PERMISSIONS_KEY
-                );
-                final Map<String, IndicesAccessControl.IndexAccessControl> accessControlByIndex = new HashMap<>();
-                for (String index : requestIndices(indicesRequest)) {
-                    IndicesAccessControl.IndexAccessControl indexAccessControl = indicesAccessControl.getIndexPermissions(index);
-                    if (indexAccessControl != null
-                        && (indexAccessControl.getFieldPermissions().hasFieldLevelSecurity()
-                            || indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions())
-                        && isDlsLicensed) {
-                        logger.trace(
-                            "intercepted request for index [{}] with field level access controls [{}] "
-                                + "document level access controls [{}]. disabling conflicting features",
-                            index,
-                            indexAccessControl.getFieldPermissions().hasFieldLevelSecurity(),
-                            indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions()
-                        );
-                        accessControlByIndex.put(index, indexAccessControl);
-                    }
-                }
-                if (false == accessControlByIndex.isEmpty()) {
-                    disableFeatures(indicesRequest, accessControlByIndex, listener);
-                    return;
+            && false == TransportActionProxy.isProxyAction(requestInfo.getAction())
+            && supports(indicesRequest)
+            && (isDlsLicensed || isFlsLicensed)) {
+            final IndicesAccessControl indicesAccessControl = threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY);
+            final Map<String, IndicesAccessControl.IndexAccessControl> accessControlByIndex = new HashMap<>();
+            for (String index : requestIndices(indicesRequest)) {
+                IndicesAccessControl.IndexAccessControl indexAccessControl = indicesAccessControl.getIndexPermissions(index);
+                if (indexAccessControl != null
+                    && (indexAccessControl.getFieldPermissions().hasFieldLevelSecurity()
+                        || indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions())) {
+                    logger.trace(
+                        "intercepted request for index [{}] with field level access controls [{}] "
+                            + "document level access controls [{}]. disabling conflicting features",
+                        index,
+                        indexAccessControl.getFieldPermissions().hasFieldLevelSecurity(),
+                        indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions()
+                    );
+                    accessControlByIndex.put(index, indexAccessControl);
                 }
             }
+            if (false == accessControlByIndex.isEmpty()) {
+                disableFeatures(indicesRequest, accessControlByIndex, listener);
+                return;
+            }
         }
         listener.onResponse(null);
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptor.java
index c1def1632430..5635cc468f2e 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/IndicesAliasesRequestInterceptor.java
@@ -31,6 +31,7 @@
 
 import static org.elasticsearch.action.support.ContextPreservingActionListener.wrapPreservingContext;
 import static org.elasticsearch.xpack.core.security.SecurityField.DOCUMENT_LEVEL_SECURITY_FEATURE;
+import static org.elasticsearch.xpack.core.security.SecurityField.FIELD_LEVEL_SECURITY_FEATURE;
 
 public final class IndicesAliasesRequestInterceptor implements RequestInterceptor {
 
@@ -55,27 +56,28 @@ public void intercept(
         AuthorizationInfo authorizationInfo,
         ActionListener<Void> listener
     ) {
-        if (requestInfo.getRequest() instanceof IndicesAliasesRequest) {
-            final IndicesAliasesRequest request = (IndicesAliasesRequest) requestInfo.getRequest();
+        if (requestInfo.getRequest()instanceof IndicesAliasesRequest request) {
             final AuditTrail auditTrail = auditTrailService.get();
             final boolean isDlsLicensed = DOCUMENT_LEVEL_SECURITY_FEATURE.checkWithoutTracking(licenseState);
+            final boolean isFlsLicensed = FIELD_LEVEL_SECURITY_FEATURE.checkWithoutTracking(licenseState);
             IndicesAccessControl indicesAccessControl = threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY);
-            for (IndicesAliasesRequest.AliasActions aliasAction : request.getAliasActions()) {
-                if (aliasAction.actionType() == IndicesAliasesRequest.AliasActions.Type.ADD) {
-                    for (String index : aliasAction.indices()) {
-                        IndicesAccessControl.IndexAccessControl indexAccessControl = indicesAccessControl.getIndexPermissions(index);
-                        if (indexAccessControl != null
-                            && (indexAccessControl.getFieldPermissions().hasFieldLevelSecurity()
-                                || indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions())
-                            && isDlsLicensed) {
-                            listener.onFailure(
-                                new ElasticsearchSecurityException(
-                                    "Alias requests are not allowed for "
-                                        + "users who have field or document level security enabled on one of the indices",
-                                    RestStatus.BAD_REQUEST
-                                )
-                            );
-                            return;
+            if (isDlsLicensed || isFlsLicensed) {
+                for (IndicesAliasesRequest.AliasActions aliasAction : request.getAliasActions()) {
+                    if (aliasAction.actionType() == IndicesAliasesRequest.AliasActions.Type.ADD) {
+                        for (String index : aliasAction.indices()) {
+                            IndicesAccessControl.IndexAccessControl indexAccessControl = indicesAccessControl.getIndexPermissions(index);
+                            if (indexAccessControl != null
+                                && (indexAccessControl.getFieldPermissions().hasFieldLevelSecurity()
+                                    || indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions())) {
+                                listener.onFailure(
+                                    new ElasticsearchSecurityException(
+                                        "Alias requests are not allowed for "
+                                            + "users who have field or document level security enabled on one of the indices",
+                                        RestStatus.BAD_REQUEST
+                                    )
+                                );
+                                return;
+                            }
                         }
                     }
                 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptor.java
index 35d7bd2cffc0..ec138c57dd86 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptor.java
@@ -26,6 +26,7 @@
 
 import static org.elasticsearch.action.support.ContextPreservingActionListener.wrapPreservingContext;
 import static org.elasticsearch.xpack.core.security.SecurityField.DOCUMENT_LEVEL_SECURITY_FEATURE;
+import static org.elasticsearch.xpack.core.security.SecurityField.FIELD_LEVEL_SECURITY_FEATURE;
 import static org.elasticsearch.xpack.security.audit.AuditUtil.extractRequestId;
 
 public final class ResizeRequestInterceptor implements RequestInterceptor {
@@ -47,23 +48,27 @@ public void intercept(
         AuthorizationInfo authorizationInfo,
         ActionListener<Void> listener
     ) {
-        if (requestInfo.getRequest() instanceof ResizeRequest) {
-            final ResizeRequest request = (ResizeRequest) requestInfo.getRequest();
+        if (requestInfo.getRequest()instanceof ResizeRequest request) {
             final AuditTrail auditTrail = auditTrailService.get();
-            IndicesAccessControl indicesAccessControl = threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY);
-            IndicesAccessControl.IndexAccessControl indexAccessControl = indicesAccessControl.getIndexPermissions(request.getSourceIndex());
-            if (indexAccessControl != null
-                && (indexAccessControl.getFieldPermissions().hasFieldLevelSecurity()
-                    || indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions())
-                && DOCUMENT_LEVEL_SECURITY_FEATURE.checkWithoutTracking(licenseState)) {
-                listener.onFailure(
-                    new ElasticsearchSecurityException(
-                        "Resize requests are not allowed for users when "
-                            + "field or document level security is enabled on the source index",
-                        RestStatus.BAD_REQUEST
-                    )
+            final boolean isDlsLicensed = DOCUMENT_LEVEL_SECURITY_FEATURE.checkWithoutTracking(licenseState);
+            final boolean isFlsLicensed = FIELD_LEVEL_SECURITY_FEATURE.checkWithoutTracking(licenseState);
+            if (isDlsLicensed || isFlsLicensed) {
+                IndicesAccessControl indicesAccessControl = threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY);
+                IndicesAccessControl.IndexAccessControl indexAccessControl = indicesAccessControl.getIndexPermissions(
+                    request.getSourceIndex()
                 );
-                return;
+                if (indexAccessControl != null
+                    && (indexAccessControl.getFieldPermissions().hasFieldLevelSecurity()
+                        || indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions())) {
+                    listener.onFailure(
+                        new ElasticsearchSecurityException(
+                            "Resize requests are not allowed for users when "
+                                + "field or document level security is enabled on the source index",
+                            RestStatus.BAD_REQUEST
+                        )
+                    );
+                    return;
+                }
             }
 
             authorizationEngine.validateIndexPermissionsAreSubset(
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestCacheDisablingInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestCacheDisablingInterceptor.java
new file mode 100644
index 000000000000..6edac2fb2d38
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestCacheDisablingInterceptor.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.security.authz.interceptor;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportActionProxy;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
+
+import java.util.Arrays;
+
+import static org.elasticsearch.transport.RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR;
+import static org.elasticsearch.xpack.core.security.SecurityField.DOCUMENT_LEVEL_SECURITY_FEATURE;
+import static org.elasticsearch.xpack.core.security.SecurityField.FIELD_LEVEL_SECURITY_FEATURE;
+
+public class SearchRequestCacheDisablingInterceptor implements RequestInterceptor {
+
+    private final ThreadContext threadContext;
+    private final XPackLicenseState licenseState;
+
+    public SearchRequestCacheDisablingInterceptor(ThreadPool threadPool, XPackLicenseState licenseState) {
+        this.threadContext = threadPool.getThreadContext();
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    public void intercept(
+        AuthorizationEngine.RequestInfo requestInfo,
+        AuthorizationEngine authorizationEngine,
+        AuthorizationEngine.AuthorizationInfo authorizationInfo,
+        ActionListener<Void> listener
+    ) {
+        final boolean isDlsLicensed = DOCUMENT_LEVEL_SECURITY_FEATURE.checkWithoutTracking(licenseState);
+        final boolean isFlsLicensed = FIELD_LEVEL_SECURITY_FEATURE.checkWithoutTracking(licenseState);
+        if (requestInfo.getRequest()instanceof SearchRequest searchRequest
+            && false == TransportActionProxy.isProxyAction(requestInfo.getAction())
+            && hasRemoteIndices(searchRequest)
+            && (isDlsLicensed || isFlsLicensed)) {
+            final IndicesAccessControl indicesAccessControl = threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY);
+            if (indicesAccessControl.getFieldAndDocumentLevelSecurityUsage() != IndicesAccessControl.DlsFlsUsage.NONE) {
+                searchRequest.requestCache(false);
+            }
+        }
+        listener.onResponse(null);
+    }
+
+    // package private for test
+    static boolean hasRemoteIndices(SearchRequest request) {
+        return Arrays.stream(request.indices()).anyMatch(name -> name.indexOf(REMOTE_CLUSTER_INDEX_SEPARATOR) >= 0);
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestInterceptor.java
index 1a3066661015..37839ae45670 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestInterceptor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestInterceptor.java
@@ -13,15 +13,11 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
 
-import java.util.Arrays;
 import java.util.Map;
 
-import static org.elasticsearch.transport.RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR;
-
 public class SearchRequestInterceptor extends FieldAndDocumentLevelSecurityRequestInterceptor {
 
     private final ClusterService clusterService;
@@ -38,21 +34,15 @@ void disableFeatures(
         ActionListener<Void> listener
     ) {
         final SearchRequest request = (SearchRequest) indicesRequest;
-        if (hasRemoteIndices(request)) {
-            request.requestCache(false);
-        }
-
-        final SearchSourceBuilder source = request.source();
-
         if (indexAccessControlByIndex.values().stream().anyMatch(iac -> iac.getDocumentPermissions().hasDocumentLevelPermissions())) {
-            if (source != null && source.suggest() != null) {
+            if (hasSuggest(request)) {
                 listener.onFailure(
                     new ElasticsearchSecurityException(
                         "Suggest isn't supported if document level security is enabled",
                         RestStatus.BAD_REQUEST
                     )
                 );
-            } else if (source != null && source.profile()) {
+            } else if (hasProfile(request)) {
                 listener.onFailure(
                     new ElasticsearchSecurityException(
                         "A search request cannot be profiled if document level security " + "is enabled",
@@ -69,11 +59,19 @@ void disableFeatures(
 
     @Override
     public boolean supports(IndicesRequest request) {
-        return request instanceof SearchRequest;
+        if (request instanceof SearchRequest searchRequest) {
+            return hasSuggest(searchRequest) || hasProfile(searchRequest);
+        } else {
+            return false;
+        }
+    }
+
+    private static boolean hasSuggest(SearchRequest searchRequest) {
+        return searchRequest.source() != null && searchRequest.source().suggest() != null;
     }
 
-    // package private for test
-    static boolean hasRemoteIndices(SearchRequest request) {
-        return Arrays.stream(request.indices()).anyMatch(name -> name.indexOf(REMOTE_CLUSTER_INDEX_SEPARATOR) >= 0);
+    private static boolean hasProfile(SearchRequest searchRequest) {
+        return searchRequest.source() != null && searchRequest.source().profile();
     }
+
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java
index 10b8df8410fa..dc595935d8b9 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java
@@ -376,8 +376,8 @@ public void getRoleDescriptorsList(Subject subject, ActionListener<Collection<Se
             () -> {
                 final List<RoleReference> roleReferences = subject.getRoleReferenceIntersection(anonymousUser).getRoleReferences();
                 final GroupedActionListener<Set<RoleDescriptor>> groupedActionListener = new GroupedActionListener<>(
-                    listener,
-                    roleReferences.size()
+                    roleReferences.size(),
+                    listener
                 );
 
                 roleReferences.forEach(roleReference -> {
@@ -435,9 +435,7 @@ public static void buildRoleFromDescriptors(
         final Map<Set<String>, MergeableIndicesPrivilege> indicesPrivilegesMap = new HashMap<>();
         final Map<Set<String>, MergeableIndicesPrivilege> restrictedIndicesPrivilegesMap = new HashMap<>();
 
-        // Outer map keyed by remote cluster alias expressions, inner map by index name expressions
-        final Map<Set<String>, Map<Set<String>, MergeableIndicesPrivilege>> remoteIndicesPrivilegesMap = new HashMap<>();
-        final Map<Set<String>, Map<Set<String>, MergeableIndicesPrivilege>> restrictedRemoteIndicesPrivilegesMap = new HashMap<>();
+        final Map<Set<String>, Set<IndicesPrivileges>> remoteIndicesPrivilegesByCluster = new HashMap<>();
 
         // Keyed by application + resource
         final Map<Tuple<String, Set<String>>, Set<String>> applicationPrivilegesMap = new HashMap<>();
@@ -459,13 +457,7 @@ public static void buildRoleFromDescriptors(
             MergeableIndicesPrivilege.collatePrivilegesByIndices(descriptor.getIndicesPrivileges(), false, indicesPrivilegesMap);
 
             if (descriptor.hasRemoteIndicesPrivileges()) {
-                final RemoteIndicesPrivileges[] remoteIndicesPrivileges = descriptor.getRemoteIndicesPrivileges();
-                MergeableIndicesPrivilege.collatePrivilegesByRemoteIndices(
-                    remoteIndicesPrivileges,
-                    true,
-                    restrictedRemoteIndicesPrivilegesMap
-                );
-                MergeableIndicesPrivilege.collatePrivilegesByRemoteIndices(remoteIndicesPrivileges, false, remoteIndicesPrivilegesMap);
+                groupIndexPrivilegesByCluster(descriptor.getRemoteIndicesPrivileges(), remoteIndicesPrivilegesByCluster);
             }
 
             for (RoleDescriptor.ApplicationResourcePrivileges appPrivilege : descriptor.getApplicationPrivileges()) {
@@ -504,27 +496,17 @@ public static void buildRoleFromDescriptors(
             )
         );
 
-        remoteIndicesPrivilegesMap.forEach((clusterAliasKey, remoteIndicesPrivilegesMapForCluster) -> {
-            remoteIndicesPrivilegesMapForCluster.forEach(
-                (key, privilege) -> builder.addRemoteGroup(
-                    clusterAliasKey,
-                    fieldPermissionsCache.getFieldPermissions(privilege.fieldPermissionsDefinition),
-                    privilege.query,
-                    IndexPrivilege.get(privilege.privileges),
-                    false,
-                    privilege.indices.toArray(Strings.EMPTY_ARRAY)
-                )
-            );
-        });
-        restrictedRemoteIndicesPrivilegesMap.forEach((clusterAliasKey, remoteIndicesPrivilegesMapForCluster) -> {
-            remoteIndicesPrivilegesMapForCluster.forEach(
-                (key, privilege) -> builder.addRemoteGroup(
+        remoteIndicesPrivilegesByCluster.forEach((clusterAliasKey, remoteIndicesPrivilegesForCluster) -> {
+            remoteIndicesPrivilegesForCluster.forEach(
+                (privilege) -> builder.addRemoteGroup(
                     clusterAliasKey,
-                    fieldPermissionsCache.getFieldPermissions(privilege.fieldPermissionsDefinition),
-                    privilege.query,
-                    IndexPrivilege.get(privilege.privileges),
-                    true,
-                    privilege.indices.toArray(Strings.EMPTY_ARRAY)
+                    fieldPermissionsCache.getFieldPermissions(
+                        new FieldPermissionsDefinition(privilege.getGrantedFields(), privilege.getDeniedFields())
+                    ),
+                    privilege.getQuery() == null ? null : newHashSet(privilege.getQuery()),
+                    IndexPrivilege.get(newHashSet(Objects.requireNonNull(privilege.getPrivileges()))),
+                    privilege.allowRestrictedIndices(),
+                    newHashSet(Objects.requireNonNull(privilege.getIndices())).toArray(new String[0])
                 )
             );
         });
@@ -591,6 +573,25 @@ boolean isValueInNegativeLookupCache(String key) {
         return negativeLookupCache.get(key) != null;
     }
 
+    private static void groupIndexPrivilegesByCluster(
+        final RemoteIndicesPrivileges[] remoteIndicesPrivileges,
+        final Map<Set<String>, Set<IndicesPrivileges>> remoteIndexPrivilegesByCluster
+    ) {
+        assert remoteIndicesPrivileges != null;
+        // if a remote index privilege is an explicit denial, then we treat it as non-existent to stay consistent with local index
+        // privileges
+        final boolean isExplicitDenial = remoteIndicesPrivileges.length == 1
+            && "none".equalsIgnoreCase(remoteIndicesPrivileges[0].indicesPrivileges().getPrivileges()[0]);
+        if (isExplicitDenial) {
+            return;
+        }
+        for (final RemoteIndicesPrivileges remoteIndicesPrivilege : remoteIndicesPrivileges) {
+            final IndicesPrivileges indicesPrivilege = remoteIndicesPrivilege.indicesPrivileges();
+            final Set<String> clusterAliasKey = newHashSet(remoteIndicesPrivilege.remoteClusters());
+            remoteIndexPrivilegesByCluster.computeIfAbsent(clusterAliasKey, k -> new HashSet<>()).add(indicesPrivilege);
+        }
+    }
+
     /**
      * A mutable class that can be used to represent the combination of one or more {@link IndicesPrivileges}
      */
@@ -630,29 +631,6 @@ void merge(MergeableIndicesPrivilege other) {
             }
         }
 
-        private static void collatePrivilegesByRemoteIndices(
-            final RemoteIndicesPrivileges[] remoteIndicesPrivileges,
-            final boolean allowsRestrictedIndices,
-            final Map<Set<String>, Map<Set<String>, MergeableIndicesPrivilege>> remoteIndicesPrivilegesMap
-        ) {
-            assert remoteIndicesPrivileges != null;
-            // if a remote index privilege is an explicit denial, then we treat it as non-existent to stay consistent with local index
-            // privileges
-            final boolean isExplicitDenial = remoteIndicesPrivileges.length == 1
-                && "none".equalsIgnoreCase(remoteIndicesPrivileges[0].indicesPrivileges().getPrivileges()[0]);
-            if (isExplicitDenial) {
-                return;
-            }
-            for (final RemoteIndicesPrivileges remoteIndicesPrivilege : remoteIndicesPrivileges) {
-                final Set<String> clusterAlias = newHashSet(remoteIndicesPrivilege.remoteClusters());
-                collatePrivilegesByIndices(
-                    remoteIndicesPrivilege.indicesPrivileges(),
-                    allowsRestrictedIndices,
-                    remoteIndicesPrivilegesMap.computeIfAbsent(clusterAlias, k -> new HashMap<>())
-                );
-            }
-        }
-
         private static void collatePrivilegesByIndices(
             final IndicesPrivileges[] indicesPrivileges,
             final boolean allowsRestrictedIndices,
@@ -666,41 +644,33 @@ private static void collatePrivilegesByIndices(
                 return;
             }
             for (final IndicesPrivileges indicesPrivilege : indicesPrivileges) {
-                collatePrivilegesByIndices(indicesPrivilege, allowsRestrictedIndices, indicesPrivilegesMap);
-            }
-        }
-
-        private static void collatePrivilegesByIndices(
-            final IndicesPrivileges indicesPrivilege,
-            final boolean allowsRestrictedIndices,
-            final Map<Set<String>, MergeableIndicesPrivilege> indicesPrivilegesMap
-        ) {
-            if (indicesPrivilege.allowRestrictedIndices() != allowsRestrictedIndices) {
-                return;
-            }
-            final Set<String> key = newHashSet(indicesPrivilege.getIndices());
-            indicesPrivilegesMap.compute(key, (k, value) -> {
-                if (value == null) {
-                    return new MergeableIndicesPrivilege(
-                        indicesPrivilege.getIndices(),
-                        indicesPrivilege.getPrivileges(),
-                        indicesPrivilege.getGrantedFields(),
-                        indicesPrivilege.getDeniedFields(),
-                        indicesPrivilege.getQuery()
-                    );
-                } else {
-                    value.merge(
-                        new MergeableIndicesPrivilege(
+                if (indicesPrivilege.allowRestrictedIndices() != allowsRestrictedIndices) {
+                    continue;
+                }
+                final Set<String> key = newHashSet(indicesPrivilege.getIndices());
+                indicesPrivilegesMap.compute(key, (k, value) -> {
+                    if (value == null) {
+                        return new MergeableIndicesPrivilege(
                             indicesPrivilege.getIndices(),
                             indicesPrivilege.getPrivileges(),
                             indicesPrivilege.getGrantedFields(),
                             indicesPrivilege.getDeniedFields(),
                             indicesPrivilege.getQuery()
-                        )
-                    );
-                    return value;
-                }
-            });
+                        );
+                    } else {
+                        value.merge(
+                            new MergeableIndicesPrivilege(
+                                indicesPrivilege.getIndices(),
+                                indicesPrivilege.getPrivileges(),
+                                indicesPrivilege.getGrantedFields(),
+                                indicesPrivilege.getDeniedFields(),
+                                indicesPrivilege.getQuery()
+                            )
+                        );
+                        return value;
+                    }
+                });
+            }
         }
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java
index ba2f78121550..f97bdc926f8b 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java
@@ -334,6 +334,7 @@ public void putPrivileges(
     ) {
         securityIndexManager.prepareIndexIfNeededThenExecute(listener::onFailure, () -> {
             ActionListener<IndexResponse> groupListener = new GroupedActionListener<>(
+                privileges.size(),
                 ActionListener.wrap((Collection<IndexResponse> responses) -> {
                     final Map<String, List<String>> createdNames = responses.stream()
                         .filter(r -> r.getResult() == DocWriteResponse.Result.CREATED)
@@ -345,8 +346,7 @@ public void putPrivileges(
                         privileges.stream().map(ApplicationPrivilegeDescriptor::getApplication).collect(Collectors.toUnmodifiableSet()),
                         createdNames
                     );
-                }, listener::onFailure),
-                privileges.size()
+                }, listener::onFailure)
             );
             for (ApplicationPrivilegeDescriptor privilege : privileges) {
                 innerPutPrivilege(privilege, refreshPolicy, groupListener);
@@ -392,14 +392,14 @@ public void deletePrivileges(
             listener.onFailure(frozenSecurityIndex.getUnavailableReason());
         } else {
             securityIndexManager.checkIndexVersionThenExecute(listener::onFailure, () -> {
-                ActionListener<DeleteResponse> groupListener = new GroupedActionListener<>(ActionListener.wrap(responses -> {
+                ActionListener<DeleteResponse> groupListener = new GroupedActionListener<>(names.size(), ActionListener.wrap(responses -> {
                     final Map<String, List<String>> deletedNames = responses.stream()
                         .filter(r -> r.getResult() == DocWriteResponse.Result.DELETED)
                         .map(r -> r.getId())
                         .map(NativePrivilegeStore::nameFromDocId)
                         .collect(TUPLES_TO_MAP);
                     clearCaches(listener, Collections.singleton(application), deletedNames);
-                }, listener::onFailure), names.size());
+                }, listener::onFailure));
                 for (String name : names) {
                     ClientHelper.executeAsyncWithOrigin(
                         client.threadPool().getThreadContext(),
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
index 8222850a06c6..b4fbe7a5bf61 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
@@ -222,7 +222,8 @@ public void putRole(final PutRoleRequest request, final RoleDescriptor role, fin
         if (role.isUsingDocumentOrFieldLevelSecurity() && DOCUMENT_LEVEL_SECURITY_FEATURE.checkWithoutTracking(licenseState) == false) {
             listener.onFailure(LicenseUtils.newComplianceException("field and document level security"));
         } else if (role.hasRemoteIndicesPrivileges()
-            && clusterService.state().nodes().getMinNodeVersion().before(RoleDescriptor.VERSION_REMOTE_INDICES)) {
+            // TODO here a node version is mixed with a transportVersion. We should look into this once Node's Version is refactored
+            && clusterService.state().nodes().getMinNodeVersion().transportVersion.before(RoleDescriptor.VERSION_REMOTE_INDICES)) {
                 listener.onFailure(
                     new IllegalStateException(
                         "all nodes must have version ["
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/RoleDescriptorStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/RoleDescriptorStore.java
index 756849193fe6..acae8a3e255f 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/RoleDescriptorStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/RoleDescriptorStore.java
@@ -132,6 +132,21 @@ public void resolveServiceAccountRoleReference(
         }));
     }
 
+    @Override
+    public void resolveRemoteAccessRoleReference(
+        RoleReference.RemoteAccessRoleReference remoteAccessRoleReference,
+        ActionListener<RolesRetrievalResult> listener
+    ) {
+        final Set<RoleDescriptor> roleDescriptors = remoteAccessRoleReference.getRoleDescriptorsBytes().toRoleDescriptors();
+        if (roleDescriptors.isEmpty()) {
+            listener.onResponse(RolesRetrievalResult.EMPTY);
+            return;
+        }
+        final RolesRetrievalResult rolesRetrievalResult = new RolesRetrievalResult();
+        rolesRetrievalResult.addDescriptors(Set.copyOf(roleDescriptors));
+        listener.onResponse(rolesRetrievalResult);
+    }
+
     private void resolveRoleNames(Set<String> roleNames, ActionListener<RolesRetrievalResult> listener) {
         roleDescriptors(roleNames, ActionListener.wrap(rolesRetrievalResult -> {
             logDeprecatedRoles(rolesRetrievalResult.getRoleDescriptors());
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/FileOperatorUsersStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/FileOperatorUsersStore.java
index c65f75b1deb8..ee3496b91aff 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/FileOperatorUsersStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/FileOperatorUsersStore.java
@@ -68,8 +68,11 @@ public boolean isOperatorUser(Authentication authentication) {
         // If not null, it will be compared exactly as well.
         // The special handling for realm name is because there can only be one file or native realm and it does
         // not matter what the name is.
+        final Authentication.RealmRef realm = authentication.getEffectiveSubject().getRealm();
+        if (realm == null) {
+            return false;
+        }
         return operatorUsersDescriptor.groups.stream().anyMatch(group -> {
-            final Authentication.RealmRef realm = authentication.getSourceRealm();
             final boolean match = group.usernames.contains(authentication.getEffectiveSubject().getUser().principal())
                 && group.authenticationType == authentication.getAuthenticationType()
                 && realm.getType().equals(group.realmType)
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java
index d0266b05bed9..f8a0a7e1a1d0 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java
@@ -398,6 +398,11 @@ private XContentBuilder getMainIndexMappings() {
                     builder.field("format", "epoch_millis");
                     builder.endObject();
 
+                    builder.startObject("invalidation_time");
+                    builder.field("type", "date");
+                    builder.field("format", "epoch_millis");
+                    builder.endObject();
+
                     builder.startObject("api_key_hash");
                     builder.field("type", "keyword");
                     builder.field("index", false);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityHttpExceptionHandler.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityHttpExceptionHandler.java
deleted file mode 100644
index ef8b0e56f313..000000000000
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityHttpExceptionHandler.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.security.transport;
-
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.component.Lifecycle;
-import org.elasticsearch.common.network.CloseableChannel;
-import org.elasticsearch.http.HttpChannel;
-
-import java.util.function.BiConsumer;
-
-import static org.elasticsearch.xpack.core.security.transport.SSLExceptionHelper.isCloseDuringHandshakeException;
-import static org.elasticsearch.xpack.core.security.transport.SSLExceptionHelper.isInsufficientBufferRemainingException;
-import static org.elasticsearch.xpack.core.security.transport.SSLExceptionHelper.isNotSslRecordException;
-import static org.elasticsearch.xpack.core.security.transport.SSLExceptionHelper.isReceivedCertificateUnknownException;
-
-public final class SecurityHttpExceptionHandler implements BiConsumer<HttpChannel, Exception> {
-
-    private final Lifecycle lifecycle;
-    private final Logger logger;
-    private final BiConsumer<HttpChannel, Exception> fallback;
-
-    public SecurityHttpExceptionHandler(Logger logger, Lifecycle lifecycle, BiConsumer<HttpChannel, Exception> fallback) {
-        this.lifecycle = lifecycle;
-        this.logger = logger;
-        this.fallback = fallback;
-    }
-
-    public void accept(HttpChannel channel, Exception e) {
-        if (lifecycle.started() == false) {
-            return;
-        }
-
-        if (isNotSslRecordException(e)) {
-            logger.warn("received plaintext http traffic on an https channel, closing connection {}", channel);
-            CloseableChannel.closeChannel(channel);
-        } else if (isCloseDuringHandshakeException(e)) {
-            logger.debug("connection {} closed during ssl handshake", channel);
-            CloseableChannel.closeChannel(channel);
-        } else if (isInsufficientBufferRemainingException(e)) {
-            logger.debug("connection {} closed abruptly", channel);
-            CloseableChannel.closeChannel(channel);
-        } else if (isReceivedCertificateUnknownException(e)) {
-            logger.warn("http client did not trust this server's certificate, closing connection {}", channel);
-            CloseableChannel.closeChannel(channel);
-        } else {
-            fallback.accept(channel, e);
-        }
-    }
-}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java
index afb0030d7a27..4d2d31153661 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java
@@ -8,8 +8,12 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsAction;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchTransportService;
 import org.elasticsearch.action.support.DestructiveOperations;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.ssl.SslConfiguration;
@@ -19,8 +23,11 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.RemoteConnectionManager;
 import org.elasticsearch.transport.SendRequestTransportException;
+import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportActionProxy;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportInterceptor;
 import org.elasticsearch.transport.TransportRequest;
@@ -32,20 +39,61 @@
 import org.elasticsearch.transport.TransportService.ContextRestoreResponseHandler;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.RemoteAccessAuthentication;
+import org.elasticsearch.xpack.core.security.authc.Subject;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection;
 import org.elasticsearch.xpack.core.security.transport.ProfileConfigurations;
+import org.elasticsearch.xpack.core.security.user.SystemUser;
+import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.security.authc.AuthenticationService;
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
 import org.elasticsearch.xpack.security.authz.AuthorizationUtils;
+import org.elasticsearch.xpack.security.authz.PreAuthorizationUtils;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
-import static org.elasticsearch.xpack.core.security.SecurityField.setting;
+import static org.elasticsearch.transport.RemoteClusterPortSettings.REMOTE_CLUSTER_PORT_ENABLED;
+import static org.elasticsearch.transport.RemoteClusterPortSettings.REMOTE_CLUSTER_PROFILE;
 
 public class SecurityServerTransportInterceptor implements TransportInterceptor {
 
+    public static final String REMOTE_ACCESS_CLUSTER_CREDENTIAL_HEADER_KEY = "_remote_access_cluster_credential";
+    private static final TransportVersion VERSION_REMOTE_ACCESS_HEADERS = TransportVersion.V_8_7_0;
     private static final Logger logger = LogManager.getLogger(SecurityServerTransportInterceptor.class);
+    // package private for testing
+    static final Set<String> REMOTE_ACCESS_ACTION_ALLOWLIST;
+    static {
+        final Stream<String> actions = Stream.of(
+            SearchAction.NAME,
+            ClusterStateAction.NAME,
+            ClusterSearchShardsAction.NAME,
+            SearchTransportService.FREE_CONTEXT_SCROLL_ACTION_NAME,
+            SearchTransportService.FREE_CONTEXT_ACTION_NAME,
+            SearchTransportService.CLEAR_SCROLL_CONTEXTS_ACTION_NAME,
+            SearchTransportService.DFS_ACTION_NAME,
+            SearchTransportService.QUERY_ACTION_NAME,
+            SearchTransportService.QUERY_ID_ACTION_NAME,
+            SearchTransportService.QUERY_SCROLL_ACTION_NAME,
+            SearchTransportService.QUERY_FETCH_SCROLL_ACTION_NAME,
+            SearchTransportService.FETCH_ID_SCROLL_ACTION_NAME,
+            SearchTransportService.FETCH_ID_ACTION_NAME,
+            SearchTransportService.QUERY_CAN_MATCH_NAME,
+            SearchTransportService.QUERY_CAN_MATCH_NODE_NAME
+        );
+        REMOTE_ACCESS_ACTION_ALLOWLIST = actions
+            // Include action, and proxy equivalent (i.e., with proxy action prefix)
+            .flatMap(name -> Stream.of(name, TransportActionProxy.getProxyAction(name)))
+            .collect(Collectors.toUnmodifiableSet());
+    }
 
     private final AuthenticationService authcService;
     private final AuthorizationService authzService;
@@ -55,6 +103,7 @@ public class SecurityServerTransportInterceptor implements TransportInterceptor
     private final Settings settings;
     private final SecurityContext securityContext;
     private final RemoteClusterAuthorizationResolver remoteClusterAuthorizationResolver;
+    private final Function<Transport.Connection, Optional<String>> remoteClusterAliasResolver;
 
     public SecurityServerTransportInterceptor(
         Settings settings,
@@ -65,6 +114,31 @@ public SecurityServerTransportInterceptor(
         SecurityContext securityContext,
         DestructiveOperations destructiveOperations,
         RemoteClusterAuthorizationResolver remoteClusterAuthorizationResolver
+    ) {
+        this(
+            settings,
+            threadPool,
+            authcService,
+            authzService,
+            sslService,
+            securityContext,
+            destructiveOperations,
+            remoteClusterAuthorizationResolver,
+            RemoteConnectionManager::resolveRemoteClusterAlias
+        );
+    }
+
+    SecurityServerTransportInterceptor(
+        Settings settings,
+        ThreadPool threadPool,
+        AuthenticationService authcService,
+        AuthorizationService authzService,
+        SSLService sslService,
+        SecurityContext securityContext,
+        DestructiveOperations destructiveOperations,
+        RemoteClusterAuthorizationResolver remoteClusterAuthorizationResolver,
+        // Inject for simplified testing
+        Function<Transport.Connection, Optional<String>> remoteClusterAliasResolver
     ) {
         this.settings = settings;
         this.threadPool = threadPool;
@@ -74,10 +148,19 @@ public SecurityServerTransportInterceptor(
         this.securityContext = securityContext;
         this.profileFilters = initializeProfileFilters(destructiveOperations);
         this.remoteClusterAuthorizationResolver = remoteClusterAuthorizationResolver;
+        this.remoteClusterAliasResolver = remoteClusterAliasResolver;
     }
 
     @Override
     public AsyncSender interceptSender(AsyncSender sender) {
+        return interceptForAllRequests(
+            // Branching based on the feature flag is not strictly necessary here, but it makes it more obvious we are not interfering with
+            // non-feature-flagged deployments
+            TcpTransport.isUntrustedRemoteClusterEnabled() ? interceptForRemoteAccessRequests(sender) : sender
+        );
+    }
+
+    private AsyncSender interceptForAllRequests(AsyncSender sender) {
         return new AsyncSender() {
             @Override
             public <T extends TransportResponse> void sendRequest(
@@ -87,55 +170,231 @@ public <T extends TransportResponse> void sendRequest(
                 TransportRequestOptions options,
                 TransportResponseHandler<T> handler
             ) {
-                // the transport in core normally does this check, BUT since we are serializing to a string header we need to do it
-                // ourselves otherwise we wind up using a version newer than what we can actually send
-                final Version minVersion = Version.min(connection.getVersion(), Version.CURRENT);
-
-                // Sometimes a system action gets executed like a internal create index request or update mappings request
-                // which means that the user is copied over to system actions so we need to change the user
-                if (AuthorizationUtils.shouldReplaceUserWithSystem(threadPool.getThreadContext(), action)) {
-                    securityContext.executeAsSystemUser(
-                        minVersion,
-                        original -> sendWithUser(
+                assertNoRemoteAccessHeadersInContext();
+                final Optional<String> remoteClusterAlias = remoteClusterAliasResolver.apply(connection);
+                if (PreAuthorizationUtils.shouldRemoveParentAuthorizationFromThreadContext(remoteClusterAlias, action, securityContext)) {
+                    securityContext.executeAfterRemovingParentAuthorization(original -> {
+                        sendRequestInner(
+                            sender,
                             connection,
                             action,
                             request,
                             options,
-                            new ContextRestoreResponseHandler<>(threadPool.getThreadContext().wrapRestorable(original), handler),
-                            sender
-                        )
+                            new ContextRestoreResponseHandler<>(threadPool.getThreadContext().wrapRestorable(original), handler)
+                        );
+                    });
+                } else {
+                    sendRequestInner(sender, connection, action, request, options, handler);
+                }
+            }
+
+            private void assertNoRemoteAccessHeadersInContext() {
+                assert securityContext.getThreadContext().getHeader(REMOTE_ACCESS_CLUSTER_CREDENTIAL_HEADER_KEY) == null
+                    : "remote access headers should not be in security context";
+                assert securityContext.getThreadContext()
+                    .getHeader(RemoteAccessAuthentication.REMOTE_ACCESS_AUTHENTICATION_HEADER_KEY) == null
+                    : "remote access headers should not be in security context";
+            }
+        };
+    }
+
+    public <T extends TransportResponse> void sendRequestInner(
+        AsyncSender sender,
+        Transport.Connection connection,
+        String action,
+        TransportRequest request,
+        TransportRequestOptions options,
+        TransportResponseHandler<T> handler
+    ) {
+        // the transport in core normally does this check, BUT since we are serializing to a string header we need to do it
+        // ourselves otherwise we wind up using a version newer than what we can actually send
+        final TransportVersion minVersion = TransportVersion.min(connection.getTransportVersion(), TransportVersion.CURRENT);
+
+        // Sometimes a system action gets executed like a internal create index request or update mappings request
+        // which means that the user is copied over to system actions so we need to change the user
+        if (AuthorizationUtils.shouldReplaceUserWithSystem(threadPool.getThreadContext(), action)) {
+            securityContext.executeAsSystemUser(
+                minVersion,
+                original -> sendWithUser(
+                    connection,
+                    action,
+                    request,
+                    options,
+                    new ContextRestoreResponseHandler<>(threadPool.getThreadContext().wrapRestorable(original), handler),
+                    sender
+                )
+            );
+        } else if (AuthorizationUtils.shouldSetUserBasedOnActionOrigin(threadPool.getThreadContext())) {
+            AuthorizationUtils.switchUserBasedOnActionOriginAndExecute(
+                threadPool.getThreadContext(),
+                securityContext,
+                minVersion,
+                (original) -> sendWithUser(
+                    connection,
+                    action,
+                    request,
+                    options,
+                    new ContextRestoreResponseHandler<>(threadPool.getThreadContext().wrapRestorable(original), handler),
+                    sender
+                )
+            );
+        } else if (securityContext.getAuthentication() != null
+            && securityContext.getAuthentication().getEffectiveSubject().getTransportVersion().equals(minVersion) == false) {
+                // re-write the authentication since we want the authentication version to match the version of the connection
+                securityContext.executeAfterRewritingAuthentication(
+                    original -> sendWithUser(
+                        connection,
+                        action,
+                        request,
+                        options,
+                        new ContextRestoreResponseHandler<>(threadPool.getThreadContext().wrapRestorable(original), handler),
+                        sender
+                    ),
+                    minVersion
+                );
+            } else {
+                sendWithUser(connection, action, request, options, handler, sender);
+            }
+    }
+
+    // Package private for testing
+    Map<String, ServerTransportFilter> getProfileFilters() {
+        return profileFilters;
+    }
+
+    private AsyncSender interceptForRemoteAccessRequests(final AsyncSender sender) {
+        return new AsyncSender() {
+            @Override
+            public <T extends TransportResponse> void sendRequest(
+                Transport.Connection connection,
+                String action,
+                TransportRequest request,
+                TransportRequestOptions options,
+                TransportResponseHandler<T> handler
+            ) {
+                final Optional<RemoteAccessCredentials> remoteAccessCredentials = getRemoteAccessCredentials(connection, action);
+                if (remoteAccessCredentials.isPresent()) {
+                    sendWithRemoteAccessHeaders(remoteAccessCredentials.get(), connection, action, request, options, handler);
+                } else {
+                    // Send regular request, without remote access headers
+                    try {
+                        sender.sendRequest(connection, action, request, options, handler);
+                    } catch (Exception e) {
+                        handler.handleException(new SendRequestTransportException(connection.getNode(), action, e));
+                    }
+                }
+            }
+
+            /**
+             * Returns cluster credentials if the connection is remote, cluster credentials are set up for the target cluster, and access
+             * via remote access headers is supported for the request type and authenticated subject. If remote access is not supported,
+             * this method does not return credentials even if they are configured, to signify that the request should be sent according to
+             * the basic security model
+             */
+            private Optional<RemoteAccessCredentials> getRemoteAccessCredentials(Transport.Connection connection, String action) {
+                final Optional<String> optionalRemoteClusterAlias = remoteClusterAliasResolver.apply(connection);
+                if (optionalRemoteClusterAlias.isEmpty()) {
+                    logger.trace("Connection is not remote");
+                    return Optional.empty();
+                }
+
+                final String remoteClusterAlias = optionalRemoteClusterAlias.get();
+                final String remoteClusterCredential = remoteClusterAuthorizationResolver.resolveAuthorization(remoteClusterAlias);
+                if (remoteClusterCredential == null) {
+                    logger.trace("No cluster credential is configured for remote cluster [{}]", remoteClusterAlias);
+                    return Optional.empty();
+                }
+
+                if (false == REMOTE_ACCESS_ACTION_ALLOWLIST.contains(action)) {
+                    logger.trace("Action [{}] towards remote cluster [{}] is not allow-listed", action, remoteClusterAlias);
+                    return Optional.empty();
+                }
+
+                final Authentication authentication = securityContext.getAuthentication();
+                assert authentication != null : "authentication must be present in security context";
+                final Subject effectiveSubject = authentication.getEffectiveSubject();
+                if (false == effectiveSubject.getType().equals(Subject.Type.USER)) {
+                    logger.trace(
+                        "Effective subject of request to remote cluster [{}] has an unsupported type [{}]",
+                        remoteClusterAlias,
+                        effectiveSubject.getType()
                     );
-                } else if (AuthorizationUtils.shouldSetUserBasedOnActionOrigin(threadPool.getThreadContext())) {
-                    AuthorizationUtils.switchUserBasedOnActionOriginAndExecute(
-                        threadPool.getThreadContext(),
-                        securityContext,
-                        minVersion,
-                        (original) -> sendWithUser(
-                            connection,
-                            action,
-                            request,
-                            options,
-                            new ContextRestoreResponseHandler<>(threadPool.getThreadContext().wrapRestorable(original), handler),
-                            sender
-                        )
+                    return Optional.empty();
+                }
+
+                return Optional.of(new RemoteAccessCredentials(remoteClusterAlias, remoteClusterCredential));
+            }
+
+            private <T extends TransportResponse> void sendWithRemoteAccessHeaders(
+                final RemoteAccessCredentials remoteAccessCredentials,
+                final Transport.Connection connection,
+                final String action,
+                final TransportRequest request,
+                final TransportRequestOptions options,
+                final TransportResponseHandler<T> handler
+            ) {
+                final String remoteClusterAlias = remoteAccessCredentials.clusterAlias();
+                if (connection.getTransportVersion().before(VERSION_REMOTE_ACCESS_HEADERS)) {
+                    throw new IllegalArgumentException(
+                        "Settings for remote cluster ["
+                            + remoteClusterAlias
+                            + "] indicate remote access headers should be sent but target cluster version ["
+                            + connection.getTransportVersion()
+                            + "] does not support receiving them"
                     );
-                } else if (securityContext.getAuthentication() != null
-                    && securityContext.getAuthentication().getEffectiveSubject().getVersion().equals(minVersion) == false) {
-                        // re-write the authentication since we want the authentication version to match the version of the connection
-                        securityContext.executeAfterRewritingAuthentication(
-                            original -> sendWithUser(
-                                connection,
-                                action,
-                                request,
-                                options,
-                                new ContextRestoreResponseHandler<>(threadPool.getThreadContext().wrapRestorable(original), handler),
-                                sender
-                            ),
-                            minVersion
-                        );
-                    } else {
-                        sendWithUser(connection, action, request, options, handler, sender);
+                }
+
+                logger.debug(
+                    "Sending [{}] request to [{}] with remote access headers for [{}] action",
+                    request.getClass(),
+                    remoteClusterAlias,
+                    action
+                );
+
+                final Authentication authentication = securityContext.getAuthentication();
+                assert authentication != null : "authentication must be present in security context";
+
+                final ThreadContext threadContext = securityContext.getThreadContext();
+                final var contextRestoreHandler = new ContextRestoreResponseHandler<>(threadContext.newRestorableContext(true), handler);
+                final User user = authentication.getEffectiveSubject().getUser();
+                if (SystemUser.is(user)) {
+                    try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
+                        remoteAccessCredentials.writeToContext(threadContext);
+                        // Access control is handled differently for the system user. Privileges are defined by the fulfilling cluster,
+                        // so we pass an empty role descriptors intersection here and let the receiver resolve privileges based on the
+                        // authentication instance
+                        new RemoteAccessAuthentication(authentication, RoleDescriptorsIntersection.EMPTY).writeToContext(threadContext);
+                        sender.sendRequest(connection, action, request, options, contextRestoreHandler);
+                    } catch (IOException e) {
+                        contextRestoreHandler.handleException(new SendRequestTransportException(connection.getNode(), action, e));
                     }
+                } else if (User.isInternal(user)) {
+                    final String message = "internal user [" + user.principal() + "] should not be used for cross cluster requests";
+                    assert false : message;
+                    throw new IllegalArgumentException(message);
+                } else {
+                    authzService.retrieveRemoteAccessRoleDescriptorsIntersection(
+                        remoteClusterAlias,
+                        authentication.getEffectiveSubject(),
+                        ActionListener.wrap(roleDescriptorsIntersection -> {
+                            try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+                                remoteAccessCredentials.writeToContext(threadContext);
+                                new RemoteAccessAuthentication(authentication, roleDescriptorsIntersection).writeToContext(threadContext);
+                                sender.sendRequest(connection, action, request, options, contextRestoreHandler);
+                            }
+                        }, e -> contextRestoreHandler.handleException(new SendRequestTransportException(connection.getNode(), action, e)))
+                    );
+                }
+            }
+
+            record RemoteAccessCredentials(String clusterAlias, String credentials) {
+                void writeToContext(final ThreadContext ctx) {
+                    ctx.putHeader(REMOTE_ACCESS_CLUSTER_CREDENTIAL_HEADER_KEY, withApiKeyPrefix(credentials));
+                }
+
+                private String withApiKeyPrefix(final String clusterCredential) {
+                    return "ApiKey " + clusterCredential;
+                }
             }
         };
     }
@@ -155,6 +414,9 @@ private <T extends TransportResponse> void sendWithUser(
             throw new IllegalStateException("there should always be a user when sending a message for action [" + action + "]");
         }
 
+        assert securityContext.getParentAuthorization() == null || remoteClusterAliasResolver.apply(connection).isPresent() == false
+            : "parent authorization header should not be set for remote cluster requests";
+
         try {
             sender.sendRequest(connection, action, request, options, handler);
         } catch (Exception e) {
@@ -178,17 +440,25 @@ public <T extends TransportRequest> TransportRequestHandler<T> interceptHandler(
     }
 
     private Map<String, ServerTransportFilter> initializeProfileFilters(DestructiveOperations destructiveOperations) {
-        final SslConfiguration sslConfiguration = sslService.getSSLConfiguration(setting("transport.ssl"));
-        final Map<String, SslConfiguration> profileConfigurations = ProfileConfigurations.get(settings, sslService, sslConfiguration);
+        final Map<String, SslConfiguration> profileConfigurations = ProfileConfigurations.get(settings, sslService, false);
 
         Map<String, ServerTransportFilter> profileFilters = Maps.newMapWithExpectedSize(profileConfigurations.size() + 1);
 
         final boolean transportSSLEnabled = XPackSettings.TRANSPORT_SSL_ENABLED.get(settings);
+        final boolean remoteClusterPortEnabled = REMOTE_CLUSTER_PORT_ENABLED.get(settings);
+        final boolean remoteClusterSSLEnabled = remoteClusterPortEnabled && XPackSettings.REMOTE_CLUSTER_SSL_ENABLED.get(settings);
+
         for (Map.Entry<String, SslConfiguration> entry : profileConfigurations.entrySet()) {
             final SslConfiguration profileConfiguration = entry.getValue();
-            final boolean extractClientCert = transportSSLEnabled && SSLService.isSSLClientAuthEnabled(profileConfiguration);
+            final String profileName = entry.getKey();
+            final boolean extractClientCert;
+            if (remoteClusterPortEnabled && REMOTE_CLUSTER_PROFILE.equals(profileName)) {
+                extractClientCert = remoteClusterSSLEnabled && SSLService.isSSLClientAuthEnabled(profileConfiguration);
+            } else {
+                extractClientCert = transportSSLEnabled && SSLService.isSSLClientAuthEnabled(profileConfiguration);
+            }
             profileFilters.put(
-                entry.getKey(),
+                profileName,
                 new ServerTransportFilter(
                     authcService,
                     authzService,
@@ -307,6 +577,7 @@ public void onResponse(Void unused) {
                 } else {
                     final Thread executingThread = Thread.currentThread();
                     filterListener = new AbstractFilterListener(receiveMessage) {
+
                         @Override
                         public void onResponse(Void unused) {
                             if (executingThread == Thread.currentThread()) {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java
index 68b68a876dc2..54cb20b6c0e0 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java
@@ -8,7 +8,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.admin.indices.close.CloseIndexAction;
@@ -100,12 +100,12 @@ requests from all the nodes are attached with a user (either a serialize
             }
         }
 
-        final Version version = transportChannel.getVersion();
+        TransportVersion version = transportChannel.getVersion();
         authcService.authenticate(securityAction, request, true, ActionListener.wrap((authentication) -> {
             if (authentication != null) {
                 if (securityAction.equals(TransportService.HANDSHAKE_ACTION_NAME)
                     && SystemUser.is(authentication.getEffectiveSubject().getUser()) == false) {
-                    securityContext.executeAsSystemUser(version, original -> {
+                    securityContext.executeAsSystemUser(TransportVersion.fromId(version.id), original -> {
                         final Authentication replaced = securityContext.getAuthentication();
                         authzService.authorize(replaced, securityAction, request, listener);
                     });
@@ -117,4 +117,9 @@ requests from all the nodes are attached with a user (either a serialize
             }
         }, listener::onFailure));
     }
+
+    // Package private for testing
+    boolean isExtractClientCert() {
+        return extractClientCert;
+    }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/filter/IPFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/filter/IPFilter.java
index 28087e64bcc8..7d834f9ce038 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/filter/IPFilter.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/filter/IPFilter.java
@@ -18,7 +18,9 @@
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.TransportSettings;
 import org.elasticsearch.xpack.security.Security;
 import org.elasticsearch.xpack.security.audit.AuditTrail;
@@ -28,6 +30,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -35,6 +38,9 @@
 import java.util.stream.Collectors;
 
 import static java.util.Collections.unmodifiableMap;
+import static org.elasticsearch.transport.RemoteClusterPortSettings.REMOTE_CLUSTER_PORT_ENABLED;
+import static org.elasticsearch.transport.RemoteClusterPortSettings.REMOTE_CLUSTER_PREFIX;
+import static org.elasticsearch.transport.RemoteClusterPortSettings.REMOTE_CLUSTER_PROFILE;
 import static org.elasticsearch.xpack.core.security.SecurityField.setting;
 
 public class IPFilter {
@@ -45,7 +51,7 @@ public class IPFilter {
      * for HTTP. This name starts withs a dot, because no profile name can ever start like that due to
      * how we handle settings
      */
-    public static final String HTTP_PROFILE_NAME = ".http";
+    public static final String HTTP_PROFILE_NAME = HttpServerTransport.HTTP_PROFILE_NAME;
 
     public static final Setting<Boolean> ALLOW_BOUND_ADDRESSES_SETTING = Setting.boolSetting(
         setting("filter.always_allow_bound_address"),
@@ -87,6 +93,26 @@ public class IPFilter {
         Property.NodeScope
     );
 
+    public static final Setting<List<String>> REMOTE_CLUSTER_FILTER_ALLOW_SETTING = Setting.listSetting(
+        setting(REMOTE_CLUSTER_PREFIX + "filter.allow"),
+        TRANSPORT_FILTER_ALLOW_SETTING,
+        Function.identity(),
+        TRANSPORT_FILTER_ALLOW_SETTING::get,
+        ALLOW_VALIDATOR,
+        Property.OperatorDynamic,
+        Property.NodeScope
+    );
+
+    public static final Setting<List<String>> REMOTE_CLUSTER_FILTER_DENY_SETTING = Setting.listSetting(
+        setting(REMOTE_CLUSTER_PREFIX + "filter.deny"),
+        TRANSPORT_FILTER_DENY_SETTING,
+        Function.identity(),
+        TRANSPORT_FILTER_DENY_SETTING::get,
+        DENY_VALIDATOR,
+        Property.OperatorDynamic,
+        Property.NodeScope
+    );
+
     public static final Setting.AffixSetting<List<String>> PROFILE_FILTER_DENY_SETTING = Setting.affixKeySetting(
         "transport.profiles.",
         "xpack.security.filter.deny",
@@ -199,21 +225,38 @@ public IPFilter(
         isHttpFilterEnabled = IP_FILTER_ENABLED_HTTP_SETTING.get(settings);
         isIpFilterEnabled = IP_FILTER_ENABLED_SETTING.get(settings);
 
-        this.profiles = settings.getGroups("transport.profiles.", true)
+        Set<String> profiles = settings.getGroups("transport.profiles.", true)
             .keySet()
             .stream()
-            .filter(k -> TransportSettings.DEFAULT_PROFILE.equals(k) == false)
-            .collect(Collectors.toSet()); // exclude default profile -- it's handled differently
+            .filter(k -> TransportSettings.DEFAULT_PROFILE.equals(k) == false) // exclude default profile -- it's handled differently
+            .collect(Collectors.toCollection(HashSet::new));
+        assert false == profiles.contains(REMOTE_CLUSTER_PROFILE);
         for (String profile : profiles) {
             Setting<List<String>> allowSetting = PROFILE_FILTER_ALLOW_SETTING.getConcreteSettingForNamespace(profile);
             profileAllowRules.put(profile, allowSetting.get(settings));
             Setting<List<String>> denySetting = PROFILE_FILTER_DENY_SETTING.getConcreteSettingForNamespace(profile);
             profileDenyRules.put(profile, denySetting.get(settings));
         }
+        if (REMOTE_CLUSTER_PORT_ENABLED.get(settings)) {
+            logger.debug(
+                "Remote access is enabled, populating filters for profile [{}] with contents of [{}] and [{}]",
+                REMOTE_CLUSTER_PROFILE,
+                REMOTE_CLUSTER_FILTER_ALLOW_SETTING.getKey(),
+                REMOTE_CLUSTER_FILTER_DENY_SETTING.getKey()
+            );
+            profiles.add(REMOTE_CLUSTER_PROFILE);
+            profileAllowRules.put(REMOTE_CLUSTER_PROFILE, REMOTE_CLUSTER_FILTER_ALLOW_SETTING.get(settings));
+            profileDenyRules.put(REMOTE_CLUSTER_PROFILE, REMOTE_CLUSTER_FILTER_DENY_SETTING.get(settings));
+        }
+        this.profiles = Collections.unmodifiableSet(profiles);
         clusterSettings.addSettingsUpdateConsumer(IP_FILTER_ENABLED_HTTP_SETTING, this::setHttpFiltering);
         clusterSettings.addSettingsUpdateConsumer(IP_FILTER_ENABLED_SETTING, this::setTransportFiltering);
         clusterSettings.addSettingsUpdateConsumer(TRANSPORT_FILTER_ALLOW_SETTING, this::setTransportAllowFilter);
         clusterSettings.addSettingsUpdateConsumer(TRANSPORT_FILTER_DENY_SETTING, this::setTransportDenyFilter);
+        if (TcpTransport.isUntrustedRemoteClusterEnabled()) {
+            clusterSettings.addSettingsUpdateConsumer(REMOTE_CLUSTER_FILTER_ALLOW_SETTING, this::setRemoteAccessAllowFilter);
+            clusterSettings.addSettingsUpdateConsumer(REMOTE_CLUSTER_FILTER_DENY_SETTING, this::setRemoteAccessDenyFilter);
+        }
         clusterSettings.addSettingsUpdateConsumer(HTTP_FILTER_ALLOW_SETTING, this::setHttpAllowFilter);
         clusterSettings.addSettingsUpdateConsumer(HTTP_FILTER_DENY_SETTING, this::setHttpDenyFilter);
         clusterSettings.addAffixUpdateConsumer(PROFILE_FILTER_ALLOW_SETTING, this::setProfileAllowRules, (a, b) -> {});
@@ -241,6 +284,16 @@ private void setProfileDenyRules(String profile, List<String> rules) {
         updateRules();
     }
 
+    private void setRemoteAccessAllowFilter(List<String> filter) {
+        profileAllowRules.put(REMOTE_CLUSTER_PROFILE, filter);
+        updateRules();
+    }
+
+    private void setRemoteAccessDenyFilter(List<String> filter) {
+        profileDenyRules.put(REMOTE_CLUSTER_PROFILE, filter);
+        updateRules();
+    }
+
     private void setHttpDenyFilter(List<String> filter) {
         this.httpDenyFilter = filter;
         updateRules();
@@ -375,6 +428,10 @@ public static void addSettings(List<Setting<?>> settings) {
         settings.add(HTTP_FILTER_DENY_SETTING);
         settings.add(TRANSPORT_FILTER_ALLOW_SETTING);
         settings.add(TRANSPORT_FILTER_DENY_SETTING);
+        if (TcpTransport.isUntrustedRemoteClusterEnabled()) {
+            settings.add(REMOTE_CLUSTER_FILTER_ALLOW_SETTING);
+            settings.add(REMOTE_CLUSTER_FILTER_DENY_SETTING);
+        }
         settings.add(PROFILE_FILTER_ALLOW_SETTING);
         settings.add(PROFILE_FILTER_DENY_SETTING);
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransport.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransport.java
deleted file mode 100644
index 3bad4773a4a6..000000000000
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransport.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.security.transport.netty4;
-
-import io.netty.channel.Channel;
-import io.netty.channel.ChannelHandler;
-import io.netty.handler.ssl.SslHandler;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.network.NetworkService;
-import org.elasticsearch.common.settings.ClusterSettings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.ssl.SslConfiguration;
-import org.elasticsearch.common.util.PageCacheRecycler;
-import org.elasticsearch.http.HttpChannel;
-import org.elasticsearch.http.netty4.Netty4HttpServerTransport;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.tracing.Tracer;
-import org.elasticsearch.transport.netty4.SharedGroupFactory;
-import org.elasticsearch.xcontent.NamedXContentRegistry;
-import org.elasticsearch.xpack.core.ssl.SSLService;
-import org.elasticsearch.xpack.security.transport.SecurityHttpExceptionHandler;
-import org.elasticsearch.xpack.security.transport.filter.IPFilter;
-
-import javax.net.ssl.SSLEngine;
-
-import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED;
-
-public class SecurityNetty4HttpServerTransport extends Netty4HttpServerTransport {
-    private static final Logger logger = LogManager.getLogger(SecurityNetty4HttpServerTransport.class);
-
-    private final SecurityHttpExceptionHandler securityExceptionHandler;
-    private final IPFilter ipFilter;
-    private final SSLService sslService;
-    private final SslConfiguration sslConfiguration;
-
-    public SecurityNetty4HttpServerTransport(
-        Settings settings,
-        NetworkService networkService,
-        PageCacheRecycler recycler,
-        IPFilter ipFilter,
-        SSLService sslService,
-        ThreadPool threadPool,
-        NamedXContentRegistry xContentRegistry,
-        Dispatcher dispatcher,
-        ClusterSettings clusterSettings,
-        SharedGroupFactory sharedGroupFactory,
-        Tracer tracer
-    ) {
-        super(settings, networkService, threadPool, xContentRegistry, dispatcher, clusterSettings, sharedGroupFactory, tracer);
-        this.securityExceptionHandler = new SecurityHttpExceptionHandler(logger, lifecycle, (c, e) -> super.onException(c, e));
-        this.ipFilter = ipFilter;
-        final boolean ssl = HTTP_SSL_ENABLED.get(settings);
-        this.sslService = sslService;
-        if (ssl) {
-            this.sslConfiguration = sslService.getHttpTransportSSLConfiguration();
-            if (SSLService.isConfigurationValidForServerUsage(sslConfiguration) == false) {
-                throw new IllegalArgumentException(
-                    "a key must be provided to run as a server. the key should be configured using the "
-                        + "[xpack.security.http.ssl.key] or [xpack.security.http.ssl.keystore.path] setting"
-                );
-            }
-        } else {
-            this.sslConfiguration = null;
-        }
-    }
-
-    @Override
-    public void onException(HttpChannel channel, Exception e) {
-        securityExceptionHandler.accept(channel, e);
-    }
-
-    @Override
-    protected void doStart() {
-        super.doStart();
-        ipFilter.setBoundHttpTransportAddress(this.boundAddress());
-    }
-
-    @Override
-    public ChannelHandler configureServerChannelHandler() {
-        return new HttpSslChannelHandler();
-    }
-
-    private final class HttpSslChannelHandler extends HttpChannelHandler {
-        HttpSslChannelHandler() {
-            super(SecurityNetty4HttpServerTransport.this, handlingSettings);
-        }
-
-        @Override
-        protected void initChannel(Channel ch) throws Exception {
-            super.initChannel(ch);
-            if (sslConfiguration != null) {
-                SSLEngine sslEngine = sslService.createSSLEngine(sslConfiguration, null, -1);
-                sslEngine.setUseClientMode(false);
-                ch.pipeline().addFirst("ssl", new SslHandler(sslEngine));
-            }
-            ch.pipeline().addFirst("ip_filter", new IpFilterRemoteAddressFilter(ipFilter, IPFilter.HTTP_PROFILE_NAME));
-        }
-    }
-}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransport.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransport.java
index c93dd93444e1..ffd7437eab2e 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransport.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransport.java
@@ -9,7 +9,7 @@
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelHandler;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
@@ -30,7 +30,7 @@ public class SecurityNetty4ServerTransport extends SecurityNetty4Transport {
 
     public SecurityNetty4ServerTransport(
         final Settings settings,
-        final Version version,
+        final TransportVersion version,
         final ThreadPool threadPool,
         final NetworkService networkService,
         final PageCacheRecycler pageCacheRecycler,
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/core/security/transport/ProfileConfigurationsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/core/security/transport/ProfileConfigurationsTests.java
index 8f9789941f44..f0702ab5c364 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/core/security/transport/ProfileConfigurationsTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/core/security/transport/ProfileConfigurationsTests.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.ssl.SslClientAuthenticationMode;
 import org.elasticsearch.common.ssl.SslConfiguration;
 import org.elasticsearch.common.ssl.SslVerificationMode;
 import org.elasticsearch.env.Environment;
@@ -18,14 +19,19 @@
 import org.hamcrest.Matchers;
 
 import java.nio.file.Path;
+import java.util.Arrays;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.anEmptyMap;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.is;
+
 public class ProfileConfigurationsTests extends ESTestCase {
 
     public void testGetSecureTransportProfileConfigurations() {
         assumeFalse("Can't run in a FIPS JVM, uses JKS/PKCS12 keystores", inFipsJvm());
         final Settings settings = getBaseSettings().put("path.home", createTempDir())
-            .put("xpack.security.transport.ssl.verification_mode", SslVerificationMode.CERTIFICATE.name())
             .put("xpack.security.transport.ssl.verification_mode", SslVerificationMode.CERTIFICATE.name())
             .put("transport.profiles.full.xpack.security.ssl.verification_mode", SslVerificationMode.FULL.name())
             .put("transport.profiles.cert.xpack.security.ssl.verification_mode", SslVerificationMode.CERTIFICATE.name())
@@ -33,7 +39,7 @@ public void testGetSecureTransportProfileConfigurations() {
         final Environment env = TestEnvironment.newEnvironment(settings);
         SSLService sslService = new SSLService(env);
         final SslConfiguration defaultConfig = sslService.getSSLConfiguration("xpack.security.transport.ssl");
-        final Map<String, SslConfiguration> profileConfigurations = ProfileConfigurations.get(settings, sslService, defaultConfig);
+        final Map<String, SslConfiguration> profileConfigurations = ProfileConfigurations.get(settings, sslService, true);
         assertThat(profileConfigurations.size(), Matchers.equalTo(3));
         assertThat(profileConfigurations.keySet(), Matchers.containsInAnyOrder("full", "cert", "default"));
         assertThat(profileConfigurations.get("full").verificationMode(), Matchers.equalTo(SslVerificationMode.FULL));
@@ -50,25 +56,134 @@ public void testGetInsecureTransportProfileConfigurations() {
         final Environment env = TestEnvironment.newEnvironment(settings);
         SSLService sslService = new SSLService(env);
         final SslConfiguration defaultConfig = sslService.getSSLConfiguration("xpack.security.transport.ssl");
-        final Map<String, SslConfiguration> profileConfigurations = ProfileConfigurations.get(settings, sslService, defaultConfig);
+        final Map<String, SslConfiguration> profileConfigurations = ProfileConfigurations.get(settings, sslService, true);
         assertThat(profileConfigurations.size(), Matchers.equalTo(2));
         assertThat(profileConfigurations.keySet(), Matchers.containsInAnyOrder("none", "default"));
         assertThat(profileConfigurations.get("none").verificationMode(), Matchers.equalTo(SslVerificationMode.NONE));
         assertThat(profileConfigurations.get("default"), Matchers.sameInstance(defaultConfig));
     }
 
+    public void testTransportAndRemoteClusterSslCanBeEnabledIndependently() {
+        assumeFalse("Can't run in a FIPS JVM with JKS/PKCS12 keystore or verification mode None", inFipsJvm());
+        final boolean transportSslEnabled = randomBoolean();
+        final boolean remoteClusterSslEnabled = randomBoolean();
+        final Settings settings = Settings.builder()
+            .put("path.home", createTempDir())
+            .put("xpack.security.transport.ssl.enabled", transportSslEnabled)
+            .put("xpack.security.transport.ssl.keystore.path", getKeystorePath().toString())
+            .put("xpack.security.transport.ssl.verification_mode", SslVerificationMode.NONE.name())
+            .put("remote_cluster.enabled", true)
+            .put("xpack.security.remote_cluster.ssl.enabled", remoteClusterSslEnabled)
+            .put("xpack.security.remote_cluster.ssl.keystore.path", getKeystorePath().toString())
+            .put("xpack.security.remote_cluster.ssl.verification_mode", SslVerificationMode.CERTIFICATE.name())
+            .setSecureSettings(getKeystoreSecureSettings("transport", "remote_cluster"))
+            .build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        SSLService sslService = new SSLService(env);
+        final Map<String, SslConfiguration> profileConfigurations = ProfileConfigurations.get(settings, sslService, true);
+
+        if (transportSslEnabled && remoteClusterSslEnabled) {
+            assertThat(profileConfigurations.size(), Matchers.equalTo(2));
+            assertThat(profileConfigurations.keySet(), Matchers.containsInAnyOrder("default", "_remote_cluster"));
+            assertThat(profileConfigurations.get("_remote_cluster").verificationMode(), Matchers.equalTo(SslVerificationMode.CERTIFICATE));
+            assertThat(
+                profileConfigurations.get("default"),
+                Matchers.sameInstance(sslService.getSSLConfiguration("xpack.security.transport.ssl"))
+            );
+        } else if (transportSslEnabled) {
+            assertThat(profileConfigurations.size(), Matchers.equalTo(1));
+            assertThat(profileConfigurations.keySet(), contains("default"));
+            assertThat(
+                profileConfigurations.get("default"),
+                Matchers.sameInstance(sslService.getSSLConfiguration("xpack.security.transport.ssl"))
+            );
+        } else if (remoteClusterSslEnabled) {
+            assertThat(profileConfigurations.size(), Matchers.equalTo(1));
+            assertThat(profileConfigurations.keySet(), contains("_remote_cluster"));
+            assertThat(profileConfigurations.get("_remote_cluster").verificationMode(), Matchers.equalTo(SslVerificationMode.CERTIFICATE));
+        } else {
+            assertThat(profileConfigurations, anEmptyMap());
+        }
+    }
+
+    public void testNoProfileConfigurationForRemoteClusterIfFeatureIsDisabled() {
+        assumeFalse("Can't run in a FIPS JVM with JKS/PKCS12 keystore or verification mode None", inFipsJvm());
+        final Settings settings = Settings.builder()
+            .put("path.home", createTempDir())
+            .put("xpack.security.transport.ssl.enabled", true)
+            .put("xpack.security.transport.ssl.keystore.path", getKeystorePath().toString())
+            .put("xpack.security.transport.ssl.verification_mode", SslVerificationMode.NONE.name())
+            .put("remote_cluster.enabled", false)
+            .put("xpack.security.remote_cluster.ssl.enabled", true)
+            .put("xpack.security.remote_cluster.ssl.keystore.path", getKeystorePath().toString())
+            .put("xpack.security.remote_cluster.ssl.verification_mode", SslVerificationMode.CERTIFICATE.name())
+            .setSecureSettings(getKeystoreSecureSettings("transport", "remote_cluster"))
+            .build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        SSLService sslService = new SSLService(env);
+        final Map<String, SslConfiguration> profileConfigurations = ProfileConfigurations.get(settings, sslService, true);
+        assertThat(profileConfigurations.size(), Matchers.equalTo(1));
+        assertThat(profileConfigurations.keySet(), contains("default"));
+        assertThat(
+            profileConfigurations.get("default"),
+            Matchers.sameInstance(sslService.getSSLConfiguration("xpack.security.transport.ssl"))
+        );
+    }
+
+    public void testGetProfileConfigurationsIrrespectiveToSslEnabled() {
+        assumeFalse("Can't run in a FIPS JVM with JKS/PKCS12 keystore or verification mode None", inFipsJvm());
+        final boolean remoteClusterPortEnabled = randomBoolean();
+        final Settings settings = Settings.builder()
+            .put("path.home", createTempDir())
+            .put("xpack.security.transport.ssl.enabled", false)
+            .put("xpack.security.transport.ssl.keystore.path", getKeystorePath().toString())
+            .put("transport.profiles.client.xpack.security.ssl.client_authentication", SslClientAuthenticationMode.NONE)
+            .put("remote_cluster.enabled", remoteClusterPortEnabled)
+            .put("xpack.security.transport.ssl.verification_mode", SslVerificationMode.NONE.name())
+            .put("xpack.security.remote_cluster.ssl.enabled", false)
+            .put("xpack.security.remote_cluster.ssl.keystore.path", getKeystorePath().toString())
+            .put("xpack.security.remote_cluster.ssl.verification_mode", SslVerificationMode.CERTIFICATE.name())
+            .setSecureSettings(getKeystoreSecureSettings("transport", "remote_cluster"))
+            .build();
+        final Environment env = TestEnvironment.newEnvironment(settings);
+        SSLService sslService = new SSLService(env);
+        final Map<String, SslConfiguration> profileConfigurations = ProfileConfigurations.get(settings, sslService, false);
+        if (remoteClusterPortEnabled) {
+            assertThat(profileConfigurations.size(), Matchers.equalTo(3));
+            assertThat(profileConfigurations.keySet(), containsInAnyOrder("default", "client", "_remote_cluster"));
+            assertThat(profileConfigurations.get("_remote_cluster").verificationMode(), is(SslVerificationMode.CERTIFICATE));
+        } else {
+            assertThat(profileConfigurations.size(), Matchers.equalTo(2));
+            assertThat(profileConfigurations.keySet(), containsInAnyOrder("default", "client"));
+        }
+        assertThat(
+            profileConfigurations.get("default"),
+            Matchers.sameInstance(sslService.getSSLConfiguration("xpack.security.transport.ssl"))
+        );
+        assertThat(profileConfigurations.get("client").clientAuth(), is(SslClientAuthenticationMode.NONE));
+    }
+
     private Settings.Builder getBaseSettings() {
+        return Settings.builder()
+            .setSecureSettings(getKeystoreSecureSettings("transport"))
+            .put("xpack.security.transport.ssl.enabled", true)
+            .put("xpack.security.transport.ssl.keystore.path", getKeystorePath().toString());
+    }
+
+    private Path getKeystorePath() {
         final Path keystore = randomBoolean()
             ? getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks")
             : getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.p12");
+        return keystore;
+    }
 
-        MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("xpack.security.transport.ssl.keystore.secure_password", "testnode");
-
-        return Settings.builder()
-            .setSecureSettings(secureSettings)
-            .put("xpack.security.transport.ssl.enabled", true)
-            .put("xpack.security.transport.ssl.keystore.path", keystore.toString());
+    private static MockSecureSettings getKeystoreSecureSettings(String... sslContexts) {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        Arrays.stream(sslContexts)
+            .forEach(
+                sslContext -> { secureSettings.setString("xpack.security." + sslContext + ".ssl.keystore.secure_password", "testnode"); }
+            );
+        return secureSettings;
     }
 
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/LocalReservedSecurityStateHandlerProvider.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/LocalReservedSecurityStateHandlerProvider.java
index e63f5993a041..619a3b73017b 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/LocalReservedSecurityStateHandlerProvider.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/LocalReservedSecurityStateHandlerProvider.java
@@ -20,7 +20,7 @@
  * for {@link org.elasticsearch.test.ESIntegTestCase} because the Security Plugin is really LocalStateSecurity in those tests.
  */
 public class LocalReservedSecurityStateHandlerProvider implements ReservedClusterStateHandlerProvider {
-    private final LocalStateSecurity plugin;
+    protected final LocalStateSecurity plugin;
 
     public LocalReservedSecurityStateHandlerProvider() {
         throw new IllegalStateException("Provider must be constructed using PluginsService");
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/LocalReservedUnstableSecurityStateHandlerProvider.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/LocalReservedUnstableSecurityStateHandlerProvider.java
new file mode 100644
index 000000000000..b4a07093e49c
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/LocalReservedUnstableSecurityStateHandlerProvider.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.reservedstate.ReservedClusterStateHandlerProvider;
+
+/**
+ * Mock Security Provider implementation for the {@link ReservedClusterStateHandlerProvider} service interface. This is used
+ * for {@link org.elasticsearch.test.ESIntegTestCase} because the Security Plugin is really LocalStateSecurity in those tests.
+ * <p>
+ * Unlike {@link LocalReservedSecurityStateHandlerProvider} this implementation is mocked to implement the
+ * {@link UnstableLocalStateSecurity}. Separate implementation is needed, because the SPI creation code matches the constructor
+ * signature when instantiating. E.g. we need to match {@link UnstableLocalStateSecurity} instead of {@link LocalStateSecurity}
+ */
+public class LocalReservedUnstableSecurityStateHandlerProvider extends LocalReservedSecurityStateHandlerProvider {
+    public LocalReservedUnstableSecurityStateHandlerProvider() {
+        throw new IllegalStateException("Provider must be constructed using PluginsService");
+    }
+
+    public LocalReservedUnstableSecurityStateHandlerProvider(UnstableLocalStateSecurity plugin) {
+        super(plugin);
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityContextTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityContextTests.java
index 9109135e469d..145532d31637 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityContextTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityContextTests.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.security;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
@@ -15,7 +15,7 @@
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
@@ -23,12 +23,17 @@
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationField;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.ParentActionAuthorization;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
+import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.security.user.AsyncSearchUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.core.security.user.XPackSecurityUser;
 import org.elasticsearch.xpack.core.security.user.XPackUser;
 import org.junit.Before;
+import org.mockito.Mockito;
 
 import java.io.EOFException;
 import java.io.IOException;
@@ -41,6 +46,8 @@
 import static org.elasticsearch.xpack.core.security.authc.Authentication.VERSION_API_KEY_ROLES_AS_BYTES;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 
 public class SecurityContextTests extends ESTestCase {
 
@@ -84,13 +91,13 @@ public void testSetInternalUser() {
         final User internalUser = randomFrom(SystemUser.INSTANCE, XPackUser.INSTANCE, XPackSecurityUser.INSTANCE, AsyncSearchUser.INSTANCE);
         assertNull(securityContext.getAuthentication());
         assertNull(securityContext.getUser());
-        securityContext.setInternalUser(internalUser, Version.CURRENT);
+        securityContext.setInternalUser(internalUser, TransportVersion.CURRENT);
         assertEquals(internalUser, securityContext.getUser());
         assertEquals(AuthenticationType.INTERNAL, securityContext.getAuthentication().getAuthenticationType());
 
         IllegalStateException e = expectThrows(
             IllegalStateException.class,
-            () -> securityContext.setInternalUser(randomFrom(internalUser, SystemUser.INSTANCE), Version.CURRENT)
+            () -> securityContext.setInternalUser(randomFrom(internalUser, SystemUser.INSTANCE), TransportVersion.CURRENT)
         );
         assertEquals("authentication ([_xpack_security_authentication]) is already present in the context", e.getMessage());
     }
@@ -116,7 +123,7 @@ public void testExecuteAsUser() throws IOException {
             AsyncSearchUser.INSTANCE
         );
         final AtomicReference<StoredContext> contextAtomicReference = new AtomicReference<>();
-        securityContext.executeAsInternalUser(executionUser, Version.CURRENT, (originalCtx) -> {
+        securityContext.executeAsInternalUser(executionUser, TransportVersion.CURRENT, (originalCtx) -> {
             assertEquals(executionUser, securityContext.getUser());
             assertEquals(AuthenticationType.INTERNAL, securityContext.getAuthentication().getAuthenticationType());
             contextAtomicReference.set(originalCtx);
@@ -162,12 +169,12 @@ public void testExecuteAfterRewritingAuthentication() throws IOException {
             assertEquals(original.getAuthenticatingSubject().getRealm(), authentication.getAuthenticatingSubject().getRealm());
             assertEquals(original.isRunAs(), authentication.isRunAs());
             assertEquals(original.getEffectiveSubject().getRealm(), authentication.getEffectiveSubject().getRealm());
-            assertEquals(VersionUtils.getPreviousVersion(), authentication.getEffectiveSubject().getVersion());
+            assertEquals(TransportVersionUtils.getPreviousVersion(), authentication.getEffectiveSubject().getTransportVersion());
             assertEquals(original.getAuthenticationType(), securityContext.getAuthentication().getAuthenticationType());
             contextAtomicReference.set(originalCtx);
             // Other request headers should be preserved
             requestHeaders.forEach((k, v) -> assertThat(threadContext.getHeader(k), equalTo(v)));
-        }, VersionUtils.getPreviousVersion());
+        }, TransportVersionUtils.getPreviousVersion());
 
         final Authentication authAfterExecution = securityContext.getAuthentication();
         assertEquals(original, authAfterExecution);
@@ -187,7 +194,11 @@ public void testExecuteAfterRewritingAuthenticationWillConditionallyRewriteNewAp
             new BytesArray("{\"limitedBy role\": {\"cluster\": [\"all\"]}}")
         );
 
-        final Authentication original = AuthenticationTestHelper.builder().apiKey().metadata(metadata).version(Version.V_8_0_0).build();
+        final Authentication original = AuthenticationTestHelper.builder()
+            .apiKey()
+            .metadata(metadata)
+            .transportVersion(TransportVersion.V_8_0_0)
+            .build();
         original.writeToContext(threadContext);
 
         // If target is old node, rewrite new style API key metadata to old format
@@ -201,17 +212,17 @@ public void testExecuteAfterRewritingAuthenticationWillConditionallyRewriteNewAp
                 Map.of("limitedBy role", Map.of("cluster", List.of("all"))),
                 authentication.getAuthenticatingSubject().getMetadata().get(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY)
             );
-        }, Version.V_7_8_0);
+        }, TransportVersion.V_7_8_0);
 
         // If target is new node, no need to rewrite the new style API key metadata
         securityContext.executeAfterRewritingAuthentication(originalCtx -> {
             Authentication authentication = securityContext.getAuthentication();
             assertSame(original.getAuthenticatingSubject().getMetadata(), authentication.getAuthenticatingSubject().getMetadata());
-        }, VersionUtils.randomVersionBetween(random(), VERSION_API_KEY_ROLES_AS_BYTES, Version.CURRENT));
+        }, TransportVersionUtils.randomVersionBetween(random(), VERSION_API_KEY_ROLES_AS_BYTES, TransportVersion.CURRENT));
     }
 
     public void testExecuteAfterRewritingAuthenticationWillConditionallyRewriteOldApiKeyMetadata() throws IOException {
-        final Authentication original = AuthenticationTestHelper.builder().apiKey().version(Version.V_7_8_0).build();
+        final Authentication original = AuthenticationTestHelper.builder().apiKey().transportVersion(TransportVersion.V_7_8_0).build();
 
         // original authentication has the old style of role descriptor maps
         assertThat(
@@ -229,7 +240,7 @@ public void testExecuteAfterRewritingAuthenticationWillConditionallyRewriteOldAp
         securityContext.executeAfterRewritingAuthentication(originalCtx -> {
             Authentication authentication = securityContext.getAuthentication();
             assertSame(original.getAuthenticatingSubject().getMetadata(), authentication.getAuthenticatingSubject().getMetadata());
-        }, Version.V_7_8_0);
+        }, TransportVersion.V_7_8_0);
 
         // If target is new node, ensure old map style API key metadata is rewritten to bytesreference
         securityContext.executeAfterRewritingAuthentication(originalCtx -> {
@@ -247,6 +258,84 @@ public void testExecuteAfterRewritingAuthenticationWillConditionallyRewriteOldAp
                         equalTo(original.getAuthenticatingSubject().getMetadata().get(key))
                     );
                 });
-        }, VersionUtils.randomVersionBetween(random(), VERSION_API_KEY_ROLES_AS_BYTES, Version.CURRENT));
+        }, TransportVersionUtils.randomVersionBetween(random(), VERSION_API_KEY_ROLES_AS_BYTES, TransportVersion.CURRENT));
+    }
+
+    public void testExecuteAfterRemovingParentAuthorization() {
+        final Map<String, String> requestHeaders = Map.ofEntries(
+            Map.entry(AuthenticationField.PRIVILEGE_CATEGORY_KEY, randomAlphaOfLengthBetween(3, 10)),
+            Map.entry(randomAlphaOfLengthBetween(3, 8), randomAlphaOfLengthBetween(3, 8)),
+            Map.entry(Task.X_OPAQUE_ID_HTTP_HEADER, randomAlphaOfLength(10)),
+            Map.entry(Task.TRACE_ID, randomAlphaOfLength(20))
+        );
+        threadContext.putHeader(requestHeaders);
+
+        final Map<String, Object> transientHeaders = Map.ofEntries(
+            Map.entry(AuthorizationServiceField.AUTHORIZATION_INFO_KEY, Mockito.mock(AuthorizationInfo.class)),
+            Map.entry(
+                AuthenticationField.AUTHENTICATION_KEY,
+                Authentication.newAnonymousAuthentication(new AnonymousUser(Settings.EMPTY), "test-node")
+            ),
+            Map.entry(randomAlphaOfLengthBetween(3, 8), randomAlphaOfLengthBetween(3, 8)),
+            Map.entry("_some_map", Map.of(randomAlphaOfLengthBetween(3, 8), randomAlphaOfLengthBetween(3, 8))),
+            Map.entry("_remote_address", "125.124.123.122"),
+            Map.entry(Task.APM_TRACE_CONTEXT, new Object())
+        );
+        transientHeaders.forEach((k, v) -> threadContext.putTransient(k, v));
+
+        final Map<String, String> responseHeaders = Map.ofEntries(
+            Map.entry(randomAlphaOfLengthBetween(3, 6), randomAlphaOfLengthBetween(3, 8)),
+            Map.entry("_response_message", "All good."),
+            Map.entry("Warning", "Some warning!")
+        );
+        responseHeaders.forEach((k, v) -> threadContext.addResponseHeader(k, v));
+
+        // mark as system context
+        boolean setSystemContext = randomBoolean();
+        if (setSystemContext) {
+            threadContext.markAsSystemContext();
+        }
+
+        final ParentActionAuthorization parentAuthorization = new ParentActionAuthorization("indices:data/read/search");
+        securityContext.setParentAuthorization(parentAuthorization);
+
+        securityContext.executeAfterRemovingParentAuthorization(original -> {
+            // parent authorization header should be removed within execute method
+            assertThat(securityContext.getParentAuthorization(), nullValue());
+            // system context boolean should be preserved
+            assertThat(threadContext.isSystemContext(), equalTo(setSystemContext));
+            // other request and transient headers should be preserved
+            assertThat(threadContext.getHeaders().size(), equalTo(requestHeaders.size()));
+            assertThat(threadContext.getResponseHeaders().size(), equalTo(responseHeaders.size()));
+            assertThat(threadContext.getTransientHeaders().size(), equalTo(transientHeaders.size()));
+            requestHeaders.forEach((k, v) -> assertThat(threadContext.getHeader(k), equalTo(v)));
+            transientHeaders.forEach((k, v) -> assertThat(threadContext.getTransient(k), equalTo(v)));
+            responseHeaders.forEach((k, v) -> assertThat(threadContext.getResponseHeaders().get(k).get(0), equalTo(v)));
+            // warning header count is still equal to 1
+            assertThat(threadContext.getResponseHeaders().get("Warning").size(), equalTo(1));
+            // add new headers
+            threadContext.addResponseHeader("_new_response_header", randomAlphaOfLengthBetween(3, 8));
+            threadContext.putTransient("_new_transient_header", randomAlphaOfLengthBetween(3, 8));
+            threadContext.putHeader("_new_request_header", randomAlphaOfLengthBetween(3, 8));
+            threadContext.addResponseHeader("Warning", randomAlphaOfLengthBetween(3, 8));
+            // warning header is now equal to 2
+            assertThat(threadContext.getResponseHeaders().get("Warning").size(), equalTo(2));
+        });
+
+        // parent authorization should be restored after execution
+        assertThat(securityContext.getParentAuthorization(), equalTo(parentAuthorization));
+        // system context boolean is unchanged
+        assertThat(threadContext.isSystemContext(), equalTo(setSystemContext));
+        // other request and transient headers should still be there
+        assertThat(threadContext.getTransientHeaders().size(), equalTo(transientHeaders.size()));
+        requestHeaders.forEach((k, v) -> assertThat(threadContext.getHeader(k), equalTo(v)));
+        transientHeaders.forEach((k, v) -> assertThat(threadContext.getTransient(k), equalTo(v)));
+        responseHeaders.forEach((k, v) -> assertThat(threadContext.getResponseHeaders().get(k).get(0), equalTo(v)));
+        // newly added transient and request headers should be removed
+        assertThat(threadContext.getTransient("_new_transient_header"), nullValue());
+        assertThat(threadContext.getHeader("_new_request_header"), nullValue());
+        // response headers should be preserved and retain newly added ones
+        assertThat(threadContext.getResponseHeaders().get("_new_response_header"), notNullValue());
+        assertThat(threadContext.getResponseHeaders().get("Warning").size(), equalTo(2));
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java
index f4786523165d..704cb1b43b8f 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java
@@ -113,6 +113,11 @@ public void testUsage() throws Exception {
         final boolean transportSSLEnabled = randomBoolean();
         settings.put("xpack.security.transport.ssl.enabled", transportSSLEnabled);
 
+        final boolean remoteClusterPortEnabled = randomBoolean();
+        settings.put("remote_cluster.enabled", remoteClusterPortEnabled);
+        final boolean remoteClusterSslEnabled = randomBoolean();
+        settings.put("xpack.security.remote_cluster.ssl.enabled", remoteClusterSslEnabled);
+
         boolean configureEnabledFlagForTokenService = randomBoolean();
         final boolean tokenServiceEnabled;
         if (configureEnabledFlagForTokenService) {
@@ -265,6 +270,11 @@ public void testUsage() throws Exception {
                 } else {
                     assertThat(source.getValue("ssl.http.enabled"), is(httpSSLEnabled));
                     assertThat(source.getValue("ssl.transport.enabled"), is(transportSSLEnabled));
+                    if (remoteClusterPortEnabled) {
+                        assertThat(source.getValue("ssl.remote_cluster.enabled"), is(remoteClusterSslEnabled));
+                    } else {
+                        assertThat(source.getValue("ssl.remote_cluster.enabled"), nullValue());
+                    }
                 }
                 assertThat(source.getValue("realms"), is(nullValue()));
                 assertThat(source.getValue("token_service"), is(nullValue()));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
index c8a5ff380df5..72aa8c3e0125 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
@@ -340,6 +340,8 @@ public void testFilteredSettings() throws Exception {
             "truststore.password",
             "truststore.path",
             "truststore.algorithm",
+            "trust_restrictions.path",
+            "trust_restrictions.x509_fields",
             "keystore.key_password"
         ).forEach(suffix -> {
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/UnstableLocalStateSecurity.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/UnstableLocalStateSecurity.java
new file mode 100644
index 000000000000..e6321af58c56
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/UnstableLocalStateSecurity.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.reservedstate.NonStateTransformResult;
+import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
+import org.elasticsearch.reservedstate.TransformState;
+import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequest;
+import org.elasticsearch.xpack.core.ssl.SSLService;
+import org.elasticsearch.xpack.security.action.rolemapping.ReservedRoleMappingAction;
+
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * A test class that allows us to Inject new type of Reserved Handler that can
+ * simulate errors in saving role mappings.
+ * <p>
+ * We can't use our regular path to simply make an extension of LocalStateSecurity
+ * in an integration test class, because the reserved handlers are injected through
+ * SPI. (see {@link LocalReservedUnstableSecurityStateHandlerProvider})
+ */
+public class UnstableLocalStateSecurity extends LocalStateSecurity {
+
+    public UnstableLocalStateSecurity(Settings settings, Path configPath) throws Exception {
+        super(settings, configPath);
+        // We reuse most of the initialization of LocalStateSecurity, we then just overwrite
+        // the security plugin with an extra method to give us a fake RoleMappingAction.
+        Optional<Plugin> security = plugins.stream().filter(p -> p instanceof Security).findFirst();
+        if (security.isPresent()) {
+            plugins.remove(security.get());
+        }
+
+        UnstableLocalStateSecurity thisVar = this;
+        var action = new ReservedUnstableRoleMappingAction();
+
+        plugins.add(new Security(settings, super.securityExtensions()) {
+            @Override
+            protected SSLService getSslService() {
+                return thisVar.getSslService();
+            }
+
+            @Override
+            protected XPackLicenseState getLicenseState() {
+                return thisVar.getLicenseState();
+            }
+
+            @Override
+            List<ReservedClusterStateHandler<?>> reservedClusterStateHandlers() {
+                // pretend the security index is initialized after 2 seconds
+                var timer = new java.util.Timer();
+                timer.schedule(new java.util.TimerTask() {
+                    @Override
+                    public void run() {
+                        action.securityIndexRecovered();
+                        timer.cancel();
+                    }
+                }, 2_000);
+                return List.of(action);
+            }
+        });
+    }
+
+    public static class ReservedUnstableRoleMappingAction extends ReservedRoleMappingAction {
+        /**
+         * Creates a fake ReservedRoleMappingAction that doesn't actually use the role mapping store
+         */
+        public ReservedUnstableRoleMappingAction() {
+            // we don't actually need a NativeRoleMappingStore
+            super(null);
+        }
+
+        /**
+         * The nonStateTransform method is the only one that uses the native store, we simply pretend
+         * something has called the onFailure method of the listener.
+         */
+        @Override
+        protected void nonStateTransform(
+            Collection<PutRoleMappingRequest> requests,
+            TransformState prevState,
+            ActionListener<NonStateTransformResult> listener
+        ) {
+            listener.onFailure(new IllegalStateException("Fake exception"));
+        }
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/reservedstate/ReservedRoleMappingActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/reservedstate/ReservedRoleMappingActionTests.java
index b6d1a0d12679..6cdca0cb3b24 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/reservedstate/ReservedRoleMappingActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/reservedstate/ReservedRoleMappingActionTests.java
@@ -76,6 +76,7 @@ public void testValidation() {
         ClusterState state = ClusterState.builder(new ClusterName("elasticsearch")).build();
         TransformState prevState = new TransformState(state, Collections.emptySet());
         ReservedRoleMappingAction action = new ReservedRoleMappingAction(nativeRoleMappingStore);
+        action.securityIndexRecovered();
 
         String badPolicyJSON = """
             {
@@ -109,6 +110,7 @@ public void testAddRemoveRoleMapping() throws Exception {
         ClusterState state = ClusterState.builder(new ClusterName("elasticsearch")).build();
         TransformState prevState = new TransformState(state, Collections.emptySet());
         ReservedRoleMappingAction action = new ReservedRoleMappingAction(nativeRoleMappingStore);
+        action.securityIndexRecovered();
 
         String emptyJSON = "";
 
@@ -181,6 +183,7 @@ public void testNonStateTransformWaitsOnAsyncActions() throws Exception {
         ClusterState state = ClusterState.builder(new ClusterName("elasticsearch")).build();
         TransformState updatedState = new TransformState(state, Collections.emptySet());
         ReservedRoleMappingAction action = new ReservedRoleMappingAction(nativeRoleMappingStore);
+        action.securityIndexRecovered();
 
         String json = """
             {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java
index a403881ff189..482da34321f2 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenActionTests.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.xpack.security.action.token;
 
 import org.elasticsearch.ElasticsearchSecurityException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.internal.Client;
@@ -104,7 +104,7 @@ public void testInvalidateTokensWhenIndexUnavailable() throws Exception {
         assertThat(ese.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE));
 
         request = new InvalidateTokenRequest(
-            TokenService.prependVersionAndEncodeRefreshToken(Version.CURRENT, UUIDs.randomBase64UUID()),
+            TokenService.prependVersionAndEncodeRefreshToken(TransportVersion.CURRENT, UUIDs.randomBase64UUID()),
             REFRESH_TOKEN.getValue(),
             null,
             null
@@ -147,7 +147,7 @@ public void testInvalidateTokensWhenIndexClosed() throws Exception {
         assertThat(ese.status(), equalTo(RestStatus.BAD_REQUEST));
 
         request = new InvalidateTokenRequest(
-            TokenService.prependVersionAndEncodeRefreshToken(Version.CURRENT, UUIDs.randomBase64UUID()),
+            TokenService.prependVersionAndEncodeRefreshToken(TransportVersion.CURRENT, UUIDs.randomBase64UUID()),
             REFRESH_TOKEN.getValue(),
             null,
             null
@@ -161,8 +161,8 @@ public void testInvalidateTokensWhenIndexClosed() throws Exception {
 
     private String generateAccessTokenString() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput(TokenService.MINIMUM_BASE64_BYTES)) {
-            out.setVersion(Version.CURRENT);
-            Version.writeVersion(Version.CURRENT, out);
+            out.setTransportVersion(TransportVersion.CURRENT);
+            TransportVersion.writeVersion(TransportVersion.CURRENT, out);
             out.writeString(UUIDs.randomBase64UUID());
             return Base64.getEncoder().encodeToString(out.bytes().toBytesRef().bytes);
         }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/ChangePasswordRequestBuilderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/ChangePasswordRequestBuilderTests.java
index 70feac92a84f..39e25a39c5d8 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/ChangePasswordRequestBuilderTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/ChangePasswordRequestBuilderTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
@@ -22,6 +23,7 @@
 import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.LinkedHashMap;
+import java.util.Locale;
 
 import static org.elasticsearch.test.SecurityIntegTestCase.getFastStoredHashAlgoForTests;
 import static org.hamcrest.Matchers.containsString;
@@ -45,7 +47,7 @@ public void testWithCleartextPassword() throws IOException {
     public void testWithHashedPassword() throws IOException {
         final Hasher hasher = getFastStoredHashAlgoForTests();
         final char[] hash = hasher.hash(new SecureString("superlongpassword".toCharArray()));
-        final String json = formatted("""
+        final String json = Strings.format("""
             {
                 "password_hash": "%s"
             }""", new String(hash));
@@ -55,29 +57,29 @@ public void testWithHashedPassword() throws IOException {
         assertThat(request.passwordHash(), equalTo(hash));
     }
 
-    public void testWithHashedPasswordWithWrongAlgo() {
+    public void testWithHashedPasswordWithDifferentAlgo() throws IOException {
         final Hasher systemHasher = getFastStoredHashAlgoForTests();
         Hasher userHasher = getFastStoredHashAlgoForTests();
         while (userHasher.name().equals(systemHasher.name())) {
             userHasher = getFastStoredHashAlgoForTests();
         }
         final char[] hash = userHasher.hash(new SecureString("superlongpassword".toCharArray()));
-        final String json = formatted("""
+        final String json = Strings.format("""
             {"password_hash": "%s"}
             """, new String(hash));
         ChangePasswordRequestBuilder builder = new ChangePasswordRequestBuilder(mock(Client.class));
-        final IllegalArgumentException e = expectThrows(
-            IllegalArgumentException.class,
-            () -> { builder.source(new BytesArray(json.getBytes(StandardCharsets.UTF_8)), XContentType.JSON, systemHasher).request(); }
-        );
-        assertThat(e.getMessage(), containsString(userHasher.name()));
-        assertThat(e.getMessage(), containsString(systemHasher.name()));
+        final ChangePasswordRequest request = builder.source(
+            new BytesArray(json.getBytes(StandardCharsets.UTF_8)),
+            XContentType.JSON,
+            systemHasher
+        ).request();
+        assertThat(request.passwordHash(), equalTo(hash));
     }
 
     public void testWithHashedPasswordNotHash() {
-        final Hasher systemHasher = getFastStoredHashAlgoForTests();
+        final Hasher systemHasher = Hasher.valueOf(randomFrom(Hasher.getAvailableAlgoStoredHash()).toUpperCase(Locale.ROOT));
         final char[] hash = randomAlphaOfLength(20).toCharArray();
-        final String json = formatted("""
+        final String json = Strings.format("""
             {
                 "password_hash": "%s"
             }""", new String(hash));
@@ -86,8 +88,10 @@ public void testWithHashedPasswordNotHash() {
             IllegalArgumentException.class,
             () -> { builder.source(new BytesArray(json.getBytes(StandardCharsets.UTF_8)), XContentType.JSON, systemHasher).request(); }
         );
-        assertThat(e.getMessage(), containsString(Hasher.NOOP.name()));
-        assertThat(e.getMessage(), containsString(systemHasher.name()));
+        assertThat(
+            e.getMessage(),
+            containsString("The provided password hash is not a hash or it could not be resolved to a supported hash algorithm.")
+        );
     }
 
     public void testWithPasswordAndHash() throws IOException {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/HasPrivilegesRequestBuilderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/HasPrivilegesRequestBuilderTests.java
index d404668a9f66..b5d728d4a3c0 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/HasPrivilegesRequestBuilderTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/HasPrivilegesRequestBuilderTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsAction;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
@@ -18,7 +19,6 @@
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 
 import java.nio.charset.StandardCharsets;
-import java.util.Locale;
 
 import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.containsString;
@@ -93,7 +93,7 @@ public void testParseValidJsonWithJustIndexPrivileges() throws Exception {
     }
 
     public void testParseValidJsonWithJustClusterPrivileges() throws Exception {
-        String json = String.format(Locale.ROOT, """
+        String json = Strings.format("""
             { "cluster": [ "manage","%s","%s"] }""", ClusterHealthAction.NAME, ClusterStatsAction.NAME);
 
         final HasPrivilegesRequestBuilder builder = new HasPrivilegesRequestBuilder(mock(Client.class));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/PutUserRequestBuilderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/PutUserRequestBuilderTests.java
index 684013921e2e..e4590fc164e0 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/PutUserRequestBuilderTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/PutUserRequestBuilderTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
@@ -23,6 +24,7 @@
 import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.LinkedHashMap;
+import java.util.Locale;
 
 import static org.elasticsearch.test.SecurityIntegTestCase.getFastStoredHashAlgoForTests;
 import static org.hamcrest.Matchers.arrayContaining;
@@ -162,7 +164,7 @@ public void testWithEnabled() throws IOException {
     public void testWithValidPasswordHash() throws IOException {
         final Hasher hasher = getFastStoredHashAlgoForTests();
         final char[] hash = hasher.hash(new SecureString("secretpassword".toCharArray()));
-        final String json = formatted("""
+        final String json = Strings.format("""
             {
               "password_hash": "%s",
               "roles": []
@@ -179,38 +181,37 @@ public void testWithValidPasswordHash() throws IOException {
         assertThat(request.username(), equalTo("hash_user"));
     }
 
-    public void testWithMismatchedPasswordHashingAlgorithm() throws IOException {
+    public void testWithDifferentPasswordHashingAlgorithm() throws IOException {
         final Hasher systemHasher = getFastStoredHashAlgoForTests();
         Hasher userHasher = getFastStoredHashAlgoForTests();
         while (userHasher.name().equals(systemHasher.name())) {
             userHasher = getFastStoredHashAlgoForTests();
         }
         final char[] hash = userHasher.hash(new SecureString("secretpassword".toCharArray()));
-        final String json = formatted("""
+        final String json = Strings.format("""
             {
               "password_hash": "%s",
               "roles": []
             }""", new String(hash));
 
         PutUserRequestBuilder builder = new PutUserRequestBuilder(mock(Client.class));
-        final IllegalArgumentException ex = expectThrows(
-            IllegalArgumentException.class,
-            () -> {
-                builder.source("hash_user", new BytesArray(json.getBytes(StandardCharsets.UTF_8)), XContentType.JSON, systemHasher)
-                    .request();
-            }
-        );
-        assertThat(ex.getMessage(), containsString(userHasher.name()));
-        assertThat(ex.getMessage(), containsString(systemHasher.name()));
+        PutUserRequest request = builder.source(
+            "hash_user",
+            new BytesArray(json.getBytes(StandardCharsets.UTF_8)),
+            XContentType.JSON,
+            systemHasher
+        ).request();
+        assertThat(request.passwordHash(), equalTo(hash));
     }
 
     public void testWithPasswordHashThatsNotReallyAHash() throws IOException {
-        final Hasher systemHasher = Hasher.PBKDF2;
-        final String json = """
+        final Hasher systemHasher = Hasher.valueOf(randomFrom(Hasher.getAvailableAlgoStoredHash()).toUpperCase(Locale.ROOT));
+        final char[] hash = randomAlphaOfLengthBetween(14, 20).toCharArray();
+        final String json = Strings.format("""
             {
-              "password_hash": "not-a-hash",
+              "password_hash": "%s",
               "roles": []
-            }""";
+            }""", new String(hash));
 
         PutUserRequestBuilder builder = new PutUserRequestBuilder(mock(Client.class));
         final IllegalArgumentException ex = expectThrows(
@@ -220,8 +221,10 @@ public void testWithPasswordHashThatsNotReallyAHash() throws IOException {
                     .request();
             }
         );
-        assertThat(ex.getMessage(), containsString(Hasher.NOOP.name()));
-        assertThat(ex.getMessage(), containsString(systemHasher.name()));
+        assertThat(
+            ex.getMessage(),
+            containsString("The provided password hash is not a hash or it could not be resolved to a supported hash algorithm.")
+        );
     }
 
     public void testWithBothPasswordAndHash() throws IOException {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateActionTests.java
index bb56dad3b4fe..d33d3ff9190c 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateActionTests.java
@@ -141,7 +141,7 @@ public void onFailure(Exception e) {
         });
 
         assertThat(responseRef.get(), notNullValue());
-        if (anonymousUser.enabled() && false == authentication.isApiKey()) {
+        if (anonymousUser.enabled() && false == (authentication.isApiKey() || authentication.isRemoteAccess())) {
             // Roles of anonymousUser are added to non api key authentication
             final Authentication auth = responseRef.get().authentication();
             final User userInResponse = auth.getEffectiveSubject().getUser();
@@ -155,7 +155,10 @@ public void onFailure(Exception e) {
             }
             assertThat(auth.getAuthenticatingSubject().getRealm(), sameInstance(authentication.getAuthenticatingSubject().getRealm()));
             assertThat(auth.getEffectiveSubject().getRealm(), sameInstance(authentication.getEffectiveSubject().getRealm()));
-            assertThat(auth.getEffectiveSubject().getVersion(), sameInstance(authentication.getEffectiveSubject().getVersion()));
+            assertThat(
+                auth.getEffectiveSubject().getTransportVersion(),
+                sameInstance(authentication.getEffectiveSubject().getTransportVersion())
+            );
             assertThat(auth.getAuthenticationType(), sameInstance(authentication.getAuthenticationType()));
             assertThat(
                 auth.getAuthenticatingSubject().getMetadata(),
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordActionTests.java
index d2b9cfa7718c..8151034390cf 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportChangePasswordActionTests.java
@@ -154,13 +154,67 @@ public void onFailure(Exception e) {
         verify(usersStore, times(1)).changePassword(eq(request), anyActionListener());
     }
 
-    public void testIncorrectPasswordHashingAlgorithm() {
+    public void testWithPasswordThatsNotAHash() {
+        final User user = randomFrom(new ElasticUser(true), new KibanaUser(true), new User("joe"));
+        NativeUsersStore usersStore = mock(NativeUsersStore.class);
+        ChangePasswordRequest request = new ChangePasswordRequest();
+        request.username(user.principal());
+        request.passwordHash(randomAlphaOfLengthBetween(14, 20).toCharArray());
+        final AtomicReference<Throwable> throwableRef = new AtomicReference<>();
+        final AtomicReference<ActionResponse.Empty> responseRef = new AtomicReference<>();
+        TransportService transportService = new TransportService(
+            Settings.EMPTY,
+            mock(Transport.class),
+            mock(ThreadPool.class),
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+            x -> null,
+            null,
+            Collections.emptySet()
+        );
+        final String systemHash = randomFrom("pbkdf2_50000", "pbkdf2_100000", "bcrypt11", "bcrypt8", "bcrypt");
+        Settings passwordHashingSettings = Settings.builder().put(XPackSettings.PASSWORD_HASHING_ALGORITHM.getKey(), systemHash).build();
+        TransportChangePasswordAction action = new TransportChangePasswordAction(
+            passwordHashingSettings,
+            transportService,
+            mock(ActionFilters.class),
+            usersStore
+        );
+        action.doExecute(mock(Task.class), request, new ActionListener<>() {
+            @Override
+            public void onResponse(ActionResponse.Empty changePasswordResponse) {
+                responseRef.set(changePasswordResponse);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throwableRef.set(e);
+            }
+        });
+
+        assertThat(responseRef.get(), is(nullValue()));
+        assertThat(throwableRef.get(), instanceOf(IllegalArgumentException.class));
+        assertThat(
+            throwableRef.get().getMessage(),
+            containsString("The provided password hash is not a hash or it could not be resolved to a supported hash algorithm.")
+        );
+        verifyNoMoreInteractions(usersStore);
+    }
+
+    public void testWithDifferentPasswordHashingAlgorithm() {
         final User user = randomFrom(new ElasticUser(true), new KibanaUser(true), new User("joe"));
         final Hasher hasher = getFastStoredHashAlgoForTests();
         NativeUsersStore usersStore = mock(NativeUsersStore.class);
         ChangePasswordRequest request = new ChangePasswordRequest();
         request.username(user.principal());
         request.passwordHash(hasher.hash(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING));
+        doAnswer(invocation -> {
+            Object[] args = invocation.getArguments();
+            assert args.length == 2;
+            @SuppressWarnings("unchecked")
+            ActionListener<Void> listener = (ActionListener<Void>) args[1];
+            listener.onResponse(null);
+            return null;
+        }).when(usersStore).changePassword(eq(request), anyActionListener());
         final AtomicReference<Throwable> throwableRef = new AtomicReference<>();
         final AtomicReference<ActionResponse.Empty> responseRef = new AtomicReference<>();
         TransportService transportService = new TransportService(
@@ -195,10 +249,10 @@ public void onFailure(Exception e) {
             }
         });
 
-        assertThat(responseRef.get(), is(nullValue()));
-        assertThat(throwableRef.get(), instanceOf(IllegalArgumentException.class));
-        assertThat(throwableRef.get().getMessage(), containsString("incorrect password hashing algorithm"));
-        verifyNoMoreInteractions(usersStore);
+        assertThat(responseRef.get(), is(notNullValue()));
+        assertSame(responseRef.get(), ActionResponse.Empty.INSTANCE);
+        assertThat(throwableRef.get(), is(nullValue()));
+        verify(usersStore, times(1)).changePassword(eq(request), anyActionListener());
     }
 
     public void testException() {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java
index c4cb5e615c0c..740bfbd7c8aa 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java
@@ -8,7 +8,7 @@
 
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterName;
@@ -2839,7 +2839,11 @@ private static Authentication createSystemUserAuthentication(boolean isFallback)
         if (isFallback) {
             return Authentication.newInternalFallbackAuthentication(SystemUser.INSTANCE, randomAlphaOfLengthBetween(3, 8));
         } else {
-            return Authentication.newInternalAuthentication(SystemUser.INSTANCE, Version.CURRENT, randomAlphaOfLengthBetween(3, 8));
+            return Authentication.newInternalAuthentication(
+                SystemUser.INSTANCE,
+                TransportVersion.CURRENT,
+                randomAlphaOfLengthBetween(3, 8)
+            );
         }
     }
 
@@ -2869,6 +2873,7 @@ private ClusterSettings mockClusterSettings() {
         final List<Setting<?>> settingsList = new ArrayList<>();
         LoggingAuditTrail.registerSettings(settingsList);
         settingsList.addAll(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        settingsList.add(ApiKeyService.DELETE_RETENTION_PERIOD);
         return new ClusterSettings(settings, new HashSet<>(settingsList));
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java
index e458d4ef7666..ce5020e44a7c 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java
@@ -320,7 +320,8 @@ public void init() throws Exception {
                 LoggingAuditTrail.FILTER_POLICY_IGNORE_ROLES,
                 LoggingAuditTrail.FILTER_POLICY_IGNORE_INDICES,
                 LoggingAuditTrail.FILTER_POLICY_IGNORE_ACTIONS,
-                Loggers.LOG_LEVEL_SETTING
+                Loggers.LOG_LEVEL_SETTING,
+                ApiKeyService.DELETE_RETENTION_PERIOD
             )
         );
         when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
@@ -602,7 +603,7 @@ public void testSecurityConfigChangeEventFormattingForRoles() throws IOException
             roleDescriptorsStringBuilder,
             createApiKeyRequest.getMetadata() == null || createApiKeyRequest.getMetadata().isEmpty()
                 ? ""
-                : formatted(",\"metadata\":%s", metadataWithSerialization.serialization())
+                : Strings.format(",\"metadata\":%s", metadataWithSerialization.serialization())
         );
         List<String> output = CapturingLogger.output(logger.getName(), Level.INFO);
         assertThat(output.size(), is(2));
@@ -634,7 +635,7 @@ public void testSecurityConfigChangeEventFormattingForRoles() throws IOException
                 """,
             keyId,
             updateApiKeyRequest.getRoleDescriptors() == null ? "" : "," + roleDescriptorsStringBuilder,
-            updateApiKeyRequest.getMetadata() == null ? "" : formatted(",\"metadata\":%s", metadataWithSerialization.serialization())
+            updateApiKeyRequest.getMetadata() == null ? "" : Strings.format(",\"metadata\":%s", metadataWithSerialization.serialization())
         );
         output = CapturingLogger.output(logger.getName(), Level.INFO);
         assertThat(output.size(), is(2));
@@ -664,9 +665,11 @@ public void testSecurityConfigChangeEventFormattingForRoles() throws IOException
             """
                 "change":{"apikeys":{"ids":[%s]%s%s}}\
                 """,
-            bulkUpdateApiKeyRequest.getIds().stream().map(s -> formatted("\"%s\"", s)).collect(Collectors.joining(",")),
+            bulkUpdateApiKeyRequest.getIds().stream().map(s -> Strings.format("\"%s\"", s)).collect(Collectors.joining(",")),
             bulkUpdateApiKeyRequest.getRoleDescriptors() == null ? "" : "," + roleDescriptorsStringBuilder,
-            bulkUpdateApiKeyRequest.getMetadata() == null ? "" : formatted(",\"metadata\":%s", metadataWithSerialization.serialization())
+            bulkUpdateApiKeyRequest.getMetadata() == null
+                ? ""
+                : Strings.format(",\"metadata\":%s", metadataWithSerialization.serialization())
         );
         output = CapturingLogger.output(logger.getName(), Level.INFO);
         assertThat(output.size(), is(2));
@@ -759,7 +762,7 @@ public void testSecurityConfigChangeEventFormattingForRoles() throws IOException
         output = CapturingLogger.output(logger.getName(), Level.INFO);
         assertThat(output.size(), is(2));
         String generatedPutRoleAuditEventString = output.get(1);
-        String expectedPutRoleAuditEventString = String.format(Locale.ROOT, """
+        String expectedPutRoleAuditEventString = Strings.format("""
             "put":{"role":{"name":"%s","role_descriptor":%s}}\
             """, putRoleRequest.name(), auditedRolesMap.get(putRoleRequest.name()));
         assertThat(generatedPutRoleAuditEventString, containsString(expectedPutRoleAuditEventString));
@@ -1237,7 +1240,7 @@ public void testSecurityConfigChangeEventFormattingForUsers() throws IOException
         output = CapturingLogger.output(logger.getName(), Level.INFO);
         assertThat(output.size(), is(2));
         String generatedEnableUserAuditEventString = output.get(1);
-        String expectedEnableUserAuditEventString = formatted("""
+        String expectedEnableUserAuditEventString = Strings.format("""
             "change":{"enable":{"user":{"name":"%s"}}}\
             """, username);
         assertThat(generatedEnableUserAuditEventString, containsString(expectedEnableUserAuditEventString));
@@ -1262,7 +1265,7 @@ public void testSecurityConfigChangeEventFormattingForUsers() throws IOException
         output = CapturingLogger.output(logger.getName(), Level.INFO);
         assertThat(output.size(), is(2));
         String generatedDisableUserAuditEventString = output.get(1);
-        String expectedDisableUserAuditEventString = formatted("""
+        String expectedDisableUserAuditEventString = Strings.format("""
             "change":{"disable":{"user":{"name":"%s"}}}\
             """, username);
         assertThat(generatedDisableUserAuditEventString, containsString(expectedDisableUserAuditEventString));
@@ -1286,7 +1289,7 @@ public void testSecurityConfigChangeEventFormattingForUsers() throws IOException
         output = CapturingLogger.output(logger.getName(), Level.INFO);
         assertThat(output.size(), is(2));
         String generatedChangePasswordAuditEventString = output.get(1);
-        String expectedChangePasswordAuditEventString = formatted("""
+        String expectedChangePasswordAuditEventString = Strings.format("""
             "change":{"password":{"user":{"name":"%s"}}}\
             """, username);
         assertThat(generatedChangePasswordAuditEventString, containsString(expectedChangePasswordAuditEventString));
@@ -1312,7 +1315,7 @@ public void testSecurityConfigChangeEventFormattingForUsers() throws IOException
         output = CapturingLogger.output(logger.getName(), Level.INFO);
         assertThat(output.size(), is(2));
         String generatedDeleteUserAuditEventString = output.get(1);
-        String expectedDeleteUserAuditEventString = formatted("""
+        String expectedDeleteUserAuditEventString = Strings.format("""
             "delete":{"user":{"name":"%s"}}\
             """, username);
         assertThat(generatedDeleteUserAuditEventString, containsString(expectedDeleteUserAuditEventString));
@@ -1353,7 +1356,7 @@ public void testSecurityConfigChangeEventFormattingForServiceAccountToken() {
         assertThat(output.size(), is(2));
         String generatedCreateServiceAccountTokenAuditEventString = output.get(1);
 
-        final String expectedCreateServiceAccountTokenAuditEventString = String.format(Locale.ROOT, """
+        final String expectedCreateServiceAccountTokenAuditEventString = Strings.format("""
             "create":{"service_token":{"namespace":"%s","service":"%s","name":"%s"}}""", namespace, serviceName, tokenName);
         assertThat(generatedCreateServiceAccountTokenAuditEventString, containsString(expectedCreateServiceAccountTokenAuditEventString));
         generatedCreateServiceAccountTokenAuditEventString = generatedCreateServiceAccountTokenAuditEventString.replace(
@@ -1388,7 +1391,7 @@ public void testSecurityConfigChangeEventFormattingForServiceAccountToken() {
         assertThat(output.size(), is(2));
         String generatedDeleteServiceAccountTokenAuditEventString = output.get(1);
 
-        final String expectedDeleteServiceAccountTokenAuditEventString = String.format(Locale.ROOT, """
+        final String expectedDeleteServiceAccountTokenAuditEventString = Strings.format("""
             "delete":{"service_token":{"namespace":"%s","service":"%s","name":"%s"}}""", namespace, serviceName, tokenName);
         assertThat(generatedDeleteServiceAccountTokenAuditEventString, containsString(expectedDeleteServiceAccountTokenAuditEventString));
         generatedDeleteServiceAccountTokenAuditEventString = generatedDeleteServiceAccountTokenAuditEventString.replace(
@@ -1467,7 +1470,7 @@ public void testSecurityConfigChangeEventFormattingForUserProfile() {
         auditTrail.accessGranted(requestId, authentication, UpdateProfileDataAction.NAME, updateProfileDataRequest, authorizationInfo);
         assertThat(output.size(), is(2));
         String generatedUpdateAuditEventString = output.get(1);
-        final String expectedUpdateAuditEventString = formatted("""
+        final String expectedUpdateAuditEventString = Strings.format("""
             "put":{"uid":"%s","labels":{"space":"production"},"data":{"theme":"default"}}""", updateProfileDataRequest.getUid());
         assertThat(generatedUpdateAuditEventString, containsString(expectedUpdateAuditEventString));
         generatedUpdateAuditEventString = generatedUpdateAuditEventString.replace(", " + expectedUpdateAuditEventString, "");
@@ -1503,12 +1506,10 @@ public void testSecurityConfigChangeEventFormattingForUserProfile() {
         checkedFields = new MapBuilder<>(commonFields);
         checkedFields.remove(LoggingAuditTrail.ORIGIN_ADDRESS_FIELD_NAME);
         checkedFields.remove(LoggingAuditTrail.ORIGIN_TYPE_FIELD_NAME);
+        Object[] args = new Object[] { setProfileEnabledRequest.isEnabled() ? "enable" : "disable" };
         checkedFields.put("type", "audit")
             .put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, "security_config_change")
-            .put(
-                LoggingAuditTrail.EVENT_ACTION_FIELD_NAME,
-                formatted("change_%s_user_profile", setProfileEnabledRequest.isEnabled() ? "enable" : "disable")
-            )
+            .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, Strings.format("change_%s_user_profile", args))
             .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId);
         assertMsg(generatedSetEnabledAuditEventString, checkedFields.map());
         // clear log
@@ -2019,7 +2020,7 @@ public void testAccessGrantedInternalSystemAction() throws Exception {
             .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "access_granted")
             .put(LoggingAuditTrail.AUTHENTICATION_TYPE_FIELD_NAME, authentication.getAuthenticationType().toString())
             .put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, systemUser.principal())
-            .put(LoggingAuditTrail.PRINCIPAL_REALM_FIELD_NAME, authentication.getSourceRealm().getName())
+            .put(LoggingAuditTrail.PRINCIPAL_REALM_FIELD_NAME, authentication.getEffectiveSubject().getRealm().getName())
             .put(LoggingAuditTrail.ACTION_FIELD_NAME, "internal:_action")
             .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, request.getClass().getSimpleName())
             .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java
index 811d415c5fca..f4a4dd39aabf 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java
@@ -10,10 +10,13 @@
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.search.TotalHits;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.bulk.BulkAction;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
@@ -29,18 +32,23 @@
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.update.UpdateAction;
+import org.elasticsearch.action.update.UpdateRequestBuilder;
+import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.cache.Cache;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.ListenableFuture;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
@@ -49,6 +57,9 @@
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockLogAppender;
@@ -65,6 +76,9 @@
 import org.elasticsearch.xcontent.json.JsonXContent;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.security.action.ClearSecurityCacheAction;
+import org.elasticsearch.xpack.core.security.action.ClearSecurityCacheRequest;
+import org.elasticsearch.xpack.core.security.action.ClearSecurityCacheResponse;
 import org.elasticsearch.xpack.core.security.action.apikey.ApiKeyTests;
 import org.elasticsearch.xpack.core.security.action.apikey.BulkUpdateApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.apikey.BulkUpdateApiKeyResponse;
@@ -97,6 +111,7 @@
 import org.elasticsearch.xpack.security.test.SecurityMocks;
 import org.junit.After;
 import org.junit.Before;
+import org.mockito.ArgumentMatcher;
 import org.mockito.Mockito;
 
 import java.io.IOException;
@@ -153,6 +168,7 @@
 import static org.hamcrest.Matchers.sameInstance;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.argThat;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
@@ -167,6 +183,7 @@ public class ApiKeyServiceTests extends ESTestCase {
     private Client client;
     private SecurityIndexManager securityIndex;
     private CacheInvalidatorRegistry cacheInvalidatorRegistry;
+    private Clock clock;
 
     @Before
     public void createThreadPool() {
@@ -195,6 +212,9 @@ public void setupMocks() {
         this.client = mock(Client.class);
         this.securityIndex = SecurityMocks.mockSecurityIndexManager();
         this.cacheInvalidatorRegistry = mock(CacheInvalidatorRegistry.class);
+        // Mock a clock that returns real clock time by default
+        clock = mock(Clock.class);
+        doAnswer(invocation -> Instant.now()).when(clock).instant();
     }
 
     public void testCreateApiKeyUsesBulkIndexAction() throws Exception {
@@ -349,6 +369,105 @@ public void testInvalidateApiKeys() throws Exception {
         assertThat(invalidateApiKeyResponse.getInvalidatedApiKeys(), emptyIterable());
     }
 
+    @SuppressWarnings("unchecked")
+    public void testInvalidateApiKeysWillSetInvalidatedFlagAndRecordTimestamp() {
+        final int docId = randomIntBetween(0, Integer.MAX_VALUE);
+        final String apiKeyId = randomAlphaOfLength(20);
+
+        // Mock the search request for keys to invalidate
+        when(client.threadPool()).thenReturn(threadPool);
+        when(client.prepareSearch(eq(SECURITY_MAIN_ALIAS))).thenReturn(new SearchRequestBuilder(client, SearchAction.INSTANCE));
+        doAnswer(invocation -> {
+            final var listener = (ActionListener<SearchResponse>) invocation.getArguments()[1];
+            final var searchHit = new SearchHit(docId, apiKeyId);
+            try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+                builder.map(buildApiKeySourceDoc("some_hash".toCharArray()));
+                searchHit.sourceRef(BytesReference.bytes(builder));
+            }
+            final var internalSearchResponse = new InternalSearchResponse(
+                new SearchHits(
+                    new SearchHit[] { searchHit },
+                    new TotalHits(1, TotalHits.Relation.EQUAL_TO),
+                    randomFloat(),
+                    null,
+                    null,
+                    null
+                ),
+                null,
+                null,
+                null,
+                false,
+                null,
+                0
+            );
+            final var searchResponse = new SearchResponse(
+                internalSearchResponse,
+                randomAlphaOfLengthBetween(3, 8),
+                1,
+                1,
+                0,
+                10,
+                null,
+                null
+            );
+            listener.onResponse(searchResponse);
+            return null;
+        }).when(client).search(any(SearchRequest.class), anyActionListener());
+
+        // Capture the Update request so that we can verify it is configured as expected
+        when(client.prepareBulk()).thenReturn(new BulkRequestBuilder(client, BulkAction.INSTANCE));
+        final var updateRequestBuilder = Mockito.spy(new UpdateRequestBuilder(client, UpdateAction.INSTANCE));
+        when(client.prepareUpdate(eq(SECURITY_MAIN_ALIAS), eq(apiKeyId))).thenReturn(updateRequestBuilder);
+
+        // Stub bulk and cache clearing calls so that the entire action flow can complete (not strictly necessary but nice to have)
+        doAnswer(invocation -> {
+            final var listener = (ActionListener<BulkResponse>) invocation.getArguments()[1];
+            listener.onResponse(
+                new BulkResponse(
+                    new BulkItemResponse[] {
+                        BulkItemResponse.success(
+                            docId,
+                            DocWriteRequest.OpType.UPDATE,
+                            new UpdateResponse(
+                                mock(ShardId.class),
+                                apiKeyId,
+                                randomLong(),
+                                randomLong(),
+                                randomLong(),
+                                DocWriteResponse.Result.UPDATED
+                            )
+                        ) },
+                    randomLongBetween(1, 100)
+                )
+            );
+            return null;
+        }).when(client).bulk(any(BulkRequest.class), anyActionListener());
+        doAnswer(invocation -> {
+            final var listener = (ActionListener<ClearSecurityCacheResponse>) invocation.getArguments()[2];
+            listener.onResponse(mock(ClearSecurityCacheResponse.class));
+            return null;
+        }).when(client).execute(eq(ClearSecurityCacheAction.INSTANCE), any(ClearSecurityCacheRequest.class), anyActionListener());
+
+        final long invalidationTime = randomMillisUpToYear9999();
+        when(clock.instant()).thenReturn(Instant.ofEpochMilli(invalidationTime));
+        final ApiKeyService service = createApiKeyService();
+        PlainActionFuture<InvalidateApiKeyResponse> future = new PlainActionFuture<>();
+        service.invalidateApiKeys(null, null, null, new String[] { apiKeyId }, future);
+        final InvalidateApiKeyResponse invalidateApiKeyResponse = future.actionGet();
+
+        assertThat(invalidateApiKeyResponse.getInvalidatedApiKeys(), equalTo(List.of(apiKeyId)));
+        verify(updateRequestBuilder).setDoc(
+            argThat(
+                (ArgumentMatcher<Map<String, Object>>) argument -> Map.of(
+                    "api_key_invalidated",
+                    true,
+                    "invalidation_time",
+                    invalidationTime
+                ).equals(argument)
+            )
+        );
+    }
+
     public void testCreateApiKeyWillCacheOnCreation() {
         final Settings settings = Settings.builder().put(XPackSettings.API_KEY_SERVICE_ENABLED_SETTING.getKey(), true).build();
         final ApiKeyService service = createApiKeyService(settings);
@@ -1396,8 +1515,8 @@ public void testGetCreatorRealm() {
 
         // Realm
         final Authentication authentication3 = AuthenticationTests.randomRealmAuthentication(randomBoolean());
-        assertThat(ApiKeyService.getCreatorRealmName(authentication3), equalTo(authentication3.getSourceRealm().getName()));
-        assertThat(ApiKeyService.getCreatorRealmType(authentication3), equalTo(authentication3.getSourceRealm().getType()));
+        assertThat(ApiKeyService.getCreatorRealmName(authentication3), equalTo(authentication3.getEffectiveSubject().getRealm().getName()));
+        assertThat(ApiKeyService.getCreatorRealmType(authentication3), equalTo(authentication3.getEffectiveSubject().getRealm().getType()));
 
         // Realm run-as
         final Authentication authentication4 = authentication3.runAs(AuthenticationTests.randomUser(), lookupRealmRef);
@@ -1410,8 +1529,19 @@ public void testGetCreatorRealm() {
             AuthenticationTests.randomAnonymousAuthentication(),
             AuthenticationTests.randomInternalAuthentication()
         );
-        assertThat(ApiKeyService.getCreatorRealmName(authentication5), equalTo(authentication5.getSourceRealm().getName()));
-        assertThat(ApiKeyService.getCreatorRealmType(authentication5), equalTo(authentication5.getSourceRealm().getType()));
+        assertThat(ApiKeyService.getCreatorRealmName(authentication5), equalTo(authentication5.getEffectiveSubject().getRealm().getName()));
+        assertThat(ApiKeyService.getCreatorRealmType(authentication5), equalTo(authentication5.getEffectiveSubject().getRealm().getType()));
+
+        // Failed run-as returns authenticating subject's realm
+        final Authentication authentication6 = authentication3.runAs(AuthenticationTests.randomUser(), null);
+        assertThat(
+            ApiKeyService.getCreatorRealmName(authentication6),
+            equalTo(authentication6.getAuthenticatingSubject().getRealm().getName())
+        );
+        assertThat(
+            ApiKeyService.getCreatorRealmType(authentication6),
+            equalTo(authentication6.getAuthenticatingSubject().getRealm().getType())
+        );
     }
 
     public void testGetOwnersRealmNames() {
@@ -1472,7 +1602,7 @@ public void testGetOwnersRealmNames() {
             randomAlphaOfLength(8),
             apiKeyCreatorRealm,
             "file",
-            Version.CURRENT
+            TransportVersion.CURRENT
         );
         assertThat(Arrays.asList(ApiKeyService.getOwnersRealmNames(authentication)), contains(apiKeyCreatorRealm));
         assertThat(Arrays.asList(ApiKeyService.getOwnersRealmNames(authentication.token())), contains(apiKeyCreatorRealm));
@@ -1908,7 +2038,7 @@ public static Authentication createApiKeyAuthentication(
             Authentication authentication,
             Set<RoleDescriptor> userRoles,
             List<RoleDescriptor> keyRoles,
-            Version version
+            TransportVersion version
         ) throws Exception {
             XContentBuilder keyDocSource = ApiKeyService.newDocument(
                 getFastStoredHashAlgoForTests().hash(new SecureString(randomAlphaOfLength(16).toCharArray())),
@@ -1977,7 +2107,7 @@ public static Authentication createApiKeyAuthentication(ApiKeyService apiKeyServ
                 authentication,
                 Collections.singleton(new RoleDescriptor("user_role_" + randomAlphaOfLength(4), new String[] { "manage" }, null, null)),
                 null,
-                Version.CURRENT
+                TransportVersion.CURRENT
             );
         }
     }
@@ -1992,12 +2122,16 @@ private ApiKeyService createApiKeyService(Settings baseSettings) {
             .put(XPackSettings.API_KEY_SERVICE_ENABLED_SETTING.getKey(), true)
             .put(baseSettings)
             .build();
+        final ClusterSettings clusterSettings = new ClusterSettings(
+            settings,
+            Sets.union(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS, Set.of(ApiKeyService.DELETE_RETENTION_PERIOD))
+        );
         final ApiKeyService service = new ApiKeyService(
             settings,
-            Clock.systemUTC(),
+            clock,
             client,
             securityIndex,
-            ClusterServiceUtils.createClusterService(threadPool),
+            ClusterServiceUtils.createClusterService(threadPool, clusterSettings),
             cacheInvalidatorRegistry,
             threadPool
         );
@@ -2017,8 +2151,11 @@ private Map<String, Object> buildApiKeySourceDoc(char[] hash) {
         sourceMap.put("api_key_hash", new String(hash));
         sourceMap.put("name", randomAlphaOfLength(12));
         sourceMap.put("version", 0);
-        sourceMap.put("role_descriptors", Collections.singletonMap("a role", Collections.singletonMap("cluster", "all")));
-        sourceMap.put("limited_by_role_descriptors", Collections.singletonMap("limited role", Collections.singletonMap("cluster", "all")));
+        sourceMap.put("role_descriptors", Collections.singletonMap("a role", Collections.singletonMap("cluster", List.of("all"))));
+        sourceMap.put(
+            "limited_by_role_descriptors",
+            Collections.singletonMap("limited role", Collections.singletonMap("cluster", List.of("all")))
+        );
         Map<String, Object> creatorMap = new HashMap<>();
         creatorMap.put("principal", "test_user");
         creatorMap.put("full_name", "test user");
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
index fdc2a613c8ad..6d5eaf3ef2df 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
@@ -37,9 +37,11 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.env.Environment;
@@ -304,7 +306,11 @@ public void init() throws Exception {
             runnable.run();
             return null;
         }).when(securityIndex).checkIndexVersionThenExecute(anyConsumer(), any(Runnable.class));
-        ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool);
+        final ClusterSettings clusterSettings = new ClusterSettings(
+            settings,
+            Sets.union(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS, Set.of(ApiKeyService.DELETE_RETENTION_PERIOD))
+        );
+        ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool, clusterSettings);
         final SecurityContext securityContext = new SecurityContext(settings, threadContext);
         apiKeyService = new ApiKeyService(
             settings,
@@ -388,7 +394,7 @@ public void testTokenFirstMissingSecondFound() throws Exception {
         service.authenticate("action", transportRequest, true, ActionListener.wrap(authentication -> {
             assertThat(threadContext.getTransient(AuthenticationResult.THREAD_CONTEXT_KEY), is(authenticationResult));
             assertThat(threadContext.getTransient(AuthenticationField.AUTHENTICATION_KEY), is(authentication));
-            assertThat(authentication.getDomain(), is(secondDomain));
+            assertThat(authentication.getEffectiveSubject().getRealm().getDomain(), is(secondDomain));
             verify(auditTrail).authenticationSuccess(anyString(), eq(authentication), eq("action"), eq(transportRequest));
             setCompletedToTrue(completed);
         }, this::logAndFail));
@@ -992,7 +998,7 @@ public void testAuthenticateTransportContextAndHeader() throws Exception {
                 assertThat(authentication, notNullValue());
                 assertThat(authentication.getEffectiveSubject().getUser(), sameInstance(user1));
                 assertThat(authentication.getAuthenticationType(), is(AuthenticationType.REALM));
-                assertThat(authentication.getDomain(), is(firstDomain));
+                assertThat(authentication.getEffectiveSubject().getRealm().getDomain(), is(firstDomain));
                 assertThreadContextContainsAuthentication(authentication);
                 authRef.set(authentication);
                 authHeaderRef.set(threadContext.getHeader(AuthenticationField.AUTHENTICATION_KEY));
@@ -1270,7 +1276,7 @@ public void testAnonymousUserRest() throws Exception {
             assertThat(result, notNullValue());
             assertThat(result.v1().getEffectiveSubject().getUser(), sameInstance((Object) anonymousUser));
             assertThat(result.v1().getAuthenticationType(), is(AuthenticationType.ANONYMOUS));
-            assertThat(result.v1().getDomain(), nullValue());
+            assertThat(result.v1().getEffectiveSubject().getRealm().getDomain(), nullValue());
             assertThreadContextContainsAuthentication(result.v1());
             assertThat(expectAuditRequestId(threadContext), is(result.v2()));
             verify(auditTrail).authenticationSuccess(result.v2(), result.v1(), request);
@@ -1345,7 +1351,7 @@ public void testAnonymousUserTransportNoDefaultUser() throws Exception {
             assertThat(expectAuditRequestId(threadContext), is(result.v2()));
             assertThat(result.v1().getEffectiveSubject().getUser(), sameInstance(anonymousUser));
             assertThat(result.v1().getAuthenticationType(), is(AuthenticationType.ANONYMOUS));
-            assertThat(result.v1().getDomain(), nullValue());
+            assertThat(result.v1().getEffectiveSubject().getRealm().getDomain(), nullValue());
             assertThreadContextContainsAuthentication(result.v1());
             verify(operatorPrivilegesService).maybeMarkOperatorUser(eq(result.v1()), eq(threadContext));
         });
@@ -1382,7 +1388,7 @@ public void testAnonymousUserTransportWithDefaultUser() throws Exception {
             assertThat(expectAuditRequestId(threadContext), is(result.v2()));
             assertThat(result.v1().getEffectiveSubject().getUser(), sameInstance(SystemUser.INSTANCE));
             assertThat(result.v1().getAuthenticationType(), is(AuthenticationType.INTERNAL));
-            assertThat(result.v1().getDomain(), nullValue());
+            assertThat(result.v1().getEffectiveSubject().getRealm().getDomain(), nullValue());
             assertThreadContextContainsAuthentication(result.v1());
             verify(operatorPrivilegesService).maybeMarkOperatorUser(eq(result.v1()), eq(threadContext));
         });
@@ -2133,7 +2139,7 @@ public void testApiKeyAuth() {
                 assertThat(result.v1().getEffectiveSubject().getUser().fullName(), is("john doe"));
                 assertThat(result.v1().getEffectiveSubject().getUser().email(), is("john@doe.com"));
                 assertThat(result.v1().getAuthenticationType(), is(AuthenticationType.API_KEY));
-                assertThat(result.v1().getDomain(), nullValue());
+                assertThat(result.v1().getEffectiveSubject().getRealm().getDomain(), nullValue());
                 verify(operatorPrivilegesService).maybeMarkOperatorUser(eq(result.v1()), eq(threadContext));
             });
         }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceMock.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceMock.java
index 47a19b74d799..f88715436318 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceMock.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceMock.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.security.authc;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.SecureString;
@@ -50,7 +50,9 @@ public TokenServiceMock(TokenService tokenService, Client client) {
 
     public MockToken mockAccessToken() throws Exception {
         final String uuid = UUIDs.randomBase64UUID();
-        final SecureString encoded = new SecureString(tokenService.prependVersionAndEncodeAccessToken(Version.CURRENT, uuid).toCharArray());
+        final SecureString encoded = new SecureString(
+            tokenService.prependVersionAndEncodeAccessToken(TransportVersion.CURRENT, uuid).toCharArray()
+        );
         final String hashedToken = TokenService.hashTokenString(uuid);
         return new MockToken(uuid, encoded, hashedToken);
     }
@@ -61,7 +63,7 @@ public void defineToken(MockToken token, Authentication authentication) throws I
 
     public void defineToken(MockToken token, Authentication authentication, boolean valid) throws IOException {
         Instant expiration = Instant.now().plusSeconds(TimeUnit.MINUTES.toSeconds(20));
-        final UserToken userToken = new UserToken(token.hashedToken, Version.CURRENT, authentication, expiration, Map.of());
+        final UserToken userToken = new UserToken(token.hashedToken, TransportVersion.CURRENT, authentication, expiration, Map.of());
         final Map<String, Object> document = new HashMap<>();
         document.put("access_token", Map.of("user_token", userToken, "invalidated", valid == false));
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java
index 8d025666829b..8efeb1ce2c71 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteRequest;
@@ -459,7 +460,7 @@ private RefreshTokenStatus newRefreshTokenStatus(
         String iv,
         String salt
     ) {
-        if (authentication.getEffectiveSubject().getVersion().onOrAfter(VERSION_CLIENT_AUTH_FOR_REFRESH)) {
+        if (authentication.getEffectiveSubject().getTransportVersion().onOrAfter(VERSION_CLIENT_AUTH_FOR_REFRESH)) {
             return new RefreshTokenStatus(invalidated, authentication, refreshed, refreshInstant, supersedingTokens, iv, salt);
         } else {
             return new RefreshTokenStatus(
@@ -674,7 +675,7 @@ public void testNotValidPre72Tokens() throws Exception {
             .build(false);
         mockGetTokenFromId(tokenService, UUIDs.randomBase64UUID(), authentication, false);
         ThreadContext requestContext = new ThreadContext(Settings.EMPTY);
-        storeTokenHeader(requestContext, generateAccessToken(tokenService, Version.V_7_1_0));
+        storeTokenHeader(requestContext, generateAccessToken(tokenService, TransportVersion.V_7_1_0));
 
         try (ThreadContext.StoredContext ignore = requestContext.newStoredContextPreservingResponseHeaders()) {
             PlainActionFuture<UserToken> future = new PlainActionFuture<>();
@@ -693,7 +694,7 @@ public void testNotValidAfter72Tokens() throws Exception {
             .build(false);
         mockGetTokenFromId(tokenService, UUIDs.randomBase64UUID(), authentication, false);
         ThreadContext requestContext = new ThreadContext(Settings.EMPTY);
-        storeTokenHeader(requestContext, generateAccessToken(tokenService, randomFrom(Version.V_7_2_0, Version.V_7_3_2)));
+        storeTokenHeader(requestContext, generateAccessToken(tokenService, randomFrom(TransportVersion.V_7_2_0, TransportVersion.V_7_3_2)));
 
         try (ThreadContext.StoredContext ignore = requestContext.newStoredContextPreservingResponseHeaders()) {
             PlainActionFuture<UserToken> future = new PlainActionFuture<>();
@@ -802,7 +803,7 @@ public void testSupersedingTokenEncryption() throws Exception {
         final String newRefreshToken = UUIDs.randomBase64UUID();
         final byte[] iv = tokenService.getRandomBytes(TokenService.IV_BYTES);
         final byte[] salt = tokenService.getRandomBytes(TokenService.SALT_BYTES);
-        final Version version = tokenService.getTokenVersionCompatibility();
+        final TransportVersion version = tokenService.getTokenVersionCompatibility();
         String encryptedTokens = tokenService.encryptSupersedingTokens(newAccessToken, newRefreshToken, refrehToken, iv, salt);
         RefreshTokenStatus refreshTokenStatus = newRefreshTokenStatus(
             false,
@@ -813,7 +814,7 @@ public void testSupersedingTokenEncryption() throws Exception {
             Base64.getEncoder().encodeToString(iv),
             Base64.getEncoder().encodeToString(salt)
         );
-        refreshTokenStatus.setVersion(version);
+        refreshTokenStatus.setTransportVersion(version);
         mockGetTokenAsyncForDecryptedToken(newAccessToken);
         tokenService.decryptAndReturnSupersedingTokens(refrehToken, refreshTokenStatus, securityTokensIndex, authentication, tokenFuture);
         if (version.onOrAfter(TokenService.VERSION_ACCESS_TOKENS_AS_UUIDS)) {
@@ -850,7 +851,7 @@ public void testCannotValidateTokenIfLicenseDoesNotAllowTokens() throws Exceptio
             userTokenId
         );
         final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
-        storeTokenHeader(threadContext, tokenService.prependVersionAndEncodeAccessToken(token.getVersion(), accessToken));
+        storeTokenHeader(threadContext, tokenService.prependVersionAndEncodeAccessToken(token.getTransportVersion(), accessToken));
 
         PlainActionFuture<UserToken> authFuture = new PlainActionFuture<>();
         when(licenseState.isAllowed(Security.TOKEN_SERVICE_FEATURE)).thenReturn(false);
@@ -895,7 +896,7 @@ public static void mockGetTokenFromId(
             @SuppressWarnings("unchecked")
             ActionListener<GetResponse> listener = (ActionListener<GetResponse>) invocationOnMock.getArguments()[1];
             GetResponse response = mock(GetResponse.class);
-            Version tokenVersion = tokenService.getTokenVersionCompatibility();
+            TransportVersion tokenVersion = tokenService.getTokenVersionCompatibility();
             final String possiblyHashedUserTokenId;
             if (tokenVersion.onOrAfter(TokenService.VERSION_ACCESS_TOKENS_AS_UUIDS)) {
                 possiblyHashedUserTokenId = TokenService.hashTokenString(userTokenId);
@@ -929,7 +930,7 @@ protected static UserToken buildUserToken(
         Authentication authentication,
         Map<String, Object> metadata
     ) {
-        final Version tokenVersion = tokenService.getTokenVersionCompatibility();
+        final TransportVersion tokenVersion = tokenService.getTokenVersionCompatibility();
         final String possiblyHashedUserTokenId;
         if (tokenVersion.onOrAfter(TokenService.VERSION_ACCESS_TOKENS_AS_UUIDS)) {
             possiblyHashedUserTokenId = TokenService.hashTokenString(userTokenId);
@@ -948,7 +949,7 @@ private void mockGetTokenFromId(UserToken userToken, boolean isExpired) {
             ActionListener<GetResponse> listener = (ActionListener<GetResponse>) invocationOnMock.getArguments()[1];
             GetResponse response = mock(GetResponse.class);
             final String possiblyHashedUserTokenId;
-            if (userToken.getVersion().onOrAfter(TokenService.VERSION_ACCESS_TOKENS_AS_UUIDS)) {
+            if (userToken.getTransportVersion().onOrAfter(TokenService.VERSION_ACCESS_TOKENS_AS_UUIDS)) {
                 possiblyHashedUserTokenId = TokenService.hashTokenString(userToken.getId());
             } else {
                 possiblyHashedUserTokenId = userToken.getId();
@@ -978,7 +979,7 @@ private void mockGetTokenFromId(UserToken userToken, boolean isExpired) {
 
     private void mockFindTokenFromRefreshToken(String refreshToken, UserToken userToken, @Nullable RefreshTokenStatus refreshTokenStatus) {
         String storedRefreshToken;
-        if (userToken.getVersion().onOrAfter(TokenService.VERSION_HASHED_TOKENS)) {
+        if (userToken.getTransportVersion().onOrAfter(TokenService.VERSION_HASHED_TOKENS)) {
             storedRefreshToken = TokenService.hashTokenString(refreshToken);
         } else {
             storedRefreshToken = refreshToken;
@@ -1071,7 +1072,7 @@ private DiscoveryNode addAnotherDataNodeWithVersion(ClusterService clusterServic
         return anotherDataNode;
     }
 
-    private String generateAccessToken(TokenService tokenService, Version version) throws Exception {
+    private String generateAccessToken(TokenService tokenService, TransportVersion version) throws Exception {
         String accessTokenString = UUIDs.randomBase64UUID();
         if (version.onOrAfter(TokenService.VERSION_ACCESS_TOKENS_AS_UUIDS)) {
             accessTokenString = TokenService.hashTokenString(accessTokenString);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserRolesStoreTests.java
index 728aa35499f8..258770b10c74 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserRolesStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileUserRolesStoreTests.java
@@ -294,7 +294,7 @@ private void assertInvalidInputIsSilentlyIgnored(String input) throws Exception
         Path file = createTempFile();
         Files.write(file, input.getBytes(StandardCharsets.UTF_8));
         Map<String, String[]> usersRoles = FileUserRolesStore.parseFile(file, null);
-        String reason = formatted("Expected userRoles to be empty, but was %s", usersRoles.keySet());
+        String reason = Strings.format("Expected userRoles to be empty, but was %s", usersRoles.keySet());
         assertThat(reason, usersRoles.keySet(), hasSize(0));
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/FallbackableClaimTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/FallbackableClaimTests.java
new file mode 100644
index 000000000000..3fccbb4e5458
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/FallbackableClaimTests.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authc.jwt;
+
+import com.nimbusds.jwt.JWTClaimsSet;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.text.ParseException;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class FallbackableClaimTests extends ESTestCase {
+
+    public void testNoFallback() throws ParseException {
+        final String name = randomAlphaOfLength(10);
+        final String value = randomAlphaOfLength(10);
+        final FallbackableClaim fallbackableClaim = new FallbackableClaim(name, null, JWTClaimsSet.parse(Map.of(name, value)));
+        assertThat(fallbackableClaim.getActualName(), equalTo(name));
+        assertThat(fallbackableClaim.toString(), equalTo(name));
+        assertThat(fallbackableClaim.getStringClaimValue(), equalTo(value));
+        assertThat(fallbackableClaim.getStringListClaimValue(), equalTo(List.of(value)));
+    }
+
+    public void testFallback() throws ParseException {
+        final String name = randomAlphaOfLength(10);
+        final String fallbackName = randomAlphaOfLength(12);
+        final String value = randomAlphaOfLength(10);
+
+        // fallback ignored
+        final JWTClaimsSet claimSet1 = JWTClaimsSet.parse(Map.of(name, value, fallbackName, randomAlphaOfLength(16)));
+        final FallbackableClaim fallbackableClaim1 = new FallbackableClaim(name, Map.of(name, fallbackName), claimSet1);
+        assertThat(fallbackableClaim1.getActualName(), equalTo(name));
+        assertThat(fallbackableClaim1.toString(), equalTo(name));
+        assertThat(fallbackableClaim1.getStringClaimValue(), equalTo(value));
+        assertThat(fallbackableClaim1.getStringListClaimValue(), equalTo(List.of(value)));
+
+        // fallback active
+        final JWTClaimsSet claimSet2 = JWTClaimsSet.parse(Map.of(fallbackName, value));
+        final FallbackableClaim fallbackableClaim2 = new FallbackableClaim(name, Map.of(name, fallbackName), claimSet2);
+        assertThat(fallbackableClaim2.getActualName(), equalTo(fallbackName));
+        assertThat(fallbackableClaim2.toString(), equalTo(fallbackName + " (fallback of " + name + ")"));
+        assertThat(fallbackableClaim2.getStringClaimValue(), equalTo(value));
+        assertThat(fallbackableClaim2.getStringListClaimValue(), equalTo(List.of(value)));
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwkSetLoaderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwkSetLoaderTests.java
new file mode 100644
index 000000000000..5f095dd0920c
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwkSetLoaderTests.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authc.jwt;
+
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.authc.RealmConfig;
+import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.IntStream;
+
+import static org.elasticsearch.test.ActionListenerUtils.anyActionListener;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class JwkSetLoaderTests extends ESTestCase {
+
+    public void testConcurrentReloadWillBeQueuedAndShareTheResults() throws IOException, InterruptedException {
+        final Path tempDir = createTempDir();
+        final Path path = tempDir.resolve("jwkset.json");
+        Files.write(path, List.of("{\"keys\":[]}"), StandardCharsets.UTF_8);
+        final RealmConfig realmConfig = mock(RealmConfig.class);
+        when(realmConfig.getSetting(JwtRealmSettings.PKC_JWKSET_PATH)).thenReturn("jwkset.json");
+        final Environment env = mock(Environment.class);
+        when(env.configFile()).thenReturn(tempDir);
+        when(realmConfig.env()).thenReturn(env);
+
+        final JwkSetLoader jwkSetLoader = spy(new JwkSetLoader(realmConfig, List.of(), null));
+
+        final int nThreads = randomIntBetween(2, 8);
+        final var futures = IntStream.range(0, nThreads)
+            .mapToObj(i -> new PlainActionFuture<Tuple<Boolean, JwkSetLoader.JwksAlgs>>())
+            .toList();
+
+        // Start the first thread for reloading
+        // Ensure it is inside the actual loading method and make it wait there to simulate slow processing
+        final var loadingLatch = new CountDownLatch(1);
+        final var readyLatch = new CountDownLatch(1);
+        doAnswer(invocation -> {
+            loadingLatch.countDown();
+            assertThat(readyLatch.await(10, TimeUnit.SECONDS), is(true));
+            invocation.callRealMethod();
+            return null;
+        }).when(jwkSetLoader).loadInternal(anyActionListener());
+
+        new Thread(() -> jwkSetLoader.reload(futures.get(0))).start();
+        assertThat(loadingLatch.await(10, TimeUnit.SECONDS), is(true));
+
+        // Start rest of the threads for racing and ensure they are all through the concurrency controlling area
+        Mockito.reset(jwkSetLoader);
+        final var threadsCountDown = new CountDownLatch(nThreads - 1);
+        doAnswer(invocation -> {
+            final Object result = invocation.callRealMethod();
+            threadsCountDown.countDown();
+            return result;
+        }).when(jwkSetLoader).getFuture();
+        IntStream.range(1, nThreads).forEach(i -> new Thread(() -> jwkSetLoader.reload(futures.get(i))).start());
+        assertThat(threadsCountDown.await(10, TimeUnit.SECONDS), is(true));
+
+        // Notify the first thread to finish the loading work
+        readyLatch.countDown();
+
+        // All concurrent reloading calls will get the same result from the first thread and skip the actual loading work
+        final Tuple<Boolean, JwkSetLoader.JwksAlgs> tuple = futures.get(0).actionGet();
+        futures.subList(1, nThreads).forEach(future -> assertThat(future.actionGet(), sameInstance(tuple)));
+        verify(jwkSetLoader, never()).loadInternal(anyActionListener());
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwkValidateUtilTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwkValidateUtilTests.java
index c260989df944..20f426cf3a8d 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwkValidateUtilTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwkValidateUtilTests.java
@@ -11,8 +11,6 @@
 import com.nimbusds.jose.jwk.JWK;
 import com.nimbusds.jose.jwk.OctetSequenceKey;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings;
 
 import java.nio.charset.StandardCharsets;
@@ -22,12 +20,9 @@
 
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
 
 public class JwkValidateUtilTests extends JwtTestCase {
 
-    private static final Logger LOGGER = LogManager.getLogger(JwkValidateUtilTests.class);
-
     // Test decode bytes as UTF8 to String, encode back to UTF8, and compare to original bytes. If same, it is safe for OIDC JWK encode.
     static boolean isJwkHmacOidcSafe(final JWK jwk) {
         if (jwk instanceof OctetSequenceKey jwkHmac) {
@@ -50,28 +45,26 @@ public void testComputeBitLengthRsa() throws Exception {
         for (final String signatureAlgorithmRsa : JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_RSA) {
             final JWK jwk = JwtTestCase.randomJwkRsa(JWSAlgorithm.parse(signatureAlgorithmRsa));
             final int minLength = JwkValidateUtil.computeBitLengthRsa(jwk.toRSAKey().toPublicKey());
-            assertThat(minLength, is(anyOf(equalTo(2048), equalTo(3072))));
+            assertThat(minLength, anyOf(equalTo(2048), equalTo(3072)));
         }
     }
 
     public void testAlgsJwksAllNotFiltered() throws Exception {
-        this.filterJwksAndAlgorithmsTestHelper(JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS);
+        filterJwksAndAlgorithmsTestHelper(JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS);
     }
 
     public void testAlgsJwksAllHmacNotFiltered() throws Exception {
-        this.filterJwksAndAlgorithmsTestHelper(JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_HMAC);
+        filterJwksAndAlgorithmsTestHelper(JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_HMAC);
     }
 
     public void testAlgsJwksAllPkcNotFiltered() throws Exception {
-        this.filterJwksAndAlgorithmsTestHelper(JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_PKC);
+        filterJwksAndAlgorithmsTestHelper(JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_PKC);
     }
 
     private void filterJwksAndAlgorithmsTestHelper(final List<String> candidateAlgs) throws JOSEException {
         final List<String> algsRandom = randomOfMinUnique(2, candidateAlgs); // duplicates allowed
         final List<JwtIssuer.AlgJwkPair> algJwkPairsAll = JwtTestCase.randomJwks(algsRandom, randomBoolean());
         final List<JWK> jwks = algJwkPairsAll.stream().map(JwtIssuer.AlgJwkPair::jwk).toList();
-        final List<String> algsAll = algJwkPairsAll.stream().map(JwtIssuer.AlgJwkPair::alg).toList();
-        final List<JWK> jwksAll = algJwkPairsAll.stream().map(JwtIssuer.AlgJwkPair::jwk).toList();
 
         // verify no filtering
         final JwkSetLoader.JwksAlgs nonFiltered = JwkValidateUtil.filterJwksAndAlgorithms(jwks, algsRandom);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtAlgorithmValidatorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtAlgorithmValidatorTests.java
index 355d39b2d9d8..516f608dad95 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtAlgorithmValidatorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtAlgorithmValidatorTests.java
@@ -10,7 +10,6 @@
 import com.nimbusds.jose.JWSHeader;
 import com.nimbusds.jwt.JWTClaimsSet;
 
-import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.test.ESTestCase;
 
 import java.text.ParseException;
@@ -39,8 +38,8 @@ public void testMismatchingAlgorithm() throws ParseException {
         final JwtAlgorithmValidator validator = new JwtAlgorithmValidator(randomList(1, 5, () -> randomAlphaOfLength(8)));
 
         final JWSHeader jwsHeader = JWSHeader.parse(Map.of("alg", algorithm));
-        final ElasticsearchSecurityException e = expectThrows(
-            ElasticsearchSecurityException.class,
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
             () -> validator.validate(jwsHeader, JWTClaimsSet.parse(Map.of()))
         );
         assertThat(e.getMessage(), containsString("invalid JWT algorithm"));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticationTokenTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticationTokenTests.java
deleted file mode 100644
index 120040b30444..000000000000
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticationTokenTests.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.security.authc.jwt;
-
-import com.nimbusds.jose.JOSEObjectType;
-import com.nimbusds.jose.jwk.JWK;
-import com.nimbusds.jwt.SignedJWT;
-
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings;
-import org.junit.Assert;
-
-import java.time.Instant;
-import java.time.temporal.ChronoUnit;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.nullValue;
-
-public class JwtAuthenticationTokenTests extends JwtTestCase {
-
-    public void testJwtAuthenticationTokenParse() throws Exception {
-        final String signatureAlgorithm = randomFrom(JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS);
-        final JWK jwk = JwtTestCase.randomJwk(signatureAlgorithm, randomBoolean());
-
-        final SecureString jwt = JwtTestCase.randomBespokeJwt(jwk, signatureAlgorithm); // bespoke JWT, not tied to any JWT realm
-        final SecureString clientSharedSecret = randomBoolean() ? null : new SecureString(randomAlphaOfLengthBetween(10, 20).toCharArray());
-
-        final List<String> principalClaimNames = List.of(randomAlphaOfLength(4), "sub", randomAlphaOfLength(4));
-        final JwtAuthenticationToken jwtAuthenticationToken = new JwtAuthenticationToken(principalClaimNames, jwt, clientSharedSecret);
-        final SecureString endUserSignedJwt = jwtAuthenticationToken.getEndUserSignedJwt();
-        final SecureString clientAuthenticationSharedSecret = jwtAuthenticationToken.getClientAuthenticationSharedSecret();
-
-        Assert.assertEquals(jwt, endUserSignedJwt);
-        Assert.assertEquals(clientSharedSecret, clientAuthenticationSharedSecret);
-
-        jwtAuthenticationToken.clearCredentials();
-
-        // verify references to SecureString throw exception when calling their methods
-        final Exception exception1 = expectThrows(IllegalStateException.class, endUserSignedJwt::length);
-        assertThat(exception1.getMessage(), equalTo("SecureString has already been closed"));
-        if (clientAuthenticationSharedSecret != null) {
-            final Exception exception2 = expectThrows(IllegalStateException.class, clientAuthenticationSharedSecret::length);
-            assertThat(exception2.getMessage(), equalTo("SecureString has already been closed"));
-        }
-
-        // verify token returns nulls
-        assertThat(jwtAuthenticationToken.principal(), is(nullValue()));
-        assertThat(jwtAuthenticationToken.credentials(), is(nullValue()));
-        assertThat(jwtAuthenticationToken.getEndUserSignedJwt(), is(nullValue()));
-        assertThat(jwtAuthenticationToken.getClientAuthenticationSharedSecret(), is(nullValue()));
-    }
-
-    public void testPrincipalForJwtWithoutSub() throws Exception {
-        final String issuer = randomAlphaOfLengthBetween(8, 24);
-        final String audience = randomAlphaOfLengthBetween(6, 12);
-
-        final String principalClaimName = randomValueOtherThan("sub", () -> randomAlphaOfLength(3));
-        final String principalClaimValue = randomAlphaOfLengthBetween(8, 32);
-
-        final String signatureAlgorithm = randomFrom(JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS);
-        final JWK jwk = JwtTestCase.randomJwk(signatureAlgorithm, randomBoolean());
-
-        final Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
-        final SignedJWT unsignedJwt = JwtTestCase.buildUnsignedJwt(
-            randomBoolean() ? null : JOSEObjectType.JWT.toString(), // kty
-            randomBoolean() ? null : jwk.getKeyID(), // kid
-            signatureAlgorithm, // alg
-            null, // jwtID
-            issuer, // iss
-            List.of(audience), // aud
-            null, // sub claim value
-            principalClaimName, // principal claim name
-            principalClaimValue, // principal claim value
-            null, // groups claim
-            List.of(), // groups
-            Date.from(now.minusSeconds(60 * randomLongBetween(10, 20))), // auth_time
-            Date.from(now.minusSeconds(randomBoolean() ? 0 : 60 * randomLongBetween(5, 10))), // iat
-            Date.from(now), // nbf
-            Date.from(now.plusSeconds(60 * randomLongBetween(3600, 7200))), // exp
-            null, // nonce
-            Map.of() // other claims
-        );
-        final SecureString jwt = JwtValidateUtil.signJwt(jwk, unsignedJwt);
-
-        final List<String> principalClaimNames = List.of(randomAlphaOfLength(4), "sub", principalClaimName);
-        final JwtAuthenticationToken jwtAuthenticationToken = new JwtAuthenticationToken(principalClaimNames, jwt, null);
-        Assert.assertEquals(issuer + "/" + audience + "/" + principalClaimValue, jwtAuthenticationToken.principal());
-    }
-}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticatorAccessTokenTypeTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticatorAccessTokenTypeTests.java
new file mode 100644
index 000000000000..40bf021a48a9
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticatorAccessTokenTypeTests.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authc.jwt;
+
+import org.elasticsearch.xpack.core.security.authc.RealmSettings;
+import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings;
+
+import java.text.ParseException;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class JwtAuthenticatorAccessTokenTypeTests extends JwtAuthenticatorTests {
+
+    @Override
+    protected JwtRealmSettings.TokenType getTokenType() {
+        return JwtRealmSettings.TokenType.ACCESS_TOKEN;
+    }
+
+    public void testSubjectIsRequired() throws ParseException {
+        final IllegalArgumentException e = doTestSubjectIsRequired(buildJwtAuthenticator());
+        if (fallbackSub != null) {
+            assertThat(e.getMessage(), containsString("missing required string claim [" + fallbackSub + " (fallback of sub)]"));
+        }
+    }
+
+    public void testAccessTokenTypeMandatesAllowedSubjects() {
+        allowedSubject = null;
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> buildJwtAuthenticator());
+
+        assertThat(
+            e.getMessage(),
+            containsString("Invalid empty list for [" + RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.ALLOWED_SUBJECTS) + "]")
+        );
+    }
+
+    public void testInvalidIssuerIsCheckedBeforeAlgorithm() throws ParseException {
+        doTestInvalidIssuerIsCheckedBeforeAlgorithm(buildJwtAuthenticator());
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticatorIdTokenTypeTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticatorIdTokenTypeTests.java
new file mode 100644
index 000000000000..c4e4cd6e6de6
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticatorIdTokenTypeTests.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authc.jwt;
+
+import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings;
+
+import java.text.ParseException;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class JwtAuthenticatorIdTokenTypeTests extends JwtAuthenticatorTests {
+
+    @Override
+    protected JwtRealmSettings.TokenType getTokenType() {
+        return JwtRealmSettings.TokenType.ID_TOKEN;
+    }
+
+    public void testSubjectIsRequired() throws ParseException {
+        final IllegalArgumentException e = doTestSubjectIsRequired(buildJwtAuthenticator());
+        assertThat(e.getMessage(), containsString("missing required string claim [sub]"));
+    }
+
+    public void testInvalidIssuerIsCheckedBeforeAlgorithm() throws ParseException {
+        doTestInvalidIssuerIsCheckedBeforeAlgorithm(buildJwtAuthenticator());
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticatorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticatorTests.java
index 0f22070458c5..4d1673210423 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticatorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticatorTests.java
@@ -12,49 +12,225 @@
 import com.nimbusds.jwt.JWTClaimsSet;
 import com.nimbusds.jwt.SignedJWT;
 
-import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.settings.MockSecureSettings;
-import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
 import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings;
+import org.junit.Before;
 
 import java.text.ParseException;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
+import static org.elasticsearch.test.ActionListenerUtils.anyActionListener;
 import static org.elasticsearch.test.TestMatchers.throwableWithMessage;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
 
-public class JwtAuthenticatorTests extends ESTestCase {
+public abstract class JwtAuthenticatorTests extends ESTestCase {
 
-    public void testInvalidIssuerIsCheckedBeforeAlgorithm() throws ParseException {
-        final String realmName = randomAlphaOfLengthBetween(3, 8);
-        final String allowedIssuer = randomAlphaOfLength(6);
-        final String allowedAlgorithm = randomFrom(JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_HMAC);
-        final JwtAuthenticator jwtAuthenticator = buildJwtAuthenticator(realmName, allowedAlgorithm, allowedIssuer);
+    protected String realmName;
+    protected String allowedAlgorithm;
+    protected String allowedIssuer;
+    @Nullable
+    protected String allowedSubject;
+    protected String allowedAudience;
+    protected String fallbackSub;
+    protected String fallbackAud;
+    protected Tuple<String, List<String>> requiredClaim;
 
+    protected abstract JwtRealmSettings.TokenType getTokenType();
+
+    @Before
+    public void beforeTest() {
+        realmName = randomAlphaOfLengthBetween(3, 8);
+        allowedIssuer = randomAlphaOfLength(6);
+        allowedAlgorithm = randomFrom(JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_HMAC);
+        if (getTokenType() == JwtRealmSettings.TokenType.ID_TOKEN) {
+            allowedSubject = randomBoolean() ? randomAlphaOfLength(8) : null;
+            fallbackSub = null;
+            fallbackAud = null;
+        } else {
+            allowedSubject = randomAlphaOfLength(8);
+            fallbackSub = randomBoolean() ? "_" + randomAlphaOfLength(5) : null;
+            fallbackAud = randomBoolean() ? "_" + randomAlphaOfLength(8) : null;
+        }
+        allowedAudience = randomAlphaOfLength(10);
+        requiredClaim = Tuple.tuple(randomAlphaOfLength(8), randomList(1, 3, () -> randomAlphaOfLengthBetween(8, 18)));
+    }
+
+    public void testRequiredClaims() throws ParseException {
+        final Instant now = Instant.now();
+        final String requireClaimValue = randomFrom(requiredClaim.v2());
+        final JWTClaimsSet claimsSet = JWTClaimsSet.parse(
+            Map.of(
+                "iss",
+                allowedIssuer,
+                "sub",
+                allowedSubject == null ? randomAlphaOfLengthBetween(10, 18) : allowedSubject,
+                "aud",
+                allowedAudience,
+                requiredClaim.v1(),
+                randomBoolean() ? requireClaimValue : List.of(requireClaimValue, "some-other-value"),
+                "iat",
+                now.minus(1, ChronoUnit.DAYS).getEpochSecond(),
+                "exp",
+                now.plus(1, ChronoUnit.DAYS).getEpochSecond()
+            )
+        );
+        final SignedJWT signedJWT = new SignedJWT(
+            JWSHeader.parse(Map.of("alg", allowedAlgorithm)).toBase64URL(),
+            claimsSet.toPayload().toBase64URL(),
+            Base64URL.encode("signature")
+        );
+
+        final JwtAuthenticationToken jwtAuthenticationToken = mock(JwtAuthenticationToken.class);
+        when(jwtAuthenticationToken.getSignedJWT()).thenReturn(signedJWT);
+        when(jwtAuthenticationToken.getJWTClaimsSet()).thenReturn(signedJWT.getJWTClaimsSet());
+
+        final PlainActionFuture<JWTClaimsSet> future = new PlainActionFuture<>();
+        final JwtAuthenticator jwtAuthenticator = buildJwtAuthenticator();
+        jwtAuthenticator.authenticate(jwtAuthenticationToken, future);
+        assertThat(future.actionGet(), equalTo(claimsSet));
+    }
+
+    public void testMismatchedRequiredClaims() throws ParseException {
+        final Instant now = Instant.now();
+        final String mismatchRequiredClaimValue = randomValueOtherThanMany(
+            requiredClaim.v2()::contains,
+            () -> randomAlphaOfLengthBetween(3, 18)
+        );
+        final JWTClaimsSet claimsSet = JWTClaimsSet.parse(
+            Map.of(
+                "iss",
+                allowedIssuer,
+                "sub",
+                allowedSubject == null ? randomAlphaOfLengthBetween(10, 18) : allowedSubject,
+                "aud",
+                allowedAudience,
+                requiredClaim.v1(),
+                mismatchRequiredClaimValue,
+                "iat",
+                now.minus(1, ChronoUnit.DAYS).getEpochSecond(),
+                "exp",
+                now.plus(1, ChronoUnit.DAYS).getEpochSecond()
+            )
+        );
+        final SignedJWT signedJWT = new SignedJWT(
+            JWSHeader.parse(Map.of("alg", allowedAlgorithm)).toBase64URL(),
+            claimsSet.toPayload().toBase64URL(),
+            Base64URL.encode("signature")
+        );
+
+        final JwtAuthenticationToken jwtAuthenticationToken = mock(JwtAuthenticationToken.class);
+        when(jwtAuthenticationToken.getSignedJWT()).thenReturn(signedJWT);
+        when(jwtAuthenticationToken.getJWTClaimsSet()).thenReturn(signedJWT.getJWTClaimsSet());
+
+        final PlainActionFuture<JWTClaimsSet> future = new PlainActionFuture<>();
+        final JwtAuthenticator jwtAuthenticator = buildJwtAuthenticator();
+        jwtAuthenticator.authenticate(jwtAuthenticationToken, future);
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, future::actionGet);
+        assertThat(
+            e.getMessage(),
+            containsString(
+                "string claim ["
+                    + requiredClaim.v1()
+                    + "] has value ["
+                    + mismatchRequiredClaimValue
+                    + "] which does not match allowed claim values ["
+                    + requiredClaim.v2().stream().collect(Collectors.joining(","))
+                    + "]"
+            )
+        );
+    }
+
+    public void testMissingRequiredClaims() throws ParseException {
+        final Instant now = Instant.now();
+        final JWTClaimsSet claimsSet = JWTClaimsSet.parse(
+            Map.of(
+                "iss",
+                allowedIssuer,
+                "sub",
+                allowedSubject == null ? randomAlphaOfLengthBetween(10, 18) : allowedSubject,
+                "aud",
+                allowedAudience,
+                "iat",
+                now.minus(1, ChronoUnit.DAYS).getEpochSecond(),
+                "exp",
+                now.plus(1, ChronoUnit.DAYS).getEpochSecond()
+            )
+        );
+        final SignedJWT signedJWT = new SignedJWT(
+            JWSHeader.parse(Map.of("alg", allowedAlgorithm)).toBase64URL(),
+            claimsSet.toPayload().toBase64URL(),
+            Base64URL.encode("signature")
+        );
+
+        final JwtAuthenticationToken jwtAuthenticationToken = mock(JwtAuthenticationToken.class);
+        when(jwtAuthenticationToken.getSignedJWT()).thenReturn(signedJWT);
+        when(jwtAuthenticationToken.getJWTClaimsSet()).thenReturn(signedJWT.getJWTClaimsSet());
+
+        // Required claim is mandatory when configured
+        final PlainActionFuture<JWTClaimsSet> future1 = new PlainActionFuture<>();
+        buildJwtAuthenticator().authenticate(jwtAuthenticationToken, future1);
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, future1::actionGet);
+        assertThat(e.getMessage(), containsString("missing required string claim [" + requiredClaim.v1() + "]"));
+
+        // Remove required claim from settings, the JWT now works
+        requiredClaim = null;
+        final PlainActionFuture<JWTClaimsSet> future2 = new PlainActionFuture<>();
+        buildJwtAuthenticator().authenticate(jwtAuthenticationToken, future2);
+        assertThat(future2.actionGet(), equalTo(claimsSet));
+    }
+
+    protected IllegalArgumentException doTestSubjectIsRequired(JwtAuthenticator jwtAuthenticator) throws ParseException {
+        final SignedJWT signedJWT = new SignedJWT(
+            JWSHeader.parse(Map.of("alg", allowedAlgorithm)).toBase64URL(),
+            JWTClaimsSet.parse(Map.of("iss", allowedIssuer)).toPayload().toBase64URL(),
+            Base64URL.encode("signature")
+        );
+        final JwtAuthenticationToken jwtAuthenticationToken = mock(JwtAuthenticationToken.class);
+        when(jwtAuthenticationToken.getSignedJWT()).thenReturn(signedJWT);
+        when(jwtAuthenticationToken.getJWTClaimsSet()).thenReturn(signedJWT.getJWTClaimsSet());
+
+        final PlainActionFuture<JWTClaimsSet> future = new PlainActionFuture<>();
+        jwtAuthenticator.authenticate(jwtAuthenticationToken, future);
+        return expectThrows(IllegalArgumentException.class, future::actionGet);
+    }
+
+    protected void doTestInvalidIssuerIsCheckedBeforeAlgorithm(JwtAuthenticator jwtAuthenticator) throws ParseException {
         // A JWT token that has mismatch for both algorithm and issuer
         final String invalidAlgorithm = randomValueOtherThan(allowedAlgorithm, () -> randomAlphaOfLengthBetween(3, 8));
         final String invalidIssuer = randomValueOtherThan(allowedIssuer, () -> randomAlphaOfLengthBetween(3, 8));
         final SignedJWT signedJWT = new SignedJWT(
             JWSHeader.parse(Map.of("alg", invalidAlgorithm)).toBase64URL(),
-            JWTClaimsSet.parse(Map.of("iss", invalidIssuer)).toPayload().toBase64URL(),
+            JWTClaimsSet.parse(Map.of("iss", invalidIssuer, "sub", randomAlphaOfLengthBetween(3, 8))).toPayload().toBase64URL(),
             Base64URL.encode("signature")
         );
         final JwtAuthenticationToken jwtAuthenticationToken = mock(JwtAuthenticationToken.class);
-        when(jwtAuthenticationToken.getEndUserSignedJwt()).thenReturn(new SecureString(signedJWT.serialize().toCharArray()));
+        when(jwtAuthenticationToken.getSignedJWT()).thenReturn(signedJWT);
+        when(jwtAuthenticationToken.getJWTClaimsSet()).thenReturn(signedJWT.getJWTClaimsSet());
 
         final PlainActionFuture<JWTClaimsSet> future = new PlainActionFuture<>();
         jwtAuthenticator.authenticate(jwtAuthenticationToken, future);
 
-        final ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, future::actionGet);
-
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, future::actionGet);
         assertThat(
             e,
             throwableWithMessage(
@@ -63,20 +239,47 @@ public void testInvalidIssuerIsCheckedBeforeAlgorithm() throws ParseException {
         );
     }
 
-    private JwtAuthenticator buildJwtAuthenticator(String realmName, String allowedAlgorithm, String allowedIssuer) {
+    protected JwtAuthenticator buildJwtAuthenticator() {
         final RealmConfig.RealmIdentifier realmIdentifier = new RealmConfig.RealmIdentifier(JwtRealmSettings.TYPE, realmName);
         final MockSecureSettings secureSettings = new MockSecureSettings();
         secureSettings.setString(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.HMAC_KEY), randomAlphaOfLength(40));
         final Settings.Builder builder = Settings.builder()
             .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.ALLOWED_SIGNATURE_ALGORITHMS), allowedAlgorithm)
             .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.ALLOWED_ISSUER), allowedIssuer)
-            .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.ALLOWED_AUDIENCES), randomAlphaOfLength(7))
+            .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.ALLOWED_AUDIENCES), allowedAudience)
             .put(RealmSettings.getFullSettingKey(realmIdentifier, RealmSettings.ORDER_SETTING), randomIntBetween(0, 99))
             .put("path.home", randomAlphaOfLength(10))
             .setSecureSettings(secureSettings);
-        if (randomBoolean()) {
-            builder.put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.TOKEN_TYPE), "id_token");
+
+        if (allowedSubject != null) {
+            builder.put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.ALLOWED_SUBJECTS), allowedSubject);
+        }
+
+        if (getTokenType() == JwtRealmSettings.TokenType.ID_TOKEN) {
+            if (randomBoolean()) {
+                builder.put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.TOKEN_TYPE), "id_token");
+            }
+        } else {
+            builder.put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.TOKEN_TYPE), "access_token");
         }
+
+        if (fallbackSub != null) {
+            builder.put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.FALLBACK_SUB_CLAIM), fallbackSub);
+        }
+        if (fallbackAud != null) {
+            builder.put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.FALLBACK_AUD_CLAIM), fallbackAud);
+        }
+
+        if (requiredClaim != null) {
+            final String requiredClaimsKey = RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.REQUIRED_CLAIMS) + requiredClaim
+                .v1();
+            if (requiredClaim.v2().size() == 1 && randomBoolean()) {
+                builder.put(requiredClaimsKey, requiredClaim.v2().get(0));
+            } else {
+                builder.putList(requiredClaimsKey, requiredClaim.v2());
+            }
+        }
+
         final Settings settings = builder.build();
 
         final RealmConfig realmConfig = new RealmConfig(
@@ -86,6 +289,14 @@ private JwtAuthenticator buildJwtAuthenticator(String realmName, String allowedA
             new ThreadContext(settings)
         );
 
-        return new JwtAuthenticator(realmConfig, null, () -> {});
+        final JwtAuthenticator jwtAuthenticator = spy(new JwtAuthenticator(realmConfig, null, () -> {}));
+        // Short circuit signature validation to be always successful since this test class does not test it
+        doAnswer(invocation -> {
+            final ActionListener<Void> listener = invocation.getArgument(2);
+            listener.onResponse(null);
+            return null;
+        }).when(jwtAuthenticator).validateSignature(any(), any(), anyActionListener());
+
+        return jwtAuthenticator;
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtDateClaimValidatorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtDateClaimValidatorTests.java
index e27e9e1ab027..7bb60fca9c23 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtDateClaimValidatorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtDateClaimValidatorTests.java
@@ -10,7 +10,6 @@
 import com.nimbusds.jose.JWSHeader;
 import com.nimbusds.jwt.JWTClaimsSet;
 
-import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
@@ -44,8 +43,8 @@ public void testClaimIsNotDate() throws ParseException {
         );
 
         final JWTClaimsSet jwtClaimsSet = JWTClaimsSet.parse(Map.of(claimName, randomAlphaOfLengthBetween(3, 8)));
-        final ElasticsearchSecurityException e = expectThrows(
-            ElasticsearchSecurityException.class,
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
             () -> validator.validate(getJwsHeader(), jwtClaimsSet)
         );
         assertThat(e.getMessage(), containsString("cannot parse date claim"));
@@ -64,8 +63,8 @@ public void testClaimDoesNotExist() throws ParseException {
         );
 
         final JWTClaimsSet jwtClaimsSet = JWTClaimsSet.parse(Map.of());
-        final ElasticsearchSecurityException e = expectThrows(
-            ElasticsearchSecurityException.class,
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
             () -> validator.validate(getJwsHeader(), jwtClaimsSet)
         );
         assertThat(e.getMessage(), containsString("missing required date claim"));
@@ -112,8 +111,8 @@ public void testBeforeNow() throws ParseException {
         }
 
         final Instant after = now.plusSeconds(randomLongBetween(1 + allowedSkewInSeconds, 600));
-        final ElasticsearchSecurityException e = expectThrows(
-            ElasticsearchSecurityException.class,
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
             () -> validator.validate(getJwsHeader(), JWTClaimsSet.parse(Map.of(claimName, after.getEpochSecond())))
         );
         assertThat(
@@ -145,8 +144,8 @@ public void testAfterNow() throws ParseException {
         when(clock.instant()).thenReturn(now);
 
         final Instant before = now.minusSeconds(randomLongBetween(1 + allowedSkewInSeconds, 600));
-        final ElasticsearchSecurityException e = expectThrows(
-            ElasticsearchSecurityException.class,
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
             () -> validator.validate(getJwsHeader(), JWTClaimsSet.parse(Map.of(claimName, before.getEpochSecond())))
         );
         assertThat(
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtIssuer.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtIssuer.java
index 008f48681855..789ac04c4062 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtIssuer.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtIssuer.java
@@ -23,7 +23,9 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.test.ESTestCase.randomAlphaOfLengthBetween;
 import static org.elasticsearch.test.ESTestCase.randomBoolean;
+import static org.elasticsearch.test.ESTestCase.randomFrom;
 
 /**
  * Test class with settings for a JWT issuer to sign JWTs for users.
@@ -37,7 +39,7 @@ record AlgJwkPair(String alg, JWK jwk) {}
     // input parameters
     final String issuerClaimValue; // claim name is hard-coded to `iss` for OIDC ID Token compatibility
     final List<String> audiencesClaimValue; // claim name is hard-coded to `aud` for OIDC ID Token compatibility
-    final String principalClaimName; // claim name is configurable, EX: Users (sub, oid, email, dn, uid), Clients (azp, appid, client_id)
+    final String principalClaimName;
     final Map<String, User> principals; // principals with roles, for sending encoded JWTs into JWT realms for authc/authz verification
     final JwtIssuerHttpsServer httpsServer;
 
@@ -56,57 +58,52 @@ record AlgJwkPair(String alg, JWK jwk) {}
     JwtIssuer(
         final String issuerClaimValue,
         final List<String> audiencesClaimValue,
-        final String principalClaimName,
         final Map<String, User> principals,
         final boolean createHttpsServer
     ) throws Exception {
         this.issuerClaimValue = issuerClaimValue;
         this.audiencesClaimValue = audiencesClaimValue;
-        this.principalClaimName = principalClaimName;
         this.principals = principals;
         this.httpsServer = createHttpsServer ? new JwtIssuerHttpsServer(null) : null;
+        this.principalClaimName = randomFrom("sub", "oid", "client_id", "appid", "azp", "email", randomAlphaOfLengthBetween(12, 18));
     }
 
     // The flag areHmacJwksOidcSafe indicates if all provided HMAC JWKs are UTF8, for HMAC OIDC JWK encoding compatibility.
     void setJwks(final List<AlgJwkPair> algAndJwks, final boolean areHmacJwksOidcSafe) throws JOSEException {
-        this.algorithmsAll = algAndJwks.stream().map(e -> e.alg).toList();
-        LOGGER.info("Setting JWKs: algorithms=[{}], areHmacJwksOidcSafe=[{}]", String.join(",", this.algorithmsAll), areHmacJwksOidcSafe);
-        this.algAndJwksAll = algAndJwks;
-        this.algAndJwksPkc = this.algAndJwksAll.stream()
-            .filter(e -> JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_PKC.contains(e.alg))
-            .toList();
-        this.algAndJwksHmac = this.algAndJwksAll.stream()
-            .filter(e -> JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_HMAC.contains(e.alg))
-            .toList();
-        if ((this.algAndJwksHmac.size() == 1) && (areHmacJwksOidcSafe) && (randomBoolean())) {
-            this.algAndJwkHmacOidc = this.algAndJwksHmac.get(0);
-            this.algAndJwksHmac = Collections.emptyList();
+        algorithmsAll = algAndJwks.stream().map(e -> e.alg).toList();
+        LOGGER.info("Setting JWKs: algorithms=[{}], areHmacJwksOidcSafe=[{}]", String.join(",", algorithmsAll), areHmacJwksOidcSafe);
+        algAndJwksAll = algAndJwks;
+        algAndJwksPkc = algAndJwksAll.stream().filter(e -> JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_PKC.contains(e.alg)).toList();
+        algAndJwksHmac = algAndJwksAll.stream().filter(e -> JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_HMAC.contains(e.alg)).toList();
+        if ((algAndJwksHmac.size() == 1) && (areHmacJwksOidcSafe) && (randomBoolean())) {
+            algAndJwkHmacOidc = algAndJwksHmac.get(0);
+            algAndJwksHmac = Collections.emptyList();
         } else {
-            this.algAndJwkHmacOidc = null;
+            algAndJwkHmacOidc = null;
         }
 
         // Encode PKC JWKSet (key material bytes are wrapped in Base64URL, and then wraps in JSON)
-        final JWKSet jwkSetPkc = new JWKSet(this.algAndJwksPkc.stream().map(p -> p.jwk).toList());
-        this.encodedJwkSetPkcPublicPrivate = JwtUtil.serializeJwkSet(jwkSetPkc, false);
-        this.encodedJwkSetPkcPublic = JwtUtil.serializeJwkSet(jwkSetPkc, true);
+        final JWKSet jwkSetPkc = new JWKSet(algAndJwksPkc.stream().map(p -> p.jwk).toList());
+        encodedJwkSetPkcPublicPrivate = JwtUtil.serializeJwkSet(jwkSetPkc, false);
+        encodedJwkSetPkcPublic = JwtUtil.serializeJwkSet(jwkSetPkc, true);
 
         // Encode HMAC JWKSet (key material bytes are wrapped in Base64URL, and then wraps in JSON)
-        final JWKSet jwkSetHmac = new JWKSet(this.algAndJwksHmac.stream().map(p -> p.jwk).toList());
-        this.encodedJwkSetHmac = JwtUtil.serializeJwkSet(jwkSetHmac, false);
+        final JWKSet jwkSetHmac = new JWKSet(algAndJwksHmac.stream().map(p -> p.jwk).toList());
+        encodedJwkSetHmac = JwtUtil.serializeJwkSet(jwkSetHmac, false);
 
         // Encode HMAC OIDC JWK (key material bytes are decoded from UTF8 to UNICODE String)
-        this.encodedKeyHmacOidc = (algAndJwkHmacOidc == null) ? null : JwtUtil.serializeJwkHmacOidc(this.algAndJwkHmacOidc.jwk);
+        encodedKeyHmacOidc = (algAndJwkHmacOidc == null) ? null : JwtUtil.serializeJwkHmacOidc(algAndJwkHmacOidc.jwk);
 
-        if (this.httpsServer != null) {
-            this.httpsServer.updateJwkSetPkcContents(this.encodedJwkSetPkcPublic.getBytes(StandardCharsets.UTF_8));
+        if (httpsServer != null) {
+            httpsServer.updateJwkSetPkcContents(encodedJwkSetPkcPublic.getBytes(StandardCharsets.UTF_8));
         }
     }
 
     @Override
     public void close() {
-        if (this.httpsServer != null) {
+        if (httpsServer != null) {
             try {
-                this.httpsServer.close();
+                httpsServer.close();
             } catch (IOException e) {
                 LOGGER.warn("Exception closing HTTPS server for issuer [" + issuerClaimValue + "]", e);
             }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtIssuerHttpsServer.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtIssuerHttpsServer.java
index 96be8c784c6b..4aac7f83c45f 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtIssuerHttpsServer.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtIssuerHttpsServer.java
@@ -69,16 +69,16 @@ public JwtIssuerHttpsServer(final byte[] encodedJwkSetPkcPublicBytes) throws Exc
     }
 
     public void updateJwkSetPkcContents(final byte[] encodedJwkSetPkcPublicBytes) {
-        this.httpsServer.removeContext(PATH);
-        this.httpsServer.createContext(PATH, new JwtIssuerHttpHandler(encodedJwkSetPkcPublicBytes));
+        httpsServer.removeContext(PATH);
+        httpsServer.createContext(PATH, new JwtIssuerHttpHandler(encodedJwkSetPkcPublicBytes));
     }
 
     @Override
     public void close() throws IOException {
-        if (this.httpsServer != null) {
-            LOGGER.trace("Stopping [{}]", this.url);
-            this.httpsServer.stop(STOP_DELAY_SECONDS);
-            LOGGER.debug("Stopped [{}]", this.url);
+        if (httpsServer != null) {
+            LOGGER.trace("Stopping [{}]", url);
+            httpsServer.stop(STOP_DELAY_SECONDS);
+            LOGGER.debug("Stopped [{}]", url);
         }
     }
 
@@ -100,8 +100,8 @@ public void handle(final HttpExchange httpExchange) throws IOException {
                     if (encodedJwkSetPkcPublicBytes == null) {
                         httpExchange.sendResponseHeaders(HttpURLConnection.HTTP_NOT_FOUND, 0);
                     } else {
-                        httpExchange.sendResponseHeaders(HttpURLConnection.HTTP_OK, this.encodedJwkSetPkcPublicBytes.length);
-                        os.write(this.encodedJwkSetPkcPublicBytes);
+                        httpExchange.sendResponseHeaders(HttpURLConnection.HTTP_OK, encodedJwkSetPkcPublicBytes.length);
+                        os.write(encodedJwkSetPkcPublicBytes);
                     }
                 }
                 LOGGER.trace("Response: [{}]", path); // Confirm client didn't disconnect before flush
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmAuthenticateAccessTokenTypeTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmAuthenticateAccessTokenTypeTests.java
new file mode 100644
index 000000000000..c242315628ae
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmAuthenticateAccessTokenTypeTests.java
@@ -0,0 +1,168 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authc.jwt;
+
+import com.nimbusds.jose.JOSEObjectType;
+import com.nimbusds.jose.jwk.JWK;
+import com.nimbusds.jwt.SignedJWT;
+import com.nimbusds.openid.connect.sdk.Nonce;
+
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.core.security.authc.RealmSettings;
+import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings;
+import org.elasticsearch.xpack.core.security.user.User;
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class JwtRealmAuthenticateAccessTokenTypeTests extends JwtRealmTestCase {
+
+    private String fallbackSub;
+    private String fallbackAud;
+    private SignedJWT unsignedJwt;
+
+    public void testAccessTokenTypeWorksWithNoFallback() throws Exception {
+        noFallback();
+
+        jwtIssuerAndRealms = generateJwtIssuerRealmPairs(
+            randomIntBetween(1, 1), // realms
+            randomIntBetween(0, 1), // authz
+            randomIntBetween(1, JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS.size()), // algorithms
+            randomIntBetween(1, 3), // audiences
+            randomIntBetween(1, 3), // users
+            randomIntBetween(0, 3), // roles
+            randomIntBetween(0, 1), // jwtCacheSize
+            randomBoolean() // createHttpsServer
+        );
+        final JwtIssuerAndRealm jwtIssuerAndRealm = randomJwtIssuerRealmPair();
+        final User user = randomUser(jwtIssuerAndRealm.issuer());
+
+        final SecureString jwt = randomJwt(jwtIssuerAndRealm, user);
+        final SecureString clientSecret = JwtRealmInspector.getClientAuthenticationSharedSecret(jwtIssuerAndRealm.realm());
+        doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt, clientSecret, randomIntBetween(1, 3));
+    }
+
+    public void testAccessTokenTypeWorksWithFallbacks() throws Exception {
+        randomFallbacks();
+
+        jwtIssuerAndRealms = generateJwtIssuerRealmPairs(
+            randomIntBetween(1, 1), // realms
+            randomIntBetween(0, 1), // authz
+            randomIntBetween(1, JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS.size()), // algorithms
+            randomIntBetween(1, 3), // audiences
+            randomIntBetween(1, 3), // users
+            randomIntBetween(0, 3), // roles
+            randomIntBetween(0, 1), // jwtCacheSize
+            randomBoolean() // createHttpsServer
+        );
+        final JwtIssuerAndRealm jwtIssuerAndRealm = randomJwtIssuerRealmPair();
+        final User user = randomUser(jwtIssuerAndRealm.issuer());
+
+        final SecureString jwt2 = randomJwt(jwtIssuerAndRealm, user);
+        final SecureString clientSecret = JwtRealmInspector.getClientAuthenticationSharedSecret(jwtIssuerAndRealm.realm());
+        doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt2, clientSecret, randomIntBetween(1, 3));
+    }
+
+    @Override
+    protected JwtRealmSettingsBuilder createJwtRealmSettingsBuilder(JwtIssuer jwtIssuer, int authzCount, int jwtCacheSize)
+        throws Exception {
+        final JwtRealmSettingsBuilder jwtRealmSettingsBuilder = super.createJwtRealmSettingsBuilder(jwtIssuer, authzCount, jwtCacheSize);
+        final String realmName = jwtRealmSettingsBuilder.name();
+        final Settings.Builder settingsBuilder = jwtRealmSettingsBuilder.settingsBuilder();
+        settingsBuilder.put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.TOKEN_TYPE), "access_token")
+            .putList(
+                RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.ALLOWED_SUBJECTS),
+                jwtIssuer.principals.keySet().stream().toList()
+            );
+
+        if (fallbackSub != null) {
+            settingsBuilder.put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.FALLBACK_SUB_CLAIM), fallbackSub);
+        }
+        if (fallbackAud != null) {
+            settingsBuilder.put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.FALLBACK_AUD_CLAIM), fallbackAud);
+        }
+
+        return jwtRealmSettingsBuilder;
+    }
+
+    @Override
+    protected SecureString randomJwt(JwtIssuerAndRealm jwtIssuerAndRealm, User user) throws Exception {
+        final JwtIssuer.AlgJwkPair algJwkPair = randomFrom(jwtIssuerAndRealm.issuer().algAndJwksAll);
+        final JWK jwk = algJwkPair.jwk();
+
+        final HashMap<String, Object> otherClaims = new HashMap<>();
+        if (randomBoolean()) {
+            otherClaims.putAll(Map.of("other1", randomAlphaOfLength(10), "other2", randomAlphaOfLength(10)));
+        }
+
+        // Randomly set the fallback claims, it can co-exist with the original one in which case it is ignored
+        String subClaimValue = user.principal();
+        if (fallbackSub != null) {
+            if (randomBoolean()) {
+                // original claim does not exist, so it's the effective fallback
+                otherClaims.put(fallbackSub, subClaimValue);
+                subClaimValue = null;
+            } else {
+                // original claim still exist, in this case, the fallback can be anything and it does not matter
+                otherClaims.put(fallbackSub, randomValueOtherThan(subClaimValue, () -> randomAlphaOfLength(15)));
+            }
+        }
+        List<String> audClaimValue = JwtRealmInspector.getAllowedAudiences(jwtIssuerAndRealm.realm());
+        if (fallbackAud != null) {
+            if (randomBoolean()) {
+                otherClaims.put(fallbackAud, audClaimValue);
+                audClaimValue = null;
+            } else {
+                otherClaims.put(fallbackAud, randomValueOtherThanMany(audClaimValue::contains, () -> randomAlphaOfLength(15)));
+            }
+        }
+
+        // A bogus auth_time but access_token type does not check it
+        if (randomBoolean()) {
+            otherClaims.put("auth_time", randomAlphaOfLengthBetween(6, 18));
+        }
+
+        final Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS);
+        unsignedJwt = JwtTestCase.buildUnsignedJwt(
+            randomBoolean() ? null : JOSEObjectType.JWT.toString(), // kty
+            randomBoolean() ? null : jwk.getKeyID(), // kid
+            algJwkPair.alg(), // alg
+            randomAlphaOfLengthBetween(10, 20), // jwtID
+            JwtRealmInspector.getAllowedIssuer(jwtIssuerAndRealm.realm()), // iss
+            audClaimValue,
+            subClaimValue,
+            JwtRealmInspector.getPrincipalClaimName(jwtIssuerAndRealm.realm()), // principal claim name
+            user.principal(), // principal claim value
+            JwtRealmInspector.getGroupsClaimName(jwtIssuerAndRealm.realm()), // group claim name
+            List.of(user.roles()), // group claim value
+            null,
+            Date.from(now.minusSeconds(randomBoolean() ? 0 : 60 * randomLongBetween(5, 10))), // iat
+            Date.from(now), // nbf
+            Date.from(now.plusSeconds(60 * randomLongBetween(3600, 7200))), // exp
+            randomBoolean() ? null : new Nonce(32).toString(),
+            otherClaims
+        );
+        final SecureString signedJWT = signJwt(jwk, unsignedJwt);
+        return signedJWT;
+    }
+
+    private void noFallback() {
+        fallbackSub = null;
+        fallbackAud = null;
+    }
+
+    private void randomFallbacks() {
+        fallbackSub = randomBoolean() ? "_" + randomAlphaOfLength(5) : null;
+        fallbackAud = randomBoolean() || fallbackSub == null ? "_" + randomAlphaOfLength(8) : null;
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmAuthenticateTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmAuthenticateTests.java
index e8dfe16e448e..51c6b677b56b 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmAuthenticateTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmAuthenticateTests.java
@@ -13,8 +13,6 @@
 import com.nimbusds.jwt.PlainJWT;
 import com.nimbusds.jwt.SignedJWT;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.MockSecureSettings;
@@ -22,8 +20,8 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.core.security.authc.Realm;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
 import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings;
@@ -42,30 +40,28 @@
 import static org.hamcrest.Matchers.nullValue;
 
 public class JwtRealmAuthenticateTests extends JwtRealmTestCase {
-    private static final Logger LOGGER = LogManager.getLogger(JwtRealmAuthenticateTests.class);
 
     /**
      * Test with empty roles.
      * @throws Exception Unexpected test failure
      */
     public void testJwtAuthcRealmAuthcAuthzWithEmptyRoles() throws Exception {
-        this.jwtIssuerAndRealms = this.generateJwtIssuerRealmPairs(
-            this.createJwtRealmsSettingsBuilder(),
-            new MinMax(1, 1), // realmsRange
-            new MinMax(0, 1), // authzRange
-            new MinMax(1, JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS.size()), // algsRange
-            new MinMax(1, 3), // audiencesRange
-            new MinMax(1, 3), // usersRange
-            new MinMax(0, 0), // rolesRange
-            new MinMax(0, 1), // jwtCacheSizeRange
+        jwtIssuerAndRealms = generateJwtIssuerRealmPairs(
+            randomIntBetween(1, 1), // realmsRange
+            randomIntBetween(0, 1), // authzRange
+            randomIntBetween(1, JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS.size()), // algsRange
+            randomIntBetween(1, 3), // audiencesRange
+            randomIntBetween(1, 3), // usersRange
+            randomIntBetween(0, 0), // rolesRange
+            randomIntBetween(0, 1), // jwtCacheSizeRange
             randomBoolean() // createHttpsServer
         );
-        final JwtIssuerAndRealm jwtIssuerAndRealm = this.randomJwtIssuerRealmPair();
-        final User user = this.randomUser(jwtIssuerAndRealm.issuer());
-        final SecureString jwt = this.randomJwt(jwtIssuerAndRealm, user);
+        final JwtIssuerAndRealm jwtIssuerAndRealm = randomJwtIssuerRealmPair();
+        final User user = randomUser(jwtIssuerAndRealm.issuer());
+        final SecureString jwt = randomJwt(jwtIssuerAndRealm, user);
         final SecureString clientSecret = JwtRealmInspector.getClientAuthenticationSharedSecret(jwtIssuerAndRealm.realm());
-        final MinMax jwtAuthcRange = new MinMax(2, 3);
-        this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt, clientSecret, jwtAuthcRange);
+        final int jwtAuthcCount = randomIntBetween(2, 3);
+        doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt, clientSecret, jwtAuthcCount);
     }
 
     /**
@@ -73,85 +69,82 @@ public void testJwtAuthcRealmAuthcAuthzWithEmptyRoles() throws Exception {
      * @throws Exception Unexpected test failure
      */
     public void testJwtAuthcRealmAuthcAuthzWithoutAuthzRealms() throws Exception {
-        this.jwtIssuerAndRealms = this.generateJwtIssuerRealmPairs(
-            this.createJwtRealmsSettingsBuilder(),
-            new MinMax(1, 3), // realmsRange
-            new MinMax(0, 0), // authzRange
-            new MinMax(1, JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS.size()), // algsRange
-            new MinMax(1, 3), // audiencesRange
-            new MinMax(1, 3), // usersRange
-            new MinMax(0, 3), // rolesRange
-            new MinMax(0, 1), // jwtCacheSizeRange
+        jwtIssuerAndRealms = generateJwtIssuerRealmPairs(
+            randomIntBetween(1, 3), // realmsRange
+            randomIntBetween(0, 0), // authzRange
+            randomIntBetween(1, JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS.size()), // algsRange
+            randomIntBetween(1, 3), // audiencesRange
+            randomIntBetween(1, 3), // usersRange
+            randomIntBetween(0, 3), // rolesRange
+            randomIntBetween(0, 1), // jwtCacheSizeRange
             randomBoolean() // createHttpsServer
         );
-        final JwtIssuerAndRealm jwtIssuerAndRealm = this.randomJwtIssuerRealmPair();
+        final JwtIssuerAndRealm jwtIssuerAndRealm = randomJwtIssuerRealmPair();
         assertThat(jwtIssuerAndRealm.realm().delegatedAuthorizationSupport.hasDelegation(), is(false));
 
-        final User user = this.randomUser(jwtIssuerAndRealm.issuer());
-        final SecureString jwt = this.randomJwt(jwtIssuerAndRealm, user);
+        final User user = randomUser(jwtIssuerAndRealm.issuer());
+        final SecureString jwt = randomJwt(jwtIssuerAndRealm, user);
         final SecureString clientSecret = JwtRealmInspector.getClientAuthenticationSharedSecret(jwtIssuerAndRealm.realm());
-        final MinMax jwtAuthcRange = new MinMax(2, 3);
-        this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt, clientSecret, jwtAuthcRange);
+        final int jwtAuthcCount = randomIntBetween(2, 3);
+        doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt, clientSecret, jwtAuthcCount);
     }
 
     /**
      * Test with updated/removed/restored JWKs.
      * @throws Exception Unexpected test failure
      */
-    @TestLogging(value = "org.elasticsearch.xpack.security.authc.jwt:trace", reason = "debug")
     public void testJwkSetUpdates() throws Exception {
-        this.jwtIssuerAndRealms = this.generateJwtIssuerRealmPairs(
-            this.createJwtRealmsSettingsBuilder(),
-            new MinMax(1, 3), // realmsRange
-            new MinMax(0, 0), // authzRange
-            new MinMax(1, JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS.size()), // algsRange
-            new MinMax(1, 3), // audiencesRange
-            new MinMax(1, 3), // usersRange
-            new MinMax(0, 3), // rolesRange
-            new MinMax(0, 1), // jwtCacheSizeRange
+        jwtIssuerAndRealms = generateJwtIssuerRealmPairs(
+            randomIntBetween(1, 3), // realmsRange
+            randomIntBetween(0, 0), // authzRange
+            randomIntBetween(1, JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS.size()), // algsRange
+            randomIntBetween(1, 3), // audiencesRange
+            randomIntBetween(1, 3), // usersRange
+            randomIntBetween(0, 3), // rolesRange
+            randomIntBetween(0, 1), // jwtCacheSizeRange
             randomBoolean() // createHttpsServer
         );
-        final JwtIssuerAndRealm jwtIssuerAndRealm = this.randomJwtIssuerRealmPair();
+        final JwtIssuerAndRealm jwtIssuerAndRealm = randomJwtIssuerRealmPair();
         assertThat(jwtIssuerAndRealm.realm().delegatedAuthorizationSupport.hasDelegation(), is(false));
 
-        final User user = this.randomUser(jwtIssuerAndRealm.issuer());
-        final SecureString jwtJwks1 = this.randomJwt(jwtIssuerAndRealm, user);
+        final User user = randomUser(jwtIssuerAndRealm.issuer());
+        final SecureString jwtJwks1 = randomJwt(jwtIssuerAndRealm, user);
         final SecureString clientSecret = JwtRealmInspector.getClientAuthenticationSharedSecret(jwtIssuerAndRealm.realm());
-        final MinMax jwtAuthcRange = new MinMax(2, 3);
-        this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks1, clientSecret, jwtAuthcRange);
+        final int jwtAuthcCount = randomIntBetween(2, 3);
+        doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks1, clientSecret, jwtAuthcCount);
 
         // Details about first JWT using the JWT issuer original JWKs
         final String jwt1JwksAlg = SignedJWT.parse(jwtJwks1.toString()).getHeader().getAlgorithm().getName();
         final boolean isPkcJwtJwks1 = JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_PKC.contains(jwt1JwksAlg);
-        LOGGER.debug("JWT alg=[{}]", jwt1JwksAlg);
+        logger.debug("JWT alg=[{}]", jwt1JwksAlg);
 
         // Backup JWKs 1
         final List<JwtIssuer.AlgJwkPair> jwtIssuerJwks1Backup = jwtIssuerAndRealm.issuer().algAndJwksAll;
         final boolean jwtIssuerJwks1OidcSafe = JwkValidateUtilTests.areJwkHmacOidcSafe(
             jwtIssuerJwks1Backup.stream().map(e -> e.jwk()).toList()
         );
-        LOGGER.debug("JWKs 1, algs=[{}]", String.join(",", jwtIssuerAndRealm.issuer().algorithmsAll));
+        logger.debug("JWKs 1, algs=[{}]", String.join(",", jwtIssuerAndRealm.issuer().algorithmsAll));
 
         // Empty all JWT issuer JWKs.
-        LOGGER.debug("JWKs 1 backed up, algs=[{}]", String.join(",", jwtIssuerAndRealm.issuer().algorithmsAll));
+        logger.debug("JWKs 1 backed up, algs=[{}]", String.join(",", jwtIssuerAndRealm.issuer().algorithmsAll));
         jwtIssuerAndRealm.issuer().setJwks(Collections.emptyList(), jwtIssuerJwks1OidcSafe);
-        super.printJwtIssuer(jwtIssuerAndRealm.issuer());
-        super.copyIssuerJwksToRealmConfig(jwtIssuerAndRealm);
-        LOGGER.debug("JWKs 1 emptied, algs=[{}]", String.join(",", jwtIssuerAndRealm.issuer().algorithmsAll));
+        printJwtIssuer(jwtIssuerAndRealm.issuer());
+        copyIssuerJwksToRealmConfig(jwtIssuerAndRealm);
+        logger.debug("JWKs 1 emptied, algs=[{}]", String.join(",", jwtIssuerAndRealm.issuer().algorithmsAll));
 
         // Original JWT continues working, because JWT realm cached old JWKs in memory.
-        this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks1, clientSecret, jwtAuthcRange);
-        LOGGER.debug("JWT 1 still worked, because JWT realm has old JWKs cached in memory");
+        doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks1, clientSecret, jwtAuthcCount);
+        logger.debug("JWT 1 still worked, because JWT realm has old JWKs cached in memory");
 
         // Restore original JWKs 1 into the JWT issuer.
         jwtIssuerAndRealm.issuer().setJwks(jwtIssuerJwks1Backup, jwtIssuerJwks1OidcSafe);
-        super.printJwtIssuer(jwtIssuerAndRealm.issuer());
-        super.copyIssuerJwksToRealmConfig(jwtIssuerAndRealm);
-        LOGGER.debug("JWKs 1 restored, algs=[{}]", String.join(",", jwtIssuerAndRealm.issuer().algorithmsAll));
+        printJwtIssuer(jwtIssuerAndRealm.issuer());
+        copyIssuerJwksToRealmConfig(jwtIssuerAndRealm);
+        logger.debug("JWKs 1 restored, algs=[{}]", String.join(",", jwtIssuerAndRealm.issuer().algorithmsAll));
 
         // Original JWT continues working, because JWT realm cached old JWKs in memory.
-        this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks1, clientSecret, jwtAuthcRange);
-        LOGGER.debug("JWT 1 still worked, because JWT realm has old JWKs cached in memory");
+        doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks1, clientSecret, jwtAuthcCount);
+        logger.debug("JWT 1 still worked, because JWT realm has old JWKs cached in memory");
 
         // Generate a replacement set of JWKs 2 for the JWT issuer.
         final List<JwtIssuer.AlgJwkPair> jwtIssuerJwks2Backup = JwtRealmTestCase.randomJwks(
@@ -159,31 +152,31 @@ public void testJwkSetUpdates() throws Exception {
             jwtIssuerJwks1OidcSafe
         );
         jwtIssuerAndRealm.issuer().setJwks(jwtIssuerJwks2Backup, jwtIssuerJwks1OidcSafe);
-        super.printJwtIssuer(jwtIssuerAndRealm.issuer());
-        super.copyIssuerJwksToRealmConfig(jwtIssuerAndRealm);
-        LOGGER.debug("JWKs 2 created, algs=[{}]", String.join(",", jwtIssuerAndRealm.issuer().algorithmsAll));
+        printJwtIssuer(jwtIssuerAndRealm.issuer());
+        copyIssuerJwksToRealmConfig(jwtIssuerAndRealm);
+        logger.debug("JWKs 2 created, algs=[{}]", String.join(",", jwtIssuerAndRealm.issuer().algorithmsAll));
 
         // Original JWT continues working, because JWT realm still has original JWKs cached in memory.
         // - jwtJwks1(PKC): Pass (Original PKC JWKs are still in the realm)
         // - jwtJwks1(HMAC): Pass (Original HMAC JWKs are still in the realm)
-        this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks1, clientSecret, jwtAuthcRange);
-        LOGGER.debug("JWT 1 still worked, because JWT realm has old JWKs cached in memory");
+        doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks1, clientSecret, jwtAuthcCount);
+        logger.debug("JWT 1 still worked, because JWT realm has old JWKs cached in memory");
 
         // Create a JWT using the new JWKs.
-        final SecureString jwtJwks2 = this.randomJwt(jwtIssuerAndRealm, user);
+        final SecureString jwtJwks2 = randomJwt(jwtIssuerAndRealm, user);
         final String jwtJwks2Alg = SignedJWT.parse(jwtJwks2.toString()).getHeader().getAlgorithm().getName();
         final boolean isPkcJwtJwks2 = JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS_PKC.contains(jwtJwks2Alg);
-        LOGGER.debug("Created JWT 2: oidcSafe=[{}], algs=[{}, {}]", jwtIssuerJwks1OidcSafe, jwt1JwksAlg, jwtJwks2Alg);
+        logger.debug("Created JWT 2: oidcSafe=[{}], algs=[{}, {}]", jwtIssuerJwks1OidcSafe, jwt1JwksAlg, jwtJwks2Alg);
 
         // Try new JWT.
         // - jwtJwks2(PKC): PKC reload triggered and loaded new JWKs, so PASS
         // - jwtJwks2(HMAC): HMAC reload triggered but it is a no-op, so FAIL
         if (isPkcJwtJwks2) {
-            this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks2, clientSecret, jwtAuthcRange);
-            LOGGER.debug("PKC JWT 2 worked with JWKs 2");
+            doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks2, clientSecret, jwtAuthcCount);
+            logger.debug("PKC JWT 2 worked with JWKs 2");
         } else {
-            this.verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtJwks2, clientSecret);
-            LOGGER.debug("HMAC JWT 2 failed with JWKs 1");
+            verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtJwks2, clientSecret);
+            logger.debug("HMAC JWT 2 failed with JWKs 1");
         }
 
         // Try old JWT.
@@ -192,30 +185,30 @@ public void testJwkSetUpdates() throws Exception {
         // - jwtJwks2(HMAC): HMAC reload triggered but it is a no-op, jwtJwks1(PKC): PKC reload not triggered, so PASS
         // - jwtJwks2(HMAC): HMAC reload triggered but it is a no-op, jwtJwks1(HMAC): HMAC reload not triggered, so PASS
         if (isPkcJwtJwks1 == false || isPkcJwtJwks2 == false) {
-            this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks1, clientSecret, jwtAuthcRange);
+            doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks1, clientSecret, jwtAuthcCount);
         } else {
-            this.verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtJwks1, clientSecret);
+            verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtJwks1, clientSecret);
         }
 
         // Empty all JWT issuer JWKs.
         jwtIssuerAndRealm.issuer().setJwks(Collections.emptyList(), jwtIssuerJwks1OidcSafe);
-        super.printJwtIssuer(jwtIssuerAndRealm.issuer());
-        super.copyIssuerJwksToRealmConfig(jwtIssuerAndRealm);
+        printJwtIssuer(jwtIssuerAndRealm.issuer());
+        copyIssuerJwksToRealmConfig(jwtIssuerAndRealm);
 
         // New JWT continues working because JWT realm will end up with PKC JWKs 2 and HMAC JWKs 1 in memory
         if (isPkcJwtJwks2) {
-            this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks2, clientSecret, jwtAuthcRange);
+            doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks2, clientSecret, jwtAuthcCount);
         } else {
-            this.verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtJwks2, clientSecret);
+            verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtJwks2, clientSecret);
         }
 
         // Trigger JWT realm to reload JWKs and go into a degraded state
         // - jwtJwks1(HMAC): HMAC reload not triggered, so PASS
         // - jwtJwks1(PKC): PKC reload triggered and loaded new JWKs, so FAIL
         if (isPkcJwtJwks1 == false || isPkcJwtJwks2 == false) {
-            this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks1, clientSecret, jwtAuthcRange);
+            doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks1, clientSecret, jwtAuthcCount);
         } else {
-            this.verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtJwks1, clientSecret);
+            verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtJwks1, clientSecret);
         }
 
         // Try new JWT and verify degraded state caused by empty PKC JWKs
@@ -224,15 +217,15 @@ public void testJwkSetUpdates() throws Exception {
         // - jwtJwks1(PKC) + jwtJwks2(HMAC): If second JWT is HMAC, it always fails because HMAC reload not supported.
         // - jwtJwks1(HMAC) + jwtJwks2(HMAC): If second JWT is HMAC, it always fails because HMAC reload not supported.
         if (isPkcJwtJwks1 == false && isPkcJwtJwks2) {
-            this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks2, clientSecret, jwtAuthcRange);
+            doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks2, clientSecret, jwtAuthcCount);
         } else {
-            this.verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtJwks2, clientSecret);
+            verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtJwks2, clientSecret);
         }
 
         // Restore JWKs 2 to the realm
         jwtIssuerAndRealm.issuer().setJwks(jwtIssuerJwks2Backup, jwtIssuerJwks1OidcSafe);
-        super.copyIssuerJwksToRealmConfig(jwtIssuerAndRealm);
-        super.printJwtIssuer(jwtIssuerAndRealm.issuer());
+        copyIssuerJwksToRealmConfig(jwtIssuerAndRealm);
+        printJwtIssuer(jwtIssuerAndRealm.issuer());
 
         // Trigger JWT realm to reload JWKs and go into a recovered state
         // - jwtJwks2(PKC): Pass (Triggers PKC reload, gets newer PKC JWKs), jwtJwks1(PKC): Fail (Triggers PKC reload, gets new PKC JWKs)
@@ -240,14 +233,14 @@ public void testJwkSetUpdates() throws Exception {
         // - jwtJwks2(HMAC): Fail (Triggers HMAC reload, but it is a no-op), jwtJwks1(PKC): Fail (Triggers PKC reload, gets new PKC JWKs)
         // - jwtJwks2(HMAC): Fail (Triggers HMAC reload, but it is a no-op), jwtJwks1(HMAC): Pass (HMAC reload was a no-op)
         if (isPkcJwtJwks2) {
-            this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks2, clientSecret, jwtAuthcRange);
+            doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks2, clientSecret, jwtAuthcCount);
         } else {
-            this.verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtJwks2, clientSecret);
+            verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtJwks2, clientSecret);
         }
         if (isPkcJwtJwks1 == false || isPkcJwtJwks2 == false) {
-            this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks1, clientSecret, jwtAuthcRange);
+            doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwtJwks1, clientSecret, jwtAuthcCount);
         } else {
-            this.verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtJwks1, clientSecret);
+            verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtJwks1, clientSecret);
         }
     }
 
@@ -256,25 +249,24 @@ public void testJwkSetUpdates() throws Exception {
      * @throws Exception Unexpected test failure
      */
     public void testJwtAuthcRealmAuthcAuthzWithAuthzRealms() throws Exception {
-        this.jwtIssuerAndRealms = this.generateJwtIssuerRealmPairs(
-            this.createJwtRealmsSettingsBuilder(),
-            new MinMax(1, 3), // realmsRange
-            new MinMax(1, 3), // authzRange
-            new MinMax(1, JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS.size()), // algsRange
-            new MinMax(1, 3), // audiencesRange
-            new MinMax(1, 3), // usersRange
-            new MinMax(0, 3), // rolesRange
-            new MinMax(0, 1), // jwtCacheSizeRange
+        jwtIssuerAndRealms = generateJwtIssuerRealmPairs(
+            randomIntBetween(1, 3), // realmsRange
+            randomIntBetween(1, 3), // authzRange
+            randomIntBetween(1, JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS.size()), // algsRange
+            randomIntBetween(1, 3), // audiencesRange
+            randomIntBetween(1, 3), // usersRange
+            randomIntBetween(0, 3), // rolesRange
+            randomIntBetween(0, 1), // jwtCacheSizeRange
             randomBoolean() // createHttpsServer
         );
-        final JwtIssuerAndRealm jwtIssuerAndRealm = this.randomJwtIssuerRealmPair();
+        final JwtIssuerAndRealm jwtIssuerAndRealm = randomJwtIssuerRealmPair();
         assertThat(jwtIssuerAndRealm.realm().delegatedAuthorizationSupport.hasDelegation(), is(true));
 
-        final User user = this.randomUser(jwtIssuerAndRealm.issuer());
-        final SecureString jwt = this.randomJwt(jwtIssuerAndRealm, user);
+        final User user = randomUser(jwtIssuerAndRealm.issuer());
+        final SecureString jwt = randomJwt(jwtIssuerAndRealm, user);
         final SecureString clientSecret = JwtRealmInspector.getClientAuthenticationSharedSecret(jwtIssuerAndRealm.realm());
-        final MinMax jwtAuthcRange = new MinMax(2, 3);
-        this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt, clientSecret, jwtAuthcRange);
+        final int jwtAuthcCount = randomIntBetween(2, 3);
+        doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt, clientSecret, jwtAuthcCount);
 
         // After the above success path test, do a negative path test for an authc user that does not exist in any authz realm.
         // In other words, above the `user` was found in an authz realm, but below `otherUser` will not be found in any authz realm.
@@ -284,13 +276,9 @@ public void testJwtAuthcRealmAuthcAuthzWithAuthzRealms() throws Exception {
                 () -> randomAlphaOfLengthBetween(4, 12)
             );
             final User otherUser = new User(otherUsername);
-            final SecureString otherJwt = this.randomJwt(jwtIssuerAndRealm, otherUser);
+            final SecureString otherJwt = randomJwt(jwtIssuerAndRealm, otherUser);
 
-            final JwtAuthenticationToken otherToken = new JwtAuthenticationToken(
-                List.of(JwtRealmInspector.getPrincipalClaimName(jwtIssuerAndRealm.realm())),
-                otherJwt,
-                clientSecret
-            );
+            final AuthenticationToken otherToken = jwtIssuerAndRealm.realm().token(createThreadContext(otherJwt, clientSecret));
             final PlainActionFuture<AuthenticationResult<User>> otherFuture = new PlainActionFuture<>();
             jwtIssuerAndRealm.realm().authenticate(otherToken, otherFuture);
             final AuthenticationResult<User> otherResult = otherFuture.actionGet();
@@ -308,21 +296,18 @@ public void testJwtAuthcRealmAuthcAuthzWithAuthzRealms() throws Exception {
      * @throws Exception Unexpected test failure
      */
     public void testPkcJwkSetUrlNotFound() throws Exception {
-        final JwtRealmsService jwtRealmsService = this.generateJwtRealmsService(this.createJwtRealmsSettingsBuilder());
-        final String principalClaimName = randomFrom(jwtRealmsService.getPrincipalClaimNames());
-
         final List<Realm> allRealms = new ArrayList<>(); // authc and authz realms
         final boolean createHttpsServer = true; // force issuer to create HTTPS server for its PKC JWKSet
-        final JwtIssuer jwtIssuer = this.createJwtIssuer(0, principalClaimName, 12, 1, 1, 1, createHttpsServer);
-        assertThat(jwtIssuer.httpsServer, is(notNullValue()));
+        final JwtIssuer jwtIssuer = createJwtIssuer(0, 12, 1, 1, 1, createHttpsServer);
+        assertThat(jwtIssuer.httpsServer, notNullValue());
         try {
-            final JwtRealmSettingsBuilder jwtRealmSettingsBuilder = this.createJwtRealmSettingsBuilder(jwtIssuer, 0, 0);
+            final JwtRealmSettingsBuilder jwtRealmSettingsBuilder = createJwtRealmSettingsBuilder(jwtIssuer, 0, 0);
             final String configKey = RealmSettings.getFullSettingKey(jwtRealmSettingsBuilder.name(), JwtRealmSettings.PKC_JWKSET_PATH);
             final String configValue = jwtIssuer.httpsServer.url.replace("/valid/", "/invalid"); // right host, wrong path
             jwtRealmSettingsBuilder.settingsBuilder().put(configKey, configValue);
             final Exception exception = expectThrows(
                 SettingsException.class,
-                () -> this.createJwtRealm(allRealms, jwtRealmsService, jwtIssuer, jwtRealmSettingsBuilder)
+                () -> createJwtRealm(allRealms, jwtIssuer, jwtRealmSettingsBuilder)
             );
             assertThat(exception.getMessage(), equalTo("Can't get contents for setting [" + configKey + "] value [" + configValue + "]."));
             assertThat(exception.getCause().getMessage(), equalTo("Get [" + configValue + "] failed, status [404], reason [Not Found]."));
@@ -336,66 +321,65 @@ public void testPkcJwkSetUrlNotFound() throws Exception {
      * @throws Exception Unexpected test failure
      */
     public void testJwtValidationFailures() throws Exception {
-        this.jwtIssuerAndRealms = this.generateJwtIssuerRealmPairs(
-            this.createJwtRealmsSettingsBuilder(),
-            new MinMax(1, 1), // realmsRange
-            new MinMax(0, 0), // authzRange
-            new MinMax(1, JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS.size()), // algsRange
-            new MinMax(1, 1), // audiencesRange
-            new MinMax(1, 1), // usersRange
-            new MinMax(1, 1), // rolesRange
-            new MinMax(0, 1), // jwtCacheSizeRange
+        jwtIssuerAndRealms = generateJwtIssuerRealmPairs(
+            randomIntBetween(1, 1), // realmsRange
+            randomIntBetween(0, 0), // authzRange
+            randomIntBetween(1, JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS.size()), // algsRange
+            randomIntBetween(1, 1), // audiencesRange
+            randomIntBetween(1, 1), // usersRange
+            randomIntBetween(1, 1), // rolesRange
+            randomIntBetween(0, 1), // jwtCacheSizeRange
             randomBoolean() // createHttpsServer
         );
-        final JwtIssuerAndRealm jwtIssuerAndRealm = this.randomJwtIssuerRealmPair();
-        final User user = this.randomUser(jwtIssuerAndRealm.issuer());
-        final SecureString jwt = this.randomJwt(jwtIssuerAndRealm, user);
+        final JwtIssuerAndRealm jwtIssuerAndRealm = randomJwtIssuerRealmPair();
+        final User user = randomUser(jwtIssuerAndRealm.issuer());
+        final SecureString jwt = randomJwt(jwtIssuerAndRealm, user);
         final SecureString clientSecret = JwtRealmInspector.getClientAuthenticationSharedSecret(jwtIssuerAndRealm.realm());
-        final MinMax jwtAuthcRange = new MinMax(2, 3);
+        final int jwtAuthcCount = randomIntBetween(2, 3);
 
         // Indirectly verify authentication works before performing any failure scenarios
-        this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt, clientSecret, jwtAuthcRange);
+        doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt, clientSecret, jwtAuthcCount);
 
         // The above confirmed JWT realm authc/authz is working.
         // Now perform negative path tests to confirm JWT validation rejects invalid JWTs for different scenarios.
 
         {   // Do one more direct SUCCESS scenario by checking token() and authenticate() directly before moving on to FAILURE scenarios.
-            final ThreadContext requestThreadContext = super.createThreadContext(jwt, clientSecret);
+            final ThreadContext requestThreadContext = createThreadContext(jwt, clientSecret);
             final JwtAuthenticationToken token = (JwtAuthenticationToken) jwtIssuerAndRealm.realm().token(requestThreadContext);
             final PlainActionFuture<AuthenticationResult<User>> plainActionFuture = PlainActionFuture.newFuture();
             jwtIssuerAndRealm.realm().authenticate(token, plainActionFuture);
-            assertThat(plainActionFuture.get(), is(notNullValue()));
+            assertThat(plainActionFuture.get(), notNullValue());
             assertThat(plainActionFuture.get().isAuthenticated(), is(true));
         }
 
         // Directly verify FAILURE scenarios for token() parsing failures and authenticate() validation failures.
 
         // Null JWT
-        final ThreadContext tc1 = super.createThreadContext(null, clientSecret);
+        final ThreadContext tc1 = createThreadContext(null, clientSecret);
         assertThat(jwtIssuerAndRealm.realm().token(tc1), nullValue());
 
         // Empty JWT string
-        final ThreadContext tc2 = super.createThreadContext("", clientSecret);
+        final ThreadContext tc2 = createThreadContext("", clientSecret);
         final Exception e2 = expectThrows(IllegalArgumentException.class, () -> jwtIssuerAndRealm.realm().token(tc2));
         assertThat(e2.getMessage(), equalTo("JWT bearer token must be non-empty"));
 
         // Non-empty whitespace JWT string
-        final ThreadContext tc3 = super.createThreadContext("", clientSecret);
+        final ThreadContext tc3 = createThreadContext("", clientSecret);
         final Exception e3 = expectThrows(IllegalArgumentException.class, () -> jwtIssuerAndRealm.realm().token(tc3));
         assertThat(e3.getMessage(), equalTo("JWT bearer token must be non-empty"));
 
         // Blank client secret
-        final ThreadContext tc4 = super.createThreadContext(jwt, "");
+        final ThreadContext tc4 = createThreadContext(jwt, "");
         final Exception e4 = expectThrows(IllegalArgumentException.class, () -> jwtIssuerAndRealm.realm().token(tc4));
         assertThat(e4.getMessage(), equalTo("Client shared secret must be non-empty"));
 
         // Non-empty whitespace JWT client secret
-        final ThreadContext tc5 = super.createThreadContext(jwt, " ");
+        final ThreadContext tc5 = createThreadContext(jwt, " ");
         final Exception e5 = expectThrows(IllegalArgumentException.class, () -> jwtIssuerAndRealm.realm().token(tc5));
         assertThat(e5.getMessage(), equalTo("Client shared secret must be non-empty"));
 
         // JWT parse exception
-        final ThreadContext tc6 = super.createThreadContext("Head.Body.Sig", clientSecret);
+        final ThreadContext tc6 = createThreadContext("Head.Body.Sig", clientSecret);
         final Exception e6 = expectThrows(IllegalArgumentException.class, () -> jwtIssuerAndRealm.realm().token(tc6));
         assertThat(e6.getMessage(), equalTo("Failed to parse JWT bearer token"));
 
@@ -407,7 +391,7 @@ public void testJwtValidationFailures() throws Exception {
 
         {   // Verify rejection of unsigned JWT
             final SecureString unsignedJwt = new SecureString(new PlainJWT(validClaimsSet).serialize().toCharArray());
-            final ThreadContext tc = super.createThreadContext(unsignedJwt, clientSecret);
+            final ThreadContext tc = createThreadContext(unsignedJwt, clientSecret);
             expectThrows(IllegalArgumentException.class, () -> jwtIssuerAndRealm.realm().token(tc));
         }
 
@@ -429,20 +413,20 @@ public void testJwtValidationFailures() throws Exception {
             // This check can only be executed if there is a flip algorithm available in the realm
             if (Strings.hasText(mixupAlg)) {
                 final JWSHeader tamperedHeader = new JWSHeader.Builder(JWSAlgorithm.parse(mixupAlg)).build();
-                final SecureString jwtTamperedHeader = JwtValidateUtil.buildJwt(tamperedHeader, validClaimsSet, validSignature);
-                this.verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtTamperedHeader, clientSecret);
+                final SecureString jwtTamperedHeader = buildJwt(tamperedHeader, validClaimsSet, validSignature);
+                verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtTamperedHeader, clientSecret);
             }
         }
 
         {   // Verify rejection of a tampered claim set
             final JWTClaimsSet tamperedClaimsSet = new JWTClaimsSet.Builder(validClaimsSet).claim("gr0up", "superuser").build();
-            final SecureString jwtTamperedClaimsSet = JwtValidateUtil.buildJwt(validHeader, tamperedClaimsSet, validSignature);
-            this.verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtTamperedClaimsSet, clientSecret);
+            final SecureString jwtTamperedClaimsSet = buildJwt(validHeader, tamperedClaimsSet, validSignature);
+            verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtTamperedClaimsSet, clientSecret);
         }
 
         {   // Verify rejection of a tampered signature
             final SecureString jwtWithTruncatedSignature = new SecureString(jwt.toString().substring(0, jwt.length() - 1).toCharArray());
-            this.verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtWithTruncatedSignature, clientSecret);
+            verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtWithTruncatedSignature, clientSecret);
         }
 
         // Get read to re-sign JWTs for time claim failure tests
@@ -454,26 +438,26 @@ public void testJwtValidationFailures() throws Exception {
 
         {   // Verify rejection of JWT auth_time > now
             final JWTClaimsSet claimsSet = new JWTClaimsSet.Builder(validClaimsSet).claim("auth_time", future).build();
-            final SecureString jwtIatFuture = JwtValidateUtil.signJwt(algJwkPair.jwk(), new SignedJWT(jwtHeader, claimsSet));
-            this.verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtIatFuture, clientSecret);
+            final SecureString jwtIatFuture = signJwt(algJwkPair.jwk(), new SignedJWT(jwtHeader, claimsSet));
+            verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtIatFuture, clientSecret);
         }
 
         {   // Verify rejection of JWT iat > now
             final JWTClaimsSet claimsSet = new JWTClaimsSet.Builder(validClaimsSet).issueTime(future).build();
-            final SecureString jwtIatFuture = JwtValidateUtil.signJwt(algJwkPair.jwk(), new SignedJWT(jwtHeader, claimsSet));
-            this.verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtIatFuture, clientSecret);
+            final SecureString jwtIatFuture = signJwt(algJwkPair.jwk(), new SignedJWT(jwtHeader, claimsSet));
+            verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtIatFuture, clientSecret);
         }
 
         {   // Verify rejection of JWT nbf > now
             final JWTClaimsSet claimsSet = new JWTClaimsSet.Builder(validClaimsSet).notBeforeTime(future).build();
-            final SecureString jwtIatFuture = JwtValidateUtil.signJwt(algJwkPair.jwk(), new SignedJWT(jwtHeader, claimsSet));
-            this.verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtIatFuture, clientSecret);
+            final SecureString jwtIatFuture = signJwt(algJwkPair.jwk(), new SignedJWT(jwtHeader, claimsSet));
+            verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtIatFuture, clientSecret);
         }
 
         {   // Verify rejection of JWT now > exp
             final JWTClaimsSet claimsSet = new JWTClaimsSet.Builder(validClaimsSet).expirationTime(past).build();
-            final SecureString jwtExpPast = JwtValidateUtil.signJwt(algJwkPair.jwk(), new SignedJWT(jwtHeader, claimsSet));
-            this.verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtExpPast, clientSecret);
+            final SecureString jwtExpPast = signJwt(algJwkPair.jwk(), new SignedJWT(jwtHeader, claimsSet));
+            verifyAuthenticateFailureHelper(jwtIssuerAndRealm, jwtExpPast, clientSecret);
         }
     }
 
@@ -483,27 +467,24 @@ public void testJwtValidationFailures() throws Exception {
      * @throws Exception Unexpected test failure
      */
     public void testSameIssuerTwoRealmsDifferentClientSecrets() throws Exception {
-        final JwtRealmsService jwtRealmsService = this.generateJwtRealmsService(this.createJwtRealmsSettingsBuilder());
-        final String principalClaimName = randomFrom(jwtRealmsService.getPrincipalClaimNames());
-
         final int realmsCount = 2;
         final List<Realm> allRealms = new ArrayList<>(realmsCount); // two identical realms for same issuer, except different client secret
-        final JwtIssuer jwtIssuer = this.createJwtIssuer(0, principalClaimName, 12, 1, 1, 1, false);
-        super.printJwtIssuer(jwtIssuer);
-        this.jwtIssuerAndRealms = new ArrayList<>(realmsCount);
+        final JwtIssuer jwtIssuer = createJwtIssuer(0, 12, 1, 1, 1, false);
+        printJwtIssuer(jwtIssuer);
+        jwtIssuerAndRealms = new ArrayList<>(realmsCount);
         for (int i = 0; i < realmsCount; i++) {
             final String realmName = "realm_" + jwtIssuer.issuerClaimValue + "_" + i;
             final String clientSecret = "clientSecret_" + jwtIssuer.issuerClaimValue + "_" + i;
 
             final Settings.Builder authcSettings = Settings.builder()
-                .put(this.globalSettings)
+                .put(globalSettings)
                 .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.ALLOWED_ISSUER), jwtIssuer.issuerClaimValue)
                 .put(
                     RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.ALLOWED_SIGNATURE_ALGORITHMS),
                     String.join(",", jwtIssuer.algorithmsAll)
                 )
                 .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.ALLOWED_AUDIENCES), jwtIssuer.audiencesClaimValue.get(0))
-                .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.CLAIMS_PRINCIPAL.getClaim()), principalClaimName)
+                .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.CLAIMS_PRINCIPAL.getClaim()), jwtIssuer.principalClaimName)
                 .put(
                     RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.CLIENT_AUTHENTICATION_TYPE),
                     JwtRealmSettings.ClientAuthenticationType.SHARED_SECRET.value()
@@ -511,7 +492,7 @@ public void testSameIssuerTwoRealmsDifferentClientSecrets() throws Exception {
             if (jwtIssuer.encodedJwkSetPkcPublic.isEmpty() == false) {
                 authcSettings.put(
                     RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.PKC_JWKSET_PATH),
-                    super.saveToTempFile("jwkset.", ".json", jwtIssuer.encodedJwkSetPkcPublic)
+                    saveToTempFile("jwkset.", ".json", jwtIssuer.encodedJwkSetPkcPublic)
                 );
             }
             // JWT authc realm secure settings
@@ -534,19 +515,19 @@ public void testSameIssuerTwoRealmsDifferentClientSecrets() throws Exception {
             );
             authcSettings.setSecureSettings(secureSettings);
             final JwtRealmSettingsBuilder jwtRealmSettingsBuilder = new JwtRealmSettingsBuilder(realmName, authcSettings);
-            final JwtRealm jwtRealm = this.createJwtRealm(allRealms, jwtRealmsService, jwtIssuer, jwtRealmSettingsBuilder);
-            jwtRealm.initialize(allRealms, super.licenseState);
+            final JwtRealm jwtRealm = createJwtRealm(allRealms, jwtIssuer, jwtRealmSettingsBuilder);
+            jwtRealm.initialize(allRealms, licenseState);
             final JwtIssuerAndRealm jwtIssuerAndRealm = new JwtIssuerAndRealm(jwtIssuer, jwtRealm, jwtRealmSettingsBuilder);
-            this.jwtIssuerAndRealms.add(jwtIssuerAndRealm); // add them so the test will clean them up
-            super.printJwtRealm(jwtRealm);
+            jwtIssuerAndRealms.add(jwtIssuerAndRealm); // add them so the test will clean them up
+            printJwtRealm(jwtRealm);
         }
 
         // pick 2nd realm and use its secret, verify 2nd realm does authc, which implies 1st realm rejects the secret
-        final JwtIssuerAndRealm jwtIssuerAndRealm = this.jwtIssuerAndRealms.get(1);
-        final User user = this.randomUser(jwtIssuerAndRealm.issuer());
-        final SecureString jwt = this.randomJwt(jwtIssuerAndRealm, user);
+        final JwtIssuerAndRealm jwtIssuerAndRealm = jwtIssuerAndRealms.get(1);
+        final User user = randomUser(jwtIssuerAndRealm.issuer());
+        final SecureString jwt = randomJwt(jwtIssuerAndRealm, user);
         final SecureString clientSecret = JwtRealmInspector.getClientAuthenticationSharedSecret(jwtIssuerAndRealm.realm());
-        final MinMax jwtAuthcRange = new MinMax(2, 3);
-        this.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt, clientSecret, jwtAuthcRange);
+        final int jwtAuthcCount = randomIntBetween(2, 3);
+        doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt, clientSecret, jwtAuthcCount);
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmGenerateTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmGenerateTests.java
index a6f4586f2588..2f77923c6c50 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmGenerateTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmGenerateTests.java
@@ -13,8 +13,6 @@
 import com.nimbusds.jose.jwk.RSAKey;
 import com.nimbusds.jwt.SignedJWT;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.SecureString;
@@ -25,7 +23,6 @@
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
 import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings;
-import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmsServiceSettings;
 import org.elasticsearch.xpack.core.security.authc.support.DelegatedAuthorizationSettings;
 import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.core.security.user.User;
@@ -41,16 +38,11 @@
 import java.util.Locale;
 import java.util.Map;
 
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-
 /**
  * Generate verified JWT configurations for integration tests or documentation.
  */
 public class JwtRealmGenerateTests extends JwtRealmTestCase {
-    private static final Logger LOGGER = LogManager.getLogger(JwtRealmGenerateTests.class);
-
-    private static final MinMax JWT_AUTHC_RANGE_1 = new MinMax(1, 1);
+    private static final int JWT_AUTHC_REPEATS_1 = 1;
     private static final Date DATE_2000_1_1 = Date.from(ZonedDateTime.of(2000, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant());
     private static final Date DATE_2099_1_1 = Date.from(ZonedDateTime.of(2099, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant());
 
@@ -67,16 +59,11 @@ public void testCreateJwtSmokeTestRealm() throws Exception {
         );
 
         final String principalClaimName = "sub";
-        final Settings.Builder jwtRealmsServiceSettings = Settings.builder()
-            .put(this.globalSettings)
-            .put(JwtRealmsServiceSettings.PRINCIPAL_CLAIMS_SETTING.getKey(), String.join(",", principalClaimName));
-        final JwtRealmsService jwtRealmsService = new JwtRealmsService(jwtRealmsServiceSettings.build());
 
         // Create issuer
         final JwtIssuer jwtIssuer = new JwtIssuer(
             "iss8", // iss
             List.of("aud8"), // aud
-            principalClaimName, // sub
             Collections.singletonMap("security_test_user", new User("security_test_user", "security_test_role")), // users
             false // createHttpsServer
         );
@@ -85,7 +72,7 @@ public void testCreateJwtSmokeTestRealm() throws Exception {
         // Create realm settings
         final String realmName = "jwt8";
         final Settings.Builder configBuilder = Settings.builder()
-            .put(this.globalSettings)
+            .put(globalSettings)
             .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.ALLOWED_ISSUER), jwtIssuer.issuerClaimValue)
             .put(
                 RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.ALLOWED_AUDIENCES),
@@ -112,18 +99,18 @@ public void testCreateJwtSmokeTestRealm() throws Exception {
         configBuilder.setSecureSettings(secureSettings);
 
         // Create realm
-        final RealmConfig config = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, configBuilder.build(), 8);
+        final RealmConfig config = buildRealmConfig(JwtRealmSettings.TYPE, realmName, configBuilder.build(), 8);
         final SSLService sslService = new SSLService(TestEnvironment.newEnvironment(configBuilder.build()));
-        final UserRoleMapper userRoleMapper = super.buildRoleMapper(jwtIssuer.principals);
-        final JwtRealm jwtRealm = new JwtRealm(config, jwtRealmsService, sslService, userRoleMapper);
-        jwtRealm.initialize(Collections.singletonList(jwtRealm), super.licenseState);
+        final UserRoleMapper userRoleMapper = buildRoleMapper(jwtIssuer.principals);
+        final JwtRealm jwtRealm = new JwtRealm(config, sslService, userRoleMapper);
+        jwtRealm.initialize(Collections.singletonList(jwtRealm), licenseState);
         final JwtRealmSettingsBuilder jwtRealmSettingsBuilder = new JwtRealmSettingsBuilder(realmName, configBuilder);
         final JwtIssuerAndRealm jwtIssuerAndRealm = new JwtIssuerAndRealm(jwtIssuer, jwtRealm, jwtRealmSettingsBuilder);
-        super.jwtIssuerAndRealms = Collections.singletonList(jwtIssuerAndRealm); // super.shutdown() closes issuer+realm if necessary
-        super.printJwtRealmAndIssuer(jwtIssuerAndRealm);
+        jwtIssuerAndRealms = Collections.singletonList(jwtIssuerAndRealm); // shutdown() closes issuer+realm if necessary
+        printJwtRealmAndIssuer(jwtIssuerAndRealm);
 
         // Create JWT
-        final User user = this.randomUser(jwtIssuerAndRealm.issuer());
+        final User user = randomUser(jwtIssuerAndRealm.issuer());
         final SignedJWT unsignedJwt = JwtTestCase.buildUnsignedJwt(
             randomBoolean() ? null : JOSEObjectType.JWT.toString(), // kty
             randomBoolean() ? null : algJwkPairHmac.jwk().getKeyID(), // kid
@@ -143,12 +130,11 @@ public void testCreateJwtSmokeTestRealm() throws Exception {
             null, // nonce
             Collections.emptyMap() // other claims
         );
-        final SecureString jwt = JwtValidateUtil.signJwt(algJwkPairHmac.jwk(), unsignedJwt);
-        assertThat(JwtValidateUtil.verifyJwt(algJwkPairHmac.jwk(), SignedJWT.parse(jwt.toString())), is(equalTo(true)));
+        final SecureString jwt = signJwt(algJwkPairHmac.jwk(), unsignedJwt);
 
         // Verify authc+authz, then print all artifacts
-        super.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt, clientSecret, JWT_AUTHC_RANGE_1);
-        this.printArtifacts(jwtIssuer, config, clientSecret, jwt);
+        doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt, clientSecret, JWT_AUTHC_REPEATS_1);
+        printArtifacts(jwtIssuer, config, clientSecret, jwt);
     }
 
     /**
@@ -161,16 +147,11 @@ public void testCreateJwtIntegrationTestRealm1() throws Exception {
         final JwtIssuer.AlgJwkPair algJwkPairPkc = new JwtIssuer.AlgJwkPair("RS256", jwk);
 
         final String principalClaimName = "sub";
-        final Settings.Builder jwtRealmsServiceSettings = Settings.builder()
-            .put(this.globalSettings)
-            .put(JwtRealmsServiceSettings.PRINCIPAL_CLAIMS_SETTING.getKey(), String.join(",", principalClaimName));
-        final JwtRealmsService jwtRealmsService = new JwtRealmsService(jwtRealmsServiceSettings.build());
 
         // Create issuer
         final JwtIssuer jwtIssuer = new JwtIssuer(
             "https://issuer.example.com/", // iss claim value
             List.of("https://audience.example.com/"), // aud claim value
-            principalClaimName, // principal claim name
             Collections.singletonMap("user1", new User("user1", "role1")), // users
             false // createHttpsServer
         );
@@ -179,7 +160,7 @@ public void testCreateJwtIntegrationTestRealm1() throws Exception {
         // Create realm settings (no secure settings)
         final String realmName = "jwt1";
         final Settings.Builder configBuilder = Settings.builder()
-            .put(this.globalSettings)
+            .put(globalSettings)
             .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.ALLOWED_ISSUER), jwtIssuer.issuerClaimValue)
             .put(
                 RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.ALLOWED_AUDIENCES),
@@ -200,22 +181,22 @@ public void testCreateJwtIntegrationTestRealm1() throws Exception {
             )
             .put(
                 RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.PKC_JWKSET_PATH),
-                super.saveToTempFile("jwkset.", ".json", jwtIssuer.encodedJwkSetPkcPublic)
+                saveToTempFile("jwkset.", ".json", jwtIssuer.encodedJwkSetPkcPublic)
             );
 
         // Create realm
-        final RealmConfig config = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, configBuilder.build(), 2);
+        final RealmConfig config = buildRealmConfig(JwtRealmSettings.TYPE, realmName, configBuilder.build(), 2);
         final SSLService sslService = new SSLService(TestEnvironment.newEnvironment(configBuilder.build()));
-        final UserRoleMapper userRoleMapper = super.buildRoleMapper(jwtIssuer.principals);
-        final JwtRealm jwtRealm = new JwtRealm(config, jwtRealmsService, sslService, userRoleMapper);
-        jwtRealm.initialize(Collections.singletonList(jwtRealm), super.licenseState);
+        final UserRoleMapper userRoleMapper = buildRoleMapper(jwtIssuer.principals);
+        final JwtRealm jwtRealm = new JwtRealm(config, sslService, userRoleMapper);
+        jwtRealm.initialize(Collections.singletonList(jwtRealm), licenseState);
         final JwtRealmSettingsBuilder jwtRealmSettingsBuilder = new JwtRealmSettingsBuilder(realmName, configBuilder);
         final JwtIssuerAndRealm jwtIssuerAndRealm = new JwtIssuerAndRealm(jwtIssuer, jwtRealm, jwtRealmSettingsBuilder);
-        super.jwtIssuerAndRealms = Collections.singletonList(jwtIssuerAndRealm); // super.shutdown() closes issuer+realm if necessary
-        super.printJwtRealmAndIssuer(jwtIssuerAndRealm);
+        jwtIssuerAndRealms = Collections.singletonList(jwtIssuerAndRealm); // shutdown() closes issuer+realm if necessary
+        printJwtRealmAndIssuer(jwtIssuerAndRealm);
 
         // Create JWT
-        final User user = this.randomUser(jwtIssuerAndRealm.issuer());
+        final User user = randomUser(jwtIssuerAndRealm.issuer());
         final SignedJWT unsignedJwt = JwtTestCase.buildUnsignedJwt(
             randomBoolean() ? null : JOSEObjectType.JWT.toString(), // kty
             randomBoolean() ? null : algJwkPairPkc.jwk().getKeyID(), // kid
@@ -235,12 +216,11 @@ public void testCreateJwtIntegrationTestRealm1() throws Exception {
             null, // nonce
             Collections.emptyMap() // other claims
         );
-        final SecureString jwt = JwtValidateUtil.signJwt(algJwkPairPkc.jwk(), unsignedJwt);
-        assertThat(JwtValidateUtil.verifyJwt(algJwkPairPkc.jwk(), SignedJWT.parse(jwt.toString())), is(equalTo(true)));
+        final SecureString jwt = signJwt(algJwkPairPkc.jwk(), unsignedJwt);
 
         // Verify authc+authz, then print all artifacts
-        super.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt, null, JWT_AUTHC_RANGE_1);
-        this.printArtifacts(jwtIssuer, config, null, jwt);
+        doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt, null, JWT_AUTHC_REPEATS_1);
+        printArtifacts(jwtIssuer, config, null, jwt);
     }
 
     /**
@@ -256,16 +236,11 @@ public void testCreateJwtIntegrationTestRealm2() throws Exception {
         );
 
         final String principalClaimName = "email";
-        final Settings.Builder jwtRealmsServiceSettings = Settings.builder()
-            .put(this.globalSettings)
-            .put(JwtRealmsServiceSettings.PRINCIPAL_CLAIMS_SETTING.getKey(), String.join(",", principalClaimName));
-        final JwtRealmsService jwtRealmsService = new JwtRealmsService(jwtRealmsServiceSettings.build());
 
         // Create issuer
         final JwtIssuer jwtIssuer = new JwtIssuer(
             "my-issuer", // iss claim value
             List.of("es01", "es02", "es03"), // aud claim value
-            principalClaimName, // principal claim name
             Collections.singletonMap("user2", new User("user2", "role2")), // users
             false // createHttpsServer
         );
@@ -275,14 +250,14 @@ public void testCreateJwtIntegrationTestRealm2() throws Exception {
         final String realmName = "jwt2";
         final String authzRealmName = "lookup_native";
         final Settings.Builder configBuilder = Settings.builder()
-            .put(this.globalSettings)
+            .put(globalSettings)
             .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.ALLOWED_ISSUER), jwtIssuer.issuerClaimValue)
             .put(
                 RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.ALLOWED_AUDIENCES),
                 String.join(",", jwtIssuer.audiencesClaimValue)
             )
             .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.ALLOWED_SIGNATURE_ALGORITHMS), "HS256,HS384")
-            .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.CLAIMS_PRINCIPAL.getClaim()), "email")
+            .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.CLAIMS_PRINCIPAL.getClaim()), principalClaimName)
             .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.CLAIMS_PRINCIPAL.getPattern()), "^(.*)@[^.]*[.]example[.]com$")
             .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.CLAIMS_MAIL.getClaim()), "email")
             .put(
@@ -305,23 +280,23 @@ public void testCreateJwtIntegrationTestRealm2() throws Exception {
         configBuilder.setSecureSettings(secureSettings);
 
         // Create authz realm
-        final RealmConfig authzConfig = this.buildRealmConfig("native", authzRealmName, Settings.EMPTY, 0);
+        final RealmConfig authzConfig = buildRealmConfig("native", authzRealmName, Settings.EMPTY, 0);
         final MockLookupRealm authzRealm = new MockLookupRealm(authzConfig);
         jwtIssuer.principals.values().forEach(authzRealm::registerUser); // authz realm will do roles lookup
 
         // Create realm
-        final RealmConfig config = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, configBuilder.build(), 3);
+        final RealmConfig config = buildRealmConfig(JwtRealmSettings.TYPE, realmName, configBuilder.build(), 3);
         final SSLService sslService = new SSLService(TestEnvironment.newEnvironment(configBuilder.build()));
-        final UserRoleMapper userRoleMapper = super.buildRoleMapper(Map.of()); // authc realm will not do role mapping
-        final JwtRealm jwtRealm = new JwtRealm(config, jwtRealmsService, sslService, userRoleMapper);
-        jwtRealm.initialize(List.of(authzRealm, jwtRealm), super.licenseState);
+        final UserRoleMapper userRoleMapper = buildRoleMapper(Map.of()); // authc realm will not do role mapping
+        final JwtRealm jwtRealm = new JwtRealm(config, sslService, userRoleMapper);
+        jwtRealm.initialize(List.of(authzRealm, jwtRealm), licenseState);
         final JwtRealmSettingsBuilder jwtRealmSettingsBuilder = new JwtRealmSettingsBuilder(realmName, configBuilder);
         final JwtIssuerAndRealm jwtIssuerAndRealm = new JwtIssuerAndRealm(jwtIssuer, jwtRealm, jwtRealmSettingsBuilder);
-        super.jwtIssuerAndRealms = Collections.singletonList(jwtIssuerAndRealm); // super.shutdown() closes issuer+realm if necessary
-        super.printJwtRealmAndIssuer(jwtIssuerAndRealm);
+        jwtIssuerAndRealms = Collections.singletonList(jwtIssuerAndRealm); // shutdown() closes issuer+realm if necessary
+        printJwtRealmAndIssuer(jwtIssuerAndRealm);
 
         // Create JWT
-        final User user = this.randomUser(jwtIssuerAndRealm.issuer());
+        final User user = randomUser(jwtIssuerAndRealm.issuer());
         final SignedJWT unsignedJwt = JwtTestCase.buildUnsignedJwt(
             randomBoolean() ? null : JOSEObjectType.JWT.toString(), // kty
             randomBoolean() ? null : algJwkPairHmac.jwk().getKeyID(), // kid
@@ -341,12 +316,11 @@ public void testCreateJwtIntegrationTestRealm2() throws Exception {
             null, // nonce
             Collections.emptyMap() // other claims
         );
-        final SecureString jwt = JwtValidateUtil.signJwt(algJwkPairHmac.jwk(), unsignedJwt);
-        assertThat(JwtValidateUtil.verifyJwt(algJwkPairHmac.jwk(), SignedJWT.parse(jwt.toString())), is(equalTo(true)));
+        final SecureString jwt = signJwt(algJwkPairHmac.jwk(), unsignedJwt);
 
         // Verify authc+authz, then print all artifacts
-        super.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt, clientSecret, JWT_AUTHC_RANGE_1);
-        this.printArtifacts(jwtIssuer, config, clientSecret, jwt);
+        doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt, clientSecret, JWT_AUTHC_REPEATS_1);
+        printArtifacts(jwtIssuer, config, clientSecret, jwt);
     }
 
     /**
@@ -361,16 +335,11 @@ public void testCreateJwtIntegrationTestRealm3() throws Exception {
         );
 
         final String principalClaimName = "sub";
-        final Settings.Builder jwtRealmsServiceSettings = Settings.builder()
-            .put(this.globalSettings)
-            .put(JwtRealmsServiceSettings.PRINCIPAL_CLAIMS_SETTING.getKey(), String.join(",", principalClaimName));
-        final JwtRealmsService jwtRealmsService = new JwtRealmsService(jwtRealmsServiceSettings.build());
 
         // Create issuer
         final JwtIssuer jwtIssuer = new JwtIssuer(
             "jwt3-issuer", // iss claim value
             List.of("jwt3-audience"), // aud claim value
-            principalClaimName, // principal claim name
             Collections.singletonMap("user3", new User("user3", "role3")), // users
             false // createHttpsServer
         );
@@ -379,7 +348,7 @@ public void testCreateJwtIntegrationTestRealm3() throws Exception {
         // Create realm settings
         final String realmName = "jwt3";
         final Settings.Builder configBuilder = Settings.builder()
-            .put(this.globalSettings)
+            .put(globalSettings)
             .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.ALLOWED_ISSUER), jwtIssuer.issuerClaimValue)
             .put(
                 RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.ALLOWED_AUDIENCES),
@@ -406,19 +375,19 @@ public void testCreateJwtIntegrationTestRealm3() throws Exception {
         configBuilder.setSecureSettings(secureSettings);
 
         // Create realm
-        final RealmConfig config = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, configBuilder.build(), 4);
+        final RealmConfig config = buildRealmConfig(JwtRealmSettings.TYPE, realmName, configBuilder.build(), 4);
         final SSLService sslService = new SSLService(TestEnvironment.newEnvironment(configBuilder.build()));
-        final UserRoleMapper userRoleMapper = super.buildRoleMapper(jwtIssuer.principals);
-        final JwtRealm jwtRealm = new JwtRealm(config, jwtRealmsService, sslService, userRoleMapper);
-        jwtRealm.initialize(List.of(jwtRealm), super.licenseState);
+        final UserRoleMapper userRoleMapper = buildRoleMapper(jwtIssuer.principals);
+        final JwtRealm jwtRealm = new JwtRealm(config, sslService, userRoleMapper);
+        jwtRealm.initialize(List.of(jwtRealm), licenseState);
         final JwtRealmSettingsBuilder jwtRealmSettingsBuilder = new JwtRealmSettingsBuilder(realmName, configBuilder);
         final JwtIssuerAndRealm jwtIssuerAndRealm = new JwtIssuerAndRealm(jwtIssuer, jwtRealm, jwtRealmSettingsBuilder);
-        super.jwtIssuerAndRealms = Collections.singletonList(jwtIssuerAndRealm); // super.shutdown() closes issuer+realm if necessary
-        super.printJwtRealmAndIssuer(jwtIssuerAndRealm);
+        jwtIssuerAndRealms = Collections.singletonList(jwtIssuerAndRealm); // shutdown() closes issuer+realm if necessary
+        printJwtRealmAndIssuer(jwtIssuerAndRealm);
 
         // Create JWT
         final JwtIssuer.AlgJwkPair selectedHmac = randomFrom(hmacKeys);
-        final User user = this.randomUser(jwtIssuerAndRealm.issuer());
+        final User user = randomUser(jwtIssuerAndRealm.issuer());
         final SignedJWT unsignedJwt = JwtTestCase.buildUnsignedJwt(
             randomBoolean() ? null : JOSEObjectType.JWT.toString(), // kty
             randomBoolean() ? null : selectedHmac.jwk().getKeyID(), // kid
@@ -438,12 +407,11 @@ public void testCreateJwtIntegrationTestRealm3() throws Exception {
             null, // nonce
             Collections.emptyMap() // other claims
         );
-        final SecureString jwt = JwtValidateUtil.signJwt(selectedHmac.jwk(), unsignedJwt);
-        assertThat(JwtValidateUtil.verifyJwt(selectedHmac.jwk(), SignedJWT.parse(jwt.toString())), is(equalTo(true)));
+        final SecureString jwt = signJwt(selectedHmac.jwk(), unsignedJwt);
 
         // Verify authc+authz, then print all artifacts
-        super.doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt, clientSecret, JWT_AUTHC_RANGE_1);
-        this.printArtifacts(jwtIssuer, config, clientSecret, jwt);
+        doMultipleAuthcAuthzAndVerifySuccess(jwtIssuerAndRealm.realm(), user, jwt, clientSecret, JWT_AUTHC_REPEATS_1);
+        printArtifacts(jwtIssuer, config, clientSecret, jwt);
     }
 
     private void printArtifacts(
@@ -453,7 +421,7 @@ private void printArtifacts(
         final SecureString jwt
     ) throws Exception {
         final SignedJWT signedJwt = SignedJWT.parse(jwt.toString());
-        LOGGER.info(
+        logger.info(
             JwtRealmGenerateTests.printIssuerSettings(jwtIssuer)
                 + JwtRealmGenerateTests.printRealmSettings(config)
                 + "\n===\nRequest Headers\n===\n"
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmInspector.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmInspector.java
index 3e0cccad7b9c..7697849179ac 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmInspector.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmInspector.java
@@ -20,6 +20,10 @@ class JwtRealmInspector {
 
     private JwtRealmInspector() {}
 
+    public static JwtRealmSettings.TokenType getTokenType(JwtRealm realm) {
+        return realm.getConfig().getSetting(JwtRealmSettings.TOKEN_TYPE);
+    }
+
     public static String getJwkSetPath(JwtRealm realm) {
         return realm.getConfig().getSetting(JwtRealmSettings.PKC_JWKSET_PATH);
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSettingsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSettingsTests.java
index 49d38038d24c..894bfc6e13d5 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSettingsTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSettingsTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
@@ -22,7 +23,9 @@
 import java.util.Locale;
 
 import static org.elasticsearch.common.Strings.capitalize;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.emptyIterable;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
@@ -40,8 +43,8 @@ public class JwtRealmSettingsTests extends JwtTestCase {
 
     public void testAllSettings() throws Exception {
         final String realmName = "jwt" + randomIntBetween(1, 9);
-        final Settings settings = super.generateRandomRealmSettings(realmName).build();
-        final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+        final Settings settings = generateRandomRealmSettings(realmName).build();
+        final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
         for (final Setting.AffixSetting<?> setting : JwtRealmSettings.getSettings()) {
             realmConfig.getConcreteSetting(setting);
         }
@@ -54,7 +57,7 @@ public void testAllowedIssuer() {
         for (final String rejectedValue : new String[] { null, "" }) {
             final Exception exception = expectThrows(IllegalArgumentException.class, () -> {
                 final Settings settings = Settings.builder().put(settingKey, rejectedValue).build();
-                final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+                final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
                 final String actualValue = realmConfig.getSetting(setting);
                 fail("No exception. Expected one for " + settingKey + "=" + rejectedValue + ". Got " + actualValue + ".");
             });
@@ -62,7 +65,7 @@ public void testAllowedIssuer() {
         }
         for (final String acceptedValue : new String[] { "http://localhost/iss1", "issuer1", "i" }) {
             final Settings settings = Settings.builder().put(settingKey, acceptedValue).build();
-            final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+            final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
             final String actualValue = realmConfig.getSetting(setting);
             assertThat(actualValue, equalTo(acceptedValue));
         }
@@ -75,7 +78,7 @@ public void testAllowedSignatureAlgorithms() {
         for (final String rejectedValue : new String[] { "unknown", "HS256,unknown" }) {
             final Exception exception = expectThrows(IllegalArgumentException.class, () -> {
                 final Settings settings = Settings.builder().put(settingKey, rejectedValue).build();
-                final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+                final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
                 final List<String> actualValue = realmConfig.getSetting(setting);
                 fail("No exception. Expected one for " + settingKey + "=" + rejectedValue + ". Got " + actualValue + ".");
             });
@@ -93,14 +96,14 @@ public void testAllowedSignatureAlgorithms() {
         }
         for (final String ignoredValue : new String[] { null, "" }) {
             final Settings settings = Settings.builder().put(settingKey, ignoredValue).build();
-            final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+            final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
             final List<String> actualValue = realmConfig.getSetting(setting);
             assertThat(actualValue, equalTo(setting.getDefault(settings)));
         }
         final String allAcceptedValues = JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS.toString().replaceAll("[\\[\\] ]", "");
         for (final String acceptedValue : List.of("HS256", "HS512,RS512", allAcceptedValues)) {
             final Settings settings = Settings.builder().put(settingKey, acceptedValue).build();
-            final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+            final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
             final List<String> actualValue = realmConfig.getSetting(setting);
             assertThat(actualValue, equalTo(Arrays.asList(acceptedValue.split(",", -1))));
         }
@@ -112,13 +115,13 @@ public void testJwtPath() {
         final String settingKey = RealmSettings.getFullSettingKey(realmName, setting);
         for (final String ignoredValue : new String[] { null, "" }) {
             final Settings settings = Settings.builder().put(settingKey, ignoredValue).build();
-            final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+            final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
             final String actualValue = realmConfig.getSetting(setting);
             assertThat(actualValue, equalTo(setting.getDefault(settings)));
         }
         for (final String acceptedValue : new String[] { "./config/jwkset.json", "http://localhost/jwkset.json" }) {
             final Settings settings = Settings.builder().put(settingKey, acceptedValue).build();
-            final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+            final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
             final String actualValue = realmConfig.getSetting(setting);
             assertThat(actualValue, equalTo(acceptedValue));
         }
@@ -131,7 +134,7 @@ public void testAllowedAudiences() {
         for (final String rejectedValue : new String[] { null, "" }) {
             final Exception exception = expectThrows(IllegalArgumentException.class, () -> {
                 final Settings settings = Settings.builder().put(settingKey, rejectedValue).build();
-                final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+                final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
                 final List<String> actualValue = realmConfig.getSetting(setting);
                 fail("No exception. Expected one for " + settingKey + "=" + rejectedValue + ". Got " + actualValue + ".");
             });
@@ -139,7 +142,7 @@ public void testAllowedAudiences() {
         }
         for (final String acceptedValue : new String[] { "elasticsearch", "elasticsearch,other" }) {
             final Settings settings = Settings.builder().put(settingKey, acceptedValue).build();
-            final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+            final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
             final List<String> actualValue = realmConfig.getSetting(setting);
             assertThat(actualValue, equalTo(Arrays.asList(acceptedValue.split(",", -1))));
         }
@@ -158,7 +161,7 @@ public void testClaimNames() {
             for (final String rejectedValue : new String[] { null, "" }) {
                 final Exception exception = expectThrows(IllegalArgumentException.class, () -> {
                     final Settings settings = Settings.builder().put(settingKey, rejectedValue).build();
-                    final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+                    final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
                     final String actualValue = realmConfig.getSetting(setting);
                     fail("No exception. Expected one for " + settingKey + "=" + rejectedValue + ". Got " + actualValue + ".");
                 });
@@ -166,7 +169,7 @@ public void testClaimNames() {
             }
             for (final String acceptedValue : new String[] { "sub", "name", "email", "dn" }) {
                 final Settings settings = Settings.builder().put(settingKey, acceptedValue).build();
-                final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+                final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
                 final String actualValue = realmConfig.getSetting(setting);
                 assertThat(actualValue, equalTo(acceptedValue));
             }
@@ -186,7 +189,7 @@ public void testClaimPatterns() {
             for (final String rejectedValue : new String[] { "[" }) {
                 final Exception exception = expectThrows(IllegalArgumentException.class, () -> {
                     final Settings settings = Settings.builder().put(settingKey, rejectedValue).build();
-                    final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+                    final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
                     final String actualValue = realmConfig.getSetting(setting);
                     fail("No exception. Expected one for " + settingKey + "=" + rejectedValue + ". Got " + actualValue + ".");
                 });
@@ -194,7 +197,7 @@ public void testClaimPatterns() {
             }
             for (final String acceptedValue : new String[] { "^([^@]+)@example\\.com$", "^Group-(.+)$" }) {
                 final Settings settings = Settings.builder().put(settingKey, acceptedValue).build();
-                final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+                final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
                 final String actualValue = realmConfig.getSetting(setting);
                 assertThat(actualValue, equalTo(acceptedValue));
             }
@@ -208,7 +211,7 @@ public void testPopulateUserMetadata() {
         for (final String rejectedValue : new String[] { "", "unknown", "t", "f", "TRUE", "FALSE", "True", "False" }) {
             final Exception exception = expectThrows(IllegalArgumentException.class, () -> {
                 final Settings settings = Settings.builder().put(settingKey, rejectedValue).build();
-                final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+                final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
                 final Boolean actualValue = realmConfig.getSetting(setting);
                 fail("No exception. Expected one for " + settingKey + "=" + rejectedValue + ". Got " + actualValue + ".");
             });
@@ -219,13 +222,13 @@ public void testPopulateUserMetadata() {
         }
         for (final String ignoredValue : new String[] { null }) {
             final Settings settings = Settings.builder().put(settingKey, ignoredValue).build();
-            final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+            final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
             final Boolean actualValue = realmConfig.getSetting(setting);
             assertThat(actualValue, equalTo(setting.getDefault(settings)));
         }
         for (final String acceptedValue : new String[] { "true", "false" }) {
             final Settings settings = Settings.builder().put(settingKey, acceptedValue).build();
-            final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+            final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
             final Boolean actualValue = realmConfig.getSetting(setting);
             assertThat(actualValue, equalTo(Boolean.valueOf(acceptedValue)));
         }
@@ -238,7 +241,7 @@ public void testClientAuthenticationType() {
         for (final String rejectedValue : new String[] { "unknown", "", randomAlphaOfLengthBetween(1, 3) }) {
             final Exception exception = expectThrows(IllegalArgumentException.class, () -> {
                 final Settings settings = Settings.builder().put(settingKey, rejectedValue).build();
-                final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+                final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
                 final ClientAuthenticationType actualValue = realmConfig.getSetting(setting);
                 fail("No exception. Expected one for " + settingKey + "=" + rejectedValue + ". Got " + actualValue + ".");
             });
@@ -249,14 +252,14 @@ public void testClientAuthenticationType() {
         }
         for (final String ignoredValue : new String[] { null }) {
             final Settings settings = Settings.builder().put(settingKey, ignoredValue).build();
-            final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+            final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
             final ClientAuthenticationType actualValue = realmConfig.getSetting(setting);
             assertThat(actualValue, equalTo(setting.getDefault(settings)));
         }
         for (final String acceptedValue : new String[] { "shared_secret", "none" }) {
             for (String inputValue : new String[] { acceptedValue, acceptedValue.toUpperCase(Locale.ROOT), capitalize(acceptedValue) }) {
                 final Settings settings = Settings.builder().put(settingKey, inputValue).build();
-                final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+                final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
                 final ClientAuthenticationType actualValue = realmConfig.getSetting(setting);
                 assertThat(actualValue.value(), equalTo(acceptedValue));
             }
@@ -269,13 +272,13 @@ public void testAuthenticationRealms() {
         final String settingKey = RealmSettings.getFullSettingKey(realmName, setting);
         for (final String ignoredValue : new String[] { null, "" }) {
             final Settings settings = Settings.builder().put(settingKey, ignoredValue).build();
-            final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+            final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
             final List<String> actualValue = realmConfig.getSetting(setting);
             assertThat(actualValue, equalTo(List.of()));
         }
         for (final String acceptedValue : new String[] { "a", "1", "native1,file1,ldap1,ad1" }) {
             final Settings settings = Settings.builder().put(settingKey, acceptedValue).build();
-            final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+            final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
             final List<String> actualValue = realmConfig.getSetting(setting);
             assertThat(actualValue, equalTo(Arrays.asList(acceptedValue.split(",", -1))));
         }
@@ -321,7 +324,7 @@ public void testTimeSettingsWithDefault() {
             for (final String rejectedValue : new String[] { "", "-2", "10", "1w", "1M", "1y" }) {
                 final Exception exception = expectThrows(IllegalArgumentException.class, () -> {
                     final Settings settings = Settings.builder().put(settingKey, rejectedValue).build();
-                    final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+                    final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
                     final TimeValue actualValue = realmConfig.getSetting(setting);
                     fail("No exception. Expected one for " + settingKey + "=" + rejectedValue + ". Got " + actualValue + ".");
                 });
@@ -338,13 +341,13 @@ public void testTimeSettingsWithDefault() {
             }
             for (final String ignoredValue : new String[] { null }) {
                 final Settings settings = Settings.builder().put(settingKey, ignoredValue).build();
-                final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+                final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
                 final TimeValue actualValue = realmConfig.getSetting(setting);
                 assertThat(actualValue, equalTo(setting.getDefault(settings)));
             }
             for (final String acceptedValue : new String[] { "-1", "0", "0s", "1s", "1m", "1h", "1d" }) {
                 final Settings settings = Settings.builder().put(settingKey, acceptedValue).build();
-                final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+                final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
                 final TimeValue actualValue = realmConfig.getSetting(setting);
                 assertThat(actualValue, equalTo(TimeValue.parseTimeValue(acceptedValue, settingKey)));
             }
@@ -362,7 +365,7 @@ public void testIntegerSettingsWithDefault() {
             for (final String rejectedValue : new String[] { "", "100_000", "NaN" }) {
                 final Exception exception = expectThrows(IllegalArgumentException.class, () -> {
                     final Settings settings = Settings.builder().put(settingKey, rejectedValue).build();
-                    final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+                    final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
                     final Integer actualValue = realmConfig.getSetting(setting);
                     fail("No exception. Expected one for " + settingKey + "=" + rejectedValue + ". Got " + actualValue + ".");
                 });
@@ -375,7 +378,7 @@ public void testIntegerSettingsWithDefault() {
             for (final String rejectedValue : new String[] { "-1", Integer.toString(Integer.MIN_VALUE) }) {
                 final Exception exception = expectThrows(IllegalArgumentException.class, () -> {
                     final Settings settings = Settings.builder().put(settingKey, rejectedValue).build();
-                    final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+                    final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
                     final Integer actualValue = realmConfig.getSetting(setting);
                     fail("No exception. Expected one for " + settingKey + "=" + rejectedValue + ". Got " + actualValue + ".");
                 });
@@ -386,13 +389,13 @@ public void testIntegerSettingsWithDefault() {
             }
             for (final String ignoredValue : new String[] { null }) {
                 final Settings settings = Settings.builder().put(settingKey, ignoredValue).build();
-                final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+                final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
                 final Integer actualValue = realmConfig.getSetting(setting);
                 assertThat(actualValue, equalTo(setting.getDefault(settings)));
             }
             for (final String acceptedValue : new String[] { "0", "1", "100000", Integer.toString(Integer.MAX_VALUE) }) {
                 final Settings settings = Settings.builder().put(settingKey, acceptedValue).build();
-                final RealmConfig realmConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
+                final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, 0);
                 final Integer actualValue = realmConfig.getSetting(setting);
                 assertThat(actualValue, equalTo(Integer.valueOf(acceptedValue)));
             }
@@ -428,4 +431,161 @@ public void testTokenTypeSetting() {
         );
         assertThat(e.getMessage(), containsString("Invalid value"));
     }
+
+    public void testFallbackClaimSettingsNotAllowedForIdTokenType() {
+        final String realmName = randomAlphaOfLengthBetween(3, 8);
+        final Settings.Builder settingsBuilder = Settings.builder();
+        if (randomBoolean()) {
+            settingsBuilder.put(
+                RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.TOKEN_TYPE),
+                JwtRealmSettings.TokenType.ID_TOKEN.value()
+            );
+        }
+        settingsBuilder.put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.FALLBACK_SUB_CLAIM), randomAlphaOfLength(8))
+            .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.FALLBACK_AUD_CLAIM), randomAlphaOfLength(8));
+
+        final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settingsBuilder.build(), randomInt());
+
+        final IllegalArgumentException e1 = expectThrows(
+            IllegalArgumentException.class,
+            () -> realmConfig.getSetting(JwtRealmSettings.FALLBACK_SUB_CLAIM)
+        );
+        assertThat(
+            e1.getMessage(),
+            containsString(
+                "fallback claim setting ["
+                    + RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.FALLBACK_SUB_CLAIM)
+                    + "] is not allowed when JWT realm ["
+                    + realmName
+                    + "] is [id_token] type"
+            )
+        );
+
+        final IllegalArgumentException e2 = expectThrows(
+            IllegalArgumentException.class,
+            () -> realmConfig.getSetting(JwtRealmSettings.FALLBACK_AUD_CLAIM)
+        );
+        assertThat(
+            e2.getMessage(),
+            containsString(
+                "fallback claim setting ["
+                    + RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.FALLBACK_AUD_CLAIM)
+                    + "] is not allowed when JWT realm ["
+                    + realmName
+                    + "] is [id_token] type"
+            )
+        );
+    }
+
+    public void testFallbackSettingsForAccessTokenType() {
+        final String realmName = randomAlphaOfLengthBetween(3, 8);
+        final String fallbackSub = randomAlphaOfLength(8);
+        final String fallbackAud = randomAlphaOfLength(8);
+        final Settings settings = Settings.builder()
+            .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.TOKEN_TYPE), JwtRealmSettings.TokenType.ACCESS_TOKEN.value())
+            .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.FALLBACK_SUB_CLAIM), fallbackSub)
+            .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.FALLBACK_AUD_CLAIM), fallbackAud)
+            .build();
+
+        final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, randomInt());
+        assertThat(realmConfig.getSetting(JwtRealmSettings.FALLBACK_SUB_CLAIM), equalTo(fallbackSub));
+        assertThat(realmConfig.getSetting(JwtRealmSettings.FALLBACK_AUD_CLAIM), equalTo(fallbackAud));
+    }
+
+    public void testRegisteredClaimsCannotBeUsedForFallbackSettings() {
+        final String realmName = randomAlphaOfLengthBetween(3, 8);
+        final String fallbackSub = randomValueOtherThan("sub", () -> randomFrom(JwtRealmSettings.REGISTERED_CLAIM_NAMES));
+        final String fallbackAud = randomValueOtherThan("aud", () -> randomFrom(JwtRealmSettings.REGISTERED_CLAIM_NAMES));
+        final Settings settings = Settings.builder()
+            .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.TOKEN_TYPE), JwtRealmSettings.TokenType.ACCESS_TOKEN.value())
+            .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.FALLBACK_SUB_CLAIM), fallbackSub)
+            .put(RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.FALLBACK_AUD_CLAIM), fallbackAud)
+            .build();
+
+        final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, randomInt());
+
+        final IllegalArgumentException e1 = expectThrows(
+            IllegalArgumentException.class,
+            () -> realmConfig.getSetting(JwtRealmSettings.FALLBACK_SUB_CLAIM)
+        );
+        assertThat(
+            e1.getMessage(),
+            containsString(
+                Strings.format(
+                    "Invalid fallback claims setting [%s]. Claim [%s] cannot fallback to a registered claim [%s]",
+                    RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.FALLBACK_SUB_CLAIM),
+                    "sub",
+                    fallbackSub
+                )
+            )
+        );
+
+        final IllegalArgumentException e2 = expectThrows(
+            IllegalArgumentException.class,
+            () -> realmConfig.getSetting(JwtRealmSettings.FALLBACK_AUD_CLAIM)
+        );
+        assertThat(
+            e2.getMessage(),
+            containsString(
+                Strings.format(
+                    "Invalid fallback claims setting [%s]. Claim [%s] cannot fallback to a registered claim [%s]",
+                    RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.FALLBACK_AUD_CLAIM),
+                    "aud",
+                    fallbackAud
+                )
+            )
+        );
+    }
+
+    public void testRequiredClaims() {
+        final String realmName = randomAlphaOfLengthBetween(3, 8);
+
+        // Required claims are optional
+        final RealmConfig realmConfig1 = buildRealmConfig(JwtRealmSettings.TYPE, realmName, Settings.EMPTY, randomInt());
+        assertThat(realmConfig1.getSetting(JwtRealmSettings.REQUIRED_CLAIMS).names(), emptyIterable());
+
+        // Multiple required claims with different value types
+        final String prefix = RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.REQUIRED_CLAIMS);
+        final Settings settings = Settings.builder()
+            .put(prefix + "extra_1", "foo")
+            .put(prefix + "extra_2", "hello,world")
+            .put(prefix + "extra_3", 42)
+            .build();
+        final RealmConfig realmConfig2 = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, randomInt());
+        final Settings requireClaimsSettings = realmConfig2.getSetting(JwtRealmSettings.REQUIRED_CLAIMS);
+        assertThat(requireClaimsSettings.names(), containsInAnyOrder("extra_1", "extra_2", "extra_3"));
+        assertThat(requireClaimsSettings.getAsList("extra_1"), equalTo(List.of("foo")));
+        assertThat(requireClaimsSettings.getAsList("extra_2"), equalTo(List.of("hello", "world")));
+        assertThat(requireClaimsSettings.getAsList("extra_3"), equalTo(List.of("42")));
+    }
+
+    public void testInvalidRequiredClaims() {
+        final String realmName = randomAlphaOfLengthBetween(3, 8);
+        final String invalidRequiredClaim = randomFrom("iss", "sub", "aud", "exp", "nbf", "iat");
+        final String fullSettingKey = RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.REQUIRED_CLAIMS) + invalidRequiredClaim;
+        final Settings settings = Settings.builder().put(fullSettingKey, randomAlphaOfLength(8)).build();
+
+        final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, randomInt());
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> realmConfig.getSetting(JwtRealmSettings.REQUIRED_CLAIMS)
+        );
+
+        assertThat(e.getMessage(), containsString("required claim [" + fullSettingKey + "] cannot be one of [iss,sub,aud,exp,nbf,iat]"));
+    }
+
+    public void testRequiredClaimsCannotBeEmpty() {
+        final String realmName = randomAlphaOfLengthBetween(3, 8);
+        final String invalidRequiredClaim = randomAlphaOfLengthBetween(4, 8);
+        final String fullSettingKey = RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.REQUIRED_CLAIMS) + invalidRequiredClaim;
+        final Settings settings = Settings.builder().put(fullSettingKey, "").build();
+
+        final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, randomInt());
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> realmConfig.getSetting(JwtRealmSettings.REQUIRED_CLAIMS)
+        );
+
+        assertThat(e.getMessage(), containsString("required claim [" + fullSettingKey + "] cannot be empty"));
+    }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmTestCase.java
index d6f170861483..ab1b7867ffa0 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmTestCase.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmTestCase.java
@@ -11,8 +11,6 @@
 import com.nimbusds.jwt.SignedJWT;
 import com.nimbusds.openid.connect.sdk.Nonce;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.SecureString;
@@ -25,13 +23,11 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
-import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.core.security.authc.Realm;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
 import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings.ClientAuthenticationType;
-import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmsServiceSettings;
 import org.elasticsearch.xpack.core.security.authc.support.DelegatedAuthorizationSettings;
 import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.core.security.user.User;
@@ -59,33 +55,20 @@
 import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.isA;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.startsWith;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 public abstract class JwtRealmTestCase extends JwtTestCase {
-    private static final Logger LOGGER = LogManager.getLogger(JwtRealmTestCase.class);
-
-    record JwtRealmsServiceSettingsBuilder(Settings.Builder settingsBuilder) {}
-
     record JwtRealmSettingsBuilder(String name, Settings.Builder settingsBuilder) {}
 
     record JwtIssuerAndRealm(JwtIssuer issuer, JwtRealm realm, JwtRealmSettingsBuilder realmSettingsBuilder) {}
 
-    record MinMax(int min, int max) {
-        MinMax {
-            assert min >= 0 && max >= min : "Invalid min=" + min + " max=" + max;
-        }
-    }
-
     protected ThreadPool threadPool;
     protected ResourceWatcherService resourceWatcherService;
     protected MockLicenseState licenseState;
@@ -93,22 +76,22 @@ record MinMax(int min, int max) {
 
     @Before
     public void init() throws Exception {
-        this.threadPool = new TestThreadPool("JWT realm tests");
-        this.resourceWatcherService = new ResourceWatcherService(Settings.EMPTY, this.threadPool);
-        this.licenseState = mock(MockLicenseState.class);
-        when(this.licenseState.isAllowed(Security.DELEGATED_AUTHORIZATION_FEATURE)).thenReturn(true);
+        threadPool = new TestThreadPool("JWT realm tests");
+        resourceWatcherService = new ResourceWatcherService(Settings.EMPTY, threadPool);
+        licenseState = mock(MockLicenseState.class);
+        when(licenseState.isAllowed(Security.DELEGATED_AUTHORIZATION_FEATURE)).thenReturn(true);
     }
 
     @After
     public void shutdown() throws Exception {
-        if (this.jwtIssuerAndRealms != null) {
-            this.jwtIssuerAndRealms.forEach(jwtIssuerAndRealm -> {
+        if (jwtIssuerAndRealms != null) {
+            jwtIssuerAndRealms.forEach(jwtIssuerAndRealm -> {
                 jwtIssuerAndRealm.realm.close(); // Close HTTPS client (if any)
                 jwtIssuerAndRealm.issuer.close(); // Close HTTPS server (if any)
             });
         }
-        this.resourceWatcherService.close();
-        terminate(this.threadPool);
+        resourceWatcherService.close();
+        terminate(threadPool);
     }
 
     protected void verifyAuthenticateFailureHelper(
@@ -116,87 +99,53 @@ protected void verifyAuthenticateFailureHelper(
         final SecureString jwt,
         final SecureString clientSecret
     ) throws InterruptedException, ExecutionException {
-        final ThreadContext tc = super.createThreadContext(jwt, clientSecret);
+        final ThreadContext tc = createThreadContext(jwt, clientSecret);
         final JwtAuthenticationToken token = (JwtAuthenticationToken) jwtIssuerAndRealm.realm.token(tc);
         final PlainActionFuture<AuthenticationResult<User>> plainActionFuture = PlainActionFuture.newFuture();
         jwtIssuerAndRealm.realm.authenticate(token, plainActionFuture);
-        assertThat(plainActionFuture.get(), is(notNullValue()));
+        assertThat(plainActionFuture.get(), notNullValue());
         assertThat(plainActionFuture.get().isAuthenticated(), is(false));
     }
 
-    protected JwtRealmsService generateJwtRealmsService(final JwtRealmsServiceSettingsBuilder jwtRealmRealmsSettingsBuilder) {
-        return new JwtRealmsService(jwtRealmRealmsSettingsBuilder.settingsBuilder.build());
-    }
-
     protected List<JwtIssuerAndRealm> generateJwtIssuerRealmPairs(
-        final JwtRealmsServiceSettingsBuilder jwtRealmsServiceSettingsBuilder,
-        final MinMax realmsRange,
-        final MinMax authzRange,
-        final MinMax algsRange,
-        final MinMax audiencesRange,
-        final MinMax usersRange,
-        final MinMax rolesRange,
-        final MinMax jwtCacheSizeRange,
+        final int realmsCount,
+        final int authzCount,
+        final int algsCount,
+        final int audiencesCount,
+        final int usersCount,
+        final int rolesCount,
+        final int jwtCacheSize,
         final boolean createHttpsServer
     ) throws Exception {
-        assertThat(realmsRange.min(), is(greaterThanOrEqualTo(1)));
-        assertThat(authzRange.min(), is(greaterThanOrEqualTo(0)));
-        assertThat(algsRange.min(), is(greaterThanOrEqualTo(1)));
-        assertThat(audiencesRange.min(), is(greaterThanOrEqualTo(1)));
-        assertThat(usersRange.min(), is(greaterThanOrEqualTo(1)));
-        assertThat(rolesRange.min(), is(greaterThanOrEqualTo(0)));
-        assertThat(jwtCacheSizeRange.min(), is(greaterThanOrEqualTo(0)));
-
         // Create JWT authc realms and mocked authz realms. Initialize each JWT realm, and test ensureInitialized() before and after.
-        final JwtRealmsService jwtRealmsService = this.generateJwtRealmsService(jwtRealmsServiceSettingsBuilder);
-        final int realmsCount = randomIntBetween(realmsRange.min(), realmsRange.max());
         final List<Realm> allRealms = new ArrayList<>(); // authc and authz realms
-        this.jwtIssuerAndRealms = new ArrayList<>(realmsCount);
+        jwtIssuerAndRealms = new ArrayList<>(realmsCount);
         for (int i = 0; i < realmsCount; i++) {
-            final int authzCount = randomIntBetween(authzRange.min(), authzRange.max());
-            final int algsCount = randomIntBetween(algsRange.min(), algsRange.max());
-            final int audiencesCount = randomIntBetween(audiencesRange.min(), audiencesRange.max());
-            final int usersCount = randomIntBetween(usersRange.min(), usersRange.max());
-            final int rolesCount = randomIntBetween(rolesRange.min(), rolesRange.max());
-            final int jwtCacheSize = randomIntBetween(jwtCacheSizeRange.min(), jwtCacheSizeRange.max());
-
-            final JwtIssuer jwtIssuer = this.createJwtIssuer(
-                i,
-                randomFrom(jwtRealmsService.getPrincipalClaimNames()),
-                algsCount,
-                audiencesCount,
-                usersCount,
-                rolesCount,
-                createHttpsServer
-            );
+
+            final JwtIssuer jwtIssuer = createJwtIssuer(i, algsCount, audiencesCount, usersCount, rolesCount, createHttpsServer);
             // If HTTPS server was created in JWT issuer, any exception after that point requires closing it to avoid a thread pool leak
             try {
-                final JwtRealmSettingsBuilder realmSettingsBuilder = this.createJwtRealmSettingsBuilder(
-                    jwtIssuer,
-                    authzCount,
-                    jwtCacheSize
-                );
-                final JwtRealm jwtRealm = this.createJwtRealm(allRealms, jwtRealmsService, jwtIssuer, realmSettingsBuilder);
+                final JwtRealmSettingsBuilder realmSettingsBuilder = createJwtRealmSettingsBuilder(jwtIssuer, authzCount, jwtCacheSize);
+                final JwtRealm jwtRealm = createJwtRealm(allRealms, jwtIssuer, realmSettingsBuilder);
 
                 // verify exception before initialize()
                 final Exception exception = expectThrows(IllegalStateException.class, jwtRealm::ensureInitialized);
                 assertThat(exception.getMessage(), equalTo("Realm has not been initialized"));
 
                 final JwtIssuerAndRealm jwtIssuerAndRealm = new JwtIssuerAndRealm(jwtIssuer, jwtRealm, realmSettingsBuilder);
-                this.jwtIssuerAndRealms.add(jwtIssuerAndRealm);
+                jwtIssuerAndRealms.add(jwtIssuerAndRealm);
             } catch (Throwable t) {
                 jwtIssuer.close();
                 throw t;
             }
         }
-        allRealms.forEach(realm -> realm.initialize(allRealms, this.licenseState)); // JWT realms and authz realms
-        this.jwtIssuerAndRealms.forEach(p -> p.realm.ensureInitialized()); // verify no exception after initialize()
-        return this.jwtIssuerAndRealms;
+        allRealms.forEach(realm -> realm.initialize(allRealms, licenseState)); // JWT realms and authz realms
+        jwtIssuerAndRealms.forEach(p -> p.realm.ensureInitialized()); // verify no exception after initialize()
+        return jwtIssuerAndRealms;
     }
 
     protected JwtIssuer createJwtIssuer(
         final int i,
-        final String principalClaimName,
         final int algsCount,
         final int audiencesCount,
         final int userCount,
@@ -207,7 +156,7 @@ protected JwtIssuer createJwtIssuer(
         final List<String> audiences = IntStream.range(0, audiencesCount).mapToObj(j -> issuer + "_aud" + (j + 1)).toList();
         final Map<String, User> users = JwtTestCase.generateTestUsersWithRoles(userCount, roleCount);
         // Allow algorithm repeats, to cover testing of multiple JWKs for same algorithm
-        final JwtIssuer jwtIssuer = new JwtIssuer(issuer, audiences, principalClaimName, users, createHttpsServer);
+        final JwtIssuer jwtIssuer = new JwtIssuer(issuer, audiences, users, createHttpsServer);
         final List<String> algorithms = randomOfMinMaxNonUnique(algsCount, algsCount, JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS);
         final boolean areHmacJwksOidcSafe = randomBoolean();
         final List<JwtIssuer.AlgJwkPair> algAndJwks = JwtRealmTestCase.randomJwks(algorithms, areHmacJwksOidcSafe);
@@ -218,7 +167,7 @@ protected JwtIssuer createJwtIssuer(
     protected void copyIssuerJwksToRealmConfig(final JwtIssuerAndRealm jwtIssuerAndRealm) throws Exception {
         if (JwtRealmInspector.isConfiguredJwkSetPkc(jwtIssuerAndRealm.realm)
             && (JwtRealmInspector.getJwkSetPathUri(jwtIssuerAndRealm.realm) == null)) {
-            LOGGER.trace("Updating JwtRealm PKC public JWKSet local file");
+            logger.trace("Updating JwtRealm PKC public JWKSet local file");
             final Path path = PathUtils.get(JwtRealmInspector.getJwkSetPath(jwtIssuerAndRealm.realm));
             Files.writeString(path, jwtIssuerAndRealm.issuer.encodedJwkSetPkcPublic);
         }
@@ -226,21 +175,6 @@ protected void copyIssuerJwksToRealmConfig(final JwtIssuerAndRealm jwtIssuerAndR
         // TODO If x-pack Security plug-in add supports for reloadable settings, update HMAC JWKSet and HMAC OIDC JWK in ES Keystore
     }
 
-    protected JwtRealmsServiceSettingsBuilder createJwtRealmsSettingsBuilder() throws Exception {
-        final List<String> principalClaimNames = randomBoolean()
-            ? List.of("principalClaim_" + randomAlphaOfLength(6))
-            : randomSubsetOf(randomIntBetween(1, 6), JwtRealmsServiceSettings.DEFAULT_PRINCIPAL_CLAIMS);
-
-        final Settings.Builder jwtRealmsServiceSettings = Settings.builder()
-            .put(this.globalSettings)
-            .put(JwtRealmsServiceSettings.PRINCIPAL_CLAIMS_SETTING.getKey(), String.join(",", principalClaimNames));
-
-        final MockSecureSettings secureSettings = new MockSecureSettings(); // none for now, placeholder for future
-        jwtRealmsServiceSettings.setSecureSettings(secureSettings);
-
-        return new JwtRealmsServiceSettingsBuilder(jwtRealmsServiceSettings);
-    }
-
     protected JwtRealmSettingsBuilder createJwtRealmSettingsBuilder(final JwtIssuer jwtIssuer, final int authzCount, final int jwtCacheSize)
         throws Exception {
         final String authcRealmName = "realm_" + jwtIssuer.issuerClaimValue;
@@ -248,7 +182,7 @@ protected JwtRealmSettingsBuilder createJwtRealmSettingsBuilder(final JwtIssuer
 
         final ClientAuthenticationType clientAuthenticationType = randomFrom(ClientAuthenticationType.values());
         final Settings.Builder authcSettings = Settings.builder()
-            .put(this.globalSettings)
+            .put(globalSettings)
             .put(RealmSettings.getFullSettingKey(authcRealmName, JwtRealmSettings.ALLOWED_ISSUER), jwtIssuer.issuerClaimValue)
             .put(
                 RealmSettings.getFullSettingKey(authcRealmName, JwtRealmSettings.ALLOWED_SIGNATURE_ALGORITHMS),
@@ -279,7 +213,7 @@ protected JwtRealmSettingsBuilder createJwtRealmSettingsBuilder(final JwtIssuer
         if (jwtIssuer.encodedJwkSetPkcPublic.isEmpty() == false) {
             final String jwkSetPath; // file or HTTPS URL
             if (jwtIssuer.httpsServer == null) {
-                jwkSetPath = super.saveToTempFile("jwkset.", ".json", jwtIssuer.encodedJwkSetPkcPublic);
+                jwkSetPath = saveToTempFile("jwkset.", ".json", jwtIssuer.encodedJwkSetPkcPublic);
             } else {
                 authcSettings.putList(
                     RealmSettings.getFullSettingKey(
@@ -389,28 +323,27 @@ protected JwtRealmSettingsBuilder createJwtRealmSettingsBuilder(final JwtIssuer
 
     protected JwtRealm createJwtRealm(
         final List<Realm> allRealms, // JWT realms and authz realms
-        final JwtRealmsService jwtRealmsService,
         final JwtIssuer jwtIssuer,
         final JwtRealmSettingsBuilder realmSettingsBuilder
     ) {
         final String authcRealmName = realmSettingsBuilder.name;
         final Settings settings = realmSettingsBuilder.settingsBuilder.build();
-        final RealmConfig authcConfig = super.buildRealmConfig(JwtRealmSettings.TYPE, authcRealmName, settings, (allRealms.size() + 1));
+        final RealmConfig authcConfig = buildRealmConfig(JwtRealmSettings.TYPE, authcRealmName, settings, (allRealms.size() + 1));
         final SSLService sslService = new SSLService(TestEnvironment.newEnvironment(settings));
         final List<String> authzRealmNames = settings.getAsList(
             RealmSettings.getFullSettingKey(authcRealmName, DelegatedAuthorizationSettings.AUTHZ_REALMS.apply(JwtRealmSettings.TYPE))
         );
-        final UserRoleMapper userRoleMapper = super.buildRoleMapper(authzRealmNames.isEmpty() ? jwtIssuer.principals : Map.of());
+        final UserRoleMapper userRoleMapper = buildRoleMapper(authzRealmNames.isEmpty() ? jwtIssuer.principals : Map.of());
 
         // If authz names is not set, register the users here in the JWT authc realm.
-        final JwtRealm jwtRealm = new JwtRealm(authcConfig, jwtRealmsService, sslService, userRoleMapper);
+        final JwtRealm jwtRealm = new JwtRealm(authcConfig, sslService, userRoleMapper);
         allRealms.add(jwtRealm);
 
         // If authz names is set, register the users here in one of the authz realms.
         if (authzRealmNames.isEmpty() == false) {
             final String selected = randomFrom(authzRealmNames);
             for (final String authzRealmName : authzRealmNames) {
-                final RealmConfig authzConfig = this.buildRealmConfig("authz", authzRealmName, Settings.EMPTY, allRealms.size() + 1);
+                final RealmConfig authzConfig = buildRealmConfig("authz", authzRealmName, Settings.EMPTY, allRealms.size() + 1);
                 final MockLookupRealm authzRealm = new MockLookupRealm(authzConfig);
                 if (authzRealmName.equals(selected)) {
                     jwtIssuer.principals.values().forEach(authzRealm::registerUser);
@@ -423,19 +356,17 @@ protected JwtRealm createJwtRealm(
 
     protected JwtIssuerAndRealm randomJwtIssuerRealmPair() throws ParseException {
         // Select random JWT issuer and JWT realm pair, and log the realm settings
-        assertThat(this.jwtIssuerAndRealms, is(notNullValue()));
-        assertThat(this.jwtIssuerAndRealms, is(not(empty())));
-        final JwtIssuerAndRealm jwtIssuerAndRealm = randomFrom(this.jwtIssuerAndRealms);
+        assertThat(jwtIssuerAndRealms, notNullValue());
+        assertThat(jwtIssuerAndRealms, not(empty()));
+        final JwtIssuerAndRealm jwtIssuerAndRealm = randomFrom(jwtIssuerAndRealms);
         final JwtRealm jwtRealm = jwtIssuerAndRealm.realm;
-        assertThat(jwtRealm, is(notNullValue()));
-        assertThat(JwtRealmInspector.getAllowedIssuer(jwtRealm), is(equalTo(jwtIssuerAndRealm.issuer.issuerClaimValue))); // assert equal,
-                                                                                                                          // don't print
-                                                                                                                          // both
+        assertThat(jwtRealm, notNullValue());
+        assertThat(JwtRealmInspector.getAllowedIssuer(jwtRealm), equalTo(jwtIssuerAndRealm.issuer.issuerClaimValue));
         assertThat(
             jwtIssuerAndRealm.issuer.audiencesClaimValue.stream().anyMatch(JwtRealmInspector.getAllowedAudiences(jwtRealm)::contains),
             is(true)
         );
-        this.printJwtRealmAndIssuer(jwtIssuerAndRealm);
+        printJwtRealmAndIssuer(jwtIssuerAndRealm);
         return jwtIssuerAndRealm;
     }
 
@@ -444,162 +375,75 @@ protected void doMultipleAuthcAuthzAndVerifySuccess(
         final User user,
         final SecureString jwt,
         final SecureString sharedSecret,
-        final MinMax jwtAuthcRange
-    ) throws Exception {
-        assertThat(jwtAuthcRange.min(), is(greaterThanOrEqualTo(1)));
-
-        // Select one JWT authc Issuer/Realm pair. Select one test user, to use inside the authc test loop.
-        final List<JwtRealm> jwtRealmsList = this.jwtIssuerAndRealms.stream().map(p -> p.realm).toList();
+        final int jwtAuthcRepeats
+    ) {
+        final List<JwtRealm> jwtRealmsList = jwtIssuerAndRealms.stream().map(p -> p.realm).toList();
 
         // Select different test JWKs from the JWT realm, and generate test JWTs for the test user. Run the JWT through the chain.
-        final int jwtAuthcRepeats = randomIntBetween(jwtAuthcRange.min(), jwtAuthcRange.max());
         for (int authcRun = 1; authcRun <= jwtAuthcRepeats; authcRun++) {
-            // Create request with headers set
-            final ThreadContext requestThreadContext = super.createThreadContext(jwt, sharedSecret);
-            LOGGER.info("REQ[" + authcRun + "/" + jwtAuthcRepeats + "] HEADERS=" + requestThreadContext.getHeaders());
-
-            // Loop through all authc/authz realms. Confirm a JWT authc realm recognizes and extracts the request headers.
-            JwtAuthenticationToken jwtAuthenticationToken = null;
-            for (final JwtRealm candidateJwtRealm : jwtRealmsList) {
-                final AuthenticationToken authenticationToken = candidateJwtRealm.token(requestThreadContext);
-                if (authenticationToken != null) {
-                    assertThat(authenticationToken, isA(JwtAuthenticationToken.class));
-                    jwtAuthenticationToken = (JwtAuthenticationToken) authenticationToken;
-                    break;
-                }
-            }
-            assertThat(jwtAuthenticationToken, is(notNullValue()));
-            final String tokenPrincipal = jwtAuthenticationToken.principal();
-            final SecureString tokenJwt = jwtAuthenticationToken.getEndUserSignedJwt();
-            final SecureString tokenSecret = jwtAuthenticationToken.getClientAuthenticationSharedSecret();
-            assertThat(tokenPrincipal, is(notNullValue()));
-            if (tokenJwt.equals(jwt) == false) {
-                assertThat(tokenJwt, is(equalTo(jwt)));
-            }
-            assertThat(tokenJwt, is(equalTo(jwt)));
-            if (tokenSecret != null) {
-                if (tokenSecret.equals(sharedSecret) == false) {
-                    assertThat(tokenSecret, is(equalTo(sharedSecret)));
-                }
-                assertThat(tokenSecret, is(equalTo(sharedSecret)));
-            }
-            LOGGER.info("GOT TOKEN: principal=[" + tokenPrincipal + "], jwt=[" + tokenJwt + "], secret=[" + tokenSecret + "].");
+            final ThreadContext requestThreadContext = createThreadContext(jwt, sharedSecret);
+            logger.info("REQ[" + authcRun + "/" + jwtAuthcRepeats + "] HEADERS=" + requestThreadContext.getHeaders());
+
+            // Any JWT realm can recognize and extract the request headers.
+            final var jwtAuthenticationToken = (JwtAuthenticationToken) randomFrom(jwtRealmsList).token(requestThreadContext);
+            assertThat(jwtAuthenticationToken, notNullValue());
+            assertThat(jwtAuthenticationToken.principal(), notNullValue());
+            assertThat(jwtAuthenticationToken.getClientAuthenticationSharedSecret(), equalTo(sharedSecret));
 
             // Loop through all authc/authz realms. Confirm user is returned with expected principal and roles.
             User authenticatedUser = null;
-            final List<String> realmAuthenticationResults = new ArrayList<>();
-            final List<String> realmUsageStats = new ArrayList<>();
-            final List<Exception> realmFailureExceptions = new ArrayList<>(jwtRealmsList.size());
-            try {
-                for (final JwtRealm candidateJwtRealm : jwtRealmsList) {
-                    LOGGER.info("TRY AUTHC: expected=[" + jwtRealm.name() + "], candidate[" + candidateJwtRealm.name() + "].");
-                    final PlainActionFuture<AuthenticationResult<User>> authenticateFuture = PlainActionFuture.newFuture();
-                    try {
-                        candidateJwtRealm.authenticate(jwtAuthenticationToken, authenticateFuture);
-                        final AuthenticationResult<User> authenticationResult = authenticateFuture.actionGet();
-                        LOGGER.info("Authentication result with realm [{}]: [{}]", candidateJwtRealm.name(), authenticationResult);
-                        final Exception authenticationResultException = authenticationResult.getException();
-                        final String realmResult = "  realms=["
-                            + jwtRealmsList.size()
-                            + "], expected=["
-                            + jwtRealm.name()
-                            + ","
-                            + jwtRealm.order()
-                            + "], current["
-                            + candidateJwtRealm.name()
-                            + ","
-                            + candidateJwtRealm.order()
-                            + "], status=["
-                            + authenticationResult.getStatus()
-                            + "], authenticated=["
-                            + authenticationResult.isAuthenticated()
-                            + "], msg=["
-                            + authenticationResult.getMessage()
-                            + "], meta=["
-                            + authenticationResult.getMetadata()
-                            + "], user=["
-                            + authenticationResult.getValue()
-                            + "].";
-                        realmAuthenticationResults.add(realmResult);
-                        realmFailureExceptions.add(new Exception(realmResult, authenticationResultException));
-                        switch (authenticationResult.getStatus()) {
-                            case SUCCESS:
-                                assertThat("Unexpected realm SUCCESS status", candidateJwtRealm.name(), is(equalTo(jwtRealm.name())));
-                                assertThat("Expected realm authc false", authenticationResult.isAuthenticated(), is(equalTo(true)));
-                                assertThat("Expected realm exception thrown", authenticationResult.getException(), is(nullValue()));
-                                assertThat("Expected realm message null", authenticationResult.getMessage(), is(nullValue()));
-                                assertThat("Expected realm metadata empty", authenticationResult.getMetadata(), is(anEmptyMap()));
-                                authenticatedUser = authenticationResult.getValue();
-                                assertThat("Expected realm user null", authenticatedUser, is(notNullValue()));
-                                break;
-                            case CONTINUE:
-                                assertThat("Expected realm CONTINUE status", candidateJwtRealm.name(), is(not(equalTo(jwtRealm.name()))));
-                                assertThat("Unexpected realm authc success", authenticationResult.isAuthenticated(), is(equalTo(false)));
-                                continue;
-                            case TERMINATE:
-                                assertThat("Expected realm TERMINATE status", candidateJwtRealm.name(), is(not(equalTo(jwtRealm.name()))));
-                                assertThat("Unexpected realm authc success", authenticationResult.isAuthenticated(), is(equalTo(false)));
-                                break;
-                            default:
-                                fail("Unexpected AuthenticationResult.Status=[" + authenticationResult.getStatus() + "]");
-                                break;
-                        }
-                        break; // Only SUCCESS falls through to here, break out of the loop
-                    } catch (Exception e) {
-                        realmFailureExceptions.add(e);
-                        throw e;
-                    } finally {
-                        final PlainActionFuture<Map<String, Object>> usageStatsFuture = PlainActionFuture.newFuture();
-                        candidateJwtRealm.usageStats(usageStatsFuture);
-                        realmUsageStats.add(
-                            "   realm["
-                                + candidateJwtRealm.name()
-                                + ","
-                                + candidateJwtRealm.order()
-                                + "/"
-                                + jwtRealmsList.size()
-                                + "], stats=["
-                                + usageStatsFuture.actionGet()
-                                + "]"
-                        );
-                    }
-                }
-                // Loop ended. Confirm user is returned with expected principal and roles.
-                assertThat("Expected realm " + jwtRealm.name() + " to authenticate.", authenticatedUser, is(notNullValue()));
-                assertThat(user.principal(), equalTo(authenticatedUser.principal()));
-                assertThat(new TreeSet<>(Arrays.asList(user.roles())), equalTo(new TreeSet<>(Arrays.asList(authenticatedUser.roles()))));
-                if (jwtRealm.delegatedAuthorizationSupport.hasDelegation()) {
-                    assertThat(user.metadata(), is(equalTo(authenticatedUser.metadata()))); // delegated authz returns user's metadata
-                } else if (JwtRealmInspector.shouldPopulateUserMetadata(jwtRealm)) {
-                    assertThat(authenticatedUser.metadata(), hasEntry("jwt_token_type", "id_token"));
-                    assertThat(authenticatedUser.metadata(), hasKey(startsWith("jwt_claim_")));
-                } else {
-                    assertThat(authenticatedUser.metadata(), equalTo(Map.of("jwt_token_type", "id_token")));
-                }
-            } catch (Throwable t) {
-                realmFailureExceptions.forEach(t::addSuppressed); // all previous realm exceptions
-                // LOGGER.error("Unexpected exception.", t);
-                throw t;
-            } finally {
-                LOGGER.info("STATS: expected=[" + jwtRealm.name() + "]\n" + String.join("\n", realmUsageStats));
-                if (authenticatedUser != null) {
-                    LOGGER.info("RESULT: expected=[" + jwtRealm.name() + "]\n" + String.join("\n", realmAuthenticationResults));
+            realmLoop: for (final JwtRealm candidateJwtRealm : jwtRealmsList) {
+                logger.info("TRY AUTHC: expected=[" + jwtRealm.name() + "], candidate[" + candidateJwtRealm.name() + "].");
+                final PlainActionFuture<AuthenticationResult<User>> authenticateFuture = PlainActionFuture.newFuture();
+                candidateJwtRealm.authenticate(jwtAuthenticationToken, authenticateFuture);
+                final AuthenticationResult<User> authenticationResult = authenticateFuture.actionGet();
+                logger.info("Authentication result with realm [{}]: [{}]", candidateJwtRealm.name(), authenticationResult);
+                switch (authenticationResult.getStatus()) {
+                    case SUCCESS:
+                        assertThat("Unexpected realm SUCCESS status", candidateJwtRealm.name(), equalTo(jwtRealm.name()));
+                        assertThat("Expected realm metadata empty", authenticationResult.getMetadata(), is(anEmptyMap()));
+                        authenticatedUser = authenticationResult.getValue();
+                        assertThat("Expected realm user null", authenticatedUser, notNullValue());
+                        break realmLoop;
+                    case CONTINUE:
+                        assertThat("Expected realm CONTINUE status", candidateJwtRealm.name(), not(equalTo(jwtRealm.name())));
+                        continue;
+                    case TERMINATE:
+                        fail("A JWT realm should never terminate the authentication process, but [" + candidateJwtRealm.name() + "] did");
+                        break;
+                    default:
+                        fail("Unexpected AuthenticationResult.Status=[" + authenticationResult.getStatus() + "]");
                 }
             }
+            // Loop ended. Confirm user is returned with expected principal and roles.
+            assertThat("Expected realm " + jwtRealm.name() + " to authenticate.", authenticatedUser, notNullValue());
+            assertThat(user.principal(), equalTo(authenticatedUser.principal()));
+            assertThat(new TreeSet<>(Arrays.asList(user.roles())), equalTo(new TreeSet<>(Arrays.asList(authenticatedUser.roles()))));
+            if (jwtRealm.delegatedAuthorizationSupport.hasDelegation()) {
+                assertThat(user.metadata(), equalTo(authenticatedUser.metadata())); // delegated authz returns user's metadata
+            } else if (JwtRealmInspector.shouldPopulateUserMetadata(jwtRealm)) {
+                assertThat(authenticatedUser.metadata(), hasEntry("jwt_token_type", JwtRealmInspector.getTokenType(jwtRealm).value()));
+                assertThat(authenticatedUser.metadata(), hasKey(startsWith("jwt_claim_")));
+            } else {
+                assertThat(
+                    authenticatedUser.metadata(),
+                    equalTo(Map.of("jwt_token_type", JwtRealmInspector.getTokenType(jwtRealm).value()))
+                );
+            }
         }
-        LOGGER.info("Test succeeded");
+        logger.info("Test succeeded");
     }
 
     protected User randomUser(final JwtIssuer jwtIssuer) {
         final User user = randomFrom(jwtIssuer.principals.values());
-        LOGGER.info("USER[" + user.principal() + "]: roles=[" + String.join(",", user.roles()) + "].");
+        logger.info("USER[" + user.principal() + "]: roles=[" + String.join(",", user.roles()) + "].");
         return user;
     }
 
     protected SecureString randomJwt(final JwtIssuerAndRealm jwtIssuerAndRealm, User user) throws Exception {
         final JwtIssuer.AlgJwkPair algJwkPair = randomFrom(jwtIssuerAndRealm.issuer.algAndJwksAll);
         final JWK jwk = algJwkPair.jwk();
-        LOGGER.info(
+        logger.info(
             "ALG["
                 + algJwkPair.alg()
                 + "]. JWK: kty=["
@@ -625,7 +469,7 @@ protected SecureString randomJwt(final JwtIssuerAndRealm jwtIssuerAndRealm, User
             randomAlphaOfLengthBetween(10, 20), // jwtID
             JwtRealmInspector.getAllowedIssuer(jwtIssuerAndRealm.realm), // iss
             JwtRealmInspector.getAllowedAudiences(jwtIssuerAndRealm.realm), // aud
-            randomBoolean() ? null : randomBoolean() ? user.principal() : user.principal() + "_" + randomInt(9), // sub claim value
+            randomBoolean() ? user.principal() : user.principal() + "_" + randomInt(9), // sub claim value
             JwtRealmInspector.getPrincipalClaimName(jwtIssuerAndRealm.realm), // principal claim name
             user.principal(), // principal claim value
             JwtRealmInspector.getGroupsClaimName(jwtIssuerAndRealm.realm), // group claim name
@@ -637,24 +481,23 @@ protected SecureString randomJwt(final JwtIssuerAndRealm jwtIssuerAndRealm, User
             randomBoolean() ? null : new Nonce(32).toString(),
             randomBoolean() ? null : Map.of("other1", randomAlphaOfLength(10), "other2", randomAlphaOfLength(10))
         );
-        final SecureString signedJWT = JwtValidateUtil.signJwt(jwk, unsignedJwt);
-        assertThat(JwtValidateUtil.verifyJwt(jwk, SignedJWT.parse(signedJWT.toString())), is(equalTo(true)));
+        final SecureString signedJWT = signJwt(jwk, unsignedJwt);
         return signedJWT;
     }
 
     protected void printJwtRealmAndIssuer(JwtIssuerAndRealm jwtIssuerAndRealm) throws ParseException {
-        this.printJwtIssuer(jwtIssuerAndRealm.issuer());
-        this.printJwtRealm(jwtIssuerAndRealm.realm());
+        printJwtIssuer(jwtIssuerAndRealm.issuer());
+        printJwtRealm(jwtIssuerAndRealm.realm());
     }
 
     protected void printJwtRealm(final JwtRealm jwtRealm) {
-        LOGGER.info(
+        logger.info(
             "REALM["
                 + jwtRealm.name()
                 + ","
                 + jwtRealm.order()
                 + "/"
-                + this.jwtIssuerAndRealms.size()
+                + jwtIssuerAndRealms.size()
                 + "]: clientType=["
                 + JwtRealmInspector.getClientAuthenticationType(jwtRealm)
                 + "], clientSecret=["
@@ -684,15 +527,15 @@ protected void printJwtRealm(final JwtRealm jwtRealm) {
                 + "]."
         );
         for (final JWK jwk : JwtRealmInspector.getJwksAlgsHmac(jwtRealm).jwks()) {
-            LOGGER.info("REALM HMAC: jwk=[{}]", jwk);
+            logger.info("REALM HMAC: jwk=[{}]", jwk);
         }
         for (final JWK jwk : JwtRealmInspector.getJwksAlgsPkc(jwtRealm).jwks()) {
-            LOGGER.info("REALM PKC: jwk=[{}]", jwk);
+            logger.info("REALM PKC: jwk=[{}]", jwk);
         }
     }
 
     protected void printJwtIssuer(final JwtIssuer jwtIssuer) {
-        LOGGER.info(
+        logger.info(
             "ISSUER: iss=["
                 + jwtIssuer.issuerClaimValue
                 + "], aud=["
@@ -706,13 +549,13 @@ protected void printJwtIssuer(final JwtIssuer jwtIssuer) {
                 + "]."
         );
         if (jwtIssuer.algAndJwkHmacOidc != null) {
-            LOGGER.info("ISSUER HMAC OIDC: alg=[{}] jwk=[{}]", jwtIssuer.algAndJwkHmacOidc.alg(), jwtIssuer.encodedKeyHmacOidc);
+            logger.info("ISSUER HMAC OIDC: alg=[{}] jwk=[{}]", jwtIssuer.algAndJwkHmacOidc.alg(), jwtIssuer.encodedKeyHmacOidc);
         }
         for (final JwtIssuer.AlgJwkPair pair : jwtIssuer.algAndJwksHmac) {
-            LOGGER.info("ISSUER HMAC: alg=[{}] jwk=[{}]", pair.alg(), pair.jwk());
+            logger.info("ISSUER HMAC: alg=[{}] jwk=[{}]", pair.alg(), pair.jwk());
         }
         for (final JwtIssuer.AlgJwkPair pair : jwtIssuer.algAndJwksPkc) {
-            LOGGER.info("ISSUER PKC: alg=[{}] jwk=[{}]", pair.alg(), pair.jwk());
+            logger.info("ISSUER PKC: alg=[{}] jwk=[{}]", pair.alg(), pair.jwk());
         }
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtStringClaimValidatorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtStringClaimValidatorTests.java
index 22a242c4d7c9..41e666db39a5 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtStringClaimValidatorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtStringClaimValidatorTests.java
@@ -10,7 +10,6 @@
 import com.nimbusds.jose.JWSHeader;
 import com.nimbusds.jwt.JWTClaimsSet;
 
-import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.test.ESTestCase;
 
 import java.text.ParseException;
@@ -23,12 +22,22 @@
 public class JwtStringClaimValidatorTests extends ESTestCase {
 
     public void testClaimIsNotString() throws ParseException {
-        final String claimName = randomAlphaOfLengthBetween(10, 18);
-        final JwtStringClaimValidator validator = new JwtStringClaimValidator(claimName, List.of(), randomBoolean());
+        final String claimName = randomAlphaOfLength(10);
+        final String fallbackClaimName = randomAlphaOfLength(12);
+
+        final JwtStringClaimValidator validator;
+        final JWTClaimsSet jwtClaimsSet;
+        if (randomBoolean()) {
+            validator = new JwtStringClaimValidator(claimName, List.of(), randomBoolean());
+            // fallback claim is ignored
+            jwtClaimsSet = JWTClaimsSet.parse(Map.of(claimName, List.of(42), fallbackClaimName, randomAlphaOfLength(8)));
+        } else {
+            validator = new JwtStringClaimValidator(claimName, Map.of(claimName, fallbackClaimName), List.of(), randomBoolean());
+            jwtClaimsSet = JWTClaimsSet.parse(Map.of(fallbackClaimName, List.of(42)));
+        }
 
-        final JWTClaimsSet jwtClaimsSet = JWTClaimsSet.parse(Map.of(claimName, List.of(42)));
-        final ElasticsearchSecurityException e = expectThrows(
-            ElasticsearchSecurityException.class,
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
             () -> validator.validate(getJwsHeader(), jwtClaimsSet)
         );
         assertThat(e.getMessage(), containsString("cannot parse string claim"));
@@ -36,12 +45,22 @@ public void testClaimIsNotString() throws ParseException {
     }
 
     public void testClaimIsNotSingleValued() throws ParseException {
-        final String claimName = randomAlphaOfLengthBetween(3, 8);
-        final JwtStringClaimValidator validator = new JwtStringClaimValidator(claimName, List.of(), true);
+        final String claimName = randomAlphaOfLengthBetween(10, 18);
+        final String fallbackClaimName = randomAlphaOfLength(12);
+
+        final JwtStringClaimValidator validator;
+        final JWTClaimsSet jwtClaimsSet;
+        if (randomBoolean()) {
+            validator = new JwtStringClaimValidator(claimName, List.of(), true);
+            // fallback claim is ignored
+            jwtClaimsSet = JWTClaimsSet.parse(Map.of(claimName, List.of("foo", "bar"), fallbackClaimName, randomAlphaOfLength(8)));
+        } else {
+            validator = new JwtStringClaimValidator(claimName, Map.of(claimName, fallbackClaimName), List.of(), true);
+            jwtClaimsSet = JWTClaimsSet.parse(Map.of(fallbackClaimName, List.of("foo", "bar")));
+        }
 
-        final JWTClaimsSet jwtClaimsSet = JWTClaimsSet.parse(Map.of(claimName, List.of("foo", "bar")));
-        final ElasticsearchSecurityException e = expectThrows(
-            ElasticsearchSecurityException.class,
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
             () -> validator.validate(getJwsHeader(), jwtClaimsSet)
         );
         assertThat(e.getMessage(), containsString("cannot parse string claim"));
@@ -49,42 +68,120 @@ public void testClaimIsNotSingleValued() throws ParseException {
     }
 
     public void testClaimDoesNotExist() throws ParseException {
-        final String claimName = randomAlphaOfLengthBetween(3, 8);
-        final JwtStringClaimValidator validator = new JwtStringClaimValidator(claimName, List.of(), randomBoolean());
+        final String claimName = randomAlphaOfLengthBetween(10, 18);
+        final String fallbackClaimName = randomAlphaOfLength(12);
+
+        final JwtStringClaimValidator validator;
+        final JWTClaimsSet jwtClaimsSet;
+        if (randomBoolean()) {
+            validator = new JwtStringClaimValidator(claimName, List.of(), randomBoolean());
+        } else {
+            validator = new JwtStringClaimValidator(claimName, Map.of(claimName, fallbackClaimName), List.of(), randomBoolean());
+        }
+        jwtClaimsSet = JWTClaimsSet.parse(Map.of());
 
-        final JWTClaimsSet jwtClaimsSet = JWTClaimsSet.parse(Map.of());
-        final ElasticsearchSecurityException e = expectThrows(
-            ElasticsearchSecurityException.class,
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
             () -> validator.validate(getJwsHeader(), jwtClaimsSet)
         );
         assertThat(e.getMessage(), containsString("missing required string claim"));
     }
 
     public void testMatchingClaimValues() throws ParseException {
-        final String claimName = randomFrom(randomAlphaOfLengthBetween(3, 8));
+        final String claimName = randomAlphaOfLengthBetween(10, 18);
+        final String fallbackClaimName = randomAlphaOfLength(12);
         final String claimValue = randomAlphaOfLength(10);
         final boolean singleValuedClaim = randomBoolean();
-        final JwtStringClaimValidator validator = new JwtStringClaimValidator(
-            claimName,
-            List.of(claimValue, randomAlphaOfLengthBetween(11, 20)),
-            singleValuedClaim
-        );
+        final List<String> allowedClaimValues = List.of(claimValue, randomAlphaOfLengthBetween(11, 20));
+        final Object incomingClaimValue = singleValuedClaim ? claimValue : randomFrom(claimValue, List.of(claimValue, "other-stuff"));
+
+        final JwtStringClaimValidator validator;
+        final JWTClaimsSet validJwtClaimsSet;
+        final boolean noFallback = randomBoolean();
+        if (noFallback) {
+            validator = new JwtStringClaimValidator(claimName, allowedClaimValues, singleValuedClaim);
+            // fallback claim is ignored
+            validJwtClaimsSet = JWTClaimsSet.parse(Map.of(claimName, incomingClaimValue, fallbackClaimName, List.of(42)));
+        } else {
+            validator = new JwtStringClaimValidator(claimName, Map.of(claimName, fallbackClaimName), allowedClaimValues, singleValuedClaim);
+            validJwtClaimsSet = JWTClaimsSet.parse(Map.of(fallbackClaimName, incomingClaimValue));
+        }
 
-        final JWTClaimsSet validJwtClaimsSet = JWTClaimsSet.parse(
-            Map.of(claimName, singleValuedClaim ? claimValue : randomFrom(claimValue, List.of(claimValue, "other-stuff")))
-        );
         try {
             validator.validate(getJwsHeader(), validJwtClaimsSet);
         } catch (Exception e) {
             throw new AssertionError("validation should have passed without exception", e);
         }
 
-        final JWTClaimsSet invalidJwtClaimsSet = JWTClaimsSet.parse(Map.of(claimName, "not-" + claimValue));
-        final ElasticsearchSecurityException e = expectThrows(
-            ElasticsearchSecurityException.class,
+        final JWTClaimsSet invalidJwtClaimsSet;
+        if (noFallback) {
+            // fallback is ignored (even when it has a valid value) since the main claim exists
+            invalidJwtClaimsSet = JWTClaimsSet.parse(Map.of(claimName, "not-" + claimValue, fallbackClaimName, claimValue));
+        } else {
+            invalidJwtClaimsSet = JWTClaimsSet.parse(Map.of(fallbackClaimName, "not-" + claimValue));
+        }
+
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> validator.validate(getJwsHeader(), invalidJwtClaimsSet)
+        );
+        assertThat(e.getMessage(), containsString("does not match allowed claim values"));
+    }
+
+    public void testDoesNotSupportWildcardOrRegex() throws ParseException {
+        final String claimName = randomAlphaOfLengthBetween(10, 18);
+        final String fallbackClaimName = randomAlphaOfLength(12);
+        final String claimValue = randomFrom("*", "/.*/");
+
+        final JwtStringClaimValidator validator;
+        final JWTClaimsSet invalidJwtClaimsSet;
+        final boolean noFallback = randomBoolean();
+        if (noFallback) {
+            validator = new JwtStringClaimValidator(claimName, List.of(claimValue), randomBoolean());
+            // fallback is ignored (even when it has a valid value) since the main claim exists
+            invalidJwtClaimsSet = JWTClaimsSet.parse(Map.of(claimName, randomAlphaOfLengthBetween(1, 10), fallbackClaimName, claimValue));
+        } else {
+            validator = new JwtStringClaimValidator(claimName, Map.of(claimName, fallbackClaimName), List.of(claimValue), randomBoolean());
+            invalidJwtClaimsSet = JWTClaimsSet.parse(Map.of(fallbackClaimName, randomAlphaOfLengthBetween(1, 10)));
+        }
+
+        // It should not match arbitrary claim value because wildcard or regex is not supported
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
             () -> validator.validate(getJwsHeader(), invalidJwtClaimsSet)
         );
         assertThat(e.getMessage(), containsString("does not match allowed claim values"));
+
+        // It should support literal matching
+        final JWTClaimsSet validJwtClaimsSet;
+        if (noFallback) {
+            // fallback claim is ignored
+            validJwtClaimsSet = JWTClaimsSet.parse(Map.of(claimName, claimValue, fallbackClaimName, randomAlphaOfLength(10)));
+        } else {
+            validJwtClaimsSet = JWTClaimsSet.parse(Map.of(fallbackClaimName, claimValue));
+        }
+        try {
+            validator.validate(getJwsHeader(), validJwtClaimsSet);
+        } catch (Exception e2) {
+            throw new AssertionError("validation should have passed without exception", e2);
+        }
+    }
+
+    public void testAllowAllSubjects() {
+        try {
+            JwtStringClaimValidator.ALLOW_ALL_SUBJECTS.validate(
+                getJwsHeader(),
+                JWTClaimsSet.parse(Map.of("sub", randomAlphaOfLengthBetween(1, 10)))
+            );
+        } catch (Exception e) {
+            throw new AssertionError("validation should have passed without exception", e);
+        }
+
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> JwtStringClaimValidator.ALLOW_ALL_SUBJECTS.validate(getJwsHeader(), JWTClaimsSet.parse(Map.of()))
+        );
+        assertThat(e.getMessage(), containsString("missing required string claim"));
     }
 
     private JWSHeader getJwsHeader() throws ParseException {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtTestCase.java
index b9821606de6e..cfb153d233e9 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtTestCase.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtTestCase.java
@@ -10,7 +10,10 @@
 import com.nimbusds.jose.JOSEObjectType;
 import com.nimbusds.jose.JWSAlgorithm;
 import com.nimbusds.jose.JWSHeader;
+import com.nimbusds.jose.JWSSigner;
+import com.nimbusds.jose.crypto.ECDSASigner;
 import com.nimbusds.jose.crypto.MACSigner;
+import com.nimbusds.jose.crypto.RSASSASigner;
 import com.nimbusds.jose.jwk.Curve;
 import com.nimbusds.jose.jwk.ECKey;
 import com.nimbusds.jose.jwk.JWK;
@@ -83,10 +86,10 @@ public abstract class JwtTestCase extends ESTestCase {
 
     @Before
     public void beforeEachTest() {
-        this.pathHome = createTempDir().toString();
-        this.globalSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), this.pathHome).build();
-        this.env = TestEnvironment.newEnvironment(this.globalSettings); // "path.home" sub-dirs: config,plugins,data,logs,bin,lib,modules
-        this.threadContext = new ThreadContext(this.globalSettings);
+        pathHome = createTempDir().toString();
+        globalSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), pathHome).build();
+        env = TestEnvironment.newEnvironment(globalSettings); // "path.home" sub-dirs: config,plugins,data,logs,bin,lib,modules
+        threadContext = new ThreadContext(globalSettings);
     }
 
     protected Settings.Builder generateRandomRealmSettings(final String name) throws IOException {
@@ -95,7 +98,7 @@ protected Settings.Builder generateRandomRealmSettings(final String name) throws
         final boolean includePublicKey = includeRsa || includeEc;
         final boolean includeHmac = randomBoolean() || (includePublicKey == false); // one of HMAC/RSA/EC must be true
         final boolean populateUserMetadata = randomBoolean();
-        final Path jwtSetPathObj = PathUtils.get(this.pathHome);
+        final Path jwtSetPathObj = PathUtils.get(pathHome);
         final String jwkSetPath = randomBoolean()
             ? "https://op.example.com/jwkset.json"
             : Files.createTempFile(jwtSetPathObj, "jwkset.", ".json").toString();
@@ -229,12 +232,11 @@ protected RealmConfig buildRealmConfig(
     ) {
         final RealmConfig.RealmIdentifier realmIdentifier = new RealmConfig.RealmIdentifier(realmType, realmName);
         final Settings settings = Settings.builder()
-            .put(this.globalSettings)
-            // .put("path.home", this.pathHome)
+            .put(globalSettings)
             .put(realmSettings)
             .put(RealmSettings.getFullSettingKey(realmIdentifier, RealmSettings.ORDER_SETTING), realmOrder)
             .build();
-        return new RealmConfig(realmIdentifier, settings, this.env, this.threadContext);
+        return new RealmConfig(realmIdentifier, settings, env, threadContext);
     }
 
     protected UserRoleMapper buildRoleMapper(final Map<String, User> registeredUsers) {
@@ -479,7 +481,46 @@ public static SignedJWT buildUnsignedJwt(
                 + otherClaims
                 + "]}."
         );
-        return JwtValidateUtil.buildUnsignedJwt(jwtHeader, jwtClaimsSet);
+        return buildUnsignedJwt(jwtHeader, jwtClaimsSet);
+    }
+
+    public static SignedJWT buildUnsignedJwt(final JWSHeader jwtHeader, final JWTClaimsSet jwtClaimsSet) {
+        return new SignedJWT(jwtHeader, jwtClaimsSet);
+    }
+
+    // Build from Base64 components. Signature may or may not be valid. Useful for negative test cases.
+    public static SecureString buildJwt(final JWSHeader header, final JWTClaimsSet claims, final Base64URL signature) throws Exception {
+        final SignedJWT signedJwt = new SignedJWT(header.toBase64URL(), claims.toPayload().toBase64URL(), signature);
+        return new SecureString(signedJwt.serialize().toCharArray());
+    }
+
+    // Convenience method to construct JWSSigner from JWK, sign the JWT, and return serialized SecureString
+    public static SecureString signJwt(final JWK jwk, final SignedJWT unsignedJwt) throws Exception {
+        // Copy the header and claims set to a new unsigned JWT, in case JWT is being re-signing
+        final SignedJWT signedJwt = new SignedJWT(unsignedJwt.getHeader(), unsignedJwt.getJWTClaimsSet());
+        signedJwt.sign(createJwsSigner(jwk));
+        return new SecureString(signedJwt.serialize().toCharArray());
+    }
+
+    public static JWSSigner createJwsSigner(final JWK jwk) throws JOSEException {
+        if (jwk instanceof RSAKey rsaKey) {
+            return new RSASSASigner(rsaKey);
+        } else if (jwk instanceof ECKey ecKey) {
+            return new ECDSASigner(ecKey);
+        } else if (jwk instanceof OctetSequenceKey octetSequenceKey) {
+            return new MACSigner(octetSequenceKey);
+        }
+        throw new JOSEException(
+            "Unsupported JWK class ["
+                + (jwk == null ? "null" : jwk.getClass().getCanonicalName())
+                + "]. Supported classes are ["
+                + RSAKey.class.getCanonicalName()
+                + ", "
+                + ECKey.class.getCanonicalName()
+                + ", "
+                + OctetSequenceKey.class.getCanonicalName()
+                + "]."
+        );
     }
 
     public static SecureString randomBespokeJwt(final JWK jwk, final String signatureAlgorithm) throws Exception {
@@ -503,7 +544,7 @@ public static SecureString randomBespokeJwt(final JWK jwk, final String signatur
             randomBoolean() ? null : new Nonce(32).toString(),
             randomBoolean() ? null : Map.of("other1", randomAlphaOfLength(10), "other2", randomAlphaOfLength(10))
         );
-        return JwtValidateUtil.signJwt(jwk, unsignedJwt);
+        return signJwt(jwk, unsignedJwt);
     }
 
     public static Map<String, User> generateTestUsersWithRoles(final int numUsers, final int numRolesPerUser) {
@@ -566,13 +607,13 @@ public static <T> List<T> randomOfMinMaxNonUnique(final int min, final int max,
     }
 
     public String saveToTempFile(final String prefix, final String suffix, final String content) throws IOException {
-        final Path path = Files.createTempFile(PathUtils.get(this.pathHome), prefix, suffix);
+        final Path path = Files.createTempFile(PathUtils.get(pathHome), prefix, suffix);
         Files.writeString(path, content);
         return path.toString();
     }
 
     public ThreadContext createThreadContext(final CharSequence jwt, final CharSequence sharedSecret) {
-        final ThreadContext requestThreadContext = new ThreadContext(this.globalSettings);
+        final ThreadContext requestThreadContext = new ThreadContext(globalSettings);
         if (jwt != null) {
             requestThreadContext.putHeader(
                 JwtRealm.HEADER_END_USER_AUTHENTICATION,
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtTypeValidatorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtTypeValidatorTests.java
index 2b491a84c8cc..7de72577970d 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtTypeValidatorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtTypeValidatorTests.java
@@ -10,7 +10,6 @@
 import com.nimbusds.jose.JWSHeader;
 import com.nimbusds.jwt.JWTClaimsSet;
 
-import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.test.ESTestCase;
 
 import java.text.ParseException;
@@ -41,8 +40,8 @@ public void testInvalidType() throws ParseException {
             Map.of("typ", randomAlphaOfLengthBetween(4, 8), "alg", randomAlphaOfLengthBetween(3, 8))
         );
 
-        final ElasticsearchSecurityException e = expectThrows(
-            ElasticsearchSecurityException.class,
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
             () -> JwtTypeValidator.INSTANCE.validate(jwsHeader, JWTClaimsSet.parse(Map.of()))
         );
         assertThat(e.getMessage(), containsString("invalid jwt typ header"));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtilTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtilTests.java
index d38f0f72df02..161e263e9978 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtilTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtilTests.java
@@ -90,17 +90,17 @@ public void testClientAuthenticationTypeValidation() {
 
     public void testParseHttpsUriAllowsFilesPassThrough() {
         // Invalid null or empty values should be rejected
-        assertThat(JwtUtil.parseHttpsUri(null), is(nullValue()));
-        assertThat(JwtUtil.parseHttpsUri(""), is(nullValue()));
+        assertThat(JwtUtil.parseHttpsUri(null), nullValue());
+        assertThat(JwtUtil.parseHttpsUri(""), nullValue());
         // Valid Windows local file paths should be rejected
-        assertThat(JwtUtil.parseHttpsUri("C:"), is(nullValue()));
-        assertThat(JwtUtil.parseHttpsUri("C:/"), is(nullValue()));
-        assertThat(JwtUtil.parseHttpsUri("C:/jwkset.json"), is(nullValue()));
+        assertThat(JwtUtil.parseHttpsUri("C:"), nullValue());
+        assertThat(JwtUtil.parseHttpsUri("C:/"), nullValue());
+        assertThat(JwtUtil.parseHttpsUri("C:/jwkset.json"), nullValue());
         // Valid Linux local file paths should be rejected
-        assertThat(JwtUtil.parseHttpsUri("/"), is(nullValue()));
-        assertThat(JwtUtil.parseHttpsUri("/tmp"), is(nullValue()));
-        assertThat(JwtUtil.parseHttpsUri("/tmp/"), is(nullValue()));
-        assertThat(JwtUtil.parseHttpsUri("/tmp/jwkset.json"), is(nullValue()));
+        assertThat(JwtUtil.parseHttpsUri("/"), nullValue());
+        assertThat(JwtUtil.parseHttpsUri("/tmp"), nullValue());
+        assertThat(JwtUtil.parseHttpsUri("/tmp/"), nullValue());
+        assertThat(JwtUtil.parseHttpsUri("/tmp/jwkset.json"), nullValue());
     }
 
     public void testParseHttpUriAllRejected() {
@@ -141,22 +141,20 @@ public void testParseHttpsUriProblemsRejected() {
 
     public void testParseHttpsUriAccepted() {
         // Accept HTTPS URIs if parse succeeds
-        assertThat(JwtUtil.parseHttpsUri("https://example.com"), is(notNullValue()));
-        assertThat(JwtUtil.parseHttpsUri("https://example.com/"), is(notNullValue()));
-        assertThat(JwtUtil.parseHttpsUri("https://example.com:443"), is(notNullValue()));
-        assertThat(JwtUtil.parseHttpsUri("https://example.com:443/"), is(notNullValue()));
-        assertThat(JwtUtil.parseHttpsUri("https://example.com:8443"), is(notNullValue()));
-        assertThat(JwtUtil.parseHttpsUri("https://example.com:8443/"), is(notNullValue()));
-        assertThat(JwtUtil.parseHttpsUri("https://example.com/path/"), is(notNullValue()));
-        assertThat(JwtUtil.parseHttpsUri("https://example.com:443/path/"), is(notNullValue()));
-        assertThat(JwtUtil.parseHttpsUri("https://example.com:8443/path/"), is(notNullValue()));
-        assertThat(JwtUtil.parseHttpsUri("https://example.com/jwkset.json"), is(notNullValue()));
-        assertThat(JwtUtil.parseHttpsUri("https://example.com:443/jwkset.json"), is(notNullValue()));
-        assertThat(JwtUtil.parseHttpsUri("https://example.com:8443/jwkset.json"), is(notNullValue()));
-        assertThat(JwtUtil.parseHttpsUri("https://example.com/path/jwkset.json"), is(notNullValue()));
-        assertThat(JwtUtil.parseHttpsUri("https://example.com:443/path/jwkset.json"), is(notNullValue()));
-        assertThat(JwtUtil.parseHttpsUri("https://example.com:8443/path/jwkset.json"), is(notNullValue()));
+        assertThat(JwtUtil.parseHttpsUri("https://example.com"), notNullValue());
+        assertThat(JwtUtil.parseHttpsUri("https://example.com/"), notNullValue());
+        assertThat(JwtUtil.parseHttpsUri("https://example.com:443"), notNullValue());
+        assertThat(JwtUtil.parseHttpsUri("https://example.com:443/"), notNullValue());
+        assertThat(JwtUtil.parseHttpsUri("https://example.com:8443"), notNullValue());
+        assertThat(JwtUtil.parseHttpsUri("https://example.com:8443/"), notNullValue());
+        assertThat(JwtUtil.parseHttpsUri("https://example.com/path/"), notNullValue());
+        assertThat(JwtUtil.parseHttpsUri("https://example.com:443/path/"), notNullValue());
+        assertThat(JwtUtil.parseHttpsUri("https://example.com:8443/path/"), notNullValue());
+        assertThat(JwtUtil.parseHttpsUri("https://example.com/jwkset.json"), notNullValue());
+        assertThat(JwtUtil.parseHttpsUri("https://example.com:443/jwkset.json"), notNullValue());
+        assertThat(JwtUtil.parseHttpsUri("https://example.com:8443/jwkset.json"), notNullValue());
+        assertThat(JwtUtil.parseHttpsUri("https://example.com/path/jwkset.json"), notNullValue());
+        assertThat(JwtUtil.parseHttpsUri("https://example.com:443/path/jwkset.json"), notNullValue());
+        assertThat(JwtUtil.parseHttpsUri("https://example.com:8443/path/jwkset.json"), notNullValue());
     }
-
-    public void testParseHttpsUri() {}
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtValidateUtilTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtValidateUtilTests.java
index e58553274f9b..11158694ca4e 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtValidateUtilTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtValidateUtilTests.java
@@ -11,41 +11,39 @@
 import com.nimbusds.jose.jwk.JWK;
 import com.nimbusds.jwt.SignedJWT;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings;
 
+import java.util.List;
+
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
 
 public class JwtValidateUtilTests extends JwtTestCase {
 
-    private static final Logger LOGGER = LogManager.getLogger(JwtValidateUtilTests.class);
-
-    private boolean helpTestSignatureAlgorithm(final String signatureAlgorithm, final boolean requireOidcSafe) throws Exception {
-        LOGGER.trace("Testing signature algorithm " + signatureAlgorithm);
+    private void helpTestSignatureAlgorithm(final String signatureAlgorithm, final boolean requireOidcSafe) throws Exception {
+        logger.trace("Testing signature algorithm " + signatureAlgorithm);
         final JWK jwk = JwtTestCase.randomJwk(signatureAlgorithm, requireOidcSafe);
         final SecureString serializedJWTOriginal = JwtTestCase.randomBespokeJwt(jwk, signatureAlgorithm);
         final SignedJWT parsedSignedJWT = SignedJWT.parse(serializedJWTOriginal.toString());
-        return JwtValidateUtil.verifyJwt(jwk, parsedSignedJWT);
+        JwtValidateUtil.validateSignature(parsedSignedJWT, List.of(jwk));
     }
 
     public void testJwtSignVerifyPassedForAllSupportedAlgorithms() throws Exception {
         // Pass: "ES256", "ES384", "ES512", RS256", "RS384", "RS512", "PS256", "PS384", "PS512, "HS256", "HS384", "HS512"
         for (final String signatureAlgorithm : JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS) {
-            assertThat(this.helpTestSignatureAlgorithm(signatureAlgorithm, false), is(true));
+            try {
+                helpTestSignatureAlgorithm(signatureAlgorithm, false);
+            } catch (Exception e) {
+                fail("signature validation with algorithm [" + signatureAlgorithm + "] should have succeeded");
+            }
         }
         // Fail: "ES256K"
-        final Exception exp1 = expectThrows(
-            JOSEException.class,
-            () -> this.helpTestSignatureAlgorithm(JWSAlgorithm.ES256K.getName(), false)
-        );
+        final Exception exp1 = expectThrows(JOSEException.class, () -> helpTestSignatureAlgorithm(JWSAlgorithm.ES256K.getName(), false));
         final String msg1 = "Unsupported signature algorithm ["
             + JWSAlgorithm.ES256K
             + "]. Supported signature algorithms are "
             + JwtRealmSettings.SUPPORTED_SIGNATURE_ALGORITHMS
             + ".";
-        assertThat(exp1.getMessage(), is(equalTo(msg1)));
+        assertThat(exp1.getMessage(), equalTo(msg1));
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectRealmTests.java
index e644cedd2361..273f8e4c111e 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectRealmTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectRealmTests.java
@@ -133,8 +133,8 @@ public void testClaimPropertyMapping() throws Exception {
             Exception.class,
             () -> authenticateWithOidc(principal, roleMapper, false, false, REALM_NAME, claimsWithNumber)
         );
-        assertThat(e.getCause().getMessage(), containsString("expects a claim with String or a String Array value"));
-        assertThat(e2.getCause().getMessage(), containsString("expects a claim with String or a String Array value"));
+        assertThat(e.getCause().getMessage(), containsString("expects claim [groups] with String or a String Array value"));
+        assertThat(e2.getCause().getMessage(), containsString("expects claim [groups] with String or a String Array value"));
     }
 
     public void testClaimMetadataMapping() throws Exception {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticatorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticatorTests.java
index 3fb4757d9f18..19ae53caca08 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticatorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticatorTests.java
@@ -345,6 +345,45 @@ public void testSuccessfullyParseContentWithMultipleValidAttributes() throws Exc
         assertThat(attributes.session(), equalTo(session));
     }
 
+    public void testSuccessfullyParseContentFromRawXmlWithSignedAssertion() throws Exception {
+        Instant now = clock.instant();
+        final String nameId = randomAlphaOfLengthBetween(12, 24);
+        final String sessionindex = randomId();
+        final String xml = getSimpleResponseFromXmlTemplate(now, nameId, sessionindex);
+
+        SamlToken token = token(signAssertions(xml));
+        final SamlAttributes attributes = authenticator.authenticate(token);
+        assertThat(attributes, notNullValue());
+        assertThat(attributes.attributes(), iterableWithSize(2));
+        final List<String> uid = attributes.getAttributeValues(UID_OID);
+        assertThat(uid, contains("daredevil"));
+        assertThat(uid, iterableWithSize(1));
+        assertThat(attributes.name(), notNullValue());
+        assertThat(attributes.name().format, equalTo(TRANSIENT));
+        assertThat(attributes.name().value, equalTo(nameId));
+    }
+
+    public void testSuccessfullyParseContentFromRawXmlWithSignedUnicodeAssertion() throws Exception {
+        Instant now = clock.instant();
+        // Picking a completely random, but valid XML, unicode char is hard. We just insert some well known chars.
+        final String nameId = randomAlphaOfLengthBetween(2, 4) + "セキュリティ" + randomAlphaOfLengthBetween(2, 4);
+        final String nameIdFormat = "urn:fake:nameid-format:soluções";
+        final String sessionindex = randomId();
+        // Randomly skip the header because the XML parser is supposed to infer UTF-8 if there is no header
+        final String xml = getSimpleResponseFromXmlTemplate(randomBoolean(), now, nameIdFormat, nameId, sessionindex);
+
+        SamlToken token = token(signAssertions(xml));
+        final SamlAttributes attributes = authenticator.authenticate(token);
+        assertThat(attributes, notNullValue());
+        assertThat(attributes.attributes(), iterableWithSize(2));
+        final List<String> uid = attributes.getAttributeValues(UID_OID);
+        assertThat(uid, contains("daredevil"));
+        assertThat(uid, iterableWithSize(1));
+        assertThat(attributes.name(), notNullValue());
+        assertThat(attributes.name().format, equalTo(nameIdFormat));
+        assertThat(attributes.name().value, equalTo(nameId));
+    }
+
     public void testSuccessfullyParseContentFromEncryptedAssertion() throws Exception {
         final Instant now = clock.instant();
         final String xml = getSimpleResponseAsString(now);
@@ -1540,9 +1579,18 @@ private Response getSimpleResponse(
     }
 
     private String getSimpleResponseFromXmlTemplate(Instant now, String nameId, String sessionindex) {
+        return getSimpleResponseFromXmlTemplate(true, now, TRANSIENT, nameId, sessionindex);
+    }
+
+    private String getSimpleResponseFromXmlTemplate(
+        boolean includeXmlHeader,
+        Instant now,
+        String nameIdFormat,
+        String nameId,
+        String sessionindex
+    ) {
         Instant validUntil = now.plusSeconds(30);
-        String xml = "<?xml version='1.0' encoding='UTF-8'?>\n"
-            + "<proto:Response"
+        String xml = "<proto:Response"
             + "    Destination='%(SP_ACS_URL)'"
             + "    ID='%(randomId)'"
             + "    InResponseTo='%(requestId)'"
@@ -1558,7 +1606,7 @@ private String getSimpleResponseFromXmlTemplate(Instant now, String nameId, Stri
             + "  <assert:Assertion ID='%(sessionindex)' IssueInstant='%(now)' Version='2.0'>"
             + "    <assert:Issuer>%(IDP_ENTITY_ID)</assert:Issuer>"
             + "    <assert:Subject>"
-            + "      <assert:NameID  Format='%(TRANSIENT)'"
+            + "      <assert:NameID  Format='%(nameIdFormat)'"
             + "        NameQualifier='%(IDP_ENTITY_ID)'"
             + "        SPNameQualifier='%(SP_ENTITY_ID)'>%(nameId)</assert:NameID>"
             + "      <assert:SubjectConfirmation Method='%(METHOD_BEARER)'>"
@@ -1586,6 +1634,10 @@ private String getSimpleResponseFromXmlTemplate(Instant now, String nameId, Stri
             + "  </assert:Assertion>"
             + "</proto:Response>";
 
+        if (includeXmlHeader) {
+            xml = "<?xml version='1.0' encoding='UTF-8'?>\n" + xml;
+        }
+
         final Map<String, Object> replacements = new HashMap<>();
         replacements.put("IDP_ENTITY_ID", IDP_ENTITY_ID);
         replacements.put("METHOD_BEARER", METHOD_BEARER);
@@ -1597,7 +1649,7 @@ private String getSimpleResponseFromXmlTemplate(Instant now, String nameId, Stri
         replacements.put("sessionindex", sessionindex);
         replacements.put("SP_ACS_URL", SP_ACS_URL);
         replacements.put("SP_ENTITY_ID", SP_ENTITY_ID);
-        replacements.put("TRANSIENT", TRANSIENT);
+        replacements.put("nameIdFormat", nameIdFormat);
         replacements.put("validUntil", validUntil);
 
         return NamedFormatter.format(xml, replacements);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStoreTests.java
index cb4279f15316..f536a696a8e2 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStoreTests.java
@@ -218,8 +218,8 @@ public void testCreateToken() throws ExecutionException, InterruptedException {
         assertThat(creatorMap.get("full_name"), equalTo(authentication.getEffectiveSubject().getUser().fullName()));
         assertThat(creatorMap.get("email"), equalTo(authentication.getEffectiveSubject().getUser().email()));
         assertThat(creatorMap.get("metadata"), equalTo(authentication.getEffectiveSubject().getUser().metadata()));
-        assertThat(creatorMap.get("realm"), equalTo(authentication.getSourceRealm().getName()));
-        assertThat(creatorMap.get("realm_type"), equalTo(authentication.getSourceRealm().getType()));
+        assertThat(creatorMap.get("realm"), equalTo(authentication.getEffectiveSubject().getRealm().getName()));
+        assertThat(creatorMap.get("realm_type"), equalTo(authentication.getEffectiveSubject().getRealm().getType()));
 
         final CreateServiceAccountTokenResponse createServiceAccountTokenResponse1 = future1.get();
         assertNotNull(createServiceAccountTokenResponse1);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/DistinguishedNamePredicateTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/DistinguishedNamePredicateTests.java
index 444cc850af02..9484615045ce 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/DistinguishedNamePredicateTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/DistinguishedNamePredicateTests.java
@@ -34,7 +34,7 @@ public void testMatching() throws Exception {
         } else {
             inputDn = randomDn;
         }
-        final Predicate<FieldValue> predicate = new UserRoleMapper.DistinguishedNamePredicate(inputDn);
+        final Predicate<FieldValue> predicate = new UserRoleMapper.DistinguishedNamePredicate(inputDn, getDnNormalizer());
 
         assertPredicate(predicate, randomDn, true);
         assertPredicate(predicate, randomDn.toLowerCase(Locale.ROOT), true);
@@ -45,6 +45,9 @@ public void testMatching() throws Exception {
         assertPredicate(predicate, "*," + new DN(inputDn).getParent().getParent().toNormalizedString(), true);
         assertPredicate(predicate, randomDn.replaceFirst(".*,", "*,"), true);
         assertPredicate(predicate, randomDn.replaceFirst("[^,]*,", "*, "), true);
+        assertPredicate(predicate, "*,", true);
+        assertPredicate(predicate, "*, ", true);
+        assertPredicate(new UserRoleMapper.DistinguishedNamePredicate("", getDnNormalizer()), "*,", false);
 
         assertPredicate(predicate, randomDn + ",CN=AU", false);
         assertPredicate(predicate, "X" + randomDn, false);
@@ -55,20 +58,20 @@ public void testMatching() throws Exception {
     }
 
     public void testParsingMalformedInput() {
-        Predicate<FieldValue> predicate = new UserRoleMapper.DistinguishedNamePredicate("");
+        Predicate<FieldValue> predicate = new UserRoleMapper.DistinguishedNamePredicate("", getDnNormalizer());
         assertPredicate(predicate, null, false);
         assertPredicate(predicate, "", true);
         assertPredicate(predicate, randomAlphaOfLengthBetween(1, 8), false);
         assertPredicate(predicate, randomAlphaOfLengthBetween(1, 8) + "*", false);
 
-        predicate = new UserRoleMapper.DistinguishedNamePredicate("foo=");
+        predicate = new UserRoleMapper.DistinguishedNamePredicate("foo=", getDnNormalizer());
         assertPredicate(predicate, null, false);
         assertPredicate(predicate, "foo", false);
         assertPredicate(predicate, "foo=", true);
         assertPredicate(predicate, randomAlphaOfLengthBetween(5, 12), false);
         assertPredicate(predicate, randomAlphaOfLengthBetween(5, 12) + "*", false);
 
-        predicate = new UserRoleMapper.DistinguishedNamePredicate("=bar");
+        predicate = new UserRoleMapper.DistinguishedNamePredicate("=bar", getDnNormalizer());
         assertPredicate(predicate, null, false);
         assertPredicate(predicate, "bar", false);
         assertPredicate(predicate, "=bar", true);
@@ -79,4 +82,8 @@ public void testParsingMalformedInput() {
     private void assertPredicate(Predicate<FieldValue> predicate, Object value, boolean expected) {
         assertThat("Predicate [" + predicate + "] match [" + value + "]", predicate.test(new FieldValue(value)), equalTo(expected));
     }
+
+    private UserRoleMapper.DistinguishedNameNormalizer getDnNormalizer() {
+        return new UserRoleMapper.DistinguishedNameNormalizer();
+    }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/HasherTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/HasherTests.java
index 762af46bbac1..c9ce5d1d1b16 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/HasherTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/HasherTests.java
@@ -249,6 +249,71 @@ public void testResolveFromHash() {
         assertThat(Hasher.resolveFromHash("notavalidhashformat".toCharArray()), sameInstance(Hasher.NOOP));
     }
 
+    public void testPbkdf2ArbitrarySaltAndKeyLengths() {
+        // In FIPS 140 mode, passwords for PBKDF2 need to be at least 14 chars (112 bits)
+        assumeFalse("This should not run in FIPS mode", inFipsJvm());
+
+        // PBKDF2withHMACSHA512, 16 byte salt, 512 bits key
+        check(
+            "{PBKDF2}10000$"
+                + "NUG78+T6yahKzMHPgTbFmw==$"
+                + "Ohp+ZCG936Q+w1XTquEz5SmQmDUJVv5ZxilRaDPpHFRzHNDMjeFl8btefZd/0yNtfQPwpfhe5DSDFlPP9WMxEQ==",
+            "passw0rd",
+            true
+        );
+        check(
+            "{PBKDF2}10000$"
+                + "NUG78+T6yahKzMHPgTbFmw==$"
+                + "Ohp+ZCG936Q+w1XTquEz5SmQmDUJVv5ZxilRaDPpHFRzHNDMjeFl8btefZd/0yNtfQPwpfhe5DSDFlPP9WMxEQ==",
+            "admin",
+            false
+        );
+        check(
+            "{PBKDF2}100000$"
+                + "V+G8sM59pOTQ77ElFvIvbQ==$"
+                + "alomPA4LKmmu5d/y8BZI1fb5SnxxI6ClTFqe6vH3JJaz3cxrWfki5EO6Wy5eLuNr2BAdMVN0jnHGdsZzS+k+vw==",
+            "ginger",
+            true
+        );
+
+        // PBKDF2withHMACSHA512, 8 byte salt, 128 bits key
+        check("{PBKDF2}10000$vT/GxENkGSc=$4/b2cEHvSeAqzEoTM+RAYA==", "s3cr3t", true);
+
+        // PBKDF2withHMACSHA512, 64 byte salt, 512 bits key
+        check(
+            "{PBKDF2}1000$"
+                + "yJ1Mdnp6AEQfWKNdo6jw6BTy1KpxsthgaOHWQGhYWrzw826GZjX9vRN6h/4jRM750WWVhju2hxAZwP7yPzYdtA==$"
+                + "U1PnM55HWq6VKk/G0EpmTZJIRAfNMVpi78/cOc0I2qEGs68Ozg1Am4frXnSpb5lbundzkLxl+cg7MmAvM14JSQ==",
+            "somelongpasswordwith$peci@lch\u00E4rs!",
+            true
+        );
+
+        // PBKDF2withHMACSHA512, 32 byte salt, 256 bits key
+        check(
+            "{PBKDF2}50000$bjxn3/UGdT7zNtCVfgRp0REhPvLkIv4PZm9fullIQxc=$wuSFQZeOxifopubSzKbIE2xAGdxJlEcUVlqvjFU1TTI=",
+            "Tr0ub4dor&3",
+            true
+        );
+
+        // 12345 iterations are not supported
+        IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> { check("{PBKDF2}12345$Rvr0LPiggps=$qBVD7TdDG3mgnLI5yZuR2g==", "s3cr3t", true); }
+        );
+        assertThat(e.getMessage(), containsString("unknown hash function [pbkdf2_12345]"));
+
+        // salt smaller than 8 bytes is not supported
+        ElasticsearchException ee = expectThrows(
+            ElasticsearchException.class,
+            () -> { check("{PBKDF2}10000$erwUfw==$Pv/wuOn49EH5nY88LOtY2g==", "s3cr3t", true); }
+        );
+        assertThat(ee.getMessage(), containsString("PBKDF2 salt must be at least [8 bytes] long"));
+
+        // derived key length is not multiple of 128 bits
+        ee = expectThrows(ElasticsearchException.class, () -> { check("{PBKDF2}10000$0jtIBOnhz7Q=$njOn++ANoMAXn0gi", "s3cr3t", true); });
+        assertThat(ee.getMessage(), containsString("PBKDF2 key length must be positive and multiple of [128 bits]"));
+    }
+
     public void testPbkdf2WithShortPasswordThrowsInFips() {
         assumeTrue("This should run only in FIPS mode", inFipsJvm());
         SecureString passwd = new SecureString(randomAlphaOfLength(between(6, 13)).toCharArray());
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/SecondaryAuthenticatorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/SecondaryAuthenticatorTests.java
index 8b80112d9caf..7fbcef61af07 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/SecondaryAuthenticatorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/SecondaryAuthenticatorTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -64,6 +65,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
@@ -122,6 +124,7 @@ public void setupMocks() throws Exception {
         final ClusterService clusterService = mock(ClusterService.class);
         final ClusterState clusterState = ClusterState.EMPTY_STATE;
         when(clusterService.state()).thenReturn(clusterState);
+        when(clusterService.getClusterSettings()).thenReturn(new ClusterSettings(settings, Set.of(ApiKeyService.DELETE_RETENTION_PERIOD)));
 
         securityContext = new SecurityContext(settings, threadContext);
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationDenialMessagesTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationDenialMessagesTests.java
index 2e0486fece9a..7cd56bc75f2d 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationDenialMessagesTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationDenialMessagesTests.java
@@ -54,7 +54,7 @@ public void testRolesDescriptionWithNullAuthorizationInfo() {
 
         assertThat(
             rolesDescription,
-            equalTo(formatted(" with assigned roles [%s]", Strings.collectionToCommaDelimitedString(assignedRoleNames)))
+            equalTo(Strings.format(" with assigned roles [%s]", Strings.collectionToCommaDelimitedString(assignedRoleNames)))
         );
     }
 
@@ -74,7 +74,7 @@ public void testRolesDescriptionWithNullRolesField() {
 
         assertThat(
             rolesDescription,
-            equalTo(formatted(" with assigned roles [%s]", Strings.collectionToCommaDelimitedString(assignedRoleNames)))
+            equalTo(Strings.format(" with assigned roles [%s]", Strings.collectionToCommaDelimitedString(assignedRoleNames)))
         );
     }
 
@@ -105,7 +105,7 @@ public void testRolesDescriptionWithIncompatibleRolesField() {
 
         assertThat(
             rolesDescription,
-            equalTo(formatted(" with assigned roles [%s]", Strings.collectionToCommaDelimitedString(assignedRoleNames)))
+            equalTo(Strings.format(" with assigned roles [%s]", Strings.collectionToCommaDelimitedString(assignedRoleNames)))
         );
     }
 
@@ -130,7 +130,7 @@ public void testRoleDescriptionWithEmptyResolvedRole() {
             assertThat(
                 rolesDescription,
                 equalTo(
-                    formatted(
+                    Strings.format(
                         " with effective roles [] (assigned roles [%s] were not found)",
                         Strings.collectionToCommaDelimitedString(assignedRoleNames.stream().sorted().toList())
                     )
@@ -164,7 +164,7 @@ public void testRoleDescriptionAllResolvedAndMaybeWithAnonymousRoles() {
         assertThat(
             rolesDescription,
             equalTo(
-                formatted(
+                Strings.format(
                     " with effective roles [%s]",
                     Strings.collectionToCommaDelimitedString(effectiveRoleNames.stream().sorted().toList())
                 )
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceIntegTests.java
new file mode 100644
index 000000000000..be12abdfeff5
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceIntegTests.java
@@ -0,0 +1,161 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authz;
+
+import org.elasticsearch.action.LatchedActionListener;
+import org.elasticsearch.action.support.ActionTestUtils;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.SecurityIntegTestCase;
+import org.elasticsearch.transport.TcpTransport;
+import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.security.action.user.AuthenticateAction;
+import org.elasticsearch.xpack.core.security.action.user.AuthenticateRequest;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection;
+import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilegeResolver;
+import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
+import org.elasticsearch.xpack.core.security.support.NativeRealmValidationUtil;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.security.audit.AuditUtil;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField.AUTHORIZATION_INFO_KEY;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
+
+public class AuthorizationServiceIntegTests extends SecurityIntegTestCase {
+
+    @Override
+    protected boolean addMockHttpTransport() {
+        return false; // need real http
+    }
+
+    public void testRetrieveRemoteAccessRoleDescriptorsIntersection() throws IOException, InterruptedException {
+        assumeTrue("untrusted remote cluster feature flag must be enabled", TcpTransport.isUntrustedRemoteClusterEnabled());
+
+        final String concreteClusterAlias = randomAlphaOfLength(10);
+        final String roleName = randomAlphaOfLength(5);
+        getSecurityClient().putRole(
+            new RoleDescriptor(
+                roleName,
+                randomSubsetOf(ClusterPrivilegeResolver.names()).toArray(String[]::new),
+                randomBoolean()
+                    ? null
+                    : new RoleDescriptor.IndicesPrivileges[] {
+                        RoleDescriptor.IndicesPrivileges.builder()
+                            .privileges(randomSubsetOf(randomIntBetween(1, 4), IndexPrivilege.names()))
+                            .indices(generateRandomStringArray(5, randomIntBetween(3, 9), false, false))
+                            .allowRestrictedIndices(randomBoolean())
+                            .build() },
+                null,
+                null,
+                generateRandomStringArray(5, randomIntBetween(2, 8), false, true),
+                null,
+                null,
+                new RoleDescriptor.RemoteIndicesPrivileges[] {
+                    new RoleDescriptor.RemoteIndicesPrivileges(
+                        RoleDescriptor.IndicesPrivileges.builder()
+                            .indices(shuffledList(List.of("index1", "index2")))
+                            .privileges(shuffledList(List.of("read", "write")))
+                            .build(),
+                        randomNonEmptySubsetOf(List.of(concreteClusterAlias, "*")).toArray(new String[0])
+                    ) }
+            )
+        );
+        final String nodeName = internalCluster().getRandomNodeName();
+        final ThreadContext threadContext = internalCluster().getInstance(SecurityContext.class, nodeName).getThreadContext();
+        final AuthorizationService authzService = internalCluster().getInstance(AuthorizationService.class, nodeName);
+        final Authentication authentication = Authentication.newRealmAuthentication(
+            new User(randomAlphaOfLengthBetween(5, 16), roleName),
+            new Authentication.RealmRef(
+                randomBoolean() ? "realm" : randomAlphaOfLengthBetween(1, 16),
+                randomAlphaOfLengthBetween(5, 16),
+                nodeName
+            )
+        );
+        final RoleDescriptorsIntersection actual = authorizeThenRetrieveRemoteAccessDescriptors(
+            threadContext,
+            authzService,
+            authentication,
+            concreteClusterAlias
+        );
+        final String generatedRoleName = actual.roleDescriptorsList().iterator().next().iterator().next().getName();
+        assertNull(NativeRealmValidationUtil.validateRoleName(generatedRoleName, false));
+        assertThat(generatedRoleName, not(equalTo(roleName)));
+        assertThat(
+            actual,
+            equalTo(
+                new RoleDescriptorsIntersection(
+                    List.of(
+                        Set.of(
+                            new RoleDescriptor(
+                                generatedRoleName,
+                                null,
+                                new RoleDescriptor.IndicesPrivileges[] {
+                                    RoleDescriptor.IndicesPrivileges.builder()
+                                        .indices("index1", "index2")
+                                        .privileges("read", "write")
+                                        .build() },
+                                null,
+                                null,
+                                null,
+                                null,
+                                null
+                            )
+                        )
+                    )
+                )
+            )
+        );
+    }
+
+    private RoleDescriptorsIntersection authorizeThenRetrieveRemoteAccessDescriptors(
+        final ThreadContext threadContext,
+        final AuthorizationService authzService,
+        final Authentication authentication,
+        final String concreteClusterAlias
+    ) throws InterruptedException {
+        try (var ignored = threadContext.stashContext()) {
+            assertThat(threadContext.getTransient(AUTHORIZATION_INFO_KEY), nullValue());
+            final AtomicReference<RoleDescriptorsIntersection> actual = new AtomicReference<>();
+            final CountDownLatch latch = new CountDownLatch(1);
+            // A request ID is set during authentication and is required for authorization; since we are not authenticating, set it
+            // explicitly
+            AuditUtil.generateRequestId(threadContext);
+            // Authorize to populate thread context with authz info
+            // Note that if the outer listener throws, we will not count down on the latch, however, we also won't get to the await call
+            // since the exception will be thrown before -- so no deadlock
+            authzService.authorize(
+                authentication,
+                AuthenticateAction.INSTANCE.name(),
+                AuthenticateRequest.INSTANCE,
+                ActionTestUtils.assertNoFailureListener(nothing -> {
+                    authzService.retrieveRemoteAccessRoleDescriptorsIntersection(
+                        concreteClusterAlias,
+                        authentication.getEffectiveSubject(),
+                        new LatchedActionListener<>(ActionTestUtils.assertNoFailureListener(newValue -> {
+                            assertThat(threadContext.getTransient(AUTHORIZATION_INFO_KEY), not(nullValue()));
+                            actual.set(newValue);
+                        }), latch)
+                    );
+                })
+            );
+            latch.await();
+            // Validate original authz info is restored after call complete
+            assertThat(threadContext.getTransient(AUTHORIZATION_INFO_KEY), nullValue());
+            return actual.get();
+        }
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java
index d69b2df3a67a..2e1c71449134 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java
@@ -240,6 +240,7 @@ public class AuthorizationServiceTests extends ESTestCase {
     private OperatorPrivileges.OperatorPrivilegesService operatorPrivilegesService;
     private boolean shouldFailOperatorPrivilegesCheck = false;
     private boolean setFakeOriginatingAction = true;
+    private SecurityContext securityContext;
 
     @SuppressWarnings("unchecked")
     @Before
@@ -258,6 +259,7 @@ public void setup() {
         when(licenseState.isAllowed(Security.AUDITING_FEATURE)).thenReturn(true);
         auditTrailService = new AuditTrailService(Collections.singletonList(auditTrail), licenseState);
         threadContext = new ThreadContext(settings);
+        securityContext = new SecurityContext(settings, threadContext);
         threadPool = mock(ThreadPool.class);
         when(threadPool.getThreadContext()).thenReturn(threadContext);
         final FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(settings);
@@ -364,7 +366,6 @@ private void authorize(
         String someRandomHeader = "test_" + UUIDs.randomBase64UUID();
         Object someRandomHeaderValue = mock(Object.class);
         threadContext.putTransient(someRandomHeader, someRandomHeaderValue);
-        SecurityContext securityContext = new SecurityContext(Settings.EMPTY, threadContext);
         // the thread context before authorization could contain any of the transient headers
         IndicesAccessControl mockAccessControlHeader = threadContext.getTransient(INDICES_PERMISSIONS_KEY);
         if (mockAccessControlHeader == null && randomBoolean()) {
@@ -1168,12 +1169,18 @@ public void testSearchAgainstIndex() {
         authorize(authentication, SearchAction.NAME, searchRequest, true, () -> {
             verify(rolesStore).getRoles(Mockito.same(authentication), Mockito.any());
             IndicesAccessControl iac = threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY);
+            // Successful search action authorization should set a parent authorization header.
+            assertThat(securityContext.getParentAuthorization().action(), equalTo(SearchAction.NAME));
             // Within the action handler, execute a child action (the query phase of search)
             authorize(authentication, SearchTransportService.QUERY_ACTION_NAME, shardRequest, false, () -> {
                 // This child action triggers a second interaction with the role store (which is cached)
                 verify(rolesStore, times(2)).getRoles(Mockito.same(authentication), Mockito.any());
                 // But it does not create a new IndicesAccessControl
                 assertThat(threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY), sameInstance(iac));
+                // The parent authorization header should only be present for direct child actions
+                // and not be carried over for a child of a child actions.
+                // Meaning, only query phase action should be pre-authorized in this case and potential sub-actions should not.
+                assertThat(securityContext.getParentAuthorization(), nullValue());
             });
         });
         verify(auditTrail).accessGranted(
@@ -1476,7 +1483,7 @@ public void testDenialErrorMessagesForBulkIngest() throws Exception {
         );
 
         TransportReplicationAction.PrimaryResult<BulkShardRequest, BulkShardResponse> result = future.get();
-        BulkShardResponse response = result.finalResponseIfSuccessful;
+        BulkShardResponse response = result.replicationResponse;
         assertThat(response, notNullValue());
         assertThat(response.getResponses(), arrayWithSize(3));
         assertThat(response.getResponses()[0].getFailureMessage(), containsString("unauthorized for user [" + user.principal() + "]"));
@@ -2879,7 +2886,7 @@ public void loadAuthorizedIndices(
                 RequestInfo requestInfo,
                 AuthorizationInfo authorizationInfo,
                 Map<String, IndexAbstraction> indicesLookup,
-                ActionListener<Set<String>> listener
+                ActionListener<AuthorizationEngine.AuthorizedIndices> listener
             ) {
                 throw new UnsupportedOperationException("not implemented");
             }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationUtilsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationUtilsTests.java
index e08f8663ae27..e4fd06a1a002 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationUtilsTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationUtilsTests.java
@@ -6,13 +6,13 @@
  */
 package org.elasticsearch.xpack.security.authz;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
@@ -111,11 +111,11 @@ public void testShouldSetUser() {
     }
 
     public void testSwitchAndExecuteXpackSecurityUser() throws Exception {
-        assertSwitchBasedOnOriginAndExecute(ClientHelper.SECURITY_ORIGIN, XPackSecurityUser.INSTANCE, randomVersion());
+        assertSwitchBasedOnOriginAndExecute(ClientHelper.SECURITY_ORIGIN, XPackSecurityUser.INSTANCE, randomTransportVersion());
     }
 
     public void testSwitchAndExecuteSecurityProfileUser() throws Exception {
-        assertSwitchBasedOnOriginAndExecute(ClientHelper.SECURITY_PROFILE_ORIGIN, SecurityProfileUser.INSTANCE, randomVersion());
+        assertSwitchBasedOnOriginAndExecute(ClientHelper.SECURITY_PROFILE_ORIGIN, SecurityProfileUser.INSTANCE, randomTransportVersion());
     }
 
     public void testSwitchAndExecuteXpackUser() throws Exception {
@@ -127,20 +127,20 @@ public void testSwitchAndExecuteXpackUser() throws Exception {
             PersistentTasksService.PERSISTENT_TASK_ORIGIN,
             ClientHelper.INDEX_LIFECYCLE_ORIGIN
         )) {
-            assertSwitchBasedOnOriginAndExecute(origin, XPackUser.INSTANCE, randomVersion());
+            assertSwitchBasedOnOriginAndExecute(origin, XPackUser.INSTANCE, randomTransportVersion());
         }
     }
 
     public void testSwitchAndExecuteAsyncSearchUser() throws Exception {
         String origin = ClientHelper.ASYNC_SEARCH_ORIGIN;
-        assertSwitchBasedOnOriginAndExecute(origin, AsyncSearchUser.INSTANCE, randomVersion());
+        assertSwitchBasedOnOriginAndExecute(origin, AsyncSearchUser.INSTANCE, randomTransportVersion());
     }
 
     public void testSwitchWithTaskOrigin() throws Exception {
-        assertSwitchBasedOnOriginAndExecute(TASKS_ORIGIN, XPackUser.INSTANCE, randomVersion());
+        assertSwitchBasedOnOriginAndExecute(TASKS_ORIGIN, XPackUser.INSTANCE, randomTransportVersion());
     }
 
-    private void assertSwitchBasedOnOriginAndExecute(String origin, User user, Version version) throws Exception {
+    private void assertSwitchBasedOnOriginAndExecute(String origin, User user, TransportVersion version) throws Exception {
         SecurityContext securityContext = new SecurityContext(Settings.EMPTY, threadContext);
         final String headerName = randomAlphaOfLengthBetween(4, 16);
         final String headerValue = randomAlphaOfLengthBetween(4, 16);
@@ -151,7 +151,7 @@ private void assertSwitchBasedOnOriginAndExecute(String origin, User user, Versi
             assertNull(threadContext.getHeader(headerName));
             final Authentication authentication = securityContext.getAuthentication();
             assertEquals(user, authentication.getEffectiveSubject().getUser());
-            assertEquals(version, authentication.getEffectiveSubject().getVersion());
+            assertEquals(version, authentication.getEffectiveSubject().getTransportVersion());
             latch.countDown();
         }, e -> fail(e.getMessage()));
 
@@ -160,7 +160,7 @@ private void assertSwitchBasedOnOriginAndExecute(String origin, User user, Versi
             assertNull(threadContext.getHeader(headerName));
             final Authentication authentication = securityContext.getAuthentication();
             assertEquals(user, authentication.getEffectiveSubject().getUser());
-            assertEquals(version, authentication.getEffectiveSubject().getVersion());
+            assertEquals(version, authentication.getEffectiveSubject().getTransportVersion());
             latch.countDown();
             listener.onResponse(null);
         };
@@ -172,7 +172,7 @@ private void assertSwitchBasedOnOriginAndExecute(String origin, User user, Versi
         }
     }
 
-    private Version randomVersion() {
-        return VersionUtils.randomCompatibleVersion(random(), Version.CURRENT);
+    private TransportVersion randomTransportVersion() {
+        return TransportVersionUtils.randomCompatibleVersion(random(), TransportVersion.CURRENT);
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java
index caaf78bf346b..23fc46b53f58 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizedIndices;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
@@ -41,17 +42,19 @@
 import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.RESTRICTED_INDICES;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 
 public class AuthorizedIndicesTests extends ESTestCase {
 
     public void testAuthorizedIndicesUserWithoutRoles() {
-        Set<String> authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(
+        AuthorizedIndices authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(
             Role.EMPTY,
             getRequestInfo(""),
-            Metadata.EMPTY_METADATA.getIndicesLookup()
+            Metadata.EMPTY_METADATA.getIndicesLookup(),
+            () -> ignore -> {}
         );
-        assertTrue(authorizedIndices.isEmpty());
+        assertTrue(authorizedIndices.all().get().isEmpty());
     }
 
     public void testAuthorizedIndicesUserWithSomeRoles() {
@@ -105,36 +108,43 @@ public void testAuthorizedIndicesUserWithSomeRoles() {
             future
         );
         Role roles = future.actionGet();
-        Set<String> list = RBACEngine.resolveAuthorizedIndicesFromRole(
+        AuthorizedIndices authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(
             roles,
             getRequestInfo(SearchAction.NAME),
-            metadata.getIndicesLookup()
+            metadata.getIndicesLookup(),
+            () -> ignore -> {}
         );
-        assertThat(list, containsInAnyOrder("a1", "a2", "aaaaaa", "b", "ab"));
-        assertFalse(list.contains("bbbbb"));
-        assertFalse(list.contains("ba"));
-        assertThat(list, not(contains(internalSecurityIndex)));
-        assertThat(list, not(contains(SecuritySystemIndices.SECURITY_MAIN_ALIAS)));
+        assertThat(authorizedIndices.all().get(), containsInAnyOrder("a1", "a2", "aaaaaa", "b", "ab"));
+        assertThat(authorizedIndices.all().get(), not(contains("bbbbb")));
+        assertThat(authorizedIndices.check("bbbbb"), is(false));
+        assertThat(authorizedIndices.all().get(), not(contains("ba")));
+        assertThat(authorizedIndices.check("ba"), is(false));
+        assertThat(authorizedIndices.all().get(), not(contains(internalSecurityIndex)));
+        assertThat(authorizedIndices.check(internalSecurityIndex), is(false));
+        assertThat(authorizedIndices.all().get(), not(contains(SecuritySystemIndices.SECURITY_MAIN_ALIAS)));
+        assertThat(authorizedIndices.check(SecuritySystemIndices.SECURITY_MAIN_ALIAS), is(false));
     }
 
     public void testAuthorizedIndicesUserWithSomeRolesEmptyMetadata() {
         Role role = Role.builder(RESTRICTED_INDICES, "role").add(IndexPrivilege.ALL, "*").build();
-        Set<String> authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(
+        AuthorizedIndices authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(
             role,
             getRequestInfo(SearchAction.NAME),
-            Metadata.EMPTY_METADATA.getIndicesLookup()
+            Metadata.EMPTY_METADATA.getIndicesLookup(),
+            () -> ignore -> {}
         );
-        assertTrue(authorizedIndices.isEmpty());
+        assertTrue(authorizedIndices.all().get().isEmpty());
     }
 
     public void testSecurityIndicesAreRemovedFromRegularUser() {
         Role role = Role.builder(RESTRICTED_INDICES, "user_role").add(IndexPrivilege.ALL, "*").cluster(Set.of("all"), Set.of()).build();
-        Set<String> authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(
+        AuthorizedIndices authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(
             role,
             getRequestInfo(SearchAction.NAME),
-            Metadata.EMPTY_METADATA.getIndicesLookup()
+            Metadata.EMPTY_METADATA.getIndicesLookup(),
+            () -> ignore -> {}
         );
-        assertTrue(authorizedIndices.isEmpty());
+        assertTrue(authorizedIndices.all().get().isEmpty());
     }
 
     public void testSecurityIndicesAreRestrictedForDefaultRole() {
@@ -160,14 +170,19 @@ public void testSecurityIndicesAreRestrictedForDefaultRole() {
             )
             .build();
 
-        Set<String> authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(
+        AuthorizedIndices authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(
             role,
             getRequestInfo(SearchAction.NAME),
-            metadata.getIndicesLookup()
+            metadata.getIndicesLookup(),
+            () -> ignore -> {}
         );
-        assertThat(authorizedIndices, containsInAnyOrder("an-index", "another-index"));
-        assertThat(authorizedIndices, not(contains(internalSecurityIndex)));
-        assertThat(authorizedIndices, not(contains(SecuritySystemIndices.SECURITY_MAIN_ALIAS)));
+        assertThat(authorizedIndices.all().get(), containsInAnyOrder("an-index", "another-index"));
+        assertThat(authorizedIndices.check("an-index"), is(true));
+        assertThat(authorizedIndices.check("another-index"), is(true));
+        assertThat(authorizedIndices.all().get(), not(contains(internalSecurityIndex)));
+        assertThat(authorizedIndices.check(internalSecurityIndex), is(false));
+        assertThat(authorizedIndices.all().get(), not(contains(SecuritySystemIndices.SECURITY_MAIN_ALIAS)));
+        assertThat(authorizedIndices.check(SecuritySystemIndices.SECURITY_MAIN_ALIAS), is(false));
     }
 
     public void testSecurityIndicesAreNotRemovedFromUnrestrictedRole() {
@@ -193,23 +208,25 @@ public void testSecurityIndicesAreNotRemovedFromUnrestrictedRole() {
             )
             .build();
 
-        Set<String> authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(
+        AuthorizedIndices authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(
             role,
             getRequestInfo(SearchAction.NAME),
-            metadata.getIndicesLookup()
+            metadata.getIndicesLookup(),
+            () -> ignore -> {}
         );
         assertThat(
-            authorizedIndices,
+            authorizedIndices.all().get(),
             containsInAnyOrder("an-index", "another-index", SecuritySystemIndices.SECURITY_MAIN_ALIAS, internalSecurityIndex)
         );
 
-        Set<String> authorizedIndicesSuperUser = RBACEngine.resolveAuthorizedIndicesFromRole(
+        AuthorizedIndices authorizedIndicesSuperUser = RBACEngine.resolveAuthorizedIndicesFromRole(
             role,
             getRequestInfo(SearchAction.NAME),
-            metadata.getIndicesLookup()
+            metadata.getIndicesLookup(),
+            () -> ignore -> {}
         );
         assertThat(
-            authorizedIndicesSuperUser,
+            authorizedIndicesSuperUser.all().get(),
             containsInAnyOrder("an-index", "another-index", SecuritySystemIndices.SECURITY_MAIN_ALIAS, internalSecurityIndex)
         );
     }
@@ -273,17 +290,28 @@ public void testDataStreamsAreNotIncludedInAuthorizedIndices() {
             future
         );
         Role roles = future.actionGet();
-        Set<String> list = RBACEngine.resolveAuthorizedIndicesFromRole(
+        AuthorizedIndices authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(
             roles,
             getRequestInfo(SearchAction.NAME),
-            metadata.getIndicesLookup()
+            metadata.getIndicesLookup(),
+            () -> ignore -> {}
         );
-        assertThat(list, containsInAnyOrder("a1", "a2", "aaaaaa", "b", "ab"));
-        assertFalse(list.contains("bbbbb"));
-        assertFalse(list.contains("ba"));
-        assertFalse(list.contains("adatastream1"));
-        assertThat(list, not(contains(internalSecurityIndex)));
-        assertThat(list, not(contains(SecuritySystemIndices.SECURITY_MAIN_ALIAS)));
+        assertThat(authorizedIndices.all().get(), containsInAnyOrder("a1", "a2", "aaaaaa", "b", "ab"));
+        for (String resource : List.of("a1", "a2", "aaaaaa", "b", "ab")) {
+            assertThat(authorizedIndices.check(resource), is(true));
+        }
+        assertThat(authorizedIndices.all().get(), not(contains("bbbbb")));
+        assertThat(authorizedIndices.check("bbbbb"), is(false));
+        assertThat(authorizedIndices.all().get(), not(contains("ba")));
+        assertThat(authorizedIndices.check("ba"), is(false));
+        // due to context, datastreams are excluded from wildcard expansion
+        assertThat(authorizedIndices.all().get(), not(contains("adatastream1")));
+        // but they are authorized when explicitly tested (they are not "unavailable" for the Security filter)
+        assertThat(authorizedIndices.check("adatastream1"), is(true));
+        assertThat(authorizedIndices.all().get(), not(contains(internalSecurityIndex)));
+        assertThat(authorizedIndices.check(internalSecurityIndex), is(false));
+        assertThat(authorizedIndices.all().get(), not(contains(SecuritySystemIndices.SECURITY_MAIN_ALIAS)));
+        assertThat(authorizedIndices.check(SecuritySystemIndices.SECURITY_MAIN_ALIAS), is(false));
     }
 
     public void testDataStreamsAreIncludedInAuthorizedIndices() {
@@ -347,12 +375,21 @@ public void testDataStreamsAreIncludedInAuthorizedIndices() {
         Role roles = future.actionGet();
         TransportRequest request = new ResolveIndexAction.Request(new String[] { "a*" });
         AuthorizationEngine.RequestInfo requestInfo = getRequestInfo(request, SearchAction.NAME);
-        Set<String> list = RBACEngine.resolveAuthorizedIndicesFromRole(roles, requestInfo, metadata.getIndicesLookup());
-        assertThat(list, containsInAnyOrder("a1", "a2", "aaaaaa", "b", "ab", "adatastream1", backingIndex));
-        assertFalse(list.contains("bbbbb"));
-        assertFalse(list.contains("ba"));
-        assertThat(list, not(contains(internalSecurityIndex)));
-        assertThat(list, not(contains(SecuritySystemIndices.SECURITY_MAIN_ALIAS)));
+        AuthorizedIndices authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(
+            roles,
+            requestInfo,
+            metadata.getIndicesLookup(),
+            () -> ignore -> {}
+        );
+        assertThat(authorizedIndices.all().get(), containsInAnyOrder("a1", "a2", "aaaaaa", "b", "ab", "adatastream1", backingIndex));
+        assertThat(authorizedIndices.all().get(), not(contains("bbbbb")));
+        assertThat(authorizedIndices.check("bbbbb"), is(false));
+        assertThat(authorizedIndices.all().get(), not(contains("ba")));
+        assertThat(authorizedIndices.check("ba"), is(false));
+        assertThat(authorizedIndices.all().get(), not(contains(internalSecurityIndex)));
+        assertThat(authorizedIndices.check(internalSecurityIndex), is(false));
+        assertThat(authorizedIndices.all().get(), not(contains(SecuritySystemIndices.SECURITY_MAIN_ALIAS)));
+        assertThat(authorizedIndices.check(SecuritySystemIndices.SECURITY_MAIN_ALIAS), is(false));
     }
 
     public static AuthorizationEngine.RequestInfo getRequestInfo(String action) {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
index 4b55d5894adc..17e73d5ca2a6 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
@@ -57,6 +57,7 @@
 import org.elasticsearch.xpack.core.graph.action.GraphExploreAction;
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
 import org.elasticsearch.xpack.core.security.authc.Subject;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizedIndices;
 import org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField;
 import org.elasticsearch.xpack.core.security.authz.ResolvedIndices;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
@@ -80,7 +81,6 @@
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -711,14 +711,11 @@ public void testResolveExplicitIndicesStrict() {
     }
 
     public void testResolveExplicitIndicesIgnoreUnavailable() {
-        SearchRequest request = new SearchRequest("missing", "bar", "barbaz");
+        SearchRequest request = new SearchRequest("missing", "missing-and-unauthorized", "bar", "barbaz");
         request.indicesOptions(IndicesOptions.fromOptions(true, randomBoolean(), randomBoolean(), randomBoolean()));
         List<String> indices = resolveIndices(request, buildAuthorizedIndices(user, SearchAction.NAME)).getLocal();
-        String[] replacedIndices = new String[] { "bar" };
-        assertThat(indices, hasSize(replacedIndices.length));
-        assertThat(request.indices().length, equalTo(replacedIndices.length));
-        assertThat(indices, hasItems(replacedIndices));
-        assertThat(request.indices(), arrayContainingInAnyOrder(replacedIndices));
+        assertThat(indices, containsInAnyOrder("bar", "missing"));
+        assertThat(request.indices(), arrayContainingInAnyOrder("bar", "missing"));
     }
 
     public void testResolveNoAuthorizedIndicesAllowNoIndices() {
@@ -749,14 +746,11 @@ public void testResolveMissingIndexStrict() {
     }
 
     public void testResolveMissingIndexIgnoreUnavailable() {
-        SearchRequest request = new SearchRequest("bar*", "missing");
+        SearchRequest request = new SearchRequest("bar*", "missing", "missing-and-unauthorized");
         request.indicesOptions(IndicesOptions.fromOptions(true, randomBoolean(), true, false));
         List<String> indices = resolveIndices(request, buildAuthorizedIndices(user, SearchAction.NAME)).getLocal();
-        String[] expectedIndices = new String[] { "bar" };
-        assertThat(indices, hasSize(expectedIndices.length));
-        assertThat(request.indices().length, equalTo(expectedIndices.length));
-        assertThat(indices, hasItems(expectedIndices));
-        assertThat(request.indices(), equalTo(expectedIndices));
+        assertThat(indices, containsInAnyOrder("bar", "missing"));
+        assertThat(request.indices(), arrayContainingInAnyOrder("bar", "missing"));
     }
 
     public void testResolveNonMatchingIndicesAndExplicit() {
@@ -804,7 +798,7 @@ public void testSearchWithRemoteAndLocalIndices() {
     public void testSearchWithRemoteAndLocalWildcards() {
         SearchRequest request = new SearchRequest("*:foo", "r*:bar*", "remote:baz*", "bar*", "foofoo");
         request.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, false));
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
         final ResolvedIndices resolved = resolveIndices(request, authorizedIndices);
         assertThat(resolved.getRemote(), containsInAnyOrder("remote:foo", "other_remote:foo", "remote:bar*", "remote:baz*"));
         assertThat(resolved.getLocal(), containsInAnyOrder("bar", "foofoo"));
@@ -922,7 +916,7 @@ public void testResolveIndicesAliasesRequestDeleteActions() {
         IndicesAliasesRequest request = new IndicesAliasesRequest();
         request.addAliasAction(AliasActions.remove().index("foo").alias("foofoobar"));
         request.addAliasAction(AliasActions.remove().index("foofoo").alias("barbaz"));
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         // the union of all indices and aliases gets returned
         String[] expectedIndices = new String[] { "foo", "foofoobar", "foofoo", "barbaz" };
@@ -938,7 +932,7 @@ public void testResolveIndicesAliasesRequestDeleteActionsMissingIndex() {
         IndicesAliasesRequest request = new IndicesAliasesRequest();
         request.addAliasAction(AliasActions.remove().index("foo").alias("foofoobar"));
         request.addAliasAction(AliasActions.remove().index("missing_index").alias("missing_alias"));
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         // the union of all indices and aliases gets returned, doesn't matter is some of them don't exist
         String[] expectedIndices = new String[] { "foo", "foofoobar", "missing_index", "missing_alias" };
@@ -954,7 +948,7 @@ public void testResolveWildcardsIndicesAliasesRequestDeleteActions() {
         IndicesAliasesRequest request = new IndicesAliasesRequest();
         request.addAliasAction(AliasActions.remove().index("foo*").alias("foofoobar"));
         request.addAliasAction(AliasActions.remove().index("bar*").alias("barbaz"));
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         // union of all resolved indices and aliases gets returned, based on what user is authorized for
         String[] expectedIndices = new String[] { "foofoobar", "foofoo", "bar", "barbaz" };
@@ -971,7 +965,7 @@ public void testResolveAliasesWildcardsIndicesAliasesRequestDeleteActions() {
         IndicesAliasesRequest request = new IndicesAliasesRequest();
         request.addAliasAction(AliasActions.remove().index("*").alias("foo*"));
         request.addAliasAction(AliasActions.remove().index("*bar").alias("foo*"));
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         // union of all resolved indices and aliases gets returned, based on what user is authorized for
         // note that the index side will end up containing matching aliases too, which is fine, as es core would do
@@ -989,7 +983,7 @@ public void testResolveAllAliasesWildcardsIndicesAliasesRequestDeleteActions() {
         IndicesAliasesRequest request = new IndicesAliasesRequest();
         request.addAliasAction(AliasActions.remove().index("*").alias("_all"));
         request.addAliasAction(AliasActions.remove().index("_all").aliases("_all", "explicit"));
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         // union of all resolved indices and aliases gets returned, based on what user is authorized for
         // note that the index side will end up containing matching aliases too, which is fine, as es core would do
@@ -1027,7 +1021,7 @@ public void testResolveWildcardsIndicesAliasesRequestAddAndDeleteActions() {
         IndicesAliasesRequest request = new IndicesAliasesRequest();
         request.addAliasAction(AliasActions.remove().index("foo*").alias("foofoobar"));
         request.addAliasAction(AliasActions.add().index("bar*").alias("foofoobar"));
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, IndicesAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         // union of all resolved indices and aliases gets returned, based on what user is authorized for
         String[] expectedIndices = new String[] { "foofoobar", "foofoo", "bar" };
@@ -1042,7 +1036,7 @@ public void testResolveWildcardsIndicesAliasesRequestAddAndDeleteActions() {
     public void testResolveGetAliasesRequestStrict() {
         GetAliasesRequest request = new GetAliasesRequest("alias1").indices("foo", "foofoo");
         request.indicesOptions(IndicesOptions.fromOptions(false, randomBoolean(), randomBoolean(), randomBoolean()));
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         // the union of all indices and aliases gets returned
         String[] expectedIndices = new String[] { "alias1", "foo", "foofoo" };
@@ -1055,7 +1049,7 @@ public void testResolveGetAliasesRequestStrict() {
     public void testResolveGetAliasesRequestIgnoreUnavailable() {
         GetAliasesRequest request = new GetAliasesRequest("alias1").indices("foo", "foofoo");
         request.indicesOptions(IndicesOptions.fromOptions(true, randomBoolean(), randomBoolean(), randomBoolean()));
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         String[] expectedIndices = new String[] { "alias1", "foofoo" };
         assertThat(indices, hasSize(expectedIndices.length));
@@ -1069,14 +1063,18 @@ public void testResolveGetAliasesRequestMissingIndexStrict() {
         request.indicesOptions(IndicesOptions.fromOptions(false, randomBoolean(), true, randomBoolean()));
         request.indices("missing");
         request.aliases("alias2");
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
-        // the union of all indices and aliases gets returned, missing is not an existing index/alias but that doesn't make any difference
-        String[] expectedIndices = new String[] { "alias2", "missing" };
-        assertThat(indices, hasSize(expectedIndices.length));
-        assertThat(indices, hasItems(expectedIndices));
+        // explicit names, regardless of whether they are missing or unauthorized,
+        // are not erased from the request if `ignoreUnavailable` is `false`
+        assertThat(indices, containsInAnyOrder("alias2", "missing"));
         assertThat(request.indices(), arrayContainingInAnyOrder("missing"));
         assertThat(request.aliases(), arrayContainingInAnyOrder("alias2"));
+        request.indices("missing-and-unauthorized");
+        indices = resolveIndices(request, authorizedIndices).getLocal();
+        assertThat(indices, containsInAnyOrder("alias2", "missing-and-unauthorized"));
+        assertThat(request.indices(), arrayContainingInAnyOrder("missing-and-unauthorized"));
+        assertThat(request.aliases(), arrayContainingInAnyOrder("alias2"));
     }
 
     public void testGetAliasesRequestMissingIndexIgnoreUnavailableDisallowNoIndices() {
@@ -1084,11 +1082,17 @@ public void testGetAliasesRequestMissingIndexIgnoreUnavailableDisallowNoIndices(
         request.indicesOptions(IndicesOptions.fromOptions(true, false, randomBoolean(), randomBoolean()));
         request.indices("missing");
         request.aliases("alias2");
+        List<String> indices = resolveIndices(request, buildAuthorizedIndices(user, GetAliasesAction.NAME)).getLocal();
+        // missing is authorized, so it is not "unavailable" from Security's POV
+        assertThat(request.indices(), arrayContainingInAnyOrder("missing"));
+        assertThat(indices, containsInAnyOrder("missing", "alias2"));
+        // but this one is both missing and unauthorized, and because it is unauthorized it counts as "unavailable"
+        request.indices("missing-and-unauthorized");
         IndexNotFoundException exception = expectThrows(
             IndexNotFoundException.class,
             () -> resolveIndices(request, buildAuthorizedIndices(user, GetAliasesAction.NAME)).getLocal()
         );
-        assertEquals("no such index [[missing]]", exception.getMessage());
+        assertEquals("no such index [[missing-and-unauthorized]]", exception.getMessage());
     }
 
     public void testGetAliasesRequestMissingIndexIgnoreUnavailableAllowNoIndices() {
@@ -1096,6 +1100,12 @@ public void testGetAliasesRequestMissingIndexIgnoreUnavailableAllowNoIndices() {
         request.indicesOptions(IndicesOptions.fromOptions(true, true, randomBoolean(), randomBoolean()));
         request.indices("missing");
         request.aliases("alias2");
+        List<String> indices = resolveIndices(request, buildAuthorizedIndices(user, GetAliasesAction.NAME)).getLocal();
+        // missing is authorized, so it is not "unavailable" from Security's POV
+        assertThat(request.indices(), arrayContainingInAnyOrder("missing"));
+        assertThat(indices, containsInAnyOrder("missing", "alias2"));
+        // but this one is both missing and unauthorized, and because it is unauthorized it counts as "unavailable"
+        request.indices("missing-and-unauthorized");
         assertNoIndices(request, resolveIndices(request, buildAuthorizedIndices(user, GetAliasesAction.NAME)));
     }
 
@@ -1104,7 +1114,7 @@ public void testGetAliasesRequestMissingIndexStrict() {
         request.indicesOptions(IndicesOptions.fromOptions(false, randomBoolean(), randomBoolean(), randomBoolean()));
         request.indices("missing");
         request.aliases("alias2");
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         String[] expectedIndices = new String[] { "alias2", "missing" };
         assertThat(indices, hasSize(expectedIndices.length));
@@ -1118,7 +1128,7 @@ public void testResolveWildcardsGetAliasesRequestStrictExpand() {
         request.indicesOptions(IndicesOptions.fromOptions(false, randomBoolean(), true, true));
         request.aliases("alias1");
         request.indices("foo*");
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         // the union of all resolved indices and aliases gets returned, based on indices and aliases that user is authorized for
         String[] expectedIndices = new String[] { "alias1", "foofoo", "foofoo-closed", "foofoobar", "foobarfoo" };
@@ -1134,7 +1144,7 @@ public void testResolveWildcardsGetAliasesRequestStrictExpandOpen() {
         request.indicesOptions(IndicesOptions.fromOptions(false, randomBoolean(), true, false));
         request.aliases("alias1");
         request.indices("foo*");
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         // the union of all resolved indices and aliases gets returned, based on indices and aliases that user is authorized for
         String[] expectedIndices = new String[] { "alias1", "foofoo", "foofoobar", "foobarfoo" };
@@ -1147,17 +1157,15 @@ public void testResolveWildcardsGetAliasesRequestStrictExpandOpen() {
 
     public void testResolveWildcardsGetAliasesRequestLenientExpandOpen() {
         GetAliasesRequest request = new GetAliasesRequest();
-        request.indicesOptions(IndicesOptions.fromOptions(true, randomBoolean(), true, false));
+        request.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), true, false));
         request.aliases("alias1");
         request.indices("foo*", "bar", "missing");
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         // the union of all resolved indices and aliases gets returned, based on indices and aliases that user is authorized for
-        String[] expectedIndices = new String[] { "alias1", "foofoo", "foofoobar", "foobarfoo", "bar" };
-        assertThat(indices, hasSize(expectedIndices.length));
-        assertThat(indices, hasItems(expectedIndices));
+        assertThat(indices, containsInAnyOrder("alias1", "foofoo", "foofoobar", "foobarfoo", "bar", "missing"));
         // wildcards get replaced on each single action
-        assertThat(request.indices(), arrayContainingInAnyOrder("foofoobar", "foobarfoo", "foofoo", "bar"));
+        assertThat(request.indices(), arrayContainingInAnyOrder("foofoobar", "foobarfoo", "foofoo", "bar", "missing"));
         assertThat(request.aliases(), arrayContainingInAnyOrder("alias1"));
     }
 
@@ -1188,7 +1196,7 @@ public void testResolveAllGetAliasesRequest() {
             request.indices("_all");
         }
         request.aliases("alias1");
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         // the union of all resolved indices and aliases gets returned, including hidden indices as Get Aliases includes hidden by default
         String[] expectedIndices = new String[] {
@@ -1233,7 +1241,7 @@ public void testResolveAllGetAliasesRequestExpandWildcardsOpenOnly() {
             request.indices("_all");
         }
         request.aliases("alias1");
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         // the union of all resolved indices and aliases gets returned
         String[] expectedIndices = new String[] { "bar", "foofoobar", "foobarfoo", "foofoo", "alias1" };
@@ -1294,7 +1302,7 @@ public void testResolveAllAliasesGetAliasesRequest() {
         if (randomBoolean()) {
             request.indices("_all");
         }
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         // the union of all resolved indices and aliases gets returned, including hidden indices as Get Aliases includes hidden by default
         String[] expectedIndices = new String[] {
@@ -1321,7 +1329,7 @@ public void testResolveAllAndExplicitAliasesGetAliasesRequest() {
         if (randomBoolean()) {
             request.indices("_all");
         }
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         // the union of all resolved indices and aliases gets returned, including hidden indices as Get Aliases includes hidden by default
         String[] expectedIndices = new String[] {
@@ -1366,7 +1374,7 @@ public void testResolveAllAndWildcardsAliasesGetAliasesRequest() {
         if (randomBoolean()) {
             request.indices("_all");
         }
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         // the union of all resolved indices and aliases gets returned, including hidden indices as Get Aliases includes hidden by default
         String[] expectedIndices = new String[] {
@@ -1392,7 +1400,7 @@ public void testResolveAliasesWildcardsGetAliasesRequest() {
         GetAliasesRequest request = new GetAliasesRequest();
         request.indices("*bar");
         request.aliases("foo*");
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         // union of all resolved indices and aliases gets returned, based on what user is authorized for
         // note that the index side will end up containing matching aliases too, which is fine, as es core would do
@@ -1416,7 +1424,7 @@ public void testResolveAliasesWildcardsGetAliasesRequestNoAuthorizedIndices() {
     public void testResolveAliasesExclusionWildcardsGetAliasesRequest() {
         GetAliasesRequest request = new GetAliasesRequest();
         request.aliases("foo*", "-foobar*");
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         // union of all resolved indices and aliases gets returned, based on what user is authorized for
         // note that the index side will end up containing matching aliases too, which is fine, as es core would do
@@ -1533,7 +1541,7 @@ public void testCompositeIndicesRequestIsNotSupported() {
     }
 
     public void testResolveAdminAction() {
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, DeleteIndexAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, DeleteIndexAction.NAME);
         {
             RefreshRequest request = new RefreshRequest("*");
             List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
@@ -1555,14 +1563,14 @@ public void testResolveAdminAction() {
     public void testXPackSecurityUserHasAccessToSecurityIndex() {
         SearchRequest request = new SearchRequest();
         {
-            final Set<String> authorizedIndices = buildAuthorizedIndices(XPackSecurityUser.INSTANCE, SearchAction.NAME);
+            final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(XPackSecurityUser.INSTANCE, SearchAction.NAME);
             List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
             assertThat(indices, hasItem(SECURITY_MAIN_ALIAS));
         }
         {
             IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
             aliasesRequest.addAliasAction(AliasActions.add().alias("security_alias").index(SECURITY_MAIN_ALIAS));
-            final Set<String> authorizedIndices = buildAuthorizedIndices(XPackSecurityUser.INSTANCE, IndicesAliasesAction.NAME);
+            final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(XPackSecurityUser.INSTANCE, IndicesAliasesAction.NAME);
             List<String> indices = resolveIndices(aliasesRequest, authorizedIndices).getLocal();
             assertThat(indices, hasItem(SECURITY_MAIN_ALIAS));
         }
@@ -1570,7 +1578,7 @@ public void testXPackSecurityUserHasAccessToSecurityIndex() {
 
     public void testXPackUserDoesNotHaveAccessToSecurityIndex() {
         SearchRequest request = new SearchRequest();
-        final Set<String> authorizedIndices = buildAuthorizedIndices(XPackUser.INSTANCE, SearchAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(XPackUser.INSTANCE, SearchAction.NAME);
         List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
         assertThat(indices, not(hasItem(SECURITY_MAIN_ALIAS)));
     }
@@ -1589,7 +1597,7 @@ public void testNonXPackUserAccessingSecurityIndex() {
 
         {
             SearchRequest request = new SearchRequest();
-            final Set<String> authorizedIndices = buildAuthorizedIndices(allAccessUser, SearchAction.NAME);
+            final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(allAccessUser, SearchAction.NAME);
             List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
             assertThat(indices, not(hasItem(SECURITY_MAIN_ALIAS)));
         }
@@ -1597,7 +1605,7 @@ public void testNonXPackUserAccessingSecurityIndex() {
         {
             IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
             aliasesRequest.addAliasAction(AliasActions.add().alias("security_alias1").index("*"));
-            final Set<String> authorizedIndices = buildAuthorizedIndices(allAccessUser, IndicesAliasesAction.NAME);
+            final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(allAccessUser, IndicesAliasesAction.NAME);
             List<String> indices = resolveIndices(aliasesRequest, authorizedIndices).getLocal();
             assertThat(indices, not(hasItem(SECURITY_MAIN_ALIAS)));
         }
@@ -1710,25 +1718,23 @@ public void testAliasDateMathExpressionNotSupported() {
     public void testDynamicPutMappingRequestFromAlias() {
         PutMappingRequest request = new PutMappingRequest(Strings.EMPTY_ARRAY).setConcreteIndex(new Index("foofoo", UUIDs.base64UUID()));
         User user = new User("alias-writer", "alias_read_write");
-        Set<String> authorizedIndices = buildAuthorizedIndices(user, PutMappingAction.NAME);
+        AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, PutMappingAction.NAME);
 
-        String putMappingIndexOrAlias = IndicesAndAliasesResolver.getPutMappingIndexOrAlias(request, authorizedIndices, metadata);
+        String putMappingIndexOrAlias = IndicesAndAliasesResolver.getPutMappingIndexOrAlias(request, authorizedIndices::check, metadata);
         assertEquals("barbaz", putMappingIndexOrAlias);
 
         // multiple indices map to an alias so we can only return the concrete index
         final String index = randomFrom("foo", "foobar");
         request = new PutMappingRequest(Strings.EMPTY_ARRAY).setConcreteIndex(new Index(index, UUIDs.base64UUID()));
-        putMappingIndexOrAlias = IndicesAndAliasesResolver.getPutMappingIndexOrAlias(request, authorizedIndices, metadata);
+        putMappingIndexOrAlias = IndicesAndAliasesResolver.getPutMappingIndexOrAlias(request, authorizedIndices::check, metadata);
         assertEquals(index, putMappingIndexOrAlias);
-
     }
 
     public void testWhenAliasToMultipleIndicesAndUserIsAuthorizedUsingAliasReturnsAliasNameForDynamicPutMappingRequestOnWriteIndex() {
         String index = "logs-00003"; // write index
         PutMappingRequest request = new PutMappingRequest(Strings.EMPTY_ARRAY).setConcreteIndex(new Index(index, UUIDs.base64UUID()));
-        Set<String> authorizedIndices = Collections.singleton("logs-alias");
         assert metadata.getIndicesLookup().get("logs-alias").getIndices().size() == 3;
-        String putMappingIndexOrAlias = IndicesAndAliasesResolver.getPutMappingIndexOrAlias(request, authorizedIndices, metadata);
+        String putMappingIndexOrAlias = IndicesAndAliasesResolver.getPutMappingIndexOrAlias(request, "logs-alias"::equals, metadata);
         String message = "user is authorized to access `logs-alias` and the put mapping request is for a write index"
             + "so this should have returned the alias name";
         assertEquals(message, "logs-alias", putMappingIndexOrAlias);
@@ -1737,9 +1743,8 @@ public void testWhenAliasToMultipleIndicesAndUserIsAuthorizedUsingAliasReturnsAl
     public void testWhenAliasToMultipleIndicesAndUserIsAuthorizedUsingAliasReturnsIndexNameForDynamicPutMappingRequestOnReadIndex() {
         String index = "logs-00002"; // read index
         PutMappingRequest request = new PutMappingRequest(Strings.EMPTY_ARRAY).setConcreteIndex(new Index(index, UUIDs.base64UUID()));
-        Set<String> authorizedIndices = Collections.singleton("logs-alias");
         assert metadata.getIndicesLookup().get("logs-alias").getIndices().size() == 3;
-        String putMappingIndexOrAlias = IndicesAndAliasesResolver.getPutMappingIndexOrAlias(request, authorizedIndices, metadata);
+        String putMappingIndexOrAlias = IndicesAndAliasesResolver.getPutMappingIndexOrAlias(request, "logs-alias"::equals, metadata);
         String message = "user is authorized to access `logs-alias` and the put mapping request is for a read index"
             + "so this should have returned the concrete index as fallback";
         assertEquals(message, index, putMappingIndexOrAlias);
@@ -1748,7 +1753,7 @@ public void testWhenAliasToMultipleIndicesAndUserIsAuthorizedUsingAliasReturnsIn
     public void testHiddenIndicesResolution() {
         SearchRequest searchRequest = new SearchRequest();
         searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, true, true));
-        Set<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
+        AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
         ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(
             SearchAction.NAME,
             searchRequest,
@@ -1835,7 +1840,7 @@ public void testHiddenIndicesResolution() {
 
     public void testHiddenAliasesResolution() {
         final User user = new User("hidden-alias-tester", "hidden_alias_test");
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
 
         // Visible only
         SearchRequest searchRequest = new SearchRequest();
@@ -1904,7 +1909,7 @@ public void testDataStreamResolution() {
             SearchRequest searchRequest = new SearchRequest();
             searchRequest.indices("logs-*");
             searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, false, false, true, true, true, true));
-            final Set<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME, searchRequest);
+            final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME, searchRequest);
             ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(
                 SearchAction.NAME,
                 searchRequest,
@@ -1935,7 +1940,7 @@ public void testDataStreamResolution() {
             SearchRequest searchRequest = new SearchRequest();
             searchRequest.indices("logs-*");
             searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, false, false, true, true, true, true));
-            final Set<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME, searchRequest);
+            final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME, searchRequest);
             ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(
                 SearchAction.NAME,
                 searchRequest,
@@ -1956,13 +1961,16 @@ public void testDataStreamsAreNotVisibleWhenNotIncludedByRequestWithWildcard() {
         // data streams and their backing indices should _not_ be in the authorized list since the backing indices
         // do not match the requested pattern
         List<String> dataStreams = List.of("logs-foo", "logs-foobar");
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
         for (String dsName : dataStreams) {
-            assertThat(authorizedIndices, hasItem(dsName));
+            assertThat(authorizedIndices.all().get(), hasItem(dsName));
+            assertThat(authorizedIndices.check(dsName), is(true));
             DataStream dataStream = metadata.dataStreams().get(dsName);
-            assertThat(authorizedIndices, hasItem(dsName));
+            assertThat(authorizedIndices.all().get(), hasItem(dsName));
+            assertThat(authorizedIndices.check(dsName), is(true));
             for (Index i : dataStream.getIndices()) {
-                assertThat(authorizedIndices, hasItem(i.getName()));
+                assertThat(authorizedIndices.all().get(), hasItem(i.getName()));
+                assertThat(authorizedIndices.check(i.getName()), is(true));
             }
         }
 
@@ -1993,12 +2001,15 @@ public void testDataStreamsAreNotVisibleWhenNotIncludedByRequestWithoutWildcard(
 
         // data streams and their backing indices should _not_ be in the authorized list since the backing indices
         // do not match the requested name
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
-        assertThat(authorizedIndices, hasItem(dataStreamName));
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
+        assertThat(authorizedIndices.all().get(), hasItem(dataStreamName));
+        assertThat(authorizedIndices.check(dataStreamName), is(true));
         DataStream dataStream = metadata.dataStreams().get(dataStreamName);
-        assertThat(authorizedIndices, hasItem(dataStreamName));
+        assertThat(authorizedIndices.all().get(), hasItem(dataStreamName));
+        assertThat(authorizedIndices.check(dataStreamName), is(true));
         for (Index i : dataStream.getIndices()) {
-            assertThat(authorizedIndices, hasItem(i.getName()));
+            assertThat(authorizedIndices.all().get(), hasItem(i.getName()));
+            assertThat(authorizedIndices.check(i.getName()), is(true));
         }
 
         // neither data streams nor their backing indices will be in the resolved list since the backing indices do not match the
@@ -2023,12 +2034,14 @@ public void testDataStreamsAreVisibleWhenIncludedByRequestWithWildcard() {
 
         // data streams and their backing indices should be in the authorized list
         List<String> expectedDataStreams = List.of("logs-foo", "logs-foobar");
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME, request);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME, request);
         for (String dsName : expectedDataStreams) {
             DataStream dataStream = metadata.dataStreams().get(dsName);
-            assertThat(authorizedIndices, hasItem(dsName));
+            assertThat(authorizedIndices.all().get(), hasItem(dsName));
+            assertThat(authorizedIndices.check(dsName), is(true));
             for (Index i : dataStream.getIndices()) {
-                assertThat(authorizedIndices, hasItem(i.getName()));
+                assertThat(authorizedIndices.all().get(), hasItem(i.getName()));
+                assertThat(authorizedIndices.check(i.getName()), is(true));
             }
         }
 
@@ -2063,11 +2076,13 @@ public void testDataStreamsAreVisibleWhenIncludedByRequestWithoutWildcard() {
         assertThat(request, instanceOf(IndicesRequest.Replaceable.class));
         assertThat(request.includeDataStreams(), is(true));
 
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME, request);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME, request);
         // data streams and their backing indices should be in the authorized list
-        assertThat(authorizedIndices, hasItem(dataStreamName));
+        assertThat(authorizedIndices.all().get(), hasItem(dataStreamName));
+        assertThat(authorizedIndices.check(dataStreamName), is(true));
         for (Index i : dataStream.getIndices()) {
-            assertThat(authorizedIndices, hasItem(i.getName()));
+            assertThat(authorizedIndices.all().get(), hasItem(i.getName()));
+            assertThat(authorizedIndices.check(i.getName()), is(true));
         }
 
         ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(
@@ -2092,12 +2107,14 @@ public void testBackingIndicesAreVisibleWhenIncludedByRequestWithWildcard() {
 
         // data streams and their backing indices should be included in the authorized list
         List<String> expectedDataStreams = List.of("logs-foo", "logs-foobar");
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME, request);
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME, request);
         for (String dsName : expectedDataStreams) {
             DataStream dataStream = metadata.dataStreams().get(dsName);
-            assertThat(authorizedIndices, hasItem(dsName));
+            assertThat(authorizedIndices.all().get(), hasItem(dsName));
+            assertThat(authorizedIndices.check(dsName), is(true));
             for (Index i : dataStream.getIndices()) {
-                assertThat(authorizedIndices, hasItem(i.getName()));
+                assertThat(authorizedIndices.all().get(), hasItem(i.getName()));
+                assertThat(authorizedIndices.check(i.getName()), is(true));
             }
         }
 
@@ -2127,12 +2144,15 @@ public void testBackingIndicesAreNotVisibleWhenNotIncludedByRequestWithoutWildca
 
         // data streams and their backing indices should _not_ be in the authorized list since the backing indices
         // did not match the requested pattern and the request does not support data streams
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
-        assertThat(authorizedIndices, hasItem(dataStreamName));
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
+        assertThat(authorizedIndices.all().get(), hasItem(dataStreamName));
+        assertThat(authorizedIndices.check(dataStreamName), is(true));
         DataStream dataStream = metadata.dataStreams().get(dataStreamName);
-        assertThat(authorizedIndices, hasItem(dataStreamName));
+        assertThat(authorizedIndices.all().get(), hasItem(dataStreamName));
+        assertThat(authorizedIndices.check(dataStreamName), is(true));
         for (Index i : dataStream.getIndices()) {
-            assertThat(authorizedIndices, hasItem(i.getName()));
+            assertThat(authorizedIndices.all().get(), hasItem(i.getName()));
+            assertThat(authorizedIndices.check(i.getName()), is(true));
         }
 
         // neither data streams nor their backing indices will be in the resolved list since the request does not support data streams
@@ -2158,12 +2178,16 @@ public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaWildcar
 
         // data streams should _not_ be in the authorized list but their backing indices that matched both the requested pattern
         // and the authorized pattern should be in the list
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
-        assertThat(authorizedIndices, not(hasItem("logs-foobar")));
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
+        assertThat(authorizedIndices.all().get(), not(hasItem("logs-foobar")));
+        assertThat(authorizedIndices.check("logs-foobar"), is(false));
         DataStream dataStream = metadata.dataStreams().get("logs-foobar");
-        assertThat(authorizedIndices, not(hasItem(indexName)));
+        assertThat(authorizedIndices.all().get(), not(hasItem(indexName)));
+        // request pattern is subset of the authorized pattern, but be aware that patterns are never passed to #check in main code
+        assertThat(authorizedIndices.check(indexName), is(true));
         for (Index i : dataStream.getIndices()) {
-            assertThat(authorizedIndices, hasItem(i.getName()));
+            assertThat(authorizedIndices.all().get(), hasItem(i.getName()));
+            assertThat(authorizedIndices.check(i.getName()), is(true));
         }
 
         // only the backing indices will be in the resolved list since the request does not support data streams
@@ -2176,7 +2200,7 @@ public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaWildcar
         );
         assertThat(resolvedIndices.getLocal(), not(hasItem(dataStream.getName())));
         for (Index i : dataStream.getIndices()) {
-            assertThat(authorizedIndices, hasItem(i.getName()));
+            assertThat(resolvedIndices.getLocal(), hasItem(i.getName()));
         }
     }
 
@@ -2189,9 +2213,11 @@ public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaNameAnd
 
         // data streams should _not_ be in the authorized list but a single backing index that matched the requested pattern
         // and the authorized name should be in the list
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
-        assertThat(authorizedIndices, not(hasItem("logs-foobar")));
-        assertThat(authorizedIndices, contains(DataStream.getDefaultBackingIndexName("logs-foobar", 1)));
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
+        assertThat(authorizedIndices.all().get(), not(hasItem("logs-foobar")));
+        assertThat(authorizedIndices.check("logs-foobar"), is(false));
+        assertThat(authorizedIndices.all().get(), contains(DataStream.getDefaultBackingIndexName("logs-foobar", 1)));
+        assertThat(authorizedIndices.check(DataStream.getDefaultBackingIndexName("logs-foobar", 1)), is(true));
 
         // only the single backing index will be in the resolved list since the request does not support data streams
         // but one of the backing indices matched the requested pattern
@@ -2214,12 +2240,16 @@ public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaWildcar
 
         // data streams should _not_ be in the authorized list but their backing indices that matched both the requested pattern
         // and the authorized pattern should be in the list
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
-        assertThat(authorizedIndices, not(hasItem("logs-foobar")));
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
+        assertThat(authorizedIndices.all().get(), not(hasItem("logs-foobar")));
+        assertThat(authorizedIndices.check("logs-foobar"), is(false));
         DataStream dataStream = metadata.dataStreams().get("logs-foobar");
-        assertThat(authorizedIndices, not(hasItem(indexName)));
+        assertThat(authorizedIndices.all().get(), not(hasItem(indexName)));
+        // request pattern is subset of the authorized pattern, but be aware that patterns are never passed to #check in main code
+        assertThat(authorizedIndices.check(indexName), is(true));
         for (Index i : dataStream.getIndices()) {
-            assertThat(authorizedIndices, hasItem(i.getName()));
+            assertThat(authorizedIndices.all().get(), hasItem(i.getName()));
+            assertThat(authorizedIndices.check(i.getName()), is(true));
         }
 
         // only the backing indices will be in the resolved list since the request does not support data streams
@@ -2232,7 +2262,7 @@ public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaWildcar
         );
         assertThat(resolvedIndices.getLocal(), not(hasItem(dataStream.getName())));
         for (Index i : dataStream.getIndices()) {
-            assertThat(authorizedIndices, hasItem(i.getName()));
+            assertThat(resolvedIndices.getLocal(), hasItem(i.getName()));
         }
     }
 
@@ -2245,9 +2275,11 @@ public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaNameAnd
 
         // data streams should _not_ be in the authorized list but a single backing index that matched the requested pattern
         // and the authorized name should be in the list
-        final Set<String> authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
-        assertThat(authorizedIndices, not(hasItem("logs-foobar")));
-        assertThat(authorizedIndices, contains(DataStream.getDefaultBackingIndexName("logs-foobar", 1)));
+        final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(user, GetAliasesAction.NAME, request);
+        assertThat(authorizedIndices.all().get(), not(hasItem("logs-foobar")));
+        assertThat(authorizedIndices.check("logs-foobar"), is(false));
+        assertThat(authorizedIndices.all().get(), contains(DataStream.getDefaultBackingIndexName("logs-foobar", 1)));
+        assertThat(authorizedIndices.check(DataStream.getDefaultBackingIndexName("logs-foobar", 1)), is(true));
 
         // only the single backing index will be in the resolved list since the request does not support data streams
         // but one of the backing indices matched the requested pattern
@@ -2261,18 +2293,19 @@ public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaNameAnd
         assertThat(resolvedIndices.getLocal(), contains(DataStream.getDefaultBackingIndexName("logs-foobar", 1)));
     }
 
-    private Set<String> buildAuthorizedIndices(User user, String action) {
+    private AuthorizedIndices buildAuthorizedIndices(User user, String action) {
         return buildAuthorizedIndices(user, action, TransportRequest.Empty.INSTANCE);
     }
 
-    private Set<String> buildAuthorizedIndices(User user, String action, TransportRequest request) {
+    private AuthorizedIndices buildAuthorizedIndices(User user, String action, TransportRequest request) {
         PlainActionFuture<Role> rolesListener = new PlainActionFuture<>();
         final Subject subject = new Subject(user, new RealmRef("test", "indices-aliases-resolver-tests", "node"));
         rolesStore.getRole(subject, rolesListener);
         return RBACEngine.resolveAuthorizedIndicesFromRole(
             rolesListener.actionGet(),
             getRequestInfo(request, action),
-            metadata.getIndicesLookup()
+            metadata.getIndicesLookup(),
+            () -> ignore -> {}
         );
     }
 
@@ -2281,11 +2314,11 @@ public static IndexMetadata.Builder indexBuilder(String index) {
             .settings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0));
     }
 
-    private ResolvedIndices resolveIndices(TransportRequest request, Set<String> authorizedIndices) {
+    private ResolvedIndices resolveIndices(TransportRequest request, AuthorizedIndices authorizedIndices) {
         return resolveIndices("indices:/" + randomAlphaOfLength(8), request, authorizedIndices);
     }
 
-    private ResolvedIndices resolveIndices(String action, TransportRequest request, Set<String> authorizedIndices) {
+    private ResolvedIndices resolveIndices(String action, TransportRequest request, AuthorizedIndices authorizedIndices) {
         return defaultIndicesResolver.resolve(action, request, this.metadata, authorizedIndices);
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/PreAuthorizationUtilsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/PreAuthorizationUtilsTests.java
new file mode 100644
index 000000000000..866626e7d01f
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/PreAuthorizationUtilsTests.java
@@ -0,0 +1,158 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authz;
+
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationContext;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.ParentActionAuthorization;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.RequestInfo;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
+import org.elasticsearch.xpack.core.security.authz.permission.Role;
+import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.security.authz.RBACEngine.RBACAuthorizationInfo;
+
+import java.util.Optional;
+
+import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.RESTRICTED_INDICES;
+import static org.elasticsearch.xpack.security.authz.PreAuthorizationUtils.maybeSkipChildrenActionAuthorization;
+import static org.elasticsearch.xpack.security.authz.PreAuthorizationUtils.shouldRemoveParentAuthorizationFromThreadContext;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+/**
+ * Unit tests for {@link PreAuthorizationUtils}.
+ */
+public class PreAuthorizationUtilsTests extends ESTestCase {
+
+    public void testMaybeSkipChildrenActionAuthorizationAddsParentAuthorizationHeader() {
+        String action = SearchAction.NAME;
+
+        Role role = Role.builder(RESTRICTED_INDICES, "test-role").add(IndexPrivilege.READ, "test-*").build();
+
+        AuthorizationContext parentAuthorizationContext = createAuthorizationContext(action, role, IndicesAccessControl.allowAll());
+        SecurityContext securityContext = new SecurityContext(Settings.EMPTY, new ThreadContext(Settings.EMPTY));
+
+        maybeSkipChildrenActionAuthorization(securityContext, parentAuthorizationContext);
+        assertThat(securityContext.getParentAuthorization(), notNullValue());
+        assertThat(securityContext.getParentAuthorization().action(), equalTo(action));
+    }
+
+    public void testMaybeSkipChildrenActionAuthorizationDoesNotAddHeaderForRandomAction() {
+        String action = "indices:data/" + randomAlphaOfLengthBetween(3, 8);
+
+        Role role = Role.builder(RESTRICTED_INDICES, "test-role").add(IndexPrivilege.READ, "test-*").build();
+
+        AuthorizationContext parentAuthorizationContext = createAuthorizationContext(action, role, IndicesAccessControl.allowAll());
+        SecurityContext securityContext = new SecurityContext(Settings.EMPTY, new ThreadContext(Settings.EMPTY));
+
+        maybeSkipChildrenActionAuthorization(securityContext, parentAuthorizationContext);
+        assertThat(securityContext.getParentAuthorization(), nullValue());
+    }
+
+    public void testShouldRemoveParentAuthorizationFromThreadContext() {
+        final String parentAction = SearchAction.NAME;
+        SecurityContext securityContextWithParentAuthorization = new SecurityContext(Settings.EMPTY, new ThreadContext(Settings.EMPTY));
+        securityContextWithParentAuthorization.setParentAuthorization(new ParentActionAuthorization(parentAction));
+
+        // We should not remove the parent authorization when child action is white-listed
+        assertThat(
+            shouldRemoveParentAuthorizationFromThreadContext(
+                Optional.empty(),
+                randomWhitelistedChildAction(parentAction),
+                securityContextWithParentAuthorization
+            ),
+            equalTo(false)
+        );
+
+        // We should not remove when there is nothing to be removed
+        assertThat(
+            shouldRemoveParentAuthorizationFromThreadContext(
+                Optional.ofNullable(randomBoolean() ? "my_remote_cluster" : null),
+                randomWhitelistedChildAction(parentAction),
+                new SecurityContext(Settings.EMPTY, new ThreadContext(Settings.EMPTY))
+            ),
+            equalTo(false)
+        );
+
+        // Even-though the child action is white-listed for the parent action,
+        // we expect to remove parent authorization when targeting remote cluster
+        assertThat(
+            shouldRemoveParentAuthorizationFromThreadContext(
+                Optional.of("my_remote_cluster"),
+                randomWhitelistedChildAction(parentAction),
+                securityContextWithParentAuthorization
+            ),
+            equalTo(true)
+        );
+
+        // The parent authorization should be removed in either case:
+        // - we are sending a transport request to a remote cluster
+        // - or the child action is not white-listed for the parent
+        assertThat(
+            shouldRemoveParentAuthorizationFromThreadContext(
+                Optional.ofNullable(randomBoolean() ? "my_remote_cluster" : null),
+                randomAlphaOfLengthBetween(3, 8),
+                securityContextWithParentAuthorization
+            ),
+            equalTo(true)
+        );
+    }
+
+    public void testShouldPreAuthorizeChildByParentAction() {
+        final String parentAction = SearchAction.NAME;
+        final String childAction = randomWhitelistedChildAction(parentAction);
+
+        ParentActionAuthorization parentAuthorization = new ParentActionAuthorization(parentAction);
+        Authentication authentication = Authentication.newRealmAuthentication(
+            new User("username1", "role1"),
+            new RealmRef("realm1", "native", "node1")
+        );
+        RequestInfo requestInfo = new RequestInfo(authentication, new SearchRequest("test-index"), childAction, null, parentAuthorization);
+
+        Role role = Role.builder(RESTRICTED_INDICES, "role1").add(IndexPrivilege.READ, "test-*").build();
+        RBACAuthorizationInfo authzInfo = new RBACAuthorizationInfo(role, null);
+
+        assertThat(PreAuthorizationUtils.shouldPreAuthorizeChildByParentAction(requestInfo, authzInfo), equalTo(true));
+    }
+
+    public void testShouldPreAuthorizeChildByParentActionWhenParentAndChildAreSame() {
+        final String parentAction = SearchAction.NAME;
+        final String childAction = parentAction;
+
+        ParentActionAuthorization parentAuthorization = new ParentActionAuthorization(parentAction);
+        Authentication authentication = Authentication.newRealmAuthentication(
+            new User("username1", "role1"),
+            new RealmRef("realm1", "native", "node1")
+        );
+        RequestInfo requestInfo = new RequestInfo(authentication, new SearchRequest("test-index"), childAction, null, parentAuthorization);
+
+        Role role = Role.builder(RESTRICTED_INDICES, "role1").add(IndexPrivilege.READ, "test-*").build();
+        RBACAuthorizationInfo authzInfo = new RBACAuthorizationInfo(role, null);
+
+        assertThat(PreAuthorizationUtils.shouldPreAuthorizeChildByParentAction(requestInfo, authzInfo), equalTo(false));
+    }
+
+    private String randomWhitelistedChildAction(String parentAction) {
+        return randomFrom(PreAuthorizationUtils.CHILD_ACTIONS_PRE_AUTHORIZED_BY_PARENT.get(parentAction));
+    }
+
+    private AuthorizationContext createAuthorizationContext(String action, Role role, IndicesAccessControl accessControl) {
+        RBACAuthorizationInfo authzInfo = new RBACAuthorizationInfo(role, null);
+        return new AuthorizationContext(action, authzInfo, accessControl);
+    }
+
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java
index 1cb84b4dcada..a60abd39441d 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.security.authz;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
@@ -25,12 +26,14 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.license.GetLicenseAction;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.XPackPlugin;
@@ -55,12 +58,19 @@
 import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.ldap.LdapRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.pki.PkiRealmSettings;
-import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AsyncSupplier;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizedIndices;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.IndexAuthorizationResult;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.ParentActionAuthorization;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.PrivilegesCheckResult;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.PrivilegesToCheck;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.RequestInfo;
+import org.elasticsearch.xpack.core.security.authz.ResolvedIndices;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.ApplicationResourcePrivileges;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection;
 import org.elasticsearch.xpack.core.security.authz.permission.ApplicationPermission;
 import org.elasticsearch.xpack.core.security.authz.permission.ClusterPermission;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
@@ -83,23 +93,29 @@
 import org.elasticsearch.xpack.security.authz.RBACEngine.RBACAuthorizationInfo;
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
 import org.hamcrest.Matchers;
+import org.junit.Assert;
 import org.junit.Before;
 import org.mockito.Mockito;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.TreeMap;
-import java.util.function.Predicate;
-import java.util.function.Supplier;
+import java.util.concurrent.ExecutionException;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static java.util.Collections.emptyMap;
+import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_VERSION_CREATED;
+import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
+import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.common.util.set.Sets.newHashSet;
 import static org.elasticsearch.test.ActionListenerUtils.anyActionListener;
 import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.RESTRICTED_INDICES;
@@ -126,6 +142,7 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
@@ -153,7 +170,7 @@ public void testResolveAuthorizationInfoForEmptyRolesWithAuthentication() {
 
         final PlainActionFuture<AuthorizationInfo> future = new PlainActionFuture<>();
         engine.resolveAuthorizationInfo(
-            new AuthorizationEngine.RequestInfo(
+            new RequestInfo(
                 AuthenticationTestHelper.builder().build(),
                 mock(TransportRequest.class),
                 randomAlphaOfLengthBetween(20, 30),
@@ -1462,7 +1479,6 @@ public void testBuildUserPrivilegeResponse() {
     public void testBackingIndicesAreIncludedForAuthorizedDataStreams() {
         final String dataStreamName = "my_data_stream";
         User user = new User(randomAlphaOfLengthBetween(4, 12));
-        Authentication authentication = AuthenticationTestHelper.builder().user(user).build();
         Role role = Role.builder(RESTRICTED_INDICES, "test1")
             .cluster(Collections.singleton("all"), Collections.emptyList())
             .add(IndexPrivilege.READ, dataStreamName)
@@ -1485,24 +1501,26 @@ public void testBackingIndicesAreIncludedForAuthorizedDataStreams() {
         }
 
         SearchRequest request = new SearchRequest("*");
-        Set<String> authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(
+        AuthorizedIndices authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(
             role,
             getRequestInfo(request, SearchAction.NAME),
-            lookup
+            lookup,
+            () -> ignore -> {}
         );
-        // The authorized indices is the lazily loading set implementation
-        assertThat(authorizedIndices, instanceOf(RBACEngine.AuthorizedIndicesSet.class));
-        assertThat(authorizedIndices, hasItem(dataStreamName));
+        assertThat(authorizedIndices.all().get(), hasItem(dataStreamName));
+        assertThat(authorizedIndices.check(dataStreamName), is(true));
         assertThat(
-            authorizedIndices,
+            authorizedIndices.all().get(),
             hasItems(backingIndices.stream().map(im -> im.getIndex().getName()).collect(Collectors.toList()).toArray(Strings.EMPTY_ARRAY))
         );
+        for (String index : backingIndices.stream().map(im -> im.getIndex().getName()).toList()) {
+            assertThat(authorizedIndices.check(index), is(true));
+        }
     }
 
     public void testExplicitMappingUpdatesAreNotGrantedWithIngestPrivileges() {
         final String dataStreamName = "my_data_stream";
         User user = new User(randomAlphaOfLengthBetween(4, 12));
-        Authentication authentication = AuthenticationTestHelper.builder().user(user).build();
         Role role = Role.builder(RESTRICTED_INDICES, "test1")
             .cluster(Collections.emptySet(), Collections.emptyList())
             .add(IndexPrivilege.CREATE, "my_*")
@@ -1527,14 +1545,13 @@ public void testExplicitMappingUpdatesAreNotGrantedWithIngestPrivileges() {
 
         PutMappingRequest request = new PutMappingRequest("*");
         request.source("{ \"properties\": { \"message\": { \"type\": \"text\" } } }", XContentType.JSON);
-        Set<String> authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(
+        AuthorizedIndices authorizedIndices = RBACEngine.resolveAuthorizedIndicesFromRole(
             role,
             getRequestInfo(request, PutMappingAction.NAME),
-            lookup
+            lookup,
+            () -> ignore -> {}
         );
-        // The authorized indices is the lazily loading set implementation
-        assertThat(authorizedIndices, instanceOf(RBACEngine.AuthorizedIndicesSet.class));
-        assertThat(authorizedIndices.isEmpty(), is(true));
+        assertThat(authorizedIndices.all().get().isEmpty(), is(true));
     }
 
     public void testNoInfiniteRecursionForRBACAuthorizationInfoHashCode() {
@@ -1544,54 +1561,6 @@ public void testNoInfiniteRecursionForRBACAuthorizationInfoHashCode() {
         new RBACAuthorizationInfo(role, null).hashCode();
     }
 
-    @SuppressWarnings("unchecked")
-    public void testLazinessForAuthorizedIndicesSet() {
-        final Set<String> authorizedNames = Set.of("foo", "bar", "baz");
-        final HashSet<String> allNames = new HashSet<>(authorizedNames);
-        allNames.addAll(Set.of("buzz", "fiz"));
-
-        final Supplier<Set<String>> supplier = mock(Supplier.class);
-        when(supplier.get()).thenReturn(authorizedNames);
-        final Predicate<String> predicate = mock(Predicate.class);
-        doAnswer(invocation -> {
-            final String name = (String) invocation.getArguments()[0];
-            return authorizedNames.contains(name);
-        }).when(predicate).test(anyString());
-        final RBACEngine.AuthorizedIndicesSet authorizedIndicesSet = new RBACEngine.AuthorizedIndicesSet(supplier, predicate);
-
-        // Check with contains or containsAll do not trigger loading
-        final String name1 = randomFrom(allNames);
-        final String name2 = randomValueOtherThan(name1, () -> randomFrom(allNames));
-        final boolean containsAll = randomBoolean();
-        if (containsAll) {
-            assertThat(authorizedIndicesSet.containsAll(Set.of(name1, name2)), equalTo(authorizedNames.containsAll(Set.of(name1, name2))));
-        } else {
-            assertThat(authorizedIndicesSet.contains(name1), equalTo(authorizedNames.contains(name1)));
-        }
-        verify(supplier, never()).get();
-        verify(predicate, atLeastOnce()).test(anyString());
-
-        // Iterating through the set triggers loading
-        Mockito.clearInvocations(predicate);
-        final Set<String> collectedNames = new HashSet<>();
-        for (String name : authorizedIndicesSet) {
-            collectedNames.add(name);
-        }
-        verify(supplier).get();
-        assertThat(collectedNames, equalTo(authorizedNames));
-
-        // Check with contains and containsAll again now uses the loaded set not the predicate anymore
-        Mockito.clearInvocations(supplier);
-        if (containsAll) {
-            assertThat(authorizedIndicesSet.containsAll(Set.of(name1, name2)), equalTo(authorizedNames.containsAll(Set.of(name1, name2))));
-        } else {
-            assertThat(authorizedIndicesSet.contains(name1), equalTo(authorizedNames.contains(name1)));
-        }
-        verify(predicate, never()).test(anyString());
-        // It also does not load twice
-        verify(supplier, never()).get();
-    }
-
     public void testGetUserPrivilegesThrowsIaeForUnsupportedOperation() {
         final RBACAuthorizationInfo authorizationInfo = mock(RBACAuthorizationInfo.class);
         final Role role = mock(Role.class);
@@ -1627,6 +1596,350 @@ public void testGetUserPrivilegesThrowsIaeForUnsupportedOperation() {
         assertThat(e.getCause(), sameInstance(unsupportedOperationException));
     }
 
+    public void testGetRemoteAccessRoleDescriptorsIntersection() throws ExecutionException, InterruptedException {
+        assumeTrue("untrusted remote cluster feature flag must be enabled", TcpTransport.isUntrustedRemoteClusterEnabled());
+
+        final RemoteIndicesPermission.Builder remoteIndicesBuilder = RemoteIndicesPermission.builder();
+        final String concreteClusterAlias = randomAlphaOfLength(10);
+        final int numGroups = randomIntBetween(1, 3);
+        final List<IndicesPrivileges> expectedIndicesPrivileges = new ArrayList<>();
+        for (int i = 0; i < numGroups; i++) {
+            final String[] indexNames = Objects.requireNonNull(generateRandomStringArray(3, 10, false, false));
+            final boolean allowRestrictedIndices = randomBoolean();
+            final boolean hasFls = randomBoolean();
+            final FieldPermissionsDefinition.FieldGrantExcludeGroup group = hasFls
+                ? randomFieldGrantExcludeGroup()
+                : new FieldPermissionsDefinition.FieldGrantExcludeGroup(null, null);
+            final BytesReference query = randomBoolean() ? randomDlsQuery() : null;
+            final IndicesPrivileges.Builder builder = IndicesPrivileges.builder()
+                .indices(Arrays.stream(indexNames).sorted().collect(Collectors.toList()))
+                .privileges("read")
+                .allowRestrictedIndices(allowRestrictedIndices)
+                .query(query);
+            if (hasFls) {
+                builder.grantedFields(group.getGrantedFields());
+                builder.deniedFields(group.getExcludedFields());
+            }
+            expectedIndicesPrivileges.add(builder.build());
+            remoteIndicesBuilder.addGroup(
+                Set.of(randomFrom(concreteClusterAlias, "*")),
+                IndexPrivilege.READ,
+                new FieldPermissions(new FieldPermissionsDefinition(Set.of(group))),
+                query == null ? null : Set.of(query),
+                allowRestrictedIndices,
+                indexNames
+            );
+        }
+        final String mismatchedConcreteClusterAlias = randomValueOtherThan(concreteClusterAlias, () -> randomAlphaOfLength(10));
+        // Add some groups that don't match the alias
+        final int numMismatchedGroups = randomIntBetween(0, 3);
+        for (int i = 0; i < numMismatchedGroups; i++) {
+            remoteIndicesBuilder.addGroup(
+                Set.of(mismatchedConcreteClusterAlias),
+                IndexPrivilege.READ,
+                new FieldPermissions(
+                    new FieldPermissionsDefinition(Set.of(new FieldPermissionsDefinition.FieldGrantExcludeGroup(null, null)))
+                ),
+                null,
+                randomBoolean(),
+                generateRandomStringArray(3, 10, false, false)
+            );
+        }
+
+        final Role role = mockRoleWithRemoteIndices(remoteIndicesBuilder.build());
+        final RBACAuthorizationInfo authorizationInfo = mock(RBACAuthorizationInfo.class);
+        when(authorizationInfo.getRole()).thenReturn(role);
+
+        final PlainActionFuture<RoleDescriptorsIntersection> future = new PlainActionFuture<>();
+        engine.getRemoteAccessRoleDescriptorsIntersection(concreteClusterAlias, authorizationInfo, future);
+        final RoleDescriptorsIntersection actual = future.get();
+
+        assertThat(
+            actual,
+            equalTo(
+                new RoleDescriptorsIntersection(
+                    List.of(
+                        Set.of(
+                            new RoleDescriptor(
+                                RBACEngine.REMOTE_USER_ROLE_NAME,
+                                null,
+                                expectedIndicesPrivileges.stream().sorted().toArray(RoleDescriptor.IndicesPrivileges[]::new),
+                                null,
+                                null,
+                                null,
+                                null,
+                                null
+                            )
+                        )
+                    )
+                )
+            )
+        );
+        verify(role, times(1)).remoteIndices();
+        verifyNoMoreInteractions(role);
+    }
+
+    public void testGetRemoteAccessRoleDescriptorsIntersectionHasDeterministicOrderForIndicesPrivileges() throws ExecutionException,
+        InterruptedException {
+        assumeTrue("untrusted remote cluster feature flag must be enabled", TcpTransport.isUntrustedRemoteClusterEnabled());
+
+        final RemoteIndicesPermission.Builder remoteIndicesBuilder = RemoteIndicesPermission.builder();
+        final String concreteClusterAlias = randomAlphaOfLength(10);
+        final int numGroups = randomIntBetween(2, 5);
+        for (int i = 0; i < numGroups; i++) {
+            remoteIndicesBuilder.addGroup(
+                Set.copyOf(randomNonEmptySubsetOf(List.of(concreteClusterAlias, "*"))),
+                IndexPrivilege.get(Set.copyOf(randomSubsetOf(randomIntBetween(1, 4), IndexPrivilege.names()))),
+                new FieldPermissions(
+                    new FieldPermissionsDefinition(
+                        Set.of(
+                            randomBoolean()
+                                ? randomFieldGrantExcludeGroup()
+                                : new FieldPermissionsDefinition.FieldGrantExcludeGroup(null, null)
+                        )
+                    )
+                ),
+                randomBoolean() ? Set.of(randomDlsQuery()) : null,
+                randomBoolean(),
+                generateRandomStringArray(3, 10, false, false)
+            );
+        }
+        final RemoteIndicesPermission permissions = remoteIndicesBuilder.build();
+        List<RemoteIndicesPermission.RemoteIndicesGroup> remoteIndicesGroups = permissions.remoteIndicesGroups();
+        final Role role1 = mockRoleWithRemoteIndices(permissions);
+        final RBACAuthorizationInfo authorizationInfo1 = mock(RBACAuthorizationInfo.class);
+        when(authorizationInfo1.getRole()).thenReturn(role1);
+        final PlainActionFuture<RoleDescriptorsIntersection> future1 = new PlainActionFuture<>();
+        engine.getRemoteAccessRoleDescriptorsIntersection(concreteClusterAlias, authorizationInfo1, future1);
+        final RoleDescriptorsIntersection actual1 = future1.get();
+        verify(role1, times(1)).remoteIndices();
+        verifyNoMoreInteractions(role1);
+
+        // Randomize the order of both remote indices groups and each of the indices permissions groups each group holds
+        final RemoteIndicesPermission shuffledPermissions = new RemoteIndicesPermission(
+            shuffledList(
+                remoteIndicesGroups.stream()
+                    .map(
+                        group -> new RemoteIndicesPermission.RemoteIndicesGroup(
+                            group.remoteClusterAliases(),
+                            shuffledList(group.indicesPermissionGroups())
+                        )
+                    )
+                    .toList()
+            )
+        );
+        final Role role2 = mockRoleWithRemoteIndices(shuffledPermissions);
+        final RBACAuthorizationInfo authorizationInfo2 = mock(RBACAuthorizationInfo.class);
+        when(authorizationInfo2.getRole()).thenReturn(role2);
+        final PlainActionFuture<RoleDescriptorsIntersection> future2 = new PlainActionFuture<>();
+        engine.getRemoteAccessRoleDescriptorsIntersection(concreteClusterAlias, authorizationInfo2, future2);
+        final RoleDescriptorsIntersection actual2 = future2.get();
+
+        verify(role2, times(1)).remoteIndices();
+        verifyNoMoreInteractions(role2);
+        assertThat(actual1, equalTo(actual2));
+        assertThat(actual1.roleDescriptorsList().iterator().next().iterator().next().getIndicesPrivileges().length, equalTo(numGroups));
+    }
+
+    public void testGetRemoteAccessRoleDescriptorsIntersectionWithoutMatchingGroups() throws ExecutionException, InterruptedException {
+        assumeTrue("untrusted remote cluster feature flag must be enabled", TcpTransport.isUntrustedRemoteClusterEnabled());
+
+        final String concreteClusterAlias = randomAlphaOfLength(10);
+        final Role role = mockRoleWithRemoteIndices(
+            RemoteIndicesPermission.builder()
+                .addGroup(
+                    Set.of(concreteClusterAlias),
+                    IndexPrivilege.READ,
+                    new FieldPermissions(new FieldPermissionsDefinition(null, null)),
+                    null,
+                    randomBoolean(),
+                    generateRandomStringArray(3, 10, false, false)
+                )
+                .build()
+        );
+        final RBACAuthorizationInfo authorizationInfo = mock(RBACAuthorizationInfo.class);
+        when(authorizationInfo.getRole()).thenReturn(role);
+
+        final PlainActionFuture<RoleDescriptorsIntersection> future = new PlainActionFuture<>();
+        engine.getRemoteAccessRoleDescriptorsIntersection(
+            randomValueOtherThan(concreteClusterAlias, () -> randomAlphaOfLength(10)),
+            authorizationInfo,
+            future
+        );
+        final RoleDescriptorsIntersection actual = future.get();
+        assertThat(actual, equalTo(RoleDescriptorsIntersection.EMPTY));
+        verify(role, times(1)).remoteIndices();
+        verifyNoMoreInteractions(role);
+    }
+
+    public void testGetRemoteAccessRoleDescriptorsIntersectionWithoutRemoteIndicesPermissions() throws ExecutionException,
+        InterruptedException {
+        assumeTrue("untrusted remote cluster feature flag must be enabled", TcpTransport.isUntrustedRemoteClusterEnabled());
+
+        final String concreteClusterAlias = randomAlphaOfLength(10);
+        final Role role = mockRoleWithRemoteIndices(RemoteIndicesPermission.NONE);
+        final RBACAuthorizationInfo authorizationInfo = mock(RBACAuthorizationInfo.class);
+        when(authorizationInfo.getRole()).thenReturn(role);
+
+        final PlainActionFuture<RoleDescriptorsIntersection> future = new PlainActionFuture<>();
+        engine.getRemoteAccessRoleDescriptorsIntersection(
+            randomValueOtherThan(concreteClusterAlias, () -> randomAlphaOfLength(10)),
+            authorizationInfo,
+            future
+        );
+        final RoleDescriptorsIntersection actual = future.get();
+        assertThat(actual, equalTo(RoleDescriptorsIntersection.EMPTY));
+    }
+
+    public void testChildSearchActionAuthorizationIsSkipped() {
+        final String[] indices = { "test-index" };
+        final Role role = Mockito.spy(Role.builder(RESTRICTED_INDICES, "test-role").add(IndexPrivilege.READ, indices).build());
+
+        final String action = randomFrom(PreAuthorizationUtils.CHILD_ACTIONS_PRE_AUTHORIZED_BY_PARENT.get(SearchAction.NAME));
+        final ParentActionAuthorization parentAuthorization = new ParentActionAuthorization(SearchAction.NAME);
+
+        authorizeIndicesAction(indices, role, action, parentAuthorization, new ActionListener<IndexAuthorizationResult>() {
+            @Override
+            public void onResponse(IndexAuthorizationResult indexAuthorizationResult) {
+                assertTrue(indexAuthorizationResult.isGranted());
+                // Child authorization should be skipped since we passed parent authorization.
+                Mockito.verify(role, never()).checkIndicesAction(action);
+                Mockito.verify(role, never()).authorize(eq(action), any(), any(), any());
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                Assert.fail(e.getMessage());
+            }
+        });
+    }
+
+    public void testChildSearchActionIsAuthorizedWithoutSkipping() {
+        final String[] indices = { "test-index" };
+        final Role role = Mockito.spy(Role.builder(RESTRICTED_INDICES, "test-role").add(IndexPrivilege.READ, indices).build());
+
+        final String action = randomFrom(PreAuthorizationUtils.CHILD_ACTIONS_PRE_AUTHORIZED_BY_PARENT.get(SearchAction.NAME));
+        final ParentActionAuthorization parentAuthorization = null;
+
+        authorizeIndicesAction(indices, role, action, parentAuthorization, new ActionListener<IndexAuthorizationResult>() {
+            @Override
+            public void onResponse(IndexAuthorizationResult indexAuthorizationResult) {
+                assertTrue(indexAuthorizationResult.isGranted());
+                // Child action should have been authorized normally since we did not pass parent authorization
+                Mockito.verify(role, atLeastOnce()).authorize(eq(action), any(), any(), any());
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                Assert.fail(e.getMessage());
+            }
+        });
+    }
+
+    public void testChildSearchActionAuthorizationIsNotSkippedWhenRoleHasDLS() {
+        final String[] indices = { "test-index" };
+        final BytesArray query = new BytesArray("""
+            {"term":{"foo":bar}}""");
+        final Role role = Mockito.spy(
+            Role.builder(RESTRICTED_INDICES, "test-role")
+                .add(
+                    new FieldPermissions(new FieldPermissionsDefinition(new String[] { "foo" }, new String[0])),
+                    Set.of(query),
+                    IndexPrivilege.READ,
+                    randomBoolean(),
+                    indices
+                )
+                .build()
+        );
+
+        final String action = randomFrom(PreAuthorizationUtils.CHILD_ACTIONS_PRE_AUTHORIZED_BY_PARENT.get(SearchAction.NAME));
+        final ParentActionAuthorization parentAuthorization = new ParentActionAuthorization(SearchAction.NAME);
+
+        authorizeIndicesAction(indices, role, action, parentAuthorization, new ActionListener<IndexAuthorizationResult>() {
+            @Override
+            public void onResponse(IndexAuthorizationResult indexAuthorizationResult) {
+                assertTrue(indexAuthorizationResult.isGranted());
+                // Child action authorization should not be skipped, even though the parent authorization was present
+                Mockito.verify(role, atLeastOnce()).authorize(eq(action), any(), any(), any());
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                Assert.fail(e.getMessage());
+            }
+        });
+    }
+
+    public void testRandomChildSearchActionAuthorizionIsNotSkipped() {
+        final String[] indices = { "test-index" };
+        final Role role = Mockito.spy(Role.builder(RESTRICTED_INDICES, "test-role").add(IndexPrivilege.READ, indices).build());
+
+        final String action = SearchAction.NAME + "[" + randomAlphaOfLength(3) + "]";
+        final ParentActionAuthorization parentAuthorization = new ParentActionAuthorization(SearchAction.NAME);
+
+        authorizeIndicesAction(indices, role, action, parentAuthorization, new ActionListener<IndexAuthorizationResult>() {
+            @Override
+            public void onResponse(IndexAuthorizationResult indexAuthorizationResult) {
+                assertTrue(indexAuthorizationResult.isGranted());
+                Mockito.verify(role, atLeastOnce()).authorize(eq(action), any(), any(), any());
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                Assert.fail(e.getMessage());
+            }
+        });
+    }
+
+    private void authorizeIndicesAction(
+        final String[] indices,
+        final Role role,
+        final String action,
+        final ParentActionAuthorization parentAuthorization,
+        final ActionListener<IndexAuthorizationResult> listener
+    ) {
+
+        final RBACAuthorizationInfo authzInfo = new RBACAuthorizationInfo(role, null);
+        final ResolvedIndices resolvedIndices = new ResolvedIndices(List.of(indices), List.of());
+        final TransportRequest searchRequest = new SearchRequest(indices);
+        final RequestInfo requestInfo = createRequestInfo(searchRequest, action, parentAuthorization);
+        final AsyncSupplier<ResolvedIndices> indicesAsyncSupplier = s -> s.onResponse(resolvedIndices);
+
+        final Map<String, IndexAbstraction> aliasOrIndexLookup = Stream.of(indices)
+            .collect(
+                Collectors.toMap(
+                    i -> i,
+                    v -> new IndexAbstraction.ConcreteIndex(
+                        IndexMetadata.builder(v)
+                            .settings(
+                                Settings.builder()
+                                    .put(SETTING_NUMBER_OF_SHARDS, 1)
+                                    .put(SETTING_NUMBER_OF_REPLICAS, 0)
+                                    .put(SETTING_INDEX_VERSION_CREATED.getKey(), Version.CURRENT)
+                            )
+                            .build()
+                    )
+                )
+            );
+
+        engine.authorizeIndexAction(requestInfo, authzInfo, indicesAsyncSupplier, aliasOrIndexLookup, listener);
+    }
+
+    private static RequestInfo createRequestInfo(TransportRequest request, String action, ParentActionAuthorization parentAuthorization) {
+        final Authentication.RealmRef realm = new Authentication.RealmRef(
+            randomAlphaOfLength(6),
+            randomAlphaOfLength(4),
+            "node0" + randomIntBetween(1, 9)
+        );
+        return new RequestInfo(
+            AuthenticationTestHelper.builder().user(new User(randomAlphaOfLength(8))).realmRef(realm).build(false),
+            request,
+            action,
+            null,
+            parentAuthorization
+        );
+    }
+
     private GetUserPrivilegesResponse.Indices findIndexPrivilege(Set<GetUserPrivilegesResponse.Indices> indices, String name) {
         return indices.stream().filter(i -> i.getIndices().contains(name)).findFirst().get();
     }
@@ -1718,4 +2031,26 @@ private PrivilegesCheckResult hasPrivileges(
     private static MapBuilder<String, Boolean> mapBuilder() {
         return MapBuilder.newMapBuilder();
     }
+
+    private BytesArray randomDlsQuery() {
+        return new BytesArray(
+            "{ \"term\": { \"" + randomAlphaOfLengthBetween(3, 24) + "\" : \"" + randomAlphaOfLengthBetween(3, 24) + "\" }"
+        );
+    }
+
+    private FieldPermissionsDefinition.FieldGrantExcludeGroup randomFieldGrantExcludeGroup() {
+        return new FieldPermissionsDefinition.FieldGrantExcludeGroup(generateRandomStringArray(3, 10, false, false), new String[] {});
+    }
+
+    private Role mockRoleWithRemoteIndices(final RemoteIndicesPermission remoteIndicesPermission) {
+        final Role role = mock(Role.class);
+        final String[] roleNames = generateRandomStringArray(3, 10, false, false);
+        when(role.names()).thenReturn(roleNames);
+        when(role.cluster()).thenReturn(ClusterPermission.NONE);
+        when(role.indices()).thenReturn(IndicesPermission.NONE);
+        when(role.application()).thenReturn(ApplicationPermission.NONE);
+        when(role.runAs()).thenReturn(RunAsPermission.NONE);
+        when(role.remoteIndices()).thenReturn(remoteIndicesPermission);
+        return role;
+    }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractorTests.java
index 6cad0802f6e9..e716df684445 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldExtractorTests.java
@@ -10,7 +10,6 @@
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queries.spans.SpanTermQuery;
-import org.apache.lucene.sandbox.search.DocValuesNumbersQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.DisjunctionMaxQuery;
@@ -110,12 +109,6 @@ public void testFieldValue() {
         assertEquals(asSet("foo"), fields);
     }
 
-    public void testDocValuesNumbers() {
-        Set<String> fields = new HashSet<>();
-        FieldExtractor.extractFields(new DocValuesNumbersQuery("foo", 5L), fields);
-        assertEquals(asSet("foo"), fields);
-    }
-
     public void testTermInSet() {
         Set<String> fields = new HashSet<>();
         FieldExtractor.extractFields(new TermInSetQuery("foo", new BytesRef("baz"), new BytesRef("baz2")), fields);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/IndicesPermissionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/IndicesPermissionTests.java
index 13f94bb104b9..b0e46a5807af 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/IndicesPermissionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/IndicesPermissionTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
@@ -34,10 +35,12 @@
 import org.elasticsearch.xpack.core.security.authz.permission.IndicesPermission;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
+import org.elasticsearch.xpack.core.security.support.StringMatcher;
 import org.elasticsearch.xpack.core.security.test.TestRestrictedIndices;
 import org.elasticsearch.xpack.security.support.SecuritySystemIndices;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -45,6 +48,10 @@
 import java.util.SortedMap;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_VERSION_CREATED;
+import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
+import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
+import static org.elasticsearch.common.settings.Settings.builder;
 import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.RESTRICTED_INDICES;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -454,7 +461,7 @@ public void testAuthorizationForBackingIndices() {
         int numBackingIndices = randomIntBetween(1, 3);
         List<IndexMetadata> backingIndices = new ArrayList<>();
         for (int backingIndexNumber = 1; backingIndexNumber <= numBackingIndices; backingIndexNumber++) {
-            backingIndices.add(createIndexMetadata(DataStream.getDefaultBackingIndexName(dataStreamName, backingIndexNumber)));
+            backingIndices.add(createBackingIndexMetadata(DataStream.getDefaultBackingIndexName(dataStreamName, backingIndexNumber)));
         }
         DataStream ds = DataStreamTestHelper.newInstance(
             dataStreamName,
@@ -519,7 +526,7 @@ public void testAuthorizationForMappingUpdates() {
         int numBackingIndices = randomIntBetween(1, 3);
         List<IndexMetadata> backingIndices = new ArrayList<>();
         for (int backingIndexNumber = 1; backingIndexNumber <= numBackingIndices; backingIndexNumber++) {
-            backingIndices.add(createIndexMetadata(DataStream.getDefaultBackingIndexName("test_write2", backingIndexNumber)));
+            backingIndices.add(createBackingIndexMetadata(DataStream.getDefaultBackingIndexName("test_write2", backingIndexNumber)));
         }
         DataStream ds = DataStreamTestHelper.newInstance(
             "test_write2",
@@ -663,7 +670,112 @@ public void testIndicesPermissionHasFieldOrDocumentLevelSecurity() {
         assertThat(indicesPermission3.hasFieldOrDocumentLevelSecurity(), is(false));
     }
 
-    private static IndexMetadata createIndexMetadata(String name) {
+    public void testResourceAuthorizedPredicateForDatastreams() {
+        String dataStreamName = "logs-datastream";
+        Metadata.Builder mb = Metadata.builder(
+            DataStreamTestHelper.getClusterStateWithDataStreams(
+                List.of(Tuple.tuple(dataStreamName, 1)),
+                List.of(),
+                Instant.now().toEpochMilli(),
+                builder().build(),
+                1
+            ).getMetadata()
+        );
+        IndexAbstraction.DataStream dataStream = new IndexAbstraction.DataStream(mb.dataStream(dataStreamName));
+        IndexAbstraction backingIndex = new IndexAbstraction.ConcreteIndex(
+            DataStreamTestHelper.createBackingIndex(dataStreamName, 1).build(),
+            dataStream
+        );
+        IndexAbstraction concreteIndex = new IndexAbstraction.ConcreteIndex(
+            IndexMetadata.builder("logs-index")
+                .settings(
+                    Settings.builder()
+                        .put(SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put(SETTING_INDEX_VERSION_CREATED.getKey(), Version.CURRENT)
+                )
+                .build()
+        );
+        AliasMetadata aliasMetadata = new AliasMetadata.Builder("logs-alias").build();
+        IndexAbstraction alias = new IndexAbstraction.Alias(
+            aliasMetadata,
+            List.of(
+                IndexMetadata.builder("logs-index")
+                    .settings(
+                        Settings.builder()
+                            .put(SETTING_NUMBER_OF_SHARDS, 1)
+                            .put(SETTING_NUMBER_OF_REPLICAS, 0)
+                            .put(SETTING_INDEX_VERSION_CREATED.getKey(), Version.CURRENT)
+                    )
+                    .putAlias(aliasMetadata)
+                    .build()
+            )
+        );
+        IndicesPermission.IsResourceAuthorizedPredicate predicate = new IndicesPermission.IsResourceAuthorizedPredicate(
+            StringMatcher.of("other"),
+            StringMatcher.of(dataStreamName, backingIndex.getName(), concreteIndex.getName(), alias.getName())
+        );
+        assertThat(predicate.test(dataStream), is(false));
+        // test authorization for a missing resource with the datastream's name
+        assertThat(predicate.test(dataStream.getName(), null), is(true));
+        assertThat(predicate.test(backingIndex), is(false));
+        // test authorization for a missing resource with the backing index's name
+        assertThat(predicate.test(backingIndex.getName(), null), is(true));
+        assertThat(predicate.test(concreteIndex), is(true));
+        assertThat(predicate.test(alias), is(true));
+    }
+
+    public void testResourceAuthorizedPredicateAnd() {
+        IndicesPermission.IsResourceAuthorizedPredicate predicate1 = new IndicesPermission.IsResourceAuthorizedPredicate(
+            StringMatcher.of("c", "a"),
+            StringMatcher.of("b", "d")
+        );
+        IndicesPermission.IsResourceAuthorizedPredicate predicate2 = new IndicesPermission.IsResourceAuthorizedPredicate(
+            StringMatcher.of("c", "b"),
+            StringMatcher.of("a", "d")
+        );
+        Metadata.Builder mb = Metadata.builder(
+            DataStreamTestHelper.getClusterStateWithDataStreams(
+                List.of(Tuple.tuple("a", 1), Tuple.tuple("b", 1), Tuple.tuple("c", 1), Tuple.tuple("d", 1)),
+                List.of(),
+                Instant.now().toEpochMilli(),
+                builder().build(),
+                1
+            ).getMetadata()
+        );
+        IndexAbstraction.DataStream dataStreamA = new IndexAbstraction.DataStream(mb.dataStream("a"));
+        IndexAbstraction.DataStream dataStreamB = new IndexAbstraction.DataStream(mb.dataStream("b"));
+        IndexAbstraction.DataStream dataStreamC = new IndexAbstraction.DataStream(mb.dataStream("c"));
+        IndexAbstraction.DataStream dataStreamD = new IndexAbstraction.DataStream(mb.dataStream("d"));
+        IndexAbstraction concreteIndexA = concreteIndexAbstraction("a");
+        IndexAbstraction concreteIndexB = concreteIndexAbstraction("b");
+        IndexAbstraction concreteIndexC = concreteIndexAbstraction("c");
+        IndexAbstraction concreteIndexD = concreteIndexAbstraction("d");
+        IndicesPermission.IsResourceAuthorizedPredicate predicate = predicate1.and(predicate2);
+        assertThat(predicate.test(dataStreamA), is(false));
+        assertThat(predicate.test(dataStreamB), is(false));
+        assertThat(predicate.test(dataStreamC), is(true));
+        assertThat(predicate.test(dataStreamD), is(false));
+        assertThat(predicate.test(concreteIndexA), is(true));
+        assertThat(predicate.test(concreteIndexB), is(true));
+        assertThat(predicate.test(concreteIndexC), is(true));
+        assertThat(predicate.test(concreteIndexD), is(true));
+    }
+
+    private static IndexAbstraction concreteIndexAbstraction(String name) {
+        return new IndexAbstraction.ConcreteIndex(
+            IndexMetadata.builder(name)
+                .settings(
+                    Settings.builder()
+                        .put(SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put(SETTING_INDEX_VERSION_CREATED.getKey(), Version.CURRENT)
+                )
+                .build()
+        );
+    }
+
+    private static IndexMetadata createBackingIndexMetadata(String name) {
         Settings.Builder settingsBuilder = Settings.builder()
             .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
             .put("index.hidden", true);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/wrapper/DlsFlsFeatureTrackingIndicesAccessControlWrapperTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/wrapper/DlsFlsFeatureTrackingIndicesAccessControlWrapperTests.java
new file mode 100644
index 000000000000..2322e1b55fb3
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/wrapper/DlsFlsFeatureTrackingIndicesAccessControlWrapperTests.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authz.accesscontrol.wrapper;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.MockLicenseState;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl.IndexAccessControl;
+import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions;
+import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
+import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition;
+import org.mockito.Mockito;
+
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.xpack.core.security.SecurityField.DOCUMENT_LEVEL_SECURITY_FEATURE;
+import static org.elasticsearch.xpack.core.security.SecurityField.FIELD_LEVEL_SECURITY_FEATURE;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+public class DlsFlsFeatureTrackingIndicesAccessControlWrapperTests extends ESTestCase {
+
+    public void testDlsFlsFeatureUsageTracking() {
+
+        MockLicenseState licenseState = MockLicenseState.createMock();
+        Mockito.when(licenseState.isAllowed(DOCUMENT_LEVEL_SECURITY_FEATURE)).thenReturn(true);
+        Mockito.when(licenseState.isAllowed(FIELD_LEVEL_SECURITY_FEATURE)).thenReturn(true);
+        Settings settings = Settings.builder().put(XPackSettings.DLS_FLS_ENABLED.getKey(), true).build();
+        DlsFlsFeatureTrackingIndicesAccessControlWrapper wrapper = new DlsFlsFeatureTrackingIndicesAccessControlWrapper(
+            settings,
+            licenseState
+        );
+
+        String flsIndexName = "fls-index";
+        String dlsIndexName = "dls-index";
+        String dlsFlsIndexName = "dls-fls-index";
+        String noDlsFlsIndexName = "no-dls-fls-restriction-index";
+
+        FieldPermissions fieldPermissions = new FieldPermissions(
+            new FieldPermissionsDefinition(new String[] { "*" }, new String[] { "private" })
+        );
+        DocumentPermissions documentPermissions = DocumentPermissions.filteredBy(Set.of(new BytesArray("""
+            {"term":{"number":1}}""")));
+
+        IndicesAccessControl indicesAccessControl = wrapper.wrap(
+            new IndicesAccessControl(
+                true,
+                Map.ofEntries(
+                    Map.entry(noDlsFlsIndexName, new IndexAccessControl(FieldPermissions.DEFAULT, DocumentPermissions.allowAll())),
+                    Map.entry(flsIndexName, new IndexAccessControl(fieldPermissions, DocumentPermissions.allowAll())),
+                    Map.entry(dlsIndexName, new IndexAccessControl(FieldPermissions.DEFAULT, documentPermissions)),
+                    Map.entry(dlsFlsIndexName, new IndexAccessControl(fieldPermissions, documentPermissions))
+                )
+            )
+        );
+
+        // Accessing index which does not have DLS nor FLS should not track usage
+        indicesAccessControl.getIndexPermissions(randomFrom(noDlsFlsIndexName, randomAlphaOfLength(3)));
+        verify(licenseState, never()).featureUsed(any());
+
+        // FLS should be tracked
+        indicesAccessControl.getIndexPermissions(flsIndexName);
+        verify(licenseState, times(1)).featureUsed(FIELD_LEVEL_SECURITY_FEATURE);
+        verify(licenseState, times(0)).featureUsed(DOCUMENT_LEVEL_SECURITY_FEATURE);
+
+        // DLS should be tracked
+        indicesAccessControl.getIndexPermissions(dlsIndexName);
+        verify(licenseState, times(1)).featureUsed(FIELD_LEVEL_SECURITY_FEATURE);
+        verify(licenseState, times(1)).featureUsed(DOCUMENT_LEVEL_SECURITY_FEATURE);
+
+        // Both DLS and FLS should be tracked
+        indicesAccessControl.getIndexPermissions(dlsFlsIndexName);
+        verify(licenseState, times(2)).featureUsed(FIELD_LEVEL_SECURITY_FEATURE);
+        verify(licenseState, times(2)).featureUsed(DOCUMENT_LEVEL_SECURITY_FEATURE);
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestCacheDisablingInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestCacheDisablingInterceptorTests.java
new file mode 100644
index 000000000000..19f4faf62064
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestCacheDisablingInterceptorTests.java
@@ -0,0 +1,137 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authz.interceptor;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.ArrayUtils;
+import org.elasticsearch.license.MockLicenseState;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.RequestInfo;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
+import org.elasticsearch.xpack.core.security.user.AnonymousUser;
+import org.junit.After;
+import org.junit.Before;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.xpack.core.security.SecurityField.DOCUMENT_LEVEL_SECURITY_FEATURE;
+import static org.hamcrest.Matchers.is;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class SearchRequestCacheDisablingInterceptorTests extends ESTestCase {
+
+    private ClusterService clusterService;
+    private ThreadPool threadPool;
+    private MockLicenseState licenseState;
+    private SearchRequestCacheDisablingInterceptor interceptor;
+
+    @Before
+    public void init() {
+        threadPool = new TestThreadPool("search request interceptor tests");
+        licenseState = mock(MockLicenseState.class);
+        when(licenseState.isAllowed(DOCUMENT_LEVEL_SECURITY_FEATURE)).thenReturn(true);
+        clusterService = mock(ClusterService.class);
+        interceptor = new SearchRequestCacheDisablingInterceptor(threadPool, licenseState);
+    }
+
+    @After
+    public void stopThreadPool() {
+        terminate(threadPool);
+    }
+
+    private void configureMinMondeVersion(Version version) {
+        final ClusterState clusterState = mock(ClusterState.class);
+        when(clusterService.state()).thenReturn(clusterState);
+        final DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class);
+        when(clusterState.nodes()).thenReturn(discoveryNodes);
+        when(discoveryNodes.getMinNodeVersion()).thenReturn(version);
+    }
+
+    public void testRequestCacheWillBeDisabledWhenSearchRemoteIndices() {
+        configureMinMondeVersion(VersionUtils.randomVersion(random()));
+        final SearchRequest searchRequest = mock(SearchRequest.class);
+        when(searchRequest.source()).thenReturn(SearchSourceBuilder.searchSource());
+        RequestInfo requestInfo = new RequestInfo(
+            Authentication.newAnonymousAuthentication(new AnonymousUser(Settings.EMPTY), randomAlphaOfLengthBetween(3, 8)),
+            searchRequest,
+            SearchAction.NAME,
+            null
+        );
+
+        final String[] localIndices = randomArray(0, 3, String[]::new, () -> randomAlphaOfLengthBetween(3, 8));
+        final String[] remoteIndices = randomArray(
+            0,
+            3,
+            String[]::new,
+            () -> randomAlphaOfLengthBetween(0, 5) + ":" + randomAlphaOfLengthBetween(3, 8)
+        );
+        final ArrayList<String> allIndices = Arrays.stream(ArrayUtils.concat(localIndices, remoteIndices))
+            .collect(Collectors.toCollection(ArrayList::new));
+        Collections.shuffle(allIndices, random());
+        when(searchRequest.indices()).thenReturn(allIndices.toArray(String[]::new));
+
+        IndicesAccessControl indicesAccessControl = Mockito.mock(IndicesAccessControl.class);
+        when(indicesAccessControl.getFieldAndDocumentLevelSecurityUsage()).thenReturn(IndicesAccessControl.DlsFlsUsage.BOTH);
+        threadPool.getThreadContext().putTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY, indicesAccessControl);
+
+        final PlainActionFuture<Void> future = new PlainActionFuture<>();
+        interceptor.intercept(requestInfo, mock(AuthorizationEngine.class), mock(AuthorizationInfo.class), future);
+        future.actionGet();
+
+        if (remoteIndices.length > 0) {
+            verify(searchRequest).requestCache(false);
+        } else {
+            verify(searchRequest, never()).requestCache(anyBoolean());
+        }
+    }
+
+    public void testHasRemoteIndices() {
+        final SearchRequest searchRequest = mock(SearchRequest.class);
+        when(searchRequest.source()).thenReturn(SearchSourceBuilder.searchSource());
+        final String[] localIndices = randomArray(0, 3, String[]::new, () -> randomAlphaOfLengthBetween(3, 8));
+        final String[] remoteIndices = randomArray(
+            0,
+            3,
+            String[]::new,
+            () -> randomAlphaOfLengthBetween(0, 5) + ":" + randomAlphaOfLengthBetween(3, 8)
+        );
+        final ArrayList<String> allIndices = Arrays.stream(ArrayUtils.concat(localIndices, remoteIndices))
+            .collect(Collectors.toCollection(ArrayList::new));
+        Collections.shuffle(allIndices, random());
+        when(searchRequest.indices()).thenReturn(allIndices.toArray(String[]::new));
+
+        if (remoteIndices.length > 0) {
+            assertThat(SearchRequestCacheDisablingInterceptor.hasRemoteIndices(searchRequest), is(true));
+        } else {
+            assertThat(SearchRequestCacheDisablingInterceptor.hasRemoteIndices(searchRequest), is(false));
+        }
+    }
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestInterceptorTests.java
deleted file mode 100644
index 3226874f1899..000000000000
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/SearchRequestInterceptorTests.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.security.authz.interceptor;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.util.ArrayUtils;
-import org.elasticsearch.license.MockLicenseState;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.threadpool.TestThreadPool;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.junit.After;
-import org.junit.Before;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import static org.elasticsearch.xpack.core.security.SecurityField.DOCUMENT_LEVEL_SECURITY_FEATURE;
-import static org.hamcrest.Matchers.is;
-import static org.mockito.ArgumentMatchers.anyBoolean;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-public class SearchRequestInterceptorTests extends ESTestCase {
-
-    private ClusterService clusterService;
-    private ThreadPool threadPool;
-    private MockLicenseState licenseState;
-    private SearchRequestInterceptor interceptor;
-
-    @Before
-    public void init() {
-        threadPool = new TestThreadPool("search request interceptor tests");
-        licenseState = mock(MockLicenseState.class);
-        when(licenseState.isAllowed(DOCUMENT_LEVEL_SECURITY_FEATURE)).thenReturn(true);
-        clusterService = mock(ClusterService.class);
-        interceptor = new SearchRequestInterceptor(threadPool, licenseState, clusterService);
-    }
-
-    @After
-    public void stopThreadPool() {
-        terminate(threadPool);
-    }
-
-    private void configureMinMondeVersion(Version version) {
-        final ClusterState clusterState = mock(ClusterState.class);
-        when(clusterService.state()).thenReturn(clusterState);
-        final DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class);
-        when(clusterState.nodes()).thenReturn(discoveryNodes);
-        when(discoveryNodes.getMinNodeVersion()).thenReturn(version);
-    }
-
-    public void testRequestCacheWillBeDisabledWhenSearchRemoteIndices() {
-        configureMinMondeVersion(VersionUtils.randomVersion(random()));
-        final SearchRequest searchRequest = mock(SearchRequest.class);
-        when(searchRequest.source()).thenReturn(SearchSourceBuilder.searchSource());
-        final String[] localIndices = randomArray(0, 3, String[]::new, () -> randomAlphaOfLengthBetween(3, 8));
-        final String[] remoteIndices = randomArray(
-            0,
-            3,
-            String[]::new,
-            () -> randomAlphaOfLengthBetween(0, 5) + ":" + randomAlphaOfLengthBetween(3, 8)
-        );
-        final ArrayList<String> allIndices = Arrays.stream(ArrayUtils.concat(localIndices, remoteIndices))
-            .collect(Collectors.toCollection(ArrayList::new));
-        Collections.shuffle(allIndices, random());
-        when(searchRequest.indices()).thenReturn(allIndices.toArray(String[]::new));
-
-        final PlainActionFuture<Void> future = new PlainActionFuture<>();
-        interceptor.disableFeatures(searchRequest, Map.of(), future);
-        future.actionGet();
-        if (remoteIndices.length > 0) {
-            verify(searchRequest).requestCache(false);
-        } else {
-            verify(searchRequest, never()).requestCache(anyBoolean());
-        }
-    }
-
-    public void testHasRemoteIndices() {
-        final SearchRequest searchRequest = mock(SearchRequest.class);
-        when(searchRequest.source()).thenReturn(SearchSourceBuilder.searchSource());
-        final String[] localIndices = randomArray(0, 3, String[]::new, () -> randomAlphaOfLengthBetween(3, 8));
-        final String[] remoteIndices = randomArray(
-            0,
-            3,
-            String[]::new,
-            () -> randomAlphaOfLengthBetween(0, 5) + ":" + randomAlphaOfLengthBetween(3, 8)
-        );
-        final ArrayList<String> allIndices = Arrays.stream(ArrayUtils.concat(localIndices, remoteIndices))
-            .collect(Collectors.toCollection(ArrayList::new));
-        Collections.shuffle(allIndices, random());
-        when(searchRequest.indices()).thenReturn(allIndices.toArray(String[]::new));
-
-        if (remoteIndices.length > 0) {
-            assertThat(SearchRequestInterceptor.hasRemoteIndices(searchRequest), is(true));
-        } else {
-            assertThat(SearchRequestInterceptor.hasRemoteIndices(searchRequest), is(false));
-        }
-    }
-}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/permission/PermissionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/permission/PermissionTests.java
index 6ff8549c5eed..a5f99f685c7a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/permission/PermissionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/permission/PermissionTests.java
@@ -12,13 +12,13 @@
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.security.authz.RestrictedIndices;
+import org.elasticsearch.xpack.core.security.authz.permission.IndicesPermission.IsResourceAuthorizedPredicate;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.authz.privilege.Privilege;
 import org.elasticsearch.xpack.core.security.support.Automatons;
 import org.junit.Before;
 
 import java.util.List;
-import java.util.function.Predicate;
 
 import static org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege.CREATE;
 import static org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege.MONITOR;
@@ -49,7 +49,7 @@ public void testAllowedIndicesMatcherAction() throws Exception {
     public void testAllowedIndicesMatcherForMappingUpdates() throws Exception {
         for (String mappingUpdateActionName : List.of(PutMappingAction.NAME, AutoPutMappingAction.NAME)) {
             IndexAbstraction mockIndexAbstraction = mock(IndexAbstraction.class);
-            Predicate<IndexAbstraction> indexPredicate = permission.indices().allowedIndicesMatcher(mappingUpdateActionName);
+            IsResourceAuthorizedPredicate indexPredicate = permission.indices().allowedIndicesMatcher(mappingUpdateActionName);
             // mapping updates are still permitted on indices and aliases
             when(mockIndexAbstraction.getName()).thenReturn("ingest_foo" + randomAlphaOfLength(3));
             when(mockIndexAbstraction.getType()).thenReturn(IndexAbstraction.Type.CONCRETE_INDEX);
@@ -66,8 +66,8 @@ public void testAllowedIndicesMatcherForMappingUpdates() throws Exception {
     }
 
     public void testAllowedIndicesMatcherActionCaching() throws Exception {
-        Predicate<IndexAbstraction> matcher1 = permission.indices().allowedIndicesMatcher(GetAction.NAME);
-        Predicate<IndexAbstraction> matcher2 = permission.indices().allowedIndicesMatcher(GetAction.NAME);
+        IsResourceAuthorizedPredicate matcher1 = permission.indices().allowedIndicesMatcher(GetAction.NAME);
+        IsResourceAuthorizedPredicate matcher2 = permission.indices().allowedIndicesMatcher(GetAction.NAME);
         assertThat(matcher1, is(matcher2));
     }
 
@@ -88,7 +88,7 @@ public void testRunAs() {
     }
 
     // "baz_*foo", "/fool.*bar/"
-    private void testAllowedIndicesMatcher(Predicate<IndexAbstraction> indicesMatcher) {
+    private void testAllowedIndicesMatcher(IsResourceAuthorizedPredicate indicesMatcher) {
         assertThat(indicesMatcher.test(mockIndexAbstraction("foobar")), is(false));
         assertThat(indicesMatcher.test(mockIndexAbstraction("fool")), is(false));
         assertThat(indicesMatcher.test(mockIndexAbstraction("fool2bar")), is(true));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
index 75e11cf0c6fc..b418740b1f01 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
@@ -10,12 +10,14 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsAction;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsAction;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsAction;
+import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
 import org.elasticsearch.action.delete.DeleteAction;
 import org.elasticsearch.action.get.GetAction;
 import org.elasticsearch.action.index.IndexAction;
@@ -33,6 +35,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.set.Sets;
@@ -45,7 +48,7 @@
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockLogAppender;
-import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequest.Empty;
@@ -63,11 +66,15 @@
 import org.elasticsearch.xpack.core.security.authc.Subject;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.DocumentSubsetBitsetCache;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
 import org.elasticsearch.xpack.core.security.authz.permission.ClusterPermission;
+import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCache;
+import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition;
 import org.elasticsearch.xpack.core.security.authz.permission.IndicesPermission;
+import org.elasticsearch.xpack.core.security.authz.permission.IndicesPermission.IsResourceAuthorizedPredicate;
 import org.elasticsearch.xpack.core.security.authz.permission.RemoteIndicesPermission;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.authz.privilege.ActionClusterPrivilege;
@@ -97,6 +104,7 @@
 import org.elasticsearch.xpack.security.audit.index.IndexNameResolver;
 import org.elasticsearch.xpack.security.authc.ApiKeyService;
 import org.elasticsearch.xpack.security.authc.service.ServiceAccountService;
+import org.elasticsearch.xpack.security.authz.RBACEngine;
 import org.elasticsearch.xpack.security.support.CacheInvalidatorRegistry;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.hamcrest.BaseMatcher;
@@ -1024,7 +1032,7 @@ public ClusterPermission.Builder buildPermission(ClusterPermission.Builder build
         assertThat(role.cluster().check(PutUserAction.NAME, randomFrom(request1, request2), authentication), equalTo(true));
         assertThat(role.cluster().check(PutUserAction.NAME, request3, authentication), equalTo(false));
 
-        final Predicate<IndexAbstraction> allowedRead = role.indices().allowedIndicesMatcher(GetAction.NAME);
+        final IsResourceAuthorizedPredicate allowedRead = role.indices().allowedIndicesMatcher(GetAction.NAME);
         assertThat(allowedRead.test(mockIndexAbstraction("abc-123")), equalTo(true));
         assertThat(allowedRead.test(mockIndexAbstraction("xyz-000")), equalTo(true));
         assertThat(allowedRead.test(mockIndexAbstraction("ind-1-a")), equalTo(true));
@@ -1037,7 +1045,7 @@ public ClusterPermission.Builder buildPermission(ClusterPermission.Builder build
         assertThat(allowedRead.test(mockIndexAbstraction("remote-idx-2-1")), equalTo(false));
         assertThat(allowedRead.test(mockIndexAbstraction("remote-idx-3-1")), equalTo(false));
 
-        final Predicate<IndexAbstraction> allowedWrite = role.indices().allowedIndicesMatcher(IndexAction.NAME);
+        final IsResourceAuthorizedPredicate allowedWrite = role.indices().allowedIndicesMatcher(IndexAction.NAME);
         assertThat(allowedWrite.test(mockIndexAbstraction("abc-123")), equalTo(true));
         assertThat(allowedWrite.test(mockIndexAbstraction("xyz-000")), equalTo(false));
         assertThat(allowedWrite.test(mockIndexAbstraction("ind-1-a")), equalTo(true));
@@ -1080,6 +1088,76 @@ public void testBuildRoleWithSingleRemoteIndicesDefinition() {
         assertHasIndexGroupsForClusters(role.remoteIndices(), Set.of(clusterAlias), indexGroup("index-1"));
     }
 
+    public void testBuildRoleWithFlsAndDlsInRemoteIndicesDefinition() {
+        String clusterAlias = randomFrom("remote-1", "*");
+        Role role = buildRole(
+            roleDescriptorWithIndicesPrivileges(
+                "r1",
+                new RoleDescriptor.RemoteIndicesPrivileges[] {
+                    RoleDescriptor.RemoteIndicesPrivileges.builder(clusterAlias)
+                        .indices("index-1")
+                        .privileges("read")
+                        .query("{\"match\":{\"field\":\"a\"}}")
+                        .grantedFields("field")
+                        .build() }
+            )
+        );
+        assertHasRemoteGroupsForClusters(role.remoteIndices(), Set.of(clusterAlias));
+        assertHasIndexGroupsForClusters(
+            role.remoteIndices(),
+            Set.of(clusterAlias),
+            indexGroup(
+                IndexPrivilege.READ,
+                false,
+                "{\"match\":{\"field\":\"a\"}}",
+                new FieldPermissionsDefinition.FieldGrantExcludeGroup(new String[] { "field" }, null),
+                "index-1"
+            )
+        );
+
+        role = buildRole(
+            roleDescriptorWithIndicesPrivileges(
+                "r1",
+                new RoleDescriptor.RemoteIndicesPrivileges[] {
+                    RoleDescriptor.RemoteIndicesPrivileges.builder(clusterAlias)
+                        .indices("index-1")
+                        .privileges("read")
+                        .query("{\"match\":{\"field\":\"a\"}}")
+                        .grantedFields("field")
+                        .build() }
+            ),
+            roleDescriptorWithIndicesPrivileges(
+                "r1",
+                new RoleDescriptor.RemoteIndicesPrivileges[] {
+                    RoleDescriptor.RemoteIndicesPrivileges.builder(clusterAlias)
+                        .indices("index-1")
+                        .privileges("read")
+                        .query("{\"match\":{\"field\":\"b\"}}")
+                        .grantedFields("other")
+                        .build() }
+            )
+        );
+        assertHasRemoteGroupsForClusters(role.remoteIndices(), Set.of(clusterAlias));
+        assertHasIndexGroupsForClusters(
+            role.remoteIndices(),
+            Set.of(clusterAlias),
+            indexGroup(
+                IndexPrivilege.READ,
+                false,
+                "{\"match\":{\"field\":\"a\"}}",
+                new FieldPermissionsDefinition.FieldGrantExcludeGroup(new String[] { "field" }, null),
+                "index-1"
+            ),
+            indexGroup(
+                IndexPrivilege.READ,
+                false,
+                "{\"match\":{\"field\":\"b\"}}",
+                new FieldPermissionsDefinition.FieldGrantExcludeGroup(new String[] { "other" }, null),
+                "index-1"
+            )
+        );
+    }
+
     public void testBuildRoleWithEmptyOrNoneRemoteIndices() {
         Role role = buildRole(
             roleDescriptorWithIndicesPrivileges(
@@ -1124,12 +1202,12 @@ public void testBuildRoleWithRemoteIndicesDoesNotMergeWhenNothingToMerge() {
         assertHasRemoteGroupsForClusters(role.remoteIndices(), Set.of("remote-1"), Set.of("*"));
         assertHasIndexGroupsForClusters(role.remoteIndices(), Set.of("*"), indexGroup("index-1"));
         assertHasIndexGroupsForClusters(role.remoteIndices(), Set.of("remote-1"), indexGroup("index-1"));
-        final Predicate<IndexAbstraction> allowedRead = role.indices().allowedIndicesMatcher(GetAction.NAME);
+        final IsResourceAuthorizedPredicate allowedRead = role.indices().allowedIndicesMatcher(GetAction.NAME);
         assertThat(allowedRead.test(mockIndexAbstraction("index-1")), equalTo(true));
         assertThat(allowedRead.test(mockIndexAbstraction("foo")), equalTo(false));
     }
 
-    public void testBuildRoleWithRemoteIndicesMergesRemotesAndLocalsSeparately() {
+    public void testBuildRoleWithRemoteIndicesDoesNotCombineRemotesAndLocals() {
         Role role = buildRole(
             roleDescriptorWithIndicesPrivileges(
                 "r1",
@@ -1143,9 +1221,9 @@ public void testBuildRoleWithRemoteIndicesMergesRemotesAndLocalsSeparately() {
         );
         assertHasRemoteGroupsForClusters(role.remoteIndices(), Set.of("*"));
         assertHasIndexGroupsForClusters(role.remoteIndices(), Set.of("*"), indexGroup("index-1"));
-        final Predicate<IndexAbstraction> allowedRead = role.indices().allowedIndicesMatcher(GetAction.NAME);
+        final IsResourceAuthorizedPredicate allowedRead = role.indices().allowedIndicesMatcher(GetAction.NAME);
         assertThat(allowedRead.test(mockIndexAbstraction("index-1")), equalTo(true));
-        final Predicate<IndexAbstraction> allowedWrite = role.indices().allowedIndicesMatcher(IndexAction.NAME);
+        final IsResourceAuthorizedPredicate allowedWrite = role.indices().allowedIndicesMatcher(IndexAction.NAME);
         assertThat(allowedWrite.test(mockIndexAbstraction("index-1")), equalTo(true));
     }
 
@@ -1246,7 +1324,8 @@ public void testBuildRoleWithMultipleRemoteMergedAcrossPrivilegesAndDescriptors(
         assertHasIndexGroupsForClusters(
             role.remoteIndices(),
             Set.of("remote-1"),
-            indexGroup(IndexPrivilege.get(Set.of("read", "none")), false, "index-1")
+            indexGroup(IndexPrivilege.get(Set.of("read")), false, "index-1"),
+            indexGroup(IndexPrivilege.get(Set.of("none")), false, "index-1")
         );
     }
 
@@ -1733,13 +1812,17 @@ public void testApiKeyAuthUsesApiKeyService() throws Exception {
         }).when(nativeRolesStore).getRoleDescriptors(isASet(), anyActionListener());
         final ReservedRolesStore reservedRolesStore = spy(new ReservedRolesStore());
         ThreadContext threadContext = new ThreadContext(SECURITY_ENABLED_SETTINGS);
+        final ClusterService clusterService = mock(ClusterService.class);
+        when(clusterService.getClusterSettings()).thenReturn(
+            new ClusterSettings(SECURITY_ENABLED_SETTINGS, Set.of(ApiKeyService.DELETE_RETENTION_PERIOD))
+        );
         ApiKeyService apiKeyService = spy(
             new ApiKeyService(
                 SECURITY_ENABLED_SETTINGS,
                 Clock.systemUTC(),
                 mock(Client.class),
                 mock(SecurityIndexManager.class),
-                mock(ClusterService.class),
+                clusterService,
                 mock(CacheInvalidatorRegistry.class),
                 mock(ThreadPool.class)
             )
@@ -1767,7 +1850,10 @@ public void testApiKeyAuthUsesApiKeyService() throws Exception {
             rds -> effectiveRoleDescriptors.set(rds)
         );
         AuditUtil.getOrGenerateRequestId(threadContext);
-        final Version version = randomFrom(Version.CURRENT, VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.V_7_8_1));
+        final TransportVersion version = randomFrom(
+            TransportVersion.CURRENT,
+            TransportVersionUtils.randomVersionBetween(random(), TransportVersion.V_7_0_0, TransportVersion.V_7_8_1)
+        );
         final Authentication authentication = createApiKeyAuthentication(
             apiKeyService,
             randomValueOtherThanMany(
@@ -1784,7 +1870,7 @@ public void testApiKeyAuthUsesApiKeyService() throws Exception {
         Role role = roleFuture.actionGet();
         assertThat(effectiveRoleDescriptors.get(), is(nullValue()));
 
-        if (version == Version.CURRENT) {
+        if (version == TransportVersion.CURRENT) {
             verify(apiKeyService, times(1)).parseRoleDescriptorsBytes(anyString(), any(BytesReference.class), any());
         } else {
             verify(apiKeyService, times(1)).parseRoleDescriptors(anyString(), anyMap(), any());
@@ -1809,13 +1895,17 @@ public void testApiKeyAuthUsesApiKeyServiceWithScopedRole() throws Exception {
         final ReservedRolesStore reservedRolesStore = spy(new ReservedRolesStore());
         ThreadContext threadContext = new ThreadContext(SECURITY_ENABLED_SETTINGS);
 
+        final ClusterService clusterService = mock(ClusterService.class);
+        when(clusterService.getClusterSettings()).thenReturn(
+            new ClusterSettings(SECURITY_ENABLED_SETTINGS, Set.of(ApiKeyService.DELETE_RETENTION_PERIOD))
+        );
         ApiKeyService apiKeyService = spy(
             new ApiKeyService(
                 SECURITY_ENABLED_SETTINGS,
                 Clock.systemUTC(),
                 mock(Client.class),
                 mock(SecurityIndexManager.class),
-                mock(ClusterService.class),
+                clusterService,
                 mock(CacheInvalidatorRegistry.class),
                 mock(ThreadPool.class)
             )
@@ -1843,7 +1933,10 @@ public void testApiKeyAuthUsesApiKeyServiceWithScopedRole() throws Exception {
             rds -> effectiveRoleDescriptors.set(rds)
         );
         AuditUtil.getOrGenerateRequestId(threadContext);
-        final Version version = randomFrom(Version.CURRENT, VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.V_7_8_1));
+        final TransportVersion version = randomFrom(
+            TransportVersion.CURRENT,
+            TransportVersionUtils.randomVersionBetween(random(), TransportVersion.V_7_0_0, TransportVersion.V_7_8_1)
+        );
         final Authentication authentication = createApiKeyAuthentication(
             apiKeyService,
             randomValueOtherThanMany(
@@ -1861,7 +1954,7 @@ public void testApiKeyAuthUsesApiKeyServiceWithScopedRole() throws Exception {
         Role role = roleFuture.actionGet();
         assertThat(role.checkClusterAction("cluster:admin/foo", Empty.INSTANCE, AuthenticationTestHelper.builder().build()), is(false));
         assertThat(effectiveRoleDescriptors.get(), is(nullValue()));
-        if (version == Version.CURRENT) {
+        if (version == TransportVersion.CURRENT) {
             verify(apiKeyService).parseRoleDescriptorsBytes(
                 apiKeyId,
                 (BytesReference) authentication.getAuthenticatingSubject().getMetadata().get(API_KEY_ROLE_DESCRIPTORS_KEY),
@@ -1888,6 +1981,144 @@ public void testApiKeyAuthUsesApiKeyServiceWithScopedRole() throws Exception {
         assertThat(role.names()[0], containsString("user_role_"));
     }
 
+    public void testGetRoleForRemoteAccessAuthentication() throws Exception {
+        final FileRolesStore fileRolesStore = mock(FileRolesStore.class);
+        doCallRealMethod().when(fileRolesStore).accept(anySet(), anyActionListener());
+        final NativeRolesStore nativeRolesStore = mock(NativeRolesStore.class);
+        doCallRealMethod().when(nativeRolesStore).accept(anySet(), anyActionListener());
+        when(fileRolesStore.roleDescriptors(anySet())).thenReturn(Collections.emptySet());
+        doAnswer((invocationOnMock) -> {
+            @SuppressWarnings("unchecked")
+            ActionListener<RoleRetrievalResult> callback = (ActionListener<RoleRetrievalResult>) invocationOnMock.getArguments()[1];
+            callback.onResponse(RoleRetrievalResult.failure(new RuntimeException("intentionally failed!")));
+            return null;
+        }).when(nativeRolesStore).getRoleDescriptors(isASet(), anyActionListener());
+        final ReservedRolesStore reservedRolesStore = spy(new ReservedRolesStore());
+        ThreadContext threadContext = new ThreadContext(SECURITY_ENABLED_SETTINGS);
+        final ClusterService clusterService = mock(ClusterService.class);
+        when(clusterService.getClusterSettings()).thenReturn(
+            new ClusterSettings(SECURITY_ENABLED_SETTINGS, Set.of(ApiKeyService.DELETE_RETENTION_PERIOD))
+        );
+        final ApiKeyService apiKeyService = spy(
+            new ApiKeyService(
+                SECURITY_ENABLED_SETTINGS,
+                Clock.systemUTC(),
+                mock(Client.class),
+                mock(SecurityIndexManager.class),
+                clusterService,
+                mock(CacheInvalidatorRegistry.class),
+                mock(ThreadPool.class)
+            )
+        );
+        final NativePrivilegeStore nativePrivStore = mock(NativePrivilegeStore.class);
+        doAnswer(invocationOnMock -> {
+            @SuppressWarnings("unchecked")
+            ActionListener<Collection<ApplicationPrivilegeDescriptor>> listener = (ActionListener<
+                Collection<ApplicationPrivilegeDescriptor>>) invocationOnMock.getArguments()[2];
+            listener.onResponse(Collections.emptyList());
+            return Void.TYPE;
+        }).when(nativePrivStore).getPrivileges(anyCollection(), anyCollection(), anyActionListener());
+
+        final AtomicReference<Collection<RoleDescriptor>> effectiveRoleDescriptors = new AtomicReference<Collection<RoleDescriptor>>();
+        final CompositeRolesStore compositeRolesStore = buildCompositeRolesStore(
+            SECURITY_ENABLED_SETTINGS,
+            fileRolesStore,
+            nativeRolesStore,
+            reservedRolesStore,
+            nativePrivStore,
+            null,
+            apiKeyService,
+            null,
+            null,
+            effectiveRoleDescriptors::set
+        );
+        AuditUtil.getOrGenerateRequestId(threadContext);
+        final TransportVersion version = TransportVersion.CURRENT;
+        final String apiKeyRoleName = "user_role_" + randomAlphaOfLength(4);
+        final Authentication apiKeyAuthentication = createApiKeyAuthentication(
+            apiKeyService,
+            randomValueOtherThanMany(
+                authc -> authc.getAuthenticationType() == AuthenticationType.API_KEY,
+                () -> AuthenticationTestHelper.builder().build()
+            ),
+            Collections.singleton(
+                new RoleDescriptor(
+                    apiKeyRoleName,
+                    null,
+                    new IndicesPrivileges[] { IndicesPrivileges.builder().indices("index*").privileges("all").build() },
+                    null
+                )
+            ),
+            null,
+            version
+        );
+        final boolean emptyRemoteRole = randomBoolean();
+        final Authentication authentication = apiKeyAuthentication.toRemoteAccess(
+            AuthenticationTestHelper.randomRemoteAccessAuthentication(
+                emptyRemoteRole
+                    ? RoleDescriptorsIntersection.EMPTY
+                    : new RoleDescriptorsIntersection(
+                        new RoleDescriptor(
+                            RBACEngine.REMOTE_USER_ROLE_NAME,
+                            null,
+                            new RoleDescriptor.IndicesPrivileges[] {
+                                RoleDescriptor.IndicesPrivileges.builder().indices("index1").privileges("read").build() },
+                            null,
+                            null,
+                            null,
+                            null,
+                            null,
+                            null
+                        )
+                    )
+            )
+        );
+
+        final PlainActionFuture<Role> roleFuture = new PlainActionFuture<>();
+        compositeRolesStore.getRole(authentication.getEffectiveSubject(), roleFuture);
+        final Role role = roleFuture.actionGet();
+        assertThat(effectiveRoleDescriptors.get(), is(nullValue()));
+
+        verify(apiKeyService, times(1)).parseRoleDescriptorsBytes(anyString(), any(BytesReference.class), any());
+        assertThat(role.names().length, is(1));
+        assertThat(role.names()[0], equalTo(apiKeyRoleName));
+
+        // Smoke-test for authorization
+        final Metadata indexMetadata = Metadata.builder()
+            .put(
+                IndexMetadata.builder("index1")
+                    .settings(
+                        Settings.builder()
+                            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+                            .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
+                    )
+            )
+            .put(
+                IndexMetadata.builder("index2")
+                    .settings(
+                        Settings.builder()
+                            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+                            .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
+                    )
+            )
+            .build();
+        final var emptyCache = new FieldPermissionsCache(Settings.EMPTY);
+        assertThat(
+            role.authorize(SearchAction.NAME, Sets.newHashSet("index1"), indexMetadata.getIndicesLookup(), emptyCache).isGranted(),
+            is(false == emptyRemoteRole)
+        );
+        assertThat(
+            role.authorize(CreateIndexAction.NAME, Sets.newHashSet("index1"), indexMetadata.getIndicesLookup(), emptyCache).isGranted(),
+            is(false)
+        );
+        assertThat(
+            role.authorize(SearchAction.NAME, Sets.newHashSet("index2"), indexMetadata.getIndicesLookup(), emptyCache).isGranted(),
+            is(false)
+        );
+    }
+
     public void testGetRolesForRunAs() {
         final ApiKeyService apiKeyService = mock(ApiKeyService.class);
         final ServiceAccountService serviceAccountService = mock(ServiceAccountService.class);
@@ -2123,7 +2354,7 @@ public void testCacheEntryIsReusedForIdenticalApiKeyRoles() {
 
     public void testXPackSecurityUserCanAccessAnyIndex() {
         for (String action : Arrays.asList(GetAction.NAME, DeleteAction.NAME, SearchAction.NAME, IndexAction.NAME)) {
-            Predicate<IndexAbstraction> predicate = getXPackSecurityRole().indices().allowedIndicesMatcher(action);
+            IsResourceAuthorizedPredicate predicate = getXPackSecurityRole().indices().allowedIndicesMatcher(action);
 
             IndexAbstraction index = mockIndexAbstraction(randomAlphaOfLengthBetween(3, 12));
             assertThat(predicate.test(index), Matchers.is(true));
@@ -2138,7 +2369,7 @@ public void testXPackSecurityUserCanAccessAnyIndex() {
 
     public void testSecurityProfileUserHasAccessForOnlyProfileIndex() {
         for (String action : Arrays.asList(GetAction.NAME, DeleteAction.NAME, SearchAction.NAME, IndexAction.NAME)) {
-            Predicate<IndexAbstraction> predicate = getSecurityProfileRole().indices().allowedIndicesMatcher(action);
+            IsResourceAuthorizedPredicate predicate = getSecurityProfileRole().indices().allowedIndicesMatcher(action);
 
             List.of(
                 ".security-profile",
@@ -2162,7 +2393,7 @@ public void testSecurityProfileUserHasAccessForOnlyProfileIndex() {
 
     public void testXPackUserCanAccessNonRestrictedIndices() {
         for (String action : Arrays.asList(GetAction.NAME, DeleteAction.NAME, SearchAction.NAME, IndexAction.NAME)) {
-            Predicate<IndexAbstraction> predicate = getXPackUserRole().indices().allowedIndicesMatcher(action);
+            IsResourceAuthorizedPredicate predicate = getXPackUserRole().indices().allowedIndicesMatcher(action);
             IndexAbstraction index = mockIndexAbstraction(randomAlphaOfLengthBetween(3, 12));
             if (false == TestRestrictedIndices.RESTRICTED_INDICES.isRestricted(index.getName())) {
                 assertThat(predicate.test(index), Matchers.is(true));
@@ -2176,7 +2407,7 @@ public void testXPackUserCanAccessNonRestrictedIndices() {
 
     public void testXPackUserCannotAccessSecurityOrAsyncSearch() {
         for (String action : Arrays.asList(GetAction.NAME, DeleteAction.NAME, SearchAction.NAME, IndexAction.NAME)) {
-            Predicate<IndexAbstraction> predicate = getXPackUserRole().indices().allowedIndicesMatcher(action);
+            IsResourceAuthorizedPredicate predicate = getXPackUserRole().indices().allowedIndicesMatcher(action);
             for (String index : TestRestrictedIndices.SAMPLE_RESTRICTED_NAMES) {
                 assertThat(predicate.test(mockIndexAbstraction(index)), Matchers.is(false));
             }
@@ -2189,19 +2420,19 @@ public void testXPackUserCannotAccessSecurityOrAsyncSearch() {
 
     public void testXPackUserCanReadAuditTrail() {
         final String action = randomFrom(GetAction.NAME, SearchAction.NAME);
-        final Predicate<IndexAbstraction> predicate = getXPackUserRole().indices().allowedIndicesMatcher(action);
+        final IsResourceAuthorizedPredicate predicate = getXPackUserRole().indices().allowedIndicesMatcher(action);
         assertThat(predicate.test(mockIndexAbstraction(getAuditLogName())), Matchers.is(true));
     }
 
     public void testXPackUserCannotWriteToAuditTrail() {
         final String action = randomFrom(IndexAction.NAME, UpdateAction.NAME);
-        final Predicate<IndexAbstraction> predicate = getXPackUserRole().indices().allowedIndicesMatcher(action);
+        final IsResourceAuthorizedPredicate predicate = getXPackUserRole().indices().allowedIndicesMatcher(action);
         assertThat(predicate.test(mockIndexAbstraction(getAuditLogName())), Matchers.is(false));
     }
 
     public void testAsyncSearchUserCannotAccessNonRestrictedIndices() {
         for (String action : Arrays.asList(GetAction.NAME, DeleteAction.NAME, SearchAction.NAME, IndexAction.NAME)) {
-            Predicate<IndexAbstraction> predicate = getAsyncSearchUserRole().indices().allowedIndicesMatcher(action);
+            IsResourceAuthorizedPredicate predicate = getAsyncSearchUserRole().indices().allowedIndicesMatcher(action);
             IndexAbstraction index = mockIndexAbstraction(randomAlphaOfLengthBetween(3, 12));
             if (false == TestRestrictedIndices.RESTRICTED_INDICES.isRestricted(index.getName())) {
                 assertThat(predicate.test(index), Matchers.is(false));
@@ -2215,7 +2446,7 @@ public void testAsyncSearchUserCannotAccessNonRestrictedIndices() {
 
     public void testAsyncSearchUserCanAccessOnlyAsyncSearchRestrictedIndices() {
         for (String action : Arrays.asList(GetAction.NAME, DeleteAction.NAME, SearchAction.NAME, IndexAction.NAME)) {
-            final Predicate<IndexAbstraction> predicate = getAsyncSearchUserRole().indices().allowedIndicesMatcher(action);
+            final IsResourceAuthorizedPredicate predicate = getAsyncSearchUserRole().indices().allowedIndicesMatcher(action);
             for (String index : TestRestrictedIndices.SAMPLE_RESTRICTED_NAMES) {
                 assertThat(predicate.test(mockIndexAbstraction(index)), Matchers.is(false));
             }
@@ -2238,13 +2469,13 @@ public void testAsyncSearchUserHasNoClusterPrivileges() {
     // async search can't read/write audit trail
     public void testAsyncSearchUserCannotReadAuditTrail() {
         final String action = randomFrom(GetAction.NAME, SearchAction.NAME);
-        final Predicate<IndexAbstraction> predicate = getAsyncSearchUserRole().indices().allowedIndicesMatcher(action);
+        final IsResourceAuthorizedPredicate predicate = getAsyncSearchUserRole().indices().allowedIndicesMatcher(action);
         assertThat(predicate.test(mockIndexAbstraction(getAuditLogName())), Matchers.is(false));
     }
 
     public void testAsyncSearchUserCannotWriteToAuditTrail() {
         final String action = randomFrom(IndexAction.NAME, UpdateAction.NAME);
-        final Predicate<IndexAbstraction> predicate = getAsyncSearchUserRole().indices().allowedIndicesMatcher(action);
+        final IsResourceAuthorizedPredicate predicate = getAsyncSearchUserRole().indices().allowedIndicesMatcher(action);
         assertThat(predicate.test(mockIndexAbstraction(getAuditLogName())), Matchers.is(false));
     }
 
@@ -2616,6 +2847,22 @@ private static Matcher<IndicesPermission.Group> indexGroup(
         final IndexPrivilege privilege,
         final boolean allowRestrictedIndices,
         final String... indices
+    ) {
+        return indexGroup(
+            privilege,
+            allowRestrictedIndices,
+            null,
+            new FieldPermissionsDefinition.FieldGrantExcludeGroup(null, null),
+            indices
+        );
+    }
+
+    private static Matcher<IndicesPermission.Group> indexGroup(
+        final IndexPrivilege privilege,
+        final boolean allowRestrictedIndices,
+        @Nullable final String query,
+        final FieldPermissionsDefinition.FieldGrantExcludeGroup flsGroup,
+        final String... indices
     ) {
         return new BaseMatcher<>() {
             @Override
@@ -2624,8 +2871,10 @@ public boolean matches(Object o) {
                     return false;
                 }
                 final IndicesPermission.Group group = (IndicesPermission.Group) o;
-                return equalTo(privilege).matches(group.privilege())
+                return equalTo(query == null ? null : Set.of(new BytesArray(query))).matches(group.getQuery())
+                    && equalTo(privilege).matches(group.privilege())
                     && equalTo(allowRestrictedIndices).matches(group.allowRestrictedIndices())
+                    && equalTo(new FieldPermissions(new FieldPermissionsDefinition(Set.of(flsGroup)))).matches(group.getFieldPermissions())
                     && arrayContaining(indices).matches(group.indices());
             }
 
@@ -2639,6 +2888,10 @@ public void describeTo(Description description) {
                         + allowRestrictedIndices
                         + ", indices="
                         + Strings.arrayToCommaDelimitedString(indices)
+                        + ", query="
+                        + query
+                        + ", fieldGrantExcludeGroup="
+                        + flsGroup
                         + '}'
                 );
             }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java
index 7463c7696e04..a099f0542d58 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java
@@ -326,7 +326,7 @@ void innerPutRole(final PutRoleRequest request, final RoleDescriptor role, final
         PlainActionFuture<Boolean> future = new PlainActionFuture<>();
         rolesStore.putRole(putRoleRequest, remoteIndicesRole, future);
         IllegalStateException e = expectThrows(IllegalStateException.class, future::actionGet);
-        assertThat(e.getMessage(), containsString("all nodes must have version [8.6.0] or higher to support remote indices privileges"));
+        assertThat(e.getMessage(), containsString("all nodes must have version [8060099] or higher to support remote indices privileges"));
     }
 
     private ClusterService mockClusterServiceWithMinNodeVersion(Version version) {
@@ -355,7 +355,8 @@ private ClusterState getClusterStateWithSecurityIndex() {
             new ShardId(index, 0),
             true,
             RecoverySource.ExistingStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(Reason.INDEX_CREATED, "")
+            new UnassignedInfo(Reason.INDEX_CREATED, ""),
+            ShardRouting.Role.DEFAULT
         );
         RoutingTable routingTable = RoutingTable.builder()
             .add(
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistryTests.java
index 217691755202..5394861b8460 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistryTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistryTests.java
@@ -12,11 +12,11 @@
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
 import java.util.HashSet;
-import java.util.Locale;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -90,7 +90,7 @@ public void testOperatorOnlySettings() {
                 violation = operatorOnlyRegistry.check(ClusterUpdateSettingsAction.NAME, request);
                 assertThat(
                     violation.message(),
-                    containsString(String.format(Locale.ROOT, "settings [%s,%s]", transientSetting.getKey(), persistentSetting.getKey()))
+                    containsString(Strings.format("settings [%s,%s]", transientSetting.getKey(), persistentSetting.getKey()))
                 );
             }
             case 1 -> {
@@ -98,14 +98,14 @@ public void testOperatorOnlySettings() {
                 persistentSetting = convertToConcreteSettingIfNecessary(randomFrom(DYNAMIC_SETTINGS));
                 request = prepareClusterUpdateSettingsRequest(transientSetting, persistentSetting);
                 violation = operatorOnlyRegistry.check(ClusterUpdateSettingsAction.NAME, request);
-                assertThat(violation.message(), containsString(formatted("setting [%s]", transientSetting.getKey())));
+                assertThat(violation.message(), containsString(Strings.format("setting [%s]", transientSetting.getKey())));
             }
             case 2 -> {
                 transientSetting = convertToConcreteSettingIfNecessary(randomFrom(DYNAMIC_SETTINGS));
                 persistentSetting = convertToConcreteSettingIfNecessary(randomFrom(IP_FILTER_SETTINGS));
                 request = prepareClusterUpdateSettingsRequest(transientSetting, persistentSetting);
                 violation = operatorOnlyRegistry.check(ClusterUpdateSettingsAction.NAME, request);
-                assertThat(violation.message(), containsString(formatted("setting [%s]", persistentSetting.getKey())));
+                assertThat(violation.message(), containsString(Strings.format("setting [%s]", persistentSetting.getKey())));
             }
             case 3 -> {
                 transientSetting = convertToConcreteSettingIfNecessary(randomFrom(DYNAMIC_SETTINGS));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestGetTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestGetTokenActionTests.java
index 17ab028eb535..2ac33a780313 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestGetTokenActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestGetTokenActionTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.AbstractRestChannel;
 import org.elasticsearch.rest.RestChannel;
@@ -142,7 +143,7 @@ public void sendResponse(RestResponse restResponse) {
     }
 
     public void testParser() throws Exception {
-        final String request = formatted("""
+        final String request = Strings.format("""
             {
               "grant_type": "password",
               "username": "user1",
@@ -160,7 +161,7 @@ public void testParser() throws Exception {
 
     public void testParserRefreshRequest() throws Exception {
         final String token = randomAlphaOfLengthBetween(4, 32);
-        final String request = formatted("""
+        final String request = Strings.format("""
             {
               "grant_type": "refresh_token",
               "refresh_token": "%s",
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/profile/RestSuggestProfilesActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/profile/RestSuggestProfilesActionTests.java
index 81b2579608ab..26412106229a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/profile/RestSuggestProfilesActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/profile/RestSuggestProfilesActionTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.license.MockLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestStatus;
@@ -98,7 +99,7 @@ public void testWillNotAllowDataInBothQueryParameterAndRequestBody() {
         )
             .withPath("/_security/profile/_suggest")
             .withParams(new HashMap<>(Map.of("data", randomAlphaOfLengthBetween(3, 8))))
-            .withContent(new BytesArray(formatted("{\"data\": \"%s\"}", randomAlphaOfLengthBetween(3, 8))), XContentType.JSON)
+            .withContent(new BytesArray(Strings.format("{\"data\": \"%s\"}", randomAlphaOfLengthBetween(3, 8))), XContentType.JSON)
             .build();
 
         final IllegalArgumentException e = expectThrows(
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUserPrivilegesActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUserPrivilegesActionTests.java
index d0c33a516599..af48949f2d31 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUserPrivilegesActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUserPrivilegesActionTests.java
@@ -33,7 +33,6 @@
 import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.List;
-import java.util.Locale;
 import java.util.Set;
 
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
@@ -186,7 +185,7 @@ public void testBuildResponse() throws Exception {
                   "clusters": [ "*", "remote-2" ]
                 }
               ]""";
-        assertThat(json, equalTo(XContentHelper.stripWhitespace(String.format(Locale.ROOT, """
+        assertThat(json, equalTo(XContentHelper.stripWhitespace(Strings.format("""
             {
               "cluster": [ "monitor", "manage_ml", "manage_watcher" ],
               "global": [
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilderTests.java
index 73393beaae50..d7f994d7499f 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilderTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilderTests.java
@@ -73,10 +73,15 @@ public void testQueryForDomainAuthentication() {
             apiKeysQuery.filter().get(1),
             is(QueryBuilders.termQuery("creator.principal", authentication.getEffectiveSubject().getUser().principal()))
         );
-        if (authentication.getDomain().realms().size() == 1) {
+        if (authentication.getEffectiveSubject().getRealm().getDomain().realms().size() == 1) {
             assertThat(
                 apiKeysQuery.filter().get(2),
-                is(QueryBuilders.termQuery("creator.realm", authentication.getDomain().realms().stream().findFirst().get().getName()))
+                is(
+                    QueryBuilders.termQuery(
+                        "creator.realm",
+                        authentication.getEffectiveSubject().getRealm().getDomain().realms().stream().findFirst().get().getName()
+                    )
+                )
             );
         } else {
             assertThat(apiKeysQuery.filter().get(2), instanceOf(BoolQueryBuilder.class));
@@ -84,7 +89,7 @@ public void testQueryForDomainAuthentication() {
             assertThat(((BoolQueryBuilder) apiKeysQuery.filter().get(2)).mustNot().size(), is(0));
             assertThat(((BoolQueryBuilder) apiKeysQuery.filter().get(2)).filter().size(), is(0));
             assertThat(((BoolQueryBuilder) apiKeysQuery.filter().get(2)).minimumShouldMatch(), is("1"));
-            for (RealmConfig.RealmIdentifier realmIdentifier : authentication.getDomain().realms()) {
+            for (RealmConfig.RealmIdentifier realmIdentifier : authentication.getEffectiveSubject().getRealm().getDomain().realms()) {
                 assertThat(
                     ((BoolQueryBuilder) apiKeysQuery.filter().get(2)).should(),
                     hasItem(QueryBuilders.termQuery("creator.realm", realmIdentifier.getName()))
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java
index b103d150b992..a45820b66de5 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java
@@ -139,7 +139,8 @@ public void testIndexWithoutPrimaryShards() {
             new ShardId(index, 0),
             true,
             RecoverySource.ExistingStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""),
+            ShardRouting.Role.DEFAULT
         );
         String nodeId = ESTestCase.randomAlphaOfLength(8);
         clusterStateBuilder.routingTable(
@@ -216,7 +217,8 @@ public void testIndexHealthChangeListeners() {
                                         new ShardId(prevIndex, 0),
                                         true,
                                         RecoverySource.ExistingStoreRecoverySource.INSTANCE,
-                                        new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")
+                                        new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""),
+                                        ShardRouting.Role.DEFAULT
                                     )
                                         .initialize(UUIDs.randomBase64UUID(random()), null, 0L)
                                         .moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, ""))
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java
index f53e18bdb91c..3f66a2880d96 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java
@@ -72,7 +72,8 @@ public static RoutingTable buildIndexRoutingTable(Index index) {
             new ShardId(index, 0),
             true,
             ExistingStoreRecoverySource.INSTANCE,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""),
+            ShardRouting.Role.DEFAULT
         );
         String nodeId = ESTestCase.randomAlphaOfLength(8);
         return RoutingTable.builder()
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java
index a9bf0fb61f9c..3cd048a3f1f9 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java
@@ -6,20 +6,31 @@
  */
 package org.elasticsearch.xpack.security.transport;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.support.DestructiveOperations;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.ssl.SslClientAuthenticationMode;
+import org.elasticsearch.common.ssl.SslConfiguration;
+import org.elasticsearch.common.ssl.SslKeyConfig;
+import org.elasticsearch.common.ssl.SslTrustConfig;
+import org.elasticsearch.common.ssl.SslVerificationMode;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.SendRequestTransportException;
+import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.Transport.Connection;
 import org.elasticsearch.transport.TransportChannel;
@@ -31,11 +42,15 @@
 import org.elasticsearch.transport.TransportResponse.Empty;
 import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction;
 import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper;
+import org.elasticsearch.xpack.core.security.authc.RemoteAccessAuthentication;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection;
+import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.core.security.user.AsyncSearchUser;
 import org.elasticsearch.xpack.core.security.user.SecurityProfileUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
@@ -46,27 +61,41 @@
 import org.elasticsearch.xpack.security.authc.AuthenticationService;
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
 import org.junit.After;
+import org.mockito.ArgumentCaptor;
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 
+import static org.elasticsearch.test.ActionListenerUtils.anyActionListener;
 import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_PROFILE_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.TRANSFORM_ORIGIN;
+import static org.elasticsearch.xpack.core.security.authc.RemoteAccessAuthentication.REMOTE_ACCESS_AUTHENTICATION_HEADER_KEY;
+import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTests.randomUniquelyNamedRoleDescriptors;
+import static org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor.REMOTE_ACCESS_ACTION_ALLOWLIST;
+import static org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor.REMOTE_ACCESS_CLUSTER_CREDENTIAL_HEADER_KEY;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
@@ -134,12 +163,12 @@ public <T extends TransportResponse> void sendRequest(
             }
         });
         Transport.Connection connection = mock(Transport.Connection.class);
-        when(connection.getVersion()).thenReturn(Version.CURRENT);
+        when(connection.getTransportVersion()).thenReturn(TransportVersion.CURRENT);
         sender.sendRequest(connection, "indices:foo", null, null, null);
         assertTrue(calledWrappedSender.get());
         assertEquals(user, sendingUser.get());
         assertEquals(user, securityContext.getUser());
-        verify(securityContext, never()).executeAsInternalUser(any(User.class), any(Version.class), anyConsumer());
+        verify(securityContext, never()).executeAsInternalUser(any(User.class), any(TransportVersion.class), anyConsumer());
     }
 
     public void testSendAsyncSwitchToSystem() throws Exception {
@@ -184,13 +213,13 @@ public <T extends TransportResponse> void sendRequest(
             }
         });
         Connection connection = mock(Connection.class);
-        when(connection.getVersion()).thenReturn(Version.CURRENT);
+        when(connection.getTransportVersion()).thenReturn(TransportVersion.CURRENT);
         sender.sendRequest(connection, "internal:foo", null, null, null);
         assertTrue(calledWrappedSender.get());
         assertNotEquals(user, sendingUser.get());
         assertEquals(SystemUser.INSTANCE, sendingUser.get());
         assertEquals(user, securityContext.getUser());
-        verify(securityContext).executeAsInternalUser(any(User.class), eq(Version.CURRENT), anyConsumer());
+        verify(securityContext).executeAsInternalUser(any(User.class), eq(TransportVersion.CURRENT), anyConsumer());
     }
 
     public void testSendWithoutUser() throws Exception {
@@ -226,14 +255,14 @@ public <T extends TransportResponse> void sendRequest(
             }
         });
         Transport.Connection connection = mock(Transport.Connection.class);
-        when(connection.getVersion()).thenReturn(Version.CURRENT);
+        when(connection.getTransportVersion()).thenReturn(TransportVersion.CURRENT);
         IllegalStateException e = expectThrows(
             IllegalStateException.class,
             () -> sender.sendRequest(connection, "indices:foo", null, null, null)
         );
         assertEquals("there should always be a user when sending a message for action [indices:foo]", e.getMessage());
         assertNull(securityContext.getUser());
-        verify(securityContext, never()).executeAsInternalUser(any(User.class), any(Version.class), anyConsumer());
+        verify(securityContext, never()).executeAsInternalUser(any(User.class), any(TransportVersion.class), anyConsumer());
     }
 
     public void testSendToNewerVersionSetsCorrectVersion() throws Exception {
@@ -290,17 +319,17 @@ public <T extends TransportResponse> void sendRequest(
             }
         };
         AsyncSender sender = interceptor.interceptSender(intercepted);
-        final Version connectionVersion = Version.fromId(Version.CURRENT.id + randomIntBetween(100, 100000));
-        assertEquals(Version.CURRENT, Version.min(connectionVersion, Version.CURRENT));
+        final TransportVersion connectionVersion = TransportVersion.fromId(Version.CURRENT.id + randomIntBetween(100, 100000));
+        assertEquals(TransportVersion.CURRENT, TransportVersion.min(connectionVersion, TransportVersion.CURRENT));
 
         Transport.Connection connection = mock(Transport.Connection.class);
-        when(connection.getVersion()).thenReturn(connectionVersion);
+        when(connection.getTransportVersion()).thenReturn(connectionVersion);
         sender.sendRequest(connection, "indices:foo[s]", null, null, null);
         assertTrue(calledWrappedSender.get());
         assertEquals(authentication.getEffectiveSubject().getUser(), sendingUser.get());
         assertEquals(authentication.getEffectiveSubject().getUser(), securityContext.getUser());
-        assertEquals(Version.CURRENT, authRef.get().getEffectiveSubject().getVersion());
-        assertEquals(Version.CURRENT, authentication.getEffectiveSubject().getVersion());
+        assertEquals(TransportVersion.CURRENT, authRef.get().getEffectiveSubject().getTransportVersion());
+        assertEquals(TransportVersion.CURRENT, authentication.getEffectiveSubject().getTransportVersion());
     }
 
     public void testSendToOlderVersionSetsCorrectVersion() throws Exception {
@@ -357,17 +386,17 @@ public <T extends TransportResponse> void sendRequest(
             }
         };
         AsyncSender sender = interceptor.interceptSender(intercepted);
-        final Version connectionVersion = Version.fromId(Version.CURRENT.id - randomIntBetween(100, 100000));
-        assertEquals(connectionVersion, Version.min(connectionVersion, Version.CURRENT));
+        final TransportVersion connectionVersion = TransportVersion.fromId(Version.CURRENT.id - randomIntBetween(100, 100000));
+        assertEquals(connectionVersion, TransportVersion.min(connectionVersion, TransportVersion.CURRENT));
 
         Transport.Connection connection = mock(Transport.Connection.class);
-        when(connection.getVersion()).thenReturn(connectionVersion);
+        when(connection.getTransportVersion()).thenReturn(connectionVersion);
         sender.sendRequest(connection, "indices:foo[s]", null, null, null);
         assertTrue(calledWrappedSender.get());
         assertEquals(authentication.getEffectiveSubject().getUser(), sendingUser.get());
         assertEquals(authentication.getEffectiveSubject().getUser(), securityContext.getUser());
-        assertEquals(connectionVersion, authRef.get().getEffectiveSubject().getVersion());
-        assertEquals(Version.CURRENT, authentication.getEffectiveSubject().getVersion());
+        assertEquals(connectionVersion, authRef.get().getEffectiveSubject().getTransportVersion());
+        assertEquals(TransportVersion.CURRENT, authentication.getEffectiveSubject().getTransportVersion());
     }
 
     public void testSetUserBasedOnActionOrigin() {
@@ -419,15 +448,15 @@ public <T extends TransportResponse> void sendRequest(
         final AsyncSender sender = interceptor.interceptSender(intercepted);
 
         Transport.Connection connection = mock(Transport.Connection.class);
-        final Version connectionVersion = VersionUtils.randomCompatibleVersion(random(), Version.CURRENT);
-        when(connection.getVersion()).thenReturn(connectionVersion);
+        final TransportVersion connectionVersion = TransportVersionUtils.randomCompatibleVersion(random(), TransportVersion.CURRENT);
+        when(connection.getTransportVersion()).thenReturn(connectionVersion);
 
         sender.sendRequest(connection, "indices:foo[s]", null, null, null);
         assertThat(calledWrappedSender.get(), is(true));
         final Authentication authentication = authenticationRef.get();
         assertThat(authentication, notNullValue());
         assertThat(authentication.getEffectiveSubject().getUser(), equalTo(originToUserMap.get(origin)));
-        assertThat(authentication.getEffectiveSubject().getVersion(), equalTo(connectionVersion));
+        assertThat(authentication.getEffectiveSubject().getTransportVersion(), equalTo(connectionVersion));
     }
 
     public void testContextRestoreResponseHandler() throws Exception {
@@ -534,6 +563,387 @@ public boolean decRef() {
         assertTrue(exceptionSent.get());
     }
 
+    public void testSendWithRemoteAccessHeaders() throws Exception {
+        assumeTrue("untrusted remote cluster feature flag must be enabled", TcpTransport.isUntrustedRemoteClusterEnabled());
+
+        final boolean internalUser = randomBoolean();
+        final Authentication authentication = internalUser
+            ? AuthenticationTestHelper.builder().internal(SystemUser.INSTANCE).build()
+            : AuthenticationTestHelper.builder().user(new User(randomAlphaOfLengthBetween(3, 10), randomRoles())).realm().build();
+        authentication.writeToContext(threadContext);
+        final RemoteClusterAuthorizationResolver remoteClusterAuthorizationResolver = mock(RemoteClusterAuthorizationResolver.class);
+        final String remoteClusterCredential = randomAlphaOfLengthBetween(10, 42);
+        when(remoteClusterAuthorizationResolver.resolveAuthorization(any())).thenReturn(remoteClusterCredential);
+        final String remoteClusterAlias = randomAlphaOfLengthBetween(5, 10);
+        final AuthorizationService authzService = mock(AuthorizationService.class);
+        // We capture the listener so that we can complete the full flow, by calling onResponse further down
+        @SuppressWarnings("unchecked")
+        final ArgumentCaptor<ActionListener<RoleDescriptorsIntersection>> listenerCaptor = ArgumentCaptor.forClass(ActionListener.class);
+        doAnswer(i -> null).when(authzService).retrieveRemoteAccessRoleDescriptorsIntersection(any(), any(), listenerCaptor.capture());
+
+        final SecurityServerTransportInterceptor interceptor = new SecurityServerTransportInterceptor(
+            settings,
+            threadPool,
+            mock(AuthenticationService.class),
+            authzService,
+            mock(SSLService.class),
+            securityContext,
+            new DestructiveOperations(
+                Settings.EMPTY,
+                new ClusterSettings(Settings.EMPTY, Collections.singleton(DestructiveOperations.REQUIRES_NAME_SETTING))
+            ),
+            remoteClusterAuthorizationResolver,
+            ignored -> Optional.of(remoteClusterAlias)
+        );
+
+        final AtomicBoolean calledWrappedSender = new AtomicBoolean(false);
+        final AtomicReference<String> sentCredential = new AtomicReference<>();
+        final AtomicReference<RemoteAccessAuthentication> sentRemoteAccessAuthentication = new AtomicReference<>();
+        final AsyncSender sender = interceptor.interceptSender(new AsyncSender() {
+            @Override
+            public <T extends TransportResponse> void sendRequest(
+                Transport.Connection connection,
+                String action,
+                TransportRequest request,
+                TransportRequestOptions options,
+                TransportResponseHandler<T> handler
+            ) {
+                if (calledWrappedSender.compareAndSet(false, true) == false) {
+                    fail("sender called more than once");
+                }
+                assertThat(securityContext.getAuthentication(), nullValue());
+                sentCredential.set(securityContext.getThreadContext().getHeader(REMOTE_ACCESS_CLUSTER_CREDENTIAL_HEADER_KEY));
+                try {
+                    sentRemoteAccessAuthentication.set(RemoteAccessAuthentication.readFromContext(securityContext.getThreadContext()));
+                } catch (IOException e) {
+                    fail("no exceptions expected but got " + e);
+                }
+                handler.handleResponse(null);
+            }
+        });
+        final Transport.Connection connection = mock(Transport.Connection.class);
+        when(connection.getTransportVersion()).thenReturn(TransportVersion.CURRENT);
+        final Tuple<String, TransportRequest> actionAndReq = randomAllowlistedActionAndRequest();
+        sender.sendRequest(connection, actionAndReq.v1(), actionAndReq.v2(), null, new TransportResponseHandler<>() {
+            @Override
+            public void handleResponse(TransportResponse response) {
+                // Headers should get restored before handle response is called
+                assertThat(securityContext.getAuthentication(), equalTo(authentication));
+                assertThat(securityContext.getThreadContext().getHeader(REMOTE_ACCESS_AUTHENTICATION_HEADER_KEY), nullValue());
+                assertThat(securityContext.getThreadContext().getHeader(REMOTE_ACCESS_CLUSTER_CREDENTIAL_HEADER_KEY), nullValue());
+            }
+
+            @Override
+            public void handleException(TransportException exp) {
+                fail("no exceptions expected but got " + exp);
+            }
+
+            @Override
+            public TransportResponse read(StreamInput in) {
+                return null;
+            }
+        });
+        final RoleDescriptorsIntersection expectedRoleDescriptorsIntersection;
+        if (internalUser) {
+            expectedRoleDescriptorsIntersection = RoleDescriptorsIntersection.EMPTY;
+        } else {
+            expectedRoleDescriptorsIntersection = new RoleDescriptorsIntersection(
+                randomList(0, 3, () -> Set.copyOf(randomUniquelyNamedRoleDescriptors(0, 1)))
+            );
+            // Call listener to complete flow
+            listenerCaptor.getValue().onResponse(expectedRoleDescriptorsIntersection);
+        }
+        assertTrue(calledWrappedSender.get());
+        assertThat(sentCredential.get(), equalTo("ApiKey " + remoteClusterCredential));
+        assertThat(
+            sentRemoteAccessAuthentication.get(),
+            equalTo(new RemoteAccessAuthentication(authentication, expectedRoleDescriptorsIntersection))
+        );
+        verify(securityContext, never()).executeAsInternalUser(any(), any(), anyConsumer());
+        verify(remoteClusterAuthorizationResolver, times(1)).resolveAuthorization(eq(remoteClusterAlias));
+        verify(authzService, times(internalUser ? 0 : 1)).retrieveRemoteAccessRoleDescriptorsIntersection(
+            eq(remoteClusterAlias),
+            eq(authentication.getEffectiveSubject()),
+            anyActionListener()
+        );
+        assertThat(securityContext.getThreadContext().getHeader(REMOTE_ACCESS_AUTHENTICATION_HEADER_KEY), nullValue());
+        assertThat(securityContext.getThreadContext().getHeader(REMOTE_ACCESS_CLUSTER_CREDENTIAL_HEADER_KEY), nullValue());
+    }
+
+    public void testSendWithUserIfRemoteAccessHeadersConditionNotMet() throws Exception {
+        assumeTrue("untrusted remote cluster feature flag must be enabled", TcpTransport.isUntrustedRemoteClusterEnabled());
+
+        boolean noCredential = randomBoolean();
+        final boolean notRemoteConnection = randomBoolean();
+        final boolean nonAllowlistedRequest = randomBoolean();
+        final boolean unsupportedAuthentication = randomBoolean();
+        // Ensure at least one condition fails
+        if (false == (notRemoteConnection || noCredential || nonAllowlistedRequest || unsupportedAuthentication)) {
+            noCredential = true;
+        }
+        final RemoteClusterAuthorizationResolver remoteClusterAuthorizationResolver = mock(RemoteClusterAuthorizationResolver.class);
+        when(remoteClusterAuthorizationResolver.resolveAuthorization(any())).thenReturn(
+            noCredential ? null : randomAlphaOfLengthBetween(10, 42)
+        );
+        final AuthenticationTestHelper.AuthenticationTestBuilder builder = AuthenticationTestHelper.builder();
+        final Authentication authentication;
+        if (unsupportedAuthentication) {
+            authentication = randomFrom(builder.apiKey().build(), builder.serviceAccount().build());
+        } else {
+            authentication = builder.user(new User(randomAlphaOfLengthBetween(3, 10), randomRoles())).realm().build();
+        }
+        authentication.writeToContext(threadContext);
+        final Tuple<String, TransportRequest> actionAndReq = nonAllowlistedRequest
+            ? new Tuple<>(FollowInfoAction.NAME, mock(TransportRequest.class))
+            : randomAllowlistedActionAndRequest();
+
+        final String remoteClusterAlias = randomAlphaOfLengthBetween(5, 10);
+        final AuthorizationService authzService = mock(AuthorizationService.class);
+        final SecurityServerTransportInterceptor interceptor = new SecurityServerTransportInterceptor(
+            settings,
+            threadPool,
+            mock(AuthenticationService.class),
+            authzService,
+            mock(SSLService.class),
+            securityContext,
+            new DestructiveOperations(
+                Settings.EMPTY,
+                new ClusterSettings(Settings.EMPTY, Collections.singleton(DestructiveOperations.REQUIRES_NAME_SETTING))
+            ),
+            remoteClusterAuthorizationResolver,
+            ignored -> notRemoteConnection ? Optional.empty() : Optional.of(remoteClusterAlias)
+        );
+
+        final AtomicBoolean calledWrappedSender = new AtomicBoolean(false);
+        final AtomicReference<Authentication> sentAuthentication = new AtomicReference<>();
+        final AsyncSender sender = interceptor.interceptSender(new AsyncSender() {
+            @Override
+            public <T extends TransportResponse> void sendRequest(
+                Transport.Connection connection,
+                String action,
+                TransportRequest request,
+                TransportRequestOptions options,
+                TransportResponseHandler<T> handler
+            ) {
+                if (calledWrappedSender.compareAndSet(false, true) == false) {
+                    fail("sender called more than once");
+                }
+                sentAuthentication.set(securityContext.getAuthentication());
+                assertThat(securityContext.getThreadContext().getHeader(REMOTE_ACCESS_CLUSTER_CREDENTIAL_HEADER_KEY), nullValue());
+                assertThat(securityContext.getThreadContext().getHeader(REMOTE_ACCESS_AUTHENTICATION_HEADER_KEY), nullValue());
+            }
+        });
+        final Transport.Connection connection = mock(Transport.Connection.class);
+        when(connection.getTransportVersion()).thenReturn(TransportVersion.CURRENT);
+        sender.sendRequest(connection, actionAndReq.v1(), actionAndReq.v2(), null, null);
+        assertTrue(calledWrappedSender.get());
+        assertThat(sentAuthentication.get(), equalTo(authentication));
+        verify(authzService, never()).retrieveRemoteAccessRoleDescriptorsIntersection(any(), any(), anyActionListener());
+        assertThat(securityContext.getThreadContext().getHeader(REMOTE_ACCESS_AUTHENTICATION_HEADER_KEY), nullValue());
+        assertThat(securityContext.getThreadContext().getHeader(REMOTE_ACCESS_CLUSTER_CREDENTIAL_HEADER_KEY), nullValue());
+    }
+
+    public void testSendWithRemoteAccessHeadersThrowsOnOldConnection() throws Exception {
+        assumeTrue("untrusted remote cluster feature flag must be enabled", TcpTransport.isUntrustedRemoteClusterEnabled());
+
+        final Authentication authentication = AuthenticationTestHelper.builder()
+            .user(
+                new User(
+                    randomAlphaOfLengthBetween(3, 10),
+                    randomArray(
+                        0,
+                        4,
+                        String[]::new,
+                        () -> randomValueOtherThanMany(ReservedRolesStore::isReserved, () -> randomAlphaOfLengthBetween(1, 20))
+                    )
+                )
+            )
+            .realm()
+            .build();
+        authentication.writeToContext(threadContext);
+        final RemoteClusterAuthorizationResolver remoteClusterAuthorizationResolver = mock(RemoteClusterAuthorizationResolver.class);
+        final String remoteClusterCredential = randomAlphaOfLengthBetween(10, 42);
+        when(remoteClusterAuthorizationResolver.resolveAuthorization(any())).thenReturn(remoteClusterCredential);
+        final String remoteClusterAlias = randomAlphaOfLengthBetween(5, 10);
+
+        final SecurityServerTransportInterceptor interceptor = new SecurityServerTransportInterceptor(
+            settings,
+            threadPool,
+            mock(AuthenticationService.class),
+            mock(AuthorizationService.class),
+            mock(SSLService.class),
+            securityContext,
+            new DestructiveOperations(
+                Settings.EMPTY,
+                new ClusterSettings(Settings.EMPTY, Collections.singleton(DestructiveOperations.REQUIRES_NAME_SETTING))
+            ),
+            remoteClusterAuthorizationResolver,
+            ignored -> Optional.of(remoteClusterAlias)
+        );
+
+        final AsyncSender sender = interceptor.interceptSender(new AsyncSender() {
+            @Override
+            public <T extends TransportResponse> void sendRequest(
+                Transport.Connection connection,
+                String action,
+                TransportRequest request,
+                TransportRequestOptions options,
+                TransportResponseHandler<T> handler
+            ) {
+                fail("sender should not be called");
+            }
+        });
+        final Transport.Connection connection = mock(Transport.Connection.class);
+        final TransportVersion versionBeforeRemoteAccessHeaders = TransportVersionUtils.getPreviousVersion(TransportVersion.V_8_7_0);
+        final TransportVersion version = TransportVersionUtils.randomVersionBetween(
+            random(),
+            versionBeforeRemoteAccessHeaders.calculateMinimumCompatVersion(),
+            versionBeforeRemoteAccessHeaders
+        );
+        when(connection.getTransportVersion()).thenReturn(version);
+        final Tuple<String, TransportRequest> actionAndReq = randomAllowlistedActionAndRequest();
+        final AtomicBoolean calledHandleException = new AtomicBoolean(false);
+        final AtomicReference<TransportException> actualException = new AtomicReference<>();
+        sender.sendRequest(connection, actionAndReq.v1(), actionAndReq.v2(), null, new TransportResponseHandler<>() {
+            @Override
+            public void handleResponse(TransportResponse response) {
+                fail("should not receive a response");
+            }
+
+            @Override
+            public void handleException(TransportException exp) {
+                if (calledHandleException.compareAndSet(false, true) == false) {
+                    fail("handle exception called more than once");
+                }
+                actualException.set(exp);
+            }
+
+            @Override
+            public TransportResponse read(StreamInput in) {
+                fail("should not receive a response");
+                return null;
+            }
+        });
+        assertThat(actualException.get(), instanceOf(SendRequestTransportException.class));
+        assertThat(actualException.get().getCause(), instanceOf(IllegalArgumentException.class));
+        assertThat(
+            actualException.get().getCause().getMessage(),
+            equalTo(
+                "Settings for remote cluster ["
+                    + remoteClusterAlias
+                    + "] indicate remote access headers should be sent but target cluster version ["
+                    + connection.getTransportVersion()
+                    + "] does not support receiving them"
+            )
+        );
+        verify(remoteClusterAuthorizationResolver, times(1)).resolveAuthorization(eq(remoteClusterAlias));
+        assertThat(securityContext.getThreadContext().getHeader(REMOTE_ACCESS_AUTHENTICATION_HEADER_KEY), nullValue());
+        assertThat(securityContext.getThreadContext().getHeader(REMOTE_ACCESS_CLUSTER_CREDENTIAL_HEADER_KEY), nullValue());
+    }
+
+    public void testProfileFiltersCreatedDifferentlyForDifferentTransportAndRemoteClusterSslSettings() {
+        // filters are created irrespective of ssl enabled
+        final boolean transportSslEnabled = randomBoolean();
+        final boolean remoteClusterSslEnabled = randomBoolean();
+        final Settings.Builder builder = Settings.builder()
+            .put(this.settings)
+            .put("xpack.security.transport.ssl.enabled", transportSslEnabled)
+            .put("remote_cluster.enabled", true)
+            .put("xpack.security.remote_cluster.ssl.enabled", remoteClusterSslEnabled);
+        final SSLService sslService = mock(SSLService.class);
+
+        when(sslService.getSSLConfiguration("xpack.security.transport.ssl.")).thenReturn(
+            new SslConfiguration(
+                "xpack.security.transport.ssl",
+                randomBoolean(),
+                mock(SslTrustConfig.class),
+                mock(SslKeyConfig.class),
+                randomFrom(SslVerificationMode.values()),
+                SslClientAuthenticationMode.REQUIRED,
+                List.of("TLS_AES_256_GCM_SHA384"),
+                List.of("TLSv1.3")
+            )
+        );
+
+        when(sslService.getSSLConfiguration("xpack.security.remote_cluster.ssl.")).thenReturn(
+            new SslConfiguration(
+                "xpack.security.remote_cluster.ssl",
+                randomBoolean(),
+                mock(SslTrustConfig.class),
+                mock(SslKeyConfig.class),
+                randomFrom(SslVerificationMode.values()),
+                SslClientAuthenticationMode.NONE,
+                List.of("TLS_RSA_WITH_AES_256_GCM_SHA384"),
+                List.of("TLSv1.2")
+            )
+        );
+
+        final var securityServerTransportInterceptor = new SecurityServerTransportInterceptor(
+            builder.build(),
+            threadPool,
+            mock(AuthenticationService.class),
+            mock(AuthorizationService.class),
+            sslService,
+            securityContext,
+            new DestructiveOperations(
+                Settings.EMPTY,
+                new ClusterSettings(Settings.EMPTY, Collections.singleton(DestructiveOperations.REQUIRES_NAME_SETTING))
+            ),
+            new RemoteClusterAuthorizationResolver(settings, clusterService.getClusterSettings())
+        );
+
+        final Map<String, ServerTransportFilter> profileFilters = securityServerTransportInterceptor.getProfileFilters();
+        assertThat(profileFilters.keySet(), containsInAnyOrder("default", "_remote_cluster"));
+        assertThat(profileFilters.get("default").isExtractClientCert(), is(transportSslEnabled));
+        assertThat(profileFilters.get("_remote_cluster").isExtractClientCert(), is(false));
+    }
+
+    public void testNoProfileFilterForRemoteClusterWhenTheFeatureIsDisabled() {
+        final boolean transportSslEnabled = randomBoolean();
+        final Settings.Builder builder = Settings.builder()
+            .put(this.settings)
+            .put("xpack.security.transport.ssl.enabled", transportSslEnabled)
+            .put("remote_cluster.enabled", false)
+            .put("xpack.security.remote_cluster.ssl.enabled", randomBoolean());
+        final SSLService sslService = mock(SSLService.class);
+
+        when(sslService.getSSLConfiguration("xpack.security.transport.ssl.")).thenReturn(
+            new SslConfiguration(
+                "xpack.security.transport.ssl",
+                randomBoolean(),
+                mock(SslTrustConfig.class),
+                mock(SslKeyConfig.class),
+                randomFrom(SslVerificationMode.values()),
+                SslClientAuthenticationMode.REQUIRED,
+                List.of("TLS_AES_256_GCM_SHA384"),
+                List.of("TLSv1.3")
+            )
+        );
+
+        final var securityServerTransportInterceptor = new SecurityServerTransportInterceptor(
+            builder.build(),
+            threadPool,
+            mock(AuthenticationService.class),
+            mock(AuthorizationService.class),
+            sslService,
+            securityContext,
+            new DestructiveOperations(
+                Settings.EMPTY,
+                new ClusterSettings(Settings.EMPTY, Collections.singleton(DestructiveOperations.REQUIRES_NAME_SETTING))
+            ),
+            new RemoteClusterAuthorizationResolver(settings, clusterService.getClusterSettings())
+        );
+
+        final Map<String, ServerTransportFilter> profileFilters = securityServerTransportInterceptor.getProfileFilters();
+        assertThat(profileFilters.keySet(), contains("default"));
+        assertThat(profileFilters.get("default").isExtractClientCert(), is(transportSslEnabled));
+    }
+
+    private Tuple<String, TransportRequest> randomAllowlistedActionAndRequest() {
+        final String action = randomFrom(REMOTE_ACCESS_ACTION_ALLOWLIST.toArray(new String[0]));
+        return new Tuple<>(action, mock(TransportRequest.class));
+    }
+
     private String[] randomRoles() {
         return generateRandomStringArray(3, 10, false, true);
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterTests.java
index ac804d8bae27..7e9c785651fa 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterTests.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.security.transport;
 
 import org.elasticsearch.ElasticsearchSecurityException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.MockIndicesRequest;
 import org.elasticsearch.action.admin.indices.close.CloseIndexAction;
@@ -65,7 +65,7 @@ public void init() throws Exception {
         authzService = mock(AuthorizationService.class);
         channel = mock(TransportChannel.class);
         when(channel.getProfileName()).thenReturn(TransportSettings.DEFAULT_PROFILE);
-        when(channel.getVersion()).thenReturn(Version.CURRENT);
+        when(channel.getVersion()).thenReturn(TransportVersion.CURRENT);
         failDestructiveOperations = randomBoolean();
         Settings settings = Settings.builder().put(DestructiveOperations.REQUIRES_NAME_SETTING.getKey(), failDestructiveOperations).build();
         destructiveOperations = new DestructiveOperations(
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/filter/IPFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/filter/IPFilterTests.java
index 04c624298c56..77035738ea2f 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/filter/IPFilterTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/filter/IPFilterTests.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.license.MockLicenseState;
 import org.elasticsearch.license.TestUtils;
@@ -21,6 +22,8 @@
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.junit.annotations.Network;
+import org.elasticsearch.transport.RemoteClusterPortSettings;
+import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.xpack.security.LocalStateSecurity;
 import org.elasticsearch.xpack.security.Security;
@@ -37,10 +40,17 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashSet;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-
+import java.util.Objects;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.transport.RemoteClusterPortSettings.REMOTE_CLUSTER_PORT_ENABLED;
+import static org.elasticsearch.transport.RemoteClusterPortSettings.REMOTE_CLUSTER_PROFILE;
+import static org.elasticsearch.xpack.security.transport.filter.IPFilter.REMOTE_CLUSTER_FILTER_ALLOW_SETTING;
+import static org.elasticsearch.xpack.security.transport.filter.IPFilter.REMOTE_CLUSTER_FILTER_DENY_SETTING;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.mockito.ArgumentMatchers.eq;
@@ -67,18 +77,19 @@ public void init() {
         auditTrailService = new AuditTrailService(Collections.singletonList(auditTrail), licenseState);
         clusterSettings = new ClusterSettings(
             Settings.EMPTY,
-            new HashSet<>(
-                Arrays.asList(
-                    IPFilter.HTTP_FILTER_ALLOW_SETTING,
-                    IPFilter.HTTP_FILTER_DENY_SETTING,
-                    IPFilter.IP_FILTER_ENABLED_HTTP_SETTING,
-                    IPFilter.IP_FILTER_ENABLED_SETTING,
-                    IPFilter.TRANSPORT_FILTER_ALLOW_SETTING,
-                    IPFilter.TRANSPORT_FILTER_DENY_SETTING,
-                    IPFilter.PROFILE_FILTER_ALLOW_SETTING,
-                    IPFilter.PROFILE_FILTER_DENY_SETTING
-                )
-            )
+            Stream.of(
+                IPFilter.HTTP_FILTER_ALLOW_SETTING,
+                IPFilter.HTTP_FILTER_DENY_SETTING,
+                IPFilter.IP_FILTER_ENABLED_HTTP_SETTING,
+                IPFilter.IP_FILTER_ENABLED_SETTING,
+                IPFilter.TRANSPORT_FILTER_ALLOW_SETTING,
+                IPFilter.TRANSPORT_FILTER_DENY_SETTING,
+                TcpTransport.isUntrustedRemoteClusterEnabled() ? IPFilter.REMOTE_CLUSTER_FILTER_ALLOW_SETTING : null,
+                TcpTransport.isUntrustedRemoteClusterEnabled() ? IPFilter.REMOTE_CLUSTER_FILTER_DENY_SETTING : null,
+                IPFilter.PROFILE_FILTER_ALLOW_SETTING,
+                IPFilter.PROFILE_FILTER_DENY_SETTING,
+                TcpTransport.isUntrustedRemoteClusterEnabled() ? RemoteClusterPortSettings.REMOTE_CLUSTER_PORT_ENABLED : null
+            ).filter(Objects::nonNull).collect(Collectors.toSet())
         );
 
         httpTransport = mock(HttpServerTransport.class);
@@ -91,10 +102,18 @@ public void init() {
         when(transport.boundAddress()).thenReturn(new BoundTransportAddress(new TransportAddress[] { address }, address));
         when(transport.lifecycleState()).thenReturn(Lifecycle.State.STARTED);
 
-        Map<String, BoundTransportAddress> profileBoundAddresses = Collections.singletonMap(
+        Map<String, BoundTransportAddress> profileBoundAddresses = new HashMap<>();
+        profileBoundAddresses.put(
             "client",
             new BoundTransportAddress(new TransportAddress[] { new TransportAddress(InetAddress.getLoopbackAddress(), 9500) }, address)
         );
+
+        if (TcpTransport.isUntrustedRemoteClusterEnabled()) {
+            profileBoundAddresses.put(
+                REMOTE_CLUSTER_PROFILE,
+                new BoundTransportAddress(new TransportAddress[] { new TransportAddress(InetAddress.getLoopbackAddress(), 9600) }, address)
+            );
+        }
         when(transport.profileBoundAddresses()).thenReturn(profileBoundAddresses);
     }
 
@@ -262,7 +281,7 @@ public void testThatAllAddressesAreAllowedWhenLicenseDisablesSecurity() {
         ipFilter.setBoundTransportAddress(transport.boundAddress(), transport.profileBoundAddresses());
 
         // don't use the assert helper because we don't want the audit trail to be invoked here
-        String message = formatted("Expected address %s to be allowed", "8.8.8.8");
+        String message = Strings.format("Expected address %s to be allowed", "8.8.8.8");
         InetAddress address = InetAddresses.forString("8.8.8.8");
         assertThat(message, ipFilter.accept("default", new InetSocketAddress(address, 0)), is(true));
         verifyNoMoreInteractions(auditTrail);
@@ -287,9 +306,72 @@ public void testThatNodeStartsWithIPFilterDisabled() throws Exception {
         }
     }
 
+    public void testRemoteAccessCanBeFilteredSeparately() throws Exception {
+        assumeTrue("tests Remote Cluster Security 2.0 functionality", TcpTransport.isUntrustedRemoteClusterEnabled());
+        Settings settings = Settings.builder()
+            .put("xpack.security.transport.filter.allow", "192.168.0.2")
+            .put("xpack.security.transport.filter.deny", "192.168.0.1")
+            .put("xpack.security.remote_cluster.filter.allow", "192.168.0.1")
+            .put("xpack.security.remote_cluster.filter.deny", "_all")
+            .put(REMOTE_CLUSTER_PORT_ENABLED.getKey(), true)
+            .build();
+        ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
+        ipFilter.setBoundTransportAddress(transport.boundAddress(), transport.profileBoundAddresses());
+        assertAddressIsAllowed("192.168.0.2");
+        assertAddressIsDenied("192.168.0.1");
+        assertAddressIsAllowedForProfile(REMOTE_CLUSTER_PROFILE, "192.168.0.1");
+        assertAddressIsDeniedForProfile(REMOTE_CLUSTER_PROFILE, randomNonLocalIPv4Address());
+    }
+
+    /**
+     * Checks that if the Remote Cluster port is enabled, it uses the IP filters configured for transport in general if the specific
+     * remote_cluster filters are not set.
+     */
+    public void testThatRemoteAccessFallsBackToDefault() throws Exception {
+        assumeTrue("tests Remote Cluster Security 2.0 functionality", TcpTransport.isUntrustedRemoteClusterEnabled());
+        Settings settings = Settings.builder()
+            .put("xpack.security.transport.filter.allow", "192.168.0.1")
+            .put("xpack.security.transport.filter.deny", "_all")
+            .put(REMOTE_CLUSTER_PORT_ENABLED.getKey(), true)
+            .build();
+        ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
+        ipFilter.setBoundTransportAddress(transport.boundAddress(), transport.profileBoundAddresses());
+        assertAddressIsDenied(randomNonLocalIPv4Address());
+        assertAddressIsAllowedForProfile(REMOTE_CLUSTER_PROFILE, "192.168.0.1");
+        assertAddressIsDeniedForProfile(REMOTE_CLUSTER_PROFILE, randomNonLocalIPv4Address());
+    }
+
+    /**
+     * This test is very similar to {@link #testThatProfilesAreUpdateable()}, but specifically checks the remote cluster port settings.
+     */
+    public void testThatRemoteAccessIsUpdateable() throws Exception {
+        assumeTrue("tests Remote Cluster Security 2.0 functionality", TcpTransport.isUntrustedRemoteClusterEnabled());
+        Settings settings = Settings.builder()
+            .put("xpack.security.transport.filter.allow", "localhost")
+            .put("xpack.security.transport.filter.deny", "_all")
+            .put(REMOTE_CLUSTER_FILTER_ALLOW_SETTING.getKey(), "192.168.0.1")
+            .put(REMOTE_CLUSTER_FILTER_DENY_SETTING.getKey(), "_all")
+            .put(REMOTE_CLUSTER_PORT_ENABLED.getKey(), true)
+            .build();
+        ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
+        ipFilter.setBoundTransportAddress(transport.boundAddress(), transport.profileBoundAddresses());
+        Settings newSettings = Settings.builder()
+            .putList(REMOTE_CLUSTER_FILTER_ALLOW_SETTING.getKey(), "192.168.0.1", "192.168.0.2")
+            .put(REMOTE_CLUSTER_FILTER_DENY_SETTING.getKey(), "192.168.0.3")
+            .build();
+        Settings.Builder updatedSettingsBuilder = Settings.builder();
+        clusterSettings.updateDynamicSettings(newSettings, updatedSettingsBuilder, Settings.builder(), "test");
+        clusterSettings.applySettings(updatedSettingsBuilder.build());
+        assertAddressIsAllowed("127.0.0.1");
+        // when "localhost" is used, ES considers all local addresses see PatternRule#isLocalhost()
+        assertAddressIsDenied(randomNonLocalIPv4Address());
+        assertAddressIsAllowedForProfile(REMOTE_CLUSTER_PROFILE, "192.168.0.1", "192.168.0.2");
+        assertAddressIsDeniedForProfile(REMOTE_CLUSTER_PROFILE, "192.168.0.3");
+    }
+
     private void assertAddressIsAllowedForProfile(String profile, String... inetAddresses) {
         for (String inetAddress : inetAddresses) {
-            String message = formatted("Expected address %s to be allowed", inetAddress);
+            String message = Strings.format("Expected address %s to be allowed", inetAddress);
             InetSocketAddress address = new InetSocketAddress(InetAddresses.forString(inetAddress), 0);
             assertTrue(message, ipFilter.accept(profile, address));
             ArgumentCaptor<SecurityIpFilterRule> ruleCaptor = ArgumentCaptor.forClass(SecurityIpFilterRule.class);
@@ -304,7 +386,7 @@ private void assertAddressIsAllowed(String... inetAddresses) {
 
     private void assertAddressIsDeniedForProfile(String profile, String... inetAddresses) {
         for (String inetAddress : inetAddresses) {
-            String message = formatted("Expected address %s to be denied", inetAddress);
+            String message = Strings.format("Expected address %s to be denied", inetAddress);
             InetSocketAddress address = new InetSocketAddress(InetAddresses.forString(inetAddress), 0);
             assertFalse(message, ipFilter.accept(profile, address));
             ArgumentCaptor<SecurityIpFilterRule> ruleCaptor = ArgumentCaptor.forClass(SecurityIpFilterRule.class);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/IpFilterRemoteAddressFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/IpFilterRemoteAddressFilterTests.java
index 1acb3da7c9f0..a4bf66358152 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/IpFilterRemoteAddressFilterTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/IpFilterRemoteAddressFilterTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
@@ -18,6 +19,8 @@
 import org.elasticsearch.license.MockLicenseState;
 import org.elasticsearch.license.TestUtils;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.RemoteClusterPortSettings;
+import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.xpack.security.Security;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
@@ -29,6 +32,8 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
 
 import static org.hamcrest.Matchers.is;
 import static org.mockito.Mockito.mock;
@@ -36,39 +41,52 @@
 
 public class IpFilterRemoteAddressFilterTests extends ESTestCase {
     private IpFilterRemoteAddressFilter handler;
+    private IPFilter ipFilter;
 
     @Before
     public void init() throws Exception {
-        Settings settings = Settings.builder()
+        final Settings.Builder settingsBuilder = Settings.builder()
             .put("xpack.security.transport.filter.allow", "127.0.0.1")
-            .put("xpack.security.transport.filter.deny", "10.0.0.0/8")
-            .build();
+            .put("xpack.security.transport.filter.deny", "10.0.0.0/8");
+        if (TcpTransport.isUntrustedRemoteClusterEnabled()) {
+            settingsBuilder.put("remote_cluster.enabled", true);
+        }
+        Settings settings = settingsBuilder.build();
 
         boolean isHttpEnabled = randomBoolean();
 
         Transport transport = mock(Transport.class);
         TransportAddress address = new TransportAddress(InetAddress.getLoopbackAddress(), 9300);
         when(transport.boundAddress()).thenReturn(new BoundTransportAddress(new TransportAddress[] { address }, address));
+        TransportAddress remoteClusterIngressPortAddress = new TransportAddress(InetAddress.getLoopbackAddress(), 9443);
+        when(transport.profileBoundAddresses()).thenReturn(
+            Map.of(
+                RemoteClusterPortSettings.REMOTE_CLUSTER_PROFILE,
+                new BoundTransportAddress(new TransportAddress[] { remoteClusterIngressPortAddress }, remoteClusterIngressPortAddress)
+            )
+        );
         when(transport.lifecycleState()).thenReturn(Lifecycle.State.STARTED);
-        ClusterSettings clusterSettings = new ClusterSettings(
-            Settings.EMPTY,
-            new HashSet<>(
-                Arrays.asList(
-                    IPFilter.HTTP_FILTER_ALLOW_SETTING,
-                    IPFilter.HTTP_FILTER_DENY_SETTING,
-                    IPFilter.IP_FILTER_ENABLED_HTTP_SETTING,
-                    IPFilter.IP_FILTER_ENABLED_SETTING,
-                    IPFilter.TRANSPORT_FILTER_ALLOW_SETTING,
-                    IPFilter.TRANSPORT_FILTER_DENY_SETTING,
-                    IPFilter.PROFILE_FILTER_ALLOW_SETTING,
-                    IPFilter.PROFILE_FILTER_DENY_SETTING
-                )
+        final Set<Setting<?>> settingsSet = new HashSet<>(
+            Arrays.asList(
+                IPFilter.HTTP_FILTER_ALLOW_SETTING,
+                IPFilter.HTTP_FILTER_DENY_SETTING,
+                IPFilter.IP_FILTER_ENABLED_HTTP_SETTING,
+                IPFilter.IP_FILTER_ENABLED_SETTING,
+                IPFilter.TRANSPORT_FILTER_ALLOW_SETTING,
+                IPFilter.TRANSPORT_FILTER_DENY_SETTING,
+                IPFilter.PROFILE_FILTER_ALLOW_SETTING,
+                IPFilter.PROFILE_FILTER_DENY_SETTING
             )
         );
+        if (TcpTransport.isUntrustedRemoteClusterEnabled()) {
+            settingsSet.add(IPFilter.REMOTE_CLUSTER_FILTER_ALLOW_SETTING);
+            settingsSet.add(IPFilter.REMOTE_CLUSTER_FILTER_DENY_SETTING);
+        }
+        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, settingsSet);
         MockLicenseState licenseState = TestUtils.newMockLicenceState();
         when(licenseState.isAllowed(Security.IP_FILTERING_FEATURE)).thenReturn(true);
         AuditTrailService auditTrailService = new AuditTrailService(Collections.emptyList(), licenseState);
-        IPFilter ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
+        ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
         ipFilter.setBoundTransportAddress(transport.boundAddress(), transport.profileBoundAddresses());
         if (isHttpEnabled) {
             HttpServerTransport httpTransport = mock(HttpServerTransport.class);
@@ -92,4 +110,14 @@ public void testThatFilteringWorksByIp() throws Exception {
         InetSocketAddress remoteAddr = new InetSocketAddress(InetAddresses.forString("10.0.0.8"), 12345);
         assertThat(handler.accept(mock(ChannelHandlerContext.class), remoteAddr), is(false));
     }
+
+    public void testFilteringWorksForRemoteClusterPort() throws Exception {
+        assumeTrue("untrusted remote cluster feature flag must be enabled", TcpTransport.isUntrustedRemoteClusterEnabled());
+        handler = new IpFilterRemoteAddressFilter(ipFilter, RemoteClusterPortSettings.REMOTE_CLUSTER_PROFILE);
+        InetSocketAddress localhostAddr = new InetSocketAddress(InetAddresses.forString("127.0.0.1"), 12345);
+        assertThat(handler.accept(mock(ChannelHandlerContext.class), localhostAddr), is(true));
+
+        InetSocketAddress remoteAddr = new InetSocketAddress(InetAddresses.forString("10.0.0.8"), 12345);
+        assertThat(handler.accept(mock(ChannelHandlerContext.class), remoteAddr), is(false));
+    }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransportTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransportTests.java
index d89eea591b13..6be9b738aaaf 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransportTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransportTests.java
@@ -14,17 +14,17 @@
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.ssl.SslClientAuthenticationMode;
-import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.http.AbstractHttpServerTransportTestCase;
 import org.elasticsearch.http.NullDispatcher;
+import org.elasticsearch.http.netty4.Netty4HttpServerTransport;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.tracing.Tracer;
 import org.elasticsearch.transport.netty4.SharedGroupFactory;
+import org.elasticsearch.transport.netty4.TLSConfig;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.ssl.SSLService;
-import org.elasticsearch.xpack.security.transport.filter.IPFilter;
 import org.junit.Before;
 
 import java.nio.file.Path;
@@ -67,18 +67,17 @@ public void createSSLService() {
     public void testDefaultClientAuth() throws Exception {
         Settings settings = Settings.builder().put(env.settings()).put(XPackSettings.HTTP_SSL_ENABLED.getKey(), true).build();
         sslService = new SSLService(TestEnvironment.newEnvironment(settings));
-        SecurityNetty4HttpServerTransport transport = new SecurityNetty4HttpServerTransport(
+        Netty4HttpServerTransport transport = new Netty4HttpServerTransport(
             settings,
             new NetworkService(Collections.emptyList()),
-            mock(PageCacheRecycler.class),
-            mock(IPFilter.class),
-            sslService,
             mock(ThreadPool.class),
             xContentRegistry(),
             new NullDispatcher(),
             randomClusterSettings(),
             new SharedGroupFactory(settings),
-            Tracer.NOOP
+            Tracer.NOOP,
+            new TLSConfig(sslService.getHttpTransportSSLConfiguration(), sslService::createSSLEngine),
+            null
         );
         ChannelHandler handler = transport.configureServerChannelHandler();
         final EmbeddedChannel ch = new EmbeddedChannel(handler);
@@ -94,18 +93,17 @@ public void testOptionalClientAuth() throws Exception {
             .put("xpack.security.http.ssl.client_authentication", value)
             .build();
         sslService = new SSLService(TestEnvironment.newEnvironment(settings));
-        SecurityNetty4HttpServerTransport transport = new SecurityNetty4HttpServerTransport(
+        Netty4HttpServerTransport transport = new Netty4HttpServerTransport(
             settings,
             new NetworkService(Collections.emptyList()),
-            mock(PageCacheRecycler.class),
-            mock(IPFilter.class),
-            sslService,
             mock(ThreadPool.class),
             xContentRegistry(),
             new NullDispatcher(),
             randomClusterSettings(),
             new SharedGroupFactory(settings),
-            Tracer.NOOP
+            Tracer.NOOP,
+            new TLSConfig(sslService.getHttpTransportSSLConfiguration(), sslService::createSSLEngine),
+            null
         );
         ChannelHandler handler = transport.configureServerChannelHandler();
         final EmbeddedChannel ch = new EmbeddedChannel(handler);
@@ -121,18 +119,17 @@ public void testRequiredClientAuth() throws Exception {
             .put("xpack.security.http.ssl.client_authentication", value)
             .build();
         sslService = new SSLService(TestEnvironment.newEnvironment(settings));
-        SecurityNetty4HttpServerTransport transport = new SecurityNetty4HttpServerTransport(
+        Netty4HttpServerTransport transport = new Netty4HttpServerTransport(
             settings,
             new NetworkService(Collections.emptyList()),
-            mock(PageCacheRecycler.class),
-            mock(IPFilter.class),
-            sslService,
             mock(ThreadPool.class),
             xContentRegistry(),
             new NullDispatcher(),
             randomClusterSettings(),
             new SharedGroupFactory(settings),
-            Tracer.NOOP
+            Tracer.NOOP,
+            new TLSConfig(sslService.getHttpTransportSSLConfiguration(), sslService::createSSLEngine),
+            null
         );
         ChannelHandler handler = transport.configureServerChannelHandler();
         final EmbeddedChannel ch = new EmbeddedChannel(handler);
@@ -148,18 +145,17 @@ public void testNoClientAuth() throws Exception {
             .put("xpack.security.http.ssl.client_authentication", value)
             .build();
         sslService = new SSLService(TestEnvironment.newEnvironment(settings));
-        SecurityNetty4HttpServerTransport transport = new SecurityNetty4HttpServerTransport(
+        Netty4HttpServerTransport transport = new Netty4HttpServerTransport(
             settings,
             new NetworkService(Collections.emptyList()),
-            mock(PageCacheRecycler.class),
-            mock(IPFilter.class),
-            sslService,
             mock(ThreadPool.class),
             xContentRegistry(),
             new NullDispatcher(),
             randomClusterSettings(),
             new SharedGroupFactory(settings),
-            Tracer.NOOP
+            Tracer.NOOP,
+            new TLSConfig(sslService.getHttpTransportSSLConfiguration(), sslService::createSSLEngine),
+            null
         );
         ChannelHandler handler = transport.configureServerChannelHandler();
         final EmbeddedChannel ch = new EmbeddedChannel(handler);
@@ -170,18 +166,17 @@ public void testNoClientAuth() throws Exception {
     public void testCustomSSLConfiguration() throws Exception {
         Settings settings = Settings.builder().put(env.settings()).put(XPackSettings.HTTP_SSL_ENABLED.getKey(), true).build();
         sslService = new SSLService(TestEnvironment.newEnvironment(settings));
-        SecurityNetty4HttpServerTransport transport = new SecurityNetty4HttpServerTransport(
+        Netty4HttpServerTransport transport = new Netty4HttpServerTransport(
             settings,
             new NetworkService(Collections.emptyList()),
-            mock(PageCacheRecycler.class),
-            mock(IPFilter.class),
-            sslService,
             mock(ThreadPool.class),
             xContentRegistry(),
             new NullDispatcher(),
             randomClusterSettings(),
             new SharedGroupFactory(settings),
-            Tracer.NOOP
+            Tracer.NOOP,
+            new TLSConfig(sslService.getHttpTransportSSLConfiguration(), sslService::createSSLEngine),
+            null
         );
         ChannelHandler handler = transport.configureServerChannelHandler();
         EmbeddedChannel ch = new EmbeddedChannel(handler);
@@ -193,18 +188,17 @@ public void testCustomSSLConfiguration() throws Exception {
             .put("xpack.security.http.ssl.supported_protocols", "TLSv1.2")
             .build();
         sslService = new SSLService(TestEnvironment.newEnvironment(settings));
-        transport = new SecurityNetty4HttpServerTransport(
+        transport = new Netty4HttpServerTransport(
             settings,
             new NetworkService(Collections.emptyList()),
-            mock(PageCacheRecycler.class),
-            mock(IPFilter.class),
-            sslService,
             mock(ThreadPool.class),
             xContentRegistry(),
             new NullDispatcher(),
             randomClusterSettings(),
             new SharedGroupFactory(settings),
-            Tracer.NOOP
+            Tracer.NOOP,
+            new TLSConfig(sslService.getHttpTransportSSLConfiguration(), sslService::createSSLEngine),
+            null
         );
         handler = transport.configureServerChannelHandler();
         ch = new EmbeddedChannel(handler);
@@ -225,18 +219,17 @@ public void testNoExceptionWhenConfiguredWithoutSslKeySSLDisabled() throws Excep
             .build();
         env = TestEnvironment.newEnvironment(settings);
         sslService = new SSLService(env);
-        SecurityNetty4HttpServerTransport transport = new SecurityNetty4HttpServerTransport(
+        Netty4HttpServerTransport transport = new Netty4HttpServerTransport(
             settings,
             new NetworkService(Collections.emptyList()),
-            mock(PageCacheRecycler.class),
-            mock(IPFilter.class),
-            sslService,
             mock(ThreadPool.class),
             xContentRegistry(),
             new NullDispatcher(),
             randomClusterSettings(),
             new SharedGroupFactory(settings),
-            Tracer.NOOP
+            Tracer.NOOP,
+            new TLSConfig(sslService.getHttpTransportSSLConfiguration(), sslService::createSSLEngine),
+            null
         );
         assertNotNull(transport.configureServerChannelHandler());
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SimpleSecurityNetty4ServerTransportTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SimpleSecurityNetty4ServerTransportTests.java
index 9ac14ab22faf..e2b62f7d5b02 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SimpleSecurityNetty4ServerTransportTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SimpleSecurityNetty4ServerTransportTests.java
@@ -6,7 +6,10 @@
  */
 package org.elasticsearch.xpack.security.transport.netty4;
 
+import io.netty.handler.ssl.SslHandshakeTimeoutException;
+
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
@@ -25,8 +28,10 @@
 import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.mocksocket.MockServerSocket;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.test.transport.StubbableTransport;
 import org.elasticsearch.transport.AbstractSimpleTransportTestCase;
@@ -36,6 +41,7 @@
 import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.TestProfiles;
 import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.netty4.SharedGroupFactory;
 import org.elasticsearch.xpack.core.XPackSettings;
@@ -47,6 +53,8 @@
 import java.io.UncheckedIOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
 import java.net.SocketTimeoutException;
 import java.net.UnknownHostException;
 import java.nio.file.Path;
@@ -74,14 +82,17 @@
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.startsWith;
 
 public class SimpleSecurityNetty4ServerTransportTests extends AbstractSimpleTransportTestCase {
     @Override
-    protected Transport build(Settings settings, final Version version, ClusterSettings clusterSettings, boolean doHandshake) {
+    protected Transport build(Settings settings, TransportVersion version, ClusterSettings clusterSettings, boolean doHandshake) {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
         NetworkService networkService = new NetworkService(Collections.emptyList());
         Settings settings1 = Settings.builder().put(settings).put("xpack.security.transport.ssl.enabled", true).build();
@@ -103,12 +114,12 @@ public void executeHandshake(
                 DiscoveryNode node,
                 TcpChannel channel,
                 ConnectionProfile profile,
-                ActionListener<Version> listener
+                ActionListener<TransportVersion> listener
             ) {
                 if (doHandshake) {
                     super.executeHandshake(node, channel, profile, listener);
                 } else {
-                    listener.onResponse(version.minimumCompatibilityVersion());
+                    listener.onResponse(version.calculateMinimumCompatVersion());
                 }
             }
         };
@@ -125,6 +136,8 @@ protected SSLService createSSLService(Settings settings) {
         secureSettings.setString("xpack.security.transport.ssl.secure_key_passphrase", "testnode");
         // Some tests use a client profile. Put the passphrase in the secure settings for the profile (secure settings cannot be set twice)
         secureSettings.setString("transport.profiles.client.xpack.security.ssl.secure_key_passphrase", "testnode");
+        // For test that enables remote cluster port
+        secureSettings.setString("xpack.security.remote_cluster.ssl.secure_key_passphrase", "testnode");
         Settings settings1 = Settings.builder()
             .put("xpack.security.transport.ssl.enabled", true)
             .put("xpack.security.transport.ssl.key", testnodeKey)
@@ -187,7 +200,7 @@ public void testTcpHandshake() {
             PlainActionFuture<Transport.Connection> future = PlainActionFuture.newFuture();
             originalTransport.openConnection(node, connectionProfile, future);
             try (TcpTransport.NodeChannels connection = (TcpTransport.NodeChannels) future.actionGet()) {
-                assertEquals(connection.getVersion(), Version.CURRENT);
+                assertEquals(TransportVersion.CURRENT, connection.getTransportVersion());
             }
         }
     }
@@ -472,6 +485,297 @@ public void testSecurityClientAuthenticationConfigs() throws Exception {
         }
     }
 
+    public void testClientChannelUsesSeparateSslConfigurationForRemoteCluster() throws Exception {
+        final Path testnodeCert = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.crt");
+        final Path testnodeKey = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.pem");
+
+        final ConnectionProfile connectionProfile = ConnectionProfile.resolveConnectionProfile(
+            new ConnectionProfile.Builder().setTransportProfile("_remote_cluster")
+                .addConnections(
+                    1,
+                    TransportRequestOptions.Type.BULK,
+                    TransportRequestOptions.Type.BULK,
+                    TransportRequestOptions.Type.PING,
+                    TransportRequestOptions.Type.RECOVERY,
+                    TransportRequestOptions.Type.REG,
+                    TransportRequestOptions.Type.STATE
+                )
+                .build(),
+            TestProfiles.LIGHT_PROFILE
+        );
+
+        final Settings fcSettings = Settings.builder()
+            .put("remote_cluster.enabled", "true")
+            .put("remote_cluster.port", "9999")
+            .put("xpack.security.remote_cluster.ssl.key", testnodeKey)
+            .put("xpack.security.remote_cluster.ssl.certificate", testnodeCert)
+            .put("xpack.security.remote_cluster.ssl.client_authentication", "none")
+            .build();
+
+        try (MockTransportService fcService = buildService("FC", Version.CURRENT, fcSettings)) {
+            final TcpTransport originalTransport = (TcpTransport) fcService.getOriginalTransport();
+            final TransportAddress remoteAccessAddress = originalTransport.profileBoundAddresses().get("_remote_cluster").publishAddress();
+            final DiscoveryNode node = new DiscoveryNode(
+                fcService.getLocalNode().getId(),
+                remoteAccessAddress,
+                fcService.getLocalNode().getVersion()
+            );
+
+            // 1. Connection will fail because FC server certificate is not trusted by default
+            final Settings qcSettings1 = Settings.builder().build();
+            try (MockTransportService qcService = buildService("QC", Version.CURRENT, qcSettings1)) {
+                final ConnectTransportException e = expectThrows(
+                    ConnectTransportException.class,
+                    () -> openConnection(qcService, node, connectionProfile)
+                );
+                assertThat(
+                    e.getRootCause().getMessage(),
+                    anyOf(containsString("unable to find valid certification path"), containsString("Unable to find certificate chain"))
+                );
+            }
+
+            // 2. Connection will success because QC does not verify FC server certificate
+            final Settings qcSettings2 = Settings.builder().put("xpack.security.remote_cluster.ssl.verification_mode", "none").build();
+            try (
+                MockTransportService qcService = buildService("QC", Version.CURRENT, qcSettings2);
+                Transport.Connection connection = openConnection(qcService, node, connectionProfile)
+            ) {
+                assertThat(connection, instanceOf(StubbableTransport.WrappedConnection.class));
+                Transport.Connection conn = ((StubbableTransport.WrappedConnection) connection).getConnection();
+                assertThat(conn, instanceOf(TcpTransport.NodeChannels.class));
+                TcpTransport.NodeChannels nodeChannels = (TcpTransport.NodeChannels) conn;
+                for (TcpChannel channel : nodeChannels.getChannels()) {
+                    assertFalse(channel.isServerChannel());
+                    assertThat(channel.getProfile(), equalTo("_remote_cluster"));
+                    final SSLEngine sslEngine = SSLEngineUtils.getSSLEngine(channel);
+                    assertThat(sslEngine.getUseClientMode(), is(true));
+                }
+
+                final TcpChannel acceptedChannel = getAcceptedChannel(originalTransport, connection);
+                assertThat(acceptedChannel.getProfile(), equalTo("_remote_cluster"));
+            }
+
+            // 3. Connection will success because QC is explicitly configured to trust FC server certificate
+            final Settings qcSettings3 = Settings.builder()
+                .put("xpack.security.remote_cluster.ssl.certificate_authorities", testnodeCert)
+                .put("xpack.security.remote_cluster.ssl.verification_mode", "full")
+                .build();
+            try (
+                MockTransportService qcService = buildService("QC", Version.CURRENT, qcSettings3);
+                Transport.Connection connection = openConnection(qcService, node, connectionProfile)
+            ) {
+                assertThat(connection, instanceOf(StubbableTransport.WrappedConnection.class));
+                Transport.Connection conn = ((StubbableTransport.WrappedConnection) connection).getConnection();
+                assertThat(conn, instanceOf(TcpTransport.NodeChannels.class));
+                TcpTransport.NodeChannels nodeChannels = (TcpTransport.NodeChannels) conn;
+                for (TcpChannel channel : nodeChannels.getChannels()) {
+                    assertFalse(channel.isServerChannel());
+                    assertThat(channel.getProfile(), equalTo("_remote_cluster"));
+                    final SSLEngine sslEngine = SSLEngineUtils.getSSLEngine(channel);
+                    assertThat(sslEngine.getUseClientMode(), is(true));
+                }
+
+                final TcpChannel acceptedChannel = getAcceptedChannel(originalTransport, connection);
+                assertThat(acceptedChannel.getProfile(), equalTo("_remote_cluster"));
+            }
+        }
+    }
+
+    public void testRemoteClusterCanWorkWithoutSSL() throws Exception {
+        final ConnectionProfile connectionProfile = ConnectionProfile.resolveConnectionProfile(
+            new ConnectionProfile.Builder().setTransportProfile("_remote_cluster")
+                .addConnections(
+                    1,
+                    TransportRequestOptions.Type.BULK,
+                    TransportRequestOptions.Type.BULK,
+                    TransportRequestOptions.Type.PING,
+                    TransportRequestOptions.Type.RECOVERY,
+                    TransportRequestOptions.Type.REG,
+                    TransportRequestOptions.Type.STATE
+                )
+                .build(),
+            TestProfiles.LIGHT_PROFILE
+        );
+
+        final Settings fcSettings = Settings.builder()
+            .put("remote_cluster.enabled", "true")
+            .put("remote_cluster.port", "9999")
+            .put("xpack.security.remote_cluster.ssl.enabled", "false")
+            .build();
+
+        try (MockTransportService fcService = buildService("FC", Version.CURRENT, fcSettings)) {
+            final TcpTransport originalTransport = (TcpTransport) fcService.getOriginalTransport();
+            final TransportAddress remoteAccessAddress = originalTransport.profileBoundAddresses().get("_remote_cluster").publishAddress();
+            final DiscoveryNode node = new DiscoveryNode(
+                fcService.getLocalNode().getId(),
+                remoteAccessAddress,
+                fcService.getLocalNode().getVersion()
+            );
+            final Settings qcSettings = Settings.builder().put("xpack.security.remote_cluster.ssl.enabled", "false").build();
+            try (
+                MockTransportService qcService = buildService("QC", Version.CURRENT, qcSettings);
+                Transport.Connection connection = openConnection(qcService, node, connectionProfile)
+            ) {
+                assertThat(connection, instanceOf(StubbableTransport.WrappedConnection.class));
+                Transport.Connection conn = ((StubbableTransport.WrappedConnection) connection).getConnection();
+                assertThat(conn, instanceOf(TcpTransport.NodeChannels.class));
+                TcpTransport.NodeChannels nodeChannels = (TcpTransport.NodeChannels) conn;
+                for (TcpChannel channel : nodeChannels.getChannels()) {
+                    assertFalse(channel.isServerChannel());
+                    assertThat(channel.getProfile(), equalTo("_remote_cluster"));
+                }
+
+                final TcpChannel acceptedChannel = getAcceptedChannel(originalTransport, connection);
+                assertThat(acceptedChannel.getProfile(), equalTo("_remote_cluster"));
+            }
+        }
+    }
+
+    public void testTcpHandshakeTimeout() throws IOException {
+        assumeFalse("Can't run in a FIPS JVM, TrustAllConfig is not a SunJSSE TrustManagers", inFipsJvm());
+        SSLService sslService = createSSLService();
+
+        final SslConfiguration sslConfiguration = sslService.getSSLConfiguration("xpack.security.transport.ssl");
+        SSLContext sslContext = sslService.sslContext(sslConfiguration);
+        final SSLServerSocketFactory serverSocketFactory = sslContext.getServerSocketFactory();
+        // use latch to to ensure that the accepted socket below isn't closed before the handshake times out
+        final CountDownLatch doneLatch = new CountDownLatch(1);
+        try (ServerSocket socket = serverSocketFactory.createServerSocket()) {
+            socket.bind(getLocalEphemeral(), 1);
+            socket.setReuseAddress(true);
+            new Thread(() -> {
+                SSLSocket acceptedSocket = null;
+                try {
+                    acceptedSocket = (SSLSocket) SocketAccess.doPrivileged(socket::accept);
+                    // A read call will execute the ssl handshake
+                    int byteRead = acceptedSocket.getInputStream().read();
+                    assertEquals('E', byteRead);
+                    doneLatch.await();
+                } catch (Exception e) {
+                    throw new AssertionError(e);
+                } finally {
+                    IOUtils.closeWhileHandlingException(acceptedSocket);
+                }
+            }).start();
+            DiscoveryNode dummy = new DiscoveryNode(
+                "TEST",
+                new TransportAddress(socket.getInetAddress(), socket.getLocalPort()),
+                emptyMap(),
+                emptySet(),
+                version0
+            );
+            ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+            builder.addConnections(
+                1,
+                TransportRequestOptions.Type.BULK,
+                TransportRequestOptions.Type.PING,
+                TransportRequestOptions.Type.RECOVERY,
+                TransportRequestOptions.Type.REG,
+                TransportRequestOptions.Type.STATE
+            );
+            builder.setHandshakeTimeout(TimeValue.timeValueMillis(1));
+            Settings settings = Settings.builder().put("xpack.security.transport.ssl.verification_mode", "none").build();
+            try (MockTransportService serviceC = buildService("TS_C", version0, settings)) {
+                ConnectTransportException ex = expectThrows(
+                    ConnectTransportException.class,
+                    () -> connectToNode(serviceC, dummy, builder.build())
+                );
+                assertEquals("[][" + dummy.getAddress() + "] handshake_timeout[1ms]", ex.getMessage());
+            }
+        } finally {
+            doneLatch.countDown();
+        }
+    }
+
+    public void testTlsHandshakeTimeout() throws IOException {
+        final CountDownLatch doneLatch = new CountDownLatch(1);
+        try (ServerSocket socket = new MockServerSocket()) {
+            socket.bind(getLocalEphemeral(), 1);
+            socket.setReuseAddress(true);
+            new Thread(() -> {
+                try (Socket ignored = socket.accept()) {
+                    doneLatch.await();
+                } catch (Exception e) {
+                    throw new AssertionError(e);
+                }
+            }).start();
+            DiscoveryNode dummy = new DiscoveryNode(
+                "TEST",
+                new TransportAddress(socket.getInetAddress(), socket.getLocalPort()),
+                emptyMap(),
+                emptySet(),
+                version0
+            );
+            ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+            builder.addConnections(
+                1,
+                TransportRequestOptions.Type.BULK,
+                TransportRequestOptions.Type.PING,
+                TransportRequestOptions.Type.RECOVERY,
+                TransportRequestOptions.Type.REG,
+                TransportRequestOptions.Type.STATE
+            );
+            ConnectTransportException ex = expectThrows(
+                ConnectTransportException.class,
+                () -> connectToNode(serviceA, dummy, builder.build())
+            );
+            assertEquals("[][" + dummy.getAddress() + "] connect_exception", ex.getMessage());
+            assertNotNull(ExceptionsHelper.unwrap(ex, SslHandshakeTimeoutException.class));
+        } finally {
+            doneLatch.countDown();
+        }
+    }
+
+    public void testTcpHandshakeConnectionReset() throws IOException, InterruptedException {
+        assumeFalse("Can't run in a FIPS JVM, TrustAllConfig is not a SunJSSE TrustManagers", inFipsJvm());
+        SSLService sslService = createSSLService();
+
+        final SslConfiguration sslConfiguration = sslService.getSSLConfiguration("xpack.security.transport.ssl");
+        SSLContext sslContext = sslService.sslContext(sslConfiguration);
+        final SSLServerSocketFactory serverSocketFactory = sslContext.getServerSocketFactory();
+        try (ServerSocket socket = serverSocketFactory.createServerSocket()) {
+            socket.bind(getLocalEphemeral(), 1);
+            socket.setReuseAddress(true);
+            DiscoveryNode dummy = new DiscoveryNode(
+                "TEST",
+                new TransportAddress(socket.getInetAddress(), socket.getLocalPort()),
+                emptyMap(),
+                emptySet(),
+                version0
+            );
+            Thread t = new Thread(() -> {
+                try (Socket accept = SocketAccess.doPrivileged(socket::accept)) {
+                    // A read call will execute the ssl handshake
+                    int byteRead = accept.getInputStream().read();
+                    assertEquals('E', byteRead);
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+            });
+            t.start();
+            ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+            builder.addConnections(
+                1,
+                TransportRequestOptions.Type.BULK,
+                TransportRequestOptions.Type.PING,
+                TransportRequestOptions.Type.RECOVERY,
+                TransportRequestOptions.Type.REG,
+                TransportRequestOptions.Type.STATE
+            );
+            builder.setHandshakeTimeout(TimeValue.timeValueHours(1));
+            Settings settings = Settings.builder().put("xpack.security.transport.ssl.verification_mode", "none").build();
+            try (MockTransportService serviceC = buildService("TS_C", version0, settings)) {
+                ConnectTransportException ex = expectThrows(
+                    ConnectTransportException.class,
+                    () -> connectToNode(serviceC, dummy, builder.build())
+                );
+                assertEquals("[][" + dummy.getAddress() + "] general node connection failure", ex.getMessage());
+                assertThat(ex.getCause().getMessage(), startsWith("handshake failed"));
+            }
+            t.join();
+        }
+    }
+
     public static String randomCapitalization(SslClientAuthenticationMode mode) {
         return randomFrom(mode.name(), mode.name().toLowerCase(Locale.ROOT));
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageFileTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageFileTests.java
index d414dac19df4..beb28a95dd67 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageFileTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLErrorMessageFileTests.java
@@ -206,6 +206,54 @@ public void testMessageForHttpsNotEnabledButTruststoreConfigured() throws Except
         );
     }
 
+    public void testMessageForRemoteClusterSslEnabledWithoutKeys() {
+        final String prefix = "xpack.security.remote_cluster.ssl";
+        final Settings.Builder builder = Settings.builder().put("remote_cluster.enabled", true);
+        // remote cluster ssl is enabled by default
+        if (randomBoolean()) {
+            builder.put(prefix + ".enabled", true);
+        }
+        if (inFipsJvm()) {
+            configureWorkingTrustedAuthorities(prefix, builder);
+        } else {
+            configureWorkingTruststore(prefix, builder);
+        }
+
+        final Throwable exception = expectFailure(builder);
+        assertThat(
+            exception,
+            throwableWithMessage(
+                "invalid SSL configuration for "
+                    + prefix
+                    + " - server ssl configuration requires a key and certificate, but these have not been configured;"
+                    + " you must set either ["
+                    + prefix
+                    + ".keystore.path], or both ["
+                    + prefix
+                    + ".key] and ["
+                    + prefix
+                    + ".certificate]"
+            )
+        );
+        assertThat(exception, instanceOf(ElasticsearchException.class));
+    }
+
+    public void testNoErrorIfRemoteClusterOrSslDisabledWithoutKeys() {
+        final String prefix = "xpack.security.remote_cluster.ssl";
+        final Settings.Builder builder = Settings.builder().put(prefix + ".enabled", false);
+        if (randomBoolean()) {
+            builder.put("remote_cluster.enabled", true);
+        } else {
+            builder.put("remote_cluster.enabled", false);
+        }
+        if (inFipsJvm()) {
+            configureWorkingTrustedAuthorities(prefix, builder);
+        } else {
+            configureWorkingTruststore(prefix, builder);
+        }
+        expectSuccess(builder);
+    }
+
     private void checkMissingKeyManagerResource(String fileType, String configKey, @Nullable Settings.Builder additionalSettings) {
         checkMissingResource(fileType, configKey, (prefix, builder) -> buildKeyConfigSettings(additionalSettings, prefix, builder));
     }
diff --git a/x-pack/plugin/security/src/test/resources/META-INF/services/org.elasticsearch.reservedstate.ReservedClusterStateHandlerProvider b/x-pack/plugin/security/src/test/resources/META-INF/services/org.elasticsearch.reservedstate.ReservedClusterStateHandlerProvider
index 3d17572429ba..77c38d302d9c 100644
--- a/x-pack/plugin/security/src/test/resources/META-INF/services/org.elasticsearch.reservedstate.ReservedClusterStateHandlerProvider
+++ b/x-pack/plugin/security/src/test/resources/META-INF/services/org.elasticsearch.reservedstate.ReservedClusterStateHandlerProvider
@@ -6,3 +6,4 @@
 #
 
 org.elasticsearch.xpack.security.LocalReservedSecurityStateHandlerProvider
+org.elasticsearch.xpack.security.LocalReservedUnstableSecurityStateHandlerProvider
diff --git a/x-pack/plugin/shutdown/qa/full-cluster-restart/build.gradle b/x-pack/plugin/shutdown/qa/full-cluster-restart/build.gradle
index 429b29bbc9fd..d9539bb668b4 100644
--- a/x-pack/plugin/shutdown/qa/full-cluster-restart/build.gradle
+++ b/x-pack/plugin/shutdown/qa/full-cluster-restart/build.gradle
@@ -1,104 +1,21 @@
-import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
-apply plugin: 'elasticsearch.internal-testclusters'
-apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.bwc-test'
 
 dependencies {
   // TODO: Remove core dependency and change tests to not use builders that are part of xpack-core.
   // Currently needed for MlConfigIndexMappingsFullClusterRestartIT and SLM classes used in
   // FullClusterRestartIT
-  testImplementation(testArtifact(project(xpackModule('core'))))
-  testImplementation(testArtifact(project(":qa:full-cluster-restart")))
-  testImplementation project(':x-pack:qa')
-}
-
-tasks.named("forbiddenPatterns") {
-  exclude '**/system_key'
-}
-
-String outputDir = "${buildDir}/generated-resources/${project.name}"
-
-tasks.register("copyTestNodeKeyMaterial", Copy) {
-  from project(':x-pack:plugin:core')
-    .files(
-      'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem',
-      'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt'
-    )
-  into outputDir
+  javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
+  javaRestTestImplementation(testArtifact(project(":qa:full-cluster-restart")))
+  javaRestTestImplementation project(':x-pack:qa')
 }
 
 BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName ->
-    def baseCluster = testClusters.register(baseName) {
-        testDistribution = "DEFAULT"
-        if (bwcVersion.before(BuildParams.bwcVersions.minimumWireCompatibleVersion)) {
-          // When testing older versions we have to first upgrade to 7.last
-          versions = [bwcVersion.toString(), BuildParams.bwcVersions.minimumWireCompatibleVersion.toString(), project.version]
-        } else {
-          versions = [bwcVersion.toString(), project.version]
-        }
-        numberOfNodes = 2
-        setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
-        user username: "test_user", password: "x-pack-test-password"
-
-        setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
-        // some tests rely on the translog not being flushed
-        setting 'indices.memory.shard_inactive_time', '60m'
-        setting 'xpack.security.enabled', 'true'
-        setting 'xpack.security.transport.ssl.enabled', 'true'
-        setting 'xpack.license.self_generated.type', 'trial'
-
-        extraConfigFile 'testnode.pem', file("${outputDir}/testnode.pem")
-        extraConfigFile 'testnode.crt', file("${outputDir}/testnode.crt")
-
-        keystore 'xpack.watcher.encryption_key', file("${project.projectDir}/src/test/resources/system_key")
-        setting 'xpack.watcher.encrypt_sensitive_data', 'true'
-
-        setting 'xpack.security.transport.ssl.key', 'testnode.pem'
-        setting 'xpack.security.transport.ssl.certificate', 'testnode.crt'
-        keystore 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
-
-        setting 'xpack.security.authc.api_key.enabled', 'true'
-
-        requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
-    }
-
-    tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) {
-        mustRunAfter("precommit")
-        useCluster baseCluster
-        dependsOn "copyTestNodeKeyMaterial"
-        doFirst {
-            delete("${buildDir}/cluster/shared/repo/${baseName}")
-        }
-        systemProperty 'tests.is_old_cluster', 'true'
-    }
-
-    tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) {
-        mustRunAfter("precommit")
-        useCluster baseCluster
-        dependsOn "${baseName}#oldClusterTest"
-        doFirst {
-            baseCluster.get().goToNextVersion()
-            if (bwcVersion.before(BuildParams.bwcVersions.minimumWireCompatibleVersion)) {
-              // When doing a full cluster restart of older versions we actually have to upgrade twice. First to 7.last, then to the current version.
-              baseCluster.get().goToNextVersion()
-            }
-        }
-        systemProperty 'tests.is_old_cluster', 'false'
-    }
-
-    String oldVersion = bwcVersion.toString().minus("-SNAPSHOT")
-    tasks.matching { it.name.startsWith("${baseName}#") && it.name.endsWith("ClusterTest") }.configureEach {
-        it.systemProperty 'tests.old_cluster_version', oldVersion
-        it.systemProperty 'tests.path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
-        it.nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
-        it.nonInputProperties.systemProperty('tests.clustername', baseName)
-    }
-
-    tasks.register(bwcTaskName(bwcVersion)) {
-        dependsOn "${baseName}#upgradedClusterTest"
-    }
-
+  tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) {
+    usesBwcDistribution(bwcVersion)
+    systemProperty("tests.old_cluster_version", bwcVersion)
+  }
 }
diff --git a/x-pack/plugin/shutdown/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/plugin/shutdown/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
new file mode 100644
index 000000000000..07ed59477064
--- /dev/null
+++ b/x-pack/plugin/shutdown/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
@@ -0,0 +1,142 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.restart;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.FeatureFlag;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.cluster.util.resource.Resource;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus;
+import org.elasticsearch.upgrades.ParameterizedFullClusterRestartTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.json.JsonXContent;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.equalToIgnoringCase;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCase {
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .version(getOldClusterTestVersion())
+        .nodes(2)
+        // some tests rely on the translog not being flushed
+        .setting("indices.memory.shard_inactive_time", "60m")
+        .setting("xpack.security.enabled", "true")
+        .setting("xpack.security.transport.ssl.enabled", "true")
+        .setting("xpack.security.transport.ssl.key", "testnode.pem")
+        .setting("xpack.security.transport.ssl.certificate", "testnode.crt")
+        .setting("xpack.license.self_generated.type", "trial")
+        .setting("xpack.watcher.encrypt_sensitive_data", "true")
+        .setting("xpack.security.authc.api_key.enabled", "true")
+        .configFile("testnode.pem", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem"))
+        .configFile("testnode.crt", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"))
+        .keystore("xpack.watcher.encryption_key", Resource.fromClasspath("system_key"))
+        .keystore("xpack.security.transport.ssl.secure_key_passphrase", "testnode")
+        .feature(FeatureFlag.TIME_SERIES_MODE)
+        .build();
+
+    public FullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) {
+        super(upgradeStatus);
+    }
+
+    @Override
+    protected ElasticsearchCluster getUpgradeCluster() {
+        return cluster;
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            // we increase the timeout here to 90 seconds to handle long waits for a green
+            // cluster health. the waits for green need to be longer than a minute to
+            // account for delayed shards
+            .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s")
+            .build();
+    }
+
+    @BeforeClass
+    public static void checkClusterVersion() {
+        assumeTrue("no shutdown in versions before " + Version.V_7_15_0, getOldClusterVersion().onOrAfter(Version.V_7_15_0));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testNodeShutdown() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+            final Request getNodesReq = new Request("GET", "_nodes");
+            final Response getNodesResp = adminClient().performRequest(getNodesReq);
+            final Map<String, Object> nodes = (Map<String, Object>) entityAsMap(getNodesResp).get("nodes");
+            final String nodeIdToShutdown = randomFrom(nodes.keySet());
+
+            final Request putShutdownRequest = new Request("PUT", "_nodes/" + nodeIdToShutdown + "/shutdown");
+            try (XContentBuilder putBody = JsonXContent.contentBuilder()) {
+                putBody.startObject();
+                {
+                    // Use the types available from as early as possible
+                    final String type = randomFrom("restart", "remove");
+                    putBody.field("type", type);
+                    putBody.field("reason", getRootTestName());
+                }
+                putBody.endObject();
+                putShutdownRequest.setJsonEntity(Strings.toString(putBody));
+            }
+            assertOK(client().performRequest(putShutdownRequest));
+        }
+
+        assertBusy(() -> {
+            final Request getShutdownsReq = new Request("GET", "_nodes/shutdown");
+            final Response getShutdownsResp = client().performRequest(getShutdownsReq);
+            final Map<String, Object> stringObjectMap = entityAsMap(getShutdownsResp);
+
+            final List<Map<String, Object>> shutdowns = (List<Map<String, Object>>) stringObjectMap.get("nodes");
+            assertThat("there should be exactly one shutdown registered", shutdowns, hasSize(1));
+            final Map<String, Object> shutdown = shutdowns.get(0);
+            assertThat(shutdown.get("node_id"), notNullValue()); // Since we randomly determine the node ID, we can't check it
+            assertThat(shutdown.get("reason"), equalTo(getRootTestName()));
+            assertThat(
+                (String) shutdown.get("status"),
+                anyOf(
+                    Arrays.stream(SingleNodeShutdownMetadata.Status.values())
+                        .map(value -> equalToIgnoringCase(value.toString()))
+                        .collect(Collectors.toList())
+                )
+            );
+        }, 30, TimeUnit.SECONDS);
+    }
+
+    @Override
+    protected void deleteAllNodeShutdownMetadata() throws IOException {
+        // do not delete node shutdown
+    }
+}
diff --git a/x-pack/plugin/shutdown/qa/full-cluster-restart/src/test/resources/system_key b/x-pack/plugin/shutdown/qa/full-cluster-restart/src/javaRestTest/resources/system_key
similarity index 100%
rename from x-pack/plugin/shutdown/qa/full-cluster-restart/src/test/resources/system_key
rename to x-pack/plugin/shutdown/qa/full-cluster-restart/src/javaRestTest/resources/system_key
diff --git a/x-pack/plugin/shutdown/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/plugin/shutdown/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
deleted file mode 100644
index df6e3ed6b938..000000000000
--- a/x-pack/plugin/shutdown/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.restart;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.test.rest.ESRestTestCase;
-import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.json.JsonXContent;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.Base64;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import static org.hamcrest.Matchers.anyOf;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.equalToIgnoringCase;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.notNullValue;
-
-public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
-
-    @Override
-    protected Settings restClientSettings() {
-        String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8));
-        return Settings.builder()
-            .put(ThreadContext.PREFIX + ".Authorization", token)
-            // we increase the timeout here to 90 seconds to handle long waits for a green
-            // cluster health. the waits for green need to be longer than a minute to
-            // account for delayed shards
-            .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s")
-            .build();
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testNodeShutdown() throws Exception {
-        assumeTrue("no shutdown in versions before " + Version.V_7_15_0, getOldClusterVersion().onOrAfter(Version.V_7_15_0));
-
-        if (isRunningAgainstOldCluster()) {
-            final Request getNodesReq = new Request("GET", "_nodes");
-            final Response getNodesResp = adminClient().performRequest(getNodesReq);
-            final Map<String, Object> nodes = (Map<String, Object>) entityAsMap(getNodesResp).get("nodes");
-            final String nodeIdToShutdown = randomFrom(nodes.keySet());
-
-            final Request putShutdownRequest = new Request("PUT", "_nodes/" + nodeIdToShutdown + "/shutdown");
-            try (XContentBuilder putBody = JsonXContent.contentBuilder()) {
-                putBody.startObject();
-                {
-                    // Use the types available from as early as possible
-                    final String type = randomFrom("restart", "remove");
-                    putBody.field("type", type);
-                    putBody.field("reason", this.getTestName());
-                }
-                putBody.endObject();
-                putShutdownRequest.setJsonEntity(Strings.toString(putBody));
-            }
-            assertOK(client().performRequest(putShutdownRequest));
-        }
-
-        assertBusy(() -> {
-            final Request getShutdownsReq = new Request("GET", "_nodes/shutdown");
-            final Response getShutdownsResp = client().performRequest(getShutdownsReq);
-            final Map<String, Object> stringObjectMap = entityAsMap(getShutdownsResp);
-
-            final List<Map<String, Object>> shutdowns = (List<Map<String, Object>>) stringObjectMap.get("nodes");
-            assertThat("there should be exactly one shutdown registered", shutdowns, hasSize(1));
-            final Map<String, Object> shutdown = shutdowns.get(0);
-            assertThat(shutdown.get("node_id"), notNullValue()); // Since we randomly determine the node ID, we can't check it
-            assertThat(shutdown.get("reason"), equalTo(this.getTestName()));
-            assertThat(
-                (String) shutdown.get("status"),
-                anyOf(
-                    Arrays.stream(SingleNodeShutdownMetadata.Status.values())
-                        .map(value -> equalToIgnoringCase(value.toString()))
-                        .collect(Collectors.toList())
-                )
-            );
-        }, 30, TimeUnit.SECONDS);
-    }
-
-    @Override
-    protected void deleteAllNodeShutdownMetadata() throws IOException {
-        // do not delete node shutdown
-    }
-}
diff --git a/x-pack/plugin/shutdown/qa/multi-node/build.gradle b/x-pack/plugin/shutdown/qa/multi-node/build.gradle
index 2e2f8e12d79e..74dbfeb5cbef 100644
--- a/x-pack/plugin/shutdown/qa/multi-node/build.gradle
+++ b/x-pack/plugin/shutdown/qa/multi-node/build.gradle
@@ -1,6 +1,6 @@
 import org.elasticsearch.gradle.VersionProperties
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.authenticated-testclusters'
 
 dependencies {
diff --git a/x-pack/plugin/shutdown/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownIT.java b/x-pack/plugin/shutdown/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownIT.java
index d99be7b2ea05..62c98befa6f4 100644
--- a/x-pack/plugin/shutdown/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownIT.java
+++ b/x-pack/plugin/shutdown/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownIT.java
@@ -344,7 +344,7 @@ public void testStalledShardMigrationProperlyDetected() throws Exception {
         // Create an index, pin the allocation to the node we're about to shut down
         final String indexName = "test-idx";
         Request createIndexRequest = new Request("PUT", indexName);
-        createIndexRequest.setJsonEntity(formatted("""
+        createIndexRequest.setJsonEntity(Strings.format("""
             {
               "settings": {
                 "number_of_shards": %s,
diff --git a/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownDelayedAllocationIT.java b/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownDelayedAllocationIT.java
index 4231d4094f19..937b67c8d787 100644
--- a/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownDelayedAllocationIT.java
+++ b/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownDelayedAllocationIT.java
@@ -24,7 +24,6 @@
 
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -230,8 +229,7 @@ private void indexRandomData() throws Exception {
     private String findIdOfNodeWithShard() {
         ClusterState state = client().admin().cluster().prepareState().get().getState();
         List<ShardRouting> startedShards = RoutingNodesHelper.shardsWithState(state.getRoutingNodes(), ShardRoutingState.STARTED);
-        Collections.shuffle(startedShards, random());
-        return startedShards.get(0).currentNodeId();
+        return randomFrom(startedShards).currentNodeId();
     }
 
     private String findNodeNameFromId(String id) {
diff --git a/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownTasksIT.java b/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownTasksIT.java
index a4079f84fcf3..270d553f3b76 100644
--- a/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownTasksIT.java
+++ b/x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/NodeShutdownTasksIT.java
@@ -10,7 +10,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ResourceAlreadyExistsException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
@@ -21,7 +21,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -146,7 +146,7 @@ public Collection<Object> createComponents(
             IndexNameExpressionResolver indexNameExpressionResolver,
             Supplier<RepositoriesService> repositoriesServiceSupplier,
             Tracer tracer,
-            AllocationDeciders allocationDeciders
+            AllocationService allocationService
         ) {
             taskExecutor = new TaskExecutor(client, clusterService, threadPool);
             return Collections.singletonList(taskExecutor);
@@ -257,8 +257,8 @@ public String getWriteableName() {
         }
 
         @Override
-        public Version getMinimalSupportedVersion() {
-            return Version.CURRENT;
+        public TransportVersion getMinimalSupportedVersion() {
+            return TransportVersion.CURRENT;
         }
 
         @Override
diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/PutShutdownNodeAction.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/PutShutdownNodeAction.java
index 2a5ff0623ab8..d9232cc9ed5f 100644
--- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/PutShutdownNodeAction.java
+++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/PutShutdownNodeAction.java
@@ -92,7 +92,7 @@ public Request(StreamInput in) throws IOException {
             this.type = in.readEnum(SingleNodeShutdownMetadata.Type.class);
             this.reason = in.readString();
             this.allocationDelay = in.readOptionalTimeValue();
-            if (in.getVersion().onOrAfter(REPLACE_SHUTDOWN_TYPE_ADDED_VERSION)) {
+            if (in.getTransportVersion().onOrAfter(REPLACE_SHUTDOWN_TYPE_ADDED_VERSION)) {
                 this.targetNodeName = in.readOptionalString();
             } else {
                 this.targetNodeName = null;
@@ -102,14 +102,15 @@ public Request(StreamInput in) throws IOException {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(nodeId);
-            if (out.getVersion().before(REPLACE_SHUTDOWN_TYPE_ADDED_VERSION) && this.type == SingleNodeShutdownMetadata.Type.REPLACE) {
+            if (out.getTransportVersion().before(REPLACE_SHUTDOWN_TYPE_ADDED_VERSION)
+                && this.type == SingleNodeShutdownMetadata.Type.REPLACE) {
                 out.writeEnum(SingleNodeShutdownMetadata.Type.REMOVE);
             } else {
                 out.writeEnum(type);
             }
             out.writeString(reason);
             out.writeOptionalTimeValue(allocationDelay);
-            if (out.getVersion().onOrAfter(REPLACE_SHUTDOWN_TYPE_ADDED_VERSION)) {
+            if (out.getTransportVersion().onOrAfter(REPLACE_SHUTDOWN_TYPE_ADDED_VERSION)) {
                 out.writeOptionalString(targetNodeName);
             }
         }
diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/ShutdownPlugin.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/ShutdownPlugin.java
index 734ec85bb96e..361ff34ff388 100644
--- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/ShutdownPlugin.java
+++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/ShutdownPlugin.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -53,7 +53,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
 
         NodeSeenService nodeSeenService = new NodeSeenService(clusterService);
diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java
index 806bf22d2688..faae39f94ce6 100644
--- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java
+++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java
@@ -267,7 +267,6 @@ static ShutdownShardMigrationStatus shardMigrationStatus(
         Optional<Tuple<ShardRouting, ShardAllocationDecision>> unmovableShard = currentState.getRoutingNodes()
             .node(nodeId)
             .shardsWithState(ShardRoutingState.STARTED)
-            .stream()
             .map(shardRouting -> new Tuple<>(shardRouting, allocationService.explainShardAllocation(shardRouting, allocation)))
             // Given that we're checking the status of a node that's shutting down, no shards should be allowed to remain
             .filter(pair -> {
diff --git a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/GetShutdownStatusRequestTests.java b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/GetShutdownStatusRequestTests.java
index b8da9341f970..769d93a94ae6 100644
--- a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/GetShutdownStatusRequestTests.java
+++ b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/GetShutdownStatusRequestTests.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
@@ -31,7 +30,7 @@ protected GetShutdownStatusAction.Request createTestInstance() {
     }
 
     @Override
-    protected GetShutdownStatusAction.Request mutateInstance(GetShutdownStatusAction.Request instance) throws IOException {
+    protected GetShutdownStatusAction.Request mutateInstance(GetShutdownStatusAction.Request instance) {
         Set<String> oldIds = new HashSet<>(Arrays.asList(instance.getNodeIds()));
         String[] newNodeIds = randomList(1, 20, () -> randomValueOtherThanMany(oldIds::contains, () -> randomAlphaOfLengthBetween(15, 25)))
             .toArray(Strings.EMPTY_ARRAY);
diff --git a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/GetShutdownStatusResponseTests.java b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/GetShutdownStatusResponseTests.java
index 1195ab279e4f..821d7e13214f 100644
--- a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/GetShutdownStatusResponseTests.java
+++ b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/GetShutdownStatusResponseTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.HashSet;
@@ -35,7 +34,7 @@ protected GetShutdownStatusAction.Response createTestInstance() {
     }
 
     @Override
-    protected GetShutdownStatusAction.Response mutateInstance(GetShutdownStatusAction.Response instance) throws IOException {
+    protected GetShutdownStatusAction.Response mutateInstance(GetShutdownStatusAction.Response instance) {
         Set<SingleNodeShutdownStatus> oldNodes = new HashSet<>(instance.getShutdownStatuses());
         List<SingleNodeShutdownStatus> newNodes = randomList(
             1,
diff --git a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java
index 10dbe670a0f2..4b8fb5c1dbe2 100644
--- a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java
+++ b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.TestShardRoutingRoleStrategies;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.NodesShutdownMetadata;
@@ -127,7 +128,8 @@ public Decision canRebalance(RoutingAllocation allocation) {
             allocationDeciders,
             new BalancedShardsAllocator(Settings.EMPTY),
             clusterInfoService,
-            snapshotsInfoService
+            snapshotsInfoService,
+            TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY
         );
         allocationService.setExistingShardsAllocators(Map.of(GatewayAllocator.ALLOCATOR_NAME, new TestGatewayAllocator()));
     }
diff --git a/x-pack/plugin/snapshot-based-recoveries/qa/azure/build.gradle b/x-pack/plugin/snapshot-based-recoveries/qa/azure/build.gradle
index 002ddfeaad8c..c9e312fe6361 100644
--- a/x-pack/plugin/snapshot-based-recoveries/qa/azure/build.gradle
+++ b/x-pack/plugin/snapshot-based-recoveries/qa/azure/build.gradle
@@ -9,7 +9,7 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 
 final Project fixture = project(':test:fixtures:azure-fixture')
diff --git a/x-pack/plugin/snapshot-based-recoveries/qa/fs/build.gradle b/x-pack/plugin/snapshot-based-recoveries/qa/fs/build.gradle
index 3e53f734d736..0c0c1930f860 100644
--- a/x-pack/plugin/snapshot-based-recoveries/qa/fs/build.gradle
+++ b/x-pack/plugin/snapshot-based-recoveries/qa/fs/build.gradle
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 
 dependencies {
diff --git a/x-pack/plugin/snapshot-based-recoveries/qa/gcs/build.gradle b/x-pack/plugin/snapshot-based-recoveries/qa/gcs/build.gradle
index 0249677a3504..3824906da3c4 100644
--- a/x-pack/plugin/snapshot-based-recoveries/qa/gcs/build.gradle
+++ b/x-pack/plugin/snapshot-based-recoveries/qa/gcs/build.gradle
@@ -6,7 +6,7 @@ import java.security.KeyPairGenerator
 
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 
 final Project fixture = project(':test:fixtures:gcs-fixture')
diff --git a/x-pack/plugin/snapshot-based-recoveries/qa/license-enforcing/build.gradle b/x-pack/plugin/snapshot-based-recoveries/qa/license-enforcing/build.gradle
index b9f843ad2670..1c9a87750779 100644
--- a/x-pack/plugin/snapshot-based-recoveries/qa/license-enforcing/build.gradle
+++ b/x-pack/plugin/snapshot-based-recoveries/qa/license-enforcing/build.gradle
@@ -8,7 +8,7 @@ import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
  * Side Public License, v 1.
  */
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 
 dependencies {
diff --git a/x-pack/plugin/snapshot-based-recoveries/qa/s3/build.gradle b/x-pack/plugin/snapshot-based-recoveries/qa/s3/build.gradle
index eccdc63386e4..dfac9046ec32 100644
--- a/x-pack/plugin/snapshot-based-recoveries/qa/s3/build.gradle
+++ b/x-pack/plugin/snapshot-based-recoveries/qa/s3/build.gradle
@@ -9,7 +9,7 @@
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 
 final Project fixture = project(':test:fixtures:s3-fixture')
diff --git a/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/ConfigurableMockSnapshotBasedRecoveriesPlugin.java b/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/ConfigurableMockSnapshotBasedRecoveriesPlugin.java
index 9320fa633c0d..2d96416b8164 100644
--- a/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/ConfigurableMockSnapshotBasedRecoveriesPlugin.java
+++ b/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/ConfigurableMockSnapshotBasedRecoveriesPlugin.java
@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.snapshotbasedrecoveries.recovery;
 
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.CheckedRunnable;
 import org.elasticsearch.xpack.snapshotbasedrecoveries.SnapshotBasedRecoveriesPlugin;
 
@@ -15,7 +16,9 @@
 public class ConfigurableMockSnapshotBasedRecoveriesPlugin extends SnapshotBasedRecoveriesPlugin {
     private static final AtomicBoolean recoveryFromSnapshotAllowed = new AtomicBoolean(true);
 
-    public ConfigurableMockSnapshotBasedRecoveriesPlugin() {}
+    public ConfigurableMockSnapshotBasedRecoveriesPlugin(Settings settings) {
+        super(settings);
+    }
 
     @Override
     public boolean isLicenseEnabled() {
diff --git a/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/IndexRecoveryWithSnapshotsIT.java b/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/IndexRecoveryWithSnapshotsIT.java
index a025db2d5dc8..25bfacbf293b 100644
--- a/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/IndexRecoveryWithSnapshotsIT.java
+++ b/x-pack/plugin/snapshot-based-recoveries/src/internalClusterTest/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/IndexRecoveryWithSnapshotsIT.java
@@ -22,7 +22,6 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return CollectionUtils.appendToCopy(super.nodePlugins(), ConfigurableMockSnapshotBasedRecoveriesPlugin.class);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/91087")
     public void testTransientErrorsDuringRecoveryAreRetried() throws Exception {
         checkTransientErrorsDuringRecoveryAreRetried(PeerRecoveryTargetService.Actions.RESTORE_FILE_FROM_SNAPSHOT);
     }
diff --git a/x-pack/plugin/snapshot-based-recoveries/src/main/java/org/elasticsearch/xpack/snapshotbasedrecoveries/SnapshotBasedRecoveriesPlugin.java b/x-pack/plugin/snapshot-based-recoveries/src/main/java/org/elasticsearch/xpack/snapshotbasedrecoveries/SnapshotBasedRecoveriesPlugin.java
index 8e2c167992d9..fdc4dad7a6e3 100644
--- a/x-pack/plugin/snapshot-based-recoveries/src/main/java/org/elasticsearch/xpack/snapshotbasedrecoveries/SnapshotBasedRecoveriesPlugin.java
+++ b/x-pack/plugin/snapshot-based-recoveries/src/main/java/org/elasticsearch/xpack/snapshotbasedrecoveries/SnapshotBasedRecoveriesPlugin.java
@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.snapshotbasedrecoveries;
 
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.recovery.plan.RecoveryPlannerService;
 import org.elasticsearch.indices.recovery.plan.ShardSnapshotsService;
 import org.elasticsearch.license.License;
@@ -16,18 +18,28 @@
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.snapshotbasedrecoveries.recovery.plan.SnapshotsRecoveryPlannerService;
 
+import java.util.Optional;
+
 public class SnapshotBasedRecoveriesPlugin extends Plugin implements RecoveryPlannerPlugin {
+
     public static final LicensedFeature.Momentary SNAPSHOT_BASED_RECOVERIES_FEATURE = LicensedFeature.momentary(
         null,
         "snapshot-based-recoveries",
         License.OperationMode.ENTERPRISE
     );
 
-    public SnapshotBasedRecoveriesPlugin() {}
+    private final Settings settings;
+
+    public SnapshotBasedRecoveriesPlugin(Settings settings) {
+        this.settings = settings;
+    }
 
     @Override
-    public RecoveryPlannerService createRecoveryPlannerService(ShardSnapshotsService shardSnapshotsService) {
-        return new SnapshotsRecoveryPlannerService(shardSnapshotsService, this::isLicenseEnabled);
+    public Optional<RecoveryPlannerService> createRecoveryPlannerService(ShardSnapshotsService shardSnapshotsService) {
+        if (DiscoveryNode.isStateless(settings)) {
+            return Optional.empty();
+        }
+        return Optional.of(new SnapshotsRecoveryPlannerService(shardSnapshotsService, this::isLicenseEnabled));
     }
 
     // Overridable for tests
diff --git a/x-pack/plugin/snapshot-based-recoveries/src/main/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/plan/SnapshotsRecoveryPlannerService.java b/x-pack/plugin/snapshot-based-recoveries/src/main/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/plan/SnapshotsRecoveryPlannerService.java
index 61facb238bc9..d858958b172f 100644
--- a/x-pack/plugin/snapshot-based-recoveries/src/main/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/plan/SnapshotsRecoveryPlannerService.java
+++ b/x-pack/plugin/snapshot-based-recoveries/src/main/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/plan/SnapshotsRecoveryPlannerService.java
@@ -11,8 +11,10 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.store.StoreFileMetadata;
 import org.elasticsearch.indices.recovery.RecoverySettings;
@@ -54,6 +56,7 @@ public void computeRecoveryPlan(
         int translogOps,
         Version targetVersion,
         boolean useSnapshots,
+        boolean primaryRelocation,
         ActionListener<ShardRecoveryPlan> listener
     ) {
         // Fallback to source only recovery if the target node is in an incompatible version
@@ -67,6 +70,7 @@ public void computeRecoveryPlan(
             latestSnapshotOpt -> ActionListener.completeWith(
                 listener,
                 () -> computeRecoveryPlanWithSnapshots(
+                    shardId,
                     shardStateIdentifier,
                     sourceMetadata,
                     targetMetadata,
@@ -79,6 +83,7 @@ public void computeRecoveryPlan(
     }
 
     private ShardRecoveryPlan computeRecoveryPlanWithSnapshots(
+        ShardId shardId,
         @Nullable String shardStateIdentifier,
         Store.MetadataSnapshot sourceMetadata,
         Store.MetadataSnapshot targetMetadata,
@@ -90,6 +95,7 @@ private ShardRecoveryPlan computeRecoveryPlanWithSnapshots(
         List<StoreFileMetadata> filesMissingInTarget = concatLists(sourceTargetDiff.missing, sourceTargetDiff.different);
 
         if (latestSnapshotOpt.isEmpty()) {
+            logger.debug("{} no snapshot suitable for recovery found, falling back to recovery from the primary", shardId);
             // If we couldn't find any valid snapshots, fallback to the source
             return getRecoveryPlanUsingSourceNode(sourceMetadata, sourceTargetDiff, filesMissingInTarget, startingSeqNo, translogOps);
         }
@@ -101,6 +107,18 @@ private ShardRecoveryPlan computeRecoveryPlanWithSnapshots(
             && latestSnapshot.hasDifferentPhysicalFiles(sourceMetadata)
             && isSnapshotVersionCompatible(latestSnapshot)
             && sourceTargetDiff.identical.isEmpty()) {
+
+            logger.debug(
+                () -> Strings.format(
+                    "%s primary has changed since snapshot completed, but snapshot still looks ok for recovery: %s",
+                    shardId,
+                    latestSnapshot.getSnapshotFiles()
+                        .stream()
+                        .map(BlobStoreIndexShardSnapshot.FileInfo::physicalName)
+                        .collect(Collectors.joining(", ", "[", "]"))
+                )
+            );
+
             // Use the current primary as a fallback if the download fails half-way
             ShardRecoveryPlan fallbackPlan = getRecoveryPlanUsingSourceNode(
                 sourceMetadata,
@@ -140,6 +158,18 @@ && isSnapshotVersionCompatible(latestSnapshot)
             );
         }
 
+        logger.debug(
+            () -> Strings.format(
+                "%s attempting snapshot-based recovery of %s based on %s",
+                shardId,
+                snapshotFilesToRecover.getSnapshotFiles()
+                    .stream()
+                    .map(BlobStoreIndexShardSnapshot.FileInfo::physicalName)
+                    .collect(Collectors.joining(", ", "[", "]")),
+                snapshotDiff
+            )
+        );
+
         return new ShardRecoveryPlan(
             snapshotFilesToRecover,
             concatLists(snapshotDiff.missing, snapshotDiff.different),
@@ -182,6 +212,7 @@ private ShardRecoveryPlan getRecoveryPlanUsingSourceNode(
 
     private void fetchLatestSnapshotsIgnoringErrors(ShardId shardId, boolean useSnapshots, Consumer<Optional<ShardSnapshot>> listener) {
         if (useSnapshots == false) {
+            logger.debug("{} recovery will not attempt to use snapshots", shardId);
             listener.accept(Optional.empty());
             return;
         }
diff --git a/x-pack/plugin/snapshot-based-recoveries/src/test/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/plan/SnapshotsRecoveryPlannerServiceTests.java b/x-pack/plugin/snapshot-based-recoveries/src/test/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/plan/SnapshotsRecoveryPlannerServiceTests.java
index d0a8387a893d..3870cfb2804e 100644
--- a/x-pack/plugin/snapshot-based-recoveries/src/test/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/plan/SnapshotsRecoveryPlannerServiceTests.java
+++ b/x-pack/plugin/snapshot-based-recoveries/src/test/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/plan/SnapshotsRecoveryPlannerServiceTests.java
@@ -105,7 +105,8 @@ public void fetchLatestSnapshotsForShard(ShardId shardId, ActionListener<Optiona
                         assert false : "Unexpected call";
                     }
                 },
-                false
+                false,
+                randomBoolean()
             );
             assertPlanIsValid(shardRecoveryPlan, sourceMetadata);
             assertAllSourceFilesAreAvailableInSource(shardRecoveryPlan, sourceMetadata);
@@ -143,7 +144,8 @@ public void fetchLatestSnapshotsForShard(ShardId shardId, ActionListener<Optiona
                         }
                     }
                 },
-                true
+                true,
+                randomBoolean()
             );
 
             assertPlanIsValid(shardRecoveryPlan, sourceMetadata);
@@ -183,7 +185,8 @@ public void fetchLatestSnapshotsForShard(ShardId shardId, ActionListener<Optiona
                         listener.onResponse(Optional.of(shardSnapshotData));
                     }
                 },
-                true
+                true,
+                randomBoolean()
             );
 
             assertPlanIsValid(shardRecoveryPlan, sourceMetadata);
@@ -245,7 +248,8 @@ public void fetchLatestSnapshotsForShard(ShardId shardId, ActionListener<Optiona
                         listener.onResponse(Optional.of(latestSnapshot));
                     }
                 },
-                true
+                true,
+                randomBoolean()
             );
 
             if (shareFilesWithSource || compatibleVersion == false) {
@@ -313,7 +317,8 @@ public void fetchLatestSnapshotsForShard(ShardId shardId, ActionListener<Optiona
                         }
                     }
                 },
-                true
+                true,
+                randomBoolean()
             );
 
             assertPlanIsValid(shardRecoveryPlan, latestSourceMetadata);
@@ -365,7 +370,8 @@ public void fetchLatestSnapshotsForShard(ShardId shardId, ActionListener<Optiona
                         listener.onResponse(Optional.of(availableSnapshots.get(availableSnapshots.size() - 1)));
                     }
                 },
-                true
+                true,
+                randomBoolean()
             );
 
             assertPlanIsValid(shardRecoveryPlan, latestSourceMetadata);
@@ -403,7 +409,8 @@ public void fetchLatestSnapshotsForShard(ShardId shardId, ActionListener<Optiona
                     }
                 },
                 true,
-                Version.V_7_14_0 // Unsupported version
+                Version.V_7_14_0, // Unsupported version,
+                randomBoolean()
             );
 
             assertPlanIsValid(shardRecoveryPlan, sourceMetadata);
@@ -424,7 +431,8 @@ private ShardRecoveryPlan computeShardRecoveryPlan(
         long startingSeqNo,
         int translogOps,
         ShardSnapshotsService shardSnapshotsService,
-        boolean snapshotRecoveriesEnabled
+        boolean snapshotRecoveriesEnabled,
+        boolean primaryRelocation
     ) throws Exception {
         return computeShardRecoveryPlan(
             shardIdentifier,
@@ -434,7 +442,8 @@ private ShardRecoveryPlan computeShardRecoveryPlan(
             translogOps,
             shardSnapshotsService,
             snapshotRecoveriesEnabled,
-            Version.CURRENT
+            Version.CURRENT,
+            primaryRelocation
         );
     }
 
@@ -446,7 +455,8 @@ private ShardRecoveryPlan computeShardRecoveryPlan(
         int translogOps,
         ShardSnapshotsService shardSnapshotsService,
         boolean snapshotRecoveriesEnabled,
-        Version version
+        Version version,
+        boolean primaryRelocation
     ) throws Exception {
         SnapshotsRecoveryPlannerService recoveryPlannerService = new SnapshotsRecoveryPlannerService(shardSnapshotsService, () -> true);
 
@@ -460,6 +470,7 @@ private ShardRecoveryPlan computeShardRecoveryPlan(
             translogOps,
             version,
             snapshotRecoveriesEnabled,
+            primaryRelocation,
             planFuture
         );
         final ShardRecoveryPlan shardRecoveryPlan = planFuture.get();
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/azure/build.gradle b/x-pack/plugin/snapshot-repo-test-kit/qa/azure/build.gradle
index 887549241ab8..92fa1a662911 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/qa/azure/build.gradle
+++ b/x-pack/plugin/snapshot-repo-test-kit/qa/azure/build.gradle
@@ -8,7 +8,7 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 
 final Project fixture = project(':test:fixtures:azure-fixture')
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/build.gradle b/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/build.gradle
index 89f76a2f8e2b..31f8bb717741 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/build.gradle
+++ b/x-pack/plugin/snapshot-repo-test-kit/qa/gcs/build.gradle
@@ -13,7 +13,7 @@ import java.security.KeyPairGenerator
 
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 
 final Project fixture = project(':test:fixtures:gcs-fixture')
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/build.gradle b/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/build.gradle
index 4acc4b6415e8..a323d18afbb3 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/build.gradle
+++ b/x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/build.gradle
@@ -17,7 +17,7 @@ import java.nio.file.Paths
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 
 apply plugin: 'elasticsearch.test.fixtures'
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 apply plugin: 'elasticsearch.internal-available-ports'
 
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/minio/build.gradle b/x-pack/plugin/snapshot-repo-test-kit/qa/minio/build.gradle
index 8e10165d243c..225e0146a6ec 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/qa/minio/build.gradle
+++ b/x-pack/plugin/snapshot-repo-test-kit/qa/minio/build.gradle
@@ -7,7 +7,7 @@
 
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
 apply plugin: 'elasticsearch.rest-resources'
 
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle
index dd5a54e828cd..6b32e11ef81d 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle
+++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 
 dependencies {
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/10_analyze.yml b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/10_analyze.yml
index 6223ca8443b0..25c1ed26bcdc 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/10_analyze.yml
+++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/10_analyze.yml
@@ -158,8 +158,8 @@ setup:
 ---
 "Timeout with large blobs":
   - skip:
-      version: "- 7.13.99"
-      reason: "abortWrites flag introduced in 7.14, and mixed-cluster support not required"
+      version: all
+      reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/90353"
 
   - do:
       catch: request
diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/s3/build.gradle b/x-pack/plugin/snapshot-repo-test-kit/qa/s3/build.gradle
index a4609cd2cc2c..36b13ef8b12a 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/qa/s3/build.gradle
+++ b/x-pack/plugin/snapshot-repo-test-kit/qa/s3/build.gradle
@@ -7,7 +7,7 @@
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 
 final Project fixture = project(':test:fixtures:s3-fixture')
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java
index 38726deb2b23..65acc3b7e2f1 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java
@@ -9,7 +9,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionListenerResponseHandler;
 import org.elasticsearch.action.ActionRequest;
@@ -254,8 +254,8 @@ static class BlobAnalysis {
 
             final StepListener<Collection<NodeResponse>> readsCompleteStep = new StepListener<>();
             readNodesListener = new GroupedActionListener<>(
-                new ThreadedActionListener<>(logger, transportService.getThreadPool(), ThreadPool.Names.SNAPSHOT, readsCompleteStep, false),
-                earlyReadNodes.size() + readNodes.size()
+                earlyReadNodes.size() + readNodes.size(),
+                new ThreadedActionListener<>(transportService.getThreadPool().executor(ThreadPool.Names.SNAPSHOT), readsCompleteStep)
             );
 
             // The order is important in this chain: if writing fails then we may never even start all the reads, and we want to cancel
@@ -676,7 +676,7 @@ public static class Request extends ActionRequest implements TaskAwareRequest {
             earlyReadNodeCount = in.readVInt();
             readEarly = in.readBoolean();
             writeAndOverwrite = in.readBoolean();
-            if (in.getVersion().onOrAfter(Version.V_7_14_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_14_0)) {
                 abortWrite = in.readBoolean();
             } else {
                 abortWrite = false;
@@ -696,10 +696,10 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeVInt(earlyReadNodeCount);
             out.writeBoolean(readEarly);
             out.writeBoolean(writeAndOverwrite);
-            if (out.getVersion().onOrAfter(Version.V_7_14_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_14_0)) {
                 out.writeBoolean(abortWrite);
             } else if (abortWrite) {
-                throw new IllegalStateException("cannot send abortWrite request to node of version [" + out.getVersion() + "]");
+                throw new IllegalStateException("cannot send abortWrite request on transport version [" + out.getTransportVersion() + "]");
             }
         }
 
diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java
index 5ef79116205d..ec79367f2b57 100644
--- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java
+++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java
@@ -10,7 +10,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionListenerResponseHandler;
 import org.elasticsearch.action.ActionRequest;
@@ -20,6 +20,7 @@
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.support.RefCountingRunnable;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -32,7 +33,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
-import org.elasticsearch.common.util.concurrent.CountDown;
 import org.elasticsearch.common.xcontent.StatusToXContentObject;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.repositories.RepositoriesService;
@@ -46,9 +46,7 @@
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.ReceiveTimeoutTransportException;
-import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequestOptions;
-import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -362,8 +360,7 @@ public static class AsyncAction {
         private final Queue<VerifyBlobTask> queue = ConcurrentCollections.newQueue();
         private final AtomicReference<Exception> failure = new AtomicReference<>();
         private final Semaphore innerFailures = new Semaphore(5); // limit the number of suppressed failures
-        private final int workerCount;
-        private final CountDown workerCountdown;
+        private final RefCountingRunnable requestRefs = new RefCountingRunnable(this::runCleanUp);
         private final Set<String> expectedBlobs = ConcurrentCollections.newConcurrentSet();
         private final List<BlobAnalyzeAction.Response> responses;
         private final RepositoryPerformanceSummary.Builder summary = new RepositoryPerformanceSummary.Builder();
@@ -386,9 +383,6 @@ public AsyncAction(
             this.timeoutTimeMillis = currentTimeMillisSupplier.getAsLong() + request.getTimeout().millis();
             this.listener = listener;
 
-            this.workerCount = request.getConcurrency();
-            this.workerCountdown = new CountDown(workerCount);
-
             responses = new ArrayList<>(request.blobCount);
         }
 
@@ -440,7 +434,6 @@ private boolean isRunning() {
         public void run() {
             assert queue.isEmpty() : "must only run action once";
             assert failure.get() == null : "must only run action once";
-            assert workerCountdown.isCountedDown() == false : "must only run action once";
 
             logger.info("running analysis of repository [{}] using path [{}]", request.getRepositoryName(), blobPath);
 
@@ -477,8 +470,10 @@ && rarely(random)
                 queue.add(verifyBlobTask);
             }
 
-            for (int i = 0; i < workerCount; i++) {
-                processNextTask();
+            try (var ignored = requestRefs) {
+                for (int i = 0; i < request.getConcurrency(); i++) {
+                    processNextTask();
+                }
             }
         }
 
@@ -488,9 +483,7 @@ private boolean rarely(Random random) {
 
         private void processNextTask() {
             final VerifyBlobTask thisTask = queue.poll();
-            if (isRunning() == false || thisTask == null) {
-                onWorkerCompletion();
-            } else {
+            if (isRunning() && thisTask != null) {
                 logger.trace("processing [{}]", thisTask);
                 // NB although all this is on the SAME thread, the per-blob verification runs on a SNAPSHOT thread so we don't have to worry
                 // about local requests resulting in a stack overflow here
@@ -503,9 +496,9 @@ private void processNextTask() {
                     thisTask.request,
                     task,
                     transportRequestOptions,
-                    new TransportResponseHandler<BlobAnalyzeAction.Response>() {
+                    new ActionListenerResponseHandler<>(ActionListener.releaseAfter(new ActionListener<>() {
                         @Override
-                        public void handleResponse(BlobAnalyzeAction.Response response) {
+                        public void onResponse(BlobAnalyzeAction.Response response) {
                             logger.trace("finished [{}]", thisTask);
                             if (thisTask.request.getAbortWrite() == false) {
                                 expectedBlobs.add(thisTask.request.getBlobName()); // each task cleans up its own mess on failure
@@ -520,17 +513,11 @@ public void handleResponse(BlobAnalyzeAction.Response response) {
                         }
 
                         @Override
-                        public void handleException(TransportException exp) {
+                        public void onFailure(Exception exp) {
                             logger.debug(() -> "failed [" + thisTask + "]", exp);
                             fail(exp);
-                            onWorkerCompletion();
                         }
-
-                        @Override
-                        public BlobAnalyzeAction.Response read(StreamInput in) throws IOException {
-                            return new BlobAnalyzeAction.Response(in);
-                        }
-                    }
+                    }, requestRefs.acquire()), BlobAnalyzeAction.Response::new)
                 );
             }
 
@@ -540,16 +527,14 @@ private BlobContainer getBlobContainer() {
             return repository.blobStore().blobContainer(repository.basePath().add(blobPath));
         }
 
-        private void onWorkerCompletion() {
-            if (workerCountdown.countDown()) {
-                transportService.getThreadPool().executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.wrap(listener, l -> {
-                    final long listingStartTimeNanos = System.nanoTime();
-                    ensureConsistentListing();
-                    final long deleteStartTimeNanos = System.nanoTime();
-                    deleteContainer();
-                    sendResponse(listingStartTimeNanos, deleteStartTimeNanos);
-                }));
-            }
+        private void runCleanUp() {
+            transportService.getThreadPool().executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.wrap(listener, l -> {
+                final long listingStartTimeNanos = System.nanoTime();
+                ensureConsistentListing();
+                final long deleteStartTimeNanos = System.nanoTime();
+                deleteContainer();
+                sendResponse(listingStartTimeNanos, deleteStartTimeNanos);
+            }));
         }
 
         private void ensureConsistentListing() {
@@ -650,20 +635,7 @@ private void sendResponse(final long listingStartTimeNanos, final long deleteSta
             }
         }
 
-        private static class VerifyBlobTask {
-            final DiscoveryNode node;
-            final BlobAnalyzeAction.Request request;
-
-            VerifyBlobTask(DiscoveryNode node, BlobAnalyzeAction.Request request) {
-                this.node = node;
-                this.request = request;
-            }
-
-            @Override
-            public String toString() {
-                return "VerifyBlobTask{" + "node=" + node + ", request=" + request + '}';
-            }
-        }
+        private record VerifyBlobTask(DiscoveryNode node, BlobAnalyzeAction.Request request) {}
     }
 
     public static class Request extends ActionRequest {
@@ -701,7 +673,7 @@ public Request(StreamInput in) throws IOException {
             maxTotalDataSize = ByteSizeValue.readFrom(in);
             detailed = in.readBoolean();
             reroutedFrom = in.readOptionalWriteable(DiscoveryNode::new);
-            if (in.getVersion().onOrAfter(Version.V_7_14_0)) {
+            if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_14_0)) {
                 abortWritePermitted = in.readBoolean();
             } else {
                 abortWritePermitted = false;
@@ -728,10 +700,12 @@ public void writeTo(StreamOutput out) throws IOException {
             maxTotalDataSize.writeTo(out);
             out.writeBoolean(detailed);
             out.writeOptionalWriteable(reroutedFrom);
-            if (out.getVersion().onOrAfter(Version.V_7_14_0)) {
+            if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_14_0)) {
                 out.writeBoolean(abortWritePermitted);
             } else if (abortWritePermitted) {
-                throw new IllegalStateException("cannot send abortWritePermitted request to node of version [" + out.getVersion() + "]");
+                throw new IllegalStateException(
+                    "cannot send abortWritePermitted request on transport version [" + out.getTransportVersion() + "]"
+                );
             }
         }
 
diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoGridAggAndQueryConsistencyIT.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoGridAggAndQueryConsistencyIT.java
index 43da13e7aa58..1d0b01f463a1 100644
--- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoGridAggAndQueryConsistencyIT.java
+++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoGridAggAndQueryConsistencyIT.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.geo.GeoBoundingBox;
 import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.geo.GeometryTestUtils;
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.Point;
@@ -34,6 +33,7 @@
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin;
+import org.elasticsearch.xpack.spatial.common.H3CartesianUtil;
 import org.elasticsearch.xpack.spatial.index.mapper.GeoShapeWithDocValuesFieldMapper;
 import org.elasticsearch.xpack.spatial.index.query.GeoGridQueryBuilder;
 import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoHexGridAggregationBuilder;
@@ -88,6 +88,102 @@ public void testGeoShapeGeoTile() throws IOException {
         );
     }
 
+    public void testGeoShapeGeoHex() throws IOException {
+        doTestGeohexGrid(GeoShapeWithDocValuesFieldMapper.CONTENT_TYPE, () -> GeometryTestUtils.randomGeometryWithoutCircle(0, false));
+    }
+
+    public void testKnownIssueWithCellLeftOfDatelineTouchingPolygonOnRightOfDateline() throws IOException {
+        XContentBuilder xcb = XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("properties")
+            .startObject("geometry")
+            .field("type", "geo_shape")
+            .endObject()
+            .endObject()
+            .endObject();
+        client().admin().indices().prepareCreate("test").setMapping(xcb).get();
+
+        BulkRequestBuilder builder = client().prepareBulk();
+        builder.add(
+            new IndexRequest("test").source("{\"geometry\" : \"BBOX (179.99999, 180.0, -11.29550, -11.29552)\"}", XContentType.JSON)
+        );
+        builder.add(
+            new IndexRequest("test").source("{\"geometry\" : \"BBOX (-180.0, -179.99999, -11.29550, -11.29552)\"}", XContentType.JSON)
+        );
+
+        assertFalse(builder.get().hasFailures());
+        client().admin().indices().prepareRefresh("test").get();
+
+        GeoBoundingBox boundingBox = new GeoBoundingBox(new GeoPoint(-11.29550, 179.999992), new GeoPoint(-11.29552, -179.999992));
+
+        GeoGridAggregationBuilder builderPoint = new GeoHexGridAggregationBuilder("geometry").field("geometry")
+            .precision(15)
+            .setGeoBoundingBox(boundingBox)
+            .size(256 * 256);
+        SearchResponse response = client().prepareSearch("test").addAggregation(builderPoint).setSize(0).get();
+        InternalGeoGrid<?> gridPoint = response.getAggregations().get("geometry");
+        for (InternalGeoGridBucket bucket : gridPoint.getBuckets()) {
+            assertThat(bucket.getDocCount(), Matchers.greaterThan(0L));
+            QueryBuilder queryBuilder = new GeoGridQueryBuilder("geometry").setGridId(
+                GeoGridQueryBuilder.Grid.GEOHEX,
+                bucket.getKeyAsString()
+            );
+            response = client().prepareSearch("test").setTrackTotalHits(true).setQuery(queryBuilder).get();
+            assertThat(
+                "Bucket " + bucket.getKeyAsString(),
+                response.getHits().getTotalHits().value,
+                Matchers.equalTo(bucket.getDocCount())
+            );
+        }
+    }
+
+    public void testKnownIssueWithCellIntersectingPolygonAndBoundingBox() throws IOException {
+        XContentBuilder xcb = XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("properties")
+            .startObject("geometry")
+            .field("type", "geo_shape")
+            .endObject()
+            .endObject()
+            .endObject();
+        client().admin().indices().prepareCreate("test").setMapping(xcb).get();
+
+        BulkRequestBuilder builder = client().prepareBulk();
+        builder.add(
+            new IndexRequest("test").source("{\"geometry\" : \"POINT (169.12088680200193 86.17678739494652)\"}", XContentType.JSON)
+        );
+        builder.add(
+            new IndexRequest("test").source("{\"geometry\" : \"POINT (169.12088680200193 86.17678739494652)\"}", XContentType.JSON)
+        );
+        String mp = "POLYGON ((150.0 70.0, 150.0 85.91811374669217, 168.77544806565834 85.91811374669217, 150.0 70.0))";
+        builder.add(new IndexRequest("test").source("{\"geometry\" : \"" + mp + "\"}", XContentType.JSON));
+
+        assertFalse(builder.get().hasFailures());
+        client().admin().indices().prepareRefresh("test").get();
+
+        // BBOX (172.21916569181505, -173.17785081207947, 86.17678739494652, 83.01600086049713)
+        GeoBoundingBox boundingBox = new GeoBoundingBox(
+            new GeoPoint(86.17678739494652, 172.21916569181505),
+            new GeoPoint(83.01600086049713, 179)
+        );
+        int precision = 4;
+        GeoGridAggregationBuilder builderPoint = new GeoHexGridAggregationBuilder("geometry").field("geometry")
+            .precision(precision)
+            .setGeoBoundingBox(boundingBox)
+            .size(256 * 256);
+        SearchResponse response = client().prepareSearch("test").addAggregation(builderPoint).setSize(0).get();
+        InternalGeoGrid<?> gridPoint = response.getAggregations().get("geometry");
+        for (InternalGeoGridBucket bucket : gridPoint.getBuckets()) {
+            assertThat(bucket.getDocCount(), Matchers.greaterThan(0L));
+            QueryBuilder queryBuilder = new GeoGridQueryBuilder("geometry").setGridId(
+                GeoGridQueryBuilder.Grid.GEOHEX,
+                bucket.getKeyAsString()
+            );
+            response = client().prepareSearch("test").setTrackTotalHits(true).setQuery(queryBuilder).get();
+            assertThat(response.getHits().getTotalHits().value, Matchers.equalTo(bucket.getDocCount()));
+        }
+    }
+
     private void doTestGeohashGrid(String fieldType, Supplier<Geometry> randomGeometriesSupplier) throws IOException {
         doTestGrid(
             1,
@@ -125,7 +221,7 @@ private void doTestGeohexGrid(String fieldType, Supplier<Geometry> randomGeometr
             }
             return points;
         },
-            h3 -> new Rectangle(GeoUtils.MIN_LON, GeoUtils.MAX_LON, GeoUtils.MAX_LAT, GeoUtils.MAX_LAT),
+            h3 -> H3CartesianUtil.toBoundingBox(H3.stringToH3(h3)),
             GeoHexGridAggregationBuilder::new,
             (s1, s2) -> new GeoGridQueryBuilder(s1).setGridId(GeoGridQueryBuilder.Grid.GEOHEX, s2),
             randomGeometriesSupplier
@@ -218,7 +314,7 @@ private void assertQuery(List<InternalGeoGridBucket> buckets, BiFunction<String,
             QueryBuilder queryBuilder = queryFunction.apply("geometry", bucket.getKeyAsString());
             SearchResponse response = client().prepareSearch("test").setTrackTotalHits(true).setQuery(queryBuilder).get();
             assertThat(
-                "Expected hits at precision " + precision,
+                "Expected hits at precision " + precision + " for H3 cell " + bucket.getKeyAsString(),
                 response.getHits().getTotalHits().value,
                 Matchers.equalTo(bucket.getDocCount())
             );
diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverShapeTests.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverShapeTests.java
index 0bcd4723effd..dbefc82702ce 100644
--- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverShapeTests.java
+++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverShapeTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.geo.GeoJson;
 import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.GeometryCollection;
 import org.elasticsearch.geometry.MultiPoint;
@@ -136,7 +137,7 @@ public void testShapeFetchingPath() throws Exception {
         String location = """
             "location" : {"type":"polygon", "coordinates":[[[-10,-10],[10,-10],[10,10],[-10,10],[-10,-10]]]}""";
 
-        client().prepareIndex(indexName).setId("1").setSource(formatted("""
+        client().prepareIndex(indexName).setId("1").setSource(Strings.format("""
             { %s, "1" : { %s, "2" : { %s, "3" : { %s } }} }
             """, location, location, location, location), XContentType.JSON).setRefreshPolicy(IMMEDIATE).get();
         client().prepareIndex(searchIndex)
@@ -230,13 +231,14 @@ public void testIgnoreMalformed() {
      * Test that the indexed shape routing can be provided if it is required
      */
     public void testIndexShapeRouting() {
-        String source = formatted("""
+        Object[] args = new Object[] { -Float.MAX_VALUE, Float.MAX_VALUE, Float.MAX_VALUE, -Float.MAX_VALUE };
+        String source = Strings.format("""
             {
                 "shape" : {
                     "type" : "bbox",
                     "coordinates" : [[%s,%s], [%s, %s]]
                 }
-            }""", -Float.MAX_VALUE, Float.MAX_VALUE, Float.MAX_VALUE, -Float.MAX_VALUE);
+            }""", args);
 
         client().prepareIndex(INDEX).setId("0").setSource(source, XContentType.JSON).setRouting("ABC").get();
         client().admin().indices().prepareRefresh(INDEX).get();
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java
index f01b7ab366d7..adedcbc3294a 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java
@@ -24,7 +24,9 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregator;
 import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregator;
 import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.CardinalityAggregator;
 import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregationBuilder;
@@ -48,20 +50,18 @@
 import org.elasticsearch.xpack.spatial.index.query.GeoGridQueryBuilder;
 import org.elasticsearch.xpack.spatial.index.query.ShapeQueryBuilder;
 import org.elasticsearch.xpack.spatial.ingest.CircleProcessor;
+import org.elasticsearch.xpack.spatial.ingest.GeoGridProcessor;
 import org.elasticsearch.xpack.spatial.search.aggregations.GeoLineAggregationBuilder;
 import org.elasticsearch.xpack.spatial.search.aggregations.InternalGeoLine;
-import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.BoundedGeoHashGridTiler;
-import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.BoundedGeoTileGridTiler;
 import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoGridTiler;
+import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoHashGridTiler;
 import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoHexCellIdSource;
 import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoHexGridAggregationBuilder;
 import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoHexGridAggregator;
+import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoHexGridTiler;
 import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoShapeCellIdSource;
-import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoShapeHashGridAggregator;
-import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoShapeTileGridAggregator;
+import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoTileGridTiler;
 import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.InternalGeoHexGrid;
-import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.UnboundedGeoHashGridTiler;
-import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.UnboundedGeoTileGridTiler;
 import org.elasticsearch.xpack.spatial.search.aggregations.metrics.CartesianBoundsAggregationBuilder;
 import org.elasticsearch.xpack.spatial.search.aggregations.metrics.CartesianBoundsAggregator;
 import org.elasticsearch.xpack.spatial.search.aggregations.metrics.CartesianCentroidAggregationBuilder;
@@ -189,7 +189,7 @@ public List<AggregationSpec> getAggregations() {
 
     @Override
     public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
-        return Map.of(CircleProcessor.TYPE, new CircleProcessor.Factory());
+        return Map.of(CircleProcessor.TYPE, new CircleProcessor.Factory(), GeoGridProcessor.TYPE, new GeoGridProcessor.Factory());
     }
 
     private static void registerGeoShapeBoundsAggregator(ValuesSourceRegistry.Builder builder) {
@@ -269,15 +269,10 @@ private void registerGeoHexGridAggregator(ValuesSourceRegistry.Builder builder)
                 cardinality,
                 metadata) -> {
                 if (GEO_HEX_AGG_FEATURE.check(getLicenseState())) {
-                    GeoHexCellIdSource cellIdSource = new GeoHexCellIdSource(
-                        (ValuesSource.GeoPoint) valuesSource,
-                        precision,
-                        geoBoundingBox
-                    );
                     return new GeoHexGridAggregator(
                         name,
                         factories,
-                        cellIdSource,
+                        cb -> new GeoHexCellIdSource((ValuesSource.GeoPoint) valuesSource, precision, geoBoundingBox, cb),
                         requiredSize,
                         shardSize,
                         aggregationContext,
@@ -310,17 +305,11 @@ private void registerGeoShapeGridAggregators(ValuesSourceRegistry.Builder builde
                 collectsFromSingleBucket,
                 metadata) -> {
                 if (GEO_GRID_AGG_FEATURE.check(getLicenseState())) {
-                    final GeoGridTiler tiler;
-                    if (geoBoundingBox.isUnbounded()) {
-                        tiler = new UnboundedGeoHashGridTiler(precision);
-                    } else {
-                        tiler = new BoundedGeoHashGridTiler(precision, geoBoundingBox);
-                    }
-                    GeoShapeCellIdSource cellIdSource = new GeoShapeCellIdSource((GeoShapeValuesSource) valuesSource, tiler);
-                    GeoShapeHashGridAggregator agg = new GeoShapeHashGridAggregator(
+                    final GeoGridTiler tiler = GeoHashGridTiler.makeGridTiler(precision, geoBoundingBox);
+                    return new GeoHashGridAggregator(
                         name,
                         factories,
-                        cellIdSource,
+                        cb -> new GeoShapeCellIdSource((GeoShapeValuesSource) valuesSource, tiler, cb),
                         requiredSize,
                         shardSize,
                         aggregationContext,
@@ -328,9 +317,6 @@ private void registerGeoShapeGridAggregators(ValuesSourceRegistry.Builder builde
                         collectsFromSingleBucket,
                         metadata
                     );
-                    // this would ideally be something set in an immutable way on the ValuesSource
-                    cellIdSource.setCircuitBreakerConsumer(agg::addRequestBytes);
-                    return agg;
                 }
                 throw LicenseUtils.newComplianceException("geohash_grid aggregation on geo_shape fields");
             },
@@ -353,17 +339,11 @@ private void registerGeoShapeGridAggregators(ValuesSourceRegistry.Builder builde
                 collectsFromSingleBucket,
                 metadata) -> {
                 if (GEO_GRID_AGG_FEATURE.check(getLicenseState())) {
-                    final GeoGridTiler tiler;
-                    if (geoBoundingBox.isUnbounded()) {
-                        tiler = new UnboundedGeoTileGridTiler(precision);
-                    } else {
-                        tiler = new BoundedGeoTileGridTiler(precision, geoBoundingBox);
-                    }
-                    GeoShapeCellIdSource cellIdSource = new GeoShapeCellIdSource((GeoShapeValuesSource) valuesSource, tiler);
-                    GeoShapeTileGridAggregator agg = new GeoShapeTileGridAggregator(
+                    final GeoGridTiler tiler = GeoTileGridTiler.makeGridTiler(precision, geoBoundingBox);
+                    return new GeoTileGridAggregator(
                         name,
                         factories,
-                        cellIdSource,
+                        cb -> new GeoShapeCellIdSource((GeoShapeValuesSource) valuesSource, tiler, cb),
                         requiredSize,
                         shardSize,
                         context,
@@ -371,14 +351,45 @@ private void registerGeoShapeGridAggregators(ValuesSourceRegistry.Builder builde
                         collectsFromSingleBucket,
                         metadata
                     );
-                    // this would ideally be something set in an immutable way on the ValuesSource
-                    cellIdSource.setCircuitBreakerConsumer(agg::addRequestBytes);
-                    return agg;
                 }
                 throw LicenseUtils.newComplianceException("geotile_grid aggregation on geo_shape fields");
             },
             true
         );
+
+        builder.register(
+            GeoHexGridAggregationBuilder.REGISTRY_KEY,
+            GeoShapeValuesSourceType.instance(),
+            (
+                name,
+                factories,
+                valuesSource,
+                precision,
+                geoBoundingBox,
+                requiredSize,
+                shardSize,
+                context,
+                parent,
+                collectsFromSingleBucket,
+                metadata) -> {
+                if (GEO_GRID_AGG_FEATURE.check(getLicenseState())) {
+                    final GeoGridTiler tiler = GeoHexGridTiler.makeGridTiler(precision, geoBoundingBox);
+                    return new GeoHexGridAggregator(
+                        name,
+                        factories,
+                        cb -> new GeoShapeCellIdSource((GeoShapeValuesSource) valuesSource, tiler, cb),
+                        requiredSize,
+                        shardSize,
+                        context,
+                        parent,
+                        collectsFromSingleBucket,
+                        metadata
+                    );
+                }
+                throw LicenseUtils.newComplianceException("geohex_grid aggregation on geo_shape fields");
+            },
+            true
+        );
     }
 
     private static void registerValueCountAggregator(ValuesSourceRegistry.Builder builder) {
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/common/H3CartesianGeometry.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/common/H3CartesianGeometry.java
new file mode 100644
index 000000000000..8b1e993ef88d
--- /dev/null
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/common/H3CartesianGeometry.java
@@ -0,0 +1,356 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.spatial.common;
+
+import org.apache.lucene.geo.Component2D;
+import org.apache.lucene.geo.LatLonGeometry;
+import org.apache.lucene.geo.Rectangle;
+import org.apache.lucene.index.PointValues;
+import org.apache.lucene.util.ArrayUtil;
+import org.elasticsearch.xpack.spatial.index.fielddata.GeoRelation;
+
+/**
+ * Lucene geometry representing an H3 bin on the cartesian space.
+ */
+class H3CartesianGeometry extends LatLonGeometry {
+
+    private final long h3;
+
+    H3CartesianGeometry(long h3) {
+        this.h3 = h3;
+    }
+
+    @Override
+    protected Component2D toComponent2D() {
+        return new H3CartesianComponent(h3);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        H3CartesianGeometry that = (H3CartesianGeometry) o;
+        return h3 == that.h3;
+    }
+
+    @Override
+    public int hashCode() {
+        return Long.hashCode(h3);
+    }
+
+    private static class H3CartesianComponent implements Component2D {
+
+        private final double[] xs, ys;
+        private final double minX, maxX, minY, maxY;
+        private final boolean crossesDateline;
+
+        H3CartesianComponent(long h3) {
+            final double[] xs = new double[H3CartesianUtil.MAX_ARRAY_SIZE];
+            final double[] ys = new double[H3CartesianUtil.MAX_ARRAY_SIZE];
+            final int numPoints = H3CartesianUtil.computePoints(h3, xs, ys);
+            this.xs = ArrayUtil.copyOfSubArray(xs, 0, numPoints);
+            this.ys = ArrayUtil.copyOfSubArray(ys, 0, numPoints);
+            double minX = Double.POSITIVE_INFINITY;
+            double maxX = Double.NEGATIVE_INFINITY;
+            double minY = Double.POSITIVE_INFINITY;
+            double maxY = Double.NEGATIVE_INFINITY;
+            for (int i = 0; i < numPoints; i++) {
+                minX = Math.min(minX, xs[i]);
+                maxX = Math.max(maxX, xs[i]);
+                minY = Math.min(minY, ys[i]);
+                maxY = Math.max(maxY, ys[i]);
+            }
+            this.minX = minX;
+            this.maxX = maxX;
+            this.minY = minY;
+            this.maxY = maxY;
+            this.crossesDateline = maxX - minX > 180d && H3CartesianUtil.isPolar(h3) == false;
+        }
+
+        @Override
+        public double getMinX() {
+            return crossesDateline ? -180d : minX;
+        }
+
+        @Override
+        public double getMaxX() {
+            return crossesDateline ? 180d : maxX;
+        }
+
+        @Override
+        public double getMinY() {
+            return minY;
+        }
+
+        @Override
+        public double getMaxY() {
+            return maxY;
+        }
+
+        @Override
+        public boolean contains(double x, double y) {
+            // fail fast if we're outside the bounding box
+            if (Rectangle.containsPoint(y, x, getMinY(), getMaxY(), getMinX(), getMaxX()) == false) {
+                return false;
+            }
+            return H3CartesianUtil.relatePoint(xs, ys, xs.length, crossesDateline, x, y) != GeoRelation.QUERY_DISJOINT;
+        }
+
+        @Override
+        public PointValues.Relation relate(double minX, double maxX, double minY, double maxY) {
+            if (Component2D.disjoint(getMinX(), getMaxX(), getMinY(), getMaxY(), minX, maxX, minY, maxY)) {
+                return PointValues.Relation.CELL_OUTSIDE_QUERY;
+            }
+            if (Component2D.within(getMinX(), getMaxX(), getMinY(), getMaxY(), minX, maxX, minY, maxY)) {
+                return PointValues.Relation.CELL_CROSSES_QUERY;
+            }
+            ContainsCorners containsCorners = containsCorners(minX, maxX, minY, maxY);
+            if (containsCorners == ContainsCorners.ALL) {
+                if (H3CartesianUtil.crossesBox(xs, ys, xs.length, crossesDateline, minX, maxX, minY, maxY, true)) {
+                    return PointValues.Relation.CELL_CROSSES_QUERY;
+                }
+                return PointValues.Relation.CELL_INSIDE_QUERY;
+            } else if (containsCorners == ContainsCorners.NONE) {
+                if (Component2D.containsPoint(xs[0], ys[0], minX, maxX, minY, maxY)) {
+                    return PointValues.Relation.CELL_CROSSES_QUERY;
+                }
+                if (H3CartesianUtil.crossesBox(xs, ys, xs.length, crossesDateline, minX, maxX, minY, maxY, true)) {
+                    return PointValues.Relation.CELL_CROSSES_QUERY;
+                }
+                return PointValues.Relation.CELL_OUTSIDE_QUERY;
+            }
+            return PointValues.Relation.CELL_CROSSES_QUERY;
+        }
+
+        @Override
+        public boolean intersectsLine(double minX, double maxX, double minY, double maxY, double aX, double aY, double bX, double bY) {
+            return contains(aX, aY) || contains(bX, bY) || crossesLine(minX, maxX, minY, maxY, aX, aY, bX, bY, true);
+        }
+
+        @Override
+        public boolean intersectsTriangle(
+            double minX,
+            double maxX,
+            double minY,
+            double maxY,
+            double aX,
+            double aY,
+            double bX,
+            double bY,
+            double cX,
+            double cY
+        ) {
+            if (Component2D.disjoint(getMinX(), getMaxX(), getMinY(), getMaxY(), minX, maxX, minY, maxY)) {
+                return false;
+            }
+            return contains(aX, aY)
+                || contains(bX, bY)
+                || contains(cX, cY)
+                || Component2D.pointInTriangle(minX, maxX, minY, maxY, xs[0], ys[0], aX, aY, bX, bY, cX, cY)
+                || H3CartesianUtil.crossesTriangle(
+                    xs,
+                    ys,
+                    xs.length,
+                    crossesDateline,
+                    minX,
+                    maxX,
+                    minY,
+                    maxY,
+                    aX,
+                    aY,
+                    bX,
+                    bY,
+                    cX,
+                    cY,
+                    true
+                );
+        }
+
+        @Override
+        public boolean containsLine(double minX, double maxX, double minY, double maxY, double aX, double aY, double bX, double bY) {
+            return contains(aX, aY) && contains(bX, bY) && crossesLine(minX, maxX, minY, maxY, aX, aY, bX, bY, false) == false;
+        }
+
+        @Override
+        public boolean containsTriangle(
+            double minX,
+            double maxX,
+            double minY,
+            double maxY,
+            double aX,
+            double aY,
+            double bX,
+            double bY,
+            double cX,
+            double cY
+        ) {
+            return (contains(aX, aY)
+                && contains(bX, bY)
+                && contains(cX, cY)
+                && H3CartesianUtil.crossesTriangle(
+                    xs,
+                    ys,
+                    xs.length,
+                    crossesDateline,
+                    minX,
+                    maxX,
+                    minY,
+                    maxY,
+                    aX,
+                    aY,
+                    bX,
+                    bY,
+                    cX,
+                    cY,
+                    false
+                ) == false);
+
+        }
+
+        @Override
+        public WithinRelation withinPoint(double x, double y) {
+            return contains(x, y) ? WithinRelation.NOTWITHIN : WithinRelation.DISJOINT;
+        }
+
+        @Override
+        public WithinRelation withinLine(
+            double minX,
+            double maxX,
+            double minY,
+            double maxY,
+            double aX,
+            double aY,
+            boolean ab,
+            double bX,
+            double bY
+        ) {
+            if (Component2D.disjoint(getMinX(), getMaxX(), getMinY(), getMaxY(), minX, maxX, minY, maxY)) {
+                return WithinRelation.DISJOINT;
+            }
+            if (contains(aX, aY) || contains(bX, bY)) {
+                return WithinRelation.NOTWITHIN;
+            }
+            if (ab && crossesLine(minX, maxX, minY, maxY, aX, aY, bX, bY, true)) {
+                return WithinRelation.NOTWITHIN;
+            }
+            return WithinRelation.DISJOINT;
+        }
+
+        @Override
+        public WithinRelation withinTriangle(
+            double minX,
+            double maxX,
+            double minY,
+            double maxY,
+            double aX,
+            double aY,
+            boolean ab,
+            double bX,
+            double bY,
+            boolean bc,
+            double cX,
+            double cY,
+            boolean ca
+        ) {
+            if (Component2D.disjoint(getMinX(), getMaxX(), getMinY(), getMaxY(), minX, maxX, minY, maxY)) {
+                return WithinRelation.DISJOINT;
+            }
+
+            // if any of the points is inside the polygon, the polygon cannot be within this indexed
+            // shape because points belong to the original indexed shape.
+            if (contains(aX, aY) || contains(bX, bY) || contains(cX, cY)) {
+                return WithinRelation.NOTWITHIN;
+            }
+
+            WithinRelation relation = WithinRelation.DISJOINT;
+            // if any of the edges intersects and the edge belongs to the shape then it cannot be within.
+            // if it only intersects edges that do not belong to the shape, then it is a candidate
+            // we skip edges at the dateline to support shapes crossing it
+            if (crossesLine(minX, maxX, minY, maxY, aX, aY, bX, bY, true)) {
+                if (ab) {
+                    return WithinRelation.NOTWITHIN;
+                } else {
+                    relation = WithinRelation.CANDIDATE;
+                }
+            }
+
+            if (crossesLine(minX, maxX, minY, maxY, bX, bY, cX, cY, true)) {
+                if (bc) {
+                    return WithinRelation.NOTWITHIN;
+                } else {
+                    relation = WithinRelation.CANDIDATE;
+                }
+            }
+            if (crossesLine(minX, maxX, minY, maxY, cX, cY, aX, aY, true)) {
+                if (ca) {
+                    return WithinRelation.NOTWITHIN;
+                } else {
+                    relation = WithinRelation.CANDIDATE;
+                }
+            }
+
+            // if any of the edges crosses and edge that does not belong to the shape
+            // then it is a candidate for within
+            if (relation == WithinRelation.CANDIDATE) {
+                return WithinRelation.CANDIDATE;
+            }
+
+            // Check if shape is within the triangle
+            if (Component2D.pointInTriangle(minX, maxX, minY, maxY, xs[0], ys[0], aX, aY, bX, bY, cX, cY)) {
+                return WithinRelation.CANDIDATE;
+            }
+            return relation;
+        }
+
+        private boolean crossesLine(
+            double minX,
+            double maxX,
+            double minY,
+            double maxY,
+            double aX,
+            double aY,
+            double bX,
+            double bY,
+            boolean includeBoundary
+        ) {
+            if (Component2D.disjoint(getMinX(), getMaxX(), getMinY(), getMaxY(), minX, maxX, minY, maxY)) {
+                return false;
+            }
+            return H3CartesianUtil.crossesLine(xs, ys, xs.length, crossesDateline, minX, maxX, minY, maxY, aX, aY, bX, bY, includeBoundary);
+        }
+
+        private ContainsCorners containsCorners(double minX, double maxX, double minY, double maxY) {
+            int containsCount = 0;
+            if (contains(minX, minY)) {
+                containsCount++;
+            }
+            if (contains(maxX, minY)) {
+                containsCount++;
+            }
+            if (containsCount == 1) {
+                return ContainsCorners.SOME;
+            }
+            if (contains(maxX, maxY)) {
+                containsCount++;
+            }
+            if (containsCount == 2) {
+                return ContainsCorners.SOME;
+            }
+            if (contains(minX, maxY)) {
+                containsCount++;
+            }
+            return containsCount == 0 ? ContainsCorners.NONE : containsCount == 4 ? ContainsCorners.ALL : ContainsCorners.SOME;
+        }
+
+        private enum ContainsCorners {
+            NONE,
+            SOME,
+            ALL
+        }
+    }
+}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/common/H3CartesianUtil.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/common/H3CartesianUtil.java
new file mode 100644
index 000000000000..d44e64e4ed72
--- /dev/null
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/common/H3CartesianUtil.java
@@ -0,0 +1,508 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.spatial.common;
+
+import org.apache.lucene.geo.GeoEncodingUtils;
+import org.apache.lucene.geo.GeoUtils;
+import org.apache.lucene.geo.LatLonGeometry;
+import org.apache.lucene.geo.Rectangle;
+import org.apache.lucene.util.IntroSorter;
+import org.elasticsearch.common.geo.GeometryNormalizer;
+import org.elasticsearch.common.geo.Orientation;
+import org.elasticsearch.common.util.ArrayUtils;
+import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.geometry.LinearRing;
+import org.elasticsearch.geometry.MultiPolygon;
+import org.elasticsearch.geometry.Polygon;
+import org.elasticsearch.h3.CellBoundary;
+import org.elasticsearch.h3.H3;
+import org.elasticsearch.h3.LatLng;
+import org.elasticsearch.xpack.spatial.index.fielddata.GeoRelation;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.DoubleUnaryOperator;
+
+import static org.apache.lucene.geo.GeoUtils.lineCrossesLine;
+import static org.apache.lucene.geo.GeoUtils.lineCrossesLineWithBoundary;
+
+/**
+ * Utility class that generates H3 bins coordinates projected on the cartesian plane (equirectangular projection).
+ * Provides spatial methods to compute spatial intersections on those coordinates.
+ */
+public final class H3CartesianUtil {
+    public static final int MAX_ARRAY_SIZE = 15;
+    private static final DoubleUnaryOperator NORMALIZE_LONG_POS = lon -> lon < 0 ? lon + 360d : lon;
+    private static final DoubleUnaryOperator NORMALIZE_LONG_NEG = lon -> lon > 0 ? lon - 360d : lon;
+
+    // we cache the first two levels and polar polygons
+    private static final Map<Long, double[][]> CACHED_H3 = new HashMap<>();
+    static {
+        for (long res0Cell : H3.getLongRes0Cells()) {
+            CACHED_H3.put(res0Cell, getCoordinates(res0Cell));
+            for (long h3 : H3.h3ToChildren(res0Cell)) {
+                CACHED_H3.put(h3, getCoordinates(h3));
+            }
+        }
+        for (int res = 2; res <= H3.MAX_H3_RES; res++) {
+            CACHED_H3.put(H3.northPolarH3(res), getCoordinates(H3.northPolarH3(res)));
+            CACHED_H3.put(H3.southPolarH3(res), getCoordinates(H3.southPolarH3(res)));
+        }
+    }
+
+    private static final double[] NORTH_BOUND = new double[H3.MAX_H3_RES + 1];
+    private static final double[] SOUTH_BOUND = new double[H3.MAX_H3_RES + 1];
+    static {
+        for (int res = 0; res <= H3.MAX_H3_RES; res++) {
+            NORTH_BOUND[res] = toBoundingBox(H3.northPolarH3(res)).getMinY();
+            SOUTH_BOUND[res] = toBoundingBox(H3.southPolarH3(res)).getMaxY();
+        }
+    }
+
+    /** For the given resolution, it returns true if the cell contains any of the poles */
+    public static boolean isPolar(long h3) {
+        final int res = H3.getResolution(h3);
+        return H3.southPolarH3(res) == h3 || H3.northPolarH3(res) == h3;
+    }
+
+    /** For the given resolution, it returns the maximum latitude of the h3 bin containing the south pole */
+    public static double getSouthPolarBound(int resolution) {
+        return SOUTH_BOUND[resolution];
+    }
+
+    /** For the given resolution, it returns the minimum latitude of the h3 bin containing the north pole */
+    public static double getNorthPolarBound(int resolution) {
+        return NORTH_BOUND[resolution];
+    }
+
+    private static double[][] getCoordinates(final long h3) {
+        final CellBoundary boundary = H3.h3ToGeoBoundary(h3);
+        final int numPoint = numPoints(h3, boundary);
+        final double[] xs = new double[numPoint];
+        final double[] ys = new double[numPoint];
+        computePoints(h3, boundary, xs, ys);
+        return new double[][] { xs, ys };
+    }
+
+    /** It stores the points for the given h3 in the provided arrays. The arrays
+     * should be at least have the length of {@link #MAX_ARRAY_SIZE}. It returns the number of point added. */
+    public static int computePoints(final long h3, final double[] xs, final double[] ys) {
+        final double[][] cached = CACHED_H3.get(h3);
+        if (cached != null) {
+            System.arraycopy(cached[0], 0, xs, 0, cached[0].length);
+            System.arraycopy(cached[1], 0, ys, 0, cached[0].length);
+            return cached[0].length;
+        }
+        return computePoints(h3, H3.h3ToGeoBoundary(h3), xs, ys);
+    }
+
+    private static int numPoints(long h3, CellBoundary cellBoundary) {
+        final int res = H3.getResolution(h3);
+        if (h3 != H3.northPolarH3(res) && h3 != H3.southPolarH3(res)) {
+            return cellBoundary.numPoints() + 1;
+        } else {
+            return cellBoundary.numPoints() + 5;
+        }
+    }
+
+    private static int computePoints(final long h3, final CellBoundary cellBoundary, final double[] xs, final double[] ys) {
+        for (int i = 0; i < cellBoundary.numPoints(); i++) {
+            final LatLng latLng = cellBoundary.getLatLon(i);
+            xs[i] = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(latLng.getLonDeg()));
+            ys[i] = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(latLng.getLatDeg()));
+        }
+        final int numPoints = numPoints(h3, cellBoundary);
+        final int res = H3.getResolution(h3);
+        if (H3.northPolarH3(res) == h3) {
+            closePolarComponent(xs, ys, cellBoundary.numPoints(), GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(90d)));
+        } else if (H3.southPolarH3(res) == h3) {
+            closePolarComponent(xs, ys, cellBoundary.numPoints(), GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(-90d)));
+            // reverse points as we are building closing the points in reverse order
+            ArrayUtils.reverseSubArray(xs, 0, numPoints);
+            ArrayUtils.reverseSubArray(ys, 0, numPoints);
+        } else {
+            xs[cellBoundary.numPoints()] = xs[0];
+            ys[cellBoundary.numPoints()] = ys[0];
+        }
+        return numPoints;
+    }
+
+    private static void closePolarComponent(double[] xs, double[] ys, int numBoundaryPoints, double pole) {
+        sort(xs, ys, numBoundaryPoints);
+        assert xs[0] > 0 != xs[numBoundaryPoints - 1] > 0 : "expected first and last element with different sign";
+        final double y = datelineIntersectionLatitude(xs[0], ys[0], xs[numBoundaryPoints - 1], ys[numBoundaryPoints - 1]);
+        xs[numBoundaryPoints] = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.MAX_LON_ENCODED);
+        ys[numBoundaryPoints] = y;
+        xs[numBoundaryPoints + 1] = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.MAX_LON_ENCODED);
+        ys[numBoundaryPoints + 1] = pole;
+        xs[numBoundaryPoints + 2] = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.MIN_LON_ENCODED);
+        ys[numBoundaryPoints + 2] = pole;
+        xs[numBoundaryPoints + 3] = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.MIN_LON_ENCODED);
+        ys[numBoundaryPoints + 3] = y;
+        xs[numBoundaryPoints + 4] = xs[0];
+        ys[numBoundaryPoints + 4] = ys[0];
+    }
+
+    private static void sort(double[] xs, double[] ys, int length) {
+        new IntroSorter() {
+            int pivotPos = -1;
+
+            @Override
+            protected void swap(int i, int j) {
+                double tmp = xs[i];
+                xs[i] = xs[j];
+                xs[j] = tmp;
+                tmp = ys[i];
+                ys[i] = ys[j];
+                ys[j] = tmp;
+            }
+
+            @Override
+            protected void setPivot(int i) {
+                pivotPos = i;
+            }
+
+            @Override
+            protected int comparePivot(int j) {
+                // all xs are different
+                return Double.compare(xs[pivotPos], xs[j]);
+            }
+        }.sort(0, length);
+    }
+
+    private static double datelineIntersectionLatitude(double x1, double y1, double x2, double y2) {
+        final double t = (180d - NORMALIZE_LONG_POS.applyAsDouble(x1)) / (NORMALIZE_LONG_POS.applyAsDouble(x2) - NORMALIZE_LONG_POS
+            .applyAsDouble(x1));
+        assert t > 0 && t <= 1;
+        return y1 + t * (y2 - y1);
+    }
+
+    /** Return the {@link LatLonGeometry} representing the provided H3 bin */
+    public static LatLonGeometry getLatLonGeometry(long h3) {
+        return new H3CartesianGeometry(h3);
+    }
+
+    /** Return the {@link Geometry} representing the provided H3 bin */
+    public static Geometry getNormalizeGeometry(long h3) {
+        final double[][] cached = CACHED_H3.get(h3);
+        final double[] xs;
+        final double[] ys;
+        if (cached != null) {
+            xs = cached[0].clone();
+            ys = cached[1].clone();
+        } else {
+            final CellBoundary boundary = H3.h3ToGeoBoundary(h3);
+            final int numPoints = numPoints(h3, boundary);
+            xs = new double[numPoints];
+            ys = new double[numPoints];
+            computePoints(h3, boundary, xs, ys);
+        }
+        final Polygon polygon = new Polygon(new LinearRing(xs, ys));
+        if (isPolar(h3) || GeometryNormalizer.needsNormalize(Orientation.CCW, polygon) == false) {
+            return polygon;
+        }
+        final Geometry geometry = GeometryNormalizer.apply(Orientation.CCW, polygon);
+        if (geometry instanceof MultiPolygon) {
+            return geometry;
+        }
+        // we shouldn't be here but one of the polygons crossing the dateline fails
+        // to normalise, so we need to normalise it this way
+        return GeometryNormalizer.apply(Orientation.CW, polygon);
+    }
+
+    /** Return the bounding box of the provided H3 bin */
+    public static org.elasticsearch.geometry.Rectangle toBoundingBox(long h3) {
+        final CellBoundary boundary = H3.h3ToGeoBoundary(h3);
+        double minLat = Double.POSITIVE_INFINITY;
+        double minLon = Double.POSITIVE_INFINITY;
+        double maxLat = Double.NEGATIVE_INFINITY;
+        double maxLon = Double.NEGATIVE_INFINITY;
+        double maxNegLon = Double.NEGATIVE_INFINITY;
+        double minPosLon = Double.POSITIVE_INFINITY;
+        for (int i = 0; i < boundary.numPoints(); i++) {
+            final double lon = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(boundary.getLatLon(i).getLonDeg()));
+            final double lat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(boundary.getLatLon(i).getLatDeg()));
+            minLat = Math.min(minLat, lat);
+            minLon = Math.min(minLon, lon);
+            maxLat = Math.max(maxLat, lat);
+            maxLon = Math.max(maxLon, lon);
+            if (lon < 0) {
+                maxNegLon = Math.max(maxNegLon, lon);
+            } else {
+                minPosLon = Math.min(minPosLon, lon);
+            }
+        }
+        final int res = H3.getResolution(h3);
+        if (h3 == H3.northPolarH3(res)) {
+            return new org.elasticsearch.geometry.Rectangle(-180d, 180d, 90d, minLat);
+        } else if (h3 == H3.southPolarH3(res)) {
+            return new org.elasticsearch.geometry.Rectangle(-180d, 180d, maxLat, -90d);
+        } else if (maxLon - minLon > 180d) {
+            return new org.elasticsearch.geometry.Rectangle(minPosLon, maxNegLon, maxLat, minLat);
+        } else {
+            return new org.elasticsearch.geometry.Rectangle(minLon, maxLon, maxLat, minLat);
+        }
+    }
+
+    /** Return the spatial relationship between an H3 and a point.*/
+    public static GeoRelation relatePoint(double[] xs, double[] ys, int numPoints, boolean crossesDateline, double x, double y) {
+        final DoubleUnaryOperator normalizeLong = crossesDateline ? NORMALIZE_LONG_POS : DoubleUnaryOperator.identity();
+        return relatePoint(xs, ys, numPoints, x, y, normalizeLong);
+    }
+
+    private static GeoRelation relatePoint(double[] xs, double[] ys, int numPoints, double x, double y, DoubleUnaryOperator normalize_lon) {
+        boolean res = false;
+        x = normalize_lon.applyAsDouble(x);
+        for (int i = 0; i < numPoints - 1; i++) {
+            final double x1 = normalize_lon.applyAsDouble(xs[i]);
+            final double x2 = normalize_lon.applyAsDouble(xs[i + 1]);
+            final double y1 = ys[i];
+            final double y2 = ys[i + 1];
+            if (y == y1 && y == y2 || (y <= y1 && y >= y2) != (y >= y1 && y <= y2)) {
+                if ((x == x1 && x == x2) || ((x <= x1 && x >= x2) != (x >= x1 && x <= x2) && GeoUtils.orient(x1, y1, x2, y2, x, y) == 0)) {
+                    return GeoRelation.QUERY_CROSSES;
+                } else if (y1 > y != y2 > y) {
+                    res ^= x < (x2 - x1) * (y - y1) / (y2 - y1) + x1;
+                }
+            }
+        }
+        return res ? GeoRelation.QUERY_CONTAINS : GeoRelation.QUERY_DISJOINT;
+    }
+
+    /** Checks if a line crosses a h3 bin.*/
+    public static boolean crossesLine(
+        double[] xs,
+        double[] ys,
+        int numPoints,
+        boolean crossesDateline,
+        double minX,
+        double maxX,
+        double minY,
+        double maxY,
+        double aX,
+        double aY,
+        double bX,
+        double bY,
+        boolean includeBoundary
+    ) {
+        if (crossesDateline) {
+            return crossesLine(xs, ys, numPoints, minX, maxX, minY, maxY, aX, aY, bX, bY, includeBoundary, NORMALIZE_LONG_POS)
+                || crossesLine(xs, ys, numPoints, minX, maxX, minY, maxY, aX, aY, bX, bY, includeBoundary, NORMALIZE_LONG_NEG);
+        } else {
+            return crossesLine(xs, ys, numPoints, minX, maxX, minY, maxY, aX, aY, bX, bY, includeBoundary, DoubleUnaryOperator.identity());
+        }
+    }
+
+    private static boolean crossesLine(
+        double[] xs,
+        double[] ys,
+        int numPoints,
+        double minX,
+        double maxX,
+        double minY,
+        double maxY,
+        double aX,
+        double aY,
+        double bX,
+        double bY,
+        boolean includeBoundary,
+        DoubleUnaryOperator normalizeLong
+    ) {
+
+        for (int i = 0; i < numPoints - 1; i++) {
+            double cy = ys[i];
+            double dy = ys[i + 1];
+            double cx = normalizeLong.applyAsDouble(xs[i]);
+            double dx = normalizeLong.applyAsDouble(xs[i + 1]);
+            // compute bounding box of line
+            double lMinX = StrictMath.min(cx, dx);
+            double lMaxX = StrictMath.max(cx, dx);
+            double lMinY = StrictMath.min(cy, dy);
+            double lMaxY = StrictMath.max(cy, dy);
+
+            // 2. check bounding boxes are disjoint
+            if (lMaxX < minX || lMinX > maxX || lMinY > maxY || lMaxY < minY) {
+                continue;
+            }
+            if (includeBoundary) {
+                if (GeoUtils.lineCrossesLineWithBoundary(cx, cy, dx, dy, aX, aY, bX, bY)) {
+                    return true;
+                }
+            } else {
+                if (GeoUtils.lineCrossesLine(cx, cy, dx, dy, aX, aY, bX, bY)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    /** Checks if a triangle crosses a h3 bin.*/
+    public static boolean crossesTriangle(
+        double[] xs,
+        double[] ys,
+        int numPoints,
+        boolean crossesDateline,
+        double minX,
+        double maxX,
+        double minY,
+        double maxY,
+        double ax,
+        double ay,
+        double bx,
+        double by,
+        double cx,
+        double cy,
+        boolean includeBoundary
+    ) {
+        if (crossesDateline) {
+            return crossesTriangle(xs, ys, numPoints, minX, maxX, minY, maxY, ax, ay, bx, by, cx, cy, includeBoundary, NORMALIZE_LONG_POS)
+                || crossesTriangle(xs, ys, numPoints, minX, maxX, minY, maxY, ax, ay, bx, by, cx, cy, includeBoundary, NORMALIZE_LONG_NEG);
+        } else {
+            return crossesTriangle(
+                xs,
+                ys,
+                numPoints,
+                minX,
+                maxX,
+                minY,
+                maxY,
+                ax,
+                ay,
+                bx,
+                by,
+                cx,
+                cy,
+                includeBoundary,
+                DoubleUnaryOperator.identity()
+            );
+        }
+    }
+
+    private static boolean crossesTriangle(
+        double[] xs,
+        double[] ys,
+        int numPoints,
+        double minX,
+        double maxX,
+        double minY,
+        double maxY,
+        double ax,
+        double ay,
+        double bx,
+        double by,
+        double cx,
+        double cy,
+        boolean includeBoundary,
+        DoubleUnaryOperator normalizeLong
+    ) {
+        for (int i = 0; i < numPoints - 1; i++) {
+            double dy = ys[i];
+            double ey = ys[i + 1];
+            double dx = normalizeLong.applyAsDouble(xs[i]);
+            double ex = normalizeLong.applyAsDouble(xs[i + 1]);
+
+            // optimization: see if the rectangle is outside of the "bounding box" of the polyline at all
+            // if not, don't waste our time trying more complicated stuff
+            boolean outside = (dy < minY && ey < minY) || (dy > maxY && ey > maxY) || (dx < minX && ex < minX) || (dx > maxX && ex > maxX);
+
+            if (outside == false) {
+                if (includeBoundary) {
+                    if (lineCrossesLineWithBoundary(dx, dy, ex, ey, ax, ay, bx, by)
+                        || lineCrossesLineWithBoundary(dx, dy, ex, ey, bx, by, cx, cy)
+                        || lineCrossesLineWithBoundary(dx, dy, ex, ey, cx, cy, ax, ay)) {
+                        return true;
+                    }
+                } else {
+                    if (lineCrossesLine(dx, dy, ex, ey, ax, ay, bx, by)
+                        || lineCrossesLine(dx, dy, ex, ey, bx, by, cx, cy)
+                        || lineCrossesLine(dx, dy, ex, ey, cx, cy, ax, ay)) {
+                        return true;
+                    }
+                }
+            }
+
+        }
+        return false;
+    }
+
+    /** Checks if a rectangle crosses a h3 bin.*/
+    public static boolean crossesBox(
+        double[] xs,
+        double[] ys,
+        int numPoints,
+        boolean crossesDateline,
+        double minX,
+        double maxX,
+        double minY,
+        double maxY,
+        boolean includeBoundary
+    ) {
+        if (crossesDateline) {
+            return crossesBox(xs, ys, numPoints, minX, maxX, minY, maxY, includeBoundary, NORMALIZE_LONG_POS)
+                || crossesBox(xs, ys, numPoints, minX, maxX, minY, maxY, includeBoundary, NORMALIZE_LONG_NEG);
+        } else {
+            return crossesBox(xs, ys, numPoints, minX, maxX, minY, maxY, includeBoundary, DoubleUnaryOperator.identity());
+        }
+    }
+
+    private static boolean crossesBox(
+        double[] xs,
+        double[] ys,
+        int numPoints,
+        double minX,
+        double maxX,
+        double minY,
+        double maxY,
+        boolean includeBoundary,
+        DoubleUnaryOperator normalizeLong
+    ) {
+        // we just have to cross one edge to answer the question, so we descend the tree and return when
+        // we do.
+        for (int i = 0; i < numPoints - 1; i++) {
+            // we compute line intersections of every polygon edge with every box line.
+            // if we find one, return true.
+            // for each box line (AB):
+            // for each poly line (CD):
+            // intersects = orient(C,D,A) * orient(C,D,B) <= 0 && orient(A,B,C) * orient(A,B,D) <= 0
+            double cy = ys[i];
+            double dy = ys[i + 1];
+            double cx = normalizeLong.applyAsDouble(xs[i]);
+            double dx = normalizeLong.applyAsDouble(xs[i + 1]);
+
+            // optimization: see if either end of the line segment is contained by the rectangle
+            if (Rectangle.containsPoint(cy, cx, minY, maxY, minX, maxX) || Rectangle.containsPoint(dy, dx, minY, maxY, minX, maxX)) {
+                return true;
+            }
+
+            // optimization: see if the rectangle is outside of the "bounding box" of the polyline at all
+            // if not, don't waste our time trying more complicated stuff
+            boolean outside = (cy < minY && dy < minY) || (cy > maxY && dy > maxY) || (cx < minX && dx < minX) || (cx > maxX && dx > maxX);
+
+            if (outside == false) {
+                if (includeBoundary) {
+                    if (lineCrossesLineWithBoundary(cx, cy, dx, dy, minX, minY, maxX, minY)
+                        || lineCrossesLineWithBoundary(cx, cy, dx, dy, maxX, minY, maxX, maxY)
+                        || lineCrossesLineWithBoundary(cx, cy, dx, dy, maxX, maxY, minX, maxY)
+                        || lineCrossesLineWithBoundary(cx, cy, dx, dy, minX, maxY, minX, minY)) {
+                        // include boundaries: ensures box edges that terminate on the polygon are included
+                        return true;
+                    }
+                } else {
+                    if (lineCrossesLine(cx, cy, dx, dy, minX, minY, maxX, minY)
+                        || lineCrossesLine(cx, cy, dx, dy, maxX, minY, maxX, maxY)
+                        || lineCrossesLine(cx, cy, dx, dy, maxX, maxY, minX, maxY)
+                        || lineCrossesLine(cx, cy, dx, dy, minX, maxY, minX, minY)) {
+                        return true;
+                    }
+                }
+            }
+        }
+        return false;
+    }
+}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/common/H3SphericalGeometry.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/common/H3SphericalGeometry.java
new file mode 100644
index 000000000000..c85fb58246af
--- /dev/null
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/common/H3SphericalGeometry.java
@@ -0,0 +1,224 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.spatial.common;
+
+import org.apache.lucene.geo.Component2D;
+import org.apache.lucene.geo.GeoUtils;
+import org.apache.lucene.geo.LatLonGeometry;
+import org.apache.lucene.index.PointValues;
+import org.apache.lucene.spatial3d.geom.GeoArea;
+import org.apache.lucene.spatial3d.geom.GeoAreaFactory;
+import org.apache.lucene.spatial3d.geom.GeoPolygon;
+import org.apache.lucene.spatial3d.geom.LatLonBounds;
+import org.apache.lucene.spatial3d.geom.PlanetModel;
+import org.elasticsearch.h3.H3;
+
+/** Implementation of a lucene {@link LatLonGeometry} that covers the extent of a provided H3 bin. Note that
+ * H3 bin are polygons on the sphere. */
+class H3SphericalGeometry extends LatLonGeometry {
+
+    private final long h3;
+
+    H3SphericalGeometry(long h3) {
+        this.h3 = h3;
+    }
+
+    @Override
+    protected Component2D toComponent2D() {
+        return new H3Polygon2D(h3);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o instanceof H3SphericalGeometry geom) {
+            return h3 == geom.h3;
+        }
+        return false;
+    }
+
+    @Override
+    public int hashCode() {
+        return Long.hashCode(h3);
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("H3 : ");
+        sb.append("\"");
+        sb.append(h3);
+        sb.append("\"");
+        return sb.toString();
+    }
+
+    private static class H3Polygon2D implements Component2D {
+
+        // We want to make are edges a bit bigger because spatial3d and h3 edges do not fully agree in
+        // membership of points around he edges.
+        private static final double BBOX_EDGE_DELTA = 1e-4;
+        private final long h3;
+        private final int res;
+        private final GeoPolygon hexagon;
+        private final double minX, maxX, minY, maxY;
+
+        private H3Polygon2D(long h3) {
+            this.h3 = h3;
+            this.res = H3.getResolution(h3);
+            this.hexagon = H3SphericalUtil.toGeoPolygon(h3);
+            final LatLonBounds bounds = new LatLonBounds();
+            this.hexagon.getBounds(bounds);
+            final double minY = bounds.checkNoBottomLatitudeBound() ? GeoUtils.MIN_LAT_INCL : Math.toDegrees(bounds.getMinLatitude());
+            final double maxY = bounds.checkNoTopLatitudeBound() ? GeoUtils.MAX_LAT_INCL : Math.toDegrees(bounds.getMaxLatitude());
+            final double minX;
+            final double maxX;
+            if (bounds.checkNoLongitudeBound() || bounds.getLeftLongitude() > bounds.getRightLongitude()) {
+                minX = GeoUtils.MIN_LON_INCL;
+                maxX = GeoUtils.MAX_LON_INCL;
+            } else {
+                minX = Math.toDegrees(bounds.getLeftLongitude());
+                maxX = Math.toDegrees(bounds.getRightLongitude());
+            }
+            // Unfortunately, h3 bin edges are fuzzy and cannot be represented easily. We need to buffer
+            // the bounding boxes to make sure we don't reject valid points
+            this.minX = Math.max(GeoUtils.MIN_LON_INCL, minX - BBOX_EDGE_DELTA);
+            this.maxX = Math.min(GeoUtils.MAX_LON_INCL, maxX + BBOX_EDGE_DELTA);
+            this.minY = Math.max(GeoUtils.MIN_LAT_INCL, minY - BBOX_EDGE_DELTA);
+            this.maxY = Math.min(GeoUtils.MAX_LAT_INCL, maxY + BBOX_EDGE_DELTA);
+
+        }
+
+        @Override
+        public double getMinX() {
+            return minX;
+        }
+
+        @Override
+        public double getMaxX() {
+            return maxX;
+        }
+
+        @Override
+        public double getMinY() {
+            return minY;
+        }
+
+        @Override
+        public double getMaxY() {
+            return maxY;
+        }
+
+        @Override
+        public boolean contains(double x, double y) {
+            return h3 == H3.geoToH3(y, x, res);
+        }
+
+        @Override
+        public PointValues.Relation relate(double minX, double maxX, double minY, double maxY) {
+            if (minX > this.maxX || maxX < this.minX || maxY < this.minY || minY > this.maxY) {
+                return PointValues.Relation.CELL_OUTSIDE_QUERY;
+            }
+            // h3 edges are fuzzy, therefore to avoid issues when bounding box are around the edges,
+            // we just buffer slightly the bounding box to check if it is inside the h3 bin, otherwise
+            // return crosses.
+            final GeoArea box = GeoAreaFactory.makeGeoArea(
+                PlanetModel.SPHERE,
+                Math.toRadians(Math.min(GeoUtils.MAX_LAT_INCL, maxY + BBOX_EDGE_DELTA)),
+                Math.toRadians(Math.max(GeoUtils.MIN_LAT_INCL, minY - BBOX_EDGE_DELTA)),
+                Math.toRadians(Math.max(GeoUtils.MIN_LON_INCL, minX - BBOX_EDGE_DELTA)),
+                Math.toRadians(Math.min(GeoUtils.MAX_LON_INCL, maxX + BBOX_EDGE_DELTA))
+            );
+            return switch (box.getRelationship(hexagon)) {
+                case GeoArea.CONTAINS -> PointValues.Relation.CELL_INSIDE_QUERY;
+                case GeoArea.DISJOINT -> PointValues.Relation.CELL_OUTSIDE_QUERY;
+                default -> PointValues.Relation.CELL_CROSSES_QUERY;
+            };
+        }
+
+        @Override
+        public boolean intersectsLine(double minX, double maxX, double minY, double maxY, double aX, double aY, double bX, double bY) {
+            throw new UnsupportedOperationException("intersectsLine not implemented in H3Polygon2D");
+        }
+
+        @Override
+        public boolean intersectsTriangle(
+            double minX,
+            double maxX,
+            double minY,
+            double maxY,
+            double aX,
+            double aY,
+            double bX,
+            double bY,
+            double cX,
+            double cY
+        ) {
+            throw new UnsupportedOperationException("intersectsTriangle not implemented in H3Polygon2D");
+        }
+
+        @Override
+        public boolean containsLine(double minX, double maxX, double minY, double maxY, double aX, double aY, double bX, double bY) {
+            throw new UnsupportedOperationException("containsLine not implemented in H3Polygon2D");
+        }
+
+        @Override
+        public boolean containsTriangle(
+            double minX,
+            double maxX,
+            double minY,
+            double maxY,
+            double aX,
+            double aY,
+            double bX,
+            double bY,
+            double cX,
+            double cY
+        ) {
+            throw new IllegalArgumentException();
+        }
+
+        @Override
+        public WithinRelation withinPoint(double x, double y) {
+            return contains(x, y) ? WithinRelation.NOTWITHIN : WithinRelation.DISJOINT;
+        }
+
+        @Override
+        public WithinRelation withinLine(
+            double minX,
+            double maxX,
+            double minY,
+            double maxY,
+            double aX,
+            double aY,
+            boolean ab,
+            double bX,
+            double bY
+        ) {
+            throw new UnsupportedOperationException("withinLine not implemented in H3Polygon2D");
+        }
+
+        @Override
+        public WithinRelation withinTriangle(
+            double minX,
+            double maxX,
+            double minY,
+            double maxY,
+            double aX,
+            double aY,
+            boolean ab,
+            double bX,
+            double bY,
+            boolean bc,
+            double cX,
+            double cY,
+            boolean ca
+        ) {
+            throw new UnsupportedOperationException("withinTriangle not implemented in H3Polygon2D");
+        }
+    }
+}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/common/H3SphericalUtil.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/common/H3SphericalUtil.java
new file mode 100644
index 000000000000..7266c3aa69bf
--- /dev/null
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/common/H3SphericalUtil.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.spatial.common;
+
+import org.apache.lucene.geo.LatLonGeometry;
+import org.apache.lucene.spatial3d.geom.GeoPoint;
+import org.apache.lucene.spatial3d.geom.GeoPolygon;
+import org.apache.lucene.spatial3d.geom.GeoPolygonFactory;
+import org.apache.lucene.spatial3d.geom.PlanetModel;
+import org.elasticsearch.common.geo.GeoBoundingBox;
+import org.elasticsearch.h3.CellBoundary;
+import org.elasticsearch.h3.H3;
+import org.elasticsearch.h3.LatLng;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.BiPredicate;
+
+/**
+ * Utility class for generating H3 spherical objects
+ */
+public final class H3SphericalUtil {
+
+    private static final BiPredicate<LatLng, LatLng> MIN_COMPARATOR = (e1, e2) -> e1.getLatRad() < e2.getLatRad();
+
+    private static final BiPredicate<LatLng, LatLng> MAX_COMPARATOR = (e1, e2) -> e1.getLatRad() > e2.getLatRad();
+
+    /**
+     * Computes the bounding box of the provided h3 cell considering edges to be great circles and
+     * stores then in the provided {@link GeoBoundingBox}.
+     */
+    public static void computeGeoBounds(long h3, GeoBoundingBox boundingBox) {
+        final CellBoundary boundary = H3.h3ToGeoBoundary(h3);
+        final int res = H3.getResolution(h3);
+        if (h3 == H3.northPolarH3(res)) {
+            // specialize north pole
+            computeNorthPoleBounds(boundary, boundingBox);
+        } else if (h3 == H3.southPolarH3(res)) {
+            // specialize south pole
+            computeSouthPoleBounds(boundary, boundingBox);
+        } else {
+            // generic case
+            computeBounds(boundary, boundingBox);
+        }
+    }
+
+    private static void computeNorthPoleBounds(CellBoundary boundary, GeoBoundingBox boundingBox) {
+        double minLat = Double.POSITIVE_INFINITY;
+        for (int i = 0; i < boundary.numPoints(); i++) {
+            minLat = Math.min(minLat, boundary.getLatLon(i).getLatRad());
+        }
+        boundingBox.topLeft().reset(90, -180d);
+        boundingBox.bottomRight().reset(Math.toDegrees(minLat), 180d);
+    }
+
+    private static void computeSouthPoleBounds(CellBoundary boundary, GeoBoundingBox boundingBox) {
+        double maxLat = Double.NEGATIVE_INFINITY;
+        for (int i = 0; i < boundary.numPoints(); i++) {
+            maxLat = Math.max(maxLat, boundary.getLatLon(i).getLatRad());
+        }
+        boundingBox.topLeft().reset(Math.toDegrees(maxLat), -180d);
+        boundingBox.bottomRight().reset(-90, 180d);
+    }
+
+    private static void computeBounds(CellBoundary boundary, GeoBoundingBox boundingBox) {
+        // This algorithm is based on the bounding box for great circle edges in
+        // https://trs.jpl.nasa.gov/bitstream/handle/2014/41271/07-0286.pdf
+        double minLat = Double.POSITIVE_INFINITY;
+        int minLatPos = -1;
+        double minLon = Double.POSITIVE_INFINITY;
+        double maxLat = Double.NEGATIVE_INFINITY;
+        int maxLatPos = -1;
+        double maxLon = Double.NEGATIVE_INFINITY;
+        double maxNegLon = Double.NEGATIVE_INFINITY;
+        double minPosLon = Double.POSITIVE_INFINITY;
+        for (int i = 0; i < boundary.numPoints(); i++) {
+            final double lon = boundary.getLatLon(i).getLonRad();
+            final double lat = boundary.getLatLon(i).getLatRad();
+            if (lat < minLat) {
+                minLat = lat;
+                minLatPos = i;
+            }
+            if (lat > maxLat) {
+                maxLat = lat;
+                maxLatPos = i;
+            }
+            minLon = Math.min(minLon, lon);
+            maxLon = Math.max(maxLon, lon);
+            if (lon < 0) {
+                maxNegLon = Math.max(maxNegLon, lon);
+            } else {
+                minPosLon = Math.min(minPosLon, lon);
+            }
+        }
+        if (minLat < 0) {
+            // we only correct the min latitude if negative
+            minLat = boundary.getLatLon(minLatPos).greatCircleMinLatitude(computeEdge(boundary, minLatPos, MIN_COMPARATOR));
+        }
+        if (maxLat > 0) {
+            // we only correct the max latitude if positive
+            maxLat = boundary.getLatLon(maxLatPos).greatCircleMaxLatitude(computeEdge(boundary, maxLatPos, MAX_COMPARATOR));
+        }
+        // the min / max longitude is computed the same way as in cartesian, being careful with polygons crossing the dateline
+        final boolean crossesDateline = maxLon - minLon > Math.PI;
+        boundingBox.topLeft().reset(Math.toDegrees(maxLat), crossesDateline ? Math.toDegrees(minPosLon) : Math.toDegrees(minLon));
+        boundingBox.bottomRight().reset(Math.toDegrees(minLat), crossesDateline ? Math.toDegrees(maxNegLon) : Math.toDegrees(maxLon));
+    }
+
+    private static LatLng computeEdge(CellBoundary boundary, int pos, BiPredicate<LatLng, LatLng> comparator) {
+        final LatLng end1 = boundary.getLatLon((pos + 1) % boundary.numPoints());
+        final LatLng end2 = boundary.getLatLon(pos == 0 ? boundary.numPoints() - 1 : pos - 1);
+        return comparator.test(end1, end2) ? end1 : end2;
+    }
+
+    /** Return the {@link GeoPolygon} representing the provided H3 bin */
+    public static GeoPolygon toGeoPolygon(long h3) {
+        final CellBoundary boundary = H3.h3ToGeoBoundary(h3);
+        List<GeoPoint> points = new ArrayList<>(boundary.numPoints());
+        for (int i = 0; i < boundary.numPoints(); i++) {
+            LatLng latLng = boundary.getLatLon(i);
+            points.add(new GeoPoint(PlanetModel.SPHERE, latLng.getLatRad(), latLng.getLonRad()));
+        }
+        return GeoPolygonFactory.makeGeoPolygon(PlanetModel.SPHERE, points);
+    }
+
+    /** Return the {@link LatLonGeometry} representing the provided H3 bin */
+    public static LatLonGeometry getLatLonGeometry(long h3) {
+        return new H3SphericalGeometry(h3);
+    }
+}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/Component2DRelationVisitor.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/Component2DRelationVisitor.java
index c84094e6eeb6..bdbf52c311df 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/Component2DRelationVisitor.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/Component2DRelationVisitor.java
@@ -17,7 +17,7 @@
  * This class supports checking {@link Component2D} relations against a serialized triangle tree.
  * It does not support bounding boxes crossing the dateline.
  */
-class Component2DRelationVisitor extends TriangleTreeReader.DecodedVisitor {
+class Component2DRelationVisitor extends TriangleTreeVisitor.TriangleTreeDecodedVisitor {
 
     private GeoRelation relation;
     private Component2D component2D;
@@ -28,51 +28,64 @@ class Component2DRelationVisitor extends TriangleTreeReader.DecodedVisitor {
 
     public void reset(Component2D component2D) {
         this.component2D = component2D;
-        relation = GeoRelation.QUERY_DISJOINT;
+        relation = null;
     }
 
     /**
      * return the computed relation.
      */
     public GeoRelation relation() {
-        return relation;
+        return relation == null ? GeoRelation.QUERY_DISJOINT : relation;
     }
 
     @Override
-    void visitDecodedPoint(double x, double y) {
+    protected void visitDecodedPoint(double x, double y) {
         if (component2D.contains(x, y)) {
-            if (component2D.withinPoint(x, y) == Component2D.WithinRelation.CANDIDATE) {
+            if (canBeInside() && component2D.withinPoint(x, y) == Component2D.WithinRelation.CANDIDATE) {
                 relation = GeoRelation.QUERY_INSIDE;
+            } else if (canBeContained()) {
+                relation = GeoRelation.QUERY_CONTAINS;
             } else {
                 relation = GeoRelation.QUERY_CROSSES;
             }
+        } else {
+            adjustRelationForNotIntersectingComponent();
         }
     }
 
     @Override
-    void visitDecodedLine(double aX, double aY, double bX, double bY, byte metadata) {
+    protected void visitDecodedLine(double aX, double aY, double bX, double bY, byte metadata) {
         if (component2D.intersectsLine(aX, aY, bX, bY)) {
             final boolean ab = (metadata & 1 << 4) == 1 << 4;
-            if (component2D.withinLine(aX, aY, ab, bX, bY) == Component2D.WithinRelation.CANDIDATE) {
+            if (canBeInside() && component2D.withinLine(aX, aY, ab, bX, bY) == Component2D.WithinRelation.CANDIDATE) {
                 relation = GeoRelation.QUERY_INSIDE;
+            } else if (canBeContained() && component2D.containsLine(aX, aY, bX, bY)) {
+                relation = GeoRelation.QUERY_CONTAINS;
             } else {
                 relation = GeoRelation.QUERY_CROSSES;
             }
+        } else {
+            adjustRelationForNotIntersectingComponent();
         }
     }
 
     @Override
-    void visitDecodedTriangle(double aX, double aY, double bX, double bY, double cX, double cY, byte metadata) {
+    protected void visitDecodedTriangle(double aX, double aY, double bX, double bY, double cX, double cY, byte metadata) {
         if (component2D.intersectsTriangle(aX, aY, bX, bY, cX, cY)) {
-            boolean ab = (metadata & 1 << 4) == 1 << 4;
-            boolean bc = (metadata & 1 << 5) == 1 << 5;
-            boolean ca = (metadata & 1 << 6) == 1 << 6;
-            if (component2D.withinTriangle(aX, aY, ab, bX, bY, bc, cX, cY, ca) == Component2D.WithinRelation.CANDIDATE) {
+            final boolean ab = (metadata & 1 << 4) == 1 << 4;
+            final boolean bc = (metadata & 1 << 5) == 1 << 5;
+            final boolean ca = (metadata & 1 << 6) == 1 << 6;
+            if (canBeInside() && component2D.withinTriangle(aX, aY, ab, bX, bY, bc, cX, cY, ca) == Component2D.WithinRelation.CANDIDATE) {
                 relation = GeoRelation.QUERY_INSIDE;
+            } else if (canBeContained() && component2D.containsTriangle(aX, aY, bX, bY, cX, cY)) {
+                relation = GeoRelation.QUERY_CONTAINS;
             } else {
                 relation = GeoRelation.QUERY_CROSSES;
             }
+        } else {
+            adjustRelationForNotIntersectingComponent();
         }
+
     }
 
     @Override
@@ -82,17 +95,29 @@ public boolean push() {
 
     @Override
     public boolean pushDecodedX(double minX) {
-        return component2D.getMaxX() >= minX;
+        if (component2D.getMaxX() >= minX) {
+            return true;
+        }
+        adjustRelationForNotIntersectingComponent();
+        return false;
     }
 
     @Override
     public boolean pushDecodedY(double minY) {
-        return component2D.getMaxY() >= minY;
+        if (component2D.getMaxY() >= minY) {
+            return true;
+        }
+        adjustRelationForNotIntersectingComponent();
+        return false;
     }
 
     @Override
     public boolean pushDecoded(double maxX, double maxY) {
-        return component2D.getMinY() <= maxY && component2D.getMinX() <= maxX;
+        if (component2D.getMinY() <= maxY && component2D.getMinX() <= maxX) {
+            return true;
+        }
+        adjustRelationForNotIntersectingComponent();
+        return false;
     }
 
     @Override
@@ -104,11 +129,23 @@ public boolean pushDecoded(double minX, double minY, double maxX, double maxY) {
             relation = GeoRelation.QUERY_DISJOINT;
             return false;
         }
-        if (rel == PointValues.Relation.CELL_INSIDE_QUERY) {
-            // the rectangle fully contains the shape
+        return true;
+    }
+
+    private void adjustRelationForNotIntersectingComponent() {
+        if (relation == null) {
+            relation = GeoRelation.QUERY_DISJOINT;
+        } else if (relation == GeoRelation.QUERY_CONTAINS) {
             relation = GeoRelation.QUERY_CROSSES;
-            return false;
         }
-        return true;
     }
+
+    private boolean canBeContained() {
+        return relation == null || relation == GeoRelation.QUERY_CONTAINS;
+    }
+
+    private boolean canBeInside() {
+        return relation != GeoRelation.QUERY_CONTAINS;
+    }
+
 }
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/Component2DVisitor.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/Component2DVisitor.java
index d6881969da82..60faf40bbd93 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/Component2DVisitor.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/Component2DVisitor.java
@@ -12,10 +12,10 @@
 import org.apache.lucene.index.PointValues;
 
 /**
- * A {@link TriangleTreeReader.Visitor} implementation for {@link Component2D} geometries.
+ * A {@link TriangleTreeVisitor.TriangleTreeDecodedVisitor} implementation for {@link Component2D} geometries.
  * It can solve spatial relationships against a serialize triangle tree.
  */
-public abstract class Component2DVisitor extends TriangleTreeReader.DecodedVisitor {
+public abstract class Component2DVisitor extends TriangleTreeVisitor.TriangleTreeDecodedVisitor {
 
     protected final Component2D component2D;
 
@@ -31,23 +31,23 @@ private Component2DVisitor(Component2D component2D, CoordinateEncoder encoder) {
     public abstract void reset();
 
     @Override
-    public boolean pushDecodedX(double minX) {
+    protected boolean pushDecodedX(double minX) {
         return component2D.getMaxX() >= minX;
     }
 
     @Override
-    public boolean pushDecodedY(double minY) {
+    protected boolean pushDecodedY(double minY) {
         return component2D.getMaxY() >= minY;
     }
 
     @Override
-    public boolean pushDecoded(double maxX, double maxY) {
+    protected boolean pushDecoded(double maxX, double maxY) {
         return component2D.getMinX() <= maxX && component2D.getMinY() <= maxY;
 
     }
 
     @Override
-    public boolean pushDecoded(double minX, double minY, double maxX, double maxY) {
+    protected boolean pushDecoded(double minX, double minY, double maxX, double maxY) {
         return pushDecoded(component2D.relate(minX, maxX, minY, maxY));
     }
 
@@ -56,7 +56,7 @@ public boolean pushDecoded(double minX, double minY, double maxX, double maxY) {
      *
      * @return if true, the visitor keeps traversing the tree, else it stops.
      * */
-    abstract boolean pushDecoded(PointValues.Relation relation);
+    protected abstract boolean pushDecoded(PointValues.Relation relation);
 
     /**
      * Creates a visitor from the provided Component2D and spatial relationship. Visitors are re-usable by
@@ -95,17 +95,17 @@ public void reset() {
         }
 
         @Override
-        void visitDecodedPoint(double x, double y) {
+        protected void visitDecodedPoint(double x, double y) {
             intersects = component2D.contains(x, y);
         }
 
         @Override
-        void visitDecodedLine(double aX, double aY, double bX, double bY, byte metadata) {
+        protected void visitDecodedLine(double aX, double aY, double bX, double bY, byte metadata) {
             intersects = component2D.intersectsLine(aX, aY, bX, bY);
         }
 
         @Override
-        void visitDecodedTriangle(double aX, double aY, double bX, double bY, double cX, double cY, byte metadata) {
+        protected void visitDecodedTriangle(double aX, double aY, double bX, double bY, double cX, double cY, byte metadata) {
             intersects = component2D.intersectsTriangle(aX, aY, bX, bY, cX, cY);
         }
 
@@ -116,7 +116,7 @@ public boolean push() {
         }
 
         @Override
-        boolean pushDecoded(PointValues.Relation relation) {
+        protected boolean pushDecoded(PointValues.Relation relation) {
             if (relation == PointValues.Relation.CELL_OUTSIDE_QUERY) {
                 // shapes are disjoint, stop traversing the tree.
                 return false;
@@ -156,17 +156,17 @@ public void reset() {
         }
 
         @Override
-        void visitDecodedPoint(double x, double y) {
+        protected void visitDecodedPoint(double x, double y) {
             disjoint = component2D.contains(x, y) == false;
         }
 
         @Override
-        void visitDecodedLine(double aX, double aY, double bX, double bY, byte metadata) {
+        protected void visitDecodedLine(double aX, double aY, double bX, double bY, byte metadata) {
             disjoint = component2D.intersectsLine(aX, aY, bX, bY) == false;
         }
 
         @Override
-        void visitDecodedTriangle(double aX, double aY, double bX, double bY, double cX, double cY, byte metadata) {
+        protected void visitDecodedTriangle(double aX, double aY, double bX, double bY, double cX, double cY, byte metadata) {
             disjoint = component2D.intersectsTriangle(aX, aY, bX, bY, cX, cY) == false;
         }
 
@@ -177,7 +177,7 @@ public boolean push() {
         }
 
         @Override
-        boolean pushDecoded(PointValues.Relation relation) {
+        protected boolean pushDecoded(PointValues.Relation relation) {
             if (relation == PointValues.Relation.CELL_OUTSIDE_QUERY) {
                 // shapes are disjoint, stop traversing the tree.
                 return false;
@@ -217,17 +217,17 @@ public void reset() {
         }
 
         @Override
-        void visitDecodedPoint(double x, double y) {
+        protected void visitDecodedPoint(double x, double y) {
             within = component2D.contains(x, y);
         }
 
         @Override
-        void visitDecodedLine(double aX, double aY, double bX, double bY, byte metadata) {
+        protected void visitDecodedLine(double aX, double aY, double bX, double bY, byte metadata) {
             within = component2D.containsLine(aX, aY, bX, bY);
         }
 
         @Override
-        void visitDecodedTriangle(double aX, double aY, double bX, double bY, double cX, double cY, byte metadata) {
+        protected void visitDecodedTriangle(double aX, double aY, double bX, double bY, double cX, double cY, byte metadata) {
             within = component2D.containsTriangle(aX, aY, bX, bY, cX, cY);
         }
 
@@ -238,31 +238,31 @@ public boolean push() {
         }
 
         @Override
-        public boolean pushX(int minX) {
+        protected boolean pushDecodedX(double minX) {
             // if any part of the tree is skipped, then the doc value is not within the shape,
             // stop traversing the tree
-            within = super.pushX(minX);
+            within = super.pushDecodedX(minX);
             return within;
         }
 
         @Override
-        public boolean pushY(int minY) {
+        protected boolean pushDecodedY(double minY) {
             // if any part of the tree is skipped, then the doc value is not within the shape,
             // stop traversing the tree
-            within = super.pushY(minY);
+            within = super.pushDecodedY(minY);
             return within;
         }
 
         @Override
-        public boolean push(int maxX, int maxY) {
+        protected boolean pushDecoded(double maxX, double maxY) {
             // if any part of the tree is skipped, then the doc value is not within the shape,
             // stop traversing the tree
-            within = super.push(maxX, maxY);
+            within = super.pushDecoded(maxX, maxY);
             return within;
         }
 
         @Override
-        boolean pushDecoded(PointValues.Relation relation) {
+        protected boolean pushDecoded(PointValues.Relation relation) {
             if (relation == PointValues.Relation.CELL_OUTSIDE_QUERY) {
                 // shapes are disjoint, stop traversing the tree.
                 within = false;
@@ -297,7 +297,7 @@ public void reset() {
         }
 
         @Override
-        void visitDecodedPoint(double x, double y) {
+        protected void visitDecodedPoint(double x, double y) {
             final Component2D.WithinRelation rel = component2D.withinPoint(x, y);
             if (rel != Component2D.WithinRelation.DISJOINT) {
                 // Only override relationship if different to DISJOINT
@@ -306,7 +306,7 @@ void visitDecodedPoint(double x, double y) {
         }
 
         @Override
-        void visitDecodedLine(double aX, double aY, double bX, double bY, byte metadata) {
+        protected void visitDecodedLine(double aX, double aY, double bX, double bY, byte metadata) {
             final boolean ab = (metadata & 1 << 4) == 1 << 4;
             final Component2D.WithinRelation rel = component2D.withinLine(aX, aY, ab, bX, bY);
             if (rel != Component2D.WithinRelation.DISJOINT) {
@@ -316,7 +316,7 @@ void visitDecodedLine(double aX, double aY, double bX, double bY, byte metadata)
         }
 
         @Override
-        void visitDecodedTriangle(double aX, double aY, double bX, double bY, double cX, double cY, byte metadata) {
+        protected void visitDecodedTriangle(double aX, double aY, double bX, double bY, double cX, double cY, byte metadata) {
             final boolean ab = (metadata & 1 << 4) == 1 << 4;
             final boolean bc = (metadata & 1 << 5) == 1 << 5;
             final boolean ca = (metadata & 1 << 6) == 1 << 6;
@@ -334,7 +334,7 @@ public boolean push() {
         }
 
         @Override
-        boolean pushDecoded(PointValues.Relation relation) {
+        protected boolean pushDecoded(PointValues.Relation relation) {
             // Only traverse the tree if the shapes intersects.
             return relation == PointValues.Relation.CELL_CROSSES_QUERY;
         }
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeoRelation.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeoRelation.java
index c940fe78ebdd..edfd730291fb 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeoRelation.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeoRelation.java
@@ -14,5 +14,6 @@
 public enum GeoRelation {
     QUERY_CROSSES,
     QUERY_INSIDE,
-    QUERY_DISJOINT
+    QUERY_DISJOINT,
+    QUERY_CONTAINS
 }
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeoShapeValues.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeoShapeValues.java
index fe0ec12e9157..b0bacf7d295f 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeoShapeValues.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeoShapeValues.java
@@ -80,7 +80,7 @@ protected Component2D centroidAsComponent2D() throws IOException {
          */
         public GeoRelation relate(int minX, int maxX, int minY, int maxY) throws IOException {
             tile2DVisitor.reset(minX, minY, maxX, maxY);
-            reader.visit(tile2DVisitor);
+            visit(tile2DVisitor);
             return tile2DVisitor.relation();
         }
 
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeometryDocValueReader.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeometryDocValueReader.java
index 0b95d5ac2666..51ce9124eac5 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeometryDocValueReader.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/GeometryDocValueReader.java
@@ -98,7 +98,7 @@ protected double getSumCentroidWeight() throws IOException {
     /**
      * Visit the triangle tree with the provided visitor
      */
-    public void visit(TriangleTreeReader.Visitor visitor) throws IOException {
+    public void visit(TriangleTreeVisitor visitor) throws IOException {
         Extent geometryExtent = getExtent();
         int thisMaxX = geometryExtent.maxX();
         int thisMinX = geometryExtent.minX();
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/LabelPositionVisitor.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/LabelPositionVisitor.java
index 16939c183cf7..6007a9df569b 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/LabelPositionVisitor.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/LabelPositionVisitor.java
@@ -17,41 +17,41 @@
  *
  * TODO: We could instead choose the point closer to the centroid which improves unbalanced trees
  */
-public class LabelPositionVisitor extends TriangleTreeReader.DecodedVisitor {
+class LabelPositionVisitor extends TriangleTreeVisitor.TriangleTreeDecodedVisitor {
 
     private SpatialPoint labelPosition;
     private final BiFunction<Double, Double, SpatialPoint> pointMaker;
 
-    public LabelPositionVisitor(CoordinateEncoder encoder, BiFunction<Double, Double, SpatialPoint> pointMaker) {
+    LabelPositionVisitor(CoordinateEncoder encoder, BiFunction<Double, Double, SpatialPoint> pointMaker) {
         super(encoder);
         this.pointMaker = pointMaker;
     }
 
     @Override
-    void visitDecodedPoint(double x, double y) {
+    protected void visitDecodedPoint(double x, double y) {
         assert labelPosition == null;
         labelPosition = pointMaker.apply(x, y);
     }
 
     @Override
-    public void visitDecodedLine(double aX, double aY, double bX, double bY, byte metadata) {
+    protected void visitDecodedLine(double aX, double aY, double bX, double bY, byte metadata) {
         assert labelPosition == null;
         labelPosition = pointMaker.apply((aX + bX) / 2.0, (aY + bY) / 2.0);
     }
 
     @Override
-    public void visitDecodedTriangle(double aX, double aY, double bX, double bY, double cX, double cY, byte metadata) {
+    protected void visitDecodedTriangle(double aX, double aY, double bX, double bY, double cX, double cY, byte metadata) {
         assert labelPosition == null;
         labelPosition = pointMaker.apply((aX + bX + cX) / 3.0, (aY + bY + cY) / 3.0);
     }
 
     @Override
-    boolean pushDecodedX(double minX) {
+    protected boolean pushDecodedX(double minX) {
         return labelPosition == null;
     }
 
     @Override
-    boolean pushDecodedY(double minX) {
+    protected boolean pushDecodedY(double minX) {
         return labelPosition == null;
     }
 
@@ -62,18 +62,12 @@ public boolean push() {
     }
 
     @Override
-    boolean pushDecoded(double maxX, double maxY) {
+    protected boolean pushDecoded(double maxX, double maxY) {
         return labelPosition == null;
     }
 
     @Override
-    public boolean push(int minX, int minY, int maxX, int maxY) {
-        // Always start the traversal
-        return labelPosition == null;
-    }
-
-    @Override
-    boolean pushDecoded(double minX, double minY, double maxX, double maxY) {
+    protected boolean pushDecoded(double minX, double minY, double maxX, double maxY) {
         return labelPosition == null;
     }
 
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/ShapeValues.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/ShapeValues.java
index 93cac2971b47..1036030546bc 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/ShapeValues.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/ShapeValues.java
@@ -91,9 +91,9 @@ public T missing(String missing) {
      * the provided decoder (could be geo or cartesian)
      */
     protected abstract static class ShapeValue implements ToXContentFragment {
-        protected final GeometryDocValueReader reader;
+        private final GeometryDocValueReader reader;
         private final BoundingBox boundingBox;
-        protected final CoordinateEncoder encoder;
+        private final CoordinateEncoder encoder;
         private final BiFunction<Double, Double, SpatialPoint> pointMaker;
         private final Component2DRelationVisitor component2DRelationVisitor;
 
@@ -117,6 +117,13 @@ public BoundingBox boundingBox() {
             return boundingBox;
         }
 
+        /**
+         * Visit the underlying tree structure using the provided {@link TriangleTreeVisitor}
+         */
+        public void visit(TriangleTreeVisitor visitor) throws IOException {
+            reader.visit(visitor);
+        }
+
         protected abstract Component2D centroidAsComponent2D() throws IOException;
 
         private boolean centroidWithinShape() throws IOException {
@@ -139,7 +146,7 @@ public SpatialPoint labelPosition() throws IOException {
             }
             // For all other cases, use the first triangle (or line or point) in the tree which will always intersect the shape
             LabelPositionVisitor visitor = new LabelPositionVisitor(this.encoder, pointMaker);
-            reader.visit(visitor);
+            visit(visitor);
             return visitor.labelPosition();
         }
 
@@ -148,7 +155,7 @@ public SpatialPoint labelPosition() throws IOException {
          * simple geometries, therefore it will fail if the LatLonGeometry is a {@link org.apache.lucene.geo.Rectangle}
          * that crosses the dateline.
          */
-        protected GeoRelation relate(Component2D component2D) throws IOException {
+        public GeoRelation relate(Component2D component2D) throws IOException {
             component2DRelationVisitor.reset(component2D);
             reader.visit(component2DRelationVisitor);
             return component2DRelationVisitor.relation();
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/Tile2DVisitor.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/Tile2DVisitor.java
index 31d50ff3f4e6..ef780d7e5d06 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/Tile2DVisitor.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/Tile2DVisitor.java
@@ -16,7 +16,7 @@
  * This class supports checking bounding box relations against a serialized triangle tree.
  *
  */
-class Tile2DVisitor implements TriangleTreeReader.Visitor {
+class Tile2DVisitor implements TriangleTreeVisitor {
 
     private GeoRelation relation;
     private int minX;
@@ -96,7 +96,7 @@ public boolean push(int minX, int minY, int maxX, int maxY) {
         }
         if (this.minX <= minX && this.maxX >= maxX && this.minY <= minY && this.maxY >= maxY) {
             // the rectangle fully contains the shape
-            relation = GeoRelation.QUERY_CROSSES;
+            relation = GeoRelation.QUERY_CONTAINS;
             return false;
         }
         return true;
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/TriangleTreeReader.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/TriangleTreeReader.java
index 0bb02f40b505..8fe4299b17a0 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/TriangleTreeReader.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/TriangleTreeReader.java
@@ -15,7 +15,7 @@
  * A tree reader for a previous serialized {@link org.elasticsearch.geometry.Geometry} using
  * {@link TriangleTreeWriter}.
  *
- * The tree structure is navigated using a {@link Visitor}.
+ * The tree structure is navigated using a {@link TriangleTreeVisitor}.
  *
  */
 class TriangleTreeReader {
@@ -23,16 +23,15 @@ class TriangleTreeReader {
     private TriangleTreeReader() {}
 
     /**
-     * Visit the Triangle tree using the {@link Visitor} provided.
+     * Visit the Triangle tree using the {@link TriangleTreeVisitor} provided.
      */
-    public static void visit(ByteArrayStreamInput input, TriangleTreeReader.Visitor visitor, int thisMaxX, int thisMaxY)
-        throws IOException {
+    public static void visit(ByteArrayStreamInput input, TriangleTreeVisitor visitor, int thisMaxX, int thisMaxY) throws IOException {
         visit(input, visitor, true, thisMaxX, thisMaxY, true);
     }
 
     private static boolean visit(
         ByteArrayStreamInput input,
-        TriangleTreeReader.Visitor visitor,
+        TriangleTreeVisitor visitor,
         boolean splitX,
         int thisMaxX,
         int thisMaxY,
@@ -90,13 +89,8 @@ private static boolean visit(
         return visitor.push();
     }
 
-    private static boolean pushLeft(
-        ByteArrayStreamInput input,
-        TriangleTreeReader.Visitor visitor,
-        int thisMaxX,
-        int thisMaxY,
-        boolean splitX
-    ) throws IOException {
+    private static boolean pushLeft(ByteArrayStreamInput input, TriangleTreeVisitor visitor, int thisMaxX, int thisMaxY, boolean splitX)
+        throws IOException {
         int nextMaxX = Math.toIntExact(thisMaxX - input.readVLong());
         int nextMaxY = Math.toIntExact(thisMaxY - input.readVLong());
         int size = input.readVInt();
@@ -110,7 +104,7 @@ private static boolean pushLeft(
 
     private static boolean pushRight(
         ByteArrayStreamInput input,
-        TriangleTreeReader.Visitor visitor,
+        TriangleTreeVisitor visitor,
         int thisMaxX,
         int thisMaxY,
         int thisMinX,
@@ -132,125 +126,4 @@ private static boolean pushRight(
         }
         return visitor.push();
     }
-
-    /** Visitor for triangle interval tree */
-    interface Visitor {
-
-        /** visit a node point. */
-        void visitPoint(int x, int y);
-
-        /** visit a node line. */
-        void visitLine(int aX, int aY, int bX, int bY, byte metadata);
-
-        /** visit a node triangle. */
-        void visitTriangle(int aX, int aY, int bX, int bY, int cX, int cY, byte metadata);
-
-        /** Should the visitor keep visiting the tree. Called after visiting a node or skipping
-         * a tree branch, if the return value is {@code false}, no more nodes will be visited. */
-        boolean push();
-
-        /** Should the visitor visit nodes that have bounds greater or equal
-         * than the {@code minX} provided. */
-        boolean pushX(int minX);
-
-        /** Should the visitor visit nodes that have bounds greater or equal
-         * than the {@code minY} provided. */
-        boolean pushY(int minY);
-
-        /** Should the visitor visit nodes that have bounds lower or equal than the
-         * {@code maxX} and {@code minX} provided. */
-        boolean push(int maxX, int maxY);
-
-        /** Should the visitor visit the tree given the bounding box of the tree. Called before
-         * visiting the tree. */
-        boolean push(int minX, int minY, int maxX, int maxY);
-    }
-
-    /** Visitor for triangle interval tree which decodes the coordinates */
-    public abstract static class DecodedVisitor implements Visitor {
-
-        private final CoordinateEncoder encoder;
-
-        DecodedVisitor(CoordinateEncoder encoder) {
-            this.encoder = encoder;
-        }
-
-        @Override
-        public void visitPoint(int x, int y) {
-            visitDecodedPoint(encoder.decodeX(x), encoder.decodeY(y));
-        }
-
-        /**
-         * Equivalent to {@link #visitPoint(int, int)} but coordinates are decoded.
-         */
-        abstract void visitDecodedPoint(double x, double y);
-
-        @Override
-        public void visitLine(int aX, int aY, int bX, int bY, byte metadata) {
-            visitDecodedLine(encoder.decodeX(aX), encoder.decodeY(aY), encoder.decodeX(bX), encoder.decodeY(bY), metadata);
-        }
-
-        /**
-         * Equivalent to {@link #visitLine(int, int, int, int, byte)} but coordinates are decoded.
-         */
-        abstract void visitDecodedLine(double aX, double aY, double bX, double bY, byte metadata);
-
-        @Override
-        public void visitTriangle(int aX, int aY, int bX, int bY, int cX, int cY, byte metadata) {
-            visitDecodedTriangle(
-                encoder.decodeX(aX),
-                encoder.decodeY(aY),
-                encoder.decodeX(bX),
-                encoder.decodeY(bY),
-                encoder.decodeX(cX),
-                encoder.decodeY(cY),
-                metadata
-            );
-        }
-
-        /**
-         * Equivalent to {@link #visitTriangle(int, int, int, int, int, int, byte)} but coordinates are decoded.
-         */
-        abstract void visitDecodedTriangle(double aX, double aY, double bX, double bY, double cX, double cY, byte metadata);
-
-        @Override
-        public boolean pushX(int minX) {
-            return pushDecodedX(encoder.decodeX(minX));
-        }
-
-        /**
-         * Equivalent to {@link #pushX(int)}  but coordinates are decoded.
-         */
-        abstract boolean pushDecodedX(double minX);
-
-        @Override
-        public boolean pushY(int minY) {
-            return pushDecodedY(encoder.decodeY(minY));
-        }
-
-        /**
-         * Equivalent to {@link #pushY(int)}  but coordinates are decoded.
-         */
-        abstract boolean pushDecodedY(double minX);
-
-        @Override
-        public boolean push(int maxX, int maxY) {
-            return pushDecoded(encoder.decodeX(maxX), encoder.decodeY(maxY));
-        }
-
-        /**
-         * Equivalent to {@link #push(int, int)} but coordinates are decoded.
-         */
-        abstract boolean pushDecoded(double maxX, double maxY);
-
-        @Override
-        public boolean push(int minX, int minY, int maxX, int maxY) {
-            return pushDecoded(encoder.decodeX(minX), encoder.decodeY(minY), encoder.decodeX(maxX), encoder.decodeY(maxY));
-        }
-
-        /**
-         * Equivalent to {@link #push(int, int, int, int)} but coordinates are decoded.
-         */
-        abstract boolean pushDecoded(double minX, double minY, double maxX, double maxY);
-    }
 }
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/TriangleTreeVisitor.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/TriangleTreeVisitor.java
new file mode 100644
index 000000000000..283e8c902a71
--- /dev/null
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/TriangleTreeVisitor.java
@@ -0,0 +1,132 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.spatial.index.fielddata;
+
+/** Visitor for triangle interval tree.
+ *
+ * @see TriangleTreeReader
+ * */
+public interface TriangleTreeVisitor {
+
+    /** visit a node point. */
+    void visitPoint(int x, int y);
+
+    /** visit a node line. */
+    void visitLine(int aX, int aY, int bX, int bY, byte metadata);
+
+    /** visit a node triangle. */
+    void visitTriangle(int aX, int aY, int bX, int bY, int cX, int cY, byte metadata);
+
+    /** Should the visitor keep visiting the tree. Called after visiting a node or skipping
+     * a tree branch, if the return value is {@code false}, no more nodes will be visited. */
+    boolean push();
+
+    /** Should the visitor visit nodes that have bounds greater or equal
+     * than the {@code minX} provided. */
+    boolean pushX(int minX);
+
+    /** Should the visitor visit nodes that have bounds greater or equal
+     * than the {@code minY} provided. */
+    boolean pushY(int minY);
+
+    /** Should the visitor visit nodes that have bounds lower or equal than the
+     * {@code maxX} and {@code minX} provided. */
+    boolean push(int maxX, int maxY);
+
+    /** Should the visitor visit the tree given the bounding box of the tree. Called before
+     * visiting the tree. */
+    boolean push(int minX, int minY, int maxX, int maxY);
+
+    /** Visitor for triangle interval tree which decodes the coordinates */
+    abstract class TriangleTreeDecodedVisitor implements TriangleTreeVisitor {
+
+        private final CoordinateEncoder encoder;
+
+        protected TriangleTreeDecodedVisitor(CoordinateEncoder encoder) {
+            this.encoder = encoder;
+        }
+
+        @Override
+        public final void visitPoint(int x, int y) {
+            visitDecodedPoint(encoder.decodeX(x), encoder.decodeY(y));
+        }
+
+        /**
+         * Equivalent to {@link #visitPoint(int, int)} but coordinates are decoded.
+         */
+        protected abstract void visitDecodedPoint(double x, double y);
+
+        @Override
+        public final void visitLine(int aX, int aY, int bX, int bY, byte metadata) {
+            visitDecodedLine(encoder.decodeX(aX), encoder.decodeY(aY), encoder.decodeX(bX), encoder.decodeY(bY), metadata);
+        }
+
+        /**
+         * Equivalent to {@link #visitLine(int, int, int, int, byte)} but coordinates are decoded.
+         */
+        protected abstract void visitDecodedLine(double aX, double aY, double bX, double bY, byte metadata);
+
+        @Override
+        public final void visitTriangle(int aX, int aY, int bX, int bY, int cX, int cY, byte metadata) {
+            visitDecodedTriangle(
+                encoder.decodeX(aX),
+                encoder.decodeY(aY),
+                encoder.decodeX(bX),
+                encoder.decodeY(bY),
+                encoder.decodeX(cX),
+                encoder.decodeY(cY),
+                metadata
+            );
+        }
+
+        /**
+         * Equivalent to {@link #visitTriangle(int, int, int, int, int, int, byte)} but coordinates are decoded.
+         */
+        protected abstract void visitDecodedTriangle(double aX, double aY, double bX, double bY, double cX, double cY, byte metadata);
+
+        @Override
+        public final boolean pushX(int minX) {
+            return pushDecodedX(encoder.decodeX(minX));
+        }
+
+        /**
+         * Equivalent to {@link #pushX(int)}  but coordinates are decoded.
+         */
+        protected abstract boolean pushDecodedX(double minX);
+
+        @Override
+        public final boolean pushY(int minY) {
+            return pushDecodedY(encoder.decodeY(minY));
+        }
+
+        /**
+         * Equivalent to {@link #pushY(int)}  but coordinates are decoded.
+         */
+        protected abstract boolean pushDecodedY(double minX);
+
+        @Override
+        public final boolean push(int maxX, int maxY) {
+            return pushDecoded(encoder.decodeX(maxX), encoder.decodeY(maxY));
+        }
+
+        /**
+         * Equivalent to {@link #push(int, int)} but coordinates are decoded.
+         */
+        protected abstract boolean pushDecoded(double maxX, double maxY);
+
+        @Override
+        public final boolean push(int minX, int minY, int maxX, int maxY) {
+            return pushDecoded(encoder.decodeX(minX), encoder.decodeY(minY), encoder.decodeX(maxX), encoder.decodeY(maxY));
+        }
+
+        /**
+         * Equivalent to {@link #push(int, int, int, int)} but coordinates are decoded.
+         */
+        protected abstract boolean pushDecoded(double minX, double minY, double maxX, double maxY);
+    }
+}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilder.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilder.java
index 663ffa5c8be4..7fb42dd024b1 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilder.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilder.java
@@ -8,10 +8,11 @@
 package org.elasticsearch.xpack.spatial.index.query;
 
 import org.apache.lucene.geo.GeoEncodingUtils;
+import org.apache.lucene.geo.LatLonGeometry;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -30,6 +31,9 @@
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xpack.spatial.common.H3CartesianUtil;
+import org.elasticsearch.xpack.spatial.common.H3SphericalUtil;
+import org.elasticsearch.xpack.spatial.index.mapper.GeoShapeWithDocValuesFieldMapper;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -102,11 +106,19 @@ protected void validate(String gridId) {
 
             @Override
             protected Query toQuery(SearchExecutionContext context, String fieldName, MappedFieldType fieldType, String id) {
-                H3LatLonGeometry geometry = new H3LatLonGeometry(id);
-                if (fieldType instanceof GeoPointFieldMapper.GeoPointFieldType pointFieldType) {
-                    return pointFieldType.geoShapeQuery(context, fieldName, ShapeRelation.INTERSECTS, geometry);
-                } else if (fieldType instanceof GeoPointScriptFieldType scriptType) {
-                    return scriptType.geoShapeQuery(context, fieldName, ShapeRelation.INTERSECTS, geometry);
+                final long h3 = H3.stringToH3(id);
+                if (fieldType instanceof GeoShapeWithDocValuesFieldMapper.GeoShapeWithDocValuesFieldType geoShapeFieldType) {
+                    // shapes are solved on the cartesian geometry
+                    final LatLonGeometry geometry = H3CartesianUtil.getLatLonGeometry(h3);
+                    return geoShapeFieldType.geoShapeQuery(context, fieldName, ShapeRelation.INTERSECTS, geometry);
+                } else {
+                    // points are solved on the spherical geometry
+                    final LatLonGeometry geometry = H3SphericalUtil.getLatLonGeometry(h3);
+                    if (fieldType instanceof GeoPointFieldMapper.GeoPointFieldType pointFieldType) {
+                        return pointFieldType.geoShapeQuery(context, fieldName, ShapeRelation.INTERSECTS, geometry);
+                    } else if (fieldType instanceof GeoPointScriptFieldType scriptType) {
+                        return scriptType.geoShapeQuery(context, fieldName, ShapeRelation.INTERSECTS, geometry);
+                    }
                 }
                 throw new QueryShardException(
                     context,
@@ -385,7 +397,7 @@ public String getWriteableName() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_3_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_3_0;
     }
 }
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/H3LatLonGeometry.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/H3LatLonGeometry.java
deleted file mode 100644
index dee8bbb28232..000000000000
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/H3LatLonGeometry.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.spatial.index.query;
-
-import org.apache.lucene.geo.Component2D;
-import org.apache.lucene.geo.GeoUtils;
-import org.apache.lucene.geo.LatLonGeometry;
-import org.apache.lucene.index.PointValues;
-import org.apache.lucene.spatial3d.geom.GeoArea;
-import org.apache.lucene.spatial3d.geom.GeoAreaFactory;
-import org.apache.lucene.spatial3d.geom.GeoPoint;
-import org.apache.lucene.spatial3d.geom.GeoPolygon;
-import org.apache.lucene.spatial3d.geom.GeoPolygonFactory;
-import org.apache.lucene.spatial3d.geom.LatLonBounds;
-import org.apache.lucene.spatial3d.geom.PlanetModel;
-import org.elasticsearch.h3.CellBoundary;
-import org.elasticsearch.h3.H3;
-import org.elasticsearch.h3.LatLng;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-
-/** Implementation of a lucene {@link LatLonGeometry} that covers the extent of a provided H3 bin. Note that
- * H3 bin are polygons on the sphere. */
-class H3LatLonGeometry extends LatLonGeometry {
-
-    private final String h3Address;
-
-    H3LatLonGeometry(String h3Address) {
-        this.h3Address = h3Address;
-    }
-
-    @Override
-    protected Component2D toComponent2D() {
-        return new H3Polygon2D(h3Address);
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o instanceof H3LatLonGeometry h3) {
-            return Objects.equals(h3Address, h3.h3Address);
-        }
-        return false;
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hashCode(h3Address);
-    }
-
-    @Override
-    public String toString() {
-        StringBuilder sb = new StringBuilder();
-        sb.append("H3 : ");
-        sb.append("\"");
-        sb.append(h3Address);
-        sb.append("\"");
-        return sb.toString();
-    }
-
-    private static class H3Polygon2D implements Component2D {
-
-        // We want to make are edges a bit bigger because spatial3d and h3 edges do not fully agree in
-        // membership of points around he edges.
-        private static final double BBOX_EDGE_DELTA = 1e-4;
-        private final long h3;
-        private final int res;
-        private final GeoPolygon hexagon;
-        private final double minX, maxX, minY, maxY;
-
-        private H3Polygon2D(String h3Address) {
-            h3 = H3.stringToH3(h3Address);
-            res = H3.getResolution(h3Address);
-            final CellBoundary cellBoundary = H3.h3ToGeoBoundary(h3Address);
-            hexagon = getGeoPolygon(cellBoundary);
-            final LatLonBounds bounds = new LatLonBounds();
-            hexagon.getBounds(bounds);
-            final double minY = bounds.checkNoBottomLatitudeBound() ? GeoUtils.MIN_LAT_INCL : Math.toDegrees(bounds.getMinLatitude());
-            final double maxY = bounds.checkNoTopLatitudeBound() ? GeoUtils.MAX_LAT_INCL : Math.toDegrees(bounds.getMaxLatitude());
-            final double minX;
-            final double maxX;
-            if (bounds.checkNoLongitudeBound() || bounds.getLeftLongitude() > bounds.getRightLongitude()) {
-                minX = GeoUtils.MIN_LON_INCL;
-                maxX = GeoUtils.MAX_LON_INCL;
-            } else {
-                minX = Math.toDegrees(bounds.getLeftLongitude());
-                maxX = Math.toDegrees(bounds.getRightLongitude());
-            }
-            // Unfortunately, h3 bin edges are fuzzy and cannot be represented easily. We need to buffer
-            // the bounding boxes to make sure we don't reject valid points
-            this.minX = Math.max(GeoUtils.MIN_LON_INCL, minX - BBOX_EDGE_DELTA);
-            this.maxX = Math.min(GeoUtils.MAX_LON_INCL, maxX + BBOX_EDGE_DELTA);
-            this.minY = Math.max(GeoUtils.MIN_LAT_INCL, minY - BBOX_EDGE_DELTA);
-            this.maxY = Math.min(GeoUtils.MAX_LAT_INCL, maxY + BBOX_EDGE_DELTA);
-
-        }
-
-        private GeoPolygon getGeoPolygon(CellBoundary cellBoundary) {
-            final List<GeoPoint> points = new ArrayList<>(cellBoundary.numPoints());
-            for (int i = 0; i < cellBoundary.numPoints(); i++) {
-                final LatLng latLng = cellBoundary.getLatLon(i);
-                points.add(new GeoPoint(PlanetModel.SPHERE, latLng.getLatRad(), latLng.getLonRad()));
-            }
-            return GeoPolygonFactory.makeGeoPolygon(PlanetModel.SPHERE, points);
-        }
-
-        @Override
-        public double getMinX() {
-            return minX;
-        }
-
-        @Override
-        public double getMaxX() {
-            return maxX;
-        }
-
-        @Override
-        public double getMinY() {
-            return minY;
-        }
-
-        @Override
-        public double getMaxY() {
-            return maxY;
-        }
-
-        @Override
-        public boolean contains(double x, double y) {
-            return h3 == H3.geoToH3(y, x, res);
-        }
-
-        @Override
-        public PointValues.Relation relate(double minX, double maxX, double minY, double maxY) {
-            if (minX > this.maxX || maxX < this.minX || maxY < this.minY || minY > this.maxY) {
-                return PointValues.Relation.CELL_OUTSIDE_QUERY;
-            }
-            // h3 edges are fuzzy, therefore to avoid issues when bounding box are around the edges,
-            // we just buffer slightly the bounding box to check if it is inside the h3 bin, otherwise
-            // return crosses.
-            final GeoArea box = GeoAreaFactory.makeGeoArea(
-                PlanetModel.SPHERE,
-                Math.toRadians(Math.min(GeoUtils.MAX_LAT_INCL, maxY + BBOX_EDGE_DELTA)),
-                Math.toRadians(Math.max(GeoUtils.MIN_LAT_INCL, minY - BBOX_EDGE_DELTA)),
-                Math.toRadians(Math.max(GeoUtils.MIN_LON_INCL, minX - BBOX_EDGE_DELTA)),
-                Math.toRadians(Math.min(GeoUtils.MAX_LON_INCL, maxX + BBOX_EDGE_DELTA))
-            );
-            return switch (box.getRelationship(hexagon)) {
-                case GeoArea.CONTAINS -> PointValues.Relation.CELL_INSIDE_QUERY;
-                case GeoArea.DISJOINT -> PointValues.Relation.CELL_OUTSIDE_QUERY;
-                default -> PointValues.Relation.CELL_CROSSES_QUERY;
-            };
-        }
-
-        @Override
-        public boolean intersectsLine(double minX, double maxX, double minY, double maxY, double aX, double aY, double bX, double bY) {
-            throw new UnsupportedOperationException("intersectsLine not implemented in H3Polygon2D");
-        }
-
-        @Override
-        public boolean intersectsTriangle(
-            double minX,
-            double maxX,
-            double minY,
-            double maxY,
-            double aX,
-            double aY,
-            double bX,
-            double bY,
-            double cX,
-            double cY
-        ) {
-            throw new UnsupportedOperationException("intersectsTriangle not implemented in H3Polygon2D");
-        }
-
-        @Override
-        public boolean containsLine(double minX, double maxX, double minY, double maxY, double aX, double aY, double bX, double bY) {
-            throw new UnsupportedOperationException("containsLine not implemented in H3Polygon2D");
-        }
-
-        @Override
-        public boolean containsTriangle(
-            double minX,
-            double maxX,
-            double minY,
-            double maxY,
-            double aX,
-            double aY,
-            double bX,
-            double bY,
-            double cX,
-            double cY
-        ) {
-            throw new IllegalArgumentException();
-        }
-
-        @Override
-        public WithinRelation withinPoint(double x, double y) {
-            return contains(x, y) ? WithinRelation.NOTWITHIN : WithinRelation.DISJOINT;
-        }
-
-        @Override
-        public WithinRelation withinLine(
-            double minX,
-            double maxX,
-            double minY,
-            double maxY,
-            double aX,
-            double aY,
-            boolean ab,
-            double bX,
-            double bY
-        ) {
-            throw new UnsupportedOperationException("withinLine not implemented in H3Polygon2D");
-        }
-
-        @Override
-        public WithinRelation withinTriangle(
-            double minX,
-            double maxX,
-            double minY,
-            double maxY,
-            double aX,
-            double aY,
-            boolean ab,
-            double bX,
-            double bY,
-            boolean bc,
-            double cX,
-            double cY,
-            boolean ca
-        ) {
-            throw new UnsupportedOperationException("withinTriangle not implemented in H3Polygon2D");
-        }
-    }
-}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilder.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilder.java
index e50c15ea12c0..53b59d17e284 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilder.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilder.java
@@ -8,7 +8,7 @@
 
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.geo.GeometryParser;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -174,7 +174,7 @@ public static ShapeQueryBuilder fromXContent(XContentParser parser) throws IOExc
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_4_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_4_0;
     }
 }
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/ingest/GeoGridProcessor.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/ingest/GeoGridProcessor.java
new file mode 100644
index 000000000000..4483ea382a82
--- /dev/null
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/ingest/GeoGridProcessor.java
@@ -0,0 +1,374 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.spatial.ingest;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.geo.GeometryParserFormat;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.geometry.utils.Geohash;
+import org.elasticsearch.h3.H3;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.spatial.common.H3CartesianUtil;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import static org.elasticsearch.common.geo.GeometryParserFormat.GEOJSON;
+import static org.elasticsearch.common.geo.GeometryParserFormat.WKT;
+
+/**
+ *  The geo-grid-processor converts a geohash, geotile or geohex tile cell definition into an indexable geometry describing the tile shape.
+ */
+public final class GeoGridProcessor extends AbstractProcessor {
+    public static final String TYPE = "geo_grid";
+
+    private final FieldConfig config;
+    private final boolean ignoreMissing;
+    private final GeometryParserFormat targetFormat;
+    private final TileFieldType tileFieldType;
+
+    GeoGridProcessor(
+        String tag,
+        String description,
+        FieldConfig config,
+        boolean ignoreMissing,
+        GeometryParserFormat targetFormat,
+        TileFieldType tileFieldType
+    ) {
+        super(tag, description);
+        this.config = config;
+        this.ignoreMissing = ignoreMissing;
+        this.targetFormat = targetFormat;
+        this.tileFieldType = tileFieldType;
+    }
+
+    public static class FieldConfig {
+        private final String field;
+        private final String targetField;
+        private final String parentField;
+        private final String childrenField;
+        private final String nonChildrenField;
+        private final String precisionField;
+
+        FieldConfig(
+            String field,
+            String targetField,
+            String parentField,
+            String childrenField,
+            String nonChildrenField,
+            String precisionField
+        ) {
+            this.field = field;
+            this.targetField = targetField;
+            this.parentField = parentField;
+            this.childrenField = childrenField;
+            this.nonChildrenField = nonChildrenField;
+            this.precisionField = precisionField;
+        }
+
+        public String field(String key) {
+            return switch (key) {
+                case "field" -> field;
+                case "target_field" -> targetField;
+                case "parent_field" -> parentField;
+                case "children_field" -> childrenField;
+                case "non_children_field" -> nonChildrenField;
+                case "precision_field" -> precisionField;
+                default -> throw new IllegalArgumentException("Invalid geo_grid processor field type [" + key + "]");
+            };
+        }
+    }
+
+    @Override
+    public IngestDocument execute(IngestDocument ingestDocument) {
+        Object obj = ingestDocument.getFieldValue(config.field, Object.class, ignoreMissing);
+
+        if (obj == null && ignoreMissing) {
+            return ingestDocument;
+        } else if (obj == null) {
+            throw new IllegalArgumentException("field [" + config.field + "] is null, cannot process it.");
+        }
+        if (obj instanceof String == false) {
+            throw new IllegalArgumentException("field [" + config.field + "] must be a String tile address");
+        }
+
+        try {
+            final TileHandler handler = switch (tileFieldType) {
+                case GEOTILE -> new GeotileHandler(obj.toString());
+                case GEOHASH -> new GeohashHandler(obj.toString());
+                case GEOHEX -> new GeohexHandler(obj.toString());
+            };
+            final Geometry geometry = handler.makeGeometry();
+            XContentBuilder newValueBuilder = XContentFactory.jsonBuilder().startObject().field("val");
+            targetFormat.toXContent(geometry, newValueBuilder, ToXContent.EMPTY_PARAMS);
+            newValueBuilder.endObject();
+            Map<String, Object> newObj = XContentHelper.convertToMap(BytesReference.bytes(newValueBuilder), true, XContentType.JSON).v2();
+            ingestDocument.setFieldValue(config.targetField, newObj.get("val"));
+            if (config.parentField != null && config.parentField.length() > 0) {
+                final String parent = handler.getParent();
+                if (parent != null && parent.length() > 0) {
+                    ingestDocument.setFieldValue(config.parentField, parent);
+                }
+            }
+            if (config.childrenField != null && config.childrenField.length() > 0) {
+                final List<String> children = handler.makeChildren();
+                if (children.size() > 0) {
+                    ingestDocument.setFieldValue(config.childrenField, children);
+                }
+            }
+            if (config.nonChildrenField != null && config.nonChildrenField.length() > 0) {
+                final List<String> nonChildren = handler.makeNonChildren();
+                if (nonChildren.size() > 0) {
+                    ingestDocument.setFieldValue(config.nonChildrenField, nonChildren);
+                }
+            }
+            if (config.precisionField != null && config.precisionField.length() > 0) {
+                final int precision = handler.getPrecision();
+                if (precision >= 0) {
+                    ingestDocument.setFieldValue(config.precisionField, precision);
+                }
+            }
+        } catch (Exception e) {
+            throw new IllegalArgumentException("invalid tile definition", e);
+        }
+
+        return ingestDocument;
+    }
+
+    /**
+     * This class handles the different tile types, rectangular or hexagonal.
+     * Each of geohash, geotile and geohex have different ways of describing geometries, and other attributes.
+     */
+    abstract static class TileHandler {
+        abstract Geometry makeGeometry();
+
+        abstract List<String> makeChildren();
+
+        // Most tilers have no intersecting non-children
+        List<String> makeNonChildren() {
+            return Collections.emptyList();
+        }
+
+        abstract int getPrecision();
+
+        abstract String getParent();
+    }
+
+    static class GeotileHandler extends TileHandler {
+        private final String address;
+        private final int zoom;
+        private final int x;
+        private final int y;
+
+        GeotileHandler(String address) {
+            this.address = address;
+            final int[] hashAsInts = GeoTileUtils.parseHash(address);
+            this.zoom = hashAsInts[0];
+            this.x = hashAsInts[1];
+            this.y = hashAsInts[2];
+        }
+
+        @Override
+        Geometry makeGeometry() {
+            return GeoTileUtils.toBoundingBox(address);
+        }
+
+        @Override
+        List<String> makeChildren() {
+            int z = zoom + 1;
+            if (z > GeoTileUtils.MAX_ZOOM) {
+                return Collections.emptyList();
+            }
+            int xo = x * 2;
+            int yo = y * 2;
+            return Arrays.asList(hash(z, xo, yo), hash(z, xo + 1, yo), hash(z, xo, yo + 1), hash(z, xo + 1, yo + 1));
+        }
+
+        private static String hash(int zoom, int x, int y) {
+            return zoom + "/" + x + "/" + y;
+        }
+
+        @Override
+        String getParent() {
+            return zoom == 0 ? "" : hash(zoom - 1, x / 2, y / 2);
+        }
+
+        @Override
+        int getPrecision() {
+            return zoom;
+        }
+    }
+
+    static class GeohashHandler extends TileHandler {
+        private final String hash;
+
+        GeohashHandler(String hash) {
+            this.hash = hash;
+        }
+
+        @Override
+        Geometry makeGeometry() {
+            return Geohash.toBoundingBox(hash);
+        }
+
+        @Override
+        String getParent() {
+            return hash.length() == 0 ? "" : hash.substring(0, hash.length() - 1);
+        }
+
+        @Override
+        List<String> makeChildren() {
+            return Arrays.asList(Geohash.getSubGeohashes(hash));
+        }
+
+        @Override
+        int getPrecision() {
+            return hash.length();
+        }
+    }
+
+    static class GeohexHandler extends TileHandler {
+        private final long h3;
+
+        GeohexHandler(String spec) {
+            this.h3 = H3.stringToH3(spec);
+        }
+
+        @Override
+        Geometry makeGeometry() {
+            return makePolygonFromH3(h3);
+        }
+
+        @Override
+        String getParent() {
+            int res = H3.getResolution(h3);
+            if (res == 0) {
+                return "";
+            }
+            long parent = H3.h3ToParent(h3);
+            return H3.h3ToString(parent);
+        }
+
+        @Override
+        List<String> makeChildren() {
+            return asStringList(H3.h3ToChildren(h3));
+        }
+
+        @Override
+        List<String> makeNonChildren() {
+            return asStringList(H3.h3ToNoChildrenIntersecting(h3));
+        }
+
+        @Override
+        int getPrecision() {
+            return H3.getResolution(h3);
+        }
+
+        private static Geometry makePolygonFromH3(long h3) {
+            return H3CartesianUtil.getNormalizeGeometry(h3);
+        }
+
+        private static List<String> asStringList(long[] cells) {
+            ArrayList<String> addresses = new ArrayList<>(cells.length);
+            for (long cell : cells) {
+                addresses.add(H3.h3ToString(cell));
+            }
+            return addresses;
+        }
+    }
+
+    @Override
+    public String getType() {
+        return TYPE;
+    }
+
+    String field(String key) {
+        return config.field(key);
+    }
+
+    GeometryParserFormat targetFormat() {
+        return targetFormat;
+    }
+
+    TileFieldType tileType() {
+        return tileFieldType;
+    }
+
+    public static final class Factory implements Processor.Factory {
+
+        public GeoGridProcessor create(
+            Map<String, Processor.Factory> registry,
+            String processorTag,
+            String description,
+            Map<String, Object> config
+        ) {
+            String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
+            String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field);
+            String parentField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "parent_field", "");
+            String childrenField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "children_field", "");
+            String nonChildrenField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "non_children_field", "");
+            String precisionField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "precision_field", "");
+            boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
+            TileFieldType tileFieldType = TileFieldType.parse(
+                ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "tile_type"),
+                processorTag
+            );
+            GeometryParserFormat targetFormat = getTargetFormat(processorTag, config);
+            FieldConfig fields = new FieldConfig(field, targetField, parentField, childrenField, nonChildrenField, precisionField);
+            return new GeoGridProcessor(processorTag, description, fields, ignoreMissing, targetFormat, tileFieldType);
+        }
+
+        private GeometryParserFormat getTargetFormat(String processorTag, Map<String, Object> config) {
+            String propertyName = "target_format";
+            String targetFormat = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, propertyName, GEOJSON.name());
+            return switch (targetFormat.toLowerCase(Locale.ROOT).trim()) {
+                case "geojson" -> GEOJSON;
+                case "wkt" -> WKT;
+                default -> throw ConfigurationUtils.newConfigurationException(
+                    TYPE,
+                    processorTag,
+                    propertyName,
+                    "illegal value [" + targetFormat + "], valid values are [WKT, GEOJSON]"
+                );
+            };
+        }
+    }
+
+    enum TileFieldType {
+        GEOHASH,
+        GEOTILE,
+        GEOHEX;
+
+        public static TileFieldType parse(String value, String tag) {
+            EnumSet<TileFieldType> validValues = EnumSet.allOf(TileFieldType.class);
+            try {
+                return valueOf(value.toUpperCase(Locale.ROOT));
+            } catch (IllegalArgumentException e) {
+                throw ConfigurationUtils.newConfigurationException(
+                    TYPE,
+                    tag,
+                    "tile_type",
+                    "illegal value [" + value + "], valid values are " + Arrays.toString(validValues.toArray())
+                );
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregationBuilder.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregationBuilder.java
index 124138cda586..c2f745951c9f 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregationBuilder.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregationBuilder.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.spatial.search.aggregations;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -50,8 +50,8 @@ public class GeoLineAggregationBuilder extends MultiValuesSourceAggregationBuild
     );
     static {
         MultiValuesSourceParseHelper.declareCommon(PARSER, true, ValueType.NUMERIC);
-        MultiValuesSourceParseHelper.declareField(POINT_FIELD.getPreferredName(), PARSER, true, false, false, false);
-        MultiValuesSourceParseHelper.declareField(SORT_FIELD.getPreferredName(), PARSER, true, false, false, false);
+        MultiValuesSourceParseHelper.declareField(POINT_FIELD.getPreferredName(), PARSER, true, false, false, false, false);
+        MultiValuesSourceParseHelper.declareField(SORT_FIELD.getPreferredName(), PARSER, true, false, false, false, false);
         PARSER.declareString((builder, order) -> builder.sortOrder(SortOrder.fromString(order)), ORDER_FIELD);
         PARSER.declareBoolean(GeoLineAggregationBuilder::includeSort, INCLUDE_SORT_FIELD);
         PARSER.declareInt(GeoLineAggregationBuilder::size, SIZE_FIELD);
@@ -174,7 +174,7 @@ public String getType() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_7_11_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_7_11_0;
     }
 }
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/AbstractGeoHashGridTiler.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/AbstractGeoHashGridTiler.java
deleted file mode 100644
index be4ca89e73ba..000000000000
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/AbstractGeoHashGridTiler.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid;
-
-import org.apache.lucene.geo.GeoEncodingUtils;
-import org.elasticsearch.geometry.Rectangle;
-import org.elasticsearch.geometry.utils.Geohash;
-import org.elasticsearch.xpack.spatial.index.fielddata.GeoRelation;
-import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeValues;
-
-import java.io.IOException;
-
-/**
- * Implements most of the logic for the GeoHash aggregation.
- */
-abstract class AbstractGeoHashGridTiler extends GeoGridTiler {
-
-    AbstractGeoHashGridTiler(int precision) {
-        super(precision);
-    }
-
-    /** check if the provided hash is in the solution space of this tiler */
-    protected abstract boolean validHash(String hash);
-
-    @Override
-    public long encode(double x, double y) {
-        return Geohash.longEncode(x, y, precision);
-    }
-
-    @Override
-    public int setValues(GeoShapeCellValues values, GeoShapeValues.GeoShapeValue geoValue) throws IOException {
-
-        if (precision == 0) {
-            return 1;
-        }
-        GeoShapeValues.BoundingBox bounds = geoValue.boundingBox();
-        assert bounds.minX() <= bounds.maxX();
-
-        // When the shape represents a point, we compute the hash directly as we do it for GeoPoint
-        if (bounds.minX() == bounds.maxX() && bounds.minY() == bounds.maxY()) {
-            return setValue(values, geoValue, bounds);
-        }
-        // TODO: optimize for when a shape fits in a single tile an
-        // for when brute-force is expected to be faster than rasterization, which
-        // is when the number of tiles expected is less than the precision
-        return setValuesByRasterization("", values, 0, geoValue);
-    }
-
-    protected int setValuesByBruteForceScan(
-        GeoShapeCellValues values,
-        GeoShapeValues.GeoShapeValue geoValue,
-        GeoShapeValues.BoundingBox bounds
-    ) throws IOException {
-        // TODO: This way to discover cells inside of a bounding box seems not to work as expected. I can
-        // see that eventually we will be visiting twice the same cell which should not happen.
-        int idx = 0;
-        String min = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision);
-        String max = Geohash.stringEncode(bounds.maxX(), bounds.maxY(), precision);
-        String minNeighborBelow = Geohash.getNeighbor(min, precision, 0, -1);
-        double minY = Geohash.decodeLatitude((minNeighborBelow == null) ? min : minNeighborBelow);
-        double minX = Geohash.decodeLongitude(min);
-        double maxY = Geohash.decodeLatitude(max);
-        double maxX = Geohash.decodeLongitude(max);
-        for (double i = minX; i <= maxX; i += Geohash.lonWidthInDegrees(precision)) {
-            for (double j = minY; j <= maxY; j += Geohash.latHeightInDegrees(precision)) {
-                String hash = Geohash.stringEncode(i, j, precision);
-                GeoRelation relation = relateTile(geoValue, hash);
-                if (relation != GeoRelation.QUERY_DISJOINT) {
-                    values.resizeCell(idx + 1);
-                    values.add(idx++, encode(i, j));
-                }
-            }
-        }
-        return idx;
-    }
-
-    /**
-     * Sets a singular doc-value for the {@link GeoShapeValues.GeoShapeValue}.
-     */
-    protected int setValue(GeoShapeCellValues docValues, GeoShapeValues.GeoShapeValue geoValue, GeoShapeValues.BoundingBox bounds)
-        throws IOException {
-        String hash = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision);
-        if (relateTile(geoValue, hash) != GeoRelation.QUERY_DISJOINT) {
-            docValues.resizeCell(1);
-            docValues.add(0, Geohash.longEncode(hash));
-            return 1;
-        }
-        return 0;
-    }
-
-    private GeoRelation relateTile(GeoShapeValues.GeoShapeValue geoValue, String hash) throws IOException {
-        if (validHash(hash)) {
-            final Rectangle rectangle = Geohash.toBoundingBox(hash);
-            int minX = GeoEncodingUtils.encodeLongitude(rectangle.getMinLon());
-            int minY = GeoEncodingUtils.encodeLatitude(rectangle.getMinLat());
-            int maxX = GeoEncodingUtils.encodeLongitude(rectangle.getMaxLon());
-            int maxY = GeoEncodingUtils.encodeLatitude(rectangle.getMaxLat());
-            return geoValue.relate(minX, maxX == Integer.MAX_VALUE ? maxX : maxX - 1, minY, maxY == Integer.MAX_VALUE ? maxY : maxY - 1);
-        }
-        return GeoRelation.QUERY_DISJOINT;
-    }
-
-    protected int setValuesByRasterization(String hash, GeoShapeCellValues values, int valuesIndex, GeoShapeValues.GeoShapeValue geoValue)
-        throws IOException {
-        String[] hashes = Geohash.getSubGeohashes(hash);
-        for (String s : hashes) {
-            GeoRelation relation = relateTile(geoValue, s);
-            if (relation == GeoRelation.QUERY_CROSSES) {
-                if (s.length() == precision) {
-                    values.resizeCell(valuesIndex + 1);
-                    values.add(valuesIndex++, Geohash.longEncode(s));
-                } else {
-                    valuesIndex = setValuesByRasterization(s, values, valuesIndex, geoValue);
-                }
-            } else if (relation == GeoRelation.QUERY_INSIDE) {
-                if (s.length() == precision) {
-                    values.resizeCell(valuesIndex + 1);
-                    values.add(valuesIndex++, Geohash.longEncode(s));
-                } else {
-                    int numTilesAtPrecision = getNumTilesAtPrecision(precision, hash.length());
-                    values.resizeCell(getNewSize(valuesIndex, numTilesAtPrecision + 1));
-                    valuesIndex = setValuesForFullyContainedTile(s, values, valuesIndex, precision);
-                }
-            }
-        }
-        return valuesIndex;
-    }
-
-    private int getNewSize(int valuesIndex, int increment) {
-        long newSize = (long) valuesIndex + increment;
-        if (newSize > Integer.MAX_VALUE) {
-            throw new IllegalArgumentException("Tile aggregation array overflow");
-        }
-        return (int) newSize;
-    }
-
-    private int getNumTilesAtPrecision(int finalPrecision, int currentPrecision) {
-        final long numTilesAtPrecision = Math.min((long) Math.pow(32, finalPrecision - currentPrecision) + 1, getMaxCells());
-        if (numTilesAtPrecision > Integer.MAX_VALUE) {
-            throw new IllegalArgumentException("Tile aggregation array overflow");
-        }
-        return (int) numTilesAtPrecision;
-    }
-
-    protected int setValuesForFullyContainedTile(String hash, GeoShapeCellValues values, int valuesIndex, int targetPrecision) {
-        String[] hashes = Geohash.getSubGeohashes(hash);
-        for (String s : hashes) {
-            if (validHash(s)) {
-                if (s.length() == targetPrecision) {
-                    values.add(valuesIndex++, Geohash.longEncode(s));
-                } else {
-                    valuesIndex = setValuesForFullyContainedTile(s, values, valuesIndex, targetPrecision);
-                }
-            }
-        }
-        return valuesIndex;
-    }
-}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/AbstractGeoTileGridTiler.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/AbstractGeoTileGridTiler.java
deleted file mode 100644
index 68717dd8b68a..000000000000
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/AbstractGeoTileGridTiler.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid;
-
-import org.apache.lucene.geo.GeoEncodingUtils;
-import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
-import org.elasticsearch.xpack.spatial.index.fielddata.GeoRelation;
-import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeValues;
-
-import java.io.IOException;
-
-/**
- * Implements most of the logic for the GeoTile aggregation.
- */
-abstract class AbstractGeoTileGridTiler extends GeoGridTiler {
-
-    protected final long tiles;
-
-    AbstractGeoTileGridTiler(int precision) {
-        super(precision);
-        tiles = 1L << precision;
-    }
-
-    /** check if the provided tile is in the solution space of this tiler */
-    protected abstract boolean validTile(int x, int y, int z);
-
-    @Override
-    public long encode(double x, double y) {
-        return GeoTileUtils.longEncode(x, y, precision);
-    }
-
-    /**
-     * Sets the values of the long[] underlying {@link GeoShapeCellValues}.
-     *
-     * If the shape resides between <code>GeoTileUtils.NORMALIZED_LATITUDE_MASK</code> and 90 or
-     * between <code>GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK</code> and -90 degree latitudes, then
-     * the shape is not accounted for since geo-tiles are only defined within those bounds.
-     *
-     * @param values           the bucket values
-     * @param geoValue         the input shape
-     *
-     * @return the number of tiles set by the shape
-     */
-    @Override
-    public int setValues(GeoShapeCellValues values, GeoShapeValues.GeoShapeValue geoValue) throws IOException {
-        GeoShapeValues.BoundingBox bounds = geoValue.boundingBox();
-        assert bounds.minX() <= bounds.maxX();
-
-        // geo tiles are not defined at the extreme latitudes due to them
-        // tiling the world as a square.
-        if (bounds.bottom > GeoTileUtils.NORMALIZED_LATITUDE_MASK || bounds.top < GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK) {
-            return 0;
-        }
-
-        if (precision == 0) {
-            return setValuesByBruteForceScan(values, geoValue, 0, 0, 0, 0);
-        }
-
-        final int minXTile = GeoTileUtils.getXTile(bounds.minX(), tiles);
-        final int minYTile = GeoTileUtils.getYTile(bounds.maxY(), tiles);
-        final int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), tiles);
-        final int maxYTile = GeoTileUtils.getYTile(bounds.minY(), tiles);
-        final long count = (long) (maxXTile - minXTile + 1) * (maxYTile - minYTile + 1);
-        if (count == 1) {
-            return setValue(values, minXTile, minYTile);
-        } else if (count <= precision) {
-            return setValuesByBruteForceScan(values, geoValue, minXTile, minYTile, maxXTile, maxYTile);
-        } else {
-            return setValuesByRasterization(0, 0, 0, values, 0, geoValue);
-        }
-    }
-
-    private GeoRelation relateTile(GeoShapeValues.GeoShapeValue geoValue, int xTile, int yTile, int precision) throws IOException {
-        if (validTile(xTile, yTile, precision)) {
-            final double tiles = 1 << precision;
-            final int minX = GeoEncodingUtils.encodeLongitude(GeoTileUtils.tileToLon(xTile, tiles));
-            final int maxX = GeoEncodingUtils.encodeLongitude(GeoTileUtils.tileToLon(xTile + 1, tiles));
-            final int minY = GeoEncodingUtils.encodeLatitude(GeoTileUtils.tileToLat(yTile + 1, tiles));
-            final int maxY = GeoEncodingUtils.encodeLatitude(GeoTileUtils.tileToLat(yTile, tiles));
-            return geoValue.relate(
-                minX,
-                maxX == Integer.MAX_VALUE ? maxX : maxX - 1,
-                minY == GeoTileUtils.ENCODED_NEGATIVE_LATITUDE_MASK ? minY : minY + 1,
-                maxY
-            );
-        }
-        return GeoRelation.QUERY_DISJOINT;
-    }
-
-    /**
-     * Sets a singular doc-value with the provided x/y.
-     */
-    protected int setValue(GeoShapeCellValues docValues, int xTile, int yTile) {
-        if (validTile(xTile, yTile, precision)) {
-            docValues.resizeCell(1);
-            docValues.add(0, GeoTileUtils.longEncodeTiles(precision, xTile, yTile));
-            return 1;
-        }
-        return 0;
-    }
-
-    /**
-     *
-     * @param values the bucket values as longs
-     * @param geoValue the shape value
-     * @return the number of buckets the geoValue is found in
-     */
-    protected int setValuesByBruteForceScan(
-        GeoShapeCellValues values,
-        GeoShapeValues.GeoShapeValue geoValue,
-        int minXTile,
-        int minYTile,
-        int maxXTile,
-        int maxYTile
-    ) throws IOException {
-        int idx = 0;
-        for (int i = minXTile; i <= maxXTile; i++) {
-            for (int j = minYTile; j <= maxYTile; j++) {
-                GeoRelation relation = relateTile(geoValue, i, j, precision);
-                if (relation != GeoRelation.QUERY_DISJOINT) {
-                    values.resizeCell(idx + 1);
-                    values.add(idx++, GeoTileUtils.longEncodeTiles(precision, i, j));
-                }
-            }
-        }
-        return idx;
-    }
-
-    protected int setValuesByRasterization(
-        int xTile,
-        int yTile,
-        int zTile,
-        GeoShapeCellValues values,
-        int valuesIndex,
-        GeoShapeValues.GeoShapeValue geoValue
-    ) throws IOException {
-        zTile++;
-        for (int i = 0; i < 2; i++) {
-            for (int j = 0; j < 2; j++) {
-                int nextX = 2 * xTile + i;
-                int nextY = 2 * yTile + j;
-                GeoRelation relation = relateTile(geoValue, nextX, nextY, zTile);
-                if (GeoRelation.QUERY_INSIDE == relation) {
-                    if (zTile == precision) {
-                        values.resizeCell(getNewSize(valuesIndex, 1));
-                        values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY));
-                    } else {
-                        int numTilesAtPrecision = getNumTilesAtPrecision(precision, zTile);
-                        values.resizeCell(getNewSize(valuesIndex, numTilesAtPrecision + 1));
-                        valuesIndex = setValuesForFullyContainedTile(nextX, nextY, zTile, values, valuesIndex);
-                    }
-                } else if (GeoRelation.QUERY_CROSSES == relation) {
-                    if (zTile == precision) {
-                        values.resizeCell(getNewSize(valuesIndex, 1));
-                        values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY));
-                    } else {
-                        valuesIndex = setValuesByRasterization(nextX, nextY, zTile, values, valuesIndex, geoValue);
-                    }
-                }
-            }
-        }
-        return valuesIndex;
-    }
-
-    private int getNewSize(int valuesIndex, int increment) {
-        long newSize = (long) valuesIndex + increment;
-        if (newSize > Integer.MAX_VALUE) {
-            throw new IllegalArgumentException("Tile aggregation array overflow");
-        }
-        return (int) newSize;
-    }
-
-    private int getNumTilesAtPrecision(int finalPrecision, int currentPrecision) {
-        final long numTilesAtPrecision = Math.min(1L << (2 * (finalPrecision - currentPrecision)), getMaxCells());
-        if (numTilesAtPrecision > Integer.MAX_VALUE) {
-            throw new IllegalArgumentException("Tile aggregation array overflow");
-        }
-        return (int) numTilesAtPrecision;
-    }
-
-    protected abstract int setValuesForFullyContainedTile(int xTile, int yTile, int zTile, GeoShapeCellValues values, int valuesIndex);
-}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/BoundedGeoHashGridTiler.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/BoundedGeoHashGridTiler.java
deleted file mode 100644
index 3973da5700f2..000000000000
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/BoundedGeoHashGridTiler.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid;
-
-import org.elasticsearch.common.geo.GeoBoundingBox;
-import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashBoundedPredicate;
-
-/**
- * Bounded geotile aggregation. It accepts hashes that intersects the provided bounds.
- */
-public class BoundedGeoHashGridTiler extends AbstractGeoHashGridTiler {
-    private final GeoHashBoundedPredicate predicate;
-
-    public BoundedGeoHashGridTiler(int precision, GeoBoundingBox bbox) {
-        super(precision);
-        this.predicate = new GeoHashBoundedPredicate(precision, bbox);
-    }
-
-    @Override
-    protected long getMaxCells() {
-        return predicate.getMaxHashes();
-    }
-
-    @Override
-    protected boolean validHash(String hash) {
-        return predicate.validHash(hash);
-    }
-}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/BoundedGeoTileGridTiler.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/BoundedGeoTileGridTiler.java
deleted file mode 100644
index d8bfb5207c95..000000000000
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/BoundedGeoTileGridTiler.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid;
-
-import org.elasticsearch.common.geo.GeoBoundingBox;
-import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileBoundedPredicate;
-import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
-
-/**
- * Bounded geotile aggregation. It accepts tiles that intersects the provided bounds.
- */
-public class BoundedGeoTileGridTiler extends AbstractGeoTileGridTiler {
-
-    private final GeoTileBoundedPredicate predicate;
-
-    public BoundedGeoTileGridTiler(int precision, GeoBoundingBox bbox) {
-        super(precision);
-        this.predicate = new GeoTileBoundedPredicate(precision, bbox);
-    }
-
-    @Override
-    protected boolean validTile(int x, int y, int z) {
-        return predicate.validTile(x, y, z);
-    }
-
-    @Override
-    protected long getMaxCells() {
-        return predicate.getMaxTiles();
-    }
-
-    @Override
-    @SuppressWarnings("HiddenField")
-    protected int setValuesForFullyContainedTile(int xTile, int yTile, int zTile, GeoShapeCellValues values, int valuesIndex) {
-        // For every level we go down, we half each dimension. The total number of splits is equal to 1 << (levelEnd - levelStart)
-        final int splits = 1 << precision - zTile;
-        // The start value of a dimension is calculated by multiplying the value of that dimension at the start level
-        // by the number of splits. Choose the max value with respect to the bounding box.
-        final int minY = Math.max(predicate.minY(), yTile * splits);
-        // The end value of a dimension is calculated by adding to the start value the number of splits.
-        // Choose the min value with respect to the bounding box.
-        final int maxY = Math.min(predicate.maxY(), yTile * splits + splits);
-        // Do the same for the X dimension taking into account that the bounding box might cross the dateline.
-        if (predicate.crossesDateline()) {
-            final int eastMinX = xTile * splits;
-            final int westMinX = Math.max(predicate.leftX(), xTile * splits);
-            // when the left and right box land in the same tile, we need to make sure we don't count then twice
-            final int eastMaxX = Math.min(westMinX, Math.min(predicate.rightX(), xTile * splits + splits));
-            final int westMaxX = xTile * splits + splits;
-            for (int i = eastMinX; i < eastMaxX; i++) {
-                for (int j = minY; j < maxY; j++) {
-                    assert predicate.validTile(i, j, precision);
-                    values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(precision, i, j));
-                }
-            }
-            for (int i = westMinX; i < westMaxX; i++) {
-                for (int j = minY; j < maxY; j++) {
-                    assert predicate.validTile(i, j, precision);
-                    values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(precision, i, j));
-                }
-            }
-        } else {
-            final int _minX = Math.max(predicate.leftX(), xTile * splits);
-            final int _maxX = Math.min(predicate.rightX(), xTile * splits + splits);
-            for (int i = _minX; i < _maxX; i++) {
-                for (int j = minY; j < maxY; j++) {
-                    assert predicate.validTile(i, j, precision);
-                    values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(precision, i, j));
-                }
-            }
-        }
-        return valuesIndex;
-    }
-}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/ByteTrackingSortingNumericDocValues.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/ByteTrackingSortingNumericDocValues.java
deleted file mode 100644
index 27efee3dfd43..000000000000
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/ByteTrackingSortingNumericDocValues.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid;
-
-import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues;
-
-import java.util.function.LongConsumer;
-
-/**
- * Wrapper class for GeoGrid to expose the protected values array for testing
- */
-abstract class ByteTrackingSortingNumericDocValues extends AbstractSortingNumericDocValues {
-
-    ByteTrackingSortingNumericDocValues(LongConsumer circuitBreakerConsumer) {
-        super(circuitBreakerConsumer);
-    }
-
-    long getValuesBytes() {
-        return values.length * Long.BYTES;
-    }
-}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHashGridTiler.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHashGridTiler.java
new file mode 100644
index 000000000000..8afbc2b90db5
--- /dev/null
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHashGridTiler.java
@@ -0,0 +1,220 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid;
+
+import org.apache.lucene.geo.GeoEncodingUtils;
+import org.elasticsearch.common.geo.GeoBoundingBox;
+import org.elasticsearch.geometry.Rectangle;
+import org.elasticsearch.geometry.utils.Geohash;
+import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashBoundedPredicate;
+import org.elasticsearch.xpack.spatial.index.fielddata.GeoRelation;
+import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeValues;
+
+import java.io.IOException;
+
+/**
+ * Implements the logic for the Geohash aggregation over a geoshape doc value.
+ */
+public abstract class GeoHashGridTiler extends GeoGridTiler {
+
+    private GeoHashGridTiler(int precision) {
+        super(precision);
+    }
+
+    /** Factory method to create GeoHashGridTiler objects */
+    public static GeoHashGridTiler makeGridTiler(int precision, GeoBoundingBox geoBoundingBox) {
+        return geoBoundingBox == null || geoBoundingBox.isUnbounded()
+            ? new UnboundedGeoHashGridTiler(precision)
+            : new BoundedGeoHashGridTiler(precision, geoBoundingBox);
+    }
+
+    /** check if the provided hash is in the solution space of this tiler */
+    protected abstract boolean validHash(String hash);
+
+    @Override
+    public long encode(double x, double y) {
+        return Geohash.longEncode(x, y, precision);
+    }
+
+    @Override
+    public int setValues(GeoShapeCellValues values, GeoShapeValues.GeoShapeValue geoValue) throws IOException {
+
+        if (precision == 0) {
+            return 1;
+        }
+        GeoShapeValues.BoundingBox bounds = geoValue.boundingBox();
+        assert bounds.minX() <= bounds.maxX();
+
+        // When the shape represents a point, we compute the hash directly as we do it for GeoPoint
+        if (bounds.minX() == bounds.maxX() && bounds.minY() == bounds.maxY()) {
+            return setValue(values, geoValue, bounds);
+        }
+        // TODO: optimize for when a shape fits in a single tile an
+        // for when brute-force is expected to be faster than rasterization, which
+        // is when the number of tiles expected is less than the precision
+        return setValuesByRasterization("", values, 0, geoValue);
+    }
+
+    int setValuesByBruteForceScan(GeoShapeCellValues values, GeoShapeValues.GeoShapeValue geoValue, GeoShapeValues.BoundingBox bounds)
+        throws IOException {
+        // TODO: This way to discover cells inside of a bounding box seems not to work as expected. I can
+        // see that eventually we will be visiting twice the same cell which should not happen.
+        int idx = 0;
+        String min = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision);
+        String max = Geohash.stringEncode(bounds.maxX(), bounds.maxY(), precision);
+        String minNeighborBelow = Geohash.getNeighbor(min, precision, 0, -1);
+        double minY = Geohash.decodeLatitude((minNeighborBelow == null) ? min : minNeighborBelow);
+        double minX = Geohash.decodeLongitude(min);
+        double maxY = Geohash.decodeLatitude(max);
+        double maxX = Geohash.decodeLongitude(max);
+        for (double i = minX; i <= maxX; i += Geohash.lonWidthInDegrees(precision)) {
+            for (double j = minY; j <= maxY; j += Geohash.latHeightInDegrees(precision)) {
+                String hash = Geohash.stringEncode(i, j, precision);
+                GeoRelation relation = relateTile(geoValue, hash);
+                if (relation != GeoRelation.QUERY_DISJOINT) {
+                    values.resizeCell(idx + 1);
+                    values.add(idx++, encode(i, j));
+                }
+            }
+        }
+        return idx;
+    }
+
+    /**
+     * Sets a singular doc-value for the {@link GeoShapeValues.GeoShapeValue}.
+     */
+    private int setValue(GeoShapeCellValues docValues, GeoShapeValues.GeoShapeValue geoValue, GeoShapeValues.BoundingBox bounds)
+        throws IOException {
+        String hash = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision);
+        if (relateTile(geoValue, hash) != GeoRelation.QUERY_DISJOINT) {
+            docValues.resizeCell(1);
+            docValues.add(0, Geohash.longEncode(hash));
+            return 1;
+        }
+        return 0;
+    }
+
+    private GeoRelation relateTile(GeoShapeValues.GeoShapeValue geoValue, String hash) throws IOException {
+        if (validHash(hash)) {
+            final Rectangle rectangle = Geohash.toBoundingBox(hash);
+            int minX = GeoEncodingUtils.encodeLongitude(rectangle.getMinLon());
+            int minY = GeoEncodingUtils.encodeLatitude(rectangle.getMinLat());
+            int maxX = GeoEncodingUtils.encodeLongitude(rectangle.getMaxLon());
+            int maxY = GeoEncodingUtils.encodeLatitude(rectangle.getMaxLat());
+            return geoValue.relate(minX, maxX == Integer.MAX_VALUE ? maxX : maxX - 1, minY, maxY == Integer.MAX_VALUE ? maxY : maxY - 1);
+        }
+        return GeoRelation.QUERY_DISJOINT;
+    }
+
+    /**
+     * Recursively search the geohash tree, only following branches that intersect the geometry.
+     * Once at the required depth, then all cells that intersect are added to the collection.
+     */
+    // pkg protected for testing
+    int setValuesByRasterization(String hash, GeoShapeCellValues values, int valuesIndex, GeoShapeValues.GeoShapeValue geoValue)
+        throws IOException {
+        String[] hashes = Geohash.getSubGeohashes(hash);
+        for (String s : hashes) {
+            GeoRelation relation = relateTile(geoValue, s);
+            if (relation == GeoRelation.QUERY_INSIDE) {
+                if (s.length() == precision) {
+                    values.resizeCell(valuesIndex + 1);
+                    values.add(valuesIndex++, Geohash.longEncode(s));
+                } else {
+                    int numTilesAtPrecision = getNumTilesAtPrecision(precision, hash.length());
+                    values.resizeCell(getNewSize(valuesIndex, numTilesAtPrecision + 1));
+                    valuesIndex = setValuesForFullyContainedTile(s, values, valuesIndex, precision);
+                }
+            } else if (relation != GeoRelation.QUERY_DISJOINT) {
+                if (s.length() == precision) {
+                    values.resizeCell(valuesIndex + 1);
+                    values.add(valuesIndex++, Geohash.longEncode(s));
+                } else {
+                    valuesIndex = setValuesByRasterization(s, values, valuesIndex, geoValue);
+                }
+            }
+        }
+        return valuesIndex;
+    }
+
+    private int getNewSize(int valuesIndex, int increment) {
+        long newSize = (long) valuesIndex + increment;
+        if (newSize > Integer.MAX_VALUE) {
+            throw new IllegalArgumentException("Tile aggregation array overflow");
+        }
+        return (int) newSize;
+    }
+
+    private int getNumTilesAtPrecision(int finalPrecision, int currentPrecision) {
+        final long numTilesAtPrecision = Math.min((long) Math.pow(32, finalPrecision - currentPrecision) + 1, getMaxCells());
+        if (numTilesAtPrecision > Integer.MAX_VALUE) {
+            throw new IllegalArgumentException("Tile aggregation array overflow");
+        }
+        return (int) numTilesAtPrecision;
+    }
+
+    // pkg protected for testing
+    int setValuesForFullyContainedTile(String hash, GeoShapeCellValues values, int valuesIndex, int targetPrecision) {
+        String[] hashes = Geohash.getSubGeohashes(hash);
+        for (String s : hashes) {
+            if (validHash(s)) {
+                if (s.length() == targetPrecision) {
+                    values.add(valuesIndex++, Geohash.longEncode(s));
+                } else {
+                    valuesIndex = setValuesForFullyContainedTile(s, values, valuesIndex, targetPrecision);
+                }
+            }
+        }
+        return valuesIndex;
+    }
+
+    /**
+     * Bounded geotile aggregation. It accepts hashes that intersects the provided bounds.
+     */
+    private static class BoundedGeoHashGridTiler extends GeoHashGridTiler {
+        private final GeoHashBoundedPredicate predicate;
+
+        BoundedGeoHashGridTiler(int precision, GeoBoundingBox bbox) {
+            super(precision);
+            this.predicate = new GeoHashBoundedPredicate(precision, bbox);
+        }
+
+        @Override
+        protected long getMaxCells() {
+            return predicate.getMaxHashes();
+        }
+
+        @Override
+        protected boolean validHash(String hash) {
+            return predicate.validHash(hash);
+        }
+    }
+
+    /**
+     * Unbounded geohash aggregation. It accepts any hash.
+     */
+    private static class UnboundedGeoHashGridTiler extends GeoHashGridTiler {
+
+        private final long maxHashes;
+
+        UnboundedGeoHashGridTiler(int precision) {
+            super(precision);
+            this.maxHashes = (long) Math.pow(32, precision);
+        }
+
+        @Override
+        protected boolean validHash(String hash) {
+            return true;
+        }
+
+        @Override
+        protected long getMaxCells() {
+            return maxHashes;
+        }
+    }
+}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexCellIdSource.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexCellIdSource.java
index 06507379be98..2b4c95e2c33f 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexCellIdSource.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexCellIdSource.java
@@ -9,19 +9,21 @@
 import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.elasticsearch.common.geo.GeoBoundingBox;
-import org.elasticsearch.h3.CellBoundary;
 import org.elasticsearch.h3.H3;
 import org.elasticsearch.index.fielddata.GeoPointValues;
 import org.elasticsearch.index.fielddata.MultiGeoPointValues;
 import org.elasticsearch.search.aggregations.bucket.geogrid.CellIdSource;
+import org.elasticsearch.xpack.spatial.common.H3SphericalUtil;
+
+import java.util.function.LongConsumer;
 
 /**
-* {@link CellIdSource} implementation for GeoHex aggregation
-*/
+ * {@link CellIdSource} implementation for GeoHex aggregation
+ */
 public class GeoHexCellIdSource extends CellIdSource {
 
-    public GeoHexCellIdSource(GeoPoint valuesSource, int precision, GeoBoundingBox geoBoundingBox) {
-        super(valuesSource, precision, geoBoundingBox);
+    public GeoHexCellIdSource(GeoPoint valuesSource, int precision, GeoBoundingBox geoBoundingBox, LongConsumer circuitBreakerConsumer) {
+        super(valuesSource, precision, geoBoundingBox, circuitBreakerConsumer);
     }
 
     @Override
@@ -37,15 +39,16 @@ protected boolean advance(org.elasticsearch.common.geo.GeoPoint target) {
 
     @Override
     protected NumericDocValues boundedCellSingleValue(GeoPointValues values, GeoBoundingBox boundingBox) {
-        final GeoHexPredicate predicate = new GeoHexPredicate(boundingBox, precision());
+        final GeoHexPredicate predicate = new GeoHexPredicate(boundingBox);
         return new CellSingleValue(values, precision()) {
             @Override
             protected boolean advance(org.elasticsearch.common.geo.GeoPoint target) {
                 final double lat = target.getLat();
                 final double lon = target.getLon();
                 final long hex = H3.geoToH3(lat, lon, precision);
-                // validPoint is a fast check, validHex is slow
-                if (validPoint(lon, lat) || predicate.validHex(hex)) {
+                // pointInBounds is a fast check, validHex is slow
+                if (pointInBounds(lon, lat) || predicate.validHex(hex)) {
+                    assert predicate.validHex(hex) : H3.h3ToString(hex) + " should be valid but it is not";
                     value = hex;
                     return true;
                 }
@@ -56,7 +59,7 @@ protected boolean advance(org.elasticsearch.common.geo.GeoPoint target) {
 
     @Override
     protected SortedNumericDocValues unboundedCellMultiValues(MultiGeoPointValues values) {
-        return new CellMultiValues(values, precision()) {
+        return new CellMultiValues(values, precision(), circuitBreakerConsumer) {
             @Override
             protected int advanceValue(org.elasticsearch.common.geo.GeoPoint target, int valuesIdx) {
                 values[valuesIdx] = H3.geoToH3(target.getLat(), target.getLon(), precision);
@@ -67,15 +70,15 @@ protected int advanceValue(org.elasticsearch.common.geo.GeoPoint target, int val
 
     @Override
     protected SortedNumericDocValues boundedCellMultiValues(MultiGeoPointValues values, GeoBoundingBox boundingBox) {
-        final GeoHexPredicate predicate = new GeoHexPredicate(boundingBox, precision());
-        return new CellMultiValues(values, precision()) {
+        final GeoHexPredicate predicate = new GeoHexPredicate(boundingBox);
+        return new CellMultiValues(values, precision(), circuitBreakerConsumer) {
             @Override
             protected int advanceValue(org.elasticsearch.common.geo.GeoPoint target, int valuesIdx) {
                 final double lat = target.getLat();
                 final double lon = target.getLon();
                 final long hex = H3.geoToH3(lat, lon, precision);
                 // validPoint is a fast check, validHex is slow
-                if (validPoint(lon, lat) || predicate.validHex(hex)) {
+                if (pointInBounds(lon, lat) || predicate.validHex(hex)) {
                     values[valuesIdx] = hex;
                     return valuesIdx + 1;
                 }
@@ -87,50 +90,32 @@ protected int advanceValue(org.elasticsearch.common.geo.GeoPoint target, int val
     private static class GeoHexPredicate {
 
         private final boolean crossesDateline;
-        private final GeoBoundingBox bbox;
-        private final long northPoleHex, southPoleHex;
+        private final GeoBoundingBox bbox, scratch;
 
-        GeoHexPredicate(GeoBoundingBox bbox, int precision) {
+        GeoHexPredicate(GeoBoundingBox bbox) {
             this.crossesDateline = bbox.right() < bbox.left();
             this.bbox = bbox;
-            northPoleHex = H3.geoToH3(90, 0, precision);
-            southPoleHex = H3.geoToH3(-90, 0, precision);
+            scratch = new GeoBoundingBox(new org.elasticsearch.common.geo.GeoPoint(), new org.elasticsearch.common.geo.GeoPoint());
         }
 
         public boolean validHex(long hex) {
-            CellBoundary boundary = H3.h3ToGeoBoundary(hex);
-            double minLat = Double.POSITIVE_INFINITY;
-            double minLon = Double.POSITIVE_INFINITY;
-            double maxLat = Double.NEGATIVE_INFINITY;
-            double maxLon = Double.NEGATIVE_INFINITY;
-            for (int i = 0; i < boundary.numPoints(); i++) {
-                double boundaryLat = boundary.getLatLon(i).getLatDeg();
-                double boundaryLon = boundary.getLatLon(i).getLonDeg();
-                minLon = Math.min(minLon, boundaryLon);
-                maxLon = Math.max(maxLon, boundaryLon);
-                minLat = Math.min(minLat, boundaryLat);
-                maxLat = Math.max(maxLat, boundaryLat);
-            }
-            if (northPoleHex == hex) {
-                return minLat < bbox.top();
-            } else if (southPoleHex == hex) {
-                return maxLat > bbox.bottom();
-            } else if (maxLon - minLon > 180) {
-                return intersects(-180, minLon, minLat, maxLat) || intersects(maxLon, 180, minLat, maxLat);
-            } else {
-                return intersects(minLon, maxLon, minLat, maxLat);
-            }
-        }
-
-        private boolean intersects(double minLon, double maxLon, double minLat, double maxLat) {
-            if (bbox.top() > minLat && bbox.bottom() < maxLat) {
-                if (crossesDateline) {
-                    return bbox.left() < maxLon || bbox.right() > minLon;
+            H3SphericalUtil.computeGeoBounds(hex, scratch);
+            if (bbox.top() > scratch.bottom() && bbox.bottom() < scratch.top()) {
+                if (scratch.left() > scratch.right()) {
+                    return intersects(-180, scratch.right()) || intersects(scratch.left(), 180);
                 } else {
-                    return bbox.left() < maxLon && bbox.right() > minLon;
+                    return intersects(scratch.left(), scratch.right());
                 }
             }
             return false;
         }
+
+        private boolean intersects(double minLon, double maxLon) {
+            if (crossesDateline) {
+                return bbox.left() < maxLon || bbox.right() > minLon;
+            } else {
+                return bbox.left() < maxLon && bbox.right() > minLon;
+            }
+        }
     }
 }
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridAggregationBuilder.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridAggregationBuilder.java
index 76525b107640..ad5a4371114f 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridAggregationBuilder.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridAggregationBuilder.java
@@ -8,7 +8,7 @@
 package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.geo.GeoBoundingBox;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
@@ -123,7 +123,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_1_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_1_0;
     }
 }
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridAggregator.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridAggregator.java
index 5f5239d1624a..1c1ca87970a3 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridAggregator.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridAggregator.java
@@ -19,6 +19,8 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
+import java.util.function.LongConsumer;
 
 /**
  * Aggregates data expressed as h3 longs (for efficiency's sake)
@@ -29,7 +31,7 @@ public class GeoHexGridAggregator extends GeoGridAggregator<InternalGeoHexGrid>
     public GeoHexGridAggregator(
         String name,
         AggregatorFactories factories,
-        ValuesSource.Numeric valuesSource,
+        Function<LongConsumer, ValuesSource.Numeric> valuesSource,
         int requiredSize,
         int shardSize,
         AggregationContext context,
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridTiler.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridTiler.java
new file mode 100644
index 000000000000..5073cb44dd5d
--- /dev/null
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridTiler.java
@@ -0,0 +1,414 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid;
+
+import org.elasticsearch.common.geo.GeoBoundingBox;
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.geo.GeoUtils;
+import org.elasticsearch.geometry.Rectangle;
+import org.elasticsearch.h3.H3;
+import org.elasticsearch.xpack.spatial.common.H3CartesianUtil;
+import org.elasticsearch.xpack.spatial.index.fielddata.GeoRelation;
+import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeValues;
+
+import java.io.IOException;
+
+/**
+ * Implements the logic for the GeoHex aggregation over a geoshape doc value.
+ */
+public abstract class GeoHexGridTiler extends GeoGridTiler {
+
+    private static final long[] RES0CELLS = H3.getLongRes0Cells();
+
+    private GeoHexGridTiler(int precision) {
+        super(precision);
+    }
+
+    /** Factory method to create GeoHexGridTiler objects */
+    public static GeoHexGridTiler makeGridTiler(int precision, GeoBoundingBox geoBoundingBox) {
+        return geoBoundingBox == null || geoBoundingBox.isUnbounded()
+            ? new GeoHexGridTiler.UnboundedGeoHexGridTiler(precision)
+            : new GeoHexGridTiler.BoundedGeoHexGridTiler(precision, geoBoundingBox);
+    }
+
+    /** check if the provided H3 bin is in the solution space of this tiler */
+    protected abstract boolean h3IntersectsBounds(long h3);
+
+    /** Return the relation between the H3 bin and the geoValue. If the h3 is out of the tiler solution (e.g.
+     * {@link #h3IntersectsBounds(long)} is false), it should return {@link GeoRelation#QUERY_DISJOINT}
+    */
+    protected abstract GeoRelation relateTile(GeoShapeValues.GeoShapeValue geoValue, long h3) throws IOException;
+
+    /** Return true if the provided {@link GeoShapeValues.GeoShapeValue} is fully contained in our solution space.
+     */
+    protected abstract boolean valueInsideBounds(GeoShapeValues.GeoShapeValue geoValue) throws IOException;
+
+    @Override
+    public long encode(double x, double y) {
+        // TODO: maybe we should remove this method from the API
+        throw new IllegalArgumentException("no supported");
+    }
+
+    @Override
+    public int setValues(GeoShapeCellValues values, GeoShapeValues.GeoShapeValue geoValue) throws IOException {
+        final GeoShapeValues.BoundingBox bounds = geoValue.boundingBox();
+        assert bounds.minX() <= bounds.maxX();
+        // first check if we are touching just fetch cells
+        if (bounds.maxX() - bounds.minX() < 180d) {
+            final long singleCell = boundsInSameCell(bounds, precision);
+            if (singleCell > 0) {
+                return setValuesFromPointResolution(singleCell, values, geoValue);
+            }
+            // TODO: specialize when they are neighbour cells.
+        }
+        // recurse tree
+        return setValuesByRecursion(values, geoValue, bounds);
+    }
+
+    /**
+     * It calls {@link #maybeAdd(long, GeoRelation, GeoShapeCellValues, int)} for {@code h3} and the neighbour cells if necessary.
+     */
+    private int setValuesFromPointResolution(long h3, GeoShapeCellValues values, GeoShapeValues.GeoShapeValue geoValue) throws IOException {
+        int valueIndex = 0;
+        {
+            final GeoRelation relation = relateTile(geoValue, h3);
+            valueIndex = maybeAdd(h3, relation, values, valueIndex);
+            if (relation == GeoRelation.QUERY_CONTAINS) {
+                return valueIndex;
+            }
+        }
+        // Point resolution is done using H3 library which uses spherical geometry. It might happen that in cartesian, the
+        // actual point value is in a neighbour cell as well.
+        {
+            final int ringSize = H3.hexRingSize(h3);
+            for (int i = 0; i < ringSize; i++) {
+                final long n = H3.hexRingPosToH3(h3, i);
+                final GeoRelation relation = relateTile(geoValue, n);
+                valueIndex = maybeAdd(n, relation, values, valueIndex);
+                if (relation == GeoRelation.QUERY_CONTAINS) {
+                    return valueIndex;
+                }
+            }
+        }
+        return valueIndex;
+    }
+
+    private long boundsInSameCell(GeoShapeValues.BoundingBox bounds, int res) {
+        final long minH3 = H3.geoToH3(bounds.minY(), bounds.minX(), res);
+        final long maxH3 = H3.geoToH3(bounds.maxY(), bounds.maxX(), res);
+        if (minH3 != maxH3) {
+            // Normally sufficient to check only bottom-left against top-right
+            return -1;
+        }
+        if (H3CartesianUtil.isPolar(minH3)) {
+            // But with polar cells we must check the other two corners too
+            final long minMax = H3.geoToH3(bounds.minY(), bounds.maxX(), res);
+            final long maxMin = H3.geoToH3(bounds.maxY(), bounds.minX(), res);
+            if (minMax != minH3 || maxMin != minH3) {
+                return -1;
+            }
+        }
+        // If all checks passed, we can use this cell in an optimization
+        return minH3;
+    }
+
+    /**
+     * Adds {@code h3} to {@link GeoShapeCellValues} if {@link #relateTile(GeoShapeValues.GeoShapeValue, long)} returns
+     * a relation different to {@link GeoRelation#QUERY_DISJOINT}.
+     */
+    private int maybeAdd(long h3, GeoRelation relation, GeoShapeCellValues values, int valueIndex) {
+        if (relation != GeoRelation.QUERY_DISJOINT) {
+            values.resizeCell(valueIndex + 1);
+            values.add(valueIndex++, h3);
+        }
+        return valueIndex;
+    }
+
+    /**
+     * Recursively search the H3 tree, only following branches that intersect the geometry.
+     * Once at the required depth, then all cells that intersect are added to the collection.
+     */
+    // package private for testing
+    int setValuesByRecursion(GeoShapeCellValues values, GeoShapeValues.GeoShapeValue geoValue, GeoShapeValues.BoundingBox bounds)
+        throws IOException {
+        // NOTE: When we recurse, we cannot shortcut for CONTAINS relationship because it might fail when visiting noChilds.
+        int valueIndex = 0;
+        if (bounds.maxX() - bounds.minX() < 180d) {
+            final long singleCell = boundsInSameCell(bounds, 0);
+            if (singleCell > 0) {
+                // When the level 0 bounds are within a single cell, we can search that cell and its immediate neighbours
+                valueIndex = setValuesByRecursion(values, geoValue, singleCell, 0, valueIndex);
+                final int ringSize = H3.hexRingSize(singleCell);
+                for (int i = 0; i < ringSize; i++) {
+                    valueIndex = setValuesByRecursion(values, geoValue, H3.hexRingPosToH3(singleCell, i), 0, valueIndex);
+                }
+                return valueIndex;
+            }
+            // TODO: specialize when they are neighbour cells.
+        }
+        for (long h3 : RES0CELLS) {
+            valueIndex = setValuesByRecursion(values, geoValue, h3, 0, valueIndex);
+        }
+        return valueIndex;
+    }
+
+    /**
+     * Recursively search the H3 tree, only following branches that intersect the geometry.
+     * Once at the required depth, then all cells that intersect are added to the collection.
+     */
+    private int setValuesByRecursion(
+        GeoShapeCellValues values,
+        GeoShapeValues.GeoShapeValue geoValue,
+        long h3,
+        int precision,
+        int valueIndex
+    ) throws IOException {
+        assert H3.getResolution(h3) == precision;
+        final GeoRelation relation = relateTile(geoValue, h3);
+        if (precision == this.precision) {
+            // When we're at the desired level
+            return maybeAdd(h3, relation, values, valueIndex);
+        } else {
+            assert precision < this.precision;
+            // When we're at higher tree levels, check if we want to keep iterating.
+            if (relation != GeoRelation.QUERY_DISJOINT) {
+                int i = 0;
+                if (relation == GeoRelation.QUERY_INSIDE) {
+                    // H3 cells do not fully contain the children. The only one we know we fully contain
+                    // is the center child which is always at position 0.
+                    final long centerChild = H3.childPosToH3(h3, i++);
+                    valueIndex = setAllValuesByRecursion(values, centerChild, precision + 1, valueIndex, valueInsideBounds(geoValue));
+                }
+                final int numChildren = H3.h3ToChildrenSize(h3);
+                for (; i < numChildren; i++) {
+                    final long child = H3.childPosToH3(h3, i);
+                    valueIndex = setValuesByRecursion(values, geoValue, child, precision + 1, valueIndex);
+                }
+                // H3 cells do intersects with other cells that are not part of the children cells. If the parent cell of those
+                // cells is disjoint, they will not be visited, therefore visit them here.
+                final int numNoChildren = H3.h3ToNotIntersectingChildrenSize(h3);
+                for (int j = 0; j < numNoChildren; j++) {
+                    final long noChild = H3.noChildIntersectingPosToH3(h3, j);
+                    if (relateTile(geoValue, H3.h3ToParent(noChild)) == GeoRelation.QUERY_DISJOINT) {
+                        valueIndex = setValuesByRecursion(values, geoValue, noChild, precision + 1, valueIndex);
+                    }
+                }
+            }
+        }
+        return valueIndex;
+    }
+
+    /**
+     * Recursively scan the H3 tree, assuming all children are fully contained in the geometry.
+     * Once at the required depth, then all cells that intersect are added to the collection.
+     */
+    private int setAllValuesByRecursion(GeoShapeCellValues values, long h3, int precision, int valueIndex, boolean valueInsideBounds) {
+        if (valueInsideBounds || h3IntersectsBounds(h3)) {
+            if (precision == this.precision) {
+                values.resizeCell(valueIndex + 1);
+                values.add(valueIndex++, h3);
+            } else {
+                final int numChildren = H3.h3ToChildrenSize(h3);
+                for (int i = 0; i < numChildren; i++) {
+                    valueIndex = setAllValuesByRecursion(values, H3.childPosToH3(h3, i), precision + 1, valueIndex, valueInsideBounds);
+                }
+            }
+        }
+        return valueIndex;
+    }
+
+    static long calcMaxAddresses(int precision) {
+        // TODO: Verify this (and perhaps move the calculation into H3 and based on NUM_BASE_CELLS and others)
+        final int baseHexagons = 110;
+        final int basePentagons = 12;
+        return baseHexagons * (long) Math.pow(7, precision) + basePentagons * (long) Math.pow(6, precision);
+    }
+
+    /**
+     * Bounded geohex aggregation. It accepts H3 addresses that intersect the provided bounds.
+     * The additional support for testing intersection with inflated bounds is used when testing
+     * parent cells, since child cells can exceed the bounds of their parent. We inflate the bounds
+     * by half of the width and half of the height.
+     */
+    static class BoundedGeoHexGridTiler extends GeoHexGridTiler {
+        private final GeoBoundingBox[] inflatedBboxes;
+        private final GeoBoundingBox bbox;
+        private final GeoHexVisitor visitor;
+        private final int resolution;
+        private static final double FACTOR = 0.35;
+
+        BoundedGeoHexGridTiler(int resolution, GeoBoundingBox bbox) {
+            super(resolution);
+            this.bbox = bbox;
+            this.visitor = new GeoHexVisitor();
+            this.resolution = resolution;
+            inflatedBboxes = new GeoBoundingBox[resolution];
+            for (int i = 0; i < resolution; i++) {
+                inflatedBboxes[i] = inflateBbox(i, bbox, FACTOR);
+            }
+        }
+
+        /**
+         * Since H3 cells do not fully contain their child cells, we need to take care that when
+         * filtering cells at a lower precision than the final precision, we must not exclude
+         * parents that do not match the filter, but their own children or descendents might match.
+         * For this reason the filter needs to be expanded to cover all descendent cells.
+         *
+         * This is done by taking the H3 cells at two corners, and expanding the filter width
+         * by 35% of the max width of those cells, and filter height by 35% of the max height of those cells.
+         *
+         * The inflation factor of 35% has been verified using test GeoHexTilerTests#testLargeShapeWithBounds
+         */
+        static GeoBoundingBox inflateBbox(int precision, GeoBoundingBox bbox, double factor) {
+            final Rectangle minMin = H3CartesianUtil.toBoundingBox(H3.geoToH3(bbox.bottom(), bbox.left(), precision));
+            final Rectangle maxMax = H3CartesianUtil.toBoundingBox(H3.geoToH3(bbox.top(), bbox.right(), precision));
+            // compute height and width at the given precision
+            final double height = Math.max(height(minMin), height(maxMax));
+            final double width = Math.max(width(minMin), width(maxMax));
+            // inflate the coordinates using the factor
+            final double minY = Math.max(bbox.bottom() - factor * height, -90d);
+            final double maxY = Math.min(bbox.top() + factor * height, 90d);
+            final double left = GeoUtils.normalizeLon(bbox.left() - factor * width);
+            final double right = GeoUtils.normalizeLon(bbox.right() + factor * width);
+            if (2 * factor * width + width(bbox) >= 360d) {
+                // if the total width bigger than the world, then it covers all longitude range.
+                return new GeoBoundingBox(new GeoPoint(maxY, -180d), new GeoPoint(minY, 180d));
+            } else {
+                return new GeoBoundingBox(new GeoPoint(maxY, left), new GeoPoint(minY, right));
+            }
+        }
+
+        static double height(Rectangle rectangle) {
+            return rectangle.getMaxY() - rectangle.getMinY();
+        }
+
+        static double width(Rectangle rectangle) {
+            if (rectangle.getMinX() > rectangle.getMaxX()) {
+                return 360d + rectangle.getMaxX() - rectangle.getMinX();
+            } else {
+                return rectangle.getMaxX() - rectangle.getMinX();
+            }
+        }
+
+        static double width(GeoBoundingBox bbox) {
+            if (bbox.left() > bbox.right()) {
+                return 360d + bbox.right() - bbox.left();
+            } else {
+                return bbox.right() - bbox.left();
+            }
+        }
+
+        @Override
+        protected long getMaxCells() {
+            // TODO: Calculate correctly based on bounds
+            return UnboundedGeoHexGridTiler.calcMaxAddresses(resolution);
+        }
+
+        @Override
+        protected boolean h3IntersectsBounds(long h3) {
+            visitor.reset(h3);
+            final int resolution = H3.getResolution(h3);
+            if (resolution != this.resolution) {
+                assert resolution < this.resolution;
+                return cellIntersectsBounds(visitor, inflatedBboxes[resolution]);
+            }
+            return cellIntersectsBounds(visitor, bbox);
+        }
+
+        @Override
+        protected GeoRelation relateTile(GeoShapeValues.GeoShapeValue geoValue, long h3) throws IOException {
+            visitor.reset(h3);
+            final int resolution = H3.getResolution(h3);
+            if (resolution != this.resolution) {
+                assert resolution < this.resolution;
+                if (cellIntersectsBounds(visitor, inflatedBboxes[resolution])) {
+                    // close to the poles, the properties of the H3 grid are lost because of the equirectangular projection,
+                    // therefore we cannot ensure that the relationship at this level make any sense in the next level.
+                    // Therefore, we just return CROSSES which just mean keep recursing.
+                    if (visitor.getMaxY() > H3CartesianUtil.getNorthPolarBound(resolution)
+                        || visitor.getMinY() < H3CartesianUtil.getSouthPolarBound(resolution)) {
+                        return GeoRelation.QUERY_CROSSES;
+                    }
+                    geoValue.visit(visitor);
+                    return visitor.relation();
+                } else {
+                    return GeoRelation.QUERY_DISJOINT;
+                }
+            }
+            if (cellIntersectsBounds(visitor, bbox)) {
+                geoValue.visit(visitor);
+                return visitor.relation();
+            }
+            return GeoRelation.QUERY_DISJOINT;
+        }
+
+        @Override
+        protected boolean valueInsideBounds(GeoShapeValues.GeoShapeValue geoValue) {
+            if (bbox.bottom() <= geoValue.boundingBox().minY() && bbox.top() >= geoValue.boundingBox().maxY()) {
+                if (bbox.right() < bbox.left()) {
+                    return bbox.left() <= geoValue.boundingBox().minX() || bbox.right() >= geoValue.boundingBox().maxX();
+                } else {
+                    return bbox.left() <= geoValue.boundingBox().minX() && bbox.right() >= geoValue.boundingBox().maxX();
+                }
+            }
+            return false;
+        }
+
+        private static boolean cellIntersectsBounds(GeoHexVisitor visitor, GeoBoundingBox bbox) {
+            return visitor.intersectsBbox(bbox.left(), bbox.right(), bbox.bottom(), bbox.top());
+        }
+    }
+
+    /**
+     * Unbounded geohex aggregation. It accepts any hash.
+     */
+    private static class UnboundedGeoHexGridTiler extends GeoHexGridTiler {
+
+        private final long maxAddresses;
+
+        private final GeoHexVisitor visitor;
+
+        UnboundedGeoHexGridTiler(int precision) {
+            super(precision);
+            this.visitor = new GeoHexVisitor();
+            maxAddresses = calcMaxAddresses(precision);
+        }
+
+        @Override
+        protected boolean h3IntersectsBounds(long h3) {
+            return true;
+        }
+
+        @Override
+        protected GeoRelation relateTile(GeoShapeValues.GeoShapeValue geoValue, long h3) throws IOException {
+            visitor.reset(h3);
+            final int resolution = H3.getResolution(h3);
+            if (resolution != precision
+                && (visitor.getMaxY() > H3CartesianUtil.getNorthPolarBound(resolution)
+                    || visitor.getMinY() < H3CartesianUtil.getSouthPolarBound(resolution))) {
+                // close to the poles, the properties of the H3 grid are lost because of the equirectangular projection,
+                // therefore we cannot ensure that the relationship at this level make any sense in the next level.
+                // Therefore, we just return CROSSES which just mean keep recursing.
+                return GeoRelation.QUERY_CROSSES;
+            }
+            geoValue.visit(visitor);
+            return visitor.relation();
+        }
+
+        @Override
+        protected boolean valueInsideBounds(GeoShapeValues.GeoShapeValue geoValue) {
+            return true;
+        }
+
+        @Override
+        protected long getMaxCells() {
+            return maxAddresses;
+        }
+    }
+
+}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexVisitor.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexVisitor.java
new file mode 100644
index 000000000000..b0b6507250bc
--- /dev/null
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexVisitor.java
@@ -0,0 +1,353 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid;
+
+import org.apache.lucene.geo.Component2D;
+import org.apache.lucene.geo.GeoUtils;
+import org.apache.lucene.util.ArrayUtil;
+import org.elasticsearch.xpack.spatial.common.H3CartesianUtil;
+import org.elasticsearch.xpack.spatial.index.fielddata.CoordinateEncoder;
+import org.elasticsearch.xpack.spatial.index.fielddata.GeoRelation;
+import org.elasticsearch.xpack.spatial.index.fielddata.TriangleTreeVisitor;
+
+/**
+ * A reusable tree reader visitor for a previous serialized {@link org.elasticsearch.geometry.Geometry}.
+ *
+ * This class supports checking H3 cells relations against a serialized triangle tree. It has the special property that if
+ * a geometry touches any of the edges, then it will never return {@link GeoRelation#QUERY_CONTAINS}, for example a point on the boundary
+ * return {@link GeoRelation#QUERY_CROSSES}.
+ */
+class GeoHexVisitor extends TriangleTreeVisitor.TriangleTreeDecodedVisitor {
+
+    private GeoRelation relation;
+    private final double[] xs, ys;
+    private double minX, maxX, minY, maxY;
+    private boolean crossesDateline;
+    private int numPoints;
+
+    GeoHexVisitor() {
+        super(CoordinateEncoder.GEO);
+        xs = new double[H3CartesianUtil.MAX_ARRAY_SIZE];
+        ys = new double[H3CartesianUtil.MAX_ARRAY_SIZE];
+    }
+
+    public double[] getXs() {
+        return ArrayUtil.copyOfSubArray(xs, 0, numPoints);
+    }
+
+    public double[] getYs() {
+        return ArrayUtil.copyOfSubArray(ys, 0, numPoints);
+    }
+
+    public double getLeftX() {
+        return crossesDateline ? maxX : minX;
+    }
+
+    public double getRightX() {
+        return crossesDateline ? minX : maxX;
+    }
+
+    public double getMinY() {
+        return minY;
+    }
+
+    public double getMaxY() {
+        return maxY;
+    }
+
+    /**
+     * reset this visitor to the provided h3 cell
+     */
+    public void reset(long h3) {
+        numPoints = H3CartesianUtil.computePoints(h3, xs, ys);
+        double minX = Double.POSITIVE_INFINITY;
+        double maxX = Double.NEGATIVE_INFINITY;
+        double minY = Double.POSITIVE_INFINITY;
+        double maxY = Double.NEGATIVE_INFINITY;
+        for (int i = 0; i < numPoints; i++) {
+            minX = Math.min(minX, xs[i]);
+            maxX = Math.max(maxX, xs[i]);
+            minY = Math.min(minY, ys[i]);
+            maxY = Math.max(maxY, ys[i]);
+        }
+        this.minX = minX;
+        this.maxX = maxX;
+        this.minY = minY;
+        this.maxY = maxY;
+        this.crossesDateline = maxX - minX > 180d && H3CartesianUtil.isPolar(h3) == false;
+    }
+
+    /**
+     * return the computed relation.
+     */
+    public GeoRelation relation() {
+        return relation;
+    }
+
+    @Override
+    public void visitDecodedPoint(double x, double y) {
+        updateRelation(relatePoint(x, y));
+    }
+
+    @Override
+    protected void visitDecodedLine(double aX, double aY, double bX, double bY, byte metadata) {
+        updateRelation(relateLine(aX, aY, bX, bY));
+    }
+
+    @Override
+    protected void visitDecodedTriangle(double aX, double aY, double bX, double bY, double cX, double cY, byte metadata) {
+        final boolean ab = (metadata & 1 << 4) == 1 << 4;
+        final boolean bc = (metadata & 1 << 5) == 1 << 5;
+        final boolean ca = (metadata & 1 << 6) == 1 << 6;
+        updateRelation(relateTriangle(aX, aY, ab, bX, bY, bc, cX, cY, ca));
+    }
+
+    private void updateRelation(GeoRelation relation) {
+        if (relation != GeoRelation.QUERY_DISJOINT) {
+            if (relation == GeoRelation.QUERY_INSIDE && canBeInside()) {
+                this.relation = GeoRelation.QUERY_INSIDE;
+            } else if (relation == GeoRelation.QUERY_CONTAINS && canBeContained()) {
+                this.relation = GeoRelation.QUERY_CONTAINS;
+            } else {
+                this.relation = GeoRelation.QUERY_CROSSES;
+            }
+        } else {
+            adjustRelationForNotIntersectingComponent();
+        }
+    }
+
+    private void adjustRelationForNotIntersectingComponent() {
+        if (relation == null) {
+            this.relation = GeoRelation.QUERY_DISJOINT;
+        } else if (relation == GeoRelation.QUERY_CONTAINS) {
+            this.relation = GeoRelation.QUERY_CROSSES;
+        }
+    }
+
+    private boolean canBeContained() {
+        return this.relation == null || this.relation == GeoRelation.QUERY_CONTAINS;
+    }
+
+    private boolean canBeInside() {
+        return this.relation != GeoRelation.QUERY_CONTAINS;
+    }
+
+    @Override
+    public boolean push() {
+        return this.relation != GeoRelation.QUERY_CROSSES;
+    }
+
+    @Override
+    public boolean pushDecodedX(double minX) {
+        if (crossesDateline || this.maxX >= minX) {
+            return true;
+        }
+        adjustRelationForNotIntersectingComponent();
+        return false;
+    }
+
+    @Override
+    public boolean pushDecodedY(double minY) {
+        if (this.maxY >= minY) {
+            return true;
+        }
+        adjustRelationForNotIntersectingComponent();
+        return false;
+    }
+
+    @Override
+    public boolean pushDecoded(double maxX, double maxY) {
+        if (this.minY <= maxY && (this.crossesDateline || minX <= maxX)) {
+            return true;
+        }
+        adjustRelationForNotIntersectingComponent();
+        return false;
+    }
+
+    @Override
+    @SuppressWarnings("HiddenField")
+    public boolean pushDecoded(double minX, double minY, double maxX, double maxY) {
+        if (boxesAreDisjoint(minX, maxX, minY, maxY)) {
+            // shapes are disjoint
+            this.relation = GeoRelation.QUERY_DISJOINT;
+            return false;
+        }
+        relation = null;
+        return true;
+    }
+
+    /** Check if the provided bounding box intersect the H3 bin. It supports bounding boxes
+     * crossing the dateline. */
+    public boolean intersectsBbox(double minX, double maxX, double minY, double maxY) {
+        if (minX > maxX) {
+            return relateBbox(minX, GeoUtils.MAX_LON_INCL, minY, maxY) || relateBbox(GeoUtils.MIN_LON_INCL, maxX, minY, maxY);
+        } else {
+            return relateBbox(minX, maxX, minY, maxY);
+        }
+    }
+
+    private boolean relateBbox(double minX, double maxX, double minY, double maxY) {
+        if (boxesAreDisjoint(minX, maxX, minY, maxY)) {
+            return false;
+        }
+        if (minX <= xs[0] && maxX >= xs[0] && minY <= ys[0] && maxY >= ys[0]) {
+            return true;
+        }
+        return relatePoint(minX, minY) != GeoRelation.QUERY_DISJOINT
+            || H3CartesianUtil.crossesBox(xs, ys, numPoints, crossesDateline, minX, maxX, minY, maxY, true);
+    }
+
+    /**
+     * Checks if the rectangle contains the provided point
+     **/
+    private GeoRelation relatePoint(double x, double y) {
+        if (boxesAreDisjoint(x, x, y, y)) {
+            return GeoRelation.QUERY_DISJOINT;
+        }
+        return H3CartesianUtil.relatePoint(xs, ys, numPoints, crossesDateline, x, y);
+    }
+
+    /**
+     * Compute the relationship between the provided line and this h3 bin
+     **/
+    private GeoRelation relateLine(double aX, double aY, double bX, double bY) {
+        // query contains any points
+        GeoRelation relation1 = relatePoint(aX, aY);
+        GeoRelation relation2 = relatePoint(bX, bY);
+        if (relation1 != relation2 || relation1 == GeoRelation.QUERY_CROSSES) {
+            return GeoRelation.QUERY_CROSSES;
+        } else if (relation1 == GeoRelation.QUERY_CONTAINS) {
+            if (crossesDateline) {
+                final double minX = Math.min(aX, bX);
+                final double maxX = Math.max(aX, bX);
+                final double minY = Math.min(aY, bY);
+                final double maxY = Math.max(aY, bY);
+                if (H3CartesianUtil.crossesLine(xs, ys, numPoints, crossesDateline, minX, maxX, minY, maxY, aX, aY, bX, bY, true)) {
+                    return GeoRelation.QUERY_CROSSES;
+                }
+            }
+            return GeoRelation.QUERY_CONTAINS;
+        }
+        // 2. check crossings
+        if (edgeIntersectsQuery(aX, aY, bX, bY)) {
+            return GeoRelation.QUERY_CROSSES;
+        }
+        return GeoRelation.QUERY_DISJOINT;
+    }
+
+    /**
+     * Compute the relationship between the provided triangle and this h3 bin
+     **/
+    private GeoRelation relateTriangle(
+        double aX,
+        double aY,
+        boolean ab,
+        double bX,
+        double bY,
+        boolean bc,
+        double cX,
+        double cY,
+        boolean ca
+    ) {
+        // compute bounding box of triangle
+        double tMinX = StrictMath.min(StrictMath.min(aX, bX), cX);
+        double tMaxX = StrictMath.max(StrictMath.max(aX, bX), cX);
+        double tMinY = StrictMath.min(StrictMath.min(aY, bY), cY);
+        double tMaxY = StrictMath.max(StrictMath.max(aY, bY), cY);
+
+        // 1. check bounding boxes are disjoint
+        if (boxesAreDisjoint(tMinX, tMaxX, tMinY, tMaxY)) {
+            return GeoRelation.QUERY_DISJOINT;
+        }
+
+        GeoRelation relation1 = relatePoint(aX, aY);
+        GeoRelation relation2 = relatePoint(bX, bY);
+        GeoRelation relation3 = relatePoint(cX, cY);
+
+        // 2. query contains any triangle points
+        if (relation1 != relation2 || relation1 != relation3 || relation1 == GeoRelation.QUERY_CROSSES) {
+            return GeoRelation.QUERY_CROSSES;
+        } else if (relation1 == GeoRelation.QUERY_CONTAINS) {
+            if (crossesDateline
+                && H3CartesianUtil.crossesTriangle(
+                    xs,
+                    ys,
+                    numPoints,
+                    crossesDateline,
+                    tMinX,
+                    tMaxX,
+                    tMinY,
+                    tMaxY,
+                    aX,
+                    aY,
+                    bX,
+                    bY,
+                    cX,
+                    cY,
+                    true
+                )) {
+                return GeoRelation.QUERY_CROSSES;
+            }
+            return GeoRelation.QUERY_CONTAINS;
+        }
+
+        boolean within = false;
+        if (edgeIntersectsQuery(aX, aY, bX, bY)) {
+            if (ab) {
+                return GeoRelation.QUERY_CROSSES;
+            }
+            within = true;
+        }
+
+        if (edgeIntersectsQuery(bX, bY, cX, cY)) {
+            if (bc) {
+                return GeoRelation.QUERY_CROSSES;
+            }
+            within = true;
+        }
+
+        if (edgeIntersectsQuery(cX, cY, aX, aY)) {
+            if (ca) {
+                return GeoRelation.QUERY_CROSSES;
+            }
+            within = true;
+        }
+
+        if (within || Component2D.pointInTriangle(tMinX, tMaxX, tMinY, tMaxY, xs[0], ys[0], aX, aY, bX, bY, cX, cY)) {
+            return GeoRelation.QUERY_INSIDE;
+        }
+
+        return GeoRelation.QUERY_DISJOINT;
+    }
+
+    /**
+     * returns true if the edge (defined by (ax, ay) (bx, by)) intersects the query
+     */
+    private boolean edgeIntersectsQuery(double ax, double ay, double bx, double by) {
+        final double minX = Math.min(ax, bx);
+        final double maxX = Math.max(ax, bx);
+        final double minY = Math.min(ay, by);
+        final double maxY = Math.max(ay, by);
+        return boxesAreDisjoint(minX, maxX, minY, maxY) == false
+            && H3CartesianUtil.crossesLine(xs, ys, numPoints, crossesDateline, minX, maxX, minY, maxY, ax, ay, bx, by, true);
+    }
+
+    /**
+     * utility method to check if two boxes are disjoint
+     */
+    private boolean boxesAreDisjoint(final double minX, final double maxX, final double minY, final double maxY) {
+        if ((maxY < this.minY || minY > this.maxY) == false) {
+            if (crossesDateline) {
+                return maxX < this.minX && minX > this.maxX;
+            } else {
+                return maxX < this.minX || minX > this.maxX;
+            }
+        }
+        return true;
+    }
+}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellIdSource.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellIdSource.java
index bfabcfbdeb2a..402bc6765ede 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellIdSource.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellIdSource.java
@@ -22,20 +22,11 @@
 public class GeoShapeCellIdSource extends ValuesSource.Numeric {
     private final GeoShapeValuesSource valuesSource;
     private final GeoGridTiler encoder;
-    private LongConsumer circuitBreakerConsumer;
+    private final LongConsumer circuitBreakerConsumer;
 
-    public GeoShapeCellIdSource(GeoShapeValuesSource valuesSource, GeoGridTiler encoder) {
+    public GeoShapeCellIdSource(GeoShapeValuesSource valuesSource, GeoGridTiler encoder, LongConsumer circuitBreakerConsumer) {
         this.valuesSource = valuesSource;
         this.encoder = encoder;
-        this.circuitBreakerConsumer = (l) -> {};
-    }
-
-    /**
-     * This setter exists since the aggregator's circuit-breaking accounting needs to be
-     * accessible from within the values-source. Problem is that this values-source needs to
-     * be created and passed to the aggregator before we have access to this functionality.
-     */
-    public void setCircuitBreakerConsumer(LongConsumer circuitBreakerConsumer) {
         this.circuitBreakerConsumer = circuitBreakerConsumer;
     }
 
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellValues.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellValues.java
index ba8d2c11681b..720d0c8e5c3d 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellValues.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellValues.java
@@ -7,17 +7,18 @@
 
 package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid;
 
+import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues;
 import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeValues;
 
 import java.io.IOException;
 import java.util.function.LongConsumer;
 
 /** Sorted numeric doc values for geo shapes */
-class GeoShapeCellValues extends ByteTrackingSortingNumericDocValues {
+final class GeoShapeCellValues extends AbstractSortingNumericDocValues {
     private final GeoShapeValues geoShapeValues;
-    protected final GeoGridTiler tiler;
+    private final GeoGridTiler tiler;
 
-    protected GeoShapeCellValues(GeoShapeValues geoShapeValues, GeoGridTiler tiler, LongConsumer circuitBreakerConsumer) {
+    GeoShapeCellValues(GeoShapeValues geoShapeValues, GeoGridTiler tiler, LongConsumer circuitBreakerConsumer) {
         super(circuitBreakerConsumer);
         this.geoShapeValues = geoShapeValues;
         this.tiler = tiler;
@@ -36,7 +37,7 @@ public boolean advanceExact(int docId) throws IOException {
     }
 
     // for testing
-    protected long[] getValues() {
+    long[] getValues() {
         return values;
     }
 
@@ -44,7 +45,7 @@ void resizeCell(int newSize) {
         resize(newSize);
     }
 
-    protected void add(int idx, long value) {
+    void add(int idx, long value) {
         values[idx] = value;
     }
 }
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeHashGridAggregator.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeHashGridAggregator.java
deleted file mode 100644
index 7ba28a46badf..000000000000
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeHashGridAggregator.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid;
-
-import org.elasticsearch.search.aggregations.Aggregator;
-import org.elasticsearch.search.aggregations.AggregatorFactories;
-import org.elasticsearch.search.aggregations.CardinalityUpperBound;
-import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
-import org.elasticsearch.search.aggregations.support.ValuesSource;
-
-import java.io.IOException;
-import java.util.Map;
-
-public class GeoShapeHashGridAggregator extends GeoHashGridAggregator {
-    public GeoShapeHashGridAggregator(
-        String name,
-        AggregatorFactories factories,
-        ValuesSource.Numeric valuesSource,
-        int requiredSize,
-        int shardSize,
-        AggregationContext context,
-        Aggregator parent,
-        CardinalityUpperBound cardinality,
-        Map<String, Object> metadata
-    ) throws IOException {
-        super(name, factories, valuesSource, requiredSize, shardSize, context, parent, cardinality, metadata);
-    }
-
-    /**
-     * This is a wrapper method to expose this protected method to {@link GeoShapeCellIdSource}
-     *
-     * @param bytes the number of bytes to register or negative to deregister the bytes
-     * @return the cumulative size in bytes allocated by this aggregator to service this request
-     */
-    public long addRequestBytes(long bytes) {
-        return addRequestCircuitBreakerBytes(bytes);
-    }
-}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeTileGridAggregator.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeTileGridAggregator.java
deleted file mode 100644
index 2552ae65c9e6..000000000000
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeTileGridAggregator.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid;
-
-import org.elasticsearch.search.aggregations.Aggregator;
-import org.elasticsearch.search.aggregations.AggregatorFactories;
-import org.elasticsearch.search.aggregations.CardinalityUpperBound;
-import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
-import org.elasticsearch.search.aggregations.support.ValuesSource;
-
-import java.io.IOException;
-import java.util.Map;
-
-public class GeoShapeTileGridAggregator extends GeoTileGridAggregator {
-    public GeoShapeTileGridAggregator(
-        String name,
-        AggregatorFactories factories,
-        ValuesSource.Numeric valuesSource,
-        int requiredSize,
-        int shardSize,
-        AggregationContext context,
-        Aggregator parent,
-        CardinalityUpperBound cardinality,
-        Map<String, Object> metadata
-    ) throws IOException {
-        super(name, factories, valuesSource, requiredSize, shardSize, context, parent, cardinality, metadata);
-    }
-
-    /**
-     * This is a wrapper method to expose this protected method to {@link GeoShapeCellIdSource}
-     *
-     * @param bytes the number of bytes to register or negative to deregister the bytes
-     * @return the cumulative size in bytes allocated by this aggregator to service this request
-     */
-    public long addRequestBytes(long bytes) {
-        return addRequestCircuitBreakerBytes(bytes);
-    }
-}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoTileGridTiler.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoTileGridTiler.java
new file mode 100644
index 000000000000..68da8fc2c859
--- /dev/null
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoTileGridTiler.java
@@ -0,0 +1,294 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid;
+
+import org.apache.lucene.geo.GeoEncodingUtils;
+import org.elasticsearch.common.geo.GeoBoundingBox;
+import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileBoundedPredicate;
+import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
+import org.elasticsearch.xpack.spatial.index.fielddata.GeoRelation;
+import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeValues;
+
+import java.io.IOException;
+
+/**
+ * Implements the logic for the GeoTile aggregation over a geoshape doc value.
+ */
+public abstract class GeoTileGridTiler extends GeoGridTiler {
+
+    protected final int tiles;
+
+    private GeoTileGridTiler(int precision) {
+        super(precision);
+        tiles = 1 << precision;
+    }
+
+    /** Factory method to create GeoTileGridTiler objects */
+    public static GeoTileGridTiler makeGridTiler(int precision, GeoBoundingBox geoBoundingBox) {
+        return geoBoundingBox == null || geoBoundingBox.isUnbounded()
+            ? new GeoTileGridTiler.UnboundedGeoTileGridTiler(precision)
+            : new GeoTileGridTiler.BoundedGeoTileGridTiler(precision, geoBoundingBox);
+    }
+
+    /** check if the provided tile is in the solution space of this tiler */
+    protected abstract boolean validTile(int x, int y, int z);
+
+    @Override
+    public long encode(double x, double y) {
+        return GeoTileUtils.longEncode(x, y, precision);
+    }
+
+    /**
+     * Sets the values of the long[] underlying {@link GeoShapeCellValues}.
+     *
+     * If the shape resides between <code>GeoTileUtils.NORMALIZED_LATITUDE_MASK</code> and 90 or
+     * between <code>GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK</code> and -90 degree latitudes, then
+     * the shape is not accounted for since geo-tiles are only defined within those bounds.
+     *
+     * @param values           the bucket values
+     * @param geoValue         the input shape
+     *
+     * @return the number of tiles set by the shape
+     */
+    @Override
+    public int setValues(GeoShapeCellValues values, GeoShapeValues.GeoShapeValue geoValue) throws IOException {
+        final GeoShapeValues.BoundingBox bounds = geoValue.boundingBox();
+        assert bounds.minX() <= bounds.maxX();
+
+        // geo tiles are not defined at the extreme latitudes due to them
+        // tiling the world as a square.
+        if (bounds.bottom > GeoTileUtils.NORMALIZED_LATITUDE_MASK || bounds.top < GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK) {
+            return 0;
+        }
+
+        if (precision == 0) {
+            return setValuesByBruteForceScan(values, geoValue, 0, 0, 0, 0);
+        }
+
+        final int minXTile = GeoTileUtils.getXTile(bounds.minX(), tiles);
+        final int minYTile = GeoTileUtils.getYTile(bounds.maxY(), tiles);
+        final int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), tiles);
+        final int maxYTile = GeoTileUtils.getYTile(bounds.minY(), tiles);
+        final long count = (long) (maxXTile - minXTile + 1) * (maxYTile - minYTile + 1);
+        if (count == 1) {
+            return setValue(values, minXTile, minYTile);
+        } else if (count <= 8L * precision) {
+            return setValuesByBruteForceScan(values, geoValue, minXTile, minYTile, maxXTile, maxYTile);
+        } else {
+            return setValuesByRasterization(0, 0, 0, values, 0, geoValue);
+        }
+    }
+
+    private GeoRelation relateTile(GeoShapeValues.GeoShapeValue geoValue, int xTile, int yTile, int precision) throws IOException {
+        if (validTile(xTile, yTile, precision)) {
+            final int tiles = 1 << precision;
+            final int minX = GeoEncodingUtils.encodeLongitude(GeoTileUtils.tileToLon(xTile, tiles));
+            final int maxX = GeoEncodingUtils.encodeLongitude(GeoTileUtils.tileToLon(xTile + 1, tiles));
+            final int minY = GeoEncodingUtils.encodeLatitude(GeoTileUtils.tileToLat(yTile + 1, tiles));
+            final int maxY = GeoEncodingUtils.encodeLatitude(GeoTileUtils.tileToLat(yTile, tiles));
+            return geoValue.relate(
+                minX,
+                maxX == Integer.MAX_VALUE ? maxX : maxX - 1,
+                minY == GeoTileUtils.ENCODED_NEGATIVE_LATITUDE_MASK ? minY : minY + 1,
+                maxY
+            );
+        }
+        return GeoRelation.QUERY_DISJOINT;
+    }
+
+    /**
+     * Sets a singular doc-value with the provided x/y.
+     */
+    private int setValue(GeoShapeCellValues docValues, int xTile, int yTile) {
+        if (validTile(xTile, yTile, precision)) {
+            docValues.resizeCell(1);
+            docValues.add(0, GeoTileUtils.longEncodeTiles(precision, xTile, yTile));
+            return 1;
+        }
+        return 0;
+    }
+
+    /**
+     * Checks all tiles between minXTile/maxXTile and minYTile/maxYTile.
+     */
+    // pack private for testing
+    int setValuesByBruteForceScan(
+        GeoShapeCellValues values,
+        GeoShapeValues.GeoShapeValue geoValue,
+        int minXTile,
+        int minYTile,
+        int maxXTile,
+        int maxYTile
+    ) throws IOException {
+        int idx = 0;
+        for (int i = minXTile; i <= maxXTile; i++) {
+            for (int j = minYTile; j <= maxYTile; j++) {
+                final GeoRelation relation = relateTile(geoValue, i, j, precision);
+                if (relation != GeoRelation.QUERY_DISJOINT) {
+                    values.resizeCell(idx + 1);
+                    values.add(idx++, GeoTileUtils.longEncodeTiles(precision, i, j));
+                }
+            }
+        }
+        return idx;
+    }
+
+    /**
+     * Recursively search the tile tree, only following branches that intersect the geometry.
+     * Once at the required depth, then all cells that intersect are added to the collection.
+     */
+    // pkg protected for testing
+    int setValuesByRasterization(
+        int xTile,
+        int yTile,
+        int zTile,
+        GeoShapeCellValues values,
+        int valuesIndex,
+        GeoShapeValues.GeoShapeValue geoValue
+    ) throws IOException {
+        zTile++;
+        for (int i = 0; i < 2; i++) {
+            for (int j = 0; j < 2; j++) {
+                final int nextX = 2 * xTile + i;
+                final int nextY = 2 * yTile + j;
+                final GeoRelation relation = relateTile(geoValue, nextX, nextY, zTile);
+                if (GeoRelation.QUERY_INSIDE == relation) {
+                    if (zTile == precision) {
+                        values.resizeCell(getNewSize(valuesIndex, 1));
+                        values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY));
+                    } else {
+                        final int numTilesAtPrecision = getNumTilesAtPrecision(precision, zTile);
+                        values.resizeCell(getNewSize(valuesIndex, numTilesAtPrecision + 1));
+                        valuesIndex = setValuesForFullyContainedTile(nextX, nextY, zTile, values, valuesIndex);
+                    }
+                } else if (GeoRelation.QUERY_DISJOINT != relation) {
+                    if (zTile == precision) {
+                        values.resizeCell(getNewSize(valuesIndex, 1));
+                        values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY));
+                    } else {
+                        valuesIndex = setValuesByRasterization(nextX, nextY, zTile, values, valuesIndex, geoValue);
+                    }
+                }
+            }
+        }
+        return valuesIndex;
+    }
+
+    private int getNewSize(int valuesIndex, int increment) {
+        final long newSize = (long) valuesIndex + increment;
+        if (newSize > Integer.MAX_VALUE) {
+            throw new IllegalArgumentException("Tile aggregation array overflow");
+        }
+        return (int) newSize;
+    }
+
+    private int getNumTilesAtPrecision(int finalPrecision, int currentPrecision) {
+        final long numTilesAtPrecision = Math.min(1L << (2 * (finalPrecision - currentPrecision)), getMaxCells());
+        if (numTilesAtPrecision > Integer.MAX_VALUE) {
+            throw new IllegalArgumentException("Tile aggregation array overflow");
+        }
+        return (int) numTilesAtPrecision;
+    }
+
+    protected abstract int setValuesForFullyContainedTile(int xTile, int yTile, int zTile, GeoShapeCellValues values, int valuesIndex);
+
+    protected int setValues(GeoShapeCellValues values, int valuesIndex, int minY, int maxY, int minX, int maxX) {
+        for (int i = minX; i < maxX; i++) {
+            for (int j = minY; j < maxY; j++) {
+                assert validTile(i, j, precision);
+                values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(precision, i, j));
+            }
+        }
+        return valuesIndex;
+    }
+
+    /**
+     * Bounded geotile aggregation. It accepts tiles that intersects the provided bounds.
+     */
+    private static class BoundedGeoTileGridTiler extends GeoTileGridTiler {
+
+        private final GeoTileBoundedPredicate predicate;
+
+        BoundedGeoTileGridTiler(int precision, GeoBoundingBox bbox) {
+            super(precision);
+            this.predicate = new GeoTileBoundedPredicate(precision, bbox);
+        }
+
+        @Override
+        protected boolean validTile(int x, int y, int z) {
+            return predicate.validTile(x, y, z);
+        }
+
+        @Override
+        protected long getMaxCells() {
+            return predicate.getMaxTiles();
+        }
+
+        @Override
+        protected int setValuesForFullyContainedTile(int xTile, int yTile, int zTile, GeoShapeCellValues values, int valuesIndex) {
+            // For every level we go down, we half each dimension. The total number of splits is equal to 1 << (levelEnd - levelStart)
+            final int splits = 1 << precision - zTile;
+            // The start value of a dimension is calculated by multiplying the value of that dimension at the start level
+            // by the number of splits. Choose the max value with respect to the bounding box.
+            final int minY = Math.max(predicate.minY(), yTile * splits);
+            // The end value of a dimension is calculated by adding to the start value the number of splits.
+            // Choose the min value with respect to the bounding box.
+            final int maxY = Math.min(predicate.maxY(), yTile * splits + splits);
+            // Do the same for the X dimension taking into account that the bounding box might cross the dateline.
+            if (predicate.crossesDateline()) {
+                final int westMinX = Math.max(predicate.leftX(), xTile * splits);
+                final int westMaxX = xTile * splits + splits;
+                valuesIndex = setValues(values, valuesIndex, minY, maxY, westMinX, westMaxX);
+                // when the left and right box land in the same tile, we need to make sure we don't count then twice
+                final int eastMaxX = Math.min(westMinX, Math.min(predicate.rightX(), xTile * splits + splits));
+                final int eastMinX = xTile * splits;
+                return setValues(values, valuesIndex, minY, maxY, eastMinX, eastMaxX);
+            } else {
+                final int minX = Math.max(predicate.leftX(), xTile * splits);
+                final int maxX = Math.min(predicate.rightX(), xTile * splits + splits);
+                return setValues(values, valuesIndex, minY, maxY, minX, maxX);
+            }
+        }
+    }
+
+    /**
+     * Unbounded geotile aggregation. It accepts any tile.
+     */
+    private static class UnboundedGeoTileGridTiler extends GeoTileGridTiler {
+        private final long maxTiles;
+
+        UnboundedGeoTileGridTiler(int precision) {
+            super(precision);
+            maxTiles = (long) tiles * tiles;
+        }
+
+        @Override
+        protected boolean validTile(int x, int y, int z) {
+            return true;
+        }
+
+        @Override
+        protected long getMaxCells() {
+            return maxTiles;
+        }
+
+        @Override
+        protected int setValuesForFullyContainedTile(int xTile, int yTile, int zTile, GeoShapeCellValues values, int valuesIndex) {
+            // For every level we go down, we half each dimension. The total number of splits is equal to 1 << (levelEnd - levelStart)
+            final int splits = 1 << precision - zTile;
+            // The start value of a dimension is calculated by multiplying the value of that dimension at the start level
+            // by the number of splits
+            final int minX = xTile * splits;
+            final int minY = yTile * splits;
+            // The end value of a dimension is calculated by adding to the start value the number of splits
+            final int maxX = minX + splits;
+            final int maxY = minY + splits;
+            return setValues(values, valuesIndex, minY, maxY, minX, maxX);
+        }
+    }
+}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/UnboundedGeoHashGridTiler.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/UnboundedGeoHashGridTiler.java
deleted file mode 100644
index 36c138b74878..000000000000
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/UnboundedGeoHashGridTiler.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid;
-
-/**
- * Unbounded geohash aggregation. It accepts any hash.
- */
-public class UnboundedGeoHashGridTiler extends AbstractGeoHashGridTiler {
-
-    private final long maxHashes;
-
-    public UnboundedGeoHashGridTiler(int precision) {
-        super(precision);
-        this.maxHashes = (long) Math.pow(32, precision);
-    }
-
-    @Override
-    protected boolean validHash(String hash) {
-        return true;
-    }
-
-    @Override
-    protected long getMaxCells() {
-        return maxHashes;
-    }
-}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/UnboundedGeoTileGridTiler.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/UnboundedGeoTileGridTiler.java
deleted file mode 100644
index 5bb18ccc5e1b..000000000000
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/UnboundedGeoTileGridTiler.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid;
-
-import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
-
-/**
- * Unbounded geotile aggregation. It accepts any tile.
- */
-public class UnboundedGeoTileGridTiler extends AbstractGeoTileGridTiler {
-    private final long maxTiles;
-
-    public UnboundedGeoTileGridTiler(int precision) {
-        super(precision);
-        maxTiles = tiles * tiles;
-    }
-
-    @Override
-    protected boolean validTile(int x, int y, int z) {
-        return true;
-    }
-
-    @Override
-    protected long getMaxCells() {
-        return maxTiles;
-    }
-
-    @Override
-    protected int setValuesForFullyContainedTile(int xTile, int yTile, int zTile, GeoShapeCellValues values, int valuesIndex) {
-        // For every level we go down, we half each dimension. The total number of splits is equal to 1 << (levelEnd - levelStart)
-        final int splits = 1 << precision - zTile;
-        // The start value of a dimension is calculated by multiplying the value of that dimension at the start level
-        // by the number of splits
-        final int minX = xTile * splits;
-        final int minY = yTile * splits;
-        // The end value of a dimension is calculated by adding to the start value the number of splits
-        final int maxX = minX + splits;
-        final int maxY = minY + splits;
-        for (int i = minX; i < maxX; i++) {
-            for (int j = minY; j < maxY; j++) {
-                assert validTile(i, j, precision);
-                values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(precision, i, j));
-            }
-        }
-        return valuesIndex;
-    }
-}
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianBoundsAggregationBuilder.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianBoundsAggregationBuilder.java
index 4e8e8083c072..5d2e63b8986c 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianBoundsAggregationBuilder.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianBoundsAggregationBuilder.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.spatial.search.aggregations.metrics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -104,7 +104,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_6_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_6_0;
     }
 }
diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianCentroidAggregationBuilder.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianCentroidAggregationBuilder.java
index 8435c8539f4f..80191803ff17 100644
--- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianCentroidAggregationBuilder.java
+++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/CartesianCentroidAggregationBuilder.java
@@ -7,7 +7,7 @@
 
 package org.elasticsearch.xpack.spatial.search.aggregations.metrics;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -111,7 +111,7 @@ protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
     }
 
     @Override
-    public Version getMinimalSupportedVersion() {
-        return Version.V_8_6_0;
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersion.V_8_6_0;
     }
 }
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/SpatialPluginTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/SpatialPluginTests.java
index 6a1fe0e54855..cdfe18516fb7 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/SpatialPluginTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/SpatialPluginTests.java
@@ -59,6 +59,16 @@ public void testGeoHexLicenseCheck() {
         }, "geohex_grid", "geo_point");
     }
 
+    public void testGeoShapeHexLicenseCheck() {
+        checkLicenseRequired(GeoShapeValuesSourceType.instance(), GeoHexGridAggregationBuilder.REGISTRY_KEY, (agg) -> {
+            try {
+                agg.build(null, AggregatorFactories.EMPTY, null, 0, null, 0, 0, null, null, CardinalityUpperBound.NONE, null);
+            } catch (IOException e) {
+                fail("Unexpected exception: " + e.getMessage());
+            }
+        }, "geohex_grid", "geo_shape");
+    }
+
     public void testGeoGridLicenseCheck() {
         for (ValuesSourceRegistry.RegistryKey<GeoGridAggregatorSupplier> registryKey : Arrays.asList(
             GeoHashGridAggregationBuilder.REGISTRY_KEY,
@@ -66,7 +76,7 @@ public void testGeoGridLicenseCheck() {
         )) {
             checkLicenseRequired(GeoShapeValuesSourceType.instance(), registryKey, (agg) -> {
                 try {
-                    agg.build(null, null, null, 0, null, 0, 0, null, null, CardinalityUpperBound.NONE, null);
+                    agg.build(null, AggregatorFactories.EMPTY, null, 0, null, 0, 0, null, null, CardinalityUpperBound.NONE, null);
                 } catch (IOException e) {
                     fail("Unexpected exception: " + e.getMessage());
                 }
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/common/H3CartesianGeometryTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/common/H3CartesianGeometryTests.java
new file mode 100644
index 000000000000..ce825aaa458e
--- /dev/null
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/common/H3CartesianGeometryTests.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.spatial.common;
+
+import org.apache.lucene.geo.Component2D;
+import org.apache.lucene.geo.LatLonGeometry;
+import org.apache.lucene.index.PointValues;
+import org.elasticsearch.h3.H3;
+import org.elasticsearch.test.ESTestCase;
+
+public class H3CartesianGeometryTests extends ESTestCase {
+    public void testPolarH3Crossing() {
+        long h3 = H3.geoToH3(-90, 0, 2);
+        Component2D component2D = LatLonGeometry.create(H3CartesianUtil.getLatLonGeometry(h3));
+        assertEquals(
+            PointValues.Relation.CELL_CROSSES_QUERY,
+            component2D.relate(-155.15317029319704, 163.99789148010314, -88.69673718698323, -88.26949733309448)
+        );
+    }
+}
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/common/H3CartesianUtilTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/common/H3CartesianUtilTests.java
new file mode 100644
index 000000000000..546c654cd494
--- /dev/null
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/common/H3CartesianUtilTests.java
@@ -0,0 +1,264 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.spatial.common;
+
+import org.apache.lucene.geo.Component2D;
+import org.apache.lucene.geo.GeoEncodingUtils;
+import org.apache.lucene.geo.LatLonGeometry;
+import org.apache.lucene.tests.geo.GeoTestUtil;
+import org.elasticsearch.common.geo.GeometryNormalizer;
+import org.elasticsearch.common.geo.Orientation;
+import org.elasticsearch.geo.GeometryTestUtils;
+import org.elasticsearch.geometry.LinearRing;
+import org.elasticsearch.geometry.MultiPolygon;
+import org.elasticsearch.geometry.Point;
+import org.elasticsearch.geometry.Polygon;
+import org.elasticsearch.geometry.Rectangle;
+import org.elasticsearch.geometry.utils.WellKnownText;
+import org.elasticsearch.h3.CellBoundary;
+import org.elasticsearch.h3.H3;
+import org.elasticsearch.h3.LatLng;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.spatial.index.fielddata.GeoRelation;
+import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeValues;
+import org.elasticsearch.xpack.spatial.util.GeoTestUtils;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+
+public class H3CartesianUtilTests extends ESTestCase {
+
+    public void testLevel1() throws IOException {
+        for (int i = 0; i < 10000; i++) {
+            Point point = GeometryTestUtils.randomPoint();
+            GeoShapeValues.GeoShapeValue geoValue = GeoTestUtils.geoShapeValue(point);
+            boolean inside = false;
+            for (long h3 : H3.getLongRes0Cells()) {
+                if (geoValue.relate(H3CartesianUtil.getLatLonGeometry(h3)) != GeoRelation.QUERY_DISJOINT) {
+                    inside = true;
+                    break;
+                }
+            }
+            if (inside == false) {
+                fail(
+                    "failing matching point: " + WellKnownText.toWKT(new org.elasticsearch.geometry.Point(point.getLon(), point.getLat()))
+                );
+            }
+        }
+    }
+
+    public void testLevel2() throws IOException {
+        for (int i = 0; i < 10000; i++) {
+            Point point = GeometryTestUtils.randomPoint();
+            GeoShapeValues.GeoShapeValue geoValue = GeoTestUtils.geoShapeValue(point);
+            boolean inside = false;
+            for (long res0Cell : H3.getLongRes0Cells()) {
+                for (long h3 : H3.h3ToChildren(res0Cell)) {
+                    if (geoValue.relate(H3CartesianUtil.getLatLonGeometry(h3)) != GeoRelation.QUERY_DISJOINT) {
+                        inside = true;
+                        break;
+                    }
+                }
+            }
+            if (inside == false) {
+                fail(
+                    "failing matching point: " + WellKnownText.toWKT(new org.elasticsearch.geometry.Point(point.getLon(), point.getLat()))
+                );
+            }
+        }
+    }
+
+    public void testNorthPole() throws IOException {
+        for (int res = 0; res <= H3.MAX_H3_RES; res++) {
+            final long h3 = H3.geoToH3(90, 0, res);
+            final LatLonGeometry latLonGeometry = H3CartesianUtil.getLatLonGeometry(h3);
+            final double lon = GeoTestUtil.nextLongitude();
+            {
+                GeoShapeValues.GeoShapeValue geoValue = GeoTestUtils.geoShapeValue(new Point(lon, 90));
+                assertThat(geoValue.relate(latLonGeometry), Matchers.equalTo(GeoRelation.QUERY_CONTAINS));
+            }
+            {
+                final double bound = H3CartesianUtil.getNorthPolarBound(res);
+                final double lat = randomValueOtherThanMany(l -> l > bound, GeoTestUtil::nextLatitude);
+                GeoShapeValues.GeoShapeValue geoValue = GeoTestUtils.geoShapeValue(new Point(lon, lat));
+                assertThat(geoValue.relate(latLonGeometry), Matchers.equalTo(GeoRelation.QUERY_DISJOINT));
+            }
+        }
+    }
+
+    public void testSouthPole() throws IOException {
+        for (int res = 0; res <= H3.MAX_H3_RES; res++) {
+            final long h3 = H3.geoToH3(-90, 0, res);
+            final LatLonGeometry latLonGeometry = H3CartesianUtil.getLatLonGeometry(h3);
+            final double lon = GeoTestUtil.nextLongitude();
+            {
+                GeoShapeValues.GeoShapeValue geoValue = GeoTestUtils.geoShapeValue(new Point(lon, -90));
+                assertThat(geoValue.relate(latLonGeometry), Matchers.equalTo(GeoRelation.QUERY_CONTAINS));
+            }
+            {
+                final double bound = H3CartesianUtil.getSouthPolarBound(res);
+                final double lat = randomValueOtherThanMany(l -> l < bound, GeoTestUtil::nextLatitude);
+                GeoShapeValues.GeoShapeValue geoValue = GeoTestUtils.geoShapeValue(new Point(lon, lat));
+                assertThat(geoValue.relate(latLonGeometry), Matchers.equalTo(GeoRelation.QUERY_DISJOINT));
+            }
+        }
+    }
+
+    public void testDateline() throws IOException {
+        final long h3 = H3.geoToH3(0, 180, 0);
+        final LatLonGeometry latLonGeometry = H3CartesianUtil.getLatLonGeometry(h3);
+        // points
+        {
+            GeoShapeValues.GeoShapeValue geoValue = GeoTestUtils.geoShapeValue(new org.elasticsearch.geometry.Point(0, 0));
+            assertThat(geoValue.relate(latLonGeometry), Matchers.equalTo(GeoRelation.QUERY_DISJOINT));
+            geoValue = GeoTestUtils.geoShapeValue(new org.elasticsearch.geometry.Point(180, 0));
+            assertThat(geoValue.relate(latLonGeometry), Matchers.equalTo(GeoRelation.QUERY_CONTAINS));
+            geoValue = GeoTestUtils.geoShapeValue(new org.elasticsearch.geometry.Point(-180, 0));
+            assertThat(geoValue.relate(latLonGeometry), Matchers.equalTo(GeoRelation.QUERY_CONTAINS));
+            geoValue = GeoTestUtils.geoShapeValue(new org.elasticsearch.geometry.Point(179, 0));
+            assertThat(geoValue.relate(latLonGeometry), Matchers.equalTo(GeoRelation.QUERY_CONTAINS));
+            geoValue = GeoTestUtils.geoShapeValue(new org.elasticsearch.geometry.Point(-179, 0));
+            assertThat(geoValue.relate(latLonGeometry), Matchers.equalTo(GeoRelation.QUERY_CONTAINS));
+        }
+        // lines
+        {
+            GeoShapeValues.GeoShapeValue geoValue = GeoTestUtils.geoShapeValue(
+                new org.elasticsearch.geometry.Line(new double[] { 0, 0 }, new double[] { -1, 1 })
+            );
+            assertThat(geoValue.relate(latLonGeometry), Matchers.equalTo(GeoRelation.QUERY_DISJOINT));
+            geoValue = GeoTestUtils.geoShapeValue(new org.elasticsearch.geometry.Line(new double[] { 180, 180 }, new double[] { -1, 1 }));
+            assertThat(geoValue.relate(latLonGeometry), Matchers.equalTo(GeoRelation.QUERY_CONTAINS));
+            geoValue = GeoTestUtils.geoShapeValue(new org.elasticsearch.geometry.Line(new double[] { -180, -180 }, new double[] { -1, 1 }));
+            assertThat(geoValue.relate(latLonGeometry), Matchers.equalTo(GeoRelation.QUERY_CONTAINS));
+            geoValue = GeoTestUtils.geoShapeValue(new org.elasticsearch.geometry.Line(new double[] { 179, 179 }, new double[] { -1, 1 }));
+            assertThat(geoValue.relate(latLonGeometry), Matchers.equalTo(GeoRelation.QUERY_CONTAINS));
+            geoValue = GeoTestUtils.geoShapeValue(new org.elasticsearch.geometry.Line(new double[] { -179, -179 }, new double[] { -1, 1 }));
+            assertThat(geoValue.relate(latLonGeometry), Matchers.equalTo(GeoRelation.QUERY_CONTAINS));
+            geoValue = GeoTestUtils.geoShapeValue(new org.elasticsearch.geometry.Line(new double[] { -179, 179 }, new double[] { -1, 1 }));
+            assertThat(geoValue.relate(latLonGeometry), Matchers.equalTo(GeoRelation.QUERY_CROSSES));
+        }
+        // polygons
+        {
+            GeoShapeValues.GeoShapeValue geoValue = GeoTestUtils.geoShapeValue(
+                new org.elasticsearch.geometry.Polygon(new LinearRing(new double[] { 0, 0, 1, 0 }, new double[] { -1, 1, 1, -1 }))
+            );
+            assertThat(geoValue.relate(latLonGeometry), Matchers.equalTo(GeoRelation.QUERY_DISJOINT));
+            geoValue = GeoTestUtils.geoShapeValue(
+                new org.elasticsearch.geometry.Polygon(new LinearRing(new double[] { 180, 180, 179, 180 }, new double[] { -1, 1, 1, -1 }))
+            );
+            assertThat(geoValue.relate(latLonGeometry), Matchers.equalTo(GeoRelation.QUERY_CONTAINS));
+            geoValue = GeoTestUtils.geoShapeValue(
+                new org.elasticsearch.geometry.Polygon(
+                    new LinearRing(new double[] { -180, -180, -179, -180 }, new double[] { -1, 1, 1, -1 })
+                )
+            );
+            assertThat(geoValue.relate(latLonGeometry), Matchers.equalTo(GeoRelation.QUERY_CONTAINS));
+            geoValue = GeoTestUtils.geoShapeValue(
+                new org.elasticsearch.geometry.Polygon(new LinearRing(new double[] { 179, 179, 179.5, 179 }, new double[] { -1, 1, 1, -1 }))
+            );
+            assertThat(geoValue.relate(latLonGeometry), Matchers.equalTo(GeoRelation.QUERY_CONTAINS));
+            geoValue = GeoTestUtils.geoShapeValue(
+                new org.elasticsearch.geometry.Polygon(
+                    new LinearRing(new double[] { -179, -179, -179.5, -179 }, new double[] { -1, 1, 1, -1 })
+                )
+            );
+            assertThat(geoValue.relate(latLonGeometry), Matchers.equalTo(GeoRelation.QUERY_CONTAINS));
+            geoValue = GeoTestUtils.geoShapeValue(
+                new org.elasticsearch.geometry.Polygon(
+                    new LinearRing(new double[] { -179, 179, -178, -179 }, new double[] { -1, 1, 1, -1 })
+                )
+            );
+            assertThat(geoValue.relate(latLonGeometry), Matchers.equalTo(GeoRelation.QUERY_CROSSES));
+        }
+    }
+
+    public void testRandomBasic() throws IOException {
+        for (int res = 0; res < H3.MAX_H3_RES; res++) {
+            final long h3 = H3.geoToH3(0, 0, res);
+            final GeoShapeValues.GeoShapeValue geoValue = GeoTestUtils.geoShapeValue(H3CartesianUtil.getNormalizeGeometry(h3));
+            final long[] children = H3.h3ToChildren(h3);
+            assertThat(geoValue.relate(getComponent(children[0])), Matchers.equalTo(GeoRelation.QUERY_INSIDE));
+            for (int i = 1; i < children.length; i++) {
+                assertThat(geoValue.relate(getComponent(children[i])), Matchers.equalTo(GeoRelation.QUERY_CROSSES));
+            }
+            for (long noChild : H3.h3ToNoChildrenIntersecting(h3)) {
+                assertThat(geoValue.relate(getComponent(noChild)), Matchers.equalTo(GeoRelation.QUERY_CROSSES));
+            }
+        }
+    }
+
+    public void testRandomDateline() throws IOException {
+        for (int res = 0; res < H3.MAX_H3_RES; res++) {
+            final long h3 = H3.geoToH3(0, 180, res);
+            final GeoShapeValues.GeoShapeValue geoValue = GeoTestUtils.geoShapeValue(H3CartesianUtil.getNormalizeGeometry(h3));
+            final long[] children = H3.h3ToChildren(h3);
+            final Component2D component2D = getComponent(children[0]);
+            // this is a current limitation because we break polygons around the dateline.
+            final GeoRelation expected = component2D.getMaxX() - component2D.getMinX() == 360d
+                ? GeoRelation.QUERY_CROSSES
+                : GeoRelation.QUERY_INSIDE;
+            assertThat(geoValue.relate(component2D), Matchers.equalTo(expected));
+            for (int i = 1; i < children.length; i++) {
+                assertThat(geoValue.relate(getComponent(children[i])), Matchers.equalTo(GeoRelation.QUERY_CROSSES));
+            }
+            for (long noChild : H3.h3ToNoChildrenIntersecting(h3)) {
+                assertThat(geoValue.relate(getComponent(noChild)), Matchers.equalTo(GeoRelation.QUERY_CROSSES));
+            }
+        }
+    }
+
+    private static Component2D getComponent(long h3) {
+        return LatLonGeometry.create(H3CartesianUtil.getLatLonGeometry(h3));
+    }
+
+    public void testBoundingBox() {
+        final double[] xs = new double[H3CartesianUtil.MAX_ARRAY_SIZE];
+        final double[] ys = new double[H3CartesianUtil.MAX_ARRAY_SIZE];
+        for (int res = 0; res <= H3.MAX_H3_RES; res++) {
+            final long h3 = H3.geoToH3(GeoTestUtil.nextLatitude(), GeoTestUtil.nextLongitude(), res);
+            final Rectangle rectangle = H3CartesianUtil.toBoundingBox(h3);
+            final int points = H3CartesianUtil.computePoints(h3, xs, ys);
+            for (int i = 0; i < points; i++) {
+                final double lat = ys[i];
+                final double lon = xs[i];
+                assertTrue(rectangle.getMaxY() >= lat && rectangle.getMinY() <= lat);
+                if (rectangle.getMinX() > rectangle.getMaxX()) {
+                    assertTrue(rectangle.getMaxX() >= lon || rectangle.getMinX() <= lon);
+                } else {
+                    assertTrue(rectangle.getMaxX() >= lon && rectangle.getMinX() <= lon);
+                }
+            }
+        }
+    }
+
+    public void testWronglyNormaliseGeometry() {
+        // this bin fails normalising in CCW, so we added hack to normalise it CW. If this test fails
+        // it might mean we fixed the bug in the GeometryNormaliser.
+        CellBoundary cellBoundary = H3.h3ToGeoBoundary(576531121047601151L);
+        double[] xs = new double[cellBoundary.numPoints() + 1];
+        double[] ys = new double[cellBoundary.numPoints() + 1];
+        for (int i = 0; i < cellBoundary.numPoints(); i++) {
+            final LatLng latLng = cellBoundary.getLatLon(i);
+            xs[i] = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(latLng.getLonDeg()));
+            ys[i] = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(latLng.getLatDeg()));
+        }
+        xs[cellBoundary.numPoints()] = xs[0];
+        ys[cellBoundary.numPoints()] = ys[0];
+        Polygon polygon = new Polygon(new LinearRing(xs, ys));
+        assertThat(GeometryNormalizer.apply(Orientation.CCW, polygon), Matchers.instanceOf(Polygon.class));
+        assertThat(GeometryNormalizer.apply(Orientation.CW, polygon), Matchers.instanceOf(MultiPolygon.class));
+    }
+
+    public void testNormaliseGeometry() throws IOException {
+        Point point = GeometryTestUtils.randomPoint();
+        for (int res = 0; res <= H3.MAX_H3_RES; res++) {
+            long h3 = H3.geoToH3(point.getLat(), point.getLon(), res);
+            assertNotNull(GeoTestUtils.geoShapeValue(H3CartesianUtil.getNormalizeGeometry(h3)));
+        }
+    }
+}
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/common/H3SphericalGeometryTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/common/H3SphericalGeometryTests.java
new file mode 100644
index 000000000000..ae381f40a1c2
--- /dev/null
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/common/H3SphericalGeometryTests.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.spatial.common;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.LatLonPoint;
+import org.apache.lucene.document.ShapeField;
+import org.apache.lucene.geo.GeoEncodingUtils;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.SerialMergeScheduler;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.geo.GeometryTestUtils;
+import org.elasticsearch.geometry.Point;
+import org.elasticsearch.h3.CellBoundary;
+import org.elasticsearch.h3.H3;
+import org.elasticsearch.h3.LatLng;
+import org.elasticsearch.test.ESTestCase;
+
+public class H3SphericalGeometryTests extends ESTestCase {
+
+    private static final String FIELD_NAME = "field";
+
+    public void testIndexPoints() throws Exception {
+        Point queryPoint = GeometryTestUtils.randomPoint();
+        long[] hexes = new long[H3.MAX_H3_RES + 1];
+        for (int res = 0; res < hexes.length; res++) {
+            hexes[res] = H3.geoToH3(queryPoint.getLat(), queryPoint.getLon(), res);
+        }
+        IndexWriterConfig iwc = newIndexWriterConfig();
+        // Else seeds may not reproduce:
+        iwc.setMergeScheduler(new SerialMergeScheduler());
+        // Else we can get O(N^2) merging:
+        iwc.setMaxBufferedDocs(10);
+        Directory dir = newDirectory();
+        // RandomIndexWriter is too slow here:
+        int[] counts = new int[H3.MAX_H3_RES + 1];
+        IndexWriter w = new IndexWriter(dir, iwc);
+        for (long hex : hexes) {
+            CellBoundary cellBoundary = H3.h3ToGeoBoundary(hex);
+            for (int i = 0; i < cellBoundary.numPoints(); i++) {
+                Document doc = new Document();
+                LatLng latLng = cellBoundary.getLatLon(i);
+                doc.add(new LatLonPoint(FIELD_NAME, latLng.getLatDeg(), latLng.getLonDeg()));
+                w.addDocument(doc);
+                computeCounts(hexes, latLng.getLonDeg(), latLng.getLatDeg(), counts);
+            }
+
+        }
+        final int numDocs = randomIntBetween(1000, 2000);
+        for (int id = 0; id < numDocs; id++) {
+            Document doc = new Document();
+            Point point = GeometryTestUtils.randomPoint();
+            doc.add(new LatLonPoint(FIELD_NAME, point.getLat(), point.getLon()));
+            w.addDocument(doc);
+            computeCounts(hexes, point.getLon(), point.getLat(), counts);
+        }
+
+        if (random().nextBoolean()) {
+            w.forceMerge(1);
+        }
+        final IndexReader r = DirectoryReader.open(w);
+        w.close();
+
+        IndexSearcher s = newSearcher(r);
+        for (int i = 0; i < H3.MAX_H3_RES + 1; i++) {
+            H3SphericalGeometry geometry = new H3SphericalGeometry(hexes[i]);
+            Query indexQuery = LatLonPoint.newGeometryQuery(FIELD_NAME, ShapeField.QueryRelation.INTERSECTS, geometry);
+            assertEquals(counts[i], s.count(indexQuery));
+        }
+        IOUtils.close(r, dir);
+    }
+
+    private void computeCounts(long[] hexes, double lon, double lat, int[] counts) {
+        double qLat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat));
+        double qLon = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lon));
+        for (int res = 0; res < hexes.length; res++) {
+            if (hexes[res] == H3.geoToH3(qLat, qLon, res)) {
+                counts[res]++;
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/common/H3SphericalUtilTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/common/H3SphericalUtilTests.java
new file mode 100644
index 000000000000..5653d46dd65b
--- /dev/null
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/common/H3SphericalUtilTests.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.spatial.common;
+
+import org.apache.lucene.spatial3d.geom.GeoPolygon;
+import org.apache.lucene.spatial3d.geom.LatLonBounds;
+import org.apache.lucene.tests.geo.GeoTestUtil;
+import org.elasticsearch.common.geo.GeoBoundingBox;
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.h3.H3;
+import org.elasticsearch.test.ESTestCase;
+
+public class H3SphericalUtilTests extends ESTestCase {
+
+    private static final double LAT_DELTA = 1e-7;
+    private static final double LON_DELTA = 1e-5;
+
+    public void testRandomBounds() {
+        GeoBoundingBox boundingBox = new GeoBoundingBox(new GeoPoint(), new GeoPoint());
+        for (int res = 0; res < H3.MAX_H3_RES; res++) {
+            final long h3 = H3.geoToH3(GeoTestUtil.nextLatitude(), GeoTestUtil.nextLongitude(), res);
+            assertBounds(h3, boundingBox);
+        }
+    }
+
+    public void testBoundsLevel0() {
+        GeoBoundingBox boundingBox = new GeoBoundingBox(new GeoPoint(), new GeoPoint());
+        for (long h3 : H3.getLongRes0Cells()) {
+            assertBounds(h3, boundingBox);
+        }
+    }
+
+    private void assertBounds(long h3, GeoBoundingBox boundingBox) {
+        H3SphericalUtil.computeGeoBounds(h3, boundingBox);
+        GeoPolygon polygon = H3SphericalUtil.toGeoPolygon(h3);
+        LatLonBounds bounds = new LatLonBounds();
+        polygon.getBounds(bounds);
+        if (bounds.checkNoLongitudeBound()) {
+            assertEquals(-180d, boundingBox.left(), LON_DELTA);
+            assertEquals(180d, boundingBox.right(), LON_DELTA);
+        } else {
+            assertEquals(Math.toDegrees(bounds.getLeftLongitude()), boundingBox.left(), LON_DELTA);
+            assertEquals(Math.toDegrees(bounds.getRightLongitude()), boundingBox.right(), LON_DELTA);
+        }
+
+        if (bounds.checkNoTopLatitudeBound()) {
+            assertEquals(90d, boundingBox.top(), LAT_DELTA);
+        } else {
+            assertEquals(Math.toDegrees(bounds.getMaxLatitude()), boundingBox.top(), LAT_DELTA);
+        }
+
+        if (bounds.checkNoBottomLatitudeBound()) {
+            assertEquals(-90d, boundingBox.bottom(), LAT_DELTA);
+        } else {
+            assertEquals(Math.toDegrees(bounds.getMinLatitude()), boundingBox.bottom(), LAT_DELTA);
+        }
+    }
+}
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/fielddata/LatLonGeometryRelationVisitorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/fielddata/LatLonGeometryRelationVisitorTests.java
index 3a0d0bcd0356..3ba3b513ebfa 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/fielddata/LatLonGeometryRelationVisitorTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/fielddata/LatLonGeometryRelationVisitorTests.java
@@ -7,17 +7,32 @@
 
 package org.elasticsearch.xpack.spatial.index.fielddata;
 
-import org.apache.lucene.document.ShapeField;
+import org.apache.lucene.document.ShapeField.QueryRelation;
 import org.apache.lucene.geo.Component2D;
 import org.apache.lucene.geo.LatLonGeometry;
+import org.apache.lucene.geo.Line;
+import org.apache.lucene.geo.Point;
+import org.apache.lucene.geo.Polygon;
 import org.apache.lucene.tests.geo.GeoTestUtil;
+import org.elasticsearch.common.geo.GeometryNormalizer;
+import org.elasticsearch.common.geo.Orientation;
 import org.elasticsearch.geo.GeometryTestUtils;
 import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.geometry.LinearRing;
+import org.elasticsearch.geometry.MultiPoint;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.spatial.util.GeoTestUtils;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.Map;
 import java.util.function.Supplier;
 
+import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude;
+import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude;
+import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude;
+import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude;
 import static org.hamcrest.Matchers.equalTo;
 
 public class LatLonGeometryRelationVisitorTests extends ESTestCase {
@@ -26,6 +41,8 @@ public void testPoint() throws Exception {
         doTestShapes(GeoTestUtil::nextPoint);
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/92142")
+    // This is waiting for a Lucene release that includes https://github.com/apache/lucene/pull/12022
     public void testLine() throws Exception {
         doTestShapes(GeoTestUtil::nextLine);
     }
@@ -34,45 +51,229 @@ public void testPolygon() throws Exception {
         doTestShapes(GeoTestUtil::nextPolygon);
     }
 
-    private void doTestShapes(Supplier<LatLonGeometry> supplier) throws Exception {
-        Geometry geometry = GeometryTestUtils.randomGeometryWithoutCircle(0, false);
+    // Specific tests for known troublesome points from https://github.com/elastic/elasticsearch/issues/92151
+    public void testTroublesomePoints() throws Exception {
+        ArrayList<org.elasticsearch.geometry.Point> points = new ArrayList<>();
+        points.add(new org.elasticsearch.geometry.Point(-6.6957112520498185, 5.337277253715181E-129));
+        points.add(new org.elasticsearch.geometry.Point(0.0, 1.6938947866910307E-202));
+        points.add(new org.elasticsearch.geometry.Point(-114.40977624485328, -37.484381576244864));
+        points.add(new org.elasticsearch.geometry.Point(49.1828546738179, 23.813793855174865));
+        points.add(new org.elasticsearch.geometry.Point(60.5683489131913, 90.0));
+        points.add(new org.elasticsearch.geometry.Point(-79.65717776327665, -39.5));
+        points.add(new org.elasticsearch.geometry.Point(1.401298464324817E-45, 0.0));
+        MultiPoint geometry = new MultiPoint(points);
         GeoShapeValues.GeoShapeValue geoShapeValue = GeoTestUtils.geoShapeValue(geometry);
         GeometryDocValueReader reader = GeoTestUtils.geometryDocValueReader(geometry, CoordinateEncoder.GEO);
-        for (int i = 0; i < 1000; i++) {
-            LatLonGeometry latLonGeometry = supplier.get();
+        Point[] troublesome = new Point[] {
+            new Point(-6.404126347681029E-213, 0.0),    // This point will match the last point in the multipoint
+            new Point(0.0, 0.0)                         // this point will match the second point in the multipoint
+        };
+        for (Point point : troublesome) {
+            doTestShape(geometry, geoShapeValue, reader, quantize(point));
+        }
+    }
+
+    public void testIdenticalPoint() throws Exception {
+        double x = quantizeLon(1);
+        double y = quantizeLat(1);
+        org.elasticsearch.geometry.Point shape = new org.elasticsearch.geometry.Point(x, y);
+        Point latLonGeometry = new Point(shape.getLat(), shape.getLon());
+
+        GeoShapeValues.GeoShapeValue geoShapeValue = GeoTestUtils.geoShapeValue(shape);
+        GeometryDocValueReader reader = GeoTestUtils.geometryDocValueReader(shape, CoordinateEncoder.GEO);
+        GeoRelation relation = geoShapeValue.relate(latLonGeometry);
+        assertThat("Identical points", relation, equalTo(GeoRelation.QUERY_INSIDE));
+        doTestShape(shape, reader, latLonGeometry, relation, true);
+    }
+
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/92142")
+    // This is waiting for a Lucene release that includes https://github.com/apache/lucene/pull/12022
+    public void testVeryFlatPolygonDoesNotContainIntersectingLine() throws Exception {
+        double[] x = new double[] { -0.001, -0.001, 0.001, 0.001, -0.001 };
+        double[] y = new double[] { 1e-10, 0, -1e-10, 0, 1e-10 };
+        Geometry geometry = new org.elasticsearch.geometry.Polygon(new LinearRing(x, y));
+        GeoShapeValues.GeoShapeValue geoShapeValue = GeoTestUtils.geoShapeValue(geometry);
+        GeometryDocValueReader reader = GeoTestUtils.geometryDocValueReader(geometry, CoordinateEncoder.GEO);
+        double[] lons = new double[] { 0.0, 0.0 };
+        double[] lats = new double[] { 0.0, 0.001 };
+        Line line = new Line(lats, lons);
+        doTestShape(geometry, geoShapeValue, reader, line);
+    }
+
+    public void testContainedPolygons() throws Exception {
+        // Create simple rectangular polygon
+        double[] x = new double[] { -1, 1, 1, -1, -1 };
+        double[] y = new double[] { -1, -1, 1, 1, -1 };
+        quantize(y, x);
+        org.elasticsearch.geometry.Polygon shape = new org.elasticsearch.geometry.Polygon(new LinearRing(x, y));
+
+        // Setup tests for contains, identical and within
+        LinkedHashMap<Double, GeoRelation> tests = new LinkedHashMap<>();
+        tests.put(0.5, GeoRelation.QUERY_INSIDE);
+        tests.put(1.0, GeoRelation.QUERY_CONTAINS);
+        tests.put(2.0, GeoRelation.QUERY_CONTAINS);
+        for (Map.Entry<Double, GeoRelation> entry : tests.entrySet()) {
+            double factor = entry.getKey();
+            GeoRelation expected = entry.getValue();
+            double[] lats = new double[y.length];
+            double[] lons = new double[x.length];
+            for (int i = 0; i < x.length; i++) {
+                lats[i] = quantizeLat(y[i] * factor);
+                lons[i] = quantizeLon(x[i] * factor);
+            }
+            Polygon latLonGeometry = new Polygon(lats, lons);
+            boolean identical = factor == 1.0;
+            // Assert that polygons are identical
+            if (identical) {
+                for (int i = 0; i < latLonGeometry.numPoints(); i++) {
+                    assertThat("Latitude[" + i + "]", latLonGeometry.getPolyLat(i), equalTo(shape.getPolygon().getLat(i)));
+                    assertThat("Longitude[" + i + "]", latLonGeometry.getPolyLon(i), equalTo(shape.getPolygon().getLon(i)));
+                }
+            }
+
+            GeoShapeValues.GeoShapeValue geoShapeValue = GeoTestUtils.geoShapeValue(shape);
+            GeometryDocValueReader reader = GeoTestUtils.geometryDocValueReader(shape, CoordinateEncoder.GEO);
             GeoRelation relation = geoShapeValue.relate(latLonGeometry);
-            Component2D component2D = LatLonGeometry.create(latLonGeometry);
-            Component2DVisitor contains = Component2DVisitor.getVisitor(
-                component2D,
-                ShapeField.QueryRelation.CONTAINS,
-                CoordinateEncoder.GEO
-            );
-            reader.visit(contains);
-            Component2DVisitor intersects = Component2DVisitor.getVisitor(
-                component2D,
-                ShapeField.QueryRelation.INTERSECTS,
-                CoordinateEncoder.GEO
-            );
-            reader.visit(intersects);
-            Component2DVisitor disjoint = Component2DVisitor.getVisitor(
-                component2D,
-                ShapeField.QueryRelation.DISJOINT,
-                CoordinateEncoder.GEO
-            );
-            reader.visit(disjoint);
-            if (relation == GeoRelation.QUERY_INSIDE) {
-                assertThat(contains.matches(), equalTo(true));
-                assertThat(intersects.matches(), equalTo(true));
-                assertThat(disjoint.matches(), equalTo(false));
-            } else if (relation == GeoRelation.QUERY_CROSSES) {
-                assertThat(contains.matches(), equalTo(false));
-                assertThat(intersects.matches(), equalTo(true));
-                assertThat(disjoint.matches(), equalTo(false));
-            } else {
-                assertThat(contains.matches(), equalTo(false));
-                assertThat(intersects.matches(), equalTo(false));
-                assertThat(disjoint.matches(), equalTo(true));
+            assertThat("Polygon scaled by " + factor, relation, equalTo(expected));
+            doTestShape(shape, reader, latLonGeometry, relation, false);
+        }
+    }
+
+    private <T extends LatLonGeometry> void doTestShapes(Supplier<T> supplier) throws Exception {
+        Geometry geometry = GeometryNormalizer.apply(Orientation.CCW, GeometryTestUtils.randomGeometryWithoutCircle(0, false));
+        GeoShapeValues.GeoShapeValue geoShapeValue = GeoTestUtils.geoShapeValue(geometry);
+        GeometryDocValueReader reader = GeoTestUtils.geometryDocValueReader(geometry, CoordinateEncoder.GEO);
+        for (int i = 0; i < 1000; i++) {
+            LatLonGeometry latLonGeometry = quantize(supplier.get());
+            doTestShape(geometry, geoShapeValue, reader, latLonGeometry);
+        }
+    }
+
+    private void doTestShape(
+        Geometry geometry,
+        GeoShapeValues.GeoShapeValue geoShapeValue,
+        GeometryDocValueReader reader,
+        LatLonGeometry latLonGeometry
+    ) throws Exception {
+        doTestShape(geometry, reader, latLonGeometry, geoShapeValue.relate(latLonGeometry));
+    }
+
+    private void doTestShape(Geometry geometry, GeometryDocValueReader reader, LatLonGeometry latLonGeometry, GeoRelation relation)
+        throws Exception {
+        doTestShape(geometry, reader, latLonGeometry, relation, isIdenticalPoint(geometry, latLonGeometry));
+    }
+
+    private boolean isIdenticalPoint(Geometry geometry, LatLonGeometry latLonGeometry) {
+        if (geometry instanceof org.elasticsearch.geometry.Point point) {
+            if (latLonGeometry instanceof Point latLonPoint) {
+                return encodeLatitude(point.getLat()) == encodeLatitude(latLonPoint.getLat())
+                    && encodeLongitude(point.getLon()) == encodeLongitude(latLonPoint.getLon());
             }
         }
+        return false;
+    }
+
+    private boolean pointsOnly(Geometry geometry) {
+        return geometry instanceof org.elasticsearch.geometry.Point || geometry instanceof org.elasticsearch.geometry.MultiPoint;
+    }
+
+    private boolean pointsOnly(LatLonGeometry geometry) {
+        return geometry instanceof Point;
+    }
+
+    private void doTestShape(
+        Geometry geometry,
+        GeometryDocValueReader reader,
+        LatLonGeometry latLonGeometry,
+        GeoRelation relation,
+        boolean identicalPoint  // When both geometries are points and identical, then CONTAINS==WITHIN
+    ) throws Exception {
+        boolean pointsOnly = pointsOnly(geometry) && pointsOnly(latLonGeometry);
+        String description = "Geometry " + latLonGeometry + " relates to shape " + geometry.getClass().getSimpleName() + ": " + relation;
+        Component2D component2D = LatLonGeometry.create(latLonGeometry);
+        Component2DVisitor contains = visitQueryRelation(component2D, QueryRelation.CONTAINS, reader);
+        Component2DVisitor intersects = visitQueryRelation(component2D, QueryRelation.INTERSECTS, reader);
+        Component2DVisitor disjoint = visitQueryRelation(component2D, QueryRelation.DISJOINT, reader);
+        Component2DVisitor within = visitQueryRelation(component2D, QueryRelation.WITHIN, reader);
+        if (relation == GeoRelation.QUERY_INSIDE) {
+            assertThat("CONTAINS/" + relation + ": " + description, contains.matches(), equalTo(true));
+            assertThat("INTERSECTS/" + relation + ": " + description, intersects.matches(), equalTo(true));
+            assertThat("DISJOINT/" + relation + ": " + description, disjoint.matches(), equalTo(false));
+            assertThat("WITHIN/" + relation + ": " + description, within.matches(), equalTo(identicalPoint));
+        } else if (relation == GeoRelation.QUERY_CROSSES) {
+            if (pointsOnly == false) {
+                // When we have point comparisons, CROSSES can also allow CONTAINS
+                assertThat("CONTAINS/" + relation + ": " + description, contains.matches(), equalTo(false));
+            }
+            assertThat("INTERSECTS/" + relation + ": " + description, intersects.matches(), equalTo(true));
+            assertThat("DISJOINT/" + relation + ": " + description, disjoint.matches(), equalTo(false));
+            assertThat("WITHIN/" + relation + ": " + description, within.matches(), equalTo(false));
+        } else if (relation == GeoRelation.QUERY_CONTAINS) {
+            assertThat("CONTAINS/" + relation + ": " + description, contains.matches(), equalTo(identicalPoint));
+            assertThat("INTERSECTS/" + relation + ": " + description, intersects.matches(), equalTo(true));
+            assertThat("DISJOINT/" + relation + ": " + description, disjoint.matches(), equalTo(false));
+            assertThat("WITHIN/" + relation + ": " + description, within.matches(), equalTo(true));
+        } else {
+            assertThat("CONTAINS/" + relation + ": " + description, contains.matches(), equalTo(false));
+            assertThat("INTERSECTS/" + relation + ": " + description, intersects.matches(), equalTo(false));
+            assertThat("DISJOINT/" + relation + ": " + description, disjoint.matches(), equalTo(true));
+            assertThat("WITHIN/" + relation + ": " + description, within.matches(), equalTo(false));
+        }
+    }
+
+    private Component2DVisitor visitQueryRelation(Component2D component2D, QueryRelation queryRelation, GeometryDocValueReader reader)
+        throws IOException {
+        Component2DVisitor contains = Component2DVisitor.getVisitor(component2D, queryRelation, CoordinateEncoder.GEO);
+        reader.visit(contains);
+        return contains;
+    }
+
+    private LatLonGeometry quantize(LatLonGeometry geometry) {
+        if (geometry instanceof Point point) {
+            return quantize(point);
+        } else if (geometry instanceof Line line) {
+            return quantize(line);
+        } else if (geometry instanceof Polygon polygon) {
+            return quantize(polygon);
+        } else {
+            throw new IllegalArgumentException("Unimplemented: quantize(" + geometry.getClass().getSimpleName() + ")");
+        }
+    }
+
+    private Point quantize(Point point) {
+        return new Point(quantizeLat(point.getLat()), quantizeLon(point.getLon()));
+    }
+
+    private Line quantize(Line line) {
+        double[] lons = line.getLons();
+        double[] lats = line.getLats();
+        quantize(lats, lons);
+        return new Line(lats, lons);
+    }
+
+    private Polygon quantize(Polygon polygon) {
+        Polygon[] holes = polygon.getHoles();
+        for (int i = 0; i < holes.length; i++) {
+            holes[i] = quantize(holes[i]);
+        }
+        double[] lats = polygon.getPolyLats();
+        double[] lons = polygon.getPolyLons();
+        quantize(lats, lons);
+        return new Polygon(lats, lons, holes);
+    }
+
+    private void quantize(double[] lats, double[] lons) {
+        for (int i = 0; i < lons.length; i++) {
+            lats[i] = quantizeLat(lats[i]);
+            lons[i] = quantizeLon(lons[i]);
+        }
+    }
+
+    private double quantizeLat(double lat) {
+        return decodeLatitude(encodeLatitude(lat));
+    }
+
+    private double quantizeLon(double lon) {
+        return decodeLongitude(encodeLongitude(lon));
     }
 }
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/fielddata/Tile2DVisitorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/fielddata/Tile2DVisitorTests.java
index 5d4685f69f88..959b91ffbc91 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/fielddata/Tile2DVisitorTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/fielddata/Tile2DVisitorTests.java
@@ -33,7 +33,7 @@
 import static org.elasticsearch.geo.GeometryTestUtils.randomMultiLine;
 import static org.elasticsearch.geo.GeometryTestUtils.randomMultiPolygon;
 import static org.elasticsearch.geo.GeometryTestUtils.randomPoint;
-import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.in;
 
 public class Tile2DVisitorTests extends ESTestCase {
 
@@ -45,10 +45,10 @@ public void testPacManPolygon() throws Exception {
         // test cell crossing poly
         Polygon pacMan = new Polygon(new LinearRing(py, px), Collections.emptyList());
         GeometryDocValueReader reader = GeoTestUtils.geometryDocValueReader(pacMan, TestCoordinateEncoder.INSTANCE);
-        assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(2, -1, 11, 1));
-        assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-12, -12, 12, 12));
-        assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-2, -1, 2, 0));
-        assertRelation(GeoRelation.QUERY_INSIDE, reader, getExtentFromBox(-5, -6, 2, -2));
+        assertRelation(reader, getExtentFromBox(2, -1, 11, 1), GeoRelation.QUERY_CROSSES);
+        assertRelation(reader, getExtentFromBox(-12, -12, 12, 12), GeoRelation.QUERY_CONTAINS);
+        assertRelation(reader, getExtentFromBox(-2, -1, 2, 0), GeoRelation.QUERY_CROSSES);
+        assertRelation(reader, getExtentFromBox(-5, -6, 2, -2), GeoRelation.QUERY_INSIDE);
     }
 
     // adapted from org.apache.lucene.geo.TestPolygon2D#testMultiPolygon
@@ -60,12 +60,12 @@ public void testPolygonWithHole() throws Exception {
 
         GeometryDocValueReader reader = GeoTestUtils.geometryDocValueReader(polyWithHole, CoordinateEncoder.GEO);
 
-        assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(6, -6, 6, -6)); // in the hole
-        assertRelation(GeoRelation.QUERY_INSIDE, reader, getExtentFromBox(25, -25, 25, -25)); // on the mainland
-        assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(51, 51, 52, 52)); // outside of mainland
-        assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-60, -60, 60, 60)); // enclosing us completely
-        assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(49, 49, 51, 51)); // overlapping the mainland
-        assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(9, 9, 11, 11)); // overlapping the hole
+        assertRelation(reader, getExtentFromBox(6, -6, 6, -6), GeoRelation.QUERY_DISJOINT); // in the hole
+        assertRelation(reader, getExtentFromBox(25, -25, 25, -25), GeoRelation.QUERY_INSIDE); // on the mainland
+        assertRelation(reader, getExtentFromBox(51, 51, 52, 52), GeoRelation.QUERY_DISJOINT); // outside of mainland
+        assertRelation(reader, getExtentFromBox(-60, -60, 60, 60), GeoRelation.QUERY_CONTAINS); // enclosing us completely
+        assertRelation(reader, getExtentFromBox(49, 49, 51, 51), GeoRelation.QUERY_CROSSES); // overlapping the mainland
+        assertRelation(reader, getExtentFromBox(9, 9, 11, 11), GeoRelation.QUERY_CROSSES); // overlapping the hole
     }
 
     public void testCombPolygon() throws Exception {
@@ -78,9 +78,9 @@ public void testCombPolygon() throws Exception {
         Polygon polyWithHole = new Polygon(new LinearRing(px, py), Collections.singletonList(new LinearRing(hx, hy)));
         GeometryDocValueReader reader = GeoTestUtils.geometryDocValueReader(polyWithHole, CoordinateEncoder.GEO);
         // test cell crossing poly
-        assertRelation(GeoRelation.QUERY_INSIDE, reader, getExtentFromBox(5, 10, 5, 10));
-        assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(15, 10, 15, 10));
-        assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(25, 10, 25, 10));
+        assertRelation(reader, getExtentFromBox(5, 10, 5, 10), GeoRelation.QUERY_INSIDE);
+        assertRelation(reader, getExtentFromBox(15, 10, 15, 10), GeoRelation.QUERY_DISJOINT);
+        assertRelation(reader, getExtentFromBox(25, 10, 25, 10), GeoRelation.QUERY_DISJOINT);
     }
 
     public void testPacManClosedLineString() throws Exception {
@@ -90,10 +90,10 @@ public void testPacManClosedLineString() throws Exception {
 
         // test cell crossing poly
         GeometryDocValueReader reader = GeoTestUtils.geometryDocValueReader(new Line(px, py), CoordinateEncoder.GEO);
-        assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(2, -1, 11, 1));
-        assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-12, -12, 12, 12));
-        assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-2, -1, 2, 0));
-        assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(-5, -6, 2, -2));
+        assertRelation(reader, getExtentFromBox(2, -1, 11, 1), GeoRelation.QUERY_CROSSES);
+        assertRelation(reader, getExtentFromBox(-12, -12, 12, 12), GeoRelation.QUERY_CONTAINS);
+        assertRelation(reader, getExtentFromBox(-2, -1, 2, 0), GeoRelation.QUERY_CROSSES);
+        assertRelation(reader, getExtentFromBox(-5, -6, 2, -2), GeoRelation.QUERY_DISJOINT);
     }
 
     public void testPacManLineString() throws Exception {
@@ -103,10 +103,10 @@ public void testPacManLineString() throws Exception {
 
         // test cell crossing poly
         GeometryDocValueReader reader = GeoTestUtils.geometryDocValueReader(new Line(px, py), CoordinateEncoder.GEO);
-        assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(2, -1, 11, 1));
-        assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-12, -12, 12, 12));
-        assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-2, -1, 2, 0));
-        assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(-5, -6, 2, -2));
+        assertRelation(reader, getExtentFromBox(2, -1, 11, 1), GeoRelation.QUERY_CROSSES);
+        assertRelation(reader, getExtentFromBox(-12, -12, 12, 12), GeoRelation.QUERY_CONTAINS);
+        assertRelation(reader, getExtentFromBox(-2, -1, 2, 0), GeoRelation.QUERY_CROSSES);
+        assertRelation(reader, getExtentFromBox(-5, -6, 2, -2), GeoRelation.QUERY_DISJOINT);
     }
 
     public void testPacManPoints() throws Exception {
@@ -132,7 +132,7 @@ public void testPacManPoints() throws Exception {
 
         // test cell crossing poly
         GeometryDocValueReader reader = GeoTestUtils.geometryDocValueReader(new MultiPoint(points), CoordinateEncoder.GEO);
-        assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(xMin, yMin, xMax, yMax));
+        assertRelation(reader, getExtentFromBox(xMin, yMin, xMax, yMax), GeoRelation.QUERY_CROSSES);
     }
 
     public void testRandomMultiLineIntersections() throws IOException {
@@ -148,23 +148,24 @@ public void testRandomMultiLineIntersections() throws IOException {
                 && lineExtent.minY() != Integer.MIN_VALUE
                 && lineExtent.maxY() != Integer.MAX_VALUE) {
                 assertRelation(
-                    GeoRelation.QUERY_CROSSES,
                     reader,
-                    Extent.fromPoints(lineExtent.minX() - 1, lineExtent.minY() - 1, lineExtent.maxX() + 1, lineExtent.maxY() + 1)
+                    Extent.fromPoints(lineExtent.minX() - 1, lineExtent.minY() - 1, lineExtent.maxX() + 1, lineExtent.maxY() + 1),
+                    GeoRelation.QUERY_CROSSES,
+                    GeoRelation.QUERY_CONTAINS
                 );
             }
         }
 
         // extent that fully encloses the MultiLine
-        assertRelation(GeoRelation.QUERY_CROSSES, reader, reader.getExtent());
+        assertRelation(reader, reader.getExtent(), GeoRelation.QUERY_CONTAINS);
         if (readerExtent.minX() != Integer.MIN_VALUE
             && readerExtent.maxX() != Integer.MAX_VALUE
             && readerExtent.minY() != Integer.MIN_VALUE
             && readerExtent.maxY() != Integer.MAX_VALUE) {
             assertRelation(
-                GeoRelation.QUERY_CROSSES,
                 reader,
-                Extent.fromPoints(readerExtent.minX() - 1, readerExtent.minY() - 1, readerExtent.maxX() + 1, readerExtent.maxY() + 1)
+                Extent.fromPoints(readerExtent.minX() - 1, readerExtent.minY() - 1, readerExtent.maxX() + 1, readerExtent.maxY() + 1),
+                GeoRelation.QUERY_CONTAINS
             );
         }
 
@@ -211,12 +212,11 @@ private static Extent getExtentFromBox(double bottomLeftX, double bottomLeftY, d
     }
 
     private boolean intersects(Geometry g, Point p, double extentSize) throws IOException {
-
         Extent bufferBounds = bufferedExtentFromGeoPoint(p.getX(), p.getY(), extentSize);
         Tile2DVisitor tile2DVisitor = new Tile2DVisitor();
         tile2DVisitor.reset(bufferBounds.minX(), bufferBounds.minY(), bufferBounds.maxX(), bufferBounds.maxY());
         GeoTestUtils.geometryDocValueReader(g, CoordinateEncoder.GEO).visit(tile2DVisitor);
-        return tile2DVisitor.relation() == GeoRelation.QUERY_CROSSES || tile2DVisitor.relation() == GeoRelation.QUERY_INSIDE;
+        return tile2DVisitor.relation() != GeoRelation.QUERY_DISJOINT;
     }
 
     /**
@@ -281,10 +281,10 @@ public R visit(Rectangle rectangle) throws E {
         });
     }
 
-    static void assertRelation(GeoRelation expectedRelation, GeometryDocValueReader reader, Extent extent) throws IOException {
+    static void assertRelation(GeometryDocValueReader reader, Extent extent, GeoRelation... expectedRelation) throws IOException {
         Tile2DVisitor tile2DVisitor = new Tile2DVisitor();
         tile2DVisitor.reset(extent.minX(), extent.minY(), extent.maxX(), extent.maxY());
         reader.visit(tile2DVisitor);
-        assertThat(expectedRelation, equalTo(tile2DVisitor.relation()));
+        assertThat(tile2DVisitor.relation(), in(expectedRelation));
     }
 }
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/fielddata/TriangleTreeTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/fielddata/TriangleTreeTests.java
index c9fa0d65cf82..435d0f314412 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/fielddata/TriangleTreeTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/fielddata/TriangleTreeTests.java
@@ -42,7 +42,7 @@ public void testVisitAllTriangles() throws IOException {
         assertThat(fieldList.size(), equalTo(visitor.counter));
     }
 
-    private static class TriangleCounterVisitor implements TriangleTreeReader.Visitor {
+    private static class TriangleCounterVisitor implements TriangleTreeVisitor {
 
         int counter;
 
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilderTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilderTests.java
index 2c18bd32a7d7..fc782adc5c5c 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilderTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilderTests.java
@@ -123,7 +123,7 @@ protected void doAssertLuceneQuery(GeoGridQueryBuilder queryBuilder, Query query
     }
 
     public void testParsingAndToQueryGeohex() throws IOException {
-        String query = formatted("""
+        String query = Strings.format("""
             {
                 "geo_grid":{
                     "%s":{
@@ -136,7 +136,7 @@ public void testParsingAndToQueryGeohex() throws IOException {
     }
 
     public void testParsingAndToQueryGeotile() throws IOException {
-        String query = formatted("""
+        String query = Strings.format("""
             {
                 "geo_grid":{
                     "%s":{
@@ -149,7 +149,7 @@ public void testParsingAndToQueryGeotile() throws IOException {
     }
 
     public void testParsingAndToQueryGeohash() throws IOException {
-        String query = formatted("""
+        String query = Strings.format("""
             {
                 "geo_grid":{
                     "%s":{
@@ -168,7 +168,7 @@ private void assertGeoGridQuery(String query) throws IOException {
     }
 
     public void testMalformedGeoTile() {
-        String query = formatted("""
+        String query = Strings.format("""
             {
                 "geo_grid":{
                     "%s":{
@@ -182,7 +182,7 @@ public void testMalformedGeoTile() {
     }
 
     public void testMalformedGeohash() {
-        String query = formatted("""
+        String query = Strings.format("""
             {
                 "geo_grid":{
                     "%s":{
@@ -196,7 +196,7 @@ public void testMalformedGeohash() {
     }
 
     public void testMalformedGeohex() {
-        String query = formatted("""
+        String query = Strings.format("""
             {
                 "geo_grid":{
                     "%s":{
@@ -210,7 +210,7 @@ public void testMalformedGeohex() {
     }
 
     public void testWrongField() {
-        String query = formatted("""
+        String query = Strings.format("""
             {
                 "geo_grid":{
                     "%s":{
@@ -224,7 +224,7 @@ public void testWrongField() {
     }
 
     public void testDuplicateField() {
-        String query = formatted("""
+        String query = Strings.format("""
             {
                 "geo_grid":{
                     "%s":{
@@ -278,6 +278,7 @@ private void assertBoundingBox(String hash, int precision, double lon, double la
         );
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/92611")
     public void testBoundingBoxQuantize() {
         double lat = randomDoubleBetween(-GeoTileUtils.LATITUDE_MASK, GeoTileUtils.LATITUDE_MASK, true);
         double lon = randomDoubleBetween(-180d, 180d, true);
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/H3LatLonGeometryTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/H3LatLonGeometryTests.java
deleted file mode 100644
index 5cd16552632f..000000000000
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/H3LatLonGeometryTests.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.spatial.index.query;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.LatLonPoint;
-import org.apache.lucene.document.ShapeField;
-import org.apache.lucene.geo.GeoEncodingUtils;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.SerialMergeScheduler;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.store.Directory;
-import org.elasticsearch.core.IOUtils;
-import org.elasticsearch.geo.GeometryTestUtils;
-import org.elasticsearch.geometry.Point;
-import org.elasticsearch.h3.CellBoundary;
-import org.elasticsearch.h3.H3;
-import org.elasticsearch.h3.LatLng;
-import org.elasticsearch.test.ESTestCase;
-
-public class H3LatLonGeometryTests extends ESTestCase {
-
-    private static final String FIELD_NAME = "field";
-
-    public void testIndexPoints() throws Exception {
-        Point queryPoint = GeometryTestUtils.randomPoint();
-        String[] hexes = new String[H3.MAX_H3_RES + 1];
-        for (int res = 0; res < hexes.length; res++) {
-            hexes[res] = H3.geoToH3Address(queryPoint.getLat(), queryPoint.getLon(), res);
-        }
-        IndexWriterConfig iwc = newIndexWriterConfig();
-        // Else seeds may not reproduce:
-        iwc.setMergeScheduler(new SerialMergeScheduler());
-        // Else we can get O(N^2) merging:
-        iwc.setMaxBufferedDocs(10);
-        Directory dir = newDirectory();
-        // RandomIndexWriter is too slow here:
-        int[] counts = new int[H3.MAX_H3_RES + 1];
-        IndexWriter w = new IndexWriter(dir, iwc);
-        for (String hex : hexes) {
-            CellBoundary cellBoundary = H3.h3ToGeoBoundary(hex);
-            for (int i = 0; i < cellBoundary.numPoints(); i++) {
-                Document doc = new Document();
-                LatLng latLng = cellBoundary.getLatLon(i);
-                doc.add(new LatLonPoint(FIELD_NAME, latLng.getLatDeg(), latLng.getLonDeg()));
-                w.addDocument(doc);
-                computeCounts(hexes, latLng.getLonDeg(), latLng.getLatDeg(), counts);
-            }
-
-        }
-        final int numDocs = randomIntBetween(1000, 2000);
-        for (int id = 0; id < numDocs; id++) {
-            Document doc = new Document();
-            Point point = GeometryTestUtils.randomPoint();
-            doc.add(new LatLonPoint(FIELD_NAME, point.getLat(), point.getLon()));
-            w.addDocument(doc);
-            computeCounts(hexes, point.getLon(), point.getLat(), counts);
-        }
-
-        if (random().nextBoolean()) {
-            w.forceMerge(1);
-        }
-        final IndexReader r = DirectoryReader.open(w);
-        w.close();
-
-        IndexSearcher s = newSearcher(r);
-        for (int i = 0; i < H3.MAX_H3_RES + 1; i++) {
-            H3LatLonGeometry geometry = new H3LatLonGeometry(hexes[i]);
-            Query indexQuery = LatLonPoint.newGeometryQuery(FIELD_NAME, ShapeField.QueryRelation.INTERSECTS, geometry);
-            assertEquals(counts[i], s.count(indexQuery));
-        }
-        IOUtils.close(r, dir);
-    }
-
-    private void computeCounts(String[] hexes, double lon, double lat, int[] counts) {
-        double qLat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat));
-        double qLon = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lon));
-        for (int res = 0; res < hexes.length; res++) {
-            if (hexes[res].equals(H3.geoToH3Address(qLat, qLon, res))) {
-                counts[res]++;
-            }
-        }
-    }
-}
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/GeoGridProcessorFactoryTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/GeoGridProcessorFactoryTests.java
new file mode 100644
index 000000000000..89d49fe9104f
--- /dev/null
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/GeoGridProcessorFactoryTests.java
@@ -0,0 +1,174 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.spatial.ingest;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.geo.GeometryParserFormat;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class GeoGridProcessorFactoryTests extends ESTestCase {
+
+    private GeoGridProcessor.Factory factory;
+
+    @Before
+    public void init() {
+        factory = new GeoGridProcessor.Factory();
+    }
+
+    public void testCreateGeohash() {
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "field1");
+        config.put("tile_type", GeoGridProcessor.TileFieldType.GEOHASH.name());
+        String processorTag = randomAlphaOfLength(10);
+        GeoGridProcessor processor = factory.create(null, processorTag, null, config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertFields(processor, "field1", "field1");
+        assertThat(processor.tileType(), equalTo(GeoGridProcessor.TileFieldType.GEOHASH));
+        assertThat(processor.targetFormat(), equalTo(GeometryParserFormat.GEOJSON));
+    }
+
+    public void testCreateGeoTile() {
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "field1");
+        config.put("tile_type", GeoGridProcessor.TileFieldType.GEOTILE.name());
+        String processorTag = randomAlphaOfLength(10);
+        GeoGridProcessor processor = factory.create(null, processorTag, null, config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertFields(processor, "field1", "field1");
+        assertThat(processor.tileType(), equalTo(GeoGridProcessor.TileFieldType.GEOTILE));
+        assertThat(processor.targetFormat(), equalTo(GeometryParserFormat.GEOJSON));
+    }
+
+    public void testCreateGeohex() {
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "field1");
+        config.put("tile_type", GeoGridProcessor.TileFieldType.GEOHEX.name());
+        String processorTag = randomAlphaOfLength(10);
+        GeoGridProcessor processor = factory.create(null, processorTag, null, config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertFields(processor, "field1", "field1");
+        assertThat(processor.tileType(), equalTo(GeoGridProcessor.TileFieldType.GEOHEX));
+        assertThat(processor.targetFormat(), equalTo(GeometryParserFormat.GEOJSON));
+    }
+
+    public void testCreateInvalidTargetFormat() {
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "field1");
+        config.put("tile_type", GeoGridProcessor.TileFieldType.GEOTILE.name());
+        config.put("target_format", "invalid");
+        String processorTag = randomAlphaOfLength(10);
+        ElasticsearchParseException e = expectThrows(
+            ElasticsearchParseException.class,
+            () -> factory.create(null, processorTag, null, config)
+        );
+        assertThat(e.getMessage(), equalTo("[target_format] illegal value [invalid], valid values are [WKT, GEOJSON]"));
+    }
+
+    public void testCreateMissingField() {
+        Map<String, Object> config = new HashMap<>();
+        String processorTag = randomAlphaOfLength(10);
+        ElasticsearchParseException e = expectThrows(
+            ElasticsearchParseException.class,
+            () -> factory.create(null, processorTag, null, config)
+        );
+        assertThat(e.getMessage(), equalTo("[field] required property is missing"));
+    }
+
+    public void testCreateWithTargetField() {
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "field1");
+        config.put("target_field", "other");
+        config.put("tile_type", GeoGridProcessor.TileFieldType.GEOTILE.name());
+        config.put("target_format", "wkt");
+        String processorTag = randomAlphaOfLength(10);
+        GeoGridProcessor processor = factory.create(null, processorTag, null, config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertFields(processor, "field1", "other");
+        assertThat(processor.tileType(), equalTo(GeoGridProcessor.TileFieldType.GEOTILE));
+        assertThat(processor.targetFormat(), equalTo(GeometryParserFormat.WKT));
+    }
+
+    public void testCreateWithChildrenField() {
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "field1");
+        config.put("tile_type", GeoGridProcessor.TileFieldType.GEOHEX.name());
+        config.put("children_field", "children");
+        String processorTag = randomAlphaOfLength(10);
+        GeoGridProcessor processor = factory.create(null, processorTag, null, config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertFields(processor, "field1", "field1", "", "children", "", "");
+        assertThat(processor.tileType(), equalTo(GeoGridProcessor.TileFieldType.GEOHEX));
+        assertThat(processor.targetFormat(), equalTo(GeometryParserFormat.GEOJSON));
+    }
+
+    public void testCreateWithExtraFields() {
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "field1");
+        config.put("tile_type", GeoGridProcessor.TileFieldType.GEOHEX.name());
+        config.put("parent_field", "parent");
+        config.put("children_field", "children");
+        config.put("non_children_field", "nonChildren");
+        config.put("precision_field", "precision");
+        config.put("target_format", "wkt");
+        String processorTag = randomAlphaOfLength(10);
+        GeoGridProcessor processor = factory.create(null, processorTag, null, config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertFields(processor, "field1", "field1", "parent", "children", "nonChildren", "precision");
+        assertThat(processor.tileType(), equalTo(GeoGridProcessor.TileFieldType.GEOHEX));
+        assertThat(processor.targetFormat(), equalTo(GeometryParserFormat.WKT));
+    }
+
+    public void testCreateWithNoTileTypeDefined() {
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "field1");
+        String processorTag = randomAlphaOfLength(10);
+        ElasticsearchParseException e = expectThrows(
+            ElasticsearchParseException.class,
+            () -> factory.create(null, processorTag, null, config)
+        );
+        assertThat(e.getMessage(), equalTo("[tile_type] required property is missing"));
+    }
+
+    public void testCreateWithInvalidTileTypeDefined() {
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "field1");
+        config.put("tile_type", "super_tiles");
+        String processorTag = randomAlphaOfLength(10);
+        ElasticsearchParseException e = expectThrows(
+            ElasticsearchParseException.class,
+            () -> factory.create(null, processorTag, null, config)
+        );
+        assertThat(e.getMessage(), equalTo("[tile_type] illegal value [super_tiles], valid values are [GEOHASH, GEOTILE, GEOHEX]"));
+    }
+
+    private void assertFields(GeoGridProcessor processor, String field, String targetField) {
+        assertFields(processor, field, targetField, "", "", "", "");
+    }
+
+    private void assertFields(
+        GeoGridProcessor processor,
+        String field,
+        String targetField,
+        String parentField,
+        String childrenField,
+        String nonChildrenField,
+        String precisionField
+    ) {
+        assertThat(processor.field("field"), equalTo(field));
+        assertThat(processor.field("target_field"), equalTo(targetField));
+        assertThat(processor.field("parent_field"), equalTo(parentField));
+        assertThat(processor.field("children_field"), equalTo(childrenField));
+        assertThat(processor.field("non_children_field"), equalTo(nonChildrenField));
+        assertThat(processor.field("precision_field"), equalTo(precisionField));
+    }
+}
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/GeoGridProcessorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/GeoGridProcessorTests.java
new file mode 100644
index 000000000000..27ebcd3aa32f
--- /dev/null
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/GeoGridProcessorTests.java
@@ -0,0 +1,314 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.spatial.ingest;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.geo.GeoJson;
+import org.elasticsearch.common.geo.GeometryParserFormat;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.geometry.Polygon;
+import org.elasticsearch.geometry.Rectangle;
+import org.elasticsearch.geometry.utils.WellKnownText;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.RandomDocumentPicks;
+import org.elasticsearch.ingest.TestIngestDocument;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
+import static org.elasticsearch.xpack.spatial.ingest.GeoGridProcessor.TileFieldType.GEOHASH;
+import static org.elasticsearch.xpack.spatial.ingest.GeoGridProcessor.TileFieldType.GEOHEX;
+import static org.elasticsearch.xpack.spatial.ingest.GeoGridProcessor.TileFieldType.GEOTILE;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class GeoGridProcessorTests extends ESTestCase {
+
+    public void testFieldNotFound() {
+        GeoGridProcessor processor = makeGridProcessor(false, GeometryParserFormat.WKT, GEOTILE);
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        Exception e = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(e.getMessage(), containsString("not present as part of path [field]"));
+    }
+
+    public void testFieldNotFoundWithIgnoreMissing() {
+        GeoGridProcessor processor = makeGridProcessor(true, GeometryParserFormat.WKT, GEOTILE);
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
+    public void testNullValue() {
+        GeoGridProcessor processor = makeGridProcessor(false, GeometryParserFormat.WKT, GEOTILE);
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
+        Exception e = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(e.getMessage(), equalTo("field [field] is null, cannot process it."));
+    }
+
+    public void testNullValueWithIgnoreMissing() {
+        GeoGridProcessor processor = makeGridProcessor(true, GeometryParserFormat.WKT, GEOTILE);
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
+    public void testGeohash2GeoJson() throws IOException {
+        HashMap<String, Object> map = new HashMap<>();
+        map.put("field", "u0");
+        Geometry expectedPoly = new GeoGridProcessor.GeohashHandler("u0").makeGeometry();
+        assertThat(expectedPoly, instanceOf(Rectangle.class));
+        IngestDocument ingestDocument = TestIngestDocument.withDefaultVersion(map);
+        GeoGridProcessor processor = makeGridProcessor(false, GeometryParserFormat.GEOJSON, GEOHASH);
+        processor.execute(ingestDocument);
+        assertFieldAsGeoJSON(ingestDocument, "field", expectedPoly);
+    }
+
+    public void testGeotile2GeoJson() throws IOException {
+        HashMap<String, Object> map = new HashMap<>();
+        map.put("field", "6/32/22");
+        Geometry expectedPoly = new GeoGridProcessor.GeotileHandler("6/32/22").makeGeometry();
+        assertThat(expectedPoly, instanceOf(Rectangle.class));
+        IngestDocument ingestDocument = TestIngestDocument.withDefaultVersion(map);
+        GeoGridProcessor processor = makeGridProcessor(false, GeometryParserFormat.GEOJSON, GEOTILE);
+        processor.execute(ingestDocument);
+        assertFieldAsGeoJSON(ingestDocument, "field", expectedPoly);
+    }
+
+    public void testGeohex2GeoJson() throws IOException {
+        HashMap<String, Object> map = new HashMap<>();
+        map.put("field", "811fbffffffffff");
+        Geometry expectedPoly = new GeoGridProcessor.GeohexHandler("811fbffffffffff").makeGeometry();
+        assertThat(expectedPoly, instanceOf(Polygon.class));
+        IngestDocument ingestDocument = TestIngestDocument.withDefaultVersion(map);
+        GeoGridProcessor processor = makeGridProcessor(false, GeometryParserFormat.GEOJSON, GEOHEX);
+        processor.execute(ingestDocument);
+        assertFieldAsGeoJSON(ingestDocument, "field", expectedPoly);
+    }
+
+    public void testGeohash2WKT() {
+        HashMap<String, Object> map = new HashMap<>();
+        map.put("field", "u0");
+        Geometry expectedPoly = new GeoGridProcessor.GeohashHandler("u0").makeGeometry();
+        IngestDocument ingestDocument = TestIngestDocument.withDefaultVersion(map);
+        GeoGridProcessor processor = makeGridProcessor(false, GeometryParserFormat.WKT, GEOHASH);
+        processor.execute(ingestDocument);
+        String polyString = ingestDocument.getFieldValue("field", String.class);
+        assertThat(polyString, equalTo(WellKnownText.toWKT(expectedPoly)));
+    }
+
+    public void testGeotile2WKT() {
+        HashMap<String, Object> map = new HashMap<>();
+        map.put("field", "6/32/22");
+        Geometry expectedPoly = new GeoGridProcessor.GeotileHandler("6/32/22").makeGeometry();
+        IngestDocument ingestDocument = TestIngestDocument.withDefaultVersion(map);
+        GeoGridProcessor processor = makeGridProcessor(false, GeometryParserFormat.WKT, GEOTILE);
+        processor.execute(ingestDocument);
+        String polyString = ingestDocument.getFieldValue("field", String.class);
+        assertThat(polyString, equalTo(WellKnownText.toWKT(expectedPoly)));
+    }
+
+    public void testGeohex2WKT() {
+        HashMap<String, Object> map = new HashMap<>();
+        map.put("field", "811fbffffffffff");
+        Geometry expectedPoly = new GeoGridProcessor.GeohexHandler("811fbffffffffff").makeGeometry();
+        IngestDocument ingestDocument = TestIngestDocument.withDefaultVersion(map);
+        GeoGridProcessor processor = makeGridProcessor(false, GeometryParserFormat.WKT, GEOHEX);
+        processor.execute(ingestDocument);
+        String polyString = ingestDocument.getFieldValue("field", String.class);
+        assertThat(polyString, equalTo(WellKnownText.toWKT(expectedPoly)));
+    }
+
+    public void testGeohex_WithChildrenAndPrecision() {
+        HashMap<String, Object> map = new HashMap<>();
+        map.put("geohex", "811fbffffffffff");
+        Geometry expectedGeometry = new GeoGridProcessor.GeohexHandler("811fbffffffffff").makeGeometry();
+        IngestDocument ingestDocument = TestIngestDocument.withDefaultVersion(map);
+        GeoGridProcessor processor = makeGridProcessor(
+            "geohex",
+            "polygon",
+            "parent",
+            "children",
+            "non_children",
+            "depth",
+            GeometryParserFormat.WKT,
+            GEOHEX
+        );
+        processor.execute(ingestDocument);
+        String polyString = ingestDocument.getFieldValue("polygon", String.class);
+        assertThat(polyString, equalTo(WellKnownText.toWKT(expectedGeometry)));
+        assertTrue("should have parent field", ingestDocument.hasField("parent"));
+        String parent = ingestDocument.getFieldValue("parent", String.class);
+        assertThat(parent, equalTo("801ffffffffffff"));
+        assertTrue("should have children field", ingestDocument.hasField("children"));
+        List<?> children = ingestDocument.getFieldValue("children", List.class);
+        assertThat(children.size(), equalTo(7));
+        assertThat(
+            children,
+            containsInAnyOrder(
+                equalTo("821f87fffffffff"),
+                equalTo("821f8ffffffffff"),
+                equalTo("821f97fffffffff"),
+                equalTo("821f9ffffffffff"),
+                equalTo("821fa7fffffffff"),
+                equalTo("821faffffffffff"),
+                equalTo("821fb7fffffffff")
+            )
+        );
+        assertTrue("should have non_children field", ingestDocument.hasField("non_children"));
+        List<?> nonChildren = ingestDocument.getFieldValue("non_children", List.class);
+        assertThat(nonChildren.size(), equalTo(6));
+        int depth = ingestDocument.getFieldValue("depth", Integer.class);
+        assertThat(depth, equalTo(1));
+    }
+
+    public void testGeotile_WithChildrenAndPrecision() {
+        HashMap<String, Object> map = new HashMap<>();
+        map.put("field", "4/8/5");
+        Geometry expectedGeometry = new GeoGridProcessor.GeotileHandler("4/8/5").makeGeometry();
+        IngestDocument ingestDocument = TestIngestDocument.withDefaultVersion(map);
+        GeoGridProcessor processor = makeGridProcessor(
+            "field",
+            "shape",
+            "encompassing",
+            "inner_tiles",
+            "intersecting",
+            "zoom",
+            GeometryParserFormat.WKT,
+            GEOTILE
+        );
+        processor.execute(ingestDocument);
+        String polyString = ingestDocument.getFieldValue("shape", String.class);
+        assertThat(polyString, equalTo(WellKnownText.toWKT(expectedGeometry)));
+        String encompassing = ingestDocument.getFieldValue("encompassing", String.class);
+        assertThat(encompassing, equalTo("3/4/2"));
+        assertTrue("should have children field", ingestDocument.hasField("inner_tiles"));
+        List<?> innerTiles = ingestDocument.getFieldValue("inner_tiles", List.class);
+        assertThat(innerTiles.size(), equalTo(4));
+        assertThat(innerTiles, containsInAnyOrder(equalTo("5/16/10"), equalTo("5/17/10"), equalTo("5/16/11"), equalTo("5/17/11")));
+        assertFalse("should have no intersecting field", ingestDocument.hasField("intersecting"));
+        int zoom = ingestDocument.getFieldValue("zoom", Integer.class);
+        assertThat(zoom, equalTo(4));
+    }
+
+    public void testGeohash_WithChildrenAndPrecision() throws IOException {
+        HashMap<String, Object> map = new HashMap<>();
+        map.put("geohash", "u0");
+        Geometry expectedGeometry = new GeoGridProcessor.GeohashHandler("u0").makeGeometry();
+        IngestDocument ingestDocument = TestIngestDocument.withDefaultVersion(map);
+        GeoGridProcessor processor = makeGridProcessor(
+            "geohash",
+            "shape",
+            "higher_level",
+            "next_level",
+            "intersecting",
+            "precision",
+            GeometryParserFormat.GEOJSON,
+            GEOHASH
+        );
+        processor.execute(ingestDocument);
+        assertFieldAsGeoJSON(ingestDocument, "shape", expectedGeometry);
+        String higher_level = ingestDocument.getFieldValue("higher_level", String.class);
+        assertThat(higher_level, equalTo("u"));
+        assertTrue("should have next_level field", ingestDocument.hasField("next_level"));
+        List<?> next_level = ingestDocument.getFieldValue("next_level", List.class);
+        assertThat(next_level.size(), equalTo(32));
+        assertFalse("should have no intersecting field", ingestDocument.hasField("intersecting"));
+        int precision = ingestDocument.getFieldValue("precision", Integer.class);
+        assertThat(precision, equalTo(2));
+    }
+
+    public void testInvalidGeohash() {
+        assertInvalidTile(GEOHASH, "unsupported symbol", "invalid geohash");
+    }
+
+    public void testInvalidGeotile() {
+        assertInvalidTile(GEOTILE, "Must be three integers in a form \"zoom/x/y\"", "invalid tile");
+    }
+
+    public void testInvalidGeohex() {
+        assertInvalidTile(GEOHEX, "NumberFormatException", "not a hex");
+    }
+
+    public void testMissingField() {
+        IngestDocument ingestDocument = TestIngestDocument.emptyIngestDocument();
+        GeoGridProcessor processor = makeGridProcessor(false, GeometryParserFormat.WKT, GEOTILE);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
+        assertThat(e.getMessage(), equalTo("field [field] not present as part of path [field]"));
+    }
+
+    @SuppressWarnings("unchecked")
+    private void assertFieldAsGeoJSON(IngestDocument ingestDocument, String field, Geometry expectedGeometry) throws IOException {
+        Map<String, Object> polyMap = ingestDocument.getFieldValue(field, Map.class);
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        GeoJson.toXContent(expectedGeometry, builder, ToXContent.EMPTY_PARAMS);
+        Tuple<XContentType, Map<String, Object>> expected = XContentHelper.convertToMap(
+            BytesReference.bytes(builder),
+            true,
+            XContentType.JSON
+        );
+        assertThat(polyMap, equalTo(expected.v2()));
+    }
+
+    private void assertInvalidTile(GeoGridProcessor.TileFieldType type, String innerError, String content) {
+        HashMap<String, Object> map = new HashMap<>();
+        map.put("field", content);
+        IngestDocument ingestDocument = TestIngestDocument.withDefaultVersion(map);
+        GeoGridProcessor processor = makeGridProcessor(false, GeometryParserFormat.WKT, type);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
+        assertThat(type + " field '" + map.get("field") + "'", e.getMessage(), equalTo("invalid tile definition"));
+        assertThat(type + " field '" + map.get("field") + "'", e.getCause().toString(), containsString(innerError));
+        map.put("field", "POINT (30 10)");
+        e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
+        assertThat(type + " field '" + map.get("field") + "'", e.getMessage(), equalTo("invalid tile definition"));
+        assertThat(type + " field '" + map.get("field") + "'", e.getCause().toString(), containsString(innerError));
+    }
+
+    private GeoGridProcessor makeGridProcessor(
+        boolean ignoreMissing,
+        GeometryParserFormat format,
+        GeoGridProcessor.TileFieldType tileFieldType
+    ) {
+        GeoGridProcessor.FieldConfig config = new GeoGridProcessor.FieldConfig("field", "field", null, null, null, null);
+        return new GeoGridProcessor("tag", null, config, ignoreMissing, format, tileFieldType);
+    }
+
+    private GeoGridProcessor makeGridProcessor(
+        String field,
+        String targetField,
+        String parentField,
+        String childrenField,
+        String nonChildrenField,
+        String precisionField,
+        GeometryParserFormat format,
+        GeoGridProcessor.TileFieldType tileFieldType
+    ) {
+        GeoGridProcessor.FieldConfig config = new GeoGridProcessor.FieldConfig(
+            field,
+            targetField,
+            parentField,
+            childrenField,
+            nonChildrenField,
+            precisionField
+        );
+        return new GeoGridProcessor("tag", null, config, false, format, tileFieldType);
+    }
+}
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregationBuilderTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregationBuilderTests.java
index ff8723b725ba..a25e38bf0bca 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregationBuilderTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregationBuilderTests.java
@@ -56,6 +56,11 @@ protected GeoLineAggregationBuilder createTestInstance() {
         return lineAggregationBuilder;
     }
 
+    @Override
+    protected GeoLineAggregationBuilder mutateInstance(GeoLineAggregationBuilder instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testInvalidSize() {
         MultiValuesSourceFieldConfig pointConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName(randomAlphaOfLength(5)).build();
         MultiValuesSourceFieldConfig sortConfig = new MultiValuesSourceFieldConfig.Builder().setFieldName(randomAlphaOfLength(6)).build();
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/BoundedGeoHexGridTilerTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/BoundedGeoHexGridTilerTests.java
new file mode 100644
index 000000000000..08850c982c20
--- /dev/null
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/BoundedGeoHexGridTilerTests.java
@@ -0,0 +1,330 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid;
+
+import org.elasticsearch.common.geo.GeoBoundingBox;
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.geo.GeoUtils;
+import org.elasticsearch.geo.GeometryTestUtils;
+import org.elasticsearch.geometry.Point;
+import org.elasticsearch.geometry.Rectangle;
+import org.elasticsearch.h3.H3;
+import org.elasticsearch.h3.LatLng;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.spatial.common.H3CartesianUtil;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
+
+import static org.elasticsearch.common.geo.GeoUtils.normalizeLon;
+import static org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoHexGridTiler.BoundedGeoHexGridTiler.height;
+import static org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoHexGridTiler.BoundedGeoHexGridTiler.width;
+import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.not;
+
+public class BoundedGeoHexGridTilerTests extends ESTestCase {
+
+    public void testBoundedTilerInflation() {
+        for (int i = 0; i < 10000; i++) {
+            Point point = GeometryTestUtils.randomPoint();
+            int precision = random().nextInt(0, 1);
+            assertH3CellInflation(point, precision);
+        }
+    }
+
+    /** Test with cell bounds crossing dateline */
+    public void testBoundedTilerInflation_805bfffffffffff() {
+        for (int precision = 0; precision < 2; precision++) {
+            long h3 = H3.stringToH3("805bfffffffffff");
+            LatLng centroid = H3.h3ToLatLng(h3);
+            assertH3CellInflation(new Point(centroid.getLonDeg(), centroid.getLatDeg()), precision);
+        }
+    }
+
+    /** Test for north pole with bounds width 360 */
+    public void testBoundedTilerInflation_8001fffffffffff() {
+        for (int precision = 0; precision < 2; precision++) {
+            long h3 = H3.stringToH3("8001fffffffffff");
+            LatLng centroid = H3.h3ToLatLng(h3);
+            assertH3CellInflation(new Point(centroid.getLonDeg(), centroid.getLatDeg()), precision);
+        }
+    }
+
+    /** Test with inner bbox crossing dateline */
+    public void testBoundedTilerInflation_80bbfffffffffff() {
+        for (int precision = 0; precision < 2; precision++) {
+            long h3 = H3.stringToH3("80bbfffffffffff");
+            LatLng centroid = H3.h3ToLatLng(h3);
+            assertH3CellInflation(new Point(centroid.getLonDeg(), centroid.getLatDeg()), precision);
+        }
+    }
+
+    /** Test with inner bbox on different side of dateline than inflated bbox */
+    public void testBoundedTilerInflation_80ebfffffffffff() {
+        for (int precision = 0; precision < 2; precision++) {
+            long h3 = H3.stringToH3("80ebfffffffffff");
+            LatLng centroid = H3.h3ToLatLng(h3);
+            assertH3CellInflation(new Point(centroid.getLonDeg(), centroid.getLatDeg()), precision);
+        }
+    }
+
+    /** Test with inner bbox and cell crossing line of zero longitude */
+    public void testBoundedTilerInflation_8075fffffffffff() {
+        for (int precision = 0; precision < 2; precision++) {
+            long h3 = H3.stringToH3("8075fffffffffff");
+            LatLng centroid = H3.h3ToLatLng(h3);
+            assertH3CellInflation(new Point(centroid.getLonDeg(), centroid.getLatDeg()), precision);
+        }
+    }
+
+    private void assertH3CellInflation(Point point, int precision) {
+        long h3 = H3.geoToH3(point.getLat(), point.getLon(), precision);
+        LatLng centroid = H3.h3ToLatLng(h3);
+        String cell = "H3[" + H3.h3ToString(h3) + "]:" + precision;
+        Rectangle cellBounds = H3CartesianUtil.toBoundingBox(h3);
+        boolean cellCrossesDateline = cellBounds.getMinX() > cellBounds.getMaxX();
+        double maxX = cellCrossesDateline ? 360 + cellBounds.getMaxX() : cellBounds.getMaxX();
+        double centroidLeft = centroid.getLonDeg();
+        double centroidRight = centroid.getLonDeg();
+        if (cellCrossesDateline) {
+            centroidLeft = normLonAgainst(centroid.getLonDeg(), cellBounds.getMinX());
+            centroidRight = normLonAgainst(centroid.getLonDeg(), maxX);
+        }
+        double left = normalizeLon(wAvg(centroidLeft, cellBounds.getMinX(), 0.1));
+        double right = normalizeLon(wAvg(centroidRight, maxX, 0.1));
+        double top = wAvg(centroid.getLatDeg(), cellBounds.getMaxY(), 0.1);
+        double bottom = wAvg(centroid.getLatDeg(), cellBounds.getMinY(), 0.1);
+        GeoBoundingBox bbox = new GeoBoundingBox(new GeoPoint(top, left), new GeoPoint(bottom, right));
+
+        // Inflating with factor 0.0 should match the original bounding box
+        GeoBoundingBox inflated0 = inflateBbox(precision, bbox, 0.0);
+        assertThat(cell + " Full-inflated", inflated0, matchesBounds(bbox));
+
+        // Test inflation factors 1.0 should match the outer cell bounds
+        GeoBoundingBox inflated1 = inflateBbox(precision, bbox, 1.0);
+        Rectangle fullyInflated = getFullBounds(cellBounds, bbox);
+        assertThat(cell + " Full-inflated", inflated1, matchesBounds(fullyInflated));
+
+        // Now test a range of inflation factors
+        for (double factor = 0.1; factor <= 0.95; factor += 0.1) {
+            GeoBoundingBox inflated = inflateBbox(precision, bbox, factor);
+            // Compare inflated to outer bounds (should be smaller)
+            assertThat(cell + " Inflated(" + factor + ")", inflated, withinBounds(fullyInflated));
+            // Compare inflated to inner bounds (should be larger)
+            assertThat(cell + " Inflated(" + factor + ")", inflated, containsBounds(bbox));
+        }
+    }
+
+    /** Calculate weighted average: 1 means second dominates, 0 means first dominates */
+    private static double wAvg(double first, double second, double weight) {
+        double width = (second - first) * weight;
+        return first + width;
+    }
+
+    // We usually got longitude values the same side of the dateline by adding 360, never subtracting 360
+    private static double normLonAgainst(double value, double other) {
+        if (other > 0 && value < 0) {
+            value += 360;
+        }
+        return value;
+    }
+
+    private GeoBoundingBox inflateBbox(int precision, GeoBoundingBox bbox, double factor) {
+        return GeoHexGridTiler.BoundedGeoHexGridTiler.inflateBbox(precision, bbox, factor);
+    }
+
+    /* Calculate the bounds of the h3 cell assuming the test bbox is entirely within the cell */
+    private Rectangle getFullBounds(Rectangle bounds, GeoBoundingBox bbox) {
+        final double height = height(bounds);
+        final double width = width(bounds);
+        // inflate the coordinates by the full width and height
+        final double minY = Math.max(bbox.bottom() - height, -90d);
+        final double maxY = Math.min(bbox.top() + height, 90d);
+        final double left = GeoUtils.normalizeLon(bbox.left() - width);
+        final double right = GeoUtils.normalizeLon(bbox.right() + width);
+        if (2 * width + width(bbox) >= 360d) {
+            // if the total width is bigger than the world, then it covers all longitude range.
+            return new Rectangle(-180, 180, maxY, minY);
+        } else {
+            return new Rectangle(left, right, maxY, minY);
+        }
+    }
+
+    public void testBoundsMatcher() {
+        GeoBoundingBox bounds = new GeoBoundingBox(new GeoPoint(10, -10), new GeoPoint(-10, 10));
+        GeoBoundingBox bigger = new GeoBoundingBox(new GeoPoint(20, -20), new GeoPoint(-20, 20));
+        GeoBoundingBox smaller = new GeoBoundingBox(new GeoPoint(5, -5), new GeoPoint(-5, 5));
+        GeoBoundingBox intersecting = new GeoBoundingBox(new GeoPoint(20, -20), new GeoPoint(0, 20));
+        GeoBoundingBox nonIntersecting = new GeoBoundingBox(new GeoPoint(20, -10), new GeoPoint(15, 10));
+        {
+            assertThat("equals", bounds, matchesBounds(bounds));
+            assertThat("contains", bounds, containsBounds(smaller));
+            assertThat("within", bounds, withinBounds(bigger));
+        }
+        {
+            assertThat("not equals", bounds, not(matchesBounds(smaller)));
+            assertThat("not equals", bounds, not(matchesBounds(bigger)));
+            assertThat("not equals", bounds, not(matchesBounds(intersecting)));
+            assertThat("not equals", bounds, not(matchesBounds(nonIntersecting)));
+        }
+        {
+            assertThat("does not contain", bounds, not(containsBounds(bigger)));
+            assertThat("does not contain", bounds, not(containsBounds(intersecting)));
+            assertThat("does not contain", bounds, not(containsBounds(nonIntersecting)));
+        }
+        {
+            assertThat("is not within", bounds, not(withinBounds(smaller)));
+            assertThat("is not within", bounds, not(withinBounds(intersecting)));
+            assertThat("is not within", bounds, not(withinBounds(nonIntersecting)));
+        }
+    }
+
+    private static TestCompareBounds matchesBounds(GeoBoundingBox other) {
+        return new TestCompareBounds(other, 0);
+    }
+
+    private static TestCompareBounds matchesBounds(Rectangle other) {
+        return new TestCompareBounds(other, 0);
+    }
+
+    private static TestCompareBounds containsBounds(GeoBoundingBox other) {
+        return new TestCompareBounds(other, 1);
+    }
+
+    private static TestCompareBounds withinBounds(Rectangle other) {
+        return new TestCompareBounds(other, -1);
+    }
+
+    private static TestCompareBounds withinBounds(GeoBoundingBox other) {
+        return new TestCompareBounds(other, -1);
+    }
+
+    private static class TestCompareBounds extends BaseMatcher<GeoBoundingBox> {
+
+        private final GeoBoundingBox other;
+        private final int comparison;
+        private boolean matchedTop;
+        private boolean matchedBottom;
+        private boolean matchedLeft;
+        private boolean matchedRight;
+
+        private TestCompareBounds(Rectangle rect, int comparison) {
+            this.other = new GeoBoundingBox(new GeoPoint(rect.getMaxY(), rect.getMinX()), new GeoPoint(rect.getMinY(), rect.getMaxX()));
+            this.comparison = comparison;
+        }
+
+        private TestCompareBounds(GeoBoundingBox other, int comparison) {
+            this.other = other;
+            this.comparison = comparison;
+        }
+
+        @Override
+        public boolean matches(Object actual) {
+            if (actual instanceof GeoBoundingBox bbox) {
+                if (comparison == 0) {
+                    matchedTop = closeTo(bbox.top(), 1e-10).matches(other.top());
+                    matchedBottom = closeTo(bbox.bottom(), 1e-10).matches(other.bottom());
+                    matchedLeft = closeTo(posLon(bbox.left()), 1e-10).matches(posLon(other.left()));
+                    matchedRight = closeTo(posLon(bbox.right()), 1e-10).matches(posLon(other.right()));
+                } else {
+                    if (comparison > 0) {
+                        // assert that 'bbox' is larger than and entirely contains 'other'
+                        setBoxWithinBox(other, bbox);
+                    } else {
+                        // assert that 'bbox' is smaller than and entirely contained within 'other'
+                        setBoxWithinBox(bbox, other);
+                    }
+                }
+                return matchedTop && matchedBottom && matchedLeft && matchedRight;
+            }
+            return false;
+        }
+
+        private void setBoxWithinBox(GeoBoundingBox smaller, GeoBoundingBox larger) {
+            double smallerRight = smaller.right();
+            double largerRight = larger.right();
+            double smallerLeft = smaller.left();
+            double largerLeft = larger.left();
+            boolean smallerCrossesDateline = smaller.left() > smaller.right();
+            boolean largerCrossesDateline = larger.left() > larger.right();
+            if (smallerCrossesDateline || largerCrossesDateline) {
+                smallerRight = smallerCrossesDateline ? smallerRight + 360 : smallerRight;
+                largerRight = largerCrossesDateline ? largerRight + 360 : largerRight;
+                if (smallerCrossesDateline == false && smallerLeft < 0) {
+                    // Larger crosses dateline, but smaller does not, make sure they have comparable signs
+                    smallerLeft += 360;
+                    smallerRight += 360;
+                }
+                if (largerCrossesDateline == false && largerLeft < 0) {
+                    // Smaller crosses dateline, but larger does not, make sure they have comparable signs
+                    largerLeft += 360;
+                    largerRight += 360;
+                }
+            }
+            assert smallerLeft < smallerRight;
+            assert largerLeft < largerRight;
+            // assert that 'smaller' is smaller than and entirely contained within 'larger'
+            matchedTop = larger.top() >= 90 || greaterThan(smaller.top()).matches(larger.top());
+            matchedBottom = larger.bottom() <= -90 || lessThan(smaller.bottom()).matches(larger.bottom());
+            // Only check smaller longitudes are within larger longitudes if larger does not cover all longitudes (eg. polar cells)
+            boolean largerGlobalLongitude = largerRight - largerLeft >= 360;
+            matchedLeft = largerGlobalLongitude || lessThan(smallerLeft).matches(largerLeft);
+            matchedRight = largerGlobalLongitude || greaterThan(smallerRight).matches(largerRight);
+        }
+
+        private static double posLon(double value) {
+            if (value < 0) return value + 360;
+            return value;
+        }
+
+        @Override
+        public void describeTo(Description description) {
+            if (comparison == 0) {
+                description.appendText("To match " + other);
+            } else if (comparison > 0) {
+                description.appendText("To contain " + other);
+            } else {
+                description.appendText("To be contained by " + other);
+            }
+        }
+
+        @Override
+        public void describeMismatch(Object item, Description description) {
+            super.describeMismatch(item, description);
+            if (item instanceof GeoBoundingBox bbox) {
+                if (matchedTop == false) {
+                    describeMismatchOf(description, "top", other.top(), bbox.top(), true);
+                }
+                if (matchedBottom == false) {
+                    describeMismatchOf(description, "bottom", other.bottom(), bbox.bottom(), false);
+                }
+                if (matchedLeft == false) {
+                    describeMismatchOf(description, "left", other.left(), bbox.left(), false);
+                }
+                if (matchedRight == false) {
+                    describeMismatchOf(description, "right", other.right(), bbox.right(), true);
+                }
+            }
+        }
+
+        private void describeMismatchOf(Description description, String field, double thisValue, double thatValue, boolean max) {
+            description.appendText("\n\t and " + field + " ");
+            description.appendValue(thisValue).appendText(" was not").appendText(describeComparison(max)).appendValue(thatValue);
+        }
+
+        private String describeComparison(boolean max) {
+            return switch (comparison) {
+                case 0 -> " equal to ";
+                case -1 -> max ? " greater than " : " less than ";
+                case 1 -> max ? " less than " : " greater than ";
+                default -> " UNKNOWN ";
+            };
+        }
+    }
+}
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoGridTilerTestCase.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoGridTilerTestCase.java
index 13174a3582b3..253eb8545044 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoGridTilerTestCase.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoGridTilerTestCase.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.geo.GeometryTestUtils;
 import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.geometry.Line;
 import org.elasticsearch.geometry.LinearRing;
 import org.elasticsearch.geometry.MultiLine;
 import org.elasticsearch.geometry.MultiPolygon;
@@ -45,18 +46,28 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
-public abstract class GeoGridTilerTestCase extends ESTestCase {
+public abstract class GeoGridTilerTestCase<T extends GeoGridTiler> extends ESTestCase {
 
     protected static final LongConsumer NOOP_BREAKER = (l) -> {};
 
-    protected abstract GeoGridTiler getUnboundedGridTiler(int precision);
+    protected final T getGridTiler(int precision) {
+        final GeoBoundingBox bbox = randomBoolean()
+            ? null
+            : new GeoBoundingBox(new GeoPoint(Double.NaN, Double.NaN), new GeoPoint(Double.NaN, Double.NaN));
+        return getGridTiler(bbox, precision);
+    }
 
-    protected abstract GeoGridTiler getBoundedGridTiler(GeoBoundingBox bbox, int precision);
+    protected abstract T getGridTiler(GeoBoundingBox bbox, int precision);
 
     protected abstract int maxPrecision();
 
     protected abstract Rectangle getCell(double lon, double lat, int precision);
 
+    /** Tilers that are not rectangular cannot run all tests, eg. H3 tiler */
+    protected boolean isRectangularTiler() {
+        return true;
+    }
+
     protected abstract long getCellsForDiffPrecision(int precisionDiff);
 
     protected abstract void assertSetValuesBruteAndRecursive(Geometry geometry) throws Exception;
@@ -73,18 +84,24 @@ public void testMaxCellsBounded() {
                 new GeoPoint(tile.getMinLat(), tile.getMaxLon())
             );
             int otherPrecision = randomIntBetween(i, maxPrecision());
-            GeoGridTiler tiler = getBoundedGridTiler(boundingBox, otherPrecision);
+            T tiler = getGridTiler(boundingBox, otherPrecision);
             assertThat(tiler.getMaxCells(), greaterThanOrEqualTo(getCellsForDiffPrecision(otherPrecision - i)));
         }
     }
 
     public void testMaxCellsUnBounded() {
         for (int i = 0; i < maxPrecision(); i++) {
-            GeoGridTiler tiler = getUnboundedGridTiler(i);
+            T tiler = getGridTiler(i);
             assertThat(tiler.getMaxCells(), greaterThanOrEqualTo(getCellsForDiffPrecision(i)));
         }
     }
 
+    public void testGeoGridSetValuesBruteAndRecursiveLine() throws Exception {
+        Line geometry = GeometryTestUtils.randomLine(false);
+        assertSetValuesBruteAndRecursive(geometry);
+
+    }
+
     public void testGeoGridSetValuesBruteAndRecursiveMultiline() throws Exception {
         MultiLine geometry = GeometryTestUtils.randomMultiLine(false);
         assertSetValuesBruteAndRecursive(geometry);
@@ -95,8 +112,13 @@ public void testGeoGridSetValuesBruteAndRecursivePolygon() throws Exception {
         assertSetValuesBruteAndRecursive(geometry);
     }
 
-    public void testGeoGridSetValuesBruteAndRecursivePoints() throws Exception {
-        Geometry geometry = randomBoolean() ? GeometryTestUtils.randomPoint(false) : GeometryTestUtils.randomMultiPoint(false);
+    public void testGeoGridSetValuesBruteAndRecursivePoint() throws Exception {
+        Geometry geometry = GeometryTestUtils.randomPoint(false);
+        assertSetValuesBruteAndRecursive(geometry);
+    }
+
+    public void testGeoGridSetValuesBruteAndRecursiveMultiPoint() throws Exception {
+        Geometry geometry = GeometryTestUtils.randomMultiPoint(false);
         assertSetValuesBruteAndRecursive(geometry);
     }
 
@@ -104,32 +126,27 @@ public void testGeoGridSetValuesBruteAndRecursivePoints() throws Exception {
     public void testGeoGridSetValuesBoundingBoxes_BoundedGeoShapeCellValues() throws Exception {
         for (int i = 0; i < 10; i++) {
             int precision = randomIntBetween(0, 3);
-            Geometry geometry = GeometryNormalizer.apply(Orientation.CCW, randomValueOtherThanMany(g -> {
-                try {
-                    // make sure is a valid shape
-                    new GeoShapeIndexer(Orientation.CCW, "test").indexShape(g);
-                    return false;
-                } catch (Exception e) {
-                    return true;
-                }
-            }, () -> boxToGeo(randomBBox())));
+            Geometry geometry = GeometryNormalizer.apply(
+                Orientation.CCW,
+                randomValueOtherThanMany(this::geometryIsInvalid, () -> boxToGeo(randomBBox()))
+            );
 
             GeoBoundingBox geoBoundingBox = randomValueOtherThanMany(b -> b.right() == -180 && b.left() == 180, () -> randomBBox());
             GeoShapeValues.GeoShapeValue value = geoShapeValue(geometry);
             GeoShapeCellValues cellValues = new GeoShapeCellValues(
                 makeGeoShapeValues(value),
-                getBoundedGridTiler(geoBoundingBox, precision),
+                getGridTiler(geoBoundingBox, precision),
                 NOOP_BREAKER
             );
 
             assertTrue(cellValues.advanceExact(0));
             int numBuckets = cellValues.docValueCount();
             int expected = expectedBuckets(value, precision, geoBoundingBox);
-            assertThat(numBuckets, equalTo(expected));
+            assertThat("[" + i + ":" + precision + "] bucket count", numBuckets, equalTo(expected));
         }
     }
 
-    // tests that bounding boxes that crosses the dateline and cover all longitude values are correctly wrapped
+    // tests that bounding boxes that cross the dateline and cover all longitude values are correctly wrapped
     public void testGeoGridSetValuesBoundingBoxes_coversAllLongitudeValues() throws Exception {
         int precision = 3;
         Geometry geometry = new Rectangle(-92, 180, 0.99, -89);
@@ -137,7 +154,7 @@ public void testGeoGridSetValuesBoundingBoxes_coversAllLongitudeValues() throws
         GeoShapeValues.GeoShapeValue value = geoShapeValue(geometry);
         GeoShapeCellValues cellValues = new GeoShapeCellValues(
             makeGeoShapeValues(value),
-            getBoundedGridTiler(geoBoundingBox, precision),
+            getGridTiler(geoBoundingBox, precision),
             NOOP_BREAKER
         );
 
@@ -148,27 +165,20 @@ public void testGeoGridSetValuesBoundingBoxes_coversAllLongitudeValues() throws
     }
 
     public void testGeoGridSetValuesBoundingBoxes_UnboundedGeoShapeCellValues() throws Exception {
-        GeoShapeIndexer indexer = new GeoShapeIndexer(Orientation.CCW, "test");
-        for (int i = 0; i < 1000; i++) {
+        for (int i = 0; i < 100; i++) {
             int precision = randomIntBetween(0, 3);
-            Geometry geometry = randomValueOtherThanMany(g -> {
-                try {
-                    indexer.indexShape(g);
-                    return false;
-                } catch (Exception e) {
-                    return true;
-                }
-            }, () -> boxToGeo(randomBBox()));
+            Geometry geometry = randomValueOtherThanMany(this::geometryIsInvalid, () -> boxToGeo(randomBBox()));
             GeoShapeValues.GeoShapeValue value = geoShapeValue(geometry);
             GeoShapeCellValues unboundedCellValues = new GeoShapeCellValues(
                 makeGeoShapeValues(value),
-                getUnboundedGridTiler(precision),
+                getGridTiler(precision),
                 NOOP_BREAKER
             );
+
             assertTrue(unboundedCellValues.advanceExact(0));
-            int numTiles = unboundedCellValues.docValueCount();
+            int numBuckets = unboundedCellValues.docValueCount();
             int expected = expectedBuckets(value, precision, null);
-            assertThat(numTiles, equalTo(expected));
+            assertThat("[" + i + ":" + precision + "] bucket count", numBuckets, equalTo(expected));
         }
     }
 
@@ -182,7 +192,7 @@ public void testGeoTileShapeContainsBoundDateLine() throws Exception {
             new GeoPoint(tile.getMinLat(), tile.getMaxLon())
         );
 
-        GeoShapeCellValues values = new GeoShapeCellValues(makeGeoShapeValues(value), getBoundedGridTiler(boundingBox, 4), NOOP_BREAKER);
+        GeoShapeCellValues values = new GeoShapeCellValues(makeGeoShapeValues(value), getGridTiler(boundingBox, 4), NOOP_BREAKER);
         assertTrue(values.advanceExact(0));
         int numTiles = values.docValueCount();
         int expectedTiles = expectedBuckets(value, 4, boundingBox);
@@ -191,6 +201,7 @@ public void testGeoTileShapeContainsBoundDateLine() throws Exception {
 
     public void testBoundsExcludeTouchingTiles() throws Exception {
         final int precision = randomIntBetween(4, maxPrecision()) - 4;
+        assumeTrue("Test only works for rectangular tilers", isRectangularTiler());
 
         final Rectangle rectangle = getCell(GeoTestUtil.nextLongitude(), GeoTestUtil.nextLatitude(), precision);
         final GeoBoundingBox box = new GeoBoundingBox(
@@ -205,17 +216,17 @@ public void testBoundsExcludeTouchingTiles() throws Exception {
         );
         final GeoShapeValues.GeoShapeValue value = geoShapeValue(other);
         for (int i = 0; i < 4; i++) {
-            final GeoGridTiler bounded = getBoundedGridTiler(box, precision + i);
+            final GeoGridTiler bounded = getGridTiler(box, precision + i);
             final GeoShapeCellValues values = new GeoShapeCellValues(makeGeoShapeValues(value), bounded, NOOP_BREAKER);
             assertTrue(values.advanceExact(0));
             final int numTiles = values.docValueCount();
             final int expected = (int) getCellsForDiffPrecision(i);
-            assertThat(numTiles, equalTo(expected));
+            assertThat("For precision " + (precision + i), numTiles, equalTo(expected));
         }
     }
 
     public void testGridCircuitBreaker() throws IOException {
-        GeoGridTiler tiler = getUnboundedGridTiler(randomIntBetween(0, 3));
+        T tiler = getGridTiler(randomIntBetween(0, 3));
         Geometry geometry = GeometryTestUtils.randomPolygon(false);
 
         GeoShapeValues.GeoShapeValue value = geoShapeValue(geometry);
@@ -229,7 +240,7 @@ public void testGridCircuitBreaker() throws IOException {
         final long maxNumBytes;
         final long curNumBytes;
         if (byteChangeHistory.size() == 1) {
-            curNumBytes = maxNumBytes = byteChangeHistory.get(byteChangeHistory.size() - 1);
+            curNumBytes = maxNumBytes = byteChangeHistory.get(0);
         } else {
             long oldNumBytes = -byteChangeHistory.get(byteChangeHistory.size() - 1);
             curNumBytes = byteChangeHistory.get(byteChangeHistory.size() - 2);
@@ -247,11 +258,21 @@ public void testGridCircuitBreaker() throws IOException {
         expectThrows(CircuitBreakingException.class, () -> {
             GeoShapeCellValues values = new GeoShapeCellValues(makeGeoShapeValues(value), tiler, circuitBreakerConsumer);
             assertTrue(values.advanceExact(0));
-            assertThat(values.getValuesBytes(), equalTo(curNumBytes));
+            assertThat((long) values.getValues().length * Long.BYTES, equalTo(curNumBytes));
             assertThat(limitedBreaker.getUsed(), equalTo(curNumBytes));
         });
     }
 
+    protected boolean geometryIsInvalid(Geometry g) {
+        try {
+            // make sure is a valid shape
+            new GeoShapeIndexer(Orientation.CCW, "test").indexShape(g);
+            return false;
+        } catch (Exception e) {
+            return true;
+        }
+    }
+
     protected GeoShapeValues makeGeoShapeValues(GeoShapeValues.GeoShapeValue... values) {
         return new GeoShapeValues() {
             int index = 0;
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHashTilerTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHashTilerTests.java
index df154b703eaf..cd9f8d10fa31 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHashTilerTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHashTilerTests.java
@@ -23,16 +23,11 @@
 import static org.elasticsearch.xpack.spatial.util.GeoTestUtils.geoShapeValue;
 import static org.hamcrest.Matchers.equalTo;
 
-public class GeoHashTilerTests extends GeoGridTilerTestCase {
+public class GeoHashTilerTests extends GeoGridTilerTestCase<GeoHashGridTiler> {
 
     @Override
-    protected GeoGridTiler getUnboundedGridTiler(int precision) {
-        return new UnboundedGeoHashGridTiler(precision);
-    }
-
-    @Override
-    protected GeoGridTiler getBoundedGridTiler(GeoBoundingBox bbox, int precision) {
-        return new BoundedGeoHashGridTiler(precision, bbox);
+    protected GeoHashGridTiler getGridTiler(GeoBoundingBox bbox, int precision) {
+        return GeoHashGridTiler.makeGridTiler(precision, bbox);
     }
 
     @Override
@@ -57,7 +52,7 @@ protected long getCellsForDiffPrecision(int precisionDiff) {
     @Override
     protected void assertSetValuesBruteAndRecursive(Geometry geometry) throws Exception {
         int precision = randomIntBetween(1, 3);
-        UnboundedGeoHashGridTiler tiler = new UnboundedGeoHashGridTiler(precision);
+        GeoHashGridTiler tiler = getGridTiler(precision);
         GeoShapeValues.GeoShapeValue value = geoShapeValue(geometry);
         GeoShapeCellValues recursiveValues = new GeoShapeCellValues(null, tiler, NOOP_BREAKER);
         int recursiveCount;
@@ -137,7 +132,7 @@ public void testGeoHash() throws Exception {
         double x = randomDouble();
         double y = randomDouble();
         int precision = randomIntBetween(0, 6);
-        assertThat(new UnboundedGeoHashGridTiler(precision).encode(x, y), equalTo(Geohash.longEncode(x, y, precision)));
+        assertThat(getGridTiler(precision).encode(x, y), equalTo(Geohash.longEncode(x, y, precision)));
 
         Rectangle tile = Geohash.toBoundingBox(Geohash.stringEncode(x, y, 5));
 
@@ -151,23 +146,22 @@ public void testGeoHash() throws Exception {
 
         // test shape within tile bounds
         {
-            GeoShapeCellValues values = new GeoShapeCellValues(makeGeoShapeValues(value), new UnboundedGeoHashGridTiler(5), NOOP_BREAKER);
+            GeoShapeCellValues values = new GeoShapeCellValues(makeGeoShapeValues(value), getGridTiler(5), NOOP_BREAKER);
             assertTrue(values.advanceExact(0));
             int count = values.docValueCount();
             assertThat(count, equalTo(1));
         }
         {
-            GeoShapeCellValues values = new GeoShapeCellValues(makeGeoShapeValues(value), new UnboundedGeoHashGridTiler(6), NOOP_BREAKER);
+            GeoShapeCellValues values = new GeoShapeCellValues(makeGeoShapeValues(value), getGridTiler(6), NOOP_BREAKER);
             assertTrue(values.advanceExact(0));
             int count = values.docValueCount();
             assertThat(count, equalTo(32));
         }
         {
-            GeoShapeCellValues values = new GeoShapeCellValues(makeGeoShapeValues(value), new UnboundedGeoHashGridTiler(7), NOOP_BREAKER);
+            GeoShapeCellValues values = new GeoShapeCellValues(makeGeoShapeValues(value), getGridTiler(7), NOOP_BREAKER);
             assertTrue(values.advanceExact(0));
             int count = values.docValueCount();
             assertThat(count, equalTo(1024));
         }
     }
-
 }
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexAggregationBuilderTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexAggregationBuilderTests.java
index 4233e210d9d2..25cda8c15775 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexAggregationBuilderTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexAggregationBuilderTests.java
@@ -57,6 +57,11 @@ protected GeoHexGridAggregationBuilder createTestInstance() {
         return geoHexGridAggregationBuilder;
     }
 
+    @Override
+    protected GeoHexGridAggregationBuilder mutateInstance(GeoHexGridAggregationBuilder instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public void testInvalidPrecision() {
         GeoHexGridAggregationBuilder geoHexGridAggregationBuilder = new GeoHexGridAggregationBuilder("_name");
         expectThrows(IllegalArgumentException.class, () -> geoHexGridAggregationBuilder.precision(16));
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexAggregatorTests.java
index 1427da4daf7d..4aa2340eef3b 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexAggregatorTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexAggregatorTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.geo.GeometryTestUtils;
 import org.elasticsearch.geometry.Point;
 import org.elasticsearch.geometry.Rectangle;
-import org.elasticsearch.h3.CellBoundary;
 import org.elasticsearch.h3.H3;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.plugins.SearchPlugin;
@@ -27,6 +26,7 @@
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin;
+import org.elasticsearch.xpack.spatial.common.H3SphericalUtil;
 import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType;
 
 import java.io.IOException;
@@ -95,24 +95,9 @@ protected GeoBoundingBox randomBBox() {
 
     @Override
     protected Rectangle getTile(double lng, double lat, int precision) {
-        CellBoundary boundary = H3.h3ToGeoBoundary(hashAsString(lng, lat, precision));
-        double minLat = Double.POSITIVE_INFINITY;
-        double minLon = Double.POSITIVE_INFINITY;
-        double maxLat = Double.NEGATIVE_INFINITY;
-        double maxLon = Double.NEGATIVE_INFINITY;
-        for (int i = 0; i < boundary.numPoints(); i++) {
-            double boundaryLat = boundary.getLatLon(i).getLatDeg();
-            double boundaryLon = boundary.getLatLon(i).getLonDeg();
-            minLon = Math.min(minLon, boundaryLon);
-            maxLon = Math.max(maxLon, boundaryLon);
-            minLat = Math.min(minLat, boundaryLat);
-            maxLat = Math.max(maxLat, boundaryLat);
-        }
-        if (maxLon - minLon > 180) {
-            return new Rectangle(maxLon, minLon, maxLat, minLat);
-        } else {
-            return new Rectangle(minLon, maxLon, maxLat, minLat);
-        }
+        final GeoBoundingBox boundingBox = new GeoBoundingBox(new GeoPoint(), new GeoPoint());
+        H3SphericalUtil.computeGeoBounds(H3.stringToH3(hashAsString(lng, lat, precision)), boundingBox);
+        return new Rectangle(boundingBox.left(), boundingBox.right(), boundingBox.top(), boundingBox.bottom());
     }
 
     @Override
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexTilerTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexTilerTests.java
new file mode 100644
index 000000000000..1ace7a958105
--- /dev/null
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexTilerTests.java
@@ -0,0 +1,321 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid;
+
+import org.apache.lucene.geo.GeoEncodingUtils;
+import org.apache.lucene.util.ArrayUtil;
+import org.elasticsearch.common.geo.GeoBoundingBox;
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.geo.GeometryTestUtils;
+import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.geometry.Point;
+import org.elasticsearch.geometry.Rectangle;
+import org.elasticsearch.geometry.utils.StandardValidator;
+import org.elasticsearch.geometry.utils.WellKnownText;
+import org.elasticsearch.h3.H3;
+import org.elasticsearch.xpack.spatial.common.H3CartesianUtil;
+import org.elasticsearch.xpack.spatial.index.fielddata.GeoRelation;
+import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeValues;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import static org.elasticsearch.xpack.spatial.util.GeoTestUtils.geoShapeValue;
+import static org.hamcrest.Matchers.equalTo;
+
+public class GeoHexTilerTests extends GeoGridTilerTestCase<GeoHexGridTiler> {
+
+    @Override
+    protected GeoHexGridTiler getGridTiler(GeoBoundingBox bbox, int precision) {
+        return GeoHexGridTiler.makeGridTiler(precision, bbox);
+    }
+
+    @Override
+    protected int maxPrecision() {
+        return H3.MAX_H3_RES;
+    }
+
+    @Override
+    protected Rectangle getCell(double lon, double lat, int precision) {
+        return H3CartesianUtil.toBoundingBox(H3.geoToH3(lat, lon, precision));
+    }
+
+    /** The H3 tilers does not produce rectangular tiles, and some tests assume this */
+    @Override
+    protected boolean isRectangularTiler() {
+        return false;
+    }
+
+    @Override
+    protected long getCellsForDiffPrecision(int precisionDiff) {
+        return GeoHexGridTiler.calcMaxAddresses(precisionDiff);
+    }
+
+    public void testLargeShape() throws Exception {
+        // We have a shape and a tile both covering all mercator space, so we expect all level0 H3 cells to match
+        Rectangle shapeRectangle = new Rectangle(-180, 180, 90, -90);
+        GeoShapeValues.GeoShapeValue value = geoShapeValue(shapeRectangle);
+
+        GeoBoundingBox boundingBox = new GeoBoundingBox(
+            new GeoPoint(shapeRectangle.getMaxLat(), shapeRectangle.getMinLon()),
+            new GeoPoint(shapeRectangle.getMinLat(), shapeRectangle.getMaxLon())
+        );
+
+        for (int precision = 0; precision < 4; precision++) {
+            GeoShapeCellValues values = new GeoShapeCellValues(
+                makeGeoShapeValues(value),
+                getGridTiler(boundingBox, precision),
+                NOOP_BREAKER
+            );
+            assertTrue(values.advanceExact(0));
+            int numTiles = values.docValueCount();
+            int expectedTiles = expectedBuckets(value, precision, boundingBox);
+            assertThat(expectedTiles, equalTo(numTiles));
+        }
+    }
+
+    public void testLargeShapeWithBounds() throws Exception {
+        // We have a shape covering all space
+        Rectangle shapeRectangle = new Rectangle(-180, 180, 90, -90);
+        GeoShapeValues.GeoShapeValue value = geoShapeValue(shapeRectangle);
+
+        Point point = GeometryTestUtils.randomPoint();
+        int res = randomIntBetween(0, H3.MAX_H3_RES - 4);
+        long h3 = H3.geoToH3(point.getLat(), point.getLon(), res);
+        Rectangle tile = H3CartesianUtil.toBoundingBox(h3);
+        GeoBoundingBox boundingBox = new GeoBoundingBox(
+            new GeoPoint(
+                GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(tile.getMaxLat())),
+                GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(tile.getMinLon()))
+            ),
+            new GeoPoint(
+                GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(tile.getMinLat())),
+                GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(tile.getMaxLon()))
+            )
+        );
+
+        for (int precision = res; precision < res + 4; precision++) {
+            String msg = "Failed " + WellKnownText.toWKT(point) + " at resolution " + res + " with precision " + precision;
+            GeoShapeCellValues values = new GeoShapeCellValues(
+                makeGeoShapeValues(value),
+                getGridTiler(boundingBox, precision),
+                NOOP_BREAKER
+            );
+            assertTrue(values.advanceExact(0));
+            long[] h3bins = ArrayUtil.copyOfSubArray(values.getValues(), 0, values.docValueCount());
+            assertCorner(h3bins, new Point(tile.getMinLon(), tile.getMinLat()), precision, msg);
+            assertCorner(h3bins, new Point(tile.getMaxLon(), tile.getMinLat()), precision, msg);
+            assertCorner(h3bins, new Point(tile.getMinLon(), tile.getMaxLat()), precision, msg);
+            assertCorner(h3bins, new Point(tile.getMaxLon(), tile.getMaxLat()), precision, msg);
+        }
+    }
+
+    // Polygons with bounds inside the South Pole cell break a tiler optimization
+    public void testTroublesomeShapeAlmostWithinSouthPole_BoundedGeoShapeCellValues() throws Exception {
+        int precision = 1;
+        String polygon = """
+            POLYGON((180.0 -90.0, 180.0 -73.80002960532788, 1.401298464324817E-45 -73.80002960532788,
+            1.401298464324817E-45 -90.0, 180.0 -90.0))""";
+        GeoBoundingBox geoBoundingBox = new GeoBoundingBox(
+            new GeoPoint(19.585157879020088, 0.9999999403953552),
+            new GeoPoint(-90.0, -26.405694642531472)
+        );
+        Geometry geometry = WellKnownText.fromWKT(StandardValidator.instance(true), true, polygon);
+        GeoShapeValues.GeoShapeValue value = geoShapeValue(geometry);
+        GeoShapeCellValues cellValues = new GeoShapeCellValues(
+            makeGeoShapeValues(value),
+            getGridTiler(geoBoundingBox, precision),
+            NOOP_BREAKER
+        );
+
+        assertTrue(cellValues.advanceExact(0));
+        int numBuckets = cellValues.docValueCount();
+        int expected = expectedBuckets(value, precision, geoBoundingBox);
+        assertThat("[" + precision + "] bucket count", numBuckets, equalTo(expected));
+    }
+
+    // Polygons with bounds inside the South Pole cell break a tiler optimization
+    public void testTroublesomeShapeAlmostWithinSouthPoleCell_UnboundedGeoShapeCellValues() throws Exception {
+        int precision = 0;
+        String polygon = """
+            POLYGON((1.7481549674935762E-110 -90.0, 180.0 -90.0, 180.0 -75.113250736563,
+            1.7481549674935762E-110 -75.113250736563, 1.7481549674935762E-110 -90.0))""";
+        Geometry geometry = WellKnownText.fromWKT(StandardValidator.instance(true), true, polygon);
+        GeoShapeValues.GeoShapeValue value = geoShapeValue(geometry);
+        GeoShapeCellValues unboundedCellValues = new GeoShapeCellValues(makeGeoShapeValues(value), getGridTiler(precision), NOOP_BREAKER);
+
+        assertTrue(unboundedCellValues.advanceExact(0));
+        int numBuckets = unboundedCellValues.docValueCount();
+        int expected = expectedBuckets(value, precision, null);
+        assertThat("[" + precision + "] bucket count", numBuckets, equalTo(expected));
+    }
+
+    // Polygons with bounds inside the North Pole cell break a tiler optimization
+    public void testTroublesomeShapeAlmostWithinNorthPoleCell_UnboundedGeoShapeCellValues() throws Exception {
+        int precision = 1;
+        String polygon = """
+            POLYGON((36.98661841690625 69.44049730644747, 180.0 69.44049730644747,
+            180.0 90.0, 36.98661841690625 90.0, 36.98661841690625 69.44049730644747))""";
+        Geometry geometry = WellKnownText.fromWKT(StandardValidator.instance(true), true, polygon);
+        GeoShapeValues.GeoShapeValue value = geoShapeValue(geometry);
+        GeoShapeCellValues unboundedCellValues = new GeoShapeCellValues(makeGeoShapeValues(value), getGridTiler(precision), NOOP_BREAKER);
+
+        assertTrue(unboundedCellValues.advanceExact(0));
+        int numBuckets = unboundedCellValues.docValueCount();
+        int expected = expectedBuckets(value, precision, null);
+        assertThat("[" + precision + "] bucket count", numBuckets, equalTo(expected));
+    }
+
+    public void testTroublesomePolarCellLevel1_UnboundedGeoShapeCellValues() throws Exception {
+        int precision = 1;
+        String polygon = "BBOX (-84.24596376729815, 43.36113427778119, 90.0, 83.51476833522361)";
+        Geometry geometry = WellKnownText.fromWKT(StandardValidator.instance(true), true, polygon);
+        GeoShapeValues.GeoShapeValue value = geoShapeValue(geometry);
+        GeoShapeCellValues unboundedCellValues = new GeoShapeCellValues(makeGeoShapeValues(value), getGridTiler(precision), NOOP_BREAKER);
+
+        assertTrue(unboundedCellValues.advanceExact(0));
+        int numBuckets = unboundedCellValues.docValueCount();
+        int expected = expectedBuckets(value, precision, null);
+        assertThat("[" + precision + "] bucket count", numBuckets, equalTo(expected));
+    }
+
+    public void testTroublesomeCellLevel2_BoundedGeoShapeCellValues() throws Exception {
+        int precision = 2;
+        String wkt = """
+            GEOMETRYCOLLECTION (
+              GEOMETRYCOLLECTION (
+                POINT(-170 0),
+                POINT (-178.5 0)
+              )
+            )
+            """;
+        GeoBoundingBox boundingBox = new GeoBoundingBox(new GeoPoint(4E-4, 179.999), new GeoPoint(-4E-4, -179.999));
+        Geometry geometry = WellKnownText.fromWKT(StandardValidator.instance(true), true, wkt);
+        GeoShapeValues.GeoShapeValue value = geoShapeValue(geometry);
+        GeoShapeCellValues cellValues = new GeoShapeCellValues(
+            makeGeoShapeValues(value),
+            getGridTiler(boundingBox, precision),
+            NOOP_BREAKER
+        );
+
+        assertTrue(cellValues.advanceExact(0));
+        int numBuckets = cellValues.docValueCount();
+        int expected = expectedBuckets(value, precision, boundingBox);
+        assertThat("[" + precision + "] bucket count", numBuckets, equalTo(expected));
+    }
+
+    public void testTroublesomeCellLevel4_BoundedGeoShapeCellValues() throws Exception {
+        int precision = 4;
+        String polygon = "POLYGON ((150.0 70.0, 150.0 85.91811374669217, 168.77544806565834 85.91811374669217, 150.0 70.0))";
+        Geometry geometry = WellKnownText.fromWKT(StandardValidator.instance(true), true, polygon);
+        GeoBoundingBox boundingBox = new GeoBoundingBox(
+            new GeoPoint(86.17678739494652, 172.21916569181505),
+            new GeoPoint(83.01600086049713, 179)
+        );
+        GeoShapeValues.GeoShapeValue value = geoShapeValue(geometry);
+        GeoShapeCellValues cellValues = new GeoShapeCellValues(
+            makeGeoShapeValues(value),
+            getGridTiler(boundingBox, precision),
+            NOOP_BREAKER
+        );
+
+        assertTrue(cellValues.advanceExact(0));
+        int numBuckets = cellValues.docValueCount();
+        int expected = expectedBuckets(value, precision, boundingBox);
+        assertThat("[" + precision + "] bucket count", numBuckets, equalTo(expected));
+    }
+
+    private void assertCorner(long[] h3bins, Point point, int precision, String msg) throws IOException {
+        GeoShapeValues.GeoShapeValue cornerValue = geoShapeValue(point);
+        GeoShapeCellValues cornerValues = new GeoShapeCellValues(makeGeoShapeValues(cornerValue), getGridTiler(precision), NOOP_BREAKER);
+        assertTrue(cornerValues.advanceExact(0));
+        long[] h3binsCorner = ArrayUtil.copyOfSubArray(cornerValues.getValues(), 0, cornerValues.docValueCount());
+        for (long corner : h3binsCorner) {
+            assertTrue(msg, Arrays.binarySearch(h3bins, corner) != -1);
+        }
+    }
+
+    @Override
+    protected void assertSetValuesBruteAndRecursive(Geometry geometry) throws Exception {
+        int precision = randomIntBetween(1, 4);
+        GeoHexGridTiler tiler = getGridTiler(precision);
+        GeoShapeValues.GeoShapeValue value = geoShapeValue(geometry);
+
+        GeoShapeCellValues recursiveValues = new GeoShapeCellValues(null, tiler, NOOP_BREAKER);
+        int recursiveCount = tiler.setValuesByRecursion(recursiveValues, value, value.boundingBox());
+
+        GeoShapeCellValues bruteForceValues = new GeoShapeCellValues(null, tiler, NOOP_BREAKER);
+        int bruteForceCount = 0;
+        for (long h3 : H3.getLongRes0Cells()) {
+            bruteForceCount = addBruteForce(tiler, bruteForceValues, value, h3, precision, bruteForceCount);
+        }
+
+        long[] recursive = Arrays.copyOf(recursiveValues.getValues(), recursiveCount);
+        long[] bruteForce = Arrays.copyOf(bruteForceValues.getValues(), bruteForceCount);
+
+        Arrays.sort(recursive);
+        Arrays.sort(bruteForce);
+        assertArrayEquals(geometry.toString(), recursive, bruteForce);
+    }
+
+    private int addBruteForce(
+        GeoHexGridTiler tiler,
+        GeoShapeCellValues values,
+        GeoShapeValues.GeoShapeValue geoValue,
+        long h3,
+        int precision,
+        int valueIndex
+    ) throws IOException {
+        if (H3.getResolution(h3) == precision) {
+            if (tiler.relateTile(geoValue, h3) != GeoRelation.QUERY_DISJOINT) {
+                values.resizeCell(valueIndex + 1);
+                values.add(valueIndex++, h3);
+            }
+        } else {
+            for (long child : H3.h3ToChildren(h3)) {
+                valueIndex = addBruteForce(tiler, values, geoValue, child, precision, valueIndex);
+            }
+        }
+        return valueIndex;
+    }
+
+    @Override
+    protected int expectedBuckets(GeoShapeValues.GeoShapeValue geoValue, int precision, GeoBoundingBox bbox) throws Exception {
+        GeoHexGridTiler bounded = bbox == null ? null : getGridTiler(bbox, precision);
+        GeoHexGridTiler predicate = getGridTiler(precision);
+        return computeBuckets(H3.getLongRes0Cells(), bounded, predicate, geoValue, precision);
+    }
+
+    private int computeBuckets(
+        long[] children,
+        GeoHexGridTiler bounded,
+        GeoHexGridTiler predicate,
+        GeoShapeValues.GeoShapeValue geoValue,
+        int finalPrecision
+    ) throws IOException {
+        int count = 0;
+        for (long child : children) {
+            if (H3.getResolution(child) == finalPrecision) {
+                if (intersects(child, geoValue, bounded, predicate)) {
+                    count++;
+                }
+            } else {
+                count += computeBuckets(H3.h3ToChildren(child), bounded, predicate, geoValue, finalPrecision);
+            }
+        }
+        return count;
+    }
+
+    private boolean intersects(long h3, GeoShapeValues.GeoShapeValue geoValue, GeoHexGridTiler bounded, GeoHexGridTiler predicate)
+        throws IOException {
+        if (bounded != null && bounded.h3IntersectsBounds(h3) == false) {
+            return false;
+        }
+        return predicate.relateTile(geoValue, h3) != GeoRelation.QUERY_DISJOINT;
+    }
+}
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexVisitorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexVisitorTests.java
new file mode 100644
index 000000000000..0dbd6323b849
--- /dev/null
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexVisitorTests.java
@@ -0,0 +1,234 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid;
+
+import org.apache.lucene.tests.geo.GeoTestUtil;
+import org.elasticsearch.common.geo.GeoBoundingBox;
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.geo.GeometryNormalizer;
+import org.elasticsearch.common.geo.Orientation;
+import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.geometry.GeometryCollection;
+import org.elasticsearch.geometry.Line;
+import org.elasticsearch.geometry.LinearRing;
+import org.elasticsearch.geometry.MultiPoint;
+import org.elasticsearch.geometry.Point;
+import org.elasticsearch.geometry.Polygon;
+import org.elasticsearch.geometry.Rectangle;
+import org.elasticsearch.h3.H3;
+import org.elasticsearch.h3.LatLng;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.spatial.index.fielddata.CoordinateEncoder;
+import org.elasticsearch.xpack.spatial.index.fielddata.GeoRelation;
+import org.elasticsearch.xpack.spatial.index.fielddata.GeometryDocValueReader;
+import org.elasticsearch.xpack.spatial.util.GeoTestUtils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.LongFunction;
+
+public class GeoHexVisitorTests extends ESTestCase {
+
+    public void testPoint() throws IOException {
+        doTestGeometry(GeoHexVisitorTests::getGeometryAsPoints, false);
+    }
+
+    public void testLine() throws IOException {
+        doTestGeometry(GeoHexVisitorTests::getGeometryAsLine, false);
+    }
+
+    public void testTriangle() throws IOException {
+        doTestGeometry(GeoHexVisitorTests::getGeometryAsPolygon, true);
+    }
+
+    private void doTestGeometry(LongFunction<Geometry> h3ToGeometry, boolean hasArea) throws IOException {
+        // we ignore polar cells are they are problematic and do not keep the relationships
+        long h3 = randomValueOtherThanMany(
+            l -> l == H3.geoToH3(90, 0, H3.getResolution(l)) || l == H3.geoToH3(-90, 0, H3.getResolution(l)),
+            () -> H3.geoToH3(GeoTestUtil.nextLatitude(), GeoTestUtil.nextLongitude(), randomIntBetween(2, 14))
+        );
+        long centerChild = H3.childPosToH3(h3, 0);
+        // children position 3 is chosen so we never use a polar polygon
+        long noChildIntersecting = H3.noChildIntersectingPosToH3(h3, 3);
+        GeoHexVisitor visitor = new GeoHexVisitor();
+        visitor.reset(h3);
+        final String failMsg = "failing h3: " + h3;
+        boolean h3CrossesDateline = visitor.getLeftX() > visitor.getRightX();
+        {
+            GeometryDocValueReader reader = GeoTestUtils.geometryDocValueReader(h3ToGeometry.apply(h3), CoordinateEncoder.GEO);
+            visitor.reset(h3);
+            reader.visit(visitor);
+            assertEquals(failMsg, GeoRelation.QUERY_CROSSES, visitor.relation());
+
+            Rectangle rectangle = getGeometryAsRectangle(h3);
+            assertTrue(failMsg, visitor.intersectsBbox(rectangle.getMinX(), rectangle.getMaxX(), rectangle.getMinY(), rectangle.getMaxY()));
+        }
+        {
+            GeometryDocValueReader reader = GeoTestUtils.geometryDocValueReader(h3ToGeometry.apply(centerChild), CoordinateEncoder.GEO);
+            visitor.reset(h3);
+            reader.visit(visitor);
+            assertEquals("failing h3: " + h3, GeoRelation.QUERY_CONTAINS, visitor.relation());
+
+            Rectangle rectangle = getGeometryAsRectangle(centerChild);
+            assertTrue(failMsg, visitor.intersectsBbox(rectangle.getMinX(), rectangle.getMaxX(), rectangle.getMinY(), rectangle.getMaxY()));
+        }
+        {
+            GeometryDocValueReader reader = GeoTestUtils.geometryDocValueReader(h3ToGeometry.apply(h3), CoordinateEncoder.GEO);
+            visitor.reset(centerChild);
+            reader.visit(visitor);
+            if (hasArea) {
+                if (h3CrossesDateline && visitor.getLeftX() > visitor.getRightX()) {
+                    // if both polygons crosses the dateline it cannot be inside due to the polygon splitting technique
+                    assertEquals("failing h3: " + h3, GeoRelation.QUERY_CROSSES, visitor.relation());
+                } else {
+                    assertEquals("failing h3: " + h3, GeoRelation.QUERY_INSIDE, visitor.relation());
+                }
+            } else {
+                assertEquals("failing h3: " + h3, GeoRelation.QUERY_DISJOINT, visitor.relation());
+            }
+        }
+        {
+            GeometryDocValueReader reader = GeoTestUtils.geometryDocValueReader(
+                h3ToGeometry.apply(noChildIntersecting),
+                CoordinateEncoder.GEO
+            );
+            visitor.reset(centerChild);
+            reader.visit(visitor);
+            assertEquals("failing h3: " + h3, GeoRelation.QUERY_DISJOINT, visitor.relation());
+
+            Rectangle rectangle = getGeometryAsRectangle(noChildIntersecting);
+            assertFalse(
+                failMsg,
+                visitor.intersectsBbox(rectangle.getMinX(), rectangle.getMaxX(), rectangle.getMinY(), rectangle.getMaxY())
+            );
+        }
+        {
+            GeometryCollection<Geometry> collection = new GeometryCollection<>(
+                List.of(h3ToGeometry.apply(centerChild), h3ToGeometry.apply(noChildIntersecting))
+            );
+            GeometryDocValueReader reader = GeoTestUtils.geometryDocValueReader(collection, CoordinateEncoder.GEO);
+            visitor.reset(h3);
+            reader.visit(visitor);
+            assertEquals("failing h3: " + h3, GeoRelation.QUERY_CROSSES, visitor.relation());
+        }
+        {
+            LatLng latLng1 = H3.h3ToLatLng(centerChild);
+            LatLng latLng2 = H3.h3ToLatLng(noChildIntersecting);
+            MultiPoint multiPoint = new MultiPoint(
+                List.of(new Point(latLng1.getLonDeg(), latLng1.getLatDeg()), new Point(latLng2.getLonDeg(), latLng2.getLatDeg()))
+            );
+            GeometryDocValueReader reader = GeoTestUtils.geometryDocValueReader(multiPoint, CoordinateEncoder.GEO);
+            visitor.reset(h3);
+            reader.visit(visitor);
+            assertEquals("failing h3: " + h3, GeoRelation.QUERY_CROSSES, visitor.relation());
+        }
+    }
+
+    private static Geometry getGeometryAsPolygon(long h3) {
+        final GeoHexVisitor visitor = new GeoHexVisitor();
+        visitor.reset(h3);
+        final Polygon polygon = new Polygon(new LinearRing(visitor.getXs(), visitor.getYs()));
+        if (visitor.getLeftX() > visitor.getRightX()) {
+            final Geometry geometry = GeometryNormalizer.apply(Orientation.CCW, polygon);
+            if (geometry instanceof Polygon) {
+                // there is a bug on the code that breaks polygons across the dateline
+                // when polygon is close to the pole (I think) so we need to try again
+                return GeometryNormalizer.apply(Orientation.CW, polygon);
+            }
+            return geometry;
+        } else {
+            return polygon;
+        }
+    }
+
+    private static Geometry getGeometryAsLine(long h3) {
+        final GeoHexVisitor visitor = new GeoHexVisitor();
+        visitor.reset(h3);
+        if (visitor.getLeftX() > visitor.getRightX()) {
+            double[] translatedXs = visitor.getXs();
+            for (int i = 0; i < translatedXs.length; i++) {
+                translatedXs[i] = translatedXs[i] < 0 ? translatedXs[i] + 360 : translatedXs[i];
+            }
+            final Geometry geometry = GeometryNormalizer.apply(Orientation.CCW, new Line(translatedXs, visitor.getYs()));
+            return GeometryNormalizer.apply(Orientation.CW, geometry);
+        } else {
+            return new Line(visitor.getXs(), visitor.getYs());
+        }
+    }
+
+    private static Geometry getGeometryAsPoints(long h3) {
+        final GeoHexVisitor visitor = new GeoHexVisitor();
+        visitor.reset(h3);
+        List<Point> points = new ArrayList<>();
+        double[] xs = visitor.getXs();
+        double[] ys = visitor.getYs();
+        for (int i = 0; i < xs.length; i++) {
+            points.add(new Point(xs[i], ys[i]));
+        }
+        return new MultiPoint(points);
+    }
+
+    private static Rectangle getGeometryAsRectangle(long h3) {
+        final GeoHexVisitor visitor = new GeoHexVisitor();
+        visitor.reset(h3);
+        return new Rectangle(visitor.getLeftX(), visitor.getRightX(), visitor.getMaxY(), visitor.getMinY());
+    }
+
+    public void testLongGeometriesWithDateline() throws IOException {
+        long h3 = H3.geoToH3(0, 180, randomIntBetween(0, 4));
+        GeoHexVisitor visitor = new GeoHexVisitor();
+        visitor.reset(h3);
+        {
+            Line line = new Line(new double[] { -180, 180 }, new double[] { 0, 0 });
+            GeometryDocValueReader reader = GeoTestUtils.geometryDocValueReader(line, CoordinateEncoder.GEO);
+            reader.visit(visitor);
+            assertEquals(GeoRelation.QUERY_CROSSES, visitor.relation());
+        }
+        {
+            Polygon polygon = new Polygon(new LinearRing(new double[] { -180, 180, 180, -180 }, new double[] { -1, -1, 1, -1 }));
+            GeometryDocValueReader reader = GeoTestUtils.geometryDocValueReader(polygon, CoordinateEncoder.GEO);
+            reader.visit(visitor);
+            assertEquals(GeoRelation.QUERY_CROSSES, visitor.relation());
+        }
+    }
+
+    // Testing issue with a specific cell left of the dateline that touches the dateline with one point
+    // Intersecting a polygon on the other side of the dateline.
+    public void testSpecificCellTouchesDateline() throws IOException {
+        long h3 = 646728346019944298L;
+        GeoBoundingBox box = new GeoBoundingBox(new GeoPoint(-11.29550, -180), new GeoPoint(-11.29552, -179.99999));
+        GeoHexVisitor visitor = new GeoHexVisitor();
+        visitor.reset(h3);
+        {
+            // Polygon on the same side of the dateline (overlaps)
+            Polygon polygon = makePolygonFromBox(179.99999, 180, -11.29550, -11.29552);
+            GeometryDocValueReader reader = GeoTestUtils.geometryDocValueReader(polygon, CoordinateEncoder.GEO);
+            reader.visit(visitor);
+            assertEquals("Polygon on the same side of the dateline", GeoRelation.QUERY_CROSSES, visitor.relation());
+        }
+        {
+            // Polygon on the other side of the dateline (just touches)
+            Polygon polygon = makePolygonFromBox(-180, -179.99999, -11.29550, -11.29552);
+            GeometryDocValueReader reader = GeoTestUtils.geometryDocValueReader(polygon, CoordinateEncoder.GEO);
+            reader.visit(visitor);
+            assertEquals(
+                "Polygon on the other side of the dateline (touches at single point)",
+                GeoRelation.QUERY_CROSSES,
+                visitor.relation()
+            );
+        }
+    }
+
+    private Polygon makePolygonFromBox(double left, double right, double top, double bottom) {
+        double[] x = new double[] { left, right, right, left, left };
+        double[] y = new double[] { bottom, bottom, top, top, bottom };
+        LinearRing ring = new LinearRing(x, y);
+        return new Polygon(ring);
+    }
+}
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java
index fe8a57171a5e..c2dd40250bae 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.MultiPoint;
 import org.elasticsearch.geometry.Point;
-import org.elasticsearch.geometry.Rectangle;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -54,7 +53,7 @@
 import static org.hamcrest.Matchers.equalTo;
 
 public abstract class GeoShapeGeoGridTestCase<T extends InternalGeoGridBucket> extends AggregatorTestCase {
-    private static final String FIELD_NAME = "location";
+    protected static final String FIELD_NAME = "location";
 
     /**
      * Generate a random precision according to the rules of the given aggregation.
@@ -62,9 +61,9 @@ public abstract class GeoShapeGeoGridTestCase<T extends InternalGeoGridBucket> e
     protected abstract int randomPrecision();
 
     /**
-     * Convert geo point into a hash string (bucket string ID)
+     * Convert geo point into an array of hash string (bucket string ID).
      */
-    protected abstract String hashAsString(double lng, double lat, int precision);
+    protected abstract String[] hashAsStrings(double lng, double lat, int precision);
 
     /**
      * Return a point within the bounds of the tile grid
@@ -77,14 +76,14 @@ public abstract class GeoShapeGeoGridTestCase<T extends InternalGeoGridBucket> e
     protected abstract GeoBoundingBox randomBBox();
 
     /**
-     * Return the bounding tile as a {@link Rectangle} for a given point
+     * Return true if the hash intersects the given shape value
      */
-    protected abstract boolean intersects(double lng, double lat, int precision, GeoShapeValues.GeoShapeValue value) throws IOException;
+    protected abstract boolean intersects(String hash, GeoShapeValues.GeoShapeValue value) throws IOException;
 
     /**
-     * Return true if the points intersects the bounds
+     * Return true if the hash intersects the given bounding box
      */
-    protected abstract boolean intersectsBounds(double lng, double lat, int precision, GeoBoundingBox box);
+    protected abstract boolean intersectsBounds(String hash, GeoBoundingBox box);
 
     /**
      * Create a new named {@link GeoGridAggregationBuilder}-derived builder
@@ -164,15 +163,17 @@ public void testGeoShapeBounds() throws IOException {
             double lon = GeoTestUtils.encodeDecodeLon(p.getX());
             double lat = GeoTestUtils.encodeDecodeLat(p.getY());
             GeoShapeValues.GeoShapeValue value = geoShapeValue(p);
-            if (intersects(lon, lat, precision, value) && intersectsBounds(lon, lat, precision, bbox)) {
-                numDocsWithin += 1;
+            String[] hashes = hashAsStrings(lon, lat, precision);
+            for (String hash : hashes) {
+                if (intersects(hash, value) && intersectsBounds(hash, bbox)) {
+                    numDocsWithin += 1;
+                }
             }
 
             docs.add(binaryGeoShapeDocValuesField(FIELD_NAME, p));
         }
 
         final long numDocsInBucket = numDocsWithin;
-
         testCase(new MatchAllDocsQuery(), FIELD_NAME, precision, bbox, iw -> {
             for (BinaryShapeDocValuesField docField : docs) {
                 iw.addDocument(Collections.singletonList(docField));
@@ -208,11 +209,13 @@ public void testGeoShapeWithSeveralDocs() throws IOException {
                 lat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat));
 
                 shapes.add(new Point(lng, lat));
-                String hash = hashAsString(lng, lat, precision);
-                if (distinctHashesPerDoc.contains(hash) == false) {
-                    expectedCountPerGeoHash.put(hash, expectedCountPerGeoHash.getOrDefault(hash, 0) + 1);
+                String[] hashes = hashAsStrings(lng, lat, precision);
+                for (String hash : hashes) {
+                    if (distinctHashesPerDoc.contains(hash) == false) {
+                        expectedCountPerGeoHash.put(hash, expectedCountPerGeoHash.getOrDefault(hash, 0) + 1);
+                    }
+                    distinctHashesPerDoc.add(hash);
                 }
-                distinctHashesPerDoc.add(hash);
                 if (usually()) {
                     Geometry geometry = new MultiPoint(new ArrayList<>(shapes));
                     document.add(binaryGeoShapeDocValuesField(FIELD_NAME, geometry));
@@ -248,7 +251,7 @@ private void testCase(
     }
 
     @SuppressWarnings("unchecked")
-    private void testCase(
+    protected void testCase(
         Query query,
         int precision,
         GeoBoundingBox geoBoundingBox,
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoHashGridAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoHashGridAggregatorTests.java
index 8826631ebad3..9622d9f77717 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoHashGridAggregatorTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoHashGridAggregatorTests.java
@@ -33,8 +33,8 @@ protected int randomPrecision() {
     }
 
     @Override
-    protected String hashAsString(double lng, double lat, int precision) {
-        return stringEncode(lng, lat, precision);
+    protected String[] hashAsStrings(double lng, double lat, int precision) {
+        return new String[] { stringEncode(lng, lat, precision) };
     }
 
     @Override
@@ -48,8 +48,8 @@ protected GeoBoundingBox randomBBox() {
     }
 
     @Override
-    protected boolean intersects(double lng, double lat, int precision, GeoShapeValues.GeoShapeValue value) throws IOException {
-        Rectangle boundingBox = GeoGridQueryBuilder.getQueryHash(hashAsString(lng, lat, precision));
+    protected boolean intersects(String hash, GeoShapeValues.GeoShapeValue value) throws IOException {
+        final Rectangle boundingBox = GeoGridQueryBuilder.getQueryHash(hash);
         return value.relate(
             GeoEncodingUtils.encodeLongitude(boundingBox.getMinLon()),
             GeoEncodingUtils.encodeLongitude(boundingBox.getMaxLon()),
@@ -59,9 +59,9 @@ protected boolean intersects(double lng, double lat, int precision, GeoShapeValu
     }
 
     @Override
-    protected boolean intersectsBounds(double lng, double lat, int precision, GeoBoundingBox box) {
-        GeoHashBoundedPredicate predicate = new GeoHashBoundedPredicate(precision, box);
-        return predicate.validHash(stringEncode(lng, lat, precision));
+    protected boolean intersectsBounds(String hash, GeoBoundingBox box) {
+        final GeoHashBoundedPredicate predicate = new GeoHashBoundedPredicate(hash.length(), box);
+        return predicate.validHash(hash);
     }
 
     @Override
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoHexGridAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoHexGridAggregatorTests.java
new file mode 100644
index 000000000000..e90bfa0adc49
--- /dev/null
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoHexGridAggregatorTests.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid;
+
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.geo.LatLonGeometry;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.geo.GeoBoundingBox;
+import org.elasticsearch.geo.GeometryTestUtils;
+import org.elasticsearch.geometry.Point;
+import org.elasticsearch.h3.H3;
+import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder;
+import org.elasticsearch.xpack.spatial.common.H3CartesianUtil;
+import org.elasticsearch.xpack.spatial.index.fielddata.GeoRelation;
+import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeValues;
+import org.elasticsearch.xpack.spatial.util.GeoTestUtils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class GeoShapeGeoHexGridAggregatorTests extends GeoShapeGeoGridTestCase<InternalGeoHexGridBucket> {
+    @Override
+    protected int randomPrecision() {
+        return randomIntBetween(0, H3.MAX_H3_RES);
+    }
+
+    @Override
+    protected String[] hashAsStrings(double lng, double lat, int precision) {
+        final List<String> strings = new ArrayList<>();
+        final long h3 = H3.geoToH3(lat, lng, precision);
+        if (LatLonGeometry.create(H3CartesianUtil.getLatLonGeometry(h3)).contains(lng, lat)) {
+            strings.add(H3.h3ToString(h3));
+        }
+        for (long n : H3.hexRing(h3)) {
+            if (LatLonGeometry.create(H3CartesianUtil.getLatLonGeometry(n)).contains(lng, lat)) {
+                strings.add(H3.h3ToString(n));
+            }
+        }
+        assertTrue("did not find any hash for point", strings.size() > 0);
+        return strings.toArray(new String[0]);
+    }
+
+    @Override
+    protected Point randomPoint() {
+        return GeometryTestUtils.randomPoint();
+    }
+
+    @Override
+    protected GeoBoundingBox randomBBox() {
+        return GeoTestUtils.randomBBox();
+    }
+
+    @Override
+    protected boolean intersects(String hash, GeoShapeValues.GeoShapeValue value) throws IOException {
+        final GeoHexVisitor visitor = new GeoHexVisitor();
+        visitor.reset(H3.stringToH3(hash));
+        value.visit(visitor);
+        return visitor.relation() != GeoRelation.QUERY_DISJOINT;
+    }
+
+    @Override
+    protected boolean intersectsBounds(String hash, GeoBoundingBox box) {
+        final GeoHexGridTiler tiler = GeoHexGridTiler.makeGridTiler(H3.getResolution(hash), box);
+        return tiler.h3IntersectsBounds(H3.stringToH3(hash));
+    }
+
+    @Override
+    protected GeoGridAggregationBuilder createBuilder(String name) {
+        return new GeoHexGridAggregationBuilder(name);
+    }
+
+    @Override
+    public void testMappedMissingGeoShape() throws IOException {
+        final String lineString = "LINESTRING (30 10, 10 30, 40 40)";
+        final GeoGridAggregationBuilder builder = createBuilder("_name").field(FIELD_NAME).missing(lineString);
+        testCase(
+            new MatchAllDocsQuery(),
+            1,
+            null,
+            iw -> { iw.addDocument(Collections.singleton(new SortedSetDocValuesField("string", new BytesRef("a")))); },
+            geoGrid -> { assertEquals(8, geoGrid.getBuckets().size()); },
+            builder
+        );
+    }
+}
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoTileGridAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoTileGridAggregatorTests.java
index 8d0d011dee1d..5bacc3657642 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoTileGridAggregatorTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoTileGridAggregatorTests.java
@@ -32,8 +32,8 @@ protected int randomPrecision() {
     }
 
     @Override
-    protected String hashAsString(double lng, double lat, int precision) {
-        return GeoTileUtils.stringEncode(GeoTileUtils.longEncode(lng, lat, precision));
+    protected String[] hashAsStrings(double lng, double lat, int precision) {
+        return new String[] { GeoTileUtils.stringEncode(GeoTileUtils.longEncode(lng, lat, precision)) };
     }
 
     @Override
@@ -61,8 +61,8 @@ protected GeoBoundingBox randomBBox() {
     }
 
     @Override
-    protected boolean intersects(double lng, double lat, int precision, GeoShapeValues.GeoShapeValue value) throws IOException {
-        Rectangle r = GeoGridQueryBuilder.getQueryTile(GeoTileUtils.stringEncode(GeoTileUtils.longEncode(lng, lat, precision)));
+    protected boolean intersects(String hash, GeoShapeValues.GeoShapeValue value) throws IOException {
+        final Rectangle r = GeoGridQueryBuilder.getQueryTile(GeoTileUtils.stringEncode(GeoTileUtils.longEncode(hash)));
         return value.relate(
             GeoEncodingUtils.encodeLongitude(r.getMinLon()),
             GeoEncodingUtils.encodeLongitude(r.getMaxLon()),
@@ -72,9 +72,10 @@ protected boolean intersects(double lng, double lat, int precision, GeoShapeValu
     }
 
     @Override
-    protected boolean intersectsBounds(double lng, double lat, int precision, GeoBoundingBox box) {
-        GeoTileBoundedPredicate predicate = new GeoTileBoundedPredicate(precision, box);
-        return predicate.validTile(GeoTileUtils.getXTile(lng, 1L << precision), GeoTileUtils.getYTile(lat, 1L << precision), precision);
+    protected boolean intersectsBounds(String hash, GeoBoundingBox box) {
+        final int[] values = GeoTileUtils.parseHash(hash);
+        final GeoTileBoundedPredicate predicate = new GeoTileBoundedPredicate(values[0], box);
+        return predicate.validTile(values[1], values[2], values[0]);
     }
 
     @Override
diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoTileTilerTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoTileTilerTests.java
index b355a57ea920..bfaf7ad63949 100644
--- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoTileTilerTests.java
+++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoTileTilerTests.java
@@ -37,16 +37,11 @@
 import static org.elasticsearch.xpack.spatial.util.GeoTestUtils.geoShapeValue;
 import static org.hamcrest.Matchers.equalTo;
 
-public class GeoTileTilerTests extends GeoGridTilerTestCase {
+public class GeoTileTilerTests extends GeoGridTilerTestCase<GeoTileGridTiler> {
 
     @Override
-    protected GeoGridTiler getUnboundedGridTiler(int precision) {
-        return new UnboundedGeoTileGridTiler(precision);
-    }
-
-    @Override
-    protected GeoGridTiler getBoundedGridTiler(GeoBoundingBox bbox, int precision) {
-        return new BoundedGeoTileGridTiler(precision, bbox);
+    protected GeoTileGridTiler getGridTiler(GeoBoundingBox bbox, int precision) {
+        return GeoTileGridTiler.makeGridTiler(precision, bbox);
     }
 
     @Override
@@ -67,7 +62,7 @@ protected long getCellsForDiffPrecision(int precisionDiff) {
     @Override
     protected void assertSetValuesBruteAndRecursive(Geometry geometry) throws Exception {
         int precision = randomIntBetween(1, 4);
-        UnboundedGeoTileGridTiler tiler = new UnboundedGeoTileGridTiler(precision);
+        GeoTileGridTiler tiler = getGridTiler(precision);
         geometry = GeometryNormalizer.apply(Orientation.CCW, geometry);
         GeoShapeValues.GeoShapeValue value = geoShapeValue(geometry);
         GeoShapeCellValues recursiveValues = new GeoShapeCellValues(null, tiler, NOOP_BREAKER);
@@ -78,12 +73,12 @@ protected void assertSetValuesBruteAndRecursive(Geometry geometry) throws Except
         GeoShapeCellValues bruteForceValues = new GeoShapeCellValues(null, tiler, NOOP_BREAKER);
         int bruteForceCount;
         {
-            final double tiles = 1 << precision;
+            final int tiles = 1 << precision;
             GeoShapeValues.BoundingBox bounds = value.boundingBox();
-            int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles);
-            int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles);
-            int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles);
-            int maxYTile = GeoTileUtils.getYTile(bounds.minY(), (long) tiles);
+            int minXTile = GeoTileUtils.getXTile(bounds.minX(), tiles);
+            int minYTile = GeoTileUtils.getYTile(bounds.maxY(), tiles);
+            int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), tiles);
+            int maxYTile = GeoTileUtils.getYTile(bounds.minY(), tiles);
             bruteForceCount = tiler.setValuesByBruteForceScan(bruteForceValues, value, minXTile, minYTile, maxXTile, maxYTile);
         }
         assertThat(geometry.toString(), recursiveCount, equalTo(bruteForceCount));
@@ -113,13 +108,13 @@ protected int expectedBuckets(GeoShapeValues.GeoShapeValue geoValue, int precisi
             return 1;
         }
 
-        final double tiles = 1 << precision;
-        int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles);
-        int maxYTile = GeoTileUtils.getYTile(bounds.minY(), (long) tiles);
+        final int tiles = 1 << precision;
+        int minYTile = GeoTileUtils.getYTile(bounds.maxY(), tiles);
+        int maxYTile = GeoTileUtils.getYTile(bounds.minY(), tiles);
         if ((bounds.posLeft >= 0 && bounds.posRight >= 0) && (bounds.negLeft < 0 && bounds.negRight < 0)) {
             // box one
-            int minXTileNeg = GeoTileUtils.getXTile(bounds.negLeft, (long) tiles);
-            int maxXTileNeg = GeoTileUtils.getXTile(bounds.negRight, (long) tiles);
+            int minXTileNeg = GeoTileUtils.getXTile(bounds.negLeft, tiles);
+            int maxXTileNeg = GeoTileUtils.getXTile(bounds.negRight, tiles);
 
             for (int x = minXTileNeg; x <= maxXTileNeg; x++) {
                 for (int y = minYTile; y <= maxYTile; y++) {
@@ -131,12 +126,12 @@ protected int expectedBuckets(GeoShapeValues.GeoShapeValue geoValue, int precisi
             }
 
             // box two
-            int minXTilePos = GeoTileUtils.getXTile(bounds.posLeft, (long) tiles);
+            int minXTilePos = GeoTileUtils.getXTile(bounds.posLeft, tiles);
             if (minXTilePos > maxXTileNeg + 1) {
                 minXTilePos -= 1;
             }
 
-            int maxXTilePos = GeoTileUtils.getXTile(bounds.posRight, (long) tiles);
+            int maxXTilePos = GeoTileUtils.getXTile(bounds.posRight, tiles);
 
             for (int x = minXTilePos; x <= maxXTilePos; x++) {
                 for (int y = minYTile; y <= maxYTile; y++) {
@@ -147,8 +142,8 @@ protected int expectedBuckets(GeoShapeValues.GeoShapeValue geoValue, int precisi
             }
             return count;
         } else {
-            int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles);
-            int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles);
+            int minXTile = GeoTileUtils.getXTile(bounds.minX(), tiles);
+            int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), tiles);
 
             if (minXTile == maxXTile && minYTile == maxYTile) {
                 return tileIntersectsBounds(minXTile, minYTile, precision, bbox) ? 1 : 0;
@@ -188,7 +183,7 @@ public void testGeoTile() throws Exception {
         double x = randomDouble();
         double y = randomDouble();
         int precision = randomIntBetween(0, GeoTileUtils.MAX_ZOOM);
-        assertThat(new UnboundedGeoTileGridTiler(precision).encode(x, y), equalTo(GeoTileUtils.longEncode(x, y, precision)));
+        assertThat(getGridTiler(precision).encode(x, y), equalTo(GeoTileUtils.longEncode(x, y, precision)));
 
         // create rectangle within tile and check bound counts
         Rectangle tile = GeoTileUtils.toBoundingBox(1309, 3166, 13);
@@ -201,17 +196,17 @@ public void testGeoTile() throws Exception {
         GeoShapeValues.GeoShapeValue value = geoShapeValue(shapeRectangle);
         // test shape within tile bounds
         {
-            GeoShapeCellValues values = new GeoShapeCellValues(makeGeoShapeValues(value), new UnboundedGeoTileGridTiler(13), NOOP_BREAKER);
+            GeoShapeCellValues values = new GeoShapeCellValues(makeGeoShapeValues(value), getGridTiler(13), NOOP_BREAKER);
             assertTrue(values.advanceExact(0));
             assertThat(values.docValueCount(), equalTo(1));
         }
         {
-            GeoShapeCellValues values = new GeoShapeCellValues(makeGeoShapeValues(value), new UnboundedGeoTileGridTiler(14), NOOP_BREAKER);
+            GeoShapeCellValues values = new GeoShapeCellValues(makeGeoShapeValues(value), getGridTiler(14), NOOP_BREAKER);
             assertTrue(values.advanceExact(0));
             assertThat(values.docValueCount(), equalTo(4));
         }
         {
-            GeoShapeCellValues values = new GeoShapeCellValues(makeGeoShapeValues(value), new UnboundedGeoTileGridTiler(15), NOOP_BREAKER);
+            GeoShapeCellValues values = new GeoShapeCellValues(makeGeoShapeValues(value), getGridTiler(15), NOOP_BREAKER);
             assertTrue(values.advanceExact(0));
             assertThat(values.docValueCount(), equalTo(16));
         }
@@ -227,7 +222,7 @@ public void testMaxCellsBoundedWithAnotherCell() {
                 new GeoPoint(tile.getMinLat(), tile.getMaxLon())
             );
             int otherPrecision = randomIntBetween(i, maxPrecision());
-            GeoGridTiler tiler = getBoundedGridTiler(boundingBox, otherPrecision);
+            GeoGridTiler tiler = getGridTiler(boundingBox, otherPrecision);
             assertThat(tiler.getMaxCells(), equalTo(getCellsForDiffPrecision(otherPrecision - i)));
         }
     }
@@ -238,11 +233,7 @@ public void testBoundGridOutOfRange() throws Exception {
         double lat = GeoTestUtil.nextLatitude();
         GeoShapeValues.GeoShapeValue value = geoShapeValue(new Point(lon, lat));
         for (int i = 0; i < maxPrecision(); i++) {
-            GeoShapeCellValues values = new GeoShapeCellValues(
-                makeGeoShapeValues(value),
-                getBoundedGridTiler(boundingBox, i),
-                NOOP_BREAKER
-            );
+            GeoShapeCellValues values = new GeoShapeCellValues(makeGeoShapeValues(value), getGridTiler(boundingBox, i), NOOP_BREAKER);
             assertTrue(values.advanceExact(0));
             int numTiles = values.docValueCount();
             assertThat(numTiles, equalTo(0));
@@ -275,7 +266,7 @@ public void testTilerMatchPoint() throws Exception {
             GeoShapeValues.GeoShapeValue value = geoShapeValue(point);
             GeoShapeCellValues unboundedCellValues = new GeoShapeCellValues(
                 makeGeoShapeValues(value),
-                new UnboundedGeoTileGridTiler(precision),
+                getGridTiler(precision),
                 NOOP_BREAKER
             );
             assertTrue(unboundedCellValues.advanceExact(0));
@@ -293,7 +284,7 @@ public void testMultiPointOutOfBounds() throws Exception {
         // points are out of bounds, should not generate any bucket
         MultiPoint points = new MultiPoint(List.of(new Point(0, maxLat), new Point(0, minLat)));
         final GeoShapeValues.GeoShapeValue value = geoShapeValue(points);
-        final GeoGridTiler tiler = getUnboundedGridTiler(0);
+        final GeoGridTiler tiler = getGridTiler(0);
         final GeoShapeCellValues values = new GeoShapeCellValues(makeGeoShapeValues(value), tiler, NOOP_BREAKER);
         assertTrue(values.advanceExact(0));
         assertThat(values.docValueCount(), equalTo(0));
diff --git a/x-pack/plugin/sql/build.gradle b/x-pack/plugin/sql/build.gradle
index 2cc09acabeec..60365d2aa2b8 100644
--- a/x-pack/plugin/sql/build.gradle
+++ b/x-pack/plugin/sql/build.gradle
@@ -11,7 +11,6 @@ esplugin {
 }
 
 ext {
-  antlrVersion = "4.9.2"
   // SQL dependency versions
   jlineVersion = "3.21.0"
 
@@ -33,7 +32,6 @@ dependencies {
   compileOnly(project(':modules:lang-painless:spi'))
   api project('sql-action')
   api project(':modules:aggregations')
-  api project(':modules:aggs-matrix-stats')
   compileOnly project(path: xpackModule('ql'))
   testImplementation project(':test:framework')
   testImplementation(testArtifact(project(xpackModule('core'))))
@@ -64,7 +62,7 @@ configurations {
 }
 
 dependencies {
-  regenerate "org.antlr:antlr4:${antlrVersion}"
+  regenerate "org.antlr:antlr4:${versions.antlr4}"
 }
 
 String grammarPath = 'src/main/antlr'
diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatementTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatementTests.java
index 4e87009c5684..dd59b63b3a00 100644
--- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatementTests.java
+++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatementTests.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.sql.jdbc;
 
 import org.elasticsearch.common.logging.LoggerMessageFormat;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 
 import java.math.BigInteger;
@@ -208,7 +209,7 @@ public void testThrownExceptionsWhenSettingIntegerValues() throws SQLException {
         int someInt = randomInt();
 
         SQLException sqle = expectThrows(SQLException.class, () -> jps.setObject(1, someInt, Types.TIMESTAMP));
-        assertEquals(formatted("Unable to convert value [%.128s] of type [INTEGER] to [TIMESTAMP]", someInt), sqle.getMessage());
+        assertEquals(Strings.format("Unable to convert value [%.128s] of type [INTEGER] to [TIMESTAMP]", someInt), sqle.getMessage());
 
         Integer randomIntNotShort = randomIntBetween(32768, Integer.MAX_VALUE);
         sqle = expectThrows(SQLException.class, () -> jps.setObject(1, randomIntNotShort, Types.SMALLINT));
@@ -298,7 +299,7 @@ public void testThrownExceptionsWhenSettingLongValues() throws SQLException {
         long someLong = randomLong();
 
         SQLException sqle = expectThrows(SQLException.class, () -> jps.setObject(1, someLong, Types.TIMESTAMP));
-        assertEquals(formatted("Unable to convert value [%.128s] of type [LONG] to [TIMESTAMP]", someLong), sqle.getMessage());
+        assertEquals(Strings.format("Unable to convert value [%.128s] of type [LONG] to [TIMESTAMP]", someLong), sqle.getMessage());
 
         Long randomLongNotShort = randomLongBetween(Integer.MAX_VALUE + 1, Long.MAX_VALUE);
         sqle = expectThrows(SQLException.class, () -> jps.setObject(1, randomLongNotShort, Types.INTEGER));
@@ -339,7 +340,7 @@ public void testThrownExceptionsWhenSettingFloatValues() throws SQLException {
         float someFloat = randomFloat();
 
         SQLException sqle = expectThrows(SQLException.class, () -> jps.setObject(1, someFloat, Types.TIMESTAMP));
-        assertEquals(formatted("Unable to convert value [%.128s] of type [FLOAT] to [TIMESTAMP]", someFloat), sqle.getMessage());
+        assertEquals(Strings.format("Unable to convert value [%.128s] of type [FLOAT] to [TIMESTAMP]", someFloat), sqle.getMessage());
 
         Float floatNotInt = 5_155_000_000f;
         sqle = expectThrows(SQLException.class, () -> jps.setObject(1, floatNotInt, Types.INTEGER));
@@ -376,7 +377,7 @@ public void testThrownExceptionsWhenSettingDoubleValues() throws SQLException {
         double someDouble = randomDouble();
 
         SQLException sqle = expectThrows(SQLException.class, () -> jps.setObject(1, someDouble, Types.TIMESTAMP));
-        assertEquals(formatted("Unable to convert value [%.128s] of type [DOUBLE] to [TIMESTAMP]", someDouble), sqle.getMessage());
+        assertEquals(Strings.format("Unable to convert value [%.128s] of type [DOUBLE] to [TIMESTAMP]", someDouble), sqle.getMessage());
 
         Double doubleNotInt = 5_155_000_000d;
         sqle = expectThrows(SQLException.class, () -> jps.setObject(1, doubleNotInt, Types.INTEGER));
diff --git a/x-pack/plugin/sql/qa/jdbc/build.gradle b/x-pack/plugin/sql/qa/jdbc/build.gradle
index ad6e71dce594..65d074538361 100644
--- a/x-pack/plugin/sql/qa/jdbc/build.gradle
+++ b/x-pack/plugin/sql/qa/jdbc/build.gradle
@@ -52,7 +52,7 @@ subprojects {
 
   if (project.name != 'security') {
     // The security project just configures its subprojects
-    apply plugin: 'elasticsearch.internal-java-rest-test'
+    apply plugin: 'elasticsearch.legacy-java-rest-test'
 
     dependencies {
       javaRestTestImplementation(project(':x-pack:plugin:sql:qa:jdbc'))
diff --git a/x-pack/plugin/sql/qa/jdbc/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/jdbc/single_node/JdbcShardFailureIT.java b/x-pack/plugin/sql/qa/jdbc/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/jdbc/single_node/JdbcShardFailureIT.java
index dd92d96e6838..f83047411f0b 100644
--- a/x-pack/plugin/sql/qa/jdbc/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/jdbc/single_node/JdbcShardFailureIT.java
+++ b/x-pack/plugin/sql/qa/jdbc/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/jdbc/single_node/JdbcShardFailureIT.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.sql.qa.jdbc.single_node;
 
 import org.elasticsearch.client.Request;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.xpack.sql.qa.jdbc.JdbcIntegrationTestCase;
 import org.junit.Before;
 
@@ -63,7 +64,7 @@ public void createTestIndex() throws IOException {
         request.addParameter("refresh", "true");
         StringBuilder bulk = new StringBuilder();
         for (int i = 0; i < 20; i++) {
-            bulk.append(formatted("""
+            bulk.append(Strings.format("""
                 {"index":{}}
                 {"test_field":%s}
                 """, i));
diff --git a/x-pack/plugin/sql/qa/mixed-node/build.gradle b/x-pack/plugin/sql/qa/mixed-node/build.gradle
index 73d02ef3c516..d7fa587dfc40 100644
--- a/x-pack/plugin/sql/qa/mixed-node/build.gradle
+++ b/x-pack/plugin/sql/qa/mixed-node/build.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.bwc-test'
 
 import org.elasticsearch.gradle.VersionProperties
diff --git a/x-pack/plugin/sql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/mixed_node/SqlCompatIT.java b/x-pack/plugin/sql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/mixed_node/SqlCompatIT.java
index c5eaa27c52c9..e7e4132052c8 100644
--- a/x-pack/plugin/sql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/mixed_node/SqlCompatIT.java
+++ b/x-pack/plugin/sql/qa/mixed-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/mixed_node/SqlCompatIT.java
@@ -157,7 +157,7 @@ private void assertCursorNotCompatibleAcrossVersions(Version version1, RestClien
         assertThat(cursor, Matchers.not(Matchers.emptyOrNullString()));
 
         Request scrollReq = new Request("POST", "_sql");
-        scrollReq.setJsonEntity(formatted("{\"cursor\": \"%s\"}", cursor));
+        scrollReq.setJsonEntity(Strings.format("{\"cursor\": \"%s\"}", cursor));
         ResponseException exception = expectThrows(ResponseException.class, () -> client2.performRequest(scrollReq));
 
         assertThat(
diff --git a/x-pack/plugin/sql/qa/server/build.gradle b/x-pack/plugin/sql/qa/server/build.gradle
index 3e8903a97965..56ee4ef971f4 100644
--- a/x-pack/plugin/sql/qa/server/build.gradle
+++ b/x-pack/plugin/sql/qa/server/build.gradle
@@ -41,7 +41,7 @@ subprojects {
 
   if (project.name != 'security') {
     // The security project just configures its subprojects
-    apply plugin: 'elasticsearch.internal-java-rest-test'
+    apply plugin: 'elasticsearch.legacy-java-rest-test'
 
     testClusters.matching { it.name == "javaRestTest" }.configureEach {
       testDistribution = 'DEFAULT'
diff --git a/x-pack/plugin/sql/qa/server/multi-cluster-with-security/build.gradle b/x-pack/plugin/sql/qa/server/multi-cluster-with-security/build.gradle
index 6755dd15badb..b56d8ff21199 100644
--- a/x-pack/plugin/sql/qa/server/multi-cluster-with-security/build.gradle
+++ b/x-pack/plugin/sql/qa/server/multi-cluster-with-security/build.gradle
@@ -1,6 +1,6 @@
 import org.elasticsearch.gradle.testclusters.DefaultTestClustersTask
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 dependencies {
   javaRestTestImplementation project(path: xpackModule('ql:test-fixtures'))
diff --git a/x-pack/plugin/sql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/multi_node/RestSqlMultinodeIT.java b/x-pack/plugin/sql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/multi_node/RestSqlMultinodeIT.java
index 5722c711c004..37e19fe428b4 100644
--- a/x-pack/plugin/sql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/multi_node/RestSqlMultinodeIT.java
+++ b/x-pack/plugin/sql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/multi_node/RestSqlMultinodeIT.java
@@ -93,7 +93,7 @@ private void createTestData(int documents) throws UnsupportedCharsetException, I
             int a = 3 * i;
             int b = a + 1;
             int c = b + 1;
-            bulk.append(formatted("""
+            bulk.append(Strings.format("""
                 {"index":{"_id":"%s"}}
                 {"a": %s, "b": %s, "c": %s}
                 """, i, a, b, c));
diff --git a/x-pack/plugin/sql/qa/server/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/SqlSecurityTestCase.java b/x-pack/plugin/sql/qa/server/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/SqlSecurityTestCase.java
index e48ea08b7147..7fd65a19b090 100644
--- a/x-pack/plugin/sql/qa/server/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/SqlSecurityTestCase.java
+++ b/x-pack/plugin/sql/qa/server/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/SqlSecurityTestCase.java
@@ -402,7 +402,7 @@ public void testShowTablesWithLimitedAccessUnaccessableIndex() throws Exception
 
         actions.expectMatchesAdmin("SHOW TABLES LIKE 'not-created'", "read_bort", "SHOW TABLES LIKE 'test'");
         createAuditLogAsserter().expect(true, SQL_ACTION_NAME, "test_admin", empty())
-            .expect(true, GetIndexAction.NAME, "test_admin", contains("*", "-*"))
+            .expect(true, GetIndexAction.NAME, "test_admin", contains("not-created"))
             .expect(true, SQL_ACTION_NAME, "read_bort", empty())
             .expect(true, GetIndexAction.NAME, "read_bort", contains("*", "-*"))
             .assertLogs();
diff --git a/x-pack/plugin/sql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/single_node/ConsistentFunctionArgHandlingIT.java b/x-pack/plugin/sql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/single_node/ConsistentFunctionArgHandlingIT.java
index c49cc5b4cf01..a3f966e712b2 100644
--- a/x-pack/plugin/sql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/single_node/ConsistentFunctionArgHandlingIT.java
+++ b/x-pack/plugin/sql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/single_node/ConsistentFunctionArgHandlingIT.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.PathUtils;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.xpack.sql.qa.jdbc.JdbcIntegrationTestCase;
 
@@ -22,7 +23,6 @@
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -204,7 +204,7 @@ public void test() throws Exception {
                 if (results.stream().map(Tuple::v2).distinct().count() > 1) {
                     int maxResultWidth = results.stream().map(Tuple::v2).mapToInt(o -> asLiteralInQuery(o).length()).max().orElse(20);
                     String resultsAsString = results.stream()
-                        .map(r -> String.format(Locale.ROOT, "%2$-" + maxResultWidth + "s // %1$s", r.v1(), asLiteralInQuery(r.v2())))
+                        .map(r -> Strings.format("%2$-" + maxResultWidth + "s // %1$s", r.v1(), asLiteralInQuery(r.v2())))
                         .collect(joining("\n"));
                     fail("The result of the last call differs from the other calls:\n" + resultsAsString);
                 }
diff --git a/x-pack/plugin/sql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcShardFailureIT.java b/x-pack/plugin/sql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcShardFailureIT.java
index daeb2891c240..492971cf2a13 100644
--- a/x-pack/plugin/sql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcShardFailureIT.java
+++ b/x-pack/plugin/sql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcShardFailureIT.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.sql.qa.single_node;
 
 import org.elasticsearch.client.Request;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.xpack.sql.qa.jdbc.JdbcIntegrationTestCase;
 
 import java.io.IOException;
@@ -15,7 +16,6 @@
 import java.sql.SQLException;
 import java.sql.SQLWarning;
 import java.sql.Statement;
-import java.util.Locale;
 import java.util.Properties;
 
 import static org.hamcrest.Matchers.containsString;
@@ -54,7 +54,7 @@ private void createTestIndex() throws IOException {
         request.addParameter("refresh", "true");
         StringBuilder bulk = new StringBuilder();
         for (int i = 0; i < 20; i++) {
-            bulk.append(formatted("""
+            bulk.append(Strings.format("""
                 {"index":{}}
                 {"test_field":%s}
                 """, i));
@@ -101,7 +101,7 @@ public void testAllowPartialSearchResults() throws Exception {
             String indexName = "/test" + i;
             Request request = new Request("PUT", indexName);
             boolean indexWithDocVals = i < okShards;
-            request.setJsonEntity(String.format(Locale.ROOT, mappingTemplate, indexWithDocVals, indexWithDocVals));
+            request.setJsonEntity(Strings.format(mappingTemplate, indexWithDocVals, indexWithDocVals));
             assertOK(provisioningClient().performRequest(request));
 
             request = new Request("POST", indexName + "/_doc");
diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/CustomDateFormatTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/CustomDateFormatTestCase.java
index 5ba9ed6c816f..861d6f158232 100644
--- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/CustomDateFormatTestCase.java
+++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/CustomDateFormatTestCase.java
@@ -49,7 +49,7 @@ public void testCustomDateFormatsWithNowFunctions() throws IOException {
         for (int i = 0; i < customFormats.length; i++) {
             String field = "date_" + i;
             String format = DateTimeFormatter.ofPattern(customFormats[i], Locale.ROOT).format(DateUtils.nowWithMillisResolution());
-            docs[i] = formatted("""
+            docs[i] = org.elasticsearch.core.Strings.format("""
                 {"%s":"%s"}
                 """, field, format);
             datesConditions.append(i > 0 ? " OR " : "").append(field + randomFrom(operators) + randomFrom(nowFunctions));
diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java
index 8c1d9fa38eab..3f8e7801118e 100644
--- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java
+++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java
@@ -1171,7 +1171,7 @@ private void executeQueryWithNextPage(String format, String expectedHeader, Stri
         int size = 20;
         String[] docs = new String[size];
         for (int i = 0; i < size; i++) {
-            docs[i] = formatted("""
+            docs[i] = org.elasticsearch.core.Strings.format("""
                 {"text":"text%s", "number":%s}
                 """, i, i);
         }
@@ -1219,7 +1219,7 @@ private static void bulkLoadTestData(int count) throws IOException {
         request.addParameter("refresh", "true");
         StringBuilder bulk = new StringBuilder();
         for (int i = 0; i < count; i++) {
-            bulk.append(formatted("""
+            bulk.append(org.elasticsearch.core.Strings.format("""
                 {"index":{"_id":"%s"}}
                 {"text":"text%s", "number":%s}
                 """, i, i, i));
diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlUsageTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlUsageTestCase.java
index 6c697f80a5b2..c979feba781c 100644
--- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlUsageTestCase.java
+++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlUsageTestCase.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xcontent.json.JsonXContent;
 import org.elasticsearch.xpack.sql.proto.Mode;
@@ -300,7 +301,7 @@ private void index(List<IndexDocument> docs) throws IOException {
         request.addParameter("refresh", "true");
         StringBuilder bulk = new StringBuilder();
         for (IndexDocument doc : docs) {
-            bulk.append(formatted("""
+            bulk.append(Strings.format("""
                 {"index":{}}
                 {"condition":"%s","name":"%s","page_count":%s}
                 """, doc.condition, doc.name, doc.pageCount));
diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java
index bbf67200c40c..2caa14aa4029 100644
--- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java
+++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/AbstractSqlQueryRequest.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.sql.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.CompositeIndicesRequest;
 import org.elasticsearch.common.Strings;
@@ -429,14 +429,14 @@ public AbstractSqlQueryRequest(StreamInput in) throws IOException {
         query = in.readString();
         params = in.readList(AbstractSqlQueryRequest::readSqlTypedParamValue);
         zoneId = in.readZoneId();
-        if (in.getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             catalog = in.readOptionalString();
         }
         fetchSize = in.readVInt();
         requestTimeout = in.readTimeValue();
         pageTimeout = in.readTimeValue();
         filter = in.readOptionalNamedWriteable(QueryBuilder.class);
-        if (in.getVersion().onOrAfter(Version.V_7_13_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0)) {
             runtimeMappings = in.readMap();
         }
     }
@@ -457,14 +457,14 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(query);
         out.writeCollection(params, AbstractSqlQueryRequest::writeSqlTypedParamValue);
         out.writeZoneId(zoneId);
-        if (out.getVersion().onOrAfter(Version.V_7_16_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_16_0)) {
             out.writeOptionalString(catalog);
         }
         out.writeVInt(fetchSize);
         out.writeTimeValue(requestTimeout);
         out.writeTimeValue(pageTimeout);
         out.writeOptionalNamedWriteable(filter);
-        if (out.getVersion().onOrAfter(Version.V_7_13_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_13_0)) {
             out.writeGenericMap(runtimeMappings);
         }
     }
diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java
index ee2902992deb..5369be6f2c0f 100644
--- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java
+++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.sql.action;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -160,12 +160,12 @@ public SqlQueryRequest(StreamInput in) throws IOException {
         fieldMultiValueLeniency = in.readBoolean();
         indexIncludeFrozen = in.readBoolean();
         binaryCommunication = in.readOptionalBoolean();
-        if (in.getVersion().onOrAfter(Version.V_7_14_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_14_0)) {
             this.waitForCompletionTimeout = in.readOptionalTimeValue();
             this.keepOnCompletion = in.readBoolean();
             this.keepAlive = in.readOptionalTimeValue();
         }
-        allowPartialSearchResults = in.getVersion().onOrAfter(Version.V_8_3_0) && in.readBoolean();
+        allowPartialSearchResults = in.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0) && in.readBoolean();
     }
 
     /**
@@ -294,12 +294,12 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(fieldMultiValueLeniency);
         out.writeBoolean(indexIncludeFrozen);
         out.writeOptionalBoolean(binaryCommunication);
-        if (out.getVersion().onOrAfter(Version.V_7_14_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_7_14_0)) {
             out.writeOptionalTimeValue(waitForCompletionTimeout);
             out.writeBoolean(keepOnCompletion);
             out.writeOptionalTimeValue(keepAlive);
         }
-        if (out.getVersion().onOrAfter(Version.V_8_3_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0)) {
             out.writeBoolean(allowPartialSearchResults);
         }
     }
diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryResponse.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryResponse.java
index 711995382936..9ea0e4b0e889 100644
--- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryResponse.java
+++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryResponse.java
@@ -8,7 +8,7 @@
 
 import com.fasterxml.jackson.core.JsonGenerator;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -83,7 +83,7 @@ public SqlQueryResponse(StreamInput in) throws IOException {
             }
         }
         this.rows = unmodifiableList(rows);
-        if (in.getVersion().onOrAfter(Version.V_7_14_0)) {
+        if (in.getTransportVersion().onOrAfter(TransportVersion.V_7_14_0)) {
             columnar = in.readBoolean();
             asyncExecutionId = in.readOptionalString();
             isPartial = in.readBoolean();
diff --git a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlClearCursorRequestTests.java b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlClearCursorRequestTests.java
index cd641cc0827c..3880f4b5f7fb 100644
--- a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlClearCursorRequestTests.java
+++ b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlClearCursorRequestTests.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.sql.proto.RequestInfo;
 import org.junit.Before;
 
-import java.io.IOException;
 import java.util.function.Consumer;
 
 import static org.elasticsearch.xpack.sql.proto.RequestInfo.CLIENT_IDS;
@@ -56,7 +55,7 @@ private RequestInfo randomRequestInfo() {
     }
 
     @Override
-    protected TestSqlClearCursorRequest mutateInstance(TestSqlClearCursorRequest instance) throws IOException {
+    protected TestSqlClearCursorRequest mutateInstance(TestSqlClearCursorRequest instance) {
         @SuppressWarnings("unchecked")
         Consumer<TestSqlClearCursorRequest> mutator = randomFrom(
             request -> request.requestInfo(randomValueOtherThan(request.requestInfo(), this::randomRequestInfo)),
diff --git a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java
index 129817b53d42..78c4ebf37877 100644
--- a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java
+++ b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java
@@ -60,6 +60,11 @@ protected SqlQueryResponse createTestInstance() {
         );
     }
 
+    @Override
+    protected SqlQueryResponse mutateInstance(SqlQueryResponse instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Writeable.Reader<SqlQueryResponse> instanceReader() {
         return SqlQueryResponse::new;
diff --git a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlRequestParsersTests.java b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlRequestParsersTests.java
index 77898a8c02a8..871e559eedf7 100644
--- a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlRequestParsersTests.java
+++ b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlRequestParsersTests.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.sql.action;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.XContentParseException;
@@ -57,7 +58,7 @@ public void testClearCursorRequestParser() throws IOException {
             {"cursor" : "whatever", "fetch_size":123}""", "unknown field [fetch_size]", SqlClearCursorRequest::fromXContent);
         Mode randomMode = randomFrom(Mode.values());
 
-        SqlClearCursorRequest request = generateRequest(formatted("""
+        SqlClearCursorRequest request = generateRequest(Strings.format("""
             {
               "cursor": "whatever",
               "mode": "%s",
@@ -72,7 +73,7 @@ public void testClearCursorRequestParser() throws IOException {
         assertTrue(request.binaryCommunication());
 
         randomMode = randomFrom(Mode.values());
-        request = generateRequest(formatted("""
+        request = generateRequest(Strings.format("""
             {
               "cursor": "whatever",
               "mode": "%s",
@@ -117,7 +118,7 @@ public void testTranslateRequestParser() throws IOException {
         assertEquals(Mode.PLAIN, request.mode());
 
         Mode randomMode = randomFrom(Mode.values());
-        request = generateRequest(formatted("""
+        request = generateRequest(Strings.format("""
             {
               "query": "whatever",
               "client_id": "foo",
@@ -162,7 +163,7 @@ public void testQueryRequestParser() throws IOException {
             list.add(new SqlTypedParamValue("integer", 123, false));
         }
 
-        SqlQueryRequest request = generateRequest(formatted("""
+        SqlQueryRequest request = generateRequest(Strings.format("""
             {
               "cursor": "whatever",
               "mode": "%s",
@@ -190,7 +191,7 @@ public void testQueryRequestParser() throws IOException {
 
     public void testParamsSuccessfulParsingInDriverMode() throws IOException {
         Mode driverMode = randomValueOtherThanMany((m) -> Mode.isDriver(m) == false, () -> randomFrom(Mode.values()));
-        String json = formatted("""
+        String json = Strings.format("""
             {
               "params": [
                 {
@@ -235,7 +236,7 @@ public void testParamsSuccessfulParsingInDriverMode() throws IOException {
 
     public void testParamsSuccessfulParsingInNonDriverMode() throws IOException {
         Mode nonDriverMode = randomValueOtherThanMany(Mode::isDriver, () -> randomFrom(Mode.values()));
-        String json = formatted("""
+        String json = Strings.format("""
             {
               "params": [ 35000, "1960-01-01", false, "foo" ],
               "mode": "%s"
@@ -263,17 +264,17 @@ public void testParamsSuccessfulParsingInNonDriverMode() throws IOException {
 
     public void testParamsParsingFailure_QueryRequest_NonDriver() throws IOException {
         Mode m = randomValueOtherThanMany(Mode::isDriver, () -> randomFrom(Mode.values()));
-        assertXContentParsingErrorMessage(formatted("""
+        assertXContentParsingErrorMessage(Strings.format("""
             {
               "params": [ { "whatever": 35000 }, "1960-01-01", false, "foo" ],
               "mode": "%s"
             }""", m), "[sql/query] failed to parse field [params]", SqlQueryRequest::fromXContent);
-        assertXContentParsingErrorMessage(formatted("""
+        assertXContentParsingErrorMessage(Strings.format("""
             {
               "params": [ 350.123, "1960-01-01", { "foobar": false }, "foo" ],
               "mode": "}%s"
             }""", m), "[sql/query] failed to parse field [params]", SqlQueryRequest::fromXContent);
-        assertXContentParsingErrorMessage(formatted("""
+        assertXContentParsingErrorMessage(Strings.format("""
             {
               "mode": "%s",
               "params": [ 350.123, "1960-01-01", false, { "type": "keyword", "value": "foo" } ]
@@ -282,18 +283,18 @@ public void testParamsParsingFailure_QueryRequest_NonDriver() throws IOException
 
     public void testParamsParsingFailure_TranslateRequest_NonDriver() throws IOException {
         Mode m = randomValueOtherThanMany(Mode::isDriver, () -> randomFrom(Mode.values()));
-        assertXContentParsingErrorMessage(formatted("""
+        assertXContentParsingErrorMessage(Strings.format("""
             {
               "params": [ { "whatever": 35000 }, "1960-01-01", false, "foo" ],
               "mode": "%s"
             }""", m), "[sql/query] failed to parse field [params]", SqlTranslateRequest::fromXContent);
-        assertXContentParsingErrorMessage(formatted("""
+        assertXContentParsingErrorMessage(Strings.format("""
             {
               "params": [ 350.123, "1960-01-01", { "foobar": false }, "foo" ],
               "mode": "}%s"
             }""", m), "[sql/query] failed to parse field [params]", SqlTranslateRequest::fromXContent);
         assertXContentParsingErrorMessage(
-            formatted("""
+            Strings.format("""
                 {
                   "mode": "%s",
                   "params": [ 350.123, "1960-01-01", false, { "type": "keyword", "value": "foo" } ]
@@ -305,7 +306,7 @@ public void testParamsParsingFailure_TranslateRequest_NonDriver() throws IOExcep
 
     public void testParamsParsingFailure_Driver() throws IOException {
         Mode m = randomValueOtherThanMany((t) -> Mode.isDriver(t) == false, () -> randomFrom(Mode.values()));
-        assertXContentParsingErrorMessage(formatted("""
+        assertXContentParsingErrorMessage(Strings.format("""
             {
               "params": [
                 35000,
@@ -320,7 +321,7 @@ public void testParamsParsingFailure_Driver() throws IOException {
               ],
               "mode": "%s"
             }""", m), "[params] must be an array where each entry is an object with a value/type pair", SqlQueryRequest::fromXContent);
-        assertXContentParsingErrorMessage(formatted("""
+        assertXContentParsingErrorMessage(Strings.format("""
             {
               "params": [
                 {
@@ -336,7 +337,7 @@ public void testParamsParsingFailure_Driver() throws IOException {
               ],
               "mode": "%s"
             }""", m), "[params] must be an array where each entry is an object with a value/type pair", SqlQueryRequest::fromXContent);
-        assertXContentParsingErrorMessage(formatted("""
+        assertXContentParsingErrorMessage(Strings.format("""
             {
               "mode": "%s",
               "params": [
diff --git a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequestTests.java b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequestTests.java
index d4ef5ae29fac..34ee5cccf50c 100644
--- a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequestTests.java
+++ b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequestTests.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.xpack.sql.proto.RequestInfo;
 import org.junit.Before;
 
-import java.io.IOException;
 import java.util.function.Consumer;
 
 import static java.util.Collections.emptyList;
@@ -85,7 +84,7 @@ protected TestSqlTranslateRequest doParseInstance(XContentParser parser) {
     }
 
     @Override
-    protected TestSqlTranslateRequest mutateInstance(TestSqlTranslateRequest instance) throws IOException {
+    protected TestSqlTranslateRequest mutateInstance(TestSqlTranslateRequest instance) {
         @SuppressWarnings("unchecked")
         Consumer<SqlTranslateRequest> mutator = randomFrom(
             request -> request.query(randomValueOtherThan(request.query(), () -> randomAlphaOfLength(5))),
diff --git a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/RemoteFailureTests.java b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/RemoteFailureTests.java
index 7750a527e647..d093332e4842 100644
--- a/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/RemoteFailureTests.java
+++ b/x-pack/plugin/sql/sql-client/src/test/java/org/elasticsearch/xpack/sql/client/RemoteFailureTests.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.sql.client;
 
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -154,9 +155,10 @@ public void testTooBig() {
             """);
         int i = 0;
         while (tooBig.length() < RemoteFailure.MAX_RAW_RESPONSE) {
-            tooBig.append(formatted("""
+            Object[] args = new Object[] { i++ };
+            tooBig.append(Strings.format("""
                 "%04d" : "lots and lots and lots and lots and lots of words",
-                """, i++));
+                """, args));
         }
         tooBig.append("""
               "end" : "lots and lots and lots and lots and lots of words"
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java
index 14a25fc2afb4..510c4f6a6856 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.xpack.ql.analyzer.AnalyzerRules.AddMissingEqualsToBoolField;
+import org.elasticsearch.xpack.ql.analyzer.AnalyzerRules.ParameterizedAnalyzerRule;
 import org.elasticsearch.xpack.ql.capabilities.Resolvables;
 import org.elasticsearch.xpack.ql.common.Failure;
 import org.elasticsearch.xpack.ql.expression.Alias;
@@ -26,9 +27,6 @@
 import org.elasticsearch.xpack.ql.expression.UnresolvedAlias;
 import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.ql.expression.UnresolvedStar;
-import org.elasticsearch.xpack.ql.expression.function.Function;
-import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition;
-import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
 import org.elasticsearch.xpack.ql.expression.function.FunctionResolutionStrategy;
 import org.elasticsearch.xpack.ql.expression.function.Functions;
 import org.elasticsearch.xpack.ql.expression.function.UnresolvedFunction;
@@ -43,11 +41,10 @@
 import org.elasticsearch.xpack.ql.plan.logical.Project;
 import org.elasticsearch.xpack.ql.plan.logical.UnaryPlan;
 import org.elasticsearch.xpack.ql.plan.logical.UnresolvedRelation;
+import org.elasticsearch.xpack.ql.rule.ParameterizedRuleExecutor;
 import org.elasticsearch.xpack.ql.rule.RuleExecutor;
 import org.elasticsearch.xpack.ql.type.DataType;
 import org.elasticsearch.xpack.ql.type.DataTypes;
-import org.elasticsearch.xpack.ql.type.InvalidMappedField;
-import org.elasticsearch.xpack.ql.type.UnsupportedEsField;
 import org.elasticsearch.xpack.ql.util.CollectionUtils;
 import org.elasticsearch.xpack.ql.util.Holder;
 import org.elasticsearch.xpack.sql.expression.SubQueryExpression;
@@ -58,7 +55,6 @@
 import org.elasticsearch.xpack.sql.plan.logical.Pivot;
 import org.elasticsearch.xpack.sql.plan.logical.SubQueryAlias;
 import org.elasticsearch.xpack.sql.plan.logical.With;
-import org.elasticsearch.xpack.sql.session.SqlConfiguration;
 import org.elasticsearch.xpack.sql.type.SqlDataTypeConverter;
 
 import java.util.ArrayList;
@@ -77,38 +73,17 @@
 import static java.util.stream.Collectors.toList;
 import static org.elasticsearch.xpack.ql.analyzer.AnalyzerRules.AnalyzerRule;
 import static org.elasticsearch.xpack.ql.analyzer.AnalyzerRules.BaseAnalyzerRule;
+import static org.elasticsearch.xpack.ql.analyzer.AnalyzerRules.maybeResolveAgainstList;
+import static org.elasticsearch.xpack.ql.analyzer.AnalyzerRules.resolveFunction;
 import static org.elasticsearch.xpack.ql.util.CollectionUtils.combine;
 
-public class Analyzer extends RuleExecutor<LogicalPlan> {
-    /**
-     * Valid functions.
-     */
-    private final FunctionRegistry functionRegistry;
-    /**
-     * Information about the index against which the SQL is being analyzed.
-     */
-    private final IndexResolution indexResolution;
-    /**
-     * Per-request specific settings needed in some of the functions (timezone, username and clustername),
-     * to which they are attached.
-     */
-    private final SqlConfiguration configuration;
-    /**
-     * The verifier has the role of checking the analyzed tree for failures and build a list of failures.
-     */
-    private final Verifier verifier;
+public class Analyzer extends ParameterizedRuleExecutor<LogicalPlan, AnalyzerContext> {
 
-    public Analyzer(SqlConfiguration configuration, FunctionRegistry functionRegistry, IndexResolution results, Verifier verifier) {
-        this.configuration = configuration;
-        this.functionRegistry = functionRegistry;
-        this.indexResolution = results;
-        this.verifier = verifier;
-    }
+    private static final Iterable<RuleExecutor.Batch<LogicalPlan>> rules;
 
-    @Override
-    protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
-        Batch substitution = new Batch("Substitution", new CTESubstitution());
-        Batch resolution = new Batch(
+    static {
+        var substitution = new Batch<>("Substitution", new CTESubstitution());
+        var resolution = new Batch<>(
             "Resolution",
             new ResolveTable(),
             new ResolveRefs(),
@@ -123,14 +98,30 @@ protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
             new ResolveAggsInOrderBy()
             // new ImplicitCasting()
         );
-        Batch finish = new Batch(
+        var finish = new Batch<>(
             "Finish Analysis",
             new ReplaceSubQueryAliases(), // Should be run before pruning SubqueryAliases
             new PruneSubQueryAliases(),
             new AddMissingEqualsToBoolField(),
             CleanAliases.INSTANCE
         );
-        return Arrays.asList(substitution, resolution, finish);
+        rules = Arrays.asList(substitution, resolution, finish);
+    }
+
+    /**
+     * The verifier has the role of checking the analyzed tree for failures and build a list of failures.
+     */
+    private final Verifier verifier;
+
+    public Analyzer(AnalyzerContext context, Verifier verifier) {
+        super(context);
+        context.analyzeWithoutVerify().set(this::execute);
+        this.verifier = verifier;
+    }
+
+    @Override
+    protected Iterable<RuleExecutor.Batch<LogicalPlan>> batches() {
+        return rules;
     }
 
     public LogicalPlan analyze(LogicalPlan plan) {
@@ -149,7 +140,7 @@ public ExecutionInfo debugAnalyze(LogicalPlan plan) {
     }
 
     public LogicalPlan verify(LogicalPlan plan) {
-        Collection<Failure> failures = verifier.verify(plan, configuration.version());
+        Collection<Failure> failures = verifier.verify(plan, context().configuration().version());
         if (failures.isEmpty() == false) {
             throw new VerificationException(failures);
         }
@@ -175,90 +166,8 @@ private static Attribute resolveAgainstList(UnresolvedAttribute u, Collection<At
     }
 
     private static Attribute resolveAgainstList(UnresolvedAttribute u, Collection<Attribute> attrList, boolean allowCompound) {
-        List<Attribute> matches = new ArrayList<>();
-
-        // first take into account the qualified version
-        boolean qualified = u.qualifier() != null;
-
-        for (Attribute attribute : attrList) {
-            if (attribute.synthetic() == false) {
-                boolean match = qualified ? Objects.equals(u.qualifiedName(), attribute.qualifiedName()) :
-                // if the field is unqualified
-                // first check the names directly
-                    (Objects.equals(u.name(), attribute.name())
-                        // but also if the qualifier might not be quoted and if there's any ambiguity with nested fields
-                        || Objects.equals(u.name(), attribute.qualifiedName()));
-                if (match) {
-                    matches.add(attribute);
-                }
-            }
-        }
-
-        // none found
-        if (matches.isEmpty()) {
-            return null;
-        }
-
-        if (matches.size() == 1) {
-            // only add the location if the match is univocal; b/c otherwise adding the location will overwrite any preexisting one
-            return handleSpecialFields(u, matches.get(0).withLocation(u.source()), allowCompound);
-        }
-
-        List<String> refs = matches.stream().sorted((a, b) -> {
-            int lineDiff = a.sourceLocation().getLineNumber() - b.sourceLocation().getLineNumber();
-            int colDiff = a.sourceLocation().getColumnNumber() - b.sourceLocation().getColumnNumber();
-            return lineDiff != 0 ? lineDiff : (colDiff != 0 ? colDiff : a.qualifiedName().compareTo(b.qualifiedName()));
-        })
-            .map(
-                a -> "line "
-                    + a.sourceLocation().toString().substring(1)
-                    + " ["
-                    + (a.qualifier() != null ? "\"" + a.qualifier() + "\".\"" + a.name() + "\"" : a.name())
-                    + "]"
-            )
-            .collect(toList());
-        return u.withUnresolvedMessage(
-            "Reference [" + u.qualifiedName() + "] is ambiguous (to disambiguate use quotes or qualifiers); " + "matches any of " + refs
-        );
-    }
-
-    private static Attribute handleSpecialFields(UnresolvedAttribute u, Attribute named, boolean allowCompound) {
-        // if it's a object/compound type, keep it unresolved with a nice error message
-        if (named instanceof FieldAttribute fa) {
-
-            // incompatible mappings
-            if (fa.field() instanceof InvalidMappedField) {
-                named = u.withUnresolvedMessage(
-                    "Cannot use field [" + fa.name() + "] due to ambiguities being " + ((InvalidMappedField) fa.field()).errorMessage()
-                );
-            }
-            // unsupported types
-            else if (DataTypes.isUnsupported(fa.dataType())) {
-                UnsupportedEsField unsupportedField = (UnsupportedEsField) fa.field();
-                if (unsupportedField.hasInherited()) {
-                    named = u.withUnresolvedMessage(
-                        "Cannot use field ["
-                            + fa.name()
-                            + "] with unsupported type ["
-                            + unsupportedField.getOriginalType()
-                            + "] in hierarchy (field ["
-                            + unsupportedField.getInherited()
-                            + "])"
-                    );
-                } else {
-                    named = u.withUnresolvedMessage(
-                        "Cannot use field [" + fa.name() + "] with unsupported type [" + unsupportedField.getOriginalType() + "]"
-                    );
-                }
-            }
-            // compound fields
-            else if (allowCompound == false && DataTypes.isPrimitive(fa.dataType()) == false) {
-                named = u.withUnresolvedMessage(
-                    "Cannot use field [" + fa.name() + "] type [" + fa.dataType().typeName() + "] only its subfields"
-                );
-            }
-        }
-        return named;
+        var matches = maybeResolveAgainstList(u, attrList, allowCompound, false);
+        return matches.isEmpty() ? null : matches.get(0);
     }
 
     private static boolean hasStar(List<? extends Expression> exprs) {
@@ -310,9 +219,10 @@ protected boolean skipResolved() {
         }
     }
 
-    private class ResolveTable extends AnalyzerRule<UnresolvedRelation> {
-        @Override
-        protected LogicalPlan rule(UnresolvedRelation plan) {
+    private static class ResolveTable extends ParameterizedAnalyzerRule<UnresolvedRelation, AnalyzerContext> {
+
+        protected LogicalPlan rule(UnresolvedRelation plan, AnalyzerContext context) {
+            IndexResolution indexResolution = context.indexResolution();
             if (indexResolution.isValid() == false) {
                 return plan.unresolvedMessage().equals(indexResolution.toString())
                     ? plan
@@ -339,7 +249,7 @@ protected LogicalPlan rule(UnresolvedRelation plan) {
         }
     }
 
-    private class ResolveRefs extends BaseAnalyzerRule {
+    private static class ResolveRefs extends BaseAnalyzerRule {
 
         @Override
         protected LogicalPlan doRule(LogicalPlan plan) {
@@ -901,17 +811,14 @@ private Expression replaceAliases(Expression condition, List<? extends NamedExpr
         }
     }
 
-    private class ResolveFunctions extends AnalyzerRule<LogicalPlan> {
+    private static class ResolveFunctions extends ParameterizedAnalyzerRule<LogicalPlan, AnalyzerContext> {
 
         @Override
-        protected LogicalPlan rule(LogicalPlan plan) {
+        protected LogicalPlan rule(LogicalPlan plan, AnalyzerContext context) {
             return plan.transformExpressionsUp(UnresolvedFunction.class, uf -> {
                 if (uf.analyzed()) {
                     return uf;
                 }
-
-                String name = uf.name();
-
                 if (hasStar(uf.arguments())) {
                     FunctionResolutionStrategy strategy = uf.resolutionStrategy();
                     if (SqlFunctionResolution.DISTINCT == strategy) {
@@ -933,18 +840,7 @@ protected LogicalPlan rule(LogicalPlan plan) {
                     }
                 }
 
-                if (uf.childrenResolved() == false) {
-                    return uf;
-                }
-
-                String functionName = functionRegistry.resolveAlias(name);
-                if (functionRegistry.functionExists(functionName) == false) {
-                    return uf.missing(functionName, functionRegistry.listFunctions());
-                }
-                // TODO: look into Generator for significant terms, etc..
-                FunctionDefinition def = functionRegistry.resolveFunction(functionName);
-                Function f = uf.buildResolved(configuration, def);
-                return f;
+                return resolveFunction(uf, context.configuration(), context.functionRegistry());
             });
         }
     }
@@ -1063,7 +959,7 @@ protected boolean skipResolved() {
     // Handle aggs in HAVING. To help folding any aggs not found in Aggregation
     // will be pushed down to the Aggregate and then projected. This also simplifies the Verifier's job.
     //
-    private class ResolveAggsInHaving extends AnalyzerRule<Filter> {
+    private static class ResolveAggsInHaving extends ParameterizedAnalyzerRule<Filter, AnalyzerContext> {
 
         @Override
         protected boolean skipResolved() {
@@ -1071,7 +967,7 @@ protected boolean skipResolved() {
         }
 
         @Override
-        protected LogicalPlan rule(Filter f) {
+        protected LogicalPlan rule(Filter f, AnalyzerContext context) {
             // HAVING = Filter followed by an Agg
             // tag::noformat - https://bugs.eclipse.org/bugs/show_bug.cgi?id=574437
             if (f.child() instanceof Aggregate agg && agg.resolved()) {
@@ -1092,7 +988,8 @@ protected LogicalPlan rule(Filter f) {
                         combine(agg.aggregates(), new Alias(f.source(), ".having", condition))
                     );
 
-                    tryResolvingCondition = (Aggregate) analyze(tryResolvingCondition, false);
+                    var analyze = context.analyzeWithoutVerify().get();
+                    tryResolvingCondition = (Aggregate) analyze.apply(tryResolvingCondition);
 
                     // if it got resolved
                     if (tryResolvingCondition.resolved()) {
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/AnalyzerContext.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/AnalyzerContext.java
new file mode 100644
index 000000000000..8c0ff5e78e94
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/AnalyzerContext.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.sql.analysis.analyzer;
+
+import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
+import org.elasticsearch.xpack.ql.index.IndexResolution;
+import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.ql.util.Holder;
+import org.elasticsearch.xpack.sql.session.SqlConfiguration;
+
+import java.util.function.Function;
+
+public record AnalyzerContext(
+    SqlConfiguration configuration,
+    FunctionRegistry functionRegistry,
+    IndexResolution indexResolution,
+    Holder<Function<LogicalPlan, LogicalPlan>> analyzeWithoutVerify
+) {
+    public AnalyzerContext(SqlConfiguration configuration, FunctionRegistry functionRegistry, IndexResolution indexResolution) {
+        this(configuration, functionRegistry, indexResolution, new Holder<>());
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java
index 8671328ddf1e..cd8c69004835 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java
@@ -216,11 +216,12 @@ protected static void logSearchResponse(SearchResponse response, Logger logger)
             aggsNames.append(aggs.get(i).getName() + (i + 1 == aggs.size() ? "" : ", "));
         }
 
+        var totalHits = response.getHits().getTotalHits();
+        var hits = totalHits != null ? "hits " + totalHits.relation + " " + totalHits.value + ", " : "";
         logger.trace(
-            "Got search response [hits {} {}, {} aggregations: [{}], {} failed shards, {} skipped shards, "
+            "Got search response [{}{} aggregations: [{}], {} failed shards, {} skipped shards, "
                 + "{} successful shards, {} total shards, took {}, timed out [{}]]",
-            response.getHits().getTotalHits().relation.toString(),
-            response.getHits().getTotalHits().value,
+            hits,
             aggs.size(),
             aggsNames,
             response.getFailedShards(),
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SearchHitCursor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SearchHitCursor.java
index b2b2f02f58d9..bad490184c09 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SearchHitCursor.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SearchHitCursor.java
@@ -8,7 +8,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
@@ -69,7 +69,7 @@ public SearchHitCursor(StreamInput in) throws IOException {
         extractors = in.readNamedWriteableList(HitExtractor.class);
         mask = BitSet.valueOf(in.readByteArray());
         includeFrozen = in.readBoolean();
-        allowPartialSearchResults = in.getVersion().onOrAfter(Version.V_8_3_0) && in.readBoolean();
+        allowPartialSearchResults = in.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0) && in.readBoolean();
     }
 
     @Override
@@ -80,7 +80,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeNamedWriteableList(extractors);
         out.writeByteArray(mask.toByteArray());
         out.writeBoolean(includeFrozen);
-        if (out.getVersion().onOrAfter(Version.V_8_3_0)) {
+        if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_3_0)) {
             out.writeBoolean(allowPartialSearchResults);
         }
     }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java
index 010d0fafade1..f44f20a942f7 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java
@@ -6,7 +6,6 @@
  */
 package org.elasticsearch.xpack.sql.execution.search.extractor;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
@@ -55,7 +54,7 @@ public CompositeKeyExtractor(String key, Property property, ZoneId zoneId, DataT
     CompositeKeyExtractor(StreamInput in) throws IOException {
         key = in.readString();
         property = in.readEnum(Property.class);
-        if (in.getVersion().onOrAfter(Version.fromId(INTRODUCING_UNSIGNED_LONG.id))) {
+        if (in.getTransportVersion().onOrAfter(INTRODUCING_UNSIGNED_LONG.transportVersion)) {
             dataType = SqlDataTypes.fromTypeName(in.readString());
         } else {
             // for pre-UNSIGNED_LONG versions, the only relevant fact about the dataType was if this isDateBased() or not.
@@ -69,7 +68,7 @@ public CompositeKeyExtractor(String key, Property property, ZoneId zoneId, DataT
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(key);
         out.writeEnum(property);
-        if (out.getVersion().onOrAfter(Version.fromId(INTRODUCING_UNSIGNED_LONG.id))) {
+        if (out.getTransportVersion().onOrAfter((INTRODUCING_UNSIGNED_LONG.transportVersion))) {
             out.writeString(dataType.typeName());
         } else {
             out.writeBoolean(isDateBased(dataType));
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/MetricAggExtractor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/MetricAggExtractor.java
index eb6288061fca..f76702f5ffe5 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/MetricAggExtractor.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/MetricAggExtractor.java
@@ -6,13 +6,13 @@
  */
 package org.elasticsearch.xpack.sql.execution.search.extractor;
 
+import org.elasticsearch.aggregations.metric.InternalMatrixStats;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
 import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter;
-import org.elasticsearch.search.aggregations.matrix.stats.InternalMatrixStats;
 import org.elasticsearch.search.aggregations.metrics.InternalAvg;
 import org.elasticsearch.search.aggregations.metrics.InternalCardinality;
 import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation;
@@ -35,7 +35,7 @@
 import java.util.Map;
 import java.util.Objects;
 
-import static org.elasticsearch.search.aggregations.matrix.stats.MatrixAggregationInspectionHelper.hasValue;
+import static org.elasticsearch.aggregations.metric.MatrixAggregationInspectionHelper.hasValue;
 import static org.elasticsearch.search.aggregations.support.AggregationInspectionHelper.hasValue;
 import static org.elasticsearch.xpack.sql.type.SqlDataTypeConverter.convert;
 import static org.elasticsearch.xpack.sql.type.SqlDataTypes.isDateBased;
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java
index 617c9226b493..cdfac737e7fe 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java
@@ -121,8 +121,8 @@ public LogicalPlan optimize(LogicalPlan verified) {
     }
 
     @Override
-    protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
-        Batch substitutions = new Batch(
+    protected Iterable<RuleExecutor.Batch<LogicalPlan>> batches() {
+        var substitutions = new Batch<>(
             "Substitutions",
             Limiter.ONCE,
             new RewritePivot(),
@@ -130,9 +130,9 @@ protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
             new ReplaceAggregatesWithLiterals()
         );
 
-        Batch refs = new Batch("Replace References", Limiter.ONCE, new ReplaceReferenceAttributeWithSource());
+        var refs = new Batch<>("Replace References", Limiter.ONCE, new ReplaceReferenceAttributeWithSource());
 
-        Batch operators = new Batch(
+        var operators = new Batch<>(
             "Operator Optimization",
             // combining
             new CombineProjections(),
@@ -166,7 +166,7 @@ protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
             new PushDownAndCombineFilters()
         );
 
-        Batch aggregate = new Batch(
+        var aggregate = new Batch<>(
             "Aggregation Rewrite",
             new ReplaceMinMaxWithTopHits(),
             new ReplaceAggsWithMatrixStats(),
@@ -178,7 +178,7 @@ protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
             new ReplaceAggsWithPercentileRanks()
         );
 
-        Batch local = new Batch(
+        var local = new Batch<>(
             "Skip Elasticsearch",
             new SkipQueryOnLimitZero(),
             new SkipQueryForLiteralAggregations(),
@@ -188,7 +188,7 @@ protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
             new PruneLiteralsInGroupBy()
         );
 
-        Batch label = new Batch("Set as Optimized", Limiter.ONCE, CleanAliases.INSTANCE, new SetAsOptimized());
+        var label = new Batch<>("Set as Optimized", Limiter.ONCE, CleanAliases.INSTANCE, new SetAsOptimized());
 
         return Arrays.asList(substitutions, refs, operators, aggregate, local, label);
     }
@@ -588,7 +588,6 @@ public LogicalPlan apply(LogicalPlan plan) {
             return rule(plan);
         }
 
-        @Override
         protected LogicalPlan rule(LogicalPlan plan) {
             Map<Attribute, Alias> aliases = new LinkedHashMap<>();
             List<Attribute> attrs = new ArrayList<>();
@@ -1262,7 +1261,6 @@ abstract static class OptimizerBasicRule extends Rule<LogicalPlan, LogicalPlan>
         @Override
         public abstract LogicalPlan apply(LogicalPlan plan);
 
-        @Override
         protected LogicalPlan rule(LogicalPlan plan) {
             return plan;
         }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseListener.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseListener.java
index 2c5662dda01f..e9f6af2da767 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseListener.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseListener.java
@@ -10,6 +10,7 @@
  * which can be extended to create a listener which only needs to handle a subset
  * of the available methods.
  */
+@SuppressWarnings("CheckReturnValue")
 class SqlBaseBaseListener implements SqlBaseListener {
     /**
      * {@inheritDoc}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseVisitor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseVisitor.java
index eaa1d81ebcf8..162f4bfb7085 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseVisitor.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseVisitor.java
@@ -11,6 +11,7 @@
  * @param <T> The return type of the visit operation. Use {@link Void} for
  * operations with no return type.
  */
+@SuppressWarnings("CheckReturnValue")
 class SqlBaseBaseVisitor<T> extends AbstractParseTreeVisitor<T> implements SqlBaseVisitor<T> {
     /**
      * {@inheritDoc}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseLexer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseLexer.java
index 9be257085a9a..7ad70c99eee2 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseLexer.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseLexer.java
@@ -8,10 +8,10 @@
 import org.antlr.v4.runtime.dfa.DFA;
 import org.antlr.v4.runtime.misc.*;
 
-@SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast" })
+@SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue" })
 class SqlBaseLexer extends Lexer {
     static {
-        RuntimeMetaData.checkVersion("4.9.2", RuntimeMetaData.VERSION);
+        RuntimeMetaData.checkVersion("4.11.1", RuntimeMetaData.VERSION);
     }
 
     protected static final DFA[] _decisionToDFA;
@@ -527,412 +527,711 @@ public ATN getATN() {
         return _ATN;
     }
 
-    public static final String _serializedATN = "\3\u608b\ua72a\u8133\ub9ed\u417c\u3be7\u7786\u5964\2\u008c\u0489\b\1\4"
-        + "\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n"
-        + "\4\13\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22"
-        + "\t\22\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31"
-        + "\t\31\4\32\t\32\4\33\t\33\4\34\t\34\4\35\t\35\4\36\t\36\4\37\t\37\4 \t"
-        + " \4!\t!\4\"\t\"\4#\t#\4$\t$\4%\t%\4&\t&\4\'\t\'\4(\t(\4)\t)\4*\t*\4+\t"
-        + "+\4,\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\4\61\t\61\4\62\t\62\4\63\t\63\4\64"
-        + "\t\64\4\65\t\65\4\66\t\66\4\67\t\67\48\t8\49\t9\4:\t:\4;\t;\4<\t<\4=\t"
-        + "=\4>\t>\4?\t?\4@\t@\4A\tA\4B\tB\4C\tC\4D\tD\4E\tE\4F\tF\4G\tG\4H\tH\4"
-        + "I\tI\4J\tJ\4K\tK\4L\tL\4M\tM\4N\tN\4O\tO\4P\tP\4Q\tQ\4R\tR\4S\tS\4T\t"
-        + "T\4U\tU\4V\tV\4W\tW\4X\tX\4Y\tY\4Z\tZ\4[\t[\4\\\t\\\4]\t]\4^\t^\4_\t_"
-        + "\4`\t`\4a\ta\4b\tb\4c\tc\4d\td\4e\te\4f\tf\4g\tg\4h\th\4i\ti\4j\tj\4k"
-        + "\tk\4l\tl\4m\tm\4n\tn\4o\to\4p\tp\4q\tq\4r\tr\4s\ts\4t\tt\4u\tu\4v\tv"
-        + "\4w\tw\4x\tx\4y\ty\4z\tz\4{\t{\4|\t|\4}\t}\4~\t~\4\177\t\177\4\u0080\t"
-        + "\u0080\4\u0081\t\u0081\4\u0082\t\u0082\4\u0083\t\u0083\4\u0084\t\u0084"
-        + "\4\u0085\t\u0085\4\u0086\t\u0086\4\u0087\t\u0087\4\u0088\t\u0088\4\u0089"
-        + "\t\u0089\4\u008a\t\u008a\4\u008b\t\u008b\4\u008c\t\u008c\4\u008d\t\u008d"
-        + "\4\u008e\t\u008e\3\2\3\2\3\3\3\3\3\4\3\4\3\5\3\5\3\6\3\6\3\6\3\6\3\7\3"
-        + "\7\3\7\3\7\3\7\3\7\3\7\3\7\3\b\3\b\3\b\3\b\3\b\3\b\3\b\3\b\3\b\3\t\3\t"
-        + "\3\t\3\t\3\n\3\n\3\n\3\n\3\13\3\13\3\13\3\f\3\f\3\f\3\f\3\r\3\r\3\r\3"
-        + "\r\3\r\3\r\3\r\3\r\3\16\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3\20\3\20\3"
-        + "\20\3\20\3\20\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3"
-        + "\22\3\22\3\22\3\22\3\22\3\22\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3"
-        + "\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\25\3\25\3\25\3\25\3\25\3\25\3"
-        + "\25\3\25\3\25\3\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\3\26\3\26\3\26\3"
-        + "\26\3\26\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\3\27\3\27\3\27\3\27\3"
-        + "\27\3\27\3\27\3\27\3\27\3\27\3\27\3\27\3\27\3\27\3\30\3\30\3\30\3\30\3"
-        + "\31\3\31\3\31\3\31\3\31\3\32\3\32\3\32\3\32\3\32\3\32\3\33\3\33\3\33\3"
-        + "\33\3\33\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3\35\3\35\3\35\3"
-        + "\35\3\35\3\35\3\35\3\35\3\35\3\36\3\36\3\36\3\36\3\36\3\37\3\37\3\37\3"
-        + "\37\3 \3 \3 \3 \3 \3 \3 \3!\3!\3!\3!\3!\3!\3!\3!\3!\3!\3!\3\"\3\"\3\""
-        + "\3\"\3\"\3\"\3\"\3#\3#\3#\3#\3#\3#\3#\3#\3$\3$\3$\3$\3$\3$\3$\3$\3%\3"
-        + "%\3%\3%\3%\3%\3&\3&\3&\3&\3&\3&\3\'\3\'\3\'\3\'\3(\3(\3(\3(\3(\3(\3(\3"
-        + ")\3)\3)\3)\3)\3*\3*\3*\3*\3*\3*\3*\3+\3+\3+\3+\3+\3,\3,\3,\3,\3,\3,\3"
-        + ",\3,\3,\3,\3-\3-\3-\3-\3-\3-\3-\3-\3-\3.\3.\3.\3.\3.\3.\3/\3/\3/\3/\3"
-        + "/\3/\3/\3\60\3\60\3\60\3\60\3\60\3\61\3\61\3\61\3\61\3\61\3\61\3\62\3"
-        + "\62\3\62\3\63\3\63\3\63\3\63\3\63\3\63\3\63\3\63\3\64\3\64\3\64\3\64\3"
-        + "\64\3\64\3\65\3\65\3\65\3\65\3\65\3\65\3\65\3\65\3\65\3\66\3\66\3\66\3"
-        + "\67\3\67\3\67\3\67\3\67\38\38\38\38\38\39\39\39\39\39\3:\3:\3:\3:\3:\3"
-        + ";\3;\3;\3;\3;\3;\3<\3<\3<\3<\3<\3<\3<\3=\3=\3=\3=\3=\3=\3>\3>\3>\3>\3"
-        + ">\3>\3>\3?\3?\3?\3?\3?\3?\3?\3?\3@\3@\3@\3@\3@\3@\3A\3A\3A\3A\3A\3A\3"
-        + "A\3B\3B\3B\3B\3B\3B\3B\3B\3C\3C\3C\3C\3D\3D\3D\3D\3D\3E\3E\3E\3E\3E\3"
-        + "E\3F\3F\3F\3G\3G\3G\3G\3G\3G\3G\3G\3G\3G\3H\3H\3H\3I\3I\3I\3I\3I\3I\3"
-        + "J\3J\3J\3J\3J\3J\3K\3K\3K\3K\3K\3K\3K\3L\3L\3L\3L\3L\3L\3L\3L\3L\3M\3"
-        + "M\3M\3M\3M\3M\3N\3N\3N\3N\3N\3O\3O\3O\3O\3O\3O\3P\3P\3P\3P\3P\3P\3Q\3"
-        + "Q\3Q\3Q\3Q\3Q\3R\3R\3R\3R\3R\3R\3R\3R\3S\3S\3S\3S\3S\3S\3S\3T\3T\3T\3"
-        + "T\3T\3T\3T\3T\3U\3U\3U\3U\3U\3U\3U\3V\3V\3V\3V\3V\3W\3W\3W\3W\3X\3X\3"
-        + "X\3X\3X\3X\3Y\3Y\3Y\3Y\3Y\3Y\3Y\3Z\3Z\3Z\3Z\3Z\3[\3[\3[\3[\3[\3\\\3\\"
-        + "\3\\\3\\\3\\\3]\3]\3]\3^\3^\3^\3^\3_\3_\3_\3_\3_\3`\3`\3`\3`\3`\3`\3a"
-        + "\3a\3a\3a\3a\3a\3b\3b\3b\3b\3b\3b\3b\3c\3c\3c\3c\3c\3d\3d\3d\3d\3d\3d"
-        + "\3e\3e\3e\3e\3e\3f\3f\3f\3f\3f\3g\3g\3g\3g\3g\3g\3h\3h\3h\3h\3h\3h\3h"
-        + "\3h\3i\3i\3i\3i\3j\3j\3j\3j\3j\3j\3j\3k\3k\3k\3l\3l\3l\3m\3m\3m\3m\3n"
-        + "\3n\3n\3n\3n\3n\3o\3o\7o\u03b9\no\fo\16o\u03bc\13o\3p\3p\3q\3q\3r\3r\3"
-        + "r\3r\3s\3s\3s\3s\5s\u03ca\ns\3t\3t\3u\3u\3u\3v\3v\3w\3w\3w\3x\3x\3y\3"
-        + "y\3z\3z\3{\3{\3|\3|\3}\3}\3}\3~\3~\3\177\3\177\3\u0080\3\u0080\3\u0080"
-        + "\3\u0080\7\u0080\u03eb\n\u0080\f\u0080\16\u0080\u03ee\13\u0080\3\u0080"
-        + "\3\u0080\3\u0081\6\u0081\u03f3\n\u0081\r\u0081\16\u0081\u03f4\3\u0082"
-        + "\6\u0082\u03f8\n\u0082\r\u0082\16\u0082\u03f9\3\u0082\3\u0082\7\u0082"
-        + "\u03fe\n\u0082\f\u0082\16\u0082\u0401\13\u0082\3\u0082\3\u0082\6\u0082"
-        + "\u0405\n\u0082\r\u0082\16\u0082\u0406\3\u0082\6\u0082\u040a\n\u0082\r"
-        + "\u0082\16\u0082\u040b\3\u0082\3\u0082\7\u0082\u0410\n\u0082\f\u0082\16"
-        + "\u0082\u0413\13\u0082\5\u0082\u0415\n\u0082\3\u0082\3\u0082\3\u0082\3"
-        + "\u0082\6\u0082\u041b\n\u0082\r\u0082\16\u0082\u041c\3\u0082\3\u0082\5"
-        + "\u0082\u0421\n\u0082\3\u0083\3\u0083\5\u0083\u0425\n\u0083\3\u0083\3\u0083"
-        + "\3\u0083\7\u0083\u042a\n\u0083\f\u0083\16\u0083\u042d\13\u0083\3\u0084"
-        + "\3\u0084\3\u0084\3\u0084\6\u0084\u0433\n\u0084\r\u0084\16\u0084\u0434"
-        + "\3\u0085\3\u0085\3\u0085\6\u0085\u043a\n\u0085\r\u0085\16\u0085\u043b"
-        + "\3\u0086\3\u0086\3\u0086\3\u0086\7\u0086\u0442\n\u0086\f\u0086\16\u0086"
-        + "\u0445\13\u0086\3\u0086\3\u0086\3\u0087\3\u0087\3\u0087\3\u0087\7\u0087"
-        + "\u044d\n\u0087\f\u0087\16\u0087\u0450\13\u0087\3\u0087\3\u0087\3\u0088"
-        + "\3\u0088\5\u0088\u0456\n\u0088\3\u0088\6\u0088\u0459\n\u0088\r\u0088\16"
-        + "\u0088\u045a\3\u0089\3\u0089\3\u008a\3\u008a\3\u008b\3\u008b\3\u008b\3"
-        + "\u008b\7\u008b\u0465\n\u008b\f\u008b\16\u008b\u0468\13\u008b\3\u008b\5"
-        + "\u008b\u046b\n\u008b\3\u008b\5\u008b\u046e\n\u008b\3\u008b\3\u008b\3\u008c"
-        + "\3\u008c\3\u008c\3\u008c\3\u008c\7\u008c\u0477\n\u008c\f\u008c\16\u008c"
-        + "\u047a\13\u008c\3\u008c\3\u008c\3\u008c\3\u008c\3\u008c\3\u008d\6\u008d"
-        + "\u0482\n\u008d\r\u008d\16\u008d\u0483\3\u008d\3\u008d\3\u008e\3\u008e"
-        + "\3\u0478\2\u008f\3\3\5\4\7\5\t\6\13\7\r\b\17\t\21\n\23\13\25\f\27\r\31"
-        + "\16\33\17\35\20\37\21!\22#\23%\24\'\25)\26+\27-\30/\31\61\32\63\33\65"
-        + "\34\67\359\36;\37= ?!A\"C#E$G%I&K\'M(O)Q*S+U,W-Y.[/]\60_\61a\62c\63e\64"
-        + "g\65i\66k\67m8o9q:s;u<w=y>{?}@\177A\u0081B\u0083C\u0085D\u0087E\u0089"
-        + "F\u008bG\u008dH\u008fI\u0091J\u0093K\u0095L\u0097M\u0099N\u009bO\u009d"
-        + "P\u009fQ\u00a1R\u00a3S\u00a5T\u00a7U\u00a9V\u00abW\u00adX\u00afY\u00b1"
-        + "Z\u00b3[\u00b5\\\u00b7]\u00b9^\u00bb_\u00bd`\u00bfa\u00c1b\u00c3c\u00c5"
-        + "d\u00c7e\u00c9f\u00cbg\u00cdh\u00cfi\u00d1j\u00d3k\u00d5l\u00d7m\u00d9"
-        + "n\u00dbo\u00ddp\u00dfq\u00e1r\u00e3s\u00e5t\u00e7u\u00e9v\u00ebw\u00ed"
-        + "x\u00efy\u00f1z\u00f3{\u00f5|\u00f7}\u00f9~\u00fb\177\u00fd\u0080\u00ff"
-        + "\u0081\u0101\u0082\u0103\u0083\u0105\u0084\u0107\u0085\u0109\u0086\u010b"
-        + "\u0087\u010d\u0088\u010f\2\u0111\2\u0113\2\u0115\u0089\u0117\u008a\u0119"
-        + "\u008b\u011b\u008c\3\2\13\3\2))\4\2BBaa\3\2$$\3\2bb\4\2--//\3\2\62;\3"
-        + "\2C\\\4\2\f\f\17\17\5\2\13\f\17\17\"\"\2\u04aa\2\3\3\2\2\2\2\5\3\2\2\2"
-        + "\2\7\3\2\2\2\2\t\3\2\2\2\2\13\3\2\2\2\2\r\3\2\2\2\2\17\3\2\2\2\2\21\3"
-        + "\2\2\2\2\23\3\2\2\2\2\25\3\2\2\2\2\27\3\2\2\2\2\31\3\2\2\2\2\33\3\2\2"
-        + "\2\2\35\3\2\2\2\2\37\3\2\2\2\2!\3\2\2\2\2#\3\2\2\2\2%\3\2\2\2\2\'\3\2"
-        + "\2\2\2)\3\2\2\2\2+\3\2\2\2\2-\3\2\2\2\2/\3\2\2\2\2\61\3\2\2\2\2\63\3\2"
-        + "\2\2\2\65\3\2\2\2\2\67\3\2\2\2\29\3\2\2\2\2;\3\2\2\2\2=\3\2\2\2\2?\3\2"
-        + "\2\2\2A\3\2\2\2\2C\3\2\2\2\2E\3\2\2\2\2G\3\2\2\2\2I\3\2\2\2\2K\3\2\2\2"
-        + "\2M\3\2\2\2\2O\3\2\2\2\2Q\3\2\2\2\2S\3\2\2\2\2U\3\2\2\2\2W\3\2\2\2\2Y"
-        + "\3\2\2\2\2[\3\2\2\2\2]\3\2\2\2\2_\3\2\2\2\2a\3\2\2\2\2c\3\2\2\2\2e\3\2"
-        + "\2\2\2g\3\2\2\2\2i\3\2\2\2\2k\3\2\2\2\2m\3\2\2\2\2o\3\2\2\2\2q\3\2\2\2"
-        + "\2s\3\2\2\2\2u\3\2\2\2\2w\3\2\2\2\2y\3\2\2\2\2{\3\2\2\2\2}\3\2\2\2\2\177"
-        + "\3\2\2\2\2\u0081\3\2\2\2\2\u0083\3\2\2\2\2\u0085\3\2\2\2\2\u0087\3\2\2"
-        + "\2\2\u0089\3\2\2\2\2\u008b\3\2\2\2\2\u008d\3\2\2\2\2\u008f\3\2\2\2\2\u0091"
-        + "\3\2\2\2\2\u0093\3\2\2\2\2\u0095\3\2\2\2\2\u0097\3\2\2\2\2\u0099\3\2\2"
-        + "\2\2\u009b\3\2\2\2\2\u009d\3\2\2\2\2\u009f\3\2\2\2\2\u00a1\3\2\2\2\2\u00a3"
-        + "\3\2\2\2\2\u00a5\3\2\2\2\2\u00a7\3\2\2\2\2\u00a9\3\2\2\2\2\u00ab\3\2\2"
-        + "\2\2\u00ad\3\2\2\2\2\u00af\3\2\2\2\2\u00b1\3\2\2\2\2\u00b3\3\2\2\2\2\u00b5"
-        + "\3\2\2\2\2\u00b7\3\2\2\2\2\u00b9\3\2\2\2\2\u00bb\3\2\2\2\2\u00bd\3\2\2"
-        + "\2\2\u00bf\3\2\2\2\2\u00c1\3\2\2\2\2\u00c3\3\2\2\2\2\u00c5\3\2\2\2\2\u00c7"
-        + "\3\2\2\2\2\u00c9\3\2\2\2\2\u00cb\3\2\2\2\2\u00cd\3\2\2\2\2\u00cf\3\2\2"
-        + "\2\2\u00d1\3\2\2\2\2\u00d3\3\2\2\2\2\u00d5\3\2\2\2\2\u00d7\3\2\2\2\2\u00d9"
-        + "\3\2\2\2\2\u00db\3\2\2\2\2\u00dd\3\2\2\2\2\u00df\3\2\2\2\2\u00e1\3\2\2"
-        + "\2\2\u00e3\3\2\2\2\2\u00e5\3\2\2\2\2\u00e7\3\2\2\2\2\u00e9\3\2\2\2\2\u00eb"
-        + "\3\2\2\2\2\u00ed\3\2\2\2\2\u00ef\3\2\2\2\2\u00f1\3\2\2\2\2\u00f3\3\2\2"
-        + "\2\2\u00f5\3\2\2\2\2\u00f7\3\2\2\2\2\u00f9\3\2\2\2\2\u00fb\3\2\2\2\2\u00fd"
-        + "\3\2\2\2\2\u00ff\3\2\2\2\2\u0101\3\2\2\2\2\u0103\3\2\2\2\2\u0105\3\2\2"
-        + "\2\2\u0107\3\2\2\2\2\u0109\3\2\2\2\2\u010b\3\2\2\2\2\u010d\3\2\2\2\2\u0115"
-        + "\3\2\2\2\2\u0117\3\2\2\2\2\u0119\3\2\2\2\2\u011b\3\2\2\2\3\u011d\3\2\2"
-        + "\2\5\u011f\3\2\2\2\7\u0121\3\2\2\2\t\u0123\3\2\2\2\13\u0125\3\2\2\2\r"
-        + "\u0129\3\2\2\2\17\u0131\3\2\2\2\21\u013a\3\2\2\2\23\u013e\3\2\2\2\25\u0142"
-        + "\3\2\2\2\27\u0145\3\2\2\2\31\u0149\3\2\2\2\33\u0151\3\2\2\2\35\u0154\3"
-        + "\2\2\2\37\u0159\3\2\2\2!\u015e\3\2\2\2#\u0166\3\2\2\2%\u016f\3\2\2\2\'"
-        + "\u0177\3\2\2\2)\u017f\3\2\2\2+\u018c\3\2\2\2-\u0199\3\2\2\2/\u01ab\3\2"
-        + "\2\2\61\u01af\3\2\2\2\63\u01b4\3\2\2\2\65\u01ba\3\2\2\2\67\u01bf\3\2\2"
-        + "\29\u01c8\3\2\2\2;\u01d1\3\2\2\2=\u01d6\3\2\2\2?\u01da\3\2\2\2A\u01e1"
-        + "\3\2\2\2C\u01ec\3\2\2\2E\u01f3\3\2\2\2G\u01fb\3\2\2\2I\u0203\3\2\2\2K"
-        + "\u0209\3\2\2\2M\u020f\3\2\2\2O\u0213\3\2\2\2Q\u021a\3\2\2\2S\u021f\3\2"
-        + "\2\2U\u0226\3\2\2\2W\u022b\3\2\2\2Y\u0235\3\2\2\2[\u023e\3\2\2\2]\u0244"
-        + "\3\2\2\2_\u024b\3\2\2\2a\u0250\3\2\2\2c\u0256\3\2\2\2e\u0259\3\2\2\2g"
-        + "\u0261\3\2\2\2i\u0267\3\2\2\2k\u0270\3\2\2\2m\u0273\3\2\2\2o\u0278\3\2"
-        + "\2\2q\u027d\3\2\2\2s\u0282\3\2\2\2u\u0287\3\2\2\2w\u028d\3\2\2\2y\u0294"
-        + "\3\2\2\2{\u029a\3\2\2\2}\u02a1\3\2\2\2\177\u02a9\3\2\2\2\u0081\u02af\3"
-        + "\2\2\2\u0083\u02b6\3\2\2\2\u0085\u02be\3\2\2\2\u0087\u02c2\3\2\2\2\u0089"
-        + "\u02c7\3\2\2\2\u008b\u02cd\3\2\2\2\u008d\u02d0\3\2\2\2\u008f\u02da\3\2"
-        + "\2\2\u0091\u02dd\3\2\2\2\u0093\u02e3\3\2\2\2\u0095\u02e9\3\2\2\2\u0097"
-        + "\u02f0\3\2\2\2\u0099\u02f9\3\2\2\2\u009b\u02ff\3\2\2\2\u009d\u0304\3\2"
-        + "\2\2\u009f\u030a\3\2\2\2\u00a1\u0310\3\2\2\2\u00a3\u0316\3\2\2\2\u00a5"
-        + "\u031e\3\2\2\2\u00a7\u0325\3\2\2\2\u00a9\u032d\3\2\2\2\u00ab\u0334\3\2"
-        + "\2\2\u00ad\u0339\3\2\2\2\u00af\u033d\3\2\2\2\u00b1\u0343\3\2\2\2\u00b3"
-        + "\u034a\3\2\2\2\u00b5\u034f\3\2\2\2\u00b7\u0354\3\2\2\2\u00b9\u0359\3\2"
-        + "\2\2\u00bb\u035c\3\2\2\2\u00bd\u0360\3\2\2\2\u00bf\u0365\3\2\2\2\u00c1"
-        + "\u036b\3\2\2\2\u00c3\u0371\3\2\2\2\u00c5\u0378\3\2\2\2\u00c7\u037d\3\2"
-        + "\2\2\u00c9\u0383\3\2\2\2\u00cb\u0388\3\2\2\2\u00cd\u038d\3\2\2\2\u00cf"
-        + "\u0393\3\2\2\2\u00d1\u039b\3\2\2\2\u00d3\u039f\3\2\2\2\u00d5\u03a6\3\2"
-        + "\2\2\u00d7\u03a9\3\2\2\2\u00d9\u03ac\3\2\2\2\u00db\u03b0\3\2\2\2\u00dd"
-        + "\u03b6\3\2\2\2\u00df\u03bd\3\2\2\2\u00e1\u03bf\3\2\2\2\u00e3\u03c1\3\2"
-        + "\2\2\u00e5\u03c9\3\2\2\2\u00e7\u03cb\3\2\2\2\u00e9\u03cd\3\2\2\2\u00eb"
-        + "\u03d0\3\2\2\2\u00ed\u03d2\3\2\2\2\u00ef\u03d5\3\2\2\2\u00f1\u03d7\3\2"
-        + "\2\2\u00f3\u03d9\3\2\2\2\u00f5\u03db\3\2\2\2\u00f7\u03dd\3\2\2\2\u00f9"
-        + "\u03df\3\2\2\2\u00fb\u03e2\3\2\2\2\u00fd\u03e4\3\2\2\2\u00ff\u03e6\3\2"
-        + "\2\2\u0101\u03f2\3\2\2\2\u0103\u0420\3\2\2\2\u0105\u0424\3\2\2\2\u0107"
-        + "\u042e\3\2\2\2\u0109\u0439\3\2\2\2\u010b\u043d\3\2\2\2\u010d\u0448\3\2"
-        + "\2\2\u010f\u0453\3\2\2\2\u0111\u045c\3\2\2\2\u0113\u045e\3\2\2\2\u0115"
-        + "\u0460\3\2\2\2\u0117\u0471\3\2\2\2\u0119\u0481\3\2\2\2\u011b\u0487\3\2"
-        + "\2\2\u011d\u011e\7*\2\2\u011e\4\3\2\2\2\u011f\u0120\7+\2\2\u0120\6\3\2"
-        + "\2\2\u0121\u0122\7.\2\2\u0122\b\3\2\2\2\u0123\u0124\7<\2\2\u0124\n\3\2"
-        + "\2\2\u0125\u0126\7C\2\2\u0126\u0127\7N\2\2\u0127\u0128\7N\2\2\u0128\f"
-        + "\3\2\2\2\u0129\u012a\7C\2\2\u012a\u012b\7P\2\2\u012b\u012c\7C\2\2\u012c"
-        + "\u012d\7N\2\2\u012d\u012e\7[\2\2\u012e\u012f\7\\\2\2\u012f\u0130\7G\2"
-        + "\2\u0130\16\3\2\2\2\u0131\u0132\7C\2\2\u0132\u0133\7P\2\2\u0133\u0134"
-        + "\7C\2\2\u0134\u0135\7N\2\2\u0135\u0136\7[\2\2\u0136\u0137\7\\\2\2\u0137"
-        + "\u0138\7G\2\2\u0138\u0139\7F\2\2\u0139\20\3\2\2\2\u013a\u013b\7C\2\2\u013b"
-        + "\u013c\7P\2\2\u013c\u013d\7F\2\2\u013d\22\3\2\2\2\u013e\u013f\7C\2\2\u013f"
-        + "\u0140\7P\2\2\u0140\u0141\7[\2\2\u0141\24\3\2\2\2\u0142\u0143\7C\2\2\u0143"
-        + "\u0144\7U\2\2\u0144\26\3\2\2\2\u0145\u0146\7C\2\2\u0146\u0147\7U\2\2\u0147"
-        + "\u0148\7E\2\2\u0148\30\3\2\2\2\u0149\u014a\7D\2\2\u014a\u014b\7G\2\2\u014b"
-        + "\u014c\7V\2\2\u014c\u014d\7Y\2\2\u014d\u014e\7G\2\2\u014e\u014f\7G\2\2"
-        + "\u014f\u0150\7P\2\2\u0150\32\3\2\2\2\u0151\u0152\7D\2\2\u0152\u0153\7"
-        + "[\2\2\u0153\34\3\2\2\2\u0154\u0155\7E\2\2\u0155\u0156\7C\2\2\u0156\u0157"
-        + "\7U\2\2\u0157\u0158\7G\2\2\u0158\36\3\2\2\2\u0159\u015a\7E\2\2\u015a\u015b"
-        + "\7C\2\2\u015b\u015c\7U\2\2\u015c\u015d\7V\2\2\u015d \3\2\2\2\u015e\u015f"
-        + "\7E\2\2\u015f\u0160\7C\2\2\u0160\u0161\7V\2\2\u0161\u0162\7C\2\2\u0162"
-        + "\u0163\7N\2\2\u0163\u0164\7Q\2\2\u0164\u0165\7I\2\2\u0165\"\3\2\2\2\u0166"
-        + "\u0167\7E\2\2\u0167\u0168\7C\2\2\u0168\u0169\7V\2\2\u0169\u016a\7C\2\2"
-        + "\u016a\u016b\7N\2\2\u016b\u016c\7Q\2\2\u016c\u016d\7I\2\2\u016d\u016e"
-        + "\7U\2\2\u016e$\3\2\2\2\u016f\u0170\7E\2\2\u0170\u0171\7Q\2\2\u0171\u0172"
-        + "\7N\2\2\u0172\u0173\7W\2\2\u0173\u0174\7O\2\2\u0174\u0175\7P\2\2\u0175"
-        + "\u0176\7U\2\2\u0176&\3\2\2\2\u0177\u0178\7E\2\2\u0178\u0179\7Q\2\2\u0179"
-        + "\u017a\7P\2\2\u017a\u017b\7X\2\2\u017b\u017c\7G\2\2\u017c\u017d\7T\2\2"
-        + "\u017d\u017e\7V\2\2\u017e(\3\2\2\2\u017f\u0180\7E\2\2\u0180\u0181\7W\2"
-        + "\2\u0181\u0182\7T\2\2\u0182\u0183\7T\2\2\u0183\u0184\7G\2\2\u0184\u0185"
-        + "\7P\2\2\u0185\u0186\7V\2\2\u0186\u0187\7a\2\2\u0187\u0188\7F\2\2\u0188"
-        + "\u0189\7C\2\2\u0189\u018a\7V\2\2\u018a\u018b\7G\2\2\u018b*\3\2\2\2\u018c"
-        + "\u018d\7E\2\2\u018d\u018e\7W\2\2\u018e\u018f\7T\2\2\u018f\u0190\7T\2\2"
-        + "\u0190\u0191\7G\2\2\u0191\u0192\7P\2\2\u0192\u0193\7V\2\2\u0193\u0194"
-        + "\7a\2\2\u0194\u0195\7V\2\2\u0195\u0196\7K\2\2\u0196\u0197\7O\2\2\u0197"
-        + "\u0198\7G\2\2\u0198,\3\2\2\2\u0199\u019a\7E\2\2\u019a\u019b\7W\2\2\u019b"
-        + "\u019c\7T\2\2\u019c\u019d\7T\2\2\u019d\u019e\7G\2\2\u019e\u019f\7P\2\2"
-        + "\u019f\u01a0\7V\2\2\u01a0\u01a1\7a\2\2\u01a1\u01a2\7V\2\2\u01a2\u01a3"
-        + "\7K\2\2\u01a3\u01a4\7O\2\2\u01a4\u01a5\7G\2\2\u01a5\u01a6\7U\2\2\u01a6"
-        + "\u01a7\7V\2\2\u01a7\u01a8\7C\2\2\u01a8\u01a9\7O\2\2\u01a9\u01aa\7R\2\2"
-        + "\u01aa.\3\2\2\2\u01ab\u01ac\7F\2\2\u01ac\u01ad\7C\2\2\u01ad\u01ae\7[\2"
-        + "\2\u01ae\60\3\2\2\2\u01af\u01b0\7F\2\2\u01b0\u01b1\7C\2\2\u01b1\u01b2"
-        + "\7[\2\2\u01b2\u01b3\7U\2\2\u01b3\62\3\2\2\2\u01b4\u01b5\7F\2\2\u01b5\u01b6"
-        + "\7G\2\2\u01b6\u01b7\7D\2\2\u01b7\u01b8\7W\2\2\u01b8\u01b9\7I\2\2\u01b9"
-        + "\64\3\2\2\2\u01ba\u01bb\7F\2\2\u01bb\u01bc\7G\2\2\u01bc\u01bd\7U\2\2\u01bd"
-        + "\u01be\7E\2\2\u01be\66\3\2\2\2\u01bf\u01c0\7F\2\2\u01c0\u01c1\7G\2\2\u01c1"
-        + "\u01c2\7U\2\2\u01c2\u01c3\7E\2\2\u01c3\u01c4\7T\2\2\u01c4\u01c5\7K\2\2"
-        + "\u01c5\u01c6\7D\2\2\u01c6\u01c7\7G\2\2\u01c78\3\2\2\2\u01c8\u01c9\7F\2"
-        + "\2\u01c9\u01ca\7K\2\2\u01ca\u01cb\7U\2\2\u01cb\u01cc\7V\2\2\u01cc\u01cd"
-        + "\7K\2\2\u01cd\u01ce\7P\2\2\u01ce\u01cf\7E\2\2\u01cf\u01d0\7V\2\2\u01d0"
-        + ":\3\2\2\2\u01d1\u01d2\7G\2\2\u01d2\u01d3\7N\2\2\u01d3\u01d4\7U\2\2\u01d4"
-        + "\u01d5\7G\2\2\u01d5<\3\2\2\2\u01d6\u01d7\7G\2\2\u01d7\u01d8\7P\2\2\u01d8"
-        + "\u01d9\7F\2\2\u01d9>\3\2\2\2\u01da\u01db\7G\2\2\u01db\u01dc\7U\2\2\u01dc"
-        + "\u01dd\7E\2\2\u01dd\u01de\7C\2\2\u01de\u01df\7R\2\2\u01df\u01e0\7G\2\2"
-        + "\u01e0@\3\2\2\2\u01e1\u01e2\7G\2\2\u01e2\u01e3\7Z\2\2\u01e3\u01e4\7G\2"
-        + "\2\u01e4\u01e5\7E\2\2\u01e5\u01e6\7W\2\2\u01e6\u01e7\7V\2\2\u01e7\u01e8"
-        + "\7C\2\2\u01e8\u01e9\7D\2\2\u01e9\u01ea\7N\2\2\u01ea\u01eb\7G\2\2\u01eb"
-        + "B\3\2\2\2\u01ec\u01ed\7G\2\2\u01ed\u01ee\7Z\2\2\u01ee\u01ef\7K\2\2\u01ef"
-        + "\u01f0\7U\2\2\u01f0\u01f1\7V\2\2\u01f1\u01f2\7U\2\2\u01f2D\3\2\2\2\u01f3"
-        + "\u01f4\7G\2\2\u01f4\u01f5\7Z\2\2\u01f5\u01f6\7R\2\2\u01f6\u01f7\7N\2\2"
-        + "\u01f7\u01f8\7C\2\2\u01f8\u01f9\7K\2\2\u01f9\u01fa\7P\2\2\u01faF\3\2\2"
-        + "\2\u01fb\u01fc\7G\2\2\u01fc\u01fd\7Z\2\2\u01fd\u01fe\7V\2\2\u01fe\u01ff"
-        + "\7T\2\2\u01ff\u0200\7C\2\2\u0200\u0201\7E\2\2\u0201\u0202\7V\2\2\u0202"
-        + "H\3\2\2\2\u0203\u0204\7H\2\2\u0204\u0205\7C\2\2\u0205\u0206\7N\2\2\u0206"
-        + "\u0207\7U\2\2\u0207\u0208\7G\2\2\u0208J\3\2\2\2\u0209\u020a\7H\2\2\u020a"
-        + "\u020b\7K\2\2\u020b\u020c\7T\2\2\u020c\u020d\7U\2\2\u020d\u020e\7V\2\2"
-        + "\u020eL\3\2\2\2\u020f\u0210\7H\2\2\u0210\u0211\7Q\2\2\u0211\u0212\7T\2"
-        + "\2\u0212N\3\2\2\2\u0213\u0214\7H\2\2\u0214\u0215\7Q\2\2\u0215\u0216\7"
-        + "T\2\2\u0216\u0217\7O\2\2\u0217\u0218\7C\2\2\u0218\u0219\7V\2\2\u0219P"
-        + "\3\2\2\2\u021a\u021b\7H\2\2\u021b\u021c\7T\2\2\u021c\u021d\7Q\2\2\u021d"
-        + "\u021e\7O\2\2\u021eR\3\2\2\2\u021f\u0220\7H\2\2\u0220\u0221\7T\2\2\u0221"
-        + "\u0222\7Q\2\2\u0222\u0223\7\\\2\2\u0223\u0224\7G\2\2\u0224\u0225\7P\2"
-        + "\2\u0225T\3\2\2\2\u0226\u0227\7H\2\2\u0227\u0228\7W\2\2\u0228\u0229\7"
-        + "N\2\2\u0229\u022a\7N\2\2\u022aV\3\2\2\2\u022b\u022c\7H\2\2\u022c\u022d"
-        + "\7W\2\2\u022d\u022e\7P\2\2\u022e\u022f\7E\2\2\u022f\u0230\7V\2\2\u0230"
-        + "\u0231\7K\2\2\u0231\u0232\7Q\2\2\u0232\u0233\7P\2\2\u0233\u0234\7U\2\2"
-        + "\u0234X\3\2\2\2\u0235\u0236\7I\2\2\u0236\u0237\7T\2\2\u0237\u0238\7C\2"
-        + "\2\u0238\u0239\7R\2\2\u0239\u023a\7J\2\2\u023a\u023b\7X\2\2\u023b\u023c"
-        + "\7K\2\2\u023c\u023d\7\\\2\2\u023dZ\3\2\2\2\u023e\u023f\7I\2\2\u023f\u0240"
-        + "\7T\2\2\u0240\u0241\7Q\2\2\u0241\u0242\7W\2\2\u0242\u0243\7R\2\2\u0243"
-        + "\\\3\2\2\2\u0244\u0245\7J\2\2\u0245\u0246\7C\2\2\u0246\u0247\7X\2\2\u0247"
-        + "\u0248\7K\2\2\u0248\u0249\7P\2\2\u0249\u024a\7I\2\2\u024a^\3\2\2\2\u024b"
-        + "\u024c\7J\2\2\u024c\u024d\7Q\2\2\u024d\u024e\7W\2\2\u024e\u024f\7T\2\2"
-        + "\u024f`\3\2\2\2\u0250\u0251\7J\2\2\u0251\u0252\7Q\2\2\u0252\u0253\7W\2"
-        + "\2\u0253\u0254\7T\2\2\u0254\u0255\7U\2\2\u0255b\3\2\2\2\u0256\u0257\7"
-        + "K\2\2\u0257\u0258\7P\2\2\u0258d\3\2\2\2\u0259\u025a\7K\2\2\u025a\u025b"
-        + "\7P\2\2\u025b\u025c\7E\2\2\u025c\u025d\7N\2\2\u025d\u025e\7W\2\2\u025e"
-        + "\u025f\7F\2\2\u025f\u0260\7G\2\2\u0260f\3\2\2\2\u0261\u0262\7K\2\2\u0262"
-        + "\u0263\7P\2\2\u0263\u0264\7P\2\2\u0264\u0265\7G\2\2\u0265\u0266\7T\2\2"
-        + "\u0266h\3\2\2\2\u0267\u0268\7K\2\2\u0268\u0269\7P\2\2\u0269\u026a\7V\2"
-        + "\2\u026a\u026b\7G\2\2\u026b\u026c\7T\2\2\u026c\u026d\7X\2\2\u026d\u026e"
-        + "\7C\2\2\u026e\u026f\7N\2\2\u026fj\3\2\2\2\u0270\u0271\7K\2\2\u0271\u0272"
-        + "\7U\2\2\u0272l\3\2\2\2\u0273\u0274\7L\2\2\u0274\u0275\7Q\2\2\u0275\u0276"
-        + "\7K\2\2\u0276\u0277\7P\2\2\u0277n\3\2\2\2\u0278\u0279\7N\2\2\u0279\u027a"
-        + "\7C\2\2\u027a\u027b\7U\2\2\u027b\u027c\7V\2\2\u027cp\3\2\2\2\u027d\u027e"
-        + "\7N\2\2\u027e\u027f\7G\2\2\u027f\u0280\7H\2\2\u0280\u0281\7V\2\2\u0281"
-        + "r\3\2\2\2\u0282\u0283\7N\2\2\u0283\u0284\7K\2\2\u0284\u0285\7M\2\2\u0285"
-        + "\u0286\7G\2\2\u0286t\3\2\2\2\u0287\u0288\7N\2\2\u0288\u0289\7K\2\2\u0289"
-        + "\u028a\7O\2\2\u028a\u028b\7K\2\2\u028b\u028c\7V\2\2\u028cv\3\2\2\2\u028d"
-        + "\u028e\7O\2\2\u028e\u028f\7C\2\2\u028f\u0290\7R\2\2\u0290\u0291\7R\2\2"
-        + "\u0291\u0292\7G\2\2\u0292\u0293\7F\2\2\u0293x\3\2\2\2\u0294\u0295\7O\2"
-        + "\2\u0295\u0296\7C\2\2\u0296\u0297\7V\2\2\u0297\u0298\7E\2\2\u0298\u0299"
-        + "\7J\2\2\u0299z\3\2\2\2\u029a\u029b\7O\2\2\u029b\u029c\7K\2\2\u029c\u029d"
-        + "\7P\2\2\u029d\u029e\7W\2\2\u029e\u029f\7V\2\2\u029f\u02a0\7G\2\2\u02a0"
-        + "|\3\2\2\2\u02a1\u02a2\7O\2\2\u02a2\u02a3\7K\2\2\u02a3\u02a4\7P\2\2\u02a4"
-        + "\u02a5\7W\2\2\u02a5\u02a6\7V\2\2\u02a6\u02a7\7G\2\2\u02a7\u02a8\7U\2\2"
-        + "\u02a8~\3\2\2\2\u02a9\u02aa\7O\2\2\u02aa\u02ab\7Q\2\2\u02ab\u02ac\7P\2"
-        + "\2\u02ac\u02ad\7V\2\2\u02ad\u02ae\7J\2\2\u02ae\u0080\3\2\2\2\u02af\u02b0"
-        + "\7O\2\2\u02b0\u02b1\7Q\2\2\u02b1\u02b2\7P\2\2\u02b2\u02b3\7V\2\2\u02b3"
-        + "\u02b4\7J\2\2\u02b4\u02b5\7U\2\2\u02b5\u0082\3\2\2\2\u02b6\u02b7\7P\2"
-        + "\2\u02b7\u02b8\7C\2\2\u02b8\u02b9\7V\2\2\u02b9\u02ba\7W\2\2\u02ba\u02bb"
-        + "\7T\2\2\u02bb\u02bc\7C\2\2\u02bc\u02bd\7N\2\2\u02bd\u0084\3\2\2\2\u02be"
-        + "\u02bf\7P\2\2\u02bf\u02c0\7Q\2\2\u02c0\u02c1\7V\2\2\u02c1\u0086\3\2\2"
-        + "\2\u02c2\u02c3\7P\2\2\u02c3\u02c4\7W\2\2\u02c4\u02c5\7N\2\2\u02c5\u02c6"
-        + "\7N\2\2\u02c6\u0088\3\2\2\2\u02c7\u02c8\7P\2\2\u02c8\u02c9\7W\2\2\u02c9"
-        + "\u02ca\7N\2\2\u02ca\u02cb\7N\2\2\u02cb\u02cc\7U\2\2\u02cc\u008a\3\2\2"
-        + "\2\u02cd\u02ce\7Q\2\2\u02ce\u02cf\7P\2\2\u02cf\u008c\3\2\2\2\u02d0\u02d1"
-        + "\7Q\2\2\u02d1\u02d2\7R\2\2\u02d2\u02d3\7V\2\2\u02d3\u02d4\7K\2\2\u02d4"
-        + "\u02d5\7O\2\2\u02d5\u02d6\7K\2\2\u02d6\u02d7\7\\\2\2\u02d7\u02d8\7G\2"
-        + "\2\u02d8\u02d9\7F\2\2\u02d9\u008e\3\2\2\2\u02da\u02db\7Q\2\2\u02db\u02dc"
-        + "\7T\2\2\u02dc\u0090\3\2\2\2\u02dd\u02de\7Q\2\2\u02de\u02df\7T\2\2\u02df"
-        + "\u02e0\7F\2\2\u02e0\u02e1\7G\2\2\u02e1\u02e2\7T\2\2\u02e2\u0092\3\2\2"
-        + "\2\u02e3\u02e4\7Q\2\2\u02e4\u02e5\7W\2\2\u02e5\u02e6\7V\2\2\u02e6\u02e7"
-        + "\7G\2\2\u02e7\u02e8\7T\2\2\u02e8\u0094\3\2\2\2\u02e9\u02ea\7R\2\2\u02ea"
-        + "\u02eb\7C\2\2\u02eb\u02ec\7T\2\2\u02ec\u02ed\7U\2\2\u02ed\u02ee\7G\2\2"
-        + "\u02ee\u02ef\7F\2\2\u02ef\u0096\3\2\2\2\u02f0\u02f1\7R\2\2\u02f1\u02f2"
-        + "\7J\2\2\u02f2\u02f3\7[\2\2\u02f3\u02f4\7U\2\2\u02f4\u02f5\7K\2\2\u02f5"
-        + "\u02f6\7E\2\2\u02f6\u02f7\7C\2\2\u02f7\u02f8\7N\2\2\u02f8\u0098\3\2\2"
-        + "\2\u02f9\u02fa\7R\2\2\u02fa\u02fb\7K\2\2\u02fb\u02fc\7X\2\2\u02fc\u02fd"
-        + "\7Q\2\2\u02fd\u02fe\7V\2\2\u02fe\u009a\3\2\2\2\u02ff\u0300\7R\2\2\u0300"
-        + "\u0301\7N\2\2\u0301\u0302\7C\2\2\u0302\u0303\7P\2\2\u0303\u009c\3\2\2"
-        + "\2\u0304\u0305\7T\2\2\u0305\u0306\7K\2\2\u0306\u0307\7I\2\2\u0307\u0308"
-        + "\7J\2\2\u0308\u0309\7V\2\2\u0309\u009e\3\2\2\2\u030a\u030b\7T\2\2\u030b"
-        + "\u030c\7N\2\2\u030c\u030d\7K\2\2\u030d\u030e\7M\2\2\u030e\u030f\7G\2\2"
-        + "\u030f\u00a0\3\2\2\2\u0310\u0311\7S\2\2\u0311\u0312\7W\2\2\u0312\u0313"
-        + "\7G\2\2\u0313\u0314\7T\2\2\u0314\u0315\7[\2\2\u0315\u00a2\3\2\2\2\u0316"
-        + "\u0317\7U\2\2\u0317\u0318\7E\2\2\u0318\u0319\7J\2\2\u0319\u031a\7G\2\2"
-        + "\u031a\u031b\7O\2\2\u031b\u031c\7C\2\2\u031c\u031d\7U\2\2\u031d\u00a4"
-        + "\3\2\2\2\u031e\u031f\7U\2\2\u031f\u0320\7G\2\2\u0320\u0321\7E\2\2\u0321"
-        + "\u0322\7Q\2\2\u0322\u0323\7P\2\2\u0323\u0324\7F\2\2\u0324\u00a6\3\2\2"
-        + "\2\u0325\u0326\7U\2\2\u0326\u0327\7G\2\2\u0327\u0328\7E\2\2\u0328\u0329"
-        + "\7Q\2\2\u0329\u032a\7P\2\2\u032a\u032b\7F\2\2\u032b\u032c\7U\2\2\u032c"
-        + "\u00a8\3\2\2\2\u032d\u032e\7U\2\2\u032e\u032f\7G\2\2\u032f\u0330\7N\2"
-        + "\2\u0330\u0331\7G\2\2\u0331\u0332\7E\2\2\u0332\u0333\7V\2\2\u0333\u00aa"
-        + "\3\2\2\2\u0334\u0335\7U\2\2\u0335\u0336\7J\2\2\u0336\u0337\7Q\2\2\u0337"
-        + "\u0338\7Y\2\2\u0338\u00ac\3\2\2\2\u0339\u033a\7U\2\2\u033a\u033b\7[\2"
-        + "\2\u033b\u033c\7U\2\2\u033c\u00ae\3\2\2\2\u033d\u033e\7V\2\2\u033e\u033f"
-        + "\7C\2\2\u033f\u0340\7D\2\2\u0340\u0341\7N\2\2\u0341\u0342\7G\2\2\u0342"
-        + "\u00b0\3\2\2\2\u0343\u0344\7V\2\2\u0344\u0345\7C\2\2\u0345\u0346\7D\2"
-        + "\2\u0346\u0347\7N\2\2\u0347\u0348\7G\2\2\u0348\u0349\7U\2\2\u0349\u00b2"
-        + "\3\2\2\2\u034a\u034b\7V\2\2\u034b\u034c\7G\2\2\u034c\u034d\7Z\2\2\u034d"
-        + "\u034e\7V\2\2\u034e\u00b4\3\2\2\2\u034f\u0350\7V\2\2\u0350\u0351\7J\2"
-        + "\2\u0351\u0352\7G\2\2\u0352\u0353\7P\2\2\u0353\u00b6\3\2\2\2\u0354\u0355"
-        + "\7V\2\2\u0355\u0356\7T\2\2\u0356\u0357\7W\2\2\u0357\u0358\7G\2\2\u0358"
-        + "\u00b8\3\2\2\2\u0359\u035a\7V\2\2\u035a\u035b\7Q\2\2\u035b\u00ba\3\2\2"
-        + "\2\u035c\u035d\7V\2\2\u035d\u035e\7Q\2\2\u035e\u035f\7R\2\2\u035f\u00bc"
-        + "\3\2\2\2\u0360\u0361\7V\2\2\u0361\u0362\7[\2\2\u0362\u0363\7R\2\2\u0363"
-        + "\u0364\7G\2\2\u0364\u00be\3\2\2\2\u0365\u0366\7V\2\2\u0366\u0367\7[\2"
-        + "\2\u0367\u0368\7R\2\2\u0368\u0369\7G\2\2\u0369\u036a\7U\2\2\u036a\u00c0"
-        + "\3\2\2\2\u036b\u036c\7W\2\2\u036c\u036d\7U\2\2\u036d\u036e\7K\2\2\u036e"
-        + "\u036f\7P\2\2\u036f\u0370\7I\2\2\u0370\u00c2\3\2\2\2\u0371\u0372\7X\2"
-        + "\2\u0372\u0373\7G\2\2\u0373\u0374\7T\2\2\u0374\u0375\7K\2\2\u0375\u0376"
-        + "\7H\2\2\u0376\u0377\7[\2\2\u0377\u00c4\3\2\2\2\u0378\u0379\7Y\2\2\u0379"
-        + "\u037a\7J\2\2\u037a\u037b\7G\2\2\u037b\u037c\7P\2\2\u037c\u00c6\3\2\2"
-        + "\2\u037d\u037e\7Y\2\2\u037e\u037f\7J\2\2\u037f\u0380\7G\2\2\u0380\u0381"
-        + "\7T\2\2\u0381\u0382\7G\2\2\u0382\u00c8\3\2\2\2\u0383\u0384\7Y\2\2\u0384"
-        + "\u0385\7K\2\2\u0385\u0386\7V\2\2\u0386\u0387\7J\2\2\u0387\u00ca\3\2\2"
-        + "\2\u0388\u0389\7[\2\2\u0389\u038a\7G\2\2\u038a\u038b\7C\2\2\u038b\u038c"
-        + "\7T\2\2\u038c\u00cc\3\2\2\2\u038d\u038e\7[\2\2\u038e\u038f\7G\2\2\u038f"
-        + "\u0390\7C\2\2\u0390\u0391\7T\2\2\u0391\u0392\7U\2\2\u0392\u00ce\3\2\2"
-        + "\2\u0393\u0394\5\u00ddo\2\u0394\u0395\7G\2\2\u0395\u0396\7U\2\2\u0396"
-        + "\u0397\7E\2\2\u0397\u0398\7C\2\2\u0398\u0399\7R\2\2\u0399\u039a\7G\2\2"
-        + "\u039a\u00d0\3\2\2\2\u039b\u039c\5\u00ddo\2\u039c\u039d\7H\2\2\u039d\u039e"
-        + "\7P\2\2\u039e\u00d2\3\2\2\2\u039f\u03a0\5\u00ddo\2\u03a0\u03a1\7N\2\2"
-        + "\u03a1\u03a2\7K\2\2\u03a2\u03a3\7O\2\2\u03a3\u03a4\7K\2\2\u03a4\u03a5"
-        + "\7V\2\2\u03a5\u00d4\3\2\2\2\u03a6\u03a7\5\u00ddo\2\u03a7\u03a8\7F\2\2"
-        + "\u03a8\u00d6\3\2\2\2\u03a9\u03aa\5\u00ddo\2\u03aa\u03ab\7V\2\2\u03ab\u00d8"
-        + "\3\2\2\2\u03ac\u03ad\5\u00ddo\2\u03ad\u03ae\7V\2\2\u03ae\u03af\7U\2\2"
-        + "\u03af\u00da\3\2\2\2\u03b0\u03b1\5\u00ddo\2\u03b1\u03b2\7I\2\2\u03b2\u03b3"
-        + "\7W\2\2\u03b3\u03b4\7K\2\2\u03b4\u03b5\7F\2\2\u03b5\u00dc\3\2\2\2\u03b6"
-        + "\u03ba\7}\2\2\u03b7\u03b9\5\u0119\u008d\2\u03b8\u03b7\3\2\2\2\u03b9\u03bc"
-        + "\3\2\2\2\u03ba\u03b8\3\2\2\2\u03ba\u03bb\3\2\2\2\u03bb\u00de\3\2\2\2\u03bc"
-        + "\u03ba\3\2\2\2\u03bd\u03be\7\177\2\2\u03be\u00e0\3\2\2\2\u03bf\u03c0\7"
-        + "?\2\2\u03c0\u00e2\3\2\2\2\u03c1\u03c2\7>\2\2\u03c2\u03c3\7?\2\2\u03c3"
-        + "\u03c4\7@\2\2\u03c4\u00e4\3\2\2\2\u03c5\u03c6\7>\2\2\u03c6\u03ca\7@\2"
-        + "\2\u03c7\u03c8\7#\2\2\u03c8\u03ca\7?\2\2\u03c9\u03c5\3\2\2\2\u03c9\u03c7"
-        + "\3\2\2\2\u03ca\u00e6\3\2\2\2\u03cb\u03cc\7>\2\2\u03cc\u00e8\3\2\2\2\u03cd"
-        + "\u03ce\7>\2\2\u03ce\u03cf\7?\2\2\u03cf\u00ea\3\2\2\2\u03d0\u03d1\7@\2"
-        + "\2\u03d1\u00ec\3\2\2\2\u03d2\u03d3\7@\2\2\u03d3\u03d4\7?\2\2\u03d4\u00ee"
-        + "\3\2\2\2\u03d5\u03d6\7-\2\2\u03d6\u00f0\3\2\2\2\u03d7\u03d8\7/\2\2\u03d8"
-        + "\u00f2\3\2\2\2\u03d9\u03da\7,\2\2\u03da\u00f4\3\2\2\2\u03db\u03dc\7\61"
-        + "\2\2\u03dc\u00f6\3\2\2\2\u03dd\u03de\7\'\2\2\u03de\u00f8\3\2\2\2\u03df"
-        + "\u03e0\7<\2\2\u03e0\u03e1\7<\2\2\u03e1\u00fa\3\2\2\2\u03e2\u03e3\7\60"
-        + "\2\2\u03e3\u00fc\3\2\2\2\u03e4\u03e5\7A\2\2\u03e5\u00fe\3\2\2\2\u03e6"
-        + "\u03ec\7)\2\2\u03e7\u03eb\n\2\2\2\u03e8\u03e9\7)\2\2\u03e9\u03eb\7)\2"
-        + "\2\u03ea\u03e7\3\2\2\2\u03ea\u03e8\3\2\2\2\u03eb\u03ee\3\2\2\2\u03ec\u03ea"
-        + "\3\2\2\2\u03ec\u03ed\3\2\2\2\u03ed\u03ef\3\2\2\2\u03ee\u03ec\3\2\2\2\u03ef"
-        + "\u03f0\7)\2\2\u03f0\u0100\3\2\2\2\u03f1\u03f3\5\u0111\u0089\2\u03f2\u03f1"
-        + "\3\2\2\2\u03f3\u03f4\3\2\2\2\u03f4\u03f2\3\2\2\2\u03f4\u03f5\3\2\2\2\u03f5"
-        + "\u0102\3\2\2\2\u03f6\u03f8\5\u0111\u0089\2\u03f7\u03f6\3\2\2\2\u03f8\u03f9"
-        + "\3\2\2\2\u03f9\u03f7\3\2\2\2\u03f9\u03fa\3\2\2\2\u03fa\u03fb\3\2\2\2\u03fb"
-        + "\u03ff\5\u00fb~\2\u03fc\u03fe\5\u0111\u0089\2\u03fd\u03fc\3\2\2\2\u03fe"
-        + "\u0401\3\2\2\2\u03ff\u03fd\3\2\2\2\u03ff\u0400\3\2\2\2\u0400\u0421\3\2"
-        + "\2\2\u0401\u03ff\3\2\2\2\u0402\u0404\5\u00fb~\2\u0403\u0405\5\u0111\u0089"
-        + "\2\u0404\u0403\3\2\2\2\u0405\u0406\3\2\2\2\u0406\u0404\3\2\2\2\u0406\u0407"
-        + "\3\2\2\2\u0407\u0421\3\2\2\2\u0408\u040a\5\u0111\u0089\2\u0409\u0408\3"
-        + "\2\2\2\u040a\u040b\3\2\2\2\u040b\u0409\3\2\2\2\u040b\u040c\3\2\2\2\u040c"
-        + "\u0414\3\2\2\2\u040d\u0411\5\u00fb~\2\u040e\u0410\5\u0111\u0089\2\u040f"
-        + "\u040e\3\2\2\2\u0410\u0413\3\2\2\2\u0411\u040f\3\2\2\2\u0411\u0412\3\2"
-        + "\2\2\u0412\u0415\3\2\2\2\u0413\u0411\3\2\2\2\u0414\u040d\3\2\2\2\u0414"
-        + "\u0415\3\2\2\2\u0415\u0416\3\2\2\2\u0416\u0417\5\u010f\u0088\2\u0417\u0421"
-        + "\3\2\2\2\u0418\u041a\5\u00fb~\2\u0419\u041b\5\u0111\u0089\2\u041a\u0419"
-        + "\3\2\2\2\u041b\u041c\3\2\2\2\u041c\u041a\3\2\2\2\u041c\u041d\3\2\2\2\u041d"
-        + "\u041e\3\2\2\2\u041e\u041f\5\u010f\u0088\2\u041f\u0421\3\2\2\2\u0420\u03f7"
-        + "\3\2\2\2\u0420\u0402\3\2\2\2\u0420\u0409\3\2\2\2\u0420\u0418\3\2\2\2\u0421"
-        + "\u0104\3\2\2\2\u0422\u0425\5\u0113\u008a\2\u0423\u0425\7a\2\2\u0424\u0422"
-        + "\3\2\2\2\u0424\u0423\3\2\2\2\u0425\u042b\3\2\2\2\u0426\u042a\5\u0113\u008a"
-        + "\2\u0427\u042a\5\u0111\u0089\2\u0428\u042a\t\3\2\2\u0429\u0426\3\2\2\2"
-        + "\u0429\u0427\3\2\2\2\u0429\u0428\3\2\2\2\u042a\u042d\3\2\2\2\u042b\u0429"
-        + "\3\2\2\2\u042b\u042c\3\2\2\2\u042c\u0106\3\2\2\2\u042d\u042b\3\2\2\2\u042e"
-        + "\u0432\5\u0111\u0089\2\u042f\u0433\5\u0113\u008a\2\u0430\u0433\5\u0111"
-        + "\u0089\2\u0431\u0433\t\3\2\2\u0432\u042f\3\2\2\2\u0432\u0430\3\2\2\2\u0432"
-        + "\u0431\3\2\2\2\u0433\u0434\3\2\2\2\u0434\u0432\3\2\2\2\u0434\u0435\3\2"
-        + "\2\2\u0435\u0108\3\2\2\2\u0436\u043a\5\u0113\u008a\2\u0437\u043a\5\u0111"
-        + "\u0089\2\u0438\u043a\7a\2\2\u0439\u0436\3\2\2\2\u0439\u0437\3\2\2\2\u0439"
-        + "\u0438\3\2\2\2\u043a\u043b\3\2\2\2\u043b\u0439\3\2\2\2\u043b\u043c\3\2"
-        + "\2\2\u043c\u010a\3\2\2\2\u043d\u0443\7$\2\2\u043e\u0442\n\4\2\2\u043f"
-        + "\u0440\7$\2\2\u0440\u0442\7$\2\2\u0441\u043e\3\2\2\2\u0441\u043f\3\2\2"
-        + "\2\u0442\u0445\3\2\2\2\u0443\u0441\3\2\2\2\u0443\u0444\3\2\2\2\u0444\u0446"
-        + "\3\2\2\2\u0445\u0443\3\2\2\2\u0446\u0447\7$\2\2\u0447\u010c\3\2\2\2\u0448"
-        + "\u044e\7b\2\2\u0449\u044d\n\5\2\2\u044a\u044b\7b\2\2\u044b\u044d\7b\2"
-        + "\2\u044c\u0449\3\2\2\2\u044c\u044a\3\2\2\2\u044d\u0450\3\2\2\2\u044e\u044c"
-        + "\3\2\2\2\u044e\u044f\3\2\2\2\u044f\u0451\3\2\2\2\u0450\u044e\3\2\2\2\u0451"
-        + "\u0452\7b\2\2\u0452\u010e\3\2\2\2\u0453\u0455\7G\2\2\u0454\u0456\t\6\2"
-        + "\2\u0455\u0454\3\2\2\2\u0455\u0456\3\2\2\2\u0456\u0458\3\2\2\2\u0457\u0459"
-        + "\5\u0111\u0089\2\u0458\u0457\3\2\2\2\u0459\u045a\3\2\2\2\u045a\u0458\3"
-        + "\2\2\2\u045a\u045b\3\2\2\2\u045b\u0110\3\2\2\2\u045c\u045d\t\7\2\2\u045d"
-        + "\u0112\3\2\2\2\u045e\u045f\t\b\2\2\u045f\u0114\3\2\2\2\u0460\u0461\7/"
-        + "\2\2\u0461\u0462\7/\2\2\u0462\u0466\3\2\2\2\u0463\u0465\n\t\2\2\u0464"
-        + "\u0463\3\2\2\2\u0465\u0468\3\2\2\2\u0466\u0464\3\2\2\2\u0466\u0467\3\2"
-        + "\2\2\u0467\u046a\3\2\2\2\u0468\u0466\3\2\2\2\u0469\u046b\7\17\2\2\u046a"
-        + "\u0469\3\2\2\2\u046a\u046b\3\2\2\2\u046b\u046d\3\2\2\2\u046c\u046e\7\f"
-        + "\2\2\u046d\u046c\3\2\2\2\u046d\u046e\3\2\2\2\u046e\u046f\3\2\2\2\u046f"
-        + "\u0470\b\u008b\2\2\u0470\u0116\3\2\2\2\u0471\u0472\7\61\2\2\u0472\u0473"
-        + "\7,\2\2\u0473\u0478\3\2\2\2\u0474\u0477\5\u0117\u008c\2\u0475\u0477\13"
-        + "\2\2\2\u0476\u0474\3\2\2\2\u0476\u0475\3\2\2\2\u0477\u047a\3\2\2\2\u0478"
-        + "\u0479\3\2\2\2\u0478\u0476\3\2\2\2\u0479\u047b\3\2\2\2\u047a\u0478\3\2"
-        + "\2\2\u047b\u047c\7,\2\2\u047c\u047d\7\61\2\2\u047d\u047e\3\2\2\2\u047e"
-        + "\u047f\b\u008c\2\2\u047f\u0118\3\2\2\2\u0480\u0482\t\n\2\2\u0481\u0480"
-        + "\3\2\2\2\u0482\u0483\3\2\2\2\u0483\u0481\3\2\2\2\u0483\u0484\3\2\2\2\u0484"
-        + "\u0485\3\2\2\2\u0485\u0486\b\u008d\2\2\u0486\u011a\3\2\2\2\u0487\u0488"
-        + "\13\2\2\2\u0488\u011c\3\2\2\2#\2\u03ba\u03c9\u03ea\u03ec\u03f4\u03f9\u03ff"
-        + "\u0406\u040b\u0411\u0414\u041c\u0420\u0424\u0429\u042b\u0432\u0434\u0439"
-        + "\u043b\u0441\u0443\u044c\u044e\u0455\u045a\u0466\u046a\u046d\u0476\u0478"
-        + "\u0483\3\2\3\2";
+    public static final String _serializedATN = "\u0004\u0000\u008a\u0487\u0006\uffff\uffff\u0002\u0000\u0007\u0000\u0002"
+        + "\u0001\u0007\u0001\u0002\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002"
+        + "\u0004\u0007\u0004\u0002\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002"
+        + "\u0007\u0007\u0007\u0002\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002"
+        + "\u000b\u0007\u000b\u0002\f\u0007\f\u0002\r\u0007\r\u0002\u000e\u0007\u000e"
+        + "\u0002\u000f\u0007\u000f\u0002\u0010\u0007\u0010\u0002\u0011\u0007\u0011"
+        + "\u0002\u0012\u0007\u0012\u0002\u0013\u0007\u0013\u0002\u0014\u0007\u0014"
+        + "\u0002\u0015\u0007\u0015\u0002\u0016\u0007\u0016\u0002\u0017\u0007\u0017"
+        + "\u0002\u0018\u0007\u0018\u0002\u0019\u0007\u0019\u0002\u001a\u0007\u001a"
+        + "\u0002\u001b\u0007\u001b\u0002\u001c\u0007\u001c\u0002\u001d\u0007\u001d"
+        + "\u0002\u001e\u0007\u001e\u0002\u001f\u0007\u001f\u0002 \u0007 \u0002!"
+        + "\u0007!\u0002\"\u0007\"\u0002#\u0007#\u0002$\u0007$\u0002%\u0007%\u0002"
+        + "&\u0007&\u0002\'\u0007\'\u0002(\u0007(\u0002)\u0007)\u0002*\u0007*\u0002"
+        + "+\u0007+\u0002,\u0007,\u0002-\u0007-\u0002.\u0007.\u0002/\u0007/\u0002"
+        + "0\u00070\u00021\u00071\u00022\u00072\u00023\u00073\u00024\u00074\u0002"
+        + "5\u00075\u00026\u00076\u00027\u00077\u00028\u00078\u00029\u00079\u0002"
+        + ":\u0007:\u0002;\u0007;\u0002<\u0007<\u0002=\u0007=\u0002>\u0007>\u0002"
+        + "?\u0007?\u0002@\u0007@\u0002A\u0007A\u0002B\u0007B\u0002C\u0007C\u0002"
+        + "D\u0007D\u0002E\u0007E\u0002F\u0007F\u0002G\u0007G\u0002H\u0007H\u0002"
+        + "I\u0007I\u0002J\u0007J\u0002K\u0007K\u0002L\u0007L\u0002M\u0007M\u0002"
+        + "N\u0007N\u0002O\u0007O\u0002P\u0007P\u0002Q\u0007Q\u0002R\u0007R\u0002"
+        + "S\u0007S\u0002T\u0007T\u0002U\u0007U\u0002V\u0007V\u0002W\u0007W\u0002"
+        + "X\u0007X\u0002Y\u0007Y\u0002Z\u0007Z\u0002[\u0007[\u0002\\\u0007\\\u0002"
+        + "]\u0007]\u0002^\u0007^\u0002_\u0007_\u0002`\u0007`\u0002a\u0007a\u0002"
+        + "b\u0007b\u0002c\u0007c\u0002d\u0007d\u0002e\u0007e\u0002f\u0007f\u0002"
+        + "g\u0007g\u0002h\u0007h\u0002i\u0007i\u0002j\u0007j\u0002k\u0007k\u0002"
+        + "l\u0007l\u0002m\u0007m\u0002n\u0007n\u0002o\u0007o\u0002p\u0007p\u0002"
+        + "q\u0007q\u0002r\u0007r\u0002s\u0007s\u0002t\u0007t\u0002u\u0007u\u0002"
+        + "v\u0007v\u0002w\u0007w\u0002x\u0007x\u0002y\u0007y\u0002z\u0007z\u0002"
+        + "{\u0007{\u0002|\u0007|\u0002}\u0007}\u0002~\u0007~\u0002\u007f\u0007\u007f"
+        + "\u0002\u0080\u0007\u0080\u0002\u0081\u0007\u0081\u0002\u0082\u0007\u0082"
+        + "\u0002\u0083\u0007\u0083\u0002\u0084\u0007\u0084\u0002\u0085\u0007\u0085"
+        + "\u0002\u0086\u0007\u0086\u0002\u0087\u0007\u0087\u0002\u0088\u0007\u0088"
+        + "\u0002\u0089\u0007\u0089\u0002\u008a\u0007\u008a\u0002\u008b\u0007\u008b"
+        + "\u0002\u008c\u0007\u008c\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001"
+        + "\u0001\u0002\u0001\u0002\u0001\u0003\u0001\u0003\u0001\u0004\u0001\u0004"
+        + "\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"
+        + "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0006\u0001\u0006"
+        + "\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006"
+        + "\u0001\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\b\u0001"
+        + "\b\u0001\b\u0001\b\u0001\t\u0001\t\u0001\t\u0001\n\u0001\n\u0001\n\u0001"
+        + "\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b"
+        + "\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\f\u0001\r\u0001\r\u0001"
+        + "\r\u0001\r\u0001\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001"
+        + "\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001"
+        + "\u000f\u0001\u000f\u0001\u000f\u0001\u0010\u0001\u0010\u0001\u0010\u0001"
+        + "\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001"
+        + "\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001"
+        + "\u0011\u0001\u0011\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001"
+        + "\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0013\u0001\u0013\u0001"
+        + "\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001"
+        + "\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0014\u0001"
+        + "\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001"
+        + "\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001"
+        + "\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001"
+        + "\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001"
+        + "\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0001"
+        + "\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0017\u0001\u0017\u0001"
+        + "\u0017\u0001\u0017\u0001\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0001"
+        + "\u0018\u0001\u0018\u0001\u0018\u0001\u0019\u0001\u0019\u0001\u0019\u0001"
+        + "\u0019\u0001\u0019\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001"
+        + "\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001a\u0001\u001b\u0001"
+        + "\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001"
+        + "\u001b\u0001\u001b\u0001\u001c\u0001\u001c\u0001\u001c\u0001\u001c\u0001"
+        + "\u001c\u0001\u001d\u0001\u001d\u0001\u001d\u0001\u001d\u0001\u001e\u0001"
+        + "\u001e\u0001\u001e\u0001\u001e\u0001\u001e\u0001\u001e\u0001\u001e\u0001"
+        + "\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001"
+        + "\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001 \u0001 \u0001"
+        + " \u0001 \u0001 \u0001 \u0001 \u0001!\u0001!\u0001!\u0001!\u0001!\u0001"
+        + "!\u0001!\u0001!\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001"
+        + "\"\u0001\"\u0001#\u0001#\u0001#\u0001#\u0001#\u0001#\u0001$\u0001$\u0001"
+        + "$\u0001$\u0001$\u0001$\u0001%\u0001%\u0001%\u0001%\u0001&\u0001&\u0001"
+        + "&\u0001&\u0001&\u0001&\u0001&\u0001\'\u0001\'\u0001\'\u0001\'\u0001\'"
+        + "\u0001(\u0001(\u0001(\u0001(\u0001(\u0001(\u0001(\u0001)\u0001)\u0001"
+        + ")\u0001)\u0001)\u0001*\u0001*\u0001*\u0001*\u0001*\u0001*\u0001*\u0001"
+        + "*\u0001*\u0001*\u0001+\u0001+\u0001+\u0001+\u0001+\u0001+\u0001+\u0001"
+        + "+\u0001+\u0001,\u0001,\u0001,\u0001,\u0001,\u0001,\u0001-\u0001-\u0001"
+        + "-\u0001-\u0001-\u0001-\u0001-\u0001.\u0001.\u0001.\u0001.\u0001.\u0001"
+        + "/\u0001/\u0001/\u0001/\u0001/\u0001/\u00010\u00010\u00010\u00011\u0001"
+        + "1\u00011\u00011\u00011\u00011\u00011\u00011\u00012\u00012\u00012\u0001"
+        + "2\u00012\u00012\u00013\u00013\u00013\u00013\u00013\u00013\u00013\u0001"
+        + "3\u00013\u00014\u00014\u00014\u00015\u00015\u00015\u00015\u00015\u0001"
+        + "6\u00016\u00016\u00016\u00016\u00017\u00017\u00017\u00017\u00017\u0001"
+        + "8\u00018\u00018\u00018\u00018\u00019\u00019\u00019\u00019\u00019\u0001"
+        + "9\u0001:\u0001:\u0001:\u0001:\u0001:\u0001:\u0001:\u0001;\u0001;\u0001"
+        + ";\u0001;\u0001;\u0001;\u0001<\u0001<\u0001<\u0001<\u0001<\u0001<\u0001"
+        + "<\u0001=\u0001=\u0001=\u0001=\u0001=\u0001=\u0001=\u0001=\u0001>\u0001"
+        + ">\u0001>\u0001>\u0001>\u0001>\u0001?\u0001?\u0001?\u0001?\u0001?\u0001"
+        + "?\u0001?\u0001@\u0001@\u0001@\u0001@\u0001@\u0001@\u0001@\u0001@\u0001"
+        + "A\u0001A\u0001A\u0001A\u0001B\u0001B\u0001B\u0001B\u0001B\u0001C\u0001"
+        + "C\u0001C\u0001C\u0001C\u0001C\u0001D\u0001D\u0001D\u0001E\u0001E\u0001"
+        + "E\u0001E\u0001E\u0001E\u0001E\u0001E\u0001E\u0001E\u0001F\u0001F\u0001"
+        + "F\u0001G\u0001G\u0001G\u0001G\u0001G\u0001G\u0001H\u0001H\u0001H\u0001"
+        + "H\u0001H\u0001H\u0001I\u0001I\u0001I\u0001I\u0001I\u0001I\u0001I\u0001"
+        + "J\u0001J\u0001J\u0001J\u0001J\u0001J\u0001J\u0001J\u0001J\u0001K\u0001"
+        + "K\u0001K\u0001K\u0001K\u0001K\u0001L\u0001L\u0001L\u0001L\u0001L\u0001"
+        + "M\u0001M\u0001M\u0001M\u0001M\u0001M\u0001N\u0001N\u0001N\u0001N\u0001"
+        + "N\u0001N\u0001O\u0001O\u0001O\u0001O\u0001O\u0001O\u0001P\u0001P\u0001"
+        + "P\u0001P\u0001P\u0001P\u0001P\u0001P\u0001Q\u0001Q\u0001Q\u0001Q\u0001"
+        + "Q\u0001Q\u0001Q\u0001R\u0001R\u0001R\u0001R\u0001R\u0001R\u0001R\u0001"
+        + "R\u0001S\u0001S\u0001S\u0001S\u0001S\u0001S\u0001S\u0001T\u0001T\u0001"
+        + "T\u0001T\u0001T\u0001U\u0001U\u0001U\u0001U\u0001V\u0001V\u0001V\u0001"
+        + "V\u0001V\u0001V\u0001W\u0001W\u0001W\u0001W\u0001W\u0001W\u0001W\u0001"
+        + "X\u0001X\u0001X\u0001X\u0001X\u0001Y\u0001Y\u0001Y\u0001Y\u0001Y\u0001"
+        + "Z\u0001Z\u0001Z\u0001Z\u0001Z\u0001[\u0001[\u0001[\u0001\\\u0001\\\u0001"
+        + "\\\u0001\\\u0001]\u0001]\u0001]\u0001]\u0001]\u0001^\u0001^\u0001^\u0001"
+        + "^\u0001^\u0001^\u0001_\u0001_\u0001_\u0001_\u0001_\u0001_\u0001`\u0001"
+        + "`\u0001`\u0001`\u0001`\u0001`\u0001`\u0001a\u0001a\u0001a\u0001a\u0001"
+        + "a\u0001b\u0001b\u0001b\u0001b\u0001b\u0001b\u0001c\u0001c\u0001c\u0001"
+        + "c\u0001c\u0001d\u0001d\u0001d\u0001d\u0001d\u0001e\u0001e\u0001e\u0001"
+        + "e\u0001e\u0001e\u0001f\u0001f\u0001f\u0001f\u0001f\u0001f\u0001f\u0001"
+        + "f\u0001g\u0001g\u0001g\u0001g\u0001h\u0001h\u0001h\u0001h\u0001h\u0001"
+        + "h\u0001h\u0001i\u0001i\u0001i\u0001j\u0001j\u0001j\u0001k\u0001k\u0001"
+        + "k\u0001k\u0001l\u0001l\u0001l\u0001l\u0001l\u0001l\u0001m\u0001m\u0005"
+        + "m\u03b7\bm\nm\fm\u03ba\tm\u0001n\u0001n\u0001o\u0001o\u0001p\u0001p\u0001"
+        + "p\u0001p\u0001q\u0001q\u0001q\u0001q\u0003q\u03c8\bq\u0001r\u0001r\u0001"
+        + "s\u0001s\u0001s\u0001t\u0001t\u0001u\u0001u\u0001u\u0001v\u0001v\u0001"
+        + "w\u0001w\u0001x\u0001x\u0001y\u0001y\u0001z\u0001z\u0001{\u0001{\u0001"
+        + "{\u0001|\u0001|\u0001}\u0001}\u0001~\u0001~\u0001~\u0001~\u0005~\u03e9"
+        + "\b~\n~\f~\u03ec\t~\u0001~\u0001~\u0001\u007f\u0004\u007f\u03f1\b\u007f"
+        + "\u000b\u007f\f\u007f\u03f2\u0001\u0080\u0004\u0080\u03f6\b\u0080\u000b"
+        + "\u0080\f\u0080\u03f7\u0001\u0080\u0001\u0080\u0005\u0080\u03fc\b\u0080"
+        + "\n\u0080\f\u0080\u03ff\t\u0080\u0001\u0080\u0001\u0080\u0004\u0080\u0403"
+        + "\b\u0080\u000b\u0080\f\u0080\u0404\u0001\u0080\u0004\u0080\u0408\b\u0080"
+        + "\u000b\u0080\f\u0080\u0409\u0001\u0080\u0001\u0080\u0005\u0080\u040e\b"
+        + "\u0080\n\u0080\f\u0080\u0411\t\u0080\u0003\u0080\u0413\b\u0080\u0001\u0080"
+        + "\u0001\u0080\u0001\u0080\u0001\u0080\u0004\u0080\u0419\b\u0080\u000b\u0080"
+        + "\f\u0080\u041a\u0001\u0080\u0001\u0080\u0003\u0080\u041f\b\u0080\u0001"
+        + "\u0081\u0001\u0081\u0003\u0081\u0423\b\u0081\u0001\u0081\u0001\u0081\u0001"
+        + "\u0081\u0005\u0081\u0428\b\u0081\n\u0081\f\u0081\u042b\t\u0081\u0001\u0082"
+        + "\u0001\u0082\u0001\u0082\u0001\u0082\u0004\u0082\u0431\b\u0082\u000b\u0082"
+        + "\f\u0082\u0432\u0001\u0083\u0001\u0083\u0001\u0083\u0004\u0083\u0438\b"
+        + "\u0083\u000b\u0083\f\u0083\u0439\u0001\u0084\u0001\u0084\u0001\u0084\u0001"
+        + "\u0084\u0005\u0084\u0440\b\u0084\n\u0084\f\u0084\u0443\t\u0084\u0001\u0084"
+        + "\u0001\u0084\u0001\u0085\u0001\u0085\u0001\u0085\u0001\u0085\u0005\u0085"
+        + "\u044b\b\u0085\n\u0085\f\u0085\u044e\t\u0085\u0001\u0085\u0001\u0085\u0001"
+        + "\u0086\u0001\u0086\u0003\u0086\u0454\b\u0086\u0001\u0086\u0004\u0086\u0457"
+        + "\b\u0086\u000b\u0086\f\u0086\u0458\u0001\u0087\u0001\u0087\u0001\u0088"
+        + "\u0001\u0088\u0001\u0089\u0001\u0089\u0001\u0089\u0001\u0089\u0005\u0089"
+        + "\u0463\b\u0089\n\u0089\f\u0089\u0466\t\u0089\u0001\u0089\u0003\u0089\u0469"
+        + "\b\u0089\u0001\u0089\u0003\u0089\u046c\b\u0089\u0001\u0089\u0001\u0089"
+        + "\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008a\u0005\u008a"
+        + "\u0475\b\u008a\n\u008a\f\u008a\u0478\t\u008a\u0001\u008a\u0001\u008a\u0001"
+        + "\u008a\u0001\u008a\u0001\u008a\u0001\u008b\u0004\u008b\u0480\b\u008b\u000b"
+        + "\u008b\f\u008b\u0481\u0001\u008b\u0001\u008b\u0001\u008c\u0001\u008c\u0001"
+        + "\u0476\u0000\u008d\u0001\u0001\u0003\u0002\u0005\u0003\u0007\u0004\t\u0005"
+        + "\u000b\u0006\r\u0007\u000f\b\u0011\t\u0013\n\u0015\u000b\u0017\f\u0019"
+        + "\r\u001b\u000e\u001d\u000f\u001f\u0010!\u0011#\u0012%\u0013\'\u0014)\u0015"
+        + "+\u0016-\u0017/\u00181\u00193\u001a5\u001b7\u001c9\u001d;\u001e=\u001f"
+        + "? A!C\"E#G$I%K&M\'O(Q)S*U+W,Y-[.]/_0a1c2e3g4i5k6m7o8q9s:u;w<y={>}?\u007f"
+        + "@\u0081A\u0083B\u0085C\u0087D\u0089E\u008bF\u008dG\u008fH\u0091I\u0093"
+        + "J\u0095K\u0097L\u0099M\u009bN\u009dO\u009fP\u00a1Q\u00a3R\u00a5S\u00a7"
+        + "T\u00a9U\u00abV\u00adW\u00afX\u00b1Y\u00b3Z\u00b5[\u00b7\\\u00b9]\u00bb"
+        + "^\u00bd_\u00bf`\u00c1a\u00c3b\u00c5c\u00c7d\u00c9e\u00cbf\u00cdg\u00cf"
+        + "h\u00d1i\u00d3j\u00d5k\u00d7l\u00d9m\u00dbn\u00ddo\u00dfp\u00e1q\u00e3"
+        + "r\u00e5s\u00e7t\u00e9u\u00ebv\u00edw\u00efx\u00f1y\u00f3z\u00f5{\u00f7"
+        + "|\u00f9}\u00fb~\u00fd\u007f\u00ff\u0080\u0101\u0081\u0103\u0082\u0105"
+        + "\u0083\u0107\u0084\u0109\u0085\u010b\u0086\u010d\u0000\u010f\u0000\u0111"
+        + "\u0000\u0113\u0087\u0115\u0088\u0117\u0089\u0119\u008a\u0001\u0000\t\u0001"
+        + "\u0000\'\'\u0002\u0000@@__\u0001\u0000\"\"\u0001\u0000``\u0002\u0000+"
+        + "+--\u0001\u000009\u0001\u0000AZ\u0002\u0000\n\n\r\r\u0003\u0000\t\n\r"
+        + "\r  \u04a8\u0000\u0001\u0001\u0000\u0000\u0000\u0000\u0003\u0001\u0000"
+        + "\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000\u0000\u0007\u0001\u0000"
+        + "\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0000\u000b\u0001\u0000\u0000"
+        + "\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000\u0000\u0000"
+        + "\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000\u0000\u0000"
+        + "\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000\u0000\u0000"
+        + "\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000\u0000\u0000"
+        + "\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000\u0000\u0000"
+        + "\u0000!\u0001\u0000\u0000\u0000\u0000#\u0001\u0000\u0000\u0000\u0000%"
+        + "\u0001\u0000\u0000\u0000\u0000\'\u0001\u0000\u0000\u0000\u0000)\u0001"
+        + "\u0000\u0000\u0000\u0000+\u0001\u0000\u0000\u0000\u0000-\u0001\u0000\u0000"
+        + "\u0000\u0000/\u0001\u0000\u0000\u0000\u00001\u0001\u0000\u0000\u0000\u0000"
+        + "3\u0001\u0000\u0000\u0000\u00005\u0001\u0000\u0000\u0000\u00007\u0001"
+        + "\u0000\u0000\u0000\u00009\u0001\u0000\u0000\u0000\u0000;\u0001\u0000\u0000"
+        + "\u0000\u0000=\u0001\u0000\u0000\u0000\u0000?\u0001\u0000\u0000\u0000\u0000"
+        + "A\u0001\u0000\u0000\u0000\u0000C\u0001\u0000\u0000\u0000\u0000E\u0001"
+        + "\u0000\u0000\u0000\u0000G\u0001\u0000\u0000\u0000\u0000I\u0001\u0000\u0000"
+        + "\u0000\u0000K\u0001\u0000\u0000\u0000\u0000M\u0001\u0000\u0000\u0000\u0000"
+        + "O\u0001\u0000\u0000\u0000\u0000Q\u0001\u0000\u0000\u0000\u0000S\u0001"
+        + "\u0000\u0000\u0000\u0000U\u0001\u0000\u0000\u0000\u0000W\u0001\u0000\u0000"
+        + "\u0000\u0000Y\u0001\u0000\u0000\u0000\u0000[\u0001\u0000\u0000\u0000\u0000"
+        + "]\u0001\u0000\u0000\u0000\u0000_\u0001\u0000\u0000\u0000\u0000a\u0001"
+        + "\u0000\u0000\u0000\u0000c\u0001\u0000\u0000\u0000\u0000e\u0001\u0000\u0000"
+        + "\u0000\u0000g\u0001\u0000\u0000\u0000\u0000i\u0001\u0000\u0000\u0000\u0000"
+        + "k\u0001\u0000\u0000\u0000\u0000m\u0001\u0000\u0000\u0000\u0000o\u0001"
+        + "\u0000\u0000\u0000\u0000q\u0001\u0000\u0000\u0000\u0000s\u0001\u0000\u0000"
+        + "\u0000\u0000u\u0001\u0000\u0000\u0000\u0000w\u0001\u0000\u0000\u0000\u0000"
+        + "y\u0001\u0000\u0000\u0000\u0000{\u0001\u0000\u0000\u0000\u0000}\u0001"
+        + "\u0000\u0000\u0000\u0000\u007f\u0001\u0000\u0000\u0000\u0000\u0081\u0001"
+        + "\u0000\u0000\u0000\u0000\u0083\u0001\u0000\u0000\u0000\u0000\u0085\u0001"
+        + "\u0000\u0000\u0000\u0000\u0087\u0001\u0000\u0000\u0000\u0000\u0089\u0001"
+        + "\u0000\u0000\u0000\u0000\u008b\u0001\u0000\u0000\u0000\u0000\u008d\u0001"
+        + "\u0000\u0000\u0000\u0000\u008f\u0001\u0000\u0000\u0000\u0000\u0091\u0001"
+        + "\u0000\u0000\u0000\u0000\u0093\u0001\u0000\u0000\u0000\u0000\u0095\u0001"
+        + "\u0000\u0000\u0000\u0000\u0097\u0001\u0000\u0000\u0000\u0000\u0099\u0001"
+        + "\u0000\u0000\u0000\u0000\u009b\u0001\u0000\u0000\u0000\u0000\u009d\u0001"
+        + "\u0000\u0000\u0000\u0000\u009f\u0001\u0000\u0000\u0000\u0000\u00a1\u0001"
+        + "\u0000\u0000\u0000\u0000\u00a3\u0001\u0000\u0000\u0000\u0000\u00a5\u0001"
+        + "\u0000\u0000\u0000\u0000\u00a7\u0001\u0000\u0000\u0000\u0000\u00a9\u0001"
+        + "\u0000\u0000\u0000\u0000\u00ab\u0001\u0000\u0000\u0000\u0000\u00ad\u0001"
+        + "\u0000\u0000\u0000\u0000\u00af\u0001\u0000\u0000\u0000\u0000\u00b1\u0001"
+        + "\u0000\u0000\u0000\u0000\u00b3\u0001\u0000\u0000\u0000\u0000\u00b5\u0001"
+        + "\u0000\u0000\u0000\u0000\u00b7\u0001\u0000\u0000\u0000\u0000\u00b9\u0001"
+        + "\u0000\u0000\u0000\u0000\u00bb\u0001\u0000\u0000\u0000\u0000\u00bd\u0001"
+        + "\u0000\u0000\u0000\u0000\u00bf\u0001\u0000\u0000\u0000\u0000\u00c1\u0001"
+        + "\u0000\u0000\u0000\u0000\u00c3\u0001\u0000\u0000\u0000\u0000\u00c5\u0001"
+        + "\u0000\u0000\u0000\u0000\u00c7\u0001\u0000\u0000\u0000\u0000\u00c9\u0001"
+        + "\u0000\u0000\u0000\u0000\u00cb\u0001\u0000\u0000\u0000\u0000\u00cd\u0001"
+        + "\u0000\u0000\u0000\u0000\u00cf\u0001\u0000\u0000\u0000\u0000\u00d1\u0001"
+        + "\u0000\u0000\u0000\u0000\u00d3\u0001\u0000\u0000\u0000\u0000\u00d5\u0001"
+        + "\u0000\u0000\u0000\u0000\u00d7\u0001\u0000\u0000\u0000\u0000\u00d9\u0001"
+        + "\u0000\u0000\u0000\u0000\u00db\u0001\u0000\u0000\u0000\u0000\u00dd\u0001"
+        + "\u0000\u0000\u0000\u0000\u00df\u0001\u0000\u0000\u0000\u0000\u00e1\u0001"
+        + "\u0000\u0000\u0000\u0000\u00e3\u0001\u0000\u0000\u0000\u0000\u00e5\u0001"
+        + "\u0000\u0000\u0000\u0000\u00e7\u0001\u0000\u0000\u0000\u0000\u00e9\u0001"
+        + "\u0000\u0000\u0000\u0000\u00eb\u0001\u0000\u0000\u0000\u0000\u00ed\u0001"
+        + "\u0000\u0000\u0000\u0000\u00ef\u0001\u0000\u0000\u0000\u0000\u00f1\u0001"
+        + "\u0000\u0000\u0000\u0000\u00f3\u0001\u0000\u0000\u0000\u0000\u00f5\u0001"
+        + "\u0000\u0000\u0000\u0000\u00f7\u0001\u0000\u0000\u0000\u0000\u00f9\u0001"
+        + "\u0000\u0000\u0000\u0000\u00fb\u0001\u0000\u0000\u0000\u0000\u00fd\u0001"
+        + "\u0000\u0000\u0000\u0000\u00ff\u0001\u0000\u0000\u0000\u0000\u0101\u0001"
+        + "\u0000\u0000\u0000\u0000\u0103\u0001\u0000\u0000\u0000\u0000\u0105\u0001"
+        + "\u0000\u0000\u0000\u0000\u0107\u0001\u0000\u0000\u0000\u0000\u0109\u0001"
+        + "\u0000\u0000\u0000\u0000\u010b\u0001\u0000\u0000\u0000\u0000\u0113\u0001"
+        + "\u0000\u0000\u0000\u0000\u0115\u0001\u0000\u0000\u0000\u0000\u0117\u0001"
+        + "\u0000\u0000\u0000\u0000\u0119\u0001\u0000\u0000\u0000\u0001\u011b\u0001"
+        + "\u0000\u0000\u0000\u0003\u011d\u0001\u0000\u0000\u0000\u0005\u011f\u0001"
+        + "\u0000\u0000\u0000\u0007\u0121\u0001\u0000\u0000\u0000\t\u0123\u0001\u0000"
+        + "\u0000\u0000\u000b\u0127\u0001\u0000\u0000\u0000\r\u012f\u0001\u0000\u0000"
+        + "\u0000\u000f\u0138\u0001\u0000\u0000\u0000\u0011\u013c\u0001\u0000\u0000"
+        + "\u0000\u0013\u0140\u0001\u0000\u0000\u0000\u0015\u0143\u0001\u0000\u0000"
+        + "\u0000\u0017\u0147\u0001\u0000\u0000\u0000\u0019\u014f\u0001\u0000\u0000"
+        + "\u0000\u001b\u0152\u0001\u0000\u0000\u0000\u001d\u0157\u0001\u0000\u0000"
+        + "\u0000\u001f\u015c\u0001\u0000\u0000\u0000!\u0164\u0001\u0000\u0000\u0000"
+        + "#\u016d\u0001\u0000\u0000\u0000%\u0175\u0001\u0000\u0000\u0000\'\u017d"
+        + "\u0001\u0000\u0000\u0000)\u018a\u0001\u0000\u0000\u0000+\u0197\u0001\u0000"
+        + "\u0000\u0000-\u01a9\u0001\u0000\u0000\u0000/\u01ad\u0001\u0000\u0000\u0000"
+        + "1\u01b2\u0001\u0000\u0000\u00003\u01b8\u0001\u0000\u0000\u00005\u01bd"
+        + "\u0001\u0000\u0000\u00007\u01c6\u0001\u0000\u0000\u00009\u01cf\u0001\u0000"
+        + "\u0000\u0000;\u01d4\u0001\u0000\u0000\u0000=\u01d8\u0001\u0000\u0000\u0000"
+        + "?\u01df\u0001\u0000\u0000\u0000A\u01ea\u0001\u0000\u0000\u0000C\u01f1"
+        + "\u0001\u0000\u0000\u0000E\u01f9\u0001\u0000\u0000\u0000G\u0201\u0001\u0000"
+        + "\u0000\u0000I\u0207\u0001\u0000\u0000\u0000K\u020d\u0001\u0000\u0000\u0000"
+        + "M\u0211\u0001\u0000\u0000\u0000O\u0218\u0001\u0000\u0000\u0000Q\u021d"
+        + "\u0001\u0000\u0000\u0000S\u0224\u0001\u0000\u0000\u0000U\u0229\u0001\u0000"
+        + "\u0000\u0000W\u0233\u0001\u0000\u0000\u0000Y\u023c\u0001\u0000\u0000\u0000"
+        + "[\u0242\u0001\u0000\u0000\u0000]\u0249\u0001\u0000\u0000\u0000_\u024e"
+        + "\u0001\u0000\u0000\u0000a\u0254\u0001\u0000\u0000\u0000c\u0257\u0001\u0000"
+        + "\u0000\u0000e\u025f\u0001\u0000\u0000\u0000g\u0265\u0001\u0000\u0000\u0000"
+        + "i\u026e\u0001\u0000\u0000\u0000k\u0271\u0001\u0000\u0000\u0000m\u0276"
+        + "\u0001\u0000\u0000\u0000o\u027b\u0001\u0000\u0000\u0000q\u0280\u0001\u0000"
+        + "\u0000\u0000s\u0285\u0001\u0000\u0000\u0000u\u028b\u0001\u0000\u0000\u0000"
+        + "w\u0292\u0001\u0000\u0000\u0000y\u0298\u0001\u0000\u0000\u0000{\u029f"
+        + "\u0001\u0000\u0000\u0000}\u02a7\u0001\u0000\u0000\u0000\u007f\u02ad\u0001"
+        + "\u0000\u0000\u0000\u0081\u02b4\u0001\u0000\u0000\u0000\u0083\u02bc\u0001"
+        + "\u0000\u0000\u0000\u0085\u02c0\u0001\u0000\u0000\u0000\u0087\u02c5\u0001"
+        + "\u0000\u0000\u0000\u0089\u02cb\u0001\u0000\u0000\u0000\u008b\u02ce\u0001"
+        + "\u0000\u0000\u0000\u008d\u02d8\u0001\u0000\u0000\u0000\u008f\u02db\u0001"
+        + "\u0000\u0000\u0000\u0091\u02e1\u0001\u0000\u0000\u0000\u0093\u02e7\u0001"
+        + "\u0000\u0000\u0000\u0095\u02ee\u0001\u0000\u0000\u0000\u0097\u02f7\u0001"
+        + "\u0000\u0000\u0000\u0099\u02fd\u0001\u0000\u0000\u0000\u009b\u0302\u0001"
+        + "\u0000\u0000\u0000\u009d\u0308\u0001\u0000\u0000\u0000\u009f\u030e\u0001"
+        + "\u0000\u0000\u0000\u00a1\u0314\u0001\u0000\u0000\u0000\u00a3\u031c\u0001"
+        + "\u0000\u0000\u0000\u00a5\u0323\u0001\u0000\u0000\u0000\u00a7\u032b\u0001"
+        + "\u0000\u0000\u0000\u00a9\u0332\u0001\u0000\u0000\u0000\u00ab\u0337\u0001"
+        + "\u0000\u0000\u0000\u00ad\u033b\u0001\u0000\u0000\u0000\u00af\u0341\u0001"
+        + "\u0000\u0000\u0000\u00b1\u0348\u0001\u0000\u0000\u0000\u00b3\u034d\u0001"
+        + "\u0000\u0000\u0000\u00b5\u0352\u0001\u0000\u0000\u0000\u00b7\u0357\u0001"
+        + "\u0000\u0000\u0000\u00b9\u035a\u0001\u0000\u0000\u0000\u00bb\u035e\u0001"
+        + "\u0000\u0000\u0000\u00bd\u0363\u0001\u0000\u0000\u0000\u00bf\u0369\u0001"
+        + "\u0000\u0000\u0000\u00c1\u036f\u0001\u0000\u0000\u0000\u00c3\u0376\u0001"
+        + "\u0000\u0000\u0000\u00c5\u037b\u0001\u0000\u0000\u0000\u00c7\u0381\u0001"
+        + "\u0000\u0000\u0000\u00c9\u0386\u0001\u0000\u0000\u0000\u00cb\u038b\u0001"
+        + "\u0000\u0000\u0000\u00cd\u0391\u0001\u0000\u0000\u0000\u00cf\u0399\u0001"
+        + "\u0000\u0000\u0000\u00d1\u039d\u0001\u0000\u0000\u0000\u00d3\u03a4\u0001"
+        + "\u0000\u0000\u0000\u00d5\u03a7\u0001\u0000\u0000\u0000\u00d7\u03aa\u0001"
+        + "\u0000\u0000\u0000\u00d9\u03ae\u0001\u0000\u0000\u0000\u00db\u03b4\u0001"
+        + "\u0000\u0000\u0000\u00dd\u03bb\u0001\u0000\u0000\u0000\u00df\u03bd\u0001"
+        + "\u0000\u0000\u0000\u00e1\u03bf\u0001\u0000\u0000\u0000\u00e3\u03c7\u0001"
+        + "\u0000\u0000\u0000\u00e5\u03c9\u0001\u0000\u0000\u0000\u00e7\u03cb\u0001"
+        + "\u0000\u0000\u0000\u00e9\u03ce\u0001\u0000\u0000\u0000\u00eb\u03d0\u0001"
+        + "\u0000\u0000\u0000\u00ed\u03d3\u0001\u0000\u0000\u0000\u00ef\u03d5\u0001"
+        + "\u0000\u0000\u0000\u00f1\u03d7\u0001\u0000\u0000\u0000\u00f3\u03d9\u0001"
+        + "\u0000\u0000\u0000\u00f5\u03db\u0001\u0000\u0000\u0000\u00f7\u03dd\u0001"
+        + "\u0000\u0000\u0000\u00f9\u03e0\u0001\u0000\u0000\u0000\u00fb\u03e2\u0001"
+        + "\u0000\u0000\u0000\u00fd\u03e4\u0001\u0000\u0000\u0000\u00ff\u03f0\u0001"
+        + "\u0000\u0000\u0000\u0101\u041e\u0001\u0000\u0000\u0000\u0103\u0422\u0001"
+        + "\u0000\u0000\u0000\u0105\u042c\u0001\u0000\u0000\u0000\u0107\u0437\u0001"
+        + "\u0000\u0000\u0000\u0109\u043b\u0001\u0000\u0000\u0000\u010b\u0446\u0001"
+        + "\u0000\u0000\u0000\u010d\u0451\u0001\u0000\u0000\u0000\u010f\u045a\u0001"
+        + "\u0000\u0000\u0000\u0111\u045c\u0001\u0000\u0000\u0000\u0113\u045e\u0001"
+        + "\u0000\u0000\u0000\u0115\u046f\u0001\u0000\u0000\u0000\u0117\u047f\u0001"
+        + "\u0000\u0000\u0000\u0119\u0485\u0001\u0000\u0000\u0000\u011b\u011c\u0005"
+        + "(\u0000\u0000\u011c\u0002\u0001\u0000\u0000\u0000\u011d\u011e\u0005)\u0000"
+        + "\u0000\u011e\u0004\u0001\u0000\u0000\u0000\u011f\u0120\u0005,\u0000\u0000"
+        + "\u0120\u0006\u0001\u0000\u0000\u0000\u0121\u0122\u0005:\u0000\u0000\u0122"
+        + "\b\u0001\u0000\u0000\u0000\u0123\u0124\u0005A\u0000\u0000\u0124\u0125"
+        + "\u0005L\u0000\u0000\u0125\u0126\u0005L\u0000\u0000\u0126\n\u0001\u0000"
+        + "\u0000\u0000\u0127\u0128\u0005A\u0000\u0000\u0128\u0129\u0005N\u0000\u0000"
+        + "\u0129\u012a\u0005A\u0000\u0000\u012a\u012b\u0005L\u0000\u0000\u012b\u012c"
+        + "\u0005Y\u0000\u0000\u012c\u012d\u0005Z\u0000\u0000\u012d\u012e\u0005E"
+        + "\u0000\u0000\u012e\f\u0001\u0000\u0000\u0000\u012f\u0130\u0005A\u0000"
+        + "\u0000\u0130\u0131\u0005N\u0000\u0000\u0131\u0132\u0005A\u0000\u0000\u0132"
+        + "\u0133\u0005L\u0000\u0000\u0133\u0134\u0005Y\u0000\u0000\u0134\u0135\u0005"
+        + "Z\u0000\u0000\u0135\u0136\u0005E\u0000\u0000\u0136\u0137\u0005D\u0000"
+        + "\u0000\u0137\u000e\u0001\u0000\u0000\u0000\u0138\u0139\u0005A\u0000\u0000"
+        + "\u0139\u013a\u0005N\u0000\u0000\u013a\u013b\u0005D\u0000\u0000\u013b\u0010"
+        + "\u0001\u0000\u0000\u0000\u013c\u013d\u0005A\u0000\u0000\u013d\u013e\u0005"
+        + "N\u0000\u0000\u013e\u013f\u0005Y\u0000\u0000\u013f\u0012\u0001\u0000\u0000"
+        + "\u0000\u0140\u0141\u0005A\u0000\u0000\u0141\u0142\u0005S\u0000\u0000\u0142"
+        + "\u0014\u0001\u0000\u0000\u0000\u0143\u0144\u0005A\u0000\u0000\u0144\u0145"
+        + "\u0005S\u0000\u0000\u0145\u0146\u0005C\u0000\u0000\u0146\u0016\u0001\u0000"
+        + "\u0000\u0000\u0147\u0148\u0005B\u0000\u0000\u0148\u0149\u0005E\u0000\u0000"
+        + "\u0149\u014a\u0005T\u0000\u0000\u014a\u014b\u0005W\u0000\u0000\u014b\u014c"
+        + "\u0005E\u0000\u0000\u014c\u014d\u0005E\u0000\u0000\u014d\u014e\u0005N"
+        + "\u0000\u0000\u014e\u0018\u0001\u0000\u0000\u0000\u014f\u0150\u0005B\u0000"
+        + "\u0000\u0150\u0151\u0005Y\u0000\u0000\u0151\u001a\u0001\u0000\u0000\u0000"
+        + "\u0152\u0153\u0005C\u0000\u0000\u0153\u0154\u0005A\u0000\u0000\u0154\u0155"
+        + "\u0005S\u0000\u0000\u0155\u0156\u0005E\u0000\u0000\u0156\u001c\u0001\u0000"
+        + "\u0000\u0000\u0157\u0158\u0005C\u0000\u0000\u0158\u0159\u0005A\u0000\u0000"
+        + "\u0159\u015a\u0005S\u0000\u0000\u015a\u015b\u0005T\u0000\u0000\u015b\u001e"
+        + "\u0001\u0000\u0000\u0000\u015c\u015d\u0005C\u0000\u0000\u015d\u015e\u0005"
+        + "A\u0000\u0000\u015e\u015f\u0005T\u0000\u0000\u015f\u0160\u0005A\u0000"
+        + "\u0000\u0160\u0161\u0005L\u0000\u0000\u0161\u0162\u0005O\u0000\u0000\u0162"
+        + "\u0163\u0005G\u0000\u0000\u0163 \u0001\u0000\u0000\u0000\u0164\u0165\u0005"
+        + "C\u0000\u0000\u0165\u0166\u0005A\u0000\u0000\u0166\u0167\u0005T\u0000"
+        + "\u0000\u0167\u0168\u0005A\u0000\u0000\u0168\u0169\u0005L\u0000\u0000\u0169"
+        + "\u016a\u0005O\u0000\u0000\u016a\u016b\u0005G\u0000\u0000\u016b\u016c\u0005"
+        + "S\u0000\u0000\u016c\"\u0001\u0000\u0000\u0000\u016d\u016e\u0005C\u0000"
+        + "\u0000\u016e\u016f\u0005O\u0000\u0000\u016f\u0170\u0005L\u0000\u0000\u0170"
+        + "\u0171\u0005U\u0000\u0000\u0171\u0172\u0005M\u0000\u0000\u0172\u0173\u0005"
+        + "N\u0000\u0000\u0173\u0174\u0005S\u0000\u0000\u0174$\u0001\u0000\u0000"
+        + "\u0000\u0175\u0176\u0005C\u0000\u0000\u0176\u0177\u0005O\u0000\u0000\u0177"
+        + "\u0178\u0005N\u0000\u0000\u0178\u0179\u0005V\u0000\u0000\u0179\u017a\u0005"
+        + "E\u0000\u0000\u017a\u017b\u0005R\u0000\u0000\u017b\u017c\u0005T\u0000"
+        + "\u0000\u017c&\u0001\u0000\u0000\u0000\u017d\u017e\u0005C\u0000\u0000\u017e"
+        + "\u017f\u0005U\u0000\u0000\u017f\u0180\u0005R\u0000\u0000\u0180\u0181\u0005"
+        + "R\u0000\u0000\u0181\u0182\u0005E\u0000\u0000\u0182\u0183\u0005N\u0000"
+        + "\u0000\u0183\u0184\u0005T\u0000\u0000\u0184\u0185\u0005_\u0000\u0000\u0185"
+        + "\u0186\u0005D\u0000\u0000\u0186\u0187\u0005A\u0000\u0000\u0187\u0188\u0005"
+        + "T\u0000\u0000\u0188\u0189\u0005E\u0000\u0000\u0189(\u0001\u0000\u0000"
+        + "\u0000\u018a\u018b\u0005C\u0000\u0000\u018b\u018c\u0005U\u0000\u0000\u018c"
+        + "\u018d\u0005R\u0000\u0000\u018d\u018e\u0005R\u0000\u0000\u018e\u018f\u0005"
+        + "E\u0000\u0000\u018f\u0190\u0005N\u0000\u0000\u0190\u0191\u0005T\u0000"
+        + "\u0000\u0191\u0192\u0005_\u0000\u0000\u0192\u0193\u0005T\u0000\u0000\u0193"
+        + "\u0194\u0005I\u0000\u0000\u0194\u0195\u0005M\u0000\u0000\u0195\u0196\u0005"
+        + "E\u0000\u0000\u0196*\u0001\u0000\u0000\u0000\u0197\u0198\u0005C\u0000"
+        + "\u0000\u0198\u0199\u0005U\u0000\u0000\u0199\u019a\u0005R\u0000\u0000\u019a"
+        + "\u019b\u0005R\u0000\u0000\u019b\u019c\u0005E\u0000\u0000\u019c\u019d\u0005"
+        + "N\u0000\u0000\u019d\u019e\u0005T\u0000\u0000\u019e\u019f\u0005_\u0000"
+        + "\u0000\u019f\u01a0\u0005T\u0000\u0000\u01a0\u01a1\u0005I\u0000\u0000\u01a1"
+        + "\u01a2\u0005M\u0000\u0000\u01a2\u01a3\u0005E\u0000\u0000\u01a3\u01a4\u0005"
+        + "S\u0000\u0000\u01a4\u01a5\u0005T\u0000\u0000\u01a5\u01a6\u0005A\u0000"
+        + "\u0000\u01a6\u01a7\u0005M\u0000\u0000\u01a7\u01a8\u0005P\u0000\u0000\u01a8"
+        + ",\u0001\u0000\u0000\u0000\u01a9\u01aa\u0005D\u0000\u0000\u01aa\u01ab\u0005"
+        + "A\u0000\u0000\u01ab\u01ac\u0005Y\u0000\u0000\u01ac.\u0001\u0000\u0000"
+        + "\u0000\u01ad\u01ae\u0005D\u0000\u0000\u01ae\u01af\u0005A\u0000\u0000\u01af"
+        + "\u01b0\u0005Y\u0000\u0000\u01b0\u01b1\u0005S\u0000\u0000\u01b10\u0001"
+        + "\u0000\u0000\u0000\u01b2\u01b3\u0005D\u0000\u0000\u01b3\u01b4\u0005E\u0000"
+        + "\u0000\u01b4\u01b5\u0005B\u0000\u0000\u01b5\u01b6\u0005U\u0000\u0000\u01b6"
+        + "\u01b7\u0005G\u0000\u0000\u01b72\u0001\u0000\u0000\u0000\u01b8\u01b9\u0005"
+        + "D\u0000\u0000\u01b9\u01ba\u0005E\u0000\u0000\u01ba\u01bb\u0005S\u0000"
+        + "\u0000\u01bb\u01bc\u0005C\u0000\u0000\u01bc4\u0001\u0000\u0000\u0000\u01bd"
+        + "\u01be\u0005D\u0000\u0000\u01be\u01bf\u0005E\u0000\u0000\u01bf\u01c0\u0005"
+        + "S\u0000\u0000\u01c0\u01c1\u0005C\u0000\u0000\u01c1\u01c2\u0005R\u0000"
+        + "\u0000\u01c2\u01c3\u0005I\u0000\u0000\u01c3\u01c4\u0005B\u0000\u0000\u01c4"
+        + "\u01c5\u0005E\u0000\u0000\u01c56\u0001\u0000\u0000\u0000\u01c6\u01c7\u0005"
+        + "D\u0000\u0000\u01c7\u01c8\u0005I\u0000\u0000\u01c8\u01c9\u0005S\u0000"
+        + "\u0000\u01c9\u01ca\u0005T\u0000\u0000\u01ca\u01cb\u0005I\u0000\u0000\u01cb"
+        + "\u01cc\u0005N\u0000\u0000\u01cc\u01cd\u0005C\u0000\u0000\u01cd\u01ce\u0005"
+        + "T\u0000\u0000\u01ce8\u0001\u0000\u0000\u0000\u01cf\u01d0\u0005E\u0000"
+        + "\u0000\u01d0\u01d1\u0005L\u0000\u0000\u01d1\u01d2\u0005S\u0000\u0000\u01d2"
+        + "\u01d3\u0005E\u0000\u0000\u01d3:\u0001\u0000\u0000\u0000\u01d4\u01d5\u0005"
+        + "E\u0000\u0000\u01d5\u01d6\u0005N\u0000\u0000\u01d6\u01d7\u0005D\u0000"
+        + "\u0000\u01d7<\u0001\u0000\u0000\u0000\u01d8\u01d9\u0005E\u0000\u0000\u01d9"
+        + "\u01da\u0005S\u0000\u0000\u01da\u01db\u0005C\u0000\u0000\u01db\u01dc\u0005"
+        + "A\u0000\u0000\u01dc\u01dd\u0005P\u0000\u0000\u01dd\u01de\u0005E\u0000"
+        + "\u0000\u01de>\u0001\u0000\u0000\u0000\u01df\u01e0\u0005E\u0000\u0000\u01e0"
+        + "\u01e1\u0005X\u0000\u0000\u01e1\u01e2\u0005E\u0000\u0000\u01e2\u01e3\u0005"
+        + "C\u0000\u0000\u01e3\u01e4\u0005U\u0000\u0000\u01e4\u01e5\u0005T\u0000"
+        + "\u0000\u01e5\u01e6\u0005A\u0000\u0000\u01e6\u01e7\u0005B\u0000\u0000\u01e7"
+        + "\u01e8\u0005L\u0000\u0000\u01e8\u01e9\u0005E\u0000\u0000\u01e9@\u0001"
+        + "\u0000\u0000\u0000\u01ea\u01eb\u0005E\u0000\u0000\u01eb\u01ec\u0005X\u0000"
+        + "\u0000\u01ec\u01ed\u0005I\u0000\u0000\u01ed\u01ee\u0005S\u0000\u0000\u01ee"
+        + "\u01ef\u0005T\u0000\u0000\u01ef\u01f0\u0005S\u0000\u0000\u01f0B\u0001"
+        + "\u0000\u0000\u0000\u01f1\u01f2\u0005E\u0000\u0000\u01f2\u01f3\u0005X\u0000"
+        + "\u0000\u01f3\u01f4\u0005P\u0000\u0000\u01f4\u01f5\u0005L\u0000\u0000\u01f5"
+        + "\u01f6\u0005A\u0000\u0000\u01f6\u01f7\u0005I\u0000\u0000\u01f7\u01f8\u0005"
+        + "N\u0000\u0000\u01f8D\u0001\u0000\u0000\u0000\u01f9\u01fa\u0005E\u0000"
+        + "\u0000\u01fa\u01fb\u0005X\u0000\u0000\u01fb\u01fc\u0005T\u0000\u0000\u01fc"
+        + "\u01fd\u0005R\u0000\u0000\u01fd\u01fe\u0005A\u0000\u0000\u01fe\u01ff\u0005"
+        + "C\u0000\u0000\u01ff\u0200\u0005T\u0000\u0000\u0200F\u0001\u0000\u0000"
+        + "\u0000\u0201\u0202\u0005F\u0000\u0000\u0202\u0203\u0005A\u0000\u0000\u0203"
+        + "\u0204\u0005L\u0000\u0000\u0204\u0205\u0005S\u0000\u0000\u0205\u0206\u0005"
+        + "E\u0000\u0000\u0206H\u0001\u0000\u0000\u0000\u0207\u0208\u0005F\u0000"
+        + "\u0000\u0208\u0209\u0005I\u0000\u0000\u0209\u020a\u0005R\u0000\u0000\u020a"
+        + "\u020b\u0005S\u0000\u0000\u020b\u020c\u0005T\u0000\u0000\u020cJ\u0001"
+        + "\u0000\u0000\u0000\u020d\u020e\u0005F\u0000\u0000\u020e\u020f\u0005O\u0000"
+        + "\u0000\u020f\u0210\u0005R\u0000\u0000\u0210L\u0001\u0000\u0000\u0000\u0211"
+        + "\u0212\u0005F\u0000\u0000\u0212\u0213\u0005O\u0000\u0000\u0213\u0214\u0005"
+        + "R\u0000\u0000\u0214\u0215\u0005M\u0000\u0000\u0215\u0216\u0005A\u0000"
+        + "\u0000\u0216\u0217\u0005T\u0000\u0000\u0217N\u0001\u0000\u0000\u0000\u0218"
+        + "\u0219\u0005F\u0000\u0000\u0219\u021a\u0005R\u0000\u0000\u021a\u021b\u0005"
+        + "O\u0000\u0000\u021b\u021c\u0005M\u0000\u0000\u021cP\u0001\u0000\u0000"
+        + "\u0000\u021d\u021e\u0005F\u0000\u0000\u021e\u021f\u0005R\u0000\u0000\u021f"
+        + "\u0220\u0005O\u0000\u0000\u0220\u0221\u0005Z\u0000\u0000\u0221\u0222\u0005"
+        + "E\u0000\u0000\u0222\u0223\u0005N\u0000\u0000\u0223R\u0001\u0000\u0000"
+        + "\u0000\u0224\u0225\u0005F\u0000\u0000\u0225\u0226\u0005U\u0000\u0000\u0226"
+        + "\u0227\u0005L\u0000\u0000\u0227\u0228\u0005L\u0000\u0000\u0228T\u0001"
+        + "\u0000\u0000\u0000\u0229\u022a\u0005F\u0000\u0000\u022a\u022b\u0005U\u0000"
+        + "\u0000\u022b\u022c\u0005N\u0000\u0000\u022c\u022d\u0005C\u0000\u0000\u022d"
+        + "\u022e\u0005T\u0000\u0000\u022e\u022f\u0005I\u0000\u0000\u022f\u0230\u0005"
+        + "O\u0000\u0000\u0230\u0231\u0005N\u0000\u0000\u0231\u0232\u0005S\u0000"
+        + "\u0000\u0232V\u0001\u0000\u0000\u0000\u0233\u0234\u0005G\u0000\u0000\u0234"
+        + "\u0235\u0005R\u0000\u0000\u0235\u0236\u0005A\u0000\u0000\u0236\u0237\u0005"
+        + "P\u0000\u0000\u0237\u0238\u0005H\u0000\u0000\u0238\u0239\u0005V\u0000"
+        + "\u0000\u0239\u023a\u0005I\u0000\u0000\u023a\u023b\u0005Z\u0000\u0000\u023b"
+        + "X\u0001\u0000\u0000\u0000\u023c\u023d\u0005G\u0000\u0000\u023d\u023e\u0005"
+        + "R\u0000\u0000\u023e\u023f\u0005O\u0000\u0000\u023f\u0240\u0005U\u0000"
+        + "\u0000\u0240\u0241\u0005P\u0000\u0000\u0241Z\u0001\u0000\u0000\u0000\u0242"
+        + "\u0243\u0005H\u0000\u0000\u0243\u0244\u0005A\u0000\u0000\u0244\u0245\u0005"
+        + "V\u0000\u0000\u0245\u0246\u0005I\u0000\u0000\u0246\u0247\u0005N\u0000"
+        + "\u0000\u0247\u0248\u0005G\u0000\u0000\u0248\\\u0001\u0000\u0000\u0000"
+        + "\u0249\u024a\u0005H\u0000\u0000\u024a\u024b\u0005O\u0000\u0000\u024b\u024c"
+        + "\u0005U\u0000\u0000\u024c\u024d\u0005R\u0000\u0000\u024d^\u0001\u0000"
+        + "\u0000\u0000\u024e\u024f\u0005H\u0000\u0000\u024f\u0250\u0005O\u0000\u0000"
+        + "\u0250\u0251\u0005U\u0000\u0000\u0251\u0252\u0005R\u0000\u0000\u0252\u0253"
+        + "\u0005S\u0000\u0000\u0253`\u0001\u0000\u0000\u0000\u0254\u0255\u0005I"
+        + "\u0000\u0000\u0255\u0256\u0005N\u0000\u0000\u0256b\u0001\u0000\u0000\u0000"
+        + "\u0257\u0258\u0005I\u0000\u0000\u0258\u0259\u0005N\u0000\u0000\u0259\u025a"
+        + "\u0005C\u0000\u0000\u025a\u025b\u0005L\u0000\u0000\u025b\u025c\u0005U"
+        + "\u0000\u0000\u025c\u025d\u0005D\u0000\u0000\u025d\u025e\u0005E\u0000\u0000"
+        + "\u025ed\u0001\u0000\u0000\u0000\u025f\u0260\u0005I\u0000\u0000\u0260\u0261"
+        + "\u0005N\u0000\u0000\u0261\u0262\u0005N\u0000\u0000\u0262\u0263\u0005E"
+        + "\u0000\u0000\u0263\u0264\u0005R\u0000\u0000\u0264f\u0001\u0000\u0000\u0000"
+        + "\u0265\u0266\u0005I\u0000\u0000\u0266\u0267\u0005N\u0000\u0000\u0267\u0268"
+        + "\u0005T\u0000\u0000\u0268\u0269\u0005E\u0000\u0000\u0269\u026a\u0005R"
+        + "\u0000\u0000\u026a\u026b\u0005V\u0000\u0000\u026b\u026c\u0005A\u0000\u0000"
+        + "\u026c\u026d\u0005L\u0000\u0000\u026dh\u0001\u0000\u0000\u0000\u026e\u026f"
+        + "\u0005I\u0000\u0000\u026f\u0270\u0005S\u0000\u0000\u0270j\u0001\u0000"
+        + "\u0000\u0000\u0271\u0272\u0005J\u0000\u0000\u0272\u0273\u0005O\u0000\u0000"
+        + "\u0273\u0274\u0005I\u0000\u0000\u0274\u0275\u0005N\u0000\u0000\u0275l"
+        + "\u0001\u0000\u0000\u0000\u0276\u0277\u0005L\u0000\u0000\u0277\u0278\u0005"
+        + "A\u0000\u0000\u0278\u0279\u0005S\u0000\u0000\u0279\u027a\u0005T\u0000"
+        + "\u0000\u027an\u0001\u0000\u0000\u0000\u027b\u027c\u0005L\u0000\u0000\u027c"
+        + "\u027d\u0005E\u0000\u0000\u027d\u027e\u0005F\u0000\u0000\u027e\u027f\u0005"
+        + "T\u0000\u0000\u027fp\u0001\u0000\u0000\u0000\u0280\u0281\u0005L\u0000"
+        + "\u0000\u0281\u0282\u0005I\u0000\u0000\u0282\u0283\u0005K\u0000\u0000\u0283"
+        + "\u0284\u0005E\u0000\u0000\u0284r\u0001\u0000\u0000\u0000\u0285\u0286\u0005"
+        + "L\u0000\u0000\u0286\u0287\u0005I\u0000\u0000\u0287\u0288\u0005M\u0000"
+        + "\u0000\u0288\u0289\u0005I\u0000\u0000\u0289\u028a\u0005T\u0000\u0000\u028a"
+        + "t\u0001\u0000\u0000\u0000\u028b\u028c\u0005M\u0000\u0000\u028c\u028d\u0005"
+        + "A\u0000\u0000\u028d\u028e\u0005P\u0000\u0000\u028e\u028f\u0005P\u0000"
+        + "\u0000\u028f\u0290\u0005E\u0000\u0000\u0290\u0291\u0005D\u0000\u0000\u0291"
+        + "v\u0001\u0000\u0000\u0000\u0292\u0293\u0005M\u0000\u0000\u0293\u0294\u0005"
+        + "A\u0000\u0000\u0294\u0295\u0005T\u0000\u0000\u0295\u0296\u0005C\u0000"
+        + "\u0000\u0296\u0297\u0005H\u0000\u0000\u0297x\u0001\u0000\u0000\u0000\u0298"
+        + "\u0299\u0005M\u0000\u0000\u0299\u029a\u0005I\u0000\u0000\u029a\u029b\u0005"
+        + "N\u0000\u0000\u029b\u029c\u0005U\u0000\u0000\u029c\u029d\u0005T\u0000"
+        + "\u0000\u029d\u029e\u0005E\u0000\u0000\u029ez\u0001\u0000\u0000\u0000\u029f"
+        + "\u02a0\u0005M\u0000\u0000\u02a0\u02a1\u0005I\u0000\u0000\u02a1\u02a2\u0005"
+        + "N\u0000\u0000\u02a2\u02a3\u0005U\u0000\u0000\u02a3\u02a4\u0005T\u0000"
+        + "\u0000\u02a4\u02a5\u0005E\u0000\u0000\u02a5\u02a6\u0005S\u0000\u0000\u02a6"
+        + "|\u0001\u0000\u0000\u0000\u02a7\u02a8\u0005M\u0000\u0000\u02a8\u02a9\u0005"
+        + "O\u0000\u0000\u02a9\u02aa\u0005N\u0000\u0000\u02aa\u02ab\u0005T\u0000"
+        + "\u0000\u02ab\u02ac\u0005H\u0000\u0000\u02ac~\u0001\u0000\u0000\u0000\u02ad"
+        + "\u02ae\u0005M\u0000\u0000\u02ae\u02af\u0005O\u0000\u0000\u02af\u02b0\u0005"
+        + "N\u0000\u0000\u02b0\u02b1\u0005T\u0000\u0000\u02b1\u02b2\u0005H\u0000"
+        + "\u0000\u02b2\u02b3\u0005S\u0000\u0000\u02b3\u0080\u0001\u0000\u0000\u0000"
+        + "\u02b4\u02b5\u0005N\u0000\u0000\u02b5\u02b6\u0005A\u0000\u0000\u02b6\u02b7"
+        + "\u0005T\u0000\u0000\u02b7\u02b8\u0005U\u0000\u0000\u02b8\u02b9\u0005R"
+        + "\u0000\u0000\u02b9\u02ba\u0005A\u0000\u0000\u02ba\u02bb\u0005L\u0000\u0000"
+        + "\u02bb\u0082\u0001\u0000\u0000\u0000\u02bc\u02bd\u0005N\u0000\u0000\u02bd"
+        + "\u02be\u0005O\u0000\u0000\u02be\u02bf\u0005T\u0000\u0000\u02bf\u0084\u0001"
+        + "\u0000\u0000\u0000\u02c0\u02c1\u0005N\u0000\u0000\u02c1\u02c2\u0005U\u0000"
+        + "\u0000\u02c2\u02c3\u0005L\u0000\u0000\u02c3\u02c4\u0005L\u0000\u0000\u02c4"
+        + "\u0086\u0001\u0000\u0000\u0000\u02c5\u02c6\u0005N\u0000\u0000\u02c6\u02c7"
+        + "\u0005U\u0000\u0000\u02c7\u02c8\u0005L\u0000\u0000\u02c8\u02c9\u0005L"
+        + "\u0000\u0000\u02c9\u02ca\u0005S\u0000\u0000\u02ca\u0088\u0001\u0000\u0000"
+        + "\u0000\u02cb\u02cc\u0005O\u0000\u0000\u02cc\u02cd\u0005N\u0000\u0000\u02cd"
+        + "\u008a\u0001\u0000\u0000\u0000\u02ce\u02cf\u0005O\u0000\u0000\u02cf\u02d0"
+        + "\u0005P\u0000\u0000\u02d0\u02d1\u0005T\u0000\u0000\u02d1\u02d2\u0005I"
+        + "\u0000\u0000\u02d2\u02d3\u0005M\u0000\u0000\u02d3\u02d4\u0005I\u0000\u0000"
+        + "\u02d4\u02d5\u0005Z\u0000\u0000\u02d5\u02d6\u0005E\u0000\u0000\u02d6\u02d7"
+        + "\u0005D\u0000\u0000\u02d7\u008c\u0001\u0000\u0000\u0000\u02d8\u02d9\u0005"
+        + "O\u0000\u0000\u02d9\u02da\u0005R\u0000\u0000\u02da\u008e\u0001\u0000\u0000"
+        + "\u0000\u02db\u02dc\u0005O\u0000\u0000\u02dc\u02dd\u0005R\u0000\u0000\u02dd"
+        + "\u02de\u0005D\u0000\u0000\u02de\u02df\u0005E\u0000\u0000\u02df\u02e0\u0005"
+        + "R\u0000\u0000\u02e0\u0090\u0001\u0000\u0000\u0000\u02e1\u02e2\u0005O\u0000"
+        + "\u0000\u02e2\u02e3\u0005U\u0000\u0000\u02e3\u02e4\u0005T\u0000\u0000\u02e4"
+        + "\u02e5\u0005E\u0000\u0000\u02e5\u02e6\u0005R\u0000\u0000\u02e6\u0092\u0001"
+        + "\u0000\u0000\u0000\u02e7\u02e8\u0005P\u0000\u0000\u02e8\u02e9\u0005A\u0000"
+        + "\u0000\u02e9\u02ea\u0005R\u0000\u0000\u02ea\u02eb\u0005S\u0000\u0000\u02eb"
+        + "\u02ec\u0005E\u0000\u0000\u02ec\u02ed\u0005D\u0000\u0000\u02ed\u0094\u0001"
+        + "\u0000\u0000\u0000\u02ee\u02ef\u0005P\u0000\u0000\u02ef\u02f0\u0005H\u0000"
+        + "\u0000\u02f0\u02f1\u0005Y\u0000\u0000\u02f1\u02f2\u0005S\u0000\u0000\u02f2"
+        + "\u02f3\u0005I\u0000\u0000\u02f3\u02f4\u0005C\u0000\u0000\u02f4\u02f5\u0005"
+        + "A\u0000\u0000\u02f5\u02f6\u0005L\u0000\u0000\u02f6\u0096\u0001\u0000\u0000"
+        + "\u0000\u02f7\u02f8\u0005P\u0000\u0000\u02f8\u02f9\u0005I\u0000\u0000\u02f9"
+        + "\u02fa\u0005V\u0000\u0000\u02fa\u02fb\u0005O\u0000\u0000\u02fb\u02fc\u0005"
+        + "T\u0000\u0000\u02fc\u0098\u0001\u0000\u0000\u0000\u02fd\u02fe\u0005P\u0000"
+        + "\u0000\u02fe\u02ff\u0005L\u0000\u0000\u02ff\u0300\u0005A\u0000\u0000\u0300"
+        + "\u0301\u0005N\u0000\u0000\u0301\u009a\u0001\u0000\u0000\u0000\u0302\u0303"
+        + "\u0005R\u0000\u0000\u0303\u0304\u0005I\u0000\u0000\u0304\u0305\u0005G"
+        + "\u0000\u0000\u0305\u0306\u0005H\u0000\u0000\u0306\u0307\u0005T\u0000\u0000"
+        + "\u0307\u009c\u0001\u0000\u0000\u0000\u0308\u0309\u0005R\u0000\u0000\u0309"
+        + "\u030a\u0005L\u0000\u0000\u030a\u030b\u0005I\u0000\u0000\u030b\u030c\u0005"
+        + "K\u0000\u0000\u030c\u030d\u0005E\u0000\u0000\u030d\u009e\u0001\u0000\u0000"
+        + "\u0000\u030e\u030f\u0005Q\u0000\u0000\u030f\u0310\u0005U\u0000\u0000\u0310"
+        + "\u0311\u0005E\u0000\u0000\u0311\u0312\u0005R\u0000\u0000\u0312\u0313\u0005"
+        + "Y\u0000\u0000\u0313\u00a0\u0001\u0000\u0000\u0000\u0314\u0315\u0005S\u0000"
+        + "\u0000\u0315\u0316\u0005C\u0000\u0000\u0316\u0317\u0005H\u0000\u0000\u0317"
+        + "\u0318\u0005E\u0000\u0000\u0318\u0319\u0005M\u0000\u0000\u0319\u031a\u0005"
+        + "A\u0000\u0000\u031a\u031b\u0005S\u0000\u0000\u031b\u00a2\u0001\u0000\u0000"
+        + "\u0000\u031c\u031d\u0005S\u0000\u0000\u031d\u031e\u0005E\u0000\u0000\u031e"
+        + "\u031f\u0005C\u0000\u0000\u031f\u0320\u0005O\u0000\u0000\u0320\u0321\u0005"
+        + "N\u0000\u0000\u0321\u0322\u0005D\u0000\u0000\u0322\u00a4\u0001\u0000\u0000"
+        + "\u0000\u0323\u0324\u0005S\u0000\u0000\u0324\u0325\u0005E\u0000\u0000\u0325"
+        + "\u0326\u0005C\u0000\u0000\u0326\u0327\u0005O\u0000\u0000\u0327\u0328\u0005"
+        + "N\u0000\u0000\u0328\u0329\u0005D\u0000\u0000\u0329\u032a\u0005S\u0000"
+        + "\u0000\u032a\u00a6\u0001\u0000\u0000\u0000\u032b\u032c\u0005S\u0000\u0000"
+        + "\u032c\u032d\u0005E\u0000\u0000\u032d\u032e\u0005L\u0000\u0000\u032e\u032f"
+        + "\u0005E\u0000\u0000\u032f\u0330\u0005C\u0000\u0000\u0330\u0331\u0005T"
+        + "\u0000\u0000\u0331\u00a8\u0001\u0000\u0000\u0000\u0332\u0333\u0005S\u0000"
+        + "\u0000\u0333\u0334\u0005H\u0000\u0000\u0334\u0335\u0005O\u0000\u0000\u0335"
+        + "\u0336\u0005W\u0000\u0000\u0336\u00aa\u0001\u0000\u0000\u0000\u0337\u0338"
+        + "\u0005S\u0000\u0000\u0338\u0339\u0005Y\u0000\u0000\u0339\u033a\u0005S"
+        + "\u0000\u0000\u033a\u00ac\u0001\u0000\u0000\u0000\u033b\u033c\u0005T\u0000"
+        + "\u0000\u033c\u033d\u0005A\u0000\u0000\u033d\u033e\u0005B\u0000\u0000\u033e"
+        + "\u033f\u0005L\u0000\u0000\u033f\u0340\u0005E\u0000\u0000\u0340\u00ae\u0001"
+        + "\u0000\u0000\u0000\u0341\u0342\u0005T\u0000\u0000\u0342\u0343\u0005A\u0000"
+        + "\u0000\u0343\u0344\u0005B\u0000\u0000\u0344\u0345\u0005L\u0000\u0000\u0345"
+        + "\u0346\u0005E\u0000\u0000\u0346\u0347\u0005S\u0000\u0000\u0347\u00b0\u0001"
+        + "\u0000\u0000\u0000\u0348\u0349\u0005T\u0000\u0000\u0349\u034a\u0005E\u0000"
+        + "\u0000\u034a\u034b\u0005X\u0000\u0000\u034b\u034c\u0005T\u0000\u0000\u034c"
+        + "\u00b2\u0001\u0000\u0000\u0000\u034d\u034e\u0005T\u0000\u0000\u034e\u034f"
+        + "\u0005H\u0000\u0000\u034f\u0350\u0005E\u0000\u0000\u0350\u0351\u0005N"
+        + "\u0000\u0000\u0351\u00b4\u0001\u0000\u0000\u0000\u0352\u0353\u0005T\u0000"
+        + "\u0000\u0353\u0354\u0005R\u0000\u0000\u0354\u0355\u0005U\u0000\u0000\u0355"
+        + "\u0356\u0005E\u0000\u0000\u0356\u00b6\u0001\u0000\u0000\u0000\u0357\u0358"
+        + "\u0005T\u0000\u0000\u0358\u0359\u0005O\u0000\u0000\u0359\u00b8\u0001\u0000"
+        + "\u0000\u0000\u035a\u035b\u0005T\u0000\u0000\u035b\u035c\u0005O\u0000\u0000"
+        + "\u035c\u035d\u0005P\u0000\u0000\u035d\u00ba\u0001\u0000\u0000\u0000\u035e"
+        + "\u035f\u0005T\u0000\u0000\u035f\u0360\u0005Y\u0000\u0000\u0360\u0361\u0005"
+        + "P\u0000\u0000\u0361\u0362\u0005E\u0000\u0000\u0362\u00bc\u0001\u0000\u0000"
+        + "\u0000\u0363\u0364\u0005T\u0000\u0000\u0364\u0365\u0005Y\u0000\u0000\u0365"
+        + "\u0366\u0005P\u0000\u0000\u0366\u0367\u0005E\u0000\u0000\u0367\u0368\u0005"
+        + "S\u0000\u0000\u0368\u00be\u0001\u0000\u0000\u0000\u0369\u036a\u0005U\u0000"
+        + "\u0000\u036a\u036b\u0005S\u0000\u0000\u036b\u036c\u0005I\u0000\u0000\u036c"
+        + "\u036d\u0005N\u0000\u0000\u036d\u036e\u0005G\u0000\u0000\u036e\u00c0\u0001"
+        + "\u0000\u0000\u0000\u036f\u0370\u0005V\u0000\u0000\u0370\u0371\u0005E\u0000"
+        + "\u0000\u0371\u0372\u0005R\u0000\u0000\u0372\u0373\u0005I\u0000\u0000\u0373"
+        + "\u0374\u0005F\u0000\u0000\u0374\u0375\u0005Y\u0000\u0000\u0375\u00c2\u0001"
+        + "\u0000\u0000\u0000\u0376\u0377\u0005W\u0000\u0000\u0377\u0378\u0005H\u0000"
+        + "\u0000\u0378\u0379\u0005E\u0000\u0000\u0379\u037a\u0005N\u0000\u0000\u037a"
+        + "\u00c4\u0001\u0000\u0000\u0000\u037b\u037c\u0005W\u0000\u0000\u037c\u037d"
+        + "\u0005H\u0000\u0000\u037d\u037e\u0005E\u0000\u0000\u037e\u037f\u0005R"
+        + "\u0000\u0000\u037f\u0380\u0005E\u0000\u0000\u0380\u00c6\u0001\u0000\u0000"
+        + "\u0000\u0381\u0382\u0005W\u0000\u0000\u0382\u0383\u0005I\u0000\u0000\u0383"
+        + "\u0384\u0005T\u0000\u0000\u0384\u0385\u0005H\u0000\u0000\u0385\u00c8\u0001"
+        + "\u0000\u0000\u0000\u0386\u0387\u0005Y\u0000\u0000\u0387\u0388\u0005E\u0000"
+        + "\u0000\u0388\u0389\u0005A\u0000\u0000\u0389\u038a\u0005R\u0000\u0000\u038a"
+        + "\u00ca\u0001\u0000\u0000\u0000\u038b\u038c\u0005Y\u0000\u0000\u038c\u038d"
+        + "\u0005E\u0000\u0000\u038d\u038e\u0005A\u0000\u0000\u038e\u038f\u0005R"
+        + "\u0000\u0000\u038f\u0390\u0005S\u0000\u0000\u0390\u00cc\u0001\u0000\u0000"
+        + "\u0000\u0391\u0392\u0003\u00dbm\u0000\u0392\u0393\u0005E\u0000\u0000\u0393"
+        + "\u0394\u0005S\u0000\u0000\u0394\u0395\u0005C\u0000\u0000\u0395\u0396\u0005"
+        + "A\u0000\u0000\u0396\u0397\u0005P\u0000\u0000\u0397\u0398\u0005E\u0000"
+        + "\u0000\u0398\u00ce\u0001\u0000\u0000\u0000\u0399\u039a\u0003\u00dbm\u0000"
+        + "\u039a\u039b\u0005F\u0000\u0000\u039b\u039c\u0005N\u0000\u0000\u039c\u00d0"
+        + "\u0001\u0000\u0000\u0000\u039d\u039e\u0003\u00dbm\u0000\u039e\u039f\u0005"
+        + "L\u0000\u0000\u039f\u03a0\u0005I\u0000\u0000\u03a0\u03a1\u0005M\u0000"
+        + "\u0000\u03a1\u03a2\u0005I\u0000\u0000\u03a2\u03a3\u0005T\u0000\u0000\u03a3"
+        + "\u00d2\u0001\u0000\u0000\u0000\u03a4\u03a5\u0003\u00dbm\u0000\u03a5\u03a6"
+        + "\u0005D\u0000\u0000\u03a6\u00d4\u0001\u0000\u0000\u0000\u03a7\u03a8\u0003"
+        + "\u00dbm\u0000\u03a8\u03a9\u0005T\u0000\u0000\u03a9\u00d6\u0001\u0000\u0000"
+        + "\u0000\u03aa\u03ab\u0003\u00dbm\u0000\u03ab\u03ac\u0005T\u0000\u0000\u03ac"
+        + "\u03ad\u0005S\u0000\u0000\u03ad\u00d8\u0001\u0000\u0000\u0000\u03ae\u03af"
+        + "\u0003\u00dbm\u0000\u03af\u03b0\u0005G\u0000\u0000\u03b0\u03b1\u0005U"
+        + "\u0000\u0000\u03b1\u03b2\u0005I\u0000\u0000\u03b2\u03b3\u0005D\u0000\u0000"
+        + "\u03b3\u00da\u0001\u0000\u0000\u0000\u03b4\u03b8\u0005{\u0000\u0000\u03b5"
+        + "\u03b7\u0003\u0117\u008b\u0000\u03b6\u03b5\u0001\u0000\u0000\u0000\u03b7"
+        + "\u03ba\u0001\u0000\u0000\u0000\u03b8\u03b6\u0001\u0000\u0000\u0000\u03b8"
+        + "\u03b9\u0001\u0000\u0000\u0000\u03b9\u00dc\u0001\u0000\u0000\u0000\u03ba"
+        + "\u03b8\u0001\u0000\u0000\u0000\u03bb\u03bc\u0005}\u0000\u0000\u03bc\u00de"
+        + "\u0001\u0000\u0000\u0000\u03bd\u03be\u0005=\u0000\u0000\u03be\u00e0\u0001"
+        + "\u0000\u0000\u0000\u03bf\u03c0\u0005<\u0000\u0000\u03c0\u03c1\u0005=\u0000"
+        + "\u0000\u03c1\u03c2\u0005>\u0000\u0000\u03c2\u00e2\u0001\u0000\u0000\u0000"
+        + "\u03c3\u03c4\u0005<\u0000\u0000\u03c4\u03c8\u0005>\u0000\u0000\u03c5\u03c6"
+        + "\u0005!\u0000\u0000\u03c6\u03c8\u0005=\u0000\u0000\u03c7\u03c3\u0001\u0000"
+        + "\u0000\u0000\u03c7\u03c5\u0001\u0000\u0000\u0000\u03c8\u00e4\u0001\u0000"
+        + "\u0000\u0000\u03c9\u03ca\u0005<\u0000\u0000\u03ca\u00e6\u0001\u0000\u0000"
+        + "\u0000\u03cb\u03cc\u0005<\u0000\u0000\u03cc\u03cd\u0005=\u0000\u0000\u03cd"
+        + "\u00e8\u0001\u0000\u0000\u0000\u03ce\u03cf\u0005>\u0000\u0000\u03cf\u00ea"
+        + "\u0001\u0000\u0000\u0000\u03d0\u03d1\u0005>\u0000\u0000\u03d1\u03d2\u0005"
+        + "=\u0000\u0000\u03d2\u00ec\u0001\u0000\u0000\u0000\u03d3\u03d4\u0005+\u0000"
+        + "\u0000\u03d4\u00ee\u0001\u0000\u0000\u0000\u03d5\u03d6\u0005-\u0000\u0000"
+        + "\u03d6\u00f0\u0001\u0000\u0000\u0000\u03d7\u03d8\u0005*\u0000\u0000\u03d8"
+        + "\u00f2\u0001\u0000\u0000\u0000\u03d9\u03da\u0005/\u0000\u0000\u03da\u00f4"
+        + "\u0001\u0000\u0000\u0000\u03db\u03dc\u0005%\u0000\u0000\u03dc\u00f6\u0001"
+        + "\u0000\u0000\u0000\u03dd\u03de\u0005:\u0000\u0000\u03de\u03df\u0005:\u0000"
+        + "\u0000\u03df\u00f8\u0001\u0000\u0000\u0000\u03e0\u03e1\u0005.\u0000\u0000"
+        + "\u03e1\u00fa\u0001\u0000\u0000\u0000\u03e2\u03e3\u0005?\u0000\u0000\u03e3"
+        + "\u00fc\u0001\u0000\u0000\u0000\u03e4\u03ea\u0005\'\u0000\u0000\u03e5\u03e9"
+        + "\b\u0000\u0000\u0000\u03e6\u03e7\u0005\'\u0000\u0000\u03e7\u03e9\u0005"
+        + "\'\u0000\u0000\u03e8\u03e5\u0001\u0000\u0000\u0000\u03e8\u03e6\u0001\u0000"
+        + "\u0000\u0000\u03e9\u03ec\u0001\u0000\u0000\u0000\u03ea\u03e8\u0001\u0000"
+        + "\u0000\u0000\u03ea\u03eb\u0001\u0000\u0000\u0000\u03eb\u03ed\u0001\u0000"
+        + "\u0000\u0000\u03ec\u03ea\u0001\u0000\u0000\u0000\u03ed\u03ee\u0005\'\u0000"
+        + "\u0000\u03ee\u00fe\u0001\u0000\u0000\u0000\u03ef\u03f1\u0003\u010f\u0087"
+        + "\u0000\u03f0\u03ef\u0001\u0000\u0000\u0000\u03f1\u03f2\u0001\u0000\u0000"
+        + "\u0000\u03f2\u03f0\u0001\u0000\u0000\u0000\u03f2\u03f3\u0001\u0000\u0000"
+        + "\u0000\u03f3\u0100\u0001\u0000\u0000\u0000\u03f4\u03f6\u0003\u010f\u0087"
+        + "\u0000\u03f5\u03f4\u0001\u0000\u0000\u0000\u03f6\u03f7\u0001\u0000\u0000"
+        + "\u0000\u03f7\u03f5\u0001\u0000\u0000\u0000\u03f7\u03f8\u0001\u0000\u0000"
+        + "\u0000\u03f8\u03f9\u0001\u0000\u0000\u0000\u03f9\u03fd\u0003\u00f9|\u0000"
+        + "\u03fa\u03fc\u0003\u010f\u0087\u0000\u03fb\u03fa\u0001\u0000\u0000\u0000"
+        + "\u03fc\u03ff\u0001\u0000\u0000\u0000\u03fd\u03fb\u0001\u0000\u0000\u0000"
+        + "\u03fd\u03fe\u0001\u0000\u0000\u0000\u03fe\u041f\u0001\u0000\u0000\u0000"
+        + "\u03ff\u03fd\u0001\u0000\u0000\u0000\u0400\u0402\u0003\u00f9|\u0000\u0401"
+        + "\u0403\u0003\u010f\u0087\u0000\u0402\u0401\u0001\u0000\u0000\u0000\u0403"
+        + "\u0404\u0001\u0000\u0000\u0000\u0404\u0402\u0001\u0000\u0000\u0000\u0404"
+        + "\u0405\u0001\u0000\u0000\u0000\u0405\u041f\u0001\u0000\u0000\u0000\u0406"
+        + "\u0408\u0003\u010f\u0087\u0000\u0407\u0406\u0001\u0000\u0000\u0000\u0408"
+        + "\u0409\u0001\u0000\u0000\u0000\u0409\u0407\u0001\u0000\u0000\u0000\u0409"
+        + "\u040a\u0001\u0000\u0000\u0000\u040a\u0412\u0001\u0000\u0000\u0000\u040b"
+        + "\u040f\u0003\u00f9|\u0000\u040c\u040e\u0003\u010f\u0087\u0000\u040d\u040c"
+        + "\u0001\u0000\u0000\u0000\u040e\u0411\u0001\u0000\u0000\u0000\u040f\u040d"
+        + "\u0001\u0000\u0000\u0000\u040f\u0410\u0001\u0000\u0000\u0000\u0410\u0413"
+        + "\u0001\u0000\u0000\u0000\u0411\u040f\u0001\u0000\u0000\u0000\u0412\u040b"
+        + "\u0001\u0000\u0000\u0000\u0412\u0413\u0001\u0000\u0000\u0000\u0413\u0414"
+        + "\u0001\u0000\u0000\u0000\u0414\u0415\u0003\u010d\u0086\u0000\u0415\u041f"
+        + "\u0001\u0000\u0000\u0000\u0416\u0418\u0003\u00f9|\u0000\u0417\u0419\u0003"
+        + "\u010f\u0087\u0000\u0418\u0417\u0001\u0000\u0000\u0000\u0419\u041a\u0001"
+        + "\u0000\u0000\u0000\u041a\u0418\u0001\u0000\u0000\u0000\u041a\u041b\u0001"
+        + "\u0000\u0000\u0000\u041b\u041c\u0001\u0000\u0000\u0000\u041c\u041d\u0003"
+        + "\u010d\u0086\u0000\u041d\u041f\u0001\u0000\u0000\u0000\u041e\u03f5\u0001"
+        + "\u0000\u0000\u0000\u041e\u0400\u0001\u0000\u0000\u0000\u041e\u0407\u0001"
+        + "\u0000\u0000\u0000\u041e\u0416\u0001\u0000\u0000\u0000\u041f\u0102\u0001"
+        + "\u0000\u0000\u0000\u0420\u0423\u0003\u0111\u0088\u0000\u0421\u0423\u0005"
+        + "_\u0000\u0000\u0422\u0420\u0001\u0000\u0000\u0000\u0422\u0421\u0001\u0000"
+        + "\u0000\u0000\u0423\u0429\u0001\u0000\u0000\u0000\u0424\u0428\u0003\u0111"
+        + "\u0088\u0000\u0425\u0428\u0003\u010f\u0087\u0000\u0426\u0428\u0007\u0001"
+        + "\u0000\u0000\u0427\u0424\u0001\u0000\u0000\u0000\u0427\u0425\u0001\u0000"
+        + "\u0000\u0000\u0427\u0426\u0001\u0000\u0000\u0000\u0428\u042b\u0001\u0000"
+        + "\u0000\u0000\u0429\u0427\u0001\u0000\u0000\u0000\u0429\u042a\u0001\u0000"
+        + "\u0000\u0000\u042a\u0104\u0001\u0000\u0000\u0000\u042b\u0429\u0001\u0000"
+        + "\u0000\u0000\u042c\u0430\u0003\u010f\u0087\u0000\u042d\u0431\u0003\u0111"
+        + "\u0088\u0000\u042e\u0431\u0003\u010f\u0087\u0000\u042f\u0431\u0007\u0001"
+        + "\u0000\u0000\u0430\u042d\u0001\u0000\u0000\u0000\u0430\u042e\u0001\u0000"
+        + "\u0000\u0000\u0430\u042f\u0001\u0000\u0000\u0000\u0431\u0432\u0001\u0000"
+        + "\u0000\u0000\u0432\u0430\u0001\u0000\u0000\u0000\u0432\u0433\u0001\u0000"
+        + "\u0000\u0000\u0433\u0106\u0001\u0000\u0000\u0000\u0434\u0438\u0003\u0111"
+        + "\u0088\u0000\u0435\u0438\u0003\u010f\u0087\u0000\u0436\u0438\u0005_\u0000"
+        + "\u0000\u0437\u0434\u0001\u0000\u0000\u0000\u0437\u0435\u0001\u0000\u0000"
+        + "\u0000\u0437\u0436\u0001\u0000\u0000\u0000\u0438\u0439\u0001\u0000\u0000"
+        + "\u0000\u0439\u0437\u0001\u0000\u0000\u0000\u0439\u043a\u0001\u0000\u0000"
+        + "\u0000\u043a\u0108\u0001\u0000\u0000\u0000\u043b\u0441\u0005\"\u0000\u0000"
+        + "\u043c\u0440\b\u0002\u0000\u0000\u043d\u043e\u0005\"\u0000\u0000\u043e"
+        + "\u0440\u0005\"\u0000\u0000\u043f\u043c\u0001\u0000\u0000\u0000\u043f\u043d"
+        + "\u0001\u0000\u0000\u0000\u0440\u0443\u0001\u0000\u0000\u0000\u0441\u043f"
+        + "\u0001\u0000\u0000\u0000\u0441\u0442\u0001\u0000\u0000\u0000\u0442\u0444"
+        + "\u0001\u0000\u0000\u0000\u0443\u0441\u0001\u0000\u0000\u0000\u0444\u0445"
+        + "\u0005\"\u0000\u0000\u0445\u010a\u0001\u0000\u0000\u0000\u0446\u044c\u0005"
+        + "`\u0000\u0000\u0447\u044b\b\u0003\u0000\u0000\u0448\u0449\u0005`\u0000"
+        + "\u0000\u0449\u044b\u0005`\u0000\u0000\u044a\u0447\u0001\u0000\u0000\u0000"
+        + "\u044a\u0448\u0001\u0000\u0000\u0000\u044b\u044e\u0001\u0000\u0000\u0000"
+        + "\u044c\u044a\u0001\u0000\u0000\u0000\u044c\u044d\u0001\u0000\u0000\u0000"
+        + "\u044d\u044f\u0001\u0000\u0000\u0000\u044e\u044c\u0001\u0000\u0000\u0000"
+        + "\u044f\u0450\u0005`\u0000\u0000\u0450\u010c\u0001\u0000\u0000\u0000\u0451"
+        + "\u0453\u0005E\u0000\u0000\u0452\u0454\u0007\u0004\u0000\u0000\u0453\u0452"
+        + "\u0001\u0000\u0000\u0000\u0453\u0454\u0001\u0000\u0000\u0000\u0454\u0456"
+        + "\u0001\u0000\u0000\u0000\u0455\u0457\u0003\u010f\u0087\u0000\u0456\u0455"
+        + "\u0001\u0000\u0000\u0000\u0457\u0458\u0001\u0000\u0000\u0000\u0458\u0456"
+        + "\u0001\u0000\u0000\u0000\u0458\u0459\u0001\u0000\u0000\u0000\u0459\u010e"
+        + "\u0001\u0000\u0000\u0000\u045a\u045b\u0007\u0005\u0000\u0000\u045b\u0110"
+        + "\u0001\u0000\u0000\u0000\u045c\u045d\u0007\u0006\u0000\u0000\u045d\u0112"
+        + "\u0001\u0000\u0000\u0000\u045e\u045f\u0005-\u0000\u0000\u045f\u0460\u0005"
+        + "-\u0000\u0000\u0460\u0464\u0001\u0000\u0000\u0000\u0461\u0463\b\u0007"
+        + "\u0000\u0000\u0462\u0461\u0001\u0000\u0000\u0000\u0463\u0466\u0001\u0000"
+        + "\u0000\u0000\u0464\u0462\u0001\u0000\u0000\u0000\u0464\u0465\u0001\u0000"
+        + "\u0000\u0000\u0465\u0468\u0001\u0000\u0000\u0000\u0466\u0464\u0001\u0000"
+        + "\u0000\u0000\u0467\u0469\u0005\r\u0000\u0000\u0468\u0467\u0001\u0000\u0000"
+        + "\u0000\u0468\u0469\u0001\u0000\u0000\u0000\u0469\u046b\u0001\u0000\u0000"
+        + "\u0000\u046a\u046c\u0005\n\u0000\u0000\u046b\u046a\u0001\u0000\u0000\u0000"
+        + "\u046b\u046c\u0001\u0000\u0000\u0000\u046c\u046d\u0001\u0000\u0000\u0000"
+        + "\u046d\u046e\u0006\u0089\u0000\u0000\u046e\u0114\u0001\u0000\u0000\u0000"
+        + "\u046f\u0470\u0005/\u0000\u0000\u0470\u0471\u0005*\u0000\u0000\u0471\u0476"
+        + "\u0001\u0000\u0000\u0000\u0472\u0475\u0003\u0115\u008a\u0000\u0473\u0475"
+        + "\t\u0000\u0000\u0000\u0474\u0472\u0001\u0000\u0000\u0000\u0474\u0473\u0001"
+        + "\u0000\u0000\u0000\u0475\u0478\u0001\u0000\u0000\u0000\u0476\u0477\u0001"
+        + "\u0000\u0000\u0000\u0476\u0474\u0001\u0000\u0000\u0000\u0477\u0479\u0001"
+        + "\u0000\u0000\u0000\u0478\u0476\u0001\u0000\u0000\u0000\u0479\u047a\u0005"
+        + "*\u0000\u0000\u047a\u047b\u0005/\u0000\u0000\u047b\u047c\u0001\u0000\u0000"
+        + "\u0000\u047c\u047d\u0006\u008a\u0000\u0000\u047d\u0116\u0001\u0000\u0000"
+        + "\u0000\u047e\u0480\u0007\b\u0000\u0000\u047f\u047e\u0001\u0000\u0000\u0000"
+        + "\u0480\u0481\u0001\u0000\u0000\u0000\u0481\u047f\u0001\u0000\u0000\u0000"
+        + "\u0481\u0482\u0001\u0000\u0000\u0000\u0482\u0483\u0001\u0000\u0000\u0000"
+        + "\u0483\u0484\u0006\u008b\u0000\u0000\u0484\u0118\u0001\u0000\u0000\u0000"
+        + "\u0485\u0486\t\u0000\u0000\u0000\u0486\u011a\u0001\u0000\u0000\u0000!"
+        + "\u0000\u03b8\u03c7\u03e8\u03ea\u03f2\u03f7\u03fd\u0404\u0409\u040f\u0412"
+        + "\u041a\u041e\u0422\u0427\u0429\u0430\u0432\u0437\u0439\u043f\u0441\u044a"
+        + "\u044c\u0453\u0458\u0464\u0468\u046b\u0474\u0476\u0481\u0001\u0000\u0001"
+        + "\u0000";
     public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray());
     static {
         _decisionToDFA = new DFA[_ATN.getNumberOfDecisions()];
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseParser.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseParser.java
index dd4fbcdd0e7a..fc5a98d3a71f 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseParser.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseParser.java
@@ -9,10 +9,10 @@
 
 import java.util.List;
 
-@SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast" })
+@SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue" })
 class SqlBaseParser extends Parser {
     static {
-        RuntimeMetaData.checkVersion("4.9.2", RuntimeMetaData.VERSION);
+        RuntimeMetaData.checkVersion("4.11.1", RuntimeMetaData.VERSION);
     }
 
     protected static final DFA[] _decisionToDFA;
@@ -424,7 +424,7 @@ public Vocabulary getVocabulary() {
 
     @Override
     public String getGrammarFileName() {
-        return "SqlBase.g4";
+        return "java-escape";
     }
 
     @Override
@@ -447,6 +447,7 @@ public SqlBaseParser(TokenStream input) {
         _interp = new ParserATNSimulator(this, _ATN, _decisionToDFA, _sharedContextCache);
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class SingleStatementContext extends ParserRuleContext {
         public StatementContext statement() {
             return getRuleContext(StatementContext.class, 0);
@@ -503,6 +504,7 @@ public final SingleStatementContext singleStatement() throws RecognitionExceptio
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class SingleExpressionContext extends ParserRuleContext {
         public ExpressionContext expression() {
             return getRuleContext(ExpressionContext.class, 0);
@@ -559,6 +561,7 @@ public final SingleExpressionContext singleExpression() throws RecognitionExcept
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class StatementContext extends ParserRuleContext {
         public StatementContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -576,6 +579,7 @@ public void copyFrom(StatementContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ExplainContext extends StatementContext {
         public Token type;
         public Token format;
@@ -706,6 +710,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class SysColumnsContext extends StatementContext {
         public StringContext cluster;
         public LikePatternContext tableLike;
@@ -765,6 +770,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class SysTypesContext extends StatementContext {
         public NumberContext type;
 
@@ -809,6 +815,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class DebugContext extends StatementContext {
         public Token type;
         public Token format;
@@ -890,6 +897,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ShowCatalogsContext extends StatementContext {
         public TerminalNode SHOW() {
             return getToken(SqlBaseParser.SHOW, 0);
@@ -920,6 +928,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class StatementDefaultContext extends StatementContext {
         public QueryContext query() {
             return getRuleContext(QueryContext.class, 0);
@@ -946,6 +955,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class SysTablesContext extends StatementContext {
         public LikePatternContext clusterLike;
         public LikePatternContext tableLike;
@@ -1008,6 +1018,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ShowFunctionsContext extends StatementContext {
         public TerminalNode SHOW() {
             return getToken(SqlBaseParser.SHOW, 0);
@@ -1042,6 +1053,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ShowTablesContext extends StatementContext {
         public LikePatternContext clusterLike;
         public StringContext cluster;
@@ -1105,6 +1117,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ShowSchemasContext extends StatementContext {
         public TerminalNode SHOW() {
             return getToken(SqlBaseParser.SHOW, 0);
@@ -1135,6 +1148,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ShowColumnsContext extends StatementContext {
         public StringContext cluster;
         public LikePatternContext tableLike;
@@ -1238,8 +1252,7 @@ public final StatementContext statement() throws RecognitionException {
                             setState(137);
                             _errHandler.sync(this);
                             _la = _input.LA(1);
-                            while (((((_la - 39)) & ~0x3f) == 0
-                                && ((1L << (_la - 39)) & ((1L << (FORMAT - 39)) | (1L << (PLAN - 39)) | (1L << (VERIFY - 39)))) != 0)) {
+                            while ((((_la - 39)) & ~0x3f) == 0 && ((1L << (_la - 39)) & 288230651029618689L) != 0) {
                                 {
                                     setState(135);
                                     _errHandler.sync(this);
@@ -1250,9 +1263,9 @@ public final StatementContext statement() throws RecognitionException {
                                             setState(130);
                                             ((ExplainContext) _localctx).type = _input.LT(1);
                                             _la = _input.LA(1);
-                                            if (!((((_la) & ~0x3f) == 0
-                                                && ((1L << _la) & ((1L << ALL) | (1L << ANALYZED) | (1L << EXECUTABLE) | (1L
-                                                    << MAPPED))) != 0) || _la == OPTIMIZED || _la == PARSED)) {
+                                            if (!(((_la) & ~0x3f) == 0 && ((1L << _la) & 576460756598390944L) != 0
+                                                || _la == OPTIMIZED
+                                                || _la == PARSED)) {
                                                 ((ExplainContext) _localctx).type = (Token) _errHandler.recoverInline(this);
                                             } else {
                                                 if (_input.LA(1) == Token.EOF) matchedEOF = true;
@@ -1886,9 +1899,7 @@ public final StatementContext statement() throws RecognitionException {
                     setState(254);
                     _errHandler.sync(this);
                     _la = _input.LA(1);
-                    if (((((_la - 119)) & ~0x3f) == 0
-                        && ((1L << (_la - 119)) & ((1L << (PLUS - 119)) | (1L << (MINUS - 119)) | (1L << (INTEGER_VALUE - 119)) | (1L
-                            << (DECIMAL_VALUE - 119)))) != 0)) {
+                    if ((((_la - 119)) & ~0x3f) == 0 && ((1L << (_la - 119)) & 1539L) != 0) {
                         {
                             setState(251);
                             _errHandler.sync(this);
@@ -1925,6 +1936,7 @@ public final StatementContext statement() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class QueryContext extends ParserRuleContext {
         public QueryNoWithContext queryNoWith() {
             return getRuleContext(QueryNoWithContext.class, 0);
@@ -2016,6 +2028,7 @@ public final QueryContext query() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class QueryNoWithContext extends ParserRuleContext {
         public QueryTermContext queryTerm() {
             return getRuleContext(QueryTermContext.class, 0);
@@ -2127,6 +2140,7 @@ public final QueryNoWithContext queryNoWith() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class LimitClauseContext extends ParserRuleContext {
         public Token limit;
 
@@ -2231,6 +2245,7 @@ public final LimitClauseContext limitClause() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class QueryTermContext extends ParserRuleContext {
         public QueryTermContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -2248,6 +2263,7 @@ public void copyFrom(QueryTermContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class SubqueryContext extends QueryTermContext {
         public QueryNoWithContext queryNoWith() {
             return getRuleContext(QueryNoWithContext.class, 0);
@@ -2274,6 +2290,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class QueryPrimaryDefaultContext extends QueryTermContext {
         public QuerySpecificationContext querySpecification() {
             return getRuleContext(QuerySpecificationContext.class, 0);
@@ -2338,6 +2355,7 @@ public final QueryTermContext queryTerm() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class OrderByContext extends ParserRuleContext {
         public Token ordering;
         public Token nullOrdering;
@@ -2450,6 +2468,7 @@ public final OrderByContext orderBy() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class QuerySpecificationContext extends ParserRuleContext {
         public BooleanExpressionContext where;
         public BooleanExpressionContext having;
@@ -2617,6 +2636,7 @@ public final QuerySpecificationContext querySpecification() throws RecognitionEx
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class FromClauseContext extends ParserRuleContext {
         public TerminalNode FROM() {
             return getToken(SqlBaseParser.FROM, 0);
@@ -2708,6 +2728,7 @@ public final FromClauseContext fromClause() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class GroupByContext extends ParserRuleContext {
         public List<GroupingElementContext> groupingElement() {
             return getRuleContexts(GroupingElementContext.class);
@@ -2793,6 +2814,7 @@ public final GroupByContext groupBy() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class GroupingElementContext extends ParserRuleContext {
         public GroupingElementContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -2810,6 +2832,7 @@ public void copyFrom(GroupingElementContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class SingleGroupingSetContext extends GroupingElementContext {
         public GroupingExpressionsContext groupingExpressions() {
             return getRuleContext(GroupingExpressionsContext.class, 0);
@@ -2856,6 +2879,7 @@ public final GroupingElementContext groupingElement() throws RecognitionExceptio
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class GroupingExpressionsContext extends ParserRuleContext {
         public List<ExpressionContext> expression() {
             return getRuleContexts(ExpressionContext.class);
@@ -2906,26 +2930,9 @@ public final GroupingExpressionsContext groupingExpressions() throws Recognition
                     setState(367);
                     _errHandler.sync(this);
                     _la = _input.LA(1);
-                    if ((((_la) & ~0x3f) == 0
-                        && ((1L << _la) & ((1L << T__0) | (1L << ANALYZE) | (1L << ANALYZED) | (1L << CASE) | (1L << CAST) | (1L
-                            << CATALOGS) | (1L << COLUMNS) | (1L << CONVERT) | (1L << CURRENT_DATE) | (1L << CURRENT_TIME) | (1L
-                                << CURRENT_TIMESTAMP) | (1L << DAY) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXISTS) | (1L << EXPLAIN)
-                            | (1L << EXTRACT) | (1L << FALSE) | (1L << FIRST) | (1L << FORMAT) | (1L << FULL) | (1L << FUNCTIONS) | (1L
-                                << GRAPHVIZ) | (1L << HOUR) | (1L << INTERVAL) | (1L << LAST) | (1L << LEFT) | (1L << LIMIT) | (1L
-                                    << MAPPED) | (1L << MATCH) | (1L << MINUTE) | (1L << MONTH))) != 0)
-                        || ((((_la - 66)) & ~0x3f) == 0
-                            && ((1L << (_la - 66)) & ((1L << (NOT - 66)) | (1L << (NULL - 66)) | (1L << (OPTIMIZED - 66)) | (1L << (PARSED
-                                - 66)) | (1L << (PHYSICAL - 66)) | (1L << (PIVOT - 66)) | (1L << (PLAN - 66)) | (1L << (RIGHT - 66)) | (1L
-                                    << (RLIKE - 66)) | (1L << (QUERY - 66)) | (1L << (SCHEMAS - 66)) | (1L << (SECOND - 66)) | (1L << (SHOW
-                                        - 66)) | (1L << (SYS - 66)) | (1L << (TABLES - 66)) | (1L << (TEXT - 66)) | (1L << (TRUE - 66))
-                                | (1L << (TOP - 66)) | (1L << (TYPE - 66)) | (1L << (TYPES - 66)) | (1L << (VERIFY - 66)) | (1L << (YEAR
-                                    - 66)) | (1L << (FUNCTION_ESC - 66)) | (1L << (DATE_ESC - 66)) | (1L << (TIME_ESC - 66)) | (1L
-                                        << (TIMESTAMP_ESC - 66)) | (1L << (GUID_ESC - 66)) | (1L << (PLUS - 66)) | (1L << (MINUS - 66))
-                                | (1L << (ASTERISK - 66)) | (1L << (PARAM - 66)) | (1L << (STRING - 66)) | (1L << (INTEGER_VALUE - 66))
-                                | (1L << (DECIMAL_VALUE - 66)))) != 0)
-                        || ((((_la - 130)) & ~0x3f) == 0
-                            && ((1L << (_la - 130)) & ((1L << (IDENTIFIER - 130)) | (1L << (DIGIT_IDENTIFIER - 130)) | (1L
-                                << (QUOTED_IDENTIFIER - 130)) | (1L << (BACKQUOTED_IDENTIFIER - 130)))) != 0)) {
+                    if (((_la) & ~0x3f) == 0 && ((1L << _la) & -4787154059691900734L) != 0
+                        || (((_la - 66)) & ~0x3f) == 0 && ((1L << (_la - 66)) & -1089854304776749293L) != 0
+                        || (((_la - 130)) & ~0x3f) == 0 && ((1L << (_la - 130)) & 27L) != 0) {
                         {
                             setState(359);
                             expression();
@@ -2969,6 +2976,7 @@ public final GroupingExpressionsContext groupingExpressions() throws Recognition
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class NamedQueryContext extends ParserRuleContext {
         public IdentifierContext name;
 
@@ -3037,6 +3045,7 @@ public final NamedQueryContext namedQuery() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class TopClauseContext extends ParserRuleContext {
         public Token top;
 
@@ -3095,6 +3104,7 @@ public final TopClauseContext topClause() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class SetQuantifierContext extends ParserRuleContext {
         public TerminalNode DISTINCT() {
             return getToken(SqlBaseParser.DISTINCT, 0);
@@ -3157,6 +3167,7 @@ public final SetQuantifierContext setQuantifier() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class SelectItemsContext extends ParserRuleContext {
         public List<SelectItemContext> selectItem() {
             return getRuleContexts(SelectItemContext.class);
@@ -3228,6 +3239,7 @@ public final SelectItemsContext selectItems() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class SelectItemContext extends ParserRuleContext {
         public SelectItemContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -3245,6 +3257,7 @@ public void copyFrom(SelectItemContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class SelectExpressionContext extends SelectItemContext {
         public ExpressionContext expression() {
             return getRuleContext(ExpressionContext.class, 0);
@@ -3319,6 +3332,7 @@ public final SelectItemContext selectItem() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class RelationContext extends ParserRuleContext {
         public RelationPrimaryContext relationPrimary() {
             return getRuleContext(RelationPrimaryContext.class, 0);
@@ -3370,9 +3384,7 @@ public final RelationContext relation() throws RecognitionException {
                 setState(403);
                 _errHandler.sync(this);
                 _la = _input.LA(1);
-                while (((((_la - 42)) & ~0x3f) == 0
-                    && ((1L << (_la - 42)) & ((1L << (FULL - 42)) | (1L << (INNER - 42)) | (1L << (JOIN - 42)) | (1L << (LEFT - 42)) | (1L
-                        << (NATURAL - 42)) | (1L << (RIGHT - 42)))) != 0)) {
+                while ((((_la - 42)) & ~0x3f) == 0 && ((1L << (_la - 42)) & 68727886337L) != 0) {
                     {
                         {
                             setState(400);
@@ -3394,6 +3406,7 @@ public final RelationContext relation() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class JoinRelationContext extends ParserRuleContext {
         public RelationPrimaryContext right;
 
@@ -3502,6 +3515,7 @@ public final JoinRelationContext joinRelation() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class JoinTypeContext extends ParserRuleContext {
         public TerminalNode INNER() {
             return getToken(SqlBaseParser.INNER, 0);
@@ -3633,6 +3647,7 @@ public final JoinTypeContext joinType() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class JoinCriteriaContext extends ParserRuleContext {
         public TerminalNode ON() {
             return getToken(SqlBaseParser.ON, 0);
@@ -3737,6 +3752,7 @@ public final JoinCriteriaContext joinCriteria() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class RelationPrimaryContext extends ParserRuleContext {
         public RelationPrimaryContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -3754,6 +3770,7 @@ public void copyFrom(RelationPrimaryContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class AliasedRelationContext extends RelationPrimaryContext {
         public RelationContext relation() {
             return getRuleContext(RelationContext.class, 0);
@@ -3788,6 +3805,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class AliasedQueryContext extends RelationPrimaryContext {
         public QueryNoWithContext queryNoWith() {
             return getRuleContext(QueryNoWithContext.class, 0);
@@ -3822,6 +3840,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class TableNameContext extends RelationPrimaryContext {
         public TableIdentifierContext tableIdentifier() {
             return getRuleContext(TableIdentifierContext.class, 0);
@@ -3975,6 +3994,7 @@ public final RelationPrimaryContext relationPrimary() throws RecognitionExceptio
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class PivotClauseContext extends ParserRuleContext {
         public PivotArgsContext aggs;
         public QualifiedNameContext column;
@@ -4067,6 +4087,7 @@ public final PivotClauseContext pivotClause() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class PivotArgsContext extends ParserRuleContext {
         public List<NamedValueExpressionContext> namedValueExpression() {
             return getRuleContexts(NamedValueExpressionContext.class);
@@ -4138,6 +4159,7 @@ public final PivotArgsContext pivotArgs() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class NamedValueExpressionContext extends ParserRuleContext {
         public ValueExpressionContext valueExpression() {
             return getRuleContext(ValueExpressionContext.class, 0);
@@ -4189,18 +4211,8 @@ public final NamedValueExpressionContext namedValueExpression() throws Recogniti
                 setState(506);
                 _errHandler.sync(this);
                 _la = _input.LA(1);
-                if ((((_la) & ~0x3f) == 0
-                    && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << AS) | (1L << CATALOGS) | (1L << COLUMNS) | (1L
-                        << CURRENT_DATE) | (1L << CURRENT_TIME) | (1L << CURRENT_TIMESTAMP) | (1L << DAY) | (1L << DEBUG) | (1L
-                            << EXECUTABLE) | (1L << EXPLAIN) | (1L << FIRST) | (1L << FORMAT) | (1L << FULL) | (1L << FUNCTIONS) | (1L
-                                << GRAPHVIZ) | (1L << HOUR) | (1L << INTERVAL) | (1L << LAST) | (1L << LIMIT) | (1L << MAPPED) | (1L
-                                    << MINUTE) | (1L << MONTH))) != 0)
-                    || ((((_la - 70)) & ~0x3f) == 0
-                        && ((1L << (_la - 70)) & ((1L << (OPTIMIZED - 70)) | (1L << (PARSED - 70)) | (1L << (PHYSICAL - 70)) | (1L << (PIVOT
-                            - 70)) | (1L << (PLAN - 70)) | (1L << (RLIKE - 70)) | (1L << (QUERY - 70)) | (1L << (SCHEMAS - 70)) | (1L
-                                << (SECOND - 70)) | (1L << (SHOW - 70)) | (1L << (SYS - 70)) | (1L << (TABLES - 70)) | (1L << (TEXT - 70))
-                            | (1L << (TOP - 70)) | (1L << (TYPE - 70)) | (1L << (TYPES - 70)) | (1L << (VERIFY - 70)) | (1L << (YEAR - 70))
-                            | (1L << (IDENTIFIER - 70)) | (1L << (DIGIT_IDENTIFIER - 70)) | (1L << (QUOTED_IDENTIFIER - 70)))) != 0)
+                if (((_la) & ~0x3f) == 0 && ((1L << _la) & -6012133270006397760L) != 0
+                    || (((_la - 70)) & ~0x3f) == 0 && ((1L << (_la - 70)) & -5764607520692920591L) != 0
                     || _la == BACKQUOTED_IDENTIFIER) {
                     {
                         setState(503);
@@ -4229,6 +4241,7 @@ public final NamedValueExpressionContext namedValueExpression() throws Recogniti
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ExpressionContext extends ParserRuleContext {
         public BooleanExpressionContext booleanExpression() {
             return getRuleContext(BooleanExpressionContext.class, 0);
@@ -4279,6 +4292,7 @@ public final ExpressionContext expression() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class BooleanExpressionContext extends ParserRuleContext {
         public BooleanExpressionContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -4296,6 +4310,7 @@ public void copyFrom(BooleanExpressionContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class LogicalNotContext extends BooleanExpressionContext {
         public TerminalNode NOT() {
             return getToken(SqlBaseParser.NOT, 0);
@@ -4326,6 +4341,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class StringQueryContext extends BooleanExpressionContext {
         public StringContext queryString;
 
@@ -4362,6 +4378,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class BooleanDefaultContext extends BooleanExpressionContext {
         public PredicatedContext predicated() {
             return getRuleContext(PredicatedContext.class, 0);
@@ -4388,6 +4405,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ExistsContext extends BooleanExpressionContext {
         public TerminalNode EXISTS() {
             return getToken(SqlBaseParser.EXISTS, 0);
@@ -4418,6 +4436,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class MultiMatchQueryContext extends BooleanExpressionContext {
         public StringContext multiFields;
         public StringContext queryString;
@@ -4459,6 +4478,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class MatchQueryContext extends BooleanExpressionContext {
         public QualifiedNameContext singleField;
         public StringContext queryString;
@@ -4500,6 +4520,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class LogicalBinaryContext extends BooleanExpressionContext {
         public BooleanExpressionContext left;
         public Token operator;
@@ -4704,6 +4725,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class MatchQueryOptionsContext extends ParserRuleContext {
         public List<StringContext> string() {
             return getRuleContexts(StringContext.class);
@@ -4773,6 +4795,7 @@ public final MatchQueryOptionsContext matchQueryOptions() throws RecognitionExce
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class PredicatedContext extends ParserRuleContext {
         public ValueExpressionContext valueExpression() {
             return getRuleContext(ValueExpressionContext.class, 0);
@@ -4836,6 +4859,7 @@ public final PredicatedContext predicated() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class PredicateContext extends ParserRuleContext {
         public Token kind;
         public ValueExpressionContext lower;
@@ -5075,6 +5099,7 @@ public final PredicateContext predicate() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class LikePatternContext extends ParserRuleContext {
         public TerminalNode LIKE() {
             return getToken(SqlBaseParser.LIKE, 0);
@@ -5131,6 +5156,7 @@ public final LikePatternContext likePattern() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class PatternContext extends ParserRuleContext {
         public StringContext value;
 
@@ -5196,6 +5222,7 @@ public final PatternContext pattern() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class PatternEscapeContext extends ParserRuleContext {
         public StringContext escape;
 
@@ -5279,6 +5306,7 @@ public final PatternEscapeContext patternEscape() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ValueExpressionContext extends ParserRuleContext {
         public ValueExpressionContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -5296,6 +5324,7 @@ public void copyFrom(ValueExpressionContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ValueExpressionDefaultContext extends ValueExpressionContext {
         public PrimaryExpressionContext primaryExpression() {
             return getRuleContext(PrimaryExpressionContext.class, 0);
@@ -5322,6 +5351,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ComparisonContext extends ValueExpressionContext {
         public ValueExpressionContext left;
         public ValueExpressionContext right;
@@ -5359,6 +5389,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ArithmeticBinaryContext extends ValueExpressionContext {
         public ValueExpressionContext left;
         public Token operator;
@@ -5413,6 +5444,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ArithmeticUnaryContext extends ValueExpressionContext {
         public Token operator;
 
@@ -5584,9 +5616,7 @@ private ValueExpressionContext valueExpression(int _p) throws RecognitionExcepti
                                     setState(635);
                                     ((ArithmeticBinaryContext) _localctx).operator = _input.LT(1);
                                     _la = _input.LA(1);
-                                    if (!(((((_la - 121)) & ~0x3f) == 0
-                                        && ((1L << (_la - 121)) & ((1L << (ASTERISK - 121)) | (1L << (SLASH - 121)) | (1L << (PERCENT
-                                            - 121)))) != 0))) {
+                                    if (!((((_la - 121)) & ~0x3f) == 0 && ((1L << (_la - 121)) & 7L) != 0)) {
                                         ((ArithmeticBinaryContext) _localctx).operator = (Token) _errHandler.recoverInline(this);
                                     } else {
                                         if (_input.LA(1) == Token.EOF) matchedEOF = true;
@@ -5647,6 +5677,7 @@ private ValueExpressionContext valueExpression(int _p) throws RecognitionExcepti
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class PrimaryExpressionContext extends ParserRuleContext {
         public PrimaryExpressionContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -5664,6 +5695,7 @@ public void copyFrom(PrimaryExpressionContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class DereferenceContext extends PrimaryExpressionContext {
         public QualifiedNameContext qualifiedName() {
             return getRuleContext(QualifiedNameContext.class, 0);
@@ -5690,6 +5722,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class CastContext extends PrimaryExpressionContext {
         public CastExpressionContext castExpression() {
             return getRuleContext(CastExpressionContext.class, 0);
@@ -5716,6 +5749,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ConstantDefaultContext extends PrimaryExpressionContext {
         public ConstantContext constant() {
             return getRuleContext(ConstantContext.class, 0);
@@ -5742,6 +5776,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ExtractContext extends PrimaryExpressionContext {
         public ExtractExpressionContext extractExpression() {
             return getRuleContext(ExtractExpressionContext.class, 0);
@@ -5768,6 +5803,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ParenthesizedExpressionContext extends PrimaryExpressionContext {
         public ExpressionContext expression() {
             return getRuleContext(ExpressionContext.class, 0);
@@ -5794,6 +5830,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class StarContext extends PrimaryExpressionContext {
         public TerminalNode ASTERISK() {
             return getToken(SqlBaseParser.ASTERISK, 0);
@@ -5828,6 +5865,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class CastOperatorExpressionContext extends PrimaryExpressionContext {
         public PrimaryExpressionContext primaryExpression() {
             return getRuleContext(PrimaryExpressionContext.class, 0);
@@ -5862,6 +5900,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class FunctionContext extends PrimaryExpressionContext {
         public FunctionExpressionContext functionExpression() {
             return getRuleContext(FunctionExpressionContext.class, 0);
@@ -5888,6 +5927,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class CurrentDateTimeFunctionContext extends PrimaryExpressionContext {
         public BuiltinDateTimeFunctionContext builtinDateTimeFunction() {
             return getRuleContext(BuiltinDateTimeFunctionContext.class, 0);
@@ -5914,6 +5954,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class SubqueryExpressionContext extends PrimaryExpressionContext {
         public QueryContext query() {
             return getRuleContext(QueryContext.class, 0);
@@ -5940,6 +5981,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class CaseContext extends PrimaryExpressionContext {
         public BooleanExpressionContext operand;
         public BooleanExpressionContext elseClause;
@@ -6052,19 +6094,8 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
                         setState(657);
                         _errHandler.sync(this);
                         _la = _input.LA(1);
-                        if ((((_la) & ~0x3f) == 0
-                            && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << CATALOGS) | (1L << COLUMNS) | (1L
-                                << CURRENT_DATE) | (1L << CURRENT_TIME) | (1L << CURRENT_TIMESTAMP) | (1L << DAY) | (1L << DEBUG) | (1L
-                                    << EXECUTABLE) | (1L << EXPLAIN) | (1L << FIRST) | (1L << FORMAT) | (1L << FULL) | (1L << FUNCTIONS)
-                                | (1L << GRAPHVIZ) | (1L << HOUR) | (1L << INTERVAL) | (1L << LAST) | (1L << LIMIT) | (1L << MAPPED) | (1L
-                                    << MINUTE) | (1L << MONTH))) != 0)
-                            || ((((_la - 70)) & ~0x3f) == 0
-                                && ((1L << (_la - 70)) & ((1L << (OPTIMIZED - 70)) | (1L << (PARSED - 70)) | (1L << (PHYSICAL - 70)) | (1L
-                                    << (PIVOT - 70)) | (1L << (PLAN - 70)) | (1L << (RLIKE - 70)) | (1L << (QUERY - 70)) | (1L << (SCHEMAS
-                                        - 70)) | (1L << (SECOND - 70)) | (1L << (SHOW - 70)) | (1L << (SYS - 70)) | (1L << (TABLES - 70))
-                                    | (1L << (TEXT - 70)) | (1L << (TOP - 70)) | (1L << (TYPE - 70)) | (1L << (TYPES - 70)) | (1L << (VERIFY
-                                        - 70)) | (1L << (YEAR - 70)) | (1L << (IDENTIFIER - 70)) | (1L << (DIGIT_IDENTIFIER - 70)) | (1L
-                                            << (QUOTED_IDENTIFIER - 70)))) != 0)
+                        if (((_la) & ~0x3f) == 0 && ((1L << _la) & -6012133270006398784L) != 0
+                            || (((_la - 70)) & ~0x3f) == 0 && ((1L << (_la - 70)) & -5764607520692920591L) != 0
                             || _la == BACKQUOTED_IDENTIFIER) {
                             {
                                 setState(654);
@@ -6127,26 +6158,9 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
                         setState(672);
                         _errHandler.sync(this);
                         _la = _input.LA(1);
-                        if ((((_la) & ~0x3f) == 0
-                            && ((1L << _la) & ((1L << T__0) | (1L << ANALYZE) | (1L << ANALYZED) | (1L << CASE) | (1L << CAST) | (1L
-                                << CATALOGS) | (1L << COLUMNS) | (1L << CONVERT) | (1L << CURRENT_DATE) | (1L << CURRENT_TIME) | (1L
-                                    << CURRENT_TIMESTAMP) | (1L << DAY) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXISTS) | (1L
-                                        << EXPLAIN) | (1L << EXTRACT) | (1L << FALSE) | (1L << FIRST) | (1L << FORMAT) | (1L << FULL) | (1L
-                                            << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << HOUR) | (1L << INTERVAL) | (1L << LAST) | (1L << LEFT)
-                                | (1L << LIMIT) | (1L << MAPPED) | (1L << MATCH) | (1L << MINUTE) | (1L << MONTH))) != 0)
-                            || ((((_la - 66)) & ~0x3f) == 0
-                                && ((1L << (_la - 66)) & ((1L << (NOT - 66)) | (1L << (NULL - 66)) | (1L << (OPTIMIZED - 66)) | (1L
-                                    << (PARSED - 66)) | (1L << (PHYSICAL - 66)) | (1L << (PIVOT - 66)) | (1L << (PLAN - 66)) | (1L << (RIGHT
-                                        - 66)) | (1L << (RLIKE - 66)) | (1L << (QUERY - 66)) | (1L << (SCHEMAS - 66)) | (1L << (SECOND
-                                            - 66)) | (1L << (SHOW - 66)) | (1L << (SYS - 66)) | (1L << (TABLES - 66)) | (1L << (TEXT - 66))
-                                    | (1L << (TRUE - 66)) | (1L << (TOP - 66)) | (1L << (TYPE - 66)) | (1L << (TYPES - 66)) | (1L << (VERIFY
-                                        - 66)) | (1L << (YEAR - 66)) | (1L << (FUNCTION_ESC - 66)) | (1L << (DATE_ESC - 66)) | (1L
-                                            << (TIME_ESC - 66)) | (1L << (TIMESTAMP_ESC - 66)) | (1L << (GUID_ESC - 66)) | (1L << (PLUS
-                                                - 66)) | (1L << (MINUS - 66)) | (1L << (ASTERISK - 66)) | (1L << (PARAM - 66)) | (1L
-                                                    << (STRING - 66)) | (1L << (INTEGER_VALUE - 66)) | (1L << (DECIMAL_VALUE - 66)))) != 0)
-                            || ((((_la - 130)) & ~0x3f) == 0
-                                && ((1L << (_la - 130)) & ((1L << (IDENTIFIER - 130)) | (1L << (DIGIT_IDENTIFIER - 130)) | (1L
-                                    << (QUOTED_IDENTIFIER - 130)) | (1L << (BACKQUOTED_IDENTIFIER - 130)))) != 0)) {
+                        if (((_la) & ~0x3f) == 0 && ((1L << _la) & -4787154059691900734L) != 0
+                            || (((_la - 66)) & ~0x3f) == 0 && ((1L << (_la - 66)) & -1089854304776749293L) != 0
+                            || (((_la - 130)) & ~0x3f) == 0 && ((1L << (_la - 130)) & 27L) != 0) {
                             {
                                 setState(671);
                                 ((CaseContext) _localctx).operand = booleanExpression(0);
@@ -6220,6 +6234,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class BuiltinDateTimeFunctionContext extends ParserRuleContext {
         public Token name;
 
@@ -6299,6 +6314,7 @@ public final BuiltinDateTimeFunctionContext builtinDateTimeFunction() throws Rec
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class CastExpressionContext extends ParserRuleContext {
         public CastTemplateContext castTemplate() {
             return getRuleContext(CastTemplateContext.class, 0);
@@ -6392,6 +6408,7 @@ public final CastExpressionContext castExpression() throws RecognitionException
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class CastTemplateContext extends ParserRuleContext {
         public TerminalNode CAST() {
             return getToken(SqlBaseParser.CAST, 0);
@@ -6464,6 +6481,7 @@ public final CastTemplateContext castTemplate() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ConvertTemplateContext extends ParserRuleContext {
         public TerminalNode CONVERT() {
             return getToken(SqlBaseParser.CONVERT, 0);
@@ -6532,6 +6550,7 @@ public final ConvertTemplateContext convertTemplate() throws RecognitionExceptio
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ExtractExpressionContext extends ParserRuleContext {
         public ExtractTemplateContext extractTemplate() {
             return getRuleContext(ExtractTemplateContext.class, 0);
@@ -6607,6 +6626,7 @@ public final ExtractExpressionContext extractExpression() throws RecognitionExce
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ExtractTemplateContext extends ParserRuleContext {
         public IdentifierContext field;
 
@@ -6681,6 +6701,7 @@ public final ExtractTemplateContext extractTemplate() throws RecognitionExceptio
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class FunctionExpressionContext extends ParserRuleContext {
         public FunctionTemplateContext functionTemplate() {
             return getRuleContext(FunctionTemplateContext.class, 0);
@@ -6802,6 +6823,7 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class FunctionTemplateContext extends ParserRuleContext {
         public FunctionNameContext functionName() {
             return getRuleContext(FunctionNameContext.class, 0);
@@ -6859,26 +6881,9 @@ public final FunctionTemplateContext functionTemplate() throws RecognitionExcept
                 setState(760);
                 _errHandler.sync(this);
                 _la = _input.LA(1);
-                if ((((_la) & ~0x3f) == 0
-                    && ((1L << _la) & ((1L << T__0) | (1L << ALL) | (1L << ANALYZE) | (1L << ANALYZED) | (1L << CASE) | (1L << CAST) | (1L
-                        << CATALOGS) | (1L << COLUMNS) | (1L << CONVERT) | (1L << CURRENT_DATE) | (1L << CURRENT_TIME) | (1L
-                            << CURRENT_TIMESTAMP) | (1L << DAY) | (1L << DEBUG) | (1L << DISTINCT) | (1L << EXECUTABLE) | (1L << EXISTS)
-                        | (1L << EXPLAIN) | (1L << EXTRACT) | (1L << FALSE) | (1L << FIRST) | (1L << FORMAT) | (1L << FULL) | (1L
-                            << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << HOUR) | (1L << INTERVAL) | (1L << LAST) | (1L << LEFT) | (1L << LIMIT)
-                        | (1L << MAPPED) | (1L << MATCH) | (1L << MINUTE) | (1L << MONTH))) != 0)
-                    || ((((_la - 66)) & ~0x3f) == 0
-                        && ((1L << (_la - 66)) & ((1L << (NOT - 66)) | (1L << (NULL - 66)) | (1L << (OPTIMIZED - 66)) | (1L << (PARSED
-                            - 66)) | (1L << (PHYSICAL - 66)) | (1L << (PIVOT - 66)) | (1L << (PLAN - 66)) | (1L << (RIGHT - 66)) | (1L
-                                << (RLIKE - 66)) | (1L << (QUERY - 66)) | (1L << (SCHEMAS - 66)) | (1L << (SECOND - 66)) | (1L << (SHOW
-                                    - 66)) | (1L << (SYS - 66)) | (1L << (TABLES - 66)) | (1L << (TEXT - 66)) | (1L << (TRUE - 66)) | (1L
-                                        << (TOP - 66)) | (1L << (TYPE - 66)) | (1L << (TYPES - 66)) | (1L << (VERIFY - 66)) | (1L << (YEAR
-                                            - 66)) | (1L << (FUNCTION_ESC - 66)) | (1L << (DATE_ESC - 66)) | (1L << (TIME_ESC - 66)) | (1L
-                                                << (TIMESTAMP_ESC - 66)) | (1L << (GUID_ESC - 66)) | (1L << (PLUS - 66)) | (1L << (MINUS
-                                                    - 66)) | (1L << (ASTERISK - 66)) | (1L << (PARAM - 66)) | (1L << (STRING - 66)) | (1L
-                                                        << (INTEGER_VALUE - 66)) | (1L << (DECIMAL_VALUE - 66)))) != 0)
-                    || ((((_la - 130)) & ~0x3f) == 0
-                        && ((1L << (_la - 130)) & ((1L << (IDENTIFIER - 130)) | (1L << (DIGIT_IDENTIFIER - 130)) | (1L << (QUOTED_IDENTIFIER
-                            - 130)) | (1L << (BACKQUOTED_IDENTIFIER - 130)))) != 0)) {
+                if (((_la) & ~0x3f) == 0 && ((1L << _la) & -4787154059423465246L) != 0
+                    || (((_la - 66)) & ~0x3f) == 0 && ((1L << (_la - 66)) & -1089854304776749293L) != 0
+                    || (((_la - 130)) & ~0x3f) == 0 && ((1L << (_la - 130)) & 27L) != 0) {
                     {
                         setState(750);
                         _errHandler.sync(this);
@@ -6924,6 +6929,7 @@ public final FunctionTemplateContext functionTemplate() throws RecognitionExcept
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class FunctionNameContext extends ParserRuleContext {
         public TerminalNode LEFT() {
             return getToken(SqlBaseParser.LEFT, 0);
@@ -7045,6 +7051,7 @@ public final FunctionNameContext functionName() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ConstantContext extends ParserRuleContext {
         public ConstantContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -7062,6 +7069,7 @@ public void copyFrom(ConstantContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class NullLiteralContext extends ConstantContext {
         public TerminalNode NULL() {
             return getToken(SqlBaseParser.NULL, 0);
@@ -7088,6 +7096,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class TimestampEscapedLiteralContext extends ConstantContext {
         public TerminalNode TIMESTAMP_ESC() {
             return getToken(SqlBaseParser.TIMESTAMP_ESC, 0);
@@ -7122,6 +7131,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class StringLiteralContext extends ConstantContext {
         public List<TerminalNode> STRING() {
             return getTokens(SqlBaseParser.STRING);
@@ -7152,6 +7162,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ParamLiteralContext extends ConstantContext {
         public TerminalNode PARAM() {
             return getToken(SqlBaseParser.PARAM, 0);
@@ -7178,6 +7189,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class TimeEscapedLiteralContext extends ConstantContext {
         public TerminalNode TIME_ESC() {
             return getToken(SqlBaseParser.TIME_ESC, 0);
@@ -7212,6 +7224,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class DateEscapedLiteralContext extends ConstantContext {
         public TerminalNode DATE_ESC() {
             return getToken(SqlBaseParser.DATE_ESC, 0);
@@ -7246,6 +7259,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class IntervalLiteralContext extends ConstantContext {
         public IntervalContext interval() {
             return getRuleContext(IntervalContext.class, 0);
@@ -7272,6 +7286,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class NumericLiteralContext extends ConstantContext {
         public NumberContext number() {
             return getRuleContext(NumberContext.class, 0);
@@ -7298,6 +7313,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class BooleanLiteralContext extends ConstantContext {
         public BooleanValueContext booleanValue() {
             return getRuleContext(BooleanValueContext.class, 0);
@@ -7324,6 +7340,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class GuidEscapedLiteralContext extends ConstantContext {
         public TerminalNode GUID_ESC() {
             return getToken(SqlBaseParser.GUID_ESC, 0);
@@ -7484,6 +7501,7 @@ public final ConstantContext constant() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class ComparisonOperatorContext extends ParserRuleContext {
         public TerminalNode EQ() {
             return getToken(SqlBaseParser.EQ, 0);
@@ -7548,9 +7566,7 @@ public final ComparisonOperatorContext comparisonOperator() throws RecognitionEx
             {
                 setState(797);
                 _la = _input.LA(1);
-                if (!(((((_la - 112)) & ~0x3f) == 0
-                    && ((1L << (_la - 112)) & ((1L << (EQ - 112)) | (1L << (NULLEQ - 112)) | (1L << (NEQ - 112)) | (1L << (LT - 112)) | (1L
-                        << (LTE - 112)) | (1L << (GT - 112)) | (1L << (GTE - 112)))) != 0))) {
+                if (!((((_la - 112)) & ~0x3f) == 0 && ((1L << (_la - 112)) & 127L) != 0)) {
                     _errHandler.recoverInline(this);
                 } else {
                     if (_input.LA(1) == Token.EOF) matchedEOF = true;
@@ -7568,6 +7584,7 @@ public final ComparisonOperatorContext comparisonOperator() throws RecognitionEx
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class BooleanValueContext extends ParserRuleContext {
         public TerminalNode TRUE() {
             return getToken(SqlBaseParser.TRUE, 0);
@@ -7630,6 +7647,7 @@ public final BooleanValueContext booleanValue() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class IntervalContext extends ParserRuleContext {
         public Token sign;
         public NumberContext valueNumeric;
@@ -7764,6 +7782,7 @@ public final IntervalContext interval() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class IntervalFieldContext extends ParserRuleContext {
         public TerminalNode YEAR() {
             return getToken(SqlBaseParser.YEAR, 0);
@@ -7848,12 +7867,8 @@ public final IntervalFieldContext intervalField() throws RecognitionException {
             {
                 setState(814);
                 _la = _input.LA(1);
-                if (!((((_la) & ~0x3f) == 0
-                    && ((1L << _la) & ((1L << DAY) | (1L << DAYS) | (1L << HOUR) | (1L << HOURS) | (1L << MINUTE) | (1L << MINUTES) | (1L
-                        << MONTH))) != 0)
-                    || ((((_la - 64)) & ~0x3f) == 0
-                        && ((1L << (_la - 64)) & ((1L << (MONTHS - 64)) | (1L << (SECOND - 64)) | (1L << (SECONDS - 64)) | (1L << (YEAR
-                            - 64)) | (1L << (YEARS - 64)))) != 0))) {
+                if (!(((_la) & ~0x3f) == 0 && ((1L << _la) & -2305420796723462144L) != 0
+                    || (((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & 412317646849L) != 0)) {
                     _errHandler.recoverInline(this);
                 } else {
                     if (_input.LA(1) == Token.EOF) matchedEOF = true;
@@ -7871,6 +7886,7 @@ public final IntervalFieldContext intervalField() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class DataTypeContext extends ParserRuleContext {
         public DataTypeContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -7888,6 +7904,7 @@ public void copyFrom(DataTypeContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class PrimitiveDataTypeContext extends DataTypeContext {
         public IdentifierContext identifier() {
             return getRuleContext(IdentifierContext.class, 0);
@@ -7934,6 +7951,7 @@ public final DataTypeContext dataType() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class QualifiedNameContext extends ParserRuleContext {
         public List<IdentifierContext> identifier() {
             return getRuleContexts(IdentifierContext.class);
@@ -8015,6 +8033,7 @@ public final QualifiedNameContext qualifiedName() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class IdentifierContext extends ParserRuleContext {
         public QuoteIdentifierContext quoteIdentifier() {
             return getRuleContext(QuoteIdentifierContext.class, 0);
@@ -8125,6 +8144,7 @@ public final IdentifierContext identifier() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class TableIdentifierContext extends ParserRuleContext {
         public IdentifierContext catalog;
         public IdentifierContext name;
@@ -8180,19 +8200,8 @@ public final TableIdentifierContext tableIdentifier() throws RecognitionExceptio
                     setState(835);
                     _errHandler.sync(this);
                     _la = _input.LA(1);
-                    if ((((_la) & ~0x3f) == 0
-                        && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << CURRENT_DATE)
-                            | (1L << CURRENT_TIME) | (1L << CURRENT_TIMESTAMP) | (1L << DAY) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L
-                                << EXPLAIN) | (1L << FIRST) | (1L << FORMAT) | (1L << FULL) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L
-                                    << HOUR) | (1L << INTERVAL) | (1L << LAST) | (1L << LIMIT) | (1L << MAPPED) | (1L << MINUTE) | (1L
-                                        << MONTH))) != 0)
-                        || ((((_la - 70)) & ~0x3f) == 0
-                            && ((1L << (_la - 70)) & ((1L << (OPTIMIZED - 70)) | (1L << (PARSED - 70)) | (1L << (PHYSICAL - 70)) | (1L
-                                << (PIVOT - 70)) | (1L << (PLAN - 70)) | (1L << (RLIKE - 70)) | (1L << (QUERY - 70)) | (1L << (SCHEMAS
-                                    - 70)) | (1L << (SECOND - 70)) | (1L << (SHOW - 70)) | (1L << (SYS - 70)) | (1L << (TABLES - 70)) | (1L
-                                        << (TEXT - 70)) | (1L << (TOP - 70)) | (1L << (TYPE - 70)) | (1L << (TYPES - 70)) | (1L << (VERIFY
-                                            - 70)) | (1L << (YEAR - 70)) | (1L << (IDENTIFIER - 70)) | (1L << (DIGIT_IDENTIFIER - 70)) | (1L
-                                                << (QUOTED_IDENTIFIER - 70)))) != 0)
+                    if (((_la) & ~0x3f) == 0 && ((1L << _la) & -6012133270006398784L) != 0
+                        || (((_la - 70)) & ~0x3f) == 0 && ((1L << (_la - 70)) & -5764607520692920591L) != 0
                         || _la == BACKQUOTED_IDENTIFIER) {
                         {
                             setState(832);
@@ -8234,6 +8243,7 @@ public final TableIdentifierContext tableIdentifier() throws RecognitionExceptio
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class QuoteIdentifierContext extends ParserRuleContext {
         public QuoteIdentifierContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -8251,6 +8261,7 @@ public void copyFrom(QuoteIdentifierContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class BackQuotedIdentifierContext extends QuoteIdentifierContext {
         public TerminalNode BACKQUOTED_IDENTIFIER() {
             return getToken(SqlBaseParser.BACKQUOTED_IDENTIFIER, 0);
@@ -8277,6 +8288,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class QuotedIdentifierContext extends QuoteIdentifierContext {
         public TerminalNode QUOTED_IDENTIFIER() {
             return getToken(SqlBaseParser.QUOTED_IDENTIFIER, 0);
@@ -8337,6 +8349,7 @@ public final QuoteIdentifierContext quoteIdentifier() throws RecognitionExceptio
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class UnquoteIdentifierContext extends ParserRuleContext {
         public UnquoteIdentifierContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -8354,6 +8367,7 @@ public void copyFrom(UnquoteIdentifierContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class DigitIdentifierContext extends UnquoteIdentifierContext {
         public TerminalNode DIGIT_IDENTIFIER() {
             return getToken(SqlBaseParser.DIGIT_IDENTIFIER, 0);
@@ -8380,6 +8394,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class UnquotedIdentifierContext extends UnquoteIdentifierContext {
         public TerminalNode IDENTIFIER() {
             return getToken(SqlBaseParser.IDENTIFIER, 0);
@@ -8491,6 +8506,7 @@ public final UnquoteIdentifierContext unquoteIdentifier() throws RecognitionExce
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class NumberContext extends ParserRuleContext {
         public NumberContext(ParserRuleContext parent, int invokingState) {
             super(parent, invokingState);
@@ -8508,6 +8524,7 @@ public void copyFrom(NumberContext ctx) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class DecimalLiteralContext extends NumberContext {
         public TerminalNode DECIMAL_VALUE() {
             return getToken(SqlBaseParser.DECIMAL_VALUE, 0);
@@ -8534,6 +8551,7 @@ public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
         }
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class IntegerLiteralContext extends NumberContext {
         public TerminalNode INTEGER_VALUE() {
             return getToken(SqlBaseParser.INTEGER_VALUE, 0);
@@ -8594,6 +8612,7 @@ public final NumberContext number() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class StringContext extends ParserRuleContext {
         public TerminalNode PARAM() {
             return getToken(SqlBaseParser.PARAM, 0);
@@ -8656,6 +8675,7 @@ public final StringContext string() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class WhenClauseContext extends ParserRuleContext {
         public ExpressionContext condition;
         public ExpressionContext result;
@@ -8727,6 +8747,7 @@ public final WhenClauseContext whenClause() throws RecognitionException {
         return _localctx;
     }
 
+    @SuppressWarnings("CheckReturnValue")
     public static class NonReservedContext extends ParserRuleContext {
         public TerminalNode ANALYZE() {
             return getToken(SqlBaseParser.ANALYZE, 0);
@@ -8927,17 +8948,8 @@ public final NonReservedContext nonReserved() throws RecognitionException {
             {
                 setState(866);
                 _la = _input.LA(1);
-                if (!((((_la) & ~0x3f) == 0
-                    && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << CURRENT_DATE) | (1L
-                        << CURRENT_TIME) | (1L << CURRENT_TIMESTAMP) | (1L << DAY) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXPLAIN)
-                        | (1L << FIRST) | (1L << FORMAT) | (1L << FULL) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << HOUR) | (1L
-                            << INTERVAL) | (1L << LAST) | (1L << LIMIT) | (1L << MAPPED) | (1L << MINUTE) | (1L << MONTH))) != 0)
-                    || ((((_la - 70)) & ~0x3f) == 0
-                        && ((1L << (_la - 70)) & ((1L << (OPTIMIZED - 70)) | (1L << (PARSED - 70)) | (1L << (PHYSICAL - 70)) | (1L << (PIVOT
-                            - 70)) | (1L << (PLAN - 70)) | (1L << (RLIKE - 70)) | (1L << (QUERY - 70)) | (1L << (SCHEMAS - 70)) | (1L
-                                << (SECOND - 70)) | (1L << (SHOW - 70)) | (1L << (SYS - 70)) | (1L << (TABLES - 70)) | (1L << (TEXT - 70))
-                            | (1L << (TOP - 70)) | (1L << (TYPE - 70)) | (1L << (TYPES - 70)) | (1L << (VERIFY - 70)) | (1L << (YEAR
-                                - 70)))) != 0))) {
+                if (!(((_la) & ~0x3f) == 0 && ((1L << _la) & -6012133270006398784L) != 0
+                    || (((_la - 70)) & ~0x3f) == 0 && ((1L << (_la - 70)) & 2341314289L) != 0)) {
                     _errHandler.recoverInline(this);
                 } else {
                     if (_input.LA(1) == Token.EOF) matchedEOF = true;
@@ -8997,350 +9009,599 @@ private boolean primaryExpression_sempred(PrimaryExpressionContext _localctx, in
         return true;
     }
 
-    public static final String _serializedATN = "\3\u608b\ua72a\u8133\ub9ed\u417c\u3be7\u7786\u5964\3\u008d\u0367\4\2\t"
-        + "\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13"
-        + "\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"
-        + "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"
-        + "\4\32\t\32\4\33\t\33\4\34\t\34\4\35\t\35\4\36\t\36\4\37\t\37\4 \t \4!"
-        + "\t!\4\"\t\"\4#\t#\4$\t$\4%\t%\4&\t&\4\'\t\'\4(\t(\4)\t)\4*\t*\4+\t+\4"
-        + ",\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\4\61\t\61\4\62\t\62\4\63\t\63\4\64\t"
-        + "\64\4\65\t\65\4\66\t\66\4\67\t\67\48\t8\49\t9\4:\t:\4;\t;\4<\t<\4=\t="
-        + "\3\2\3\2\3\2\3\3\3\3\3\3\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\7\4\u008a"
-        + "\n\4\f\4\16\4\u008d\13\4\3\4\5\4\u0090\n\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4"
-        + "\7\4\u0099\n\4\f\4\16\4\u009c\13\4\3\4\5\4\u009f\n\4\3\4\3\4\3\4\3\4\3"
-        + "\4\3\4\5\4\u00a7\n\4\5\4\u00a9\n\4\3\4\3\4\5\4\u00ad\n\4\3\4\3\4\5\4\u00b1"
-        + "\n\4\3\4\3\4\3\4\3\4\5\4\u00b7\n\4\3\4\3\4\5\4\u00bb\n\4\3\4\3\4\3\4\5"
-        + "\4\u00c0\n\4\3\4\3\4\3\4\5\4\u00c5\n\4\3\4\3\4\5\4\u00c9\n\4\3\4\3\4\5"
-        + "\4\u00cd\n\4\3\4\3\4\3\4\5\4\u00d2\n\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4"
-        + "\5\4\u00dc\n\4\3\4\3\4\5\4\u00e0\n\4\3\4\3\4\3\4\3\4\7\4\u00e6\n\4\f\4"
-        + "\16\4\u00e9\13\4\5\4\u00eb\n\4\3\4\3\4\3\4\3\4\5\4\u00f1\n\4\3\4\3\4\3"
-        + "\4\5\4\u00f6\n\4\3\4\5\4\u00f9\n\4\3\4\3\4\3\4\5\4\u00fe\n\4\3\4\5\4\u0101"
-        + "\n\4\5\4\u0103\n\4\3\5\3\5\3\5\3\5\7\5\u0109\n\5\f\5\16\5\u010c\13\5\5"
-        + "\5\u010e\n\5\3\5\3\5\3\6\3\6\3\6\3\6\3\6\3\6\7\6\u0118\n\6\f\6\16\6\u011b"
-        + "\13\6\5\6\u011d\n\6\3\6\5\6\u0120\n\6\3\7\3\7\3\7\3\7\3\7\5\7\u0127\n"
-        + "\7\3\b\3\b\3\b\3\b\3\b\5\b\u012e\n\b\3\t\3\t\5\t\u0132\n\t\3\t\3\t\5\t"
-        + "\u0136\n\t\3\n\3\n\5\n\u013a\n\n\3\n\5\n\u013d\n\n\3\n\3\n\5\n\u0141\n"
-        + "\n\3\n\3\n\5\n\u0145\n\n\3\n\3\n\3\n\5\n\u014a\n\n\3\n\3\n\5\n\u014e\n"
-        + "\n\3\13\3\13\3\13\3\13\7\13\u0154\n\13\f\13\16\13\u0157\13\13\3\13\5\13"
-        + "\u015a\n\13\3\f\5\f\u015d\n\f\3\f\3\f\3\f\7\f\u0162\n\f\f\f\16\f\u0165"
-        + "\13\f\3\r\3\r\3\16\3\16\3\16\3\16\7\16\u016d\n\16\f\16\16\16\u0170\13"
-        + "\16\5\16\u0172\n\16\3\16\3\16\5\16\u0176\n\16\3\17\3\17\3\17\3\17\3\17"
-        + "\3\17\3\20\3\20\3\20\3\21\3\21\3\22\3\22\3\22\7\22\u0186\n\22\f\22\16"
-        + "\22\u0189\13\22\3\23\3\23\5\23\u018d\n\23\3\23\5\23\u0190\n\23\3\24\3"
-        + "\24\7\24\u0194\n\24\f\24\16\24\u0197\13\24\3\25\3\25\3\25\3\25\5\25\u019d"
-        + "\n\25\3\25\3\25\3\25\3\25\3\25\5\25\u01a4\n\25\3\26\5\26\u01a7\n\26\3"
-        + "\26\3\26\5\26\u01ab\n\26\3\26\3\26\5\26\u01af\n\26\3\26\3\26\5\26\u01b3"
-        + "\n\26\5\26\u01b5\n\26\3\27\3\27\3\27\3\27\3\27\3\27\3\27\7\27\u01be\n"
-        + "\27\f\27\16\27\u01c1\13\27\3\27\3\27\5\27\u01c5\n\27\3\30\5\30\u01c8\n"
-        + "\30\3\30\3\30\5\30\u01cc\n\30\3\30\5\30\u01cf\n\30\3\30\3\30\3\30\3\30"
-        + "\5\30\u01d5\n\30\3\30\5\30\u01d8\n\30\3\30\3\30\3\30\3\30\5\30\u01de\n"
-        + "\30\3\30\5\30\u01e1\n\30\5\30\u01e3\n\30\3\31\3\31\3\31\3\31\3\31\3\31"
-        + "\3\31\3\31\3\31\3\31\3\31\3\32\3\32\3\32\7\32\u01f3\n\32\f\32\16\32\u01f6"
-        + "\13\32\3\33\3\33\5\33\u01fa\n\33\3\33\5\33\u01fd\n\33\3\34\3\34\3\35\3"
-        + "\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3"
-        + "\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3"
-        + "\35\3\35\5\35\u0220\n\35\3\35\3\35\3\35\3\35\3\35\3\35\7\35\u0228\n\35"
-        + "\f\35\16\35\u022b\13\35\3\36\3\36\7\36\u022f\n\36\f\36\16\36\u0232\13"
-        + "\36\3\37\3\37\5\37\u0236\n\37\3 \5 \u0239\n \3 \3 \3 \3 \3 \3 \5 \u0241"
-        + "\n \3 \3 \3 \3 \3 \7 \u0248\n \f \16 \u024b\13 \3 \3 \3 \5 \u0250\n \3"
-        + " \3 \3 \3 \3 \3 \5 \u0258\n \3 \3 \3 \5 \u025d\n \3 \3 \3 \3 \5 \u0263"
-        + "\n \3 \5 \u0266\n \3!\3!\3!\3\"\3\"\5\"\u026d\n\"\3#\3#\3#\3#\3#\3#\5"
-        + "#\u0275\n#\3$\3$\3$\3$\5$\u027b\n$\3$\3$\3$\3$\3$\3$\3$\3$\3$\3$\7$\u0287"
-        + "\n$\f$\16$\u028a\13$\3%\3%\3%\3%\3%\3%\3%\3%\5%\u0294\n%\3%\3%\3%\3%\3"
-        + "%\3%\3%\3%\3%\3%\3%\3%\3%\5%\u02a3\n%\3%\6%\u02a6\n%\r%\16%\u02a7\3%\3"
-        + "%\5%\u02ac\n%\3%\3%\5%\u02b0\n%\3%\3%\3%\7%\u02b5\n%\f%\16%\u02b8\13%"
-        + "\3&\3&\3&\5&\u02bd\n&\3\'\3\'\3\'\3\'\3\'\3\'\3\'\3\'\3\'\3\'\5\'\u02c9"
-        + "\n\'\3(\3(\3(\3(\3(\3(\3(\3)\3)\3)\3)\3)\3)\3)\3*\3*\3*\3*\3*\5*\u02de"
-        + "\n*\3+\3+\3+\3+\3+\3+\3+\3,\3,\3,\3,\3,\5,\u02ec\n,\3-\3-\3-\5-\u02f1"
-        + "\n-\3-\3-\3-\7-\u02f6\n-\f-\16-\u02f9\13-\5-\u02fb\n-\3-\3-\3.\3.\3.\5"
-        + ".\u0302\n.\3/\3/\3/\3/\3/\6/\u0309\n/\r/\16/\u030a\3/\3/\3/\3/\3/\3/\3"
-        + "/\3/\3/\3/\3/\3/\3/\3/\3/\3/\3/\5/\u031e\n/\3\60\3\60\3\61\3\61\3\62\3"
-        + "\62\5\62\u0326\n\62\3\62\3\62\5\62\u032a\n\62\3\62\3\62\3\62\5\62\u032f"
-        + "\n\62\3\63\3\63\3\64\3\64\3\65\3\65\3\65\7\65\u0338\n\65\f\65\16\65\u033b"
-        + "\13\65\3\65\3\65\3\66\3\66\5\66\u0341\n\66\3\67\3\67\3\67\5\67\u0346\n"
-        + "\67\3\67\3\67\3\67\3\67\5\67\u034c\n\67\3\67\5\67\u034f\n\67\38\38\58"
-        + "\u0353\n8\39\39\39\59\u0358\n9\3:\3:\5:\u035c\n:\3;\3;\3<\3<\3<\3<\3<"
-        + "\3=\3=\3=\2\58FH>\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,.\60\62"
-        + "\64\668:<>@BDFHJLNPRTVXZ\\^`bdfhjlnprtvx\2\22\b\2\7\7\t\t\"\"==HHLL\4"
-        + "\2..[[\4\2\t\tHH\4\2**\63\63\3\2\34\35\3\2yz\4\2\7\7\u0082\u0082\4\2\r"
-        + "\r\34\34\4\2\'\'99\4\2\7\7\36\36\3\2{}\3\2rx\4\2&&]]\7\2\31\32\61\62?"
-        + "BTUgh\3\2\u0080\u0081\31\2\b\t\23\24\26\31\33\33\"\"$$\'\')),.\61\61\66"
-        + "\6699<=??AAHHLOQTWXZ[_accgg\2\u03cf\2z\3\2\2\2\4}\3\2\2\2\6\u0102\3\2"
-        + "\2\2\b\u010d\3\2\2\2\n\u0111\3\2\2\2\f\u0126\3\2\2\2\16\u012d\3\2\2\2"
-        + "\20\u012f\3\2\2\2\22\u0137\3\2\2\2\24\u014f\3\2\2\2\26\u015c\3\2\2\2\30"
-        + "\u0166\3\2\2\2\32\u0175\3\2\2\2\34\u0177\3\2\2\2\36\u017d\3\2\2\2 \u0180"
-        + "\3\2\2\2\"\u0182\3\2\2\2$\u018a\3\2\2\2&\u0191\3\2\2\2(\u01a3\3\2\2\2"
-        + "*\u01b4\3\2\2\2,\u01c4\3\2\2\2.\u01e2\3\2\2\2\60\u01e4\3\2\2\2\62\u01ef"
-        + "\3\2\2\2\64\u01f7\3\2\2\2\66\u01fe\3\2\2\28\u021f\3\2\2\2:\u0230\3\2\2"
-        + "\2<\u0233\3\2\2\2>\u0265\3\2\2\2@\u0267\3\2\2\2B\u026a\3\2\2\2D\u0274"
-        + "\3\2\2\2F\u027a\3\2\2\2H\u02af\3\2\2\2J\u02bc\3\2\2\2L\u02c8\3\2\2\2N"
-        + "\u02ca\3\2\2\2P\u02d1\3\2\2\2R\u02dd\3\2\2\2T\u02df\3\2\2\2V\u02eb\3\2"
-        + "\2\2X\u02ed\3\2\2\2Z\u0301\3\2\2\2\\\u031d\3\2\2\2^\u031f\3\2\2\2`\u0321"
-        + "\3\2\2\2b\u0323\3\2\2\2d\u0330\3\2\2\2f\u0332\3\2\2\2h\u0339\3\2\2\2j"
-        + "\u0340\3\2\2\2l\u034e\3\2\2\2n\u0352\3\2\2\2p\u0357\3\2\2\2r\u035b\3\2"
-        + "\2\2t\u035d\3\2\2\2v\u035f\3\2\2\2x\u0364\3\2\2\2z{\5\6\4\2{|\7\2\2\3"
-        + "|\3\3\2\2\2}~\5\66\34\2~\177\7\2\2\3\177\5\3\2\2\2\u0080\u0103\5\b\5\2"
-        + "\u0081\u008f\7$\2\2\u0082\u008b\7\3\2\2\u0083\u0084\7O\2\2\u0084\u008a"
-        + "\t\2\2\2\u0085\u0086\7)\2\2\u0086\u008a\t\3\2\2\u0087\u0088\7c\2\2\u0088"
-        + "\u008a\5`\61\2\u0089\u0083\3\2\2\2\u0089\u0085\3\2\2\2\u0089\u0087\3\2"
-        + "\2\2\u008a\u008d\3\2\2\2\u008b\u0089\3\2\2\2\u008b\u008c\3\2\2\2\u008c"
-        + "\u008e\3\2\2\2\u008d\u008b\3\2\2\2\u008e\u0090\7\4\2\2\u008f\u0082\3\2"
-        + "\2\2\u008f\u0090\3\2\2\2\u0090\u0091\3\2\2\2\u0091\u0103\5\6\4\2\u0092"
-        + "\u009e\7\33\2\2\u0093\u009a\7\3\2\2\u0094\u0095\7O\2\2\u0095\u0099\t\4"
-        + "\2\2\u0096\u0097\7)\2\2\u0097\u0099\t\3\2\2\u0098\u0094\3\2\2\2\u0098"
-        + "\u0096\3\2\2\2\u0099\u009c\3\2\2\2\u009a\u0098\3\2\2\2\u009a\u009b\3\2"
-        + "\2\2\u009b\u009d\3\2\2\2\u009c\u009a\3\2\2\2\u009d\u009f\7\4\2\2\u009e"
-        + "\u0093\3\2\2\2\u009e\u009f\3\2\2\2\u009f\u00a0\3\2\2\2\u00a0\u0103\5\6"
-        + "\4\2\u00a1\u00a2\7W\2\2\u00a2\u00a8\7Z\2\2\u00a3\u00a6\7\22\2\2\u00a4"
-        + "\u00a7\5@!\2\u00a5\u00a7\5t;\2\u00a6\u00a4\3\2\2\2\u00a6\u00a5\3\2\2\2"
-        + "\u00a7\u00a9\3\2\2\2\u00a8\u00a3\3\2\2\2\u00a8\u00a9\3\2\2\2\u00a9\u00ac"
-        + "\3\2\2\2\u00aa\u00ab\7\64\2\2\u00ab\u00ad\7+\2\2\u00ac\u00aa\3\2\2\2\u00ac"
-        + "\u00ad\3\2\2\2\u00ad\u00b0\3\2\2\2\u00ae\u00b1\5@!\2\u00af\u00b1\5l\67"
-        + "\2\u00b0\u00ae\3\2\2\2\u00b0\u00af\3\2\2\2\u00b0\u00b1\3\2\2\2\u00b1\u0103"
-        + "\3\2\2\2\u00b2\u00b3\7W\2\2\u00b3\u00b6\7\24\2\2\u00b4\u00b5\7\22\2\2"
-        + "\u00b5\u00b7\5t;\2\u00b6\u00b4\3\2\2\2\u00b6\u00b7\3\2\2\2\u00b7\u00ba"
-        + "\3\2\2\2\u00b8\u00b9\7\64\2\2\u00b9\u00bb\7+\2\2\u00ba\u00b8\3\2\2\2\u00ba"
-        + "\u00bb\3\2\2\2\u00bb\u00bc\3\2\2\2\u00bc\u00bf\t\5\2\2\u00bd\u00c0\5@"
-        + "!\2\u00be\u00c0\5l\67\2\u00bf\u00bd\3\2\2\2\u00bf\u00be\3\2\2\2\u00c0"
-        + "\u0103\3\2\2\2\u00c1\u00c4\t\6\2\2\u00c2\u00c3\7\22\2\2\u00c3\u00c5\5"
-        + "t;\2\u00c4\u00c2\3\2\2\2\u00c4\u00c5\3\2\2\2\u00c5\u00c8\3\2\2\2\u00c6"
-        + "\u00c7\7\64\2\2\u00c7\u00c9\7+\2\2\u00c8\u00c6\3\2\2\2\u00c8\u00c9\3\2"
-        + "\2\2\u00c9\u00cc\3\2\2\2\u00ca\u00cd\5@!\2\u00cb\u00cd\5l\67\2\u00cc\u00ca"
-        + "\3\2\2\2\u00cc\u00cb\3\2\2\2\u00cd\u0103\3\2\2\2\u00ce\u00cf\7W\2\2\u00cf"
-        + "\u00d1\7-\2\2\u00d0\u00d2\5@!\2\u00d1\u00d0\3\2\2\2\u00d1\u00d2\3\2\2"
-        + "\2\u00d2\u0103\3\2\2\2\u00d3\u00d4\7W\2\2\u00d4\u0103\7S\2\2\u00d5\u00d6"
-        + "\7W\2\2\u00d6\u0103\7\23\2\2\u00d7\u00d8\7X\2\2\u00d8\u00db\7Z\2\2\u00d9"
-        + "\u00da\7\22\2\2\u00da\u00dc\5@!\2\u00db\u00d9\3\2\2\2\u00db\u00dc\3\2"
-        + "\2\2\u00dc\u00df\3\2\2\2\u00dd\u00e0\5@!\2\u00de\u00e0\5l\67\2\u00df\u00dd"
-        + "\3\2\2\2\u00df\u00de\3\2\2\2\u00df\u00e0\3\2\2\2\u00e0\u00ea\3\2\2\2\u00e1"
-        + "\u00e2\7`\2\2\u00e2\u00e7\5t;\2\u00e3\u00e4\7\5\2\2\u00e4\u00e6\5t;\2"
-        + "\u00e5\u00e3\3\2\2\2\u00e6\u00e9\3\2\2\2\u00e7\u00e5\3\2\2\2\u00e7\u00e8"
-        + "\3\2\2\2\u00e8\u00eb\3\2\2\2\u00e9\u00e7\3\2\2\2\u00ea\u00e1\3\2\2\2\u00ea"
-        + "\u00eb\3\2\2\2\u00eb\u0103\3\2\2\2\u00ec\u00ed\7X\2\2\u00ed\u00f0\7\24"
-        + "\2\2\u00ee\u00ef\7\22\2\2\u00ef\u00f1\5t;\2\u00f0\u00ee\3\2\2\2\u00f0"
-        + "\u00f1\3\2\2\2\u00f1\u00f5\3\2\2\2\u00f2\u00f3\7Y\2\2\u00f3\u00f6\5@!"
-        + "\2\u00f4\u00f6\5l\67\2\u00f5\u00f2\3\2\2\2\u00f5\u00f4\3\2\2\2\u00f5\u00f6"
-        + "\3\2\2\2\u00f6\u00f8\3\2\2\2\u00f7\u00f9\5@!\2\u00f8\u00f7\3\2\2\2\u00f8"
-        + "\u00f9\3\2\2\2\u00f9\u0103\3\2\2\2\u00fa\u00fb\7X\2\2\u00fb\u0100\7a\2"
-        + "\2\u00fc\u00fe\t\7\2\2\u00fd\u00fc\3\2\2\2\u00fd\u00fe\3\2\2\2\u00fe\u00ff"
-        + "\3\2\2\2\u00ff\u0101\5r:\2\u0100\u00fd\3\2\2\2\u0100\u0101\3\2\2\2\u0101"
-        + "\u0103\3\2\2\2\u0102\u0080\3\2\2\2\u0102\u0081\3\2\2\2\u0102\u0092\3\2"
-        + "\2\2\u0102\u00a1\3\2\2\2\u0102\u00b2\3\2\2\2\u0102\u00c1\3\2\2\2\u0102"
-        + "\u00ce\3\2\2\2\u0102\u00d3\3\2\2\2\u0102\u00d5\3\2\2\2\u0102\u00d7\3\2"
-        + "\2\2\u0102\u00ec\3\2\2\2\u0102\u00fa\3\2\2\2\u0103\7\3\2\2\2\u0104\u0105"
-        + "\7f\2\2\u0105\u010a\5\34\17\2\u0106\u0107\7\5\2\2\u0107\u0109\5\34\17"
-        + "\2\u0108\u0106\3\2\2\2\u0109\u010c\3\2\2\2\u010a\u0108\3\2\2\2\u010a\u010b"
-        + "\3\2\2\2\u010b\u010e\3\2\2\2\u010c\u010a\3\2\2\2\u010d\u0104\3\2\2\2\u010d"
-        + "\u010e\3\2\2\2\u010e\u010f\3\2\2\2\u010f\u0110\5\n\6\2\u0110\t\3\2\2\2"
-        + "\u0111\u011c\5\16\b\2\u0112\u0113\7J\2\2\u0113\u0114\7\17\2\2\u0114\u0119"
-        + "\5\20\t\2\u0115\u0116\7\5\2\2\u0116\u0118\5\20\t\2\u0117\u0115\3\2\2\2"
-        + "\u0118\u011b\3\2\2\2\u0119\u0117\3\2\2\2\u0119\u011a\3\2\2\2\u011a\u011d"
-        + "\3\2\2\2\u011b\u0119\3\2\2\2\u011c\u0112\3\2\2\2\u011c\u011d\3\2\2\2\u011d"
-        + "\u011f\3\2\2\2\u011e\u0120\5\f\7\2\u011f\u011e\3\2\2\2\u011f\u0120\3\2"
-        + "\2\2\u0120\13\3\2\2\2\u0121\u0122\7<\2\2\u0122\u0127\t\b\2\2\u0123\u0124"
-        + "\7k\2\2\u0124\u0125\t\b\2\2\u0125\u0127\7q\2\2\u0126\u0121\3\2\2\2\u0126"
-        + "\u0123\3\2\2\2\u0127\r\3\2\2\2\u0128\u012e\5\22\n\2\u0129\u012a\7\3\2"
-        + "\2\u012a\u012b\5\n\6\2\u012b\u012c\7\4\2\2\u012c\u012e\3\2\2\2\u012d\u0128"
-        + "\3\2\2\2\u012d\u0129\3\2\2\2\u012e\17\3\2\2\2\u012f\u0131\5\66\34\2\u0130"
-        + "\u0132\t\t\2\2\u0131\u0130\3\2\2\2\u0131\u0132\3\2\2\2\u0132\u0135\3\2"
-        + "\2\2\u0133\u0134\7F\2\2\u0134\u0136\t\n\2\2\u0135\u0133\3\2\2\2\u0135"
-        + "\u0136\3\2\2\2\u0136\21\3\2\2\2\u0137\u0139\7V\2\2\u0138\u013a\5\36\20"
-        + "\2\u0139\u0138\3\2\2\2\u0139\u013a\3\2\2\2\u013a\u013c\3\2\2\2\u013b\u013d"
-        + "\5 \21\2\u013c\u013b\3\2\2\2\u013c\u013d\3\2\2\2\u013d\u013e\3\2\2\2\u013e"
-        + "\u0140\5\"\22\2\u013f\u0141\5\24\13\2\u0140\u013f\3\2\2\2\u0140\u0141"
-        + "\3\2\2\2\u0141\u0144\3\2\2\2\u0142\u0143\7e\2\2\u0143\u0145\58\35\2\u0144"
-        + "\u0142\3\2\2\2\u0144\u0145\3\2\2\2\u0145\u0149\3\2\2\2\u0146\u0147\7/"
-        + "\2\2\u0147\u0148\7\17\2\2\u0148\u014a\5\26\f\2\u0149\u0146\3\2\2\2\u0149"
-        + "\u014a\3\2\2\2\u014a\u014d\3\2\2\2\u014b\u014c\7\60\2\2\u014c\u014e\5"
-        + "8\35\2\u014d\u014b\3\2\2\2\u014d\u014e\3\2\2\2\u014e\23\3\2\2\2\u014f"
-        + "\u0150\7*\2\2\u0150\u0155\5&\24\2\u0151\u0152\7\5\2\2\u0152\u0154\5&\24"
-        + "\2\u0153\u0151\3\2\2\2\u0154\u0157\3\2\2\2\u0155\u0153\3\2\2\2\u0155\u0156"
-        + "\3\2\2\2\u0156\u0159\3\2\2\2\u0157\u0155\3\2\2\2\u0158\u015a\5\60\31\2"
-        + "\u0159\u0158\3\2\2\2\u0159\u015a\3\2\2\2\u015a\25\3\2\2\2\u015b\u015d"
-        + "\5 \21\2\u015c\u015b\3\2\2\2\u015c\u015d\3\2\2\2\u015d\u015e\3\2\2\2\u015e"
-        + "\u0163\5\30\r\2\u015f\u0160\7\5\2\2\u0160\u0162\5\30\r\2\u0161\u015f\3"
-        + "\2\2\2\u0162\u0165\3\2\2\2\u0163\u0161\3\2\2\2\u0163\u0164\3\2\2\2\u0164"
-        + "\27\3\2\2\2\u0165\u0163\3\2\2\2\u0166\u0167\5\32\16\2\u0167\31\3\2\2\2"
-        + "\u0168\u0171\7\3\2\2\u0169\u016e\5\66\34\2\u016a\u016b\7\5\2\2\u016b\u016d"
-        + "\5\66\34\2\u016c\u016a\3\2\2\2\u016d\u0170\3\2\2\2\u016e\u016c\3\2\2\2"
-        + "\u016e\u016f\3\2\2\2\u016f\u0172\3\2\2\2\u0170\u016e\3\2\2\2\u0171\u0169"
-        + "\3\2\2\2\u0171\u0172\3\2\2\2\u0172\u0173\3\2\2\2\u0173\u0176\7\4\2\2\u0174"
-        + "\u0176\5\66\34\2\u0175\u0168\3\2\2\2\u0175\u0174\3\2\2\2\u0176\33\3\2"
-        + "\2\2\u0177\u0178\5j\66\2\u0178\u0179\7\f\2\2\u0179\u017a\7\3\2\2\u017a"
-        + "\u017b\5\n\6\2\u017b\u017c\7\4\2\2\u017c\35\3\2\2\2\u017d\u017e\7_\2\2"
-        + "\u017e\u017f\7\u0082\2\2\u017f\37\3\2\2\2\u0180\u0181\t\13\2\2\u0181!"
-        + "\3\2\2\2\u0182\u0187\5$\23\2\u0183\u0184\7\5\2\2\u0184\u0186\5$\23\2\u0185"
-        + "\u0183\3\2\2\2\u0186\u0189\3\2\2\2\u0187\u0185\3\2\2\2\u0187\u0188\3\2"
-        + "\2\2\u0188#\3\2\2\2\u0189\u0187\3\2\2\2\u018a\u018f\5\66\34\2\u018b\u018d"
-        + "\7\f\2\2\u018c\u018b\3\2\2\2\u018c\u018d\3\2\2\2\u018d\u018e\3\2\2\2\u018e"
-        + "\u0190\5j\66\2\u018f\u018c\3\2\2\2\u018f\u0190\3\2\2\2\u0190%\3\2\2\2"
-        + "\u0191\u0195\5.\30\2\u0192\u0194\5(\25\2\u0193\u0192\3\2\2\2\u0194\u0197"
-        + "\3\2\2\2\u0195\u0193\3\2\2\2\u0195\u0196\3\2\2\2\u0196\'\3\2\2\2\u0197"
-        + "\u0195\3\2\2\2\u0198\u0199\5*\26\2\u0199\u019a\78\2\2\u019a\u019c\5.\30"
-        + "\2\u019b\u019d\5,\27\2\u019c\u019b\3\2\2\2\u019c\u019d\3\2\2\2\u019d\u01a4"
-        + "\3\2\2\2\u019e\u019f\7C\2\2\u019f\u01a0\5*\26\2\u01a0\u01a1\78\2\2\u01a1"
-        + "\u01a2\5.\30\2\u01a2\u01a4\3\2\2\2\u01a3\u0198\3\2\2\2\u01a3\u019e\3\2"
-        + "\2\2\u01a4)\3\2\2\2\u01a5\u01a7\7\65\2\2\u01a6\u01a5\3\2\2\2\u01a6\u01a7"
-        + "\3\2\2\2\u01a7\u01b5\3\2\2\2\u01a8\u01aa\7:\2\2\u01a9\u01ab\7K\2\2\u01aa"
-        + "\u01a9\3\2\2\2\u01aa\u01ab\3\2\2\2\u01ab\u01b5\3\2\2\2\u01ac\u01ae\7P"
-        + "\2\2\u01ad\u01af\7K\2\2\u01ae\u01ad\3\2\2\2\u01ae\u01af\3\2\2\2\u01af"
-        + "\u01b5\3\2\2\2\u01b0\u01b2\7,\2\2\u01b1\u01b3\7K\2\2\u01b2\u01b1\3\2\2"
-        + "\2\u01b2\u01b3\3\2\2\2\u01b3\u01b5\3\2\2\2\u01b4\u01a6\3\2\2\2\u01b4\u01a8"
-        + "\3\2\2\2\u01b4\u01ac\3\2\2\2\u01b4\u01b0\3\2\2\2\u01b5+\3\2\2\2\u01b6"
-        + "\u01b7\7G\2\2\u01b7\u01c5\58\35\2\u01b8\u01b9\7b\2\2\u01b9\u01ba\7\3\2"
-        + "\2\u01ba\u01bf\5j\66\2\u01bb\u01bc\7\5\2\2\u01bc\u01be\5j\66\2\u01bd\u01bb"
-        + "\3\2\2\2\u01be\u01c1\3\2\2\2\u01bf\u01bd\3\2\2\2\u01bf\u01c0\3\2\2\2\u01c0"
-        + "\u01c2\3\2\2\2\u01c1\u01bf\3\2\2\2\u01c2\u01c3\7\4\2\2\u01c3\u01c5\3\2"
-        + "\2\2\u01c4\u01b6\3\2\2\2\u01c4\u01b8\3\2\2\2\u01c5-\3\2\2\2\u01c6\u01c8"
-        + "\7+\2\2\u01c7\u01c6\3\2\2\2\u01c7\u01c8\3\2\2\2\u01c8\u01c9\3\2\2\2\u01c9"
-        + "\u01ce\5l\67\2\u01ca\u01cc\7\f\2\2\u01cb\u01ca\3\2\2\2\u01cb\u01cc\3\2"
-        + "\2\2\u01cc\u01cd\3\2\2\2\u01cd\u01cf\5h\65\2\u01ce\u01cb\3\2\2\2\u01ce"
-        + "\u01cf\3\2\2\2\u01cf\u01e3\3\2\2\2\u01d0\u01d1\7\3\2\2\u01d1\u01d2\5\n"
-        + "\6\2\u01d2\u01d7\7\4\2\2\u01d3\u01d5\7\f\2\2\u01d4\u01d3\3\2\2\2\u01d4"
-        + "\u01d5\3\2\2\2\u01d5\u01d6\3\2\2\2\u01d6\u01d8\5h\65\2\u01d7\u01d4\3\2"
-        + "\2\2\u01d7\u01d8\3\2\2\2\u01d8\u01e3\3\2\2\2\u01d9\u01da\7\3\2\2\u01da"
-        + "\u01db\5&\24\2\u01db\u01e0\7\4\2\2\u01dc\u01de\7\f\2\2\u01dd\u01dc\3\2"
-        + "\2\2\u01dd\u01de\3\2\2\2\u01de\u01df\3\2\2\2\u01df\u01e1\5h\65\2\u01e0"
-        + "\u01dd\3\2\2\2\u01e0\u01e1\3\2\2\2\u01e1\u01e3\3\2\2\2\u01e2\u01c7\3\2"
-        + "\2\2\u01e2\u01d0\3\2\2\2\u01e2\u01d9\3\2\2\2\u01e3/\3\2\2\2\u01e4\u01e5"
-        + "\7N\2\2\u01e5\u01e6\7\3\2\2\u01e6\u01e7\5\62\32\2\u01e7\u01e8\7(\2\2\u01e8"
-        + "\u01e9\5h\65\2\u01e9\u01ea\7\63\2\2\u01ea\u01eb\7\3\2\2\u01eb\u01ec\5"
-        + "\62\32\2\u01ec\u01ed\7\4\2\2\u01ed\u01ee\7\4\2\2\u01ee\61\3\2\2\2\u01ef"
-        + "\u01f4\5\64\33\2\u01f0\u01f1\7\5\2\2\u01f1\u01f3\5\64\33\2\u01f2\u01f0"
-        + "\3\2\2\2\u01f3\u01f6\3\2\2\2\u01f4\u01f2\3\2\2\2\u01f4\u01f5\3\2\2\2\u01f5"
-        + "\63\3\2\2\2\u01f6\u01f4\3\2\2\2\u01f7\u01fc\5F$\2\u01f8\u01fa\7\f\2\2"
-        + "\u01f9\u01f8\3\2\2\2\u01f9\u01fa\3\2\2\2\u01fa\u01fb\3\2\2\2\u01fb\u01fd"
-        + "\5j\66\2\u01fc\u01f9\3\2\2\2\u01fc\u01fd\3\2\2\2\u01fd\65\3\2\2\2\u01fe"
-        + "\u01ff\58\35\2\u01ff\67\3\2\2\2\u0200\u0201\b\35\1\2\u0201\u0202\7D\2"
-        + "\2\u0202\u0220\58\35\n\u0203\u0204\7#\2\2\u0204\u0205\7\3\2\2\u0205\u0206"
-        + "\5\b\5\2\u0206\u0207\7\4\2\2\u0207\u0220\3\2\2\2\u0208\u0209\7R\2\2\u0209"
-        + "\u020a\7\3\2\2\u020a\u020b\5t;\2\u020b\u020c\5:\36\2\u020c\u020d\7\4\2"
-        + "\2\u020d\u0220\3\2\2\2\u020e\u020f\7>\2\2\u020f\u0210\7\3\2\2\u0210\u0211"
-        + "\5h\65\2\u0211\u0212\7\5\2\2\u0212\u0213\5t;\2\u0213\u0214\5:\36\2\u0214"
-        + "\u0215\7\4\2\2\u0215\u0220\3\2\2\2\u0216\u0217\7>\2\2\u0217\u0218\7\3"
-        + "\2\2\u0218\u0219\5t;\2\u0219\u021a\7\5\2\2\u021a\u021b\5t;\2\u021b\u021c"
-        + "\5:\36\2\u021c\u021d\7\4\2\2\u021d\u0220\3\2\2\2\u021e\u0220\5<\37\2\u021f"
-        + "\u0200\3\2\2\2\u021f\u0203\3\2\2\2\u021f\u0208\3\2\2\2\u021f\u020e\3\2"
-        + "\2\2\u021f\u0216\3\2\2\2\u021f\u021e\3\2\2\2\u0220\u0229\3\2\2\2\u0221"
-        + "\u0222\f\4\2\2\u0222\u0223\7\n\2\2\u0223\u0228\58\35\5\u0224\u0225\f\3"
-        + "\2\2\u0225\u0226\7I\2\2\u0226\u0228\58\35\4\u0227\u0221\3\2\2\2\u0227"
-        + "\u0224\3\2\2\2\u0228\u022b\3\2\2\2\u0229\u0227\3\2\2\2\u0229\u022a\3\2"
-        + "\2\2\u022a9\3\2\2\2\u022b\u0229\3\2\2\2\u022c\u022d\7\5\2\2\u022d\u022f"
-        + "\5t;\2\u022e\u022c\3\2\2\2\u022f\u0232\3\2\2\2\u0230\u022e\3\2\2\2\u0230"
-        + "\u0231\3\2\2\2\u0231;\3\2\2\2\u0232\u0230\3\2\2\2\u0233\u0235\5F$\2\u0234"
-        + "\u0236\5> \2\u0235\u0234\3\2\2\2\u0235\u0236\3\2\2\2\u0236=\3\2\2\2\u0237"
-        + "\u0239\7D\2\2\u0238\u0237\3\2\2\2\u0238\u0239\3\2\2\2\u0239\u023a\3\2"
-        + "\2\2\u023a\u023b\7\16\2\2\u023b\u023c\5F$\2\u023c\u023d\7\n\2\2\u023d"
-        + "\u023e\5F$\2\u023e\u0266\3\2\2\2\u023f\u0241\7D\2\2\u0240\u023f\3\2\2"
-        + "\2\u0240\u0241\3\2\2\2\u0241\u0242\3\2\2\2\u0242\u0243\7\63\2\2\u0243"
-        + "\u0244\7\3\2\2\u0244\u0249\5F$\2\u0245\u0246\7\5\2\2\u0246\u0248\5F$\2"
-        + "\u0247\u0245\3\2\2\2\u0248\u024b\3\2\2\2\u0249\u0247\3\2\2\2\u0249\u024a"
-        + "\3\2\2\2\u024a\u024c\3\2\2\2\u024b\u0249\3\2\2\2\u024c\u024d\7\4\2\2\u024d"
-        + "\u0266\3\2\2\2\u024e\u0250\7D\2\2\u024f\u024e\3\2\2\2\u024f\u0250\3\2"
-        + "\2\2\u0250\u0251\3\2\2\2\u0251\u0252\7\63\2\2\u0252\u0253\7\3\2\2\u0253"
-        + "\u0254\5\b\5\2\u0254\u0255\7\4\2\2\u0255\u0266\3\2\2\2\u0256\u0258\7D"
-        + "\2\2\u0257\u0256\3\2\2\2\u0257\u0258\3\2\2\2\u0258\u0259\3\2\2\2\u0259"
-        + "\u025a\7;\2\2\u025a\u0266\5B\"\2\u025b\u025d\7D\2\2\u025c\u025b\3\2\2"
-        + "\2\u025c\u025d\3\2\2\2\u025d\u025e\3\2\2\2\u025e\u025f\7Q\2\2\u025f\u0266"
-        + "\5t;\2\u0260\u0262\7\67\2\2\u0261\u0263\7D\2\2\u0262\u0261\3\2\2\2\u0262"
-        + "\u0263\3\2\2\2\u0263\u0264\3\2\2\2\u0264\u0266\7E\2\2\u0265\u0238\3\2"
-        + "\2\2\u0265\u0240\3\2\2\2\u0265\u024f\3\2\2\2\u0265\u0257\3\2\2\2\u0265"
-        + "\u025c\3\2\2\2\u0265\u0260\3\2\2\2\u0266?\3\2\2\2\u0267\u0268\7;\2\2\u0268"
-        + "\u0269\5B\"\2\u0269A\3\2\2\2\u026a\u026c\5t;\2\u026b\u026d\5D#\2\u026c"
-        + "\u026b\3\2\2\2\u026c\u026d\3\2\2\2\u026dC\3\2\2\2\u026e\u026f\7!\2\2\u026f"
-        + "\u0275\5t;\2\u0270\u0271\7i\2\2\u0271\u0272\5t;\2\u0272\u0273\7q\2\2\u0273"
-        + "\u0275\3\2\2\2\u0274\u026e\3\2\2\2\u0274\u0270\3\2\2\2\u0275E\3\2\2\2"
-        + "\u0276\u0277\b$\1\2\u0277\u027b\5H%\2\u0278\u0279\t\7\2\2\u0279\u027b"
-        + "\5F$\6\u027a\u0276\3\2\2\2\u027a\u0278\3\2\2\2\u027b\u0288\3\2\2\2\u027c"
-        + "\u027d\f\5\2\2\u027d\u027e\t\f\2\2\u027e\u0287\5F$\6\u027f\u0280\f\4\2"
-        + "\2\u0280\u0281\t\7\2\2\u0281\u0287\5F$\5\u0282\u0283\f\3\2\2\u0283\u0284"
-        + "\5^\60\2\u0284\u0285\5F$\4\u0285\u0287\3\2\2\2\u0286\u027c\3\2\2\2\u0286"
-        + "\u027f\3\2\2\2\u0286\u0282\3\2\2\2\u0287\u028a\3\2\2\2\u0288\u0286\3\2"
-        + "\2\2\u0288\u0289\3\2\2\2\u0289G\3\2\2\2\u028a\u0288\3\2\2\2\u028b\u028c"
-        + "\b%\1\2\u028c\u02b0\5L\'\2\u028d\u02b0\5R*\2\u028e\u02b0\5J&\2\u028f\u02b0"
-        + "\5\\/\2\u0290\u0291\5h\65\2\u0291\u0292\7\177\2\2\u0292\u0294\3\2\2\2"
-        + "\u0293\u0290\3\2\2\2\u0293\u0294\3\2\2\2\u0294\u0295\3\2\2\2\u0295\u02b0"
-        + "\7{\2\2\u0296\u02b0\5V,\2\u0297\u0298\7\3\2\2\u0298\u0299\5\b\5\2\u0299"
-        + "\u029a\7\4\2\2\u029a\u02b0\3\2\2\2\u029b\u02b0\5h\65\2\u029c\u029d\7\3"
-        + "\2\2\u029d\u029e\5\66\34\2\u029e\u029f\7\4\2\2\u029f\u02b0\3\2\2\2\u02a0"
-        + "\u02a2\7\20\2\2\u02a1\u02a3\58\35\2\u02a2\u02a1\3\2\2\2\u02a2\u02a3\3"
-        + "\2\2\2\u02a3\u02a5\3\2\2\2\u02a4\u02a6\5v<\2\u02a5\u02a4\3\2\2\2\u02a6"
-        + "\u02a7\3\2\2\2\u02a7\u02a5\3\2\2\2\u02a7\u02a8\3\2\2\2\u02a8\u02ab\3\2"
-        + "\2\2\u02a9\u02aa\7\37\2\2\u02aa\u02ac\58\35\2\u02ab\u02a9\3\2\2\2\u02ab"
-        + "\u02ac\3\2\2\2\u02ac\u02ad\3\2\2\2\u02ad\u02ae\7 \2\2\u02ae\u02b0\3\2"
-        + "\2\2\u02af\u028b\3\2\2\2\u02af\u028d\3\2\2\2\u02af\u028e\3\2\2\2\u02af"
-        + "\u028f\3\2\2\2\u02af\u0293\3\2\2\2\u02af\u0296\3\2\2\2\u02af\u0297\3\2"
-        + "\2\2\u02af\u029b\3\2\2\2\u02af\u029c\3\2\2\2\u02af\u02a0\3\2\2\2\u02b0"
-        + "\u02b6\3\2\2\2\u02b1\u02b2\f\f\2\2\u02b2\u02b3\7~\2\2\u02b3\u02b5\5f\64"
-        + "\2\u02b4\u02b1\3\2\2\2\u02b5\u02b8\3\2\2\2\u02b6\u02b4\3\2\2\2\u02b6\u02b7"
-        + "\3\2\2\2\u02b7I\3\2\2\2\u02b8\u02b6\3\2\2\2\u02b9\u02bd\7\30\2\2\u02ba"
-        + "\u02bd\7\26\2\2\u02bb\u02bd\7\27\2\2\u02bc\u02b9\3\2\2\2\u02bc\u02ba\3"
-        + "\2\2\2\u02bc\u02bb\3\2\2\2\u02bdK\3\2\2\2\u02be\u02c9\5N(\2\u02bf\u02c0"
-        + "\7j\2\2\u02c0\u02c1\5N(\2\u02c1\u02c2\7q\2\2\u02c2\u02c9\3\2\2\2\u02c3"
-        + "\u02c9\5P)\2\u02c4\u02c5\7j\2\2\u02c5\u02c6\5P)\2\u02c6\u02c7\7q\2\2\u02c7"
-        + "\u02c9\3\2\2\2\u02c8\u02be\3\2\2\2\u02c8\u02bf\3\2\2\2\u02c8\u02c3\3\2"
-        + "\2\2\u02c8\u02c4\3\2\2\2\u02c9M\3\2\2\2\u02ca\u02cb\7\21\2\2\u02cb\u02cc"
-        + "\7\3\2\2\u02cc\u02cd\5\66\34\2\u02cd\u02ce\7\f\2\2\u02ce\u02cf\5f\64\2"
-        + "\u02cf\u02d0\7\4\2\2\u02d0O\3\2\2\2\u02d1\u02d2\7\25\2\2\u02d2\u02d3\7"
-        + "\3\2\2\u02d3\u02d4\5\66\34\2\u02d4\u02d5\7\5\2\2\u02d5\u02d6\5f\64\2\u02d6"
-        + "\u02d7\7\4\2\2\u02d7Q\3\2\2\2\u02d8\u02de\5T+\2\u02d9\u02da\7j\2\2\u02da"
-        + "\u02db\5T+\2\u02db\u02dc\7q\2\2\u02dc\u02de\3\2\2\2\u02dd\u02d8\3\2\2"
-        + "\2\u02dd\u02d9\3\2\2\2\u02deS\3\2\2\2\u02df\u02e0\7%\2\2\u02e0\u02e1\7"
-        + "\3\2\2\u02e1\u02e2\5j\66\2\u02e2\u02e3\7*\2\2\u02e3\u02e4\5F$\2\u02e4"
-        + "\u02e5\7\4\2\2\u02e5U\3\2\2\2\u02e6\u02ec\5X-\2\u02e7\u02e8\7j\2\2\u02e8"
-        + "\u02e9\5X-\2\u02e9\u02ea\7q\2\2\u02ea\u02ec\3\2\2\2\u02eb\u02e6\3\2\2"
-        + "\2\u02eb\u02e7\3\2\2\2\u02ecW\3\2\2\2\u02ed\u02ee\5Z.\2\u02ee\u02fa\7"
-        + "\3\2\2\u02ef\u02f1\5 \21\2\u02f0\u02ef\3\2\2\2\u02f0\u02f1\3\2\2\2\u02f1"
-        + "\u02f2\3\2\2\2\u02f2\u02f7\5\66\34\2\u02f3\u02f4\7\5\2\2\u02f4\u02f6\5"
-        + "\66\34\2\u02f5\u02f3\3\2\2\2\u02f6\u02f9\3\2\2\2\u02f7\u02f5\3\2\2\2\u02f7"
-        + "\u02f8\3\2\2\2\u02f8\u02fb\3\2\2\2\u02f9\u02f7\3\2\2\2\u02fa\u02f0\3\2"
-        + "\2\2\u02fa\u02fb\3\2\2\2\u02fb\u02fc\3\2\2\2\u02fc\u02fd\7\4\2\2\u02fd"
-        + "Y\3\2\2\2\u02fe\u0302\7:\2\2\u02ff\u0302\7P\2\2\u0300\u0302\5j\66\2\u0301"
-        + "\u02fe\3\2\2\2\u0301\u02ff\3\2\2\2\u0301\u0300\3\2\2\2\u0302[\3\2\2\2"
-        + "\u0303\u031e\7E\2\2\u0304\u031e\5b\62\2\u0305\u031e\5r:\2\u0306\u031e"
-        + "\5`\61\2\u0307\u0309\7\u0081\2\2\u0308\u0307\3\2\2\2\u0309\u030a\3\2\2"
-        + "\2\u030a\u0308\3\2\2\2\u030a\u030b\3\2\2\2\u030b\u031e\3\2\2\2\u030c\u031e"
-        + "\7\u0080\2\2\u030d\u030e\7l\2\2\u030e\u030f\5t;\2\u030f\u0310\7q\2\2\u0310"
-        + "\u031e\3\2\2\2\u0311\u0312\7m\2\2\u0312\u0313\5t;\2\u0313\u0314\7q\2\2"
-        + "\u0314\u031e\3\2\2\2\u0315\u0316\7n\2\2\u0316\u0317\5t;\2\u0317\u0318"
-        + "\7q\2\2\u0318\u031e\3\2\2\2\u0319\u031a\7o\2\2\u031a\u031b\5t;\2\u031b"
-        + "\u031c\7q\2\2\u031c\u031e\3\2\2\2\u031d\u0303\3\2\2\2\u031d\u0304\3\2"
-        + "\2\2\u031d\u0305\3\2\2\2\u031d\u0306\3\2\2\2\u031d\u0308\3\2\2\2\u031d"
-        + "\u030c\3\2\2\2\u031d\u030d\3\2\2\2\u031d\u0311\3\2\2\2\u031d\u0315\3\2"
-        + "\2\2\u031d\u0319\3\2\2\2\u031e]\3\2\2\2\u031f\u0320\t\r\2\2\u0320_\3\2"
-        + "\2\2\u0321\u0322\t\16\2\2\u0322a\3\2\2\2\u0323\u0325\7\66\2\2\u0324\u0326"
-        + "\t\7\2\2\u0325\u0324\3\2\2\2\u0325\u0326\3\2\2\2\u0326\u0329\3\2\2\2\u0327"
-        + "\u032a\5r:\2\u0328\u032a\5t;\2\u0329\u0327\3\2\2\2\u0329\u0328\3\2\2\2"
-        + "\u032a\u032b\3\2\2\2\u032b\u032e\5d\63\2\u032c\u032d\7^\2\2\u032d\u032f"
-        + "\5d\63\2\u032e\u032c\3\2\2\2\u032e\u032f\3\2\2\2\u032fc\3\2\2\2\u0330"
-        + "\u0331\t\17\2\2\u0331e\3\2\2\2\u0332\u0333\5j\66\2\u0333g\3\2\2\2\u0334"
-        + "\u0335\5j\66\2\u0335\u0336\7\177\2\2\u0336\u0338\3\2\2\2\u0337\u0334\3"
-        + "\2\2\2\u0338\u033b\3\2\2\2\u0339\u0337\3\2\2\2\u0339\u033a\3\2\2\2\u033a"
-        + "\u033c\3\2\2\2\u033b\u0339\3\2\2\2\u033c\u033d\5j\66\2\u033di\3\2\2\2"
-        + "\u033e\u0341\5n8\2\u033f\u0341\5p9\2\u0340\u033e\3\2\2\2\u0340\u033f\3"
-        + "\2\2\2\u0341k\3\2\2\2\u0342\u0343\5j\66\2\u0343\u0344\7\6\2\2\u0344\u0346"
-        + "\3\2\2\2\u0345\u0342\3\2\2\2\u0345\u0346\3\2\2\2\u0346\u0347\3\2\2\2\u0347"
-        + "\u034f\7\u0086\2\2\u0348\u0349\5j\66\2\u0349\u034a\7\6\2\2\u034a\u034c"
-        + "\3\2\2\2\u034b\u0348\3\2\2\2\u034b\u034c\3\2\2\2\u034c\u034d\3\2\2\2\u034d"
-        + "\u034f\5j\66\2\u034e\u0345\3\2\2\2\u034e\u034b\3\2\2\2\u034fm\3\2\2\2"
-        + "\u0350\u0353\7\u0087\2\2\u0351\u0353\7\u0088\2\2\u0352\u0350\3\2\2\2\u0352"
-        + "\u0351\3\2\2\2\u0353o\3\2\2\2\u0354\u0358\7\u0084\2\2\u0355\u0358\5x="
-        + "\2\u0356\u0358\7\u0085\2\2\u0357\u0354\3\2\2\2\u0357\u0355\3\2\2\2\u0357"
-        + "\u0356\3\2\2\2\u0358q\3\2\2\2\u0359\u035c\7\u0083\2\2\u035a\u035c\7\u0082"
-        + "\2\2\u035b\u0359\3\2\2\2\u035b\u035a\3\2\2\2\u035cs\3\2\2\2\u035d\u035e"
-        + "\t\20\2\2\u035eu\3\2\2\2\u035f\u0360\7d\2\2\u0360\u0361\5\66\34\2\u0361"
-        + "\u0362\7\\\2\2\u0362\u0363\5\66\34\2\u0363w\3\2\2\2\u0364\u0365\t\21\2"
-        + "\2\u0365y\3\2\2\2x\u0089\u008b\u008f\u0098\u009a\u009e\u00a6\u00a8\u00ac"
-        + "\u00b0\u00b6\u00ba\u00bf\u00c4\u00c8\u00cc\u00d1\u00db\u00df\u00e7\u00ea"
-        + "\u00f0\u00f5\u00f8\u00fd\u0100\u0102\u010a\u010d\u0119\u011c\u011f\u0126"
-        + "\u012d\u0131\u0135\u0139\u013c\u0140\u0144\u0149\u014d\u0155\u0159\u015c"
-        + "\u0163\u016e\u0171\u0175\u0187\u018c\u018f\u0195\u019c\u01a3\u01a6\u01aa"
-        + "\u01ae\u01b2\u01b4\u01bf\u01c4\u01c7\u01cb\u01ce\u01d4\u01d7\u01dd\u01e0"
-        + "\u01e2\u01f4\u01f9\u01fc\u021f\u0227\u0229\u0230\u0235\u0238\u0240\u0249"
-        + "\u024f\u0257\u025c\u0262\u0265\u026c\u0274\u027a\u0286\u0288\u0293\u02a2"
-        + "\u02a7\u02ab\u02af\u02b6\u02bc\u02c8\u02dd\u02eb\u02f0\u02f7\u02fa\u0301"
-        + "\u030a\u031d\u0325\u0329\u032e\u0339\u0340\u0345\u034b\u034e\u0352\u0357"
-        + "\u035b";
+    public static final String _serializedATN = "\u0004\u0001\u008b\u0365\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001"
+        + "\u0002\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004"
+        + "\u0002\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007"
+        + "\u0002\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b"
+        + "\u0002\f\u0007\f\u0002\r\u0007\r\u0002\u000e\u0007\u000e\u0002\u000f\u0007"
+        + "\u000f\u0002\u0010\u0007\u0010\u0002\u0011\u0007\u0011\u0002\u0012\u0007"
+        + "\u0012\u0002\u0013\u0007\u0013\u0002\u0014\u0007\u0014\u0002\u0015\u0007"
+        + "\u0015\u0002\u0016\u0007\u0016\u0002\u0017\u0007\u0017\u0002\u0018\u0007"
+        + "\u0018\u0002\u0019\u0007\u0019\u0002\u001a\u0007\u001a\u0002\u001b\u0007"
+        + "\u001b\u0002\u001c\u0007\u001c\u0002\u001d\u0007\u001d\u0002\u001e\u0007"
+        + "\u001e\u0002\u001f\u0007\u001f\u0002 \u0007 \u0002!\u0007!\u0002\"\u0007"
+        + "\"\u0002#\u0007#\u0002$\u0007$\u0002%\u0007%\u0002&\u0007&\u0002\'\u0007"
+        + "\'\u0002(\u0007(\u0002)\u0007)\u0002*\u0007*\u0002+\u0007+\u0002,\u0007"
+        + ",\u0002-\u0007-\u0002.\u0007.\u0002/\u0007/\u00020\u00070\u00021\u0007"
+        + "1\u00022\u00072\u00023\u00073\u00024\u00074\u00025\u00075\u00026\u0007"
+        + "6\u00027\u00077\u00028\u00078\u00029\u00079\u0002:\u0007:\u0002;\u0007"
+        + ";\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001"
+        + "\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002"
+        + "\u0001\u0002\u0001\u0002\u0001\u0002\u0005\u0002\u0088\b\u0002\n\u0002"
+        + "\f\u0002\u008b\t\u0002\u0001\u0002\u0003\u0002\u008e\b\u0002\u0001\u0002"
+        + "\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002"
+        + "\u0005\u0002\u0097\b\u0002\n\u0002\f\u0002\u009a\t\u0002\u0001\u0002\u0003"
+        + "\u0002\u009d\b\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001"
+        + "\u0002\u0001\u0002\u0003\u0002\u00a5\b\u0002\u0003\u0002\u00a7\b\u0002"
+        + "\u0001\u0002\u0001\u0002\u0003\u0002\u00ab\b\u0002\u0001\u0002\u0001\u0002"
+        + "\u0003\u0002\u00af\b\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002"
+        + "\u0003\u0002\u00b5\b\u0002\u0001\u0002\u0001\u0002\u0003\u0002\u00b9\b"
+        + "\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0003\u0002\u00be\b\u0002\u0001"
+        + "\u0002\u0001\u0002\u0001\u0002\u0003\u0002\u00c3\b\u0002\u0001\u0002\u0001"
+        + "\u0002\u0003\u0002\u00c7\b\u0002\u0001\u0002\u0001\u0002\u0003\u0002\u00cb"
+        + "\b\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0003\u0002\u00d0\b\u0002"
+        + "\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002"
+        + "\u0001\u0002\u0001\u0002\u0003\u0002\u00da\b\u0002\u0001\u0002\u0001\u0002"
+        + "\u0003\u0002\u00de\b\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002"
+        + "\u0005\u0002\u00e4\b\u0002\n\u0002\f\u0002\u00e7\t\u0002\u0003\u0002\u00e9"
+        + "\b\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0003\u0002\u00ef"
+        + "\b\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0003\u0002\u00f4\b\u0002"
+        + "\u0001\u0002\u0003\u0002\u00f7\b\u0002\u0001\u0002\u0001\u0002\u0001\u0002"
+        + "\u0003\u0002\u00fc\b\u0002\u0001\u0002\u0003\u0002\u00ff\b\u0002\u0003"
+        + "\u0002\u0101\b\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0005"
+        + "\u0003\u0107\b\u0003\n\u0003\f\u0003\u010a\t\u0003\u0003\u0003\u010c\b"
+        + "\u0003\u0001\u0003\u0001\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001"
+        + "\u0004\u0001\u0004\u0001\u0004\u0005\u0004\u0116\b\u0004\n\u0004\f\u0004"
+        + "\u0119\t\u0004\u0003\u0004\u011b\b\u0004\u0001\u0004\u0003\u0004\u011e"
+        + "\b\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003"
+        + "\u0005\u0125\b\u0005\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001"
+        + "\u0006\u0003\u0006\u012c\b\u0006\u0001\u0007\u0001\u0007\u0003\u0007\u0130"
+        + "\b\u0007\u0001\u0007\u0001\u0007\u0003\u0007\u0134\b\u0007\u0001\b\u0001"
+        + "\b\u0003\b\u0138\b\b\u0001\b\u0003\b\u013b\b\b\u0001\b\u0001\b\u0003\b"
+        + "\u013f\b\b\u0001\b\u0001\b\u0003\b\u0143\b\b\u0001\b\u0001\b\u0001\b\u0003"
+        + "\b\u0148\b\b\u0001\b\u0001\b\u0003\b\u014c\b\b\u0001\t\u0001\t\u0001\t"
+        + "\u0001\t\u0005\t\u0152\b\t\n\t\f\t\u0155\t\t\u0001\t\u0003\t\u0158\b\t"
+        + "\u0001\n\u0003\n\u015b\b\n\u0001\n\u0001\n\u0001\n\u0005\n\u0160\b\n\n"
+        + "\n\f\n\u0163\t\n\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\f\u0001"
+        + "\f\u0005\f\u016b\b\f\n\f\f\f\u016e\t\f\u0003\f\u0170\b\f\u0001\f\u0001"
+        + "\f\u0003\f\u0174\b\f\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001"
+        + "\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f\u0001\u0010\u0001"
+        + "\u0010\u0001\u0010\u0005\u0010\u0184\b\u0010\n\u0010\f\u0010\u0187\t\u0010"
+        + "\u0001\u0011\u0001\u0011\u0003\u0011\u018b\b\u0011\u0001\u0011\u0003\u0011"
+        + "\u018e\b\u0011\u0001\u0012\u0001\u0012\u0005\u0012\u0192\b\u0012\n\u0012"
+        + "\f\u0012\u0195\t\u0012\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013"
+        + "\u0003\u0013\u019b\b\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013"
+        + "\u0001\u0013\u0003\u0013\u01a2\b\u0013\u0001\u0014\u0003\u0014\u01a5\b"
+        + "\u0014\u0001\u0014\u0001\u0014\u0003\u0014\u01a9\b\u0014\u0001\u0014\u0001"
+        + "\u0014\u0003\u0014\u01ad\b\u0014\u0001\u0014\u0001\u0014\u0003\u0014\u01b1"
+        + "\b\u0014\u0003\u0014\u01b3\b\u0014\u0001\u0015\u0001\u0015\u0001\u0015"
+        + "\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0005\u0015\u01bc\b\u0015"
+        + "\n\u0015\f\u0015\u01bf\t\u0015\u0001\u0015\u0001\u0015\u0003\u0015\u01c3"
+        + "\b\u0015\u0001\u0016\u0003\u0016\u01c6\b\u0016\u0001\u0016\u0001\u0016"
+        + "\u0003\u0016\u01ca\b\u0016\u0001\u0016\u0003\u0016\u01cd\b\u0016\u0001"
+        + "\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0003\u0016\u01d3\b\u0016\u0001"
+        + "\u0016\u0003\u0016\u01d6\b\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001"
+        + "\u0016\u0003\u0016\u01dc\b\u0016\u0001\u0016\u0003\u0016\u01df\b\u0016"
+        + "\u0003\u0016\u01e1\b\u0016\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017"
+        + "\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017"
+        + "\u0001\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0005\u0018\u01f1\b\u0018"
+        + "\n\u0018\f\u0018\u01f4\t\u0018\u0001\u0019\u0001\u0019\u0003\u0019\u01f8"
+        + "\b\u0019\u0001\u0019\u0003\u0019\u01fb\b\u0019\u0001\u001a\u0001\u001a"
+        + "\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b"
+        + "\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b"
+        + "\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b"
+        + "\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b"
+        + "\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b\u0001\u001b"
+        + "\u0001\u001b\u0003\u001b\u021e\b\u001b\u0001\u001b\u0001\u001b\u0001\u001b"
+        + "\u0001\u001b\u0001\u001b\u0001\u001b\u0005\u001b\u0226\b\u001b\n\u001b"
+        + "\f\u001b\u0229\t\u001b\u0001\u001c\u0001\u001c\u0005\u001c\u022d\b\u001c"
+        + "\n\u001c\f\u001c\u0230\t\u001c\u0001\u001d\u0001\u001d\u0003\u001d\u0234"
+        + "\b\u001d\u0001\u001e\u0003\u001e\u0237\b\u001e\u0001\u001e\u0001\u001e"
+        + "\u0001\u001e\u0001\u001e\u0001\u001e\u0001\u001e\u0003\u001e\u023f\b\u001e"
+        + "\u0001\u001e\u0001\u001e\u0001\u001e\u0001\u001e\u0001\u001e\u0005\u001e"
+        + "\u0246\b\u001e\n\u001e\f\u001e\u0249\t\u001e\u0001\u001e\u0001\u001e\u0001"
+        + "\u001e\u0003\u001e\u024e\b\u001e\u0001\u001e\u0001\u001e\u0001\u001e\u0001"
+        + "\u001e\u0001\u001e\u0001\u001e\u0003\u001e\u0256\b\u001e\u0001\u001e\u0001"
+        + "\u001e\u0001\u001e\u0003\u001e\u025b\b\u001e\u0001\u001e\u0001\u001e\u0001"
+        + "\u001e\u0001\u001e\u0003\u001e\u0261\b\u001e\u0001\u001e\u0003\u001e\u0264"
+        + "\b\u001e\u0001\u001f\u0001\u001f\u0001\u001f\u0001 \u0001 \u0003 \u026b"
+        + "\b \u0001!\u0001!\u0001!\u0001!\u0001!\u0001!\u0003!\u0273\b!\u0001\""
+        + "\u0001\"\u0001\"\u0001\"\u0003\"\u0279\b\"\u0001\"\u0001\"\u0001\"\u0001"
+        + "\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u0285\b\"\n"
+        + "\"\f\"\u0288\t\"\u0001#\u0001#\u0001#\u0001#\u0001#\u0001#\u0001#\u0001"
+        + "#\u0003#\u0292\b#\u0001#\u0001#\u0001#\u0001#\u0001#\u0001#\u0001#\u0001"
+        + "#\u0001#\u0001#\u0001#\u0001#\u0001#\u0003#\u02a1\b#\u0001#\u0004#\u02a4"
+        + "\b#\u000b#\f#\u02a5\u0001#\u0001#\u0003#\u02aa\b#\u0001#\u0001#\u0003"
+        + "#\u02ae\b#\u0001#\u0001#\u0001#\u0005#\u02b3\b#\n#\f#\u02b6\t#\u0001$"
+        + "\u0001$\u0001$\u0003$\u02bb\b$\u0001%\u0001%\u0001%\u0001%\u0001%\u0001"
+        + "%\u0001%\u0001%\u0001%\u0001%\u0003%\u02c7\b%\u0001&\u0001&\u0001&\u0001"
+        + "&\u0001&\u0001&\u0001&\u0001\'\u0001\'\u0001\'\u0001\'\u0001\'\u0001\'"
+        + "\u0001\'\u0001(\u0001(\u0001(\u0001(\u0001(\u0003(\u02dc\b(\u0001)\u0001"
+        + ")\u0001)\u0001)\u0001)\u0001)\u0001)\u0001*\u0001*\u0001*\u0001*\u0001"
+        + "*\u0003*\u02ea\b*\u0001+\u0001+\u0001+\u0003+\u02ef\b+\u0001+\u0001+\u0001"
+        + "+\u0005+\u02f4\b+\n+\f+\u02f7\t+\u0003+\u02f9\b+\u0001+\u0001+\u0001,"
+        + "\u0001,\u0001,\u0003,\u0300\b,\u0001-\u0001-\u0001-\u0001-\u0001-\u0004"
+        + "-\u0307\b-\u000b-\f-\u0308\u0001-\u0001-\u0001-\u0001-\u0001-\u0001-\u0001"
+        + "-\u0001-\u0001-\u0001-\u0001-\u0001-\u0001-\u0001-\u0001-\u0001-\u0001"
+        + "-\u0003-\u031c\b-\u0001.\u0001.\u0001/\u0001/\u00010\u00010\u00030\u0324"
+        + "\b0\u00010\u00010\u00030\u0328\b0\u00010\u00010\u00010\u00030\u032d\b"
+        + "0\u00011\u00011\u00012\u00012\u00013\u00013\u00013\u00053\u0336\b3\n3"
+        + "\f3\u0339\t3\u00013\u00013\u00014\u00014\u00034\u033f\b4\u00015\u0001"
+        + "5\u00015\u00035\u0344\b5\u00015\u00015\u00015\u00015\u00035\u034a\b5\u0001"
+        + "5\u00035\u034d\b5\u00016\u00016\u00036\u0351\b6\u00017\u00017\u00017\u0003"
+        + "7\u0356\b7\u00018\u00018\u00038\u035a\b8\u00019\u00019\u0001:\u0001:\u0001"
+        + ":\u0001:\u0001:\u0001;\u0001;\u0001;\u0000\u00036DF<\u0000\u0002\u0004"
+        + "\u0006\b\n\f\u000e\u0010\u0012\u0014\u0016\u0018\u001a\u001c\u001e \""
+        + "$&(*,.02468:<>@BDFHJLNPRTVXZ\\^`bdfhjlnprtv\u0000\u0010\u0006\u0000\u0005"
+        + "\u0005\u0007\u0007  ;;FFJJ\u0002\u0000,,YY\u0002\u0000\u0007\u0007FF\u0002"
+        + "\u0000((11\u0001\u0000\u001a\u001b\u0001\u0000wx\u0002\u0000\u0005\u0005"
+        + "\u0080\u0080\u0002\u0000\u000b\u000b\u001a\u001a\u0002\u0000%%77\u0002"
+        + "\u0000\u0005\u0005\u001c\u001c\u0001\u0000y{\u0001\u0000pv\u0002\u0000"
+        + "$$[[\u0005\u0000\u0017\u0018/0=@RSef\u0001\u0000~\u007f\u0017\u0000\u0006"
+        + "\u0007\u0011\u0012\u0014\u0017\u0019\u0019  \"\"%%\'\'*,//4477:;==??F"
+        + "FJMORUVXY]_aaee\u03cd\u0000x\u0001\u0000\u0000\u0000\u0002{\u0001\u0000"
+        + "\u0000\u0000\u0004\u0100\u0001\u0000\u0000\u0000\u0006\u010b\u0001\u0000"
+        + "\u0000\u0000\b\u010f\u0001\u0000\u0000\u0000\n\u0124\u0001\u0000\u0000"
+        + "\u0000\f\u012b\u0001\u0000\u0000\u0000\u000e\u012d\u0001\u0000\u0000\u0000"
+        + "\u0010\u0135\u0001\u0000\u0000\u0000\u0012\u014d\u0001\u0000\u0000\u0000"
+        + "\u0014\u015a\u0001\u0000\u0000\u0000\u0016\u0164\u0001\u0000\u0000\u0000"
+        + "\u0018\u0173\u0001\u0000\u0000\u0000\u001a\u0175\u0001\u0000\u0000\u0000"
+        + "\u001c\u017b\u0001\u0000\u0000\u0000\u001e\u017e\u0001\u0000\u0000\u0000"
+        + " \u0180\u0001\u0000\u0000\u0000\"\u0188\u0001\u0000\u0000\u0000$\u018f"
+        + "\u0001\u0000\u0000\u0000&\u01a1\u0001\u0000\u0000\u0000(\u01b2\u0001\u0000"
+        + "\u0000\u0000*\u01c2\u0001\u0000\u0000\u0000,\u01e0\u0001\u0000\u0000\u0000"
+        + ".\u01e2\u0001\u0000\u0000\u00000\u01ed\u0001\u0000\u0000\u00002\u01f5"
+        + "\u0001\u0000\u0000\u00004\u01fc\u0001\u0000\u0000\u00006\u021d\u0001\u0000"
+        + "\u0000\u00008\u022e\u0001\u0000\u0000\u0000:\u0231\u0001\u0000\u0000\u0000"
+        + "<\u0263\u0001\u0000\u0000\u0000>\u0265\u0001\u0000\u0000\u0000@\u0268"
+        + "\u0001\u0000\u0000\u0000B\u0272\u0001\u0000\u0000\u0000D\u0278\u0001\u0000"
+        + "\u0000\u0000F\u02ad\u0001\u0000\u0000\u0000H\u02ba\u0001\u0000\u0000\u0000"
+        + "J\u02c6\u0001\u0000\u0000\u0000L\u02c8\u0001\u0000\u0000\u0000N\u02cf"
+        + "\u0001\u0000\u0000\u0000P\u02db\u0001\u0000\u0000\u0000R\u02dd\u0001\u0000"
+        + "\u0000\u0000T\u02e9\u0001\u0000\u0000\u0000V\u02eb\u0001\u0000\u0000\u0000"
+        + "X\u02ff\u0001\u0000\u0000\u0000Z\u031b\u0001\u0000\u0000\u0000\\\u031d"
+        + "\u0001\u0000\u0000\u0000^\u031f\u0001\u0000\u0000\u0000`\u0321\u0001\u0000"
+        + "\u0000\u0000b\u032e\u0001\u0000\u0000\u0000d\u0330\u0001\u0000\u0000\u0000"
+        + "f\u0337\u0001\u0000\u0000\u0000h\u033e\u0001\u0000\u0000\u0000j\u034c"
+        + "\u0001\u0000\u0000\u0000l\u0350\u0001\u0000\u0000\u0000n\u0355\u0001\u0000"
+        + "\u0000\u0000p\u0359\u0001\u0000\u0000\u0000r\u035b\u0001\u0000\u0000\u0000"
+        + "t\u035d\u0001\u0000\u0000\u0000v\u0362\u0001\u0000\u0000\u0000xy\u0003"
+        + "\u0004\u0002\u0000yz\u0005\u0000\u0000\u0001z\u0001\u0001\u0000\u0000"
+        + "\u0000{|\u00034\u001a\u0000|}\u0005\u0000\u0000\u0001}\u0003\u0001\u0000"
+        + "\u0000\u0000~\u0101\u0003\u0006\u0003\u0000\u007f\u008d\u0005\"\u0000"
+        + "\u0000\u0080\u0089\u0005\u0001\u0000\u0000\u0081\u0082\u0005M\u0000\u0000"
+        + "\u0082\u0088\u0007\u0000\u0000\u0000\u0083\u0084\u0005\'\u0000\u0000\u0084"
+        + "\u0088\u0007\u0001\u0000\u0000\u0085\u0086\u0005a\u0000\u0000\u0086\u0088"
+        + "\u0003^/\u0000\u0087\u0081\u0001\u0000\u0000\u0000\u0087\u0083\u0001\u0000"
+        + "\u0000\u0000\u0087\u0085\u0001\u0000\u0000\u0000\u0088\u008b\u0001\u0000"
+        + "\u0000\u0000\u0089\u0087\u0001\u0000\u0000\u0000\u0089\u008a\u0001\u0000"
+        + "\u0000\u0000\u008a\u008c\u0001\u0000\u0000\u0000\u008b\u0089\u0001\u0000"
+        + "\u0000\u0000\u008c\u008e\u0005\u0002\u0000\u0000\u008d\u0080\u0001\u0000"
+        + "\u0000\u0000\u008d\u008e\u0001\u0000\u0000\u0000\u008e\u008f\u0001\u0000"
+        + "\u0000\u0000\u008f\u0101\u0003\u0004\u0002\u0000\u0090\u009c\u0005\u0019"
+        + "\u0000\u0000\u0091\u0098\u0005\u0001\u0000\u0000\u0092\u0093\u0005M\u0000"
+        + "\u0000\u0093\u0097\u0007\u0002\u0000\u0000\u0094\u0095\u0005\'\u0000\u0000"
+        + "\u0095\u0097\u0007\u0001\u0000\u0000\u0096\u0092\u0001\u0000\u0000\u0000"
+        + "\u0096\u0094\u0001\u0000\u0000\u0000\u0097\u009a\u0001\u0000\u0000\u0000"
+        + "\u0098\u0096\u0001\u0000\u0000\u0000\u0098\u0099\u0001\u0000\u0000\u0000"
+        + "\u0099\u009b\u0001\u0000\u0000\u0000\u009a\u0098\u0001\u0000\u0000\u0000"
+        + "\u009b\u009d\u0005\u0002\u0000\u0000\u009c\u0091\u0001\u0000\u0000\u0000"
+        + "\u009c\u009d\u0001\u0000\u0000\u0000\u009d\u009e\u0001\u0000\u0000\u0000"
+        + "\u009e\u0101\u0003\u0004\u0002\u0000\u009f\u00a0\u0005U\u0000\u0000\u00a0"
+        + "\u00a6\u0005X\u0000\u0000\u00a1\u00a4\u0005\u0010\u0000\u0000\u00a2\u00a5"
+        + "\u0003>\u001f\u0000\u00a3\u00a5\u0003r9\u0000\u00a4\u00a2\u0001\u0000"
+        + "\u0000\u0000\u00a4\u00a3\u0001\u0000\u0000\u0000\u00a5\u00a7\u0001\u0000"
+        + "\u0000\u0000\u00a6\u00a1\u0001\u0000\u0000\u0000\u00a6\u00a7\u0001\u0000"
+        + "\u0000\u0000\u00a7\u00aa\u0001\u0000\u0000\u0000\u00a8\u00a9\u00052\u0000"
+        + "\u0000\u00a9\u00ab\u0005)\u0000\u0000\u00aa\u00a8\u0001\u0000\u0000\u0000"
+        + "\u00aa\u00ab\u0001\u0000\u0000\u0000\u00ab\u00ae\u0001\u0000\u0000\u0000"
+        + "\u00ac\u00af\u0003>\u001f\u0000\u00ad\u00af\u0003j5\u0000\u00ae\u00ac"
+        + "\u0001\u0000\u0000\u0000\u00ae\u00ad\u0001\u0000\u0000\u0000\u00ae\u00af"
+        + "\u0001\u0000\u0000\u0000\u00af\u0101\u0001\u0000\u0000\u0000\u00b0\u00b1"
+        + "\u0005U\u0000\u0000\u00b1\u00b4\u0005\u0012\u0000\u0000\u00b2\u00b3\u0005"
+        + "\u0010\u0000\u0000\u00b3\u00b5\u0003r9\u0000\u00b4\u00b2\u0001\u0000\u0000"
+        + "\u0000\u00b4\u00b5\u0001\u0000\u0000\u0000\u00b5\u00b8\u0001\u0000\u0000"
+        + "\u0000\u00b6\u00b7\u00052\u0000\u0000\u00b7\u00b9\u0005)\u0000\u0000\u00b8"
+        + "\u00b6\u0001\u0000\u0000\u0000\u00b8\u00b9\u0001\u0000\u0000\u0000\u00b9"
+        + "\u00ba\u0001\u0000\u0000\u0000\u00ba\u00bd\u0007\u0003\u0000\u0000\u00bb"
+        + "\u00be\u0003>\u001f\u0000\u00bc\u00be\u0003j5\u0000\u00bd\u00bb\u0001"
+        + "\u0000\u0000\u0000\u00bd\u00bc\u0001\u0000\u0000\u0000\u00be\u0101\u0001"
+        + "\u0000\u0000\u0000\u00bf\u00c2\u0007\u0004\u0000\u0000\u00c0\u00c1\u0005"
+        + "\u0010\u0000\u0000\u00c1\u00c3\u0003r9\u0000\u00c2\u00c0\u0001\u0000\u0000"
+        + "\u0000\u00c2\u00c3\u0001\u0000\u0000\u0000\u00c3\u00c6\u0001\u0000\u0000"
+        + "\u0000\u00c4\u00c5\u00052\u0000\u0000\u00c5\u00c7\u0005)\u0000\u0000\u00c6"
+        + "\u00c4\u0001\u0000\u0000\u0000\u00c6\u00c7\u0001\u0000\u0000\u0000\u00c7"
+        + "\u00ca\u0001\u0000\u0000\u0000\u00c8\u00cb\u0003>\u001f\u0000\u00c9\u00cb"
+        + "\u0003j5\u0000\u00ca\u00c8\u0001\u0000\u0000\u0000\u00ca\u00c9\u0001\u0000"
+        + "\u0000\u0000\u00cb\u0101\u0001\u0000\u0000\u0000\u00cc\u00cd\u0005U\u0000"
+        + "\u0000\u00cd\u00cf\u0005+\u0000\u0000\u00ce\u00d0\u0003>\u001f\u0000\u00cf"
+        + "\u00ce\u0001\u0000\u0000\u0000\u00cf\u00d0\u0001\u0000\u0000\u0000\u00d0"
+        + "\u0101\u0001\u0000\u0000\u0000\u00d1\u00d2\u0005U\u0000\u0000\u00d2\u0101"
+        + "\u0005Q\u0000\u0000\u00d3\u00d4\u0005U\u0000\u0000\u00d4\u0101\u0005\u0011"
+        + "\u0000\u0000\u00d5\u00d6\u0005V\u0000\u0000\u00d6\u00d9\u0005X\u0000\u0000"
+        + "\u00d7\u00d8\u0005\u0010\u0000\u0000\u00d8\u00da\u0003>\u001f\u0000\u00d9"
+        + "\u00d7\u0001\u0000\u0000\u0000\u00d9\u00da\u0001\u0000\u0000\u0000\u00da"
+        + "\u00dd\u0001\u0000\u0000\u0000\u00db\u00de\u0003>\u001f\u0000\u00dc\u00de"
+        + "\u0003j5\u0000\u00dd\u00db\u0001\u0000\u0000\u0000\u00dd\u00dc\u0001\u0000"
+        + "\u0000\u0000\u00dd\u00de\u0001\u0000\u0000\u0000\u00de\u00e8\u0001\u0000"
+        + "\u0000\u0000\u00df\u00e0\u0005^\u0000\u0000\u00e0\u00e5\u0003r9\u0000"
+        + "\u00e1\u00e2\u0005\u0003\u0000\u0000\u00e2\u00e4\u0003r9\u0000\u00e3\u00e1"
+        + "\u0001\u0000\u0000\u0000\u00e4\u00e7\u0001\u0000\u0000\u0000\u00e5\u00e3"
+        + "\u0001\u0000\u0000\u0000\u00e5\u00e6\u0001\u0000\u0000\u0000\u00e6\u00e9"
+        + "\u0001\u0000\u0000\u0000\u00e7\u00e5\u0001\u0000\u0000\u0000\u00e8\u00df"
+        + "\u0001\u0000\u0000\u0000\u00e8\u00e9\u0001\u0000\u0000\u0000\u00e9\u0101"
+        + "\u0001\u0000\u0000\u0000\u00ea\u00eb\u0005V\u0000\u0000\u00eb\u00ee\u0005"
+        + "\u0012\u0000\u0000\u00ec\u00ed\u0005\u0010\u0000\u0000\u00ed\u00ef\u0003"
+        + "r9\u0000\u00ee\u00ec\u0001\u0000\u0000\u0000\u00ee\u00ef\u0001\u0000\u0000"
+        + "\u0000\u00ef\u00f3\u0001\u0000\u0000\u0000\u00f0\u00f1\u0005W\u0000\u0000"
+        + "\u00f1\u00f4\u0003>\u001f\u0000\u00f2\u00f4\u0003j5\u0000\u00f3\u00f0"
+        + "\u0001\u0000\u0000\u0000\u00f3\u00f2\u0001\u0000\u0000\u0000\u00f3\u00f4"
+        + "\u0001\u0000\u0000\u0000\u00f4\u00f6\u0001\u0000\u0000\u0000\u00f5\u00f7"
+        + "\u0003>\u001f\u0000\u00f6\u00f5\u0001\u0000\u0000\u0000\u00f6\u00f7\u0001"
+        + "\u0000\u0000\u0000\u00f7\u0101\u0001\u0000\u0000\u0000\u00f8\u00f9\u0005"
+        + "V\u0000\u0000\u00f9\u00fe\u0005_\u0000\u0000\u00fa\u00fc\u0007\u0005\u0000"
+        + "\u0000\u00fb\u00fa\u0001\u0000\u0000\u0000\u00fb\u00fc\u0001\u0000\u0000"
+        + "\u0000\u00fc\u00fd\u0001\u0000\u0000\u0000\u00fd\u00ff\u0003p8\u0000\u00fe"
+        + "\u00fb\u0001\u0000\u0000\u0000\u00fe\u00ff\u0001\u0000\u0000\u0000\u00ff"
+        + "\u0101\u0001\u0000\u0000\u0000\u0100~\u0001\u0000\u0000\u0000\u0100\u007f"
+        + "\u0001\u0000\u0000\u0000\u0100\u0090\u0001\u0000\u0000\u0000\u0100\u009f"
+        + "\u0001\u0000\u0000\u0000\u0100\u00b0\u0001\u0000\u0000\u0000\u0100\u00bf"
+        + "\u0001\u0000\u0000\u0000\u0100\u00cc\u0001\u0000\u0000\u0000\u0100\u00d1"
+        + "\u0001\u0000\u0000\u0000\u0100\u00d3\u0001\u0000\u0000\u0000\u0100\u00d5"
+        + "\u0001\u0000\u0000\u0000\u0100\u00ea\u0001\u0000\u0000\u0000\u0100\u00f8"
+        + "\u0001\u0000\u0000\u0000\u0101\u0005\u0001\u0000\u0000\u0000\u0102\u0103"
+        + "\u0005d\u0000\u0000\u0103\u0108\u0003\u001a\r\u0000\u0104\u0105\u0005"
+        + "\u0003\u0000\u0000\u0105\u0107\u0003\u001a\r\u0000\u0106\u0104\u0001\u0000"
+        + "\u0000\u0000\u0107\u010a\u0001\u0000\u0000\u0000\u0108\u0106\u0001\u0000"
+        + "\u0000\u0000\u0108\u0109\u0001\u0000\u0000\u0000\u0109\u010c\u0001\u0000"
+        + "\u0000\u0000\u010a\u0108\u0001\u0000\u0000\u0000\u010b\u0102\u0001\u0000"
+        + "\u0000\u0000\u010b\u010c\u0001\u0000\u0000\u0000\u010c\u010d\u0001\u0000"
+        + "\u0000\u0000\u010d\u010e\u0003\b\u0004\u0000\u010e\u0007\u0001\u0000\u0000"
+        + "\u0000\u010f\u011a\u0003\f\u0006\u0000\u0110\u0111\u0005H\u0000\u0000"
+        + "\u0111\u0112\u0005\r\u0000\u0000\u0112\u0117\u0003\u000e\u0007\u0000\u0113"
+        + "\u0114\u0005\u0003\u0000\u0000\u0114\u0116\u0003\u000e\u0007\u0000\u0115"
+        + "\u0113\u0001\u0000\u0000\u0000\u0116\u0119\u0001\u0000\u0000\u0000\u0117"
+        + "\u0115\u0001\u0000\u0000\u0000\u0117\u0118\u0001\u0000\u0000\u0000\u0118"
+        + "\u011b\u0001\u0000\u0000\u0000\u0119\u0117\u0001\u0000\u0000\u0000\u011a"
+        + "\u0110\u0001\u0000\u0000\u0000\u011a\u011b\u0001\u0000\u0000\u0000\u011b"
+        + "\u011d\u0001\u0000\u0000\u0000\u011c\u011e\u0003\n\u0005\u0000\u011d\u011c"
+        + "\u0001\u0000\u0000\u0000\u011d\u011e\u0001\u0000\u0000\u0000\u011e\t\u0001"
+        + "\u0000\u0000\u0000\u011f\u0120\u0005:\u0000\u0000\u0120\u0125\u0007\u0006"
+        + "\u0000\u0000\u0121\u0122\u0005i\u0000\u0000\u0122\u0123\u0007\u0006\u0000"
+        + "\u0000\u0123\u0125\u0005o\u0000\u0000\u0124\u011f\u0001\u0000\u0000\u0000"
+        + "\u0124\u0121\u0001\u0000\u0000\u0000\u0125\u000b\u0001\u0000\u0000\u0000"
+        + "\u0126\u012c\u0003\u0010\b\u0000\u0127\u0128\u0005\u0001\u0000\u0000\u0128"
+        + "\u0129\u0003\b\u0004\u0000\u0129\u012a\u0005\u0002\u0000\u0000\u012a\u012c"
+        + "\u0001\u0000\u0000\u0000\u012b\u0126\u0001\u0000\u0000\u0000\u012b\u0127"
+        + "\u0001\u0000\u0000\u0000\u012c\r\u0001\u0000\u0000\u0000\u012d\u012f\u0003"
+        + "4\u001a\u0000\u012e\u0130\u0007\u0007\u0000\u0000\u012f\u012e\u0001\u0000"
+        + "\u0000\u0000\u012f\u0130\u0001\u0000\u0000\u0000\u0130\u0133\u0001\u0000"
+        + "\u0000\u0000\u0131\u0132\u0005D\u0000\u0000\u0132\u0134\u0007\b\u0000"
+        + "\u0000\u0133\u0131\u0001\u0000\u0000\u0000\u0133\u0134\u0001\u0000\u0000"
+        + "\u0000\u0134\u000f\u0001\u0000\u0000\u0000\u0135\u0137\u0005T\u0000\u0000"
+        + "\u0136\u0138\u0003\u001c\u000e\u0000\u0137\u0136\u0001\u0000\u0000\u0000"
+        + "\u0137\u0138\u0001\u0000\u0000\u0000\u0138\u013a\u0001\u0000\u0000\u0000"
+        + "\u0139\u013b\u0003\u001e\u000f\u0000\u013a\u0139\u0001\u0000\u0000\u0000"
+        + "\u013a\u013b\u0001\u0000\u0000\u0000\u013b\u013c\u0001\u0000\u0000\u0000"
+        + "\u013c\u013e\u0003 \u0010\u0000\u013d\u013f\u0003\u0012\t\u0000\u013e"
+        + "\u013d\u0001\u0000\u0000\u0000\u013e\u013f\u0001\u0000\u0000\u0000\u013f"
+        + "\u0142\u0001\u0000\u0000\u0000\u0140\u0141\u0005c\u0000\u0000\u0141\u0143"
+        + "\u00036\u001b\u0000\u0142\u0140\u0001\u0000\u0000\u0000\u0142\u0143\u0001"
+        + "\u0000\u0000\u0000\u0143\u0147\u0001\u0000\u0000\u0000\u0144\u0145\u0005"
+        + "-\u0000\u0000\u0145\u0146\u0005\r\u0000\u0000\u0146\u0148\u0003\u0014"
+        + "\n\u0000\u0147\u0144\u0001\u0000\u0000\u0000\u0147\u0148\u0001\u0000\u0000"
+        + "\u0000\u0148\u014b\u0001\u0000\u0000\u0000\u0149\u014a\u0005.\u0000\u0000"
+        + "\u014a\u014c\u00036\u001b\u0000\u014b\u0149\u0001\u0000\u0000\u0000\u014b"
+        + "\u014c\u0001\u0000\u0000\u0000\u014c\u0011\u0001\u0000\u0000\u0000\u014d"
+        + "\u014e\u0005(\u0000\u0000\u014e\u0153\u0003$\u0012\u0000\u014f\u0150\u0005"
+        + "\u0003\u0000\u0000\u0150\u0152\u0003$\u0012\u0000\u0151\u014f\u0001\u0000"
+        + "\u0000\u0000\u0152\u0155\u0001\u0000\u0000\u0000\u0153\u0151\u0001\u0000"
+        + "\u0000\u0000\u0153\u0154\u0001\u0000\u0000\u0000\u0154\u0157\u0001\u0000"
+        + "\u0000\u0000\u0155\u0153\u0001\u0000\u0000\u0000\u0156\u0158\u0003.\u0017"
+        + "\u0000\u0157\u0156\u0001\u0000\u0000\u0000\u0157\u0158\u0001\u0000\u0000"
+        + "\u0000\u0158\u0013\u0001\u0000\u0000\u0000\u0159\u015b\u0003\u001e\u000f"
+        + "\u0000\u015a\u0159\u0001\u0000\u0000\u0000\u015a\u015b\u0001\u0000\u0000"
+        + "\u0000\u015b\u015c\u0001\u0000\u0000\u0000\u015c\u0161\u0003\u0016\u000b"
+        + "\u0000\u015d\u015e\u0005\u0003\u0000\u0000\u015e\u0160\u0003\u0016\u000b"
+        + "\u0000\u015f\u015d\u0001\u0000\u0000\u0000\u0160\u0163\u0001\u0000\u0000"
+        + "\u0000\u0161\u015f\u0001\u0000\u0000\u0000\u0161\u0162\u0001\u0000\u0000"
+        + "\u0000\u0162\u0015\u0001\u0000\u0000\u0000\u0163\u0161\u0001\u0000\u0000"
+        + "\u0000\u0164\u0165\u0003\u0018\f\u0000\u0165\u0017\u0001\u0000\u0000\u0000"
+        + "\u0166\u016f\u0005\u0001\u0000\u0000\u0167\u016c\u00034\u001a\u0000\u0168"
+        + "\u0169\u0005\u0003\u0000\u0000\u0169\u016b\u00034\u001a\u0000\u016a\u0168"
+        + "\u0001\u0000\u0000\u0000\u016b\u016e\u0001\u0000\u0000\u0000\u016c\u016a"
+        + "\u0001\u0000\u0000\u0000\u016c\u016d\u0001\u0000\u0000\u0000\u016d\u0170"
+        + "\u0001\u0000\u0000\u0000\u016e\u016c\u0001\u0000\u0000\u0000\u016f\u0167"
+        + "\u0001\u0000\u0000\u0000\u016f\u0170\u0001\u0000\u0000\u0000\u0170\u0171"
+        + "\u0001\u0000\u0000\u0000\u0171\u0174\u0005\u0002\u0000\u0000\u0172\u0174"
+        + "\u00034\u001a\u0000\u0173\u0166\u0001\u0000\u0000\u0000\u0173\u0172\u0001"
+        + "\u0000\u0000\u0000\u0174\u0019\u0001\u0000\u0000\u0000\u0175\u0176\u0003"
+        + "h4\u0000\u0176\u0177\u0005\n\u0000\u0000\u0177\u0178\u0005\u0001\u0000"
+        + "\u0000\u0178\u0179\u0003\b\u0004\u0000\u0179\u017a\u0005\u0002\u0000\u0000"
+        + "\u017a\u001b\u0001\u0000\u0000\u0000\u017b\u017c\u0005]\u0000\u0000\u017c"
+        + "\u017d\u0005\u0080\u0000\u0000\u017d\u001d\u0001\u0000\u0000\u0000\u017e"
+        + "\u017f\u0007\t\u0000\u0000\u017f\u001f\u0001\u0000\u0000\u0000\u0180\u0185"
+        + "\u0003\"\u0011\u0000\u0181\u0182\u0005\u0003\u0000\u0000\u0182\u0184\u0003"
+        + "\"\u0011\u0000\u0183\u0181\u0001\u0000\u0000\u0000\u0184\u0187\u0001\u0000"
+        + "\u0000\u0000\u0185\u0183\u0001\u0000\u0000\u0000\u0185\u0186\u0001\u0000"
+        + "\u0000\u0000\u0186!\u0001\u0000\u0000\u0000\u0187\u0185\u0001\u0000\u0000"
+        + "\u0000\u0188\u018d\u00034\u001a\u0000\u0189\u018b\u0005\n\u0000\u0000"
+        + "\u018a\u0189\u0001\u0000\u0000\u0000\u018a\u018b\u0001\u0000\u0000\u0000"
+        + "\u018b\u018c\u0001\u0000\u0000\u0000\u018c\u018e\u0003h4\u0000\u018d\u018a"
+        + "\u0001\u0000\u0000\u0000\u018d\u018e\u0001\u0000\u0000\u0000\u018e#\u0001"
+        + "\u0000\u0000\u0000\u018f\u0193\u0003,\u0016\u0000\u0190\u0192\u0003&\u0013"
+        + "\u0000\u0191\u0190\u0001\u0000\u0000\u0000\u0192\u0195\u0001\u0000\u0000"
+        + "\u0000\u0193\u0191\u0001\u0000\u0000\u0000\u0193\u0194\u0001\u0000\u0000"
+        + "\u0000\u0194%\u0001\u0000\u0000\u0000\u0195\u0193\u0001\u0000\u0000\u0000"
+        + "\u0196\u0197\u0003(\u0014\u0000\u0197\u0198\u00056\u0000\u0000\u0198\u019a"
+        + "\u0003,\u0016\u0000\u0199\u019b\u0003*\u0015\u0000\u019a\u0199\u0001\u0000"
+        + "\u0000\u0000\u019a\u019b\u0001\u0000\u0000\u0000\u019b\u01a2\u0001\u0000"
+        + "\u0000\u0000\u019c\u019d\u0005A\u0000\u0000\u019d\u019e\u0003(\u0014\u0000"
+        + "\u019e\u019f\u00056\u0000\u0000\u019f\u01a0\u0003,\u0016\u0000\u01a0\u01a2"
+        + "\u0001\u0000\u0000\u0000\u01a1\u0196\u0001\u0000\u0000\u0000\u01a1\u019c"
+        + "\u0001\u0000\u0000\u0000\u01a2\'\u0001\u0000\u0000\u0000\u01a3\u01a5\u0005"
+        + "3\u0000\u0000\u01a4\u01a3\u0001\u0000\u0000\u0000\u01a4\u01a5\u0001\u0000"
+        + "\u0000\u0000\u01a5\u01b3\u0001\u0000\u0000\u0000\u01a6\u01a8\u00058\u0000"
+        + "\u0000\u01a7\u01a9\u0005I\u0000\u0000\u01a8\u01a7\u0001\u0000\u0000\u0000"
+        + "\u01a8\u01a9\u0001\u0000\u0000\u0000\u01a9\u01b3\u0001\u0000\u0000\u0000"
+        + "\u01aa\u01ac\u0005N\u0000\u0000\u01ab\u01ad\u0005I\u0000\u0000\u01ac\u01ab"
+        + "\u0001\u0000\u0000\u0000\u01ac\u01ad\u0001\u0000\u0000\u0000\u01ad\u01b3"
+        + "\u0001\u0000\u0000\u0000\u01ae\u01b0\u0005*\u0000\u0000\u01af\u01b1\u0005"
+        + "I\u0000\u0000\u01b0\u01af\u0001\u0000\u0000\u0000\u01b0\u01b1\u0001\u0000"
+        + "\u0000\u0000\u01b1\u01b3\u0001\u0000\u0000\u0000\u01b2\u01a4\u0001\u0000"
+        + "\u0000\u0000\u01b2\u01a6\u0001\u0000\u0000\u0000\u01b2\u01aa\u0001\u0000"
+        + "\u0000\u0000\u01b2\u01ae\u0001\u0000\u0000\u0000\u01b3)\u0001\u0000\u0000"
+        + "\u0000\u01b4\u01b5\u0005E\u0000\u0000\u01b5\u01c3\u00036\u001b\u0000\u01b6"
+        + "\u01b7\u0005`\u0000\u0000\u01b7\u01b8\u0005\u0001\u0000\u0000\u01b8\u01bd"
+        + "\u0003h4\u0000\u01b9\u01ba\u0005\u0003\u0000\u0000\u01ba\u01bc\u0003h"
+        + "4\u0000\u01bb\u01b9\u0001\u0000\u0000\u0000\u01bc\u01bf\u0001\u0000\u0000"
+        + "\u0000\u01bd\u01bb\u0001\u0000\u0000\u0000\u01bd\u01be\u0001\u0000\u0000"
+        + "\u0000\u01be\u01c0\u0001\u0000\u0000\u0000\u01bf\u01bd\u0001\u0000\u0000"
+        + "\u0000\u01c0\u01c1\u0005\u0002\u0000\u0000\u01c1\u01c3\u0001\u0000\u0000"
+        + "\u0000\u01c2\u01b4\u0001\u0000\u0000\u0000\u01c2\u01b6\u0001\u0000\u0000"
+        + "\u0000\u01c3+\u0001\u0000\u0000\u0000\u01c4\u01c6\u0005)\u0000\u0000\u01c5"
+        + "\u01c4\u0001\u0000\u0000\u0000\u01c5\u01c6\u0001\u0000\u0000\u0000\u01c6"
+        + "\u01c7\u0001\u0000\u0000\u0000\u01c7\u01cc\u0003j5\u0000\u01c8\u01ca\u0005"
+        + "\n\u0000\u0000\u01c9\u01c8\u0001\u0000\u0000\u0000\u01c9\u01ca\u0001\u0000"
+        + "\u0000\u0000\u01ca\u01cb\u0001\u0000\u0000\u0000\u01cb\u01cd\u0003f3\u0000"
+        + "\u01cc\u01c9\u0001\u0000\u0000\u0000\u01cc\u01cd\u0001\u0000\u0000\u0000"
+        + "\u01cd\u01e1\u0001\u0000\u0000\u0000\u01ce\u01cf\u0005\u0001\u0000\u0000"
+        + "\u01cf\u01d0\u0003\b\u0004\u0000\u01d0\u01d5\u0005\u0002\u0000\u0000\u01d1"
+        + "\u01d3\u0005\n\u0000\u0000\u01d2\u01d1\u0001\u0000\u0000\u0000\u01d2\u01d3"
+        + "\u0001\u0000\u0000\u0000\u01d3\u01d4\u0001\u0000\u0000\u0000\u01d4\u01d6"
+        + "\u0003f3\u0000\u01d5\u01d2\u0001\u0000\u0000\u0000\u01d5\u01d6\u0001\u0000"
+        + "\u0000\u0000\u01d6\u01e1\u0001\u0000\u0000\u0000\u01d7\u01d8\u0005\u0001"
+        + "\u0000\u0000\u01d8\u01d9\u0003$\u0012\u0000\u01d9\u01de\u0005\u0002\u0000"
+        + "\u0000\u01da\u01dc\u0005\n\u0000\u0000\u01db\u01da\u0001\u0000\u0000\u0000"
+        + "\u01db\u01dc\u0001\u0000\u0000\u0000\u01dc\u01dd\u0001\u0000\u0000\u0000"
+        + "\u01dd\u01df\u0003f3\u0000\u01de\u01db\u0001\u0000\u0000\u0000\u01de\u01df"
+        + "\u0001\u0000\u0000\u0000\u01df\u01e1\u0001\u0000\u0000\u0000\u01e0\u01c5"
+        + "\u0001\u0000\u0000\u0000\u01e0\u01ce\u0001\u0000\u0000\u0000\u01e0\u01d7"
+        + "\u0001\u0000\u0000\u0000\u01e1-\u0001\u0000\u0000\u0000\u01e2\u01e3\u0005"
+        + "L\u0000\u0000\u01e3\u01e4\u0005\u0001\u0000\u0000\u01e4\u01e5\u00030\u0018"
+        + "\u0000\u01e5\u01e6\u0005&\u0000\u0000\u01e6\u01e7\u0003f3\u0000\u01e7"
+        + "\u01e8\u00051\u0000\u0000\u01e8\u01e9\u0005\u0001\u0000\u0000\u01e9\u01ea"
+        + "\u00030\u0018\u0000\u01ea\u01eb\u0005\u0002\u0000\u0000\u01eb\u01ec\u0005"
+        + "\u0002\u0000\u0000\u01ec/\u0001\u0000\u0000\u0000\u01ed\u01f2\u00032\u0019"
+        + "\u0000\u01ee\u01ef\u0005\u0003\u0000\u0000\u01ef\u01f1\u00032\u0019\u0000"
+        + "\u01f0\u01ee\u0001\u0000\u0000\u0000\u01f1\u01f4\u0001\u0000\u0000\u0000"
+        + "\u01f2\u01f0\u0001\u0000\u0000\u0000\u01f2\u01f3\u0001\u0000\u0000\u0000"
+        + "\u01f31\u0001\u0000\u0000\u0000\u01f4\u01f2\u0001\u0000\u0000\u0000\u01f5"
+        + "\u01fa\u0003D\"\u0000\u01f6\u01f8\u0005\n\u0000\u0000\u01f7\u01f6\u0001"
+        + "\u0000\u0000\u0000\u01f7\u01f8\u0001\u0000\u0000\u0000\u01f8\u01f9\u0001"
+        + "\u0000\u0000\u0000\u01f9\u01fb\u0003h4\u0000\u01fa\u01f7\u0001\u0000\u0000"
+        + "\u0000\u01fa\u01fb\u0001\u0000\u0000\u0000\u01fb3\u0001\u0000\u0000\u0000"
+        + "\u01fc\u01fd\u00036\u001b\u0000\u01fd5\u0001\u0000\u0000\u0000\u01fe\u01ff"
+        + "\u0006\u001b\uffff\uffff\u0000\u01ff\u0200\u0005B\u0000\u0000\u0200\u021e"
+        + "\u00036\u001b\b\u0201\u0202\u0005!\u0000\u0000\u0202\u0203\u0005\u0001"
+        + "\u0000\u0000\u0203\u0204\u0003\u0006\u0003\u0000\u0204\u0205\u0005\u0002"
+        + "\u0000\u0000\u0205\u021e\u0001\u0000\u0000\u0000\u0206\u0207\u0005P\u0000"
+        + "\u0000\u0207\u0208\u0005\u0001\u0000\u0000\u0208\u0209\u0003r9\u0000\u0209"
+        + "\u020a\u00038\u001c\u0000\u020a\u020b\u0005\u0002\u0000\u0000\u020b\u021e"
+        + "\u0001\u0000\u0000\u0000\u020c\u020d\u0005<\u0000\u0000\u020d\u020e\u0005"
+        + "\u0001\u0000\u0000\u020e\u020f\u0003f3\u0000\u020f\u0210\u0005\u0003\u0000"
+        + "\u0000\u0210\u0211\u0003r9\u0000\u0211\u0212\u00038\u001c\u0000\u0212"
+        + "\u0213\u0005\u0002\u0000\u0000\u0213\u021e\u0001\u0000\u0000\u0000\u0214"
+        + "\u0215\u0005<\u0000\u0000\u0215\u0216\u0005\u0001\u0000\u0000\u0216\u0217"
+        + "\u0003r9\u0000\u0217\u0218\u0005\u0003\u0000\u0000\u0218\u0219\u0003r"
+        + "9\u0000\u0219\u021a\u00038\u001c\u0000\u021a\u021b\u0005\u0002\u0000\u0000"
+        + "\u021b\u021e\u0001\u0000\u0000\u0000\u021c\u021e\u0003:\u001d\u0000\u021d"
+        + "\u01fe\u0001\u0000\u0000\u0000\u021d\u0201\u0001\u0000\u0000\u0000\u021d"
+        + "\u0206\u0001\u0000\u0000\u0000\u021d\u020c\u0001\u0000\u0000\u0000\u021d"
+        + "\u0214\u0001\u0000\u0000\u0000\u021d\u021c\u0001\u0000\u0000\u0000\u021e"
+        + "\u0227\u0001\u0000\u0000\u0000\u021f\u0220\n\u0002\u0000\u0000\u0220\u0221"
+        + "\u0005\b\u0000\u0000\u0221\u0226\u00036\u001b\u0003\u0222\u0223\n\u0001"
+        + "\u0000\u0000\u0223\u0224\u0005G\u0000\u0000\u0224\u0226\u00036\u001b\u0002"
+        + "\u0225\u021f\u0001\u0000\u0000\u0000\u0225\u0222\u0001\u0000\u0000\u0000"
+        + "\u0226\u0229\u0001\u0000\u0000\u0000\u0227\u0225\u0001\u0000\u0000\u0000"
+        + "\u0227\u0228\u0001\u0000\u0000\u0000\u02287\u0001\u0000\u0000\u0000\u0229"
+        + "\u0227\u0001\u0000\u0000\u0000\u022a\u022b\u0005\u0003\u0000\u0000\u022b"
+        + "\u022d\u0003r9\u0000\u022c\u022a\u0001\u0000\u0000\u0000\u022d\u0230\u0001"
+        + "\u0000\u0000\u0000\u022e\u022c\u0001\u0000\u0000\u0000\u022e\u022f\u0001"
+        + "\u0000\u0000\u0000\u022f9\u0001\u0000\u0000\u0000\u0230\u022e\u0001\u0000"
+        + "\u0000\u0000\u0231\u0233\u0003D\"\u0000\u0232\u0234\u0003<\u001e\u0000"
+        + "\u0233\u0232\u0001\u0000\u0000\u0000\u0233\u0234\u0001\u0000\u0000\u0000"
+        + "\u0234;\u0001\u0000\u0000\u0000\u0235\u0237\u0005B\u0000\u0000\u0236\u0235"
+        + "\u0001\u0000\u0000\u0000\u0236\u0237\u0001\u0000\u0000\u0000\u0237\u0238"
+        + "\u0001\u0000\u0000\u0000\u0238\u0239\u0005\f\u0000\u0000\u0239\u023a\u0003"
+        + "D\"\u0000\u023a\u023b\u0005\b\u0000\u0000\u023b\u023c\u0003D\"\u0000\u023c"
+        + "\u0264\u0001\u0000\u0000\u0000\u023d\u023f\u0005B\u0000\u0000\u023e\u023d"
+        + "\u0001\u0000\u0000\u0000\u023e\u023f\u0001\u0000\u0000\u0000\u023f\u0240"
+        + "\u0001\u0000\u0000\u0000\u0240\u0241\u00051\u0000\u0000\u0241\u0242\u0005"
+        + "\u0001\u0000\u0000\u0242\u0247\u0003D\"\u0000\u0243\u0244\u0005\u0003"
+        + "\u0000\u0000\u0244\u0246\u0003D\"\u0000\u0245\u0243\u0001\u0000\u0000"
+        + "\u0000\u0246\u0249\u0001\u0000\u0000\u0000\u0247\u0245\u0001\u0000\u0000"
+        + "\u0000\u0247\u0248\u0001\u0000\u0000\u0000\u0248\u024a\u0001\u0000\u0000"
+        + "\u0000\u0249\u0247\u0001\u0000\u0000\u0000\u024a\u024b\u0005\u0002\u0000"
+        + "\u0000\u024b\u0264\u0001\u0000\u0000\u0000\u024c\u024e\u0005B\u0000\u0000"
+        + "\u024d\u024c\u0001\u0000\u0000\u0000\u024d\u024e\u0001\u0000\u0000\u0000"
+        + "\u024e\u024f\u0001\u0000\u0000\u0000\u024f\u0250\u00051\u0000\u0000\u0250"
+        + "\u0251\u0005\u0001\u0000\u0000\u0251\u0252\u0003\u0006\u0003\u0000\u0252"
+        + "\u0253\u0005\u0002\u0000\u0000\u0253\u0264\u0001\u0000\u0000\u0000\u0254"
+        + "\u0256\u0005B\u0000\u0000\u0255\u0254\u0001\u0000\u0000\u0000\u0255\u0256"
+        + "\u0001\u0000\u0000\u0000\u0256\u0257\u0001\u0000\u0000\u0000\u0257\u0258"
+        + "\u00059\u0000\u0000\u0258\u0264\u0003@ \u0000\u0259\u025b\u0005B\u0000"
+        + "\u0000\u025a\u0259\u0001\u0000\u0000\u0000\u025a\u025b\u0001\u0000\u0000"
+        + "\u0000\u025b\u025c\u0001\u0000\u0000\u0000\u025c\u025d\u0005O\u0000\u0000"
+        + "\u025d\u0264\u0003r9\u0000\u025e\u0260\u00055\u0000\u0000\u025f\u0261"
+        + "\u0005B\u0000\u0000\u0260\u025f\u0001\u0000\u0000\u0000\u0260\u0261\u0001"
+        + "\u0000\u0000\u0000\u0261\u0262\u0001\u0000\u0000\u0000\u0262\u0264\u0005"
+        + "C\u0000\u0000\u0263\u0236\u0001\u0000\u0000\u0000\u0263\u023e\u0001\u0000"
+        + "\u0000\u0000\u0263\u024d\u0001\u0000\u0000\u0000\u0263\u0255\u0001\u0000"
+        + "\u0000\u0000\u0263\u025a\u0001\u0000\u0000\u0000\u0263\u025e\u0001\u0000"
+        + "\u0000\u0000\u0264=\u0001\u0000\u0000\u0000\u0265\u0266\u00059\u0000\u0000"
+        + "\u0266\u0267\u0003@ \u0000\u0267?\u0001\u0000\u0000\u0000\u0268\u026a"
+        + "\u0003r9\u0000\u0269\u026b\u0003B!\u0000\u026a\u0269\u0001\u0000\u0000"
+        + "\u0000\u026a\u026b\u0001\u0000\u0000\u0000\u026bA\u0001\u0000\u0000\u0000"
+        + "\u026c\u026d\u0005\u001f\u0000\u0000\u026d\u0273\u0003r9\u0000\u026e\u026f"
+        + "\u0005g\u0000\u0000\u026f\u0270\u0003r9\u0000\u0270\u0271\u0005o\u0000"
+        + "\u0000\u0271\u0273\u0001\u0000\u0000\u0000\u0272\u026c\u0001\u0000\u0000"
+        + "\u0000\u0272\u026e\u0001\u0000\u0000\u0000\u0273C\u0001\u0000\u0000\u0000"
+        + "\u0274\u0275\u0006\"\uffff\uffff\u0000\u0275\u0279\u0003F#\u0000\u0276"
+        + "\u0277\u0007\u0005\u0000\u0000\u0277\u0279\u0003D\"\u0004\u0278\u0274"
+        + "\u0001\u0000\u0000\u0000\u0278\u0276\u0001\u0000\u0000\u0000\u0279\u0286"
+        + "\u0001\u0000\u0000\u0000\u027a\u027b\n\u0003\u0000\u0000\u027b\u027c\u0007"
+        + "\n\u0000\u0000\u027c\u0285\u0003D\"\u0004\u027d\u027e\n\u0002\u0000\u0000"
+        + "\u027e\u027f\u0007\u0005\u0000\u0000\u027f\u0285\u0003D\"\u0003\u0280"
+        + "\u0281\n\u0001\u0000\u0000\u0281\u0282\u0003\\.\u0000\u0282\u0283\u0003"
+        + "D\"\u0002\u0283\u0285\u0001\u0000\u0000\u0000\u0284\u027a\u0001\u0000"
+        + "\u0000\u0000\u0284\u027d\u0001\u0000\u0000\u0000\u0284\u0280\u0001\u0000"
+        + "\u0000\u0000\u0285\u0288\u0001\u0000\u0000\u0000\u0286\u0284\u0001\u0000"
+        + "\u0000\u0000\u0286\u0287\u0001\u0000\u0000\u0000\u0287E\u0001\u0000\u0000"
+        + "\u0000\u0288\u0286\u0001\u0000\u0000\u0000\u0289\u028a\u0006#\uffff\uffff"
+        + "\u0000\u028a\u02ae\u0003J%\u0000\u028b\u02ae\u0003P(\u0000\u028c\u02ae"
+        + "\u0003H$\u0000\u028d\u02ae\u0003Z-\u0000\u028e\u028f\u0003f3\u0000\u028f"
+        + "\u0290\u0005}\u0000\u0000\u0290\u0292\u0001\u0000\u0000\u0000\u0291\u028e"
+        + "\u0001\u0000\u0000\u0000\u0291\u0292\u0001\u0000\u0000\u0000\u0292\u0293"
+        + "\u0001\u0000\u0000\u0000\u0293\u02ae\u0005y\u0000\u0000\u0294\u02ae\u0003"
+        + "T*\u0000\u0295\u0296\u0005\u0001\u0000\u0000\u0296\u0297\u0003\u0006\u0003"
+        + "\u0000\u0297\u0298\u0005\u0002\u0000\u0000\u0298\u02ae\u0001\u0000\u0000"
+        + "\u0000\u0299\u02ae\u0003f3\u0000\u029a\u029b\u0005\u0001\u0000\u0000\u029b"
+        + "\u029c\u00034\u001a\u0000\u029c\u029d\u0005\u0002\u0000\u0000\u029d\u02ae"
+        + "\u0001\u0000\u0000\u0000\u029e\u02a0\u0005\u000e\u0000\u0000\u029f\u02a1"
+        + "\u00036\u001b\u0000\u02a0\u029f\u0001\u0000\u0000\u0000\u02a0\u02a1\u0001"
+        + "\u0000\u0000\u0000\u02a1\u02a3\u0001\u0000\u0000\u0000\u02a2\u02a4\u0003"
+        + "t:\u0000\u02a3\u02a2\u0001\u0000\u0000\u0000\u02a4\u02a5\u0001\u0000\u0000"
+        + "\u0000\u02a5\u02a3\u0001\u0000\u0000\u0000\u02a5\u02a6\u0001\u0000\u0000"
+        + "\u0000\u02a6\u02a9\u0001\u0000\u0000\u0000\u02a7\u02a8\u0005\u001d\u0000"
+        + "\u0000\u02a8\u02aa\u00036\u001b\u0000\u02a9\u02a7\u0001\u0000\u0000\u0000"
+        + "\u02a9\u02aa\u0001\u0000\u0000\u0000\u02aa\u02ab\u0001\u0000\u0000\u0000"
+        + "\u02ab\u02ac\u0005\u001e\u0000\u0000\u02ac\u02ae\u0001\u0000\u0000\u0000"
+        + "\u02ad\u0289\u0001\u0000\u0000\u0000\u02ad\u028b\u0001\u0000\u0000\u0000"
+        + "\u02ad\u028c\u0001\u0000\u0000\u0000\u02ad\u028d\u0001\u0000\u0000\u0000"
+        + "\u02ad\u0291\u0001\u0000\u0000\u0000\u02ad\u0294\u0001\u0000\u0000\u0000"
+        + "\u02ad\u0295\u0001\u0000\u0000\u0000\u02ad\u0299\u0001\u0000\u0000\u0000"
+        + "\u02ad\u029a\u0001\u0000\u0000\u0000\u02ad\u029e\u0001\u0000\u0000\u0000"
+        + "\u02ae\u02b4\u0001\u0000\u0000\u0000\u02af\u02b0\n\n\u0000\u0000\u02b0"
+        + "\u02b1\u0005|\u0000\u0000\u02b1\u02b3\u0003d2\u0000\u02b2\u02af\u0001"
+        + "\u0000\u0000\u0000\u02b3\u02b6\u0001\u0000\u0000\u0000\u02b4\u02b2\u0001"
+        + "\u0000\u0000\u0000\u02b4\u02b5\u0001\u0000\u0000\u0000\u02b5G\u0001\u0000"
+        + "\u0000\u0000\u02b6\u02b4\u0001\u0000\u0000\u0000\u02b7\u02bb\u0005\u0016"
+        + "\u0000\u0000\u02b8\u02bb\u0005\u0014\u0000\u0000\u02b9\u02bb\u0005\u0015"
+        + "\u0000\u0000\u02ba\u02b7\u0001\u0000\u0000\u0000\u02ba\u02b8\u0001\u0000"
+        + "\u0000\u0000\u02ba\u02b9\u0001\u0000\u0000\u0000\u02bbI\u0001\u0000\u0000"
+        + "\u0000\u02bc\u02c7\u0003L&\u0000\u02bd\u02be\u0005h\u0000\u0000\u02be"
+        + "\u02bf\u0003L&\u0000\u02bf\u02c0\u0005o\u0000\u0000\u02c0\u02c7\u0001"
+        + "\u0000\u0000\u0000\u02c1\u02c7\u0003N\'\u0000\u02c2\u02c3\u0005h\u0000"
+        + "\u0000\u02c3\u02c4\u0003N\'\u0000\u02c4\u02c5\u0005o\u0000\u0000\u02c5"
+        + "\u02c7\u0001\u0000\u0000\u0000\u02c6\u02bc\u0001\u0000\u0000\u0000\u02c6"
+        + "\u02bd\u0001\u0000\u0000\u0000\u02c6\u02c1\u0001\u0000\u0000\u0000\u02c6"
+        + "\u02c2\u0001\u0000\u0000\u0000\u02c7K\u0001\u0000\u0000\u0000\u02c8\u02c9"
+        + "\u0005\u000f\u0000\u0000\u02c9\u02ca\u0005\u0001\u0000\u0000\u02ca\u02cb"
+        + "\u00034\u001a\u0000\u02cb\u02cc\u0005\n\u0000\u0000\u02cc\u02cd\u0003"
+        + "d2\u0000\u02cd\u02ce\u0005\u0002\u0000\u0000\u02ceM\u0001\u0000\u0000"
+        + "\u0000\u02cf\u02d0\u0005\u0013\u0000\u0000\u02d0\u02d1\u0005\u0001\u0000"
+        + "\u0000\u02d1\u02d2\u00034\u001a\u0000\u02d2\u02d3\u0005\u0003\u0000\u0000"
+        + "\u02d3\u02d4\u0003d2\u0000\u02d4\u02d5\u0005\u0002\u0000\u0000\u02d5O"
+        + "\u0001\u0000\u0000\u0000\u02d6\u02dc\u0003R)\u0000\u02d7\u02d8\u0005h"
+        + "\u0000\u0000\u02d8\u02d9\u0003R)\u0000\u02d9\u02da\u0005o\u0000\u0000"
+        + "\u02da\u02dc\u0001\u0000\u0000\u0000\u02db\u02d6\u0001\u0000\u0000\u0000"
+        + "\u02db\u02d7\u0001\u0000\u0000\u0000\u02dcQ\u0001\u0000\u0000\u0000\u02dd"
+        + "\u02de\u0005#\u0000\u0000\u02de\u02df\u0005\u0001\u0000\u0000\u02df\u02e0"
+        + "\u0003h4\u0000\u02e0\u02e1\u0005(\u0000\u0000\u02e1\u02e2\u0003D\"\u0000"
+        + "\u02e2\u02e3\u0005\u0002\u0000\u0000\u02e3S\u0001\u0000\u0000\u0000\u02e4"
+        + "\u02ea\u0003V+\u0000\u02e5\u02e6\u0005h\u0000\u0000\u02e6\u02e7\u0003"
+        + "V+\u0000\u02e7\u02e8\u0005o\u0000\u0000\u02e8\u02ea\u0001\u0000\u0000"
+        + "\u0000\u02e9\u02e4\u0001\u0000\u0000\u0000\u02e9\u02e5\u0001\u0000\u0000"
+        + "\u0000\u02eaU\u0001\u0000\u0000\u0000\u02eb\u02ec\u0003X,\u0000\u02ec"
+        + "\u02f8\u0005\u0001\u0000\u0000\u02ed\u02ef\u0003\u001e\u000f\u0000\u02ee"
+        + "\u02ed\u0001\u0000\u0000\u0000\u02ee\u02ef\u0001\u0000\u0000\u0000\u02ef"
+        + "\u02f0\u0001\u0000\u0000\u0000\u02f0\u02f5\u00034\u001a\u0000\u02f1\u02f2"
+        + "\u0005\u0003\u0000\u0000\u02f2\u02f4\u00034\u001a\u0000\u02f3\u02f1\u0001"
+        + "\u0000\u0000\u0000\u02f4\u02f7\u0001\u0000\u0000\u0000\u02f5\u02f3\u0001"
+        + "\u0000\u0000\u0000\u02f5\u02f6\u0001\u0000\u0000\u0000\u02f6\u02f9\u0001"
+        + "\u0000\u0000\u0000\u02f7\u02f5\u0001\u0000\u0000\u0000\u02f8\u02ee\u0001"
+        + "\u0000\u0000\u0000\u02f8\u02f9\u0001\u0000\u0000\u0000\u02f9\u02fa\u0001"
+        + "\u0000\u0000\u0000\u02fa\u02fb\u0005\u0002\u0000\u0000\u02fbW\u0001\u0000"
+        + "\u0000\u0000\u02fc\u0300\u00058\u0000\u0000\u02fd\u0300\u0005N\u0000\u0000"
+        + "\u02fe\u0300\u0003h4\u0000\u02ff\u02fc\u0001\u0000\u0000\u0000\u02ff\u02fd"
+        + "\u0001\u0000\u0000\u0000\u02ff\u02fe\u0001\u0000\u0000\u0000\u0300Y\u0001"
+        + "\u0000\u0000\u0000\u0301\u031c\u0005C\u0000\u0000\u0302\u031c\u0003`0"
+        + "\u0000\u0303\u031c\u0003p8\u0000\u0304\u031c\u0003^/\u0000\u0305\u0307"
+        + "\u0005\u007f\u0000\u0000\u0306\u0305\u0001\u0000\u0000\u0000\u0307\u0308"
+        + "\u0001\u0000\u0000\u0000\u0308\u0306\u0001\u0000\u0000\u0000\u0308\u0309"
+        + "\u0001\u0000\u0000\u0000\u0309\u031c\u0001\u0000\u0000\u0000\u030a\u031c"
+        + "\u0005~\u0000\u0000\u030b\u030c\u0005j\u0000\u0000\u030c\u030d\u0003r"
+        + "9\u0000\u030d\u030e\u0005o\u0000\u0000\u030e\u031c\u0001\u0000\u0000\u0000"
+        + "\u030f\u0310\u0005k\u0000\u0000\u0310\u0311\u0003r9\u0000\u0311\u0312"
+        + "\u0005o\u0000\u0000\u0312\u031c\u0001\u0000\u0000\u0000\u0313\u0314\u0005"
+        + "l\u0000\u0000\u0314\u0315\u0003r9\u0000\u0315\u0316\u0005o\u0000\u0000"
+        + "\u0316\u031c\u0001\u0000\u0000\u0000\u0317\u0318\u0005m\u0000\u0000\u0318"
+        + "\u0319\u0003r9\u0000\u0319\u031a\u0005o\u0000\u0000\u031a\u031c\u0001"
+        + "\u0000\u0000\u0000\u031b\u0301\u0001\u0000\u0000\u0000\u031b\u0302\u0001"
+        + "\u0000\u0000\u0000\u031b\u0303\u0001\u0000\u0000\u0000\u031b\u0304\u0001"
+        + "\u0000\u0000\u0000\u031b\u0306\u0001\u0000\u0000\u0000\u031b\u030a\u0001"
+        + "\u0000\u0000\u0000\u031b\u030b\u0001\u0000\u0000\u0000\u031b\u030f\u0001"
+        + "\u0000\u0000\u0000\u031b\u0313\u0001\u0000\u0000\u0000\u031b\u0317\u0001"
+        + "\u0000\u0000\u0000\u031c[\u0001\u0000\u0000\u0000\u031d\u031e\u0007\u000b"
+        + "\u0000\u0000\u031e]\u0001\u0000\u0000\u0000\u031f\u0320\u0007\f\u0000"
+        + "\u0000\u0320_\u0001\u0000\u0000\u0000\u0321\u0323\u00054\u0000\u0000\u0322"
+        + "\u0324\u0007\u0005\u0000\u0000\u0323\u0322\u0001\u0000\u0000\u0000\u0323"
+        + "\u0324\u0001\u0000\u0000\u0000\u0324\u0327\u0001\u0000\u0000\u0000\u0325"
+        + "\u0328\u0003p8\u0000\u0326\u0328\u0003r9\u0000\u0327\u0325\u0001\u0000"
+        + "\u0000\u0000\u0327\u0326\u0001\u0000\u0000\u0000\u0328\u0329\u0001\u0000"
+        + "\u0000\u0000\u0329\u032c\u0003b1\u0000\u032a\u032b\u0005\\\u0000\u0000"
+        + "\u032b\u032d\u0003b1\u0000\u032c\u032a\u0001\u0000\u0000\u0000\u032c\u032d"
+        + "\u0001\u0000\u0000\u0000\u032da\u0001\u0000\u0000\u0000\u032e\u032f\u0007"
+        + "\r\u0000\u0000\u032fc\u0001\u0000\u0000\u0000\u0330\u0331\u0003h4\u0000"
+        + "\u0331e\u0001\u0000\u0000\u0000\u0332\u0333\u0003h4\u0000\u0333\u0334"
+        + "\u0005}\u0000\u0000\u0334\u0336\u0001\u0000\u0000\u0000\u0335\u0332\u0001"
+        + "\u0000\u0000\u0000\u0336\u0339\u0001\u0000\u0000\u0000\u0337\u0335\u0001"
+        + "\u0000\u0000\u0000\u0337\u0338\u0001\u0000\u0000\u0000\u0338\u033a\u0001"
+        + "\u0000\u0000\u0000\u0339\u0337\u0001\u0000\u0000\u0000\u033a\u033b\u0003"
+        + "h4\u0000\u033bg\u0001\u0000\u0000\u0000\u033c\u033f\u0003l6\u0000\u033d"
+        + "\u033f\u0003n7\u0000\u033e\u033c\u0001\u0000\u0000\u0000\u033e\u033d\u0001"
+        + "\u0000\u0000\u0000\u033fi\u0001\u0000\u0000\u0000\u0340\u0341\u0003h4"
+        + "\u0000\u0341\u0342\u0005\u0004\u0000\u0000\u0342\u0344\u0001\u0000\u0000"
+        + "\u0000\u0343\u0340\u0001\u0000\u0000\u0000\u0343\u0344\u0001\u0000\u0000"
+        + "\u0000\u0344\u0345\u0001\u0000\u0000\u0000\u0345\u034d\u0005\u0084\u0000"
+        + "\u0000\u0346\u0347\u0003h4\u0000\u0347\u0348\u0005\u0004\u0000\u0000\u0348"
+        + "\u034a\u0001\u0000\u0000\u0000\u0349\u0346\u0001\u0000\u0000\u0000\u0349"
+        + "\u034a\u0001\u0000\u0000\u0000\u034a\u034b\u0001\u0000\u0000\u0000\u034b"
+        + "\u034d\u0003h4\u0000\u034c\u0343\u0001\u0000\u0000\u0000\u034c\u0349\u0001"
+        + "\u0000\u0000\u0000\u034dk\u0001\u0000\u0000\u0000\u034e\u0351\u0005\u0085"
+        + "\u0000\u0000\u034f\u0351\u0005\u0086\u0000\u0000\u0350\u034e\u0001\u0000"
+        + "\u0000\u0000\u0350\u034f\u0001\u0000\u0000\u0000\u0351m\u0001\u0000\u0000"
+        + "\u0000\u0352\u0356\u0005\u0082\u0000\u0000\u0353\u0356\u0003v;\u0000\u0354"
+        + "\u0356\u0005\u0083\u0000\u0000\u0355\u0352\u0001\u0000\u0000\u0000\u0355"
+        + "\u0353\u0001\u0000\u0000\u0000\u0355\u0354\u0001\u0000\u0000\u0000\u0356"
+        + "o\u0001\u0000\u0000\u0000\u0357\u035a\u0005\u0081\u0000\u0000\u0358\u035a"
+        + "\u0005\u0080\u0000\u0000\u0359\u0357\u0001\u0000\u0000\u0000\u0359\u0358"
+        + "\u0001\u0000\u0000\u0000\u035aq\u0001\u0000\u0000\u0000\u035b\u035c\u0007"
+        + "\u000e\u0000\u0000\u035cs\u0001\u0000\u0000\u0000\u035d\u035e\u0005b\u0000"
+        + "\u0000\u035e\u035f\u00034\u001a\u0000\u035f\u0360\u0005Z\u0000\u0000\u0360"
+        + "\u0361\u00034\u001a\u0000\u0361u\u0001\u0000\u0000\u0000\u0362\u0363\u0007"
+        + "\u000f\u0000\u0000\u0363w\u0001\u0000\u0000\u0000v\u0087\u0089\u008d\u0096"
+        + "\u0098\u009c\u00a4\u00a6\u00aa\u00ae\u00b4\u00b8\u00bd\u00c2\u00c6\u00ca"
+        + "\u00cf\u00d9\u00dd\u00e5\u00e8\u00ee\u00f3\u00f6\u00fb\u00fe\u0100\u0108"
+        + "\u010b\u0117\u011a\u011d\u0124\u012b\u012f\u0133\u0137\u013a\u013e\u0142"
+        + "\u0147\u014b\u0153\u0157\u015a\u0161\u016c\u016f\u0173\u0185\u018a\u018d"
+        + "\u0193\u019a\u01a1\u01a4\u01a8\u01ac\u01b0\u01b2\u01bd\u01c2\u01c5\u01c9"
+        + "\u01cc\u01d2\u01d5\u01db\u01de\u01e0\u01f2\u01f7\u01fa\u021d\u0225\u0227"
+        + "\u022e\u0233\u0236\u023e\u0247\u024d\u0255\u025a\u0260\u0263\u026a\u0272"
+        + "\u0278\u0284\u0286\u0291\u02a0\u02a5\u02a9\u02ad\u02b4\u02ba\u02c6\u02db"
+        + "\u02e9\u02ee\u02f5\u02f8\u02ff\u0308\u031b\u0323\u0327\u032c\u0337\u033e"
+        + "\u0343\u0349\u034c\u0350\u0355\u0359";
     public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray());
     static {
         _decisionToDFA = new DFA[_ATN.getNumberOfDecisions()];
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/Debug.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/Debug.java
index cbc97856682a..56754c48442b 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/Debug.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/Debug.java
@@ -106,7 +106,7 @@ private void handleInfo(ExecutionInfo info, ActionListener<Page> listener) {
                     sb.append(entry.getKey().name());
                     sb.append("***");
                     for (Transformation tf : entry.getValue()) {
-                        sb.append(tf.ruleName());
+                        sb.append(tf.name());
                         sb.append("\n");
                         sb.append(NodeUtils.diffString(tf.before(), tf.after()));
                         sb.append("\n");
@@ -127,7 +127,7 @@ private void handleInfo(ExecutionInfo info, ActionListener<Page> listener) {
                     int counter = 0;
                     for (Transformation tf : entry.getValue()) {
                         if (tf.hasChanged()) {
-                            plans.put(tf.ruleName() + "#" + ++counter, tf.after());
+                            plans.put(tf.name() + "#" + ++counter, tf.after());
                         }
                     }
                 }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/Mapper.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/Mapper.java
index 50012d607699..2f94b5e869a3 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/Mapper.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/Mapper.java
@@ -45,8 +45,8 @@ public PhysicalPlan map(LogicalPlan plan) {
     }
 
     @Override
-    protected Iterable<RuleExecutor<PhysicalPlan>.Batch> batches() {
-        Batch conversion = new Batch("Mapping", new JoinMapper(), new SimpleExecMapper());
+    protected Iterable<RuleExecutor.Batch<PhysicalPlan>> batches() {
+        var conversion = new Batch<>("Mapping", new JoinMapper(), new SimpleExecMapper());
 
         return Arrays.asList(conversion);
     }
@@ -136,7 +136,6 @@ public final PhysicalPlan apply(PhysicalPlan plan) {
         }
 
         @SuppressWarnings("unchecked")
-        @Override
         protected final PhysicalPlan rule(UnplannedExec plan) {
             LogicalPlan subPlan = plan.plan();
             if (subPlanToken.isInstance(subPlan)) {
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java
index ecdc6fe86a7d..e566f1dd0c34 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java
@@ -116,8 +116,8 @@ PhysicalPlan fold(PhysicalPlan plan) {
     }
 
     @Override
-    protected Iterable<RuleExecutor<PhysicalPlan>.Batch> batches() {
-        Batch rollup = new Batch(
+    protected Iterable<RuleExecutor.Batch<PhysicalPlan>> batches() {
+        var rollup = new Batch<>(
             "Fold queries",
             new FoldPivot(),
             new FoldAggregate(),
@@ -127,9 +127,8 @@ protected Iterable<RuleExecutor<PhysicalPlan>.Batch> batches() {
             new FoldLimit()
         );
 
-        Batch local = new Batch("Local queries", new LocalLimit(), new PropagateEmptyLocal());
-
-        Batch finish = new Batch("Finish query", Limiter.ONCE, new PlanOutputToQueryRef());
+        var local = new Batch<>("Local queries", new LocalLimit(), new PropagateEmptyLocal());
+        var finish = new Batch<>("Finish query", Limiter.ONCE, new PlanOutputToQueryRef());
 
         return Arrays.asList(rollup, local, finish);
     }
@@ -943,7 +942,6 @@ public final PhysicalPlan apply(PhysicalPlan plan) {
             return plan.transformUp(typeToken(), this::rule);
         }
 
-        @Override
         protected abstract PhysicalPlan rule(SubPlan plan);
     }
 }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlPlugin.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlPlugin.java
index c09fb9e1090b..7a18a886e298 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlPlugin.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlPlugin.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -99,7 +99,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver expressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
 
         return createComponents(client, environment.settings(), clusterService, namedWriteableRegistry);
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/MatrixStatsAgg.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/MatrixStatsAgg.java
index da4bea6ab89e..546dd06f287a 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/MatrixStatsAgg.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/MatrixStatsAgg.java
@@ -6,12 +6,11 @@
  */
 package org.elasticsearch.xpack.sql.querydsl.agg;
 
+import org.elasticsearch.aggregations.metric.MatrixStatsAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 
 import java.util.List;
 
-import static org.elasticsearch.search.aggregations.matrix.MatrixStatsAggregationBuilders.matrixStats;
-
 public class MatrixStatsAgg extends LeafAgg {
 
     private final List<String> fields;
@@ -23,6 +22,6 @@ public MatrixStatsAgg(String id, List<String> fields) {
 
     @Override
     AggregationBuilder toBuilder() {
-        return matrixStats(id()).fields(fields);
+        return new MatrixStatsAggregationBuilder(id()).fields(fields);
     }
 }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java
index 85b411344989..2b333f41047a 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.ql.rule.RuleExecutor;
 import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer;
+import org.elasticsearch.xpack.sql.analysis.analyzer.AnalyzerContext;
 import org.elasticsearch.xpack.sql.analysis.analyzer.PreAnalyzer;
 import org.elasticsearch.xpack.sql.analysis.analyzer.PreAnalyzer.PreAnalysis;
 import org.elasticsearch.xpack.sql.analysis.analyzer.TableInfo;
@@ -116,12 +117,12 @@ public void analyzedPlan(LogicalPlan parsed, boolean verify, ActionListener<Logi
         }
 
         preAnalyze(parsed, r -> {
-            Analyzer analyzer = new Analyzer(
+            AnalyzerContext context = new AnalyzerContext(
                 configuration,
                 functionRegistry,
-                IndexCompatibility.compatible(r, Version.fromId(configuration.version().id)),
-                verifier
+                IndexCompatibility.compatible(r, Version.fromId(configuration.version().id))
             );
+            Analyzer analyzer = new Analyzer(context, verifier);
             return analyzer.analyze(parsed, verify);
         }, listener);
     }
@@ -133,7 +134,8 @@ public void debugAnalyzedPlan(LogicalPlan parsed, ActionListener<RuleExecutor<Lo
         }
 
         preAnalyze(parsed, r -> {
-            Analyzer analyzer = new Analyzer(configuration, functionRegistry, r, verifier);
+            AnalyzerContext context = new AnalyzerContext(configuration, functionRegistry, r);
+            Analyzer analyzer = new Analyzer(context, verifier);
             return analyzer.debugAnalyze(parsed);
         }, listener);
     }
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/AnalyzerTestUtils.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/AnalyzerTestUtils.java
new file mode 100644
index 000000000000..38e14e9ab401
--- /dev/null
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/AnalyzerTestUtils.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.sql.analysis.analyzer;
+
+import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
+import org.elasticsearch.xpack.ql.index.IndexResolution;
+import org.elasticsearch.xpack.sql.expression.function.SqlFunctionRegistry;
+import org.elasticsearch.xpack.sql.session.SqlConfiguration;
+import org.elasticsearch.xpack.sql.stats.Metrics;
+
+import static org.elasticsearch.xpack.sql.SqlTestUtils.TEST_CFG;
+
+public final class AnalyzerTestUtils {
+
+    private AnalyzerTestUtils() {}
+
+    public static Analyzer analyzer(IndexResolution resolution) {
+        return analyzer(TEST_CFG, new SqlFunctionRegistry(), resolution);
+    }
+
+    public static Analyzer analyzer(IndexResolution resolution, Verifier verifier) {
+        return analyzer(TEST_CFG, new SqlFunctionRegistry(), resolution, verifier);
+    }
+
+    public static Analyzer analyzer(SqlConfiguration configuration, IndexResolution resolution) {
+        return analyzer(configuration, new SqlFunctionRegistry(), resolution);
+    }
+
+    public static Analyzer analyzer(SqlConfiguration configuration, FunctionRegistry registry, IndexResolution resolution) {
+        return analyzer(configuration, registry, resolution, new Verifier(new Metrics()));
+    }
+
+    public static Analyzer analyzer(
+        SqlConfiguration configuration,
+        FunctionRegistry registry,
+        IndexResolution resolution,
+        Verifier verifier
+    ) {
+        return new Analyzer(new AnalyzerContext(configuration, registry, resolution), verifier);
+    }
+}
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/AnalyzerTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/AnalyzerTests.java
index ff68f6f3b11c..bac11297fef8 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/AnalyzerTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/AnalyzerTests.java
@@ -35,13 +35,14 @@
 public class AnalyzerTests extends ESTestCase {
 
     private final SqlParser parser = new SqlParser();
-    private final Analyzer analyzer = new Analyzer(
+    private final AnalyzerContext context = new AnalyzerContext(
         SqlTestUtils.TEST_CFG,
         new SqlFunctionRegistry(),
-        IndexResolution.valid(new EsIndex("test", loadMapping("mapping-basic.json"))),
-        new Verifier(new Metrics())
+        IndexResolution.valid(new EsIndex("test", loadMapping("mapping-basic.json")))
     );
 
+    private final Analyzer analyzer = new Analyzer(context, new Verifier(new Metrics()));
+
     private LogicalPlan analyze(String sql) {
         return analyzer.analyze(parser.createStatement(sql), false);
     }
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java
index fa7151e1f875..41518ce093d2 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java
@@ -76,7 +76,7 @@ public FieldAttributeTests() {
 
         EsIndex test = new EsIndex("test", mapping);
         getIndexResult = IndexResolution.valid(test);
-        analyzer = new Analyzer(SqlTestUtils.TEST_CFG, functionRegistry, getIndexResult, verifier);
+        analyzer = analyzer(functionRegistry, getIndexResult, verifier);
     }
 
     private LogicalPlan plan(String sql) {
@@ -197,7 +197,7 @@ public void testFieldAmbiguity() {
 
         EsIndex index = new EsIndex("test", mapping);
         getIndexResult = IndexResolution.valid(index);
-        analyzer = new Analyzer(SqlTestUtils.TEST_CFG, functionRegistry, getIndexResult, verifier);
+        analyzer = analyzer(functionRegistry, getIndexResult, verifier);
 
         VerificationException ex = expectThrows(VerificationException.class, () -> plan("SELECT test.bar FROM test"));
         assertEquals(
@@ -232,7 +232,7 @@ public void testAggregations() {
         Map<String, EsField> mapping = TypesTests.loadMapping("mapping-basic.json");
         EsIndex index = new EsIndex("test", mapping);
         getIndexResult = IndexResolution.valid(index);
-        analyzer = new Analyzer(SqlTestUtils.TEST_CFG, functionRegistry, getIndexResult, verifier);
+        analyzer = analyzer(functionRegistry, getIndexResult, verifier);
 
         LogicalPlan plan = plan("SELECT sum(salary) AS s FROM test");
         assertThat(plan, instanceOf(Aggregate.class));
@@ -265,7 +265,7 @@ public void testGroupByAmbiguity() {
         Map<String, EsField> mapping = TypesTests.loadMapping("mapping-basic.json");
         EsIndex index = new EsIndex("test", mapping);
         getIndexResult = IndexResolution.valid(index);
-        analyzer = new Analyzer(SqlTestUtils.TEST_CFG, functionRegistry, getIndexResult, verifier);
+        analyzer = analyzer(functionRegistry, getIndexResult, verifier);
 
         VerificationException ex = expectThrows(
             VerificationException.class,
@@ -323,7 +323,7 @@ public void testUnsignedLongVersionCompatibility() {
         SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(SqlVersion.fromId(preUnsignedLong.id));
 
         for (String sql : List.of(query, queryWithLiteral, queryWithCastLiteral, queryWithAlias, queryWithArithmetic, queryWithCast)) {
-            analyzer = new Analyzer(
+            analyzer = analyzer(
                 sqlConfig,
                 functionRegistry,
                 loadCompatibleIndexResolution("mapping-numeric.json", preUnsignedLong),
@@ -333,7 +333,7 @@ public void testUnsignedLongVersionCompatibility() {
             assertThat(ex.getMessage(), containsString("Found 1 problem\nline 1:8: Cannot use field [unsigned_long]"));
 
             for (Version v : List.of(INTRODUCING_UNSIGNED_LONG, postUnsignedLong)) {
-                analyzer = new Analyzer(
+                analyzer = analyzer(
                     SqlTestUtils.randomConfiguration(SqlVersion.fromId(v.id)),
                     functionRegistry,
                     loadCompatibleIndexResolution("mapping-numeric.json", v),
@@ -362,7 +362,7 @@ public void testVersionTypeVersionCompatibility() {
         SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(SqlVersion.fromId(preVersion.id));
 
         for (String sql : List.of(query, queryWithCastLiteral, queryWithAlias, queryWithCast)) {
-            analyzer = new Analyzer(
+            analyzer = analyzer(
                 sqlConfig,
                 functionRegistry,
                 loadCompatibleIndexResolution("mapping-version.json", preVersion),
@@ -372,7 +372,7 @@ public void testVersionTypeVersionCompatibility() {
             assertThat(ex.getMessage(), containsString("Cannot use field [version_number]"));
 
             for (Version v : List.of(INTRODUCING_VERSION_FIELD_TYPE, postVersion)) {
-                analyzer = new Analyzer(
+                analyzer = analyzer(
                     SqlTestUtils.randomConfiguration(SqlVersion.fromId(v.id)),
                     functionRegistry,
                     loadCompatibleIndexResolution("mapping-version.json", v),
@@ -393,7 +393,7 @@ public void testVersionTypeVersionCompatibility() {
     public void testNonProjectedUnsignedLongVersionCompatibility() {
         Version preUnsignedLong = Version.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER);
         SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(SqlVersion.fromId(preUnsignedLong.id));
-        analyzer = new Analyzer(
+        analyzer = analyzer(
             sqlConfig,
             functionRegistry,
             loadCompatibleIndexResolution("mapping-numeric.json", preUnsignedLong),
@@ -427,7 +427,7 @@ public void testNestedUnsignedLongVersionCompatibility() {
         String sql = "SELECT container.ul as unsigned_long FROM test";
 
         Version preUnsignedLong = Version.fromId(INTRODUCING_UNSIGNED_LONG.id - SqlVersion.MINOR_MULTIPLIER);
-        analyzer = new Analyzer(
+        analyzer = analyzer(
             SqlTestUtils.randomConfiguration(SqlVersion.fromId(preUnsignedLong.id)),
             functionRegistry,
             compatibleIndexResolution(props, preUnsignedLong),
@@ -438,7 +438,7 @@ public void testNestedUnsignedLongVersionCompatibility() {
 
         Version postUnsignedLong = Version.fromId(INTRODUCING_UNSIGNED_LONG.id + SqlVersion.MINOR_MULTIPLIER);
         for (Version v : List.of(INTRODUCING_UNSIGNED_LONG, postUnsignedLong)) {
-            analyzer = new Analyzer(
+            analyzer = analyzer(
                 SqlTestUtils.randomConfiguration(SqlVersion.fromId(v.id)),
                 functionRegistry,
                 compatibleIndexResolution(props, v),
@@ -463,7 +463,7 @@ public void testUnsignedLongStarExpandedVersionControlled() {
         for (SqlVersion version : List.of(preUnsignedLong, SqlVersion.fromId(INTRODUCING_UNSIGNED_LONG.id), postUnsignedLong)) {
             SqlConfiguration config = SqlTestUtils.randomConfiguration(version);
             // the mapping is mutated when making it "compatible", so it needs to be reloaded inside the loop.
-            analyzer = new Analyzer(
+            analyzer = analyzer(
                 config,
                 functionRegistry,
                 loadCompatibleIndexResolution("mapping-numeric.json", Version.fromId(version.id)),
@@ -481,7 +481,7 @@ public void testUnsignedLongStarExpandedVersionControlled() {
     }
 
     public void testFunctionOverNonExistingFieldAsArgumentAndSameAlias() throws Exception {
-        analyzer = new Analyzer(SqlTestUtils.TEST_CFG, functionRegistry, loadIndexResolution("mapping-basic.json"), verifier);
+        analyzer = analyzer(SqlTestUtils.TEST_CFG, functionRegistry, loadIndexResolution("mapping-basic.json"), verifier);
 
         VerificationException ex = expectThrows(
             VerificationException.class,
@@ -491,7 +491,7 @@ public void testFunctionOverNonExistingFieldAsArgumentAndSameAlias() throws Exce
     }
 
     public void testFunctionWithExpressionOverNonExistingFieldAsArgumentAndSameAlias() throws Exception {
-        analyzer = new Analyzer(SqlTestUtils.TEST_CFG, functionRegistry, loadIndexResolution("mapping-basic.json"), verifier);
+        analyzer = analyzer(SqlTestUtils.TEST_CFG, functionRegistry, loadIndexResolution("mapping-basic.json"), verifier);
 
         VerificationException ex = expectThrows(
             VerificationException.class,
@@ -503,7 +503,7 @@ public void testFunctionWithExpressionOverNonExistingFieldAsArgumentAndSameAlias
     public void testExpandStarOnIndexWithoutColumns() {
         EsIndex test = new EsIndex("test", Collections.emptyMap());
         getIndexResult = IndexResolution.valid(test);
-        analyzer = new Analyzer(SqlTestUtils.TEST_CFG, functionRegistry, getIndexResult, verifier);
+        analyzer = analyzer(SqlTestUtils.TEST_CFG, functionRegistry, getIndexResult, verifier);
 
         LogicalPlan plan = plan("SELECT * FROM test");
 
@@ -529,4 +529,17 @@ private static IndexResolution compatibleIndexResolution(String properties, Vers
         EsIndex index = new EsIndex("test", mapping);
         return IndexCompatibility.compatible(IndexResolution.valid(index), version);
     }
+
+    private static Analyzer analyzer(
+        SqlConfiguration configuration,
+        FunctionRegistry functionRegistry,
+        IndexResolution resolution,
+        Verifier verifier
+    ) {
+        return new Analyzer(new AnalyzerContext(configuration, functionRegistry, resolution), verifier);
+    }
+
+    private static Analyzer analyzer(FunctionRegistry functionRegistry, IndexResolution resolution, Verifier verifier) {
+        return analyzer(SqlTestUtils.TEST_CFG, functionRegistry, resolution, verifier);
+    }
 }
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java
index 480f955b68f2..6fb3122fb0eb 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.ql.type.EsField;
 import org.elasticsearch.xpack.sql.analysis.index.IndexResolverTests;
-import org.elasticsearch.xpack.sql.expression.function.SqlFunctionRegistry;
 import org.elasticsearch.xpack.sql.expression.function.aggregate.First;
 import org.elasticsearch.xpack.sql.expression.function.aggregate.Last;
 import org.elasticsearch.xpack.sql.expression.function.scalar.math.Round;
@@ -25,7 +24,6 @@
 import org.elasticsearch.xpack.sql.expression.predicate.conditional.Least;
 import org.elasticsearch.xpack.sql.expression.predicate.conditional.NullIf;
 import org.elasticsearch.xpack.sql.parser.SqlParser;
-import org.elasticsearch.xpack.sql.stats.Metrics;
 
 import java.util.Arrays;
 import java.util.HashMap;
@@ -39,7 +37,7 @@
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.OBJECT;
-import static org.elasticsearch.xpack.sql.SqlTestUtils.TEST_CFG;
+import static org.elasticsearch.xpack.sql.analysis.analyzer.AnalyzerTestUtils.analyzer;
 import static org.elasticsearch.xpack.sql.types.SqlTypesTests.loadMapping;
 
 public class VerifierErrorMessagesTests extends ESTestCase {
@@ -54,7 +52,7 @@ private String error(String sql) {
     }
 
     private String error(IndexResolution getIndexResult, String sql) {
-        Analyzer analyzer = new Analyzer(TEST_CFG, new SqlFunctionRegistry(), getIndexResult, new Verifier(new Metrics()));
+        Analyzer analyzer = analyzer(getIndexResult);
         VerificationException e = expectThrows(VerificationException.class, () -> analyzer.analyze(parser.createStatement(sql), true));
         String message = e.getMessage();
         assertTrue(message.startsWith("Found "));
@@ -74,7 +72,7 @@ private EsIndex getTestEsIndex() {
     }
 
     private LogicalPlan accept(IndexResolution resolution, String sql) {
-        Analyzer analyzer = new Analyzer(TEST_CFG, new SqlFunctionRegistry(), resolution, new Verifier(new Metrics()));
+        Analyzer analyzer = analyzer(resolution);
         return analyzer.analyze(parser.createStatement(sql), true);
     }
 
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursorTests.java
index e8dd1fd03195..5126aba35688 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursorTests.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.sql.execution.search.extractor.CompositeKeyExtractorTests;
 import org.elasticsearch.xpack.sql.execution.search.extractor.MetricAggExtractorTests;
 
-import java.io.IOException;
 import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.BitSet;
@@ -49,7 +48,7 @@ static BucketExtractor randomBucketExtractor(ZoneId zoneId) {
     }
 
     @Override
-    protected CompositeAggCursor mutateInstance(CompositeAggCursor instance) throws IOException {
+    protected CompositeAggCursor mutateInstance(CompositeAggCursor instance) {
         return new CompositeAggCursor(
             instance.next(),
             instance.extractors(),
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/SearchHitCursorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/SearchHitCursorTests.java
index fa0c2005f023..a1d1ad7a5fd0 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/SearchHitCursorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/SearchHitCursorTests.java
@@ -49,7 +49,7 @@ static HitExtractor randomHitExtractor(int depth) {
     }
 
     @Override
-    protected SearchHitCursor mutateInstance(SearchHitCursor instance) throws IOException {
+    protected SearchHitCursor mutateInstance(SearchHitCursor instance) {
         return new SearchHitCursor(
             instance.next(),
             instance.extractors(),
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingExtractorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingExtractorTests.java
index bdc913095548..d06a239e61ce 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingExtractorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingExtractorTests.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor;
 import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Supplier;
@@ -64,7 +63,7 @@ protected Reader<ComputingExtractor> instanceReader() {
     }
 
     @Override
-    protected ComputingExtractor mutateInstance(ComputingExtractor instance) throws IOException {
+    protected ComputingExtractor mutateInstance(ComputingExtractor instance) {
         return new ComputingExtractor(
             randomValueOtherThan(instance.processor(), () -> randomProcessor()),
             randomValueOtherThan(instance.hitName(), () -> randomAlphaOfLength(10))
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/MetricAggExtractorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/MetricAggExtractorTests.java
index e0539e2bfcff..32de6cd30396 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/MetricAggExtractorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/MetricAggExtractorTests.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.xpack.sql.type.SqlDataTypes;
 import org.elasticsearch.xpack.sql.util.DateUtils;
 
-import java.io.IOException;
 import java.time.ZoneId;
 
 import static java.util.Collections.emptyList;
@@ -65,7 +64,7 @@ protected ZoneId instanceZoneId(MetricAggExtractor instance) {
     }
 
     @Override
-    protected MetricAggExtractor mutateInstance(MetricAggExtractor instance) throws IOException {
+    protected MetricAggExtractor mutateInstance(MetricAggExtractor instance) {
         return new MetricAggExtractor(
             instance.name() + "mutated",
             instance.property() + "mutated",
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/CastProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/CastProcessorTests.java
index 4e191ffe1c46..b8fa3e4def77 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/CastProcessorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/CastProcessorTests.java
@@ -13,8 +13,6 @@
 import org.elasticsearch.xpack.ql.type.DataTypeConverter.DefaultConverter;
 import org.elasticsearch.xpack.sql.type.SqlDataTypeConverter.SqlConverter;
 
-import java.io.IOException;
-
 public class CastProcessorTests extends AbstractWireSerializingTestCase<CastProcessor> {
     public static CastProcessor randomCastProcessor() {
         return new CastProcessor(randomFrom(SqlConverter.values()));
@@ -31,7 +29,7 @@ protected Reader<CastProcessor> instanceReader() {
     }
 
     @Override
-    protected CastProcessor mutateInstance(CastProcessor instance) throws IOException {
+    protected CastProcessor mutateInstance(CastProcessor instance) {
         return new CastProcessor(randomValueOtherThan(instance.converter(), () -> randomFrom(SqlConverter.values())));
     }
 
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java
index fb3a42aca0db..e62eb570f9a8 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java
@@ -15,15 +15,14 @@
 import org.elasticsearch.xpack.ql.plan.logical.Project;
 import org.elasticsearch.xpack.sql.action.Protocol;
 import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer;
-import org.elasticsearch.xpack.sql.analysis.analyzer.Verifier;
-import org.elasticsearch.xpack.sql.expression.function.SqlFunctionRegistry;
 import org.elasticsearch.xpack.sql.parser.SqlParser;
 import org.elasticsearch.xpack.sql.proto.Mode;
 import org.elasticsearch.xpack.sql.session.SqlConfiguration;
-import org.elasticsearch.xpack.sql.stats.Metrics;
 import org.elasticsearch.xpack.sql.types.SqlTypesTests;
 import org.elasticsearch.xpack.sql.util.DateUtils;
 
+import static org.elasticsearch.xpack.sql.analysis.analyzer.AnalyzerTestUtils.analyzer;
+
 public class DatabaseFunctionTests extends ESTestCase {
 
     public void testDatabaseFunctionOutput() {
@@ -49,7 +48,7 @@ public void testDatabaseFunctionOutput() {
             null,
             randomBoolean()
         );
-        Analyzer analyzer = new Analyzer(sqlConfig, new SqlFunctionRegistry(), IndexResolution.valid(test), new Verifier(new Metrics()));
+        Analyzer analyzer = analyzer(sqlConfig, IndexResolution.valid(test));
 
         Project result = (Project) analyzer.analyze(parser.createStatement("SELECT DATABASE()"), true);
         NamedExpression ne = result.projections().get(0);
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java
index 21a73f1b445a..a939b3c5540a 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java
@@ -15,15 +15,14 @@
 import org.elasticsearch.xpack.ql.plan.logical.Project;
 import org.elasticsearch.xpack.sql.action.Protocol;
 import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer;
-import org.elasticsearch.xpack.sql.analysis.analyzer.Verifier;
-import org.elasticsearch.xpack.sql.expression.function.SqlFunctionRegistry;
 import org.elasticsearch.xpack.sql.parser.SqlParser;
 import org.elasticsearch.xpack.sql.proto.Mode;
 import org.elasticsearch.xpack.sql.session.SqlConfiguration;
-import org.elasticsearch.xpack.sql.stats.Metrics;
 import org.elasticsearch.xpack.sql.types.SqlTypesTests;
 import org.elasticsearch.xpack.sql.util.DateUtils;
 
+import static org.elasticsearch.xpack.sql.analysis.analyzer.AnalyzerTestUtils.analyzer;
+
 public class UserFunctionTests extends ESTestCase {
 
     public void testNoUsernameFunctionOutput() {
@@ -48,7 +47,7 @@ public void testNoUsernameFunctionOutput() {
             null,
             randomBoolean()
         );
-        Analyzer analyzer = new Analyzer(sqlConfig, new SqlFunctionRegistry(), IndexResolution.valid(test), new Verifier(new Metrics()));
+        Analyzer analyzer = analyzer(sqlConfig, IndexResolution.valid(test));
 
         Project result = (Project) analyzer.analyze(parser.createStatement("SELECT USER()"), true);
         NamedExpression ne = result.projections().get(0);
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java
index 090794b86941..c14daee2c143 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java
@@ -15,11 +15,8 @@
 import org.elasticsearch.xpack.ql.tree.AbstractNodeTestCase;
 import org.elasticsearch.xpack.sql.SqlTestUtils;
 import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer;
-import org.elasticsearch.xpack.sql.analysis.analyzer.Verifier;
-import org.elasticsearch.xpack.sql.expression.function.SqlFunctionRegistry;
 import org.elasticsearch.xpack.sql.parser.ParsingException;
 import org.elasticsearch.xpack.sql.parser.SqlParser;
-import org.elasticsearch.xpack.sql.stats.Metrics;
 import org.elasticsearch.xpack.sql.types.SqlTypesTests;
 
 import java.time.ZoneId;
@@ -29,6 +26,7 @@
 
 import static org.elasticsearch.xpack.ql.tree.Source.EMPTY;
 import static org.elasticsearch.xpack.sql.SqlTestUtils.literal;
+import static org.elasticsearch.xpack.sql.analysis.analyzer.AnalyzerTestUtils.analyzer;
 
 public class CurrentDateTimeTests extends AbstractNodeTestCase<CurrentDateTime, Expression> {
 
@@ -93,7 +91,7 @@ public void testInvalidPrecision() {
             new EsIndex("test", SqlTypesTests.loadMapping("mapping-multi-field-with-nested.json"))
         );
 
-        Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), indexResolution, new Verifier(new Metrics()));
+        Analyzer analyzer = analyzer(indexResolution);
         ParsingException e = expectThrows(
             ParsingException.class,
             () -> analyzer.analyze(parser.createStatement("SELECT CURRENT_TIMESTAMP(100000000000000)"), true)
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentTimeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentTimeTests.java
index 4c845728f1c2..6f4f9a646e71 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentTimeTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentTimeTests.java
@@ -15,11 +15,8 @@
 import org.elasticsearch.xpack.ql.tree.AbstractNodeTestCase;
 import org.elasticsearch.xpack.sql.SqlTestUtils;
 import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer;
-import org.elasticsearch.xpack.sql.analysis.analyzer.Verifier;
-import org.elasticsearch.xpack.sql.expression.function.SqlFunctionRegistry;
 import org.elasticsearch.xpack.sql.parser.ParsingException;
 import org.elasticsearch.xpack.sql.parser.SqlParser;
-import org.elasticsearch.xpack.sql.stats.Metrics;
 import org.elasticsearch.xpack.sql.types.SqlTypesTests;
 
 import java.time.OffsetTime;
@@ -30,6 +27,7 @@
 
 import static org.elasticsearch.xpack.ql.tree.Source.EMPTY;
 import static org.elasticsearch.xpack.sql.SqlTestUtils.literal;
+import static org.elasticsearch.xpack.sql.analysis.analyzer.AnalyzerTestUtils.analyzer;
 
 public class CurrentTimeTests extends AbstractNodeTestCase<CurrentTime, Expression> {
 
@@ -94,7 +92,7 @@ public void testInvalidPrecision() {
             new EsIndex("test", SqlTypesTests.loadMapping("mapping-multi-field-with-nested.json"))
         );
 
-        Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), indexResolution, new Verifier(new Metrics()));
+        Analyzer analyzer = analyzer(indexResolution);
         ParsingException e = expectThrows(
             ParsingException.class,
             () -> analyzer.analyze(parser.createStatement("SELECT CURRENT_TIME(100000000000000)"), true)
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeDateFormatProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeDateFormatProcessorTests.java
index a44f641a5ff4..459b7758f18a 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeDateFormatProcessorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeDateFormatProcessorTests.java
@@ -10,6 +10,7 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.core.PathUtils;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.ql.tree.Source;
 
@@ -94,7 +95,7 @@ public void test() {
             .makePipe()
             .asProcessor()
             .process(null);
-        assertEquals(String.format(Locale.ROOT, """
+        assertEquals(Strings.format("""
 
             Testfile: %s
             Line number: %s
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NamedDateTimeProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NamedDateTimeProcessorTests.java
index c4bb7e4b5bf7..c03f853e72f8 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NamedDateTimeProcessorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NamedDateTimeProcessorTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.NamedDateTimeProcessor.NameExtractor;
 import org.junit.Assume;
 
-import java.io.IOException;
 import java.time.ZoneId;
 
 import static org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeTestUtils.dateTime;
@@ -35,7 +34,7 @@ protected Reader<NamedDateTimeProcessor> instanceReader() {
     }
 
     @Override
-    protected NamedDateTimeProcessor mutateInstance(NamedDateTimeProcessor instance) throws IOException {
+    protected NamedDateTimeProcessor mutateInstance(NamedDateTimeProcessor instance) {
         NameExtractor replaced = randomValueOtherThan(instance.extractor(), () -> randomFrom(NameExtractor.values()));
         return new NamedDateTimeProcessor(replaced, UTC);
     }
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NonIsoDateTimeProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NonIsoDateTimeProcessorTests.java
index 953a0709467b..3845e0cfb325 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NonIsoDateTimeProcessorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/NonIsoDateTimeProcessorTests.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.xpack.sql.AbstractSqlWireSerializingTestCase;
 import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.NonIsoDateTimeProcessor.NonIsoDateTimeExtractor;
 
-import java.io.IOException;
 import java.time.ZoneId;
 
 import static org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeTestUtils.dateTime;
@@ -33,7 +32,7 @@ protected Reader<NonIsoDateTimeProcessor> instanceReader() {
     }
 
     @Override
-    protected NonIsoDateTimeProcessor mutateInstance(NonIsoDateTimeProcessor instance) throws IOException {
+    protected NonIsoDateTimeProcessor mutateInstance(NonIsoDateTimeProcessor instance) {
         NonIsoDateTimeExtractor replaced = randomValueOtherThan(instance.extractor(), () -> randomFrom(NonIsoDateTimeExtractor.values()));
         return new NonIsoDateTimeProcessor(replaced, UTC);
     }
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/geo/GeoProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/geo/GeoProcessorTests.java
index ddd3fc268f6b..b628a5565215 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/geo/GeoProcessorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/geo/GeoProcessorTests.java
@@ -12,8 +12,6 @@
 import org.elasticsearch.xpack.sql.expression.function.scalar.geo.GeoProcessor.GeoOperation;
 import org.elasticsearch.xpack.sql.expression.literal.geo.GeoShape;
 
-import java.io.IOException;
-
 public class GeoProcessorTests extends AbstractWireSerializingTestCase<GeoProcessor> {
     public static GeoProcessor randomGeoProcessor() {
         return new GeoProcessor(randomFrom(GeoOperation.values()));
@@ -30,7 +28,7 @@ protected Reader<GeoProcessor> instanceReader() {
     }
 
     @Override
-    protected GeoProcessor mutateInstance(GeoProcessor instance) throws IOException {
+    protected GeoProcessor mutateInstance(GeoProcessor instance) {
         return new GeoProcessor(randomValueOtherThan(instance.processor(), () -> randomFrom(GeoOperation.values())));
     }
 
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/geo/StDistanceProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/geo/StDistanceProcessorTests.java
index 2ff801e24496..3d7a49759d52 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/geo/StDistanceProcessorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/geo/StDistanceProcessorTests.java
@@ -32,6 +32,11 @@ public StDistanceProcessor createTestInstance() {
         );
     }
 
+    @Override
+    protected StDistanceProcessor mutateInstance(StDistanceProcessor instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     public static Processor constantPoint(double lon, double lat) {
         return new ChainingProcessor(new ConstantProcessor("point (" + lon + " " + lat + ")"), StWkttosqlProcessor.INSTANCE);
     }
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/BinaryMathProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/BinaryMathProcessorTests.java
index 1bd0c74e1440..e0b151f7c953 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/BinaryMathProcessorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/BinaryMathProcessorTests.java
@@ -32,6 +32,11 @@ protected BinaryMathProcessor createTestInstance() {
         return randomProcessor();
     }
 
+    @Override
+    protected BinaryMathProcessor mutateInstance(BinaryMathProcessor instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<BinaryMathProcessor> instanceReader() {
         return BinaryMathProcessor::new;
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathFunctionProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathFunctionProcessorTests.java
index a21b6e33443d..1b929ad4fa90 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathFunctionProcessorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/MathFunctionProcessorTests.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
 import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation;
 
-import java.io.IOException;
 import java.math.BigInteger;
 import java.util.Arrays;
 
@@ -31,7 +30,7 @@ protected Reader<MathProcessor> instanceReader() {
     }
 
     @Override
-    protected MathProcessor mutateInstance(MathProcessor instance) throws IOException {
+    protected MathProcessor mutateInstance(MathProcessor instance) {
         return new MathProcessor(randomValueOtherThan(instance.processor(), () -> randomFrom(MathOperation.values())));
     }
 
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringNumericProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringNumericProcessorTests.java
index eb8d365c8f17..c0e436cc4890 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringNumericProcessorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringNumericProcessorTests.java
@@ -29,6 +29,11 @@ protected BinaryStringNumericProcessor createTestInstance() {
         );
     }
 
+    @Override
+    protected BinaryStringNumericProcessor mutateInstance(BinaryStringNumericProcessor instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<BinaryStringNumericProcessor> instanceReader() {
         return BinaryStringNumericProcessor::new;
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringStringProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringStringProcessorTests.java
index f03fccbc336f..272ad4bc45d0 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringStringProcessorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BinaryStringStringProcessorTests.java
@@ -29,6 +29,11 @@ protected BinaryStringStringProcessor createTestInstance() {
         );
     }
 
+    @Override
+    protected BinaryStringStringProcessor mutateInstance(BinaryStringStringProcessor instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<BinaryStringStringProcessor> instanceReader() {
         return BinaryStringStringProcessor::new;
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ConcatProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ConcatProcessorTests.java
index b7d524e9c8e8..3f5699dab716 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ConcatProcessorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ConcatProcessorTests.java
@@ -27,6 +27,11 @@ protected ConcatFunctionProcessor createTestInstance() {
         );
     }
 
+    @Override
+    protected ConcatFunctionProcessor mutateInstance(ConcatFunctionProcessor instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<ConcatFunctionProcessor> instanceReader() {
         return ConcatFunctionProcessor::new;
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/InsertProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/InsertProcessorTests.java
index df3c801efa64..b938876fc981 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/InsertProcessorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/InsertProcessorTests.java
@@ -29,6 +29,11 @@ protected InsertFunctionProcessor createTestInstance() {
         );
     }
 
+    @Override
+    protected InsertFunctionProcessor mutateInstance(InsertFunctionProcessor instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<InsertFunctionProcessor> instanceReader() {
         return InsertFunctionProcessor::new;
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LocateProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LocateProcessorTests.java
index 01223940bd28..3b8098978f01 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LocateProcessorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LocateProcessorTests.java
@@ -32,6 +32,11 @@ protected LocateFunctionProcessor createTestInstance() {
         );
     }
 
+    @Override
+    protected LocateFunctionProcessor mutateInstance(LocateFunctionProcessor instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<LocateFunctionProcessor> instanceReader() {
         return LocateFunctionProcessor::new;
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ReplaceProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ReplaceProcessorTests.java
index 0961587a3e63..83f20cfd77ee 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ReplaceProcessorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/ReplaceProcessorTests.java
@@ -28,6 +28,11 @@ protected ReplaceFunctionProcessor createTestInstance() {
         );
     }
 
+    @Override
+    protected ReplaceFunctionProcessor mutateInstance(ReplaceFunctionProcessor instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<ReplaceFunctionProcessor> instanceReader() {
         return ReplaceFunctionProcessor::new;
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java
index af839ff7e78d..c1dc98571999 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
 import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor.StringOperation;
 
-import java.io.IOException;
 import java.util.Locale;
 
 public class StringFunctionProcessorTests extends AbstractWireSerializingTestCase<StringProcessor> {
@@ -30,7 +29,7 @@ protected Reader<StringProcessor> instanceReader() {
     }
 
     @Override
-    protected StringProcessor mutateInstance(StringProcessor instance) throws IOException {
+    protected StringProcessor mutateInstance(StringProcessor instance) {
         return new StringProcessor(randomValueOtherThan(instance.processor(), () -> randomFrom(StringOperation.values())));
     }
 
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/SubstringProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/SubstringProcessorTests.java
index b3d3f2ce6a5f..2ce90e121b0e 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/SubstringProcessorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/SubstringProcessorTests.java
@@ -28,6 +28,11 @@ protected SubstringFunctionProcessor createTestInstance() {
         );
     }
 
+    @Override
+    protected SubstringFunctionProcessor mutateInstance(SubstringFunctionProcessor instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected Reader<SubstringFunctionProcessor> instanceReader() {
         return SubstringFunctionProcessor::new;
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/literal/interval/IntervalsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/literal/interval/IntervalsTests.java
index e3505e2b6f85..d8238201744a 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/literal/interval/IntervalsTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/literal/interval/IntervalsTests.java
@@ -9,6 +9,7 @@
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.ql.ParsingException;
 import org.elasticsearch.xpack.sql.expression.literal.interval.Intervals.TimeUnit;
@@ -86,7 +87,7 @@ public void testSecondInterval() throws Exception {
         int randomSeconds = randomNonNegativeInt();
         int randomMillis = randomBoolean() ? (randomBoolean() ? 0 : 999) : randomInt(999);
         String value = format(Locale.ROOT, "%s%d", sign, randomSeconds);
-        value += randomMillis > 0 ? formatted(".%03d", randomMillis) : "";
+        value += randomMillis > 0 ? Strings.format(".%03d", randomMillis) : "";
         TemporalAmount amount = parseInterval(EMPTY, value, INTERVAL_SECOND);
         assertEquals(maybeNegate(sign, Duration.ofSeconds(randomSeconds).plusMillis(randomMillis)), amount);
     }
@@ -131,7 +132,7 @@ public void testDayToSecond() throws Exception {
 
         boolean withMillis = randomBoolean();
         int randomMilli = withMillis ? randomInt(999) : 0;
-        String millisString = withMillis && randomMilli > 0 ? formatted(".%03d", randomMilli) : "";
+        String millisString = withMillis && randomMilli > 0 ? Strings.format(".%03d", randomMilli) : "";
 
         String value = format(Locale.ROOT, "%s%d %d:%d:%d%s", sign, randomDay, randomHour, randomMinute, randomSecond, millisString);
         TemporalAmount amount = parseInterval(EMPTY, value, INTERVAL_DAY_TO_SECOND);
@@ -159,7 +160,7 @@ public void testHourToSecond() throws Exception {
 
         boolean withMillis = randomBoolean();
         int randomMilli = withMillis ? randomInt(999) : 0;
-        String millisString = withMillis && randomMilli > 0 ? formatted(".%03d", randomMilli) : "";
+        String millisString = withMillis && randomMilli > 0 ? Strings.format(".%03d", randomMilli) : "";
 
         String value = format(Locale.ROOT, "%s%d:%d:%d%s", sign, randomHour, randomMinute, randomSecond, millisString);
         TemporalAmount amount = parseInterval(EMPTY, value, INTERVAL_HOUR_TO_SECOND);
@@ -175,7 +176,7 @@ public void testMinuteToSecond() throws Exception {
 
         boolean withMillis = randomBoolean();
         int randomMilli = withMillis ? randomInt(999) : 0;
-        String millisString = withMillis && randomMilli > 0 ? formatted(".%03d", randomMilli) : "";
+        String millisString = withMillis && randomMilli > 0 ? Strings.format(".%03d", randomMilli) : "";
 
         String value = format(Locale.ROOT, "%s%d:%d%s", sign, randomMinute, randomSecond, millisString);
         TemporalAmount amount = parseInterval(EMPTY, value, INTERVAL_MINUTE_TO_SECOND);
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerRunTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerRunTests.java
index c56a2eaeae34..9ce49721ba2a 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerRunTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerRunTests.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.xpack.ql.expression.FieldAttribute;
 import org.elasticsearch.xpack.ql.expression.Literal;
 import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute;
-import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan;
@@ -28,11 +27,8 @@
 import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.ql.plan.logical.UnaryPlan;
 import org.elasticsearch.xpack.ql.type.EsField;
-import org.elasticsearch.xpack.sql.SqlTestUtils;
 import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer;
-import org.elasticsearch.xpack.sql.analysis.analyzer.Verifier;
 import org.elasticsearch.xpack.sql.parser.SqlParser;
-import org.elasticsearch.xpack.sql.stats.Metrics;
 import org.elasticsearch.xpack.sql.types.SqlTypesTests;
 
 import java.time.ZonedDateTime;
@@ -48,12 +44,12 @@
 import static org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor.BinaryComparisonOperation.LTE;
 import static org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor.BinaryComparisonOperation.NEQ;
 import static org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor.BinaryComparisonOperation.NULLEQ;
+import static org.elasticsearch.xpack.sql.analysis.analyzer.AnalyzerTestUtils.analyzer;
 
 public class OptimizerRunTests extends ESTestCase {
 
     private final SqlParser parser;
     private final IndexResolution getIndexResult;
-    private final FunctionRegistry functionRegistry;
     private final Analyzer analyzer;
     private final Optimizer optimizer;
     private static final Map<String, Class<? extends BinaryComparison>> COMPARISONS = new HashMap<>() {
@@ -71,13 +67,12 @@ public class OptimizerRunTests extends ESTestCase {
 
     public OptimizerRunTests() {
         parser = new SqlParser();
-        functionRegistry = new FunctionRegistry();
 
         Map<String, EsField> mapping = SqlTypesTests.loadMapping("mapping-multi-field-variation.json");
 
         EsIndex test = new EsIndex("test", mapping);
         getIndexResult = IndexResolution.valid(test);
-        analyzer = new Analyzer(SqlTestUtils.TEST_CFG, functionRegistry, getIndexResult, new Verifier(new Metrics()));
+        analyzer = analyzer(getIndexResult);
         optimizer = new Optimizer();
     }
 
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java
index 4cb717b501ce..a7a36fe65ae5 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java
@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.xpack.sql.parser;
 
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.ql.expression.Expression;
 import org.elasticsearch.xpack.ql.expression.Literal;
@@ -119,7 +120,7 @@ private Limit limit(int limit) {
     }
 
     private LikePattern likeEscape(String like, String character) {
-        Expression exp = parser.createExpression(formatted("exp LIKE '%s' ", like) + buildExpression("escape", "'%s'", character));
+        Expression exp = parser.createExpression(Strings.format("exp LIKE '%s' ", like) + buildExpression("escape", "'%s'", character));
         assertThat(exp, instanceOf(Like.class));
         return ((Like) exp).pattern();
     }
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java
index fe903ff45571..df14cb132af9 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
 import org.elasticsearch.xpack.ql.index.EsIndex;
 import org.elasticsearch.xpack.ql.index.IndexCompatibility;
 import org.elasticsearch.xpack.ql.index.IndexResolution;
@@ -18,7 +17,6 @@
 import org.elasticsearch.xpack.ql.type.EsField;
 import org.elasticsearch.xpack.sql.action.Protocol;
 import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer;
-import org.elasticsearch.xpack.sql.analysis.analyzer.Verifier;
 import org.elasticsearch.xpack.sql.parser.SqlParser;
 import org.elasticsearch.xpack.sql.plan.logical.command.Command;
 import org.elasticsearch.xpack.sql.proto.Mode;
@@ -28,7 +26,6 @@
 import org.elasticsearch.xpack.sql.session.SchemaRowSet;
 import org.elasticsearch.xpack.sql.session.SqlConfiguration;
 import org.elasticsearch.xpack.sql.session.SqlSession;
-import org.elasticsearch.xpack.sql.stats.Metrics;
 import org.elasticsearch.xpack.sql.util.DateUtils;
 
 import java.sql.Types;
@@ -50,6 +47,7 @@
 import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.isTypeSupportedInVersion;
 import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.ql.type.DataTypes.VERSION;
+import static org.elasticsearch.xpack.sql.analysis.analyzer.AnalyzerTestUtils.analyzer;
 import static org.elasticsearch.xpack.sql.proto.Mode.isDriver;
 import static org.elasticsearch.xpack.sql.types.SqlTypesTests.loadMapping;
 import static org.mockito.ArgumentMatchers.any;
@@ -392,7 +390,7 @@ private Tuple<Command, SqlSession> sql(
         Map<String, EsField> mapping
     ) {
         EsIndex test = new EsIndex("test", mapping);
-        Analyzer analyzer = new Analyzer(config, new FunctionRegistry(), IndexResolution.valid(test), new Verifier(new Metrics()));
+        Analyzer analyzer = analyzer(config, IndexResolution.valid(test));
         Command cmd = (Command) analyzer.analyze(parser.createStatement(sql, params, UTC), true);
 
         IndexResolver resolver = mock(IndexResolver.class);
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java
index 11f2d09be613..9316c663bdfc 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java
@@ -9,7 +9,6 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
 import org.elasticsearch.xpack.ql.index.EsIndex;
 import org.elasticsearch.xpack.ql.index.IndexResolution;
 import org.elasticsearch.xpack.ql.index.IndexResolver;
@@ -20,7 +19,6 @@
 import org.elasticsearch.xpack.sql.SqlTestUtils;
 import org.elasticsearch.xpack.sql.action.Protocol;
 import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer;
-import org.elasticsearch.xpack.sql.analysis.analyzer.Verifier;
 import org.elasticsearch.xpack.sql.parser.SqlParser;
 import org.elasticsearch.xpack.sql.plan.logical.command.Command;
 import org.elasticsearch.xpack.sql.proto.Mode;
@@ -28,7 +26,6 @@
 import org.elasticsearch.xpack.sql.session.SchemaRowSet;
 import org.elasticsearch.xpack.sql.session.SqlConfiguration;
 import org.elasticsearch.xpack.sql.session.SqlSession;
-import org.elasticsearch.xpack.sql.stats.Metrics;
 import org.elasticsearch.xpack.sql.types.SqlTypesTests;
 import org.elasticsearch.xpack.sql.util.DateUtils;
 
@@ -44,6 +41,7 @@
 import static org.elasticsearch.action.ActionListener.wrap;
 import static org.elasticsearch.xpack.ql.index.IndexResolver.SQL_TABLE;
 import static org.elasticsearch.xpack.ql.index.IndexResolver.SQL_VIEW;
+import static org.elasticsearch.xpack.sql.analysis.analyzer.AnalyzerTestUtils.analyzer;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
@@ -368,12 +366,7 @@ private SqlTypedParamValue param(Object value) {
 
     private Tuple<Command, SqlSession> sql(String sql, List<SqlTypedParamValue> params, SqlConfiguration cfg) {
         EsIndex test = new EsIndex("test", mapping);
-        Analyzer analyzer = new Analyzer(
-            SqlTestUtils.TEST_CFG,
-            new FunctionRegistry(),
-            IndexResolution.valid(test),
-            new Verifier(new Metrics())
-        );
+        Analyzer analyzer = analyzer(IndexResolution.valid(test));
         Command cmd = (Command) analyzer.analyze(parser.createStatement(sql, params, cfg.zoneId()), true);
 
         IndexResolver resolver = mock(IndexResolver.class);
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java
index 7a6481983615..922ee7f81243 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java
@@ -9,7 +9,6 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
 import org.elasticsearch.xpack.ql.index.EsIndex;
 import org.elasticsearch.xpack.ql.index.IndexResolution;
 import org.elasticsearch.xpack.ql.index.IndexResolver;
@@ -38,6 +37,7 @@
 import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.isTypeSupportedInVersion;
 import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.ql.type.DataTypes.VERSION;
+import static org.elasticsearch.xpack.sql.analysis.analyzer.AnalyzerTestUtils.analyzer;
 import static org.elasticsearch.xpack.sql.plan.logical.command.sys.SysColumnsTests.UNSIGNED_LONG_TEST_VERSIONS;
 import static org.elasticsearch.xpack.sql.plan.logical.command.sys.SysColumnsTests.VERSION_FIELD_TEST_VERSIONS;
 import static org.mockito.Mockito.mock;
@@ -67,7 +67,7 @@ private Tuple<Command, SqlSession> sql(String sql, Mode mode, SqlVersion version
             false
         );
         EsIndex test = new EsIndex("test", SqlTypesTests.loadMapping("mapping-multi-field-with-nested.json", true));
-        Analyzer analyzer = new Analyzer(configuration, new FunctionRegistry(), IndexResolution.valid(test), null);
+        Analyzer analyzer = analyzer(configuration, IndexResolution.valid(test));
         Command cmd = (Command) analyzer.analyze(parser.createStatement(sql), false);
 
         IndexResolver resolver = mock(IndexResolver.class);
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java
index bc9402401ee1..72d5fc286465 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.xpack.ql.type.EsField;
 import org.elasticsearch.xpack.sql.SqlTestUtils;
 import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer;
-import org.elasticsearch.xpack.sql.analysis.analyzer.Verifier;
 import org.elasticsearch.xpack.sql.expression.function.SqlFunctionRegistry;
 import org.elasticsearch.xpack.sql.optimizer.Optimizer;
 import org.elasticsearch.xpack.sql.parser.SqlParser;
@@ -31,7 +30,6 @@
 import org.elasticsearch.xpack.sql.querydsl.container.QueryContainer;
 import org.elasticsearch.xpack.sql.session.EmptyExecutable;
 import org.elasticsearch.xpack.sql.session.SingletonExecutable;
-import org.elasticsearch.xpack.sql.stats.Metrics;
 import org.elasticsearch.xpack.sql.types.SqlTypesTests;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -42,6 +40,7 @@
 
 import static java.util.Arrays.asList;
 import static java.util.stream.Collectors.toList;
+import static org.elasticsearch.xpack.sql.analysis.analyzer.AnalyzerTestUtils.analyzer;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.startsWith;
@@ -60,7 +59,7 @@ public static void init() {
         Map<String, EsField> mapping = SqlTypesTests.loadMapping("mapping-multi-field-variation.json");
         EsIndex test = new EsIndex("test", mapping);
         IndexResolution getIndexResult = IndexResolution.valid(test);
-        analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), getIndexResult, new Verifier(new Metrics()));
+        analyzer = analyzer(getIndexResult);
         optimizer = new Optimizer();
         planner = new Planner();
     }
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorSpecTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorSpecTests.java
index 3cc1023e37d1..1df5058f65c4 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorSpecTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorSpecTests.java
@@ -16,13 +16,10 @@
 import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.ql.type.EsField;
 import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer;
-import org.elasticsearch.xpack.sql.analysis.analyzer.Verifier;
-import org.elasticsearch.xpack.sql.expression.function.SqlFunctionRegistry;
 import org.elasticsearch.xpack.sql.optimizer.Optimizer;
 import org.elasticsearch.xpack.sql.parser.SqlParser;
 import org.elasticsearch.xpack.sql.plan.physical.EsQueryExec;
 import org.elasticsearch.xpack.sql.plan.physical.PhysicalPlan;
-import org.elasticsearch.xpack.sql.stats.Metrics;
 import org.elasticsearch.xpack.sql.types.SqlTypesTests;
 import org.elasticsearch.xpack.sql.util.DateUtils;
 import org.hamcrest.Matcher;
@@ -32,7 +29,7 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.sql.SqlTestUtils.TEST_CFG;
+import static org.elasticsearch.xpack.sql.analysis.analyzer.AnalyzerTestUtils.analyzer;
 
 public class QueryTranslatorSpecTests extends ESTestCase {
 
@@ -49,7 +46,7 @@ private static class TestContext {
             Map<String, EsField> mapping = SqlTypesTests.loadMapping(mappingFile);
             EsIndex test = new EsIndex("test", mapping);
             IndexResolution getIndexResult = IndexResolution.valid(test);
-            analyzer = new Analyzer(TEST_CFG, new SqlFunctionRegistry(), getIndexResult, new Verifier(new Metrics()));
+            analyzer = analyzer(getIndexResult);
             optimizer = new Optimizer();
             planner = new Planner();
         }
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java
index b18d3c52f98c..d5b4a31e9e38 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.sql.planner;
 
 import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.query.ExistsQueryBuilder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -48,7 +49,6 @@
 import org.elasticsearch.xpack.ql.querydsl.query.WildcardQuery;
 import org.elasticsearch.xpack.ql.type.EsField;
 import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer;
-import org.elasticsearch.xpack.sql.analysis.analyzer.Verifier;
 import org.elasticsearch.xpack.sql.expression.function.SqlFunctionRegistry;
 import org.elasticsearch.xpack.sql.expression.function.aggregate.ExtendedStatsEnclosed;
 import org.elasticsearch.xpack.sql.expression.function.aggregate.MatrixStatsEnclosed;
@@ -75,7 +75,6 @@
 import org.elasticsearch.xpack.sql.querydsl.agg.GroupByDateHistogram;
 import org.elasticsearch.xpack.sql.querydsl.container.MetricAggRef;
 import org.elasticsearch.xpack.sql.session.SingletonExecutable;
-import org.elasticsearch.xpack.sql.stats.Metrics;
 import org.elasticsearch.xpack.sql.types.SqlTypesTests;
 import org.elasticsearch.xpack.sql.util.DateUtils;
 import org.hamcrest.Matcher;
@@ -102,6 +101,7 @@
 import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT;
 import static org.elasticsearch.xpack.sql.SqlTestUtils.TEST_CFG;
 import static org.elasticsearch.xpack.sql.SqlTestUtils.literal;
+import static org.elasticsearch.xpack.sql.analysis.analyzer.AnalyzerTestUtils.analyzer;
 import static org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation.E;
 import static org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation.PI;
 import static org.elasticsearch.xpack.sql.planner.QueryTranslator.DATE_FORMAT;
@@ -130,7 +130,7 @@ private static class TestContext {
             Map<String, EsField> mapping = SqlTypesTests.loadMapping(mappingFile);
             EsIndex test = new EsIndex("test", mapping);
             IndexResolution getIndexResult = IndexResolution.valid(test);
-            analyzer = new Analyzer(TEST_CFG, sqlFunctionRegistry, getIndexResult, new Verifier(new Metrics()));
+            analyzer = analyzer(getIndexResult);
             optimizer = new Optimizer();
             planner = new Planner();
         }
@@ -770,7 +770,7 @@ public void testChronoFieldBasedDateTimeFunctionsWithMathIntervalAndGroupBy() {
                 + randomFunction.name()
                 + "(date + INTERVAL 1 YEAR)"
         );
-        assertESQuery(p, containsString(formatted("""
+        assertESQuery(p, containsString(Strings.format("""
             {
               "terms": {
                 "script": {
@@ -803,7 +803,7 @@ public void testDateTimeFunctionsWithMathIntervalAndGroupBy() {
         );
         assertEquals(EsQueryExec.class, p.getClass());
         EsQueryExec eqe = (EsQueryExec) p;
-        assertThat(eqe.queryContainer().toString().replaceAll("\\s+", ""), containsString(formatted("""
+        assertThat(eqe.queryContainer().toString().replaceAll("\\s+", ""), containsString(Strings.format("""
             {
               "terms": {
                 "script": {
@@ -1131,7 +1131,7 @@ public void testAllCountVariantsWithHavingGenerateCorrectAggregations() {
         assertThat(
             ee.queryContainer().aggs().asAggBuilder().toString().replaceAll("\\s+", ""),
             endsWith(
-                formatted(
+                Strings.format(
                     """
                         {
                           "buckets_path": {
@@ -1204,7 +1204,7 @@ public void testExtendedStatsAggsStddevAndVar() {
             assertEquals(((MetricAggRef) fe).property(), metricToAgg.get(funcName));
 
             String aggName = eqe.queryContainer().aggs().asAggBuilder().getSubAggregations().iterator().next().getName();
-            assertESQuery(p, endsWith(formatted("""
+            assertESQuery(p, endsWith(Strings.format("""
                 "aggregations":{"%s":{"extended_stats":{"field":"int","sigma":2.0}}}}}}\
                 """, aggName)));
         }
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/VerifierTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/VerifierTests.java
index fdb87f9ab1c9..0e2e50cb73fc 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/VerifierTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/VerifierTests.java
@@ -11,14 +11,11 @@
 import org.elasticsearch.xpack.ql.index.EsIndex;
 import org.elasticsearch.xpack.ql.index.IndexResolution;
 import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer;
-import org.elasticsearch.xpack.sql.analysis.analyzer.Verifier;
-import org.elasticsearch.xpack.sql.expression.function.SqlFunctionRegistry;
 import org.elasticsearch.xpack.sql.parser.SqlParser;
 import org.elasticsearch.xpack.sql.plan.physical.EsQueryExec;
 import org.elasticsearch.xpack.sql.plan.physical.PhysicalPlan;
-import org.elasticsearch.xpack.sql.stats.Metrics;
 
-import static org.elasticsearch.xpack.sql.SqlTestUtils.TEST_CFG;
+import static org.elasticsearch.xpack.sql.analysis.analyzer.AnalyzerTestUtils.analyzer;
 import static org.elasticsearch.xpack.sql.types.SqlTypesTests.loadMapping;
 
 public class VerifierTests extends ESTestCase {
@@ -27,7 +24,7 @@ public class VerifierTests extends ESTestCase {
     private final IndexResolution indexResolution = IndexResolution.valid(
         new EsIndex("test", loadMapping("mapping-multi-field-with-nested.json"))
     );
-    private final Analyzer analyzer = new Analyzer(TEST_CFG, new SqlFunctionRegistry(), indexResolution, new Verifier(new Metrics()));
+    private final Analyzer analyzer = analyzer(indexResolution);
     private final Planner planner = new Planner();
 
     private PhysicalPlan verify(String sql) {
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/session/ListCursorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/session/ListCursorTests.java
index ba08e25c381d..26312ef135ba 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/session/ListCursorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/session/ListCursorTests.java
@@ -31,7 +31,7 @@ public static ListCursor randomPagingListCursor() {
     }
 
     @Override
-    protected ListCursor mutateInstance(ListCursor instance) throws IOException {
+    protected ListCursor mutateInstance(ListCursor instance) {
         return new ListCursor(instance.data(), randomValueOtherThan(instance.pageSize(), () -> between(1, 20)), instance.columnCount());
     }
 
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/stats/VerifierMetricsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/stats/VerifierMetricsTests.java
index da6c96a84684..7612c453d699 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/stats/VerifierMetricsTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/stats/VerifierMetricsTests.java
@@ -12,15 +12,14 @@
 import org.elasticsearch.xpack.ql.index.EsIndex;
 import org.elasticsearch.xpack.ql.index.IndexResolution;
 import org.elasticsearch.xpack.ql.type.EsField;
-import org.elasticsearch.xpack.sql.SqlTestUtils;
 import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer;
 import org.elasticsearch.xpack.sql.analysis.analyzer.Verifier;
-import org.elasticsearch.xpack.sql.expression.function.SqlFunctionRegistry;
 import org.elasticsearch.xpack.sql.parser.SqlParser;
 import org.elasticsearch.xpack.sql.types.SqlTypesTests;
 
 import java.util.Map;
 
+import static org.elasticsearch.xpack.sql.analysis.analyzer.AnalyzerTestUtils.analyzer;
 import static org.elasticsearch.xpack.sql.stats.FeatureMetric.COMMAND;
 import static org.elasticsearch.xpack.sql.stats.FeatureMetric.GROUPBY;
 import static org.elasticsearch.xpack.sql.stats.FeatureMetric.HAVING;
@@ -258,7 +257,7 @@ private Counters sql(String sql, Verifier v) {
             verifier = new Verifier(metrics);
         }
 
-        Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), IndexResolution.valid(test), verifier);
+        Analyzer analyzer = analyzer(IndexResolution.valid(test), verifier);
         analyzer.analyze(parser.createStatement(sql), true);
 
         return metrics == null ? null : metrics.stats();
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/reset_tracking_rate.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/reset_tracking_rate.yml
new file mode 100644
index 000000000000..7027e0f90391
--- /dev/null
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/reset_tracking_rate.yml
@@ -0,0 +1,135 @@
+setup:
+  - skip:
+      version: " - 8.5.0"
+      reason: Reset tracking rate agg added in 8.6
+
+  - do:
+      indices.create:
+        index: tsdb-01
+        body:
+          settings:
+            number_of_replicas: 0
+            mode: time_series
+            routing_path: [key]
+            time_series:
+              start_time: "2021-01-01T00:00:00Z"
+              end_time: "2021-01-01T01:00:00Z"
+          mappings:
+            properties:
+              val:
+                type: long
+                time_series_metric: counter
+              key:
+                type: keyword
+                time_series_dimension: true
+              "@timestamp":
+                type: date
+  - do:
+      indices.create:
+        index: tsdb-02
+        body:
+          settings:
+            number_of_replicas: 0
+            mode: time_series
+            routing_path: [key]
+            time_series:
+              start_time: "2021-01-01T00:01:00Z"
+              end_time: "2021-01-01T02:00:00Z"
+          mappings:
+            properties:
+              val:
+                type: long
+                time_series_metric: counter
+              key:
+                type: keyword
+                time_series_dimension: true
+              "@timestamp":
+                type: date
+  - do:
+      indices.create:
+        index: tsdb-03
+        body:
+          settings:
+            number_of_replicas: 0
+            mode: time_series
+            routing_path: [key]
+            time_series:
+              start_time: "2021-01-01T00:02:00Z"
+              end_time: "2021-01-01T03:00:00Z"
+          mappings:
+            properties:
+              val:
+                type: long
+                time_series_metric: counter
+              key:
+                type: keyword
+                time_series_dimension: true
+              "@timestamp":
+                type: date
+  - do:
+      cluster.health:
+        wait_for_status: green
+
+  - do:
+      bulk:
+        index: tsdb-01
+        refresh: true
+        body:
+          - '{ "index": {} }'
+          - '{ "key": "bar", "val": 40, "@timestamp": "2021-01-01T00:00:00Z" }'
+          - '{ "index": {}}'
+          - '{ "key": "bar", "val": 50, "@timestamp": "2021-01-01T00:20:00Z" }'
+          - '{ "index": {}}'
+          - '{ "key": "bar", "val": 60, "@timestamp": "2021-01-01T00:40:00Z" }'
+  - do:
+      bulk:
+        index: tsdb-02
+        refresh: true
+        body:
+          - '{ "index": {} }'
+          - '{ "key": "bar", "val": 70, "@timestamp": "2021-01-01T01:00:00Z" }'
+          - '{ "index": {}}'
+          - '{ "key": "bar", "val": 0, "@timestamp": "2021-01-01T01:20:00Z" }'
+          - '{ "index": {}}'
+          - '{ "key": "bar", "val": 10, "@timestamp": "2021-01-01T01:40:00Z" }'
+
+  - do:
+      bulk:
+        index: tsdb-03
+        refresh: true
+        body:
+          - '{ "index": {} }'
+          - '{ "key": "bar", "val": 0, "@timestamp": "2021-01-01T02:00:00Z" }'
+          - '{ "index": {}}'
+          - '{ "key": "bar", "val": 10, "@timestamp": "2021-01-01T02:20:00Z" }'
+          - '{ "index": {}}'
+          - '{ "key": "bar", "val": 20, "@timestamp": "2021-01-01T02:40:00Z" }'
+---
+"test resets do not lead to negative rate":
+  - do:
+      search:
+        index: tsdb-*
+        body:
+          query:
+            range:
+              "@timestamp":
+                gte: "2020-01-01T00:10:00Z"
+          size: 0
+          aggs:
+            ts:
+              time_series:
+                keyed: false
+              aggs:
+                rate:
+                  rate:
+                    field: val
+
+
+
+  - match: { hits.total.value: 9 }
+  - length: { aggregations.ts.buckets: 1 }
+
+  - match: { aggregations.ts.buckets.0.key: { "key": "bar" } }
+  - match: { aggregations.ts.buckets.0.doc_count: 9 }
+  - gte: { aggregations.ts.buckets.0.rate.value: 0.0 }
+
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/top_metrics.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/top_metrics.yml
index ab8914840198..07cddcbf9eea 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/top_metrics.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/top_metrics.yml
@@ -707,3 +707,53 @@
                 size: 3
   - match: { aggregations.tm.top.0.sort: [1] }
   - match: { aggregations.tm.top.0.metrics.host\.hostname: abc }
+
+---
+"Counter field":
+  - skip:
+      version: " - 8.6.99"
+      reason: "counter field support added in 8.7"
+
+  - do:
+      indices.create:
+        index: myindex
+        body:
+          mappings:
+            properties:
+              counter_field:
+                type: long
+                time_series_metric: counter
+              number_field:
+                type: long
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: myindex
+              _id:    "1"
+          - number_field: 1
+            counter_field: 2
+          - index:
+              _index: myindex
+              _id:    "2"
+          - number_field: 2
+            counter_field: 4
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: myindex
+        body:
+          aggs:
+            the_counter_top_metrics:
+              top_metrics:
+                metrics:
+                  field: counter_field
+                sort:
+                  number_field: asc
+
+  - match: { hits.total: 2 }
+  - length: { hits.hits: 2 }
+  - match: { aggregations.the_counter_top_metrics.top.0.sort: [1] }
+  - match: { aggregations.the_counter_top_metrics.top.0.metrics.counter_field: 2 }
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/change_password/12_custom_hash.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/change_password/12_custom_hash.yml
new file mode 100644
index 000000000000..cb876fba5d2a
--- /dev/null
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/change_password/12_custom_hash.yml
@@ -0,0 +1,171 @@
+---
+setup:
+  - skip:
+      features:  headers
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+  - do:
+      security.put_user:
+        username: "test_user"
+        body: >
+          {
+            "password": "s3cr3t-password",
+            "roles" : [ "superuser" ]
+          }
+  - do:
+      security.put_user:
+        username: "pre_hashed_user1"
+        body:  >
+          {
+            "password": "new-test-password",
+            "roles" : [ "superuser" ]
+          }
+---
+teardown:
+  - do:
+      security.delete_user:
+        username: "test_user"
+        ignore: 404
+  - do:
+      security.delete_user:
+        username: "pre_hashed_user1"
+        ignore: 404
+  - do:
+      security.delete_user:
+        username: "pre_hashed_user2"
+        ignore: 404
+
+---
+"Test creating users with pre-hashed password":
+  - skip:
+      features: ["catch_unauthorized", "warnings"]
+  - do:
+      headers:
+        Authorization: "Basic dGVzdF91c2VyOnMzY3IzdC1wYXNzd29yZA=="
+      cluster.health: {}
+  - match: { timed_out: false }
+
+  # create user with pre-hashed PBKDF2 password with 16 bytes salt and 512 bits key lengths
+  - do:
+      security.put_user:
+        username: "pre_hashed_user2"
+        body:  >
+          {
+            "password_hash": "{PBKDF2}50000$SGRCCCwQ9vz105opttTJAQ==$9FVXEWMXeiwlFTwbvkiQ6h0vN3REnWvukzWlle7ql8QosHdusRhFMBWhRjM7CLFZzq/owGhFh5BmrXRkO6IHZA==",
+            "roles" : [ "superuser" ]
+          }
+
+  # login with valid credentials
+  - do:
+      headers:
+        Authorization: "Basic cHJlX2hhc2hlZF91c2VyMjpwcmVfaGFzaGVkX3VzZXIyLXBhc3N3b3Jk"
+      cluster.health: {}
+  - match: { timed_out: false }
+
+  # change password to PBKDF2 with 24 bytes salt and 128 bits key lengths
+  - do:
+      security.change_password:
+        username: "pre_hashed_user2"
+        body:  >
+          {
+            "password_hash" : "{PBKDF2}10000$1C7Dlwg/5awqxgi+f6czbBAi/kQxKiDA$3AhIGtGiwYbFBIAosoT3LA=="
+          }
+
+  # login with new credentials
+  - do:
+      headers:
+        Authorization: "Basic cHJlX2hhc2hlZF91c2VyMjpwcmVfaGFzaGVkX3VzZXIyLXBhc3N3b3JkMg=="
+      cluster.health: {}
+  - match: { timed_out: false }
+
+---
+"Test changing users password with pre-hashed password":
+  - skip:
+      features: ["catch_unauthorized", "warnings"]
+  - do:
+      headers:
+        Authorization: "Basic dGVzdF91c2VyOnMzY3IzdC1wYXNzd29yZA=="
+      cluster.health: {}
+  - match: { timed_out: false }
+  # We need this so that we can get hold of the password hash of the user because, for bcrypt,
+  # the stored hash string starts with a "$" and it would otherwise be
+  # interpreted as a stashed value, if we tried to use it as the value of "password_hash".
+  - do:
+      headers:
+        Authorization: "Basic dGVzdF91c2VyOnMzY3IzdC1wYXNzd29yZA=="
+      warnings:
+        - "this request accesses system indices: [.security-7], but in a future major version, direct access to system indices will be prevented by default"
+      get:
+        index: .security
+        id: user-pre_hashed_user1
+  - set: { _source.password: "pre_hashed_user1_hash" }
+
+  # change password to "new-test-password2"
+  - do:
+      headers:
+        Authorization: "Basic cHJlX2hhc2hlZF91c2VyMTpuZXctdGVzdC1wYXNzd29yZA=="
+      security.change_password:
+        username: "pre_hashed_user1"
+        body: >
+          {
+            "password_hash" : "{PBKDF2}10000$7M5rzp0qLpIP3qecOmUl2TTbr4yQ/sTmBMkQ2EtFrYk=$kElwxM4tFXU2eElGaqgOWVg95uAVUgVrkl45Zpzr+ZU="
+          }
+
+  # login with new credentials
+  - do:
+      headers:
+        Authorization: "Basic cHJlX2hhc2hlZF91c2VyMTpuZXctdGVzdC1wYXNzd29yZDI="
+      cluster.health: {}
+  - match: { timed_out: false }
+
+  # change the password to the original one
+  - do:
+      headers:
+        Authorization: "Basic cHJlX2hhc2hlZF91c2VyMTpuZXctdGVzdC1wYXNzd29yZDI="
+      security.change_password:
+        username: "pre_hashed_user1"
+        body:  >
+          {
+            "password_hash" : "$pre_hashed_user1_hash"
+          }
+
+  # login with original credentials
+  - do:
+      headers:
+        Authorization: "Basic cHJlX2hhc2hlZF91c2VyMTpuZXctdGVzdC1wYXNzd29yZA=="
+      cluster.health: {}
+  - match: { timed_out: false }
+
+---
+"Test creating users and changing passwords with clear text as hash":
+  - skip:
+      features: ["catch_unauthorized", "warnings"]
+  - do:
+      headers:
+        Authorization: "Basic dGVzdF91c2VyOnMzY3IzdC1wYXNzd29yZA=="
+      cluster.health: {}
+  - match: { timed_out: false }
+
+  - do:
+      headers:
+        Authorization: "Basic dGVzdF91c2VyOnMzY3IzdC1wYXNzd29yZA=="
+      catch: /The provided password hash is not a hash or it could not be resolved to a supported hash algorithm.*/
+      security.put_user:
+        username: "clear_text_hash_user1"
+        body:  >
+          {
+            "password_hash": "som3-1ong-passw0rd",
+            "roles" : [ "superuser" ]
+          }
+
+  - do:
+      headers:
+        Authorization: "Basic dGVzdF91c2VyOnMzY3IzdC1wYXNzd29yZA=="
+      catch: /The provided password hash is not a hash or it could not be resolved to a supported hash algorithm..*/
+      security.change_password:
+        username: "test_user"
+        body: >
+          {
+            "password_hash" : "som3-1ong-passw0rd"
+          }
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/health/10_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/health/10_usage.yml
new file mode 100644
index 000000000000..207b70367766
--- /dev/null
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/health/10_usage.yml
@@ -0,0 +1,37 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: red_index
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+            index.routing.allocation.enable: none
+
+  - do:
+      health_report: {}
+
+  - do:
+      health_report:
+        feature: disk
+---
+"Usage stats on the health API":
+  - skip:
+      version: "- 8.6.99"
+      reason: "the health api stats were only added to the usage api in 8.7"
+
+  - do:
+      xpack.usage: {}
+
+  - match: { health_api.available: true }
+  - match: { health_api.enabled: true }
+  - match: { health_api.invocations.total: 2 }
+  - match: { health_api.invocations.verbose_true: 2 }
+  - match: { health_api.statuses.red: 1 }
+  - match: { health_api.statuses.green: 1 }
+  - match: { health_api.statuses.values: ["green", "red"] }
+  - match: { health_api.indicators.red.values: ["shards_availability"] }
+  - match: { health_api.indicators.red.shards_availability: 1 }
+  - match: { health_api.diagnoses.red.elasticsearch:health:shards_availability:diagnosis:enable_index_allocations: 1}
+  - match: { health_api.diagnoses.red.values: ["elasticsearch:health:shards_availability:diagnosis:enable_index_allocations"]}
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/frequent_item_sets_agg.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/frequent_item_sets_agg.yml
new file mode 100644
index 000000000000..f5244d271abe
--- /dev/null
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/frequent_item_sets_agg.yml
@@ -0,0 +1,542 @@
+setup:
+  - skip:
+      features: headers
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      indices.create:
+        index: store
+        body:
+          mappings:
+            properties:
+              features:
+                type: keyword
+              error_message:
+                type: keyword
+              timestamp:
+                type: date
+              geo_point:
+                type: geo_point
+              histogram:
+                type: histogram
+
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+        Content-Type: application/json
+      bulk:
+        index: store
+        refresh: true
+        body: |
+          { "index": {} }
+          { "features": ["turbocharger", "electric mirror", "heated steering wheel", "matrix led", "gps", "sunroof", "autopilot", "air condition"], "error_message": "compressor low pressure", "timestamp": 1020229654000 }
+          { "index": {} }
+          { "features": ["pink wheels", "turbocharger", "lane assist", "matrix led", "coming home", "heated seats", "blind spot monitoring", "air condition"], "error_message": "engine overheated", "timestamp": 1020316054000 }
+          { "index": {} }
+          { "features": ["awd", "tow-bar", "matrix led", "gps", "sunroof", "air condition"], "error_message": "compressor low pressure", "timestamp": 1022742044000 }
+          { "index": {} }
+          { "features": ["rwd", "tow-bar", "turbocharger", "lane assist", "pink wheels"], "error_message": "engine overheated", "timestamp": 1021532444000 }
+          { "index": {} }
+          { "features": ["pink wheels", "sunroof", "air condition"], "error_message": "compressor low pressure", "timestamp": 1020575254000 }
+          { "index": {} }
+          { "features": ["turbocharger", "pink wheels", "air condition" ], "error_message": "engine overheated", "timestamp": 1020322844000 }
+          { "index": {} }
+          { "features": ["awd", "heated steering wheel", "sunroof", "air condition"], "error_message": "compressor low pressure", "timestamp": 1020748054000 }
+          { "index": {} }
+          { "features": ["turbocharger", "sunroof", "autopilot", "air condition" ], "error_message": "compressor low pressure", "timestamp": 1020229654000 }
+          { "index": {} }
+          { "features": ["rwd", "turbocharger", "autopilot", "pink wheels"], "error_message": "engine overheated", "timestamp": 1022137244000 }
+          { "index": {} }
+          { "features": ["fwd", "tow-bar", "turbocharger", "pink wheels", "electric mirror", "heated steering wheel", "lane assist", "sunroof", "autopilot", "air condition" ], "error_message": "low tire pressure", "timestamp": 1020920854000 }
+  # same data, but stored in a flattened field
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      indices.create:
+        index: store-flattened
+        body:
+          mappings:
+            properties:
+              data:
+                type: flattened
+              timestamp:
+                type: date
+
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+        Content-Type: application/json
+      bulk:
+        index: store-flattened
+        refresh: true
+        body: |
+          { "index": {} }
+          { "data": {"features": ["turbocharger", "electric mirror", "heated steering wheel", "matrix led", "gps", "sunroof", "autopilot", "air condition"], "error_message": "compressor low pressure"}, "timestamp": 1020229654000 }
+          { "index": {} }
+          { "data": {"features": ["pink wheels", "turbocharger", "lane assist", "matrix led", "coming home", "heated seats", "blind spot monitoring", "air condition"], "error_message": "engine overheated"}, "timestamp": 1020316054000 }
+          { "index": {} }
+          { "data": {"features": ["awd", "tow-bar", "matrix led", "gps", "sunroof", "air condition"], "error_message": "compressor low pressure"}, "timestamp": 1022742044000 }
+          { "index": {} }
+          { "data": {"features": ["rwd", "tow-bar", "turbocharger", "lane assist", "pink wheels"], "error_message": "engine overheated"}, "timestamp": 1021532444000 }
+          { "index": {} }
+          { "data": {"features": ["pink wheels", "sunroof", "air condition"], "error_message": "compressor low pressure"}, "timestamp": 1020575254000 }
+          { "index": {} }
+          { "data": {"features": ["turbocharger", "pink wheels", "air condition" ], "error_message": "engine overheated"}, "timestamp": 1020322844000 }
+          { "index": {} }
+          { "data": {"features": ["awd", "heated steering wheel", "sunroof", "air condition"], "error_message": "compressor low pressure"}, "timestamp": 1020748054000 }
+          { "index": {} }
+          { "data": {"features": ["turbocharger", "sunroof", "autopilot", "air condition" ], "error_message": "compressor low pressure"}, "timestamp": 1020229654000 }
+          { "index": {} }
+          { "data": {"features": ["rwd", "turbocharger", "autopilot", "pink wheels"], "error_message": "engine overheated"}, "timestamp": 1022137244000 }
+          { "index": {} }
+          { "data": {"features": ["fwd", "tow-bar", "turbocharger", "pink wheels", "electric mirror", "heated steering wheel", "lane assist", "sunroof", "autopilot", "air condition" ], "error_message": "low tire pressure"}, "timestamp": 1020920854000 }
+
+
+---
+"Test frequent item sets array fields":
+
+  - do:
+      search:
+        index: store
+        body: >
+          {
+            "size": 0,
+            "aggs": {
+              "fi": {
+                "frequent_item_sets": {
+                  "minimum_set_size": 3,
+                  "minimum_support": 0.3,
+                  "fields": [
+                    {"field": "features"},
+                    {"field": "error_message"}
+                  ]
+                }
+              }
+            }
+          }
+  - length: { aggregations.fi.buckets: 4 }
+  - match: { aggregations.fi.buckets.0.doc_count: 5 }
+  - match: { aggregations.fi.buckets.0.support: 0.5 }
+  - match: { aggregations.fi.buckets.0.key.error_message: ["compressor low pressure"] }
+  - match: { aggregations.fi.buckets.1.doc_count: 4 }
+  - match: { aggregations.fi.buckets.1.support: 0.4 }
+  - match: { aggregations.fi.buckets.1.key.error_message: ["engine overheated"] }
+
+---
+"Test frequent item sets date format":
+
+  - do:
+      search:
+        index: store
+        body: >
+          {
+            "runtime_mappings": {
+              "weekday": {
+                "type": "keyword",
+                "script": {
+                "source": "emit(doc['timestamp'].value.format(DateTimeFormatter.ofPattern('EEE')));"
+                }
+              }
+            },
+            "size": 0,
+            "aggs": {
+              "fi": {
+                "frequent_item_sets": {
+                  "minimum_set_size": 3,
+                  "minimum_support": 0.3,
+                  "fields": [
+                    {"field": "features"},
+                    {"field": "weekday"}
+                  ]
+                }
+              }
+            }
+          }
+  - length: { aggregations.fi.buckets: 3 }
+  - match: { aggregations.fi.buckets.0.doc_count: 5 }
+  - match: { aggregations.fi.buckets.0.support: 0.5 }
+  - match: { aggregations.fi.buckets.0.key.weekday: ["Thu"] }
+
+
+---
+"Test frequent item sets date format 2":
+
+  - do:
+      search:
+        index: store
+        body: >
+          {
+            "runtime_mappings": {
+              "weekday": {
+                "type": "keyword",
+                "script": {
+                "source": "emit(doc['timestamp'].value.format(DateTimeFormatter.ofPattern('EEE')));"
+                }
+              }
+            },
+            "size": 0,
+            "aggs": {
+              "fi": {
+                "frequent_item_sets": {
+                  "minimum_set_size": 2,
+                  "minimum_support": 0.3,
+                  "fields": [
+                    {"field": "error_message"},
+                    {"field": "weekday"}
+                  ]
+                }
+              }
+            }
+          }
+  - length: { aggregations.fi.buckets: 1 }
+  - match: { aggregations.fi.buckets.0.doc_count: 4 }
+  - match: { aggregations.fi.buckets.0.support: 0.4 }
+  - match: { aggregations.fi.buckets.0.key.weekday: ["Thu"] }
+  - match: { aggregations.fi.buckets.0.key.error_message: ["engine overheated"] }
+
+---
+"Test frequent item sets array fields profile":
+
+  - do:
+      search:
+        index: store
+        body: >
+          {
+            "profile": true,
+            "size": 0,
+            "aggs": {
+              "fi": {
+                "frequent_item_sets": {
+                  "minimum_set_size": 3,
+                  "minimum_support": 0.2,
+                  "fields": [
+                    {"field": "features"},
+                    {"field": "error_message"}
+                  ]
+                }
+              }
+            }
+          }
+  - length: { aggregations.fi.buckets: 10 }
+  - match: { aggregations.fi.buckets.0.doc_count: 5 }
+  - match: { aggregations.fi.buckets.0.support: 0.5 }
+  - match: { aggregations.fi.buckets.0.key.error_message: ["compressor low pressure"] }
+  - match: { aggregations.fi.buckets.1.doc_count: 4 }
+  - match: { aggregations.fi.buckets.1.support: 0.4 }
+  - match: { aggregations.fi.buckets.1.key.error_message: ["engine overheated"] }
+  - match: { aggregations.fi.buckets.1.support: 0.4 }
+  - match: { aggregations.fi.profile.unique_transactions_after_prune: 10 }
+  - match: { aggregations.fi.profile.unique_items_after_prune: 11 }
+
+---
+"Test frequent item sets flattened fields":
+
+  - do:
+      search:
+        index: store-flattened
+        body: >
+          {
+            "size": 0,
+            "aggs": {
+              "fi": {
+                "frequent_item_sets": {
+                  "minimum_set_size": 3,
+                  "minimum_support": 0.3,
+                  "fields": [
+                    {"field": "data.features"},
+                    {"field": "data.error_message"}
+                  ]
+                }
+              }
+            }
+          }
+  - length: { aggregations.fi.buckets: 4 }
+  - match: { aggregations.fi.buckets.0.doc_count: 5 }
+  - match: { aggregations.fi.buckets.0.support: 0.5 }
+  - match: { aggregations.fi.buckets.0.key.data\.error_message: ["compressor low pressure"] }
+  - match: { aggregations.fi.buckets.1.doc_count: 4 }
+  - match: { aggregations.fi.buckets.1.support: 0.4 }
+  - match: { aggregations.fi.buckets.1.key.data\.error_message: ["engine overheated"] }
+
+---
+"Test frequent item sets as subagg":
+
+  - do:
+      search:
+        index: store
+        body: >
+          {
+            "size": 0,
+            "aggs": {
+              "filter_error": {
+                "filter": {
+                  "term": {
+                    "error_message": "compressor low pressure"
+                  }
+                },
+                "aggs": {
+                  "fi": {
+                    "frequent_item_sets": {
+                      "minimum_set_size": 3,
+                      "minimum_support": 0.3,
+                      "fields": [
+                        {"field": "features"},
+                        {"field": "error_message"}
+                      ]
+                    }
+                  }
+                }
+              }
+            }
+          }
+  - length: { aggregations.filter_error.fi.buckets: 5 }
+  - match: { aggregations.filter_error.fi.buckets.0.doc_count: 5 }
+  - match: { aggregations.filter_error.fi.buckets.0.support: 1.0 }
+  - match: { aggregations.filter_error.fi.buckets.0.key.error_message: ["compressor low pressure"] }
+  - match: { aggregations.filter_error.fi.buckets.1.doc_count: 2 }
+  - match: { aggregations.filter_error.fi.buckets.1.support: 0.4 }
+  - match: { aggregations.filter_error.fi.buckets.0.key.error_message: ["compressor low pressure"] }
+
+---
+"Test frequent item sets as multi-bucket subagg":
+
+  - do:
+      search:
+        index: store
+        body: >
+          {
+            "size": 0,
+            "aggs": {
+              "weekly": {
+                "date_histogram": {
+                  "field": "timestamp",
+                  "calendar_interval": "1w"
+                },
+                "aggs": {
+                  "fi": {
+                    "frequent_item_sets": {
+                      "minimum_set_size": 3,
+                      "minimum_support": 0.3,
+                      "fields": [
+                        {"field": "features"},
+                        {"field": "error_message"}
+                      ]
+                    }
+                  }
+                }
+              }
+            }
+          }
+  - length: { aggregations.weekly.buckets: 5 }
+  - match: { aggregations.weekly.buckets.0.fi.buckets.0.doc_count: 3 }
+  - match: { aggregations.weekly.buckets.0.fi.buckets.0.support: 0.6 }
+  - match: { aggregations.weekly.buckets.0.fi.buckets.0.key.error_message: ["compressor low pressure"] }
+  - length: { aggregations.weekly.buckets.0.fi.buckets: 4 }
+  - match: { aggregations.weekly.buckets.2.fi.buckets.0.doc_count: 1 }
+
+---
+"Test frequent item sets filter":
+
+  - do:
+      search:
+        index: store
+        body: >
+          {
+            "size": 0,
+            "aggs": {
+              "fi": {
+                "frequent_item_sets": {
+                  "minimum_set_size": 3,
+                  "minimum_support": 0.3,
+                  "fields": [
+                    {"field": "features"},
+                    {"field": "error_message"}
+                  ],
+                  "filter": {
+                    "bool": {
+                      "must_not": {
+                        "term": {"features": "pink wheels"}
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+  - length: { aggregations.fi.buckets: 1 }
+  - match: { aggregations.fi.buckets.0.doc_count: 4 }
+  - match: { aggregations.fi.buckets.0.support: 0.4 }
+  - match: { aggregations.fi.buckets.0.key.error_message: ["compressor low pressure"] }
+
+---
+"Test frequent item sets exclude":
+
+  - do:
+      search:
+        index: store
+        body: >
+          {
+            "size": 0,
+            "aggs": {
+              "fi": {
+                "frequent_item_sets": {
+                  "minimum_set_size": 3,
+                  "minimum_support": 0.3,
+                  "fields": [
+                    {"field": "features"},
+                    {
+                      "field": "error_message",
+                      "exclude": "engine overheated"
+                    }
+                  ]
+                }
+              }
+            }
+          }
+  - length: { aggregations.fi.buckets: 3 }
+  - match: { aggregations.fi.buckets.0.doc_count: 5 }
+  - match: { aggregations.fi.buckets.0.support: 0.5 }
+  - match: { aggregations.fi.buckets.0.key.error_message: ["compressor low pressure"] }
+  - match: { aggregations.fi.buckets.1.doc_count: 3 }
+  - match: { aggregations.fi.buckets.1.support: 0.3 }
+
+---
+"Test frequent item sets include":
+
+  - do:
+      search:
+        index: store
+        body: >
+          {
+            "size": 0,
+            "aggs": {
+              "fi": {
+                "frequent_item_sets": {
+                  "minimum_set_size": 3,
+                  "minimum_support": 0.3,
+                  "fields": [
+                    {"field": "features"},
+                    {
+                      "field": "error_message",
+                      "include": "en.*ed"
+                    }
+                  ]
+                }
+              }
+            }
+          }
+  - length: { aggregations.fi.buckets: 3 }
+  - match: { aggregations.fi.buckets.0.doc_count: 4 }
+  - match: { aggregations.fi.buckets.0.support: 0.4 }
+  - match: { aggregations.fi.buckets.0.key.error_message: ["engine overheated"] }
+
+---
+"Test frequent item sets unsupported types":
+  - do:
+      catch: /Field \[geo_point\] of type \[geo_point\] is not supported for aggregation \[frequent_item_sets\]/
+      search:
+        index: store
+        body: >
+          {
+            "size": 0,
+            "aggs": {
+              "fi": {
+                "frequent_item_sets": {
+                  "minimum_set_size": 3,
+                  "minimum_support": 0.3,
+                  "fields": [
+                    {"field": "features"},
+                    {"field": "error_message"},
+                    {"field": "geo_point"}
+                  ]
+                }
+              }
+            }
+          }
+  - do:
+      catch: /Field \[histogram\] of type \[histogram\] is not supported for aggregation \[frequent_item_sets\]/
+      search:
+        index: store
+        body: >
+          {
+            "size": 0,
+            "aggs": {
+              "fi": {
+                "frequent_item_sets": {
+                  "minimum_set_size": 3,
+                  "minimum_support": 0.3,
+                  "fields": [
+                    {"field": "features"},
+                    {"field": "error_message"},
+                    {"field": "histogram"}
+                  ]
+                }
+              }
+            }
+          }
+
+---
+"Test frequent item sets unsupported subaggs":
+  - do:
+      catch: /Aggregator \[fi\] of type \[frequent_item_sets\] cannot accept sub-aggregations/
+      search:
+        index: store
+        body: >
+          {
+            "size": 0,
+            "aggs": {
+              "fi": {
+                "frequent_item_sets": {
+                  "minimum_set_size": 3,
+                  "minimum_support": 0.3,
+                  "fields": [
+                    {"field": "features"},
+                    {"field": "error_message"}
+                  ]
+                },
+              "aggs": {
+                "sum": {
+                  "sum": {
+                    "field": "some_field"
+                  }
+                }
+              }
+            }
+            }
+          }
+
+---
+"Test deprecated frequent items":
+  - skip:
+      features:
+        - "allowed_warnings"
+
+  - do:
+      allowed_warnings:
+        - 'Deprecated field [frequent_items] used, expected [frequent_item_sets] instead'
+
+      search:
+        index: store
+        body: >
+          {
+            "size": 0,
+            "aggs": {
+              "fi": {
+                "frequent_items": {
+                  "minimum_set_size": 3,
+                  "minimum_support": 0.3,
+                  "fields": [
+                    {"field": "features"},
+                    {"field": "error_message"}
+                  ]
+                }
+              }
+            }
+          }
+  - length: { aggregations.fi.buckets: 4 }
+  - match: { aggregations.fi.buckets.0.doc_count: 5 }
+  - match: { aggregations.fi.buckets.0.support: 0.5 }
+  - match: { aggregations.fi.buckets.0.key.error_message: ["compressor low pressure"] }
+  - match: { aggregations.fi.buckets.1.doc_count: 4 }
+  - match: { aggregations.fi.buckets.1.support: 0.4 }
+  - match: { aggregations.fi.buckets.1.key.error_message: ["engine overheated"] }
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/frequent_items_agg.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/frequent_items_agg.yml
deleted file mode 100644
index 72d7a08de684..000000000000
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/frequent_items_agg.yml
+++ /dev/null
@@ -1,444 +0,0 @@
-setup:
-  - skip:
-      features: headers
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-      indices.create:
-        index: store
-        body:
-          mappings:
-            properties:
-              features:
-                type: keyword
-              error_message:
-                type: keyword
-              timestamp:
-                type: date
-              geo_point:
-                type: geo_point
-              histogram:
-                type: histogram
-
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-        Content-Type: application/json
-      bulk:
-        index: store
-        refresh: true
-        body: |
-          { "index": {} }
-          { "features": ["turbocharger", "electric mirror", "heated steering wheel", "matrix led", "gps", "sunroof", "autopilot", "air condition"], "error_message": "compressor low pressure", "timestamp": 1020229654000 }
-          { "index": {} }
-          { "features": ["pink wheels", "turbocharger", "lane assist", "matrix led", "coming home", "heated seats", "blind spot monitoring", "air condition"], "error_message": "engine overheated", "timestamp": 1020316054000 }
-          { "index": {} }
-          { "features": ["awd", "tow-bar", "matrix led", "gps", "sunroof", "air condition"], "error_message": "compressor low pressure", "timestamp": 1022742044000 }
-          { "index": {} }
-          { "features": ["rwd", "tow-bar", "turbocharger", "lane assist", "pink wheels"], "error_message": "engine overheated", "timestamp": 1021532444000 }
-          { "index": {} }
-          { "features": ["pink wheels", "sunroof", "air condition"], "error_message": "compressor low pressure", "timestamp": 1020575254000 }
-          { "index": {} }
-          { "features": ["turbocharger", "pink wheels", "air condition" ], "error_message": "engine overheated", "timestamp": 1020322844000 }
-          { "index": {} }
-          { "features": ["awd", "heated steering wheel", "sunroof", "air condition"], "error_message": "compressor low pressure", "timestamp": 1020748054000 }
-          { "index": {} }
-          { "features": ["turbocharger", "sunroof", "autopilot", "air condition" ], "error_message": "compressor low pressure", "timestamp": 1020229654000 }
-          { "index": {} }
-          { "features": ["rwd", "turbocharger", "autopilot", "pink wheels"], "error_message": "engine overheated", "timestamp": 1022137244000 }
-          { "index": {} }
-          { "features": ["fwd", "tow-bar", "turbocharger", "pink wheels", "electric mirror", "heated steering wheel", "lane assist", "sunroof", "autopilot", "air condition" ], "error_message": "low tire pressure", "timestamp": 1020920854000 }
-  # same data, but stored in a flattened field
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-      indices.create:
-        index: store-flattened
-        body:
-          mappings:
-            properties:
-              data:
-                type: flattened
-              timestamp:
-                type: date
-
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-        Content-Type: application/json
-      bulk:
-        index: store-flattened
-        refresh: true
-        body: |
-          { "index": {} }
-          { "data": {"features": ["turbocharger", "electric mirror", "heated steering wheel", "matrix led", "gps", "sunroof", "autopilot", "air condition"], "error_message": "compressor low pressure"}, "timestamp": 1020229654000 }
-          { "index": {} }
-          { "data": {"features": ["pink wheels", "turbocharger", "lane assist", "matrix led", "coming home", "heated seats", "blind spot monitoring", "air condition"], "error_message": "engine overheated"}, "timestamp": 1020316054000 }
-          { "index": {} }
-          { "data": {"features": ["awd", "tow-bar", "matrix led", "gps", "sunroof", "air condition"], "error_message": "compressor low pressure"}, "timestamp": 1022742044000 }
-          { "index": {} }
-          { "data": {"features": ["rwd", "tow-bar", "turbocharger", "lane assist", "pink wheels"], "error_message": "engine overheated"}, "timestamp": 1021532444000 }
-          { "index": {} }
-          { "data": {"features": ["pink wheels", "sunroof", "air condition"], "error_message": "compressor low pressure"}, "timestamp": 1020575254000 }
-          { "index": {} }
-          { "data": {"features": ["turbocharger", "pink wheels", "air condition" ], "error_message": "engine overheated"}, "timestamp": 1020322844000 }
-          { "index": {} }
-          { "data": {"features": ["awd", "heated steering wheel", "sunroof", "air condition"], "error_message": "compressor low pressure"}, "timestamp": 1020748054000 }
-          { "index": {} }
-          { "data": {"features": ["turbocharger", "sunroof", "autopilot", "air condition" ], "error_message": "compressor low pressure"}, "timestamp": 1020229654000 }
-          { "index": {} }
-          { "data": {"features": ["rwd", "turbocharger", "autopilot", "pink wheels"], "error_message": "engine overheated"}, "timestamp": 1022137244000 }
-          { "index": {} }
-          { "data": {"features": ["fwd", "tow-bar", "turbocharger", "pink wheels", "electric mirror", "heated steering wheel", "lane assist", "sunroof", "autopilot", "air condition" ], "error_message": "low tire pressure"}, "timestamp": 1020920854000 }
-
-
----
-"Test frequent items array fields":
-
-  - do:
-      search:
-        index: store
-        body: >
-          {
-            "size": 0,
-            "aggs": {
-              "fi": {
-                "frequent_items": {
-                  "minimum_set_size": 3,
-                  "minimum_support": 0.3,
-                  "fields": [
-                    {"field": "features"},
-                    {"field": "error_message"}
-                  ]
-                }
-              }
-            }
-          }
-  - length: { aggregations.fi.buckets: 4 }
-  - match: { aggregations.fi.buckets.0.doc_count: 5 }
-  - match: { aggregations.fi.buckets.0.support: 0.5 }
-  - match: { aggregations.fi.buckets.0.key.error_message: ["compressor low pressure"] }
-  - match: { aggregations.fi.buckets.1.doc_count: 4 }
-  - match: { aggregations.fi.buckets.1.support: 0.4 }
-  - match: { aggregations.fi.buckets.1.key.error_message: ["engine overheated"] }
-
----
-"Test frequent items date format":
-
-  - do:
-      search:
-        index: store
-        body: >
-          {
-            "runtime_mappings": {
-              "weekday": {
-                "type": "keyword",
-                "script": {
-                "source": "emit(doc['timestamp'].value.format(DateTimeFormatter.ofPattern('EEE')));"
-                }
-              }
-            },
-            "size": 0,
-            "aggs": {
-              "fi": {
-                "frequent_items": {
-                  "minimum_set_size": 3,
-                  "minimum_support": 0.3,
-                  "fields": [
-                    {"field": "features"},
-                    {"field": "weekday"}
-                  ]
-                }
-              }
-            }
-          }
-  - length: { aggregations.fi.buckets: 3 }
-  - match: { aggregations.fi.buckets.0.doc_count: 5 }
-  - match: { aggregations.fi.buckets.0.support: 0.5 }
-  - match: { aggregations.fi.buckets.0.key.weekday: ["Thu"] }
-
-
----
-"Test frequent items date format 2":
-
-  - do:
-      search:
-        index: store
-        body: >
-          {
-            "runtime_mappings": {
-              "weekday": {
-                "type": "keyword",
-                "script": {
-                "source": "emit(doc['timestamp'].value.format(DateTimeFormatter.ofPattern('EEE')));"
-                }
-              }
-            },
-            "size": 0,
-            "aggs": {
-              "fi": {
-                "frequent_items": {
-                  "minimum_set_size": 2,
-                  "minimum_support": 0.3,
-                  "fields": [
-                    {"field": "error_message"},
-                    {"field": "weekday"}
-                  ]
-                }
-              }
-            }
-          }
-  - length: { aggregations.fi.buckets: 1 }
-  - match: { aggregations.fi.buckets.0.doc_count: 4 }
-  - match: { aggregations.fi.buckets.0.support: 0.4 }
-  - match: { aggregations.fi.buckets.0.key.weekday: ["Thu"] }
-  - match: { aggregations.fi.buckets.0.key.error_message: ["engine overheated"] }
-
----
-"Test frequent items array fields profile":
-
-  - do:
-      search:
-        index: store
-        body: >
-          {
-            "profile": true,
-            "size": 0,
-            "aggs": {
-              "fi": {
-                "frequent_items": {
-                  "minimum_set_size": 3,
-                  "minimum_support": 0.2,
-                  "fields": [
-                    {"field": "features"},
-                    {"field": "error_message"}
-                  ]
-                }
-              }
-            }
-          }
-  - length: { aggregations.fi.buckets: 10 }
-  - match: { aggregations.fi.buckets.0.doc_count: 5 }
-  - match: { aggregations.fi.buckets.0.support: 0.5 }
-  - match: { aggregations.fi.buckets.0.key.error_message: ["compressor low pressure"] }
-  - match: { aggregations.fi.buckets.1.doc_count: 4 }
-  - match: { aggregations.fi.buckets.1.support: 0.4 }
-  - match: { aggregations.fi.buckets.1.key.error_message: ["engine overheated"] }
-  - match: { aggregations.fi.buckets.1.support: 0.4 }
-  - match: { aggregations.fi.profile.unique_transactions_after_prune: 10 }
-  - match: { aggregations.fi.profile.unique_items_after_prune: 11 }
-
----
-"Test frequent items flattened fields":
-
-  - do:
-      search:
-        index: store-flattened
-        body: >
-          {
-            "size": 0,
-            "aggs": {
-              "fi": {
-                "frequent_items": {
-                  "minimum_set_size": 3,
-                  "minimum_support": 0.3,
-                  "fields": [
-                    {"field": "data.features"},
-                    {"field": "data.error_message"}
-                  ]
-                }
-              }
-            }
-          }
-  - length: { aggregations.fi.buckets: 4 }
-  - match: { aggregations.fi.buckets.0.doc_count: 5 }
-  - match: { aggregations.fi.buckets.0.support: 0.5 }
-  - match: { aggregations.fi.buckets.0.key.data\.error_message: ["compressor low pressure"] }
-  - match: { aggregations.fi.buckets.1.doc_count: 4 }
-  - match: { aggregations.fi.buckets.1.support: 0.4 }
-  - match: { aggregations.fi.buckets.1.key.data\.error_message: ["engine overheated"] }
-
----
-"Test frequent items as subagg":
-
-  - do:
-      search:
-        index: store
-        body: >
-          {
-            "size": 0,
-            "aggs": {
-              "filter_error": {
-                "filter": {
-                  "term": {
-                    "error_message": "compressor low pressure"
-                  }
-                },
-                "aggs": {
-                  "fi": {
-                    "frequent_items": {
-                      "minimum_set_size": 3,
-                      "minimum_support": 0.3,
-                      "fields": [
-                        {"field": "features"},
-                        {"field": "error_message"}
-                      ]
-                    }
-                  }
-                }
-              }
-            }
-          }
-  - length: { aggregations.filter_error.fi.buckets: 5 }
-  - match: { aggregations.filter_error.fi.buckets.0.doc_count: 5 }
-  - match: { aggregations.filter_error.fi.buckets.0.support: 1.0 }
-  - match: { aggregations.filter_error.fi.buckets.0.key.error_message: ["compressor low pressure"] }
-  - match: { aggregations.filter_error.fi.buckets.1.doc_count: 2 }
-  - match: { aggregations.filter_error.fi.buckets.1.support: 0.4 }
-  - match: { aggregations.filter_error.fi.buckets.0.key.error_message: ["compressor low pressure"] }
-
----
-"Test frequent items as multi-bucket subagg":
-
-  - do:
-      search:
-        index: store
-        body: >
-          {
-            "size": 0,
-            "aggs": {
-              "weekly": {
-                "date_histogram": {
-                  "field": "timestamp",
-                  "calendar_interval": "1w"
-                },
-                "aggs": {
-                  "fi": {
-                    "frequent_items": {
-                      "minimum_set_size": 3,
-                      "minimum_support": 0.3,
-                      "fields": [
-                        {"field": "features"},
-                        {"field": "error_message"}
-                      ]
-                    }
-                  }
-                }
-              }
-            }
-          }
-  - length: { aggregations.weekly.buckets: 5 }
-  - match: { aggregations.weekly.buckets.0.fi.buckets.0.doc_count: 3 }
-  - match: { aggregations.weekly.buckets.0.fi.buckets.0.support: 0.6 }
-  - match: { aggregations.weekly.buckets.0.fi.buckets.0.key.error_message: ["compressor low pressure"] }
-  - length: { aggregations.weekly.buckets.0.fi.buckets: 4 }
-  - match: { aggregations.weekly.buckets.2.fi.buckets.0.doc_count: 1 }
-
----
-"Test frequent items filter":
-
-  - do:
-      search:
-        index: store
-        body: >
-          {
-            "size": 0,
-            "aggs": {
-              "fi": {
-                "frequent_items": {
-                  "minimum_set_size": 3,
-                  "minimum_support": 0.3,
-                  "fields": [
-                    {"field": "features"},
-                    {"field": "error_message"}
-                  ],
-                  "filter": {
-                    "bool": {
-                      "must_not": {
-                        "term": {"features": "pink wheels"}
-                      }
-                    }
-                  }
-                }
-              }
-            }
-          }
-  - length: { aggregations.fi.buckets: 1 }
-  - match: { aggregations.fi.buckets.0.doc_count: 4 }
-  - match: { aggregations.fi.buckets.0.support: 0.4 }
-  - match: { aggregations.fi.buckets.0.key.error_message: ["compressor low pressure"] }
-
----
-"Test frequent items unsupported types":
-  - do:
-      catch: /Field \[geo_point\] of type \[geo_point\] is not supported for aggregation \[frequent_items\]/
-      search:
-        index: store
-        body: >
-          {
-            "size": 0,
-            "aggs": {
-              "fi": {
-                "frequent_items": {
-                  "minimum_set_size": 3,
-                  "minimum_support": 0.3,
-                  "fields": [
-                    {"field": "features"},
-                    {"field": "error_message"},
-                    {"field": "geo_point"}
-                  ]
-                }
-              }
-            }
-          }
-  - do:
-      catch: /Field \[histogram\] of type \[histogram\] is not supported for aggregation \[frequent_items\]/
-      search:
-        index: store
-        body: >
-          {
-            "size": 0,
-            "aggs": {
-              "fi": {
-                "frequent_items": {
-                  "minimum_set_size": 3,
-                  "minimum_support": 0.3,
-                  "fields": [
-                    {"field": "features"},
-                    {"field": "error_message"},
-                    {"field": "histogram"}
-                  ]
-                }
-              }
-            }
-          }
-
----
-"Test frequent items unsupported subaggs":
-  - do:
-      catch: /Aggregator \[fi\] of type \[frequent_items\] cannot accept sub-aggregations/
-      search:
-        index: store
-        body: >
-          {
-            "size": 0,
-            "aggs": {
-              "fi": {
-                "frequent_items": {
-                  "minimum_set_size": 3,
-                  "minimum_support": 0.3,
-                  "fields": [
-                    {"field": "features"},
-                    {"field": "error_message"}
-                  ]
-                },
-              "aggs": {
-                "sum": {
-                  "sum": {
-                    "field": "some_field"
-                  }
-                }
-              }
-            }
-            }
-          }
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_crud.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_crud.yml
index 6113569f6303..3c4439444d1a 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_crud.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_crud.yml
@@ -1670,3 +1670,35 @@
           {
             "allow_no_match" : true
           }
+
+---
+"Test job with rule referencing missing filter":
+
+  - do:
+      ml.put_job:
+        job_id: jobs-crud-rule-missing-filter
+        body:  >
+          {
+            "analysis_config": {
+              "detectors": [
+                {
+                  "function": "count",
+                  "by_field_name": "country",
+                  "custom_rules": [
+                    {
+                      "actions": ["skip_result"],
+                      "scope": {
+                        "country": {"filter_id": "safe_countries"}
+                      }
+                    }
+                  ]
+                }
+              ]
+            },
+            "data_description" : {}
+          }
+
+  - do:
+      catch: /Unable to find filter \[safe_countries\]/
+      ml.open_job:
+        job_id: jobs-crud-rule-missing-filter
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_stats.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_stats.yml
index 798288d72700..6aab1fb9e894 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_stats.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/jobs_get_stats.yml
@@ -397,3 +397,57 @@ setup:
   - is_false: jobs.1.timing_stats.maximum_bucket_processing_time_ms
   - is_false: jobs.1.timing_stats.average_bucket_processing_time_ms
   - is_false: jobs.1.timing_stats.exponential_average_bucket_processing_time_ms
+
+---
+"Test closed results index":
+
+  - skip:
+      features:
+        - "warnings"
+
+  - do:
+      warnings:
+        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
+      ml.post_data:
+        job_id: job-stats-test
+        body: >
+          {"airline":"AAL","responsetime":"132.2046","time":"1403481600"}
+          {"airline":"JZA","responsetime":"990.4628","time":"1403481600"}
+
+  - do:
+      ml.close_job:
+        job_id: jobs-get-stats-datafeed-job
+  - match: { closed: true }
+
+  - do:
+      ml.close_job:
+        job_id: job-stats-test
+  - match: { closed: true }
+
+  - do:
+      ml.get_job_stats: {}
+  - length: { jobs : 2 }
+
+  - do:
+      xpack.usage: {}
+  - match: { ml.available: true }
+  - match: { ml.enabled: true }
+  - match: { ml.jobs.closed.count: 2 }
+
+  - do:
+      indices.close:
+        index: .ml-anomalies-shared
+
+  # With the index closed the low level ML API reports a problem
+  - do:
+      catch: /type=cluster_block_exception, reason=index \[.ml-anomalies-shared\] blocked by. \[FORBIDDEN\/.\/index closed\]/
+      ml.get_job_stats: {}
+
+  # But the high level X-Pack API returns what it can - we do this
+  # so that corruption to ML doesn't blind observers of the general
+  # cluster status
+  - do:
+      xpack.usage: {}
+  - match: { ml.available: true }
+  - match: { ml.enabled: true }
+  - is_false: ml.jobs.closed.count
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/semantic_search.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/semantic_search.yml
deleted file mode 100644
index f4f9cf18f3d7..000000000000
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/semantic_search.yml
+++ /dev/null
@@ -1,256 +0,0 @@
-# This test uses the simple text embedding model from
-# SemanticSearchIT.java to create the dense vectors. The
-# model creates random tensors seeded by the hash of
-# input string, we expect the same input to produce
-# the same output. However, not all platforms produce
-# the same random tensor for the same seed. The model
-# creates different tensors on different platforms and
-# OS's, therefore the top search hits will be different
-# on different platforms meaning the test cannot assert
-# on the top hits. SemanticSearchIT does not suffer
-# from this limitation as it dynamically creates the
-# embeddings and documents rather than using hard-coded
-# values. It is important to keep this test for the
-# language clients to snoop the response format but the
-# assertions can be skipped here because of the test
-# coverage in SemanticSearchIT
-
-setup:
-  - skip:
-      features: headers
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-      indices.create:
-        index: embedded_text
-        body:
-          settings:
-            number_of_replicas: 0
-          mappings:
-            properties:
-              source_text:
-                type: keyword
-              embedding:
-                type: dense_vector
-                dims: 100
-                index: true
-                similarity: cosine
-
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-      indices.create:
-        index: embedded_text_10_dims
-        body:
-          settings:
-            number_of_replicas: 0
-          mappings:
-            properties:
-              source_text:
-                type: keyword
-              embedding:
-                type: dense_vector
-                dims: 10
-                index: true
-                similarity: cosine
-
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-      ml.put_trained_model:
-        model_id: "text_embedding_model"
-        body: >
-          {
-            "description": "simple model for testing",
-            "model_type": "pytorch",
-            "inference_config": {
-              "text_embedding": {
-                "tokenization": {
-                  "bert": {
-                    "with_special_tokens": false
-                  }
-                }
-              }
-            }
-          }
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-      ml.put_trained_model_vocabulary:
-        model_id: "text_embedding_model"
-        body: >
-          { "vocabulary": ["[PAD]", "[UNK]", "these", "are", "my", "words", "the", "washing", "machine", "is", "leaking", "octopus", "comforter", "smells"] }
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-      ml.put_trained_model_definition_part:
-        model_id: "text_embedding_model"
-        part: 0
-        body: >
-          {
-            "total_definition_length":1694,
-            "definition": "UEsDBAAACAgAAAAAAAAAAAAAAAAAAAAAAAAUAA4Ac2ltcGxlbW9kZWwvZGF0YS5wa2xGQgoAWlpaWlpaWlpaWoACY19fdG9yY2hfXwpUaW55VGV4dEVtYmVkZGluZwpxACmBfShYCAAAAHRyYWluaW5ncQGJWBYAAABfaXNfZnVsbF9iYWNrd2FyZF9ob29rcQJOdWJxAy5QSwcIsFTQsFgAAABYAAAAUEsDBBQACAgIAAAAAAAAAAAAAAAAAAAAAAAdAB0Ac2ltcGxlbW9kZWwvY29kZS9fX3RvcmNoX18ucHlGQhkAWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWoWPMWvDMBCF9/yKGy1IQ7Ia0q1j2yWbMYdsnWphWWd0Em3+fS3bBEopXd99j/dd77UI3Fy43+grvUwdGePCR/XKJntS9QEAcdZRT5QoCiJcoWnXtMvW/ohS1C4sZaihY/YFcoI2e4+d7sdPHQ0OzONyf5+T46B9U8DSNWTBcixMJeRtvQwkjv2AePpld1wKAC7MOaEzUsONgnDc4sQjBUz3mbbbY2qD2usbB9rQmcWV47/gOiVIReAvUsHT8y5S7yKL/mnSIWuPQmSqLRm0DJWkWD0eUEqtjUgpx7AXow6mai5HuJzPrTp8A1BLBwiD/6yJ6gAAAKkBAABQSwMEFAAICAgAAAAAAAAAAAAAAAAAAAAAACcAQQBzaW1wbGVtb2RlbC9jb2RlL19fdG9yY2hfXy5weS5kZWJ1Z19wa2xGQj0AWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWo2Qz0rDQBDGk/5RmjfwlmMCbWivBZ9gWL0IFkRCdLcmmOwmuxu0N08O3r2rCOrdx9CDgm/hWUUQMdugzUk6LCwzv++bGeak5YE1saoorNgCCwsbzFc9sm1PvivQo2zqToU8iiT1FEunfadXRcLzUocJVWN3i3ElZF3W4pDxUM9yVrPNXCeCR+lOLdp1190NwVktzoVKDF5COh+nQpbtsX+0/tjpOWYJuR8HMuJUZEEW8TJKQ8UY9eJIxZ7S0vvb3vf9yiCZLiV3Fz5v1HdHw6HvFK3JWnUElWR5ygbz8TThB4NMUJYG+axowyoWHbiHBwQbSWbHHXiEJ4QWkmOTPMMLQhvJaZOgSX49Z3a8uPq5Ia/whtBBctEkl4a8wwdCF8lVk1wb8glfCCtIbprkttntrkF00Q1+AFBLBwi4BIswOAEAAP0BAABQSwMEAAAICAAAAAAAAAAAAAAAAAAAAAAAABkAQQBzaW1wbGVtb2RlbC9jb25zdGFudHMucGtsRkI9AFpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlqAAikuUEsHCG0vCVcEAAAABAAAAFBLAwQAAAgIAAAAAAAAAAAAAAAAAAAAAAAAEwA7AHNpbXBsZW1vZGVsL3ZlcnNpb25GQjcAWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWjMKUEsHCNGeZ1UCAAAAAgAAAFBLAQIAAAAACAgAAAAAAACwVNCwWAAAAFgAAAAUAAAAAAAAAAAAAAAAAAAAAABzaW1wbGVtb2RlbC9kYXRhLnBrbFBLAQIAABQACAgIAAAAAACD/6yJ6gAAAKkBAAAdAAAAAAAAAAAAAAAAAKgAAABzaW1wbGVtb2RlbC9jb2RlL19fdG9yY2hfXy5weVBLAQIAABQACAgIAAAAAAC4BIswOAEAAP0BAAAnAAAAAAAAAAAAAAAAAPoBAABzaW1wbGVtb2RlbC9jb2RlL19fdG9yY2hfXy5weS5kZWJ1Z19wa2xQSwECAAAAAAgIAAAAAAAAbS8JVwQAAAAEAAAAGQAAAAAAAAAAAAAAAADIAwAAc2ltcGxlbW9kZWwvY29uc3RhbnRzLnBrbFBLAQIAAAAACAgAAAAAAADRnmdVAgAAAAIAAAATAAAAAAAAAAAAAAAAAFQEAABzaW1wbGVtb2RlbC92ZXJzaW9uUEsGBiwAAAAAAAAAHgMtAAAAAAAAAAAABQAAAAAAAAAFAAAAAAAAAGoBAAAAAAAA0gQAAAAAAABQSwYHAAAAADwGAAAAAAAAAQAAAFBLBQYAAAAABQAFAGoBAADSBAAAAAA=",
-            "total_parts": 1
-          }
-
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-        Content-Type: application/json
-      bulk:
-        index: embedded_text
-        refresh: true
-        body: |
-          {"index": {}}
-          {"source_text": "the octopus comforter smells", "embedding":[0.3925197124481201, 0.9145996570587158, 0.01372915506362915, 0.9987854957580566, 0.3240084648132324, 0.6188188195228577, 0.926924467086792, 0.12143599987030029, 0.175662100315094, 0.16076070070266724, 0.7671306133270264, 0.9518267512321472, 0.4557478427886963, 0.5410670638084412, 0.7594802975654602, 0.5035953521728516, 0.4115469455718994, 0.038427770137786865, 0.5419668555259705, 0.6362232565879822, 0.17554593086242676, 0.01821446418762207, 0.2931918501853943, 0.294437050819397, 0.6901726722717285, 0.1679999828338623, 0.7995195984840393, 0.8781598210334778, 0.18507736921310425, 0.8614458441734314, 0.690071702003479, 0.7859554886817932, 0.803643524646759, 0.0048452019691467285, 0.19700628519058228, 0.22210919857025146, 0.7043975591659546, 0.6320799589157104, 0.542057991027832, 0.8704766035079956, 0.32195907831192017, 0.7272325158119202, 0.4066658020019531, 0.89588862657547, 0.7947880029678345, 0.06543421745300293, 0.2873639464378357, 0.8773637413978577, 0.36480581760406494, 0.692948043346405, 0.19171112775802612, 0.14275449514389038, 0.17054951190948486, 0.8969640135765076, 0.39838290214538574, 0.26756417751312256, 0.5369327664375305, 0.4736328721046448, 0.21181154251098633, 0.2695402503013611, 0.8651300072669983, 0.8051849603652954, 0.7073134779930115, 0.5963589549064636, 0.09601861238479614, 0.5362404584884644, 0.23020631074905396, 0.8515381813049316, 0.5730932354927063, 0.7235705256462097, 0.08228331804275513, 0.5840849280357361, 0.6030013561248779, 0.2084050178527832, 0.7312950491905212, 0.6159517168998718, 0.6482340693473816, 0.07220339775085449, 0.5136227607727051, 0.9152160286903381, 0.8169018030166626, 0.15515869855880737, 0.7978536486625671, 0.564482569694519, 0.4757157564163208, 0.2718064785003662, 0.6910138726234436, 0.5675734877586365, 0.702862024307251, 0.19079893827438354, 0.8995556235313416, 0.4988499879837036, 0.6378234028816223, 0.2683940529823303, 0.21990180015563965, 0.8442690372467041, 0.8502047061920166, 0.9857811331748962, 0.3549607992172241, 0.7605474591255188]}
-          {"index": {}}
-          {"source_text": "the machine is leaking", "embedding":[0.09775793552398682, 0.9594467282295227, 0.7915146946907043, 0.9140479564666748, 0.5148435235023499, 0.8556410670280457, 0.6022665500640869, 0.05222177505493164, 0.9821935296058655, 0.49276530742645264, 0.23147249221801758, 0.2428399920463562, 0.3865380883216858, 0.5778483748435974, 0.5600519776344299, 0.9427472352981567, 0.48832541704177856, 0.8807493448257446, 0.32909590005874634, 0.8452557325363159, 0.811530590057373, 0.13344216346740723, 0.15256845951080322, 0.5025331974029541, 0.4288772940635681, 0.6590417623519897, 0.9282752871513367, 0.8842046856880188, 0.7873250842094421, 0.356731653213501, 0.9959152936935425, 0.07572609186172485, 0.5062583088874817, 0.36245888471603394, 0.6189196705818176, 0.7766605019569397, 0.5198523998260498, 0.7379586100578308, 0.0553441047668457, 0.5035901665687561, 0.24139636754989624, 0.10798943042755127, 0.272808313369751, 0.38171595335006714, 0.24275553226470947, 0.956981897354126, 0.8182021379470825, 0.9383817315101624, 0.06551980972290039, 0.6892690658569336, 0.7068917751312256, 0.5184322595596313, 0.6103079319000244, 0.7020677328109741, 0.7181660532951355, 0.6477184295654297, 0.26282840967178345, 0.9316624402999878, 0.8318467140197754, 0.1487215757369995, 0.39937925338745117, 0.6842989921569824, 0.3496543765068054, 0.6008991003036499, 0.9530165791511536, 0.4209877848625183, 0.5675879716873169, 0.7883706092834473, 0.9547191858291626, 0.6292906403541565, 0.49566715955734253, 0.6907342672348022, 0.0834314227104187, 0.19785481691360474, 0.4896165728569031, 0.8460168838500977, 0.9680339097976685, 0.43386441469192505, 0.7068926095962524, 0.19123870134353638, 0.5661664009094238, 0.610595166683197, 0.23599380254745483, 0.2831611633300781, 0.7919651865959167, 0.0018386244773864746, 0.15559959411621094, 0.4622604250907898, 0.02038663625717163, 0.42241227626800537, 0.4200526475906372, 0.1223069429397583, 0.7035380005836487, 0.09902423620223999, 0.7804107666015625, 0.05339455604553223, 0.6485095024108887, 0.29347676038742065, 0.9716366529464722, 0.30257928371429443]}
-          {"index": {}}
-          {"source_text": "these are my words", "embedding":[0.7000167369842529, 0.590781033039093, 0.009879708290100098, 0.7874260544776917, 0.797156572341919, 0.1791083812713623, 0.07826781272888184, 0.25102007389068604, 0.09334254264831543, 0.3819708824157715, 0.7312374711036682, 0.02819347381591797, 0.20099765062332153, 0.7702597975730896, 0.9443559050559998, 0.35520339012145996, 0.25699591636657715, 0.5596823692321777, 0.23947590589523315, 0.47478222846984863, 0.23411548137664795, 0.9809996485710144, 0.3806597590446472, 0.5006771087646484, 0.5724453926086426, 0.21510547399520874, 0.07062828540802002, 0.9858258962631226, 0.9636645317077637, 0.36034029722213745, 0.07260054349899292, 0.06882566213607788, 0.18354403972625732, 0.06756395101547241, 0.5749042630195618, 0.05275309085845947, 0.1865217685699463, 0.5852730870246887, 0.1086682677268982, 0.10090464353561401, 0.32582908868789673, 0.5494027733802795, 0.873362123966217, 0.02236837148666382, 0.37973177433013916, 0.5556552410125732, 0.5083678364753723, 0.8081125020980835, 0.09164196252822876, 0.2207810878753662, 0.8086426258087158, 0.271828293800354, 0.5981417298316956, 0.7745779156684875, 0.40872830152511597, 0.6035888195037842, 0.5598325133323669, 0.19086670875549316, 0.02406853437423706, 0.8299782872200012, 0.4994274377822876, 0.0300295352935791, 0.47190529108047485, 0.8889331817626953, 0.34195321798324585, 0.9380808472633362, 0.4418332576751709, 0.5789303779602051, 0.0526617169380188, 0.7349719405174255, 0.44571834802627563, 0.6602563261985779, 0.3819742202758789, 0.16881734132766724, 0.45588219165802, 0.028081774711608887, 0.6681976914405823, 0.8183007836341858, 0.7887755632400513, 0.4506028890609741, 0.8040162324905396, 0.431918203830719, 0.7408918738365173, 0.39756304025650024, 0.7438145875930786, 0.6120601892471313, 0.5724676251411438, 0.08701330423355103, 0.18344634771347046, 0.7226220369338989, 0.3648560643196106, 0.9813777208328247, 0.2615315318107605, 0.9847549796104431, 0.32967478036880493, 0.47099196910858154, 0.3591546416282654, 0.4132147431373596, 0.48631107807159424, 0.04420149326324463]}
-          {"index": {}}
-          {"source_text": "washing machine", "embedding":[0.7044712901115417, 0.12284207344055176, 0.5008929967880249, 0.04643195867538452, 0.3666788339614868, 0.26660799980163574, 0.24114298820495605, 0.0761682391166687, 0.5294214487075806, 0.16935181617736816, 0.6257967948913574, 0.2804388999938965, 0.6417903900146484, 0.169958233833313, 0.4216839075088501, 0.6773303747177124, 0.9472144842147827, 0.21874648332595825, 0.5095921754837036, 0.839306116104126, 0.6176233291625977, 0.5847064852714539, 0.6748610734939575, 0.3264034390449524, 0.4112023115158081, 0.13818275928497314, 0.08356589078903198, 0.4147903323173523, 0.5626787543296814, 0.7167286276817322, 0.6314535737037659, 0.23092854022979736, 0.34547603130340576, 0.7425565719604492, 0.2837678790092468, 0.47037917375564575, 0.1555209755897522, 0.5618507266044617, 0.2076261043548584, 0.3026384711265564, 0.04561811685562134, 0.1691250205039978, 0.2504339814186096, 0.5350574851036072, 0.26857447624206543, 0.23607933521270752, 0.16938960552215576, 0.23708534240722656, 0.026302993297576904, 0.16901731491088867, 0.2847784757614136, 0.944273829460144, 0.28171658515930176, 0.9864799380302429, 0.6811433434486389, 0.9383156895637512, 0.5682582259178162, 0.14361613988876343, 0.7900274395942688, 0.27808505296707153, 0.05677521228790283, 0.08594226837158203, 0.6450491547584534, 0.06500720977783203, 0.36045730113983154, 0.1987738013267517, 0.07287931442260742, 0.5315744280815125, 0.04742676019668579, 0.7842378616333008, 0.0881078839302063, 0.7612627744674683, 0.2528950572013855, 0.27305954694747925, 0.03027820587158203, 0.4686838984489441, 0.13311690092086792, 0.048372089862823486, 0.808062732219696, 0.44010263681411743, 0.5726178288459778, 0.15828031301498413, 0.4597446322441101, 0.6375324130058289, 0.8452948927879333, 0.9763500690460205, 0.5094607472419739, 0.3535742163658142, 0.664739191532135, 0.40749913454055786, 0.8537857532501221, 0.5830079913139343, 0.7949922680854797, 0.6309236288070679, 0.07258343696594238, 0.1224660873413086, 0.24250483512878418, 0.36189037561416626, 0.5156043171882629, 0.1819135546684265]}
-          {"index": {}}
-          {"source_text": "washing machine smells", "embedding":[0.7249823808670044, 0.3981819152832031, 0.4572623372077942, 0.7442894577980042, 0.15898281335830688, 0.6481881737709045, 0.1513708233833313, 0.8945682644844055, 0.7708938121795654, 0.5494217276573181, 0.48253726959228516, 0.39402270317077637, 0.6369197368621826, 0.7152248024940491, 0.6326345205307007, 0.7362181544303894, 0.350342333316803, 0.16101288795471191, 0.4180338382720947, 0.04114532470703125, 0.002633512020111084, 0.20396709442138672, 0.8963556885719299, 0.1552276611328125, 0.7476853728294373, 0.9651047587394714, 0.7527561187744141, 0.7041972279548645, 0.12461084127426147, 0.6282403469085693, 0.9631509184837341, 0.16590750217437744, 0.4101366400718689, 0.31320667266845703, 0.13579899072647095, 0.2895740270614624, 0.9905323386192322, 0.02118372917175293, 0.637545645236969, 0.5133231282234192, 0.679695188999176, 0.04641437530517578, 0.21913814544677734, 0.16534924507141113, 0.02987360954284668, 0.14805591106414795, 0.16874665021896362, 0.9378783702850342, 0.8607399463653564, 0.7287217974662781, 0.5402306318283081, 0.9973209500312805, 0.26169413328170776, 0.3835873603820801, 0.1874808669090271, 0.8038567304611206, 0.18557673692703247, 0.8631893992424011, 0.7676172256469727, 0.3599127531051636, 0.48698097467422485, 0.926689088344574, 0.6542723774909973, 0.49722349643707275, 0.7027173638343811, 0.13385021686553955, 0.9873734712600708, 0.17187494039535522, 0.7995050549507141, 0.5259199142456055, 0.33804380893707275, 0.21665722131729126, 0.952264130115509, 0.8337767720222473, 0.879487156867981, 0.5553549528121948, 0.6160674095153809, 0.1315295696258545, 0.8010737895965576, 0.834412693977356, 0.20340144634246826, 0.8993185758590698, 0.6493895649909973, 0.9454924464225769, 0.38529330492019653, 0.6891772150993347, 0.5530646443367004, 0.18555349111557007, 0.8361382484436035, 0.11815804243087769, 0.38942235708236694, 0.945141613483429, 0.6417409181594849, 0.39776402711868286, 0.5133314728736877, 0.5431299805641174, 0.2615429759025574, 0.8987119793891907, 0.023733675479888916, 0.4941052794456482]}
-          {"index": {}}
-          {"source_text": "my words", "embedding":[0.19087255001068115, 0.5498749017715454, 0.9536173939704895, 0.25011056661605835, 0.37642204761505127, 0.18271470069885254, 0.670674741268158, 0.5553990006446838, 0.3306507468223572, 0.3368762731552124, 0.053364574909210205, 0.047215282917022705, 0.4221981167793274, 0.7591024041175842, 0.998794436454773, 0.6113318204879761, 0.8178470730781555, 0.8554672598838806, 0.40100908279418945, 0.6486459374427795, 0.804382860660553, 0.6775466799736023, 0.2916865944862366, 0.7019925117492676, 0.9812073707580566, 0.4414554834365845, 0.08203905820846558, 0.9167835116386414, 0.3082762360572815, 0.5454868674278259, 0.6665160655975342, 0.06828844547271729, 0.36014634370803833, 0.01810687780380249, 0.2640475630760193, 0.1856365203857422, 0.4734996557235718, 0.8153479695320129, 0.9614933133125305, 0.4851576089859009, 0.003343045711517334, 0.17352384328842163, 0.26423048973083496, 0.24217921495437622, 0.5694647431373596, 0.8538861274719238, 0.06464511156082153, 0.038984060287475586, 0.7695011496543884, 0.008188009262084961, 0.3858819007873535, 0.7950196862220764, 0.7225212454795837, 0.3982154130935669, 0.4996080994606018, 0.28709208965301514, 0.6753579378128052, 0.6779837608337402, 0.4815831184387207, 0.27917128801345825, 0.8400004506111145, 0.9022405743598938, 0.8253144025802612, 0.6251398324966431, 0.25444501638412476, 0.7694959044456482, 0.006821691989898682, 0.7958594560623169, 0.9144708514213562, 0.8688076138496399, 0.9641174077987671, 0.44437146186828613, 0.06135892868041992, 0.2638128399848938, 0.05436718463897705, 0.9926314353942871, 0.8661795854568481, 0.9176243543624878, 0.5521496534347534, 0.6017677783966064, 0.22096896171569824, 0.7030748128890991, 0.16923701763153076, 0.8178754448890686, 0.47008246183395386, 0.28875309228897095, 0.14314061403274536, 0.3431167006492615, 0.9301973581314087, 0.5416158437728882, 0.563427209854126, 0.7897542119026184, 0.2761036157608032, 0.16855067014694214, 0.42684781551361084, 0.7562968730926514, 0.2551668882369995, 0.7754542827606201, 0.218039870262146, 0.7080662846565247]}
-
-  - do:
-      headers:
-        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
-        Content-Type: application/json
-      indices.forcemerge:
-        index: embedded_text
-        max_num_segments: 1
-
----
-"Test semantic search":
-  - do:
-      ml.start_trained_model_deployment:
-        model_id: text_embedding_model
-        wait_for: started
-  - match: {assignment.assignment_state: started}
-
-  - do:
-      semantic_search:
-        index: embedded_text
-        body:
-          model_id: text_embedding_model
-          query_string: "the octopus comforter smells"
-          knn:
-            field: embedding
-            k: 3
-            num_candidates: 10
-  - gte: { inference_took: 0 }
-  - match: { hits.total.value: 3 }
-    # See comment at the top of the file for the reason why
-    # the test cannot match on the text field
-
-  - do:
-      semantic_search:
-        index: embedded_text
-        body:
-          model_id: text_embedding_model
-          query_string: "the octopus comforter smells"
-          text_embedding_config:
-            tokenization:
-              bert:
-                truncate: none
-          knn:
-            field: embedding
-            k: 3
-            num_candidates: 10
-  - gte: { inference_took: 0 }
-  - match: { hits.total.value: 3 }
-    # See comment at the top of the file for the reason why
-    # the test cannot match on the text field
-
----
-"Knn field is not a vector":
-  - do:
-      ml.start_trained_model_deployment:
-        model_id: text_embedding_model
-        wait_for: started
-  - match: {assignment.assignment_state: started}
-
-  - do:
-      catch: bad_request
-      semantic_search:
-        index: embedded_text
-        body:
-          model_id: text_embedding_model
-          query_string: "kids pyjamas with picture of a tractor"
-          knn:
-            field: source_text
-            k: 2
-            num_candidates: 3
-
----
-"Text embedding has the wrong number of dimensions":
-  - do:
-      ml.start_trained_model_deployment:
-        model_id: text_embedding_model
-        wait_for: started
-  - match: {assignment.assignment_state: started}
-
-  - do:
-      catch: bad_request
-      semantic_search:
-        index: embedded_text_10_dims
-        body:
-          model_id: text_embedding_model
-          query_string: "kids pyjamas with picture of a tractor"
-          knn:
-            field: embedding
-            k: 2
-            num_candidates: 3
-
----
-"Model not deployed":
-  - do:
-      catch: conflict
-      semantic_search:
-        index: embedded_text
-        body:
-          model_id: text_embedding_model
-          query_string: "kids pyjamas with picture of a tractor"
-          knn:
-            field: embedding
-            k: 2
-            num_candidates: 3
-
----
-"Unknown model":
-  - do:
-      catch: missing
-      semantic_search:
-        index: embedded_text
-        body:
-          model_id: missing_model
-          query_string: "kids pyjamas with picture of a tractor"
-          knn:
-            field: embedding
-            k: 2
-            num_candidates: 3
-
----
-"missing query string":
-  - do:
-      catch: bad_request
-      semantic_search:
-        index: embedded_text
-        body:
-          model_id: text_embedding_model
-          knn:
-            field: embedding
-            k: 2
-            num_candidates: 3
-
----
-"missing knn":
-  - do:
-      catch: bad_request
-      semantic_search:
-        index: embedded_text
-        body:
-          model_id: text_embedding_model
-          query_string: "kids pyjamas with picture of a tractor"
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/monitoring/bulk/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/monitoring/bulk/10_basic.yml
index 278080788bf3..04c7c0011e6b 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/monitoring/bulk/10_basic.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/monitoring/bulk/10_basic.yml
@@ -1,14 +1,20 @@
 ---
 "Bulk indexing of monitoring data":
-
   - skip:
-      version: "all"
-      reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/30101"
+      features: ["allowed_warnings"]
+
+  - do:
+      allowed_warnings:
+        - "[xpack.monitoring.collection.enabled] setting was deprecated in Elasticsearch and will be removed in a future release."
+      cluster.put_settings:
+        body:
+          persistent:
+            xpack.monitoring.collection.enabled: true
 
   - do:
       monitoring.bulk:
         system_id:          "kibana"
-        system_api_version: "6"
+        system_api_version: "7"
         interval:           "10s"
         body:
           - index:
@@ -44,15 +50,14 @@
   - do:
       monitoring.bulk:
         system_id:          "kibana"
-        system_api_version: "6"
+        system_api_version: "7"
         interval:           "123456ms"
-        type:               "default_type"
         body:
-          - '{"index": {}}'
+          - '{"index": {"_type":"default_type"}}'
           - '{"field_1": "value_1"}'
           - '{"index": {"_type": "custom_type"}}'
           - '{"field_1": "value_2"}'
-          - '{"index": {}}'
+          - '{"index": {"_type":"default_type"}}'
           - '{"field_1": "value_3"}'
           - '{"index": {"_index": "_data", "_type": "kibana"}}'
           - '{"field_1": "value_4"}'
@@ -93,15 +98,14 @@
   - do:
       monitoring.bulk:
         system_id:          "kibana"
-        system_api_version: "2"
+        system_api_version: "6"
         interval:           "10000ms"
-        type:               "default_type"
         body:
-          - '{"index": {}}'
+          - '{"index": {"_type": "default_type"}}'
           - '{"field_1": "value_1"}'
           - '{"index": {"_type": "custom_type"}}'
           - '{"field_1": "value_2"}'
-          - '{"index": {}}'
+          - '{"index": {"_type": "default_type"}}'
           - '{"field_1": "value_3"}'
           - '{"index": {"_index": "_data", "_type": "kibana"}}'
           - '{"field_1": "value_4"}'
@@ -139,11 +143,10 @@
   - do:
       catch: bad_request
       monitoring.bulk:
-        system_api_version: "6"
+        system_api_version: "7"
         interval:           "10s"
-        type:               "default_type"
         body:
-          - '{"index": {}}'
+          - '{"index": {"_type": "default_type"}}'
           - '{"field_1": "value_1"}'
 
   # Missing a system_api_version causes it to fail
@@ -152,9 +155,8 @@
       monitoring.bulk:
         system_id:          "kibana"
         interval:           "10s"
-        type:               "default_type"
         body:
-          - '{"index": {}}'
+          - '{"index": {"_type": "default_type"}}'
           - '{"field_1": "value_1"}'
 
   # Missing an interval causes it to fail
@@ -162,24 +164,28 @@
       catch: bad_request
       monitoring.bulk:
         system_id:          "kibana"
-        system_api_version: "6"
-        type:               "default_type"
+        system_api_version: "7"
         body:
-          - '{"index": {}}'
+          - '{"index": {"_type": "default_type"}}'
           - '{"field_1": "value_1"}'
 
 ---
 "Bulk indexing of monitoring data on closed indices should throw an export exception":
-
   - skip:
-      version: "all"
-      reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/30101"
       features: ["allowed_warnings"]
 
+  - do:
+      allowed_warnings:
+        - "[xpack.monitoring.collection.enabled] setting was deprecated in Elasticsearch and will be removed in a future release."
+      cluster.put_settings:
+        body:
+          persistent:
+            xpack.monitoring.collection.enabled: true
+
   - do:
       monitoring.bulk:
         system_id:          "beats"
-        system_api_version: "6"
+        system_api_version: "7"
         interval:           "5s"
         body:
           - index:
@@ -214,7 +220,7 @@
       catch: /export_exception/
       monitoring.bulk:
         system_id:          "beats"
-        system_api_version: "6"
+        system_api_version: "7"
         interval:           "5s"
         body:
           - index:
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/monitoring/bulk/20_privileges.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/monitoring/bulk/20_privileges.yml
index 5d4a702ddeee..93e5c44b83e2 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/monitoring/bulk/20_privileges.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/monitoring/bulk/20_privileges.yml
@@ -76,9 +76,15 @@ teardown:
 ---
 "Monitoring Bulk API":
   - skip:
-      features: catch_unauthorized
-      version: "all"
-      reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/30101"
+      features: ["catch_unauthorized", "allowed_warnings"]
+
+  - do:
+      allowed_warnings:
+        - "[xpack.monitoring.collection.enabled] setting was deprecated in Elasticsearch and will be removed in a future release."
+      cluster.put_settings:
+        body:
+          persistent:
+            xpack.monitoring.collection.enabled: true
 
   - do:
       headers:
@@ -86,7 +92,7 @@ teardown:
         Authorization: "Basic bG9nc3Rhc2hfYWdlbnQ6czNrcml0LXBhc3N3b3Jk"
       monitoring.bulk:
         system_id:          "logstash"
-        system_api_version: "6"
+        system_api_version: "7"
         interval:           "10s"
         body:
           - index:
@@ -132,7 +138,7 @@ teardown:
           - metric:
               queue:  10
   - match: { "error.type": "security_exception" }
-  - match: { "error.reason": "action [cluster:admin/xpack/monitoring/bulk] is unauthorized for user [unknown_agent]" }
+  - match: { "error.reason": "action [cluster:admin/xpack/monitoring/bulk] is unauthorized for user [unknown_agent] with effective roles [unknown_agent_role], this action is granted by the cluster privileges [manage,all]" }
 
   - do:
       indices.refresh: {}
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/100_geo_grid_ingest.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/100_geo_grid_ingest.yml
new file mode 100644
index 000000000000..77252887a23e
--- /dev/null
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/100_geo_grid_ingest.yml
@@ -0,0 +1,465 @@
+---
+teardown:
+  - do:
+      ingest.delete_pipeline:
+        id: "my_pipeline"
+        ignore: 404
+
+---
+"Test geo_grid on geotile using default target format":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body: >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "geo_grid": {
+                  "field" : "geocell",
+                  "tile_type": "geotile"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        pipeline: "my_pipeline"
+        body: >
+          {
+            "geocell": "4/8/5"
+          }
+
+  - do:
+      get:
+        index: test
+        id: "1"
+  - match:
+      _source:
+        geocell:
+          type: "Envelope"
+          coordinates:
+            - [ 0.0, 55.77657301866769 ]
+            - [ 22.5, 40.979898069620134 ]
+
+---
+"Test geo_grid on geotile using WKT":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body: >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "geo_grid": {
+                  "field" : "geocell",
+                  "tile_type": "geotile",
+                  "target_format": "wkt"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        pipeline: "my_pipeline"
+        body: >
+          {
+            "geocell": "4/8/5"
+          }
+
+  - do:
+      get:
+        index: test
+        id: "1"
+  - match: { _source.geocell: "BBOX (0.0, 22.5, 55.77657301866769, 40.979898069620134)" }
+
+---
+"Test geo_grid on geohash using WKT":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body: >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "geo_grid": {
+                  "field" : "geocell",
+                  "tile_type": "geohash",
+                  "target_format": "wkt"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        pipeline: "my_pipeline"
+        body: >
+          {
+            "geocell": "u0"
+          }
+
+  - do:
+      get:
+        index: test
+        id: "1"
+  - match: { _source.geocell: "BBOX (0.0, 11.25, 50.625, 45.0)" }
+
+---
+"Test geo_grid on geohex using WKT":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body: >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "geo_grid": {
+                  "field" : "geocell",
+                  "tile_type": "geohex",
+                  "target_format": "wkt"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        pipeline: "my_pipeline"
+        body: >
+          {
+            "geocell": "811fbffffffffff"
+          }
+
+  - do:
+      get:
+        index: test
+        id: "1"
+  - match: { _source.geocell: "POLYGON ((1.1885095294564962 49.470279179513454, 2.0265689212828875 45.18424864858389, 7.509948452934623 43.786609335802495, 12.6773177459836 46.40695743262768, 12.345747342333198 50.55427505169064, 6.259687012061477 51.964770150370896, 3.6300085578113794 50.610463307239115, 1.1885095294564962 49.470279179513454))" }
+
+---
+"Test geo_grid on geotile with advanced pipeline and cell hierarchy using WKT":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body: >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "geo_grid": {
+                  "field": "geocell",
+                  "tile_type": "geotile",
+                  "target_format": "wkt",
+                  "target_field": "shape",
+                  "parent_field": "parent",
+                  "children_field": "children",
+                  "precision_field": "precision"
+                }
+              },
+              {
+                "set": {
+                  "field": "childrenShapes",
+                  "copy_from": "children"
+                }
+              },
+              {
+                "foreach" : {
+                  "field" : "childrenShapes",
+                  "processor" : {
+                    "geo_grid": {
+                      "field": "_ingest._value",
+                      "tile_type": "geotile",
+                      "target_format": "wkt"
+                    }
+                  }
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        pipeline: "my_pipeline"
+        body: >
+          {
+            "geocell": "4/8/5"
+          }
+
+  - do:
+      get:
+        index: test
+        id: "1"
+  - match:
+      _source:
+        geocell: "4/8/5"
+        shape: "BBOX (0.0, 22.5, 55.77657301866769, 40.979898069620134)"
+        parent: "3/4/2"
+        precision: 4
+        children: [ "5/16/10", "5/17/10", "5/16/11", "5/17/11" ]
+        childrenShapes:
+          - "BBOX (0.0, 11.25, 55.77657301866769, 48.922499263758255)"
+          - "BBOX (11.25, 22.5, 55.77657301866769, 48.922499263758255)"
+          - "BBOX (0.0, 11.25, 48.922499263758255, 40.979898069620134)"
+          - "BBOX (11.25, 22.5, 48.922499263758255, 40.979898069620134)"
+
+---
+"Test geo_grid on geohash with advanced pipeline and cell hierarchy using WKT":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body: >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "geo_grid": {
+                  "field": "geocell",
+                  "tile_type": "geohash",
+                  "target_format": "wkt",
+                  "target_field": "shape",
+                  "parent_field": "parent",
+                  "children_field": "children",
+                  "precision_field": "precision"
+                }
+              },
+              {
+                "set": {
+                  "field": "childrenShapes",
+                  "copy_from": "children"
+                }
+              },
+              {
+                "foreach" : {
+                  "field" : "childrenShapes",
+                  "processor" : {
+                    "geo_grid": {
+                      "field": "_ingest._value",
+                      "tile_type": "geohash",
+                      "target_format": "wkt"
+                    }
+                  }
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        pipeline: "my_pipeline"
+        body: >
+          {
+            "geocell": "u0"
+          }
+
+  - do:
+      get:
+        index: test
+        id: "1"
+  - match:
+      _source:
+        geocell: "u0"
+        shape: "BBOX (0.0, 11.25, 50.625, 45.0)"
+        parent: "u"
+        precision: 2
+        children:
+          - "u00"
+          - "u01"
+          - "u02"
+          - "u03"
+          - "u04"
+          - "u05"
+          - "u06"
+          - "u07"
+          - "u08"
+          - "u09"
+          - "u0b"
+          - "u0c"
+          - "u0d"
+          - "u0e"
+          - "u0f"
+          - "u0g"
+          - "u0h"
+          - "u0j"
+          - "u0k"
+          - "u0m"
+          - "u0n"
+          - "u0p"
+          - "u0q"
+          - "u0r"
+          - "u0s"
+          - "u0t"
+          - "u0u"
+          - "u0v"
+          - "u0w"
+          - "u0x"
+          - "u0y"
+          - "u0z"
+        childrenShapes:
+          - "BBOX (0.0, 1.40625, 46.40625, 45.0)"
+          - "BBOX (1.40625, 2.8125, 46.40625, 45.0)"
+          - "BBOX (0.0, 1.40625, 47.8125, 46.40625)"
+          - "BBOX (1.40625, 2.8125, 47.8125, 46.40625)"
+          - "BBOX (2.8125, 4.21875, 46.40625, 45.0)"
+          - "BBOX (4.21875, 5.625, 46.40625, 45.0)"
+          - "BBOX (2.8125, 4.21875, 47.8125, 46.40625)"
+          - "BBOX (4.21875, 5.625, 47.8125, 46.40625)"
+          - "BBOX (0.0, 1.40625, 49.21875, 47.8125)"
+          - "BBOX (1.40625, 2.8125, 49.21875, 47.8125)"
+          - "BBOX (0.0, 1.40625, 50.625, 49.21875)"
+          - "BBOX (1.40625, 2.8125, 50.625, 49.21875)"
+          - "BBOX (2.8125, 4.21875, 49.21875, 47.8125)"
+          - "BBOX (4.21875, 5.625, 49.21875, 47.8125)"
+          - "BBOX (2.8125, 4.21875, 50.625, 49.21875)"
+          - "BBOX (4.21875, 5.625, 50.625, 49.21875)"
+          - "BBOX (5.625, 7.03125, 46.40625, 45.0)"
+          - "BBOX (7.03125, 8.4375, 46.40625, 45.0)"
+          - "BBOX (5.625, 7.03125, 47.8125, 46.40625)"
+          - "BBOX (7.03125, 8.4375, 47.8125, 46.40625)"
+          - "BBOX (8.4375, 9.84375, 46.40625, 45.0)"
+          - "BBOX (9.84375, 11.25, 46.40625, 45.0)"
+          - "BBOX (8.4375, 9.84375, 47.8125, 46.40625)"
+          - "BBOX (9.84375, 11.25, 47.8125, 46.40625)"
+          - "BBOX (5.625, 7.03125, 49.21875, 47.8125)"
+          - "BBOX (7.03125, 8.4375, 49.21875, 47.8125)"
+          - "BBOX (5.625, 7.03125, 50.625, 49.21875)"
+          - "BBOX (7.03125, 8.4375, 50.625, 49.21875)"
+          - "BBOX (8.4375, 9.84375, 49.21875, 47.8125)"
+          - "BBOX (9.84375, 11.25, 49.21875, 47.8125)"
+          - "BBOX (8.4375, 9.84375, 50.625, 49.21875)"
+          - "BBOX (9.84375, 11.25, 50.625, 49.21875)"
+
+---
+"Test geo_grid on geohex with advanced pipeline and cell hierarchy using WKT":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body: >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "geo_grid": {
+                  "field": "geocell",
+                  "tile_type": "geohex",
+                  "target_format": "wkt",
+                  "target_field": "shape",
+                  "parent_field": "parent",
+                  "children_field": "children",
+                  "non_children_field": "nonChildren",
+                  "precision_field": "precision"
+                }
+              },
+              {
+                "set": {
+                  "field": "childrenShapes",
+                  "copy_from": "children"
+                }
+              },
+              {
+                "set": {
+                  "field": "nonChildrenShapes",
+                  "copy_from": "nonChildren"
+                }
+              },
+              {
+                "foreach" : {
+                  "field" : "childrenShapes",
+                  "processor" : {
+                    "geo_grid": {
+                      "field": "_ingest._value",
+                      "tile_type": "geohex",
+                      "target_format": "wkt"
+                    }
+                  }
+                }
+              },
+              {
+                "foreach" : {
+                  "field" : "nonChildrenShapes",
+                  "processor" : {
+                    "geo_grid": {
+                      "field": "_ingest._value",
+                      "tile_type": "geohex",
+                      "target_format": "wkt"
+                    }
+                  }
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        pipeline: "my_pipeline"
+        body: >
+          {
+            "geocell": "811fbffffffffff"
+          }
+
+  - do:
+      get:
+        index: test
+        id: "1"
+  - match:
+      _source:
+        geocell: "811fbffffffffff"
+        shape: "POLYGON ((1.1885095294564962 49.470279179513454, 2.0265689212828875 45.18424864858389, 7.509948452934623 43.786609335802495, 12.6773177459836 46.40695743262768, 12.345747342333198 50.55427505169064, 6.259687012061477 51.964770150370896, 3.6300085578113794 50.610463307239115, 1.1885095294564962 49.470279179513454))"
+        parent: "801ffffffffffff"
+        precision: 1
+        children:
+          - "821f87fffffffff"
+          - "821f8ffffffffff"
+          - "821f97fffffffff"
+          - "821f9ffffffffff"
+          - "821fa7fffffffff"
+          - "821faffffffffff"
+          - "821fb7fffffffff"
+        nonChildren:
+          - "821ea7fffffffff"
+          - "82186ffffffffff"
+          - "82396ffffffffff"
+          - "821f17fffffffff"
+          - "821e37fffffffff"
+          - "82194ffffffffff"
+        nonChildrenShapes:
+          - "POLYGON ((10.343212774023414 45.92192775104195, 9.731941111385822 44.323804839514196, 11.369192842394114 43.20897659752518, 13.642238667234778 43.67433743085712, 14.337206231430173 45.273432224057615, 12.6773177459836 46.40695743262768, 10.343212774023414 45.92192775104195))"
+          - "POLYGON ((0.7640258967876434 47.907450324855745, -1.3966082129627466 47.44351194240153, -1.7527981102466583 45.828917124308646, -0.028004944324493408 44.71297777723521, 2.0265689212828875 45.18424864858389, 2.4589017033576965 46.76373050082475, 0.7640258967876434 47.907450324855745))"
+          - "POLYGON ((3.696701768785715 44.22123494558036, 3.2493495382368565 42.60479216929525, 4.868332073092461 41.58031489234418, 6.977840261533856 42.158615668304265, 7.509948452934623 43.786609335802495, 5.848476877436042 44.82546979095787, 3.696701768785715 44.22123494558036))"
+          - "POLYGON ((6.848228881135583 53.43156033195555, 6.259687012061477 51.964770150370896, 8.08863085694611 51.039695707149804, 10.549622755497694 51.56398999504745, 11.24517043121159 53.03203972522169, 9.374530809000134 53.975513661280274, 6.848228881135583 53.43156033195555))"
+          - "POLYGON ((12.345747342333198 50.55427505169064, 11.648396300151944 49.039345127530396, 13.37434759363532 47.96593840699643, 15.817383527755737 48.38812246453017, 16.606046771630645 49.899182128719985, 14.863026738166809 50.99274770356715, 12.345747342333198 50.55427505169064))"
+          - "POLYGON ((2.101313956081867 52.51713396515697, -0.21913748234510422 52.14168077800423, -0.6310947798192501 50.60191108379513, 1.1885095294564962 49.470279179513454, 3.384771505370736 49.85377031844109, 3.8811334781348705 51.36037617456168, 2.101313956081867 52.51713396515697))"
+        childrenShapes:
+          - "POLYGON ((5.163510935381055 48.943763938732445, 4.6524353697896 47.393027362413704, 6.371338488534093 46.417016638442874, 8.645314201712608 46.97706841863692, 9.251571390777826 48.53467794600874, 7.489805733785033 49.52621237374842, 5.163510935381055 48.943763938732445))"
+          - "POLYGON ((9.251571390777826 48.53467794600874, 8.645314201712608 46.97706841863692, 10.343212774023414 45.92192775104195, 12.6773177459836 46.40695743262768, 13.37434759363532 47.96593840699643, 11.648396300151944 49.039345127530396, 9.251571390777826 48.53467794600874))"
+          - "POLYGON ((2.4589017033576965 46.76373050082475, 2.0265689212828875 45.18424864858389, 3.696701768785715 44.22123494558036, 5.848476877436042 44.82546979095787, 6.371338488534093 46.417016638442874, 4.6524353697896 47.393027362413704, 2.4589017033576965 46.76373050082475))"
+          - "POLYGON ((6.371338488534093 46.417016638442874, 5.848476877436042 44.82546979095787, 7.509948452934623 43.786609335802495, 9.731941111385822 44.323804839514196, 10.343212774023414 45.92192775104195, 8.645314201712608 46.97706841863692, 6.371338488534093 46.417016638442874))"
+          - "POLYGON ((3.8811334781348705 51.36037617456168, 3.384771505370736 49.85377031844109, 5.163510935381055 48.943763938732445, 7.489805733785033 49.52621237374842, 8.08863085694611 51.039695707149804, 6.259687012061477 51.964770150370896, 3.8811334781348705 51.36037617456168))"
+          - "POLYGON ((8.08863085694611 51.039695707149804, 7.489805733785033 49.52621237374842, 9.251571390777826 48.53467794600874, 11.648396300151944 49.039345127530396, 12.345747342333198 50.55427505169064, 10.549622755497694 51.56398999504745, 8.08863085694611 51.039695707149804))"
+          - "POLYGON ((1.1885095294564962 49.470279179513454, 0.7640258967876434 47.907450324855745, 2.4589017033576965 46.76373050082475, 4.6524353697896 47.393027362413704, 5.163510935381055 48.943763938732445, 3.384771505370736 49.85377031844109, 1.1885095294564962 49.470279179513454))"
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/preview_transforms.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/preview_transforms.yml
index 70656a3f1ff7..2d3cfa764e98 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/preview_transforms.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/preview_transforms.yml
@@ -11,6 +11,10 @@ setup:
                 type: keyword
               responsetime:
                 type: float
+              agg_metric:
+                type: aggregate_metric_double
+                metrics: [ "min", "max" ]
+                default_metric: "max"
               event_rate:
                 type: integer
               _isDeleted:
@@ -156,6 +160,45 @@ setup:
   - match: { generated_dest_index.mappings.properties.by-hour.type: "date" }
   - match: { generated_dest_index.mappings.properties.avg_response.type: "double" }
 
+---
+"Test preview transform with offset":
+  - do:
+      transform.preview_transform:
+        body: >
+          {
+            "source": { "index": "airline-data" },
+            "pivot": {
+              "group_by": {
+                "airline": {"terms": {"field": "airline"}},
+                "by-hour": {"date_histogram": {"fixed_interval": "1h", "field": "time", "offset": "-15m"}}},
+              "aggs": {
+                "avg_response": {"avg": {"field": "responsetime"}},
+                "time.max": {"max": {"field": "time"}},
+                "time.min": {"min": {"field": "time"}}
+              }
+            }
+          }
+  - match: { preview.0.airline: foo }
+  - match: { preview.0.by-hour: "2017-02-17T23:45:00.000Z" }
+  - match: { preview.0.avg_response: 1.0 }
+  - match: { preview.0.time.max: "2017-02-18T00:30:00.000Z" }
+  - match: { preview.0.time.min: "2017-02-18T00:00:00.000Z" }
+  - match: { preview.1.airline: bar }
+  - match: { preview.1.by-hour: "2017-02-18T00:45:00.000Z" }
+  - match: { preview.1.avg_response: 42.0 }
+  - match: { preview.1.time.max: "2017-02-18T01:00:00.000Z" }
+  - match: { preview.1.time.min: "2017-02-18T01:00:00.000Z" }
+  - match: { preview.2.airline: foo }
+  - match: { preview.2.by-hour: "2017-02-18T00:45:00.000Z" }
+  - match: { preview.2.avg_response: 42.0 }
+  - match: { preview.2.time.max: "2017-02-18T01:01:00.000Z" }
+  - match: { preview.2.time.min: "2017-02-18T01:01:00.000Z" }
+  - match: { generated_dest_index.mappings.properties.airline.type: "keyword" }
+  - match: { generated_dest_index.mappings.properties.by-hour.type: "date" }
+  - match: { generated_dest_index.mappings.properties.avg_response.type: "double" }
+  - match: { generated_dest_index.mappings.properties.time\.max.type: "date" }
+  - match: { generated_dest_index.mappings.properties.time\.min.type: "date" }
+
 ---
 "Test preview transform with timeout":
   - do:
@@ -638,3 +681,19 @@ setup:
               "unattended": true
             }
           }
+
+---
+"Test preview transform latest with unique key field of aggregate_metric_double type":
+  - do:
+      catch: /Field \[agg_metric\] of type \[aggregate_metric_double\] is not supported for aggregation \[terms\]/
+      transform.preview_transform:
+        body:  >
+          {
+            "source": {
+              "index": "airline-data"
+            },
+            "latest": {
+              "unique_key": ["agg_metric"],
+              "sort": "time"
+            }
+          }
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_start_stop.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_start_stop.yml
index 8bf95b8ccf5d..aebc28ce0bfe 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_start_stop.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_start_stop.yml
@@ -137,6 +137,39 @@ teardown:
       transform.start_transform:
         transform_id: "airline-transform-start-stop"
 
+---
+"Test schedule_now on a stopped transform":
+  - do:
+      transform.schedule_now_transform:
+        transform_id: "airline-transform-start-stop"
+  - match: { acknowledged: true }
+
+---
+"Test schedule_now on an already started transform":
+  - do:
+      transform.start_transform:
+        transform_id: "airline-transform-start-stop"
+  - match: { acknowledged: true }
+
+  - do:
+      transform.schedule_now_transform:
+        transform_id: "airline-transform-start-stop"
+  - match: { acknowledged: true }
+
+---
+"Test schedule_now all transforms":
+  - do:
+      catch: /_schedule_now API does not support _all wildcard/
+      transform.schedule_now_transform:
+        transform_id: "_all"
+
+---
+"Test schedule_now missing transform":
+  - do:
+      catch: missing
+      transform.schedule_now_transform:
+        transform_id: "missing"
+
 ---
 "Verify start transform creates destination index with appropriate mapping":
   - do:
@@ -256,6 +289,31 @@ teardown:
         transform_id: "airline-transform-start-stop-continuous"
         wait_for_completion: true
   - match: { acknowledged: true }
+
+---
+"Test start transform with empty value of from parameter":
+  - do:
+      catch: /Failed to parse date for \[from\]/
+      transform.start_transform:
+        from: ""
+        transform_id: "airline-transform-start-stop-continuous"
+
+---
+"Test start transform with invalid value of from parameter":
+  - do:
+      catch: /Failed to parse date for \[from\]/
+      transform.start_transform:
+        from: "not-a-valid-timestamp"
+        transform_id: "airline-transform-start-stop-continuous"
+
+---
+"Test start batch transform with from parameter":
+  - do:
+      catch: /\[from\] parameter is currently not supported for batch \(non-continuous\) transforms/
+      transform.start_transform:
+        from: "2023-01-11T12:00:00"
+        transform_id: "airline-transform-start-stop"
+
 ---
 "Test stop missing transform":
   - do:
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_stats.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_stats.yml
index e23852572041..c3fe2114ff84 100644
--- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_stats.yml
+++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_stats.yml
@@ -90,6 +90,16 @@ teardown:
   - lte: { transforms.0.stats.search_total: 1 }
   - match: { transforms.0.stats.search_failures: 0 }
 
+---
+"Test get transform stats with timeout":
+  - do:
+      transform.get_transform_stats:
+        transform_id: "airline-transform-stats"
+        timeout: "10s"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "airline-transform-stats" }
+  - match: { transforms.0.state: "/started|indexing|stopping|stopped/" }
+
 ---
 "Test get transform stats on missing transform":
   - do:
diff --git a/x-pack/plugin/stack/qa/rest/build.gradle b/x-pack/plugin/stack/qa/rest/build.gradle
index 770a52fde68b..a5bdb388d610 100644
--- a/x-pack/plugin/stack/qa/rest/build.gradle
+++ b/x-pack/plugin/stack/qa/rest/build.gradle
@@ -1,5 +1,5 @@
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 apply plugin: 'elasticsearch.authenticated-testclusters'
 
 dependencies {
diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java
index c2fdb50ca924..fb647f851139 100644
--- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java
+++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java
@@ -8,7 +8,7 @@
 
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Setting;
@@ -55,7 +55,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         StackTemplateRegistry templateRegistry = new StackTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry);
         templateRegistry.initialize();
diff --git a/x-pack/plugin/text-structure/qa/text-structure-with-security/build.gradle b/x-pack/plugin/text-structure/qa/text-structure-with-security/build.gradle
index 4431351e5618..5fc76885aa7e 100644
--- a/x-pack/plugin/text-structure/qa/text-structure-with-security/build.gradle
+++ b/x-pack/plugin/text-structure/qa/text-structure-with-security/build.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 
 dependencies {
   yamlRestTestImplementation(testArtifact(project(xpackModule('core'))))
diff --git a/x-pack/plugin/text-structure/src/test/java/org/elasticsearch/xpack/textstructure/structurefinder/TimestampFormatFinderTests.java b/x-pack/plugin/text-structure/src/test/java/org/elasticsearch/xpack/textstructure/structurefinder/TimestampFormatFinderTests.java
index dd8bd7f5cc17..87bb63f442ac 100644
--- a/x-pack/plugin/text-structure/src/test/java/org/elasticsearch/xpack/textstructure/structurefinder/TimestampFormatFinderTests.java
+++ b/x-pack/plugin/text-structure/src/test/java/org/elasticsearch/xpack/textstructure/structurefinder/TimestampFormatFinderTests.java
@@ -1590,7 +1590,7 @@ public void testSelectBestMatchGivenAllSame() {
                 [2018-06-27T11:59:22,202][INFO ][o.e.e.NodeEnvironment    ] [node-0] heap size [494.9mb], compressed ordinary object pointers [true]
                 [2018-06-27T11:59:22,204][INFO ][o.e.n.Node               ] [node-0] node name [node-0], node ID [Ha1gD8nNSDqjd6PIyu3DJA]
                 [2018-06-27T11:59:22,204][INFO ][o.e.n.Node               ] [node-0] version[6.4.0-SNAPSHOT], pid[2785], build[default/zip/3c60efa/2018-06-26T14:55:15.206676Z], OS[Mac OS X/10.12.6/x86_64], JVM["Oracle Corporation"/Java HotSpot(TM) 64-Bit Server VM/10/10+46]
-                [2018-06-27T11:59:22,205][INFO ][o.e.n.Node               ] [node-0] JVM arguments [-Xms1g, -Xmx1g, -XX:+UseConcMarkSweepGC, -XX:CMSInitiatingOccupancyFraction=75, -XX:+UseCMSInitiatingOccupancyOnly, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.io.tmpdir=/var/folders/k5/5sqcdlps5sg3cvlp783gcz740000h0/T/elasticsearch.nFUyeMH1, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Djava.locale.providers=COMPAT, -Dio.netty.allocator.type=unpooled, -ea, -esa, -Xms512m, -Xmx512m, -Des.path.home=/Users/dave/elasticsearch/distribution/build/cluster/run node0/elasticsearch-6.4.0-SNAPSHOT, -Des.path.conf=/Users/dave/elasticsearch/distribution/build/cluster/run node0/elasticsearch-6.4.0-SNAPSHOT/config, -Des.distribution.flavor=default, -Des.distribution.type=zip]
+                [2018-06-27T11:59:22,205][INFO ][o.e.n.Node               ] [node-0] JVM arguments [-Xms1g, -Xmx1g, -XX:+UseConcMarkSweepGC, -XX:CMSInitiatingOccupancyFraction=75, -XX:+UseCMSInitiatingOccupancyOnly, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.io.tmpdir=/var/folders/k5/5sqcdlps5sg3cvlp783gcz740000h0/T/elasticsearch.nFUyeMH1, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,level,pid,tags:filecount=32,filesize=64m, -Djava.locale.providers=COMPAT, -Dio.netty.allocator.type=unpooled, -ea, -esa, -Xms512m, -Xmx512m, -Des.path.home=/Users/dave/elasticsearch/distribution/build/cluster/run node0/elasticsearch-6.4.0-SNAPSHOT, -Des.path.conf=/Users/dave/elasticsearch/distribution/build/cluster/run node0/elasticsearch-6.4.0-SNAPSHOT/config, -Des.distribution.flavor=default, -Des.distribution.type=zip]
                 [2018-06-27T11:59:22,205][WARN ][o.e.n.Node               ] [node-0] version [6.4.0-SNAPSHOT] is a pre-release version of Elasticsearch and is not suitable for production
                 [2018-06-27T11:59:23,585][INFO ][o.e.p.PluginsService     ] [node-0] loaded module [aggs-matrix-stats]
                 [2018-06-27T11:59:23,586][INFO ][o.e.p.PluginsService     ] [node-0] loaded module [analysis-common]
diff --git a/x-pack/plugin/transform/build.gradle b/x-pack/plugin/transform/build.gradle
index 7115cb0c3450..9e50aea3a8fc 100644
--- a/x-pack/plugin/transform/build.gradle
+++ b/x-pack/plugin/transform/build.gradle
@@ -13,7 +13,6 @@ dependencies {
 
   testImplementation(testArtifact(project(xpackModule('core'))))
   testImplementation project(path: xpackModule('analytics'))
-  testImplementation project(path: ':modules:aggs-matrix-stats')
   testImplementation project(path: ':modules:aggregations')
   testImplementation project(path: xpackModule('spatial'))
   testImplementation project(":test:x-content")
diff --git a/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/80_transform.yml b/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/80_transform.yml
index fe0b930cc193..0fc5b7fa3b3a 100644
--- a/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/80_transform.yml
+++ b/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/80_transform.yml
@@ -26,7 +26,6 @@ setup:
   - do:
       security.put_role:
         name: "x_cluster_role"
-        # gh#72715: the my_remote_cluster privileges should not be needed
         body:  >
           {
             "cluster": [],
@@ -38,10 +37,6 @@ setup:
               {
                 "names": ["simple-remote-transform*", "simple-local-remote-transform", "same-index-local-and-remote-transform"],
                 "privileges": ["create_index", "index", "read"]
-              },
-              {
-                "names": ["my_remote_cluster:remote_test_i*", "my_remote_cluster:aliased_test_index", "my_remote_cluster:same_index_local_and_remote"],
-                "privileges": ["read", "view_index_metadata"]
               }
             ]
           }
@@ -156,7 +151,7 @@ teardown:
         transform_id: "simple-remote-transform"
 
   - do:
-      catch: /Cannot preview transform \[simple-remote-transform\] because user bob lacks the required permissions \[my_remote_cluster:remote_test_index\*:\[read, view_index_metadata\], simple-remote-transform:\[\]\]/
+      catch: /Source indices have been deleted or closed./
       headers: { Authorization: "Basic Ym9iOnRyYW5zZm9ybS1wYXNzd29yZA==" }  # This is bob
       transform.preview_transform:
         transform_id: "simple-remote-transform"
@@ -311,14 +306,13 @@ teardown:
 ---
 "Batch transform from remote cluster when the user is not authorized":
   - do:
-      catch: /Cannot create transform \[simple-remote-transform\] because user bob lacks the required permissions \[my_remote_cluster:remote_test_index:\[read, view_index_metadata\], simple-remote-transform:\[\]\]/
       headers: { Authorization: "Basic Ym9iOnRyYW5zZm9ybS1wYXNzd29yZA==" }  # This is bob
       transform.put_transform:
-        transform_id: "simple-remote-transform"
+        transform_id: "simple-remote-transform-3"
         body: >
           {
             "source": { "index": "my_remote_cluster:remote_test_index" },
-            "dest": { "index": "simple-remote-transform" },
+            "dest": { "index": "simple-remote-transform-3" },
             "pivot": {
               "group_by": { "user": {"terms": {"field": "user"}}},
               "aggs": { "avg_stars": {"avg": {"field": "stars"}}}
@@ -342,7 +336,6 @@ teardown:
           }
   - match: { acknowledged: true }
   - do:
-      catch: /Cannot update transform \[simple-remote-transform-2\] because user bob lacks the required permissions \[my_remote_cluster:remote_test_index:\[read, view_index_metadata\], simple-remote-transform-2:\[\]\]/
       headers: { Authorization: "Basic Ym9iOnRyYW5zZm9ybS1wYXNzd29yZA==" }  # This is bob
       transform.update_transform:
         transform_id: "simple-remote-transform-2"
@@ -355,7 +348,7 @@ teardown:
 ---
 "Batch transform preview from remote cluster when the user is not authorized":
   - do:
-      catch: /Cannot preview transform \[transform-preview\] because user bob lacks the required permissions \[my_remote_cluster:remote_test_index:\[read, view_index_metadata\], simple-remote-transform-2:\[\]\]/
+      catch: /Source indices have been deleted or closed./
       headers: { Authorization: "Basic Ym9iOnRyYW5zZm9ybS1wYXNzd29yZA==" }  # This is bob
       transform.preview_transform:
         body: >
@@ -368,7 +361,7 @@ teardown:
             }
           }
   - do:
-      catch: /Cannot preview transform \[transform-preview\] because user bob lacks the required permissions \[my_remote_cluster:test_index:\[read, view_index_metadata\]\]/
+      catch: /Source indices have been deleted or closed./
       headers: { Authorization: "Basic Ym9iOnRyYW5zZm9ybS1wYXNzd29yZA==" }  # This is bob
       transform.preview_transform:
         body: >
diff --git a/x-pack/plugin/transform/qa/multi-node-tests/build.gradle b/x-pack/plugin/transform/qa/multi-node-tests/build.gradle
index 6cd6c529a879..4f1dfd334589 100644
--- a/x-pack/plugin/transform/qa/multi-node-tests/build.gradle
+++ b/x-pack/plugin/transform/qa/multi-node-tests/build.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 dependencies {
   javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java
index a0cb50ea3f66..bd5e1cb06893 100644
--- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java
+++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java
@@ -220,7 +220,7 @@ public void testContinuousTransformUpdate() throws Exception {
         putPipeline.setEntity(new StringEntity(Strings.toString(pipelineBuilder), ContentType.APPLICATION_JSON));
         assertOK(client().performRequest(putPipeline));
 
-        String update = formatted("""
+        String update = Strings.format("""
             {
                 "description": "updated config",
                 "dest": {
@@ -407,7 +407,7 @@ public void testContinuousTransformRethrottle() throws Exception {
         // test randomly: with explicit settings and reset to default
         String reqsPerSec = randomBoolean() ? "1000" : "null";
         String maxPageSize = randomBoolean() ? "1000" : "null";
-        String update = formatted("""
+        String update = Strings.format("""
             {
                 "settings" : {
                     "docs_per_second": %s,
@@ -482,14 +482,14 @@ public void testStartTransform_GivenTimeout_Returns408() throws Exception {
     private void indexMoreDocs(long timestamp, long userId, String index) throws Exception {
         StringBuilder bulkBuilder = new StringBuilder();
         for (int i = 0; i < 25; i++) {
-            bulkBuilder.append(formatted("""
+            bulkBuilder.append(Strings.format("""
                 {"create":{"_index":"%s"}}
                 """, index));
 
             int stars = (i + 20) % 5;
             long business = (i + 100) % 50;
 
-            String source = formatted("""
+            String source = Strings.format("""
                 {"user_id":"user_%s","count":%s,"business_id":"business_%s","stars":%s,"timestamp":%s}
                 """, userId, i, business, stars, timestamp);
             bulkBuilder.append(source);
diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java
index 7561c68e8eed..6b6dc6b5c21d 100644
--- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java
+++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java
@@ -290,7 +290,7 @@ protected DateHistogramGroupSource createDateHistogramGroupSourceWithFixedInterv
         DateHistogramInterval interval,
         ZoneId zone
     ) {
-        return new DateHistogramGroupSource(field, null, false, new DateHistogramGroupSource.FixedInterval(interval), zone);
+        return new DateHistogramGroupSource(field, null, false, new DateHistogramGroupSource.FixedInterval(interval), zone, null);
     }
 
     protected DateHistogramGroupSource createDateHistogramGroupSourceWithCalendarInterval(
@@ -298,7 +298,7 @@ protected DateHistogramGroupSource createDateHistogramGroupSourceWithCalendarInt
         DateHistogramInterval interval,
         ZoneId zone
     ) {
-        return new DateHistogramGroupSource(field, null, false, new DateHistogramGroupSource.CalendarInterval(interval), zone);
+        return new DateHistogramGroupSource(field, null, false, new DateHistogramGroupSource.CalendarInterval(interval), zone, null);
     }
 
     /**
@@ -464,7 +464,7 @@ protected void createReviewsIndex(
             long business = i % 50;
             String dateString = dateStringProvider.apply(i);
 
-            sourceBuilder.append(formatted("""
+            sourceBuilder.append(Strings.format("""
                 {"create":{"_index":"%s"}}
                 """, indexName));
 
@@ -472,7 +472,7 @@ protected void createReviewsIndex(
             if (user != null) {
                 sourceBuilder.append("\"user_id\":\"").append("user_").append(user).append("\",");
             }
-            sourceBuilder.append(formatted("""
+            sourceBuilder.append(Strings.format("""
                 "count":%s,"business_id":"business_%s","stars":%s,"comment":"Great stuff, deserves %s stars","regular_object":\
                 {"foo": 42},"nested_object":{"bar": 43},"timestamp":"%s"}
                 """, i, business, stars, stars, dateString));
diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java
index 870eb9e715f3..416f078c626d 100644
--- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java
+++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.xpack.core.transform.transforms.DestConfig;
@@ -66,6 +67,7 @@ public TransformConfig createConfig() throws IOException {
                     null,
                     missing,
                     new DateHistogramGroupSource.FixedInterval(DateHistogramInterval.SECOND),
+                    null,
                     null
                 )
             ),
@@ -93,7 +95,8 @@ public String getName() {
     @Override
     @SuppressWarnings("unchecked")
     public void testIteration(int iteration, Set<String> modifiedEvents) throws IOException {
-        String query = formatted("""
+        Object[] args = new Object[] { timestampField, missing ? "\"missing\": \"" + MISSING_BUCKET_KEY + "\"," : "" };
+        String query = Strings.format("""
             {
               "aggs": {
                 "second": {
@@ -106,7 +109,7 @@ public void testIteration(int iteration, Set<String> modifiedEvents) throws IOEx
                 }
               }
             }
-            """, timestampField, missing ? "\"missing\": \"" + MISSING_BUCKET_KEY + "\"," : "");
+            """, args);
 
         Response searchResponse = search(
             CONTINUOUS_EVENTS_SOURCE_INDEX,
diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByOtherTimeFieldIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByOtherTimeFieldIT.java
index 409022842e18..9da958dfc331 100644
--- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByOtherTimeFieldIT.java
+++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByOtherTimeFieldIT.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.xpack.core.transform.transforms.DestConfig;
@@ -72,6 +73,7 @@ public TransformConfig createConfig() throws IOException {
                 null,
                 false,
                 new DateHistogramGroupSource.FixedInterval(DateHistogramInterval.SECOND),
+                null,
                 null
             )
         );
@@ -100,11 +102,12 @@ public String getName() {
 
     @Override
     public void testIteration(int iteration, Set<String> modifiedEvents) throws IOException {
-        String eventAgg = formatted("""
+        String eventAgg = Strings.format("""
             , "aggs" : {"event": {"terms": {"field": "%s", "size": 1000, "order": {"_key": "asc"}}}}
             """, termsField);
 
-        String querySource = formatted("""
+        Object[] args = new Object[] { metricTimestampField, addGroupByTerms ? eventAgg : "" };
+        String querySource = Strings.format("""
             {
               "aggs": {
                 "second": {
@@ -116,7 +119,7 @@ public void testIteration(int iteration, Set<String> modifiedEvents) throws IOEx
                 }
               }
             }
-            """, metricTimestampField, addGroupByTerms ? eventAgg : "");
+            """, args);
 
         Response searchResponseSource = search(
             CONTINUOUS_EVENTS_SOURCE_INDEX,
diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/HistogramGroupByIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/HistogramGroupByIT.java
index 3f9219ff1fa0..c9aabd8aa069 100644
--- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/HistogramGroupByIT.java
+++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/HistogramGroupByIT.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.xpack.core.transform.transforms.DestConfig;
 import org.elasticsearch.xpack.core.transform.transforms.SourceConfig;
@@ -70,7 +71,7 @@ public TransformConfig createConfig() throws IOException {
     @Override
     @SuppressWarnings("unchecked")
     public void testIteration(int iteration, Set<String> modifiedEvents) throws IOException {
-        String querySource = formatted("""
+        String querySource = Strings.format("""
             {
               "aggs": {
                 "metric": {
diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/LatestContinuousIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/LatestContinuousIT.java
index a5de3be478e2..81e6e2328f7b 100644
--- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/LatestContinuousIT.java
+++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/LatestContinuousIT.java
@@ -74,7 +74,7 @@ public String getName() {
     public void testIteration(int iteration, Set<String> modifiedEvents) throws IOException {
 
         var runtimeMappings = toJson(RUNTIME_MAPPINGS);
-        String query = formatted("""
+        String query = format("""
             {
               "runtime_mappings" : %s,
               "aggs": {
diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsGroupByIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsGroupByIT.java
index af38fd6cee4a..c0617795fba9 100644
--- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsGroupByIT.java
+++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsGroupByIT.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.transform.integration.continuous;
 
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.xpack.core.transform.transforms.DestConfig;
@@ -74,7 +75,8 @@ public String getName() {
     @Override
     @SuppressWarnings("unchecked")
     public void testIteration(int iteration, Set<String> modifiedEvents) throws IOException {
-        String query = formatted("""
+        Object[] args = new Object[] { termsField, missing ? "\"missing\": \"" + MISSING_BUCKET_KEY + "\"," : "", metricField };
+        String query = Strings.format("""
             {
               "aggs": {
                 "event": {
@@ -95,7 +97,7 @@ public void testIteration(int iteration, Set<String> modifiedEvents) throws IOEx
               },
               "sort": ["event"]
             }
-            """, termsField, missing ? "\"missing\": \"" + MISSING_BUCKET_KEY + "\"," : "", metricField);
+            """, args);
 
         var searchResponseSource = entityAsMap(
             search(CONTINUOUS_EVENTS_SOURCE_INDEX, query, Map.of("allow_partial_search_results", "false", "size", "0"))
diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsOnDateGroupByIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsOnDateGroupByIT.java
index 658a60985f4e..d9e159b0b9b2 100644
--- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsOnDateGroupByIT.java
+++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsOnDateGroupByIT.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.xpack.core.transform.transforms.DestConfig;
@@ -83,7 +84,8 @@ public String getName() {
     @Override
     @SuppressWarnings("unchecked")
     public void testIteration(int iteration, Set<String> modifiedEvents) throws IOException {
-        String query = formatted("""
+        Object[] args = new Object[] { timestampField, missing ? "\"missing\": \"" + MISSING_BUCKET_KEY + "\"," : "", metricField };
+        String query = Strings.format("""
             {
               "aggs": {
                 "some-timestamp": {
@@ -103,7 +105,7 @@ public void testIteration(int iteration, Set<String> modifiedEvents) throws IOEx
                 }
               }
             }
-            """, timestampField, missing ? "\"missing\": \"" + MISSING_BUCKET_KEY + "\"," : "", metricField);
+            """, args);
 
         Response searchResponseSource = search(
             CONTINUOUS_EVENTS_SOURCE_INDEX,
diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TransformContinuousIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TransformContinuousIT.java
index bc9ab27b62eb..229b0e8719e0 100644
--- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TransformContinuousIT.java
+++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TransformContinuousIT.java
@@ -198,7 +198,7 @@ public void testContinuousEvents() throws Exception {
 
             int numDocs = randomIntBetween(1000, 20000);
             Set<String> modifiedEvents = new HashSet<>();
-            String action = formatted("""
+            String action = Strings.format("""
                 {"create":{"_index":"%s"}}
                 """, sourceIndexName);
             for (int numDoc = 0; numDoc < numDocs; numDoc++) {
@@ -415,7 +415,7 @@ private void putIndex(String indexName, String dateType, boolean isDataStream) t
             logger.info("Creating source index with: {}", indexSettingsAndMappings);
             if (isDataStream) {
                 Request createCompositeTemplate = new Request("PUT", "_index_template/" + indexName + "_template");
-                createCompositeTemplate.setJsonEntity(formatted("""
+                createCompositeTemplate.setJsonEntity(Strings.format("""
                     {
                       "index_patterns": [ "%s" ],
                       "data_stream": {
diff --git a/x-pack/plugin/transform/qa/single-node-tests/build.gradle b/x-pack/plugin/transform/qa/single-node-tests/build.gradle
index be073f64a2ef..c321b3aeb028 100644
--- a/x-pack/plugin/transform/qa/single-node-tests/build.gradle
+++ b/x-pack/plugin/transform/qa/single-node-tests/build.gradle
@@ -1,5 +1,5 @@
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 dependencies {
   javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformGetAndGetStatsIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformGetAndGetStatsIT.java
index bebf3076b345..0f1af34ed5ee 100644
--- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformGetAndGetStatsIT.java
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformGetAndGetStatsIT.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.WarningsHandler;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.xpack.core.transform.TransformField;
 import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants;
 import org.junit.After;
@@ -380,7 +381,7 @@ public void testGetStatsWithContinuous() throws Exception {
         String transformSrc = "reviews_cont_pivot_test";
         createReviewsIndex(transformSrc);
         final Request createTransformRequest = createRequestWithAuth("PUT", getTransformEndpoint() + transformId, null);
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "dest": {
                 "index": "%s"
@@ -450,7 +451,7 @@ public void testGetStatsWithContinuous() throws Exception {
         long now = Instant.now().toEpochMilli() - 1_000;
         for (int i = 0; i < numDocs; i++) {
             // Doing only new users so that there is a deterministic number of docs for progress
-            bulk.append(formatted("""
+            bulk.append(Strings.format("""
                 {"index":{"_index":"%s"}}
                 {"user_id":"user_%s","business_id":"business_%s","stars":%s,"timestamp":%s}
                 """, transformSrc, randomFrom(42, 47, 113), 10, 5, now));
@@ -497,7 +498,7 @@ public void testManyTransforms() throws IOException {
 
         int numberOfTransforms = randomIntBetween(1_500, 4_000);
         for (int i = 0; i < numberOfTransforms; ++i) {
-            String transformId = formatted("t-%05d", i);
+            String transformId = Strings.format("t-%05d", i);
             final Request createTransformRequest = createRequestWithAuth("PUT", getTransformEndpoint() + transformId, null);
             createTransformRequest.setJsonEntity(config);
             assertOK(client().performRequest(createTransformRequest));
@@ -519,20 +520,20 @@ public void testManyTransforms() throws IOException {
             assertEquals(size, configs.size());
             assertEquals(size, stats.size());
 
-            assertThat(configs.get(0).get("id"), equalTo(formatted("t-%05d", from)));
-            assertThat(configs.get(configs.size() - 1).get("id"), equalTo(formatted("t-%05d", from + size - 1)));
-            assertThat(stats.get(0).get("id"), equalTo(formatted("t-%05d", from)));
-            assertThat(stats.get(stats.size() - 1).get("id"), equalTo(formatted("t-%05d", from + size - 1)));
+            assertThat(configs.get(0).get("id"), equalTo(Strings.format("t-%05d", from)));
+            assertThat(configs.get(configs.size() - 1).get("id"), equalTo(Strings.format("t-%05d", from + size - 1)));
+            assertThat(stats.get(0).get("id"), equalTo(Strings.format("t-%05d", from)));
+            assertThat(stats.get(stats.size() - 1).get("id"), equalTo(Strings.format("t-%05d", from + size - 1)));
 
             if (size > 2) {
                 int randomElement = randomIntBetween(1, size - 1);
-                assertThat(configs.get(randomElement).get("id"), equalTo(formatted("t-%05d", from + randomElement)));
-                assertThat(stats.get(randomElement).get("id"), equalTo(formatted("t-%05d", from + randomElement)));
+                assertThat(configs.get(randomElement).get("id"), equalTo(Strings.format("t-%05d", from + randomElement)));
+                assertThat(stats.get(randomElement).get("id"), equalTo(Strings.format("t-%05d", from + randomElement)));
             }
         }
 
         for (int i = 0; i < numberOfTransforms; ++i) {
-            deleteTransform(formatted("t-%05d", i));
+            deleteTransform(Strings.format("t-%05d", i));
         }
     }
 
diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformLatestRestIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformLatestRestIT.java
index 579cc549ed03..9224b838b058 100644
--- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformLatestRestIT.java
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformLatestRestIT.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -16,6 +17,7 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
@@ -53,10 +55,9 @@ public void createIndexes() throws IOException {
     public void testLatestWithAggregateMetricDouble() throws Exception {
         String transformId = "aggregate_metric_double_latest_transform";
         String transformIndex = "aggregate_metric_double_latest_reviews";
-        String statsField = "stars_stats";
         setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformIndex);
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -102,4 +103,48 @@ public void testLatestWithAggregateMetricDouble() throws Exception {
         assertThat(((List<?>) XContentMapValues.extractValue("hits.hits._source.stars_stats.max", searchResult)).get(0), is(equalTo(3)));
         assertThat(((List<?>) XContentMapValues.extractValue("hits.hits._source.stars_stats.sum", searchResult)).get(0), is(equalTo(20)));
     }
+
+    public void testLatestWithAggregateMetricDoubleAsUniqueKey() throws Exception {
+        String transformId = "aggregate_metric_double_latest_transform";
+        String transformIndex = "aggregate_metric_double_latest_reviews";
+        String statsField = "stars_stats";
+        setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformIndex);
+
+        String config = Strings.format("""
+            {
+              "source": {
+                "index": "%s"
+              },
+              "dest": {
+                "index": "%s"
+              },
+              "latest": {
+                "unique_key": [ "%s" ],
+                "sort": "@timestamp"
+              }
+            }""", REVIEWS_INDEX_NAME, transformIndex, statsField);
+
+        {
+            final Request createPreviewRequest = createRequestWithAuth("POST", getTransformEndpoint() + "_preview", null);
+            createPreviewRequest.setJsonEntity(config);
+            Exception e = expectThrows(Exception.class, () -> client().performRequest(createPreviewRequest));
+            assertThat(
+                e.getMessage(),
+                containsString("Field [stars_stats] of type [aggregate_metric_double] is not supported for aggregation [terms]")
+            );
+        }
+        {
+            final Request createTransformRequest = createRequestWithAuth(
+                "PUT",
+                getTransformEndpoint() + transformId,
+                BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
+            );
+            createTransformRequest.setJsonEntity(config);
+            Exception e = expectThrows(Exception.class, () -> client().performRequest(createTransformRequest));
+            assertThat(
+                e.getMessage(),
+                containsString("Field [stars_stats] of type [aggregate_metric_double] is not supported for aggregation [terms]")
+            );
+        }
+    }
 }
diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java
index d3c0cf026e53..c63ffe252608 100644
--- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java
@@ -31,6 +31,7 @@
 
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
@@ -245,7 +246,7 @@ public void testSimplePivotWithScript() throws Exception {
         );
 
         // same pivot as testSimplePivot, but we retrieve the grouping key using a script and add prefix
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "dest": {
                 "index": "%s"
@@ -298,7 +299,7 @@ public void testPivotWithPipeline() throws Exception {
         String pipelineId = "my-pivot-pipeline";
         int pipelineValue = 42;
         Request pipelineRequest = new Request("PUT", "/_ingest/pipeline/" + pipelineId);
-        pipelineRequest.setJsonEntity(formatted("""
+        pipelineRequest.setJsonEntity(Strings.format("""
             {
                "description" : "my pivot pipeline",
                "processors" : [
@@ -342,7 +343,7 @@ public void testBucketSelectorPivot() throws Exception {
             getTransformEndpoint() + transformId,
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -402,7 +403,7 @@ public void testContinuousPivot() throws Exception {
             getTransformEndpoint() + transformId,
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -462,17 +463,17 @@ public void testContinuousPivot() throws Exception {
             int stars = (i * 32) % 5;
             long business = (stars * user) % 13;
             String location = (user + 10) + "," + (user + 15);
-            bulk.append(formatted("""
+            bulk.append(Strings.format("""
                 {"index":{"_index":"%s"}}
                 {"user_id":"user_%s","business_id":"business_%s","stars":%s,"location":"%s","timestamp":%s}
                 """, indexName, user, business, stars, location, dateStamp));
             stars = 5;
             business = 11;
-            bulk.append(formatted("""
+            bulk.append(Strings.format("""
                 {"index":{"_index":"%s"}}
                 {"user_id":"user_%s","business_id":"business_%s","stars":%s,"location":"%s","timestamp":%s}
                 """, indexName, user26, business, stars, location, dateStamp));
-            bulk.append(formatted("""
+            bulk.append(Strings.format("""
                 {"index":{"_index":"%s"}}
                 {"business_id":"business_%s","stars":%s,"location":"%s","timestamp":%s}
                 """, indexName, business, stars, location, dateStamp));
@@ -499,6 +500,82 @@ public void testContinuousPivot() throws Exception {
         assertOnePivotValue(transformIndex + "/_search?q=reviewer:user_42", 2.0);
     }
 
+    public void testContinuousPivotFrom() throws Exception {
+        String indexName = "continuous_reviews_from";
+        createReviewsIndex(indexName);
+        String transformId = "continuous_pivot_from";
+        String transformIndex = "pivot_reviews_continuous_from";
+        setupDataAccessRole(DATA_ACCESS_ROLE, indexName, transformIndex);
+        final Request createTransformRequest = createRequestWithAuth(
+            "PUT",
+            getTransformEndpoint() + transformId,
+            BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
+        );
+        String config = Strings.format("""
+            {
+              "source": {
+                "index": "%s"
+              },
+              "dest": {
+                "index": "%s"
+              },
+              "frequency": "1s",
+              "sync": {
+                "time": {
+                  "field": "timestamp",
+                  "delay": "1s"
+                }
+              },
+              "pivot": {
+                "group_by": {
+                  "reviewer": {
+                    "terms": {
+                      "field": "user_id",
+                      "missing_bucket": true
+                    }
+                  }
+                },
+                "aggregations": {
+                  "avg_rating": {
+                    "avg": {
+                      "field": "stars"
+                    }
+                  }
+                }
+              }
+            }""", indexName, transformIndex);
+        createTransformRequest.setJsonEntity(config);
+        Map<String, Object> createTransformResponse = entityAsMap(client().performRequest(createTransformRequest));
+        assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
+
+        final StringBuilder bulk = new StringBuilder();
+        bulk.append(Strings.format("""
+            {"index":{"_index":"%s"}}
+            {"user_id":"user_%s","business_id":"business_%s","stars":%s,"location":"%s","timestamp":%s}
+            """, indexName, 666, 777, 7, 888, "\"2017-01-20\""));
+        bulk.append("\r\n");
+
+        final Request bulkRequest = new Request("POST", "/_bulk");
+        bulkRequest.addParameter("refresh", "true");
+        bulkRequest.setJsonEntity(bulk.toString());
+        Map<String, Object> bulkResponse = entityAsMap(client().performRequest(bulkRequest));
+        assertThat(bulkResponse.get("errors"), equalTo(Boolean.FALSE));
+
+        startAndWaitForContinuousTransform(transformId, transformIndex, null, "2017-01-23", 1L);
+        assertTrue(indexExists(transformIndex));
+
+        assertEquals(27, XContentMapValues.extractValue("_all.total.docs.count", getAsMap(transformIndex + "/_stats")));
+
+        // get and check some users
+        assertOnePivotValue(transformIndex + "/_search?q=reviewer:user_0", 3.776978417);
+        assertOnePivotValue(transformIndex + "/_search?q=reviewer:user_11", 3.846153846);
+        assertOnePivotValue(transformIndex + "/_search?q=reviewer:user_20", 3.769230769);
+        assertOnePivotValue(transformIndex + "/_search?q=reviewer:user_26", 3.918918918);
+
+        stopTransform(transformId, false);
+        deleteIndex(indexName);
+    }
+
     public void testHistogramPivot() throws Exception {
         String transformId = "simple_histogram_pivot";
         String transformIndex = "pivot_reviews_via_histogram";
@@ -510,7 +587,7 @@ public void testHistogramPivot() throws Exception {
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -561,7 +638,7 @@ public void testContinuousPivotHistogram() throws Exception {
             getTransformEndpoint() + transformId,
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -624,7 +701,7 @@ public void testContinuousPivotHistogram() throws Exception {
         // add 5 data points with 3 new users: 27, 28, 29
         for (int i = 25; i < 30; i++) {
             String location = (i + 10) + "," + (i + 15);
-            bulk.append(formatted("""
+            bulk.append(Strings.format("""
                 {"index":{"_index":"%s"}}
                 {"user_id":"user_%s","business_id":"business_%s","stars":%s,"location":"%s","timestamp":%s}
                 """, indexName, i, i, 3, location, dateStamp));
@@ -673,7 +750,7 @@ public void testBiggerPivot() throws Exception {
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -780,7 +857,7 @@ public void testPivotWithTermsAgg() throws Exception {
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -880,7 +957,7 @@ private void assertDateHistogramPivot(String indexName) throws Exception {
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -929,7 +1006,7 @@ public void testPreviewTransform() throws Exception {
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -980,7 +1057,7 @@ public void testPreviewTransformWithPipeline() throws Exception {
         String pipelineId = "my-preview-pivot-pipeline";
         int pipelineValue = 42;
         Request pipelineRequest = new Request("PUT", "/_ingest/pipeline/" + pipelineId);
-        pipelineRequest.setJsonEntity(formatted("""
+        pipelineRequest.setJsonEntity(Strings.format("""
             {
               "description": "my pivot preview pipeline",
               "processors": [
@@ -998,7 +1075,7 @@ public void testPreviewTransformWithPipeline() throws Exception {
         setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME);
         final Request createPreviewRequest = createRequestWithAuth("POST", getTransformEndpoint() + "_preview", null);
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -1070,7 +1147,7 @@ public void testPreviewTransformWithPipelineScript() throws Exception {
         final Request createPreviewRequest = createRequestWithAuth("POST", getTransformEndpoint() + "_preview", null);
         createPreviewRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE));
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -1115,6 +1192,63 @@ public void testPreviewTransformWithPipelineScript() throws Exception {
         );
     }
 
+    public void testPreviewTransformWithDateHistogramOffset() throws Exception {
+        assertThat(
+            previewWithOffset("+2d"),
+            contains("2017-01-04T00:00:00.000Z", "2017-01-11T00:00:00.000Z", "2017-01-18T00:00:00.000Z", "2017-01-25T00:00:00.000Z")
+        );
+        assertThat(previewWithOffset("+1d"), contains("2017-01-10T00:00:00.000Z", "2017-01-17T00:00:00.000Z", "2017-01-24T00:00:00.000Z"));
+        assertThat(
+            previewWithOffset("0"),
+            contains("2017-01-09T00:00:00.000Z", "2017-01-16T00:00:00.000Z", "2017-01-23T00:00:00.000Z", "2017-01-30T00:00:00.000Z")
+        );
+        assertThat(
+            previewWithOffset("-1d"),
+            contains("2017-01-08T00:00:00.000Z", "2017-01-15T00:00:00.000Z", "2017-01-22T00:00:00.000Z", "2017-01-29T00:00:00.000Z")
+        );
+        assertThat(
+            previewWithOffset("-2d"),
+            contains("2017-01-07T00:00:00.000Z", "2017-01-14T00:00:00.000Z", "2017-01-21T00:00:00.000Z", "2017-01-28T00:00:00.000Z")
+        );
+    }
+
+    @SuppressWarnings("unchecked")
+    private List<String> previewWithOffset(String offset) throws IOException {
+        setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME);
+        final Request createPreviewRequest = createRequestWithAuth("POST", getTransformEndpoint() + "_preview", null);
+        createPreviewRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE));
+
+        String config = Strings.format("""
+            {
+              "source": {
+                "index": "%s"
+              },
+              "pivot": {
+                "group_by": {
+                  "by_week": {
+                    "date_histogram": {
+                      "calendar_interval": "1w",
+                      "field": "timestamp",
+                      "offset": "%s"
+                    }
+                  }
+                },
+                "aggregations": {
+                  "user.avg_rating": {
+                    "avg": {
+                      "field": "stars"
+                    }
+                  }
+                }
+              }
+            }""", REVIEWS_INDEX_NAME, offset);
+        createPreviewRequest.setJsonEntity(config);
+
+        Map<String, Object> previewTransformResponse = entityAsMap(client().performRequest(createPreviewRequest));
+        List<Map<String, Object>> preview = (List<Map<String, Object>>) previewTransformResponse.get("preview");
+        return preview.stream().map(p -> (String) p.get("by_week")).toList();
+    }
+
     public void testPivotWithMaxOnDateField() throws Exception {
         String transformId = "simple_date_histogram_pivot_with_max_time";
         String transformIndex = "pivot_reviews_via_date_histogram_with_max_time";
@@ -1126,7 +1260,7 @@ public void testPivotWithMaxOnDateField() throws Exception {
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -1187,7 +1321,7 @@ public void testPivotWithScriptedMetricAgg() throws Exception {
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -1252,7 +1386,7 @@ public void testPivotWithBucketScriptAgg() throws Exception {
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -1323,7 +1457,7 @@ public void testPivotWithGeoBoundsAgg() throws Exception {
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -1392,7 +1526,7 @@ public void testPivotWithGeoCentroidAgg() throws Exception {
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -1457,7 +1591,7 @@ public void testPivotWithGeoLineAgg() throws Exception {
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -1528,7 +1662,7 @@ public void testPivotWithGeotileGroupBy() throws Exception {
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -1601,7 +1735,7 @@ public void testPivotWithWeightedAvgAgg() throws Exception {
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -1656,7 +1790,7 @@ public void testPivotWithTopMetrics() throws Exception {
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -1713,7 +1847,7 @@ public void testPivotWithAggregateMetricDouble() throws Exception {
         String statsField = "stars_stats";
         setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformIndex);
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -1796,7 +1930,7 @@ public void testManyBucketsWithSmallPageSize() throws Exception {
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -1890,7 +2024,7 @@ public void testContinuousStopWaitForCheckpoint() throws Exception {
             getTransformEndpoint() + transformId,
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -1957,7 +2091,7 @@ public void testContinuousDateNanos() throws Exception {
             getTransformEndpoint() + transformId,
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -2005,7 +2139,7 @@ public void testContinuousDateNanos() throws Exception {
 
         final StringBuilder bulk = new StringBuilder();
         for (int i = 0; i < 20; i++) {
-            bulk.append(formatted("""
+            bulk.append(Strings.format("""
                 {"index":{"_index":"%s"}}
                 {"id":"id_%s","rating":%s,"timestamp":"%s"}
                 """, indexName, i % 5, 7, nanoResolutionTimeStamp));
@@ -2042,7 +2176,7 @@ public void testPivotWithPercentile() throws Exception {
             getTransformEndpoint() + transformId,
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -2113,7 +2247,7 @@ public void testPivotWithRanges() throws Exception {
             getTransformEndpoint() + transformId,
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -2210,7 +2344,7 @@ public void testPivotWithFilter() throws Exception {
             getTransformEndpoint() + transformId,
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
         );
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -2363,7 +2497,7 @@ private void createDateNanoIndex(String indexName, int numDocs) throws IOExcepti
         String randomNanos = "," + randomIntBetween(100000000, 999999999);
         final StringBuilder bulk = new StringBuilder();
         for (int i = 0; i < numDocs; i++) {
-            bulk.append(formatted("""
+            bulk.append(Strings.format("""
                 {"index":{"_index":"%s"}}
                 {"id":"id_%s","rating":%s,"timestamp":"2020-01-27T01:59:00%sZ"}
                 """, indexName, i % 10, i % 7, randomNanos));
diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestSpecialCasesIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestSpecialCasesIT.java
index 66d5890268ae..d3dbc2d53a59 100644
--- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestSpecialCasesIT.java
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestSpecialCasesIT.java
@@ -72,7 +72,7 @@ public void testIndexTemplateMappingClash() throws Exception {
 
         final Request createTransformRequest = new Request("PUT", getTransformEndpoint() + transformId);
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -144,7 +144,7 @@ public void testSparseDataPercentiles() throws Exception {
         }
 
         final StringBuilder bulk = new StringBuilder();
-        bulk.append(formatted("""
+        bulk.append(Strings.format("""
             {"index":{"_index":"%s"}}
             {"host":"host-1","cpu": 22}
             {"index":{"_index":"%s"}}
@@ -162,7 +162,7 @@ public void testSparseDataPercentiles() throws Exception {
             """, indexName, indexName, indexName, indexName, indexName, indexName, indexName));
 
         // missing value for cpu
-        bulk.append(formatted("""
+        bulk.append(Strings.format("""
             {"index":{"_index":"%s"}}
             {"host":"host-3"}
 
@@ -174,7 +174,7 @@ public void testSparseDataPercentiles() throws Exception {
 
         final Request createTransformRequest = new Request("PUT", getTransformEndpoint() + transformId);
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -299,7 +299,7 @@ public void testEmptyKeyInTermsAgg() throws Exception {
         String transformIndex = "empty-terms-agg-key";
         String transformId = "empty-terms-agg-key";
         final Request createTransformRequest = new Request("PUT", getTransformEndpoint() + transformId);
-        final String config = formatted("""
+        final String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
@@ -363,7 +363,7 @@ public void testEmptyKeyInTermsAgg() throws Exception {
     private void verifyDestIndexHitsCount(String sourceIndex, String transformId, int maxPageSearchSize, long expectedDestIndexCount)
         throws Exception {
         String transformIndex = transformId;
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformResetIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformResetIT.java
index a6900a7b95ff..caa4fc8d0b0a 100644
--- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformResetIT.java
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformResetIT.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -79,7 +80,7 @@ public void testReset() throws Exception {
             getTransformEndpoint() + transformId,
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_1
         );
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "dest": {
                 "index": "%s"
diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java
index 84554c0d8ae2..5e741bd662e8 100644
--- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java
@@ -73,7 +73,7 @@ protected void createReviewsIndex(
         int hour = 10;
         int min = 10;
         for (int i = 0; i < numDocs; i++) {
-            bulk.append(formatted("""
+            bulk.append(Strings.format("""
                 {"create":{"_index":"%s"}}
                 """, indexName));
             long user = Math.round(Math.pow(i * 31 % 1000, distributionTable[i % distributionTable.length]) % numUsers);
@@ -206,7 +206,7 @@ protected void putReviewsIndex(String indexName, String dateType, boolean isData
             builder.endObject();
             if (isDataStream) {
                 Request createCompositeTemplate = new Request("PUT", "_index_template/" + indexName + "_template");
-                createCompositeTemplate.setJsonEntity(formatted("""
+                createCompositeTemplate.setJsonEntity(Strings.format("""
                     {
                       "index_patterns": [ "%s" ],
                       "data_stream": {
@@ -252,7 +252,7 @@ protected void createReviewsIndexNano() throws IOException {
     protected void createContinuousPivotReviewsTransform(String transformId, String transformIndex, String authHeader) throws IOException {
 
         // Set frequency high for testing
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "dest": {
                 "index": "%s"
@@ -311,18 +311,18 @@ protected void createPivotReviewsTransform(
         String config = "{";
 
         if (pipeline != null) {
-            config += formatted("""
+            config += Strings.format("""
                 "dest": {"index":"%s", "pipeline":"%s"},""", transformIndex, pipeline);
         } else {
-            config += formatted("""
+            config += Strings.format("""
                 "dest": {"index":"%s"},""", transformIndex);
         }
 
         if (query != null) {
-            config += formatted("""
+            config += Strings.format("""
                 "source": {"index":"%s", "query":{%s}},""", sourceIndex, query);
         } else {
-            config += formatted("""
+            config += Strings.format("""
                 "source": {"index":"%s"},""", sourceIndex);
         }
 
@@ -360,7 +360,7 @@ protected void createPivotReviewsTransform(
     }
 
     protected void createLatestReviewsTransform(String transformId, String transformIndex) throws IOException {
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "dest": {
                 "index": "%s"
@@ -400,15 +400,10 @@ protected void createPivotReviewsTransform(String transformId, String transformI
     }
 
     protected void startTransform(String transformId) throws IOException {
-        startTransform(transformId, null);
+        startTransform(transformId, null, null, null);
     }
 
-    protected void startTransform(String transformId, String authHeader, String... warnings) throws IOException {
-        // start the transform
-        startTransform(transformId, authHeader, null, warnings);
-    }
-
-    protected void startTransform(String transformId, String authHeader, String secondaryAuthHeader, String... warnings)
+    protected void startTransform(String transformId, String authHeader, String secondaryAuthHeader, String from, String... warnings)
         throws IOException {
         // start the transform
         final Request startTransformRequest = createRequestWithSecondaryAuth(
@@ -417,6 +412,9 @@ protected void startTransform(String transformId, String authHeader, String seco
             authHeader,
             secondaryAuthHeader
         );
+        if (from != null) {
+            startTransformRequest.addParameter(TransformField.FROM.getPreferredName(), from);
+        }
         if (warnings.length > 0) {
             startTransformRequest.setOptions(expectWarnings(warnings));
         }
@@ -462,7 +460,7 @@ protected void startAndWaitForTransform(
         String... warnings
     ) throws Exception {
         // start the transform
-        startTransform(transformId, authHeader, secondaryAuthHeader, warnings);
+        startTransform(transformId, authHeader, secondaryAuthHeader, null, warnings);
         assertTrue(indexExists(transformIndex));
         // wait until the transform has been created and all data is available
         waitForTransformCheckpoint(transformId);
@@ -472,13 +470,18 @@ protected void startAndWaitForTransform(
     }
 
     protected void startAndWaitForContinuousTransform(String transformId, String transformIndex, String authHeader) throws Exception {
-        startAndWaitForContinuousTransform(transformId, transformIndex, authHeader, 1L);
+        startAndWaitForContinuousTransform(transformId, transformIndex, authHeader, null, 1L);
     }
 
-    protected void startAndWaitForContinuousTransform(String transformId, String transformIndex, String authHeader, long checkpoint)
-        throws Exception {
+    protected void startAndWaitForContinuousTransform(
+        String transformId,
+        String transformIndex,
+        String authHeader,
+        String from,
+        long checkpoint
+    ) throws Exception {
         // start the transform
-        startTransform(transformId, authHeader, new String[0]);
+        startTransform(transformId, authHeader, null, from, new String[0]);
         assertTrue(indexExists(transformIndex));
         // wait until the transform has been created and all data is available
         waitForTransformCheckpoint(transformId, checkpoint);
@@ -492,6 +495,16 @@ protected void resetTransform(String transformId, boolean force) throws IOExcept
         assertThat(resetTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
     }
 
+    protected void scheduleNowTransform(String transformId) throws IOException {
+        final Request scheduleNowTransformRequest = createRequestWithAuth(
+            "POST",
+            getTransformEndpoint() + transformId + "/_schedule_now",
+            null
+        );
+        Map<String, Object> scheduleNowTransformResponse = entityAsMap(client().performRequest(scheduleNowTransformRequest));
+        assertThat(scheduleNowTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
+    }
+
     protected Request createRequestWithSecondaryAuth(
         final String method,
         final String endpoint,
@@ -605,7 +618,7 @@ static int getTransformCheckpoint(String transformId) throws IOException {
     protected void setupDataAccessRole(String role, String... indices) throws IOException {
         String indicesStr = Arrays.stream(indices).collect(Collectors.joining("\",\"", "\"", "\""));
         Request request = new Request("PUT", "/_security/role/" + role);
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {
               "indices": [
                 {
@@ -622,7 +635,7 @@ protected void setupUser(String user, List<String> roles) throws IOException {
 
         String rolesStr = roles.stream().collect(Collectors.joining("\",\"", "\"", "\""));
         Request request = new Request("PUT", "/_security/user/" + user);
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {  "password" : "%s",  "roles" : [ %s ]}
             """, password, rolesStr));
         client().performRequest(request);
diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRobustnessIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRobustnessIT.java
index 8022bb6b48d6..1ed27453fadd 100644
--- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRobustnessIT.java
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRobustnessIT.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.xpack.core.transform.TransformField;
 import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants;
 
@@ -32,7 +33,7 @@ public void testTaskRemovalAfterInternalIndexGotDeleted() throws Exception {
         String transformId = "simple_continuous_pivot";
         String transformIndex = "pivot_reviews_continuous";
         final Request createTransformRequest = new Request("PUT", TransformField.REST_BASE_PATH_TRANSFORMS + transformId);
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": "%s"
diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformScheduleNowIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformScheduleNowIT.java
new file mode 100644
index 000000000000..2a5236445fa1
--- /dev/null
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformScheduleNowIT.java
@@ -0,0 +1,202 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.transform.integration;
+
+import org.apache.http.HttpHost;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+public class TransformScheduleNowIT extends TransformRestTestCase {
+
+    private static final String TEST_USER_NAME = "transform_user";
+    private static final String TEST_ADMIN_USER_NAME_1 = "transform_admin_1";
+    private static final String BASIC_AUTH_VALUE_TRANSFORM_ADMIN_1 = basicAuthHeaderValue(
+        TEST_ADMIN_USER_NAME_1,
+        TEST_PASSWORD_SECURE_STRING
+    );
+    private static final String DATA_ACCESS_ROLE = "test_data_access";
+
+    private static boolean indicesCreated = false;
+
+    // preserve indices in order to reuse source indices in several test cases
+    @Override
+    protected boolean preserveIndicesUponCompletion() {
+        return true;
+    }
+
+    @Override
+    protected boolean enableWarningsCheck() {
+        return false;
+    }
+
+    @Override
+    protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException {
+        RestClientBuilder builder = RestClient.builder(hosts);
+        configureClient(builder, settings);
+        builder.setStrictDeprecationMode(false);
+        return builder.build();
+    }
+
+    @Before
+    public void createIndexes() throws IOException {
+        setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME);
+        setupUser(TEST_USER_NAME, Arrays.asList("transform_user", DATA_ACCESS_ROLE));
+        setupUser(TEST_ADMIN_USER_NAME_1, Arrays.asList("transform_admin", DATA_ACCESS_ROLE));
+
+        // it's not possible to run it as @BeforeClass as clients aren't initialized then, so we need this little hack
+        if (indicesCreated) {
+            return;
+        }
+
+        createReviewsIndex();
+        indicesCreated = true;
+    }
+
+    public void testScheduleNow() throws Exception {
+        String sourceIndex = REVIEWS_INDEX_NAME;
+        String transformId = "old_transform";
+        String destIndex = transformId + "_idx";
+        setupDataAccessRole(DATA_ACCESS_ROLE, sourceIndex, destIndex);
+
+        final Request createTransformRequest = createRequestWithAuth(
+            "PUT",
+            getTransformEndpoint() + transformId,
+            BASIC_AUTH_VALUE_TRANSFORM_ADMIN_1
+        );
+        String config = Strings.format("""
+            {
+              "dest": {
+                "index": "%s"
+              },
+              "source": {
+                "index": "%s"
+              },
+              "pivot": {
+                "group_by": {
+                  "reviewer": {
+                    "terms": {
+                      "field": "user_id"
+                    }
+                  }
+                },
+                "aggregations": {
+                  "avg_rating": {
+                    "avg": {
+                      "field": "stars"
+                    }
+                  }
+                }
+              },
+              "sync": {
+                "time": {
+                  "field": "timestamp",
+                  "delay": "1s"
+                }
+              },
+              "frequency": "1h"
+            }""", destIndex, sourceIndex);
+        createTransformRequest.setJsonEntity(config);
+        Map<String, Object> createTransformResponse = entityAsMap(client().performRequest(createTransformRequest));
+        assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
+
+        // Verify that _schedule_now is a no-op on a new transform
+        scheduleNowTransform(transformId);
+
+        // Start the transform, notice that frequency is set pretty high
+        startAndWaitForContinuousTransform(transformId, destIndex, null, null, 1L);
+
+        String newUser = "user_666";
+        verifyNumberOfSourceDocs(sourceIndex, newUser, 0);
+        verifyDestDoc(destIndex, newUser, 0, null);
+
+        // Ingest a new document to *source* index
+        indexSourceDoc(sourceIndex, newUser, 7);
+
+        // Wait a little bit to accommodate differences between "now" timestamp in the test and the current time in the server
+        Thread.sleep(5_000);
+
+        // Verify the new data is in the source index but not yet in the destination index
+        verifyNumberOfSourceDocs(sourceIndex, newUser, 1);
+        verifyDestDoc(destIndex, newUser, 0, null);
+
+        // Schedule now the transform to force processing the new data despite 1h-long interval
+        scheduleNowTransform(transformId);
+        waitForTransformCheckpoint(transformId, 2L);
+
+        // Verify that the new data is available in the destination index after _schedule_now
+        verifyNumberOfSourceDocs(sourceIndex, newUser, 1);
+        verifyDestDoc(destIndex, newUser, 1, 7.0);
+
+        // Ingest a new document to *source* index
+        indexSourceDoc(sourceIndex, newUser, 9);
+
+        // Wait a little bit to accommodate differences between "now" timestamp in the test and the current time in the server
+        Thread.sleep(5_000);
+
+        // Verify the new data is in the source index but not yet in the destination index
+        verifyNumberOfSourceDocs(sourceIndex, newUser, 2);
+        verifyDestDoc(destIndex, newUser, 1, 7.0);
+
+        // Try scheduling now all the transforms at once using _all wildcard, it is *not* supported
+        ResponseException e = expectThrows(ResponseException.class, () -> scheduleNowTransform("_all"));
+        assertThat(e.getMessage(), containsString("_schedule_now API does not support _all wildcard"));
+
+        // Schedule now the transform to force processing the new data despite 1h-long interval
+        scheduleNowTransform(transformId);
+        waitForTransformCheckpoint(transformId, 3L);
+
+        // Verify that the new data is available in the destination index after _schedule_now
+        verifyNumberOfSourceDocs(sourceIndex, newUser, 2);
+        verifyDestDoc(destIndex, newUser, 1, 8.0);  // 8.0 = (7.0 + 9.0) / 2
+
+        // Verify that _schedule_now works on a stopped transform
+        stopTransform(transformId, false);
+        scheduleNowTransform(transformId);
+    }
+
+    private void indexSourceDoc(String sourceIndex, String user, int stars) throws IOException {
+        String doc = Strings.format("""
+            {"user_id":"%s","stars":%s,"timestamp":%s}
+            """, user, stars, Instant.now().toEpochMilli());
+
+        Request indexRequest = new Request("POST", sourceIndex + "/_doc");
+        indexRequest.addParameter("refresh", "true");
+        indexRequest.setJsonEntity(doc);
+
+        Map<String, Object> indexResponse = entityAsMap(client().performRequest(indexRequest));
+        assertThat(indexResponse.get("result"), equalTo("created"));
+    }
+
+    private void verifyNumberOfSourceDocs(String sourceIndex, String user, int expectedDocCount) throws IOException {
+        Map<String, Object> searchResult = getAsMap(sourceIndex + "/_search?q=user_id:" + user);
+        assertEquals(expectedDocCount, XContentMapValues.extractValue("hits.total.value", searchResult));
+    }
+
+    private void verifyDestDoc(String destIndex, String user, int expectedDocCount, Double expectedAvgRating) throws IOException {
+        Map<String, Object> searchResult = getAsMap(destIndex + "/_search?q=reviewer:" + user);
+        assertEquals(expectedDocCount, XContentMapValues.extractValue("hits.total.value", searchResult));
+        if (expectedAvgRating != null) {
+            assertEquals(List.of(expectedAvgRating), XContentMapValues.extractValue("hits.hits._source.avg_rating", searchResult));
+        }
+    }
+}
diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUpdateIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUpdateIT.java
index 78b1ab7ed378..11f12c38b2ab 100644
--- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUpdateIT.java
+++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUpdateIT.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -87,7 +88,7 @@ public void testUpdateDeprecatedSettings() throws Exception {
             getTransformEndpoint() + transformId,
             BASIC_AUTH_VALUE_TRANSFORM_ADMIN_1
         );
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "dest": {
                 "index": "%s"
@@ -181,7 +182,7 @@ private void updateTransferRightsTester(boolean useSecondaryAuthHeaders) throws
             )
             : createRequestWithAuth("PUT", getTransformEndpoint() + transformIdCloned, BASIC_AUTH_VALUE_TRANSFORM_ADMIN_2);
 
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "dest": {
                 "index": "%s"
diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCheckpointServiceNodeTests.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCheckpointServiceNodeTests.java
index a4d52f0d278e..bd52af4331a6 100644
--- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCheckpointServiceNodeTests.java
+++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCheckpointServiceNodeTests.java
@@ -379,7 +379,8 @@ private static ShardStats[] createShardStats(Map<String, long[]> checkpoints) {
                     shardId,
                     true,
                     RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)
+                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null),
+                    ShardRouting.Role.DEFAULT
                 );
                 Path path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve(String.valueOf(i));
 
diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformInternalIndexIT.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformInternalIndexIT.java
index e71265eb18e5..5b1348f1b1ab 100644
--- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformInternalIndexIT.java
+++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformInternalIndexIT.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
@@ -65,7 +66,7 @@ public void testUpdateDeletesOldTransformConfig() throws Exception {
         String transformIndex = "transform-index-deletes-old";
         createSourceIndex(transformIndex);
         String transformId = "transform-update-deletes-old-transform-config";
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "dest": {
                 "index": "bar"
diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformNoTransformNodeIT.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformNoTransformNodeIT.java
index a1abfbddaf33..710dcd424e69 100644
--- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformNoTransformNodeIT.java
+++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformNoTransformNodeIT.java
@@ -41,7 +41,7 @@ protected Settings nodeSettings() {
     }
 
     public void testGetTransformStats() {
-        GetTransformStatsAction.Request request = new GetTransformStatsAction.Request("_all");
+        GetTransformStatsAction.Request request = new GetTransformStatsAction.Request("_all", AcknowledgedRequest.DEFAULT_ACK_TIMEOUT);
         GetTransformStatsAction.Response response = client().execute(GetTransformStatsAction.INSTANCE, request).actionGet();
         assertThat(response.getTransformsStats(), is(empty()));
 
diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformOldTransformsIT.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformOldTransformsIT.java
index e5642f2209f0..2c63596e3f83 100644
--- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformOldTransformsIT.java
+++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformOldTransformsIT.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
@@ -77,7 +78,7 @@ public void testStopThrowsForDeprecatedTransformConfig() throws Exception {
             Version.V_7_2_0,
             VersionUtils.getPreviousVersion(TransformDeprecations.MIN_TRANSFORM_VERSION)
         );
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "dest": {
                 "index": "bar"
@@ -122,6 +123,7 @@ public void testStopThrowsForDeprecatedTransformConfig() throws Exception {
 
         StartTransformAction.Request startTransformRequest = new StartTransformAction.Request(
             transformId,
+            null,
             AcknowledgedRequest.DEFAULT_ACK_TIMEOUT
         );
 
diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManagerTests.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManagerTests.java
index 404dfa51d665..fafb905198d1 100644
--- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManagerTests.java
+++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManagerTests.java
@@ -233,7 +233,13 @@ public void testExpandIds() throws Exception {
 
         // expand 1 id
         assertAsync(
-            listener -> transformConfigManager.expandTransformIds(transformConfig1.getId(), PageParams.defaultParams(), true, listener),
+            listener -> transformConfigManager.expandTransformIds(
+                transformConfig1.getId(),
+                PageParams.defaultParams(),
+                null,
+                true,
+                listener
+            ),
             tuple(1L, tuple(singletonList("transform1_expand"), singletonList(transformConfig1))),
             null,
             null
@@ -244,6 +250,7 @@ public void testExpandIds() throws Exception {
             listener -> transformConfigManager.expandTransformIds(
                 "transform1_expand,transform2_expand",
                 PageParams.defaultParams(),
+                null,
                 true,
                 listener
             ),
@@ -257,6 +264,7 @@ public void testExpandIds() throws Exception {
             listener -> transformConfigManager.expandTransformIds(
                 "transform1*,transform2_expand,transform3_expand",
                 PageParams.defaultParams(),
+                null,
                 true,
                 listener
             ),
@@ -273,7 +281,7 @@ public void testExpandIds() throws Exception {
 
         // expand 3 ids _all
         assertAsync(
-            listener -> transformConfigManager.expandTransformIds("_all", PageParams.defaultParams(), true, listener),
+            listener -> transformConfigManager.expandTransformIds("_all", PageParams.defaultParams(), null, true, listener),
             tuple(
                 3L,
                 tuple(
@@ -287,7 +295,7 @@ public void testExpandIds() throws Exception {
 
         // expand 1 id _all with pagination
         assertAsync(
-            listener -> transformConfigManager.expandTransformIds("_all", new PageParams(0, 1), true, listener),
+            listener -> transformConfigManager.expandTransformIds("_all", new PageParams(0, 1), null, true, listener),
             tuple(3L, tuple(singletonList("transform1_expand"), singletonList(transformConfig1))),
             null,
             null
@@ -295,7 +303,7 @@ public void testExpandIds() throws Exception {
 
         // expand 2 later ids _all with pagination
         assertAsync(
-            listener -> transformConfigManager.expandTransformIds("_all", new PageParams(1, 2), true, listener),
+            listener -> transformConfigManager.expandTransformIds("_all", new PageParams(1, 2), null, true, listener),
             tuple(3L, tuple(Arrays.asList("transform2_expand", "transform3_expand"), Arrays.asList(transformConfig2, transformConfig3))),
             null,
             null
@@ -303,7 +311,7 @@ public void testExpandIds() throws Exception {
 
         // expand 1 id explicitly that does not exist
         assertAsync(
-            listener -> transformConfigManager.expandTransformIds("unknown,unknown2", new PageParams(1, 2), true, listener),
+            listener -> transformConfigManager.expandTransformIds("unknown,unknown2", new PageParams(1, 2), null, true, listener),
             (Tuple<Long, Tuple<List<String>, List<TransformConfig>>>) null,
             null,
             e -> {
@@ -317,7 +325,7 @@ public void testExpandIds() throws Exception {
 
         // expand 1 id implicitly that does not exist
         assertAsync(
-            listener -> transformConfigManager.expandTransformIds("unknown*", new PageParams(1, 2), false, listener),
+            listener -> transformConfigManager.expandTransformIds("unknown*", new PageParams(1, 2), null, false, listener),
             (Tuple<Long, Tuple<List<String>, List<TransformConfig>>>) null,
             null,
             e -> {
@@ -348,6 +356,7 @@ public void testExpandIds() throws Exception {
             listener -> transformConfigManager.expandTransformIds(
                 "transform1_expand,transform2_expand",
                 PageParams.defaultParams(),
+                null,
                 true,
                 listener
             ),
@@ -368,25 +377,25 @@ public void testGetAllTransformIdsAndGetAllOutdatedTransformIds() throws Excepti
             TransformConfig transformConfig = TransformConfigTests.randomTransformConfig(id);
             assertAsync(listener -> transformConfigManager.putTransformConfiguration(transformConfig, listener), true, null, null);
         }
-        assertAsync(listener -> transformConfigManager.getAllTransformIds(listener), transformIds, null, null);
+        assertAsync(listener -> transformConfigManager.getAllTransformIds(null, listener), transformIds, null, null);
 
         // test recursive retrieval
         assertAsync(
-            listener -> transformConfigManager.expandAllTransformIds(false, 10, listener),
+            listener -> transformConfigManager.expandAllTransformIds(false, 10, null, listener),
             tuple(Long.valueOf(numberOfTransformsToGenerate), transformIds),
             null,
             null
         );
 
         assertAsync(
-            listener -> transformConfigManager.getAllOutdatedTransformIds(listener),
+            listener -> transformConfigManager.getAllOutdatedTransformIds(null, listener),
             tuple(Long.valueOf(numberOfTransformsToGenerate), Collections.<String>emptySet()),
             null,
             null
         );
 
         assertAsync(
-            listener -> transformConfigManager.expandAllTransformIds(true, 10, listener),
+            listener -> transformConfigManager.expandAllTransformIds(true, 10, null, listener),
             tuple(Long.valueOf(numberOfTransformsToGenerate), Collections.<String>emptySet()),
             null,
             null
@@ -410,9 +419,9 @@ public void testGetAllTransformIdsAndGetAllOutdatedTransformIds() throws Excepti
             client().index(request).actionGet();
         }
 
-        assertAsync(listener -> transformConfigManager.getAllTransformIds(listener), transformIds, null, null);
+        assertAsync(listener -> transformConfigManager.getAllTransformIds(null, listener), transformIds, null, null);
         assertAsync(
-            listener -> transformConfigManager.getAllOutdatedTransformIds(listener),
+            listener -> transformConfigManager.getAllOutdatedTransformIds(null, listener),
             tuple(Long.valueOf(numberOfTransformsToGenerate), Collections.<String>emptySet()),
             null,
             null
@@ -443,16 +452,16 @@ public void testGetAllTransformIdsAndGetAllOutdatedTransformIds() throws Excepti
         );
 
         transformIds.add(oldTransformId);
-        assertAsync(listener -> transformConfigManager.getAllTransformIds(listener), transformIds, null, null);
+        assertAsync(listener -> transformConfigManager.getAllTransformIds(null, listener), transformIds, null, null);
         assertAsync(
-            listener -> transformConfigManager.getAllOutdatedTransformIds(listener),
+            listener -> transformConfigManager.getAllOutdatedTransformIds(null, listener),
             tuple(Long.valueOf(numberOfTransformsToGenerate + 1), Collections.singleton(oldTransformId)),
             null,
             null
         );
 
         assertAsync(
-            listener -> transformConfigManager.expandAllTransformIds(true, 10, listener),
+            listener -> transformConfigManager.expandAllTransformIds(true, 10, null, listener),
             tuple(Long.valueOf(numberOfTransformsToGenerate + 1), Collections.singleton(oldTransformId)),
             null,
             null
@@ -517,7 +526,7 @@ public void testGetStoredDocMultiple() throws InterruptedException {
 
         // returned docs will be ordered by id
         expectedDocs.sort(Comparator.comparing(TransformStoredDoc::getId));
-        assertAsync(listener -> transformConfigManager.getTransformStoredDocs(ids, listener), expectedDocs, null, null);
+        assertAsync(listener -> transformConfigManager.getTransformStoredDocs(ids, null, listener), expectedDocs, null, null);
     }
 
     public void testDeleteOldTransformConfigurations() throws Exception {
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java
index 6ea369b90d99..d5ac73afde1a 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java
@@ -24,7 +24,7 @@
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -68,6 +68,7 @@
 import org.elasticsearch.xpack.core.transform.action.PreviewTransformAction;
 import org.elasticsearch.xpack.core.transform.action.PutTransformAction;
 import org.elasticsearch.xpack.core.transform.action.ResetTransformAction;
+import org.elasticsearch.xpack.core.transform.action.ScheduleNowTransformAction;
 import org.elasticsearch.xpack.core.transform.action.SetResetModeAction;
 import org.elasticsearch.xpack.core.transform.action.StartTransformAction;
 import org.elasticsearch.xpack.core.transform.action.StopTransformAction;
@@ -83,6 +84,7 @@
 import org.elasticsearch.xpack.transform.action.TransportPreviewTransformAction;
 import org.elasticsearch.xpack.transform.action.TransportPutTransformAction;
 import org.elasticsearch.xpack.transform.action.TransportResetTransformAction;
+import org.elasticsearch.xpack.transform.action.TransportScheduleNowTransformAction;
 import org.elasticsearch.xpack.transform.action.TransportSetTransformResetModeAction;
 import org.elasticsearch.xpack.transform.action.TransportStartTransformAction;
 import org.elasticsearch.xpack.transform.action.TransportStopTransformAction;
@@ -101,6 +103,7 @@
 import org.elasticsearch.xpack.transform.rest.action.RestPreviewTransformAction;
 import org.elasticsearch.xpack.transform.rest.action.RestPutTransformAction;
 import org.elasticsearch.xpack.transform.rest.action.RestResetTransformAction;
+import org.elasticsearch.xpack.transform.rest.action.RestScheduleNowTransformAction;
 import org.elasticsearch.xpack.transform.rest.action.RestStartTransformAction;
 import org.elasticsearch.xpack.transform.rest.action.RestStopTransformAction;
 import org.elasticsearch.xpack.transform.rest.action.RestUpdateTransformAction;
@@ -190,7 +193,8 @@ public List<RestHandler> getRestHandlers(
             new RestUpdateTransformAction(),
             new RestCatTransformAction(),
             new RestUpgradeTransformsAction(),
-            new RestResetTransformAction()
+            new RestResetTransformAction(),
+            new RestScheduleNowTransformAction()
         );
     }
 
@@ -209,6 +213,7 @@ public List<RestHandler> getRestHandlers(
             new ActionHandler<>(SetResetModeAction.INSTANCE, TransportSetTransformResetModeAction.class),
             new ActionHandler<>(UpgradeTransformsAction.INSTANCE, TransportUpgradeTransformsAction.class),
             new ActionHandler<>(ResetTransformAction.INSTANCE, TransportResetTransformAction.class),
+            new ActionHandler<>(ScheduleNowTransformAction.INSTANCE, TransportScheduleNowTransformAction.class),
 
             // internal, no rest endpoint
             new ActionHandler<>(ValidateTransformAction.INSTANCE, TransportValidateTransformAction.class),
@@ -235,7 +240,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver expressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         TransformConfigManager configManager = new IndexBasedTransformConfigManager(
             clusterService,
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransformPrivilegeChecker.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransformPrivilegeChecker.java
index 07ef74396672..453877e0ed31 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransformPrivilegeChecker.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransformPrivilegeChecker.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.license.RemoteClusterLicenseChecker;
 import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesAction;
 import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
@@ -23,9 +24,11 @@
 import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
+import static java.util.function.Predicate.not;
 import static java.util.stream.Collectors.joining;
 import static java.util.stream.Collectors.toList;
 import static org.elasticsearch.xpack.transform.utils.SecondaryAuthorizationUtils.useSecondaryAuthIfAvailable;
@@ -60,7 +63,11 @@ static void checkPrivileges(
                 username,
                 checkDestIndexPrivileges
             );
-            client.execute(HasPrivilegesAction.INSTANCE, hasPrivilegesRequest, hasPrivilegesResponseListener);
+            if (hasPrivilegesRequest.indexPrivileges().length == 0) {
+                listener.onResponse(null);
+            } else {
+                client.execute(HasPrivilegesAction.INSTANCE, hasPrivilegesRequest, hasPrivilegesResponseListener);
+            }
         });
     }
 
@@ -73,13 +80,19 @@ private static HasPrivilegesRequest buildPrivilegesRequest(
     ) {
         List<RoleDescriptor.IndicesPrivileges> indicesPrivileges = new ArrayList<>(2);
 
-        RoleDescriptor.IndicesPrivileges sourceIndexPrivileges = RoleDescriptor.IndicesPrivileges.builder()
-            .indices(config.getSource().getIndex())
-            // We need to read the source indices mapping to deduce the destination mapping, hence the need for view_index_metadata
-            .privileges("read", "view_index_metadata")
-            .build();
-        indicesPrivileges.add(sourceIndexPrivileges);
+        // TODO: Remove this filter once https://github.com/elastic/elasticsearch/issues/67798 is fixed.
+        String[] sourceIndex = Arrays.stream(config.getSource().getIndex())
+            .filter(not(RemoteClusterLicenseChecker::isRemoteIndex))
+            .toArray(String[]::new);
 
+        if (sourceIndex.length > 0) {
+            RoleDescriptor.IndicesPrivileges sourceIndexPrivileges = RoleDescriptor.IndicesPrivileges.builder()
+                .indices(sourceIndex)
+                // We need to read the source indices mapping to deduce the destination mapping, hence the need for view_index_metadata
+                .privileges("read", "view_index_metadata")
+                .build();
+            indicesPrivileges.add(sourceIndexPrivileges);
+        }
         if (checkDestIndexPrivileges) {
             final String destIndex = config.getDestination().getIndex();
             final String[] concreteDest = indexNameExpressionResolver.concreteIndexNames(
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointAction.java
index 579a76626d16..538779ea7659 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointAction.java
@@ -143,6 +143,7 @@ protected AsyncGetCheckpointsFromNodesAction(
 
         public void start() {
             GroupedActionListener<GetCheckpointNodeAction.Response> groupedListener = new GroupedActionListener<>(
+                nodesAndShards.size(),
                 ActionListener.wrap(responses -> {
                     // the final list should be ordered by key
                     Map<String, long[]> checkpointsByIndexReduced = new TreeMap<>();
@@ -162,8 +163,7 @@ public void start() {
                     }
 
                     listener.onResponse(new Response(checkpointsByIndexReduced));
-                }, listener::onFailure),
-                nodesAndShards.size()
+                }, listener::onFailure)
             );
 
             for (Entry<String, Set<ShardId>> oneNodeAndItsShards : nodesAndShards.entrySet()) {
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java
index faf910a25cf2..cfa532735d32 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.client.internal.ParentTaskAssigningClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
@@ -27,6 +28,7 @@
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata.Assignment;
 import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.transform.action.GetTransformStatsAction;
@@ -138,12 +140,15 @@ protected void taskOperation(Task actionTask, Request request, TransformTask tas
 
     @Override
     protected void doExecute(Task task, Request request, ActionListener<Response> finalListener) {
+        TaskId parentTaskId = new TaskId(clusterService.localNode().getId(), task.getId());
+
         final ClusterState clusterState = clusterService.state();
         TransformNodes.warnIfNoTransformNodes(clusterState);
 
         transformConfigManager.expandTransformIds(
             request.getId(),
             request.getPageParams(),
+            request.getTimeout(),
             request.isAllowNoMatch(),
             ActionListener.wrap(hitsAndIds -> {
                 boolean hasAnyTransformNode = TransformNodes.hasAnyTransformNode(clusterState.getNodes());
@@ -175,6 +180,7 @@ protected void doExecute(Task task, Request request, ActionListener<Response> fi
                         response.getTransformsStats().forEach(dtsasi -> setNodeAttributes(dtsasi, tasksInProgress, clusterState));
                     }
                     collectStatsForTransformsWithoutTasks(
+                        parentTaskId,
                         request,
                         response,
                         transformNodeAssignments.getWaitingForAssignment(),
@@ -246,6 +252,7 @@ static TransformStats deriveStats(TransformTask task, @Nullable TransformCheckpo
     }
 
     private void collectStatsForTransformsWithoutTasks(
+        TaskId parentTaskId,
         Request request,
         Response response,
         Set<String> transformsWaitingForAssignment,
@@ -270,6 +277,7 @@ private void collectStatsForTransformsWithoutTasks(
             // copy the list as it might be immutable
             List<TransformStats> allStateAndStats = new ArrayList<>(response.getTransformsStats());
             addCheckpointingInfoForTransformsWithoutTasks(
+                parentTaskId,
                 allStateAndStats,
                 statsForTransformsWithoutTasks,
                 transformsWaitingForAssignment,
@@ -299,12 +307,16 @@ private void collectStatsForTransformsWithoutTasks(
             }
         });
 
-        transformConfigManager.getTransformStoredDocs(transformsWithoutTasks, searchStatsListener);
+        transformConfigManager.getTransformStoredDocs(transformsWithoutTasks, request.getTimeout(), searchStatsListener);
     }
 
-    private void populateSingleStoppedTransformStat(TransformStoredDoc transform, ActionListener<TransformCheckpointingInfo> listener) {
+    private void populateSingleStoppedTransformStat(
+        TransformStoredDoc transform,
+        TaskId parentTaskId,
+        ActionListener<TransformCheckpointingInfo> listener
+    ) {
         transformCheckpointService.getCheckpointingInfo(
-            client,
+            new ParentTaskAssigningClient(client, parentTaskId),
             transform.getId(),
             transform.getTransformState().getCheckpoint(),
             transform.getTransformState().getPosition(),
@@ -317,6 +329,7 @@ private void populateSingleStoppedTransformStat(TransformStoredDoc transform, Ac
     }
 
     private void addCheckpointingInfoForTransformsWithoutTasks(
+        TaskId parentTaskId,
         List<TransformStats> allStateAndStats,
         List<TransformStoredDoc> statsForTransformsWithoutTasks,
         Set<String> transformsWaitingForAssignment,
@@ -333,42 +346,44 @@ private void addCheckpointingInfoForTransformsWithoutTasks(
         AtomicInteger numberRemaining = new AtomicInteger(statsForTransformsWithoutTasks.size());
         AtomicBoolean isExceptionReported = new AtomicBoolean(false);
 
-        statsForTransformsWithoutTasks.forEach(stat -> populateSingleStoppedTransformStat(stat, ActionListener.wrap(checkpointingInfo -> {
-            synchronized (allStateAndStats) {
-                if (transformsWaitingForAssignment.contains(stat.getId())) {
-                    Assignment assignment = TransformNodes.getAssignment(stat.getId(), clusterState);
-                    allStateAndStats.add(
-                        new TransformStats(
-                            stat.getId(),
-                            TransformStats.State.WAITING,
-                            assignment.getExplanation(),
-                            null,
-                            stat.getTransformStats(),
-                            checkpointingInfo,
-                            TransformHealthChecker.checkUnassignedTransform(stat.getId(), clusterState)
-                        )
-                    );
-                } else {
-                    allStateAndStats.add(
-                        new TransformStats(
-                            stat.getId(),
-                            TransformStats.State.STOPPED,
-                            null,
-                            null,
-                            stat.getTransformStats(),
-                            checkpointingInfo,
-                            TransformHealth.GREEN
-                        )
-                    );
+        statsForTransformsWithoutTasks.forEach(
+            stat -> populateSingleStoppedTransformStat(stat, parentTaskId, ActionListener.wrap(checkpointingInfo -> {
+                synchronized (allStateAndStats) {
+                    if (transformsWaitingForAssignment.contains(stat.getId())) {
+                        Assignment assignment = TransformNodes.getAssignment(stat.getId(), clusterState);
+                        allStateAndStats.add(
+                            new TransformStats(
+                                stat.getId(),
+                                TransformStats.State.WAITING,
+                                assignment.getExplanation(),
+                                null,
+                                stat.getTransformStats(),
+                                checkpointingInfo,
+                                TransformHealthChecker.checkUnassignedTransform(stat.getId(), clusterState)
+                            )
+                        );
+                    } else {
+                        allStateAndStats.add(
+                            new TransformStats(
+                                stat.getId(),
+                                TransformStats.State.STOPPED,
+                                null,
+                                null,
+                                stat.getTransformStats(),
+                                checkpointingInfo,
+                                TransformHealth.GREEN
+                            )
+                        );
+                    }
                 }
-            }
-            if (numberRemaining.decrementAndGet() == 0) {
-                listener.onResponse(null);
-            }
-        }, e -> {
-            if (isExceptionReported.compareAndSet(false, true)) {
-                listener.onFailure(e);
-            }
-        })));
+                if (numberRemaining.decrementAndGet() == 0) {
+                    listener.onResponse(null);
+                }
+            }, e -> {
+                if (isExceptionReported.compareAndSet(false, true)) {
+                    listener.onFailure(e);
+                }
+            }))
+        );
     }
 }
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportScheduleNowTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportScheduleNowTransformAction.java
new file mode 100644
index 000000000000..98918bb7dcc8
--- /dev/null
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportScheduleNowTransformAction.java
@@ -0,0 +1,179 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.transform.action;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.FailedNodeException;
+import org.elasticsearch.action.TaskOperationFailure;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.tasks.TransportTasksAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.ActionNotFoundTransportException;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.transform.action.ScheduleNowTransformAction;
+import org.elasticsearch.xpack.core.transform.action.ScheduleNowTransformAction.Request;
+import org.elasticsearch.xpack.core.transform.action.ScheduleNowTransformAction.Response;
+import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
+import org.elasticsearch.xpack.core.transform.transforms.TransformState;
+import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState;
+import org.elasticsearch.xpack.transform.TransformServices;
+import org.elasticsearch.xpack.transform.persistence.TransformConfigManager;
+import org.elasticsearch.xpack.transform.transforms.TransformTask;
+import org.elasticsearch.xpack.transform.transforms.scheduling.TransformScheduler;
+
+import java.util.List;
+
+import static org.elasticsearch.core.Strings.format;
+import static org.elasticsearch.xpack.transform.utils.SecondaryAuthorizationUtils.useSecondaryAuthIfAvailable;
+
+public class TransportScheduleNowTransformAction extends TransportTasksAction<TransformTask, Request, Response, Response> {
+
+    private static final Logger logger = LogManager.getLogger(TransportScheduleNowTransformAction.class);
+    private final TransformConfigManager transformConfigManager;
+    private final TransformScheduler transformScheduler;
+    private final SecurityContext securityContext;
+
+    @Inject
+    public TransportScheduleNowTransformAction(
+        Settings settings,
+        TransportService transportService,
+        ThreadPool threadPool,
+        ActionFilters actionFilters,
+        ClusterService clusterService,
+        TransformServices transformServices
+    ) {
+        super(
+            ScheduleNowTransformAction.NAME,
+            clusterService,
+            transportService,
+            actionFilters,
+            Request::new,
+            Response::new,
+            Response::new,
+            ThreadPool.Names.SAME
+        );
+
+        this.transformConfigManager = transformServices.getConfigManager();
+        this.transformScheduler = transformServices.getScheduler();
+        this.securityContext = XPackSettings.SECURITY_ENABLED.get(settings)
+            ? new SecurityContext(settings, threadPool.getThreadContext())
+            : null;
+    }
+
+    @Override
+    protected void doExecute(Task task, Request request, ActionListener<Response> listener) {
+        final ClusterState clusterState = clusterService.state();
+        XPackPlugin.checkReadyForXPackCustomMetadata(clusterState);
+
+        useSecondaryAuthIfAvailable(securityContext, () -> {
+            ActionListener<TransformConfig> getTransformListener = ActionListener.wrap(unusedConfig -> {
+                PersistentTasksCustomMetadata.PersistentTask<?> transformTask = TransformTask.getTransformTask(
+                    request.getId(),
+                    clusterState
+                );
+
+                // to send a request to schedule now the transform at runtime, several requirements must be met:
+                // - transform must be running, meaning a task exists
+                // - transform is not failed (stopped transforms do not have a task)
+                if (transformTask != null
+                    && transformTask.isAssigned()
+                    && transformTask.getState() instanceof TransformState
+                    && ((TransformState) transformTask.getState()).getTaskState() != TransformTaskState.FAILED) {
+
+                    ActionListener<Response> taskScheduleNowListener = ActionListener.wrap(listener::onResponse, e -> {
+                        // benign: A transform might have been stopped meanwhile, this is not a problem
+                        if (e instanceof TransformTaskDisappearedDuringScheduleNowException) {
+                            logger.debug(
+                                () -> format("[%s] transform task disappeared during schedule_now, ignoring.", request.getId()),
+                                e
+                            );
+                            listener.onResponse(Response.TRUE);
+                            return;
+                        }
+                        if (e instanceof TransformTaskScheduleNowException) {
+                            logger.warn(() -> format("[%s] failed to schedule now the running transform.", request.getId()), e);
+                            listener.onResponse(Response.TRUE);
+                            return;
+                        }
+                        listener.onFailure(e);
+                    });
+                    request.setNodes(transformTask.getExecutorNode());
+                    super.doExecute(task, request, taskScheduleNowListener);
+                } else {
+                    listener.onResponse(Response.TRUE);
+                }
+            }, listener::onFailure);
+
+            // <1> Get the config to verify it exists and is valid
+            transformConfigManager.getTransformConfiguration(request.getId(), getTransformListener);
+        });
+    }
+
+    @Override
+    protected void taskOperation(Task actionTask, Request request, TransformTask transformTask, ActionListener<Response> listener) {
+        transformScheduler.scheduleNow(request.getId());
+        listener.onResponse(Response.TRUE);
+    }
+
+    @Override
+    protected Response newResponse(
+        Request request,
+        List<Response> tasks,
+        List<TaskOperationFailure> taskOperationFailures,
+        List<FailedNodeException> failedNodeExceptions
+    ) {
+        if (tasks.isEmpty()) {
+            if (taskOperationFailures.isEmpty() == false) {
+                throw new TransformTaskScheduleNowException(
+                    "Failed to schedule now the running transform due to task operation failure.",
+                    taskOperationFailures.get(0).getCause()
+                );
+            } else if (failedNodeExceptions.isEmpty() == false) {
+                FailedNodeException failedNodeException = failedNodeExceptions.get(0);
+                Throwable failedNodeExceptionCause = ExceptionsHelper.unwrapCause(failedNodeException.getCause());
+                if (failedNodeExceptionCause instanceof ActionNotFoundTransportException) {
+                    throw (ActionNotFoundTransportException) failedNodeExceptionCause;
+                }
+                throw new TransformTaskScheduleNowException(
+                    "Failed to schedule now the running transform due to failed node exception.",
+                    failedNodeExceptions.get(0)
+                );
+            } else {
+                throw new TransformTaskDisappearedDuringScheduleNowException(
+                    "Could not schedule now the running transform as it has been stopped."
+                );
+            }
+        }
+        return tasks.get(0);
+    }
+
+    private static class TransformTaskScheduleNowException extends ElasticsearchException {
+        TransformTaskScheduleNowException(String msg, Throwable cause, Object... args) {
+            super(msg, cause, args);
+        }
+    }
+
+    private static class TransformTaskDisappearedDuringScheduleNowException extends ElasticsearchException {
+        TransformTaskDisappearedDuringScheduleNowException(String msg) {
+            super(msg);
+        }
+    }
+}
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java
index 5c6099455c2b..cc9463de310e 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStartTransformAction.java
@@ -11,6 +11,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -21,7 +22,6 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
@@ -52,6 +52,7 @@
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 
+import static org.elasticsearch.action.ValidateActions.addValidationError;
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.xpack.core.transform.TransformMessages.CANNOT_START_FAILED_TRANSFORM;
 
@@ -203,7 +204,13 @@ protected void masterOperation(
 
         // <2> run transform validations
         ActionListener<TransformConfig> getTransformListener = ActionListener.wrap(config -> {
-            ValidationException validationException = config.validate(null);
+            ActionRequestValidationException validationException = config.validate(null);
+            if (request.from() != null && config.getSyncConfig() == null) {
+                validationException = addValidationError(
+                    "[from] parameter is currently not supported for batch (non-continuous) transforms",
+                    validationException
+                );
+            }
             if (validationException != null) {
                 listener.onFailure(
                     new ElasticsearchStatusException(
@@ -221,6 +228,7 @@ protected void masterOperation(
                 new TransformTaskParams(
                     config.getId(),
                     config.getVersion(),
+                    request.from(),
                     config.getFrequency(),
                     config.getSource().requiresRemoteCluster()
                 )
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStopTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStopTransformAction.java
index a20fb4dcbe08..5acb3e541039 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStopTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStopTransformAction.java
@@ -150,6 +150,7 @@ protected void doExecute(Task task, Request request, ActionListener<Response> li
             transformConfigManager.expandTransformIds(
                 request.getId(),
                 new PageParams(0, 10_000),
+                request.getTimeout(),
                 request.isAllowNoMatch(),
                 ActionListener.wrap(hitsAndIds -> {
                     validateTaskState(state, hitsAndIds.v2().v1(), request.isForce());
@@ -483,8 +484,8 @@ private ActionListener<Response> cancelTransformTasksWithNoAssignment(
     ) {
         final ActionListener<Response> doExecuteListener = ActionListener.wrap(response -> {
             GroupedActionListener<PersistentTask<?>> groupedListener = new GroupedActionListener<>(
-                ActionListener.wrap(r -> { finalListener.onResponse(response); }, finalListener::onFailure),
-                transformNodeAssignments.getWaitingForAssignment().size()
+                transformNodeAssignments.getWaitingForAssignment().size(),
+                ActionListener.wrap(r -> { finalListener.onResponse(response); }, finalListener::onFailure)
             );
 
             for (String unassignedTaskId : transformNodeAssignments.getWaitingForAssignment()) {
@@ -493,8 +494,8 @@ private ActionListener<Response> cancelTransformTasksWithNoAssignment(
 
         }, e -> {
             GroupedActionListener<PersistentTask<?>> groupedListener = new GroupedActionListener<>(
-                ActionListener.wrap(r -> { finalListener.onFailure(e); }, finalListener::onFailure),
-                transformNodeAssignments.getWaitingForAssignment().size()
+                transformNodeAssignments.getWaitingForAssignment().size(),
+                ActionListener.wrap(r -> { finalListener.onFailure(e); }, finalListener::onFailure)
             );
 
             for (String unassignedTaskId : transformNodeAssignments.getWaitingForAssignment()) {
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java
index d4db343a1c3c..e8a6a243a3f6 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java
@@ -211,7 +211,7 @@ private void recursiveExpandTransformIdsAndUpgrade(
         TimeValue timeout,
         ActionListener<Map<UpdateResult.Status, Long>> listener
     ) {
-        transformConfigManager.getAllOutdatedTransformIds(ActionListener.wrap(totalAndIds -> {
+        transformConfigManager.getAllOutdatedTransformIds(timeout, ActionListener.wrap(totalAndIds -> {
 
             // exit quickly if there is nothing to do
             if (totalAndIds.v2().isEmpty()) {
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/DefaultCheckpointProvider.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/DefaultCheckpointProvider.java
index 10321efbce32..0ba57764a7af 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/DefaultCheckpointProvider.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/DefaultCheckpointProvider.java
@@ -118,7 +118,7 @@ protected void getIndexCheckpoints(ActionListener<Map<String, long[]>> listener)
                     );
                 }, listener::onFailure);
 
-                groupedListener = new GroupedActionListener<>(mergeMapsListener, resolvedIndexes.numClusters());
+                groupedListener = new GroupedActionListener<>(resolvedIndexes.numClusters(), mergeMapsListener);
             }
 
             if (resolvedIndexes.getLocalIndices().isEmpty() == false) {
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java
index 7831405e5c8d..5d7e706447f9 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
@@ -518,6 +519,7 @@ public void getTransformConfigurationForUpdate(
     public void expandTransformIds(
         String transformIdsExpression,
         PageParams pageParams,
+        TimeValue timeout,
         boolean allowNoMatch,
         ActionListener<Tuple<Long, Tuple<List<String>, List<TransformConfig>>>> foundConfigsListener
     ) {
@@ -533,6 +535,7 @@ public void expandTransformIds(
             .setFrom(pageParams.getFrom())
             .setTrackTotalHits(true)
             .setSize(pageParams.getSize())
+            .setTimeout(timeout)
             .setQuery(queryBuilder)
             .request();
 
@@ -588,13 +591,18 @@ public void expandTransformIds(
     }
 
     @Override
-    public void getAllTransformIds(ActionListener<Set<String>> listener) {
-        expandAllTransformIds(false, MAX_RESULTS_WINDOW, ActionListener.wrap(r -> listener.onResponse(r.v2()), listener::onFailure));
+    public void getAllTransformIds(TimeValue timeout, ActionListener<Set<String>> listener) {
+        expandAllTransformIds(
+            false,
+            MAX_RESULTS_WINDOW,
+            timeout,
+            ActionListener.wrap(r -> listener.onResponse(r.v2()), listener::onFailure)
+        );
     }
 
     @Override
-    public void getAllOutdatedTransformIds(ActionListener<Tuple<Long, Set<String>>> listener) {
-        expandAllTransformIds(true, MAX_RESULTS_WINDOW, listener);
+    public void getAllOutdatedTransformIds(TimeValue timeout, ActionListener<Tuple<Long, Set<String>>> listener) {
+        expandAllTransformIds(true, MAX_RESULTS_WINDOW, timeout, listener);
     }
 
     @Override
@@ -781,7 +789,11 @@ public void getTransformStoredDoc(
     }
 
     @Override
-    public void getTransformStoredDocs(Collection<String> transformIds, ActionListener<List<TransformStoredDoc>> listener) {
+    public void getTransformStoredDocs(
+        Collection<String> transformIds,
+        TimeValue timeout,
+        ActionListener<List<TransformStoredDoc>> listener
+    ) {
         QueryBuilder builder = QueryBuilders.constantScoreQuery(
             QueryBuilders.boolQuery()
                 .filter(QueryBuilders.termsQuery(TransformField.ID.getPreferredName(), transformIds))
@@ -797,6 +809,7 @@ public void getTransformStoredDocs(Collection<String> transformIds, ActionListen
             .setQuery(builder)
             // the limit for getting stats and transforms is 1000, as long as we do not have 10 indices this works
             .setSize(Math.min(transformIds.size(), 10_000))
+            .setTimeout(timeout)
             .request();
 
         executeAsyncWithOrigin(
@@ -904,13 +917,19 @@ private QueryBuilder buildQueryFromTokenizedIds(String[] idTokens, String resour
      *
      * @param filterForOutdated if true, only returns outdated ids (after de-duplication)
      * @param maxResultWindow the max result window size (exposed for testing)
+     * @param timeout timeout applied to all the spawned requests
      * @param listener listener to call containing transform ids
      */
-    void expandAllTransformIds(boolean filterForOutdated, int maxResultWindow, ActionListener<Tuple<Long, Set<String>>> listener) {
+    void expandAllTransformIds(
+        boolean filterForOutdated,
+        int maxResultWindow,
+        TimeValue timeout,
+        ActionListener<Tuple<Long, Set<String>>> listener
+    ) {
         PageParams startPage = new PageParams(0, maxResultWindow);
 
         Set<String> collectedIds = new HashSet<>();
-        recursiveExpandAllTransformIds(collectedIds, 0, filterForOutdated, maxResultWindow, null, startPage, listener);
+        recursiveExpandAllTransformIds(collectedIds, 0, filterForOutdated, maxResultWindow, null, startPage, timeout, listener);
     }
 
     private void recursiveExpandAllTransformIds(
@@ -920,6 +939,7 @@ private void recursiveExpandAllTransformIds(
         int maxResultWindow,
         String lastId,
         PageParams page,
+        TimeValue timeout,
         ActionListener<Tuple<Long, Set<String>>> listener
     ) {
         SearchRequest request = client.prepareSearch(
@@ -930,6 +950,7 @@ private void recursiveExpandAllTransformIds(
             .addSort("_index", SortOrder.DESC)
             .setFrom(page.getFrom())
             .setSize(page.getSize())
+            .setTimeout(timeout)
             .setFetchSource(false)
             .addDocValueField(TransformField.ID.getPreferredName())
             .setQuery(
@@ -973,6 +994,7 @@ private void recursiveExpandAllTransformIds(
                         maxResultWindow,
                         idOfLastHit,
                         nextPage,
+                        timeout,
                         listener
                     );
                     return;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManager.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManager.java
index 71e748762190..5ed1a7254268 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManager.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManager.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.transform.persistence;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.xpack.core.action.util.PageParams;
 import org.elasticsearch.xpack.core.transform.TransformField;
@@ -146,11 +147,13 @@ void getTransformConfigurationForUpdate(
      *
      * @param transformIdsExpression The id expression. Can be _all, *, or comma delimited list of simple regex strings
      * @param pageParams             The paging params
+     * @param timeout                The timeout applied to all the spawned requests
      * @param foundConfigsListener   The listener on signal on success or failure
      */
     void expandTransformIds(
         String transformIdsExpression,
         PageParams pageParams,
+        TimeValue timeout,
         boolean allowNoMatch,
         ActionListener<Tuple<Long, Tuple<List<String>, List<TransformConfig>>>> foundConfigsListener
     );
@@ -158,16 +161,18 @@ void expandTransformIds(
     /**
      * Get all transform ids
      *
+     * @param timeout  The timeout applied to all the spawned requests
      * @param listener The listener to call with the collected ids
      */
-    void getAllTransformIds(ActionListener<Set<String>> listener);
+    void getAllTransformIds(TimeValue timeout, ActionListener<Set<String>> listener);
 
     /**
      * Get all transform ids that aren't using the latest index.
      *
+     * @param timeout  The timeout applied to all the spawned requests
      * @param listener The listener to call with total number of transforms and the list of transform ids.
      */
-    void getAllOutdatedTransformIds(ActionListener<Tuple<Long, Set<String>>> listener);
+    void getAllOutdatedTransformIds(TimeValue timeout, ActionListener<Tuple<Long, Set<String>>> listener);
 
     /**
      * This deletes documents corresponding to the transform id (e.g. checkpoints).
@@ -198,7 +203,7 @@ void getTransformStoredDoc(
         ActionListener<Tuple<TransformStoredDoc, SeqNoPrimaryTermAndIndex>> resultListener
     );
 
-    void getTransformStoredDocs(Collection<String> transformIds, ActionListener<List<TransformStoredDoc>> listener);
+    void getTransformStoredDocs(Collection<String> transformIds, TimeValue timeout, ActionListener<List<TransformStoredDoc>> listener);
 
     void refresh(ActionListener<Boolean> listener);
 }
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java
index 9b5964ac1779..db5f7f9799b4 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java
@@ -59,7 +59,7 @@ protected RestChannelConsumer doCatRequest(RestRequest restRequest, NodeClient c
         GetTransformAction.Request request = new GetTransformAction.Request(id);
         request.setAllowNoResources(restRequest.paramAsBoolean(ALLOW_NO_MATCH.getPreferredName(), true));
 
-        GetTransformStatsAction.Request statsRequest = new GetTransformStatsAction.Request(id);
+        GetTransformStatsAction.Request statsRequest = new GetTransformStatsAction.Request(id, null);
         statsRequest.setAllowNoMatch(restRequest.paramAsBoolean(ALLOW_NO_MATCH.getPreferredName(), true));
 
         if (restRequest.hasParam(PageParams.FROM.getPreferredName()) || restRequest.hasParam(PageParams.SIZE.getPreferredName())) {
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestGetTransformStatsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestGetTransformStatsAction.java
index f929f886199c..1e263a229922 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestGetTransformStatsAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestGetTransformStatsAction.java
@@ -7,9 +7,13 @@
 
 package org.elasticsearch.xpack.transform.rest.action;
 
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestCancellableNodeClient;
 import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.xpack.core.action.util.PageParams;
 import org.elasticsearch.xpack.core.transform.TransformField;
@@ -31,9 +35,10 @@ public List<Route> routes() {
     }
 
     @Override
-    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient nodeClient) {
         String id = restRequest.param(TransformField.ID.getPreferredName());
-        GetTransformStatsAction.Request request = new GetTransformStatsAction.Request(id);
+        TimeValue timeout = restRequest.paramAsTime(TransformField.TIMEOUT.getPreferredName(), AcknowledgedRequest.DEFAULT_ACK_TIMEOUT);
+        GetTransformStatsAction.Request request = new GetTransformStatsAction.Request(id, timeout);
         request.setAllowNoMatch(restRequest.paramAsBoolean(ALLOW_NO_MATCH.getPreferredName(), true));
         if (restRequest.hasParam(PageParams.FROM.getPreferredName()) || restRequest.hasParam(PageParams.SIZE.getPreferredName())) {
             request.setPageParams(
@@ -43,6 +48,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
                 )
             );
         }
+        Client client = new RestCancellableNodeClient(nodeClient, restRequest.getHttpChannel());
         return channel -> client.execute(GetTransformStatsAction.INSTANCE, request, new RestToXContentListener<>(channel));
     }
 
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestScheduleNowTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestScheduleNowTransformAction.java
new file mode 100644
index 000000000000..1ae65747de80
--- /dev/null
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestScheduleNowTransformAction.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.transform.rest.action;
+
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.transform.TransformField;
+import org.elasticsearch.xpack.core.transform.action.ScheduleNowTransformAction;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+
+public class RestScheduleNowTransformAction extends BaseRestHandler {
+
+    @Override
+    public List<Route> routes() {
+        return List.of(new Route(POST, TransformField.REST_BASE_PATH_TRANSFORMS_BY_ID + "_schedule_now"));
+    }
+
+    @Override
+    public String getName() {
+        return "transform_schedule_now_transform_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        String id = restRequest.param(TransformField.ID.getPreferredName());
+        TimeValue timeout = restRequest.paramAsTime(TransformField.TIMEOUT.getPreferredName(), AcknowledgedRequest.DEFAULT_ACK_TIMEOUT);
+
+        ScheduleNowTransformAction.Request request = ScheduleNowTransformAction.Request.fromXContent(id, timeout);
+
+        return channel -> client.execute(ScheduleNowTransformAction.INSTANCE, request, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestStartTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestStartTransformAction.java
index 29834a306ab0..36d8124356c7 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestStartTransformAction.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestStartTransformAction.java
@@ -7,16 +7,23 @@
 
 package org.elasticsearch.xpack.transform.rest.action;
 
+import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.common.time.DateMathParser;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xpack.core.transform.TransformField;
+import org.elasticsearch.xpack.core.transform.TransformMessages;
 import org.elasticsearch.xpack.core.transform.action.StartTransformAction;
 
+import java.time.Instant;
 import java.util.List;
+import java.util.function.LongSupplier;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
@@ -30,12 +37,24 @@ public List<Route> routes() {
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
         String id = restRequest.param(TransformField.ID.getPreferredName());
+        String fromAsString = restRequest.param(TransformField.FROM.getPreferredName());
+        Instant from = fromAsString != null ? parseDateOrThrow(fromAsString, TransformField.FROM, System::currentTimeMillis) : null;
         TimeValue timeout = restRequest.paramAsTime(TransformField.TIMEOUT.getPreferredName(), AcknowledgedRequest.DEFAULT_ACK_TIMEOUT);
 
-        StartTransformAction.Request request = new StartTransformAction.Request(id, timeout);
+        StartTransformAction.Request request = new StartTransformAction.Request(id, from, timeout);
         return channel -> client.execute(StartTransformAction.INSTANCE, request, new RestToXContentListener<>(channel));
     }
 
+    private static Instant parseDateOrThrow(String date, ParseField paramName, LongSupplier now) {
+        DateMathParser dateMathParser = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.toDateMathParser();
+        try {
+            return dateMathParser.parse(date, now);
+        } catch (Exception e) {
+            String msg = TransformMessages.getMessage(TransformMessages.FAILED_TO_PARSE_DATE, paramName.getPreferredName(), date);
+            throw new ElasticsearchParseException(msg, e);
+        }
+    }
+
     @Override
     public String getName() {
         return "transform_start_transform_action";
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerBuilder.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerBuilder.java
index 6fdf21ae1311..402f2489698e 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerBuilder.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexerBuilder.java
@@ -52,7 +52,10 @@ ClientTransformIndexer build(ThreadPool threadPool, TransformContext context) {
             initialStats,
             transformConfig,
             progress,
-            TransformCheckpoint.isNullOrEmpty(lastCheckpoint) ? TransformCheckpoint.EMPTY : lastCheckpoint,
+            // If there already exists at least one checkpoint, the "from" setting is effectively ignored.
+            TransformCheckpoint.isNullOrEmpty(lastCheckpoint)
+                ? (context.from() != null ? TransformCheckpoint.createEmpty(context.from().toEpochMilli()) : TransformCheckpoint.EMPTY)
+                : lastCheckpoint,
             TransformCheckpoint.isNullOrEmpty(nextCheckpoint) ? TransformCheckpoint.EMPTY : nextCheckpoint,
             seqNoPrimaryTermAndIndex,
             context,
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java
index ae350a443d82..4e20ec4dcb61 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java
@@ -47,10 +47,17 @@ public interface Listener {
     // Note: Each indexer run creates a new future checkpoint which becomes the current checkpoint only after the indexer run finished
     private final AtomicLong currentCheckpoint;
 
+    private final Instant from;
+
     public TransformContext(TransformTaskState taskState, String stateReason, long currentCheckpoint, Listener taskListener) {
+        this(taskState, stateReason, currentCheckpoint, null, taskListener);
+    }
+
+    public TransformContext(TransformTaskState taskState, String stateReason, long currentCheckpoint, Instant from, Listener taskListener) {
         this.taskState = new AtomicReference<>(taskState);
         this.stateReason = new AtomicReference<>(stateReason);
         this.currentCheckpoint = new AtomicLong(currentCheckpoint);
+        this.from = from;
         this.taskListener = taskListener;
         this.failureCount = new AtomicInteger(0);
     }
@@ -99,6 +106,10 @@ long getCheckpoint() {
         return currentCheckpoint.get();
     }
 
+    Instant from() {
+        return from;
+    }
+
     long incrementAndGetCheckpoint() {
         return currentCheckpoint.incrementAndGet();
     }
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java
index 23569920f303..a3faed34ebfc 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java
@@ -84,6 +84,9 @@ private enum RunState {
     private static final long RETENTION_OF_CHECKPOINTS_MS = 864000000L; // 10 days
     private static final long CHECKPOINT_CLEANUP_INTERVAL = 100L; // every 100 checkpoints
 
+    // constant for triggering state persistence, hardcoded for now
+    public static final long DEFAULT_TRIGGER_SAVE_STATE_INTERVAL_MS = 60_000; // 60s
+
     protected final TransformConfigManager transformsConfigManager;
     private final CheckpointProvider checkpointProvider;
     protected final TransformFailureHandler failureHandler;
@@ -119,6 +122,7 @@ private enum RunState {
     private volatile RunState runState;
 
     private volatile long lastCheckpointCleanup = 0L;
+    private volatile long lastSaveStateMilliseconds;
 
     protected volatile boolean indexerThreadShuttingDown = false;
     protected volatile boolean saveStateRequestedDuringIndexerThreadShutdown = false;
@@ -154,6 +158,7 @@ public TransformIndexer(
         if (transformConfig.getSettings() != null && transformConfig.getSettings().getDocsPerSecond() != null) {
             docsPerSecond = transformConfig.getSettings().getDocsPerSecond();
         }
+        this.lastSaveStateMilliseconds = TimeUnit.NANOSECONDS.toMillis(getTimeNanos());
     }
 
     abstract void doGetInitialProgress(SearchRequest request, ActionListener<SearchResponse> responseListener);
@@ -181,7 +186,11 @@ protected float getMaxDocsPerSecond() {
     @Override
     protected boolean triggerSaveState() {
         // trigger in case of listeners waiting for state being saved
-        return saveStateListeners.get() != null || super.triggerSaveState();
+        if (saveStateListeners.get() != null) {
+            return true;
+        }
+
+        return TimeUnit.NANOSECONDS.toMillis(getTimeNanos()) > lastSaveStateMilliseconds + DEFAULT_TRIGGER_SAVE_STATE_INTERVAL_MS;
     }
 
     public TransformConfig getConfig() {
@@ -357,7 +366,7 @@ protected void onStart(long now, ActionListener<Boolean> listener) {
         // If we are not on the initial batch checkpoint and its the first pass of whatever continuous checkpoint we are on,
         // we should verify if there are local changes based on the sync config. If not, do not proceed further and exit.
         if (context.getCheckpoint() > 0 && initialRun()) {
-            sourceHasChanged(ActionListener.wrap(hasChanged -> {
+            checkpointProvider.sourceHasChanged(getLastCheckpoint(), ActionListener.wrap(hasChanged -> {
                 context.setLastSearchTime(instantOfTrigger);
                 hasSourceChanged = hasChanged;
                 if (hasChanged) {
@@ -738,6 +747,7 @@ protected void doSaveState(IndexerState indexerState, TransformIndexerPosition p
                 String msg = LoggerMessageFormat.format("[{}] failed notifying saveState listeners, ignoring.", getJobId());
                 logger.warn(msg, onResponseException);
             } finally {
+                lastSaveStateMilliseconds = TimeUnit.NANOSECONDS.toMillis(getTimeNanos());
                 next.run();
             }
         }, e -> {
@@ -959,20 +969,6 @@ private void cleanupOldCheckpoints(ActionListener<Void> listener) {
         }
     }
 
-    private void sourceHasChanged(ActionListener<Boolean> hasChangedListener) {
-        checkpointProvider.sourceHasChanged(getLastCheckpoint(), ActionListener.wrap(hasChanged -> {
-            logger.trace("[{}] change detected [{}].", getJobId(), hasChanged);
-            hasChangedListener.onResponse(hasChanged);
-        }, e -> {
-            logger.warn(() -> "[" + getJobId() + "] failed to detect changes for transform. Skipping update till next check.", e);
-            auditor.warning(
-                getJobId(),
-                "Failed to detect changes for transform, skipping update till next check. Exception: " + e.getMessage()
-            );
-            hasChangedListener.onResponse(false);
-        }));
-    }
-
     private IterationResult<TransformIndexerPosition> processBuckets(final SearchResponse searchResponse) {
         Tuple<Stream<IndexRequest>, Map<String, Object>> indexRequestStreamAndCursor = function.processSearchResponse(
             searchResponse,
@@ -1111,7 +1107,7 @@ private SearchRequest buildQueryToUpdateDestinationIndex() {
                 .filter(config.getSyncConfig().getRangeQuery(nextCheckpoint));
 
             // Only apply extra filter if it is the subsequent run of the continuous transform
-            if (nextCheckpoint.getCheckpoint() > 1 && changeCollector != null) {
+            if (changeCollector != null) {
                 QueryBuilder filter = changeCollector.buildFilterQuery(lastCheckpoint, nextCheckpoint);
                 if (filter != null) {
                     filteredQuery.filter(filter);
@@ -1168,6 +1164,10 @@ protected boolean shouldAuditOnFinish(long completedCheckpoint) {
     }
 
     private RunState determineRunStateAtStart() {
+        if (context.from() != null) {
+            return RunState.IDENTIFY_CHANGES;
+        }
+
         // either 1st run or not a continuous transform
         if (nextCheckpoint.getCheckpoint() == 1 || isContinuous() == false) {
             return RunState.APPLY_RESULTS;
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformTask.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformTask.java
index 1964a42aac2e..4ade3a31608a 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformTask.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformTask.java
@@ -114,7 +114,7 @@ public TransformTask(
         this.initialIndexerState = initialState;
         this.initialPosition = initialPosition;
 
-        this.context = new TransformContext(initialTaskState, initialReason, initialCheckpoint, this);
+        this.context = new TransformContext(initialTaskState, initialReason, initialCheckpoint, transform.from(), this);
     }
 
     public ParentTaskAssigningClient getParentTaskClient() {
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java
index 9d00df70d094..5c479629a199 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java
@@ -384,6 +384,10 @@ public QueryBuilder filterByChanges(long lastCheckpointTimestamp, long nextcheck
             if (missingBucket) {
                 return null;
             }
+            // filterByChanges has been called before collectChangesFromAggregations
+            if (lowerBound == 0 && upperBound == 0) {
+                return null;
+            }
             return new RangeQueryBuilder(sourceFieldName).gte(lowerBound).lte(upperBound).format("epoch_millis");
         }
 
@@ -480,11 +484,15 @@ public boolean collectChangesFromCompositeBuckets(Collection<? extends Bucket> b
 
         @Override
         public QueryBuilder filterByChanges(long lastCheckpointTimestamp, long nextcheckpointTimestamp) {
+
             if (missingBucket) {
                 return null;
             }
-
-            // (upperBound - lowerBound) >= interval, so never 0
+            // filterByChanges has been called before collectChangesFromAggregations
+            if (lowerBound == 0 && upperBound == 0) {
+                return null;
+            }
+            // (upperBound - lowerBound) >= interval, so never 0.
             if ((maxUpperBound - minLowerBound) / (upperBound - lowerBound) < MIN_CUT_OFF) {
                 return null;
             }
diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/scheduling/TransformScheduler.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/scheduling/TransformScheduler.java
index 59b77af08aa7..aa7a5f9873dc 100644
--- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/scheduling/TransformScheduler.java
+++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/scheduling/TransformScheduler.java
@@ -115,9 +115,12 @@ void processScheduledTasks() {
         }
         if (isTraceEnabled) {
             Instant processingFinished = clock.instant();
-            logger.trace(
-                format("Processing scheduled tasks finished, took %dms", Duration.between(processingStarted, processingFinished).toMillis())
-            );
+            long tookMs = Duration.between(processingStarted, processingFinished).toMillis();
+            if (taskWasProcessed) {
+                logger.trace(format("Processing one scheduled task finished, took %dms", tookMs));
+            } else {
+                logger.trace(format("Looking for scheduled tasks to process finished, took %dms", tookMs));
+            }
         }
         if (taskWasProcessed == false) {
             return;
@@ -228,6 +231,30 @@ public void handleTransformFailureCountChanged(String transformId, int failureCo
         );
     }
 
+    /**
+     * Updates the transform task's next_scheduled_time so that it is set to now.
+     * Doing so may result in the task being processed earlier that it would normally (i.e.: according to its frequency) be.
+     *
+     * @param transformId id of the transform to schedule now
+     */
+    public void scheduleNow(String transformId) {
+        logger.trace(() -> format("[%s] schedule_now transform", transformId));
+        long currentTimeMillis = clock.millis();
+        // Update the task's next_scheduled_time
+        scheduledTasks.update(
+            transformId,
+            task -> new TransformScheduledTask(
+                task.getTransformId(),
+                task.getFrequency(),
+                task.getLastTriggeredTimeMillis(),
+                task.getFailureCount(),
+                currentTimeMillis,  // we schedule this task at current clock time so that it is processed ASAP
+                task.getListener()
+            )
+        );
+        processScheduledTasks();
+    }
+
     /**
      * De-registers the given transform by removing it from the queue.
      *
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformMetadataTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformMetadataTests.java
index e2bc0c74dcdb..39ea10ef570f 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformMetadataTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformMetadataTests.java
@@ -8,11 +8,11 @@
 package org.elasticsearch.xpack.transform;
 
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.test.AbstractXContentSerializingTestCase;
+import org.elasticsearch.test.AbstractChunkedSerializingTestCase;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.transform.TransformMetadata;
 
-public class TransformMetadataTests extends AbstractXContentSerializingTestCase<TransformMetadata> {
+public class TransformMetadataTests extends AbstractChunkedSerializingTestCase<TransformMetadata> {
 
     @Override
     protected TransformMetadata createTestInstance() {
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransformPrivilegeCheckerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransformPrivilegeCheckerTests.java
index 1386a1aa1ea5..6892e68015f1 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransformPrivilegeCheckerTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransformPrivilegeCheckerTests.java
@@ -45,6 +45,7 @@
 import static org.hamcrest.Matchers.emptyArray;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
 
 public class TransformPrivilegeCheckerTests extends ESTestCase {
 
@@ -61,6 +62,7 @@ public class TransformPrivilegeCheckerTests extends ESTestCase {
         .addPrivilege("view_index_metadata", true)
         .addPrivilege("read", false)
         .build();
+    private static final String REMOTE_SOURCE_INDEX_NAME = "some-remote-cluster:some-remote-source-index";
     private static final String DEST_INDEX_NAME = "some-dest-index";
     private static final ResourcePrivileges DEST_INDEX_NAME_PRIVILEGES = ResourcePrivileges.builder(DEST_INDEX_NAME)
         .addPrivilege("index", true)
@@ -103,20 +105,23 @@ public void tearDownClient() {
     }
 
     public void testCheckPrivileges_NoCheckDestIndexPrivileges() {
+        TransformConfig config = new TransformConfig.Builder(TRANSFORM_CONFIG).setSource(
+            new SourceConfig(SOURCE_INDEX_NAME, REMOTE_SOURCE_INDEX_NAME)
+        ).build();
         TransformPrivilegeChecker.checkPrivileges(
             OPERATION_NAME,
             securityContext,
             indexNameExpressionResolver,
             ClusterState.EMPTY_STATE,
             client,
-            TRANSFORM_CONFIG,
+            config,
             false,
             ActionListener.wrap(aVoid -> {
                 HasPrivilegesRequest request = client.lastHasPrivilegesRequest;
                 assertThat(request.username(), is(equalTo(USER_NAME)));
                 assertThat(request.applicationPrivileges(), is(emptyArray()));
                 assertThat(request.clusterPrivileges(), is(emptyArray()));
-                assertThat(request.indexPrivileges(), is(arrayWithSize(1)));
+                assertThat(request.indexPrivileges(), is(arrayWithSize(1)));  // remote index is filtered out
                 RoleDescriptor.IndicesPrivileges sourceIndicesPrivileges = request.indexPrivileges()[0];
                 assertThat(sourceIndicesPrivileges.getIndices(), is(arrayContaining(SOURCE_INDEX_NAME)));
                 assertThat(sourceIndicesPrivileges.getPrivileges(), is(arrayContaining("read", "view_index_metadata")));
@@ -124,6 +129,24 @@ public void testCheckPrivileges_NoCheckDestIndexPrivileges() {
         );
     }
 
+    public void testCheckPrivileges_NoLocalIndices_NoCheckDestIndexPrivileges() {
+        TransformConfig config = new TransformConfig.Builder(TRANSFORM_CONFIG).setSource(new SourceConfig(REMOTE_SOURCE_INDEX_NAME))
+            .build();
+        TransformPrivilegeChecker.checkPrivileges(
+            OPERATION_NAME,
+            securityContext,
+            indexNameExpressionResolver,
+            ClusterState.EMPTY_STATE,
+            client,
+            config,
+            false,
+            ActionListener.wrap(aVoid -> {
+                // _has_privileges API is not called for the remote index
+                assertThat(client.lastHasPrivilegesRequest, is(nullValue()));
+            }, e -> fail(e.getMessage()))
+        );
+    }
+
     public void testCheckPrivileges_CheckDestIndexPrivileges_DestIndexDoesNotExist() {
         TransformPrivilegeChecker.checkPrivileges(
             OPERATION_NAME,
@@ -180,6 +203,36 @@ public void testCheckPrivileges_CheckDestIndexPrivileges_DestIndexExists() {
         );
     }
 
+    public void testCheckPrivileges_NoLocalIndices_CheckDestIndexPrivileges_DestIndexExists() {
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metadata(
+                Metadata.builder()
+                    .put(IndexMetadata.builder(DEST_INDEX_NAME).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
+            )
+            .build();
+        TransformConfig config = new TransformConfig.Builder(TRANSFORM_CONFIG).setSource(new SourceConfig(REMOTE_SOURCE_INDEX_NAME))
+            .build();
+        TransformPrivilegeChecker.checkPrivileges(
+            OPERATION_NAME,
+            securityContext,
+            indexNameExpressionResolver,
+            clusterState,
+            client,
+            config,
+            true,
+            ActionListener.wrap(aVoid -> {
+                HasPrivilegesRequest request = client.lastHasPrivilegesRequest;
+                assertThat(request.username(), is(equalTo(USER_NAME)));
+                assertThat(request.applicationPrivileges(), is(emptyArray()));
+                assertThat(request.clusterPrivileges(), is(emptyArray()));
+                assertThat(request.indexPrivileges(), is(arrayWithSize(1)));
+                RoleDescriptor.IndicesPrivileges destIndicesPrivileges = request.indexPrivileges()[0];
+                assertThat(destIndicesPrivileges.getIndices(), is(arrayContaining(DEST_INDEX_NAME)));
+                assertThat(destIndicesPrivileges.getPrivileges(), is(arrayContaining("read", "index")));
+            }, e -> fail(e.getMessage()))
+        );
+    }
+
     public void testCheckPrivileges_CheckDestIndexPrivileges_DestIndexExistsWithRetentionPolicy() {
         ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
             .metadata(
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/TimeBasedCheckpointProviderTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/TimeBasedCheckpointProviderTests.java
index 902a69b88507..7521fc63cbf1 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/TimeBasedCheckpointProviderTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/TimeBasedCheckpointProviderTests.java
@@ -298,6 +298,7 @@ private static TransformConfig newTransformConfigWithDateHistogram(
             null,
             false,
             new DateHistogramGroupSource.FixedInterval(new DateHistogramInterval(dateHistogramInterval.getStringRep())),
+            null,
             null
         );
         Supplier<SingleGroupSource> singleGroupSourceSupplier = new Supplier<>() {
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/TransformsCheckpointServiceTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/TransformsCheckpointServiceTests.java
index 3e89ae21f0cb..bb4f0450f051 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/TransformsCheckpointServiceTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/TransformsCheckpointServiceTests.java
@@ -207,7 +207,8 @@ private static ShardStats[] createRandomShardStats(
                         shardId,
                         primary,
                         primary ? RecoverySource.EmptyStoreRecoverySource.INSTANCE : PeerRecoverySource.INSTANCE,
-                        new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)
+                        new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null),
+                        ShardRouting.Role.DEFAULT
                     );
                     shardRouting = shardRouting.initialize("node-0", null, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
                     shardRouting = shardRouting.moveToStarted(ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/InMemoryTransformConfigManager.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/InMemoryTransformConfigManager.java
index 104ad7c466de..1b9dedf22a80 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/InMemoryTransformConfigManager.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/InMemoryTransformConfigManager.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.xpack.core.action.util.PageParams;
 import org.elasticsearch.xpack.core.transform.TransformMessages;
@@ -243,6 +244,7 @@ public void getTransformConfigurationForUpdate(
     public void expandTransformIds(
         String transformIdsExpression,
         PageParams pageParams,
+        TimeValue timeout,
         boolean allowNoMatch,
         ActionListener<Tuple<Long, Tuple<List<String>, List<TransformConfig>>>> foundConfigsListener
     ) {
@@ -359,7 +361,11 @@ public void getTransformStoredDoc(
     }
 
     @Override
-    public void getTransformStoredDocs(Collection<String> transformIds, ActionListener<List<TransformStoredDoc>> listener) {
+    public void getTransformStoredDocs(
+        Collection<String> transformIds,
+        TimeValue timeout,
+        ActionListener<List<TransformStoredDoc>> listener
+    ) {
         List<TransformStoredDoc> docs = new ArrayList<>();
         for (String transformId : transformIds) {
             TransformStoredDoc storedDoc = transformStoredDocs.get(transformId);
@@ -381,17 +387,16 @@ public void refresh(ActionListener<Boolean> listener) {
     }
 
     @Override
-    public void getAllTransformIds(ActionListener<Set<String>> listener) {
+    public void getAllTransformIds(TimeValue timeout, ActionListener<Set<String>> listener) {
         Set<String> allIds = new HashSet<>(configs.keySet());
         allIds.addAll(oldConfigs.keySet());
         listener.onResponse(allIds);
     }
 
     @Override
-    public void getAllOutdatedTransformIds(ActionListener<Tuple<Long, Set<String>>> listener) {
+    public void getAllOutdatedTransformIds(TimeValue timeout, ActionListener<Tuple<Long, Set<String>>> listener) {
         Set<String> outdatedIds = new HashSet<>(oldConfigs.keySet());
         outdatedIds.removeAll(configs.keySet());
         listener.onResponse(new Tuple<>(Long.valueOf(configs.size() + outdatedIds.size()), outdatedIds));
     }
-
 }
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/rest/action/RestScheduleNowTransformActionTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/rest/action/RestScheduleNowTransformActionTests.java
new file mode 100644
index 000000000000..9aa792e0bf2e
--- /dev/null
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/rest/action/RestScheduleNowTransformActionTests.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.transform.rest.action;
+
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xpack.core.transform.action.ScheduleNowTransformAction;
+import org.junit.Before;
+
+import java.util.Map;
+
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+public class RestScheduleNowTransformActionTests extends ESTestCase {
+
+    private static final String ID = "id";
+    private static final String TIMEOUT = "timeout";
+
+    private RestChannel channel;
+    private NodeClient client;
+
+    @Before
+    public void initializeMocks() {
+        channel = mock(RestChannel.class);
+        client = mock(NodeClient.class);
+    }
+
+    public void testHandleRequest() throws Exception {
+        RestScheduleNowTransformAction handler = new RestScheduleNowTransformAction();
+        FakeRestRequest request = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).withParams(Map.of(ID, "my-id")).build();
+
+        handler.handleRequest(request, channel, client);
+
+        ScheduleNowTransformAction.Request expectedActionRequest = new ScheduleNowTransformAction.Request(
+            "my-id",
+            TimeValue.timeValueSeconds(30)
+        );
+        verify(client).execute(eq(ScheduleNowTransformAction.INSTANCE), eq(expectedActionRequest), any());
+        verifyNoMoreInteractions(client);
+    }
+
+    public void testHandleRequestWithTimeout() throws Exception {
+        RestScheduleNowTransformAction handler = new RestScheduleNowTransformAction();
+        FakeRestRequest request = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).withParams(Map.of(ID, "my-id", TIMEOUT, "45s"))
+            .build();
+
+        handler.handleRequest(request, channel, client);
+
+        ScheduleNowTransformAction.Request expectedActionRequest = new ScheduleNowTransformAction.Request(
+            "my-id",
+            TimeValue.timeValueSeconds(45)
+        );
+        verify(client).execute(eq(ScheduleNowTransformAction.INSTANCE), eq(expectedActionRequest), any());
+        verifyNoMoreInteractions(client);
+    }
+}
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/rest/action/RestStartTransformActionTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/rest/action/RestStartTransformActionTests.java
new file mode 100644
index 000000000000..1f36efc513e1
--- /dev/null
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/rest/action/RestStartTransformActionTests.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.transform.rest.action;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
+
+public class RestStartTransformActionTests extends ESTestCase {
+
+    private static final String ID = "id";
+    private static final String FROM = "from";
+
+    public void testFromValid() throws Exception {
+        testFromValid(null);
+        testFromValid("12345678");
+        testFromValid("2022-10-25");
+        testFromValid("now-1d");
+    }
+
+    private void testFromValid(String from) {
+        RestStartTransformAction handler = new RestStartTransformAction();
+        FakeRestRequest.Builder requestBuilder = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY);
+        if (from == null) {
+            requestBuilder.withParams(Map.of(ID, "my-id"));
+        } else {
+            requestBuilder.withParams(Map.of(ID, "my-id", FROM, from));
+        }
+        FakeRestRequest request = requestBuilder.build();
+        handler.prepareRequest(request, mock(NodeClient.class));
+    }
+
+    public void testFromInvalid() {
+        testFromInvalid("");
+        testFromInvalid("not-a-valid-timestamp");
+        testFromInvalid("2023-17-42");
+    }
+
+    private void testFromInvalid(String from) {
+        final RestStartTransformAction handler = new RestStartTransformAction();
+        final FakeRestRequest request = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).withParams(Map.of(ID, "my-id", FROM, from))
+            .build();
+        ElasticsearchParseException e = expectThrows(
+            ElasticsearchParseException.class,
+            () -> handler.prepareRequest(request, mock(NodeClient.class))
+        );
+        assertThat(e.getMessage(), equalTo("Failed to parse date for [from]"));
+    }
+}
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformContextTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformContextTests.java
index f1db6f21d023..b2ca0769ea97 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformContextTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformContextTests.java
@@ -12,6 +12,8 @@
 import org.junit.After;
 import org.junit.Before;
 
+import java.time.Instant;
+
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
@@ -88,4 +90,10 @@ public void testStateReason() {
 
         verify(listener).failureCountChanged();
     }
+
+    public void testFrom() {
+        Instant from = Instant.ofEpochMilli(randomLongBetween(0, 1_000_000_000_000L));
+        TransformContext context = new TransformContext(TransformTaskState.STARTED, null, 0, from, listener);
+        assertThat(context.from(), is(equalTo(from)));
+    }
 }
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java
index bedde8047a28..7759f153ee75 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.client.NoOpClient;
+import org.elasticsearch.test.junit.annotations.TestIssueLogging;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.indexing.IndexerState;
@@ -254,6 +255,97 @@ void validate(ActionListener<Void> listener) {
         }
     }
 
+    class MockedTransformIndexerForStatePersistenceTesting extends TransformIndexer {
+
+        private long timeNanos = 0;
+
+        MockedTransformIndexerForStatePersistenceTesting(
+            ThreadPool threadPool,
+            TransformServices transformServices,
+            CheckpointProvider checkpointProvider,
+            TransformConfig transformConfig,
+            AtomicReference<IndexerState> initialState,
+            TransformIndexerPosition initialPosition,
+            TransformIndexerStats jobStats,
+            TransformContext context
+        ) {
+            super(
+                threadPool,
+                transformServices,
+                checkpointProvider,
+                transformConfig,
+                initialState,
+                initialPosition,
+                jobStats,
+                /* TransformProgress */ null,
+                TransformCheckpoint.EMPTY,
+                TransformCheckpoint.EMPTY,
+                context
+            );
+        }
+
+        public void setTimeMillis(long millis) {
+            this.timeNanos = TimeUnit.MILLISECONDS.toNanos(millis);
+        }
+
+        @Override
+        protected long getTimeNanos() {
+            return timeNanos;
+        }
+
+        @Override
+        protected void doNextSearch(long waitTimeInNanos, ActionListener<SearchResponse> nextPhase) {
+            threadPool.generic().execute(() -> nextPhase.onResponse(ONE_HIT_SEARCH_RESPONSE));
+        }
+
+        @Override
+        protected void doNextBulk(BulkRequest request, ActionListener<BulkResponse> nextPhase) {
+            threadPool.generic().execute(() -> nextPhase.onResponse(new BulkResponse(new BulkItemResponse[0], 100)));
+        }
+
+        @Override
+        void doGetInitialProgress(SearchRequest request, ActionListener<SearchResponse> responseListener) {
+            responseListener.onResponse(ONE_HIT_SEARCH_RESPONSE);
+        }
+
+        @Override
+        void doGetFieldMappings(ActionListener<Map<String, String>> fieldMappingsListener) {
+            fieldMappingsListener.onResponse(Collections.emptyMap());
+        }
+
+        @Override
+        void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener<BulkByScrollResponse> responseListener) {
+            responseListener.onResponse(
+                new BulkByScrollResponse(
+                    TimeValue.ZERO,
+                    new BulkByScrollTask.Status(Collections.emptyList(), null),
+                    Collections.emptyList(),
+                    Collections.emptyList(),
+                    false
+                )
+            );
+        }
+
+        @Override
+        void refreshDestinationIndex(ActionListener<RefreshResponse> responseListener) {
+            responseListener.onResponse(new RefreshResponse(1, 1, 0, Collections.emptyList()));
+        }
+
+        @Override
+        void persistState(TransformState state, ActionListener<Void> listener) {
+            listener.onResponse(null);
+        }
+
+        @Override
+        void validate(ActionListener<Void> listener) {
+            listener.onResponse(null);
+        }
+
+        public void initialize() {
+            this.initializeFunction();
+        }
+    }
+
     @Before
     public void setUpMocks() {
         auditor = MockTransformAuditor.createMockAuditor();
@@ -268,6 +360,87 @@ public void tearDownClient() {
         ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS);
     }
 
+    public void testTriggerStatePersistence() {
+        TransformConfig config = new TransformConfig(
+            randomAlphaOfLength(10),
+            randomSourceConfig(),
+            randomDestConfig(),
+            null,
+            new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)),
+            null,
+            randomPivotConfig(),
+            null,
+            randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000),
+            null,
+            null,
+            null,
+            null,
+            null
+        );
+        AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.INDEXING);
+
+        TransformContext context = new TransformContext(TransformTaskState.STARTED, "", 0, mock(TransformContext.Listener.class));
+        final MockedTransformIndexerForStatePersistenceTesting indexer = createMockIndexerForStatePersistenceTesting(
+            config,
+            state,
+            null,
+            threadPool,
+            auditor,
+            null,
+            new TransformIndexerStats(),
+            context
+        );
+
+        assertFalse(indexer.triggerSaveState());
+        // simple: advancing the time should trigger state persistence
+        indexer.setTimeMillis(80_000);
+        assertTrue(indexer.triggerSaveState());
+        // still true as state persistence has not been executed
+        assertTrue(indexer.triggerSaveState());
+        indexer.doSaveState(IndexerState.INDEXING, null, () -> {});
+        // after state persistence, the trigger should return false
+        assertFalse(indexer.triggerSaveState());
+        // advance time twice, but don't persist
+        indexer.setTimeMillis(81_000);
+        assertFalse(indexer.triggerSaveState());
+        indexer.setTimeMillis(140_000);
+        assertFalse(indexer.triggerSaveState());
+        // now trigger should return false as last persistence was 80_000
+        indexer.setTimeMillis(140_001);
+        assertTrue(indexer.triggerSaveState());
+        // persist and check trigger
+        indexer.doSaveState(IndexerState.INDEXING, null, () -> {});
+        assertFalse(indexer.triggerSaveState());
+        // check trigger but persist later
+        indexer.setTimeMillis(200_001);
+        assertFalse(indexer.triggerSaveState());
+        indexer.setTimeMillis(240_000);
+        indexer.doSaveState(IndexerState.INDEXING, null, () -> {});
+        assertFalse(indexer.triggerSaveState());
+        // last persistence should be 240_000, so don't trigger
+        indexer.setTimeMillis(270_000);
+        assertFalse(indexer.triggerSaveState());
+        indexer.setTimeMillis(300_001);
+        assertTrue(indexer.triggerSaveState());
+        indexer.doSaveState(IndexerState.INDEXING, null, () -> {});
+        assertFalse(indexer.triggerSaveState());
+        // advance again, it shouldn't trigger
+        indexer.setTimeMillis(310_000);
+        assertFalse(indexer.triggerSaveState());
+
+        // set stop at checkpoint, which must trigger state persistence
+        setStopAtCheckpoint(indexer, true, ActionListener.noop());
+        assertTrue(indexer.triggerSaveState());
+        indexer.setTimeMillis(311_000);
+        // after state persistence, trigger should return false
+        indexer.doSaveState(IndexerState.INDEXING, null, () -> {});
+        indexer.setTimeMillis(310_200);
+        assertFalse(indexer.triggerSaveState());
+
+        // after time has passed, trigger should work again
+        indexer.setTimeMillis(371_001);
+    }
+
     public void testStopAtCheckpoint() throws Exception {
         TransformConfig config = new TransformConfig(
             randomAlphaOfLength(10),
@@ -501,6 +674,10 @@ public void testStopAtCheckpoint() throws Exception {
         }
     }
 
+    @TestIssueLogging(
+        value = "org.elasticsearch.xpack.transform.transforms:DEBUG",
+        issueUrl = "https://github.com/elastic/elasticsearch/issues/92069"
+    )
     public void testStopAtCheckpointForThrottledTransform() throws Exception {
         TransformConfig config = new TransformConfig(
             randomAlphaOfLength(10),
@@ -648,4 +825,38 @@ private MockedTransformIndexer createMockIndexer(
         indexer.initialize();
         return indexer;
     }
+
+    private MockedTransformIndexerForStatePersistenceTesting createMockIndexerForStatePersistenceTesting(
+        TransformConfig config,
+        AtomicReference<IndexerState> state,
+        Consumer<String> failureConsumer,
+        ThreadPool threadPool,
+        TransformAuditor transformAuditor,
+        TransformIndexerPosition initialPosition,
+        TransformIndexerStats jobStats,
+        TransformContext context
+    ) {
+        CheckpointProvider checkpointProvider = new MockTimebasedCheckpointProvider(config);
+        transformConfigManager.putTransformConfiguration(config, ActionListener.noop());
+        TransformServices transformServices = new TransformServices(
+            transformConfigManager,
+            mock(TransformCheckpointService.class),
+            transformAuditor,
+            new TransformScheduler(Clock.systemUTC(), threadPool, Settings.EMPTY)
+        );
+
+        MockedTransformIndexerForStatePersistenceTesting indexer = new MockedTransformIndexerForStatePersistenceTesting(
+            threadPool,
+            transformServices,
+            checkpointProvider,
+            config,
+            state,
+            initialPosition,
+            jobStats,
+            context
+        );
+
+        indexer.initialize();
+        return indexer;
+    }
 }
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformNodesTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformNodesTests.java
index 7b8375fd02ca..07f37e8effba 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformNodesTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformNodesTests.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.transform.transforms;
 
 import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
@@ -69,7 +70,7 @@ public String getWriteableName() {
             }
 
             @Override
-            public Version getMinimalSupportedVersion() {
+            public TransportVersion getMinimalSupportedVersion() {
                 return null;
             }
 
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java
index eb5c74a02bef..6dafe09c34c4 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java
@@ -201,7 +201,8 @@ public void testVerifyIndicesPrimaryShardsAreActive() {
                 shardId,
                 true,
                 RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")
+                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""),
+                ShardRouting.Role.DEFAULT
             );
             shardRouting = shardRouting.initialize("node_id", null, 0L);
             routingTable.add(
@@ -239,7 +240,8 @@ private void addIndices(Metadata.Builder metadata, RoutingTable.Builder routingT
                 shardId,
                 true,
                 RecoverySource.EmptyStoreRecoverySource.INSTANCE,
-                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")
+                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""),
+                ShardRouting.Role.DEFAULT
             );
             shardRouting = shardRouting.initialize("node_id", null, 0L);
             shardRouting = shardRouting.moveToStarted(ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java
index ccb6369d7515..7dc441141793 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java
@@ -9,6 +9,7 @@
 
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.Aggregation;
@@ -224,7 +225,7 @@ protected NamedXContentRegistry xContentRegistry() {
     public void testExtractCompositeAggregationResults() throws IOException {
         String targetField = randomAlphaOfLengthBetween(5, 10);
 
-        GroupConfig groupBy = parseGroupConfig(formatted("""
+        GroupConfig groupBy = parseGroupConfig(Strings.format("""
             { "%s" : {"terms" : {   "field" : "doesn't_matter_for_this_test"} } }
             """, targetField));
 
@@ -254,7 +255,7 @@ public void testExtractCompositeAggregationResultsMultipleGroups() throws IOExce
         String targetField = randomAlphaOfLengthBetween(5, 10);
         String targetField2 = randomAlphaOfLengthBetween(5, 10) + "_2";
 
-        GroupConfig groupBy = parseGroupConfig(formatted("""
+        GroupConfig groupBy = parseGroupConfig(Strings.format("""
             {
               "%s": {
                 "terms": {
@@ -295,7 +296,7 @@ public void testExtractCompositeAggregationResultsMultipleGroups() throws IOExce
     public void testExtractCompositeAggregationResultsMultiAggregations() throws IOException {
         String targetField = randomAlphaOfLengthBetween(5, 10);
 
-        GroupConfig groupBy = parseGroupConfig(formatted("""
+        GroupConfig groupBy = parseGroupConfig(Strings.format("""
             {
               "%s": {
                 "terms": {
@@ -361,7 +362,7 @@ public void testExtractCompositeAggregationResultsMultiAggregationsAndTypes() th
         String targetField = randomAlphaOfLengthBetween(5, 10);
         String targetField2 = randomAlphaOfLengthBetween(5, 10) + "_2";
 
-        GroupConfig groupBy = parseGroupConfig(formatted("""
+        GroupConfig groupBy = parseGroupConfig(Strings.format("""
             {
               "%s": {
                 "terms": {
@@ -452,7 +453,7 @@ public void testExtractCompositeAggregationResultsWithDynamicType() throws IOExc
         String targetField = randomAlphaOfLengthBetween(5, 10);
         String targetField2 = randomAlphaOfLengthBetween(5, 10) + "_2";
 
-        GroupConfig groupBy = parseGroupConfig(formatted("""
+        GroupConfig groupBy = parseGroupConfig(Strings.format("""
             {
               "%s": {
                 "terms": {
@@ -516,7 +517,7 @@ public void testExtractCompositeAggregationResultsWithPipelineAggregation() thro
         String targetField = randomAlphaOfLengthBetween(5, 10);
         String targetField2 = randomAlphaOfLengthBetween(5, 10) + "_2";
 
-        GroupConfig groupBy = parseGroupConfig(formatted("""
+        GroupConfig groupBy = parseGroupConfig(Strings.format("""
             {
               "%s": {
                 "terms": {
@@ -604,7 +605,7 @@ public void testExtractCompositeAggregationResultsDocIDs() throws IOException {
         String targetField = randomAlphaOfLengthBetween(5, 10);
         String targetField2 = randomAlphaOfLengthBetween(5, 10) + "_2";
 
-        GroupConfig groupBy = parseGroupConfig(formatted("""
+        GroupConfig groupBy = parseGroupConfig(Strings.format("""
             {
               "%s": {
                 "terms": {
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollectorTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollectorTests.java
index adeef0af15f2..a92428697895 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollectorTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollectorTests.java
@@ -140,6 +140,7 @@ public void testDateHistogramFieldCollector() throws IOException {
             null,
             false,
             new DateHistogramGroupSource.FixedInterval(DateHistogramInterval.MINUTE),
+            null,
             null
         );
         groups.put("output_timestamp", groupBy);
@@ -224,6 +225,7 @@ public void testDateHistogramFieldCollector() throws IOException {
             null,
             true,
             new DateHistogramGroupSource.FixedInterval(DateHistogramInterval.MINUTE),
+            null,
             null
         );
         groups.put("output_timestamp", groupBy);
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java
index 9e31445fb7b8..6919cc0a982d 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.search.SearchHit;
@@ -419,7 +420,7 @@ private AggregationConfig getAggregationConfig(String agg) throws IOException {
                 {"pivot_global": {"global": {}}}""");
         }
 
-        return parseAggregations(formatted("""
+        return parseAggregations(Strings.format("""
             {
               "pivot_%s": {
                 "%s": {
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregationsTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregationsTests.java
index 3b187cb277b3..1666fde56df1 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregationsTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/TransformAggregationsTests.java
@@ -7,8 +7,10 @@
 
 package org.elasticsearch.xpack.transform.transforms.pivot;
 
+import org.elasticsearch.aggregations.AggregationsPlugin;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.search.SearchModule;
@@ -16,7 +18,6 @@
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
-import org.elasticsearch.search.aggregations.matrix.MatrixAggregationPlugin;
 import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.PercentilesAggregationBuilder;
@@ -141,7 +142,7 @@ public void testResolveTargetMapping() {
 
     public void testAggregationsVsTransforms() {
         // Note: if a new plugin is added, it must be added here
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, Arrays.asList((new AnalyticsPlugin()), new MatrixAggregationPlugin()));
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Arrays.asList((new AnalyticsPlugin()), new AggregationsPlugin()));
         List<NamedWriteableRegistry.Entry> namedWriteables = searchModule.getNamedWriteables();
 
         List<String> aggregationNames = namedWriteables.stream()
@@ -150,7 +151,7 @@ public void testAggregationsVsTransforms() {
             .collect(Collectors.toList());
 
         for (String aggregationName : aggregationNames) {
-            String message = formatted("""
+            String message = Strings.format("""
                 The following aggregation is unknown to transform: [%s]. If this is a newly added aggregation, \
                 please open an issue to add transform support for it. Afterwards add "%s" to the list in %s. \
                 Thanks!\
diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/scheduling/TransformSchedulerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/scheduling/TransformSchedulerTests.java
index 96f497f04dc2..4fa5b5eac001 100644
--- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/scheduling/TransformSchedulerTests.java
+++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/scheduling/TransformSchedulerTests.java
@@ -29,12 +29,15 @@
 import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 
+import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 
@@ -171,6 +174,55 @@ public void testSchedulingWithFailures() {
         transformScheduler.stop();
     }
 
+    public void testScheduleNow() {
+        String transformId = "test-schedule-now-with-fake-clock";
+        TimeValue frequency = TimeValue.timeValueHours(1);
+        TransformTaskParams transformTaskParams = new TransformTaskParams(transformId, Version.CURRENT, frequency, false);
+        FakeClock clock = new FakeClock(Instant.ofEpochMilli(0));
+        CopyOnWriteArrayList<TransformScheduler.Event> events = new CopyOnWriteArrayList<>();
+        TransformScheduler.Listener listener = events::add;
+
+        TransformScheduler transformScheduler = new TransformScheduler(clock, threadPool, SETTINGS);
+        transformScheduler.registerTransform(transformTaskParams, listener);
+        assertThat(
+            transformScheduler.getTransformScheduledTasks(),
+            contains(new TransformScheduledTask(transformId, frequency, 0L, 0, 60 * 60 * 1000, listener))
+        );
+        assertThat(events, hasSize(1));
+
+        // Advance time by 30 minutes (half of the configured transform frequency).
+        clock.advanceTimeBy(Duration.ofMillis(frequency.millis() / 2));
+        assertThat(
+            transformScheduler.getTransformScheduledTasks(),
+            contains(new TransformScheduledTask(transformId, frequency, 0L, 0, 60 * 60 * 1000, listener))
+        );
+        assertThat(events, hasSize(1));
+
+        // Schedule the transform now even though it is half-way through between checkpoints.
+        transformScheduler.scheduleNow(transformId);
+        assertThat(
+            transformScheduler.getTransformScheduledTasks(),
+            contains(new TransformScheduledTask(transformId, frequency, 30 * 60 * 1000L, 0, 90 * 60 * 1000, listener))
+        );
+        assertThat(events, hasSize(2));
+
+        clock.advanceTimeBy(Duration.ofMinutes(1));
+        transformScheduler.scheduleNow(transformId);
+        assertThat(
+            transformScheduler.getTransformScheduledTasks(),
+            contains(new TransformScheduledTask(transformId, frequency, 31 * 60 * 1000L, 0, 91 * 60 * 1000, listener))
+        );
+        assertThat(events, hasSize(3));
+        assertThat(events.get(0), is(equalTo(new TransformScheduler.Event(transformId, 0, 0))));
+        assertThat(events.get(1), is(equalTo(new TransformScheduler.Event(transformId, 30 * 60 * 1000, 30 * 60 * 1000))));
+        assertThat(events.get(2), is(equalTo(new TransformScheduler.Event(transformId, 31 * 60 * 1000, 31 * 60 * 1000))));
+
+        transformScheduler.deregisterTransform(transformId);
+        assertThat(transformScheduler.getTransformScheduledTasks(), is(empty()));
+
+        transformScheduler.stop();
+    }
+
     public void testConcurrentProcessing() throws Exception {
         String transformId = "test-with-fake-clock-concurrent";
         int frequencySeconds = 5;
@@ -252,7 +304,7 @@ public void triggered(TransformScheduler.Event event) {
         );
     }
 
-    public void testWithSystemClock() throws Exception {
+    public void testSchedulingWithSystemClock() throws Exception {
         String transformId = "test-with-system-clock";
         TimeValue frequency = TimeValue.timeValueSeconds(1);
         TransformTaskParams transformTaskParams = new TransformTaskParams(transformId, Version.CURRENT, frequency, false);
@@ -277,6 +329,30 @@ public void testWithSystemClock() throws Exception {
         transformScheduler.stop();
     }
 
+    public void testScheduleNowWithSystemClock() throws Exception {
+        String transformId = "test-schedule-now-with-system-clock";
+        TimeValue frequency = TimeValue.timeValueHours(1);  // Very long pause between checkpoints
+        TransformTaskParams transformTaskParams = new TransformTaskParams(transformId, Version.CURRENT, frequency, false);
+        Clock clock = Clock.systemUTC();
+        CopyOnWriteArrayList<TransformScheduler.Event> events = new CopyOnWriteArrayList<>();
+
+        TransformScheduler transformScheduler = new TransformScheduler(clock, threadPool, SETTINGS);
+        transformScheduler.start();
+        transformScheduler.registerTransform(transformTaskParams, events::add);
+        assertThat(events, hasSize(1));
+
+        Thread.sleep(5 * 1000L);
+        transformScheduler.scheduleNow(transformId);
+
+        assertThat(events, hasSize(2));
+        assertThat(events.get(0).transformId(), is(equalTo(transformId)));
+        assertThat(events.get(1).transformId(), is(equalTo(transformId)));
+        assertThat(events.get(1).scheduledTime() - events.get(0).triggeredTime(), is(allOf(greaterThan(4 * 1000L), lessThan(6 * 1000L))));
+
+        transformScheduler.deregisterTransform(transformId);
+        transformScheduler.stop();
+    }
+
     public void testScheduledTransformTaskEqualsAndHashCode() {
         Supplier<TransformScheduler.Listener> listenerSupplier = () -> new TransformScheduler.Listener() {
             @Override
diff --git a/x-pack/plugin/vector-tile/build.gradle b/x-pack/plugin/vector-tile/build.gradle
index e9ed8848cc38..017685e32dd7 100644
--- a/x-pack/plugin/vector-tile/build.gradle
+++ b/x-pack/plugin/vector-tile/build.gradle
@@ -14,7 +14,7 @@
  */
 
 apply plugin: 'elasticsearch.internal-es-plugin'
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 esplugin {
   name 'vector-tile'
@@ -29,11 +29,11 @@ dependencies {
   testImplementation(testArtifact(project(xpackModule('core'))))
   compileOnly "org.locationtech.jts:jts-core:${versions.jts}"
   api "com.wdtinc:mapbox-vector-tile:3.1.0"
-  api "com.google.protobuf:protobuf-java:3.16.1"
+  api "com.google.protobuf:protobuf-java:${versions.protobuf}"
   runtimeOnly("org.slf4j:slf4j-api:${versions.slf4j}")
   runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}")
   javaRestTestImplementation("com.wdtinc:mapbox-vector-tile:3.1.0")
-  javaRestTestImplementation("com.google.protobuf:protobuf-java:3.16.1")
+  javaRestTestImplementation("com.google.protobuf:protobuf-java:${versions.protobuf}")
 }
 
 testClusters.configureEach {
diff --git a/x-pack/plugin/vector-tile/qa/multi-cluster/build.gradle b/x-pack/plugin/vector-tile/qa/multi-cluster/build.gradle
index 0a987a3faa1e..ea20c8eacd17 100644
--- a/x-pack/plugin/vector-tile/qa/multi-cluster/build.gradle
+++ b/x-pack/plugin/vector-tile/qa/multi-cluster/build.gradle
@@ -7,7 +7,7 @@
 
 import org.elasticsearch.gradle.testclusters.DefaultTestClustersTask
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 dependencies {
   javaRestTestImplementation project(':x-pack:plugin:vector-tile')
diff --git a/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java b/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java
index 2ab1310d0769..192f051f6f1c 100644
--- a/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java
+++ b/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.LinearRing;
 import org.elasticsearch.geometry.MultiPolygon;
@@ -35,7 +36,6 @@
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Locale;
 
 /**
  * Rest test for _mvt end point. The tests only check that the structure of the vector tiles is sound in
@@ -46,6 +46,7 @@ public class VectorTileRestIT extends ESRestTestCase {
     private static final String INDEX_POINTS = "index-points";
     private static final String INDEX_POLYGON = "index-polygon";
     private static final String INDEX_COLLECTION = "index-collection";
+    private static final String INDEX_BIG_POLYGON = "index-big-polygon";
     private static final String INDEX_POINTS_SHAPES = INDEX_POINTS + "," + INDEX_POLYGON;
     private static final String INDEX_ALL = "index*";
     private static final String META_LAYER = "meta";
@@ -62,7 +63,8 @@ public void indexDocuments() throws IOException {
             x = randomIntBetween(0, (1 << z) - 1);
             y = randomIntBetween(0, (1 << z) - 1);
             indexPoints();
-            indexShapes();
+            indexPolygon();
+            indexBigPolygon();
             indexCollection();
             oneTimeSetup = true;
         }
@@ -92,7 +94,7 @@ private void indexPoints() throws IOException {
         for (int i = 0; i < 30; i += 10) {
             for (int j = 0; j <= i; j++) {
                 final Request putRequest = new Request(HttpPost.METHOD_NAME, INDEX_POINTS + "/_doc/");
-                putRequest.setJsonEntity(String.format(Locale.ROOT, """
+                putRequest.setJsonEntity(Strings.format("""
                     {
                       "location": "POINT(%s %s)", "name": "point%s", "value1": %s, "value2": %s
                     }""", x, y, i, i, i + 1));
@@ -106,11 +108,16 @@ private void indexPoints() throws IOException {
         assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK));
     }
 
-    private void indexShapes() throws IOException {
+    private void indexPolygon() throws IOException {
         final Rectangle r = GeoTileUtils.toBoundingBox(x, y, z);
         createIndexAndPutGeometry(INDEX_POLYGON, toPolygon(r), "polygon");
     }
 
+    private void indexBigPolygon() throws IOException {
+        final Rectangle r = new Rectangle(-180, 180, 90, -90);
+        createIndexAndPutGeometry(INDEX_BIG_POLYGON, toPolygon(r), "polygon");
+    }
+
     private void createIndexAndPutGeometry(String indexName, Geometry geometry, String id) throws IOException {
         final Request createRequest = new Request(HttpPut.METHOD_NAME, indexName);
         Response response = client().performRequest(createRequest);
@@ -135,7 +142,7 @@ private void createIndexAndPutGeometry(String indexName, Geometry geometry, Stri
         assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK));
 
         final Request putRequest = new Request(HttpPost.METHOD_NAME, indexName + "/_doc/" + id);
-        putRequest.setJsonEntity(String.format(Locale.ROOT, """
+        putRequest.setJsonEntity(Strings.format("""
             {
               "location": "%s", "name": "geometry", "value1": %s, "value2": %s, "nullField" : null, "ignore_value" : ""
             }""", WellKnownText.toWKT(geometry), 1, 2));
@@ -192,7 +199,7 @@ private void indexCollection() throws IOException {
             + " "
             + y
             + "))";
-        putRequest.setJsonEntity(String.format(Locale.ROOT, """
+        putRequest.setJsonEntity(Strings.format("""
             {
               "location": "%s", "name": "collection", "value1": %s, "value2": %s
             }""", collection, 1, 2));
@@ -234,8 +241,8 @@ public void testIndexAllGet() throws Exception {
         mvtRequest.setJsonEntity("{\"size\" : 100}");
         final VectorTile.Tile tile = execute(mvtRequest);
         assertThat(tile.getLayersCount(), Matchers.equalTo(3));
-        // 33 points, 1 polygon and two from geometry collection
-        assertLayer(tile, HITS_LAYER, 4096, 36, 2);
+        // 33 points, 1 big polygon, 1 polygon and two from geometry collection
+        assertLayer(tile, HITS_LAYER, 4096, 37, 2);
         assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 2);
         assertLayer(tile, META_LAYER, 4096, 1, 13);
     }
@@ -325,6 +332,51 @@ public void testGridPrecision() throws Exception {
         }
     }
 
+    public void testGridPrecisionGeoTile() throws Exception {
+        final int z = randomIntBetween(0, GeoTileUtils.MAX_ZOOM - 10);
+        final int x = randomIntBetween(0, (1 << z) - 1);
+        final int y = randomIntBetween(0, (1 << z) - 1);
+        for (int i = 1; i <= 8; i++) {
+            final Request mvtRequest = new Request(getHttpMethod(), INDEX_BIG_POLYGON + "/_mvt/location/" + z + "/" + x + "/" + y);
+            mvtRequest.setJsonEntity("{\"size\" : 0, \"grid_agg\" : \"geotile\", \"grid_precision\" : " + i + " }");
+            final VectorTile.Tile tile = execute(mvtRequest);
+            assertThat(tile.getLayersCount(), Matchers.equalTo(2));
+            assertLayer(tile, AGGS_LAYER, 4096, (1 << i) * (1 << i), 2);
+            assertLayer(tile, META_LAYER, 4096, 1, 13);
+        }
+    }
+
+    public void testGridPrecisionGeoHex() throws Exception {
+        // the number of hex depends on the position of the tile, therefore we just check some of them.
+        final int[] expected_zoom_0 = new int[] { 122, 122, 842, 842, 5872, 5872, 41058, 41058 };
+        for (int i = 1; i <= 8; i++) {
+            final Request mvtRequest = new Request(getHttpMethod(), INDEX_BIG_POLYGON + "/_mvt/location/0/0/0");
+            mvtRequest.setJsonEntity("{\"size\" : 0, \"grid_agg\" : \"geohex\", \"grid_precision\" : " + i + " }");
+            final VectorTile.Tile tile = execute(mvtRequest);
+            assertThat(tile.getLayersCount(), Matchers.equalTo(2));
+            assertLayer(tile, AGGS_LAYER, 4096, expected_zoom_0[i - 1], 2);
+            assertLayer(tile, META_LAYER, 4096, 1, 13);
+        }
+        final int[] expected_zoom_1 = new int[] { 45, 241, 241, 1559, 1559, 10531, 10531, 10531 };
+        for (int i = 1; i <= 8; i++) {
+            final Request mvtRequest = new Request(getHttpMethod(), INDEX_BIG_POLYGON + "/_mvt/location/1/0/0");
+            mvtRequest.setJsonEntity("{\"size\" : 0, \"grid_agg\" : \"geohex\", \"grid_precision\" : " + i + " }");
+            final VectorTile.Tile tile = execute(mvtRequest);
+            assertThat(tile.getLayersCount(), Matchers.equalTo(2));
+            assertLayer(tile, AGGS_LAYER, 4096, expected_zoom_1[i - 1], 2);
+            assertLayer(tile, META_LAYER, 4096, 1, 13);
+        }
+        final int[] expected_zoom_5 = new int[] { 12, 55, 55, 55, 292, 292, 1873, 12673 };
+        for (int i = 1; i <= 8; i++) {
+            final Request mvtRequest = new Request(getHttpMethod(), INDEX_BIG_POLYGON + "/_mvt/location/5/16/8");
+            mvtRequest.setJsonEntity("{\"size\" : 0, \"grid_agg\" : \"geohex\", \"grid_precision\" : " + i + " }");
+            final VectorTile.Tile tile = execute(mvtRequest);
+            assertThat(tile.getLayersCount(), Matchers.equalTo(2));
+            assertLayer(tile, AGGS_LAYER, 4096, expected_zoom_5[i - 1], 2);
+            assertLayer(tile, META_LAYER, 4096, 1, 13);
+        }
+    }
+
     public void testWithLabels() throws Exception {
         final Request mvtRequest = new Request(getHttpMethod(), INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y);
         mvtRequest.setJsonEntity("{\"size\" : 100, \"with_labels\": true}");
@@ -678,7 +730,7 @@ public void testRuntimeFieldWithSort() throws Exception {
         {
             // desc order, polygon should be the first hit
             final Request mvtRequest = new Request(getHttpMethod(), INDEX_POINTS_SHAPES + "/_mvt/location/" + z + "/" + x + "/" + y);
-            mvtRequest.setJsonEntity(formatted("""
+            mvtRequest.setJsonEntity(Strings.format("""
                 {
                   "size" : 100,
                   "grid_precision" : 0,
@@ -701,7 +753,7 @@ public void testRuntimeFieldWithSort() throws Exception {
         {
             // asc order, polygon should be the last hit
             final Request mvtRequest = new Request(getHttpMethod(), INDEX_POINTS_SHAPES + "/_mvt/location/" + z + "/" + x + "/" + y);
-            mvtRequest.setJsonEntity(formatted("""
+            mvtRequest.setJsonEntity(Strings.format("""
                 {
                   "size" : 100,
                   "grid_precision" : 0,
diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java
index 45a756976ea3..4504c0a4532b 100644
--- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java
+++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/GridAggregation.java
@@ -8,17 +8,12 @@
 package org.elasticsearch.xpack.vectortile.rest;
 
 import org.elasticsearch.common.geo.GeoUtils;
-import org.elasticsearch.common.geo.GeometryNormalizer;
-import org.elasticsearch.common.geo.Orientation;
-import org.elasticsearch.geometry.LinearRing;
-import org.elasticsearch.geometry.Polygon;
 import org.elasticsearch.geometry.Rectangle;
-import org.elasticsearch.h3.CellBoundary;
 import org.elasticsearch.h3.H3;
-import org.elasticsearch.h3.LatLng;
 import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
+import org.elasticsearch.xpack.spatial.common.H3CartesianUtil;
 import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoHexGridAggregationBuilder;
 import org.elasticsearch.xpack.vectortile.feature.FeatureFactory;
 
@@ -57,6 +52,12 @@ public byte[] toGrid(String bucketKey, FeatureFactory featureFactory) throws IOE
         public Rectangle toRectangle(String bucketKey) {
             return GeoTileUtils.toBoundingBox(bucketKey);
         }
+
+        @Override
+        public boolean needsBounding(int z, int gridPrecision) {
+            // we always bound it as it is pretty efficient anyway.
+            return true;
+        }
     },
     GEOHEX {
 
@@ -118,6 +119,7 @@ public Rectangle toRectangle(String bucketKey) {
             2,
             3,
             3,
+            3,
             4,
             4,
             5,
@@ -125,16 +127,15 @@ public Rectangle toRectangle(String bucketKey) {
             6,
             7,
             8,
-            8,
+            9,
             9,
             10,
             11,
             11,
             12,
             13,
-            13,
             14,
-            15,
+            14,
             15,
             15,
             15,
@@ -171,36 +172,24 @@ public int gridPrecisionToAggPrecision(int z, int gridPrecision) {
 
         @Override
         public byte[] toGrid(String bucketKey, FeatureFactory featureFactory) {
-            final CellBoundary boundary = H3.h3ToGeoBoundary(bucketKey);
-            final double[] lats = new double[boundary.numPoints() + 1];
-            final double[] lons = new double[boundary.numPoints() + 1];
-            for (int i = 0; i < boundary.numPoints(); i++) {
-                final LatLng latLng = boundary.getLatLon(i);
-                lats[i] = latLng.getLatDeg();
-                lons[i] = latLng.getLonDeg();
-            }
-            lats[boundary.numPoints()] = lats[0];
-            lons[boundary.numPoints()] = lons[0];
-            final Polygon polygon = new Polygon(new LinearRing(lons, lats));
-            final List<byte[]> x = featureFactory.getFeatures(GeometryNormalizer.apply(Orientation.CCW, polygon));
+            final List<byte[]> x = featureFactory.getFeatures(H3CartesianUtil.getNormalizeGeometry(H3.stringToH3(bucketKey)));
             return x.size() > 0 ? x.get(0) : null;
         }
 
         @Override
         public Rectangle toRectangle(String bucketKey) {
-            final CellBoundary boundary = H3.h3ToGeoBoundary(bucketKey);
-            double minLat = Double.POSITIVE_INFINITY;
-            double minLon = Double.POSITIVE_INFINITY;
-            double maxLat = Double.NEGATIVE_INFINITY;
-            double maxLon = Double.NEGATIVE_INFINITY;
-            for (int i = 0; i < boundary.numPoints(); i++) {
-                final LatLng latLng = boundary.getLatLon(i);
-                minLat = Math.min(minLat, latLng.getLatDeg());
-                minLon = Math.min(minLon, latLng.getLonDeg());
-                maxLat = Math.max(maxLat, latLng.getLatDeg());
-                maxLon = Math.max(maxLon, latLng.getLonDeg());
-            }
-            return new Rectangle(minLon, maxLon, maxLat, minLat);
+            return H3CartesianUtil.toBoundingBox(H3.stringToH3(bucketKey));
+        }
+
+        @Override
+        public boolean needsBounding(int z, int gridPrecision) {
+            /*
+              Bounded geohex aggregation can be expensive, in particular where there is lots of data outside the bounding
+              box. Because we are buffering our queries, this is magnified for low precision tiles. Because the total number
+              of buckets up to precision 3 is lower than the default max buckets, we better not bound those aggregations
+              which results in much better performance.
+             */
+            return gridPrecisionToAggPrecision(z, gridPrecision) > 3;
         }
     };
 
@@ -231,6 +220,11 @@ public Rectangle toRectangle(String bucketKey) {
      */
     public abstract Rectangle toRectangle(String bucketKey);
 
+    /**
+     * If false, the aggregation at the given zoom and grid precision is not bound.
+     */
+    public abstract boolean needsBounding(int z, int gridPrecision);
+
     public static GridAggregation fromString(String type) {
         return switch (type.toLowerCase(Locale.ROOT)) {
             case "geotile" -> GEOTILE;
diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java
index a98c4d9a4b03..effebcca27ec 100644
--- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java
+++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java
@@ -215,11 +215,17 @@ private static SearchRequestBuilder searchRequestBuilder(RestCancellableNodeClie
         }
         searchRequestBuilder.setQuery(qBuilder);
         if (request.getGridPrecision() > 0) {
-            final Rectangle rectangle = request.getBoundingBox();
-            final GeoBoundingBox boundingBox = new GeoBoundingBox(
-                new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()),
-                new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon())
-            );
+            final GeoBoundingBox boundingBox;
+            if (request.getGridAgg().needsBounding(request.getZ(), request.getGridPrecision())) {
+                final Rectangle rectangle = request.getBoundingBox();
+                boundingBox = new GeoBoundingBox(
+                    new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()),
+                    new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon())
+                );
+            } else {
+                // unbounded
+                boundingBox = new GeoBoundingBox(new GeoPoint(Double.NaN, Double.NaN), new GeoPoint(Double.NaN, Double.NaN));
+            }
             final GeoGridAggregationBuilder tileAggBuilder = request.getGridAgg()
                 .newAgg(GRID_FIELD)
                 .field(request.getField())
diff --git a/x-pack/plugin/voting-only-node/src/main/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodePlugin.java b/x-pack/plugin/voting-only-node/src/main/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodePlugin.java
index 35e125210785..bee5eb859a35 100644
--- a/x-pack/plugin/voting-only-node/src/main/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodePlugin.java
+++ b/x-pack/plugin/voting-only-node/src/main/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodePlugin.java
@@ -20,7 +20,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -95,7 +95,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver expressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         this.threadPool.set(threadPool);
         return Collections.emptyList();
diff --git a/x-pack/plugin/watcher/qa/rest/build.gradle b/x-pack/plugin/watcher/qa/rest/build.gradle
index 007c11c1d4b6..3f61bdcb3c2e 100644
--- a/x-pack/plugin/watcher/qa/rest/build.gradle
+++ b/x-pack/plugin/watcher/qa/rest/build.gradle
@@ -1,8 +1,8 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 dependencies {
   yamlRestTestImplementation project(path: ':x-pack:plugin:watcher:qa:common')
@@ -40,6 +40,7 @@ tasks.named("yamlRestTestV7CompatTransform").configure{ task ->
   task.skipTest("mustache/30_search_input/Test search input mustache integration (using request body and rest_total_hits_as_int)", "remove JodaCompatibleDateTime -- ZonedDateTime doesn't output millis/nanos if they're 0 (#78417)")
   task.skipTest("mustache/30_search_input/Test search input mustache integration (using request body)", "remove JodaCompatibleDateTime -- ZonedDateTime doesn't output millis/nanos if they're 0 (#78417)")
   task.skipTest("mustache/40_search_transform/Test search transform mustache integration (using request body)", "remove JodaCompatibleDateTime -- ZonedDateTime doesn't output millis/nanos if they're 0 (#78417)")
+  task.skipTest("painless/40_exception/Test painless exceptions are returned when logging a broken response", "Exceptions are no longer thrown from Mustache, but from the transform action itself")
   task.replaceKeyInDo("watcher.ack_watch", "xpack-watcher.ack_watch")
   task.replaceKeyInDo("watcher.activate_watch", "xpack-watcher.activate_watch")
   task.replaceKeyInDo("watcher.deactivate_watch", "xpack-watcher.deactivate_watch")
diff --git a/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/painless/40_exception.yml b/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/painless/40_exception.yml
index 411ef8426552..e65d7280e117 100644
--- a/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/painless/40_exception.yml
+++ b/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/painless/40_exception.yml
@@ -37,6 +37,10 @@
 
 ---
 "Test painless exceptions are returned when logging a broken response":
+  - skip:
+      version: " - 8.6.99"
+      reason:  "self-referencing objects were fixed to be in Painless instead of Mustache in 8.7"
+
   - do:
       cluster.health:
           wait_for_status: green
@@ -75,36 +79,5 @@
   - match: { watch_record.trigger_event.type: "manual" }
   - match: { watch_record.state: "executed" }
   - match: { watch_record.result.actions.0.status: "failure" }
-  - match: { watch_record.result.actions.0.error.caused_by.caused_by.type: "illegal_argument_exception" }
-  - match: { watch_record.result.actions.0.error.caused_by.caused_by.reason: "Iterable object is self-referencing itself (CustomReflectionObjectHandler stringify)" }
-
-  - do:
-      catch: bad_request
-      watcher.execute_watch:
-        body: >
-          {
-            "watch": {
-              "trigger": {
-                "schedule": {
-                  "interval": "10s"
-                }
-              },
-              "input": {
-                "simple": {
-                  "foo": "bar"
-                }
-              },
-              "actions": {
-                "my-logging": {
-                  "transform": {
-                    "script": {
-                      "source": "def x = [:] ; def y = [:] ; x.a = y ; y.a = x ; return x"
-                    }
-                  },
-                  "logging": {
-                    "text": "{{#join}}ctx.payload{{/join}}"
-                  }
-                }
-              }
-            }
-          }
+  - match: { watch_record.result.actions.0.reason: "Failed to transform payload" }
+  - match: { watch_record.result.actions.0.transform.error.reason: "Iterable object is self-referencing itself (watcher action payload)" }
diff --git a/x-pack/plugin/watcher/qa/with-monitoring/build.gradle b/x-pack/plugin/watcher/qa/with-monitoring/build.gradle
index b1f8d157394e..6a2ef6fbd7bd 100644
--- a/x-pack/plugin/watcher/qa/with-monitoring/build.gradle
+++ b/x-pack/plugin/watcher/qa/with-monitoring/build.gradle
@@ -1,6 +1,6 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 dependencies {
   javaRestTestImplementation project(':x-pack:qa')
diff --git a/x-pack/plugin/watcher/qa/with-monitoring/src/javaRestTest/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java b/x-pack/plugin/watcher/qa/with-monitoring/src/javaRestTest/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java
index e854b524aa72..d8ccbfd7688f 100644
--- a/x-pack/plugin/watcher/qa/with-monitoring/src/javaRestTest/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java
+++ b/x-pack/plugin/watcher/qa/with-monitoring/src/javaRestTest/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java
@@ -44,28 +44,6 @@ public void cleanExporters() throws Exception {
         deleteAllWatcherData();
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/59132")
-    public void testThatLocalExporterAddsWatches() throws Exception {
-        String watchId = createMonitoringWatch();
-
-        Request request = new Request("PUT", "/_cluster/settings");
-        request.setJsonEntity(
-            Strings.toString(
-                jsonBuilder().startObject()
-                    .startObject("persistent")
-                    .field("xpack.monitoring.exporters.my_local_exporter.type", "local")
-                    .field("xpack.monitoring.exporters.my_local_exporter.cluster_alerts.management.enabled", true)
-                    .endObject()
-                    .endObject()
-            )
-        );
-        adminClient().performRequest(request);
-
-        assertTotalWatchCount(WATCH_IDS.length);
-
-        assertMonitoringWatchHasBeenOverWritten(watchId);
-    }
-
     private void assertMonitoringWatchHasBeenOverWritten(String watchId) throws Exception {
         assertBusy(() -> {
             ObjectPath path = ObjectPath.createFromResponse(client().performRequest(new Request("GET", "/_watcher/watch/" + watchId)));
diff --git a/x-pack/plugin/watcher/qa/with-security/build.gradle b/x-pack/plugin/watcher/qa/with-security/build.gradle
index d3b064b271b5..5c01573e0941 100644
--- a/x-pack/plugin/watcher/qa/with-security/build.gradle
+++ b/x-pack/plugin/watcher/qa/with-security/build.gradle
@@ -1,6 +1,6 @@
-apply plugin: 'elasticsearch.internal-java-rest-test'
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 dependencies {
   yamlRestTestImplementation project(path: ':x-pack:plugin:watcher:qa:common')
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookTokenIntegrationTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookTokenIntegrationTests.java
new file mode 100644
index 000000000000..8fba22b44f3f
--- /dev/null
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookTokenIntegrationTests.java
@@ -0,0 +1,143 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.watcher.actions.webhook;
+
+import org.elasticsearch.action.admin.cluster.node.reload.NodesReloadSecureSettingsAction;
+import org.elasticsearch.action.admin.cluster.node.reload.NodesReloadSecureSettingsRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.KeyStoreWrapper;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.http.MockRequest;
+import org.elasticsearch.test.http.MockResponse;
+import org.elasticsearch.test.http.MockWebServer;
+import org.elasticsearch.transport.netty4.Netty4Plugin;
+import org.elasticsearch.xpack.core.watcher.history.WatchRecord;
+import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
+import org.elasticsearch.xpack.watcher.actions.ActionBuilders;
+import org.elasticsearch.xpack.watcher.common.http.BasicAuth;
+import org.elasticsearch.xpack.watcher.common.http.HttpMethod;
+import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
+import org.elasticsearch.xpack.watcher.common.text.TextTemplate;
+import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
+import org.elasticsearch.xpack.watcher.notification.WebhookService;
+import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+import java.nio.file.Path;
+import java.util.Collection;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.elasticsearch.xpack.watcher.client.WatchSourceBuilders.watchBuilder;
+import static org.elasticsearch.xpack.watcher.input.InputBuilders.simpleInput;
+import static org.elasticsearch.xpack.watcher.test.WatcherTestUtils.xContentSource;
+import static org.elasticsearch.xpack.watcher.trigger.TriggerBuilders.schedule;
+import static org.elasticsearch.xpack.watcher.trigger.schedule.Schedules.interval;
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class WebhookTokenIntegrationTests extends AbstractWatcherIntegrationTestCase {
+
+    private MockWebServer webServer = new MockWebServer();
+
+    @Override
+    protected boolean addMockHttpTransport() {
+        return false; // enable http
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return CollectionUtils.appendToCopy(super.nodePlugins(), Netty4Plugin.class); // for http
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        Settings.Builder builder = Settings.builder();
+        builder.put(super.nodeSettings(nodeOrdinal, otherSettings));
+        builder.put(WebhookService.SETTING_WEBHOOK_TOKEN_ENABLED.getKey(), true);
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString(WebhookService.SETTING_WEBHOOK_HOST_TOKEN_PAIRS.getKey(), "localhost:0=oldtoken");
+        builder.setSecureSettings(secureSettings);
+        return builder.build();
+    }
+
+    @Before
+    public void startWebservice() throws Exception {
+        webServer.start();
+    }
+
+    @After
+    public void stopWebservice() throws Exception {
+        webServer.close();
+    }
+
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/93536")
+    public void testWebhook() throws Exception {
+        String localServer = "localhost:" + webServer.getPort();
+        logger.info("--> updating keystore token hosts to: {}", localServer);
+        Path configPath = internalCluster().configPaths().stream().findFirst().orElseThrow();
+        try (KeyStoreWrapper ksw = KeyStoreWrapper.bootstrap(configPath, () -> new SecureString("".toCharArray()))) {
+            ksw.setString(WebhookService.SETTING_WEBHOOK_HOST_TOKEN_PAIRS.getKey(), (localServer + "=token1234").toCharArray());
+            ksw.save(configPath, "".toCharArray());
+        }
+        // Reload the keystore to load the new settings
+        NodesReloadSecureSettingsRequest reloadReq = new NodesReloadSecureSettingsRequest();
+        reloadReq.setSecureStorePassword(new SecureString("".toCharArray()));
+        client().execute(NodesReloadSecureSettingsAction.INSTANCE, reloadReq).get();
+
+        webServer.enqueue(new MockResponse().setResponseCode(200).setBody("body"));
+        HttpRequestTemplate.Builder builder = HttpRequestTemplate.builder("localhost", webServer.getPort())
+            .path(new TextTemplate("/test/_id"))
+            .putParam("param1", new TextTemplate("value1"))
+            .putParam("watch_id", new TextTemplate("_id"))
+            .body(new TextTemplate("_body"))
+            .auth(new BasicAuth("user", "pass".toCharArray()))
+            .method(HttpMethod.POST);
+
+        new PutWatchRequestBuilder(client(), "_id").setSource(
+            watchBuilder().trigger(schedule(interval("5s")))
+                .input(simpleInput("key", "value"))
+                .condition(InternalAlwaysCondition.INSTANCE)
+                .addAction("_id", ActionBuilders.webhookAction(builder))
+        ).get();
+
+        timeWarp().trigger("_id");
+        refresh();
+
+        assertWatchWithMinimumPerformedActionsCount("_id", 1, false);
+        assertThat(webServer.requests(), hasSize(1));
+        MockRequest req = webServer.requests().get(0);
+        assertThat(
+            webServer.requests().get(0).getUri().getQuery(),
+            anyOf(equalTo("watch_id=_id&param1=value1"), equalTo("param1=value1&watch_id=_id"))
+        );
+        assertThat("token header should be set", req.getHeader(WebhookService.TOKEN_HEADER_NAME), equalTo("token1234"));
+
+        assertThat(webServer.requests().get(0).getBody(), is("_body"));
+
+        SearchResponse response = searchWatchRecords(b -> QueryBuilders.termQuery(WatchRecord.STATE.getPreferredName(), "executed"));
+
+        assertNoFailures(response);
+        XContentSource source = xContentSource(response.getHits().getAt(0).getSourceRef());
+        String body = source.getValue("result.actions.0.webhook.response.body");
+        assertThat(body, notNullValue());
+        assertThat(body, is("body"));
+        Number status = source.getValue("result.actions.0.webhook.response.status");
+        assertThat(status, notNullValue());
+        assertThat(status.intValue(), is(200));
+    }
+}
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java
index 10a61a188acb..d962cddb4f8f 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.datastreams.DataStreamsPlugin;
@@ -633,7 +634,7 @@ public ClockMock clock() {
         public void trigger(String watchId, int times, TimeValue timeValue) throws Exception {
             assertBusy(() -> {
                 long triggeredCount = schedulers.stream().filter(scheduler -> scheduler.trigger(watchId, times, timeValue)).count();
-                String msg = formatted("watch was triggered on [%d] schedulers, expected [1]", triggeredCount);
+                String msg = Strings.format("watch was triggered on [%d] schedulers, expected [1]", triggeredCount);
                 if (triggeredCount > 1) {
                     logger.warn(msg);
                 }
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/SingleNodeTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/SingleNodeTests.java
index 4c47907ef645..16437b22940e 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/SingleNodeTests.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/SingleNodeTests.java
@@ -40,7 +40,6 @@ protected boolean timeWarped() {
     // this is the standard setup when starting watcher in a regular cluster
     // the index does not exist, a watch gets added
     // the watch should be executed properly, despite the index being created and the cluster state listener being reloaded
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/54096")
     public void testThatLoadingWithNonExistingIndexWorks() throws Exception {
         stopWatcher();
         ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/transform/TransformIntegrationTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/transform/TransformIntegrationTests.java
index 0e8918e8bc6d..19af70d76d9f 100644
--- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/transform/TransformIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/transform/TransformIntegrationTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.script.MockScriptPlugin;
@@ -107,7 +108,7 @@ public void testScriptTransform() throws Exception {
             script = mockScript("['key3' : ctx.payload.key1 + ctx.payload.key2]");
         } else {
             logger.info("testing script transform with an indexed script");
-            assertAcked(client().admin().cluster().preparePutStoredScript().setId("my-script").setContent(new BytesArray(formatted("""
+            assertAcked(client().admin().cluster().preparePutStoredScript().setId("my-script").setContent(new BytesArray(Strings.format("""
                 {
                   "script": {
                     "lang": "%s",
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
index 89283cfc7a54..4553db34069b 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
@@ -24,7 +24,7 @@
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -131,6 +131,7 @@
 import org.elasticsearch.xpack.watcher.input.transform.TransformInput;
 import org.elasticsearch.xpack.watcher.input.transform.TransformInputFactory;
 import org.elasticsearch.xpack.watcher.notification.NotificationService;
+import org.elasticsearch.xpack.watcher.notification.WebhookService;
 import org.elasticsearch.xpack.watcher.notification.email.Account;
 import org.elasticsearch.xpack.watcher.notification.email.EmailService;
 import org.elasticsearch.xpack.watcher.notification.email.HtmlSanitizer;
@@ -306,7 +307,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver expressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         Tracer tracer,
-        AllocationDeciders allocationDeciders
+        AllocationService allocationService
     ) {
         if (enabled == false) {
             return Collections.emptyList();
@@ -341,11 +342,13 @@ public Collection<Object> createComponents(
         JiraService jiraService = new JiraService(settings, httpClient, clusterService.getClusterSettings());
         SlackService slackService = new SlackService(settings, httpClient, clusterService.getClusterSettings());
         PagerDutyService pagerDutyService = new PagerDutyService(settings, httpClient, clusterService.getClusterSettings());
+        WebhookService webhookService = new WebhookService(settings, httpClient, clusterService.getClusterSettings());
 
         reloadableServices.add(emailService);
         reloadableServices.add(jiraService);
         reloadableServices.add(slackService);
         reloadableServices.add(pagerDutyService);
+        reloadableServices.add(webhookService);
 
         TextTemplateEngine templateEngine = new TextTemplateEngine(scriptService);
         Map<String, EmailAttachmentParser<?>> emailAttachmentParsers = new HashMap<>();
@@ -386,7 +389,7 @@ public Collection<Object> createComponents(
         // actions
         final Map<String, ActionFactory> actionFactoryMap = new HashMap<>();
         actionFactoryMap.put(EmailAction.TYPE, new EmailActionFactory(settings, emailService, templateEngine, emailAttachmentsParser));
-        actionFactoryMap.put(WebhookAction.TYPE, new WebhookActionFactory(httpClient, templateEngine));
+        actionFactoryMap.put(WebhookAction.TYPE, new WebhookActionFactory(webhookService, templateEngine));
         actionFactoryMap.put(IndexAction.TYPE, new IndexActionFactory(settings, client));
         actionFactoryMap.put(LoggingAction.TYPE, new LoggingActionFactory(templateEngine));
         actionFactoryMap.put(JiraAction.TYPE, new JiraActionFactory(templateEngine, jiraService));
@@ -595,6 +598,7 @@ public List<Setting<?>> getSettings() {
         settings.addAll(JiraService.getSettings());
         settings.addAll(PagerDutyService.getSettings());
         settings.addAll(ReportingAttachmentParser.getSettings());
+        settings.addAll(WebhookService.getSettings());
 
         // http settings
         settings.addAll(HttpSettings.getSettings());
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java
index 26ee1a85a271..ca91d51a2d12 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java
@@ -234,7 +234,7 @@ private void checkWatchIndexHasChanged(IndexMetadata metadata, ClusterChangedEve
         RoutingNode routingNode = state.getRoutingNodes().node(localNodeId);
 
         // no local shards, exit early
-        List<ShardRouting> localShardRouting = routingNode.shardsWithState(watchIndex, STARTED, RELOCATING);
+        List<ShardRouting> localShardRouting = routingNode.shardsWithState(watchIndex, STARTED, RELOCATING).toList();
         if (localShardRouting.isEmpty()) {
             configuration = INACTIVE;
         } else {
@@ -281,7 +281,6 @@ private boolean hasShardAllocationIdChanged(String watchIndex, ClusterState stat
         Set<ShardId> clusterStateLocalShardIds = state.getRoutingNodes()
             .node(localNodeId)
             .shardsWithState(watchIndex, STARTED, RELOCATING)
-            .stream()
             .map(ShardRouting::shardId)
             .collect(Collectors.toSet());
         Set<ShardId> configuredLocalShardIds = new HashSet<>(configuration.localShards.keySet());
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleService.java
index b5f29085fd5f..73c3a4c093fd 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleService.java
@@ -144,7 +144,7 @@ public void clusterChanged(ClusterChangedEvent event) {
         }
 
         String watchIndex = watcherIndexMetadata.getIndex().getName();
-        List<ShardRouting> localShards = routingNode.shardsWithState(watchIndex, RELOCATING, STARTED);
+        List<ShardRouting> localShards = routingNode.shardsWithState(watchIndex, RELOCATING, STARTED).toList();
         // no local shards, empty out watcher and dont waste resources!
         if (localShards.isEmpty()) {
             pauseExecution("no local watcher shards found");
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherService.java
index 347ca20f7a94..9a0fa93f0605 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherService.java
@@ -323,7 +323,7 @@ private Collection<Watch> loadWatches(ClusterState clusterState) {
             if (routingNode == null) {
                 return Collections.emptyList();
             }
-            List<ShardRouting> localShards = routingNode.shardsWithState(watchIndexName, RELOCATING, STARTED);
+            List<ShardRouting> localShards = routingNode.shardsWithState(watchIndexName, RELOCATING, STARTED).toList();
 
             // find out all allocation ids
             List<ShardRouting> watchIndexShardRoutings = clusterState.getRoutingTable().allShards(watchIndexName);
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/ExecutableWebhookAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/ExecutableWebhookAction.java
index 45fce5cce890..e0787b200bc1 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/ExecutableWebhookAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/ExecutableWebhookAction.java
@@ -11,41 +11,22 @@
 import org.elasticsearch.xpack.core.watcher.actions.ExecutableAction;
 import org.elasticsearch.xpack.core.watcher.execution.WatchExecutionContext;
 import org.elasticsearch.xpack.core.watcher.watch.Payload;
-import org.elasticsearch.xpack.watcher.common.http.HttpClient;
-import org.elasticsearch.xpack.watcher.common.http.HttpRequest;
-import org.elasticsearch.xpack.watcher.common.http.HttpResponse;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
-import org.elasticsearch.xpack.watcher.support.Variables;
-
-import java.util.Map;
+import org.elasticsearch.xpack.watcher.notification.WebhookService;
 
 public class ExecutableWebhookAction extends ExecutableAction<WebhookAction> {
 
-    private final HttpClient httpClient;
+    private final WebhookService webhookService;
     private final TextTemplateEngine templateEngine;
 
-    public ExecutableWebhookAction(WebhookAction action, Logger logger, HttpClient httpClient, TextTemplateEngine templateEngine) {
+    public ExecutableWebhookAction(WebhookAction action, Logger logger, WebhookService webhookService, TextTemplateEngine templateEngine) {
         super(action, logger);
-        this.httpClient = httpClient;
+        this.webhookService = webhookService;
         this.templateEngine = templateEngine;
     }
 
     @Override
     public Action.Result execute(String actionId, WatchExecutionContext ctx, Payload payload) throws Exception {
-        Map<String, Object> model = Variables.createCtxParamsMap(ctx, payload);
-
-        HttpRequest request = action.requestTemplate.render(templateEngine, model);
-
-        if (ctx.simulateAction(actionId)) {
-            return new WebhookAction.Result.Simulated(request);
-        }
-
-        HttpResponse response = httpClient.execute(request);
-
-        if (response.status() >= 400) {
-            return new WebhookAction.Result.Failure(request, response);
-        } else {
-            return new WebhookAction.Result.Success(request, response);
-        }
+        return webhookService.execute(actionId, action, templateEngine, ctx, payload);
     }
 }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionFactory.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionFactory.java
index 5cd648a148ce..2937b19276a5 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionFactory.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionFactory.java
@@ -9,25 +9,25 @@
 import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.watcher.actions.ActionFactory;
-import org.elasticsearch.xpack.watcher.common.http.HttpClient;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
+import org.elasticsearch.xpack.watcher.notification.WebhookService;
 
 import java.io.IOException;
 
 public class WebhookActionFactory extends ActionFactory {
 
-    private final HttpClient httpClient;
     private final TextTemplateEngine templateEngine;
+    private final WebhookService webhookService;
 
-    public WebhookActionFactory(HttpClient httpClient, TextTemplateEngine templateEngine) {
+    public WebhookActionFactory(WebhookService webhookService, TextTemplateEngine templateEngine) {
         super(LogManager.getLogger(ExecutableWebhookAction.class));
-        this.httpClient = httpClient;
         this.templateEngine = templateEngine;
+        this.webhookService = webhookService;
     }
 
     @Override
     public ExecutableWebhookAction parseExecutable(String watchId, String actionId, XContentParser parser) throws IOException {
-        return new ExecutableWebhookAction(WebhookAction.parse(watchId, actionId, parser), actionLogger, httpClient, templateEngine);
+        return new ExecutableWebhookAction(WebhookAction.parse(watchId, actionId, parser), actionLogger, webhookService, templateEngine);
 
     }
 }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java
index 1ec4d727d0c3..7dd9ce8c1012 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.xpack.core.watcher.support.WatcherUtils;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherParams;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherXContentParser;
+import org.elasticsearch.xpack.watcher.notification.WebhookService;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -166,7 +167,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params toX
             if (WatcherParams.hideSecrets(toXContentParams)) {
                 builder.field(Field.HEADERS.getPreferredName(), sanitizeHeaders(headers));
             } else {
-                builder.field(Field.HEADERS.getPreferredName(), headers);
+                builder.field(Field.HEADERS.getPreferredName(), sanitizeInternalHeaders(headers));
             }
         }
         if (auth != null) {
@@ -195,6 +196,10 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params toX
         return builder.endObject();
     }
 
+    /**
+     * Sanitize both internal (see {@link #sanitizeInternalHeaders(Map)} and
+     * user-added sensitive headers that should not be shown.
+     */
     private Map<String, String> sanitizeHeaders(Map<String, String> headers) {
         String authorizationHeader = headers.containsKey("Authorization") ? "Authorization" : null;
         if (authorizationHeader == null) {
@@ -205,7 +210,22 @@ private Map<String, String> sanitizeHeaders(Map<String, String> headers) {
         }
         Map<String, String> sanitizedHeaders = new HashMap<>(headers);
         sanitizedHeaders.put(authorizationHeader, WatcherXContentParser.REDACTED_PASSWORD);
-        return sanitizedHeaders;
+        return sanitizeInternalHeaders(sanitizedHeaders);
+    }
+
+    /**
+     * Sanitize headers that the user may not have added, but were automatically
+     * added by Elasticsearch.
+     */
+    private Map<String, String> sanitizeInternalHeaders(Map<String, String> headers) {
+        // Redact the additional webhook password, if present.
+        if (headers.containsKey(WebhookService.TOKEN_HEADER_NAME)) {
+            Map<String, String> sanitizedHeaders = new HashMap<>(headers);
+            sanitizedHeaders.put(WebhookService.TOKEN_HEADER_NAME, WatcherXContentParser.REDACTED_PASSWORD);
+            return sanitizedHeaders;
+        } else {
+            return headers;
+        }
     }
 
     @Override
@@ -268,6 +288,13 @@ public static Builder builder(String host, int port) {
         return new Builder(host, port);
     }
 
+    /**
+     * Create a new builder modeled on this HttpRequest
+     */
+    public Builder copy() {
+        return new Builder(this);
+    }
+
     static Builder builder() {
         return new Builder();
     }
@@ -402,6 +429,21 @@ private Builder(String host, int port) {
             this.port = port;
         }
 
+        private Builder(HttpRequest original) {
+            this.host = original.host;
+            this.port = original.port;
+            this.scheme = original.scheme;
+            this.method = original.method;
+            this.path = original.path;
+            this.params = new HashMap<>(original.params);
+            this.headers = new HashMap<>(original.headers);
+            this.auth = original.auth;
+            this.body = original.body;
+            this.connectionTimeout = original.connectionTimeout;
+            this.readTimeout = original.readTimeout;
+            this.proxy = original.proxy;
+        }
+
         private Builder() {}
 
         public Builder scheme(Scheme scheme) {
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/NotificationService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/NotificationService.java
index 03e733cc8417..4e62e4152b4e 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/NotificationService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/NotificationService.java
@@ -8,6 +8,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.SecureString;
@@ -141,11 +142,12 @@ private Set<String> getAccountNames(Settings settings) {
         return settings.getByPrefix(getNotificationsAccountPrefix()).names();
     }
 
-    private @Nullable String getDefaultAccountName(Settings settings) {
+    @Nullable
+    protected String getDefaultAccountName(Settings settings) {
         return settings.get("xpack.notification." + type + ".default_account");
     }
 
-    private Map<String, LazyInitializable<Account, SettingsException>> createAccounts(
+    protected Map<String, LazyInitializable<Account, SettingsException>> createAccounts(
         Settings settings,
         Set<String> accountNames,
         BiFunction<String, Settings, Account> accountFactory
@@ -153,7 +155,7 @@ private Map<String, LazyInitializable<Account, SettingsException>> createAccount
         final Map<String, LazyInitializable<Account, SettingsException>> accounts = new HashMap<>();
         for (final String accountName : accountNames) {
             final Settings accountSettings = settings.getAsSettings(getNotificationsAccountPrefix() + accountName);
-            accounts.put(accountName, new LazyInitializable<>(() -> { return accountFactory.apply(accountName, accountSettings); }));
+            accounts.put(accountName, new LazyInitializable<>(() -> accountFactory.apply(accountName, accountSettings)));
         }
         return Collections.unmodifiableMap(accounts);
     }
@@ -209,7 +211,6 @@ private static SecureSettings extractSecureSettings(Settings source, List<Settin
             }
         }
         return new SecureSettings() {
-
             @Override
             public boolean isLoaded() {
                 return true;
@@ -237,6 +238,11 @@ public byte[] getSHA256Digest(String setting) {
 
             @Override
             public void close() throws IOException {}
+
+            @Override
+            public void writeTo(StreamOutput out) throws IOException {
+                throw new IllegalStateException("Unsupported operation");
+            }
         };
     }
 }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/WebhookService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/WebhookService.java
new file mode 100644
index 000000000000..f4e2ce84bded
--- /dev/null
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/WebhookService.java
@@ -0,0 +1,194 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.watcher.notification;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.SecureSetting;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.common.util.LazyInitializable;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+import org.elasticsearch.xpack.core.watcher.actions.Action;
+import org.elasticsearch.xpack.core.watcher.execution.WatchExecutionContext;
+import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherXContentParser;
+import org.elasticsearch.xpack.core.watcher.watch.Payload;
+import org.elasticsearch.xpack.watcher.actions.webhook.WebhookAction;
+import org.elasticsearch.xpack.watcher.common.http.HttpClient;
+import org.elasticsearch.xpack.watcher.common.http.HttpRequest;
+import org.elasticsearch.xpack.watcher.common.http.HttpResponse;
+import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
+import org.elasticsearch.xpack.watcher.support.Variables;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+
+/**
+ * The WebhookService class handles executing webhook requests for Watcher actions. These can be
+ * regular "webhook" actions as well as parts of an "email" action with attachments that make HTTP
+ * requests.
+ */
+public class WebhookService extends NotificationService<WebhookService.WebhookAccount> {
+    public static final String NAME = "webhook";
+
+    private static final Logger logger = LogManager.getLogger(WebhookService.class);
+
+    public static final String TOKEN_HEADER_NAME = "X-Elastic-App-Auth";
+
+    // Token URLs should be in the form of "example.com:8200=<token>,other.com:80=<token>",
+    // i.e., a list of comma-separated <host>:<port>=<token> pairs
+    public static final Setting<SecureString> SETTING_WEBHOOK_HOST_TOKEN_PAIRS = SecureSetting.secureString(
+        "xpack.notification.webhook.host_token_pairs",
+        null
+    );
+
+    // Boolean setting for whether token should be sent if present.
+    public static final Setting<Boolean> SETTING_WEBHOOK_TOKEN_ENABLED = Setting.boolSetting(
+        "xpack.notification.webhook.additional_token_enabled",
+        false,
+        Setting.Property.NodeScope
+    );
+
+    private final HttpClient httpClient;
+    private final boolean additionalTokenEnabled;
+
+    public WebhookService(Settings settings, HttpClient httpClient, ClusterSettings clusterSettings) {
+        super(NAME, settings, clusterSettings, List.of(), getSecureSettings());
+        this.httpClient = httpClient;
+        // do an initial load of all settings
+        reload(settings);
+        this.additionalTokenEnabled = SETTING_WEBHOOK_TOKEN_ENABLED.get(settings);
+    }
+
+    public static List<Setting<?>> getSettings() {
+        List<Setting<?>> settings = new ArrayList<>(getSecureSettings());
+        settings.add(SETTING_WEBHOOK_TOKEN_ENABLED);
+        return settings;
+    }
+
+    private static List<Setting<?>> getSecureSettings() {
+        return List.of(SETTING_WEBHOOK_HOST_TOKEN_PAIRS);
+    }
+
+    @Override
+    protected String getDefaultAccountName(Settings settings) {
+        // There are no accounts for webhooks, but we still want a default
+        // account because Watcher's notification services infrastructure
+        // expects one, so use "webhook" as the name.
+        return NAME;
+    }
+
+    @Override
+    protected WebhookAccount createAccount(String name, Settings accountSettings) {
+        throw new UnsupportedOperationException("this should never be called");
+    }
+
+    @Override
+    protected Map<String, LazyInitializable<WebhookAccount, SettingsException>> createAccounts(
+        Settings settings,
+        Set<String> accountNames,
+        BiFunction<String, Settings, WebhookAccount> accountFactory
+    ) {
+        // We override the createAccounts here because there are no real "accounts" for webhooks.
+        // Instead, we create the single account. This simplifies a great deal around dynamic
+        // setting keys for accounts
+        final WebhookAccount defaultAccount = new WebhookAccount(settings);
+        return Map.of(NAME, new LazyInitializable<>(() -> defaultAccount));
+    }
+
+    public Action.Result execute(
+        String actionId,
+        WebhookAction action,
+        TextTemplateEngine templateEngine,
+        WatchExecutionContext ctx,
+        Payload payload
+    ) throws IOException {
+        Map<String, Object> model = Variables.createCtxParamsMap(ctx, payload);
+
+        // Render the original request
+        HttpRequest request = action.getRequest().render(templateEngine, model);
+
+        // If applicable, add the extra token to the headers
+        boolean tokenAdded = false;
+        WebhookAccount account = getAccount(NAME);
+        if (this.additionalTokenEnabled && account.hostTokenMap.size() > 0) {
+            // Generate a string like example.com:9200 to match against the list of hosts where the
+            // additional token should be provided. The token will only be added to the headers if
+            // the request matches the list.
+            String reqHostAndPort = request.host() + ":" + request.port();
+            if (Strings.hasText(account.hostTokenMap.get(reqHostAndPort))) {
+                // Add the additional token
+                tokenAdded = true;
+                request = request.copy().setHeader(TOKEN_HEADER_NAME, account.hostTokenMap.get(reqHostAndPort)).build();
+            }
+        }
+
+        final Function<HttpRequest, HttpRequest> redactToken = tokenAdded
+            ? req -> req.copy().setHeader(TOKEN_HEADER_NAME, WatcherXContentParser.REDACTED_PASSWORD).build()
+            : Function.identity();
+
+        if (ctx.simulateAction(actionId)) {
+            // Skip execution, return only the simulated (and redacted if necessary) response
+            return new WebhookAction.Result.Simulated(redactToken.apply(request));
+        }
+
+        HttpResponse response = httpClient.execute(request);
+
+        if (response.status() >= 400) {
+            return new WebhookAction.Result.Failure(redactToken.apply(request), response);
+        } else {
+            return new WebhookAction.Result.Success(redactToken.apply(request), response);
+        }
+    }
+
+    public static final class WebhookAccount {
+        private final Map<String, String> hostTokenMap;
+
+        public WebhookAccount(Settings settings) {
+            SecureString validTokenHosts = SETTING_WEBHOOK_HOST_TOKEN_PAIRS.get(settings);
+            if (Strings.hasText(validTokenHosts)) {
+                Set<String> hostAndTokens = Strings.commaDelimitedListToSet(validTokenHosts.toString());
+                Map<String, String> hostAndPortToToken = new HashMap<>(hostAndTokens.size());
+                for (String hostPortToken : hostAndTokens) {
+                    int equalsIndex = hostPortToken.indexOf('=');
+                    if (equalsIndex == -1) {
+                        // This is an invalid format, and we can skip this token
+                        break;
+                    }
+                    if (equalsIndex + 1 == hostPortToken.length()) {
+                        // This is also invalid, because it ends in a trailing =
+                        break;
+                    }
+                    // The first part becomes the <host>:<port> pair
+                    String hostAndPort = hostPortToken.substring(0, equalsIndex);
+                    // The second part after the '=' is the <token>
+                    String token = hostPortToken.substring(equalsIndex + 1);
+                    hostAndPortToToken.put(hostAndPort, token);
+                }
+                this.hostTokenMap = Collections.unmodifiableMap(hostAndPortToToken);
+            } else {
+                this.hostTokenMap = Map.of();
+            }
+        }
+
+        @Override
+        public String toString() {
+            return "WebhookAccount[" + this.hostTokenMap.keySet().stream().map(s -> s + "=********") + "]";
+        }
+    }
+}
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java
index c682ca93488d..69be65996c8f 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateUtils;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.shard.ShardId;
@@ -52,10 +53,10 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.SortedMap;
 import java.util.TreeMap;
+import java.util.stream.Stream;
 
 import static java.util.Arrays.asList;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
@@ -330,14 +331,13 @@ public void testClusterChangedWatchAliasChanged() throws Exception {
         boolean emptyShards = randomBoolean();
 
         if (emptyShards) {
-            when(routingNode.shardsWithState(eq(newActiveWatchIndex), any())).thenReturn(Collections.emptyList());
+            when(routingNode.shardsWithState(eq(newActiveWatchIndex), any(ShardRoutingState[].class))).thenReturn(Stream.empty());
         } else {
             Index index = new Index(newActiveWatchIndex, "uuid");
             ShardId shardId = new ShardId(index, 0);
             ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, "node_1", true, STARTED);
-            List<ShardRouting> routing = Collections.singletonList(shardRouting);
-            when(routingNode.shardsWithState(eq(newActiveWatchIndex), eq(STARTED), eq(RELOCATING))).thenReturn(routing);
-            when(routingTable.allShards(eq(newActiveWatchIndex))).thenReturn(routing);
+            when(routingNode.shardsWithState(eq(newActiveWatchIndex), eq(STARTED), eq(RELOCATING))).thenReturn(Stream.of(shardRouting));
+            when(routingTable.allShards(eq(newActiveWatchIndex))).thenReturn(List.of(shardRouting));
             IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(index).addShard(shardRouting).build();
             when(routingTable.index(newActiveWatchIndex)).thenReturn(indexRoutingTable);
         }
@@ -455,7 +455,7 @@ public void testShardConfigurationShouldBeTriggeredExactlyOnce() throws Exceptio
                 boolean shouldBeTriggered = sac.shouldBeTriggered("watch_" + i);
                 boolean hasAlreadyBeenTriggered = bitSet.get(i);
                 if (shouldBeTriggered) {
-                    String message = String.format(Locale.ROOT, "Watch [%s] has already been " + "triggered", i);
+                    String message = Strings.format("Watch [%s] has already been " + "triggered", i);
                     assertThat(message, hasAlreadyBeenTriggered, is(false));
                     bitSet.set(i);
                 }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherMetadataSerializationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherMetadataSerializationTests.java
index 3be81d688335..498d198bb33e 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherMetadataSerializationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherMetadataSerializationTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.cluster.metadata.RepositoriesMetadata;
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ChunkedToXContent;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ToXContent;
@@ -35,7 +36,7 @@ public void testXContentSerializationOneSignedWatcher() throws Exception {
         XContentBuilder builder = XContentFactory.jsonBuilder();
         builder.startObject();
         builder.startObject("watcher");
-        watcherMetadata.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        ChunkedToXContent.wrapAsToXContent(watcherMetadata).toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
         builder.endObject();
         WatcherMetadata watchersMetadataFromXContent = getWatcherMetadataFromXContent(createParser(builder));
@@ -63,7 +64,7 @@ public void testWatcherMetadataParsingDoesNotSwallowOtherMetadata() throws Excep
             Collections.singletonMap(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY)
         );
         builder.startObject();
-        builder = metadataBuilder.build().toXContent(builder, params);
+        builder = ChunkedToXContent.wrapAsToXContent(metadataBuilder.build()).toXContent(builder, params);
         builder.endObject();
         // deserialize metadata again
         Metadata metadata = Metadata.Builder.fromXContent(createParser(builder));
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionTests.java
index 371560b1631f..f1019614d229 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionTests.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
@@ -33,6 +34,7 @@
 import org.elasticsearch.xpack.watcher.common.text.TextTemplate;
 import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine;
 import org.elasticsearch.xpack.watcher.execution.TriggeredExecutionContext;
+import org.elasticsearch.xpack.watcher.notification.WebhookService;
 import org.elasticsearch.xpack.watcher.notification.email.Attachment;
 import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateService;
 import org.elasticsearch.xpack.watcher.test.MockTextTemplateEngine;
@@ -92,7 +94,12 @@ public void testExecute() throws Exception {
         HttpRequestTemplate httpRequest = getHttpRequestTemplate(method, TEST_HOST, TEST_PORT, testPath, testBody, null);
 
         WebhookAction action = new WebhookAction(httpRequest);
-        ExecutableWebhookAction executable = new ExecutableWebhookAction(action, logger, httpClient, templateEngine);
+        WebhookService webhookService = new WebhookService(
+            Settings.EMPTY,
+            httpClient,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+        );
+        ExecutableWebhookAction executable = new ExecutableWebhookAction(action, logger, webhookService, templateEngine);
         WatchExecutionContext ctx = WatcherTestUtils.mockExecutionContext("_id", new Payload.Simple("foo", "bar"));
 
         Action.Result actionResult = executable.execute("_id", ctx, Payload.EMPTY);
@@ -155,7 +162,12 @@ public void testParserSelfGenerated() throws Exception {
 
         HttpRequestTemplate request = getHttpRequestTemplate(method, host, TEST_PORT, path, body, null);
         WebhookAction action = new WebhookAction(request);
-        ExecutableWebhookAction executable = new ExecutableWebhookAction(action, logger, ExecuteScenario.Success.client(), templateEngine);
+        WebhookService webhookService = new WebhookService(
+            Settings.EMPTY,
+            ExecuteScenario.Success.client(),
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+        );
+        ExecutableWebhookAction executable = new ExecutableWebhookAction(action, logger, webhookService, templateEngine);
 
         XContentBuilder builder = jsonBuilder();
         executable.toXContent(builder, ToXContent.EMPTY_PARAMS);
@@ -217,7 +229,12 @@ public void testParserFailure() throws Exception {
     }
 
     private WebhookActionFactory webhookFactory(HttpClient client) {
-        return new WebhookActionFactory(client, templateEngine);
+        WebhookService webhookService = new WebhookService(
+            Settings.EMPTY,
+            client,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+        );
+        return new WebhookActionFactory(webhookService, templateEngine);
     }
 
     public void testThatSelectingProxyWorks() throws Exception {
@@ -235,7 +252,12 @@ public void testThatSelectingProxyWorks() throws Exception {
                 .proxy(new HttpProxy("localhost", proxyServer.getPort()));
             WebhookAction action = new WebhookAction(builder.build());
 
-            ExecutableWebhookAction executable = new ExecutableWebhookAction(action, logger, httpClient, templateEngine);
+            WebhookService webhookService = new WebhookService(
+                Settings.EMPTY,
+                httpClient,
+                new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+            );
+            ExecutableWebhookAction executable = new ExecutableWebhookAction(action, logger, webhookService, templateEngine);
             String watchId = "test_url_encode" + randomAlphaOfLength(10);
             ScheduleTriggerEvent triggerEvent = new ScheduleTriggerEvent(
                 watchId,
@@ -268,7 +290,12 @@ public void testValidUrls() throws Exception {
         HttpRequestTemplate requestTemplate = getHttpRequestTemplate(method, host, TEST_PORT, path, testBody, null);
         WebhookAction action = new WebhookAction(requestTemplate);
 
-        ExecutableWebhookAction executable = new ExecutableWebhookAction(action, logger, client, templateEngine);
+        WebhookService webhookService = new WebhookService(
+            Settings.EMPTY,
+            client,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+        );
+        ExecutableWebhookAction executable = new ExecutableWebhookAction(action, logger, webhookService, templateEngine);
 
         ScheduleTriggerEvent triggerEvent = new ScheduleTriggerEvent(
             watchId,
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java
index d8481840e9d8..f52a7f8a65aa 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.common.ssl.SslVerificationMode;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.env.Environment;
@@ -48,7 +49,6 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
-import java.util.Locale;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicReference;
@@ -483,7 +483,7 @@ public void testThatUrlPathIsNotEncoded() throws Exception {
             request = HttpRequest.builder("localhost", webServer.getPort()).path(path).build();
         } else {
             // ensure that fromUrl acts the same way than the above builder
-            request = HttpRequest.builder().fromUrl(String.format(Locale.ROOT, "http://localhost:%s%s", webServer.getPort(), path)).build();
+            request = HttpRequest.builder().fromUrl(Strings.format("http://localhost:%s%s", webServer.getPort(), path)).build();
         }
         httpClient.execute(request);
 
@@ -822,6 +822,6 @@ public static ClusterService mockClusterService() {
     }
 
     private String getWebserverUri() {
-        return String.format(Locale.ROOT, "http://%s:%s", webServer.getHostName(), webServer.getPort());
+        return Strings.format("http://%s:%s", webServer.getHostName(), webServer.getPort());
     }
 }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java
index 348f552465c3..0bbc67585f80 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java
@@ -34,9 +34,9 @@
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
-import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
@@ -173,17 +173,10 @@ public void testValidateNoActivePrimaryShards() {
             ShardId shardId = new ShardId(index, 0);
             indexRoutingTableBuilder.addIndexShard(
                 new IndexShardRoutingTable.Builder(shardId).addShard(
-                    TestShardRouting.newShardRouting(
-                        shardId,
-                        currentNodeId,
-                        null,
-                        true,
-                        state,
-                        new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")
-                    )
+                    TestShardRouting.newShardRouting(shardId, currentNodeId, null, true, state)
                 )
             );
-            indexRoutingTableBuilder.addReplica();
+            indexRoutingTableBuilder.addReplica(ShardRouting.Role.DEFAULT);
         }
         routingTableBuilder.add(indexRoutingTableBuilder.build());
 
@@ -208,7 +201,7 @@ public void testFindTriggeredWatchesGoodCase() {
                 TestShardRouting.newShardRouting(shardId, "_node_id", null, true, ShardRoutingState.STARTED)
             )
         );
-        indexRoutingTableBuilder.addReplica();
+        indexRoutingTableBuilder.addReplica(ShardRouting.Role.DEFAULT);
         routingTableBuilder.add(indexRoutingTableBuilder.build());
         csBuilder.metadata(metadataBuilder);
         csBuilder.routingTable(routingTableBuilder.build());
@@ -340,7 +333,7 @@ public void testLoadStoreAsAlias() {
                 TestShardRouting.newShardRouting(shardId, "_node_id", null, true, ShardRoutingState.STARTED)
             )
         );
-        indexRoutingTableBuilder.addReplica();
+        indexRoutingTableBuilder.addReplica(ShardRouting.Role.DEFAULT);
         routingTableBuilder.add(indexRoutingTableBuilder.build());
         csBuilder.metadata(metadataBuilder);
         csBuilder.routingTable(routingTableBuilder.build());
@@ -374,7 +367,7 @@ public void testLoadingFailsWithTwoAliases() {
                 TestShardRouting.newShardRouting("triggered-watches-alias", 0, "_node_id", null, true, ShardRoutingState.STARTED)
             )
         );
-        indexRoutingTableBuilder.addReplica();
+        indexRoutingTableBuilder.addReplica(ShardRouting.Role.DEFAULT);
         final Index otherIndex = metadataBuilder.get("whatever").getIndex();
         IndexRoutingTable.Builder otherIndexRoutingTableBuilder = IndexRoutingTable.builder(otherIndex);
         otherIndexRoutingTableBuilder.addIndexShard(
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/NotificationServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/NotificationServiceTests.java
index b8bfdb7e1738..9280b47bddcd 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/NotificationServiceTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/NotificationServiceTests.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.watcher.notification;
 
 import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.SecureSetting;
 import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.SecureString;
@@ -253,7 +254,7 @@ public boolean isLoaded() {
             }
 
             @Override
-            public SecureString getString(String setting) throws GeneralSecurityException {
+            public SecureString getString(String setting) {
                 return new SecureString(secureSettingsMap.get(setting));
             }
 
@@ -274,6 +275,11 @@ public byte[] getSHA256Digest(String setting) throws GeneralSecurityException {
 
             @Override
             public void close() throws IOException {}
+
+            @Override
+            public void writeTo(StreamOutput out) throws IOException {
+                throw new IllegalStateException("Not supported");
+            }
         };
     }
 }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParserTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParserTests.java
index 86733d5de874..075182af8798 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParserTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/attachment/ReportingAttachmentParserTests.java
@@ -44,7 +44,6 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -529,7 +528,7 @@ public void testDefaultWarnings() throws Exception {
         // parameterize the messages
         assertEquals(
             attachment.getWarnings(),
-            WARNINGS.values().stream().map(s -> String.format(Locale.ROOT, s, reportId)).collect(Collectors.toSet())
+            WARNINGS.values().stream().map(s -> Strings.format(s, reportId)).collect(Collectors.toSet())
         );
 
         Attachment.Bytes bytesAttachment = (Attachment.Bytes) attachment;
@@ -610,7 +609,7 @@ public void testCustomWarningsWithParams() throws Exception {
         // parameterize the messages
         assertEquals(
             attachment.getWarnings(),
-            customWarnings.values().stream().map(s -> String.format(Locale.ROOT, s, reportId)).collect(Collectors.toSet())
+            customWarnings.values().stream().map(s -> Strings.format(s, reportId)).collect(Collectors.toSet())
         );
         // ensure the reportId is parameterized in
         attachment.getWarnings().forEach(s -> { assertThat(s, containsString(reportId)); });
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/LocalStateWatcher.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/LocalStateWatcher.java
index 17657ff8e0d2..e641df2a46f0 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/LocalStateWatcher.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/LocalStateWatcher.java
@@ -8,18 +8,21 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.plugins.ReloadablePlugin;
 import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.watcher.Watcher;
 
 import java.nio.file.Path;
 
-public class LocalStateWatcher extends LocalStateCompositeXPackPlugin {
+public class LocalStateWatcher extends LocalStateCompositeXPackPlugin implements ReloadablePlugin {
+
+    private final Watcher watcher;
+
     public LocalStateWatcher(final Settings settings, final Path configPath) throws Exception {
         super(settings, configPath);
         LocalStateWatcher thisVar = this;
-
-        plugins.add(new Watcher(settings) {
+        this.watcher = new Watcher(settings) {
             @Override
             protected SSLService getSslService() {
                 return thisVar.getSslService();
@@ -30,6 +33,12 @@ protected XPackLicenseState getLicenseState() {
                 return thisVar.getLicenseState();
             }
 
-        });
+        };
+        plugins.add(watcher);
+    }
+
+    @Override
+    public void reload(Settings settings) throws Exception {
+        this.watcher.reload(settings);
     }
 }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/TimeWarpedWatcher.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/TimeWarpedWatcher.java
index 434ddfb3e5eb..c4fbce3f1eff 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/TimeWarpedWatcher.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/TimeWarpedWatcher.java
@@ -10,6 +10,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.plugins.ReloadablePlugin;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
 import org.elasticsearch.xpack.core.ssl.SSLService;
@@ -30,19 +31,19 @@
 import java.util.function.Consumer;
 import java.util.stream.Stream;
 
-public class TimeWarpedWatcher extends LocalStateCompositeXPackPlugin {
+public class TimeWarpedWatcher extends LocalStateCompositeXPackPlugin implements ReloadablePlugin {
     private static final Logger logger = LogManager.getLogger(TimeWarpedWatcher.class);
 
     // use a single clock across all nodes using this plugin, this lets keep it static
     private static final ClockMock clock = new ClockMock();
+    private final Watcher watcher;
 
     public TimeWarpedWatcher(final Settings settings, final Path configPath) throws Exception {
         super(settings, configPath);
         logger.info("using time warped watchers plugin");
 
         TimeWarpedWatcher thisVar = this;
-
-        plugins.add(new Watcher(settings) {
+        this.watcher = new Watcher(settings) {
             @Override
             protected SSLService getSslService() {
                 return thisVar.getSslService();
@@ -72,7 +73,13 @@ protected WatchExecutor getWatchExecutor(ThreadPool threadPool) {
             protected Consumer<Iterable<TriggerEvent>> getTriggerEngineListener(ExecutionService executionService) {
                 return new SyncTriggerEventConsumer(executionService);
             }
-        });
+        };
+        plugins.add(watcher);
+    }
+
+    @Override
+    public void reload(Settings settings) throws Exception {
+        this.watcher.reload(settings);
     }
 
     public static class SameThreadExecutor implements WatchExecutor {
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java
index f201289a5d1a..91124771bdaf 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.TimeValue;
@@ -43,6 +44,7 @@
 import org.elasticsearch.xpack.watcher.execution.TriggeredExecutionContext;
 import org.elasticsearch.xpack.watcher.input.simple.ExecutableSimpleInput;
 import org.elasticsearch.xpack.watcher.input.simple.SimpleInput;
+import org.elasticsearch.xpack.watcher.notification.WebhookService;
 import org.elasticsearch.xpack.watcher.notification.email.Authentication;
 import org.elasticsearch.xpack.watcher.notification.email.EmailService;
 import org.elasticsearch.xpack.watcher.notification.email.EmailTemplate;
@@ -179,13 +181,19 @@ public static Watch createTestWatch(
         httpRequest.method(HttpMethod.POST);
         httpRequest.path(new TextTemplate("/foobarbaz/{{ctx.watch_id}}"));
         httpRequest.body(new TextTemplate("{{ctx.watch_id}} executed with {{ctx.payload.response.hits.total_hits}} hits"));
+
+        WebhookService webhookService = new WebhookService(
+            Settings.EMPTY,
+            httpClient,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+        );
         actions.add(
             new ActionWrapper(
                 "_webhook",
                 actionThrottler,
                 null,
                 null,
-                new ExecutableWebhookAction(new WebhookAction(httpRequest.build()), logger, httpClient, engine),
+                new ExecutableWebhookAction(new WebhookAction(httpRequest.build()), logger, webhookService, engine),
                 null,
                 null
             )
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/QueryWatchesRequestTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/QueryWatchesRequestTests.java
index 3ce104175ba2..df838d1b5673 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/QueryWatchesRequestTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/QueryWatchesRequestTests.java
@@ -67,6 +67,11 @@ protected QueryWatchesAction.Request createTestInstance() {
         );
     }
 
+    @Override
+    protected QueryWatchesAction.Request mutateInstance(QueryWatchesAction.Request instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
         SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/QueryWatchesResponseTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/QueryWatchesResponseTests.java
index f522255129e5..f35038e1a427 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/QueryWatchesResponseTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/QueryWatchesResponseTests.java
@@ -106,6 +106,11 @@ protected QueryWatchesAction.Response createTestInstance() {
         return new QueryWatchesAction.Response(numWatches + randomIntBetween(0, 100), items);
     }
 
+    @Override
+    protected QueryWatchesAction.Response mutateInstance(QueryWatchesAction.Response instance) {
+        return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929
+    }
+
     private Watch createWatch(String watchId) {
         return WatcherTestUtils.createTestWatch(
             watchId,
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java
index 64ca51128e72..b29dfa182cb2 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java
@@ -79,6 +79,7 @@
 import org.elasticsearch.xpack.watcher.input.simple.ExecutableSimpleInput;
 import org.elasticsearch.xpack.watcher.input.simple.SimpleInput;
 import org.elasticsearch.xpack.watcher.input.simple.SimpleInputFactory;
+import org.elasticsearch.xpack.watcher.notification.WebhookService;
 import org.elasticsearch.xpack.watcher.notification.email.DataAttachment;
 import org.elasticsearch.xpack.watcher.notification.email.EmailService;
 import org.elasticsearch.xpack.watcher.notification.email.EmailTemplate;
@@ -153,6 +154,7 @@ public class WatchTests extends ESTestCase {
     private Client client;
     private HttpClient httpClient;
     private EmailService emailService;
+    private WebhookService webhookService;
     private TextTemplateEngine templateEngine;
     private HtmlSanitizer htmlSanitizer;
     private XPackLicenseState licenseState;
@@ -166,6 +168,7 @@ public void init() throws Exception {
         client = mock(Client.class);
         httpClient = mock(HttpClient.class);
         emailService = mock(EmailService.class);
+        webhookService = mock(WebhookService.class);
         templateEngine = mock(TextTemplateEngine.class);
         htmlSanitizer = mock(HtmlSanitizer.class);
         licenseState = mock(XPackLicenseState.class);
@@ -658,7 +661,7 @@ private List<ActionWrapper> randomActions() {
                     randomThrottler(),
                     AlwaysConditionTests.randomCondition(scriptService),
                     randomTransform(),
-                    new ExecutableWebhookAction(action, logger, httpClient, templateEngine),
+                    new ExecutableWebhookAction(action, logger, webhookService, templateEngine),
                     null,
                     null
                 )
@@ -676,7 +679,7 @@ private ActionRegistry registry(List<ActionWrapper> actions, ConditionRegistry c
                     new EmailActionFactory(settings, emailService, templateEngine, new EmailAttachmentsParser(Collections.emptyMap()))
                 );
                 case IndexAction.TYPE -> parsers.put(IndexAction.TYPE, new IndexActionFactory(settings, client));
-                case WebhookAction.TYPE -> parsers.put(WebhookAction.TYPE, new WebhookActionFactory(httpClient, templateEngine));
+                case WebhookAction.TYPE -> parsers.put(WebhookAction.TYPE, new WebhookActionFactory(webhookService, templateEngine));
                 case LoggingAction.TYPE -> parsers.put(LoggingAction.TYPE, new LoggingActionFactory(new MockTextTemplateEngine()));
             }
         }
diff --git a/x-pack/plugin/wildcard/build.gradle b/x-pack/plugin/wildcard/build.gradle
index 82f2d33ac004..63437cb2f132 100644
--- a/x-pack/plugin/wildcard/build.gradle
+++ b/x-pack/plugin/wildcard/build.gradle
@@ -1,7 +1,7 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 
 apply plugin: 'elasticsearch.internal-es-plugin'
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 
 esplugin {
   name 'wildcard'
diff --git a/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java b/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java
index 6d7164ec770c..6821a29654c8 100644
--- a/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java
+++ b/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.datastreams.DataStreamsPlugin;
 import org.elasticsearch.index.Index;
@@ -38,7 +39,6 @@
 
 import java.util.Collection;
 import java.util.List;
-import java.util.Locale;
 import java.util.OptionalDouble;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -200,7 +200,7 @@ static void indexDocs(String dataStream, int numDocs) {
             String value = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis());
             bulkRequest.add(
                 new IndexRequest(dataStream).opType(DocWriteRequest.OpType.CREATE)
-                    .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON)
+                    .source(Strings.format("{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON)
             );
         }
         client().bulk(bulkRequest).actionGet();
diff --git a/x-pack/qa/core-rest-tests-with-security/build.gradle b/x-pack/qa/core-rest-tests-with-security/build.gradle
index 87cb38469465..647766a3a585 100644
--- a/x-pack/qa/core-rest-tests-with-security/build.gradle
+++ b/x-pack/qa/core-rest-tests-with-security/build.gradle
@@ -1,11 +1,12 @@
-import org.elasticsearch.gradle.Version
-import org.elasticsearch.gradle.internal.info.BuildParams
-
 apply plugin: 'elasticsearch.internal-yaml-rest-test'
-apply plugin: 'elasticsearch.authenticated-testclusters'
 
 dependencies {
   testImplementation project(':x-pack:qa')
+  clusterModules project(':modules:mapper-extras')
+  clusterModules project(':modules:rank-eval')
+  clusterModules project(xpackModule('stack'))
+  clusterModules project(xpackModule('ilm'))
+  clusterModules project(xpackModule('mapper-constant-keyword'))
 }
 
 restResources {
@@ -21,13 +22,3 @@ tasks.named("yamlRestTest").configure {
         'indices.get_alias/10_basic/Get alias against closed indices'
       ].join(',')
 }
-
-testClusters.matching { it.name == "yamlRestTest" }.configureEach {
-  testDistribution = 'DEFAULT'
-  setting 'xpack.watcher.enabled', 'false'
-  setting 'xpack.ml.enabled', 'false'
-  setting 'xpack.license.self_generated.type', 'trial'
-  setting 'indices.lifecycle.history_index_enabled', 'false'
-  setting 'xpack.security.autoconfiguration.enabled', 'false'
-  requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
-}
diff --git a/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java b/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java
index c6ea16bb278e..81533552765d 100644
--- a/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java
+++ b/x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java
@@ -15,16 +15,34 @@
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.ClassRule;
 
 import java.util.Objects;
 
 @TimeoutSuite(millis = 30 * TimeUnits.MINUTE) // as default timeout seems not enough on the jenkins VMs
 public class CoreWithSecurityClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+    private static final String USER = Objects.requireNonNull(System.getProperty("tests.rest.cluster.username", "test_admin"));
+    private static final String PASS = Objects.requireNonNull(System.getProperty("tests.rest.cluster.password", "x-pack-test-password"));
 
-    private static final String USER = Objects.requireNonNull(System.getProperty("tests.rest.cluster.username"));
-    private static final String PASS = Objects.requireNonNull(System.getProperty("tests.rest.cluster.password"));
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .module("constant-keyword")
+        .module("mapper-extras")
+        .module("rank-eval")
+        .module("x-pack-ilm")
+        .module("x-pack-stack")
+        .setting("xpack.security.enabled", "true")
+        .setting("xpack.watcher.enabled", "false")
+        .setting("xpack.ml.enabled", "false")
+        .setting("xpack.license.self_generated.type", "trial")
+        .setting("xpack.security.autoconfiguration.enabled", "false")
+        .user(USER, PASS)
+        .feature(FeatureFlag.TIME_SERIES_MODE)
+        .build();
 
     public CoreWithSecurityClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
@@ -40,4 +58,9 @@ protected Settings restClientSettings() {
         String token = basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray()));
         return Settings.builder().put(super.restClientSettings()).put(ThreadContext.PREFIX + ".Authorization", token).build();
     }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
 }
diff --git a/x-pack/qa/full-cluster-restart/build.gradle b/x-pack/qa/full-cluster-restart/build.gradle
index 3923d439d394..d9539bb668b4 100644
--- a/x-pack/qa/full-cluster-restart/build.gradle
+++ b/x-pack/qa/full-cluster-restart/build.gradle
@@ -1,110 +1,21 @@
-import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
-apply plugin: 'elasticsearch.internal-testclusters'
-apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.bwc-test'
 
 dependencies {
   // TODO: Remove core dependency and change tests to not use builders that are part of xpack-core.
   // Currently needed for MlConfigIndexMappingsFullClusterRestartIT and SLM classes used in
   // FullClusterRestartIT
-  testImplementation(testArtifact(project(xpackModule('core'))))
-  testImplementation(testArtifact(project(":qa:full-cluster-restart")))
-  testImplementation project(':x-pack:qa')
-}
-
-tasks.named("forbiddenPatterns") {
-  exclude '**/system_key'
-}
-
-String outputDir = "${buildDir}/generated-resources/${project.name}"
-
-tasks.register("copyTestNodeKeyMaterial", Copy) {
-  from project(':x-pack:plugin:core')
-    .files(
-      'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem',
-      'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt'
-    )
-  into outputDir
+  javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
+  javaRestTestImplementation(testArtifact(project(":qa:full-cluster-restart")))
+  javaRestTestImplementation project(':x-pack:qa')
 }
 
 BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName ->
-    def baseCluster = testClusters.register(baseName) {
-        testDistribution = "DEFAULT"
-        if (bwcVersion.before(BuildParams.bwcVersions.minimumWireCompatibleVersion)) {
-          // When testing older versions we have to first upgrade to 7.last
-          versions = [bwcVersion.toString(), BuildParams.bwcVersions.minimumWireCompatibleVersion.toString(), project.version]
-        } else {
-          versions = [bwcVersion.toString(), project.version]
-        }
-        numberOfNodes = 2
-        setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
-        user username: "test_user", password: "x-pack-test-password"
-
-        setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
-        // some tests rely on the translog not being flushed
-        setting 'indices.memory.shard_inactive_time', '60m'
-        setting 'xpack.security.enabled', 'true'
-        setting 'xpack.security.transport.ssl.enabled', 'true'
-        setting 'xpack.license.self_generated.type', 'trial'
-
-        extraConfigFile 'testnode.pem', file("${outputDir}/testnode.pem")
-        extraConfigFile 'testnode.crt', file("${outputDir}/testnode.crt")
-
-        keystore 'xpack.watcher.encryption_key', file("${project.projectDir}/src/test/resources/system_key")
-        setting 'xpack.watcher.encrypt_sensitive_data', 'true'
-
-        setting 'xpack.security.transport.ssl.key', 'testnode.pem'
-        setting 'xpack.security.transport.ssl.certificate', 'testnode.crt'
-        keystore 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
-
-        setting 'xpack.security.authc.api_key.enabled', 'true'
-
-        requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
-    }
-
-    tasks.register("${baseName}#oldClusterTest", StandaloneRestIntegTestTask) {
-        mustRunAfter("precommit")
-        useCluster baseCluster
-        dependsOn "copyTestNodeKeyMaterial"
-        doFirst {
-            delete("${buildDir}/cluster/shared/repo/${baseName}")
-        }
-        systemProperty 'tests.is_old_cluster', 'true'
-        exclude 'org/elasticsearch/upgrades/FullClusterRestartIT.class'
-        exclude 'org/elasticsearch/upgrades/FullClusterRestartSettingsUpgradeIT.class'
-        exclude 'org/elasticsearch/upgrades/QueryBuilderBWCIT.class'
-    }
-
-    tasks.register("${baseName}#upgradedClusterTest", StandaloneRestIntegTestTask) {
-        mustRunAfter("precommit")
-        useCluster baseCluster
-        dependsOn "${baseName}#oldClusterTest"
-        doFirst {
-            baseCluster.get().goToNextVersion()
-            if (bwcVersion.before(BuildParams.bwcVersions.minimumWireCompatibleVersion)) {
-              // When doing a full cluster restart of older versions we actually have to upgrade twice. First to 7.last, then to the current version.
-              baseCluster.get().goToNextVersion()
-            }
-        }
-        systemProperty 'tests.is_old_cluster', 'false'
-        exclude 'org/elasticsearch/upgrades/FullClusterRestartIT.class'
-        exclude 'org/elasticsearch/upgrades/FullClusterRestartSettingsUpgradeIT.class'
-        exclude 'org/elasticsearch/upgrades/QueryBuilderBWCIT.class'
-    }
-
-    String oldVersion = bwcVersion.toString().minus("-SNAPSHOT")
-    tasks.matching { it.name.startsWith("${baseName}#") && it.name.endsWith("ClusterTest") }.configureEach {
-        it.systemProperty 'tests.old_cluster_version', oldVersion
-        it.systemProperty 'tests.path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
-        it.nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
-        it.nonInputProperties.systemProperty('tests.clustername', baseName)
-    }
-
-    tasks.register(bwcTaskName(bwcVersion)) {
-        dependsOn "${baseName}#upgradedClusterTest"
-    }
-
+  tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) {
+    usesBwcDistribution(bwcVersion)
+    systemProperty("tests.old_cluster_version", bwcVersion)
+  }
 }
diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/AbstractXpackFullClusterRestartTestCase.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/AbstractXpackFullClusterRestartTestCase.java
new file mode 100644
index 000000000000..0bc9101301a5
--- /dev/null
+++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/AbstractXpackFullClusterRestartTestCase.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.restart;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.FeatureFlag;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.cluster.util.resource.Resource;
+import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus;
+import org.elasticsearch.upgrades.ParameterizedFullClusterRestartTestCase;
+import org.junit.ClassRule;
+
+public abstract class AbstractXpackFullClusterRestartTestCase extends ParameterizedFullClusterRestartTestCase {
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .version(getOldClusterTestVersion())
+        .nodes(2)
+        // some tests rely on the translog not being flushed
+        .setting("indices.memory.shard_inactive_time", "60m")
+        .setting("xpack.security.enabled", "true")
+        .setting("xpack.security.transport.ssl.enabled", "true")
+        .setting("xpack.security.transport.ssl.key", "testnode.pem")
+        .setting("xpack.security.transport.ssl.certificate", "testnode.crt")
+        .setting("xpack.license.self_generated.type", "trial")
+        .setting("xpack.watcher.encrypt_sensitive_data", "true")
+        .setting("xpack.security.authc.api_key.enabled", "true")
+        .configFile("testnode.pem", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem"))
+        .configFile("testnode.crt", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"))
+        .keystore("xpack.watcher.encryption_key", Resource.fromClasspath("system_key"))
+        .keystore("xpack.security.transport.ssl.secure_key_passphrase", "testnode")
+        .feature(FeatureFlag.TIME_SERIES_MODE)
+        .build();
+
+    public AbstractXpackFullClusterRestartTestCase(FullClusterRestartUpgradeStatus upgradeStatus) {
+        super(upgradeStatus);
+    }
+
+    @Override
+    protected ElasticsearchCluster getUpgradeCluster() {
+        return cluster;
+    }
+}
diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/CoreFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/CoreFullClusterRestartIT.java
new file mode 100644
index 000000000000..dcdc12707963
--- /dev/null
+++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/CoreFullClusterRestartIT.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.restart;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.cluster.util.resource.Resource;
+import org.elasticsearch.upgrades.FullClusterRestartIT;
+import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+public class CoreFullClusterRestartIT extends FullClusterRestartIT {
+
+    static {
+        clusterConfig = c -> c.setting("xpack.security.enabled", "true")
+            .setting("xpack.security.transport.ssl.enabled", "true")
+            .setting("xpack.security.transport.ssl.key", "testnode.pem")
+            .setting("xpack.security.transport.ssl.certificate", "testnode.crt")
+            .setting("xpack.license.self_generated.type", "trial")
+            .setting("xpack.watcher.encrypt_sensitive_data", "true")
+            .setting("xpack.security.authc.api_key.enabled", "true")
+            .configFile("testnode.pem", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem"))
+            .configFile("testnode.crt", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"))
+            .keystore("xpack.watcher.encryption_key", Resource.fromClasspath("system_key"))
+            .keystore("xpack.security.transport.ssl.secure_key_passphrase", "testnode");
+    }
+
+    public CoreFullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) {
+        super(upgradeStatus);
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8));
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+
+}
diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
new file mode 100644
index 000000000000..ab48825ed983
--- /dev/null
+++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
@@ -0,0 +1,997 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.restart;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.search.RestSearchAction;
+import org.elasticsearch.test.StreamsUtils;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus;
+import org.elasticsearch.xcontent.ObjectPath;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xcontent.json.JsonXContent;
+import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
+import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicy;
+import org.elasticsearch.xpack.core.slm.SnapshotLifecycleStats;
+import org.hamcrest.Matcher;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.core.TimeValue.timeValueSeconds;
+import static org.elasticsearch.upgrades.FullClusterRestartIT.assertNumHits;
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.everyItem;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasItems;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.startsWith;
+
+public class FullClusterRestartIT extends AbstractXpackFullClusterRestartTestCase {
+
+    public static final int UPGRADE_FIELD_EXPECTED_INDEX_FORMAT_VERSION = 6;
+    public static final int SECURITY_EXPECTED_INDEX_FORMAT_VERSION = 6;
+
+    public FullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) {
+        super(upgradeStatus);
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            // we increase the timeout here to 90 seconds to handle long waits for a green
+            // cluster health. the waits for green need to be longer than a minute to
+            // account for delayed shards
+            .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s")
+            .build();
+    }
+
+    /**
+     * Tests that a single document survives. Super basic smoke test.
+     */
+    public void testSingleDoc() throws IOException {
+        String docLocation = "/testsingledoc/_doc/1";
+        String doc = "{\"test\": \"test\"}";
+
+        if (isRunningAgainstOldCluster()) {
+            Request createDoc = new Request("PUT", docLocation);
+            createDoc.addParameter("refresh", "true");
+            createDoc.setJsonEntity(doc);
+            client().performRequest(createDoc);
+        }
+
+        Request getRequest = new Request("GET", docLocation);
+        assertThat(toStr(client().performRequest(getRequest)), containsString(doc));
+    }
+
+    public void testSecurityNativeRealm() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+            createUser(true);
+            createRole(true);
+        } else {
+            waitForYellow(".security");
+            final Request getSettingsRequest = new Request("GET", "/.security/_settings/index.format");
+            getSettingsRequest.setOptions(systemIndexWarningHandlerOptions(".security-7"));
+            Response settingsResponse = client().performRequest(getSettingsRequest);
+            Map<String, Object> settingsResponseMap = entityAsMap(settingsResponse);
+            logger.info("settings response map {}", settingsResponseMap);
+            final String concreteSecurityIndex;
+            if (settingsResponseMap.isEmpty()) {
+                fail("The security index does not have the expected setting [index.format]");
+            } else {
+                concreteSecurityIndex = settingsResponseMap.keySet().iterator().next();
+                Map<?, ?> indexSettingsMap = (Map<?, ?>) settingsResponseMap.get(concreteSecurityIndex);
+                Map<?, ?> settingsMap = (Map<?, ?>) indexSettingsMap.get("settings");
+                logger.info("settings map {}", settingsMap);
+                if (settingsMap.containsKey("index")) {
+                    int format = Integer.parseInt(String.valueOf(((Map<?, ?>) settingsMap.get("index")).get("format")));
+                    assertEquals("The security index needs to be upgraded", SECURITY_EXPECTED_INDEX_FORMAT_VERSION, format);
+                }
+            }
+
+            // create additional user and role
+            createUser(false);
+            createRole(false);
+        }
+
+        assertUserInfo(isRunningAgainstOldCluster());
+        assertRoleInfo(isRunningAgainstOldCluster());
+    }
+
+    public void testWatcher() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+            logger.info("Adding a watch on old cluster {}", getOldClusterVersion());
+            Request createBwcWatch = new Request("PUT", "/_watcher/watch/bwc_watch");
+            createBwcWatch.setJsonEntity(loadWatch("simple-watch.json"));
+            client().performRequest(createBwcWatch);
+
+            logger.info("Adding a watch with \"fun\" throttle periods on old cluster");
+            Request createBwcThrottlePeriod = new Request("PUT", "/_watcher/watch/bwc_throttle_period");
+            createBwcThrottlePeriod.setJsonEntity(loadWatch("throttle-period-watch.json"));
+            client().performRequest(createBwcThrottlePeriod);
+
+            logger.info("Adding a watch with \"fun\" read timeout on old cluster");
+            Request createFunnyTimeout = new Request("PUT", "/_watcher/watch/bwc_funny_timeout");
+            createFunnyTimeout.setJsonEntity(loadWatch("funny-timeout-watch.json"));
+            client().performRequest(createFunnyTimeout);
+
+            logger.info("Waiting for watch results index to fill up...");
+            try {
+                waitForYellow(".watches,bwc_watch_index,.watcher-history*");
+            } catch (ResponseException e) {
+                {
+                    String rsp = toStr(client().performRequest(new Request("GET", "/_cluster/state")));
+                    logger.info("cluster_state_response=\n{}", rsp);
+                }
+                {
+                    Request request = new Request("GET", "/_watcher/stats/_all");
+                    request.addParameter("emit_stacktraces", "true");
+                    String rsp = toStr(client().performRequest(request));
+                    logger.info("watcher_stats_response=\n{}", rsp);
+                }
+                throw e;
+            }
+            waitForHits("bwc_watch_index", 2);
+            waitForHits(".watcher-history*", 2);
+            logger.info("Done creating watcher-related indices");
+        } else {
+            logger.info("testing against {}", getOldClusterVersion());
+            try {
+                waitForYellow(".watches,bwc_watch_index,.watcher-history*");
+            } catch (ResponseException e) {
+                String rsp = toStr(client().performRequest(new Request("GET", "/_cluster/state")));
+                logger.info("cluster_state_response=\n{}", rsp);
+                throw e;
+            }
+
+            logger.info("checking that the Watches index is the correct version");
+
+            // Verify .watches index format:
+            var getClusterStateResponse = entityAsMap(client().performRequest(new Request("GET", "/_cluster/state/metadata/.watches")));
+            Map<?, ?> indices = ObjectPath.eval("metadata.indices", getClusterStateResponse);
+            var dotWatchesIndex = indices.get(".watches"); // ObjectPath.eval(...) doesn't handle keys containing .
+            var indexFormat = Integer.parseInt(ObjectPath.eval("settings.index.format", dotWatchesIndex));
+            assertEquals("The watches index needs to be upgraded", UPGRADE_FIELD_EXPECTED_INDEX_FORMAT_VERSION, indexFormat);
+
+            // Wait for watcher to actually start....
+            startWatcher();
+
+            try {
+                assertOldTemplatesAreDeleted();
+                assertWatchIndexContentsWork();
+                assertBasicWatchInteractions();
+            } finally {
+                /* Shut down watcher after every test because watcher can be a bit finicky about shutting down when the node shuts
+                 * down. This makes super sure it shuts down *and* causes the test to fail in a sensible spot if it doesn't shut down.
+                 */
+                stopWatcher();
+            }
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testWatcherWithApiKey() throws Exception {
+        final Request getWatchStatusRequest = new Request("GET", "/_watcher/watch/watch_with_api_key");
+
+        if (isRunningAgainstOldCluster()) {
+            final Request createApiKeyRequest = new Request("PUT", "/_security/api_key");
+            createApiKeyRequest.setJsonEntity("""
+                {
+                   "name": "key-1",
+                   "role_descriptors": {
+                     "r": {
+                       "cluster": [ "all" ],
+                       "indices": [
+                         {
+                           "names": [ "*" ],
+                           "privileges": [ "all" ]
+                         }
+                       ]
+                     }
+                   }
+                 }""");
+            final Response response = client().performRequest(createApiKeyRequest);
+            final Map<String, Object> createApiKeyResponse = entityAsMap(response);
+
+            Request createWatchWithApiKeyRequest = new Request("PUT", "/_watcher/watch/watch_with_api_key");
+            createWatchWithApiKeyRequest.setJsonEntity(loadWatch("logging-watch.json"));
+            final byte[] keyBytes = (createApiKeyResponse.get("id") + ":" + createApiKeyResponse.get("api_key")).getBytes(
+                StandardCharsets.UTF_8
+            );
+            final String authHeader = "ApiKey " + Base64.getEncoder().encodeToString(keyBytes);
+            createWatchWithApiKeyRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", authHeader));
+            client().performRequest(createWatchWithApiKeyRequest);
+
+            assertBusy(() -> {
+                final Map<String, Object> getWatchStatusResponse = entityAsMap(client().performRequest(getWatchStatusRequest));
+                final Map<String, Object> status = (Map<String, Object>) getWatchStatusResponse.get("status");
+                assertEquals("executed", status.get("execution_state"));
+            }, 30, TimeUnit.SECONDS);
+
+        } else {
+            logger.info("testing against {}", getOldClusterVersion());
+            try {
+                waitForYellow(".watches,.watcher-history*");
+            } catch (ResponseException e) {
+                String rsp = toStr(client().performRequest(new Request("GET", "/_cluster/state")));
+                logger.info("cluster_state_response=\n{}", rsp);
+                throw e;
+            }
+
+            // Wait for watcher to actually start....
+            startWatcher();
+
+            try {
+                final Map<String, Object> getWatchStatusResponse = entityAsMap(client().performRequest(getWatchStatusRequest));
+                final Map<String, Object> status = (Map<String, Object>) getWatchStatusResponse.get("status");
+                final int version = (int) status.get("version");
+
+                final AtomicBoolean versionIncreased = new AtomicBoolean();
+                final AtomicBoolean executed = new AtomicBoolean();
+                assertBusy(() -> {
+                    final Map<String, Object> newGetWatchStatusResponse = entityAsMap(client().performRequest(getWatchStatusRequest));
+                    final Map<String, Object> newStatus = (Map<String, Object>) newGetWatchStatusResponse.get("status");
+                    if (false == versionIncreased.get() && version < (int) newStatus.get("version")) {
+                        versionIncreased.set(true);
+                    }
+                    if (false == executed.get() && "executed".equals(newStatus.get("execution_state"))) {
+                        executed.set(true);
+                    }
+                    assertThat(
+                        "version increased: [" + versionIncreased.get() + "], executed: [" + executed.get() + "]",
+                        versionIncreased.get() && executed.get(),
+                        is(true)
+                    );
+                }, 30, TimeUnit.SECONDS);
+            } finally {
+                stopWatcher();
+            }
+        }
+    }
+
+    public void testServiceAccountApiKey() throws IOException {
+        assumeTrue("no service accounts in versions before " + Version.V_7_13_0, getOldClusterVersion().onOrAfter(Version.V_7_13_0));
+        if (isRunningAgainstOldCluster()) {
+            final Request createServiceTokenRequest = new Request("POST", "/_security/service/elastic/fleet-server/credential/token");
+            final Response createServiceTokenResponse = client().performRequest(createServiceTokenRequest);
+            assertOK(createServiceTokenResponse);
+            @SuppressWarnings("unchecked")
+            final String serviceToken = ((Map<String, String>) responseAsMap(createServiceTokenResponse).get("token")).get("value");
+            final Request createApiKeyRequest = new Request("PUT", "/_security/api_key");
+            createApiKeyRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", "Bearer " + serviceToken));
+            createApiKeyRequest.setJsonEntity("{\"name\":\"key-1\"}");
+            final Response createApiKeyResponse = client().performRequest(createApiKeyRequest);
+            final Map<String, Object> createApiKeyResponseMap = entityAsMap(createApiKeyResponse);
+            final String authHeader = "ApiKey "
+                + Base64.getEncoder()
+                    .encodeToString(
+                        (createApiKeyResponseMap.get("id") + ":" + createApiKeyResponseMap.get("api_key")).getBytes(StandardCharsets.UTF_8)
+                    );
+
+            final Request indexRequest = new Request("PUT", "/api_keys/_doc/key-1");
+            indexRequest.setJsonEntity("{\"auth_header\":\"" + authHeader + "\"}");
+            assertOK(client().performRequest(indexRequest));
+        } else {
+            final Request getRequest = new Request("GET", "/api_keys/_doc/key-1");
+            final Response getResponse = client().performRequest(getRequest);
+            assertOK(getResponse);
+            final Map<String, Object> getResponseMap = responseAsMap(getResponse);
+            @SuppressWarnings("unchecked")
+            final String authHeader = ((Map<String, String>) getResponseMap.get("_source")).get("auth_header");
+
+            final Request mainRequest = new Request("GET", "/");
+            mainRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", authHeader));
+            assertOK(client().performRequest(mainRequest));
+
+            final Request getUserRequest = new Request("GET", "/_security/user");
+            getUserRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", authHeader));
+            final ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(getUserRequest));
+            assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(403));
+            assertThat(e.getMessage(), containsString("is unauthorized"));
+        }
+    }
+
+    public void testApiKeySuperuser() throws IOException {
+        if (isRunningAgainstOldCluster()) {
+            final Request createUserRequest = new Request("PUT", "/_security/user/api_key_super_creator");
+            createUserRequest.setJsonEntity("""
+                {
+                   "password" : "l0ng-r4nd0m-p@ssw0rd",
+                   "roles" : [ "superuser", "monitoring_user" ]
+                }""");
+            client().performRequest(createUserRequest);
+
+            // Create API key
+            final Request createApiKeyRequest = new Request("PUT", "/_security/api_key");
+            createApiKeyRequest.setOptions(
+                RequestOptions.DEFAULT.toBuilder()
+                    .addHeader(
+                        "Authorization",
+                        UsernamePasswordToken.basicAuthHeaderValue(
+                            "api_key_super_creator",
+                            new SecureString("l0ng-r4nd0m-p@ssw0rd".toCharArray())
+                        )
+                    )
+            );
+            if (getOldClusterVersion().onOrAfter(Version.V_7_3_0)) {
+                createApiKeyRequest.setJsonEntity("""
+                    {
+                       "name": "super_legacy_key"
+                    }""");
+            } else {
+                createApiKeyRequest.setJsonEntity("""
+                    {
+                       "name": "super_legacy_key",
+                       "role_descriptors": {
+                         "super": {
+                           "cluster": [ "all" ],
+                           "indices": [
+                             {
+                               "names": [ "*" ],
+                               "privileges": [ "all" ],
+                               "allow_restricted_indices": true
+                             }
+                           ]
+                         }
+                       }
+                    }""");
+            }
+            final Map<String, Object> createApiKeyResponse = entityAsMap(client().performRequest(createApiKeyRequest));
+            final byte[] keyBytes = (createApiKeyResponse.get("id") + ":" + createApiKeyResponse.get("api_key")).getBytes(
+                StandardCharsets.UTF_8
+            );
+            final String apiKeyAuthHeader = "ApiKey " + Base64.getEncoder().encodeToString(keyBytes);
+            // Save the API key info across restart
+            final Request saveApiKeyRequest = new Request("PUT", "/api_keys/_doc/super_legacy_key");
+            saveApiKeyRequest.setJsonEntity("{\"auth_header\":\"" + apiKeyAuthHeader + "\"}");
+            assertOK(client().performRequest(saveApiKeyRequest));
+
+            if (getOldClusterVersion().before(Version.V_8_0_0)) {
+                final Request indexRequest = new Request("POST", ".security/_doc");
+                indexRequest.setJsonEntity("""
+                    {
+                      "doc_type": "foo"
+                    }""");
+                if (getOldClusterVersion().onOrAfter(Version.V_7_10_0)) {
+                    indexRequest.setOptions(systemIndexWarningHandlerOptions(".security-7").addHeader("Authorization", apiKeyAuthHeader));
+                } else {
+                    indexRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", apiKeyAuthHeader));
+                }
+                assertOK(client().performRequest(indexRequest));
+            }
+        } else {
+            final Request getRequest = new Request("GET", "/api_keys/_doc/super_legacy_key");
+            final Map<String, Object> getResponseMap = responseAsMap(client().performRequest(getRequest));
+            @SuppressWarnings("unchecked")
+            final String apiKeyAuthHeader = ((Map<String, String>) getResponseMap.get("_source")).get("auth_header");
+
+            // read is ok
+            final Request searchRequest = new Request("GET", ".security/_search");
+            searchRequest.setOptions(systemIndexWarningHandlerOptions(".security-7").addHeader("Authorization", apiKeyAuthHeader));
+            assertOK(client().performRequest(searchRequest));
+
+            // write must not be allowed
+            final Request indexRequest = new Request("POST", ".security/_doc");
+            indexRequest.setJsonEntity("""
+                {
+                  "doc_type": "foo"
+                }""");
+            indexRequest.setOptions(systemIndexWarningHandlerOptions(".security-7").addHeader("Authorization", apiKeyAuthHeader));
+            final ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(indexRequest));
+            assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(403));
+            assertThat(e.getMessage(), containsString("is unauthorized"));
+        }
+    }
+
+    /**
+     * Tests that a RollUp job created on a old cluster is correctly restarted after the upgrade.
+     */
+    public void testRollupAfterRestart() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+            final int numDocs = 59;
+            final int year = randomIntBetween(1970, 2018);
+
+            // index documents for the rollup job
+            final StringBuilder bulk = new StringBuilder();
+            for (int i = 0; i < numDocs; i++) {
+                bulk.append("{\"index\":{\"_index\":\"rollup-docs\"}}\n");
+                String date = Strings.format("%04d-01-01T00:%02d:00Z", year, i);
+                bulk.append("{\"timestamp\":\"").append(date).append("\",\"value\":").append(i).append("}\n");
+            }
+            bulk.append("\r\n");
+
+            final Request bulkRequest = new Request("POST", "/_bulk");
+            bulkRequest.setJsonEntity(bulk.toString());
+            client().performRequest(bulkRequest);
+
+            // create the rollup job
+            final Request createRollupJobRequest = new Request("PUT", "/_rollup/job/rollup-job-test");
+
+            String intervalType;
+            if (getOldClusterVersion().onOrAfter(Version.V_7_2_0)) {
+                intervalType = "fixed_interval";
+            } else {
+                intervalType = "interval";
+            }
+
+            createRollupJobRequest.setJsonEntity(Strings.format("""
+                {
+                  "index_pattern": "rollup-*",
+                  "rollup_index": "results-rollup",
+                  "cron": "*/30 * * * * ?",
+                  "page_size": 100,
+                  "groups": {
+                    "date_histogram": {
+                      "field": "timestamp",
+                      "%s": "5m"
+                    }
+                  },
+                  "metrics": [
+                    {
+                      "field": "value",
+                      "metrics": [ "min", "max", "sum" ]
+                    }
+                  ]
+                }""", intervalType));
+
+            Map<String, Object> createRollupJobResponse = entityAsMap(client().performRequest(createRollupJobRequest));
+            assertThat(createRollupJobResponse.get("acknowledged"), equalTo(Boolean.TRUE));
+
+            // start the rollup job
+            final Request startRollupJobRequest = new Request("POST", "/_rollup/job/rollup-job-test/_start");
+            Map<String, Object> startRollupJobResponse = entityAsMap(client().performRequest(startRollupJobRequest));
+            assertThat(startRollupJobResponse.get("started"), equalTo(Boolean.TRUE));
+
+            assertRollUpJob("rollup-job-test");
+
+        } else {
+
+            final Request clusterHealthRequest = new Request("GET", "/_cluster/health");
+            clusterHealthRequest.addParameter("wait_for_status", "yellow");
+            clusterHealthRequest.addParameter("wait_for_no_relocating_shards", "true");
+            clusterHealthRequest.addParameter("wait_for_no_initializing_shards", "true");
+            Map<String, Object> clusterHealthResponse = entityAsMap(client().performRequest(clusterHealthRequest));
+            assertThat(clusterHealthResponse.get("timed_out"), equalTo(Boolean.FALSE));
+
+            assertRollUpJob("rollup-job-test");
+        }
+    }
+
+    public void testTransformLegacyTemplateCleanup() throws Exception {
+        assumeTrue("Before 7.2 transforms didn't exist", getOldClusterVersion().onOrAfter(Version.V_7_2_0));
+        if (isRunningAgainstOldCluster()) {
+
+            // create the source index
+            final Request createIndexRequest = new Request("PUT", "customers");
+            createIndexRequest.setJsonEntity("""
+                {
+                  "mappings": {
+                    "properties": {
+                      "customer_id": {
+                        "type": "keyword"
+                      },
+                      "price": {
+                        "type": "double"
+                      }
+                    }
+                  }
+                }""");
+
+            Map<String, Object> createIndexResponse = entityAsMap(client().performRequest(createIndexRequest));
+            assertThat(createIndexResponse.get("acknowledged"), equalTo(Boolean.TRUE));
+
+            // create a transform
+            String endpoint = getOldClusterVersion().onOrAfter(Version.V_7_5_0)
+                ? "_transform/transform-full-cluster-restart-test"
+                : "_data_frame/transforms/transform-full-cluster-restart-test";
+            final Request createTransformRequest = new Request("PUT", endpoint);
+
+            createTransformRequest.setJsonEntity("""
+                {
+                  "source": {
+                    "index": "customers"
+                  },
+                  "description": "testing",
+                  "dest": {
+                    "index": "max_price"
+                  },
+                  "pivot": {
+                    "group_by": {
+                      "customer_id": {
+                        "terms": {
+                          "field": "customer_id"
+                        }
+                      }
+                    },
+                    "aggregations": {
+                      "max_price": {
+                        "max": {
+                          "field": "price"
+                        }
+                      }
+                    }
+                  }
+                }""");
+
+            Map<String, Object> createTransformResponse = entityAsMap(client().performRequest(createTransformRequest));
+            assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
+        } else {
+            // legacy index templates created in previous releases should not be present anymore
+            assertBusy(() -> {
+                Request request = new Request("GET", "/_template/.transform-*,.data-frame-*");
+                try {
+                    Response response = client().performRequest(request);
+                    Map<String, Object> responseLevel = entityAsMap(response);
+                    assertNotNull(responseLevel);
+                    assertThat(responseLevel.keySet(), empty());
+                } catch (ResponseException e) {
+                    // not found is fine
+                    assertThat(
+                        "Unexpected failure getting templates: " + e.getResponse().getStatusLine(),
+                        e.getResponse().getStatusLine().getStatusCode(),
+                        is(404)
+                    );
+                }
+            });
+        }
+    }
+
+    public void testSlmPolicyAndStats() throws IOException {
+        SnapshotLifecyclePolicy slmPolicy = new SnapshotLifecyclePolicy(
+            "test-policy",
+            "test-policy",
+            "* * * 31 FEB ? *",
+            "test-repo",
+            Collections.singletonMap("indices", Collections.singletonList("*")),
+            null
+        );
+        if (isRunningAgainstOldCluster() && getOldClusterVersion().onOrAfter(Version.V_7_4_0)) {
+            Request createRepoRequest = new Request("PUT", "_snapshot/test-repo");
+            String repoCreateJson = "{" + " \"type\": \"fs\"," + " \"settings\": {" + "   \"location\": \"test-repo\"" + "  }" + "}";
+            createRepoRequest.setJsonEntity(repoCreateJson);
+            Request createSlmPolicyRequest = new Request("PUT", "_slm/policy/test-policy");
+            try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+                String createSlmPolicyJson = Strings.toString(slmPolicy.toXContent(builder, null));
+                createSlmPolicyRequest.setJsonEntity(createSlmPolicyJson);
+            }
+
+            client().performRequest(createRepoRequest);
+            client().performRequest(createSlmPolicyRequest);
+        }
+
+        if (isRunningAgainstOldCluster() == false && getOldClusterVersion().onOrAfter(Version.V_7_4_0)) {
+            Request getSlmPolicyRequest = new Request("GET", "_slm/policy/test-policy");
+            Response response = client().performRequest(getSlmPolicyRequest);
+            Map<String, Object> responseMap = entityAsMap(response);
+            Map<?, ?> policy = (Map<?, ?>) ((Map<?, ?>) responseMap.get("test-policy")).get("policy");
+            assertEquals(slmPolicy.getName(), policy.get("name"));
+            assertEquals(slmPolicy.getRepository(), policy.get("repository"));
+            assertEquals(slmPolicy.getSchedule(), policy.get("schedule"));
+            assertEquals(slmPolicy.getConfig(), policy.get("config"));
+        }
+
+        if (isRunningAgainstOldCluster() == false) {
+            Response response = client().performRequest(new Request("GET", "_slm/stats"));
+            XContentType xContentType = XContentType.fromMediaType(response.getEntity().getContentType().getValue());
+            try (
+                XContentParser parser = xContentType.xContent()
+                    .createParser(XContentParserConfiguration.EMPTY, response.getEntity().getContent())
+            ) {
+                assertEquals(new SnapshotLifecycleStats(), SnapshotLifecycleStats.parse(parser));
+            }
+        }
+    }
+
+    private String loadWatch(String watch) throws IOException {
+        return StreamsUtils.copyToStringFromClasspath("/org/elasticsearch/xpack/restart/" + watch);
+    }
+
+    private void assertOldTemplatesAreDeleted() throws IOException {
+        Map<String, Object> templates = entityAsMap(client().performRequest(new Request("GET", "/_template")));
+        assertThat(templates.keySet(), not(hasItems(is("watches"), startsWith("watch-history"), is("triggered_watches"))));
+    }
+
+    private void assertWatchIndexContentsWork() throws Exception {
+        // Fetch a basic watch
+        Request getRequest = new Request("GET", "_watcher/watch/bwc_watch");
+
+        Map<String, Object> bwcWatch = entityAsMap(client().performRequest(getRequest));
+
+        logger.error("-----> {}", bwcWatch);
+
+        assertThat(bwcWatch.get("found"), equalTo(true));
+        Map<?, ?> source = (Map<?, ?>) bwcWatch.get("watch");
+        assertEquals(1000, source.get("throttle_period_in_millis"));
+        int timeout = (int) timeValueSeconds(100).millis();
+        assertThat(ObjectPath.eval("input.search.timeout_in_millis", source), equalTo(timeout));
+        assertThat(ObjectPath.eval("actions.index_payload.transform.search.timeout_in_millis", source), equalTo(timeout));
+        assertThat(ObjectPath.eval("actions.index_payload.index.index", source), equalTo("bwc_watch_index"));
+        assertThat(ObjectPath.eval("actions.index_payload.index.timeout_in_millis", source), equalTo(timeout));
+
+        // Fetch a watch with "fun" throttle periods
+        getRequest = new Request("GET", "_watcher/watch/bwc_throttle_period");
+
+        bwcWatch = entityAsMap(client().performRequest(getRequest));
+        assertThat(bwcWatch.get("found"), equalTo(true));
+        source = (Map<?, ?>) bwcWatch.get("watch");
+        assertEquals(timeout, source.get("throttle_period_in_millis"));
+        assertThat(ObjectPath.eval("actions.index_payload.throttle_period_in_millis", source), equalTo(timeout));
+
+        /*
+         * Fetch a watch with a funny timeout to verify loading fractional time
+         * values.
+         */
+        bwcWatch = entityAsMap(client().performRequest(new Request("GET", "_watcher/watch/bwc_funny_timeout")));
+        assertThat(bwcWatch.get("found"), equalTo(true));
+        source = (Map<?, ?>) bwcWatch.get("watch");
+
+        Map<String, Object> attachments = ObjectPath.eval("actions.work.email.attachments", source);
+        Map<?, ?> attachment = (Map<?, ?>) attachments.get("test_report.pdf");
+        Map<String, Object> request = ObjectPath.eval("http.request", attachment);
+        assertEquals(timeout, request.get("read_timeout_millis"));
+        assertEquals("https", request.get("scheme"));
+        assertEquals("example.com", request.get("host"));
+        assertEquals("{{ctx.metadata.report_url}}", request.get("path"));
+        assertEquals(8443, request.get("port"));
+        Map<String, String> basic = ObjectPath.eval("auth.basic", request);
+        assertThat(basic, hasEntry("username", "Aladdin"));
+        // password doesn't come back because it is hidden
+        assertThat(basic, hasEntry(is("password"), anyOf(startsWith("::es_encrypted::"), is("::es_redacted::"))));
+        Request searchRequest = new Request("GET", ".watcher-history*/_search");
+        if (isRunningAgainstOldCluster() == false) {
+            searchRequest.addParameter(RestSearchAction.TOTAL_HITS_AS_INT_PARAM, "true");
+        }
+        Map<String, Object> history = entityAsMap(client().performRequest(searchRequest));
+        Map<?, ?> hits = (Map<?, ?>) history.get("hits");
+        assertThat((Integer) hits.get("total"), greaterThanOrEqualTo(2));
+    }
+
+    private void assertBasicWatchInteractions() throws Exception {
+        String watch = """
+            {
+              "trigger": {
+                "schedule": {
+                  "interval": "1s"
+                }
+              },
+              "input": {
+                "none": {}
+              },
+              "condition": {
+                "always": {}
+              },
+              "actions": {
+                "awesome": {
+                  "logging": {
+                    "level": "info",
+                    "text": "test"
+                  }
+                }
+              }
+            }""";
+        Request createWatchRequest = new Request("PUT", "_watcher/watch/new_watch");
+        createWatchRequest.setJsonEntity(watch);
+        Map<String, Object> createWatch = entityAsMap(client().performRequest(createWatchRequest));
+
+        logger.info("create watch {}", createWatch);
+
+        assertThat(createWatch.get("created"), equalTo(true));
+        assertThat(createWatch.get("_version"), equalTo(1));
+
+        Map<String, Object> updateWatch = entityAsMap(client().performRequest(createWatchRequest));
+        assertThat(updateWatch.get("created"), equalTo(false));
+        assertThat(updateWatch.get("_version"), equalTo(2));
+
+        Map<String, Object> get = entityAsMap(client().performRequest(new Request("GET", "_watcher/watch/new_watch")));
+        assertThat(get.get("found"), equalTo(true));
+        Map<?, ?> source = (Map<?, ?>) get.get("watch");
+        Map<String, Object> logging = ObjectPath.eval("actions.awesome.logging", source);
+        assertEquals("info", logging.get("level"));
+        assertEquals("test", logging.get("text"));
+    }
+
+    private void waitForYellow(String indexName) throws IOException {
+        Request request = new Request("GET", "/_cluster/health/" + indexName);
+        request.addParameter("wait_for_status", "yellow");
+        request.addParameter("timeout", "30s");
+        request.addParameter("wait_for_no_relocating_shards", "true");
+        request.addParameter("wait_for_no_initializing_shards", "true");
+        Map<String, Object> response = entityAsMap(client().performRequest(request));
+        assertThat(response.get("timed_out"), equalTo(Boolean.FALSE));
+    }
+
+    private void waitForHits(String indexName, int expectedHits) throws Exception {
+        Request request = new Request("GET", "/" + indexName + "/_search");
+        request.addParameter("ignore_unavailable", "true");
+        request.addParameter("size", "0");
+        assertBusy(() -> {
+            try {
+                Map<String, Object> response = entityAsMap(client().performRequest(request));
+                Map<?, ?> hits = (Map<?, ?>) response.get("hits");
+                logger.info("Hits are: {}", hits);
+                Integer total;
+                if (getOldClusterVersion().onOrAfter(Version.V_7_0_0) || isRunningAgainstOldCluster() == false) {
+                    total = (Integer) ((Map<?, ?>) hits.get("total")).get("value");
+                } else {
+                    total = (Integer) hits.get("total");
+                }
+                assertThat(total, greaterThanOrEqualTo(expectedHits));
+            } catch (IOException ioe) {
+                if (ioe instanceof ResponseException) {
+                    Response response = ((ResponseException) ioe).getResponse();
+                    if (RestStatus.fromCode(response.getStatusLine().getStatusCode()) == RestStatus.SERVICE_UNAVAILABLE) {
+                        fail("shards are not yet active");
+                    }
+                }
+                throw ioe;
+            }
+        }, 30, TimeUnit.SECONDS);
+    }
+
+    private void startWatcher() throws Exception {
+        Map<String, Object> startWatchResponse = entityAsMap(client().performRequest(new Request("POST", "_watcher/_start")));
+        assertThat(startWatchResponse.get("acknowledged"), equalTo(Boolean.TRUE));
+        assertBusy(() -> {
+            Map<String, Object> statsWatchResponse = entityAsMap(client().performRequest(new Request("GET", "_watcher/stats")));
+            List<?> states = ((List<?>) statsWatchResponse.get("stats")).stream()
+                .map(o -> ((Map<?, ?>) o).get("watcher_state"))
+                .collect(Collectors.toList());
+            assertThat(states, everyItem(is("started")));
+        });
+    }
+
+    private void stopWatcher() throws Exception {
+        Map<String, Object> stopWatchResponse = entityAsMap(client().performRequest(new Request("POST", "_watcher/_stop")));
+        assertThat(stopWatchResponse.get("acknowledged"), equalTo(Boolean.TRUE));
+        assertBusy(() -> {
+            Map<String, Object> statsStoppedWatchResponse = entityAsMap(client().performRequest(new Request("GET", "_watcher/stats")));
+            List<?> states = ((List<?>) statsStoppedWatchResponse.get("stats")).stream()
+                .map(o -> ((Map<?, ?>) o).get("watcher_state"))
+                .collect(Collectors.toList());
+            assertThat(states, everyItem(is("stopped")));
+        });
+    }
+
+    static String toStr(Response response) throws IOException {
+        return EntityUtils.toString(response.getEntity());
+    }
+
+    private void createUser(final boolean oldCluster) throws Exception {
+        final String id = oldCluster ? "preupgrade_user" : "postupgrade_user";
+        Request request = new Request("PUT", "/_security/user/" + id);
+        request.setJsonEntity(Strings.format("""
+            {
+               "password" : "l0ng-r4nd0m-p@ssw0rd",
+               "roles" : [ "admin", "other_role1" ],
+               "full_name" : "%s",
+               "email" : "%s@example.com",
+               "enabled": true
+            }""", randomAlphaOfLength(5), id));
+        client().performRequest(request);
+    }
+
+    private void createRole(final boolean oldCluster) throws Exception {
+        final String id = oldCluster ? "preupgrade_role" : "postupgrade_role";
+        Request request = new Request("PUT", "/_security/role/" + id);
+        request.setJsonEntity("""
+            {
+              "run_as": [ "abc" ],
+              "cluster": [ "monitor" ],
+              "indices": [
+                {
+                  "names": [ "events-*" ],
+                  "privileges": [ "read" ],
+                  "field_security" : {
+                    "grant" : [ "category", "@timestamp", "message" ]
+                  },
+                  "query": "{\\"match\\": {\\"category\\": \\"click\\"}}"
+                }
+              ]
+            }""");
+        client().performRequest(request);
+    }
+
+    private void assertUserInfo(final boolean oldCluster) throws Exception {
+        final String user = oldCluster ? "preupgrade_user" : "postupgrade_user";
+        Request request = new Request("GET", "/_security/user/" + user);
+        ;
+        Map<String, Object> response = entityAsMap(client().performRequest(request));
+        Map<?, ?> userInfo = (Map<?, ?>) response.get(user);
+        assertEquals(user + "@example.com", userInfo.get("email"));
+        assertNotNull(userInfo.get("full_name"));
+        assertNotNull(userInfo.get("roles"));
+    }
+
+    private void assertRoleInfo(final boolean oldCluster) throws Exception {
+        final String role = oldCluster ? "preupgrade_role" : "postupgrade_role";
+        Map<?, ?> response = (Map<?, ?>) entityAsMap(client().performRequest(new Request("GET", "/_security/role/" + role))).get(role);
+        assertNotNull(response.get("run_as"));
+        assertNotNull(response.get("cluster"));
+        assertNotNull(response.get("indices"));
+    }
+
+    private void assertRollUpJob(final String rollupJob) throws Exception {
+        final Matcher<?> expectedStates = anyOf(equalTo("indexing"), equalTo("started"));
+        waitForRollUpJob(rollupJob, expectedStates);
+
+        // check that the rollup job is started using the RollUp API
+        final Request getRollupJobRequest = new Request("GET", "_rollup/job/" + rollupJob);
+        Map<String, Object> getRollupJobResponse = entityAsMap(client().performRequest(getRollupJobRequest));
+        Map<?, ?> job = getJob(getRollupJobResponse, rollupJob);
+        assertNotNull(job);
+        assertThat(ObjectPath.eval("status.job_state", job), expectedStates);
+
+        // check that the rollup job is started using the Tasks API
+        final Request taskRequest = new Request("GET", "_tasks");
+        taskRequest.addParameter("detailed", "true");
+        taskRequest.addParameter("actions", "xpack/rollup/*");
+        Map<String, Object> taskResponse = entityAsMap(client().performRequest(taskRequest));
+        Map<?, ?> taskResponseNodes = (Map<?, ?>) taskResponse.get("nodes");
+        Map<?, ?> taskResponseNode = (Map<?, ?>) taskResponseNodes.values().iterator().next();
+        Map<?, ?> taskResponseTasks = (Map<?, ?>) taskResponseNode.get("tasks");
+        Map<?, ?> taskResponseStatus = (Map<?, ?>) taskResponseTasks.values().iterator().next();
+        assertThat(ObjectPath.eval("status.job_state", taskResponseStatus), expectedStates);
+
+        // check that the rollup job is started using the Cluster State API
+        final Request clusterStateRequest = new Request("GET", "_cluster/state/metadata");
+        Map<String, Object> clusterStateResponse = entityAsMap(client().performRequest(clusterStateRequest));
+        List<Map<String, Object>> rollupJobTasks = ObjectPath.eval("metadata.persistent_tasks.tasks", clusterStateResponse);
+
+        boolean hasRollupTask = false;
+        for (Map<String, Object> task : rollupJobTasks) {
+            if (ObjectPath.eval("id", task).equals(rollupJob)) {
+                hasRollupTask = true;
+                final String jobStateField = "task.xpack/rollup/job.state.job_state";
+                assertThat(
+                    "Expected field [" + jobStateField + "] to be started or indexing in " + task.get("id"),
+                    ObjectPath.eval(jobStateField, task),
+                    expectedStates
+                );
+                break;
+            }
+        }
+        if (hasRollupTask == false) {
+            fail("Expected persistent task for [" + rollupJob + "] but none found.");
+        }
+    }
+
+    private void waitForRollUpJob(final String rollupJob, final Matcher<?> expectedStates) throws Exception {
+        assertBusy(() -> {
+            final Request getRollupJobRequest = new Request("GET", "/_rollup/job/" + rollupJob);
+
+            Response getRollupJobResponse = client().performRequest(getRollupJobRequest);
+            assertThat(getRollupJobResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
+
+            Map<?, ?> job = getJob(getRollupJobResponse, rollupJob);
+            assertNotNull(job);
+            assertThat(ObjectPath.eval("status.job_state", job), expectedStates);
+        }, 30L, TimeUnit.SECONDS);
+    }
+
+    private Map<?, ?> getJob(Response response, String targetJobId) throws IOException {
+        return getJob(ESRestTestCase.entityAsMap(response), targetJobId);
+    }
+
+    private Map<?, ?> getJob(Map<String, Object> jobsMap, String targetJobId) throws IOException {
+        List<?> jobs = (List<?>) XContentMapValues.extractValue("jobs", jobsMap);
+        if (jobs == null) {
+            return null;
+        }
+
+        for (Object entry : jobs) {
+            Map<?, ?> job = (Map<?, ?>) entry;
+            String jobId = (String) ((Map<?, ?>) job.get("config")).get("id");
+            if (jobId.equals(targetJobId)) {
+                return job;
+            }
+        }
+        return null;
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testDataStreams() throws Exception {
+        assumeTrue("no data streams in versions before " + Version.V_7_9_0, getOldClusterVersion().onOrAfter(Version.V_7_9_0));
+        if (isRunningAgainstOldCluster()) {
+            createComposableTemplate(client(), "dst", "ds");
+
+            Request indexRequest = new Request("POST", "/ds/_doc/1?op_type=create&refresh");
+            XContentBuilder builder = JsonXContent.contentBuilder()
+                .startObject()
+                .field("f", "v")
+                .field("@timestamp", System.currentTimeMillis())
+                .endObject();
+            indexRequest.setJsonEntity(Strings.toString(builder));
+            assertOK(client().performRequest(indexRequest));
+        }
+
+        // It's quite possible that this test will run where the data stream backing index is
+        // created on one day, and then checked on a subsequent day. To avoid this failing the
+        // test, we store the timestamp used when the document is indexed, then when we go to
+        // check the backing index name, we retrieve the time and use it for the backing index
+        // name resolution.
+        Request getDoc = new Request("GET", "/ds/_search");
+        Map<String, Object> doc = entityAsMap(client().performRequest(getDoc));
+        logger.info("--> doc: {}", doc);
+        Map<String, Object> hits = (Map<String, Object>) doc.get("hits");
+        Map<String, Object> docBody = (Map<String, Object>) ((List<Object>) hits.get("hits")).get(0);
+        Long timestamp = (Long) ((Map<String, Object>) docBody.get("_source")).get("@timestamp");
+        logger.info("--> parsed out timestamp of {}", timestamp);
+
+        Request getDataStream = new Request("GET", "/_data_stream/ds");
+        Response response = client().performRequest(getDataStream);
+        assertOK(response);
+        List<Object> dataStreams = (List<Object>) entityAsMap(response).get("data_streams");
+        assertEquals(1, dataStreams.size());
+        Map<String, Object> ds = (Map<String, Object>) dataStreams.get(0);
+        List<Map<String, String>> indices = (List<Map<String, String>>) ds.get("indices");
+        assertEquals("ds", ds.get("name"));
+        assertEquals(1, indices.size());
+        assertEquals(
+            DataStreamTestHelper.getLegacyDefaultBackingIndexName("ds", 1, timestamp, getOldClusterVersion()),
+            indices.get(0).get("index_name")
+        );
+        assertNumHits("ds", 1, 1);
+    }
+
+    private static void createComposableTemplate(RestClient client, String templateName, String indexPattern) throws IOException {
+        StringEntity templateJSON = new StringEntity(Strings.format("""
+            {
+              "index_patterns": "%s",
+              "data_stream": {}
+            }""", indexPattern), ContentType.APPLICATION_JSON);
+        Request createIndexTemplateRequest = new Request("PUT", "_index_template/" + templateName);
+        createIndexTemplateRequest.setEntity(templateJSON);
+        client.performRequest(createIndexTemplateRequest);
+    }
+
+    private RequestOptions.Builder systemIndexWarningHandlerOptions(String index) {
+        return RequestOptions.DEFAULT.toBuilder()
+            .setWarningsHandler(
+                w -> w.size() > 0
+                    && w.contains(
+                        "this request accesses system indices: ["
+                            + index
+                            + "], but in a future major "
+                            + "version, direct access to system indices will be prevented by default"
+                    ) == false
+            );
+    }
+}
diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java
similarity index 93%
rename from x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java
rename to x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java
index 875ac4cef777..25a14c47e52c 100644
--- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java
+++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MLModelDeploymentFullClusterRestartIT.java
@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.restart;
 
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
@@ -14,7 +16,8 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
-import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus;
 import org.elasticsearch.xpack.core.ml.inference.assignment.AllocationStatus;
 import org.junit.Before;
 
@@ -31,7 +34,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 
-public class MLModelDeploymentFullClusterRestartIT extends AbstractFullClusterRestartTestCase {
+public class MLModelDeploymentFullClusterRestartIT extends AbstractXpackFullClusterRestartTestCase {
 
     // See PyTorchModelIT for how this model was created
     static final String BASE_64_ENCODED_MODEL =
@@ -62,6 +65,10 @@ public class MLModelDeploymentFullClusterRestartIT extends AbstractFullClusterRe
         RAW_MODEL_SIZE = Base64.getDecoder().decode(BASE_64_ENCODED_MODEL).length;
     }
 
+    public MLModelDeploymentFullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) {
+        super(upgradeStatus);
+    }
+
     @Before
     public void setLogging() throws IOException {
         Request loggingSettings = new Request("PUT", "_cluster/settings");
@@ -98,8 +105,10 @@ public void testDeploymentSurvivesRestart() throws Exception {
                 request.addParameter("timeout", "70s");
             }));
             waitForDeploymentStarted(modelId);
-            assertInfer(modelId);
-            assertNewInfer(modelId);
+            assertBusy(() -> {
+                assertInfer(modelId);
+                assertNewInfer(modelId);
+            }, 90, TimeUnit.SECONDS);
             stopDeployment(modelId);
         }
     }
@@ -133,7 +142,7 @@ private void assertNewInfer(String modelId) throws IOException {
 
     private void putModelDefinition(String modelId) throws IOException {
         Request request = new Request("PUT", "_ml/trained_models/" + modelId + "/definition/0");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {"total_definition_length":%s,"definition": "%s","total_parts": 1}""", RAW_MODEL_SIZE, BASE_64_ENCODED_MODEL));
         client().performRequest(request);
     }
@@ -146,7 +155,7 @@ private void putVocabulary(List<String> vocabulary, String modelId) throws IOExc
         String quotedWords = vocabularyWithPad.stream().map(s -> "\"" + s + "\"").collect(Collectors.joining(","));
 
         Request request = new Request("PUT", "_ml/trained_models/" + modelId + "/vocabulary");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             { "vocabulary": [%s] }
             """, quotedWords));
         client().performRequest(request);
@@ -205,7 +214,7 @@ private Response getTrainedModelStats(String modelId) throws IOException {
 
     private Response infer(String input, String modelId) throws IOException {
         Request request = new Request("POST", "/_ml/trained_models/" + modelId + "/deployment/_infer");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {  "docs": [{"input":"%s"}] }
             """, input));
 
@@ -217,7 +226,7 @@ private Response infer(String input, String modelId) throws IOException {
 
     private Response newInfer(String input, String modelId) throws IOException {
         Request request = new Request("POST", "/_ml/trained_models/" + modelId + "/_infer");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {  "docs": [{"input":"%s"}] }
             """, input));
         var response = client().performRequest(request);
diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java
similarity index 92%
rename from x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java
rename to x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java
index fe952e555e83..e4ab3957f262 100644
--- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java
+++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java
@@ -6,13 +6,16 @@
  */
 package org.elasticsearch.xpack.restart;
 
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
-import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus;
 import org.elasticsearch.xpack.test.rest.IndexMappingTemplateAsserter;
 import org.elasticsearch.xpack.test.rest.XPackRestTestConstants;
 import org.elasticsearch.xpack.test.rest.XPackRestTestHelper;
@@ -28,11 +31,15 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 
-public class MlConfigIndexMappingsFullClusterRestartIT extends AbstractFullClusterRestartTestCase {
+public class MlConfigIndexMappingsFullClusterRestartIT extends AbstractXpackFullClusterRestartTestCase {
 
     private static final String OLD_CLUSTER_JOB_ID = "ml-config-mappings-old-cluster-job";
     private static final String NEW_CLUSTER_JOB_ID = "ml-config-mappings-new-cluster-job";
 
+    public MlConfigIndexMappingsFullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) {
+        super(upgradeStatus);
+    }
+
     @Override
     protected Settings restClientSettings() {
         String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8));
@@ -72,7 +79,7 @@ public void testMlConfigIndexMappingsAfterMigration() throws Exception {
     }
 
     private void createAnomalyDetectorJob(String jobId) throws IOException {
-        String jobConfig = formatted("""
+        String jobConfig = Strings.format("""
             {
                 "job_id": "%s",
                 "analysis_config": {
diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java
new file mode 100644
index 000000000000..79a2be51197e
--- /dev/null
+++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java
@@ -0,0 +1,205 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.restart;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.WarningsHandler;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
+import org.elasticsearch.xcontent.spi.XContentProvider;
+import org.elasticsearch.xpack.test.rest.XPackRestTestConstants;
+import org.elasticsearch.xpack.test.rest.XPackRestTestHelper;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.aMapWithSize;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class MlHiddenIndicesFullClusterRestartIT extends AbstractXpackFullClusterRestartTestCase {
+
+    private static final String JOB_ID = "ml-hidden-indices-old-cluster-job";
+    private static final List<Tuple<List<String>, String>> EXPECTED_INDEX_ALIAS_PAIRS = List.of(
+        Tuple.tuple(List.of(".ml-annotations-000001"), ".ml-annotations-read"),
+        Tuple.tuple(List.of(".ml-annotations-000001"), ".ml-annotations-write"),
+        Tuple.tuple(List.of(".ml-state", ".ml-state-000001"), ".ml-state-write"),
+        Tuple.tuple(List.of(".ml-anomalies-shared"), ".ml-anomalies-" + JOB_ID),
+        Tuple.tuple(List.of(".ml-anomalies-shared"), ".ml-anomalies-.write-" + JOB_ID)
+    );
+
+    public MlHiddenIndicesFullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) {
+        super(upgradeStatus);
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8));
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+
+    @Before
+    public void waitForMlTemplates() throws Exception {
+        // We shouldn't wait for ML templates during the upgrade - production won't
+        if (isRunningAgainstOldCluster()) {
+            XPackRestTestHelper.waitForTemplates(
+                client(),
+                XPackRestTestConstants.ML_POST_V7120_TEMPLATES,
+                getOldClusterVersion().onOrAfter(Version.V_7_8_0)
+            );
+        }
+    }
+
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/93521")
+    public void testMlIndicesBecomeHidden() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+            // trigger ML indices creation
+            createAnomalyDetectorJob(JOB_ID);
+            openAnomalyDetectorJob(JOB_ID);
+
+            if (getOldClusterVersion().before(Version.V_7_7_0)) {
+                Map<String, Object> indexSettingsMap = contentAsMap(getMlIndicesSettings());
+                Map<String, Object> aliasesMap = contentAsMap(getMlAliases());
+
+                assertThat("Index settings map was: " + indexSettingsMap, indexSettingsMap, is(aMapWithSize(greaterThanOrEqualTo(4))));
+                for (Map.Entry<String, Object> e : indexSettingsMap.entrySet()) {
+                    String indexName = e.getKey();
+                    @SuppressWarnings("unchecked")
+                    Map<String, Object> settings = (Map<String, Object>) e.getValue();
+                    assertThat(settings, is(notNullValue()));
+                    assertThat(
+                        "Index " + indexName + " expected not to be hidden but was, settings = " + settings,
+                        XContentMapValues.extractValue(settings, "settings", "index", "hidden"),
+                        is(nullValue())
+                    );
+                }
+
+                for (Tuple<List<String>, String> indexAndAlias : EXPECTED_INDEX_ALIAS_PAIRS) {
+                    List<String> indices = indexAndAlias.v1();
+                    String alias = indexAndAlias.v2();
+                    for (String index : indices) {
+                        assertThat(
+                            indexAndAlias + " expected not be hidden but was, aliasesMap = " + aliasesMap,
+                            XContentMapValues.extractValue(aliasesMap, index, "aliases", alias, "is_hidden"),
+                            is(nullValue())
+                        );
+                    }
+                }
+            }
+        } else {
+            // The 5 operations in MlInitializationService.makeMlInternalIndicesHidden() run sequentially, so might
+            // not all be finished when this test runs. The desired state should exist within a few seconds of startup,
+            // hence the assertBusy().
+            assertBusy(() -> {
+                Map<String, Object> indexSettingsMap = contentAsMap(getMlIndicesSettings());
+                Map<String, Object> aliasesMap = contentAsMap(getMlAliases());
+
+                assertThat("Index settings map was: " + indexSettingsMap, indexSettingsMap, is(aMapWithSize(greaterThanOrEqualTo(4))));
+                for (Map.Entry<String, Object> e : indexSettingsMap.entrySet()) {
+                    String indexName = e.getKey();
+                    @SuppressWarnings("unchecked")
+                    Map<String, Object> settings = (Map<String, Object>) e.getValue();
+                    assertThat(settings, is(notNullValue()));
+                    assertThat(
+                        "Index " + indexName + " expected to be hidden but wasn't, settings = " + settings,
+                        XContentMapValues.extractValue(settings, "settings", "index", "hidden"),
+                        is(equalTo("true"))
+                    );
+                }
+
+                for (Tuple<List<String>, String> indexAndAlias : EXPECTED_INDEX_ALIAS_PAIRS) {
+                    List<String> indices = indexAndAlias.v1();
+                    String alias = indexAndAlias.v2();
+                    assertThat(
+                        indexAndAlias + " expected to be hidden but wasn't, aliasesMap = " + aliasesMap,
+                        indices.stream()
+                            .anyMatch(
+                                index -> Boolean.TRUE.equals(
+                                    XContentMapValues.extractValue(aliasesMap, index, "aliases", alias, "is_hidden")
+                                )
+                            ),
+                        is(true)
+                    );
+                }
+            });
+        }
+    }
+
+    private Response getMlIndicesSettings() throws IOException {
+        Request getSettingsRequest = new Request(
+            "GET",
+            ".ml-anomalies-*,.ml-state*,.ml-stats-*,.ml-notifications*,.ml-annotations*/_settings"
+        );
+        getSettingsRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE).build());
+        Response getSettingsResponse = client().performRequest(getSettingsRequest);
+        assertThat(getSettingsResponse, is(notNullValue()));
+        return getSettingsResponse;
+    }
+
+    private Response getMlAliases() throws IOException {
+        Request getAliasesRequest = new Request("GET", ".ml-anomalies-*,.ml-state*,.ml-stats-*,.ml-notifications*,.ml-annotations*/_alias");
+        getAliasesRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE).build());
+        Response getAliasesResponse = client().performRequest(getAliasesRequest);
+        assertThat(getAliasesResponse, is(notNullValue()));
+        return getAliasesResponse;
+    }
+
+    @SuppressWarnings("unchecked")
+    private static Map<String, Object> contentAsMap(Response response) throws IOException {
+        InputStreamReader reader = new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8);
+        XContentParser parser = XContentProvider.provider()
+            .getJsonXContent()
+            .XContent()
+            .createParser(XContentParserConfiguration.EMPTY, reader);
+        return parser.map();
+    }
+
+    private void createAnomalyDetectorJob(String jobId) throws IOException {
+        String jobConfig = Strings.format("""
+            {
+                "job_id": "%s",
+                "analysis_config": {
+                    "bucket_span": "10m",
+                    "detectors": [{
+                        "function": "metric",
+                        "field_name": "responsetime"
+                    }]
+                },
+                "data_description": {}
+            }""", jobId);
+
+        Request putJobRequest = new Request("PUT", "/_ml/anomaly_detectors/" + jobId);
+        putJobRequest.setJsonEntity(jobConfig);
+        Response putJobResponse = client().performRequest(putJobRequest);
+        assertThat(putJobResponse.getStatusLine().getStatusCode(), equalTo(200));
+    }
+
+    private void openAnomalyDetectorJob(String jobId) throws IOException {
+        Request openJobRequest = new Request("POST", "/_ml/anomaly_detectors/" + jobId + "/_open");
+        Response openJobResponse = client().performRequest(openJobRequest);
+        assertThat(openJobResponse.getStatusLine().getStatusCode(), equalTo(200));
+    }
+}
diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java
similarity index 96%
rename from x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java
rename to x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java
index ddfbe1dc017d..2bbda9123ae3 100644
--- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java
+++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java
@@ -6,6 +6,8 @@
  */
 package org.elasticsearch.xpack.restart;
 
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
@@ -17,7 +19,7 @@
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
-import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase;
+import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus;
 import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.test.rest.XPackRestTestConstants;
@@ -35,13 +37,17 @@
 import static org.hamcrest.Matchers.emptyOrNullString;
 import static org.hamcrest.Matchers.is;
 
-public class MlMigrationFullClusterRestartIT extends AbstractFullClusterRestartTestCase {
+public class MlMigrationFullClusterRestartIT extends AbstractXpackFullClusterRestartTestCase {
 
     private static final String OLD_CLUSTER_OPEN_JOB_ID = "migration-old-cluster-open-job";
     private static final String OLD_CLUSTER_STARTED_DATAFEED_ID = "migration-old-cluster-started-datafeed";
     private static final String OLD_CLUSTER_CLOSED_JOB_ID = "migration-old-cluster-closed-job";
     private static final String OLD_CLUSTER_STOPPED_DATAFEED_ID = "migration-old-cluster-stopped-datafeed";
 
+    public MlMigrationFullClusterRestartIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) {
+        super(upgradeStatus);
+    }
+
     @Override
     protected Settings restClientSettings() {
         String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8));
@@ -205,7 +211,7 @@ private void addAggregations(DatafeedConfig.Builder dfBuilder) {
     }
 
     private void putJob(String jobId) throws IOException {
-        String jobConfig = formatted("""
+        String jobConfig = Strings.format("""
             {
                 "job_id": "%s",
                 "analysis_config": {
diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/QueryBuilderBWCIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/QueryBuilderBWCIT.java
new file mode 100644
index 000000000000..563cde322b72
--- /dev/null
+++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/QueryBuilderBWCIT.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.restart;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.cluster.util.resource.Resource;
+import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+public class QueryBuilderBWCIT extends org.elasticsearch.upgrades.QueryBuilderBWCIT {
+
+    static {
+        clusterConfig = c -> c.setting("xpack.security.enabled", "true")
+            .setting("xpack.security.transport.ssl.enabled", "true")
+            .setting("xpack.security.transport.ssl.key", "testnode.pem")
+            .setting("xpack.security.transport.ssl.certificate", "testnode.crt")
+            .setting("xpack.license.self_generated.type", "trial")
+            .setting("xpack.watcher.encrypt_sensitive_data", "true")
+            .setting("xpack.security.authc.api_key.enabled", "true")
+            .configFile("testnode.pem", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem"))
+            .configFile("testnode.crt", Resource.fromClasspath("org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"))
+            .keystore("xpack.watcher.encryption_key", Resource.fromClasspath("system_key"))
+            .keystore("xpack.security.transport.ssl.secure_key_passphrase", "testnode");
+    }
+
+    public QueryBuilderBWCIT(FullClusterRestartUpgradeStatus upgradeStatus) {
+        super(upgradeStatus);
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8));
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+}
diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java
similarity index 89%
rename from x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java
rename to x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java
index ea926e964360..043b3f49a882 100644
--- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java
+++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java
@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.restart;
 
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
@@ -14,7 +16,7 @@
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase;
+import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus;
 
 import java.nio.charset.StandardCharsets;
 import java.util.Base64;
@@ -23,7 +25,11 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.not;
 
-public class WatcherMappingUpdateIT extends AbstractFullClusterRestartTestCase {
+public class WatcherMappingUpdateIT extends AbstractXpackFullClusterRestartTestCase {
+
+    public WatcherMappingUpdateIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) {
+        super(upgradeStatus);
+    }
 
     @Override
     protected Settings restClientSettings() {
@@ -91,7 +97,7 @@ private void assertNoMappingVersion(String index) throws Exception {
 
     private RequestOptions.Builder getWarningHandlerOptions(String index) {
         return RequestOptions.DEFAULT.toBuilder()
-            .setWarningsHandler(w -> w.contains(getWatcherSystemIndexWarning(index)) == false || w.size() != 1);
+            .setWarningsHandler(w -> w.size() > 0 && w.contains(getWatcherSystemIndexWarning(index)) == false);
     }
 
     private String getWatcherSystemIndexWarning(String index) {
diff --git a/x-pack/qa/full-cluster-restart/src/test/resources/org/elasticsearch/xpack/restart/funny-timeout-watch.json b/x-pack/qa/full-cluster-restart/src/javaRestTest/resources/org/elasticsearch/xpack/restart/funny-timeout-watch.json
similarity index 100%
rename from x-pack/qa/full-cluster-restart/src/test/resources/org/elasticsearch/xpack/restart/funny-timeout-watch.json
rename to x-pack/qa/full-cluster-restart/src/javaRestTest/resources/org/elasticsearch/xpack/restart/funny-timeout-watch.json
diff --git a/x-pack/qa/full-cluster-restart/src/test/resources/org/elasticsearch/xpack/restart/logging-watch.json b/x-pack/qa/full-cluster-restart/src/javaRestTest/resources/org/elasticsearch/xpack/restart/logging-watch.json
similarity index 100%
rename from x-pack/qa/full-cluster-restart/src/test/resources/org/elasticsearch/xpack/restart/logging-watch.json
rename to x-pack/qa/full-cluster-restart/src/javaRestTest/resources/org/elasticsearch/xpack/restart/logging-watch.json
diff --git a/x-pack/qa/full-cluster-restart/src/test/resources/org/elasticsearch/xpack/restart/simple-watch.json b/x-pack/qa/full-cluster-restart/src/javaRestTest/resources/org/elasticsearch/xpack/restart/simple-watch.json
similarity index 100%
rename from x-pack/qa/full-cluster-restart/src/test/resources/org/elasticsearch/xpack/restart/simple-watch.json
rename to x-pack/qa/full-cluster-restart/src/javaRestTest/resources/org/elasticsearch/xpack/restart/simple-watch.json
diff --git a/x-pack/qa/full-cluster-restart/src/test/resources/org/elasticsearch/xpack/restart/throttle-period-watch.json b/x-pack/qa/full-cluster-restart/src/javaRestTest/resources/org/elasticsearch/xpack/restart/throttle-period-watch.json
similarity index 100%
rename from x-pack/qa/full-cluster-restart/src/test/resources/org/elasticsearch/xpack/restart/throttle-period-watch.json
rename to x-pack/qa/full-cluster-restart/src/javaRestTest/resources/org/elasticsearch/xpack/restart/throttle-period-watch.json
diff --git a/x-pack/qa/full-cluster-restart/src/test/resources/system_key b/x-pack/qa/full-cluster-restart/src/javaRestTest/resources/system_key
similarity index 100%
rename from x-pack/qa/full-cluster-restart/src/test/resources/system_key
rename to x-pack/qa/full-cluster-restart/src/javaRestTest/resources/system_key
diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/CoreFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/CoreFullClusterRestartIT.java
deleted file mode 100644
index e06cb12f747a..000000000000
--- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/CoreFullClusterRestartIT.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.restart;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.upgrades.FullClusterRestartIT;
-
-import java.nio.charset.StandardCharsets;
-import java.util.Base64;
-
-public class CoreFullClusterRestartIT extends FullClusterRestartIT {
-
-    @Override
-    protected Settings restClientSettings() {
-        String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8));
-        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
-    }
-
-}
diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
deleted file mode 100644
index 56ad255eb102..000000000000
--- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
+++ /dev/null
@@ -1,999 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.restart;
-
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.util.EntityUtils;
-import org.elasticsearch.Version;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.client.ResponseException;
-import org.elasticsearch.client.RestClient;
-import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.common.xcontent.support.XContentMapValues;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.rest.action.search.RestSearchAction;
-import org.elasticsearch.test.StreamsUtils;
-import org.elasticsearch.test.rest.ESRestTestCase;
-import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase;
-import org.elasticsearch.xcontent.ObjectPath;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xcontent.XContentParserConfiguration;
-import org.elasticsearch.xcontent.XContentType;
-import org.elasticsearch.xcontent.json.JsonXContent;
-import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.core.slm.SnapshotLifecyclePolicy;
-import org.elasticsearch.xpack.core.slm.SnapshotLifecycleStats;
-import org.hamcrest.Matcher;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.Base64;
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.stream.Collectors;
-
-import static org.elasticsearch.core.TimeValue.timeValueSeconds;
-import static org.elasticsearch.upgrades.FullClusterRestartIT.assertNumHits;
-import static org.hamcrest.Matchers.anyOf;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.empty;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.everyItem;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.hasEntry;
-import static org.hamcrest.Matchers.hasItems;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.startsWith;
-
-public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
-
-    public static final int UPGRADE_FIELD_EXPECTED_INDEX_FORMAT_VERSION = 6;
-    public static final int SECURITY_EXPECTED_INDEX_FORMAT_VERSION = 6;
-
-    @Override
-    protected Settings restClientSettings() {
-        String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8));
-        return Settings.builder()
-            .put(ThreadContext.PREFIX + ".Authorization", token)
-            // we increase the timeout here to 90 seconds to handle long waits for a green
-            // cluster health. the waits for green need to be longer than a minute to
-            // account for delayed shards
-            .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s")
-            .build();
-    }
-
-    /**
-     * Tests that a single document survives. Super basic smoke test.
-     */
-    public void testSingleDoc() throws IOException {
-        String docLocation = "/testsingledoc/_doc/1";
-        String doc = "{\"test\": \"test\"}";
-
-        if (isRunningAgainstOldCluster()) {
-            Request createDoc = new Request("PUT", docLocation);
-            createDoc.addParameter("refresh", "true");
-            createDoc.setJsonEntity(doc);
-            client().performRequest(createDoc);
-        }
-
-        Request getRequest = new Request("GET", docLocation);
-        assertThat(toStr(client().performRequest(getRequest)), containsString(doc));
-    }
-
-    public void testSecurityNativeRealm() throws Exception {
-        if (isRunningAgainstOldCluster()) {
-            createUser(true);
-            createRole(true);
-        } else {
-            waitForYellow(".security");
-            final Request getSettingsRequest = new Request("GET", "/.security/_settings/index.format");
-            getSettingsRequest.setOptions(
-                expectWarnings(
-                    "this request accesses system indices: [.security-7], but in a future major "
-                        + "version, direct access to system indices will be prevented by default"
-                )
-            );
-            Response settingsResponse = client().performRequest(getSettingsRequest);
-            Map<String, Object> settingsResponseMap = entityAsMap(settingsResponse);
-            logger.info("settings response map {}", settingsResponseMap);
-            final String concreteSecurityIndex;
-            if (settingsResponseMap.isEmpty()) {
-                fail("The security index does not have the expected setting [index.format]");
-            } else {
-                concreteSecurityIndex = settingsResponseMap.keySet().iterator().next();
-                Map<?, ?> indexSettingsMap = (Map<?, ?>) settingsResponseMap.get(concreteSecurityIndex);
-                Map<?, ?> settingsMap = (Map<?, ?>) indexSettingsMap.get("settings");
-                logger.info("settings map {}", settingsMap);
-                if (settingsMap.containsKey("index")) {
-                    int format = Integer.parseInt(String.valueOf(((Map<?, ?>) settingsMap.get("index")).get("format")));
-                    assertEquals("The security index needs to be upgraded", SECURITY_EXPECTED_INDEX_FORMAT_VERSION, format);
-                }
-            }
-
-            // create additional user and role
-            createUser(false);
-            createRole(false);
-        }
-
-        assertUserInfo(isRunningAgainstOldCluster());
-        assertRoleInfo(isRunningAgainstOldCluster());
-    }
-
-    public void testWatcher() throws Exception {
-        if (isRunningAgainstOldCluster()) {
-            logger.info("Adding a watch on old cluster {}", getOldClusterVersion());
-            Request createBwcWatch = new Request("PUT", "/_watcher/watch/bwc_watch");
-            createBwcWatch.setJsonEntity(loadWatch("simple-watch.json"));
-            client().performRequest(createBwcWatch);
-
-            logger.info("Adding a watch with \"fun\" throttle periods on old cluster");
-            Request createBwcThrottlePeriod = new Request("PUT", "/_watcher/watch/bwc_throttle_period");
-            createBwcThrottlePeriod.setJsonEntity(loadWatch("throttle-period-watch.json"));
-            client().performRequest(createBwcThrottlePeriod);
-
-            logger.info("Adding a watch with \"fun\" read timeout on old cluster");
-            Request createFunnyTimeout = new Request("PUT", "/_watcher/watch/bwc_funny_timeout");
-            createFunnyTimeout.setJsonEntity(loadWatch("funny-timeout-watch.json"));
-            client().performRequest(createFunnyTimeout);
-
-            logger.info("Waiting for watch results index to fill up...");
-            try {
-                waitForYellow(".watches,bwc_watch_index,.watcher-history*");
-            } catch (ResponseException e) {
-                {
-                    String rsp = toStr(client().performRequest(new Request("GET", "/_cluster/state")));
-                    logger.info("cluster_state_response=\n{}", rsp);
-                }
-                {
-                    Request request = new Request("GET", "/_watcher/stats/_all");
-                    request.addParameter("emit_stacktraces", "true");
-                    String rsp = toStr(client().performRequest(request));
-                    logger.info("watcher_stats_response=\n{}", rsp);
-                }
-                throw e;
-            }
-            waitForHits("bwc_watch_index", 2);
-            waitForHits(".watcher-history*", 2);
-            logger.info("Done creating watcher-related indices");
-        } else {
-            logger.info("testing against {}", getOldClusterVersion());
-            try {
-                waitForYellow(".watches,bwc_watch_index,.watcher-history*");
-            } catch (ResponseException e) {
-                String rsp = toStr(client().performRequest(new Request("GET", "/_cluster/state")));
-                logger.info("cluster_state_response=\n{}", rsp);
-                throw e;
-            }
-
-            logger.info("checking that the Watches index is the correct version");
-
-            // Verify .watches index format:
-            var getClusterStateResponse = entityAsMap(client().performRequest(new Request("GET", "/_cluster/state/metadata/.watches")));
-            Map<?, ?> indices = ObjectPath.eval("metadata.indices", getClusterStateResponse);
-            var dotWatchesIndex = indices.get(".watches"); // ObjectPath.eval(...) doesn't handle keys containing .
-            var indexFormat = Integer.parseInt(ObjectPath.eval("settings.index.format", dotWatchesIndex));
-            assertEquals("The watches index needs to be upgraded", UPGRADE_FIELD_EXPECTED_INDEX_FORMAT_VERSION, indexFormat);
-
-            // Wait for watcher to actually start....
-            startWatcher();
-
-            try {
-                assertOldTemplatesAreDeleted();
-                assertWatchIndexContentsWork();
-                assertBasicWatchInteractions();
-            } finally {
-                /* Shut down watcher after every test because watcher can be a bit finicky about shutting down when the node shuts
-                 * down. This makes super sure it shuts down *and* causes the test to fail in a sensible spot if it doesn't shut down.
-                 */
-                stopWatcher();
-            }
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testWatcherWithApiKey() throws Exception {
-        final Request getWatchStatusRequest = new Request("GET", "/_watcher/watch/watch_with_api_key");
-
-        if (isRunningAgainstOldCluster()) {
-            final Request createApiKeyRequest = new Request("PUT", "/_security/api_key");
-            createApiKeyRequest.setJsonEntity("""
-                {
-                   "name": "key-1",
-                   "role_descriptors": {
-                     "r": {
-                       "cluster": [ "all" ],
-                       "indices": [
-                         {
-                           "names": [ "*" ],
-                           "privileges": [ "all" ]
-                         }
-                       ]
-                     }
-                   }
-                 }""");
-            final Response response = client().performRequest(createApiKeyRequest);
-            final Map<String, Object> createApiKeyResponse = entityAsMap(response);
-
-            Request createWatchWithApiKeyRequest = new Request("PUT", "/_watcher/watch/watch_with_api_key");
-            createWatchWithApiKeyRequest.setJsonEntity(loadWatch("logging-watch.json"));
-            final byte[] keyBytes = (createApiKeyResponse.get("id") + ":" + createApiKeyResponse.get("api_key")).getBytes(
-                StandardCharsets.UTF_8
-            );
-            final String authHeader = "ApiKey " + Base64.getEncoder().encodeToString(keyBytes);
-            createWatchWithApiKeyRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", authHeader));
-            client().performRequest(createWatchWithApiKeyRequest);
-
-            assertBusy(() -> {
-                final Map<String, Object> getWatchStatusResponse = entityAsMap(client().performRequest(getWatchStatusRequest));
-                final Map<String, Object> status = (Map<String, Object>) getWatchStatusResponse.get("status");
-                assertEquals("executed", status.get("execution_state"));
-            }, 30, TimeUnit.SECONDS);
-
-        } else {
-            logger.info("testing against {}", getOldClusterVersion());
-            try {
-                waitForYellow(".watches,.watcher-history*");
-            } catch (ResponseException e) {
-                String rsp = toStr(client().performRequest(new Request("GET", "/_cluster/state")));
-                logger.info("cluster_state_response=\n{}", rsp);
-                throw e;
-            }
-
-            // Wait for watcher to actually start....
-            startWatcher();
-
-            try {
-                final Map<String, Object> getWatchStatusResponse = entityAsMap(client().performRequest(getWatchStatusRequest));
-                final Map<String, Object> status = (Map<String, Object>) getWatchStatusResponse.get("status");
-                final int version = (int) status.get("version");
-
-                final AtomicBoolean versionIncreased = new AtomicBoolean();
-                final AtomicBoolean executed = new AtomicBoolean();
-                assertBusy(() -> {
-                    final Map<String, Object> newGetWatchStatusResponse = entityAsMap(client().performRequest(getWatchStatusRequest));
-                    final Map<String, Object> newStatus = (Map<String, Object>) newGetWatchStatusResponse.get("status");
-                    if (false == versionIncreased.get() && version < (int) newStatus.get("version")) {
-                        versionIncreased.set(true);
-                    }
-                    if (false == executed.get() && "executed".equals(newStatus.get("execution_state"))) {
-                        executed.set(true);
-                    }
-                    assertThat(
-                        "version increased: [" + versionIncreased.get() + "], executed: [" + executed.get() + "]",
-                        versionIncreased.get() && executed.get(),
-                        is(true)
-                    );
-                }, 30, TimeUnit.SECONDS);
-            } finally {
-                stopWatcher();
-            }
-        }
-    }
-
-    public void testServiceAccountApiKey() throws IOException {
-        assumeTrue("no service accounts in versions before " + Version.V_7_13_0, getOldClusterVersion().onOrAfter(Version.V_7_13_0));
-        if (isRunningAgainstOldCluster()) {
-            final Request createServiceTokenRequest = new Request("POST", "/_security/service/elastic/fleet-server/credential/token");
-            final Response createServiceTokenResponse = client().performRequest(createServiceTokenRequest);
-            assertOK(createServiceTokenResponse);
-            @SuppressWarnings("unchecked")
-            final String serviceToken = ((Map<String, String>) responseAsMap(createServiceTokenResponse).get("token")).get("value");
-            final Request createApiKeyRequest = new Request("PUT", "/_security/api_key");
-            createApiKeyRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", "Bearer " + serviceToken));
-            createApiKeyRequest.setJsonEntity("{\"name\":\"key-1\"}");
-            final Response createApiKeyResponse = client().performRequest(createApiKeyRequest);
-            final Map<String, Object> createApiKeyResponseMap = entityAsMap(createApiKeyResponse);
-            final String authHeader = "ApiKey "
-                + Base64.getEncoder()
-                    .encodeToString(
-                        (createApiKeyResponseMap.get("id") + ":" + createApiKeyResponseMap.get("api_key")).getBytes(StandardCharsets.UTF_8)
-                    );
-
-            final Request indexRequest = new Request("PUT", "/api_keys/_doc/key-1");
-            indexRequest.setJsonEntity("{\"auth_header\":\"" + authHeader + "\"}");
-            assertOK(client().performRequest(indexRequest));
-        } else {
-            final Request getRequest = new Request("GET", "/api_keys/_doc/key-1");
-            final Response getResponse = client().performRequest(getRequest);
-            assertOK(getResponse);
-            final Map<String, Object> getResponseMap = responseAsMap(getResponse);
-            @SuppressWarnings("unchecked")
-            final String authHeader = ((Map<String, String>) getResponseMap.get("_source")).get("auth_header");
-
-            final Request mainRequest = new Request("GET", "/");
-            mainRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", authHeader));
-            assertOK(client().performRequest(mainRequest));
-
-            final Request getUserRequest = new Request("GET", "/_security/user");
-            getUserRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", authHeader));
-            final ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(getUserRequest));
-            assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(403));
-            assertThat(e.getMessage(), containsString("is unauthorized"));
-        }
-    }
-
-    public void testApiKeySuperuser() throws IOException {
-        if (isRunningAgainstOldCluster()) {
-            final Request createUserRequest = new Request("PUT", "/_security/user/api_key_super_creator");
-            createUserRequest.setJsonEntity("""
-                {
-                   "password" : "l0ng-r4nd0m-p@ssw0rd",
-                   "roles" : [ "superuser", "monitoring_user" ]
-                }""");
-            client().performRequest(createUserRequest);
-
-            // Create API key
-            final Request createApiKeyRequest = new Request("PUT", "/_security/api_key");
-            createApiKeyRequest.setOptions(
-                RequestOptions.DEFAULT.toBuilder()
-                    .addHeader(
-                        "Authorization",
-                        UsernamePasswordToken.basicAuthHeaderValue(
-                            "api_key_super_creator",
-                            new SecureString("l0ng-r4nd0m-p@ssw0rd".toCharArray())
-                        )
-                    )
-            );
-            if (getOldClusterVersion().onOrAfter(Version.V_7_3_0)) {
-                createApiKeyRequest.setJsonEntity("""
-                    {
-                       "name": "super_legacy_key"
-                    }""");
-            } else {
-                createApiKeyRequest.setJsonEntity("""
-                    {
-                       "name": "super_legacy_key",
-                       "role_descriptors": {
-                         "super": {
-                           "cluster": [ "all" ],
-                           "indices": [
-                             {
-                               "names": [ "*" ],
-                               "privileges": [ "all" ],
-                               "allow_restricted_indices": true
-                             }
-                           ]
-                         }
-                       }
-                    }""");
-            }
-            final Map<String, Object> createApiKeyResponse = entityAsMap(client().performRequest(createApiKeyRequest));
-            final byte[] keyBytes = (createApiKeyResponse.get("id") + ":" + createApiKeyResponse.get("api_key")).getBytes(
-                StandardCharsets.UTF_8
-            );
-            final String apiKeyAuthHeader = "ApiKey " + Base64.getEncoder().encodeToString(keyBytes);
-            // Save the API key info across restart
-            final Request saveApiKeyRequest = new Request("PUT", "/api_keys/_doc/super_legacy_key");
-            saveApiKeyRequest.setJsonEntity("{\"auth_header\":\"" + apiKeyAuthHeader + "\"}");
-            assertOK(client().performRequest(saveApiKeyRequest));
-
-            if (getOldClusterVersion().before(Version.V_8_0_0)) {
-                final Request indexRequest = new Request("POST", ".security/_doc");
-                indexRequest.setJsonEntity("""
-                    {
-                      "doc_type": "foo"
-                    }""");
-                if (getOldClusterVersion().onOrAfter(Version.V_7_10_0)) {
-                    indexRequest.setOptions(
-                        expectWarnings(
-                            "this request accesses system indices: [.security-7], but in a future major "
-                                + "version, direct access to system indices will be prevented by default"
-                        ).toBuilder().addHeader("Authorization", apiKeyAuthHeader)
-                    );
-                } else {
-                    indexRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", apiKeyAuthHeader));
-                }
-                assertOK(client().performRequest(indexRequest));
-            }
-        } else {
-            final Request getRequest = new Request("GET", "/api_keys/_doc/super_legacy_key");
-            final Map<String, Object> getResponseMap = responseAsMap(client().performRequest(getRequest));
-            @SuppressWarnings("unchecked")
-            final String apiKeyAuthHeader = ((Map<String, String>) getResponseMap.get("_source")).get("auth_header");
-
-            // read is ok
-            final Request searchRequest = new Request("GET", ".security/_search");
-            searchRequest.setOptions(
-                expectWarnings(
-                    "this request accesses system indices: [.security-7], but in a future major "
-                        + "version, direct access to system indices will be prevented by default"
-                ).toBuilder().addHeader("Authorization", apiKeyAuthHeader)
-            );
-            assertOK(client().performRequest(searchRequest));
-
-            // write must not be allowed
-            final Request indexRequest = new Request("POST", ".security/_doc");
-            indexRequest.setJsonEntity("""
-                {
-                  "doc_type": "foo"
-                }""");
-            indexRequest.setOptions(
-                expectWarnings(
-                    "this request accesses system indices: [.security-7], but in a future major "
-                        + "version, direct access to system indices will be prevented by default"
-                ).toBuilder().addHeader("Authorization", apiKeyAuthHeader)
-            );
-            final ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(indexRequest));
-            assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(403));
-            assertThat(e.getMessage(), containsString("is unauthorized"));
-        }
-    }
-
-    /**
-     * Tests that a RollUp job created on a old cluster is correctly restarted after the upgrade.
-     */
-    public void testRollupAfterRestart() throws Exception {
-        if (isRunningAgainstOldCluster()) {
-            final int numDocs = 59;
-            final int year = randomIntBetween(1970, 2018);
-
-            // index documents for the rollup job
-            final StringBuilder bulk = new StringBuilder();
-            for (int i = 0; i < numDocs; i++) {
-                bulk.append("{\"index\":{\"_index\":\"rollup-docs\"}}\n");
-                String date = String.format(Locale.ROOT, "%04d-01-01T00:%02d:00Z", year, i);
-                bulk.append("{\"timestamp\":\"").append(date).append("\",\"value\":").append(i).append("}\n");
-            }
-            bulk.append("\r\n");
-
-            final Request bulkRequest = new Request("POST", "/_bulk");
-            bulkRequest.setJsonEntity(bulk.toString());
-            client().performRequest(bulkRequest);
-
-            // create the rollup job
-            final Request createRollupJobRequest = new Request("PUT", "/_rollup/job/rollup-job-test");
-
-            String intervalType;
-            if (getOldClusterVersion().onOrAfter(Version.V_7_2_0)) {
-                intervalType = "fixed_interval";
-            } else {
-                intervalType = "interval";
-            }
-
-            createRollupJobRequest.setJsonEntity(formatted("""
-                {
-                  "index_pattern": "rollup-*",
-                  "rollup_index": "results-rollup",
-                  "cron": "*/30 * * * * ?",
-                  "page_size": 100,
-                  "groups": {
-                    "date_histogram": {
-                      "field": "timestamp",
-                      "%s": "5m"
-                    }
-                  },
-                  "metrics": [
-                    {
-                      "field": "value",
-                      "metrics": [ "min", "max", "sum" ]
-                    }
-                  ]
-                }""", intervalType));
-
-            Map<String, Object> createRollupJobResponse = entityAsMap(client().performRequest(createRollupJobRequest));
-            assertThat(createRollupJobResponse.get("acknowledged"), equalTo(Boolean.TRUE));
-
-            // start the rollup job
-            final Request startRollupJobRequest = new Request("POST", "/_rollup/job/rollup-job-test/_start");
-            Map<String, Object> startRollupJobResponse = entityAsMap(client().performRequest(startRollupJobRequest));
-            assertThat(startRollupJobResponse.get("started"), equalTo(Boolean.TRUE));
-
-            assertRollUpJob("rollup-job-test");
-
-        } else {
-
-            final Request clusterHealthRequest = new Request("GET", "/_cluster/health");
-            clusterHealthRequest.addParameter("wait_for_status", "yellow");
-            clusterHealthRequest.addParameter("wait_for_no_relocating_shards", "true");
-            clusterHealthRequest.addParameter("wait_for_no_initializing_shards", "true");
-            Map<String, Object> clusterHealthResponse = entityAsMap(client().performRequest(clusterHealthRequest));
-            assertThat(clusterHealthResponse.get("timed_out"), equalTo(Boolean.FALSE));
-
-            assertRollUpJob("rollup-job-test");
-        }
-    }
-
-    public void testTransformLegacyTemplateCleanup() throws Exception {
-        assumeTrue("Before 7.2 transforms didn't exist", getOldClusterVersion().onOrAfter(Version.V_7_2_0));
-        if (isRunningAgainstOldCluster()) {
-
-            // create the source index
-            final Request createIndexRequest = new Request("PUT", "customers");
-            createIndexRequest.setJsonEntity("""
-                {
-                  "mappings": {
-                    "properties": {
-                      "customer_id": {
-                        "type": "keyword"
-                      },
-                      "price": {
-                        "type": "double"
-                      }
-                    }
-                  }
-                }""");
-
-            Map<String, Object> createIndexResponse = entityAsMap(client().performRequest(createIndexRequest));
-            assertThat(createIndexResponse.get("acknowledged"), equalTo(Boolean.TRUE));
-
-            // create a transform
-            String endpoint = getOldClusterVersion().onOrAfter(Version.V_7_5_0)
-                ? "_transform/transform-full-cluster-restart-test"
-                : "_data_frame/transforms/transform-full-cluster-restart-test";
-            final Request createTransformRequest = new Request("PUT", endpoint);
-
-            createTransformRequest.setJsonEntity("""
-                {
-                  "source": {
-                    "index": "customers"
-                  },
-                  "description": "testing",
-                  "dest": {
-                    "index": "max_price"
-                  },
-                  "pivot": {
-                    "group_by": {
-                      "customer_id": {
-                        "terms": {
-                          "field": "customer_id"
-                        }
-                      }
-                    },
-                    "aggregations": {
-                      "max_price": {
-                        "max": {
-                          "field": "price"
-                        }
-                      }
-                    }
-                  }
-                }""");
-
-            Map<String, Object> createTransformResponse = entityAsMap(client().performRequest(createTransformRequest));
-            assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
-        } else {
-            // legacy index templates created in previous releases should not be present anymore
-            assertBusy(() -> {
-                Request request = new Request("GET", "/_template/.transform-*,.data-frame-*");
-                try {
-                    Response response = client().performRequest(request);
-                    Map<String, Object> responseLevel = entityAsMap(response);
-                    assertNotNull(responseLevel);
-                    assertThat(responseLevel.keySet(), empty());
-                } catch (ResponseException e) {
-                    // not found is fine
-                    assertThat(
-                        "Unexpected failure getting templates: " + e.getResponse().getStatusLine(),
-                        e.getResponse().getStatusLine().getStatusCode(),
-                        is(404)
-                    );
-                }
-            });
-        }
-    }
-
-    public void testSlmPolicyAndStats() throws IOException {
-        SnapshotLifecyclePolicy slmPolicy = new SnapshotLifecyclePolicy(
-            "test-policy",
-            "test-policy",
-            "* * * 31 FEB ? *",
-            "test-repo",
-            Collections.singletonMap("indices", Collections.singletonList("*")),
-            null
-        );
-        if (isRunningAgainstOldCluster() && getOldClusterVersion().onOrAfter(Version.V_7_4_0)) {
-            Request createRepoRequest = new Request("PUT", "_snapshot/test-repo");
-            String repoCreateJson = "{" + " \"type\": \"fs\"," + " \"settings\": {" + "   \"location\": \"test-repo\"" + "  }" + "}";
-            createRepoRequest.setJsonEntity(repoCreateJson);
-            Request createSlmPolicyRequest = new Request("PUT", "_slm/policy/test-policy");
-            try (XContentBuilder builder = JsonXContent.contentBuilder()) {
-                String createSlmPolicyJson = Strings.toString(slmPolicy.toXContent(builder, null));
-                createSlmPolicyRequest.setJsonEntity(createSlmPolicyJson);
-            }
-
-            client().performRequest(createRepoRequest);
-            client().performRequest(createSlmPolicyRequest);
-        }
-
-        if (isRunningAgainstOldCluster() == false && getOldClusterVersion().onOrAfter(Version.V_7_4_0)) {
-            Request getSlmPolicyRequest = new Request("GET", "_slm/policy/test-policy");
-            Response response = client().performRequest(getSlmPolicyRequest);
-            Map<String, Object> responseMap = entityAsMap(response);
-            Map<?, ?> policy = (Map<?, ?>) ((Map<?, ?>) responseMap.get("test-policy")).get("policy");
-            assertEquals(slmPolicy.getName(), policy.get("name"));
-            assertEquals(slmPolicy.getRepository(), policy.get("repository"));
-            assertEquals(slmPolicy.getSchedule(), policy.get("schedule"));
-            assertEquals(slmPolicy.getConfig(), policy.get("config"));
-        }
-
-        if (isRunningAgainstOldCluster() == false) {
-            Response response = client().performRequest(new Request("GET", "_slm/stats"));
-            XContentType xContentType = XContentType.fromMediaType(response.getEntity().getContentType().getValue());
-            try (
-                XContentParser parser = xContentType.xContent()
-                    .createParser(XContentParserConfiguration.EMPTY, response.getEntity().getContent())
-            ) {
-                assertEquals(new SnapshotLifecycleStats(), SnapshotLifecycleStats.parse(parser));
-            }
-        }
-    }
-
-    private String loadWatch(String watch) throws IOException {
-        return StreamsUtils.copyToStringFromClasspath("/org/elasticsearch/xpack/restart/" + watch);
-    }
-
-    private void assertOldTemplatesAreDeleted() throws IOException {
-        Map<String, Object> templates = entityAsMap(client().performRequest(new Request("GET", "/_template")));
-        assertThat(templates.keySet(), not(hasItems(is("watches"), startsWith("watch-history"), is("triggered_watches"))));
-    }
-
-    private void assertWatchIndexContentsWork() throws Exception {
-        // Fetch a basic watch
-        Request getRequest = new Request("GET", "_watcher/watch/bwc_watch");
-
-        Map<String, Object> bwcWatch = entityAsMap(client().performRequest(getRequest));
-
-        logger.error("-----> {}", bwcWatch);
-
-        assertThat(bwcWatch.get("found"), equalTo(true));
-        Map<?, ?> source = (Map<?, ?>) bwcWatch.get("watch");
-        assertEquals(1000, source.get("throttle_period_in_millis"));
-        int timeout = (int) timeValueSeconds(100).millis();
-        assertThat(ObjectPath.eval("input.search.timeout_in_millis", source), equalTo(timeout));
-        assertThat(ObjectPath.eval("actions.index_payload.transform.search.timeout_in_millis", source), equalTo(timeout));
-        assertThat(ObjectPath.eval("actions.index_payload.index.index", source), equalTo("bwc_watch_index"));
-        assertThat(ObjectPath.eval("actions.index_payload.index.timeout_in_millis", source), equalTo(timeout));
-
-        // Fetch a watch with "fun" throttle periods
-        getRequest = new Request("GET", "_watcher/watch/bwc_throttle_period");
-
-        bwcWatch = entityAsMap(client().performRequest(getRequest));
-        assertThat(bwcWatch.get("found"), equalTo(true));
-        source = (Map<?, ?>) bwcWatch.get("watch");
-        assertEquals(timeout, source.get("throttle_period_in_millis"));
-        assertThat(ObjectPath.eval("actions.index_payload.throttle_period_in_millis", source), equalTo(timeout));
-
-        /*
-         * Fetch a watch with a funny timeout to verify loading fractional time
-         * values.
-         */
-        bwcWatch = entityAsMap(client().performRequest(new Request("GET", "_watcher/watch/bwc_funny_timeout")));
-        assertThat(bwcWatch.get("found"), equalTo(true));
-        source = (Map<?, ?>) bwcWatch.get("watch");
-
-        Map<String, Object> attachments = ObjectPath.eval("actions.work.email.attachments", source);
-        Map<?, ?> attachment = (Map<?, ?>) attachments.get("test_report.pdf");
-        Map<String, Object> request = ObjectPath.eval("http.request", attachment);
-        assertEquals(timeout, request.get("read_timeout_millis"));
-        assertEquals("https", request.get("scheme"));
-        assertEquals("example.com", request.get("host"));
-        assertEquals("{{ctx.metadata.report_url}}", request.get("path"));
-        assertEquals(8443, request.get("port"));
-        Map<String, String> basic = ObjectPath.eval("auth.basic", request);
-        assertThat(basic, hasEntry("username", "Aladdin"));
-        // password doesn't come back because it is hidden
-        assertThat(basic, hasEntry(is("password"), anyOf(startsWith("::es_encrypted::"), is("::es_redacted::"))));
-        Request searchRequest = new Request("GET", ".watcher-history*/_search");
-        if (isRunningAgainstOldCluster() == false) {
-            searchRequest.addParameter(RestSearchAction.TOTAL_HITS_AS_INT_PARAM, "true");
-        }
-        Map<String, Object> history = entityAsMap(client().performRequest(searchRequest));
-        Map<?, ?> hits = (Map<?, ?>) history.get("hits");
-        assertThat((Integer) hits.get("total"), greaterThanOrEqualTo(2));
-    }
-
-    private void assertBasicWatchInteractions() throws Exception {
-        String watch = """
-            {
-              "trigger": {
-                "schedule": {
-                  "interval": "1s"
-                }
-              },
-              "input": {
-                "none": {}
-              },
-              "condition": {
-                "always": {}
-              },
-              "actions": {
-                "awesome": {
-                  "logging": {
-                    "level": "info",
-                    "text": "test"
-                  }
-                }
-              }
-            }""";
-        Request createWatchRequest = new Request("PUT", "_watcher/watch/new_watch");
-        createWatchRequest.setJsonEntity(watch);
-        Map<String, Object> createWatch = entityAsMap(client().performRequest(createWatchRequest));
-
-        logger.info("create watch {}", createWatch);
-
-        assertThat(createWatch.get("created"), equalTo(true));
-        assertThat(createWatch.get("_version"), equalTo(1));
-
-        Map<String, Object> updateWatch = entityAsMap(client().performRequest(createWatchRequest));
-        assertThat(updateWatch.get("created"), equalTo(false));
-        assertThat(updateWatch.get("_version"), equalTo(2));
-
-        Map<String, Object> get = entityAsMap(client().performRequest(new Request("GET", "_watcher/watch/new_watch")));
-        assertThat(get.get("found"), equalTo(true));
-        Map<?, ?> source = (Map<?, ?>) get.get("watch");
-        Map<String, Object> logging = ObjectPath.eval("actions.awesome.logging", source);
-        assertEquals("info", logging.get("level"));
-        assertEquals("test", logging.get("text"));
-    }
-
-    private void waitForYellow(String indexName) throws IOException {
-        Request request = new Request("GET", "/_cluster/health/" + indexName);
-        request.addParameter("wait_for_status", "yellow");
-        request.addParameter("timeout", "30s");
-        request.addParameter("wait_for_no_relocating_shards", "true");
-        request.addParameter("wait_for_no_initializing_shards", "true");
-        Map<String, Object> response = entityAsMap(client().performRequest(request));
-        assertThat(response.get("timed_out"), equalTo(Boolean.FALSE));
-    }
-
-    private void waitForHits(String indexName, int expectedHits) throws Exception {
-        Request request = new Request("GET", "/" + indexName + "/_search");
-        request.addParameter("ignore_unavailable", "true");
-        request.addParameter("size", "0");
-        assertBusy(() -> {
-            try {
-                Map<String, Object> response = entityAsMap(client().performRequest(request));
-                Map<?, ?> hits = (Map<?, ?>) response.get("hits");
-                logger.info("Hits are: {}", hits);
-                Integer total;
-                if (getOldClusterVersion().onOrAfter(Version.V_7_0_0) || isRunningAgainstOldCluster() == false) {
-                    total = (Integer) ((Map<?, ?>) hits.get("total")).get("value");
-                } else {
-                    total = (Integer) hits.get("total");
-                }
-                assertThat(total, greaterThanOrEqualTo(expectedHits));
-            } catch (IOException ioe) {
-                if (ioe instanceof ResponseException) {
-                    Response response = ((ResponseException) ioe).getResponse();
-                    if (RestStatus.fromCode(response.getStatusLine().getStatusCode()) == RestStatus.SERVICE_UNAVAILABLE) {
-                        fail("shards are not yet active");
-                    }
-                }
-                throw ioe;
-            }
-        }, 30, TimeUnit.SECONDS);
-    }
-
-    private void startWatcher() throws Exception {
-        Map<String, Object> startWatchResponse = entityAsMap(client().performRequest(new Request("POST", "_watcher/_start")));
-        assertThat(startWatchResponse.get("acknowledged"), equalTo(Boolean.TRUE));
-        assertBusy(() -> {
-            Map<String, Object> statsWatchResponse = entityAsMap(client().performRequest(new Request("GET", "_watcher/stats")));
-            List<?> states = ((List<?>) statsWatchResponse.get("stats")).stream()
-                .map(o -> ((Map<?, ?>) o).get("watcher_state"))
-                .collect(Collectors.toList());
-            assertThat(states, everyItem(is("started")));
-        });
-    }
-
-    private void stopWatcher() throws Exception {
-        Map<String, Object> stopWatchResponse = entityAsMap(client().performRequest(new Request("POST", "_watcher/_stop")));
-        assertThat(stopWatchResponse.get("acknowledged"), equalTo(Boolean.TRUE));
-        assertBusy(() -> {
-            Map<String, Object> statsStoppedWatchResponse = entityAsMap(client().performRequest(new Request("GET", "_watcher/stats")));
-            List<?> states = ((List<?>) statsStoppedWatchResponse.get("stats")).stream()
-                .map(o -> ((Map<?, ?>) o).get("watcher_state"))
-                .collect(Collectors.toList());
-            assertThat(states, everyItem(is("stopped")));
-        });
-    }
-
-    static String toStr(Response response) throws IOException {
-        return EntityUtils.toString(response.getEntity());
-    }
-
-    private void createUser(final boolean oldCluster) throws Exception {
-        final String id = oldCluster ? "preupgrade_user" : "postupgrade_user";
-        Request request = new Request("PUT", "/_security/user/" + id);
-        request.setJsonEntity(formatted("""
-            {
-               "password" : "l0ng-r4nd0m-p@ssw0rd",
-               "roles" : [ "admin", "other_role1" ],
-               "full_name" : "%s",
-               "email" : "%s@example.com",
-               "enabled": true
-            }""", randomAlphaOfLength(5), id));
-        client().performRequest(request);
-    }
-
-    private void createRole(final boolean oldCluster) throws Exception {
-        final String id = oldCluster ? "preupgrade_role" : "postupgrade_role";
-        Request request = new Request("PUT", "/_security/role/" + id);
-        request.setJsonEntity("""
-            {
-              "run_as": [ "abc" ],
-              "cluster": [ "monitor" ],
-              "indices": [
-                {
-                  "names": [ "events-*" ],
-                  "privileges": [ "read" ],
-                  "field_security" : {
-                    "grant" : [ "category", "@timestamp", "message" ]
-                  },
-                  "query": "{\\"match\\": {\\"category\\": \\"click\\"}}"
-                }
-              ]
-            }""");
-        client().performRequest(request);
-    }
-
-    private void assertUserInfo(final boolean oldCluster) throws Exception {
-        final String user = oldCluster ? "preupgrade_user" : "postupgrade_user";
-        Request request = new Request("GET", "/_security/user/" + user);
-        ;
-        Map<String, Object> response = entityAsMap(client().performRequest(request));
-        Map<?, ?> userInfo = (Map<?, ?>) response.get(user);
-        assertEquals(user + "@example.com", userInfo.get("email"));
-        assertNotNull(userInfo.get("full_name"));
-        assertNotNull(userInfo.get("roles"));
-    }
-
-    private void assertRoleInfo(final boolean oldCluster) throws Exception {
-        final String role = oldCluster ? "preupgrade_role" : "postupgrade_role";
-        Map<?, ?> response = (Map<?, ?>) entityAsMap(client().performRequest(new Request("GET", "/_security/role/" + role))).get(role);
-        assertNotNull(response.get("run_as"));
-        assertNotNull(response.get("cluster"));
-        assertNotNull(response.get("indices"));
-    }
-
-    private void assertRollUpJob(final String rollupJob) throws Exception {
-        final Matcher<?> expectedStates = anyOf(equalTo("indexing"), equalTo("started"));
-        waitForRollUpJob(rollupJob, expectedStates);
-
-        // check that the rollup job is started using the RollUp API
-        final Request getRollupJobRequest = new Request("GET", "_rollup/job/" + rollupJob);
-        Map<String, Object> getRollupJobResponse = entityAsMap(client().performRequest(getRollupJobRequest));
-        Map<?, ?> job = getJob(getRollupJobResponse, rollupJob);
-        assertNotNull(job);
-        assertThat(ObjectPath.eval("status.job_state", job), expectedStates);
-
-        // check that the rollup job is started using the Tasks API
-        final Request taskRequest = new Request("GET", "_tasks");
-        taskRequest.addParameter("detailed", "true");
-        taskRequest.addParameter("actions", "xpack/rollup/*");
-        Map<String, Object> taskResponse = entityAsMap(client().performRequest(taskRequest));
-        Map<?, ?> taskResponseNodes = (Map<?, ?>) taskResponse.get("nodes");
-        Map<?, ?> taskResponseNode = (Map<?, ?>) taskResponseNodes.values().iterator().next();
-        Map<?, ?> taskResponseTasks = (Map<?, ?>) taskResponseNode.get("tasks");
-        Map<?, ?> taskResponseStatus = (Map<?, ?>) taskResponseTasks.values().iterator().next();
-        assertThat(ObjectPath.eval("status.job_state", taskResponseStatus), expectedStates);
-
-        // check that the rollup job is started using the Cluster State API
-        final Request clusterStateRequest = new Request("GET", "_cluster/state/metadata");
-        Map<String, Object> clusterStateResponse = entityAsMap(client().performRequest(clusterStateRequest));
-        List<Map<String, Object>> rollupJobTasks = ObjectPath.eval("metadata.persistent_tasks.tasks", clusterStateResponse);
-
-        boolean hasRollupTask = false;
-        for (Map<String, Object> task : rollupJobTasks) {
-            if (ObjectPath.eval("id", task).equals(rollupJob)) {
-                hasRollupTask = true;
-                final String jobStateField = "task.xpack/rollup/job.state.job_state";
-                assertThat(
-                    "Expected field [" + jobStateField + "] to be started or indexing in " + task.get("id"),
-                    ObjectPath.eval(jobStateField, task),
-                    expectedStates
-                );
-                break;
-            }
-        }
-        if (hasRollupTask == false) {
-            fail("Expected persistent task for [" + rollupJob + "] but none found.");
-        }
-    }
-
-    private void waitForRollUpJob(final String rollupJob, final Matcher<?> expectedStates) throws Exception {
-        assertBusy(() -> {
-            final Request getRollupJobRequest = new Request("GET", "/_rollup/job/" + rollupJob);
-
-            Response getRollupJobResponse = client().performRequest(getRollupJobRequest);
-            assertThat(getRollupJobResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
-
-            Map<?, ?> job = getJob(getRollupJobResponse, rollupJob);
-            assertNotNull(job);
-            assertThat(ObjectPath.eval("status.job_state", job), expectedStates);
-        }, 30L, TimeUnit.SECONDS);
-    }
-
-    private Map<?, ?> getJob(Response response, String targetJobId) throws IOException {
-        return getJob(ESRestTestCase.entityAsMap(response), targetJobId);
-    }
-
-    private Map<?, ?> getJob(Map<String, Object> jobsMap, String targetJobId) throws IOException {
-        List<?> jobs = (List<?>) XContentMapValues.extractValue("jobs", jobsMap);
-        if (jobs == null) {
-            return null;
-        }
-
-        for (Object entry : jobs) {
-            Map<?, ?> job = (Map<?, ?>) entry;
-            String jobId = (String) ((Map<?, ?>) job.get("config")).get("id");
-            if (jobId.equals(targetJobId)) {
-                return job;
-            }
-        }
-        return null;
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testDataStreams() throws Exception {
-        assumeTrue("no data streams in versions before " + Version.V_7_9_0, getOldClusterVersion().onOrAfter(Version.V_7_9_0));
-        if (isRunningAgainstOldCluster()) {
-            createComposableTemplate(client(), "dst", "ds");
-
-            Request indexRequest = new Request("POST", "/ds/_doc/1?op_type=create&refresh");
-            XContentBuilder builder = JsonXContent.contentBuilder()
-                .startObject()
-                .field("f", "v")
-                .field("@timestamp", System.currentTimeMillis())
-                .endObject();
-            indexRequest.setJsonEntity(Strings.toString(builder));
-            assertOK(client().performRequest(indexRequest));
-        }
-
-        // It's quite possible that this test will run where the data stream backing index is
-        // created on one day, and then checked on a subsequent day. To avoid this failing the
-        // test, we store the timestamp used when the document is indexed, then when we go to
-        // check the backing index name, we retrieve the time and use it for the backing index
-        // name resolution.
-        Request getDoc = new Request("GET", "/ds/_search");
-        Map<String, Object> doc = entityAsMap(client().performRequest(getDoc));
-        logger.info("--> doc: {}", doc);
-        Map<String, Object> hits = (Map<String, Object>) doc.get("hits");
-        Map<String, Object> docBody = (Map<String, Object>) ((List<Object>) hits.get("hits")).get(0);
-        Long timestamp = (Long) ((Map<String, Object>) docBody.get("_source")).get("@timestamp");
-        logger.info("--> parsed out timestamp of {}", timestamp);
-
-        Request getDataStream = new Request("GET", "/_data_stream/ds");
-        Response response = client().performRequest(getDataStream);
-        assertOK(response);
-        List<Object> dataStreams = (List<Object>) entityAsMap(response).get("data_streams");
-        assertEquals(1, dataStreams.size());
-        Map<String, Object> ds = (Map<String, Object>) dataStreams.get(0);
-        List<Map<String, String>> indices = (List<Map<String, String>>) ds.get("indices");
-        assertEquals("ds", ds.get("name"));
-        assertEquals(1, indices.size());
-        assertEquals(
-            DataStreamTestHelper.getLegacyDefaultBackingIndexName("ds", 1, timestamp, getOldClusterVersion()),
-            indices.get(0).get("index_name")
-        );
-        assertNumHits("ds", 1, 1);
-    }
-
-    private static void createComposableTemplate(RestClient client, String templateName, String indexPattern) throws IOException {
-        StringEntity templateJSON = new StringEntity(formatted("""
-            {
-              "index_patterns": "%s",
-              "data_stream": {}
-            }""", indexPattern), ContentType.APPLICATION_JSON);
-        Request createIndexTemplateRequest = new Request("PUT", "_index_template/" + templateName);
-        createIndexTemplateRequest.setEntity(templateJSON);
-        client.performRequest(createIndexTemplateRequest);
-    }
-}
diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java
deleted file mode 100644
index b4e3cbe0b1d7..000000000000
--- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.restart;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.client.WarningsHandler;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.common.xcontent.support.XContentMapValues;
-import org.elasticsearch.core.Tuple;
-import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase;
-import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xcontent.XContentParserConfiguration;
-import org.elasticsearch.xcontent.spi.XContentProvider;
-import org.elasticsearch.xpack.test.rest.XPackRestTestConstants;
-import org.elasticsearch.xpack.test.rest.XPackRestTestHelper;
-import org.junit.Before;
-
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-import java.util.Base64;
-import java.util.List;
-import java.util.Map;
-
-import static org.hamcrest.Matchers.aMapWithSize;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-
-public class MlHiddenIndicesFullClusterRestartIT extends AbstractFullClusterRestartTestCase {
-
-    private static final String JOB_ID = "ml-hidden-indices-old-cluster-job";
-    private static final List<Tuple<List<String>, String>> EXPECTED_INDEX_ALIAS_PAIRS = List.of(
-        Tuple.tuple(List.of(".ml-annotations-000001"), ".ml-annotations-read"),
-        Tuple.tuple(List.of(".ml-annotations-000001"), ".ml-annotations-write"),
-        Tuple.tuple(List.of(".ml-state", ".ml-state-000001"), ".ml-state-write"),
-        Tuple.tuple(List.of(".ml-anomalies-shared"), ".ml-anomalies-" + JOB_ID),
-        Tuple.tuple(List.of(".ml-anomalies-shared"), ".ml-anomalies-.write-" + JOB_ID)
-    );
-
-    @Override
-    protected Settings restClientSettings() {
-        String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8));
-        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
-    }
-
-    @Before
-    public void waitForMlTemplates() throws Exception {
-        // We shouldn't wait for ML templates during the upgrade - production won't
-        if (isRunningAgainstOldCluster()) {
-            XPackRestTestHelper.waitForTemplates(
-                client(),
-                XPackRestTestConstants.ML_POST_V7120_TEMPLATES,
-                getOldClusterVersion().onOrAfter(Version.V_7_8_0)
-            );
-        }
-    }
-
-    public void testMlIndicesBecomeHidden() throws Exception {
-        if (isRunningAgainstOldCluster()) {
-            // trigger ML indices creation
-            createAnomalyDetectorJob(JOB_ID);
-            openAnomalyDetectorJob(JOB_ID);
-
-            if (getOldClusterVersion().before(Version.V_7_7_0)) {
-                Map<String, Object> indexSettingsMap = contentAsMap(getMlIndicesSettings());
-                Map<String, Object> aliasesMap = contentAsMap(getMlAliases());
-
-                assertThat("Index settings map was: " + indexSettingsMap, indexSettingsMap, is(aMapWithSize(greaterThanOrEqualTo(4))));
-                for (Map.Entry<String, Object> e : indexSettingsMap.entrySet()) {
-                    String indexName = e.getKey();
-                    @SuppressWarnings("unchecked")
-                    Map<String, Object> settings = (Map<String, Object>) e.getValue();
-                    assertThat(settings, is(notNullValue()));
-                    assertThat(
-                        "Index " + indexName + " expected not to be hidden but was, settings = " + settings,
-                        XContentMapValues.extractValue(settings, "settings", "index", "hidden"),
-                        is(nullValue())
-                    );
-                }
-
-                for (Tuple<List<String>, String> indexAndAlias : EXPECTED_INDEX_ALIAS_PAIRS) {
-                    List<String> indices = indexAndAlias.v1();
-                    String alias = indexAndAlias.v2();
-                    for (String index : indices) {
-                        assertThat(
-                            indexAndAlias + " expected not be hidden but was, aliasesMap = " + aliasesMap,
-                            XContentMapValues.extractValue(aliasesMap, index, "aliases", alias, "is_hidden"),
-                            is(nullValue())
-                        );
-                    }
-                }
-            }
-        } else {
-            Map<String, Object> indexSettingsMap = contentAsMap(getMlIndicesSettings());
-            Map<String, Object> aliasesMap = contentAsMap(getMlAliases());
-
-            assertThat("Index settings map was: " + indexSettingsMap, indexSettingsMap, is(aMapWithSize(greaterThanOrEqualTo(4))));
-            for (Map.Entry<String, Object> e : indexSettingsMap.entrySet()) {
-                String indexName = e.getKey();
-                @SuppressWarnings("unchecked")
-                Map<String, Object> settings = (Map<String, Object>) e.getValue();
-                assertThat(settings, is(notNullValue()));
-                assertThat(
-                    "Index " + indexName + " expected to be hidden but wasn't, settings = " + settings,
-                    XContentMapValues.extractValue(settings, "settings", "index", "hidden"),
-                    is(equalTo("true"))
-                );
-            }
-
-            for (Tuple<List<String>, String> indexAndAlias : EXPECTED_INDEX_ALIAS_PAIRS) {
-                List<String> indices = indexAndAlias.v1();
-                String alias = indexAndAlias.v2();
-                assertThat(
-                    indexAndAlias + " expected to be hidden but wasn't, aliasesMap = " + aliasesMap,
-                    indices.stream()
-                        .anyMatch(
-                            index -> Boolean.TRUE.equals(XContentMapValues.extractValue(aliasesMap, index, "aliases", alias, "is_hidden"))
-                        ),
-                    is(true)
-                );
-            }
-        }
-    }
-
-    private Response getMlIndicesSettings() throws IOException {
-        Request getSettingsRequest = new Request(
-            "GET",
-            ".ml-anomalies-*,.ml-state*,.ml-stats-*,.ml-notifications*,.ml-annotations*/_settings"
-        );
-        getSettingsRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE).build());
-        Response getSettingsResponse = client().performRequest(getSettingsRequest);
-        assertThat(getSettingsResponse, is(notNullValue()));
-        return getSettingsResponse;
-    }
-
-    private Response getMlAliases() throws IOException {
-        Request getAliasesRequest = new Request("GET", ".ml-anomalies-*,.ml-state*,.ml-stats-*,.ml-notifications*,.ml-annotations*/_alias");
-        getAliasesRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE).build());
-        Response getAliasesResponse = client().performRequest(getAliasesRequest);
-        assertThat(getAliasesResponse, is(notNullValue()));
-        return getAliasesResponse;
-    }
-
-    @SuppressWarnings("unchecked")
-    private static Map<String, Object> contentAsMap(Response response) throws IOException {
-        InputStreamReader reader = new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8);
-        XContentParser parser = XContentProvider.provider()
-            .getJsonXContent()
-            .XContent()
-            .createParser(XContentParserConfiguration.EMPTY, reader);
-        return parser.map();
-    }
-
-    private void createAnomalyDetectorJob(String jobId) throws IOException {
-        String jobConfig = formatted("""
-            {
-                "job_id": "%s",
-                "analysis_config": {
-                    "bucket_span": "10m",
-                    "detectors": [{
-                        "function": "metric",
-                        "field_name": "responsetime"
-                    }]
-                },
-                "data_description": {}
-            }""", jobId);
-
-        Request putJobRequest = new Request("PUT", "/_ml/anomaly_detectors/" + jobId);
-        putJobRequest.setJsonEntity(jobConfig);
-        Response putJobResponse = client().performRequest(putJobRequest);
-        assertThat(putJobResponse.getStatusLine().getStatusCode(), equalTo(200));
-    }
-
-    private void openAnomalyDetectorJob(String jobId) throws IOException {
-        Request openJobRequest = new Request("POST", "/_ml/anomaly_detectors/" + jobId + "/_open");
-        Response openJobResponse = client().performRequest(openJobRequest);
-        assertThat(openJobResponse.getStatusLine().getStatusCode(), equalTo(200));
-    }
-}
diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/QueryBuilderBWCIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/QueryBuilderBWCIT.java
deleted file mode 100644
index cffc6881df64..000000000000
--- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/QueryBuilderBWCIT.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-package org.elasticsearch.xpack.restart;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-
-import java.nio.charset.StandardCharsets;
-import java.util.Base64;
-
-public class QueryBuilderBWCIT extends org.elasticsearch.upgrades.QueryBuilderBWCIT {
-
-    @Override
-    protected Settings restClientSettings() {
-        String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8));
-        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
-    }
-}
diff --git a/x-pack/qa/kerberos-tests/build.gradle b/x-pack/qa/kerberos-tests/build.gradle
index b04cdc8078d8..536758dcfb71 100644
--- a/x-pack/qa/kerberos-tests/build.gradle
+++ b/x-pack/qa/kerberos-tests/build.gradle
@@ -1,7 +1,7 @@
 import java.nio.file.Path
 import java.nio.file.Paths
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
 
 testFixtures.useFixture ":test:fixtures:krb5kdc-fixture", "peppa"
diff --git a/x-pack/qa/kerberos-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java b/x-pack/qa/kerberos-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java
index 3477f356078c..4e2d37fda209 100644
--- a/x-pack/qa/kerberos-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java
+++ b/x-pack/qa/kerberos-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java
@@ -133,7 +133,7 @@ public void testGetOauth2TokenInExchangeForKerberosTickets() throws PrivilegedAc
         final String kerberosTicket = callbackHandler.getBase64EncodedTokenForSpnegoHeader(host);
 
         final Request request = new Request("POST", "/_security/oauth2/token");
-        String json = formatted("""
+        String json = Strings.format("""
             { "grant_type" : "_kerberos", "kerberos_ticket" : "%s"}
             """, kerberosTicket);
         request.setJsonEntity(json);
diff --git a/x-pack/qa/mixed-tier-cluster/build.gradle b/x-pack/qa/mixed-tier-cluster/build.gradle
index c94175dcfc75..862b33a5402b 100644
--- a/x-pack/qa/mixed-tier-cluster/build.gradle
+++ b/x-pack/qa/mixed-tier-cluster/build.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.bwc-test'
 
 import org.elasticsearch.gradle.VersionProperties
diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle
index 9bd526c3616a..f9d60181fc81 100644
--- a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle
+++ b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/build.gradle
@@ -12,7 +12,7 @@ restResources {
 }
 
 // randomise between sniff and proxy modes
-boolean proxyMode = (new Random(Long.parseUnsignedLong(BuildParams.testSeed.tokenize(':').get(0), 16))).nextBoolean()
+boolean proxyMode = BuildParams.random.nextBoolean()
 
 def fulfillingCluster = testClusters.register('fulfilling-cluster') {
   setting 'xpack.security.enabled', 'true'
@@ -55,8 +55,8 @@ tasks.register('fulfilling-cluster', RestIntegTestTask) {
 
 tasks.register('querying-cluster', RestIntegTestTask) {
   dependsOn 'fulfilling-cluster'
-  useCluster queryingCluster
   useCluster fulfillingCluster
+  useCluster queryingCluster
   systemProperty 'tests.rest.suite', 'querying_cluster'
   if (proxyMode) {
     systemProperty 'tests.rest.blacklist', [
diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-full-license/build.gradle b/x-pack/qa/multi-cluster-search-security/legacy-with-full-license/build.gradle
index 07701a46d5c2..35b7a981fca3 100644
--- a/x-pack/qa/multi-cluster-search-security/legacy-with-full-license/build.gradle
+++ b/x-pack/qa/multi-cluster-search-security/legacy-with-full-license/build.gradle
@@ -12,7 +12,7 @@ restResources {
 }
 
 // randomise between sniff and proxy modes
-boolean proxyMode = (new Random(Long.parseUnsignedLong(BuildParams.testSeed.tokenize(':').get(0), 16))).nextBoolean()
+boolean proxyMode = BuildParams.random.nextBoolean()
 
 def fulfillingCluster = testClusters.register('fulfilling-cluster') {
   setting 'xpack.security.enabled', 'true'
diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-restricted-trust/build.gradle b/x-pack/qa/multi-cluster-search-security/legacy-with-restricted-trust/build.gradle
new file mode 100644
index 000000000000..810895abe1e8
--- /dev/null
+++ b/x-pack/qa/multi-cluster-search-security/legacy-with-restricted-trust/build.gradle
@@ -0,0 +1,94 @@
+import org.elasticsearch.gradle.internal.info.BuildParams
+import org.elasticsearch.gradle.internal.test.RestIntegTestTask
+
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-resources'
+
+configurations {
+  signedCerts
+  rootCert
+}
+
+dependencies {
+  signedCerts project(path: ':x-pack:plugin:core', configuration: 'signedCerts')
+  rootCert project(path: ':x-pack:plugin:core', configuration: 'rootCert')
+}
+
+tasks.register("copyCerts", Sync) {
+  dependsOn configurations.signedCerts
+  from(configurations.signedCerts)
+  from(configurations.rootCert)
+  into "${buildDir}/certs"
+}
+
+// randomise between sniff and proxy modes
+boolean proxyMode = BuildParams.random.nextBoolean()
+
+def fulfillingCluster = testClusters.register('fulfilling-cluster') {
+  setting 'xpack.security.enabled', 'true'
+  setting 'xpack.license.self_generated.type', 'basic'
+  setting 'xpack.security.transport.ssl.enabled', 'true'
+  setting 'xpack.security.transport.ssl.client_authentication', 'required'
+  extraConfigFile 'transport.key', file("${buildDir}/certs/n1.c1.key")
+  extraConfigFile 'transport.cert', file("${buildDir}/certs/n1.c1.crt")
+  extraConfigFile 'transport.ca', file("${buildDir}/certs/ca.crt")
+  extraConfigFile 'trust.yml', file("${buildDir}/resources/test/trust.yml")
+  setting 'xpack.security.transport.ssl.key', 'transport.key'
+  setting 'xpack.security.transport.ssl.certificate', 'transport.cert'
+  setting 'xpack.security.transport.ssl.certificate_authorities', 'transport.ca'
+  setting 'xpack.security.transport.ssl.verification_mode', 'certificate'
+  setting 'xpack.security.transport.ssl.trust_restrictions.path', 'trust.yml'
+  user username: "test_user", password: "x-pack-test-password"
+}
+
+def queryingCluster = testClusters.register('querying-cluster') {
+  setting 'xpack.security.enabled', 'true'
+  setting 'xpack.license.self_generated.type', 'basic'
+  setting 'xpack.security.transport.ssl.enabled', 'true'
+  setting 'xpack.security.transport.ssl.client_authentication', 'required'
+  extraConfigFile 'transport.key', file("${buildDir}/certs/n1.c2.key")
+  extraConfigFile 'transport.cert', file("${buildDir}/certs/n1.c2.crt")
+  extraConfigFile 'transport.ca', file("${buildDir}/certs/ca.crt")
+  extraConfigFile 'trust.yml', file("${buildDir}/resources/test/trust.yml")
+  setting 'xpack.security.transport.ssl.key', 'transport.key'
+  setting 'xpack.security.transport.ssl.certificate', 'transport.cert'
+  setting 'xpack.security.transport.ssl.certificate_authorities', 'transport.ca'
+  setting 'xpack.security.transport.ssl.verification_mode', 'certificate'
+  setting 'xpack.security.transport.ssl.trust_restrictions.path', 'trust.yml'
+  setting 'xpack.security.transport.ssl.trust_restrictions.x509_fields', 'subjectAltName.dnsName'
+  setting 'cluster.remote.connections_per_cluster', "1"
+  user username: "test_user", password: "x-pack-test-password"
+
+  if (proxyMode) {
+    setting 'cluster.remote.my_remote_cluster.mode', 'proxy'
+    setting 'cluster.remote.my_remote_cluster.proxy_address', {
+      "\"${fulfillingCluster.get().getAllTransportPortURI().get(0)}\""
+    }
+  } else {
+    setting 'cluster.remote.my_remote_cluster.seeds', {
+      fulfillingCluster.get().getAllTransportPortURI().collect { "\"$it\"" }.toString()
+    }
+  }
+}
+
+tasks.register('fulfilling-cluster', RestIntegTestTask) {
+  dependsOn 'copyCerts'
+  useCluster fulfillingCluster
+  systemProperty 'tests.rest.suite', 'fulfilling_cluster'
+}
+
+tasks.register('querying-cluster', RestIntegTestTask) {
+  dependsOn 'copyCerts'
+  dependsOn 'fulfilling-cluster'
+  useCluster queryingCluster
+  useCluster fulfillingCluster
+  systemProperty 'tests.rest.suite', 'querying_cluster'
+}
+
+// runs the fulfilling-cluster cluster tests then the querying-cluster tests
+tasks.register("integTest") {
+  dependsOn 'copyCerts'
+  dependsOn 'querying-cluster'
+}
+
+tasks.named("check").configure { dependsOn("integTest") }
diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-restricted-trust/src/test/java/org/elasticsearch/xpack/remotecluster/LegacyRemoteClusterSecuritySmokeIT.java b/x-pack/qa/multi-cluster-search-security/legacy-with-restricted-trust/src/test/java/org/elasticsearch/xpack/remotecluster/LegacyRemoteClusterSecuritySmokeIT.java
new file mode 100644
index 000000000000..f070b497251a
--- /dev/null
+++ b/x-pack/qa/multi-cluster-search-security/legacy-with-restricted-trust/src/test/java/org/elasticsearch/xpack/remotecluster/LegacyRemoteClusterSecuritySmokeIT.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.remotecluster;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.ObjectPath;
+
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * This test suite will be run twice: Once against the fulfilling cluster, then again against the querying cluster.
+ */
+public class LegacyRemoteClusterSecuritySmokeIT extends ESRestTestCase {
+
+    private static final String USER = "test_user";
+    private static final SecureString PASS = new SecureString("x-pack-test-password".toCharArray());
+
+    @Override
+    protected boolean preserveIndicesUponCompletion() {
+        return true;
+    }
+
+    @Override
+    protected boolean preserveDataStreamsUponCompletion() {
+        return true;
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue(USER, PASS);
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+
+    private boolean isFulfillingCluster() {
+        return "fulfilling_cluster".equals(System.getProperty("tests.rest.suite"));
+    }
+
+    /**
+     * This test really depends on the local build.gradle, which configures cross-cluster search using the `remote_cluster.*` settings.
+     */
+    public void testRemoteAccessPortFunctions() throws Exception {
+        if (isFulfillingCluster()) {
+            // Index some documents, so we can search them from the querying cluster
+            Request indexDocRequest = new Request("POST", "/test_idx/_doc");
+            indexDocRequest.setJsonEntity("{\"foo\": \"bar\"}");
+            Response response = client().performRequest(indexDocRequest);
+            assertOK(response);
+
+        } else {
+            // Check that we can search the fulfilling cluster from the querying cluster
+            Request searchRequest = new Request("GET", "/my_remote_cluster:test_idx/_search");
+            Response response = client().performRequest(searchRequest);
+            assertOK(response);
+            ObjectPath responseObj = ObjectPath.createFromResponse(response);
+            int totalHits = responseObj.evaluate("hits.total.value");
+            assertThat(totalHits, equalTo(1));
+        }
+    }
+}
diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-restricted-trust/src/test/resources/trust.yml b/x-pack/qa/multi-cluster-search-security/legacy-with-restricted-trust/src/test/resources/trust.yml
new file mode 100644
index 000000000000..64b16a00e353
--- /dev/null
+++ b/x-pack/qa/multi-cluster-search-security/legacy-with-restricted-trust/src/test/resources/trust.yml
@@ -0,0 +1,2 @@
+trust.subject_name:
+  - "*.elasticsearch"
diff --git a/x-pack/qa/multi-node/build.gradle b/x-pack/qa/multi-node/build.gradle
index 16aa1fc50053..f4af52b99459 100644
--- a/x-pack/qa/multi-node/build.gradle
+++ b/x-pack/qa/multi-node/build.gradle
@@ -2,16 +2,5 @@ apply plugin: 'elasticsearch.internal-java-rest-test'
 
 dependencies {
   javaRestTestImplementation project(':x-pack:qa')
-}
-
-testClusters.matching { it.name == "javaRestTest" }.configureEach {
-  testDistribution = 'DEFAULT'
-  numberOfNodes = 2
-  setting 'xpack.security.enabled', 'true'
-  setting 'xpack.watcher.enabled', 'false'
-  setting 'xpack.ml.enabled', 'false'
-  setting 'xpack.license.self_generated.type', 'trial'
-  rolesFile file('roles.yml')
-  user username: "test-user", password: "x-pack-test-password", role: "test"
-  user username: "super-user", password: "x-pack-super-password"
+  clusterModules project(xpackModule('rollup'))
 }
diff --git a/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/GlobalCheckpointSyncActionIT.java b/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/GlobalCheckpointSyncActionIT.java
index f42b4681ed19..12cfa8e48b3e 100644
--- a/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/GlobalCheckpointSyncActionIT.java
+++ b/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/GlobalCheckpointSyncActionIT.java
@@ -12,14 +12,28 @@
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.util.resource.Resource;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.ObjectPath;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.junit.ClassRule;
 
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.equalTo;
 
 public class GlobalCheckpointSyncActionIT extends ESRestTestCase {
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .nodes(2)
+        .setting("xpack.security.enabled", "true")
+        .setting("xpack.watcher.enabled", "false")
+        .setting("xpack.ml.enabled", "false")
+        .setting("xpack.license.self_generated.type", "trial")
+        .rolesFile(Resource.fromClasspath("roles.yml"))
+        .user("test-user", "x-pack-test-password", "test")
+        .user("super-user", "x-pack-super-password")
+        .build();
 
     @Override
     protected Settings restClientSettings() {
@@ -31,6 +45,11 @@ protected Settings restAdminSettings() {
         return getClientSettings("super-user", "x-pack-super-password");
     }
 
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
     private Settings getClientSettings(final String username, final String password) {
         final String token = basicAuthHeaderValue(username, new SecureString(password.toCharArray()));
         return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
diff --git a/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/RollupIT.java b/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/RollupIT.java
index 0e4fea305cfb..91e273f7e34b 100644
--- a/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/RollupIT.java
+++ b/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/RollupIT.java
@@ -16,9 +16,11 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xcontent.ObjectPath;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.junit.ClassRule;
 
 import java.io.IOException;
 import java.time.Instant;
@@ -35,6 +37,16 @@
 import static org.hamcrest.Matchers.oneOf;
 
 public class RollupIT extends ESRestTestCase {
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .nodes(2)
+        .module("x-pack-rollup")
+        .setting("xpack.security.enabled", "true")
+        .setting("xpack.watcher.enabled", "false")
+        .setting("xpack.ml.enabled", "false")
+        .setting("xpack.license.self_generated.type", "trial")
+        .user("super-user", "x-pack-super-password")
+        .build();
 
     @Override
     protected Settings restClientSettings() {
@@ -46,6 +58,11 @@ protected Settings restAdminSettings() {
         return getClientSettings("super-user", "x-pack-super-password");
     }
 
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
     private Settings getClientSettings(final String username, final String password) {
         final String token = basicAuthHeaderValue(username, new SecureString(password.toCharArray()));
         return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
@@ -97,7 +114,7 @@ public void testBigRollup() throws Exception {
         final Request createRollupJobRequest = new Request("PUT", "/_rollup/job/rollup-job-test");
         int pageSize = randomIntBetween(2, 50);
         // fast cron so test runs quickly
-        createRollupJobRequest.setJsonEntity(formatted("""
+        createRollupJobRequest.setJsonEntity(Strings.format("""
             {
                 "index_pattern": "rollup-*",
                 "rollup_index": "results-rollup",
diff --git a/x-pack/qa/multi-node/roles.yml b/x-pack/qa/multi-node/src/javaRestTest/resources/roles.yml
similarity index 100%
rename from x-pack/qa/multi-node/roles.yml
rename to x-pack/qa/multi-node/src/javaRestTest/resources/roles.yml
diff --git a/x-pack/qa/oidc-op-tests/build.gradle b/x-pack/qa/oidc-op-tests/build.gradle
index d9394a8b3ce3..aa4c1e2d0d22 100644
--- a/x-pack/qa/oidc-op-tests/build.gradle
+++ b/x-pack/qa/oidc-op-tests/build.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
 
 dependencies {
diff --git a/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtWithOidcAuthIT.java b/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtWithOidcAuthIT.java
index 1044e81af67f..47e4b02d6364 100644
--- a/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtWithOidcAuthIT.java
+++ b/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtWithOidcAuthIT.java
@@ -17,6 +17,7 @@
 
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.test.TestMatchers;
@@ -65,7 +66,7 @@ public static void registerClient() throws IOException {
         clientId = randomFrom(ALLOWED_AUDIENCES);
         redirectUri = "https://" + randomAlphaOfLength(4) + ".rp.example.com/" + randomAlphaOfLength(6);
         String clientSecret = randomAlphaOfLength(24);
-        String clientSetup = formatted("""
+        String clientSetup = Strings.format("""
             {
               "grant_types": [ "implicit" ],
               "response_types": [ "token id_token" ],
@@ -80,7 +81,7 @@ public static void registerClient() throws IOException {
     public void setupRoleMapping() throws Exception {
         try (var restClient = getElasticsearchClient()) {
             var client = new TestSecurityClient(restClient);
-            final String mappingJson = formatted("""
+            final String mappingJson = Strings.format("""
                 {
                   "roles": [ "%s" ],
                   "enabled": true,
diff --git a/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/C2IdOpTestCase.java b/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/C2IdOpTestCase.java
index da8d9e800bf6..9723ba708c9b 100644
--- a/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/C2IdOpTestCase.java
+++ b/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/C2IdOpTestCase.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.core.PathUtils;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xpack.core.common.socket.SocketAccess;
@@ -125,7 +126,7 @@ protected String authenticateAtOP(URI opAuthUri) throws Exception {
             final BasicHttpContext context = new BasicHttpContext();
             // Initiate the authentication process
             HttpPost httpPost = new HttpPost(C2ID_LOGIN_API + "initAuthRequest");
-            String initJson = formatted("""
+            String initJson = Strings.format("""
                 {"qs":"%s"}
                 """, opAuthUri.getRawQuery());
             configureJsonRequest(httpPost, initJson);
diff --git a/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java b/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java
index a62268402808..d8016ec68540 100644
--- a/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java
+++ b/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
@@ -58,7 +59,7 @@ public void setupUserAndRoles() throws Exception {
      */
     @BeforeClass
     public static void registerClients() throws Exception {
-        String codeClient = formatted("""
+        String codeClient = Strings.format("""
             {
               "grant_types": [ "authorization_code" ],
               "response_types": [ "code" ],
@@ -67,7 +68,7 @@ public static void registerClients() throws Exception {
               "redirect_uris": [ "https://my.fantastic.rp/cb" ],
               "token_endpoint_auth_method": "client_secret_basic"
             }""", CLIENT_SECRET);
-        String implicitClient = formatted("""
+        String implicitClient = Strings.format("""
             {
               "grant_types": [ "implicit" ],
               "response_types": [ "token id_token" ],
@@ -75,7 +76,7 @@ public static void registerClients() throws Exception {
               "preferred_client_secret": "%s",
               "redirect_uris": [ "https://my.fantastic.rp/cb" ]
             }""", CLIENT_SECRET);
-        String postClient = formatted("""
+        String postClient = Strings.format("""
             {
               "grant_types": [ "authorization_code" ],
               "response_types": [ "code" ],
@@ -84,7 +85,7 @@ public static void registerClients() throws Exception {
               "redirect_uris": [ "https://my.fantastic.rp/cb" ],
               "token_endpoint_auth_method": "client_secret_post"
             }""", CLIENT_SECRET);
-        String jwtClient = formatted("""
+        String jwtClient = Strings.format("""
             {
               "grant_types": [ "authorization_code" ],
               "response_types": [ "code" ],
@@ -269,7 +270,7 @@ private void setFacilitatorUser() throws Exception {
                 { "cluster" : ["manage_oidc", "manage_token"] }""");
             restClient.performRequest(createRoleRequest);
             Request createUserRequest = new Request("PUT", "/_security/user/facilitator");
-            createUserRequest.setJsonEntity(formatted("""
+            createUserRequest.setJsonEntity(Strings.format("""
                 { "password" : "%s", "roles" : ["facilitator"] }""", FACILITATOR_PASSWORD));
             restClient.performRequest(createUserRequest);
         }
@@ -278,7 +279,7 @@ private void setFacilitatorUser() throws Exception {
     private void setRoleMappings() throws Exception {
         try (RestClient restClient = getElasticsearchClient()) {
             Request createRoleMappingRequest = new Request("PUT", "/_security/role_mapping/oidc_kibana");
-            createRoleMappingRequest.setJsonEntity(formatted("""
+            createRoleMappingRequest.setJsonEntity(Strings.format("""
                 {
                   "roles": [ "kibana_admin" ],
                   "enabled": true,
@@ -310,7 +311,7 @@ private void setRoleMappings() throws Exception {
             restClient.performRequest(createRoleMappingRequest);
 
             createRoleMappingRequest = new Request("PUT", "/_security/role_mapping/oidc_limited");
-            createRoleMappingRequest.setJsonEntity(formatted("""
+            createRoleMappingRequest.setJsonEntity(Strings.format("""
                 {
                   "roles": [ "limited_user" ],
                   "enabled": true,
diff --git a/x-pack/qa/password-protected-keystore/build.gradle b/x-pack/qa/password-protected-keystore/build.gradle
index 7e6dd4bade15..61d373b5a42e 100644
--- a/x-pack/qa/password-protected-keystore/build.gradle
+++ b/x-pack/qa/password-protected-keystore/build.gradle
@@ -3,7 +3,7 @@
  * is using a password protected keystore in its nodes.
  */
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 dependencies {
   javaRestTestImplementation project(path: xpackModule('core'))
diff --git a/x-pack/qa/reindex-tests-with-security/build.gradle b/x-pack/qa/reindex-tests-with-security/build.gradle
index bec7c8b1b0be..fdb61e02f818 100644
--- a/x-pack/qa/reindex-tests-with-security/build.gradle
+++ b/x-pack/qa/reindex-tests-with-security/build.gradle
@@ -1,6 +1,6 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 
 dependencies {
diff --git a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java
index ed8b6dbfd2e1..968262448c87 100644
--- a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java
+++ b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/DocValueOnlyFieldsIT.java
@@ -176,7 +176,7 @@ public void setupIndex() throws IOException {
 
             // register repo on old ES and take snapshot
             Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName);
-            createRepoRequest.setJsonEntity(formatted("""
+            createRepoRequest.setJsonEntity(Strings.format("""
                 {"type":"fs","settings":{"location":"%s"}}
                 """, repoLocation));
             assertOK(oldEs.performRequest(createRepoRequest));
@@ -189,7 +189,7 @@ public void setupIndex() throws IOException {
 
         // register repo on new ES and restore snapshot
         Request createRepoRequest2 = new Request("PUT", "/_snapshot/" + repoName);
-        createRepoRequest2.setJsonEntity(formatted("""
+        createRepoRequest2.setJsonEntity(Strings.format("""
             {"type":"fs","settings":{"location":"%s"}}
             """, repoLocation));
         assertOK(client().performRequest(createRepoRequest2));
diff --git a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldMappingsIT.java b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldMappingsIT.java
index 65a578f07125..67dbdec6b839 100644
--- a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldMappingsIT.java
+++ b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldMappingsIT.java
@@ -144,7 +144,7 @@ public void setupIndex() throws IOException {
 
             // register repo on old ES and take snapshot
             Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName);
-            createRepoRequest.setJsonEntity(formatted("""
+            createRepoRequest.setJsonEntity(Strings.format("""
                 {"type":"fs","settings":{"location":"%s"}}
                 """, repoLocation));
             assertOK(oldEs.performRequest(createRepoRequest));
@@ -157,7 +157,7 @@ public void setupIndex() throws IOException {
 
         // register repo on new ES and restore snapshot
         Request createRepoRequest2 = new Request("PUT", "/_snapshot/" + repoName);
-        createRepoRequest2.setJsonEntity(formatted("""
+        createRepoRequest2.setJsonEntity(Strings.format("""
             {"type":"fs","settings":{"location":"%s"}}
             """, repoLocation));
         assertOK(client().performRequest(createRepoRequest2));
diff --git a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java
index fa5465ee2a2c..f4acc4491def 100644
--- a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java
+++ b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java
@@ -178,9 +178,9 @@ private void beforeRestart(
 
         // register repo on old ES and take snapshot
         Request createRepoRequest = new Request("PUT", "/_snapshot/" + repoName);
-        createRepoRequest.setJsonEntity(sourceOnlyRepository ? formatted("""
+        createRepoRequest.setJsonEntity(sourceOnlyRepository ? Strings.format("""
             {"type":"source","settings":{"location":"%s","delegate_type":"fs"}}
-            """, repoLocation) : formatted("""
+            """, repoLocation) : Strings.format("""
             {"type":"fs","settings":{"location":"%s"}}
             """, repoLocation));
         assertOK(oldEs.performRequest(createRepoRequest));
@@ -281,7 +281,7 @@ private String getType(Version oldVersion, String id) {
     }
 
     private static String sourceForDoc(int i) {
-        return "{\"test\":\"test" + i + "\",\"val\":" + i + ",\"create_date\":\"2020-01-" + formatted("%02d", i + 1) + "\"}";
+        return "{\"test\":\"test" + i + "\",\"val\":" + i + ",\"create_date\":\"2020-01-" + Strings.format("%02d", i + 1) + "\"}";
     }
 
     @SuppressWarnings("removal")
diff --git a/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/AbstractMultiClusterUpgradeTestCase.java b/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/AbstractMultiClusterUpgradeTestCase.java
index ae803257c5fa..1149a88e0e5a 100644
--- a/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/AbstractMultiClusterUpgradeTestCase.java
+++ b/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/AbstractMultiClusterUpgradeTestCase.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xcontent.json.JsonXContent;
 import org.junit.AfterClass;
@@ -110,7 +111,7 @@ private void configureLeaderRemoteClusters() throws IOException {
         if (leaderRemoteClusterSeed != null) {
             logger.info("Configuring leader remote cluster [{}]", leaderRemoteClusterSeed);
             Request request = new Request("PUT", "/_cluster/settings");
-            request.setJsonEntity(formatted("""
+            request.setJsonEntity(Strings.format("""
                 {
                   "persistent": {
                     "cluster.remote.leader.seeds": "%s"
@@ -130,7 +131,7 @@ private void configureFollowerRemoteClusters() throws IOException {
         if (followerRemoteClusterSeed != null) {
             logger.info("Configuring follower remote cluster [{}]", followerRemoteClusterSeed);
             Request request = new Request("PUT", "/_cluster/settings");
-            request.setJsonEntity(formatted("""
+            request.setJsonEntity(Strings.format("""
                 {
                   "persistent": {
                     "cluster.remote.follower.seeds": "%s"
diff --git a/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java b/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java
index b67f78c28e7e..c51814cd1935 100644
--- a/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java
+++ b/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.xcontent.ObjectPath;
 
 import java.io.IOException;
@@ -293,7 +294,7 @@ private static void createLeaderIndex(RestClient client, String indexName) throw
 
     private static void followIndex(RestClient client, String leaderCluster, String leaderIndex, String followIndex) throws IOException {
         final Request request = new Request("PUT", "/" + followIndex + "/_ccr/follow?wait_for_active_shards=1");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {"remote_cluster": "%s", "leader_index": "%s", "read_poll_timeout": "10ms"}
             """, leaderCluster, leaderIndex));
         assertOK(client.performRequest(request));
@@ -301,7 +302,7 @@ private static void followIndex(RestClient client, String leaderCluster, String
 
     private static void putAutoFollowPattern(RestClient client, String name, String remoteCluster, String pattern) throws IOException {
         Request request = new Request("PUT", "/_ccr/auto_follow/" + name);
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {
               "leader_index_patterns": [ "%s" ],
               "remote_cluster": "%s",
diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle
index 1a55844b41a4..02a2074b2b5b 100644
--- a/x-pack/qa/rolling-upgrade/build.gradle
+++ b/x-pack/qa/rolling-upgrade/build.gradle
@@ -8,6 +8,7 @@ apply plugin: 'elasticsearch.bwc-test'
 apply plugin: 'elasticsearch.rest-resources'
 
 dependencies {
+  testImplementation testArtifact(project(xpackModule('core')))
   testImplementation project(':x-pack:qa')
   testImplementation project(':client:rest-high-level')
 }
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java
index 8fba728cf55e..4c399d0a01a9 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
 import org.elasticsearch.core.Booleans;
+import org.elasticsearch.core.Strings;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
@@ -47,7 +48,7 @@ public void testDataStreams() throws IOException {
 
             StringBuilder b = new StringBuilder();
             for (int i = 0; i < 1000; i++) {
-                b.append(formatted("""
+                b.append(Strings.format("""
                     {"create":{"_index":"logs-foobar"}}
                     {"@timestamp":"2020-12-12","test":"value%s"}
                     """, i));
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/GeoIpUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/GeoIpUpgradeIT.java
index 3dedd041d646..eb0e97e1ecce 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/GeoIpUpgradeIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/GeoIpUpgradeIT.java
@@ -26,7 +26,9 @@ public void testGeoIpDownloader() throws Exception {
             assertBusy(() -> {
                 Response response = client().performRequest(new Request("GET", "_ingest/geoip/stats"));
                 String tasks = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
-                assertThat(tasks, Matchers.containsString("failed_downloads\":1"));
+                // The geoip downloader doesn't actually do anything since there are no geoip processors:
+                assertThat(tasks, Matchers.containsString("failed_downloads\":0"));
+                assertThat(tasks, Matchers.containsString("successful_downloads\":0"));
             });
         }
     }
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java
index 69e6d7ba1407..18d9148edfc4 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.core.Booleans;
+import org.elasticsearch.core.Strings;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -111,7 +112,7 @@ public void testIndexing() throws IOException {
     private void bulk(String index, String valueSuffix, int count) throws IOException {
         StringBuilder b = new StringBuilder();
         for (int i = 0; i < count; i++) {
-            b.append(formatted("""
+            b.append(Strings.format("""
                 {"index": {"_index": "%s"}}
                 {"f1": "v%s%s", "f2": %s}
                 """, index, i, valueSuffix, i));
@@ -127,7 +128,7 @@ static void assertCount(String index, int count) throws IOException {
         searchTestIndexRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true");
         searchTestIndexRequest.addParameter("filter_path", "hits.total");
         Response searchTestIndexResponse = client().performRequest(searchTestIndexRequest);
-        assertEquals(formatted("""
+        assertEquals(Strings.format("""
             {"hits":{"total":%s}}\
             """, count), EntityUtils.toString(searchTestIndexResponse.getEntity(), StandardCharsets.UTF_8));
     }
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java
index fd6bffaf5bee..a5d3e3ea43cd 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.core.Strings;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -191,7 +192,7 @@ private void assertNewInfer(String modelId) throws IOException {
 
     private void putModelDefinition(String modelId) throws IOException {
         Request request = new Request("PUT", "_ml/trained_models/" + modelId + "/definition/0");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {"total_definition_length":%s,"definition": "%s","total_parts": 1}""", RAW_MODEL_SIZE, BASE_64_ENCODED_MODEL));
         client().performRequest(request);
     }
@@ -204,7 +205,7 @@ private void putVocabulary(List<String> vocabulary, String modelId) throws IOExc
         String quotedWords = vocabularyWithPad.stream().map(s -> "\"" + s + "\"").collect(Collectors.joining(","));
 
         Request request = new Request("PUT", "_ml/trained_models/" + modelId + "/vocabulary");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             { "vocabulary": [%s] }
             """, quotedWords));
         client().performRequest(request);
@@ -276,7 +277,7 @@ private Response getTrainedModelStats(String modelId) throws IOException {
 
     private Response infer(String input, String modelId) throws IOException {
         Request request = new Request("POST", "/_ml/trained_models/" + modelId + "/deployment/_infer");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {  "docs": [{"input":"%s"}] }
             """, input));
         request.setOptions(request.getOptions().toBuilder().setWarningsHandler(PERMISSIVE).build());
@@ -287,7 +288,7 @@ private Response infer(String input, String modelId) throws IOException {
 
     private Response newInfer(String input, String modelId) throws IOException {
         Request request = new Request("POST", "/_ml/trained_models/" + modelId + "/_infer");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {  "docs": [{"input":"%s"}] }
             """, input));
         var response = client().performRequest(request);
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlTrainedModelsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlTrainedModelsUpgradeIT.java
index 78f57ad22d5a..23ea59811a24 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlTrainedModelsUpgradeIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlTrainedModelsUpgradeIT.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.xpack.test.rest.XPackRestTestConstants;
 import org.junit.BeforeClass;
 
@@ -61,7 +62,7 @@ public void testTrainedModelInference() throws Exception {
             case OLD -> {
                 createIndexWithName(INDEX_NAME);
                 indexData(INDEX_NAME, 1000);
-                createAndRunClassificationJob();
+                createAndRunClassificationJob("classification-upgrade-job");
                 createAndRunRegressionJob();
                 List<String> oldModels = getTrainedModels();
                 createPipelines(oldModels);
@@ -153,7 +154,7 @@ void testInfer(List<String> modelIds) throws Exception {
     }
 
     void createAndRunRegressionJob() throws Exception {
-        String config = formatted("""
+        String config = Strings.format("""
             {
               "source": {
                 "index": [ "%s" ]
@@ -171,8 +172,8 @@ void createAndRunRegressionJob() throws Exception {
         putAndStartDFAAndWaitForFinish(config, "regression");
     }
 
-    void createAndRunClassificationJob() throws Exception {
-        String config = formatted("""
+    void createAndRunClassificationJob(String jobName) throws Exception {
+        String config = Strings.format("""
             {
               "source": {
                 "index": [ "%s" ]
@@ -187,7 +188,7 @@ void createAndRunClassificationJob() throws Exception {
               },
               "model_memory_limit": "18mb"
             }""", INDEX_NAME, KEYWORD_FIELD);
-        putAndStartDFAAndWaitForFinish(config, "classification");
+        putAndStartDFAAndWaitForFinish(config, jobName);
     }
 
     @SuppressWarnings("unchecked")
@@ -201,11 +202,11 @@ void putAndStartDFAAndWaitForFinish(String config, String id) throws Exception {
                 client().performRequest(new Request("GET", "_ml/data_frame/analytics/" + id + "/_stats"))
             ).get("data_frame_analytics")).get(0);
             assertThat(state.get("state"), equalTo("stopped"));
-        }, 1, TimeUnit.MINUTES);
+        }, 2, TimeUnit.MINUTES);
     }
 
     void createPipeline(String id, String modelType, String modelId) throws Exception {
-        String body = formatted("""
+        String body = Strings.format("""
             {
               "processors": [
                 {
@@ -225,7 +226,7 @@ void createPipeline(String id, String modelType, String modelId) throws Exceptio
     }
 
     void createIndexWithName(String index) throws IOException {
-        String mapping = formatted("""
+        String mapping = Strings.format("""
             "properties": {
                 "%s": {
                   "type": "boolean"
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SearchableSnapshotsRollingUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SearchableSnapshotsRollingUpgradeIT.java
index bd52bea94685..025136c418b3 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SearchableSnapshotsRollingUpgradeIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SearchableSnapshotsRollingUpgradeIT.java
@@ -409,7 +409,7 @@ private static void mountSnapshot(
         } else {
             assertThat("Parameter 'storage' was introduced in 7.12.0 with " + Storage.SHARED_CACHE, storage, equalTo(Storage.FULL_COPY));
         }
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {
               "index": "%s",
               "renamed_index": "%s",
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java
index 7d737662eba3..6d65aff668ff 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.WarningsHandler;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.test.rest.ObjectPath;
 import org.junit.After;
@@ -299,7 +300,7 @@ private void assertAccessTokenDoesNotWork(String token) {
     private void assertRefreshTokenInvalidated(String refreshToken) throws IOException {
         for (RestClient client : twoClients) {
             Request refreshTokenRequest = new Request("POST", "/_security/oauth2/token");
-            refreshTokenRequest.setJsonEntity(formatted("""
+            refreshTokenRequest.setJsonEntity(Strings.format("""
                 {
                   "refresh_token": "%s",
                   "grant_type": "refresh_token"
@@ -336,7 +337,7 @@ private Map<Version, RestClient> getRestClientByVersion() throws IOException {
 
     private Map<String, Object> createTokens(RestClient client, String username, String password) throws IOException {
         final Request createTokenRequest = new Request("POST", "/_security/oauth2/token");
-        createTokenRequest.setJsonEntity(formatted("""
+        createTokenRequest.setJsonEntity(Strings.format("""
             {
               "username": "%s",
               "password": "%s",
@@ -349,7 +350,7 @@ private Map<String, Object> createTokens(RestClient client, String username, Str
 
     private void storeTokens(RestClient client, int idx, String accessToken, String refreshToken) throws IOException {
         final Request indexRequest = new Request("PUT", "token_backwards_compatibility_it/_doc/old_cluster_token" + idx);
-        indexRequest.setJsonEntity(formatted("""
+        indexRequest.setJsonEntity(Strings.format("""
             {
               "token": "%s",
               "refresh_token": "%s"
@@ -368,7 +369,7 @@ private Map<String, Object> retrieveStoredTokens(RestClient client, int tokenIdx
 
     private Map<String, Object> refreshToken(RestClient client, String refreshToken) throws IOException {
         final Request refreshTokenRequest = new Request("POST", "/_security/oauth2/token");
-        refreshTokenRequest.setJsonEntity(formatted("""
+        refreshTokenRequest.setJsonEntity(Strings.format("""
             {
               "refresh_token": "%s",
               "grant_type": "refresh_token"
@@ -408,7 +409,7 @@ private void extendExpirationTimeForAllTokens() throws Exception {
         final var bulkRequest = new Request("POST", "/.security-tokens/_bulk?refresh=true");
         bulkRequest.setOptions(bulkRequest.getOptions().toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE));
         final long newExpirationTime = Instant.now().plus(1, ChronoUnit.HOURS).toEpochMilli();
-        bulkRequest.setJsonEntity(tokensIds.stream().map(tokenId -> formatted("""
+        bulkRequest.setJsonEntity(tokensIds.stream().map(tokenId -> Strings.format("""
             {"update": {"_id": "%s"}}
             {"doc": {"access_token": {"user_token": {"expiration_time": %s}}}}
             """, tokenId, newExpirationTime)).collect(Collectors.joining("\n")));
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java
index 0deec0317b3a..3f4eb491d6cf 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java
@@ -14,13 +14,13 @@
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
-import org.elasticsearch.client.core.IndexerState;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.indexing.IndexerState;
 
 import java.io.IOException;
 import java.time.Instant;
@@ -256,7 +256,7 @@ private void awaitWrittenIndexerState(String id, Consumer<Map<?, ?>> responseAss
             TRANSFORM_INTERNAL_INDEX_PREFIX + "*," + TRANSFORM_INTERNAL_INDEX_PREFIX_DEPRECATED + "*" + "/_search"
         );
 
-        getStatsDocsRequest.setJsonEntity(formatted("""
+        getStatsDocsRequest.setJsonEntity(Strings.format("""
             {
                "query": {
                  "bool": {
@@ -373,7 +373,7 @@ private void putData(String indexName, int numDocs, TimeValue fromTime, List<Str
         final StringBuilder bulk = new StringBuilder();
         for (int i = 0; i < numDocs; i++) {
             for (String entity : entityIds) {
-                bulk.append(formatted("""
+                bulk.append(Strings.format("""
                     {"index":{"_index":"%s"}}
                     {"user_id":"%s","stars":%s,"timestamp":%s}
                     """, indexName, entity, randomLongBetween(0, 5), timeStamp));
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml
index d4aec6ac1f0a..cab9f19363d4 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml
@@ -1,5 +1,8 @@
 ---
 "Continue scroll after upgrade":
+  - skip:
+      version: "all"
+      reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/91637"
   - do:
       get:
         index: scroll_index
diff --git a/x-pack/qa/runtime-fields/build.gradle b/x-pack/qa/runtime-fields/build.gradle
index 4e9eb2d23905..5bee9f08c2a7 100644
--- a/x-pack/qa/runtime-fields/build.gradle
+++ b/x-pack/qa/runtime-fields/build.gradle
@@ -21,7 +21,7 @@ tasks.named("test").configure { enabled = false }
 
 subprojects {
   if (project.name.startsWith('core-with-')) {
-    apply plugin: 'elasticsearch.internal-yaml-rest-test'
+    apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 
     dependencies {
       yamlRestTestImplementation project(":x-pack:qa:runtime-fields")
@@ -114,6 +114,13 @@ subprojects {
           'search/400_synthetic_source/*',
           'search.highlight/50_synthetic_source/*',
           'aggregations/top_hits/synthetic _source',
+          // Runtime fields produce a slightly different result for multi valued matrix_stats yaml tests:
+          // (this appears to always have been the case. muted when matrix_stats was moved from its own
+          // module to aggregations module)
+          'aggregations/matrix_stats_multi_value_field/Multi value field Min',
+          'aggregations/matrix_stats_multi_value_field/Multi value field Max',
+          'aggregations/matrix_stats_multi_value_field/Partially unmapped',
+          'aggregations/matrix_stats_multi_value_field/Partially unmapped with missing defaults',
           /////// NOT SUPPORTED ///////
         ].join(',')
     }
diff --git a/x-pack/qa/runtime-fields/with-security/build.gradle b/x-pack/qa/runtime-fields/with-security/build.gradle
index 41ae03d6692d..e664ee888a31 100644
--- a/x-pack/qa/runtime-fields/with-security/build.gradle
+++ b/x-pack/qa/runtime-fields/with-security/build.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.authenticated-testclusters'
 
 dependencies {
diff --git a/x-pack/qa/saml-idp-tests/build.gradle b/x-pack/qa/saml-idp-tests/build.gradle
index 813733c0b4aa..10e6848dfb73 100644
--- a/x-pack/qa/saml-idp-tests/build.gradle
+++ b/x-pack/qa/saml-idp-tests/build.gradle
@@ -2,7 +2,7 @@ import org.elasticsearch.gradle.LazyPropertyMap
 
 Project idpFixtureProject = project(':x-pack:test:idp-fixture')
 
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
 
 dependencies {
diff --git a/x-pack/qa/security-example-spi-extension/build.gradle b/x-pack/qa/security-example-spi-extension/build.gradle
index cac57d78344b..3fb73d615b8c 100644
--- a/x-pack/qa/security-example-spi-extension/build.gradle
+++ b/x-pack/qa/security-example-spi-extension/build.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 apply plugin: 'elasticsearch.base-internal-es-plugin'
 
 esplugin {
diff --git a/x-pack/qa/security-example-spi-extension/src/javaRestTest/java/org/elasticsearch/example/realm/CustomRoleMappingRealmIT.java b/x-pack/qa/security-example-spi-extension/src/javaRestTest/java/org/elasticsearch/example/realm/CustomRoleMappingRealmIT.java
index 7404fbd4b8b2..30eae681990b 100644
--- a/x-pack/qa/security-example-spi-extension/src/javaRestTest/java/org/elasticsearch/example/realm/CustomRoleMappingRealmIT.java
+++ b/x-pack/qa/security-example-spi-extension/src/javaRestTest/java/org/elasticsearch/example/realm/CustomRoleMappingRealmIT.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.hamcrest.Matchers;
 import org.junit.Before;
@@ -40,7 +41,7 @@ protected Settings restClientSettings() {
     public void setupRoleMapping() throws Exception {
         expectedRole = randomAlphaOfLengthBetween(4, 16);
         Request request = new Request("PUT", "/_security/role_mapping/test");
-        request.setJsonEntity(formatted("""
+        request.setJsonEntity(Strings.format("""
             {
               "enabled": true,
               "roles": [ "%s" ],
diff --git a/x-pack/qa/security-example-spi-extension/src/javaRestTest/java/org/elasticsearch/example/role/CustomRolesProviderIT.java b/x-pack/qa/security-example-spi-extension/src/javaRestTest/java/org/elasticsearch/example/role/CustomRolesProviderIT.java
index 2d7d8ff76aed..f11b6d0a63a0 100644
--- a/x-pack/qa/security-example-spi-extension/src/javaRestTest/java/org/elasticsearch/example/role/CustomRolesProviderIT.java
+++ b/x-pack/qa/security-example-spi-extension/src/javaRestTest/java/org/elasticsearch/example/role/CustomRolesProviderIT.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.example.realm.CustomRealm;
 import org.elasticsearch.example.realm.CustomRealmIT;
 import org.elasticsearch.test.rest.ESRestTestCase;
@@ -55,7 +56,7 @@ protected Settings restClientSettings() {
     public void setupTestUser(String role) throws IOException {
         final String endpoint = "/_security/user/" + TEST_USER;
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
-        final String body = formatted("""
+        final String body = Strings.format("""
             {
                 "username": "%s",
                 "password": "%s",
diff --git a/x-pack/qa/security-setup-password-tests/build.gradle b/x-pack/qa/security-setup-password-tests/build.gradle
index 29455fe41dd9..8f17e6d92291 100644
--- a/x-pack/qa/security-setup-password-tests/build.gradle
+++ b/x-pack/qa/security-setup-password-tests/build.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.legacy-java-rest-test'
 
 dependencies {
   javaRestTestImplementation project(path: xpackModule('security'))
diff --git a/x-pack/qa/security-setup-password-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/esnative/tool/ResetPasswordToolIT.java b/x-pack/qa/security-setup-password-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/esnative/tool/ResetPasswordToolIT.java
index 9bc42bc742e3..ab849872cc6c 100644
--- a/x-pack/qa/security-setup-password-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/esnative/tool/ResetPasswordToolIT.java
+++ b/x-pack/qa/security-setup-password-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/esnative/tool/ResetPasswordToolIT.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.core.Strings;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -70,7 +71,7 @@ public void testResetNativeUser() throws Exception {
             + Base64.getEncoder().encodeToString(("test_admin:x-pack-test-password").getBytes(StandardCharsets.UTF_8));
         try {
             Request putUserRequest = new Request("PUT", "/_security/user/" + nativeUser);
-            putUserRequest.setJsonEntity(formatted("""
+            putUserRequest.setJsonEntity(Strings.format("""
                 {
                    "password" : "l0ng-r4nd0m-p@ssw0rd",
                    "roles" : [ "admin", "other_role1" ],
diff --git a/x-pack/qa/smoke-test-plugins-ssl/build.gradle b/x-pack/qa/smoke-test-plugins-ssl/build.gradle
index bd0f9112e2a4..1e8e88b91c33 100644
--- a/x-pack/qa/smoke-test-plugins-ssl/build.gradle
+++ b/x-pack/qa/smoke-test-plugins-ssl/build.gradle
@@ -2,7 +2,7 @@ import org.apache.tools.ant.filters.ReplaceTokens
 
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 
 dependencies {
   yamlRestTestImplementation project(':x-pack:plugin:core')
diff --git a/x-pack/qa/smoke-test-plugins/build.gradle b/x-pack/qa/smoke-test-plugins/build.gradle
index 46ae4b9015db..427aa39f02e4 100644
--- a/x-pack/qa/smoke-test-plugins/build.gradle
+++ b/x-pack/qa/smoke-test-plugins/build.gradle
@@ -1,7 +1,7 @@
 import org.apache.tools.ant.filters.ReplaceTokens
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 
 dependencies {
diff --git a/x-pack/qa/smoke-test-security-with-mustache/build.gradle b/x-pack/qa/smoke-test-security-with-mustache/build.gradle
index c3bebeb17817..373e6966df51 100644
--- a/x-pack/qa/smoke-test-security-with-mustache/build.gradle
+++ b/x-pack/qa/smoke-test-security-with-mustache/build.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 
 dependencies {
   yamlRestTestImplementation project(':x-pack:qa')
diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/MultiGroupMappingIT.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/MultiGroupMappingIT.java
index e90b16f705a2..3349d52c493a 100644
--- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/MultiGroupMappingIT.java
+++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/MultiGroupMappingIT.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.security.authc.ldap;
 
 import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.core.Strings;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
@@ -34,7 +35,7 @@ public static void setRoleMappingType() {
 
     @Override
     protected String configRoles() {
-        return formatted("""
+        return Strings.format("""
             %s
             MarvelCharacters:
               cluster: [ NONE ]
diff --git a/x-pack/qa/third-party/jira/build.gradle b/x-pack/qa/third-party/jira/build.gradle
index 84e0d7dc50bb..b7268af80753 100644
--- a/x-pack/qa/third-party/jira/build.gradle
+++ b/x-pack/qa/third-party/jira/build.gradle
@@ -5,7 +5,7 @@ import java.nio.charset.StandardCharsets
 
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 
 dependencies {
   yamlRestTestImplementation project(':x-pack:plugin:core')
diff --git a/x-pack/qa/third-party/pagerduty/build.gradle b/x-pack/qa/third-party/pagerduty/build.gradle
index b7adeb9c6e66..4b5a0bbeeeb4 100644
--- a/x-pack/qa/third-party/pagerduty/build.gradle
+++ b/x-pack/qa/third-party/pagerduty/build.gradle
@@ -1,6 +1,6 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 
 dependencies {
   yamlRestTestImplementation project(':x-pack:plugin:core')
diff --git a/x-pack/qa/third-party/slack/build.gradle b/x-pack/qa/third-party/slack/build.gradle
index a298a1531a8d..b2b0478da047 100644
--- a/x-pack/qa/third-party/slack/build.gradle
+++ b/x-pack/qa/third-party/slack/build.gradle
@@ -1,6 +1,6 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 
-apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 
 dependencies {
   yamlRestTestImplementation project(':x-pack:plugin:core')
diff --git a/x-pack/qa/xpack-prefix-rest-compat/build.gradle b/x-pack/qa/xpack-prefix-rest-compat/build.gradle
index b17463183bd2..d609ede0efd2 100644
--- a/x-pack/qa/xpack-prefix-rest-compat/build.gradle
+++ b/x-pack/qa/xpack-prefix-rest-compat/build.gradle
@@ -13,7 +13,7 @@ import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.internal.test.rest.CopyRestTestsTask
 import org.elasticsearch.gradle.util.GradleUtils
 
-apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
 /**
  * This project exists to test the _xpack prefix for REST compatibility. The _xpack prefix was removed from the specification, but still supported